Merge pull request #3872 from matrix-org/hawkowl/timeouts-2

timeouts 2: electric boogaloo
This commit is contained in:
Erik Johnston 2018-09-14 16:58:44 +01:00 committed by GitHub
commit 3e6e94fe9f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 13 additions and 7 deletions

1
changelog.d/3872.misc Normal file
View File

@ -0,0 +1 @@
Disable buffering and automatic retrying in treq requests to prevent timeouts.

View File

@ -38,12 +38,12 @@ def cancelled_to_request_timed_out_error(value, timeout):
return value return value
ACCESS_TOKEN_RE = re.compile(br'(\?.*access(_|%5[Ff])token=)[^&]*(.*)$') ACCESS_TOKEN_RE = re.compile(r'(\?.*access(_|%5[Ff])token=)[^&]*(.*)$')
def redact_uri(uri): def redact_uri(uri):
"""Strips access tokens from the uri replaces with <redacted>""" """Strips access tokens from the uri replaces with <redacted>"""
return ACCESS_TOKEN_RE.sub( return ACCESS_TOKEN_RE.sub(
br'\1<redacted>\3', r'\1<redacted>\3',
uri uri
) )

View File

@ -93,7 +93,7 @@ class SimpleHttpClient(object):
outgoing_requests_counter.labels(method).inc() outgoing_requests_counter.labels(method).inc()
# log request but strip `access_token` (AS requests for example include this) # log request but strip `access_token` (AS requests for example include this)
logger.info("Sending request %s %s", method, redact_uri(uri.encode('ascii'))) logger.info("Sending request %s %s", method, redact_uri(uri))
try: try:
request_deferred = treq.request( request_deferred = treq.request(
@ -108,14 +108,14 @@ class SimpleHttpClient(object):
incoming_responses_counter.labels(method, response.code).inc() incoming_responses_counter.labels(method, response.code).inc()
logger.info( logger.info(
"Received response to %s %s: %s", "Received response to %s %s: %s",
method, redact_uri(uri.encode('ascii')), response.code method, redact_uri(uri), response.code
) )
defer.returnValue(response) defer.returnValue(response)
except Exception as e: except Exception as e:
incoming_responses_counter.labels(method, "ERR").inc() incoming_responses_counter.labels(method, "ERR").inc()
logger.info( logger.info(
"Error sending request to %s %s: %s %s", "Error sending request to %s %s: %s %s",
method, redact_uri(uri.encode('ascii')), type(e).__name__, e.args[0] method, redact_uri(uri), type(e).__name__, e.args[0]
) )
raise raise

View File

@ -92,6 +92,7 @@ class MatrixFederationHttpClient(object):
self.server_name = hs.hostname self.server_name = hs.hostname
reactor = hs.get_reactor() reactor = hs.get_reactor()
pool = HTTPConnectionPool(reactor) pool = HTTPConnectionPool(reactor)
pool.retryAutomatically = False
pool.maxPersistentPerHost = 5 pool.maxPersistentPerHost = 5
pool.cachedConnectionTimeout = 2 * 60 pool.cachedConnectionTimeout = 2 * 60
self.agent = Agent.usingEndpointFactory( self.agent = Agent.usingEndpointFactory(
@ -222,7 +223,8 @@ class MatrixFederationHttpClient(object):
headers=Headers(headers_dict), headers=Headers(headers_dict),
data=data, data=data,
agent=self.agent, agent=self.agent,
reactor=self.hs.get_reactor() reactor=self.hs.get_reactor(),
unbuffered=True
) )
request_deferred.addTimeout(_sec_timeout, self.hs.get_reactor()) request_deferred.addTimeout(_sec_timeout, self.hs.get_reactor())

View File

@ -85,7 +85,10 @@ class SynapseRequest(Request):
return "%s-%i" % (self.method, self.request_seq) return "%s-%i" % (self.method, self.request_seq)
def get_redacted_uri(self): def get_redacted_uri(self):
return redact_uri(self.uri) uri = self.uri
if isinstance(uri, bytes):
uri = self.uri.decode('ascii')
return redact_uri(uri)
def get_user_agent(self): def get_user_agent(self):
return self.requestHeaders.getRawHeaders(b"User-Agent", [None])[-1] return self.requestHeaders.getRawHeaders(b"User-Agent", [None])[-1]