handle requests with missing content-length headers (e.g. YouTube)

This commit is contained in:
Matthew Hodgson 2016-03-31 01:55:21 +01:00
parent 7178ab7da0
commit a8a5dd3b44
2 changed files with 28 additions and 9 deletions

View File

@ -23,8 +23,9 @@ from canonicaljson import encode_canonical_json
from twisted.internet import defer, reactor, ssl, protocol
from twisted.web.client import (
RedirectAgent, Agent, readBody, FileBodyProducer, PartialDownloadError,
BrowserLikeRedirectAgent, Agent, readBody, FileBodyProducer, PartialDownloadError,
)
from twisted.web.http import PotentialDataLoss
from twisted.web.http_headers import Headers
from twisted.web._newclient import ResponseDone
@ -59,11 +60,11 @@ class SimpleHttpClient(object):
# The default context factory in Twisted 14.0.0 (which we require) is
# BrowserLikePolicyForHTTPS which will do regular cert validation
# 'like a browser'
self.agent = RedirectAgent(Agent(
self.agent = Agent(
reactor,
connectTimeout=15,
contextFactory=hs.get_http_client_context_factory()
))
)
self.user_agent = hs.version_string
if hs.config.user_agent_suffix:
self.user_agent = "%s %s" % (self.user_agent, hs.config.user_agent_suffix,)
@ -253,10 +254,6 @@ class SimpleHttpClient(object):
headers.
"""
def body_callback(method, url_bytes, headers_dict):
self.sign_request(destination, method, url_bytes, headers_dict)
return None
response = yield self.request(
"GET",
url.encode("ascii"),
@ -309,6 +306,10 @@ class _ReadBodyToFileProtocol(protocol.Protocol):
def connectionLost(self, reason):
if reason.check(ResponseDone):
self.deferred.callback(self.length)
elif reason.check(PotentialDataLoss):
# stolen from https://github.com/twisted/treq/pull/49/files
# http://twistedmatrix.com/trac/ticket/4840
self.deferred.callback(self.length)
else:
self.deferred.errback(reason)
@ -350,6 +351,24 @@ class CaptchaServerHttpClient(SimpleHttpClient):
# twisted dislikes google's response, no content length.
defer.returnValue(e.response)
class SpiderHttpClient(SimpleHttpClient):
"""
Separate HTTP client for spidering arbitrary URLs.
Special in that it follows retries and has a UA that looks
like a browser.
used by the preview_url endpoint in the content repo.
"""
def __init__(self, hs):
SimpleHttpClient.__init__(self, hs)
# clobber the base class's agent and UA:
self.agent = BrowserLikeRedirectAgent(Agent(
reactor,
connectTimeout=15,
contextFactory=hs.get_http_client_context_factory()
))
# Look like Chrome for now
#self.user_agent = ("Mozilla/5.0 (%s) (KHTML, like Gecko) Chrome Safari" % hs.version_string)
def encode_urlencode_args(args):
return {k: encode_urlencode_arg(v) for k, v in args.items()}

View File

@ -19,7 +19,7 @@ from twisted.web.server import NOT_DONE_YET
from twisted.internet import defer
from lxml import html
from synapse.util.stringutils import random_string
from synapse.http.client import SimpleHttpClient
from synapse.http.client import SpiderHttpClient
from synapse.http.server import request_handler, respond_with_json, respond_with_json_bytes
import os
@ -33,7 +33,7 @@ class PreviewUrlResource(BaseMediaResource):
def __init__(self, hs, filepaths):
BaseMediaResource.__init__(self, hs, filepaths)
self.client = SimpleHttpClient(hs)
self.client = SpiderHttpClient(hs)
def render_GET(self, request):
self._async_render_GET(request)