From ff7c2e41de2056ab959a2d560c89d397425c61be Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 23 Jul 2015 14:12:49 +0100 Subject: [PATCH 01/29] Always return a thumbnail of the requested size. Before, we returned a thumbnail that was at least as big (if possible) as the requested size. Now, if we don't have a thumbnail of the given size we generate (and persist) one of that size. --- synapse/rest/media/v1/base_resource.py | 83 +++++++++++++++++++++ synapse/rest/media/v1/thumbnail_resource.py | 81 +++++++++++++++++++- 2 files changed, 162 insertions(+), 2 deletions(-) diff --git a/synapse/rest/media/v1/base_resource.py b/synapse/rest/media/v1/base_resource.py index c43ae0314b..00668b3862 100644 --- a/synapse/rest/media/v1/base_resource.py +++ b/synapse/rest/media/v1/base_resource.py @@ -225,6 +225,89 @@ class BaseMediaResource(Resource): else: return () + @defer.inlineCallbacks + def _generate_local_exact_thumbnail(self, media_id, t_width, t_height, + t_method, t_type): + input_path = self.filepaths.local_media_filepath(media_id) + + def thumbnail(): + thumbnailer = Thumbnailer(input_path) + m_width = thumbnailer.width + m_height = thumbnailer.height + + if m_width * m_height >= self.max_image_pixels: + logger.info( + "Image too large to thumbnail %r x %r > %r", + m_width, m_height, self.max_image_pixels + ) + return + + t_path = self.filepaths.local_media_thumbnail( + media_id, t_width, t_height, t_type, t_method + ) + self._makedirs(t_path) + + if t_method == "crop": + t_len = thumbnailer.crop(t_path, t_width, t_height, t_type) + elif t_method == "scale": + t_len = thumbnailer.scale(t_path, t_width, t_height, t_type) + else: + t_len = None + + return t_len, t_path + + res = yield threads.deferToThread(thumbnail) + + if res: + t_len, t_path = res + yield self.store.store_local_thumbnail( + media_id, t_width, t_height, t_type, t_method, t_len + ) + + defer.returnValue(t_path) + + @defer.inlineCallbacks + def _generate_remote_exact_thumbnail(self, server_name, file_id, media_id, + t_width, t_height, t_method, t_type): + input_path = self.filepaths.remote_media_filepath(server_name, file_id) + + def thumbnail(): + thumbnailer = Thumbnailer(input_path) + m_width = thumbnailer.width + m_height = thumbnailer.height + + if m_width * m_height >= self.max_image_pixels: + logger.info( + "Image too large to thumbnail %r x %r > %r", + m_width, m_height, self.max_image_pixels + ) + return + + t_path = self.filepaths.remote_media_thumbnail( + media_id, t_width, t_height, t_type, t_method + ) + self._makedirs(t_path) + + if t_method == "crop": + t_len = thumbnailer.crop(t_path, t_width, t_height, t_type) + elif t_method == "scale": + t_len = thumbnailer.scale(t_path, t_width, t_height, t_type) + else: + t_len = None + + return t_path, t_len + + res = yield threads.deferToThread(thumbnail) + + if res: + t_path, t_len = res + yield self.store.store_remote_media_thumbnail( + server_name, media_id, file_id, + t_width, t_height, t_type, t_method, t_len + ) + + defer.returnValue(t_path) + @defer.inlineCallbacks def _generate_local_thumbnails(self, media_id, media_info): media_type = media_info["media_type"] diff --git a/synapse/rest/media/v1/thumbnail_resource.py b/synapse/rest/media/v1/thumbnail_resource.py index 61f88e486e..58621f45df 100644 --- a/synapse/rest/media/v1/thumbnail_resource.py +++ b/synapse/rest/media/v1/thumbnail_resource.py @@ -43,11 +43,11 @@ class ThumbnailResource(BaseMediaResource): m_type = parse_string(request, "type", "image/png") if server_name == self.server_name: - yield self._respond_local_thumbnail( + yield self._select_or_generate_local_thumbnail( request, media_id, width, height, method, m_type ) else: - yield self._respond_remote_thumbnail( + yield self._select_or_generate_remote_thumbnail( request, server_name, media_id, width, height, method, m_type ) @@ -82,6 +82,83 @@ class ThumbnailResource(BaseMediaResource): request, media_info, width, height, method, m_type, ) + @defer.inlineCallbacks + def _select_or_generate_local_thumbnail(self, request, media_id, desired_width, + desired_height, desired_method, + desired_type): + media_info = yield self.store.get_local_media(media_id) + + if not media_info: + self._respond_404(request) + return + + thumbnail_infos = yield self.store.get_local_media_thumbnails(media_id) + for info in thumbnail_infos: + t_w = info["thumbnail_width"] == desired_width + t_h = info["thumbnail_height"] == desired_height + t_method = info["thumbnail_method"] == desired_method + t_type = info["thumbnail_type"] == desired_type + + if t_w and t_h and t_method and t_type: + file_path = self.filepaths.local_media_thumbnail( + media_id, desired_width, desired_height, desired_type, desired_method, + ) + yield self._respond_with_file(request, desired_type, file_path) + return + + logger.debug("We don't have a local thumbnail of that size. Generating") + + # Okay, so we generate one. + file_path = yield self._generate_local_exact_thumbnail( + media_id, desired_width, desired_height, desired_method, desired_type + ) + + if file_path: + yield self._respond_with_file(request, desired_type, file_path) + else: + yield self._respond_default_thumbnail( + request, media_info, desired_width, desired_height, + desired_method, desired_type, + ) + + @defer.inlineCallbacks + def _select_or_generate_remote_thumbnail(self, request, server_name, media_id, + desired_width, desired_height, + desired_method, desired_type): + media_info = yield self._get_remote_media(server_name, media_id) + + thumbnail_infos = yield self.store.get_remote_media_thumbnails( + server_name, media_id, + ) + + for info in thumbnail_infos: + t_w = info["thumbnail_width"] == desired_width + t_h = info["thumbnail_height"] == desired_height + t_method = info["thumbnail_method"] == desired_method + t_type = info["thumbnail_type"] == desired_type + + if t_w and t_h and t_method and t_type: + file_path = self.filepaths.remote_media_thumbnail( + media_id, desired_width, desired_height, desired_type, desired_method, + ) + yield self._respond_with_file(request, desired_type, file_path) + + logger.debug("We don't have a local thumbnail of that size. Generating") + + # Okay, so we generate one. + path = yield self._generate_remote_exact_thumbnail( + server_name, media_id, desired_width, desired_height, + desired_method, desired_type + ) + + if path: + yield self._respond_with_file(request, t_type, file_path) + else: + yield self._respond_default_thumbnail( + request, media_info, desired_width, desired_height, + desired_method, desired_type, + ) + @defer.inlineCallbacks def _respond_remote_thumbnail(self, request, server_name, media_id, width, height, method, m_type): From 33d83f36158a98111c3dceb605a40d962a9e5812 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 23 Jul 2015 14:24:21 +0100 Subject: [PATCH 02/29] Fix remote thumbnailing --- synapse/rest/media/v1/base_resource.py | 2 +- synapse/rest/media/v1/thumbnail_resource.py | 16 ++++++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/synapse/rest/media/v1/base_resource.py b/synapse/rest/media/v1/base_resource.py index 00668b3862..74c0cd093c 100644 --- a/synapse/rest/media/v1/base_resource.py +++ b/synapse/rest/media/v1/base_resource.py @@ -284,7 +284,7 @@ class BaseMediaResource(Resource): return t_path = self.filepaths.remote_media_thumbnail( - media_id, t_width, t_height, t_type, t_method + server_name, file_id, t_width, t_height, t_type, t_method ) self._makedirs(t_path) diff --git a/synapse/rest/media/v1/thumbnail_resource.py b/synapse/rest/media/v1/thumbnail_resource.py index 58621f45df..9387258a7a 100644 --- a/synapse/rest/media/v1/thumbnail_resource.py +++ b/synapse/rest/media/v1/thumbnail_resource.py @@ -131,6 +131,8 @@ class ThumbnailResource(BaseMediaResource): server_name, media_id, ) + file_id = media_info["filesystem_id"] + for info in thumbnail_infos: t_w = info["thumbnail_width"] == desired_width t_h = info["thumbnail_height"] == desired_height @@ -139,20 +141,22 @@ class ThumbnailResource(BaseMediaResource): if t_w and t_h and t_method and t_type: file_path = self.filepaths.remote_media_thumbnail( - media_id, desired_width, desired_height, desired_type, desired_method, + server_name, file_id, desired_width, desired_height, + desired_type, desired_method, ) yield self._respond_with_file(request, desired_type, file_path) + return logger.debug("We don't have a local thumbnail of that size. Generating") # Okay, so we generate one. - path = yield self._generate_remote_exact_thumbnail( - server_name, media_id, desired_width, desired_height, - desired_method, desired_type + file_path = yield self._generate_remote_exact_thumbnail( + server_name, file_id, media_id, desired_width, + desired_height, desired_method, desired_type ) - if path: - yield self._respond_with_file(request, t_type, file_path) + if file_path: + yield self._respond_with_file(request, desired_type, file_path) else: yield self._respond_default_thumbnail( request, media_info, desired_width, desired_height, From 459085184ce80c67584bee4e5d3bc43add99bb0b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 23 Jul 2015 15:59:32 +0100 Subject: [PATCH 03/29] Factor out thumbnail() --- synapse/rest/media/v1/base_resource.py | 96 +++++++++++--------------- 1 file changed, 40 insertions(+), 56 deletions(-) diff --git a/synapse/rest/media/v1/base_resource.py b/synapse/rest/media/v1/base_resource.py index 74c0cd093c..093ba847d3 100644 --- a/synapse/rest/media/v1/base_resource.py +++ b/synapse/rest/media/v1/base_resource.py @@ -225,41 +225,44 @@ class BaseMediaResource(Resource): else: return () + def _generate_thumbnail(self, input_path, t_path, t_width, t_height, + t_method, t_type): + thumbnailer = Thumbnailer(input_path) + m_width = thumbnailer.width + m_height = thumbnailer.height + + if m_width * m_height >= self.max_image_pixels: + logger.info( + "Image too large to thumbnail %r x %r > %r", + m_width, m_height, self.max_image_pixels + ) + return + + if t_method == "crop": + t_len = thumbnailer.crop(t_path, t_width, t_height, t_type) + elif t_method == "scale": + t_len = thumbnailer.scale(t_path, t_width, t_height, t_type) + else: + t_len = None + + return t_len + @defer.inlineCallbacks def _generate_local_exact_thumbnail(self, media_id, t_width, t_height, t_method, t_type): input_path = self.filepaths.local_media_filepath(media_id) - def thumbnail(): - thumbnailer = Thumbnailer(input_path) - m_width = thumbnailer.width - m_height = thumbnailer.height + t_path = self.filepaths.local_media_thumbnail( + media_id, t_width, t_height, t_type, t_method + ) + self._makedirs(t_path) - if m_width * m_height >= self.max_image_pixels: - logger.info( - "Image too large to thumbnail %r x %r > %r", - m_width, m_height, self.max_image_pixels - ) - return + t_len = yield threads.deferToThread( + self._generate_thumbnail, + input_path, t_path, t_width, t_height, t_method, t_type + ) - t_path = self.filepaths.local_media_thumbnail( - media_id, t_width, t_height, t_type, t_method - ) - self._makedirs(t_path) - - if t_method == "crop": - t_len = thumbnailer.crop(t_path, t_width, t_height, t_type) - elif t_method == "scale": - t_len = thumbnailer.scale(t_path, t_width, t_height, t_type) - else: - t_len = None - - return t_len, t_path - - res = yield threads.deferToThread(thumbnail) - - if res: - t_len, t_path = res + if t_len: yield self.store.store_local_thumbnail( media_id, t_width, t_height, t_type, t_method, t_len ) @@ -271,36 +274,17 @@ class BaseMediaResource(Resource): t_width, t_height, t_method, t_type): input_path = self.filepaths.remote_media_filepath(server_name, file_id) - def thumbnail(): - thumbnailer = Thumbnailer(input_path) - m_width = thumbnailer.width - m_height = thumbnailer.height + t_path = self.filepaths.remote_media_thumbnail( + server_name, file_id, t_width, t_height, t_type, t_method + ) + self._makedirs(t_path) - if m_width * m_height >= self.max_image_pixels: - logger.info( - "Image too large to thumbnail %r x %r > %r", - m_width, m_height, self.max_image_pixels - ) - return + t_len = yield threads.deferToThread( + self._generate_thumbnail, + input_path, t_path, t_width, t_height, t_method, t_type + ) - t_path = self.filepaths.remote_media_thumbnail( - server_name, file_id, t_width, t_height, t_type, t_method - ) - self._makedirs(t_path) - - if t_method == "crop": - t_len = thumbnailer.crop(t_path, t_width, t_height, t_type) - elif t_method == "scale": - t_len = thumbnailer.scale(t_path, t_width, t_height, t_type) - else: - t_len = None - - return t_path, t_len - - res = yield threads.deferToThread(thumbnail) - - if res: - t_path, t_len = res + if t_len: yield self.store.store_remote_media_thumbnail( server_name, media_id, file_id, t_width, t_height, t_type, t_method, t_len From c77048e12f032842cebbb0f1a0639bb62db88418 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 4 Aug 2015 14:37:09 +0100 Subject: [PATCH 04/29] Add endpoint that proxies ID server request token and errors if the given email is in use on this Home Server. --- synapse/api/errors.py | 1 + synapse/handlers/identity.py | 25 +++++++++++++++++ synapse/rest/client/v2_alpha/register.py | 27 ++++++++++++++++++- .../schema/delta/22/user_threepids_unique.sql | 19 +++++++++++++ 4 files changed, 71 insertions(+), 1 deletion(-) create mode 100644 synapse/storage/schema/delta/22/user_threepids_unique.sql diff --git a/synapse/api/errors.py b/synapse/api/errors.py index 0b3320e62c..c3b4d971a8 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -40,6 +40,7 @@ class Codes(object): TOO_LARGE = "M_TOO_LARGE" EXCLUSIVE = "M_EXCLUSIVE" THREEPID_AUTH_FAILED = "M_THREEPID_AUTH_FAILED" + THREEPID_IN_USE = "THREEPID_IN_USE" class CodeMessageException(RuntimeError): diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py index c1095708a0..2a99921d5f 100644 --- a/synapse/handlers/identity.py +++ b/synapse/handlers/identity.py @@ -117,3 +117,28 @@ class IdentityHandler(BaseHandler): except CodeMessageException as e: data = json.loads(e.msg) defer.returnValue(data) + + @defer.inlineCallbacks + def requestEmailToken(self, id_server, email, client_secret, send_attempt, **kwargs): + yield run_on_reactor() + http_client = SimpleHttpClient(self.hs) + + params = { + 'email': email, + 'client_secret': client_secret, + 'send_attempt': send_attempt, + } + params.update(kwargs) + + try: + data = yield http_client.post_urlencoded_get_json( + "https://%s%s" % ( + id_server, + "/_matrix/identity/api/v1/validate/email/requestToken" + ), + params + ) + defer.returnValue(data) + except CodeMessageException as e: + logger.info("Proxied requestToken failed: %r", e) + raise e diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index b5926f9ca6..7b97a73df6 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -41,7 +41,7 @@ logger = logging.getLogger(__name__) class RegisterRestServlet(RestServlet): - PATTERN = client_v2_pattern("/register") + PATTERN = client_v2_pattern("/register*") def __init__(self, hs): super(RegisterRestServlet, self).__init__() @@ -55,6 +55,11 @@ class RegisterRestServlet(RestServlet): @defer.inlineCallbacks def on_POST(self, request): yield run_on_reactor() + + if '/register/email/requestToken' in request.path: + ret = yield self.onEmailTokenRequest(request) + defer.returnValue(ret) + body = parse_json_dict_from_request(request) # we do basic sanity checks here because the auth layer will store these @@ -209,6 +214,26 @@ class RegisterRestServlet(RestServlet): "home_server": self.hs.hostname, } + @defer.inlineCallbacks + def onEmailTokenRequest(self, request): + body = parse_json_dict_from_request(request) + + required = ['id_server', 'client_secret', 'email', 'send_attempt'] + absent = [] + for k in required: + if k not in body: + absent.append(k) + + existingUid = self.hs.get_datastore().get_user_id_by_threepid('email', body['email']) + if existingUid is not None: + raise SynapseError(400, "Email is already in use", Codes.THREEPID_IN_USE) + + if len(absent) > 0: + raise SynapseError(400, "Missing params: %r" % absent, Codes.MISSING_PARAM) + + ret = yield self.identity_handler.requestEmailToken(**body) + defer.returnValue((200, ret)) + def register_servlets(hs, http_server): RegisterRestServlet(hs).register(http_server) diff --git a/synapse/storage/schema/delta/22/user_threepids_unique.sql b/synapse/storage/schema/delta/22/user_threepids_unique.sql new file mode 100644 index 0000000000..87edfa454c --- /dev/null +++ b/synapse/storage/schema/delta/22/user_threepids_unique.sql @@ -0,0 +1,19 @@ +CREATE TABLE IF NOT EXISTS user_threepids2 ( + user_id TEXT NOT NULL, + medium TEXT NOT NULL, + address TEXT NOT NULL, + validated_at BIGINT NOT NULL, + added_at BIGINT NOT NULL, + CONSTRAINT medium_address UNIQUE (medium, address) +); + +INSERT INTO user_threepids2 + SELECT * FROM user_threepids WHERE added_at IN ( + SELECT max(added_at) FROM user_threepids GROUP BY medium, address + ) +; + +DROP TABLE user_threepids; +ALTER TABLE user_threepids2 RENAME TO user_threepids; + +CREATE INDEX user_threepids_user_id ON user_threepids(user_id); From e1241285420556b8afb16aacb3eeb8f29e9859f0 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 4 Aug 2015 14:50:31 +0100 Subject: [PATCH 05/29] Bump schema version --- synapse/storage/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 71d5d92500..99467dde02 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -54,7 +54,7 @@ logger = logging.getLogger(__name__) # Remember to update this number every time a change is made to database # schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 21 +SCHEMA_VERSION = 22 dir_path = os.path.abspath(os.path.dirname(__file__)) From 07ad03d5df0e3e797a23c5994308d77bf60bada3 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 4 Aug 2015 15:18:40 +0100 Subject: [PATCH 06/29] Fix tests --- tests/rest/client/v2_alpha/test_register.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/rest/client/v2_alpha/test_register.py b/tests/rest/client/v2_alpha/test_register.py index 66fd25964d..f9a2b22485 100644 --- a/tests/rest/client/v2_alpha/test_register.py +++ b/tests/rest/client/v2_alpha/test_register.py @@ -13,6 +13,7 @@ class RegisterRestServletTestCase(unittest.TestCase): self.request_data = "" self.request = Mock( content=Mock(read=Mock(side_effect=lambda: self.request_data)), + path='/_matrix/api/v2_alpha/register' ) self.request.args = {} @@ -131,4 +132,4 @@ class RegisterRestServletTestCase(unittest.TestCase): }) self.registration_handler.register = Mock(return_value=("@user:id", "t")) d = self.servlet.on_POST(self.request) - return self.assertFailure(d, SynapseError) \ No newline at end of file + return self.assertFailure(d, SynapseError) From 883aabe4236f466b404bae7982dcb3b375dc53e1 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 4 Aug 2015 15:20:35 +0100 Subject: [PATCH 07/29] splt long line --- synapse/rest/client/v2_alpha/register.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index 7b97a73df6..93f9223708 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -224,7 +224,9 @@ class RegisterRestServlet(RestServlet): if k not in body: absent.append(k) - existingUid = self.hs.get_datastore().get_user_id_by_threepid('email', body['email']) + existingUid = self.hs.get_datastore().get_user_id_by_threepid( + 'email', body['email'] + ) if existingUid is not None: raise SynapseError(400, "Email is already in use", Codes.THREEPID_IN_USE) From a0dea6eaed2315bff018f86820ed7a866ab0d2ef Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 4 Aug 2015 16:18:17 +0100 Subject: [PATCH 08/29] Remember to yield: not much point testing is a deferred is not None --- synapse/rest/client/v2_alpha/register.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index 93f9223708..25dab6f9c0 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -224,9 +224,10 @@ class RegisterRestServlet(RestServlet): if k not in body: absent.append(k) - existingUid = self.hs.get_datastore().get_user_id_by_threepid( + existingUid = yield self.hs.get_datastore().get_user_id_by_threepid( 'email', body['email'] ) + if existingUid is not None: raise SynapseError(400, "Email is already in use", Codes.THREEPID_IN_USE) From 185ac7ee6cde22b9b491ac97013b029071ec9d53 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 4 Aug 2015 16:29:54 +0100 Subject: [PATCH 09/29] Allow sign in using email address --- synapse/rest/client/v1/login.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/synapse/rest/client/v1/login.py b/synapse/rest/client/v1/login.py index 998d4d44c6..8ce3364820 100644 --- a/synapse/rest/client/v1/login.py +++ b/synapse/rest/client/v1/login.py @@ -74,17 +74,24 @@ class LoginRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def do_password_login(self, login_submission): - if not login_submission["user"].startswith('@'): - login_submission["user"] = UserID.create( - login_submission["user"], self.hs.hostname).to_string() + if 'medium' in login_submission and 'address' in login_submission: + user_id = yield self.hs.get_datastore().get_user_id_by_threepid( + login_submission['medium'], login_submission['address'] + ) + else: + user_id = login_submission['user'] + + if not user_id.startswith('@'): + user_id = UserID.create( + user_id, self.hs.hostname).to_string() handler = self.handlers.login_handler token = yield handler.login( - user=login_submission["user"], + user=user_id, password=login_submission["password"]) result = { - "user_id": login_submission["user"], # may have changed + "user_id": user_id, # may have changed "access_token": token, "home_server": self.hs.hostname, } From 7e3d1c7d92157a3cce8ed975f2a982a6a80693d0 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 12 Aug 2015 10:54:38 +0100 Subject: [PATCH 10/29] Make a config option for whether to generate new thumbnail sizes dynamically --- synapse/config/repository.py | 8 +++++++ synapse/rest/media/v1/base_resource.py | 1 + synapse/rest/media/v1/thumbnail_resource.py | 25 +++++++++++++++------ 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/synapse/config/repository.py b/synapse/config/repository.py index 6891abd71d..748dd14e23 100644 --- a/synapse/config/repository.py +++ b/synapse/config/repository.py @@ -22,6 +22,7 @@ class ContentRepositoryConfig(Config): self.max_image_pixels = self.parse_size(config["max_image_pixels"]) self.media_store_path = self.ensure_directory(config["media_store_path"]) self.uploads_path = self.ensure_directory(config["uploads_path"]) + self.dynamic_thumbnails = config["dynamic_thumbnails"] def default_config(self, config_dir_path, server_name): media_store = self.default_path("media_store") @@ -38,4 +39,11 @@ class ContentRepositoryConfig(Config): # Maximum number of pixels that will be thumbnailed max_image_pixels: "32M" + + # Whether to generate new thumbnails on the fly to precisely match + # the resolution requested by the client. If true then whenever + # a new resolution is requested by the client the server will + # generate a new thumbnail. If false the server will pick a thumbnail + # from a precalcualted list. + dynamic_thumbnails: false """ % locals() diff --git a/synapse/rest/media/v1/base_resource.py b/synapse/rest/media/v1/base_resource.py index 093ba847d3..e39729489e 100644 --- a/synapse/rest/media/v1/base_resource.py +++ b/synapse/rest/media/v1/base_resource.py @@ -69,6 +69,7 @@ class BaseMediaResource(Resource): self.filepaths = filepaths self.version_string = hs.version_string self.downloads = {} + self.dynamic_thumbnails = hs.config.dynamic_thumbnails def _respond_404(self, request): respond_with_json( diff --git a/synapse/rest/media/v1/thumbnail_resource.py b/synapse/rest/media/v1/thumbnail_resource.py index 9387258a7a..e506dad934 100644 --- a/synapse/rest/media/v1/thumbnail_resource.py +++ b/synapse/rest/media/v1/thumbnail_resource.py @@ -43,14 +43,25 @@ class ThumbnailResource(BaseMediaResource): m_type = parse_string(request, "type", "image/png") if server_name == self.server_name: - yield self._select_or_generate_local_thumbnail( - request, media_id, width, height, method, m_type - ) + if self.dynamic_thumbnails: + yield self._select_or_generate_local_thumbnail( + request, media_id, width, height, method, m_type + ) + else: + yield self._respond_local_thumbnail( + request, media_id, width, height, method, m_type + ) else: - yield self._select_or_generate_remote_thumbnail( - request, server_name, media_id, - width, height, method, m_type - ) + if self.dynamic_thumbnails: + yield self._select_or_generate_remote_thumbnail( + request, server_name, media_id, + width, height, method, m_type + ) + else: + yield self._respond_remote_thumbnail( + request, server_name, media_id, + width, height, method, m_type + ) @defer.inlineCallbacks def _respond_local_thumbnail(self, request, media_id, width, height, From fdb724cb7040a7746b2a6c6d8aabbf3654daf8dd Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 12 Aug 2015 10:55:27 +0100 Subject: [PATCH 11/29] Add config option for setting the list of thumbnail sizes to precalculate --- synapse/config/repository.py | 39 ++++++++++++++++++++++++++ synapse/rest/media/v1/base_resource.py | 18 ++---------- 2 files changed, 41 insertions(+), 16 deletions(-) diff --git a/synapse/config/repository.py b/synapse/config/repository.py index 748dd14e23..7cab874422 100644 --- a/synapse/config/repository.py +++ b/synapse/config/repository.py @@ -14,6 +14,27 @@ # limitations under the License. from ._base import Config +from collections import namedtuple + +ThumbnailRequirement = namedtuple( + "ThumbnailRequirement", ["width", "height", "method", "media_type"] +) + +def parse_thumbnail_requirements(thumbnail_sizes): + requirements = {} + for size in thumbnail_sizes: + width = size["width"] + height = size["height"] + method = size["method"] + jpeg_thumbnail = ThumbnailRequirement(width, height, method, "image/jpeg") + png_thumbnail = ThumbnailRequirement(width, height, method, "image/png") + requirements.setdefault("image/jpeg", []).append(jpeg_thumbnail) + requirements.setdefault("image/gif", []).append(png_thumbnail) + requirements.setdefault("image/png", []).append(png_thumbnail) + return { + media_type: tuple(thumbnails) + for media_type, thumbnails in requirements.items() + } class ContentRepositoryConfig(Config): @@ -23,6 +44,9 @@ class ContentRepositoryConfig(Config): self.media_store_path = self.ensure_directory(config["media_store_path"]) self.uploads_path = self.ensure_directory(config["uploads_path"]) self.dynamic_thumbnails = config["dynamic_thumbnails"] + self.thumbnail_requirements = parse_thumbnail_requirements( + config["thumbnail_sizes"] + ) def default_config(self, config_dir_path, server_name): media_store = self.default_path("media_store") @@ -46,4 +70,19 @@ class ContentRepositoryConfig(Config): # generate a new thumbnail. If false the server will pick a thumbnail # from a precalcualted list. dynamic_thumbnails: false + + # List of thumbnail to precalculate when an image is uploaded. + thumbnail_sizes: + - width: 32 + height: 32 + method: crop + - width: 96 + height: 96 + method: crop + - width: 320 + height: 240 + method: scale + - width: 640 + height: 480 + method: scale """ % locals() diff --git a/synapse/rest/media/v1/base_resource.py b/synapse/rest/media/v1/base_resource.py index e39729489e..271cbca2d6 100644 --- a/synapse/rest/media/v1/base_resource.py +++ b/synapse/rest/media/v1/base_resource.py @@ -70,6 +70,7 @@ class BaseMediaResource(Resource): self.version_string = hs.version_string self.downloads = {} self.dynamic_thumbnails = hs.config.dynamic_thumbnails + self.thumbnail_requirements = hs.config.thumbnail_requirements def _respond_404(self, request): respond_with_json( @@ -209,22 +210,7 @@ class BaseMediaResource(Resource): self._respond_404(request) def _get_thumbnail_requirements(self, media_type): - if media_type == "image/jpeg": - return ( - (32, 32, "crop", "image/jpeg"), - (96, 96, "crop", "image/jpeg"), - (320, 240, "scale", "image/jpeg"), - (640, 480, "scale", "image/jpeg"), - ) - elif (media_type == "image/png") or (media_type == "image/gif"): - return ( - (32, 32, "crop", "image/png"), - (96, 96, "crop", "image/png"), - (320, 240, "scale", "image/png"), - (640, 480, "scale", "image/png"), - ) - else: - return () + return self.thumbnail_requirements.get(media_type, ()) def _generate_thumbnail(self, input_path, t_path, t_width, t_height, t_method, t_type): From de3b7b55d68ebad3d535dad2643e747460dc22a3 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 12 Aug 2015 14:29:17 +0100 Subject: [PATCH 12/29] Doc-string for config ultility function --- synapse/config/repository.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/synapse/config/repository.py b/synapse/config/repository.py index 7cab874422..2cb0812d7d 100644 --- a/synapse/config/repository.py +++ b/synapse/config/repository.py @@ -21,6 +21,17 @@ ThumbnailRequirement = namedtuple( ) def parse_thumbnail_requirements(thumbnail_sizes): + """ Takes a list of dictionaries with "width", "height", and "method" keys + and creates a map from image media types to the thumbnail size, thumnailing + method, and thumbnail media type to precalculate + + Args: + thumbnail_sizes(list): List of dicts with "width", "height", and + "method" keys + Returns: + Dictionary mapping from media type string to list of + ThumbnailRequirement tuples. + """ requirements = {} for size in thumbnail_sizes: width = size["width"] From 73605f80705f4a0c7271cabfee40a47c7764d72d Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 12 Aug 2015 15:40:54 +0100 Subject: [PATCH 13/29] Just leaving off the $ is fine. r* == registerrrrrrrrr --- synapse/rest/client/v2_alpha/register.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index 25dab6f9c0..e6ad35aa16 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -41,7 +41,7 @@ logger = logging.getLogger(__name__) class RegisterRestServlet(RestServlet): - PATTERN = client_v2_pattern("/register*") + PATTERN = client_v2_pattern("/register") def __init__(self, hs): super(RegisterRestServlet, self).__init__() From f43041aacd36b7d9052476bcb0d083ea4213a9f9 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 12 Aug 2015 15:44:08 +0100 Subject: [PATCH 14/29] Check absent before trying to access keys --- synapse/rest/client/v2_alpha/register.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index e6ad35aa16..254c5f1ddf 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -224,6 +224,9 @@ class RegisterRestServlet(RestServlet): if k not in body: absent.append(k) + if len(absent) > 0: + raise SynapseError(400, "Missing params: %r" % absent, Codes.MISSING_PARAM) + existingUid = yield self.hs.get_datastore().get_user_id_by_threepid( 'email', body['email'] ) @@ -231,9 +234,6 @@ class RegisterRestServlet(RestServlet): if existingUid is not None: raise SynapseError(400, "Email is already in use", Codes.THREEPID_IN_USE) - if len(absent) > 0: - raise SynapseError(400, "Missing params: %r" % absent, Codes.MISSING_PARAM) - ret = yield self.identity_handler.requestEmailToken(**body) defer.returnValue((200, ret)) From 95b0f5449d2b7539a7817b231090d607e55d6ac7 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 13 Aug 2015 17:34:22 +0100 Subject: [PATCH 15/29] Fix flake8 warning --- synapse/config/repository.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/config/repository.py b/synapse/config/repository.py index 2cb0812d7d..64644b9a7a 100644 --- a/synapse/config/repository.py +++ b/synapse/config/repository.py @@ -20,6 +20,7 @@ ThumbnailRequirement = namedtuple( "ThumbnailRequirement", ["width", "height", "method", "media_type"] ) + def parse_thumbnail_requirements(thumbnail_sizes): """ Takes a list of dictionaries with "width", "height", and "method" keys and creates a map from image media types to the thumbnail size, thumnailing From a45ec7c651abdbe54bba4a8fe420e02c9059f390 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 19 Aug 2015 10:08:12 +0100 Subject: [PATCH 16/29] Block on storing the current last_tokens --- synapse/push/__init__.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 36f450c31d..483deb624c 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -345,7 +345,7 @@ class Pusher(object): if processed: self.backoff_delay = Pusher.INITIAL_BACKOFF self.last_token = chunk['end'] - self.store.update_pusher_last_token_and_success( + yield self.store.update_pusher_last_token_and_success( self.app_id, self.pushkey, self.user_name, @@ -354,7 +354,7 @@ class Pusher(object): ) if self.failing_since: self.failing_since = None - self.store.update_pusher_failing_since( + yield self.store.update_pusher_failing_since( self.app_id, self.pushkey, self.user_name, @@ -362,7 +362,7 @@ class Pusher(object): else: if not self.failing_since: self.failing_since = self.clock.time_msec() - self.store.update_pusher_failing_since( + yield self.store.update_pusher_failing_since( self.app_id, self.pushkey, self.user_name, @@ -380,7 +380,7 @@ class Pusher(object): self.user_name, self.pushkey) self.backoff_delay = Pusher.INITIAL_BACKOFF self.last_token = chunk['end'] - self.store.update_pusher_last_token( + yield self.store.update_pusher_last_token( self.app_id, self.pushkey, self.user_name, @@ -388,7 +388,7 @@ class Pusher(object): ) self.failing_since = None - self.store.update_pusher_failing_since( + yield self.store.update_pusher_failing_since( self.app_id, self.pushkey, self.user_name, From 78fa346b0781cf42ef8772638f9f8abb26b9a36f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 19 Aug 2015 10:08:31 +0100 Subject: [PATCH 17/29] Store the 'last_token' in the db, even if we processed no events --- synapse/push/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 483deb624c..13002e0db4 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -294,6 +294,12 @@ class Pusher(object): if not single_event: self.last_token = chunk['end'] logger.debug("Event stream timeout for pushkey %s", self.pushkey) + yield self.store.update_pusher_last_token( + self.app_id, + self.pushkey, + self.user_name, + self.last_token + ) return if not self.alive: From d7272f8d9d0ce3ac9a4095969453efef5aecce40 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 19 Aug 2015 12:03:09 +0100 Subject: [PATCH 18/29] Add canonical alias to the default power levels --- synapse/api/constants.py | 1 + synapse/handlers/room.py | 1 + 2 files changed, 2 insertions(+) diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 7156ee4e7d..60a0d336da 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -76,6 +76,7 @@ class EventTypes(object): Feedback = "m.room.message.feedback" RoomHistoryVisibility = "m.room.history_visibility" + CanonicalAlias = "m.room.canonical_alias" # These are used for validation Message = "m.room.message" diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 7511d294f3..c56112a92a 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -250,6 +250,7 @@ class RoomCreationHandler(BaseHandler): EventTypes.Name: 100, EventTypes.PowerLevels: 100, EventTypes.RoomHistoryVisibility: 100, + EventTypes.CanonicalAlias: 100, }, "events_default": 0, "state_default": 50, From aadb2238c9647186711933666851def5e37a8dbf Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 20 Aug 2015 09:55:04 +0100 Subject: [PATCH 19/29] Check that the canonical room alias actually points to the room --- synapse/handlers/_base.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index d6c064b398..e91f1129db 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -18,7 +18,7 @@ from twisted.internet import defer from synapse.api.errors import LimitExceededError, SynapseError from synapse.crypto.event_signing import add_hashes_and_signatures from synapse.api.constants import Membership, EventTypes -from synapse.types import UserID +from synapse.types import UserID, RoomAlias from synapse.util.logcontext import PreserveLoggingContext @@ -130,6 +130,22 @@ class BaseHandler(object): returned_invite.signatures ) + if event.type == EventTypes.CanonicalAlias: + # Check the alias is acually valid (at this time at least) + room_alias_str = event.content.get("alias", None) + if room_alias_str: + room_alias = RoomAlias.from_string(room_alias_str) + directory_handler = self.hs.get_handlers().directory_handler + mapping = yield directory_handler.get_association(room_alias) + + if mapping["room_id"] != event.room_id: + raise SynapseError( + 400, + "Room alias %s does not point to the room" % ( + room_alias_str, + ) + ) + destinations = set(extra_destinations) for k, s in context.current_state.items(): try: From 4cf302de5b1b4f4494a9445e0d85e4c9c24ff73d Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 20 Aug 2015 10:31:18 +0100 Subject: [PATCH 20/29] Comma comma comma comma comma chameleon --- synapse/rest/client/v1/login.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/rest/client/v1/login.py b/synapse/rest/client/v1/login.py index cb97917608..0d5eafd0fa 100644 --- a/synapse/rest/client/v1/login.py +++ b/synapse/rest/client/v1/login.py @@ -86,7 +86,7 @@ class LoginRestServlet(ClientV1RestServlet): user_id, self.hs.hostname).to_string() token = yield self.handlers.auth_handler.login_with_password( - user_id=user_id + user_id=user_id, password=login_submission["password"]) result = { From 9d720223f25982500ba4076caef9b70b608e0292 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 20 Aug 2015 14:12:01 +0100 Subject: [PATCH 21/29] Bump version and changelog --- CHANGES.rst | 73 +++++++++++++++++++++++++++++++++++++++++++++ synapse/__init__.py | 2 +- 2 files changed, 74 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 6a5fce899a..e2c6afb6ae 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,76 @@ +Changes in synapse v0.10.0-rc1 (2015-08-20) +=========================================== + +Also see v0.9.4-rc1 changelog, which has been amalgamated into this release. + +General: + +* Upgrade to Twisted 15 (PR #173) +* Add support for serving and fetching encryption keys over federation. + (PR #208) +* Add support for logging in with email address (PR #234) +* Add support for new ``m.room.canonical_alias`` event. (PR #233) +* Error if a user tries to register with an email already in use. (PR #211) +* Add extra and improve existing caches (PR #212, #219, #226, #228) +* Batch various storage request (PR #226, #228) +* Fix bug where we didn't correctly log the entity that triggered the request + if the request came in via an application service (PR #230) +* Fix bug where we needlessly regenerated the full list of rooms an AS is + interested in. (PR #232) + + +Configuration: + +* Add ``--generate-keys`` that will generate any missing cert and key files in + the configuration files. This is equivalent to running ``--generate-config`` + on an existing configuration file. (PR #220) +* ``--generate-config`` now no longer requires a ``--server-name`` parameter + when used on existing configuration files. (PR #220) +* Add ``--print-pidfile`` flag that controls the printing of the pid to stdout + of the demonised process. (PR #213) + +Media Repository: + +* Fix bug where we picked a lower resolution image than requested. (PR #205) +* Add support for specifying if a the media repository should dynamically + thumbnail images or not. (PR #206) + +Metrics: + +* Add statistics from the reactor to the metrics API. (PR #224, #225) + +Demo Homeservers: + +* Fix starting the demo homeservers without rate-limiting enabled. (PR #182) +* Fix enabling registration on demo homeservers (PR #223) + + +Changes in synapse v0.9.4-rc1 (2015-07-21) +========================================== + +General: + +* Add basic implementation of receipts. (SPEC-99) +* Add support for configuration presets in room creation API. (PR #203) +* Add auth event that limits the visibility of history for new users. + (SPEC-134) +* Add SAML2 login/registration support. (PR #201. Thanks Muthu Subramanian!) +* Add client side key management APIs for end to end encryption. (PR #198) +* Change power level semantics so that you cannot kick, ban or change power + levels of users that have equal or greater power level than you. (SYN-192) +* Improve performance by bulk inserting events where possible. (PR #193) +* Improve performance by bulk verifying signatures where possible. (PR #194) + + +Configuration: + +* Add support for including TLS certificate chains. + +Media Repository: + +* Add Content-Disposition headers to content repository responses. (SYN-150) + + Changes in synapse v0.9.3 (2015-07-01) ====================================== diff --git a/synapse/__init__.py b/synapse/__init__.py index 96e37308d6..5853165a21 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.9.3" +__version__ = "0.10.0-rc1" From 23b21e5215d3a4d88925aa28e5bcaeda169d81cc Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 20 Aug 2015 14:25:57 +0100 Subject: [PATCH 22/29] Update changelog --- CHANGES.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES.rst b/CHANGES.rst index e2c6afb6ae..e6d1a37307 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -17,6 +17,7 @@ General: if the request came in via an application service (PR #230) * Fix bug where we needlessly regenerated the full list of rooms an AS is interested in. (PR #232) +* Add support for AS's to use v2_alpha registration API (PR #210) Configuration: From 9b63def3887779c7c9a1aeadd2d16df506155953 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 20 Aug 2015 14:35:40 +0100 Subject: [PATCH 23/29] Add m.room.avatar to default power levels. Change default required power levels of such events to 50 --- synapse/api/constants.py | 1 + synapse/handlers/room.py | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 60a0d336da..1423986c1e 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -77,6 +77,7 @@ class EventTypes(object): RoomHistoryVisibility = "m.room.history_visibility" CanonicalAlias = "m.room.canonical_alias" + RoomAvatar = "m.room.avatar" # These are used for validation Message = "m.room.message" diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 8108c2763d..c5d1001b50 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -247,10 +247,11 @@ class RoomCreationHandler(BaseHandler): }, "users_default": 0, "events": { - EventTypes.Name: 100, + EventTypes.Name: 50, EventTypes.PowerLevels: 100, EventTypes.RoomHistoryVisibility: 100, - EventTypes.CanonicalAlias: 100, + EventTypes.CanonicalAlias: 50, + EventTypes.RoomAvatar: 50, }, "events_default": 0, "state_default": 50, From bb9611bd46c77252e7ecf3ef90251cf4cd2f446e Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 20 Aug 2015 15:08:18 +0100 Subject: [PATCH 24/29] Add generic update instructions to UPGRADE.rst and add link to them from the README.rst --- README.rst | 11 ++++------- UPGRADE.rst | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/README.rst b/README.rst index 5ff53f2df7..b01d693a47 100644 --- a/README.rst +++ b/README.rst @@ -362,14 +362,11 @@ This should end with a 'PASSED' result:: Upgrading an existing Synapse ============================= -IMPORTANT: Before upgrading an existing synapse to a new version, please -refer to UPGRADE.rst for any additional instructions. - -Otherwise, simply re-install the new codebase over the current one - e.g. -by ``pip install --process-dependency-links -https://github.com/matrix-org/synapse/tarball/master`` -if using pip, or by ``git pull`` if running off a git working copy. +The instructions for upgrading synapse are in `UPGRADE.rst`_. +Please check these instructions as upgrading may require extra steps for some +versions of synapse. +.. _UPGRADE.rst: UPGRADE.rst Setting up Federation ===================== diff --git a/UPGRADE.rst b/UPGRADE.rst index d98460f643..a82dabbd3c 100644 --- a/UPGRADE.rst +++ b/UPGRADE.rst @@ -1,3 +1,36 @@ +Upgrading Synapse +================= + +Before upgrading check if any special steps are required to upgrade from the +what you currently have installed to current version of synapse. The extra +instructions that may be required are listed later in this document. + +If you installed synapse in a virtualenv then active that virtualenv before +upgrading. If synapse is installed in a virtualenv in ``~/.synapse/`` then run: + +.. code:: bash + + source ~/.synapse/bin/activate + +If you installed synapse using pip then upgrade to the latest version by +running: + +.. code:: bash + + pip install --upgrade --process-dependency-links https://github.com/matrix-org/synapse/tarball/master + +If you installed synapse using git then upgrade to the latest version by +running: + +.. code:: bash + + # Pull the latest version of the master branch. + git pull + # Update the versions of synapses python dependencies. + python synapse/python_dependencies.py | xargs -n1 pip install + + + Upgrading to v0.9.0 =================== From fd88ea19c0702d970af3a737a094234be1809b9e Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 20 Aug 2015 15:12:44 +0100 Subject: [PATCH 25/29] Tweak the wording a bit --- UPGRADE.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/UPGRADE.rst b/UPGRADE.rst index a82dabbd3c..35a0333a7e 100644 --- a/UPGRADE.rst +++ b/UPGRADE.rst @@ -5,28 +5,28 @@ Before upgrading check if any special steps are required to upgrade from the what you currently have installed to current version of synapse. The extra instructions that may be required are listed later in this document. -If you installed synapse in a virtualenv then active that virtualenv before +If synapse was installed in a virtualenv then active that virtualenv before upgrading. If synapse is installed in a virtualenv in ``~/.synapse/`` then run: .. code:: bash source ~/.synapse/bin/activate -If you installed synapse using pip then upgrade to the latest version by +If synapse was installed using pip then upgrade to the latest version by running: .. code:: bash pip install --upgrade --process-dependency-links https://github.com/matrix-org/synapse/tarball/master -If you installed synapse using git then upgrade to the latest version by +If synapse was installed using git then upgrade to the latest version by running: .. code:: bash # Pull the latest version of the master branch. git pull - # Update the versions of synapses python dependencies. + # Update the versions of synapse's python dependencies. python synapse/python_dependencies.py | xargs -n1 pip install From 482648123f608fb6e77dc47960ddda501c8f5c70 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 20 Aug 2015 15:20:07 +0100 Subject: [PATCH 26/29] Clean up some of restructured text formatting in the README.rst --- README.rst | 62 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 32 insertions(+), 30 deletions(-) diff --git a/README.rst b/README.rst index 5ff53f2df7..01d8cb1a78 100644 --- a/README.rst +++ b/README.rst @@ -7,7 +7,7 @@ Matrix is an ambitious new ecosystem for open federated Instant Messaging and VoIP. The basics you need to know to get up and running are: - Everything in Matrix happens in a room. Rooms are distributed and do not - exist on any single server. Rooms can be located using convenience aliases + exist on any single server. Rooms can be located using convenience aliases like ``#matrix:matrix.org`` or ``#test:localhost:8448``. - Matrix user IDs look like ``@matthew:matrix.org`` (although in the future @@ -23,7 +23,7 @@ The overall architecture is:: accessed by the web client at http://matrix.org/beta or via an IRC bridge at irc://irc.freenode.net/matrix. -Synapse is currently in rapid development, but as of version 0.5 we believe it +Synapse is currently in rapid development, but as of version 0.5 we believe it is sufficiently stable to be run as an internet-facing service for real usage! About Matrix @@ -104,7 +104,7 @@ Installing prerequisites on Ubuntu or Debian:: sudo apt-get install build-essential python2.7-dev libffi-dev \ python-pip python-setuptools sqlite3 \ libssl-dev python-virtualenv libjpeg-dev - + Installing prerequisites on ArchLinux:: sudo pacman -S base-devel python2 python-pip \ @@ -115,7 +115,7 @@ Installing prerequisites on Mac OS X:: xcode-select --install sudo easy_install pip sudo pip install virtualenv - + To install the synapse homeserver run:: virtualenv -p python2.7 ~/.synapse @@ -180,7 +180,7 @@ The advantages of Postgres include: * allowing basic active/backup high-availability with a "hot spare" synapse pointing at the same DB master, as well as enabling DB replication in synapse itself. - + The only disadvantage is that the code is relatively new as of April 2015 and may have a few regressions relative to SQLite. @@ -190,8 +190,8 @@ For information on how to install and use PostgreSQL, please see Running Synapse =============== -To actually run your new homeserver, pick a working directory for Synapse to run -(e.g. ``~/.synapse``), and:: +To actually run your new homeserver, pick a working directory for Synapse to +run (e.g. ``~/.synapse``), and:: cd ~/.synapse source ./bin/activate @@ -214,13 +214,13 @@ defaults to python 3, but synapse currently assumes python 2.7 by default: pip may be outdated (6.0.7-1 and needs to be upgraded to 6.0.8-1 ):: sudo pip2.7 install --upgrade pip - + You also may need to explicitly specify python 2.7 again during the install request:: pip2.7 install --process-dependency-links \ https://github.com/matrix-org/synapse/tarball/master - + If you encounter an error with lib bcrypt causing an Wrong ELF Class: ELFCLASS32 (x64 Systems), you may need to reinstall py-bcrypt to correctly compile it under the right architecture. (This should not be needed if @@ -228,7 +228,7 @@ installing under virtualenv):: sudo pip2.7 uninstall py-bcrypt sudo pip2.7 install py-bcrypt - + During setup of Synapse you need to call python2.7 directly again:: cd ~/.synapse @@ -236,25 +236,27 @@ During setup of Synapse you need to call python2.7 directly again:: --server-name machine.my.domain.name \ --config-path homeserver.yaml \ --generate-config - + ...substituting your host and domain name as appropriate. Windows Install --------------- Synapse can be installed on Cygwin. It requires the following Cygwin packages: - - gcc - - git - - libffi-devel - - openssl (and openssl-devel, python-openssl) - - python - - python-setuptools +- gcc +- git +- libffi-devel +- openssl (and openssl-devel, python-openssl) +- python +- python-setuptools The content repository requires additional packages and will be unable to process uploads without them: - - libjpeg8 - - libjpeg8-devel - - zlib + +- libjpeg8 +- libjpeg8-devel +- zlib + If you choose to install Synapse without these packages, you will need to reinstall ``pillow`` for changes to be applied, e.g. ``pip uninstall pillow`` ``pip install pillow --user`` @@ -276,8 +278,8 @@ Troubleshooting Troubleshooting Installation ---------------------------- -Synapse requires pip 1.7 or later, so if your OS provides too old a version and -you get errors about ``error: no such option: --process-dependency-links`` you +Synapse requires pip 1.7 or later, so if your OS provides too old a version and +you get errors about ``error: no such option: --process-dependency-links`` you may need to manually upgrade it:: sudo pip install --upgrade pip @@ -288,9 +290,9 @@ created. To reset the installation:: rm -rf /tmp/pip_install_matrix -pip seems to leak *lots* of memory during installation. For instance, a Linux -host with 512MB of RAM may run out of memory whilst installing Twisted. If this -happens, you will have to individually install the dependencies which are +pip seems to leak *lots* of memory during installation. For instance, a Linux +host with 512MB of RAM may run out of memory whilst installing Twisted. If this +happens, you will have to individually install the dependencies which are failing, e.g.:: pip install twisted @@ -301,8 +303,8 @@ will need to export CFLAGS=-Qunused-arguments. Troubleshooting Running ----------------------- -If synapse fails with ``missing "sodium.h"`` crypto errors, you may need -to manually upgrade PyNaCL, as synapse uses NaCl (http://nacl.cr.yp.to/) for +If synapse fails with ``missing "sodium.h"`` crypto errors, you may need +to manually upgrade PyNaCL, as synapse uses NaCl (http://nacl.cr.yp.to/) for encryption and digital signatures. Unfortunately PyNACL currently has a few issues (https://github.com/pyca/pynacl/issues/53) and @@ -313,7 +315,7 @@ fix try re-installing from PyPI or directly from # Install from PyPI pip install --user --upgrade --force pynacl - + # Install from github pip install --user https://github.com/pyca/pynacl/tarball/master @@ -431,7 +433,7 @@ private federation (``localhost:8080``, ``localhost:8081`` and http://localhost:8080. Simply run:: demo/start.sh - + This is mainly useful just for development purposes. Running The Demo Web Client @@ -494,7 +496,7 @@ time. Where's the spec?! ================== -The source of the matrix spec lives at https://github.com/matrix-org/matrix-doc. +The source of the matrix spec lives at https://github.com/matrix-org/matrix-doc. A recent HTML snapshot of this lives at http://matrix.org/docs/spec From 8a951540f6b47c8cc33dc41f3178295c55350c74 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 20 Aug 2015 15:22:26 +0100 Subject: [PATCH 27/29] Further formatting clean ups --- README.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.rst b/README.rst index 01d8cb1a78..e63f86b607 100644 --- a/README.rst +++ b/README.rst @@ -174,12 +174,12 @@ traditionally used for convenience and simplicity. The advantages of Postgres include: - * significant performance improvements due to the superior threading and - caching model, smarter query optimiser - * allowing the DB to be run on separate hardware - * allowing basic active/backup high-availability with a "hot spare" synapse - pointing at the same DB master, as well as enabling DB replication in - synapse itself. +* significant performance improvements due to the superior threading and + caching model, smarter query optimiser +* allowing the DB to be run on separate hardware +* allowing basic active/backup high-availability with a "hot spare" synapse + pointing at the same DB master, as well as enabling DB replication in + synapse itself. The only disadvantage is that the code is relatively new as of April 2015 and may have a few regressions relative to SQLite. @@ -326,7 +326,7 @@ If running `$ synctl start` fails with 'returned non-zero exit status 1', you will need to explicitly call Python2.7 - either running as:: python2.7 -m synapse.app.homeserver --daemonize -c homeserver.yaml - + ...or by editing synctl with the correct python executable. Synapse Development From ca0d28ef34022874ebc9168146df53a10bcb925e Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 20 Aug 2015 15:35:14 +0100 Subject: [PATCH 28/29] Another use of check_password that got missed in the yield fix --- synapse/handlers/auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index be2baeaece..ff2c66f442 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -162,7 +162,7 @@ class AuthHandler(BaseHandler): if not user_id.startswith('@'): user_id = UserID.create(user_id, self.hs.hostname).to_string() - self._check_password(user_id, password) + yield self._check_password(user_id, password) defer.returnValue(user_id) @defer.inlineCallbacks From f764f9264734fdcf83869022fed23bdab5e4c8dc Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 20 Aug 2015 15:35:54 +0100 Subject: [PATCH 29/29] Remove spurious extra arg to set_password --- synapse/rest/client/v2_alpha/account.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py index 897c54b539..522a312c9e 100644 --- a/synapse/rest/client/v2_alpha/account.py +++ b/synapse/rest/client/v2_alpha/account.py @@ -79,7 +79,7 @@ class PasswordRestServlet(RestServlet): new_password = params['new_password'] yield self.auth_handler.set_password( - user_id, new_password, None + user_id, new_password ) defer.returnValue((200, {}))