From ab97b6e33c6e141d185ec203d24ea8266f924900 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Thu, 17 Jan 2019 15:52:57 +0200 Subject: [PATCH 01/66] Fix a test docstring in frontend proxy tests Signed-off-by: Jason Robinson --- tests/app/test_frontend_proxy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/app/test_frontend_proxy.py b/tests/app/test_frontend_proxy.py index a83f567ebd..8bdbc608a9 100644 --- a/tests/app/test_frontend_proxy.py +++ b/tests/app/test_frontend_proxy.py @@ -59,7 +59,7 @@ class FrontendProxyTests(HomeserverTestCase): def test_listen_http_with_presence_disabled(self): """ - When presence is on, the stub servlet will register. + When presence is off, the stub servlet will register. """ # Presence is off self.hs.config.use_presence = False From 899e60be80736e3b747eb567171296829e96f716 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Thu, 17 Jan 2019 15:55:37 +0200 Subject: [PATCH 02/66] Add parameterized Python module to test dependencies Allows running parameterized tests. BSD license. Signed-off-by: Jason Robinson --- synapse/python_dependencies.py | 2 +- tox.ini | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 882e844eb1..b5555fbba9 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -81,7 +81,7 @@ CONDITIONAL_REQUIREMENTS = { "saml2": ["pysaml2>=4.5.0"], "url_preview": ["lxml>=3.5.0"], - "test": ["mock>=2.0"], + "test": ["mock>=2.0", "parameterized"], } diff --git a/tox.ini b/tox.ini index a0f5486829..9b6cb2036c 100644 --- a/tox.ini +++ b/tox.ini @@ -8,6 +8,7 @@ deps = python-subunit junitxml coverage + parameterized # cyptography 2.2 requires setuptools >= 18.5 # From 4f8f41c824dc326903863625ce79e5386ad0f8e1 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Thu, 17 Jan 2019 15:58:27 +0200 Subject: [PATCH 03/66] Make FederationReaderServer _http_listen use self.get_reactor() For all the homeserver classes, only the FrontendProxyServer passes its reactor when doing the http listen. Looking at previous PR's looks like this was introduced to make it possible to write a test, otherwise when you try to run a test with the test homeserver it tries to do a real bind to a port. Passing the reactor that the homeserver is instantiated with should probably be the right thing to do anyway? Signed-off-by: Jason Robinson --- synapse/app/federation_reader.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/app/federation_reader.py b/synapse/app/federation_reader.py index 228a297fb8..ea594f0f1a 100644 --- a/synapse/app/federation_reader.py +++ b/synapse/app/federation_reader.py @@ -99,7 +99,8 @@ class FederationReaderServer(HomeServer): listener_config, root_resource, self.version_string, - ) + ), + reactor=self.get_reactor() ) logger.info("Synapse federation reader now listening on port %d", port) From 6d255990983405fe7eb1322c9d02414c859d1e96 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Thu, 17 Jan 2019 15:59:28 +0200 Subject: [PATCH 04/66] Add tests for the openid lister for FederationReaderServer Check all possible variants of openid and federation listener on/off possibilities. Signed-off-by: Jason Robinson --- tests/app/test_openid_listener.py | 66 +++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 tests/app/test_openid_listener.py diff --git a/tests/app/test_openid_listener.py b/tests/app/test_openid_listener.py new file mode 100644 index 0000000000..cf3baad96f --- /dev/null +++ b/tests/app/test_openid_listener.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from mock import patch, Mock +from parameterized import parameterized + +from synapse.app.federation_reader import FederationReaderServer + +from tests.unittest import HomeserverTestCase + + +@patch("synapse.app.homeserver.KeyApiV2Resource", new=Mock()) +class FederationReaderOpenIDListenerTests(HomeserverTestCase): + def make_homeserver(self, reactor, clock): + hs = self.setup_test_homeserver( + http_client=None, homeserverToUse=FederationReaderServer, + ) + return hs + + @parameterized.expand([ + (["federation"], "auth_fail"), + ([], "no_resource"), + (["openid", "federation"], "auth_fail"), + (["openid"], "auth_fail"), + ]) + def test_openid_listener(self, names, expectation): + """ + Test different openid listener configurations. + + 401 is success here since it means we hit the handler and auth failed. + """ + config = { + "port": 8080, + "bind_addresses": ["0.0.0.0"], + "resources": [{"names": names}], + } + + # Listen with the config + self.hs._listen_http(config) + + # Grab the resource from the site that was told to listen + site = self.reactor.tcpServers[0][1] + try: + self.resource = ( + site.resource.children[b"_matrix"].children[b"federation"].children[b"v1"] + ) + except KeyError: + if expectation == "no_resource": + return + raise + + request, channel = self.make_request("GET", "/_matrix/federation/v1/openid/userinfo") + self.render(request) + + self.assertEqual(channel.code, 401) From a17bac171f301e54d6bd3d4c769f4f005c38f112 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Fri, 18 Jan 2019 10:02:08 +0200 Subject: [PATCH 05/66] Make SynapseHomeServer _http_listener use self.get_reactor() For all the homeserver classes, only the FrontendProxyServer passes its reactor when doing the http listen. Looking at previous PR's looks like this was introduced to make it possible to write a test, otherwise when you try to run a test with the test homeserver it tries to do a real bind to a port. Passing the reactor that the homeserver is instantiated with should probably be the right thing to do anyway? Signed-off-by: Jason Robinson --- synapse/app/homeserver.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index f3ac3d19f0..5652c6201c 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -130,6 +130,7 @@ class SynapseHomeServer(HomeServer): self.version_string, ), self.tls_server_context_factory, + reactor=self.get_reactor(), ) else: @@ -142,7 +143,8 @@ class SynapseHomeServer(HomeServer): listener_config, root_resource, self.version_string, - ) + ), + reactor=self.get_reactor(), ) logger.info("Synapse now listening on port %d", port) From 5336e49b39b6ad73491531c68694c92dade028be Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Fri, 18 Jan 2019 10:03:32 +0200 Subject: [PATCH 06/66] Add tests for the openid lister for SynapseHomeServer Check all possible variants of openid and federation listener on/off possibilities. Signed-off-by: Jason Robinson --- tests/app/test_openid_listener.py | 49 ++++++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/tests/app/test_openid_listener.py b/tests/app/test_openid_listener.py index cf3baad96f..329aadc3d3 100644 --- a/tests/app/test_openid_listener.py +++ b/tests/app/test_openid_listener.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2018 New Vector Ltd +# Copyright 2019 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,6 +16,7 @@ from mock import patch, Mock from parameterized import parameterized from synapse.app.federation_reader import FederationReaderServer +from synapse.app.homeserver import SynapseHomeServer from tests.unittest import HomeserverTestCase @@ -64,3 +65,49 @@ class FederationReaderOpenIDListenerTests(HomeserverTestCase): self.render(request) self.assertEqual(channel.code, 401) + + +@patch("synapse.app.homeserver.KeyApiV2Resource", new=Mock()) +class SynapseHomeserverOpenIDListenerTests(HomeserverTestCase): + def make_homeserver(self, reactor, clock): + hs = self.setup_test_homeserver( + http_client=None, homeserverToUse=SynapseHomeServer, + ) + return hs + + @parameterized.expand([ + (["federation"], "auth_fail"), + ([], "no_resource"), + (["openid", "federation"], "auth_fail"), + (["openid"], "auth_fail"), + ]) + def test_openid_listener(self, names, expectation): + """ + Test different openid listener configurations. + + 401 is success here since it means we hit the handler and auth failed. + """ + config = { + "port": 8080, + "bind_addresses": ["0.0.0.0"], + "resources": [{"names": names}], + } + + # Listen with the config + self.hs._listener_http(config, config) + + # Grab the resource from the site that was told to listen + site = self.reactor.tcpServers[0][1] + try: + self.resource = ( + site.resource.children[b"_matrix"].children[b"federation"].children[b"v1"] + ) + except KeyError: + if expectation == "no_resource": + return + raise + + request, channel = self.make_request("GET", "/_matrix/federation/v1/openid/userinfo") + self.render(request) + + self.assertEqual(channel.code, 401) From 82e13662c03a41c085e784a594b423711e0caffa Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Mon, 21 Jan 2019 01:54:43 +0200 Subject: [PATCH 07/66] Split federation OpenID userinfo endpoint out of the federation resource This allows the OpenID userinfo endpoint to be active even if the federation resource is not active. The OpenID userinfo endpoint is called by integration managers to verify user actions using the client API OpenID access token. Without this verification, the integration manager cannot know that the access token is valid. The OpenID userinfo endpoint will be loaded in the case that either "federation" or "openid" resource is defined. The new "openid" resource is defaulted to active in default configuration. Signed-off-by: Jason Robinson --- synapse/app/federation_reader.py | 7 ++ synapse/app/homeserver.py | 9 +++ synapse/config/server.py | 9 ++- synapse/federation/transport/server.py | 106 ++++++++++++++++--------- 4 files changed, 89 insertions(+), 42 deletions(-) diff --git a/synapse/app/federation_reader.py b/synapse/app/federation_reader.py index ea594f0f1a..99e8a4cf6a 100644 --- a/synapse/app/federation_reader.py +++ b/synapse/app/federation_reader.py @@ -87,6 +87,13 @@ class FederationReaderServer(HomeServer): resources.update({ FEDERATION_PREFIX: TransportLayerServer(self), }) + if name == "openid" and "federation" not in res["names"]: + # Only load the openid resource separately if federation resource + # is not specified since federation resource includes openid + # resource. + resources.update({ + FEDERATION_PREFIX: TransportLayerServer(self, servlet_groups=["openid"]), + }) root_resource = create_resource_tree(resources, NoResource()) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 5652c6201c..0a924d7a80 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -95,6 +95,10 @@ class SynapseHomeServer(HomeServer): resources = {} for res in listener_config["resources"]: for name in res["names"]: + if name == "openid" and "federation" in res["names"]: + # Skip loading openid resource if federation is defined + # since federation resource will include openid + continue resources.update(self._configure_named_resource( name, res.get("compress", False), )) @@ -192,6 +196,11 @@ class SynapseHomeServer(HomeServer): FEDERATION_PREFIX: TransportLayerServer(self), }) + if name == "openid": + resources.update({ + FEDERATION_PREFIX: TransportLayerServer(self, servlet_groups=["openid"]), + }) + if name in ["static", "client"]: resources.update({ STATIC_PREFIX: File( diff --git a/synapse/config/server.py b/synapse/config/server.py index fb57791098..556f1efee5 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -151,7 +151,7 @@ class ServerConfig(Config): "compress": gzip_responses, }, { - "names": ["federation"], + "names": ["federation", "openid"], "compress": False, } ] @@ -170,7 +170,7 @@ class ServerConfig(Config): "compress": gzip_responses, }, { - "names": ["federation"], + "names": ["federation", "openid"], "compress": False, } ] @@ -328,7 +328,7 @@ class ServerConfig(Config): # that can do automatic compression. compress: true - - names: [federation] # Federation APIs + - names: [federation, openid] # Federation APIs compress: false # optional list of additional endpoints which can be loaded via @@ -350,7 +350,7 @@ class ServerConfig(Config): resources: - names: [client] compress: true - - names: [federation] + - names: [federation, openid] compress: false # Turn on the twisted ssh manhole service on localhost on the given @@ -477,6 +477,7 @@ KNOWN_RESOURCES = ( 'keys', 'media', 'metrics', + 'openid', 'replication', 'static', 'webclient', diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py index 4557a9e66e..49b80beecb 100644 --- a/synapse/federation/transport/server.py +++ b/synapse/federation/transport/server.py @@ -43,9 +43,10 @@ logger = logging.getLogger(__name__) class TransportLayerServer(JsonResource): """Handles incoming federation HTTP requests""" - def __init__(self, hs): + def __init__(self, hs, servlet_groups=None): self.hs = hs self.clock = hs.get_clock() + self.servlet_groups = servlet_groups super(TransportLayerServer, self).__init__(hs, canonical_json=False) @@ -67,6 +68,7 @@ class TransportLayerServer(JsonResource): resource=self, ratelimiter=self.ratelimiter, authenticator=self.authenticator, + servlet_groups=self.servlet_groups, ) @@ -1308,10 +1310,12 @@ FEDERATION_SERVLET_CLASSES = ( FederationClientKeysClaimServlet, FederationThirdPartyInviteExchangeServlet, On3pidBindServlet, - OpenIdUserInfo, FederationVersionServlet, ) +OPENID_SERVLET_CLASSES = ( + OpenIdUserInfo, +) ROOM_LIST_CLASSES = ( PublicRoomList, @@ -1350,44 +1354,70 @@ GROUP_ATTESTATION_SERVLET_CLASSES = ( FederationGroupsRenewAttestaionServlet, ) +DEFAULT_SERVLET_GROUPS = ( + "federation", + "room_list", + "group_server", + "group_local", + "group_attestation", + "openid", +) -def register_servlets(hs, resource, authenticator, ratelimiter): - for servletclass in FEDERATION_SERVLET_CLASSES: - servletclass( - handler=hs.get_federation_server(), - authenticator=authenticator, - ratelimiter=ratelimiter, - server_name=hs.hostname, - ).register(resource) - for servletclass in ROOM_LIST_CLASSES: - servletclass( - handler=hs.get_room_list_handler(), - authenticator=authenticator, - ratelimiter=ratelimiter, - server_name=hs.hostname, - ).register(resource) +def register_servlets(hs, resource, authenticator, ratelimiter, servlet_groups=None): + if not servlet_groups: + servlet_groups = DEFAULT_SERVLET_GROUPS - for servletclass in GROUP_SERVER_SERVLET_CLASSES: - servletclass( - handler=hs.get_groups_server_handler(), - authenticator=authenticator, - ratelimiter=ratelimiter, - server_name=hs.hostname, - ).register(resource) + if "federation" in servlet_groups: + for servletclass in FEDERATION_SERVLET_CLASSES: + servletclass( + handler=hs.get_federation_server(), + authenticator=authenticator, + ratelimiter=ratelimiter, + server_name=hs.hostname, + ).register(resource) - for servletclass in GROUP_LOCAL_SERVLET_CLASSES: - servletclass( - handler=hs.get_groups_local_handler(), - authenticator=authenticator, - ratelimiter=ratelimiter, - server_name=hs.hostname, - ).register(resource) + if "openid" in servlet_groups: + for servletclass in OPENID_SERVLET_CLASSES: + servletclass( + handler=hs.get_federation_server(), + authenticator=authenticator, + ratelimiter=ratelimiter, + server_name=hs.hostname, + ).register(resource) - for servletclass in GROUP_ATTESTATION_SERVLET_CLASSES: - servletclass( - handler=hs.get_groups_attestation_renewer(), - authenticator=authenticator, - ratelimiter=ratelimiter, - server_name=hs.hostname, - ).register(resource) + if "room_list" in servlet_groups: + for servletclass in ROOM_LIST_CLASSES: + servletclass( + handler=hs.get_room_list_handler(), + authenticator=authenticator, + ratelimiter=ratelimiter, + server_name=hs.hostname, + ).register(resource) + + if "group_server" in servlet_groups: + for servletclass in GROUP_SERVER_SERVLET_CLASSES: + servletclass( + handler=hs.get_groups_server_handler(), + authenticator=authenticator, + ratelimiter=ratelimiter, + server_name=hs.hostname, + ).register(resource) + + if "group_local" in servlet_groups: + for servletclass in GROUP_LOCAL_SERVLET_CLASSES: + servletclass( + handler=hs.get_groups_local_handler(), + authenticator=authenticator, + ratelimiter=ratelimiter, + server_name=hs.hostname, + ).register(resource) + + if "group_attestation" in servlet_groups: + for servletclass in GROUP_ATTESTATION_SERVLET_CLASSES: + servletclass( + handler=hs.get_groups_attestation_renewer(), + authenticator=authenticator, + ratelimiter=ratelimiter, + server_name=hs.hostname, + ).register(resource) From 1d2c69fee897cf052cfa03f0cc6f9f419c898bb1 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Mon, 21 Jan 2019 01:59:18 +0200 Subject: [PATCH 08/66] Add changelog for openid resource addition Signed-off-by: Jason Robinson --- changelog.d/4420.feature | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 changelog.d/4420.feature diff --git a/changelog.d/4420.feature b/changelog.d/4420.feature new file mode 100644 index 0000000000..5e684d01e0 --- /dev/null +++ b/changelog.d/4420.feature @@ -0,0 +1,13 @@ +New listener resource for the federation API "openid/userinfo" endpoint + +Integration managers use the OpenID userinfo endpoint in the federation API to verify that user +OpenID access tokens are valid. If the federation resource is disabled, integration managers will not be able +to verify the access token, causing a broken experience for users. The OpenID userinfo endpoint has now been split +to a separate `openid` resource, which is enabled by default in newly generated configuration. It is also enabled +automatically if the federation resource is enabled. + +If your homeserver runs federation enabled, this change does not require any actions. + +If you run a homeserver with federation disabled, we recommend adding the `openid` resource to your homeserver +configuration in the `type: http` listener `resources` list to allow your users access to +integration manager features. From d39b7b6d38b0d9876ad305491ea05af92ea35b04 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Tue, 22 Jan 2019 11:00:17 +0200 Subject: [PATCH 09/66] Document `servlet_groups` parameters Signed-off-by: Jason Robinson --- synapse/federation/transport/server.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py index 49b80beecb..9c1c9c39e1 100644 --- a/synapse/federation/transport/server.py +++ b/synapse/federation/transport/server.py @@ -44,6 +44,16 @@ class TransportLayerServer(JsonResource): """Handles incoming federation HTTP requests""" def __init__(self, hs, servlet_groups=None): + """Initialize the TransportLayerServer + + Will by default register all servlets. For custom behaviour, pass in + a list of servlet_groups to register. + + Args: + hs (synapse.server.HomeServer): homeserver + servlet_groups (list[str], optional): List of servlet groups to register. + Defaults to ``DEFAULT_SERVLET_GROUPS``. + """ self.hs = hs self.clock = hs.get_clock() self.servlet_groups = servlet_groups @@ -1365,6 +1375,19 @@ DEFAULT_SERVLET_GROUPS = ( def register_servlets(hs, resource, authenticator, ratelimiter, servlet_groups=None): + """Initialize and register servlet classes. + + Will by default register all servlets. For custom behaviour, pass in + a list of servlet_groups to register. + + Args: + hs (synapse.server.HomeServer): homeserver + resource (TransportLayerServer): resource class to register to + authenticator (Authenticator): authenticator to use + ratelimiter (util.ratelimitutils.FederationRateLimiter): ratelimiter to use + servlet_groups (list[str], optional): List of servlet groups to register. + Defaults to ``DEFAULT_SERVLET_GROUPS``. + """ if not servlet_groups: servlet_groups = DEFAULT_SERVLET_GROUPS From 0516dc4d85f1018beb241df6833117b8b49d08d3 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Tue, 22 Jan 2019 11:04:10 +0200 Subject: [PATCH 10/66] Remove openid resource from default config Instead document it commented out. Signed-off-by: Jason Robinson --- synapse/config/server.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/synapse/config/server.py b/synapse/config/server.py index 556f1efee5..eebbfccafe 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -151,7 +151,7 @@ class ServerConfig(Config): "compress": gzip_responses, }, { - "names": ["federation", "openid"], + "names": ["federation"], "compress": False, } ] @@ -170,7 +170,7 @@ class ServerConfig(Config): "compress": gzip_responses, }, { - "names": ["federation", "openid"], + "names": ["federation"], "compress": False, } ] @@ -328,8 +328,13 @@ class ServerConfig(Config): # that can do automatic compression. compress: true - - names: [federation, openid] # Federation APIs + - names: [federation] # Federation APIs compress: false + + # # If federation is disabled synapse can still expose the open ID endpoint + # # to allow integrations to authenticate users + # - names: [openid] + # compress: false # optional list of additional endpoints which can be loaded via # dynamic modules @@ -350,8 +355,12 @@ class ServerConfig(Config): resources: - names: [client] compress: true - - names: [federation, openid] + - names: [federation] compress: false + # # If federation is disabled synapse can still expose the open ID endpoint + # # to allow integrations to authenticate users + # - names: [openid] + # compress: false # Turn on the twisted ssh manhole service on localhost on the given # port. From db33634b1dc47167cffce31ff4ae44c5d3fae2af Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Tue, 22 Jan 2019 11:05:22 +0200 Subject: [PATCH 11/66] Collapse changelog to one line Signed-off-by: Jason Robinson --- changelog.d/4420.feature | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/changelog.d/4420.feature b/changelog.d/4420.feature index 5e684d01e0..05e777c624 100644 --- a/changelog.d/4420.feature +++ b/changelog.d/4420.feature @@ -1,13 +1 @@ -New listener resource for the federation API "openid/userinfo" endpoint - -Integration managers use the OpenID userinfo endpoint in the federation API to verify that user -OpenID access tokens are valid. If the federation resource is disabled, integration managers will not be able -to verify the access token, causing a broken experience for users. The OpenID userinfo endpoint has now been split -to a separate `openid` resource, which is enabled by default in newly generated configuration. It is also enabled -automatically if the federation resource is enabled. - -If your homeserver runs federation enabled, this change does not require any actions. - -If you run a homeserver with federation disabled, we recommend adding the `openid` resource to your homeserver -configuration in the `type: http` listener `resources` list to allow your users access to -integration manager features. +Federation OpenID listener resource can now be activated even if federation is disabled From a47fac9af6b17ae60c381675caece6e846faefff Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Tue, 22 Jan 2019 11:07:28 +0200 Subject: [PATCH 12/66] Fix sorting of imports in tests. Remove an unnecessary mock Signed-off-by: Jason Robinson --- tests/app/test_openid_listener.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/app/test_openid_listener.py b/tests/app/test_openid_listener.py index 329aadc3d3..c1ebc93f52 100644 --- a/tests/app/test_openid_listener.py +++ b/tests/app/test_openid_listener.py @@ -12,7 +12,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from mock import patch, Mock +from mock import Mock, patch + from parameterized import parameterized from synapse.app.federation_reader import FederationReaderServer @@ -21,7 +22,6 @@ from synapse.app.homeserver import SynapseHomeServer from tests.unittest import HomeserverTestCase -@patch("synapse.app.homeserver.KeyApiV2Resource", new=Mock()) class FederationReaderOpenIDListenerTests(HomeserverTestCase): def make_homeserver(self, reactor, clock): hs = self.setup_test_homeserver( From 1838ef1ac39d624e21786120b402d04a0c4485ba Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Wed, 23 Jan 2019 10:38:13 +0200 Subject: [PATCH 13/66] Fix openid tests after rebase Signed-off-by: Jason Robinson --- tests/app/test_openid_listener.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/app/test_openid_listener.py b/tests/app/test_openid_listener.py index c1ebc93f52..46a3b61a3c 100644 --- a/tests/app/test_openid_listener.py +++ b/tests/app/test_openid_listener.py @@ -54,7 +54,7 @@ class FederationReaderOpenIDListenerTests(HomeserverTestCase): site = self.reactor.tcpServers[0][1] try: self.resource = ( - site.resource.children[b"_matrix"].children[b"federation"].children[b"v1"] + site.resource.children[b"_matrix"].children[b"federation"] ) except KeyError: if expectation == "no_resource": @@ -100,7 +100,7 @@ class SynapseHomeserverOpenIDListenerTests(HomeserverTestCase): site = self.reactor.tcpServers[0][1] try: self.resource = ( - site.resource.children[b"_matrix"].children[b"federation"].children[b"v1"] + site.resource.children[b"_matrix"].children[b"federation"] ) except KeyError: if expectation == "no_resource": From 6f680241bd7f53c3a3f912c257d2bfa437dfd486 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Wed, 23 Jan 2019 10:53:48 +0200 Subject: [PATCH 14/66] Fix flake8 issues Signed-off-by: Jason Robinson --- synapse/app/federation_reader.py | 5 ++++- synapse/config/server.py | 2 +- tests/app/test_openid_listener.py | 10 ++++++++-- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/synapse/app/federation_reader.py b/synapse/app/federation_reader.py index 99e8a4cf6a..2c99ce8c64 100644 --- a/synapse/app/federation_reader.py +++ b/synapse/app/federation_reader.py @@ -92,7 +92,10 @@ class FederationReaderServer(HomeServer): # is not specified since federation resource includes openid # resource. resources.update({ - FEDERATION_PREFIX: TransportLayerServer(self, servlet_groups=["openid"]), + FEDERATION_PREFIX: TransportLayerServer( + self, + servlet_groups=["openid"], + ), }) root_resource = create_resource_tree(resources, NoResource()) diff --git a/synapse/config/server.py b/synapse/config/server.py index eebbfccafe..4eefd06f4a 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -330,7 +330,7 @@ class ServerConfig(Config): - names: [federation] # Federation APIs compress: false - + # # If federation is disabled synapse can still expose the open ID endpoint # # to allow integrations to authenticate users # - names: [openid] diff --git a/tests/app/test_openid_listener.py b/tests/app/test_openid_listener.py index 46a3b61a3c..590abc1e92 100644 --- a/tests/app/test_openid_listener.py +++ b/tests/app/test_openid_listener.py @@ -61,7 +61,10 @@ class FederationReaderOpenIDListenerTests(HomeserverTestCase): return raise - request, channel = self.make_request("GET", "/_matrix/federation/v1/openid/userinfo") + request, channel = self.make_request( + "GET", + "/_matrix/federation/v1/openid/userinfo", + ) self.render(request) self.assertEqual(channel.code, 401) @@ -107,7 +110,10 @@ class SynapseHomeserverOpenIDListenerTests(HomeserverTestCase): return raise - request, channel = self.make_request("GET", "/_matrix/federation/v1/openid/userinfo") + request, channel = self.make_request( + "GET", + "/_matrix/federation/v1/openid/userinfo", + ) self.render(request) self.assertEqual(channel.code, 401) From ad7ac8853cab27cb7f5aedcde4f1aaae82f8d5c9 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 30 Jan 2019 16:26:13 +0000 Subject: [PATCH 15/66] by default include m.room.encryption on invites (#3902) * by default include m.room.encryption on invites * fix constant * changelog --- changelog.d/3902.feature | 1 + synapse/api/constants.py | 1 + synapse/config/api.py | 2 ++ 3 files changed, 4 insertions(+) create mode 100644 changelog.d/3902.feature diff --git a/changelog.d/3902.feature b/changelog.d/3902.feature new file mode 100644 index 0000000000..eb8d9f2393 --- /dev/null +++ b/changelog.d/3902.feature @@ -0,0 +1 @@ +Include m.room.encryption on invites by default diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 0cbae9429b..39ff4f62eb 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -73,6 +73,7 @@ class EventTypes(object): RoomHistoryVisibility = "m.room.history_visibility" CanonicalAlias = "m.room.canonical_alias" RoomAvatar = "m.room.avatar" + RoomEncryption = "m.room.encryption" GuestAccess = "m.room.guest_access" # These are used for validation diff --git a/synapse/config/api.py b/synapse/config/api.py index 403d96ba76..9f25bbc5cb 100644 --- a/synapse/config/api.py +++ b/synapse/config/api.py @@ -24,6 +24,7 @@ class ApiConfig(Config): EventTypes.JoinRules, EventTypes.CanonicalAlias, EventTypes.RoomAvatar, + EventTypes.RoomEncryption, EventTypes.Name, ]) @@ -36,5 +37,6 @@ class ApiConfig(Config): - "{JoinRules}" - "{CanonicalAlias}" - "{RoomAvatar}" + - "{RoomEncryption}" - "{Name}" """.format(**vars(EventTypes)) From d621c5562ea6bce4fd8282da642b3123ae016d94 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Wed, 30 Jan 2019 16:33:51 +0000 Subject: [PATCH 16/66] Copy over non-federatable trait on room upgrade --- synapse/handlers/room.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 13ba9291b0..c04ba3a0c5 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -263,6 +263,20 @@ class RoomCreationHandler(BaseHandler): } } + # Check if old room was non-federatable + + # Get old room's create event + old_room_create_event_ids = yield self.store.get_filtered_current_state_ids( + old_room_id, StateFilter.from_types(((EventTypes.Create, ""),)), + ) + old_room_create_event_dict = yield self.store.get_events(old_room_create_event_ids.values()) + old_room_create_event = list(old_room_create_event_dict.values())[0] + + # Check if the create event specified a non-federatable room + if old_room_create_event.content.get("m.federate", True) == False: + # If so, mark the new room as non-federatable as well + creation_content["m.federate"] = False + initial_state = dict() # Replicate relevant room events From cf9a2676d026c7864de44fe3440475a9cf1dc31d Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Wed, 30 Jan 2019 19:04:48 +0000 Subject: [PATCH 17/66] Add changelog --- changelog.d/4530.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/4530.bugfix diff --git a/changelog.d/4530.bugfix b/changelog.d/4530.bugfix new file mode 100644 index 0000000000..d010af927e --- /dev/null +++ b/changelog.d/4530.bugfix @@ -0,0 +1 @@ +Copy over room federation ability on room upgrade. \ No newline at end of file From fb50934b8ff605a70b2a325eaaf51148a9651de5 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 31 Jan 2019 11:34:45 +0000 Subject: [PATCH 18/66] lint --- synapse/handlers/room.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index c04ba3a0c5..a69441b96f 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -269,11 +269,13 @@ class RoomCreationHandler(BaseHandler): old_room_create_event_ids = yield self.store.get_filtered_current_state_ids( old_room_id, StateFilter.from_types(((EventTypes.Create, ""),)), ) - old_room_create_event_dict = yield self.store.get_events(old_room_create_event_ids.values()) + old_room_create_event_dict = yield self.store.get_events( + old_room_create_event_ids.values(), + ) old_room_create_event = list(old_room_create_event_dict.values())[0] # Check if the create event specified a non-federatable room - if old_room_create_event.content.get("m.federate", True) == False: + if not old_room_create_event.content.get("m.federate", True): # If so, mark the new room as non-federatable as well creation_content["m.federate"] = False From 563f6a832b379e2cde6b5618a7c344c2bcd793a1 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Thu, 31 Jan 2019 11:44:04 +0000 Subject: [PATCH 19/66] Reject large transactions on federation (#4513) * Reject large transactions on federation * Add changelog * lint * Simplify large transaction handling --- changelog.d/4513.misc | 1 + synapse/federation/federation_server.py | 16 ++++++++++++++++ 2 files changed, 17 insertions(+) create mode 100644 changelog.d/4513.misc diff --git a/changelog.d/4513.misc b/changelog.d/4513.misc new file mode 100644 index 0000000000..1f64a96465 --- /dev/null +++ b/changelog.d/4513.misc @@ -0,0 +1 @@ +Reject federation transactions if they include more than 50 PDUs or 100 EDUs. \ No newline at end of file diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index aeadc9c564..3da86d4ba6 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -148,6 +148,22 @@ class FederationServer(FederationBase): logger.debug("[%s] Transaction is new", transaction.transaction_id) + # Reject if PDU count > 50 and EDU count > 100 + if (len(transaction.pdus) > 50 + or (hasattr(transaction, "edus") and len(transaction.edus) > 100)): + + logger.info( + "Transaction PDU or EDU count too large. Returning 400", + ) + + response = {} + yield self.transaction_actions.set_response( + origin, + transaction, + 400, response + ) + defer.returnValue((400, response)) + received_pdus_counter.inc(len(transaction.pdus)) origin_host, _ = parse_server_name(origin) From 3ed3cb43394b41e76f4739f22760c1d8ebfed3c7 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 31 Jan 2019 18:21:39 +0000 Subject: [PATCH 20/66] New function for getting room's create event --- synapse/handlers/room.py | 8 +------- synapse/storage/state.py | 31 ++++++++++++++++++++----------- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index a69441b96f..5e40e9ea46 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -266,13 +266,7 @@ class RoomCreationHandler(BaseHandler): # Check if old room was non-federatable # Get old room's create event - old_room_create_event_ids = yield self.store.get_filtered_current_state_ids( - old_room_id, StateFilter.from_types(((EventTypes.Create, ""),)), - ) - old_room_create_event_dict = yield self.store.get_events( - old_room_create_event_ids.values(), - ) - old_room_create_event = list(old_room_create_event_dict.values())[0] + old_room_create_event = yield self.store.get_create_event_for_room(old_room_id) # Check if the create event specified a non-federatable room if not old_room_create_event.content.get("m.federate", True): diff --git a/synapse/storage/state.py b/synapse/storage/state.py index c3ab7db7ae..522aaee918 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -24,7 +24,6 @@ import attr from twisted.internet import defer from synapse.api.constants import EventTypes -from synapse.api.errors import NotFoundError from synapse.storage._base import SQLBaseStore from synapse.storage.background_updates import BackgroundUpdateStore from synapse.storage.engines import PostgresEngine @@ -428,13 +427,9 @@ class StateGroupWorkerStore(EventsWorkerStore, SQLBaseStore): """ # for now we do this by looking at the create event. We may want to cache this # more intelligently in future. - state_ids = yield self.get_current_state_ids(room_id) - create_id = state_ids.get((EventTypes.Create, "")) - if not create_id: - raise NotFoundError("Unknown room %s" % (room_id)) - - create_event = yield self.get_event(create_id) + # Retrieve the room's create event + create_event = yield self.get_create_event_for_room(room_id) defer.returnValue(create_event.content.get("room_version", "1")) @defer.inlineCallbacks @@ -448,6 +443,22 @@ class StateGroupWorkerStore(EventsWorkerStore, SQLBaseStore): Returns: Deferred[unicode|None]: predecessor room id """ + # Retrieve the room's create event + create_event = yield self.get_create_event_for_room(room_id) + + # Return predecessor if present + defer.returnValue(create_event.content.get("predecessor", None)) + + @defer.inlineCallbacks + def get_create_event_for_room(self, room_id): + """Get the create state event for a room. + + Args: + room_id (str) + + Returns: + Deferred[EventBase|None]: The room creation event. None if can not be found + """ state_ids = yield self.get_current_state_ids(room_id) create_id = state_ids.get((EventTypes.Create, "")) @@ -455,11 +466,9 @@ class StateGroupWorkerStore(EventsWorkerStore, SQLBaseStore): if not create_id: defer.returnValue(None) - # Retrieve the room's create event + # Retrieve the room's create event and return create_event = yield self.get_event(create_id) - - # Return predecessor if present - defer.returnValue(create_event.content.get("predecessor", None)) + defer.returnValue(create_event) @cached(max_entries=100000, iterable=True) def get_current_state_ids(self, room_id): From d239f67c25a31257a11852be08e6615e99423fe7 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 31 Jan 2019 18:34:15 +0000 Subject: [PATCH 21/66] Raise an exception instead of returning None --- synapse/storage/state.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/synapse/storage/state.py b/synapse/storage/state.py index 522aaee918..d14a7b2538 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -24,6 +24,7 @@ import attr from twisted.internet import defer from synapse.api.constants import EventTypes +from synapse.api.errors import NotFoundError from synapse.storage._base import SQLBaseStore from synapse.storage.background_updates import BackgroundUpdateStore from synapse.storage.engines import PostgresEngine @@ -442,6 +443,9 @@ class StateGroupWorkerStore(EventsWorkerStore, SQLBaseStore): Returns: Deferred[unicode|None]: predecessor room id + + Raises: + NotFoundError if the room is unknown """ # Retrieve the room's create event create_event = yield self.get_create_event_for_room(room_id) @@ -457,14 +461,17 @@ class StateGroupWorkerStore(EventsWorkerStore, SQLBaseStore): room_id (str) Returns: - Deferred[EventBase|None]: The room creation event. None if can not be found + Deferred[EventBase]: The room creation event. + + Raises: + NotFoundError if the room is unknown """ state_ids = yield self.get_current_state_ids(room_id) create_id = state_ids.get((EventTypes.Create, "")) # If we can't find the create event, assume we've hit a dead end if not create_id: - defer.returnValue(None) + raise NotFoundError("Unknown room %s" % (room_id)) # Retrieve the room's create event and return create_event = yield self.get_event(create_id) From f0ba34f581906e3809de6313bc48b696ac4ef07c Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 1 Feb 2019 12:22:57 +0000 Subject: [PATCH 22/66] Fix noisy "twisted.internet.task.TaskStopped" errors in logs Fixes #4003 --- changelog.d/4546.bugfix | 1 + synapse/http/matrixfederationclient.py | 17 +++++++++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) create mode 100644 changelog.d/4546.bugfix diff --git a/changelog.d/4546.bugfix b/changelog.d/4546.bugfix new file mode 100644 index 0000000000..056f2848ed --- /dev/null +++ b/changelog.d/4546.bugfix @@ -0,0 +1 @@ +Fix noisy "twisted.internet.task.TaskStopped" errors in logs diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index bb2e64ed80..5ee4d528d2 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -28,7 +28,7 @@ from canonicaljson import encode_canonical_json from prometheus_client import Counter from signedjson.sign import sign_json -from twisted.internet import defer, protocol +from twisted.internet import defer, protocol, task from twisted.internet.error import DNSLookupError from twisted.internet.task import _EPSILON, Cooperator from twisted.web._newclient import ResponseDone @@ -286,7 +286,7 @@ class MatrixFederationHttpClient(object): json, ) data = encode_canonical_json(json) - producer = FileBodyProducer( + producer = QuieterFileBodyProducer( BytesIO(data), cooperator=self._cooperator, ) @@ -839,3 +839,16 @@ def encode_query_args(args): query_bytes = urllib.parse.urlencode(encoded_args, True) return query_bytes.encode('utf8') + + +class QuieterFileBodyProducer(FileBodyProducer): + """Wrapper for FileBodyProducer that avoids CRITICAL errors when the connection drops. + + Workaround for https://github.com/matrix-org/synapse/issues/4003 / + https://twistedmatrix.com/trac/ticket/6528 + """ + def stopProducing(self): + try: + FileBodyProducer.stopProducing(self) + except task.TaskStopped: + pass From 627ecd358eda66f81afd676523f520762571728c Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Tue, 5 Feb 2019 12:16:28 +0000 Subject: [PATCH 23/66] Filter user directory state query to a subset of state events (#4462) * Filter user directory state query to a subset of state events * Add changelog --- changelog.d/4462.misc | 1 + synapse/storage/user_directory.py | 16 ++++++++++++---- 2 files changed, 13 insertions(+), 4 deletions(-) create mode 100644 changelog.d/4462.misc diff --git a/changelog.d/4462.misc b/changelog.d/4462.misc new file mode 100644 index 0000000000..03a4d7ae1c --- /dev/null +++ b/changelog.d/4462.misc @@ -0,0 +1 @@ +Change the user directory state query to use a filtered call to the db instead of a generic one. \ No newline at end of file diff --git a/synapse/storage/user_directory.py b/synapse/storage/user_directory.py index ce48212265..e8b574ee5e 100644 --- a/synapse/storage/user_directory.py +++ b/synapse/storage/user_directory.py @@ -22,6 +22,7 @@ from twisted.internet import defer from synapse.api.constants import EventTypes, JoinRules from synapse.storage.engines import PostgresEngine, Sqlite3Engine +from synapse.storage.state import StateFilter from synapse.types import get_domain_from_id, get_localpart_from_id from synapse.util.caches.descriptors import cached, cachedInlineCallbacks @@ -31,12 +32,19 @@ logger = logging.getLogger(__name__) class UserDirectoryStore(SQLBaseStore): - @cachedInlineCallbacks(cache_context=True) - def is_room_world_readable_or_publicly_joinable(self, room_id, cache_context): + @defer.inlineCallbacks + def is_room_world_readable_or_publicly_joinable(self, room_id): """Check if the room is either world_readable or publically joinable """ - current_state_ids = yield self.get_current_state_ids( - room_id, on_invalidate=cache_context.invalidate + + # Create a state filter that only queries join and history state event + types_to_filter = ( + (EventTypes.JoinRules, ""), + (EventTypes.RoomHistoryVisibility, ""), + ) + + current_state_ids = yield self.get_filtered_current_state_ids( + room_id, StateFilter.from_types(types_to_filter), ) join_rules_id = current_state_ids.get((EventTypes.JoinRules, "")) From 664c81e8b7525bfaa5c3a7620f2f831ade0754a2 Mon Sep 17 00:00:00 2001 From: Hubert Chathi Date: Wed, 6 Feb 2019 17:47:22 -0500 Subject: [PATCH 24/66] return proper error codes for some 404s --- synapse/handlers/e2e_room_keys.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py index 42b040375f..c5d7bf0c29 100644 --- a/synapse/handlers/e2e_room_keys.py +++ b/synapse/handlers/e2e_room_keys.py @@ -267,7 +267,7 @@ class E2eRoomKeysHandler(object): version(str): Optional; if None gives the most recent version otherwise a historical one. Raises: - StoreError: code 404 if the requested backup version doesn't exist + NotFoundError: if the requested backup version doesn't exist Returns: A deferred of a info dict that gives the info about the new version. @@ -279,7 +279,13 @@ class E2eRoomKeysHandler(object): """ with (yield self._upload_linearizer.queue(user_id)): - res = yield self.store.get_e2e_room_keys_version_info(user_id, version) + try: + res = yield self.store.get_e2e_room_keys_version_info(user_id, version) + except StoreError as e: + if e.code == 404: + raise NotFoundError("Unknown backup version") + else: + raise defer.returnValue(res) @defer.inlineCallbacks @@ -290,8 +296,14 @@ class E2eRoomKeysHandler(object): user_id(str): the user whose current backup version we're deleting version(str): the version id of the backup being deleted Raises: - StoreError: code 404 if this backup version doesn't exist + NotFoundError: if this backup version doesn't exist """ with (yield self._upload_linearizer.queue(user_id)): - yield self.store.delete_e2e_room_keys_version(user_id, version) + try: + yield self.store.delete_e2e_room_keys_version(user_id, version) + except StoreError as e: + if e.code == 404: + raise NotFoundError("Unknown backup version") + else: + raise From 82486371738a130322f3a1829bc4b49f79c1a3e4 Mon Sep 17 00:00:00 2001 From: Hubert Chathi Date: Wed, 6 Feb 2019 17:57:10 -0500 Subject: [PATCH 25/66] add new endpoint to update backup versions --- synapse/handlers/e2e_room_keys.py | 34 ++++++++++++++++++++++- synapse/rest/client/v2_alpha/room_keys.py | 33 ++++++++++++++++++++++ synapse/storage/e2e_room_keys.py | 21 ++++++++++++++ 3 files changed, 87 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py index c5d7bf0c29..e0e5ece747 100644 --- a/synapse/handlers/e2e_room_keys.py +++ b/synapse/handlers/e2e_room_keys.py @@ -19,7 +19,8 @@ from six import iteritems from twisted.internet import defer -from synapse.api.errors import NotFoundError, RoomKeysVersionError, StoreError +from synapse.api.errors import Codes, NotFoundError, RoomKeysVersionError, \ + StoreError, SynapseError from synapse.util.async_helpers import Linearizer logger = logging.getLogger(__name__) @@ -307,3 +308,34 @@ class E2eRoomKeysHandler(object): raise NotFoundError("Unknown backup version") else: raise + + @defer.inlineCallbacks + def update_version(self, user_id, version, version_info): + """Update the info about a given version of the user's backup + + Args: + user_id(str): the user whose current backup version we're updating + version(str): the backup version we're updating + version_info(dict): the new information about the backup + Raises: + NotFoundError: if the requested backup version doesn't exist + Returns: + A deferred of an empty dict. + """ + try: + old_info = yield self.store.get_e2e_room_keys_version_info(user_id, version) + except StoreError as e: + if e.code == 404: + raise NotFoundError("Unknown backup version") + else: + raise + if old_info["algorithm"] != version_info["algorithm"]: + raise SynapseError( + 400, + "Algorithm does not match", + Codes.INVALID_PARAM + ) + + yield self.store.update_e2e_room_keys_version(user_id, version, version_info) + + defer.returnValue({}) diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py index ab3f1bd21a..1c39d2af1c 100644 --- a/synapse/rest/client/v2_alpha/room_keys.py +++ b/synapse/rest/client/v2_alpha/room_keys.py @@ -380,6 +380,39 @@ class RoomKeysVersionServlet(RestServlet): ) defer.returnValue((200, {})) + @defer.inlineCallbacks + def on_PUT(self, request, version): + """ + Update the information about a given version of the user's room_keys backup. + + POST /room_keys/version/12345 HTTP/1.1 + Content-Type: application/json + { + "algorithm": "m.megolm_backup.v1", + "auth_data": { + "public_key": "abcdefg", + "signatures": { + "ed25519:something": "hijklmnop" + } + } + } + + HTTP/1.1 200 OK + Content-Type: application/json + {} + """ + requester = yield self.auth.get_user_by_req(request, allow_guest=False) + user_id = requester.user.to_string() + info = parse_json_object_from_request(request) + + if version is None: + raise SynapseError(400, "No version specified to update", Codes.MISSING_PARAM) + + yield self.e2e_room_keys_handler.update_version( + user_id, version, info + ) + defer.returnValue((200, {})) + def register_servlets(hs, http_server): RoomKeysServlet(hs).register(http_server) diff --git a/synapse/storage/e2e_room_keys.py b/synapse/storage/e2e_room_keys.py index 45cebe61d1..9a3aec759e 100644 --- a/synapse/storage/e2e_room_keys.py +++ b/synapse/storage/e2e_room_keys.py @@ -298,6 +298,27 @@ class EndToEndRoomKeyStore(SQLBaseStore): "create_e2e_room_keys_version_txn", _create_e2e_room_keys_version_txn ) + def update_e2e_room_keys_version(self, user_id, version, info): + """Update a given backup version + + Args: + user_id(str): the user whose backup version we're updating + version(str): the version ID of the backup version we're updating + info(dict): the new backup version info to store + """ + + return self._simple_update( + table="e2e_room_keys_versions", + keyvalues={ + "user_id": user_id, + "version": version, + }, + updatevalues={ + "auth_data": json.dumps(info["auth_data"]), + }, + desc="update_e2e_room_keys_version" + ) + def delete_e2e_room_keys_version(self, user_id, version=None): """Delete a given backup version of the user's room keys. Doesn't delete their actual key data. From 9ff620a518739bfe7417bac20365e4acf9c5906e Mon Sep 17 00:00:00 2001 From: Hubert Chathi Date: Wed, 6 Feb 2019 21:32:52 -0500 Subject: [PATCH 26/66] fix import to make isort happy --- synapse/handlers/e2e_room_keys.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py index e0e5ece747..6c43c9db76 100644 --- a/synapse/handlers/e2e_room_keys.py +++ b/synapse/handlers/e2e_room_keys.py @@ -19,8 +19,13 @@ from six import iteritems from twisted.internet import defer -from synapse.api.errors import Codes, NotFoundError, RoomKeysVersionError, \ - StoreError, SynapseError +from synapse.api.errors import ( + Codes, + NotFoundError, + RoomKeysVersionError, + StoreError, + SynapseError +) from synapse.util.async_helpers import Linearizer logger = logging.getLogger(__name__) From 51b73be63bd6931018f98ea00f6b723c6196219a Mon Sep 17 00:00:00 2001 From: Hubert Chathi Date: Wed, 6 Feb 2019 21:39:56 -0500 Subject: [PATCH 27/66] add changelog entry --- changelog.d/4580.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/4580.feature diff --git a/changelog.d/4580.feature b/changelog.d/4580.feature new file mode 100644 index 0000000000..a2a5a77dbe --- /dev/null +++ b/changelog.d/4580.feature @@ -0,0 +1 @@ +Add ability to update backup versions \ No newline at end of file From d9e424bf649c1d4ba7b9da7fd64db84cda389e11 Mon Sep 17 00:00:00 2001 From: Hubert Chathi Date: Wed, 6 Feb 2019 22:18:41 -0500 Subject: [PATCH 28/66] re-try to make isort happy --- synapse/handlers/e2e_room_keys.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py index 6c43c9db76..546050f8e7 100644 --- a/synapse/handlers/e2e_room_keys.py +++ b/synapse/handlers/e2e_room_keys.py @@ -24,7 +24,7 @@ from synapse.api.errors import ( NotFoundError, RoomKeysVersionError, StoreError, - SynapseError + SynapseError, ) from synapse.util.async_helpers import Linearizer From acb2ac5863d34e2d01fa53598aecda274dc2fb26 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 7 Feb 2019 19:30:32 +0000 Subject: [PATCH 29/66] Update MSC1711 FAQ to be explicit about well-known (#4584) A surprising number of people are using the well-known method, and are simply copying the example configuration. This is problematic as the example includes an explicit port, which causes inbound federation requests to have the HTTP Host header include the port, upsetting some reverse proxies. Given that, we update the well-known example to be more explicit about the various ways you can set it up, and the consequence of using an explict port. --- changelog.d/4584.misc | 1 + docs/MSC1711_certificates_FAQ.md | 40 +++++++++++++++++++++----------- 2 files changed, 28 insertions(+), 13 deletions(-) create mode 100644 changelog.d/4584.misc diff --git a/changelog.d/4584.misc b/changelog.d/4584.misc new file mode 100644 index 0000000000..4dec2e2b5c --- /dev/null +++ b/changelog.d/4584.misc @@ -0,0 +1 @@ +Update MSC1711 FAQ to calrify .well-known usage diff --git a/docs/MSC1711_certificates_FAQ.md b/docs/MSC1711_certificates_FAQ.md index 414af96ef3..0a781d00e3 100644 --- a/docs/MSC1711_certificates_FAQ.md +++ b/docs/MSC1711_certificates_FAQ.md @@ -107,12 +107,12 @@ hosted at a target domain of `customer.example.net`. Currently you should have an SRV record which looks like: ``` -_matrix._tcp.example.com. IN SRV 10 5 443 customer.example.net. +_matrix._tcp.example.com. IN SRV 10 5 8000 customer.example.net. ``` -In this situation, you have two choices for how to proceed: +In this situation, you have three choices for how to proceed: -#### Option 1: give Synapse (or a reverse-proxy) a certificate for your matrix domain +#### Option 1: give Synapse a certificate for your matrix domain Synapse 1.0 will expect your server to present a TLS certificate for your `server_name` (`example.com` in the above example). You can achieve this by @@ -123,12 +123,16 @@ doing one of the following: and `tls_private_key_path`, or: * Use Synapse's [ACME support](./ACME.md), and forward port 80 on the - `server_name` domain to your Synapse instance, or: + `server_name` domain to your Synapse instance. - * Set up a reverse-proxy on port 8448 on the `server_name` domain, which - forwards to Synapse. Once it is set up, you can remove the SRV record. +### Option 2: run Synapse behind a reverse proxy -#### Option 2: add a .well-known file to delegate your matrix traffic +If you have an existing reverse proxy set up with correct TLS certificates for +your domain, you can simply route all traffic through the reverse proxy by +updating the SRV record appropriately (or removing it, if the proxy listens on +8448). + +#### Option 3: add a .well-known file to delegate your matrix traffic This will allow you to keep Synapse on a separate domain, without having to give it a certificate for the matrix domain. @@ -151,15 +155,25 @@ You can do this with a `.well-known` file as follows: `https:///.well-known/matrix/server` with contents: ```json - {"m.server": ":"} + {"m.server": ""} ``` - In the above example, `https://example.com/.well-known/matrix/server` - should have the contents: + where the target server name is resolved as usual (i.e. SRV lookup, falling + back to talking to port 8448). + + In the above example, where synapse is listening on port 8000, + `https://example.com/.well-known/matrix/server` should have `m.server` set to one of: + + 1. `customer.example.net` ─ with a SRV record on + `_matrix._tcp.customer.example.com` pointing to port 8000, or: + + 2. `customer.example.net` ─ updating synapse to listen on the default port + 8448, or: + + 3. `customer.example.net:8000` ─ ensuring that if there is a reverse proxy + on `customer.example.net:8000` it correctly handles HTTP requests with + Host header set to `customer.example.net:8000`. - ```json - {"m.server": "customer.example.net:443"} - ``` ## FAQ From afae8442b56d2e2466812916654396341038c38c Mon Sep 17 00:00:00 2001 From: Hubert Chathi Date: Fri, 8 Feb 2019 01:32:45 -0500 Subject: [PATCH 30/66] make sure version is in body and wrap in linearizer queue also add tests --- synapse/handlers/e2e_room_keys.py | 37 ++++++++---- synapse/rest/client/v2_alpha/room_keys.py | 3 +- tests/handlers/test_e2e_room_keys.py | 72 +++++++++++++++++++++++ 3 files changed, 100 insertions(+), 12 deletions(-) diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py index 546050f8e7..7bc174070e 100644 --- a/synapse/handlers/e2e_room_keys.py +++ b/synapse/handlers/e2e_room_keys.py @@ -327,20 +327,35 @@ class E2eRoomKeysHandler(object): Returns: A deferred of an empty dict. """ - try: - old_info = yield self.store.get_e2e_room_keys_version_info(user_id, version) - except StoreError as e: - if e.code == 404: - raise NotFoundError("Unknown backup version") - else: - raise - if old_info["algorithm"] != version_info["algorithm"]: + if "version" not in version_info: raise SynapseError( 400, - "Algorithm does not match", + "Missing version in body", + Codes.MISSING_PARAM + ) + if version_info["version"] != version: + raise SynapseError( + 400, + "Version in body does not match", Codes.INVALID_PARAM ) + with (yield self._upload_linearizer.queue(user_id)): + try: + old_info = yield self.store.get_e2e_room_keys_version_info( + user_id, version + ) + except StoreError as e: + if e.code == 404: + raise NotFoundError("Unknown backup version") + else: + raise + if old_info["algorithm"] != version_info["algorithm"]: + raise SynapseError( + 400, + "Algorithm does not match", + Codes.INVALID_PARAM + ) - yield self.store.update_e2e_room_keys_version(user_id, version, version_info) + yield self.store.update_e2e_room_keys_version(user_id, version, version_info) - defer.returnValue({}) + defer.returnValue({}) diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py index 1c39d2af1c..220a0de30b 100644 --- a/synapse/rest/client/v2_alpha/room_keys.py +++ b/synapse/rest/client/v2_alpha/room_keys.py @@ -394,7 +394,8 @@ class RoomKeysVersionServlet(RestServlet): "signatures": { "ed25519:something": "hijklmnop" } - } + }, + "version": "42" } HTTP/1.1 200 OK diff --git a/tests/handlers/test_e2e_room_keys.py b/tests/handlers/test_e2e_room_keys.py index c8994f416e..1c49bbbc3c 100644 --- a/tests/handlers/test_e2e_room_keys.py +++ b/tests/handlers/test_e2e_room_keys.py @@ -125,6 +125,78 @@ class E2eRoomKeysHandlerTestCase(unittest.TestCase): "auth_data": "second_version_auth_data", }) + @defer.inlineCallbacks + def test_update_version(self): + """Check that we can update versions. + """ + version = yield self.handler.create_version(self.local_user, { + "algorithm": "m.megolm_backup.v1", + "auth_data": "first_version_auth_data", + }) + self.assertEqual(version, "1") + + res = yield self.handler.update_version(self.local_user, version, { + "algorithm": "m.megolm_backup.v1", + "auth_data": "revised_first_version_auth_data", + "version": version + }) + self.assertDictEqual(res, {}) + + # check we can retrieve it as the current version + res = yield self.handler.get_version_info(self.local_user) + self.assertDictEqual(res, { + "algorithm": "m.megolm_backup.v1", + "auth_data": "revised_first_version_auth_data", + "version": version + }) + + @defer.inlineCallbacks + def test_update_missing_version(self): + """Check that we get a 404 on updating nonexistent versions + """ + res = None + try: + yield self.handler.update_version(self.local_user, "1", { + "algorithm": "m.megolm_backup.v1", + "auth_data": "revised_first_version_auth_data", + "version": "1" + }) + except errors.SynapseError as e: + res = e.code + self.assertEqual(res, 404) + + @defer.inlineCallbacks + def test_update_bad_version(self): + """Check that we get a 400 if the version in the body is missing or + doesn't match + """ + version = yield self.handler.create_version(self.local_user, { + "algorithm": "m.megolm_backup.v1", + "auth_data": "first_version_auth_data", + }) + self.assertEqual(version, "1") + + res = None + try: + yield self.handler.update_version(self.local_user, version, { + "algorithm": "m.megolm_backup.v1", + "auth_data": "revised_first_version_auth_data" + }) + except errors.SynapseError as e: + res = e.code + self.assertEqual(res, 400) + + res = None + try: + yield self.handler.update_version(self.local_user, version, { + "algorithm": "m.megolm_backup.v1", + "auth_data": "revised_first_version_auth_data", + "version": "incorrect" + }) + except errors.SynapseError as e: + res = e.code + self.assertEqual(res, 400) + @defer.inlineCallbacks def test_delete_missing_version(self): """Check that we get a 404 on deleting nonexistent versions From 9cd33d2f4bc14a165f693e2d3dfe231179e968e8 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Fri, 8 Feb 2019 17:25:57 +0000 Subject: [PATCH 31/66] Deduplicate some code in synapse.app (#4567) --- changelog.d/4567.misc | 1 + synapse/app/_base.py | 63 ++++++++++++++++++++++++++++++++ synapse/app/appservice.py | 7 +--- synapse/app/client_reader.py | 13 +------ synapse/app/event_creator.py | 13 +------ synapse/app/federation_reader.py | 13 +------ synapse/app/federation_sender.py | 12 +----- synapse/app/frontend_proxy.py | 13 +------ synapse/app/homeserver.py | 54 ++------------------------- synapse/app/media_repository.py | 13 +------ synapse/app/pusher.py | 3 +- synapse/app/synchrotron.py | 7 +--- synapse/app/user_dir.py | 13 +------ synapse/config/logger.py | 16 +++----- 14 files changed, 83 insertions(+), 158 deletions(-) create mode 100644 changelog.d/4567.misc diff --git a/changelog.d/4567.misc b/changelog.d/4567.misc new file mode 100644 index 0000000000..96a2e0aefc --- /dev/null +++ b/changelog.d/4567.misc @@ -0,0 +1 @@ +Reduce duplication of ``synapse.app`` code. diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 5b97a54d45..3cbb003035 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -15,7 +15,9 @@ import gc import logging +import signal import sys +import traceback import psutil from daemonize import Daemonize @@ -23,11 +25,25 @@ from daemonize import Daemonize from twisted.internet import error, reactor from synapse.app import check_bind_error +from synapse.crypto import context_factory from synapse.util import PreserveLoggingContext from synapse.util.rlimit import change_resource_limit logger = logging.getLogger(__name__) +_sighup_callbacks = [] + + +def register_sighup(func): + """ + Register a function to be called when a SIGHUP occurs. + + Args: + func (function): Function to be called when sent a SIGHUP signal. + Will be called with a single argument, the homeserver. + """ + _sighup_callbacks.append(func) + def start_worker_reactor(appname, config): """ Run the reactor in the main process @@ -189,3 +205,50 @@ def listen_ssl( logger.info("Synapse now listening on port %d (TLS)", port) return r + + +def refresh_certificate(hs): + """ + Refresh the TLS certificates that Synapse is using by re-reading them from + disk and updating the TLS context factories to use them. + """ + logging.info("Loading certificate from disk...") + hs.config.read_certificate_from_disk() + hs.tls_server_context_factory = context_factory.ServerContextFactory(hs.config) + hs.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( + hs.config + ) + logging.info("Certificate loaded.") + + +def start(hs, listeners=None): + """ + Start a Synapse server or worker. + + Args: + hs (synapse.server.HomeServer) + listeners (list[dict]): Listener configuration ('listeners' in homeserver.yaml) + """ + try: + # Set up the SIGHUP machinery. + if hasattr(signal, "SIGHUP"): + def handle_sighup(*args, **kwargs): + for i in _sighup_callbacks: + i(hs) + + signal.signal(signal.SIGHUP, handle_sighup) + + register_sighup(refresh_certificate) + + # Load the certificate from disk. + refresh_certificate(hs) + + # It is now safe to start your Synapse. + hs.start_listening(listeners) + hs.get_datastore().start_profiling() + except Exception: + traceback.print_exc(file=sys.stderr) + reactor = hs.get_reactor() + if reactor.running: + reactor.stop() + sys.exit(1) diff --git a/synapse/app/appservice.py b/synapse/app/appservice.py index 8559e141af..33107f56d1 100644 --- a/synapse/app/appservice.py +++ b/synapse/app/appservice.py @@ -168,12 +168,7 @@ def start(config_options): ) ps.setup() - ps.start_listening(config.worker_listeners) - - def start(): - ps.get_datastore().start_profiling() - - reactor.callWhenRunning(start) + reactor.callWhenRunning(_base.start, ps, config.worker_listeners) _base.start_worker_reactor("synapse-appservice", config) diff --git a/synapse/app/client_reader.py b/synapse/app/client_reader.py index f8a417cb60..a9d2147022 100644 --- a/synapse/app/client_reader.py +++ b/synapse/app/client_reader.py @@ -25,7 +25,6 @@ from synapse.app import _base from synapse.config._base import ConfigError from synapse.config.homeserver import HomeServerConfig from synapse.config.logger import setup_logging -from synapse.crypto import context_factory from synapse.http.server import JsonResource from synapse.http.site import SynapseSite from synapse.metrics import RegistryProxy @@ -173,17 +172,7 @@ def start(config_options): ) ss.setup() - - def start(): - ss.config.read_certificate_from_disk() - ss.tls_server_context_factory = context_factory.ServerContextFactory(config) - ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( - config - ) - ss.start_listening(config.worker_listeners) - ss.get_datastore().start_profiling() - - reactor.callWhenRunning(start) + reactor.callWhenRunning(_base.start, ss, config.worker_listeners) _base.start_worker_reactor("synapse-client-reader", config) diff --git a/synapse/app/event_creator.py b/synapse/app/event_creator.py index 656e0edc0f..b8e5196152 100644 --- a/synapse/app/event_creator.py +++ b/synapse/app/event_creator.py @@ -25,7 +25,6 @@ from synapse.app import _base from synapse.config._base import ConfigError from synapse.config.homeserver import HomeServerConfig from synapse.config.logger import setup_logging -from synapse.crypto import context_factory from synapse.http.server import JsonResource from synapse.http.site import SynapseSite from synapse.metrics import RegistryProxy @@ -194,17 +193,7 @@ def start(config_options): ) ss.setup() - - def start(): - ss.config.read_certificate_from_disk() - ss.tls_server_context_factory = context_factory.ServerContextFactory(config) - ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( - config - ) - ss.start_listening(config.worker_listeners) - ss.get_datastore().start_profiling() - - reactor.callWhenRunning(start) + reactor.callWhenRunning(_base.start, ss, config.worker_listeners) _base.start_worker_reactor("synapse-event-creator", config) diff --git a/synapse/app/federation_reader.py b/synapse/app/federation_reader.py index 3de2715132..42886dbfc1 100644 --- a/synapse/app/federation_reader.py +++ b/synapse/app/federation_reader.py @@ -26,7 +26,6 @@ from synapse.app import _base from synapse.config._base import ConfigError from synapse.config.homeserver import HomeServerConfig from synapse.config.logger import setup_logging -from synapse.crypto import context_factory from synapse.federation.transport.server import TransportLayerServer from synapse.http.site import SynapseSite from synapse.metrics import RegistryProxy @@ -160,17 +159,7 @@ def start(config_options): ) ss.setup() - - def start(): - ss.config.read_certificate_from_disk() - ss.tls_server_context_factory = context_factory.ServerContextFactory(config) - ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( - config - ) - ss.start_listening(config.worker_listeners) - ss.get_datastore().start_profiling() - - reactor.callWhenRunning(start) + reactor.callWhenRunning(_base.start, ss, config.worker_listeners) _base.start_worker_reactor("synapse-federation-reader", config) diff --git a/synapse/app/federation_sender.py b/synapse/app/federation_sender.py index d944e0517f..a461442fdc 100644 --- a/synapse/app/federation_sender.py +++ b/synapse/app/federation_sender.py @@ -25,7 +25,6 @@ from synapse.app import _base from synapse.config._base import ConfigError from synapse.config.homeserver import HomeServerConfig from synapse.config.logger import setup_logging -from synapse.crypto import context_factory from synapse.federation import send_queue from synapse.http.site import SynapseSite from synapse.metrics import RegistryProxy @@ -192,17 +191,8 @@ def start(config_options): ) ss.setup() + reactor.callWhenRunning(_base.start, ss, config.worker_listeners) - def start(): - ss.config.read_certificate_from_disk() - ss.tls_server_context_factory = context_factory.ServerContextFactory(config) - ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( - config - ) - ss.start_listening(config.worker_listeners) - ss.get_datastore().start_profiling() - - reactor.callWhenRunning(start) _base.start_worker_reactor("synapse-federation-sender", config) diff --git a/synapse/app/frontend_proxy.py b/synapse/app/frontend_proxy.py index d9ef6edc3c..d5b954361d 100644 --- a/synapse/app/frontend_proxy.py +++ b/synapse/app/frontend_proxy.py @@ -26,7 +26,6 @@ from synapse.app import _base from synapse.config._base import ConfigError from synapse.config.homeserver import HomeServerConfig from synapse.config.logger import setup_logging -from synapse.crypto import context_factory from synapse.http.server import JsonResource from synapse.http.servlet import RestServlet, parse_json_object_from_request from synapse.http.site import SynapseSite @@ -250,17 +249,7 @@ def start(config_options): ) ss.setup() - - def start(): - ss.config.read_certificate_from_disk() - ss.tls_server_context_factory = context_factory.ServerContextFactory(config) - ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( - config - ) - ss.start_listening(config.worker_listeners) - ss.get_datastore().start_profiling() - - reactor.callWhenRunning(start) + reactor.callWhenRunning(_base.start, ss, config.worker_listeners) _base.start_worker_reactor("synapse-frontend-proxy", config) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 250a17cef8..1a341568ac 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -17,7 +17,6 @@ import gc import logging import os -import signal import sys import traceback @@ -28,7 +27,6 @@ from prometheus_client import Gauge from twisted.application import service from twisted.internet import defer, reactor -from twisted.protocols.tls import TLSMemoryBIOFactory from twisted.web.resource import EncodingResourceWrapper, NoResource from twisted.web.server import GzipEncoderFactory from twisted.web.static import File @@ -49,7 +47,6 @@ from synapse.app import _base from synapse.app._base import listen_ssl, listen_tcp, quit_with_error from synapse.config._base import ConfigError from synapse.config.homeserver import HomeServerConfig -from synapse.crypto import context_factory from synapse.federation.transport.server import TransportLayerServer from synapse.http.additional_resource import AdditionalResource from synapse.http.server import RootRedirect @@ -241,10 +238,10 @@ class SynapseHomeServer(HomeServer): return resources - def start_listening(self): + def start_listening(self, listeners): config = self.get_config() - for listener in config.listeners: + for listener in listeners: if listener["type"] == "http": self._listening_services.extend( self._listener_http(config, listener) @@ -328,20 +325,11 @@ def setup(config_options): # generating config files and shouldn't try to continue. sys.exit(0) - sighup_callbacks = [] synapse.config.logger.setup_logging( config, - use_worker_options=False, - register_sighup=sighup_callbacks.append + use_worker_options=False ) - def handle_sighup(*args, **kwargs): - for i in sighup_callbacks: - i(*args, **kwargs) - - if hasattr(signal, "SIGHUP"): - signal.signal(signal.SIGHUP, handle_sighup) - events.USE_FROZEN_DICTS = config.use_frozen_dicts database_engine = create_engine(config.database_config) @@ -377,31 +365,6 @@ def setup(config_options): hs.setup() - def refresh_certificate(*args): - """ - Refresh the TLS certificates that Synapse is using by re-reading them - from disk and updating the TLS context factories to use them. - """ - logging.info("Reloading certificate from disk...") - hs.config.read_certificate_from_disk() - hs.tls_server_context_factory = context_factory.ServerContextFactory(config) - hs.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( - config - ) - logging.info("Certificate reloaded.") - - logging.info("Updating context factories...") - for i in hs._listening_services: - if isinstance(i.factory, TLSMemoryBIOFactory): - i.factory = TLSMemoryBIOFactory( - hs.tls_server_context_factory, - False, - i.factory.wrappedFactory - ) - logging.info("Context factories updated.") - - sighup_callbacks.append(refresh_certificate) - @defer.inlineCallbacks def start(): try: @@ -425,18 +388,9 @@ def setup(config_options): ): yield acme.provision_certificate() - # Read the certificate from disk and build the context factories for - # TLS. - hs.config.read_certificate_from_disk() - hs.tls_server_context_factory = context_factory.ServerContextFactory(config) - hs.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( - config - ) + _base.start(hs, config.listeners) - # It is now safe to start your Synapse. - hs.start_listening() hs.get_pusherpool().start() - hs.get_datastore().start_profiling() hs.get_datastore().start_doing_background_updates() except Exception as e: # If a DeferredList failed (like in listening on the ACME listener), diff --git a/synapse/app/media_repository.py b/synapse/app/media_repository.py index 4ecf64031b..d4cc4e9443 100644 --- a/synapse/app/media_repository.py +++ b/synapse/app/media_repository.py @@ -26,7 +26,6 @@ from synapse.app import _base from synapse.config._base import ConfigError from synapse.config.homeserver import HomeServerConfig from synapse.config.logger import setup_logging -from synapse.crypto import context_factory from synapse.http.site import SynapseSite from synapse.metrics import RegistryProxy from synapse.metrics.resource import METRICS_PREFIX, MetricsResource @@ -160,17 +159,7 @@ def start(config_options): ) ss.setup() - - def start(): - ss.config.read_certificate_from_disk() - ss.tls_server_context_factory = context_factory.ServerContextFactory(config) - ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( - config - ) - ss.start_listening(config.worker_listeners) - ss.get_datastore().start_profiling() - - reactor.callWhenRunning(start) + reactor.callWhenRunning(_base.start, ss, config.worker_listeners) _base.start_worker_reactor("synapse-media-repository", config) diff --git a/synapse/app/pusher.py b/synapse/app/pusher.py index 83b0863f00..cbf0d67f51 100644 --- a/synapse/app/pusher.py +++ b/synapse/app/pusher.py @@ -224,11 +224,10 @@ def start(config_options): ) ps.setup() - ps.start_listening(config.worker_listeners) def start(): + _base.start(ps, config.worker_listeners) ps.get_pusherpool().start() - ps.get_datastore().start_profiling() reactor.callWhenRunning(start) diff --git a/synapse/app/synchrotron.py b/synapse/app/synchrotron.py index 0354e82bf8..9163b56d86 100644 --- a/synapse/app/synchrotron.py +++ b/synapse/app/synchrotron.py @@ -445,12 +445,7 @@ def start(config_options): ) ss.setup() - ss.start_listening(config.worker_listeners) - - def start(): - ss.get_datastore().start_profiling() - - reactor.callWhenRunning(start) + reactor.callWhenRunning(_base.start, ss, config.worker_listeners) _base.start_worker_reactor("synapse-synchrotron", config) diff --git a/synapse/app/user_dir.py b/synapse/app/user_dir.py index 176d55a783..d1ab9512cd 100644 --- a/synapse/app/user_dir.py +++ b/synapse/app/user_dir.py @@ -26,7 +26,6 @@ from synapse.app import _base from synapse.config._base import ConfigError from synapse.config.homeserver import HomeServerConfig from synapse.config.logger import setup_logging -from synapse.crypto import context_factory from synapse.http.server import JsonResource from synapse.http.site import SynapseSite from synapse.metrics import RegistryProxy @@ -220,17 +219,7 @@ def start(config_options): ) ss.setup() - - def start(): - ss.config.read_certificate_from_disk() - ss.tls_server_context_factory = context_factory.ServerContextFactory(config) - ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( - config - ) - ss.start_listening(config.worker_listeners) - ss.get_datastore().start_profiling() - - reactor.callWhenRunning(start) + reactor.callWhenRunning(_base.start, ss, config.worker_listeners) _base.start_worker_reactor("synapse-user-dir", config) diff --git a/synapse/config/logger.py b/synapse/config/logger.py index a795e39b1a..4b938053fb 100644 --- a/synapse/config/logger.py +++ b/synapse/config/logger.py @@ -15,7 +15,6 @@ import logging import logging.config import os -import signal import sys from string import Template @@ -24,6 +23,7 @@ import yaml from twisted.logger import STDLibLogObserver, globalLogBeginner import synapse +from synapse.app import _base as appbase from synapse.util.logcontext import LoggingContextFilter from synapse.util.versionstring import get_version_string @@ -127,7 +127,7 @@ class LoggingConfig(Config): ) -def setup_logging(config, use_worker_options=False, register_sighup=None): +def setup_logging(config, use_worker_options=False): """ Set up python logging Args: @@ -140,12 +140,6 @@ def setup_logging(config, use_worker_options=False, register_sighup=None): register_sighup (func | None): Function to call to register a sighup handler. """ - if not register_sighup: - if getattr(signal, "SIGHUP"): - register_sighup = lambda x: signal.signal(signal.SIGHUP, x) - else: - register_sighup = lambda x: None - log_config = (config.worker_log_config if use_worker_options else config.log_config) log_file = (config.worker_log_file if use_worker_options @@ -187,7 +181,7 @@ def setup_logging(config, use_worker_options=False, register_sighup=None): else: handler = logging.StreamHandler() - def sighup(signum, stack): + def sighup(*args): pass handler.setFormatter(formatter) @@ -200,14 +194,14 @@ def setup_logging(config, use_worker_options=False, register_sighup=None): with open(log_config, 'r') as f: logging.config.dictConfig(yaml.load(f)) - def sighup(signum, stack): + def sighup(*args): # it might be better to use a file watcher or something for this. load_log_config() logging.info("Reloaded log config from %s due to SIGHUP", log_config) load_log_config() - register_sighup(sighup) + appbase.register_sighup(sighup) # make sure that the first thing we log is a thing we can grep backwards # for From 56710c7df55edc782f6562d520c4ebc839dfe500 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Fri, 8 Feb 2019 18:30:46 +0000 Subject: [PATCH 32/66] Fix 'no unique or exclusion constraint' error (#4591) Add more tables to the list of tables which need a background update to complete before we can upsert into them, which fixes a race against the background updates. --- changelog.d/4591.bugfix | 1 + synapse/storage/_base.py | 27 +++++++++++++++++++++------ 2 files changed, 22 insertions(+), 6 deletions(-) create mode 100644 changelog.d/4591.bugfix diff --git a/changelog.d/4591.bugfix b/changelog.d/4591.bugfix new file mode 100644 index 0000000000..628bbb6d81 --- /dev/null +++ b/changelog.d/4591.bugfix @@ -0,0 +1 @@ +Fix 'no unique or exclusion constraint' error diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 4872ff55b6..e124161845 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -50,6 +50,21 @@ sql_query_timer = Histogram("synapse_storage_query_time", "sec", ["verb"]) sql_txn_timer = Histogram("synapse_storage_transaction_time", "sec", ["desc"]) +# Unique indexes which have been added in background updates. Maps from table name +# to the name of the background update which added the unique index to that table. +# +# This is used by the upsert logic to figure out which tables are safe to do a proper +# UPSERT on: until the relevant background update has completed, we +# have to emulate an upsert by locking the table. +# +UNIQUE_INDEX_BACKGROUND_UPDATES = { + "user_ips": "user_ips_device_unique_index", + "device_lists_remote_extremeties": "device_lists_remote_extremeties_unique_idx", + "device_lists_remote_cache": "device_lists_remote_cache_unique_idx", + "event_search": "event_search_event_id_idx", +} + + class LoggingTransaction(object): """An object that almost-transparently proxies for the 'txn' object passed to the constructor. Adds logging and metrics to the .execute() @@ -194,7 +209,7 @@ class SQLBaseStore(object): self.database_engine = hs.database_engine # A set of tables that are not safe to use native upserts in. - self._unsafe_to_upsert_tables = {"user_ips"} + self._unsafe_to_upsert_tables = set(UNIQUE_INDEX_BACKGROUND_UPDATES.keys()) # We add the user_directory_search table to the blacklist on SQLite # because the existing search table does not have an index, making it @@ -230,12 +245,12 @@ class SQLBaseStore(object): ) updates = [x["update_name"] for x in updates] - # The User IPs table in schema #53 was missing a unique index, which we - # run as a background update. - if "user_ips_device_unique_index" not in updates: - self._unsafe_to_upsert_tables.discard("user_ips") + for table, update_name in UNIQUE_INDEX_BACKGROUND_UPDATES.items(): + if update_name not in updates: + logger.debug("Now safe to upsert in %s", table) + self._unsafe_to_upsert_tables.discard(table) - # If there's any tables left to check, reschedule to run. + # If there's any updates still running, reschedule to run. if updates: self._clock.call_later( 15.0, From 2dc2b6e9f1b85163c8ebf4d9062253a54d531f1e Mon Sep 17 00:00:00 2001 From: Valentin Anger Date: Fri, 8 Feb 2019 21:09:56 +0000 Subject: [PATCH 33/66] Allow "unavailable" presence status for /sync (#4592) * Allow "unavailable" presence status for /sync Closes #3772, closes #3779 Signed-off-by: Valentin Anger * Add changelog for PR 4592 --- changelog.d/4592.feature | 2 ++ synapse/rest/client/v2_alpha/sync.py | 2 +- synapse/rest/client/versions.py | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 changelog.d/4592.feature diff --git a/changelog.d/4592.feature b/changelog.d/4592.feature new file mode 100644 index 0000000000..112005ded4 --- /dev/null +++ b/changelog.d/4592.feature @@ -0,0 +1,2 @@ +Allow the "unavailable" presence status for /sync. +This change makes Synapse compliant with r0.4.0 of the Client-Server specification. diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index 0251146722..39d157a44b 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -75,7 +75,7 @@ class SyncRestServlet(RestServlet): """ PATTERNS = client_v2_patterns("/sync$") - ALLOWED_PRESENCE = set(["online", "offline"]) + ALLOWED_PRESENCE = set(["online", "offline", "unavailable"]) def __init__(self, hs): super(SyncRestServlet, self).__init__() diff --git a/synapse/rest/client/versions.py b/synapse/rest/client/versions.py index 29e62bfcdd..27e7cbf3cc 100644 --- a/synapse/rest/client/versions.py +++ b/synapse/rest/client/versions.py @@ -38,6 +38,7 @@ class VersionsRestServlet(RestServlet): "r0.1.0", "r0.2.0", "r0.3.0", + "r0.4.0", ], # as per MSC1497: "unstable_features": { From 4ffd10f46d62f7bc5430d148260e3240b7a0598b Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Mon, 11 Feb 2019 21:04:27 +1100 Subject: [PATCH 34/66] Be tolerant of blank TLS fingerprints config (#4589) --- changelog.d/4589.bugfix | 1 + synapse/config/tls.py | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 changelog.d/4589.bugfix diff --git a/changelog.d/4589.bugfix b/changelog.d/4589.bugfix new file mode 100644 index 0000000000..d5783f46e8 --- /dev/null +++ b/changelog.d/4589.bugfix @@ -0,0 +1 @@ +Synapse is now tolerant of the tls_fingerprints option being None or not specified. diff --git a/synapse/config/tls.py b/synapse/config/tls.py index b5f2cfd9b7..81b3a659fe 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -45,7 +45,11 @@ class TlsConfig(Config): self.tls_certificate_file = self.abspath(config.get("tls_certificate_path")) self.tls_private_key_file = self.abspath(config.get("tls_private_key_path")) - self._original_tls_fingerprints = config["tls_fingerprints"] + self._original_tls_fingerprints = config.get("tls_fingerprints", []) + + if self._original_tls_fingerprints is None: + self._original_tls_fingerprints = [] + self.tls_fingerprints = list(self._original_tls_fingerprints) self.no_tls = config.get("no_tls", False) From 6e2a5aa050fc132a7dee6b3e33a7a368207d7e5a Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Mon, 11 Feb 2019 21:36:26 +1100 Subject: [PATCH 35/66] ACME Reprovisioning (#4522) --- changelog.d/4522.feature | 1 + synapse/app/_base.py | 19 ++++++++++ synapse/app/homeserver.py | 79 +++++++++++++++++++++++++++------------ synapse/config/tls.py | 12 +++++- synapse/server.py | 3 ++ 5 files changed, 89 insertions(+), 25 deletions(-) create mode 100644 changelog.d/4522.feature diff --git a/changelog.d/4522.feature b/changelog.d/4522.feature new file mode 100644 index 0000000000..ef18daf60b --- /dev/null +++ b/changelog.d/4522.feature @@ -0,0 +1 @@ +Synapse's ACME support will now correctly reprovision a certificate that approaches its expiry while Synapse is running. diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 3cbb003035..62c633146f 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -23,6 +23,7 @@ import psutil from daemonize import Daemonize from twisted.internet import error, reactor +from twisted.protocols.tls import TLSMemoryBIOFactory from synapse.app import check_bind_error from synapse.crypto import context_factory @@ -220,6 +221,24 @@ def refresh_certificate(hs): ) logging.info("Certificate loaded.") + if hs._listening_services: + logging.info("Updating context factories...") + for i in hs._listening_services: + # When you listenSSL, it doesn't make an SSL port but a TCP one with + # a TLS wrapping factory around the factory you actually want to get + # requests. This factory attribute is public but missing from + # Twisted's documentation. + if isinstance(i.factory, TLSMemoryBIOFactory): + # We want to replace TLS factories with a new one, with the new + # TLS configuration. We do this by reaching in and pulling out + # the wrappedFactory, and then re-wrapping it. + i.factory = TLSMemoryBIOFactory( + hs.tls_server_context_factory, + False, + i.factory.wrappedFactory + ) + logging.info("Context factories updated.") + def start(hs, listeners=None): """ diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index d1cab07bb6..b4476bf16e 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -83,7 +83,6 @@ def gz_wrap(r): class SynapseHomeServer(HomeServer): DATASTORE_CLASS = DataStore - _listening_services = [] def _listener_http(self, config, listener_config): port = listener_config["port"] @@ -376,42 +375,73 @@ def setup(config_options): hs.setup() + @defer.inlineCallbacks + def do_acme(): + """ + Reprovision an ACME certificate, if it's required. + + Returns: + Deferred[bool]: Whether the cert has been updated. + """ + acme = hs.get_acme_handler() + + # Check how long the certificate is active for. + cert_days_remaining = hs.config.is_disk_cert_valid( + allow_self_signed=False + ) + + # We want to reprovision if cert_days_remaining is None (meaning no + # certificate exists), or the days remaining number it returns + # is less than our re-registration threshold. + provision = False + + if (cert_days_remaining is None): + provision = True + + if cert_days_remaining > hs.config.acme_reprovision_threshold: + provision = True + + if provision: + yield acme.provision_certificate() + + defer.returnValue(provision) + + @defer.inlineCallbacks + def reprovision_acme(): + """ + Provision a certificate from ACME, if required, and reload the TLS + certificate if it's renewed. + """ + reprovisioned = yield do_acme() + if reprovisioned: + _base.refresh_certificate(hs) + @defer.inlineCallbacks def start(): try: - # Check if the certificate is still valid. - cert_days_remaining = hs.config.is_disk_cert_valid() - + # Run the ACME provisioning code, if it's enabled. if hs.config.acme_enabled: - # If ACME is enabled, we might need to provision a certificate - # before starting. acme = hs.get_acme_handler() - # Start up the webservices which we will respond to ACME - # challenges with. + # challenges with, and then provision. yield acme.start_listening() + yield do_acme() - # We want to reprovision if cert_days_remaining is None (meaning no - # certificate exists), or the days remaining number it returns - # is less than our re-registration threshold. - if (cert_days_remaining is None) or ( - not cert_days_remaining > hs.config.acme_reprovision_threshold - ): - yield acme.provision_certificate() + # Check if it needs to be reprovisioned every day. + hs.get_clock().looping_call( + reprovision_acme, + 24 * 60 * 60 * 1000 + ) _base.start(hs, config.listeners) hs.get_pusherpool().start() hs.get_datastore().start_doing_background_updates() - except Exception as e: - # If a DeferredList failed (like in listening on the ACME listener), - # we need to print the subfailure explicitly. - if isinstance(e, defer.FirstError): - e.subFailure.printTraceback(sys.stderr) - sys.exit(1) - - # Something else went wrong when starting. Print it and bail out. + except Exception: + # Print the exception and bail out. traceback.print_exc(file=sys.stderr) + if reactor.running: + reactor.stop() sys.exit(1) reactor.callWhenRunning(start) @@ -420,7 +450,8 @@ def setup(config_options): class SynapseService(service.Service): - """A twisted Service class that will start synapse. Used to run synapse + """ + A twisted Service class that will start synapse. Used to run synapse via twistd and a .tac. """ def __init__(self, config): diff --git a/synapse/config/tls.py b/synapse/config/tls.py index 81b3a659fe..9fcc79816d 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -64,10 +64,14 @@ class TlsConfig(Config): self.tls_certificate = None self.tls_private_key = None - def is_disk_cert_valid(self): + def is_disk_cert_valid(self, allow_self_signed=True): """ Is the certificate we have on disk valid, and if so, for how long? + Args: + allow_self_signed (bool): Should we allow the certificate we + read to be self signed? + Returns: int: Days remaining of certificate validity. None: No certificate exists. @@ -88,6 +92,12 @@ class TlsConfig(Config): logger.exception("Failed to parse existing certificate off disk!") raise + if not allow_self_signed: + if tls_certificate.get_subject() == tls_certificate.get_issuer(): + raise ValueError( + "TLS Certificate is self signed, and this is not permitted" + ) + # YYYYMMDDhhmmssZ -- in UTC expires_on = datetime.strptime( tls_certificate.get_notAfter().decode('ascii'), "%Y%m%d%H%M%SZ" diff --git a/synapse/server.py b/synapse/server.py index 6c52101616..a2cf8a91cd 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -112,6 +112,8 @@ class HomeServer(object): Attributes: config (synapse.config.homeserver.HomeserverConfig): + _listening_services (list[twisted.internet.tcp.Port]): TCP ports that + we are listening on to provide HTTP services. """ __metaclass__ = abc.ABCMeta @@ -196,6 +198,7 @@ class HomeServer(object): self._reactor = reactor self.hostname = hostname self._building = {} + self._listening_services = [] self.clock = Clock(reactor) self.distributor = Distributor() From a126f86eec53cd4807a1300bea885122f8559944 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Mon, 11 Feb 2019 11:30:37 +0000 Subject: [PATCH 36/66] Transfer Server ACLs on room upgrade --- synapse/handlers/room.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 5e40e9ea46..f9af1f0046 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -284,6 +284,7 @@ class RoomCreationHandler(BaseHandler): (EventTypes.GuestAccess, ""), (EventTypes.RoomAvatar, ""), (EventTypes.Encryption, ""), + (EventTypes.ServerACL, ""), ) old_room_state_ids = yield self.store.get_filtered_current_state_ids( From eff204221748c7cfa0c88162f6fd1758f03a2f4a Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Mon, 11 Feb 2019 11:41:57 +0000 Subject: [PATCH 37/66] Changelog --- changelog.d/4608.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/4608.bugfix diff --git a/changelog.d/4608.bugfix b/changelog.d/4608.bugfix new file mode 100644 index 0000000000..e331a362c4 --- /dev/null +++ b/changelog.d/4608.bugfix @@ -0,0 +1 @@ +Transfer Server ACLs on room upgrade. \ No newline at end of file From c475275926aeee906b76621444468280d5bf569b Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Mon, 11 Feb 2019 11:44:28 +0000 Subject: [PATCH 38/66] Clarifications for reverse proxy docs (#4607) Factor out the reverse proxy info to a separate file, add some more info on reverse-proxying the federation port. --- INSTALL.md | 5 +- README.rst | 52 +----------------- changelog.d/4607.misc | 1 + docs/MSC1711_certificates_FAQ.md | 22 +++++--- docs/reverse_proxy.rst | 94 ++++++++++++++++++++++++++++++++ docs/workers.rst | 5 +- 6 files changed, 117 insertions(+), 62 deletions(-) create mode 100644 changelog.d/4607.misc create mode 100644 docs/reverse_proxy.rst diff --git a/INSTALL.md b/INSTALL.md index cbe4bda120..e496a13b21 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -359,8 +359,9 @@ Synapse v1.0. Instructions for having Synapse automatically provision and renew If you would like to use your own certificates, you can do so by changing `tls_certificate_path` and `tls_private_key_path` in `homeserver.yaml`; -alternatively, you can use a reverse-proxy. Apart from port 8448 using TLS, -both ports are the same in the default configuration. +alternatively, you can use a reverse proxy. See +[docs/reverse_proxy.rst](docs/reverse_proxy.rst) for information on configuring +a reverse proxy. ## Registering a user diff --git a/README.rst b/README.rst index e666b3b427..bc7cb5f784 100644 --- a/README.rst +++ b/README.rst @@ -263,6 +263,8 @@ So, things to check are: (it should be ``_matrix._tcp.``), and that the port and hostname it specifies are reachable from outside your network. +.. TODO: add a note about forgetting ``nocanon`` on a reverse-proxy config + Running a Demo Federation of Synapses ------------------------------------- @@ -290,7 +292,6 @@ The advantages of Postgres include: For information on how to install and use PostgreSQL, please see `docs/postgres.rst `_. - .. _reverse-proxy: Using a reverse proxy with Synapse @@ -304,54 +305,7 @@ It is recommended to put a reverse proxy such as doing so is that it means that you can expose the default https port (443) to Matrix clients without needing to run Synapse with root privileges. -The most important thing to know here is that Matrix clients and other Matrix -servers do not necessarily need to connect to your server via the same -port. Indeed, clients will use port 443 by default, whereas servers default to -port 8448. Where these are different, we refer to the 'client port' and the -'federation port'. - -All Matrix endpoints begin with ``/_matrix``, so an example nginx -configuration for forwarding client connections to Synapse might look like:: - - server { - listen 443 ssl; - listen [::]:443 ssl; - server_name matrix.example.com; - - location /_matrix { - proxy_pass http://localhost:8008; - proxy_set_header X-Forwarded-For $remote_addr; - } - } - -an example Caddy configuration might look like:: - - matrix.example.com { - proxy /_matrix http://localhost:8008 { - transparent - } - } - -and an example Apache configuration might look like:: - - - SSLEngine on - ServerName matrix.example.com; - - - ProxyPass http://127.0.0.1:8008/_matrix nocanon - ProxyPassReverse http://127.0.0.1:8008/_matrix - - - -You will also want to set ``bind_addresses: ['127.0.0.1']`` and ``x_forwarded: true`` -for port 8008 in ``homeserver.yaml`` to ensure that client IP addresses are -recorded correctly. - -Having done so, you can then use ``https://matrix.example.com`` (instead of -``https://matrix.example.com:8448``) as the "Custom server" when `Connecting to -Synapse from a client`_. - +For information on configuring one, see ``_. Identity Servers ================ diff --git a/changelog.d/4607.misc b/changelog.d/4607.misc new file mode 100644 index 0000000000..160a824378 --- /dev/null +++ b/changelog.d/4607.misc @@ -0,0 +1 @@ +Clarifications for reverse proxy docs diff --git a/docs/MSC1711_certificates_FAQ.md b/docs/MSC1711_certificates_FAQ.md index 0a781d00e3..2c52b0d517 100644 --- a/docs/MSC1711_certificates_FAQ.md +++ b/docs/MSC1711_certificates_FAQ.md @@ -42,9 +42,9 @@ imminent Matrix 1.0 release, you can also see our * It used to work just fine, why are you breaking everything? * Can I manage my own certificates rather than having Synapse renew certificates itself? - * Do you still recommend against using a reverse-proxy on the federation port? + * Do you still recommend against using a reverse proxy on the federation port? * Do I still need to give my TLS certificates to Synapse if I am using a - reverse-proxy? + reverse proxy? * Do I need the same certificate for the client and federation port? * How do I tell Synapse to reload my keys/certificates after I replace them? @@ -132,6 +132,9 @@ your domain, you can simply route all traffic through the reverse proxy by updating the SRV record appropriately (or removing it, if the proxy listens on 8448). +See [reverse_proxy.rst](reverse_proxy.rst) for information on setting up a +reverse proxy. + #### Option 3: add a .well-known file to delegate your matrix traffic This will allow you to keep Synapse on a separate domain, without having to @@ -297,17 +300,20 @@ attempt to obtain certificates from Let's Encrypt if you configure it to do so.The only requirement is that there is a valid TLS cert present for federation end points. -### Do you still recommend against using a reverse-proxy on the federation port? +### Do you still recommend against using a reverse proxy on the federation port? We no longer actively recommend against using a reverse proxy. Many admins will -find it easier to direct federation traffic to a reverse-proxy and manage their +find it easier to direct federation traffic to a reverse proxy and manage their own TLS certificates, and this is a supported configuration. +See [reverse_proxy.rst](reverse_proxy.rst) for information on setting up a +reverse proxy. + ### Do I still need to give my TLS certificates to Synapse if I am using a reverse proxy? Practically speaking, this is no longer necessary. -If you are using a reverse-proxy for all of your TLS traffic, then you can set +If you are using a reverse proxy for all of your TLS traffic, then you can set `no_tls: True`. In that case, the only reason Synapse needs the certificate is to populate a legacy 'tls_fingerprints' field in the federation API. This is ignored by Synapse 0.99.0 and later, and the only time pre-0.99 Synapses will @@ -321,9 +327,9 @@ this, you can give it any TLS certificate at all. This will be fixed soon. ### Do I need the same certificate for the client and federation port? -No. There is nothing stopping you doing so, particularly if you are using a -reverse-proxy. However, Synapse will use the same certificate on any ports -where TLS is configured. +No. There is nothing stopping you from using different certificates, +particularly if you are using a reverse proxy. However, Synapse will use the +same certificate on any ports where TLS is configured. ### How do I tell Synapse to reload my keys/certificates after I replace them? diff --git a/docs/reverse_proxy.rst b/docs/reverse_proxy.rst new file mode 100644 index 0000000000..d8aaac8a08 --- /dev/null +++ b/docs/reverse_proxy.rst @@ -0,0 +1,94 @@ +Using a reverse proxy with Synapse +================================== + +It is recommended to put a reverse proxy such as +`nginx `_, +`Apache `_, +`Caddy `_ or +`HAProxy `_ in front of Synapse. One advantage of +doing so is that it means that you can expose the default https port (443) to +Matrix clients without needing to run Synapse with root privileges. + +**NOTE**: Your reverse proxy must not 'canonicalise' or 'normalise' the +requested URI in any way (for example, by decoding ``%xx`` escapes). Beware +that Apache *will* canonicalise URIs unless you specifify ``nocanon``. + +When setting up a reverse proxy, remember that Matrix clients and other Matrix +servers do not necessarily need to connect to your server via the same server +name or port. Indeed, clients will use port 443 by default, whereas servers +default to port 8448. Where these are different, we refer to the 'client port' +and the 'federation port'. See `Setting up federation +<../README.rst#setting-up-federation>`_ for more details of the algorithm used for +federation connections. + +Let's assume that we expect clients to connect to our server at +``https://matrix.example.com``, and other servers to connect at +``https://example.com:8448``. Here are some example configurations: + +* nginx:: + + server { + listen 443 ssl; + listen [::]:443 ssl; + server_name matrix.example.com; + + location /_matrix { + proxy_pass http://localhost:8008; + proxy_set_header X-Forwarded-For $remote_addr; + } + } + + server { + listen 8448 ssl default_server; + listen [::]:8448 ssl default_server; + server_name example.com; + + location / { + proxy_pass http://localhost:8008; + proxy_set_header X-Forwarded-For $remote_addr; + } + } + +* Caddy:: + + matrix.example.com { + proxy /_matrix http://localhost:8008 { + transparent + } + } + + example.com:8448 { + proxy / http://localhost:8008 { + transparent + } + } + +* Apache (note the ``nocanon`` options here!):: + + + SSLEngine on + ServerName matrix.example.com; + + + ProxyPass http://127.0.0.1:8008/_matrix nocanon + ProxyPassReverse http://127.0.0.1:8008/_matrix + + + + + SSLEngine on + ServerName example.com; + + + ProxyPass http://127.0.0.1:8008/_matrix nocanon + ProxyPassReverse http://127.0.0.1:8008/_matrix + + + +You will also want to set ``bind_addresses: ['127.0.0.1']`` and ``x_forwarded: true`` +for port 8008 in ``homeserver.yaml`` to ensure that client IP addresses are +recorded correctly. + +Having done so, you can then use ``https://matrix.example.com`` (instead of +``https://matrix.example.com:8448``) as the "Custom server" when connecting to +Synapse from a client. diff --git a/docs/workers.rst b/docs/workers.rst index 101e950020..dd3a84ba0d 100644 --- a/docs/workers.rst +++ b/docs/workers.rst @@ -26,9 +26,8 @@ Configuration To make effective use of the workers, you will need to configure an HTTP reverse-proxy such as nginx or haproxy, which will direct incoming requests to the correct worker, or to the main synapse instance. Note that this includes -requests made to the federation port. The caveats regarding running a -reverse-proxy on the federation port still apply (see -https://github.com/matrix-org/synapse/blob/master/README.rst#reverse-proxying-the-federation-port). +requests made to the federation port. See ``_ for +information on setting up a reverse proxy. To enable workers, you need to add two replication listeners to the master synapse, e.g.:: From 24b7f3916d514f570985b6ebc3936938bb5adf45 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Mon, 11 Feb 2019 12:50:30 +0000 Subject: [PATCH 39/66] Clean up default listener configuration (#4586) Rearrange the comments to try to clarify them, and expand on what some of it means. Use a sensible default 'bind_addresses' setting. For the insecure port, only bind to localhost, and enable x_forwarded, since apparently it's for use behind a load-balancer. --- changelog.d/4586.misc | 1 + synapse/config/server.py | 129 ++++++++++++++++++++++++--------------- 2 files changed, 82 insertions(+), 48 deletions(-) create mode 100644 changelog.d/4586.misc diff --git a/changelog.d/4586.misc b/changelog.d/4586.misc new file mode 100644 index 0000000000..37af371ccf --- /dev/null +++ b/changelog.d/4586.misc @@ -0,0 +1 @@ +Clean up default listener configuration diff --git a/synapse/config/server.py b/synapse/config/server.py index f0a60cc712..ce0458195c 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -24,6 +24,14 @@ from ._base import Config, ConfigError logger = logging.Logger(__name__) +# by default, we attempt to listen on both '::' *and* '0.0.0.0' because some OSes +# (Windows, macOS, other BSD/Linux where net.ipv6.bindv6only is set) will only listen +# on IPv6 when '::' is set. +# +# We later check for errors when binding to 0.0.0.0 and ignore them if :: is also in +# in the list. +DEFAULT_BIND_ADDRESSES = ['::', '0.0.0.0'] + class ServerConfig(Config): @@ -124,10 +132,13 @@ class ServerConfig(Config): bind_address = listener.pop("bind_address", None) bind_addresses = listener.setdefault("bind_addresses", []) + # if bind_address was specified, add it to the list of addresses if bind_address: bind_addresses.append(bind_address) - elif not bind_addresses: - bind_addresses.append('') + + # if we still have an empty list of addresses, use the default list + if not bind_addresses: + bind_addresses.extend(DEFAULT_BIND_ADDRESSES) if not self.web_client_location: _warn_if_webclient_configured(self.listeners) @@ -295,76 +306,98 @@ class ServerConfig(Config): # List of ports that Synapse should listen on, their purpose and their # configuration. + # + # Options for each listener include: + # + # port: the TCP port to bind to + # + # bind_addresses: a list of local addresses to listen on. The default is + # 'all local interfaces'. + # + # type: the type of listener. Normally 'http', but other valid options are: + # 'manhole' (see docs/manhole.md), + # 'metrics' (see docs/metrics-howto.rst), + # 'replication' (see docs/workers.rst). + # + # tls: set to true to enable TLS for this listener. Will use the TLS + # key/cert specified in tls_private_key_path / tls_certificate_path. + # + # x_forwarded: Only valid for an 'http' listener. Set to true to use the + # X-Forwarded-For header as the client IP. Useful when Synapse is + # behind a reverse-proxy. + # + # resources: Only valid for an 'http' listener. A list of resources to host + # on this port. Options for each resource are: + # + # names: a list of names of HTTP resources. See below for a list of + # valid resource names. + # + # compress: set to true to enable HTTP comression for this resource. + # + # additional_resources: Only valid for an 'http' listener. A map of + # additional endpoints which should be loaded via dynamic modules. + # + # Valid resource names are: + # + # client: the client-server API (/_matrix/client). Also implies 'media' and + # 'static'. + # + # consent: user consent forms (/_matrix/consent). See + # docs/consent_tracking.md. + # + # federation: the server-server API (/_matrix/federation). Also implies + # 'media', 'keys', 'openid' + # + # keys: the key discovery API (/_matrix/keys). + # + # media: the media API (/_matrix/media). + # + # metrics: the metrics interface. See docs/metrics-howto.rst. + # + # openid: OpenID authentication. + # + # replication: the HTTP replication API (/_synapse/replication). See + # docs/workers.rst. + # + # static: static resources under synapse/static (/_matrix/static). (Mostly + # useful for 'fallback authentication'.) + # + # webclient: A web client. Requires web_client_location to be set. + # listeners: - # Main HTTPS listener + # Main HTTPS listener. # For when matrix traffic is sent directly to synapse. - - - # The port to listen for HTTPS requests on. - port: %(bind_port)s - - # Local addresses to listen on. - # On Linux and Mac OS, `::` will listen on all IPv4 and IPv6 - # addresses by default. For most other OSes, this will only listen - # on IPv6. - bind_addresses: - - '::' - - '0.0.0.0' - - # This is a 'http' listener, allows us to specify 'resources'. + - port: %(bind_port)s type: http - tls: true - # Use the X-Forwarded-For (XFF) header as the client IP and not the - # actual client IP. - x_forwarded: false - # List of HTTP resources to serve on this listener. resources: - - - # List of resources to host on this listener. - names: - - client # The client-server APIs, both v1 and v2 - # - webclient # A web client. Requires web_client_location to be set. - - # Should synapse compress HTTP responses to clients that support it? - # This should be disabled if running synapse behind a load balancer - # that can do automatic compression. + - names: [client] compress: true - - - names: [federation] # Federation APIs + - names: [federation] compress: false - # # If federation is disabled synapse can still expose the open ID endpoint - # # to allow integrations to authenticate users - # - names: [openid] - # compress: false - - # optional list of additional endpoints which can be loaded via - # dynamic modules + # example addional_resources: + # # additional_resources: # "/_matrix/my/custom/endpoint": # module: my_module.CustomRequestHandler # config: {} - # Unsecure HTTP listener, - # For when matrix traffic passes through loadbalancer that unwraps TLS. + # Unsecure HTTP listener + # For when matrix traffic passes through a reverse-proxy that unwraps TLS. - port: %(unsecure_port)s tls: false - bind_addresses: ['::', '0.0.0.0'] + bind_addresses: ['::1', '127.0.0.1'] type: http - - x_forwarded: false + x_forwarded: true resources: - names: [client] compress: true - names: [federation] compress: false - # # If federation is disabled synapse can still expose the open ID endpoint - # # to allow integrations to authenticate users - # - names: [openid] - # compress: false # Turn on the twisted ssh manhole service on localhost on the given # port. From 5d27730a73d01bd3ff7eb42a21f2f024493c5b89 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Mon, 11 Feb 2019 18:03:30 +0000 Subject: [PATCH 40/66] Move ClientTLSOptionsFactory init out of refresh_certificates (#4611) It's nothing to do with refreshing the certificates. No idea why it was here. --- changelog.d/4611.misc | 1 + synapse/app/_base.py | 3 --- synapse/http/matrixfederationclient.py | 4 ++-- synapse/server.py | 6 +++++- tests/http/test_fedclient.py | 4 +--- 5 files changed, 9 insertions(+), 9 deletions(-) create mode 100644 changelog.d/4611.misc diff --git a/changelog.d/4611.misc b/changelog.d/4611.misc new file mode 100644 index 0000000000..d2e0a05daa --- /dev/null +++ b/changelog.d/4611.misc @@ -0,0 +1 @@ +Move ClientTLSOptionsFactory init out of refresh_certificates diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 62c633146f..e1fc1afd5b 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -216,9 +216,6 @@ def refresh_certificate(hs): logging.info("Loading certificate from disk...") hs.config.read_certificate_from_disk() hs.tls_server_context_factory = context_factory.ServerContextFactory(hs.config) - hs.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( - hs.config - ) logging.info("Certificate loaded.") if hs._listening_services: diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 5ee4d528d2..3c24bf3805 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -168,7 +168,7 @@ class MatrixFederationHttpClient(object): requests. """ - def __init__(self, hs): + def __init__(self, hs, tls_client_options_factory): self.hs = hs self.signing_key = hs.config.signing_key[0] self.server_name = hs.hostname @@ -176,7 +176,7 @@ class MatrixFederationHttpClient(object): self.agent = MatrixFederationAgent( hs.get_reactor(), - hs.tls_client_options_factory, + tls_client_options_factory, ) self.clock = hs.get_clock() self._store = hs.get_datastore() diff --git a/synapse/server.py b/synapse/server.py index a2cf8a91cd..8615b67ad4 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -31,6 +31,7 @@ from synapse.api.filtering import Filtering from synapse.api.ratelimiting import Ratelimiter from synapse.appservice.api import ApplicationServiceApi from synapse.appservice.scheduler import ApplicationServiceScheduler +from synapse.crypto import context_factory from synapse.crypto.keyring import Keyring from synapse.events.builder import EventBuilderFactory from synapse.events.spamcheck import SpamChecker @@ -367,7 +368,10 @@ class HomeServer(object): return PusherPool(self) def build_http_client(self): - return MatrixFederationHttpClient(self) + tls_client_options_factory = context_factory.ClientTLSOptionsFactory( + self.config + ) + return MatrixFederationHttpClient(self, tls_client_options_factory) def build_db_pool(self): name = self.db_config["name"] diff --git a/tests/http/test_fedclient.py b/tests/http/test_fedclient.py index 018c77ebcd..b03b37affe 100644 --- a/tests/http/test_fedclient.py +++ b/tests/http/test_fedclient.py @@ -43,13 +43,11 @@ def check_logcontext(context): class FederationClientTests(HomeserverTestCase): def make_homeserver(self, reactor, clock): - hs = self.setup_test_homeserver(reactor=reactor, clock=clock) - hs.tls_client_options_factory = None return hs def prepare(self, reactor, clock, homeserver): - self.cl = MatrixFederationHttpClient(self.hs) + self.cl = MatrixFederationHttpClient(self.hs, None) self.reactor.lookups["testserv"] = "1.2.3.4" def test_client_get(self): From 086f6f27d409520e71556cad4707cb2f70476e20 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 11 Feb 2019 21:00:41 +0000 Subject: [PATCH 41/66] Logging improvements around TLS certs Log which file we're reading keys and certs from, and refactor the code a bit in preparation for other work --- changelog.d/4615.misc | 1 + synapse/app/_base.py | 6 ++--- synapse/config/tls.py | 54 ++++++++++++++++++++++++++++--------------- 3 files changed, 39 insertions(+), 22 deletions(-) create mode 100644 changelog.d/4615.misc diff --git a/changelog.d/4615.misc b/changelog.d/4615.misc new file mode 100644 index 0000000000..c7266fcfc7 --- /dev/null +++ b/changelog.d/4615.misc @@ -0,0 +1 @@ +Logging improvements around TLS certs diff --git a/synapse/app/_base.py b/synapse/app/_base.py index e1fc1afd5b..6d72de1daa 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -213,13 +213,11 @@ def refresh_certificate(hs): Refresh the TLS certificates that Synapse is using by re-reading them from disk and updating the TLS context factories to use them. """ - logging.info("Loading certificate from disk...") hs.config.read_certificate_from_disk() hs.tls_server_context_factory = context_factory.ServerContextFactory(hs.config) - logging.info("Certificate loaded.") if hs._listening_services: - logging.info("Updating context factories...") + logger.info("Updating context factories...") for i in hs._listening_services: # When you listenSSL, it doesn't make an SSL port but a TCP one with # a TLS wrapping factory around the factory you actually want to get @@ -234,7 +232,7 @@ def refresh_certificate(hs): False, i.factory.wrappedFactory ) - logging.info("Context factories updated.") + logger.info("Context factories updated.") def start(hs, listeners=None): diff --git a/synapse/config/tls.py b/synapse/config/tls.py index 9fcc79816d..76d2add4fe 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -25,7 +25,7 @@ from OpenSSL import crypto from synapse.config._base import Config -logger = logging.getLogger() +logger = logging.getLogger(__name__) class TlsConfig(Config): @@ -110,20 +110,10 @@ class TlsConfig(Config): """ Read the certificates from disk. """ - self.tls_certificate = self.read_tls_certificate(self.tls_certificate_file) - - # Check if it is self-signed, and issue a warning if so. - if self.tls_certificate.get_issuer() == self.tls_certificate.get_subject(): - warnings.warn( - ( - "Self-signed TLS certificates will not be accepted by Synapse 1.0. " - "Please either provide a valid certificate, or use Synapse's ACME " - "support to provision one." - ) - ) + self.tls_certificate = self.read_tls_certificate() if not self.no_tls: - self.tls_private_key = self.read_tls_private_key(self.tls_private_key_file) + self.tls_private_key = self.read_tls_private_key() self.tls_fingerprints = list(self._original_tls_fingerprints) @@ -250,10 +240,38 @@ class TlsConfig(Config): % locals() ) - def read_tls_certificate(self, cert_path): - cert_pem = self.read_file(cert_path, "tls_certificate") - return crypto.load_certificate(crypto.FILETYPE_PEM, cert_pem) + def read_tls_certificate(self): + """Reads the TLS certificate from the configured file, and returns it - def read_tls_private_key(self, private_key_path): - private_key_pem = self.read_file(private_key_path, "tls_private_key") + Also checks if it is self-signed, and warns if so + + Returns: + OpenSSL.crypto.X509: the certificate + """ + cert_path = self.tls_certificate_file + logger.info("Loading TLS certificate from %s", cert_path) + cert_pem = self.read_file(cert_path, "tls_certificate_path") + cert = crypto.load_certificate(crypto.FILETYPE_PEM, cert_pem) + + # Check if it is self-signed, and issue a warning if so. + if cert.get_issuer() == cert.get_subject(): + warnings.warn( + ( + "Self-signed TLS certificates will not be accepted by Synapse 1.0. " + "Please either provide a valid certificate, or use Synapse's ACME " + "support to provision one." + ) + ) + + return cert + + def read_tls_private_key(self): + """Reads the TLS private key from the configured file, and returns it + + Returns: + OpenSSL.crypto.PKey: the private key + """ + private_key_path = self.tls_private_key_file + logger.info("Loading TLS key from %s", private_key_path) + private_key_pem = self.read_file(private_key_path, "tls_private_key_path") return crypto.load_privatekey(crypto.FILETYPE_PEM, private_key_pem) From 2129dd1a023d1e221dab8753be3fbd7024963634 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 11 Feb 2019 21:13:53 +0000 Subject: [PATCH 42/66] Fail cleanly if listener config lacks a 'port' ... otherwise we would fail with a mysterious KeyError or something later. --- changelog.d/4616.misc | 1 + synapse/config/server.py | 5 +++++ 2 files changed, 6 insertions(+) create mode 100644 changelog.d/4616.misc diff --git a/changelog.d/4616.misc b/changelog.d/4616.misc new file mode 100644 index 0000000000..ee79e208e3 --- /dev/null +++ b/changelog.d/4616.misc @@ -0,0 +1 @@ +Fail cleanly if listener config lacks a 'port' diff --git a/synapse/config/server.py b/synapse/config/server.py index ce0458195c..eed9d7c81e 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -129,6 +129,11 @@ class ServerConfig(Config): self.listeners = config.get("listeners", []) for listener in self.listeners: + if not isinstance(listener.get("port", None), int): + raise ConfigError( + "Listener configuration is lacking a valid 'port' option" + ) + bind_address = listener.pop("bind_address", None) bind_addresses = listener.setdefault("bind_addresses", []) From 9645728619828fda050fa08aaa25628f5db5d775 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 11 Feb 2019 21:30:59 +0000 Subject: [PATCH 43/66] Don't create server contexts when TLS is disabled we aren't going to use them anyway. --- changelog.d/4617.misc | 1 + synapse/app/_base.py | 5 +++++ synapse/crypto/context_factory.py | 4 +--- 3 files changed, 7 insertions(+), 3 deletions(-) create mode 100644 changelog.d/4617.misc diff --git a/changelog.d/4617.misc b/changelog.d/4617.misc new file mode 100644 index 0000000000..6d751865c9 --- /dev/null +++ b/changelog.d/4617.misc @@ -0,0 +1 @@ +Don't create server contexts when TLS is disabled diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 6d72de1daa..6b3cb61ae9 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -214,6 +214,11 @@ def refresh_certificate(hs): disk and updating the TLS context factories to use them. """ hs.config.read_certificate_from_disk() + + if hs.config.no_tls: + # nothing else to do here + return + hs.tls_server_context_factory = context_factory.ServerContextFactory(hs.config) if hs._listening_services: diff --git a/synapse/crypto/context_factory.py b/synapse/crypto/context_factory.py index 286ad80100..85f2848fb1 100644 --- a/synapse/crypto/context_factory.py +++ b/synapse/crypto/context_factory.py @@ -43,9 +43,7 @@ class ServerContextFactory(ContextFactory): logger.exception("Failed to enable elliptic curve for TLS") context.set_options(SSL.OP_NO_SSLv2 | SSL.OP_NO_SSLv3) context.use_certificate_chain_file(config.tls_certificate_file) - - if not config.no_tls: - context.use_privatekey(config.tls_private_key) + context.use_privatekey(config.tls_private_key) # https://hynek.me/articles/hardening-your-web-servers-ssl-ciphers/ context.set_cipher_list( From 4fddf8fc77496d9bb3b5fa8835f0e5ba9a5a9926 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 11 Feb 2019 17:57:58 +0000 Subject: [PATCH 44/66] Infer no_tls from presence of TLS listeners Rather than have to specify `no_tls` explicitly, infer whether we need to load the TLS keys etc from whether we have any TLS-enabled listeners. --- changelog.d/4613.feature | 1 + changelog.d/4615.feature | 1 + changelog.d/4615.misc | 1 - changelog.d/4617.feature | 1 + changelog.d/4617.misc | 1 - synapse/app/_base.py | 2 +- synapse/app/homeserver.py | 5 ----- synapse/config/homeserver.py | 2 +- synapse/config/server.py | 23 ++++++++++++++++++++--- synapse/config/tls.py | 10 ++-------- 10 files changed, 27 insertions(+), 20 deletions(-) create mode 100644 changelog.d/4613.feature create mode 100644 changelog.d/4615.feature delete mode 100644 changelog.d/4615.misc create mode 100644 changelog.d/4617.feature delete mode 100644 changelog.d/4617.misc diff --git a/changelog.d/4613.feature b/changelog.d/4613.feature new file mode 100644 index 0000000000..098f906af2 --- /dev/null +++ b/changelog.d/4613.feature @@ -0,0 +1 @@ +There is no longer any need to specify `no_tls`: it is inferred from the absence of TLS listeners diff --git a/changelog.d/4615.feature b/changelog.d/4615.feature new file mode 100644 index 0000000000..098f906af2 --- /dev/null +++ b/changelog.d/4615.feature @@ -0,0 +1 @@ +There is no longer any need to specify `no_tls`: it is inferred from the absence of TLS listeners diff --git a/changelog.d/4615.misc b/changelog.d/4615.misc deleted file mode 100644 index c7266fcfc7..0000000000 --- a/changelog.d/4615.misc +++ /dev/null @@ -1 +0,0 @@ -Logging improvements around TLS certs diff --git a/changelog.d/4617.feature b/changelog.d/4617.feature new file mode 100644 index 0000000000..098f906af2 --- /dev/null +++ b/changelog.d/4617.feature @@ -0,0 +1 @@ +There is no longer any need to specify `no_tls`: it is inferred from the absence of TLS listeners diff --git a/changelog.d/4617.misc b/changelog.d/4617.misc deleted file mode 100644 index 6d751865c9..0000000000 --- a/changelog.d/4617.misc +++ /dev/null @@ -1 +0,0 @@ -Don't create server contexts when TLS is disabled diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 6b3cb61ae9..50fd17c0be 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -215,7 +215,7 @@ def refresh_certificate(hs): """ hs.config.read_certificate_from_disk() - if hs.config.no_tls: + if not hs.config.has_tls_listener(): # nothing else to do here return diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index b4476bf16e..dbd98d394f 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -90,11 +90,6 @@ class SynapseHomeServer(HomeServer): tls = listener_config.get("tls", False) site_tag = listener_config.get("tag", port) - if tls and config.no_tls: - raise ConfigError( - "Listener on port %i has TLS enabled, but no_tls is set" % (port,), - ) - resources = {} for res in listener_config["resources"]: for name in res["names"]: diff --git a/synapse/config/homeserver.py b/synapse/config/homeserver.py index 5aad062c36..727fdc54d8 100644 --- a/synapse/config/homeserver.py +++ b/synapse/config/homeserver.py @@ -42,7 +42,7 @@ from .voip import VoipConfig from .workers import WorkerConfig -class HomeServerConfig(TlsConfig, ServerConfig, DatabaseConfig, LoggingConfig, +class HomeServerConfig(ServerConfig, TlsConfig, DatabaseConfig, LoggingConfig, RatelimitConfig, ContentRepositoryConfig, CaptchaConfig, VoipConfig, RegistrationConfig, MetricsConfig, ApiConfig, AppServiceConfig, KeyConfig, SAML2Config, CasConfig, diff --git a/synapse/config/server.py b/synapse/config/server.py index eed9d7c81e..767897c419 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -126,14 +126,22 @@ class ServerConfig(Config): self.public_baseurl += '/' self.start_pushers = config.get("start_pushers", True) - self.listeners = config.get("listeners", []) - - for listener in self.listeners: + self.listeners = [] + for listener in config.get("listeners", []): if not isinstance(listener.get("port", None), int): raise ConfigError( "Listener configuration is lacking a valid 'port' option" ) + if listener.setdefault("tls", False): + # no_tls is not really supported any more, but let's grandfather it in + # here. + if config.get("no_tls", False): + logger.info( + "Ignoring TLS-enabled listener on port %i due to no_tls" + ) + continue + bind_address = listener.pop("bind_address", None) bind_addresses = listener.setdefault("bind_addresses", []) @@ -145,6 +153,8 @@ class ServerConfig(Config): if not bind_addresses: bind_addresses.extend(DEFAULT_BIND_ADDRESSES) + self.listeners.append(listener) + if not self.web_client_location: _warn_if_webclient_configured(self.listeners) @@ -152,6 +162,9 @@ class ServerConfig(Config): bind_port = config.get("bind_port") if bind_port: + if config.get("no_tls", False): + raise ConfigError("no_tls is incompatible with bind_port") + self.listeners = [] bind_host = config.get("bind_host", "") gzip_responses = config.get("gzip_responses", True) @@ -198,6 +211,7 @@ class ServerConfig(Config): "port": manhole, "bind_addresses": ["127.0.0.1"], "type": "manhole", + "tls": False, }) metrics_port = config.get("metrics_port") @@ -223,6 +237,9 @@ class ServerConfig(Config): _check_resource_config(self.listeners) + def has_tls_listener(self): + return any(l["tls"] for l in self.listeners) + def default_config(self, server_name, data_dir_path, **kwargs): _, bind_port = parse_and_validate_server_name(server_name) if bind_port is not None: diff --git a/synapse/config/tls.py b/synapse/config/tls.py index 76d2add4fe..e37a41eff4 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -51,7 +51,6 @@ class TlsConfig(Config): self._original_tls_fingerprints = [] self.tls_fingerprints = list(self._original_tls_fingerprints) - self.no_tls = config.get("no_tls", False) # This config option applies to non-federation HTTP clients # (e.g. for talking to recaptcha, identity servers, and such) @@ -141,6 +140,8 @@ class TlsConfig(Config): return ( """\ + ## TLS ## + # PEM-encoded X509 certificate for TLS. # This certificate, as of Synapse 1.0, will need to be a valid and verifiable # certificate, signed by a recognised Certificate Authority. @@ -201,13 +202,6 @@ class TlsConfig(Config): # # reprovision_threshold: 30 - # If your server runs behind a reverse-proxy which terminates TLS connections - # (for both client and federation connections), it may be useful to disable - # All TLS support for incoming connections. Setting no_tls to True will - # do so (and avoid the need to give synapse a TLS private key). - # - # no_tls: True - # List of allowed TLS fingerprints for this server to publish along # with the signing keys for this server. Other matrix servers that # make HTTPS requests to this server will check that the TLS From 0ca290865350212e1834730c918973162a3067f4 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 11 Feb 2019 22:01:27 +0000 Subject: [PATCH 45/66] fix tests --- synapse/config/tls.py | 2 +- tests/config/test_tls.py | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/synapse/config/tls.py b/synapse/config/tls.py index e37a41eff4..86e6eb80db 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -111,7 +111,7 @@ class TlsConfig(Config): """ self.tls_certificate = self.read_tls_certificate() - if not self.no_tls: + if self.has_tls_listener(): self.tls_private_key = self.read_tls_private_key() self.tls_fingerprints = list(self._original_tls_fingerprints) diff --git a/tests/config/test_tls.py b/tests/config/test_tls.py index 4ccaf35603..d8fd18a9cb 100644 --- a/tests/config/test_tls.py +++ b/tests/config/test_tls.py @@ -20,6 +20,11 @@ from synapse.config.tls import TlsConfig from tests.unittest import TestCase +class TestConfig(TlsConfig): + def has_tls_listener(self): + return False + + class TLSConfigTests(TestCase): def test_warn_self_signed(self): @@ -55,11 +60,10 @@ s4niecZKPBizL6aucT59CsunNmmb5Glq8rlAcU+1ZTZZzGYqVYhF6axB9Qg= config = { "tls_certificate_path": os.path.join(config_dir, "cert.pem"), - "no_tls": True, "tls_fingerprints": [] } - t = TlsConfig() + t = TestConfig() t.read_config(config) t.read_certificate_from_disk() From 91f8cd3307fee502c6bfb064f825952b05c4617b Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 11 Feb 2019 22:15:08 +0000 Subject: [PATCH 46/66] Remove redundant entries from docker config * no_tls is now redundant (#4613) * we don't need a dummy cert any more (#4618) --- changelog.d/4619.misc | 1 + docker/conf/dummy.tls.crt | 17 ----------------- docker/conf/homeserver.yaml | 8 +------- 3 files changed, 2 insertions(+), 24 deletions(-) create mode 100644 changelog.d/4619.misc delete mode 100644 docker/conf/dummy.tls.crt diff --git a/changelog.d/4619.misc b/changelog.d/4619.misc new file mode 100644 index 0000000000..886fedf198 --- /dev/null +++ b/changelog.d/4619.misc @@ -0,0 +1 @@ +Remove redundant entries from docker config diff --git a/docker/conf/dummy.tls.crt b/docker/conf/dummy.tls.crt deleted file mode 100644 index 8e3b1a9aaa..0000000000 --- a/docker/conf/dummy.tls.crt +++ /dev/null @@ -1,17 +0,0 @@ ------BEGIN CERTIFICATE----- -MIICnTCCAYUCAgPoMA0GCSqGSIb3DQEBCwUAMBQxEjAQBgNVBAMMCWxvY2FsaG9z -dDAeFw0xOTAxMTUwMDQxNTBaFw0yOTAxMTIwMDQxNTBaMBQxEjAQBgNVBAMMCWxv -Y2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMKqm81/8j5d -R1s7VZ8ueg12gJrPVCCAOkp0UnuC/ZlXhN0HTvnhQ+B0IlSgB4CcQZyf4jnA6o4M -rwSc7VX0MPE9x/idoA0g/0WoC6tsxugOrvbzCw8Tv+fnXglm6uVc7aFPfx69wU3q -lUHGD/8jtEoHxmCG177Pt2lHAfiVLBAyMQGtETzxt/yAfkloaybe316qoljgK5WK -cokdAt9G84EEqxNeEnx5FG3Vc100bAqJS4GvQlFgtF9KFEqZKEyB1yKBpPMDfPIS -V9hIV0gswSmYI8dpyBlGf5lPElY68ZGABmOQgr0RI5qHK/h28OpFPE0q3v4AMHgZ -I36wii4NrAUCAwEAATANBgkqhkiG9w0BAQsFAAOCAQEAfD8kcpZ+dn08xh1qtKtp -X+/YNZaOBIeVdlCzfoZKNblSFAFD/jCfObNJYvZMUQ8NX2UtEJp1lTA6m7ltSsdY -gpC2k1VD8iN+ooXklJmL0kxc7UUqho8I0l9vn35h+lhLF0ihT6XfZVi/lDHWl+4G -rG+v9oxvCSCWrNWLearSlFPtQQ8xPtOE0nLwfXtOI/H/2kOuC38ihaIWM4jjbWXK -E/ksgUfuDv0mFiwf1YdBF5/M3/qOowqzU8HgMJ3WoT/9Po5Ya1pWc+3BcxxytUDf -XdMu0tWHKX84tZxLcR1nZHzluyvFFM8xNtLi9xV0Z7WbfT76V0C/ulEOybGInYsv -nQ== ------END CERTIFICATE----- diff --git a/docker/conf/homeserver.yaml b/docker/conf/homeserver.yaml index f07d5c1001..babd5bef9e 100644 --- a/docker/conf/homeserver.yaml +++ b/docker/conf/homeserver.yaml @@ -2,13 +2,7 @@ ## TLS ## -{% if SYNAPSE_NO_TLS %} -no_tls: True - -# workaround for https://github.com/matrix-org/synapse/issues/4554 -tls_certificate_path: "/conf/dummy.tls.crt" - -{% else %} +{% if not SYNAPSE_NO_TLS %} tls_certificate_path: "/data/{{ SYNAPSE_SERVER_NAME }}.tls.crt" tls_private_key_path: "/data/{{ SYNAPSE_SERVER_NAME }}.tls.key" From dfc846a316a92a750133ed04f3625da70f7944d7 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 12 Feb 2019 10:37:00 +0000 Subject: [PATCH 47/66] fix self-signed cert notice from generate-config fixes #4620 --- changelog.d/4625.bugfix | 1 + synapse/config/_base.py | 9 ++------- 2 files changed, 3 insertions(+), 7 deletions(-) create mode 100644 changelog.d/4625.bugfix diff --git a/changelog.d/4625.bugfix b/changelog.d/4625.bugfix new file mode 100644 index 0000000000..3dc0ecf24c --- /dev/null +++ b/changelog.d/4625.bugfix @@ -0,0 +1 @@ +fix self-signed cert notice from generate-config. diff --git a/synapse/config/_base.py b/synapse/config/_base.py index 5858fb92b4..5aec43b702 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -257,7 +257,7 @@ class Config(object): "--keys-directory", metavar="DIRECTORY", help="Used with 'generate-*' options to specify where files such as" - " certs and signing keys should be stored in, unless explicitly" + " signing keys should be stored, unless explicitly" " specified in the config.", ) config_parser.add_argument( @@ -313,16 +313,11 @@ class Config(object): print( ( "A config file has been generated in %r for server name" - " %r with corresponding SSL keys and self-signed" - " certificates. Please review this file and customise it" + " %r. Please review this file and customise it" " to your needs." ) % (config_path, server_name) ) - print( - "If this server name is incorrect, you will need to" - " regenerate the SSL certificates" - ) return else: print( From 32b781bfe20034052d71558c0ed9b0df511a1f77 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 12 Feb 2019 10:51:31 +0000 Subject: [PATCH 48/66] Fix error when loading cert if tls is disabled (#4618) If TLS is disabled, it should not be an error if no cert is given. Fixes #4554. --- changelog.d/4618.bugfix | 1 + synapse/app/_base.py | 5 ++-- synapse/config/tls.py | 57 +++++++++++++++++++++++++++++----------- tests/config/test_tls.py | 2 +- 4 files changed, 47 insertions(+), 18 deletions(-) create mode 100644 changelog.d/4618.bugfix diff --git a/changelog.d/4618.bugfix b/changelog.d/4618.bugfix new file mode 100644 index 0000000000..22115a020e --- /dev/null +++ b/changelog.d/4618.bugfix @@ -0,0 +1 @@ +Fix failure to start when not TLS certificate was given even if TLS was disabled. diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 50fd17c0be..5b0ca312e2 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -213,12 +213,13 @@ def refresh_certificate(hs): Refresh the TLS certificates that Synapse is using by re-reading them from disk and updating the TLS context factories to use them. """ - hs.config.read_certificate_from_disk() if not hs.config.has_tls_listener(): - # nothing else to do here + # attempt to reload the certs for the good of the tls_fingerprints + hs.config.read_certificate_from_disk(require_cert_and_key=False) return + hs.config.read_certificate_from_disk(require_cert_and_key=True) hs.tls_server_context_factory = context_factory.ServerContextFactory(hs.config) if hs._listening_services: diff --git a/synapse/config/tls.py b/synapse/config/tls.py index 86e6eb80db..57f117a14d 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -23,7 +23,7 @@ from unpaddedbase64 import encode_base64 from OpenSSL import crypto -from synapse.config._base import Config +from synapse.config._base import Config, ConfigError logger = logging.getLogger(__name__) @@ -45,6 +45,19 @@ class TlsConfig(Config): self.tls_certificate_file = self.abspath(config.get("tls_certificate_path")) self.tls_private_key_file = self.abspath(config.get("tls_private_key_path")) + + if self.has_tls_listener(): + if not self.tls_certificate_file: + raise ConfigError( + "tls_certificate_path must be specified if TLS-enabled listeners are " + "configured." + ) + if not self.tls_private_key_file: + raise ConfigError( + "tls_certificate_path must be specified if TLS-enabled listeners are " + "configured." + ) + self._original_tls_fingerprints = config.get("tls_fingerprints", []) if self._original_tls_fingerprints is None: @@ -105,26 +118,40 @@ class TlsConfig(Config): days_remaining = (expires_on - now).days return days_remaining - def read_certificate_from_disk(self): + def read_certificate_from_disk(self, require_cert_and_key): """ - Read the certificates from disk. - """ - self.tls_certificate = self.read_tls_certificate() + Read the certificates and private key from disk. - if self.has_tls_listener(): + Args: + require_cert_and_key (bool): set to True to throw an error if the certificate + and key file are not given + """ + if require_cert_and_key: self.tls_private_key = self.read_tls_private_key() + self.tls_certificate = self.read_tls_certificate() + elif self.tls_certificate_file: + # we only need the certificate for the tls_fingerprints. Reload it if we + # can, but it's not a fatal error if we can't. + try: + self.tls_certificate = self.read_tls_certificate() + except Exception as e: + logger.info( + "Unable to read TLS certificate (%s). Ignoring as no " + "tls listeners enabled.", e, + ) self.tls_fingerprints = list(self._original_tls_fingerprints) - # Check that our own certificate is included in the list of fingerprints - # and include it if it is not. - x509_certificate_bytes = crypto.dump_certificate( - crypto.FILETYPE_ASN1, self.tls_certificate - ) - sha256_fingerprint = encode_base64(sha256(x509_certificate_bytes).digest()) - sha256_fingerprints = set(f["sha256"] for f in self.tls_fingerprints) - if sha256_fingerprint not in sha256_fingerprints: - self.tls_fingerprints.append({u"sha256": sha256_fingerprint}) + if self.tls_certificate: + # Check that our own certificate is included in the list of fingerprints + # and include it if it is not. + x509_certificate_bytes = crypto.dump_certificate( + crypto.FILETYPE_ASN1, self.tls_certificate + ) + sha256_fingerprint = encode_base64(sha256(x509_certificate_bytes).digest()) + sha256_fingerprints = set(f["sha256"] for f in self.tls_fingerprints) + if sha256_fingerprint not in sha256_fingerprints: + self.tls_fingerprints.append({u"sha256": sha256_fingerprint}) def default_config(self, config_dir_path, server_name, **kwargs): base_key_name = os.path.join(config_dir_path, server_name) diff --git a/tests/config/test_tls.py b/tests/config/test_tls.py index d8fd18a9cb..c260d3359f 100644 --- a/tests/config/test_tls.py +++ b/tests/config/test_tls.py @@ -65,7 +65,7 @@ s4niecZKPBizL6aucT59CsunNmmb5Glq8rlAcU+1ZTZZzGYqVYhF6axB9Qg= t = TestConfig() t.read_config(config) - t.read_certificate_from_disk() + t.read_certificate_from_disk(require_cert_and_key=False) warnings = self.flushWarnings() self.assertEqual(len(warnings), 1) From a4ce91396bda0c6a6e3a2392355f8297cc97071b Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 12 Feb 2019 10:52:08 +0000 Subject: [PATCH 49/66] Disable TLS by default (#4614) --- changelog.d/4614.feature | 1 + synapse/config/server.py | 50 ++++++++++++++++++++-------------------- synapse/config/tls.py | 6 ++--- 3 files changed, 29 insertions(+), 28 deletions(-) create mode 100644 changelog.d/4614.feature diff --git a/changelog.d/4614.feature b/changelog.d/4614.feature new file mode 100644 index 0000000000..18e16dbc7b --- /dev/null +++ b/changelog.d/4614.feature @@ -0,0 +1 @@ +The default configuration no longer requires TLS certificates. diff --git a/synapse/config/server.py b/synapse/config/server.py index 767897c419..c5c3aac8ed 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -387,28 +387,24 @@ class ServerConfig(Config): # webclient: A web client. Requires web_client_location to be set. # listeners: - # Main HTTPS listener. - # For when matrix traffic is sent directly to synapse. - - port: %(bind_port)s - type: http - tls: true + # TLS-enabled listener: for when matrix traffic is sent directly to synapse. + # + # Disabled by default. To enable it, uncomment the following. (Note that you + # will also need to give Synapse a TLS key and certificate: see the TLS section + # below.) + # + # - port: %(bind_port)s + # type: http + # tls: true + # resources: + # - names: [client, federation] - # List of HTTP resources to serve on this listener. - resources: - - names: [client] - compress: true - - names: [federation] - compress: false - - # example addional_resources: - # - # additional_resources: - # "/_matrix/my/custom/endpoint": - # module: my_module.CustomRequestHandler - # config: {} - - # Unsecure HTTP listener - # For when matrix traffic passes through a reverse-proxy that unwraps TLS. + # Unsecure HTTP listener: for when matrix traffic passes through a reverse proxy + # that unwraps TLS. + # + # If you plan to use a reverse proxy, please see + # https://github.com/matrix-org/synapse/blob/master/docs/reverse_proxy.rst. + # - port: %(unsecure_port)s tls: false bind_addresses: ['::1', '127.0.0.1'] @@ -416,18 +412,22 @@ class ServerConfig(Config): x_forwarded: true resources: - - names: [client] - compress: true - - names: [federation] + - names: [client, federation] compress: false + # example additonal_resources: + # + # additional_resources: + # "/_matrix/my/custom/endpoint": + # module: my_module.CustomRequestHandler + # config: {} + # Turn on the twisted ssh manhole service on localhost on the given # port. # - port: 9000 # bind_addresses: ['::1', '127.0.0.1'] # type: manhole - # Homeserver blocking # # How to reach the server admin, used in ResourceLimitError diff --git a/synapse/config/tls.py b/synapse/config/tls.py index 57f117a14d..5fb3486db1 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -176,10 +176,10 @@ class TlsConfig(Config): # See 'ACME support' below to enable auto-provisioning this certificate via # Let's Encrypt. # - tls_certificate_path: "%(tls_certificate_path)s" + # tls_certificate_path: "%(tls_certificate_path)s" # PEM-encoded private key for TLS - tls_private_key_path: "%(tls_private_key_path)s" + # tls_private_key_path: "%(tls_private_key_path)s" # ACME support: This will configure Synapse to request a valid TLS certificate # for your configured `server_name` via Let's Encrypt. @@ -204,7 +204,7 @@ class TlsConfig(Config): # acme: # ACME support is disabled by default. Uncomment the following line - # to enable it. + # (and tls_certificate_path and tls_private_key_path above) to enable it. # # enabled: true From 2418b91bb7d64bfe572647565a6f1c80e82e1f5a Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 12 Feb 2019 10:53:28 +0000 Subject: [PATCH 50/66] README updates (#4621) Lots of updates to the README/INSTALL.md. Fixes #4601. --- INSTALL.md | 40 ++++++++++++------ README.rst | 98 +++++++++++++++++++++---------------------- changelog.d/4621.misc | 1 + 3 files changed, 78 insertions(+), 61 deletions(-) create mode 100644 changelog.d/4621.misc diff --git a/INSTALL.md b/INSTALL.md index e496a13b21..fb6a5e4e99 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -350,18 +350,34 @@ Once you have installed synapse as above, you will need to configure it. ## TLS certificates -The default configuration exposes two HTTP ports: 8008 and 8448. Port 8008 is -configured without TLS; it should be behind a reverse proxy for TLS/SSL -termination on port 443 which in turn should be used for clients. Port 8448 -is configured to use TLS for Federation with a self-signed or verified -certificate, but please be aware that a valid certificate will be required in -Synapse v1.0. Instructions for having Synapse automatically provision and renew federation certificates through ACME can be found at [ACME.md](docs/ACME.md). +The default configuration exposes a single HTTP port: http://localhost:8008. It +is suitable for local testing, but for any practical use, you will either need +to enable a reverse proxy, or configure Synapse to expose an HTTPS port. -If you would like to use your own certificates, you can do so by changing -`tls_certificate_path` and `tls_private_key_path` in `homeserver.yaml`; -alternatively, you can use a reverse proxy. See -[docs/reverse_proxy.rst](docs/reverse_proxy.rst) for information on configuring -a reverse proxy. +For information on using a reverse proxy, see +[docs/reverse_proxy.rst](docs/reverse_proxy.rst). + +To configure Synapse to expose an HTTPS port, you will need to edit +`homeserver.yaml`. + +First, under the `listeners` section, uncomment the configuration for the +TLS-enabled listener. (Remove the hash sign (`#`) and space at the start of +each line). The relevant lines are like this: + +``` + - port: 8448 + type: http + tls: true + resources: + - names: [client, federation] +``` + +You will also need to uncomment the `tls_certificate_path` and +`tls_private_key_path` lines under the `TLS` section. You can either point +these settings at an existing certificate and key, or you can enable Synapse's +built-in ACME (Let's Encrypt) support. Instructions for having Synapse +automatically provision and renew federation certificates through ACME can be +found at [ACME.md](docs/ACME.md). ## Registering a user @@ -375,7 +391,7 @@ users. This can be done as follows: ``` $ source ~/synapse/env/bin/activate $ synctl start # if not already running -$ register_new_matrix_user -c homeserver.yaml https://localhost:8448 +$ register_new_matrix_user -c homeserver.yaml http://localhost:8008 New user localpart: erikj Password: Confirm password: diff --git a/README.rst b/README.rst index bc7cb5f784..9a7c04b55e 100644 --- a/README.rst +++ b/README.rst @@ -26,7 +26,6 @@ via IRC bridge at irc://irc.freenode.net/matrix. Synapse is currently in rapid development, but as of version 0.5 we believe it is sufficiently stable to be run as an internet-facing service for real usage! - About Matrix ============ @@ -88,18 +87,20 @@ Connecting to Synapse from a client =================================== The easiest way to try out your new Synapse installation is by connecting to it -from a web client. The easiest option is probably the one at -https://riot.im/app. You will need to specify a "Custom server" when you log on -or register: set this to ``https://domain.tld`` if you setup a reverse proxy -following the recommended setup, or ``https://localhost:8448`` - remember to specify the -port (``:8448``) if not ``:443`` unless you changed the configuration. (Leave the identity -server as the default - see `Identity servers`_.) +from a web client. -If using port 8448 you will run into errors if you are using a self-signed -certificate. To overcome this, simply go to ``https://localhost:8448`` -directly with your browser and accept the presented certificate. You can then -go back in your web client and proceed further. Valid federation certificates -should not have this problem. +Unless you are running a test instance of Synapse on your local machine, in +general, you will need to enable TLS support before you can successfully +connect from a client: see ``_. + +An easy way to get started is to login or register via Riot at +https://riot.im/app/#/login or https://riot.im/app/#/register respectively. +You will need to change the server you are logging into from ``matrix.org`` +and instead specify a Homeserver URL of ``https://:8448`` +(or just ``https://`` if you are using a reverse proxy). +(Leave the identity server as the default - see `Identity servers`_.) +If you prefer to use another client, refer to our +`client breakdown `_. If all goes well you should at least be able to log in, create a room, and start sending messages. @@ -174,9 +175,30 @@ Separately, Synapse may leak file handles if inbound HTTP requests get stuck during processing - e.g. blocked behind a lock or talking to a remote server etc. This is best diagnosed by matching up the 'Received request' and 'Processed request' log lines and looking for any 'Processed request' lines which take more than -a few seconds to execute. Please let us know at #matrix-dev:matrix.org if +a few seconds to execute. Please let us know at #synapse:matrix.org if you see this failure mode so we can help debug it, however. +Help!! Synapse eats all my RAM! +------------------------------- + +Synapse's architecture is quite RAM hungry currently - we deliberately +cache a lot of recent room data and metadata in RAM in order to speed up +common requests. We'll improve this in future, but for now the easiest +way to either reduce the RAM usage (at the risk of slowing things down) +is to set the almost-undocumented ``SYNAPSE_CACHE_FACTOR`` environment +variable. The default is 0.5, which can be decreased to reduce RAM usage +in memory constrained enviroments, or increased if performance starts to +degrade. + +Using `libjemalloc `_ can also yield a significant +improvement in overall amount, and especially in terms of giving back RAM +to the OS. To use it, the library must simply be put in the LD_PRELOAD +environment variable when launching Synapse. On Debian, this can be done +by installing the ``libjemalloc1`` package and adding this line to +``/etc/default/matrix-synapse``:: + + LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1 + Upgrading an existing Synapse ============================= @@ -196,12 +218,12 @@ Federation is the process by which users on different servers can participate in the same room. For this to work, those other servers must be able to contact yours to send messages. -The ``server_name`` in your -``homeserver.yaml`` file determines the way that other servers will reach -yours. By default, they will treat it as a hostname and try to connect to -port 8448. This is easy to set up and will work with the default configuration, -provided you set the ``server_name`` to match your machine's public DNS -hostname. +The ``server_name`` in your ``homeserver.yaml`` file determines the way that +other servers will reach yours. By default, they will treat it as a hostname +and try to connect to port 8448. This is easy to set up and will work with the +default configuration, provided you set the ``server_name`` to match your +machine's public DNS hostname, and give Synapse a TLS certificate which is +valid for your ``server_name``. For a more flexible configuration, you can set up a DNS SRV record. This allows you to run your server on a machine that might not have the same name as your @@ -243,11 +265,8 @@ largest boxes pause for thought.) Troubleshooting --------------- -You can use the federation tester to check if your homeserver is all set: -``https://matrix.org/federationtester/api/report?server_name=`` -If any of the attributes under "checks" is false, federation won't work. -There is also a nicer interface available from a community member at -``_. +You can use the `federation tester `_ to +check if your homeserver is all set. The typical failure mode with federation is that when you try to join a room, it is rejected with "401: Unauthorized". Generally this means that other @@ -263,7 +282,10 @@ So, things to check are: (it should be ``_matrix._tcp.``), and that the port and hostname it specifies are reachable from outside your network. -.. TODO: add a note about forgetting ``nocanon`` on a reverse-proxy config +Another common problem is that people on other servers can't join rooms that +you invite them to. This can be caused by an incorrectly-configured reverse +proxy: see ``_ for instructions on how to correctly +configure a reverse proxy. Running a Demo Federation of Synapses ------------------------------------- @@ -363,7 +385,7 @@ Synapse Development Before setting up a development environment for synapse, make sure you have the system dependencies (such as the python header files) installed - see -`Installing from source`_. +`Installing from source `_. To check out a synapse for development, clone the git repo into a working directory of your choice:: @@ -374,7 +396,7 @@ directory of your choice:: Synapse has a number of external dependencies, that are easiest to install using pip and a virtualenv:: - virtualenv -p python2.7 env + virtualenv -p python3 env source env/bin/activate python -m pip install -e .[all] @@ -416,25 +438,3 @@ sphinxcontrib-napoleon:: Building internal API documentation:: python setup.py build_sphinx - - -Help!! Synapse eats all my RAM! -=============================== - -Synapse's architecture is quite RAM hungry currently - we deliberately -cache a lot of recent room data and metadata in RAM in order to speed up -common requests. We'll improve this in future, but for now the easiest -way to either reduce the RAM usage (at the risk of slowing things down) -is to set the almost-undocumented ``SYNAPSE_CACHE_FACTOR`` environment -variable. The default is 0.5, which can be decreased to reduce RAM usage -in memory constrained enviroments, or increased if performance starts to -degrade. - -Using `libjemalloc `_ can also yield a significant -improvement in overall amount, and especially in terms of giving back RAM -to the OS. To use it, the library must simply be put in the LD_PRELOAD -environment variable when launching Synapse. On Debian, this can be done -by installing the ``libjemalloc1`` package and adding this line to -``/etc/default/matrix-synapse``:: - - LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1 diff --git a/changelog.d/4621.misc b/changelog.d/4621.misc new file mode 100644 index 0000000000..60e45cb70c --- /dev/null +++ b/changelog.d/4621.misc @@ -0,0 +1 @@ +README updates From 362d80b7706c7bcd57ea5ee3e8e38b31c7d1ad8b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 12 Feb 2019 11:28:08 +0000 Subject: [PATCH 51/66] Reduce user_ips bloat during dedupe background update The background update to remove duplicate rows naively deleted and reinserted the duplicates. For large tables with a large number of duplicates this causes a lot of bloat (with postgres), as the inserted rows are appended to the table, since deleted rows will not be overwritten until a VACUUM has happened. This should hopefully also help ensure that the query in the last batch uses the correct index, as inserting a large number of new rows without analyzing will upset the query planner. --- synapse/storage/client_ips.py | 63 +++++++++++++++++++++++++++++++++-- 1 file changed, 60 insertions(+), 3 deletions(-) diff --git a/synapse/storage/client_ips.py b/synapse/storage/client_ips.py index 091d7116c5..a20cc8231f 100644 --- a/synapse/storage/client_ips.py +++ b/synapse/storage/client_ips.py @@ -167,12 +167,16 @@ class ClientIpStore(background_updates.BackgroundUpdateStore): clause = "? <= last_seen AND last_seen < ?" args = (begin_last_seen, end_last_seen) + # (Note: The DISTINCT in the inner query is important to ensure that + # the COUNT(*) is accurate, otherwise double counting may happen due + # to the join effectively being a cross product) txn.execute( """ SELECT user_id, access_token, ip, - MAX(device_id), MAX(user_agent), MAX(last_seen) + MAX(device_id), MAX(user_agent), MAX(last_seen), + COUNT(*) FROM ( - SELECT user_id, access_token, ip + SELECT DISTINCT user_id, access_token, ip FROM user_ips WHERE {} ) c @@ -186,7 +190,60 @@ class ClientIpStore(background_updates.BackgroundUpdateStore): # We've got some duplicates for i in res: - user_id, access_token, ip, device_id, user_agent, last_seen = i + user_id, access_token, ip, device_id, user_agent, last_seen, count = i + + # We want to delete the duplicates so we end up with only a + # single row. + # + # The naive way of doing this would be just to delete all rows + # and reinsert a constructed row. However, if there are a lot of + # duplicate rows this can cause the table to grow a lot, which + # can be problematic in two ways: + # 1. If user_ips is already large then this can cause the + # table to rapidly grow, potentially filling the disk. + # 2. Reinserting a lot of rows can confuse the table + # statistics for postgres, causing it to not use the + # correct indices for the query above, resulting in a full + # table scan. This is incredibly slow for large tables and + # can kill database performance. (This seems to mainly + # happen for the last query where the clause is simply `? < + # last_seen`) + # + # So instead we want to delete all but *one* of the duplicate + # rows. That is hard to do reliably, so we cheat and do a two + # step process: + # 1. Delete all rows with a last_seen strictly less than the + # max last_seen. This hopefully results in deleting all but + # one row the majority of the time, but there may be + # duplicate last_seen + # 2. If multiple rows remain, we fall back to the naive method + # and simply delete all rows and reinsert. + # + # Note that this relies on no new duplicate rows being inserted, + # but if that is happening then this entire process is futile + # anyway. + + # Do step 1: + + txn.execute( + """ + DELETE FROM user_ips + WHERE user_id = ? AND access_token = ? AND ip = ? AND last_seen < ? + """, + (user_id, access_token, ip, last_seen) + ) + if txn.rowcount == count - 1: + # We deleted all but one of the duplicate rows, i.e. there + # is exactly one remaining and so there is nothing left to + # do. + continue + elif txn.rowcount >= count: + raise Exception( + "We deleted more duplicate rows from 'user_ips' than expected", + ) + + # The previous step didn't delete enough rows, so we fallback to + # step 2: # Drop all the duplicates txn.execute( From 218cc071c5ef027361eddc72ef1863b43b2cdd67 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 12 Feb 2019 11:37:00 +0000 Subject: [PATCH 52/66] Newsfile --- changelog.d/4626.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/4626.misc diff --git a/changelog.d/4626.misc b/changelog.d/4626.misc new file mode 100644 index 0000000000..f1a57dcf9a --- /dev/null +++ b/changelog.d/4626.misc @@ -0,0 +1 @@ +Improve 'user_ips' table deduplication background update From 483ba85c7a1a8ee9b7eebcc5c07d522c71229c9f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 12 Feb 2019 11:55:27 +0000 Subject: [PATCH 53/66] Analyze user_ips before running deduplication Due to the table locks taken out by the naive upsert, the table statistics may be out of date. During deduplication it is important that the correct index is used as otherwise a full table scan may be incorrectly used, which can end up thrashing the database badly. --- synapse/storage/client_ips.py | 24 +++++++++++++++++++ .../schema/delta/53/user_ips_index.sql | 10 +++++--- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/synapse/storage/client_ips.py b/synapse/storage/client_ips.py index 091d7116c5..6f81406269 100644 --- a/synapse/storage/client_ips.py +++ b/synapse/storage/client_ips.py @@ -65,6 +65,11 @@ class ClientIpStore(background_updates.BackgroundUpdateStore): columns=["last_seen"], ) + self.register_background_update_handler( + "user_ips_analyze", + self._analyze_user_ip, + ) + self.register_background_update_handler( "user_ips_remove_dupes", self._remove_user_ip_dupes, @@ -108,6 +113,25 @@ class ClientIpStore(background_updates.BackgroundUpdateStore): yield self._end_background_update("user_ips_drop_nonunique_index") defer.returnValue(1) + @defer.inlineCallbacks + def _analyze_user_ip(self, progress, batch_size): + # Background update to analyze user_ips table before we run the + # deduplication background update. The table may not have been analyzed + # for ages due to the table locks. + # + # This will lock out the naive upserts to user_ips while it happens, but + # the analyze should be quick (28GB table takes ~10s) + def user_ips_analyze(txn): + txn.execute("ANALYZE user_ips") + + end_last_seen = yield self.runInteraction( + "user_ips_analyze", user_ips_analyze + ) + + yield self._end_background_update("user_ips_analyze") + + defer.returnValue(1) + @defer.inlineCallbacks def _remove_user_ip_dupes(self, progress, batch_size): # This works function works by scanning the user_ips table in batches diff --git a/synapse/storage/schema/delta/53/user_ips_index.sql b/synapse/storage/schema/delta/53/user_ips_index.sql index 4ca346c111..b812c5794f 100644 --- a/synapse/storage/schema/delta/53/user_ips_index.sql +++ b/synapse/storage/schema/delta/53/user_ips_index.sql @@ -13,9 +13,13 @@ * limitations under the License. */ --- delete duplicates + -- analyze user_ips, to help ensure the correct indices are used INSERT INTO background_updates (update_name, progress_json) VALUES - ('user_ips_remove_dupes', '{}'); + ('user_ips_analyze', '{}'); + +-- delete duplicates +INSERT INTO background_updates (update_name, progress_json, depends_on) VALUES + ('user_ips_remove_dupes', '{}', 'user_ips_analyze'); -- add a new unique index to user_ips table INSERT INTO background_updates (update_name, progress_json, depends_on) VALUES @@ -23,4 +27,4 @@ INSERT INTO background_updates (update_name, progress_json, depends_on) VALUES -- drop the old original index INSERT INTO background_updates (update_name, progress_json, depends_on) VALUES - ('user_ips_drop_nonunique_index', '{}', 'user_ips_device_unique_index'); \ No newline at end of file + ('user_ips_drop_nonunique_index', '{}', 'user_ips_device_unique_index'); From b2327eb9cb2c03037acd58b320063ec34968ea81 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 12 Feb 2019 11:58:36 +0000 Subject: [PATCH 54/66] Newsfile --- changelog.d/4627.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/4627.misc diff --git a/changelog.d/4627.misc b/changelog.d/4627.misc new file mode 100644 index 0000000000..f1a57dcf9a --- /dev/null +++ b/changelog.d/4627.misc @@ -0,0 +1 @@ +Improve 'user_ips' table deduplication background update From 495ea92350c4a3f6bd92cded5da939326b858d9b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 12 Feb 2019 12:40:42 +0000 Subject: [PATCH 55/66] Fix pep8 --- synapse/storage/client_ips.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/client_ips.py b/synapse/storage/client_ips.py index 6f81406269..cc23d7cdbe 100644 --- a/synapse/storage/client_ips.py +++ b/synapse/storage/client_ips.py @@ -124,7 +124,7 @@ class ClientIpStore(background_updates.BackgroundUpdateStore): def user_ips_analyze(txn): txn.execute("ANALYZE user_ips") - end_last_seen = yield self.runInteraction( + yield self.runInteraction( "user_ips_analyze", user_ips_analyze ) From b18cd25e42711d99c62a098203f59f4e12f3fca7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 12 Feb 2019 13:05:31 +0000 Subject: [PATCH 56/66] Fixup changelog entries --- changelog.d/4626.bugfix | 1 + changelog.d/4626.misc | 1 - changelog.d/4627.bugfix | 1 + changelog.d/4627.misc | 1 - 4 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 changelog.d/4626.bugfix delete mode 100644 changelog.d/4626.misc create mode 100644 changelog.d/4627.bugfix delete mode 100644 changelog.d/4627.misc diff --git a/changelog.d/4626.bugfix b/changelog.d/4626.bugfix new file mode 100644 index 0000000000..cc71df44ca --- /dev/null +++ b/changelog.d/4626.bugfix @@ -0,0 +1 @@ +Fix performance of 'user_ips' table deduplication background update diff --git a/changelog.d/4626.misc b/changelog.d/4626.misc deleted file mode 100644 index f1a57dcf9a..0000000000 --- a/changelog.d/4626.misc +++ /dev/null @@ -1 +0,0 @@ -Improve 'user_ips' table deduplication background update diff --git a/changelog.d/4627.bugfix b/changelog.d/4627.bugfix new file mode 100644 index 0000000000..cc71df44ca --- /dev/null +++ b/changelog.d/4627.bugfix @@ -0,0 +1 @@ +Fix performance of 'user_ips' table deduplication background update diff --git a/changelog.d/4627.misc b/changelog.d/4627.misc deleted file mode 100644 index f1a57dcf9a..0000000000 --- a/changelog.d/4627.misc +++ /dev/null @@ -1 +0,0 @@ -Improve 'user_ips' table deduplication background update From d2fa7b7e99a50b47b73de7e673fbe12a6cc12adc Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 12 Feb 2019 13:22:25 +0000 Subject: [PATCH 57/66] Update changelog and version --- CHANGES.md | 46 ++++++++++++++++++++++++++++++++++++++++ changelog.d/3902.feature | 1 - changelog.d/4420.feature | 1 - changelog.d/4462.misc | 1 - changelog.d/4513.misc | 1 - changelog.d/4522.feature | 1 - changelog.d/4530.bugfix | 1 - changelog.d/4546.bugfix | 1 - changelog.d/4567.misc | 1 - changelog.d/4576.misc | 1 - changelog.d/4578.misc | 1 - changelog.d/4580.feature | 1 - changelog.d/4584.misc | 1 - changelog.d/4586.misc | 1 - changelog.d/4589.bugfix | 1 - changelog.d/4591.bugfix | 1 - changelog.d/4592.feature | 2 -- changelog.d/4607.misc | 1 - changelog.d/4608.bugfix | 1 - changelog.d/4611.misc | 1 - changelog.d/4613.feature | 1 - changelog.d/4614.feature | 1 - changelog.d/4615.feature | 1 - changelog.d/4616.misc | 1 - changelog.d/4617.feature | 1 - changelog.d/4618.bugfix | 1 - changelog.d/4619.misc | 1 - changelog.d/4621.misc | 1 - changelog.d/4625.bugfix | 1 - changelog.d/4626.bugfix | 1 - changelog.d/4627.bugfix | 1 - synapse/__init__.py | 2 +- 32 files changed, 47 insertions(+), 32 deletions(-) delete mode 100644 changelog.d/3902.feature delete mode 100644 changelog.d/4420.feature delete mode 100644 changelog.d/4462.misc delete mode 100644 changelog.d/4513.misc delete mode 100644 changelog.d/4522.feature delete mode 100644 changelog.d/4530.bugfix delete mode 100644 changelog.d/4546.bugfix delete mode 100644 changelog.d/4567.misc delete mode 100644 changelog.d/4576.misc delete mode 100644 changelog.d/4578.misc delete mode 100644 changelog.d/4580.feature delete mode 100644 changelog.d/4584.misc delete mode 100644 changelog.d/4586.misc delete mode 100644 changelog.d/4589.bugfix delete mode 100644 changelog.d/4591.bugfix delete mode 100644 changelog.d/4592.feature delete mode 100644 changelog.d/4607.misc delete mode 100644 changelog.d/4608.bugfix delete mode 100644 changelog.d/4611.misc delete mode 100644 changelog.d/4613.feature delete mode 100644 changelog.d/4614.feature delete mode 100644 changelog.d/4615.feature delete mode 100644 changelog.d/4616.misc delete mode 100644 changelog.d/4617.feature delete mode 100644 changelog.d/4618.bugfix delete mode 100644 changelog.d/4619.misc delete mode 100644 changelog.d/4621.misc delete mode 100644 changelog.d/4625.bugfix delete mode 100644 changelog.d/4626.bugfix delete mode 100644 changelog.d/4627.bugfix diff --git a/CHANGES.md b/CHANGES.md index e330aea9e3..5120631fb6 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,49 @@ +Synapse 0.99.1rc1 (2019-02-12) +============================== + +Features +-------- + +- Include m.room.encryption on invites by default ([\#3902](https://github.com/matrix-org/synapse/issues/3902)) +- Federation OpenID listener resource can now be activated even if federation is disabled ([\#4420](https://github.com/matrix-org/synapse/issues/4420)) +- Synapse's ACME support will now correctly reprovision a certificate that approaches its expiry while Synapse is running. ([\#4522](https://github.com/matrix-org/synapse/issues/4522)) +- Add ability to update backup versions ([\#4580](https://github.com/matrix-org/synapse/issues/4580)) +- Allow the "unavailable" presence status for /sync. + This change makes Synapse compliant with r0.4.0 of the Client-Server specification. ([\#4592](https://github.com/matrix-org/synapse/issues/4592)) +- There is no longer any need to specify `no_tls`: it is inferred from the absence of TLS listeners ([\#4613](https://github.com/matrix-org/synapse/issues/4613), [\#4615](https://github.com/matrix-org/synapse/issues/4615), [\#4617](https://github.com/matrix-org/synapse/issues/4617)) +- The default configuration no longer requires TLS certificates. ([\#4614](https://github.com/matrix-org/synapse/issues/4614)) + + +Bugfixes +-------- + +- Copy over room federation ability on room upgrade. ([\#4530](https://github.com/matrix-org/synapse/issues/4530)) +- Fix noisy "twisted.internet.task.TaskStopped" errors in logs ([\#4546](https://github.com/matrix-org/synapse/issues/4546)) +- Synapse is now tolerant of the tls_fingerprints option being None or not specified. ([\#4589](https://github.com/matrix-org/synapse/issues/4589)) +- Fix 'no unique or exclusion constraint' error ([\#4591](https://github.com/matrix-org/synapse/issues/4591)) +- Transfer Server ACLs on room upgrade. ([\#4608](https://github.com/matrix-org/synapse/issues/4608)) +- Fix failure to start when not TLS certificate was given even if TLS was disabled. ([\#4618](https://github.com/matrix-org/synapse/issues/4618)) +- fix self-signed cert notice from generate-config. ([\#4625](https://github.com/matrix-org/synapse/issues/4625)) +- Fix performance of 'user_ips' table deduplication background update ([\#4626](https://github.com/matrix-org/synapse/issues/4626), [\#4627](https://github.com/matrix-org/synapse/issues/4627)) + + +Internal Changes +---------------- + +- Change the user directory state query to use a filtered call to the db instead of a generic one. ([\#4462](https://github.com/matrix-org/synapse/issues/4462)) +- Reject federation transactions if they include more than 50 PDUs or 100 EDUs. ([\#4513](https://github.com/matrix-org/synapse/issues/4513)) +- Reduce duplication of ``synapse.app`` code. ([\#4567](https://github.com/matrix-org/synapse/issues/4567)) +- Fix docker upload job to push -py2 images. ([\#4576](https://github.com/matrix-org/synapse/issues/4576)) +- Add port configuration information to ACME instructions. ([\#4578](https://github.com/matrix-org/synapse/issues/4578)) +- Update MSC1711 FAQ to calrify .well-known usage ([\#4584](https://github.com/matrix-org/synapse/issues/4584)) +- Clean up default listener configuration ([\#4586](https://github.com/matrix-org/synapse/issues/4586)) +- Clarifications for reverse proxy docs ([\#4607](https://github.com/matrix-org/synapse/issues/4607)) +- Move ClientTLSOptionsFactory init out of refresh_certificates ([\#4611](https://github.com/matrix-org/synapse/issues/4611)) +- Fail cleanly if listener config lacks a 'port' ([\#4616](https://github.com/matrix-org/synapse/issues/4616)) +- Remove redundant entries from docker config ([\#4619](https://github.com/matrix-org/synapse/issues/4619)) +- README updates ([\#4621](https://github.com/matrix-org/synapse/issues/4621)) + + Synapse 0.99.0 (2019-02-05) =========================== diff --git a/changelog.d/3902.feature b/changelog.d/3902.feature deleted file mode 100644 index eb8d9f2393..0000000000 --- a/changelog.d/3902.feature +++ /dev/null @@ -1 +0,0 @@ -Include m.room.encryption on invites by default diff --git a/changelog.d/4420.feature b/changelog.d/4420.feature deleted file mode 100644 index 05e777c624..0000000000 --- a/changelog.d/4420.feature +++ /dev/null @@ -1 +0,0 @@ -Federation OpenID listener resource can now be activated even if federation is disabled diff --git a/changelog.d/4462.misc b/changelog.d/4462.misc deleted file mode 100644 index 03a4d7ae1c..0000000000 --- a/changelog.d/4462.misc +++ /dev/null @@ -1 +0,0 @@ -Change the user directory state query to use a filtered call to the db instead of a generic one. \ No newline at end of file diff --git a/changelog.d/4513.misc b/changelog.d/4513.misc deleted file mode 100644 index 1f64a96465..0000000000 --- a/changelog.d/4513.misc +++ /dev/null @@ -1 +0,0 @@ -Reject federation transactions if they include more than 50 PDUs or 100 EDUs. \ No newline at end of file diff --git a/changelog.d/4522.feature b/changelog.d/4522.feature deleted file mode 100644 index ef18daf60b..0000000000 --- a/changelog.d/4522.feature +++ /dev/null @@ -1 +0,0 @@ -Synapse's ACME support will now correctly reprovision a certificate that approaches its expiry while Synapse is running. diff --git a/changelog.d/4530.bugfix b/changelog.d/4530.bugfix deleted file mode 100644 index d010af927e..0000000000 --- a/changelog.d/4530.bugfix +++ /dev/null @@ -1 +0,0 @@ -Copy over room federation ability on room upgrade. \ No newline at end of file diff --git a/changelog.d/4546.bugfix b/changelog.d/4546.bugfix deleted file mode 100644 index 056f2848ed..0000000000 --- a/changelog.d/4546.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix noisy "twisted.internet.task.TaskStopped" errors in logs diff --git a/changelog.d/4567.misc b/changelog.d/4567.misc deleted file mode 100644 index 96a2e0aefc..0000000000 --- a/changelog.d/4567.misc +++ /dev/null @@ -1 +0,0 @@ -Reduce duplication of ``synapse.app`` code. diff --git a/changelog.d/4576.misc b/changelog.d/4576.misc deleted file mode 100644 index 94b1ade2e3..0000000000 --- a/changelog.d/4576.misc +++ /dev/null @@ -1 +0,0 @@ -Fix docker upload job to push -py2 images. diff --git a/changelog.d/4578.misc b/changelog.d/4578.misc deleted file mode 100644 index d1c006bb6b..0000000000 --- a/changelog.d/4578.misc +++ /dev/null @@ -1 +0,0 @@ -Add port configuration information to ACME instructions. \ No newline at end of file diff --git a/changelog.d/4580.feature b/changelog.d/4580.feature deleted file mode 100644 index a2a5a77dbe..0000000000 --- a/changelog.d/4580.feature +++ /dev/null @@ -1 +0,0 @@ -Add ability to update backup versions \ No newline at end of file diff --git a/changelog.d/4584.misc b/changelog.d/4584.misc deleted file mode 100644 index 4dec2e2b5c..0000000000 --- a/changelog.d/4584.misc +++ /dev/null @@ -1 +0,0 @@ -Update MSC1711 FAQ to calrify .well-known usage diff --git a/changelog.d/4586.misc b/changelog.d/4586.misc deleted file mode 100644 index 37af371ccf..0000000000 --- a/changelog.d/4586.misc +++ /dev/null @@ -1 +0,0 @@ -Clean up default listener configuration diff --git a/changelog.d/4589.bugfix b/changelog.d/4589.bugfix deleted file mode 100644 index d5783f46e8..0000000000 --- a/changelog.d/4589.bugfix +++ /dev/null @@ -1 +0,0 @@ -Synapse is now tolerant of the tls_fingerprints option being None or not specified. diff --git a/changelog.d/4591.bugfix b/changelog.d/4591.bugfix deleted file mode 100644 index 628bbb6d81..0000000000 --- a/changelog.d/4591.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix 'no unique or exclusion constraint' error diff --git a/changelog.d/4592.feature b/changelog.d/4592.feature deleted file mode 100644 index 112005ded4..0000000000 --- a/changelog.d/4592.feature +++ /dev/null @@ -1,2 +0,0 @@ -Allow the "unavailable" presence status for /sync. -This change makes Synapse compliant with r0.4.0 of the Client-Server specification. diff --git a/changelog.d/4607.misc b/changelog.d/4607.misc deleted file mode 100644 index 160a824378..0000000000 --- a/changelog.d/4607.misc +++ /dev/null @@ -1 +0,0 @@ -Clarifications for reverse proxy docs diff --git a/changelog.d/4608.bugfix b/changelog.d/4608.bugfix deleted file mode 100644 index e331a362c4..0000000000 --- a/changelog.d/4608.bugfix +++ /dev/null @@ -1 +0,0 @@ -Transfer Server ACLs on room upgrade. \ No newline at end of file diff --git a/changelog.d/4611.misc b/changelog.d/4611.misc deleted file mode 100644 index d2e0a05daa..0000000000 --- a/changelog.d/4611.misc +++ /dev/null @@ -1 +0,0 @@ -Move ClientTLSOptionsFactory init out of refresh_certificates diff --git a/changelog.d/4613.feature b/changelog.d/4613.feature deleted file mode 100644 index 098f906af2..0000000000 --- a/changelog.d/4613.feature +++ /dev/null @@ -1 +0,0 @@ -There is no longer any need to specify `no_tls`: it is inferred from the absence of TLS listeners diff --git a/changelog.d/4614.feature b/changelog.d/4614.feature deleted file mode 100644 index 18e16dbc7b..0000000000 --- a/changelog.d/4614.feature +++ /dev/null @@ -1 +0,0 @@ -The default configuration no longer requires TLS certificates. diff --git a/changelog.d/4615.feature b/changelog.d/4615.feature deleted file mode 100644 index 098f906af2..0000000000 --- a/changelog.d/4615.feature +++ /dev/null @@ -1 +0,0 @@ -There is no longer any need to specify `no_tls`: it is inferred from the absence of TLS listeners diff --git a/changelog.d/4616.misc b/changelog.d/4616.misc deleted file mode 100644 index ee79e208e3..0000000000 --- a/changelog.d/4616.misc +++ /dev/null @@ -1 +0,0 @@ -Fail cleanly if listener config lacks a 'port' diff --git a/changelog.d/4617.feature b/changelog.d/4617.feature deleted file mode 100644 index 098f906af2..0000000000 --- a/changelog.d/4617.feature +++ /dev/null @@ -1 +0,0 @@ -There is no longer any need to specify `no_tls`: it is inferred from the absence of TLS listeners diff --git a/changelog.d/4618.bugfix b/changelog.d/4618.bugfix deleted file mode 100644 index 22115a020e..0000000000 --- a/changelog.d/4618.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix failure to start when not TLS certificate was given even if TLS was disabled. diff --git a/changelog.d/4619.misc b/changelog.d/4619.misc deleted file mode 100644 index 886fedf198..0000000000 --- a/changelog.d/4619.misc +++ /dev/null @@ -1 +0,0 @@ -Remove redundant entries from docker config diff --git a/changelog.d/4621.misc b/changelog.d/4621.misc deleted file mode 100644 index 60e45cb70c..0000000000 --- a/changelog.d/4621.misc +++ /dev/null @@ -1 +0,0 @@ -README updates diff --git a/changelog.d/4625.bugfix b/changelog.d/4625.bugfix deleted file mode 100644 index 3dc0ecf24c..0000000000 --- a/changelog.d/4625.bugfix +++ /dev/null @@ -1 +0,0 @@ -fix self-signed cert notice from generate-config. diff --git a/changelog.d/4626.bugfix b/changelog.d/4626.bugfix deleted file mode 100644 index cc71df44ca..0000000000 --- a/changelog.d/4626.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix performance of 'user_ips' table deduplication background update diff --git a/changelog.d/4627.bugfix b/changelog.d/4627.bugfix deleted file mode 100644 index cc71df44ca..0000000000 --- a/changelog.d/4627.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix performance of 'user_ips' table deduplication background update diff --git a/synapse/__init__.py b/synapse/__init__.py index 048d6e572f..c211cb4e6f 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -27,4 +27,4 @@ try: except ImportError: pass -__version__ = "0.99.0" +__version__ = "0.99.1rc1" From 19818d66af9d67731e19020efae3a244d276b2c6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 12 Feb 2019 13:25:05 +0000 Subject: [PATCH 58/66] Fixup changelog --- CHANGES.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 5120631fb6..cc629978c2 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -19,12 +19,12 @@ Bugfixes - Copy over room federation ability on room upgrade. ([\#4530](https://github.com/matrix-org/synapse/issues/4530)) - Fix noisy "twisted.internet.task.TaskStopped" errors in logs ([\#4546](https://github.com/matrix-org/synapse/issues/4546)) -- Synapse is now tolerant of the tls_fingerprints option being None or not specified. ([\#4589](https://github.com/matrix-org/synapse/issues/4589)) +- Synapse is now tolerant of the `tls_fingerprints` option being None or not specified. ([\#4589](https://github.com/matrix-org/synapse/issues/4589)) - Fix 'no unique or exclusion constraint' error ([\#4591](https://github.com/matrix-org/synapse/issues/4591)) - Transfer Server ACLs on room upgrade. ([\#4608](https://github.com/matrix-org/synapse/issues/4608)) - Fix failure to start when not TLS certificate was given even if TLS was disabled. ([\#4618](https://github.com/matrix-org/synapse/issues/4618)) -- fix self-signed cert notice from generate-config. ([\#4625](https://github.com/matrix-org/synapse/issues/4625)) -- Fix performance of 'user_ips' table deduplication background update ([\#4626](https://github.com/matrix-org/synapse/issues/4626), [\#4627](https://github.com/matrix-org/synapse/issues/4627)) +- Fix self-signed cert notice from generate-config. ([\#4625](https://github.com/matrix-org/synapse/issues/4625)) +- Fix performance of `user_ips` table deduplication background update ([\#4626](https://github.com/matrix-org/synapse/issues/4626), [\#4627](https://github.com/matrix-org/synapse/issues/4627)) Internal Changes @@ -38,7 +38,7 @@ Internal Changes - Update MSC1711 FAQ to calrify .well-known usage ([\#4584](https://github.com/matrix-org/synapse/issues/4584)) - Clean up default listener configuration ([\#4586](https://github.com/matrix-org/synapse/issues/4586)) - Clarifications for reverse proxy docs ([\#4607](https://github.com/matrix-org/synapse/issues/4607)) -- Move ClientTLSOptionsFactory init out of refresh_certificates ([\#4611](https://github.com/matrix-org/synapse/issues/4611)) +- Move ClientTLSOptionsFactory init out of `refresh_certificates` ([\#4611](https://github.com/matrix-org/synapse/issues/4611)) - Fail cleanly if listener config lacks a 'port' ([\#4616](https://github.com/matrix-org/synapse/issues/4616)) - Remove redundant entries from docker config ([\#4619](https://github.com/matrix-org/synapse/issues/4619)) - README updates ([\#4621](https://github.com/matrix-org/synapse/issues/4621)) From e3a0300431e6f641297cb90e936cb2d35fb51850 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 13 Feb 2019 11:48:56 +0000 Subject: [PATCH 59/66] Special-case the default bind_addresses for metrics listener turns out it doesn't really support ipv6, so let's hack around that by only listening on ipv4 by default. --- synapse/app/_base.py | 5 ++--- synapse/config/server.py | 6 +++++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 5b0ca312e2..1d2c3339ff 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -153,9 +153,8 @@ def listen_metrics(bind_addresses, port): from prometheus_client import start_http_server for host in bind_addresses: - reactor.callInThread(start_http_server, int(port), - addr=host, registry=RegistryProxy) - logger.info("Metrics now reporting on %s:%d", host, port) + logger.info("Starting metrics listener on %s:%d", host, port) + start_http_server(port, addr=host, registry=RegistryProxy) def listen_tcp(bind_addresses, port, factory, reactor=reactor, backlog=50): diff --git a/synapse/config/server.py b/synapse/config/server.py index c5c3aac8ed..93a30e4cfa 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -151,7 +151,11 @@ class ServerConfig(Config): # if we still have an empty list of addresses, use the default list if not bind_addresses: - bind_addresses.extend(DEFAULT_BIND_ADDRESSES) + if listener['type'] == 'metrics': + # the metrics listener doesn't support IPv6 + bind_addresses.append('0.0.0.0') + else: + bind_addresses.extend(DEFAULT_BIND_ADDRESSES) self.listeners.append(listener) From 2a5a15aff8f8d3146df52d5421a6c818637eb629 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 13 Feb 2019 11:53:43 +0000 Subject: [PATCH 60/66] Improve logging around listening services I wanted to bring listen_tcp into line with listen_ssl in terms of returning a list of ports, and wanted to check that was a safe thing to do - hence the logging in `refresh_certificate`. Also, pull the 'Synapse now listening' message up to homeserver.py, because it was being duplicated everywhere else. --- synapse/app/_base.py | 23 ++++++++++++++--------- synapse/app/homeserver.py | 8 ++++++-- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 1d2c3339ff..c53b644932 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -162,21 +162,23 @@ def listen_tcp(bind_addresses, port, factory, reactor=reactor, backlog=50): Create a TCP socket for a port and several addresses Returns: - list (empty) + list of twisted.internet.tcp.Port listening for TLS connections """ + r = [] for address in bind_addresses: try: - reactor.listenTCP( - port, - factory, - backlog, - address + r.append( + reactor.listenTCP( + port, + factory, + backlog, + address + ) ) except error.CannotListenError as e: check_bind_error(e, address, bind_addresses) - logger.info("Synapse now listening on TCP port %d", port) - return [] + return r def listen_ssl( @@ -203,7 +205,6 @@ def listen_ssl( except error.CannotListenError as e: check_bind_error(e, address, bind_addresses) - logger.info("Synapse now listening on port %d (TLS)", port) return r @@ -229,6 +230,10 @@ def refresh_certificate(hs): # requests. This factory attribute is public but missing from # Twisted's documentation. if isinstance(i.factory, TLSMemoryBIOFactory): + addr = i.getHost() + logger.info( + "Replacing TLS context factory on [%s]:%i", addr.host, addr.port, + ) # We want to replace TLS factories with a new one, with the new # TLS configuration. We do this by reaching in and pulling out # the wrappedFactory, and then re-wrapping it. diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index dbd98d394f..2b008e3402 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -121,7 +121,7 @@ class SynapseHomeServer(HomeServer): root_resource = create_resource_tree(resources, root_resource) if tls: - return listen_ssl( + ports = listen_ssl( bind_addresses, port, SynapseSite( @@ -134,9 +134,10 @@ class SynapseHomeServer(HomeServer): self.tls_server_context_factory, reactor=self.get_reactor(), ) + logger.info("Synapse now listening on TCP port %d (TLS)", port) else: - return listen_tcp( + ports = listen_tcp( bind_addresses, port, SynapseSite( @@ -148,6 +149,9 @@ class SynapseHomeServer(HomeServer): ), reactor=self.get_reactor(), ) + logger.info("Synapse now listening on TCP port %d", port) + + return ports def _configure_named_resource(self, name, compress=False): """Build a resource map for a named resource From 767686af48ecbf5fe999830e0b0ce13b7fd46b1b Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 13 Feb 2019 11:59:04 +0000 Subject: [PATCH 61/66] Use `listen_tcp` for the replication listener Fixes the "can't listen on 0.0.0.0" error. Also makes it more consistent with what we do elsewhere. --- synapse/app/homeserver.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 2b008e3402..dbd9a83877 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -266,14 +266,14 @@ class SynapseHomeServer(HomeServer): ) ) elif listener["type"] == "replication": - bind_addresses = listener["bind_addresses"] - for address in bind_addresses: - factory = ReplicationStreamProtocolFactory(self) - server_listener = reactor.listenTCP( - listener["port"], factory, interface=address - ) + services = listen_tcp( + listener["bind_addresses"], + listener["port"], + ReplicationStreamProtocolFactory(self), + ) + for s in services: reactor.addSystemEventTrigger( - "before", "shutdown", server_listener.stopListening, + "before", "shutdown", s.stopListening, ) elif listener["type"] == "metrics": if not self.get_config().enable_metrics: From 2d0e0a40441f299f06033c20d7ae801100eb0028 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 13 Feb 2019 12:00:34 +0000 Subject: [PATCH 62/66] changelog --- changelog.d/4636.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/4636.bugfix diff --git a/changelog.d/4636.bugfix b/changelog.d/4636.bugfix new file mode 100644 index 0000000000..7607aa1d53 --- /dev/null +++ b/changelog.d/4636.bugfix @@ -0,0 +1 @@ +Fix errors when using default bind_addresses with replication/metrics listeners. \ No newline at end of file From 309f3bb322ea407b29651bd31c12183ea81b05b1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 13 Feb 2019 13:24:27 +0000 Subject: [PATCH 63/66] Update synapse/app/_base.py Co-Authored-By: richvdh <1389908+richvdh@users.noreply.github.com> --- synapse/app/_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/app/_base.py b/synapse/app/_base.py index c53b644932..73ca52bd8c 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -162,7 +162,7 @@ def listen_tcp(bind_addresses, port, factory, reactor=reactor, backlog=50): Create a TCP socket for a port and several addresses Returns: - list of twisted.internet.tcp.Port listening for TLS connections + list[twisted.internet.tcp.Port]: listening for TCP connections """ r = [] for address in bind_addresses: From 3bc238629eb61e6a3f6c652e3ddae3261179d624 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 13 Feb 2019 14:46:18 +0000 Subject: [PATCH 64/66] 0.99.1rc2 --- CHANGES.md | 9 +++++++++ changelog.d/4636.bugfix | 1 - synapse/__init__.py | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/4636.bugfix diff --git a/CHANGES.md b/CHANGES.md index cc629978c2..d0b3437b2d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,12 @@ +Synapse 0.99.1rc2 (2019-02-13) +============================== + +Bugfixes +-------- + +- Fix errors when using default bind_addresses with replication/metrics listeners. ([\#4636](https://github.com/matrix-org/synapse/issues/4636)) + + Synapse 0.99.1rc1 (2019-02-12) ============================== diff --git a/changelog.d/4636.bugfix b/changelog.d/4636.bugfix deleted file mode 100644 index 7607aa1d53..0000000000 --- a/changelog.d/4636.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix errors when using default bind_addresses with replication/metrics listeners. \ No newline at end of file diff --git a/synapse/__init__.py b/synapse/__init__.py index c211cb4e6f..4629fa76ec 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -27,4 +27,4 @@ try: except ImportError: pass -__version__ = "0.99.1rc1" +__version__ = "0.99.1rc2" From a214ba93e0770f87c1a13a62375a11fd6c60bf64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=94=D0=B0=D0=BC=D1=98=D0=B0=D0=BD=20=D0=93=D0=B5=D0=BE?= =?UTF-8?q?=D1=80=D0=B3=D0=B8=D0=B5=D0=B2=D1=81=D0=BA=D0=B8?= Date: Thu, 14 Feb 2019 14:44:22 +0100 Subject: [PATCH 65/66] implement `reload` by sending the HUP signal (#4622) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * implement `reload` by sending the HUP signal According to the 0.99 release info* synapse now uses the HUP signal to reload certificates: > Synapse will now reload TLS certificates from disk upon SIGHUP. (#4495, #4524) So the matrix-synapse.service unit file should include a reload directive. Signed-off-by: Дамјан Георгиевски --- debian/changelog | 6 ++++++ debian/matrix-synapse.service | 1 + 2 files changed, 7 insertions(+) diff --git a/debian/changelog b/debian/changelog index 04b5d69053..d4b9ed8ed7 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (0.99.1) UNRELEASED; urgency=medium + + * Added ExecReload= in service unit file to send a HUP signal + + -- Damjan Georgievski Thu, 14 Feb 2019 12:53:13 +0000 + matrix-synapse-py3 (0.99.0) stable; urgency=medium * New synapse release 0.99.0 diff --git a/debian/matrix-synapse.service b/debian/matrix-synapse.service index 2e9cd83b5f..942e4b83fe 100644 --- a/debian/matrix-synapse.service +++ b/debian/matrix-synapse.service @@ -8,6 +8,7 @@ WorkingDirectory=/var/lib/matrix-synapse EnvironmentFile=/etc/default/matrix-synapse ExecStartPre=/opt/venvs/matrix-synapse/bin/python -m synapse.app.homeserver --config-path=/etc/matrix-synapse/homeserver.yaml --config-path=/etc/matrix-synapse/conf.d/ --generate-keys ExecStart=/opt/venvs/matrix-synapse/bin/python -m synapse.app.homeserver --config-path=/etc/matrix-synapse/homeserver.yaml --config-path=/etc/matrix-synapse/conf.d/ +ExecReload=/bin/kill -HUP $MAINPID Restart=always RestartSec=3 From 06cd757ae718ca4ec01c8e6480547f48f9410cfa Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 14 Feb 2019 14:24:24 +0000 Subject: [PATCH 66/66] 0.99.1 --- CHANGES.md | 15 +++------------ debian/changelog | 8 ++++++-- synapse/__init__.py | 4 ++-- 3 files changed, 11 insertions(+), 16 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index d0b3437b2d..872ac440b5 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,14 +1,5 @@ -Synapse 0.99.1rc2 (2019-02-13) -============================== - -Bugfixes --------- - -- Fix errors when using default bind_addresses with replication/metrics listeners. ([\#4636](https://github.com/matrix-org/synapse/issues/4636)) - - -Synapse 0.99.1rc1 (2019-02-12) -============================== +Synapse 0.99.1 (2019-02-14) +=========================== Features -------- @@ -19,7 +10,7 @@ Features - Add ability to update backup versions ([\#4580](https://github.com/matrix-org/synapse/issues/4580)) - Allow the "unavailable" presence status for /sync. This change makes Synapse compliant with r0.4.0 of the Client-Server specification. ([\#4592](https://github.com/matrix-org/synapse/issues/4592)) -- There is no longer any need to specify `no_tls`: it is inferred from the absence of TLS listeners ([\#4613](https://github.com/matrix-org/synapse/issues/4613), [\#4615](https://github.com/matrix-org/synapse/issues/4615), [\#4617](https://github.com/matrix-org/synapse/issues/4617)) +- There is no longer any need to specify `no_tls`: it is inferred from the absence of TLS listeners ([\#4613](https://github.com/matrix-org/synapse/issues/4613), [\#4615](https://github.com/matrix-org/synapse/issues/4615), [\#4617](https://github.com/matrix-org/synapse/issues/4617), [\#4636](https://github.com/matrix-org/synapse/issues/4636)) - The default configuration no longer requires TLS certificates. ([\#4614](https://github.com/matrix-org/synapse/issues/4614)) diff --git a/debian/changelog b/debian/changelog index d4b9ed8ed7..bc1ba153e3 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,8 +1,12 @@ -matrix-synapse-py3 (0.99.1) UNRELEASED; urgency=medium +matrix-synapse-py3 (0.99.1) stable; urgency=medium + [ Damjan Georgievski ] * Added ExecReload= in service unit file to send a HUP signal - -- Damjan Georgievski Thu, 14 Feb 2019 12:53:13 +0000 + [ Synapse Packaging team ] + * New synapse release 0.99.1 + + -- Synapse Packaging team Thu, 14 Feb 2019 14:12:26 +0000 matrix-synapse-py3 (0.99.0) stable; urgency=medium diff --git a/synapse/__init__.py b/synapse/__init__.py index 4629fa76ec..f7bac0ea4e 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # Copyright 2014-2016 OpenMarket Ltd -# Copyright 2018 New Vector Ltd +# Copyright 2018-9 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -27,4 +27,4 @@ try: except ImportError: pass -__version__ = "0.99.1rc2" +__version__ = "0.99.1"