From 7d5b1a60a3aa9f10e285e74295f407d19a3a9ad5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 16 Aug 2018 15:32:20 +0100 Subject: [PATCH 1/9] Fix inbound federation on reader worker Inbound federation requires calculating push, which in turn relies on having access to account data. --- synapse/app/federation_reader.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/synapse/app/federation_reader.py b/synapse/app/federation_reader.py index 52522e9d33..7d8105778d 100644 --- a/synapse/app/federation_reader.py +++ b/synapse/app/federation_reader.py @@ -32,6 +32,7 @@ from synapse.http.site import SynapseSite from synapse.metrics import RegistryProxy from synapse.metrics.resource import METRICS_PREFIX, MetricsResource from synapse.replication.slave.storage._base import BaseSlavedStore +from synapse.replication.slave.storage.account_data import SlavedAccountDataStore from synapse.replication.slave.storage.appservice import SlavedApplicationServiceStore from synapse.replication.slave.storage.directory import DirectoryStore from synapse.replication.slave.storage.events import SlavedEventStore @@ -54,6 +55,7 @@ logger = logging.getLogger("synapse.app.federation_reader") class FederationReaderSlavedStore( + SlavedAccountDataStore, SlavedProfileStore, SlavedApplicationServiceStore, SlavedPusherStore, From aae86a81eff9e5099efc97078ce35f26332f0ff4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 16 Aug 2018 15:41:48 +0100 Subject: [PATCH 2/9] Newsfile --- changelog.d/3705.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/3705.bugfix diff --git a/changelog.d/3705.bugfix b/changelog.d/3705.bugfix new file mode 100644 index 0000000000..6c5422994f --- /dev/null +++ b/changelog.d/3705.bugfix @@ -0,0 +1 @@ +Support more federation endpoints on workers From 7edd11623dac8952d6af04b49cc1390cf579cb68 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Thu, 16 Aug 2018 21:04:30 +0100 Subject: [PATCH 3/9] add new error type ResourceLimit --- changelog.d/3707.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/3707.misc diff --git a/changelog.d/3707.misc b/changelog.d/3707.misc new file mode 100644 index 0000000000..8123ca6543 --- /dev/null +++ b/changelog.d/3707.misc @@ -0,0 +1 @@ +add new error type ResourceLimit From 372bf073c1a5ac5e66cca717ce9aa72c43ba9404 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Thu, 16 Aug 2018 21:25:16 +0100 Subject: [PATCH 4/9] block event creation and room creation on hitting resource limits --- synapse/handlers/message.py | 6 +++++- synapse/handlers/room.py | 4 ++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 893c9bcdc4..4d006df63c 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -276,10 +276,14 @@ class EventCreationHandler(object): where *hashes* is a map from algorithm to hash. If None, they will be requested from the database. - + Raises: + ResourceLimitError if server is blocked to some resource being + exceeded Returns: Tuple of created event (FrozenEvent), Context """ + yield self.auth.check_auth_blocking(requester.user.to_string()) + builder = self.event_builder_factory.new(event_dict) self.validator.validate_new(builder) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 6a17c42238..c3f820b975 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -98,9 +98,13 @@ class RoomCreationHandler(BaseHandler): Raises: SynapseError if the room ID couldn't be stored, or something went horribly wrong. + ResourceLimitError if server is blocked to some resource being + exceeded """ user_id = requester.user.to_string() + self.auth.check_auth_blocking(user_id) + if not self.spam_checker.user_may_create_room(user_id): raise SynapseError(403, "You are not permitted to create rooms") From bcfeb44afe750dadd4199e7c02302b0157bdab11 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Thu, 16 Aug 2018 22:55:32 +0100 Subject: [PATCH 5/9] call reap on start up and fix under reaping bug --- synapse/api/auth.py | 2 +- synapse/app/homeserver.py | 1 + synapse/storage/monthly_active_users.py | 5 ++++- tests/storage/test_monthly_active_users.py | 13 +++++++++++++ 4 files changed, 19 insertions(+), 2 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 3b2a2ab77a..ab1e3a4e35 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -799,7 +799,7 @@ class Auth(object): current_mau = yield self.store.get_monthly_active_count() if current_mau >= self.hs.config.max_mau_value: raise AuthError( - 403, "Monthly Active User Limits AU Limit Exceeded", + 403, "Monthly Active User Limit Exceeded", admin_uri=self.hs.config.admin_uri, errcode=Codes.RESOURCE_LIMIT_EXCEED ) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index a98bb506e5..800b9c0e3c 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -525,6 +525,7 @@ def run(hs): clock.looping_call( hs.get_datastore().reap_monthly_active_users, 1000 * 60 * 60 ) + yield hs.get_datastore().reap_monthly_active_users() @defer.inlineCallbacks def generate_monthly_active_users(): diff --git a/synapse/storage/monthly_active_users.py b/synapse/storage/monthly_active_users.py index 7e417f811e..06f9a75a97 100644 --- a/synapse/storage/monthly_active_users.py +++ b/synapse/storage/monthly_active_users.py @@ -96,7 +96,10 @@ class MonthlyActiveUsersStore(SQLBaseStore): # While Postgres does not require 'LIMIT', but also does not support # negative LIMIT values. So there is no way to write it that both can # support - query_args = [self.hs.config.max_mau_value] + safe_guard = self.hs.config.max_mau_value - len(self.reserved_users) + # Must be greater than zero for postgres + safe_guard = safe_guard if safe_guard > 0 else 0 + query_args = [safe_guard] base_sql = """ DELETE FROM monthly_active_users diff --git a/tests/storage/test_monthly_active_users.py b/tests/storage/test_monthly_active_users.py index 511acbde9b..f2ed866ae7 100644 --- a/tests/storage/test_monthly_active_users.py +++ b/tests/storage/test_monthly_active_users.py @@ -75,6 +75,19 @@ class MonthlyActiveUsersTestCase(tests.unittest.TestCase): active_count = yield self.store.get_monthly_active_count() self.assertEquals(active_count, user_num) + # Test that regalar users are removed from the db + ru_count = 2 + yield self.store.upsert_monthly_active_user("@ru1:server") + yield self.store.upsert_monthly_active_user("@ru2:server") + active_count = yield self.store.get_monthly_active_count() + + self.assertEqual(active_count, user_num + ru_count) + self.hs.config.max_mau_value = user_num + yield self.store.reap_monthly_active_users() + + active_count = yield self.store.get_monthly_active_count() + self.assertEquals(active_count, user_num) + @defer.inlineCallbacks def test_can_insert_and_count_mau(self): count = yield self.store.get_monthly_active_count() From 7e513421969738428c40d9d6032c798edd9ebab6 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Thu, 16 Aug 2018 23:05:20 +0100 Subject: [PATCH 6/9] For resource limit blocked users, prevent writing into rooms --- changelog.d/3708.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/3708.feature diff --git a/changelog.d/3708.feature b/changelog.d/3708.feature new file mode 100644 index 0000000000..2f146ba62b --- /dev/null +++ b/changelog.d/3708.feature @@ -0,0 +1 @@ +For resource limit blocked users, prevent writing into rooms From a8ffc27db76b9805d7c91e7256c278306ab54761 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 17 Aug 2018 02:46:25 -0600 Subject: [PATCH 7/9] Update the admin register documentation to return a real user ID Presumably this is the intention anyways. I've also updated the domain part to be something more along the lines of what people might expect. --- docs/admin_api/register_api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/admin_api/register_api.rst b/docs/admin_api/register_api.rst index 209cd140fd..16d65c86b3 100644 --- a/docs/admin_api/register_api.rst +++ b/docs/admin_api/register_api.rst @@ -33,7 +33,7 @@ As an example:: < { "access_token": "token_here", - "user_id": "@pepper_roni@test", + "user_id": "@pepper_roni:localhost", "home_server": "test", "device_id": "device_id_here" } From b99a0f39415e42aeebb1dbc0e860324bb8f5b37e Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 17 Aug 2018 02:47:31 -0600 Subject: [PATCH 8/9] Create 3712.misc --- changelog.d/3712.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/3712.misc diff --git a/changelog.d/3712.misc b/changelog.d/3712.misc new file mode 100644 index 0000000000..30f8c2af21 --- /dev/null +++ b/changelog.d/3712.misc @@ -0,0 +1 @@ +Update admin register API documentation to reference a real user ID. From 521d369e7a6d6497427a03f3e4f81d20bd2e5761 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Fri, 17 Aug 2018 10:12:11 +0100 Subject: [PATCH 9/9] remove errant yield --- synapse/app/homeserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 800b9c0e3c..005921dcf7 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -525,7 +525,7 @@ def run(hs): clock.looping_call( hs.get_datastore().reap_monthly_active_users, 1000 * 60 * 60 ) - yield hs.get_datastore().reap_monthly_active_users() + hs.get_datastore().reap_monthly_active_users() @defer.inlineCallbacks def generate_monthly_active_users():