From 9db4c1b1758c0d6092161c67f9e3a3cf4ff05bdb Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Fri, 18 Sep 2020 07:56:20 -0400 Subject: [PATCH 1/7] Add flags to /versions about whether new rooms are encrypted by default. (#8343) --- changelog.d/8343.feature | 1 + synapse/rest/client/versions.py | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 changelog.d/8343.feature diff --git a/changelog.d/8343.feature b/changelog.d/8343.feature new file mode 100644 index 0000000000..ccecb22f37 --- /dev/null +++ b/changelog.d/8343.feature @@ -0,0 +1 @@ +Add flags to the `/versions` endpoint that includes whether new rooms default to using E2EE. diff --git a/synapse/rest/client/versions.py b/synapse/rest/client/versions.py index 24ac57f35d..c560edbc59 100644 --- a/synapse/rest/client/versions.py +++ b/synapse/rest/client/versions.py @@ -19,6 +19,7 @@ import logging import re +from synapse.api.constants import RoomCreationPreset from synapse.http.servlet import RestServlet logger = logging.getLogger(__name__) @@ -31,6 +32,20 @@ class VersionsRestServlet(RestServlet): super(VersionsRestServlet, self).__init__() self.config = hs.config + # Calculate these once since they shouldn't change after start-up. + self.e2ee_forced_public = ( + RoomCreationPreset.PUBLIC_CHAT + in self.config.encryption_enabled_by_default_for_room_presets + ) + self.e2ee_forced_private = ( + RoomCreationPreset.PRIVATE_CHAT + in self.config.encryption_enabled_by_default_for_room_presets + ) + self.e2ee_forced_trusted_private = ( + RoomCreationPreset.TRUSTED_PRIVATE_CHAT + in self.config.encryption_enabled_by_default_for_room_presets + ) + def on_GET(self, request): return ( 200, @@ -62,6 +77,10 @@ class VersionsRestServlet(RestServlet): "org.matrix.msc2432": True, # Implements additional endpoints as described in MSC2666 "uk.half-shot.msc2666": True, + # Whether new rooms will be set to encrypted or not (based on presets). + "io.element.e2ee_forced.public": self.e2ee_forced_public, + "io.element.e2ee_forced.private": self.e2ee_forced_private, + "io.element.e2ee_forced.trusted_private": self.e2ee_forced_trusted_private, }, }, ) From 43f2b67e4d2ce95b3b13d88e755afc7e3907e82b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 18 Sep 2020 14:25:52 +0100 Subject: [PATCH 2/7] Intelligently select extremities used in backfill. (#8349) Instead of just using the most recent extremities let's pick the ones that will give us results that the pagination request cares about, i.e. pick extremities only if they have a smaller depth than the pagination token. This is useful when we fail to backfill an extremity, as we no longer get stuck requesting that same extremity repeatedly. --- changelog.d/8349.bugfix | 1 + synapse/handlers/federation.py | 65 +++++++++++++++++++++--- synapse/handlers/pagination.py | 8 +-- synapse/storage/databases/main/stream.py | 13 ++--- 4 files changed, 67 insertions(+), 20 deletions(-) create mode 100644 changelog.d/8349.bugfix diff --git a/changelog.d/8349.bugfix b/changelog.d/8349.bugfix new file mode 100644 index 0000000000..cf2f531b14 --- /dev/null +++ b/changelog.d/8349.bugfix @@ -0,0 +1 @@ +Fix a longstanding bug where back pagination over federation could get stuck if it failed to handle a received event. diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 43f2986f89..014dab2940 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -943,15 +943,26 @@ class FederationHandler(BaseHandler): return events - async def maybe_backfill(self, room_id, current_depth): + async def maybe_backfill( + self, room_id: str, current_depth: int, limit: int + ) -> bool: """Checks the database to see if we should backfill before paginating, and if so do. + + Args: + room_id + current_depth: The depth from which we're paginating from. This is + used to decide if we should backfill and what extremities to + use. + limit: The number of events that the pagination request will + return. This is used as part of the heuristic to decide if we + should back paginate. """ extremities = await self.store.get_oldest_events_with_depth_in_room(room_id) if not extremities: logger.debug("Not backfilling as no extremeties found.") - return + return False # We only want to paginate if we can actually see the events we'll get, # as otherwise we'll just spend a lot of resources to get redacted @@ -1004,16 +1015,54 @@ class FederationHandler(BaseHandler): sorted_extremeties_tuple = sorted(extremities.items(), key=lambda e: -int(e[1])) max_depth = sorted_extremeties_tuple[0][1] + # If we're approaching an extremity we trigger a backfill, otherwise we + # no-op. + # + # We chose twice the limit here as then clients paginating backwards + # will send pagination requests that trigger backfill at least twice + # using the most recent extremity before it gets removed (see below). We + # chose more than one times the limit in case of failure, but choosing a + # much larger factor will result in triggering a backfill request much + # earlier than necessary. + if current_depth - 2 * limit > max_depth: + logger.debug( + "Not backfilling as we don't need to. %d < %d - 2 * %d", + max_depth, + current_depth, + limit, + ) + return False + + logger.debug( + "room_id: %s, backfill: current_depth: %s, max_depth: %s, extrems: %s", + room_id, + current_depth, + max_depth, + sorted_extremeties_tuple, + ) + + # We ignore extremities that have a greater depth than our current depth + # as: + # 1. we don't really care about getting events that have happened + # before our current position; and + # 2. we have likely previously tried and failed to backfill from that + # extremity, so to avoid getting "stuck" requesting the same + # backfill repeatedly we drop those extremities. + filtered_sorted_extremeties_tuple = [ + t for t in sorted_extremeties_tuple if int(t[1]) <= current_depth + ] + + # However, we need to check that the filtered extremities are non-empty. + # If they are empty then either we can a) bail or b) still attempt to + # backill. We opt to try backfilling anyway just in case we do get + # relevant events. + if filtered_sorted_extremeties_tuple: + sorted_extremeties_tuple = filtered_sorted_extremeties_tuple + # We don't want to specify too many extremities as it causes the backfill # request URI to be too long. extremities = dict(sorted_extremeties_tuple[:5]) - if current_depth > max_depth: - logger.debug( - "Not backfilling as we don't need to. %d < %d", max_depth, current_depth - ) - return - # Now we need to decide which hosts to hit first. # First we try hosts that are already in the room diff --git a/synapse/handlers/pagination.py b/synapse/handlers/pagination.py index 34ed0e2921..6067585f9b 100644 --- a/synapse/handlers/pagination.py +++ b/synapse/handlers/pagination.py @@ -362,9 +362,9 @@ class PaginationHandler: # if we're going backwards, we might need to backfill. This # requires that we have a topo token. if room_token.topological: - max_topo = room_token.topological + curr_topo = room_token.topological else: - max_topo = await self.store.get_max_topological_token( + curr_topo = await self.store.get_current_topological_token( room_id, room_token.stream ) @@ -380,11 +380,11 @@ class PaginationHandler: leave_token = await self.store.get_topological_token_for_event( member_event_id ) - if RoomStreamToken.parse(leave_token).topological < max_topo: + if RoomStreamToken.parse(leave_token).topological < curr_topo: source_config.from_key = str(leave_token) await self.hs.get_handlers().federation_handler.maybe_backfill( - room_id, max_topo + room_id, curr_topo, limit=source_config.limit, ) events, next_key = await self.store.paginate_room_events( diff --git a/synapse/storage/databases/main/stream.py b/synapse/storage/databases/main/stream.py index be6df8a6d1..db20a3db30 100644 --- a/synapse/storage/databases/main/stream.py +++ b/synapse/storage/databases/main/stream.py @@ -648,23 +648,20 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): ) return "t%d-%d" % (row["topological_ordering"], row["stream_ordering"]) - async def get_max_topological_token(self, room_id: str, stream_key: int) -> int: - """Get the max topological token in a room before the given stream + async def get_current_topological_token(self, room_id: str, stream_key: int) -> int: + """Gets the topological token in a room after or at the given stream ordering. Args: room_id stream_key - - Returns: - The maximum topological token. """ sql = ( - "SELECT coalesce(max(topological_ordering), 0) FROM events" - " WHERE room_id = ? AND stream_ordering < ?" + "SELECT coalesce(MIN(topological_ordering), 0) FROM events" + " WHERE room_id = ? AND stream_ordering >= ?" ) row = await self.db_pool.execute( - "get_max_topological_token", None, sql, room_id, stream_key + "get_current_topological_token", None, sql, room_id, stream_key ) return row[0][0] if row else 0 From 27c1abc7b876ef3e73c8b418a89b987f86147829 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Fri, 18 Sep 2020 14:51:11 +0100 Subject: [PATCH 3/7] Use _check_sigs_and_hash_and_fetch to validate backfill requests (#8350) This is a bit of a hack, as `_check_sigs_and_hash_and_fetch` is intended for attempting to pull an event from the database/(re)pull it from the server that originally sent the event if checking the signature of the event fails. During backfill we *know* that we won't have the event in our database, however it is still useful to be able to query the original sending server as the server we're backfilling from may be acting maliciously. The main benefit and reason for this change however is that `_check_sigs_and_hash_and_fetch` will drop an event during backfill if it cannot be successfully validated, whereas the current code will simply fail the backfill request - resulting in the client's /messages request silently being dropped. This is a quick patch to fix backfilling rooms that contain malformed events. A better implementation in planned in future. --- changelog.d/8350.bugfix | 1 + synapse/federation/federation_client.py | 8 +++----- 2 files changed, 4 insertions(+), 5 deletions(-) create mode 100644 changelog.d/8350.bugfix diff --git a/changelog.d/8350.bugfix b/changelog.d/8350.bugfix new file mode 100644 index 0000000000..0e493c0282 --- /dev/null +++ b/changelog.d/8350.bugfix @@ -0,0 +1 @@ +Partially mitigate bug where newly joined servers couldn't get past events in a room when there is a malformed event. \ No newline at end of file diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index a2e8d96ea2..d42930d1b9 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -217,11 +217,9 @@ class FederationClient(FederationBase): for p in transaction_data["pdus"] ] - # FIXME: We should handle signature failures more gracefully. - pdus[:] = await make_deferred_yieldable( - defer.gatherResults( - self._check_sigs_and_hashes(room_version, pdus), consumeErrors=True, - ).addErrback(unwrapFirstError) + # Check signatures and hash of pdus, removing any from the list that fail checks + pdus[:] = await self._check_sigs_and_hash_and_fetch( + dest, pdus, outlier=True, room_version=room_version ) return pdus From 5b70acb44c730109029a5624238cdda80e109b17 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Fri, 18 Sep 2020 15:00:07 +0100 Subject: [PATCH 4/7] 1.19.3 --- CHANGES.md | 9 +++++++++ changelog.d/8350.bugfix | 1 - debian/changelog | 6 ++++++ synapse/__init__.py | 2 +- 4 files changed, 16 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/8350.bugfix diff --git a/CHANGES.md b/CHANGES.md index d82b30c66c..f20566e71e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,12 @@ +Synapse 1.19.3 (2020-09-18) +=========================== + +Bugfixes +-------- + +- Partially mitigate bug where newly joined servers couldn't get past events in a room when there is a malformed event. ([\#8350](https://github.com/matrix-org/synapse/issues/8350)) + + Synapse 1.19.2 (2020-09-16) =========================== diff --git a/changelog.d/8350.bugfix b/changelog.d/8350.bugfix deleted file mode 100644 index 0e493c0282..0000000000 --- a/changelog.d/8350.bugfix +++ /dev/null @@ -1 +0,0 @@ -Partially mitigate bug where newly joined servers couldn't get past events in a room when there is a malformed event. \ No newline at end of file diff --git a/debian/changelog b/debian/changelog index 6d60db6084..82125220aa 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.19.3) stable; urgency=medium + + * New synapse release 1.19.3. + + -- Synapse Packaging team Fri, 18 Sep 2020 14:59:30 +0100 + matrix-synapse-py3 (1.19.2) stable; urgency=medium * New synapse release 1.19.2. diff --git a/synapse/__init__.py b/synapse/__init__.py index 078914695a..56705707e0 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -48,7 +48,7 @@ try: except ImportError: pass -__version__ = "1.19.2" +__version__ = "1.19.3" if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)): # We import here so that we don't have to install a bunch of deps when From d5f7182ba15647f1c900883b7edbe898e32f012b Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Fri, 18 Sep 2020 10:56:50 -0400 Subject: [PATCH 5/7] 1.20.0rc5 --- CHANGES.md | 22 ++++++++++++++++++++++ changelog.d/8285.misc | 1 - changelog.d/8342.bugfix | 1 - changelog.d/8343.feature | 1 - changelog.d/8349.bugfix | 1 - synapse/__init__.py | 2 +- 6 files changed, 23 insertions(+), 5 deletions(-) delete mode 100644 changelog.d/8285.misc delete mode 100644 changelog.d/8342.bugfix delete mode 100644 changelog.d/8343.feature delete mode 100644 changelog.d/8349.bugfix diff --git a/CHANGES.md b/CHANGES.md index aade896bd5..de869001b3 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,25 @@ +Synapse 1.20.0rc5 (2020-09-18) +============================== + +Features +-------- + +- Add flags to the `/versions` endpoint that includes whether new rooms default to using E2EE. ([\#8343](https://github.com/matrix-org/synapse/issues/8343)) + + +Bugfixes +-------- + +- Fix ratelimitng of federation `/send` requests. ([\#8342](https://github.com/matrix-org/synapse/issues/8342)) +- Fix a longstanding bug where back pagination over federation could get stuck if it failed to handle a received event. ([\#8349](https://github.com/matrix-org/synapse/issues/8349)) + + +Internal Changes +---------------- + +- Blacklist [MSC2753](https://github.com/matrix-org/matrix-doc/pull/2753) SyTests until it is implemented. ([\#8285](https://github.com/matrix-org/synapse/issues/8285)) + + Synapse 1.19.3 (2020-09-18) =========================== diff --git a/changelog.d/8285.misc b/changelog.d/8285.misc deleted file mode 100644 index 4646664ba1..0000000000 --- a/changelog.d/8285.misc +++ /dev/null @@ -1 +0,0 @@ -Blacklist [MSC2753](https://github.com/matrix-org/matrix-doc/pull/2753) SyTests until it is implemented. \ No newline at end of file diff --git a/changelog.d/8342.bugfix b/changelog.d/8342.bugfix deleted file mode 100644 index 786057facb..0000000000 --- a/changelog.d/8342.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix ratelimitng of federation `/send` requests. diff --git a/changelog.d/8343.feature b/changelog.d/8343.feature deleted file mode 100644 index ccecb22f37..0000000000 --- a/changelog.d/8343.feature +++ /dev/null @@ -1 +0,0 @@ -Add flags to the `/versions` endpoint that includes whether new rooms default to using E2EE. diff --git a/changelog.d/8349.bugfix b/changelog.d/8349.bugfix deleted file mode 100644 index cf2f531b14..0000000000 --- a/changelog.d/8349.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a longstanding bug where back pagination over federation could get stuck if it failed to handle a received event. diff --git a/synapse/__init__.py b/synapse/__init__.py index 6b11c5681b..a95753dcc7 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -48,7 +48,7 @@ try: except ImportError: pass -__version__ = "1.20.0rc4" +__version__ = "1.20.0rc5" if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)): # We import here so that we don't have to install a bunch of deps when From c4e8b18c72365c9d8f592e74519f8fe5a6cf402b Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Fri, 18 Sep 2020 10:57:29 -0400 Subject: [PATCH 6/7] Tweak wording in the changelog. --- CHANGES.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index de869001b3..7909386b47 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,13 +4,13 @@ Synapse 1.20.0rc5 (2020-09-18) Features -------- -- Add flags to the `/versions` endpoint that includes whether new rooms default to using E2EE. ([\#8343](https://github.com/matrix-org/synapse/issues/8343)) +- Add flags to the `/versions` endpoint for whether new rooms default to using E2EE. ([\#8343](https://github.com/matrix-org/synapse/issues/8343)) Bugfixes -------- -- Fix ratelimitng of federation `/send` requests. ([\#8342](https://github.com/matrix-org/synapse/issues/8342)) +- Fix rate limiting of federation `/send` requests. ([\#8342](https://github.com/matrix-org/synapse/issues/8342)) - Fix a longstanding bug where back pagination over federation could get stuck if it failed to handle a received event. ([\#8349](https://github.com/matrix-org/synapse/issues/8349)) From c7e060bfee89ebea599b22fc402fae8336b1afed Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Fri, 18 Sep 2020 11:10:59 -0400 Subject: [PATCH 7/7] Add a note about including the changes from 1.19.3. --- CHANGES.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 7909386b47..84976ab2bd 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,8 @@ Synapse 1.20.0rc5 (2020-09-18) ============================== +In addition to the below, Synapse 1.20.0rc5 also includes the bug fix that was included in 1.19.3. + Features --------