Merge remote-tracking branch 'origin/develop' into matrix-org-hotfixes

.github/workflows/release-artifacts.yml

@@ -130,7 +130,7 @@ jobs:
           python-version: "3.x"
 
       - name: Install cibuildwheel
-        run: python -m pip install cibuildwheel==2.9.0
+        run: python -m pip install cibuildwheel==2.16.2
 
       - name: Set up QEMU to emulate aarch64
         if: matrix.arch == 'aarch64'

CHANGES.md

@@ -1,3 +1,28 @@
+# Synapse 1.96.1 (2023-11-17)
+
+Synapse will soon be forked by Element under an AGPLv3.0 licence (with CLA, for
+proprietary dual licensing). You can read more about this here:
+
+* https://matrix.org/blog/2023/11/06/future-of-synapse-dendrite/
+* https://element.io/blog/element-to-adopt-agplv3/
+
+The Matrix.org Foundation copy of the project will be archived. Any changes needed
+by server administrators will be communicated via our usual
+[announcements channels](https://matrix.to/#/#homeowners:matrix.org), but we are
+striving to make this as seamless as possible.
+
+This minor release was needed only because of CI-related trouble on [v1.96.0](https://github.com/matrix-org/synapse/releases/tag/v1.96.0), which was never released.
+
+### Internal Changes
+
+- Fix building of wheels in CI. ([\#16653](https://github.com/matrix-org/synapse/issues/16653))
+
+# Synapse 1.96.0 (2023-11-16)
+
+### Bugfixes
+
+- Fix "'int' object is not iterable" error in `set_device_id_for_pushers` background update introduced in Synapse 1.95.0. ([\#16594](https://github.com/matrix-org/synapse/issues/16594))
+
 # Synapse 1.96.0rc1 (2023-10-31)
 
 ### Features

changelog.d/16654.doc

@@ -0,0 +1 @@
+Provide an example for a common encrypted media response from the admin user media API and mention possible null values.

changelog.d/16655.misc

@@ -0,0 +1 @@
+More efficiently handle no-op `POSITION` over replication.

changelog.d/16656.misc

@@ -0,0 +1 @@
+Reduce max concurrency of background tasks, reducing potential max DB load.

changelog.d/16657.misc

@@ -0,0 +1 @@
+Speed up purge room by adding an index to `event_push_summary`.

changelog.d/16659.misc

@@ -0,0 +1 @@
+Bump requests-toolbelt from 0.10.1 to 1.0.0.

changelog.d/16660.misc

@@ -0,0 +1 @@
+Reduce max concurrency of background tasks, reducing potential max DB load.

debian/changelog

@@ -1,3 +1,15 @@
+matrix-synapse-py3 (1.96.1) stable; urgency=medium
+
+  * New synapse release 1.96.1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Fri, 17 Nov 2023 12:48:45 +0000
+
+matrix-synapse-py3 (1.96.0) stable; urgency=medium
+
+  * New synapse release 1.96.0.
+
+ -- Synapse Packaging team <packages@matrix.org>  Thu, 16 Nov 2023 17:54:26 +0000
+
 matrix-synapse-py3 (1.96.0~rc1) stable; urgency=medium
 
   * New Synapse release 1.96.0rc1.

docs/admin_api/user_admin_api.md

@@ -618,6 +618,16 @@ A response body like the following is returned:
       "quarantined_by": null,
       "safe_from_quarantine": false,
       "upload_name": "test2.png"
+    },
+    {
+      "created_ts": 300400,
+      "last_access_ts": 300700,
+      "media_id": "BzYNLRUgGHphBkdKGbzXwbjX",
+      "media_length": 1337,
+      "media_type": "application/octet-stream",
+      "quarantined_by": null,
+      "safe_from_quarantine": false,
+      "upload_name": null
     }
   ],
   "next_token": 3,
@@ -679,16 +689,17 @@ The following fields are returned in the JSON response body:
 - `media` - An array of objects, each containing information about a media.
   Media objects contain the following fields:
   - `created_ts` - integer - Timestamp when the content was uploaded in ms.
-  - `last_access_ts` - integer - Timestamp when the content was last accessed in ms.
+  - `last_access_ts` - integer or null - Timestamp when the content was last accessed in ms.
+     Null if there was no access, yet.
   - `media_id` - string - The id used to refer to the media. Details about the format
     are documented under
     [media repository](../media_repository.md).
   - `media_length` - integer - Length of the media in bytes.
   - `media_type` - string - The MIME-type of the media.
-  - `quarantined_by` - string - The user ID that initiated the quarantine request
-    for this media.
+  - `quarantined_by` - string or null - The user ID that initiated the quarantine request
+    for this media. Null if not quarantined.
   - `safe_from_quarantine` - bool - Status if this media is safe from quarantining.
-  - `upload_name` - string - The name the media was uploaded with.
+  - `upload_name` - string or null - The name the media was uploaded with. Null if not provided during upload.
 - `next_token`: integer - Indication for pagination. See above.
 - `total` - integer - Total number of media.
 

poetry.lock

@@ -1,4 +1,4 @@
-# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand.
+# This file is automatically @generated by Poetry 1.7.0 and should not be changed by hand.
 
 [[package]]
 name = "alabaster"
@@ -2273,13 +2273,13 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"]
 
 [[package]]
 name = "requests-toolbelt"
-version = "0.10.1"
+version = "1.0.0"
 description = "A utility belt for advanced users of python-requests"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
 files = [
-    {file = "requests-toolbelt-0.10.1.tar.gz", hash = "sha256:62e09f7ff5ccbda92772a29f394a49c3ad6cb181d568b1337626b2abb628a63d"},
-    {file = "requests_toolbelt-0.10.1-py2.py3-none-any.whl", hash = "sha256:18565aa58116d9951ac39baa288d3adb5b3ff975c4f25eee78555d89e8f247f7"},
+    {file = "requests-toolbelt-1.0.0.tar.gz", hash = "sha256:7681a0a3d047012b5bdc0ee37d7f8f07ebe76ab08caeccfc3921ce23c88d5bc6"},
+    {file = "requests_toolbelt-1.0.0-py2.py3-none-any.whl", hash = "sha256:cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06"},
 ]
 
 [package.dependencies]

pyproject.toml

@@ -96,7 +96,7 @@ module-name = "synapse.synapse_rust"
 
 [tool.poetry]
 name = "matrix-synapse"
-version = "1.96.0rc1"
+version = "1.96.1"
 description = "Homeserver for the Matrix decentralised comms protocol"
 authors = ["Matrix.org Team and Contributors <packages@matrix.org>"]
 license = "Apache-2.0"

synapse/handlers/device.py

@@ -383,7 +383,7 @@ class DeviceWorkerHandler:
         )
 
     DEVICE_MSGS_DELETE_BATCH_LIMIT = 1000
-    DEVICE_MSGS_DELETE_SLEEP_MS = 1000
+    DEVICE_MSGS_DELETE_SLEEP_MS = 100
 
     async def _delete_device_messages(
         self,

synapse/replication/tcp/streams/_base.py

@@ -305,6 +305,14 @@ class BackfillStream(Stream):
         # which means we need to negate it.
         return -self.store._backfill_id_gen.get_minimal_local_current_token()
 
+    def can_discard_position(
+        self, instance_name: str, prev_token: int, new_token: int
+    ) -> bool:
+        # Backfill stream can't go backwards, so we know we can ignore any
+        # positions where the tokens are from before the current token.
+
+        return new_token <= self.current_token(instance_name)
+
 
 class PresenceStream(_StreamFromIdGen):
     @attr.s(slots=True, frozen=True, auto_attribs=True)
@@ -519,6 +527,14 @@ class CachesStream(Stream):
             return self.store._cache_id_gen.get_minimal_local_current_token()
         return self.current_token(self.local_instance_name)
 
+    def can_discard_position(
+        self, instance_name: str, prev_token: int, new_token: int
+    ) -> bool:
+        # Caches streams can't go backwards, so we know we can ignore any
+        # positions where the tokens are from before the current token.
+
+        return new_token <= self.current_token(instance_name)
+
 
 class DeviceListsStream(_StreamFromIdGen):
     """Either a user has updated their devices or a remote server needs to be

synapse/storage/databases/main/event_push_actions.py

@@ -311,6 +311,14 @@ class EventPushActionsWorkerStore(ReceiptsWorkerStore, StreamWorkerStore, SQLBas
             self._background_drop_null_thread_id_indexes,
         )
 
+        # Add a room ID index to speed up room deletion
+        self.db_pool.updates.register_background_index_update(
+            "event_push_summary_index_room_id",
+            index_name="event_push_summary_index_room_id",
+            table="event_push_summary",
+            columns=["room_id"],
+        )
+
     async def _background_drop_null_thread_id_indexes(
         self, progress: JsonDict, batch_size: int
     ) -> int:

synapse/storage/databases/main/pusher.py

@@ -601,7 +601,7 @@ class PusherBackgroundUpdatesStore(SQLBaseStore):
                 (last_pusher_id, batch_size),
             )
 
-            rows = txn.fetchall()
+            rows = cast(List[Tuple[int, Optional[str], Optional[str]]], txn.fetchall())
             if len(rows) == 0:
                 return 0
 
@@ -617,7 +617,7 @@ class PusherBackgroundUpdatesStore(SQLBaseStore):
                 txn=txn,
                 table="pushers",
                 key_names=("id",),
-                key_values=[row[0] for row in rows],
+                key_values=[(row[0],) for row in rows],
                 value_names=("device_id", "access_token"),
                 # If there was already a device_id on the pusher, we only want to clear
                 # the access_token column, so we keep the existing device_id. Otherwise,

synapse/storage/schema/main/delta/83/06_event_push_summary_room.sql

@@ -0,0 +1,17 @@
+/* Copyright 2023 The Matrix.org Foundation C.I.C
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+INSERT INTO background_updates (ordering, update_name, progress_json) VALUES
+    (8306, 'event_push_summary_index_room_id', '{}');

synapse/util/task_scheduler.py

@@ -71,7 +71,7 @@ class TaskScheduler:
     # Time before a complete or failed task is deleted from the DB
     KEEP_TASKS_FOR_MS = 7 * 24 * 60 * 60 * 1000  # 1 week
     # Maximum number of tasks that can run at the same time
-    MAX_CONCURRENT_RUNNING_TASKS = 10
+    MAX_CONCURRENT_RUNNING_TASKS = 5
     # Time from the last task update after which we will log a warning
     LAST_UPDATE_BEFORE_WARNING_MS = 24 * 60 * 60 * 1000  # 24hrs
 
@@ -377,7 +377,7 @@ class TaskScheduler:
                 self._running_tasks.remove(task.id)
 
             # Try launch a new task since we've finished with this one.
-            self._clock.call_later(1, self._launch_scheduled_tasks)
+            self._clock.call_later(0.1, self._launch_scheduled_tasks)
 
         if len(self._running_tasks) >= TaskScheduler.MAX_CONCURRENT_RUNNING_TASKS:
             return