Synapse 1.47.0rc3 (2021-11-16)
============================== Bugfixes -------- - Fix a bug introduced in 1.47.0rc1 which caused worker processes to not halt startup in the presence of outstanding database migrations. ([\#11346](https://github.com/matrix-org/synapse/issues/11346)) - Fix a bug introduced in 1.47.0rc1 which prevented the 'remove deleted devices from `device_inbox` column' background process from running when updating from a recent Synapse version. ([\#11303](https://github.com/matrix-org/synapse/issues/11303), [\#11353](https://github.com/matrix-org/synapse/issues/11353)) -----BEGIN PGP SIGNATURE----- iQJHBAABCAAxFiEEgQG31Z317NrSMt0QiISIDS7+X/QFAmGTw7wTHGFuZHJld0Bh bW9yZ2FuLnh5egAKCRCIhIgNLv5f9FUdD/9EgimhmOUuwcbNQS+zJJo6qhbnxngA uOIWGhuoBq9vyZjx3s1mwRO/sCuBc6KRs2JYzhntfHwV6FCgkC0QJXPkrYO7mkbg L/QsET3QYrWwClMN1y/j2/rstOqGtgrtg6pPgd9LiQH+LjdM+vzAyrgxJRyhk9Sx 7184Y9QpEt/6ApqFIbXZewP+zH1z1QyVeZ1WiuHsqhjcrrcpv1t8Rq0fv4FxgIVP Tmy3T2uw0FeVoQIL8BnNVmRGbLjlSgpDfiXZTwuXGS/uLtTe1tnHf0QsKEK7XZxY yUsRKQVDy76GW9AJuWKFXa9Voy5gkjLWnvMwHvr+B1WcJi/5cjz9lP2A0Bsm6vhQ ivrktlhlw7O3x0EvrL1r4z2gRY4GXxyZKOBLRsY4AAIMblNR/e94SmK0rLoKIQn8 Rp50VV2B4cf3WLxoqHBpJh7CDFisth1nXMnETU3y0VgAYMg2+cxqVDawTSTH9v7Q QeGI10QCMmeFwGoL6lyVJ+qhyUqFeFOqTrNqzigsiB1qule+w1fEEl7cvI+Om/QM Wjkvw6fpOvUGBN1OOARL5Qnm4rjxa0Ld2vw5vugkGiDTV8P3TMJP1JFQ44IX2GKP yGJH92ac0qy6vQ2za1gL/w/U3miNblHUIvAmpkaIIlvovQJH9dU5dMsxnvi+QZU+ Auyqh9rwiWiizA== =iNww -----END PGP SIGNATURE----- gpgsig -----BEGIN PGP SIGNATURE----- iQIzBAABCAAdFiEE1508oLYUKainYFJakD7OEIo53t0FAmGT0mcACgkQkD7OEIo5 3t0sSA/+KjLlnYe3xO9Pbeeq7/E6Qs3VTMbwERcVVj0WZQU8Hik/EH+p0zsnG9jU 8t9UVs0xnu1r5ZMNsrMnfOU5SR9o7RM8UBCHnIYde8yFf1N6FI0YgGJ8nOP96gSG dKZQOCKA9psQqd0pvhwCem4ITmBvVz2CrnUZYkKp9rHjUZeBYwE7cWmHL5W/WzT+ Sv0zafhvYi76PxMKdXu08MArMZ9JpCLbZnlzaoW/sY+xvnQBRfQviqYkj+qZhQjr tPnJlQN5FS1pO1ZHd9o0mVcdbDBwLqeQlqC1toFuZXbed6e558KtCzvizM5VAfF8 peRGVFarbZpD/QcvoFljoH9qIECrcAYIN1HeE8aX7pIedh2AVROyINumdnbmsMJH F4OyX/aLb3KbbecHJaVLQ2c/KyDnSnxr6Fs3HlEslD7DRdA0TS7XsI/BVER0pscp j2Q5nLOthpV2d1SVekj4Ge/Hr++AuTnriHldEx9OlGI3/74ldrNL2c/L5gM3ue1W qnPQfr5ehtIitqa/ROiexxSoWS5OC953UujTCwBgil9mAD9gC7mhoEFOAFVuMHfN zBjsunMGBRYsGcw2umTPGPd7D3Gi1FXQrduN1xUoV8g8vmJld4GDjqJsitiaR3lv NUsQ5JzttakDUwAJU7qijOo2Y/HtSs5E2nF66bMCSmwHvl9AkUc= =c7gi -----END PGP SIGNATURE----- Merge tag 'v1.47.0rc3' into develop Synapse 1.47.0rc3 (2021-11-16) ============================== Bugfixes -------- - Fix a bug introduced in 1.47.0rc1 which caused worker processes to not halt startup in the presence of outstanding database migrations. ([\#11346](https://github.com/matrix-org/synapse/issues/11346)) - Fix a bug introduced in 1.47.0rc1 which prevented the 'remove deleted devices from `device_inbox` column' background process from running when updating from a recent Synapse version. ([\#11303](https://github.com/matrix-org/synapse/issues/11303), [\#11353](https://github.com/matrix-org/synapse/issues/11353))
This commit is contained in:
David Robertson
commit
0caf20883c
10
CHANGES.md
10
CHANGES.md
|
|
@ -1,3 +1,13 @@
|
|||
Synapse 1.47.0rc3 (2021-11-16)
|
||||
==============================
|
||||
|
||||
Bugfixes
|
||||
--------
|
||||
|
||||
- Fix a bug introduced in 1.47.0rc1 which caused worker processes to not halt startup in the presence of outstanding database migrations. ([\#11346](https://github.com/matrix-org/synapse/issues/11346))
|
||||
- Fix a bug introduced in 1.47.0rc1 which prevented the 'remove deleted devices from `device_inbox` column' background process from running when updating from a recent Synapse version. ([\#11303](https://github.com/matrix-org/synapse/issues/11303), [\#11353](https://github.com/matrix-org/synapse/issues/11353))
|
||||
|
||||
|
||||
Synapse 1.47.0rc2 (2021-11-10)
|
||||
==============================
|
||||
|
||||
|
|
|
|||
|
|
@ -1,3 +1,9 @@
|
|||
matrix-synapse-py3 (1.47.0~rc3) stable; urgency=medium
|
||||
|
||||
* New synapse release 1.47.0~rc3.
|
||||
|
||||
-- Synapse Packaging team <packages@matrix.org> Tue, 16 Nov 2021 14:32:47 +0000
|
||||
|
||||
matrix-synapse-py3 (1.47.0~rc2) stable; urgency=medium
|
||||
|
||||
[ Dan Callahan ]
|
||||
|
|
|
|||
|
|
@ -47,7 +47,7 @@ try:
|
|||
except ImportError:
|
||||
pass
|
||||
|
||||
__version__ = "1.47.0rc2"
|
||||
__version__ = "1.47.0rc3"
|
||||
|
||||
if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)):
|
||||
# We import here so that we don't have to install a bunch of deps when
|
||||
|
|
|
|||
|
|
@ -131,24 +131,16 @@ def prepare_database(
|
|||
"config==None in prepare_database, but database is not empty"
|
||||
)
|
||||
|
||||
# if it's a worker app, refuse to upgrade the database, to avoid multiple
|
||||
# workers doing it at once.
|
||||
if config.worker.worker_app is None:
|
||||
_upgrade_existing_database(
|
||||
cur,
|
||||
version_info,
|
||||
database_engine,
|
||||
config,
|
||||
databases=databases,
|
||||
)
|
||||
elif version_info.current_version < SCHEMA_VERSION:
|
||||
# If the DB is on an older version than we expect then we refuse
|
||||
# to start the worker (as the main process needs to run first to
|
||||
# update the schema).
|
||||
raise UpgradeDatabaseException(
|
||||
OUTDATED_SCHEMA_ON_WORKER_ERROR
|
||||
% (SCHEMA_VERSION, version_info.current_version)
|
||||
)
|
||||
# This should be run on all processes, master or worker. The master will
|
||||
# apply the deltas, while workers will check if any outstanding deltas
|
||||
# exist and raise an PrepareDatabaseException if they do.
|
||||
_upgrade_existing_database(
|
||||
cur,
|
||||
version_info,
|
||||
database_engine,
|
||||
config,
|
||||
databases=databases,
|
||||
)
|
||||
|
||||
else:
|
||||
logger.info("%r: Initialising new database", databases)
|
||||
|
|
@ -358,6 +350,18 @@ def _upgrade_existing_database(
|
|||
|
||||
is_worker = config and config.worker.worker_app is not None
|
||||
|
||||
# If the schema version needs to be updated, and we are on a worker, we immediately
|
||||
# know to bail out as workers cannot update the database schema. Only one process
|
||||
# must update the database at the time, therefore we delegate this task to the master.
|
||||
if is_worker and current_schema_state.current_version < SCHEMA_VERSION:
|
||||
# If the DB is on an older version than we expect then we refuse
|
||||
# to start the worker (as the main process needs to run first to
|
||||
# update the schema).
|
||||
raise UpgradeDatabaseException(
|
||||
OUTDATED_SCHEMA_ON_WORKER_ERROR
|
||||
% (SCHEMA_VERSION, current_schema_state.current_version)
|
||||
)
|
||||
|
||||
if (
|
||||
current_schema_state.compat_version is not None
|
||||
and current_schema_state.compat_version > SCHEMA_VERSION
|
||||
|
|
|
|||
|
|
@ -18,5 +18,17 @@
|
|||
-- when a device was deleted using Synapse earlier than 1.47.0.
|
||||
-- This runs as background task, but may take a bit to finish.
|
||||
|
||||
-- Remove any existing instances of this job running. It's OK to stop and restart this job,
|
||||
-- as it's just deleting entries from a table - no progress will be lost.
|
||||
--
|
||||
-- This is necessary due a similar migration running the job accidentally
|
||||
-- being included in schema version 64 during v1.47.0rc1,rc2. If a
|
||||
-- homeserver had updated from Synapse <=v1.45.0 (schema version <=64),
|
||||
-- then they would have started running this background update already.
|
||||
-- If that update was still running, then simply inserting it again would
|
||||
-- cause an SQL failure. So we effectively do an "upsert" here instead.
|
||||
|
||||
DELETE FROM background_updates WHERE update_name = 'remove_deleted_devices_from_device_inbox';
|
||||
|
||||
INSERT INTO background_updates (ordering, update_name, progress_json) VALUES
|
||||
(6505, 'remove_deleted_devices_from_device_inbox', '{}');
|
||||
(6506, 'remove_deleted_devices_from_device_inbox', '{}');
|
||||
|
|
@ -11,6 +11,9 @@
|
|||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from typing import List
|
||||
from unittest import mock
|
||||
|
||||
from synapse.app.generic_worker import GenericWorkerServer
|
||||
from synapse.storage.database import LoggingDatabaseConnection
|
||||
from synapse.storage.prepare_database import PrepareDatabaseException, prepare_database
|
||||
|
|
@ -19,6 +22,22 @@ from synapse.storage.schema import SCHEMA_VERSION
|
|||
from tests.unittest import HomeserverTestCase
|
||||
|
||||
|
||||
def fake_listdir(filepath: str) -> List[str]:
|
||||
"""
|
||||
A fake implementation of os.listdir which we can use to mock out the filesystem.
|
||||
|
||||
Args:
|
||||
filepath: The directory to list files for.
|
||||
|
||||
Returns:
|
||||
A list of files and folders in the directory.
|
||||
"""
|
||||
if filepath.endswith("full_schemas"):
|
||||
return [SCHEMA_VERSION]
|
||||
|
||||
return ["99_add_unicorn_to_database.sql"]
|
||||
|
||||
|
||||
class WorkerSchemaTests(HomeserverTestCase):
|
||||
def make_homeserver(self, reactor, clock):
|
||||
hs = self.setup_test_homeserver(
|
||||
|
|
@ -51,7 +70,7 @@ class WorkerSchemaTests(HomeserverTestCase):
|
|||
|
||||
prepare_database(db_conn, db_pool.engine, self.hs.config)
|
||||
|
||||
def test_not_upgraded(self):
|
||||
def test_not_upgraded_old_schema_version(self):
|
||||
"""Test that workers don't start if the DB has an older schema version"""
|
||||
db_pool = self.hs.get_datastore().db_pool
|
||||
db_conn = LoggingDatabaseConnection(
|
||||
|
|
@ -67,3 +86,34 @@ class WorkerSchemaTests(HomeserverTestCase):
|
|||
|
||||
with self.assertRaises(PrepareDatabaseException):
|
||||
prepare_database(db_conn, db_pool.engine, self.hs.config)
|
||||
|
||||
def test_not_upgraded_current_schema_version_with_outstanding_deltas(self):
|
||||
"""
|
||||
Test that workers don't start if the DB is on the current schema version,
|
||||
but there are still outstanding delta migrations to run.
|
||||
"""
|
||||
db_pool = self.hs.get_datastore().db_pool
|
||||
db_conn = LoggingDatabaseConnection(
|
||||
db_pool._db_pool.connect(),
|
||||
db_pool.engine,
|
||||
"tests",
|
||||
)
|
||||
|
||||
# Set the schema version of the database to the current version
|
||||
cur = db_conn.cursor()
|
||||
cur.execute("UPDATE schema_version SET version = ?", (SCHEMA_VERSION,))
|
||||
|
||||
db_conn.commit()
|
||||
|
||||
# Path `os.listdir` here to make synapse think that there is a migration
|
||||
# file ready to be run.
|
||||
# Note that we can't patch this function for the whole method, else Synapse
|
||||
# will try to find the file when building the database initially.
|
||||
with mock.patch("os.listdir", mock.Mock(side_effect=fake_listdir)):
|
||||
with self.assertRaises(PrepareDatabaseException):
|
||||
# Synapse should think that there is an outstanding migration file due to
|
||||
# patching 'os.listdir' in the function decorator.
|
||||
#
|
||||
# We expect Synapse to raise an exception to indicate the master process
|
||||
# needs to apply this migration file.
|
||||
prepare_database(db_conn, db_pool.engine, self.hs.config)
|
||||
|
|
|
|||
Loading…
Reference in New Issue