diff --git a/develop/404.html b/develop/404.html index f7dd922449..7692d1c8df 100644 --- a/develop/404.html +++ b/develop/404.html @@ -101,7 +101,7 @@ diff --git a/develop/CAPTCHA_SETUP.html b/develop/CAPTCHA_SETUP.html index 3672e08d6d..a5ea438bd2 100644 --- a/develop/CAPTCHA_SETUP.html +++ b/develop/CAPTCHA_SETUP.html @@ -99,7 +99,7 @@ diff --git a/develop/MSC1711_certificates_FAQ.html b/develop/MSC1711_certificates_FAQ.html index 966230c684..ed024d7daf 100644 --- a/develop/MSC1711_certificates_FAQ.html +++ b/develop/MSC1711_certificates_FAQ.html @@ -99,7 +99,7 @@ diff --git a/develop/admin_api/account_validity.html b/develop/admin_api/account_validity.html index e41490f9f2..1e55a34cc4 100644 --- a/develop/admin_api/account_validity.html +++ b/develop/admin_api/account_validity.html @@ -99,7 +99,7 @@ diff --git a/develop/admin_api/delete_group.html b/develop/admin_api/delete_group.html index 142be17a34..df4409e2b2 100644 --- a/develop/admin_api/delete_group.html +++ b/develop/admin_api/delete_group.html @@ -99,7 +99,7 @@ diff --git a/develop/admin_api/event_reports.html b/develop/admin_api/event_reports.html index 25227c8ecf..868d6b1bf0 100644 --- a/develop/admin_api/event_reports.html +++ b/develop/admin_api/event_reports.html @@ -99,7 +99,7 @@ diff --git a/develop/admin_api/media_admin_api.html b/develop/admin_api/media_admin_api.html index 2d0db60530..86a4c64a79 100644 --- a/develop/admin_api/media_admin_api.html +++ b/develop/admin_api/media_admin_api.html @@ -99,7 +99,7 @@ diff --git a/develop/admin_api/purge_history_api.html b/develop/admin_api/purge_history_api.html index 6084577934..dc411ec5a7 100644 --- a/develop/admin_api/purge_history_api.html +++ b/develop/admin_api/purge_history_api.html @@ -99,7 +99,7 @@ diff --git a/develop/admin_api/purge_room.html b/develop/admin_api/purge_room.html index d7046c4be3..1adb4a6a63 100644 --- a/develop/admin_api/purge_room.html +++ b/develop/admin_api/purge_room.html @@ -99,7 +99,7 @@ diff --git a/develop/admin_api/register_api.html b/develop/admin_api/register_api.html index c96405613f..8219ebd0bf 100644 --- a/develop/admin_api/register_api.html +++ b/develop/admin_api/register_api.html @@ -99,7 +99,7 @@ diff --git a/develop/admin_api/room_membership.html b/develop/admin_api/room_membership.html index 972f9b4194..08bec4be62 100644 --- a/develop/admin_api/room_membership.html +++ b/develop/admin_api/room_membership.html @@ -99,7 +99,7 @@ diff --git a/develop/admin_api/rooms.html b/develop/admin_api/rooms.html index 6bdd7b80a0..2c1b5752ba 100644 --- a/develop/admin_api/rooms.html +++ b/develop/admin_api/rooms.html @@ -99,7 +99,7 @@ diff --git a/develop/admin_api/server_notices.html b/develop/admin_api/server_notices.html index 6da23848c0..6c738d4073 100644 --- a/develop/admin_api/server_notices.html +++ b/develop/admin_api/server_notices.html @@ -99,7 +99,7 @@ diff --git a/develop/admin_api/shutdown_room.html b/develop/admin_api/shutdown_room.html index 9c5304d63d..6f606b1626 100644 --- a/develop/admin_api/shutdown_room.html +++ b/develop/admin_api/shutdown_room.html @@ -99,7 +99,7 @@ diff --git a/develop/admin_api/statistics.html b/develop/admin_api/statistics.html index d569ac6e96..79309cc3ec 100644 --- a/develop/admin_api/statistics.html +++ b/develop/admin_api/statistics.html @@ -99,7 +99,7 @@ diff --git a/develop/admin_api/user_admin_api.html b/develop/admin_api/user_admin_api.html index b756d9ba7e..7a1cc1143c 100644 --- a/develop/admin_api/user_admin_api.html +++ b/develop/admin_api/user_admin_api.html @@ -99,7 +99,7 @@ diff --git a/develop/admin_api/version_api.html b/develop/admin_api/version_api.html index 1f35eb19b1..79e13da824 100644 --- a/develop/admin_api/version_api.html +++ b/develop/admin_api/version_api.html @@ -99,7 +99,7 @@ diff --git a/develop/application_services.html b/develop/application_services.html index 1a12f9cdfe..d5feeb0737 100644 --- a/develop/application_services.html +++ b/develop/application_services.html @@ -99,7 +99,7 @@ diff --git a/develop/auth_chain_difference_algorithm.html b/develop/auth_chain_difference_algorithm.html index 5525f823b1..902bec8992 100644 --- a/develop/auth_chain_difference_algorithm.html +++ b/develop/auth_chain_difference_algorithm.html @@ -99,7 +99,7 @@ diff --git a/develop/code_style.html b/develop/code_style.html index 35bd7a908d..3ad6ab4884 100644 --- a/develop/code_style.html +++ b/develop/code_style.html @@ -99,7 +99,7 @@ diff --git a/develop/consent_tracking.html b/develop/consent_tracking.html index 7b15ae9528..57fc99bf61 100644 --- a/develop/consent_tracking.html +++ b/develop/consent_tracking.html @@ -99,7 +99,7 @@ diff --git a/develop/delegate.html b/develop/delegate.html index 057755a69f..c50e5d863a 100644 --- a/develop/delegate.html +++ b/develop/delegate.html @@ -99,7 +99,7 @@ diff --git a/develop/deprecation_policy.html b/develop/deprecation_policy.html index 0cf2e668ed..18e76ce1d5 100644 --- a/develop/deprecation_policy.html +++ b/develop/deprecation_policy.html @@ -99,7 +99,7 @@ diff --git a/develop/dev/cas.html b/develop/dev/cas.html index 0325e96b5e..cd6a54ccf8 100644 --- a/develop/dev/cas.html +++ b/develop/dev/cas.html @@ -99,7 +99,7 @@ diff --git a/develop/dev/git.html b/develop/dev/git.html index fa91d5f6a2..95b091e7a3 100644 --- a/develop/dev/git.html +++ b/develop/dev/git.html @@ -99,7 +99,7 @@ diff --git a/develop/dev/saml.html b/develop/dev/saml.html index 464095f1c7..9c93332424 100644 --- a/develop/dev/saml.html +++ b/develop/dev/saml.html @@ -99,7 +99,7 @@ diff --git a/develop/development/contributing_guide.html b/develop/development/contributing_guide.html index 13d3bf1438..586fdbbf51 100644 --- a/develop/development/contributing_guide.html +++ b/develop/development/contributing_guide.html @@ -99,7 +99,7 @@ diff --git a/develop/development/database_schema.html b/develop/development/database_schema.html new file mode 100644 index 0000000000..dccc6ecafa --- /dev/null +++ b/develop/development/database_schema.html @@ -0,0 +1,340 @@ + + + + + + Database Schemas - Synapse + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + + + + + + + +
+
+ +
+ +
+ +

Synapse database schema files

+

Synapse's database schema is stored in the synapse.storage.schema module.

+

Logical databases

+

Synapse supports splitting its datastore across multiple physical databases (which can +be useful for large installations), and the schema files are therefore split according +to the logical database they apply to.

+

At the time of writing, the following "logical" databases are supported:

+
    +
  • state - used to store Matrix room state (more specifically, state_groups, +their relationships and contents).
  • +
  • main - stores everything else.
  • +
+

Additionally, the common directory contains schema files for tables which must be +present on all physical databases.

+

Synapse schema versions

+

Synapse manages its database schema via "schema versions". These are mainly used to +help avoid confusion if the Synapse codebase is rolled back after the database is +updated. They work as follows:

+
    +
  • +

    The Synapse codebase defines a constant synapse.storage.schema.SCHEMA_VERSION +which represents the expectations made about the database by that version. For +example, as of Synapse v1.36, this is 59.

    +
  • +
  • +

    The database stores a "compatibility version" in +schema_compat_version.compat_version which defines the SCHEMA_VERSION of the +oldest version of Synapse which will work with the database. On startup, if +compat_version is found to be newer than SCHEMA_VERSION, Synapse will refuse to +start.

    +

    Synapse automatically updates this field from +synapse.storage.schema.SCHEMA_COMPAT_VERSION.

    +
  • +
  • +

    Whenever a backwards-incompatible change is made to the database format (normally +via a delta file), synapse.storage.schema.SCHEMA_COMPAT_VERSION is also updated +so that administrators can not accidentally roll back to a too-old version of Synapse.

    +
  • +
+

Generally, the goal is to maintain compatibility with at least one or two previous +releases of Synapse, so any substantial change tends to require multiple releases and a +bit of forward-planning to get right.

+

As a worked example: we want to remove the room_stats_historical table. Here is how it +might pan out.

+
    +
  1. +

    Replace any code that reads from room_stats_historical with alternative +implementations, but keep writing to it in case of rollback to an earlier version. +Also, increase synapse.storage.schema.SCHEMA_VERSION. In this +instance, there is no existing code which reads from room_stats_historical, so +our starting point is:

    +

    v1.36.0: SCHEMA_VERSION=59, SCHEMA_COMPAT_VERSION=59

    +
  2. +
  3. +

    Next (say in Synapse v1.37.0): remove the code that writes to +room_stats_historical, but don’t yet remove the table in case of rollback to +v1.36.0. Again, we increase synapse.storage.schema.SCHEMA_VERSION, but +because we have not broken compatibility with v1.36, we do not yet update +SCHEMA_COMPAT_VERSION. We now have:

    +

    v1.37.0: SCHEMA_VERSION=60, SCHEMA_COMPAT_VERSION=59.

    +
  4. +
  5. +

    Later (say in Synapse v1.38.0): we can remove the table altogether. This will +break compatibility with v1.36.0, so we must update SCHEMA_COMPAT_VERSION accordingly. +There is no need to update synapse.storage.schema.SCHEMA_VERSION, since there is no +change to the Synapse codebase here. So we end up with:

    +

    v1.38.0: SCHEMA_VERSION=60, SCHEMA_COMPAT_VERSION=60.

    +
  6. +
+

If in doubt about whether to update SCHEMA_VERSION or not, it is generally best to +lean towards doing so.

+

Full schema dumps

+

In the full_schemas directories, only the most recently-numbered snapshot is used +(54 at the time of writing). Older snapshots (eg, 16) are present for historical +reference only.

+

Building full schema dumps

+

If you want to recreate these schemas, they need to be made from a database that +has had all background updates run.

+

To do so, use scripts-dev/make_full_schema.sh. This will produce new +full.sql.postgres and full.sql.sqlite files.

+

Ensure postgres is installed, then run:

+
./scripts-dev/make_full_schema.sh -p postgres_username -o output_dir/
+
+

NB at the time of writing, this script predates the split into separate state/main +databases so will require updates to handle that correctly.

+ +
+ + +
+
+ + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/develop/development/internal_documentation/index.html b/develop/development/internal_documentation/index.html index d03c1f09cf..98e80c5acb 100644 --- a/develop/development/internal_documentation/index.html +++ b/develop/development/internal_documentation/index.html @@ -99,7 +99,7 @@ diff --git a/develop/federate.html b/develop/federate.html index f626f0f25f..088134818d 100644 --- a/develop/federate.html +++ b/develop/federate.html @@ -99,7 +99,7 @@ diff --git a/develop/jwt.html b/develop/jwt.html index daa785ab88..169775e2da 100644 --- a/develop/jwt.html +++ b/develop/jwt.html @@ -99,7 +99,7 @@ diff --git a/develop/log_contexts.html b/develop/log_contexts.html index 8293831058..baec2f4ee5 100644 --- a/develop/log_contexts.html +++ b/develop/log_contexts.html @@ -99,7 +99,7 @@ @@ -588,7 +588,7 @@ lead to leaked logcontexts which are incredibly hard to track down.