diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 5355bfa208..193cb505c3 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -19,6 +19,14 @@ jobs:
       - run: scripts-dev/generate_sample_config.sh --check
       - run: scripts-dev/config-lint.sh
 
+  check-schema-delta:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+      - run: "pip install 'click==8.1.1' 'GitPython>=3.1.20'"
+      - run: scripts-dev/check_schema_delta.py --force-colors
+
   lint:
     uses: "matrix-org/backend-meta/.github/workflows/python-poetry-ci.yml@v1"
     with:
@@ -48,7 +56,7 @@ jobs:
   # Dummy step to gate other tests on without repeating the whole list
   linting-done:
     if: ${{ !cancelled() }} # Run this even if prior jobs were skipped
-    needs: [lint, lint-crlf, lint-newsfile, check-sampleconfig]
+    needs: [lint, lint-crlf, lint-newsfile, check-sampleconfig, check-schema-delta]
     runs-on: ubuntu-latest
     steps:
       - run: "true"
diff --git a/changelog.d/13063.misc b/changelog.d/13063.misc
new file mode 100644
index 0000000000..167d6d2cd5
--- /dev/null
+++ b/changelog.d/13063.misc
@@ -0,0 +1 @@
+Add a CI job to check that schema deltas are in the correct folder.
diff --git a/poetry.lock b/poetry.lock
index 8a54a939fe..6a67f59bca 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -139,7 +139,7 @@ unicode_backport = ["unicodedata2"]
 
 [[package]]
 name = "click"
-version = "8.1.0"
+version = "8.1.1"
 description = "Composable command line interface toolkit"
 category = "dev"
 optional = false
@@ -1563,7 +1563,7 @@ url_preview = ["lxml"]
 [metadata]
 lock-version = "1.1"
 python-versions = "^3.7.1"
-content-hash = "c1bb4dabba1e87517e25ca7bf778e8082fbc960a51d83819aec3a154110a374f"
+content-hash = "37bd4bccfdb5a869635f2135a85bea4a0729af7375a27de153b4fd9a4aebc195"
 
 [metadata.files]
 attrs = [
@@ -1684,8 +1684,8 @@ charset-normalizer = [
     {file = "charset_normalizer-2.0.12-py3-none-any.whl", hash = "sha256:6881edbebdb17b39b4eaaa821b438bf6eddffb4468cf344f09f89def34a8b1df"},
 ]
 click = [
-    {file = "click-8.1.0-py3-none-any.whl", hash = "sha256:19a4baa64da924c5e0cd889aba8e947f280309f1a2ce0947a3e3a7bcb7cc72d6"},
-    {file = "click-8.1.0.tar.gz", hash = "sha256:977c213473c7665d3aa092b41ff12063227751c41d7b17165013e10069cc5cd2"},
+    {file = "click-8.1.1-py3-none-any.whl", hash = "sha256:5e0d195c2067da3136efb897449ec1e9e6c98282fbf30d7f9e164af9be901a6b"},
+    {file = "click-8.1.1.tar.gz", hash = "sha256:7ab900e38149c9872376e8f9b5986ddcaf68c0f413cf73678a0bca5547e6f976"},
 ]
 click-default-group = [
     {file = "click-default-group-1.2.2.tar.gz", hash = "sha256:d9560e8e8dfa44b3562fbc9425042a0fd6d21956fcc2db0077f63f34253ab904"},
diff --git a/pyproject.toml b/pyproject.toml
index 3c64e248ab..85c2c9534f 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -272,7 +272,7 @@ parameterized = ">=0.7.4"
 idna = ">=2.5"
 
 # The following are used by the release script
-click = "==8.1.0"
+click = "==8.1.1"
 # GitPython was == 3.1.14; bumped to 3.1.20, the first release with type hints.
 GitPython = ">=3.1.20"
 commonmark = "==0.9.1"
diff --git a/scripts-dev/check_schema_delta.py b/scripts-dev/check_schema_delta.py
new file mode 100755
index 0000000000..32fe7f50de
--- /dev/null
+++ b/scripts-dev/check_schema_delta.py
@@ -0,0 +1,111 @@
+#!/usr/bin/env python3
+
+# Check that no schema deltas have been added to the wrong version.
+
+import re
+from typing import Any, Dict, List
+
+import click
+import git
+
+SCHEMA_FILE_REGEX = re.compile(r"^synapse/storage/schema/(.*)/delta/(.*)/(.*)$")
+
+
+@click.command()
+@click.option(
+    "--force-colors",
+    is_flag=True,
+    flag_value=True,
+    default=None,
+    help="Always output ANSI colours",
+)
+def main(force_colors: bool) -> None:
+    click.secho(
+        "+++ Checking schema deltas are in the right folder",
+        fg="green",
+        bold=True,
+        color=force_colors,
+    )
+
+    click.secho("Updating repo...")
+
+    repo = git.Repo()
+    repo.remote().fetch()
+
+    click.secho("Getting current schema version...")
+
+    r = repo.git.show("origin/develop:synapse/storage/schema/__init__.py")
+
+    locals: Dict[str, Any] = {}
+    exec(r, locals)
+    current_schema_version = locals["SCHEMA_VERSION"]
+
+    click.secho(f"Current schema version: {current_schema_version}")
+
+    diffs: List[git.Diff] = repo.remote().refs.develop.commit.diff(None)
+
+    seen_deltas = False
+    bad_files = []
+    for diff in diffs:
+        if not diff.new_file or diff.b_path is None:
+            continue
+
+        match = SCHEMA_FILE_REGEX.match(diff.b_path)
+        if not match:
+            continue
+
+        seen_deltas = True
+
+        _, delta_version, _ = match.groups()
+
+        if delta_version != str(current_schema_version):
+            bad_files.append(diff.b_path)
+
+    if not seen_deltas:
+        click.secho(
+            "No deltas found.",
+            fg="green",
+            bold=True,
+            color=force_colors,
+        )
+        return
+
+    if not bad_files:
+        click.secho(
+            f"All deltas are in the correct folder: {current_schema_version}!",
+            fg="green",
+            bold=True,
+            color=force_colors,
+        )
+        return
+
+    bad_files.sort()
+
+    click.secho(
+        "Found deltas in the wrong folder!",
+        fg="red",
+        bold=True,
+        color=force_colors,
+    )
+
+    for f in bad_files:
+        click.secho(
+            f"\t{f}",
+            fg="red",
+            bold=True,
+            color=force_colors,
+        )
+
+    click.secho()
+    click.secho(
+        f"Please move these files to delta/{current_schema_version}/",
+        fg="red",
+        bold=True,
+        color=force_colors,
+    )
+
+    click.get_current_context().exit(1)
+
+
+if __name__ == "__main__":
+    main()