diff --git a/changelog.d/11632.bugfix b/changelog.d/11632.bugfix
new file mode 100644
index 0000000000..c73d41652a
--- /dev/null
+++ b/changelog.d/11632.bugfix
@@ -0,0 +1 @@
+Fix a bug introduced in Synapse 1.19.3 which could sometimes cause `AssertionError`s when backfilling rooms over federation.
diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py
index fee1477ab6..7353c2b6b1 100644
--- a/synapse/federation/federation_client.py
+++ b/synapse/federation/federation_client.py
@@ -272,7 +272,7 @@ class FederationClient(FederationBase):
 
         # Check signatures and hash of pdus, removing any from the list that fail checks
         pdus[:] = await self._check_sigs_and_hash_and_fetch(
-            dest, pdus, outlier=True, room_version=room_version
+            dest, pdus, room_version=room_version
         )
 
         return pdus
diff --git a/synapse/handlers/federation_event.py b/synapse/handlers/federation_event.py
index d08e48da58..7c81e3651f 100644
--- a/synapse/handlers/federation_event.py
+++ b/synapse/handlers/federation_event.py
@@ -666,7 +666,9 @@ class FederationEventHandler:
         logger.info("Processing pulled event %s", event)
 
         # these should not be outliers.
-        assert not event.internal_metadata.is_outlier()
+        assert (
+            not event.internal_metadata.is_outlier()
+        ), "pulled event unexpectedly flagged as outlier"
 
         event_id = event.event_id