Merge pull request #871 from matrix-org/erikj/linearize_state_fetch_on_pdu

Linearize fetching of gaps on incoming events

synapse/federation/federation_base.py

@@ -31,6 +31,9 @@ logger = logging.getLogger(__name__)
 
 
 class FederationBase(object):
+    def __init__(self, hs):
+        pass
+
     @defer.inlineCallbacks
     def _check_sigs_and_hash_and_fetch(self, origin, pdus, outlier=False,
                                        include_none=False):

synapse/federation/federation_client.py

@@ -52,6 +52,8 @@ sent_queries_counter = metrics.register_counter("sent_queries", labels=["type"])
 
 
 class FederationClient(FederationBase):
+    def __init__(self, hs):
+        super(FederationClient, self).__init__(hs)
 
     def start_get_pdu_cache(self):
         self._get_pdu_cache = ExpiringCache(

synapse/federation/federation_server.py

@@ -19,6 +19,7 @@ from twisted.internet import defer
 from .federation_base import FederationBase
 from .units import Transaction, Edu
 
+from synapse.util.async import Linearizer
 from synapse.util.logutils import log_function
 from synapse.events import FrozenEvent
 import synapse.metrics
@@ -44,6 +45,11 @@ received_queries_counter = metrics.register_counter("received_queries", labels=[
 
 
 class FederationServer(FederationBase):
+    def __init__(self, hs):
+        super(FederationServer, self).__init__(hs)
+
+        self._room_pdu_linearizer = Linearizer()
+
     def set_handler(self, handler):
         """Sets the handler that the replication layer will use to communicate
         receipt of new PDUs from other home servers. The required methods are
@@ -491,43 +497,51 @@ class FederationServer(FederationBase):
                 pdu.internal_metadata.outlier = True
             elif min_depth and pdu.depth > min_depth:
                 if get_missing and prevs - seen:
-                    latest = yield self.store.get_latest_event_ids_in_room(
+                    # If we're missing stuff, ensure we only fetch stuff one
-                        pdu.room_id
+                    # at a time.
-                    )
+                    with (yield self._room_pdu_linearizer.queue(pdu.room_id)):
+                        # We recalculate seen, since it may have changed.
+                        have_seen = yield self.store.have_events(prevs)
+                        seen = set(have_seen.keys())
 
-                    # We add the prev events that we have seen to the latest
+                        if prevs - seen:
-                    # list to ensure the remote server doesn't give them to us
+                            latest = yield self.store.get_latest_event_ids_in_room(
-                    latest = set(latest)
+                                pdu.room_id
-                    latest |= seen
+                            )
 
-                    logger.info(
+                            # We add the prev events that we have seen to the latest
-                        "Missing %d events for room %r: %r...",
+                            # list to ensure the remote server doesn't give them to us
-                        len(prevs - seen), pdu.room_id, list(prevs - seen)[:5]
+                            latest = set(latest)
-                    )
+                            latest |= seen
 
-                    missing_events = yield self.get_missing_events(
+                            logger.info(
-                        origin,
+                                "Missing %d events for room %r: %r...",
-                        pdu.room_id,
+                                len(prevs - seen), pdu.room_id, list(prevs - seen)[:5]
-                        earliest_events_ids=list(latest),
+                            )
-                        latest_events=[pdu],
-                        limit=10,
-                        min_depth=min_depth,
-                    )
 
-                    # We want to sort these by depth so we process them and
+                            missing_events = yield self.get_missing_events(
-                    # tell clients about them in order.
+                                origin,
-                    missing_events.sort(key=lambda x: x.depth)
+                                pdu.room_id,
+                                earliest_events_ids=list(latest),
+                                latest_events=[pdu],
+                                limit=10,
+                                min_depth=min_depth,
+                            )
 
-                    for e in missing_events:
+                            # We want to sort these by depth so we process them and
-                        yield self._handle_new_pdu(
+                            # tell clients about them in order.
-                            origin,
+                            missing_events.sort(key=lambda x: x.depth)
-                            e,
-                            get_missing=False
-                        )
 
-                    have_seen = yield self.store.have_events(
+                            for e in missing_events:
-                        [ev for ev, _ in pdu.prev_events]
+                                yield self._handle_new_pdu(
-                    )
+                                    origin,
+                                    e,
+                                    get_missing=False
+                                )
+
+                            have_seen = yield self.store.have_events(
+                                [ev for ev, _ in pdu.prev_events]
+                            )
 
             prevs = {e_id for e_id, _ in pdu.prev_events}
             seen = set(have_seen.keys())

synapse/federation/replication.py

@@ -72,5 +72,7 @@ class ReplicationLayer(FederationClient, FederationServer):
 
         self.hs = hs
 
+        super(ReplicationLayer, self).__init__(hs)
+
     def __str__(self):
         return "<ReplicationLayer(%s)>" % self.server_name