Merge pull request #3521 from matrix-org/rav/optimise_stream_change_cache

Reduce set building in get_entities_changed

changelog.d/3521.feature

@@ -0,0 +1 @@
+Cache optimisation for /sync requests

synapse/util/caches/stream_change_cache.py

@@ -74,14 +74,18 @@ class StreamChangeCache(object):
         assert type(stream_pos) is int
 
         if stream_pos >= self._earliest_known_stream_pos:
-            not_known_entities = set(entities) - set(self._entity_to_key)
+            changed_entities = {
-
+                self._cache[k] for k in self._cache.islice(
-            result = (
+                    start=self._cache.bisect_right(stream_pos),
-                {self._cache[k] for k in self._cache.islice(
-                    start=self._cache.bisect_right(stream_pos))}
-                .intersection(entities)
-                .union(not_known_entities)
                 )
+            }
+
+            # we need to include entities which we don't know about, as well as
+            # those which are known to have changed since the stream pos.
+            result = {
+                e for e in entities
+                if e in changed_entities or e not in self._entity_to_key
+            }
 
             self.metrics.inc_hits()
         else: