From 1bc9985eb7feca2a8eb4a0125cd03dfa3ac631fe Mon Sep 17 00:00:00 2001 From: David Robertson Date: Wed, 22 Mar 2023 12:53:55 +0000 Subject: [PATCH] Have replication clients remove _INT_STREAM_POS (#15309) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Have replication clients remove _INT_STREAM_POS Suppose worker A makes an internal http request from worker B. B may make changes that A later learns about over replication. We want A's request to block until it has seen those changes—mainly to ensure A's caches are invalidated promptly. This helps provide read-after-write consistency, eliminating entire categories of races and test flakes. To implement this, B includes a top-level field `_INT_STREAM_POS` in its response JSON. Roughly speaking, the field's value tells A what to wait for. But we weren't removing that internal field before A's request completed! Introduced in https://github.com/matrix-org/synapse/pull/14820. Fixes #15308. * Changelog --- changelog.d/15309.bugfix | 1 + synapse/replication/http/_base.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/15309.bugfix diff --git a/changelog.d/15309.bugfix b/changelog.d/15309.bugfix new file mode 100644 index 0000000000..4d3fe4e4b1 --- /dev/null +++ b/changelog.d/15309.bugfix @@ -0,0 +1 @@ +Fix a bug introduced in Synapse 1.76.0 where responses from worker deployments could include an internal `_INT_STREAM_POS` key. diff --git a/synapse/replication/http/_base.py b/synapse/replication/http/_base.py index c20d9c7e9d..8c2c54c07a 100644 --- a/synapse/replication/http/_base.py +++ b/synapse/replication/http/_base.py @@ -345,7 +345,7 @@ class ReplicationEndpoint(metaclass=abc.ABCMeta): _outgoing_request_counter.labels(cls.NAME, 200).inc() # Wait on any streams that the remote may have written to. - for stream_name, position in result.get( + for stream_name, position in result.pop( _STREAM_POSITION_KEY, {} ).items(): await replication.wait_for_stream_position(