From 499d4a32cda7bc26d82140d84a7279372c519538 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 17 Jun 2019 16:20:20 +0100 Subject: [PATCH 1/7] Add metrics for len of new extremities persisted. Of new events being persisted add metrics for total size of forward extremities and number of unchanged, "stale" extremities. --- synapse/storage/events.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index f631fb1733..f2374bc3f4 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -24,7 +24,7 @@ from six import iteritems, text_type from six.moves import range from canonicaljson import json -from prometheus_client import Counter +from prometheus_client import Counter, Histogram from twisted.internet import defer @@ -74,6 +74,19 @@ state_delta_reuse_delta_counter = Counter( "synapse_storage_events_state_delta_reuse_delta", "" ) +# The number of forward extremities for each new event. +forward_extremities_counter = Histogram( + "synapse_storage_events_forward_extremities_persisted", "", + buckets=(1, 2, 3, 5, 7, 10, 15, 20, 50, 100, 200, 500, "+Inf") +) + +# The number of stale forward extremities for each new event. Stale extremities +# are those that were in the previous set of extremities as well as the new. +stale_forward_extremities_counter = Histogram( + "synapse_storage_events_stale_forward_extremities_persisted", "", + buckets=(0, 1, 2, 3, 5, 7, 10, 15, 20, 50, 100, 200, 500, "+Inf") +) + def encode_json(json_object): """ @@ -541,6 +554,8 @@ class EventsStore( and not event.internal_metadata.is_soft_failed() ] + latest_event_ids = set(latest_event_ids) + # start with the existing forward extremities result = set(latest_event_ids) @@ -564,6 +579,13 @@ class EventsStore( ) result.difference_update(existing_prevs) + # We only update metrics for events that change forward extremities + # (e.g. we ignore backfill/outliers/etc) + if result != latest_event_ids: + forward_extremities_counter.observe(len(result)) + stale = set(latest_event_ids) & result + stale_forward_extremities_counter.observe(len(stale)) + defer.returnValue(result) @defer.inlineCallbacks From ff88d36dcb3113b14a6c8d8fe4b3db467c38c06e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 17 Jun 2019 16:29:00 +0100 Subject: [PATCH 2/7] Add metric fo number of state groups in resolution --- synapse/state/__init__.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/synapse/state/__init__.py b/synapse/state/__init__.py index 36684ef9f6..3415a34367 100644 --- a/synapse/state/__init__.py +++ b/synapse/state/__init__.py @@ -21,6 +21,7 @@ from six import iteritems, itervalues import attr from frozendict import frozendict +from prometheus_client import Histogram from twisted.internet import defer @@ -37,6 +38,13 @@ from synapse.util.metrics import Measure logger = logging.getLogger(__name__) +# Metrics for number of state groups involved in a resolution. +state_groups_histogram = Histogram( + "synapse_state_number_state_groups", "", + buckets=(0, 1, 2, 3, 5, 7, 10, 20, 50, 100, "+Inf"), +) + + KeyStateTuple = namedtuple("KeyStateTuple", ("context", "type", "state_key")) @@ -364,6 +372,8 @@ class StateHandler(object): room_id, event_ids ) + state_groups_histogram.observe(len(state_groups_ids)) + if len(state_groups_ids) == 0: defer.returnValue(_StateCacheEntry( state={}, From 4c4877bbc1dc73cd51caa7da53218509dabdb8b2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 17 Jun 2019 16:23:26 +0100 Subject: [PATCH 3/7] Newsfile --- changelog.d/5476.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/5476.misc diff --git a/changelog.d/5476.misc b/changelog.d/5476.misc new file mode 100644 index 0000000000..7955c14744 --- /dev/null +++ b/changelog.d/5476.misc @@ -0,0 +1 @@ +Add new metrics for number of forward extremities being persisted and number of state groups involved in resolution. From c9385dd23874261dd8f01da1d659afa33f552a83 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 18 Jun 2019 12:43:41 +0100 Subject: [PATCH 4/7] Use consistent buckets --- synapse/state/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/state/__init__.py b/synapse/state/__init__.py index 3415a34367..8099297cfe 100644 --- a/synapse/state/__init__.py +++ b/synapse/state/__init__.py @@ -41,7 +41,7 @@ logger = logging.getLogger(__name__) # Metrics for number of state groups involved in a resolution. state_groups_histogram = Histogram( "synapse_state_number_state_groups", "", - buckets=(0, 1, 2, 3, 5, 7, 10, 20, 50, 100, "+Inf"), + buckets=(0, 1, 2, 3, 5, 7, 10, 15, 20, 50, 100, 200, 500, "+Inf"), ) From 16a3124b767a851e219777464a694b04dcc1aae7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 18 Jun 2019 13:01:52 +0100 Subject: [PATCH 5/7] Only count non-cache state resolution --- synapse/state/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/state/__init__.py b/synapse/state/__init__.py index 8099297cfe..b914a5ba03 100644 --- a/synapse/state/__init__.py +++ b/synapse/state/__init__.py @@ -41,7 +41,7 @@ logger = logging.getLogger(__name__) # Metrics for number of state groups involved in a resolution. state_groups_histogram = Histogram( "synapse_state_number_state_groups", "", - buckets=(0, 1, 2, 3, 5, 7, 10, 15, 20, 50, 100, 200, 500, "+Inf"), + buckets=(1, 2, 3, 5, 7, 10, 15, 20, 50, 100, 200, 500, "+Inf"), ) @@ -372,8 +372,6 @@ class StateHandler(object): room_id, event_ids ) - state_groups_histogram.observe(len(state_groups_ids)) - if len(state_groups_ids) == 0: defer.returnValue(_StateCacheEntry( state={}, @@ -498,6 +496,8 @@ class StateResolutionHandler(object): "Resolving state for %s with %d groups", room_id, len(state_groups_ids) ) + state_groups_histogram.observe(len(state_groups_ids)) + # start by assuming we won't have any conflicted state, and build up the new # state map by iterating through the state groups. If we discover a conflict, # we give up and instead use `resolve_events_with_store`. From 65787b0f7c8b1c1725da63ceeca4d3e25607af1f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 19 Jun 2019 11:49:39 +0100 Subject: [PATCH 6/7] Add descriptions and remove redundant set(..) --- synapse/state/__init__.py | 3 ++- synapse/storage/events.py | 8 +++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/synapse/state/__init__.py b/synapse/state/__init__.py index b914a5ba03..4b136b3054 100644 --- a/synapse/state/__init__.py +++ b/synapse/state/__init__.py @@ -40,7 +40,8 @@ logger = logging.getLogger(__name__) # Metrics for number of state groups involved in a resolution. state_groups_histogram = Histogram( - "synapse_state_number_state_groups", "", + "synapse_state_number_state_groups_in_resolution", + "Number of state groups used when performing a state resolution", buckets=(1, 2, 3, 5, 7, 10, 15, 20, 50, 100, 200, 500, "+Inf"), ) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index f2374bc3f4..9903786c48 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -76,14 +76,16 @@ state_delta_reuse_delta_counter = Counter( # The number of forward extremities for each new event. forward_extremities_counter = Histogram( - "synapse_storage_events_forward_extremities_persisted", "", + "synapse_storage_events_forward_extremities_persisted", + "Number of forward extremities for each new event", buckets=(1, 2, 3, 5, 7, 10, 15, 20, 50, 100, 200, 500, "+Inf") ) # The number of stale forward extremities for each new event. Stale extremities # are those that were in the previous set of extremities as well as the new. stale_forward_extremities_counter = Histogram( - "synapse_storage_events_stale_forward_extremities_persisted", "", + "synapse_storage_events_stale_forward_extremities_persisted", + "Number of unchanged forward extremities for each new event", buckets=(0, 1, 2, 3, 5, 7, 10, 15, 20, 50, 100, 200, 500, "+Inf") ) @@ -583,7 +585,7 @@ class EventsStore( # (e.g. we ignore backfill/outliers/etc) if result != latest_event_ids: forward_extremities_counter.observe(len(result)) - stale = set(latest_event_ids) & result + stale = latest_event_ids & result stale_forward_extremities_counter.observe(len(stale)) defer.returnValue(result) From f8bd30af2a02ce6956d6f3070a98a3d75e0f1f33 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 20 Jun 2019 13:10:06 +0100 Subject: [PATCH 7/7] Black --- synapse/storage/events.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 934310f637..fefba39ea1 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -78,7 +78,7 @@ state_delta_reuse_delta_counter = Counter( forward_extremities_counter = Histogram( "synapse_storage_events_forward_extremities_persisted", "Number of forward extremities for each new event", - buckets=(1, 2, 3, 5, 7, 10, 15, 20, 50, 100, 200, 500, "+Inf") + buckets=(1, 2, 3, 5, 7, 10, 15, 20, 50, 100, 200, 500, "+Inf"), ) # The number of stale forward extremities for each new event. Stale extremities @@ -86,7 +86,7 @@ forward_extremities_counter = Histogram( stale_forward_extremities_counter = Histogram( "synapse_storage_events_stale_forward_extremities_persisted", "Number of unchanged forward extremities for each new event", - buckets=(0, 1, 2, 3, 5, 7, 10, 15, 20, 50, 100, 200, 500, "+Inf") + buckets=(0, 1, 2, 3, 5, 7, 10, 15, 20, 50, 100, 200, 500, "+Inf"), )