From b31daac01c9cc757d7c73da2f23e1b7251c54b79 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Fri, 11 Jun 2021 04:12:35 -0500 Subject: [PATCH] Add metrics to track how often events are `soft_failed` (#10156) Spawned from missing messages we were seeing on `matrix.org` from a federated Gtiter bridged room, https://gitlab.com/gitterHQ/webapp/-/issues/2770. The underlying issue in Synapse is tracked by https://github.com/matrix-org/synapse/issues/10066 where the message and join event race and the message is `soft_failed` before the `join` event reaches the remote federated server. Less soft_failed events = better and usually this should only trigger for events where people are doing bad things and trying to fuzz and fake everything. --- changelog.d/10156.misc | 1 + synapse/handlers/federation.py | 7 +++++++ 2 files changed, 8 insertions(+) create mode 100644 changelog.d/10156.misc diff --git a/changelog.d/10156.misc b/changelog.d/10156.misc new file mode 100644 index 0000000000..92a188b87b --- /dev/null +++ b/changelog.d/10156.misc @@ -0,0 +1 @@ +Add `synapse_federation_soft_failed_events_total` metric to track how often events are soft failed. diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 6e40e2c216..6647063485 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -33,6 +33,7 @@ from typing import ( ) import attr +from prometheus_client import Counter from signedjson.key import decode_verify_key_bytes from signedjson.sign import verify_signed_json from unpaddedbase64 import decode_base64 @@ -101,6 +102,11 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) +soft_failed_event_counter = Counter( + "synapse_federation_soft_failed_events_total", + "Events received over federation that we marked as soft_failed", +) + @attr.s(slots=True) class _NewEventInfo: @@ -2498,6 +2504,7 @@ class FederationHandler(BaseHandler): event_auth.check(room_version_obj, event, auth_events=current_auth_events) except AuthError as e: logger.warning("Soft-failing %r because %s", event, e) + soft_failed_event_counter.inc() event.internal_metadata.soft_failed = True async def on_get_missing_events(