From b7367c339db153824fb47728d3eebe2f944530e6 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
Date: Mon, 25 Nov 2019 13:26:59 +0000
Subject: [PATCH] Fix exceptions from background database update for event
 labels. (#6407)

Add some exception handling here so that events whose json cannot be parsed are
ignored rather than getting us stuck in a loop.

Fixes #6404.
---
 changelog.d/6407.bugfix                       |  1 +
 .../data_stores/main/events_bg_updates.py     | 41 +++++++++++--------
 2 files changed, 25 insertions(+), 17 deletions(-)
 create mode 100644 changelog.d/6407.bugfix

diff --git a/changelog.d/6407.bugfix b/changelog.d/6407.bugfix
new file mode 100644
index 0000000000..0fdbf2a781
--- /dev/null
+++ b/changelog.d/6407.bugfix
@@ -0,0 +1 @@
+Fix a bug which could cause the background database update hander for event labels to get stuck in a loop raising exceptions.
diff --git a/synapse/storage/data_stores/main/events_bg_updates.py b/synapse/storage/data_stores/main/events_bg_updates.py
index 0ed59ef48e..aa87f9abc5 100644
--- a/synapse/storage/data_stores/main/events_bg_updates.py
+++ b/synapse/storage/data_stores/main/events_bg_updates.py
@@ -530,24 +530,31 @@ class EventsBackgroundUpdatesStore(BackgroundUpdateStore):
             nbrows = 0
             last_row_event_id = ""
             for (event_id, event_json_raw) in results:
-                event_json = json.loads(event_json_raw)
+                try:
+                    event_json = json.loads(event_json_raw)
 
-                self._simple_insert_many_txn(
-                    txn=txn,
-                    table="event_labels",
-                    values=[
-                        {
-                            "event_id": event_id,
-                            "label": label,
-                            "room_id": event_json["room_id"],
-                            "topological_ordering": event_json["depth"],
-                        }
-                        for label in event_json["content"].get(
-                            EventContentFields.LABELS, []
-                        )
-                        if isinstance(label, str)
-                    ],
-                )
+                    self._simple_insert_many_txn(
+                        txn=txn,
+                        table="event_labels",
+                        values=[
+                            {
+                                "event_id": event_id,
+                                "label": label,
+                                "room_id": event_json["room_id"],
+                                "topological_ordering": event_json["depth"],
+                            }
+                            for label in event_json["content"].get(
+                                EventContentFields.LABELS, []
+                            )
+                            if isinstance(label, str)
+                        ],
+                    )
+                except Exception as e:
+                    logger.warning(
+                        "Unable to load event %s (no labels will be imported): %s",
+                        event_id,
+                        e,
+                    )
 
                 nbrows += 1
                 last_row_event_id = event_id