2015-03-20 07:52:56 -06:00
|
|
|
# -*- coding: utf-8 -*-
|
2016-01-06 21:26:29 -07:00
|
|
|
# Copyright 2014-2016 OpenMarket Ltd
|
2018-02-23 03:52:10 -07:00
|
|
|
# Copyright 2018 New Vector Ltd
|
2015-03-20 07:52:56 -06:00
|
|
|
#
|
|
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
# you may not use this file except in compliance with the License.
|
|
|
|
# You may obtain a copy of the License at
|
|
|
|
#
|
|
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
#
|
|
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
# See the License for the specific language governing permissions and
|
|
|
|
# limitations under the License.
|
|
|
|
|
2018-04-17 11:30:53 -06:00
|
|
|
import itertools
|
2018-03-29 15:57:28 -06:00
|
|
|
import logging
|
2018-07-09 00:09:20 -06:00
|
|
|
from collections import OrderedDict, deque, namedtuple
|
|
|
|
from functools import wraps
|
|
|
|
|
2018-08-30 08:19:58 -06:00
|
|
|
from six import iteritems, text_type
|
2018-07-09 00:09:20 -06:00
|
|
|
from six.moves import range
|
2015-03-20 07:52:56 -06:00
|
|
|
|
2018-06-28 07:49:57 -06:00
|
|
|
from canonicaljson import json
|
2018-07-09 00:09:20 -06:00
|
|
|
from prometheus_client import Counter
|
2018-06-28 07:49:57 -06:00
|
|
|
|
2018-02-23 04:01:21 -07:00
|
|
|
from twisted.internet import defer
|
|
|
|
|
2018-07-09 00:09:20 -06:00
|
|
|
import synapse.metrics
|
|
|
|
from synapse.api.constants import EventTypes
|
|
|
|
from synapse.api.errors import SynapseError
|
|
|
|
# these are only included to make the type annotations work
|
|
|
|
from synapse.events import EventBase # noqa: F401
|
|
|
|
from synapse.events.snapshot import EventContext # noqa: F401
|
2018-07-18 07:35:24 -06:00
|
|
|
from synapse.metrics.background_process_metrics import run_as_background_process
|
2018-09-03 08:13:17 -06:00
|
|
|
from synapse.state import StateResolutionStore
|
2018-07-31 06:11:04 -06:00
|
|
|
from synapse.storage.background_updates import BackgroundUpdateStore
|
2018-07-26 05:48:51 -06:00
|
|
|
from synapse.storage.event_federation import EventFederationStore
|
2018-03-29 15:57:28 -06:00
|
|
|
from synapse.storage.events_worker import EventsWorkerStore
|
2018-07-09 00:09:20 -06:00
|
|
|
from synapse.types import RoomStreamToken, get_domain_from_id
|
2018-10-02 16:33:29 -06:00
|
|
|
from synapse.util import batch_iter
|
2018-08-10 07:50:21 -06:00
|
|
|
from synapse.util.async_helpers import ObservableDeferred
|
2018-07-09 00:09:20 -06:00
|
|
|
from synapse.util.caches.descriptors import cached, cachedInlineCallbacks
|
2018-03-29 15:57:28 -06:00
|
|
|
from synapse.util.frozenutils import frozendict_json_encoder
|
2018-07-23 17:37:17 -06:00
|
|
|
from synapse.util.logcontext import PreserveLoggingContext, make_deferred_yieldable
|
2015-03-20 07:52:56 -06:00
|
|
|
from synapse.util.logutils import log_function
|
2016-08-19 10:40:23 -06:00
|
|
|
from synapse.util.metrics import Measure
|
2015-03-20 07:52:56 -06:00
|
|
|
|
2018-05-21 18:48:57 -06:00
|
|
|
logger = logging.getLogger(__name__)
|
2015-03-20 07:52:56 -06:00
|
|
|
|
2018-05-21 18:48:57 -06:00
|
|
|
persist_event_counter = Counter("synapse_storage_events_persisted_events", "")
|
2018-05-22 16:32:57 -06:00
|
|
|
event_counter = Counter("synapse_storage_events_persisted_events_sep", "",
|
|
|
|
["type", "origin_type", "origin_entity"])
|
2018-03-27 06:13:38 -06:00
|
|
|
|
|
|
|
# The number of times we are recalculating the current state
|
2018-05-21 18:48:57 -06:00
|
|
|
state_delta_counter = Counter("synapse_storage_events_state_delta", "")
|
|
|
|
|
2018-03-27 06:13:38 -06:00
|
|
|
# The number of times we are recalculating state when there is only a
|
|
|
|
# single forward extremity
|
2018-05-22 16:32:57 -06:00
|
|
|
state_delta_single_event_counter = Counter(
|
|
|
|
"synapse_storage_events_state_delta_single_event", "")
|
2018-05-21 18:48:57 -06:00
|
|
|
|
2018-03-27 06:13:38 -06:00
|
|
|
# The number of times we are reculating state when we could have resonably
|
|
|
|
# calculated the delta when we calculated the state for an event we were
|
|
|
|
# persisting.
|
2018-05-22 16:32:57 -06:00
|
|
|
state_delta_reuse_delta_counter = Counter(
|
|
|
|
"synapse_storage_events_state_delta_reuse_delta", "")
|
2016-06-06 04:58:09 -06:00
|
|
|
|
|
|
|
|
2015-08-24 09:17:38 -06:00
|
|
|
def encode_json(json_object):
|
2018-08-01 08:54:06 -06:00
|
|
|
"""
|
|
|
|
Encode a Python object as JSON and return it in a Unicode string.
|
|
|
|
"""
|
|
|
|
out = frozendict_json_encoder.encode(json_object)
|
|
|
|
if isinstance(out, bytes):
|
|
|
|
out = out.decode('utf8')
|
|
|
|
return out
|
2015-08-24 09:17:38 -06:00
|
|
|
|
2016-11-15 04:22:29 -07:00
|
|
|
|
2016-05-06 07:31:38 -06:00
|
|
|
class _EventPeristenceQueue(object):
|
|
|
|
"""Queues up events so that they can be persisted in bulk with only one
|
|
|
|
concurrent transaction per room.
|
|
|
|
"""
|
|
|
|
|
|
|
|
_EventPersistQueueItem = namedtuple("_EventPersistQueueItem", (
|
2017-01-20 04:52:51 -07:00
|
|
|
"events_and_contexts", "backfilled", "deferred",
|
2016-05-06 07:31:38 -06:00
|
|
|
))
|
|
|
|
|
|
|
|
def __init__(self):
|
|
|
|
self._event_persist_queues = {}
|
|
|
|
self._currently_persisting_rooms = set()
|
|
|
|
|
2017-01-20 04:52:51 -07:00
|
|
|
def add_to_queue(self, room_id, events_and_contexts, backfilled):
|
2016-05-06 07:31:38 -06:00
|
|
|
"""Add events to the queue, with the given persist_event options.
|
2017-03-17 05:51:13 -06:00
|
|
|
|
2017-10-17 03:59:30 -06:00
|
|
|
NB: due to the normal usage pattern of this method, it does *not*
|
|
|
|
follow the synapse logcontext rules, and leaves the logcontext in
|
|
|
|
place whether or not the returned deferred is ready.
|
|
|
|
|
2017-03-17 05:51:13 -06:00
|
|
|
Args:
|
|
|
|
room_id (str):
|
|
|
|
events_and_contexts (list[(EventBase, EventContext)]):
|
|
|
|
backfilled (bool):
|
2017-10-17 03:59:30 -06:00
|
|
|
|
|
|
|
Returns:
|
|
|
|
defer.Deferred: a deferred which will resolve once the events are
|
|
|
|
persisted. Runs its callbacks *without* a logcontext.
|
2016-05-06 07:31:38 -06:00
|
|
|
"""
|
|
|
|
queue = self._event_persist_queues.setdefault(room_id, deque())
|
|
|
|
if queue:
|
2017-10-17 03:59:30 -06:00
|
|
|
# if the last item in the queue has the same `backfilled` setting,
|
|
|
|
# we can just add these new events to that item.
|
2016-05-06 07:31:38 -06:00
|
|
|
end_item = queue[-1]
|
|
|
|
if end_item.backfilled == backfilled:
|
|
|
|
end_item.events_and_contexts.extend(events_and_contexts)
|
|
|
|
return end_item.deferred.observe()
|
|
|
|
|
2018-01-27 07:00:11 -07:00
|
|
|
deferred = ObservableDeferred(defer.Deferred(), consumeErrors=True)
|
2016-05-06 07:31:38 -06:00
|
|
|
|
|
|
|
queue.append(self._EventPersistQueueItem(
|
|
|
|
events_and_contexts=events_and_contexts,
|
|
|
|
backfilled=backfilled,
|
|
|
|
deferred=deferred,
|
|
|
|
))
|
|
|
|
|
|
|
|
return deferred.observe()
|
|
|
|
|
2016-05-06 08:38:42 -06:00
|
|
|
def handle_queue(self, room_id, per_item_callback):
|
2016-05-06 07:31:38 -06:00
|
|
|
"""Attempts to handle the queue for a room if not already being handled.
|
|
|
|
|
2017-10-17 03:59:30 -06:00
|
|
|
The given callback will be invoked with for each item in the queue,
|
2016-05-06 08:38:42 -06:00
|
|
|
of type _EventPersistQueueItem. The per_item_callback will continuously
|
|
|
|
be called with new items, unless the queue becomnes empty. The return
|
|
|
|
value of the function will be given to the deferreds waiting on the item,
|
2017-10-17 03:59:30 -06:00
|
|
|
exceptions will be passed to the deferreds as well.
|
2016-05-06 07:31:38 -06:00
|
|
|
|
|
|
|
This function should therefore be called whenever anything is added
|
|
|
|
to the queue.
|
|
|
|
|
|
|
|
If another callback is currently handling the queue then it will not be
|
|
|
|
invoked.
|
|
|
|
"""
|
|
|
|
|
|
|
|
if room_id in self._currently_persisting_rooms:
|
|
|
|
return
|
|
|
|
|
|
|
|
self._currently_persisting_rooms.add(room_id)
|
|
|
|
|
2016-05-06 08:38:42 -06:00
|
|
|
@defer.inlineCallbacks
|
|
|
|
def handle_queue_loop():
|
|
|
|
try:
|
|
|
|
queue = self._get_drainining_queue(room_id)
|
|
|
|
for item in queue:
|
|
|
|
try:
|
|
|
|
ret = yield per_item_callback(item)
|
2018-07-25 03:47:57 -06:00
|
|
|
except Exception:
|
|
|
|
with PreserveLoggingContext():
|
|
|
|
item.deferred.errback()
|
|
|
|
else:
|
2018-07-23 17:37:17 -06:00
|
|
|
with PreserveLoggingContext():
|
|
|
|
item.deferred.callback(ret)
|
2016-05-06 08:38:42 -06:00
|
|
|
finally:
|
|
|
|
queue = self._event_persist_queues.pop(room_id, None)
|
|
|
|
if queue:
|
|
|
|
self._event_persist_queues[room_id] = queue
|
|
|
|
self._currently_persisting_rooms.discard(room_id)
|
|
|
|
|
2018-07-18 07:35:24 -06:00
|
|
|
# set handle_queue_loop off in the background
|
|
|
|
run_as_background_process("persist_events", handle_queue_loop)
|
2016-05-06 07:31:38 -06:00
|
|
|
|
|
|
|
def _get_drainining_queue(self, room_id):
|
2016-05-06 08:38:42 -06:00
|
|
|
queue = self._event_persist_queues.setdefault(room_id, deque())
|
2016-05-06 07:31:38 -06:00
|
|
|
|
|
|
|
try:
|
|
|
|
while True:
|
|
|
|
yield queue.popleft()
|
|
|
|
except IndexError:
|
|
|
|
# Queue has been drained.
|
|
|
|
pass
|
|
|
|
|
|
|
|
|
2016-06-03 10:55:32 -06:00
|
|
|
_EventCacheEntry = namedtuple("_EventCacheEntry", ("event", "redacted_event"))
|
|
|
|
|
|
|
|
|
2016-08-04 08:02:15 -06:00
|
|
|
def _retry_on_integrity_error(func):
|
|
|
|
"""Wraps a database function so that it gets retried on IntegrityError,
|
|
|
|
with `delete_existing=True` passed in.
|
|
|
|
|
|
|
|
Args:
|
|
|
|
func: function that returns a Deferred and accepts a `delete_existing` arg
|
|
|
|
"""
|
|
|
|
@wraps(func)
|
|
|
|
@defer.inlineCallbacks
|
|
|
|
def f(self, *args, **kwargs):
|
|
|
|
try:
|
|
|
|
res = yield func(self, *args, **kwargs)
|
|
|
|
except self.database_engine.module.IntegrityError:
|
|
|
|
logger.exception("IntegrityError, retrying.")
|
|
|
|
res = yield func(self, *args, delete_existing=True, **kwargs)
|
2016-08-04 08:21:29 -06:00
|
|
|
defer.returnValue(res)
|
2016-08-04 08:02:15 -06:00
|
|
|
|
|
|
|
return f
|
|
|
|
|
|
|
|
|
2018-07-26 05:48:51 -06:00
|
|
|
# inherits from EventFederationStore so that we can call _update_backward_extremities
|
|
|
|
# and _handle_mult_prev_events (though arguably those could both be moved in here)
|
2018-07-31 06:11:04 -06:00
|
|
|
class EventsStore(EventFederationStore, EventsWorkerStore, BackgroundUpdateStore):
|
2015-11-30 10:45:31 -07:00
|
|
|
EVENT_ORIGIN_SERVER_TS_NAME = "event_origin_server_ts"
|
2016-07-14 08:15:22 -06:00
|
|
|
EVENT_FIELDS_SENDER_URL_UPDATE_NAME = "event_fields_sender_url"
|
2015-11-30 10:45:31 -07:00
|
|
|
|
2017-11-09 11:51:27 -07:00
|
|
|
def __init__(self, db_conn, hs):
|
|
|
|
super(EventsStore, self).__init__(db_conn, hs)
|
2015-11-30 10:45:31 -07:00
|
|
|
self.register_background_update_handler(
|
|
|
|
self.EVENT_ORIGIN_SERVER_TS_NAME, self._background_reindex_origin_server_ts
|
|
|
|
)
|
2016-07-14 08:15:22 -06:00
|
|
|
self.register_background_update_handler(
|
|
|
|
self.EVENT_FIELDS_SENDER_URL_UPDATE_NAME,
|
|
|
|
self._background_reindex_fields_sender,
|
|
|
|
)
|
2015-11-30 10:45:31 -07:00
|
|
|
|
2016-09-12 09:57:05 -06:00
|
|
|
self.register_background_index_update(
|
|
|
|
"event_contains_url_index",
|
|
|
|
index_name="event_contains_url_index",
|
|
|
|
table="events",
|
|
|
|
columns=["room_id", "topological_ordering", "stream_ordering"],
|
|
|
|
where_clause="contains_url = true AND outlier = false",
|
|
|
|
)
|
|
|
|
|
2017-05-11 04:57:02 -06:00
|
|
|
# an event_id index on event_search is useful for the purge_history
|
|
|
|
# api. Plus it means we get to enforce some integrity with a UNIQUE
|
|
|
|
# clause
|
|
|
|
self.register_background_index_update(
|
|
|
|
"event_search_event_id_idx",
|
|
|
|
index_name="event_search_event_id_idx",
|
|
|
|
table="event_search",
|
|
|
|
columns=["event_id"],
|
|
|
|
unique=True,
|
2017-05-11 05:46:55 -06:00
|
|
|
psql_only=True,
|
2017-05-11 04:57:02 -06:00
|
|
|
)
|
|
|
|
|
2016-05-06 07:31:38 -06:00
|
|
|
self._event_persist_queue = _EventPeristenceQueue()
|
|
|
|
|
2018-01-30 04:06:15 -07:00
|
|
|
self._state_resolution_handler = hs.get_state_resolution_handler()
|
|
|
|
|
2018-07-25 09:00:38 -06:00
|
|
|
@defer.inlineCallbacks
|
2016-03-31 08:00:42 -06:00
|
|
|
def persist_events(self, events_and_contexts, backfilled=False):
|
2016-04-06 08:42:15 -06:00
|
|
|
"""
|
|
|
|
Write events to the database
|
|
|
|
Args:
|
|
|
|
events_and_contexts: list of tuples of (event, context)
|
2018-07-25 09:00:38 -06:00
|
|
|
backfilled (bool): Whether the results are retrieved from federation
|
|
|
|
via backfill or not. Used to determine if they're "new" events
|
|
|
|
which might update the current state etc.
|
|
|
|
|
|
|
|
Returns:
|
2018-08-01 06:39:14 -06:00
|
|
|
Deferred[int]: the stream ordering of the latest persisted event
|
2016-05-06 07:31:38 -06:00
|
|
|
"""
|
|
|
|
partitioned = {}
|
|
|
|
for event, ctx in events_and_contexts:
|
|
|
|
partitioned.setdefault(event.room_id, []).append((event, ctx))
|
2016-04-06 08:42:15 -06:00
|
|
|
|
2016-05-06 07:31:38 -06:00
|
|
|
deferreds = []
|
2018-05-31 03:03:47 -06:00
|
|
|
for room_id, evs_ctxs in iteritems(partitioned):
|
2017-10-17 03:59:30 -06:00
|
|
|
d = self._event_persist_queue.add_to_queue(
|
2016-05-06 07:31:38 -06:00
|
|
|
room_id, evs_ctxs,
|
|
|
|
backfilled=backfilled,
|
|
|
|
)
|
|
|
|
deferreds.append(d)
|
2016-04-06 08:42:15 -06:00
|
|
|
|
2017-03-24 04:57:02 -06:00
|
|
|
for room_id in partitioned:
|
2016-05-06 07:31:38 -06:00
|
|
|
self._maybe_start_persisting(room_id)
|
|
|
|
|
2018-07-25 09:00:38 -06:00
|
|
|
yield make_deferred_yieldable(
|
2016-08-23 08:23:39 -06:00
|
|
|
defer.gatherResults(deferreds, consumeErrors=True)
|
|
|
|
)
|
2016-05-06 07:31:38 -06:00
|
|
|
|
2018-07-25 09:00:38 -06:00
|
|
|
max_persisted_id = yield self._stream_id_gen.get_current_token()
|
|
|
|
|
|
|
|
defer.returnValue(max_persisted_id)
|
|
|
|
|
2016-05-06 07:31:38 -06:00
|
|
|
@defer.inlineCallbacks
|
|
|
|
@log_function
|
2017-01-20 04:52:51 -07:00
|
|
|
def persist_event(self, event, context, backfilled=False):
|
2017-03-17 05:51:13 -06:00
|
|
|
"""
|
|
|
|
|
|
|
|
Args:
|
|
|
|
event (EventBase):
|
|
|
|
context (EventContext):
|
|
|
|
backfilled (bool):
|
|
|
|
|
|
|
|
Returns:
|
|
|
|
Deferred: resolves to (int, int): the stream ordering of ``event``,
|
|
|
|
and the stream ordering of the latest persisted event
|
|
|
|
"""
|
2016-05-06 07:31:38 -06:00
|
|
|
deferred = self._event_persist_queue.add_to_queue(
|
|
|
|
event.room_id, [(event, context)],
|
|
|
|
backfilled=backfilled,
|
|
|
|
)
|
|
|
|
|
|
|
|
self._maybe_start_persisting(event.room_id)
|
|
|
|
|
2017-10-17 03:59:30 -06:00
|
|
|
yield make_deferred_yieldable(deferred)
|
2016-05-06 07:31:38 -06:00
|
|
|
|
|
|
|
max_persisted_id = yield self._stream_id_gen.get_current_token()
|
|
|
|
defer.returnValue((event.internal_metadata.stream_ordering, max_persisted_id))
|
|
|
|
|
|
|
|
def _maybe_start_persisting(self, room_id):
|
|
|
|
@defer.inlineCallbacks
|
2016-05-06 08:38:42 -06:00
|
|
|
def persisting_queue(item):
|
2018-03-13 03:57:54 -06:00
|
|
|
with Measure(self._clock, "persist_events"):
|
|
|
|
yield self._persist_events(
|
|
|
|
item.events_and_contexts,
|
|
|
|
backfilled=item.backfilled,
|
|
|
|
)
|
2016-05-06 07:31:38 -06:00
|
|
|
|
|
|
|
self._event_persist_queue.handle_queue(room_id, persisting_queue)
|
|
|
|
|
2016-08-04 08:02:15 -06:00
|
|
|
@_retry_on_integrity_error
|
2016-05-06 07:31:38 -06:00
|
|
|
@defer.inlineCallbacks
|
2016-08-04 08:02:15 -06:00
|
|
|
def _persist_events(self, events_and_contexts, backfilled=False,
|
|
|
|
delete_existing=False):
|
2017-03-17 05:51:13 -06:00
|
|
|
"""Persist events to db
|
|
|
|
|
|
|
|
Args:
|
|
|
|
events_and_contexts (list[(EventBase, EventContext)]):
|
|
|
|
backfilled (bool):
|
|
|
|
delete_existing (bool):
|
|
|
|
|
|
|
|
Returns:
|
|
|
|
Deferred: resolves when the events have been persisted
|
|
|
|
"""
|
2015-06-25 10:18:19 -06:00
|
|
|
if not events_and_contexts:
|
|
|
|
return
|
|
|
|
|
|
|
|
if backfilled:
|
2016-04-01 06:29:05 -06:00
|
|
|
stream_ordering_manager = self._backfill_id_gen.get_next_mult(
|
|
|
|
len(events_and_contexts)
|
|
|
|
)
|
2015-06-25 10:18:19 -06:00
|
|
|
else:
|
2016-03-01 07:32:56 -07:00
|
|
|
stream_ordering_manager = self._stream_id_gen.get_next_mult(
|
|
|
|
len(events_and_contexts)
|
2015-06-25 10:18:19 -06:00
|
|
|
)
|
|
|
|
|
|
|
|
with stream_ordering_manager as stream_orderings:
|
2016-08-30 09:54:40 -06:00
|
|
|
for (event, context), stream, in zip(
|
|
|
|
events_and_contexts, stream_orderings
|
|
|
|
):
|
|
|
|
event.internal_metadata.stream_ordering = stream
|
|
|
|
|
|
|
|
chunks = [
|
|
|
|
events_and_contexts[x:x + 100]
|
2018-05-31 03:03:47 -06:00
|
|
|
for x in range(0, len(events_and_contexts), 100)
|
2016-08-30 09:54:40 -06:00
|
|
|
]
|
|
|
|
|
|
|
|
for chunk in chunks:
|
|
|
|
# We can't easily parallelize these since different chunks
|
|
|
|
# might contain the same event. :(
|
2017-01-20 04:52:51 -07:00
|
|
|
|
2017-01-20 07:38:13 -07:00
|
|
|
# NB: Assumes that we are only persisting events for one room
|
|
|
|
# at a time.
|
2018-01-19 17:23:36 -07:00
|
|
|
|
|
|
|
# map room_id->list[event_ids] giving the new forward
|
|
|
|
# extremities in each room
|
2017-01-20 07:28:53 -07:00
|
|
|
new_forward_extremeties = {}
|
2018-01-19 17:23:36 -07:00
|
|
|
|
|
|
|
# map room_id->(type,state_key)->event_id tracking the full
|
2018-07-24 04:13:47 -06:00
|
|
|
# state in each room after adding these events.
|
|
|
|
# This is simply used to prefill the get_current_state_ids
|
|
|
|
# cache
|
2017-01-20 04:52:51 -07:00
|
|
|
current_state_for_room = {}
|
2018-01-19 17:23:36 -07:00
|
|
|
|
2018-07-24 04:59:16 -06:00
|
|
|
# map room_id->(to_delete, to_insert) where to_delete is a list
|
|
|
|
# of type/state keys to remove from current state, and to_insert
|
|
|
|
# is a map (type,key)->event_id giving the state delta in each
|
2018-01-19 17:23:36 -07:00
|
|
|
# room
|
|
|
|
state_delta_for_room = {}
|
|
|
|
|
2017-01-20 04:52:51 -07:00
|
|
|
if not backfilled:
|
2017-01-23 07:51:33 -07:00
|
|
|
with Measure(self._clock, "_calculate_state_and_extrem"):
|
|
|
|
# Work out the new "current state" for each room.
|
|
|
|
# We do this by working out what the new extremities are and then
|
|
|
|
# calculating the state from that.
|
|
|
|
events_by_room = {}
|
|
|
|
for event, context in chunk:
|
|
|
|
events_by_room.setdefault(event.room_id, []).append(
|
|
|
|
(event, context)
|
2017-01-20 04:52:51 -07:00
|
|
|
)
|
|
|
|
|
2018-05-31 03:03:47 -06:00
|
|
|
for room_id, ev_ctx_rm in iteritems(events_by_room):
|
2017-01-23 07:51:33 -07:00
|
|
|
latest_event_ids = yield self.get_latest_event_ids_in_room(
|
|
|
|
room_id
|
|
|
|
)
|
2018-10-02 16:33:29 -06:00
|
|
|
new_latest_event_ids = yield self._calculate_new_extremities(
|
2017-02-08 07:48:06 -07:00
|
|
|
room_id, ev_ctx_rm, latest_event_ids
|
2017-01-23 07:51:33 -07:00
|
|
|
)
|
2017-01-20 04:52:51 -07:00
|
|
|
|
2018-03-27 03:30:03 -06:00
|
|
|
latest_event_ids = set(latest_event_ids)
|
|
|
|
if new_latest_event_ids == latest_event_ids:
|
2017-01-23 07:51:33 -07:00
|
|
|
# No change in extremities, so no change in state
|
|
|
|
continue
|
2017-01-20 04:52:51 -07:00
|
|
|
|
2018-10-02 16:33:29 -06:00
|
|
|
# there should always be at least one forward extremity.
|
|
|
|
# (except during the initial persistence of the send_join
|
|
|
|
# results, in which case there will be no existing
|
|
|
|
# extremities, so we'll `continue` above and skip this bit.)
|
|
|
|
assert new_latest_event_ids, "No forward extremities left!"
|
|
|
|
|
2017-01-23 07:51:33 -07:00
|
|
|
new_forward_extremeties[room_id] = new_latest_event_ids
|
|
|
|
|
2017-02-27 11:33:41 -07:00
|
|
|
len_1 = (
|
|
|
|
len(latest_event_ids) == 1
|
|
|
|
and len(new_latest_event_ids) == 1
|
|
|
|
)
|
|
|
|
if len_1:
|
2017-02-27 11:45:24 -07:00
|
|
|
all_single_prev_not_state = all(
|
2017-02-27 11:33:41 -07:00
|
|
|
len(event.prev_events) == 1
|
|
|
|
and not event.is_state()
|
|
|
|
for event, ctx in ev_ctx_rm
|
|
|
|
)
|
|
|
|
# Don't bother calculating state if they're just
|
|
|
|
# a long chain of single ancestor non-state events.
|
|
|
|
if all_single_prev_not_state:
|
|
|
|
continue
|
|
|
|
|
2018-03-27 03:30:03 -06:00
|
|
|
state_delta_counter.inc()
|
|
|
|
if len(new_latest_event_ids) == 1:
|
|
|
|
state_delta_single_event_counter.inc()
|
|
|
|
|
|
|
|
# This is a fairly handwavey check to see if we could
|
|
|
|
# have guessed what the delta would have been when
|
|
|
|
# processing one of these events.
|
|
|
|
# What we're interested in is if the latest extremities
|
|
|
|
# were the same when we created the event as they are
|
2018-03-27 06:30:39 -06:00
|
|
|
# now. When this server creates a new event (as opposed
|
|
|
|
# to receiving it over federation) it will use the
|
|
|
|
# forward extremities as the prev_events, so we can
|
|
|
|
# guess this by looking at the prev_events and checking
|
|
|
|
# if they match the current forward extremities.
|
2018-03-27 03:30:03 -06:00
|
|
|
for ev, _ in ev_ctx_rm:
|
|
|
|
prev_event_ids = set(e for e, _ in ev.prev_events)
|
|
|
|
if latest_event_ids == prev_event_ids:
|
|
|
|
state_delta_reuse_delta_counter.inc()
|
2018-03-27 06:13:46 -06:00
|
|
|
break
|
2018-03-27 03:30:03 -06:00
|
|
|
|
2018-01-19 10:43:40 -07:00
|
|
|
logger.info(
|
|
|
|
"Calculating state delta for room %s", room_id,
|
2017-01-20 04:52:51 -07:00
|
|
|
)
|
2018-07-23 17:04:44 -06:00
|
|
|
with Measure(
|
2018-07-24 04:13:47 -06:00
|
|
|
self._clock,
|
|
|
|
"persist_events.get_new_state_after_events",
|
2018-07-23 17:04:44 -06:00
|
|
|
):
|
2018-07-24 04:13:47 -06:00
|
|
|
res = yield self._get_new_state_after_events(
|
2018-07-23 16:48:19 -06:00
|
|
|
room_id,
|
|
|
|
ev_ctx_rm,
|
|
|
|
latest_event_ids,
|
|
|
|
new_latest_event_ids,
|
|
|
|
)
|
2018-07-24 04:13:47 -06:00
|
|
|
current_state, delta_ids = res
|
|
|
|
|
|
|
|
# If either are not None then there has been a change,
|
|
|
|
# and we need to work out the delta (or use that
|
|
|
|
# given)
|
|
|
|
if delta_ids is not None:
|
|
|
|
# If there is a delta we know that we've
|
|
|
|
# only added or replaced state, never
|
|
|
|
# removed keys entirely.
|
|
|
|
state_delta_for_room[room_id] = ([], delta_ids)
|
|
|
|
elif current_state is not None:
|
2018-07-23 17:04:44 -06:00
|
|
|
with Measure(
|
2018-07-24 04:13:47 -06:00
|
|
|
self._clock,
|
|
|
|
"persist_events.calculate_state_delta",
|
2018-07-23 17:04:44 -06:00
|
|
|
):
|
2018-07-23 16:48:19 -06:00
|
|
|
delta = yield self._calculate_state_delta(
|
|
|
|
room_id, current_state,
|
|
|
|
)
|
2018-07-24 04:13:47 -06:00
|
|
|
state_delta_for_room[room_id] = delta
|
|
|
|
|
|
|
|
# If we have the current_state then lets prefill
|
|
|
|
# the cache with it.
|
|
|
|
if current_state is not None:
|
|
|
|
current_state_for_room[room_id] = current_state
|
2017-01-20 04:52:51 -07:00
|
|
|
|
2016-08-30 09:54:40 -06:00
|
|
|
yield self.runInteraction(
|
|
|
|
"persist_events",
|
|
|
|
self._persist_events_txn,
|
|
|
|
events_and_contexts=chunk,
|
|
|
|
backfilled=backfilled,
|
|
|
|
delete_existing=delete_existing,
|
2018-01-19 17:23:36 -07:00
|
|
|
state_delta_for_room=state_delta_for_room,
|
2017-01-20 07:28:53 -07:00
|
|
|
new_forward_extremeties=new_forward_extremeties,
|
2016-08-30 09:54:40 -06:00
|
|
|
)
|
2018-05-21 18:48:57 -06:00
|
|
|
persist_event_counter.inc(len(chunk))
|
2018-08-07 03:00:03 -06:00
|
|
|
|
|
|
|
if not backfilled:
|
|
|
|
# backfilled events have negative stream orderings, so we don't
|
|
|
|
# want to set the event_persisted_position to that.
|
|
|
|
synapse.metrics.event_persisted_position.set(
|
|
|
|
chunk[-1][0].internal_metadata.stream_ordering,
|
|
|
|
)
|
|
|
|
|
2017-05-02 04:36:11 -06:00
|
|
|
for event, context in chunk:
|
|
|
|
if context.app_service:
|
|
|
|
origin_type = "local"
|
|
|
|
origin_entity = context.app_service.id
|
|
|
|
elif self.hs.is_mine_id(event.sender):
|
|
|
|
origin_type = "local"
|
|
|
|
origin_entity = "*client*"
|
|
|
|
else:
|
|
|
|
origin_type = "remote"
|
|
|
|
origin_entity = get_domain_from_id(event.sender)
|
|
|
|
|
2018-05-21 18:48:57 -06:00
|
|
|
event_counter.labels(event.type, origin_type, origin_entity).inc()
|
2015-06-25 10:18:19 -06:00
|
|
|
|
2018-05-31 03:03:47 -06:00
|
|
|
for room_id, new_state in iteritems(current_state_for_room):
|
2017-05-02 03:40:31 -06:00
|
|
|
self.get_current_state_ids.prefill(
|
|
|
|
(room_id, ), new_state
|
|
|
|
)
|
|
|
|
|
2018-05-31 03:03:47 -06:00
|
|
|
for room_id, latest_event_ids in iteritems(new_forward_extremeties):
|
2017-06-29 08:38:48 -06:00
|
|
|
self.get_latest_event_ids_in_room.prefill(
|
2017-06-29 08:47:37 -06:00
|
|
|
(room_id,), list(latest_event_ids)
|
2017-06-29 08:38:48 -06:00
|
|
|
)
|
|
|
|
|
2017-01-20 07:28:53 -07:00
|
|
|
@defer.inlineCallbacks
|
2018-10-02 16:33:29 -06:00
|
|
|
def _calculate_new_extremities(self, room_id, event_contexts, latest_event_ids):
|
|
|
|
"""Calculates the new forward extremities for a room given events to
|
2017-01-20 07:38:13 -07:00
|
|
|
persist.
|
|
|
|
|
|
|
|
Assumes that we are only persisting events for one room at a time.
|
|
|
|
"""
|
2018-10-02 16:33:29 -06:00
|
|
|
|
|
|
|
# we're only interested in new events which aren't outliers and which aren't
|
|
|
|
# being rejected.
|
|
|
|
new_events = [
|
|
|
|
event for event, ctx in event_contexts
|
2017-02-08 07:48:06 -07:00
|
|
|
if not event.internal_metadata.is_outlier() and not ctx.rejected
|
2018-10-02 16:33:29 -06:00
|
|
|
]
|
|
|
|
|
|
|
|
# start with the existing forward extremities
|
|
|
|
result = set(latest_event_ids)
|
|
|
|
|
|
|
|
# add all the new events to the list
|
|
|
|
result.update(
|
|
|
|
event.event_id for event in new_events
|
2017-01-20 07:28:53 -07:00
|
|
|
)
|
2018-10-02 16:33:29 -06:00
|
|
|
|
|
|
|
# Now remove all events which are prev_events of any of the new events
|
|
|
|
result.difference_update(
|
2017-01-20 07:28:53 -07:00
|
|
|
e_id
|
2018-10-02 16:33:29 -06:00
|
|
|
for event in new_events
|
2017-01-20 07:28:53 -07:00
|
|
|
for e_id, _ in event.prev_events
|
|
|
|
)
|
|
|
|
|
2018-10-02 16:33:29 -06:00
|
|
|
# Finally, remove any events which are prev_events of any existing events.
|
|
|
|
existing_prevs = yield self._get_events_which_are_prevs(result)
|
|
|
|
result.difference_update(existing_prevs)
|
|
|
|
|
|
|
|
defer.returnValue(result)
|
2017-01-20 07:28:53 -07:00
|
|
|
|
2018-10-02 16:33:29 -06:00
|
|
|
@defer.inlineCallbacks
|
|
|
|
def _get_events_which_are_prevs(self, event_ids):
|
|
|
|
"""Filter the supplied list of event_ids to get those which are prev_events of
|
2018-10-03 03:19:41 -06:00
|
|
|
existing (non-outlier/rejected) events.
|
2018-10-02 16:33:29 -06:00
|
|
|
|
|
|
|
Args:
|
|
|
|
event_ids (Iterable[str]): event ids to filter
|
|
|
|
|
|
|
|
Returns:
|
|
|
|
Deferred[List[str]]: filtered event ids
|
|
|
|
"""
|
|
|
|
results = []
|
|
|
|
|
|
|
|
def _get_events(txn, batch):
|
|
|
|
sql = """
|
|
|
|
SELECT prev_event_id
|
|
|
|
FROM event_edges
|
|
|
|
INNER JOIN events USING (event_id)
|
|
|
|
LEFT JOIN rejections USING (event_id)
|
|
|
|
WHERE
|
|
|
|
prev_event_id IN (%s)
|
2018-10-03 03:19:41 -06:00
|
|
|
AND NOT events.outlier
|
2018-10-02 16:33:29 -06:00
|
|
|
AND rejections.event_id IS NULL
|
|
|
|
""" % (
|
|
|
|
",".join("?" for _ in batch),
|
|
|
|
)
|
2017-01-20 07:28:53 -07:00
|
|
|
|
2018-10-02 16:33:29 -06:00
|
|
|
txn.execute(sql, batch)
|
|
|
|
results.extend(r[0] for r in txn)
|
|
|
|
|
|
|
|
for chunk in batch_iter(event_ids, 100):
|
|
|
|
yield self.runInteraction(
|
|
|
|
"_get_events_which_are_prevs",
|
|
|
|
_get_events,
|
|
|
|
chunk,
|
|
|
|
)
|
|
|
|
|
|
|
|
defer.returnValue(results)
|
2017-01-20 07:28:53 -07:00
|
|
|
|
2017-01-23 07:51:33 -07:00
|
|
|
@defer.inlineCallbacks
|
2018-02-16 04:16:29 -07:00
|
|
|
def _get_new_state_after_events(self, room_id, events_context, old_latest_event_ids,
|
|
|
|
new_latest_event_ids):
|
2018-01-19 10:43:40 -07:00
|
|
|
"""Calculate the current state dict after adding some new events to
|
|
|
|
a room
|
2017-01-23 07:51:33 -07:00
|
|
|
|
2018-01-19 10:43:40 -07:00
|
|
|
Args:
|
2018-01-30 04:06:15 -07:00
|
|
|
room_id (str):
|
|
|
|
room to which the events are being added. Used for logging etc
|
|
|
|
|
2018-01-19 10:43:40 -07:00
|
|
|
events_context (list[(EventBase, EventContext)]):
|
|
|
|
events and contexts which are being added to the room
|
|
|
|
|
2018-02-16 04:16:29 -07:00
|
|
|
old_latest_event_ids (iterable[str]):
|
|
|
|
the old forward extremities for the room.
|
|
|
|
|
2018-01-19 10:43:40 -07:00
|
|
|
new_latest_event_ids (iterable[str]):
|
|
|
|
the new forward extremities for the room.
|
2017-01-23 07:51:33 -07:00
|
|
|
|
|
|
|
Returns:
|
2018-07-24 04:13:47 -06:00
|
|
|
Deferred[tuple[dict[(str,str), str]|None, dict[(str,str), str]|None]]:
|
|
|
|
Returns a tuple of two state maps, the first being the full new current
|
|
|
|
state and the second being the delta to the existing current state.
|
|
|
|
If both are None then there has been no change.
|
2018-07-24 08:12:50 -06:00
|
|
|
|
|
|
|
If there has been a change then we only return the delta if its
|
|
|
|
already been calculated. Conversely if we do know the delta then
|
|
|
|
the new current state is only returned if we've already calculated
|
|
|
|
it.
|
2017-01-23 07:51:33 -07:00
|
|
|
"""
|
2018-01-30 04:06:15 -07:00
|
|
|
# map from state_group to ((type, key) -> event_id) state map
|
2018-02-16 04:16:29 -07:00
|
|
|
state_groups_map = {}
|
2018-07-24 04:13:47 -06:00
|
|
|
|
2018-07-24 07:31:38 -06:00
|
|
|
# Map from (prev state group, new state group) -> delta state dict
|
2018-07-24 04:13:47 -06:00
|
|
|
state_group_deltas = {}
|
|
|
|
|
2018-02-16 04:16:29 -07:00
|
|
|
for ev, ctx in events_context:
|
|
|
|
if ctx.state_group is None:
|
2018-07-25 02:35:02 -06:00
|
|
|
# This should only happen for outlier events.
|
2018-07-25 02:48:01 -06:00
|
|
|
if not ev.internal_metadata.is_outlier():
|
2018-07-25 02:35:02 -06:00
|
|
|
raise Exception(
|
|
|
|
"Context for new event %s has no state "
|
|
|
|
"group" % (ev.event_id, ),
|
|
|
|
)
|
2018-07-25 04:13:20 -06:00
|
|
|
continue
|
2018-02-16 04:16:29 -07:00
|
|
|
|
|
|
|
if ctx.state_group in state_groups_map:
|
|
|
|
continue
|
|
|
|
|
2018-07-23 10:43:01 -06:00
|
|
|
# We're only interested in pulling out state that has already
|
|
|
|
# been cached in the context. We'll pull stuff out of the DB later
|
|
|
|
# if necessary.
|
2018-07-23 10:21:40 -06:00
|
|
|
current_state_ids = ctx.get_cached_current_state_ids()
|
|
|
|
if current_state_ids is not None:
|
|
|
|
state_groups_map[ctx.state_group] = current_state_ids
|
2018-02-16 04:16:29 -07:00
|
|
|
|
2018-07-24 04:13:47 -06:00
|
|
|
if ctx.prev_group:
|
|
|
|
state_group_deltas[(ctx.prev_group, ctx.state_group)] = ctx.delta_ids
|
|
|
|
|
2018-05-03 03:38:08 -06:00
|
|
|
# We need to map the event_ids to their state groups. First, let's
|
|
|
|
# check if the event is one we're persisting, in which case we can
|
|
|
|
# pull the state group from its context.
|
2018-02-16 04:16:29 -07:00
|
|
|
# Otherwise we need to pull the state group from the database.
|
|
|
|
|
|
|
|
# Set of events we need to fetch groups for. (We know none of the old
|
|
|
|
# extremities are going to be in events_context).
|
|
|
|
missing_event_ids = set(old_latest_event_ids)
|
|
|
|
|
|
|
|
event_id_to_state_group = {}
|
2017-01-23 07:51:33 -07:00
|
|
|
for event_id in new_latest_event_ids:
|
2018-02-16 04:16:29 -07:00
|
|
|
# First search in the list of new events we're adding.
|
2017-01-23 07:51:33 -07:00
|
|
|
for ev, ctx in events_context:
|
2018-07-25 02:35:02 -06:00
|
|
|
if event_id == ev.event_id and ctx.state_group is not None:
|
2018-02-16 04:16:29 -07:00
|
|
|
event_id_to_state_group[event_id] = ctx.state_group
|
2017-01-23 07:51:33 -07:00
|
|
|
break
|
|
|
|
else:
|
|
|
|
# If we couldn't find it, then we'll need to pull
|
|
|
|
# the state from the database
|
2018-02-16 04:16:29 -07:00
|
|
|
missing_event_ids.add(event_id)
|
2018-01-30 03:17:55 -07:00
|
|
|
|
2017-01-23 07:51:33 -07:00
|
|
|
if missing_event_ids:
|
2018-02-16 04:16:29 -07:00
|
|
|
# Now pull out the state groups for any missing events from DB
|
2017-01-23 07:51:33 -07:00
|
|
|
event_to_groups = yield self._get_state_group_for_events(
|
|
|
|
missing_event_ids,
|
|
|
|
)
|
2018-02-16 04:16:29 -07:00
|
|
|
event_id_to_state_group.update(event_to_groups)
|
2017-01-23 07:51:33 -07:00
|
|
|
|
2018-02-16 04:16:29 -07:00
|
|
|
# State groups of old_latest_event_ids
|
|
|
|
old_state_groups = set(
|
|
|
|
event_id_to_state_group[evid] for evid in old_latest_event_ids
|
|
|
|
)
|
2017-01-23 07:51:33 -07:00
|
|
|
|
2018-02-16 04:16:29 -07:00
|
|
|
# State groups of new_latest_event_ids
|
|
|
|
new_state_groups = set(
|
|
|
|
event_id_to_state_group[evid] for evid in new_latest_event_ids
|
|
|
|
)
|
|
|
|
|
|
|
|
# If they old and new groups are the same then we don't need to do
|
|
|
|
# anything.
|
|
|
|
if old_state_groups == new_state_groups:
|
2018-07-24 04:13:47 -06:00
|
|
|
defer.returnValue((None, None))
|
2017-01-23 07:51:33 -07:00
|
|
|
|
2018-07-24 07:31:38 -06:00
|
|
|
if len(new_state_groups) == 1 and len(old_state_groups) == 1:
|
|
|
|
# If we're going from one state group to another, lets check if
|
|
|
|
# we have a delta for that transition. If we do then we can just
|
|
|
|
# return that.
|
2018-07-24 04:13:47 -06:00
|
|
|
|
2018-07-24 07:31:38 -06:00
|
|
|
new_state_group = next(iter(new_state_groups))
|
|
|
|
old_state_group = next(iter(old_state_groups))
|
|
|
|
|
|
|
|
delta_ids = state_group_deltas.get(
|
|
|
|
(old_state_group, new_state_group,), None
|
|
|
|
)
|
|
|
|
if delta_ids is not None:
|
|
|
|
# We have a delta from the existing to new current state,
|
|
|
|
# so lets just return that. If we happen to already have
|
|
|
|
# the current state in memory then lets also return that,
|
|
|
|
# but it doesn't matter if we don't.
|
|
|
|
new_state = state_groups_map.get(new_state_group)
|
|
|
|
defer.returnValue((new_state, delta_ids))
|
2018-07-24 04:13:47 -06:00
|
|
|
|
2018-07-24 06:40:42 -06:00
|
|
|
# Now that we have calculated new_state_groups we need to get
|
|
|
|
# their state IDs so we can resolve to a single state set.
|
|
|
|
missing_state = new_state_groups - set(state_groups_map)
|
|
|
|
if missing_state:
|
|
|
|
group_to_state = yield self._get_state_for_groups(missing_state)
|
|
|
|
state_groups_map.update(group_to_state)
|
|
|
|
|
|
|
|
if len(new_state_groups) == 1:
|
2018-07-24 07:31:38 -06:00
|
|
|
# If there is only one state group, then we know what the current
|
|
|
|
# state is.
|
2018-07-24 06:40:42 -06:00
|
|
|
defer.returnValue((state_groups_map[new_state_groups.pop()], None))
|
2018-02-16 04:16:29 -07:00
|
|
|
|
|
|
|
# Ok, we need to defer to the state handler to resolve our state sets.
|
2018-01-17 09:01:59 -07:00
|
|
|
|
2018-02-16 04:16:29 -07:00
|
|
|
state_groups = {
|
|
|
|
sg: state_groups_map[sg] for sg in new_state_groups
|
|
|
|
}
|
|
|
|
|
2018-01-30 04:06:15 -07:00
|
|
|
events_map = {ev.event_id: ev for ev, _ in events_context}
|
2018-08-08 10:01:57 -06:00
|
|
|
room_version = yield self.get_room_version(room_id)
|
|
|
|
|
2018-01-30 04:06:15 -07:00
|
|
|
logger.debug("calling resolve_state_groups from preserve_events")
|
|
|
|
res = yield self._state_resolution_handler.resolve_state_groups(
|
2018-09-03 08:13:17 -06:00
|
|
|
room_id, room_version, state_groups, events_map,
|
|
|
|
state_res_store=StateResolutionStore(self)
|
2018-01-30 03:17:55 -07:00
|
|
|
)
|
2017-03-27 11:00:47 -06:00
|
|
|
|
2018-07-24 04:13:47 -06:00
|
|
|
defer.returnValue((res.state, None))
|
2017-01-23 07:51:33 -07:00
|
|
|
|
2018-01-19 10:43:40 -07:00
|
|
|
@defer.inlineCallbacks
|
|
|
|
def _calculate_state_delta(self, room_id, current_state):
|
|
|
|
"""Calculate the new state deltas for a room.
|
|
|
|
|
|
|
|
Assumes that we are only persisting events for one room at a time.
|
|
|
|
|
|
|
|
Returns:
|
2018-07-24 04:59:16 -06:00
|
|
|
tuple[list, dict] (to_delete, to_insert): where to_delete are the
|
|
|
|
type/state_keys to remove from current_state_events and `to_insert`
|
|
|
|
are the updates to current_state_events.
|
2018-01-19 10:43:40 -07:00
|
|
|
"""
|
2017-03-10 10:39:35 -07:00
|
|
|
existing_state = yield self.get_current_state_ids(room_id)
|
2017-01-23 07:51:33 -07:00
|
|
|
|
2018-07-24 04:59:16 -06:00
|
|
|
to_delete = [
|
2018-07-24 08:08:01 -06:00
|
|
|
key for key in existing_state
|
2018-07-24 04:59:16 -06:00
|
|
|
if key not in current_state
|
|
|
|
]
|
2018-07-24 03:55:11 -06:00
|
|
|
|
2017-01-23 07:51:33 -07:00
|
|
|
to_insert = {
|
2018-05-31 03:03:47 -06:00
|
|
|
key: ev_id for key, ev_id in iteritems(current_state)
|
2018-07-24 03:55:11 -06:00
|
|
|
if ev_id != existing_state.get(key)
|
2017-01-23 07:51:33 -07:00
|
|
|
}
|
|
|
|
|
2018-01-19 17:23:36 -07:00
|
|
|
defer.returnValue((to_delete, to_insert))
|
2017-01-23 07:51:33 -07:00
|
|
|
|
2015-06-25 10:18:19 -06:00
|
|
|
@log_function
|
2016-08-04 08:02:15 -06:00
|
|
|
def _persist_events_txn(self, txn, events_and_contexts, backfilled,
|
2018-01-19 17:23:36 -07:00
|
|
|
delete_existing=False, state_delta_for_room={},
|
2017-01-20 07:28:53 -07:00
|
|
|
new_forward_extremeties={}):
|
2016-07-26 03:49:52 -06:00
|
|
|
"""Insert some number of room events into the necessary database tables.
|
|
|
|
|
|
|
|
Rejected events are only inserted into the events table, the events_json table,
|
|
|
|
and the rejections table. Things reading from those table will need to check
|
|
|
|
whether the event was rejected.
|
2016-08-04 08:02:15 -06:00
|
|
|
|
2017-03-17 05:51:13 -06:00
|
|
|
Args:
|
|
|
|
txn (twisted.enterprise.adbapi.Connection): db connection
|
|
|
|
events_and_contexts (list[(EventBase, EventContext)]):
|
|
|
|
events to persist
|
|
|
|
backfilled (bool): True if the events were backfilled
|
|
|
|
delete_existing (bool): True to purge existing table rows for the
|
|
|
|
events from the database. This is useful when retrying due to
|
|
|
|
IntegrityError.
|
2018-07-24 04:59:16 -06:00
|
|
|
state_delta_for_room (dict[str, (list, dict)]):
|
2017-03-17 05:51:13 -06:00
|
|
|
The current-state delta for each room. For each room, a tuple
|
2018-07-24 04:59:16 -06:00
|
|
|
(to_delete, to_insert), being a list of type/state keys to be
|
|
|
|
removed from the current state, and a state set to be added to
|
2017-03-17 05:51:13 -06:00
|
|
|
the current state.
|
|
|
|
new_forward_extremeties (dict[str, list[str]]):
|
|
|
|
The new forward extremities for each room. For each room, a
|
|
|
|
list of the event ids which are the forward extremities.
|
|
|
|
|
2016-07-26 03:49:52 -06:00
|
|
|
"""
|
2018-02-20 05:33:04 -07:00
|
|
|
all_events_and_contexts = events_and_contexts
|
|
|
|
|
2017-01-20 07:28:53 -07:00
|
|
|
max_stream_order = events_and_contexts[-1][0].internal_metadata.stream_ordering
|
2017-05-30 07:41:42 -06:00
|
|
|
|
2018-01-19 17:23:36 -07:00
|
|
|
self._update_current_state_txn(txn, state_delta_for_room, max_stream_order)
|
2017-05-30 07:41:42 -06:00
|
|
|
|
2017-03-17 05:51:13 -06:00
|
|
|
self._update_forward_extremities_txn(
|
|
|
|
txn,
|
|
|
|
new_forward_extremities=new_forward_extremeties,
|
|
|
|
max_stream_order=max_stream_order,
|
|
|
|
)
|
|
|
|
|
|
|
|
# Ensure that we don't have the same event twice.
|
|
|
|
events_and_contexts = self._filter_events_and_contexts_for_duplicates(
|
|
|
|
events_and_contexts,
|
|
|
|
)
|
|
|
|
|
|
|
|
self._update_room_depths_txn(
|
|
|
|
txn,
|
|
|
|
events_and_contexts=events_and_contexts,
|
|
|
|
backfilled=backfilled,
|
|
|
|
)
|
|
|
|
|
|
|
|
# _update_outliers_txn filters out any events which have already been
|
|
|
|
# persisted, and returns the filtered list.
|
|
|
|
events_and_contexts = self._update_outliers_txn(
|
|
|
|
txn,
|
|
|
|
events_and_contexts=events_and_contexts,
|
|
|
|
)
|
|
|
|
|
|
|
|
# From this point onwards the events are only events that we haven't
|
|
|
|
# seen before.
|
|
|
|
|
|
|
|
if delete_existing:
|
|
|
|
# For paranoia reasons, we go and delete all the existing entries
|
|
|
|
# for these events so we can reinsert them.
|
|
|
|
# This gets around any problems with some tables already having
|
|
|
|
# entries.
|
|
|
|
self._delete_existing_rows_txn(
|
|
|
|
txn,
|
|
|
|
events_and_contexts=events_and_contexts,
|
|
|
|
)
|
|
|
|
|
|
|
|
self._store_event_txn(
|
|
|
|
txn,
|
|
|
|
events_and_contexts=events_and_contexts,
|
|
|
|
)
|
|
|
|
|
2018-02-06 07:31:24 -07:00
|
|
|
# Insert into event_to_state_groups.
|
|
|
|
self._store_event_state_mappings_txn(txn, events_and_contexts)
|
2017-03-17 08:30:16 -06:00
|
|
|
|
2018-10-16 07:01:53 -06:00
|
|
|
# We want to store event_auth mappings for rejected events, as they're
|
|
|
|
# used in state res v2.
|
|
|
|
# This is only necessary if the rejected event appears in an accepted
|
|
|
|
# event's auth chain, but its easier for now just to store them (and
|
|
|
|
# it doesn't take much storage compared to storing the entire event
|
|
|
|
# anyway).
|
|
|
|
self._simple_insert_many_txn(
|
|
|
|
txn,
|
|
|
|
table="event_auth",
|
|
|
|
values=[
|
|
|
|
{
|
|
|
|
"event_id": event.event_id,
|
|
|
|
"room_id": event.room_id,
|
|
|
|
"auth_id": auth_id,
|
|
|
|
}
|
|
|
|
for event, _ in events_and_contexts
|
|
|
|
for auth_id, _ in event.auth_events
|
|
|
|
if event.is_state()
|
|
|
|
],
|
|
|
|
)
|
|
|
|
|
2017-03-17 05:51:13 -06:00
|
|
|
# _store_rejected_events_txn filters out any events which were
|
|
|
|
# rejected, and returns the filtered list.
|
|
|
|
events_and_contexts = self._store_rejected_events_txn(
|
|
|
|
txn,
|
|
|
|
events_and_contexts=events_and_contexts,
|
|
|
|
)
|
|
|
|
|
|
|
|
# From this point onwards the events are only ones that weren't
|
|
|
|
# rejected.
|
|
|
|
|
|
|
|
self._update_metadata_tables_txn(
|
|
|
|
txn,
|
|
|
|
events_and_contexts=events_and_contexts,
|
2018-02-20 05:33:04 -07:00
|
|
|
all_events_and_contexts=all_events_and_contexts,
|
2017-03-17 05:51:13 -06:00
|
|
|
backfilled=backfilled,
|
|
|
|
)
|
|
|
|
|
2017-05-30 07:41:42 -06:00
|
|
|
def _update_current_state_txn(self, txn, state_delta_by_room, max_stream_order):
|
2018-05-31 03:03:47 -06:00
|
|
|
for room_id, current_state_tuple in iteritems(state_delta_by_room):
|
2018-01-19 17:23:36 -07:00
|
|
|
to_delete, to_insert = current_state_tuple
|
2018-07-24 04:59:16 -06:00
|
|
|
|
|
|
|
# First we add entries to the current_state_delta_stream. We
|
|
|
|
# do this before updating the current_state_events table so
|
|
|
|
# that we can use it to calculate the `prev_event_id`. (This
|
|
|
|
# allows us to not have to pull out the existing state
|
|
|
|
# unnecessarily).
|
|
|
|
sql = """
|
|
|
|
INSERT INTO current_state_delta_stream
|
|
|
|
(stream_id, room_id, type, state_key, event_id, prev_event_id)
|
|
|
|
SELECT ?, ?, ?, ?, ?, (
|
|
|
|
SELECT event_id FROM current_state_events
|
|
|
|
WHERE room_id = ? AND type = ? AND state_key = ?
|
|
|
|
)
|
|
|
|
"""
|
|
|
|
txn.executemany(sql, (
|
|
|
|
(
|
|
|
|
max_stream_order, room_id, etype, state_key, None,
|
|
|
|
room_id, etype, state_key,
|
|
|
|
)
|
|
|
|
for etype, state_key in to_delete
|
|
|
|
# We sanity check that we're deleting rather than updating
|
|
|
|
if (etype, state_key) not in to_insert
|
|
|
|
))
|
|
|
|
txn.executemany(sql, (
|
|
|
|
(
|
|
|
|
max_stream_order, room_id, etype, state_key, ev_id,
|
|
|
|
room_id, etype, state_key,
|
|
|
|
)
|
|
|
|
for (etype, state_key), ev_id in iteritems(to_insert)
|
|
|
|
))
|
|
|
|
|
|
|
|
# Now we actually update the current_state_events table
|
|
|
|
|
2017-01-20 08:40:04 -07:00
|
|
|
txn.executemany(
|
2018-07-24 04:59:16 -06:00
|
|
|
"DELETE FROM current_state_events"
|
|
|
|
" WHERE room_id = ? AND type = ? AND state_key = ?",
|
|
|
|
(
|
|
|
|
(room_id, etype, state_key)
|
|
|
|
for etype, state_key in itertools.chain(to_delete, to_insert)
|
|
|
|
),
|
2017-01-20 08:40:04 -07:00
|
|
|
)
|
|
|
|
|
|
|
|
self._simple_insert_many_txn(
|
|
|
|
txn,
|
|
|
|
table="current_state_events",
|
|
|
|
values=[
|
|
|
|
{
|
|
|
|
"event_id": ev_id,
|
|
|
|
"room_id": room_id,
|
|
|
|
"type": key[0],
|
|
|
|
"state_key": key[1],
|
|
|
|
}
|
2018-05-31 03:03:47 -06:00
|
|
|
for key, ev_id in iteritems(to_insert)
|
2017-01-20 08:40:04 -07:00
|
|
|
],
|
|
|
|
)
|
|
|
|
|
2018-07-02 04:36:44 -06:00
|
|
|
txn.call_after(
|
|
|
|
self._curr_state_delta_stream_cache.entity_has_changed,
|
2017-05-31 08:46:36 -06:00
|
|
|
room_id, max_stream_order,
|
|
|
|
)
|
|
|
|
|
2017-01-20 10:13:24 -07:00
|
|
|
# Invalidate the various caches
|
|
|
|
|
|
|
|
# Figure out the changes of membership to invalidate the
|
|
|
|
# `get_rooms_for_user` cache.
|
|
|
|
# We find out which membership events we may have deleted
|
|
|
|
# and which we have added, then we invlidate the caches for all
|
|
|
|
# those users.
|
2017-01-20 08:40:04 -07:00
|
|
|
members_changed = set(
|
2018-07-24 04:59:16 -06:00
|
|
|
state_key
|
|
|
|
for ev_type, state_key in itertools.chain(to_delete, to_insert)
|
2017-01-23 07:51:33 -07:00
|
|
|
if ev_type == EventTypes.Member
|
2017-01-20 08:40:04 -07:00
|
|
|
)
|
|
|
|
|
|
|
|
for member in members_changed:
|
2017-01-30 10:24:32 -07:00
|
|
|
self._invalidate_cache_and_stream(
|
2018-03-05 05:06:19 -07:00
|
|
|
txn, self.get_rooms_for_user_with_stream_ordering, (member,)
|
2017-01-30 10:24:32 -07:00
|
|
|
)
|
2017-01-20 08:40:04 -07:00
|
|
|
|
2017-06-13 02:56:18 -06:00
|
|
|
for host in set(get_domain_from_id(u) for u in members_changed):
|
|
|
|
self._invalidate_cache_and_stream(
|
|
|
|
txn, self.is_host_joined, (room_id, host)
|
|
|
|
)
|
2017-10-03 07:12:28 -06:00
|
|
|
self._invalidate_cache_and_stream(
|
|
|
|
txn, self.was_host_joined, (room_id, host)
|
|
|
|
)
|
2017-06-13 02:56:18 -06:00
|
|
|
|
2017-01-30 10:24:32 -07:00
|
|
|
self._invalidate_cache_and_stream(
|
|
|
|
txn, self.get_users_in_room, (room_id,)
|
|
|
|
)
|
2017-01-20 04:52:51 -07:00
|
|
|
|
2018-09-11 17:50:39 -06:00
|
|
|
self._invalidate_cache_and_stream(
|
|
|
|
txn, self.get_room_summary, (room_id,)
|
|
|
|
)
|
|
|
|
|
2017-03-10 10:39:35 -07:00
|
|
|
self._invalidate_cache_and_stream(
|
|
|
|
txn, self.get_current_state_ids, (room_id,)
|
|
|
|
)
|
|
|
|
|
2017-03-17 05:51:13 -06:00
|
|
|
def _update_forward_extremities_txn(self, txn, new_forward_extremities,
|
|
|
|
max_stream_order):
|
2018-05-31 03:03:47 -06:00
|
|
|
for room_id, new_extrem in iteritems(new_forward_extremities):
|
2017-01-20 07:28:53 -07:00
|
|
|
self._simple_delete_txn(
|
|
|
|
txn,
|
|
|
|
table="event_forward_extremities",
|
|
|
|
keyvalues={"room_id": room_id},
|
|
|
|
)
|
|
|
|
txn.call_after(
|
|
|
|
self.get_latest_event_ids_in_room.invalidate, (room_id,)
|
|
|
|
)
|
|
|
|
|
|
|
|
self._simple_insert_many_txn(
|
|
|
|
txn,
|
|
|
|
table="event_forward_extremities",
|
|
|
|
values=[
|
|
|
|
{
|
|
|
|
"event_id": ev_id,
|
|
|
|
"room_id": room_id,
|
|
|
|
}
|
2018-05-31 03:03:47 -06:00
|
|
|
for room_id, new_extrem in iteritems(new_forward_extremities)
|
2017-01-20 07:28:53 -07:00
|
|
|
for ev_id in new_extrem
|
|
|
|
],
|
|
|
|
)
|
|
|
|
# We now insert into stream_ordering_to_exterm a mapping from room_id,
|
|
|
|
# new stream_ordering to new forward extremeties in the room.
|
|
|
|
# This allows us to later efficiently look up the forward extremeties
|
|
|
|
# for a room before a given stream_ordering
|
|
|
|
self._simple_insert_many_txn(
|
|
|
|
txn,
|
|
|
|
table="stream_ordering_to_exterm",
|
|
|
|
values=[
|
|
|
|
{
|
|
|
|
"room_id": room_id,
|
|
|
|
"event_id": event_id,
|
|
|
|
"stream_ordering": max_stream_order,
|
|
|
|
}
|
2018-05-31 03:03:47 -06:00
|
|
|
for room_id, new_extrem in iteritems(new_forward_extremities)
|
2017-01-20 07:28:53 -07:00
|
|
|
for event_id in new_extrem
|
|
|
|
]
|
|
|
|
)
|
|
|
|
|
2017-03-17 05:51:13 -06:00
|
|
|
@classmethod
|
|
|
|
def _filter_events_and_contexts_for_duplicates(cls, events_and_contexts):
|
|
|
|
"""Ensure that we don't have the same event twice.
|
|
|
|
|
|
|
|
Pick the earliest non-outlier if there is one, else the earliest one.
|
|
|
|
|
|
|
|
Args:
|
|
|
|
events_and_contexts (list[(EventBase, EventContext)]):
|
|
|
|
Returns:
|
|
|
|
list[(EventBase, EventContext)]: filtered list
|
|
|
|
"""
|
2016-08-03 04:23:39 -06:00
|
|
|
new_events_and_contexts = OrderedDict()
|
|
|
|
for event, context in events_and_contexts:
|
|
|
|
prev_event_context = new_events_and_contexts.get(event.event_id)
|
|
|
|
if prev_event_context:
|
|
|
|
if not event.internal_metadata.is_outlier():
|
|
|
|
if prev_event_context[0].internal_metadata.is_outlier():
|
|
|
|
# To ensure correct ordering we pop, as OrderedDict is
|
|
|
|
# ordered by first insertion.
|
|
|
|
new_events_and_contexts.pop(event.event_id, None)
|
|
|
|
new_events_and_contexts[event.event_id] = (event, context)
|
|
|
|
else:
|
|
|
|
new_events_and_contexts[event.event_id] = (event, context)
|
2018-05-31 03:03:47 -06:00
|
|
|
return list(new_events_and_contexts.values())
|
2016-08-03 04:23:39 -06:00
|
|
|
|
2017-03-17 05:51:13 -06:00
|
|
|
def _update_room_depths_txn(self, txn, events_and_contexts, backfilled):
|
|
|
|
"""Update min_depth for each room
|
2016-08-03 04:23:39 -06:00
|
|
|
|
2017-03-17 05:51:13 -06:00
|
|
|
Args:
|
|
|
|
txn (twisted.enterprise.adbapi.Connection): db connection
|
|
|
|
events_and_contexts (list[(EventBase, EventContext)]): events
|
|
|
|
we are persisting
|
|
|
|
backfilled (bool): True if the events were backfilled
|
|
|
|
"""
|
2016-02-09 09:19:15 -07:00
|
|
|
depth_updates = {}
|
|
|
|
for event, context in events_and_contexts:
|
|
|
|
# Remove the any existing cache entries for the event_ids
|
2015-06-25 10:18:19 -06:00
|
|
|
txn.call_after(self._invalidate_get_event_cache, event.event_id)
|
2016-01-28 08:02:37 -07:00
|
|
|
if not backfilled:
|
|
|
|
txn.call_after(
|
2016-01-28 09:37:41 -07:00
|
|
|
self._events_stream_cache.entity_has_changed,
|
2016-01-28 08:02:37 -07:00
|
|
|
event.room_id, event.internal_metadata.stream_ordering,
|
|
|
|
)
|
|
|
|
|
2016-07-25 11:44:30 -06:00
|
|
|
if not event.internal_metadata.is_outlier() and not context.rejected:
|
2016-02-09 09:19:15 -07:00
|
|
|
depth_updates[event.room_id] = max(
|
|
|
|
event.depth, depth_updates.get(event.room_id, event.depth)
|
|
|
|
)
|
|
|
|
|
2018-05-31 03:03:47 -06:00
|
|
|
for room_id, depth in iteritems(depth_updates):
|
2015-06-25 10:18:19 -06:00
|
|
|
self._update_min_depth_for_room_txn(txn, room_id, depth)
|
|
|
|
|
2017-03-17 05:51:13 -06:00
|
|
|
def _update_outliers_txn(self, txn, events_and_contexts):
|
|
|
|
"""Update any outliers with new event info.
|
|
|
|
|
|
|
|
This turns outliers into ex-outliers (unless the new event was
|
|
|
|
rejected).
|
|
|
|
|
|
|
|
Args:
|
|
|
|
txn (twisted.enterprise.adbapi.Connection): db connection
|
|
|
|
events_and_contexts (list[(EventBase, EventContext)]): events
|
|
|
|
we are persisting
|
|
|
|
|
|
|
|
Returns:
|
|
|
|
list[(EventBase, EventContext)] new list, without events which
|
|
|
|
are already in the events table.
|
|
|
|
"""
|
2015-06-25 10:18:19 -06:00
|
|
|
txn.execute(
|
|
|
|
"SELECT event_id, outlier FROM events WHERE event_id in (%s)" % (
|
|
|
|
",".join(["?"] * len(events_and_contexts)),
|
|
|
|
),
|
|
|
|
[event.event_id for event, _ in events_and_contexts]
|
2015-03-20 07:52:56 -06:00
|
|
|
)
|
2016-07-25 11:44:30 -06:00
|
|
|
|
2015-06-25 10:18:19 -06:00
|
|
|
have_persisted = {
|
|
|
|
event_id: outlier
|
2017-03-23 11:53:49 -06:00
|
|
|
for event_id, outlier in txn
|
2015-06-25 10:18:19 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
to_remove = set()
|
|
|
|
for event, context in events_and_contexts:
|
|
|
|
if event.event_id not in have_persisted:
|
|
|
|
continue
|
|
|
|
|
|
|
|
to_remove.add(event)
|
|
|
|
|
2017-03-17 05:51:13 -06:00
|
|
|
if context.rejected:
|
|
|
|
# If the event is rejected then we don't care if the event
|
|
|
|
# was an outlier or not.
|
|
|
|
continue
|
|
|
|
|
2015-06-25 10:18:19 -06:00
|
|
|
outlier_persisted = have_persisted[event.event_id]
|
|
|
|
if not event.internal_metadata.is_outlier() and outlier_persisted:
|
2016-07-26 03:49:52 -06:00
|
|
|
# We received a copy of an event that we had already stored as
|
|
|
|
# an outlier in the database. We now have some state at that
|
|
|
|
# so we need to update the state_groups table with that state.
|
|
|
|
|
2018-02-06 07:31:24 -07:00
|
|
|
# insert into event_to_state_groups.
|
2016-09-02 03:41:38 -06:00
|
|
|
try:
|
2018-02-06 07:31:24 -07:00
|
|
|
self._store_event_state_mappings_txn(txn, ((event, context),))
|
2016-09-02 03:41:38 -06:00
|
|
|
except Exception:
|
|
|
|
logger.exception("")
|
|
|
|
raise
|
2015-03-20 07:52:56 -06:00
|
|
|
|
2015-06-25 10:18:19 -06:00
|
|
|
metadata_json = encode_json(
|
2015-08-24 09:17:38 -06:00
|
|
|
event.internal_metadata.get_dict()
|
2018-08-01 08:54:06 -06:00
|
|
|
)
|
2015-05-12 07:14:58 -06:00
|
|
|
|
2015-03-20 07:52:56 -06:00
|
|
|
sql = (
|
|
|
|
"UPDATE event_json SET internal_metadata = ?"
|
|
|
|
" WHERE event_id = ?"
|
|
|
|
)
|
|
|
|
txn.execute(
|
|
|
|
sql,
|
2015-04-16 04:17:52 -06:00
|
|
|
(metadata_json, event.event_id,)
|
2015-03-20 07:52:56 -06:00
|
|
|
)
|
|
|
|
|
2016-07-26 03:49:52 -06:00
|
|
|
# Add an entry to the ex_outlier_stream table to replicate the
|
|
|
|
# change in outlier status to our workers.
|
2016-03-30 10:19:56 -06:00
|
|
|
stream_order = event.internal_metadata.stream_ordering
|
2016-08-31 03:09:46 -06:00
|
|
|
state_group_id = context.state_group
|
2016-03-30 10:19:56 -06:00
|
|
|
self._simple_insert_txn(
|
|
|
|
txn,
|
|
|
|
table="ex_outlier_stream",
|
|
|
|
values={
|
|
|
|
"event_stream_ordering": stream_order,
|
|
|
|
"event_id": event.event_id,
|
|
|
|
"state_group": state_group_id,
|
|
|
|
}
|
|
|
|
)
|
|
|
|
|
2015-03-20 07:52:56 -06:00
|
|
|
sql = (
|
2015-04-14 06:53:20 -06:00
|
|
|
"UPDATE events SET outlier = ?"
|
2015-03-20 07:52:56 -06:00
|
|
|
" WHERE event_id = ?"
|
|
|
|
)
|
|
|
|
txn.execute(
|
|
|
|
sql,
|
2015-04-14 06:53:20 -06:00
|
|
|
(False, event.event_id,)
|
2015-03-20 07:52:56 -06:00
|
|
|
)
|
2016-07-25 11:44:30 -06:00
|
|
|
|
2016-07-26 03:49:52 -06:00
|
|
|
# Update the event_backward_extremities table now that this
|
|
|
|
# event isn't an outlier any more.
|
2017-01-20 07:40:31 -07:00
|
|
|
self._update_backward_extremeties(txn, [event])
|
2015-09-15 09:34:42 -06:00
|
|
|
|
2017-03-17 05:51:13 -06:00
|
|
|
return [
|
2016-03-30 05:55:02 -06:00
|
|
|
ec for ec in events_and_contexts if ec[0] not in to_remove
|
|
|
|
]
|
2015-03-24 10:20:26 -06:00
|
|
|
|
2017-03-17 05:51:13 -06:00
|
|
|
@classmethod
|
|
|
|
def _delete_existing_rows_txn(cls, txn, events_and_contexts):
|
2015-06-25 10:18:19 -06:00
|
|
|
if not events_and_contexts:
|
2017-03-17 05:51:13 -06:00
|
|
|
# nothing to do here
|
2015-06-25 10:18:19 -06:00
|
|
|
return
|
2015-03-20 07:52:56 -06:00
|
|
|
|
2017-03-17 05:51:13 -06:00
|
|
|
logger.info("Deleting existing")
|
2015-03-20 07:52:56 -06:00
|
|
|
|
2017-03-17 05:51:13 -06:00
|
|
|
for table in (
|
2016-08-04 08:02:15 -06:00
|
|
|
"events",
|
2016-08-04 21:38:08 -06:00
|
|
|
"event_auth",
|
2016-08-04 08:02:15 -06:00
|
|
|
"event_json",
|
|
|
|
"event_content_hashes",
|
|
|
|
"event_destinations",
|
|
|
|
"event_edge_hashes",
|
|
|
|
"event_edges",
|
|
|
|
"event_forward_extremities",
|
|
|
|
"event_reference_hashes",
|
|
|
|
"event_search",
|
|
|
|
"event_signatures",
|
|
|
|
"event_to_state_groups",
|
2016-08-05 03:39:44 -06:00
|
|
|
"guest_access",
|
|
|
|
"history_visibility",
|
|
|
|
"local_invites",
|
|
|
|
"room_names",
|
|
|
|
"state_events",
|
2016-08-04 08:02:15 -06:00
|
|
|
"rejections",
|
|
|
|
"redactions",
|
2016-08-04 21:38:08 -06:00
|
|
|
"room_memberships",
|
2016-08-14 12:50:22 -06:00
|
|
|
"topics"
|
2017-03-17 05:51:13 -06:00
|
|
|
):
|
|
|
|
txn.executemany(
|
|
|
|
"DELETE FROM %s WHERE event_id = ?" % (table,),
|
|
|
|
[(ev.event_id,) for ev, _ in events_and_contexts]
|
|
|
|
)
|
|
|
|
|
2016-12-29 09:54:03 -07:00
|
|
|
for table in (
|
|
|
|
"event_push_actions",
|
|
|
|
):
|
|
|
|
txn.executemany(
|
|
|
|
"DELETE FROM %s WHERE room_id = ? AND event_id = ?" % (table,),
|
2018-07-25 04:01:36 -06:00
|
|
|
[(ev.room_id, ev.event_id) for ev, _ in events_and_contexts]
|
2016-12-29 09:54:03 -07:00
|
|
|
)
|
|
|
|
|
2017-03-17 05:51:13 -06:00
|
|
|
def _store_event_txn(self, txn, events_and_contexts):
|
|
|
|
"""Insert new events into the event and event_json tables
|
|
|
|
|
|
|
|
Args:
|
|
|
|
txn (twisted.enterprise.adbapi.Connection): db connection
|
|
|
|
events_and_contexts (list[(EventBase, EventContext)]): events
|
|
|
|
we are persisting
|
|
|
|
"""
|
|
|
|
|
|
|
|
if not events_and_contexts:
|
|
|
|
# nothing to do here
|
|
|
|
return
|
|
|
|
|
|
|
|
def event_dict(event):
|
2017-03-24 04:57:02 -06:00
|
|
|
d = event.get_dict()
|
|
|
|
d.pop("redacted", None)
|
|
|
|
d.pop("redacted_because", None)
|
|
|
|
return d
|
2016-08-04 08:02:15 -06:00
|
|
|
|
2015-06-25 10:18:19 -06:00
|
|
|
self._simple_insert_many_txn(
|
|
|
|
txn,
|
|
|
|
table="event_json",
|
|
|
|
values=[
|
|
|
|
{
|
|
|
|
"event_id": event.event_id,
|
|
|
|
"room_id": event.room_id,
|
|
|
|
"internal_metadata": encode_json(
|
2015-08-24 09:17:38 -06:00
|
|
|
event.internal_metadata.get_dict()
|
2018-08-01 08:54:06 -06:00
|
|
|
),
|
|
|
|
"json": encode_json(event_dict(event)),
|
2015-06-25 10:18:19 -06:00
|
|
|
}
|
|
|
|
for event, _ in events_and_contexts
|
|
|
|
],
|
2015-04-15 03:24:24 -06:00
|
|
|
)
|
2015-03-20 07:52:56 -06:00
|
|
|
|
2015-06-25 10:18:19 -06:00
|
|
|
self._simple_insert_many_txn(
|
|
|
|
txn,
|
|
|
|
table="events",
|
|
|
|
values=[
|
|
|
|
{
|
|
|
|
"stream_ordering": event.internal_metadata.stream_ordering,
|
|
|
|
"topological_ordering": event.depth,
|
|
|
|
"depth": event.depth,
|
|
|
|
"event_id": event.event_id,
|
|
|
|
"room_id": event.room_id,
|
|
|
|
"type": event.type,
|
|
|
|
"processed": True,
|
|
|
|
"outlier": event.internal_metadata.is_outlier(),
|
2015-11-30 10:45:31 -07:00
|
|
|
"origin_server_ts": int(event.origin_server_ts),
|
2016-04-19 07:24:36 -06:00
|
|
|
"received_ts": self._clock.time_msec(),
|
2016-07-14 08:15:22 -06:00
|
|
|
"sender": event.sender,
|
|
|
|
"contains_url": (
|
|
|
|
"url" in event.content
|
2018-08-30 08:19:58 -06:00
|
|
|
and isinstance(event.content["url"], text_type)
|
2016-07-14 08:15:22 -06:00
|
|
|
),
|
2015-06-25 10:18:19 -06:00
|
|
|
}
|
|
|
|
for event, _ in events_and_contexts
|
|
|
|
],
|
2015-03-20 07:52:56 -06:00
|
|
|
)
|
|
|
|
|
2017-03-17 05:51:13 -06:00
|
|
|
def _store_rejected_events_txn(self, txn, events_and_contexts):
|
|
|
|
"""Add rows to the 'rejections' table for received events which were
|
|
|
|
rejected
|
|
|
|
|
|
|
|
Args:
|
|
|
|
txn (twisted.enterprise.adbapi.Connection): db connection
|
|
|
|
events_and_contexts (list[(EventBase, EventContext)]): events
|
|
|
|
we are persisting
|
|
|
|
|
|
|
|
Returns:
|
|
|
|
list[(EventBase, EventContext)] new list, without the rejected
|
|
|
|
events.
|
|
|
|
"""
|
2016-07-26 03:49:52 -06:00
|
|
|
# Remove the rejected events from the list now that we've added them
|
|
|
|
# to the events table and the events_json table.
|
2016-07-25 11:44:30 -06:00
|
|
|
to_remove = set()
|
2016-07-25 09:12:16 -06:00
|
|
|
for event, context in events_and_contexts:
|
|
|
|
if context.rejected:
|
2016-07-26 03:49:52 -06:00
|
|
|
# Insert the event_id into the rejections table
|
2016-07-25 09:12:16 -06:00
|
|
|
self._store_rejections_txn(
|
|
|
|
txn, event.event_id, context.rejected
|
|
|
|
)
|
2016-07-26 03:49:52 -06:00
|
|
|
to_remove.add(event)
|
2016-07-25 11:44:30 -06:00
|
|
|
|
2017-03-17 05:51:13 -06:00
|
|
|
return [
|
2016-07-26 03:49:52 -06:00
|
|
|
ec for ec in events_and_contexts if ec[0] not in to_remove
|
2016-07-25 11:44:30 -06:00
|
|
|
]
|
|
|
|
|
2018-02-20 05:33:04 -07:00
|
|
|
def _update_metadata_tables_txn(self, txn, events_and_contexts,
|
|
|
|
all_events_and_contexts, backfilled):
|
2017-03-17 05:51:13 -06:00
|
|
|
"""Update all the miscellaneous tables for new events
|
|
|
|
|
|
|
|
Args:
|
|
|
|
txn (twisted.enterprise.adbapi.Connection): db connection
|
|
|
|
events_and_contexts (list[(EventBase, EventContext)]): events
|
|
|
|
we are persisting
|
2018-02-20 05:33:04 -07:00
|
|
|
all_events_and_contexts (list[(EventBase, EventContext)]): all
|
|
|
|
events that we were going to persist. This includes events
|
|
|
|
we've already persisted, etc, that wouldn't appear in
|
|
|
|
events_and_context.
|
2017-03-17 05:51:13 -06:00
|
|
|
backfilled (bool): True if the events were backfilled
|
|
|
|
"""
|
|
|
|
|
2018-02-20 05:29:50 -07:00
|
|
|
# Insert all the push actions into the event_push_actions table.
|
|
|
|
self._set_push_actions_for_event_and_users_txn(
|
|
|
|
txn,
|
|
|
|
events_and_contexts=events_and_contexts,
|
2018-02-20 05:33:04 -07:00
|
|
|
all_events_and_contexts=all_events_and_contexts,
|
2018-02-20 05:29:50 -07:00
|
|
|
)
|
|
|
|
|
2016-07-25 11:44:30 -06:00
|
|
|
if not events_and_contexts:
|
2017-03-17 05:51:13 -06:00
|
|
|
# nothing to do here
|
2016-07-25 11:44:30 -06:00
|
|
|
return
|
|
|
|
|
|
|
|
for event, context in events_and_contexts:
|
2016-07-26 04:05:39 -06:00
|
|
|
if event.type == EventTypes.Redaction and event.redacts is not None:
|
|
|
|
# Remove the entries in the event_push_actions table for the
|
|
|
|
# redacted event.
|
|
|
|
self._remove_push_actions_for_event_id_txn(
|
|
|
|
txn, event.room_id, event.redacts
|
|
|
|
)
|
|
|
|
|
2016-07-26 03:49:52 -06:00
|
|
|
# Update the event_forward_extremities, event_backward_extremities and
|
|
|
|
# event_edges tables.
|
2016-07-25 11:44:30 -06:00
|
|
|
self._handle_mult_prev_events(
|
|
|
|
txn,
|
|
|
|
events=[event for event, _ in events_and_contexts],
|
|
|
|
)
|
|
|
|
|
|
|
|
for event, _ in events_and_contexts:
|
|
|
|
if event.type == EventTypes.Name:
|
2016-07-26 03:49:52 -06:00
|
|
|
# Insert into the room_names and event_search tables.
|
2016-07-25 11:44:30 -06:00
|
|
|
self._store_room_name_txn(txn, event)
|
|
|
|
elif event.type == EventTypes.Topic:
|
2016-07-26 03:49:52 -06:00
|
|
|
# Insert into the topics table and event_search table.
|
2016-07-25 11:44:30 -06:00
|
|
|
self._store_room_topic_txn(txn, event)
|
|
|
|
elif event.type == EventTypes.Message:
|
2016-07-26 03:49:52 -06:00
|
|
|
# Insert into the event_search table.
|
2016-07-25 11:44:30 -06:00
|
|
|
self._store_room_message_txn(txn, event)
|
|
|
|
elif event.type == EventTypes.Redaction:
|
2016-07-26 03:49:52 -06:00
|
|
|
# Insert into the redactions table.
|
2016-07-25 11:44:30 -06:00
|
|
|
self._store_redaction(txn, event)
|
|
|
|
elif event.type == EventTypes.RoomHistoryVisibility:
|
2016-07-26 03:49:52 -06:00
|
|
|
# Insert into the event_search table.
|
2016-07-25 11:44:30 -06:00
|
|
|
self._store_history_visibility_txn(txn, event)
|
|
|
|
elif event.type == EventTypes.GuestAccess:
|
2016-07-26 03:49:52 -06:00
|
|
|
# Insert into the event_search table.
|
2016-07-25 11:44:30 -06:00
|
|
|
self._store_guest_access_txn(txn, event)
|
|
|
|
|
2016-07-26 03:49:52 -06:00
|
|
|
# Insert into the room_memberships table.
|
2016-07-25 11:44:30 -06:00
|
|
|
self._store_room_members_txn(
|
|
|
|
txn,
|
|
|
|
[
|
|
|
|
event
|
|
|
|
for event, _ in events_and_contexts
|
|
|
|
if event.type == EventTypes.Member
|
|
|
|
],
|
|
|
|
backfilled=backfilled,
|
|
|
|
)
|
|
|
|
|
2016-07-26 03:49:52 -06:00
|
|
|
# Insert event_reference_hashes table.
|
2015-06-25 10:18:19 -06:00
|
|
|
self._store_event_reference_hashes_txn(
|
|
|
|
txn, [event for event, _ in events_and_contexts]
|
|
|
|
)
|
|
|
|
|
2016-03-31 08:00:42 -06:00
|
|
|
state_events_and_contexts = [
|
|
|
|
ec for ec in events_and_contexts if ec[0].is_state()
|
|
|
|
]
|
2015-04-27 06:22:30 -06:00
|
|
|
|
2015-06-25 10:18:19 -06:00
|
|
|
state_values = []
|
|
|
|
for event, context in state_events_and_contexts:
|
2015-03-20 07:52:56 -06:00
|
|
|
vals = {
|
|
|
|
"event_id": event.event_id,
|
|
|
|
"room_id": event.room_id,
|
|
|
|
"type": event.type,
|
|
|
|
"state_key": event.state_key,
|
|
|
|
}
|
|
|
|
|
|
|
|
# TODO: How does this work with backfilling?
|
|
|
|
if hasattr(event, "replaces_state"):
|
|
|
|
vals["prev_state"] = event.replaces_state
|
|
|
|
|
2015-06-25 10:18:19 -06:00
|
|
|
state_values.append(vals)
|
2015-03-20 07:52:56 -06:00
|
|
|
|
2015-06-25 10:18:19 -06:00
|
|
|
self._simple_insert_many_txn(
|
|
|
|
txn,
|
|
|
|
table="state_events",
|
|
|
|
values=state_values,
|
|
|
|
)
|
2015-03-20 07:52:56 -06:00
|
|
|
|
2015-06-25 10:18:19 -06:00
|
|
|
self._simple_insert_many_txn(
|
|
|
|
txn,
|
|
|
|
table="event_edges",
|
|
|
|
values=[
|
|
|
|
{
|
|
|
|
"event_id": event.event_id,
|
|
|
|
"prev_event_id": prev_id,
|
|
|
|
"room_id": event.room_id,
|
|
|
|
"is_state": True,
|
|
|
|
}
|
|
|
|
for event, _ in state_events_and_contexts
|
|
|
|
for prev_id, _ in event.prev_state
|
|
|
|
],
|
|
|
|
)
|
2015-05-21 08:14:26 -06:00
|
|
|
|
2016-07-26 04:05:39 -06:00
|
|
|
# Prefill the event cache
|
2016-06-06 04:08:12 -06:00
|
|
|
self._add_to_cache(txn, events_and_contexts)
|
|
|
|
|
|
|
|
def _add_to_cache(self, txn, events_and_contexts):
|
|
|
|
to_prefill = []
|
|
|
|
|
|
|
|
rows = []
|
|
|
|
N = 200
|
|
|
|
for i in range(0, len(events_and_contexts), N):
|
|
|
|
ev_map = {
|
|
|
|
e[0].event_id: e[0]
|
|
|
|
for e in events_and_contexts[i:i + N]
|
|
|
|
}
|
|
|
|
if not ev_map:
|
|
|
|
break
|
|
|
|
|
|
|
|
sql = (
|
|
|
|
"SELECT "
|
|
|
|
" e.event_id as event_id, "
|
|
|
|
" r.redacts as redacts,"
|
|
|
|
" rej.event_id as rejects "
|
|
|
|
" FROM events as e"
|
|
|
|
" LEFT JOIN rejections as rej USING (event_id)"
|
|
|
|
" LEFT JOIN redactions as r ON e.event_id = r.redacts"
|
|
|
|
" WHERE e.event_id IN (%s)"
|
|
|
|
) % (",".join(["?"] * len(ev_map)),)
|
|
|
|
|
2018-05-31 03:03:47 -06:00
|
|
|
txn.execute(sql, list(ev_map))
|
2016-06-06 04:08:12 -06:00
|
|
|
rows = self.cursor_to_dict(txn)
|
|
|
|
for row in rows:
|
|
|
|
event = ev_map[row["event_id"]]
|
|
|
|
if not row["rejects"] and not row["redacts"]:
|
|
|
|
to_prefill.append(_EventCacheEntry(
|
|
|
|
event=event,
|
|
|
|
redacted_event=None,
|
|
|
|
))
|
|
|
|
|
|
|
|
def prefill():
|
|
|
|
for cache_entry in to_prefill:
|
|
|
|
self._get_event_cache.prefill((cache_entry[0].event_id,), cache_entry)
|
|
|
|
txn.call_after(prefill)
|
|
|
|
|
2015-03-20 07:52:56 -06:00
|
|
|
def _store_redaction(self, txn, event):
|
|
|
|
# invalidate the cache for the redacted event
|
2015-05-05 10:32:21 -06:00
|
|
|
txn.call_after(self._invalidate_get_event_cache, event.redacts)
|
2015-03-20 07:52:56 -06:00
|
|
|
txn.execute(
|
|
|
|
"INSERT INTO redactions (event_id, redacts) VALUES (?,?)",
|
|
|
|
(event.event_id, event.redacts)
|
|
|
|
)
|
|
|
|
|
2015-09-22 05:57:40 -06:00
|
|
|
@defer.inlineCallbacks
|
|
|
|
def count_daily_messages(self):
|
2015-09-22 06:29:36 -06:00
|
|
|
"""
|
|
|
|
Returns an estimate of the number of messages sent in the last day.
|
|
|
|
|
|
|
|
If it has been significantly less or more than one day since the last
|
|
|
|
call to this function, it will return None.
|
|
|
|
"""
|
2015-09-22 05:57:40 -06:00
|
|
|
def _count_messages(txn):
|
2017-06-14 12:37:17 -06:00
|
|
|
sql = """
|
|
|
|
SELECT COALESCE(COUNT(*), 0) FROM events
|
|
|
|
WHERE type = 'm.room.message'
|
|
|
|
AND stream_ordering > ?
|
|
|
|
"""
|
|
|
|
txn.execute(sql, (self.stream_ordering_day_ago,))
|
|
|
|
count, = txn.fetchone()
|
|
|
|
return count
|
2015-09-22 05:57:40 -06:00
|
|
|
|
|
|
|
ret = yield self.runInteraction("count_messages", _count_messages)
|
|
|
|
defer.returnValue(ret)
|
|
|
|
|
2017-06-14 12:37:17 -06:00
|
|
|
@defer.inlineCallbacks
|
|
|
|
def count_daily_sent_messages(self):
|
|
|
|
def _count_messages(txn):
|
|
|
|
# This is good enough as if you have silly characters in your own
|
|
|
|
# hostname then thats your own fault.
|
|
|
|
like_clause = "%:" + self.hs.hostname
|
|
|
|
|
|
|
|
sql = """
|
|
|
|
SELECT COALESCE(COUNT(*), 0) FROM events
|
|
|
|
WHERE type = 'm.room.message'
|
|
|
|
AND sender LIKE ?
|
|
|
|
AND stream_ordering > ?
|
|
|
|
"""
|
|
|
|
|
|
|
|
txn.execute(sql, (like_clause, self.stream_ordering_day_ago,))
|
|
|
|
count, = txn.fetchone()
|
|
|
|
return count
|
|
|
|
|
|
|
|
ret = yield self.runInteraction("count_daily_sent_messages", _count_messages)
|
|
|
|
defer.returnValue(ret)
|
2015-09-22 05:57:40 -06:00
|
|
|
|
2017-06-15 02:39:39 -06:00
|
|
|
@defer.inlineCallbacks
|
|
|
|
def count_daily_active_rooms(self):
|
|
|
|
def _count(txn):
|
|
|
|
sql = """
|
|
|
|
SELECT COALESCE(COUNT(DISTINCT room_id), 0) FROM events
|
|
|
|
WHERE type = 'm.room.message'
|
|
|
|
AND stream_ordering > ?
|
|
|
|
"""
|
|
|
|
txn.execute(sql, (self.stream_ordering_day_ago,))
|
|
|
|
count, = txn.fetchone()
|
|
|
|
return count
|
|
|
|
|
|
|
|
ret = yield self.runInteraction("count_daily_active_rooms", _count)
|
2015-09-22 05:57:40 -06:00
|
|
|
defer.returnValue(ret)
|
2015-11-30 10:45:31 -07:00
|
|
|
|
2016-07-14 08:15:22 -06:00
|
|
|
@defer.inlineCallbacks
|
|
|
|
def _background_reindex_fields_sender(self, progress, batch_size):
|
|
|
|
target_min_stream_id = progress["target_min_stream_id_inclusive"]
|
|
|
|
max_stream_id = progress["max_stream_id_exclusive"]
|
|
|
|
rows_inserted = progress.get("rows_inserted", 0)
|
|
|
|
|
|
|
|
INSERT_CLUMP_SIZE = 1000
|
|
|
|
|
|
|
|
def reindex_txn(txn):
|
|
|
|
sql = (
|
|
|
|
"SELECT stream_ordering, event_id, json FROM events"
|
|
|
|
" INNER JOIN event_json USING (event_id)"
|
|
|
|
" WHERE ? <= stream_ordering AND stream_ordering < ?"
|
|
|
|
" ORDER BY stream_ordering DESC"
|
|
|
|
" LIMIT ?"
|
|
|
|
)
|
|
|
|
|
|
|
|
txn.execute(sql, (target_min_stream_id, max_stream_id, batch_size))
|
|
|
|
|
|
|
|
rows = txn.fetchall()
|
|
|
|
if not rows:
|
|
|
|
return 0
|
|
|
|
|
|
|
|
min_stream_id = rows[-1][0]
|
|
|
|
|
|
|
|
update_rows = []
|
|
|
|
for row in rows:
|
|
|
|
try:
|
|
|
|
event_id = row[1]
|
|
|
|
event_json = json.loads(row[2])
|
|
|
|
sender = event_json["sender"]
|
|
|
|
content = event_json["content"]
|
|
|
|
|
|
|
|
contains_url = "url" in content
|
|
|
|
if contains_url:
|
2018-08-30 08:19:58 -06:00
|
|
|
contains_url &= isinstance(content["url"], text_type)
|
2016-07-14 08:15:22 -06:00
|
|
|
except (KeyError, AttributeError):
|
|
|
|
# If the event is missing a necessary field then
|
|
|
|
# skip over it.
|
|
|
|
continue
|
|
|
|
|
|
|
|
update_rows.append((sender, contains_url, event_id))
|
|
|
|
|
|
|
|
sql = (
|
|
|
|
"UPDATE events SET sender = ?, contains_url = ? WHERE event_id = ?"
|
|
|
|
)
|
|
|
|
|
|
|
|
for index in range(0, len(update_rows), INSERT_CLUMP_SIZE):
|
|
|
|
clump = update_rows[index:index + INSERT_CLUMP_SIZE]
|
|
|
|
txn.executemany(sql, clump)
|
|
|
|
|
|
|
|
progress = {
|
|
|
|
"target_min_stream_id_inclusive": target_min_stream_id,
|
|
|
|
"max_stream_id_exclusive": min_stream_id,
|
|
|
|
"rows_inserted": rows_inserted + len(rows)
|
|
|
|
}
|
|
|
|
|
|
|
|
self._background_update_progress_txn(
|
|
|
|
txn, self.EVENT_FIELDS_SENDER_URL_UPDATE_NAME, progress
|
|
|
|
)
|
|
|
|
|
|
|
|
return len(rows)
|
|
|
|
|
|
|
|
result = yield self.runInteraction(
|
|
|
|
self.EVENT_FIELDS_SENDER_URL_UPDATE_NAME, reindex_txn
|
|
|
|
)
|
|
|
|
|
|
|
|
if not result:
|
|
|
|
yield self._end_background_update(self.EVENT_FIELDS_SENDER_URL_UPDATE_NAME)
|
|
|
|
|
|
|
|
defer.returnValue(result)
|
|
|
|
|
2015-11-30 10:45:31 -07:00
|
|
|
@defer.inlineCallbacks
|
|
|
|
def _background_reindex_origin_server_ts(self, progress, batch_size):
|
|
|
|
target_min_stream_id = progress["target_min_stream_id_inclusive"]
|
|
|
|
max_stream_id = progress["max_stream_id_exclusive"]
|
|
|
|
rows_inserted = progress.get("rows_inserted", 0)
|
|
|
|
|
|
|
|
INSERT_CLUMP_SIZE = 1000
|
|
|
|
|
|
|
|
def reindex_search_txn(txn):
|
|
|
|
sql = (
|
|
|
|
"SELECT stream_ordering, event_id FROM events"
|
|
|
|
" WHERE ? <= stream_ordering AND stream_ordering < ?"
|
|
|
|
" ORDER BY stream_ordering DESC"
|
|
|
|
" LIMIT ?"
|
|
|
|
)
|
|
|
|
|
|
|
|
txn.execute(sql, (target_min_stream_id, max_stream_id, batch_size))
|
|
|
|
|
|
|
|
rows = txn.fetchall()
|
|
|
|
if not rows:
|
|
|
|
return 0
|
|
|
|
|
|
|
|
min_stream_id = rows[-1][0]
|
|
|
|
event_ids = [row[1] for row in rows]
|
|
|
|
|
2016-09-27 02:43:27 -06:00
|
|
|
rows_to_update = []
|
2015-11-30 10:45:31 -07:00
|
|
|
|
2016-09-27 02:43:27 -06:00
|
|
|
chunks = [
|
|
|
|
event_ids[i:i + 100]
|
2018-05-31 03:03:47 -06:00
|
|
|
for i in range(0, len(event_ids), 100)
|
2016-09-27 02:43:27 -06:00
|
|
|
]
|
|
|
|
for chunk in chunks:
|
|
|
|
ev_rows = self._simple_select_many_txn(
|
|
|
|
txn,
|
|
|
|
table="event_json",
|
|
|
|
column="event_id",
|
|
|
|
iterable=chunk,
|
|
|
|
retcols=["event_id", "json"],
|
|
|
|
keyvalues={},
|
|
|
|
)
|
2015-11-30 10:45:31 -07:00
|
|
|
|
2016-09-27 02:43:27 -06:00
|
|
|
for row in ev_rows:
|
|
|
|
event_id = row["event_id"]
|
|
|
|
event_json = json.loads(row["json"])
|
|
|
|
try:
|
|
|
|
origin_server_ts = event_json["origin_server_ts"]
|
|
|
|
except (KeyError, AttributeError):
|
|
|
|
# If the event is missing a necessary field then
|
|
|
|
# skip over it.
|
|
|
|
continue
|
|
|
|
|
|
|
|
rows_to_update.append((origin_server_ts, event_id))
|
2015-11-30 10:45:31 -07:00
|
|
|
|
|
|
|
sql = (
|
|
|
|
"UPDATE events SET origin_server_ts = ? WHERE event_id = ?"
|
|
|
|
)
|
|
|
|
|
2016-09-27 02:43:27 -06:00
|
|
|
for index in range(0, len(rows_to_update), INSERT_CLUMP_SIZE):
|
|
|
|
clump = rows_to_update[index:index + INSERT_CLUMP_SIZE]
|
2015-11-30 10:45:31 -07:00
|
|
|
txn.executemany(sql, clump)
|
|
|
|
|
|
|
|
progress = {
|
|
|
|
"target_min_stream_id_inclusive": target_min_stream_id,
|
|
|
|
"max_stream_id_exclusive": min_stream_id,
|
2016-09-27 02:43:27 -06:00
|
|
|
"rows_inserted": rows_inserted + len(rows_to_update)
|
2015-11-30 10:45:31 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
self._background_update_progress_txn(
|
|
|
|
txn, self.EVENT_ORIGIN_SERVER_TS_NAME, progress
|
|
|
|
)
|
|
|
|
|
2016-09-27 02:43:27 -06:00
|
|
|
return len(rows_to_update)
|
2015-11-30 10:45:31 -07:00
|
|
|
|
|
|
|
result = yield self.runInteraction(
|
|
|
|
self.EVENT_ORIGIN_SERVER_TS_NAME, reindex_search_txn
|
|
|
|
)
|
|
|
|
|
|
|
|
if not result:
|
|
|
|
yield self._end_background_update(self.EVENT_ORIGIN_SERVER_TS_NAME)
|
|
|
|
|
|
|
|
defer.returnValue(result)
|
2016-03-01 07:49:41 -07:00
|
|
|
|
|
|
|
def get_current_backfill_token(self):
|
|
|
|
"""The current minimum token that backfilled events have reached"""
|
2016-04-01 06:29:05 -06:00
|
|
|
return -self._backfill_id_gen.get_current_token()
|
2016-03-01 07:49:41 -07:00
|
|
|
|
2017-03-27 07:03:38 -06:00
|
|
|
def get_current_events_token(self):
|
|
|
|
"""The current maximum token that events have reached"""
|
|
|
|
return self._stream_id_gen.get_current_token()
|
|
|
|
|
|
|
|
def get_all_new_forward_event_rows(self, last_id, current_id, limit):
|
|
|
|
if last_id == current_id:
|
|
|
|
return defer.succeed([])
|
|
|
|
|
|
|
|
def get_all_new_forward_event_rows(txn):
|
|
|
|
sql = (
|
|
|
|
"SELECT e.stream_ordering, e.event_id, e.room_id, e.type,"
|
|
|
|
" state_key, redacts"
|
|
|
|
" FROM events AS e"
|
|
|
|
" LEFT JOIN redactions USING (event_id)"
|
|
|
|
" LEFT JOIN state_events USING (event_id)"
|
|
|
|
" WHERE ? < stream_ordering AND stream_ordering <= ?"
|
|
|
|
" ORDER BY stream_ordering ASC"
|
|
|
|
" LIMIT ?"
|
|
|
|
)
|
|
|
|
txn.execute(sql, (last_id, current_id, limit))
|
|
|
|
new_event_updates = txn.fetchall()
|
|
|
|
|
|
|
|
if len(new_event_updates) == limit:
|
|
|
|
upper_bound = new_event_updates[-1][0]
|
|
|
|
else:
|
|
|
|
upper_bound = current_id
|
|
|
|
|
|
|
|
sql = (
|
|
|
|
"SELECT event_stream_ordering, e.event_id, e.room_id, e.type,"
|
|
|
|
" state_key, redacts"
|
|
|
|
" FROM events AS e"
|
|
|
|
" INNER JOIN ex_outlier_stream USING (event_id)"
|
|
|
|
" LEFT JOIN redactions USING (event_id)"
|
|
|
|
" LEFT JOIN state_events USING (event_id)"
|
|
|
|
" WHERE ? < event_stream_ordering"
|
|
|
|
" AND event_stream_ordering <= ?"
|
|
|
|
" ORDER BY event_stream_ordering DESC"
|
|
|
|
)
|
|
|
|
txn.execute(sql, (last_id, upper_bound))
|
|
|
|
new_event_updates.extend(txn)
|
|
|
|
|
|
|
|
return new_event_updates
|
|
|
|
return self.runInteraction(
|
|
|
|
"get_all_new_forward_event_rows", get_all_new_forward_event_rows
|
|
|
|
)
|
|
|
|
|
|
|
|
def get_all_new_backfill_event_rows(self, last_id, current_id, limit):
|
|
|
|
if last_id == current_id:
|
|
|
|
return defer.succeed([])
|
|
|
|
|
|
|
|
def get_all_new_backfill_event_rows(txn):
|
|
|
|
sql = (
|
|
|
|
"SELECT -e.stream_ordering, e.event_id, e.room_id, e.type,"
|
|
|
|
" state_key, redacts"
|
|
|
|
" FROM events AS e"
|
|
|
|
" LEFT JOIN redactions USING (event_id)"
|
|
|
|
" LEFT JOIN state_events USING (event_id)"
|
|
|
|
" WHERE ? > stream_ordering AND stream_ordering >= ?"
|
|
|
|
" ORDER BY stream_ordering ASC"
|
|
|
|
" LIMIT ?"
|
|
|
|
)
|
|
|
|
txn.execute(sql, (-last_id, -current_id, limit))
|
|
|
|
new_event_updates = txn.fetchall()
|
|
|
|
|
|
|
|
if len(new_event_updates) == limit:
|
|
|
|
upper_bound = new_event_updates[-1][0]
|
|
|
|
else:
|
|
|
|
upper_bound = current_id
|
|
|
|
|
|
|
|
sql = (
|
|
|
|
"SELECT -event_stream_ordering, e.event_id, e.room_id, e.type,"
|
|
|
|
" state_key, redacts"
|
|
|
|
" FROM events AS e"
|
|
|
|
" INNER JOIN ex_outlier_stream USING (event_id)"
|
|
|
|
" LEFT JOIN redactions USING (event_id)"
|
|
|
|
" LEFT JOIN state_events USING (event_id)"
|
|
|
|
" WHERE ? > event_stream_ordering"
|
|
|
|
" AND event_stream_ordering >= ?"
|
|
|
|
" ORDER BY event_stream_ordering DESC"
|
|
|
|
)
|
|
|
|
txn.execute(sql, (-last_id, -upper_bound))
|
|
|
|
new_event_updates.extend(txn.fetchall())
|
|
|
|
|
|
|
|
return new_event_updates
|
|
|
|
return self.runInteraction(
|
|
|
|
"get_all_new_backfill_event_rows", get_all_new_backfill_event_rows
|
|
|
|
)
|
|
|
|
|
2017-02-01 03:42:37 -07:00
|
|
|
@cached(num_args=5, max_entries=10)
|
2016-03-01 07:49:41 -07:00
|
|
|
def get_all_new_events(self, last_backfill_id, last_forward_id,
|
|
|
|
current_backfill_id, current_forward_id, limit):
|
|
|
|
"""Get all the new events that have arrived at the server either as
|
|
|
|
new events or as backfilled events"""
|
2016-04-27 04:54:13 -06:00
|
|
|
have_backfill_events = last_backfill_id != current_backfill_id
|
|
|
|
have_forward_events = last_forward_id != current_forward_id
|
|
|
|
|
|
|
|
if not have_backfill_events and not have_forward_events:
|
|
|
|
return defer.succeed(AllNewEventsResult([], [], [], [], []))
|
|
|
|
|
2016-03-01 07:49:41 -07:00
|
|
|
def get_all_new_events_txn(txn):
|
|
|
|
sql = (
|
2017-03-17 09:47:51 -06:00
|
|
|
"SELECT e.stream_ordering, e.event_id, e.room_id, e.type,"
|
|
|
|
" state_key, redacts"
|
|
|
|
" FROM events AS e"
|
|
|
|
" LEFT JOIN redactions USING (event_id)"
|
|
|
|
" LEFT JOIN state_events USING (event_id)"
|
|
|
|
" WHERE ? < stream_ordering AND stream_ordering <= ?"
|
|
|
|
" ORDER BY stream_ordering ASC"
|
2016-03-01 07:49:41 -07:00
|
|
|
" LIMIT ?"
|
|
|
|
)
|
2016-04-27 04:54:13 -06:00
|
|
|
if have_forward_events:
|
2016-03-01 07:49:41 -07:00
|
|
|
txn.execute(sql, (last_forward_id, current_forward_id, limit))
|
|
|
|
new_forward_events = txn.fetchall()
|
2016-03-30 10:19:56 -06:00
|
|
|
|
|
|
|
if len(new_forward_events) == limit:
|
|
|
|
upper_bound = new_forward_events[-1][0]
|
|
|
|
else:
|
|
|
|
upper_bound = current_forward_id
|
|
|
|
|
|
|
|
sql = (
|
2016-04-06 07:12:51 -06:00
|
|
|
"SELECT event_stream_ordering, event_id, state_group"
|
2016-03-30 10:19:56 -06:00
|
|
|
" FROM ex_outlier_stream"
|
|
|
|
" WHERE ? > event_stream_ordering"
|
|
|
|
" AND event_stream_ordering >= ?"
|
|
|
|
" ORDER BY event_stream_ordering DESC"
|
|
|
|
)
|
|
|
|
txn.execute(sql, (last_forward_id, upper_bound))
|
|
|
|
forward_ex_outliers = txn.fetchall()
|
2016-03-01 07:49:41 -07:00
|
|
|
else:
|
|
|
|
new_forward_events = []
|
2016-03-30 10:19:56 -06:00
|
|
|
forward_ex_outliers = []
|
2016-03-01 07:49:41 -07:00
|
|
|
|
|
|
|
sql = (
|
2017-03-17 09:47:51 -06:00
|
|
|
"SELECT -e.stream_ordering, e.event_id, e.room_id, e.type,"
|
|
|
|
" state_key, redacts"
|
|
|
|
" FROM events AS e"
|
|
|
|
" LEFT JOIN redactions USING (event_id)"
|
|
|
|
" LEFT JOIN state_events USING (event_id)"
|
|
|
|
" WHERE ? > stream_ordering AND stream_ordering >= ?"
|
|
|
|
" ORDER BY stream_ordering DESC"
|
2016-03-01 07:49:41 -07:00
|
|
|
" LIMIT ?"
|
|
|
|
)
|
2016-04-27 04:54:13 -06:00
|
|
|
if have_backfill_events:
|
2016-03-01 07:49:41 -07:00
|
|
|
txn.execute(sql, (-last_backfill_id, -current_backfill_id, limit))
|
|
|
|
new_backfill_events = txn.fetchall()
|
2016-03-30 10:19:56 -06:00
|
|
|
|
|
|
|
if len(new_backfill_events) == limit:
|
|
|
|
upper_bound = new_backfill_events[-1][0]
|
|
|
|
else:
|
|
|
|
upper_bound = current_backfill_id
|
|
|
|
|
|
|
|
sql = (
|
|
|
|
"SELECT -event_stream_ordering, event_id, state_group"
|
|
|
|
" FROM ex_outlier_stream"
|
|
|
|
" WHERE ? > event_stream_ordering"
|
|
|
|
" AND event_stream_ordering >= ?"
|
|
|
|
" ORDER BY event_stream_ordering DESC"
|
|
|
|
)
|
|
|
|
txn.execute(sql, (-last_backfill_id, -upper_bound))
|
|
|
|
backward_ex_outliers = txn.fetchall()
|
2016-03-01 07:49:41 -07:00
|
|
|
else:
|
|
|
|
new_backfill_events = []
|
2016-03-30 10:19:56 -06:00
|
|
|
backward_ex_outliers = []
|
2016-03-01 07:49:41 -07:00
|
|
|
|
2016-03-31 03:33:02 -06:00
|
|
|
return AllNewEventsResult(
|
2016-03-30 10:19:56 -06:00
|
|
|
new_forward_events, new_backfill_events,
|
|
|
|
forward_ex_outliers, backward_ex_outliers,
|
|
|
|
)
|
2016-03-01 07:49:41 -07:00
|
|
|
return self.runInteraction("get_all_new_events", get_all_new_events_txn)
|
2016-03-31 03:33:02 -06:00
|
|
|
|
2018-02-08 11:44:52 -07:00
|
|
|
def purge_history(
|
2018-05-15 09:06:30 -06:00
|
|
|
self, room_id, token, delete_local_events,
|
2018-02-08 11:44:52 -07:00
|
|
|
):
|
2018-02-07 10:27:08 -07:00
|
|
|
"""Deletes room history before a certain point
|
2018-02-08 11:44:52 -07:00
|
|
|
|
|
|
|
Args:
|
|
|
|
room_id (str):
|
|
|
|
|
2018-05-15 09:06:30 -06:00
|
|
|
token (str): A topological token to delete events before
|
2016-07-05 03:28:51 -06:00
|
|
|
|
2018-02-08 11:44:52 -07:00
|
|
|
delete_local_events (bool):
|
|
|
|
if True, we will delete local events as well as remote ones
|
|
|
|
(instead of just marking them as outliers and deleting their
|
|
|
|
state groups).
|
2016-07-04 09:02:50 -06:00
|
|
|
"""
|
|
|
|
|
2016-07-05 03:28:51 -06:00
|
|
|
return self.runInteraction(
|
2018-02-07 10:27:08 -07:00
|
|
|
"purge_history",
|
2018-05-15 09:06:30 -06:00
|
|
|
self._purge_history_txn, room_id, token,
|
2018-02-08 11:44:52 -07:00
|
|
|
delete_local_events,
|
2016-07-05 03:28:51 -06:00
|
|
|
)
|
|
|
|
|
2018-02-08 11:44:52 -07:00
|
|
|
def _purge_history_txn(
|
2018-05-16 03:52:06 -06:00
|
|
|
self, txn, room_id, token_str, delete_local_events,
|
2018-02-08 11:44:52 -07:00
|
|
|
):
|
2018-05-16 03:52:06 -06:00
|
|
|
token = RoomStreamToken.parse(token_str)
|
|
|
|
|
2016-07-04 09:02:50 -06:00
|
|
|
# Tables that should be pruned:
|
|
|
|
# event_auth
|
|
|
|
# event_backward_extremities
|
|
|
|
# event_content_hashes
|
|
|
|
# event_destinations
|
|
|
|
# event_edge_hashes
|
|
|
|
# event_edges
|
|
|
|
# event_forward_extremities
|
|
|
|
# event_json
|
|
|
|
# event_push_actions
|
|
|
|
# event_reference_hashes
|
|
|
|
# event_search
|
|
|
|
# event_signatures
|
|
|
|
# event_to_state_groups
|
|
|
|
# events
|
|
|
|
# rejections
|
|
|
|
# room_depth
|
|
|
|
# state_groups
|
|
|
|
# state_groups_state
|
|
|
|
|
2018-02-14 09:41:12 -07:00
|
|
|
# we will build a temporary table listing the events so that we don't
|
|
|
|
# have to keep shovelling the list back and forth across the
|
|
|
|
# connection. Annoyingly the python sqlite driver commits the
|
|
|
|
# transaction on CREATE, so let's do this first.
|
|
|
|
#
|
|
|
|
# furthermore, we might already have the table from a previous (failed)
|
|
|
|
# purge attempt, so let's drop the table first.
|
|
|
|
|
|
|
|
txn.execute("DROP TABLE IF EXISTS events_to_purge")
|
|
|
|
|
|
|
|
txn.execute(
|
|
|
|
"CREATE TEMPORARY TABLE events_to_purge ("
|
|
|
|
" event_id TEXT NOT NULL,"
|
|
|
|
" should_delete BOOLEAN NOT NULL"
|
|
|
|
")"
|
|
|
|
)
|
|
|
|
|
2016-07-04 09:02:50 -06:00
|
|
|
# First ensure that we're not about to delete all the forward extremeties
|
|
|
|
txn.execute(
|
|
|
|
"SELECT e.event_id, e.depth FROM events as e "
|
|
|
|
"INNER JOIN event_forward_extremities as f "
|
|
|
|
"ON e.event_id = f.event_id "
|
|
|
|
"AND e.room_id = f.room_id "
|
|
|
|
"WHERE f.room_id = ?",
|
|
|
|
(room_id,)
|
|
|
|
)
|
|
|
|
rows = txn.fetchall()
|
2018-08-30 08:19:58 -06:00
|
|
|
max_depth = max(row[1] for row in rows)
|
2016-07-04 09:02:50 -06:00
|
|
|
|
2018-08-30 08:19:58 -06:00
|
|
|
if max_depth < token.topological:
|
2018-08-15 09:35:22 -06:00
|
|
|
# We need to ensure we don't delete all the events from the database
|
2016-07-07 04:42:15 -06:00
|
|
|
# otherwise we wouldn't be able to send any events (due to not
|
|
|
|
# having any backwards extremeties)
|
2016-07-07 04:41:07 -06:00
|
|
|
raise SynapseError(
|
|
|
|
400, "topological_ordering is greater than forward extremeties"
|
|
|
|
)
|
2016-07-04 09:02:50 -06:00
|
|
|
|
2018-02-07 10:34:35 -07:00
|
|
|
logger.info("[purge] looking for events to delete")
|
2017-05-11 05:06:28 -06:00
|
|
|
|
2018-02-14 04:02:22 -07:00
|
|
|
should_delete_expr = "state_key IS NULL"
|
|
|
|
should_delete_params = ()
|
|
|
|
if not delete_local_events:
|
|
|
|
should_delete_expr += " AND event_id NOT LIKE ?"
|
2018-09-13 09:10:56 -06:00
|
|
|
|
|
|
|
# We include the parameter twice since we use the expression twice
|
2018-09-13 05:48:10 -06:00
|
|
|
should_delete_params += (
|
|
|
|
"%:" + self.hs.hostname,
|
|
|
|
"%:" + self.hs.hostname,
|
|
|
|
)
|
2018-02-14 04:02:22 -07:00
|
|
|
|
2018-05-15 09:06:30 -06:00
|
|
|
should_delete_params += (room_id, token.topological)
|
2018-02-14 04:02:22 -07:00
|
|
|
|
2018-09-13 05:48:10 -06:00
|
|
|
# Note that we insert events that are outliers and aren't going to be
|
|
|
|
# deleted, as nothing will happen to them.
|
2018-02-14 04:02:22 -07:00
|
|
|
txn.execute(
|
|
|
|
"INSERT INTO events_to_purge"
|
|
|
|
" SELECT event_id, %s"
|
|
|
|
" FROM events AS e LEFT JOIN state_events USING (event_id)"
|
2018-09-13 05:48:10 -06:00
|
|
|
" WHERE (NOT outlier OR (%s)) AND e.room_id = ? AND topological_ordering < ?"
|
|
|
|
% (
|
|
|
|
should_delete_expr,
|
2018-02-14 04:02:22 -07:00
|
|
|
should_delete_expr,
|
|
|
|
),
|
|
|
|
should_delete_params,
|
|
|
|
)
|
2018-09-13 08:05:52 -06:00
|
|
|
|
|
|
|
# We create the indices *after* insertion as that's a lot faster.
|
|
|
|
|
|
|
|
# create an index on should_delete because later we'll be looking for
|
|
|
|
# the should_delete / shouldn't_delete subsets
|
|
|
|
txn.execute(
|
|
|
|
"CREATE INDEX events_to_purge_should_delete"
|
|
|
|
" ON events_to_purge(should_delete)",
|
|
|
|
)
|
|
|
|
|
|
|
|
# We do joins against events_to_purge for e.g. calculating state
|
|
|
|
# groups to purge, etc., so lets make an index.
|
|
|
|
txn.execute(
|
|
|
|
"CREATE INDEX events_to_purge_id"
|
|
|
|
" ON events_to_purge(event_id)",
|
|
|
|
)
|
|
|
|
|
2018-02-14 04:02:22 -07:00
|
|
|
txn.execute(
|
|
|
|
"SELECT event_id, should_delete FROM events_to_purge"
|
|
|
|
)
|
|
|
|
event_rows = txn.fetchall()
|
2017-05-11 05:06:28 -06:00
|
|
|
logger.info(
|
2018-02-08 11:44:52 -07:00
|
|
|
"[purge] found %i events before cutoff, of which %i can be deleted",
|
2018-02-14 04:02:22 -07:00
|
|
|
len(event_rows), sum(1 for e in event_rows if e[1]),
|
2018-02-08 11:44:52 -07:00
|
|
|
)
|
2016-09-02 03:41:38 -06:00
|
|
|
|
2018-02-07 10:34:35 -07:00
|
|
|
logger.info("[purge] Finding new backward extremities")
|
2017-05-10 10:46:41 -06:00
|
|
|
|
2016-07-04 09:02:50 -06:00
|
|
|
# We calculate the new entries for the backward extremeties by finding
|
2018-05-16 04:47:26 -06:00
|
|
|
# events to be purged that are pointed to by events we're not going to
|
|
|
|
# purge.
|
2016-07-04 09:02:50 -06:00
|
|
|
txn.execute(
|
2018-02-14 04:02:22 -07:00
|
|
|
"SELECT DISTINCT e.event_id FROM events_to_purge AS e"
|
|
|
|
" INNER JOIN event_edges AS ed ON e.event_id = ed.prev_event_id"
|
2018-05-15 08:48:40 -06:00
|
|
|
" LEFT JOIN events_to_purge AS ep2 ON ed.event_id = ep2.event_id"
|
|
|
|
" WHERE ep2.event_id IS NULL",
|
2016-07-04 09:02:50 -06:00
|
|
|
)
|
|
|
|
new_backwards_extrems = txn.fetchall()
|
|
|
|
|
2018-02-07 10:34:35 -07:00
|
|
|
logger.info("[purge] replacing backward extremities: %r", new_backwards_extrems)
|
2017-05-10 10:46:41 -06:00
|
|
|
|
2016-07-15 07:23:15 -06:00
|
|
|
txn.execute(
|
|
|
|
"DELETE FROM event_backward_extremities WHERE room_id = ?",
|
|
|
|
(room_id,)
|
|
|
|
)
|
|
|
|
|
|
|
|
# Update backward extremeties
|
|
|
|
txn.executemany(
|
|
|
|
"INSERT INTO event_backward_extremities (room_id, event_id)"
|
|
|
|
" VALUES (?, ?)",
|
|
|
|
[
|
|
|
|
(room_id, event_id) for event_id, in new_backwards_extrems
|
|
|
|
]
|
|
|
|
)
|
|
|
|
|
2018-02-07 10:34:35 -07:00
|
|
|
logger.info("[purge] finding redundant state groups")
|
2017-05-10 10:46:41 -06:00
|
|
|
|
2016-07-04 09:02:50 -06:00
|
|
|
# Get all state groups that are only referenced by events that are
|
|
|
|
# to be deleted.
|
2018-05-15 08:48:40 -06:00
|
|
|
# This works by first getting state groups that we may want to delete,
|
|
|
|
# joining against event_to_state_groups to get events that use that
|
|
|
|
# state group, then left joining against events_to_purge again. Any
|
|
|
|
# state group where the left join produce *no nulls* are referenced
|
|
|
|
# only by events that are going to be purged.
|
|
|
|
txn.execute("""
|
|
|
|
SELECT state_group FROM
|
|
|
|
(
|
|
|
|
SELECT DISTINCT state_group FROM events_to_purge
|
|
|
|
INNER JOIN event_to_state_groups USING (event_id)
|
|
|
|
) AS sp
|
|
|
|
INNER JOIN event_to_state_groups USING (state_group)
|
|
|
|
LEFT JOIN events_to_purge AS ep USING (event_id)
|
|
|
|
GROUP BY state_group
|
|
|
|
HAVING SUM(CASE WHEN ep.event_id IS NULL THEN 1 ELSE 0 END) = 0
|
|
|
|
""")
|
2016-09-05 07:49:08 -06:00
|
|
|
|
|
|
|
state_rows = txn.fetchall()
|
2018-02-07 10:34:35 -07:00
|
|
|
logger.info("[purge] found %i redundant state groups", len(state_rows))
|
2017-05-11 03:56:12 -06:00
|
|
|
|
|
|
|
# make a set of the redundant state groups, so that we can look them up
|
|
|
|
# efficiently
|
2017-05-10 11:17:41 -06:00
|
|
|
state_groups_to_delete = set([sg for sg, in state_rows])
|
2017-05-10 10:46:41 -06:00
|
|
|
|
2016-09-05 07:49:08 -06:00
|
|
|
# Now we get all the state groups that rely on these state groups
|
2018-02-07 10:34:35 -07:00
|
|
|
logger.info("[purge] finding state groups which depend on redundant"
|
|
|
|
" state groups")
|
2017-05-11 03:56:12 -06:00
|
|
|
remaining_state_groups = []
|
2018-05-31 03:03:47 -06:00
|
|
|
for i in range(0, len(state_rows), 100):
|
2017-05-10 11:17:41 -06:00
|
|
|
chunk = [sg for sg, in state_rows[i:i + 100]]
|
2017-05-11 03:56:12 -06:00
|
|
|
# look for state groups whose prev_state_group is one we are about
|
|
|
|
# to delete
|
2016-09-05 07:49:08 -06:00
|
|
|
rows = self._simple_select_many_txn(
|
|
|
|
txn,
|
|
|
|
table="state_group_edges",
|
|
|
|
column="prev_state_group",
|
|
|
|
iterable=chunk,
|
|
|
|
retcols=["state_group"],
|
|
|
|
keyvalues={},
|
|
|
|
)
|
2017-05-11 03:56:12 -06:00
|
|
|
remaining_state_groups.extend(
|
2017-05-10 11:17:41 -06:00
|
|
|
row["state_group"] for row in rows
|
2017-05-11 03:56:12 -06:00
|
|
|
|
|
|
|
# exclude state groups we are about to delete: no point in
|
|
|
|
# updating them
|
2017-05-10 11:17:41 -06:00
|
|
|
if row["state_group"] not in state_groups_to_delete
|
|
|
|
)
|
2016-09-05 07:49:08 -06:00
|
|
|
|
2017-05-11 03:56:12 -06:00
|
|
|
# Now we turn the state groups that reference to-be-deleted state
|
|
|
|
# groups to non delta versions.
|
|
|
|
for sg in remaining_state_groups:
|
2018-02-07 10:34:35 -07:00
|
|
|
logger.info("[purge] de-delta-ing remaining state group %s", sg)
|
2016-09-05 07:49:08 -06:00
|
|
|
curr_state = self._get_state_groups_from_groups_txn(
|
2017-05-11 03:56:12 -06:00
|
|
|
txn, [sg], types=None
|
2016-09-05 07:49:08 -06:00
|
|
|
)
|
2017-05-11 03:56:12 -06:00
|
|
|
curr_state = curr_state[sg]
|
2016-09-05 07:49:08 -06:00
|
|
|
|
|
|
|
self._simple_delete_txn(
|
|
|
|
txn,
|
|
|
|
table="state_groups_state",
|
|
|
|
keyvalues={
|
2017-05-11 03:56:12 -06:00
|
|
|
"state_group": sg,
|
2016-09-05 07:49:08 -06:00
|
|
|
}
|
|
|
|
)
|
|
|
|
|
|
|
|
self._simple_delete_txn(
|
|
|
|
txn,
|
|
|
|
table="state_group_edges",
|
|
|
|
keyvalues={
|
2017-05-11 03:56:12 -06:00
|
|
|
"state_group": sg,
|
2016-09-05 07:49:08 -06:00
|
|
|
}
|
|
|
|
)
|
|
|
|
|
|
|
|
self._simple_insert_many_txn(
|
|
|
|
txn,
|
|
|
|
table="state_groups_state",
|
|
|
|
values=[
|
|
|
|
{
|
2017-05-11 03:56:12 -06:00
|
|
|
"state_group": sg,
|
2016-09-05 07:49:08 -06:00
|
|
|
"room_id": room_id,
|
|
|
|
"type": key[0],
|
|
|
|
"state_key": key[1],
|
|
|
|
"event_id": state_id,
|
|
|
|
}
|
2018-05-31 03:03:47 -06:00
|
|
|
for key, state_id in iteritems(curr_state)
|
2016-09-05 07:49:08 -06:00
|
|
|
],
|
|
|
|
)
|
|
|
|
|
2018-02-07 10:34:35 -07:00
|
|
|
logger.info("[purge] removing redundant state groups")
|
2016-09-05 07:49:08 -06:00
|
|
|
txn.executemany(
|
|
|
|
"DELETE FROM state_groups_state WHERE state_group = ?",
|
|
|
|
state_rows
|
|
|
|
)
|
|
|
|
txn.executemany(
|
|
|
|
"DELETE FROM state_groups WHERE id = ?",
|
|
|
|
state_rows
|
|
|
|
)
|
2017-05-10 10:46:41 -06:00
|
|
|
|
2018-02-07 10:34:35 -07:00
|
|
|
logger.info("[purge] removing events from event_to_state_groups")
|
2018-02-14 04:02:22 -07:00
|
|
|
txn.execute(
|
|
|
|
"DELETE FROM event_to_state_groups "
|
|
|
|
"WHERE event_id IN (SELECT event_id from events_to_purge)"
|
2016-07-04 09:02:50 -06:00
|
|
|
)
|
2018-02-07 10:40:29 -07:00
|
|
|
for event_id, _ in event_rows:
|
|
|
|
txn.call_after(self._get_state_group_for_event.invalidate, (
|
|
|
|
event_id,
|
|
|
|
))
|
2016-07-04 09:02:50 -06:00
|
|
|
|
|
|
|
# Delete all remote non-state events
|
|
|
|
for table in (
|
|
|
|
"events",
|
|
|
|
"event_json",
|
|
|
|
"event_auth",
|
|
|
|
"event_content_hashes",
|
|
|
|
"event_destinations",
|
|
|
|
"event_edge_hashes",
|
|
|
|
"event_edges",
|
|
|
|
"event_forward_extremities",
|
|
|
|
"event_reference_hashes",
|
|
|
|
"event_search",
|
|
|
|
"event_signatures",
|
|
|
|
"rejections",
|
|
|
|
):
|
2018-02-13 09:51:21 -07:00
|
|
|
logger.info("[purge] removing events from %s", table)
|
2017-05-10 10:46:41 -06:00
|
|
|
|
2018-02-14 04:02:22 -07:00
|
|
|
txn.execute(
|
|
|
|
"DELETE FROM %s WHERE event_id IN ("
|
|
|
|
" SELECT event_id FROM events_to_purge WHERE should_delete"
|
|
|
|
")" % (table,),
|
2016-07-04 09:02:50 -06:00
|
|
|
)
|
|
|
|
|
2018-02-14 08:44:51 -07:00
|
|
|
# event_push_actions lacks an index on event_id, and has one on
|
|
|
|
# (room_id, event_id) instead.
|
|
|
|
for table in (
|
|
|
|
"event_push_actions",
|
|
|
|
):
|
|
|
|
logger.info("[purge] removing events from %s", table)
|
|
|
|
|
|
|
|
txn.execute(
|
|
|
|
"DELETE FROM %s WHERE room_id = ? AND event_id IN ("
|
|
|
|
" SELECT event_id FROM events_to_purge WHERE should_delete"
|
|
|
|
")" % (table,),
|
|
|
|
(room_id, )
|
|
|
|
)
|
|
|
|
|
2016-07-04 09:02:50 -06:00
|
|
|
# Mark all state and own events as outliers
|
2018-02-07 10:34:35 -07:00
|
|
|
logger.info("[purge] marking remaining events as outliers")
|
2018-02-14 04:02:22 -07:00
|
|
|
txn.execute(
|
2016-07-04 09:02:50 -06:00
|
|
|
"UPDATE events SET outlier = ?"
|
2018-02-14 04:02:22 -07:00
|
|
|
" WHERE event_id IN ("
|
|
|
|
" SELECT event_id FROM events_to_purge "
|
|
|
|
" WHERE NOT should_delete"
|
|
|
|
")",
|
|
|
|
(True,),
|
|
|
|
)
|
|
|
|
|
2018-02-09 05:13:34 -07:00
|
|
|
# synapse tries to take out an exclusive lock on room_depth whenever it
|
|
|
|
# persists events (because upsert), and once we run this update, we
|
|
|
|
# will block that for the rest of our transaction.
|
|
|
|
#
|
|
|
|
# So, let's stick it at the end so that we don't block event
|
|
|
|
# persistence.
|
2018-05-16 04:13:31 -06:00
|
|
|
#
|
|
|
|
# We do this by calculating the minimum depth of the backwards
|
|
|
|
# extremities. However, the events in event_backward_extremities
|
|
|
|
# are ones we don't have yet so we need to look at the events that
|
|
|
|
# point to it via event_edges table.
|
2018-05-15 08:48:40 -06:00
|
|
|
txn.execute("""
|
|
|
|
SELECT COALESCE(MIN(depth), 0)
|
|
|
|
FROM event_backward_extremities AS eb
|
|
|
|
INNER JOIN event_edges AS eg ON eg.prev_event_id = eb.event_id
|
|
|
|
INNER JOIN events AS e ON e.event_id = eg.event_id
|
|
|
|
WHERE eb.room_id = ?
|
|
|
|
""", (room_id,))
|
|
|
|
min_depth, = txn.fetchone()
|
|
|
|
|
|
|
|
logger.info("[purge] updating room_depth to %d", min_depth)
|
|
|
|
|
2018-02-09 05:13:34 -07:00
|
|
|
txn.execute(
|
|
|
|
"UPDATE room_depth SET min_depth = ? WHERE room_id = ?",
|
2018-05-15 08:48:40 -06:00
|
|
|
(min_depth, room_id,)
|
2018-02-09 05:13:34 -07:00
|
|
|
)
|
|
|
|
|
2018-02-14 09:41:12 -07:00
|
|
|
# finally, drop the temp table. this will commit the txn in sqlite,
|
|
|
|
# so make sure to keep this actually last.
|
|
|
|
txn.execute(
|
|
|
|
"DROP TABLE events_to_purge"
|
|
|
|
)
|
|
|
|
|
2017-05-11 05:06:28 -06:00
|
|
|
logger.info("[purge] done")
|
2017-05-10 10:46:41 -06:00
|
|
|
|
2017-04-11 10:34:09 -06:00
|
|
|
@defer.inlineCallbacks
|
|
|
|
def is_event_after(self, event_id1, event_id2):
|
2017-04-12 07:36:20 -06:00
|
|
|
"""Returns True if event_id1 is after event_id2 in the stream
|
|
|
|
"""
|
2017-04-11 10:34:09 -06:00
|
|
|
to_1, so_1 = yield self._get_event_ordering(event_id1)
|
|
|
|
to_2, so_2 = yield self._get_event_ordering(event_id2)
|
2017-04-13 06:46:17 -06:00
|
|
|
defer.returnValue((to_1, so_1) > (to_2, so_2))
|
2017-04-11 10:34:09 -06:00
|
|
|
|
2018-03-01 08:30:57 -07:00
|
|
|
@cachedInlineCallbacks(max_entries=5000)
|
2017-04-11 10:34:09 -06:00
|
|
|
def _get_event_ordering(self, event_id):
|
|
|
|
res = yield self._simple_select_one(
|
|
|
|
table="events",
|
|
|
|
retcols=["topological_ordering", "stream_ordering"],
|
|
|
|
keyvalues={"event_id": event_id},
|
|
|
|
allow_none=True
|
|
|
|
)
|
|
|
|
|
|
|
|
if not res:
|
|
|
|
raise SynapseError(404, "Could not find event %s" % (event_id,))
|
|
|
|
|
|
|
|
defer.returnValue((int(res["topological_ordering"]), int(res["stream_ordering"])))
|
2016-03-31 03:33:02 -06:00
|
|
|
|
2017-06-15 05:47:05 -06:00
|
|
|
def get_max_current_state_delta_stream_id(self):
|
|
|
|
return self._stream_id_gen.get_current_token()
|
|
|
|
|
|
|
|
def get_all_updated_current_state_deltas(self, from_token, to_token, limit):
|
|
|
|
def get_all_updated_current_state_deltas_txn(txn):
|
|
|
|
sql = """
|
|
|
|
SELECT stream_id, room_id, type, state_key, event_id
|
|
|
|
FROM current_state_delta_stream
|
|
|
|
WHERE ? < stream_id AND stream_id <= ?
|
|
|
|
ORDER BY stream_id ASC LIMIT ?
|
|
|
|
"""
|
|
|
|
txn.execute(sql, (from_token, to_token, limit))
|
|
|
|
return txn.fetchall()
|
|
|
|
return self.runInteraction(
|
|
|
|
"get_all_updated_current_state_deltas",
|
|
|
|
get_all_updated_current_state_deltas_txn,
|
|
|
|
)
|
|
|
|
|
2017-04-11 11:35:45 -06:00
|
|
|
|
2016-03-31 03:33:02 -06:00
|
|
|
AllNewEventsResult = namedtuple("AllNewEventsResult", [
|
|
|
|
"new_forward_events", "new_backfill_events",
|
|
|
|
"forward_ex_outliers", "backward_ex_outliers",
|
|
|
|
])
|