From ec2b5d8c284451179c0f7b72c46dda0dd81f6f5f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 1 Dec 2014 16:21:17 +0000 Subject: [PATCH 001/158] Store full JSON of events in db --- synapse/storage/__init__.py | 19 +++++++++ synapse/storage/_base.py | 75 ++++++++++------------------------- synapse/storage/schema/im.sql | 10 +++++ 3 files changed, 49 insertions(+), 55 deletions(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index f15e3dfe62..205d125642 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -155,6 +155,25 @@ class DataStore(RoomMemberStore, RoomStore, if hasattr(event, "outlier"): outlier = event.outlier + event_dict = { + k: v + for k, v in event.get_full_dict().items() + if k not in [ + "redacted", + "redacted_because", + ] + } + + self._simple_insert_txn( + txn, + table="event_json", + values={ + "event_id": event.event_id, + "json": json.dumps(event_dict, separators=(',', ':')), + }, + or_replace=True, + ) + vals = { "topological_ordering": event.depth, "event_id": event.event_id, diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 4881f03368..bb61c20150 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -479,66 +479,30 @@ class SQLBaseStore(object): ) def _parse_events_txn(self, txn, rows): - events = [self._parse_event_from_row(r) for r in rows] + event_ids = [r["event_id"] for r in rows] - select_event_sql = ( - "SELECT * FROM events WHERE event_id = ? ORDER BY rowid asc" - ) - - for i, ev in enumerate(events): - signatures = self._get_event_signatures_txn( - txn, ev.event_id, + events = [] + for event_id in event_ids: + js = self._simple_select_one_onecol_txn( + txn, + table="event_json", + keyvalues={"event_id": event_id}, + retcol="json", + allow_none=True, ) - ev.signatures = { - n: { - k: encode_base64(v) for k, v in s.items() - } - for n, s in signatures.items() - } + if not js: + # FIXME (erikj): What should we actually do here? + continue - hashes = self._get_event_content_hashes_txn( - txn, ev.event_id, + d = json.loads(js) + + ev = self.event_factory.create_event( + etype=d["type"], + **d ) - ev.hashes = { - k: encode_base64(v) for k, v in hashes.items() - } - - prevs = self._get_prev_events_and_state(txn, ev.event_id) - - ev.prev_events = [ - (e_id, h) - for e_id, h, is_state in prevs - if is_state == 0 - ] - - ev.auth_events = self._get_auth_events(txn, ev.event_id) - - if hasattr(ev, "state_key"): - ev.prev_state = [ - (e_id, h) - for e_id, h, is_state in prevs - if is_state == 1 - ] - - if hasattr(ev, "replaces_state"): - # Load previous state_content. - # FIXME (erikj): Handle multiple prev_states. - cursor = txn.execute( - select_event_sql, - (ev.replaces_state,) - ) - prevs = self.cursor_to_dict(cursor) - if prevs: - prev = self._parse_event_from_row(prevs[0]) - ev.prev_content = prev.content - - if not hasattr(ev, "redacted"): - logger.debug("Doesn't have redacted key: %s", ev) - ev.redacted = self._has_been_redacted_txn(txn, ev) - - if ev.redacted: + if hasattr(ev, "redacted") and ev.redacted: # Get the redaction event. select_event_sql = "SELECT * FROM events WHERE event_id = ?" txn.execute(select_event_sql, (ev.redacted,)) @@ -549,9 +513,10 @@ class SQLBaseStore(object): if del_evs: ev = prune_event(ev) - events[i] = ev ev.redacted_because = del_evs[0] + events.append(ev) + return events def _has_been_redacted_txn(self, txn, event): diff --git a/synapse/storage/schema/im.sql b/synapse/storage/schema/im.sql index 8ba732a23b..cb0c494ddf 100644 --- a/synapse/storage/schema/im.sql +++ b/synapse/storage/schema/im.sql @@ -32,6 +32,16 @@ CREATE INDEX IF NOT EXISTS events_stream_ordering ON events (stream_ordering); CREATE INDEX IF NOT EXISTS events_topological_ordering ON events (topological_ordering); CREATE INDEX IF NOT EXISTS events_room_id ON events (room_id); + +CREATE TABLE IF NOT EXISTS event_json( + event_id TEXT NOT NULL, + json BLOB NOT NULL, + CONSTRAINT ev_j_uniq UNIQUE (event_id) +); + +CREATE INDEX IF NOT EXISTS event_json_id ON event_json(event_id); + + CREATE TABLE IF NOT EXISTS state_events( event_id TEXT NOT NULL, room_id TEXT NOT NULL, From 9d53228158bd94093eea4407b93b564ee065930b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 2 Dec 2014 10:42:28 +0000 Subject: [PATCH 002/158] Change DomainSpecificString so that it doesn't use a HomeServer object --- synapse/api/events/factory.py | 2 +- synapse/handlers/directory.py | 8 ++++---- synapse/handlers/message.py | 2 +- synapse/handlers/presence.py | 38 +++++++++++++++++------------------ synapse/handlers/profile.py | 14 ++++++------- synapse/handlers/room.py | 19 ++++++++++-------- synapse/handlers/typing.py | 4 ++-- synapse/rest/admin.py | 2 +- synapse/rest/login.py | 4 ++-- synapse/rest/presence.py | 4 ++-- synapse/server.py | 11 ++++++---- synapse/types.py | 21 ++++++------------- 12 files changed, 63 insertions(+), 66 deletions(-) diff --git a/synapse/api/events/factory.py b/synapse/api/events/factory.py index a1ec708a81..1b84e2b445 100644 --- a/synapse/api/events/factory.py +++ b/synapse/api/events/factory.py @@ -58,7 +58,7 @@ class EventFactory(object): local_part = str(int(self.clock.time())) + i + random_string(5) - e_id = EventID.create_local(local_part, self.hs) + e_id = EventID.create(local_part, self.hs.hostname) return e_id.to_string() diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index 3b37e49e6f..b95c4b8bf7 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -40,7 +40,7 @@ class DirectoryHandler(BaseHandler): # TODO(erikj): Do auth. - if not room_alias.is_mine: + if not self.hs.is_mine(room_alias): raise SynapseError(400, "Room alias must be local") # TODO(erikj): Change this. @@ -64,7 +64,7 @@ class DirectoryHandler(BaseHandler): def delete_association(self, user_id, room_alias): # TODO Check if server admin - if not room_alias.is_mine: + if not self.hs.is_mine(room_alias): raise SynapseError(400, "Room alias must be local") room_id = yield self.store.delete_room_alias(room_alias) @@ -75,7 +75,7 @@ class DirectoryHandler(BaseHandler): @defer.inlineCallbacks def get_association(self, room_alias): room_id = None - if room_alias.is_mine: + if self.hs.is_mine(room_alias): result = yield self.store.get_association_from_room_alias( room_alias ) @@ -123,7 +123,7 @@ class DirectoryHandler(BaseHandler): @defer.inlineCallbacks def on_directory_query(self, args): room_alias = self.hs.parse_roomalias(args["room_alias"]) - if not room_alias.is_mine: + if not self.hs.is_mine(room_alias): raise SynapseError( 400, "Room Alias is not hosted on this Home Server" ) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 42dc4d46f3..269d6622e1 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -79,7 +79,7 @@ class MessageHandler(BaseHandler): self.ratelimit(event.user_id) # TODO(paul): Why does 'event' not have a 'user' object? user = self.hs.parse_userid(event.user_id) - assert user.is_mine, "User must be our own: %s" % (user,) + assert self.hs.is_mine(user), "User must be our own: %s" % (user,) snapshot = yield self.store.snapshot_room(event) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index b55d589daf..179b431aa5 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -147,7 +147,7 @@ class PresenceHandler(BaseHandler): @defer.inlineCallbacks def is_presence_visible(self, observer_user, observed_user): - assert(observed_user.is_mine) + assert(self.hs.is_mine(observed_user)) if observer_user == observed_user: defer.returnValue(True) @@ -165,7 +165,7 @@ class PresenceHandler(BaseHandler): @defer.inlineCallbacks def get_state(self, target_user, auth_user, as_event=False): - if target_user.is_mine: + if self.hs.is_mine(target_user): visible = yield self.is_presence_visible( observer_user=auth_user, observed_user=target_user @@ -212,7 +212,7 @@ class PresenceHandler(BaseHandler): # TODO (erikj): Turn this back on. Why did we end up sending EDUs # everywhere? - if not target_user.is_mine: + if not self.hs.is_mine(target_user): raise SynapseError(400, "User is not hosted on this Home Server") if target_user != auth_user: @@ -291,7 +291,7 @@ class PresenceHandler(BaseHandler): @defer.inlineCallbacks def user_joined_room(self, user, room_id): - if user.is_mine: + if self.hs.is_mine(user): statuscache = self._get_or_make_usercache(user) # No actual update but we need to bump the serial anyway for the @@ -309,7 +309,7 @@ class PresenceHandler(BaseHandler): rm_handler = self.homeserver.get_handlers().room_member_handler curr_users = yield rm_handler.get_room_members(room_id) - for local_user in [c for c in curr_users if c.is_mine]: + for local_user in [c for c in curr_users if self.hs.is_mine(c)]: self.push_update_to_local_and_remote( observed_user=local_user, users_to_push=[user], @@ -318,14 +318,14 @@ class PresenceHandler(BaseHandler): @defer.inlineCallbacks def send_invite(self, observer_user, observed_user): - if not observer_user.is_mine: + if not self.hs.is_mine(observer_user): raise SynapseError(400, "User is not hosted on this Home Server") yield self.store.add_presence_list_pending( observer_user.localpart, observed_user.to_string() ) - if observed_user.is_mine: + if self.hs.is_mine(observed_user): yield self.invite_presence(observed_user, observer_user) else: yield self.federation.send_edu( @@ -339,7 +339,7 @@ class PresenceHandler(BaseHandler): @defer.inlineCallbacks def _should_accept_invite(self, observed_user, observer_user): - if not observed_user.is_mine: + if not self.hs.is_mine(observed_user): defer.returnValue(False) row = yield self.store.has_presence_state(observed_user.localpart) @@ -359,7 +359,7 @@ class PresenceHandler(BaseHandler): observed_user.localpart, observer_user.to_string() ) - if observer_user.is_mine: + if self.hs.is_mine(observer_user): if accept: yield self.accept_presence(observed_user, observer_user) else: @@ -396,7 +396,7 @@ class PresenceHandler(BaseHandler): @defer.inlineCallbacks def drop(self, observed_user, observer_user): - if not observer_user.is_mine: + if not self.hs.is_mine(observer_user): raise SynapseError(400, "User is not hosted on this Home Server") yield self.store.del_presence_list( @@ -410,7 +410,7 @@ class PresenceHandler(BaseHandler): @defer.inlineCallbacks def get_presence_list(self, observer_user, accepted=None): - if not observer_user.is_mine: + if not self.hs.is_mine(observer_user): raise SynapseError(400, "User is not hosted on this Home Server") presence = yield self.store.get_presence_list( @@ -465,7 +465,7 @@ class PresenceHandler(BaseHandler): ) for target_user in target_users: - if target_user.is_mine: + if self.hs.is_mine(target_user): self._start_polling_local(user, target_user) # We want to tell the person that just came online @@ -477,7 +477,7 @@ class PresenceHandler(BaseHandler): ) deferreds = [] - remote_users = [u for u in target_users if not u.is_mine] + remote_users = [u for u in target_users if not self.hs.is_mine(u)] remoteusers_by_domain = partition(remote_users, lambda u: u.domain) # Only poll for people in our get_presence_list for domain in remoteusers_by_domain: @@ -520,7 +520,7 @@ class PresenceHandler(BaseHandler): def stop_polling_presence(self, user, target_user=None): logger.debug("Stop polling for presence from %s", user) - if not target_user or target_user.is_mine: + if not target_user or self.hs.is_mine(target_user): self._stop_polling_local(user, target_user=target_user) deferreds = [] @@ -579,7 +579,7 @@ class PresenceHandler(BaseHandler): @defer.inlineCallbacks @log_function def push_presence(self, user, statuscache): - assert(user.is_mine) + assert(self.hs.is_mine(user)) logger.debug("Pushing presence update from %s", user) @@ -691,7 +691,7 @@ class PresenceHandler(BaseHandler): for poll in content.get("poll", []): user = self.hs.parse_userid(poll) - if not user.is_mine: + if not self.hs.is_mine(user): continue # TODO(paul) permissions checks @@ -706,7 +706,7 @@ class PresenceHandler(BaseHandler): for unpoll in content.get("unpoll", []): user = self.hs.parse_userid(unpoll) - if not user.is_mine: + if not self.hs.is_mine(user): continue if user in self._remote_sendmap: @@ -725,7 +725,7 @@ class PresenceHandler(BaseHandler): localusers, remoteusers = partitionbool( users_to_push, - lambda u: u.is_mine + lambda u: self.hs.is_mine(u) ) localusers = set(localusers) @@ -783,7 +783,7 @@ class PresenceEventSource(object): [u.to_string() for u in observer_user, observed_user])): defer.returnValue(True) - if observed_user.is_mine: + if self.hs.is_mine(observed_user): pushmap = presence._local_pushmap defer.returnValue( diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index 814b3b68fe..0116ba5358 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -51,7 +51,7 @@ class ProfileHandler(BaseHandler): @defer.inlineCallbacks def get_displayname(self, target_user): - if target_user.is_mine: + if self.hs.is_mine(target_user): displayname = yield self.store.get_profile_displayname( target_user.localpart ) @@ -81,7 +81,7 @@ class ProfileHandler(BaseHandler): def set_displayname(self, target_user, auth_user, new_displayname): """target_user is the user whose displayname is to be changed; auth_user is the user attempting to make this change.""" - if not target_user.is_mine: + if not self.hs.is_mine(target_user): raise SynapseError(400, "User is not hosted on this Home Server") if target_user != auth_user: @@ -101,7 +101,7 @@ class ProfileHandler(BaseHandler): @defer.inlineCallbacks def get_avatar_url(self, target_user): - if target_user.is_mine: + if self.hs.is_mine(target_user): avatar_url = yield self.store.get_profile_avatar_url( target_user.localpart ) @@ -130,7 +130,7 @@ class ProfileHandler(BaseHandler): def set_avatar_url(self, target_user, auth_user, new_avatar_url): """target_user is the user whose avatar_url is to be changed; auth_user is the user attempting to make this change.""" - if not target_user.is_mine: + if not self.hs.is_mine(target_user): raise SynapseError(400, "User is not hosted on this Home Server") if target_user != auth_user: @@ -150,7 +150,7 @@ class ProfileHandler(BaseHandler): @defer.inlineCallbacks def collect_presencelike_data(self, user, state): - if not user.is_mine: + if not self.hs.is_mine(user): defer.returnValue(None) with PreserveLoggingContext(): @@ -170,7 +170,7 @@ class ProfileHandler(BaseHandler): @defer.inlineCallbacks def on_profile_query(self, args): user = self.hs.parse_userid(args["user_id"]) - if not user.is_mine: + if not self.hs.is_mine(user): raise SynapseError(400, "User is not hosted on this Home Server") just_field = args.get("field", None) @@ -191,7 +191,7 @@ class ProfileHandler(BaseHandler): @defer.inlineCallbacks def _update_join_states(self, user): - if not user.is_mine: + if not self.hs.is_mine(user): return joins = yield self.store.get_rooms_for_user_where_membership_is( diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 88955160c5..315fb02e7b 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -52,9 +52,9 @@ class RoomCreationHandler(BaseHandler): self.ratelimit(user_id) if "room_alias_name" in config: - room_alias = RoomAlias.create_local( + room_alias = RoomAlias.create( config["room_alias_name"], - self.hs + self.hs.hostname, ) mapping = yield self.store.get_association_from_room_alias( room_alias @@ -77,7 +77,7 @@ class RoomCreationHandler(BaseHandler): if room_id: # Ensure room_id is the correct type room_id_obj = RoomID.from_string(room_id, self.hs) - if not room_id_obj.is_mine: + if not self.hs.is_mine(room_id_obj): raise SynapseError(400, "Room id must be local") yield self.store.store_room( @@ -93,7 +93,10 @@ class RoomCreationHandler(BaseHandler): while attempts < 5: try: random_string = stringutils.random_string(18) - gen_room_id = RoomID.create_local(random_string, self.hs) + gen_room_id = RoomID.create( + random_string, + self.hs.hostname, + ) yield self.store.store_room( room_id=gen_room_id.to_string(), room_creator_user_id=user_id, @@ -287,7 +290,7 @@ class RoomMemberHandler(BaseHandler): if ignore_user is not None and member == ignore_user: continue - if member.is_mine: + if self.hs.is_mine(member): if localusers is not None: localusers.add(member) else: @@ -457,7 +460,7 @@ class RoomMemberHandler(BaseHandler): prev_state.user_id, self.hs ) - should_do_dance = not inviter.is_mine and not room + should_do_dance = not self.hs.is_mine(inviter) and not room room_host = inviter.domain else: should_do_dance = False @@ -504,7 +507,7 @@ class RoomMemberHandler(BaseHandler): prev_state.sender, self.hs ) - is_remote_invite_join = not inviter.is_mine and not room + is_remote_invite_join = not self.hs.is_mine(inviter) and not room room_host = inviter.domain else: is_remote_invite_join = False @@ -530,7 +533,7 @@ class RoomMemberHandler(BaseHandler): # HS. target_user_id = event.state_key target_user = self.hs.parse_userid(target_user_id) - if membership == Membership.INVITE and not target_user.is_mine: + if membership == Membership.INVITE and not self.hs.is_mine(target_user): do_invite_host = target_user.domain else: do_invite_host = None diff --git a/synapse/handlers/typing.py b/synapse/handlers/typing.py index d88a53242c..be67fb2fc2 100644 --- a/synapse/handlers/typing.py +++ b/synapse/handlers/typing.py @@ -47,7 +47,7 @@ class TypingNotificationHandler(BaseHandler): @defer.inlineCallbacks def started_typing(self, target_user, auth_user, room_id, timeout): - if not target_user.is_mine: + if not self.hs.is_mine(target_user): raise SynapseError(400, "User is not hosted on this Home Server") if target_user != auth_user: @@ -72,7 +72,7 @@ class TypingNotificationHandler(BaseHandler): @defer.inlineCallbacks def stopped_typing(self, target_user, auth_user, room_id): - if not target_user.is_mine: + if not self.hs.is_mine(target_user): raise SynapseError(400, "User is not hosted on this Home Server") if target_user != auth_user: diff --git a/synapse/rest/admin.py b/synapse/rest/admin.py index ed9b484623..d74c551512 100644 --- a/synapse/rest/admin.py +++ b/synapse/rest/admin.py @@ -35,7 +35,7 @@ class WhoisRestServlet(RestServlet): if not is_admin and target_user != auth_user: raise AuthError(403, "You are not a server admin") - if not target_user.is_mine: + if not self.hs.is_mine(target_user): raise SynapseError(400, "Can only whois a local user") ret = yield self.handlers.admin_handler.get_whois(target_user) diff --git a/synapse/rest/login.py b/synapse/rest/login.py index ad71f6c61d..875da076af 100644 --- a/synapse/rest/login.py +++ b/synapse/rest/login.py @@ -47,8 +47,8 @@ class LoginRestServlet(RestServlet): @defer.inlineCallbacks def do_password_login(self, login_submission): if not login_submission["user"].startswith('@'): - login_submission["user"] = UserID.create_local( - login_submission["user"], self.hs).to_string() + login_submission["user"] = UserID.create( + login_submission["user"], self.hs.hostname).to_string() handler = self.handlers.login_handler token = yield handler.login( diff --git a/synapse/rest/presence.py b/synapse/rest/presence.py index 502ed0d4ca..062c895595 100644 --- a/synapse/rest/presence.py +++ b/synapse/rest/presence.py @@ -83,7 +83,7 @@ class PresenceListRestServlet(RestServlet): user_id = urllib.unquote(user_id) user = self.hs.parse_userid(user_id) - if not user.is_mine: + if not self.hs.is_mine(user): raise SynapseError(400, "User not hosted on this Home Server") if auth_user != user: @@ -104,7 +104,7 @@ class PresenceListRestServlet(RestServlet): user_id = urllib.unquote(user_id) user = self.hs.parse_userid(user_id) - if not user.is_mine: + if not self.hs.is_mine(user): raise SynapseError(400, "User not hosted on this Home Server") if auth_user != user: diff --git a/synapse/server.py b/synapse/server.py index da0a44433a..c3b54221d6 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -133,22 +133,22 @@ class BaseHomeServer(object): def parse_userid(self, s): """Parse the string given by 's' as a User ID and return a UserID object.""" - return UserID.from_string(s, hs=self) + return UserID.from_string(s) def parse_roomalias(self, s): """Parse the string given by 's' as a Room Alias and return a RoomAlias object.""" - return RoomAlias.from_string(s, hs=self) + return RoomAlias.from_string(s) def parse_roomid(self, s): """Parse the string given by 's' as a Room ID and return a RoomID object.""" - return RoomID.from_string(s, hs=self) + return RoomID.from_string(s) def parse_eventid(self, s): """Parse the string given by 's' as a Event ID and return a EventID object.""" - return EventID.from_string(s, hs=self) + return EventID.from_string(s) def serialize_event(self, e): return serialize_event(self, e) @@ -165,6 +165,9 @@ class BaseHomeServer(object): return ip_addr + def is_mine(self, domain_specific_string): + return domain_specific_string.domain == self.hostname + # Build magic accessors for every dependency for depname in BaseHomeServer.DEPENDENCIES: BaseHomeServer._make_dependency_method(depname) diff --git a/synapse/types.py b/synapse/types.py index 649ff2f7d7..7c533193e1 100644 --- a/synapse/types.py +++ b/synapse/types.py @@ -19,7 +19,7 @@ from collections import namedtuple class DomainSpecificString( - namedtuple("DomainSpecificString", ("localpart", "domain", "is_mine")) + namedtuple("DomainSpecificString", ("localpart", "domain")) ): """Common base class among ID/name strings that have a local part and a domain name, prefixed with a sigil. @@ -28,15 +28,13 @@ class DomainSpecificString( 'localpart' : The local part of the name (without the leading sigil) 'domain' : The domain part of the name - 'is_mine' : Boolean indicating if the domain name is recognised by the - HomeServer as being its own """ # Deny iteration because it will bite you if you try to create a singleton # set by: # users = set(user) def __iter__(self): - raise ValueError("Attempted to iterate a %s" % (type(self).__name__)) + raise ValueError("Attempted to iterate a %s" % (type(self).__name__,)) # Because this class is a namedtuple of strings and booleans, it is deeply # immutable. @@ -47,7 +45,7 @@ class DomainSpecificString( return self @classmethod - def from_string(cls, s, hs): + def from_string(cls, s): """Parse the string given by 's' into a structure object.""" if s[0] != cls.SIGIL: raise SynapseError(400, "Expected %s string to start with '%s'" % ( @@ -66,22 +64,15 @@ class DomainSpecificString( # This code will need changing if we want to support multiple domain # names on one HS - is_mine = domain == hs.hostname - return cls(localpart=parts[0], domain=domain, is_mine=is_mine) + return cls(localpart=parts[0], domain=domain) def to_string(self): """Return a string encoding the fields of the structure object.""" return "%s%s:%s" % (self.SIGIL, self.localpart, self.domain) @classmethod - def create_local(cls, localpart, hs): - """Create a structure on the local domain""" - return cls(localpart=localpart, domain=hs.hostname, is_mine=True) - - @classmethod - def create(cls, localpart, domain, hs): - is_mine = domain == hs.hostname - return cls(localpart=localpart, domain=domain, is_mine=is_mine) + def create(cls, localpart, domain,): + return cls(localpart=localpart, domain=domain) class UserID(DomainSpecificString): From c1e66800a937351614b2d19cdebbb141f786b03a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 2 Dec 2014 11:40:22 +0000 Subject: [PATCH 003/158] Begin fleshing out a new Event object --- setup.py | 1 + synapse/events/__init__.py | 120 +++++++++++++++++++++++++++++++++++++ synapse/events/builder.py | 74 +++++++++++++++++++++++ 3 files changed, 195 insertions(+) create mode 100644 synapse/events/__init__.py create mode 100644 synapse/events/builder.py diff --git a/setup.py b/setup.py index 9b38f790b9..d1b8f0680a 100755 --- a/setup.py +++ b/setup.py @@ -41,6 +41,7 @@ setup( "pynacl", "daemonize", "py-bcrypt", + "frozendict>=0.4", ], dependency_links=[ "https://github.com/matrix-org/syutil/tarball/v0.0.2#egg=syutil-0.0.2", diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py new file mode 100644 index 0000000000..eefc9d3b30 --- /dev/null +++ b/synapse/events/__init__.py @@ -0,0 +1,120 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from frozendict import frozendict + + +class _EventInternalMetadata(object): + def __init__(self, internal_metadata_dict): + self.__dict__ = internal_metadata_dict + + def get_dict(self): + return dict(self.__dict__) + + +class Event(object): + def __init__(self, event_dict, internal_metadata_dict={}): + self._signatures = event_dict.get("signatures", {}) + self._unsigned = event_dict.get("unsigned", {}) + + self._original = { + k: v + for k, v in event_dict.items() + if k not in ["signatures", "unsigned"] + } + + self._event_dict = frozendict(self._original) + + self.internal_metadata = _EventInternalMetadata( + internal_metadata_dict + ) + + @property + def auth_events(self): + return self._event_dict["auth_events"] + + @property + def content(self): + return self._event_dict["content"] + + @property + def event_id(self): + return self._event_dict["event_id"] + + @property + def hashes(self): + return self._event_dict["hashes"] + + @property + def origin(self): + return self._event_dict["origin"] + + @property + def prev_events(self): + return self._event_dict["prev_events"] + + @property + def prev_state(self): + return self._event_dict["prev_state"] + + @property + def room_id(self): + return self._event_dict["room_id"] + + @property + def signatures(self): + return self._signatures + + @property + def state_key(self): + return self._event_dict["state_key"] + + @property + def type(self): + return self._event_dict["type"] + + @property + def unsigned(self): + return self._unsigned + + @property + def user_id(self): + return self._event_dict["sender"] + + @property + def sender(self): + return self._event_dict["sender"] + + def get_dict(self): + d = dict(self._original) + d.update({ + "signatures": self._signatures, + "unsigned": self._unsigned, + }) + + return d + + def get_internal_metadata_dict(self): + return self.internal_metadata.get_dict() + + def get_pdu_json(self, time_now=None): + pdu_json = self.get_dict() + + if time_now is not None and "age_ts" in pdu_json["unsigned"]: + age = time_now - pdu_json["unsigned"]["age_ts"] + pdu_json.setdefault("unsigned", {})["age"] = int(age) + del pdu_json["unsigned"]["age_ts"] + + return pdu_json \ No newline at end of file diff --git a/synapse/events/builder.py b/synapse/events/builder.py new file mode 100644 index 0000000000..d741795bc5 --- /dev/null +++ b/synapse/events/builder.py @@ -0,0 +1,74 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from . import Event + +from synapse.types import EventID + +from synapse.util.stringutils import random_string + + +class EventBuilder(object): + def __init__(self, key_values={}): + self._event_dict = dict(key_values) + self._metadata = {} + + def update_event_key(self, key, value): + self._event_dict[key] = value + + def update_event_keys(self, other_dict): + self._event_dict.update(other_dict) + + def update_internal_key(self, key, value): + self._metadata[key] = value + + def build(self): + return Event( + self._event_dict, + self._metadata, + ) + + +class EventBuilderFactory(object): + def __init__(self, clock, hostname): + self.clock = clock + self.hostname = hostname + + self.event_id_count = 0 + + def create_event_id(self): + i = str(self.event_id_count) + self.event_id_count += 1 + + local_part = str(int(self.clock.time())) + i + random_string(5) + + e_id = EventID.create(local_part, self.hostname) + + return e_id.to_string() + + def new(self, key_values={}): + if "event_id" not in key_values: + key_values["event_id"] = self.create_event_id() + + time_now = self.clock.time_msec() + + key_values.setdefault("origin", self.hostname) + key_values.setdefault("origin_server_ts", time_now) + + if "unsigned" in key_values: + age = key_values["unsigned"].pop("age", 0) + key_values["unsigned"].setdefault("age_ts", time_now - age) + + return EventBuilder(key_values=key_values,) \ No newline at end of file From 279c48c8b442ec726fb5088e56ce9c1d2ed4bfb5 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 2 Dec 2014 15:09:51 +0000 Subject: [PATCH 004/158] Write the upload portion of version 1 of the media repository --- docs/media_repository.rst | 20 +++ synapse/http/server.py | 21 ++-- .../{http => media/v0}/content_repository.py | 0 synapse/media/v1/filepath.py | 53 ++++++++ synapse/media/v1/media_repository.py | 72 +++++++++++ synapse/media/v1/upload_resource.py | 110 +++++++++++++++++ synapse/storage/media_repository.py | 116 ++++++++++++++++++ synapse/storage/schema/media_repository.sql | 66 ++++++++++ 8 files changed, 451 insertions(+), 7 deletions(-) create mode 100644 docs/media_repository.rst rename synapse/{http => media/v0}/content_repository.py (100%) create mode 100644 synapse/media/v1/filepath.py create mode 100644 synapse/media/v1/media_repository.py create mode 100644 synapse/media/v1/upload_resource.py create mode 100644 synapse/storage/media_repository.py create mode 100644 synapse/storage/schema/media_repository.sql diff --git a/docs/media_repository.rst b/docs/media_repository.rst new file mode 100644 index 0000000000..e554d0f495 --- /dev/null +++ b/docs/media_repository.rst @@ -0,0 +1,20 @@ +Media Repository +================ + +The media repository is where attachments and avatar photos are stored. +It stores attachment content and thumbnails for media uploaded by local users. +It caches attachment content and thumbnails for media uploaded by remote users. + +Storage +------- + +Each item of media is assigned a ``media_id`` when it is uploaded. +The ``media_id`` is a randomly chosen, URL safe 24 character string. +Metadata such as the MIME type, upload time and length are stored in the +sqlite3 database indexed by ``media_id``. +Content is stored on the filesystem under a "content" directory. Thumbnails are +stored under a "thumbnails" directory. +The item with ``media_id`` ``"aabbccccccccdddddddddddd"`` is stored under +``"local/content/aa/bb/ccccccccdddddddddddd"``. Its thumbnail with width +``128`` and height ``96`` and type ``"image/jpeg"`` is stored under +``"local/thumbnails/aa/bb/ccccccccdddddddddddd/128-96-image-jpeg"`` diff --git a/synapse/http/server.py b/synapse/http/server.py index 8024ff5bde..046e230361 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -166,14 +166,10 @@ class JsonResource(HttpServer, resource.Resource): request) return - if not self._request_user_agent_is_curl(request): - json_bytes = encode_canonical_json(response_json_object) - else: - json_bytes = encode_pretty_printed_json(response_json_object) - # TODO: Only enable CORS for the requests that need it. - respond_with_json_bytes(request, code, json_bytes, send_cors=True, - response_code_message=response_code_message) + respond_with_json(request, code, response_json_object, send_cors=True, + response_code_message=response_code_message, + pretty_print=self._request_user_agent_is_curl) @staticmethod def _request_user_agent_is_curl(request): @@ -202,6 +198,17 @@ class RootRedirect(resource.Resource): return resource.Resource.getChild(self, name, request) +def respond_with_json(request, code, json_object, send_cors=False, + response_code_message=None, pretty_print=False): + if not pretty_print: + json_bytes = encode_pretty_printed_json(response_json_object) + else: + json_bytes = encode_canonical_json(response_json_object) + + return respond_with_json_bytes(request, code, json_bytes, send_cors, + response_code_message=response_code_message) + + def respond_with_json_bytes(request, code, json_bytes, send_cors=False, response_code_message=None): """Sends encoded JSON in response to the given request. diff --git a/synapse/http/content_repository.py b/synapse/media/v0/content_repository.py similarity index 100% rename from synapse/http/content_repository.py rename to synapse/media/v0/content_repository.py diff --git a/synapse/media/v1/filepath.py b/synapse/media/v1/filepath.py new file mode 100644 index 0000000000..d23564e03e --- /dev/null +++ b/synapse/media/v1/filepath.py @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + + +class MediaFilePaths(object): + + def __init__(self, base_path): + self.base_path = base_path + + def local_media_filepath(self, media_id): + return os.path.join( + self.base_path, "local", "content", + media_id[0:2], media_id[2:4], media_id[4:] + ) + + def local_media_thumbnail(self, media_id, width, height, content_type): + top_level_type, sub_type = content_type.split("/") + file_name = "%i-%i-%s-%s" % (width, height, top_level_type, sub_type) + return os.path.join( + self.base_path, "local", "thumbnails", + media_id[0:2], media_id[2:4], media_id[4:], + file_name + ) + + def remote_media_filepath(self, server_name, file_id): + return os.path.join( + self.base_path, "remote", "content", server_name, + file_id[0:2], file_id[2:4], file_id[4:] + ) + + def remote_media_thumbnail(self, server_name, file_id, width, height, + content_type): + top_level_type, sub_type = content_type.split("/") + file_name = "%i-%i-%s-%s" % (width, height, top_level_type, sub_type) + return os.path.join( + self.base_path, "remote", "content", server_name, + file_id[0:2], file_id[2:4], file_id[4:], + file_name + ) diff --git a/synapse/media/v1/media_repository.py b/synapse/media/v1/media_repository.py new file mode 100644 index 0000000000..9c36a8e933 --- /dev/null +++ b/synapse/media/v1/media_repository.py @@ -0,0 +1,72 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from synapse.http.server import respond_with_json_bytes + +from synapse.util.stringutils import random_string +from synapse.api.errors import ( + cs_exception, SynapseError, CodeMessageException, Codes, cs_error +) + +from twisted.protocols.basic import FileSender +from twisted.web import server, resource +from twisted.internet import defer + +import base64 +import json +import logging +import os +import re + +logger = logging.getLogger(__name__) + + +class MediaRepository(): + """Profiles file uploading and downloading. + + Uploads are POSTed to a resource which returns a token which is used to GET + the download:: + + => POST /_matrix/media/v1/upload HTTP/1.1 + Content-Type: + + + + <= HTTP/1.1 200 OK + Content-Type: application/json + + { "token": } + + => GET /_matrix/media/v1/download/ HTTP/1.1 + + <= HTTP/1.1 200 OK + Content-Type: + Content-Disposition: attachment;filename= + + + + Clients can get thumbnails by supplying a desired width and height:: + + => GET /_matrix/media/v1/thumbnail/?width=&height= HTTP/1.1 + + <= HTTP/1.1 200 OK + Content-Type: image/jpeg or image/png + + + """ + + def __init__(self, hs): + filepaths = MediaFilePaths + diff --git a/synapse/media/v1/upload_resource.py b/synapse/media/v1/upload_resource.py new file mode 100644 index 0000000000..3721a0173d --- /dev/null +++ b/synapse/media/v1/upload_resource.py @@ -0,0 +1,110 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from synapse.http.server import respond_with_json + +from synapse.util.stringutils import random_string +from synapse.api.errors import ( + cs_exception, SynapseError, CodeMessageException +) + +from twisted.web import server, resource +from twisted.internet import defer + +import logging + +logger = logging.getLogger(__name__) + +class UploadResource(resource.Resource): + + def __init__(self, hs, filepaths): + self.auth = hs.get_auth() + self.store = hs.get_datastore() + self.max_upload_size = hs.config.max_upload_size() + self.filepaths = filepaths + + def render_POST(self, request): + self._async_render_POST(request) + return server.NOT_DONE_YET + + def render_OPTIONS(self, request): + respond_with_json(request, 200, {}, send_cors=True) + return server.NOT_DONE_YET + + @defer.inlineCallbacks + def _async_render_POST(self, request): + + auth_user = yield self.auth.get_user_by_req(request) + + try: + # TODO: The checks here are a bit late. The content will have + # already been uploaded to a tmp file at this point + content_length = request.getHeader("Content-Length") + if content_length is None: + raise SynapseError( + msg="Request must specify a Content-Length", code=400 + ) + if int(content_length) > self.max_upload_size: + raise SynapseError( + msg="Upload request body is too large", + code=413, + ) + + headers = request.requestHeaders() + + if headers.hasHeader("Content-Type"): + media_type = headers.getRawHeaders("Content-Type")[0] + else: + raise SynapseError( + msg="Upload request missing 'Content-Type'", + code=400, + ) + + #if headers.hasHeader("Content-Disposition"): + # disposition = headers.getRawHeaders("Content-Disposition")[0] + # TODO(markjh): parse content-dispostion + + media_id = random_string(24) + + fname = self.filepaths.local_media_file_path(media_id) + + # This shouldn't block for very long because the content will have + # already been uploaded at this point. + with open(fname, "wb") as f: + f.write(request.content.read()) + + yield self.store.store_local_media( + media_id=media_id, + media_type=media_type, + time_now_ms=self.clock.time_msec(), + upload_name=None, + media_length=content_length, + user_id=auth_user, + ) + + respond_with_json( + request, 200, {"content_token": media_id}, send_cors=True + ) + except CodeMessageException as e: + logger.exception(e) + respond_with_json(request, e.code, cs_exception(e), send_cors=True) + except: + logger.exception("Failed to store file") + respond_with_json( + request, + 500, + {"error": "Internal server error"}, + send_cors=True + ) diff --git a/synapse/storage/media_repository.py b/synapse/storage/media_repository.py new file mode 100644 index 0000000000..73ceba3f2c --- /dev/null +++ b/synapse/storage/media_repository.py @@ -0,0 +1,116 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from _base import SQLBaseStore + + +class MediaRepositoryStore(SQLBaseStore): + """Persistence for attachments and avatars""" + + def get_local_media(self, media_id): + return self._simple_select_one( + "local_media_repository", + {"media_id": media_id}, + ("media_type", "media_length", "upload_name", "created_ts"), + ) + + def store_local_media(self, media_id, media_type, time_now_ms, upload_name, + media_length, user_id): + return self._simple_insert( + "local_media_repository", + { + "media_id": media_id, + "media_type": media_type, + "created_ts": time_now_ms, + "upload_name": upload_name, + "media_length": media_length, + "user_id": user_id, + } + ) + + def get_local_media_thumbnails(self, media_id): + return self._simple_select_list( + "local_media_thumbnails", + {"media_id": media_id}, + ( + "thumbnail_width", "thumbnail_height", + "thumbnail_type", "thumbnail_length", + ) + ) + + def store_local_thumbnail(self, media_id, thumbnail_width, + thumbnail_height, thumbnail_type, + thumbnail_length): + return self._simple_insert( + "local_media_thumbnails", + { + "media_id": media_id, + "thumbnail_width": thumbnail_width, + "thumbnail_height": thumbnail_height, + "thumbnail_type": thumbnail_type, + "thumbnail_length": thumbnail_length, + } + ) + + def get_cached_remote_media(self, origin, media_id): + return self._simple_select_one( + "remote_media_cache", + {"media_origin": origin, "media_id": media_id}, + ("media_type", "media_length", "upload_name", "created_ts"), + ) + + def store_cached_remote_media(self, origin, media_id, media_type, + media_length, time_now_ms, upload_name, + filesytem_id): + return self._simple_insert( + "remote_media_cache", + { + "media_origin": origin, + "media_id": media_id, + "media_type": media_type, + "media_length": media_length, + "created_ts": time_now_ms, + "upload_name": upload_name, + "filesystem_id": filesystem_id, + } + ) + + def get_remote_media_thumbnails(self, origin, media_id): + return self._simple_select_list( + "remote_media_cache_thumbnails", + {"origin": origin, "media_id": media_id}, + ( + "thumbnail_width", "thumbnail_height", + "thumbnail_type", "thumbnail_length", + "filesystem_id" + ) + ) + + + def store_remote_media_thumbnail(self, origin, media_id, thumbnail_width, + thumbnail_height, thumbnail_type, + thumbnail_length, filesystem_id): + return self._simple_insert( + "remote_media_cache_thumbnails", + { + "media_origin": origin, + "media_id": media_id, + "thumbnail_width": thumbnail_width, + "thumbnail_height": thumbnail_height, + "thumbnail_type": thumbnail_type, + "thumbnail_length": thumbnail_length, + "filesystem_id": filesystem_id, + } + ) diff --git a/synapse/storage/schema/media_repository.sql b/synapse/storage/schema/media_repository.sql new file mode 100644 index 0000000000..768752296f --- /dev/null +++ b/synapse/storage/schema/media_repository.sql @@ -0,0 +1,66 @@ +/* Copyright 2014 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS local_media_repository ( + media_id TEXT, -- The id used to refer to the media. + media_type TEXT, -- The MIME-type of the media. + media_length INTEGER, -- Length of the media in bytes. + created_ts INTEGER, -- When the content was uploaded in ms. + upload_name TEXT, -- The name the media was uploaded with. + user_id TEXT, -- The user who uploaded the file. + CONSTRAINT uniqueness UNIQUE (media_id) +); + +CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails ( + media_id TEXT, -- The id used to refer to the media. + thumbnail_width INTEGER, -- The width of the thumbnail in pixels. + thumbnail_height INTEGER, -- The height of the thumbnail in pixels. + thumbnail_type TEXT, -- The MIME-type of the thumbnail. + thumbnail_length INTEGER, -- The length of the thumbnail in bytes. + CONSTRAINT uniqueness UNIQUE ( + media_id, thumbnail_width, thumbnail_height, thumbnail_type + ) +); + +CREATE INDEX IF NOT EXISTS local_media_repository_thumbnails_media_id + ON local_media_repository_thumbnails (media_id); + +CREATE TABLE IF NOT EXISTS remote_media_cache ( + media_origin TEXT, -- The remote HS the media came from. + media_id TEXT, -- The id used to refer to the media on that server. + media_type TEXT, -- The MIME-type of the media. + created_ts INTEGER, -- When the content was uploaded in ms. + upload_name TEXT, -- The name the media was uploaded with. + media_length INTEGER, -- Length of the media in bytes. + filesystem_id TEXT, -- The name used to store the media on disk. + CONSTRAINT uniqueness UNIQUE (media_origin, media_id) +); + +CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails ( + media_origin TEXT, -- The remote HS the media came from. + media_id TEXT, -- The id used to refer to the media. + thumbnail_width INTEGER, -- The width of the thumbnail in pixels. + thumbnail_height INTEGER, -- The height of the thumbnail in pixels. + thumbnail_type TEXT, -- The MIME-type of the thumbnail. + thumbnail_length INTEGER, -- The length of the thumbnail in bytes. + filesystem_id TEXT, -- The name used to store the media on disk. + CONSTRAINT uniqueness UNIQUE ( + media_origin, media_id, thumbnail_width, thumbnail_height, + thumbnail_type + ) +); + +CREATE INDEX IF NOT EXISTS remote_media_cache_thumbnails_media_id + ON local_media_repository_thumbnails (media_id); From 5da65085d106e98cf7b762836cb300d01226bf92 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 2 Dec 2014 19:51:47 +0000 Subject: [PATCH 005/158] Get uploads working with new media repo --- synapse/api/urls.py | 1 + synapse/app/homeserver.py | 9 +++++++-- synapse/config/_base.py | 14 ++++++++++++++ synapse/config/repository.py | 4 ++++ synapse/http/server.py | 4 ++-- synapse/media/__init__.py | 0 synapse/media/v0/__init__.py | 0 synapse/media/v0/content_repository.py | 2 +- synapse/media/v1/__init__.py | 0 synapse/media/v1/media_repository.py | 23 +++++++---------------- synapse/media/v1/upload_resource.py | 14 ++++++++------ synapse/server.py | 1 + synapse/storage/__init__.py | 6 +++++- synapse/storage/media_repository.py | 7 ++++++- 14 files changed, 56 insertions(+), 29 deletions(-) create mode 100644 synapse/media/__init__.py create mode 100644 synapse/media/v0/__init__.py create mode 100644 synapse/media/v1/__init__.py diff --git a/synapse/api/urls.py b/synapse/api/urls.py index 6dc19305b7..d7625127f8 100644 --- a/synapse/api/urls.py +++ b/synapse/api/urls.py @@ -20,3 +20,4 @@ FEDERATION_PREFIX = "/_matrix/federation/v1" WEB_CLIENT_PREFIX = "/_matrix/client" CONTENT_REPO_PREFIX = "/_matrix/content" SERVER_KEY_PREFIX = "/_matrix/key/v1" +MEDIA_PREFIX = "/_matrix/media/v1" diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 855fe8e170..a6e29c0860 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -24,12 +24,13 @@ from twisted.web.resource import Resource from twisted.web.static import File from twisted.web.server import Site from synapse.http.server import JsonResource, RootRedirect -from synapse.http.content_repository import ContentRepoResource +from synapse.media.v0.content_repository import ContentRepoResource +from synapse.media.v1.media_repository import MediaRepositoryResource from synapse.http.server_key_resource import LocalKey from synapse.http.matrixfederationclient import MatrixFederationHttpClient from synapse.api.urls import ( CLIENT_PREFIX, FEDERATION_PREFIX, WEB_CLIENT_PREFIX, CONTENT_REPO_PREFIX, - SERVER_KEY_PREFIX, + SERVER_KEY_PREFIX, MEDIA_PREFIX ) from synapse.config.homeserver import HomeServerConfig from synapse.crypto import context_factory @@ -69,6 +70,9 @@ class SynapseHomeServer(HomeServer): self, self.upload_dir, self.auth, self.content_addr ) + def build_resource_for_media_repository(self): + return MediaRepositoryResource(self) + def build_resource_for_server_key(self): return LocalKey(self) @@ -99,6 +103,7 @@ class SynapseHomeServer(HomeServer): (FEDERATION_PREFIX, self.get_resource_for_federation()), (CONTENT_REPO_PREFIX, self.get_resource_for_content_repo()), (SERVER_KEY_PREFIX, self.get_resource_for_server_key()), + (MEDIA_PREFIX, self.get_resource_for_media_repository()), ] if web_client: logger.info("Adding the web client.") diff --git a/synapse/config/_base.py b/synapse/config/_base.py index 6870af10e8..1426436dcb 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -50,12 +50,26 @@ class Config(object): ) return cls.abspath(file_path) + @staticmethod + def ensure_directory(dir_path): + if not os.path.exists(dir_path): + os.makedirs(dir_path) + if not os.path.isdir(dir_path): + raise ConfigError( + "%s is not a directory" % (dir_path,) + ) + return dir_path + @classmethod def read_file(cls, file_path, config_name): cls.check_file(file_path, config_name) with open(file_path) as file_stream: return file_stream.read() + @staticmethod + def default_path(name): + return os.path.abspath(os.path.join(os.path.curdir, name)) + @staticmethod def read_config_file(file_path): with open(file_path) as file_stream: diff --git a/synapse/config/repository.py b/synapse/config/repository.py index 743bc26474..6eec930a03 100644 --- a/synapse/config/repository.py +++ b/synapse/config/repository.py @@ -20,6 +20,7 @@ class ContentRepositoryConfig(Config): def __init__(self, args): super(ContentRepositoryConfig, self).__init__(args) self.max_upload_size = self.parse_size(args.max_upload_size) + self.media_store_path = self.ensure_directory(args.media_store_path) def parse_size(self, string): sizes = {"K": 1024, "M": 1024 * 1024} @@ -37,3 +38,6 @@ class ContentRepositoryConfig(Config): db_group.add_argument( "--max-upload-size", default="1M" ) + db_group.add_argument( + "--media-store-path", default=cls.default_path("media_store") + ) diff --git a/synapse/http/server.py b/synapse/http/server.py index 046e230361..02277c4998 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -201,9 +201,9 @@ class RootRedirect(resource.Resource): def respond_with_json(request, code, json_object, send_cors=False, response_code_message=None, pretty_print=False): if not pretty_print: - json_bytes = encode_pretty_printed_json(response_json_object) + json_bytes = encode_pretty_printed_json(json_object) else: - json_bytes = encode_canonical_json(response_json_object) + json_bytes = encode_canonical_json(json_object) return respond_with_json_bytes(request, code, json_bytes, send_cors, response_code_message=response_code_message) diff --git a/synapse/media/__init__.py b/synapse/media/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/synapse/media/v0/__init__.py b/synapse/media/v0/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/synapse/media/v0/content_repository.py b/synapse/media/v0/content_repository.py index 64ecb5346e..ce5d3d153e 100644 --- a/synapse/media/v0/content_repository.py +++ b/synapse/media/v0/content_repository.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from .server import respond_with_json_bytes +from synapse.http.server import respond_with_json_bytes from synapse.util.stringutils import random_string from synapse.api.errors import ( diff --git a/synapse/media/v1/__init__.py b/synapse/media/v1/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/synapse/media/v1/media_repository.py b/synapse/media/v1/media_repository.py index 9c36a8e933..0f4eeef278 100644 --- a/synapse/media/v1/media_repository.py +++ b/synapse/media/v1/media_repository.py @@ -13,27 +13,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -from synapse.http.server import respond_with_json_bytes +from .upload_resource import UploadResource +from .filepath import MediaFilePaths -from synapse.util.stringutils import random_string -from synapse.api.errors import ( - cs_exception, SynapseError, CodeMessageException, Codes, cs_error -) +from twisted.web.resource import Resource -from twisted.protocols.basic import FileSender -from twisted.web import server, resource -from twisted.internet import defer - -import base64 -import json import logging -import os -import re logger = logging.getLogger(__name__) -class MediaRepository(): +class MediaRepositoryResource(Resource): """Profiles file uploading and downloading. Uploads are POSTed to a resource which returns a token which is used to GET @@ -68,5 +58,6 @@ class MediaRepository(): """ def __init__(self, hs): - filepaths = MediaFilePaths - + Resource.__init__(self) + filepaths = MediaFilePaths(hs.config.media_store_path) + self.putChild("upload", UploadResource(hs, filepaths)) diff --git a/synapse/media/v1/upload_resource.py b/synapse/media/v1/upload_resource.py index 3721a0173d..d9d7825b2b 100644 --- a/synapse/media/v1/upload_resource.py +++ b/synapse/media/v1/upload_resource.py @@ -23,6 +23,8 @@ from synapse.api.errors import ( from twisted.web import server, resource from twisted.internet import defer +import os + import logging logger = logging.getLogger(__name__) @@ -31,8 +33,9 @@ class UploadResource(resource.Resource): def __init__(self, hs, filepaths): self.auth = hs.get_auth() + self.clock = hs.get_clock() self.store = hs.get_datastore() - self.max_upload_size = hs.config.max_upload_size() + self.max_upload_size = hs.config.max_upload_size self.filepaths = filepaths def render_POST(self, request): @@ -45,10 +48,8 @@ class UploadResource(resource.Resource): @defer.inlineCallbacks def _async_render_POST(self, request): - - auth_user = yield self.auth.get_user_by_req(request) - try: + auth_user = yield self.auth.get_user_by_req(request) # TODO: The checks here are a bit late. The content will have # already been uploaded to a tmp file at this point content_length = request.getHeader("Content-Length") @@ -62,7 +63,7 @@ class UploadResource(resource.Resource): code=413, ) - headers = request.requestHeaders() + headers = request.requestHeaders if headers.hasHeader("Content-Type"): media_type = headers.getRawHeaders("Content-Type")[0] @@ -78,7 +79,8 @@ class UploadResource(resource.Resource): media_id = random_string(24) - fname = self.filepaths.local_media_file_path(media_id) + fname = self.filepaths.local_media_filepath(media_id) + os.makedirs(os.path.dirname(fname)) # This shouldn't block for very long because the content will have # already been uploaded at this point. diff --git a/synapse/server.py b/synapse/server.py index da0a44433a..7eb15270fc 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -78,6 +78,7 @@ class BaseHomeServer(object): 'resource_for_web_client', 'resource_for_content_repo', 'resource_for_server_key', + 'resource_for_media_repository', 'event_sources', 'ratelimiter', 'keyring', diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 1231794de0..f6811a8117 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -33,6 +33,7 @@ from .stream import StreamStore from .transactions import TransactionStore from .keys import KeyStore from .event_federation import EventFederationStore +from .media_repository import MediaRepositoryStore from .state import StateStore from .signatures import SignatureStore @@ -62,6 +63,7 @@ SCHEMAS = [ "state", "event_edges", "event_signatures", + "media_repository", ] @@ -81,7 +83,9 @@ class DataStore(RoomMemberStore, RoomStore, RegistrationStore, StreamStore, ProfileStore, FeedbackStore, PresenceStore, TransactionStore, DirectoryStore, KeyStore, StateStore, SignatureStore, - EventFederationStore, ): + EventFederationStore, + MediaRepositoryStore, + ): def __init__(self, hs): super(DataStore, self).__init__(hs) diff --git a/synapse/storage/media_repository.py b/synapse/storage/media_repository.py index 73ceba3f2c..db03619a80 100644 --- a/synapse/storage/media_repository.py +++ b/synapse/storage/media_repository.py @@ -20,10 +20,15 @@ class MediaRepositoryStore(SQLBaseStore): """Persistence for attachments and avatars""" def get_local_media(self, media_id): + """Get the metadata for a local piece of media + Returns: + None if the media_id doesn't exist. + """ return self._simple_select_one( "local_media_repository", {"media_id": media_id}, ("media_type", "media_length", "upload_name", "created_ts"), + True, ) def store_local_media(self, media_id, media_type, time_now_ms, upload_name, @@ -36,7 +41,7 @@ class MediaRepositoryStore(SQLBaseStore): "created_ts": time_now_ms, "upload_name": upload_name, "media_length": media_length, - "user_id": user_id, + "user_id": user_id.to_string(), } ) From 2f804a70723a911bbc1d1bafbeb33f8462980151 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 2 Dec 2014 19:55:18 +0000 Subject: [PATCH 006/158] Fix pyflakes and pep8 warnings --- synapse/media/v1/media_repository.py | 3 ++- synapse/media/v1/upload_resource.py | 1 + synapse/storage/media_repository.py | 3 +-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/synapse/media/v1/media_repository.py b/synapse/media/v1/media_repository.py index 0f4eeef278..afd92874cf 100644 --- a/synapse/media/v1/media_repository.py +++ b/synapse/media/v1/media_repository.py @@ -49,7 +49,8 @@ class MediaRepositoryResource(Resource): Clients can get thumbnails by supplying a desired width and height:: - => GET /_matrix/media/v1/thumbnail/?width=&height= HTTP/1.1 + => GET /_matrix/media/v1 + /thumbnail/?width=&height= HTTP/1.1 <= HTTP/1.1 200 OK Content-Type: image/jpeg or image/png diff --git a/synapse/media/v1/upload_resource.py b/synapse/media/v1/upload_resource.py index d9d7825b2b..2919fee12f 100644 --- a/synapse/media/v1/upload_resource.py +++ b/synapse/media/v1/upload_resource.py @@ -29,6 +29,7 @@ import logging logger = logging.getLogger(__name__) + class UploadResource(resource.Resource): def __init__(self, hs, filepaths): diff --git a/synapse/storage/media_repository.py b/synapse/storage/media_repository.py index db03619a80..eda191ad5b 100644 --- a/synapse/storage/media_repository.py +++ b/synapse/storage/media_repository.py @@ -78,7 +78,7 @@ class MediaRepositoryStore(SQLBaseStore): def store_cached_remote_media(self, origin, media_id, media_type, media_length, time_now_ms, upload_name, - filesytem_id): + filesystem_id): return self._simple_insert( "remote_media_cache", { @@ -103,7 +103,6 @@ class MediaRepositoryStore(SQLBaseStore): ) ) - def store_remote_media_thumbnail(self, origin, media_id, thumbnail_width, thumbnail_height, thumbnail_type, thumbnail_length, filesystem_id): From 75b4329aaad400abcee4b23e9c88ff845e0d73c7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 3 Dec 2014 16:07:21 +0000 Subject: [PATCH 007/158] WIP for new way of managing events. --- synapse/api/auth.py | 13 +-- synapse/api/constants.py | 9 ++ synapse/crypto/event_signing.py | 39 +++++---- synapse/events/__init__.py | 145 +++++++++++++++++--------------- synapse/events/builder.py | 20 ++--- synapse/events/snapshot.py | 63 ++++++++++++++ synapse/events/utils.py | 82 ++++++++++++++++++ synapse/events/validator.py | 58 +++++++++++++ synapse/handlers/_base.py | 51 +++++++++++ 9 files changed, 376 insertions(+), 104 deletions(-) create mode 100644 synapse/events/snapshot.py create mode 100644 synapse/events/utils.py create mode 100644 synapse/events/validator.py diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 2b0475543d..50d4be113f 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -351,7 +351,7 @@ class Auth(object): return self.store.is_server_admin(user) @defer.inlineCallbacks - def add_auth_events(self, event): + def get_auth_events(self, event, current_state): if event.type == RoomCreateEvent.TYPE: event.auth_events = [] return @@ -359,19 +359,19 @@ class Auth(object): auth_events = [] key = (RoomPowerLevelsEvent.TYPE, "", ) - power_level_event = event.old_state_events.get(key) + power_level_event = current_state.get(key) if power_level_event: auth_events.append(power_level_event.event_id) key = (RoomJoinRulesEvent.TYPE, "", ) - join_rule_event = event.old_state_events.get(key) + join_rule_event = current_state.get(key) key = (RoomMemberEvent.TYPE, event.user_id, ) - member_event = event.old_state_events.get(key) + member_event = current_state.get(key) key = (RoomCreateEvent.TYPE, "", ) - create_event = event.old_state_events.get(key) + create_event = current_state.get(key) if create_event: auth_events.append(create_event.event_id) @@ -403,7 +403,8 @@ class Auth(object): } for h in hashes ] - event.auth_events = zip(auth_events, hashes) + + defer.returnValue(zip(auth_events, hashes)) @log_function def _can_send_event(self, event, auth_events): diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 3cafff0e32..acf50e42ab 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -59,3 +59,12 @@ class LoginType(object): EMAIL_URL = u"m.login.email.url" EMAIL_IDENTITY = u"m.login.email.identity" RECAPTCHA = u"m.login.recaptcha" + + +class EventTypes(object): + Member = "m.room.member" + Create = "m.room.create" + JoinRules = "m.room.join_rules" + PowerLevels = "m.room.power_levels" + Aliases = "m.room.aliases" + Redaction = "m.room.redaction" \ No newline at end of file diff --git a/synapse/crypto/event_signing.py b/synapse/crypto/event_signing.py index a9d8953239..209f9d73fe 100644 --- a/synapse/crypto/event_signing.py +++ b/synapse/crypto/event_signing.py @@ -29,17 +29,17 @@ logger = logging.getLogger(__name__) def check_event_content_hash(event, hash_algorithm=hashlib.sha256): """Check whether the hash for this PDU matches the contents""" - computed_hash = _compute_content_hash(event, hash_algorithm) - logger.debug("Expecting hash: %s", encode_base64(computed_hash.digest())) - if computed_hash.name not in event.hashes: + name, expected_hash = compute_content_hash(event, hash_algorithm) + logger.debug("Expecting hash: %s", encode_base64(expected_hash)) + if name not in event.hashes: raise SynapseError( 400, "Algorithm %s not in hashes %s" % ( - computed_hash.name, list(event.hashes), + name, list(event.hashes), ), Codes.UNAUTHORIZED, ) - message_hash_base64 = event.hashes[computed_hash.name] + message_hash_base64 = event.hashes[name.name] try: message_hash_bytes = decode_base64(message_hash_base64) except: @@ -48,10 +48,10 @@ def check_event_content_hash(event, hash_algorithm=hashlib.sha256): "Invalid base64: %s" % (message_hash_base64,), Codes.UNAUTHORIZED, ) - return message_hash_bytes == computed_hash.digest() + return message_hash_bytes == expected_hash -def _compute_content_hash(event, hash_algorithm): +def compute_content_hash(event, hash_algorithm): event_json = event.get_pdu_json() event_json.pop("age_ts", None) event_json.pop("unsigned", None) @@ -59,8 +59,11 @@ def _compute_content_hash(event, hash_algorithm): event_json.pop("hashes", None) event_json.pop("outlier", None) event_json.pop("destinations", None) + event_json_bytes = encode_canonical_json(event_json) - return hash_algorithm(event_json_bytes) + + hashed = hash_algorithm(event_json_bytes) + return (hashed.name, hashed.digest()) def compute_event_reference_hash(event, hash_algorithm=hashlib.sha256): @@ -86,20 +89,20 @@ def compute_event_signature(event, signature_name, signing_key): def add_hashes_and_signatures(event, signature_name, signing_key, hash_algorithm=hashlib.sha256): - if hasattr(event, "old_state_events"): - state_json_bytes = encode_canonical_json( - [e.event_id for e in event.old_state_events.values()] - ) - hashed = hash_algorithm(state_json_bytes) - event.state_hash = { - hashed.name: encode_base64(hashed.digest()) - } + # if hasattr(event, "old_state_events"): + # state_json_bytes = encode_canonical_json( + # [e.event_id for e in event.old_state_events.values()] + # ) + # hashed = hash_algorithm(state_json_bytes) + # event.state_hash = { + # hashed.name: encode_base64(hashed.digest()) + # } - hashed = _compute_content_hash(event, hash_algorithm=hash_algorithm) + name, digest = compute_content_hash(event, hash_algorithm=hash_algorithm) if not hasattr(event, "hashes"): event.hashes = {} - event.hashes[hashed.name] = encode_base64(hashed.digest()) + event.hashes[name] = encode_base64(digest) event.signatures = compute_event_signature( event, diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index eefc9d3b30..6748198917 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -16,6 +16,21 @@ from frozendict import frozendict +def _freeze(o): + if isinstance(o, dict): + return frozendict({k: _freeze(v) for k,v in o.items()}) + + if isinstance(o, basestring): + return o + + try: + return tuple([_freeze(i) for i in o]) + except TypeError: + pass + + return o + + class _EventInternalMetadata(object): def __init__(self, internal_metadata_dict): self.__dict__ = internal_metadata_dict @@ -24,78 +39,47 @@ class _EventInternalMetadata(object): return dict(self.__dict__) -class Event(object): - def __init__(self, event_dict, internal_metadata_dict={}): - self._signatures = event_dict.get("signatures", {}) - self._unsigned = event_dict.get("unsigned", {}) +def _event_dict_property(key): + def getter(self): + return self._event_dict[key] - self._original = { - k: v - for k, v in event_dict.items() - if k not in ["signatures", "unsigned"] - } + def setter(self, v): + self._event_dict[key] = v - self._event_dict = frozendict(self._original) + def delete(self): + del self._event_dict[key] + + return property( + getter, + setter, + delete, + ) + + +class EventBase(object): + def __init__(self, event_dict, signatures={}, unsigned={}, + internal_metadata_dict={}): + self.signatures = signatures + self.unsigned = unsigned + + self._event_dict = event_dict self.internal_metadata = _EventInternalMetadata( internal_metadata_dict ) - @property - def auth_events(self): - return self._event_dict["auth_events"] - - @property - def content(self): - return self._event_dict["content"] - - @property - def event_id(self): - return self._event_dict["event_id"] - - @property - def hashes(self): - return self._event_dict["hashes"] - - @property - def origin(self): - return self._event_dict["origin"] - - @property - def prev_events(self): - return self._event_dict["prev_events"] - - @property - def prev_state(self): - return self._event_dict["prev_state"] - - @property - def room_id(self): - return self._event_dict["room_id"] - - @property - def signatures(self): - return self._signatures - - @property - def state_key(self): - return self._event_dict["state_key"] - - @property - def type(self): - return self._event_dict["type"] - - @property - def unsigned(self): - return self._unsigned - - @property - def user_id(self): - return self._event_dict["sender"] - - @property - def sender(self): - return self._event_dict["sender"] + auth_events = _event_dict_property("auth_events") + content = _event_dict_property("content") + event_id = _event_dict_property("event_id") + hashes = _event_dict_property("hashes") + origin = _event_dict_property("origin") + prev_events = _event_dict_property("prev_events") + prev_state = _event_dict_property("prev_state") + room_id = _event_dict_property("room_id") + sender = _event_dict_property("sender") + state_key = _event_dict_property("state_key") + type = _event_dict_property("type") + user_id = _event_dict_property("sender") def get_dict(self): d = dict(self._original) @@ -117,4 +101,33 @@ class Event(object): pdu_json.setdefault("unsigned", {})["age"] = int(age) del pdu_json["unsigned"]["age_ts"] - return pdu_json \ No newline at end of file + return pdu_json + + def __set__(self, instance, value): + raise AttributeError("Unrecognized attribute %s" % (instance,)) + + +class FrozenEvent(EventBase): + def __init__(self, event_dict, signatures={}, unsigned={}): + event_dict = dict(event_dict) + + signatures.update(event_dict.pop("signatures", {})) + unsigned.update(event_dict.pop("unsigned", {})) + + frozen_dict = _freeze(event_dict) + + super(FrozenEvent, self).__init__( + frozen_dict, + signatures=signatures, + unsigned=unsigned + ) + + @staticmethod + def from_event(event): + e = FrozenEvent( + event.event_dict() + ) + + e.internal_metadata = event.internal_metadata + + return e diff --git a/synapse/events/builder.py b/synapse/events/builder.py index d741795bc5..39b4d2a2ab 100644 --- a/synapse/events/builder.py +++ b/synapse/events/builder.py @@ -13,32 +13,24 @@ # See the License for the specific language governing permissions and # limitations under the License. -from . import Event +from . import EventBase, FrozenEvent from synapse.types import EventID from synapse.util.stringutils import random_string -class EventBuilder(object): +class EventBuilder(EventBase): def __init__(self, key_values={}): - self._event_dict = dict(key_values) - self._metadata = {} - - def update_event_key(self, key, value): - self._event_dict[key] = value + super(FrozenEvent, self).__init__( + key_values, + ) def update_event_keys(self, other_dict): self._event_dict.update(other_dict) - def update_internal_key(self, key, value): - self._metadata[key] = value - def build(self): - return Event( - self._event_dict, - self._metadata, - ) + return FrozenEvent.from_event(self) class EventBuilderFactory(object): diff --git a/synapse/events/snapshot.py b/synapse/events/snapshot.py new file mode 100644 index 0000000000..ca15ec09ae --- /dev/null +++ b/synapse/events/snapshot.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + + +class EventSnapshot(object): + def __init__(self, prev_events, depth, current_state, + current_state_group): + self._prev_events = prev_events + self._depth = depth + self._current_state = current_state + self._current_state_group = current_state_group + + +class EventCache(object): + def __init__(self, store): + self._store = store + + self._cache = {} + + @defer.inlineCallbacks + def load_event(self, event_id): + event = self._cache.get(event_id, None) + + if not event: + event = yield self._store.get_event( + event_id, + allow_none=True + ) + + if event: + self._cache[event_id] = event + + defer.returnValue(event) + + def load_event_from_cache(self, event_id): + return self._cache.get(event_id, None) + + def add_to_cache(self, *events): + self._cache.update({ + event.event_id: event + for event in events + }) + + +class EventContext(object): + + def __init__(self, current_state, auth_events): + self.current_state = current_state + self.auth_events = auth_events diff --git a/synapse/events/utils.py b/synapse/events/utils.py new file mode 100644 index 0000000000..412f690f08 --- /dev/null +++ b/synapse/events/utils.py @@ -0,0 +1,82 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from synapse.api.constants import EventTypes + + +def prune_event(event): + """ Returns a pruned version of the given event, which removes all keys we + don't know about or think could potentially be dodgy. + + This is used when we "redact" an event. We want to remove all fields that + the user has specified, but we do want to keep necessary information like + type, state_key etc. + """ + event_type = event.type + + allowed_keys = [ + "event_id", + "sender", + "room_id", + "hashes", + "signatures", + "content", + "type", + "state_key", + "depth", + "prev_events", + "prev_state", + "auth_events", + "origin", + "origin_server_ts", + ] + + new_content = {} + + def add_fields(*fields): + for field in fields: + if field in event.content: + new_content[field] = event.content[field] + + if event_type == EventTypes.Member: + add_fields("membership") + elif event_type == EventTypes.Create: + add_fields("creator") + elif event_type == EventTypes.JoinRules: + add_fields("join_rule") + elif event_type == EventTypes.PowerLevels: + add_fields( + "users", + "users_default", + "events", + "events_default", + "events_default", + "state_default", + "ban", + "kick", + "redact", + ) + elif event_type == EventTypes.Aliases: + add_fields("aliases") + + allowed_fields = { + k: v + for k, v in event.get_dict().items() + if k in allowed_keys + } + + allowed_fields["content"] = new_content + + return type(event)(allowed_fields) diff --git a/synapse/events/validator.py b/synapse/events/validator.py new file mode 100644 index 0000000000..7dc9506ec4 --- /dev/null +++ b/synapse/events/validator.py @@ -0,0 +1,58 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from synapse.types import EventID, RoomID, UserID +from synapse.api.errors import SynapseError + + +class EventValidator(object): + + def validate(self, event): + EventID.from_string(event.event_id) + RoomID.from_string(event.room_id) + + hasattr(event, "auth_events") + hasattr(event, "content") + hasattr(event, "hashes") + hasattr(event, "origin") + hasattr(event, "prev_events") + hasattr(event, "prev_events") + hasattr(event, "sender") + hasattr(event, "type") + + # Check that the following keys have string values + strings = [ + "origin", + "sender", + "type", + ] + + if hasattr(event, "state_key"): + strings.append("state_key") + + for s in strings: + if not isinstance(getattr(event, s), basestring): + raise SynapseError(400, "Not '%s' a string type" % (s,)) + + # Check that the following keys have dictionary values + # TODO + + # Check that the following keys have the correct format for DAGs + # TODO + + def validate_new(self, event): + self.validate(event) + + UserID.from_string(event.sender) \ No newline at end of file diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index 15adc9dc2c..a45715bf60 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -21,6 +21,8 @@ from synapse.crypto.event_signing import add_hashes_and_signatures from synapse.api.events.room import RoomMemberEvent from synapse.api.constants import Membership +from synapse.events.snapshot import EventSnapshot, EventContext + import logging @@ -56,6 +58,55 @@ class BaseHandler(object): retry_after_ms=int(1000*(time_allowed - time_now)), ) + @defer.inlineCallbacks + def _handle_new_client_event(self, builder): + latest_ret = yield self.store.get_latest_events_in_room( + builder.room_id, + ) + + depth = max([d for _, _, d in latest_ret]) + prev_events = [(e, h) for e, h, _ in latest_ret] + + group, curr_state = yield self.state_handler.resolve_state_groups( + [e for e, _ in prev_events] + ) + + snapshot = EventSnapshot( + prev_events=prev_events, + depth=depth, + current_state=curr_state, + current_state_group=group, + ) + + builder.prev_events = prev_events + builder.depth = depth + + auth_events = yield self.auth.get_event_auth(builder, curr_state) + + builder.update_event_key("auth_events", auth_events) + + add_hashes_and_signatures( + builder, self.server_name, self.signing_key + ) + + event = builder.build() + + auth_ids = zip(*auth_events)[0] + curr_auth_events = { + k: v + for k, v in curr_state + if v.event_id in auth_ids + } + + context = EventContext( + current_state=curr_state, + auth_events=curr_auth_events, + ) + + self.auth.check(event, auth_events=context.auth_events) + + + @defer.inlineCallbacks def _on_new_room_event(self, event, snapshot, extra_destinations=[], extra_users=[], suppress_auth=False, From 52f1d3c886abacda06c18deab5b76a1c7f6d99ca Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Wed, 3 Dec 2014 19:06:24 +0000 Subject: [PATCH 008/158] Store any incoming presence push in the local cache anyway, even if there's no interested observers (yet *hint*) (SYN-115) --- synapse/handlers/presence.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 815d40f166..d40218dd28 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -659,10 +659,6 @@ class PresenceHandler(BaseHandler): if room_ids: logger.debug(" | %d interested room IDs %r", len(room_ids), room_ids) - if not observers and not room_ids: - logger.debug(" | no interested observers or room IDs") - continue - state = dict(push) del state["user_id"] @@ -683,6 +679,10 @@ class PresenceHandler(BaseHandler): self._user_cachemap_latest_serial += 1 statuscache.update(state, serial=self._user_cachemap_latest_serial) + if not observers and not room_ids: + logger.debug(" | no interested observers or room IDs") + continue + self.push_update_to_clients( observed_user=user, users_to_push=observers, From f5d2514fc05e062b1425139c5064cd5bae3c3ca7 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Wed, 3 Dec 2014 19:48:14 +0000 Subject: [PATCH 009/158] @log_function on PresenceStream's get_new_events_for_user() --- synapse/handlers/presence.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index d40218dd28..84a039489f 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -804,6 +804,7 @@ class PresenceEventSource(object): ) @defer.inlineCallbacks + @log_function def get_new_events_for_user(self, user, from_key, limit): from_key = int(from_key) @@ -816,7 +817,8 @@ class PresenceEventSource(object): # TODO(paul): use a DeferredList ? How to limit concurrency. for observed_user in cachemap.keys(): cached = cachemap[observed_user] - if not (from_key < cached.serial): + + if cached.serial <= from_key: continue if (yield self.is_visible(observer_user, observed_user)): From 5d7c9ab7898f2721aa3f60ab76c53dc44322be77 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 4 Dec 2014 11:27:59 +0000 Subject: [PATCH 010/158] Begin converting things to use the new Event structure --- synapse/api/auth.py | 11 +----- synapse/events/__init__.py | 3 ++ synapse/federation/replication.py | 12 +++--- synapse/handlers/_base.py | 65 +++++++++++++++++++++++++------ synapse/handlers/federation.py | 9 +---- synapse/handlers/room.py | 1 + synapse/state.py | 18 +++++++-- synapse/storage/signatures.py | 16 ++++++++ 8 files changed, 96 insertions(+), 39 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 50d4be113f..5261c3e3bf 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -393,18 +393,11 @@ class Auth(object): if member_event.content["membership"] == Membership.JOIN: auth_events.append(member_event.event_id) - hashes = yield self.store.get_event_reference_hashes( + auth_events = yield self.store.add_event_hashes( auth_events ) - hashes = [ - { - k: encode_base64(v) for k, v in h.items() - if k == "sha256" - } - for h in hashes - ] - defer.returnValue(zip(auth_events, hashes)) + defer.returnValue(auth_events) @log_function def _can_send_event(self, event, auth_events): diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index 6748198917..6a05ba2d16 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -81,6 +81,9 @@ class EventBase(object): type = _event_dict_property("type") user_id = _event_dict_property("sender") + def is_state(self): + return hasattr(self, "state_key") + def get_dict(self): d = dict(self._original) d.update({ diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index 01f87fe423..bd56a4c108 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -112,7 +112,7 @@ class ReplicationLayer(object): self.query_handlers[query_type] = handler @log_function - def send_pdu(self, pdu): + def send_pdu(self, pdu, destinations): """Informs the replication layer about a new PDU generated within the home server that should be transmitted to others. @@ -131,7 +131,7 @@ class ReplicationLayer(object): logger.debug("[%s] transaction_layer.enqueue_pdu... ", pdu.event_id) # TODO, add errback, etc. - self._transaction_queue.enqueue_pdu(pdu, order) + self._transaction_queue.enqueue_pdu(pdu, destinations, order) logger.debug( "[%s] transaction_layer.enqueue_pdu... done", @@ -705,15 +705,13 @@ class _TransactionQueue(object): @defer.inlineCallbacks @log_function - def enqueue_pdu(self, pdu, order): + def enqueue_pdu(self, pdu, destinations, order): # We loop through all destinations to see whether we already have # a transaction in progress. If we do, stick it in the pending_pdus # table and we'll get back to it later. - destinations = set([ - d for d in pdu.destinations - if d != self.server_name - ]) + destinations = set(destinations) + destinations.remove(self.server_name) logger.debug("Sending to: %s", str(destinations)) diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index a45715bf60..890b51be30 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -15,11 +15,11 @@ from twisted.internet import defer -from synapse.api.errors import LimitExceededError +from synapse.api.errors import LimitExceededError, SynapseError from synapse.util.async import run_on_reactor from synapse.crypto.event_signing import add_hashes_and_signatures from synapse.api.events.room import RoomMemberEvent -from synapse.api.constants import Membership +from synapse.api.constants import Membership, EventTypes from synapse.events.snapshot import EventSnapshot, EventContext @@ -59,7 +59,7 @@ class BaseHandler(object): ) @defer.inlineCallbacks - def _handle_new_client_event(self, builder): + def _create_new_client_event(self, builder): latest_ret = yield self.store.get_latest_events_in_room( builder.room_id, ) @@ -67,16 +67,27 @@ class BaseHandler(object): depth = max([d for _, _, d in latest_ret]) prev_events = [(e, h) for e, h, _ in latest_ret] - group, curr_state = yield self.state_handler.resolve_state_groups( - [e for e, _ in prev_events] - ) + state_handler = self.state_handler + if builder.is_state(): + ret = yield state_handler.resolve_state_groups( + [e for e, _ in prev_events], + event_type=builder.event_type, + state_key=builder.state_key, + ) - snapshot = EventSnapshot( - prev_events=prev_events, - depth=depth, - current_state=curr_state, - current_state_group=group, - ) + group, curr_state, prev_state = ret + + prev_state = yield self.store.add_event_hashes( + prev_state + ) + + builder.prev_state = prev_state + else: + group, curr_state, _ = yield state_handler.resolve_state_groups( + [e for e, _ in prev_events], + ) + + builder.internal_metadata.state_group = group builder.prev_events = prev_events builder.depth = depth @@ -103,9 +114,39 @@ class BaseHandler(object): auth_events=curr_auth_events, ) + defer.returnValue( + (event, context,) + ) + + @defer.inlineCallbacks + def _handle_new_client_event(self, event, context): + # We now need to go and hit out to wherever we need to hit out to. + self.auth.check(event, auth_events=context.auth_events) + yield self.store.persist_event(event) + destinations = set() + for k, s in context.current_state.items(): + try: + if k[0] == EventTypes.Member: + if s.content["membership"] == Membership.JOIN: + destinations.add( + self.hs.parse_userid(s.state_key).domain + ) + except SynapseError: + logger.warn( + "Failed to get destination from event %s", s.event_id + ) + + yield self.notifier.on_new_room_event(event) + + federation_handler = self.hs.get_handlers().federation_handler + yield federation_handler.handle_new_event( + event, + None, + destinations=destinations, + ) @defer.inlineCallbacks def _on_new_room_event(self, event, snapshot, extra_destinations=[], diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 925eb5376e..7bd36e415e 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -76,7 +76,7 @@ class FederationHandler(BaseHandler): @log_function @defer.inlineCallbacks - def handle_new_event(self, event, snapshot): + def handle_new_event(self, event, snapshot, destinations): """ Takes in an event from the client to server side, that has already been authed and handled by the state module, and sends it to any remote home servers that may be interested. @@ -92,12 +92,7 @@ class FederationHandler(BaseHandler): yield run_on_reactor() - pdu = event - - if not hasattr(pdu, "destinations") or not pdu.destinations: - pdu.destinations = [] - - yield self.replication_layer.send_pdu(pdu) + yield self.replication_layer.send_pdu(event, destinations) @log_function @defer.inlineCallbacks diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 6e1c37df03..52a9788823 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -378,6 +378,7 @@ class RoomMemberHandler(BaseHandler): else: # This is not a JOIN, so we can handle it normally. + # FIXME: This isn't idempotency. if prev_state and prev_state.membership == event.membership: # double same action, treat this event as a NOOP. defer.returnValue({}) diff --git a/synapse/state.py b/synapse/state.py index 430665f7ba..8a556a27f6 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -89,7 +89,7 @@ class StateHandler(object): ids = [e for e, _ in event.prev_events] ret = yield self.resolve_state_groups(ids) - state_group, new_state = ret + state_group, new_state, _ = ret event.old_state_events = copy.deepcopy(new_state) @@ -137,7 +137,7 @@ class StateHandler(object): @defer.inlineCallbacks @log_function - def resolve_state_groups(self, event_ids): + def resolve_state_groups(self, event_ids, event_type=None, state_key=""): """ Given a list of event_ids this method fetches the state at each event, resolves conflicts between them and returns them. @@ -156,7 +156,10 @@ class StateHandler(object): (e.type, e.state_key): e for e in state_list } - defer.returnValue((name, state)) + prev_state = state.get((event_type, state_key), None) + if prev_state: + prev_state = prev_state.event_id + defer.returnValue((name, state, [prev_state])) state = {} for group, g_state in state_groups.items(): @@ -177,6 +180,13 @@ class StateHandler(object): if len(v.values()) > 1 } + if event_type: + prev_states = conflicted_state.get( + (event_type, state_key), {} + ).keys() + else: + prev_states = [] + try: new_state = {} new_state.update(unconflicted_state) @@ -186,7 +196,7 @@ class StateHandler(object): logger.exception("Failed to resolve state") raise - defer.returnValue((None, new_state)) + defer.returnValue((None, new_state, prev_states)) def _get_power_level_from_event_state(self, event, user_id): if hasattr(event, "old_state_events") and event.old_state_events: diff --git a/synapse/storage/signatures.py b/synapse/storage/signatures.py index eea4f21065..e2f11c7ffc 100644 --- a/synapse/storage/signatures.py +++ b/synapse/storage/signatures.py @@ -15,6 +15,8 @@ from _base import SQLBaseStore +from syutil.base64util import encode_base64 + class SignatureStore(SQLBaseStore): """Persistence for event signatures and hashes""" @@ -67,6 +69,20 @@ class SignatureStore(SQLBaseStore): f ) + def add_event_hashes(self, event_ids): + hashes = yield self.store.get_event_reference_hashes( + event_ids + ) + hashes = [ + { + k: encode_base64(v) for k, v in h.items() + if k == "sha256" + } + for h in hashes + ] + + defer.returnValue(zip(event_ids, hashes)) + def _get_event_reference_hashes_txn(self, txn, event_id): """Get all the hashes for a given PDU. Args: From c01fd5573c92c7c6da258bac7ff377a91cbebfd1 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 4 Dec 2014 14:22:31 +0000 Subject: [PATCH 011/158] Implement download support for media_repository --- synapse/http/matrixfederationclient.py | 73 +++++++++- synapse/media/v1/download_resource.py | 194 +++++++++++++++++++++++++ synapse/media/v1/media_repository.py | 2 + synapse/media/v1/upload_resource.py | 11 +- synapse/storage/media_repository.py | 10 +- 5 files changed, 278 insertions(+), 12 deletions(-) create mode 100644 synapse/media/v1/download_resource.py diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 510f07dd7b..c7082b83a7 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -14,10 +14,11 @@ # limitations under the License. -from twisted.internet import defer, reactor +from twisted.internet import defer, reactor, protocol from twisted.internet.error import DNSLookupError from twisted.web.client import readBody, _AgentBase, _URI from twisted.web.http_headers import Headers +from twisted.web._newclient import ResponseDone from synapse.http.endpoint import matrix_federation_endpoint from synapse.util.async import sleep @@ -227,7 +228,7 @@ class MatrixFederationHttpClient(object): @defer.inlineCallbacks def get_json(self, destination, path, args={}, retry_on_dns_fail=True): - """ Get's some json from the given host homeserver and path + """ GETs some json from the given host homeserver and path Args: destination (str): The remote server to send the HTTP request @@ -235,9 +236,6 @@ class MatrixFederationHttpClient(object): path (str): The HTTP path. args (dict): A dictionary used to create query strings, defaults to None. - **Note**: The value of each key is assumed to be an iterable - and *not* a string. - Returns: Deferred: Succeeds when we get *any* HTTP response. @@ -272,6 +270,48 @@ class MatrixFederationHttpClient(object): defer.returnValue(json.loads(body)) + @defer.inlineCallbacks + def get_file(self, destination, path, output_stream, args={}, + retry_on_dns_fail=True): + """GETs a file from a given homeserver + Args: + destination (str): The remote server to send the HTTP request to. + path (str): The HTTP path to GET. + output_stream (file): File to write the response body to. + args (dict): Optional dictionary used to create the query string. + Returns: + A (int,dict) tuple of the file length and a dict of the response + headers. + """ + + encoded_args = {} + for k, vs in args.items(): + if isinstance(vs, basestring): + vs = [vs] + encoded_args[k] = [v.encode("UTF-8") for v in vs] + + query_bytes = urllib.urlencode(encoded_args, True) + logger.debug("Query bytes: %s Retry DNS: %s", args, retry_on_dns_fail) + + def body_callback(method, url_bytes, headers_dict): + self.sign_request(destination, method, url_bytes, headers_dict) + return None + + response = yield self._create_request( + destination.encode("ascii"), + "GET", + path.encode("ascii"), + query_bytes=query_bytes, + body_callback=body_callback, + retry_on_dns_fail=retry_on_dns_fail + ) + + headers = dict(response.headers.getAllRawHeaders()) + + length = yield _readBodyToFile(response, output_stream) + + defer.returnValue((length, headers)) + def _getEndpoint(self, reactor, destination): return matrix_federation_endpoint( reactor, destination, timeout=10, @@ -279,6 +319,29 @@ class MatrixFederationHttpClient(object): ) +class _ReadBodyToFileProtocol(protocol.Protocol): + def __init__(self, stream, deferred): + self.stream = stream + self.deferred = deferred + self.length = 0 + + def dataReceived(self, data): + self.stream.write(data) + self.length += len(data) + + def connectionLost(self, reason): + if reason.check(ResponseDone): + self.deferred.callback(self.length) + else: + self.deferred.errback(reason) + + +def _readBodyToFile(response, stream): + d = defer.Deferred() + response.deliverBody(_ReadBodyToFileProtocol(stream, d)) + return d + + def _print_ex(e): if hasattr(e, "reasons") and e.reasons: for ex in e.reasons: diff --git a/synapse/media/v1/download_resource.py b/synapse/media/v1/download_resource.py new file mode 100644 index 0000000000..c243f16a74 --- /dev/null +++ b/synapse/media/v1/download_resource.py @@ -0,0 +1,194 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from synapse.http.server import respond_with_json +from synapse.util.stringutils import random_string +from synapse.api.errors import ( + cs_exception, CodeMessageException, cs_error, Codes +) + +from twisted.protocols.basic import FileSender +from twisted.web.resource import Resource +from twisted.web.server import NOT_DONE_YET +from twisted.internet import defer + +import os + +import logging + +logger = logging.getLogger(__name__) + + +class DownloadResource(Resource): + isLeaf = True + + def __init__(self, hs, filepaths): + Resource.__init__(self) + self.client = hs.get_http_client() + self.clock = hs.get_clock() + self.server_name = hs.hostname + self.store = hs.get_datastore() + self.filepaths = filepaths + + def render_GET(self, request): + self._async_render_GET(request) + return NOT_DONE_YET + + def _respond_404(self, request): + respond_with_json( + request, 404, + cs_error( + "Not found %r" % (request.postpath,), + code=Codes.NOT_FOUND, + ), + send_cors=True + ) + + @defer.inlineCallbacks + def _async_render_GET(self, request): + + try: + server_name, media_id = request.postpath + except: + self._respond_404(request) + return + + try: + if server_name == self.server_name: + yield self._respond_local_file(request, media_id) + else: + yield self._respond_remote_file(request, server_name, media_id) + except CodeMessageException as e: + logger.exception(e) + respond_with_json(request, e.code, cs_exception(e), send_cors=True) + except: + logger.exception("Failed to serve file") + respond_with_json( + request, + 500, + {"error": "Internal server error"}, + send_cors=True + ) + + @defer.inlineCallbacks + def _download_remote_file(self, server_name, media_id): + filesystem_id = random_string(24) + + fname = self.filepaths.remote_media_filepath( + server_name, filesystem_id + ) + os.makedirs(os.path.dirname(fname)) + + try: + with open(fname, "wb") as f: + length, headers = yield self.client.get_file( + server_name, + "/".join(( + "/_matrix/media/v1/download", server_name, media_id, + )), + output_stream=f, + ) + except: + os.remove(fname) + raise + + media_type = headers["Content-Type"][0] + time_now_ms = self.clock.time_msec() + + yield self.store.store_cached_remote_media( + origin=server_name, + media_id=media_id, + media_type=media_type, + time_now_ms=self.clock.time_msec(), + upload_name=None, + media_length=length, + filesystem_id=filesystem_id, + ) + + defer.returnValue({ + "media_type": media_type, + "media_length": length, + "upload_name": None, + "created_ts": time_now_ms, + "filesystem_id": filesystem_id, + }) + + @defer.inlineCallbacks + def _respond_remote_file(self, request, server_name, media_id): + media_info = yield self.store.get_cached_remote_media( + server_name, media_id + ) + + if not media_info: + media_info = yield self._download_remote_file( + server_name, media_id + ) + + filesystem_id = media_info["filesystem_id"] + + file_path = self.filepaths.remote_media_filepath( + server_name, filesystem_id + ) + + if os.path.isfile(file_path): + media_type = media_info["media_type"] + request.setHeader(b"Content-Type", media_type.encode("UTF-8")) + + # cache for at least a day. + # XXX: we might want to turn this off for data we don't want to + # recommend caching as it's sensitive or private - or at least + # select private. don't bother setting Expires as all our + # clients are smart enough to be happy with Cache-Control + request.setHeader( + b"Cache-Control", b"public,max-age=86400,s-maxage=86400" + ) + + with open(file_path, "rb") as f: + yield FileSender().beginFileTransfer(f, request) + + request.finish() + else: + self._respond_404() + + @defer.inlineCallbacks + def _respond_local_file(self, request, media_id): + media_info = yield self.store.get_local_media(media_id) + if not media_info: + self._respond_404() + return + + file_path = self.filepaths.local_media_filepath(media_id) + + logger.debug("Searching for %s", file_path) + + if os.path.isfile(file_path): + media_type = media_info["media_type"] + request.setHeader(b"Content-Type", media_type.encode("UTF-8")) + + # cache for at least a day. + # XXX: we might want to turn this off for data we don't want to + # recommend caching as it's sensitive or private - or at least + # select private. don't bother setting Expires as all our + # clients are smart enough to be happy with Cache-Control + request.setHeader( + b"Cache-Control", b"public,max-age=86400,s-maxage=86400" + ) + + with open(file_path, "rb") as f: + yield FileSender().beginFileTransfer(f, request) + + request.finish() + else: + self._respond_404() diff --git a/synapse/media/v1/media_repository.py b/synapse/media/v1/media_repository.py index afd92874cf..e0a4cd01ee 100644 --- a/synapse/media/v1/media_repository.py +++ b/synapse/media/v1/media_repository.py @@ -14,6 +14,7 @@ # limitations under the License. from .upload_resource import UploadResource +from .download_resource import DownloadResource from .filepath import MediaFilePaths from twisted.web.resource import Resource @@ -62,3 +63,4 @@ class MediaRepositoryResource(Resource): Resource.__init__(self) filepaths = MediaFilePaths(hs.config.media_store_path) self.putChild("upload", UploadResource(hs, filepaths)) + self.putChild("download", DownloadResource(hs, filepaths)) diff --git a/synapse/media/v1/upload_resource.py b/synapse/media/v1/upload_resource.py index 2919fee12f..91bcc5caff 100644 --- a/synapse/media/v1/upload_resource.py +++ b/synapse/media/v1/upload_resource.py @@ -20,7 +20,8 @@ from synapse.api.errors import ( cs_exception, SynapseError, CodeMessageException ) -from twisted.web import server, resource +from twisted.web.resource import Resource +from twisted.web.server import NOT_DONE_YET from twisted.internet import defer import os @@ -30,9 +31,11 @@ import logging logger = logging.getLogger(__name__) -class UploadResource(resource.Resource): +class UploadResource(Resource): + isLeaf = True def __init__(self, hs, filepaths): + Resource.__init__(self) self.auth = hs.get_auth() self.clock = hs.get_clock() self.store = hs.get_datastore() @@ -41,11 +44,11 @@ class UploadResource(resource.Resource): def render_POST(self, request): self._async_render_POST(request) - return server.NOT_DONE_YET + return NOT_DONE_YET def render_OPTIONS(self, request): respond_with_json(request, 200, {}, send_cors=True) - return server.NOT_DONE_YET + return NOT_DONE_YET @defer.inlineCallbacks def _async_render_POST(self, request): diff --git a/synapse/storage/media_repository.py b/synapse/storage/media_repository.py index eda191ad5b..2d3a2d1ccb 100644 --- a/synapse/storage/media_repository.py +++ b/synapse/storage/media_repository.py @@ -22,13 +22,13 @@ class MediaRepositoryStore(SQLBaseStore): def get_local_media(self, media_id): """Get the metadata for a local piece of media Returns: - None if the media_id doesn't exist. + None if the meia_id doesn't exist. """ return self._simple_select_one( "local_media_repository", {"media_id": media_id}, ("media_type", "media_length", "upload_name", "created_ts"), - True, + allow_none=True, ) def store_local_media(self, media_id, media_type, time_now_ms, upload_name, @@ -73,7 +73,11 @@ class MediaRepositoryStore(SQLBaseStore): return self._simple_select_one( "remote_media_cache", {"media_origin": origin, "media_id": media_id}, - ("media_type", "media_length", "upload_name", "created_ts"), + ( + "media_type", "media_length", "upload_name", "created_ts", + "filesystem_id", + ), + allow_none=True, ) def store_cached_remote_media(self, origin, media_id, media_type, From c31dba86ec40853f27c70ae13409ca3332052cc1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 4 Dec 2014 15:50:01 +0000 Subject: [PATCH 012/158] Convert rest and handlers to use new event structure --- synapse/crypto/event_signing.py | 2 +- synapse/events/__init__.py | 8 +- synapse/events/builder.py | 5 +- synapse/federation/replication.py | 18 ++--- synapse/handlers/_base.py | 16 ++-- synapse/handlers/directory.py | 20 ++--- synapse/handlers/federation.py | 19 ++--- synapse/handlers/message.py | 23 +++++- synapse/handlers/profile.py | 19 ++--- synapse/handlers/room.py | 130 ++++++++++++------------------ synapse/rest/base.py | 2 +- synapse/rest/room.py | 112 +++++++++++-------------- synapse/server.py | 8 ++ synapse/storage/signatures.py | 5 +- 14 files changed, 188 insertions(+), 199 deletions(-) diff --git a/synapse/crypto/event_signing.py b/synapse/crypto/event_signing.py index 209f9d73fe..b189f0bb2b 100644 --- a/synapse/crypto/event_signing.py +++ b/synapse/crypto/event_signing.py @@ -15,7 +15,7 @@ # limitations under the License. -from synapse.api.events.utils import prune_event +from synapse.events.utils import prune_event from syutil.jsonutil import encode_canonical_json from syutil.base64util import encode_base64, decode_base64 from syutil.crypto.jsonsign import sign_json diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index 6a05ba2d16..58edf2bc8f 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -85,10 +85,10 @@ class EventBase(object): return hasattr(self, "state_key") def get_dict(self): - d = dict(self._original) + d = dict(self._event_dict) d.update({ - "signatures": self._signatures, - "unsigned": self._unsigned, + "signatures": self.signatures, + "unsigned": self.unsigned, }) return d @@ -128,7 +128,7 @@ class FrozenEvent(EventBase): @staticmethod def from_event(event): e = FrozenEvent( - event.event_dict() + event.get_pdu_json() ) e.internal_metadata = event.internal_metadata diff --git a/synapse/events/builder.py b/synapse/events/builder.py index 39b4d2a2ab..0b8caf9318 100644 --- a/synapse/events/builder.py +++ b/synapse/events/builder.py @@ -22,10 +22,13 @@ from synapse.util.stringutils import random_string class EventBuilder(EventBase): def __init__(self, key_values={}): - super(FrozenEvent, self).__init__( + super(EventBuilder, self).__init__( key_values, ) + def update_event_key(self, key, value): + self._event_dict[key] = value + def update_event_keys(self, other_dict): self._event_dict.update(other_dict) diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index bd56a4c108..b11df9e5c6 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -74,6 +74,7 @@ class ReplicationLayer(object): self._clock = hs.get_clock() self.event_factory = hs.get_event_factory() + self.event_builder_factory = hs.get_event_builder_factory() def set_handler(self, handler): """Sets the handler that the replication layer will use to communicate @@ -658,19 +659,14 @@ class ReplicationLayer(object): return "" % self.server_name def event_from_pdu_json(self, pdu_json, outlier=False): - #TODO: Check we have all the PDU keys here - pdu_json.setdefault("hashes", {}) - pdu_json.setdefault("signatures", {}) - sender = pdu_json.pop("sender", None) - if sender is not None: - pdu_json["user_id"] = sender - state_hash = pdu_json.get("unsigned", {}).pop("state_hash", None) - if state_hash is not None: - pdu_json["state_hash"] = state_hash - return self.event_factory.create_event( - pdu_json["type"], outlier=outlier, **pdu_json + builder = self.event_builder_factory.new( + pdu_json ) + builder.internal_metadata = outlier + + return builder.build() + class _TransactionQueue(object): """This class makes sure we only have one transaction in flight at diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index 890b51be30..4052d0e1e7 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -46,6 +46,8 @@ class BaseHandler(object): self.signing_key = hs.config.signing_key[0] self.server_name = hs.hostname + self.event_builder_factory = hs.get_event_builder_factory() + def ratelimit(self, user_id): time_now = self.clock.time() allowed, time_allowed = self.ratelimiter.send_message( @@ -92,7 +94,7 @@ class BaseHandler(object): builder.prev_events = prev_events builder.depth = depth - auth_events = yield self.auth.get_event_auth(builder, curr_state) + auth_events = yield self.auth.get_auth_events(builder, curr_state) builder.update_event_key("auth_events", auth_events) @@ -105,7 +107,7 @@ class BaseHandler(object): auth_ids = zip(*auth_events)[0] curr_auth_events = { k: v - for k, v in curr_state + for k, v in curr_state.items() if v.event_id in auth_ids } @@ -119,14 +121,16 @@ class BaseHandler(object): ) @defer.inlineCallbacks - def _handle_new_client_event(self, event, context): + def handle_new_client_event(self, event, context, extra_destinations=[], + extra_users=[], suppress_auth=False): # We now need to go and hit out to wherever we need to hit out to. - self.auth.check(event, auth_events=context.auth_events) + if not suppress_auth: + self.auth.check(event, auth_events=context.auth_events) yield self.store.persist_event(event) - destinations = set() + destinations = set(extra_destinations) for k, s in context.current_state.items(): try: if k[0] == EventTypes.Member: @@ -139,7 +143,7 @@ class BaseHandler(object): "Failed to get destination from event %s", s.event_id ) - yield self.notifier.on_new_room_event(event) + yield self.notifier.on_new_room_event(event, extra_users=extra_users) federation_handler = self.hs.get_handlers().federation_handler yield federation_handler.handle_new_event( diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index b95c4b8bf7..76fb897f20 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -148,16 +148,12 @@ class DirectoryHandler(BaseHandler): def send_room_alias_update_event(self, user_id, room_id): aliases = yield self.store.get_aliases_for_room(room_id) - event = self.event_factory.create_event( - etype=RoomAliasesEvent.TYPE, - state_key=self.hs.hostname, - room_id=room_id, - user_id=user_id, - content={"aliases": aliases}, - ) + msg_handler = self.hs.get_handlers().message_handler + yield msg_handler.handle_event({ + "type": RoomAliasesEvent.TYPE, + "state_key": self.hs.hostname, + "room_id": room_id, + "sender": user_id, + "content": {"aliases": aliases}, + }) - snapshot = yield self.store.snapshot_room(event) - - yield self._on_new_room_event( - event, snapshot, extra_users=[user_id], suppress_auth=True - ) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 7bd36e415e..b4a28ea3cb 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -421,16 +421,17 @@ class FederationHandler(BaseHandler): join event for the room and return that. We don *not* persist or process it until the other server has signed it and sent it back. """ - event = self.event_factory.create_event( - etype=RoomMemberEvent.TYPE, - content={"membership": Membership.JOIN}, - room_id=context, - user_id=user_id, - state_key=user_id, - ) + builder = self.event_builder_factory.new({ + "type": RoomMemberEvent.TYPE, + "content": {"membership": Membership.JOIN}, + "room_id": context, + "sender": user_id, + "state_key": user_id, + }) - snapshot = yield self.store.snapshot_room(event) - snapshot.fill_out_prev_events(event) + event, context = yield self._create_new_client_event( + builder=builder, + ) yield self.state_handler.annotate_event_with_state(event) yield self.auth.add_auth_events(event) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 269d6622e1..485d8e8179 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -15,7 +15,7 @@ from twisted.internet import defer -from synapse.api.constants import Membership +from synapse.api.constants import EventTypes, Membership from synapse.api.errors import RoomError from synapse.streams.config import PaginationConfig from synapse.util.logcontext import PreserveLoggingContext @@ -133,6 +133,27 @@ class MessageHandler(BaseHandler): defer.returnValue(chunk) + @defer.inlineCallbacks + def handle_event(self, event_dict): + builder = self.event_builder_factory.new(event_dict) + + event, context = yield self._create_new_client_event( + builder=builder, + ) + + # TODO: self.validator.validate(event) + + if event.type == EventTypes.Member: + member_handler = self.hs.get_handlers().room_member_handler + yield member_handler.change_membership(event, context) + else: + yield self.handle_new_client_event( + event=event, + context=context, + ) + + defer.returnValue(event) + @defer.inlineCallbacks def store_room_data(self, event=None): """ Stores data for a room. diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index 0116ba5358..f2abbc5df9 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -210,14 +210,11 @@ class ProfileHandler(BaseHandler): "collect_presencelike_data", user, content ) - new_event = self.event_factory.create_event( - etype=j.type, - room_id=j.room_id, - state_key=j.state_key, - content=content, - user_id=j.state_key, - ) - - yield self._on_new_room_event( - new_event, snapshot, suppress_auth=True - ) + msg_handler = self.hs.get_handlers().message_handler + yield msg_handler.handle_event({ + "type": j.type, + "room_id": j.room_id, + "state_key": j.state_key, + "content": content, + "sender": j.state_key, + }) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 52a9788823..f0ffd62b7f 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -123,59 +123,37 @@ class RoomCreationHandler(BaseHandler): user, room_id, is_public=is_public ) - room_member_handler = self.hs.get_handlers().room_member_handler - - @defer.inlineCallbacks - def handle_event(event): - snapshot = yield self.store.snapshot_room(event) - - logger.debug("Event: %s", event) - - if event.type == RoomMemberEvent.TYPE: - yield room_member_handler.change_membership( - event, - do_auth=True - ) - else: - yield self._on_new_room_event( - event, snapshot, extra_users=[user], suppress_auth=True - ) + msg_handler = self.hs.get_handlers().message_handler for event in creation_events: - yield handle_event(event) + yield msg_handler.handle_event(event) if "name" in config: name = config["name"] - name_event = self.event_factory.create_event( - etype=RoomNameEvent.TYPE, - room_id=room_id, - user_id=user_id, - content={"name": name}, - ) - - yield handle_event(name_event) + yield msg_handler.handle_event({ + "type": RoomNameEvent.TYPE, + "room_id": room_id, + "sender": user_id, + "content": {"name": name}, + }) if "topic" in config: topic = config["topic"] - topic_event = self.event_factory.create_event( - etype=RoomTopicEvent.TYPE, - room_id=room_id, - user_id=user_id, - content={"topic": topic}, - ) + yield msg_handler.handle_event({ + "type": RoomTopicEvent.TYPE, + "room_id": room_id, + "sender": user_id, + "content": {"topic": topic}, + }) - yield handle_event(topic_event) - - content = {"membership": Membership.INVITE} for invitee in invite_list: - invite_event = self.event_factory.create_event( - etype=RoomMemberEvent.TYPE, - state_key=invitee, - room_id=room_id, - user_id=user_id, - content=content - ) - yield handle_event(invite_event) + yield msg_handler.handle_event({ + "type": RoomMemberEvent.TYPE, + "state_key": invitee, + "room_id": room_id, + "user_id": user_id, + "content": {"membership": Membership.INVITE}, + }) result = {"room_id": room_id} @@ -192,22 +170,25 @@ class RoomCreationHandler(BaseHandler): event_keys = { "room_id": room_id, - "user_id": creator_id, + "sender": creator_id, } - def create(etype, **content): - return self.event_factory.create_event( - etype=etype, - content=content, - **event_keys - ) + def create(etype, content): + e = { + "type": etype, + "content": content, + } + + e.update(event_keys) + + return e creation_event = create( etype=RoomCreateEvent.TYPE, - creator=creator.to_string(), + content={"creator": creator.to_string()}, ) - join_event = self.event_factory.create_event( + join_event = create( etype=RoomMemberEvent.TYPE, state_key=creator_id, content={ @@ -216,7 +197,7 @@ class RoomCreationHandler(BaseHandler): **event_keys ) - power_levels_event = self.event_factory.create_event( + power_levels_event = create( etype=RoomPowerLevelsEvent.TYPE, content={ "users": { @@ -233,13 +214,12 @@ class RoomCreationHandler(BaseHandler): "kick": 50, "redact": 50 }, - **event_keys ) join_rule = JoinRules.PUBLIC if is_public else JoinRules.INVITE join_rules_event = create( etype=RoomJoinRulesEvent.TYPE, - join_rule=join_rule, + content={"join_rule": join_rule}, ) return [ @@ -351,7 +331,7 @@ class RoomMemberHandler(BaseHandler): defer.returnValue(member) @defer.inlineCallbacks - def change_membership(self, event=None, do_auth=True): + def change_membership(self, event, context, do_auth=True): """ Change the membership status of a user in a room. Args: @@ -361,8 +341,6 @@ class RoomMemberHandler(BaseHandler): """ target_user_id = event.state_key - snapshot = yield self.store.snapshot_room(event) - ## TODO(markjh): get prev state from snapshot. prev_state = yield self.store.get_room_member( target_user_id, event.room_id @@ -374,7 +352,7 @@ class RoomMemberHandler(BaseHandler): # if this HS is not currently in the room, i.e. we have to do the # invite/join dance. if event.membership == Membership.JOIN: - yield self._do_join(event, snapshot, do_auth=do_auth) + yield self._do_join(event, context, do_auth=do_auth) else: # This is not a JOIN, so we can handle it normally. @@ -387,7 +365,7 @@ class RoomMemberHandler(BaseHandler): yield self._do_local_membership_update( event, membership=event.content["membership"], - snapshot=snapshot, + context=context, do_auth=do_auth, ) @@ -409,23 +387,21 @@ class RoomMemberHandler(BaseHandler): host = hosts[0] content.update({"membership": Membership.JOIN}) - new_event = self.event_factory.create_event( - etype=RoomMemberEvent.TYPE, - state_key=joinee.to_string(), - room_id=room_id, - user_id=joinee.to_string(), - membership=Membership.JOIN, - content=content, - ) + event, context = yield self.create_new_client_event({ + "type": RoomMemberEvent.TYPE, + "state_key": joinee.to_string(), + "room_id": room_id, + "sender": joinee.to_string(), + "membership": Membership.JOIN, + "content": content, + }) - snapshot = yield self.store.snapshot_room(new_event) - - yield self._do_join(new_event, snapshot, room_host=host, do_auth=True) + yield self._do_join(event, context, room_host=host, do_auth=True) defer.returnValue({"room_id": room_id}) @defer.inlineCallbacks - def _do_join(self, event, snapshot, room_host=None, do_auth=True): + def _do_join(self, event, context, room_host=None, do_auth=True): joinee = self.hs.parse_userid(event.state_key) # room_id = RoomID.from_string(event.room_id, self.hs) room_id = event.room_id @@ -470,7 +446,7 @@ class RoomMemberHandler(BaseHandler): if should_do_dance: handler = self.hs.get_handlers().federation_handler have_joined = yield handler.do_invite_join( - room_host, room_id, event.user_id, event.content, snapshot + room_host, room_id, event.user_id, event.content, context ) # We want to do the _do_update inside the room lock. @@ -480,7 +456,7 @@ class RoomMemberHandler(BaseHandler): yield self._do_local_membership_update( event, membership=event.content["membership"], - snapshot=snapshot, + context=context, do_auth=do_auth, ) @@ -530,7 +506,7 @@ class RoomMemberHandler(BaseHandler): defer.returnValue(room_ids) @defer.inlineCallbacks - def _do_local_membership_update(self, event, membership, snapshot, + def _do_local_membership_update(self, event, membership, context, do_auth): yield run_on_reactor() @@ -543,9 +519,9 @@ class RoomMemberHandler(BaseHandler): else: do_invite_host = None - yield self._on_new_room_event( + yield self.handle_new_client_event( event, - snapshot, + context, extra_users=[target_user], suppress_auth=(not do_auth), do_invite_host=do_invite_host, diff --git a/synapse/rest/base.py b/synapse/rest/base.py index 79fc4dfb84..72bb66ddda 100644 --- a/synapse/rest/base.py +++ b/synapse/rest/base.py @@ -63,7 +63,7 @@ class RestServlet(object): self.hs = hs self.handlers = hs.get_handlers() - self.event_factory = hs.get_event_factory() + self.builder_factory = hs.get_event_builder_factory() self.auth = hs.get_auth() self.txns = HttpTransactionStore() diff --git a/synapse/rest/room.py b/synapse/rest/room.py index 3147d7a60b..3d78b4ff5c 100644 --- a/synapse/rest/room.py +++ b/synapse/rest/room.py @@ -117,10 +117,10 @@ class RoomStateEventRestServlet(RestServlet): self.on_PUT_no_state_key) def on_GET_no_state_key(self, request, room_id, event_type): - return self.on_GET(request, room_id, event_type, "") + return self.on_GET(request, room_id, event_type, None) def on_PUT_no_state_key(self, request, room_id, event_type): - return self.on_PUT(request, room_id, event_type, "") + return self.on_PUT(request, room_id, event_type, None) @defer.inlineCallbacks def on_GET(self, request, room_id, event_type, state_key): @@ -147,28 +147,18 @@ class RoomStateEventRestServlet(RestServlet): content = _parse_json(request) - event = self.event_factory.create_event( - etype=event_type, # already urldecoded - content=content, - room_id=urllib.unquote(room_id), - user_id=user.to_string(), - state_key=urllib.unquote(state_key) - ) + msg_handler = self.handlers.message_handler + yield msg_handler.handle_event( + { + "type": event_type, + "content": content, + "room_id": room_id, + "sender": user.to_string(), + "state_key": urllib.unquote(state_key), + } + ) - self.validator.validate(event) - - if event_type == RoomMemberEvent.TYPE: - # membership events are special - handler = self.handlers.room_member_handler - yield handler.change_membership(event) - defer.returnValue((200, {})) - else: - # store random bits of state - msg_handler = self.handlers.message_handler - yield msg_handler.store_room_data( - event=event - ) - defer.returnValue((200, {})) + defer.returnValue((200, {})) # TODO: Needs unit testing for generic events + feedback @@ -184,17 +174,15 @@ class RoomSendEventRestServlet(RestServlet): user = yield self.auth.get_user_by_req(request) content = _parse_json(request) - event = self.event_factory.create_event( - etype=urllib.unquote(event_type), - room_id=urllib.unquote(room_id), - user_id=user.to_string(), - content=content - ) - - self.validator.validate(event) - msg_handler = self.handlers.message_handler - yield msg_handler.send_message(event) + event = yield msg_handler.handle_event( + { + "type": urllib.unquote(event_type), + "content": content, + "room_id": urllib.unquote(room_id), + "sender": user.to_string(), + } + ) defer.returnValue((200, {"event_id": event.event_id})) @@ -251,18 +239,17 @@ class JoinRoomAliasServlet(RestServlet): ret_dict = yield handler.join_room_alias(user, identifier) defer.returnValue((200, ret_dict)) else: # room id - event = self.event_factory.create_event( - etype=RoomMemberEvent.TYPE, - content={"membership": Membership.JOIN}, - room_id=urllib.unquote(identifier.to_string()), - user_id=user.to_string(), - state_key=user.to_string() + msg_handler = self.handlers.message_handler + yield msg_handler.handle_event( + { + "type": RoomMemberEvent.TYPE, + "content": {"membership": Membership.JOIN}, + "room_id": urllib.unquote(identifier.to_string()), + "sender": user.to_string(), + "state_key": user.to_string(), + } ) - self.validator.validate(event) - - handler = self.handlers.room_member_handler - yield handler.change_membership(event) defer.returnValue((200, {})) @defer.inlineCallbacks @@ -414,18 +401,17 @@ class RoomMembershipRestServlet(RestServlet): if membership_action == "kick": membership_action = "leave" - event = self.event_factory.create_event( - etype=RoomMemberEvent.TYPE, - content={"membership": unicode(membership_action)}, - room_id=urllib.unquote(room_id), - user_id=user.to_string(), - state_key=state_key + msg_handler = self.handlers.message_handler + yield msg_handler.handle_event( + { + "type": RoomMemberEvent.TYPE, + "content": {"membership": unicode(membership_action)}, + "room_id": urllib.unquote(room_id), + "sender": user.to_string(), + "state_key": state_key, + } ) - self.validator.validate(event) - - handler = self.handlers.room_member_handler - yield handler.change_membership(event) defer.returnValue((200, {})) @defer.inlineCallbacks @@ -453,18 +439,16 @@ class RoomRedactEventRestServlet(RestServlet): user = yield self.auth.get_user_by_req(request) content = _parse_json(request) - event = self.event_factory.create_event( - etype=RoomRedactionEvent.TYPE, - room_id=urllib.unquote(room_id), - user_id=user.to_string(), - content=content, - redacts=urllib.unquote(event_id), - ) - - self.validator.validate(event) - msg_handler = self.handlers.message_handler - yield msg_handler.send_message(event) + event = yield msg_handler.handle_event( + { + "type": RoomRedactionEvent.TYPE, + "content": content, + "room_id": urllib.unquote(room_id), + "sender": user.to_string(), + "redacts": urllib.unquote(event_id), + } + ) defer.returnValue((200, {"event_id": event.event_id})) diff --git a/synapse/server.py b/synapse/server.py index c3b54221d6..8bc27bbc3c 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -36,6 +36,7 @@ from synapse.util.lockutils import LockManager from synapse.streams.events import EventSources from synapse.api.ratelimiting import Ratelimiter from synapse.crypto.keyring import Keyring +from synapse.events.builder import EventBuilderFactory class BaseHomeServer(object): @@ -82,6 +83,7 @@ class BaseHomeServer(object): 'ratelimiter', 'keyring', 'event_validator', + 'event_builder_factory', ] def __init__(self, hostname, **kwargs): @@ -231,6 +233,12 @@ class HomeServer(BaseHomeServer): def build_event_validator(self): return EventValidator(self) + def build_event_builder_factory(self): + return EventBuilderFactory( + clock=self.get_clock(), + hostname=self.hostname, + ) + def register_servlets(self): """ Register all servlets associated with this HomeServer. """ diff --git a/synapse/storage/signatures.py b/synapse/storage/signatures.py index e2f11c7ffc..3a705119fd 100644 --- a/synapse/storage/signatures.py +++ b/synapse/storage/signatures.py @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from twisted.internet import defer + from _base import SQLBaseStore from syutil.base64util import encode_base64 @@ -69,8 +71,9 @@ class SignatureStore(SQLBaseStore): f ) + @defer.inlineCallbacks def add_event_hashes(self, event_ids): - hashes = yield self.store.get_event_reference_hashes( + hashes = yield self.get_event_reference_hashes( event_ids ) hashes = [ From 1c72e22c4f66061e4528a1e8a593bb28bd415cd3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 5 Dec 2014 11:21:56 +0000 Subject: [PATCH 013/158] Pull in latest matrix-angular_sdk --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 9b38f790b9..1be02daece 100755 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ setup( description="Reference Synapse Home Server", install_requires=[ "syutil==0.0.2", - "matrix_angular_sdk==0.5.1", + "matrix_angular_sdk==0.5.3", "Twisted>=14.0.0", "service_identity>=1.0.0", "pyopenssl>=0.14", @@ -45,7 +45,7 @@ setup( dependency_links=[ "https://github.com/matrix-org/syutil/tarball/v0.0.2#egg=syutil-0.0.2", "https://github.com/pyca/pynacl/tarball/d4d3175589b892f6ea7c22f466e0e223853516fa#egg=pynacl-0.3.0", - "https://github.com/matrix-org/matrix-angular-sdk/tarball/v0.5.1/#egg=matrix_angular_sdk-0.5.1", + "https://github.com/matrix-org/matrix-angular-sdk/tarball/v0.5.3/#egg=matrix_angular_sdk-0.5.3", ], setup_requires=[ "setuptools_trial", From ce212eb83aacfca3d6dee0cc6de339871efe6a2d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 5 Dec 2014 11:55:24 +0000 Subject: [PATCH 014/158] Pull in latest matrix-angular_sdk --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 1be02daece..8358092e30 100755 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ setup( description="Reference Synapse Home Server", install_requires=[ "syutil==0.0.2", - "matrix_angular_sdk==0.5.3", + "matrix_angular_sdk==0.5.3b", "Twisted>=14.0.0", "service_identity>=1.0.0", "pyopenssl>=0.14", @@ -45,7 +45,7 @@ setup( dependency_links=[ "https://github.com/matrix-org/syutil/tarball/v0.0.2#egg=syutil-0.0.2", "https://github.com/pyca/pynacl/tarball/d4d3175589b892f6ea7c22f466e0e223853516fa#egg=pynacl-0.3.0", - "https://github.com/matrix-org/matrix-angular-sdk/tarball/v0.5.3/#egg=matrix_angular_sdk-0.5.3", + "https://github.com/matrix-org/matrix-angular-sdk/tarball/v0.5.3b/#egg=matrix_angular_sdk-0.5.3b", ], setup_requires=[ "setuptools_trial", From 036382012227916718f28a63b882dd341a309b68 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 5 Dec 2014 16:12:37 +0000 Subject: [PATCH 015/158] Add a class for generating thumbnails using PIL --- synapse/media/v1/thumbnailer.py | 78 +++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 synapse/media/v1/thumbnailer.py diff --git a/synapse/media/v1/thumbnailer.py b/synapse/media/v1/thumbnailer.py new file mode 100644 index 0000000000..ed09283b27 --- /dev/null +++ b/synapse/media/v1/thumbnailer.py @@ -0,0 +1,78 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import PIL.Image + +class Thumbnailer(object): + + FORMAT_JPEG="JPEG" + FORMAT_PNG="PNG" + + def __init__(self, input_path): + self.image = PIL.Image.open(input_path) + self.width, self.height = self.image.size + + def size_preserve(self, max_width, max_height): + """Calculate the largest size that preserves aspect ratio which + fits within the given rectangle:: + + (w_in / h_in) = (w_out / h_out) + w_out = min(w_max, h_max * (w_in / h_in)) + h_out = min(h_max, w_max * (h_in / w_in)) + + Args: + max_width: The largest possible width. + max_height: The larget possible height. + """ + + if max_width * self.height < max_height * self.width: + return (max_width, (max_width * self.height) // self.width) + else: + return ((max_height * self.width) // self.height, max_height) + + def thumbnail_scale(self, output_path, output_format, width, height): + """Rescales the image to the given dimensions""" + output = self.image.resize((width, height), PIL.Image.BILINEAR) + output.save(output_path, output_format) + + def thumbnail_crop(self, output_path, output_format, width, height): + """Rescales and crops the image to the given dimensions preserving + aspect:: + (w_in / h_in) = (w_scaled / h_scaled) + w_scaled = max(w_out, h_out * (w_in / h_in)) + h_scaled = max(h_out, w_out * (h_in / w_in)) + + Args: + max_width: The largest possible width. + max_height: The larget possible height. + """ + if width * self.height > height * self.width: + scaled_height = (width * self.height) // self.width + scaled_image = self.image.resize( + (width, scaled_height), PIL.Image.BILINEAR + ) + crop_top = (scaled_height - height) // 2 + crop_bottom = height + crop_top + cropped = scaled_image.crop((0, crop_top, width, crop_bottom)) + cropped.save(output_path, output_format) + else: + scaled_width = (height * self.width) // self.height + scaled_image = self.image.resize( + (scaled_width, height), PIL.Image.BILINEAR + ) + crop_left = (scaled_width - width) // 2 + crop_right = width + crop_left + cropped = scaled_image.crop((crop_left, 0, crop_right, height)) + cropped.save(output_path, output_format) From 6630e1b5795667fd947cc5b0d5d2b00da97325e3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 5 Dec 2014 16:20:48 +0000 Subject: [PATCH 016/158] Start making more things use EventContext rather than event.* --- synapse/api/auth.py | 33 ++++--- synapse/events/__init__.py | 35 ++++++- synapse/events/utils.py | 16 ++++ synapse/handlers/_base.py | 164 +++++++++++++++------------------ synapse/handlers/federation.py | 19 ++-- synapse/server.py | 2 +- synapse/state.py | 33 +++++++ synapse/storage/__init__.py | 23 +++-- synapse/storage/_base.py | 8 +- synapse/storage/state.py | 13 ++- 10 files changed, 212 insertions(+), 134 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 5261c3e3bf..3f2e58a5ef 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -351,27 +351,27 @@ class Auth(object): return self.store.is_server_admin(user) @defer.inlineCallbacks - def get_auth_events(self, event, current_state): - if event.type == RoomCreateEvent.TYPE: - event.auth_events = [] + def add_auth_events(self, builder, context): + if builder.type == RoomCreateEvent.TYPE: + builder.auth_events = [] return auth_events = [] key = (RoomPowerLevelsEvent.TYPE, "", ) - power_level_event = current_state.get(key) + power_level_event = context.current_state.get(key) if power_level_event: auth_events.append(power_level_event.event_id) key = (RoomJoinRulesEvent.TYPE, "", ) - join_rule_event = current_state.get(key) + join_rule_event = context.current_state.get(key) - key = (RoomMemberEvent.TYPE, event.user_id, ) - member_event = current_state.get(key) + key = (RoomMemberEvent.TYPE, builder.user_id, ) + member_event = context.current_state.get(key) key = (RoomCreateEvent.TYPE, "", ) - create_event = current_state.get(key) + create_event = context.current_state.get(key) if create_event: auth_events.append(create_event.event_id) @@ -381,8 +381,8 @@ class Auth(object): else: is_public = False - if event.type == RoomMemberEvent.TYPE: - e_type = event.content["membership"] + if builder.type == RoomMemberEvent.TYPE: + e_type = builder.content["membership"] if e_type in [Membership.JOIN, Membership.INVITE]: if join_rule_event: auth_events.append(join_rule_event.event_id) @@ -393,11 +393,18 @@ class Auth(object): if member_event.content["membership"] == Membership.JOIN: auth_events.append(member_event.event_id) - auth_events = yield self.store.add_event_hashes( - auth_events + auth_ids = [(a.event_id, h) for a, h in auth_events] + auth_events_entries = yield self.store.add_event_hashes( + auth_ids ) - defer.returnValue(auth_events) + builder.auth_events = auth_events_entries + + context.auth_events = { + k: v + for k, v in context.current_state.items() + if v.event_id in auth_ids + } @log_function def _can_send_event(self, event, auth_events): diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index 58edf2bc8f..e81b995d39 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -17,8 +17,8 @@ from frozendict import frozendict def _freeze(o): - if isinstance(o, dict): - return frozendict({k: _freeze(v) for k,v in o.items()}) + if isinstance(o, dict) or isinstance(o, frozendict): + return frozendict({k: _freeze(v) for k, v in o.items()}) if isinstance(o, basestring): return o @@ -31,6 +31,21 @@ def _freeze(o): return o +def _unfreeze(o): + if isinstance(o, frozendict) or isinstance(o, dict): + return dict({k: _unfreeze(v) for k, v in o.items()}) + + if isinstance(o, basestring): + return o + + try: + return [_unfreeze(i) for i in o] + except TypeError: + pass + + return o + + class _EventInternalMetadata(object): def __init__(self, internal_metadata_dict): self.__dict__ = internal_metadata_dict @@ -69,6 +84,7 @@ class EventBase(object): ) auth_events = _event_dict_property("auth_events") + depth = _event_dict_property("depth") content = _event_dict_property("content") event_id = _event_dict_property("event_id") hashes = _event_dict_property("hashes") @@ -81,6 +97,10 @@ class EventBase(object): type = _event_dict_property("type") user_id = _event_dict_property("sender") + @property + def membership(self): + return self.content["membership"] + def is_state(self): return hasattr(self, "state_key") @@ -134,3 +154,14 @@ class FrozenEvent(EventBase): e.internal_metadata = event.internal_metadata return e + + def get_dict(self): + # We need to unfreeze what we return + + d = _unfreeze(self._event_dict) + d.update({ + "signatures": self.signatures, + "unsigned": self.unsigned, + }) + + return d diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 412f690f08..1b05ee0a95 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -14,6 +14,7 @@ # limitations under the License. from synapse.api.constants import EventTypes +from . import EventBase def prune_event(event): @@ -80,3 +81,18 @@ def prune_event(event): allowed_fields["content"] = new_content return type(event)(allowed_fields) + + +def serialize_event(hs, e): + # FIXME(erikj): To handle the case of presence events and the like + if not isinstance(e, EventBase): + return e + + # Should this strip out None's? + d = {k: v for k, v in e.get_dict().items()} + if "age_ts" in d["unsigned"]: + now = int(hs.get_clock().time_msec()) + d["unsigned"]["age"] = now - d["unsigned"]["age_ts"] + del d["unsigned"]["age_ts"] + + return d \ No newline at end of file diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index 4052d0e1e7..810ce138ff 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -62,6 +62,8 @@ class BaseHandler(object): @defer.inlineCallbacks def _create_new_client_event(self, builder): + context = EventContext() + latest_ret = yield self.store.get_latest_events_in_room( builder.room_id, ) @@ -69,34 +71,26 @@ class BaseHandler(object): depth = max([d for _, _, d in latest_ret]) prev_events = [(e, h) for e, h, _ in latest_ret] + builder.prev_events = prev_events + builder.depth = depth + state_handler = self.state_handler + ret = yield state_handler.annotate_context_with_state( + builder, + context, + ) + group, prev_state = ret + if builder.is_state(): - ret = yield state_handler.resolve_state_groups( - [e for e, _ in prev_events], - event_type=builder.event_type, - state_key=builder.state_key, - ) - - group, curr_state, prev_state = ret - prev_state = yield self.store.add_event_hashes( prev_state ) builder.prev_state = prev_state - else: - group, curr_state, _ = yield state_handler.resolve_state_groups( - [e for e, _ in prev_events], - ) builder.internal_metadata.state_group = group - builder.prev_events = prev_events - builder.depth = depth - - auth_events = yield self.auth.get_auth_events(builder, curr_state) - - builder.update_event_key("auth_events", auth_events) + yield self.auth.add_auth_events(builder, context) add_hashes_and_signatures( builder, self.server_name, self.signing_key @@ -104,18 +98,6 @@ class BaseHandler(object): event = builder.build() - auth_ids = zip(*auth_events)[0] - curr_auth_events = { - k: v - for k, v in curr_state.items() - if v.event_id in auth_ids - } - - context = EventContext( - current_state=curr_state, - auth_events=curr_auth_events, - ) - defer.returnValue( (event, context,) ) @@ -128,7 +110,7 @@ class BaseHandler(object): if not suppress_auth: self.auth.check(event, auth_events=context.auth_events) - yield self.store.persist_event(event) + yield self.store.persist_event(event, context=context) destinations = set(extra_destinations) for k, s in context.current_state.items(): @@ -152,63 +134,63 @@ class BaseHandler(object): destinations=destinations, ) - @defer.inlineCallbacks - def _on_new_room_event(self, event, snapshot, extra_destinations=[], - extra_users=[], suppress_auth=False, - do_invite_host=None): - yield run_on_reactor() - - snapshot.fill_out_prev_events(event) - - yield self.state_handler.annotate_event_with_state(event) - - yield self.auth.add_auth_events(event) - - logger.debug("Signing event...") - - add_hashes_and_signatures( - event, self.server_name, self.signing_key - ) - - logger.debug("Signed event.") - - if not suppress_auth: - logger.debug("Authing...") - self.auth.check(event, auth_events=event.old_state_events) - logger.debug("Authed") - else: - logger.debug("Suppressed auth.") - - if do_invite_host: - federation_handler = self.hs.get_handlers().federation_handler - invite_event = yield federation_handler.send_invite( - do_invite_host, - event - ) - - # FIXME: We need to check if the remote changed anything else - event.signatures = invite_event.signatures - - yield self.store.persist_event(event) - - destinations = set(extra_destinations) - # Send a PDU to all hosts who have joined the room. - - for k, s in event.state_events.items(): - try: - if k[0] == RoomMemberEvent.TYPE: - if s.content["membership"] == Membership.JOIN: - destinations.add( - self.hs.parse_userid(s.state_key).domain - ) - except: - logger.warn( - "Failed to get destination from event %s", s.event_id - ) - - event.destinations = list(destinations) - - yield self.notifier.on_new_room_event(event, extra_users=extra_users) - - federation_handler = self.hs.get_handlers().federation_handler - yield federation_handler.handle_new_event(event, snapshot) + # @defer.inlineCallbacks + # def _on_new_room_event(self, event, snapshot, extra_destinations=[], + # extra_users=[], suppress_auth=False, + # do_invite_host=None): + # yield run_on_reactor() + # + # snapshot.fill_out_prev_events(event) + # + # yield self.state_handler.annotate_event_with_state(event) + # + # yield self.auth.add_auth_events(event) + # + # logger.debug("Signing event...") + # + # add_hashes_and_signatures( + # event, self.server_name, self.signing_key + # ) + # + # logger.debug("Signed event.") + # + # if not suppress_auth: + # logger.debug("Authing...") + # self.auth.check(event, auth_events=event.old_state_events) + # logger.debug("Authed") + # else: + # logger.debug("Suppressed auth.") + # + # if do_invite_host: + # federation_handler = self.hs.get_handlers().federation_handler + # invite_event = yield federation_handler.send_invite( + # do_invite_host, + # event + # ) + # + # # FIXME: We need to check if the remote changed anything else + # event.signatures = invite_event.signatures + # + # yield self.store.persist_event(event) + # + # destinations = set(extra_destinations) + # # Send a PDU to all hosts who have joined the room. + # + # for k, s in event.state_events.items(): + # try: + # if k[0] == RoomMemberEvent.TYPE: + # if s.content["membership"] == Membership.JOIN: + # destinations.add( + # self.hs.parse_userid(s.state_key).domain + # ) + # except: + # logger.warn( + # "Failed to get destination from event %s", s.event_id + # ) + # + # event.destinations = list(destinations) + # + # yield self.notifier.on_new_room_event(event, extra_users=extra_users) + # + # federation_handler = self.hs.get_handlers().federation_handler + # yield federation_handler.handle_new_event(event, snapshot) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index b4a28ea3cb..5264e3eafc 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -17,7 +17,8 @@ from ._base import BaseHandler -from synapse.api.events.utils import prune_event +from synapse.events.snapshot import EventContext +from synapse.events.utils import prune_event from synapse.api.errors import ( AuthError, FederationError, SynapseError, StoreError, ) @@ -416,7 +417,7 @@ class FederationHandler(BaseHandler): @defer.inlineCallbacks @log_function - def on_make_join_request(self, context, user_id): + def on_make_join_request(self, room_id, user_id): """ We've received a /make_join/ request, so we create a partial join event for the room and return that. We don *not* persist or process it until the other server has signed it and sent it back. @@ -424,7 +425,7 @@ class FederationHandler(BaseHandler): builder = self.event_builder_factory.new({ "type": RoomMemberEvent.TYPE, "content": {"membership": Membership.JOIN}, - "room_id": context, + "room_id": room_id, "sender": user_id, "state_key": user_id, }) @@ -433,9 +434,7 @@ class FederationHandler(BaseHandler): builder=builder, ) - yield self.state_handler.annotate_event_with_state(event) - yield self.auth.add_auth_events(event) - self.auth.check(event, auth_events=event.old_state_events) + self.auth.check(event, auth_events=context.auth_events) pdu = event @@ -505,7 +504,9 @@ class FederationHandler(BaseHandler): """ event = pdu - event.outlier = True + context = EventContext() + + event.internal_metadata.outlier = True event.signatures.update( compute_event_signature( @@ -515,10 +516,11 @@ class FederationHandler(BaseHandler): ) ) - yield self.state_handler.annotate_event_with_state(event) + yield self.state_handler.annotate_context_with_state(event, context) yield self.store.persist_event( event, + context=context, backfilled=False, ) @@ -640,6 +642,7 @@ class FederationHandler(BaseHandler): @defer.inlineCallbacks def _handle_new_event(self, event, state=None, backfilled=False, current_state=None, fetch_missing=True): + context = EventContext() is_new_state = yield self.state_handler.annotate_event_with_state( event, old_state=state diff --git a/synapse/server.py b/synapse/server.py index 8bc27bbc3c..0d0f3af3f4 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -20,7 +20,7 @@ # Imports required for the default HomeServer() implementation from synapse.federation import initialize_http_replication -from synapse.api.events import serialize_event +from synapse.events.utils import serialize_event from synapse.api.events.factory import EventFactory from synapse.api.events.validator import EventValidator from synapse.notifier import Notifier diff --git a/synapse/state.py b/synapse/state.py index 8a556a27f6..cbb4243fad 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -135,6 +135,39 @@ class StateHandler(object): defer.returnValue(res[1].values()) + @defer.inlineCallbacks + def annotate_context_with_state(self, event, context): + if event.is_state(): + ret = yield self.resolve_state_groups( + [e for e, _ in event.prev_events], + event_type=event.event_type, + state_key=event.state_key, + ) + else: + ret = yield self.resolve_state_groups( + [e for e, _ in event.prev_events], + ) + + group, curr_state, prev_state = ret + + context.current_state = curr_state + + prev_state = yield self.store.add_event_hashes( + prev_state + ) + + if hasattr(event, "auth_events") and event.auth_events: + auth_ids = zip(*event.auth_events)[0] + context.auth_events = { + k: v + for k, v in context.current_state.items() + if v.event_id in auth_ids + } + + defer.returnValue( + (group, prev_state) + ) + @defer.inlineCallbacks @log_function def resolve_state_groups(self, event_ids, event_type=None, state_key=""): diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 205d125642..f172c2690a 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -21,6 +21,7 @@ from synapse.api.events.room import ( ) from synapse.util.logutils import log_function +from synapse.util.frozenutils import FrozenEncoder from .directory import DirectoryStore from .feedback import FeedbackStore @@ -93,8 +94,8 @@ class DataStore(RoomMemberStore, RoomStore, @defer.inlineCallbacks @log_function - def persist_event(self, event, backfilled=False, is_new_state=True, - current_state=None): + def persist_event(self, event, context, backfilled=False, + is_new_state=True, current_state=None): stream_ordering = None if backfilled: if not self.min_token_deferred.called: @@ -107,6 +108,7 @@ class DataStore(RoomMemberStore, RoomStore, "persist_event", self._persist_event_txn, event=event, + context=context, backfilled=backfilled, stream_ordering=stream_ordering, is_new_state=is_new_state, @@ -138,8 +140,9 @@ class DataStore(RoomMemberStore, RoomStore, defer.returnValue(event[0]) @log_function - def _persist_event_txn(self, txn, event, backfilled, stream_ordering=None, - is_new_state=True, current_state=None): + def _persist_event_txn(self, txn, event, context, backfilled, + stream_ordering=None, is_new_state=True, + current_state=None): if event.type == RoomMemberEvent.TYPE: self._store_room_member_txn(txn, event) elif event.type == FeedbackEvent.TYPE: @@ -152,12 +155,12 @@ class DataStore(RoomMemberStore, RoomStore, self._store_redaction(txn, event) outlier = False - if hasattr(event, "outlier"): - outlier = event.outlier + if hasattr(event.internal_metadata, "outlier"): + outlier = event.internal_metadata.outlier event_dict = { k: v - for k, v in event.get_full_dict().items() + for k, v in event.get_dict().items() if k not in [ "redacted", "redacted_because", @@ -179,7 +182,7 @@ class DataStore(RoomMemberStore, RoomStore, "event_id": event.event_id, "type": event.type, "room_id": event.room_id, - "content": json.dumps(event.content), + "content": json.dumps(event.content, cls=FrozenEncoder), "processed": True, "outlier": outlier, "depth": event.depth, @@ -190,7 +193,7 @@ class DataStore(RoomMemberStore, RoomStore, unrec = { k: v - for k, v in event.get_full_dict().items() + for k, v in event.get_dict().items() if k not in vals.keys() and k not in [ "redacted", "redacted_because", @@ -225,7 +228,7 @@ class DataStore(RoomMemberStore, RoomStore, room_id=event.room_id, ) - self._store_state_groups_txn(txn, event) + self._store_state_groups_txn(txn, event, context) if current_state: txn.execute( diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index bb61c20150..c56c3a0b0f 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -15,7 +15,8 @@ import logging from synapse.api.errors import StoreError -from synapse.api.events.utils import prune_event +from synapse.events import FrozenEvent +from synapse.events.utils import prune_event from synapse.util.logutils import log_function from synapse.util.logcontext import PreserveLoggingContext, LoggingContext from syutil.base64util import encode_base64 @@ -497,10 +498,7 @@ class SQLBaseStore(object): d = json.loads(js) - ev = self.event_factory.create_event( - etype=d["type"], - **d - ) + ev = FrozenEvent(d) if hasattr(ev, "redacted") and ev.redacted: # Get the redaction event. diff --git a/synapse/storage/state.py b/synapse/storage/state.py index e0f44b3e59..b8e721ad72 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -86,11 +86,16 @@ class StateStore(SQLBaseStore): self._store_state_groups_txn, event ) - def _store_state_groups_txn(self, txn, event): - if event.state_events is None: + def _store_state_groups_txn(self, txn, event, context): + if context.current_state_events is None: return - state_group = event.state_group + state_events = context.current_state_events + + if event.is_state(): + state_events[(event.type, event.state_key)] = event + + state_group = context.state_group if not state_group: state_group = self._simple_insert_txn( txn, @@ -102,7 +107,7 @@ class StateStore(SQLBaseStore): or_ignore=True, ) - for state in event.state_events.values(): + for state in context.state_events.values(): self._simple_insert_txn( txn, table="state_groups_state", From 05e48c5d4ba9e446896f6fba3483826952758f46 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 5 Dec 2014 16:29:36 +0000 Subject: [PATCH 017/158] Add pillow to dependencies --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index d0d649612d..249819cafd 100755 --- a/setup.py +++ b/setup.py @@ -41,6 +41,7 @@ setup( "pynacl", "daemonize", "py-bcrypt", + "pillow", ], dependency_links=[ "https://github.com/matrix-org/syutil/tarball/v0.0.2#egg=syutil-0.0.2", From a953be097f3431481875dc04662adcc5ba2226a9 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 5 Dec 2014 16:30:18 +0000 Subject: [PATCH 018/158] Add a method field to thumbnail storage --- synapse/storage/media_repository.py | 15 +++++++++------ synapse/storage/schema/media_repository.sql | 4 +++- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/synapse/storage/media_repository.py b/synapse/storage/media_repository.py index 2d3a2d1ccb..b3f1fc087b 100644 --- a/synapse/storage/media_repository.py +++ b/synapse/storage/media_repository.py @@ -50,20 +50,21 @@ class MediaRepositoryStore(SQLBaseStore): "local_media_thumbnails", {"media_id": media_id}, ( - "thumbnail_width", "thumbnail_height", + "thumbnail_width", "thumbnail_height", "thumbnail_method", "thumbnail_type", "thumbnail_length", ) ) def store_local_thumbnail(self, media_id, thumbnail_width, - thumbnail_height, thumbnail_type, - thumbnail_length): + thumbnail_height, thumbnail_method, + thumbnail_type, thumbnail_length): return self._simple_insert( "local_media_thumbnails", { "media_id": media_id, "thumbnail_width": thumbnail_width, "thumbnail_height": thumbnail_height, + "thumbnail_method": thumbnail_method, "thumbnail_type": thumbnail_type, "thumbnail_length": thumbnail_length, } @@ -101,15 +102,16 @@ class MediaRepositoryStore(SQLBaseStore): "remote_media_cache_thumbnails", {"origin": origin, "media_id": media_id}, ( - "thumbnail_width", "thumbnail_height", + "thumbnail_width", "thumbnail_height", "thumbnail_method" "thumbnail_type", "thumbnail_length", "filesystem_id" ) ) def store_remote_media_thumbnail(self, origin, media_id, thumbnail_width, - thumbnail_height, thumbnail_type, - thumbnail_length, filesystem_id): + thumbnail_height, thumbnail_method, + thumbnail_type, thumbnail_length, + filesystem_id): return self._simple_insert( "remote_media_cache_thumbnails", { @@ -117,6 +119,7 @@ class MediaRepositoryStore(SQLBaseStore): "media_id": media_id, "thumbnail_width": thumbnail_width, "thumbnail_height": thumbnail_height, + "thumbnail_method": thumbnail_method, "thumbnail_type": thumbnail_type, "thumbnail_length": thumbnail_length, "filesystem_id": filesystem_id, diff --git a/synapse/storage/schema/media_repository.sql b/synapse/storage/schema/media_repository.sql index 768752296f..b785fa0208 100644 --- a/synapse/storage/schema/media_repository.sql +++ b/synapse/storage/schema/media_repository.sql @@ -28,6 +28,7 @@ CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails ( thumbnail_width INTEGER, -- The width of the thumbnail in pixels. thumbnail_height INTEGER, -- The height of the thumbnail in pixels. thumbnail_type TEXT, -- The MIME-type of the thumbnail. + thumbnail_method TEXT, -- The method used to make the thumbnail. thumbnail_length INTEGER, -- The length of the thumbnail in bytes. CONSTRAINT uniqueness UNIQUE ( media_id, thumbnail_width, thumbnail_height, thumbnail_type @@ -53,12 +54,13 @@ CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails ( media_id TEXT, -- The id used to refer to the media. thumbnail_width INTEGER, -- The width of the thumbnail in pixels. thumbnail_height INTEGER, -- The height of the thumbnail in pixels. + thumbnail_method TEXT, -- The method used to make the thumbnail thumbnail_type TEXT, -- The MIME-type of the thumbnail. thumbnail_length INTEGER, -- The length of the thumbnail in bytes. filesystem_id TEXT, -- The name used to store the media on disk. CONSTRAINT uniqueness UNIQUE ( media_origin, media_id, thumbnail_width, thumbnail_height, - thumbnail_type + thumbnail_type, thumbnail_type ) ); From 63b0b946be3df331ab4a3994e87d2318914c37c7 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 5 Dec 2014 18:01:05 +0000 Subject: [PATCH 019/158] point the entry_point for synapse-homeserver at the right method --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8358092e30..9bee64e368 100755 --- a/setup.py +++ b/setup.py @@ -59,6 +59,6 @@ setup( entry_points=""" [console_scripts] synctl=synapse.app.synctl:main - synapse-homeserver=synapse.app.homeserver:run + synapse-homeserver=synapse.app.homeserver:main """ ) From aed62a35832a3ec1c7425ecc99cab06a781263ba Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sun, 7 Dec 2014 02:26:07 +0000 Subject: [PATCH 020/158] track replication destination health, and perform exponential back-off when sending transactions. does *not* yet retry transactions, but drops them on the floor if waiting for a server to recover. --- synapse/federation/replication.py | 43 +++++++++++--- synapse/federation/transport.py | 2 +- synapse/http/matrixfederationclient.py | 16 +++-- synapse/rest/transactions.py | 2 +- synapse/storage/__init__.py | 2 +- synapse/storage/_base.py | 2 +- synapse/storage/schema/delta/v9.sql | 23 ++++++++ synapse/storage/schema/transactions.sql | 6 ++ synapse/storage/transactions.py | 78 ++++++++++++++++++++++++- 9 files changed, 156 insertions(+), 18 deletions(-) create mode 100644 synapse/storage/schema/delta/v9.sql diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index 01f87fe423..f9c05b5ea3 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -723,6 +723,8 @@ class _TransactionQueue(object): deferreds = [] for destination in destinations: + # XXX: why don't we specify an errback for this deferred + # like we do for EDUs? --matthew deferred = defer.Deferred() self.pending_pdus_by_dest.setdefault(destination, []).append( (pdu, deferred, order) @@ -738,6 +740,9 @@ class _TransactionQueue(object): # NO inlineCallbacks def enqueue_edu(self, edu): destination = edu.destination + + if destination == self.server_name: + return deferred = defer.Deferred() self.pending_edus_by_dest.setdefault(destination, []).append( @@ -766,14 +771,23 @@ class _TransactionQueue(object): ) yield deferred - + @defer.inlineCallbacks @log_function def _attempt_new_transaction(self, destination): + + (retry_last_ts, retry_interval) = self.store.get_destination_retry_timings(destination) + if retry_last_ts + retry_interval > int(self._clock.time_msec()): + logger.info("TX [%s] not ready for retry yet - dropping transaction for now") + return + if destination in self.pending_transactions: + # XXX: pending_transactions can get stuck on by a never-ending request + # at which point pending_pdus_by_dest just keeps growing. + # we need application-layer timeouts of some flavour of these requests return - # list of (pending_pdu, deferred, order) + # list of (pending_pdu, deferred, order) pending_pdus = self.pending_pdus_by_dest.pop(destination, []) pending_edus = self.pending_edus_by_dest.pop(destination, []) pending_failures = self.pending_failures_by_dest.pop(destination, []) @@ -781,7 +795,8 @@ class _TransactionQueue(object): if not pending_pdus and not pending_edus and not pending_failures: return - logger.debug("TX [%s] Attempting new transaction", destination) + logger.debug("TX [%s] Attempting new transaction (pdus: %d, edus: %d, failures: %d)", + destination, len(pending_pdus), len(pending_edus), len(pending_failures)) # Sort based on the order field pending_pdus.sort(key=lambda t: t[2]) @@ -814,7 +829,7 @@ class _TransactionQueue(object): yield self.transaction_actions.prepare_to_send(transaction) logger.debug("TX [%s] Persisted transaction", destination) - logger.debug("TX [%s] Sending transaction...", destination) + logger.info("TX [%s] Sending transaction [%s]", destination, transaction.transaction_id) # Actually send the transaction @@ -835,6 +850,8 @@ class _TransactionQueue(object): transaction, json_data_cb ) + logger.info("TX [%s] got %d response", destination, code) + logger.debug("TX [%s] Sent transaction", destination) logger.debug("TX [%s] Marking as delivered...", destination) @@ -849,6 +866,7 @@ class _TransactionQueue(object): if code == 200: deferred.callback(None) else: + start_retrying(destination, retry_interval) deferred.errback(RuntimeError("Got status %d" % code)) # Ensures we don't continue until all callbacks on that @@ -861,12 +879,12 @@ class _TransactionQueue(object): logger.debug("TX [%s] Yielded to callbacks", destination) except Exception as e: - logger.error("TX Problem in _attempt_transaction") - # We capture this here as there as nothing actually listens # for this finishing functions deferred. - logger.exception(e) + logger.exception("TX [%s] Problem in _attempt_transaction: %s", destination, e) + start_retrying(destination, retry_interval) + for deferred in deferreds: if not deferred.called: deferred.errback(e) @@ -877,3 +895,14 @@ class _TransactionQueue(object): # Check to see if there is anything else to send. self._attempt_new_transaction(destination) + +def start_retrying(destination, retry_interval): + # track that this destination is having problems and we should + # give it a chance to recover before trying it again + if retry_interval: + retry_interval *= 2 + else: + retry_interval = 2 # try again at first after 2 seconds + self.store.set_destination_retry_timings(destination, + int(self._clock.time_msec()), retry_interval) + \ No newline at end of file diff --git a/synapse/federation/transport.py b/synapse/federation/transport.py index 8d86152085..0f11c6d491 100644 --- a/synapse/federation/transport.py +++ b/synapse/federation/transport.py @@ -155,7 +155,7 @@ class TransportLayer(object): @defer.inlineCallbacks @log_function def send_transaction(self, transaction, json_data_callback=None): - """ Sends the given Transaction to it's destination + """ Sends the given Transaction to its destination Args: transaction (Transaction) diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 510f07dd7b..3edc59dbab 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -89,7 +89,7 @@ class MatrixFederationHttpClient(object): ("", "", path_bytes, param_bytes, query_bytes, "",) ) - logger.debug("Sending request to %s: %s %s", + logger.info("Sending request to %s: %s %s", destination, method, url_bytes) logger.debug( @@ -101,7 +101,10 @@ class MatrixFederationHttpClient(object): ] ) - retries_left = 5 + # was 5; for now, let's only try once at the HTTP layer and then + # rely on transaction-layer retries for exponential backoff and + # getting the message through. + retries_left = 0 endpoint = self._getEndpoint(reactor, destination) @@ -131,7 +134,8 @@ class MatrixFederationHttpClient(object): e) raise SynapseError(400, "Domain specified not found.") - logger.exception("Got error in _create_request") + logger.exception("Sending request failed to %s: %s %s : %s", + destination, method, url_bytes, e) _print_ex(e) if retries_left: @@ -140,15 +144,15 @@ class MatrixFederationHttpClient(object): else: raise + logger.info("Received response %d %s for %s: %s %s", + response.code, response.phrase, destination, method, url_bytes) + if 200 <= response.code < 300: # We need to update the transactions table to say it was sent? pass else: # :'( # Update transactions table? - logger.error( - "Got response %d %s", response.code, response.phrase - ) raise CodeMessageException( response.code, response.phrase ) diff --git a/synapse/rest/transactions.py b/synapse/rest/transactions.py index 93c0122f30..8c41ab4edb 100644 --- a/synapse/rest/transactions.py +++ b/synapse/rest/transactions.py @@ -19,7 +19,7 @@ import logging logger = logging.getLogger(__name__) - +# FIXME: elsewhere we use FooStore to indicate something in the storage layer... class HttpTransactionStore(object): def __init__(self): diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index f15e3dfe62..04ab39341d 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -67,7 +67,7 @@ SCHEMAS = [ # Remember to update this number every time an incompatible change is made to # database schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 8 +SCHEMA_VERSION = 9 class _RollbackButIsFineException(Exception): diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 4881f03368..e72200e2f7 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -650,7 +650,7 @@ class JoinHelper(object): to dump the results into. Attributes: - taples (list): List of `Table` classes + tables (list): List of `Table` classes EntryType (type) """ diff --git a/synapse/storage/schema/delta/v9.sql b/synapse/storage/schema/delta/v9.sql new file mode 100644 index 0000000000..ad680c64da --- /dev/null +++ b/synapse/storage/schema/delta/v9.sql @@ -0,0 +1,23 @@ +/* Copyright 2014 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- To track destination health +CREATE TABLE IF NOT EXISTS destinations( + destination TEXT PRIMARY KEY, + retry_last_ts INTEGER, + retry_interval INTEGER +); + +PRAGMA user_version = 9; \ No newline at end of file diff --git a/synapse/storage/schema/transactions.sql b/synapse/storage/schema/transactions.sql index 88e3e4e04d..de461bfa15 100644 --- a/synapse/storage/schema/transactions.sql +++ b/synapse/storage/schema/transactions.sql @@ -59,3 +59,9 @@ CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_tx ON transaction_id_to_pdu(tra CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination); CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_index ON transaction_id_to_pdu(transaction_id, destination); +-- To track destination health +CREATE TABLE IF NOT EXISTS destinations( + destination TEXT PRIMARY KEY, + retry_last_ts INTEGER, + retry_interval INTEGER +); diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index 00d0f48082..47b73f7458 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -114,7 +114,7 @@ class TransactionStore(SQLBaseStore): def _prep_send_transaction(self, txn, transaction_id, destination, origin_server_ts): - # First we find out what the prev_txs should be. + # First we find out what the prev_txns should be. # Since we know that we are only sending one transaction at a time, # we can simply take the last one. query = "%s ORDER BY id DESC LIMIT 1" % ( @@ -205,6 +205,71 @@ class TransactionStore(SQLBaseStore): return ReceivedTransactionsTable.decode_results(txn.fetchall()) + def get_destination_retry_timings(self, destination): + """Gets the current retry timings (if any) for a given destination. + + Args: + destination (str) + + Returns: + None if not retrying + tuple: (retry_last_ts, retry_interval) + retry_ts: time of last retry attempt in unix epoch ms + retry_interval: how long until next retry in ms + """ + return self.runInteraction( + "get_destination_retry_timings", + self._get_destination_retry_timings, destination) + + def _get_destination_retry_timings(cls, txn, destination): + query = DestinationsTable.select_statement("destination = ?") + txn.execute(query, (destination,)) + result = DestinationsTable.decode_single_result(txn.fetchone()) + if result and result[0] > 0: + return result + else: + return None + + def set_destination_retry_timings(self, destination): + """Sets the current retry timings for a given destination. + Both timings should be zero if retrying is no longer occuring. + + Args: + destination (str) + retry_last_ts (int) - time of last retry attempt in unix epoch ms + retry_interval (int) - how long until next retry in ms + """ + return self.runInteraction( + "set_destination_retry_timings", + self._set_destination_retry_timings, destination, retry_last_ts, retry_interval) + + def _set_destination_retry_timings(cls, txn, destination, retry_last_ts, retry_interval): + + query = ( + "INSERT OR REPLACE INTO %s " + "(retry_last_ts, retry_interval) " + "VALUES (?, ?) " + "WHERE destination = ?" + ) % DestinationsTable.table_name + + txn.execute(query, (retry_last_ts, retry_interval, destination)) + + def get_destinations_needing_retry(self): + """Get all destinations which are due a retry for sending a transaction. + + Returns: + list: A list of `DestinationsTable.EntryType` + """ + return self.runInteraction( + "get_destinations_needing_retry", + self._get_destinations_needing_retry + ) + + def _get_destinations_needing_retry(cls, txn): + where = "retry_last_ts > 0 and retry_next_ts < now()" + query = DestinationsTable.select_statement(where) + txn.execute(query) + return DestinationsTable.decode_results(txn.fetchall()) class ReceivedTransactionsTable(Table): table_name = "received_transactions" @@ -247,3 +312,14 @@ class TransactionsToPduTable(Table): ] EntryType = namedtuple("TransactionsToPduEntry", fields) + +class DestinationsTable(Table): + table_name = "destinations" + + fields = [ + "destination", + "retry_last_ts", + "retry_interval", + ] + + EntryType = namedtuple("DestinationsEntry", fields) \ No newline at end of file From 5cd43d4b9f3c41b21ced0ab44cf24c2cf7dab817 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sun, 7 Dec 2014 23:44:16 +0000 Subject: [PATCH 021/158] fix stupid syntax thinkos --- synapse/federation/replication.py | 23 +++++++++++------------ synapse/storage/transactions.py | 2 +- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index f9c05b5ea3..1b9e3ece09 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -866,7 +866,7 @@ class _TransactionQueue(object): if code == 200: deferred.callback(None) else: - start_retrying(destination, retry_interval) + self.start_retrying(destination, retry_interval) deferred.errback(RuntimeError("Got status %d" % code)) # Ensures we don't continue until all callbacks on that @@ -883,7 +883,7 @@ class _TransactionQueue(object): # for this finishing functions deferred. logger.exception("TX [%s] Problem in _attempt_transaction: %s", destination, e) - start_retrying(destination, retry_interval) + self.start_retrying(destination, retry_interval) for deferred in deferreds: if not deferred.called: @@ -896,13 +896,12 @@ class _TransactionQueue(object): # Check to see if there is anything else to send. self._attempt_new_transaction(destination) -def start_retrying(destination, retry_interval): - # track that this destination is having problems and we should - # give it a chance to recover before trying it again - if retry_interval: - retry_interval *= 2 - else: - retry_interval = 2 # try again at first after 2 seconds - self.store.set_destination_retry_timings(destination, - int(self._clock.time_msec()), retry_interval) - \ No newline at end of file + def start_retrying(self, destination, retry_interval): + # track that this destination is having problems and we should + # give it a chance to recover before trying it again + if retry_interval: + retry_interval *= 2 + else: + retry_interval = 2 # try again at first after 2 seconds + self.store.set_destination_retry_timings(destination, + int(self._clock.time_msec()), retry_interval) diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index 47b73f7458..cacd948302 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -230,7 +230,7 @@ class TransactionStore(SQLBaseStore): else: return None - def set_destination_retry_timings(self, destination): + def set_destination_retry_timings(self, destination, retry_last_ts, retry_interval): """Sets the current retry timings for a given destination. Both timings should be zero if retrying is no longer occuring. From 9c43b258ecc493b126ef2858b9bb8fda0f01478a Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Mon, 8 Dec 2014 00:17:12 +0000 Subject: [PATCH 022/158] actually reset retry schedule if we can successfuly talk to it --- synapse/federation/replication.py | 3 +++ synapse/handlers/federation.py | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index 1b9e3ece09..88184caecd 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -864,6 +864,9 @@ class _TransactionQueue(object): for deferred in deferreds: if code == 200: + if retry_last_ts: + # this host is alive! reset retry schedule + self.store.set_destination_retry_timings(destination, 0, 0) deferred.callback(None) else: self.start_retrying(destination, retry_interval) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 925eb5376e..7a79e2d117 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -207,6 +207,12 @@ class FederationHandler(BaseHandler): e.msg, affected=event.event_id, ) + + # if we're receiving valid events from an origin, + # it's probably a good idea to mark it as not in retry-state + # for sending (although this is a bit of a leap) + if ((self.store.get_destination_retry_timings(origin))[0]): + self.store.set_destination_retry_timings(origin, 0, 0) room = yield self.store.get_room(event.room_id) From d044121168672c657e595525af9b588c8769e9bb Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 8 Dec 2014 09:08:26 +0000 Subject: [PATCH 023/158] Various typos and bug fixes. --- synapse/api/auth.py | 16 ++--- synapse/events/snapshot.py | 3 +- synapse/events/utils.py | 2 + synapse/federation/replication.py | 12 +++- synapse/handlers/_base.py | 8 ++- synapse/handlers/federation.py | 97 ++++++++++++------------------- synapse/state.py | 16 +++-- synapse/storage/state.py | 6 +- 8 files changed, 80 insertions(+), 80 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 3f2e58a5ef..821e3ba5e2 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -24,6 +24,7 @@ from synapse.api.events.room import ( RoomJoinRulesEvent, RoomCreateEvent, RoomAliasesEvent, ) from synapse.util.logutils import log_function +from synapse.util.async import run_on_reactor from syutil.base64util import encode_base64 import logging @@ -352,17 +353,19 @@ class Auth(object): @defer.inlineCallbacks def add_auth_events(self, builder, context): + yield run_on_reactor() + if builder.type == RoomCreateEvent.TYPE: builder.auth_events = [] return - auth_events = [] + auth_ids = [] key = (RoomPowerLevelsEvent.TYPE, "", ) power_level_event = context.current_state.get(key) if power_level_event: - auth_events.append(power_level_event.event_id) + auth_ids.append(power_level_event.event_id) key = (RoomJoinRulesEvent.TYPE, "", ) join_rule_event = context.current_state.get(key) @@ -373,7 +376,7 @@ class Auth(object): key = (RoomCreateEvent.TYPE, "", ) create_event = context.current_state.get(key) if create_event: - auth_events.append(create_event.event_id) + auth_ids.append(create_event.event_id) if join_rule_event: join_rule = join_rule_event.content.get("join_rule") @@ -385,15 +388,14 @@ class Auth(object): e_type = builder.content["membership"] if e_type in [Membership.JOIN, Membership.INVITE]: if join_rule_event: - auth_events.append(join_rule_event.event_id) + auth_ids.append(join_rule_event.event_id) if member_event and not is_public: - auth_events.append(member_event.event_id) + auth_ids.append(member_event.event_id) elif member_event: if member_event.content["membership"] == Membership.JOIN: - auth_events.append(member_event.event_id) + auth_ids.append(member_event.event_id) - auth_ids = [(a.event_id, h) for a, h in auth_events] auth_events_entries = yield self.store.add_event_hashes( auth_ids ) diff --git a/synapse/events/snapshot.py b/synapse/events/snapshot.py index ca15ec09ae..e0cbacc19c 100644 --- a/synapse/events/snapshot.py +++ b/synapse/events/snapshot.py @@ -58,6 +58,7 @@ class EventCache(object): class EventContext(object): - def __init__(self, current_state, auth_events): + def __init__(self, current_state=None, auth_events=None): self.current_state = current_state self.auth_events = auth_events + self.state_group = None diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 1b05ee0a95..485f075406 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -95,4 +95,6 @@ def serialize_event(hs, e): d["unsigned"]["age"] = now - d["unsigned"]["age_ts"] del d["unsigned"]["age_ts"] + d["user_id"] = d.pop("sender", None) + return d \ No newline at end of file diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index b11df9e5c6..3af24ee46d 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -558,7 +558,13 @@ class ReplicationLayer(object): origin, pdu.event_id, do_auth=False ) - if existing and (not existing.outlier or pdu.outlier): + already_seen = ( + existing and ( + not existing.internal_metadata.outlier + or pdu.internal_metadata.outlier + ) + ) + if already_seen: logger.debug("Already seen pdu %s", pdu.event_id) defer.returnValue({}) return @@ -596,7 +602,7 @@ class ReplicationLayer(object): # ) # Get missing pdus if necessary. - if not pdu.outlier: + if not pdu.internal_metadata.outlier: # We only backfill backwards to the min depth. min_depth = yield self.handler.get_min_depth_for_context( pdu.room_id @@ -663,7 +669,7 @@ class ReplicationLayer(object): pdu_json ) - builder.internal_metadata = outlier + builder.internal_metadata.outlier = outlier return builder.build() diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index 810ce138ff..0bff644192 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -62,6 +62,8 @@ class BaseHandler(object): @defer.inlineCallbacks def _create_new_client_event(self, builder): + yield run_on_reactor() + context = EventContext() latest_ret = yield self.store.get_latest_events_in_room( @@ -79,7 +81,7 @@ class BaseHandler(object): builder, context, ) - group, prev_state = ret + prev_state = ret if builder.is_state(): prev_state = yield self.store.add_event_hashes( @@ -88,8 +90,6 @@ class BaseHandler(object): builder.prev_state = prev_state - builder.internal_metadata.state_group = group - yield self.auth.add_auth_events(builder, context) add_hashes_and_signatures( @@ -105,6 +105,8 @@ class BaseHandler(object): @defer.inlineCallbacks def handle_new_client_event(self, event, context, extra_destinations=[], extra_users=[], suppress_auth=False): + yield run_on_reactor() + # We now need to go and hit out to wherever we need to hit out to. if not suppress_auth: diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 5264e3eafc..38ee32d26e 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -149,7 +149,7 @@ class FederationHandler(BaseHandler): event.room_id, self.server_name ) - if not is_in_room and not event.outlier: + if not is_in_room and not event.internal_metadata.outlier: logger.debug("Got event for room we're not in.") replication_layer = self.replication_layer @@ -160,7 +160,7 @@ class FederationHandler(BaseHandler): ) for e in auth_chain: - e.outlier = True + e.internal_metadata.outlier = True try: yield self._handle_new_event(e, fetch_missing=False) except: @@ -180,7 +180,7 @@ class FederationHandler(BaseHandler): if state: for e in state: - e.outlier = True + e.internal_metadata.outlier = True try: yield self._handle_new_event(e) except: @@ -254,11 +254,18 @@ class FederationHandler(BaseHandler): event = pdu # FIXME (erikj): Not sure this actually works :/ - yield self.state_handler.annotate_event_with_state(event) + context = EventContext() + yield self.state_handler.annotate_context_with_state(event, context) - events.append(event) + events.append( + (event, context) + ) - yield self.store.persist_event(event, backfilled=True) + yield self.store.persist_event( + event, + context=context, + backfilled=True + ) defer.returnValue(events) @@ -326,7 +333,7 @@ class FederationHandler(BaseHandler): assert(event.state_key == joinee) assert(event.room_id == room_id) - event.outlier = False + event.internal_metadata.outlier = False self.room_queues[room_id] = [] @@ -369,7 +376,7 @@ class FederationHandler(BaseHandler): pass for e in auth_chain: - e.outlier = True + e.internal_metadata.outlier = True try: yield self._handle_new_event(e, fetch_missing=False) except: @@ -380,7 +387,7 @@ class FederationHandler(BaseHandler): for e in state: # FIXME: Auth these. - e.outlier = True + e.internal_metadata.outlier = True try: yield self._handle_new_event( e, @@ -448,7 +455,7 @@ class FederationHandler(BaseHandler): """ event = pdu - event.outlier = False + event.internal_metadata.outlier = False yield self._handle_new_event(event) @@ -643,70 +650,42 @@ class FederationHandler(BaseHandler): def _handle_new_event(self, event, state=None, backfilled=False, current_state=None, fetch_missing=True): context = EventContext() - is_new_state = yield self.state_handler.annotate_event_with_state( + yield self.state_handler.annotate_context_with_state( event, old_state=state ) - if event.old_state_events: - known_ids = set( - [s.event_id for s in event.old_state_events.values()] - ) - for e_id, _ in event.auth_events: - if e_id not in known_ids: - e = yield self.store.get_event( - e_id, - allow_none=True, - ) + is_new_state = not event.internal_metadata.outlier - if not e: - # TODO: Do some conflict res to make sure that we're - # not the ones who are wrong. - logger.info( - "Rejecting %s as %s not in %s", - event.event_id, e_id, known_ids, - ) - raise AuthError(403, "Auth events are stale") - - auth_events = event.old_state_events - else: - # We need to get the auth events from somewhere. - - # TODO: Don't just hit the DBs? - - auth_events = {} - for e_id, _ in event.auth_events: + known_ids = set( + [s.event_id for s in context.auth_events.values()] + ) + for e_id, _ in event.auth_events: + if e_id not in known_ids: e = yield self.store.get_event( e_id, + context, allow_none=True, ) if not e: - e = yield self.replication_layer.get_pdu( - event.origin, e_id, outlier=True + # TODO: Do some conflict res to make sure that we're + # not the ones who are wrong. + logger.info( + "Rejecting %s as %s not in %s", + event.event_id, e_id, known_ids, ) + raise AuthError(403, "Auth events are stale") - if e and fetch_missing: - try: - yield self.on_receive_pdu(event.origin, e, False) - except: - logger.exception( - "Failed to parse auth event %s", - e_id, - ) + context.auth_events[(e.type, e.state_key)] = e - if not e: - logger.warn("Can't find auth event %s.", e_id) + if event.type == RoomMemberEvent.TYPE and not event.auth_events: + if len(event.prev_events) == 1: + c = yield self.store.get_event(event.prev_events[0][0]) + if c.type == RoomCreateEvent.TYPE: + context.auth_events[(c.type, c.state_key)] = c - auth_events[(e.type, e.state_key)] = e - - if event.type == RoomMemberEvent.TYPE and not event.auth_events: - if len(event.prev_events) == 1: - c = yield self.store.get_event(event.prev_events[0][0]) - if c.type == RoomCreateEvent.TYPE: - auth_events[(c.type, c.state_key)] = c - - self.auth.check(event, auth_events=auth_events) + self.auth.check(event, auth_events=context.auth_events) yield self.store.persist_event( event, diff --git a/synapse/state.py b/synapse/state.py index cbb4243fad..464cbae564 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -136,7 +136,16 @@ class StateHandler(object): defer.returnValue(res[1].values()) @defer.inlineCallbacks - def annotate_context_with_state(self, event, context): + def annotate_context_with_state(self, event, context, old_state=None): + yield run_on_reactor() + + if old_state: + context.current_state = { + (s.type, s.state_key): s for s in old_state + } + context.state_group = None + defer.returnValue([]) + if event.is_state(): ret = yield self.resolve_state_groups( [e for e, _ in event.prev_events], @@ -151,6 +160,7 @@ class StateHandler(object): group, curr_state, prev_state = ret context.current_state = curr_state + context.state_group = group prev_state = yield self.store.add_event_hashes( prev_state @@ -164,9 +174,7 @@ class StateHandler(object): if v.event_id in auth_ids } - defer.returnValue( - (group, prev_state) - ) + defer.returnValue(prev_state) @defer.inlineCallbacks @log_function diff --git a/synapse/storage/state.py b/synapse/storage/state.py index b8e721ad72..afe3e5edea 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -87,10 +87,10 @@ class StateStore(SQLBaseStore): ) def _store_state_groups_txn(self, txn, event, context): - if context.current_state_events is None: + if context.current_state is None: return - state_events = context.current_state_events + state_events = context.current_state if event.is_state(): state_events[(event.type, event.state_key)] = event @@ -107,7 +107,7 @@ class StateStore(SQLBaseStore): or_ignore=True, ) - for state in context.state_events.values(): + for state in state_events.values(): self._simple_insert_txn( txn, table="state_groups_state", From 721482c83e2b08200d5a8976dd85d45b04d8b39a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 8 Dec 2014 09:10:12 +0000 Subject: [PATCH 024/158] Add forgotten file --- synapse/util/frozenutils.py | 56 +++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 synapse/util/frozenutils.py diff --git a/synapse/util/frozenutils.py b/synapse/util/frozenutils.py new file mode 100644 index 0000000000..46d3ab4f7d --- /dev/null +++ b/synapse/util/frozenutils.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from frozendict import frozendict + +import json + + +def freeze(o): + if isinstance(o, dict) or isinstance(o, frozendict): + return frozendict({k: freeze(v) for k, v in o.items()}) + + if isinstance(o, basestring): + return o + + try: + return tuple([freeze(i) for i in o]) + except TypeError: + pass + + return o + + +def unfreeze(o): + if isinstance(o, frozendict) or isinstance(o, dict): + return dict({k: unfreeze(v) for k, v in o.items()}) + + if isinstance(o, basestring): + return o + + try: + return [unfreeze(i) for i in o] + except TypeError: + pass + + return o + + +class FrozenEncoder(json.JSONEncoder): + def default(self, o): + if isinstance(o, frozendict): + return dict(o) + + return json.JSONEncoder(self, o) \ No newline at end of file From d45f28f8bde9f45c006f6d74d24f29fca95e9b91 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 8 Dec 2014 09:24:08 +0000 Subject: [PATCH 025/158] Ignore pycharm dir --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 339a99e0d6..af90668c89 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,5 @@ graph/*.dot **/webclient/test/environment-protractor.js uploads + +.idea/ From a295a3c6910b31ec45e42084b91d22ca30da4bde Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 8 Dec 2014 09:24:37 +0000 Subject: [PATCH 026/158] Fix registration --- synapse/handlers/register.py | 6 ++++-- synapse/rest/register.py | 4 +++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 48c326ebf0..15d8716455 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -22,6 +22,7 @@ from synapse.api.errors import ( ) from ._base import BaseHandler import synapse.util.stringutils as stringutils +from synapse.util.async import run_on_reactor from synapse.http.client import SimpleHttpClient from synapse.http.client import CaptchaServerHttpClient @@ -54,12 +55,13 @@ class RegistrationHandler(BaseHandler): Raises: RegistrationError if there was a problem registering. """ + yield run_on_reactor() password_hash = None if password: password_hash = bcrypt.hashpw(password, bcrypt.gensalt()) if localpart: - user = UserID(localpart, self.hs.hostname, True) + user = UserID(localpart, self.hs.hostname) user_id = user.to_string() token = self._generate_token(user_id) @@ -78,7 +80,7 @@ class RegistrationHandler(BaseHandler): while not user_id and not token: try: localpart = self._generate_user_id() - user = UserID(localpart, self.hs.hostname, True) + user = UserID(localpart, self.hs.hostname) user_id = user.to_string() token = self._generate_token(user_id) diff --git a/synapse/rest/register.py b/synapse/rest/register.py index f25e23a158..4f0f5a7531 100644 --- a/synapse/rest/register.py +++ b/synapse/rest/register.py @@ -21,6 +21,8 @@ from synapse.api.constants import LoginType from base import RestServlet, client_path_pattern import synapse.util.stringutils as stringutils +from synapse.util.async import run_on_reactor + from hashlib import sha1 import hmac import json @@ -233,7 +235,7 @@ class RegisterRestServlet(RestServlet): @defer.inlineCallbacks def _do_password(self, request, register_json, session): - yield + yield run_on_reactor() if (self.hs.config.enable_registration_captcha and not session[LoginType.RECAPTCHA]): # captcha should've been done by this stage! From e8323b9e344a557452b17f4e0dea9d95feff747f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 8 Dec 2014 10:16:18 +0000 Subject: [PATCH 027/158] More bug fixes --- synapse/handlers/_base.py | 6 +++++- synapse/handlers/room.py | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index 0bff644192..871564a3a2 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -70,7 +70,11 @@ class BaseHandler(object): builder.room_id, ) - depth = max([d for _, _, d in latest_ret]) + if latest_ret: + depth = max([d for _, _, d in latest_ret]) + else: + depth = 1 + prev_events = [(e, h) for e, h, _ in latest_ret] builder.prev_events = prev_events diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index f0ffd62b7f..ffcdbcfdf7 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -173,13 +173,14 @@ class RoomCreationHandler(BaseHandler): "sender": creator_id, } - def create(etype, content): + def create(etype, content, **kwargs): e = { "type": etype, "content": content, } e.update(event_keys) + e.update(kwargs) return e @@ -194,7 +195,6 @@ class RoomCreationHandler(BaseHandler): content={ "membership": Membership.JOIN, }, - **event_keys ) power_levels_event = create( From 617dde2ba9db6bad815bce20ca2383d5801dd691 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 8 Dec 2014 10:18:50 +0000 Subject: [PATCH 028/158] Ignore pycharm dir --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 339a99e0d6..af90668c89 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,5 @@ graph/*.dot **/webclient/test/environment-protractor.js uploads + +.idea/ From ba3d1e2fc04c2b2de6a309c5aa1a1e8024d192e1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 8 Dec 2014 12:01:25 +0000 Subject: [PATCH 029/158] Remove unused import --- synapse/api/auth.py | 1 - synapse/storage/_base.py | 1 - 2 files changed, 2 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 821e3ba5e2..cd0deeb0e6 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -25,7 +25,6 @@ from synapse.api.events.room import ( ) from synapse.util.logutils import log_function from synapse.util.async import run_on_reactor -from syutil.base64util import encode_base64 import logging diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index c56c3a0b0f..b0f454b90b 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -19,7 +19,6 @@ from synapse.events import FrozenEvent from synapse.events.utils import prune_event from synapse.util.logutils import log_function from synapse.util.logcontext import PreserveLoggingContext, LoggingContext -from syutil.base64util import encode_base64 from twisted.internet import defer From ee3df06183cbebfe04f51cdd3a1a85a6b50efa9a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 8 Dec 2014 14:50:48 +0000 Subject: [PATCH 030/158] More bug fixes --- synapse/federation/replication.py | 2 +- synapse/handlers/_base.py | 4 ---- synapse/handlers/message.py | 12 ++++++++++++ synapse/handlers/room.py | 7 +------ synapse/state.py | 8 ++++++-- 5 files changed, 20 insertions(+), 13 deletions(-) diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index 3af24ee46d..d4cd79b7ac 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -713,7 +713,7 @@ class _TransactionQueue(object): # table and we'll get back to it later. destinations = set(destinations) - destinations.remove(self.server_name) + destinations.discard(self.server_name) logger.debug("Sending to: %s", str(destinations)) diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index 871564a3a2..4cbc0c027c 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -88,10 +88,6 @@ class BaseHandler(object): prev_state = ret if builder.is_state(): - prev_state = yield self.store.add_event_hashes( - prev_state - ) - builder.prev_state = prev_state yield self.auth.add_auth_events(builder, context) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 485d8e8179..8ee560d79a 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -137,6 +137,18 @@ class MessageHandler(BaseHandler): def handle_event(self, event_dict): builder = self.event_builder_factory.new(event_dict) + + if builder.type == EventTypes.Member: + membership = builder.content.get("membership", None) + if membership == Membership.JOIN: + joinee = self.hs.parse_userid(builder.state_key) + # If event doesn't include a display name, add one. + yield self.distributor.fire( + "collect_presencelike_data", + joinee, + builder.content + ) + event, context = yield self._create_new_client_event( builder=builder, ) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index ffcdbcfdf7..4f4b275290 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -187,6 +187,7 @@ class RoomCreationHandler(BaseHandler): creation_event = create( etype=RoomCreateEvent.TYPE, content={"creator": creator.to_string()}, + state_key="", ) join_event = create( @@ -406,11 +407,6 @@ class RoomMemberHandler(BaseHandler): # room_id = RoomID.from_string(event.room_id, self.hs) room_id = event.room_id - # If event doesn't include a display name, add one. - yield self.distributor.fire( - "collect_presencelike_data", joinee, event.content - ) - # XXX: We don't do an auth check if we are doing an invite # join dance for now, since we're kinda implicitly checking # that we are allowed to join when we decide whether or not we @@ -524,7 +520,6 @@ class RoomMemberHandler(BaseHandler): context, extra_users=[target_user], suppress_auth=(not do_auth), - do_invite_host=do_invite_host, ) diff --git a/synapse/state.py b/synapse/state.py index 464cbae564..19b408db45 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -149,7 +149,7 @@ class StateHandler(object): if event.is_state(): ret = yield self.resolve_state_groups( [e for e, _ in event.prev_events], - event_type=event.event_type, + event_type=event.type, state_key=event.state_key, ) else: @@ -200,7 +200,11 @@ class StateHandler(object): prev_state = state.get((event_type, state_key), None) if prev_state: prev_state = prev_state.event_id - defer.returnValue((name, state, [prev_state])) + prev_states = [prev_state] + else: + prev_states = [] + + defer.returnValue((name, state, prev_states)) state = {} for group, g_state in state_groups.items(): From 0d3fa1ac6e5257218a0c0dbda8cc015e77fe0a30 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Mon, 8 Dec 2014 17:48:57 +0000 Subject: [PATCH 031/158] add a write-through cache on the retry schedule --- synapse/storage/transactions.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index cacd948302..fa51766e05 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -25,6 +25,9 @@ logger = logging.getLogger(__name__) class TransactionStore(SQLBaseStore): """A collection of queries for handling PDUs. """ + + # a write-through cache of DestinationsTable.EntryType indexed by destination string + destination_retry_cache = {} def get_received_txn_response(self, transaction_id, origin): """For an incoming transaction from a given origin, check if we have @@ -213,10 +216,11 @@ class TransactionStore(SQLBaseStore): Returns: None if not retrying - tuple: (retry_last_ts, retry_interval) - retry_ts: time of last retry attempt in unix epoch ms - retry_interval: how long until next retry in ms + Otherwise a DestinationsTable.EntryType for the retry scheme """ + if self.destination_retry_cache[destination]: + return self.destination_retry_cache[destination] + return self.runInteraction( "get_destination_retry_timings", self._get_destination_retry_timings, destination) @@ -225,7 +229,7 @@ class TransactionStore(SQLBaseStore): query = DestinationsTable.select_statement("destination = ?") txn.execute(query, (destination,)) result = DestinationsTable.decode_single_result(txn.fetchone()) - if result and result[0] > 0: + if result and result.retry_last_ts > 0: return result else: return None @@ -239,6 +243,12 @@ class TransactionStore(SQLBaseStore): retry_last_ts (int) - time of last retry attempt in unix epoch ms retry_interval (int) - how long until next retry in ms """ + + self.destination_retry_cache[destination] = ( + DestinationsTable.EntryType(destination, retry_last_ts, retry_interval) + ) + + # xxx: we could chose to not bother persisting this if our cache things this is a NOOP return self.runInteraction( "set_destination_retry_timings", self._set_destination_retry_timings, destination, retry_last_ts, retry_interval) @@ -260,6 +270,7 @@ class TransactionStore(SQLBaseStore): Returns: list: A list of `DestinationsTable.EntryType` """ + return self.runInteraction( "get_destinations_needing_retry", self._get_destinations_needing_retry From 609c31e8dfa23bce3b34500f28df4f8eaf740a91 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 8 Dec 2014 17:50:56 +0000 Subject: [PATCH 032/158] More bug fixes --- synapse/crypto/event_signing.py | 2 +- synapse/events/__init__.py | 9 +------- synapse/handlers/federation.py | 39 +++++++++++++++++++-------------- synapse/handlers/room.py | 11 +++++++--- synapse/state.py | 4 ++-- 5 files changed, 35 insertions(+), 30 deletions(-) diff --git a/synapse/crypto/event_signing.py b/synapse/crypto/event_signing.py index b189f0bb2b..15de0f5ae3 100644 --- a/synapse/crypto/event_signing.py +++ b/synapse/crypto/event_signing.py @@ -39,7 +39,7 @@ def check_event_content_hash(event, hash_algorithm=hashlib.sha256): ), Codes.UNAUTHORIZED, ) - message_hash_base64 = event.hashes[name.name] + message_hash_base64 = event.hashes[name] try: message_hash_bytes = decode_base64(message_hash_base64) except: diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index e81b995d39..230daf30d6 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -157,11 +157,4 @@ class FrozenEvent(EventBase): def get_dict(self): # We need to unfreeze what we return - - d = _unfreeze(self._event_dict) - d.update({ - "signatures": self.signatures, - "unsigned": self.unsigned, - }) - - return d + return _unfreeze(super(FrozenEvent, self).get_dict()) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 38ee32d26e..2d015ccce6 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -337,23 +337,29 @@ class FederationHandler(BaseHandler): self.room_queues[room_id] = [] + builder = self.event_builder_factory.new( + event.get_pdu_json() + ) + try: - event.event_id = self.event_factory.create_event_id() - event.origin = self.hs.hostname - event.content = content + builder.event_id = self.event_factory.create_event_id() + builder.origin = self.hs.hostname + builder.content = content if not hasattr(event, "signatures"): - event.signatures = {} + builder.signatures = {} add_hashes_and_signatures( - event, + builder, self.hs.hostname, self.hs.config.signing_key[0], ) + new_event = builder.build() + ret = yield self.replication_layer.send_join( target_host, - event + new_event ) state = ret["state"] @@ -363,7 +369,7 @@ class FederationHandler(BaseHandler): logger.debug("do_invite_join auth_chain: %s", auth_chain) logger.debug("do_invite_join state: %s", state) - logger.debug("do_invite_join event: %s", event) + logger.debug("do_invite_join event: %s", new_event) try: yield self.store.store_room( @@ -400,13 +406,13 @@ class FederationHandler(BaseHandler): ) yield self._handle_new_event( - event, + new_event, state=state, current_state=state, ) yield self.notifier.on_new_room_event( - event, extra_users=[joinee] + new_event, extra_users=[joinee] ) logger.debug("Finished joining %s to %s", joinee, room_id) @@ -457,7 +463,7 @@ class FederationHandler(BaseHandler): event.internal_metadata.outlier = False - yield self._handle_new_event(event) + context = yield self._handle_new_event(event) extra_users = [] if event.type == RoomMemberEvent.TYPE: @@ -480,7 +486,7 @@ class FederationHandler(BaseHandler): destinations = set() - for k, s in event.state_events.items(): + for k, s in context.current_state.items(): try: if k[0] == RoomMemberEvent.TYPE: if s.content["membership"] == Membership.JOIN: @@ -492,14 +498,12 @@ class FederationHandler(BaseHandler): "Failed to get destination from event %s", s.event_id ) - new_pdu.destinations = list(destinations) - - yield self.replication_layer.send_pdu(new_pdu) + yield self.replication_layer.send_pdu(new_pdu, destinations) auth_chain = yield self.store.get_auth_chain(event.event_id) defer.returnValue({ - "state": event.state_events.values(), + "state": context.current_state.values(), "auth_chain": auth_chain, }) @@ -652,6 +656,7 @@ class FederationHandler(BaseHandler): context = EventContext() yield self.state_handler.annotate_context_with_state( event, + context, old_state=state ) @@ -664,7 +669,6 @@ class FederationHandler(BaseHandler): if e_id not in known_ids: e = yield self.store.get_event( e_id, - context, allow_none=True, ) @@ -689,7 +693,10 @@ class FederationHandler(BaseHandler): yield self.store.persist_event( event, + context=context, backfilled=backfilled, is_new_state=(is_new_state and not backfilled), current_state=current_state, ) + + defer.returnValue(context) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 4f4b275290..6da084b3ac 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -171,6 +171,7 @@ class RoomCreationHandler(BaseHandler): event_keys = { "room_id": room_id, "sender": creator_id, + "state_key": "", } def create(etype, content, **kwargs): @@ -187,7 +188,6 @@ class RoomCreationHandler(BaseHandler): creation_event = create( etype=RoomCreateEvent.TYPE, content={"creator": creator.to_string()}, - state_key="", ) join_event = create( @@ -388,7 +388,7 @@ class RoomMemberHandler(BaseHandler): host = hosts[0] content.update({"membership": Membership.JOIN}) - event, context = yield self.create_new_client_event({ + builder = self.event_builder_factory.new({ "type": RoomMemberEvent.TYPE, "state_key": joinee.to_string(), "room_id": room_id, @@ -396,6 +396,7 @@ class RoomMemberHandler(BaseHandler): "membership": Membership.JOIN, "content": content, }) + event, context = yield self._create_new_client_event(builder) yield self._do_join(event, context, room_host=host, do_auth=True) @@ -442,7 +443,11 @@ class RoomMemberHandler(BaseHandler): if should_do_dance: handler = self.hs.get_handlers().federation_handler have_joined = yield handler.do_invite_join( - room_host, room_id, event.user_id, event.content, context + room_host, + room_id, + event.user_id, + event.get_dict()["content"], # FIXME To get a non-frozen dict + context ) # We want to do the _do_update inside the room lock. diff --git a/synapse/state.py b/synapse/state.py index 19b408db45..d1d6f95246 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -160,13 +160,13 @@ class StateHandler(object): group, curr_state, prev_state = ret context.current_state = curr_state - context.state_group = group + context.state_group = group if not event.is_state() else None prev_state = yield self.store.add_event_hashes( prev_state ) - if hasattr(event, "auth_events") and event.auth_events: + if hasattr(event, "auth_events"): auth_ids = zip(*event.auth_events)[0] context.auth_events = { k: v From 8529fba02d93ed1d0d08873f0cbbd58a3194e4af Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Mon, 8 Dec 2014 19:34:51 +0000 Subject: [PATCH 033/158] fix a million stupid bugs and make it actually work --- synapse/federation/replication.py | 25 +++++++++++++++++-------- synapse/handlers/federation.py | 3 ++- synapse/storage/transactions.py | 25 ++++++++++++++----------- 3 files changed, 33 insertions(+), 20 deletions(-) diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index 88184caecd..c4c6667b62 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -685,6 +685,7 @@ class _TransactionQueue(object): self.transport_layer = transport_layer self._clock = hs.get_clock() + self.store = hs.get_datastore() # Is a mapping from destinations -> deferreds. Used to keep track # of which destinations have transactions in flight and when they are @@ -775,11 +776,18 @@ class _TransactionQueue(object): @defer.inlineCallbacks @log_function def _attempt_new_transaction(self, destination): - - (retry_last_ts, retry_interval) = self.store.get_destination_retry_timings(destination) - if retry_last_ts + retry_interval > int(self._clock.time_msec()): - logger.info("TX [%s] not ready for retry yet - dropping transaction for now") - return + + (retry_last_ts, retry_interval) = (0, 0) + retry_timings = yield self.store.get_destination_retry_timings(destination) + if retry_timings: + (retry_last_ts, retry_interval) = ( + retry_timings.retry_last_ts, retry_timings.retry_interval + ) + if retry_last_ts + retry_interval > int(self._clock.time_msec()): + logger.info("TX [%s] not ready for retry yet - dropping transaction for now", destination) + return + else: + logger.info("TX [%s] is ready for retry", destination) if destination in self.pending_transactions: # XXX: pending_transactions can get stuck on by a never-ending request @@ -866,7 +874,7 @@ class _TransactionQueue(object): if code == 200: if retry_last_ts: # this host is alive! reset retry schedule - self.store.set_destination_retry_timings(destination, 0, 0) + yield self.store.set_destination_retry_timings(destination, 0, 0) deferred.callback(None) else: self.start_retrying(destination, retry_interval) @@ -899,12 +907,13 @@ class _TransactionQueue(object): # Check to see if there is anything else to send. self._attempt_new_transaction(destination) + @defer.inlineCallbacks def start_retrying(self, destination, retry_interval): # track that this destination is having problems and we should # give it a chance to recover before trying it again if retry_interval: retry_interval *= 2 else: - retry_interval = 2 # try again at first after 2 seconds - self.store.set_destination_retry_timings(destination, + retry_interval = 2000 # try again at first after 2 seconds + yield self.store.set_destination_retry_timings(destination, int(self._clock.time_msec()), retry_interval) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 7a79e2d117..cfb5029774 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -211,7 +211,8 @@ class FederationHandler(BaseHandler): # if we're receiving valid events from an origin, # it's probably a good idea to mark it as not in retry-state # for sending (although this is a bit of a leap) - if ((self.store.get_destination_retry_timings(origin))[0]): + retry_timings = yield self.store.get_destination_retry_timings(origin) + if (retry_timings and retry_timings.retry_last_ts): self.store.set_destination_retry_timings(origin, 0, 0) room = yield self.store.get_room(event.room_id) diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index fa51766e05..237b024451 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -17,6 +17,8 @@ from ._base import SQLBaseStore, Table from collections import namedtuple +from twisted.internet import defer + import logging logger = logging.getLogger(__name__) @@ -218,8 +220,8 @@ class TransactionStore(SQLBaseStore): None if not retrying Otherwise a DestinationsTable.EntryType for the retry scheme """ - if self.destination_retry_cache[destination]: - return self.destination_retry_cache[destination] + if destination in self.destination_retry_cache: + return defer.succeed(self.destination_retry_cache[destination]) return self.runInteraction( "get_destination_retry_timings", @@ -228,11 +230,13 @@ class TransactionStore(SQLBaseStore): def _get_destination_retry_timings(cls, txn, destination): query = DestinationsTable.select_statement("destination = ?") txn.execute(query, (destination,)) - result = DestinationsTable.decode_single_result(txn.fetchone()) - if result and result.retry_last_ts > 0: - return result - else: - return None + result = txn.fetchall() + if result: + result = DestinationsTable.decode_single_result(result) + if result.retry_last_ts > 0: + return result + else: + return None def set_destination_retry_timings(self, destination, retry_last_ts, retry_interval): """Sets the current retry timings for a given destination. @@ -257,12 +261,11 @@ class TransactionStore(SQLBaseStore): query = ( "INSERT OR REPLACE INTO %s " - "(retry_last_ts, retry_interval) " - "VALUES (?, ?) " - "WHERE destination = ?" + "(destination, retry_last_ts, retry_interval) " + "VALUES (?, ?, ?) " ) % DestinationsTable.table_name - txn.execute(query, (retry_last_ts, retry_interval, destination)) + txn.execute(query, (destination, retry_last_ts, retry_interval)) def get_destinations_needing_retry(self): """Get all destinations which are due a retry for sending a transaction. From 4e57943cc5bc6454891c9dba65d3fcd43cb4bee4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 9 Dec 2014 10:51:36 +0000 Subject: [PATCH 034/158] Remove unused import --- synapse/handlers/_base.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index 4cbc0c027c..46c2b0a690 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -18,10 +18,9 @@ from twisted.internet import defer from synapse.api.errors import LimitExceededError, SynapseError from synapse.util.async import run_on_reactor from synapse.crypto.event_signing import add_hashes_and_signatures -from synapse.api.events.room import RoomMemberEvent from synapse.api.constants import Membership, EventTypes -from synapse.events.snapshot import EventSnapshot, EventContext +from synapse.events.snapshot import EventContext import logging From 1c8ee06877ce11ac40e1c2a7468a0788992d9879 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 9 Dec 2014 10:53:34 +0000 Subject: [PATCH 035/158] Remove unused snapshot --- synapse/handlers/profile.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index f2abbc5df9..18fd0914e2 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -200,8 +200,6 @@ class ProfileHandler(BaseHandler): ) for j in joins: - snapshot = yield self.store.snapshot_room(j) - content = { "membership": j.content["membership"], } From 8c4845068298cf6e2b8a662ffa2340b655abf9b6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 9 Dec 2014 10:58:31 +0000 Subject: [PATCH 036/158] Add PEP8 newlines --- synapse/events/builder.py | 2 +- synapse/events/utils.py | 2 +- synapse/events/validator.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/events/builder.py b/synapse/events/builder.py index 0b8caf9318..127b8fa904 100644 --- a/synapse/events/builder.py +++ b/synapse/events/builder.py @@ -66,4 +66,4 @@ class EventBuilderFactory(object): age = key_values["unsigned"].pop("age", 0) key_values["unsigned"].setdefault("age_ts", time_now - age) - return EventBuilder(key_values=key_values,) \ No newline at end of file + return EventBuilder(key_values=key_values,) diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 485f075406..f5e135e3d0 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -97,4 +97,4 @@ def serialize_event(hs, e): d["user_id"] = d.pop("sender", None) - return d \ No newline at end of file + return d diff --git a/synapse/events/validator.py b/synapse/events/validator.py index 7dc9506ec4..f319072d37 100644 --- a/synapse/events/validator.py +++ b/synapse/events/validator.py @@ -55,4 +55,4 @@ class EventValidator(object): def validate_new(self, event): self.validate(event) - UserID.from_string(event.sender) \ No newline at end of file + UserID.from_string(event.sender) From bc6564bac0010cda8f75e7f5325f5dc2c49b8ebd Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 9 Dec 2014 11:01:44 +0000 Subject: [PATCH 037/158] Add PEP8 newlines --- synapse/util/frozenutils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/util/frozenutils.py b/synapse/util/frozenutils.py index 46d3ab4f7d..1fb67df6b9 100644 --- a/synapse/util/frozenutils.py +++ b/synapse/util/frozenutils.py @@ -53,4 +53,4 @@ class FrozenEncoder(json.JSONEncoder): if isinstance(o, frozendict): return dict(o) - return json.JSONEncoder(self, o) \ No newline at end of file + return json.JSONEncoder(self, o) From 3986c775c4f742c663e8223b156e87e5679c3733 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 9 Dec 2014 11:16:03 +0000 Subject: [PATCH 038/158] This is to test jenkins --- TEST | 1 + 1 file changed, 1 insertion(+) create mode 100644 TEST diff --git a/TEST b/TEST new file mode 100644 index 0000000000..9daeafb986 --- /dev/null +++ b/TEST @@ -0,0 +1 @@ +test From 2ef499ab84ee6f7145e6a1752c5777c5996bcb83 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 9 Dec 2014 11:19:39 +0000 Subject: [PATCH 039/158] This is to test jenkins --- TEST | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TEST b/TEST index 9daeafb986..180cf83280 100644 --- a/TEST +++ b/TEST @@ -1 +1 @@ -test +test2 From 3654825b02de691587cc1fb2c438f075e47247cc Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 9 Dec 2014 11:25:41 +0000 Subject: [PATCH 040/158] This is to test jenkins --- TEST | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TEST b/TEST index 180cf83280..df6b0d2bcc 100644 --- a/TEST +++ b/TEST @@ -1 +1 @@ -test2 +test3 From bdbcd8a638fb882b36f466c2e3b9f4649d71deec Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 9 Dec 2014 11:36:38 +0000 Subject: [PATCH 041/158] This is to test jenkins --- TEST | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TEST b/TEST index df6b0d2bcc..d234c5e057 100644 --- a/TEST +++ b/TEST @@ -1 +1 @@ -test3 +test4 From 4a7a0ed94905907d8df92168e444aa0d0625c682 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 9 Dec 2014 11:41:32 +0000 Subject: [PATCH 042/158] This is to test jenkins --- TEST | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TEST b/TEST index d234c5e057..4f346f1ad5 100644 --- a/TEST +++ b/TEST @@ -1 +1 @@ -test4 +test5 From d7277398b90b3716b773c13d1e6baa71ff22520d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 9 Dec 2014 12:20:49 +0000 Subject: [PATCH 043/158] This is to test jenkins --- TEST | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TEST b/TEST index 4f346f1ad5..8390c32b5e 100644 --- a/TEST +++ b/TEST @@ -1 +1 @@ -test5 +test6 From 90d022441fad7c5e1a6c8ea21bf4cf7f272a18ae Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 9 Dec 2014 13:14:38 +0000 Subject: [PATCH 044/158] Delete test file --- TEST | 1 - 1 file changed, 1 deletion(-) delete mode 100644 TEST diff --git a/TEST b/TEST deleted file mode 100644 index 8390c32b5e..0000000000 --- a/TEST +++ /dev/null @@ -1 +0,0 @@ -test6 From aa3f66cf7ff15b292afc6f251c1a77e154b8b675 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 9 Dec 2014 13:35:26 +0000 Subject: [PATCH 045/158] Change the way we implement get_events to be less sucky --- synapse/storage/__init__.py | 24 +++++++----------------- synapse/storage/_base.py | 34 +++++++++++++++------------------- 2 files changed, 22 insertions(+), 36 deletions(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index f172c2690a..7a09c33613 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -119,25 +119,15 @@ class DataStore(RoomMemberStore, RoomStore, @defer.inlineCallbacks def get_event(self, event_id, allow_none=False): - events_dict = yield self._simple_select_one( - "events", - {"event_id": event_id}, - [ - "event_id", - "type", - "room_id", - "content", - "unrecognized_keys", - "depth", - ], - allow_none=allow_none, - ) + events = yield self._get_events([event_id]) - if not events_dict: - defer.returnValue(None) + if not events: + if allow_none: + defer.returnValue(None) + else: + raise RuntimeError("Could not find event %s" % (event_id,)) - event = yield self._parse_events([events_dict]) - defer.returnValue(event[0]) + defer.returnValue(events[0]) @log_function def _persist_event_txn(self, txn, event, context, backfilled, diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index b0f454b90b..72f88cb2aa 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -461,32 +461,18 @@ class SQLBaseStore(object): **d ) - def _get_events_txn(self, txn, event_ids): - # FIXME (erikj): This should be batched? - - sql = "SELECT * FROM events WHERE event_id = ? ORDER BY rowid asc" - - event_rows = [] - for e_id in event_ids: - c = txn.execute(sql, (e_id,)) - event_rows.extend(self.cursor_to_dict(c)) - - return self._parse_events_txn(txn, event_rows) - - def _parse_events(self, rows): + def _get_events(self, event_ids): return self.runInteraction( - "_parse_events", self._parse_events_txn, rows + "_get_events", self._get_events_txn, event_ids ) - def _parse_events_txn(self, txn, rows): - event_ids = [r["event_id"] for r in rows] - + def _get_events_txn(self, txn, event_ids): events = [] - for event_id in event_ids: + for e_id in event_ids: js = self._simple_select_one_onecol_txn( txn, table="event_json", - keyvalues={"event_id": event_id}, + keyvalues={"event_id": e_id}, retcol="json", allow_none=True, ) @@ -516,6 +502,16 @@ class SQLBaseStore(object): return events + def _parse_events(self, rows): + return self.runInteraction( + "_parse_events", self._parse_events_txn, rows + ) + + def _parse_events_txn(self, txn, rows): + event_ids = [r["event_id"] for r in rows] + + return self._get_events_txn(txn, event_ids) + def _has_been_redacted_txn(self, txn, event): sql = "SELECT event_id FROM redactions WHERE redacts = ?" txn.execute(sql, (event.event_id,)) From 5eca288d280659e69fce5d5eca3e7bf09faac5f0 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 9 Dec 2014 14:47:27 +0000 Subject: [PATCH 046/158] Fix joining from an invite --- synapse/handlers/_base.py | 15 ++++++++++++++- synapse/handlers/federation.py | 2 -- synapse/handlers/room.py | 16 +++------------- synapse/state.py | 4 +++- 4 files changed, 20 insertions(+), 17 deletions(-) diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index 46c2b0a690..2c737c8bf9 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -113,6 +113,20 @@ class BaseHandler(object): yield self.store.persist_event(event, context=context) + federation_handler = self.hs.get_handlers().federation_handler + + if event.type == EventTypes.Member: + if event.content["membership"] == Membership.INVITE: + invitee = self.hs.parse_userid(event.state_key) + if not self.hs.is_mine(invitee): + returned_invite = yield federation_handler.send_invite( + invitee.domain, + event, + ) + event.signatures.update( + returned_invite.signatures + ) + destinations = set(extra_destinations) for k, s in context.current_state.items(): try: @@ -128,7 +142,6 @@ class BaseHandler(object): yield self.notifier.on_new_room_event(event, extra_users=extra_users) - federation_handler = self.hs.get_handlers().federation_handler yield federation_handler.handle_new_event( event, None, diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 2d015ccce6..827c86c9da 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -282,8 +282,6 @@ class FederationHandler(BaseHandler): pdu=event ) - - defer.returnValue(pdu) @defer.inlineCallbacks diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 6da084b3ac..215b9cc5ac 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -429,12 +429,9 @@ class RoomMemberHandler(BaseHandler): ) if prev_state and prev_state.membership == Membership.INVITE: - room = yield self.store.get_room(room_id) - inviter = UserID.from_string( - prev_state.user_id, self.hs - ) + inviter = UserID.from_string(prev_state.user_id) - should_do_dance = not self.hs.is_mine(inviter) and not room + should_do_dance = not self.hs.is_mine(inviter) room_host = inviter.domain else: should_do_dance = False @@ -511,14 +508,7 @@ class RoomMemberHandler(BaseHandler): do_auth): yield run_on_reactor() - # If we're inviting someone, then we should also send it to that - # HS. - target_user_id = event.state_key - target_user = self.hs.parse_userid(target_user_id) - if membership == Membership.INVITE and not self.hs.is_mine(target_user): - do_invite_host = target_user.domain - else: - do_invite_host = None + target_user = self.hs.parse_userid(event.state_key) yield self.handle_new_client_event( event, diff --git a/synapse/state.py b/synapse/state.py index d1d6f95246..ebec0ad9dc 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -166,13 +166,15 @@ class StateHandler(object): prev_state ) - if hasattr(event, "auth_events"): + if hasattr(event, "auth_events") and event.auth_events: auth_ids = zip(*event.auth_events)[0] context.auth_events = { k: v for k, v in context.current_state.items() if v.event_id in auth_ids } + else: + context.auth_events = {} defer.returnValue(prev_state) From 008303b245944a41bca644a5ffdd2e9dfdebb338 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 9 Dec 2014 14:49:11 +0000 Subject: [PATCH 047/158] PEP8 --- synapse/api/constants.py | 2 +- synapse/handlers/room.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/api/constants.py b/synapse/api/constants.py index acf50e42ab..7e8c892b64 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -67,4 +67,4 @@ class EventTypes(object): JoinRules = "m.room.join_rules" PowerLevels = "m.room.power_levels" Aliases = "m.room.aliases" - Redaction = "m.room.redaction" \ No newline at end of file + Redaction = "m.room.redaction" diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 215b9cc5ac..509763ebc7 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -342,7 +342,7 @@ class RoomMemberHandler(BaseHandler): """ target_user_id = event.state_key - ## TODO(markjh): get prev state from snapshot. + # TODO(markjh): get prev state from snapshot. prev_state = yield self.store.get_room_member( target_user_id, event.room_id ) From 7e8d5c2606a8109054eba9b1ec319bc0cb286ec3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 9 Dec 2014 16:31:27 +0000 Subject: [PATCH 048/158] This is to test jenkins --- TEST | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 TEST diff --git a/TEST b/TEST new file mode 100644 index 0000000000..e69de29bb2 From 50fd5014c2ef42fbe7b9b1109f3d2470cea6957c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 9 Dec 2014 16:33:04 +0000 Subject: [PATCH 049/158] This is to test jenkins --- TEST | 1 + 1 file changed, 1 insertion(+) diff --git a/TEST b/TEST index e69de29bb2..9daeafb986 100644 --- a/TEST +++ b/TEST @@ -0,0 +1 @@ +test From 26e293abbed138a4a555753910209f9de152b0b3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 9 Dec 2014 16:33:47 +0000 Subject: [PATCH 050/158] This is to test jenkins --- TEST | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TEST b/TEST index 9daeafb986..a5bce3fd25 100644 --- a/TEST +++ b/TEST @@ -1 +1 @@ -test +test1 From b63cea96600bf60a6eb627d63d91b7759374ca42 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 9 Dec 2014 16:35:00 +0000 Subject: [PATCH 051/158] This is to test jenkins --- TEST | 1 - 1 file changed, 1 deletion(-) delete mode 100644 TEST diff --git a/TEST b/TEST deleted file mode 100644 index a5bce3fd25..0000000000 --- a/TEST +++ /dev/null @@ -1 +0,0 @@ -test1 From 8ada2d20183da9aac8182fbe80bdb897c5a4d224 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Tue, 9 Dec 2014 23:53:07 +0000 Subject: [PATCH 052/158] fix UTs by telling all the mock stores about the new methods for tracking retries --- tests/federation/test_federation.py | 5 +++++ tests/handlers/test_federation.py | 2 ++ tests/handlers/test_presence.py | 10 +++++++++- tests/handlers/test_typing.py | 9 +++++++-- 4 files changed, 23 insertions(+), 3 deletions(-) diff --git a/tests/federation/test_federation.py b/tests/federation/test_federation.py index 73dd289276..f6b41e2c4c 100644 --- a/tests/federation/test_federation.py +++ b/tests/federation/test_federation.py @@ -25,6 +25,7 @@ from synapse.server import HomeServer from synapse.federation import initialize_http_replication from synapse.api.events import SynapseEvent +from synapse.storage.transactions import DestinationsTable def make_pdu(prev_pdus=[], **kwargs): """Provide some default fields for making a PduTuple.""" @@ -55,10 +56,14 @@ class FederationTestCase(unittest.TestCase): "delivered_txn", "get_received_txn_response", "set_received_txn_response", + "get_destination_retry_timings", ]) self.mock_persistence.get_received_txn_response.return_value = ( defer.succeed(None) ) + self.mock_persistence.get_destination_retry_timings.return_value = ( + defer.succeed(DestinationsTable.EntryType("", 0, 0)) + ) self.mock_config = Mock() self.mock_config.signing_key = [MockKey()] self.clock = MockClock() diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py index 33016c16ef..fae33716a3 100644 --- a/tests/handlers/test_federation.py +++ b/tests/handlers/test_federation.py @@ -53,6 +53,8 @@ class FederationTestCase(unittest.TestCase): "persist_event", "store_room", "get_room", + "get_destination_retry_timings", + "set_destination_retry_timings", ]), resource_for_federation=NonCallableMock(), http_client=NonCallableMock(spec_set=[]), diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py index fe69ce47eb..b85a89052a 100644 --- a/tests/handlers/test_presence.py +++ b/tests/handlers/test_presence.py @@ -30,7 +30,7 @@ from synapse.api.constants import PresenceState from synapse.api.errors import SynapseError from synapse.handlers.presence import PresenceHandler, UserPresenceCache from synapse.streams.config import SourcePaginationConfig - +from synapse.storage.transactions import DestinationsTable OFFLINE = PresenceState.OFFLINE UNAVAILABLE = PresenceState.UNAVAILABLE @@ -528,6 +528,7 @@ class PresencePushTestCase(unittest.TestCase): "delivered_txn", "get_received_txn_response", "set_received_txn_response", + "get_destination_retry_timings", ]), handlers=None, resource_for_client=Mock(), @@ -539,6 +540,9 @@ class PresencePushTestCase(unittest.TestCase): hs.handlers = JustPresenceHandlers(hs) self.datastore = hs.get_datastore() + self.datastore.get_destination_retry_timings.return_value = ( + defer.succeed(DestinationsTable.EntryType("", 0, 0)) + ) def get_received_txn_response(*args): return defer.succeed(None) @@ -1037,6 +1041,7 @@ class PresencePollingTestCase(unittest.TestCase): "delivered_txn", "get_received_txn_response", "set_received_txn_response", + "get_destination_retry_timings", ]), handlers=None, resource_for_client=Mock(), @@ -1048,6 +1053,9 @@ class PresencePollingTestCase(unittest.TestCase): hs.handlers = JustPresenceHandlers(hs) self.datastore = hs.get_datastore() + self.datastore.get_destination_retry_timings.return_value = ( + defer.succeed(DestinationsTable.EntryType("", 0, 0)) + ) def get_received_txn_response(*args): return defer.succeed(None) diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py index adb5148351..2f170ac3bf 100644 --- a/tests/handlers/test_typing.py +++ b/tests/handlers/test_typing.py @@ -25,6 +25,8 @@ from ..utils import MockHttpResource, MockClock, DeferredMockCallable, MockKey from synapse.server import HomeServer from synapse.handlers.typing import TypingNotificationHandler +from synapse.storage.transactions import DestinationsTable + def _expect_edu(destination, edu_type, content, origin="test"): return { @@ -49,7 +51,6 @@ class JustTypingNotificationHandlers(object): def __init__(self, hs): self.typing_notification_handler = TypingNotificationHandler(hs) - class TypingNotificationsTestCase(unittest.TestCase): """Tests typing notifications to rooms.""" def setUp(self): @@ -72,6 +73,7 @@ class TypingNotificationsTestCase(unittest.TestCase): "delivered_txn", "get_received_txn_response", "set_received_txn_response", + "get_destination_retry_timings", ]), handlers=None, resource_for_client=Mock(), @@ -89,6 +91,9 @@ class TypingNotificationsTestCase(unittest.TestCase): self.handler.push_update_to_clients = self.mock_update_client self.datastore = hs.get_datastore() + self.datastore.get_destination_retry_timings.return_value = ( + defer.succeed(DestinationsTable.EntryType("", 0, 0)) + ) def get_received_txn_response(*args): return defer.succeed(None) @@ -162,7 +167,7 @@ class TypingNotificationsTestCase(unittest.TestCase): @defer.inlineCallbacks def test_started_typing_remote_send(self): self.room_members = [self.u_apple, self.u_onion] - + put_json = self.mock_http_client.put_json put_json.expect_call_and_return( call("farm", From 2b1acb7671e33baeb01be2f0facd20cd6ea7e3b5 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 10 Dec 2014 00:03:55 +0000 Subject: [PATCH 053/158] squidge to 79 columns as per pep8 --- synapse/federation/replication.py | 30 +++++++++++++++++--------- synapse/http/matrixfederationclient.py | 7 +++--- synapse/storage/transactions.py | 18 ++++++++++------ 3 files changed, 36 insertions(+), 19 deletions(-) diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index c4c6667b62..c242488483 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -778,21 +778,25 @@ class _TransactionQueue(object): def _attempt_new_transaction(self, destination): (retry_last_ts, retry_interval) = (0, 0) - retry_timings = yield self.store.get_destination_retry_timings(destination) + retry_timings = yield self.store.get_destination_retry_timings( + destination + ) if retry_timings: (retry_last_ts, retry_interval) = ( retry_timings.retry_last_ts, retry_timings.retry_interval ) if retry_last_ts + retry_interval > int(self._clock.time_msec()): - logger.info("TX [%s] not ready for retry yet - dropping transaction for now", destination) + logger.info("TX [%s] not ready for retry yet - " + "dropping transaction for now", destination) return else: logger.info("TX [%s] is ready for retry", destination) if destination in self.pending_transactions: - # XXX: pending_transactions can get stuck on by a never-ending request - # at which point pending_pdus_by_dest just keeps growing. - # we need application-layer timeouts of some flavour of these requests + # XXX: pending_transactions can get stuck on by a never-ending + # request at which point pending_pdus_by_dest just keeps growing. + # we need application-layer timeouts of some flavour of these + # requests return # list of (pending_pdu, deferred, order) @@ -803,8 +807,10 @@ class _TransactionQueue(object): if not pending_pdus and not pending_edus and not pending_failures: return - logger.debug("TX [%s] Attempting new transaction (pdus: %d, edus: %d, failures: %d)", - destination, len(pending_pdus), len(pending_edus), len(pending_failures)) + logger.debug("TX [%s] Attempting new transaction " + "(pdus: %d, edus: %d, failures: %d)", + destination, + len(pending_pdus), len(pending_edus), len(pending_failures)) # Sort based on the order field pending_pdus.sort(key=lambda t: t[2]) @@ -837,7 +843,8 @@ class _TransactionQueue(object): yield self.transaction_actions.prepare_to_send(transaction) logger.debug("TX [%s] Persisted transaction", destination) - logger.info("TX [%s] Sending transaction [%s]", destination, transaction.transaction_id) + logger.info("TX [%s] Sending transaction [%s]", destination, + transaction.transaction_id) # Actually send the transaction @@ -874,7 +881,9 @@ class _TransactionQueue(object): if code == 200: if retry_last_ts: # this host is alive! reset retry schedule - yield self.store.set_destination_retry_timings(destination, 0, 0) + yield self.store.set_destination_retry_timings( + destination, 0, 0 + ) deferred.callback(None) else: self.start_retrying(destination, retry_interval) @@ -892,7 +901,8 @@ class _TransactionQueue(object): except Exception as e: # We capture this here as there as nothing actually listens # for this finishing functions deferred. - logger.exception("TX [%s] Problem in _attempt_transaction: %s", destination, e) + logger.exception("TX [%s] Problem in _attempt_transaction: %s", + destination, e) self.start_retrying(destination, retry_interval) diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 3edc59dbab..c76990904d 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -90,7 +90,7 @@ class MatrixFederationHttpClient(object): ) logger.info("Sending request to %s: %s %s", - destination, method, url_bytes) + destination, method, url_bytes) logger.debug( "Types: %s", @@ -135,7 +135,7 @@ class MatrixFederationHttpClient(object): raise SynapseError(400, "Domain specified not found.") logger.exception("Sending request failed to %s: %s %s : %s", - destination, method, url_bytes, e) + destination, method, url_bytes, e) _print_ex(e) if retries_left: @@ -145,7 +145,8 @@ class MatrixFederationHttpClient(object): raise logger.info("Received response %d %s for %s: %s %s", - response.code, response.phrase, destination, method, url_bytes) + response.code, response.phrase, + destination, method, url_bytes) if 200 <= response.code < 300: # We need to update the transactions table to say it was sent? diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index 237b024451..2b16787695 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -28,7 +28,8 @@ class TransactionStore(SQLBaseStore): """A collection of queries for handling PDUs. """ - # a write-through cache of DestinationsTable.EntryType indexed by destination string + # a write-through cache of DestinationsTable.EntryType indexed by + # destination string destination_retry_cache = {} def get_received_txn_response(self, transaction_id, origin): @@ -238,7 +239,8 @@ class TransactionStore(SQLBaseStore): else: return None - def set_destination_retry_timings(self, destination, retry_last_ts, retry_interval): + def set_destination_retry_timings(self, destination, + retry_last_ts, retry_interval): """Sets the current retry timings for a given destination. Both timings should be zero if retrying is no longer occuring. @@ -249,15 +251,19 @@ class TransactionStore(SQLBaseStore): """ self.destination_retry_cache[destination] = ( - DestinationsTable.EntryType(destination, retry_last_ts, retry_interval) + DestinationsTable.EntryType(destination, + retry_last_ts, retry_interval) ) - # xxx: we could chose to not bother persisting this if our cache things this is a NOOP + # XXX: we could chose to not bother persisting this if our cache thinks + # this is a NOOP return self.runInteraction( "set_destination_retry_timings", - self._set_destination_retry_timings, destination, retry_last_ts, retry_interval) + self._set_destination_retry_timings, destination, + retry_last_ts, retry_interval) - def _set_destination_retry_timings(cls, txn, destination, retry_last_ts, retry_interval): + def _set_destination_retry_timings(cls, txn, destination, + retry_last_ts, retry_interval): query = ( "INSERT OR REPLACE INTO %s " From faf12b64f81627d92cb1ac49b6eb58f9d3f4837d Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 10 Dec 2014 00:12:51 +0000 Subject: [PATCH 054/158] add errbacks to enqueue_pdu deferreds; change logging for failed federation sends to warn rather than exception --- synapse/federation/replication.py | 16 ++++++++++------ synapse/http/matrixfederationclient.py | 4 ++-- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index c242488483..346b5f04c0 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -334,7 +334,7 @@ class ReplicationLayer(object): defer.returnValue(response) return - logger.debug("[%s] Transacition is new", transaction.transaction_id) + logger.debug("[%s] Transaction is new", transaction.transaction_id) with PreserveLoggingContext(): dl = [] @@ -724,15 +724,19 @@ class _TransactionQueue(object): deferreds = [] for destination in destinations: - # XXX: why don't we specify an errback for this deferred - # like we do for EDUs? --matthew deferred = defer.Deferred() self.pending_pdus_by_dest.setdefault(destination, []).append( (pdu, deferred, order) ) + + def eb(failure): + if not deferred.called: + deferred.errback(failure) + else: + logger.warn("Failed to send pdu", failure) with PreserveLoggingContext(): - self._attempt_new_transaction(destination) + self._attempt_new_transaction(destination).addErrback(eb) deferreds.append(deferred) @@ -754,7 +758,7 @@ class _TransactionQueue(object): if not deferred.called: deferred.errback(failure) else: - logger.exception("Failed to send edu", failure) + logger.warn("Failed to send edu", failure) with PreserveLoggingContext(): self._attempt_new_transaction(destination).addErrback(eb) @@ -901,7 +905,7 @@ class _TransactionQueue(object): except Exception as e: # We capture this here as there as nothing actually listens # for this finishing functions deferred. - logger.exception("TX [%s] Problem in _attempt_transaction: %s", + logger.warn("TX [%s] Problem in _attempt_transaction: %s", destination, e) self.start_retrying(destination, retry_interval) diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index c76990904d..8fc6bf8f97 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -134,7 +134,7 @@ class MatrixFederationHttpClient(object): e) raise SynapseError(400, "Domain specified not found.") - logger.exception("Sending request failed to %s: %s %s : %s", + logger.warn("Sending request failed to %s: %s %s : %s", destination, method, url_bytes, e) _print_ex(e) @@ -289,7 +289,7 @@ def _print_ex(e): for ex in e.reasons: _print_ex(ex) else: - logger.exception(e) + logger.warn(e) class _JsonProducer(object): From 71da2bed5585cf66a7707cfc5a000eb28a56ff34 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 10 Dec 2014 00:18:44 +0000 Subject: [PATCH 055/158] plateau retries after 1h --- synapse/federation/replication.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index 346b5f04c0..589a3f581b 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -890,7 +890,7 @@ class _TransactionQueue(object): ) deferred.callback(None) else: - self.start_retrying(destination, retry_interval) + self.set_retrying(destination, retry_interval) deferred.errback(RuntimeError("Got status %d" % code)) # Ensures we don't continue until all callbacks on that @@ -908,7 +908,7 @@ class _TransactionQueue(object): logger.warn("TX [%s] Problem in _attempt_transaction: %s", destination, e) - self.start_retrying(destination, retry_interval) + self.set_retrying(destination, retry_interval) for deferred in deferreds: if not deferred.called: @@ -922,11 +922,14 @@ class _TransactionQueue(object): self._attempt_new_transaction(destination) @defer.inlineCallbacks - def start_retrying(self, destination, retry_interval): + def set_retrying(self, destination, retry_interval): # track that this destination is having problems and we should # give it a chance to recover before trying it again if retry_interval: retry_interval *= 2 + # plateau at hourly retries for now + if retry_interval >= 60 * 60 * 1000: + retry_interval = 60 * 60 * 1000 else: retry_interval = 2000 # try again at first after 2 seconds yield self.store.set_destination_retry_timings(destination, From b8d30899b1296347a75d5a59e32d73a5236e6ea2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 10 Dec 2014 10:16:09 +0000 Subject: [PATCH 056/158] Code style. --- synapse/federation/replication.py | 52 +++++++++++++++++--------- synapse/http/matrixfederationclient.py | 29 ++++++++++---- synapse/storage/transactions.py | 50 ++++++++++++++----------- 3 files changed, 85 insertions(+), 46 deletions(-) diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index 589a3f581b..0cb632fb08 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -728,7 +728,7 @@ class _TransactionQueue(object): self.pending_pdus_by_dest.setdefault(destination, []).append( (pdu, deferred, order) ) - + def eb(failure): if not deferred.called: deferred.errback(failure) @@ -745,7 +745,7 @@ class _TransactionQueue(object): # NO inlineCallbacks def enqueue_edu(self, edu): destination = edu.destination - + if destination == self.server_name: return @@ -776,7 +776,7 @@ class _TransactionQueue(object): ) yield deferred - + @defer.inlineCallbacks @log_function def _attempt_new_transaction(self, destination): @@ -790,12 +790,15 @@ class _TransactionQueue(object): retry_timings.retry_last_ts, retry_timings.retry_interval ) if retry_last_ts + retry_interval > int(self._clock.time_msec()): - logger.info("TX [%s] not ready for retry yet - " - "dropping transaction for now", destination) + logger.info( + "TX [%s] not ready for retry yet - " + "dropping transaction for now", + destination, + ) return else: logger.info("TX [%s] is ready for retry", destination) - + if destination in self.pending_transactions: # XXX: pending_transactions can get stuck on by a never-ending # request at which point pending_pdus_by_dest just keeps growing. @@ -811,10 +814,14 @@ class _TransactionQueue(object): if not pending_pdus and not pending_edus and not pending_failures: return - logger.debug("TX [%s] Attempting new transaction " - "(pdus: %d, edus: %d, failures: %d)", + logger.debug( + "TX [%s] Attempting new transaction " + "(pdus: %d, edus: %d, failures: %d)", destination, - len(pending_pdus), len(pending_edus), len(pending_failures)) + len(pending_pdus), + len(pending_edus), + len(pending_failures) + ) # Sort based on the order field pending_pdus.sort(key=lambda t: t[2]) @@ -847,8 +854,11 @@ class _TransactionQueue(object): yield self.transaction_actions.prepare_to_send(transaction) logger.debug("TX [%s] Persisted transaction", destination) - logger.info("TX [%s] Sending transaction [%s]", destination, - transaction.transaction_id) + logger.info( + "TX [%s] Sending transaction [%s]", + destination, + transaction.transaction_id, + ) # Actually send the transaction @@ -905,11 +915,14 @@ class _TransactionQueue(object): except Exception as e: # We capture this here as there as nothing actually listens # for this finishing functions deferred. - logger.warn("TX [%s] Problem in _attempt_transaction: %s", - destination, e) + logger.warn( + "TX [%s] Problem in _attempt_transaction: %s", + destination, + e, + ) self.set_retrying(destination, retry_interval) - + for deferred in deferreds: if not deferred.called: deferred.errback(e) @@ -925,12 +938,17 @@ class _TransactionQueue(object): def set_retrying(self, destination, retry_interval): # track that this destination is having problems and we should # give it a chance to recover before trying it again + if retry_interval: retry_interval *= 2 # plateau at hourly retries for now if retry_interval >= 60 * 60 * 1000: retry_interval = 60 * 60 * 1000 else: - retry_interval = 2000 # try again at first after 2 seconds - yield self.store.set_destination_retry_timings(destination, - int(self._clock.time_msec()), retry_interval) + retry_interval = 2000 # try again at first after 2 seconds + + yield self.store.set_destination_retry_timings( + destination, + int(self._clock.time_msec()), + retry_interval + ) diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 8fc6bf8f97..16fb2adab5 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -130,12 +130,20 @@ class MatrixFederationHttpClient(object): break except Exception as e: if not retry_on_dns_fail and isinstance(e, DNSLookupError): - logger.warn("DNS Lookup failed to %s with %s", destination, - e) + logger.warn( + "DNS Lookup failed to %s with %s", + destination, + e + ) raise SynapseError(400, "Domain specified not found.") - logger.warn("Sending request failed to %s: %s %s : %s", - destination, method, url_bytes, e) + logger.warn( + "Sending request failed to %s: %s %s : %s", + destination, + method, + url_bytes, + e + ) _print_ex(e) if retries_left: @@ -144,10 +152,15 @@ class MatrixFederationHttpClient(object): else: raise - logger.info("Received response %d %s for %s: %s %s", - response.code, response.phrase, - destination, method, url_bytes) - + logger.info( + "Received response %d %s for %s: %s %s", + response.code, + response.phrase, + destination, + method, + url_bytes + ) + if 200 <= response.code < 300: # We need to update the transactions table to say it was sent? pass diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index 2b16787695..423cc3f02a 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -27,7 +27,7 @@ logger = logging.getLogger(__name__) class TransactionStore(SQLBaseStore): """A collection of queries for handling PDUs. """ - + # a write-through cache of DestinationsTable.EntryType indexed by # destination string destination_retry_cache = {} @@ -213,21 +213,21 @@ class TransactionStore(SQLBaseStore): def get_destination_retry_timings(self, destination): """Gets the current retry timings (if any) for a given destination. - + Args: destination (str) - + Returns: None if not retrying Otherwise a DestinationsTable.EntryType for the retry scheme """ if destination in self.destination_retry_cache: return defer.succeed(self.destination_retry_cache[destination]) - + return self.runInteraction( "get_destination_retry_timings", self._get_destination_retry_timings, destination) - + def _get_destination_retry_timings(cls, txn, destination): query = DestinationsTable.select_statement("destination = ?") txn.execute(query, (destination,)) @@ -238,30 +238,36 @@ class TransactionStore(SQLBaseStore): return result else: return None - + def set_destination_retry_timings(self, destination, retry_last_ts, retry_interval): """Sets the current retry timings for a given destination. Both timings should be zero if retrying is no longer occuring. - + Args: destination (str) retry_last_ts (int) - time of last retry attempt in unix epoch ms retry_interval (int) - how long until next retry in ms """ - + self.destination_retry_cache[destination] = ( - DestinationsTable.EntryType(destination, - retry_last_ts, retry_interval) + DestinationsTable.EntryType( + destination, + retry_last_ts, + retry_interval + ) ) - + # XXX: we could chose to not bother persisting this if our cache thinks # this is a NOOP return self.runInteraction( "set_destination_retry_timings", - self._set_destination_retry_timings, destination, - retry_last_ts, retry_interval) - + self._set_destination_retry_timings, + destination, + retry_last_ts, + retry_interval, + ) + def _set_destination_retry_timings(cls, txn, destination, retry_last_ts, retry_interval): @@ -275,21 +281,22 @@ class TransactionStore(SQLBaseStore): def get_destinations_needing_retry(self): """Get all destinations which are due a retry for sending a transaction. - + Returns: list: A list of `DestinationsTable.EntryType` """ - + return self.runInteraction( "get_destinations_needing_retry", self._get_destinations_needing_retry ) - + def _get_destinations_needing_retry(cls, txn): where = "retry_last_ts > 0 and retry_next_ts < now()" query = DestinationsTable.select_statement(where) txn.execute(query) - return DestinationsTable.decode_results(txn.fetchall()) + return DestinationsTable.decode_results(txn.fetchall()) + class ReceivedTransactionsTable(Table): table_name = "received_transactions" @@ -332,14 +339,15 @@ class TransactionsToPduTable(Table): ] EntryType = namedtuple("TransactionsToPduEntry", fields) - + + class DestinationsTable(Table): table_name = "destinations" - + fields = [ "destination", "retry_last_ts", "retry_interval", ] - EntryType = namedtuple("DestinationsEntry", fields) \ No newline at end of file + EntryType = namedtuple("DestinationsEntry", fields) From f26ec14b2167a2a52e30f3e6d52f50cbb5cdbf52 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 10 Dec 2014 10:25:21 +0000 Subject: [PATCH 057/158] Remove whitespace --- tests/handlers/test_typing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py index 2f170ac3bf..7b390e4346 100644 --- a/tests/handlers/test_typing.py +++ b/tests/handlers/test_typing.py @@ -167,7 +167,7 @@ class TypingNotificationsTestCase(unittest.TestCase): @defer.inlineCallbacks def test_started_typing_remote_send(self): self.room_members = [self.u_apple, self.u_onion] - + put_json = self.mock_http_client.put_json put_json.expect_call_and_return( call("farm", From 08aceea82e834119a0152198edef738d8350cba1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 10 Dec 2014 10:26:12 +0000 Subject: [PATCH 058/158] Add newline back in --- tests/handlers/test_typing.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py index 7b390e4346..7e6ed9a42f 100644 --- a/tests/handlers/test_typing.py +++ b/tests/handlers/test_typing.py @@ -51,6 +51,7 @@ class JustTypingNotificationHandlers(object): def __init__(self, hs): self.typing_notification_handler = TypingNotificationHandler(hs) + class TypingNotificationsTestCase(unittest.TestCase): """Tests typing notifications to rooms.""" def setUp(self): From 0f4dcab238b029407080cb02cc2cf14e22d8fe89 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 10 Dec 2014 10:28:27 +0000 Subject: [PATCH 059/158] turn back on per-request transaction retries, so that every time we try to hit a dead server we actually end up hammering 5 times :| --- synapse/http/matrixfederationclient.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 16fb2adab5..fc5b5ab809 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -101,10 +101,9 @@ class MatrixFederationHttpClient(object): ] ) - # was 5; for now, let's only try once at the HTTP layer and then - # rely on transaction-layer retries for exponential backoff and - # getting the message through. - retries_left = 0 + # XXX: Would be much nicer to retry only at the transaction-layer + # (once we have reliable transactions in place) + retries_left = 5 endpoint = self._getEndpoint(reactor, destination) From 95aa903ffa77effcbca2a510744c3c3fa9b46ed3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 10 Dec 2014 10:06:12 +0000 Subject: [PATCH 060/158] Try and figure out how and why signatures are being changed. --- synapse/crypto/event_signing.py | 3 +- synapse/events/__init__.py | 18 ++++++----- synapse/events/builder.py | 7 ++-- synapse/federation/replication.py | 9 ++++-- synapse/handlers/federation.py | 54 +++++++++++++++++++++++++++++-- synapse/handlers/message.py | 1 - synapse/handlers/room.py | 7 ++-- synapse/state.py | 11 +++++++ synapse/storage/__init__.py | 10 ------ 9 files changed, 86 insertions(+), 34 deletions(-) diff --git a/synapse/crypto/event_signing.py b/synapse/crypto/event_signing.py index 15de0f5ae3..21c19c971d 100644 --- a/synapse/crypto/event_signing.py +++ b/synapse/crypto/event_signing.py @@ -82,8 +82,9 @@ def compute_event_signature(event, signature_name, signing_key): redact_json = tmp_event.get_pdu_json() redact_json.pop("age_ts", None) redact_json.pop("unsigned", None) - logger.debug("Signing event: %s", redact_json) + logger.debug("Signing event: %s", encode_canonical_json(redact_json)) redact_json = sign_json(redact_json, signature_name, signing_key) + logger.debug("Signed event: %s", encode_canonical_json(redact_json)) return redact_json["signatures"] diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index 230daf30d6..ed02138706 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -15,6 +15,8 @@ from frozendict import frozendict +import copy + def _freeze(o): if isinstance(o, dict) or isinstance(o, frozendict): @@ -48,7 +50,7 @@ def _unfreeze(o): class _EventInternalMetadata(object): def __init__(self, internal_metadata_dict): - self.__dict__ = internal_metadata_dict + self.__dict__ = copy.deepcopy(internal_metadata_dict) def get_dict(self): return dict(self.__dict__) @@ -74,10 +76,10 @@ def _event_dict_property(key): class EventBase(object): def __init__(self, event_dict, signatures={}, unsigned={}, internal_metadata_dict={}): - self.signatures = signatures - self.unsigned = unsigned + self.signatures = copy.deepcopy(signatures) + self.unsigned = copy.deepcopy(unsigned) - self._event_dict = event_dict + self._event_dict = copy.deepcopy(event_dict) self.internal_metadata = _EventInternalMetadata( internal_metadata_dict @@ -131,11 +133,11 @@ class EventBase(object): class FrozenEvent(EventBase): - def __init__(self, event_dict, signatures={}, unsigned={}): - event_dict = dict(event_dict) + def __init__(self, event_dict): + event_dict = copy.deepcopy(event_dict) - signatures.update(event_dict.pop("signatures", {})) - unsigned.update(event_dict.pop("unsigned", {})) + signatures = copy.deepcopy(event_dict.pop("signatures", {})) + unsigned = copy.deepcopy(event_dict.pop("unsigned", {})) frozen_dict = _freeze(event_dict) diff --git a/synapse/events/builder.py b/synapse/events/builder.py index 127b8fa904..642264e9f3 100644 --- a/synapse/events/builder.py +++ b/synapse/events/builder.py @@ -54,10 +54,9 @@ class EventBuilderFactory(object): return e_id.to_string() def new(self, key_values={}): - if "event_id" not in key_values: - key_values["event_id"] = self.create_event_id() + key_values["event_id"] = self.create_event_id() - time_now = self.clock.time_msec() + time_now = int(self.clock.time_msec()) key_values.setdefault("origin", self.hostname) key_values.setdefault("origin_server_ts", time_now) @@ -66,4 +65,6 @@ class EventBuilderFactory(object): age = key_values["unsigned"].pop("age", 0) key_values["unsigned"].setdefault("age_ts", time_now - age) + key_values["signatures"] = {} + return EventBuilder(key_values=key_values,) diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index d4cd79b7ac..a4600b0b40 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -25,6 +25,7 @@ from .persistence import TransactionActions from synapse.util.logutils import log_function from synapse.util.logcontext import PreserveLoggingContext +from synapse.events import FrozenEvent import logging @@ -439,7 +440,9 @@ class ReplicationLayer(object): @defer.inlineCallbacks def on_send_join_request(self, origin, content): + logger.debug("on_send_join_request: content: %s", content) pdu = self.event_from_pdu_json(content) + logger.debug("on_send_join_request: pdu sigs: %s", pdu.signatures) res_pdus = yield self.handler.on_send_join_request(origin, pdu) time_now = self._clock.time_msec() defer.returnValue((200, { @@ -665,13 +668,13 @@ class ReplicationLayer(object): return "" % self.server_name def event_from_pdu_json(self, pdu_json, outlier=False): - builder = self.event_builder_factory.new( + event = FrozenEvent( pdu_json ) - builder.internal_metadata.outlier = outlier + event.internal_metadata.outlier = outlier - return builder.build() + return event class _TransactionQueue(object): diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 827c86c9da..9ae3e5eca4 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -459,10 +459,22 @@ class FederationHandler(BaseHandler): """ event = pdu + logger.debug( + "on_send_join_request: Got event: %s, signatures: %s", + event.event_id, + event.signatures, + ) + event.internal_metadata.outlier = False context = yield self._handle_new_event(event) + logger.debug( + "on_send_join_request: After _handle_new_event: %s, sigs: %s", + event.event_id, + event.signatures, + ) + extra_users = [] if event.type == RoomMemberEvent.TYPE: target_user_id = event.state_key @@ -496,6 +508,12 @@ class FederationHandler(BaseHandler): "Failed to get destination from event %s", s.event_id ) + logger.debug( + "on_send_join_request: Sending event: %s, signatures: %s", + event.event_id, + event.signatures, + ) + yield self.replication_layer.send_pdu(new_pdu, destinations) auth_chain = yield self.store.get_auth_chain(event.event_id) @@ -652,12 +670,23 @@ class FederationHandler(BaseHandler): def _handle_new_event(self, event, state=None, backfilled=False, current_state=None, fetch_missing=True): context = EventContext() + + logger.debug( + "_handle_new_event: Before annotate: %s, sigs: %s", + event.event_id, event.signatures, + ) + yield self.state_handler.annotate_context_with_state( event, context, old_state=state ) + logger.debug( + "_handle_new_event: Before auth fetch: %s, sigs: %s", + event.event_id, event.signatures, + ) + is_new_state = not event.internal_metadata.outlier known_ids = set( @@ -666,29 +695,43 @@ class FederationHandler(BaseHandler): for e_id, _ in event.auth_events: if e_id not in known_ids: e = yield self.store.get_event( - e_id, - allow_none=True, + e_id, allow_none=True, ) if not e: # TODO: Do some conflict res to make sure that we're # not the ones who are wrong. logger.info( - "Rejecting %s as %s not in %s", + "Rejecting %s as %s not in db or %s", event.event_id, e_id, known_ids, ) raise AuthError(403, "Auth events are stale") context.auth_events[(e.type, e.state_key)] = e + logger.debug( + "_handle_new_event: Before hack: %s, sigs: %s", + event.event_id, event.signatures, + ) + if event.type == RoomMemberEvent.TYPE and not event.auth_events: if len(event.prev_events) == 1: c = yield self.store.get_event(event.prev_events[0][0]) if c.type == RoomCreateEvent.TYPE: context.auth_events[(c.type, c.state_key)] = c + logger.debug( + "_handle_new_event: Before auth check: %s, sigs: %s", + event.event_id, event.signatures, + ) + self.auth.check(event, auth_events=context.auth_events) + logger.debug( + "_handle_new_event: Before persist_event: %s, sigs: %s", + event.event_id, event.signatures, + ) + yield self.store.persist_event( event, context=context, @@ -697,4 +740,9 @@ class FederationHandler(BaseHandler): current_state=current_state, ) + logger.debug( + "_handle_new_event: After persist_event: %s, sigs: %s", + event.event_id, event.signatures, + ) + defer.returnValue(context) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 8ee560d79a..13fa0be7b4 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -137,7 +137,6 @@ class MessageHandler(BaseHandler): def handle_event(self, event_dict): builder = self.event_builder_factory.new(event_dict) - if builder.type == EventTypes.Member: membership = builder.content.get("membership", None) if membership == Membership.JOIN: diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 509763ebc7..93732a9c87 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -436,19 +436,16 @@ class RoomMemberHandler(BaseHandler): else: should_do_dance = False - have_joined = False if should_do_dance: handler = self.hs.get_handlers().federation_handler - have_joined = yield handler.do_invite_join( + yield handler.do_invite_join( room_host, room_id, event.user_id, event.get_dict()["content"], # FIXME To get a non-frozen dict context ) - - # We want to do the _do_update inside the room lock. - if not have_joined: + else: logger.debug("Doing normal join") yield self._do_local_membership_update( diff --git a/synapse/state.py b/synapse/state.py index ebec0ad9dc..7fdf596006 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -144,6 +144,17 @@ class StateHandler(object): (s.type, s.state_key): s for s in old_state } context.state_group = None + + if hasattr(event, "auth_events") and event.auth_events: + auth_ids = zip(*event.auth_events)[0] + context.auth_events = { + k: v + for k, v in context.current_state.items() + if v.event_id in auth_ids + } + else: + context.auth_events = {} + defer.returnValue([]) if event.is_state(): diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 7a09c33613..07b1665bf6 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -312,16 +312,6 @@ class DataStore(RoomMemberStore, RoomStore, txn, event.event_id, hash_alg, hash_bytes, ) - if hasattr(event, "signatures"): - logger.debug("sigs: %s", event.signatures) - for name, sigs in event.signatures.items(): - for key_id, signature_base64 in sigs.items(): - signature_bytes = decode_base64(signature_base64) - self._store_event_signature_txn( - txn, event.event_id, name, key_id, - signature_bytes, - ) - for prev_event_id, prev_hashes in event.prev_events: for alg, hash_base64 in prev_hashes.items(): hash_bytes = decode_base64(hash_base64) From 102d2373b4c5e46252ca9e2845a2415c5d00d193 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 10 Dec 2014 11:38:08 +0000 Subject: [PATCH 061/158] Add __str__ to FrozenEvent --- synapse/events/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index ed02138706..c43367fa20 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -160,3 +160,8 @@ class FrozenEvent(EventBase): def get_dict(self): # We need to unfreeze what we return return _unfreeze(super(FrozenEvent, self).get_dict()) + + def __str__(self): + return "" % ( + self.event_id, self.type, self.state_key, + ) From 018443cb5914f8c4a5488397cc1c19689e949aff Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 10 Dec 2014 11:59:53 +0000 Subject: [PATCH 062/158] Make depth increase. --- synapse/handlers/_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index 2c737c8bf9..6b5f6e7cd8 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -70,7 +70,7 @@ class BaseHandler(object): ) if latest_ret: - depth = max([d for _, _, d in latest_ret]) + depth = max([d for _, _, d in latest_ret]) + 1 else: depth = 1 From 02db1fd2e7cfbd9741684a0a875789cb5d599f54 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 10 Dec 2014 12:00:05 +0000 Subject: [PATCH 063/158] Fix AttributeError --- synapse/events/__init__.py | 3 +++ synapse/handlers/federation.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index c43367fa20..5f41933174 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -55,6 +55,9 @@ class _EventInternalMetadata(object): def get_dict(self): return dict(self.__dict__) + def is_outlier(self): + return hasattr(self, "outlier") and self.outlier + def _event_dict_property(key): def getter(self): diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 9ae3e5eca4..f38d9a48c8 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -687,7 +687,7 @@ class FederationHandler(BaseHandler): event.event_id, event.signatures, ) - is_new_state = not event.internal_metadata.outlier + is_new_state = not event.internal_metadata.is_outlier() known_ids = set( [s.event_id for s in context.auth_events.values()] From 32bc2b4fc1cd63e3011fcfd36012daf24c650838 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 10 Dec 2014 13:11:43 +0000 Subject: [PATCH 064/158] update codestyle based on debate on #matrix-dev --- docs/code_style.rst | 37 ++++++++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/docs/code_style.rst b/docs/code_style.rst index d7e2d5e69e..6e53f3eb09 100644 --- a/docs/code_style.rst +++ b/docs/code_style.rst @@ -1,10 +1,14 @@ Basically, PEP8 -- Max line width: 80 chars. +- NEVER tabs. 4 spaces to indent. +- Max line width: 79 chars (with flexibility to overflow by a "few chars" if + the overflowing content is not semantically significant and avoids an + explosion of vertical whitespace). - Use camel case for class and type names - Use underscores for functions and variables. - Use double quotes. -- Use parentheses instead of '\' for line continuation where ever possible (which is pretty much everywhere) +- Use parentheses instead of '\\' for line continuation where ever possible + (which is pretty much everywhere) - There should be max a single new line between: - statements - functions in a class @@ -14,5 +18,32 @@ Basically, PEP8 - a single space after a comma - a single space before and after for '=' when used as assignment - no spaces before and after for '=' for default values and keyword arguments. +- Indenting must follow PEP8; either hanging indent or multiline-visual indent + depending on the size and shape of the arguments and what makes more sense to + the author. In other words, both this:: -Comments should follow the google code style. This is so that we can generate documentation with sphinx (http://sphinxcontrib-napoleon.readthedocs.org/en/latest/) + print("I am a fish %s", "moo") + + and this:: + + print("I am a fish %s", + "moo") + + and this:: + + print( + "I am a fish %s", + "moo" + ) + + ...are valid, although given each one takes up 2x more vertical space than + the previous, it's up to the author's discretion as to which layout makes most + sense for their function invocation. (e.g. if they want to add comments + per-argument, or put expressions in the arguments, or group related arguments + together, or want to deliberately extend or preserve vertical/horizontal + space) + +Comments should follow the google code style. This is so that we can generate +documentation with sphinx (http://sphinxcontrib-napoleon.readthedocs.org/en/latest/) + +Code should pass pep8 --max-line-length=100 without any warnings. From 8ffbb52eeea4827220fc851189d95ebed824285a Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 10 Dec 2014 13:43:34 +0000 Subject: [PATCH 065/158] oops --- docs/code_style.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/code_style.rst b/docs/code_style.rst index 6e53f3eb09..dc40a7ab7b 100644 --- a/docs/code_style.rst +++ b/docs/code_style.rst @@ -22,17 +22,17 @@ Basically, PEP8 depending on the size and shape of the arguments and what makes more sense to the author. In other words, both this:: - print("I am a fish %s", "moo") + print("I am a fish %s" % "moo") and this:: - print("I am a fish %s", + print("I am a fish %s" % "moo") and this:: print( - "I am a fish %s", + "I am a fish %s" % "moo" ) From 02db7eb209af879a0168a371b4cb1c2ad0fcab49 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 10 Dec 2014 14:02:48 +0000 Subject: [PATCH 066/158] Fix bug when uploading state with empty state_key --- synapse/events/__init__.py | 10 ++++-- synapse/handlers/_base.py | 66 +++-------------------------------- synapse/handlers/message.py | 15 -------- synapse/rest/room.py | 20 ++++++----- synapse/storage/__init__.py | 4 ++- synapse/storage/schema/im.sql | 2 ++ 6 files changed, 29 insertions(+), 88 deletions(-) diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index 5f41933174..bd3ffb59cc 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -118,6 +118,9 @@ class EventBase(object): return d + def get(self, key, default): + return self._event_dict.get(key, default) + def get_internal_metadata_dict(self): return self.internal_metadata.get_dict() @@ -165,6 +168,9 @@ class FrozenEvent(EventBase): return _unfreeze(super(FrozenEvent, self).get_dict()) def __str__(self): + return self.__repr__() + + def __repr__(self): return "" % ( - self.event_id, self.type, self.state_key, - ) + self.event_id, self.type, self.get("state_key", None), + ) \ No newline at end of file diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index 6b5f6e7cd8..fdd3151877 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -97,6 +97,11 @@ class BaseHandler(object): event = builder.build() + logger.debug( + "Created event %s with auth_events: %s, current state: %s", + event.event_id, context.auth_events, context.current_state, + ) + defer.returnValue( (event, context,) ) @@ -147,64 +152,3 @@ class BaseHandler(object): None, destinations=destinations, ) - - # @defer.inlineCallbacks - # def _on_new_room_event(self, event, snapshot, extra_destinations=[], - # extra_users=[], suppress_auth=False, - # do_invite_host=None): - # yield run_on_reactor() - # - # snapshot.fill_out_prev_events(event) - # - # yield self.state_handler.annotate_event_with_state(event) - # - # yield self.auth.add_auth_events(event) - # - # logger.debug("Signing event...") - # - # add_hashes_and_signatures( - # event, self.server_name, self.signing_key - # ) - # - # logger.debug("Signed event.") - # - # if not suppress_auth: - # logger.debug("Authing...") - # self.auth.check(event, auth_events=event.old_state_events) - # logger.debug("Authed") - # else: - # logger.debug("Suppressed auth.") - # - # if do_invite_host: - # federation_handler = self.hs.get_handlers().federation_handler - # invite_event = yield federation_handler.send_invite( - # do_invite_host, - # event - # ) - # - # # FIXME: We need to check if the remote changed anything else - # event.signatures = invite_event.signatures - # - # yield self.store.persist_event(event) - # - # destinations = set(extra_destinations) - # # Send a PDU to all hosts who have joined the room. - # - # for k, s in event.state_events.items(): - # try: - # if k[0] == RoomMemberEvent.TYPE: - # if s.content["membership"] == Membership.JOIN: - # destinations.add( - # self.hs.parse_userid(s.state_key).domain - # ) - # except: - # logger.warn( - # "Failed to get destination from event %s", s.event_id - # ) - # - # event.destinations = list(destinations) - # - # yield self.notifier.on_new_room_event(event, extra_users=extra_users) - # - # federation_handler = self.hs.get_handlers().federation_handler - # yield federation_handler.handle_new_event(event, snapshot) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 13fa0be7b4..9043b945e4 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -165,21 +165,6 @@ class MessageHandler(BaseHandler): defer.returnValue(event) - @defer.inlineCallbacks - def store_room_data(self, event=None): - """ Stores data for a room. - - Args: - event : The room path event - stamp_event (bool) : True to stamp event content with server keys. - Raises: - SynapseError if something went wrong. - """ - - snapshot = yield self.store.snapshot_room(event) - - yield self._on_new_room_event(event, snapshot) - @defer.inlineCallbacks def get_room_data(self, user_id=None, room_id=None, event_type=None, state_key=""): diff --git a/synapse/rest/room.py b/synapse/rest/room.py index 3d78b4ff5c..22e1244e57 100644 --- a/synapse/rest/room.py +++ b/synapse/rest/room.py @@ -147,16 +147,18 @@ class RoomStateEventRestServlet(RestServlet): content = _parse_json(request) + event_dict = { + "type": event_type, + "content": content, + "room_id": urllib.unquote(room_id), + "sender": user.to_string(), + } + + if state_key is not None: + event_dict["state_key"] = urllib.unquote(state_key) + msg_handler = self.handlers.message_handler - yield msg_handler.handle_event( - { - "type": event_type, - "content": content, - "room_id": room_id, - "sender": user.to_string(), - "state_key": urllib.unquote(state_key), - } - ) + yield msg_handler.handle_event(event_dict) defer.returnValue((200, {})) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 7068424441..f8d895082d 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -39,6 +39,7 @@ from .state import StateStore from .signatures import SignatureStore from syutil.base64util import decode_base64 +from syutil.jsonutil import encode_canonical_json from synapse.crypto.event_signing import compute_event_reference_hash @@ -162,7 +163,8 @@ class DataStore(RoomMemberStore, RoomStore, table="event_json", values={ "event_id": event.event_id, - "json": json.dumps(event_dict, separators=(',', ':')), + "room_id": event.room_id, + "json": encode_canonical_json(event_dict), }, or_replace=True, ) diff --git a/synapse/storage/schema/im.sql b/synapse/storage/schema/im.sql index cb0c494ddf..0300bb29e1 100644 --- a/synapse/storage/schema/im.sql +++ b/synapse/storage/schema/im.sql @@ -35,11 +35,13 @@ CREATE INDEX IF NOT EXISTS events_room_id ON events (room_id); CREATE TABLE IF NOT EXISTS event_json( event_id TEXT NOT NULL, + room_id TEXT NOT NULL, json BLOB NOT NULL, CONSTRAINT ev_j_uniq UNIQUE (event_id) ); CREATE INDEX IF NOT EXISTS event_json_id ON event_json(event_id); +CREATE INDEX IF NOT EXISTS event_json_room_id ON event_json(room_id); CREATE TABLE IF NOT EXISTS state_events( From cabead6194c2946c2d05d6662c7ffe237d60291d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 10 Dec 2014 14:49:52 +0000 Subject: [PATCH 067/158] Actually fix bug when uploading state with empty state_key --- synapse/events/__init__.py | 1 + synapse/rest/room.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index bd3ffb59cc..7103b937af 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -94,6 +94,7 @@ class EventBase(object): event_id = _event_dict_property("event_id") hashes = _event_dict_property("hashes") origin = _event_dict_property("origin") + origin_server_ts = _event_dict_property("origin_server_ts") prev_events = _event_dict_property("prev_events") prev_state = _event_dict_property("prev_state") room_id = _event_dict_property("room_id") diff --git a/synapse/rest/room.py b/synapse/rest/room.py index 22e1244e57..c526e9bc72 100644 --- a/synapse/rest/room.py +++ b/synapse/rest/room.py @@ -117,10 +117,10 @@ class RoomStateEventRestServlet(RestServlet): self.on_PUT_no_state_key) def on_GET_no_state_key(self, request, room_id, event_type): - return self.on_GET(request, room_id, event_type, None) + return self.on_GET(request, room_id, event_type, "") def on_PUT_no_state_key(self, request, room_id, event_type): - return self.on_PUT(request, room_id, event_type, None) + return self.on_PUT(request, room_id, event_type, "") @defer.inlineCallbacks def on_GET(self, request, room_id, event_type, state_key): From cc84d3ea78eaf50c20ad84b3df99ecf4547e08a8 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 10 Dec 2014 14:46:55 +0000 Subject: [PATCH 068/158] Thumbnail uploaded and cached images --- synapse/media/v1/base_resource.py | 318 +++++++++++++++++++++++++ synapse/media/v1/download_resource.py | 153 ++---------- synapse/media/v1/filepath.py | 28 ++- synapse/media/v1/thumbnail_resource.py | 191 +++++++++++++++ synapse/media/v1/thumbnailer.py | 37 ++- synapse/media/v1/upload_resource.py | 21 +- synapse/storage/media_repository.py | 12 +- 7 files changed, 586 insertions(+), 174 deletions(-) create mode 100644 synapse/media/v1/base_resource.py create mode 100644 synapse/media/v1/thumbnail_resource.py diff --git a/synapse/media/v1/base_resource.py b/synapse/media/v1/base_resource.py new file mode 100644 index 0000000000..1e57a1465f --- /dev/null +++ b/synapse/media/v1/base_resource.py @@ -0,0 +1,318 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .thumbnailer import Thumbnailer + +from synapse.http.server import respond_with_json +from synapse.util.stringutils import random_string +from synapse.api.errors import ( + cs_exception, CodeMessageException, cs_error, Codes, SynapseError +) + +from twisted.internet import defer +from twisted.web.resource import Resource +from twisted.protocols.basic import FileSender + +import os + +import logging + +logger = logging.getLogger(__name__) + + +class BaseMediaResource(Resource): + isLeaf = True + + def __init__(self, hs, filepaths): + Resource.__init__(self) + self.client = hs.get_http_client() + self.clock = hs.get_clock() + self.server_name = hs.hostname + self.store = hs.get_datastore() + self.max_upload_size = hs.config.max_upload_size + self.filepaths = filepaths + + @staticmethod + def catch_errors(request_handler): + @defer.inlineCallbacks + def wrapped_request_handler(self, request): + try: + yield request_handler(self, request) + except CodeMessageException as e: + logger.exception(e) + respond_with_json( + request, e.code, cs_exception(e), send_cors=True + ) + except: + logger.exception( + "Failed handle request %s.%s on %r", + request_handler.__module__, + request_handler.__name__, + self, + ) + respond_with_json( + request, + 500, + {"error": "Internal server error"}, + send_cors=True + ) + return wrapped_request_handler + + @staticmethod + def _parse_media_id(request): + try: + server_name, media_id = request.postpath + return (server_name, media_id) + except: + raise SynapseError( + 404, + "Invalid media id token %r" % (request.postpath,), + Codes.UNKKOWN, + ) + + @staticmethod + def _parse_integer(request, arg_name, default=None): + try: + if default is None: + return int(request.args[arg_name][0]) + else: + return int(request.args.get(arg_name, [default])[0]) + except: + raise SynapseError( + 400, + "Missing integer argument %r" % (arg_name), + Codes.UNKNOWN, + ) + + @staticmethod + def _parse_string(request, arg_name, default=None): + try: + if default is None: + return request.args[arg_name][0] + else: + return request.args.get(arg_name, [default])[0] + except: + raise SynapseError( + 400, + "Missing string argument %r" % (arg_name), + Codes.UNKNOWN, + ) + + def _respond_404(self, request): + respond_with_json( + request, 404, + cs_error( + "Not found %r" % (request.postpath,), + code=Codes.NOT_FOUND, + ), + send_cors=True + ) + + @defer.inlineCallbacks + def _download_remote_file(self, server_name, media_id): + file_id = random_string(24) + + fname = self.filepaths.remote_media_filepath( + server_name, file_id + ) + os.makedirs(os.path.dirname(fname)) + + try: + with open(fname, "wb") as f: + request_path = "/".join(( + "/_matrix/media/v1/download", server_name, media_id, + )), + length, headers = yield self.client.get_file( + server_name, request_path, output_stream=f, + ) + media_type = headers["Content-Type"][0] + time_now_ms = self.clock.time_msec() + + yield self.store.store_cached_remote_media( + origin=server_name, + media_id=media_id, + media_type=media_type, + time_now_ms=self.clock.time_msec(), + upload_name=None, + media_length=length, + file_id=file_id, + ) + except: + os.remove(fname) + raise + + media_info = { + "media_type": media_type, + "media_length": length, + "upload_name": None, + "created_ts": time_now_ms, + "file_id": file_id, + } + + yield self._generate_remote_thumbnails( + server_name, media_id, media_info + ) + + defer.returnValue(media_info) + + @defer.inlineCallbacks + def _respond_with_file(self, request, media_type, file_path): + logger.debug("Responding with %r", file_path) + + if os.path.isfile(file_path): + request.setHeader(b"Content-Type", media_type.encode("UTF-8")) + + # cache for at least a day. + # XXX: we might want to turn this off for data we don't want to + # recommend caching as it's sensitive or private - or at least + # select private. don't bother setting Expires as all our + # clients are smart enough to be happy with Cache-Control + request.setHeader( + b"Cache-Control", b"public,max-age=86400,s-maxage=86400" + ) + + with open(file_path, "rb") as f: + yield FileSender().beginFileTransfer(f, request) + + request.finish() + else: + self._respond_404() + + def _get_thumbnail_requirements(self, media_type): + if media_type == "image/jpeg": + return ( + (32, 32, "crop", "image/jpeg"), + (96, 96, "crop", "image/jpeg"), + (320, 240, "scale", "image/jpeg"), + (640, 480, "scale", "image/jpeg"), + ) + elif (media_type == "image/png") or (media_type == "image/gif"): + return ( + (32, 32, "crop", "image/png"), + (96, 96, "crop", "image/png"), + (320, 240, "scale", "image/png"), + (640, 480, "scale", "image/png"), + ) + else: + return () + + @defer.inlineCallbacks + def _generate_local_thumbnails(self, media_id, media_info): + media_type = media_info["media_type"] + requirements = self._get_thumbnail_requirements(media_type) + if not requirements: + return + + input_path = self.filepaths.local_media_path(media_id) + thumbnailer = Thumbnailer(input_path) + m_width = thumbnailer.width + m_height = thumbnailer.height + scales = set() + crops = set() + for r_width, r_height, r_method, r_type in requirements: + if r_method == "scale": + t_width, t_height = thumbnailer.aspect(r_width, r_height) + scales.add(( + min(m_width, t_width), min(m_height, t_height), r_type, + )) + elif r_method == "crop": + crops.add((r_width, r_height, r_type)) + + for t_width, t_height, t_type in scales: + t_method = "scale" + t_path = self.filepaths.local_media_thumbnail( + media_id, t_width, t_height, t_type, t_method + ) + t_len = thumbnailer.scale(t_path, t_width, t_height, t_type) + yield self.store.store_local_thumbnail( + media_id, t_width, t_height, t_type, t_method, t_len + ) + + for t_width, t_height, t_type in crops: + if (t_width, t_height, t_type) in scales: + # If the aspect ratio of the cropped thumbnail matches a purely + # scaled one then there is no point in calculating a separate + # thumbnail. + continue + t_method = "crop" + t_path = self.filepaths.local_media_thumbnail( + media_id, t_width, t_height, t_type, t_method + ) + t_len = thumbnailer.crop(t_path, t_width, t_height, t_type) + yield self.store.store_local_thumbnail( + media_id, t_width, t_height, t_type, t_method, t_len + ) + + defer.returnValue({ + "width": m_width, + "height": m_height, + }) + + @defer.inlineCallbacks + def _generate_remote_thumbnails(self, server_name, media_id, media_info): + media_type = media_info["media_type"] + file_id = media_info["filesystem_id"] + requirements = self._get_requirements(media_type) + if not requirements: + return + + input_path = self.filepaths.remote_media_path(server_name, file_id) + thumbnailer = Thumbnailer(input_path) + m_width = thumbnailer.width + m_height = thumbnailer.height + scales = set() + crops = set() + for r_width, r_height, r_method, r_type in requirements: + if r_method == "scale": + t_width, t_height = thumbnailer.aspect(r_width, r_height) + scales.add(( + min(m_width, t_width), min(m_height, t_height), r_type, + )) + elif r_method == "crop": + crops.add((r_width, r_height, r_type)) + + for t_width, t_height, t_type in scales: + t_method = "scale" + t_path = self.filepaths.remote_media_thumbnail( + server_name, media_id, file_id, + media_id, t_width, t_height, t_type, t_method + ) + t_len = thumbnailer.scale(t_path, t_width, t_height, t_type) + yield self.store.store_remote_media_thumbnail( + server_name, media_id, file_id, + t_width, t_height, t_type, t_method, t_len + ) + + for t_width, t_height, t_type in crops: + if (t_width, t_height, t_type) in scales: + # If the aspect ratio of the cropped thumbnail matches a purely + # scaled one then there is no point in calculating a separate + # thumbnail. + continue + t_method = "crop" + t_path = self.filepaths.remote_media_thumbnail( + server_name, media_id, file_id, + t_width, t_height, t_type, t_method + ) + t_len = thumbnailer.crop(t_path, t_width, t_height, t_type) + yield self.store.store_remote_media_thumbnail( + server_name, media_id, file_id, + t_width, t_height, t_type, t_method, t_len + ) + + defer.returnValue({ + "width": m_width, + "height": m_height, + }) diff --git a/synapse/media/v1/download_resource.py b/synapse/media/v1/download_resource.py index c243f16a74..31c6f25968 100644 --- a/synapse/media/v1/download_resource.py +++ b/synapse/media/v1/download_resource.py @@ -13,117 +13,46 @@ # See the License for the specific language governing permissions and # limitations under the License. -from synapse.http.server import respond_with_json -from synapse.util.stringutils import random_string -from synapse.api.errors import ( - cs_exception, CodeMessageException, cs_error, Codes -) +from .base_media_resource import BaseMediaResource -from twisted.protocols.basic import FileSender -from twisted.web.resource import Resource from twisted.web.server import NOT_DONE_YET from twisted.internet import defer -import os - import logging logger = logging.getLogger(__name__) -class DownloadResource(Resource): - isLeaf = True - - def __init__(self, hs, filepaths): - Resource.__init__(self) - self.client = hs.get_http_client() - self.clock = hs.get_clock() - self.server_name = hs.hostname - self.store = hs.get_datastore() - self.filepaths = filepaths - +class DownloadResource(BaseMediaResource): def render_GET(self, request): self._async_render_GET(request) return NOT_DONE_YET - def _respond_404(self, request): - respond_with_json( - request, 404, - cs_error( - "Not found %r" % (request.postpath,), - code=Codes.NOT_FOUND, - ), - send_cors=True - ) - + @BaseMediaResource.catch_errors @defer.inlineCallbacks def _async_render_GET(self, request): - try: server_name, media_id = request.postpath except: self._respond_404(request) return - try: - if server_name == self.server_name: - yield self._respond_local_file(request, media_id) - else: - yield self._respond_remote_file(request, server_name, media_id) - except CodeMessageException as e: - logger.exception(e) - respond_with_json(request, e.code, cs_exception(e), send_cors=True) - except: - logger.exception("Failed to serve file") - respond_with_json( - request, - 500, - {"error": "Internal server error"}, - send_cors=True - ) + if server_name == self.server_name: + yield self._respond_local_file(request, media_id) + else: + yield self._respond_remote_file(request, server_name, media_id) @defer.inlineCallbacks - def _download_remote_file(self, server_name, media_id): - filesystem_id = random_string(24) + def _respond_local_file(self, request, media_id): + media_info = yield self.store.get_local_media(media_id) + if not media_info: + self._respond_404() + return - fname = self.filepaths.remote_media_filepath( - server_name, filesystem_id - ) - os.makedirs(os.path.dirname(fname)) + media_type = media_info["media_type"] + file_path = self.filepaths.local_media_filepath(media_id) - try: - with open(fname, "wb") as f: - length, headers = yield self.client.get_file( - server_name, - "/".join(( - "/_matrix/media/v1/download", server_name, media_id, - )), - output_stream=f, - ) - except: - os.remove(fname) - raise - - media_type = headers["Content-Type"][0] - time_now_ms = self.clock.time_msec() - - yield self.store.store_cached_remote_media( - origin=server_name, - media_id=media_id, - media_type=media_type, - time_now_ms=self.clock.time_msec(), - upload_name=None, - media_length=length, - filesystem_id=filesystem_id, - ) - - defer.returnValue({ - "media_type": media_type, - "media_length": length, - "upload_name": None, - "created_ts": time_now_ms, - "filesystem_id": filesystem_id, - }) + yield self.respond_with_file(request, media_type, file_path) @defer.inlineCallbacks def _respond_remote_file(self, request, server_name, media_id): @@ -136,59 +65,11 @@ class DownloadResource(Resource): server_name, media_id ) + media_type = media_info["media_type"] filesystem_id = media_info["filesystem_id"] file_path = self.filepaths.remote_media_filepath( server_name, filesystem_id ) - if os.path.isfile(file_path): - media_type = media_info["media_type"] - request.setHeader(b"Content-Type", media_type.encode("UTF-8")) - - # cache for at least a day. - # XXX: we might want to turn this off for data we don't want to - # recommend caching as it's sensitive or private - or at least - # select private. don't bother setting Expires as all our - # clients are smart enough to be happy with Cache-Control - request.setHeader( - b"Cache-Control", b"public,max-age=86400,s-maxage=86400" - ) - - with open(file_path, "rb") as f: - yield FileSender().beginFileTransfer(f, request) - - request.finish() - else: - self._respond_404() - - @defer.inlineCallbacks - def _respond_local_file(self, request, media_id): - media_info = yield self.store.get_local_media(media_id) - if not media_info: - self._respond_404() - return - - file_path = self.filepaths.local_media_filepath(media_id) - - logger.debug("Searching for %s", file_path) - - if os.path.isfile(file_path): - media_type = media_info["media_type"] - request.setHeader(b"Content-Type", media_type.encode("UTF-8")) - - # cache for at least a day. - # XXX: we might want to turn this off for data we don't want to - # recommend caching as it's sensitive or private - or at least - # select private. don't bother setting Expires as all our - # clients are smart enough to be happy with Cache-Control - request.setHeader( - b"Cache-Control", b"public,max-age=86400,s-maxage=86400" - ) - - with open(file_path, "rb") as f: - yield FileSender().beginFileTransfer(f, request) - - request.finish() - else: - self._respond_404() + yield self.respond_with_file(request, media_type, file_path) diff --git a/synapse/media/v1/filepath.py b/synapse/media/v1/filepath.py index d23564e03e..0078bc3d40 100644 --- a/synapse/media/v1/filepath.py +++ b/synapse/media/v1/filepath.py @@ -21,33 +21,47 @@ class MediaFilePaths(object): def __init__(self, base_path): self.base_path = base_path + def default_thumbnail(self, default_top_level, default_sub_type, width, + height, content_type, method): + top_level_type, sub_type = content_type.split("/") + file_name = "%i-%i-%s-%s-%s" % ( + width, height, top_level_type, sub_type, method + ) + return os.path.join( + self.base_path, "default_thumbnails", default_top_level, + default_sub_type, file_name + ) + def local_media_filepath(self, media_id): return os.path.join( - self.base_path, "local", "content", + self.base_path, "local_content", media_id[0:2], media_id[2:4], media_id[4:] ) - def local_media_thumbnail(self, media_id, width, height, content_type): + def local_media_thumbnail(self, media_id, width, height, content_type, + method): top_level_type, sub_type = content_type.split("/") - file_name = "%i-%i-%s-%s" % (width, height, top_level_type, sub_type) + file_name = "%i-%i-%s-%s-%s" % ( + width, height, top_level_type, sub_type, method + ) return os.path.join( - self.base_path, "local", "thumbnails", + self.base_path, "local_thumbnails", media_id[0:2], media_id[2:4], media_id[4:], file_name ) def remote_media_filepath(self, server_name, file_id): return os.path.join( - self.base_path, "remote", "content", server_name, + self.base_path, "remote_content", server_name, file_id[0:2], file_id[2:4], file_id[4:] ) def remote_media_thumbnail(self, server_name, file_id, width, height, - content_type): + content_type, method): top_level_type, sub_type = content_type.split("/") file_name = "%i-%i-%s-%s" % (width, height, top_level_type, sub_type) return os.path.join( - self.base_path, "remote", "content", server_name, + self.base_path, "remote_thumbnail", server_name, file_id[0:2], file_id[2:4], file_id[4:], file_name ) diff --git a/synapse/media/v1/thumbnail_resource.py b/synapse/media/v1/thumbnail_resource.py new file mode 100644 index 0000000000..331ba87e00 --- /dev/null +++ b/synapse/media/v1/thumbnail_resource.py @@ -0,0 +1,191 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .base_media_resource import BaseMediaResource + +from twisted.web.server import NOT_DONE_YET +from twisted.internet import defer + +import logging + +logger = logging.getLogger(__name__) + + +class ThumbnailResource(BaseMediaResource): + isLeaf = True + + def render_GET(self, request): + self._async_render_GET(request) + return NOT_DONE_YET + + @BaseMediaResource.catch_errors + @defer.inlineCallbacks + def _async_render_GET(self, request): + server_name, media_id = self._parse_media_id(request) + width = self._parse_integer(request, "width") + height = self._parse_integer(request, "height") + method = self._parse_string(request, "method", "scale") + m_type = self._parse_string(request, "type", "image/png") + + if server_name == self.server_name: + yield self._respond_local_thumbnail( + request, media_id, width, height, method, m_type + ) + else: + yield self._respond_remote_thumbnail( + request, server_name, media_id, + width, height, method, m_type + ) + + @defer.inlineCallbacks + def _respond_local_thumbnail(self, request, media_id, width, height, + method, m_type): + media_info = yield self.store.get_local_media(media_id) + + if not media_info: + self._respond_404(request) + return + + thumbnail_infos = yield self.store.get_local_thumbnail(media_id) + + if thumbnail_infos: + thumbnail_info = self._select_thumbnail( + width, height, method, m_type, thumbnail_infos + ) + thumbnail_width = thumbnail_info["thumbnail_width"] + thumbnail_height = thumbnail_info["thumbnail_height"] + thumbnail_type = thumbnail_info["thumbnail_type"] + thumbnail_method = thumbnail_info["thumbnail_method"] + + file_path = self.filepaths.local_media_thumbnail( + media_id, thumbnail_width, thumbnail_height, thumbnail_type, + thumbnail_method, + ) + yield self._respond_with_file(request, thumbnail_type, file_path) + + else: + yield self._respond_default_thumbnail( + self, request, media_info, width, height, method, m_type, + ) + + @defer.inlineCallbacks + def _respond_remote_thumbnail(self, request, server_name, media_id, width, + height, method, m_type): + media_info = yield self.store.get_cached_remote_media( + server_name, media_id + ) + + if not media_info: + # TODO: Don't download the whole remote file + # We should proxy the thumbnail from the remote server instead. + media_info = yield self._download_remote_file( + server_name, media_id + ) + + thumbnail_infos = yield self.store.get_remote_media_thumbnails( + server_name, media_id, + ) + + if thumbnail_infos: + thumbnail_info = self._select_thumbnail( + width, height, method, m_type, thumbnail_infos + ) + thumbnail_width = thumbnail_info["thumbnail_width"] + thumbnail_height = thumbnail_info["thumbnail_height"] + thumbnail_type = thumbnail_info["thumbnail_type"] + thumbnail_method = thumbnail_info["thumbnail_method"] + + file_path = self.filepaths.remote_media_thumbnail( + server_name, media_id, thumbnail_width, thumbnail_height, + thumbnail_type, thumbnail_method, + ) + yield self._respond_with_file(request, thumbnail_type, file_path) + else: + yield self._respond_default_thumbnail( + self, request, media_info, width, height, method, m_type, + ) + + @defer.inlineCallbacks + def _respond_default_thumbnail(self, request, media_info, width, height, + method, m_type): + media_type = media_info["media_type"] + top_level_type = media_type.split("/")[0] + sub_type = media_type.split("/")[-1].split(";")[0] + thumbnail_infos = yield self.store.get_default_thumbnails( + top_level_type, sub_type, + ) + if not thumbnail_infos: + thumbnail_infos = yield self.store.get_default_thumbnails( + top_level_type, "_default", + ) + if not thumbnail_infos: + thumbnail_infos = yield self.store.get_default_thumbnails( + "_default", "_default", + ) + if not thumbnail_infos: + self._respond_404(request) + return + + thumbnail_info = self._select_thumbnail( + width, height, "crop", m_type, thumbnail_infos + ) + + thumbnail_width = thumbnail_info["thumbnail_width"] + thumbnail_height = thumbnail_info["thumbnail_height"] + thumbnail_type = thumbnail_info["thumbnail_type"] + thumbnail_method = thumbnail_info["thumbnail_method"] + + file_path = self.filepaths.default_thumbnail( + top_level_type, sub_type, thumbnail_width, thumbnail_height, + thumbnail_type, thumbnail_method, + ) + yield self.respond_with_file(request, thumbnail_type, file_path) + + def _select_thumbnail(self, desired_width, desired_height, desired_method, + desired_type, thumbnail_infos): + d_w = desired_width + d_h = desired_height + + if desired_method.lower() == "crop": + info_list = [] + for info in thumbnail_infos: + t_w = info["thumbnail_width"] + t_h = info["thumbnail_height"] + t_method = info["thumnail_method"] + if t_method == "scale" or t_method == "crop": + aspect_quality = abs(d_w * t_h - d_h * t_w) + size_quality = abs((d_w - t_w) * (d_h - t_h)) + type_quality = desired_type != info["thumbnail_type"] + length_quality = info["thumbnail_length"] + info_list.append(( + aspect_quality, size_quality, type_quality, + length_quality, info + )) + return min(info_list)[-1] + else: + info_list = [] + for info in thumbnail_infos: + t_w = info["thumbnail_width"] + t_h = info["thumbnail_height"] + t_method = info["thumnail_method"] + if t_method == "scale" and (t_w >= d_w or t_h >= d_h): + size_quality = abs((d_w - t_w) * (d_h - t_h)) + type_quality = desired_type != info["thumbnail_type"] + length_quality = info["thumbnail_length"] + info_list.append(( + size_quality, type_quality, length_quality, info + )) + return min(info_list)[-1] diff --git a/synapse/media/v1/thumbnailer.py b/synapse/media/v1/thumbnailer.py index ed09283b27..47160721e7 100644 --- a/synapse/media/v1/thumbnailer.py +++ b/synapse/media/v1/thumbnailer.py @@ -13,18 +13,22 @@ # See the License for the specific language governing permissions and # limitations under the License. -import PIL.Image +import Image +from io import BytesIO + class Thumbnailer(object): - FORMAT_JPEG="JPEG" - FORMAT_PNG="PNG" + FORMATS = { + "image/jpeg": "JPEG", + "image/png": "PNG", + } def __init__(self, input_path): - self.image = PIL.Image.open(input_path) + self.image = Image.open(input_path) self.width, self.height = self.image.size - def size_preserve(self, max_width, max_height): + def aspect(self, max_width, max_height): """Calculate the largest size that preserves aspect ratio which fits within the given rectangle:: @@ -42,12 +46,12 @@ class Thumbnailer(object): else: return ((max_height * self.width) // self.height, max_height) - def thumbnail_scale(self, output_path, output_format, width, height): + def scale(self, output_path, width, height, output_type): """Rescales the image to the given dimensions""" - output = self.image.resize((width, height), PIL.Image.BILINEAR) - output.save(output_path, output_format) + scaled = self.image.resize((width, height), Image.BILINEAR) + return self.save_image(scaled, output_type, output_path) - def thumbnail_crop(self, output_path, output_format, width, height): + def crop(self, output_path, width, height, output_type): """Rescales and crops the image to the given dimensions preserving aspect:: (w_in / h_in) = (w_scaled / h_scaled) @@ -61,18 +65,25 @@ class Thumbnailer(object): if width * self.height > height * self.width: scaled_height = (width * self.height) // self.width scaled_image = self.image.resize( - (width, scaled_height), PIL.Image.BILINEAR + (width, scaled_height), Image.BILINEAR ) crop_top = (scaled_height - height) // 2 crop_bottom = height + crop_top cropped = scaled_image.crop((0, crop_top, width, crop_bottom)) - cropped.save(output_path, output_format) else: scaled_width = (height * self.width) // self.height scaled_image = self.image.resize( - (scaled_width, height), PIL.Image.BILINEAR + (scaled_width, height), Image.BILINEAR ) crop_left = (scaled_width - width) // 2 crop_right = width + crop_left cropped = scaled_image.crop((crop_left, 0, crop_right, height)) - cropped.save(output_path, output_format) + return self.save_image(cropped, output_type, output_path) + + def save_image(self, output_image, output_type, output_path): + output_bytes_io = BytesIO() + output_image.save(output_bytes_io, self.FORMATS[output_type]) + output_bytes = output_bytes_io.getvalue() + with open(output_path, "wb") as output_file: + output_file.write(output_bytes) + return len(output_bytes) diff --git a/synapse/media/v1/upload_resource.py b/synapse/media/v1/upload_resource.py index 91bcc5caff..a78cc3cff3 100644 --- a/synapse/media/v1/upload_resource.py +++ b/synapse/media/v1/upload_resource.py @@ -20,10 +20,11 @@ from synapse.api.errors import ( cs_exception, SynapseError, CodeMessageException ) -from twisted.web.resource import Resource from twisted.web.server import NOT_DONE_YET from twisted.internet import defer +from .baseresource import BaseMediaResource + import os import logging @@ -31,17 +32,7 @@ import logging logger = logging.getLogger(__name__) -class UploadResource(Resource): - isLeaf = True - - def __init__(self, hs, filepaths): - Resource.__init__(self) - self.auth = hs.get_auth() - self.clock = hs.get_clock() - self.store = hs.get_datastore() - self.max_upload_size = hs.config.max_upload_size - self.filepaths = filepaths - +class UploadResource(BaseMediaResource): def render_POST(self, request): self._async_render_POST(request) return NOT_DONE_YET @@ -99,6 +90,12 @@ class UploadResource(Resource): media_length=content_length, user_id=auth_user, ) + media_info = { + "media_type": media_type, + "media_length": content_length, + } + + yield self._generate_local_thumbnails(self, media_id, media_info) respond_with_json( request, 200, {"content_token": media_id}, send_cors=True diff --git a/synapse/storage/media_repository.py b/synapse/storage/media_repository.py index b3f1fc087b..a848662716 100644 --- a/synapse/storage/media_repository.py +++ b/synapse/storage/media_repository.py @@ -56,8 +56,8 @@ class MediaRepositoryStore(SQLBaseStore): ) def store_local_thumbnail(self, media_id, thumbnail_width, - thumbnail_height, thumbnail_method, - thumbnail_type, thumbnail_length): + thumbnail_height, thumbnail_type, + thumbnail_method, thumbnail_length): return self._simple_insert( "local_media_thumbnails", { @@ -108,10 +108,10 @@ class MediaRepositoryStore(SQLBaseStore): ) ) - def store_remote_media_thumbnail(self, origin, media_id, thumbnail_width, - thumbnail_height, thumbnail_method, - thumbnail_type, thumbnail_length, - filesystem_id): + def store_remote_media_thumbnail(self, origin, media_id, filesystem_id, + thumbnail_width, thumbnail_height, + thumbnail_type, thumbnail_method, + thumbnail_length): return self._simple_insert( "remote_media_cache_thumbnails", { From e5275d856ee7a1d7aeccd3ea6ab97b49456d24c9 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 10 Dec 2014 15:46:18 +0000 Subject: [PATCH 069/158] Get the code actually working --- synapse/media/v1/base_resource.py | 31 +++++++++------ synapse/media/v1/download_resource.py | 6 +-- synapse/media/v1/media_repository.py | 2 + synapse/media/v1/thumbnail_resource.py | 52 +++++++++++++------------- synapse/media/v1/upload_resource.py | 8 ++-- synapse/storage/media_repository.py | 14 ++++--- 6 files changed, 61 insertions(+), 52 deletions(-) diff --git a/synapse/media/v1/base_resource.py b/synapse/media/v1/base_resource.py index 1e57a1465f..81f2456343 100644 --- a/synapse/media/v1/base_resource.py +++ b/synapse/media/v1/base_resource.py @@ -37,6 +37,7 @@ class BaseMediaResource(Resource): def __init__(self, hs, filepaths): Resource.__init__(self) + self.auth = hs.get_auth() self.client = hs.get_http_client() self.clock = hs.get_clock() self.server_name = hs.hostname @@ -120,6 +121,12 @@ class BaseMediaResource(Resource): send_cors=True ) + @staticmethod + def _makedirs(filepath): + dirname = os.path.dirname(filepath) + if not os.path.exists(dirname): + os.makedirs(dirname) + @defer.inlineCallbacks def _download_remote_file(self, server_name, media_id): file_id = random_string(24) @@ -127,13 +134,13 @@ class BaseMediaResource(Resource): fname = self.filepaths.remote_media_filepath( server_name, file_id ) - os.makedirs(os.path.dirname(fname)) + self._makedirs(fname) try: with open(fname, "wb") as f: request_path = "/".join(( "/_matrix/media/v1/download", server_name, media_id, - )), + )) length, headers = yield self.client.get_file( server_name, request_path, output_stream=f, ) @@ -147,7 +154,7 @@ class BaseMediaResource(Resource): time_now_ms=self.clock.time_msec(), upload_name=None, media_length=length, - file_id=file_id, + filesystem_id=file_id, ) except: os.remove(fname) @@ -158,7 +165,7 @@ class BaseMediaResource(Resource): "media_length": length, "upload_name": None, "created_ts": time_now_ms, - "file_id": file_id, + "filesystem_id": file_id, } yield self._generate_remote_thumbnails( @@ -215,7 +222,7 @@ class BaseMediaResource(Resource): if not requirements: return - input_path = self.filepaths.local_media_path(media_id) + input_path = self.filepaths.local_media_filepath(media_id) thumbnailer = Thumbnailer(input_path) m_width = thumbnailer.width m_height = thumbnailer.height @@ -235,6 +242,7 @@ class BaseMediaResource(Resource): t_path = self.filepaths.local_media_thumbnail( media_id, t_width, t_height, t_type, t_method ) + self._makedirs(t_path) t_len = thumbnailer.scale(t_path, t_width, t_height, t_type) yield self.store.store_local_thumbnail( media_id, t_width, t_height, t_type, t_method, t_len @@ -250,6 +258,7 @@ class BaseMediaResource(Resource): t_path = self.filepaths.local_media_thumbnail( media_id, t_width, t_height, t_type, t_method ) + self._makedirs(t_path) t_len = thumbnailer.crop(t_path, t_width, t_height, t_type) yield self.store.store_local_thumbnail( media_id, t_width, t_height, t_type, t_method, t_len @@ -264,11 +273,11 @@ class BaseMediaResource(Resource): def _generate_remote_thumbnails(self, server_name, media_id, media_info): media_type = media_info["media_type"] file_id = media_info["filesystem_id"] - requirements = self._get_requirements(media_type) + requirements = self._get_thumbnail_requirements(media_type) if not requirements: return - input_path = self.filepaths.remote_media_path(server_name, file_id) + input_path = self.filepaths.remote_media_filepath(server_name, file_id) thumbnailer = Thumbnailer(input_path) m_width = thumbnailer.width m_height = thumbnailer.height @@ -286,9 +295,9 @@ class BaseMediaResource(Resource): for t_width, t_height, t_type in scales: t_method = "scale" t_path = self.filepaths.remote_media_thumbnail( - server_name, media_id, file_id, - media_id, t_width, t_height, t_type, t_method + server_name, file_id, t_width, t_height, t_type, t_method ) + self._makedirs(t_path) t_len = thumbnailer.scale(t_path, t_width, t_height, t_type) yield self.store.store_remote_media_thumbnail( server_name, media_id, file_id, @@ -303,9 +312,9 @@ class BaseMediaResource(Resource): continue t_method = "crop" t_path = self.filepaths.remote_media_thumbnail( - server_name, media_id, file_id, - t_width, t_height, t_type, t_method + server_name, file_id, t_width, t_height, t_type, t_method ) + self._makedirs(t_path) t_len = thumbnailer.crop(t_path, t_width, t_height, t_type) yield self.store.store_remote_media_thumbnail( server_name, media_id, file_id, diff --git a/synapse/media/v1/download_resource.py b/synapse/media/v1/download_resource.py index 31c6f25968..6de0932ba3 100644 --- a/synapse/media/v1/download_resource.py +++ b/synapse/media/v1/download_resource.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from .base_media_resource import BaseMediaResource +from .base_resource import BaseMediaResource from twisted.web.server import NOT_DONE_YET from twisted.internet import defer @@ -52,7 +52,7 @@ class DownloadResource(BaseMediaResource): media_type = media_info["media_type"] file_path = self.filepaths.local_media_filepath(media_id) - yield self.respond_with_file(request, media_type, file_path) + yield self._respond_with_file(request, media_type, file_path) @defer.inlineCallbacks def _respond_remote_file(self, request, server_name, media_id): @@ -72,4 +72,4 @@ class DownloadResource(BaseMediaResource): server_name, filesystem_id ) - yield self.respond_with_file(request, media_type, file_path) + yield self._respond_with_file(request, media_type, file_path) diff --git a/synapse/media/v1/media_repository.py b/synapse/media/v1/media_repository.py index e0a4cd01ee..2bd29d2288 100644 --- a/synapse/media/v1/media_repository.py +++ b/synapse/media/v1/media_repository.py @@ -15,6 +15,7 @@ from .upload_resource import UploadResource from .download_resource import DownloadResource +from .thumbnail_resource import ThumbnailResource from .filepath import MediaFilePaths from twisted.web.resource import Resource @@ -64,3 +65,4 @@ class MediaRepositoryResource(Resource): filepaths = MediaFilePaths(hs.config.media_store_path) self.putChild("upload", UploadResource(hs, filepaths)) self.putChild("download", DownloadResource(hs, filepaths)) + self.putChild("thumbnail", ThumbnailResource(hs, filepaths)) diff --git a/synapse/media/v1/thumbnail_resource.py b/synapse/media/v1/thumbnail_resource.py index 331ba87e00..fd08c7ecd2 100644 --- a/synapse/media/v1/thumbnail_resource.py +++ b/synapse/media/v1/thumbnail_resource.py @@ -14,7 +14,7 @@ # limitations under the License. -from .base_media_resource import BaseMediaResource +from .base_resource import BaseMediaResource from twisted.web.server import NOT_DONE_YET from twisted.internet import defer @@ -59,26 +59,25 @@ class ThumbnailResource(BaseMediaResource): self._respond_404(request) return - thumbnail_infos = yield self.store.get_local_thumbnail(media_id) + thumbnail_infos = yield self.store.get_local_media_thumbnails(media_id) if thumbnail_infos: thumbnail_info = self._select_thumbnail( width, height, method, m_type, thumbnail_infos ) - thumbnail_width = thumbnail_info["thumbnail_width"] - thumbnail_height = thumbnail_info["thumbnail_height"] - thumbnail_type = thumbnail_info["thumbnail_type"] - thumbnail_method = thumbnail_info["thumbnail_method"] + t_width = thumbnail_info["thumbnail_width"] + t_height = thumbnail_info["thumbnail_height"] + t_type = thumbnail_info["thumbnail_type"] + t_method = thumbnail_info["thumbnail_method"] file_path = self.filepaths.local_media_thumbnail( - media_id, thumbnail_width, thumbnail_height, thumbnail_type, - thumbnail_method, + media_id, t_width, t_height, t_type, t_method, ) - yield self._respond_with_file(request, thumbnail_type, file_path) + yield self._respond_with_file(request, t_type, file_path) else: yield self._respond_default_thumbnail( - self, request, media_info, width, height, method, m_type, + request, media_info, width, height, method, m_type, ) @defer.inlineCallbacks @@ -103,19 +102,19 @@ class ThumbnailResource(BaseMediaResource): thumbnail_info = self._select_thumbnail( width, height, method, m_type, thumbnail_infos ) - thumbnail_width = thumbnail_info["thumbnail_width"] - thumbnail_height = thumbnail_info["thumbnail_height"] - thumbnail_type = thumbnail_info["thumbnail_type"] - thumbnail_method = thumbnail_info["thumbnail_method"] + t_width = thumbnail_info["thumbnail_width"] + t_height = thumbnail_info["thumbnail_height"] + t_type = thumbnail_info["thumbnail_type"] + t_method = thumbnail_info["thumbnail_method"] + file_id = thumbnail_info["filesystem_id"] file_path = self.filepaths.remote_media_thumbnail( - server_name, media_id, thumbnail_width, thumbnail_height, - thumbnail_type, thumbnail_method, + server_name, file_id, t_width, t_height, t_type, t_method, ) - yield self._respond_with_file(request, thumbnail_type, file_path) + yield self._respond_with_file(request, t_type, file_path) else: yield self._respond_default_thumbnail( - self, request, media_info, width, height, method, m_type, + request, media_info, width, height, method, m_type, ) @defer.inlineCallbacks @@ -143,16 +142,15 @@ class ThumbnailResource(BaseMediaResource): width, height, "crop", m_type, thumbnail_infos ) - thumbnail_width = thumbnail_info["thumbnail_width"] - thumbnail_height = thumbnail_info["thumbnail_height"] - thumbnail_type = thumbnail_info["thumbnail_type"] - thumbnail_method = thumbnail_info["thumbnail_method"] + t_width = thumbnail_info["thumbnail_width"] + t_height = thumbnail_info["thumbnail_height"] + t_type = thumbnail_info["thumbnail_type"] + t_method = thumbnail_info["thumbnail_method"] file_path = self.filepaths.default_thumbnail( - top_level_type, sub_type, thumbnail_width, thumbnail_height, - thumbnail_type, thumbnail_method, + top_level_type, sub_type, t_width, t_height, t_type, t_method, ) - yield self.respond_with_file(request, thumbnail_type, file_path) + yield self.respond_with_file(request, t_type, file_path) def _select_thumbnail(self, desired_width, desired_height, desired_method, desired_type, thumbnail_infos): @@ -164,7 +162,7 @@ class ThumbnailResource(BaseMediaResource): for info in thumbnail_infos: t_w = info["thumbnail_width"] t_h = info["thumbnail_height"] - t_method = info["thumnail_method"] + t_method = info["thumbnail_method"] if t_method == "scale" or t_method == "crop": aspect_quality = abs(d_w * t_h - d_h * t_w) size_quality = abs((d_w - t_w) * (d_h - t_h)) @@ -180,7 +178,7 @@ class ThumbnailResource(BaseMediaResource): for info in thumbnail_infos: t_w = info["thumbnail_width"] t_h = info["thumbnail_height"] - t_method = info["thumnail_method"] + t_method = info["thumbnail_method"] if t_method == "scale" and (t_w >= d_w or t_h >= d_h): size_quality = abs((d_w - t_w) * (d_h - t_h)) type_quality = desired_type != info["thumbnail_type"] diff --git a/synapse/media/v1/upload_resource.py b/synapse/media/v1/upload_resource.py index a78cc3cff3..b2449ff03d 100644 --- a/synapse/media/v1/upload_resource.py +++ b/synapse/media/v1/upload_resource.py @@ -23,9 +23,7 @@ from synapse.api.errors import ( from twisted.web.server import NOT_DONE_YET from twisted.internet import defer -from .baseresource import BaseMediaResource - -import os +from .base_resource import BaseMediaResource import logging @@ -75,7 +73,7 @@ class UploadResource(BaseMediaResource): media_id = random_string(24) fname = self.filepaths.local_media_filepath(media_id) - os.makedirs(os.path.dirname(fname)) + self._makedirs(fname) # This shouldn't block for very long because the content will have # already been uploaded at this point. @@ -95,7 +93,7 @@ class UploadResource(BaseMediaResource): "media_length": content_length, } - yield self._generate_local_thumbnails(self, media_id, media_info) + yield self._generate_local_thumbnails(media_id, media_info) respond_with_json( request, 200, {"content_token": media_id}, send_cors=True diff --git a/synapse/storage/media_repository.py b/synapse/storage/media_repository.py index a848662716..18c068d3d9 100644 --- a/synapse/storage/media_repository.py +++ b/synapse/storage/media_repository.py @@ -19,6 +19,9 @@ from _base import SQLBaseStore class MediaRepositoryStore(SQLBaseStore): """Persistence for attachments and avatars""" + def get_default_thumbnails(self, top_level_type, sub_type): + return [] + def get_local_media(self, media_id): """Get the metadata for a local piece of media Returns: @@ -47,7 +50,7 @@ class MediaRepositoryStore(SQLBaseStore): def get_local_media_thumbnails(self, media_id): return self._simple_select_list( - "local_media_thumbnails", + "local_media_repository_thumbnails", {"media_id": media_id}, ( "thumbnail_width", "thumbnail_height", "thumbnail_method", @@ -59,7 +62,7 @@ class MediaRepositoryStore(SQLBaseStore): thumbnail_height, thumbnail_type, thumbnail_method, thumbnail_length): return self._simple_insert( - "local_media_thumbnails", + "local_media_repository_thumbnails", { "media_id": media_id, "thumbnail_width": thumbnail_width, @@ -100,11 +103,10 @@ class MediaRepositoryStore(SQLBaseStore): def get_remote_media_thumbnails(self, origin, media_id): return self._simple_select_list( "remote_media_cache_thumbnails", - {"origin": origin, "media_id": media_id}, + {"media_origin": origin, "media_id": media_id}, ( - "thumbnail_width", "thumbnail_height", "thumbnail_method" - "thumbnail_type", "thumbnail_length", - "filesystem_id" + "thumbnail_width", "thumbnail_height", "thumbnail_method", + "thumbnail_type", "thumbnail_length", "filesystem_id", ) ) From 1d2a0040cff8d04cdc7d7d09d8f04a5d628fa9dd Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 10 Dec 2014 15:55:03 +0000 Subject: [PATCH 070/158] Fix bug where we clobbered old state group values --- synapse/handlers/federation.py | 9 +++++++++ synapse/storage/__init__.py | 3 ++- synapse/storage/schema/state.sql | 3 ++- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index e5deb8a9ef..2201cd977e 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -346,6 +346,8 @@ class FederationHandler(BaseHandler): event.get_pdu_json() ) + handled_events = set() + try: builder.event_id = self.event_factory.create_event_id() builder.origin = self.hs.hostname @@ -371,6 +373,10 @@ class FederationHandler(BaseHandler): auth_chain = ret["auth_chain"] auth_chain.sort(key=lambda e: e.depth) + handled_events.update([s.event_id for s in state]) + handled_events.update([a.event_id for a in auth_chain]) + handled_events.add(new_event.event_id) + logger.debug("do_invite_join auth_chain: %s", auth_chain) logger.debug("do_invite_join state: %s", state) @@ -426,6 +432,9 @@ class FederationHandler(BaseHandler): del self.room_queues[room_id] for p, origin in room_queue: + if p.event_id in handled_events: + continue + try: self.on_receive_pdu(origin, p, backfilled=False) except: diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index f8d895082d..2db2e9720f 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -220,7 +220,8 @@ class DataStore(RoomMemberStore, RoomStore, room_id=event.room_id, ) - self._store_state_groups_txn(txn, event, context) + if not outlier: + self._store_state_groups_txn(txn, event, context) if current_state: txn.execute( diff --git a/synapse/storage/schema/state.sql b/synapse/storage/schema/state.sql index 44f7aafb27..2c48d6daca 100644 --- a/synapse/storage/schema/state.sql +++ b/synapse/storage/schema/state.sql @@ -29,7 +29,8 @@ CREATE TABLE IF NOT EXISTS state_groups_state( CREATE TABLE IF NOT EXISTS event_to_state_groups( event_id TEXT NOT NULL, - state_group INTEGER NOT NULL + state_group INTEGER NOT NULL, + CONSTRAINT event_to_state_groups_uniq UNIQUE (event_id) ); CREATE INDEX IF NOT EXISTS state_groups_id ON state_groups(id); From 2d265ef3bd99e84ad37057e8fb69c5cddc91f0ce Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 10 Dec 2014 16:09:09 +0000 Subject: [PATCH 071/158] import Image as PIL.Image. --- synapse/media/v1/thumbnailer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/media/v1/thumbnailer.py b/synapse/media/v1/thumbnailer.py index 47160721e7..774ae4538f 100644 --- a/synapse/media/v1/thumbnailer.py +++ b/synapse/media/v1/thumbnailer.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import Image +import PIL.Image as Image from io import BytesIO From 6a8148f15ba51ef788e6579bace695aa990cd645 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 10 Dec 2014 16:10:25 +0000 Subject: [PATCH 072/158] Add new event graphing tool --- graph/graph2.py | 138 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 138 insertions(+) create mode 100644 graph/graph2.py diff --git a/graph/graph2.py b/graph/graph2.py new file mode 100644 index 0000000000..b9b8a562a0 --- /dev/null +++ b/graph/graph2.py @@ -0,0 +1,138 @@ +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import sqlite3 +import pydot +import cgi +import json +import datetime +import argparse + +from synapse.events import FrozenEvent + + +def make_graph(db_name, room_id, file_prefix): + conn = sqlite3.connect(db_name) + + c = conn.execute( + "SELECT json FROM event_json where room_id = ?", + (room_id,) + ) + + events = [FrozenEvent(json.loads(e[0])) for e in c.fetchall()] + + events.sort(key=lambda e: e.depth) + + node_map = {} + state_groups = {} + + graph = pydot.Dot(graph_name="Test") + + for event in events: + c = conn.execute( + "SELECT state_group FROM event_to_state_groups " + "WHERE event_id = ?", + (event.event_id,) + ) + + res = c.fetchone() + state_group = res[0] if res else None + + if state_group is not None: + state_groups.setdefault(state_group, []).append(event.event_id) + + t = datetime.datetime.fromtimestamp( + float(event.origin_server_ts) / 1000 + ).strftime('%Y-%m-%d %H:%M:%S,%f') + + content = json.dumps(event.get_dict()["content"]) + + label = ( + "<" + "%(name)s
" + "Type: %(type)s
" + "State key: %(state_key)s
" + "Content: %(content)s
" + "Time: %(time)s
" + "Depth: %(depth)s
" + "State group: %(state_group)s
" + ">" + ) % { + "name": event.event_id, + "type": event.type, + "state_key": event.get("state_key", None), + "content": cgi.escape(content, quote=True), + "time": t, + "depth": event.depth, + "state_group": state_group, + } + + node = pydot.Node( + name=event.event_id, + label=label, + ) + + node_map[event.event_id] = node + graph.add_node(node) + + for event in events: + for prev_id, _ in event.prev_events: + try: + end_node = node_map[prev_id] + except: + end_node = pydot.Node( + name=prev_id, + label="<%s>" % (prev_id,), + ) + + node_map[prev_id] = end_node + graph.add_node(end_node) + + edge = pydot.Edge(node_map[event.event_id], end_node) + graph.add_edge(edge) + + for group, event_ids in state_groups.items(): + if len(event_ids) <= 1: + continue + + cluster = pydot.Cluster( + str(group), + label="" % (str(group),) + ) + + for event_id in event_ids: + cluster.add_node(node_map[event_id]) + + graph.add_subgraph(cluster) + + graph.write('%s.dot' % file_prefix, format='raw', prog='dot') + graph.write_svg("%s.svg" % file_prefix, prog='dot') + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Generate a PDU graph for a given room by talking " + "to the given homeserver to get the list of PDUs. \n" + "Requires pydot." + ) + parser.add_argument( + "-p", "--prefix", dest="prefix", + help="String to prefix output files with" + ) + parser.add_argument('db') + parser.add_argument('room') + + args = parser.parse_args() + + make_graph(args.db, args.room, args.prefix) From 7f193b9958f9fc3e039564a9360767530c09b803 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 10 Dec 2014 16:54:37 +0000 Subject: [PATCH 073/158] update media repository implementation docs --- docs/media_repository.rst | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/docs/media_repository.rst b/docs/media_repository.rst index e554d0f495..e4a6974041 100644 --- a/docs/media_repository.rst +++ b/docs/media_repository.rst @@ -12,9 +12,14 @@ Each item of media is assigned a ``media_id`` when it is uploaded. The ``media_id`` is a randomly chosen, URL safe 24 character string. Metadata such as the MIME type, upload time and length are stored in the sqlite3 database indexed by ``media_id``. -Content is stored on the filesystem under a "content" directory. Thumbnails are -stored under a "thumbnails" directory. +Content is stored on the filesystem under a ``"local_content"`` directory. +Thumbnails are stored under a ``"local_thumbnails"`` directory. The item with ``media_id`` ``"aabbccccccccdddddddddddd"`` is stored under -``"local/content/aa/bb/ccccccccdddddddddddd"``. Its thumbnail with width +``"local_content/aa/bb/ccccccccdddddddddddd"``. Its thumbnail with width ``128`` and height ``96`` and type ``"image/jpeg"`` is stored under -``"local/thumbnails/aa/bb/ccccccccdddddddddddd/128-96-image-jpeg"`` +``"local_thumbnails/aa/bb/ccccccccdddddddddddd/128-96-image-jpeg"`` +Remote content is cached under ``"remote_content"`` directory. Each item of +remote content is assigned a local "``filesystem_id``" to ensure that the +directory structure ``"remote_content/server_name/aa/bb/ccccccccdddddddddddd"`` +is appropriate. Thumbnails for remote content are stored under +``"remote_thumbnails/server_name/..."`` From b245ee34ed70854d0802921feb13822cd07996fa Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 10 Dec 2014 17:59:47 +0000 Subject: [PATCH 074/158] Add some basic event validation --- synapse/events/validator.py | 29 +++++++++++++++++++++-------- synapse/handlers/message.py | 8 ++++++-- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/synapse/events/validator.py b/synapse/events/validator.py index f319072d37..47830aa985 100644 --- a/synapse/events/validator.py +++ b/synapse/events/validator.py @@ -15,6 +15,7 @@ from synapse.types import EventID, RoomID, UserID from synapse.api.errors import SynapseError +from synapse.api.constants import EventTypes, Membership class EventValidator(object): @@ -23,14 +24,19 @@ class EventValidator(object): EventID.from_string(event.event_id) RoomID.from_string(event.room_id) - hasattr(event, "auth_events") - hasattr(event, "content") - hasattr(event, "hashes") - hasattr(event, "origin") - hasattr(event, "prev_events") - hasattr(event, "prev_events") - hasattr(event, "sender") - hasattr(event, "type") + required = [ + # "auth_events", + "content", + # "hashes", + "origin", + # "prev_events", + "sender", + "type", + ] + + for k in required: + if not hasattr(event, k): + raise SynapseError(400, "Event does not have key %s" % (k,)) # Check that the following keys have string values strings = [ @@ -46,6 +52,13 @@ class EventValidator(object): if not isinstance(getattr(event, s), basestring): raise SynapseError(400, "Not '%s' a string type" % (s,)) + if event.type == EventTypes.Member: + if "membership" not in event.content: + raise SynapseError(400, "Content has not membership key") + + if event.content["membership"] not in Membership.LIST: + raise SynapseError(400, "Invalid membership key") + # Check that the following keys have dictionary values # TODO diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 9043b945e4..f92b01a50f 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -19,6 +19,9 @@ from synapse.api.constants import EventTypes, Membership from synapse.api.errors import RoomError from synapse.streams.config import PaginationConfig from synapse.util.logcontext import PreserveLoggingContext + +from synapse.events.validator import EventValidator + from ._base import BaseHandler import logging @@ -33,6 +36,7 @@ class MessageHandler(BaseHandler): self.hs = hs self.clock = hs.get_clock() self.event_factory = hs.get_event_factory() + self.validator = EventValidator() @defer.inlineCallbacks def get_message(self, msg_id=None, room_id=None, sender_id=None, @@ -137,6 +141,8 @@ class MessageHandler(BaseHandler): def handle_event(self, event_dict): builder = self.event_builder_factory.new(event_dict) + self.validator.validate(builder) + if builder.type == EventTypes.Member: membership = builder.content.get("membership", None) if membership == Membership.JOIN: @@ -152,8 +158,6 @@ class MessageHandler(BaseHandler): builder=builder, ) - # TODO: self.validator.validate(event) - if event.type == EventTypes.Member: member_handler = self.hs.get_handlers().room_member_handler yield member_handler.change_membership(event, context) From 02e4c181714b94e86cf621307f2dfc765a62ad83 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 10 Dec 2014 18:00:36 +0000 Subject: [PATCH 075/158] Remove dead code --- synapse/handlers/_base.py | 2 -- synapse/storage/_base.py | 25 ------------------------- 2 files changed, 27 deletions(-) diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index fdd3151877..14f75ecbc2 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -32,10 +32,8 @@ class BaseHandler(object): def __init__(self, hs): self.store = hs.get_datastore() - self.event_factory = hs.get_event_factory() self.auth = hs.get_auth() self.notifier = hs.get_notifier() - self.room_lock = hs.get_room_lock_manager() self.state_handler = hs.get_state_handler() self.distributor = hs.get_distributor() self.ratelimiter = hs.get_ratelimiter() diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 935a167f6f..12239fa074 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -436,31 +436,6 @@ class SQLBaseStore(object): return self.runInteraction("_simple_max_id", func) - def _parse_event_from_row(self, row_dict): - d = copy.deepcopy({k: v for k, v in row_dict.items()}) - - d.pop("stream_ordering", None) - d.pop("topological_ordering", None) - d.pop("processed", None) - d["origin_server_ts"] = d.pop("ts", 0) - replaces_state = d.pop("prev_state", None) - - if replaces_state: - d["replaces_state"] = replaces_state - - d.update(json.loads(row_dict["unrecognized_keys"])) - d["content"] = json.loads(d["content"]) - del d["unrecognized_keys"] - - if "age_ts" not in d: - # For compatibility - d["age_ts"] = d.get("origin_server_ts", 0) - - return self.event_factory.create_event( - etype=d["type"], - **d - ) - def _get_events(self, event_ids): return self.runInteraction( "_get_events", self._get_events_txn, event_ids From 4c682143c897b09f17ceac61fb2480d430dde929 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 10 Dec 2014 18:00:49 +0000 Subject: [PATCH 076/158] .from_string() no longer takes a HS --- synapse/handlers/room.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 93732a9c87..e771cf317b 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -76,7 +76,7 @@ class RoomCreationHandler(BaseHandler): if room_id: # Ensure room_id is the correct type - room_id_obj = RoomID.from_string(room_id, self.hs) + room_id_obj = RoomID.from_string(room_id) if not self.hs.is_mine(room_id_obj): raise SynapseError(400, "Room id must be local") @@ -476,7 +476,7 @@ class RoomMemberHandler(BaseHandler): if prev_state and prev_state.membership == Membership.INVITE: room = yield self.store.get_room(room_id) inviter = UserID.from_string( - prev_state.sender, self.hs + prev_state.sender ) is_remote_invite_join = not self.hs.is_mine(inviter) and not room From 4d6af0dde3c8489928312060b7d2c69181a8dc41 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 10 Dec 2014 18:00:57 +0000 Subject: [PATCH 077/158] Fix some tests --- tests/handlers/test_typing.py | 2 +- tests/storage/test_redaction.py | 106 ++++++++++++++----------------- tests/storage/test_roommember.py | 51 ++++++++------- tests/storage/test_stream.py | 73 ++++++++++----------- tests/test_types.py | 18 +++--- 5 files changed, 120 insertions(+), 130 deletions(-) diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py index 7e6ed9a42f..af466c474f 100644 --- a/tests/handlers/test_typing.py +++ b/tests/handlers/test_typing.py @@ -131,7 +131,7 @@ class TypingNotificationsTestCase(unittest.TestCase): if ignore_user is not None and member == ignore_user: continue - if member.is_mine: + if hs.is_mine(member): if localusers is not None: localusers.add(member) else: diff --git a/tests/storage/test_redaction.py b/tests/storage/test_redaction.py index adfe64a980..e8671ae3a2 100644 --- a/tests/storage/test_redaction.py +++ b/tests/storage/test_redaction.py @@ -23,7 +23,9 @@ from synapse.api.events.room import ( RoomMemberEvent, MessageEvent, RoomRedactionEvent, ) -from tests.utils import SQLiteMemoryDbPool +from tests.utils import SQLiteMemoryDbPool, MockKey + +from mock import Mock class RedactionTestCase(unittest.TestCase): @@ -33,13 +35,21 @@ class RedactionTestCase(unittest.TestCase): db_pool = SQLiteMemoryDbPool() yield db_pool.prepare() + self.mock_config = Mock() + self.mock_config.signing_key = [MockKey()] + hs = HomeServer( "test", db_pool=db_pool, + config=self.mock_config, + resource_for_federation=Mock(), + http_client=None, ) self.store = hs.get_datastore() - self.event_factory = hs.get_event_factory() + self.event_builder_factory = hs.get_event_builder_factory() + self.handlers = hs.get_handlers() + self.message_handler = self.handlers.message_handler self.u_alice = hs.parse_userid("@alice:test") self.u_bob = hs.parse_userid("@bob:test") @@ -49,35 +59,23 @@ class RedactionTestCase(unittest.TestCase): self.depth = 1 @defer.inlineCallbacks - def inject_room_member(self, room, user, membership, prev_state=None, + def inject_room_member(self, room, user, membership, replaces_state=None, extra_content={}): - self.depth += 1 + content = {"membership": membership} + content.update(extra_content) + builder = self.event_builder_factory.new({ + "type": RoomMemberEvent.TYPE, + "sender": user.to_string(), + "state_key": user.to_string(), + "room_id": room.to_string(), + "content": content, + }) - event = self.event_factory.create_event( - etype=RoomMemberEvent.TYPE, - user_id=user.to_string(), - state_key=user.to_string(), - room_id=room.to_string(), - membership=membership, - content={"membership": membership}, - depth=self.depth, - prev_events=[], + event, context = yield self.message_handler._create_new_client_event( + builder ) - event.content.update(extra_content) - - if prev_state: - event.prev_state = prev_state - - event.state_events = None - event.hashes = {} - event.prev_state = [] - event.auth_events = [] - - # Have to create a join event using the eventfactory - yield self.store.persist_event( - event - ) + yield self.store.persist_event(event, context) defer.returnValue(event) @@ -85,46 +83,38 @@ class RedactionTestCase(unittest.TestCase): def inject_message(self, room, user, body): self.depth += 1 - event = self.event_factory.create_event( - etype=MessageEvent.TYPE, - user_id=user.to_string(), - room_id=room.to_string(), - content={"body": body, "msgtype": u"message"}, - depth=self.depth, - prev_events=[], + builder = self.event_builder_factory.new({ + "type": MessageEvent.TYPE, + "sender": user.to_string(), + "state_key": user.to_string(), + "room_id": room.to_string(), + "content": {"body": body, "msgtype": u"message"}, + }) + + event, context = yield self.message_handler._create_new_client_event( + builder ) - event.state_events = None - event.hashes = {} - event.auth_events = [] - - yield self.store.persist_event( - event - ) + yield self.store.persist_event(event, context) defer.returnValue(event) @defer.inlineCallbacks def inject_redaction(self, room, event_id, user, reason): - event = self.event_factory.create_event( - etype=RoomRedactionEvent.TYPE, - user_id=user.to_string(), - room_id=room.to_string(), - content={"reason": reason}, - depth=self.depth, - redacts=event_id, - prev_events=[], + builder = self.event_builder_factory.new({ + "type": MessageEvent.TYPE, + "sender": user.to_string(), + "state_key": user.to_string(), + "room_id": room.to_string(), + "content": {"reason": reason}, + "redacts": event_id, + }) + + event, context = yield self.message_handler._create_new_client_event( + builder ) - event.state_events = None - event.hashes = {} - event.auth_events = [] - - yield self.store.persist_event( - event - ) - - defer.returnValue(event) + yield self.store.persist_event(event, context) @defer.inlineCallbacks def test_redact(self): diff --git a/tests/storage/test_roommember.py b/tests/storage/test_roommember.py index 8614e5ca9d..6df09952d1 100644 --- a/tests/storage/test_roommember.py +++ b/tests/storage/test_roommember.py @@ -21,7 +21,9 @@ from synapse.server import HomeServer from synapse.api.constants import Membership from synapse.api.events.room import RoomMemberEvent -from tests.utils import SQLiteMemoryDbPool +from tests.utils import SQLiteMemoryDbPool, MockKey + +from mock import Mock class RoomMemberStoreTestCase(unittest.TestCase): @@ -31,14 +33,22 @@ class RoomMemberStoreTestCase(unittest.TestCase): db_pool = SQLiteMemoryDbPool() yield db_pool.prepare() - hs = HomeServer("test", - db_pool=db_pool, - ) + self.mock_config = Mock() + self.mock_config.signing_key = [MockKey()] + hs = HomeServer( + "test", + db_pool=db_pool, + config=self.mock_config, + resource_for_federation=Mock(), + http_client=None, + ) # We can't test the RoomMemberStore on its own without the other event # storage logic self.store = hs.get_datastore() - self.event_factory = hs.get_event_factory() + self.event_builder_factory = hs.get_event_builder_factory() + self.handlers = hs.get_handlers() + self.message_handler = self.handlers.message_handler self.u_alice = hs.parse_userid("@alice:test") self.u_bob = hs.parse_userid("@bob:test") @@ -49,27 +59,22 @@ class RoomMemberStoreTestCase(unittest.TestCase): self.room = hs.parse_roomid("!abc123:test") @defer.inlineCallbacks - def inject_room_member(self, room, user, membership): - # Have to create a join event using the eventfactory - event = self.event_factory.create_event( - etype=RoomMemberEvent.TYPE, - user_id=user.to_string(), - state_key=user.to_string(), - room_id=room.to_string(), - membership=membership, - content={"membership": membership}, - depth=1, - prev_events=[], + def inject_room_member(self, room, user, membership, replaces_state=None): + builder = self.event_builder_factory.new({ + "type": RoomMemberEvent.TYPE, + "sender": user.to_string(), + "state_key": user.to_string(), + "room_id": room.to_string(), + "content": {"membership": membership}, + }) + + event, context = yield self.message_handler._create_new_client_event( + builder ) - event.state_events = None - event.hashes = {} - event.prev_state = {} - event.auth_events = {} + yield self.store.persist_event(event, context) - yield self.store.persist_event( - event - ) + defer.returnValue(event) @defer.inlineCallbacks def test_one_member(self): diff --git a/tests/storage/test_stream.py b/tests/storage/test_stream.py index 5038546aee..cba65bb9f0 100644 --- a/tests/storage/test_stream.py +++ b/tests/storage/test_stream.py @@ -21,7 +21,9 @@ from synapse.server import HomeServer from synapse.api.constants import Membership from synapse.api.events.room import RoomMemberEvent, MessageEvent -from tests.utils import SQLiteMemoryDbPool +from tests.utils import SQLiteMemoryDbPool, MockKey + +from mock import Mock class StreamStoreTestCase(unittest.TestCase): @@ -31,13 +33,21 @@ class StreamStoreTestCase(unittest.TestCase): db_pool = SQLiteMemoryDbPool() yield db_pool.prepare() + self.mock_config = Mock() + self.mock_config.signing_key = [MockKey()] + hs = HomeServer( "test", db_pool=db_pool, + config=self.mock_config, + resource_for_federation=Mock(), + http_client=None, ) self.store = hs.get_datastore() - self.event_factory = hs.get_event_factory() + self.event_builder_factory = hs.get_event_builder_factory() + self.handlers = hs.get_handlers() + self.message_handler = self.handlers.message_handler self.u_alice = hs.parse_userid("@alice:test") self.u_bob = hs.parse_userid("@bob:test") @@ -51,30 +61,19 @@ class StreamStoreTestCase(unittest.TestCase): def inject_room_member(self, room, user, membership, replaces_state=None): self.depth += 1 - event = self.event_factory.create_event( - etype=RoomMemberEvent.TYPE, - user_id=user.to_string(), - state_key=user.to_string(), - room_id=room.to_string(), - membership=membership, - content={"membership": membership}, - depth=self.depth, - prev_events=[], + builder = self.event_builder_factory.new({ + "type": RoomMemberEvent.TYPE, + "sender": user.to_string(), + "state_key": user.to_string(), + "room_id": room.to_string(), + "content": {"membership": membership}, + }) + + event, context = yield self.message_handler._create_new_client_event( + builder ) - event.state_events = None - event.hashes = {} - event.prev_state = [] - event.auth_events = [] - - if replaces_state: - event.prev_state = [(replaces_state, "hash")] - event.replaces_state = replaces_state - - # Have to create a join event using the eventfactory - yield self.store.persist_event( - event - ) + yield self.store.persist_event(event, context) defer.returnValue(event) @@ -82,23 +81,19 @@ class StreamStoreTestCase(unittest.TestCase): def inject_message(self, room, user, body): self.depth += 1 - event = self.event_factory.create_event( - etype=MessageEvent.TYPE, - user_id=user.to_string(), - room_id=room.to_string(), - content={"body": body, "msgtype": u"message"}, - depth=self.depth, - prev_events=[], + builder = self.event_builder_factory.new({ + "type": MessageEvent.TYPE, + "sender": user.to_string(), + "state_key": user.to_string(), + "room_id": room.to_string(), + "content": {"body": body, "msgtype": u"message"}, + }) + + event, context = yield self.message_handler._create_new_client_event( + builder ) - event.state_events = None - event.hashes = {} - event.auth_events = [] - - # Have to create a join event using the eventfactory - yield self.store.persist_event( - event - ) + yield self.store.persist_event(event, context) @defer.inlineCallbacks def test_event_stream_get_other(self): diff --git a/tests/test_types.py b/tests/test_types.py index 276ecc91fd..bfb9e6f548 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -23,21 +23,21 @@ mock_homeserver = BaseHomeServer(hostname="my.domain") class UserIDTestCase(unittest.TestCase): def test_parse(self): - user = UserID.from_string("@1234abcd:my.domain", hs=mock_homeserver) + user = UserID.from_string("@1234abcd:my.domain") self.assertEquals("1234abcd", user.localpart) self.assertEquals("my.domain", user.domain) - self.assertEquals(True, user.is_mine) + self.assertEquals(True, mock_homeserver.is_mine(user)) def test_build(self): - user = UserID("5678efgh", "my.domain", True) + user = UserID("5678efgh", "my.domain") self.assertEquals(user.to_string(), "@5678efgh:my.domain") def test_compare(self): - userA = UserID.from_string("@userA:my.domain", hs=mock_homeserver) - userAagain = UserID.from_string("@userA:my.domain", hs=mock_homeserver) - userB = UserID.from_string("@userB:my.domain", hs=mock_homeserver) + userA = UserID.from_string("@userA:my.domain") + userAagain = UserID.from_string("@userA:my.domain") + userB = UserID.from_string("@userB:my.domain") self.assertTrue(userA == userAagain) self.assertTrue(userA != userB) @@ -52,14 +52,14 @@ class UserIDTestCase(unittest.TestCase): class RoomAliasTestCase(unittest.TestCase): def test_parse(self): - room = RoomAlias.from_string("#channel:my.domain", hs=mock_homeserver) + room = RoomAlias.from_string("#channel:my.domain") self.assertEquals("channel", room.localpart) self.assertEquals("my.domain", room.domain) - self.assertEquals(True, room.is_mine) + self.assertEquals(True, mock_homeserver.is_mine(room)) def test_build(self): - room = RoomAlias("channel", "my.domain", True) + room = RoomAlias("channel", "my.domain") self.assertEquals(room.to_string(), "#channel:my.domain") From be9a8d68e078a20967885df5cc25d38b3c1220d6 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Wed, 10 Dec 2014 19:13:50 +0000 Subject: [PATCH 078/158] Trivial test of MockClock() --- tests/test_test_utils.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 tests/test_test_utils.py diff --git a/tests/test_test_utils.py b/tests/test_test_utils.py new file mode 100644 index 0000000000..11c5db3cd9 --- /dev/null +++ b/tests/test_test_utils.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from tests import unittest + +from tests.utils import MockClock + +class MockClockTestCase(unittest.TestCase): + + def setUp(self): + self.clock = MockClock() + + def test_advance_time(self): + start_time = self.clock.time() + + self.clock.advance_time(20) + + self.assertEquals(20, self.clock.time() - start_time) From 38da9884e70e8e44bde14c67a7a8a9d49a8b87ac Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Wed, 10 Dec 2014 19:24:12 +0000 Subject: [PATCH 079/158] Implement .call_later() in MockClock --- tests/test_test_utils.py | 22 ++++++++++++++++++++++ tests/utils.py | 26 ++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/tests/test_test_utils.py b/tests/test_test_utils.py index 11c5db3cd9..dcf946fc1a 100644 --- a/tests/test_test_utils.py +++ b/tests/test_test_utils.py @@ -28,3 +28,25 @@ class MockClockTestCase(unittest.TestCase): self.clock.advance_time(20) self.assertEquals(20, self.clock.time() - start_time) + + def test_later(self): + invoked = [0, 0] + + def _cb0(): + invoked[0] = 1 + self.clock.call_later(10, _cb0) + + def _cb1(): + invoked[1] = 1 + self.clock.call_later(20, _cb1) + + self.assertFalse(invoked[0]) + + self.clock.advance_time(15) + + self.assertTrue(invoked[0]) + self.assertFalse(invoked[1]) + + self.clock.advance_time(5) + + self.assertTrue(invoked[1]) diff --git a/tests/utils.py b/tests/utils.py index d8be73dba8..72843714fd 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -18,6 +18,8 @@ from synapse.api.errors import cs_error, CodeMessageException, StoreError from synapse.api.constants import Membership from synapse.storage import prepare_database +from synapse.util.logcontext import LoggingContext + from synapse.api.events.room import ( RoomMemberEvent, MessageEvent ) @@ -134,16 +136,40 @@ class MockKey(object): class MockClock(object): now = 1000 + def __init__(self): + # list of tuples of (absolute_time, callback) in no particular order + self.timers = [] + def time(self): return self.now def time_msec(self): return self.time() * 1000 + def call_later(self, delay, callback): + current_context = LoggingContext.current_context() + + def wrapped_callback(): + LoggingContext.thread_local.current_context = current_context + callback() + self.timers.append((self.now + delay, wrapped_callback)) + + def cancel_call_later(self, timer): + raise NotImplementedError("Oopsie") + # For unit testing def advance_time(self, secs): self.now += secs + timers = self.timers + self.timers = [] + + for time, callback in timers: + if self.now >= time: + callback() + else: + self.timers.append((time, callback)) + class SQLiteMemoryDbPool(ConnectionPool, object): def __init__(self): From 4551afc6d2f69ae3aa0ac37217fa0a94c0b2d62a Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Wed, 10 Dec 2014 19:26:52 +0000 Subject: [PATCH 080/158] Implement .cancel_call_later() in MockClock --- tests/test_test_utils.py | 18 ++++++++++++++++++ tests/utils.py | 7 +++++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/tests/test_test_utils.py b/tests/test_test_utils.py index dcf946fc1a..b42787dd25 100644 --- a/tests/test_test_utils.py +++ b/tests/test_test_utils.py @@ -50,3 +50,21 @@ class MockClockTestCase(unittest.TestCase): self.clock.advance_time(5) self.assertTrue(invoked[1]) + + def test_cancel_later(self): + invoked = [0, 0] + + def _cb0(): + invoked[0] = 1 + t0 = self.clock.call_later(10, _cb0) + + def _cb1(): + invoked[1] = 1 + t1 = self.clock.call_later(20, _cb1) + + self.clock.cancel_call_later(t0) + + self.clock.advance_time(30) + + self.assertFalse(invoked[0]) + self.assertTrue(invoked[1]) diff --git a/tests/utils.py b/tests/utils.py index 72843714fd..f9a34748cd 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -152,10 +152,13 @@ class MockClock(object): def wrapped_callback(): LoggingContext.thread_local.current_context = current_context callback() - self.timers.append((self.now + delay, wrapped_callback)) + + t = (self.now + delay, wrapped_callback) + self.timers.append(t) + return t def cancel_call_later(self, timer): - raise NotImplementedError("Oopsie") + self.timers = [t for t in self.timers if t != timer] # For unit testing def advance_time(self, secs): From 9eb819e82801091362de7bd57c126ca5f60f1a8d Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Wed, 10 Dec 2014 19:39:01 +0000 Subject: [PATCH 081/158] First hack at implementing timeouts in typing notification handler --- synapse/handlers/typing.py | 22 ++++++++++++++++++--- tests/handlers/test_typing.py | 36 ++++++++++++++++++++++++++++++++--- 2 files changed, 52 insertions(+), 6 deletions(-) diff --git a/synapse/handlers/typing.py b/synapse/handlers/typing.py index d88a53242c..912cfd708b 100644 --- a/synapse/handlers/typing.py +++ b/synapse/handlers/typing.py @@ -43,7 +43,8 @@ class TypingNotificationHandler(BaseHandler): self.federation.register_edu_handler("m.typing", self._recv_edu) - self._member_typing_until = {} + self._member_typing_until = {} # clock time we expect to stop + self._member_typing_timer = {} # deferreds to manage theabove @defer.inlineCallbacks def started_typing(self, target_user, auth_user, room_id, timeout): @@ -58,7 +59,13 @@ class TypingNotificationHandler(BaseHandler): was_present = member in self._member_typing_until + if member in self._member_typing_timer: + self.clock.cancel_call_later(self._member_typing_timer[member]) + self._member_typing_until[member] = until + self._member_typing_timer[member] = self.clock.call_later( + timeout / 1000, lambda: self._stopped_typing(member) + ) if was_present: # No point sending another notification @@ -80,16 +87,25 @@ class TypingNotificationHandler(BaseHandler): member = RoomMember(room_id=room_id, user=target_user) + yield self._stopped_typing(member) + + @defer.inlineCallbacks + def _stopped_typing(self, member): if member not in self._member_typing_until: # No point defer.returnValue(None) yield self._push_update( - room_id=room_id, - user=target_user, + room_id=member.room_id, + user=member.user, typing=False, ) + del self._member_typing_until[member] + + self.clock.cancel_call_later(self._member_typing_timer[member]) + del self._member_typing_timer[member] + @defer.inlineCallbacks def _push_update(self, room_id, user, typing): localusers = set() diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py index 7e6ed9a42f..6b9e22d396 100644 --- a/tests/handlers/test_typing.py +++ b/tests/handlers/test_typing.py @@ -238,9 +238,11 @@ class TypingNotificationsTestCase(unittest.TestCase): # Gut-wrenching from synapse.handlers.typing import RoomMember - self.handler._member_typing_until[ - RoomMember(self.room_id, self.u_apple) - ] = 1002000 + member = RoomMember(self.room_id, self.u_apple) + self.handler._member_typing_until[member] = 1002000 + self.handler._member_typing_timer[member] = ( + self.clock.call_later(1002, lambda: 0) + ) yield self.handler.stopped_typing( target_user=self.u_apple, @@ -256,3 +258,31 @@ class TypingNotificationsTestCase(unittest.TestCase): ]) yield put_json.await_calls() + + @defer.inlineCallbacks + def test_typing_timeout(self): + self.room_members = [self.u_apple, self.u_banana] + + yield self.handler.started_typing( + target_user=self.u_apple, + auth_user=self.u_apple, + room_id=self.room_id, + timeout=10000, + ) + + self.mock_update_client.assert_has_calls([ + call(observer_user=self.u_banana, + observed_user=self.u_apple, + room_id=self.room_id, + typing=True), + ]) + self.mock_update_client.reset_mock() + + self.clock.advance_time(11) + + self.mock_update_client.assert_has_calls([ + call(observer_user=self.u_banana, + observed_user=self.u_apple, + room_id=self.room_id, + typing=False), + ]) From 4006d58335a7dad88a30b74d0b50d2d6e80cc72d Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Wed, 10 Dec 2014 20:48:12 +0000 Subject: [PATCH 082/158] Store serial numbers per room for typing event stream purposes --- synapse/handlers/typing.py | 41 +++++++++++++++++++++---------- tests/handlers/test_typing.py | 46 ++++++++++++----------------------- 2 files changed, 44 insertions(+), 43 deletions(-) diff --git a/synapse/handlers/typing.py b/synapse/handlers/typing.py index 912cfd708b..2370ff7134 100644 --- a/synapse/handlers/typing.py +++ b/synapse/handlers/typing.py @@ -46,6 +46,12 @@ class TypingNotificationHandler(BaseHandler): self._member_typing_until = {} # clock time we expect to stop self._member_typing_timer = {} # deferreds to manage theabove + # map room IDs to serial numbers + self._room_serials = {} + self._latest_room_serial = 0 + # map room IDs to sets of users currently typing + self._room_typing = {} + @defer.inlineCallbacks def started_typing(self, target_user, auth_user, room_id, timeout): if not target_user.is_mine: @@ -117,12 +123,11 @@ class TypingNotificationHandler(BaseHandler): ignore_user=user ) - for u in localusers: - self.push_update_to_clients( + if localusers: + self._push_update_local( room_id=room_id, - observer_user=u, - observed_user=user, - typing=typing, + user=user, + typing=typing ) deferreds = [] @@ -151,18 +156,28 @@ class TypingNotificationHandler(BaseHandler): room_id, localusers=localusers ) - for u in localusers: - self.push_update_to_clients( + if localusers: + self._push_update_local( room_id=room_id, - observer_user=u, - observed_user=user, + user=user, typing=content["typing"] ) - def push_update_to_clients(self, room_id, observer_user, observed_user, - typing): - # TODO(paul) steal this from presence.py - pass + def _push_update_local(self, room_id, user, typing): + if room_id not in self._room_serials: + self._room_serials[room_id] = 0 + self._room_typing[room_id] = set() + + room_set = self._room_typing[room_id] + if typing: + room_set.add(user) + elif user in room_set: + room_set.remove(user) + + self._latest_room_serial += 1 + self._room_serials[room_id] = self._latest_room_serial + + self.notifier.on_new_user_event(rooms=[room_id]) class TypingNotificationEventSource(object): diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py index 6b9e22d396..898977ed8d 100644 --- a/tests/handlers/test_typing.py +++ b/tests/handlers/test_typing.py @@ -65,6 +65,9 @@ class TypingNotificationsTestCase(unittest.TestCase): self.mock_config = Mock() self.mock_config.signing_key = [MockKey()] + mock_notifier = Mock(spec=["on_new_user_event"]) + self.on_new_user_event = mock_notifier.on_new_user_event + hs = HomeServer("test", clock=self.clock, db_pool=None, @@ -77,6 +80,7 @@ class TypingNotificationsTestCase(unittest.TestCase): "get_destination_retry_timings", ]), handlers=None, + notifier=mock_notifier, resource_for_client=Mock(), resource_for_federation=self.mock_federation_resource, http_client=self.mock_http_client, @@ -85,11 +89,7 @@ class TypingNotificationsTestCase(unittest.TestCase): ) hs.handlers = JustTypingNotificationHandlers(hs) - self.mock_update_client = Mock() - self.mock_update_client.return_value = defer.succeed(None) - self.handler = hs.get_handlers().typing_notification_handler - self.handler.push_update_to_clients = self.mock_update_client self.datastore = hs.get_datastore() self.datastore.get_destination_retry_timings.return_value = ( @@ -158,11 +158,8 @@ class TypingNotificationsTestCase(unittest.TestCase): timeout=20000, ) - self.mock_update_client.assert_has_calls([ - call(observer_user=self.u_banana, - observed_user=self.u_apple, - room_id=self.room_id, - typing=True), + self.on_new_user_event.assert_has_calls([ + call(rooms=[self.room_id]), ]) @defer.inlineCallbacks @@ -209,11 +206,8 @@ class TypingNotificationsTestCase(unittest.TestCase): ) ) - self.mock_update_client.assert_has_calls([ - call(observer_user=self.u_apple, - observed_user=self.u_onion, - room_id=self.room_id, - typing=True), + self.on_new_user_event.assert_has_calls([ + call(rooms=[self.room_id]), ]) @defer.inlineCallbacks @@ -243,6 +237,7 @@ class TypingNotificationsTestCase(unittest.TestCase): self.handler._member_typing_timer[member] = ( self.clock.call_later(1002, lambda: 0) ) + self.handler._room_typing[self.room_id] = set((self.u_apple,)) yield self.handler.stopped_typing( target_user=self.u_apple, @@ -250,11 +245,8 @@ class TypingNotificationsTestCase(unittest.TestCase): room_id=self.room_id, ) - self.mock_update_client.assert_has_calls([ - call(observer_user=self.u_banana, - observed_user=self.u_apple, - room_id=self.room_id, - typing=False), + self.on_new_user_event.assert_has_calls([ + call(rooms=[self.room_id]), ]) yield put_json.await_calls() @@ -270,19 +262,13 @@ class TypingNotificationsTestCase(unittest.TestCase): timeout=10000, ) - self.mock_update_client.assert_has_calls([ - call(observer_user=self.u_banana, - observed_user=self.u_apple, - room_id=self.room_id, - typing=True), + self.on_new_user_event.assert_has_calls([ + call(rooms=[self.room_id]), ]) - self.mock_update_client.reset_mock() + self.on_new_user_event.reset_mock() self.clock.advance_time(11) - self.mock_update_client.assert_has_calls([ - call(observer_user=self.u_banana, - observed_user=self.u_apple, - room_id=self.room_id, - typing=False), + self.on_new_user_event.assert_has_calls([ + call(rooms=[self.room_id]), ]) From 1a75ff5c23e4d85714927f995e185ddf2d7eaff8 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Wed, 10 Dec 2014 21:01:49 +0000 Subject: [PATCH 083/158] Hook up the event stream to typing notifications --- synapse/handlers/typing.py | 24 ++++++++++++-- tests/handlers/test_typing.py | 60 +++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/typing.py b/synapse/handlers/typing.py index 2370ff7134..7426bda960 100644 --- a/synapse/handlers/typing.py +++ b/synapse/handlers/typing.py @@ -183,12 +183,32 @@ class TypingNotificationHandler(BaseHandler): class TypingNotificationEventSource(object): def __init__(self, hs): self.hs = hs + self.handler = hs.get_handlers().typing_notification_handler + + def _make_event_for(self, room_id): + typing = self.handler._room_typing[room_id] + return { + "type": "m.typing", + "room_id": room_id, + "typing": [u.to_string() for u in typing], + } def get_new_events_for_user(self, user, from_key, limit): - return ([], from_key) + from_key = int(from_key) + handler = self.handler + + events = [] + for room_id in handler._room_serials: + if handler._room_serials[room_id] <= from_key: + continue + + # TODO: check if user is in room + events.append(self._make_event_for(room_id)) + + return (events, handler._latest_room_serial) def get_current_key(self): - return 0 + return self.handler._latest_room_serial def get_pagination_rows(self, user, pagination_config, key): return ([], pagination_config.from_key) diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py index 898977ed8d..b858f96323 100644 --- a/tests/handlers/test_typing.py +++ b/tests/handlers/test_typing.py @@ -91,6 +91,8 @@ class TypingNotificationsTestCase(unittest.TestCase): self.handler = hs.get_handlers().typing_notification_handler + self.event_source = hs.get_event_sources().sources["typing"] + self.datastore = hs.get_datastore() self.datastore.get_destination_retry_timings.return_value = ( defer.succeed(DestinationsTable.EntryType("", 0, 0)) @@ -151,6 +153,8 @@ class TypingNotificationsTestCase(unittest.TestCase): def test_started_typing_local(self): self.room_members = [self.u_apple, self.u_banana] + self.assertEquals(self.event_source.get_current_key(), 0) + yield self.handler.started_typing( target_user=self.u_apple, auth_user=self.u_apple, @@ -162,6 +166,16 @@ class TypingNotificationsTestCase(unittest.TestCase): call(rooms=[self.room_id]), ]) + self.assertEquals(self.event_source.get_current_key(), 1) + self.assertEquals( + self.event_source.get_new_events_for_user(self.u_apple, 0, None)[0], + [ + {"type": "m.typing", + "room_id": self.room_id, + "typing": [self.u_apple.to_string()]}, + ] + ) + @defer.inlineCallbacks def test_started_typing_remote_send(self): self.room_members = [self.u_apple, self.u_onion] @@ -195,6 +209,8 @@ class TypingNotificationsTestCase(unittest.TestCase): def test_started_typing_remote_recv(self): self.room_members = [self.u_apple, self.u_onion] + self.assertEquals(self.event_source.get_current_key(), 0) + yield self.mock_federation_resource.trigger("PUT", "/_matrix/federation/v1/send/1000000/", _make_edu_json("farm", "m.typing", @@ -210,6 +226,16 @@ class TypingNotificationsTestCase(unittest.TestCase): call(rooms=[self.room_id]), ]) + self.assertEquals(self.event_source.get_current_key(), 1) + self.assertEquals( + self.event_source.get_new_events_for_user(self.u_apple, 0, None)[0], + [ + {"type": "m.typing", + "room_id": self.room_id, + "typing": [self.u_onion.to_string()]}, + ] + ) + @defer.inlineCallbacks def test_stopped_typing(self): self.room_members = [self.u_apple, self.u_banana, self.u_onion] @@ -239,6 +265,8 @@ class TypingNotificationsTestCase(unittest.TestCase): ) self.handler._room_typing[self.room_id] = set((self.u_apple,)) + self.assertEquals(self.event_source.get_current_key(), 0) + yield self.handler.stopped_typing( target_user=self.u_apple, auth_user=self.u_apple, @@ -251,10 +279,22 @@ class TypingNotificationsTestCase(unittest.TestCase): yield put_json.await_calls() + self.assertEquals(self.event_source.get_current_key(), 1) + self.assertEquals( + self.event_source.get_new_events_for_user(self.u_apple, 0, None)[0], + [ + {"type": "m.typing", + "room_id": self.room_id, + "typing": []}, + ] + ) + @defer.inlineCallbacks def test_typing_timeout(self): self.room_members = [self.u_apple, self.u_banana] + self.assertEquals(self.event_source.get_current_key(), 0) + yield self.handler.started_typing( target_user=self.u_apple, auth_user=self.u_apple, @@ -267,8 +307,28 @@ class TypingNotificationsTestCase(unittest.TestCase): ]) self.on_new_user_event.reset_mock() + self.assertEquals(self.event_source.get_current_key(), 1) + self.assertEquals( + self.event_source.get_new_events_for_user(self.u_apple, 0, None)[0], + [ + {"type": "m.typing", + "room_id": self.room_id, + "typing": [self.u_apple.to_string()]}, + ] + ) + self.clock.advance_time(11) self.on_new_user_event.assert_has_calls([ call(rooms=[self.room_id]), ]) + + self.assertEquals(self.event_source.get_current_key(), 2) + self.assertEquals( + self.event_source.get_new_events_for_user(self.u_apple, 1, None)[0], + [ + {"type": "m.typing", + "room_id": self.room_id, + "typing": []}, + ] + ) From 5f49914dee80c7572c549bb17a0099cdad7cdb32 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Wed, 10 Dec 2014 21:17:48 +0000 Subject: [PATCH 084/158] Avoid cyclic dependency in handler setup --- synapse/handlers/typing.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/synapse/handlers/typing.py b/synapse/handlers/typing.py index 7426bda960..989f73947f 100644 --- a/synapse/handlers/typing.py +++ b/synapse/handlers/typing.py @@ -183,10 +183,16 @@ class TypingNotificationHandler(BaseHandler): class TypingNotificationEventSource(object): def __init__(self, hs): self.hs = hs - self.handler = hs.get_handlers().typing_notification_handler + self._handler = None + + def handler(self): + # Avoid cyclic dependency in handler setup + if not self._handler: + self._handler = self.hs.get_handlers().typing_notification_handler + return self._handler def _make_event_for(self, room_id): - typing = self.handler._room_typing[room_id] + typing = self.handler()._room_typing[room_id] return { "type": "m.typing", "room_id": room_id, @@ -195,7 +201,7 @@ class TypingNotificationEventSource(object): def get_new_events_for_user(self, user, from_key, limit): from_key = int(from_key) - handler = self.handler + handler = self.handler() events = [] for room_id in handler._room_serials: @@ -208,7 +214,7 @@ class TypingNotificationEventSource(object): return (events, handler._latest_room_serial) def get_current_key(self): - return self.handler._latest_room_serial + return self.handler()._latest_room_serial def get_pagination_rows(self, user, pagination_config, key): return ([], pagination_config.from_key) From b5eb9124f70a0f5196f8508f24e4566e8a5c0e90 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 11 Dec 2014 10:08:09 +0000 Subject: [PATCH 085/158] Make sure we pass a tuple to string '%' formatting --- synapse/media/v1/base_resource.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/media/v1/base_resource.py b/synapse/media/v1/base_resource.py index 81f2456343..8c62ecd597 100644 --- a/synapse/media/v1/base_resource.py +++ b/synapse/media/v1/base_resource.py @@ -93,7 +93,7 @@ class BaseMediaResource(Resource): except: raise SynapseError( 400, - "Missing integer argument %r" % (arg_name), + "Missing integer argument %r" % (arg_name,), Codes.UNKNOWN, ) @@ -107,7 +107,7 @@ class BaseMediaResource(Resource): except: raise SynapseError( 400, - "Missing string argument %r" % (arg_name), + "Missing string argument %r" % (arg_name,), Codes.UNKNOWN, ) From ead8fc5e388cf03b83b5470430a1c3d70af00df3 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 11 Dec 2014 10:41:43 +0000 Subject: [PATCH 086/158] doc the thumbnail methods --- synapse/media/v1/media_repository.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/synapse/media/v1/media_repository.py b/synapse/media/v1/media_repository.py index 2bd29d2288..a0dc56be4b 100644 --- a/synapse/media/v1/media_repository.py +++ b/synapse/media/v1/media_repository.py @@ -49,15 +49,24 @@ class MediaRepositoryResource(Resource): - Clients can get thumbnails by supplying a desired width and height:: + Clients can get thumbnails by supplying a desired width and height and + thumbnailing method:: => GET /_matrix/media/v1 - /thumbnail/?width=&height= HTTP/1.1 + /thumbnail/?width=&height=&method= HTTP/1.1 <= HTTP/1.1 200 OK Content-Type: image/jpeg or image/png + + The thumbnail methods are "crop" and "scale". "scale" trys to return an + image where either the width or the height is smaller than the requested + size. The client should then scale and letterbox the image if it needs to + fit within a given rectangle. "crop" trys to return an image where the + width and height are close to the requested size and the aspect matches + the requested size. The client should scale the image if it needs to fit + within a given rectangle. """ def __init__(self, hs): From 0ca072b3b606b54537e1d963c8e0d4b70c2348a4 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 11 Dec 2014 10:55:36 +0000 Subject: [PATCH 087/158] Initial tiny hack at REST API for setting room typing notification status --- synapse/rest/room.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/synapse/rest/room.py b/synapse/rest/room.py index 3147d7a60b..e7f28c2784 100644 --- a/synapse/rest/room.py +++ b/synapse/rest/room.py @@ -483,6 +483,37 @@ class RoomRedactEventRestServlet(RestServlet): defer.returnValue(response) +class RoomTypingRestServlet(RestServlet): + PATTERN = client_path_pattern("/rooms/(?P[^/]*)/typing/(?P[^/]*)$") + + @defer.inlineCallbacks + def on_PUT(self, request, room_id, user_id): + auth_user = yield self.auth.get_user_by_req(request) + + room_id = urllib.unquote(room_id) + target_user = self.hs.parse_userid(urllib.unquote(user_id)) + + content = _parse_json(request) + + typing_handler = self.handlers.typing_notification_handler + + if content["typing"]: + yield typing_handler.started_typing( + target_user=target_user, + auth_user=auth_user, + room_id=room_id, + timeout=content.get("timeout", 30000), + ) + else: + yield typing_handler.stopped_typing( + target_user=target_user, + auth_user=auth_user, + room_id=room_id, + ) + + defer.returnValue((200, {})) + + def _parse_json(request): try: content = json.loads(request.content.read()) @@ -538,3 +569,4 @@ def register_servlets(hs, http_server): RoomStateRestServlet(hs).register(http_server) RoomInitialSyncRestServlet(hs).register(http_server) RoomRedactEventRestServlet(hs).register(http_server) + RoomTypingRestServlet(hs).register(http_server) From 8cdebce470869613658543cb79ed5dd97a5f0548 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 11 Dec 2014 13:25:19 +0000 Subject: [PATCH 088/158] Fix redactions. Fix 'age' key --- synapse/events/__init__.py | 1 + synapse/events/builder.py | 13 ++++-- synapse/events/utils.py | 21 ++++++++++ synapse/state.py | 12 ++++++ synapse/storage/_base.py | 70 +++++++++++++++++++-------------- tests/storage/test_redaction.py | 6 +-- 6 files changed, 88 insertions(+), 35 deletions(-) diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index 7103b937af..98d7f0e324 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -97,6 +97,7 @@ class EventBase(object): origin_server_ts = _event_dict_property("origin_server_ts") prev_events = _event_dict_property("prev_events") prev_state = _event_dict_property("prev_state") + redacts = _event_dict_property("redacts") room_id = _event_dict_property("room_id") sender = _event_dict_property("sender") state_key = _event_dict_property("state_key") diff --git a/synapse/events/builder.py b/synapse/events/builder.py index 642264e9f3..9579b1fe8b 100644 --- a/synapse/events/builder.py +++ b/synapse/events/builder.py @@ -19,11 +19,18 @@ from synapse.types import EventID from synapse.util.stringutils import random_string +import copy + class EventBuilder(EventBase): def __init__(self, key_values={}): + signatures = copy.deepcopy(key_values.pop("signatures", {})) + unsigned = copy.deepcopy(key_values.pop("unsigned", {})) + super(EventBuilder, self).__init__( key_values, + signatures=signatures, + unsigned=unsigned ) def update_event_key(self, key, value): @@ -61,9 +68,9 @@ class EventBuilderFactory(object): key_values.setdefault("origin", self.hostname) key_values.setdefault("origin_server_ts", time_now) - if "unsigned" in key_values: - age = key_values["unsigned"].pop("age", 0) - key_values["unsigned"].setdefault("age_ts", time_now - age) + key_values.setdefault("unsigned", {}) + age = key_values["unsigned"].pop("age", 0) + key_values["unsigned"].setdefault("age_ts", time_now - age) key_values["signatures"] = {} diff --git a/synapse/events/utils.py b/synapse/events/utils.py index f5e135e3d0..6d9c9352e2 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -80,6 +80,11 @@ def prune_event(event): allowed_fields["content"] = new_content + allowed_fields["unsigned"] = {} + + if "age_ts" in event.unsigned: + allowed_fields["unsigned"]["age_ts"] = event.unsigned["age_ts"] + return type(event)(allowed_fields) @@ -97,4 +102,20 @@ def serialize_event(hs, e): d["user_id"] = d.pop("sender", None) + if "redacted_because" in e.unsigned: + d["redacted_because"] = serialize_event( + hs, e.unsigned["redacted_because"] + ) + + del d["unsigned"]["redacted_because"] + + if "redacted_by" in e.unsigned: + d["redacted_by"] = e.unsigned["redacted_by"] + del d["unsigned"]["redacted_by"] + + del d["auth_events"] + del d["prev_events"] + del d["hashes"] + del d["signatures"] + return d diff --git a/synapse/state.py b/synapse/state.py index 7fdf596006..5bfa73fb46 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -155,6 +155,12 @@ class StateHandler(object): else: context.auth_events = {} + if event.is_state(): + key = (event.type, event.state_key) + if key in context.current_state: + replaces = context.current_state[key] + event.unsigned["replaces_state"] = replaces.event_id + defer.returnValue([]) if event.is_state(): @@ -177,6 +183,12 @@ class StateHandler(object): prev_state ) + if event.is_state(): + key = (event.type, event.state_key) + if key in context.current_state: + replaces = context.current_state[key] + event.unsigned["replaces_state"] = replaces.event_id + if hasattr(event, "auth_events") and event.auth_events: auth_ids = zip(*event.auth_events)[0] context.auth_events = { diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 12239fa074..ffc26d4a61 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -444,39 +444,51 @@ class SQLBaseStore(object): def _get_events_txn(self, txn, event_ids): events = [] for e_id in event_ids: - js = self._simple_select_one_onecol_txn( - txn, - table="event_json", - keyvalues={"event_id": e_id}, - retcol="json", - allow_none=True, - ) + ev = self._get_event_txn(txn, e_id) - if not js: - # FIXME (erikj): What should we actually do here? - continue - - d = json.loads(js) - - ev = FrozenEvent(d) - - if hasattr(ev, "redacted") and ev.redacted: - # Get the redaction event. - select_event_sql = "SELECT * FROM events WHERE event_id = ?" - txn.execute(select_event_sql, (ev.redacted,)) - - del_evs = self._parse_events_txn( - txn, self.cursor_to_dict(txn) - ) - - if del_evs: - ev = prune_event(ev) - ev.redacted_because = del_evs[0] - - events.append(ev) + if ev: + events.append(ev) return events + def _get_event_txn(self, txn, event_id, check_redacted=True): + sql = ( + "SELECT json, r.event_id FROM event_json as e " + "LEFT JOIN redactions as r ON e.event_id = r.redacts " + "WHERE e.event_id = ? " + "LIMIT 1 " + ) + + txn.execute(sql, (event_id,)) + + res = txn.fetchone() + + if not res: + return None + + js, redacted = res + + d = json.loads(js) + + ev = FrozenEvent(d) + + if check_redacted and redacted: + ev = prune_event(ev) + + ev.unsigned["redacted_by"] = redacted + # Get the redaction event. + + because = self._get_event_txn( + txn, + redacted, + check_redacted=False + ) + + if because: + ev.unsigned["redacted_because"] = because + + return ev + def _parse_events(self, rows): return self.runInteraction( "_parse_events", self._parse_events_txn, rows diff --git a/tests/storage/test_redaction.py b/tests/storage/test_redaction.py index e8671ae3a2..d81f7add1c 100644 --- a/tests/storage/test_redaction.py +++ b/tests/storage/test_redaction.py @@ -149,7 +149,7 @@ class RedactionTestCase(unittest.TestCase): event, ) - self.assertFalse(hasattr(event, "redacted_because")) + self.assertFalse("redacted_because" in event.unsigned) # Redact event reason = "Because I said so" @@ -179,7 +179,7 @@ class RedactionTestCase(unittest.TestCase): event, ) - self.assertTrue(hasattr(event, "redacted_because")) + self.assertTrue("redacted_because" in event.unsigned) self.assertObjectHasAttributes( { @@ -187,7 +187,7 @@ class RedactionTestCase(unittest.TestCase): "user_id": self.u_alice.to_string(), "content": {"reason": reason}, }, - event.redacted_because, + event.unsigned["redacted_because"], ) @defer.inlineCallbacks From e72b16f9a37ca335fdc24224a3b8e403ff7d3085 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 11 Dec 2014 13:38:52 +0000 Subject: [PATCH 089/158] Fix redaction tests --- tests/storage/test_redaction.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/storage/test_redaction.py b/tests/storage/test_redaction.py index d81f7add1c..f670e154c3 100644 --- a/tests/storage/test_redaction.py +++ b/tests/storage/test_redaction.py @@ -102,7 +102,7 @@ class RedactionTestCase(unittest.TestCase): @defer.inlineCallbacks def inject_redaction(self, room, event_id, user, reason): builder = self.event_builder_factory.new({ - "type": MessageEvent.TYPE, + "type": RoomRedactionEvent.TYPE, "sender": user.to_string(), "state_key": user.to_string(), "room_id": room.to_string(), @@ -170,6 +170,10 @@ class RedactionTestCase(unittest.TestCase): event = results[0] + self.assertEqual(msg_event.event_id, event.event_id) + + self.assertTrue("redacted_because" in event.unsigned) + self.assertObjectHasAttributes( { "type": MessageEvent.TYPE, @@ -179,8 +183,6 @@ class RedactionTestCase(unittest.TestCase): event, ) - self.assertTrue("redacted_because" in event.unsigned) - self.assertObjectHasAttributes( { "type": RoomRedactionEvent.TYPE, @@ -247,6 +249,8 @@ class RedactionTestCase(unittest.TestCase): event = results[0] + self.assertTrue("redacted_because" in event.unsigned) + self.assertObjectHasAttributes( { "type": RoomMemberEvent.TYPE, @@ -256,13 +260,11 @@ class RedactionTestCase(unittest.TestCase): event, ) - self.assertTrue(hasattr(event, "redacted_because")) - self.assertObjectHasAttributes( { "type": RoomRedactionEvent.TYPE, "user_id": self.u_alice.to_string(), "content": {"reason": reason}, }, - event.redacted_because, + event.unsigned["redacted_because"], ) From d80d505b1f70eae128990ce1a9517e5c5edead73 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 11 Dec 2014 14:19:32 +0000 Subject: [PATCH 090/158] Limit the size of images that are thumbnailed serverside. Limit the size of file that a server will download from a remote server --- synapse/api/errors.py | 1 + synapse/config/repository.py | 5 +++++ synapse/http/matrixfederationclient.py | 25 +++++++++++++++++++------ synapse/media/v1/base_resource.py | 18 ++++++++++++++++++ 4 files changed, 43 insertions(+), 6 deletions(-) diff --git a/synapse/api/errors.py b/synapse/api/errors.py index 581439ceb3..e250b9b211 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -34,6 +34,7 @@ class Codes(object): LIMIT_EXCEEDED = "M_LIMIT_EXCEEDED" CAPTCHA_NEEDED = "M_CAPTCHA_NEEDED" CAPTCHA_INVALID = "M_CAPTCHA_INVALID" + TOO_LARGE = "M_TOO_LARGE" class CodeMessageException(Exception): diff --git a/synapse/config/repository.py b/synapse/config/repository.py index 6eec930a03..f1b7b1b74e 100644 --- a/synapse/config/repository.py +++ b/synapse/config/repository.py @@ -20,6 +20,7 @@ class ContentRepositoryConfig(Config): def __init__(self, args): super(ContentRepositoryConfig, self).__init__(args) self.max_upload_size = self.parse_size(args.max_upload_size) + self.max_image_pixels = self.parse_size(args.max_image_pixels) self.media_store_path = self.ensure_directory(args.media_store_path) def parse_size(self, string): @@ -41,3 +42,7 @@ class ContentRepositoryConfig(Config): db_group.add_argument( "--media-store-path", default=cls.default_path("media_store") ) + db_group.add_argument( + "--max-image-pixels", default="32M", + help="Maximum number of pixels that will be thumbnailed" + ) diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index f05269cdfb..8f4db59c75 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -26,7 +26,7 @@ from synapse.util.logcontext import PreserveLoggingContext from syutil.jsonutil import encode_canonical_json -from synapse.api.errors import CodeMessageException, SynapseError +from synapse.api.errors import CodeMessageException, SynapseError, Codes from syutil.crypto.jsonsign import sign_json @@ -289,7 +289,7 @@ class MatrixFederationHttpClient(object): @defer.inlineCallbacks def get_file(self, destination, path, output_stream, args={}, - retry_on_dns_fail=True): + retry_on_dns_fail=True, max_size=None): """GETs a file from a given homeserver Args: destination (str): The remote server to send the HTTP request to. @@ -325,7 +325,11 @@ class MatrixFederationHttpClient(object): headers = dict(response.headers.getAllRawHeaders()) - length = yield _readBodyToFile(response, output_stream) + try: + length = yield _readBodyToFile(response, output_stream, max_size) + except: + logger.exception("Failed to download body") + raise defer.returnValue((length, headers)) @@ -337,14 +341,23 @@ class MatrixFederationHttpClient(object): class _ReadBodyToFileProtocol(protocol.Protocol): - def __init__(self, stream, deferred): + def __init__(self, stream, deferred, max_size): self.stream = stream self.deferred = deferred self.length = 0 + self.max_size = max_size def dataReceived(self, data): self.stream.write(data) self.length += len(data) + if self.max_size is not None and self.length >= self.max_size: + self.deferred.errback(SynapseError( + 502, + "Requested file is too large > %r bytes" % (self.max_size,), + Codes.TOO_LARGE, + )) + self.deferred = defer.Deferred() + self.transport.loseConnection() def connectionLost(self, reason): if reason.check(ResponseDone): @@ -353,9 +366,9 @@ class _ReadBodyToFileProtocol(protocol.Protocol): self.deferred.errback(reason) -def _readBodyToFile(response, stream): +def _readBodyToFile(response, stream, max_size): d = defer.Deferred() - response.deliverBody(_ReadBodyToFileProtocol(stream, d)) + response.deliverBody(_ReadBodyToFileProtocol(stream, d, max_size)) return d diff --git a/synapse/media/v1/base_resource.py b/synapse/media/v1/base_resource.py index 8c62ecd597..77b05c6548 100644 --- a/synapse/media/v1/base_resource.py +++ b/synapse/media/v1/base_resource.py @@ -43,6 +43,7 @@ class BaseMediaResource(Resource): self.server_name = hs.hostname self.store = hs.get_datastore() self.max_upload_size = hs.config.max_upload_size + self.max_image_pixels = hs.config.max_image_pixels self.filepaths = filepaths @staticmethod @@ -143,6 +144,7 @@ class BaseMediaResource(Resource): )) length, headers = yield self.client.get_file( server_name, request_path, output_stream=f, + max_size=self.max_upload_size, ) media_type = headers["Content-Type"][0] time_now_ms = self.clock.time_msec() @@ -226,6 +228,14 @@ class BaseMediaResource(Resource): thumbnailer = Thumbnailer(input_path) m_width = thumbnailer.width m_height = thumbnailer.height + + if m_width * m_height >= self.max_image_pixels: + logger.info( + "Image too large to thumbnail %r x %r > %r" + m_width, m_height, self.max_image_pixels + ) + return + scales = set() crops = set() for r_width, r_height, r_method, r_type in requirements: @@ -281,6 +291,14 @@ class BaseMediaResource(Resource): thumbnailer = Thumbnailer(input_path) m_width = thumbnailer.width m_height = thumbnailer.height + + if m_width * m_height >= self.max_image_pixels: + logger.info( + "Image too large to thumbnail %r x %r > %r" + m_width, m_height, self.max_image_pixels + ) + return + scales = set() crops = set() for r_width, r_height, r_method, r_type in requirements: From 9191292b0f657f6210e88f16ffd9a182bfab8170 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 11 Dec 2014 15:16:55 +0000 Subject: [PATCH 091/158] Fix prev_content --- synapse/events/utils.py | 8 ++++++++ synapse/storage/_base.py | 6 ++++++ tests/storage/test_stream.py | 5 ++--- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 6d9c9352e2..4ab770dd5f 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -113,6 +113,14 @@ def serialize_event(hs, e): d["redacted_by"] = e.unsigned["redacted_by"] del d["unsigned"]["redacted_by"] + if "replaces_state" in e.unsigned: + d["replaces_state"] = e.unsigned["replaces_state"] + del d["unsigned"]["replaces_state"] + + if "prev_content" in e.unsigned: + d["prev_content"] = e.unsigned["prev_content"] + del d["unsigned"]["prev_content"] + del d["auth_events"] del d["prev_events"] del d["hashes"] diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index ffc26d4a61..e9cf73a8e2 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -487,6 +487,12 @@ class SQLBaseStore(object): if because: ev.unsigned["redacted_because"] = because + if "replaces_state" in ev.unsigned: + ev.unsigned["prev_content"] = self._get_event_txn( + txn, + ev.unsigned["replaces_state"], + ).get_dict()["content"] + return ev def _parse_events(self, rows): diff --git a/tests/storage/test_stream.py b/tests/storage/test_stream.py index cba65bb9f0..4865a5c142 100644 --- a/tests/storage/test_stream.py +++ b/tests/storage/test_stream.py @@ -58,7 +58,7 @@ class StreamStoreTestCase(unittest.TestCase): self.depth = 1 @defer.inlineCallbacks - def inject_room_member(self, room, user, membership, replaces_state=None): + def inject_room_member(self, room, user, membership): self.depth += 1 builder = self.event_builder_factory.new({ @@ -215,7 +215,6 @@ class StreamStoreTestCase(unittest.TestCase): event2 = yield self.inject_room_member( self.room1, self.u_alice, Membership.JOIN, - replaces_state=event1.event_id, ) end = yield self.store.get_room_events_max_id() @@ -233,6 +232,6 @@ class StreamStoreTestCase(unittest.TestCase): event = results[0] self.assertTrue( - hasattr(event, "prev_content"), + "prev_content" in event.unsigned, msg="No prev_content key" ) From 0b0436923819a0252a7a2a6f70a1f929b45b9114 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 11 Dec 2014 15:56:01 +0000 Subject: [PATCH 092/158] Fix public room joining by making sure replaces_state never points to itself. --- synapse/handlers/federation.py | 14 +++++++------- synapse/state.py | 3 ++- synapse/storage/_base.py | 6 ++++-- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 2201cd977e..17779475b8 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -349,7 +349,7 @@ class FederationHandler(BaseHandler): handled_events = set() try: - builder.event_id = self.event_factory.create_event_id() + builder.event_id = self.event_builder_factory.create_event_id() builder.origin = self.hs.hostname builder.content = content @@ -593,13 +593,13 @@ class FederationHandler(BaseHandler): } event = yield self.store.get_event(event_id) - if hasattr(event, "state_key"): + if event and event.is_state(): # Get previous state - if hasattr(event, "replaces_state") and event.replaces_state: - prev_event = yield self.store.get_event( - event.replaces_state - ) - results[(event.type, event.state_key)] = prev_event + if "replaces_state" in event.unsigned: + prev_id = event.unsigned["replaces_state"] + if prev_id != event.event_id: + prev_event = yield self.store.get_event(prev_id) + results[(event.type, event.state_key)] = prev_event else: del results[(event.type, event.state_key)] diff --git a/synapse/state.py b/synapse/state.py index 5bfa73fb46..f9ab5faf9e 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -159,7 +159,8 @@ class StateHandler(object): key = (event.type, event.state_key) if key in context.current_state: replaces = context.current_state[key] - event.unsigned["replaces_state"] = replaces.event_id + if replaces.event_id != event.event_id: # Paranoia check + event.unsigned["replaces_state"] = replaces.event_id defer.returnValue([]) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index e9cf73a8e2..b6f8817b62 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -451,7 +451,8 @@ class SQLBaseStore(object): return events - def _get_event_txn(self, txn, event_id, check_redacted=True): + def _get_event_txn(self, txn, event_id, check_redacted=True, + get_prev_content=True): sql = ( "SELECT json, r.event_id FROM event_json as e " "LEFT JOIN redactions as r ON e.event_id = r.redacts " @@ -487,10 +488,11 @@ class SQLBaseStore(object): if because: ev.unsigned["redacted_because"] = because - if "replaces_state" in ev.unsigned: + if get_prev_content and "replaces_state" in ev.unsigned: ev.unsigned["prev_content"] = self._get_event_txn( txn, ev.unsigned["replaces_state"], + get_prev_content=False, ).get_dict()["content"] return ev From 3b2cc260531c18681c29e771a79f3374013b5c63 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 11 Dec 2014 16:03:04 +0000 Subject: [PATCH 093/158] Initial hack at unit tests of room typing REST API --- synapse/handlers/typing.py | 7 ++++ tests/rest/test_rooms.py | 71 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 75 insertions(+), 3 deletions(-) diff --git a/synapse/handlers/typing.py b/synapse/handlers/typing.py index 989f73947f..c2b890261a 100644 --- a/synapse/handlers/typing.py +++ b/synapse/handlers/typing.py @@ -52,6 +52,13 @@ class TypingNotificationHandler(BaseHandler): # map room IDs to sets of users currently typing self._room_typing = {} + def tearDown(self): + """Cancels all the pending timers. + Normally this shouldn't be needed, but it's required from unit tests + to avoid a "Reactor was unclean" warning.""" + for t in self._member_typing_timer.values(): + self.clock.cancel_call_later(t) + @defer.inlineCallbacks def started_typing(self, target_user, auth_user, room_id, timeout): if not target_user.is_mine: diff --git a/tests/rest/test_rooms.py b/tests/rest/test_rooms.py index ff7c9f0530..5273105304 100644 --- a/tests/rest/test_rooms.py +++ b/tests/rest/test_rooms.py @@ -1067,6 +1067,71 @@ class RoomInitialSyncTestCase(RestTestCase): self.assertTrue(self.user_id in presence_by_user) self.assertEquals("m.presence", presence_by_user[self.user_id]["type"]) -# (code, response) = yield self.mock_resource.trigger("GET", path, None) -# self.assertEquals(200, code, msg=str(response)) -# self.assert_dict(json.loads(content), response) + +class RoomTypingTestCase(RestTestCase): + """ Tests /rooms/$room_id/typing/$user_id REST API. """ + user_id = "@sid:red" + + @defer.inlineCallbacks + def setUp(self): + self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) + self.auth_user_id = self.user_id + + self.mock_config = NonCallableMock() + self.mock_config.signing_key = [MockKey()] + + db_pool = SQLiteMemoryDbPool() + yield db_pool.prepare() + + hs = HomeServer( + "red", + db_pool=db_pool, + http_client=None, + replication_layer=Mock(), + ratelimiter=NonCallableMock(spec_set=[ + "send_message", + ]), + config=self.mock_config, + ) + self.hs = hs + + self.ratelimiter = hs.get_ratelimiter() + self.ratelimiter.send_message.return_value = (True, 0) + + hs.get_handlers().federation_handler = Mock() + + def _get_user_by_token(token=None): + return { + "user": hs.parse_userid(self.auth_user_id), + "admin": False, + "device_id": None, + } + + hs.get_auth().get_user_by_token = _get_user_by_token + + def _insert_client_ip(*args, **kwargs): + return defer.succeed(None) + hs.get_datastore().insert_client_ip = _insert_client_ip + + synapse.rest.room.register_servlets(hs, self.mock_resource) + + self.room_id = yield self.create_room_as(self.user_id) + + def tearDown(self): + self.hs.get_handlers().typing_notification_handler.tearDown() + + @defer.inlineCallbacks + def test_set_typing(self): + (code, _) = yield self.mock_resource.trigger("PUT", + "/rooms/%s/typing/%s" % (self.room_id, self.user_id), + '{"typing": true, "timeout": 30000}' + ) + self.assertEquals(200, code) + + @defer.inlineCallbacks + def test_set_not_typing(self): + (code, _) = yield self.mock_resource.trigger("PUT", + "/rooms/%s/typing/%s" % (self.room_id, self.user_id), + '{"typing": false}' + ) + self.assertEquals(200, code) From c161b6cf96992e6fa3af11fac770f1d6472103a4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 11 Dec 2014 16:43:30 +0000 Subject: [PATCH 094/158] Fix room creation test --- tests/handlers/test_room.py | 68 ++++++++++++++++++------------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/tests/handlers/test_room.py b/tests/handlers/test_room.py index 0279ab703a..1e6e7c3602 100644 --- a/tests/handlers/test_room.py +++ b/tests/handlers/test_room.py @@ -20,7 +20,7 @@ from tests import unittest from synapse.api.events.room import ( RoomMemberEvent, ) -from synapse.api.constants import Membership +from synapse.api.constants import EventTypes, Membership from synapse.handlers.room import RoomMemberHandler, RoomCreationHandler from synapse.handlers.profile import ProfileHandler from synapse.server import HomeServer @@ -254,13 +254,9 @@ class RoomCreationTest(unittest.TestCase): notifier=NonCallableMock(spec_set=["on_new_room_event"]), handlers=NonCallableMock(spec_set=[ "room_creation_handler", - "room_member_handler", - "federation_handler", + "message_handler", ]), auth=NonCallableMock(spec_set=["check", "add_auth_events"]), - state_handler=NonCallableMock(spec_set=[ - "annotate_event_with_state", - ]), ratelimiter=NonCallableMock(spec_set=[ "send_message", ]), @@ -271,30 +267,12 @@ class RoomCreationTest(unittest.TestCase): "handle_new_event", ]) - self.datastore = hs.get_datastore() self.handlers = hs.get_handlers() - self.notifier = hs.get_notifier() - self.state_handler = hs.get_state_handler() - self.hs = hs - self.handlers.federation_handler = self.federation - - self.handlers.room_creation_handler = RoomCreationHandler(self.hs) + self.handlers.room_creation_handler = RoomCreationHandler(hs) self.room_creation_handler = self.handlers.room_creation_handler - self.handlers.room_member_handler = NonCallableMock(spec_set=[ - "change_membership" - ]) - self.room_member_handler = self.handlers.room_member_handler - - def annotate(event): - event.state_events = {} - return defer.succeed(None) - self.state_handler.annotate_event_with_state.side_effect = annotate - - def hosts(room): - return defer.succeed([]) - self.datastore.get_joined_hosts_for_room.side_effect = hosts + self.message_handler = self.handlers.message_handler self.ratelimiter = hs.get_ratelimiter() self.ratelimiter.send_message.return_value = (True, 0) @@ -311,14 +289,36 @@ class RoomCreationTest(unittest.TestCase): config=config, ) - self.assertTrue(self.room_member_handler.change_membership.called) - join_event = self.room_member_handler.change_membership.call_args[0][0] + self.assertTrue(self.message_handler.handle_event.called) - self.assertEquals(RoomMemberEvent.TYPE, join_event.type) - self.assertEquals(room_id, join_event.room_id) - self.assertEquals(user_id, join_event.user_id) - self.assertEquals(user_id, join_event.state_key) + event_dicts = [ + e[0][0] for e in self.message_handler.handle_event.call_args_list + ] - self.assertTrue(self.state_handler.annotate_event_with_state.called) + self.assertTrue(len(event_dicts) > 3) - self.assertTrue(self.federation.handle_new_event.called) + self.assertDictContainsSubset( + { + "type": EventTypes.Create, + "sender": user_id, + "room_id": room_id, + }, + event_dicts[0] + ) + + self.assertEqual(user_id, event_dicts[0]["content"]["creator"]) + + self.assertDictContainsSubset( + { + "type": EventTypes.Member, + "sender": user_id, + "room_id": room_id, + "state_key": user_id, + }, + event_dicts[1] + ) + + self.assertEqual( + Membership.JOIN, + event_dicts[1]["content"]["membership"] + ) From 03d9024cbcb4957b223d1a36e7ae2ad668b1859d Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 11 Dec 2014 16:48:11 +0000 Subject: [PATCH 095/158] Allow only one download for a given image at a time, so that we don't end up downloading the same image twice if two clients request a remote image at the same time --- synapse/media/v1/base_resource.py | 27 ++++++++++++++++++++++++-- synapse/media/v1/download_resource.py | 9 +-------- synapse/media/v1/thumbnail_resource.py | 13 +++---------- 3 files changed, 29 insertions(+), 20 deletions(-) diff --git a/synapse/media/v1/base_resource.py b/synapse/media/v1/base_resource.py index 77b05c6548..14735ff375 100644 --- a/synapse/media/v1/base_resource.py +++ b/synapse/media/v1/base_resource.py @@ -45,6 +45,7 @@ class BaseMediaResource(Resource): self.max_upload_size = hs.config.max_upload_size self.max_image_pixels = hs.config.max_image_pixels self.filepaths = filepaths + self.downloads = {} @staticmethod def catch_errors(request_handler): @@ -128,6 +129,28 @@ class BaseMediaResource(Resource): if not os.path.exists(dirname): os.makedirs(dirname) + def _get_remote_media(self, server_name, media_id): + key = (server_name, media_id) + download = self.downloads.get(key) + if download is None: + download = self._get_remote_media_impl(server_name, media_id) + self.downloads[key] = download + @download.addBoth + def callback(media_info): + del self.downloads[key] + return download + + @defer.inlineCallbacks + def _get_remote_media_impl(self, server_name, media_id): + media_info = yield self.store.get_cached_remote_media( + server_name, media_id + ) + if not media_info: + media_info = yield self._download_remote_file( + server_name, media_id + ) + defer.returnValue(media_info) + @defer.inlineCallbacks def _download_remote_file(self, server_name, media_id): file_id = random_string(24) @@ -231,7 +254,7 @@ class BaseMediaResource(Resource): if m_width * m_height >= self.max_image_pixels: logger.info( - "Image too large to thumbnail %r x %r > %r" + "Image too large to thumbnail %r x %r > %r", m_width, m_height, self.max_image_pixels ) return @@ -294,7 +317,7 @@ class BaseMediaResource(Resource): if m_width * m_height >= self.max_image_pixels: logger.info( - "Image too large to thumbnail %r x %r > %r" + "Image too large to thumbnail %r x %r > %r", m_width, m_height, self.max_image_pixels ) return diff --git a/synapse/media/v1/download_resource.py b/synapse/media/v1/download_resource.py index 6de0932ba3..f3a6804e05 100644 --- a/synapse/media/v1/download_resource.py +++ b/synapse/media/v1/download_resource.py @@ -56,14 +56,7 @@ class DownloadResource(BaseMediaResource): @defer.inlineCallbacks def _respond_remote_file(self, request, server_name, media_id): - media_info = yield self.store.get_cached_remote_media( - server_name, media_id - ) - - if not media_info: - media_info = yield self._download_remote_file( - server_name, media_id - ) + media_info = yield self._get_remote_media(server_name, media_id) media_type = media_info["media_type"] filesystem_id = media_info["filesystem_id"] diff --git a/synapse/media/v1/thumbnail_resource.py b/synapse/media/v1/thumbnail_resource.py index fd08c7ecd2..e19620d456 100644 --- a/synapse/media/v1/thumbnail_resource.py +++ b/synapse/media/v1/thumbnail_resource.py @@ -83,16 +83,9 @@ class ThumbnailResource(BaseMediaResource): @defer.inlineCallbacks def _respond_remote_thumbnail(self, request, server_name, media_id, width, height, method, m_type): - media_info = yield self.store.get_cached_remote_media( - server_name, media_id - ) - - if not media_info: - # TODO: Don't download the whole remote file - # We should proxy the thumbnail from the remote server instead. - media_info = yield self._download_remote_file( - server_name, media_id - ) + # TODO: Don't download the whole remote file + # We should proxy the thumbnail from the remote server instead. + media_info = yield self._get_remote_media(server_name, media_id) thumbnail_infos = yield self.store.get_remote_media_thumbnails( server_name, media_id, From d3eb12c7b8292f2879e5aa11cb5ffec05ce2a3a5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 11 Dec 2014 17:01:27 +0000 Subject: [PATCH 096/158] Fix federation test --- synapse/handlers/federation.py | 2 +- tests/handlers/test_federation.py | 42 ++++++++++++++++++------------- 2 files changed, 25 insertions(+), 19 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 17779475b8..ddcb28c8e2 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -136,7 +136,7 @@ class FederationHandler(BaseHandler): if not check_event_content_hash(event): logger.warn( "Event content has been tampered, redacting %s, %s", - event.event_id, encode_canonical_json(event.get_full_dict()) + event.event_id, encode_canonical_json(event.get_dict()) ) event = redacted_event diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py index fae33716a3..91f7351087 100644 --- a/tests/handlers/test_federation.py +++ b/tests/handlers/test_federation.py @@ -20,7 +20,7 @@ from synapse.api.events.room import ( MessageEvent, ) -from synapse.api.events import SynapseEvent +from synapse.events import FrozenEvent from synapse.handlers.federation import FederationHandler from synapse.server import HomeServer @@ -37,7 +37,7 @@ class FederationTestCase(unittest.TestCase): self.mock_config.signing_key = [MockKey()] self.state_handler = NonCallableMock(spec_set=[ - "annotate_event_with_state", + "annotate_context_with_state", ]) self.auth = NonCallableMock(spec_set=[ @@ -78,36 +78,42 @@ class FederationTestCase(unittest.TestCase): @defer.inlineCallbacks def test_msg(self): - pdu = SynapseEvent( - type=MessageEvent.TYPE, - room_id="foo", - content={"msgtype": u"fooo"}, - origin_server_ts=0, - event_id="$a:b", - user_id="@a:b", - origin="b", - auth_events=[], - hashes={"sha256":"AcLrgtUIqqwaGoHhrEvYG1YLDIsVPYJdSRGhkp3jJp8"}, - ) + pdu = FrozenEvent({ + "type": MessageEvent.TYPE, + "room_id": "foo", + "content": {"msgtype": u"fooo"}, + "origin_server_ts": 0, + "event_id": "$a:b", + "user_id":"@a:b", + "origin": "b", + "auth_events": [], + "hashes": {"sha256":"AcLrgtUIqqwaGoHhrEvYG1YLDIsVPYJdSRGhkp3jJp8"}, + }) self.datastore.persist_event.return_value = defer.succeed(None) self.datastore.get_room.return_value = defer.succeed(True) self.auth.check_host_in_room.return_value = defer.succeed(True) - def annotate(ev, old_state=None): - ev.old_state_events = [] + def annotate(ev, context, old_state=None): + context.current_state = {} + context.auth_events = {} return defer.succeed(False) - self.state_handler.annotate_event_with_state.side_effect = annotate + self.state_handler.annotate_context_with_state.side_effect = annotate yield self.handlers.federation_handler.on_receive_pdu( "fo", pdu, False ) self.datastore.persist_event.assert_called_once_with( - ANY, is_new_state=False, backfilled=False, current_state=None + ANY, + is_new_state=True, + backfilled=False, + current_state=None, + context=ANY, ) - self.state_handler.annotate_event_with_state.assert_called_once_with( + self.state_handler.annotate_context_with_state.assert_called_once_with( + ANY, ANY, old_state=None, ) From 3fecacd86b32ab8d5a2b16bc124167227eb8dcb7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 11 Dec 2014 17:11:06 +0000 Subject: [PATCH 097/158] Fix replication tests --- tests/federation/test_federation.py | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/tests/federation/test_federation.py b/tests/federation/test_federation.py index f6b41e2c4c..79ac1ce10d 100644 --- a/tests/federation/test_federation.py +++ b/tests/federation/test_federation.py @@ -23,25 +23,20 @@ from ..utils import MockHttpResource, MockClock, MockKey from synapse.server import HomeServer from synapse.federation import initialize_http_replication -from synapse.api.events import SynapseEvent +from synapse.events import FrozenEvent from synapse.storage.transactions import DestinationsTable + def make_pdu(prev_pdus=[], **kwargs): """Provide some default fields for making a PduTuple.""" pdu_fields = { - "is_state": False, - "unrecognized_keys": [], - "outlier": False, - "have_processed": True, "state_key": None, - "power_level": None, - "prev_state_id": None, - "prev_state_origin": None, + "prev_events": prev_pdus, } pdu_fields.update(kwargs) - return SynapseEvent(prev_pdus=prev_pdus, **pdu_fields) + return FrozenEvent(pdu_fields) class FederationTestCase(unittest.TestCase): @@ -176,7 +171,7 @@ class FederationTestCase(unittest.TestCase): (200, "OK") ) - pdu = SynapseEvent( + pdu = make_pdu( event_id="abc123def456", origin="red", user_id="@a:red", @@ -185,10 +180,9 @@ class FederationTestCase(unittest.TestCase): origin_server_ts=123456789001, depth=1, content={"text": "Here is the message"}, - destinations=["remote"], ) - yield self.federation.send_pdu(pdu) + yield self.federation.send_pdu(pdu, ["remote"]) self.mock_http_client.put_json.assert_called_with( "remote", From 1f26e56de003fcf0fe8a8f064611ca40e997a3e6 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 11 Dec 2014 17:31:56 +0000 Subject: [PATCH 098/158] Actually unit-test the event stream around REST typing tests --- tests/rest/test_rooms.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/rest/test_rooms.py b/tests/rest/test_rooms.py index 5273105304..a34f832758 100644 --- a/tests/rest/test_rooms.py +++ b/tests/rest/test_rooms.py @@ -1095,6 +1095,8 @@ class RoomTypingTestCase(RestTestCase): ) self.hs = hs + self.event_source = hs.get_event_sources().sources["typing"] + self.ratelimiter = hs.get_ratelimiter() self.ratelimiter.send_message.return_value = (True, 0) @@ -1116,6 +1118,8 @@ class RoomTypingTestCase(RestTestCase): synapse.rest.room.register_servlets(hs, self.mock_resource) self.room_id = yield self.create_room_as(self.user_id) + # Need another user to make notifications actually work + yield self.join(self.room_id, user="@jim:red") def tearDown(self): self.hs.get_handlers().typing_notification_handler.tearDown() @@ -1128,6 +1132,16 @@ class RoomTypingTestCase(RestTestCase): ) self.assertEquals(200, code) + self.assertEquals(self.event_source.get_current_key(), 1) + self.assertEquals( + self.event_source.get_new_events_for_user(self.user_id, 0, None)[0], + [ + {"type": "m.typing", + "room_id": self.room_id, + "typing": [self.user_id]}, + ] + ) + @defer.inlineCallbacks def test_set_not_typing(self): (code, _) = yield self.mock_resource.trigger("PUT", From 6e1531682b694f8db13244e08241f5f0476ea1fc Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 11 Dec 2014 17:39:08 +0000 Subject: [PATCH 099/158] Move typing-notification REST tests into their own .py file --- tests/rest/test_rooms.py | 83 ---------------------------- tests/rest/test_typing.py | 113 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+), 83 deletions(-) create mode 100644 tests/rest/test_typing.py diff --git a/tests/rest/test_rooms.py b/tests/rest/test_rooms.py index a34f832758..1f719beb0a 100644 --- a/tests/rest/test_rooms.py +++ b/tests/rest/test_rooms.py @@ -1066,86 +1066,3 @@ class RoomInitialSyncTestCase(RestTestCase): } self.assertTrue(self.user_id in presence_by_user) self.assertEquals("m.presence", presence_by_user[self.user_id]["type"]) - - -class RoomTypingTestCase(RestTestCase): - """ Tests /rooms/$room_id/typing/$user_id REST API. """ - user_id = "@sid:red" - - @defer.inlineCallbacks - def setUp(self): - self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) - self.auth_user_id = self.user_id - - self.mock_config = NonCallableMock() - self.mock_config.signing_key = [MockKey()] - - db_pool = SQLiteMemoryDbPool() - yield db_pool.prepare() - - hs = HomeServer( - "red", - db_pool=db_pool, - http_client=None, - replication_layer=Mock(), - ratelimiter=NonCallableMock(spec_set=[ - "send_message", - ]), - config=self.mock_config, - ) - self.hs = hs - - self.event_source = hs.get_event_sources().sources["typing"] - - self.ratelimiter = hs.get_ratelimiter() - self.ratelimiter.send_message.return_value = (True, 0) - - hs.get_handlers().federation_handler = Mock() - - def _get_user_by_token(token=None): - return { - "user": hs.parse_userid(self.auth_user_id), - "admin": False, - "device_id": None, - } - - hs.get_auth().get_user_by_token = _get_user_by_token - - def _insert_client_ip(*args, **kwargs): - return defer.succeed(None) - hs.get_datastore().insert_client_ip = _insert_client_ip - - synapse.rest.room.register_servlets(hs, self.mock_resource) - - self.room_id = yield self.create_room_as(self.user_id) - # Need another user to make notifications actually work - yield self.join(self.room_id, user="@jim:red") - - def tearDown(self): - self.hs.get_handlers().typing_notification_handler.tearDown() - - @defer.inlineCallbacks - def test_set_typing(self): - (code, _) = yield self.mock_resource.trigger("PUT", - "/rooms/%s/typing/%s" % (self.room_id, self.user_id), - '{"typing": true, "timeout": 30000}' - ) - self.assertEquals(200, code) - - self.assertEquals(self.event_source.get_current_key(), 1) - self.assertEquals( - self.event_source.get_new_events_for_user(self.user_id, 0, None)[0], - [ - {"type": "m.typing", - "room_id": self.room_id, - "typing": [self.user_id]}, - ] - ) - - @defer.inlineCallbacks - def test_set_not_typing(self): - (code, _) = yield self.mock_resource.trigger("PUT", - "/rooms/%s/typing/%s" % (self.room_id, self.user_id), - '{"typing": false}' - ) - self.assertEquals(200, code) diff --git a/tests/rest/test_typing.py b/tests/rest/test_typing.py new file mode 100644 index 0000000000..0b95d70718 --- /dev/null +++ b/tests/rest/test_typing.py @@ -0,0 +1,113 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests REST events for /rooms paths.""" + +# twisted imports +from twisted.internet import defer + +import synapse.rest.room +from synapse.server import HomeServer + +from ..utils import MockHttpResource, SQLiteMemoryDbPool, MockKey +from .utils import RestTestCase + +from mock import Mock, NonCallableMock + + +PATH_PREFIX = "/_matrix/client/api/v1" + + +class RoomTypingTestCase(RestTestCase): + """ Tests /rooms/$room_id/typing/$user_id REST API. """ + user_id = "@sid:red" + + @defer.inlineCallbacks + def setUp(self): + self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) + self.auth_user_id = self.user_id + + self.mock_config = NonCallableMock() + self.mock_config.signing_key = [MockKey()] + + db_pool = SQLiteMemoryDbPool() + yield db_pool.prepare() + + hs = HomeServer( + "red", + db_pool=db_pool, + http_client=None, + replication_layer=Mock(), + ratelimiter=NonCallableMock(spec_set=[ + "send_message", + ]), + config=self.mock_config, + ) + self.hs = hs + + self.event_source = hs.get_event_sources().sources["typing"] + + self.ratelimiter = hs.get_ratelimiter() + self.ratelimiter.send_message.return_value = (True, 0) + + hs.get_handlers().federation_handler = Mock() + + def _get_user_by_token(token=None): + return { + "user": hs.parse_userid(self.auth_user_id), + "admin": False, + "device_id": None, + } + + hs.get_auth().get_user_by_token = _get_user_by_token + + def _insert_client_ip(*args, **kwargs): + return defer.succeed(None) + hs.get_datastore().insert_client_ip = _insert_client_ip + + synapse.rest.room.register_servlets(hs, self.mock_resource) + + self.room_id = yield self.create_room_as(self.user_id) + # Need another user to make notifications actually work + yield self.join(self.room_id, user="@jim:red") + + def tearDown(self): + self.hs.get_handlers().typing_notification_handler.tearDown() + + @defer.inlineCallbacks + def test_set_typing(self): + (code, _) = yield self.mock_resource.trigger("PUT", + "/rooms/%s/typing/%s" % (self.room_id, self.user_id), + '{"typing": true, "timeout": 30000}' + ) + self.assertEquals(200, code) + + self.assertEquals(self.event_source.get_current_key(), 1) + self.assertEquals( + self.event_source.get_new_events_for_user(self.user_id, 0, None)[0], + [ + {"type": "m.typing", + "room_id": self.room_id, + "typing": [self.user_id]}, + ] + ) + + @defer.inlineCallbacks + def test_set_not_typing(self): + (code, _) = yield self.mock_resource.trigger("PUT", + "/rooms/%s/typing/%s" % (self.room_id, self.user_id), + '{"typing": false}' + ) + self.assertEquals(200, code) From 966c4b2b04fe5dcaa53a6b33b86ac9dbc6f94eb2 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 11 Dec 2014 18:00:15 +0000 Subject: [PATCH 100/158] Add a sprinkling of logger.debug() into typing notification handler --- synapse/handlers/typing.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/synapse/handlers/typing.py b/synapse/handlers/typing.py index c2b890261a..c55221c6df 100644 --- a/synapse/handlers/typing.py +++ b/synapse/handlers/typing.py @@ -67,6 +67,10 @@ class TypingNotificationHandler(BaseHandler): if target_user != auth_user: raise AuthError(400, "Cannot set another user's typing state") + logger.debug( + "%s has started typing in %s", target_user.to_string(), room_id + ) + until = self.clock.time_msec() + timeout member = RoomMember(room_id=room_id, user=target_user) @@ -98,6 +102,10 @@ class TypingNotificationHandler(BaseHandler): if target_user != auth_user: raise AuthError(400, "Cannot set another user's typing state") + logger.debug( + "%s has stopped typing in %s", target_user.to_string(), room_id + ) + member = RoomMember(room_id=room_id, user=target_user) yield self._stopped_typing(member) From 5ebc994f841508beb48f06d22073e08845c5f593 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 11 Dec 2014 18:11:43 +0000 Subject: [PATCH 101/158] Actually auth-check to ensure people can only send typing notifications for rooms they're actually in --- synapse/handlers/typing.py | 4 ++++ tests/handlers/test_typing.py | 10 ++++++++++ 2 files changed, 14 insertions(+) diff --git a/synapse/handlers/typing.py b/synapse/handlers/typing.py index c55221c6df..fa903c251c 100644 --- a/synapse/handlers/typing.py +++ b/synapse/handlers/typing.py @@ -67,6 +67,8 @@ class TypingNotificationHandler(BaseHandler): if target_user != auth_user: raise AuthError(400, "Cannot set another user's typing state") + yield self.auth.check_joined_room(room_id, target_user.to_string()) + logger.debug( "%s has started typing in %s", target_user.to_string(), room_id ) @@ -102,6 +104,8 @@ class TypingNotificationHandler(BaseHandler): if target_user != auth_user: raise AuthError(400, "Cannot set another user's typing state") + yield self.auth.check_joined_room(room_id, target_user.to_string()) + logger.debug( "%s has stopped typing in %s", target_user.to_string(), room_id ) diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py index b858f96323..bc19db8dfa 100644 --- a/tests/handlers/test_typing.py +++ b/tests/handlers/test_typing.py @@ -22,6 +22,7 @@ import json from ..utils import MockHttpResource, MockClock, DeferredMockCallable, MockKey +from synapse.api.errors import AuthError from synapse.server import HomeServer from synapse.handlers.typing import TypingNotificationHandler @@ -68,7 +69,10 @@ class TypingNotificationsTestCase(unittest.TestCase): mock_notifier = Mock(spec=["on_new_user_event"]) self.on_new_user_event = mock_notifier.on_new_user_event + self.auth = Mock(spec=[]) + hs = HomeServer("test", + auth=self.auth, clock=self.clock, db_pool=None, datastore=Mock(spec=[ @@ -142,6 +146,12 @@ class TypingNotificationsTestCase(unittest.TestCase): self.room_member_handler.fetch_room_distributions_into = ( fetch_room_distributions_into) + def check_joined_room(room_id, user_id): + if user_id not in [u.to_string() for u in self.room_members]: + raise AuthError(401, "User is not in the room") + + self.auth.check_joined_room = check_joined_room + # Some local users to test with self.u_apple = hs.parse_userid("@apple:test") self.u_banana = hs.parse_userid("@banana:test") From 64bf9f54ccb4facb9b0a1ecef85e9d82b3268eda Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 11 Dec 2014 18:18:58 +0000 Subject: [PATCH 102/158] Fix media repository doc string to include server_name --- synapse/media/v1/media_repository.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/media/v1/media_repository.py b/synapse/media/v1/media_repository.py index a0dc56be4b..afbe07ad74 100644 --- a/synapse/media/v1/media_repository.py +++ b/synapse/media/v1/media_repository.py @@ -41,7 +41,7 @@ class MediaRepositoryResource(Resource): { "token": } - => GET /_matrix/media/v1/download/ HTTP/1.1 + => GET /_matrix/media/v1/download// HTTP/1.1 <= HTTP/1.1 200 OK Content-Type: @@ -52,8 +52,8 @@ class MediaRepositoryResource(Resource): Clients can get thumbnails by supplying a desired width and height and thumbnailing method:: - => GET /_matrix/media/v1 - /thumbnail/?width=&height=&method= HTTP/1.1 + => GET /_matrix/media/v1/thumbnail/ + /?width=&height=&method= HTTP/1.1 <= HTTP/1.1 200 OK Content-Type: image/jpeg or image/png From b3e34a53997e52d148ebc9fd3f3b0f589e0338d0 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 11 Dec 2014 18:21:08 +0000 Subject: [PATCH 103/158] Fix typo in media repository doc string --- synapse/media/v1/media_repository.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/media/v1/media_repository.py b/synapse/media/v1/media_repository.py index afbe07ad74..7e446fd827 100644 --- a/synapse/media/v1/media_repository.py +++ b/synapse/media/v1/media_repository.py @@ -26,7 +26,7 @@ logger = logging.getLogger(__name__) class MediaRepositoryResource(Resource): - """Profiles file uploading and downloading. + """File uploading and downloading. Uploads are POSTed to a resource which returns a token which is used to GET the download:: From f25764943c81c145a8e787444c3b16c973e2286c Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 11 Dec 2014 18:27:01 +0000 Subject: [PATCH 104/158] Add a 'user_left_room' distributor signal analogous to 'user_joined_room' --- synapse/handlers/room.py | 7 ++++++ tests/handlers/test_room.py | 44 ++++++++++++++++++++++++++++++++++--- 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index a000b44036..c802e8f69d 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -260,6 +260,7 @@ class RoomMemberHandler(BaseHandler): self.distributor = hs.get_distributor() self.distributor.declare("user_joined_room") + self.distributor.declare("user_left_room") @defer.inlineCallbacks def get_room_members(self, room_id, membership=Membership.JOIN): @@ -387,6 +388,12 @@ class RoomMemberHandler(BaseHandler): do_auth=do_auth, ) + if prev_state and prev_state.membership == Membership.JOIN: + user = self.hs.parse_userid(event.user_id) + self.distributor.fire( + "user_left_room", user=user, room_id=event.room_id + ) + defer.returnValue({"room_id": room_id}) @defer.inlineCallbacks diff --git a/tests/handlers/test_room.py b/tests/handlers/test_room.py index 0279ab703a..3a71ed0aed 100644 --- a/tests/handlers/test_room.py +++ b/tests/handlers/test_room.py @@ -222,14 +222,52 @@ class RoomMemberHandlerTestCase(unittest.TestCase): user=user, room_id=room_id ) - def _create_member(self, user_id, room_id): + @defer.inlineCallbacks + def test_simple_leave(self): + room_id = "!foo:red" + user_id = "@bob:red" + user = self.hs.parse_userid(user_id) + + event = self._create_member( + user_id=user_id, + room_id=room_id, + membership=Membership.LEAVE, + ) + + prev_state = NonCallableMock() + prev_state.membership = Membership.JOIN + prev_state.sender = user_id + self.datastore.get_room_member.return_value = defer.succeed(prev_state) + + event.state_events = { + (RoomMemberEvent.TYPE, user_id): event, + } + + event.old_state_events = { + (RoomMemberEvent.TYPE, user_id): self._create_member( + user_id=user_id, + room_id=room_id, + ), + } + + leave_signal_observer = Mock() + self.distributor.observe("user_left_room", leave_signal_observer) + + # Actual invocation + yield self.room_member_handler.change_membership(event) + + leave_signal_observer.assert_called_with( + user=user, room_id=room_id + ) + + def _create_member(self, user_id, room_id, membership=Membership.JOIN): return self.hs.get_event_factory().create_event( etype=RoomMemberEvent.TYPE, user_id=user_id, state_key=user_id, room_id=room_id, - membership=Membership.JOIN, - content={"membership": Membership.JOIN}, + membership=membership, + content={"membership": membership}, ) From cfb963af0376a02fa527c458530d08c92893d21a Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Thu, 11 Dec 2014 18:33:09 +0000 Subject: [PATCH 105/158] When users leave rooms mark them as no longer typing in them --- synapse/handlers/typing.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/synapse/handlers/typing.py b/synapse/handlers/typing.py index fa903c251c..46a0b299a1 100644 --- a/synapse/handlers/typing.py +++ b/synapse/handlers/typing.py @@ -43,6 +43,8 @@ class TypingNotificationHandler(BaseHandler): self.federation.register_edu_handler("m.typing", self._recv_edu) + hs.get_distributor().observe("user_left_room", self.user_left_room) + self._member_typing_until = {} # clock time we expect to stop self._member_typing_timer = {} # deferreds to manage theabove @@ -114,6 +116,12 @@ class TypingNotificationHandler(BaseHandler): yield self._stopped_typing(member) + @defer.inlineCallbacks + def user_left_room(self, user, room_id): + if user.is_mine: + member = RoomMember(room_id=room_id, user=user) + yield self._stopped_typing(member) + @defer.inlineCallbacks def _stopped_typing(self, member): if member not in self._member_typing_until: From fa4b610ae383bdced11068832d880d14b1df576d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 12 Dec 2014 10:42:27 +0000 Subject: [PATCH 106/158] Fix stream test. Make sure we add join to auth_events for invitiations --- synapse/api/auth.py | 6 ++++++ tests/rest/test_events.py | 4 ---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index cd0deeb0e6..c76f5180f6 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -61,6 +61,8 @@ class Auth(object): if event.type == RoomAliasesEvent.TYPE: return True + logger.debug("Auth events: %s", auth_events) + if event.type == RoomMemberEvent.TYPE: allowed = self.is_membership_change_allowed( event, auth_events @@ -389,8 +391,12 @@ class Auth(object): if join_rule_event: auth_ids.append(join_rule_event.event_id) + if e_type == Membership.JOIN: if member_event and not is_public: auth_ids.append(member_event.event_id) + else: + if member_event: + auth_ids.append(member_event.event_id) elif member_event: if member_event.content["membership"] == Membership.JOIN: auth_ids.append(member_event.event_id) diff --git a/tests/rest/test_events.py b/tests/rest/test_events.py index 4a3234c332..d3159e2cf4 100644 --- a/tests/rest/test_events.py +++ b/tests/rest/test_events.py @@ -113,9 +113,6 @@ class EventStreamPermissionsTestCase(RestTestCase): def setUp(self): self.mock_resource = MockHttpResource(prefix=PATH_PREFIX) - persistence_service = Mock(spec=["get_latest_pdus_in_context"]) - persistence_service.get_latest_pdus_in_context.return_value = [] - self.mock_config = NonCallableMock() self.mock_config.signing_key = [MockKey()] @@ -127,7 +124,6 @@ class EventStreamPermissionsTestCase(RestTestCase): db_pool=db_pool, http_client=None, replication_layer=Mock(), - persistence_service=persistence_service, clock=Mock(spec=[ "call_later", "cancel_call_later", From 63810c777d8c02bc6fefa2a4bcfacb7f4df21ba2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 12 Dec 2014 10:56:14 +0000 Subject: [PATCH 107/158] Validate message, topic and name event contents --- synapse/api/constants.py | 5 +++++ synapse/events/validator.py | 21 +++++++++++++++++++++ synapse/handlers/message.py | 2 +- 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 7e8c892b64..b668da4a26 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -68,3 +68,8 @@ class EventTypes(object): PowerLevels = "m.room.power_levels" Aliases = "m.room.aliases" Redaction = "m.room.redaction" + + # These are used for validation + Message = "m.room.message" + Topic = "m.room.topic" + Name = "m.room.name" diff --git a/synapse/events/validator.py b/synapse/events/validator.py index 47830aa985..ebc6c30e62 100644 --- a/synapse/events/validator.py +++ b/synapse/events/validator.py @@ -69,3 +69,24 @@ class EventValidator(object): self.validate(event) UserID.from_string(event.sender) + + if event.type == EventTypes.Message: + strings = [ + "body", + "msgtype", + ] + + self._ensure_strings(event.content, strings) + + elif event.type == EventTypes.Topic: + self._ensure_strings(event.content, ["topic"]) + + elif event.type == EventTypes.Name: + self._ensure_strings(event.content, ["name"]) + + def _ensure_strings(self, d, keys): + for s in keys: + if s not in d: + raise SynapseError(400, "'%s' not in content" % (s,)) + if not isinstance(d[s], basestring): + raise SynapseError(400, "Not '%s' a string type" % (s,)) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index f92b01a50f..4fa4ffea25 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -141,7 +141,7 @@ class MessageHandler(BaseHandler): def handle_event(self, event_dict): builder = self.event_builder_factory.new(event_dict) - self.validator.validate(builder) + self.validator.validate_new(builder) if builder.type == EventTypes.Member: membership = builder.content.get("membership", None) From b0bb1756a9a85f476c6ebd03a7e78ad5f403311c Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Fri, 12 Dec 2014 11:59:46 +0000 Subject: [PATCH 108/158] Send list of typing user IDs as 'user_ids' list within 'content', so that m.typing stream events have a toplevel content, for consistency with others --- synapse/handlers/typing.py | 4 +++- tests/handlers/test_typing.py | 20 +++++++++++++++----- tests/rest/test_typing.py | 4 +++- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/synapse/handlers/typing.py b/synapse/handlers/typing.py index 46a0b299a1..253fec514d 100644 --- a/synapse/handlers/typing.py +++ b/synapse/handlers/typing.py @@ -223,7 +223,9 @@ class TypingNotificationEventSource(object): return { "type": "m.typing", "room_id": room_id, - "typing": [u.to_string() for u in typing], + "content": { + "user_ids": [u.to_string() for u in typing], + }, } def get_new_events_for_user(self, user, from_key, limit): diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py index bc19db8dfa..391e287adc 100644 --- a/tests/handlers/test_typing.py +++ b/tests/handlers/test_typing.py @@ -182,7 +182,9 @@ class TypingNotificationsTestCase(unittest.TestCase): [ {"type": "m.typing", "room_id": self.room_id, - "typing": [self.u_apple.to_string()]}, + "content": { + "user_ids": [self.u_apple.to_string()], + }}, ] ) @@ -242,7 +244,9 @@ class TypingNotificationsTestCase(unittest.TestCase): [ {"type": "m.typing", "room_id": self.room_id, - "typing": [self.u_onion.to_string()]}, + "content": { + "user_ids": [self.u_onion.to_string()], + }}, ] ) @@ -295,7 +299,9 @@ class TypingNotificationsTestCase(unittest.TestCase): [ {"type": "m.typing", "room_id": self.room_id, - "typing": []}, + "content": { + "user_ids": [], + }}, ] ) @@ -323,7 +329,9 @@ class TypingNotificationsTestCase(unittest.TestCase): [ {"type": "m.typing", "room_id": self.room_id, - "typing": [self.u_apple.to_string()]}, + "content": { + "user_ids": [self.u_apple.to_string()], + }}, ] ) @@ -339,6 +347,8 @@ class TypingNotificationsTestCase(unittest.TestCase): [ {"type": "m.typing", "room_id": self.room_id, - "typing": []}, + "content": { + "user_ids": [], + }}, ] ) diff --git a/tests/rest/test_typing.py b/tests/rest/test_typing.py index 0b95d70718..c550294d59 100644 --- a/tests/rest/test_typing.py +++ b/tests/rest/test_typing.py @@ -100,7 +100,9 @@ class RoomTypingTestCase(RestTestCase): [ {"type": "m.typing", "room_id": self.room_id, - "typing": [self.user_id]}, + "content": { + "user_ids": [self.user_id], + }}, ] ) From 41ff21c9074ae2859eb49dfba1bda21f4ec3ab6e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 12 Dec 2014 14:10:32 +0000 Subject: [PATCH 109/158] Fix test. --- tests/rest/test_rooms.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/rest/test_rooms.py b/tests/rest/test_rooms.py index ff7c9f0530..e2172366ef 100644 --- a/tests/rest/test_rooms.py +++ b/tests/rest/test_rooms.py @@ -503,7 +503,7 @@ class RoomsMemberListTestCase(RestTestCase): @defer.inlineCallbacks def test_get_member_list_mixed_memberships(self): - room_creator = "@some_other_guy:blue" + room_creator = "@some_other_guy:red" room_id = yield self.create_room_as(room_creator) room_path = "/rooms/%s/members" % room_id yield self.invite(room=room_id, src=room_creator, From ebf2ec3ce6ffe04d75d1d221fcf7d5bac268a1ab Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 12 Dec 2014 14:32:39 +0000 Subject: [PATCH 110/158] Fix membership handler test --- tests/handlers/test_room.py | 189 +++++++++++++++++++----------------- 1 file changed, 102 insertions(+), 87 deletions(-) diff --git a/tests/handlers/test_room.py b/tests/handlers/test_room.py index 1e6e7c3602..15b2c265b6 100644 --- a/tests/handlers/test_room.py +++ b/tests/handlers/test_room.py @@ -47,7 +47,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase): "get_room_member", "get_room", "store_room", - "snapshot_room", + "get_latest_events_in_room", ]), resource_for_federation=NonCallableMock(), http_client=NonCallableMock(spec_set=[]), @@ -63,7 +63,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase): "check_host_in_room", ]), state_handler=NonCallableMock(spec_set=[ - "annotate_event_with_state", + "annotate_context_with_state", "get_current_state", ]), config=self.mock_config, @@ -91,9 +91,6 @@ class RoomMemberHandlerTestCase(unittest.TestCase): self.handlers.profile_handler = ProfileHandler(self.hs) self.room_member_handler = self.handlers.room_member_handler - self.snapshot = Mock() - self.datastore.snapshot_room.return_value = self.snapshot - self.ratelimiter = hs.get_ratelimiter() self.ratelimiter.send_message.return_value = (True, 0) @@ -104,50 +101,68 @@ class RoomMemberHandlerTestCase(unittest.TestCase): target_user_id = "@red:blue" content = {"membership": Membership.INVITE} - event = self.hs.get_event_factory().create_event( - etype=RoomMemberEvent.TYPE, - user_id=user_id, - state_key=target_user_id, - room_id=room_id, - membership=Membership.INVITE, - content=content, + builder = self.hs.get_event_builder_factory().new({ + "type": RoomMemberEvent.TYPE, + "sender": user_id, + "state_key": target_user_id, + "room_id": room_id, + "content": content, + }) + + self.datastore.get_latest_events_in_room.return_value = ( + defer.succeed([]) ) - self.auth.check_host_in_room.return_value = defer.succeed(True) + def annotate(_, ctx): + ctx.current_state = { + (RoomMemberEvent.TYPE, "@alice:green"): self._create_member( + user_id="@alice:green", + room_id=room_id, + ), + (RoomMemberEvent.TYPE, "@bob:red"): self._create_member( + user_id="@bob:red", + room_id=room_id, + ), + } - store_id = "store_id_fooo" - self.datastore.persist_event.return_value = defer.succeed(store_id) + return defer.succeed(True) - self.datastore.get_room_member.return_value = defer.succeed(None) + self.state_handler.annotate_context_with_state.side_effect = annotate - event.old_state_events = { - (RoomMemberEvent.TYPE, "@alice:green"): self._create_member( - user_id="@alice:green", - room_id=room_id, - ), - (RoomMemberEvent.TYPE, "@bob:red"): self._create_member( - user_id="@bob:red", - room_id=room_id, - ), - } + def add_auth(_, ctx): + ctx.auth_events = ctx.current_state[ + (RoomMemberEvent.TYPE, "@bob:red") + ] - event.state_events = event.old_state_events - event.state_events[(RoomMemberEvent.TYPE, target_user_id)] = event + return defer.succeed(True) + self.auth.add_auth_events.side_effect = add_auth - # Actual invocation - yield self.room_member_handler.change_membership(event) + def send_invite(domain, event): + return defer.succeed(event) - self.federation.handle_new_event.assert_called_once_with( - event, self.snapshot, + self.federation.send_invite.side_effect = send_invite + + room_handler = self.room_member_handler + event, context = yield room_handler._create_new_client_event( + builder ) - self.assertEquals( - set(["red", "green"]), - set(event.destinations) + yield room_handler.change_membership(event, context) + + self.state_handler.annotate_context_with_state.assert_called_once_with( + builder, context + ) + + self.auth.add_auth_events.assert_called_once_with( + builder, context + ) + + self.federation.send_invite.assert_called_once_with( + "blue", event, ) self.datastore.persist_event.assert_called_once_with( - event + event, context=context, ) self.notifier.on_new_room_event.assert_called_once_with( event, extra_users=[self.hs.parse_userid(target_user_id)] @@ -162,57 +177,56 @@ class RoomMemberHandlerTestCase(unittest.TestCase): user_id = "@bob:red" user = self.hs.parse_userid(user_id) - event = self._create_member( - user_id=user_id, - room_id=room_id, - ) - - self.auth.check_host_in_room.return_value = defer.succeed(True) - - store_id = "store_id_fooo" - self.datastore.persist_event.return_value = defer.succeed(store_id) - self.datastore.get_room.return_value = defer.succeed(1) # Not None. - - prev_state = NonCallableMock() - prev_state.membership = Membership.INVITE - prev_state.sender = "@foo:red" - self.datastore.get_room_member.return_value = defer.succeed(prev_state) - join_signal_observer = Mock() self.distributor.observe("user_joined_room", join_signal_observer) - event.state_events = { - (RoomMemberEvent.TYPE, "@alice:green"): self._create_member( - user_id="@alice:green", - room_id=room_id, - ), - (RoomMemberEvent.TYPE, user_id): event, - } + builder = self.hs.get_event_builder_factory().new({ + "type": RoomMemberEvent.TYPE, + "sender": user_id, + "state_key": user_id, + "room_id": room_id, + "content": {"membership": Membership.JOIN}, + }) - event.old_state_events = { - (RoomMemberEvent.TYPE, "@alice:green"): self._create_member( - user_id="@alice:green", - room_id=room_id, - ), - } - - event.state_events = event.old_state_events - event.state_events[(RoomMemberEvent.TYPE, user_id)] = event - - # Actual invocation - yield self.room_member_handler.change_membership(event) - - self.federation.handle_new_event.assert_called_once_with( - event, self.snapshot + self.datastore.get_latest_events_in_room.return_value = ( + defer.succeed([]) ) - self.assertEquals( - set(["red", "green"]), - set(event.destinations) + def annotate(_, ctx): + ctx.current_state = { + (RoomMemberEvent.TYPE, "@bob:red"): self._create_member( + user_id="@bob:red", + room_id=room_id, + membership=Membership.INVITE + ), + } + + return defer.succeed(True) + + self.state_handler.annotate_context_with_state.side_effect = annotate + + def add_auth(_, ctx): + ctx.auth_events = ctx.current_state[ + (RoomMemberEvent.TYPE, "@bob:red") + ] + + return defer.succeed(True) + self.auth.add_auth_events.side_effect = add_auth + + room_handler = self.room_member_handler + event, context = yield room_handler._create_new_client_event( + builder + ) + + # Actual invocation + yield room_handler.change_membership(event, context) + + self.federation.handle_new_event.assert_called_once_with( + event, None, destinations=set() ) self.datastore.persist_event.assert_called_once_with( - event + event, context=context ) self.notifier.on_new_room_event.assert_called_once_with( event, extra_users=[user] @@ -222,15 +236,16 @@ class RoomMemberHandlerTestCase(unittest.TestCase): user=user, room_id=room_id ) - def _create_member(self, user_id, room_id): - return self.hs.get_event_factory().create_event( - etype=RoomMemberEvent.TYPE, - user_id=user_id, - state_key=user_id, - room_id=room_id, - membership=Membership.JOIN, - content={"membership": Membership.JOIN}, - ) + def _create_member(self, user_id, room_id, membership=Membership.JOIN): + builder = self.hs.get_event_builder_factory().new({ + "type": RoomMemberEvent.TYPE, + "sender": user_id, + "state_key": user_id, + "room_id": room_id, + "content": {"membership": membership}, + }) + + return builder.build() class RoomCreationTest(unittest.TestCase): From 75085bb4d16d73df019a671ba483f28b9071d0ff Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 12 Dec 2014 14:34:34 +0000 Subject: [PATCH 111/158] Pyflakes --- synapse/storage/_base.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index b6f8817b62..1967290ad2 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -23,7 +23,6 @@ from synapse.util.logcontext import PreserveLoggingContext, LoggingContext from twisted.internet import defer import collections -import copy import json import sys import time From c39beb55593cd08abe67eddd902627d6171cb245 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 12 Dec 2014 14:53:37 +0000 Subject: [PATCH 112/158] Store json as UTF-8 and not bytes --- synapse/storage/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 3bc3622169..079ddac605 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -168,7 +168,7 @@ class DataStore(RoomMemberStore, RoomStore, values={ "event_id": event.event_id, "room_id": event.room_id, - "json": encode_canonical_json(event_dict), + "json": encode_canonical_json(event_dict).decode("UTF-8"), }, or_replace=True, ) From 7b43a503f31e47b0eae9fe2b12fbea5e7fd280f5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 12 Dec 2014 15:05:37 +0000 Subject: [PATCH 113/158] Consistently url decode and decode as utf 8 the URL parts --- synapse/http/server.py | 12 ++++++++++- synapse/rest/directory.py | 12 +++-------- synapse/rest/presence.py | 4 ---- synapse/rest/profile.py | 5 ----- synapse/rest/room.py | 45 ++++++++++++++++++--------------------- 5 files changed, 35 insertions(+), 43 deletions(-) diff --git a/synapse/http/server.py b/synapse/http/server.py index 02277c4998..2d5d71c8aa 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -29,6 +29,7 @@ from twisted.web.util import redirectTo import collections import logging +import urllib logger = logging.getLogger(__name__) @@ -122,9 +123,18 @@ class JsonResource(HttpServer, resource.Resource): # We found a match! Trigger callback and then return the # returned response. We pass both the request and any # matched groups from the regex to the callback. + + logger.debug("url things: %r", m.groups()) + + args = [ + urllib.unquote(u).decode("UTF-8") for u in m.groups() + ] + + logger.debug("url things args: %r", args) + code, response = yield path_entry.callback( request, - *m.groups() + *args ) self._send_response(request, code, response) diff --git a/synapse/rest/directory.py b/synapse/rest/directory.py index 35300c6a6f..7b2b18dbcd 100644 --- a/synapse/rest/directory.py +++ b/synapse/rest/directory.py @@ -36,9 +36,7 @@ class ClientDirectoryServer(RestServlet): @defer.inlineCallbacks def on_GET(self, request, room_alias): - room_alias = self.hs.parse_roomalias( - urllib.unquote(room_alias).decode("utf-8") - ) + room_alias = self.hs.parse_roomalias(room_alias) dir_handler = self.handlers.directory_handler res = yield dir_handler.get_association(room_alias) @@ -56,9 +54,7 @@ class ClientDirectoryServer(RestServlet): logger.debug("Got content: %s", content) - room_alias = self.hs.parse_roomalias( - urllib.unquote(room_alias).decode("utf-8") - ) + room_alias = self.hs.parse_roomalias(room_alias) logger.debug("Got room name: %s", room_alias.to_string()) @@ -97,9 +93,7 @@ class ClientDirectoryServer(RestServlet): dir_handler = self.handlers.directory_handler - room_alias = self.hs.parse_roomalias( - urllib.unquote(room_alias).decode("utf-8") - ) + room_alias = self.hs.parse_roomalias(room_alias) yield dir_handler.delete_association( user.to_string(), room_alias diff --git a/synapse/rest/presence.py b/synapse/rest/presence.py index 062c895595..4bcd7ef635 100644 --- a/synapse/rest/presence.py +++ b/synapse/rest/presence.py @@ -33,7 +33,6 @@ class PresenceStatusRestServlet(RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id): auth_user = yield self.auth.get_user_by_req(request) - user_id = urllib.unquote(user_id) user = self.hs.parse_userid(user_id) state = yield self.handlers.presence_handler.get_state( @@ -44,7 +43,6 @@ class PresenceStatusRestServlet(RestServlet): @defer.inlineCallbacks def on_PUT(self, request, user_id): auth_user = yield self.auth.get_user_by_req(request) - user_id = urllib.unquote(user_id) user = self.hs.parse_userid(user_id) state = {} @@ -80,7 +78,6 @@ class PresenceListRestServlet(RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id): auth_user = yield self.auth.get_user_by_req(request) - user_id = urllib.unquote(user_id) user = self.hs.parse_userid(user_id) if not self.hs.is_mine(user): @@ -101,7 +98,6 @@ class PresenceListRestServlet(RestServlet): @defer.inlineCallbacks def on_POST(self, request, user_id): auth_user = yield self.auth.get_user_by_req(request) - user_id = urllib.unquote(user_id) user = self.hs.parse_userid(user_id) if not self.hs.is_mine(user): diff --git a/synapse/rest/profile.py b/synapse/rest/profile.py index 72e02d8dd8..fa1be2c289 100644 --- a/synapse/rest/profile.py +++ b/synapse/rest/profile.py @@ -27,7 +27,6 @@ class ProfileDisplaynameRestServlet(RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id): - user_id = urllib.unquote(user_id) user = self.hs.parse_userid(user_id) displayname = yield self.handlers.profile_handler.get_displayname( @@ -39,7 +38,6 @@ class ProfileDisplaynameRestServlet(RestServlet): @defer.inlineCallbacks def on_PUT(self, request, user_id): auth_user = yield self.auth.get_user_by_req(request) - user_id = urllib.unquote(user_id) user = self.hs.parse_userid(user_id) try: @@ -62,7 +60,6 @@ class ProfileAvatarURLRestServlet(RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id): - user_id = urllib.unquote(user_id) user = self.hs.parse_userid(user_id) avatar_url = yield self.handlers.profile_handler.get_avatar_url( @@ -74,7 +71,6 @@ class ProfileAvatarURLRestServlet(RestServlet): @defer.inlineCallbacks def on_PUT(self, request, user_id): auth_user = yield self.auth.get_user_by_req(request) - user_id = urllib.unquote(user_id) user = self.hs.parse_userid(user_id) try: @@ -97,7 +93,6 @@ class ProfileRestServlet(RestServlet): @defer.inlineCallbacks def on_GET(self, request, user_id): - user_id = urllib.unquote(user_id) user = self.hs.parse_userid(user_id) displayname = yield self.handlers.profile_handler.get_displayname( diff --git a/synapse/rest/room.py b/synapse/rest/room.py index c526e9bc72..7fb5aca0a7 100644 --- a/synapse/rest/room.py +++ b/synapse/rest/room.py @@ -129,9 +129,9 @@ class RoomStateEventRestServlet(RestServlet): msg_handler = self.handlers.message_handler data = yield msg_handler.get_room_data( user_id=user.to_string(), - room_id=urllib.unquote(room_id), - event_type=urllib.unquote(event_type), - state_key=urllib.unquote(state_key), + room_id=room_id, + event_type=event_type, + state_key=state_key, ) if not data: @@ -143,19 +143,18 @@ class RoomStateEventRestServlet(RestServlet): @defer.inlineCallbacks def on_PUT(self, request, room_id, event_type, state_key): user = yield self.auth.get_user_by_req(request) - event_type = urllib.unquote(event_type) content = _parse_json(request) event_dict = { "type": event_type, "content": content, - "room_id": urllib.unquote(room_id), + "room_id": room_id, "sender": user.to_string(), } if state_key is not None: - event_dict["state_key"] = urllib.unquote(state_key) + event_dict["state_key"] = state_key msg_handler = self.handlers.message_handler yield msg_handler.handle_event(event_dict) @@ -179,9 +178,9 @@ class RoomSendEventRestServlet(RestServlet): msg_handler = self.handlers.message_handler event = yield msg_handler.handle_event( { - "type": urllib.unquote(event_type), + "type": event_type, "content": content, - "room_id": urllib.unquote(room_id), + "room_id": room_id, "sender": user.to_string(), } ) @@ -225,14 +224,10 @@ class JoinRoomAliasServlet(RestServlet): identifier = None is_room_alias = False try: - identifier = self.hs.parse_roomalias( - urllib.unquote(room_identifier) - ) + identifier = self.hs.parse_roomalias(room_identifier) is_room_alias = True except SynapseError: - identifier = self.hs.parse_roomid( - urllib.unquote(room_identifier) - ) + identifier = self.hs.parse_roomid(room_identifier) # TODO: Support for specifying the home server to join with? @@ -246,7 +241,7 @@ class JoinRoomAliasServlet(RestServlet): { "type": RoomMemberEvent.TYPE, "content": {"membership": Membership.JOIN}, - "room_id": urllib.unquote(identifier.to_string()), + "room_id": identifier.to_string(), "sender": user.to_string(), "state_key": user.to_string(), } @@ -290,7 +285,7 @@ class RoomMemberListRestServlet(RestServlet): user = yield self.auth.get_user_by_req(request) handler = self.handlers.room_member_handler members = yield handler.get_room_members_as_pagination_chunk( - room_id=urllib.unquote(room_id), + room_id=room_id, user_id=user.to_string()) for event in members["chunk"]: @@ -322,7 +317,7 @@ class RoomMessageListRestServlet(RestServlet): with_feedback = "feedback" in request.args handler = self.handlers.message_handler msgs = yield handler.get_messages( - room_id=urllib.unquote(room_id), + room_id=room_id, user_id=user.to_string(), pagin_config=pagination_config, feedback=with_feedback) @@ -340,7 +335,7 @@ class RoomStateRestServlet(RestServlet): handler = self.handlers.message_handler # Get all the current state for this room events = yield handler.get_state_events( - room_id=urllib.unquote(room_id), + room_id=room_id, user_id=user.to_string(), ) defer.returnValue((200, events)) @@ -355,7 +350,7 @@ class RoomInitialSyncRestServlet(RestServlet): user = yield self.auth.get_user_by_req(request) pagination_config = PaginationConfig.from_request(request) content = yield self.handlers.message_handler.room_initial_sync( - room_id=urllib.unquote(room_id), + room_id=room_id, user_id=user.to_string(), pagin_config=pagination_config, ) @@ -367,8 +362,10 @@ class RoomTriggerBackfill(RestServlet): @defer.inlineCallbacks def on_GET(self, request, room_id): - remote_server = urllib.unquote(request.args["remote"][0]) - room_id = urllib.unquote(room_id) + remote_server = urllib.unquote( + request.args["remote"][0] + ).decode("UTF-8") + limit = int(request.args["limit"][0]) handler = self.handlers.federation_handler @@ -408,7 +405,7 @@ class RoomMembershipRestServlet(RestServlet): { "type": RoomMemberEvent.TYPE, "content": {"membership": unicode(membership_action)}, - "room_id": urllib.unquote(room_id), + "room_id": room_id, "sender": user.to_string(), "state_key": state_key, } @@ -446,9 +443,9 @@ class RoomRedactEventRestServlet(RestServlet): { "type": RoomRedactionEvent.TYPE, "content": content, - "room_id": urllib.unquote(room_id), + "room_id": room_id, "sender": user.to_string(), - "redacts": urllib.unquote(event_id), + "redacts": event_id, } ) From 1fc2a0e33ef6e88a79dbf4325468d8eb77db0f65 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 12 Dec 2014 15:08:29 +0000 Subject: [PATCH 114/158] Fix tests and remove debug logging --- synapse/http/server.py | 4 ---- tests/utils.py | 9 +++++++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/synapse/http/server.py b/synapse/http/server.py index 2d5d71c8aa..f33859cf76 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -124,14 +124,10 @@ class JsonResource(HttpServer, resource.Resource): # returned response. We pass both the request and any # matched groups from the regex to the callback. - logger.debug("url things: %r", m.groups()) - args = [ urllib.unquote(u).decode("UTF-8") for u in m.groups() ] - logger.debug("url things args: %r", args) - code, response = yield path_entry.callback( request, *args diff --git a/tests/utils.py b/tests/utils.py index f9a34748cd..70a221550c 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -29,7 +29,7 @@ from twisted.enterprise.adbapi import ConnectionPool from collections import namedtuple from mock import patch, Mock -import json +import urllib import urlparse from inspect import getcallargs @@ -103,9 +103,14 @@ class MockHttpResource(HttpServer): matcher = pattern.match(path) if matcher: try: + args = [ + urllib.unquote(u).decode("UTF-8") + for u in matcher.groups() + ] + (code, response) = yield func( mock_request, - *matcher.groups() + *args ) defer.returnValue((code, response)) except CodeMessageException as e: From fc409096ac03f8a422d0efd7cca1b5fc3c182306 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 12 Dec 2014 16:31:50 +0000 Subject: [PATCH 115/158] Make auth module use EventTypes constants --- synapse/api/auth.py | 50 +++++++++++++++++++++------------------------ 1 file changed, 23 insertions(+), 27 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index c76f5180f6..70245aba04 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -17,12 +17,8 @@ from twisted.internet import defer -from synapse.api.constants import Membership, JoinRules +from synapse.api.constants import EventTypes, Membership, JoinRules from synapse.api.errors import AuthError, StoreError, Codes, SynapseError -from synapse.api.events.room import ( - RoomMemberEvent, RoomPowerLevelsEvent, RoomRedactionEvent, - RoomJoinRulesEvent, RoomCreateEvent, RoomAliasesEvent, -) from synapse.util.logutils import log_function from synapse.util.async import run_on_reactor @@ -53,17 +49,17 @@ class Auth(object): logger.warn("Trusting event: %s", event.event_id) return True - if event.type == RoomCreateEvent.TYPE: + if event.type == EventTypes.Create: # FIXME return True # FIXME: Temp hack - if event.type == RoomAliasesEvent.TYPE: + if event.type == EventTypes.Aliases: return True logger.debug("Auth events: %s", auth_events) - if event.type == RoomMemberEvent.TYPE: + if event.type == EventTypes.Member: allowed = self.is_membership_change_allowed( event, auth_events ) @@ -76,10 +72,10 @@ class Auth(object): self.check_event_sender_in_room(event, auth_events) self._can_send_event(event, auth_events) - if event.type == RoomPowerLevelsEvent.TYPE: + if event.type == EventTypes.PowerLevels: self._check_power_levels(event, auth_events) - if event.type == RoomRedactionEvent.TYPE: + if event.type == EventTypes.Redaction: self._check_redaction(event, auth_events) logger.debug("Allowing! %s", event) @@ -95,7 +91,7 @@ class Auth(object): def check_joined_room(self, room_id, user_id): member = yield self.state.get_current_state( room_id=room_id, - event_type=RoomMemberEvent.TYPE, + event_type=EventTypes.Member, state_key=user_id ) self._check_joined_room(member, user_id, room_id) @@ -106,7 +102,7 @@ class Auth(object): curr_state = yield self.state.get_current_state(room_id) for event in curr_state: - if event.type == RoomMemberEvent.TYPE: + if event.type == EventTypes.Member: try: if self.hs.parse_userid(event.state_key).domain != host: continue @@ -120,7 +116,7 @@ class Auth(object): defer.returnValue(False) def check_event_sender_in_room(self, event, auth_events): - key = (RoomMemberEvent.TYPE, event.user_id, ) + key = (EventTypes.Member, event.user_id, ) member_event = auth_events.get(key) return self._check_joined_room( @@ -142,7 +138,7 @@ class Auth(object): # Check if this is the room creator joining: if len(event.prev_events) == 1 and Membership.JOIN == membership: # Get room creation event: - key = (RoomCreateEvent.TYPE, "", ) + key = (EventTypes.Create, "", ) create = auth_events.get(key) if create and event.prev_events[0][0] == create.event_id: if create.content["creator"] == event.state_key: @@ -151,19 +147,19 @@ class Auth(object): target_user_id = event.state_key # get info about the caller - key = (RoomMemberEvent.TYPE, event.user_id, ) + key = (EventTypes.Member, event.user_id, ) caller = auth_events.get(key) caller_in_room = caller and caller.membership == Membership.JOIN caller_invited = caller and caller.membership == Membership.INVITE # get info about the target - key = (RoomMemberEvent.TYPE, target_user_id, ) + key = (EventTypes.Member, target_user_id, ) target = auth_events.get(key) target_in_room = target and target.membership == Membership.JOIN - key = (RoomJoinRulesEvent.TYPE, "", ) + key = (EventTypes.JoinRules, "", ) join_rule_event = auth_events.get(key) if join_rule_event: join_rule = join_rule_event.content.get( @@ -258,7 +254,7 @@ class Auth(object): return True def _get_power_level_from_event_state(self, event, user_id, auth_events): - key = (RoomPowerLevelsEvent.TYPE, "", ) + key = (EventTypes.PowerLevels, "", ) power_level_event = auth_events.get(key) level = None if power_level_event: @@ -266,7 +262,7 @@ class Auth(object): if not level: level = power_level_event.content.get("users_default", 0) else: - key = (RoomCreateEvent.TYPE, "", ) + key = (EventTypes.Create, "", ) create_event = auth_events.get(key) if (create_event is not None and create_event.content["creator"] == user_id): @@ -275,7 +271,7 @@ class Auth(object): return level def _get_ops_level_from_event_state(self, event, auth_events): - key = (RoomPowerLevelsEvent.TYPE, "", ) + key = (EventTypes.PowerLevels, "", ) power_level_event = auth_events.get(key) if power_level_event: @@ -356,25 +352,25 @@ class Auth(object): def add_auth_events(self, builder, context): yield run_on_reactor() - if builder.type == RoomCreateEvent.TYPE: + if builder.type == EventTypes.Create: builder.auth_events = [] return auth_ids = [] - key = (RoomPowerLevelsEvent.TYPE, "", ) + key = (EventTypes.PowerLevels, "", ) power_level_event = context.current_state.get(key) if power_level_event: auth_ids.append(power_level_event.event_id) - key = (RoomJoinRulesEvent.TYPE, "", ) + key = (EventTypes.JoinRules, "", ) join_rule_event = context.current_state.get(key) - key = (RoomMemberEvent.TYPE, builder.user_id, ) + key = (EventTypes.Member, builder.user_id, ) member_event = context.current_state.get(key) - key = (RoomCreateEvent.TYPE, "", ) + key = (EventTypes.Create, "", ) create_event = context.current_state.get(key) if create_event: auth_ids.append(create_event.event_id) @@ -385,7 +381,7 @@ class Auth(object): else: is_public = False - if builder.type == RoomMemberEvent.TYPE: + if builder.type == EventTypes.Member: e_type = builder.content["membership"] if e_type in [Membership.JOIN, Membership.INVITE]: if join_rule_event: @@ -415,7 +411,7 @@ class Auth(object): @log_function def _can_send_event(self, event, auth_events): - key = (RoomPowerLevelsEvent.TYPE, "", ) + key = (EventTypes.PowerLevels, "", ) send_level_event = auth_events.get(key) send_level = None if send_level_event: From 23c7cb6220fecad558f63c07cc98c6fe8c016756 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 12 Dec 2014 16:31:59 +0000 Subject: [PATCH 116/158] Remove unused imports --- synapse/rest/directory.py | 1 - synapse/rest/presence.py | 1 - synapse/rest/profile.py | 1 - 3 files changed, 3 deletions(-) diff --git a/synapse/rest/directory.py b/synapse/rest/directory.py index 7b2b18dbcd..868fa7abed 100644 --- a/synapse/rest/directory.py +++ b/synapse/rest/directory.py @@ -21,7 +21,6 @@ from base import RestServlet, client_path_pattern import json import logging -import urllib logger = logging.getLogger(__name__) diff --git a/synapse/rest/presence.py b/synapse/rest/presence.py index 4bcd7ef635..9b42131628 100644 --- a/synapse/rest/presence.py +++ b/synapse/rest/presence.py @@ -22,7 +22,6 @@ from base import RestServlet, client_path_pattern import json import logging -import urllib logger = logging.getLogger(__name__) diff --git a/synapse/rest/profile.py b/synapse/rest/profile.py index fa1be2c289..351863ab5d 100644 --- a/synapse/rest/profile.py +++ b/synapse/rest/profile.py @@ -19,7 +19,6 @@ from twisted.internet import defer from base import RestServlet, client_path_pattern import json -import urllib class ProfileDisplaynameRestServlet(RestServlet): From 58fa6d3fc64c789a10f407532c0e7c419b0c1650 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 15 Dec 2014 13:53:58 +0000 Subject: [PATCH 117/158] return an mxc uri rather than a content_token. --- synapse/media/v1/upload_resource.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/media/v1/upload_resource.py b/synapse/media/v1/upload_resource.py index b2449ff03d..5645b0df46 100644 --- a/synapse/media/v1/upload_resource.py +++ b/synapse/media/v1/upload_resource.py @@ -95,8 +95,10 @@ class UploadResource(BaseMediaResource): yield self._generate_local_thumbnails(media_id, media_info) + content_uri = "mxc://%s/%s" % (self.server_name, media_id) + respond_with_json( - request, 200, {"content_token": media_id}, send_cors=True + request, 200, {"content_uri": content_uri}, send_cors=True ) except CodeMessageException as e: logger.exception(e) From c8dd3314d673fce90a53520475cdb19d5358dd34 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 15 Dec 2014 13:55:22 +0000 Subject: [PATCH 118/158] Fix bug where we ignored event_edge_hashes table --- scripts/check_event_hash.py | 3 +++ synapse/events/utils.py | 1 + synapse/storage/__init__.py | 1 - synapse/storage/_base.py | 1 - synapse/storage/event_federation.py | 11 ++++++----- 5 files changed, 10 insertions(+), 7 deletions(-) diff --git a/scripts/check_event_hash.py b/scripts/check_event_hash.py index 7c32f8102a..679afbd268 100644 --- a/scripts/check_event_hash.py +++ b/scripts/check_event_hash.py @@ -18,6 +18,9 @@ class dictobj(dict): def get_full_dict(self): return dict(self) + def get_pdu_json(self): + return dict(self) + def main(): parser = argparse.ArgumentParser() diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 4ab770dd5f..94f3f15f52 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -42,6 +42,7 @@ def prune_event(event): "auth_events", "origin", "origin_server_ts", + "membership", ] new_content = {} diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 079ddac605..d0ea304b3d 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -91,7 +91,6 @@ class DataStore(RoomMemberStore, RoomStore, def __init__(self, hs): super(DataStore, self).__init__(hs) - self.event_factory = hs.get_event_factory() self.hs = hs self.min_token_deferred = self._get_min_token() diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 1967290ad2..31d5163c19 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -83,7 +83,6 @@ class SQLBaseStore(object): def __init__(self, hs): self.hs = hs self._db_pool = hs.get_db_pool() - self.event_factory = hs.get_event_factory() self._clock = hs.get_clock() @defer.inlineCallbacks diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 6c559f8f63..ced066f407 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -177,14 +177,15 @@ class EventFederationStore(SQLBaseStore): retcols=["prev_event_id", "is_state"], ) + hashes = self._get_prev_event_hashes_txn(txn, event_id) + results = [] for d in res: - hashes = self._get_event_reference_hashes_txn( - txn, - d["prev_event_id"] - ) + edge_hash = self._get_event_reference_hashes_txn(txn, d["prev_event_id"]) + edge_hash.update(hashes.get(d["prev_event_id"], {})) prev_hashes = { - k: encode_base64(v) for k, v in hashes.items() + k: encode_base64(v) + for k, v in edge_hash.items() if k == "sha256" } results.append((d["prev_event_id"], prev_hashes, d["is_state"])) From 616f88027cbd6f75af987215ad03074caf9fc05a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 15 Dec 2014 13:55:41 +0000 Subject: [PATCH 119/158] Add beginnings of upgrade script --- scripts/upgrade_db_to_v0.5.5.py | 223 ++++++++++++++++++++++++++++++++ 1 file changed, 223 insertions(+) create mode 100644 scripts/upgrade_db_to_v0.5.5.py diff --git a/scripts/upgrade_db_to_v0.5.5.py b/scripts/upgrade_db_to_v0.5.5.py new file mode 100644 index 0000000000..5898341d6e --- /dev/null +++ b/scripts/upgrade_db_to_v0.5.5.py @@ -0,0 +1,223 @@ +from synapse.storage._base import SQLBaseStore +from synapse.storage.signatures import SignatureStore +from synapse.storage.event_federation import EventFederationStore + +from syutil.base64util import encode_base64, decode_base64 + +from synapse.events import FrozenEvent +from synapse.events.builder import EventBuilder +from synapse.events.utils import prune_event + +from synapse.crypto.event_signing import check_event_content_hash + +from syutil.crypto.jsonsign import verify_signed_json, SignatureVerifyException +from syutil.crypto.signing_key import ( + decode_verify_key_bytes, write_signing_keys +) + +import dns.resolver +import hashlib +import json +import sqlite3 +import sys +import urllib2 + + +class Store(object): + _get_event_signatures_txn = SignatureStore.__dict__["_get_event_signatures_txn"] + _get_event_content_hashes_txn = SignatureStore.__dict__["_get_event_content_hashes_txn"] + _get_event_reference_hashes_txn = SignatureStore.__dict__["_get_event_reference_hashes_txn"] + _get_prev_event_hashes_txn = SignatureStore.__dict__["_get_prev_event_hashes_txn"] + _get_prev_events_and_state = EventFederationStore.__dict__["_get_prev_events_and_state"] + _get_auth_events = EventFederationStore.__dict__["_get_auth_events"] + cursor_to_dict = SQLBaseStore.__dict__["cursor_to_dict"] + _simple_select_onecol_txn = SQLBaseStore.__dict__["_simple_select_onecol_txn"] + _simple_select_list_txn = SQLBaseStore.__dict__["_simple_select_list_txn"] + + def _generate_event_json(self, txn, rows): + sql = "SELECT * FROM events WHERE event_id = ? ORDER BY rowid asc" + + events = [] + for row in rows: + d = dict(row) + + d.pop("stream_ordering", None) + d.pop("topological_ordering", None) + d.pop("processed", None) + + if "origin_server_ts" not in d: + d["origin_server_ts"] = d.pop("ts", 0) + else: + d.pop("ts", 0) + + d.pop("prev_state", None) + d.update(json.loads(d.pop("unrecognized_keys"))) + + d["sender"] = d.pop("user_id") + + d["content"] = json.loads(d["content"]) + + if "age_ts" not in d: + # For compatibility + d["age_ts"] = d.get("origin_server_ts", 0) + + d.setdefault("unsigned", {})["age_ts"] = d.pop("age_ts") + + d.pop("outlier", None) + + # d.pop("membership", None) + + d.pop("state_hash", None) + + d.pop("replaces_state", None) + + events.append(EventBuilder(d)) + + for i, ev in enumerate(events): + signatures = self._get_event_signatures_txn( + txn, ev.event_id, + ) + + ev.signatures = { + n: { + k: encode_base64(v) for k, v in s.items() + } + for n, s in signatures.items() + } + + hashes = self._get_event_content_hashes_txn( + txn, ev.event_id, + ) + + ev.hashes = { + k: encode_base64(v) for k, v in hashes.items() + } + + prevs = self._get_prev_events_and_state(txn, ev.event_id) + + ev.prev_events = [ + (e_id, h) + for e_id, h, is_state in prevs + if is_state == 0 + ] + + # ev.auth_events = self._get_auth_events(txn, ev.event_id) + + hashes = dict(ev.auth_events) + + for e_id, hash in ev.prev_events: + if e_id in hashes and not hash: + hash.update(hashes[e_id]) + # + # if hasattr(ev, "state_key"): + # ev.prev_state = [ + # (e_id, h) + # for e_id, h, is_state in prevs + # if is_state == 1 + # ] + + return [e.build() for e in events] + + +store = Store() + + +def get_key(server_name): + print "Getting keys for: %s" % (server_name,) + targets = [] + if ":" in server_name: + target, port = server_name.split(":") + targets.append((target, int(port))) + return + try: + answers = dns.resolver.query("_matrix._tcp." + server_name, "SRV") + for srv in answers: + targets.append((srv.target, srv.port)) + except dns.resolver.NXDOMAIN: + targets.append((server_name, 8448)) + except: + print "Failed to lookup keys for %s" % (server_name,) + return {} + + for target, port in targets: + url = "https://%s:%i/_matrix/key/v1" % (target, port) + try: + keys = json.load(urllib2.urlopen(url, timeout=2)) + verify_keys = {} + for key_id, key_base64 in keys["verify_keys"].items(): + verify_key = decode_verify_key_bytes(key_id, decode_base64(key_base64)) + verify_signed_json(keys, server_name, verify_key) + verify_keys[key_id] = verify_key + print "Got keys for: %s" % (server_name,) + return verify_keys + except urllib2.URLError: + pass + + print "Failed to get keys for %s" % (server_name,) + return {} + + +def get_events(cursor): + # cursor.execute( + # "SELECT * FROM events WHERE event_id = ? ORDER BY rowid DESC", + # ("$14182049031533SMfTT:matrix.org",) + # ) + + # cursor.execute( + # "SELECT * FROM events ORDER BY rowid DESC LIMIT 10000" + # ) + + cursor.execute( + "SELECT * FROM events ORDER BY rowid DESC" + ) + + rows = store.cursor_to_dict(cursor) + + events = store._generate_event_json(cursor, rows) + + print "Got events from DB." + + algorithms = { + "sha256": hashlib.sha256, + } + + server_keys = {} + + for event in events: + for alg_name in event.hashes: + if check_event_content_hash(event, algorithms[alg_name]): + # print "PASS content hash %s" % (alg_name,) + pass + else: + pass + print "FAIL content hash %s %s" % (alg_name, event.event_id, ) + # print "%s %d" % (event.event_id, event.origin_server_ts) + # print json.dumps(event.get_pdu_json(), indent=4, sort_keys=True) + + for host, sigs in event.signatures.items(): + pruned = prune_event(event) + + for key_id in sigs: + if host not in server_keys: + server_keys[host] = get_key(host) + if key_id in server_keys[host]: + try: + verify_signed_json( + pruned.get_pdu_json(), + host, + server_keys[host][key_id] + ) + except SignatureVerifyException as e: + # print e + print "FAIL signature check %s %s" % (key_id, event.event_id) + # print json.dumps(pruned.get_pdu_json(), indent=4, sort_keys=True) + +def main(): + conn = sqlite3.connect(sys.argv[1]) + cursor = conn.cursor() + get_events(cursor) + conn.commit() + + +if __name__ == "__main__": + main() \ No newline at end of file From 3610641a62ab781afb6e8eded0b57e2ce2f6b6b5 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 15 Dec 2014 13:56:43 +0000 Subject: [PATCH 120/158] Update docs in media_repository --- synapse/media/v1/media_repository.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/media/v1/media_repository.py b/synapse/media/v1/media_repository.py index 7e446fd827..cbc49aa325 100644 --- a/synapse/media/v1/media_repository.py +++ b/synapse/media/v1/media_repository.py @@ -39,9 +39,9 @@ class MediaRepositoryResource(Resource): <= HTTP/1.1 200 OK Content-Type: application/json - { "token": } + { "content-uri": "mxc:///" } - => GET /_matrix/media/v1/download// HTTP/1.1 + => GET /_matrix/media/v1/download// HTTP/1.1 <= HTTP/1.1 200 OK Content-Type: From 20beed9dd4b4b129641423caebaef090b30795b8 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 15 Dec 2014 14:37:12 +0000 Subject: [PATCH 121/158] Still send typing notifications to myself if I'm the only one in the room (it's a lonely life...) --- synapse/handlers/typing.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/synapse/handlers/typing.py b/synapse/handlers/typing.py index 253fec514d..3f2f250dd2 100644 --- a/synapse/handlers/typing.py +++ b/synapse/handlers/typing.py @@ -146,8 +146,7 @@ class TypingNotificationHandler(BaseHandler): rm_handler = self.homeserver.get_handlers().room_member_handler yield rm_handler.fetch_room_distributions_into( - room_id, localusers=localusers, remotedomains=remotedomains, - ignore_user=user + room_id, localusers=localusers, remotedomains=remotedomains ) if localusers: From 65cdf4e724f3c528aa38d578abd94334d55ce593 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 15 Dec 2014 15:03:27 +0000 Subject: [PATCH 122/158] Get current member state from current_state snapshot. Fix leave test. --- synapse/handlers/room.py | 6 +++--- tests/handlers/test_room.py | 13 ++++++++++++- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index a1d542854d..d317f2b30e 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -343,9 +343,9 @@ class RoomMemberHandler(BaseHandler): """ target_user_id = event.state_key - # TODO(markjh): get prev state from snapshot. - prev_state = yield self.store.get_room_member( - target_user_id, event.room_id + prev_state = context.current_state.get( + (RoomMemberEvent.TYPE, target_user_id), + None ) room_id = event.room_id diff --git a/tests/handlers/test_room.py b/tests/handlers/test_room.py index 21853fdc50..9ae9335e29 100644 --- a/tests/handlers/test_room.py +++ b/tests/handlers/test_room.py @@ -270,7 +270,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase): (RoomMemberEvent.TYPE, "@bob:red"): self._create_member( user_id="@bob:red", room_id=room_id, - membership=Membership.INVITE + membership=Membership.JOIN ), } @@ -297,6 +297,17 @@ class RoomMemberHandlerTestCase(unittest.TestCase): # Actual invocation yield room_handler.change_membership(event, context) + self.federation.handle_new_event.assert_called_once_with( + event, None, destinations=set(['red']) + ) + + self.datastore.persist_event.assert_called_once_with( + event, context=context + ) + self.notifier.on_new_room_event.assert_called_once_with( + event, extra_users=[user] + ) + leave_signal_observer.assert_called_with( user=user, room_id=room_id ) From b75adaedcaa0f153557557217844cd06f92635ec Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 15 Dec 2014 16:14:34 +0000 Subject: [PATCH 123/158] Finish up upgrade script --- scripts/upgrade_db_to_v0.5.5.py | 124 ++++++++++++++++++++++++-------- synapse/events/__init__.py | 2 +- 2 files changed, 95 insertions(+), 31 deletions(-) diff --git a/scripts/upgrade_db_to_v0.5.5.py b/scripts/upgrade_db_to_v0.5.5.py index 5898341d6e..be9d07b2df 100644 --- a/scripts/upgrade_db_to_v0.5.5.py +++ b/scripts/upgrade_db_to_v0.5.5.py @@ -4,25 +4,42 @@ from synapse.storage.event_federation import EventFederationStore from syutil.base64util import encode_base64, decode_base64 -from synapse.events import FrozenEvent +from synapse.crypto.event_signing import compute_event_signature + from synapse.events.builder import EventBuilder from synapse.events.utils import prune_event from synapse.crypto.event_signing import check_event_content_hash -from syutil.crypto.jsonsign import verify_signed_json, SignatureVerifyException -from syutil.crypto.signing_key import ( - decode_verify_key_bytes, write_signing_keys +from syutil.crypto.jsonsign import ( + verify_signed_json, SignatureVerifyException, ) +from syutil.crypto.signing_key import decode_verify_key_bytes +from syutil.jsonutil import encode_canonical_json + +import argparse import dns.resolver import hashlib import json import sqlite3 -import sys +import syutil import urllib2 +delta_sql = """ +CREATE TABLE IF NOT EXISTS event_json( + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + json BLOB NOT NULL, + CONSTRAINT ev_j_uniq UNIQUE (event_id) +); + +CREATE INDEX IF NOT EXISTS event_json_id ON event_json(event_id); +CREATE INDEX IF NOT EXISTS event_json_room_id ON event_json(room_id); +""" + + class Store(object): _get_event_signatures_txn = SignatureStore.__dict__["_get_event_signatures_txn"] _get_event_content_hashes_txn = SignatureStore.__dict__["_get_event_content_hashes_txn"] @@ -33,10 +50,9 @@ class Store(object): cursor_to_dict = SQLBaseStore.__dict__["cursor_to_dict"] _simple_select_onecol_txn = SQLBaseStore.__dict__["_simple_select_onecol_txn"] _simple_select_list_txn = SQLBaseStore.__dict__["_simple_select_list_txn"] + _simple_insert_txn = SQLBaseStore.__dict__["_simple_insert_txn"] def _generate_event_json(self, txn, rows): - sql = "SELECT * FROM events WHERE event_id = ? ORDER BY rowid asc" - events = [] for row in rows: d = dict(row) @@ -145,7 +161,9 @@ def get_key(server_name): keys = json.load(urllib2.urlopen(url, timeout=2)) verify_keys = {} for key_id, key_base64 in keys["verify_keys"].items(): - verify_key = decode_verify_key_bytes(key_id, decode_base64(key_base64)) + verify_key = decode_verify_key_bytes( + key_id, decode_base64(key_base64) + ) verify_signed_json(keys, server_name, verify_key) verify_keys[key_id] = verify_key print "Got keys for: %s" % (server_name,) @@ -157,18 +175,11 @@ def get_key(server_name): return {} -def get_events(cursor): - # cursor.execute( - # "SELECT * FROM events WHERE event_id = ? ORDER BY rowid DESC", - # ("$14182049031533SMfTT:matrix.org",) - # ) - - # cursor.execute( - # "SELECT * FROM events ORDER BY rowid DESC LIMIT 10000" - # ) +def reinsert_events(cursor, server_name, signing_key): + cursor.executescript(delta_sql) cursor.execute( - "SELECT * FROM events ORDER BY rowid DESC" + "SELECT * FROM events ORDER BY rowid ASC" ) rows = store.cursor_to_dict(cursor) @@ -181,19 +192,26 @@ def get_events(cursor): "sha256": hashlib.sha256, } - server_keys = {} + key_id = "%s:%s" % (signing_key.alg, signing_key.version) + verify_key = signing_key.verify_key + verify_key.alg = signing_key.alg + verify_key.version = signing_key.version + + server_keys = { + server_name: { + key_id: verify_key + } + } for event in events: for alg_name in event.hashes: if check_event_content_hash(event, algorithms[alg_name]): - # print "PASS content hash %s" % (alg_name,) pass else: pass print "FAIL content hash %s %s" % (alg_name, event.event_id, ) - # print "%s %d" % (event.event_id, event.origin_server_ts) - # print json.dumps(event.get_pdu_json(), indent=4, sort_keys=True) + have_own_correctly_signed = False for host, sigs in event.signatures.items(): pruned = prune_event(event) @@ -207,17 +225,63 @@ def get_events(cursor): host, server_keys[host][key_id] ) - except SignatureVerifyException as e: - # print e - print "FAIL signature check %s %s" % (key_id, event.event_id) - # print json.dumps(pruned.get_pdu_json(), indent=4, sort_keys=True) -def main(): - conn = sqlite3.connect(sys.argv[1]) + if host == server_name: + have_own_correctly_signed = True + except SignatureVerifyException: + print "FAIL signature check %s %s" % ( + key_id, event.event_id + ) + + # TODO: Re sign with our own server key + if not have_own_correctly_signed: + sigs = compute_event_signature(event, server_name, signing_key) + event.signatures.update(sigs) + + pruned = prune_event(event) + + for key_id in event.signatures[server_name]: + verify_signed_json( + pruned.get_pdu_json(), + server_name, + server_keys[server_name][key_id] + ) + + event_json = encode_canonical_json( + event.get_dict() + ).decode("UTF-8") + + store._simple_insert_txn( + cursor, + table="event_json", + values={ + "event_id": event.event_id, + "room_id": event.room_id, + "json": event_json, + }, + or_replace=True, + ) + + +def main(database, server_name, signing_key): + conn = sqlite3.connect(database) cursor = conn.cursor() - get_events(cursor) + reinsert_events(cursor, server_name, signing_key) conn.commit() if __name__ == "__main__": - main() \ No newline at end of file + parser = argparse.ArgumentParser() + + parser.add_argument("database") + parser.add_argument("server_name") + parser.add_argument( + "signing_key", type=argparse.FileType('r'), + ) + args = parser.parse_args() + + signing_key = syutil.crypto.signing_key.read_signing_keys( + args.signing_key + ) + + main(args.database, args.server_name, signing_key[0]) diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index 98d7f0e324..d9dfe5e3f3 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -175,4 +175,4 @@ class FrozenEvent(EventBase): def __repr__(self): return "" % ( self.event_id, self.type, self.get("state_key", None), - ) \ No newline at end of file + ) From 0c1deca5745d40859b368a9d6978845eb9811fcc Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 15 Dec 2014 16:14:53 +0000 Subject: [PATCH 124/158] Remember to hook up the typing event stream to the notifier as well --- synapse/notifier.py | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/synapse/notifier.py b/synapse/notifier.py index 5e14950449..383230caf1 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -146,7 +146,11 @@ class Notifier(object): Will wake up all listeners for the given users and rooms. """ yield run_on_reactor() + + # TODO(paul): This is horrible, having to manually list every event + # source here individually presence_source = self.event_sources.sources["presence"] + typing_source = self.event_sources.sources["typing"] listeners = set() @@ -158,19 +162,33 @@ class Notifier(object): @defer.inlineCallbacks def notify(listener): - events, end_key = yield presence_source.get_new_events_for_user( - listener.user, - listener.from_token.presence_key, - listener.limit, + presence_events, presence_end_key = ( + yield presence_source.get_new_events_for_user( + listener.user, + listener.from_token.presence_key, + listener.limit, + ) + ) + typing_events, typing_end_key = ( + yield typing_source.get_new_events_for_user( + listener.user, + listener.from_token.typing_key, + listener.limit, + ) ) - if events: + if presence_events or typing_events: end_token = listener.from_token.copy_and_replace( - "presence_key", end_key + "presence_key", presence_end_key + ).copy_and_replace( + "typing_key", typing_end_key ) listener.notify( - self, events, listener.from_token, end_token + self, + presence_events + typing_events, + listener.from_token, + end_token ) def eb(failure): From 670dcdfc14d855c45b076f852673906dc450f515 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 15 Dec 2014 16:16:58 +0000 Subject: [PATCH 125/158] Remove unused functions --- synapse/events/snapshot.py | 42 -------------------------------------- 1 file changed, 42 deletions(-) diff --git a/synapse/events/snapshot.py b/synapse/events/snapshot.py index e0cbacc19c..b9fb29be01 100644 --- a/synapse/events/snapshot.py +++ b/synapse/events/snapshot.py @@ -13,48 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from twisted.internet import defer - - -class EventSnapshot(object): - def __init__(self, prev_events, depth, current_state, - current_state_group): - self._prev_events = prev_events - self._depth = depth - self._current_state = current_state - self._current_state_group = current_state_group - - -class EventCache(object): - def __init__(self, store): - self._store = store - - self._cache = {} - - @defer.inlineCallbacks - def load_event(self, event_id): - event = self._cache.get(event_id, None) - - if not event: - event = yield self._store.get_event( - event_id, - allow_none=True - ) - - if event: - self._cache[event_id] = event - - defer.returnValue(event) - - def load_event_from_cache(self, event_id): - return self._cache.get(event_id, None) - - def add_to_cache(self, *events): - self._cache.update({ - event.event_id: event - for event in events - }) - class EventContext(object): From 67c9585656d2c241abefbdea79f0edbb543d9eb5 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Mon, 15 Dec 2014 16:57:53 +0000 Subject: [PATCH 126/158] Update media_repository.py _ not - --- synapse/media/v1/media_repository.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/media/v1/media_repository.py b/synapse/media/v1/media_repository.py index cbc49aa325..2070ec3c7f 100644 --- a/synapse/media/v1/media_repository.py +++ b/synapse/media/v1/media_repository.py @@ -39,7 +39,7 @@ class MediaRepositoryResource(Resource): <= HTTP/1.1 200 OK Content-Type: application/json - { "content-uri": "mxc:///" } + { "content_uri": "mxc:///" } => GET /_matrix/media/v1/download// HTTP/1.1 From cf6e5f1dbf3cecf693e45f293535e71ee31801ed Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 15 Dec 2014 17:01:12 +0000 Subject: [PATCH 127/158] Rename MessageHandler.handle_event. Add a few comments. --- synapse/handlers/_base.py | 5 +++++ synapse/handlers/directory.py | 2 +- synapse/handlers/message.py | 14 +++++++++++++- synapse/handlers/profile.py | 2 +- synapse/handlers/room.py | 8 ++++---- synapse/rest/room.py | 10 +++++----- tests/handlers/test_room.py | 5 +++-- 7 files changed, 32 insertions(+), 14 deletions(-) diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index 14f75ecbc2..af8eb5f0f5 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -122,10 +122,15 @@ class BaseHandler(object): if event.content["membership"] == Membership.INVITE: invitee = self.hs.parse_userid(event.state_key) if not self.hs.is_mine(invitee): + # TODO: Can we add signature from remote server in a nicer + # way? If we have been invited by a remote server, we need + # to get them to sign the event. returned_invite = yield federation_handler.send_invite( invitee.domain, event, ) + + # TODO: Make sure the signatures actually are correct. event.signatures.update( returned_invite.signatures ) diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index 76fb897f20..4b0869cd9f 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -149,7 +149,7 @@ class DirectoryHandler(BaseHandler): aliases = yield self.store.get_aliases_for_room(room_id) msg_handler = self.hs.get_handlers().message_handler - yield msg_handler.handle_event({ + yield msg_handler.create_and_send_event({ "type": RoomAliasesEvent.TYPE, "state_key": self.hs.hostname, "room_id": room_id, diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 4fa4ffea25..1eed38c6d1 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -138,7 +138,19 @@ class MessageHandler(BaseHandler): defer.returnValue(chunk) @defer.inlineCallbacks - def handle_event(self, event_dict): + def create_and_send_event(self, event_dict): + """ Given a dict from a client, create and handle a new event. + + Creates an FrozenEvent object, filling out auth_events, prev_events, + etc. + + Adds display names to Join membership events. + + Persists and notifies local clients and federation. + + Args: + event_dict (dict): An entire event + """ builder = self.event_builder_factory.new(event_dict) self.validator.validate_new(builder) diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index 18fd0914e2..33a2c167ec 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -209,7 +209,7 @@ class ProfileHandler(BaseHandler): ) msg_handler = self.hs.get_handlers().message_handler - yield msg_handler.handle_event({ + yield msg_handler.create_and_send_event({ "type": j.type, "room_id": j.room_id, "state_key": j.state_key, diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index d317f2b30e..f7cc869225 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -126,11 +126,11 @@ class RoomCreationHandler(BaseHandler): msg_handler = self.hs.get_handlers().message_handler for event in creation_events: - yield msg_handler.handle_event(event) + yield msg_handler.create_and_send_event(event) if "name" in config: name = config["name"] - yield msg_handler.handle_event({ + yield msg_handler.create_and_send_event({ "type": RoomNameEvent.TYPE, "room_id": room_id, "sender": user_id, @@ -139,7 +139,7 @@ class RoomCreationHandler(BaseHandler): if "topic" in config: topic = config["topic"] - yield msg_handler.handle_event({ + yield msg_handler.create_and_send_event({ "type": RoomTopicEvent.TYPE, "room_id": room_id, "sender": user_id, @@ -147,7 +147,7 @@ class RoomCreationHandler(BaseHandler): }) for invitee in invite_list: - yield msg_handler.handle_event({ + yield msg_handler.create_and_send_event({ "type": RoomMemberEvent.TYPE, "state_key": invitee, "room_id": room_id, diff --git a/synapse/rest/room.py b/synapse/rest/room.py index 25ee964555..1a527d27c1 100644 --- a/synapse/rest/room.py +++ b/synapse/rest/room.py @@ -157,7 +157,7 @@ class RoomStateEventRestServlet(RestServlet): event_dict["state_key"] = state_key msg_handler = self.handlers.message_handler - yield msg_handler.handle_event(event_dict) + yield msg_handler.create_and_send_event(event_dict) defer.returnValue((200, {})) @@ -176,7 +176,7 @@ class RoomSendEventRestServlet(RestServlet): content = _parse_json(request) msg_handler = self.handlers.message_handler - event = yield msg_handler.handle_event( + event = yield msg_handler.create_and_send_event( { "type": event_type, "content": content, @@ -237,7 +237,7 @@ class JoinRoomAliasServlet(RestServlet): defer.returnValue((200, ret_dict)) else: # room id msg_handler = self.handlers.message_handler - yield msg_handler.handle_event( + yield msg_handler.create_and_send_event( { "type": RoomMemberEvent.TYPE, "content": {"membership": Membership.JOIN}, @@ -401,7 +401,7 @@ class RoomMembershipRestServlet(RestServlet): membership_action = "leave" msg_handler = self.handlers.message_handler - yield msg_handler.handle_event( + yield msg_handler.create_and_send_event( { "type": RoomMemberEvent.TYPE, "content": {"membership": unicode(membership_action)}, @@ -439,7 +439,7 @@ class RoomRedactEventRestServlet(RestServlet): content = _parse_json(request) msg_handler = self.handlers.message_handler - event = yield msg_handler.handle_event( + event = yield msg_handler.create_and_send_event( { "type": RoomRedactionEvent.TYPE, "content": content, diff --git a/tests/handlers/test_room.py b/tests/handlers/test_room.py index 9ae9335e29..9c63f2a882 100644 --- a/tests/handlers/test_room.py +++ b/tests/handlers/test_room.py @@ -369,10 +369,11 @@ class RoomCreationTest(unittest.TestCase): config=config, ) - self.assertTrue(self.message_handler.handle_event.called) + self.assertTrue(self.message_handler.create_and_send_event.called) event_dicts = [ - e[0][0] for e in self.message_handler.handle_event.call_args_list + e[0][0] + for e in self.message_handler.create_and_send_event.call_args_list ] self.assertTrue(len(event_dicts) > 3) From 009e4b5637709ff1c18d2e38843b6e6f5934ade4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 15 Dec 2014 17:17:51 +0000 Subject: [PATCH 128/158] User.is_mine is no longer a thing. Use hs.is_mine instead. --- synapse/handlers/typing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/typing.py b/synapse/handlers/typing.py index 34bc955c15..77d66f66ff 100644 --- a/synapse/handlers/typing.py +++ b/synapse/handlers/typing.py @@ -118,7 +118,7 @@ class TypingNotificationHandler(BaseHandler): @defer.inlineCallbacks def user_left_room(self, user, room_id): - if user.is_mine: + if self.hs.is_mine(user): member = RoomMember(room_id=room_id, user=user) yield self._stopped_typing(member) From f280929a12314cfdb680881e2458bb047b7346ce Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 15 Dec 2014 17:31:36 +0000 Subject: [PATCH 129/158] Use frozenutils --- synapse/events/__init__.py | 36 ++++-------------------------------- synapse/storage/__init__.py | 3 +-- 2 files changed, 5 insertions(+), 34 deletions(-) diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index d9dfe5e3f3..f8fbb18e3c 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -15,39 +15,11 @@ from frozendict import frozendict +from synapse.util.frozenutils import freeze, unfreeze + import copy -def _freeze(o): - if isinstance(o, dict) or isinstance(o, frozendict): - return frozendict({k: _freeze(v) for k, v in o.items()}) - - if isinstance(o, basestring): - return o - - try: - return tuple([_freeze(i) for i in o]) - except TypeError: - pass - - return o - - -def _unfreeze(o): - if isinstance(o, frozendict) or isinstance(o, dict): - return dict({k: _unfreeze(v) for k, v in o.items()}) - - if isinstance(o, basestring): - return o - - try: - return [_unfreeze(i) for i in o] - except TypeError: - pass - - return o - - class _EventInternalMetadata(object): def __init__(self, internal_metadata_dict): self.__dict__ = copy.deepcopy(internal_metadata_dict) @@ -147,7 +119,7 @@ class FrozenEvent(EventBase): signatures = copy.deepcopy(event_dict.pop("signatures", {})) unsigned = copy.deepcopy(event_dict.pop("unsigned", {})) - frozen_dict = _freeze(event_dict) + frozen_dict = freeze(event_dict) super(FrozenEvent, self).__init__( frozen_dict, @@ -167,7 +139,7 @@ class FrozenEvent(EventBase): def get_dict(self): # We need to unfreeze what we return - return _unfreeze(super(FrozenEvent, self).get_dict()) + return unfreeze(super(FrozenEvent, self).get_dict()) def __str__(self): return self.__repr__() diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index d0ea304b3d..e75eaa92d5 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -21,7 +21,6 @@ from synapse.api.events.room import ( ) from synapse.util.logutils import log_function -from synapse.util.frozenutils import FrozenEncoder from .directory import DirectoryStore from .feedback import FeedbackStore @@ -177,7 +176,7 @@ class DataStore(RoomMemberStore, RoomStore, "event_id": event.event_id, "type": event.type, "room_id": event.room_id, - "content": json.dumps(event.content, cls=FrozenEncoder), + "content": json.dumps(event.get_dict()["content"]), "processed": True, "outlier": outlier, "depth": event.depth, From 65b2e494294269db6026a8095ee364fbe1377620 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 15 Dec 2014 17:35:37 +0000 Subject: [PATCH 130/158] Fix pyflakes --- synapse/events/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index f8fbb18e3c..1ec79a6788 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -13,8 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from frozendict import frozendict - from synapse.util.frozenutils import freeze, unfreeze import copy From 400327d12849678f440cf30962458184b4db6319 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 15 Dec 2014 17:37:40 +0000 Subject: [PATCH 131/158] Add a script for talking matrix federation adding X-Matrix Authorization headers. --- scripts/federation_client.py | 143 +++++++++++++++++++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100644 scripts/federation_client.py diff --git a/scripts/federation_client.py b/scripts/federation_client.py new file mode 100644 index 0000000000..3139c61761 --- /dev/null +++ b/scripts/federation_client.py @@ -0,0 +1,143 @@ +import nacl.signing +import json +import base64 +import requests +import sys +import srvlookup + + +def encode_base64(input_bytes): + """Encode bytes as a base64 string without any padding.""" + + input_len = len(input_bytes) + output_len = 4 * ((input_len + 2) // 3) + (input_len + 2) % 3 - 2 + output_bytes = base64.b64encode(input_bytes) + output_string = output_bytes[:output_len].decode("ascii") + return output_string + + +def decode_base64(input_string): + """Decode a base64 string to bytes inferring padding from the length of the + string.""" + + input_bytes = input_string.encode("ascii") + input_len = len(input_bytes) + padding = b"=" * (3 - ((input_len + 3) % 4)) + output_len = 3 * ((input_len + 2) // 4) + (input_len + 2) % 4 - 2 + output_bytes = base64.b64decode(input_bytes + padding) + return output_bytes[:output_len] + + +def encode_canonical_json(value): + return json.dumps( + value, + # Encode code-points outside of ASCII as UTF-8 rather than \u escapes + ensure_ascii=False, + # Remove unecessary white space. + separators=(',',':'), + # Sort the keys of dictionaries. + sort_keys=True, + # Encode the resulting unicode as UTF-8 bytes. + ).encode("UTF-8") + + +def sign_json(json_object, signing_key, signing_name): + signatures = json_object.pop("signatures", {}) + unsigned = json_object.pop("unsigned", None) + + signed = signing_key.sign(encode_canonical_json(json_object)) + signature_base64 = encode_base64(signed.signature) + + key_id = "%s:%s" % (signing_key.alg, signing_key.version) + signatures.setdefault(signing_name, {})[key_id] = signature_base64 + + json_object["signatures"] = signatures + if unsigned is not None: + json_object["unsigned"] = unsigned + + return json_object + + +NACL_ED25519 = "ed25519" + +def decode_signing_key_base64(algorithm, version, key_base64): + """Decode a base64 encoded signing key + Args: + algorithm (str): The algorithm the key is for (currently "ed25519"). + version (str): Identifies this key out of the keys for this entity. + key_base64 (str): Base64 encoded bytes of the key. + Returns: + A SigningKey object. + """ + if algorithm == NACL_ED25519: + key_bytes = decode_base64(key_base64) + key = nacl.signing.SigningKey(key_bytes) + key.version = version + key.alg = NACL_ED25519 + return key + else: + raise ValueError("Unsupported algorithm %s" % (algorithm,)) + + +def read_signing_keys(stream): + """Reads a list of keys from a stream + Args: + stream : A stream to iterate for keys. + Returns: + list of SigningKey objects. + """ + keys = [] + for line in stream: + algorithm, version, key_base64 = line.split() + keys.append(decode_signing_key_base64(algorithm, version, key_base64)) + return keys + + +def lookup(destination, path): + if ":" in destination: + return "https://%s%s" % (destination, path) + else: + srv = srvlookup.lookup("matrix", "tcp", destination)[0] + return "https://%s:%d%s" % (srv.host, srv.port, path) + +def get_json(origin_name, origin_key, destination, path): + request_json = { + "method": "GET", + "uri": path, + "origin": origin_name, + "destination": destination, + } + + signed_json = sign_json(request_json, origin_key, origin_name) + + authorization_headers = [] + + for key, sig in signed_json["signatures"][origin_name].items(): + authorization_headers.append(bytes( + "X-Matrix origin=%s,key=\"%s\",sig=\"%s\"" % ( + origin_name, key, sig, + ) + )) + + result = requests.get( + lookup(destination, path), + headers={"Authorization": authorization_headers[0]}, + verify=False, + ) + return result.json() + + +def main(): + origin_name, keyfile, destination, path = sys.argv[1:] + + with open(keyfile) as f: + key = read_signing_keys(f)[0] + + result = get_json( + origin_name, key, destination, "/_matrix/federation/v1/" + path + ) + + json.dump(result, sys.stdout) + +if __name__ == "__main__": + main() From 6a1da99fab064eb96aa1ae0566dbef4454b16567 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 16 Dec 2014 09:35:31 +0000 Subject: [PATCH 132/158] Add fixme to raising of AuthError in federation land --- synapse/handlers/federation.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index ddcb28c8e2..c00f5a7031 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -721,6 +721,7 @@ class FederationHandler(BaseHandler): "Rejecting %s as %s not in db or %s", event.event_id, e_id, known_ids, ) + # FIXME: How does raising AuthError work with federation? raise AuthError(403, "Auth events are stale") context.auth_events[(e.type, e.state_key)] = e From 3c77d13aa5375274e267a0ea898ce6267fb67cdc Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 16 Dec 2014 11:29:05 +0000 Subject: [PATCH 133/158] Kill off synapse.api.events.* --- synapse/api/constants.py | 1 + synapse/api/events/__init__.py | 148 -------------------- synapse/api/events/factory.py | 90 ------------- synapse/api/events/room.py | 170 ----------------------- synapse/api/events/utils.py | 85 ------------ synapse/api/events/validator.py | 87 ------------ synapse/federation/replication.py | 1 - synapse/handlers/directory.py | 4 +- synapse/handlers/federation.py | 22 +-- synapse/handlers/message.py | 1 - synapse/handlers/room.py | 28 ++-- synapse/rest/__init__.py | 2 +- synapse/rest/base.py | 2 - synapse/rest/room.py | 9 +- synapse/server.py | 10 -- synapse/state.py | 4 +- synapse/storage/__init__.py | 16 +-- tests/events/__init__.py | 15 --- tests/events/test_events.py | 217 ------------------------------ tests/handlers/test_federation.py | 7 +- tests/handlers/test_room.py | 25 ++-- tests/storage/test_redaction.py | 23 ++-- tests/storage/test_room.py | 8 +- tests/storage/test_roommember.py | 5 +- tests/storage/test_stream.py | 11 +- tests/utils.py | 8 +- 26 files changed, 74 insertions(+), 925 deletions(-) delete mode 100644 synapse/api/events/__init__.py delete mode 100644 synapse/api/events/factory.py delete mode 100644 synapse/api/events/room.py delete mode 100644 synapse/api/events/utils.py delete mode 100644 synapse/api/events/validator.py delete mode 100644 tests/events/__init__.py delete mode 100644 tests/events/test_events.py diff --git a/synapse/api/constants.py b/synapse/api/constants.py index b668da4a26..4fc8b79a40 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -68,6 +68,7 @@ class EventTypes(object): PowerLevels = "m.room.power_levels" Aliases = "m.room.aliases" Redaction = "m.room.redaction" + Feedback = "m.room.message.feedback" # These are used for validation Message = "m.room.message" diff --git a/synapse/api/events/__init__.py b/synapse/api/events/__init__.py deleted file mode 100644 index 22939d011a..0000000000 --- a/synapse/api/events/__init__.py +++ /dev/null @@ -1,148 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from synapse.util.jsonobject import JsonEncodedObject - - -def serialize_event(hs, e): - # FIXME(erikj): To handle the case of presence events and the like - if not isinstance(e, SynapseEvent): - return e - - # Should this strip out None's? - d = {k: v for k, v in e.get_dict().items()} - if "age_ts" in d: - d["age"] = int(hs.get_clock().time_msec()) - d["age_ts"] - del d["age_ts"] - - return d - - -class SynapseEvent(JsonEncodedObject): - - """Base class for Synapse events. These are JSON objects which must abide - by a certain well-defined structure. - """ - - # Attributes that are currently assumed by the federation side: - # Mandatory: - # - event_id - # - room_id - # - type - # - is_state - # - # Optional: - # - state_key (mandatory when is_state is True) - # - prev_events (these can be filled out by the federation layer itself.) - # - prev_state - - valid_keys = [ - "event_id", - "type", - "room_id", - "user_id", # sender/initiator - "content", # HTTP body, JSON - "state_key", - "age_ts", - "prev_content", - "replaces_state", - "redacted_because", - "origin_server_ts", - ] - - internal_keys = [ - "is_state", - "depth", - "destinations", - "origin", - "outlier", - "redacted", - "prev_events", - "hashes", - "signatures", - "prev_state", - "auth_events", - "state_hash", - ] - - required_keys = [ - "event_id", - "room_id", - "content", - ] - - outlier = False - - def __init__(self, raises=True, **kwargs): - super(SynapseEvent, self).__init__(**kwargs) - # if "content" in kwargs: - # self.check_json(self.content, raises=raises) - - def get_content_template(self): - """ Retrieve the JSON template for this event as a dict. - - The template must be a dict representing the JSON to match. Only - required keys should be present. The values of the keys in the template - are checked via type() to the values of the same keys in the actual - event JSON. - - NB: If loading content via json.loads, you MUST define strings as - unicode. - - For example: - Content: - { - "name": u"bob", - "age": 18, - "friends": [u"mike", u"jill"] - } - Template: - { - "name": u"string", - "age": 0, - "friends": [u"string"] - } - The values "string" and 0 could be anything, so long as the types - are the same as the content. - """ - raise NotImplementedError("get_content_template not implemented.") - - def get_pdu_json(self, time_now=None): - pdu_json = self.get_full_dict() - pdu_json.pop("destinations", None) - pdu_json.pop("outlier", None) - pdu_json.pop("replaces_state", None) - pdu_json.pop("redacted", None) - pdu_json.pop("prev_content", None) - state_hash = pdu_json.pop("state_hash", None) - if state_hash is not None: - pdu_json.setdefault("unsigned", {})["state_hash"] = state_hash - content = pdu_json.get("content", {}) - content.pop("prev", None) - if time_now is not None and "age_ts" in pdu_json: - age = time_now - pdu_json["age_ts"] - pdu_json.setdefault("unsigned", {})["age"] = int(age) - del pdu_json["age_ts"] - user_id = pdu_json.pop("user_id") - pdu_json["sender"] = user_id - return pdu_json - - -class SynapseStateEvent(SynapseEvent): - - def __init__(self, **kwargs): - if "state_key" not in kwargs: - kwargs["state_key"] = "" - super(SynapseStateEvent, self).__init__(**kwargs) diff --git a/synapse/api/events/factory.py b/synapse/api/events/factory.py deleted file mode 100644 index 1b84e2b445..0000000000 --- a/synapse/api/events/factory.py +++ /dev/null @@ -1,90 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from synapse.api.events.room import ( - RoomTopicEvent, MessageEvent, RoomMemberEvent, FeedbackEvent, - InviteJoinEvent, RoomConfigEvent, RoomNameEvent, GenericEvent, - RoomPowerLevelsEvent, RoomJoinRulesEvent, - RoomCreateEvent, - RoomRedactionEvent, -) - -from synapse.types import EventID - -from synapse.util.stringutils import random_string - - -class EventFactory(object): - - _event_classes = [ - RoomTopicEvent, - RoomNameEvent, - MessageEvent, - RoomMemberEvent, - FeedbackEvent, - InviteJoinEvent, - RoomConfigEvent, - RoomPowerLevelsEvent, - RoomJoinRulesEvent, - RoomCreateEvent, - RoomRedactionEvent, - ] - - def __init__(self, hs): - self._event_list = {} # dict of TYPE to event class - for event_class in EventFactory._event_classes: - self._event_list[event_class.TYPE] = event_class - - self.clock = hs.get_clock() - self.hs = hs - - self.event_id_count = 0 - - def create_event_id(self): - i = str(self.event_id_count) - self.event_id_count += 1 - - local_part = str(int(self.clock.time())) + i + random_string(5) - - e_id = EventID.create(local_part, self.hs.hostname) - - return e_id.to_string() - - def create_event(self, etype=None, **kwargs): - kwargs["type"] = etype - if "event_id" not in kwargs: - kwargs["event_id"] = self.create_event_id() - kwargs["origin"] = self.hs.hostname - else: - ev_id = self.hs.parse_eventid(kwargs["event_id"]) - kwargs["origin"] = ev_id.domain - - if "origin_server_ts" not in kwargs: - kwargs["origin_server_ts"] = int(self.clock.time_msec()) - - # The "age" key is a delta timestamp that should be converted into an - # absolute timestamp the minute we see it. - if "age" in kwargs: - kwargs["age_ts"] = int(self.clock.time_msec()) - int(kwargs["age"]) - del kwargs["age"] - elif "age_ts" not in kwargs: - kwargs["age_ts"] = int(self.clock.time_msec()) - - if etype in self._event_list: - handler = self._event_list[etype] - else: - handler = GenericEvent - - return handler(**kwargs) diff --git a/synapse/api/events/room.py b/synapse/api/events/room.py deleted file mode 100644 index 8c4ac45d02..0000000000 --- a/synapse/api/events/room.py +++ /dev/null @@ -1,170 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from synapse.api.constants import Feedback, Membership -from synapse.api.errors import SynapseError -from . import SynapseEvent, SynapseStateEvent - - -class GenericEvent(SynapseEvent): - def get_content_template(self): - return {} - - -class RoomTopicEvent(SynapseEvent): - TYPE = "m.room.topic" - - internal_keys = SynapseEvent.internal_keys + [ - "topic", - ] - - def __init__(self, **kwargs): - kwargs["state_key"] = "" - if "topic" in kwargs["content"]: - kwargs["topic"] = kwargs["content"]["topic"] - super(RoomTopicEvent, self).__init__(**kwargs) - - def get_content_template(self): - return {"topic": u"string"} - - -class RoomNameEvent(SynapseEvent): - TYPE = "m.room.name" - - internal_keys = SynapseEvent.internal_keys + [ - "name", - ] - - def __init__(self, **kwargs): - kwargs["state_key"] = "" - if "name" in kwargs["content"]: - kwargs["name"] = kwargs["content"]["name"] - super(RoomNameEvent, self).__init__(**kwargs) - - def get_content_template(self): - return {"name": u"string"} - - -class RoomMemberEvent(SynapseEvent): - TYPE = "m.room.member" - - valid_keys = SynapseEvent.valid_keys + [ - # target is the state_key - "membership", # action - ] - - def __init__(self, **kwargs): - if "membership" not in kwargs: - kwargs["membership"] = kwargs.get("content", {}).get("membership") - if not kwargs["membership"] in Membership.LIST: - raise SynapseError(400, "Bad membership value.") - super(RoomMemberEvent, self).__init__(**kwargs) - - def get_content_template(self): - return {"membership": u"string"} - - -class MessageEvent(SynapseEvent): - TYPE = "m.room.message" - - valid_keys = SynapseEvent.valid_keys + [ - "msg_id", # unique per room + user combo - ] - - def __init__(self, **kwargs): - super(MessageEvent, self).__init__(**kwargs) - - def get_content_template(self): - return {"msgtype": u"string"} - - -class FeedbackEvent(SynapseEvent): - TYPE = "m.room.message.feedback" - - valid_keys = SynapseEvent.valid_keys - - def __init__(self, **kwargs): - super(FeedbackEvent, self).__init__(**kwargs) - if not kwargs["content"]["type"] in Feedback.LIST: - raise SynapseError(400, "Bad feedback value.") - - def get_content_template(self): - return { - "type": u"string", - "target_event_id": u"string" - } - - -class InviteJoinEvent(SynapseEvent): - TYPE = "m.room.invite_join" - - valid_keys = SynapseEvent.valid_keys + [ - # target_user_id is the state_key - "target_host", - ] - - def __init__(self, **kwargs): - super(InviteJoinEvent, self).__init__(**kwargs) - - def get_content_template(self): - return {} - - -class RoomConfigEvent(SynapseEvent): - TYPE = "m.room.config" - - def __init__(self, **kwargs): - kwargs["state_key"] = "" - super(RoomConfigEvent, self).__init__(**kwargs) - - def get_content_template(self): - return {} - - -class RoomCreateEvent(SynapseStateEvent): - TYPE = "m.room.create" - - def get_content_template(self): - return {} - - -class RoomJoinRulesEvent(SynapseStateEvent): - TYPE = "m.room.join_rules" - - def get_content_template(self): - return {} - - -class RoomPowerLevelsEvent(SynapseStateEvent): - TYPE = "m.room.power_levels" - - def get_content_template(self): - return {} - - -class RoomAliasesEvent(SynapseStateEvent): - TYPE = "m.room.aliases" - - def get_content_template(self): - return {} - - -class RoomRedactionEvent(SynapseEvent): - TYPE = "m.room.redaction" - - valid_keys = SynapseEvent.valid_keys + ["redacts"] - - def get_content_template(self): - return {} diff --git a/synapse/api/events/utils.py b/synapse/api/events/utils.py deleted file mode 100644 index d6019d56eb..0000000000 --- a/synapse/api/events/utils.py +++ /dev/null @@ -1,85 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from .room import ( - RoomMemberEvent, RoomJoinRulesEvent, RoomPowerLevelsEvent, - RoomAliasesEvent, RoomCreateEvent, -) - - -def prune_event(event): - """ Returns a pruned version of the given event, which removes all keys we - don't know about or think could potentially be dodgy. - - This is used when we "redact" an event. We want to remove all fields that - the user has specified, but we do want to keep necessary information like - type, state_key etc. - """ - event_type = event.type - - allowed_keys = [ - "event_id", - "user_id", - "room_id", - "hashes", - "signatures", - "content", - "type", - "state_key", - "depth", - "prev_events", - "prev_state", - "auth_events", - "origin", - "origin_server_ts", - ] - - new_content = {} - - def add_fields(*fields): - for field in fields: - if field in event.content: - new_content[field] = event.content[field] - - if event_type == RoomMemberEvent.TYPE: - add_fields("membership") - elif event_type == RoomCreateEvent.TYPE: - add_fields("creator") - elif event_type == RoomJoinRulesEvent.TYPE: - add_fields("join_rule") - elif event_type == RoomPowerLevelsEvent.TYPE: - add_fields( - "users", - "users_default", - "events", - "events_default", - "events_default", - "state_default", - "ban", - "kick", - "redact", - ) - elif event_type == RoomAliasesEvent.TYPE: - add_fields("aliases") - - allowed_fields = { - k: v - for k, v in event.get_full_dict().items() - if k in allowed_keys - } - - allowed_fields["content"] = new_content - - return type(event)(**allowed_fields) diff --git a/synapse/api/events/validator.py b/synapse/api/events/validator.py deleted file mode 100644 index 067215f6ef..0000000000 --- a/synapse/api/events/validator.py +++ /dev/null @@ -1,87 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from synapse.api.errors import SynapseError, Codes - - -class EventValidator(object): - def __init__(self, hs): - pass - - def validate(self, event): - """Checks the given JSON content abides by the rules of the template. - - Args: - content : A JSON object to check. - raises: True to raise a SynapseError if the check fails. - Returns: - True if the content passes the template. Returns False if the check - fails and raises=False. - Raises: - SynapseError if the check fails and raises=True. - """ - # recursively call to inspect each layer - err_msg = self._check_json_template( - event.content, - event.get_content_template() - ) - if err_msg: - raise SynapseError(400, err_msg, Codes.BAD_JSON) - else: - return True - - def _check_json_template(self, content, template): - """Check content and template matches. - - If the template is a dict, each key in the dict will be validated with - the content, else it will just compare the types of content and - template. This basic type check is required because this function will - be recursively called and could be called with just strs or ints. - - Args: - content: The content to validate. - template: The validation template. - Returns: - str: An error message if the validation fails, else None. - """ - if type(content) != type(template): - return "Mismatched types: %s" % template - - if type(template) == dict: - for key in template: - if key not in content: - return "Missing %s key" % key - - if type(content[key]) != type(template[key]): - return "Key %s is of the wrong type (got %s, want %s)" % ( - key, type(content[key]), type(template[key])) - - if type(content[key]) == dict: - # we must go deeper - msg = self._check_json_template( - content[key], - template[key] - ) - if msg: - return msg - elif type(content[key]) == list: - # make sure each item type in content matches the template - for entry in content[key]: - msg = self._check_json_template( - entry, - template[key][0] - ) - if msg: - return msg diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index 6388bb98e2..9f8aadccca 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -74,7 +74,6 @@ class ReplicationLayer(object): self._clock = hs.get_clock() - self.event_factory = hs.get_event_factory() self.event_builder_factory = hs.get_event_builder_factory() def set_handler(self, handler): diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index 4b0869cd9f..404baea796 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -18,7 +18,7 @@ from twisted.internet import defer from ._base import BaseHandler from synapse.api.errors import SynapseError, Codes, CodeMessageException -from synapse.api.events.room import RoomAliasesEvent +from synapse.api.constants import EventTypes import logging @@ -150,7 +150,7 @@ class DirectoryHandler(BaseHandler): msg_handler = self.hs.get_handlers().message_handler yield msg_handler.create_and_send_event({ - "type": RoomAliasesEvent.TYPE, + "type": EventTypes.Aliases, "state_key": self.hs.hostname, "room_id": room_id, "sender": user_id, diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index c00f5a7031..16a104c0e2 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -22,8 +22,7 @@ from synapse.events.utils import prune_event from synapse.api.errors import ( AuthError, FederationError, SynapseError, StoreError, ) -from synapse.api.events.room import RoomMemberEvent, RoomCreateEvent -from synapse.api.constants import Membership +from synapse.api.constants import EventTypes, Membership from synapse.util.logutils import log_function from synapse.util.async import run_on_reactor from synapse.crypto.event_signing import ( @@ -225,7 +224,7 @@ class FederationHandler(BaseHandler): if not backfilled: extra_users = [] - if event.type == RoomMemberEvent.TYPE: + if event.type == EventTypes.Member: target_user_id = event.state_key target_user = self.hs.parse_userid(target_user_id) extra_users.append(target_user) @@ -234,7 +233,7 @@ class FederationHandler(BaseHandler): event, extra_users=extra_users ) - if event.type == RoomMemberEvent.TYPE: + if event.type == EventTypes.Member: if event.membership == Membership.JOIN: user = self.hs.parse_userid(event.state_key) yield self.distributor.fire( @@ -333,7 +332,8 @@ class FederationHandler(BaseHandler): event = pdu # We should assert some things. - assert(event.type == RoomMemberEvent.TYPE) + # FIXME: Do this in a nicer way + assert(event.type == EventTypes.Member) assert(event.user_id == joinee) assert(event.state_key == joinee) assert(event.room_id == room_id) @@ -450,7 +450,7 @@ class FederationHandler(BaseHandler): process it until the other server has signed it and sent it back. """ builder = self.event_builder_factory.new({ - "type": RoomMemberEvent.TYPE, + "type": EventTypes.Member, "content": {"membership": Membership.JOIN}, "room_id": room_id, "sender": user_id, @@ -492,7 +492,7 @@ class FederationHandler(BaseHandler): ) extra_users = [] - if event.type == RoomMemberEvent.TYPE: + if event.type == EventTypes.Member: target_user_id = event.state_key target_user = self.hs.parse_userid(target_user_id) extra_users.append(target_user) @@ -501,7 +501,7 @@ class FederationHandler(BaseHandler): event, extra_users=extra_users ) - if event.type == RoomMemberEvent.TYPE: + if event.type == EventTypes.Member: if event.content["membership"] == Membership.JOIN: user = self.hs.parse_userid(event.state_key) yield self.distributor.fire( @@ -514,7 +514,7 @@ class FederationHandler(BaseHandler): for k, s in context.current_state.items(): try: - if k[0] == RoomMemberEvent.TYPE: + if k[0] == EventTypes.Member: if s.content["membership"] == Membership.JOIN: destinations.add( self.hs.parse_userid(s.state_key).domain @@ -731,10 +731,10 @@ class FederationHandler(BaseHandler): event.event_id, event.signatures, ) - if event.type == RoomMemberEvent.TYPE and not event.auth_events: + if event.type == EventTypes.Member and not event.auth_events: if len(event.prev_events) == 1: c = yield self.store.get_event(event.prev_events[0][0]) - if c.type == RoomCreateEvent.TYPE: + if c.type == EventTypes.Create: context.auth_events[(c.type, c.state_key)] = c logger.debug( diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 1eed38c6d1..baf372fdad 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -35,7 +35,6 @@ class MessageHandler(BaseHandler): super(MessageHandler, self).__init__(hs) self.hs = hs self.clock = hs.get_clock() - self.event_factory = hs.get_event_factory() self.validator = EventValidator() @defer.inlineCallbacks diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index f7cc869225..8567d7409d 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -17,12 +17,8 @@ from twisted.internet import defer from synapse.types import UserID, RoomAlias, RoomID -from synapse.api.constants import Membership, JoinRules +from synapse.api.constants import EventTypes, Membership, JoinRules from synapse.api.errors import StoreError, SynapseError -from synapse.api.events.room import ( - RoomMemberEvent, RoomCreateEvent, RoomPowerLevelsEvent, - RoomTopicEvent, RoomNameEvent, RoomJoinRulesEvent, -) from synapse.util import stringutils from synapse.util.async import run_on_reactor from ._base import BaseHandler @@ -131,7 +127,7 @@ class RoomCreationHandler(BaseHandler): if "name" in config: name = config["name"] yield msg_handler.create_and_send_event({ - "type": RoomNameEvent.TYPE, + "type": EventTypes.Name, "room_id": room_id, "sender": user_id, "content": {"name": name}, @@ -140,7 +136,7 @@ class RoomCreationHandler(BaseHandler): if "topic" in config: topic = config["topic"] yield msg_handler.create_and_send_event({ - "type": RoomTopicEvent.TYPE, + "type": EventTypes.Topic, "room_id": room_id, "sender": user_id, "content": {"topic": topic}, @@ -148,7 +144,7 @@ class RoomCreationHandler(BaseHandler): for invitee in invite_list: yield msg_handler.create_and_send_event({ - "type": RoomMemberEvent.TYPE, + "type": EventTypes.Member, "state_key": invitee, "room_id": room_id, "user_id": user_id, @@ -186,12 +182,12 @@ class RoomCreationHandler(BaseHandler): return e creation_event = create( - etype=RoomCreateEvent.TYPE, + etype=EventTypes.Create, content={"creator": creator.to_string()}, ) join_event = create( - etype=RoomMemberEvent.TYPE, + etype=EventTypes.Member, state_key=creator_id, content={ "membership": Membership.JOIN, @@ -199,15 +195,15 @@ class RoomCreationHandler(BaseHandler): ) power_levels_event = create( - etype=RoomPowerLevelsEvent.TYPE, + etype=EventTypes.PowerLevels, content={ "users": { creator.to_string(): 100, }, "users_default": 0, "events": { - RoomNameEvent.TYPE: 100, - RoomPowerLevelsEvent.TYPE: 100, + EventTypes.Name: 100, + EventTypes.PowerLevels: 100, }, "events_default": 0, "state_default": 50, @@ -219,7 +215,7 @@ class RoomCreationHandler(BaseHandler): join_rule = JoinRules.PUBLIC if is_public else JoinRules.INVITE join_rules_event = create( - etype=RoomJoinRulesEvent.TYPE, + etype=EventTypes.JoinRules, content={"join_rule": join_rule}, ) @@ -344,7 +340,7 @@ class RoomMemberHandler(BaseHandler): target_user_id = event.state_key prev_state = context.current_state.get( - (RoomMemberEvent.TYPE, target_user_id), + (EventTypes.Member, target_user_id), None ) @@ -396,7 +392,7 @@ class RoomMemberHandler(BaseHandler): content.update({"membership": Membership.JOIN}) builder = self.event_builder_factory.new({ - "type": RoomMemberEvent.TYPE, + "type": EventTypes.Member, "state_key": joinee.to_string(), "room_id": room_id, "sender": joinee.to_string(), diff --git a/synapse/rest/__init__.py b/synapse/rest/__init__.py index e391e5678d..a59630ec96 100644 --- a/synapse/rest/__init__.py +++ b/synapse/rest/__init__.py @@ -28,7 +28,7 @@ class RestServletFactory(object): speaking, they serve as wrappers around events and the handlers that process them. - See synapse.api.events for information on synapse events. + See synapse.events for information on synapse events. """ def __init__(self, hs): diff --git a/synapse/rest/base.py b/synapse/rest/base.py index 72bb66ddda..06eda2587c 100644 --- a/synapse/rest/base.py +++ b/synapse/rest/base.py @@ -67,8 +67,6 @@ class RestServlet(object): self.auth = hs.get_auth() self.txns = HttpTransactionStore() - self.validator = hs.get_event_validator() - def register(self, http_server): """ Register this servlet with the given HTTP server. """ if hasattr(self, "PATTERN"): diff --git a/synapse/rest/room.py b/synapse/rest/room.py index 1a527d27c1..0e2d5fbaae 100644 --- a/synapse/rest/room.py +++ b/synapse/rest/room.py @@ -19,8 +19,7 @@ from twisted.internet import defer from base import RestServlet, client_path_pattern from synapse.api.errors import SynapseError, Codes from synapse.streams.config import PaginationConfig -from synapse.api.events.room import RoomMemberEvent, RoomRedactionEvent -from synapse.api.constants import Membership +from synapse.api.constants import EventTypes, Membership import json import logging @@ -239,7 +238,7 @@ class JoinRoomAliasServlet(RestServlet): msg_handler = self.handlers.message_handler yield msg_handler.create_and_send_event( { - "type": RoomMemberEvent.TYPE, + "type": EventTypes.Member, "content": {"membership": Membership.JOIN}, "room_id": identifier.to_string(), "sender": user.to_string(), @@ -403,7 +402,7 @@ class RoomMembershipRestServlet(RestServlet): msg_handler = self.handlers.message_handler yield msg_handler.create_and_send_event( { - "type": RoomMemberEvent.TYPE, + "type": EventTypes.Member, "content": {"membership": unicode(membership_action)}, "room_id": room_id, "sender": user.to_string(), @@ -441,7 +440,7 @@ class RoomRedactEventRestServlet(RestServlet): msg_handler = self.handlers.message_handler event = yield msg_handler.create_and_send_event( { - "type": RoomRedactionEvent.TYPE, + "type": EventTypes.Redaction, "content": content, "room_id": room_id, "sender": user.to_string(), diff --git a/synapse/server.py b/synapse/server.py index 1c0703c51f..e4021481e8 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -21,8 +21,6 @@ # Imports required for the default HomeServer() implementation from synapse.federation import initialize_http_replication from synapse.events.utils import serialize_event -from synapse.api.events.factory import EventFactory -from synapse.api.events.validator import EventValidator from synapse.notifier import Notifier from synapse.api.auth import Auth from synapse.handlers import Handlers @@ -66,7 +64,6 @@ class BaseHomeServer(object): 'persistence_service', 'replication_layer', 'datastore', - 'event_factory', 'handlers', 'auth', 'rest_servlet_factory', @@ -83,7 +80,6 @@ class BaseHomeServer(object): 'event_sources', 'ratelimiter', 'keyring', - 'event_validator', 'event_builder_factory', ] @@ -198,9 +194,6 @@ class HomeServer(BaseHomeServer): def build_datastore(self): return DataStore(self) - def build_event_factory(self): - return EventFactory(self) - def build_handlers(self): return Handlers(self) @@ -231,9 +224,6 @@ class HomeServer(BaseHomeServer): def build_keyring(self): return Keyring(self) - def build_event_validator(self): - return EventValidator(self) - def build_event_builder_factory(self): return EventBuilderFactory( clock=self.get_clock(), diff --git a/synapse/state.py b/synapse/state.py index f9ab5faf9e..d2763cdd9a 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -18,7 +18,7 @@ from twisted.internet import defer from synapse.util.logutils import log_function from synapse.util.async import run_on_reactor -from synapse.api.events.room import RoomPowerLevelsEvent +from synapse.api.constants import EventTypes from collections import namedtuple @@ -271,7 +271,7 @@ class StateHandler(object): def _get_power_level_from_event_state(self, event, user_id): if hasattr(event, "old_state_events") and event.old_state_events: - key = (RoomPowerLevelsEvent.TYPE, "", ) + key = (EventTypes.PowerLevels, "", ) power_level_event = event.old_state_events.get(key) level = None if power_level_event: diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index e75eaa92d5..5c079da5ba 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -15,12 +15,8 @@ from twisted.internet import defer -from synapse.api.events.room import ( - RoomMemberEvent, RoomTopicEvent, FeedbackEvent, RoomNameEvent, - RoomRedactionEvent, -) - from synapse.util.logutils import log_function +from synapse.api.constants import EventTypes from .directory import DirectoryStore from .feedback import FeedbackStore @@ -136,15 +132,15 @@ class DataStore(RoomMemberStore, RoomStore, def _persist_event_txn(self, txn, event, context, backfilled, stream_ordering=None, is_new_state=True, current_state=None): - if event.type == RoomMemberEvent.TYPE: + if event.type == EventTypes.Member: self._store_room_member_txn(txn, event) - elif event.type == FeedbackEvent.TYPE: + elif event.type == EventTypes.Feedback: self._store_feedback_txn(txn, event) - elif event.type == RoomNameEvent.TYPE: + elif event.type == EventTypes.Name: self._store_room_name_txn(txn, event) - elif event.type == RoomTopicEvent.TYPE: + elif event.type == EventTypes.Topic: self._store_room_topic_txn(txn, event) - elif event.type == RoomRedactionEvent.TYPE: + elif event.type == EventTypes.Redaction: self._store_redaction(txn, event) outlier = False diff --git a/tests/events/__init__.py b/tests/events/__init__.py deleted file mode 100644 index 9bff9ec169..0000000000 --- a/tests/events/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - diff --git a/tests/events/test_events.py b/tests/events/test_events.py deleted file mode 100644 index 91d1d44fee..0000000000 --- a/tests/events/test_events.py +++ /dev/null @@ -1,217 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from synapse.api.events import SynapseEvent -from synapse.api.events.validator import EventValidator -from synapse.api.errors import SynapseError - -from tests import unittest - - -class SynapseTemplateCheckTestCase(unittest.TestCase): - - def setUp(self): - self.validator = EventValidator(None) - - def tearDown(self): - pass - - def test_top_level_keys(self): - template = { - "person": {}, - "friends": ["string"] - } - - content = { - "person": {"name": "bob"}, - "friends": ["jill", "mike"] - } - - event = MockSynapseEvent(template) - event.content = content - self.assertTrue(self.validator.validate(event)) - - content = { - "person": {"name": "bob"}, - "friends": ["jill"], - "enemies": ["mike"] - } - event.content = content - self.assertTrue(self.validator.validate(event)) - - content = { - "person": {"name": "bob"}, - # missing friends - "enemies": ["mike", "jill"] - } - event.content = content - self.assertRaises( - SynapseError, - self.validator.validate, - event - ) - - def test_lists(self): - template = { - "person": {}, - "friends": [{"name":"string"}] - } - - content = { - "person": {"name": "bob"}, - "friends": ["jill", "mike"] # should be in objects - } - - event = MockSynapseEvent(template) - event.content = content - self.assertRaises( - SynapseError, - self.validator.validate, - event - ) - - content = { - "person": {"name": "bob"}, - "friends": [{"name": "jill"}, {"name": "mike"}] - } - event.content = content - self.assertTrue(self.validator.validate(event)) - - def test_nested_lists(self): - template = { - "results": { - "families": [ - { - "name": "string", - "members": [ - {} - ] - } - ] - } - } - - content = { - "results": { - "families": [ - { - "name": "Smith", - "members": [ - "Alice", "Bob" # wrong types - ] - } - ] - } - } - - event = MockSynapseEvent(template) - event.content = content - self.assertRaises( - SynapseError, - self.validator.validate, - event - ) - - content = { - "results": { - "families": [ - { - "name": "Smith", - "members": [ - {"name": "Alice"}, {"name": "Bob"} - ] - } - ] - } - } - event.content = content - self.assertTrue(self.validator.validate(event)) - - def test_nested_keys(self): - template = { - "person": { - "attributes": { - "hair": "string", - "eye": "string" - }, - "age": 0, - "fav_books": ["string"] - } - } - event = MockSynapseEvent(template) - - content = { - "person": { - "attributes": { - "hair": "brown", - "eye": "green", - "skin": "purple" - }, - "age": 33, - "fav_books": ["lotr", "hobbit"], - "fav_music": ["abba", "beatles"] - } - } - - event.content = content - self.assertTrue(self.validator.validate(event)) - - content = { - "person": { - "attributes": { - "hair": "brown" - # missing eye - }, - "age": 33, - "fav_books": ["lotr", "hobbit"], - "fav_music": ["abba", "beatles"] - } - } - - event.content = content - self.assertRaises( - SynapseError, - self.validator.validate, - event - ) - - content = { - "person": { - "attributes": { - "hair": "brown", - "eye": "green", - "skin": "purple" - }, - "age": 33, - "fav_books": "nothing", # should be a list - } - } - - event.content = content - self.assertRaises( - SynapseError, - self.validator.validate, - event - ) - - -class MockSynapseEvent(SynapseEvent): - - def __init__(self, template): - self.template = template - - def get_content_template(self): - return self.template - diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py index 91f7351087..ed351367cc 100644 --- a/tests/handlers/test_federation.py +++ b/tests/handlers/test_federation.py @@ -16,10 +16,7 @@ from twisted.internet import defer from tests import unittest -from synapse.api.events.room import ( - MessageEvent, -) - +from synapse.api.constants import EventTypes from synapse.events import FrozenEvent from synapse.handlers.federation import FederationHandler from synapse.server import HomeServer @@ -79,7 +76,7 @@ class FederationTestCase(unittest.TestCase): @defer.inlineCallbacks def test_msg(self): pdu = FrozenEvent({ - "type": MessageEvent.TYPE, + "type": EventTypes.Message, "room_id": "foo", "content": {"msgtype": u"fooo"}, "origin_server_ts": 0, diff --git a/tests/handlers/test_room.py b/tests/handlers/test_room.py index 9c63f2a882..83493cae20 100644 --- a/tests/handlers/test_room.py +++ b/tests/handlers/test_room.py @@ -17,9 +17,6 @@ from twisted.internet import defer from tests import unittest -from synapse.api.events.room import ( - RoomMemberEvent, -) from synapse.api.constants import EventTypes, Membership from synapse.handlers.room import RoomMemberHandler, RoomCreationHandler from synapse.handlers.profile import ProfileHandler @@ -102,7 +99,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase): content = {"membership": Membership.INVITE} builder = self.hs.get_event_builder_factory().new({ - "type": RoomMemberEvent.TYPE, + "type": EventTypes.Member, "sender": user_id, "state_key": target_user_id, "room_id": room_id, @@ -115,11 +112,11 @@ class RoomMemberHandlerTestCase(unittest.TestCase): def annotate(_, ctx): ctx.current_state = { - (RoomMemberEvent.TYPE, "@alice:green"): self._create_member( + (EventTypes.Member, "@alice:green"): self._create_member( user_id="@alice:green", room_id=room_id, ), - (RoomMemberEvent.TYPE, "@bob:red"): self._create_member( + (EventTypes.Member, "@bob:red"): self._create_member( user_id="@bob:red", room_id=room_id, ), @@ -131,7 +128,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase): def add_auth(_, ctx): ctx.auth_events = ctx.current_state[ - (RoomMemberEvent.TYPE, "@bob:red") + (EventTypes.Member, "@bob:red") ] return defer.succeed(True) @@ -181,7 +178,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase): self.distributor.observe("user_joined_room", join_signal_observer) builder = self.hs.get_event_builder_factory().new({ - "type": RoomMemberEvent.TYPE, + "type": EventTypes.Member, "sender": user_id, "state_key": user_id, "room_id": room_id, @@ -194,7 +191,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase): def annotate(_, ctx): ctx.current_state = { - (RoomMemberEvent.TYPE, "@bob:red"): self._create_member( + (EventTypes.Member, "@bob:red"): self._create_member( user_id="@bob:red", room_id=room_id, membership=Membership.INVITE @@ -207,7 +204,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase): def add_auth(_, ctx): ctx.auth_events = ctx.current_state[ - (RoomMemberEvent.TYPE, "@bob:red") + (EventTypes.Member, "@bob:red") ] return defer.succeed(True) @@ -238,7 +235,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase): def _create_member(self, user_id, room_id, membership=Membership.JOIN): builder = self.hs.get_event_builder_factory().new({ - "type": RoomMemberEvent.TYPE, + "type": EventTypes.Member, "sender": user_id, "state_key": user_id, "room_id": room_id, @@ -254,7 +251,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase): user = self.hs.parse_userid(user_id) builder = self.hs.get_event_builder_factory().new({ - "type": RoomMemberEvent.TYPE, + "type": EventTypes.Member, "sender": user_id, "state_key": user_id, "room_id": room_id, @@ -267,7 +264,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase): def annotate(_, ctx): ctx.current_state = { - (RoomMemberEvent.TYPE, "@bob:red"): self._create_member( + (EventTypes.Member, "@bob:red"): self._create_member( user_id="@bob:red", room_id=room_id, membership=Membership.JOIN @@ -280,7 +277,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase): def add_auth(_, ctx): ctx.auth_events = ctx.current_state[ - (RoomMemberEvent.TYPE, "@bob:red") + (EventTypes.Member, "@bob:red") ] return defer.succeed(True) diff --git a/tests/storage/test_redaction.py b/tests/storage/test_redaction.py index f670e154c3..9806fbc69b 100644 --- a/tests/storage/test_redaction.py +++ b/tests/storage/test_redaction.py @@ -18,10 +18,7 @@ from tests import unittest from twisted.internet import defer from synapse.server import HomeServer -from synapse.api.constants import Membership -from synapse.api.events.room import ( - RoomMemberEvent, MessageEvent, RoomRedactionEvent, -) +from synapse.api.constants import EventTypes, Membership from tests.utils import SQLiteMemoryDbPool, MockKey @@ -64,7 +61,7 @@ class RedactionTestCase(unittest.TestCase): content = {"membership": membership} content.update(extra_content) builder = self.event_builder_factory.new({ - "type": RoomMemberEvent.TYPE, + "type": EventTypes.Member, "sender": user.to_string(), "state_key": user.to_string(), "room_id": room.to_string(), @@ -84,7 +81,7 @@ class RedactionTestCase(unittest.TestCase): self.depth += 1 builder = self.event_builder_factory.new({ - "type": MessageEvent.TYPE, + "type": EventTypes.Message, "sender": user.to_string(), "state_key": user.to_string(), "room_id": room.to_string(), @@ -102,7 +99,7 @@ class RedactionTestCase(unittest.TestCase): @defer.inlineCallbacks def inject_redaction(self, room, event_id, user, reason): builder = self.event_builder_factory.new({ - "type": RoomRedactionEvent.TYPE, + "type": EventTypes.Redaction, "sender": user.to_string(), "state_key": user.to_string(), "room_id": room.to_string(), @@ -142,7 +139,7 @@ class RedactionTestCase(unittest.TestCase): self.assertObjectHasAttributes( { - "type": MessageEvent.TYPE, + "type": EventTypes.Message, "user_id": self.u_alice.to_string(), "content": {"body": "t", "msgtype": "message"}, }, @@ -176,7 +173,7 @@ class RedactionTestCase(unittest.TestCase): self.assertObjectHasAttributes( { - "type": MessageEvent.TYPE, + "type": EventTypes.Message, "user_id": self.u_alice.to_string(), "content": {}, }, @@ -185,7 +182,7 @@ class RedactionTestCase(unittest.TestCase): self.assertObjectHasAttributes( { - "type": RoomRedactionEvent.TYPE, + "type": EventTypes.Redaction, "user_id": self.u_alice.to_string(), "content": {"reason": reason}, }, @@ -221,7 +218,7 @@ class RedactionTestCase(unittest.TestCase): self.assertObjectHasAttributes( { - "type": RoomMemberEvent.TYPE, + "type": EventTypes.Member, "user_id": self.u_bob.to_string(), "content": {"membership": Membership.JOIN, "blue": "red"}, }, @@ -253,7 +250,7 @@ class RedactionTestCase(unittest.TestCase): self.assertObjectHasAttributes( { - "type": RoomMemberEvent.TYPE, + "type": EventTypes.Member, "user_id": self.u_bob.to_string(), "content": {"membership": Membership.JOIN}, }, @@ -262,7 +259,7 @@ class RedactionTestCase(unittest.TestCase): self.assertObjectHasAttributes( { - "type": RoomRedactionEvent.TYPE, + "type": EventTypes.Redaction, "user_id": self.u_alice.to_string(), "content": {"reason": reason}, }, diff --git a/tests/storage/test_room.py b/tests/storage/test_room.py index 4ff02c306b..11761fe29a 100644 --- a/tests/storage/test_room.py +++ b/tests/storage/test_room.py @@ -18,9 +18,7 @@ from tests import unittest from twisted.internet import defer from synapse.server import HomeServer -from synapse.api.events.room import ( - RoomNameEvent, RoomTopicEvent -) +from synapse.api.constants import EventTypes from tests.utils import SQLiteMemoryDbPool @@ -131,7 +129,7 @@ class RoomEventsStoreTestCase(unittest.TestCase): name = u"A-Room-Name" yield self.inject_room_event( - etype=RoomNameEvent.TYPE, + etype=EventTypes.Name, name=name, content={"name": name}, depth=1, @@ -154,7 +152,7 @@ class RoomEventsStoreTestCase(unittest.TestCase): topic = u"A place for things" yield self.inject_room_event( - etype=RoomTopicEvent.TYPE, + etype=EventTypes.Topic, topic=topic, content={"topic": topic}, depth=1, diff --git a/tests/storage/test_roommember.py b/tests/storage/test_roommember.py index 6df09952d1..a23a8189df 100644 --- a/tests/storage/test_roommember.py +++ b/tests/storage/test_roommember.py @@ -18,8 +18,7 @@ from tests import unittest from twisted.internet import defer from synapse.server import HomeServer -from synapse.api.constants import Membership -from synapse.api.events.room import RoomMemberEvent +from synapse.api.constants import EventTypes, Membership from tests.utils import SQLiteMemoryDbPool, MockKey @@ -61,7 +60,7 @@ class RoomMemberStoreTestCase(unittest.TestCase): @defer.inlineCallbacks def inject_room_member(self, room, user, membership, replaces_state=None): builder = self.event_builder_factory.new({ - "type": RoomMemberEvent.TYPE, + "type": EventTypes.Member, "sender": user.to_string(), "state_key": user.to_string(), "room_id": room.to_string(), diff --git a/tests/storage/test_stream.py b/tests/storage/test_stream.py index 4865a5c142..9247fc579e 100644 --- a/tests/storage/test_stream.py +++ b/tests/storage/test_stream.py @@ -18,8 +18,7 @@ from tests import unittest from twisted.internet import defer from synapse.server import HomeServer -from synapse.api.constants import Membership -from synapse.api.events.room import RoomMemberEvent, MessageEvent +from synapse.api.constants import EventTypes, Membership from tests.utils import SQLiteMemoryDbPool, MockKey @@ -62,7 +61,7 @@ class StreamStoreTestCase(unittest.TestCase): self.depth += 1 builder = self.event_builder_factory.new({ - "type": RoomMemberEvent.TYPE, + "type": EventTypes.Member, "sender": user.to_string(), "state_key": user.to_string(), "room_id": room.to_string(), @@ -82,7 +81,7 @@ class StreamStoreTestCase(unittest.TestCase): self.depth += 1 builder = self.event_builder_factory.new({ - "type": MessageEvent.TYPE, + "type": EventTypes.Message, "sender": user.to_string(), "state_key": user.to_string(), "room_id": room.to_string(), @@ -125,7 +124,7 @@ class StreamStoreTestCase(unittest.TestCase): self.assertObjectHasAttributes( { - "type": MessageEvent.TYPE, + "type": EventTypes.Message, "user_id": self.u_alice.to_string(), "content": {"body": "test", "msgtype": "message"}, }, @@ -162,7 +161,7 @@ class StreamStoreTestCase(unittest.TestCase): self.assertObjectHasAttributes( { - "type": MessageEvent.TYPE, + "type": EventTypes.Message, "user_id": self.u_alice.to_string(), "content": {"body": "test", "msgtype": "message"}, }, diff --git a/tests/utils.py b/tests/utils.py index 70a221550c..731e03f517 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -15,15 +15,11 @@ from synapse.http.server import HttpServer from synapse.api.errors import cs_error, CodeMessageException, StoreError -from synapse.api.constants import Membership +from synapse.api.constants import EventTypes from synapse.storage import prepare_database from synapse.util.logcontext import LoggingContext -from synapse.api.events.room import ( - RoomMemberEvent, MessageEvent -) - from twisted.internet import defer, reactor from twisted.enterprise.adbapi import ConnectionPool @@ -276,7 +272,7 @@ class MemoryDataStore(object): return defer.succeed([]) def persist_event(self, event): - if event.type == RoomMemberEvent.TYPE: + if event.type == EventTypes.Member: room_id = event.room_id user = event.state_key membership = event.membership From 4afac883900fe07c99552bbb6f56414b8b0ec81d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 16 Dec 2014 13:07:48 +0000 Subject: [PATCH 134/158] Add basic docstring to annotate_context_with_state --- synapse/state.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/synapse/state.py b/synapse/state.py index d2763cdd9a..2f6eea09aa 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -137,6 +137,17 @@ class StateHandler(object): @defer.inlineCallbacks def annotate_context_with_state(self, event, context, old_state=None): + """ Fills out the context with the `current state` of the graph. The + `current state` here is defined to be exclusive of the given event, + i.e. its the state of the event graph just before the event. + + If `event` has `auth_events` then this will also fill out the + `auth_events` field on `context` from the `current_state`. + + Args: + event (EventBase) + context (EventContext) + """ yield run_on_reactor() if old_state: From 882dc8dcab434d24b0e902be3b4545ceabe9bdbe Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 16 Dec 2014 13:17:09 +0000 Subject: [PATCH 135/158] Persist internal_metadata --- synapse/events/__init__.py | 5 +++-- synapse/storage/__init__.py | 5 +++++ synapse/storage/_base.py | 7 ++++--- synapse/storage/schema/im.sql | 1 + 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index 1ec79a6788..984f14fce4 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -111,7 +111,7 @@ class EventBase(object): class FrozenEvent(EventBase): - def __init__(self, event_dict): + def __init__(self, event_dict, internal_metadata_dict={}): event_dict = copy.deepcopy(event_dict) signatures = copy.deepcopy(event_dict.pop("signatures", {})) @@ -122,7 +122,8 @@ class FrozenEvent(EventBase): super(FrozenEvent, self).__init__( frozen_dict, signatures=signatures, - unsigned=unsigned + unsigned=unsigned, + internal_metadata_dict=internal_metadata_dict, ) @staticmethod diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 5c079da5ba..26f205ae8f 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -156,12 +156,17 @@ class DataStore(RoomMemberStore, RoomStore, ] } + metadata_json = encode_canonical_json( + event.internal_metadata.get_dict() + ) + self._simple_insert_txn( txn, table="event_json", values={ "event_id": event.event_id, "room_id": event.room_id, + "internal_metadata": metadata_json.decode("UTF-8"), "json": encode_canonical_json(event_dict).decode("UTF-8"), }, or_replace=True, diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 31d5163c19..6dc857c4aa 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -452,7 +452,7 @@ class SQLBaseStore(object): def _get_event_txn(self, txn, event_id, check_redacted=True, get_prev_content=True): sql = ( - "SELECT json, r.event_id FROM event_json as e " + "SELECT internal_metadata, json, r.event_id FROM event_json as e " "LEFT JOIN redactions as r ON e.event_id = r.redacts " "WHERE e.event_id = ? " "LIMIT 1 " @@ -465,11 +465,12 @@ class SQLBaseStore(object): if not res: return None - js, redacted = res + internal_metadata, js, redacted = res d = json.loads(js) + internal_metadata = json.loads(internal_metadata) - ev = FrozenEvent(d) + ev = FrozenEvent(d, internal_metadata_dict=internal_metadata) if check_redacted and redacted: ev = prune_event(ev) diff --git a/synapse/storage/schema/im.sql b/synapse/storage/schema/im.sql index 0300bb29e1..253f9f779b 100644 --- a/synapse/storage/schema/im.sql +++ b/synapse/storage/schema/im.sql @@ -36,6 +36,7 @@ CREATE INDEX IF NOT EXISTS events_room_id ON events (room_id); CREATE TABLE IF NOT EXISTS event_json( event_id TEXT NOT NULL, room_id TEXT NOT NULL, + internal_metadata NOT NULL, json BLOB NOT NULL, CONSTRAINT ev_j_uniq UNIQUE (event_id) ); From 35f4f6b07019a9815fb06529412c35a65c3cd285 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 16 Dec 2014 13:27:53 +0000 Subject: [PATCH 136/158] Update upgrade script --- scripts/upgrade_db_to_v0.5.5.py | 13 +++++++++++-- synapse/storage/__init__.py | 2 +- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/scripts/upgrade_db_to_v0.5.5.py b/scripts/upgrade_db_to_v0.5.5.py index be9d07b2df..aa0d7667ac 100644 --- a/scripts/upgrade_db_to_v0.5.5.py +++ b/scripts/upgrade_db_to_v0.5.5.py @@ -31,6 +31,7 @@ delta_sql = """ CREATE TABLE IF NOT EXISTS event_json( event_id TEXT NOT NULL, room_id TEXT NOT NULL, + internal_metadata NOT NULL, json BLOB NOT NULL, CONSTRAINT ev_j_uniq UNIQUE (event_id) ); @@ -79,7 +80,7 @@ class Store(object): d.setdefault("unsigned", {})["age_ts"] = d.pop("age_ts") - d.pop("outlier", None) + outlier = d.pop("outlier", False) # d.pop("membership", None) @@ -87,7 +88,10 @@ class Store(object): d.pop("replaces_state", None) - events.append(EventBuilder(d)) + b = EventBuilder(d) + b.internal_metadata.outlier = outlier + + events.append(b) for i, ev in enumerate(events): signatures = self._get_event_signatures_txn( @@ -251,12 +255,17 @@ def reinsert_events(cursor, server_name, signing_key): event.get_dict() ).decode("UTF-8") + metadata_json = encode_canonical_json( + event.internal_metadata.get_dict() + ).decode("UTF-8") + store._simple_insert_txn( cursor, table="event_json", values={ "event_id": event.event_id, "room_id": event.room_id, + "internal_metadata": metadata_json, "json": event_json, }, or_replace=True, diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 26f205ae8f..cc1dcc2e74 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -156,7 +156,7 @@ class DataStore(RoomMemberStore, RoomStore, ] } - metadata_json = encode_canonical_json( + metadata_json = encode_canonical_json( event.internal_metadata.get_dict() ) From 8133cdcc88aa54480bfab41eb017e6b024b55c7e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 16 Dec 2014 13:32:06 +0000 Subject: [PATCH 137/158] Better english in docstrings are helpful. --- synapse/state.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/state.py b/synapse/state.py index 2f6eea09aa..38adde4dc9 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -138,8 +138,8 @@ class StateHandler(object): @defer.inlineCallbacks def annotate_context_with_state(self, event, context, old_state=None): """ Fills out the context with the `current state` of the graph. The - `current state` here is defined to be exclusive of the given event, - i.e. its the state of the event graph just before the event. + `current state` here is defined to be the state of the event graph + just before the event - i.e. it never includes `event` If `event` has `auth_events` then this will also fill out the `auth_events` field on `context` from the `current_state`. From 58168498b04eb9d0e85019fe485fa6b2ee79b422 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 16 Dec 2014 13:38:38 +0000 Subject: [PATCH 138/158] Remove FrozenEncoder --- synapse/util/frozenutils.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/synapse/util/frozenutils.py b/synapse/util/frozenutils.py index 1fb67df6b9..1874464c8b 100644 --- a/synapse/util/frozenutils.py +++ b/synapse/util/frozenutils.py @@ -46,11 +46,3 @@ def unfreeze(o): pass return o - - -class FrozenEncoder(json.JSONEncoder): - def default(self, o): - if isinstance(o, frozendict): - return dict(o) - - return json.JSONEncoder(self, o) From 5a465b67ba5938f7a18dd5411da0e31c4e5fe71c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 16 Dec 2014 13:41:43 +0000 Subject: [PATCH 139/158] Fix pyflakes --- synapse/util/frozenutils.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/synapse/util/frozenutils.py b/synapse/util/frozenutils.py index 1874464c8b..fcfb38b3b4 100644 --- a/synapse/util/frozenutils.py +++ b/synapse/util/frozenutils.py @@ -15,8 +15,6 @@ from frozendict import frozendict -import json - def freeze(o): if isinstance(o, dict) or isinstance(o, frozendict): From 96cc7c87405e87aacc2318f4634e1c6e5e2f46d3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 16 Dec 2014 13:57:27 +0000 Subject: [PATCH 140/158] Bump version --- VERSION | 2 +- synapse/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index 64c6e2f460..a918a2aa18 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.5.4a +0.6.0 diff --git a/synapse/__init__.py b/synapse/__init__.py index 7e49e1fd08..1cef40925f 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a synapse home server. """ -__version__ = "0.5.4a" +__version__ = "0.6.0" From ef5a14105064070a794d83a6542bf01eb62e1512 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 16 Dec 2014 13:57:47 +0000 Subject: [PATCH 141/158] Bump database version --- synapse/storage/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index cc1dcc2e74..62f89b8d8c 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -66,7 +66,7 @@ SCHEMAS = [ # Remember to update this number every time an incompatible change is made to # database schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 9 +SCHEMA_VERSION = 10 class _RollbackButIsFineException(Exception): From bab1e790aeac3d0047cea18a473735ec85d7bb6b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 16 Dec 2014 13:58:38 +0000 Subject: [PATCH 142/158] Include database bump in upgrade script --- scripts/upgrade_db_to_v0.5.5.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/upgrade_db_to_v0.5.5.py b/scripts/upgrade_db_to_v0.5.5.py index aa0d7667ac..5f6fa3a5aa 100644 --- a/scripts/upgrade_db_to_v0.5.5.py +++ b/scripts/upgrade_db_to_v0.5.5.py @@ -38,6 +38,8 @@ CREATE TABLE IF NOT EXISTS event_json( CREATE INDEX IF NOT EXISTS event_json_id ON event_json(event_id); CREATE INDEX IF NOT EXISTS event_json_room_id ON event_json(room_id); + +PRAGMA user_version = 10; """ From cb91ce5bba7ad8644b1e34e627c4238cd3f1aa41 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 16 Dec 2014 13:58:57 +0000 Subject: [PATCH 143/158] Rename upgrade script --- scripts/{upgrade_db_to_v0.5.5.py => upgrade_db_to_v0.6.0.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename scripts/{upgrade_db_to_v0.5.5.py => upgrade_db_to_v0.6.0.py} (100%) diff --git a/scripts/upgrade_db_to_v0.5.5.py b/scripts/upgrade_db_to_v0.6.0.py similarity index 100% rename from scripts/upgrade_db_to_v0.5.5.py rename to scripts/upgrade_db_to_v0.6.0.py From 592ba14b36bca88aa4517aa4885f89a706cc3b06 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 16 Dec 2014 14:07:05 +0000 Subject: [PATCH 144/158] Fix bugs in upgrade script. Handle the case when there are colons in server_name. Handle http exceptions more gracefully. --- scripts/upgrade_db_to_v0.6.0.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/upgrade_db_to_v0.6.0.py b/scripts/upgrade_db_to_v0.6.0.py index 5f6fa3a5aa..8f371bb4bb 100644 --- a/scripts/upgrade_db_to_v0.6.0.py +++ b/scripts/upgrade_db_to_v0.6.0.py @@ -21,6 +21,7 @@ from syutil.jsonutil import encode_canonical_json import argparse import dns.resolver import hashlib +import httplib import json import sqlite3 import syutil @@ -150,7 +151,6 @@ def get_key(server_name): if ":" in server_name: target, port = server_name.split(":") targets.append((target, int(port))) - return try: answers = dns.resolver.query("_matrix._tcp." + server_name, "SRV") for srv in answers: @@ -176,6 +176,10 @@ def get_key(server_name): return verify_keys except urllib2.URLError: pass + except urllib2.HTTPError: + pass + except httplib.HTTPException: + pass print "Failed to get keys for %s" % (server_name,) return {} From 2e44714214af801906c9fbc3e1cb1bc8d6b92427 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 16 Dec 2014 14:20:32 +0000 Subject: [PATCH 145/158] Make failure to run appropraite upgrade scripts more helpful. --- synapse/app/homeserver.py | 13 ++++++++++--- synapse/storage/__init__.py | 12 ++++++++++++ 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index a6e29c0860..140c99f18a 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from synapse.storage import prepare_database +from synapse.storage import prepare_database, UpgradeDatabaseException from synapse.server import HomeServer @@ -228,8 +228,15 @@ def setup(): logger.info("Preparing database: %s...", db_name) - with sqlite3.connect(db_name) as db_conn: - prepare_database(db_conn) + try: + with sqlite3.connect(db_name) as db_conn: + prepare_database(db_conn) + except UpgradeDatabaseException: + sys.stderr.write( + "\nFailed to upgrade database.\n" + "Have you followed any instructions in UPGRADES.rst?\n" + ) + sys.exit(1) logger.info("Database prepared in %s.", db_name) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 62f89b8d8c..2e97bbab3b 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -526,6 +526,14 @@ def read_schema(schema): return schema_file.read() +class PrepareDatabaseException(Exception): + pass + + +class UpgradeDatabaseException(PrepareDatabaseException): + pass + + def prepare_database(db_conn): """ Set up all the dbs. Since all the *.sql have IF NOT EXISTS, so we don't have to worry about overwriting existing content. @@ -542,6 +550,10 @@ def prepare_database(db_conn): "Cannot use this database as it is too " + "new for the server to understand" ) + elif user_version < 10: + raise UpgradeDatabaseException( + "No delta for versions less than 10" + ) elif user_version < SCHEMA_VERSION: logger.info( "Upgrading database from version %d", From d2ca24087fe6fdd91535602ffe493a92c3a468c6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 16 Dec 2014 14:36:31 +0000 Subject: [PATCH 146/158] Bump UPGRADES and CHANGES --- CHANGES.rst | 9 +++++++++ UPGRADE.rst | 16 ++++++++++++++++ scripts/upgrade_db_to_v0.6.0.py | 2 ++ 3 files changed, 27 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 0d36e8eeff..23bdac6a8a 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,12 @@ +Changes in synapse 0.6.0 (2014-12-16) +===================================== + + * Add new API for media upload and download that supports thumbnailing. + * Implement typing notifications. + * Fix bugs where we sent events with invalid signatures due to bugs where + we incorrectly persisted events. + * Improve performance of database queries involving retrieving events. + Changes in synapse 0.5.4a (2014-12-13) ====================================== diff --git a/UPGRADE.rst b/UPGRADE.rst index 5ebdd455c1..a602a9f3eb 100644 --- a/UPGRADE.rst +++ b/UPGRADE.rst @@ -1,3 +1,19 @@ +Upgrading to v0.6.0 +=================== + +This update includes a change to the database schema. To upgrade you first need +to upgrade the database by running:: + + python scripts/upgrade_db_to_v0.6.0.py + +Where `` is the location of the database, `` is the +server name as specified in the synapse configuration, and `` is +the location of the signing key as specified in the synapse configuration. + +This may take some time to complete. Failures of signatures and content hashes +can safely be ignored. + + Upgrading to v0.5.1 =================== diff --git a/scripts/upgrade_db_to_v0.6.0.py b/scripts/upgrade_db_to_v0.6.0.py index 8f371bb4bb..add088a818 100644 --- a/scripts/upgrade_db_to_v0.6.0.py +++ b/scripts/upgrade_db_to_v0.6.0.py @@ -284,6 +284,8 @@ def main(database, server_name, signing_key): reinsert_events(cursor, server_name, signing_key) conn.commit() + print "Success!" + if __name__ == "__main__": parser = argparse.ArgumentParser() From b3c793e362004f736c84388008a2aad07f61a492 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 16 Dec 2014 14:44:53 +0000 Subject: [PATCH 147/158] Do run all deltas up to missing delta 10 --- synapse/storage/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 2e97bbab3b..2a683b25f7 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -550,10 +550,6 @@ def prepare_database(db_conn): "Cannot use this database as it is too " + "new for the server to understand" ) - elif user_version < 10: - raise UpgradeDatabaseException( - "No delta for versions less than 10" - ) elif user_version < SCHEMA_VERSION: logger.info( "Upgrading database from version %d", @@ -562,6 +558,10 @@ def prepare_database(db_conn): # Run every version since after the current version. for v in range(user_version + 1, SCHEMA_VERSION + 1): + if v == 10: + raise UpgradeDatabaseException( + "No delta for version 10" + ) sql_script = read_schema("delta/v%d" % (v)) c.executescript(sql_script) From 8b8beba194f45b171345a44d4b42841d5e7311d3 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 16 Dec 2014 15:07:38 +0000 Subject: [PATCH 148/158] Remove annotate_event_with_state as nothing was using it. Update state tests to call annotate_context_with_state --- synapse/state.py | 66 --------------------- tests/test_state.py | 140 +++++++++++++++++--------------------------- 2 files changed, 55 insertions(+), 151 deletions(-) diff --git a/synapse/state.py b/synapse/state.py index 38adde4dc9..61b14b939f 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -22,7 +22,6 @@ from synapse.api.constants import EventTypes from collections import namedtuple -import copy import logging import hashlib @@ -43,71 +42,6 @@ class StateHandler(object): def __init__(self, hs): self.store = hs.get_datastore() - @defer.inlineCallbacks - @log_function - def annotate_event_with_state(self, event, old_state=None): - """ Annotates the event with the current state events as of that event. - - This method adds three new attributes to the event: - * `state_events`: The state up to and including the event. Encoded - as a dict mapping tuple (type, state_key) -> event. - * `old_state_events`: The state up to, but excluding, the event. - Encoded similarly as `state_events`. - * `state_group`: If there is an existing state group that can be - used, then return that. Otherwise return `None`. See state - storage for more information. - - If the argument `old_state` is given (in the form of a list of - events), then they are used as a the values for `old_state_events` and - the value for `state_events` is generated from it. `state_group` is - set to None. - - This needs to be called before persisting the event. - """ - yield run_on_reactor() - - if old_state: - event.state_group = None - event.old_state_events = { - (s.type, s.state_key): s for s in old_state - } - event.state_events = event.old_state_events - - if hasattr(event, "state_key"): - event.state_events[(event.type, event.state_key)] = event - - defer.returnValue(False) - return - - if hasattr(event, "outlier") and event.outlier: - event.state_group = None - event.old_state_events = None - event.state_events = None - defer.returnValue(False) - return - - ids = [e for e, _ in event.prev_events] - - ret = yield self.resolve_state_groups(ids) - state_group, new_state, _ = ret - - event.old_state_events = copy.deepcopy(new_state) - - if hasattr(event, "state_key"): - key = (event.type, event.state_key) - if key in new_state: - event.replaces_state = new_state[key].event_id - new_state[key] = event - elif state_group: - event.state_group = state_group - event.state_events = new_state - defer.returnValue(False) - - event.state_group = None - event.state_events = new_state - - defer.returnValue(hasattr(event, "state_key")) - @defer.inlineCallbacks def get_current_state(self, room_id, event_type=None, state_key=""): """ Returns the current state for the room as a list. This is done by diff --git a/tests/test_state.py b/tests/test_state.py index 7979b54a35..197e35f140 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -26,6 +26,7 @@ class StateTestCase(unittest.TestCase): self.store = Mock( spec_set=[ "get_state_groups", + "add_event_hashes", ] ) hs = Mock(spec=["get_datastore"]) @@ -37,6 +38,7 @@ class StateTestCase(unittest.TestCase): @defer.inlineCallbacks def test_annotate_with_old_message(self): event = self.create_event(type="test_message", name="event") + context = Mock() old_state = [ self.create_event(type="test1", state_key="1"), @@ -44,48 +46,53 @@ class StateTestCase(unittest.TestCase): self.create_event(type="test2", state_key=""), ] - yield self.state.annotate_event_with_state(event, old_state=old_state) + yield self.state.annotate_context_with_state( + event, context, old_state=old_state + ) - for k, v in event.old_state_events.items(): - type, state_key = k - self.assertEqual(type, v.type) - self.assertEqual(state_key, v.state_key) - - self.assertEqual(set(old_state), set(event.old_state_events.values())) - self.assertDictEqual(event.old_state_events, event.state_events) - - self.assertIsNone(event.state_group) - - @defer.inlineCallbacks - def test_annotate_with_old_state(self): - event = self.create_event(type="state", state_key="", name="event") - - old_state = [ - self.create_event(type="test1", state_key="1"), - self.create_event(type="test1", state_key="2"), - self.create_event(type="test2", state_key=""), - ] - - yield self.state.annotate_event_with_state(event, old_state=old_state) - - for k, v in event.old_state_events.items(): + for k, v in context.current_state.items(): type, state_key = k self.assertEqual(type, v.type) self.assertEqual(state_key, v.state_key) self.assertEqual( - set(old_state + [event]), - set(event.old_state_events.values()) + set(old_state), set(context.current_state.values()) ) - self.assertDictEqual(event.old_state_events, event.state_events) + self.assertIsNone(context.state_group) - self.assertIsNone(event.state_group) + @defer.inlineCallbacks + def test_annotate_with_old_state(self): + event = self.create_event(type="state", state_key="", name="event") + context = Mock() + + old_state = [ + self.create_event(type="test1", state_key="1"), + self.create_event(type="test1", state_key="2"), + self.create_event(type="test2", state_key=""), + ] + + yield self.state.annotate_context_with_state( + event, context, old_state=old_state + ) + + for k, v in context.current_state.items(): + type, state_key = k + self.assertEqual(type, v.type) + self.assertEqual(state_key, v.state_key) + + self.assertEqual( + set(old_state), + set(context.current_state.values()) + ) + + self.assertIsNone(context.state_group) @defer.inlineCallbacks def test_trivial_annotate_message(self): event = self.create_event(type="test_message", name="event") event.prev_events = [] + context = Mock() old_state = [ self.create_event(type="test1", state_key="1"), @@ -99,35 +106,25 @@ class StateTestCase(unittest.TestCase): group_name: old_state, } - yield self.state.annotate_event_with_state(event) + yield self.state.annotate_context_with_state(event, context) - for k, v in event.old_state_events.items(): + for k, v in context.current_state.items(): type, state_key = k self.assertEqual(type, v.type) self.assertEqual(state_key, v.state_key) self.assertEqual( set([e.event_id for e in old_state]), - set([e.event_id for e in event.old_state_events.values()]) + set([e.event_id for e in context.current_state.values()]) ) - self.assertDictEqual( - { - k: v.event_id - for k, v in event.old_state_events.items() - }, - { - k: v.event_id - for k, v in event.state_events.items() - } - ) - - self.assertEqual(group_name, event.state_group) + self.assertEqual(group_name, context.state_group) @defer.inlineCallbacks def test_trivial_annotate_state(self): event = self.create_event(type="state", state_key="", name="event") event.prev_events = [] + context = Mock() old_state = [ self.create_event(type="test1", state_key="1"), @@ -141,43 +138,25 @@ class StateTestCase(unittest.TestCase): group_name: old_state, } - yield self.state.annotate_event_with_state(event) + yield self.state.annotate_context_with_state(event, context) - for k, v in event.old_state_events.items(): + for k, v in context.current_state.items(): type, state_key = k self.assertEqual(type, v.type) self.assertEqual(state_key, v.state_key) self.assertEqual( set([e.event_id for e in old_state]), - set([e.event_id for e in event.old_state_events.values()]) + set([e.event_id for e in context.current_state.values()]) ) - self.assertEqual( - set([e.event_id for e in old_state] + [event.event_id]), - set([e.event_id for e in event.state_events.values()]) - ) - - new_state = { - k: v.event_id - for k, v in event.state_events.items() - } - old_state = { - k: v.event_id - for k, v in event.old_state_events.items() - } - old_state[(event.type, event.state_key)] = event.event_id - self.assertDictEqual( - old_state, - new_state - ) - - self.assertIsNone(event.state_group) + self.assertIsNone(context.state_group) @defer.inlineCallbacks def test_resolve_message_conflict(self): event = self.create_event(type="test_message", name="event") event.prev_events = [] + context = Mock() old_state_1 = [ self.create_event(type="test1", state_key="1"), @@ -199,21 +178,17 @@ class StateTestCase(unittest.TestCase): group_name_2: old_state_2, } - yield self.state.annotate_event_with_state(event) + yield self.state.annotate_context_with_state(event, context) - self.assertEqual(len(event.old_state_events), 5) + self.assertEqual(len(context.current_state), 5) - self.assertEqual( - set([e.event_id for e in event.state_events.values()]), - set([e.event_id for e in event.old_state_events.values()]) - ) - - self.assertIsNone(event.state_group) + self.assertIsNone(context.state_group) @defer.inlineCallbacks def test_resolve_state_conflict(self): event = self.create_event(type="test4", state_key="", name="event") event.prev_events = [] + context = Mock() old_state_1 = [ self.create_event(type="test1", state_key="1"), @@ -235,19 +210,11 @@ class StateTestCase(unittest.TestCase): group_name_2: old_state_2, } - yield self.state.annotate_event_with_state(event) + yield self.state.annotate_context_with_state(event, context) - self.assertEqual(len(event.old_state_events), 5) + self.assertEqual(len(context.current_state), 5) - expected_new = event.old_state_events - expected_new[(event.type, event.state_key)] = event - - self.assertEqual( - set([e.event_id for e in expected_new.values()]), - set([e.event_id for e in event.state_events.values()]), - ) - - self.assertIsNone(event.state_group) + self.assertIsNone(context.state_group) def create_event(self, name=None, type=None, state_key=None): self.event_id += 1 @@ -266,6 +233,9 @@ class StateTestCase(unittest.TestCase): event.state_key = state_key event.event_id = event_id + event.is_state = lambda: (state_key is not None) + event.unsigned = {} + event.user_id = "@user_id:example.com" event.room_id = "!room_id:example.com" From 42b725ce52844b3e858193aa12ddc06933c7584a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 16 Dec 2014 15:13:34 +0000 Subject: [PATCH 149/158] Fix upgrade script to run all the missing deltas. --- scripts/upgrade_db_to_v0.6.0.py | 20 ++++++++++ synapse/storage/schema/delta/v9.sql | 58 ++++++++++++++++++++++++++++- 2 files changed, 77 insertions(+), 1 deletion(-) diff --git a/scripts/upgrade_db_to_v0.6.0.py b/scripts/upgrade_db_to_v0.6.0.py index add088a818..32c415a662 100644 --- a/scripts/upgrade_db_to_v0.6.0.py +++ b/scripts/upgrade_db_to_v0.6.0.py @@ -1,3 +1,5 @@ + +from synapse.storage import SCHEMA_VERSION, read_schema from synapse.storage._base import SQLBaseStore from synapse.storage.signatures import SignatureStore from synapse.storage.event_federation import EventFederationStore @@ -186,12 +188,16 @@ def get_key(server_name): def reinsert_events(cursor, server_name, signing_key): + print "Running delta: v10" + cursor.executescript(delta_sql) cursor.execute( "SELECT * FROM events ORDER BY rowid ASC" ) + print "Getting events..." + rows = store.cursor_to_dict(cursor) events = store._generate_event_json(cursor, rows) @@ -281,7 +287,21 @@ def reinsert_events(cursor, server_name, signing_key): def main(database, server_name, signing_key): conn = sqlite3.connect(database) cursor = conn.cursor() + + # Do other deltas: + cursor.execute("PRAGMA user_version") + row = cursor.fetchone() + + if row and row[0]: + user_version = row[0] + # Run every version since after the current version. + for v in range(user_version + 1, 10): + print "Running delta: %d" % (v,) + sql_script = read_schema("delta/v%d" % (v,)) + cursor.executescript(sql_script) + reinsert_events(cursor, server_name, signing_key) + conn.commit() print "Success!" diff --git a/synapse/storage/schema/delta/v9.sql b/synapse/storage/schema/delta/v9.sql index ad680c64da..0af29733a0 100644 --- a/synapse/storage/schema/delta/v9.sql +++ b/synapse/storage/schema/delta/v9.sql @@ -20,4 +20,60 @@ CREATE TABLE IF NOT EXISTS destinations( retry_interval INTEGER ); -PRAGMA user_version = 9; \ No newline at end of file + +CREATE TABLE IF NOT EXISTS local_media_repository ( + media_id TEXT, -- The id used to refer to the media. + media_type TEXT, -- The MIME-type of the media. + media_length INTEGER, -- Length of the media in bytes. + created_ts INTEGER, -- When the content was uploaded in ms. + upload_name TEXT, -- The name the media was uploaded with. + user_id TEXT, -- The user who uploaded the file. + CONSTRAINT uniqueness UNIQUE (media_id) +); + +CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails ( + media_id TEXT, -- The id used to refer to the media. + thumbnail_width INTEGER, -- The width of the thumbnail in pixels. + thumbnail_height INTEGER, -- The height of the thumbnail in pixels. + thumbnail_type TEXT, -- The MIME-type of the thumbnail. + thumbnail_method TEXT, -- The method used to make the thumbnail. + thumbnail_length INTEGER, -- The length of the thumbnail in bytes. + CONSTRAINT uniqueness UNIQUE ( + media_id, thumbnail_width, thumbnail_height, thumbnail_type + ) +); + +CREATE INDEX IF NOT EXISTS local_media_repository_thumbnails_media_id + ON local_media_repository_thumbnails (media_id); + +CREATE TABLE IF NOT EXISTS remote_media_cache ( + media_origin TEXT, -- The remote HS the media came from. + media_id TEXT, -- The id used to refer to the media on that server. + media_type TEXT, -- The MIME-type of the media. + created_ts INTEGER, -- When the content was uploaded in ms. + upload_name TEXT, -- The name the media was uploaded with. + media_length INTEGER, -- Length of the media in bytes. + filesystem_id TEXT, -- The name used to store the media on disk. + CONSTRAINT uniqueness UNIQUE (media_origin, media_id) +); + +CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails ( + media_origin TEXT, -- The remote HS the media came from. + media_id TEXT, -- The id used to refer to the media. + thumbnail_width INTEGER, -- The width of the thumbnail in pixels. + thumbnail_height INTEGER, -- The height of the thumbnail in pixels. + thumbnail_method TEXT, -- The method used to make the thumbnail + thumbnail_type TEXT, -- The MIME-type of the thumbnail. + thumbnail_length INTEGER, -- The length of the thumbnail in bytes. + filesystem_id TEXT, -- The name used to store the media on disk. + CONSTRAINT uniqueness UNIQUE ( + media_origin, media_id, thumbnail_width, thumbnail_height, + thumbnail_type, thumbnail_type + ) +); + +CREATE INDEX IF NOT EXISTS remote_media_cache_thumbnails_media_id + ON local_media_repository_thumbnails (media_id); + + +PRAGMA user_version = 9; From 3c7857e49b8dcad723d52174aba77c47453c0298 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 16 Dec 2014 15:24:03 +0000 Subject: [PATCH 150/158] clean up coding style a bit --- synapse/events/__init__.py | 22 +++++++++++----------- synapse/handlers/_base.py | 4 ++-- synapse/handlers/directory.py | 1 - synapse/handlers/federation.py | 11 ++++------- synapse/handlers/typing.py | 4 ++-- synapse/media/v1/base_resource.py | 1 + synapse/rest/room.py | 8 +++++--- synapse/rest/transactions.py | 1 + 8 files changed, 26 insertions(+), 26 deletions(-) diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index 984f14fce4..34b1b944ab 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -30,20 +30,20 @@ class _EventInternalMetadata(object): def _event_dict_property(key): - def getter(self): - return self._event_dict[key] + def getter(self): + return self._event_dict[key] - def setter(self, v): - self._event_dict[key] = v + def setter(self, v): + self._event_dict[key] = v - def delete(self): - del self._event_dict[key] + def delete(self): + del self._event_dict[key] - return property( - getter, - setter, - delete, - ) + return property( + getter, + setter, + delete, + ) class EventBase(object): diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index af8eb5f0f5..567769253e 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -61,8 +61,6 @@ class BaseHandler(object): def _create_new_client_event(self, builder): yield run_on_reactor() - context = EventContext() - latest_ret = yield self.store.get_latest_events_in_room( builder.room_id, ) @@ -78,6 +76,8 @@ class BaseHandler(object): builder.depth = depth state_handler = self.state_handler + + context = EventContext() ret = yield state_handler.annotate_context_with_state( builder, context, diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index 404baea796..66d3b533d9 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -156,4 +156,3 @@ class DirectoryHandler(BaseHandler): "sender": user_id, "content": {"aliases": aliases}, }) - diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 16a104c0e2..d80a54bdea 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -202,7 +202,7 @@ class FederationHandler(BaseHandler): e.msg, affected=event.event_id, ) - + # if we're receiving valid events from an origin, # it's probably a good idea to mark it as not in retry-state # for sending (although this is a bit of a leap) @@ -263,9 +263,7 @@ class FederationHandler(BaseHandler): context = EventContext() yield self.state_handler.annotate_context_with_state(event, context) - events.append( - (event, context) - ) + events.append((event, context)) yield self.store.persist_event( event, @@ -547,8 +545,6 @@ class FederationHandler(BaseHandler): """ event = pdu - context = EventContext() - event.internal_metadata.outlier = True event.signatures.update( @@ -559,6 +555,7 @@ class FederationHandler(BaseHandler): ) ) + context = EventContext() yield self.state_handler.annotate_context_with_state(event, context) yield self.store.persist_event( @@ -685,13 +682,13 @@ class FederationHandler(BaseHandler): @defer.inlineCallbacks def _handle_new_event(self, event, state=None, backfilled=False, current_state=None, fetch_missing=True): - context = EventContext() logger.debug( "_handle_new_event: Before annotate: %s, sigs: %s", event.event_id, event.signatures, ) + context = EventContext() yield self.state_handler.annotate_context_with_state( event, context, diff --git a/synapse/handlers/typing.py b/synapse/handlers/typing.py index 77d66f66ff..7626b07280 100644 --- a/synapse/handlers/typing.py +++ b/synapse/handlers/typing.py @@ -45,8 +45,8 @@ class TypingNotificationHandler(BaseHandler): hs.get_distributor().observe("user_left_room", self.user_left_room) - self._member_typing_until = {} # clock time we expect to stop - self._member_typing_timer = {} # deferreds to manage theabove + self._member_typing_until = {} # clock time we expect to stop + self._member_typing_timer = {} # deferreds to manage theabove # map room IDs to serial numbers self._room_serials = {} diff --git a/synapse/media/v1/base_resource.py b/synapse/media/v1/base_resource.py index 14735ff375..2f5440ab64 100644 --- a/synapse/media/v1/base_resource.py +++ b/synapse/media/v1/base_resource.py @@ -135,6 +135,7 @@ class BaseMediaResource(Resource): if download is None: download = self._get_remote_media_impl(server_name, media_id) self.downloads[key] = download + @download.addBoth def callback(media_info): del self.downloads[key] diff --git a/synapse/rest/room.py b/synapse/rest/room.py index 0e2d5fbaae..005a9f6f44 100644 --- a/synapse/rest/room.py +++ b/synapse/rest/room.py @@ -310,8 +310,8 @@ class RoomMessageListRestServlet(RestServlet): @defer.inlineCallbacks def on_GET(self, request, room_id): user = yield self.auth.get_user_by_req(request) - pagination_config = PaginationConfig.from_request(request, - default_limit=10, + pagination_config = PaginationConfig.from_request( + request, default_limit=10, ) with_feedback = "feedback" in request.args handler = self.handlers.message_handler @@ -466,7 +466,9 @@ class RoomRedactEventRestServlet(RestServlet): class RoomTypingRestServlet(RestServlet): - PATTERN = client_path_pattern("/rooms/(?P[^/]*)/typing/(?P[^/]*)$") + PATTERN = client_path_pattern( + "/rooms/(?P[^/]*)/typing/(?P[^/]*)$" + ) @defer.inlineCallbacks def on_PUT(self, request, room_id, user_id): diff --git a/synapse/rest/transactions.py b/synapse/rest/transactions.py index 8c41ab4edb..31377bd41d 100644 --- a/synapse/rest/transactions.py +++ b/synapse/rest/transactions.py @@ -19,6 +19,7 @@ import logging logger = logging.getLogger(__name__) + # FIXME: elsewhere we use FooStore to indicate something in the storage layer... class HttpTransactionStore(object): From c3eae8a88c21cf99b0109ebcb3f0f49714617060 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 16 Dec 2014 15:59:17 +0000 Subject: [PATCH 151/158] Construct the EventContext in the state handler rather than constructing one and then immediately calling state_handler.annotate_context_with_state --- synapse/handlers/_base.py | 11 ++--------- synapse/handlers/federation.py | 14 ++++---------- synapse/state.py | 14 ++++++++++---- tests/handlers/test_federation.py | 18 ++++++++---------- tests/handlers/test_room.py | 30 ++++++++++++++++++------------ tests/test_state.py | 22 ++++++++-------------- 6 files changed, 50 insertions(+), 59 deletions(-) diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index 567769253e..97ebd98917 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -20,8 +20,6 @@ from synapse.util.async import run_on_reactor from synapse.crypto.event_signing import add_hashes_and_signatures from synapse.api.constants import Membership, EventTypes -from synapse.events.snapshot import EventContext - import logging @@ -77,15 +75,10 @@ class BaseHandler(object): state_handler = self.state_handler - context = EventContext() - ret = yield state_handler.annotate_context_with_state( - builder, - context, - ) - prev_state = ret + context = yield state_handler.compute_event_context(builder) if builder.is_state(): - builder.prev_state = prev_state + builder.prev_state = context.prev_state_events yield self.auth.add_auth_events(builder, context) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index d80a54bdea..4aec3563ac 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -17,7 +17,6 @@ from ._base import BaseHandler -from synapse.events.snapshot import EventContext from synapse.events.utils import prune_event from synapse.api.errors import ( AuthError, FederationError, SynapseError, StoreError, @@ -260,8 +259,7 @@ class FederationHandler(BaseHandler): event = pdu # FIXME (erikj): Not sure this actually works :/ - context = EventContext() - yield self.state_handler.annotate_context_with_state(event, context) + context = yield self.state_handler.compute_event_context(event) events.append((event, context)) @@ -555,8 +553,7 @@ class FederationHandler(BaseHandler): ) ) - context = EventContext() - yield self.state_handler.annotate_context_with_state(event, context) + context = yield self.state_handler.compute_event_context(event) yield self.store.persist_event( event, @@ -688,11 +685,8 @@ class FederationHandler(BaseHandler): event.event_id, event.signatures, ) - context = EventContext() - yield self.state_handler.annotate_context_with_state( - event, - context, - old_state=state + context = yield self.state_handler.compute_event_context( + event, old_state=state ) logger.debug( diff --git a/synapse/state.py b/synapse/state.py index 61b14b939f..551058b510 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -19,6 +19,7 @@ from twisted.internet import defer from synapse.util.logutils import log_function from synapse.util.async import run_on_reactor from synapse.api.constants import EventTypes +from synapse.events.snapshot import EventContext from collections import namedtuple @@ -70,7 +71,7 @@ class StateHandler(object): defer.returnValue(res[1].values()) @defer.inlineCallbacks - def annotate_context_with_state(self, event, context, old_state=None): + def compute_event_context(self, event, old_state=None): """ Fills out the context with the `current state` of the graph. The `current state` here is defined to be the state of the event graph just before the event - i.e. it never includes `event` @@ -80,8 +81,11 @@ class StateHandler(object): Args: event (EventBase) - context (EventContext) + Returns: + an EventContext """ + context = EventContext() + yield run_on_reactor() if old_state: @@ -107,7 +111,8 @@ class StateHandler(object): if replaces.event_id != event.event_id: # Paranoia check event.unsigned["replaces_state"] = replaces.event_id - defer.returnValue([]) + context.prev_state_events = [] + defer.returnValue(context) if event.is_state(): ret = yield self.resolve_state_groups( @@ -145,7 +150,8 @@ class StateHandler(object): else: context.auth_events = {} - defer.returnValue(prev_state) + context.prev_state_events = prev_state + defer.returnValue(context) @defer.inlineCallbacks @log_function diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py index ed351367cc..ed21defd13 100644 --- a/tests/handlers/test_federation.py +++ b/tests/handlers/test_federation.py @@ -34,7 +34,7 @@ class FederationTestCase(unittest.TestCase): self.mock_config.signing_key = [MockKey()] self.state_handler = NonCallableMock(spec_set=[ - "annotate_context_with_state", + "compute_event_context", ]) self.auth = NonCallableMock(spec_set=[ @@ -91,11 +91,12 @@ class FederationTestCase(unittest.TestCase): self.datastore.get_room.return_value = defer.succeed(True) self.auth.check_host_in_room.return_value = defer.succeed(True) - def annotate(ev, context, old_state=None): + def annotate(ev, old_state=None): + context = Mock() context.current_state = {} context.auth_events = {} - return defer.succeed(False) - self.state_handler.annotate_context_with_state.side_effect = annotate + return defer.succeed(context) + self.state_handler.compute_event_context.side_effect = annotate yield self.handlers.federation_handler.on_receive_pdu( "fo", pdu, False @@ -109,15 +110,12 @@ class FederationTestCase(unittest.TestCase): context=ANY, ) - self.state_handler.annotate_context_with_state.assert_called_once_with( - ANY, - ANY, - old_state=None, + self.state_handler.compute_event_context.assert_called_once_with( + ANY, old_state=None, ) self.auth.check.assert_called_once_with(ANY, auth_events={}) self.notifier.on_new_room_event.assert_called_once_with( - ANY, - extra_users=[] + ANY, extra_users=[] ) diff --git a/tests/handlers/test_room.py b/tests/handlers/test_room.py index 83493cae20..0cb8aa4fbc 100644 --- a/tests/handlers/test_room.py +++ b/tests/handlers/test_room.py @@ -60,7 +60,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase): "check_host_in_room", ]), state_handler=NonCallableMock(spec_set=[ - "annotate_context_with_state", + "compute_event_context", "get_current_state", ]), config=self.mock_config, @@ -110,7 +110,8 @@ class RoomMemberHandlerTestCase(unittest.TestCase): defer.succeed([]) ) - def annotate(_, ctx): + def annotate(_): + ctx = Mock() ctx.current_state = { (EventTypes.Member, "@alice:green"): self._create_member( user_id="@alice:green", @@ -121,10 +122,11 @@ class RoomMemberHandlerTestCase(unittest.TestCase): room_id=room_id, ), } + ctx.prev_state_events = [] - return defer.succeed(True) + return defer.succeed(ctx) - self.state_handler.annotate_context_with_state.side_effect = annotate + self.state_handler.compute_event_context.side_effect = annotate def add_auth(_, ctx): ctx.auth_events = ctx.current_state[ @@ -146,8 +148,8 @@ class RoomMemberHandlerTestCase(unittest.TestCase): yield room_handler.change_membership(event, context) - self.state_handler.annotate_context_with_state.assert_called_once_with( - builder, context + self.state_handler.compute_event_context.assert_called_once_with( + builder ) self.auth.add_auth_events.assert_called_once_with( @@ -189,7 +191,8 @@ class RoomMemberHandlerTestCase(unittest.TestCase): defer.succeed([]) ) - def annotate(_, ctx): + def annotate(_): + ctx = Mock() ctx.current_state = { (EventTypes.Member, "@bob:red"): self._create_member( user_id="@bob:red", @@ -197,10 +200,11 @@ class RoomMemberHandlerTestCase(unittest.TestCase): membership=Membership.INVITE ), } + ctx.prev_state_events = [] - return defer.succeed(True) + return defer.succeed(ctx) - self.state_handler.annotate_context_with_state.side_effect = annotate + self.state_handler.compute_event_context.side_effect = annotate def add_auth(_, ctx): ctx.auth_events = ctx.current_state[ @@ -262,7 +266,8 @@ class RoomMemberHandlerTestCase(unittest.TestCase): defer.succeed([]) ) - def annotate(_, ctx): + def annotate(_): + ctx = Mock() ctx.current_state = { (EventTypes.Member, "@bob:red"): self._create_member( user_id="@bob:red", @@ -270,10 +275,11 @@ class RoomMemberHandlerTestCase(unittest.TestCase): membership=Membership.JOIN ), } + ctx.prev_state_events = [] - return defer.succeed(True) + return defer.succeed(ctx) - self.state_handler.annotate_context_with_state.side_effect = annotate + self.state_handler.compute_event_context.side_effect = annotate def add_auth(_, ctx): ctx.auth_events = ctx.current_state[ diff --git a/tests/test_state.py b/tests/test_state.py index 197e35f140..98ad9e54cd 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -38,7 +38,6 @@ class StateTestCase(unittest.TestCase): @defer.inlineCallbacks def test_annotate_with_old_message(self): event = self.create_event(type="test_message", name="event") - context = Mock() old_state = [ self.create_event(type="test1", state_key="1"), @@ -46,8 +45,8 @@ class StateTestCase(unittest.TestCase): self.create_event(type="test2", state_key=""), ] - yield self.state.annotate_context_with_state( - event, context, old_state=old_state + context = yield self.state.compute_event_context( + event, old_state=old_state ) for k, v in context.current_state.items(): @@ -64,7 +63,6 @@ class StateTestCase(unittest.TestCase): @defer.inlineCallbacks def test_annotate_with_old_state(self): event = self.create_event(type="state", state_key="", name="event") - context = Mock() old_state = [ self.create_event(type="test1", state_key="1"), @@ -72,8 +70,8 @@ class StateTestCase(unittest.TestCase): self.create_event(type="test2", state_key=""), ] - yield self.state.annotate_context_with_state( - event, context, old_state=old_state + context = yield self.state.compute_event_context( + event, old_state=old_state ) for k, v in context.current_state.items(): @@ -92,7 +90,6 @@ class StateTestCase(unittest.TestCase): def test_trivial_annotate_message(self): event = self.create_event(type="test_message", name="event") event.prev_events = [] - context = Mock() old_state = [ self.create_event(type="test1", state_key="1"), @@ -106,7 +103,7 @@ class StateTestCase(unittest.TestCase): group_name: old_state, } - yield self.state.annotate_context_with_state(event, context) + context = yield self.state.compute_event_context(event) for k, v in context.current_state.items(): type, state_key = k @@ -124,7 +121,6 @@ class StateTestCase(unittest.TestCase): def test_trivial_annotate_state(self): event = self.create_event(type="state", state_key="", name="event") event.prev_events = [] - context = Mock() old_state = [ self.create_event(type="test1", state_key="1"), @@ -138,7 +134,7 @@ class StateTestCase(unittest.TestCase): group_name: old_state, } - yield self.state.annotate_context_with_state(event, context) + context = yield self.state.compute_event_context(event) for k, v in context.current_state.items(): type, state_key = k @@ -156,7 +152,6 @@ class StateTestCase(unittest.TestCase): def test_resolve_message_conflict(self): event = self.create_event(type="test_message", name="event") event.prev_events = [] - context = Mock() old_state_1 = [ self.create_event(type="test1", state_key="1"), @@ -178,7 +173,7 @@ class StateTestCase(unittest.TestCase): group_name_2: old_state_2, } - yield self.state.annotate_context_with_state(event, context) + context = yield self.state.compute_event_context(event) self.assertEqual(len(context.current_state), 5) @@ -188,7 +183,6 @@ class StateTestCase(unittest.TestCase): def test_resolve_state_conflict(self): event = self.create_event(type="test4", state_key="", name="event") event.prev_events = [] - context = Mock() old_state_1 = [ self.create_event(type="test1", state_key="1"), @@ -210,7 +204,7 @@ class StateTestCase(unittest.TestCase): group_name_2: old_state_2, } - yield self.state.annotate_context_with_state(event, context) + context = yield self.state.compute_event_context(event) self.assertEqual(len(context.current_state), 5) From 23da4a40510bfda5c617946cf6f2b7ab3f95b480 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 16 Dec 2014 15:59:40 +0000 Subject: [PATCH 152/158] Fix typo where we thought a list was a dict --- synapse/state.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/synapse/state.py b/synapse/state.py index 38adde4dc9..99f873b6e5 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -263,9 +263,10 @@ class StateHandler(object): } if event_type: - prev_states = conflicted_state.get( - (event_type, state_key), {} - ).keys() + prev_states_events = conflicted_state.get( + (event_type, state_key), [] + ) + prev_states = [s.event_id for s in prev_states_events] else: prev_states = [] From 627e4f01d2bf2516ddb3f8737bb682753e9c21f1 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 16 Dec 2014 16:07:41 +0000 Subject: [PATCH 153/158] Remove send_message since nothing was calling it. Remove Snapshot because only send_message was using it --- synapse/handlers/message.py | 38 ------------------ synapse/storage/__init__.py | 80 ------------------------------------- 2 files changed, 118 deletions(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index baf372fdad..b529d890bb 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -18,8 +18,6 @@ from twisted.internet import defer from synapse.api.constants import EventTypes, Membership from synapse.api.errors import RoomError from synapse.streams.config import PaginationConfig -from synapse.util.logcontext import PreserveLoggingContext - from synapse.events.validator import EventValidator from ._base import BaseHandler @@ -66,35 +64,6 @@ class MessageHandler(BaseHandler): defer.returnValue(None) - @defer.inlineCallbacks - def send_message(self, event=None, suppress_auth=False): - """ Send a message. - - Args: - event : The message event to store. - suppress_auth (bool) : True to suppress auth for this message. This - is primarily so the home server can inject messages into rooms at - will. - Raises: - SynapseError if something went wrong. - """ - - self.ratelimit(event.user_id) - # TODO(paul): Why does 'event' not have a 'user' object? - user = self.hs.parse_userid(event.user_id) - assert self.hs.is_mine(user), "User must be our own: %s" % (user,) - - snapshot = yield self.store.snapshot_room(event) - - yield self._on_new_room_event( - event, snapshot, suppress_auth=suppress_auth - ) - - with PreserveLoggingContext(): - self.hs.get_handlers().presence_handler.bump_presence_active_time( - user - ) - @defer.inlineCallbacks def get_messages(self, user_id=None, room_id=None, pagin_config=None, feedback=False): @@ -212,13 +181,6 @@ class MessageHandler(BaseHandler): defer.returnValue(fb) defer.returnValue(None) - @defer.inlineCallbacks - def send_feedback(self, event): - snapshot = yield self.store.snapshot_room(event) - - # store message in db - yield self._on_new_room_event(event, snapshot) - @defer.inlineCallbacks def get_state_events(self, user_id, room_id): """Retrieve all state events for a given room. diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index cc1dcc2e74..c051f33a5c 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -419,86 +419,6 @@ class DataStore(RoomMemberStore, RoomStore, ], ) - def snapshot_room(self, event): - """Snapshot the room for an update by a user - Args: - room_id (synapse.types.RoomId): The room to snapshot. - user_id (synapse.types.UserId): The user to snapshot the room for. - state_type (str): Optional state type to snapshot. - state_key (str): Optional state key to snapshot. - Returns: - synapse.storage.Snapshot: A snapshot of the state of the room. - """ - def _snapshot(txn): - prev_events = self._get_latest_events_in_room( - txn, - event.room_id - ) - - prev_state = None - state_key = None - if hasattr(event, "state_key"): - state_key = event.state_key - prev_state = self._get_latest_state_in_room( - txn, - event.room_id, - type=event.type, - state_key=state_key, - ) - - return Snapshot( - store=self, - room_id=event.room_id, - user_id=event.user_id, - prev_events=prev_events, - prev_state=prev_state, - state_type=event.type, - state_key=state_key, - ) - - return self.runInteraction("snapshot_room", _snapshot) - - -class Snapshot(object): - """Snapshot of the state of a room - Args: - store (DataStore): The datastore. - room_id (RoomId): The room of the snapshot. - user_id (UserId): The user this snapshot is for. - prev_events (list): The list of event ids this snapshot is after. - membership_state (RoomMemberEvent): The current state of the user in - the room. - state_type (str, optional): State type captured by the snapshot - state_key (str, optional): State key captured by the snapshot - prev_state_pdu (PduEntry, optional): pdu id of - the previous value of the state type and key in the room. - """ - - def __init__(self, store, room_id, user_id, prev_events, - prev_state, state_type=None, state_key=None): - self.store = store - self.room_id = room_id - self.user_id = user_id - self.prev_events = prev_events - self.prev_state = prev_state - self.state_type = state_type - self.state_key = state_key - - def fill_out_prev_events(self, event): - if not hasattr(event, "prev_events"): - event.prev_events = [ - (event_id, hashes) - for event_id, hashes, _ in self.prev_events - ] - - if self.prev_events: - event.depth = max([int(v) for _, _, v in self.prev_events]) + 1 - else: - event.depth = 0 - - if not hasattr(event, "prev_state") and self.prev_state is not None: - event.prev_state = self.prev_state - def schema_path(schema): """ Get a filesystem path for the named database schema From f06161a3077723a47772cbefc08a4bf2f14b176b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 16 Dec 2014 16:10:17 +0000 Subject: [PATCH 154/158] Enable rate limiting for all events --- synapse/handlers/message.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index baf372fdad..7939d2bffa 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -154,6 +154,11 @@ class MessageHandler(BaseHandler): self.validator.validate_new(builder) + self.ratelimit(builder.user_id) + # TODO(paul): Why does 'event' not have a 'user' object? + user = self.hs.parse_userid(builder.user_id) + assert self.hs.is_mine(user), "User must be our own: %s" % (user,) + if builder.type == EventTypes.Member: membership = builder.content.get("membership", None) if membership == Membership.JOIN: From 4dcad143dd3fc48aceae792fcbe2f331b83776a3 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 16 Dec 2014 17:24:49 +0000 Subject: [PATCH 155/158] SYN-142: Use a default log file 'homeserver.log' so people get logging by default. --- synapse/config/logger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/config/logger.py b/synapse/config/logger.py index 089d906fa5..d352ea9be9 100644 --- a/synapse/config/logger.py +++ b/synapse/config/logger.py @@ -36,7 +36,7 @@ class LoggingConfig(Config): help="The verbosity level." ) logging_group.add_argument( - '-f', '--log-file', dest="log_file", default=None, + '-f', '--log-file', dest="log_file", default="homeserver.log", help="File to log to." ) logging_group.add_argument( From 28f71ecf0da6e162f1f3aeb03a80723f9ff1fdd6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 16 Dec 2014 17:29:22 +0000 Subject: [PATCH 156/158] Change upgrade script to not check hashes or signatures --- scripts/upgrade_db_to_v0.6.0.py | 99 ++++++++++++++++++--------------- 1 file changed, 53 insertions(+), 46 deletions(-) diff --git a/scripts/upgrade_db_to_v0.6.0.py b/scripts/upgrade_db_to_v0.6.0.py index 32c415a662..298e9c0789 100644 --- a/scripts/upgrade_db_to_v0.6.0.py +++ b/scripts/upgrade_db_to_v0.6.0.py @@ -21,7 +21,7 @@ from syutil.crypto.signing_key import decode_verify_key_bytes from syutil.jsonutil import encode_canonical_json import argparse -import dns.resolver +# import dns.resolver import hashlib import httplib import json @@ -147,44 +147,44 @@ class Store(object): store = Store() -def get_key(server_name): - print "Getting keys for: %s" % (server_name,) - targets = [] - if ":" in server_name: - target, port = server_name.split(":") - targets.append((target, int(port))) - try: - answers = dns.resolver.query("_matrix._tcp." + server_name, "SRV") - for srv in answers: - targets.append((srv.target, srv.port)) - except dns.resolver.NXDOMAIN: - targets.append((server_name, 8448)) - except: - print "Failed to lookup keys for %s" % (server_name,) - return {} - - for target, port in targets: - url = "https://%s:%i/_matrix/key/v1" % (target, port) - try: - keys = json.load(urllib2.urlopen(url, timeout=2)) - verify_keys = {} - for key_id, key_base64 in keys["verify_keys"].items(): - verify_key = decode_verify_key_bytes( - key_id, decode_base64(key_base64) - ) - verify_signed_json(keys, server_name, verify_key) - verify_keys[key_id] = verify_key - print "Got keys for: %s" % (server_name,) - return verify_keys - except urllib2.URLError: - pass - except urllib2.HTTPError: - pass - except httplib.HTTPException: - pass - - print "Failed to get keys for %s" % (server_name,) - return {} +# def get_key(server_name): +# print "Getting keys for: %s" % (server_name,) +# targets = [] +# if ":" in server_name: +# target, port = server_name.split(":") +# targets.append((target, int(port))) +# try: +# answers = dns.resolver.query("_matrix._tcp." + server_name, "SRV") +# for srv in answers: +# targets.append((srv.target, srv.port)) +# except dns.resolver.NXDOMAIN: +# targets.append((server_name, 8448)) +# except: +# print "Failed to lookup keys for %s" % (server_name,) +# return {} +# +# for target, port in targets: +# url = "https://%s:%i/_matrix/key/v1" % (target, port) +# try: +# keys = json.load(urllib2.urlopen(url, timeout=2)) +# verify_keys = {} +# for key_id, key_base64 in keys["verify_keys"].items(): +# verify_key = decode_verify_key_bytes( +# key_id, decode_base64(key_base64) +# ) +# verify_signed_json(keys, server_name, verify_key) +# verify_keys[key_id] = verify_key +# print "Got keys for: %s" % (server_name,) +# return verify_keys +# except urllib2.URLError: +# pass +# except urllib2.HTTPError: +# pass +# except httplib.HTTPException: +# pass +# +# print "Failed to get keys for %s" % (server_name,) +# return {} def reinsert_events(cursor, server_name, signing_key): @@ -219,13 +219,20 @@ def reinsert_events(cursor, server_name, signing_key): } } + i = 0 + N = len(events) + for event in events: - for alg_name in event.hashes: - if check_event_content_hash(event, algorithms[alg_name]): - pass - else: - pass - print "FAIL content hash %s %s" % (alg_name, event.event_id, ) + if i % 100 == 0: + print "Processed: %d/%d events" % (i,N,) + i += 1 + + # for alg_name in event.hashes: + # if check_event_content_hash(event, algorithms[alg_name]): + # pass + # else: + # pass + # print "FAIL content hash %s %s" % (alg_name, event.event_id, ) have_own_correctly_signed = False for host, sigs in event.signatures.items(): @@ -233,7 +240,7 @@ def reinsert_events(cursor, server_name, signing_key): for key_id in sigs: if host not in server_keys: - server_keys[host] = get_key(host) + server_keys[host] = {} # get_key(host) if key_id in server_keys[host]: try: verify_signed_json( From f8cc8a66b406d6c36d8a06b722738ea19b3e967e Mon Sep 17 00:00:00 2001 From: Kegsay Date: Thu, 18 Dec 2014 14:16:31 +0000 Subject: [PATCH 157/158] Update README.rst Add windows (cygwin) install instructions. --- README.rst | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/README.rst b/README.rst index 8459bcac2e..7497f1584b 100644 --- a/README.rst +++ b/README.rst @@ -133,6 +133,28 @@ failing, e.g.:: On OSX, if you encounter clang: error: unknown argument: '-mno-fused-madd' you will need to export CFLAGS=-Qunused-arguments. +Windows Install +--------------- +Synapse can be installed on Cygwin. It requires the following Cygwin packages: + + - gcc + - git + - libffi-devel + - openssl (and openssl-devel, python-openssl) + - python + - python-setuptools + +Troubleshooting: + +- You may need to upgrade ``setuptools`` to get this to work correctly: + ``pip install setuptools --upgrade``. +- You may encounter errors indicating that ``ffi.h`` is missing, even with + ``libffi-devel`` installed. If you do, copy the ``.h`` files: + ``cp /usr/lib/libffi-3.0.13/include/*.h /usr/include`` +- You may need to install libsodium from source in order to install PyNacl. If + you do, you may need to create a symlink to ``libsodium.a`` so ``ld`` can find + it: ``ln -s /usr/local/lib/libsodium.a /usr/lib/libsodium.a`` + Running Your Homeserver ======================= From 20923ffd43cc49b3f532012d9465c07a707534e3 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Thu, 18 Dec 2014 14:44:48 +0000 Subject: [PATCH 158/158] Update README.rst Add gotcha: The content repository requires additional cygwin packages. --- README.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.rst b/README.rst index 7497f1584b..f5d2b0af38 100644 --- a/README.rst +++ b/README.rst @@ -143,6 +143,15 @@ Synapse can be installed on Cygwin. It requires the following Cygwin packages: - openssl (and openssl-devel, python-openssl) - python - python-setuptools + +The content repository requires additional packages and will be unable to process +uploads without them: + - libjpeg8 + - libjpeg8-devel + - zlib +If you choose to install Synapse without these packages, you will need to reinstall +``pillow`` for changes to be applied, e.g. ``pip uninstall pillow`` ``pip install +pillow --user`` Troubleshooting: