From cc0532a4bf328478d508242d830433a33c4f22b2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 6 Feb 2015 15:16:26 +0000 Subject: [PATCH 01/22] Explicitly list the RejectedReasons that we can prove --- synapse/handlers/federation.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index b13b7c7701..0f9c82fd06 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -860,9 +860,14 @@ class FederationHandler(BaseHandler): # Only do auth resolution if we have something new to say. # We can't rove an auth failure. do_resolution = False + + provable = [ + RejectedReason.NOT_ANCESTOR, RejectedReason.NOT_ANCESTOR, + ] + for e_id in different_auth: if e_id in have_events: - if have_events[e_id] != RejectedReason.AUTH_ERROR: + if have_events[e_id] in provable: do_resolution = True break From 20db147ef377cefb2b5cc138c2bd95291e26f3cc Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 6 Feb 2015 15:58:40 +0000 Subject: [PATCH 02/22] SYN-258: get_recent_events_for_room only accepts stream tokens, convert the topological token to a stream token before passing it to get_recent_events_for_room --- synapse/handlers/sync.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 962686f4bb..439164ae39 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -298,15 +298,17 @@ class SyncHandler(BaseHandler): load_limit = max(sync_config.limit * filtering_factor, 100) max_repeat = 3 # Only try a few times per room, otherwise room_key = now_token.room_key + end_key = room_key while limited and len(recents) < sync_config.limit and max_repeat: events, keys = yield self.store.get_recent_events_for_room( room_id, limit=load_limit + 1, from_token=since_token.room_key if since_token else None, - end_token=room_key, + end_token=end_key, ) (room_key, _) = keys + end_key = "s" + room_key.split('-')[-1] loaded_recents = sync_config.filter.filter_room_events(events) loaded_recents.extend(recents) recents = loaded_recents From a578251b4800f20424e8b294a42cc6c65ef568a2 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Mon, 9 Feb 2015 16:44:47 +0000 Subject: [PATCH 03/22] only do word-boundary patches on bodies for now --- synapse/push/__init__.py | 5 ++++- synapse/python_dependencies.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 07b5f0187c..58c8cf700b 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -147,7 +147,10 @@ class Pusher(object): logger.warn("event_match condition with no pattern") return False # XXX: optimisation: cache our pattern regexps - r = r'\b%s\b' % self._glob_to_regexp(condition['pattern']) + if condition['key'] == 'content.body': + r = r'\b%s\b' % self._glob_to_regexp(condition['pattern']) + else: + r = r'^%s$' % self._glob_to_regexp(condition['pattern']) val = _value_for_dotted_key(condition['key'], ev) if val is None: return False diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index a89d618606..f429d1beda 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -6,7 +6,7 @@ logger = logging.getLogger(__name__) REQUIREMENTS = { "syutil==0.0.2": ["syutil"], "matrix_angular_sdk==0.6.0": ["syweb>=0.6.0"], - "Twisted==14.0.2": ["twisted==14.0.2"], + "Twisted==14.0.0": ["twisted==14.0.0"], "service_identity>=1.0.0": ["service_identity>=1.0.0"], "pyopenssl>=0.14": ["OpenSSL>=0.14"], "pyyaml": ["yaml"], From bd2373277de8fd571d8eb0af38b107065e567a02 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Mon, 9 Feb 2015 16:48:09 +0000 Subject: [PATCH 04/22] oops --- synapse/python_dependencies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index f429d1beda..44d1a0404d 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -6,7 +6,7 @@ logger = logging.getLogger(__name__) REQUIREMENTS = { "syutil==0.0.2": ["syutil"], "matrix_angular_sdk==0.6.0": ["syweb>=0.6.0"], - "Twisted==14.0.0": ["twisted==14.0.0"], + "Twisted==14.0.0": ["twisted==14.0.2"], "service_identity>=1.0.0": ["service_identity>=1.0.0"], "pyopenssl>=0.14": ["OpenSSL>=0.14"], "pyyaml": ["yaml"], From 0b725f5c4f8f0550205c09bbdc775f6ad613a1fd Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Mon, 9 Feb 2015 16:48:31 +0000 Subject: [PATCH 05/22] oops --- synapse/python_dependencies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 44d1a0404d..a89d618606 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -6,7 +6,7 @@ logger = logging.getLogger(__name__) REQUIREMENTS = { "syutil==0.0.2": ["syutil"], "matrix_angular_sdk==0.6.0": ["syweb>=0.6.0"], - "Twisted==14.0.0": ["twisted==14.0.2"], + "Twisted==14.0.2": ["twisted==14.0.2"], "service_identity>=1.0.0": ["service_identity>=1.0.0"], "pyopenssl>=0.14": ["OpenSSL>=0.14"], "pyyaml": ["yaml"], From 8f616684a3a712ccf06349c67bb64779f06d8da1 Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 9 Feb 2015 17:01:40 +0000 Subject: [PATCH 06/22] Need to use re.search if looking for matches not at the start of the string. Also comparisons with None should be 'is'. --- synapse/push/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 58c8cf700b..6f143a5df9 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -154,7 +154,7 @@ class Pusher(object): val = _value_for_dotted_key(condition['key'], ev) if val is None: return False - return re.match(r, val, flags=re.IGNORECASE) != None + return re.search(r, val, flags=re.IGNORECASE) is not None elif condition['kind'] == 'device': if 'profile_tag' not in condition: @@ -170,8 +170,8 @@ class Pusher(object): return False if not display_name: return False - return re.match("\b%s\b" % re.escape(display_name), - ev['content']['body'], flags=re.IGNORECASE) != None + return re.search("\b%s\b" % re.escape(display_name), + ev['content']['body'], flags=re.IGNORECASE) is not None elif condition['kind'] == 'room_member_count': if 'is' not in condition: From d94f682a4c3e7bab5079d516582f0ee44a3d3f06 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 9 Feb 2015 17:41:29 +0000 Subject: [PATCH 07/22] During room intial sync, only calculate current state once. --- synapse/api/auth.py | 21 ++++++++++++++------- synapse/handlers/message.py | 32 ++++++++++++++++++++++---------- synapse/handlers/sync.py | 9 ++++++--- synapse/state.py | 2 +- 4 files changed, 43 insertions(+), 21 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 7105ee21dc..0054745363 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -89,12 +89,19 @@ class Auth(object): raise @defer.inlineCallbacks - def check_joined_room(self, room_id, user_id): - member = yield self.state.get_current_state( - room_id=room_id, - event_type=EventTypes.Member, - state_key=user_id - ) + def check_joined_room(self, room_id, user_id, current_state=None): + if current_state: + member = current_state.get( + (EventTypes.Member, user_id), + None + ) + else: + member = yield self.state.get_current_state( + room_id=room_id, + event_type=EventTypes.Member, + state_key=user_id + ) + self._check_joined_room(member, user_id, room_id) defer.returnValue(member) @@ -102,7 +109,7 @@ class Auth(object): def check_host_in_room(self, room_id, host): curr_state = yield self.state.get_current_state(room_id) - for event in curr_state: + for event in curr_state.values(): if event.type == EventTypes.Member: try: if UserID.from_string(event.state_key).domain != host: diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 3f51f38f18..3355adefcf 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -35,6 +35,7 @@ class MessageHandler(BaseHandler): def __init__(self, hs): super(MessageHandler, self).__init__(hs) self.hs = hs + self.state = hs.get_state_handler() self.clock = hs.get_clock() self.validator = EventValidator() @@ -225,7 +226,9 @@ class MessageHandler(BaseHandler): # TODO: This is duplicating logic from snapshot_all_rooms current_state = yield self.state_handler.get_current_state(room_id) now = self.clock.time_msec() - defer.returnValue([serialize_event(c, now) for c in current_state]) + defer.returnValue( + [serialize_event(c, now) for c in current_state.values()] + ) @defer.inlineCallbacks def snapshot_all_rooms(self, user_id=None, pagin_config=None, @@ -313,7 +316,7 @@ class MessageHandler(BaseHandler): ) d["state"] = [ serialize_event(c, time_now, as_client_event) - for c in current_state + for c in current_state.values() ] except: logger.exception("Failed to get snapshot") @@ -329,7 +332,14 @@ class MessageHandler(BaseHandler): @defer.inlineCallbacks def room_initial_sync(self, user_id, room_id, pagin_config=None, feedback=False): - yield self.auth.check_joined_room(room_id, user_id) + current_state = yield self.state.get_current_state( + room_id=room_id, + ) + + yield self.auth.check_joined_room( + room_id, user_id, + current_state=current_state + ) # TODO(paul): I wish I was called with user objects not user_id # strings... @@ -337,13 +347,12 @@ class MessageHandler(BaseHandler): # TODO: These concurrently time_now = self.clock.time_msec() - state_tuples = yield self.state_handler.get_current_state(room_id) - state = [serialize_event(x, time_now) for x in state_tuples] + state = [ + serialize_event(x, time_now) + for x in current_state.values() + ] - member_event = (yield self.store.get_room_member( - user_id=user_id, - room_id=room_id - )) + member_event = current_state.get((EventTypes.Member, user_id,)) now_token = yield self.hs.get_event_sources().get_current_token() @@ -360,7 +369,10 @@ class MessageHandler(BaseHandler): start_token = now_token.copy_and_replace("room_key", token[0]) end_token = now_token.copy_and_replace("room_key", token[1]) - room_members = yield self.store.get_room_members(room_id) + room_members = [ + m for m in current_state.values() + if m.type == EventTypes.Member + ] presence_handler = self.hs.get_handlers().presence_handler presence = [] diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 439164ae39..5af90cc5d1 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -175,9 +175,10 @@ class SyncHandler(BaseHandler): room_id, sync_config, now_token, ) - current_state_events = yield self.state_handler.get_current_state( + current_state = yield self.state_handler.get_current_state( room_id ) + current_state_events = current_state.values() defer.returnValue(RoomSyncResult( room_id=room_id, @@ -347,9 +348,10 @@ class SyncHandler(BaseHandler): # TODO(mjark): This seems racy since this isn't being passed a # token to indicate what point in the stream this is - current_state_events = yield self.state_handler.get_current_state( + current_state = yield self.state_handler.get_current_state( room_id ) + current_state_events = current_state.values() state_at_previous_sync = yield self.get_state_at_previous_sync( room_id, since_token=since_token @@ -431,6 +433,7 @@ class SyncHandler(BaseHandler): joined = True if joined: - state_delta = yield self.state_handler.get_current_state(room_id) + res = yield self.state_handler.get_current_state(room_id) + state_delta = res.values() defer.returnValue(state_delta) diff --git a/synapse/state.py b/synapse/state.py index 695a5e7ac4..54380b9e5c 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -76,7 +76,7 @@ class StateHandler(object): defer.returnValue(res[1].get((event_type, state_key))) return - defer.returnValue(res[1].values()) + defer.returnValue(res[1]) @defer.inlineCallbacks def compute_event_context(self, event, old_state=None): From 3a5ad7dbd5a375023c96ee65c901f8be5ab02341 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 9 Feb 2015 17:55:56 +0000 Subject: [PATCH 08/22] Performance counters for database transaction names --- synapse/storage/_base.py | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 310ee0104c..bcb03cbdcb 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -39,6 +39,7 @@ class LoggingTransaction(object): passed to the constructor. Adds logging to the .execute() method.""" __slots__ = ["txn", "name"] + def __init__(self, txn, name): object.__setattr__(self, "txn", txn) object.__setattr__(self, "name", name) @@ -88,6 +89,8 @@ class SQLBaseStore(object): self._previous_txn_total_time = 0 self._current_txn_total_time = 0 self._previous_loop_ts = 0 + self._txn_perf_counters = {} + self._previous_txn_perf_counters = {} def start_profiling(self): self._previous_loop_ts = self._clock.time_msec() @@ -103,7 +106,29 @@ class SQLBaseStore(object): ratio = (curr - prev)/(time_now - time_then) - logger.info("Total database time: %.3f%%", ratio * 100) + txn_counters = [] + for name, (count, cum_time) in self._txn_perf_counters.items(): + prev_count, prev_time = self._previous_txn_perf_counters.get( + name, (0,0) + ) + txn_counters.append(( + (cum_time - prev_time) / (time_now - time_then), + count - prev_count, + name + )) + + self._previous_txn_perf_counters = dict(self._txn_perf_counters) + + txn_counters.sort(reverse=True) + top_three_counters = ", ".join( + "%s(%d): %.3f%%" % (name, count, 100 * ratio) + for ratio, count, name in txn_counters[:3] + ) + + logger.info( + "Total database time: %.3f%% {%s}", + ratio * 100, top_three_counters + ) self._clock.looping_call(loop, 10000) @@ -139,6 +164,11 @@ class SQLBaseStore(object): self._current_txn_total_time += end - start + count, cum_time = self._txn_perf_counters.get(name, (0,0)) + count += 1 + cum_time += end - start + self._txn_perf_counters[name] = (count, cum_time) + with PreserveLoggingContext(): result = yield self._db_pool.runInteraction( inner_func, *args, **kwargs From 347b497db0355fe4e26ae3a51967aa91bec090d3 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 9 Feb 2015 17:57:09 +0000 Subject: [PATCH 09/22] Formatting --- synapse/storage/_base.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index bcb03cbdcb..45f4b994eb 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -39,7 +39,6 @@ class LoggingTransaction(object): passed to the constructor. Adds logging to the .execute() method.""" __slots__ = ["txn", "name"] - def __init__(self, txn, name): object.__setattr__(self, "txn", txn) object.__setattr__(self, "name", name) From 0c4536da8fe75a207052fb558414b4408aa857ec Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 9 Feb 2015 18:06:31 +0000 Subject: [PATCH 10/22] Use the transaction 'desc' rather than 'name', increment the txn_ids in txn names --- synapse/storage/_base.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 45f4b994eb..5ddd410607 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -140,7 +140,7 @@ class SQLBaseStore(object): with LoggingContext("runInteraction") as context: current_context.copy_to(context) start = time.time() * 1000 - txn_id = SQLBaseStore._TXN_ID + txn_id = self._TXN_ID # We don't really need these to be unique, so lets stop it from # growing really large. @@ -163,10 +163,10 @@ class SQLBaseStore(object): self._current_txn_total_time += end - start - count, cum_time = self._txn_perf_counters.get(name, (0,0)) + count, cum_time = self._txn_perf_counters.get(desc, (0,0)) count += 1 cum_time += end - start - self._txn_perf_counters[name] = (count, cum_time) + self._txn_perf_counters[desc] = (count, cum_time) with PreserveLoggingContext(): result = yield self._db_pool.runInteraction( From 8ce100c7b48a38e5f74ed6ac560de1f50c288379 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 9 Feb 2015 18:29:36 +0000 Subject: [PATCH 11/22] Convert directory paths to absolute paths before daemonizing --- synapse/config/_base.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/synapse/config/_base.py b/synapse/config/_base.py index dfc115d8e8..9b0f8c3c32 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -50,8 +50,9 @@ class Config(object): ) return cls.abspath(file_path) - @staticmethod - def ensure_directory(dir_path): + @classmethod + def ensure_directory(cls, dir_path): + dir_path = cls.abspath(dir_path) if not os.path.exists(dir_path): os.makedirs(dir_path) if not os.path.isdir(dir_path): From 30595b466f1f6b61eef15dc41562f6336e57e6fe Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 10 Feb 2015 13:50:33 +0000 Subject: [PATCH 12/22] Use yaml logging config format because it is much nicer --- synapse/config/logger.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/config/logger.py b/synapse/config/logger.py index f9568ebd21..b18d08ec61 100644 --- a/synapse/config/logger.py +++ b/synapse/config/logger.py @@ -18,6 +18,7 @@ from synapse.util.logcontext import LoggingContextFilter from twisted.python.log import PythonLoggingObserver import logging import logging.config +import yaml class LoggingConfig(Config): @@ -79,7 +80,7 @@ class LoggingConfig(Config): logger.addHandler(handler) logger.info("Test") else: - logging.config.fileConfig(self.log_config) + logging.config.dictConfig(yaml.load(self.log_config)) observer = PythonLoggingObserver() observer.start() From f91345bdb5b76ce76b99d4fc969ee4f41cbbba4c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 10 Feb 2015 13:57:31 +0000 Subject: [PATCH 13/22] yaml.load expects strings to be a yaml rather than file --- synapse/config/logger.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/config/logger.py b/synapse/config/logger.py index b18d08ec61..63c8e36930 100644 --- a/synapse/config/logger.py +++ b/synapse/config/logger.py @@ -80,7 +80,8 @@ class LoggingConfig(Config): logger.addHandler(handler) logger.info("Test") else: - logging.config.dictConfig(yaml.load(self.log_config)) + with open(self.log_config, 'r') as f: + logging.config.dictConfig(yaml.load(f)) observer = PythonLoggingObserver() observer.start() From d7c7efb691bd726ec3e8879e289546fbcfd7dabd Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 10 Feb 2015 14:50:53 +0000 Subject: [PATCH 14/22] Add performance counters for different stages of loading events --- synapse/storage/_base.py | 82 +++++++++++++++++++++++++++++----------- 1 file changed, 59 insertions(+), 23 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 5ddd410607..c79399fe5e 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -77,6 +77,43 @@ class LoggingTransaction(object): sql_logger.debug("[SQL time] {%s} %f", self.name, end - start) +class PerformanceCounters(object): + def __init__(self): + self.current_counters = {} + self.previous_counters = {} + + def update(self, key, start_time, end_time=None): + if end_time is None: + end_time = time.time() * 1000; + duration = end_time - start_time + count, cum_time = self.current_counters.get(key, (0, 0)) + count += 1 + cum_time += duration + self.current_counters[key] = (count, cum_time) + return end_time + + def interval(self, interval_duration, limit=3): + counters = [] + for name, (count, cum_time) in self.current_counters.items(): + prev_count, prev_time = self.previous_counters.get(name, (0, 0)) + counters.append(( + (cum_time - prev_time) / interval_duration, + count - prev_count, + name + )) + + self.previous_counters = dict(self.current_counters) + + counters.sort(reverse=True) + + top_n_counters = ", ".join( + "%s(%d): %.3f%%" % (name, count, 100 * ratio) + for ratio, count, name in txn_counters[:limit] + ) + + return top_n_counters + + class SQLBaseStore(object): _TXN_ID = 0 @@ -88,8 +125,8 @@ class SQLBaseStore(object): self._previous_txn_total_time = 0 self._current_txn_total_time = 0 self._previous_loop_ts = 0 - self._txn_perf_counters = {} - self._previous_txn_perf_counters = {} + self._txn_perf_counters = PerformanceCounters() + self._get_event_counters = PerformanceCounters() def start_profiling(self): self._previous_loop_ts = self._clock.time_msec() @@ -105,23 +142,12 @@ class SQLBaseStore(object): ratio = (curr - prev)/(time_now - time_then) - txn_counters = [] - for name, (count, cum_time) in self._txn_perf_counters.items(): - prev_count, prev_time = self._previous_txn_perf_counters.get( - name, (0,0) - ) - txn_counters.append(( - (cum_time - prev_time) / (time_now - time_then), - count - prev_count, - name - )) + top_three_counters = self._txn_perf_counters.interval( + time_now - time_then, limit=3 + ) - self._previous_txn_perf_counters = dict(self._txn_perf_counters) - - txn_counters.sort(reverse=True) - top_three_counters = ", ".join( - "%s(%d): %.3f%%" % (name, count, 100 * ratio) - for ratio, count, name in txn_counters[:3] + top_3_event_counters = self._get_event_counters.interval( + time_now - time_then, limit=3 ) logger.info( @@ -162,11 +188,7 @@ class SQLBaseStore(object): ) self._current_txn_total_time += end - start - - count, cum_time = self._txn_perf_counters.get(desc, (0,0)) - count += 1 - cum_time += end - start - self._txn_perf_counters[desc] = (count, cum_time) + self._txn_perf_counters.update(desc, start, end) with PreserveLoggingContext(): result = yield self._db_pool.runInteraction( @@ -566,6 +588,8 @@ class SQLBaseStore(object): "LIMIT 1 " ) + start_time = time.time() * 1000; + txn.execute(sql, (event_id,)) res = txn.fetchone() @@ -575,6 +599,8 @@ class SQLBaseStore(object): internal_metadata, js, redacted, rejected_reason = res + self._get_event_counters.update("select_event", start_time) + if allow_rejected or not rejected_reason: return self._get_event_from_row_txn( txn, internal_metadata, js, redacted, @@ -586,10 +612,18 @@ class SQLBaseStore(object): def _get_event_from_row_txn(self, txn, internal_metadata, js, redacted, check_redacted=True, get_prev_content=False): + + start_time = time.time() * 1000; + update_counter = self._get_event_counters.update + d = json.loads(js) + start_time = update_counter("decode_json", start_time) + internal_metadata = json.loads(internal_metadata) + start_time = update_counter("decode_internal", start_time) ev = FrozenEvent(d, internal_metadata_dict=internal_metadata) + start_time = update_counter("build_frozen_event", start_time) if check_redacted and redacted: ev = prune_event(ev) @@ -605,6 +639,7 @@ class SQLBaseStore(object): if because: ev.unsigned["redacted_because"] = because + start_time = update_counter("redact_event", start_time) if get_prev_content and "replaces_state" in ev.unsigned: prev = self._get_event_txn( @@ -614,6 +649,7 @@ class SQLBaseStore(object): ) if prev: ev.unsigned["prev_content"] = prev.get_dict()["content"] + start_time = update_counter("get_prev_content", start_time) return ev From fda4422bc9d9f2974d7185011d6d905eea372b09 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 10 Feb 2015 14:54:07 +0000 Subject: [PATCH 15/22] Fix pyflakes --- synapse/storage/_base.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index c79399fe5e..36455ef93c 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -108,7 +108,7 @@ class PerformanceCounters(object): top_n_counters = ", ".join( "%s(%d): %.3f%%" % (name, count, 100 * ratio) - for ratio, count, name in txn_counters[:limit] + for ratio, count, name in counters[:limit] ) return top_n_counters @@ -151,8 +151,8 @@ class SQLBaseStore(object): ) logger.info( - "Total database time: %.3f%% {%s}", - ratio * 100, top_three_counters + "Total database time: %.3f%% {%s} {%s}", + ratio * 100, top_three_counters, top_3_event_counters ) self._clock.looping_call(loop, 10000) From 697ab75a34b14da6d872d153f36017a6dc6d5b99 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 10 Feb 2015 15:46:24 +0000 Subject: [PATCH 16/22] Sign auth_chains when returned by /state/ requests --- synapse/federation/federation_server.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 4742ca9390..b23f72c7fa 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -25,6 +25,8 @@ from synapse.events import FrozenEvent from synapse.api.errors import FederationError, SynapseError +from synapse.crypto.event_signing import compute_event_signature + import logging @@ -156,6 +158,15 @@ class FederationServer(FederationBase): auth_chain = yield self.store.get_auth_chain( [pdu.event_id for pdu in pdus] ) + + for event in auth_chain: + event.signatures.update( + compute_event_signature( + event, + self.hs.hostname, + self.hs.config.signing_key[0] + ) + ) else: raise NotImplementedError("Specify an event") From b085fac7353e1cd395b89f9334c8273a8e996f48 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 10 Feb 2015 16:30:48 +0000 Subject: [PATCH 17/22] Code-style fixes --- synapse/handlers/presence.py | 4 +++- synapse/handlers/register.py | 13 ++++++++----- synapse/push/__init__.py | 8 +++++--- synapse/push/baserules.py | 11 ++++++----- synapse/push/rulekinds.py | 12 ++++++------ synapse/python_dependencies.py | 6 ++++-- synapse/rest/client/v1/push_rule.py | 18 +++++++++++++----- synapse/rest/client/v1/pusher.py | 4 ++-- synapse/rest/media/v1/upload_resource.py | 7 ++++--- synapse/storage/_base.py | 6 +++--- synapse/storage/push_rule.py | 4 +++- 11 files changed, 57 insertions(+), 36 deletions(-) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index cd0798c2b0..6a266ee0fe 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -658,7 +658,9 @@ class PresenceHandler(BaseHandler): observers = set(self._remote_recvmap.get(user, set())) if observers: - logger.debug(" | %d interested local observers %r", len(observers), observers) + logger.debug( + " | %d interested local observers %r", len(observers), observers + ) rm_handler = self.homeserver.get_handlers().room_member_handler room_ids = yield rm_handler.get_rooms_for_user(user) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 4f06c487b1..0247327eb9 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -105,17 +105,20 @@ class RegistrationHandler(BaseHandler): # do it here. try: auth_user = UserID.from_string(user_id) - identicon_resource = self.hs.get_resource_for_media_repository().getChildWithDefault("identicon", None) - upload_resource = self.hs.get_resource_for_media_repository().getChildWithDefault("upload", None) + media_repository = self.hs.get_resource_for_media_repository() + identicon_resource = media_repository.getChildWithDefault("identicon", None) + upload_resource = media_repository.getChildWithDefault("upload", None) identicon_bytes = identicon_resource.generate_identicon(user_id, 320, 320) content_uri = yield upload_resource.create_content( "image/png", None, identicon_bytes, len(identicon_bytes), auth_user ) profile_handler = self.hs.get_handlers().profile_handler - profile_handler.set_avatar_url(auth_user, auth_user, ("%s#auto" % content_uri)) + profile_handler.set_avatar_url( + auth_user, auth_user, ("%s#auto" % (content_uri,)) + ) except NotImplementedError: - pass # make tests pass without messing around creating default avatars - + pass # make tests pass without messing around creating default avatars + defer.returnValue((user_id, token)) @defer.inlineCallbacks diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 6f143a5df9..418a348a58 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -140,7 +140,7 @@ class Pusher(object): lambda x: ('[%s%s]' % (x.group(1) and '^' or '', re.sub(r'\\\-', '-', x.group(2)))), r) return r - + def _event_fulfills_condition(self, ev, condition, display_name, room_member_count): if condition['kind'] == 'event_match': if 'pattern' not in condition: @@ -170,8 +170,10 @@ class Pusher(object): return False if not display_name: return False - return re.search("\b%s\b" % re.escape(display_name), - ev['content']['body'], flags=re.IGNORECASE) is not None + return re.search( + "\b%s\b" % re.escape(display_name), ev['content']['body'], + flags=re.IGNORECASE + ) is not None elif condition['kind'] == 'room_member_count': if 'is' not in condition: diff --git a/synapse/push/baserules.py b/synapse/push/baserules.py index 37878f1e0b..162d265f66 100644 --- a/synapse/push/baserules.py +++ b/synapse/push/baserules.py @@ -1,5 +1,6 @@ from synapse.push.rulekinds import PRIORITY_CLASS_MAP, PRIORITY_CLASS_INVERSE_MAP + def list_with_base_rules(rawrules, user_name): ruleslist = [] @@ -9,9 +10,9 @@ def list_with_base_rules(rawrules, user_name): if r['priority_class'] < current_prio_class: while r['priority_class'] < current_prio_class: ruleslist.extend(make_base_rules( - user_name, - PRIORITY_CLASS_INVERSE_MAP[current_prio_class]) - ) + user_name, + PRIORITY_CLASS_INVERSE_MAP[current_prio_class] + )) current_prio_class -= 1 ruleslist.append(r) @@ -19,8 +20,8 @@ def list_with_base_rules(rawrules, user_name): while current_prio_class > 0: ruleslist.extend(make_base_rules( user_name, - PRIORITY_CLASS_INVERSE_MAP[current_prio_class]) - ) + PRIORITY_CLASS_INVERSE_MAP[current_prio_class] + )) current_prio_class -= 1 return ruleslist diff --git a/synapse/push/rulekinds.py b/synapse/push/rulekinds.py index 763bdee58e..660aa4e10e 100644 --- a/synapse/push/rulekinds.py +++ b/synapse/push/rulekinds.py @@ -1,8 +1,8 @@ PRIORITY_CLASS_MAP = { - 'underride': 1, - 'sender': 2, - 'room': 3, - 'content': 4, - 'override': 5, - } + 'underride': 1, + 'sender': 2, + 'room': 3, + 'content': 4, + 'override': 5, +} PRIORITY_CLASS_INVERSE_MAP = {v: k for k, v in PRIORITY_CLASS_MAP.items()} diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index a89d618606..fd68da9dfb 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -19,10 +19,11 @@ REQUIREMENTS = { "pydenticon": ["pydenticon"], } + def github_link(project, version, egg): return "https://github.com/%s/tarball/%s/#egg=%s" % (project, version, egg) -DEPENDENCY_LINKS=[ +DEPENDENCY_LINKS = [ github_link( project="matrix-org/syutil", version="v0.0.2", @@ -101,6 +102,7 @@ def check_requirements(): % (dependency, file_path, version, required_version) ) + def list_requirements(): result = [] linked = [] @@ -111,7 +113,7 @@ def list_requirements(): for requirement in REQUIREMENTS: is_linked = False for link in linked: - if requirement.replace('-','_').startswith(link): + if requirement.replace('-', '_').startswith(link): is_linked = True if not is_linked: result.append(requirement) diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index d43ade39dd..c4e7dfcf0e 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -15,12 +15,17 @@ from twisted.internet import defer -from synapse.api.errors import SynapseError, Codes, UnrecognizedRequestError, NotFoundError, \ - StoreError +from synapse.api.errors import ( + SynapseError, Codes, UnrecognizedRequestError, NotFoundError, StoreError +) from .base import ClientV1RestServlet, client_path_pattern -from synapse.storage.push_rule import InconsistentRuleException, RuleNotFoundException +from synapse.storage.push_rule import ( + InconsistentRuleException, RuleNotFoundException +) import synapse.push.baserules as baserules -from synapse.push.rulekinds import PRIORITY_CLASS_MAP, PRIORITY_CLASS_INVERSE_MAP +from synapse.push.rulekinds import ( + PRIORITY_CLASS_MAP, PRIORITY_CLASS_INVERSE_MAP +) import json @@ -105,7 +110,9 @@ class PushRuleRestServlet(ClientV1RestServlet): # we build up the full structure and then decide which bits of it # to send which means doing unnecessary work sometimes but is # is probably not going to make a whole lot of difference - rawrules = yield self.hs.get_datastore().get_push_rules_for_user_name(user.to_string()) + rawrules = yield self.hs.get_datastore().get_push_rules_for_user_name( + user.to_string() + ) for r in rawrules: r["conditions"] = json.loads(r["conditions"]) @@ -383,6 +390,7 @@ def _namespaced_rule_id_from_spec(spec): def _rule_id_from_namespaced(in_rule_id): return in_rule_id.split('/')[-1] + class InvalidRuleException(Exception): pass diff --git a/synapse/rest/client/v1/pusher.py b/synapse/rest/client/v1/pusher.py index e10d2576d2..80e9939b79 100644 --- a/synapse/rest/client/v1/pusher.py +++ b/synapse/rest/client/v1/pusher.py @@ -34,8 +34,8 @@ class PusherRestServlet(ClientV1RestServlet): pusher_pool = self.hs.get_pusherpool() if ('pushkey' in content and 'app_id' in content - and 'kind' in content and - content['kind'] is None): + and 'kind' in content and + content['kind'] is None): yield pusher_pool.remove_pusher( content['app_id'], content['pushkey'] ) diff --git a/synapse/rest/media/v1/upload_resource.py b/synapse/rest/media/v1/upload_resource.py index 5b42782331..6df52ca434 100644 --- a/synapse/rest/media/v1/upload_resource.py +++ b/synapse/rest/media/v1/upload_resource.py @@ -38,9 +38,10 @@ class UploadResource(BaseMediaResource): def render_OPTIONS(self, request): respond_with_json(request, 200, {}, send_cors=True) return NOT_DONE_YET - + @defer.inlineCallbacks - def create_content(self, media_type, upload_name, content, content_length, auth_user): + def create_content(self, media_type, upload_name, content, content_length, + auth_user): media_id = random_string(24) fname = self.filepaths.local_media_filepath(media_id) @@ -65,7 +66,7 @@ class UploadResource(BaseMediaResource): } yield self._generate_local_thumbnails(media_id, media_info) - + defer.returnValue("mxc://%s/%s" % (self.server_name, media_id)) @defer.inlineCallbacks diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 36455ef93c..3e1ab0a159 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -84,7 +84,7 @@ class PerformanceCounters(object): def update(self, key, start_time, end_time=None): if end_time is None: - end_time = time.time() * 1000; + end_time = time.time() * 1000 duration = end_time - start_time count, cum_time = self.current_counters.get(key, (0, 0)) count += 1 @@ -588,7 +588,7 @@ class SQLBaseStore(object): "LIMIT 1 " ) - start_time = time.time() * 1000; + start_time = time.time() * 1000 txn.execute(sql, (event_id,)) @@ -613,7 +613,7 @@ class SQLBaseStore(object): def _get_event_from_row_txn(self, txn, internal_metadata, js, redacted, check_redacted=True, get_prev_content=False): - start_time = time.time() * 1000; + start_time = time.time() * 1000 update_counter = self._get_event_counters.update d = json.loads(js) diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py index 30e23445d9..620de71398 100644 --- a/synapse/storage/push_rule.py +++ b/synapse/storage/push_rule.py @@ -91,7 +91,9 @@ class PushRuleStore(SQLBaseStore): txn.execute(sql, (user_name, relative_to_rule)) res = txn.fetchall() if not res: - raise RuleNotFoundException("before/after rule not found: %s" % (relative_to_rule)) + raise RuleNotFoundException( + "before/after rule not found: %s" % (relative_to_rule,) + ) priority_class, base_rule_priority = res[0] if 'priority_class' in kwargs and kwargs['priority_class'] != priority_class: From c8e1da930dd4ffbc4ca409208b2d715182cd100f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 10 Feb 2015 17:30:46 +0000 Subject: [PATCH 18/22] Log all the exits from _attempt_new_transaction --- synapse/federation/transaction_queue.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py index f38aeba7cc..731019ad9f 100644 --- a/synapse/federation/transaction_queue.py +++ b/synapse/federation/transaction_queue.py @@ -157,15 +157,19 @@ class TransactionQueue(object): else: logger.info("TX [%s] is ready for retry", destination) - logger.info("TX [%s] _attempt_new_transaction", destination) - if destination in self.pending_transactions: # XXX: pending_transactions can get stuck on by a never-ending # request at which point pending_pdus_by_dest just keeps growing. # we need application-layer timeouts of some flavour of these # requests + logger.info( + "TX [%s] Transaction already in progress", + destination + ) return + logger.info("TX [%s] _attempt_new_transaction", destination) + # list of (pending_pdu, deferred, order) pending_pdus = self.pending_pdus_by_dest.pop(destination, []) pending_edus = self.pending_edus_by_dest.pop(destination, []) @@ -176,6 +180,7 @@ class TransactionQueue(object): destination, len(pending_pdus)) if not pending_pdus and not pending_edus and not pending_failures: + logger.info("TX [%s] Nothing to send", destination) return logger.debug( From 7ed971d9b206a70f44c55588bdb1a4a33906fbe1 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 10 Feb 2015 17:42:36 +0000 Subject: [PATCH 19/22] Single source version and python dependencies, prevent people accidentally installing with easy_install, use scripts rather than entry_points to install synctl --- VERSION | 1 - setup.py | 80 +++++++++++++++++----------------- synapse/python_dependencies.py | 6 +-- 3 files changed, 43 insertions(+), 44 deletions(-) delete mode 100644 VERSION diff --git a/VERSION b/VERSION deleted file mode 100644 index 1c29ff4d36..0000000000 --- a/VERSION +++ /dev/null @@ -1 +0,0 @@ -0.6.1d diff --git a/setup.py b/setup.py index bd2766b24c..0215fba4c4 100755 --- a/setup.py +++ b/setup.py @@ -17,52 +17,52 @@ import os from setuptools import setup, find_packages +import setuptools.command.easy_install -# Utility function to read the README file. -# Used for the long_description. It's nice, because now 1) we have a top level -# README file and 2) it's easier to type in the README file than to put a raw -# string in below ... -def read(fname): - return open(os.path.join(os.path.dirname(__file__), fname)).read() + +def no_easy_install(self, spec, *args, **kargs): + raise RuntimeError( + "Missing requirement %r. Please use pip to install this dependency." + " See README.rst for instructions." + % (spec,) + ) + +# Patch the easy_install command to prevent people accidentally installing +# depedencies using it. +setuptools.command.easy_install.easy_install.easy_install = no_easy_install + + +here = os.path.abspath(os.path.dirname(__file__)) + + +def read_file(path): + """Read a file from the package. Takes a list of strings to join to + make the path""" + file_path = os.path.join(here, *path) + with open(file_path) as f: + return f.read() + + +def exec_file(path): + """Execute a single python file to get the variables defined in it""" + result = {} + code = read_file(path) + exec(code, result) + return result + +version = exec_file(("synapse", "__init__.py"))["__version__"] +dependencies = exec_file(("synapse", "python_dependencies.py")) +long_description = read_file(("README.rst",)) setup( name="matrix-synapse", - version=read("VERSION").strip(), + version=version, packages=find_packages(exclude=["tests", "tests.*"]), description="Reference Synapse Home Server", - install_requires=[ - "syutil==0.0.2", - "matrix_angular_sdk>=0.6.1", - "Twisted==14.0.2", - "service_identity>=1.0.0", - "pyopenssl>=0.14", - "pyyaml", - "pyasn1", - "pynacl", - "daemonize", - "py-bcrypt", - "frozendict>=0.4", - "pillow", - "pydenticon", - ], - dependency_links=[ - "https://github.com/matrix-org/syutil/tarball/v0.0.2#egg=syutil-0.0.2", - "https://github.com/pyca/pynacl/tarball/d4d3175589b892f6ea7c22f466e0e223853516fa#egg=pynacl-0.3.0", - "https://github.com/matrix-org/matrix-angular-sdk/tarball/v0.6.1/#egg=matrix_angular_sdk-0.6.1", - ], - setup_requires=[ - "Twisted==14.0.2", # Here to override setuptools_trial's dependency on Twisted>=2.4.0 - "setuptools_trial", - "setuptools>=1.0.0", # Needs setuptools that supports git+ssh. - # TODO: Do we need this now? we don't use git+ssh. - "mock" - ], + install_requires=dependencies["REQUIREMENTS"].keys(), + dependency_links=dependencies["DEPENDENCY_LINKS"], include_package_data=True, zip_safe=False, - long_description=read("README.rst"), - entry_points=""" - [console_scripts] - synctl=synapse.app.synctl:main - synapse-homeserver=synapse.app.homeserver:main - """ + long_description=long_description, + scripts=["synctl"], ) diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index fd68da9dfb..e2a9d1f6a7 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -5,7 +5,7 @@ logger = logging.getLogger(__name__) REQUIREMENTS = { "syutil==0.0.2": ["syutil"], - "matrix_angular_sdk==0.6.0": ["syweb>=0.6.0"], + "matrix_angular_sdk>=0.6.1": ["syweb>=0.6.1"], "Twisted==14.0.2": ["twisted==14.0.2"], "service_identity>=1.0.0": ["service_identity>=1.0.0"], "pyopenssl>=0.14": ["OpenSSL>=0.14"], @@ -31,8 +31,8 @@ DEPENDENCY_LINKS = [ ), github_link( project="matrix-org/matrix-angular-sdk", - version="v0.6.0", - egg="matrix_angular_sdk-0.6.0", + version="v0.6.1", + egg="matrix_angular_sdk-0.6.1", ), github_link( project="pyca/pynacl", From a9684730acdb98ebe9102da985765d8f309388fa Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 10 Feb 2015 17:48:16 +0000 Subject: [PATCH 20/22] Add the 'setup_requires' and allow easy_install since jenkins uses them --- setup.py | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/setup.py b/setup.py index 0215fba4c4..cc7a3f5b69 100755 --- a/setup.py +++ b/setup.py @@ -17,20 +17,6 @@ import os from setuptools import setup, find_packages -import setuptools.command.easy_install - - -def no_easy_install(self, spec, *args, **kargs): - raise RuntimeError( - "Missing requirement %r. Please use pip to install this dependency." - " See README.rst for instructions." - % (spec,) - ) - -# Patch the easy_install command to prevent people accidentally installing -# depedencies using it. -setuptools.command.easy_install.easy_install.easy_install = no_easy_install - here = os.path.abspath(os.path.dirname(__file__)) @@ -60,6 +46,11 @@ setup( packages=find_packages(exclude=["tests", "tests.*"]), description="Reference Synapse Home Server", install_requires=dependencies["REQUIREMENTS"].keys(), + setup_requires=[ + "Twisted==14.0.2", # Here to override setuptools_trial's dependency on Twisted>=2.4.0 + "setuptools_trial", + "mock" + ], dependency_links=dependencies["DEPENDENCY_LINKS"], include_package_data=True, zip_safe=False, From 84a769cdb7321481954c386a040d7e7ff504d02b Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 10 Feb 2015 17:58:36 +0000 Subject: [PATCH 21/22] Fix code-style --- synapse/app/homeserver.py | 2 +- synapse/crypto/keyclient.py | 4 ++-- synapse/handlers/presence.py | 8 ++++---- synapse/handlers/sync.py | 2 +- synapse/rest/client/v1/directory.py | 2 +- synapse/rest/client/v2_alpha/sync.py | 2 +- synapse/rest/media/v1/upload_resource.py | 4 ++-- 7 files changed, 12 insertions(+), 12 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 8976ff2e82..ff6680e7da 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -134,7 +134,7 @@ class SynapseHomeServer(HomeServer): logger.info("Attaching %s to path %s", resource, full_path) last_resource = self.root_resource for path_seg in full_path.split('/')[1:-1]: - if not path_seg in last_resource.listNames(): + if path_seg not in last_resource.listNames(): # resource doesn't exist, so make a "dummy resource" child_resource = Resource() last_resource.putChild(path_seg, child_resource) diff --git a/synapse/crypto/keyclient.py b/synapse/crypto/keyclient.py index cdb6279764..cd12349f67 100644 --- a/synapse/crypto/keyclient.py +++ b/synapse/crypto/keyclient.py @@ -75,7 +75,7 @@ class SynapseKeyClientProtocol(HTTPClient): def handleStatus(self, version, status, message): if status != b"200": - #logger.info("Non-200 response from %s: %s %s", + # logger.info("Non-200 response from %s: %s %s", # self.transport.getHost(), status, message) self.transport.abortConnection() @@ -83,7 +83,7 @@ class SynapseKeyClientProtocol(HTTPClient): try: json_response = json.loads(response_body_bytes) except ValueError: - #logger.info("Invalid JSON response from %s", + # logger.info("Invalid JSON response from %s", # self.transport.getHost()) self.transport.abortConnection() return diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 6a266ee0fe..59287010ed 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -457,9 +457,9 @@ class PresenceHandler(BaseHandler): if state is None: state = yield self.store.get_presence_state(user.localpart) else: -# statuscache = self._get_or_make_usercache(user) -# self._user_cachemap_latest_serial += 1 -# statuscache.update(state, self._user_cachemap_latest_serial) + # statuscache = self._get_or_make_usercache(user) + # self._user_cachemap_latest_serial += 1 + # statuscache.update(state, self._user_cachemap_latest_serial) pass yield self.push_update_to_local_and_remote( @@ -709,7 +709,7 @@ class PresenceHandler(BaseHandler): # TODO(paul) permissions checks - if not user in self._remote_sendmap: + if user not in self._remote_sendmap: self._remote_sendmap[user] = set() self._remote_sendmap[user].add(origin) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 5af90cc5d1..7883bbd834 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -114,7 +114,7 @@ class SyncHandler(BaseHandler): if sync_config.gap: return self.incremental_sync_with_gap(sync_config, since_token) else: - #TODO(mjark): Handle gapless sync + # TODO(mjark): Handle gapless sync raise NotImplementedError() @defer.inlineCallbacks diff --git a/synapse/rest/client/v1/directory.py b/synapse/rest/client/v1/directory.py index 8f65efec5f..8ed7e2d669 100644 --- a/synapse/rest/client/v1/directory.py +++ b/synapse/rest/client/v1/directory.py @@ -48,7 +48,7 @@ class ClientDirectoryServer(ClientV1RestServlet): user, client = yield self.auth.get_user_by_req(request) content = _parse_json(request) - if not "room_id" in content: + if "room_id" not in content: raise SynapseError(400, "Missing room_id key", errcode=Codes.BAD_JSON) diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index 81d5cf8ead..3056ec45cf 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -118,7 +118,7 @@ class SyncRestServlet(RestServlet): except: filter = Filter({}) # filter = filter.apply_overrides(http_request) - #if filter.matches(event): + # if filter.matches(event): # # stuff sync_config = SyncConfig( diff --git a/synapse/rest/media/v1/upload_resource.py b/synapse/rest/media/v1/upload_resource.py index 6df52ca434..e5aba3af4c 100644 --- a/synapse/rest/media/v1/upload_resource.py +++ b/synapse/rest/media/v1/upload_resource.py @@ -96,8 +96,8 @@ class UploadResource(BaseMediaResource): code=400, ) - #if headers.hasHeader("Content-Disposition"): - # disposition = headers.getRawHeaders("Content-Disposition")[0] + # if headers.hasHeader("Content-Disposition"): + # disposition = headers.getRawHeaders("Content-Disposition")[0] # TODO(markjh): parse content-dispostion content_uri = yield self.create_content( From eab141ee676c87f6d1ca3d853e3f9ef9d49e28ca Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 10 Feb 2015 18:25:54 +0000 Subject: [PATCH 22/22] Rename path to path_segments to make it clearer that it is a list --- setup.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index cc7a3f5b69..2d812fa389 100755 --- a/setup.py +++ b/setup.py @@ -21,18 +21,18 @@ from setuptools import setup, find_packages here = os.path.abspath(os.path.dirname(__file__)) -def read_file(path): +def read_file(path_segments): """Read a file from the package. Takes a list of strings to join to make the path""" - file_path = os.path.join(here, *path) + file_path = os.path.join(here, *path_segments) with open(file_path) as f: return f.read() -def exec_file(path): +def exec_file(path_segments): """Execute a single python file to get the variables defined in it""" result = {} - code = read_file(path) + code = read_file(path_segments) exec(code, result) return result