From cc3ab0c214156cdad9731beb27790e8510b4b023 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Tue, 22 Sep 2015 18:13:06 +0100
Subject: [PATCH 01/24] Add dev script for finding where functions are called
 from, and finding functions that aren't called at all

---
 scripts-dev/definitions.py | 130 +++++++++++++++++++++++++++++++++++++
 1 file changed, 130 insertions(+)
 create mode 100755 scripts-dev/definitions.py

diff --git a/scripts-dev/definitions.py b/scripts-dev/definitions.py
new file mode 100755
index 0000000000..09cec9d8ec
--- /dev/null
+++ b/scripts-dev/definitions.py
@@ -0,0 +1,130 @@
+#! /usr/bin/python
+
+import ast
+import yaml
+
+class DefinitionVisitor(ast.NodeVisitor):
+    def __init__(self):
+        super(DefinitionVisitor, self).__init__()
+        self.functions = {}
+        self.classes = {}
+        self.names = {}
+        self.attrs = set()
+        self.definitions = {
+            'def': self.functions,
+            'class': self.classes,
+            'names': self.names,
+            'attrs': self.attrs,
+        }
+
+    def visit_Name(self, node):
+        self.names.setdefault(type(node.ctx).__name__, set()).add(node.id)
+
+    def visit_Attribute(self, node):
+        self.attrs.add(node.attr)
+        for child in ast.iter_child_nodes(node):
+            self.visit(child)
+
+    def visit_ClassDef(self, node):
+        visitor = DefinitionVisitor()
+        self.classes[node.name] = visitor.definitions
+        for child in ast.iter_child_nodes(node):
+            visitor.visit(child)
+
+    def visit_FunctionDef(self, node):
+        visitor = DefinitionVisitor()
+        self.functions[node.name] = visitor.definitions
+        for child in ast.iter_child_nodes(node):
+            visitor.visit(child)
+
+
+def non_empty(defs):
+    functions = {name: non_empty(f) for name, f in defs['def'].items()}
+    classes = {name: non_empty(f) for name, f in defs['class'].items()}
+    result = {}
+    if functions: result['def'] = functions
+    if classes: result['class'] = classes
+    names = defs['names']
+    uses = []
+    for name in names.get('Load', ()):
+        if name not in names.get('Param', ()) and name not in names.get('Store', ()):
+            uses.append(name)
+    uses.extend(defs['attrs'])
+    if uses: result['uses'] = uses
+    result['names'] = names
+    result['attrs'] = defs['attrs']
+    return result
+
+
+def definitions_in_code(input_code):
+    input_ast = ast.parse(input_code)
+    visitor = DefinitionVisitor()
+    visitor.visit(input_ast)
+    definitions = non_empty(visitor.definitions)
+    return definitions
+
+
+def definitions_in_file(filepath):
+    with open(filepath) as f:
+        return definitions_in_code(f.read())
+
+
+def defined_names(prefix, defs, names):
+    for name, funcs in defs.get('def', {}).items():
+        names.setdefault(name, {'defined': []})['defined'].append(prefix + name)
+        defined_names(prefix + name + ".", funcs, names)
+
+    for name, funcs in defs.get('class', {}).items():
+        names.setdefault(name, {'defined': []})['defined'].append(prefix + name)
+        defined_names(prefix + name + ".", funcs, names)
+
+
+def used_names(prefix, defs, names):
+    for name, funcs in defs.get('def', {}).items():
+        used_names(prefix + name + ".", funcs, names)
+
+    for name, funcs in defs.get('class', {}).items():
+        used_names(prefix + name + ".", funcs, names)
+
+    for used in defs.get('uses', ()):
+        if used in names:
+            names[used].setdefault('used', []).append(prefix.rstrip('.'))
+
+
+if __name__ == '__main__':
+    import sys, os
+    if not sys.argv[1:]:
+        sys.stderr.write(
+            "Usage: definitions.py <directory> <regexp>\n"
+            "       definitions.py <directory>\n"
+            "Either list the definitions matching the regexp or list\n"
+            " 'unused' definitions\n"
+        )
+
+    definitions = {}
+    for root, dirs, files in os.walk(sys.argv[1]):
+        for filename in files:
+            if filename.endswith(".py"):
+                filepath = os.path.join(root, filename)
+                definitions[filepath] = definitions_in_file(filepath)
+
+    names = {}
+    for filepath, defs in definitions.items():
+        defined_names(filepath + ":", defs, names)
+
+    for filepath, defs in definitions.items():
+        used_names(filepath + ":", defs, names)
+
+    if sys.argv[2:]:
+        import re
+        pattern = re.compile(sys.argv[2])
+        for name in list(names):
+            if not pattern.match(name):
+                del names[name]
+    else:
+        for name in list(names):
+            if 'used' in names[name]:
+                del names[name]
+
+    yaml.dump(names, sys.stdout, default_flow_style=False)
+    #yaml.dump(definitions, sys.stdout, default_flow_style=False)

From 527d95dea0d55bd1932639e61387ef60d834134e Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Tue, 22 Sep 2015 18:14:15 +0100
Subject: [PATCH 02/24] synapse/storage/_base.py:Table was unused

---
 synapse/storage/_base.py  | 128 --------------------------------------
 synapse/storage/pusher.py |   4 +-
 2 files changed, 2 insertions(+), 130 deletions(-)

diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index 495ef087c9..c1b5423bd6 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -25,8 +25,6 @@ from util.id_generators import IdGenerator, StreamIdGenerator
 
 from twisted.internet import defer
 
-from collections import namedtuple
-
 import sys
 import time
 import threading
@@ -791,129 +789,3 @@ class _RollbackButIsFineException(Exception):
     something went wrong.
     """
     pass
-
-
-class Table(object):
-    """ A base class used to store information about a particular table.
-    """
-
-    table_name = None
-    """ str: The name of the table """
-
-    fields = None
-    """ list: The field names """
-
-    EntryType = None
-    """ Type: A tuple type used to decode the results """
-
-    _select_where_clause = "SELECT %s FROM %s WHERE %s"
-    _select_clause = "SELECT %s FROM %s"
-    _insert_clause = "REPLACE INTO %s (%s) VALUES (%s)"
-
-    @classmethod
-    def select_statement(cls, where_clause=None):
-        """
-        Args:
-            where_clause (str): The WHERE clause to use.
-
-        Returns:
-            str: An SQL statement to select rows from the table with the given
-            WHERE clause.
-        """
-        if where_clause:
-            return cls._select_where_clause % (
-                ", ".join(cls.fields),
-                cls.table_name,
-                where_clause
-            )
-        else:
-            return cls._select_clause % (
-                ", ".join(cls.fields),
-                cls.table_name,
-            )
-
-    @classmethod
-    def insert_statement(cls):
-        return cls._insert_clause % (
-            cls.table_name,
-            ", ".join(cls.fields),
-            ", ".join(["?"] * len(cls.fields)),
-        )
-
-    @classmethod
-    def decode_single_result(cls, results):
-        """ Given an iterable of tuples, return a single instance of
-            `EntryType` or None if the iterable is empty
-        Args:
-            results (list): The results list to convert to `EntryType`
-        Returns:
-            EntryType: An instance of `EntryType`
-        """
-        results = list(results)
-        if results:
-            return cls.EntryType(*results[0])
-        else:
-            return None
-
-    @classmethod
-    def decode_results(cls, results):
-        """ Given an iterable of tuples, return a list of `EntryType`
-        Args:
-            results (list): The results list to convert to `EntryType`
-
-        Returns:
-            list: A list of `EntryType`
-        """
-        return [cls.EntryType(*row) for row in results]
-
-    @classmethod
-    def get_fields_string(cls, prefix=None):
-        if prefix:
-            to_join = ("%s.%s" % (prefix, f) for f in cls.fields)
-        else:
-            to_join = cls.fields
-
-        return ", ".join(to_join)
-
-
-class JoinHelper(object):
-    """ Used to help do joins on tables by looking at the tables' fields and
-    creating a list of unique fields to use with SELECTs and a namedtuple
-    to dump the results into.
-
-    Attributes:
-        tables (list): List of `Table` classes
-        EntryType (type)
-    """
-
-    def __init__(self, *tables):
-        self.tables = tables
-
-        res = []
-        for table in self.tables:
-            res += [f for f in table.fields if f not in res]
-
-        self.EntryType = namedtuple("JoinHelperEntry", res)
-
-    def get_fields(self, **prefixes):
-        """Get a string representing a list of fields for use in SELECT
-        statements with the given prefixes applied to each.
-
-        For example::
-
-            JoinHelper(PdusTable, StateTable).get_fields(
-                PdusTable="pdus",
-                StateTable="state"
-            )
-        """
-        res = []
-        for field in self.EntryType._fields:
-            for table in self.tables:
-                if field in table.fields:
-                    res.append("%s.%s" % (prefixes[table.__name__], field))
-                    break
-
-        return ", ".join(res)
-
-    def decode_results(self, rows):
-        return [self.EntryType(*row) for row in rows]
diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py
index 00b748f131..345c4e1104 100644
--- a/synapse/storage/pusher.py
+++ b/synapse/storage/pusher.py
@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from ._base import SQLBaseStore, Table
+from ._base import SQLBaseStore
 from twisted.internet import defer
 
 from synapse.api.errors import StoreError
@@ -149,5 +149,5 @@ class PusherStore(SQLBaseStore):
         )
 
 
-class PushersTable(Table):
+class PushersTable(object):
     table_name = "pushers"

From 372ac60375973f03bcfce7f857c85b6ed2388881 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Tue, 22 Sep 2015 18:16:07 +0100
Subject: [PATCH 03/24] synapse/util/__init__.py:unwrap_deferred was unused

---
 synapse/util/__init__.py | 28 ----------------------------
 1 file changed, 28 deletions(-)

diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py
index 07ff25cef3..1d123ccefc 100644
--- a/synapse/util/__init__.py
+++ b/synapse/util/__init__.py
@@ -29,34 +29,6 @@ def unwrapFirstError(failure):
     return failure.value.subFailure
 
 
-def unwrap_deferred(d):
-    """Given a deferred that we know has completed, return its value or raise
-    the failure as an exception
-    """
-    if not d.called:
-        raise RuntimeError("deferred has not finished")
-
-    res = []
-
-    def f(r):
-        res.append(r)
-        return r
-    d.addCallback(f)
-
-    if res:
-        return res[0]
-
-    def f(r):
-        res.append(r)
-        return r
-    d.addErrback(f)
-
-    if res:
-        res[0].raiseException()
-    else:
-        raise RuntimeError("deferred did not call callbacks")
-
-
 class Clock(object):
     """A small utility that obtains current time-of-day so that time may be
     mocked during unit-tests.

From f2fcc0a8cff8b11ba0201c2bebea37649960b6a3 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Tue, 22 Sep 2015 18:18:45 +0100
Subject: [PATCH 04/24] synapse/api/errors.py:RoomError was unused

---
 synapse/api/errors.py | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/synapse/api/errors.py b/synapse/api/errors.py
index c3b4d971a8..ee3045268f 100644
--- a/synapse/api/errors.py
+++ b/synapse/api/errors.py
@@ -77,11 +77,6 @@ class SynapseError(CodeMessageException):
         )
 
 
-class RoomError(SynapseError):
-    """An error raised when a room event fails."""
-    pass
-
-
 class RegistrationError(SynapseError):
     """An error raised when a registration event fails."""
     pass

From a247729806a1cf7093b3c0819094338bf22affa8 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Tue, 22 Sep 2015 18:19:49 +0100
Subject: [PATCH 05/24] synapse/streams/events.py:StreamSource was unused

---
 synapse/streams/events.py | 12 ------------
 1 file changed, 12 deletions(-)

diff --git a/synapse/streams/events.py b/synapse/streams/events.py
index aaa3609aa5..8671a8fa4e 100644
--- a/synapse/streams/events.py
+++ b/synapse/streams/events.py
@@ -70,15 +70,3 @@ class EventSources(object):
             ),
         )
         defer.returnValue(token)
-
-
-class StreamSource(object):
-    def get_new_events_for_user(self, user, from_key, limit):
-        """from_key is the key within this event source."""
-        raise NotImplementedError("get_new_events_for_user")
-
-    def get_current_key(self):
-        raise NotImplementedError("get_current_key")
-
-    def get_pagination_rows(self, user, pagination_config, key):
-        raise NotImplementedError("get_rows")

From 184ba0968a08e537dccc65d8fe75f65b86e22504 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Tue, 22 Sep 2015 18:25:30 +0100
Subject: [PATCH 06/24] synapse/app/homeserver.py:GzipFile was unused

---
 synapse/app/homeserver.py | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index 21840e4a28..349a4c1774 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -85,12 +85,6 @@ import time
 logger = logging.getLogger("synapse.app.homeserver")
 
 
-class GzipFile(File):
-    def getChild(self, path, request):
-        child = File.getChild(self, path, request)
-        return EncodingResourceWrapper(child, [GzipEncoderFactory()])
-
-
 def gz_wrap(r):
     return EncodingResourceWrapper(r, [GzipEncoderFactory()])
 

From 7a5818ed811c6da046e60bcc7eacb7a28a92cd73 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Tue, 22 Sep 2015 18:27:22 +0100
Subject: [PATCH 07/24] Note that GzipFile was removed in comment that
 referenced it

---
 synapse/app/homeserver.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index 349a4c1774..190b03e2f7 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -128,6 +128,7 @@ class SynapseHomeServer(HomeServer):
         # (It can stay enabled for the API resources: they call
         # write() with the whole body and then finish() straight
         # after and so do not trigger the bug.
+        # GzipFile was removed in commit 184ba09
         # return GzipFile(webclient_path)  # TODO configurable?
         return File(webclient_path)  # TODO configurable?
 

From bb4dddd6c4f85bc5b07119d3f9dec31964b5b6f9 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Tue, 22 Sep 2015 18:33:34 +0100
Subject: [PATCH 08/24] Move NullSource out of synapse and into tests since it
 is only used by the tests

---
 synapse/streams/events.py             | 16 ----------------
 tests/rest/client/v1/test_presence.py | 18 +++++++++++++++++-
 2 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/synapse/streams/events.py b/synapse/streams/events.py
index 8671a8fa4e..699083ae12 100644
--- a/synapse/streams/events.py
+++ b/synapse/streams/events.py
@@ -23,22 +23,6 @@ from synapse.handlers.typing import TypingNotificationEventSource
 from synapse.handlers.receipts import ReceiptEventSource
 
 
-class NullSource(object):
-    """This event source never yields any events and its token remains at
-    zero. It may be useful for unit-testing."""
-    def __init__(self, hs):
-        pass
-
-    def get_new_events_for_user(self, user, from_key, limit):
-        return defer.succeed(([], from_key))
-
-    def get_current_key(self, direction='f'):
-        return defer.succeed(0)
-
-    def get_pagination_rows(self, user, pagination_config, key):
-        return defer.succeed(([], pagination_config.from_key))
-
-
 class EventSources(object):
     SOURCE_TYPES = {
         "room": RoomEventSource,
diff --git a/tests/rest/client/v1/test_presence.py b/tests/rest/client/v1/test_presence.py
index 2ee3da0b34..29d9bbaad4 100644
--- a/tests/rest/client/v1/test_presence.py
+++ b/tests/rest/client/v1/test_presence.py
@@ -41,6 +41,22 @@ myid = "@apple:test"
 PATH_PREFIX = "/_matrix/client/api/v1"
 
 
+class NullSource(object):
+    """This event source never yields any events and its token remains at
+    zero. It may be useful for unit-testing."""
+    def __init__(self, hs):
+        pass
+
+    def get_new_events_for_user(self, user, from_key, limit):
+        return defer.succeed(([], from_key))
+
+    def get_current_key(self, direction='f'):
+        return defer.succeed(0)
+
+    def get_pagination_rows(self, user, pagination_config, key):
+        return defer.succeed(([], pagination_config.from_key))
+
+
 class JustPresenceHandlers(object):
     def __init__(self, hs):
         self.presence_handler = PresenceHandler(hs)
@@ -243,7 +259,7 @@ class PresenceEventStreamTestCase(unittest.TestCase):
         # HIDEOUS HACKERY
         # TODO(paul): This should be injected in via the HomeServer DI system
         from synapse.streams.events import (
-            PresenceEventSource, NullSource, EventSources
+            PresenceEventSource, EventSources
         )
 
         old_SOURCE_TYPES = EventSources.SOURCE_TYPES

From 7dd4f79c49e1a1ba4cf2edf8b45ed841a32a33b0 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Tue, 22 Sep 2015 18:37:07 +0100
Subject: [PATCH 09/24] synapse/storage/_base.py:_execute_and_decode was unused

---
 synapse/storage/_base.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index c1b5423bd6..cf4ec30f48 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -374,9 +374,6 @@ class SQLBaseStore(object):
 
         return self.runInteraction(desc, interaction)
 
-    def _execute_and_decode(self, desc, query, *args):
-        return self._execute(desc, self.cursor_to_dict, query, *args)
-
     # "Simple" SQL API methods that operate on a single table with no JOINs,
     # no complex WHERE clauses, just a dict of values for columns.
 

From 3559a835a22be3ae474fc2a14363a59b89ec05b8 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Tue, 22 Sep 2015 18:39:46 +0100
Subject: [PATCH 10/24] synapse/storage/event_federation.py:_get_auth_events is
 unused

---
 synapse/storage/event_federation.py | 21 ---------------------
 1 file changed, 21 deletions(-)

diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py
index c1cabbaa60..7ed0d9ae1b 100644
--- a/synapse/storage/event_federation.py
+++ b/synapse/storage/event_federation.py
@@ -225,27 +225,6 @@ class EventFederationStore(SQLBaseStore):
 
         return results
 
-    def _get_auth_events(self, txn, event_id):
-        auth_ids = self._simple_select_onecol_txn(
-            txn,
-            table="event_auth",
-            keyvalues={
-                "event_id": event_id,
-            },
-            retcol="auth_id",
-        )
-
-        results = []
-        for auth_id in auth_ids:
-            hashes = self._get_event_reference_hashes_txn(txn, auth_id)
-            prev_hashes = {
-                k: encode_base64(v) for k, v in hashes.items()
-                if k == "sha256"
-            }
-            results.append((auth_id, prev_hashes))
-
-        return results
-
     def get_min_depth(self, room_id):
         """ For hte given room, get the minimum depth we have seen for it.
         """

From 04abf53a5633d8220b01772aedc573caa0e71f6e Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Wed, 23 Sep 2015 10:17:50 +0100
Subject: [PATCH 11/24] Use argparse for definition finder

---
 scripts-dev/definitions.py | 62 +++++++++++++++++++++++---------------
 1 file changed, 37 insertions(+), 25 deletions(-)

diff --git a/scripts-dev/definitions.py b/scripts-dev/definitions.py
index 09cec9d8ec..934bc4a3d8 100755
--- a/scripts-dev/definitions.py
+++ b/scripts-dev/definitions.py
@@ -92,21 +92,32 @@ def used_names(prefix, defs, names):
 
 
 if __name__ == '__main__':
-    import sys, os
-    if not sys.argv[1:]:
-        sys.stderr.write(
-            "Usage: definitions.py <directory> <regexp>\n"
-            "       definitions.py <directory>\n"
-            "Either list the definitions matching the regexp or list\n"
-            " 'unused' definitions\n"
-        )
+    import sys, os, argparse, re
+
+    parser = argparse.ArgumentParser(description='Find definitions.')
+    parser.add_argument(
+        "--unused", action="store_true", help="Only list unused definitions"
+    )
+    parser.add_argument(
+        "--ignore", action="append", metavar="REGEXP", help="Ignore a pattern"
+    )
+    parser.add_argument(
+        "--pattern", nargs='+', action="append", metavar="REGEXP",
+        help="Search for a pattern"
+    )
+    parser.add_argument(
+        "directories", nargs='+', metavar="DIR",
+        help="Directories to search for definitions"
+    )
+    args = parser.parse_args()
 
     definitions = {}
-    for root, dirs, files in os.walk(sys.argv[1]):
-        for filename in files:
-            if filename.endswith(".py"):
-                filepath = os.path.join(root, filename)
-                definitions[filepath] = definitions_in_file(filepath)
+    for directory in args.directories:
+        for root, dirs, files in os.walk(directory):
+            for filename in files:
+                if filename.endswith(".py"):
+                    filepath = os.path.join(root, filename)
+                    definitions[filepath] = definitions_in_file(filepath)
 
     names = {}
     for filepath, defs in definitions.items():
@@ -115,16 +126,17 @@ if __name__ == '__main__':
     for filepath, defs in definitions.items():
         used_names(filepath + ":", defs, names)
 
-    if sys.argv[2:]:
-        import re
-        pattern = re.compile(sys.argv[2])
-        for name in list(names):
-            if not pattern.match(name):
-                del names[name]
-    else:
-        for name in list(names):
-            if 'used' in names[name]:
-                del names[name]
+    patterns = [re.compile(pattern) for pattern in args.pattern or ()]
+    ignore = [re.compile(pattern) for pattern in args.ignore or ()]
 
-    yaml.dump(names, sys.stdout, default_flow_style=False)
-    #yaml.dump(definitions, sys.stdout, default_flow_style=False)
+    result = {}
+    for name, definition in names.items():
+        if patterns and not any(pattern.match(name) for pattern in patterns):
+            continue
+        if ignore and any(pattern.match(name) for pattern in ignore):
+            continue
+        if args.unused and definition.get('used'):
+            continue
+        result[name] = definition
+
+    yaml.dump(result, sys.stdout, default_flow_style=False)

From 60728c8c9ebc80c58454639c3f021d43091453ca Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Wed, 23 Sep 2015 10:25:26 +0100
Subject: [PATCH 12/24] synapse/handlers/federation.py:_handle_auth_events was
 unused

---
 synapse/handlers/federation.py | 49 ----------------------------------
 1 file changed, 49 deletions(-)

diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 4ff20599d6..f4dce712f9 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -1456,52 +1456,3 @@ class FederationHandler(BaseHandler):
             },
             "missing": [e.event_id for e in missing_locals],
         })
-
-    @defer.inlineCallbacks
-    def _handle_auth_events(self, origin, auth_events):
-        auth_ids_to_deferred = {}
-
-        def process_auth_ev(ev):
-            auth_ids = [e_id for e_id, _ in ev.auth_events]
-
-            prev_ds = [
-                auth_ids_to_deferred[i]
-                for i in auth_ids
-                if i in auth_ids_to_deferred
-            ]
-
-            d = defer.Deferred()
-
-            auth_ids_to_deferred[ev.event_id] = d
-
-            @defer.inlineCallbacks
-            def f(*_):
-                ev.internal_metadata.outlier = True
-
-                try:
-                    auth = {
-                        (e.type, e.state_key): e for e in auth_events
-                        if e.event_id in auth_ids
-                    }
-
-                    yield self._handle_new_event(
-                        origin, ev, auth_events=auth
-                    )
-                except:
-                    logger.exception(
-                        "Failed to handle auth event %s",
-                        ev.event_id,
-                    )
-
-                d.callback(None)
-
-            if prev_ds:
-                dx = defer.DeferredList(prev_ds)
-                dx.addBoth(f)
-            else:
-                f()
-
-        for e in auth_events:
-            process_auth_ev(e)
-
-        yield defer.DeferredList(auth_ids_to_deferred.values())

From 57338a97683de16b325a2e62b4dee4b4a6b77aa2 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Wed, 23 Sep 2015 10:26:45 +0100
Subject: [PATCH 13/24] synapse/handlers/room.py:_should_invite_join was unused

---
 synapse/handlers/room.py | 26 --------------------------
 1 file changed, 26 deletions(-)

diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py
index 243623190f..d60f86bbd0 100644
--- a/synapse/handlers/room.py
+++ b/synapse/handlers/room.py
@@ -492,32 +492,6 @@ class RoomMemberHandler(BaseHandler):
             "user_joined_room", user=user, room_id=room_id
         )
 
-    @defer.inlineCallbacks
-    def _should_invite_join(self, room_id, prev_state, do_auth):
-        logger.debug("_should_invite_join: room_id: %s", room_id)
-
-        # XXX: We don't do an auth check if we are doing an invite
-        # join dance for now, since we're kinda implicitly checking
-        # that we are allowed to join when we decide whether or not we
-        # need to do the invite/join dance.
-
-        # Only do an invite join dance if a) we were invited,
-        # b) the person inviting was from a differnt HS and c) we are
-        # not currently in the room
-        room_host = None
-        if prev_state and prev_state.membership == Membership.INVITE:
-            room = yield self.store.get_room(room_id)
-            inviter = UserID.from_string(
-                prev_state.sender
-            )
-
-            is_remote_invite_join = not self.hs.is_mine(inviter) and not room
-            room_host = inviter.domain
-        else:
-            is_remote_invite_join = False
-
-        defer.returnValue((is_remote_invite_join, room_host))
-
     @defer.inlineCallbacks
     def get_joined_rooms_for_user(self, user):
         """Returns a list of roomids that the user has any of the given

From 82b8d4b86a981559570fbd1e8ee3dcd38faf553f Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Wed, 23 Sep 2015 10:27:47 +0100
Subject: [PATCH 14/24] synapse/state.py:_get_state_key_from_event was unused

---
 synapse/state.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/synapse/state.py b/synapse/state.py
index ed36f844cb..bb225c39cf 100644
--- a/synapse/state.py
+++ b/synapse/state.py
@@ -31,10 +31,6 @@ import hashlib
 logger = logging.getLogger(__name__)
 
 
-def _get_state_key_from_event(event):
-    return event.state_key
-
-
 KeyStateTuple = namedtuple("KeyStateTuple", ("context", "type", "state_key"))
 
 

From 1ee3d26432d87ff312350f21da982f646b5af49a Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Wed, 23 Sep 2015 10:30:03 +0100
Subject: [PATCH 15/24] synapse/storage/_base.py:_simple_selectupdate_one was
 unused

---
 synapse/storage/_base.py   | 31 -------------------------------
 tests/storage/test_base.py | 20 --------------------
 2 files changed, 51 deletions(-)

diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index cf4ec30f48..79021bde6b 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -686,37 +686,6 @@ class SQLBaseStore(object):
 
         return dict(zip(retcols, row))
 
-    def _simple_selectupdate_one(self, table, keyvalues, updatevalues=None,
-                                 retcols=None, allow_none=False,
-                                 desc="_simple_selectupdate_one"):
-        """ Combined SELECT then UPDATE."""
-        def func(txn):
-            ret = None
-            if retcols:
-                ret = self._simple_select_one_txn(
-                    txn,
-                    table=table,
-                    keyvalues=keyvalues,
-                    retcols=retcols,
-                    allow_none=allow_none,
-                )
-
-            if updatevalues:
-                self._simple_update_one_txn(
-                    txn,
-                    table=table,
-                    keyvalues=keyvalues,
-                    updatevalues=updatevalues,
-                )
-
-                # if txn.rowcount == 0:
-                #     raise StoreError(404, "No row found")
-                if txn.rowcount > 1:
-                    raise StoreError(500, "More than one row matched")
-
-            return ret
-        return self.runInteraction(desc, func)
-
     def _simple_delete_one(self, table, keyvalues, desc="_simple_delete_one"):
         """Executes a DELETE query on the named table, expecting to delete a
         single row.
diff --git a/tests/storage/test_base.py b/tests/storage/test_base.py
index 8573f18b55..1ddca1da4c 100644
--- a/tests/storage/test_base.py
+++ b/tests/storage/test_base.py
@@ -185,26 +185,6 @@ class SQLBaseStoreTestCase(unittest.TestCase):
                 [3, 4, 1, 2]
         )
 
-    @defer.inlineCallbacks
-    def test_update_one_with_return(self):
-        self.mock_txn.rowcount = 1
-        self.mock_txn.fetchone.return_value = ("Old Value",)
-
-        ret = yield self.datastore._simple_selectupdate_one(
-                table="tablename",
-                keyvalues={"keycol": "TheKey"},
-                updatevalues={"columname": "New Value"},
-                retcols=["columname"]
-        )
-
-        self.assertEquals({"columname": "Old Value"}, ret)
-        self.mock_txn.execute.assert_has_calls([
-                call('SELECT columname FROM tablename WHERE keycol = ?',
-                    ['TheKey']),
-                call("UPDATE tablename SET columname = ? WHERE keycol = ?",
-                    ["New Value", "TheKey"])
-        ])
-
     @defer.inlineCallbacks
     def test_delete_one(self):
         self.mock_txn.rowcount = 1

From 1d9036aff2be9ce089114b1389cc86a34ccd8490 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Wed, 23 Sep 2015 10:30:25 +0100
Subject: [PATCH 16/24] synapse/storage/_base.py:_simple_delete was unused

---
 synapse/storage/_base.py | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index 79021bde6b..33751f3092 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -707,16 +707,6 @@ class SQLBaseStore(object):
                 raise StoreError(500, "more than one row matched")
         return self.runInteraction(desc, func)
 
-    def _simple_delete(self, table, keyvalues, desc="_simple_delete"):
-        """Executes a DELETE query on the named table.
-
-        Args:
-            table : string giving the table name
-            keyvalues : dict of column names and values to select the row with
-        """
-
-        return self.runInteraction(desc, self._simple_delete_txn)
-
     def _simple_delete_txn(self, txn, table, keyvalues):
         sql = "DELETE FROM %s WHERE %s" % (
             table,

From 396834f1c010899cae87aac45143bbd9911adb4f Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Wed, 23 Sep 2015 10:30:38 +0100
Subject: [PATCH 17/24] synapse/storage/_base.py:_simple_max_id was unused

---
 synapse/storage/_base.py | 18 ------------------
 1 file changed, 18 deletions(-)

diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index 33751f3092..693784ad38 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -715,24 +715,6 @@ class SQLBaseStore(object):
 
         return txn.execute(sql, keyvalues.values())
 
-    def _simple_max_id(self, table):
-        """Executes a SELECT query on the named table, expecting to return the
-        max value for the column "id".
-
-        Args:
-            table : string giving the table name
-        """
-        sql = "SELECT MAX(id) AS id FROM %s" % table
-
-        def func(txn):
-            txn.execute(sql)
-            max_id = self.cursor_to_dict(txn)[0]["id"]
-            if max_id is None:
-                return 0
-            return max_id
-
-        return self.runInteraction("_simple_max_id", func)
-
     def get_next_stream_id(self):
         with self._next_stream_id_lock:
             i = self._next_stream_id

From c292dba70cca26751cdab90a6e50200828959642 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Wed, 23 Sep 2015 10:31:25 +0100
Subject: [PATCH 18/24] Remove unused functions from
 synapse/storage/event_federation.py

---
 synapse/storage/event_federation.py | 71 -----------------------------
 1 file changed, 71 deletions(-)

diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py
index 7ed0d9ae1b..6d4421dd8f 100644
--- a/synapse/storage/event_federation.py
+++ b/synapse/storage/event_federation.py
@@ -154,77 +154,6 @@ class EventFederationStore(SQLBaseStore):
 
         return results
 
-    def _get_latest_state_in_room(self, txn, room_id, type, state_key):
-        event_ids = self._simple_select_onecol_txn(
-            txn,
-            table="state_forward_extremities",
-            keyvalues={
-                "room_id": room_id,
-                "type": type,
-                "state_key": state_key,
-            },
-            retcol="event_id",
-        )
-
-        results = []
-        for event_id in event_ids:
-            hashes = self._get_event_reference_hashes_txn(txn, event_id)
-            prev_hashes = {
-                k: encode_base64(v) for k, v in hashes.items()
-                if k == "sha256"
-            }
-            results.append((event_id, prev_hashes))
-
-        return results
-
-    def _get_prev_events(self, txn, event_id):
-        results = self._get_prev_events_and_state(
-            txn,
-            event_id,
-            is_state=0,
-        )
-
-        return [(e_id, h, ) for e_id, h, _ in results]
-
-    def _get_prev_state(self, txn, event_id):
-        results = self._get_prev_events_and_state(
-            txn,
-            event_id,
-            is_state=True,
-        )
-
-        return [(e_id, h, ) for e_id, h, _ in results]
-
-    def _get_prev_events_and_state(self, txn, event_id, is_state=None):
-        keyvalues = {
-            "event_id": event_id,
-        }
-
-        if is_state is not None:
-            keyvalues["is_state"] = bool(is_state)
-
-        res = self._simple_select_list_txn(
-            txn,
-            table="event_edges",
-            keyvalues=keyvalues,
-            retcols=["prev_event_id", "is_state"],
-        )
-
-        hashes = self._get_prev_event_hashes_txn(txn, event_id)
-
-        results = []
-        for d in res:
-            edge_hash = self._get_event_reference_hashes_txn(txn, d["prev_event_id"])
-            edge_hash.update(hashes.get(d["prev_event_id"], {}))
-            prev_hashes = {
-                k: encode_base64(v)
-                for k, v in edge_hash.items()
-                if k == "sha256"
-            }
-            results.append((d["prev_event_id"], prev_hashes, d["is_state"]))
-
-        return results
-
     def get_min_depth(self, room_id):
         """ For hte given room, get the minimum depth we have seen for it.
         """

From 92d8d724c5c34f0a83cbd8c5dce7f0c0c21a1568 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Wed, 23 Sep 2015 10:33:06 +0100
Subject: [PATCH 19/24] Remove unused functions from synapse/storage/events.py

---
 synapse/storage/events.py | 11 -----------
 1 file changed, 11 deletions(-)

diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index 46df6b4d6d..416ef6af93 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -890,22 +890,11 @@ class EventsStore(SQLBaseStore):
 
         return ev
 
-    def _parse_events(self, rows):
-        return self.runInteraction(
-            "_parse_events", self._parse_events_txn, rows
-        )
-
     def _parse_events_txn(self, txn, rows):
         event_ids = [r["event_id"] for r in rows]
 
         return self._get_events_txn(txn, event_ids)
 
-    def _has_been_redacted_txn(self, txn, event):
-        sql = "SELECT event_id FROM redactions WHERE redacts = ?"
-        txn.execute(sql, (event.event_id,))
-        result = txn.fetchone()
-        return result[0] if result else None
-
     @defer.inlineCallbacks
     def count_daily_messages(self):
         """

From e51aa4be9691e7c6918d72810ee0b751f2e48797 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Wed, 23 Sep 2015 10:35:10 +0100
Subject: [PATCH 20/24] synapse/storage/roommember.py:_get_members_query was
 unused

---
 synapse/storage/roommember.py | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py
index cd9eefbd9f..e17cbe677a 100644
--- a/synapse/storage/roommember.py
+++ b/synapse/storage/roommember.py
@@ -178,12 +178,6 @@ class RoomMemberStore(SQLBaseStore):
 
         return joined_domains
 
-    def _get_members_query(self, where_clause, where_values):
-        return self.runInteraction(
-            "get_members_query", self._get_members_events_txn,
-            where_clause, where_values
-        ).addCallbacks(self._get_events)
-
     def _get_members_events_txn(self, txn, room_id, membership=None, user_id=None):
         rows = self._get_members_rows_txn(
             txn,

From 973ebb66bacfeece1f88f8ea71b86186f9c9163e Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Wed, 23 Sep 2015 10:36:33 +0100
Subject: [PATCH 21/24] Remove unused functions from
 synapse/storage/signatures.py

---
 synapse/storage/signatures.py | 112 ----------------------------------
 1 file changed, 112 deletions(-)

diff --git a/synapse/storage/signatures.py b/synapse/storage/signatures.py
index ab57b92174..b070be504d 100644
--- a/synapse/storage/signatures.py
+++ b/synapse/storage/signatures.py
@@ -24,41 +24,6 @@ from synapse.crypto.event_signing import compute_event_reference_hash
 class SignatureStore(SQLBaseStore):
     """Persistence for event signatures and hashes"""
 
-    def _get_event_content_hashes_txn(self, txn, event_id):
-        """Get all the hashes for a given Event.
-        Args:
-            txn (cursor):
-            event_id (str): Id for the Event.
-        Returns:
-            A dict of algorithm -> hash.
-        """
-        query = (
-            "SELECT algorithm, hash"
-            " FROM event_content_hashes"
-            " WHERE event_id = ?"
-        )
-        txn.execute(query, (event_id, ))
-        return dict(txn.fetchall())
-
-    def _store_event_content_hash_txn(self, txn, event_id, algorithm,
-                                      hash_bytes):
-        """Store a hash for a Event
-        Args:
-            txn (cursor):
-            event_id (str): Id for the Event.
-            algorithm (str): Hashing algorithm.
-            hash_bytes (bytes): Hash function output bytes.
-        """
-        self._simple_insert_txn(
-            txn,
-            "event_content_hashes",
-            {
-                "event_id": event_id,
-                "algorithm": algorithm,
-                "hash": buffer(hash_bytes),
-            },
-        )
-
     def get_event_reference_hashes(self, event_ids):
         def f(txn):
             return [
@@ -123,80 +88,3 @@ class SignatureStore(SQLBaseStore):
             table="event_reference_hashes",
             values=vals,
         )
-
-    def _get_event_signatures_txn(self, txn, event_id):
-        """Get all the signatures for a given PDU.
-        Args:
-            txn (cursor):
-            event_id (str): Id for the Event.
-        Returns:
-            A dict of sig name -> dict(key_id -> signature_bytes)
-        """
-        query = (
-            "SELECT signature_name, key_id, signature"
-            " FROM event_signatures"
-            " WHERE event_id = ? "
-        )
-        txn.execute(query, (event_id, ))
-        rows = txn.fetchall()
-
-        res = {}
-
-        for name, key, sig in rows:
-            res.setdefault(name, {})[key] = sig
-
-        return res
-
-    def _store_event_signature_txn(self, txn, event_id, signature_name, key_id,
-                                   signature_bytes):
-        """Store a signature from the origin server for a PDU.
-        Args:
-            txn (cursor):
-            event_id (str): Id for the Event.
-            origin (str): origin of the Event.
-            key_id (str): Id for the signing key.
-            signature (bytes): The signature.
-        """
-        self._simple_insert_txn(
-            txn,
-            "event_signatures",
-            {
-                "event_id": event_id,
-                "signature_name": signature_name,
-                "key_id": key_id,
-                "signature": buffer(signature_bytes),
-            },
-        )
-
-    def _get_prev_event_hashes_txn(self, txn, event_id):
-        """Get all the hashes for previous PDUs of a PDU
-        Args:
-            txn (cursor):
-            event_id (str): Id for the Event.
-        Returns:
-            dict of (pdu_id, origin) -> dict of algorithm -> hash_bytes.
-        """
-        query = (
-            "SELECT prev_event_id, algorithm, hash"
-            " FROM event_edge_hashes"
-            " WHERE event_id = ?"
-        )
-        txn.execute(query, (event_id, ))
-        results = {}
-        for prev_event_id, algorithm, hash_bytes in txn.fetchall():
-            hashes = results.setdefault(prev_event_id, {})
-            hashes[algorithm] = hash_bytes
-        return results
-
-    def _store_prev_event_hash_txn(self, txn, event_id, prev_event_id,
-                                   algorithm, hash_bytes):
-        self._simple_insert_txn(
-            txn,
-            "event_edge_hashes",
-            {
-                "event_id": event_id,
-                "prev_event_id": prev_event_id,
-                "algorithm": algorithm,
-                "hash": buffer(hash_bytes),
-            },
-        )

From 1cd65a8d1e9fe30506aa7f19c60e6271a24e6ae9 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Wed, 23 Sep 2015 10:37:58 +0100
Subject: [PATCH 22/24] synapse/storage/state.py: _make_group_id was unused

---
 synapse/storage/state.py | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/synapse/storage/state.py b/synapse/storage/state.py
index 9630efcfcc..e935b9443b 100644
--- a/synapse/storage/state.py
+++ b/synapse/storage/state.py
@@ -20,8 +20,6 @@ from synapse.util.caches.descriptors import (
 
 from twisted.internet import defer
 
-from synapse.util.stringutils import random_string
-
 import logging
 
 logger = logging.getLogger(__name__)
@@ -428,7 +426,3 @@ class StateStore(SQLBaseStore):
             }
 
         defer.returnValue(results)
-
-
-def _make_group_id(clock):
-    return str(int(clock.time_msec())) + random_string(5)

From 7d55314277e29afb25da9c0910b8b0422d422671 Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Wed, 23 Sep 2015 10:42:02 +0100
Subject: [PATCH 23/24] Remove unused _execute_and_decode from
 scripts/synapse_port_db

---
 scripts/synapse_port_db | 2 --
 1 file changed, 2 deletions(-)

diff --git a/scripts/synapse_port_db b/scripts/synapse_port_db
index 6aba72e459..62515997b1 100755
--- a/scripts/synapse_port_db
+++ b/scripts/synapse_port_db
@@ -95,8 +95,6 @@ class Store(object):
     _simple_update_one = SQLBaseStore.__dict__["_simple_update_one"]
     _simple_update_one_txn = SQLBaseStore.__dict__["_simple_update_one_txn"]
 
-    _execute_and_decode = SQLBaseStore.__dict__["_execute_and_decode"]
-
     def runInteraction(self, desc, func, *args, **kwargs):
         def r(conn):
             try:

From 314aabba82d382d36f9ac91050e77deb6018fbcd Mon Sep 17 00:00:00 2001
From: Mark Haines <mark.haines@matrix.org>
Date: Wed, 23 Sep 2015 10:45:33 +0100
Subject: [PATCH 24/24] Fix scripts-dev/definitions.py argparse options

---
 scripts-dev/definitions.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts-dev/definitions.py b/scripts-dev/definitions.py
index 934bc4a3d8..f0d0cd8a3f 100755
--- a/scripts-dev/definitions.py
+++ b/scripts-dev/definitions.py
@@ -102,7 +102,7 @@ if __name__ == '__main__':
         "--ignore", action="append", metavar="REGEXP", help="Ignore a pattern"
     )
     parser.add_argument(
-        "--pattern", nargs='+', action="append", metavar="REGEXP",
+        "--pattern", action="append", metavar="REGEXP",
         help="Search for a pattern"
     )
     parser.add_argument(