From 6a4b650d8ad3e6c095020cac3861e430d643d53d Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 26 Aug 2015 13:22:23 +0100 Subject: [PATCH 01/82] Attempt to validate macaroons A couple of weird caveats: * If we can't validate your macaroon, we fall back to checking that your access token is in the DB, and ignoring the failure * Even if we can validate your macaroon, we still have to hit the DB to get the access token ID, which we pretend is a device ID all over the codebase. This mostly adds the interesting code, and points out the two pieces we need to delete (and necessary conditions) in order to fix the above caveats. --- synapse/api/auth.py | 104 ++++++++++++++++-- tests/api/test_auth.py | 142 ++++++++++++++++++++++++- tests/rest/client/v1/test_presence.py | 8 +- tests/rest/client/v1/test_rooms.py | 28 ++--- tests/rest/client/v1/test_typing.py | 4 +- tests/rest/client/v1/utils.py | 3 - tests/rest/client/v2_alpha/__init__.py | 4 +- 7 files changed, 257 insertions(+), 36 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 65ee1452ce..f8ea1e2c69 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -23,6 +23,7 @@ from synapse.util.logutils import log_function from synapse.types import UserID import logging +import pymacaroons logger = logging.getLogger(__name__) @@ -40,6 +41,12 @@ class Auth(object): self.store = hs.get_datastore() self.state = hs.get_state_handler() self.TOKEN_NOT_FOUND_HTTP_STATUS = 401 + self._KNOWN_CAVEAT_PREFIXES = set([ + "gen = ", + "type = ", + "time < ", + "user_id = ", + ]) def check(self, event, auth_events): """ Checks if this event is correctly authed. @@ -359,8 +366,8 @@ class Auth(object): except KeyError: pass # normal users won't have the user_id query parameter set. - user_info = yield self.get_user_by_access_token(access_token) - user = user_info["user"] + user_info = yield self._get_user_by_access_token(access_token) + user_id = user_info["user_id"] token_id = user_info["token_id"] ip_addr = self.hs.get_ip_from_request(request) @@ -368,17 +375,17 @@ class Auth(object): "User-Agent", default=[""] )[0] - if user and access_token and ip_addr: + if user_id and access_token and ip_addr: self.store.insert_client_ip( - user=user, + user=user_id, access_token=access_token, ip=ip_addr, user_agent=user_agent ) - request.authenticated_entity = user.to_string() + request.authenticated_entity = user_id.to_string() - defer.returnValue((user, token_id,)) + defer.returnValue((user_id, token_id,)) except KeyError: raise AuthError( self.TOKEN_NOT_FOUND_HTTP_STATUS, "Missing access token.", @@ -386,7 +393,7 @@ class Auth(object): ) @defer.inlineCallbacks - def get_user_by_access_token(self, token): + def _get_user_by_access_token(self, token): """ Get a registered user's ID. Args: @@ -396,6 +403,86 @@ class Auth(object): Raises: AuthError if no user by that token exists or the token is invalid. """ + try: + ret = yield self._get_user_from_macaroon(token) + except AuthError: + # TODO(daniel): Remove this fallback when all existing access tokens + # have been re-issued as macaroons. + ret = yield self._look_up_user_by_access_token(token) + defer.returnValue(ret) + + @defer.inlineCallbacks + def _get_user_from_macaroon(self, macaroon_str): + try: + macaroon = pymacaroons.Macaroon.deserialize(macaroon_str) + self._validate_macaroon(macaroon) + + user_prefix = "user_id = " + for caveat in macaroon.caveats: + if caveat.caveat_id.startswith(user_prefix): + user_id = UserID.from_string(caveat.caveat_id[len(user_prefix):]) + # This codepath exists so that we can actually return a + # token ID, because we use token IDs in place of device + # identifiers throughout the codebase. + # TODO(daniel): Remove this fallback when device IDs are + # properly implemented. + ret = yield self._look_up_user_by_access_token(macaroon_str) + if ret["user_id"] != user_id: + logger.error( + "Macaroon user (%s) != DB user (%s)", + user_id, + ret["user_id"] + ) + raise AuthError( + self.TOKEN_NOT_FOUND_HTTP_STATUS, + "User mismatch in macaroon", + errcode=Codes.UNKNOWN_TOKEN + ) + defer.returnValue(ret) + raise AuthError( + self.TOKEN_NOT_FOUND_HTTP_STATUS, "No user caveat in macaroon", + errcode=Codes.UNKNOWN_TOKEN + ) + except (pymacaroons.exceptions.MacaroonException, TypeError, ValueError): + raise AuthError( + self.TOKEN_NOT_FOUND_HTTP_STATUS, "Invalid macaroon passed.", + errcode=Codes.UNKNOWN_TOKEN + ) + + def _validate_macaroon(self, macaroon): + v = pymacaroons.Verifier() + v.satisfy_exact("gen = 1") + v.satisfy_exact("type = access") + v.satisfy_general(lambda c: c.startswith("user_id = ")) + v.satisfy_general(self._verify_expiry) + v.verify(macaroon, self.hs.config.macaroon_secret_key) + + v = pymacaroons.Verifier() + v.satisfy_general(self._verify_recognizes_caveats) + v.verify(macaroon, self.hs.config.macaroon_secret_key) + + def _verify_expiry(self, caveat): + prefix = "time < " + if not caveat.startswith(prefix): + return False + # TODO(daniel): Enable expiry check when clients actually know how to + # refresh tokens. (And remember to enable the tests) + return True + expiry = int(caveat[len(prefix):]) + now = self.hs.get_clock().time_msec() + return now < expiry + + def _verify_recognizes_caveats(self, caveat): + first_space = caveat.find(" ") + if first_space < 0: + return False + second_space = caveat.find(" ", first_space + 1) + if second_space < 0: + return False + return caveat[:second_space + 1] in self._KNOWN_CAVEAT_PREFIXES + + @defer.inlineCallbacks + def _look_up_user_by_access_token(self, token): ret = yield self.store.get_user_by_access_token(token) if not ret: raise AuthError( @@ -403,10 +490,9 @@ class Auth(object): errcode=Codes.UNKNOWN_TOKEN ) user_info = { - "user": UserID.from_string(ret.get("name")), + "user_id": UserID.from_string(ret.get("name")), "token_id": ret.get("token_id", None), } - defer.returnValue(user_info) @defer.inlineCallbacks diff --git a/tests/api/test_auth.py b/tests/api/test_auth.py index 22fc804331..1ba85d6f83 100644 --- a/tests/api/test_auth.py +++ b/tests/api/test_auth.py @@ -14,22 +14,27 @@ # limitations under the License. from tests import unittest from twisted.internet import defer +from twisted.trial.unittest import FailTest from mock import Mock from synapse.api.auth import Auth from synapse.api.errors import AuthError +from synapse.types import UserID +from tests.utils import setup_test_homeserver + +import pymacaroons class AuthTestCase(unittest.TestCase): + @defer.inlineCallbacks def setUp(self): self.state_handler = Mock() self.store = Mock() - self.hs = Mock() + self.hs = yield setup_test_homeserver(handlers=None) self.hs.get_datastore = Mock(return_value=self.store) - self.hs.get_state_handler = Mock(return_value=self.state_handler) self.auth = Auth(self.hs) self.test_user = "@foo:bar" @@ -133,3 +138,136 @@ class AuthTestCase(unittest.TestCase): request.requestHeaders.getRawHeaders = Mock(return_value=[""]) d = self.auth.get_user_by_req(request) self.failureResultOf(d, AuthError) + + @defer.inlineCallbacks + def test_get_user_from_macaroon(self): + # TODO(danielwh): Remove this mock when we remove the + # get_user_by_access_token fallback. + self.store.get_user_by_access_token = Mock( + return_value={"name": "@baldrick:matrix.org"} + ) + + user = "@baldrick:matrix.org" + macaroon = pymacaroons.Macaroon( + location=self.hs.config.server_name, + identifier="key", + key=self.hs.config.macaroon_secret_key) + macaroon.add_first_party_caveat("gen = 1") + macaroon.add_first_party_caveat("type = access") + macaroon.add_first_party_caveat("user_id = %s" % (user,)) + user_info = yield self.auth._get_user_from_macaroon(macaroon.serialize()) + user_id = user_info["user_id"] + self.assertEqual(UserID.from_string(user), user_id) + + @defer.inlineCallbacks + def test_get_user_from_macaroon_user_db_mismatch(self): + self.store.get_user_by_access_token = Mock( + return_value={"name": "@percy:matrix.org"} + ) + + user = "@baldrick:matrix.org" + macaroon = pymacaroons.Macaroon( + location=self.hs.config.server_name, + identifier="key", + key=self.hs.config.macaroon_secret_key) + macaroon.add_first_party_caveat("gen = 1") + macaroon.add_first_party_caveat("type = access") + macaroon.add_first_party_caveat("user_id = %s" % (user,)) + with self.assertRaises(AuthError) as cm: + yield self.auth._get_user_from_macaroon(macaroon.serialize()) + self.assertEqual(401, cm.exception.code) + self.assertIn("User mismatch", cm.exception.msg) + + @defer.inlineCallbacks + def test_get_user_from_macaroon_missing_caveat(self): + # TODO(danielwh): Remove this mock when we remove the + # get_user_by_access_token fallback. + self.store.get_user_by_access_token = Mock( + return_value={"name": "@baldrick:matrix.org"} + ) + + macaroon = pymacaroons.Macaroon( + location=self.hs.config.server_name, + identifier="key", + key=self.hs.config.macaroon_secret_key) + macaroon.add_first_party_caveat("gen = 1") + macaroon.add_first_party_caveat("type = access") + + with self.assertRaises(AuthError) as cm: + yield self.auth._get_user_from_macaroon(macaroon.serialize()) + self.assertEqual(401, cm.exception.code) + self.assertIn("No user caveat", cm.exception.msg) + + @defer.inlineCallbacks + def test_get_user_from_macaroon_wrong_key(self): + # TODO(danielwh): Remove this mock when we remove the + # get_user_by_access_token fallback. + self.store.get_user_by_access_token = Mock( + return_value={"name": "@baldrick:matrix.org"} + ) + + user = "@baldrick:matrix.org" + macaroon = pymacaroons.Macaroon( + location=self.hs.config.server_name, + identifier="key", + key=self.hs.config.macaroon_secret_key + "wrong") + macaroon.add_first_party_caveat("gen = 1") + macaroon.add_first_party_caveat("type = access") + macaroon.add_first_party_caveat("user_id = %s" % (user,)) + + with self.assertRaises(AuthError) as cm: + yield self.auth._get_user_from_macaroon(macaroon.serialize()) + self.assertEqual(401, cm.exception.code) + self.assertIn("Invalid macaroon", cm.exception.msg) + + @defer.inlineCallbacks + def test_get_user_from_macaroon_unknown_caveat(self): + # TODO(danielwh): Remove this mock when we remove the + # get_user_by_access_token fallback. + self.store.get_user_by_access_token = Mock( + return_value={"name": "@baldrick:matrix.org"} + ) + + user = "@baldrick:matrix.org" + macaroon = pymacaroons.Macaroon( + location=self.hs.config.server_name, + identifier="key", + key=self.hs.config.macaroon_secret_key) + macaroon.add_first_party_caveat("gen = 1") + macaroon.add_first_party_caveat("type = access") + macaroon.add_first_party_caveat("user_id = %s" % (user,)) + macaroon.add_first_party_caveat("cunning > fox") + + with self.assertRaises(AuthError) as cm: + yield self.auth._get_user_from_macaroon(macaroon.serialize()) + self.assertEqual(401, cm.exception.code) + self.assertIn("Invalid macaroon", cm.exception.msg) + + @defer.inlineCallbacks + def test_get_user_from_macaroon_expired(self): + # TODO(danielwh): Remove this mock when we remove the + # get_user_by_access_token fallback. + self.store.get_user_by_access_token = Mock( + return_value={"name": "@baldrick:matrix.org"} + ) + + self.todo = (FailTest, "Token expiry isn't currently enabled",) + self.store.get_user_by_access_token = Mock( + return_value={"name": "@baldrick:matrix.org"} + ) + + user = "@baldrick:matrix.org" + macaroon = pymacaroons.Macaroon( + location=self.hs.config.server_name, + identifier="key", + key=self.hs.config.macaroon_secret_key) + macaroon.add_first_party_caveat("gen = 1") + macaroon.add_first_party_caveat("type = access") + macaroon.add_first_party_caveat("user_id = %s" % (user,)) + macaroon.add_first_party_caveat("time < 1") # ms + + self.hs.clock.now = 5000 # seconds + with self.assertRaises(AuthError) as cm: + yield self.auth._get_user_from_macaroon(macaroon.serialize()) + self.assertEqual(401, cm.exception.code) + self.assertIn("Invalid macaroon", cm.exception.msg) diff --git a/tests/rest/client/v1/test_presence.py b/tests/rest/client/v1/test_presence.py index 91547bdd06..d8d1416f59 100644 --- a/tests/rest/client/v1/test_presence.py +++ b/tests/rest/client/v1/test_presence.py @@ -72,11 +72,11 @@ class PresenceStateTestCase(unittest.TestCase): def _get_user_by_access_token(token=None): return { - "user": UserID.from_string(myid), + "user_id": UserID.from_string(myid), "token_id": 1, } - hs.get_v1auth().get_user_by_access_token = _get_user_by_access_token + hs.get_v1auth()._get_user_by_access_token = _get_user_by_access_token room_member_handler = hs.handlers.room_member_handler = Mock( spec=[ @@ -159,7 +159,7 @@ class PresenceListTestCase(unittest.TestCase): def _get_user_by_access_token(token=None): return { - "user": UserID.from_string(myid), + "user_id": UserID.from_string(myid), "token_id": 1, } @@ -169,7 +169,7 @@ class PresenceListTestCase(unittest.TestCase): ] ) - hs.get_v1auth().get_user_by_access_token = _get_user_by_access_token + hs.get_v1auth()._get_user_by_access_token = _get_user_by_access_token presence.register_servlets(hs, self.mock_resource) diff --git a/tests/rest/client/v1/test_rooms.py b/tests/rest/client/v1/test_rooms.py index 34ab47d02e..be1d52f720 100644 --- a/tests/rest/client/v1/test_rooms.py +++ b/tests/rest/client/v1/test_rooms.py @@ -56,10 +56,10 @@ class RoomPermissionsTestCase(RestTestCase): def _get_user_by_access_token(token=None): return { - "user": UserID.from_string(self.auth_user_id), + "user_id": UserID.from_string(self.auth_user_id), "token_id": 1, } - hs.get_v1auth().get_user_by_access_token = _get_user_by_access_token + hs.get_v1auth()._get_user_by_access_token = _get_user_by_access_token def _insert_client_ip(*args, **kwargs): return defer.succeed(None) @@ -441,10 +441,10 @@ class RoomsMemberListTestCase(RestTestCase): def _get_user_by_access_token(token=None): return { - "user": UserID.from_string(self.auth_user_id), + "user_id": UserID.from_string(self.auth_user_id), "token_id": 1, } - hs.get_v1auth().get_user_by_access_token = _get_user_by_access_token + hs.get_v1auth()._get_user_by_access_token = _get_user_by_access_token def _insert_client_ip(*args, **kwargs): return defer.succeed(None) @@ -519,10 +519,10 @@ class RoomsCreateTestCase(RestTestCase): def _get_user_by_access_token(token=None): return { - "user": UserID.from_string(self.auth_user_id), + "user_id": UserID.from_string(self.auth_user_id), "token_id": 1, } - hs.get_v1auth().get_user_by_access_token = _get_user_by_access_token + hs.get_v1auth()._get_user_by_access_token = _get_user_by_access_token def _insert_client_ip(*args, **kwargs): return defer.succeed(None) @@ -610,11 +610,11 @@ class RoomTopicTestCase(RestTestCase): def _get_user_by_access_token(token=None): return { - "user": UserID.from_string(self.auth_user_id), + "user_id": UserID.from_string(self.auth_user_id), "token_id": 1, } - hs.get_v1auth().get_user_by_access_token = _get_user_by_access_token + hs.get_v1auth()._get_user_by_access_token = _get_user_by_access_token def _insert_client_ip(*args, **kwargs): return defer.succeed(None) @@ -715,10 +715,10 @@ class RoomMemberStateTestCase(RestTestCase): def _get_user_by_access_token(token=None): return { - "user": UserID.from_string(self.auth_user_id), + "user_id": UserID.from_string(self.auth_user_id), "token_id": 1, } - hs.get_v1auth().get_user_by_access_token = _get_user_by_access_token + hs.get_v1auth()._get_user_by_access_token = _get_user_by_access_token def _insert_client_ip(*args, **kwargs): return defer.succeed(None) @@ -840,10 +840,10 @@ class RoomMessagesTestCase(RestTestCase): def _get_user_by_access_token(token=None): return { - "user": UserID.from_string(self.auth_user_id), + "user_id": UserID.from_string(self.auth_user_id), "token_id": 1, } - hs.get_v1auth().get_user_by_access_token = _get_user_by_access_token + hs.get_v1auth()._get_user_by_access_token = _get_user_by_access_token def _insert_client_ip(*args, **kwargs): return defer.succeed(None) @@ -935,10 +935,10 @@ class RoomInitialSyncTestCase(RestTestCase): def _get_user_by_access_token(token=None): return { - "user": UserID.from_string(self.auth_user_id), + "user_id": UserID.from_string(self.auth_user_id), "token_id": 1, } - hs.get_v1auth().get_user_by_access_token = _get_user_by_access_token + hs.get_v1auth()._get_user_by_access_token = _get_user_by_access_token def _insert_client_ip(*args, **kwargs): return defer.succeed(None) diff --git a/tests/rest/client/v1/test_typing.py b/tests/rest/client/v1/test_typing.py index 1c4519406d..da6fc975f7 100644 --- a/tests/rest/client/v1/test_typing.py +++ b/tests/rest/client/v1/test_typing.py @@ -63,11 +63,11 @@ class RoomTypingTestCase(RestTestCase): def _get_user_by_access_token(token=None): return { - "user": UserID.from_string(self.auth_user_id), + "user_id": UserID.from_string(self.auth_user_id), "token_id": 1, } - hs.get_v1auth().get_user_by_access_token = _get_user_by_access_token + hs.get_v1auth()._get_user_by_access_token = _get_user_by_access_token def _insert_client_ip(*args, **kwargs): return defer.succeed(None) diff --git a/tests/rest/client/v1/utils.py b/tests/rest/client/v1/utils.py index c472d53043..85096a0326 100644 --- a/tests/rest/client/v1/utils.py +++ b/tests/rest/client/v1/utils.py @@ -37,9 +37,6 @@ class RestTestCase(unittest.TestCase): self.mock_resource = None self.auth_user_id = None - def mock_get_user_by_access_token(self, token=None): - return self.auth_user_id - @defer.inlineCallbacks def create_room_as(self, room_creator, is_public=True, tok=None): temp_id = self.auth_user_id diff --git a/tests/rest/client/v2_alpha/__init__.py b/tests/rest/client/v2_alpha/__init__.py index ef972a53aa..7d0f77a3ee 100644 --- a/tests/rest/client/v2_alpha/__init__.py +++ b/tests/rest/client/v2_alpha/__init__.py @@ -45,10 +45,10 @@ class V2AlphaRestTestCase(unittest.TestCase): def _get_user_by_access_token(token=None): return { - "user": UserID.from_string(self.USER_ID), + "user_id": UserID.from_string(self.USER_ID), "token_id": 1, } - hs.get_auth().get_user_by_access_token = _get_user_by_access_token + hs.get_auth()._get_user_by_access_token = _get_user_by_access_token for r in self.TO_REGISTER: r.register_servlets(hs, self.mock_resource) From 81450fded8c4d2a0f4a914251cc2d11a366efdbd Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 26 Aug 2015 13:56:01 +0100 Subject: [PATCH 02/82] Turn TODO into thing which actually will fail --- tests/api/test_auth.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/api/test_auth.py b/tests/api/test_auth.py index 1ba85d6f83..2e2d0c428a 100644 --- a/tests/api/test_auth.py +++ b/tests/api/test_auth.py @@ -14,7 +14,6 @@ # limitations under the License. from tests import unittest from twisted.internet import defer -from twisted.trial.unittest import FailTest from mock import Mock @@ -251,7 +250,6 @@ class AuthTestCase(unittest.TestCase): return_value={"name": "@baldrick:matrix.org"} ) - self.todo = (FailTest, "Token expiry isn't currently enabled",) self.store.get_user_by_access_token = Mock( return_value={"name": "@baldrick:matrix.org"} ) @@ -267,7 +265,12 @@ class AuthTestCase(unittest.TestCase): macaroon.add_first_party_caveat("time < 1") # ms self.hs.clock.now = 5000 # seconds - with self.assertRaises(AuthError) as cm: - yield self.auth._get_user_from_macaroon(macaroon.serialize()) - self.assertEqual(401, cm.exception.code) - self.assertIn("Invalid macaroon", cm.exception.msg) + + yield self.auth._get_user_from_macaroon(macaroon.serialize()) + # TODO(daniel): Turn on the check that we validate expiration, when we + # validate expiration (and remove the above line, which will start + # throwing). + # with self.assertRaises(AuthError) as cm: + # yield self.auth._get_user_from_macaroon(macaroon.serialize()) + # self.assertEqual(401, cm.exception.code) + # self.assertIn("Invalid macaroon", cm.exception.msg) From e255c2c32ff85db03abbf2dac184b2949f481cfb Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 1 Sep 2015 12:41:16 +0100 Subject: [PATCH 03/82] s/user_id/user/g for consistency --- synapse/api/auth.py | 20 ++++++++++---------- tests/api/test_auth.py | 8 ++++---- tests/rest/client/v1/test_presence.py | 4 ++-- tests/rest/client/v1/test_rooms.py | 14 +++++++------- tests/rest/client/v1/test_typing.py | 2 +- tests/rest/client/v2_alpha/__init__.py | 2 +- 6 files changed, 25 insertions(+), 25 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index f8ea1e2c69..0a77a76cb8 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -367,7 +367,7 @@ class Auth(object): pass # normal users won't have the user_id query parameter set. user_info = yield self._get_user_by_access_token(access_token) - user_id = user_info["user_id"] + user = user_info["user"] token_id = user_info["token_id"] ip_addr = self.hs.get_ip_from_request(request) @@ -375,17 +375,17 @@ class Auth(object): "User-Agent", default=[""] )[0] - if user_id and access_token and ip_addr: + if user and access_token and ip_addr: self.store.insert_client_ip( - user=user_id, + user=user, access_token=access_token, ip=ip_addr, user_agent=user_agent ) - request.authenticated_entity = user_id.to_string() + request.authenticated_entity = user.to_string() - defer.returnValue((user_id, token_id,)) + defer.returnValue((user, token_id,)) except KeyError: raise AuthError( self.TOKEN_NOT_FOUND_HTTP_STATUS, "Missing access token.", @@ -420,18 +420,18 @@ class Auth(object): user_prefix = "user_id = " for caveat in macaroon.caveats: if caveat.caveat_id.startswith(user_prefix): - user_id = UserID.from_string(caveat.caveat_id[len(user_prefix):]) + user = UserID.from_string(caveat.caveat_id[len(user_prefix):]) # This codepath exists so that we can actually return a # token ID, because we use token IDs in place of device # identifiers throughout the codebase. # TODO(daniel): Remove this fallback when device IDs are # properly implemented. ret = yield self._look_up_user_by_access_token(macaroon_str) - if ret["user_id"] != user_id: + if ret["user"] != user: logger.error( "Macaroon user (%s) != DB user (%s)", - user_id, - ret["user_id"] + user, + ret["user"] ) raise AuthError( self.TOKEN_NOT_FOUND_HTTP_STATUS, @@ -490,7 +490,7 @@ class Auth(object): errcode=Codes.UNKNOWN_TOKEN ) user_info = { - "user_id": UserID.from_string(ret.get("name")), + "user": UserID.from_string(ret.get("name")), "token_id": ret.get("token_id", None), } defer.returnValue(user_info) diff --git a/tests/api/test_auth.py b/tests/api/test_auth.py index 2e2d0c428a..c96273480d 100644 --- a/tests/api/test_auth.py +++ b/tests/api/test_auth.py @@ -146,17 +146,17 @@ class AuthTestCase(unittest.TestCase): return_value={"name": "@baldrick:matrix.org"} ) - user = "@baldrick:matrix.org" + user_id = "@baldrick:matrix.org" macaroon = pymacaroons.Macaroon( location=self.hs.config.server_name, identifier="key", key=self.hs.config.macaroon_secret_key) macaroon.add_first_party_caveat("gen = 1") macaroon.add_first_party_caveat("type = access") - macaroon.add_first_party_caveat("user_id = %s" % (user,)) + macaroon.add_first_party_caveat("user_id = %s" % (user_id,)) user_info = yield self.auth._get_user_from_macaroon(macaroon.serialize()) - user_id = user_info["user_id"] - self.assertEqual(UserID.from_string(user), user_id) + user = user_info["user"] + self.assertEqual(UserID.from_string(user_id), user) @defer.inlineCallbacks def test_get_user_from_macaroon_user_db_mismatch(self): diff --git a/tests/rest/client/v1/test_presence.py b/tests/rest/client/v1/test_presence.py index d8d1416f59..2ee3da0b34 100644 --- a/tests/rest/client/v1/test_presence.py +++ b/tests/rest/client/v1/test_presence.py @@ -72,7 +72,7 @@ class PresenceStateTestCase(unittest.TestCase): def _get_user_by_access_token(token=None): return { - "user_id": UserID.from_string(myid), + "user": UserID.from_string(myid), "token_id": 1, } @@ -159,7 +159,7 @@ class PresenceListTestCase(unittest.TestCase): def _get_user_by_access_token(token=None): return { - "user_id": UserID.from_string(myid), + "user": UserID.from_string(myid), "token_id": 1, } diff --git a/tests/rest/client/v1/test_rooms.py b/tests/rest/client/v1/test_rooms.py index be1d52f720..9fb2bfb315 100644 --- a/tests/rest/client/v1/test_rooms.py +++ b/tests/rest/client/v1/test_rooms.py @@ -56,7 +56,7 @@ class RoomPermissionsTestCase(RestTestCase): def _get_user_by_access_token(token=None): return { - "user_id": UserID.from_string(self.auth_user_id), + "user": UserID.from_string(self.auth_user_id), "token_id": 1, } hs.get_v1auth()._get_user_by_access_token = _get_user_by_access_token @@ -441,7 +441,7 @@ class RoomsMemberListTestCase(RestTestCase): def _get_user_by_access_token(token=None): return { - "user_id": UserID.from_string(self.auth_user_id), + "user": UserID.from_string(self.auth_user_id), "token_id": 1, } hs.get_v1auth()._get_user_by_access_token = _get_user_by_access_token @@ -519,7 +519,7 @@ class RoomsCreateTestCase(RestTestCase): def _get_user_by_access_token(token=None): return { - "user_id": UserID.from_string(self.auth_user_id), + "user": UserID.from_string(self.auth_user_id), "token_id": 1, } hs.get_v1auth()._get_user_by_access_token = _get_user_by_access_token @@ -610,7 +610,7 @@ class RoomTopicTestCase(RestTestCase): def _get_user_by_access_token(token=None): return { - "user_id": UserID.from_string(self.auth_user_id), + "user": UserID.from_string(self.auth_user_id), "token_id": 1, } @@ -715,7 +715,7 @@ class RoomMemberStateTestCase(RestTestCase): def _get_user_by_access_token(token=None): return { - "user_id": UserID.from_string(self.auth_user_id), + "user": UserID.from_string(self.auth_user_id), "token_id": 1, } hs.get_v1auth()._get_user_by_access_token = _get_user_by_access_token @@ -840,7 +840,7 @@ class RoomMessagesTestCase(RestTestCase): def _get_user_by_access_token(token=None): return { - "user_id": UserID.from_string(self.auth_user_id), + "user": UserID.from_string(self.auth_user_id), "token_id": 1, } hs.get_v1auth()._get_user_by_access_token = _get_user_by_access_token @@ -935,7 +935,7 @@ class RoomInitialSyncTestCase(RestTestCase): def _get_user_by_access_token(token=None): return { - "user_id": UserID.from_string(self.auth_user_id), + "user": UserID.from_string(self.auth_user_id), "token_id": 1, } hs.get_v1auth()._get_user_by_access_token = _get_user_by_access_token diff --git a/tests/rest/client/v1/test_typing.py b/tests/rest/client/v1/test_typing.py index da6fc975f7..6395ce79db 100644 --- a/tests/rest/client/v1/test_typing.py +++ b/tests/rest/client/v1/test_typing.py @@ -63,7 +63,7 @@ class RoomTypingTestCase(RestTestCase): def _get_user_by_access_token(token=None): return { - "user_id": UserID.from_string(self.auth_user_id), + "user": UserID.from_string(self.auth_user_id), "token_id": 1, } diff --git a/tests/rest/client/v2_alpha/__init__.py b/tests/rest/client/v2_alpha/__init__.py index 7d0f77a3ee..f45570a1c0 100644 --- a/tests/rest/client/v2_alpha/__init__.py +++ b/tests/rest/client/v2_alpha/__init__.py @@ -45,7 +45,7 @@ class V2AlphaRestTestCase(unittest.TestCase): def _get_user_by_access_token(token=None): return { - "user_id": UserID.from_string(self.USER_ID), + "user": UserID.from_string(self.USER_ID), "token_id": 1, } hs.get_auth()._get_user_by_access_token = _get_user_by_access_token From 3cdfd37d95484b9ebca012a8169b8d6613385656 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 1 Sep 2015 16:47:26 +0100 Subject: [PATCH 04/82] Print an example "pip install" line for a missing requirement --- synapse/app/homeserver.py | 17 +++++++++++++++-- synapse/python_dependencies.py | 17 +++++++++++------ 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index ffc6299146..2a85932b86 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -16,10 +16,23 @@ import sys sys.dont_write_bytecode = True -from synapse.python_dependencies import check_requirements, DEPENDENCY_LINKS +from synapse.python_dependencies import ( + check_requirements, DEPENDENCY_LINKS, MissingRequirementError +) if __name__ == '__main__': - check_requirements() + try: + check_requirements() + except MissingRequirementError as e: + message = "\n".join([ + "Missing Requirement: %s" % (e.message,), + "To install run:", + " pip install --upgrade --force \"%s\"" % (e.dependency,), + "", + ]) + sys.stderr.writelines(message) + sys.exit(1) + from synapse.storage.engines import create_engine, IncorrectDatabaseSetup from synapse.storage import ( diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 795ef27182..387183b50b 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -60,7 +60,10 @@ DEPENDENCY_LINKS = { class MissingRequirementError(Exception): - pass + def __init__(self, message, module_name, dependency): + super(MissingRequirementError, self).__init__(message) + self.module_name = module_name + self.dependency = dependency def check_requirements(config=None): @@ -88,7 +91,7 @@ def check_requirements(config=None): ) raise MissingRequirementError( "Can't import %r which is part of %r" - % (module_name, dependency) + % (module_name, dependency), module_name, dependency ) version = getattr(module, "__version__", None) file_path = getattr(module, "__file__", None) @@ -101,23 +104,25 @@ def check_requirements(config=None): if version is None: raise MissingRequirementError( "Version of %r isn't set as __version__ of module %r" - % (dependency, module_name) + % (dependency, module_name), module_name, dependency ) if LooseVersion(version) < LooseVersion(required_version): raise MissingRequirementError( "Version of %r in %r is too old. %r < %r" - % (dependency, file_path, version, required_version) + % (dependency, file_path, version, required_version), + module_name, dependency ) elif version_test == "==": if version is None: raise MissingRequirementError( "Version of %r isn't set as __version__ of module %r" - % (dependency, module_name) + % (dependency, module_name), module_name, dependency ) if LooseVersion(version) != LooseVersion(required_version): raise MissingRequirementError( "Unexpected version of %r in %r. %r != %r" - % (dependency, file_path, version, required_version) + % (dependency, file_path, version, required_version), + module_name, dependency ) From 8bab7abddda956b4131d8e4ad04b623c93df02ad Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 1 Sep 2015 16:51:10 +0100 Subject: [PATCH 05/82] Add nacl.bindings to the list of modules checked. Re-arrange import order to check packages after the packages they depend on --- synapse/python_dependencies.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 387183b50b..e95316720e 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -18,18 +18,18 @@ from distutils.version import LooseVersion logger = logging.getLogger(__name__) REQUIREMENTS = { + "frozendict>=0.4": ["frozendict"], "unpaddedbase64>=1.0.1": ["unpaddedbase64>=1.0.1"], "canonicaljson>=1.0.0": ["canonicaljson>=1.0.0"], "signedjson>=1.0.0": ["signedjson>=1.0.0"], - "Twisted>=15.1.0": ["twisted>=15.1.0"], + "pynacl>=0.3.0": ["nacl>=0.3.0", "nacl.bindings"], "service_identity>=1.0.0": ["service_identity>=1.0.0"], + "Twisted>=15.1.0": ["twisted>=15.1.0"], "pyopenssl>=0.14": ["OpenSSL>=0.14"], "pyyaml": ["yaml"], "pyasn1": ["pyasn1"], - "pynacl>=0.3.0": ["nacl>=0.3.0"], "daemonize": ["daemonize"], "py-bcrypt": ["bcrypt"], - "frozendict>=0.4": ["frozendict"], "pillow": ["PIL"], "pydenticon": ["pydenticon"], "ujson": ["ujson"], From 2ff439cff7dd666beeaaeae647a34031e285331e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 8 Sep 2015 11:01:48 +0100 Subject: [PATCH 06/82] Bump version/changelog --- CHANGES.rst | 5 +++++ synapse/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 2ec10516fd..a0a12ca4a0 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,8 @@ +Changes in synapse v0.10.0-r1 (2015-09-08) +========================================== + +* Fix bug with python packaging + Changes in synapse v0.10.0 (2015-09-03) ======================================= diff --git a/synapse/__init__.py b/synapse/__init__.py index d85bb3dce0..5b5ff6c3f3 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.10.0" +__version__ = "0.10.0-r1" From 417485eefaff86206b5f961102f882b3fbe44651 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 8 Sep 2015 18:14:54 +0100 Subject: [PATCH 07/82] Include the event_id and stream_ordering of membership events when looking up which rooms a user is in --- synapse/storage/roommember.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 8eee2dfbcc..cd9eefbd9f 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -30,7 +30,7 @@ logger = logging.getLogger(__name__) RoomsForUser = namedtuple( "RoomsForUser", - ("room_id", "sender", "membership") + ("room_id", "sender", "membership", "event_id", "stream_ordering") ) @@ -141,9 +141,11 @@ class RoomMemberStore(SQLBaseStore): args.extend(membership_list) sql = ( - "SELECT m.room_id, m.sender, m.membership" + "SELECT m.room_id, m.sender, m.membership, m.event_id, e.stream_ordering" " FROM room_memberships as m" " INNER JOIN current_state_events as c" + " ON e.event_id = c.event_id " + " INNER JOIN events as e " " ON m.event_id = c.event_id " " AND m.room_id = c.room_id " " AND m.user_id = c.state_key" From dd42bb78d088c719aaa288b892fe58caa5850deb Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 8 Sep 2015 18:16:09 +0100 Subject: [PATCH 08/82] Include rooms that a user has left in an initialSync. Include the state and messages at the point they left the room --- synapse/handlers/message.py | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 23b779ad7c..5447c97e83 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -316,7 +316,9 @@ class MessageHandler(BaseHandler): """ room_list = yield self.store.get_rooms_for_user_where_membership_is( user_id=user_id, - membership_list=[Membership.INVITE, Membership.JOIN] + membership_list=[ + Membership.INVITE, Membership.JOIN, Membership.LEAVE + ] ) user = UserID.from_string(user_id) @@ -358,19 +360,32 @@ class MessageHandler(BaseHandler): rooms_ret.append(d) - if event.membership != Membership.JOIN: + if event.membership not in (Membership.JOIN, Membership.LEAVE): return try: + if event.membership == Membership.JOIN: + room_end_token = now_token.room_key + deferred_room_state = self.state_handler.get_current_state( + event.room_id + ) + else: + room_end_token = "s%d" % (event.stream_ordering,) + deferred_room_state = self.store.get_state_for_events( + event.room_id, [event.event_id], None + ) + deferred_room_state.addCallback( + lambda states: states[event.event_id] + ) + + (messages, token), current_state = yield defer.gatherResults( [ self.store.get_recent_events_for_room( event.room_id, limit=limit, - end_token=now_token.room_key, - ), - self.state_handler.get_current_state( - event.room_id + end_token=room_end_token, ), + deferred_room_state, ] ).addErrback(unwrapFirstError) From e530208e68a2d0c37732855e4ed8f22a9f84d334 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 9 Sep 2015 09:57:49 +0100 Subject: [PATCH 09/82] Change default history visibility for private rooms --- synapse/handlers/room.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index c5d1001b50..bb3d428288 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -39,7 +39,7 @@ class RoomCreationHandler(BaseHandler): PRESETS_DICT = { RoomCreationPreset.PRIVATE_CHAT: { "join_rules": JoinRules.INVITE, - "history_visibility": "invited", + "history_visibility": "shared", "original_invitees_have_ops": False, }, RoomCreationPreset.PUBLIC_CHAT: { From 81a93ddcc8798568276582ed9c7a63bc64dc5bc0 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 9 Sep 2015 12:02:07 +0100 Subject: [PATCH 10/82] Allow configuration to ignore invalid SSL certs This will be useful for sytest, and sytest only, hence the aggressive config key name. --- synapse/app/homeserver.py | 8 ++++---- synapse/config/tls.py | 4 ++++ synapse/crypto/keyring.py | 4 ++-- synapse/handlers/auth.py | 3 +-- synapse/http/client.py | 25 +++++++++++++++++++++++-- synapse/http/matrixfederationclient.py | 4 ++-- synapse/server.py | 14 ++++++++++++++ 7 files changed, 50 insertions(+), 12 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index ffc6299146..ba76ee362a 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -15,6 +15,7 @@ # limitations under the License. import sys + sys.dont_write_bytecode = True from synapse.python_dependencies import check_requirements, DEPENDENCY_LINKS @@ -221,7 +222,7 @@ class SynapseHomeServer(HomeServer): listener_config, root_resource, ), - self.tls_context_factory, + self.tls_server_context_factory, interface=bind_address ) else: @@ -365,7 +366,6 @@ def setup(config_options): Args: config_options_options: The options passed to Synapse. Usually `sys.argv[1:]`. - should_run (bool): Whether to start the reactor. Returns: HomeServer @@ -388,7 +388,7 @@ def setup(config_options): events.USE_FROZEN_DICTS = config.use_frozen_dicts - tls_context_factory = context_factory.ServerContextFactory(config) + tls_server_context_factory = context_factory.ServerContextFactory(config) database_engine = create_engine(config.database_config["name"]) config.database_config["args"]["cp_openfun"] = database_engine.on_new_connection @@ -396,7 +396,7 @@ def setup(config_options): hs = SynapseHomeServer( config.server_name, db_config=config.database_config, - tls_context_factory=tls_context_factory, + tls_server_context_factory=tls_server_context_factory, config=config, content_addr=config.content_addr, version_string=version_string, diff --git a/synapse/config/tls.py b/synapse/config/tls.py index 4751d39bc9..472cf7ac4a 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -42,6 +42,10 @@ class TlsConfig(Config): config.get("tls_dh_params_path"), "tls_dh_params" ) + self.use_insecure_ssl_client = config.get( + "i_really_want_to_ignore_ssl_certs_when_i_am_an_http_client_even_" + "though_it_is_woefully_insecure_because_i_hate_my_users", False) + def default_config(self, config_dir_path, server_name): base_key_name = os.path.join(config_dir_path, server_name) diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index a692cdbe55..e98a625fea 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -463,7 +463,7 @@ class Keyring(object): continue (response, tls_certificate) = yield fetch_server_key( - server_name, self.hs.tls_context_factory, + server_name, self.hs.tls_server_context_factory, path=(b"/_matrix/key/v2/server/%s" % ( urllib.quote(requested_key_id), )).encode("ascii"), @@ -597,7 +597,7 @@ class Keyring(object): # Try to fetch the key from the remote server. (response, tls_certificate) = yield fetch_server_key( - server_name, self.hs.tls_context_factory + server_name, self.hs.tls_server_context_factory ) # Check the response. diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 59f687e0f1..793b3fcd8b 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -19,7 +19,6 @@ from ._base import BaseHandler from synapse.api.constants import LoginType from synapse.types import UserID from synapse.api.errors import LoginError, Codes -from synapse.http.client import SimpleHttpClient from synapse.util.async import run_on_reactor from twisted.web.client import PartialDownloadError @@ -187,7 +186,7 @@ class AuthHandler(BaseHandler): # TODO: get this from the homeserver rather than creating a new one for # each request try: - client = SimpleHttpClient(self.hs) + client = self.hs.get_simple_http_client() resp_body = yield client.post_urlencoded_get_json( self.hs.config.recaptcha_siteverify_api, args={ diff --git a/synapse/http/client.py b/synapse/http/client.py index 4b8fd3d3a3..da77c8b0ac 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -12,6 +12,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from OpenSSL import SSL +from OpenSSL.SSL import VERIFY_NONE from synapse.api.errors import CodeMessageException from synapse.util.logcontext import preserve_context_over_fn @@ -19,7 +21,7 @@ import synapse.metrics from canonicaljson import encode_canonical_json -from twisted.internet import defer, reactor +from twisted.internet import defer, reactor, ssl from twisted.web.client import ( Agent, readBody, FileBodyProducer, PartialDownloadError, HTTPConnectionPool, @@ -59,7 +61,12 @@ class SimpleHttpClient(object): # 'like a browser' pool = HTTPConnectionPool(reactor) pool.maxPersistentPerHost = 10 - self.agent = Agent(reactor, pool=pool) + self.agent = Agent( + reactor, + pool=pool, + connectTimeout=15, + contextFactory=hs.get_http_client_context_factory() + ) self.version_string = hs.version_string def request(self, method, uri, *args, **kwargs): @@ -252,3 +259,17 @@ def _print_ex(e): _print_ex(ex) else: logger.exception(e) + + +class WoefullyInsecureContextFactory(ssl.ContextFactory): + """ + Factory for PyOpenSSL SSL contexts which does absolutely no certificate verification. + + Do not use this unless you really, really hate your users.""" + + def __init__(self): + self._context = SSL.Context(SSL.SSLv23_METHOD) + self._context.set_verify(VERIFY_NONE, lambda *_: None) + + def getContext(self, hostname, port): + return self._context diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 1c9e552788..b50a0c445c 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -57,14 +57,14 @@ incoming_responses_counter = metrics.register_counter( class MatrixFederationEndpointFactory(object): def __init__(self, hs): - self.tls_context_factory = hs.tls_context_factory + self.tls_server_context_factory = hs.tls_server_context_factory def endpointForURI(self, uri): destination = uri.netloc return matrix_federation_endpoint( reactor, destination, timeout=10, - ssl_context_factory=self.tls_context_factory + ssl_context_factory=self.tls_server_context_factory ) diff --git a/synapse/server.py b/synapse/server.py index 4d1fb1cbf6..656e534dff 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -19,7 +19,9 @@ # partial one for unit test mocking. # Imports required for the default HomeServer() implementation +from twisted.web.client import BrowserLikePolicyForHTTPS from synapse.federation import initialize_http_replication +from synapse.http.client import SimpleHttpClient, WoefullyInsecureContextFactory from synapse.notifier import Notifier from synapse.api.auth import Auth from synapse.handlers import Handlers @@ -87,6 +89,8 @@ class BaseHomeServer(object): 'pusherpool', 'event_builder_factory', 'filtering', + 'http_client_context_factory', + 'simple_http_client', ] def __init__(self, hostname, **kwargs): @@ -174,6 +178,16 @@ class HomeServer(BaseHomeServer): def build_auth(self): return Auth(self) + def build_http_client_context_factory(self): + config = self.get_config() + return ( + WoefullyInsecureContextFactory() if config.use_insecure_ssl_client + else BrowserLikePolicyForHTTPS() + ) + + def build_simple_http_client(self): + return SimpleHttpClient(self) + def build_v1auth(self): orf = Auth(self) # Matrix spec makes no reference to what HTTP status code is returned, From 6485f03d91a5f96da28f9dcc8e9ebc3adb213f6f Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 9 Sep 2015 13:05:00 +0100 Subject: [PATCH 11/82] Fix random formatting --- synapse/app/homeserver.py | 1 - synapse/http/client.py | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index ba76ee362a..8e60304e29 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -15,7 +15,6 @@ # limitations under the License. import sys - sys.dont_write_bytecode = True from synapse.python_dependencies import check_requirements, DEPENDENCY_LINKS diff --git a/synapse/http/client.py b/synapse/http/client.py index da77c8b0ac..815a838729 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -265,7 +265,8 @@ class WoefullyInsecureContextFactory(ssl.ContextFactory): """ Factory for PyOpenSSL SSL contexts which does absolutely no certificate verification. - Do not use this unless you really, really hate your users.""" + Do not use this unless you really, really hate your users. + """ def __init__(self): self._context = SSL.Context(SSL.SSLv23_METHOD) From 89ae0166ded093be2343409cfe42f475dea83139 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 9 Sep 2015 13:25:22 +0100 Subject: [PATCH 12/82] Allow room initialSync for users that have left the room, returning a snapshot of how the room was when they left it --- synapse/api/auth.py | 49 +++++++++++++++++++++ synapse/handlers/message.py | 85 +++++++++++++++++++++++++++++++++---- synapse/storage/stream.py | 15 +++++++ 3 files changed, 140 insertions(+), 9 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 0c0d678562..9b614a12bb 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -104,6 +104,20 @@ class Auth(object): @defer.inlineCallbacks def check_joined_room(self, room_id, user_id, current_state=None): + """Check if the user is currently joined in the room + Args: + room_id(str): The room to check. + user_id(str): The user to check. + current_state(dict): Optional map of the current state of the room. + If provided then that map is used to check whether they are a + member of the room. Otherwise the current membership is + loaded from the database. + Raises: + AuthError if the user is not in the room. + Returns: + A deferred membership event for the user if the user is in + the room. + """ if current_state: member = current_state.get( (EventTypes.Member, user_id), @@ -119,6 +133,41 @@ class Auth(object): self._check_joined_room(member, user_id, room_id) defer.returnValue(member) + @defer.inlineCallbacks + def check_user_was_in_room(self, room_id, user_id, current_state=None): + """Check if the user was in the room at some point. + Args: + room_id(str): The room to check. + user_id(str): The user to check. + current_state(dict): Optional map of the current state of the room. + If provided then that map is used to check whether they are a + member of the room. Otherwise the current membership is + loaded from the database. + Raises: + AuthError if the user was never in the room. + Returns: + A deferred membership event for the user if the user was in + the room. + """ + if current_state: + member = current_state.get( + (EventTypes.Member, user_id), + None + ) + else: + member = yield self.state.get_current_state( + room_id=room_id, + event_type=EventTypes.Member, + state_key=user_id + ) + + if not member: + raise AuthError(403, "User %s not in room %s" % ( + user_id, room_id + )) + + defer.returnValue(member) + @defer.inlineCallbacks def check_host_in_room(self, room_id, host): curr_state = yield self.state.get_current_state(room_id) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 5447c97e83..fc9a234333 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -22,7 +22,7 @@ from synapse.events.utils import serialize_event from synapse.events.validator import EventValidator from synapse.util import unwrapFirstError from synapse.util.logcontext import PreserveLoggingContext -from synapse.types import UserID, RoomStreamToken +from synapse.types import UserID, RoomStreamToken, StreamToken from ._base import BaseHandler @@ -377,7 +377,6 @@ class MessageHandler(BaseHandler): lambda states: states[event.event_id] ) - (messages, token), current_state = yield defer.gatherResults( [ self.store.get_recent_events_for_room( @@ -434,13 +433,83 @@ class MessageHandler(BaseHandler): @defer.inlineCallbacks def room_initial_sync(self, user_id, room_id, pagin_config=None, feedback=False): - current_state = yield self.state.get_current_state( - room_id=room_id, + """Capture the a snapshot of a room. If user is currently a member of + the room this will be what is currently in the room. If the user left + the room this will be what was in the room when they left. + + Args: + user_id(str): The user to get a snapshot for. + room_id(str): The room to get a snapshot of. + pagin_config(synapse.api.streams.PaginationConfig): The pagination + config used to determine how many messages to return. + Raises: + AuthError if the user wasn't in the room. + Returns: + A JSON object with the snapshot of the room. + """ + + member_event = yield self.auth.check_user_was_in_room(room_id, user_id) + + if member_event.membership == Membership.JOIN: + result = yield self._room_initial_sync_joined( + user_id, room_id, pagin_config, member_event + ) + elif member_event.membership == Membership.LEAVE: + result = yield self._room_initial_sync_parted( + user_id, room_id, pagin_config, member_event + ) + defer.returnValue(result) + + @defer.inlineCallbacks + def _room_initial_sync_parted(self, user_id, room_id, pagin_config, + member_event): + room_state = yield self.store.get_state_for_events( + member_event.room_id, [member_event.event_id], None ) - yield self.auth.check_joined_room( - room_id, user_id, - current_state=current_state + room_state = room_state[member_event.event_id] + + limit = pagin_config.limit if pagin_config else None + if limit is None: + limit = 10 + + stream_token = yield self.store.get_stream_token_for_event( + member_event.event_id + ) + + messages, token = yield self.store.get_recent_events_for_room( + room_id, + limit=limit, + end_token=stream_token + ) + + messages = yield self._filter_events_for_client( + user_id, room_id, messages + ) + + start_token = StreamToken(token[0], 0, 0, 0) + end_token = StreamToken(token[1], 0, 0, 0) + + time_now = self.clock.time_msec() + + defer.returnValue({ + "membership": member_event.membership, + "room_id": room_id, + "messages": { + "chunk": [serialize_event(m, time_now) for m in messages], + "start": start_token.to_string(), + "end": end_token.to_string(), + }, + "state": [serialize_event(s, time_now) for s in room_state.values()], + "presence": [], + "receipts": [], + }) + + @defer.inlineCallbacks + def _room_initial_sync_joined(self, user_id, room_id, pagin_config, + member_event): + current_state = yield self.state.get_current_state( + room_id=room_id, ) # TODO(paul): I wish I was called with user objects not user_id @@ -454,8 +523,6 @@ class MessageHandler(BaseHandler): for x in current_state.values() ] - member_event = current_state.get((EventTypes.Member, user_id,)) - now_token = yield self.hs.get_event_sources().get_current_token() limit = pagin_config.limit if pagin_config else None diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index d7fe423f5a..0abfa86cd2 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -379,6 +379,21 @@ class StreamStore(SQLBaseStore): ) defer.returnValue("t%d-%d" % (topo, token)) + def get_stream_token_for_event(self, event_id): + """The stream token for an event + Args: + event_id(str): The id of the event to look up a stream token for. + Raises: + StoreError if the event wasn't in the database. + Returns: + A deferred "s%d" stream token. + """ + return self._simple_select_one_onecol( + table="events", + keyvalues={"event_id": event_id}, + retcol="stream_ordering", + ).addCallback(lambda stream_ordering: "s%d" % (stream_ordering,)) + def _get_max_topological_txn(self, txn): txn.execute( "SELECT MAX(topological_ordering) FROM events" From ddfe30ba835da4357670f2a2a39386b8b8e65b60 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 9 Sep 2015 13:26:23 +0100 Subject: [PATCH 13/82] Better document the intent of the insecure SSL setting --- synapse/config/tls.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/synapse/config/tls.py b/synapse/config/tls.py index 472cf7ac4a..35ff13f4ba 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -42,9 +42,13 @@ class TlsConfig(Config): config.get("tls_dh_params_path"), "tls_dh_params" ) + # This config option applies to non-federation HTTP clients + # (e.g. for talking to recaptcha, identity servers, and such) + # It should never be used in production, and is intended for + # use only when running tests. self.use_insecure_ssl_client = config.get( - "i_really_want_to_ignore_ssl_certs_when_i_am_an_http_client_even_" - "though_it_is_woefully_insecure_because_i_hate_my_users", False) + "i_really_want_to_ignore_ssl_certs_when_i_am_an_https_client_even_" + "though_it_is_woefully_insecure_because_i_am_testing_i_promise", False) def default_config(self, config_dir_path, server_name): base_key_name = os.path.join(config_dir_path, server_name) From 1d579df66475c342e0bf5fed338808bdbfd03c94 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 9 Sep 2015 14:12:24 +0100 Subject: [PATCH 14/82] Allow rooms/{roomId}/state for a room that has been left --- synapse/api/auth.py | 3 ++- synapse/handlers/message.py | 18 +++++++++++++----- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 9b614a12bb..d8bb64a4af 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -160,8 +160,9 @@ class Auth(object): event_type=EventTypes.Member, state_key=user_id ) + membership = member.membership if member else None - if not member: + if membership not in (Membership.JOIN, Membership.LEAVE): raise AuthError(403, "User %s not in room %s" % ( user_id, room_id )) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index fc9a234333..171e9d72ac 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -277,7 +277,9 @@ class MessageHandler(BaseHandler): @defer.inlineCallbacks def get_state_events(self, user_id, room_id): - """Retrieve all state events for a given room. + """Retrieve all state events for a given room. If the user is + joined to the room then return the current state. If the user has + left the room return the state events from when they left. Args: user_id(str): The user requesting state events. @@ -285,13 +287,19 @@ class MessageHandler(BaseHandler): Returns: A list of dicts representing state events. [{}, {}, {}] """ - yield self.auth.check_joined_room(room_id, user_id) + member_event = yield self.auth.check_user_was_in_room(room_id, user_id) + + if member_event.membership == Membership.JOIN: + room_state = yield self.state_handler.get_current_state(room_id) + elif member_event.membership == Membership.LEAVE: + room_state = yield self.store.get_state_for_events( + room_id, [member_event.event_id], None + ) + room_state = room_state[member_event.event_id] - # TODO: This is duplicating logic from snapshot_all_rooms - current_state = yield self.state_handler.get_current_state(room_id) now = self.clock.time_msec() defer.returnValue( - [serialize_event(c, now) for c in current_state.values()] + [serialize_event(c, now) for c in room_state.values()] ) @defer.inlineCallbacks From bc8b25eb56bf4fcec3546c2ea28741189a519da5 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 9 Sep 2015 15:42:16 +0100 Subject: [PATCH 15/82] Allow users that have left the room to view the member list from the point they left --- synapse/handlers/room.py | 36 ------------------------------ synapse/rest/client/v1/room.py | 18 ++++++++++----- tests/rest/client/v1/test_rooms.py | 4 ++-- 3 files changed, 15 insertions(+), 43 deletions(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index c5d1001b50..0ff816d53e 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -25,7 +25,6 @@ from synapse.api.constants import ( from synapse.api.errors import StoreError, SynapseError from synapse.util import stringutils, unwrapFirstError from synapse.util.async import run_on_reactor -from synapse.events.utils import serialize_event from collections import OrderedDict import logging @@ -342,41 +341,6 @@ class RoomMemberHandler(BaseHandler): if remotedomains is not None: remotedomains.add(member.domain) - @defer.inlineCallbacks - def get_room_members_as_pagination_chunk(self, room_id=None, user_id=None, - limit=0, start_tok=None, - end_tok=None): - """Retrieve a list of room members in the room. - - Args: - room_id (str): The room to get the member list for. - user_id (str): The ID of the user making the request. - limit (int): The max number of members to return. - start_tok (str): Optional. The start token if known. - end_tok (str): Optional. The end token if known. - Returns: - dict: A Pagination streamable dict. - Raises: - SynapseError if something goes wrong. - """ - yield self.auth.check_joined_room(room_id, user_id) - - member_list = yield self.store.get_room_members(room_id=room_id) - time_now = self.clock.time_msec() - event_list = [ - serialize_event(entry, time_now) - for entry in member_list - ] - chunk_data = { - "start": "START", # FIXME (erikj): START is no longer valid - "end": "END", - "chunk": event_list - } - # TODO honor Pagination stream params - # TODO snapshot this list to return on subsequent requests when - # paginating - defer.returnValue(chunk_data) - @defer.inlineCallbacks def change_membership(self, event, context, do_auth=True): """ Change the membership status of a user in a room. diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index c9c27dd5a0..f4558b95a7 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -290,12 +290,18 @@ class RoomMemberListRestServlet(ClientV1RestServlet): def on_GET(self, request, room_id): # TODO support Pagination stream API (limit/tokens) user, _ = yield self.auth.get_user_by_req(request) - handler = self.handlers.room_member_handler - members = yield handler.get_room_members_as_pagination_chunk( + handler = self.handlers.message_handler + events = yield handler.get_state_events( room_id=room_id, - user_id=user.to_string()) + user_id=user.to_string(), + ) - for event in members["chunk"]: + chunk = [] + + for event in events: + if event["type"] != EventTypes.Member: + continue + chunk.append(event) # FIXME: should probably be state_key here, not user_id target_user = UserID.from_string(event["user_id"]) # Presence is an optional cache; don't fail if we can't fetch it @@ -308,7 +314,9 @@ class RoomMemberListRestServlet(ClientV1RestServlet): except: pass - defer.returnValue((200, members)) + defer.returnValue((200, { + "chunk": chunk + })) # TODO: Needs unit testing diff --git a/tests/rest/client/v1/test_rooms.py b/tests/rest/client/v1/test_rooms.py index 34ab47d02e..d50cfe4298 100644 --- a/tests/rest/client/v1/test_rooms.py +++ b/tests/rest/client/v1/test_rooms.py @@ -492,9 +492,9 @@ class RoomsMemberListTestCase(RestTestCase): self.assertEquals(200, code, msg=str(response)) yield self.leave(room=room_id, user=self.user_id) - # can no longer see list, you've left. + # can see old list once left (code, response) = yield self.mock_resource.trigger_get(room_path) - self.assertEquals(403, code, msg=str(response)) + self.assertEquals(200, code, msg=str(response)) class RoomsCreateTestCase(RestTestCase): From 3c166a24c591afdc851de3c6c754c90471b1b0a9 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 9 Sep 2015 16:05:09 +0100 Subject: [PATCH 16/82] Remove undocumented and unimplemented 'feedback' parameter from the Client-Server API --- synapse/api/constants.py | 11 ----------- synapse/handlers/message.py | 21 +++------------------ synapse/handlers/room.py | 1 - synapse/rest/client/v1/initial_sync.py | 2 -- synapse/rest/client/v1/room.py | 2 -- synapse/storage/stream.py | 10 ++-------- 6 files changed, 5 insertions(+), 42 deletions(-) diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 1423986c1e..3385664394 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -27,16 +27,6 @@ class Membership(object): LIST = (INVITE, JOIN, KNOCK, LEAVE, BAN) -class Feedback(object): - - """Represents the types of feedback a user can send in response to a - message.""" - - DELIVERED = u"delivered" - READ = u"read" - LIST = (DELIVERED, READ) - - class PresenceState(object): """Represents the presence state of a user.""" OFFLINE = u"offline" @@ -73,7 +63,6 @@ class EventTypes(object): PowerLevels = "m.room.power_levels" Aliases = "m.room.aliases" Redaction = "m.room.redaction" - Feedback = "m.room.message.feedback" RoomHistoryVisibility = "m.room.history_visibility" CanonicalAlias = "m.room.canonical_alias" diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 171e9d72ac..72ebac047f 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -71,7 +71,7 @@ class MessageHandler(BaseHandler): @defer.inlineCallbacks def get_messages(self, user_id=None, room_id=None, pagin_config=None, - feedback=False, as_client_event=True): + as_client_event=True): """Get messages in a room. Args: @@ -79,7 +79,6 @@ class MessageHandler(BaseHandler): room_id (str): The room they want messages from. pagin_config (synapse.api.streams.PaginationConfig): The pagination config rules to apply, if any. - feedback (bool): True to get compressed feedback with the messages as_client_event (bool): True to get events in client-server format. Returns: dict: Pagination API results @@ -264,17 +263,6 @@ class MessageHandler(BaseHandler): ) defer.returnValue(data) - @defer.inlineCallbacks - def get_feedback(self, event_id): - # yield self.auth.check_joined_room(room_id, user_id) - - # Pull out the feedback from the db - fb = yield self.store.get_feedback(event_id) - - if fb: - defer.returnValue(fb) - defer.returnValue(None) - @defer.inlineCallbacks def get_state_events(self, user_id, room_id): """Retrieve all state events for a given room. If the user is @@ -303,8 +291,7 @@ class MessageHandler(BaseHandler): ) @defer.inlineCallbacks - def snapshot_all_rooms(self, user_id=None, pagin_config=None, - feedback=False, as_client_event=True): + def snapshot_all_rooms(self, user_id=None, pagin_config=None, as_client_event=True): """Retrieve a snapshot of all rooms the user is invited or has joined. This snapshot may include messages for all rooms where the user is @@ -314,7 +301,6 @@ class MessageHandler(BaseHandler): user_id (str): The ID of the user making the request. pagin_config (synapse.api.streams.PaginationConfig): The pagination config used to determine how many messages *PER ROOM* to return. - feedback (bool): True to get feedback along with these messages. as_client_event (bool): True to get events in client-server format. Returns: A list of dicts with "room_id" and "membership" keys for all rooms @@ -439,8 +425,7 @@ class MessageHandler(BaseHandler): defer.returnValue(ret) @defer.inlineCallbacks - def room_initial_sync(self, user_id, room_id, pagin_config=None, - feedback=False): + def room_initial_sync(self, user_id, room_id, pagin_config=None): """Capture the a snapshot of a room. If user is currently a member of the room this will be what is currently in the room. If the user left the room this will be what was in the room when they left. diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 0ff816d53e..243623190f 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -610,7 +610,6 @@ class RoomEventSource(object): to_key=config.to_key, direction=config.direction, limit=config.limit, - with_feedback=True ) defer.returnValue((events, next_key)) diff --git a/synapse/rest/client/v1/initial_sync.py b/synapse/rest/client/v1/initial_sync.py index 4ea4da653c..bac68cc29f 100644 --- a/synapse/rest/client/v1/initial_sync.py +++ b/synapse/rest/client/v1/initial_sync.py @@ -26,14 +26,12 @@ class InitialSyncRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request): user, _ = yield self.auth.get_user_by_req(request) - with_feedback = "feedback" in request.args as_client_event = "raw" not in request.args pagination_config = PaginationConfig.from_request(request) handler = self.handlers.message_handler content = yield handler.snapshot_all_rooms( user_id=user.to_string(), pagin_config=pagination_config, - feedback=with_feedback, as_client_event=as_client_event ) diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index f4558b95a7..23871f161e 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -329,14 +329,12 @@ class RoomMessageListRestServlet(ClientV1RestServlet): pagination_config = PaginationConfig.from_request( request, default_limit=10, ) - with_feedback = "feedback" in request.args as_client_event = "raw" not in request.args handler = self.handlers.message_handler msgs = yield handler.get_messages( room_id=room_id, user_id=user.to_string(), pagin_config=pagination_config, - feedback=with_feedback, as_client_event=as_client_event ) diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 0abfa86cd2..5763c462af 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -159,9 +159,7 @@ class StreamStore(SQLBaseStore): @log_function def get_room_events_stream(self, user_id, from_key, to_key, room_id, - limit=0, with_feedback=False): - # TODO (erikj): Handle compressed feedback - + limit=0): current_room_membership_sql = ( "SELECT m.room_id FROM room_memberships as m " " INNER JOIN current_state_events as c" @@ -227,10 +225,7 @@ class StreamStore(SQLBaseStore): @defer.inlineCallbacks def paginate_room_events(self, room_id, from_key, to_key=None, - direction='b', limit=-1, - with_feedback=False): - # TODO (erikj): Handle compressed feedback - + direction='b', limit=-1): # Tokens really represent positions between elements, but we use # the convention of pointing to the event before the gap. Hence # we have a bit of asymmetry when it comes to equalities. @@ -302,7 +297,6 @@ class StreamStore(SQLBaseStore): @cachedInlineCallbacks(num_args=4) def get_recent_events_for_room(self, room_id, limit, end_token, from_token=None): - # TODO (erikj): Handle compressed feedback end_token = RoomStreamToken.parse_stream_token(end_token) From 09cb5c7d33c32e2cbf5a5b6f6f0e2780338491d2 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 9 Sep 2015 17:31:09 +0100 Subject: [PATCH 17/82] Allow users that have left a room to get the messages that happend in the room before they left --- synapse/handlers/message.py | 31 +++++++++++++++++++++++++++---- synapse/storage/stream.py | 19 ++++++++++++++++++- 2 files changed, 45 insertions(+), 5 deletions(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 72ebac047f..db89491b46 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -83,21 +83,44 @@ class MessageHandler(BaseHandler): Returns: dict: Pagination API results """ - yield self.auth.check_joined_room(room_id, user_id) + member_event = yield self.auth.check_user_was_in_room(room_id, user_id) data_source = self.hs.get_event_sources().sources["room"] - if not pagin_config.from_token: + if pagin_config.from_token: + room_token = pagin_config.from_token.room_key + else: pagin_config.from_token = ( yield self.hs.get_event_sources().get_current_token( direction='b' ) ) + room_token = pagin_config.from_token.room_key - room_token = RoomStreamToken.parse(pagin_config.from_token.room_key) + room_token = RoomStreamToken.parse(room_token) if room_token.topological is None: raise SynapseError(400, "Invalid token") + pagin_config.from_token = pagin_config.from_token.copy_and_replace( + "room_key", str(room_token) + ) + + source_config = pagin_config.get_source_config("room") + + if member_event.membership == Membership.LEAVE: + # If they have left the room then clamp the token to be before + # they left the room + leave_token = yield self.store.get_topological_token_for_event( + member_event.event_id + ) + leave_token = RoomStreamToken.parse(leave_token) + if leave_token.topological < room_token.topological: + source_config.from_key = str(leave_token) + + if source_config.direction == "f": + if source_config.to_key is None: + source_config.to_key = str(leave_token) + yield self.hs.get_handlers().federation_handler.maybe_backfill( room_id, room_token.topological ) @@ -105,7 +128,7 @@ class MessageHandler(BaseHandler): user = UserID.from_string(user_id) events, next_key = yield data_source.get_pagination_rows( - user, pagin_config.get_source_config("room"), room_id + user, source_config, room_id ) next_token = pagin_config.from_token.copy_and_replace( diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index 5763c462af..3cab06fdef 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -386,7 +386,24 @@ class StreamStore(SQLBaseStore): table="events", keyvalues={"event_id": event_id}, retcol="stream_ordering", - ).addCallback(lambda stream_ordering: "s%d" % (stream_ordering,)) + ).addCallback(lambda row: "s%d" % (row,)) + + def get_topological_token_for_event(self, event_id): + """The stream token for an event + Args: + event_id(str): The id of the event to look up a stream token for. + Raises: + StoreError if the event wasn't in the database. + Returns: + A deferred "t%d-%d" topological token. + """ + return self._simple_select_one( + table="events", + keyvalues={"event_id": event_id}, + retcols=("stream_ordering", "topological_ordering"), + ).addCallback(lambda row: "t%d-%d" % ( + row["topological_ordering"], row["stream_ordering"],) + ) def _get_max_topological_txn(self, txn): txn.execute( From e1eb1f3fb959468ddacb46e0766ce0c2ce938778 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 9 Sep 2015 17:02:39 +0100 Subject: [PATCH 18/82] Various bug fixes to crypto.keyring --- synapse/crypto/keyring.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index aa74d4d0cb..644c7b14a9 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -162,7 +162,9 @@ class Keyring(object): def remove_deferreds(res, server_name, group_id): server_to_gids[server_name].discard(group_id) if not server_to_gids[server_name]: - server_to_deferred.pop(server_name).callback(None) + d = server_to_deferred.pop(server_name, None) + if d: + d.callback(None) return res for g_id, deferred in deferreds.items(): @@ -200,8 +202,15 @@ class Keyring(object): else: break - for server_name, deferred in server_to_deferred: - self.key_downloads[server_name] = ObservableDeferred(deferred) + for server_name, deferred in server_to_deferred.items(): + d = ObservableDeferred(deferred) + self.key_downloads[server_name] = d + + def rm(r, server_name): + self.key_downloads.pop(server_name, None) + return r + + d.addBoth(rm, server_name) def get_server_verify_keys(self, group_id_to_group, group_id_to_deferred): """Takes a dict of KeyGroups and tries to find at least one key for @@ -220,9 +229,8 @@ class Keyring(object): merged_results = {} missing_keys = { - group.server_name: key_id + group.server_name: set(group.key_ids) for group in group_id_to_group.values() - for key_id in group.key_ids } for fn in key_fetch_fns: @@ -279,16 +287,15 @@ class Keyring(object): def get_keys_from_store(self, server_name_and_key_ids): res = yield defer.gatherResults( [ - self.store.get_server_verify_keys(server_name, key_ids) + self.store.get_server_verify_keys( + server_name, key_ids + ).addCallback(lambda ks, server: (server, ks), server_name) for server_name, key_ids in server_name_and_key_ids ], consumeErrors=True, ).addErrback(unwrapFirstError) - defer.returnValue(dict(zip( - [server_name for server_name, _ in server_name_and_key_ids], - res - ))) + defer.returnValue(dict(res)) @defer.inlineCallbacks def get_keys_from_perspectives(self, server_name_and_key_ids): From 3f604816555040a501874c8638d861b66dcd960b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 10 Sep 2015 09:58:32 +0100 Subject: [PATCH 19/82] Bump version and change log --- CHANGES.rst | 6 ++++++ synapse/__init__.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index a0a12ca4a0..1f6b712cbb 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,9 @@ +Changes in synapse v0.10.0-r2 (2015-09-10) +========================================== + +* Fix bug where we always fetched remote server signing keys instead of using + ones in our cache. + Changes in synapse v0.10.0-r1 (2015-09-08) ========================================== diff --git a/synapse/__init__.py b/synapse/__init__.py index 5b5ff6c3f3..d62294e6bb 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.10.0-r1" +__version__ = "0.10.0-r2" From ca32c7a0657c056a4d503b92e4120b880639663f Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 10 Sep 2015 10:33:48 +0100 Subject: [PATCH 20/82] Fix adding threepids to an existing account --- synapse/rest/client/v2_alpha/account.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py index 522a312c9e..6281e2d029 100644 --- a/synapse/rest/client/v2_alpha/account.py +++ b/synapse/rest/client/v2_alpha/account.py @@ -96,6 +96,7 @@ class ThreepidRestServlet(RestServlet): self.hs = hs self.identity_handler = hs.get_handlers().identity_handler self.auth = hs.get_auth() + self.auth_handler = hs.get_handlers().auth_handler @defer.inlineCallbacks def on_GET(self, request): From 4ba8189b744ab942322b7da5136292d3c4bb4e58 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 10 Sep 2015 10:45:22 +0100 Subject: [PATCH 21/82] Bump change log --- CHANGES.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES.rst b/CHANGES.rst index 1f6b712cbb..cab5e495c3 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -3,6 +3,7 @@ Changes in synapse v0.10.0-r2 (2015-09-10) * Fix bug where we always fetched remote server signing keys instead of using ones in our cache. +* Fix adding threepids to an existing account. Changes in synapse v0.10.0-r1 (2015-09-08) ========================================== From e2054ce21a04f3d741293f50b283c01bbe2b0591 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 10 Sep 2015 15:06:47 +0100 Subject: [PATCH 22/82] Allow users to GET individual state events for rooms that they have left --- synapse/handlers/message.py | 20 +++++++++++++------- tests/rest/client/v1/test_rooms.py | 10 +++++----- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index db89491b46..5d18aaacf0 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -16,7 +16,7 @@ from twisted.internet import defer from synapse.api.constants import EventTypes, Membership -from synapse.api.errors import RoomError, SynapseError +from synapse.api.errors import SynapseError from synapse.streams.config import PaginationConfig from synapse.events.utils import serialize_event from synapse.events.validator import EventValidator @@ -277,13 +277,19 @@ class MessageHandler(BaseHandler): Raises: SynapseError if something went wrong. """ - have_joined = yield self.auth.check_joined_room(room_id, user_id) - if not have_joined: - raise RoomError(403, "User not in room.") + member_event = yield self.auth.check_user_was_in_room(room_id, user_id) + + if member_event.membership == Membership.JOIN: + data = yield self.state_handler.get_current_state( + room_id, event_type, state_key + ) + elif member_event.membership == Membership.LEAVE: + key = (event_type, state_key) + room_state = yield self.store.get_state_for_events( + room_id, [member_event.event_id], [key] + ) + data = room_state[member_event.event_id].get(key) - data = yield self.state_handler.get_current_state( - room_id, event_type, state_key - ) defer.returnValue(data) @defer.inlineCallbacks diff --git a/tests/rest/client/v1/test_rooms.py b/tests/rest/client/v1/test_rooms.py index d50cfe4298..ed0ac8d5c8 100644 --- a/tests/rest/client/v1/test_rooms.py +++ b/tests/rest/client/v1/test_rooms.py @@ -239,7 +239,7 @@ class RoomPermissionsTestCase(RestTestCase): "PUT", topic_path, topic_content) self.assertEquals(403, code, msg=str(response)) (code, response) = yield self.mock_resource.trigger_get(topic_path) - self.assertEquals(403, code, msg=str(response)) + self.assertEquals(200, code, msg=str(response)) # get topic in PUBLIC room, not joined, expect 403 (code, response) = yield self.mock_resource.trigger_get( @@ -301,11 +301,11 @@ class RoomPermissionsTestCase(RestTestCase): room=room, expect_code=200) # get membership of self, get membership of other, private room + left - # expect all 403s + # expect all 200s yield self.leave(room=room, user=self.user_id) yield self._test_get_membership( members=[self.user_id, self.rmcreator_id], - room=room, expect_code=403) + room=room, expect_code=200) @defer.inlineCallbacks def test_membership_public_room_perms(self): @@ -326,11 +326,11 @@ class RoomPermissionsTestCase(RestTestCase): room=room, expect_code=200) # get membership of self, get membership of other, public room + left - # expect 403. + # expect 200. yield self.leave(room=room, user=self.user_id) yield self._test_get_membership( members=[self.user_id, self.rmcreator_id], - room=room, expect_code=403) + room=room, expect_code=200) @defer.inlineCallbacks def test_invited_permissions(self): From 9cd5b9a802f4c38f716a507a40188557103993f3 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Mon, 14 Sep 2015 19:03:53 +0100 Subject: [PATCH 23/82] Hacky attempt at catching SIGHUP and rotating the logfile around --- synapse/config/logger.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/synapse/config/logger.py b/synapse/config/logger.py index fa542623b7..daca698d0c 100644 --- a/synapse/config/logger.py +++ b/synapse/config/logger.py @@ -21,6 +21,7 @@ import logging.config import yaml from string import Template import os +import signal DEFAULT_LOG_CONFIG = Template(""" @@ -142,6 +143,19 @@ class LoggingConfig(Config): handler = logging.handlers.RotatingFileHandler( self.log_file, maxBytes=(1000 * 1000 * 100), backupCount=3 ) + + def sighup(signum, stack): + logger.info("Closing log file due to SIGHUP") + handler.doRollover() + logger.info("Opened new log file due to SIGHUP") + + # TODO(paul): obviously this is a terrible mechanism for + # stealing SIGHUP, because it means no other part of synapse + # can use it instead. If we want to catch SIGHUP anywhere + # else as well, I'd suggest we find a nicer way to broadcast + # it around. + if getattr(signal, "SIGHUP"): + signal.signal(signal.SIGHUP, sighup) else: handler = logging.StreamHandler() handler.setFormatter(formatter) From 3bcbabc9fb5446e74a675352e22963d528189957 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 15 Sep 2015 15:46:22 +0100 Subject: [PATCH 24/82] Rename context factory Mjark is officially no fun. --- synapse/http/client.py | 6 +++--- synapse/server.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/synapse/http/client.py b/synapse/http/client.py index 815a838729..0933388c04 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -261,11 +261,11 @@ def _print_ex(e): logger.exception(e) -class WoefullyInsecureContextFactory(ssl.ContextFactory): +class InsecureInterceptableContextFactory(ssl.ContextFactory): """ - Factory for PyOpenSSL SSL contexts which does absolutely no certificate verification. + Factory for PyOpenSSL SSL contexts which accepts any certificate for any domain. - Do not use this unless you really, really hate your users. + Do not use this since it allows an attacker to intercept your communications. """ def __init__(self): diff --git a/synapse/server.py b/synapse/server.py index 656e534dff..d96c5a573a 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -21,7 +21,7 @@ # Imports required for the default HomeServer() implementation from twisted.web.client import BrowserLikePolicyForHTTPS from synapse.federation import initialize_http_replication -from synapse.http.client import SimpleHttpClient, WoefullyInsecureContextFactory +from synapse.http.client import SimpleHttpClient, InsecureInterceptableContextFactory from synapse.notifier import Notifier from synapse.api.auth import Auth from synapse.handlers import Handlers @@ -181,7 +181,7 @@ class HomeServer(BaseHomeServer): def build_http_client_context_factory(self): config = self.get_config() return ( - WoefullyInsecureContextFactory() if config.use_insecure_ssl_client + InsecureInterceptableContextFactory() if config.use_insecure_ssl_client else BrowserLikePolicyForHTTPS() ) From d4af08a167cb5351110036c35bdfc267242d8131 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 15 Sep 2015 15:50:13 +0100 Subject: [PATCH 25/82] Use shorter config key name --- synapse/config/tls.py | 6 +++--- synapse/server.py | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/synapse/config/tls.py b/synapse/config/tls.py index 35ff13f4ba..e6023a718d 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -46,9 +46,9 @@ class TlsConfig(Config): # (e.g. for talking to recaptcha, identity servers, and such) # It should never be used in production, and is intended for # use only when running tests. - self.use_insecure_ssl_client = config.get( - "i_really_want_to_ignore_ssl_certs_when_i_am_an_https_client_even_" - "though_it_is_woefully_insecure_because_i_am_testing_i_promise", False) + self.use_insecure_ssl_client_just_for_testing_do_not_use = config.get( + "use_insecure_ssl_client_just_for_testing_do_not_use" + ) def default_config(self, config_dir_path, server_name): base_key_name = os.path.join(config_dir_path, server_name) diff --git a/synapse/server.py b/synapse/server.py index d96c5a573a..8424798b1b 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -181,7 +181,8 @@ class HomeServer(BaseHomeServer): def build_http_client_context_factory(self): config = self.get_config() return ( - InsecureInterceptableContextFactory() if config.use_insecure_ssl_client + InsecureInterceptableContextFactory() + if config.use_insecure_ssl_client_just_for_testing_do_not_use else BrowserLikePolicyForHTTPS() ) From 3b05b67c89bacfa729f14ad7c6de80b32d87cc98 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 15 Sep 2015 16:34:42 +0100 Subject: [PATCH 26/82] When updating a stored event from outlier to non-outlier, remember to update the extremeties --- synapse/storage/event_federation.py | 3 +++ synapse/storage/events.py | 2 ++ 2 files changed, 5 insertions(+) diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index dda3027b61..27284cd08c 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -303,6 +303,9 @@ class EventFederationStore(SQLBaseStore): ], ) + self._update_extremeties(txn, events) + + def _update_extremeties(self, txn, events): events_by_room = {} for ev in events: events_by_room.setdefault(ev.room_id, []).append(ev) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index e3eabab13d..8774b3b388 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -275,6 +275,8 @@ class EventsStore(SQLBaseStore): (False, event.event_id,) ) + self._update_extremeties(txn, [event]) + events_and_contexts = filter( lambda ec: ec[0] not in to_remove, events_and_contexts From 8148c48f11845af1747dc9447e47f9cce70c615f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 15 Sep 2015 16:54:43 +0100 Subject: [PATCH 27/82] "Comments" --- synapse/storage/event_federation.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 27284cd08c..7cb314dee8 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -306,6 +306,12 @@ class EventFederationStore(SQLBaseStore): self._update_extremeties(txn, events) def _update_extremeties(self, txn, events): + """Updates the event_*_extremities tables based on the new/updated + events being persisted. + + This is called for new events *and* for events that were outliers, but + are are now being persisted as non-outliers. + """ events_by_room = {} for ev in events: events_by_room.setdefault(ev.room_id, []).append(ev) From 0b1a55c60a7b10a987583dc2c231e8b1cfd29f30 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 16 Sep 2015 09:55:44 +0100 Subject: [PATCH 28/82] Update changelog --- CHANGES.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index cab5e495c3..f1d2c7a765 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,9 +1,11 @@ -Changes in synapse v0.10.0-r2 (2015-09-10) +Changes in synapse v0.10.0-r2 (2015-09-16) ========================================== * Fix bug where we always fetched remote server signing keys instead of using ones in our cache. * Fix adding threepids to an existing account. +* Fix bug with invinting over federation where remote server was already in + the room. (PR #281, SYN-392) Changes in synapse v0.10.0-r1 (2015-09-08) ========================================== From ffe8cf7e59bab7c9da98534f27b1e486cb642223 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 11 Sep 2015 11:51:25 +0100 Subject: [PATCH 29/82] Fix bug where we sometimes didn't fetch all the keys requested for a server. --- synapse/crypto/keyring.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index 1b1b31c5c0..8b6a59866f 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -228,10 +228,9 @@ class Keyring(object): def do_iterations(): merged_results = {} - missing_keys = { - group.server_name: set(group.key_ids) - for group in group_id_to_group.values() - } + missing_keys = {} + for group in group_id_to_group.values(): + missing_keys.setdefault(group.server_name, set()).union(group.key_ids) for fn in key_fetch_fns: results = yield fn(missing_keys.items()) From 4678055173636f9940e77f1af35b888f99506030 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 11 Sep 2015 11:07:22 +0100 Subject: [PATCH 30/82] Refactor do_invite_join --- synapse/handlers/federation.py | 84 ++++++++++++++++++++++------------ synapse/state.py | 3 -- 2 files changed, 56 insertions(+), 31 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 4ff20599d6..30b9982e25 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -649,35 +649,10 @@ class FederationHandler(BaseHandler): # FIXME pass - ev_infos = [] - for e in itertools.chain(state, auth_chain): - if e.event_id == event.event_id: - continue + self._check_auth_tree(auth_chain, event) - e.internal_metadata.outlier = True - auth_ids = [e_id for e_id, _ in e.auth_events] - ev_infos.append({ - "event": e, - "auth_events": { - (e.type, e.state_key): e for e in auth_chain - if e.event_id in auth_ids - } - }) - - yield self._handle_new_events(origin, ev_infos, outliers=True) - - auth_ids = [e_id for e_id, _ in event.auth_events] - auth_events = { - (e.type, e.state_key): e for e in auth_chain - if e.event_id in auth_ids - } - - _, event_stream_id, max_stream_id = yield self._handle_new_event( - origin, - new_event, - state=state, - current_state=state, - auth_events=auth_events, + event_stream_id, max_stream_id = yield self._persist_auth_tree( + auth_chain, state, event ) with PreserveLoggingContext(): @@ -1026,6 +1001,59 @@ class FederationHandler(BaseHandler): is_new_state=(not outliers and not backfilled), ) + def _check_auth_tree(self, auth_events, event): + event_map = { + e.event_id: e + for e in auth_events + } + + create_event = None + for e in auth_events: + if (e.type, e.state_key) == (EventTypes.Create, ""): + create_event = e + break + + for e in auth_events + [event]: + a = { + (event_map[e_id].type, event_map[e_id].state_key): event_map[e_id] + for e_id, _ in e.auth_events + } + if create_event: + a[(EventTypes.Create, "")] = create_event + + self.auth.check(e, auth_events=a) + + @defer.inlineCallbacks + def _persist_auth_tree(self, auth_events, state, event): + events_to_context = {} + for e in auth_events: + ctx = yield self.state_handler.compute_event_context( + e, outlier=True, + ) + events_to_context[e.event_id] = ctx + e.internal_metadata.outlier = True + + yield self.store.persist_events( + [ + (e, events_to_context[e.event_id]) + for e in auth_events + ], + is_new_state=False, + ) + + new_event_context = yield self.state_handler.compute_event_context( + event, old_state=state, outlier=False, + ) + + event_stream_id, max_stream_id = yield self.store.persist_event( + event, new_event_context, + backfilled=False, + is_new_state=True, + current_state=state, + ) + + defer.returnValue((event_stream_id, max_stream_id)) + @defer.inlineCallbacks def _prep_event(self, origin, event, state=None, backfilled=False, current_state=None, auth_events=None): diff --git a/synapse/state.py b/synapse/state.py index 1fe4d066bd..ed36f844cb 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -17,7 +17,6 @@ from twisted.internet import defer from synapse.util.logutils import log_function -from synapse.util.async import run_on_reactor from synapse.util.caches.expiringcache import ExpiringCache from synapse.api.constants import EventTypes from synapse.api.errors import AuthError @@ -119,8 +118,6 @@ class StateHandler(object): Returns: an EventContext """ - yield run_on_reactor() - context = EventContext() if outlier: From a3e332af1930c103b8b2ece9d50edd94193761e4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 11 Sep 2015 13:41:07 +0100 Subject: [PATCH 31/82] Don't bail out of joining if we encounter a rejected event --- scripts-dev/check_auth.py | 2 +- synapse/handlers/federation.py | 33 ++++++++++++++++++++------------- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/scripts-dev/check_auth.py b/scripts-dev/check_auth.py index 4fa8792a5f..b362aad722 100644 --- a/scripts-dev/check_auth.py +++ b/scripts-dev/check_auth.py @@ -38,7 +38,7 @@ def check_auth(auth, auth_chain, events): print "Failed:", e.event_id, e.type, e.state_key print "Auth_events:", auth_events print ex - print json.dumps(e.get_dict(), sort_keys=True, indent=4) + # print json.dumps(e.get_dict(), sort_keys=True, indent=4) # raise print "Success:", e.event_id, e.type, e.state_key diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 30b9982e25..0f11fa390f 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -649,8 +649,6 @@ class FederationHandler(BaseHandler): # FIXME pass - self._check_auth_tree(auth_chain, event) - event_stream_id, max_stream_id = yield self._persist_auth_tree( auth_chain, state, event ) @@ -1001,7 +999,16 @@ class FederationHandler(BaseHandler): is_new_state=(not outliers and not backfilled), ) - def _check_auth_tree(self, auth_events, event): + @defer.inlineCallbacks + def _persist_auth_tree(self, auth_events, state, event): + events_to_context = {} + for e in auth_events: + ctx = yield self.state_handler.compute_event_context( + e, outlier=True, + ) + events_to_context[e.event_id] = ctx + e.internal_metadata.outlier = True + event_map = { e.event_id: e for e in auth_events @@ -1021,17 +1028,17 @@ class FederationHandler(BaseHandler): if create_event: a[(EventTypes.Create, "")] = create_event - self.auth.check(e, auth_events=a) + try: + self.auth.check(e, auth_events=a) + except AuthError: + logger.warn( + "Rejecting %s because %s", + event.event_id, e.msg + ) - @defer.inlineCallbacks - def _persist_auth_tree(self, auth_events, state, event): - events_to_context = {} - for e in auth_events: - ctx = yield self.state_handler.compute_event_context( - e, outlier=True, - ) - events_to_context[e.event_id] = ctx - e.internal_metadata.outlier = True + if e == event: + raise + events_to_context[e.event_id].rejected = RejectedReason.AUTH_ERROR yield self.store.persist_events( [ From 744e7d2790380b20260c0740fc68f7f49d07136b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 11 Sep 2015 14:26:15 +0100 Subject: [PATCH 32/82] Also handle state --- synapse/handlers/federation.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 0f11fa390f..b148af5390 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -1002,7 +1002,7 @@ class FederationHandler(BaseHandler): @defer.inlineCallbacks def _persist_auth_tree(self, auth_events, state, event): events_to_context = {} - for e in auth_events: + for e in itertools.chain(auth_events, state): ctx = yield self.state_handler.compute_event_context( e, outlier=True, ) @@ -1020,7 +1020,7 @@ class FederationHandler(BaseHandler): create_event = e break - for e in auth_events + [event]: + for e in itertools.chain(auth_events, state, [event]): a = { (event_map[e_id].type, event_map[e_id].state_key): event_map[e_id] for e_id, _ in e.auth_events @@ -1033,7 +1033,7 @@ class FederationHandler(BaseHandler): except AuthError: logger.warn( "Rejecting %s because %s", - event.event_id, e.msg + e.event_id, e.msg ) if e == event: From 3a01901d6c59f540540e00835f0716dfa7f03846 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 11 Sep 2015 14:28:57 +0100 Subject: [PATCH 33/82] Capture err --- synapse/handlers/federation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index b148af5390..fd8a86ea99 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -1030,10 +1030,10 @@ class FederationHandler(BaseHandler): try: self.auth.check(e, auth_events=a) - except AuthError: + except AuthError as err: logger.warn( "Rejecting %s because %s", - e.event_id, e.msg + e.event_id, err.msg ) if e == event: From 54e688277ad86b95345c3e8d1306c7e08b0ed484 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 11 Sep 2015 14:32:31 +0100 Subject: [PATCH 34/82] Also persist state --- synapse/handlers/federation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index fd8a86ea99..85fdf94d06 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -1043,7 +1043,7 @@ class FederationHandler(BaseHandler): yield self.store.persist_events( [ (e, events_to_context[e.event_id]) - for e in auth_events + for e in itertools.chain(auth_events, state) ], is_new_state=False, ) From c34ffd2736a2042484a3593a0174df5e2d118252 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 16 Sep 2015 16:26:03 +0100 Subject: [PATCH 35/82] Fix getting an event for a room the server forgot it was in --- synapse/handlers/federation.py | 110 ++++++++++++++++++--------------- 1 file changed, 61 insertions(+), 49 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 85fdf94d06..e79a82cfc0 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -125,60 +125,72 @@ class FederationHandler(BaseHandler): ) if not is_in_room and not event.internal_metadata.is_outlier(): logger.debug("Got event for room we're not in.") - current_state = state - event_ids = set() - if state: - event_ids |= {e.event_id for e in state} - if auth_chain: - event_ids |= {e.event_id for e in auth_chain} + try: + event_stream_id, max_stream_id = yield self._persist_auth_tree( + auth_chain, state, event + ) + except AuthError as e: + raise FederationError( + "ERROR", + e.code, + e.msg, + affected=event.event_id, + ) - seen_ids = set( - (yield self.store.have_events(event_ids)).keys() - ) + else: + event_ids = set() + if state: + event_ids |= {e.event_id for e in state} + if auth_chain: + event_ids |= {e.event_id for e in auth_chain} - if state and auth_chain is not None: - # If we have any state or auth_chain given to us by the replication - # layer, then we should handle them (if we haven't before.) - - event_infos = [] - - for e in itertools.chain(auth_chain, state): - if e.event_id in seen_ids: - continue - e.internal_metadata.outlier = True - auth_ids = [e_id for e_id, _ in e.auth_events] - auth = { - (e.type, e.state_key): e for e in auth_chain - if e.event_id in auth_ids - } - event_infos.append({ - "event": e, - "auth_events": auth, - }) - seen_ids.add(e.event_id) - - yield self._handle_new_events( - origin, - event_infos, - outliers=True + seen_ids = set( + (yield self.store.have_events(event_ids)).keys() ) - try: - _, event_stream_id, max_stream_id = yield self._handle_new_event( - origin, - event, - state=state, - backfilled=backfilled, - current_state=current_state, - ) - except AuthError as e: - raise FederationError( - "ERROR", - e.code, - e.msg, - affected=event.event_id, - ) + if state and auth_chain is not None: + # If we have any state or auth_chain given to us by the replication + # layer, then we should handle them (if we haven't before.) + + event_infos = [] + + for e in itertools.chain(auth_chain, state): + if e.event_id in seen_ids: + continue + e.internal_metadata.outlier = True + auth_ids = [e_id for e_id, _ in e.auth_events] + auth = { + (e.type, e.state_key): e for e in auth_chain + if e.event_id in auth_ids + } + event_infos.append({ + "event": e, + "auth_events": auth, + }) + seen_ids.add(e.event_id) + + yield self._handle_new_events( + origin, + event_infos, + outliers=True + ) + + try: + _, event_stream_id, max_stream_id = yield self._handle_new_event( + origin, + event, + state=state, + backfilled=backfilled, + current_state=current_state, + ) + except AuthError as e: + raise FederationError( + "ERROR", + e.code, + e.msg, + affected=event.event_id, + ) # if we're receiving valid events from an origin, # it's probably a good idea to mark it as not in retry-state From 51b2448e050d4944d1a5176bcfbf30a33953ca68 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 17 Sep 2015 10:11:15 +0100 Subject: [PATCH 36/82] Revert change of scripts/check_auth.py --- scripts-dev/check_auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts-dev/check_auth.py b/scripts-dev/check_auth.py index b362aad722..4fa8792a5f 100644 --- a/scripts-dev/check_auth.py +++ b/scripts-dev/check_auth.py @@ -38,7 +38,7 @@ def check_auth(auth, auth_chain, events): print "Failed:", e.event_id, e.type, e.state_key print "Auth_events:", auth_events print ex - # print json.dumps(e.get_dict(), sort_keys=True, indent=4) + print json.dumps(e.get_dict(), sort_keys=True, indent=4) # raise print "Success:", e.event_id, e.type, e.state_key From b105996fc16ac264457c5a9f8e00c6a8263142d6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 17 Sep 2015 10:28:36 +0100 Subject: [PATCH 37/82] Remove run_on_reactor --- synapse/state.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/synapse/state.py b/synapse/state.py index 1fe4d066bd..ed36f844cb 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -17,7 +17,6 @@ from twisted.internet import defer from synapse.util.logutils import log_function -from synapse.util.async import run_on_reactor from synapse.util.caches.expiringcache import ExpiringCache from synapse.api.constants import EventTypes from synapse.api.errors import AuthError @@ -119,8 +118,6 @@ class StateHandler(object): Returns: an EventContext """ - yield run_on_reactor() - context = EventContext() if outlier: From 8e3bbc9bd01b8d0bd75e28d9c8862e555384e124 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 21 Sep 2015 13:47:44 +0100 Subject: [PATCH 38/82] Clarify which event is returned by check_user_was_in_room --- synapse/api/auth.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/synapse/api/auth.py b/synapse/api/auth.py index d8bb64a4af..49a068afb1 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -146,8 +146,9 @@ class Auth(object): Raises: AuthError if the user was never in the room. Returns: - A deferred membership event for the user if the user was in - the room. + A deferred membership event for the user if the user was in the + room. This will be the join event if they are currently joined to + the room. This will be the leave event if they have left the room. """ if current_state: member = current_state.get( From 1e101ed4a45bad68f4865229e0c8d617fed67065 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 21 Sep 2015 14:13:10 +0100 Subject: [PATCH 39/82] Clamp the "to" token for /rooms/{roomId}/messages to when the user left the room. There isn't a way for the client to learn a valid "to" token for a room that they have left in the C-S API but that doesn't stop a client making one up. --- synapse/handlers/message.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 5d18aaacf0..bca592f5d7 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -120,6 +120,10 @@ class MessageHandler(BaseHandler): if source_config.direction == "f": if source_config.to_key is None: source_config.to_key = str(leave_token) + else: + to_token = RoomStreamToken.parse(source_config.to_key) + if leave_token.topological < to_token.topological: + source_config.to_key = str(leave_token) yield self.hs.get_handlers().federation_handler.maybe_backfill( room_id, room_token.topological From 0c162859897cb0cae5e501109e3d51d0e861e194 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 21 Sep 2015 14:17:16 +0100 Subject: [PATCH 40/82] Add explicit "elif event.membership == Membership.LEAVE" for clarity --- synapse/handlers/message.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index bca592f5d7..5593a8d4f7 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -389,13 +389,14 @@ class MessageHandler(BaseHandler): if event.membership not in (Membership.JOIN, Membership.LEAVE): return + try: if event.membership == Membership.JOIN: room_end_token = now_token.room_key deferred_room_state = self.state_handler.get_current_state( event.room_id ) - else: + elif event.membership == Membership.LEAVE: room_end_token = "s%d" % (event.stream_ordering,) deferred_room_state = self.store.get_state_for_events( event.room_id, [event.event_id], None From 95c304e3f93bc1113c2b4ac64d85c9fdeb7120b9 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 21 Sep 2015 14:18:47 +0100 Subject: [PATCH 41/82] Fix doc string to point at the right class --- synapse/handlers/message.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 5593a8d4f7..bb73286729 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -467,8 +467,9 @@ class MessageHandler(BaseHandler): Args: user_id(str): The user to get a snapshot for. room_id(str): The room to get a snapshot of. - pagin_config(synapse.api.streams.PaginationConfig): The pagination - config used to determine how many messages to return. + pagin_config(synapse.streams.config.PaginationConfig): + The pagination config used to determine how many messages to + return. Raises: AuthError if the user wasn't in the room. Returns: From 49c0a0b5c4c2385fdf6755b1a5e1a3f0b04ef503 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 21 Sep 2015 14:21:03 +0100 Subject: [PATCH 42/82] Clarify that room_initial_sync returns a python dict --- synapse/handlers/message.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index bb73286729..bda8eb5f3f 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -473,7 +473,7 @@ class MessageHandler(BaseHandler): Raises: AuthError if the user wasn't in the room. Returns: - A JSON object with the snapshot of the room. + A JSON serialisable dict with the snapshot of the room. """ member_event = yield self.auth.check_user_was_in_room(room_id, user_id) From 7213588083dd9a721b0cd623fe22b308f25f19a5 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 22 Sep 2015 12:57:40 +0100 Subject: [PATCH 43/82] Implement configurable stats reporting SYN-287 This requires that HS owners either opt in or out of stats reporting. When --generate-config is passed, --report-stats must be specified If an already-generated config is used, and doesn't have the report_stats key, it is requested to be set. --- synapse/app/homeserver.py | 35 +++++- synapse/app/synctl.py | 12 +- synapse/config/_base.py | 45 ++++++- synapse/config/appservice.py | 2 +- synapse/config/captcha.py | 2 +- synapse/config/database.py | 2 +- synapse/config/key.py | 2 +- synapse/config/logger.py | 2 +- synapse/config/metrics.py | 8 +- synapse/config/ratelimiting.py | 2 +- synapse/config/registration.py | 2 +- synapse/config/repository.py | 2 +- synapse/config/saml2.py | 2 +- synapse/config/server.py | 2 +- synapse/config/tls.py | 2 +- synapse/config/voip.py | 2 +- synapse/storage/__init__.py | 20 ++- synapse/storage/events.py | 58 ++++++++- synapse/storage/registration.py | 12 ++ .../schema/delta/24/stats_reporting.sql | 22 ++++ tests/storage/event_injector.py | 81 ++++++++++++ tests/storage/test_events.py | 116 ++++++++++++++++++ tests/storage/test_room.py | 2 +- tests/storage/test_stream.py | 68 +++------- 24 files changed, 425 insertions(+), 78 deletions(-) create mode 100644 synapse/storage/schema/delta/24/stats_reporting.sql create mode 100644 tests/storage/event_injector.py create mode 100644 tests/storage/test_events.py diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 15c0a4a003..b4429bd4f3 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -42,7 +42,7 @@ from synapse.storage import ( from synapse.server import HomeServer -from twisted.internet import reactor +from twisted.internet import reactor, task, defer from twisted.application import service from twisted.enterprise import adbapi from twisted.web.resource import Resource, EncodingResourceWrapper @@ -677,6 +677,39 @@ def run(hs): ThreadPool._worker = profile(ThreadPool._worker) reactor.run = profile(reactor.run) + start_time = hs.get_clock().time() + + @defer.inlineCallbacks + def phone_stats_home(): + now = int(hs.get_clock().time()) + uptime = int(now - start_time) + if uptime < 0: + uptime = 0 + + stats = {} + stats["homeserver"] = hs.config.server_name + stats["timestamp"] = now + stats["uptime_seconds"] = uptime + stats["total_users"] = yield hs.get_datastore().count_all_users() + + all_rooms = yield hs.get_datastore().get_rooms(False) + stats["total_room_count"] = len(all_rooms) + + stats["daily_active_users"] = yield hs.get_datastore().count_daily_users() + daily_messages = yield hs.get_datastore().count_daily_messages() + if daily_messages is not None: + stats["daily_messages"] = daily_messages + + logger.info("Reporting stats to matrix.org: %s" % (stats,)) + hs.get_simple_http_client().put_json( + "https://matrix.org/report-usage-stats/push", + stats + ) + + if hs.config.report_stats: + phone_home_task = task.LoopingCall(phone_stats_home) + phone_home_task.start(60 * 60 * 24, now=False) + def in_thread(): with LoggingContext("run"): change_resource_limit(hs.config.soft_file_limit) diff --git a/synapse/app/synctl.py b/synapse/app/synctl.py index 1f7d543c31..6bcc437591 100755 --- a/synapse/app/synctl.py +++ b/synapse/app/synctl.py @@ -25,6 +25,7 @@ SYNAPSE = ["python", "-B", "-m", "synapse.app.homeserver"] CONFIGFILE = "homeserver.yaml" GREEN = "\x1b[1;32m" +RED = "\x1b[1;31m" NORMAL = "\x1b[m" if not os.path.exists(CONFIGFILE): @@ -45,8 +46,15 @@ def start(): print "Starting ...", args = SYNAPSE args.extend(["--daemonize", "-c", CONFIGFILE]) - subprocess.check_call(args) - print GREEN + "started" + NORMAL + try: + subprocess.check_call(args) + print GREEN + "started" + NORMAL + except subprocess.CalledProcessError as e: + print ( + RED + + "error starting (exit code: %d); see above for logs" % e.returncode + + NORMAL + ) def stop(): diff --git a/synapse/config/_base.py b/synapse/config/_base.py index 8a75c48733..b9983f72a2 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -26,6 +26,16 @@ class ConfigError(Exception): class Config(object): + stats_reporting_begging_spiel = ( + "We would really appreciate it if you could help our project out by " + "reporting anonymized usage statistics from your homeserver. Only very " + "basic aggregate data (e.g. number of users) will be reported, but it " + "helps us to track the growth of the Matrix community, and helps us to " + "make Matrix a success, as well as to convince other networks that they " + "should peer with us.\n" + "Thank you." + ) + @staticmethod def parse_size(value): if isinstance(value, int) or isinstance(value, long): @@ -111,11 +121,14 @@ class Config(object): results.append(getattr(cls, name)(self, *args, **kargs)) return results - def generate_config(self, config_dir_path, server_name): + def generate_config(self, config_dir_path, server_name, report_stats=None): default_config = "# vim:ft=yaml\n" default_config += "\n\n".join(dedent(conf) for conf in self.invoke_all( - "default_config", config_dir_path, server_name + "default_config", + config_dir_path=config_dir_path, + server_name=server_name, + report_stats=report_stats, )) config = yaml.load(default_config) @@ -139,6 +152,12 @@ class Config(object): action="store_true", help="Generate a config file for the server name" ) + config_parser.add_argument( + "--report-stats", + action="store", + help="Stuff", + choices=["yes", "no"] + ) config_parser.add_argument( "--generate-keys", action="store_true", @@ -189,6 +208,11 @@ class Config(object): config_files.append(config_path) if config_args.generate_config: + if config_args.report_stats is None: + config_parser.error( + "Please specify either --report-stats=yes or --report-stats=no\n\n" + + cls.stats_reporting_begging_spiel + ) if not config_files: config_parser.error( "Must supply a config file.\nA config file can be automatically" @@ -211,7 +235,9 @@ class Config(object): os.makedirs(config_dir_path) with open(config_path, "wb") as config_file: config_bytes, config = obj.generate_config( - config_dir_path, server_name + config_dir_path=config_dir_path, + server_name=server_name, + report_stats=(config_args.report_stats == "yes"), ) obj.invoke_all("generate_files", config) config_file.write(config_bytes) @@ -261,9 +287,20 @@ class Config(object): specified_config.update(yaml_config) server_name = specified_config["server_name"] - _, config = obj.generate_config(config_dir_path, server_name) + _, config = obj.generate_config( + config_dir_path=config_dir_path, + server_name=server_name + ) config.pop("log_config") config.update(specified_config) + if "report_stats" not in config: + sys.stderr.write( + "Please opt in or out of reporting anonymized homeserver usage " + "statistics, by setting the report_stats key in your config file " + " ( " + config_path + " ) " + + "to either True or False.\n\n" + + Config.stats_reporting_begging_spiel + "\n") + sys.exit(1) if generate_keys: obj.invoke_all("generate_files", config) diff --git a/synapse/config/appservice.py b/synapse/config/appservice.py index 38f41933b7..b8d301995e 100644 --- a/synapse/config/appservice.py +++ b/synapse/config/appservice.py @@ -20,7 +20,7 @@ class AppServiceConfig(Config): def read_config(self, config): self.app_service_config_files = config.get("app_service_config_files", []) - def default_config(cls, config_dir_path, server_name): + def default_config(cls, **kwargs): return """\ # A list of application service config file to use app_service_config_files: [] diff --git a/synapse/config/captcha.py b/synapse/config/captcha.py index 15a132b4e3..dd92fcd0dc 100644 --- a/synapse/config/captcha.py +++ b/synapse/config/captcha.py @@ -24,7 +24,7 @@ class CaptchaConfig(Config): self.captcha_bypass_secret = config.get("captcha_bypass_secret") self.recaptcha_siteverify_api = config["recaptcha_siteverify_api"] - def default_config(self, config_dir_path, server_name): + def default_config(self, **kwargs): return """\ ## Captcha ## diff --git a/synapse/config/database.py b/synapse/config/database.py index f0611e8884..baeda8f300 100644 --- a/synapse/config/database.py +++ b/synapse/config/database.py @@ -45,7 +45,7 @@ class DatabaseConfig(Config): self.set_databasepath(config.get("database_path")) - def default_config(self, config, config_dir_path): + def default_config(self, **kwargs): database_path = self.abspath("homeserver.db") return """\ # Database configuration diff --git a/synapse/config/key.py b/synapse/config/key.py index 23ac8a3fca..2c187065e5 100644 --- a/synapse/config/key.py +++ b/synapse/config/key.py @@ -40,7 +40,7 @@ class KeyConfig(Config): config["perspectives"] ) - def default_config(self, config_dir_path, server_name): + def default_config(self, config_dir_path, server_name, **kwargs): base_key_name = os.path.join(config_dir_path, server_name) return """\ ## Signing Keys ## diff --git a/synapse/config/logger.py b/synapse/config/logger.py index daca698d0c..bd0c17c861 100644 --- a/synapse/config/logger.py +++ b/synapse/config/logger.py @@ -70,7 +70,7 @@ class LoggingConfig(Config): self.log_config = self.abspath(config.get("log_config")) self.log_file = self.abspath(config.get("log_file")) - def default_config(self, config_dir_path, server_name): + def default_config(self, config_dir_path, server_name, **kwargs): log_file = self.abspath("homeserver.log") log_config = self.abspath( os.path.join(config_dir_path, server_name + ".log.config") diff --git a/synapse/config/metrics.py b/synapse/config/metrics.py index ae5a691527..825fec9a38 100644 --- a/synapse/config/metrics.py +++ b/synapse/config/metrics.py @@ -19,13 +19,15 @@ from ._base import Config class MetricsConfig(Config): def read_config(self, config): self.enable_metrics = config["enable_metrics"] + self.report_stats = config.get("report_stats", None) self.metrics_port = config.get("metrics_port") self.metrics_bind_host = config.get("metrics_bind_host", "127.0.0.1") - def default_config(self, config_dir_path, server_name): - return """\ + def default_config(self, report_stats=None, **kwargs): + suffix = "" if report_stats is None else "report_stats: %(report_stats)s\n" + return ("""\ ## Metrics ### # Enable collection and rendering of performance metrics enable_metrics: False - """ + """ + suffix) % locals() diff --git a/synapse/config/ratelimiting.py b/synapse/config/ratelimiting.py index 76d9970e5b..611b598ec7 100644 --- a/synapse/config/ratelimiting.py +++ b/synapse/config/ratelimiting.py @@ -27,7 +27,7 @@ class RatelimitConfig(Config): self.federation_rc_reject_limit = config["federation_rc_reject_limit"] self.federation_rc_concurrent = config["federation_rc_concurrent"] - def default_config(self, config_dir_path, server_name): + def default_config(self, **kwargs): return """\ ## Ratelimiting ## diff --git a/synapse/config/registration.py b/synapse/config/registration.py index 62de4b399f..fa98eced34 100644 --- a/synapse/config/registration.py +++ b/synapse/config/registration.py @@ -34,7 +34,7 @@ class RegistrationConfig(Config): self.registration_shared_secret = config.get("registration_shared_secret") self.macaroon_secret_key = config.get("macaroon_secret_key") - def default_config(self, config_dir, server_name): + def default_config(self, **kwargs): registration_shared_secret = random_string_with_symbols(50) macaroon_secret_key = random_string_with_symbols(50) return """\ diff --git a/synapse/config/repository.py b/synapse/config/repository.py index 64644b9a7a..2fcf872449 100644 --- a/synapse/config/repository.py +++ b/synapse/config/repository.py @@ -60,7 +60,7 @@ class ContentRepositoryConfig(Config): config["thumbnail_sizes"] ) - def default_config(self, config_dir_path, server_name): + def default_config(self, **kwargs): media_store = self.default_path("media_store") uploads_path = self.default_path("uploads") return """ diff --git a/synapse/config/saml2.py b/synapse/config/saml2.py index 1532036876..4c6133cf22 100644 --- a/synapse/config/saml2.py +++ b/synapse/config/saml2.py @@ -41,7 +41,7 @@ class SAML2Config(Config): self.saml2_config_path = None self.saml2_idp_redirect_url = None - def default_config(self, config_dir_path, server_name): + def default_config(self, config_dir_path, server_name, **kwargs): return """ # Enable SAML2 for registration and login. Uses pysaml2 # config_path: Path to the sp_conf.py configuration file diff --git a/synapse/config/server.py b/synapse/config/server.py index a03e55c223..4d12d49857 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -117,7 +117,7 @@ class ServerConfig(Config): self.content_addr = content_addr - def default_config(self, config_dir_path, server_name): + def default_config(self, server_name, **kwargs): if ":" in server_name: bind_port = int(server_name.split(":")[1]) unsecure_port = bind_port - 400 diff --git a/synapse/config/tls.py b/synapse/config/tls.py index e6023a718d..0ac2698293 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -50,7 +50,7 @@ class TlsConfig(Config): "use_insecure_ssl_client_just_for_testing_do_not_use" ) - def default_config(self, config_dir_path, server_name): + def default_config(self, config_dir_path, server_name, **kwargs): base_key_name = os.path.join(config_dir_path, server_name) tls_certificate_path = base_key_name + ".tls.crt" diff --git a/synapse/config/voip.py b/synapse/config/voip.py index a1707223d3..a093354ccd 100644 --- a/synapse/config/voip.py +++ b/synapse/config/voip.py @@ -22,7 +22,7 @@ class VoipConfig(Config): self.turn_shared_secret = config["turn_shared_secret"] self.turn_user_lifetime = self.parse_duration(config["turn_user_lifetime"]) - def default_config(self, config_dir_path, server_name): + def default_config(self, **kwargs): return """\ ## Turn ## diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 77cb1dbd81..b64c90d631 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -54,7 +54,7 @@ logger = logging.getLogger(__name__) # Remember to update this number every time a change is made to database # schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 23 +SCHEMA_VERSION = 24 dir_path = os.path.abspath(os.path.dirname(__file__)) @@ -126,6 +126,24 @@ class DataStore(RoomMemberStore, RoomStore, lock=False, ) + @defer.inlineCallbacks + def count_daily_users(self): + def _count_users(txn): + txn.execute( + "SELECT COUNT(DISTINCT user_id) AS users" + " FROM user_ips" + " WHERE last_seen > ?", + # This is close enough to a day for our purposes. + (int(self._clock.time_msec()) - (1000 * 60 * 60 * 24),) + ) + rows = self.cursor_to_dict(txn) + if rows: + return rows[0]["users"] + return 0 + + ret = yield self.runInteraction("count_users", _count_users) + defer.returnValue(ret) + def get_user_ip_and_agents(self, user): return self._simple_select_list( table="user_ips", diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 0a477e3122..2b51db9940 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -12,7 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - from _base import SQLBaseStore, _RollbackButIsFineException from twisted.internet import defer, reactor @@ -28,6 +27,7 @@ from canonicaljson import encode_canonical_json from contextlib import contextmanager import logging +import math import ujson as json logger = logging.getLogger(__name__) @@ -905,3 +905,59 @@ class EventsStore(SQLBaseStore): txn.execute(sql, (event.event_id,)) result = txn.fetchone() return result[0] if result else None + + @defer.inlineCallbacks + def count_daily_messages(self): + def _count_messages(txn): + now = self.hs.get_clock().time() + + txn.execute( + "SELECT reported_stream_token, reported_time FROM stats_reporting" + ) + last_reported = self.cursor_to_dict(txn) + + txn.execute( + "SELECT stream_ordering" + " FROM events" + " ORDER BY stream_ordering DESC" + " LIMIT 1" + ) + now_reporting = self.cursor_to_dict(txn) + if not now_reporting: + return None + now_reporting = now_reporting[0]["stream_ordering"] + + txn.execute("DELETE FROM stats_reporting") + txn.execute( + "INSERT INTO stats_reporting" + " (reported_stream_token, reported_time)" + " VALUES (?, ?)", + (now_reporting, now,) + ) + + if not last_reported: + return None + + # Close enough to correct for our purposes. + yesterday = (now - 24 * 60 * 60) + if math.fabs(yesterday - last_reported[0]["reported_time"]) > 60 * 60: + return None + + txn.execute( + "SELECT COUNT(*) as messages" + " FROM events NATURAL JOIN event_json" + " WHERE json like '%m.room.message%'" + " AND stream_ordering > ?" + " AND stream_ordering <= ?", + ( + last_reported[0]["reported_stream_token"], + now_reporting, + ) + ) + rows = self.cursor_to_dict(txn) + if not rows: + return None + return rows[0]["messages"] + + ret = yield self.runInteraction("count_messages", _count_messages) + defer.returnValue(ret) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index c9ceb132ae..6d76237658 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -289,3 +289,15 @@ class RegistrationStore(SQLBaseStore): if ret: defer.returnValue(ret['user_id']) defer.returnValue(None) + + @defer.inlineCallbacks + def count_all_users(self): + def _count_users(txn): + txn.execute("SELECT COUNT(*) AS users FROM users") + rows = self.cursor_to_dict(txn) + if rows: + return rows[0]["users"] + return 0 + + ret = yield self.runInteraction("count_users", _count_users) + defer.returnValue(ret) diff --git a/synapse/storage/schema/delta/24/stats_reporting.sql b/synapse/storage/schema/delta/24/stats_reporting.sql new file mode 100644 index 0000000000..e9165d2917 --- /dev/null +++ b/synapse/storage/schema/delta/24/stats_reporting.sql @@ -0,0 +1,22 @@ +/* Copyright 2015 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- Should only ever contain one row +CREATE TABLE IF NOT EXISTS stats_reporting( + -- The stream ordering token which was most recently reported as stats + reported_stream_token INTEGER, + -- The time (seconds since epoch) stats were most recently reported + reported_time BIGINT +); diff --git a/tests/storage/event_injector.py b/tests/storage/event_injector.py new file mode 100644 index 0000000000..42bd8928bd --- /dev/null +++ b/tests/storage/event_injector.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from tests import unittest +from twisted.internet import defer + +from synapse.api.constants import EventTypes, Membership +from synapse.types import UserID, RoomID + +from tests.utils import setup_test_homeserver + +from mock import Mock + + +class EventInjector: + def __init__(self, hs): + self.hs = hs + self.store = hs.get_datastore() + self.message_handler = hs.get_handlers().message_handler + self.event_builder_factory = hs.get_event_builder_factory() + + @defer.inlineCallbacks + def create_room(self, room): + builder = self.event_builder_factory.new({ + "type": EventTypes.Create, + "room_id": room.to_string(), + "content": {}, + }) + + event, context = yield self.message_handler._create_new_client_event( + builder + ) + + yield self.store.persist_event(event, context) + + @defer.inlineCallbacks + def inject_room_member(self, room, user, membership): + builder = self.event_builder_factory.new({ + "type": EventTypes.Member, + "sender": user.to_string(), + "state_key": user.to_string(), + "room_id": room.to_string(), + "content": {"membership": membership}, + }) + + event, context = yield self.message_handler._create_new_client_event( + builder + ) + + yield self.store.persist_event(event, context) + + defer.returnValue(event) + + @defer.inlineCallbacks + def inject_message(self, room, user, body): + builder = self.event_builder_factory.new({ + "type": EventTypes.Message, + "sender": user.to_string(), + "state_key": user.to_string(), + "room_id": room.to_string(), + "content": {"body": body, "msgtype": u"message"}, + }) + + event, context = yield self.message_handler._create_new_client_event( + builder + ) + + yield self.store.persist_event(event, context) diff --git a/tests/storage/test_events.py b/tests/storage/test_events.py new file mode 100644 index 0000000000..313013009e --- /dev/null +++ b/tests/storage/test_events.py @@ -0,0 +1,116 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import uuid +from mock.mock import Mock +from synapse.types import RoomID, UserID + +from tests import unittest +from twisted.internet import defer +from tests.storage.event_injector import EventInjector + +from tests.utils import setup_test_homeserver + + +class EventsStoreTestCase(unittest.TestCase): + + @defer.inlineCallbacks + def setUp(self): + self.hs = yield setup_test_homeserver( + resource_for_federation=Mock(), + http_client=None, + ) + self.store = self.hs.get_datastore() + self.db_pool = self.hs.get_db_pool() + self.message_handler = self.hs.get_handlers().message_handler + self.event_injector = EventInjector(self.hs) + + @defer.inlineCallbacks + def test_count_daily_messages(self): + self.db_pool.runQuery("DELETE FROM stats_reporting") + + self.hs.clock.now = 100 + + # Never reported before, and nothing which could be reported + count = yield self.store.count_daily_messages() + self.assertIsNone(count) + count = yield self.db_pool.runQuery("SELECT COUNT(*) FROM stats_reporting") + self.assertEqual([(0,)], count) + + # Create something to report + room = RoomID.from_string("!abc123:test") + user = UserID.from_string("@raccoonlover:test") + yield self.event_injector.create_room(room) + + self.base_event = yield self._get_last_stream_token() + + yield self.event_injector.inject_message(room, user, "Raccoons are really cute") + + # Never reported before, something could be reported, but isn't because + # it isn't old enough. + count = yield self.store.count_daily_messages() + self.assertIsNone(count) + self._assert_stats_reporting(1, self.hs.clock.now) + + # Already reported yesterday, two new events from today. + yield self.event_injector.inject_message(room, user, "Yeah they are!") + yield self.event_injector.inject_message(room, user, "Incredibly!") + self.hs.clock.now += 60 * 60 * 24 + count = yield self.store.count_daily_messages() + self.assertEqual(2, count) # 2 since yesterday + self._assert_stats_reporting(3, self.hs.clock.now) # 3 ever + + # Last reported too recently. + yield self.event_injector.inject_message(room, user, "Who could disagree?") + self.hs.clock.now += 60 * 60 * 22 + count = yield self.store.count_daily_messages() + self.assertIsNone(count) + self._assert_stats_reporting(4, self.hs.clock.now) + + # Last reported too long ago + yield self.event_injector.inject_message(room, user, "No one.") + self.hs.clock.now += 60 * 60 * 26 + count = yield self.store.count_daily_messages() + self.assertIsNone(count) + self._assert_stats_reporting(5, self.hs.clock.now) + + # And now let's actually report something + yield self.event_injector.inject_message(room, user, "Indeed.") + yield self.event_injector.inject_message(room, user, "Indeed.") + yield self.event_injector.inject_message(room, user, "Indeed.") + # A little over 24 hours is fine :) + self.hs.clock.now += (60 * 60 * 24) + 50 + count = yield self.store.count_daily_messages() + self.assertEqual(3, count) + self._assert_stats_reporting(8, self.hs.clock.now) + + @defer.inlineCallbacks + def _get_last_stream_token(self): + rows = yield self.db_pool.runQuery( + "SELECT stream_ordering" + " FROM events" + " ORDER BY stream_ordering DESC" + " LIMIT 1" + ) + if not rows: + defer.returnValue(0) + else: + defer.returnValue(rows[0][0]) + + @defer.inlineCallbacks + def _assert_stats_reporting(self, messages, time): + rows = yield self.db_pool.runQuery( + "SELECT reported_stream_token, reported_time FROM stats_reporting" + ) + self.assertEqual([(self.base_event + messages, time,)], rows) diff --git a/tests/storage/test_room.py b/tests/storage/test_room.py index ab7625a3ca..caffce64e3 100644 --- a/tests/storage/test_room.py +++ b/tests/storage/test_room.py @@ -85,7 +85,7 @@ class RoomEventsStoreTestCase(unittest.TestCase): # Room events need the full datastore, for persist_event() and # get_room_state() self.store = hs.get_datastore() - self.event_factory = hs.get_event_factory(); + self.event_factory = hs.get_event_factory() self.room = RoomID.from_string("!abcde:test") diff --git a/tests/storage/test_stream.py b/tests/storage/test_stream.py index 0c9b89d765..a658a789aa 100644 --- a/tests/storage/test_stream.py +++ b/tests/storage/test_stream.py @@ -19,6 +19,7 @@ from twisted.internet import defer from synapse.api.constants import EventTypes, Membership from synapse.types import UserID, RoomID +from tests.storage.event_injector import EventInjector from tests.utils import setup_test_homeserver @@ -36,6 +37,7 @@ class StreamStoreTestCase(unittest.TestCase): self.store = hs.get_datastore() self.event_builder_factory = hs.get_event_builder_factory() + self.event_injector = EventInjector(hs) self.handlers = hs.get_handlers() self.message_handler = self.handlers.message_handler @@ -45,60 +47,20 @@ class StreamStoreTestCase(unittest.TestCase): self.room1 = RoomID.from_string("!abc123:test") self.room2 = RoomID.from_string("!xyx987:test") - self.depth = 1 - - @defer.inlineCallbacks - def inject_room_member(self, room, user, membership): - self.depth += 1 - - builder = self.event_builder_factory.new({ - "type": EventTypes.Member, - "sender": user.to_string(), - "state_key": user.to_string(), - "room_id": room.to_string(), - "content": {"membership": membership}, - }) - - event, context = yield self.message_handler._create_new_client_event( - builder - ) - - yield self.store.persist_event(event, context) - - defer.returnValue(event) - - @defer.inlineCallbacks - def inject_message(self, room, user, body): - self.depth += 1 - - builder = self.event_builder_factory.new({ - "type": EventTypes.Message, - "sender": user.to_string(), - "state_key": user.to_string(), - "room_id": room.to_string(), - "content": {"body": body, "msgtype": u"message"}, - }) - - event, context = yield self.message_handler._create_new_client_event( - builder - ) - - yield self.store.persist_event(event, context) - @defer.inlineCallbacks def test_event_stream_get_other(self): # Both bob and alice joins the room - yield self.inject_room_member( + yield self.event_injector.inject_room_member( self.room1, self.u_alice, Membership.JOIN ) - yield self.inject_room_member( + yield self.event_injector.inject_room_member( self.room1, self.u_bob, Membership.JOIN ) # Initial stream key: start = yield self.store.get_room_events_max_id() - yield self.inject_message(self.room1, self.u_alice, u"test") + yield self.event_injector.inject_message(self.room1, self.u_alice, u"test") end = yield self.store.get_room_events_max_id() @@ -125,17 +87,17 @@ class StreamStoreTestCase(unittest.TestCase): @defer.inlineCallbacks def test_event_stream_get_own(self): # Both bob and alice joins the room - yield self.inject_room_member( + yield self.event_injector.inject_room_member( self.room1, self.u_alice, Membership.JOIN ) - yield self.inject_room_member( + yield self.event_injector.inject_room_member( self.room1, self.u_bob, Membership.JOIN ) # Initial stream key: start = yield self.store.get_room_events_max_id() - yield self.inject_message(self.room1, self.u_alice, u"test") + yield self.event_injector.inject_message(self.room1, self.u_alice, u"test") end = yield self.store.get_room_events_max_id() @@ -162,22 +124,22 @@ class StreamStoreTestCase(unittest.TestCase): @defer.inlineCallbacks def test_event_stream_join_leave(self): # Both bob and alice joins the room - yield self.inject_room_member( + yield self.event_injector.inject_room_member( self.room1, self.u_alice, Membership.JOIN ) - yield self.inject_room_member( + yield self.event_injector.inject_room_member( self.room1, self.u_bob, Membership.JOIN ) # Then bob leaves again. - yield self.inject_room_member( + yield self.event_injector.inject_room_member( self.room1, self.u_bob, Membership.LEAVE ) # Initial stream key: start = yield self.store.get_room_events_max_id() - yield self.inject_message(self.room1, self.u_alice, u"test") + yield self.event_injector.inject_message(self.room1, self.u_alice, u"test") end = yield self.store.get_room_events_max_id() @@ -193,17 +155,17 @@ class StreamStoreTestCase(unittest.TestCase): @defer.inlineCallbacks def test_event_stream_prev_content(self): - yield self.inject_room_member( + yield self.event_injector.inject_room_member( self.room1, self.u_bob, Membership.JOIN ) - event1 = yield self.inject_room_member( + event1 = yield self.event_injector.inject_room_member( self.room1, self.u_alice, Membership.JOIN ) start = yield self.store.get_room_events_max_id() - event2 = yield self.inject_room_member( + event2 = yield self.event_injector.inject_room_member( self.room1, self.u_alice, Membership.JOIN, ) From 6d7f291b93c363ecee7a3f1e5f5e2cdf81a0fbae Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 22 Sep 2015 13:13:07 +0100 Subject: [PATCH 44/82] Front-load spaces --- synapse/config/_base.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/synapse/config/_base.py b/synapse/config/_base.py index b9983f72a2..ceef309afc 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -27,13 +27,13 @@ class ConfigError(Exception): class Config(object): stats_reporting_begging_spiel = ( - "We would really appreciate it if you could help our project out by " - "reporting anonymized usage statistics from your homeserver. Only very " - "basic aggregate data (e.g. number of users) will be reported, but it " - "helps us to track the growth of the Matrix community, and helps us to " - "make Matrix a success, as well as to convince other networks that they " - "should peer with us.\n" - "Thank you." + "We would really appreciate it if you could help our project out by" + " reporting anonymized usage statistics from your homeserver. Only very" + " basic aggregate data (e.g. number of users) will be reported, but it" + " helps us to track the growth of the Matrix community, and helps us to" + " make Matrix a success, as well as to convince other networks that they" + " should peer with us." + "\nThank you." ) @staticmethod From eb011cd99ba03f40f4ed7a023b64f93dfa2cbdc9 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 22 Sep 2015 13:29:36 +0100 Subject: [PATCH 45/82] Add docstring --- synapse/storage/events.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 2b51db9940..46df6b4d6d 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -908,6 +908,12 @@ class EventsStore(SQLBaseStore): @defer.inlineCallbacks def count_daily_messages(self): + """ + Returns an estimate of the number of messages sent in the last day. + + If it has been significantly less or more than one day since the last + call to this function, it will return None. + """ def _count_messages(txn): now = self.hs.get_clock().time() From b6e0303c833bb37891294046598d48cafe2b1c9e Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 22 Sep 2015 13:34:29 +0100 Subject: [PATCH 46/82] Catch stats-reporting errors --- synapse/app/homeserver.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index b4429bd4f3..21840e4a28 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -701,10 +701,13 @@ def run(hs): stats["daily_messages"] = daily_messages logger.info("Reporting stats to matrix.org: %s" % (stats,)) - hs.get_simple_http_client().put_json( - "https://matrix.org/report-usage-stats/push", - stats - ) + try: + yield hs.get_simple_http_client().put_json( + "https://matrix.org/report-usage-stats/push", + stats + ) + except Exception as e: + logger.warn("Error reporting stats: %s", e) if hs.config.report_stats: phone_home_task = task.LoopingCall(phone_stats_home) From 6d59ffe1ce9a821d50d491f97bf05950198f6f53 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 22 Sep 2015 13:47:40 +0100 Subject: [PATCH 47/82] Add some docstrings --- synapse/storage/__init__.py | 3 +++ synapse/storage/registration.py | 1 + 2 files changed, 4 insertions(+) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index b64c90d631..340e59afcb 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -128,6 +128,9 @@ class DataStore(RoomMemberStore, RoomStore, @defer.inlineCallbacks def count_daily_users(self): + """ + Counts the number of users who used this homeserver in the last 24 hours. + """ def _count_users(txn): txn.execute( "SELECT COUNT(DISTINCT user_id) AS users" diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 6d76237658..b454dd5b3a 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -292,6 +292,7 @@ class RegistrationStore(SQLBaseStore): @defer.inlineCallbacks def count_all_users(self): + """Counts all users registered on the homeserver.""" def _count_users(txn): txn.execute("SELECT COUNT(*) AS users FROM users") rows = self.cursor_to_dict(txn) From b35baf6f3c5e9f24e9af241eb2423c4b94dd5a14 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 22 Sep 2015 15:13:10 +0100 Subject: [PATCH 48/82] Define __repr__ methods for StreamConfig and PaginationConfig So that they can be used with "%r" log formats. --- synapse/streams/config.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/synapse/streams/config.py b/synapse/streams/config.py index 2ec7c5403b..167bfe0de3 100644 --- a/synapse/streams/config.py +++ b/synapse/streams/config.py @@ -34,6 +34,11 @@ class SourcePaginationConfig(object): self.direction = 'f' if direction == 'f' else 'b' self.limit = int(limit) if limit is not None else None + def __repr__(self): + return ( + "StreamConfig(from_key=%r, to_key=%r, direction=%r, limit=%r)" + ) % (self.from_key, self.to_key, self.direction, self.limit) + class PaginationConfig(object): @@ -94,10 +99,10 @@ class PaginationConfig(object): logger.exception("Failed to create pagination config") raise SynapseError(400, "Invalid request.") - def __str__(self): + def __repr__(self): return ( - "" + "PaginationConfig(from_tok=%r, to_tok=%r," + " direction=%r, limit=%r)" ) % (self.from_token, self.to_token, self.direction, self.limit) def get_source_config(self, source_name): From cc3ab0c214156cdad9731beb27790e8510b4b023 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 22 Sep 2015 18:13:06 +0100 Subject: [PATCH 49/82] Add dev script for finding where functions are called from, and finding functions that aren't called at all --- scripts-dev/definitions.py | 130 +++++++++++++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100755 scripts-dev/definitions.py diff --git a/scripts-dev/definitions.py b/scripts-dev/definitions.py new file mode 100755 index 0000000000..09cec9d8ec --- /dev/null +++ b/scripts-dev/definitions.py @@ -0,0 +1,130 @@ +#! /usr/bin/python + +import ast +import yaml + +class DefinitionVisitor(ast.NodeVisitor): + def __init__(self): + super(DefinitionVisitor, self).__init__() + self.functions = {} + self.classes = {} + self.names = {} + self.attrs = set() + self.definitions = { + 'def': self.functions, + 'class': self.classes, + 'names': self.names, + 'attrs': self.attrs, + } + + def visit_Name(self, node): + self.names.setdefault(type(node.ctx).__name__, set()).add(node.id) + + def visit_Attribute(self, node): + self.attrs.add(node.attr) + for child in ast.iter_child_nodes(node): + self.visit(child) + + def visit_ClassDef(self, node): + visitor = DefinitionVisitor() + self.classes[node.name] = visitor.definitions + for child in ast.iter_child_nodes(node): + visitor.visit(child) + + def visit_FunctionDef(self, node): + visitor = DefinitionVisitor() + self.functions[node.name] = visitor.definitions + for child in ast.iter_child_nodes(node): + visitor.visit(child) + + +def non_empty(defs): + functions = {name: non_empty(f) for name, f in defs['def'].items()} + classes = {name: non_empty(f) for name, f in defs['class'].items()} + result = {} + if functions: result['def'] = functions + if classes: result['class'] = classes + names = defs['names'] + uses = [] + for name in names.get('Load', ()): + if name not in names.get('Param', ()) and name not in names.get('Store', ()): + uses.append(name) + uses.extend(defs['attrs']) + if uses: result['uses'] = uses + result['names'] = names + result['attrs'] = defs['attrs'] + return result + + +def definitions_in_code(input_code): + input_ast = ast.parse(input_code) + visitor = DefinitionVisitor() + visitor.visit(input_ast) + definitions = non_empty(visitor.definitions) + return definitions + + +def definitions_in_file(filepath): + with open(filepath) as f: + return definitions_in_code(f.read()) + + +def defined_names(prefix, defs, names): + for name, funcs in defs.get('def', {}).items(): + names.setdefault(name, {'defined': []})['defined'].append(prefix + name) + defined_names(prefix + name + ".", funcs, names) + + for name, funcs in defs.get('class', {}).items(): + names.setdefault(name, {'defined': []})['defined'].append(prefix + name) + defined_names(prefix + name + ".", funcs, names) + + +def used_names(prefix, defs, names): + for name, funcs in defs.get('def', {}).items(): + used_names(prefix + name + ".", funcs, names) + + for name, funcs in defs.get('class', {}).items(): + used_names(prefix + name + ".", funcs, names) + + for used in defs.get('uses', ()): + if used in names: + names[used].setdefault('used', []).append(prefix.rstrip('.')) + + +if __name__ == '__main__': + import sys, os + if not sys.argv[1:]: + sys.stderr.write( + "Usage: definitions.py \n" + " definitions.py \n" + "Either list the definitions matching the regexp or list\n" + " 'unused' definitions\n" + ) + + definitions = {} + for root, dirs, files in os.walk(sys.argv[1]): + for filename in files: + if filename.endswith(".py"): + filepath = os.path.join(root, filename) + definitions[filepath] = definitions_in_file(filepath) + + names = {} + for filepath, defs in definitions.items(): + defined_names(filepath + ":", defs, names) + + for filepath, defs in definitions.items(): + used_names(filepath + ":", defs, names) + + if sys.argv[2:]: + import re + pattern = re.compile(sys.argv[2]) + for name in list(names): + if not pattern.match(name): + del names[name] + else: + for name in list(names): + if 'used' in names[name]: + del names[name] + + yaml.dump(names, sys.stdout, default_flow_style=False) + #yaml.dump(definitions, sys.stdout, default_flow_style=False) From 527d95dea0d55bd1932639e61387ef60d834134e Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 22 Sep 2015 18:14:15 +0100 Subject: [PATCH 50/82] synapse/storage/_base.py:Table was unused --- synapse/storage/_base.py | 128 -------------------------------------- synapse/storage/pusher.py | 4 +- 2 files changed, 2 insertions(+), 130 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 495ef087c9..c1b5423bd6 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -25,8 +25,6 @@ from util.id_generators import IdGenerator, StreamIdGenerator from twisted.internet import defer -from collections import namedtuple - import sys import time import threading @@ -791,129 +789,3 @@ class _RollbackButIsFineException(Exception): something went wrong. """ pass - - -class Table(object): - """ A base class used to store information about a particular table. - """ - - table_name = None - """ str: The name of the table """ - - fields = None - """ list: The field names """ - - EntryType = None - """ Type: A tuple type used to decode the results """ - - _select_where_clause = "SELECT %s FROM %s WHERE %s" - _select_clause = "SELECT %s FROM %s" - _insert_clause = "REPLACE INTO %s (%s) VALUES (%s)" - - @classmethod - def select_statement(cls, where_clause=None): - """ - Args: - where_clause (str): The WHERE clause to use. - - Returns: - str: An SQL statement to select rows from the table with the given - WHERE clause. - """ - if where_clause: - return cls._select_where_clause % ( - ", ".join(cls.fields), - cls.table_name, - where_clause - ) - else: - return cls._select_clause % ( - ", ".join(cls.fields), - cls.table_name, - ) - - @classmethod - def insert_statement(cls): - return cls._insert_clause % ( - cls.table_name, - ", ".join(cls.fields), - ", ".join(["?"] * len(cls.fields)), - ) - - @classmethod - def decode_single_result(cls, results): - """ Given an iterable of tuples, return a single instance of - `EntryType` or None if the iterable is empty - Args: - results (list): The results list to convert to `EntryType` - Returns: - EntryType: An instance of `EntryType` - """ - results = list(results) - if results: - return cls.EntryType(*results[0]) - else: - return None - - @classmethod - def decode_results(cls, results): - """ Given an iterable of tuples, return a list of `EntryType` - Args: - results (list): The results list to convert to `EntryType` - - Returns: - list: A list of `EntryType` - """ - return [cls.EntryType(*row) for row in results] - - @classmethod - def get_fields_string(cls, prefix=None): - if prefix: - to_join = ("%s.%s" % (prefix, f) for f in cls.fields) - else: - to_join = cls.fields - - return ", ".join(to_join) - - -class JoinHelper(object): - """ Used to help do joins on tables by looking at the tables' fields and - creating a list of unique fields to use with SELECTs and a namedtuple - to dump the results into. - - Attributes: - tables (list): List of `Table` classes - EntryType (type) - """ - - def __init__(self, *tables): - self.tables = tables - - res = [] - for table in self.tables: - res += [f for f in table.fields if f not in res] - - self.EntryType = namedtuple("JoinHelperEntry", res) - - def get_fields(self, **prefixes): - """Get a string representing a list of fields for use in SELECT - statements with the given prefixes applied to each. - - For example:: - - JoinHelper(PdusTable, StateTable).get_fields( - PdusTable="pdus", - StateTable="state" - ) - """ - res = [] - for field in self.EntryType._fields: - for table in self.tables: - if field in table.fields: - res.append("%s.%s" % (prefixes[table.__name__], field)) - break - - return ", ".join(res) - - def decode_results(self, rows): - return [self.EntryType(*row) for row in rows] diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index 00b748f131..345c4e1104 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ._base import SQLBaseStore, Table +from ._base import SQLBaseStore from twisted.internet import defer from synapse.api.errors import StoreError @@ -149,5 +149,5 @@ class PusherStore(SQLBaseStore): ) -class PushersTable(Table): +class PushersTable(object): table_name = "pushers" From 372ac60375973f03bcfce7f857c85b6ed2388881 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 22 Sep 2015 18:16:07 +0100 Subject: [PATCH 51/82] synapse/util/__init__.py:unwrap_deferred was unused --- synapse/util/__init__.py | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index 07ff25cef3..1d123ccefc 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -29,34 +29,6 @@ def unwrapFirstError(failure): return failure.value.subFailure -def unwrap_deferred(d): - """Given a deferred that we know has completed, return its value or raise - the failure as an exception - """ - if not d.called: - raise RuntimeError("deferred has not finished") - - res = [] - - def f(r): - res.append(r) - return r - d.addCallback(f) - - if res: - return res[0] - - def f(r): - res.append(r) - return r - d.addErrback(f) - - if res: - res[0].raiseException() - else: - raise RuntimeError("deferred did not call callbacks") - - class Clock(object): """A small utility that obtains current time-of-day so that time may be mocked during unit-tests. From f2fcc0a8cff8b11ba0201c2bebea37649960b6a3 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 22 Sep 2015 18:18:45 +0100 Subject: [PATCH 52/82] synapse/api/errors.py:RoomError was unused --- synapse/api/errors.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/synapse/api/errors.py b/synapse/api/errors.py index c3b4d971a8..ee3045268f 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -77,11 +77,6 @@ class SynapseError(CodeMessageException): ) -class RoomError(SynapseError): - """An error raised when a room event fails.""" - pass - - class RegistrationError(SynapseError): """An error raised when a registration event fails.""" pass From a247729806a1cf7093b3c0819094338bf22affa8 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 22 Sep 2015 18:19:49 +0100 Subject: [PATCH 53/82] synapse/streams/events.py:StreamSource was unused --- synapse/streams/events.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/synapse/streams/events.py b/synapse/streams/events.py index aaa3609aa5..8671a8fa4e 100644 --- a/synapse/streams/events.py +++ b/synapse/streams/events.py @@ -70,15 +70,3 @@ class EventSources(object): ), ) defer.returnValue(token) - - -class StreamSource(object): - def get_new_events_for_user(self, user, from_key, limit): - """from_key is the key within this event source.""" - raise NotImplementedError("get_new_events_for_user") - - def get_current_key(self): - raise NotImplementedError("get_current_key") - - def get_pagination_rows(self, user, pagination_config, key): - raise NotImplementedError("get_rows") From 184ba0968a08e537dccc65d8fe75f65b86e22504 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 22 Sep 2015 18:25:30 +0100 Subject: [PATCH 54/82] synapse/app/homeserver.py:GzipFile was unused --- synapse/app/homeserver.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 21840e4a28..349a4c1774 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -85,12 +85,6 @@ import time logger = logging.getLogger("synapse.app.homeserver") -class GzipFile(File): - def getChild(self, path, request): - child = File.getChild(self, path, request) - return EncodingResourceWrapper(child, [GzipEncoderFactory()]) - - def gz_wrap(r): return EncodingResourceWrapper(r, [GzipEncoderFactory()]) From 7a5818ed811c6da046e60bcc7eacb7a28a92cd73 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 22 Sep 2015 18:27:22 +0100 Subject: [PATCH 55/82] Note that GzipFile was removed in comment that referenced it --- synapse/app/homeserver.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 349a4c1774..190b03e2f7 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -128,6 +128,7 @@ class SynapseHomeServer(HomeServer): # (It can stay enabled for the API resources: they call # write() with the whole body and then finish() straight # after and so do not trigger the bug. + # GzipFile was removed in commit 184ba09 # return GzipFile(webclient_path) # TODO configurable? return File(webclient_path) # TODO configurable? From bb4dddd6c4f85bc5b07119d3f9dec31964b5b6f9 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 22 Sep 2015 18:33:34 +0100 Subject: [PATCH 56/82] Move NullSource out of synapse and into tests since it is only used by the tests --- synapse/streams/events.py | 16 ---------------- tests/rest/client/v1/test_presence.py | 18 +++++++++++++++++- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/synapse/streams/events.py b/synapse/streams/events.py index 8671a8fa4e..699083ae12 100644 --- a/synapse/streams/events.py +++ b/synapse/streams/events.py @@ -23,22 +23,6 @@ from synapse.handlers.typing import TypingNotificationEventSource from synapse.handlers.receipts import ReceiptEventSource -class NullSource(object): - """This event source never yields any events and its token remains at - zero. It may be useful for unit-testing.""" - def __init__(self, hs): - pass - - def get_new_events_for_user(self, user, from_key, limit): - return defer.succeed(([], from_key)) - - def get_current_key(self, direction='f'): - return defer.succeed(0) - - def get_pagination_rows(self, user, pagination_config, key): - return defer.succeed(([], pagination_config.from_key)) - - class EventSources(object): SOURCE_TYPES = { "room": RoomEventSource, diff --git a/tests/rest/client/v1/test_presence.py b/tests/rest/client/v1/test_presence.py index 2ee3da0b34..29d9bbaad4 100644 --- a/tests/rest/client/v1/test_presence.py +++ b/tests/rest/client/v1/test_presence.py @@ -41,6 +41,22 @@ myid = "@apple:test" PATH_PREFIX = "/_matrix/client/api/v1" +class NullSource(object): + """This event source never yields any events and its token remains at + zero. It may be useful for unit-testing.""" + def __init__(self, hs): + pass + + def get_new_events_for_user(self, user, from_key, limit): + return defer.succeed(([], from_key)) + + def get_current_key(self, direction='f'): + return defer.succeed(0) + + def get_pagination_rows(self, user, pagination_config, key): + return defer.succeed(([], pagination_config.from_key)) + + class JustPresenceHandlers(object): def __init__(self, hs): self.presence_handler = PresenceHandler(hs) @@ -243,7 +259,7 @@ class PresenceEventStreamTestCase(unittest.TestCase): # HIDEOUS HACKERY # TODO(paul): This should be injected in via the HomeServer DI system from synapse.streams.events import ( - PresenceEventSource, NullSource, EventSources + PresenceEventSource, EventSources ) old_SOURCE_TYPES = EventSources.SOURCE_TYPES From 7dd4f79c49e1a1ba4cf2edf8b45ed841a32a33b0 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 22 Sep 2015 18:37:07 +0100 Subject: [PATCH 57/82] synapse/storage/_base.py:_execute_and_decode was unused --- synapse/storage/_base.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index c1b5423bd6..cf4ec30f48 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -374,9 +374,6 @@ class SQLBaseStore(object): return self.runInteraction(desc, interaction) - def _execute_and_decode(self, desc, query, *args): - return self._execute(desc, self.cursor_to_dict, query, *args) - # "Simple" SQL API methods that operate on a single table with no JOINs, # no complex WHERE clauses, just a dict of values for columns. From 3559a835a22be3ae474fc2a14363a59b89ec05b8 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 22 Sep 2015 18:39:46 +0100 Subject: [PATCH 58/82] synapse/storage/event_federation.py:_get_auth_events is unused --- synapse/storage/event_federation.py | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index c1cabbaa60..7ed0d9ae1b 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -225,27 +225,6 @@ class EventFederationStore(SQLBaseStore): return results - def _get_auth_events(self, txn, event_id): - auth_ids = self._simple_select_onecol_txn( - txn, - table="event_auth", - keyvalues={ - "event_id": event_id, - }, - retcol="auth_id", - ) - - results = [] - for auth_id in auth_ids: - hashes = self._get_event_reference_hashes_txn(txn, auth_id) - prev_hashes = { - k: encode_base64(v) for k, v in hashes.items() - if k == "sha256" - } - results.append((auth_id, prev_hashes)) - - return results - def get_min_depth(self, room_id): """ For hte given room, get the minimum depth we have seen for it. """ From 8a519ac76d646c991c4f77973a157ac7bea25665 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 23 Sep 2015 09:55:24 +0100 Subject: [PATCH 59/82] Fix demo/start.sh to work with --report-stats --- demo/start.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/demo/start.sh b/demo/start.sh index 572dbfab0b..a90561488d 100755 --- a/demo/start.sh +++ b/demo/start.sh @@ -25,6 +25,7 @@ for port in 8080 8081 8082; do --generate-config \ -H "localhost:$https_port" \ --config-path "$DIR/etc/$port.config" \ + --report-stats no # Check script parameters if [ $# -eq 1 ]; then From 257fa1c53e20b4394ff1493f6112a011c2727e7b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 23 Sep 2015 10:07:31 +0100 Subject: [PATCH 60/82] Set m.room.canonical_alias on room creation. --- synapse/handlers/room.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index bb5eef6bbd..e194f39e70 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -155,6 +155,7 @@ class RoomCreationHandler(BaseHandler): preset_config=preset_config, invite_list=invite_list, initial_state=initial_state, + room_alias=room_alias, ) msg_handler = self.hs.get_handlers().message_handler @@ -202,7 +203,7 @@ class RoomCreationHandler(BaseHandler): defer.returnValue(result) def _create_events_for_new_room(self, creator, room_id, preset_config, - invite_list, initial_state): + invite_list, initial_state, room_alias): config = RoomCreationHandler.PRESETS_DICT[preset_config] creator_id = creator.to_string() @@ -271,6 +272,15 @@ class RoomCreationHandler(BaseHandler): returned_events.append(power_levels_event) + if room_alias: + if (EventTypes.CanonicalAlias, '') not in initial_state: + room_alias_event = create( + etype=EventTypes.CanonicalAlias, + content={"alias": room_alias.to_string()}, + ) + + returned_events.append(room_alias_event) + if (EventTypes.JoinRules, '') not in initial_state: join_rules_event = create( etype=EventTypes.JoinRules, From 04abf53a5633d8220b01772aedc573caa0e71f6e Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 23 Sep 2015 10:17:50 +0100 Subject: [PATCH 61/82] Use argparse for definition finder --- scripts-dev/definitions.py | 62 +++++++++++++++++++++++--------------- 1 file changed, 37 insertions(+), 25 deletions(-) diff --git a/scripts-dev/definitions.py b/scripts-dev/definitions.py index 09cec9d8ec..934bc4a3d8 100755 --- a/scripts-dev/definitions.py +++ b/scripts-dev/definitions.py @@ -92,21 +92,32 @@ def used_names(prefix, defs, names): if __name__ == '__main__': - import sys, os - if not sys.argv[1:]: - sys.stderr.write( - "Usage: definitions.py \n" - " definitions.py \n" - "Either list the definitions matching the regexp or list\n" - " 'unused' definitions\n" - ) + import sys, os, argparse, re + + parser = argparse.ArgumentParser(description='Find definitions.') + parser.add_argument( + "--unused", action="store_true", help="Only list unused definitions" + ) + parser.add_argument( + "--ignore", action="append", metavar="REGEXP", help="Ignore a pattern" + ) + parser.add_argument( + "--pattern", nargs='+', action="append", metavar="REGEXP", + help="Search for a pattern" + ) + parser.add_argument( + "directories", nargs='+', metavar="DIR", + help="Directories to search for definitions" + ) + args = parser.parse_args() definitions = {} - for root, dirs, files in os.walk(sys.argv[1]): - for filename in files: - if filename.endswith(".py"): - filepath = os.path.join(root, filename) - definitions[filepath] = definitions_in_file(filepath) + for directory in args.directories: + for root, dirs, files in os.walk(directory): + for filename in files: + if filename.endswith(".py"): + filepath = os.path.join(root, filename) + definitions[filepath] = definitions_in_file(filepath) names = {} for filepath, defs in definitions.items(): @@ -115,16 +126,17 @@ if __name__ == '__main__': for filepath, defs in definitions.items(): used_names(filepath + ":", defs, names) - if sys.argv[2:]: - import re - pattern = re.compile(sys.argv[2]) - for name in list(names): - if not pattern.match(name): - del names[name] - else: - for name in list(names): - if 'used' in names[name]: - del names[name] + patterns = [re.compile(pattern) for pattern in args.pattern or ()] + ignore = [re.compile(pattern) for pattern in args.ignore or ()] - yaml.dump(names, sys.stdout, default_flow_style=False) - #yaml.dump(definitions, sys.stdout, default_flow_style=False) + result = {} + for name, definition in names.items(): + if patterns and not any(pattern.match(name) for pattern in patterns): + continue + if ignore and any(pattern.match(name) for pattern in ignore): + continue + if args.unused and definition.get('used'): + continue + result[name] = definition + + yaml.dump(result, sys.stdout, default_flow_style=False) From 60728c8c9ebc80c58454639c3f021d43091453ca Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 23 Sep 2015 10:25:26 +0100 Subject: [PATCH 62/82] synapse/handlers/federation.py:_handle_auth_events was unused --- synapse/handlers/federation.py | 49 ---------------------------------- 1 file changed, 49 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 4ff20599d6..f4dce712f9 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -1456,52 +1456,3 @@ class FederationHandler(BaseHandler): }, "missing": [e.event_id for e in missing_locals], }) - - @defer.inlineCallbacks - def _handle_auth_events(self, origin, auth_events): - auth_ids_to_deferred = {} - - def process_auth_ev(ev): - auth_ids = [e_id for e_id, _ in ev.auth_events] - - prev_ds = [ - auth_ids_to_deferred[i] - for i in auth_ids - if i in auth_ids_to_deferred - ] - - d = defer.Deferred() - - auth_ids_to_deferred[ev.event_id] = d - - @defer.inlineCallbacks - def f(*_): - ev.internal_metadata.outlier = True - - try: - auth = { - (e.type, e.state_key): e for e in auth_events - if e.event_id in auth_ids - } - - yield self._handle_new_event( - origin, ev, auth_events=auth - ) - except: - logger.exception( - "Failed to handle auth event %s", - ev.event_id, - ) - - d.callback(None) - - if prev_ds: - dx = defer.DeferredList(prev_ds) - dx.addBoth(f) - else: - f() - - for e in auth_events: - process_auth_ev(e) - - yield defer.DeferredList(auth_ids_to_deferred.values()) From 57338a97683de16b325a2e62b4dee4b4a6b77aa2 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 23 Sep 2015 10:26:45 +0100 Subject: [PATCH 63/82] synapse/handlers/room.py:_should_invite_join was unused --- synapse/handlers/room.py | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 243623190f..d60f86bbd0 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -492,32 +492,6 @@ class RoomMemberHandler(BaseHandler): "user_joined_room", user=user, room_id=room_id ) - @defer.inlineCallbacks - def _should_invite_join(self, room_id, prev_state, do_auth): - logger.debug("_should_invite_join: room_id: %s", room_id) - - # XXX: We don't do an auth check if we are doing an invite - # join dance for now, since we're kinda implicitly checking - # that we are allowed to join when we decide whether or not we - # need to do the invite/join dance. - - # Only do an invite join dance if a) we were invited, - # b) the person inviting was from a differnt HS and c) we are - # not currently in the room - room_host = None - if prev_state and prev_state.membership == Membership.INVITE: - room = yield self.store.get_room(room_id) - inviter = UserID.from_string( - prev_state.sender - ) - - is_remote_invite_join = not self.hs.is_mine(inviter) and not room - room_host = inviter.domain - else: - is_remote_invite_join = False - - defer.returnValue((is_remote_invite_join, room_host)) - @defer.inlineCallbacks def get_joined_rooms_for_user(self, user): """Returns a list of roomids that the user has any of the given From 82b8d4b86a981559570fbd1e8ee3dcd38faf553f Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 23 Sep 2015 10:27:47 +0100 Subject: [PATCH 64/82] synapse/state.py:_get_state_key_from_event was unused --- synapse/state.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/synapse/state.py b/synapse/state.py index ed36f844cb..bb225c39cf 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -31,10 +31,6 @@ import hashlib logger = logging.getLogger(__name__) -def _get_state_key_from_event(event): - return event.state_key - - KeyStateTuple = namedtuple("KeyStateTuple", ("context", "type", "state_key")) From 1ee3d26432d87ff312350f21da982f646b5af49a Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 23 Sep 2015 10:30:03 +0100 Subject: [PATCH 65/82] synapse/storage/_base.py:_simple_selectupdate_one was unused --- synapse/storage/_base.py | 31 ------------------------------- tests/storage/test_base.py | 20 -------------------- 2 files changed, 51 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index cf4ec30f48..79021bde6b 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -686,37 +686,6 @@ class SQLBaseStore(object): return dict(zip(retcols, row)) - def _simple_selectupdate_one(self, table, keyvalues, updatevalues=None, - retcols=None, allow_none=False, - desc="_simple_selectupdate_one"): - """ Combined SELECT then UPDATE.""" - def func(txn): - ret = None - if retcols: - ret = self._simple_select_one_txn( - txn, - table=table, - keyvalues=keyvalues, - retcols=retcols, - allow_none=allow_none, - ) - - if updatevalues: - self._simple_update_one_txn( - txn, - table=table, - keyvalues=keyvalues, - updatevalues=updatevalues, - ) - - # if txn.rowcount == 0: - # raise StoreError(404, "No row found") - if txn.rowcount > 1: - raise StoreError(500, "More than one row matched") - - return ret - return self.runInteraction(desc, func) - def _simple_delete_one(self, table, keyvalues, desc="_simple_delete_one"): """Executes a DELETE query on the named table, expecting to delete a single row. diff --git a/tests/storage/test_base.py b/tests/storage/test_base.py index 8573f18b55..1ddca1da4c 100644 --- a/tests/storage/test_base.py +++ b/tests/storage/test_base.py @@ -185,26 +185,6 @@ class SQLBaseStoreTestCase(unittest.TestCase): [3, 4, 1, 2] ) - @defer.inlineCallbacks - def test_update_one_with_return(self): - self.mock_txn.rowcount = 1 - self.mock_txn.fetchone.return_value = ("Old Value",) - - ret = yield self.datastore._simple_selectupdate_one( - table="tablename", - keyvalues={"keycol": "TheKey"}, - updatevalues={"columname": "New Value"}, - retcols=["columname"] - ) - - self.assertEquals({"columname": "Old Value"}, ret) - self.mock_txn.execute.assert_has_calls([ - call('SELECT columname FROM tablename WHERE keycol = ?', - ['TheKey']), - call("UPDATE tablename SET columname = ? WHERE keycol = ?", - ["New Value", "TheKey"]) - ]) - @defer.inlineCallbacks def test_delete_one(self): self.mock_txn.rowcount = 1 From 1d9036aff2be9ce089114b1389cc86a34ccd8490 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 23 Sep 2015 10:30:25 +0100 Subject: [PATCH 66/82] synapse/storage/_base.py:_simple_delete was unused --- synapse/storage/_base.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 79021bde6b..33751f3092 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -707,16 +707,6 @@ class SQLBaseStore(object): raise StoreError(500, "more than one row matched") return self.runInteraction(desc, func) - def _simple_delete(self, table, keyvalues, desc="_simple_delete"): - """Executes a DELETE query on the named table. - - Args: - table : string giving the table name - keyvalues : dict of column names and values to select the row with - """ - - return self.runInteraction(desc, self._simple_delete_txn) - def _simple_delete_txn(self, txn, table, keyvalues): sql = "DELETE FROM %s WHERE %s" % ( table, From 396834f1c010899cae87aac45143bbd9911adb4f Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 23 Sep 2015 10:30:38 +0100 Subject: [PATCH 67/82] synapse/storage/_base.py:_simple_max_id was unused --- synapse/storage/_base.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 33751f3092..693784ad38 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -715,24 +715,6 @@ class SQLBaseStore(object): return txn.execute(sql, keyvalues.values()) - def _simple_max_id(self, table): - """Executes a SELECT query on the named table, expecting to return the - max value for the column "id". - - Args: - table : string giving the table name - """ - sql = "SELECT MAX(id) AS id FROM %s" % table - - def func(txn): - txn.execute(sql) - max_id = self.cursor_to_dict(txn)[0]["id"] - if max_id is None: - return 0 - return max_id - - return self.runInteraction("_simple_max_id", func) - def get_next_stream_id(self): with self._next_stream_id_lock: i = self._next_stream_id From c292dba70cca26751cdab90a6e50200828959642 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 23 Sep 2015 10:31:25 +0100 Subject: [PATCH 68/82] Remove unused functions from synapse/storage/event_federation.py --- synapse/storage/event_federation.py | 71 ----------------------------- 1 file changed, 71 deletions(-) diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py index 7ed0d9ae1b..6d4421dd8f 100644 --- a/synapse/storage/event_federation.py +++ b/synapse/storage/event_federation.py @@ -154,77 +154,6 @@ class EventFederationStore(SQLBaseStore): return results - def _get_latest_state_in_room(self, txn, room_id, type, state_key): - event_ids = self._simple_select_onecol_txn( - txn, - table="state_forward_extremities", - keyvalues={ - "room_id": room_id, - "type": type, - "state_key": state_key, - }, - retcol="event_id", - ) - - results = [] - for event_id in event_ids: - hashes = self._get_event_reference_hashes_txn(txn, event_id) - prev_hashes = { - k: encode_base64(v) for k, v in hashes.items() - if k == "sha256" - } - results.append((event_id, prev_hashes)) - - return results - - def _get_prev_events(self, txn, event_id): - results = self._get_prev_events_and_state( - txn, - event_id, - is_state=0, - ) - - return [(e_id, h, ) for e_id, h, _ in results] - - def _get_prev_state(self, txn, event_id): - results = self._get_prev_events_and_state( - txn, - event_id, - is_state=True, - ) - - return [(e_id, h, ) for e_id, h, _ in results] - - def _get_prev_events_and_state(self, txn, event_id, is_state=None): - keyvalues = { - "event_id": event_id, - } - - if is_state is not None: - keyvalues["is_state"] = bool(is_state) - - res = self._simple_select_list_txn( - txn, - table="event_edges", - keyvalues=keyvalues, - retcols=["prev_event_id", "is_state"], - ) - - hashes = self._get_prev_event_hashes_txn(txn, event_id) - - results = [] - for d in res: - edge_hash = self._get_event_reference_hashes_txn(txn, d["prev_event_id"]) - edge_hash.update(hashes.get(d["prev_event_id"], {})) - prev_hashes = { - k: encode_base64(v) - for k, v in edge_hash.items() - if k == "sha256" - } - results.append((d["prev_event_id"], prev_hashes, d["is_state"])) - - return results - def get_min_depth(self, room_id): """ For hte given room, get the minimum depth we have seen for it. """ From 92d8d724c5c34f0a83cbd8c5dce7f0c0c21a1568 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 23 Sep 2015 10:33:06 +0100 Subject: [PATCH 69/82] Remove unused functions from synapse/storage/events.py --- synapse/storage/events.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 46df6b4d6d..416ef6af93 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -890,22 +890,11 @@ class EventsStore(SQLBaseStore): return ev - def _parse_events(self, rows): - return self.runInteraction( - "_parse_events", self._parse_events_txn, rows - ) - def _parse_events_txn(self, txn, rows): event_ids = [r["event_id"] for r in rows] return self._get_events_txn(txn, event_ids) - def _has_been_redacted_txn(self, txn, event): - sql = "SELECT event_id FROM redactions WHERE redacts = ?" - txn.execute(sql, (event.event_id,)) - result = txn.fetchone() - return result[0] if result else None - @defer.inlineCallbacks def count_daily_messages(self): """ From e51aa4be9691e7c6918d72810ee0b751f2e48797 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 23 Sep 2015 10:35:10 +0100 Subject: [PATCH 70/82] synapse/storage/roommember.py:_get_members_query was unused --- synapse/storage/roommember.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index cd9eefbd9f..e17cbe677a 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -178,12 +178,6 @@ class RoomMemberStore(SQLBaseStore): return joined_domains - def _get_members_query(self, where_clause, where_values): - return self.runInteraction( - "get_members_query", self._get_members_events_txn, - where_clause, where_values - ).addCallbacks(self._get_events) - def _get_members_events_txn(self, txn, room_id, membership=None, user_id=None): rows = self._get_members_rows_txn( txn, From 973ebb66bacfeece1f88f8ea71b86186f9c9163e Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 23 Sep 2015 10:36:33 +0100 Subject: [PATCH 71/82] Remove unused functions from synapse/storage/signatures.py --- synapse/storage/signatures.py | 112 ---------------------------------- 1 file changed, 112 deletions(-) diff --git a/synapse/storage/signatures.py b/synapse/storage/signatures.py index ab57b92174..b070be504d 100644 --- a/synapse/storage/signatures.py +++ b/synapse/storage/signatures.py @@ -24,41 +24,6 @@ from synapse.crypto.event_signing import compute_event_reference_hash class SignatureStore(SQLBaseStore): """Persistence for event signatures and hashes""" - def _get_event_content_hashes_txn(self, txn, event_id): - """Get all the hashes for a given Event. - Args: - txn (cursor): - event_id (str): Id for the Event. - Returns: - A dict of algorithm -> hash. - """ - query = ( - "SELECT algorithm, hash" - " FROM event_content_hashes" - " WHERE event_id = ?" - ) - txn.execute(query, (event_id, )) - return dict(txn.fetchall()) - - def _store_event_content_hash_txn(self, txn, event_id, algorithm, - hash_bytes): - """Store a hash for a Event - Args: - txn (cursor): - event_id (str): Id for the Event. - algorithm (str): Hashing algorithm. - hash_bytes (bytes): Hash function output bytes. - """ - self._simple_insert_txn( - txn, - "event_content_hashes", - { - "event_id": event_id, - "algorithm": algorithm, - "hash": buffer(hash_bytes), - }, - ) - def get_event_reference_hashes(self, event_ids): def f(txn): return [ @@ -123,80 +88,3 @@ class SignatureStore(SQLBaseStore): table="event_reference_hashes", values=vals, ) - - def _get_event_signatures_txn(self, txn, event_id): - """Get all the signatures for a given PDU. - Args: - txn (cursor): - event_id (str): Id for the Event. - Returns: - A dict of sig name -> dict(key_id -> signature_bytes) - """ - query = ( - "SELECT signature_name, key_id, signature" - " FROM event_signatures" - " WHERE event_id = ? " - ) - txn.execute(query, (event_id, )) - rows = txn.fetchall() - - res = {} - - for name, key, sig in rows: - res.setdefault(name, {})[key] = sig - - return res - - def _store_event_signature_txn(self, txn, event_id, signature_name, key_id, - signature_bytes): - """Store a signature from the origin server for a PDU. - Args: - txn (cursor): - event_id (str): Id for the Event. - origin (str): origin of the Event. - key_id (str): Id for the signing key. - signature (bytes): The signature. - """ - self._simple_insert_txn( - txn, - "event_signatures", - { - "event_id": event_id, - "signature_name": signature_name, - "key_id": key_id, - "signature": buffer(signature_bytes), - }, - ) - - def _get_prev_event_hashes_txn(self, txn, event_id): - """Get all the hashes for previous PDUs of a PDU - Args: - txn (cursor): - event_id (str): Id for the Event. - Returns: - dict of (pdu_id, origin) -> dict of algorithm -> hash_bytes. - """ - query = ( - "SELECT prev_event_id, algorithm, hash" - " FROM event_edge_hashes" - " WHERE event_id = ?" - ) - txn.execute(query, (event_id, )) - results = {} - for prev_event_id, algorithm, hash_bytes in txn.fetchall(): - hashes = results.setdefault(prev_event_id, {}) - hashes[algorithm] = hash_bytes - return results - - def _store_prev_event_hash_txn(self, txn, event_id, prev_event_id, - algorithm, hash_bytes): - self._simple_insert_txn( - txn, - "event_edge_hashes", - { - "event_id": event_id, - "prev_event_id": prev_event_id, - "algorithm": algorithm, - "hash": buffer(hash_bytes), - }, - ) From 1cd65a8d1e9fe30506aa7f19c60e6271a24e6ae9 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 23 Sep 2015 10:37:58 +0100 Subject: [PATCH 72/82] synapse/storage/state.py: _make_group_id was unused --- synapse/storage/state.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/synapse/storage/state.py b/synapse/storage/state.py index 9630efcfcc..e935b9443b 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -20,8 +20,6 @@ from synapse.util.caches.descriptors import ( from twisted.internet import defer -from synapse.util.stringutils import random_string - import logging logger = logging.getLogger(__name__) @@ -428,7 +426,3 @@ class StateStore(SQLBaseStore): } defer.returnValue(results) - - -def _make_group_id(clock): - return str(int(clock.time_msec())) + random_string(5) From 7d55314277e29afb25da9c0910b8b0422d422671 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 23 Sep 2015 10:42:02 +0100 Subject: [PATCH 73/82] Remove unused _execute_and_decode from scripts/synapse_port_db --- scripts/synapse_port_db | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/synapse_port_db b/scripts/synapse_port_db index 6aba72e459..62515997b1 100755 --- a/scripts/synapse_port_db +++ b/scripts/synapse_port_db @@ -95,8 +95,6 @@ class Store(object): _simple_update_one = SQLBaseStore.__dict__["_simple_update_one"] _simple_update_one_txn = SQLBaseStore.__dict__["_simple_update_one_txn"] - _execute_and_decode = SQLBaseStore.__dict__["_execute_and_decode"] - def runInteraction(self, desc, func, *args, **kwargs): def r(conn): try: From 314aabba82d382d36f9ac91050e77deb6018fbcd Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 23 Sep 2015 10:45:33 +0100 Subject: [PATCH 74/82] Fix scripts-dev/definitions.py argparse options --- scripts-dev/definitions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts-dev/definitions.py b/scripts-dev/definitions.py index 934bc4a3d8..f0d0cd8a3f 100755 --- a/scripts-dev/definitions.py +++ b/scripts-dev/definitions.py @@ -102,7 +102,7 @@ if __name__ == '__main__': "--ignore", action="append", metavar="REGEXP", help="Ignore a pattern" ) parser.add_argument( - "--pattern", nargs='+', action="append", metavar="REGEXP", + "--pattern", action="append", metavar="REGEXP", help="Search for a pattern" ) parser.add_argument( From cf1100887b454535e25dfeb67d649c2a4673eab7 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 24 Sep 2015 17:34:02 +0100 Subject: [PATCH 75/82] Fix order of ON constraints in _get_rooms_for_user_where_membership_is_txn --- synapse/storage/roommember.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index cd9eefbd9f..41c939efb1 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -142,12 +142,12 @@ class RoomMemberStore(SQLBaseStore): sql = ( "SELECT m.room_id, m.sender, m.membership, m.event_id, e.stream_ordering" - " FROM room_memberships as m" - " INNER JOIN current_state_events as c" - " ON e.event_id = c.event_id " - " INNER JOIN events as e " - " ON m.event_id = c.event_id " - " AND m.room_id = c.room_id " + " FROM current_state_events as c" + " INNER JOIN room_memberships as m" + " ON m.event_id = c.event_id" + " INNER JOIN events as e" + " ON e.event_id = c.event_id" + " AND m.room_id = c.room_id" " AND m.user_id = c.state_key" " WHERE %s" ) % (where_clause,) From 76328b85f66f03908a36904102b6b8437fa41663 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 24 Sep 2015 21:50:20 +0000 Subject: [PATCH 76/82] Allow config file path to be configurable in in synctl Also, allow it to be run from directories other than the synapse directory --- synapse/app/synctl.py | 50 ++++++++++++++++++++++--------------------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/synapse/app/synctl.py b/synapse/app/synctl.py index 6bcc437591..af4f9ff7e0 100755 --- a/synapse/app/synctl.py +++ b/synapse/app/synctl.py @@ -16,38 +16,25 @@ import sys import os +import os.path import subprocess import signal import yaml SYNAPSE = ["python", "-B", "-m", "synapse.app.homeserver"] -CONFIGFILE = "homeserver.yaml" - GREEN = "\x1b[1;32m" RED = "\x1b[1;31m" NORMAL = "\x1b[m" -if not os.path.exists(CONFIGFILE): - sys.stderr.write( - "No config file found\n" - "To generate a config file, run '%s -c %s --generate-config" - " --server-name='\n" % ( - " ".join(SYNAPSE), CONFIGFILE - ) - ) - sys.exit(1) -CONFIG = yaml.load(open(CONFIGFILE)) -PIDFILE = CONFIG["pid_file"] - - -def start(): +def start(configfile): print "Starting ...", args = SYNAPSE - args.extend(["--daemonize", "-c", CONFIGFILE]) + args.extend(["--daemonize", "-c", configfile]) + cwd = os.path.dirname(os.path.abspath(__file__)) try: - subprocess.check_call(args) + subprocess.check_call(args, cwd=cwd) print GREEN + "started" + NORMAL except subprocess.CalledProcessError as e: print ( @@ -57,24 +44,39 @@ def start(): ) -def stop(): - if os.path.exists(PIDFILE): - pid = int(open(PIDFILE).read()) +def stop(pidfile): + if os.path.exists(pidfile): + pid = int(open(pidfile).read()) os.kill(pid, signal.SIGTERM) print GREEN + "stopped" + NORMAL def main(): + configfile = sys.argv[2] if len(sys.argv) == 3 else "homeserver.yaml" + + if not os.path.exists(configfile): + sys.stderr.write( + "No config file found\n" + "To generate a config file, run '%s -c %s --generate-config" + " --server-name='\n" % ( + " ".join(SYNAPSE), configfile + ) + ) + sys.exit(1) + + config = yaml.load(open(configfile)) + pidfile = config["pid_file"] + action = sys.argv[1] if sys.argv[1:] else "usage" if action == "start": - start() + start(configfile) elif action == "stop": - stop() + stop(pidfile) elif action == "restart": stop() start() else: - sys.stderr.write("Usage: %s [start|stop|restart]\n" % (sys.argv[0],)) + sys.stderr.write("Usage: %s [start|stop|restart] [configfile]\n" % (sys.argv[0],)) sys.exit(1) From f87a11e0fdf4c1a6ebca1ab3f5e8cf33ac34c2b4 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 24 Sep 2015 21:59:38 +0000 Subject: [PATCH 77/82] Fix restart --- synapse/app/synctl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/app/synctl.py b/synapse/app/synctl.py index af4f9ff7e0..1078d19b79 100755 --- a/synapse/app/synctl.py +++ b/synapse/app/synctl.py @@ -73,8 +73,8 @@ def main(): elif action == "stop": stop(pidfile) elif action == "restart": - stop() - start() + stop(pidfile) + start(configfile) else: sys.stderr.write("Usage: %s [start|stop|restart] [configfile]\n" % (sys.argv[0],)) sys.exit(1) From 9d39615b7d4d5525ab814d2d84e7f2b4523d0417 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 30 Sep 2015 16:37:59 +0100 Subject: [PATCH 78/82] Rename var --- synapse/handlers/federation.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index e79a82cfc0..3ce1aee52c 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -1033,15 +1033,15 @@ class FederationHandler(BaseHandler): break for e in itertools.chain(auth_events, state, [event]): - a = { + auth_for_e = { (event_map[e_id].type, event_map[e_id].state_key): event_map[e_id] for e_id, _ in e.auth_events } if create_event: - a[(EventTypes.Create, "")] = create_event + auth_for_e[(EventTypes.Create, "")] = create_event try: - self.auth.check(e, auth_events=a) + self.auth.check(e, auth_events=auth_for_e) except AuthError as err: logger.warn( "Rejecting %s because %s", From 83892d0d3039965ae3075df166cbdbd7339cb0bc Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 30 Sep 2015 16:41:48 +0100 Subject: [PATCH 79/82] Comment --- synapse/handlers/federation.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 3ce1aee52c..17f4ddd325 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -1013,6 +1013,14 @@ class FederationHandler(BaseHandler): @defer.inlineCallbacks def _persist_auth_tree(self, auth_events, state, event): + """Checks the auth chain is valid (and passes auth checks) for the + state and event. Then persists the auth chain and state atomically. + Persists the event seperately. + + Returns: + 2-tuple of (event_stream_id, max_stream_id) from the persist_event + call for `event` + """ events_to_context = {} for e in itertools.chain(auth_events, state): ctx = yield self.state_handler.compute_event_context( From ecd0c0dfc50ceed16aa47cf066bc412211af2335 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 30 Sep 2015 16:46:24 +0100 Subject: [PATCH 80/82] Remove double indentation --- synapse/handlers/room.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index e194f39e70..2b15136bda 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -272,14 +272,13 @@ class RoomCreationHandler(BaseHandler): returned_events.append(power_levels_event) - if room_alias: - if (EventTypes.CanonicalAlias, '') not in initial_state: - room_alias_event = create( - etype=EventTypes.CanonicalAlias, - content={"alias": room_alias.to_string()}, - ) + if room_alias and (EventTypes.CanonicalAlias, '') not in initial_state: + room_alias_event = create( + etype=EventTypes.CanonicalAlias, + content={"alias": room_alias.to_string()}, + ) - returned_events.append(room_alias_event) + returned_events.append(room_alias_event) if (EventTypes.JoinRules, '') not in initial_state: join_rules_event = create( From 0a4b7226fc0ce163c7ba2a1a62d6125b3fd1e55d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 1 Oct 2015 09:21:27 +0100 Subject: [PATCH 81/82] Don't change cwd in synctl --- synapse/app/synctl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/app/synctl.py b/synapse/app/synctl.py index 1078d19b79..5d82beed0e 100755 --- a/synapse/app/synctl.py +++ b/synapse/app/synctl.py @@ -32,9 +32,9 @@ def start(configfile): print "Starting ...", args = SYNAPSE args.extend(["--daemonize", "-c", configfile]) - cwd = os.path.dirname(os.path.abspath(__file__)) + try: - subprocess.check_call(args, cwd=cwd) + subprocess.check_call(args) print GREEN + "started" + NORMAL except subprocess.CalledProcessError as e: print ( From bad780a19705cbffcdd181d3ffc81f10980ed109 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 1 Oct 2015 14:01:52 +0100 Subject: [PATCH 82/82] Validate the receipt type before passing it on to the receipt handler --- synapse/rest/client/v2_alpha/receipts.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/synapse/rest/client/v2_alpha/receipts.py b/synapse/rest/client/v2_alpha/receipts.py index 52e99f54d5..b107b7ce17 100644 --- a/synapse/rest/client/v2_alpha/receipts.py +++ b/synapse/rest/client/v2_alpha/receipts.py @@ -15,6 +15,7 @@ from twisted.internet import defer +from synapse.api.errors import SynapseError from synapse.http.servlet import RestServlet from ._base import client_v2_pattern @@ -41,6 +42,9 @@ class ReceiptRestServlet(RestServlet): def on_POST(self, request, room_id, receipt_type, event_id): user, _ = yield self.auth.get_user_by_req(request) + if receipt_type != "m.read": + raise SynapseError(400, "Receipt type must be 'm.read'") + yield self.receipts_handler.received_client_receipt( room_id, receipt_type,