Merge remote-tracking branch 'origin/develop' into store_event_actions

jenkins.sh

@@ -6,7 +6,6 @@ export PYTHONDONTWRITEBYTECODE=yep
 export TRIAL_FLAGS="--reporter=subunit"
 export TOXSUFFIX="| subunit-1to2 | subunit2junitxml --no-passthrough --output-to=results.xml"
 # Write coverage reports to a separate file for each process
-# Include branch coverage
 export COVERAGE_OPTS="-p"
 export DUMP_COVERAGE_COMMAND="coverage help"
 
@@ -16,15 +15,13 @@ export DUMP_COVERAGE_COMMAND="coverage help"
 # UNSTABLE or FAILURE this build.
 export PEP8SUFFIX="--output-file=violations.flake8.log || echo flake8 finished with status code \$?"
 
-rm .coverage.* || echo "No files to remove"
+rm .coverage* || echo "No coverage files to remove"
 
 tox
 
 : ${GIT_BRANCH:="origin/$(git rev-parse --abbrev-ref HEAD)"}
 
-set +u
-. .tox/py27/bin/activate
-set -u
+TOX_BIN=$WORKSPACE/.tox/py27/bin
 
 if [[ ! -e .sytest-base ]]; then
   git clone https://github.com/matrix-org/sytest.git .sytest-base --mirror
@@ -48,7 +45,8 @@ export PERL5LIB PERL_MB_OPT PERL_MM_OPT
 : ${PORT_BASE:=8000}
 
 echo >&2 "Running sytest with SQLite3";
-./run-tests.pl --coverage -O tap --synapse-directory .. --all --port-base $PORT_BASE > results-sqlite3.tap
+./run-tests.pl --coverage -O tap --synapse-directory $WORKSPACE \
+    --python $TOX_BIN/python --all --port-base $PORT_BASE > results-sqlite3.tap
 
 RUN_POSTGRES=""
 
@@ -66,8 +64,9 @@ done
 # Run if both postgresql databases exist
 if test $RUN_POSTGRES = ":$(($PORT_BASE + 1)):$(($PORT_BASE + 2))"; then
     echo >&2 "Running sytest with PostgreSQL";
-    pip install psycopg2
-    ./run-tests.pl --coverage -O tap --synapse-directory .. --all --port-base $PORT_BASE > results-postgresql.tap
+    $TOX_BIN/pip install psycopg2
+    ./run-tests.pl --coverage -O tap --synapse-directory $WORKSPACE \
+        --python $TOX_BIN/python --all --port-base $PORT_BASE > results-postgresql.tap
 else
     echo >&2 "Skipping running sytest with PostgreSQL, $RUN_POSTGRES"
 fi
@@ -76,6 +75,7 @@ cd ..
 cp sytest/.coverage.* .
 
 # Combine the coverage reports
-python -m coverage combine
+echo "Combining:" .coverage.*
+$TOX_BIN/python -m coverage combine
 # Output coverage to coverage.xml
-coverage xml -o coverage.xml
+$TOX_BIN/coverage xml -o coverage.xml

synapse/api/errors.py

@@ -120,6 +120,22 @@ class AuthError(SynapseError):
         super(AuthError, self).__init__(*args, **kwargs)
 
 
+class GuestAccessError(AuthError):
+    """An error raised when a there is a problem with a guest user accessing
+    a room"""
+
+    def __init__(self, rooms, *args, **kwargs):
+        self.rooms = rooms
+        super(GuestAccessError, self).__init__(*args, **kwargs)
+
+    def error_dict(self):
+        return cs_error(
+            self.msg,
+            self.errcode,
+            rooms=self.rooms,
+        )
+
+
 class EventSizeError(SynapseError):
     """An error raised when an event is too big."""
 

synapse/api/filtering.py

@@ -62,10 +62,29 @@ class Filtering(object):
                 self._check_definition(user_filter_json[key])
 
         if "room" in user_filter_json:
+            self._check_definition_room_lists(user_filter_json["room"])
             for key in room_level_definitions:
                 if key in user_filter_json["room"]:
                     self._check_definition(user_filter_json["room"][key])
 
+    def _check_definition_room_lists(self, definition):
+        """Check that "rooms" and "not_rooms" are lists of room ids if they
+        are present
+
+        Args:
+            definition(dict): The filter definition
+        Raises:
+            SynapseError: If there was a problem with this definition.
+        """
+        # check rooms are valid room IDs
+        room_id_keys = ["rooms", "not_rooms"]
+        for key in room_id_keys:
+            if key in definition:
+                if type(definition[key]) != list:
+                    raise SynapseError(400, "Expected %s to be a list." % key)
+                for room_id in definition[key]:
+                    RoomID.from_string(room_id)
+
     def _check_definition(self, definition):
         """Check if the provided definition is valid.
 
@@ -85,14 +104,7 @@ class Filtering(object):
                 400, "Expected JSON object, not %s" % (definition,)
             )
 
-        # check rooms are valid room IDs
-        room_id_keys = ["rooms", "not_rooms"]
-        for key in room_id_keys:
-            if key in definition:
-                if type(definition[key]) != list:
-                    raise SynapseError(400, "Expected %s to be a list." % key)
-                for room_id in definition[key]:
-                    RoomID.from_string(room_id)
+        self._check_definition_room_lists(definition)
 
         # check senders are valid user IDs
         user_id_keys = ["senders", "not_senders"]
@@ -119,34 +131,27 @@ class FilterCollection(object):
     def __init__(self, filter_json):
         self.filter_json = filter_json
 
-        self.room_timeline_filter = Filter(
-            self.filter_json.get("room", {}).get("timeline", {})
-        )
+        room_filter_json = self.filter_json.get("room", {})
 
-        self.room_state_filter = Filter(
-            self.filter_json.get("room", {}).get("state", {})
-        )
+        self.room_filter = Filter({
+            k: v for k, v in room_filter_json.items()
+            if k in ("rooms", "not_rooms")
+        })
 
-        self.room_ephemeral_filter = Filter(
-            self.filter_json.get("room", {}).get("ephemeral", {})
-        )
-
-        self.room_account_data = Filter(
-            self.filter_json.get("room", {}).get("account_data", {})
-        )
-
-        self.presence_filter = Filter(
-            self.filter_json.get("presence", {})
-        )
-
-        self.account_data = Filter(
-            self.filter_json.get("account_data", {})
-        )
+        self.room_timeline_filter = Filter(room_filter_json.get("timeline", {}))
+        self.room_state_filter = Filter(room_filter_json.get("state", {}))
+        self.room_ephemeral_filter = Filter(room_filter_json.get("ephemeral", {}))
+        self.room_account_data = Filter(room_filter_json.get("account_data", {}))
+        self.presence_filter = Filter(self.filter_json.get("presence", {}))
+        self.account_data = Filter(self.filter_json.get("account_data", {}))
 
         self.include_leave = self.filter_json.get("room", {}).get(
             "include_leave", False
         )
 
+    def list_rooms(self):
+        return self.room_filter.list_rooms()
+
     def timeline_limit(self):
         return self.room_timeline_filter.limit()
 
@@ -163,22 +168,31 @@ class FilterCollection(object):
         return self.account_data.filter(events)
 
     def filter_room_state(self, events):
-        return self.room_state_filter.filter(events)
+        return self.room_state_filter.filter(self.room_filter.filter(events))
 
     def filter_room_timeline(self, events):
-        return self.room_timeline_filter.filter(events)
+        return self.room_timeline_filter.filter(self.room_filter.filter(events))
 
     def filter_room_ephemeral(self, events):
-        return self.room_ephemeral_filter.filter(events)
+        return self.room_ephemeral_filter.filter(self.room_filter.filter(events))
 
     def filter_room_account_data(self, events):
-        return self.room_account_data.filter(events)
+        return self.room_account_data.filter(self.room_filter.filter(events))
 
 
 class Filter(object):
     def __init__(self, filter_json):
         self.filter_json = filter_json
 
+    def list_rooms(self):
+        """The list of room_id strings this filter restricts the output to
+        or None if the this filter doesn't list the room ids.
+        """
+        if "rooms" in self.filter_json:
+            return list(set(self.filter_json["rooms"]))
+        else:
+            return None
+
     def check(self, event):
         """Checks whether the filter matches the given event.
 

synapse/handlers/room.py

@@ -816,7 +816,8 @@ class RoomListHandler(BaseHandler):
     @defer.inlineCallbacks
     def get_public_room_list(self):
         chunk = yield self.store.get_rooms(is_public=True)
-        results = yield defer.gatherResults(
+
+        room_members = yield defer.gatherResults(
             [
                 self.store.get_users_in_room(room["room_id"])
                 for room in chunk
@@ -824,12 +825,30 @@ class RoomListHandler(BaseHandler):
             consumeErrors=True,
         ).addErrback(unwrapFirstError)
 
+        avatar_urls = yield defer.gatherResults(
+            [
+                self.get_room_avatar_url(room["room_id"])
+                for room in chunk
+            ],
+            consumeErrors=True,
+        ).addErrback(unwrapFirstError)
+
         for i, room in enumerate(chunk):
-            room["num_joined_members"] = len(results[i])
+            room["num_joined_members"] = len(room_members[i])
+            if avatar_urls[i]:
+                room["avatar_url"] = avatar_urls[i]
 
         # FIXME (erikj): START is no longer a valid value
         defer.returnValue({"start": "START", "end": "END", "chunk": chunk})
 
+    @defer.inlineCallbacks
+    def get_room_avatar_url(self, room_id):
+        event = yield self.hs.get_state_handler().get_current_state(
+            room_id, "m.room.avatar"
+        )
+        if event and "url" in event.content:
+            defer.returnValue(event.content["url"])
+
 
 class RoomContextHandler(BaseHandler):
     @defer.inlineCallbacks

synapse/handlers/sync.py

@@ -15,8 +15,8 @@
 
 from ._base import BaseHandler
 
-from synapse.streams.config import PaginationConfig
 from synapse.api.constants import Membership, EventTypes
+from synapse.api.errors import GuestAccessError
 from synapse.util import unwrapFirstError
 
 from twisted.internet import defer
@@ -29,6 +29,7 @@ logger = logging.getLogger(__name__)
 
 SyncConfig = collections.namedtuple("SyncConfig", [
     "user",
+    "is_guest",
     "filter",
 ])
 
@@ -120,6 +121,8 @@ class SyncResult(collections.namedtuple("SyncResult", [
             self.presence or self.joined or self.invited
         )
 
+GuestRoom = collections.namedtuple("GuestRoom", ("room_id", "membership"))
+
 
 class SyncHandler(BaseHandler):
 
@@ -138,6 +141,18 @@ class SyncHandler(BaseHandler):
             A Deferred SyncResult.
         """
 
+        if sync_config.is_guest:
+            bad_rooms = []
+            for room_id in sync_config.filter.list_rooms():
+                world_readable = yield self._is_world_readable(room_id)
+                if not world_readable:
+                    bad_rooms.append(room_id)
+
+            if bad_rooms:
+                raise GuestAccessError(
+                    bad_rooms, 403, "Guest access not allowed"
+                )
+
         if timeout == 0 or since_token is None or full_state:
             # we are going to return immediately, so don't bother calling
             # notifier.wait_for_events.
@@ -154,6 +169,17 @@ class SyncHandler(BaseHandler):
             )
             defer.returnValue(result)
 
+    @defer.inlineCallbacks
+    def _is_world_readable(self, room_id):
+        state = yield self.hs.get_state_handler().get_current_state(
+            room_id,
+            EventTypes.RoomHistoryVisibility
+        )
+        if state and "history_visibility" in state.content:
+            defer.returnValue(state.content["history_visibility"] == "world_readable")
+        else:
+            defer.returnValue(False)
+
     def current_sync_for_user(self, sync_config, since_token=None,
                               full_state=False):
         """Get the sync for client needed to match what the server has now.
@@ -189,20 +215,17 @@ class SyncHandler(BaseHandler):
         """
         now_token = yield self.event_sources.get_current_token()
 
-        now_token, ephemeral_by_room = yield self.ephemeral_by_room(
-            sync_config, now_token
-        )
+        if sync_config.is_guest:
+            room_list = [
+                GuestRoom(room_id, Membership.JOIN)
+                for room_id in sync_config.filter.list_rooms()
+            ]
 
-        presence_stream = self.event_sources.sources["presence"]
-        # TODO (mjark): This looks wrong, shouldn't we be getting the presence
-        # UP to the present rather than after the present?
-        pagination_config = PaginationConfig(from_token=now_token)
-        presence, _ = yield presence_stream.get_pagination_rows(
-            user=sync_config.user,
-            pagination_config=pagination_config.get_source_config("presence"),
-            key=None
-        )
+            account_data = {}
+            account_data_by_room = {}
+            tags_by_room = {}
 
+        else:
             membership_list = (Membership.INVITE, Membership.JOIN)
             if sync_config.filter.include_leave:
                 membership_list += (Membership.LEAVE, Membership.BAN)
@@ -222,6 +245,24 @@ class SyncHandler(BaseHandler):
                 sync_config.user.to_string()
             )
 
+        presence_stream = self.event_sources.sources["presence"]
+
+        joined_room_ids = [
+            room.room_id for room in room_list
+            if room.membership == Membership.JOIN
+        ]
+
+        presence, _ = yield presence_stream.get_new_events(
+            from_key=0,
+            user=sync_config.user,
+            room_ids=joined_room_ids,
+            is_guest=sync_config.is_guest,
+        )
+
+        now_token, ephemeral_by_room = yield self.ephemeral_by_room(
+            sync_config, now_token, joined_room_ids
+        )
+
         joined = []
         invited = []
         archived = []
@@ -338,11 +379,13 @@ class SyncHandler(BaseHandler):
         return account_data_events
 
     @defer.inlineCallbacks
-    def ephemeral_by_room(self, sync_config, now_token, since_token=None):
+    def ephemeral_by_room(self, sync_config, now_token, room_ids,
+                          since_token=None):
         """Get the ephemeral events for each room the user is in
         Args:
             sync_config (SyncConfig): The flags, filters and user for the sync.
             now_token (StreamToken): Where the server is currently up to.
+            room_ids (list): List of room id strings to get data for.
             since_token (StreamToken): Where the server was when the client
                 last synced.
         Returns:
@@ -353,9 +396,6 @@ class SyncHandler(BaseHandler):
 
         typing_key = since_token.typing_key if since_token else "0"
 
-        rooms = yield self.store.get_rooms_for_user(sync_config.user.to_string())
-        room_ids = [room.room_id for room in rooms]
-
         typing_source = self.event_sources.sources["typing"]
         typing, typing_key = yield typing_source.get_new_events(
             user=sync_config.user,
@@ -433,17 +473,46 @@ class SyncHandler(BaseHandler):
         """
         now_token = yield self.event_sources.get_current_token()
 
-        rooms = yield self.store.get_rooms_for_user(sync_config.user.to_string())
+        if sync_config.is_guest:
+            room_ids = sync_config.filter.list_rooms()
+
+            tags_by_room = {}
+            account_data = {}
+            account_data_by_room = {}
+
+        else:
+            rooms = yield self.store.get_rooms_for_user(
+                sync_config.user.to_string()
+            )
             room_ids = [room.room_id for room in rooms]
 
+            now_token, ephemeral_by_room = yield self.ephemeral_by_room(
+                sync_config, now_token, since_token
+            )
+
+            tags_by_room = yield self.store.get_updated_tags(
+                sync_config.user.to_string(),
+                since_token.account_data_key,
+            )
+
+            account_data, account_data_by_room = (
+                yield self.store.get_updated_account_data_for_user(
+                    sync_config.user.to_string(),
+                    since_token.account_data_key,
+                )
+            )
+
+        now_token, ephemeral_by_room = yield self.ephemeral_by_room(
+            sync_config, now_token, room_ids, since_token
+        )
+
         presence_source = self.event_sources.sources["presence"]
         presence, presence_key = yield presence_source.get_new_events(
             user=sync_config.user,
             from_key=since_token.presence_key,
             limit=sync_config.filter.presence_limit(),
             room_ids=room_ids,
-            # /sync doesn't support guest access, they can't get to this point in code
-            is_guest=False,
+            is_guest=sync_config.is_guest,
         )
         now_token = now_token.copy_and_replace("presence_key", presence_key)
 
@@ -477,18 +546,8 @@ class SyncHandler(BaseHandler):
             from_key=since_token.room_key,
             to_key=now_token.room_key,
             limit=timeline_limit + 1,
-        )
-
-        tags_by_room = yield self.store.get_updated_tags(
-            sync_config.user.to_string(),
-            since_token.account_data_key,
-        )
-
-        account_data, account_data_by_room = (
-            yield self.store.get_updated_account_data_for_user(
-                sync_config.user.to_string(),
-                since_token.account_data_key,
-            )
+            room_ids=room_ids if sync_config.is_guest else (),
+            is_guest=sync_config.is_guest,
         )
 
         joined = []
@@ -628,7 +687,10 @@ class SyncHandler(BaseHandler):
             end_key = "s" + room_key.split('-')[-1]
             loaded_recents = sync_config.filter.filter_room_timeline(events)
             loaded_recents = yield self._filter_events_for_client(
-                sync_config.user.to_string(), loaded_recents,
+                sync_config.user.to_string(),
+                loaded_recents,
+                is_guest=sync_config.is_guest,
+                require_all_visible_for_guests=False
             )
             loaded_recents.extend(recents)
             recents = loaded_recents

synapse/rest/client/v2_alpha/sync.py

@@ -85,7 +85,9 @@ class SyncRestServlet(RestServlet):
 
     @defer.inlineCallbacks
     def on_GET(self, request):
-        user, token_id, _ = yield self.auth.get_user_by_req(request)
+        user, token_id, is_guest = yield self.auth.get_user_by_req(
+            request, allow_guest=True
+        )
 
         timeout = parse_integer(request, "timeout", default=0)
         since = parse_string(request, "since")
@@ -118,8 +120,14 @@ class SyncRestServlet(RestServlet):
             except:
                 filter = FilterCollection({})
 
+        if is_guest and filter.list_rooms() is None:
+            raise SynapseError(
+                400, "Guest users must provide a list of rooms in the filter"
+            )
+
         sync_config = SyncConfig(
             user=user,
+            is_guest=is_guest,
             filter=filter,
         )