From 069ae2a5d62976eab50c27e6e746c3fb1eeb0761 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 13 Dec 2017 15:11:22 +0000 Subject: [PATCH 1/9] Bump changelog and version --- CHANGES.rst | 37 +++++++++++++++++++++++++++++++++++++ synapse/__init__.py | 2 +- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index aff6688b93..4bbc7a7536 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,40 @@ +Changes in synapse v0.26.0 (2017-12-13) +======================================= + +Features: + +* Add ability for ASes to publicise groups for their users (PR #2686) +* Add all local users to the user_directory and optionally search them (PR + #2723) +* Add support for custom login types for validating users (PR #2729) + + +Changes: + +* Update example Prometheus config to new format (PR #2648) Thanks to + @krombel! +* Rename redact_content option to include_content in Push API (PR #2650) +* Declare support for r0.3.0 (PR #2677) +* Improve upserts (PR #2684, #2688, #2689, #2713) +* Improve documentation of workers (PR #2700) +* Improve tracebacks on exceptions (PR #2705) +* Allow guest access to group APIs for reading (PR #2715) +* Support for posting content in federation_client script (PR #2716) +* Delete devices and pushers on logouts etc (PR #2722) + + +Bug fixes: + +* Fix database port script (PR #2673) +* Fix internal server error on login with ldap_auth_provider (PR #2678) Thanks + to @jkolo! +* Fix error on sqlite 3.7 (PR #2697) +* Fix OPTIONS on preview_url (PR #2707) +* Fix error handling on dns lookup (PR #2711) +* Fix wrong avatars when inviting multiple users when creating room (PR #2717) +* Fix 500 when joining matrix-dev (PR #2719) + + Changes in synapse v0.25.1 (2017-11-17) ======================================= diff --git a/synapse/__init__.py b/synapse/__init__.py index 8c3d7a210a..fb2a88a3f1 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.25.1" +__version__ = "0.26.0-rc1" From 80573e39005fbc9795cadf03b42f6e1a342b6623 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 13 Dec 2017 15:15:33 +0000 Subject: [PATCH 2/9] Fix rc version number --- CHANGES.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 4bbc7a7536..77b65ebe04 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,5 +1,5 @@ -Changes in synapse v0.26.0 (2017-12-13) -======================================= +Changes in synapse v0.26.0-rc1 (2017-12-13) +=========================================== Features: From 9419bb5776d656bbc0a47e8123099794da6da40e Mon Sep 17 00:00:00 2001 From: Vincent Breitmoser Date: Sat, 16 Dec 2017 22:09:53 +0200 Subject: [PATCH 3/9] mention federation tester more prominently in the readme --- README.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.rst b/README.rst index 6f146b63b1..76fe2e4139 100644 --- a/README.rst +++ b/README.rst @@ -632,6 +632,11 @@ largest boxes pause for thought.) Troubleshooting --------------- + +You can use the federation tester to check if your homeserver is all set: +``https://matrix.org/federationtester/api/report?server_name=`` +If any of the attributes under "checks" is false, federation won't work. + The typical failure mode with federation is that when you try to join a room, it is rejected with "401: Unauthorized". Generally this means that other servers in the room couldn't access yours. (Joining a room over federation is a From 864a6d297767dd6b91e66a64f74d1ddefcd10ef1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 5 Jan 2018 10:54:01 +0000 Subject: [PATCH 4/9] Bump version and changelog --- CHANGES.rst | 6 ++++++ synapse/__init__.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 77b65ebe04..dcf9adc95c 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,9 @@ +Changes in synapse v0.26.0 (2018-01-05) +======================================= + +No changes since v0.26.0-rc1 + + Changes in synapse v0.26.0-rc1 (2017-12-13) =========================================== diff --git a/synapse/__init__.py b/synapse/__init__.py index fb2a88a3f1..ef8853bd24 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.26.0-rc1" +__version__ = "0.26.0" From 51c9d9ed65af58d4fb5657bad877dbf1f0dcaf39 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 9 Jan 2018 14:39:12 +0000 Subject: [PATCH 5/9] Add /room/{id}/event/{id} to synapse Turns out that there is a valid usecase for retrieving event by id (notably having received a push), but event ids should be scoped to room, so /event/{id} is wrong. --- synapse/rest/client/v1/room.py | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index 75b735b47d..682a0af9fc 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -487,13 +487,35 @@ class RoomInitialSyncRestServlet(ClientV1RestServlet): defer.returnValue((200, content)) -class RoomEventContext(ClientV1RestServlet): +class RoomEventServlet(ClientV1RestServlet): + PATTERNS = client_path_patterns( + "/rooms/(?P[^/]*)/event/(?P[^/]*)$" + ) + + def __init__(self, hs): + super(RoomEventServlet, self).__init__(hs) + self.clock = hs.get_clock() + self.event_handler = hs.get_event_handler() + + @defer.inlineCallbacks + def on_GET(self, request, room_id, event_id): + requester = yield self.auth.get_user_by_req(request) + event = yield self.event_handler.get_event(requester.user, event_id) + + time_now = self.clock.time_msec() + if event: + defer.returnValue((200, serialize_event(event, time_now))) + else: + defer.returnValue((404, "Event not found.")) + + +class RoomEventContextServlet(ClientV1RestServlet): PATTERNS = client_path_patterns( "/rooms/(?P[^/]*)/context/(?P[^/]*)$" ) def __init__(self, hs): - super(RoomEventContext, self).__init__(hs) + super(RoomEventContextServlet, self).__init__(hs) self.clock = hs.get_clock() self.handlers = hs.get_handlers() @@ -803,4 +825,5 @@ def register_servlets(hs, http_server): RoomTypingRestServlet(hs).register(http_server) SearchRestServlet(hs).register(http_server) JoinedRoomsRestServlet(hs).register(http_server) - RoomEventContext(hs).register(http_server) + RoomEventServlet(hs).register(http_server) + RoomEventContextServlet(hs).register(http_server) From d90e8ea444092f500c721567fe055d48eb110528 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 9 Jan 2018 18:27:35 +0000 Subject: [PATCH 6/9] Update http request metrics before calling servlet Make sure that we set the servlet name in the metrics object *before* calling the servlet, in case the servlet throws an exception. --- synapse/http/server.py | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/synapse/http/server.py b/synapse/http/server.py index 25466cd292..6e8f4c9c5f 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -107,6 +107,10 @@ def wrap_request_handler(request_handler, include_metrics=False): with LoggingContext(request_id) as request_context: with Measure(self.clock, "wrapped_request_handler"): request_metrics = RequestMetrics() + # we start the request metrics timer here with an initial stab + # at the servlet name. For most requests that name will be + # JsonResource (or a subclass), and JsonResource._async_render + # will update it once it picks a servlet. request_metrics.start(self.clock, name=self.__class__.__name__) request_context.request = request_id @@ -249,12 +253,23 @@ class JsonResource(HttpServer, resource.Resource): if not m: continue - # We found a match! Trigger callback and then return the - # returned response. We pass both the request and any - # matched groups from the regex to the callback. + # We found a match! First update the metrics object to indicate + # which servlet is handling the request. callback = path_entry.callback + servlet_instance = getattr(callback, "__self__", None) + if servlet_instance is not None: + servlet_classname = servlet_instance.__class__.__name__ + else: + servlet_classname = "%r" % callback + + request_metrics.name = servlet_classname + + # Now trigger the callback. If it returns a response, we send it + # here. If it throws an exception, that is handled by the wrapper + # installed by @request_handler. + kwargs = intern_dict({ name: urllib.unquote(value).decode("UTF-8") if value else value for name, value in m.groupdict().items() @@ -265,17 +280,10 @@ class JsonResource(HttpServer, resource.Resource): code, response = callback_return self._send_response(request, code, response) - servlet_instance = getattr(callback, "__self__", None) - if servlet_instance is not None: - servlet_classname = servlet_instance.__class__.__name__ - else: - servlet_classname = "%r" % callback - - request_metrics.name = servlet_classname - return # Huh. No one wanted to handle that? Fiiiiiine. Send 400. + request_metrics.name = self.__class__.__name__ + ".UnrecognizedRequest" raise UnrecognizedRequestError() def _send_response(self, request, code, response_json_object, From 8c9d5b4873dc5e54c1383e3d31060fd32b08fae1 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Wed, 10 Jan 2018 15:04:02 +0000 Subject: [PATCH 7/9] Fix publicised groups API (singular) over federation which was missing its fed client API, since there is no other API it might as well reuse the bulk one and unwrap it Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --- synapse/handlers/groups_local.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/synapse/handlers/groups_local.py b/synapse/handlers/groups_local.py index 7e5d3f148d..0d3feacd47 100644 --- a/synapse/handlers/groups_local.py +++ b/synapse/handlers/groups_local.py @@ -383,11 +383,11 @@ class GroupsLocalHandler(object): defer.returnValue({"groups": result}) else: - result = yield self.transport_client.get_publicised_groups_for_user( - get_domain_from_id(user_id), user_id - ) + result = yield self.transport_client.bulk_get_publicised_groups( + get_domain_from_id(user_id), [user_id], + )["users"][user_id] # TODO: Verify attestations - defer.returnValue(result) + defer.returnValue({"groups": result}) @defer.inlineCallbacks def bulk_get_publicised_groups(self, user_ids, proxy=True): From 5f07f5694c9a0535e64262c3726007e966ba72ea Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Wed, 10 Jan 2018 15:11:35 +0000 Subject: [PATCH 8/9] fix order of operations derp and also use `.get` to default to {} Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --- synapse/handlers/groups_local.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/groups_local.py b/synapse/handlers/groups_local.py index 0d3feacd47..e4d0cc8b02 100644 --- a/synapse/handlers/groups_local.py +++ b/synapse/handlers/groups_local.py @@ -383,9 +383,10 @@ class GroupsLocalHandler(object): defer.returnValue({"groups": result}) else: - result = yield self.transport_client.bulk_get_publicised_groups( + bulk_result = yield self.transport_client.bulk_get_publicised_groups( get_domain_from_id(user_id), [user_id], - )["users"][user_id] + ) + result = bulk_result.get("users", {}).get(user_id) # TODO: Verify attestations defer.returnValue({"groups": result}) From f8e1ab5fee708bea93c54b39dc802a08c7ed7e21 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 10 Jan 2018 18:01:28 +0000 Subject: [PATCH 9/9] Do bcrypt hashing in a background thread --- synapse/handlers/auth.py | 24 ++++++++++++++++-------- synapse/handlers/register.py | 2 +- synapse/handlers/set_password.py | 2 +- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 2f30f183ce..af9060fdbf 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -13,7 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from twisted.internet import defer +from twisted.internet import defer, threads from ._base import BaseHandler from synapse.api.constants import LoginType @@ -22,6 +22,7 @@ from synapse.module_api import ModuleApi from synapse.types import UserID from synapse.util.async import run_on_reactor from synapse.util.caches.expiringcache import ExpiringCache +from synapse.util.logcontext import make_deferred_yieldable from twisted.web.client import PartialDownloadError @@ -626,7 +627,7 @@ class AuthHandler(BaseHandler): if not lookupres: defer.returnValue(None) (user_id, password_hash) = lookupres - result = self.validate_hash(password, password_hash) + result = yield self.validate_hash(password, password_hash) if not result: logger.warn("Failed password login for user %s", user_id) defer.returnValue(None) @@ -754,10 +755,13 @@ class AuthHandler(BaseHandler): password (str): Password to hash. Returns: - Hashed password (str). + Deferred(str): Hashed password. """ - return bcrypt.hashpw(password.encode('utf8') + self.hs.config.password_pepper, - bcrypt.gensalt(self.bcrypt_rounds)) + def _do_hash(): + return bcrypt.hashpw(password.encode('utf8') + self.hs.config.password_pepper, + bcrypt.gensalt(self.bcrypt_rounds)) + + return make_deferred_yieldable(threads.deferToThread(_do_hash)) def validate_hash(self, password, stored_hash): """Validates that self.hash(password) == stored_hash. @@ -767,13 +771,17 @@ class AuthHandler(BaseHandler): stored_hash (str): Expected hash value. Returns: - Whether self.hash(password) == stored_hash (bool). + Deferred(bool): Whether self.hash(password) == stored_hash. """ - if stored_hash: + + def _do_validate_hash(): return bcrypt.hashpw(password.encode('utf8') + self.hs.config.password_pepper, stored_hash.encode('utf8')) == stored_hash + + if stored_hash: + return make_deferred_yieldable(threads.deferToThread(_do_validate_hash)) else: - return False + return defer.succeed(False) class MacaroonGeneartor(object): diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index f6e7e58563..132e2052c0 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -130,7 +130,7 @@ class RegistrationHandler(BaseHandler): yield run_on_reactor() password_hash = None if password: - password_hash = self.auth_handler().hash(password) + password_hash = yield self.auth_handler().hash(password) if localpart: yield self.check_username(localpart, guest_access_token=guest_access_token) diff --git a/synapse/handlers/set_password.py b/synapse/handlers/set_password.py index 44414e1dc1..e057ae54c9 100644 --- a/synapse/handlers/set_password.py +++ b/synapse/handlers/set_password.py @@ -31,7 +31,7 @@ class SetPasswordHandler(BaseHandler): @defer.inlineCallbacks def set_password(self, user_id, newpassword, requester=None): - password_hash = self._auth_handler.hash(newpassword) + password_hash = yield self._auth_handler.hash(newpassword) except_device_id = requester.device_id if requester else None except_access_token_id = requester.access_token_id if requester else None