Mirrors
/
synapse
mirror of https://github.com/element-hq/synapse.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'develop' into matthew/fix_overzealous_ll_state

This commit is contained in:
Matthew Hodgson 2018-09-18 16:50:39 +01:00
parent 095d4f27a1 286d6930b7
commit a078de955f
204 changed files with 2935 additions and 1688 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

81
.circleci/config.yml
View File

@ -9,6 +9,8 @@ jobs:
- store_artifacts: - store_artifacts:
path: ~/project/logs path: ~/project/logs
destination: logs destination: logs
- store_test_results:
path: logs
sytestpy2postgres: sytestpy2postgres:
machine: true machine: true
steps: steps:
@ -18,15 +20,45 @@ jobs:
- store_artifacts: - store_artifacts:
path: ~/project/logs path: ~/project/logs
destination: logs destination: logs
- store_test_results:
path: logs
sytestpy2merged:
machine: true
steps:
- checkout
- run: bash .circleci/merge_base_branch.sh
- run: docker pull matrixdotorg/sytest-synapsepy2
- run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs matrixdotorg/sytest-synapsepy2
- store_artifacts:
path: ~/project/logs
destination: logs
- store_test_results:
path: logs
sytestpy2postgresmerged:
machine: true
steps:
- checkout
- run: bash .circleci/merge_base_branch.sh
- run: docker pull matrixdotorg/sytest-synapsepy2
- run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs -e POSTGRES=1 matrixdotorg/sytest-synapsepy2
- store_artifacts:
path: ~/project/logs
destination: logs
- store_test_results:
path: logs
sytestpy3: sytestpy3:
machine: true machine: true
steps: steps:
- checkout - checkout
- run: docker pull matrixdotorg/sytest-synapsepy3 - run: docker pull matrixdotorg/sytest-synapsepy3
- run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs hawkowl/sytestpy3 - run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs matrixdotorg/sytest-synapsepy3
- store_artifacts: - store_artifacts:
path: ~/project/logs path: ~/project/logs
destination: logs destination: logs
- store_test_results:
path: logs
sytestpy3postgres: sytestpy3postgres:
machine: true machine: true
steps: steps:
@ -36,6 +68,32 @@ jobs:
- store_artifacts: - store_artifacts:
path: ~/project/logs path: ~/project/logs
destination: logs destination: logs
- store_test_results:
path: logs
sytestpy3merged:
machine: true
steps:
- checkout
- run: bash .circleci/merge_base_branch.sh
- run: docker pull matrixdotorg/sytest-synapsepy3
- run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs matrixdotorg/sytest-synapsepy3
- store_artifacts:
path: ~/project/logs
destination: logs
- store_test_results:
path: logs
sytestpy3postgresmerged:
machine: true
steps:
- checkout
- run: bash .circleci/merge_base_branch.sh
- run: docker pull matrixdotorg/sytest-synapsepy3
- run: docker run --rm -it -v $(pwd)\:/src -v $(pwd)/logs\:/logs -e POSTGRES=1 matrixdotorg/sytest-synapsepy3
- store_artifacts:
path: ~/project/logs
destination: logs
- store_test_results:
path: logs
workflows: workflows:
version: 2 version: 2
@ -43,6 +101,21 @@ workflows:
jobs: jobs:
- sytestpy2 - sytestpy2
- sytestpy2postgres - sytestpy2postgres
# Currently broken while the Python 3 port is incomplete - sytestpy3
# - sytestpy3 - sytestpy3postgres
# - sytestpy3postgres - sytestpy2merged:
filters:
branches:
ignore: /develop|master/
- sytestpy2postgresmerged:
filters:
branches:
ignore: /develop|master/
- sytestpy3merged:
filters:
branches:
ignore: /develop|master/
- sytestpy3postgresmerged:
filters:
branches:
ignore: /develop|master/

31
.circleci/merge_base_branch.sh Executable file
View File

@ -0,0 +1,31 @@
#!/usr/bin/env bash
set -e
# CircleCI doesn't give CIRCLE_PR_NUMBER in the environment for non-forked PRs. Wonderful.
# In this case, we just need to do some ~shell magic~ to strip it out of the PULL_REQUEST URL.
echo 'export CIRCLE_PR_NUMBER="${CIRCLE_PR_NUMBER:-${CIRCLE_PULL_REQUEST##*/}}"' >> $BASH_ENV
source $BASH_ENV
if [[ -z "${CIRCLE_PR_NUMBER}" ]]
then
echo "Can't figure out what the PR number is!"
exit 1
fi
# Get the reference, using the GitHub API
GITBASE=`curl -q https://api.github.com/repos/matrix-org/synapse/pulls/${CIRCLE_PR_NUMBER} | jq -r '.base.ref'`
# Show what we are before
git show -s
# Set up username so it can do a merge
git config --global user.email bot@matrix.org
git config --global user.name "A robot"
# Fetch and merge. If it doesn't work, it will raise due to set -e.
git fetch -u origin $GITBASE
git merge --no-edit origin/$GITBASE
# Show what we are after.
git show -s

1
.dockerignore
View File

@ -3,6 +3,5 @@ Dockerfile
.gitignore .gitignore
demo/etc demo/etc
tox.ini tox.ini
synctl
.git/* .git/*
.tox/* .tox/*

1
.gitignore vendored
View File

@ -44,6 +44,7 @@ media_store/
build/ build/
venv/ venv/
venv*/ venv*/
*venv/
localhost-800*/ localhost-800*/
static/client/register/register_config.js static/client/register/register_config.js

12
.travis.yml
View File

@ -8,9 +8,6 @@ before_script:
- git remote set-branches --add origin develop - git remote set-branches --add origin develop
- git fetch origin develop - git fetch origin develop
services:
- postgresql
matrix: matrix:
fast_finish: true fast_finish: true
include: include:
@ -25,6 +22,11 @@ matrix:
- python: 2.7 - python: 2.7
env: TOX_ENV=py27-postgres TRIAL_FLAGS="-j 4" env: TOX_ENV=py27-postgres TRIAL_FLAGS="-j 4"
services:
- postgresql
- python: 3.5
env: TOX_ENV=py35
- python: 3.6 - python: 3.6
env: TOX_ENV=py36 env: TOX_ENV=py36
@ -35,10 +37,6 @@ matrix:
- python: 3.6 - python: 3.6
env: TOX_ENV=check-newsfragment env: TOX_ENV=check-newsfragment
allow_failures:
- python: 2.7
env: TOX_ENV=py27-postgres TRIAL_FLAGS="-j 4"
install: install:
- pip install tox - pip install tox

74
CHANGES.md
View File

@ -1,3 +1,77 @@
Synapse 0.33.4 (2018-09-07)
===========================
Internal Changes
----------------
- Unignore synctl in .dockerignore to fix docker builds ([\#3802](https://github.com/matrix-org/synapse/issues/3802))
Synapse 0.33.4rc2 (2018-09-06)
==============================
Pull in security fixes from v0.33.3.1
Synapse 0.33.3.1 (2018-09-06)
=============================
SECURITY FIXES
--------------
- Fix an issue where event signatures were not always correctly validated ([\#3796](https://github.com/matrix-org/synapse/issues/3796))
- Fix an issue where server_acls could be circumvented for incoming events ([\#3796](https://github.com/matrix-org/synapse/issues/3796))
Internal Changes
----------------
- Unignore synctl in .dockerignore to fix docker builds ([\#3802](https://github.com/matrix-org/synapse/issues/3802))
Synapse 0.33.4rc1 (2018-09-04)
==============================
Features
--------
- Support profile API endpoints on workers ([\#3659](https://github.com/matrix-org/synapse/issues/3659))
- Server notices for resource limit blocking ([\#3680](https://github.com/matrix-org/synapse/issues/3680))
- Allow guests to use /rooms/:roomId/event/:eventId ([\#3724](https://github.com/matrix-org/synapse/issues/3724))
- Add mau_trial_days config param, so that users only get counted as MAU after N days. ([\#3749](https://github.com/matrix-org/synapse/issues/3749))
- Require twisted 17.1 or later (fixes [#3741](https://github.com/matrix-org/synapse/issues/3741)). ([\#3751](https://github.com/matrix-org/synapse/issues/3751))
Bugfixes
--------
- Fix error collecting prometheus metrics when run on dedicated thread due to threading concurrency issues ([\#3722](https://github.com/matrix-org/synapse/issues/3722))
- Fix bug where we resent "limit exceeded" server notices repeatedly ([\#3747](https://github.com/matrix-org/synapse/issues/3747))
- Fix bug where we broke sync when using limit_usage_by_mau but hadn't configured server notices ([\#3753](https://github.com/matrix-org/synapse/issues/3753))
- Fix 'federation_domain_whitelist' such that an empty list correctly blocks all outbound federation traffic ([\#3754](https://github.com/matrix-org/synapse/issues/3754))
- Fix tagging of server notice rooms ([\#3755](https://github.com/matrix-org/synapse/issues/3755), [\#3756](https://github.com/matrix-org/synapse/issues/3756))
- Fix 'admin_uri' config variable and error parameter to be 'admin_contact' to match the spec. ([\#3758](https://github.com/matrix-org/synapse/issues/3758))
- Don't return non-LL-member state in incremental sync state blocks ([\#3760](https://github.com/matrix-org/synapse/issues/3760))
- Fix bug in sending presence over federation ([\#3768](https://github.com/matrix-org/synapse/issues/3768))
- Fix bug where preserved threepid user comes to sign up and server is mau blocked ([\#3777](https://github.com/matrix-org/synapse/issues/3777))
Internal Changes
----------------
- Removed the link to the unmaintained matrix-synapse-auto-deploy project from the readme. ([\#3378](https://github.com/matrix-org/synapse/issues/3378))
- Refactor state module to support multiple room versions ([\#3673](https://github.com/matrix-org/synapse/issues/3673))
- The synapse.storage module has been ported to Python 3. ([\#3725](https://github.com/matrix-org/synapse/issues/3725))
- Split the state_group_cache into member and non-member state events (and so speed up LL /sync) ([\#3726](https://github.com/matrix-org/synapse/issues/3726))
- Log failure to authenticate remote servers as warnings (without stack traces) ([\#3727](https://github.com/matrix-org/synapse/issues/3727))
- The CONTRIBUTING guidelines have been updated to mention our use of Markdown and that .misc files have content. ([\#3730](https://github.com/matrix-org/synapse/issues/3730))
- Reference the need for an HTTP replication port when using the federation_reader worker ([\#3734](https://github.com/matrix-org/synapse/issues/3734))
- Fix minor spelling error in federation client documentation. ([\#3735](https://github.com/matrix-org/synapse/issues/3735))
- Remove redundant state resolution function ([\#3737](https://github.com/matrix-org/synapse/issues/3737))
- The test suite now passes on PostgreSQL. ([\#3740](https://github.com/matrix-org/synapse/issues/3740))
- Fix MAU cache invalidation due to missing yield ([\#3746](https://github.com/matrix-org/synapse/issues/3746))
- Make sure that we close db connections opened during init ([\#3764](https://github.com/matrix-org/synapse/issues/3764))
Synapse 0.33.3 (2018-08-22) Synapse 0.33.3 (2018-08-22)
=========================== ===========================

7
CONTRIBUTING.rst
View File

@ -59,9 +59,10 @@ To create a changelog entry, make a new file in the ``changelog.d``
file named in the format of ``PRnumber.type``. The type can be file named in the format of ``PRnumber.type``. The type can be
one of ``feature``, ``bugfix``, ``removal`` (also used for one of ``feature``, ``bugfix``, ``removal`` (also used for
deprecations), or ``misc`` (for internal-only changes). The content of deprecations), or ``misc`` (for internal-only changes). The content of
the file is your changelog entry, which can contain RestructuredText the file is your changelog entry, which can contain Markdown
formatting. A note of contributors is welcomed in changelogs for formatting. Adding credits to the changelog is encouraged, we value
non-misc changes (the content of misc changes is not displayed). your contributions and would like to have you shouted out in the
release notes!
For example, a fix in PR #1234 would have its changelog entry in For example, a fix in PR #1234 would have its changelog entry in
``changelog.d/1234.bugfix``, and contain content like "The security levels of ``changelog.d/1234.bugfix``, and contain content like "The security levels of

27
README.rst
View File

@ -167,11 +167,6 @@ Alternatively, Andreas Peters (previously Silvio Fricke) has contributed a
Dockerfile to automate a synapse server in a single Docker image, at Dockerfile to automate a synapse server in a single Docker image, at
https://hub.docker.com/r/avhost/docker-matrix/tags/ https://hub.docker.com/r/avhost/docker-matrix/tags/
Also, Martin Giess has created an auto-deployment process with vagrant/ansible,
tested with VirtualBox/AWS/DigitalOcean - see
https://github.com/EMnify/matrix-synapse-auto-deploy
for details.
Configuring synapse Configuring synapse
------------------- -------------------
@ -747,6 +742,18 @@ so an example nginx configuration might look like::
} }
} }
and an example apache configuration may look like::
<VirtualHost *:443>
SSLEngine on
ServerName matrix.example.com;
<Location /_matrix>
ProxyPass http://127.0.0.1:8008/_matrix nocanon
ProxyPassReverse http://127.0.0.1:8008/_matrix
</Location>
</VirtualHost>
You will also want to set ``bind_addresses: ['127.0.0.1']`` and ``x_forwarded: true`` You will also want to set ``bind_addresses: ['127.0.0.1']`` and ``x_forwarded: true``
for port 8008 in ``homeserver.yaml`` to ensure that client IP addresses are for port 8008 in ``homeserver.yaml`` to ensure that client IP addresses are
recorded correctly. recorded correctly.
@ -901,7 +908,7 @@ to install using pip and a virtualenv::
virtualenv -p python2.7 env virtualenv -p python2.7 env
source env/bin/activate source env/bin/activate
python synapse/python_dependencies.py | xargs pip install python -m synapse.python_dependencies | xargs pip install
pip install lxml mock pip install lxml mock
This will run a process of downloading and installing all the needed This will run a process of downloading and installing all the needed
@ -956,5 +963,13 @@ variable. The default is 0.5, which can be decreased to reduce RAM usage
in memory constrained enviroments, or increased if performance starts to in memory constrained enviroments, or increased if performance starts to
degrade. degrade.
Using `libjemalloc <http://jemalloc.net/>`_ can also yield a significant
improvement in overall amount, and especially in terms of giving back RAM
to the OS. To use it, the library must simply be put in the LD_PRELOAD
environment variable when launching Synapse. On Debian, this can be done
by installing the ``libjemalloc1`` package and adding this line to
``/etc/default/matrix-synapse``::
LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1
.. _`key_management`: https://matrix.org/docs/spec/server_server/unstable.html#retrieving-server-keys .. _`key_management`: https://matrix.org/docs/spec/server_server/unstable.html#retrieving-server-keys

1
changelog.d/3576.feature Normal file
View File

@ -0,0 +1 @@
Python 3.5+ is now supported.

1
changelog.d/3659.feature
View File

@ -1 +0,0 @@
Support profile API endpoints on workers

1
changelog.d/3673.misc
View File

@ -1 +0,0 @@
Refactor state module to support multiple room versions

1
changelog.d/3680.feature
View File

@ -1 +0,0 @@
Server notices for resource limit blocking

1
changelog.d/3704.misc Normal file
View File

@ -0,0 +1 @@
CircleCI tests now run on the potential merge of a PR.

1
changelog.d/3722.bugfix
View File

@ -1 +0,0 @@
Fix error collecting prometheus metrics when run on dedicated thread due to threading concurrency issues

1
changelog.d/3724.feature
View File

@ -1 +0,0 @@
Allow guests to use /rooms/:roomId/event/:eventId

1
changelog.d/3726.misc
View File

@ -1 +0,0 @@
Split the state_group_cache into member and non-member state events (and so speed up LL /sync)

1
changelog.d/3727.misc
View File

@ -1 +0,0 @@
Log failure to authenticate remote servers as warnings (without stack traces)

1
changelog.d/3734.misc
View File

@ -1 +0,0 @@
Reference the need for an HTTP replication port when using the federation_reader worker

1
changelog.d/3735.misc
View File

@ -1 +0,0 @@
Fix minor spelling error in federation client documentation.

1
changelog.d/3746.misc
View File

@ -1 +0,0 @@
Fix MAU cache invalidation due to missing yield

1
changelog.d/3747.bugfix
View File

@ -1 +0,0 @@
Fix bug where we resent "limit exceeded" server notices repeatedly

1
changelog.d/3749.feature
View File

@ -1 +0,0 @@
Add mau_trial_days config param, so that users only get counted as MAU after N days.

1
changelog.d/3751.feature
View File

@ -1 +0,0 @@
Require twisted 17.1 or later (fixes [#3741](https://github.com/matrix-org/synapse/issues/3741)).

1
changelog.d/3753.bugfix
View File

@ -1 +0,0 @@
Fix bug where we broke sync when using limit_usage_by_mau but hadn't configured server notices

1
changelog.d/3754.bugfix
View File

@ -1 +0,0 @@
Fix 'federation_domain_whitelist' such that an empty list correctly blocks all outbound federation traffic

1
changelog.d/3755.bugfix
View File

@ -1 +0,0 @@
Fix tagging of server notice rooms

1
changelog.d/3756.bugfix
View File

@ -1 +0,0 @@
Fix tagging of server notice rooms

1
changelog.d/3758.bugfix
View File

@ -1 +0,0 @@
Fix 'admin_uri' config variable and error parameter to be 'admin_contact' to match the spec.

1
changelog.d/3771.misc Normal file
View File

@ -0,0 +1 @@
http/ is now ported to Python 3.

1
changelog.d/3788.bugfix Normal file
View File

@ -0,0 +1 @@
Remove connection ID for replication prometheus metrics, as it creates a large number of new series.

1
changelog.d/3789.misc Normal file
View File

@ -0,0 +1 @@
Improve human readable error messages for threepid registration/account update

1
changelog.d/3790.feature Normal file
View File

@ -0,0 +1 @@
Implement `event_format` filter param in `/sync`

1
changelog.d/3795.misc Normal file
View File

@ -0,0 +1 @@
Make /sync slightly faster by avoiding needless copies

1
changelog.d/3800.bugfix Normal file
View File

@ -0,0 +1 @@
guest users should not be part of mau total

1
changelog.d/3803.misc Normal file
View File

@ -0,0 +1 @@
handlers/ is now ported to Python 3.

1
changelog.d/3804.bugfix Normal file
View File

@ -0,0 +1 @@
Bump dependency on pyopenssl 16.x, to avoid incompatibility with recent Twisted.

1
changelog.d/3805.misc Normal file
View File

@ -0,0 +1 @@
Limit the number of PDUs/EDUs per federation transaction

1
changelog.d/3806.misc Normal file
View File

@ -0,0 +1 @@
Only start postgres instance for postgres tests on Travis CI

1
changelog.d/3808.misc Normal file
View File

@ -0,0 +1 @@
tests/ is now ported to Python 3.

1
changelog.d/3810.bugfix Normal file
View File

@ -0,0 +1 @@
Fix existing room tags not coming down sync when joining a room

1
changelog.d/3822.misc Normal file
View File

@ -0,0 +1 @@
crypto/ is now ported to Python 3.

1
changelog.d/3823.misc Normal file
View File

@ -0,0 +1 @@
rest/ is now ported to Python 3.

1
changelog.d/3824.bugfix Normal file
View File

@ -0,0 +1 @@
Fix jwt import check

1
changelog.d/3826.misc Normal file
View File

@ -0,0 +1 @@
add some logging for the keyring queue

1
changelog.d/3827.misc Normal file
View File

@ -0,0 +1 @@
speed up lazy loading by 2-3x

1
changelog.d/3834.misc Normal file
View File

@ -0,0 +1 @@
Improved Dockerfile to remove build requirements after building reducing the image size.

1
changelog.d/3835.bugfix Normal file
View File

@ -0,0 +1 @@
fix VOIP crashes under Python 3 (#3821)

1
changelog.d/3840.misc Normal file
View File

@ -0,0 +1 @@
Disable lazy loading for incremental syncs for now

1
changelog.d/3841.bugfix Normal file
View File

@ -0,0 +1 @@
Fix manhole so that it works with latest openssh clients

1
changelog.d/3845.bugfix Normal file
View File

@ -0,0 +1 @@
Fix outbound requests occasionally wedging, which can result in federation breaking between servers.

1
changelog.d/3846.feature Normal file
View File

@ -0,0 +1 @@
Add synapse_admin_mau:registered_reserved_users metric to expose number of real reaserved users

1
changelog.d/3847.misc Normal file
View File

@ -0,0 +1 @@
federation/ is now ported to Python 3.

1
changelog.d/3851.bugfix Normal file
View File

@ -0,0 +1 @@
Show heroes if room name/canonical alias has been deleted

1
changelog.d/3853.misc Normal file
View File

@ -0,0 +1 @@
Log when we retry outbound requests

1
changelog.d/3855.misc Normal file
View File

@ -0,0 +1 @@
Removed some excess logging messages.

1
changelog.d/3856.misc Normal file
View File

@ -0,0 +1 @@
Speed up purge history for rooms that have been previously purged

1
changelog.d/3857.misc Normal file
View File

@ -0,0 +1 @@
Refactor some HTTP timeout code.

1
changelog.d/3858.misc Normal file
View File

@ -0,0 +1 @@
Fix running merged builds on CircleCI

1
changelog.d/3859.bugfix Normal file
View File

@ -0,0 +1 @@
Fix handling of redacted events from federation

1
changelog.d/3860.misc Normal file
View File

@ -0,0 +1 @@
Fix typo in replication stream exception.

1
changelog.d/3871.misc Normal file
View File

@ -0,0 +1 @@
Add in flight real time metrics for Measure blocks

1
changelog.d/3872.misc Normal file
View File

@ -0,0 +1 @@
Disable buffering and automatic retrying in treq requests to prevent timeouts.

0
changelog.d/3874.bugfix Normal file
View File

1
changelog.d/3875.bugfix Normal file
View File

@ -0,0 +1 @@
Mitigate outbound federation randomly becoming wedged

1
changelog.d/3877.misc Normal file
View File

@ -0,0 +1 @@
mention jemalloc in the README

1
changelog.d/3879.bugfix Normal file
View File

@ -0,0 +1 @@
Don't ratelimit autojoins

1
changelog.d/3883.feature Normal file
View File

@ -0,0 +1 @@
Adding the ability to change MAX_UPLOAD_SIZE for the docker container variables.

1
changelog.d/3888.misc Normal file
View File

@ -0,0 +1 @@
Remove unmaintained "nuke-room-from-db.sh" script

1
changelog.d/3889.bugfix Normal file
View File

@ -0,0 +1 @@
Fix 500 error when deleting unknown room alias

1
changelog.d/3892.bugfix Normal file
View File

@ -0,0 +1 @@
Fix some b'abcd' noise in logs and metrics

1
changelog.d/3895.bugfix Normal file
View File

@ -0,0 +1 @@
Fix some b'abcd' noise in logs and metrics

1
changelog.d/3897.misc Normal file
View File

@ -0,0 +1 @@
Fix typo in README, synaspse -> synapse

22
docker/Dockerfile
View File

@ -1,6 +1,8 @@
FROM docker.io/python:2-alpine3.8 FROM docker.io/python:2-alpine3.8
RUN apk add --no-cache --virtual .nacl_deps \ COPY . /synapse
RUN apk add --no-cache --virtual .build_deps \
build-base \ build-base \
libffi-dev \ libffi-dev \
libjpeg-turbo-dev \ libjpeg-turbo-dev \
@ -8,13 +10,16 @@ RUN apk add --no-cache --virtual .nacl_deps \
libxslt-dev \ libxslt-dev \
linux-headers \ linux-headers \
postgresql-dev \ postgresql-dev \
zlib-dev \
&& cd /synapse \
&& apk add --no-cache --virtual .runtime_deps \
libffi \
libjpeg-turbo \
libressl \
libxslt \
libpq \
zlib \
su-exec \ su-exec \
zlib-dev
COPY . /synapse
# A wheel cache may be provided in ./cache for faster build
RUN cd /synapse \
&& pip install --upgrade \ && pip install --upgrade \
lxml \ lxml \
pip \ pip \
@ -26,7 +31,8 @@ RUN cd /synapse \
&& rm -rf \ && rm -rf \
setup.cfg \ setup.cfg \
setup.py \ setup.py \
synapse synapse \
&& apk del .build_deps
VOLUME ["/data"] VOLUME ["/data"]

1
docker/README.md
View File

@ -88,6 +88,7 @@ variables are available for configuration:
* ``SYNAPSE_TURN_URIS``, set this variable to the coma-separated list of TURN * ``SYNAPSE_TURN_URIS``, set this variable to the coma-separated list of TURN
uris to enable TURN for this homeserver. uris to enable TURN for this homeserver.
* ``SYNAPSE_TURN_SECRET``, set this to the TURN shared secret if required. * ``SYNAPSE_TURN_SECRET``, set this to the TURN shared secret if required.
* ``SYNAPSE_MAX_UPLOAD_SIZE``, set this variable to change the max upload size [default `10M`].
Shared secrets, that will be initialized to random values if not set: Shared secrets, that will be initialized to random values if not set:

2
docker/conf/homeserver.yaml
View File

@ -85,7 +85,7 @@ federation_rc_concurrent: 3
media_store_path: "/data/media" media_store_path: "/data/media"
uploads_path: "/data/uploads" uploads_path: "/data/uploads"
max_upload_size: "10M" max_upload_size: "{{ SYNAPSE_MAX_UPLOAD_SIZE or "10M" }}"
max_image_pixels: "32M" max_image_pixels: "32M"
dynamic_thumbnails: false dynamic_thumbnails: false

2
jenkins/prepare_synapse.sh
View File

@ -31,5 +31,5 @@ $TOX_BIN/pip install 'setuptools>=18.5'
$TOX_BIN/pip install 'pip>=10' $TOX_BIN/pip install 'pip>=10'
{ python synapse/python_dependencies.py { python synapse/python_dependencies.py
echo lxml psycopg2 echo lxml
} | xargs $TOX_BIN/pip install } | xargs $TOX_BIN/pip install

57
scripts-dev/nuke-room-from-db.sh
View File

@ -1,57 +0,0 @@
#!/bin/bash
## CAUTION:
## This script will remove (hopefully) all trace of the given room ID from
## your homeserver.db
## Do not run it lightly.
set -e
if [ "$1" == "-h" ] || [ "$1" == "" ]; then
echo "Call with ROOM_ID as first option and then pipe it into the database. So for instance you might run"
echo " nuke-room-from-db.sh <room_id> | sqlite3 homeserver.db"
echo "or"
echo " nuke-room-from-db.sh <room_id> | psql --dbname=synapse"
exit
fi
ROOMID="$1"
cat <<EOF
DELETE FROM event_forward_extremities WHERE room_id = '$ROOMID';
DELETE FROM event_backward_extremities WHERE room_id = '$ROOMID';
DELETE FROM event_edges WHERE room_id = '$ROOMID';
DELETE FROM room_depth WHERE room_id = '$ROOMID';
DELETE FROM state_forward_extremities WHERE room_id = '$ROOMID';
DELETE FROM events WHERE room_id = '$ROOMID';
DELETE FROM event_json WHERE room_id = '$ROOMID';
DELETE FROM state_events WHERE room_id = '$ROOMID';
DELETE FROM current_state_events WHERE room_id = '$ROOMID';
DELETE FROM room_memberships WHERE room_id = '$ROOMID';
DELETE FROM feedback WHERE room_id = '$ROOMID';
DELETE FROM topics WHERE room_id = '$ROOMID';
DELETE FROM room_names WHERE room_id = '$ROOMID';
DELETE FROM rooms WHERE room_id = '$ROOMID';
DELETE FROM room_hosts WHERE room_id = '$ROOMID';
DELETE FROM room_aliases WHERE room_id = '$ROOMID';
DELETE FROM state_groups WHERE room_id = '$ROOMID';
DELETE FROM state_groups_state WHERE room_id = '$ROOMID';
DELETE FROM receipts_graph WHERE room_id = '$ROOMID';
DELETE FROM receipts_linearized WHERE room_id = '$ROOMID';
DELETE FROM event_search WHERE room_id = '$ROOMID';
DELETE FROM guest_access WHERE room_id = '$ROOMID';
DELETE FROM history_visibility WHERE room_id = '$ROOMID';
DELETE FROM room_tags WHERE room_id = '$ROOMID';
DELETE FROM room_tags_revisions WHERE room_id = '$ROOMID';
DELETE FROM room_account_data WHERE room_id = '$ROOMID';
DELETE FROM event_push_actions WHERE room_id = '$ROOMID';
DELETE FROM local_invites WHERE room_id = '$ROOMID';
DELETE FROM pusher_throttle WHERE room_id = '$ROOMID';
DELETE FROM event_reports WHERE room_id = '$ROOMID';
DELETE FROM public_room_list_stream WHERE room_id = '$ROOMID';
DELETE FROM stream_ordering_to_exterm WHERE room_id = '$ROOMID';
DELETE FROM event_auth WHERE room_id = '$ROOMID';
DELETE FROM appservice_room_list WHERE room_id = '$ROOMID';
VACUUM;
EOF

5
setup.cfg
View File

@ -17,13 +17,14 @@ ignore =
[pep8] [pep8]
max-line-length = 90 max-line-length = 90
# W503 requires that binary operators be at the end, not start, of lines. Erik # W503 requires that binary operators be at the end, not start, of lines. Erik
# doesn't like it. E203 is contrary to PEP8. # doesn't like it. E203 is contrary to PEP8. E731 is silly.
ignore = W503,E203 ignore = W503,E203,E731
[flake8] [flake8]
# note that flake8 inherits the "ignore" settings from "pep8" (because it uses # note that flake8 inherits the "ignore" settings from "pep8" (because it uses
# pep8 to do those checks), but not the "max-line-length" setting # pep8 to do those checks), but not the "max-line-length" setting
max-line-length = 90 max-line-length = 90
ignore=W503,E203,E731
[isort] [isort]
line_length = 89 line_length = 89

12
synapse/__init__.py
View File

@ -17,4 +17,14 @@
""" This is a reference implementation of a Matrix home server. """ This is a reference implementation of a Matrix home server.
""" """
__version__ = "0.33.3" try:
from twisted.internet import protocol
from twisted.internet.protocol import Factory
from twisted.names.dns import DNSDatagramProtocol
protocol.Factory.noisy = False
Factory.noisy = False
DNSDatagramProtocol.noisy = False
except ImportError:
pass
__version__ = "0.33.4"

17
synapse/api/auth.py
View File

@ -26,6 +26,7 @@ import synapse.types
from synapse import event_auth from synapse import event_auth
from synapse.api.constants import EventTypes, JoinRules, Membership from synapse.api.constants import EventTypes, JoinRules, Membership
from synapse.api.errors import AuthError, Codes, ResourceLimitError from synapse.api.errors import AuthError, Codes, ResourceLimitError
from synapse.config.server import is_threepid_reserved
from synapse.types import UserID from synapse.types import UserID
from synapse.util.caches import CACHE_SIZE_FACTOR, register_cache from synapse.util.caches import CACHE_SIZE_FACTOR, register_cache
from synapse.util.caches.lrucache import LruCache from synapse.util.caches.lrucache import LruCache
@ -775,13 +776,19 @@ class Auth(object):
) )
@defer.inlineCallbacks @defer.inlineCallbacks
def check_auth_blocking(self, user_id=None): def check_auth_blocking(self, user_id=None, threepid=None):
"""Checks if the user should be rejected for some external reason, """Checks if the user should be rejected for some external reason,
such as monthly active user limiting or global disable flag such as monthly active user limiting or global disable flag
Args: Args:
user_id(str|None): If present, checks for presence against existing user_id(str|None): If present, checks for presence against existing
MAU cohort MAU cohort
threepid(dict|None): If present, checks for presence against configured
reserved threepid. Used in cases where the user is trying register
with a MAU blocked server, normally they would be rejected but their
threepid is on the reserved list. user_id and
threepid should never be set at the same time.
""" """
# Never fail an auth check for the server notices users # Never fail an auth check for the server notices users
@ -797,6 +804,8 @@ class Auth(object):
limit_type=self.hs.config.hs_disabled_limit_type limit_type=self.hs.config.hs_disabled_limit_type
) )
if self.hs.config.limit_usage_by_mau is True: if self.hs.config.limit_usage_by_mau is True:
assert not (user_id and threepid)
# If the user is already part of the MAU cohort or a trial user # If the user is already part of the MAU cohort or a trial user
if user_id: if user_id:
timestamp = yield self.store.user_last_seen_monthly_active(user_id) timestamp = yield self.store.user_last_seen_monthly_active(user_id)
@ -806,12 +815,16 @@ class Auth(object):
is_trial = yield self.store.is_trial_user(user_id) is_trial = yield self.store.is_trial_user(user_id)
if is_trial: if is_trial:
return return
elif threepid:
# If the user does not exist yet, but is signing up with a
# reserved threepid then pass auth check
if is_threepid_reserved(self.hs.config, threepid):
return
# Else if there is no room in the MAU bucket, bail # Else if there is no room in the MAU bucket, bail
current_mau = yield self.store.get_monthly_active_count() current_mau = yield self.store.get_monthly_active_count()
if current_mau >= self.hs.config.max_mau_value: if current_mau >= self.hs.config.max_mau_value:
raise ResourceLimitError( raise ResourceLimitError(
403, "Monthly Active User Limit Exceeded", 403, "Monthly Active User Limit Exceeded",
admin_contact=self.hs.config.admin_contact, admin_contact=self.hs.config.admin_contact,
errcode=Codes.RESOURCE_LIMIT_EXCEEDED, errcode=Codes.RESOURCE_LIMIT_EXCEEDED,
limit_type="monthly_active_user" limit_type="monthly_active_user"

1
synapse/api/filtering.py
View File

@ -251,6 +251,7 @@ class FilterCollection(object):
"include_leave", False "include_leave", False
) )
self.event_fields = filter_json.get("event_fields", []) self.event_fields = filter_json.get("event_fields", [])
self.event_format = filter_json.get("event_format", "client")
def __repr__(self): def __repr__(self):
return "<FilterCollection %s>" % (json.dumps(self._filter_json),) return "<FilterCollection %s>" % (json.dumps(self._filter_json),)

5
synapse/app/appservice.py
View File

@ -51,10 +51,7 @@ class AppserviceSlaveStore(
class AppserviceServer(HomeServer): class AppserviceServer(HomeServer):
def setup(self): DATASTORE_CLASS = AppserviceSlaveStore
logger.info("Setting up.")
self.datastore = AppserviceSlaveStore(self.get_db_conn(), self)
logger.info("Finished setting up.")
def _listen_http(self, listener_config): def _listen_http(self, listener_config):
port = listener_config["port"] port = listener_config["port"]

5
synapse/app/client_reader.py
View File

@ -74,10 +74,7 @@ class ClientReaderSlavedStore(
class ClientReaderServer(HomeServer): class ClientReaderServer(HomeServer):
def setup(self): DATASTORE_CLASS = ClientReaderSlavedStore
logger.info("Setting up.")
self.datastore = ClientReaderSlavedStore(self.get_db_conn(), self)
logger.info("Finished setting up.")
def _listen_http(self, listener_config): def _listen_http(self, listener_config):
port = listener_config["port"] port = listener_config["port"]

5
synapse/app/event_creator.py
View File

@ -90,10 +90,7 @@ class EventCreatorSlavedStore(
class EventCreatorServer(HomeServer): class EventCreatorServer(HomeServer):
def setup(self): DATASTORE_CLASS = EventCreatorSlavedStore
logger.info("Setting up.")
self.datastore = EventCreatorSlavedStore(self.get_db_conn(), self)
logger.info("Finished setting up.")
def _listen_http(self, listener_config): def _listen_http(self, listener_config):
port = listener_config["port"] port = listener_config["port"]

5
synapse/app/federation_reader.py
View File

@ -72,10 +72,7 @@ class FederationReaderSlavedStore(
class FederationReaderServer(HomeServer): class FederationReaderServer(HomeServer):
def setup(self): DATASTORE_CLASS = FederationReaderSlavedStore
logger.info("Setting up.")
self.datastore = FederationReaderSlavedStore(self.get_db_conn(), self)
logger.info("Finished setting up.")
def _listen_http(self, listener_config): def _listen_http(self, listener_config):
port = listener_config["port"] port = listener_config["port"]

5
synapse/app/federation_sender.py
View File

@ -78,10 +78,7 @@ class FederationSenderSlaveStore(
class FederationSenderServer(HomeServer): class FederationSenderServer(HomeServer):
def setup(self): DATASTORE_CLASS = FederationSenderSlaveStore
logger.info("Setting up.")
self.datastore = FederationSenderSlaveStore(self.get_db_conn(), self)
logger.info("Finished setting up.")
def _listen_http(self, listener_config): def _listen_http(self, listener_config):
port = listener_config["port"] port = listener_config["port"]

5
synapse/app/frontend_proxy.py
View File

@ -148,10 +148,7 @@ class FrontendProxySlavedStore(
class FrontendProxyServer(HomeServer): class FrontendProxyServer(HomeServer):
def setup(self): DATASTORE_CLASS = FrontendProxySlavedStore
logger.info("Setting up.")
self.datastore = FrontendProxySlavedStore(self.get_db_conn(), self)
logger.info("Finished setting up.")
def _listen_http(self, listener_config): def _listen_http(self, listener_config):
port = listener_config["port"] port = listener_config["port"]

20
synapse/app/homeserver.py
View File

@ -62,7 +62,7 @@ from synapse.rest.key.v1.server_key_resource import LocalKey
from synapse.rest.key.v2 import KeyApiV2Resource from synapse.rest.key.v2 import KeyApiV2Resource
from synapse.rest.media.v0.content_repository import ContentRepoResource from synapse.rest.media.v0.content_repository import ContentRepoResource
from synapse.server import HomeServer from synapse.server import HomeServer
from synapse.storage import are_all_users_on_domain from synapse.storage import DataStore, are_all_users_on_domain
from synapse.storage.engines import IncorrectDatabaseSetup, create_engine from synapse.storage.engines import IncorrectDatabaseSetup, create_engine
from synapse.storage.prepare_database import UpgradeDatabaseException, prepare_database from synapse.storage.prepare_database import UpgradeDatabaseException, prepare_database
from synapse.util.caches import CACHE_SIZE_FACTOR from synapse.util.caches import CACHE_SIZE_FACTOR
@ -111,6 +111,8 @@ def build_resource_for_web_client(hs):
class SynapseHomeServer(HomeServer): class SynapseHomeServer(HomeServer):
DATASTORE_CLASS = DataStore
def _listener_http(self, config, listener_config): def _listener_http(self, config, listener_config):
port = listener_config["port"] port = listener_config["port"]
bind_addresses = listener_config["bind_addresses"] bind_addresses = listener_config["bind_addresses"]
@ -305,6 +307,10 @@ class SynapseHomeServer(HomeServer):
# Gauges to expose monthly active user control metrics # Gauges to expose monthly active user control metrics
current_mau_gauge = Gauge("synapse_admin_mau:current", "Current MAU") current_mau_gauge = Gauge("synapse_admin_mau:current", "Current MAU")
max_mau_gauge = Gauge("synapse_admin_mau:max", "MAU Limit") max_mau_gauge = Gauge("synapse_admin_mau:max", "MAU Limit")
registered_reserved_users_mau_gauge = Gauge(
"synapse_admin_mau:registered_reserved_users",
"Registered users with reserved threepids"
)
def setup(config_options): def setup(config_options):
@ -356,7 +362,7 @@ def setup(config_options):
logger.info("Preparing database: %s...", config.database_config['name']) logger.info("Preparing database: %s...", config.database_config['name'])
try: try:
db_conn = hs.get_db_conn(run_new_connection=False) with hs.get_db_conn(run_new_connection=False) as db_conn:
prepare_database(db_conn, database_engine, config=config) prepare_database(db_conn, database_engine, config=config)
database_engine.on_new_connection(db_conn) database_engine.on_new_connection(db_conn)
@ -529,10 +535,14 @@ def run(hs):
@defer.inlineCallbacks @defer.inlineCallbacks
def generate_monthly_active_users(): def generate_monthly_active_users():
count = 0 current_mau_count = 0
reserved_count = 0
store = hs.get_datastore()
if hs.config.limit_usage_by_mau: if hs.config.limit_usage_by_mau:
count = yield hs.get_datastore().get_monthly_active_count() current_mau_count = yield store.get_monthly_active_count()
current_mau_gauge.set(float(count)) reserved_count = yield store.get_registered_reserved_users_count()
current_mau_gauge.set(float(current_mau_count))
registered_reserved_users_mau_gauge.set(float(reserved_count))
max_mau_gauge.set(float(hs.config.max_mau_value)) max_mau_gauge.set(float(hs.config.max_mau_value))
hs.get_datastore().initialise_reserved_users( hs.get_datastore().initialise_reserved_users(

5
synapse/app/media_repository.py
View File

@ -60,10 +60,7 @@ class MediaRepositorySlavedStore(
class MediaRepositoryServer(HomeServer): class MediaRepositoryServer(HomeServer):
def setup(self): DATASTORE_CLASS = MediaRepositorySlavedStore
logger.info("Setting up.")
self.datastore = MediaRepositorySlavedStore(self.get_db_conn(), self)
logger.info("Finished setting up.")
def _listen_http(self, listener_config): def _listen_http(self, listener_config):
port = listener_config["port"] port = listener_config["port"]

5
synapse/app/pusher.py
View File

@ -78,10 +78,7 @@ class PusherSlaveStore(
class PusherServer(HomeServer): class PusherServer(HomeServer):
def setup(self): DATASTORE_CLASS = PusherSlaveStore
logger.info("Setting up.")
self.datastore = PusherSlaveStore(self.get_db_conn(), self)
logger.info("Finished setting up.")
def remove_pusher(self, app_id, push_key, user_id): def remove_pusher(self, app_id, push_key, user_id):
self.get_tcp_replication().send_remove_pusher(app_id, push_key, user_id) self.get_tcp_replication().send_remove_pusher(app_id, push_key, user_id)

5
synapse/app/synchrotron.py
View File

@ -249,10 +249,7 @@ class SynchrotronApplicationService(object):
class SynchrotronServer(HomeServer): class SynchrotronServer(HomeServer):
def setup(self): DATASTORE_CLASS = SynchrotronSlavedStore
logger.info("Setting up.")
self.datastore = SynchrotronSlavedStore(self.get_db_conn(), self)
logger.info("Finished setting up.")
def _listen_http(self, listener_config): def _listen_http(self, listener_config):
port = listener_config["port"] port = listener_config["port"]

5
synapse/app/user_dir.py
View File

@ -94,10 +94,7 @@ class UserDirectorySlaveStore(
class UserDirectoryServer(HomeServer): class UserDirectoryServer(HomeServer):
def setup(self): DATASTORE_CLASS = UserDirectorySlaveStore
logger.info("Setting up.")
self.datastore = UserDirectorySlaveStore(self.get_db_conn(), self)
logger.info("Finished setting up.")
def _listen_http(self, listener_config): def _listen_http(self, listener_config):
port = listener_config["port"] port = listener_config["port"]

13
synapse/appservice/api.py
View File

@ -13,7 +13,8 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import logging import logging
import urllib
from six.moves import urllib
from prometheus_client import Counter from prometheus_client import Counter
@ -98,7 +99,7 @@ class ApplicationServiceApi(SimpleHttpClient):
def query_user(self, service, user_id): def query_user(self, service, user_id):
if service.url is None: if service.url is None:
defer.returnValue(False) defer.returnValue(False)
uri = service.url + ("/users/%s" % urllib.quote(user_id)) uri = service.url + ("/users/%s" % urllib.parse.quote(user_id))
response = None response = None
try: try:
response = yield self.get_json(uri, { response = yield self.get_json(uri, {
@ -119,7 +120,7 @@ class ApplicationServiceApi(SimpleHttpClient):
def query_alias(self, service, alias): def query_alias(self, service, alias):
if service.url is None: if service.url is None:
defer.returnValue(False) defer.returnValue(False)
uri = service.url + ("/rooms/%s" % urllib.quote(alias)) uri = service.url + ("/rooms/%s" % urllib.parse.quote(alias))
response = None response = None
try: try:
response = yield self.get_json(uri, { response = yield self.get_json(uri, {
@ -153,7 +154,7 @@ class ApplicationServiceApi(SimpleHttpClient):
service.url, service.url,
APP_SERVICE_PREFIX, APP_SERVICE_PREFIX,
kind, kind,
urllib.quote(protocol) urllib.parse.quote(protocol)
) )
try: try:
response = yield self.get_json(uri, fields) response = yield self.get_json(uri, fields)
@ -188,7 +189,7 @@ class ApplicationServiceApi(SimpleHttpClient):
uri = "%s%s/thirdparty/protocol/%s" % ( uri = "%s%s/thirdparty/protocol/%s" % (
service.url, service.url,
APP_SERVICE_PREFIX, APP_SERVICE_PREFIX,
urllib.quote(protocol) urllib.parse.quote(protocol)
) )
try: try:
info = yield self.get_json(uri, {}) info = yield self.get_json(uri, {})
@ -228,7 +229,7 @@ class ApplicationServiceApi(SimpleHttpClient):
txn_id = str(txn_id) txn_id = str(txn_id)
uri = service.url + ("/transactions/%s" % uri = service.url + ("/transactions/%s" %
urllib.quote(txn_id)) urllib.parse.quote(txn_id))
try: try:
yield self.put_json( yield self.put_json(
uri=uri, uri=uri,

2
synapse/config/homeserver.py
View File

@ -21,7 +21,7 @@ from .consent_config import ConsentConfig
from .database import DatabaseConfig from .database import DatabaseConfig
from .emailconfig import EmailConfig from .emailconfig import EmailConfig
from .groups import GroupsConfig from .groups import GroupsConfig
from .jwt import JWTConfig from .jwt_config import JWTConfig
from .key import KeyConfig from .key import KeyConfig
from .logger import LoggingConfig from .logger import LoggingConfig
from .metrics import MetricsConfig from .metrics import MetricsConfig

0
synapse/config/jwt.py → synapse/config/jwt_config.py
View File

17
synapse/config/logger.py
View File

@ -227,7 +227,22 @@ def setup_logging(config, use_worker_options=False):
# #
# However this may not be too much of a problem if we are just writing to a file. # However this may not be too much of a problem if we are just writing to a file.
observer = STDLibLogObserver() observer = STDLibLogObserver()
def _log(event):
if "log_text" in event:
if event["log_text"].startswith("DNSDatagramProtocol starting on "):
return
if event["log_text"].startswith("(UDP Port "):
return
if event["log_text"].startswith("Timing out client"):
return
return observer(event)
globalLogBeginner.beginLoggingTo( globalLogBeginner.beginLoggingTo(
[observer], [_log],
redirectStandardIO=not config.no_redirect_stdio, redirectStandardIO=not config.no_redirect_stdio,
) )

17
synapse/config/server.py
View File

@ -404,6 +404,23 @@ class ServerConfig(Config):
" service on the given port.") " service on the given port.")
def is_threepid_reserved(config, threepid):
"""Check the threepid against the reserved threepid config
Args:
config(ServerConfig) - to access server config attributes
threepid(dict) - The threepid to test for
Returns:
boolean Is the threepid undertest reserved_user
"""
for tp in config.mau_limits_reserved_threepids:
if (threepid['medium'] == tp['medium']
and threepid['address'] == tp['address']):
return True
return False
def read_gc_thresholds(thresholds): def read_gc_thresholds(thresholds):
"""Reads the three integer thresholds for garbage collection. Ensures that """Reads the three integer thresholds for garbage collection. Ensures that
the thresholds are integers if thresholds are supplied. the thresholds are integers if thresholds are supplied.

2
synapse/crypto/context_factory.py
View File

@ -123,6 +123,6 @@ class ClientTLSOptionsFactory(object):
def get_options(self, host): def get_options(self, host):
return ClientTLSOptions( return ClientTLSOptions(
host.decode('utf-8'), host,
CertificateOptions(verify=False).getContext() CertificateOptions(verify=False).getContext()
) )

8
synapse/crypto/keyclient.py
View File

@ -50,7 +50,7 @@ def fetch_server_key(server_name, tls_client_options_factory, path=KEY_API_V1):
defer.returnValue((server_response, server_certificate)) defer.returnValue((server_response, server_certificate))
except SynapseKeyClientError as e: except SynapseKeyClientError as e:
logger.warn("Error getting key for %r: %s", server_name, e) logger.warn("Error getting key for %r: %s", server_name, e)
if e.status.startswith("4"): if e.status.startswith(b"4"):
# Don't retry for 4xx responses. # Don't retry for 4xx responses.
raise IOError("Cannot get key for %r" % server_name) raise IOError("Cannot get key for %r" % server_name)
except (ConnectError, DomainError) as e: except (ConnectError, DomainError) as e:
@ -82,6 +82,12 @@ class SynapseKeyClientProtocol(HTTPClient):
self._peer = self.transport.getPeer() self._peer = self.transport.getPeer()
logger.debug("Connected to %s", self._peer) logger.debug("Connected to %s", self._peer)
if not isinstance(self.path, bytes):
self.path = self.path.encode('ascii')
if not isinstance(self.host, bytes):
self.host = self.host.encode('ascii')
self.sendCommand(b"GET", self.path) self.sendCommand(b"GET", self.path)
if self.host: if self.host:
self.sendHeader(b"Host", self.host) self.sendHeader(b"Host", self.host)

31
synapse/crypto/keyring.py
View File

@ -16,9 +16,10 @@
import hashlib import hashlib
import logging import logging
import urllib
from collections import namedtuple from collections import namedtuple
from six.moves import urllib
from signedjson.key import ( from signedjson.key import (
decode_verify_key_bytes, decode_verify_key_bytes,
encode_verify_key_base64, encode_verify_key_base64,
@ -40,6 +41,7 @@ from synapse.api.errors import Codes, SynapseError
from synapse.crypto.keyclient import fetch_server_key from synapse.crypto.keyclient import fetch_server_key
from synapse.util import logcontext, unwrapFirstError from synapse.util import logcontext, unwrapFirstError
from synapse.util.logcontext import ( from synapse.util.logcontext import (
LoggingContext,
PreserveLoggingContext, PreserveLoggingContext,
preserve_fn, preserve_fn,
run_in_background, run_in_background,
@ -216,23 +218,34 @@ class Keyring(object):
servers have completed. Follows the synapse rules of logcontext servers have completed. Follows the synapse rules of logcontext
preservation. preservation.
""" """
loop_count = 1
while True: while True:
wait_on = [ wait_on = [
self.key_downloads[server_name] (server_name, self.key_downloads[server_name])
for server_name in server_names for server_name in server_names
if server_name in self.key_downloads if server_name in self.key_downloads
] ]
if wait_on: if not wait_on:
with PreserveLoggingContext():
yield defer.DeferredList(wait_on)
else:
break break
logger.info(
"Waiting for existing lookups for %s to complete [loop %i]",
[w[0] for w in wait_on], loop_count,
)
with PreserveLoggingContext():
yield defer.DeferredList((w[1] for w in wait_on))
loop_count += 1
ctx = LoggingContext.current_context()
def rm(r, server_name_): def rm(r, server_name_):
with PreserveLoggingContext(ctx):
logger.debug("Releasing key lookup lock on %s", server_name_)
self.key_downloads.pop(server_name_, None) self.key_downloads.pop(server_name_, None)
return r return r
for server_name, deferred in server_to_deferred.items(): for server_name, deferred in server_to_deferred.items():
logger.debug("Got key lookup lock on %s", server_name)
self.key_downloads[server_name] = deferred self.key_downloads[server_name] = deferred
deferred.addBoth(rm, server_name) deferred.addBoth(rm, server_name)
@ -432,7 +445,7 @@ class Keyring(object):
# an incoming request. # an incoming request.
query_response = yield self.client.post_json( query_response = yield self.client.post_json(
destination=perspective_name, destination=perspective_name,
path=b"/_matrix/key/v2/query", path="/_matrix/key/v2/query",
data={ data={
u"server_keys": { u"server_keys": {
server_name: { server_name: {
@ -513,8 +526,8 @@ class Keyring(object):
(response, tls_certificate) = yield fetch_server_key( (response, tls_certificate) = yield fetch_server_key(
server_name, self.hs.tls_client_options_factory, server_name, self.hs.tls_client_options_factory,
path=(b"/_matrix/key/v2/server/%s" % ( path=("/_matrix/key/v2/server/%s" % (
urllib.quote(requested_key_id), urllib.parse.quote(requested_key_id),
)).encode("ascii"), )).encode("ascii"),
) )

Some files were not shown because too many files have changed in this diff Show More