From 95e02b856b2ac1409dc0d5575ee0b7be55105e9a Mon Sep 17 00:00:00 2001 From: Robert Swain Date: Thu, 28 Sep 2017 12:12:47 +0200 Subject: [PATCH 01/84] docker: Initial Dockerfile and docker-compose.yaml --- Dockerfile | 58 ++++++++++++++++++ docker/README.md | 70 ++++++++++++++++++++++ docker/docker-compose.yaml | 39 ++++++++++++ docker/rootfs/etc/service/synapse/finish | 17 ++++++ docker/rootfs/etc/service/synapse/run | 75 ++++++++++++++++++++++++ 5 files changed, 259 insertions(+) create mode 100644 Dockerfile create mode 100644 docker/README.md create mode 100644 docker/docker-compose.yaml create mode 100755 docker/rootfs/etc/service/synapse/finish create mode 100755 docker/rootfs/etc/service/synapse/run diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000..9b11a143f6 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,58 @@ +# Copyright 2017 Vector Creations Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM phusion/baseimage:0.9.22 + +COPY ./ /synapse/source/ + +RUN apt-get update -y \ + && DEBIAN_FRONTEND=noninteractive apt-get upgrade -y \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y \ + build-essential \ + libffi-dev \ + libjpeg-dev \ + libpq-dev \ + libssl-dev \ + libxslt1-dev \ + python-pip \ + python-setuptools \ + python-virtualenv \ + python2.7-dev \ + sqlite3 \ + && virtualenv -p python2.7 /synapse \ + && . /synapse/bin/activate \ + && pip install --upgrade pip \ + && pip install --upgrade setuptools \ + && pip install --upgrade psycopg2 \ + && cd /synapse/source \ + && pip install --upgrade ./ \ + && cd / \ + && rm -rf /synapse/source \ + && apt-get autoremove -y \ + build-essential \ + libffi-dev \ + libjpeg-dev \ + libpq-dev \ + libssl-dev \ + libxslt1-dev \ + python2.7-dev \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +COPY docker/rootfs/ / + +VOLUME /synapse/config/ +VOLUME /synapse/data/ + +CMD ["/sbin/my_init"] diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 0000000000..c9e6fd216d --- /dev/null +++ b/docker/README.md @@ -0,0 +1,70 @@ +# Synapse Docker + +## Build + +Build the docker image with the `docker build` command from the root of the synapse repository. + +``` +docker build -t matrix-org/synapse:v0.22.1 . +``` + +The `-t` option sets the image tag. Official images are tagged `matrix-org/synapse:` where `` is the same as the release tag in the synapse git repository. + +## Configure + +Synapse provides a command for generating homeserver configuration files. These are a good starting point for setting up your own deployment. + +The documentation below will refer to a `CONFIG_PATH` shell variable. This is a path to a directory where synapse configuration will be stored. It needs to be mapped into the container as a volume at `/synapse/config/` as can be seen in the example `docker run` command. + +Docker container environment variables: +* `GENERATE_CONFIG` - Set this to any non-empty string, such as `yes`, to trigger generation of configuration files. Existing files in the `CONFIG_PATH` will **not** be overwritten. +* `POSTGRES_DATABASE` - The database name for the synapse postgres database. [default: `synapse`] +* `POSTGRES_HOST` - The host of the postgres database if you wish to use postgresql instead of sqlite3. [default: `postgres` which is useful when using a container on the same docker network in a compose file where the postgres service is called `postgres`] **NOTE**: `localhost` and `127.0.0.1` refer to the container itself unless running the container with `host` networking. +* `POSTGRES_PASSWORD` - The password for the synapse postgres database. **If this is set then postgres will be used instead of sqlite3.** [default: none] **NOTE**: You are highly encouraged to use postgresql! Please use the compose file to make it easier to deploy. +* `POSTGRES_USER` - The user for the synapse postgres database. [default: `postgres`] +* `REPORT_STATS` - Whether to send anonymous usage statistics back to the Matrix project which helps us to get funding! Must be `yes` or `no`. [default: `yes`] +* `SERVER_NAME` - The domain used for the Matrix homeserver. If you intend to run this synapse instance on a public domain, use that domain. [default: `localhost`] + +``` +CONFIG_PATH=/my/magical/config/path/ +mkdir -p ${CONFIG_PATH} +docker run \ + --rm \ + -e GENERATE_CONFIG=yes \ + -e POSTGRES_PASSWORD=MyVerySecretPassword \ + -e REPORT_STATS=yes \ + -e SERVER_NAME=example.com \ + -v ${CONFIG_PATH}:/synapse/config/ \ + matrix-org/synapse:develop +``` + +This will create a temporary container from the image and use the synapse code for generating configuration files and TLS keys and certificates for the specified `SERVER_NAME` domain. The files are written to `CONFIG_PATH`. + +## Run + +**NOTE**: If you are not using postgresql and are using sqlite3 as your database, you will need to make a directory to store the sqlite3 database file in and then mount this volume into the container at `/synapse/data/`. As it is so easy to use postgresql, when using Docker containers, this is not documented to somewhat discourage it. Choose a `POSTGRES_PASSWORD` instead. + +### Docker Compose + +A `docker-compose.yaml` file is included to ease deployment of the basic synapse and postgres setup. Remember to set a `POSTGRES_PASSWORD` when generating your configuration above. You will need it for running the containers in the composition. + +From the `docker/` subdirectory of the synapse repository: +``` +CONFIG_PATH=/my/magical/config/path/ +POSTGRES_PASSWORD=MyVerySecretPassword \ +docker-compose \ + -p synapse \ + up -d +``` + +### Docker + +Note that the following is just a guideline and you may need to add parameters to the docker run command to account for the network situation with your postgres database. + +``` +docker run \ + -d \ + --name synapse \ + -v ${CONFIG_PATH}:/synapse/config/ \ + matrix-org/synapse:v0.22.1 +``` diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml new file mode 100644 index 0000000000..ff36081a9b --- /dev/null +++ b/docker/docker-compose.yaml @@ -0,0 +1,39 @@ +# Copyright 2017 Vector Creations Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: '3' + +services: + postgres: + image: postgres:9.6.5-alpine + environment: + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_DB: synapse + expose: + - 5432 + restart: unless-stopped + volumes: + - postgres-data:/var/lib/postgresql/data/ + + synapse: + image: matrix-org/synapse:develop + ports: + - 8008:8008 + - 8448:8448 + restart: unless-stopped + volumes: + - ${CONFIG_PATH}:/synapse/config/ + +volumes: + postgres-data: diff --git a/docker/rootfs/etc/service/synapse/finish b/docker/rootfs/etc/service/synapse/finish new file mode 100755 index 0000000000..2aace581a1 --- /dev/null +++ b/docker/rootfs/etc/service/synapse/finish @@ -0,0 +1,17 @@ +#!/bin/bash +# +# Copyright 2017 Vector Creations Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +kill -TERM 1 diff --git a/docker/rootfs/etc/service/synapse/run b/docker/rootfs/etc/service/synapse/run new file mode 100755 index 0000000000..dd797d3ef9 --- /dev/null +++ b/docker/rootfs/etc/service/synapse/run @@ -0,0 +1,75 @@ +#!/bin/bash +# +# Copyright 2017 Vector Creations Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +: ${CONFIG_PATH:="/synapse/config"} +: ${POSTGRES_DATABASE:="synapse"} +: ${POSTGRES_HOST:="postgres"} +: ${POSTGRES_USER:="postgres"} +: ${REPORT_STATS:="yes"} +: ${SERVER_NAME:="localhost"} + +DATABASE_CONFIG_PATH="${CONFIG_PATH}/database.yaml" +HOMESERVER_CONFIG_PATH="${CONFIG_PATH}/homeserver.yaml" +SYNAPSE_COMMAND="python -m synapse.app.homeserver" + +. /synapse/bin/activate +cd /synapse + +if [[ -n "${GENERATE_CONFIG}" ]]; then + ${SYNAPSE_COMMAND} \ + --server-name ${SERVER_NAME} \ + --config-path ${HOMESERVER_CONFIG_PATH} \ + --generate-config \ + --report-stats=${REPORT_STATS} + + if [[ -f "${DATABASE_CONFIG_PATH}" ]]; then + echo "Config file '${DATABASE_CONFIG_PATH}' already exists. Remove it if you want it to be generated." + else + echo "Generating ${DATABASE_CONFIG_PATH}..." + if [[ -n "${POSTGRES_PASSWORD}" ]]; then + (cat > ${DATABASE_CONFIG_PATH}) < ${DATABASE_CONFIG_PATH}) < Date: Fri, 29 Sep 2017 11:40:15 +0200 Subject: [PATCH 02/84] docker: s/matrix-org/matrixdotorg/g --- docker/README.md | 8 ++++---- docker/docker-compose.yaml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docker/README.md b/docker/README.md index c9e6fd216d..c15517d0e0 100644 --- a/docker/README.md +++ b/docker/README.md @@ -5,10 +5,10 @@ Build the docker image with the `docker build` command from the root of the synapse repository. ``` -docker build -t matrix-org/synapse:v0.22.1 . +docker build -t matrixdotorg/synapse:v0.22.1 . ``` -The `-t` option sets the image tag. Official images are tagged `matrix-org/synapse:` where `` is the same as the release tag in the synapse git repository. +The `-t` option sets the image tag. Official images are tagged `matrixdotorg/synapse:` where `` is the same as the release tag in the synapse git repository. ## Configure @@ -35,7 +35,7 @@ docker run \ -e REPORT_STATS=yes \ -e SERVER_NAME=example.com \ -v ${CONFIG_PATH}:/synapse/config/ \ - matrix-org/synapse:develop + matrixdotorg/synapse:v0.22.1 ``` This will create a temporary container from the image and use the synapse code for generating configuration files and TLS keys and certificates for the specified `SERVER_NAME` domain. The files are written to `CONFIG_PATH`. @@ -66,5 +66,5 @@ docker run \ -d \ --name synapse \ -v ${CONFIG_PATH}:/synapse/config/ \ - matrix-org/synapse:v0.22.1 + matrixdotorg/synapse:v0.22.1 ``` diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index ff36081a9b..73cc29f8fd 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -27,7 +27,7 @@ services: - postgres-data:/var/lib/postgresql/data/ synapse: - image: matrix-org/synapse:develop + image: matrixdotorg/synapse:v0.22.1 ports: - 8008:8008 - 8448:8448 From 431476fbc4ef0c740e33e19ccc73996c2412e4f9 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sat, 3 Feb 2018 20:18:36 +0100 Subject: [PATCH 03/84] Initial commit including a Dockerfile for synapse --- Dockerfile | 16 ++++++++++++++++ contrib/docker/start.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 Dockerfile create mode 100755 contrib/docker/start.py diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000..5f0433004f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,16 @@ +FROM python:2-alpine + +RUN apk add --no-cache --virtual .nacl_deps build-base libffi-dev zlib-dev openssl-dev libjpeg-turbo-dev linux-headers + +COPY synapse /usr/local/src/synapse +COPY setup.py setup.cfg README.rst synctl /usr/local/src/ + +RUN cd /usr/local/src \ + && pip install --upgrade --process-dependency-links . \ + && rm -rf setup.py setup.cfg synapse + +COPY contrib/docker / + +VOLUME ["/data"] + +ENTRYPOINT ["/start.py"] diff --git a/contrib/docker/start.py b/contrib/docker/start.py new file mode 100755 index 0000000000..4f63ea1ad5 --- /dev/null +++ b/contrib/docker/start.py @@ -0,0 +1,29 @@ +#!/usr/local/bin/python + +import jinja2 +import os +import sys +import socket + +convert = lambda src, dst: open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ)) +mode = sys.argv[1] if len(sys.argv) > 1 else None + +if "SYNAPSE_SERVER_NAME" not in os.environ: + print("Environment variable SYNAPSE_SERVER_NAME is mandatory, exiting.") + sys.exit(2) + +params = ["--server-name", os.environ.get("SYNAPSE_SERVER_NAME"), + "--report-stats", os.environ.get("SYNAPSE_REPORT_STATS", "no"), + "--config-path", os.environ.get("SYNAPSE_CONFIG_PATH", "/compiled/homeserver.yaml")] + +if mode == "generate": + params.append("--generate-config") + +# Parse the configuration file +if not os.path.exists("/compiled"): + os.mkdir("/compiled") +convert("/conf/homeserver.yaml", "/compiled/homeserver.yaml") +convert("/conf/log.config", "/compiled/%s.log.config" % os.environ.get("SYNAPSE_SERVER_NAME")) + +# TODO, replace with a call to synapse.app.homeserver.run() +os.execv("/usr/local/bin/python", ["python", "-m", "synapse.app.homeserver"] + params) From d434ae33875b4c7d5ee04c45e454ce237ce578f8 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sat, 3 Feb 2018 20:30:08 +0100 Subject: [PATCH 04/84] Add template config files for the Docker image --- contrib/docker/conf/homeserver.yaml | 181 ++++++++++++++++++++++++++++ contrib/docker/conf/log.config | 36 ++++++ 2 files changed, 217 insertions(+) create mode 100644 contrib/docker/conf/homeserver.yaml create mode 100644 contrib/docker/conf/log.config diff --git a/contrib/docker/conf/homeserver.yaml b/contrib/docker/conf/homeserver.yaml new file mode 100644 index 0000000000..851c389c19 --- /dev/null +++ b/contrib/docker/conf/homeserver.yaml @@ -0,0 +1,181 @@ +# vim:ft=yaml + +## TLS ## + +tls_certificate_path: "/data/{{ SYNAPSE_SERVER_NAME }}.tls.crt" +tls_private_key_path: "/data/{{ SYNAPSE_SERVER_NAME }}.tls.key" +tls_dh_params_path: "/data/{{ SYNAPSE_SERVER_NAME }}.tls.dh" +no_tls: {{ "True" if SYNAPSE_NO_TLS else "False" }} +tls_fingerprints: [] + +## Server ## + +server_name: "{{ SYNAPSE_SERVER_NAME }}" +pid_file: /homeserver.pid +web_client: {{ "True" if SYNAPSE_WEB_CLIENT else "False" }} +soft_file_limit: 0 + +## Ports ## + +listeners: + {% if not SYNAPSE_NO_TLS %} + - + port: 8448 + bind_addresses: ['0.0.0.0'] + type: http + tls: true + x_forwarded: false + resources: + - names: [client, webclient] + compress: true + - names: [federation] # Federation APIs + compress: false + {% endif %} + + - port: 8008 + tls: false + bind_addresses: ['0.0.0.0'] + type: http + x_forwarded: false + + resources: + - names: [client, webclient] + compress: true + - names: [federation] + compress: false + +## Database ## + +{% if SYNAPSE_DB_HOST %} + +{% else %} +database: + name: "sqlite3" + args: + database: "/data/homeserver.db" +{% endif %} + +## Performance ## + +event_cache_size: "{{ SYNAPSE_EVENT_CACHE_SIZE or "10K" }}" +verbose: 0 +log_file: "/data/homeserver.log" +log_config: "/data/{{ SYNAPSE_SERVER_NAME }}.log.config" + +## Ratelimiting ## + +rc_messages_per_second: 0.2 +rc_message_burst_count: 10.0 +federation_rc_window_size: 1000 +federation_rc_sleep_limit: 10 +federation_rc_sleep_delay: 500 +federation_rc_reject_limit: 50 +federation_rc_concurrent: 3 + +## Files ## + +media_store_path: "/data/media" +uploads_path: "/data/uploads" +max_upload_size: "10M" +max_image_pixels: "32M" +dynamic_thumbnails: false + +# List of thumbnail to precalculate when an image is uploaded. +thumbnail_sizes: +- width: 32 + height: 32 + method: crop +- width: 96 + height: 96 + method: crop +- width: 320 + height: 240 + method: scale +- width: 640 + height: 480 + method: scale +- width: 800 + height: 600 + method: scale + +url_preview_enabled: False +max_spider_size: "10M" + +## Captcha ## + +recaptcha_public_key: "YOUR_PUBLIC_KEY" +recaptcha_private_key: "YOUR_PRIVATE_KEY" +enable_registration_captcha: False +recaptcha_siteverify_api: "https://www.google.com/recaptcha/api/siteverify" + +## Turn ## + +turn_uris: [] +turn_shared_secret: "YOUR_SHARED_SECRET" +turn_user_lifetime: "1h" +turn_allow_guests: True + +## Registration ## + +enable_registration: {{ "True" if SYNAPSE_ENABLE_REGISTRATION else "False" }} +registration_shared_secret: "{{ SYNAPSE_REGISTRATION_SHARED_SECRET }}" +bcrypt_rounds: 12 +allow_guest_access: {{ "True" if SYNAPSE_ALLOW_GUEST else "False" }} + +# The list of identity servers trusted to verify third party +# identifiers by this server. +trusted_third_party_id_servers: + - matrix.org + - vector.im + - riot.im + +## Metrics ### + +enable_metrics: False +report_stats: False + +## API Configuration ## + +room_invite_state_types: + - "m.room.join_rules" + - "m.room.canonical_alias" + - "m.room.avatar" + - "m.room.name" + +app_service_config_files: [] +macaroon_secret_key: "{{ SYNAPSE_MACAROON_SECRET_KEY }}" +expire_access_token: False + +## Signing Keys ## + +signing_key_path: "/data/{{ SYNAPSE_SERVER_NAME }}.signing.key" +old_signing_keys: {} +key_refresh_interval: "1d" # 1 Day. + +# The trusted servers to download signing keys from. +perspectives: + servers: + "matrix.org": + verify_keys: + "ed25519:auto": + key: "Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw" + +password_config: + enabled: true + +#email: +# enable_notifs: false +# smtp_host: "localhost" +# smtp_port: 25 +# smtp_user: "exampleusername" +# smtp_pass: "examplepassword" +# require_transport_security: False +# notif_from: "Your Friendly %(app)s Home Server " +# app_name: Matrix +# template_dir: res/templates +# notif_template_html: notif_mail.html +# notif_template_text: notif_mail.txt +# notif_for_new_users: True +# riot_base_url: "http://localhost/riot" + +enable_group_creation: true diff --git a/contrib/docker/conf/log.config b/contrib/docker/conf/log.config new file mode 100644 index 0000000000..45e7eef953 --- /dev/null +++ b/contrib/docker/conf/log.config @@ -0,0 +1,36 @@ +version: 1 + +formatters: + precise: + format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s- %(message)s' + +filters: + context: + (): synapse.util.logcontext.LoggingContextFilter + request: "" + +handlers: + file: + class: logging.handlers.RotatingFileHandler + formatter: precise + filename: /data/homeserver.log + maxBytes: 104857600 + backupCount: 10 + filters: [context] + console: + class: logging.StreamHandler + formatter: precise + filters: [context] + +loggers: + synapse: + level: INFO + + synapse.storage.SQL: + # beware: increasing this to DEBUG will make synapse log sensitive + # information such as access tokens. + level: INFO + +root: + level: INFO + handlers: [file, console] From 48bc22f89dadb8278cf2b8c940604534999d246f Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sun, 4 Feb 2018 10:58:07 +0100 Subject: [PATCH 05/84] Allow for a wheel cache and include missing files in the build --- .dockerignore | 5 +++++ Dockerfile | 13 +++++++------ contrib/docker/start.py | 1 + 3 files changed, 13 insertions(+), 6 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000..f36f86fbb7 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,5 @@ +Dockerfile +.travis.yml +.gitignore +demo/etc +tox.ini diff --git a/Dockerfile b/Dockerfile index 5f0433004f..277246b697 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,15 +2,16 @@ FROM python:2-alpine RUN apk add --no-cache --virtual .nacl_deps build-base libffi-dev zlib-dev openssl-dev libjpeg-turbo-dev linux-headers -COPY synapse /usr/local/src/synapse -COPY setup.py setup.cfg README.rst synctl /usr/local/src/ +COPY . /synapse -RUN cd /usr/local/src \ - && pip install --upgrade --process-dependency-links . \ +# A wheel cache may be provided in ./cache for faster build +RUN cd /synapse \ + && pip install --upgrade pip setuptools \ + && mkdir -p /synapse/cache \ + && pip install -f /synapse/cache --upgrade --process-dependency-links . \ + && mv /synapse/contrib/docker/* / \ && rm -rf setup.py setup.cfg synapse -COPY contrib/docker / - VOLUME ["/data"] ENTRYPOINT ["/start.py"] diff --git a/contrib/docker/start.py b/contrib/docker/start.py index 4f63ea1ad5..2c427ba1b7 100755 --- a/contrib/docker/start.py +++ b/contrib/docker/start.py @@ -22,6 +22,7 @@ if mode == "generate": # Parse the configuration file if not os.path.exists("/compiled"): os.mkdir("/compiled") + convert("/conf/homeserver.yaml", "/compiled/homeserver.yaml") convert("/conf/log.config", "/compiled/%s.log.config" % os.environ.get("SYNAPSE_SERVER_NAME")) From 6d1e28a8426da9e954a3edec25a8717376c583f3 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sun, 4 Feb 2018 11:14:06 +0100 Subject: [PATCH 06/84] Generate any missing keys before starting synapse --- contrib/docker/start.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/contrib/docker/start.py b/contrib/docker/start.py index 2c427ba1b7..e50d23be5f 100755 --- a/contrib/docker/start.py +++ b/contrib/docker/start.py @@ -3,7 +3,7 @@ import jinja2 import os import sys -import socket +import subprocess convert = lambda src, dst: open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ)) mode = sys.argv[1] if len(sys.argv) > 1 else None @@ -12,19 +12,20 @@ if "SYNAPSE_SERVER_NAME" not in os.environ: print("Environment variable SYNAPSE_SERVER_NAME is mandatory, exiting.") sys.exit(2) -params = ["--server-name", os.environ.get("SYNAPSE_SERVER_NAME"), - "--report-stats", os.environ.get("SYNAPSE_REPORT_STATS", "no"), - "--config-path", os.environ.get("SYNAPSE_CONFIG_PATH", "/compiled/homeserver.yaml")] - -if mode == "generate": - params.append("--generate-config") +args = ["python", "-m", "synapse.app.homeserver", + "--server-name", os.environ.get("SYNAPSE_SERVER_NAME"), + "--report-stats", os.environ.get("SYNAPSE_REPORT_STATS", "no"), + "--config-path", os.environ.get("SYNAPSE_CONFIG_PATH", "/compiled/homeserver.yaml")] # Parse the configuration file -if not os.path.exists("/compiled"): - os.mkdir("/compiled") - +if not os.path.exists("/compiled"): os.mkdir("/compiled") convert("/conf/homeserver.yaml", "/compiled/homeserver.yaml") convert("/conf/log.config", "/compiled/%s.log.config" % os.environ.get("SYNAPSE_SERVER_NAME")) -# TODO, replace with a call to synapse.app.homeserver.run() -os.execv("/usr/local/bin/python", ["python", "-m", "synapse.app.homeserver"] + params) +# In generate mode, generate a configuration, missing keys, then exit +if mode == "generate": + os.execv("/usr/local/bin/python", args + ["--generate-config"]) +# In normal mode, generate missing keys if any, then run synapse +else: + subprocess.check_output(args + ["--generate-keys"]) + os.execv("/usr/local/bin/python", args) From f2bf0cda02fef358172033b28dab5f1805c31cad Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sun, 4 Feb 2018 11:40:20 +0100 Subject: [PATCH 07/84] Generate shared secrets if not defined in the environment --- contrib/docker/start.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/contrib/docker/start.py b/contrib/docker/start.py index e50d23be5f..7057f85f61 100755 --- a/contrib/docker/start.py +++ b/contrib/docker/start.py @@ -5,10 +5,11 @@ import os import sys import subprocess -convert = lambda src, dst: open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ)) +convert = lambda src, dst, environ: open(dst, "w").write(jinja2.Template(open(src).read()).render(**environ)) mode = sys.argv[1] if len(sys.argv) > 1 else None +environ = os.environ.copy() -if "SYNAPSE_SERVER_NAME" not in os.environ: +if "SYNAPSE_SERVER_NAME" not in environ: print("Environment variable SYNAPSE_SERVER_NAME is mandatory, exiting.") sys.exit(2) @@ -17,10 +18,16 @@ args = ["python", "-m", "synapse.app.homeserver", "--report-stats", os.environ.get("SYNAPSE_REPORT_STATS", "no"), "--config-path", os.environ.get("SYNAPSE_CONFIG_PATH", "/compiled/homeserver.yaml")] +# Generate any missing shared secret +for secret in ("SYNAPSE_REGISTRATION_SHARED_SECRET", "SYNAPSE_MACAROON_SECRET_KEY"): + if secret not in environ: + print("Generating a random secret for {}".format(secret)) + environ[secret] = os.urandom(32).encode("hex") + # Parse the configuration file if not os.path.exists("/compiled"): os.mkdir("/compiled") -convert("/conf/homeserver.yaml", "/compiled/homeserver.yaml") -convert("/conf/log.config", "/compiled/%s.log.config" % os.environ.get("SYNAPSE_SERVER_NAME")) +convert("/conf/homeserver.yaml", "/compiled/homeserver.yaml", environ) +convert("/conf/log.config", "/compiled/%s.log.config" % environ.get("SYNAPSE_SERVER_NAME"), environ) # In generate mode, generate a configuration, missing keys, then exit if mode == "generate": From 886c2d50197bb3558168ce45b6975777780c7aad Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sun, 4 Feb 2018 12:20:29 +0100 Subject: [PATCH 08/84] Support an external postgresql config in the Docker image --- contrib/docker/conf/homeserver.yaml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/contrib/docker/conf/homeserver.yaml b/contrib/docker/conf/homeserver.yaml index 851c389c19..1a685320c0 100644 --- a/contrib/docker/conf/homeserver.yaml +++ b/contrib/docker/conf/homeserver.yaml @@ -46,8 +46,17 @@ listeners: ## Database ## -{% if SYNAPSE_DB_HOST %} - +{% if SYNAPSE_DB_PASSWORD %} +database: + name: "psycopg2" + args: + user: "{{ SYNAPSE_DB_USER or "matrix" }}" + password: "{{ SYNAPSE_DB_PASSWORD }}" + database: "{{ SYNAPSE_DB_DATABASE or "matrix" }}" + host: "{{ SYNAPSE_DB_HOST or "db" }}" + port: "{{ SYNAPSE_DB_PORT or "5432" }}" + cp_min: 5 + cp_max: 10 {% else %} database: name: "sqlite3" From 042757feb25ba9f0cccfdaa6c8775957880c2d2e Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sun, 4 Feb 2018 12:28:42 +0100 Subject: [PATCH 09/84] Install the postgres dependencies --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 277246b697..881c25c243 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,12 @@ FROM python:2-alpine -RUN apk add --no-cache --virtual .nacl_deps build-base libffi-dev zlib-dev openssl-dev libjpeg-turbo-dev linux-headers +RUN apk add --no-cache --virtual .nacl_deps build-base libffi-dev zlib-dev openssl-dev libjpeg-turbo-dev linux-headers postgresql-dev COPY . /synapse # A wheel cache may be provided in ./cache for faster build RUN cd /synapse \ - && pip install --upgrade pip setuptools \ + && pip install --upgrade pip setuptools psycopg2 \ && mkdir -p /synapse/cache \ && pip install -f /synapse/cache --upgrade --process-dependency-links . \ && mv /synapse/contrib/docker/* / \ From 1ba2fe114c79dd0f2f2fb8868f8cb0a9f5893652 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sun, 4 Feb 2018 12:55:20 +0100 Subject: [PATCH 10/84] Provide an example docker compose file --- Dockerfile | 2 ++ contrib/docker/docker-compose.yml | 32 +++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 contrib/docker/docker-compose.yml diff --git a/Dockerfile b/Dockerfile index 881c25c243..25f3746303 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,4 +14,6 @@ RUN cd /synapse \ VOLUME ["/data"] +EXPOSE 8448 + ENTRYPOINT ["/start.py"] diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml new file mode 100644 index 0000000000..659bee2979 --- /dev/null +++ b/contrib/docker/docker-compose.yml @@ -0,0 +1,32 @@ +# This compose file is compatible with Copose itself, it might need some +# adjustments to run properly with stack. + +version: '3' + +services: + + synapse: + image: matrix/synapse + # See the readme for a full documentation of the environment settings + environment: + - SYNAPSE_SERVER_NAME=my.matrix.host + - SYNAPSE_DB_PASSWORD=changeme + volumes: + - ./files:/data + # One may either expose ports directly + ports: + - 8448:8448/tcp + # ... or use a reverse proxy, here is an example for traefik + labels: + - traefik.enable=true + - traefik.frontend.rule=Host:my.matrix.Host + - traefik.port=8448 + + db: + image: postgres:latest + # Change that password, of course! + environment: + - POSTGRES_USER=matrix + - POSTGRES_PASSWORD=changeme + volumes: + - ./schemas:/var/lib/postgres From a207cccb059451682564cddf0e39c7c45b06cb72 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sun, 4 Feb 2018 15:04:26 +0100 Subject: [PATCH 11/84] Reuse environment variables of the postgres container --- contrib/docker/conf/homeserver.yaml | 12 ++++++------ contrib/docker/docker-compose.yml | 11 ++++++++--- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/contrib/docker/conf/homeserver.yaml b/contrib/docker/conf/homeserver.yaml index 1a685320c0..7450cc1228 100644 --- a/contrib/docker/conf/homeserver.yaml +++ b/contrib/docker/conf/homeserver.yaml @@ -46,15 +46,15 @@ listeners: ## Database ## -{% if SYNAPSE_DB_PASSWORD %} +{% if POSTGRES_PASSWORD %} database: name: "psycopg2" args: - user: "{{ SYNAPSE_DB_USER or "matrix" }}" - password: "{{ SYNAPSE_DB_PASSWORD }}" - database: "{{ SYNAPSE_DB_DATABASE or "matrix" }}" - host: "{{ SYNAPSE_DB_HOST or "db" }}" - port: "{{ SYNAPSE_DB_PORT or "5432" }}" + user: "{{ POSTGRES_USER or "matrix" }}" + password: "{{ POSTGRES_PASSWORD }}" + database: "{{ POSTGRES_DB or "matrix" }}" + host: "{{ POSTGRES_HOST or "db" }}" + port: "{{ POSTGRES_PORT or "5432" }}" cp_min: 5 cp_max: 10 {% else %} diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml index 659bee2979..b8f9741f05 100644 --- a/contrib/docker/docker-compose.yml +++ b/contrib/docker/docker-compose.yml @@ -6,13 +6,18 @@ version: '3' services: synapse: - image: matrix/synapse + image: synapse + # Since snyapse does not retry to connect to the database, restart upon + # failure + restart: unless-stopped # See the readme for a full documentation of the environment settings environment: - SYNAPSE_SERVER_NAME=my.matrix.host - - SYNAPSE_DB_PASSWORD=changeme + - SYNAPSE_ENABLE_REGISTRATION=yes volumes: - ./files:/data + depends_on: + - db # One may either expose ports directly ports: - 8448:8448/tcp @@ -29,4 +34,4 @@ services: - POSTGRES_USER=matrix - POSTGRES_PASSWORD=changeme volumes: - - ./schemas:/var/lib/postgres + - ./schemas:/var/lib/postgresql/data From 84a9209ba7294243e5bbefede46a3dee7994da9b Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sun, 4 Feb 2018 15:08:43 +0100 Subject: [PATCH 12/84] Remove etc/service files from rob's branch --- contrib/docker/rob/docker-compose.yaml | 39 ---------- .../rob/rootfs/etc/service/synapse/finish | 17 ----- .../docker/rob/rootfs/etc/service/synapse/run | 75 ------------------- 3 files changed, 131 deletions(-) delete mode 100644 contrib/docker/rob/docker-compose.yaml delete mode 100755 contrib/docker/rob/rootfs/etc/service/synapse/finish delete mode 100755 contrib/docker/rob/rootfs/etc/service/synapse/run diff --git a/contrib/docker/rob/docker-compose.yaml b/contrib/docker/rob/docker-compose.yaml deleted file mode 100644 index 73cc29f8fd..0000000000 --- a/contrib/docker/rob/docker-compose.yaml +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2017 Vector Creations Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: '3' - -services: - postgres: - image: postgres:9.6.5-alpine - environment: - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_DB: synapse - expose: - - 5432 - restart: unless-stopped - volumes: - - postgres-data:/var/lib/postgresql/data/ - - synapse: - image: matrixdotorg/synapse:v0.22.1 - ports: - - 8008:8008 - - 8448:8448 - restart: unless-stopped - volumes: - - ${CONFIG_PATH}:/synapse/config/ - -volumes: - postgres-data: diff --git a/contrib/docker/rob/rootfs/etc/service/synapse/finish b/contrib/docker/rob/rootfs/etc/service/synapse/finish deleted file mode 100755 index 2aace581a1..0000000000 --- a/contrib/docker/rob/rootfs/etc/service/synapse/finish +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -# -# Copyright 2017 Vector Creations Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -kill -TERM 1 diff --git a/contrib/docker/rob/rootfs/etc/service/synapse/run b/contrib/docker/rob/rootfs/etc/service/synapse/run deleted file mode 100755 index dd797d3ef9..0000000000 --- a/contrib/docker/rob/rootfs/etc/service/synapse/run +++ /dev/null @@ -1,75 +0,0 @@ -#!/bin/bash -# -# Copyright 2017 Vector Creations Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e - -: ${CONFIG_PATH:="/synapse/config"} -: ${POSTGRES_DATABASE:="synapse"} -: ${POSTGRES_HOST:="postgres"} -: ${POSTGRES_USER:="postgres"} -: ${REPORT_STATS:="yes"} -: ${SERVER_NAME:="localhost"} - -DATABASE_CONFIG_PATH="${CONFIG_PATH}/database.yaml" -HOMESERVER_CONFIG_PATH="${CONFIG_PATH}/homeserver.yaml" -SYNAPSE_COMMAND="python -m synapse.app.homeserver" - -. /synapse/bin/activate -cd /synapse - -if [[ -n "${GENERATE_CONFIG}" ]]; then - ${SYNAPSE_COMMAND} \ - --server-name ${SERVER_NAME} \ - --config-path ${HOMESERVER_CONFIG_PATH} \ - --generate-config \ - --report-stats=${REPORT_STATS} - - if [[ -f "${DATABASE_CONFIG_PATH}" ]]; then - echo "Config file '${DATABASE_CONFIG_PATH}' already exists. Remove it if you want it to be generated." - else - echo "Generating ${DATABASE_CONFIG_PATH}..." - if [[ -n "${POSTGRES_PASSWORD}" ]]; then - (cat > ${DATABASE_CONFIG_PATH}) < ${DATABASE_CONFIG_PATH}) < Date: Sun, 4 Feb 2018 15:27:32 +0100 Subject: [PATCH 13/84] Update sumperdump Docker readme to match this image properties --- contrib/docker/README.md | 108 ++++++++++++++++++++++++--------------- 1 file changed, 67 insertions(+), 41 deletions(-) diff --git a/contrib/docker/README.md b/contrib/docker/README.md index c15517d0e0..8ba5f79692 100644 --- a/contrib/docker/README.md +++ b/contrib/docker/README.md @@ -1,5 +1,12 @@ # Synapse Docker +This Docker image will run Synapse as a single process. It does not provide any +database server or TURN server that you should run separately. + +If you run a Postgres server, you should simply have it in the same Compose +project or set the proper environment variables and the image will automatically +use that server. + ## Build Build the docker image with the `docker build` command from the root of the synapse repository. @@ -10,54 +17,29 @@ docker build -t matrixdotorg/synapse:v0.22.1 . The `-t` option sets the image tag. Official images are tagged `matrixdotorg/synapse:` where `` is the same as the release tag in the synapse git repository. -## Configure - -Synapse provides a command for generating homeserver configuration files. These are a good starting point for setting up your own deployment. - -The documentation below will refer to a `CONFIG_PATH` shell variable. This is a path to a directory where synapse configuration will be stored. It needs to be mapped into the container as a volume at `/synapse/config/` as can be seen in the example `docker run` command. - -Docker container environment variables: -* `GENERATE_CONFIG` - Set this to any non-empty string, such as `yes`, to trigger generation of configuration files. Existing files in the `CONFIG_PATH` will **not** be overwritten. -* `POSTGRES_DATABASE` - The database name for the synapse postgres database. [default: `synapse`] -* `POSTGRES_HOST` - The host of the postgres database if you wish to use postgresql instead of sqlite3. [default: `postgres` which is useful when using a container on the same docker network in a compose file where the postgres service is called `postgres`] **NOTE**: `localhost` and `127.0.0.1` refer to the container itself unless running the container with `host` networking. -* `POSTGRES_PASSWORD` - The password for the synapse postgres database. **If this is set then postgres will be used instead of sqlite3.** [default: none] **NOTE**: You are highly encouraged to use postgresql! Please use the compose file to make it easier to deploy. -* `POSTGRES_USER` - The user for the synapse postgres database. [default: `postgres`] -* `REPORT_STATS` - Whether to send anonymous usage statistics back to the Matrix project which helps us to get funding! Must be `yes` or `no`. [default: `yes`] -* `SERVER_NAME` - The domain used for the Matrix homeserver. If you intend to run this synapse instance on a public domain, use that domain. [default: `localhost`] - -``` -CONFIG_PATH=/my/magical/config/path/ -mkdir -p ${CONFIG_PATH} -docker run \ - --rm \ - -e GENERATE_CONFIG=yes \ - -e POSTGRES_PASSWORD=MyVerySecretPassword \ - -e REPORT_STATS=yes \ - -e SERVER_NAME=example.com \ - -v ${CONFIG_PATH}:/synapse/config/ \ - matrixdotorg/synapse:v0.22.1 -``` - -This will create a temporary container from the image and use the synapse code for generating configuration files and TLS keys and certificates for the specified `SERVER_NAME` domain. The files are written to `CONFIG_PATH`. +You may have a local Python wheel cache available, in which case copy the relevant packages in the ``cache/`` directory at the root of the project. ## Run -**NOTE**: If you are not using postgresql and are using sqlite3 as your database, you will need to make a directory to store the sqlite3 database file in and then mount this volume into the container at `/synapse/data/`. As it is so easy to use postgresql, when using Docker containers, this is not documented to somewhat discourage it. Choose a `POSTGRES_PASSWORD` instead. +It is recommended that you use Docker Compose to run your containers, including +this image and a Postgres server. A sample ``docker-compose.yml`` is provided, +with example labels for a reverse proxy and other artifacts. -### Docker Compose +Then, to run the server: -A `docker-compose.yaml` file is included to ease deployment of the basic synapse and postgres setup. Remember to set a `POSTGRES_PASSWORD` when generating your configuration above. You will need it for running the containers in the composition. - -From the `docker/` subdirectory of the synapse repository: ``` -CONFIG_PATH=/my/magical/config/path/ -POSTGRES_PASSWORD=MyVerySecretPassword \ -docker-compose \ - -p synapse \ - up -d +docker-compose up -d ``` -### Docker +In the case you specified a custom path for you configuration file and wish to +generate a fresh ``homeserver.yaml``, simply run: + +``` +docker-compose run synapse generate +``` + +If you do not wish to use Compose, you may still run this image using plain +Docker commands: Note that the following is just a guideline and you may need to add parameters to the docker run command to account for the network situation with your postgres database. @@ -65,6 +47,50 @@ Note that the following is just a guideline and you may need to add parameters t docker run \ -d \ --name synapse \ - -v ${CONFIG_PATH}:/synapse/config/ \ + -v ${DATA_PATH}:/data \ + -e SYNAPSE_SERVER_NAME=my.matrix.host \ matrixdotorg/synapse:v0.22.1 ``` + + +## Volumes + +The image expects a single volue, located at ``/data``, that will hold: + +* temporary files during uploads; +* uploaded media and thumbnais; +* the SQLite database if you do not configure postgres. + +## Environment + +If you do not specify a custom path for the configuration file, a very generic +file will be generated, based on the following environment settings. +These are a good starting point for setting up your own deployment. + +Synapse specific settings: + +* ``SYNAPSE_SERVER_NAME`` (mandatory), the current server public hostname. +* ``SYNAPSE_CONFIG_PATH``, path to a custom config file (will ignore all + other options then). +* ``SYNAPSE_NO_TLS``, set this variable to disable TLS in Synapse (use this if + you run your own TLS-capable reverse proxy). +* ``SYNAPSE_WEB_CLIENT``, set this variable to enable the embedded Web client. +* ``SYNAPSE_ENABLE_REGISTRATION``, set this variable to enable registration on + the Synapse instance. +* ``SYNAPSE_ALLOW_GUEST``, set this variable to allow guest joining this server. +* ``SYNAPSE_EVENT_CACHE_SIZE``, the event cache size [default `10K`]. +* ``SYNAPSE_REPORT_STATS``, set this variable to `yes` to enable anonymous + statistics reporting back to the Matrix project which helps us to get funding. + +Shared secrets, these will be initialized to random values if not set: + +* ``SYNAPSE_REGISTRATION_SHARED_SECRET``, secret for registrering users if + registration is disable. +* ``SYNAPSE_MACAROON_SECRET_KEY``, secret for Macaroon. + +Database specific values (will use SQLite if not set): + +* `POSTGRES_DATABASE` - The database name for the synapse postgres database. [default: `matrix`] +* `POSTGRES_HOST` - The host of the postgres database if you wish to use postgresql instead of sqlite3. [default: `db` which is useful when using a container on the same docker network in a compose file where the postgres service is called `db`] +* `POSTGRES_PASSWORD` - The password for the synapse postgres database. **If this is set then postgres will be used instead of sqlite3.** [default: none] **NOTE**: You are highly encouraged to use postgresql! Please use the compose file to make it easier to deploy. +* `POSTGRES_USER` - The user for the synapse postgres database. [default: `matrix`] From b8ab78b82c60ab9c897891cf9f53c9cc44873a25 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sun, 4 Feb 2018 15:41:54 +0100 Subject: [PATCH 14/84] Add the build cache/ folder to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 491047c352..d13017f400 100644 --- a/.gitignore +++ b/.gitignore @@ -32,6 +32,7 @@ demo/media_store.* demo/etc uploads +cache .idea/ media_store/ From f72c9c1fb650c7f3bac4d77d4e24fa0469d09ebb Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sun, 4 Feb 2018 16:18:40 +0100 Subject: [PATCH 15/84] Fix multiple typos --- MANIFEST.in | 1 + contrib/docker/README.md | 19 ++++++++++++------- contrib/docker/docker-compose.yml | 7 ++++--- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index afb60e12ee..8c9a57a9ca 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -25,6 +25,7 @@ recursive-include synapse/static *.js exclude jenkins.sh exclude jenkins*.sh exclude jenkins* +exclude Dockerfile recursive-exclude jenkins *.sh prune .github diff --git a/contrib/docker/README.md b/contrib/docker/README.md index 8ba5f79692..73e53e4306 100644 --- a/contrib/docker/README.md +++ b/contrib/docker/README.md @@ -23,7 +23,7 @@ You may have a local Python wheel cache available, in which case copy the releva It is recommended that you use Docker Compose to run your containers, including this image and a Postgres server. A sample ``docker-compose.yml`` is provided, -with example labels for a reverse proxy and other artifacts. +including example labels for reverse proxying and other artifacts. Then, to run the server: @@ -35,13 +35,13 @@ In the case you specified a custom path for you configuration file and wish to generate a fresh ``homeserver.yaml``, simply run: ``` -docker-compose run synapse generate +docker-compose run --rm synapse generate ``` If you do not wish to use Compose, you may still run this image using plain -Docker commands: - -Note that the following is just a guideline and you may need to add parameters to the docker run command to account for the network situation with your postgres database. +Docker commands. Note that the following is just a guideline and you may need +to add parameters to the docker run command to account for the network situation +with your postgres database. ``` docker run \ @@ -55,7 +55,7 @@ docker run \ ## Volumes -The image expects a single volue, located at ``/data``, that will hold: +The image expects a single volume, located at ``/data``, that will hold: * temporary files during uploads; * uploaded media and thumbnais; @@ -63,10 +63,15 @@ The image expects a single volue, located at ``/data``, that will hold: ## Environment -If you do not specify a custom path for the configuration file, a very generic +Unless you specify a custom path for the configuration file, a very generic file will be generated, based on the following environment settings. These are a good starting point for setting up your own deployment. +Global settings: + +* ``UID``, the user id Synapse will run as [default 991] +* ``GID``, the group id Synapse will run as [default 991] + Synapse specific settings: * ``SYNAPSE_SERVER_NAME`` (mandatory), the current server public hostname. diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml index b8f9741f05..727743effa 100644 --- a/contrib/docker/docker-compose.yml +++ b/contrib/docker/docker-compose.yml @@ -1,4 +1,4 @@ -# This compose file is compatible with Copose itself, it might need some +# This compose file is compatible with Compose itself, it might need some # adjustments to run properly with stack. version: '3' @@ -18,10 +18,11 @@ services: - ./files:/data depends_on: - db - # One may either expose ports directly + # In order to expose Synapse, remove one of the following, you might for + # instance expose the TLS port directly: ports: - 8448:8448/tcp - # ... or use a reverse proxy, here is an example for traefik + # ... or use a reverse proxy, here is an example for traefik: labels: - traefik.enable=true - traefik.frontend.rule=Host:my.matrix.Host From e9021e16c49c4224782040449b43fb0015c9f05c Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sun, 4 Feb 2018 23:19:08 +0100 Subject: [PATCH 16/84] Run the server as an unprivileged user --- Dockerfile | 2 +- contrib/docker/start.py | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 25f3746303..f687a4f2bb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM python:2-alpine -RUN apk add --no-cache --virtual .nacl_deps build-base libffi-dev zlib-dev openssl-dev libjpeg-turbo-dev linux-headers postgresql-dev +RUN apk add --no-cache --virtual .nacl_deps su-exec build-base libffi-dev zlib-dev openssl-dev libjpeg-turbo-dev linux-headers postgresql-dev COPY . /synapse diff --git a/contrib/docker/start.py b/contrib/docker/start.py index 7057f85f61..8bc72bf428 100755 --- a/contrib/docker/start.py +++ b/contrib/docker/start.py @@ -9,14 +9,16 @@ convert = lambda src, dst, environ: open(dst, "w").write(jinja2.Template(open(sr mode = sys.argv[1] if len(sys.argv) > 1 else None environ = os.environ.copy() +# Check mandatory parameters and build the base start arguments if "SYNAPSE_SERVER_NAME" not in environ: print("Environment variable SYNAPSE_SERVER_NAME is mandatory, exiting.") sys.exit(2) +permissions = "{}:{}".format(environ.get("UID", 991), environ.get("GID", 991)) args = ["python", "-m", "synapse.app.homeserver", - "--server-name", os.environ.get("SYNAPSE_SERVER_NAME"), - "--report-stats", os.environ.get("SYNAPSE_REPORT_STATS", "no"), - "--config-path", os.environ.get("SYNAPSE_CONFIG_PATH", "/compiled/homeserver.yaml")] + "--server-name", environ.get("SYNAPSE_SERVER_NAME"), + "--report-stats", environ.get("SYNAPSE_REPORT_STATS", "no"), + "--config-path", environ.get("SYNAPSE_CONFIG_PATH", "/compiled/homeserver.yaml")] # Generate any missing shared secret for secret in ("SYNAPSE_REGISTRATION_SHARED_SECRET", "SYNAPSE_MACAROON_SECRET_KEY"): @@ -35,4 +37,5 @@ if mode == "generate": # In normal mode, generate missing keys if any, then run synapse else: subprocess.check_output(args + ["--generate-keys"]) - os.execv("/usr/local/bin/python", args) + subprocess.check_output(["chown", "-R", permissions, "/data"]) + os.execv("/sbin/su-exec", ["su-exec", permissions] + args) From 8db84e9b2198ad4a80f6e9e66e2fc8c2d44fbeca Mon Sep 17 00:00:00 2001 From: kaiyou Date: Mon, 5 Feb 2018 20:08:35 +0100 Subject: [PATCH 17/84] Remove docker related files from the python manifest --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index 8c9a57a9ca..e2a6623a63 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -26,6 +26,7 @@ exclude jenkins.sh exclude jenkins*.sh exclude jenkins* exclude Dockerfile +exclude .dockerignore recursive-exclude jenkins *.sh prune .github From 81010a126e9bbcce018b104921ff5221a67f99cc Mon Sep 17 00:00:00 2001 From: kaiyou Date: Mon, 5 Feb 2018 21:28:15 +0100 Subject: [PATCH 18/84] Add dynamic recaptcha configuration in the Docker image --- contrib/docker/README.md | 4 ++++ contrib/docker/conf/homeserver.yaml | 7 +++++++ 2 files changed, 11 insertions(+) diff --git a/contrib/docker/README.md b/contrib/docker/README.md index 73e53e4306..f4bc78908d 100644 --- a/contrib/docker/README.md +++ b/contrib/docker/README.md @@ -86,6 +86,10 @@ Synapse specific settings: * ``SYNAPSE_EVENT_CACHE_SIZE``, the event cache size [default `10K`]. * ``SYNAPSE_REPORT_STATS``, set this variable to `yes` to enable anonymous statistics reporting back to the Matrix project which helps us to get funding. +* ``SYNAPSE_RECAPTCHA_PUBLIC_KEY``, set this variable to the recaptcha public + key in order to enable recaptcha upon registration +* ``SYNAPSE_RECAPTCHA_PRIVATE_KEY``, set this variable to the recaptcha private + key in order to enable recaptcha upon registration Shared secrets, these will be initialized to random values if not set: diff --git a/contrib/docker/conf/homeserver.yaml b/contrib/docker/conf/homeserver.yaml index 7450cc1228..6f1de24aad 100644 --- a/contrib/docker/conf/homeserver.yaml +++ b/contrib/docker/conf/homeserver.yaml @@ -112,10 +112,17 @@ max_spider_size: "10M" ## Captcha ## +{% if SYNAPSE_RECAPTCHA_PUBLIC_KEY %} +recaptcha_public_key: "{{ SYNAPSE_RECAPTCHA_PUBLIC_KEY }}" +recaptcha_private_key: "{{ SYNAPSE_RECAPTCHA_PRIVATE_KEY }}" +enable_registration_captcha: True +recaptcha_siteverify_api: "https://www.google.com/recaptcha/api/siteverify" +{% else %} recaptcha_public_key: "YOUR_PUBLIC_KEY" recaptcha_private_key: "YOUR_PRIVATE_KEY" enable_registration_captcha: False recaptcha_siteverify_api: "https://www.google.com/recaptcha/api/siteverify" +{% endif %} ## Turn ## From cd51931b62aef63dacf4d79cdfa5de56da4eeda6 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Mon, 5 Feb 2018 21:53:53 +0100 Subject: [PATCH 19/84] Add dynamic TURN configuration in the Docker image --- contrib/docker/README.md | 7 +++++-- contrib/docker/conf/homeserver.yaml | 9 +++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/contrib/docker/README.md b/contrib/docker/README.md index f4bc78908d..0da7b56628 100644 --- a/contrib/docker/README.md +++ b/contrib/docker/README.md @@ -87,9 +87,12 @@ Synapse specific settings: * ``SYNAPSE_REPORT_STATS``, set this variable to `yes` to enable anonymous statistics reporting back to the Matrix project which helps us to get funding. * ``SYNAPSE_RECAPTCHA_PUBLIC_KEY``, set this variable to the recaptcha public - key in order to enable recaptcha upon registration + key in order to enable recaptcha upon registration. * ``SYNAPSE_RECAPTCHA_PRIVATE_KEY``, set this variable to the recaptcha private - key in order to enable recaptcha upon registration + key in order to enable recaptcha upon registration. +* ``SYNAPSE_TURN_URIS``, set this variable to the coma-separated list of TURN + uris to enable TURN for this homeserver. +* ``SYNAPSE_TURN_SECRET``, set this to the TURN shared secret if required. Shared secrets, these will be initialized to random values if not set: diff --git a/contrib/docker/conf/homeserver.yaml b/contrib/docker/conf/homeserver.yaml index 6f1de24aad..6f8fb24e5f 100644 --- a/contrib/docker/conf/homeserver.yaml +++ b/contrib/docker/conf/homeserver.yaml @@ -126,10 +126,19 @@ recaptcha_siteverify_api: "https://www.google.com/recaptcha/api/siteverify" ## Turn ## +{% if SYNAPSE_TURN_URIS %} +turn_uris: +{% for uri in SYNAPSE_TURN_URIS.split(',') %} - {{ uri }} +{% endfor %} +turn_shared_secret: "{{ SYNAPSE_TURN_SECRET }}" +turn_user_lifetime: "1h" +turn_allow_guests: True +{% else %} turn_uris: [] turn_shared_secret: "YOUR_SHARED_SECRET" turn_user_lifetime: "1h" turn_allow_guests: True +{% endif %} ## Registration ## From cf4ef60e287a6a61844ab260606721db2b7ee0cd Mon Sep 17 00:00:00 2001 From: kaiyou Date: Mon, 5 Feb 2018 22:10:03 +0100 Subject: [PATCH 20/84] Document the cache factor environment variable for Docker --- contrib/docker/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/docker/README.md b/contrib/docker/README.md index 0da7b56628..8b1e0afee6 100644 --- a/contrib/docker/README.md +++ b/contrib/docker/README.md @@ -84,6 +84,7 @@ Synapse specific settings: the Synapse instance. * ``SYNAPSE_ALLOW_GUEST``, set this variable to allow guest joining this server. * ``SYNAPSE_EVENT_CACHE_SIZE``, the event cache size [default `10K`]. +* ``SYNAPSE_CACHE_FACTOR``, the cache factor [default `0.5`]. * ``SYNAPSE_REPORT_STATS``, set this variable to `yes` to enable anonymous statistics reporting back to the Matrix project which helps us to get funding. * ``SYNAPSE_RECAPTCHA_PUBLIC_KEY``, set this variable to the recaptcha public From d8c7da5dca907bf65293b6b967200141cad69410 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Mon, 5 Feb 2018 22:12:50 +0100 Subject: [PATCH 21/84] Fix a typo in the Docker README --- contrib/docker/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/docker/README.md b/contrib/docker/README.md index 8b1e0afee6..624dca747a 100644 --- a/contrib/docker/README.md +++ b/contrib/docker/README.md @@ -58,7 +58,7 @@ docker run \ The image expects a single volume, located at ``/data``, that will hold: * temporary files during uploads; -* uploaded media and thumbnais; +* uploaded media and thumbnails; * the SQLite database if you do not configure postgres. ## Environment From f5364b47ec4b67a552976b9dbb9594d448744e30 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Mon, 5 Feb 2018 22:14:40 +0100 Subject: [PATCH 22/84] Point to the 'latest' tag in the Docker documentation --- contrib/docker/README.md | 2 +- contrib/docker/docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/docker/README.md b/contrib/docker/README.md index 624dca747a..87354b9bc3 100644 --- a/contrib/docker/README.md +++ b/contrib/docker/README.md @@ -49,7 +49,7 @@ docker run \ --name synapse \ -v ${DATA_PATH}:/data \ -e SYNAPSE_SERVER_NAME=my.matrix.host \ - matrixdotorg/synapse:v0.22.1 + matrixdotorg/synapse:latest ``` diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml index 727743effa..3fb156db47 100644 --- a/contrib/docker/docker-compose.yml +++ b/contrib/docker/docker-compose.yml @@ -6,7 +6,7 @@ version: '3' services: synapse: - image: synapse + image: matrixdotorg/synapse:latest # Since snyapse does not retry to connect to the database, restart upon # failure restart: unless-stopped From 630573a9325f826e81f04650fd83e64d1c831035 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Mon, 5 Feb 2018 22:57:22 +0100 Subject: [PATCH 23/84] Do not copy documentation files to the Docker root folder --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index f687a4f2bb..2ae503ce79 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ RUN cd /synapse \ && pip install --upgrade pip setuptools psycopg2 \ && mkdir -p /synapse/cache \ && pip install -f /synapse/cache --upgrade --process-dependency-links . \ - && mv /synapse/contrib/docker/* / \ + && mv /synapse/contrib/docker/start.py /synapse/contrib/docker/conf / \ && rm -rf setup.py setup.cfg synapse VOLUME ["/data"] From ee3b160a2ad375223b4304184304605a35f1b406 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Mon, 5 Feb 2018 22:57:35 +0100 Subject: [PATCH 24/84] Only generate configuration files when necessary --- contrib/docker/start.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/contrib/docker/start.py b/contrib/docker/start.py index 8bc72bf428..d3364e4226 100755 --- a/contrib/docker/start.py +++ b/contrib/docker/start.py @@ -26,16 +26,18 @@ for secret in ("SYNAPSE_REGISTRATION_SHARED_SECRET", "SYNAPSE_MACAROON_SECRET_KE print("Generating a random secret for {}".format(secret)) environ[secret] = os.urandom(32).encode("hex") -# Parse the configuration file -if not os.path.exists("/compiled"): os.mkdir("/compiled") -convert("/conf/homeserver.yaml", "/compiled/homeserver.yaml", environ) -convert("/conf/log.config", "/compiled/%s.log.config" % environ.get("SYNAPSE_SERVER_NAME"), environ) - # In generate mode, generate a configuration, missing keys, then exit if mode == "generate": os.execv("/usr/local/bin/python", args + ["--generate-config"]) + # In normal mode, generate missing keys if any, then run synapse else: + # Parse the configuration file + if "SYNAPSE_CONFIG_PATH" not in environ: + if not os.path.exists("/compiled"): os.mkdir("/compiled") + convert("/conf/homeserver.yaml", "/compiled/homeserver.yaml", environ) + convert("/conf/log.config", "/compiled/%s.log.config" % environ.get("SYNAPSE_SERVER_NAME"), environ) + # Generate missing keys and start synapse subprocess.check_output(args + ["--generate-keys"]) subprocess.check_output(["chown", "-R", permissions, "/data"]) os.execv("/sbin/su-exec", ["su-exec", permissions] + args) From 107a5c94418de55d52a0232115d5c3c7efc6d285 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Mon, 5 Feb 2018 23:02:33 +0100 Subject: [PATCH 25/84] Add the non-tls port to the expose list --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 2ae503ce79..bec884ce9b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,6 +14,6 @@ RUN cd /synapse \ VOLUME ["/data"] -EXPOSE 8448 +EXPOSE 8008/tcp 8448/tcp ENTRYPOINT ["/start.py"] From 1ffd9cb93617fe9bb2367d575786c0ff222cd415 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Mon, 5 Feb 2018 23:13:27 +0100 Subject: [PATCH 26/84] Support loading application service files from /data/appservices/ --- contrib/docker/README.md | 7 ++++++- contrib/docker/conf/homeserver.yaml | 9 ++++++++- contrib/docker/start.py | 4 ++++ 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/contrib/docker/README.md b/contrib/docker/README.md index 87354b9bc3..c1724fe269 100644 --- a/contrib/docker/README.md +++ b/contrib/docker/README.md @@ -59,7 +59,12 @@ The image expects a single volume, located at ``/data``, that will hold: * temporary files during uploads; * uploaded media and thumbnails; -* the SQLite database if you do not configure postgres. +* the SQLite database if you do not configure postgres; +* the appservices configuration. + +In order to setup an application service, simply create an ``appservices`` +directory in the data volume and write the application service Yaml +configuration file there. Multiple application services are supported. ## Environment diff --git a/contrib/docker/conf/homeserver.yaml b/contrib/docker/conf/homeserver.yaml index 6f8fb24e5f..e5d3f965e4 100644 --- a/contrib/docker/conf/homeserver.yaml +++ b/contrib/docker/conf/homeserver.yaml @@ -128,7 +128,7 @@ recaptcha_siteverify_api: "https://www.google.com/recaptcha/api/siteverify" {% if SYNAPSE_TURN_URIS %} turn_uris: -{% for uri in SYNAPSE_TURN_URIS.split(',') %} - {{ uri }} +{% for uri in SYNAPSE_TURN_URIS.split(',') %} - "{{ uri }}" {% endfor %} turn_shared_secret: "{{ SYNAPSE_TURN_SECRET }}" turn_user_lifetime: "1h" @@ -167,7 +167,14 @@ room_invite_state_types: - "m.room.avatar" - "m.room.name" +{% if SYNAPSE_APPSERVICES %} +app_service_config_files: +{% for appservice in SYNAPSE_APPSERVICES %} - "{{ appservice }}" +{% endfor %} +{% else %} app_service_config_files: [] +{% endif %} + macaroon_secret_key: "{{ SYNAPSE_MACAROON_SECRET_KEY }}" expire_access_token: False diff --git a/contrib/docker/start.py b/contrib/docker/start.py index d3364e4226..8ade0f227d 100755 --- a/contrib/docker/start.py +++ b/contrib/docker/start.py @@ -4,6 +4,7 @@ import jinja2 import os import sys import subprocess +import glob convert = lambda src, dst, environ: open(dst, "w").write(jinja2.Template(open(src).read()).render(**environ)) mode = sys.argv[1] if len(sys.argv) > 1 else None @@ -26,6 +27,9 @@ for secret in ("SYNAPSE_REGISTRATION_SHARED_SECRET", "SYNAPSE_MACAROON_SECRET_KE print("Generating a random secret for {}".format(secret)) environ[secret] = os.urandom(32).encode("hex") +# Load appservices configurations +environ["SYNAPSE_APPSERVICES"] = glob.glob("/data/appservices/*.yaml") + # In generate mode, generate a configuration, missing keys, then exit if mode == "generate": os.execv("/usr/local/bin/python", args + ["--generate-config"]) From 63fd148724399d52f3435b1c172435d7cabcde4c Mon Sep 17 00:00:00 2001 From: kaiyou Date: Thu, 8 Feb 2018 19:46:11 +0100 Subject: [PATCH 27/84] Make it clear that two modes are avaiable in the documentation, improve the compose file --- contrib/docker/README.md | 17 ++++++++++------- contrib/docker/docker-compose.yml | 12 ++++++++++-- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/contrib/docker/README.md b/contrib/docker/README.md index c1724fe269..197bad103e 100644 --- a/contrib/docker/README.md +++ b/contrib/docker/README.md @@ -12,7 +12,7 @@ use that server. Build the docker image with the `docker build` command from the root of the synapse repository. ``` -docker build -t matrixdotorg/synapse:v0.22.1 . +docker build -t matrixdotorg/synapse . ``` The `-t` option sets the image tag. Official images are tagged `matrixdotorg/synapse:` where `` is the same as the release tag in the synapse git repository. @@ -76,12 +76,17 @@ Global settings: * ``UID``, the user id Synapse will run as [default 991] * ``GID``, the group id Synapse will run as [default 991] +* ``SYNAPSE_CONFIG_PATH``, path to a custom config file -Synapse specific settings: +If ``SYNAPSE_CONFIG_PATH`` is set, you should generate a configuration file +then customize it manually. No other environment variable is required. + +Otherwise, a dynamic configuration file will be used. The following environment +variables are available for configuration: * ``SYNAPSE_SERVER_NAME`` (mandatory), the current server public hostname. -* ``SYNAPSE_CONFIG_PATH``, path to a custom config file (will ignore all - other options then). +* ``SYNAPSE_REPORT_STATS``, (mandatory, ``yes`` or ``not``), enable anonymous + statistics reporting back to the Matrix project which helps us to get funding. * ``SYNAPSE_NO_TLS``, set this variable to disable TLS in Synapse (use this if you run your own TLS-capable reverse proxy). * ``SYNAPSE_WEB_CLIENT``, set this variable to enable the embedded Web client. @@ -90,8 +95,6 @@ Synapse specific settings: * ``SYNAPSE_ALLOW_GUEST``, set this variable to allow guest joining this server. * ``SYNAPSE_EVENT_CACHE_SIZE``, the event cache size [default `10K`]. * ``SYNAPSE_CACHE_FACTOR``, the cache factor [default `0.5`]. -* ``SYNAPSE_REPORT_STATS``, set this variable to `yes` to enable anonymous - statistics reporting back to the Matrix project which helps us to get funding. * ``SYNAPSE_RECAPTCHA_PUBLIC_KEY``, set this variable to the recaptcha public key in order to enable recaptcha upon registration. * ``SYNAPSE_RECAPTCHA_PRIVATE_KEY``, set this variable to the recaptcha private @@ -100,7 +103,7 @@ Synapse specific settings: uris to enable TURN for this homeserver. * ``SYNAPSE_TURN_SECRET``, set this to the TURN shared secret if required. -Shared secrets, these will be initialized to random values if not set: +Shared secrets, that will be initialized to random values if not set: * ``SYNAPSE_REGISTRATION_SHARED_SECRET``, secret for registrering users if registration is disable. diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml index 3fb156db47..b07984ea34 100644 --- a/contrib/docker/docker-compose.yml +++ b/contrib/docker/docker-compose.yml @@ -6,7 +6,7 @@ version: '3' services: synapse: - image: matrixdotorg/synapse:latest + image: docker.io/matrixdotorg/synapse:latest # Since snyapse does not retry to connect to the database, restart upon # failure restart: unless-stopped @@ -15,7 +15,12 @@ services: - SYNAPSE_SERVER_NAME=my.matrix.host - SYNAPSE_ENABLE_REGISTRATION=yes volumes: + # You may either store all the files in a local folder - ./files:/data + # .. or you may split this between different storage points + # - ./files:/data + # - /path/to/ssd:/data/uploads + # - /path/to/large_hdd:/data/media depends_on: - db # In order to expose Synapse, remove one of the following, you might for @@ -29,10 +34,13 @@ services: - traefik.port=8448 db: - image: postgres:latest + image: postgres:10-alpine # Change that password, of course! environment: - POSTGRES_USER=matrix - POSTGRES_PASSWORD=changeme volumes: + # You may store the database tables in a local folder.. - ./schemas:/var/lib/postgresql/data + # .. or store them on some high performance storage for better results + # - /path/to/ssd/storage:/var/lib/postfesql/data From 58df3a8c5dcf5de12bcf9190551dc8241d32b8a1 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Thu, 8 Feb 2018 19:48:53 +0100 Subject: [PATCH 28/84] Add some documentation about high performance storage --- contrib/docker/README.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/contrib/docker/README.md b/contrib/docker/README.md index 197bad103e..3710afb0cf 100644 --- a/contrib/docker/README.md +++ b/contrib/docker/README.md @@ -31,11 +31,11 @@ Then, to run the server: docker-compose up -d ``` -In the case you specified a custom path for you configuration file and wish to +In the case you specify a custom path for you configuration file and wish to generate a fresh ``homeserver.yaml``, simply run: ``` -docker-compose run --rm synapse generate +docker-compose run --rm -e SYNAPSE_SERVER_NAME=my.matrix.host synapse generate ``` If you do not wish to use Compose, you may still run this image using plain @@ -62,6 +62,11 @@ The image expects a single volume, located at ``/data``, that will hold: * the SQLite database if you do not configure postgres; * the appservices configuration. +You are free to use separate volumes depending on storage endpoints at your +disposal. For instance, ``/data/media`` coud be stored on a large but low +performance hdd storage while other files could be stored on high performance +endpoints. + In order to setup an application service, simply create an ``appservices`` directory in the data volume and write the application service Yaml configuration file there. Multiple application services are supported. From 084afbb6a06f2661bed503bf49b0291ce999c6c1 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Thu, 8 Feb 2018 19:50:04 +0100 Subject: [PATCH 29/84] Rename the permissions variable to avoid confusion --- contrib/docker/start.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/contrib/docker/start.py b/contrib/docker/start.py index 8ade0f227d..13a10a11bb 100755 --- a/contrib/docker/start.py +++ b/contrib/docker/start.py @@ -15,7 +15,7 @@ if "SYNAPSE_SERVER_NAME" not in environ: print("Environment variable SYNAPSE_SERVER_NAME is mandatory, exiting.") sys.exit(2) -permissions = "{}:{}".format(environ.get("UID", 991), environ.get("GID", 991)) +ownership = "{}:{}".format(environ.get("UID", 991), environ.get("GID", 991)) args = ["python", "-m", "synapse.app.homeserver", "--server-name", environ.get("SYNAPSE_SERVER_NAME"), "--report-stats", environ.get("SYNAPSE_REPORT_STATS", "no"), @@ -43,5 +43,5 @@ else: convert("/conf/log.config", "/compiled/%s.log.config" % environ.get("SYNAPSE_SERVER_NAME"), environ) # Generate missing keys and start synapse subprocess.check_output(args + ["--generate-keys"]) - subprocess.check_output(["chown", "-R", permissions, "/data"]) - os.execv("/sbin/su-exec", ["su-exec", permissions] + args) + subprocess.check_output(["chown", "-R", ownership, "/data"]) + os.execv("/sbin/su-exec", ["su-exec", ownership] + args) From b8a4dceb3cee6b69d1b1b882cef1f96a3ff6249f Mon Sep 17 00:00:00 2001 From: kaiyou Date: Thu, 8 Feb 2018 20:41:41 +0100 Subject: [PATCH 30/84] Refactor the start script to better handle mandatory parameters --- contrib/docker/README.md | 2 +- contrib/docker/docker-compose.yml | 1 + contrib/docker/start.py | 56 ++++++++++++++++++------------- 3 files changed, 35 insertions(+), 24 deletions(-) diff --git a/contrib/docker/README.md b/contrib/docker/README.md index 3710afb0cf..0493d2ee6e 100644 --- a/contrib/docker/README.md +++ b/contrib/docker/README.md @@ -90,7 +90,7 @@ Otherwise, a dynamic configuration file will be used. The following environment variables are available for configuration: * ``SYNAPSE_SERVER_NAME`` (mandatory), the current server public hostname. -* ``SYNAPSE_REPORT_STATS``, (mandatory, ``yes`` or ``not``), enable anonymous +* ``SYNAPSE_REPORT_STATS``, (mandatory, ``yes`` or ``no``), enable anonymous statistics reporting back to the Matrix project which helps us to get funding. * ``SYNAPSE_NO_TLS``, set this variable to disable TLS in Synapse (use this if you run your own TLS-capable reverse proxy). diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml index b07984ea34..3d0b3c0ea4 100644 --- a/contrib/docker/docker-compose.yml +++ b/contrib/docker/docker-compose.yml @@ -13,6 +13,7 @@ services: # See the readme for a full documentation of the environment settings environment: - SYNAPSE_SERVER_NAME=my.matrix.host + - SYNAPSE_REPORT_STATS=no - SYNAPSE_ENABLE_REGISTRATION=yes volumes: # You may either store all the files in a local folder diff --git a/contrib/docker/start.py b/contrib/docker/start.py index 13a10a11bb..32142bbe00 100755 --- a/contrib/docker/start.py +++ b/contrib/docker/start.py @@ -6,42 +6,52 @@ import sys import subprocess import glob +# Utility functions convert = lambda src, dst, environ: open(dst, "w").write(jinja2.Template(open(src).read()).render(**environ)) + +def check_arguments(environ, args): + for argument in args: + if argument not in environ: + print("Environment variable %s is mandatory, exiting." % argument) + sys.exit(2) + +def generate_secrets(environ, secrets): + for secret in secrets: + if secret not in environ: + print("Generating a random secret for {}".format(secret)) + environ[secret] = os.urandom(32).encode("hex") + +# Prepare the configuration mode = sys.argv[1] if len(sys.argv) > 1 else None environ = os.environ.copy() - -# Check mandatory parameters and build the base start arguments -if "SYNAPSE_SERVER_NAME" not in environ: - print("Environment variable SYNAPSE_SERVER_NAME is mandatory, exiting.") - sys.exit(2) - ownership = "{}:{}".format(environ.get("UID", 991), environ.get("GID", 991)) -args = ["python", "-m", "synapse.app.homeserver", - "--server-name", environ.get("SYNAPSE_SERVER_NAME"), - "--report-stats", environ.get("SYNAPSE_REPORT_STATS", "no"), - "--config-path", environ.get("SYNAPSE_CONFIG_PATH", "/compiled/homeserver.yaml")] - -# Generate any missing shared secret -for secret in ("SYNAPSE_REGISTRATION_SHARED_SECRET", "SYNAPSE_MACAROON_SECRET_KEY"): - if secret not in environ: - print("Generating a random secret for {}".format(secret)) - environ[secret] = os.urandom(32).encode("hex") - -# Load appservices configurations -environ["SYNAPSE_APPSERVICES"] = glob.glob("/data/appservices/*.yaml") +args = ["python", "-m", "synapse.app.homeserver"] # In generate mode, generate a configuration, missing keys, then exit if mode == "generate": - os.execv("/usr/local/bin/python", args + ["--generate-config"]) + check_arguments(environ, ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS", "SYNAPSE_CONFIG_PATH")) + args += [ + "--server-name", environ["SYNAPSE_SERVER_NAME"], + "--report-stats", environ["SYNAPSE_REPORT_STATS"], + "--config-path", environ["SYNAPSE_CONFIG_PATH"], + "--generate-config" + ] + os.execv("/usr/local/bin/python", args) # In normal mode, generate missing keys if any, then run synapse else: # Parse the configuration file - if "SYNAPSE_CONFIG_PATH" not in environ: + if "SYNAPSE_CONFIG_PATH" in environ: + args += ["--config-path", environ["SYNAPSE_CONFIG_PATH"]] + else: + check_arguments(environ, ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS")) + generate_secrets(environ, ("SYNAPSE_REGISTRATION_SHARED_SECRET", "SYNAPSE_MACAROON_SECRET_KEY")) + environ["SYNAPSE_APPSERVICES"] = glob.glob("/data/appservices/*.yaml") if not os.path.exists("/compiled"): os.mkdir("/compiled") convert("/conf/homeserver.yaml", "/compiled/homeserver.yaml", environ) - convert("/conf/log.config", "/compiled/%s.log.config" % environ.get("SYNAPSE_SERVER_NAME"), environ) + convert("/conf/log.config", "/compiled/%s.log.config" % environ["SYNAPSE_SERVER_NAME"], environ) + subprocess.check_output(["chown", "-R", ownership, "/data"]) + args += ["--config-path", "/compiled/homeserver.yaml"] # Generate missing keys and start synapse subprocess.check_output(args + ["--generate-keys"]) - subprocess.check_output(["chown", "-R", ownership, "/data"]) os.execv("/sbin/su-exec", ["su-exec", ownership] + args) From e174c46a295ca6e06b217b5dcbcf995b890e6d07 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Thu, 8 Feb 2018 20:42:57 +0100 Subject: [PATCH 31/84] Use 'synapse' as a default postgres user in Docker examples --- contrib/docker/conf/homeserver.yaml | 4 ++-- contrib/docker/docker-compose.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/contrib/docker/conf/homeserver.yaml b/contrib/docker/conf/homeserver.yaml index e5d3f965e4..1ca1fe991f 100644 --- a/contrib/docker/conf/homeserver.yaml +++ b/contrib/docker/conf/homeserver.yaml @@ -50,9 +50,9 @@ listeners: database: name: "psycopg2" args: - user: "{{ POSTGRES_USER or "matrix" }}" + user: "{{ POSTGRES_USER or "synapse" }}" password: "{{ POSTGRES_PASSWORD }}" - database: "{{ POSTGRES_DB or "matrix" }}" + database: "{{ POSTGRES_DB or "synapse" }}" host: "{{ POSTGRES_HOST or "db" }}" port: "{{ POSTGRES_PORT or "5432" }}" cp_min: 5 diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml index 3d0b3c0ea4..e447bf1212 100644 --- a/contrib/docker/docker-compose.yml +++ b/contrib/docker/docker-compose.yml @@ -38,7 +38,7 @@ services: image: postgres:10-alpine # Change that password, of course! environment: - - POSTGRES_USER=matrix + - POSTGRES_USER=synapse - POSTGRES_PASSWORD=changeme volumes: # You may store the database tables in a local folder.. From 914a59cb8c12b25d77ed9a81e4543c23d7e10b5e Mon Sep 17 00:00:00 2001 From: kaiyou Date: Thu, 8 Feb 2018 20:43:45 +0100 Subject: [PATCH 32/84] Disable the Web client in the Docker image --- contrib/docker/README.md | 1 - contrib/docker/conf/homeserver.yaml | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/contrib/docker/README.md b/contrib/docker/README.md index 0493d2ee6e..9f40dc0d58 100644 --- a/contrib/docker/README.md +++ b/contrib/docker/README.md @@ -94,7 +94,6 @@ variables are available for configuration: statistics reporting back to the Matrix project which helps us to get funding. * ``SYNAPSE_NO_TLS``, set this variable to disable TLS in Synapse (use this if you run your own TLS-capable reverse proxy). -* ``SYNAPSE_WEB_CLIENT``, set this variable to enable the embedded Web client. * ``SYNAPSE_ENABLE_REGISTRATION``, set this variable to enable registration on the Synapse instance. * ``SYNAPSE_ALLOW_GUEST``, set this variable to allow guest joining this server. diff --git a/contrib/docker/conf/homeserver.yaml b/contrib/docker/conf/homeserver.yaml index 1ca1fe991f..19a2cbad29 100644 --- a/contrib/docker/conf/homeserver.yaml +++ b/contrib/docker/conf/homeserver.yaml @@ -12,7 +12,7 @@ tls_fingerprints: [] server_name: "{{ SYNAPSE_SERVER_NAME }}" pid_file: /homeserver.pid -web_client: {{ "True" if SYNAPSE_WEB_CLIENT else "False" }} +web_client: False soft_file_limit: 0 ## Ports ## @@ -26,7 +26,7 @@ listeners: tls: true x_forwarded: false resources: - - names: [client, webclient] + - names: [client] compress: true - names: [federation] # Federation APIs compress: false @@ -39,7 +39,7 @@ listeners: x_forwarded: false resources: - - names: [client, webclient] + - names: [client] compress: true - names: [federation] compress: false From a0af0054ec91e92a6843c121cd27e92ea63c1034 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Thu, 8 Feb 2018 20:46:11 +0100 Subject: [PATCH 33/84] Honor the SYNAPSE_REPORT_STATS parameter in the Docker image --- contrib/docker/conf/homeserver.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/contrib/docker/conf/homeserver.yaml b/contrib/docker/conf/homeserver.yaml index 19a2cbad29..3b57f7174d 100644 --- a/contrib/docker/conf/homeserver.yaml +++ b/contrib/docker/conf/homeserver.yaml @@ -156,8 +156,13 @@ trusted_third_party_id_servers: ## Metrics ### +{% if SYNAPSE_REPORT_STATS.lower() == "yes" %} +enable_metrics: True +report_stats: True +{% else %} enable_metrics: False report_stats: False +{% endif %} ## API Configuration ## From ef1f8d4be6e970043b8283f5caa1ca764652ad56 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Thu, 8 Feb 2018 20:53:12 +0100 Subject: [PATCH 34/84] Enable email server configuration from environment variables --- contrib/docker/README.md | 7 ++++++ contrib/docker/conf/homeserver.yaml | 33 +++++++++++++++-------------- 2 files changed, 24 insertions(+), 16 deletions(-) diff --git a/contrib/docker/README.md b/contrib/docker/README.md index 9f40dc0d58..b74c72698c 100644 --- a/contrib/docker/README.md +++ b/contrib/docker/README.md @@ -119,3 +119,10 @@ Database specific values (will use SQLite if not set): * `POSTGRES_HOST` - The host of the postgres database if you wish to use postgresql instead of sqlite3. [default: `db` which is useful when using a container on the same docker network in a compose file where the postgres service is called `db`] * `POSTGRES_PASSWORD` - The password for the synapse postgres database. **If this is set then postgres will be used instead of sqlite3.** [default: none] **NOTE**: You are highly encouraged to use postgresql! Please use the compose file to make it easier to deploy. * `POSTGRES_USER` - The user for the synapse postgres database. [default: `matrix`] + +Mail server specific values (will not send emails if not set): + +* ``SYNAPSE_SMTP_HOST``, hostname to the mail server. +* ``SYNAPSE_SMTP_PORT``, TCP port for accessing the mail server [default ``25``]. +* ``SYNAPSE_SMTP_USER``, username for authenticating against the mail server if any. +* ``SYNAPSE_SMTP_PASSWORD``, password for authenticating against the mail server if any. diff --git a/contrib/docker/conf/homeserver.yaml b/contrib/docker/conf/homeserver.yaml index 3b57f7174d..198b8ddee7 100644 --- a/contrib/docker/conf/homeserver.yaml +++ b/contrib/docker/conf/homeserver.yaml @@ -146,6 +146,7 @@ enable_registration: {{ "True" if SYNAPSE_ENABLE_REGISTRATION else "False" }} registration_shared_secret: "{{ SYNAPSE_REGISTRATION_SHARED_SECRET }}" bcrypt_rounds: 12 allow_guest_access: {{ "True" if SYNAPSE_ALLOW_GUEST else "False" }} +enable_group_creation: true # The list of identity servers trusted to verify third party # identifiers by this server. @@ -200,19 +201,19 @@ perspectives: password_config: enabled: true -#email: -# enable_notifs: false -# smtp_host: "localhost" -# smtp_port: 25 -# smtp_user: "exampleusername" -# smtp_pass: "examplepassword" -# require_transport_security: False -# notif_from: "Your Friendly %(app)s Home Server " -# app_name: Matrix -# template_dir: res/templates -# notif_template_html: notif_mail.html -# notif_template_text: notif_mail.txt -# notif_for_new_users: True -# riot_base_url: "http://localhost/riot" - -enable_group_creation: true +{% if SYNAPSE_SMTP_HOST %} +email: + enable_notifs: false + smtp_host: "{{ SYNAPSE_SMTP_HOST }}" + smtp_port: {{ SYNAPSE_SMTP_PORT or "25" }} + smtp_user: "{{ SYNAPSE_SMTP_USER }}" + smtp_pass: "{{ SYNAPSE_SMTP_PASSWORD }}" + require_transport_security: False + notif_from: "{{ SYNAPSE_SMTP_FROM or "hostmaster@" + SYNAPSE_SERVER_NAME }}" + app_name: Matrix + template_dir: res/templates + notif_template_html: notif_mail.html + notif_template_text: notif_mail.txt + notif_for_new_users: True + riot_base_url: "https://{{ SYNAPSE_SERVER_NAME }}" +{% endif %} From b9b668e4bb6ec3c0ae6c696a4b7abf6f537e0504 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Thu, 8 Feb 2018 21:39:36 +0100 Subject: [PATCH 35/84] Update to Alpine 3.7 and switch to libressl --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index bec884ce9b..8085f3d354 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ -FROM python:2-alpine +FROM docker.io/python:2-alpine3.7 -RUN apk add --no-cache --virtual .nacl_deps su-exec build-base libffi-dev zlib-dev openssl-dev libjpeg-turbo-dev linux-headers postgresql-dev +RUN apk add --no-cache --virtual .nacl_deps su-exec build-base libffi-dev zlib-dev libressl-dev libjpeg-turbo-dev linux-headers postgresql-dev COPY . /synapse From d8680c969bb2e8248436d7352ebf0f5cd1daea7b Mon Sep 17 00:00:00 2001 From: kaiyou Date: Thu, 8 Feb 2018 21:55:35 +0100 Subject: [PATCH 36/84] Make it clear that the image has two modes of operation --- contrib/docker/README.md | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/contrib/docker/README.md b/contrib/docker/README.md index b74c72698c..27e25afcef 100644 --- a/contrib/docker/README.md +++ b/contrib/docker/README.md @@ -21,23 +21,42 @@ You may have a local Python wheel cache available, in which case copy the releva ## Run +This image is designed to run either with an automatically generated configuration +file or with a custom configuration that requires manual edition. + +### Automated configuration + It is recommended that you use Docker Compose to run your containers, including this image and a Postgres server. A sample ``docker-compose.yml`` is provided, including example labels for reverse proxying and other artifacts. -Then, to run the server: +Read the section about environment variables and set at least mandatory variables, +then run the server: ``` docker-compose up -d ``` -In the case you specify a custom path for you configuration file and wish to -generate a fresh ``homeserver.yaml``, simply run: +### Manual configuration + +A sample ``docker-compose.yml`` is provided, including example labels for +reverse proxying and other artifacts. + +Specify a ``SYNAPSE_CONFIG_PATH``, preferably to a persistent path, +to use manual configuration. To generate a fresh ``homeserver.yaml``, simply run: ``` docker-compose run --rm -e SYNAPSE_SERVER_NAME=my.matrix.host synapse generate ``` +Then, customize your configuration and run the server: + +``` +docker-compose up -d +``` + +### Without Compose + If you do not wish to use Compose, you may still run this image using plain Docker commands. Note that the following is just a guideline and you may need to add parameters to the docker run command to account for the network situation @@ -49,10 +68,10 @@ docker run \ --name synapse \ -v ${DATA_PATH}:/data \ -e SYNAPSE_SERVER_NAME=my.matrix.host \ - matrixdotorg/synapse:latest + -e SYNAPSE_REPORT_STATS=yes \ + docker.io/matrixdotorg/synapse:latest ``` - ## Volumes The image expects a single volume, located at ``/data``, that will hold: From 48e2c641b80ac57d16701e4a27f348ea4f2c66cc Mon Sep 17 00:00:00 2001 From: kaiyou Date: Thu, 8 Feb 2018 21:58:12 +0100 Subject: [PATCH 37/84] Specify the Docker registry in the build tag --- contrib/docker/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/docker/README.md b/contrib/docker/README.md index 27e25afcef..221d9c53b5 100644 --- a/contrib/docker/README.md +++ b/contrib/docker/README.md @@ -12,7 +12,7 @@ use that server. Build the docker image with the `docker build` command from the root of the synapse repository. ``` -docker build -t matrixdotorg/synapse . +docker build -t docker.io/matrixdotorg/synapse . ``` The `-t` option sets the image tag. Official images are tagged `matrixdotorg/synapse:` where `` is the same as the release tag in the synapse git repository. From a03c382966a3219acd26851db9cc6558e5c53310 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Thu, 8 Feb 2018 22:00:43 +0100 Subject: [PATCH 38/84] Specify the Docker registry for the postgres image --- contrib/docker/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml index e447bf1212..1d2aebbcd3 100644 --- a/contrib/docker/docker-compose.yml +++ b/contrib/docker/docker-compose.yml @@ -35,7 +35,7 @@ services: - traefik.port=8448 db: - image: postgres:10-alpine + image: docker.io/postgres:10-alpine # Change that password, of course! environment: - POSTGRES_USER=synapse From e511979fe6c4a03da3e9c1d16672e263f54ee2d3 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Fri, 9 Feb 2018 00:13:26 +0100 Subject: [PATCH 39/84] Make SYNAPSE_MACAROON_SECRET_KEY a mandatory option --- contrib/docker/README.md | 3 ++- contrib/docker/start.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/contrib/docker/README.md b/contrib/docker/README.md index 221d9c53b5..25c358c847 100644 --- a/contrib/docker/README.md +++ b/contrib/docker/README.md @@ -111,6 +111,8 @@ variables are available for configuration: * ``SYNAPSE_SERVER_NAME`` (mandatory), the current server public hostname. * ``SYNAPSE_REPORT_STATS``, (mandatory, ``yes`` or ``no``), enable anonymous statistics reporting back to the Matrix project which helps us to get funding. +* ``SYNAPSE_MACAROON_SECRET_KEY`` (mandatory) secret for signing access tokens + to the server, set this to a proper random key. * ``SYNAPSE_NO_TLS``, set this variable to disable TLS in Synapse (use this if you run your own TLS-capable reverse proxy). * ``SYNAPSE_ENABLE_REGISTRATION``, set this variable to enable registration on @@ -130,7 +132,6 @@ Shared secrets, that will be initialized to random values if not set: * ``SYNAPSE_REGISTRATION_SHARED_SECRET``, secret for registrering users if registration is disable. -* ``SYNAPSE_MACAROON_SECRET_KEY``, secret for Macaroon. Database specific values (will use SQLite if not set): diff --git a/contrib/docker/start.py b/contrib/docker/start.py index 32142bbe00..d4c1140b1d 100755 --- a/contrib/docker/start.py +++ b/contrib/docker/start.py @@ -44,8 +44,8 @@ else: if "SYNAPSE_CONFIG_PATH" in environ: args += ["--config-path", environ["SYNAPSE_CONFIG_PATH"]] else: - check_arguments(environ, ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS")) - generate_secrets(environ, ("SYNAPSE_REGISTRATION_SHARED_SECRET", "SYNAPSE_MACAROON_SECRET_KEY")) + check_arguments(environ, ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS", "SYNAPSE_MACAROON_SECRET_KEY")) + generate_secrets(environ, ("SYNAPSE_REGISTRATION_SHARED_SECRET",)) environ["SYNAPSE_APPSERVICES"] = glob.glob("/data/appservices/*.yaml") if not os.path.exists("/compiled"): os.mkdir("/compiled") convert("/conf/homeserver.yaml", "/compiled/homeserver.yaml", environ) From ca70148c0569295a2b9ecdd1cd9cd85a203f20e7 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Fri, 9 Feb 2018 00:23:19 +0100 Subject: [PATCH 40/84] Fix the path to the log config file --- contrib/docker/conf/homeserver.yaml | 2 +- contrib/docker/start.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/docker/conf/homeserver.yaml b/contrib/docker/conf/homeserver.yaml index 198b8ddee7..6bc25bb45f 100644 --- a/contrib/docker/conf/homeserver.yaml +++ b/contrib/docker/conf/homeserver.yaml @@ -69,7 +69,7 @@ database: event_cache_size: "{{ SYNAPSE_EVENT_CACHE_SIZE or "10K" }}" verbose: 0 log_file: "/data/homeserver.log" -log_config: "/data/{{ SYNAPSE_SERVER_NAME }}.log.config" +log_config: "/compiled/log.config" ## Ratelimiting ## diff --git a/contrib/docker/start.py b/contrib/docker/start.py index d4c1140b1d..75c30b8ac0 100755 --- a/contrib/docker/start.py +++ b/contrib/docker/start.py @@ -49,7 +49,7 @@ else: environ["SYNAPSE_APPSERVICES"] = glob.glob("/data/appservices/*.yaml") if not os.path.exists("/compiled"): os.mkdir("/compiled") convert("/conf/homeserver.yaml", "/compiled/homeserver.yaml", environ) - convert("/conf/log.config", "/compiled/%s.log.config" % environ["SYNAPSE_SERVER_NAME"], environ) + convert("/conf/log.config", "/compiled/log.config", environ) subprocess.check_output(["chown", "-R", ownership, "/data"]) args += ["--config-path", "/compiled/homeserver.yaml"] # Generate missing keys and start synapse From 6f0b1f85f9f34401219eab4b4977a63c698ce987 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sat, 10 Feb 2018 00:05:03 +0100 Subject: [PATCH 41/84] Generate macaroon and registration secrets, then store the results to the data dir --- contrib/docker/docker-compose.yml | 2 +- contrib/docker/start.py | 19 ++++++++++++++----- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml index 1d2aebbcd3..9e32dd87de 100644 --- a/contrib/docker/docker-compose.yml +++ b/contrib/docker/docker-compose.yml @@ -6,7 +6,7 @@ version: '3' services: synapse: - image: docker.io/matrixdotorg/synapse:latest + image: synapse #docker.io/matrixdotorg/synapse:latest # Since snyapse does not retry to connect to the database, restart upon # failure restart: unless-stopped diff --git a/contrib/docker/start.py b/contrib/docker/start.py index 75c30b8ac0..90e8b9c51a 100755 --- a/contrib/docker/start.py +++ b/contrib/docker/start.py @@ -16,10 +16,16 @@ def check_arguments(environ, args): sys.exit(2) def generate_secrets(environ, secrets): - for secret in secrets: + for name, secret in secrets.items(): if secret not in environ: - print("Generating a random secret for {}".format(secret)) - environ[secret] = os.urandom(32).encode("hex") + filename = "/data/%s.%s.key" % (environ["SYNAPSE_SERVER_NAME"], name) + if os.path.exists(filename): + with open(filename) as handle: value = handle.read() + else: + print("Generating a random secret for {}".format(name)) + value = os.urandom(32).encode("hex") + with open(filename, "w") as handle: handle.write(value) + environ[secret] = value # Prepare the configuration mode = sys.argv[1] if len(sys.argv) > 1 else None @@ -44,8 +50,11 @@ else: if "SYNAPSE_CONFIG_PATH" in environ: args += ["--config-path", environ["SYNAPSE_CONFIG_PATH"]] else: - check_arguments(environ, ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS", "SYNAPSE_MACAROON_SECRET_KEY")) - generate_secrets(environ, ("SYNAPSE_REGISTRATION_SHARED_SECRET",)) + check_arguments(environ, ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS")) + generate_secrets(environ, { + "registration": "SYNAPSE_REGISTRATION_SHARED_SECRET", + "macaroon": "SYNAPSE_MACAROON_SECRET_KEY" + }) environ["SYNAPSE_APPSERVICES"] = glob.glob("/data/appservices/*.yaml") if not os.path.exists("/compiled"): os.mkdir("/compiled") convert("/conf/homeserver.yaml", "/compiled/homeserver.yaml", environ) From b815aa0e2db8f50116b1443b559ca13fe6ad1750 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sat, 10 Feb 2018 21:59:58 +0100 Subject: [PATCH 42/84] Remove an accidentally committed test configuration --- contrib/docker/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml index 9e32dd87de..1d2aebbcd3 100644 --- a/contrib/docker/docker-compose.yml +++ b/contrib/docker/docker-compose.yml @@ -6,7 +6,7 @@ version: '3' services: synapse: - image: synapse #docker.io/matrixdotorg/synapse:latest + image: docker.io/matrixdotorg/synapse:latest # Since snyapse does not retry to connect to the database, restart upon # failure restart: unless-stopped From 07f1b7181997dca91b67dca7561ce4c532caf253 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sat, 10 Feb 2018 23:57:36 +0100 Subject: [PATCH 43/84] Explicitely provide the postgres password to synapse in the Compose example --- contrib/docker/docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml index 1d2aebbcd3..46e72601d3 100644 --- a/contrib/docker/docker-compose.yml +++ b/contrib/docker/docker-compose.yml @@ -15,6 +15,7 @@ services: - SYNAPSE_SERVER_NAME=my.matrix.host - SYNAPSE_REPORT_STATS=no - SYNAPSE_ENABLE_REGISTRATION=yes + - POSTGRES_PASSWORD=changeme volumes: # You may either store all the files in a local folder - ./files:/data From f44b7c022f6bc8b30cb8c446e0922b26b8b8eb5a Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sat, 10 Feb 2018 23:57:51 +0100 Subject: [PATCH 44/84] Disable logging to file and rely on the console when using Docker --- contrib/docker/conf/log.config | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/contrib/docker/conf/log.config b/contrib/docker/conf/log.config index 45e7eef953..b5c907c4f9 100644 --- a/contrib/docker/conf/log.config +++ b/contrib/docker/conf/log.config @@ -10,13 +10,6 @@ filters: request: "" handlers: - file: - class: logging.handlers.RotatingFileHandler - formatter: precise - filename: /data/homeserver.log - maxBytes: 104857600 - backupCount: 10 - filters: [context] console: class: logging.StreamHandler formatter: precise @@ -33,4 +26,4 @@ loggers: root: level: INFO - handlers: [file, console] + handlers: [console] From 48c01ae8511dee744879ac7207872770a0e24705 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Wed, 25 Apr 2018 11:02:28 +0100 Subject: [PATCH 45/84] ignore atom editor python ide files --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index c8901eb206..d438c6aeea 100644 --- a/.gitignore +++ b/.gitignore @@ -48,3 +48,4 @@ env/ *.config .vscode/ +.ropeproject/ From 617bf40924bc8d627734eb2fb25a9f3980ede5d5 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Wed, 25 Apr 2018 17:37:29 +0100 Subject: [PATCH 46/84] Generate user daily stats --- synapse/app/homeserver.py | 6 ++ synapse/storage/__init__.py | 60 +++++++++++++++++-- synapse/storage/client_ips.py | 7 +++ synapse/storage/prepare_database.py | 2 +- .../schema/delta/49/add_user_daily_visits.sql | 25 ++++++++ .../49/add_user_ips_last_seen_only_index.sql | 17 ++++++ 6 files changed, 112 insertions(+), 5 deletions(-) create mode 100644 synapse/storage/schema/delta/49/add_user_daily_visits.sql create mode 100644 synapse/storage/schema/delta/49/add_user_ips_last_seen_only_index.sql diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index a0e465d644..808567365a 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -473,6 +473,9 @@ def run(hs): " changes across releases." ) + def generate_user_daily_visit_stats(): + hs.get_datastore().generate_user_daily_visits() + if hs.config.report_stats: logger.info("Scheduling stats reporting for 3 hour intervals") clock.looping_call(phone_stats_home, 3 * 60 * 60 * 1000) @@ -485,6 +488,9 @@ def run(hs): # be quite busy the first few minutes clock.call_later(5 * 60, phone_stats_home) + clock.looping_call(generate_user_daily_visit_stats, 60 * 1000) + clock.call_later(5 * 60, generate_user_daily_visit_stats) + if hs.config.daemonize and hs.config.print_pidfile: print (hs.config.pid_file) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 8cdfd50f90..6f2f947433 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -14,6 +14,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +import datetime +import time +import logging + from synapse.storage.devices import DeviceStore from .appservice import ( ApplicationServiceStore, ApplicationServiceTransactionStore @@ -55,10 +59,6 @@ from .engines import PostgresEngine from synapse.api.constants import PresenceState from synapse.util.caches.stream_change_cache import StreamChangeCache - -import logging - - logger = logging.getLogger(__name__) @@ -347,6 +347,58 @@ class DataStore(RoomMemberStore, RoomStore, return self.runInteraction("count_r30_users", _count_r30_users) + + def generate_user_daily_visits(self): + """ + Generates daily visit data for use in cohort/ retention analysis + """ + def _generate_user_daily_visits(txn): + logger.info("Calling _generate_user_daily_visits") + # determine timestamp of previous days + yesterday = datetime.datetime.now() - datetime.timedelta(days=1) + yesterday_start = datetime.datetime(yesterday.year, + yesterday.month, + yesterday.day, 0, 0, 0, 0) + yesterday_start_time = int(time.mktime(yesterday_start.timetuple())) * 1000 + + # Check that this job has not already been completed + sql = """ + SELECT timestamp + FROM user_daily_visits + ORDER by timestamp desc limit 1 + """ + txn.execute(sql) + row = txn.fetchone() + + # Bail if the most recent time is yesterday + if row and row[0] == yesterday_start_time: + logger.info("Bailing from _generate_user_daily_visits, already completed") + return + logger.info("inserting into user_daily_visits") + # Not specificying an upper bound means that if the update is run at + # 10 mins past midnight and the user is active during a 30 min session + # that the user is still included in the previous days stats + # This does mean that if the update is run hours late, then it is possible + # to overstate the cohort, but this seems a reasonable trade off + # The alternative is to insert on every request - but prefer to avoid + # for performance reasons + sql = """ + SELECT user_id, user_agent, device_id + FROM user_ips + WHERE last_seen > ? + """ + txn.execute(sql, (yesterday_start_time,)) + + sql = """ + INSERT INTO user_daily_visits (user_id, user_agent, device_id, timestamp) + VALUES (?, ?, ?, ?) + """ + + for row in txn: + txn.execute(sql, (row + (yesterday_start_time,))) + + return self.runInteraction("generate_user_daily_visits", _generate_user_daily_visits) + def get_users(self): """Function to reterive a list of users in users table. diff --git a/synapse/storage/client_ips.py b/synapse/storage/client_ips.py index 7b44dae0fc..ba46907737 100644 --- a/synapse/storage/client_ips.py +++ b/synapse/storage/client_ips.py @@ -55,6 +55,13 @@ class ClientIpStore(background_updates.BackgroundUpdateStore): columns=["user_id", "last_seen"], ) + self.register_background_index_update( + "user_ips_last_seen_only_index", + index_name="user_ips_last_seen_only", + table="user_ips", + columns=["last_seen"], + ) + # (user_id, access_token, ip) -> (user_agent, device_id, last_seen) self._batch_row_update = {} diff --git a/synapse/storage/prepare_database.py b/synapse/storage/prepare_database.py index 04411a665f..c08e9cd65a 100644 --- a/synapse/storage/prepare_database.py +++ b/synapse/storage/prepare_database.py @@ -26,7 +26,7 @@ logger = logging.getLogger(__name__) # Remember to update this number every time a change is made to database # schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 48 +SCHEMA_VERSION = 49 dir_path = os.path.abspath(os.path.dirname(__file__)) diff --git a/synapse/storage/schema/delta/49/add_user_daily_visits.sql b/synapse/storage/schema/delta/49/add_user_daily_visits.sql new file mode 100644 index 0000000000..11356a97fc --- /dev/null +++ b/synapse/storage/schema/delta/49/add_user_daily_visits.sql @@ -0,0 +1,25 @@ +/* Copyright 2018 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +CREATE TABLE user_daily_visits ( user_id TEXT NOT NULL, + device_id TEXT, + user_agent TEXT NOT NULL, + timestamp BIGINT NOT NULL ); + +/* What indexes should I include? + * Reads are offline so should optimise for writes + * Need to check if already an entry so user,day + */ diff --git a/synapse/storage/schema/delta/49/add_user_ips_last_seen_only_index.sql b/synapse/storage/schema/delta/49/add_user_ips_last_seen_only_index.sql new file mode 100644 index 0000000000..3a4ed59b5b --- /dev/null +++ b/synapse/storage/schema/delta/49/add_user_ips_last_seen_only_index.sql @@ -0,0 +1,17 @@ +/* Copyright 2018 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +INSERT into background_updates (update_name, progress_json) + VALUES ('user_ips_last_seen_only_index', '{}'); From fb6015d0a6828d57c18d601ea36b1a8a4cebc0e2 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Wed, 25 Apr 2018 17:56:11 +0100 Subject: [PATCH 47/84] pep8 --- synapse/storage/__init__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 6f2f947433..c22d38800c 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -347,7 +347,6 @@ class DataStore(RoomMemberStore, RoomStore, return self.runInteraction("count_r30_users", _count_r30_users) - def generate_user_daily_visits(self): """ Generates daily visit data for use in cohort/ retention analysis @@ -390,14 +389,16 @@ class DataStore(RoomMemberStore, RoomStore, txn.execute(sql, (yesterday_start_time,)) sql = """ - INSERT INTO user_daily_visits (user_id, user_agent, device_id, timestamp) + INSERT INTO user_daily_visits (user_id. user_agent, + device_id, timestamp) VALUES (?, ?, ?, ?) """ for row in txn: txn.execute(sql, (row + (yesterday_start_time,))) - return self.runInteraction("generate_user_daily_visits", _generate_user_daily_visits) + return self.runInteraction("generate_user_daily_visits", + _generate_user_daily_visits) def get_users(self): """Function to reterive a list of users in users table. From 5917562b6040448cd26b67fac52d61b41ba08f6d Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Tue, 1 May 2018 12:12:22 +0100 Subject: [PATCH 48/84] 10 mins seems more reasonable that every minute --- synapse/app/homeserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 808567365a..328ab8032c 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -488,7 +488,7 @@ def run(hs): # be quite busy the first few minutes clock.call_later(5 * 60, phone_stats_home) - clock.looping_call(generate_user_daily_visit_stats, 60 * 1000) + clock.looping_call(generate_user_daily_visit_stats, 10 * 60 * 1000) clock.call_later(5 * 60, generate_user_daily_visit_stats) if hs.config.daemonize and hs.config.print_pidfile: From d0857702e8e22b8bd3dae7fc3cd3718da1e94f19 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Tue, 1 May 2018 12:12:57 +0100 Subject: [PATCH 49/84] add inidexes based on usage --- synapse/storage/schema/delta/49/add_user_daily_visits.sql | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/synapse/storage/schema/delta/49/add_user_daily_visits.sql b/synapse/storage/schema/delta/49/add_user_daily_visits.sql index 11356a97fc..3dd478196f 100644 --- a/synapse/storage/schema/delta/49/add_user_daily_visits.sql +++ b/synapse/storage/schema/delta/49/add_user_daily_visits.sql @@ -16,10 +16,6 @@ CREATE TABLE user_daily_visits ( user_id TEXT NOT NULL, device_id TEXT, - user_agent TEXT NOT NULL, timestamp BIGINT NOT NULL ); - -/* What indexes should I include? - * Reads are offline so should optimise for writes - * Need to check if already an entry so user,day - */ +CREATE INDEX user_daily_visits_uts_idx ON user_daily_visits(user_id, timestamp); +CREATE INDEX user_daily_visits_ts_idx ON user_daily_visits(timestamp); From dd1a8324195f37d508d518db10ba37d9028afc86 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Tue, 1 May 2018 12:13:49 +0100 Subject: [PATCH 50/84] remove user agent from data model, will just join on user_ips --- synapse/storage/__init__.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index c22d38800c..b51cf70336 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -15,6 +15,7 @@ # limitations under the License. import datetime +from dateutil import tz import time import logging @@ -354,10 +355,9 @@ class DataStore(RoomMemberStore, RoomStore, def _generate_user_daily_visits(txn): logger.info("Calling _generate_user_daily_visits") # determine timestamp of previous days - yesterday = datetime.datetime.now() - datetime.timedelta(days=1) - yesterday_start = datetime.datetime(yesterday.year, - yesterday.month, - yesterday.day, 0, 0, 0, 0) + yesterday = datetime.datetime.utcnow() - datetime.timedelta(days=1) + yesterday_start = datetime.datetime(yesterday.year, yesterday.month, + yesterday.day, tzinfo=tz.tzutc()) yesterday_start_time = int(time.mktime(yesterday_start.timetuple())) * 1000 # Check that this job has not already been completed @@ -371,9 +371,8 @@ class DataStore(RoomMemberStore, RoomStore, # Bail if the most recent time is yesterday if row and row[0] == yesterday_start_time: - logger.info("Bailing from _generate_user_daily_visits, already completed") return - logger.info("inserting into user_daily_visits") + # Not specificying an upper bound means that if the update is run at # 10 mins past midnight and the user is active during a 30 min session # that the user is still included in the previous days stats @@ -382,20 +381,20 @@ class DataStore(RoomMemberStore, RoomStore, # The alternative is to insert on every request - but prefer to avoid # for performance reasons sql = """ - SELECT user_id, user_agent, device_id + SELECT user_id, device_id FROM user_ips WHERE last_seen > ? """ txn.execute(sql, (yesterday_start_time,)) + user_visits = txn.fetchall() sql = """ - INSERT INTO user_daily_visits (user_id. user_agent, - device_id, timestamp) - VALUES (?, ?, ?, ?) + INSERT INTO user_daily_visits (user_id, device_id, timestamp) + VALUES (?, ?, ?) """ - for row in txn: - txn.execute(sql, (row + (yesterday_start_time,))) + for visit in user_visits: + txn.execute(sql, (visit + (yesterday_start_time,))) return self.runInteraction("generate_user_daily_visits", _generate_user_daily_visits) From da602419b277308d06ba5e2a147e6d128eaa2802 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Tue, 1 May 2018 19:19:23 +0100 Subject: [PATCH 51/84] missing word :| --- CHANGES.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 0dd2efd9d2..317846d2a2 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -10,7 +10,7 @@ SECURITY UPDATE of events in such a room will default to using stream_ordering rather than depth (topological_ordering). - This is a temporary solution to mitigate abuse in the wild, whilst a long solution + This is a temporary solution to mitigate abuse in the wild, whilst a long term solution is being implemented to improve how the depth parameter is used. Full details at From d4c14e143825c679a7909b009aec2e51f829857f Mon Sep 17 00:00:00 2001 From: kaiyou Date: Tue, 1 May 2018 20:47:58 +0200 Subject: [PATCH 52/84] Fix the documentation about 'POSTGRES_DB' --- contrib/docker/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/docker/README.md b/contrib/docker/README.md index 25c358c847..aed56646c2 100644 --- a/contrib/docker/README.md +++ b/contrib/docker/README.md @@ -135,7 +135,7 @@ Shared secrets, that will be initialized to random values if not set: Database specific values (will use SQLite if not set): -* `POSTGRES_DATABASE` - The database name for the synapse postgres database. [default: `matrix`] +* `POSTGRES_DB` - The database name for the synapse postgres database. [default: `synapse`] * `POSTGRES_HOST` - The host of the postgres database if you wish to use postgresql instead of sqlite3. [default: `db` which is useful when using a container on the same docker network in a compose file where the postgres service is called `db`] * `POSTGRES_PASSWORD` - The password for the synapse postgres database. **If this is set then postgres will be used instead of sqlite3.** [default: none] **NOTE**: You are highly encouraged to use postgresql! Please use the compose file to make it easier to deploy. * `POSTGRES_USER` - The user for the synapse postgres database. [default: `matrix`] From 4f2e898c29de5b51e8c5893afa64e5b43d25048f Mon Sep 17 00:00:00 2001 From: kaiyou Date: Tue, 1 May 2018 20:49:52 +0200 Subject: [PATCH 53/84] Make the logging level configurable --- contrib/docker/conf/log.config | 6 +++--- contrib/docker/docker-compose.yml | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/contrib/docker/conf/log.config b/contrib/docker/conf/log.config index b5c907c4f9..1851995802 100644 --- a/contrib/docker/conf/log.config +++ b/contrib/docker/conf/log.config @@ -17,13 +17,13 @@ handlers: loggers: synapse: - level: INFO + level: {{ SYNAPSE_LOG_LEVEL or "WARNING" }} synapse.storage.SQL: # beware: increasing this to DEBUG will make synapse log sensitive # information such as access tokens. - level: INFO + level: {{ SYNAPSE_LOG_LEVEL or "WARNING" }} root: - level: INFO + level: {{ SYNAPSE_LOG_LEVEL or "WARNING" }} handlers: [console] diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml index 46e72601d3..0b531949e0 100644 --- a/contrib/docker/docker-compose.yml +++ b/contrib/docker/docker-compose.yml @@ -15,6 +15,7 @@ services: - SYNAPSE_SERVER_NAME=my.matrix.host - SYNAPSE_REPORT_STATS=no - SYNAPSE_ENABLE_REGISTRATION=yes + - SYNAPSE_LOG_LEVEL=INFO - POSTGRES_PASSWORD=changeme volumes: # You may either store all the files in a local folder From 5addeaa02c432241af282900b19172ba7158118c Mon Sep 17 00:00:00 2001 From: kaiyou Date: Fri, 4 May 2018 21:23:01 +0200 Subject: [PATCH 54/84] Add Docker packaging in the author list --- AUTHORS.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/AUTHORS.rst b/AUTHORS.rst index 3dcb1c2a89..e13ac5ad34 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -60,3 +60,6 @@ Niklas Riekenbrauck Christoph Witzany * Add LDAP support for authentication + +Pierre Jaury +* Docker packaging \ No newline at end of file From bf98fa0864fe62dbf1303acc778912d69fd4a738 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 8 May 2018 15:58:35 +0100 Subject: [PATCH 55/84] Part user from rooms on account deactivate This implements this very crudely: this probably isn't viable because parting a user from all their rooms could take a long time, and if the HS gets restarted in that time the process will be aborted. --- synapse/handlers/deactivate_account.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/deactivate_account.py b/synapse/handlers/deactivate_account.py index b1d3814909..387620c9c6 100644 --- a/synapse/handlers/deactivate_account.py +++ b/synapse/handlers/deactivate_account.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2017 New Vector Ltd +# Copyright 2017, 2018 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,6 +15,7 @@ from twisted.internet import defer from ._base import BaseHandler +from synapse.types import UserID, create_requester import logging @@ -27,6 +28,7 @@ class DeactivateAccountHandler(BaseHandler): super(DeactivateAccountHandler, self).__init__(hs) self._auth_handler = hs.get_auth_handler() self._device_handler = hs.get_device_handler() + self._room_member_handler = hs.get_room_member_handler() @defer.inlineCallbacks def deactivate_account(self, user_id): @@ -50,3 +52,15 @@ class DeactivateAccountHandler(BaseHandler): yield self.store.user_delete_threepids(user_id) yield self.store.user_set_password_hash(user_id, None) + + user = UserID.from_string(user_id) + + rooms_for_user = yield self.store.get_rooms_for_user(user_id) + for room_id in rooms_for_user: + yield self._room_member_handler.update_membership( + create_requester(user), + user, + room_id, + "leave", + ratelimit=False, + ) From 7e8726b8fb297f255ed1fd78f8e3be1d7f21dcc7 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 9 May 2018 14:54:28 +0100 Subject: [PATCH 56/84] Part deactivated users in the background One room at a time so we don't take out the whole server with leave events, and restart at server restart. --- synapse/handlers/deactivate_account.py | 35 +++++++++++++++++++++++++- synapse/storage/registration.py | 27 ++++++++++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/deactivate_account.py b/synapse/handlers/deactivate_account.py index 387620c9c6..5bb6c11e8f 100644 --- a/synapse/handlers/deactivate_account.py +++ b/synapse/handlers/deactivate_account.py @@ -12,10 +12,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from twisted.internet import defer +from twisted.internet import defer, reactor from ._base import BaseHandler from synapse.types import UserID, create_requester +from synapse.util.logcontext import run_in_background import logging @@ -30,6 +31,10 @@ class DeactivateAccountHandler(BaseHandler): self._device_handler = hs.get_device_handler() self._room_member_handler = hs.get_room_member_handler() + self._user_parter_running = False + + reactor.callWhenRunning(self.start_user_parting) + @defer.inlineCallbacks def deactivate_account(self, user_id): """Deactivate a user's account @@ -53,10 +58,38 @@ class DeactivateAccountHandler(BaseHandler): yield self.store.user_delete_threepids(user_id) yield self.store.user_set_password_hash(user_id, None) + yield self.store.add_user_pending_deactivation(user_id) + + self.start_user_parting() + + def start_user_parting(self): + if not self._user_parter_running: + run_in_background(self.user_parter_loop()) + + @defer.inlineCallbacks + def user_parter_loop(self): + self._user_parter_running = True + logger.info("Starting user parter") + try: + while True: + user_id = yield self.store.get_user_pending_deactivation() + if user_id is None: + break + logger.info("User parter parting %r", user_id) + yield self.part_user(user_id) + yield self.store.del_user_pending_deactivation(user_id) + logger.info("User parter finished parting %r", user_id) + logger.info("User parter finished: stopping") + finally: + self._user_parter_running = False + + @defer.inlineCallbacks + def part_user(self, user_id): user = UserID.from_string(user_id) rooms_for_user = yield self.store.get_rooms_for_user(user_id) for room_id in rooms_for_user: + logger.info("User parter parting %r from %r", user_id, room_id) yield self._room_member_handler.update_membership( create_requester(user), user, diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index a50717db2d..de068a55d2 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -526,3 +526,30 @@ class RegistrationStore(RegistrationWorkerStore, except self.database_engine.module.IntegrityError: ret = yield self.get_3pid_guest_access_token(medium, address) defer.returnValue(ret) + + def add_user_pending_deactivation(self, user_id): + return self._simple_insert( + "users_pending_deactivation", + values={ + "user_id": user_id, + }, + desc="add_user_pending_deactivation", + ) + + def del_user_pending_deactivation(self, user_id): + return self._simple_delete_one( + "users_pending_deactivation", + keyvalues={ + "user_id": user_id, + }, + desc="del_user_pending_deactivation", + ) + + def get_user_pending_deactivation(self): + return self._simple_select_one_onecol( + "users_pending_deactivation", + keyvalues={}, + retcol="user_id", + allow_none=True, + desc="get_users_pending_deactivation", + ) From 52281e4c548233087f03a014fdb21783a471b2a7 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 9 May 2018 15:06:16 +0100 Subject: [PATCH 57/84] Indent fail --- synapse/handlers/deactivate_account.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/deactivate_account.py b/synapse/handlers/deactivate_account.py index 5bb6c11e8f..f99935b4ee 100644 --- a/synapse/handlers/deactivate_account.py +++ b/synapse/handlers/deactivate_account.py @@ -63,7 +63,7 @@ class DeactivateAccountHandler(BaseHandler): self.start_user_parting() def start_user_parting(self): - if not self._user_parter_running: + if not self._user_parter_running: run_in_background(self.user_parter_loop()) @defer.inlineCallbacks From 46df23f581bc52313cb1c9366e967ebcddef07cf Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 9 May 2018 15:07:54 +0100 Subject: [PATCH 58/84] Add the schema file --- .../schema/delta/48/deactivated_users.sql | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 synapse/storage/schema/delta/48/deactivated_users.sql diff --git a/synapse/storage/schema/delta/48/deactivated_users.sql b/synapse/storage/schema/delta/48/deactivated_users.sql new file mode 100644 index 0000000000..e9013a6969 --- /dev/null +++ b/synapse/storage/schema/delta/48/deactivated_users.sql @@ -0,0 +1,25 @@ +/* Copyright 2018 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Store any accounts that have been requested to be deactivated. + * We part the account from all the rooms its in when its + * deactivated. This can take some time and synapse may be restarted + * before it completes, so store the user IDs here until the process + * is complete. + */ +CREATE TABLE users_pending_deactivation ( + user_id TEXT NOT NULL +); From 294e9a0c9ba0c4ad69605eb781d9ae56b8982a96 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 9 May 2018 15:10:37 +0100 Subject: [PATCH 59/84] Prefix internal functions --- synapse/handlers/deactivate_account.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/synapse/handlers/deactivate_account.py b/synapse/handlers/deactivate_account.py index f99935b4ee..81e0a75e78 100644 --- a/synapse/handlers/deactivate_account.py +++ b/synapse/handlers/deactivate_account.py @@ -33,7 +33,7 @@ class DeactivateAccountHandler(BaseHandler): self._user_parter_running = False - reactor.callWhenRunning(self.start_user_parting) + reactor.callWhenRunning(self._start_user_parting) @defer.inlineCallbacks def deactivate_account(self, user_id): @@ -60,14 +60,14 @@ class DeactivateAccountHandler(BaseHandler): yield self.store.add_user_pending_deactivation(user_id) - self.start_user_parting() + self._start_user_parting() - def start_user_parting(self): + def _start_user_parting(self): if not self._user_parter_running: - run_in_background(self.user_parter_loop()) + run_in_background(self._user_parter_loop()) @defer.inlineCallbacks - def user_parter_loop(self): + def _user_parter_loop(self): self._user_parter_running = True logger.info("Starting user parter") try: @@ -76,7 +76,7 @@ class DeactivateAccountHandler(BaseHandler): if user_id is None: break logger.info("User parter parting %r", user_id) - yield self.part_user(user_id) + yield self._part_user(user_id) yield self.store.del_user_pending_deactivation(user_id) logger.info("User parter finished parting %r", user_id) logger.info("User parter finished: stopping") @@ -84,7 +84,7 @@ class DeactivateAccountHandler(BaseHandler): self._user_parter_running = False @defer.inlineCallbacks - def part_user(self, user_id): + def _part_user(self, user_id): user = UserID.from_string(user_id) rooms_for_user = yield self.store.get_rooms_for_user(user_id) From 18e144fe088bda9e28697062cda62dca1e03724b Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 9 May 2018 19:55:03 +0100 Subject: [PATCH 60/84] Move RequestsMetrics to its own file This is useful in its own right, because server.py is full of stuff; but more importantly, I want to do some refactoring that will cause a circular reference as it is. --- synapse/http/request_metrics.py | 147 ++++++++++++++++++++++++++++++++ synapse/http/server.py | 128 +-------------------------- 2 files changed, 151 insertions(+), 124 deletions(-) create mode 100644 synapse/http/request_metrics.py diff --git a/synapse/http/request_metrics.py b/synapse/http/request_metrics.py new file mode 100644 index 0000000000..4a843a36a7 --- /dev/null +++ b/synapse/http/request_metrics.py @@ -0,0 +1,147 @@ +# -*- coding: utf-8 -*- +# Copyright 2014-2016 OpenMarket Ltd +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +import synapse.metrics +from synapse.util.logcontext import LoggingContext + +logger = logging.getLogger(__name__) + +metrics = synapse.metrics.get_metrics_for("synapse.http.server") + +# total number of responses served, split by method/servlet/tag +response_count = metrics.register_counter( + "response_count", + labels=["method", "servlet", "tag"], + alternative_names=( + # the following are all deprecated aliases for the same metric + metrics.name_prefix + x for x in ( + "_requests", + "_response_time:count", + "_response_ru_utime:count", + "_response_ru_stime:count", + "_response_db_txn_count:count", + "_response_db_txn_duration:count", + ) + ) +) + +requests_counter = metrics.register_counter( + "requests_received", + labels=["method", "servlet", ], +) + +outgoing_responses_counter = metrics.register_counter( + "responses", + labels=["method", "code"], +) + +response_timer = metrics.register_counter( + "response_time_seconds", + labels=["method", "servlet", "tag"], + alternative_names=( + metrics.name_prefix + "_response_time:total", + ), +) + +response_ru_utime = metrics.register_counter( + "response_ru_utime_seconds", labels=["method", "servlet", "tag"], + alternative_names=( + metrics.name_prefix + "_response_ru_utime:total", + ), +) + +response_ru_stime = metrics.register_counter( + "response_ru_stime_seconds", labels=["method", "servlet", "tag"], + alternative_names=( + metrics.name_prefix + "_response_ru_stime:total", + ), +) + +response_db_txn_count = metrics.register_counter( + "response_db_txn_count", labels=["method", "servlet", "tag"], + alternative_names=( + metrics.name_prefix + "_response_db_txn_count:total", + ), +) + +# seconds spent waiting for db txns, excluding scheduling time, when processing +# this request +response_db_txn_duration = metrics.register_counter( + "response_db_txn_duration_seconds", labels=["method", "servlet", "tag"], + alternative_names=( + metrics.name_prefix + "_response_db_txn_duration:total", + ), +) + +# seconds spent waiting for a db connection, when processing this request +response_db_sched_duration = metrics.register_counter( + "response_db_sched_duration_seconds", labels=["method", "servlet", "tag"] +) + +# size in bytes of the response written +response_size = metrics.register_counter( + "response_size", labels=["method", "servlet", "tag"] +) + + +class RequestMetrics(object): + def start(self, clock, name): + self.start = clock.time_msec() + self.start_context = LoggingContext.current_context() + self.name = name + + def stop(self, clock, request): + context = LoggingContext.current_context() + + tag = "" + if context: + tag = context.tag + + if context != self.start_context: + logger.warn( + "Context have unexpectedly changed %r, %r", + context, self.start_context + ) + return + + response_count.inc(request.method, self.name, tag) + + response_timer.inc_by( + clock.time_msec() - self.start, request.method, + self.name, tag + ) + + ru_utime, ru_stime = context.get_resource_usage() + + response_ru_utime.inc_by( + ru_utime, request.method, self.name, tag + ) + response_ru_stime.inc_by( + ru_stime, request.method, self.name, tag + ) + response_db_txn_count.inc_by( + context.db_txn_count, request.method, self.name, tag + ) + response_db_txn_duration.inc_by( + context.db_txn_duration_ms / 1000., request.method, self.name, tag + ) + response_db_sched_duration.inc_by( + context.db_sched_duration_ms / 1000., request.method, self.name, tag + ) + + response_size.inc_by(request.sentLength, request.method, self.name, tag) diff --git a/synapse/http/server.py b/synapse/http/server.py index 55b9ad5251..37b26b908e 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -18,6 +18,10 @@ from synapse.api.errors import ( cs_exception, SynapseError, CodeMessageException, UnrecognizedRequestError, Codes ) +from synapse.http.request_metrics import ( + RequestMetrics, requests_counter, + outgoing_responses_counter, +) from synapse.util.logcontext import LoggingContext, PreserveLoggingContext from synapse.util.caches import intern_dict from synapse.util.metrics import Measure @@ -41,82 +45,6 @@ import simplejson logger = logging.getLogger(__name__) -metrics = synapse.metrics.get_metrics_for(__name__) - -# total number of responses served, split by method/servlet/tag -response_count = metrics.register_counter( - "response_count", - labels=["method", "servlet", "tag"], - alternative_names=( - # the following are all deprecated aliases for the same metric - metrics.name_prefix + x for x in ( - "_requests", - "_response_time:count", - "_response_ru_utime:count", - "_response_ru_stime:count", - "_response_db_txn_count:count", - "_response_db_txn_duration:count", - ) - ) -) - -requests_counter = metrics.register_counter( - "requests_received", - labels=["method", "servlet", ], -) - -outgoing_responses_counter = metrics.register_counter( - "responses", - labels=["method", "code"], -) - -response_timer = metrics.register_counter( - "response_time_seconds", - labels=["method", "servlet", "tag"], - alternative_names=( - metrics.name_prefix + "_response_time:total", - ), -) - -response_ru_utime = metrics.register_counter( - "response_ru_utime_seconds", labels=["method", "servlet", "tag"], - alternative_names=( - metrics.name_prefix + "_response_ru_utime:total", - ), -) - -response_ru_stime = metrics.register_counter( - "response_ru_stime_seconds", labels=["method", "servlet", "tag"], - alternative_names=( - metrics.name_prefix + "_response_ru_stime:total", - ), -) - -response_db_txn_count = metrics.register_counter( - "response_db_txn_count", labels=["method", "servlet", "tag"], - alternative_names=( - metrics.name_prefix + "_response_db_txn_count:total", - ), -) - -# seconds spent waiting for db txns, excluding scheduling time, when processing -# this request -response_db_txn_duration = metrics.register_counter( - "response_db_txn_duration_seconds", labels=["method", "servlet", "tag"], - alternative_names=( - metrics.name_prefix + "_response_db_txn_duration:total", - ), -) - -# seconds spent waiting for a db connection, when processing this request -response_db_sched_duration = metrics.register_counter( - "response_db_sched_duration_seconds", labels=["method", "servlet", "tag"] -) - -# size in bytes of the response written -response_size = metrics.register_counter( - "response_size", labels=["method", "servlet", "tag"] -) _next_request_id = 0 @@ -386,54 +314,6 @@ def _unrecognised_request_handler(request): raise UnrecognizedRequestError() -class RequestMetrics(object): - def start(self, clock, name): - self.start = clock.time_msec() - self.start_context = LoggingContext.current_context() - self.name = name - - def stop(self, clock, request): - context = LoggingContext.current_context() - - tag = "" - if context: - tag = context.tag - - if context != self.start_context: - logger.warn( - "Context have unexpectedly changed %r, %r", - context, self.start_context - ) - return - - response_count.inc(request.method, self.name, tag) - - response_timer.inc_by( - clock.time_msec() - self.start, request.method, - self.name, tag - ) - - ru_utime, ru_stime = context.get_resource_usage() - - response_ru_utime.inc_by( - ru_utime, request.method, self.name, tag - ) - response_ru_stime.inc_by( - ru_stime, request.method, self.name, tag - ) - response_db_txn_count.inc_by( - context.db_txn_count, request.method, self.name, tag - ) - response_db_txn_duration.inc_by( - context.db_txn_duration_ms / 1000., request.method, self.name, tag - ) - response_db_sched_duration.inc_by( - context.db_sched_duration_ms / 1000., request.method, self.name, tag - ) - - response_size.inc_by(request.sentLength, request.method, self.name, tag) - - class RootRedirect(resource.Resource): """Redirects the root '/' path to another path.""" From 8460e48d0639a7f575fd3e57dcd5cf263a6d6a19 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 9 May 2018 23:00:11 +0100 Subject: [PATCH 61/84] Move request_id management into SynapseRequest --- synapse/http/server.py | 31 ++++++++++++++++--------------- synapse/http/site.py | 9 +++++++++ 2 files changed, 25 insertions(+), 15 deletions(-) diff --git a/synapse/http/server.py b/synapse/http/server.py index 37b26b908e..369f7bbdd1 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -46,38 +46,40 @@ import simplejson logger = logging.getLogger(__name__) -_next_request_id = 0 - - def request_handler(include_metrics=False): """Decorator for ``wrap_request_handler``""" return lambda request_handler: wrap_request_handler(request_handler, include_metrics) def wrap_request_handler(request_handler, include_metrics=False): - """Wraps a method that acts as a request handler with the necessary logging - and exception handling. + """Wraps a request handler method with the necessary logging and exception + handling. - The method must have a signature of "handle_foo(self, request)". The - argument "self" must have "version_string" and "clock" attributes. The - argument "request" must be a twisted HTTP request. + The handler method must have a signature of "handle_foo(self, request)", + where "self" must have "version_string" and "clock" attributes (and + "request" must be a SynapseRequest). - The method must return a deferred. If the deferred succeeds we assume that + The handler must return a deferred. If the deferred succeeds we assume that a response has been sent. If the deferred fails with a SynapseError we use it to send a JSON response with the appropriate HTTP reponse code. If the deferred fails with any other type of error we send a 500 reponse. - We insert a unique request-id into the logging context for this request and - log the response and duration for this request. + As well as calling `request.processing` (which will log the response and + duration for this request), the wrapped request handler will insert the + request id into the logging context. """ @defer.inlineCallbacks def wrapped_request_handler(self, request): - global _next_request_id - request_id = "%s-%s" % (request.method, _next_request_id) - _next_request_id += 1 + """ + Args: + self: + request (synapse.http.site.SynapseRequest): + """ + request_id = request.get_request_id() with LoggingContext(request_id) as request_context: + request_context.request = request_id with Measure(self.clock, "wrapped_request_handler"): request_metrics = RequestMetrics() # we start the request metrics timer here with an initial stab @@ -87,7 +89,6 @@ def wrap_request_handler(request_handler, include_metrics=False): servlet_name = self.__class__.__name__ request_metrics.start(self.clock, name=servlet_name) - request_context.request = request_id with request.processing(): try: with PreserveLoggingContext(request_context): diff --git a/synapse/http/site.py b/synapse/http/site.py index c8b46e1af2..6af276e69a 100644 --- a/synapse/http/site.py +++ b/synapse/http/site.py @@ -22,6 +22,8 @@ import time ACCESS_TOKEN_RE = re.compile(br'(\?.*access(_|%5[Ff])token=)[^&]*(.*)$') +_next_request_seq = 0 + class SynapseRequest(Request): def __init__(self, site, *args, **kw): @@ -30,6 +32,10 @@ class SynapseRequest(Request): self.authenticated_entity = None self.start_time = 0 + global _next_request_seq + self.request_seq = _next_request_seq + _next_request_seq += 1 + def __repr__(self): # We overwrite this so that we don't log ``access_token`` return '<%s at 0x%x method=%s uri=%s clientproto=%s site=%s>' % ( @@ -41,6 +47,9 @@ class SynapseRequest(Request): self.site.site_tag, ) + def get_request_id(self): + return "%s-%i" % (self.method, self.request_seq) + def get_redacted_uri(self): return ACCESS_TOKEN_RE.sub( br'\1\3', From 4d298506ddb65f642d899dffa236e4499f0c427b Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 10 May 2018 11:57:13 +0100 Subject: [PATCH 62/84] Oops, don't call function passed to run_in_background --- synapse/handlers/deactivate_account.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/deactivate_account.py b/synapse/handlers/deactivate_account.py index 81e0a75e78..78558dd309 100644 --- a/synapse/handlers/deactivate_account.py +++ b/synapse/handlers/deactivate_account.py @@ -64,7 +64,7 @@ class DeactivateAccountHandler(BaseHandler): def _start_user_parting(self): if not self._user_parter_running: - run_in_background(self._user_parter_loop()) + run_in_background(self._user_parter_loop) @defer.inlineCallbacks def _user_parter_loop(self): From 09b29f9c4a941ae5721b1f5e296156dfe92e3395 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 9 May 2018 23:03:11 +0100 Subject: [PATCH 63/84] Make RequestMetrics take a raw time rather than a clock ... which is going to make it easier to move around. --- synapse/http/request_metrics.py | 8 ++++---- synapse/http/server.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/synapse/http/request_metrics.py b/synapse/http/request_metrics.py index 4a843a36a7..4e8a5f5306 100644 --- a/synapse/http/request_metrics.py +++ b/synapse/http/request_metrics.py @@ -100,12 +100,12 @@ response_size = metrics.register_counter( class RequestMetrics(object): - def start(self, clock, name): - self.start = clock.time_msec() + def start(self, time_msec, name): + self.start = time_msec self.start_context = LoggingContext.current_context() self.name = name - def stop(self, clock, request): + def stop(self, time_msec, request): context = LoggingContext.current_context() tag = "" @@ -122,7 +122,7 @@ class RequestMetrics(object): response_count.inc(request.method, self.name, tag) response_timer.inc_by( - clock.time_msec() - self.start, request.method, + time_msec - self.start, request.method, self.name, tag ) diff --git a/synapse/http/server.py b/synapse/http/server.py index 369f7bbdd1..b16c9c17f6 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -87,7 +87,7 @@ def wrap_request_handler(request_handler, include_metrics=False): # JsonResource (or a subclass), and JsonResource._async_render # will update it once it picks a servlet. servlet_name = self.__class__.__name__ - request_metrics.start(self.clock, name=servlet_name) + request_metrics.start(self.clock.time_msec(), name=servlet_name) with request.processing(): try: @@ -138,7 +138,7 @@ def wrap_request_handler(request_handler, include_metrics=False): finally: try: request_metrics.stop( - self.clock, request + self.clock.time_msec(), request ) except Exception as e: logger.warn("Failed to stop metrics: %r", e) From c6f730282c3a25468df0e624293aefd60cef7840 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 9 May 2018 23:05:14 +0100 Subject: [PATCH 64/84] Move RequestMetrics handling into SynapseRequest.processing() It fits quite nicely here, and opens the path to getting rid of the "include_metrics" mess. --- synapse/http/server.py | 19 ++++-------- synapse/http/site.py | 69 +++++++++++++++++++++++++++++++++++------- 2 files changed, 64 insertions(+), 24 deletions(-) diff --git a/synapse/http/server.py b/synapse/http/server.py index b16c9c17f6..8e5d1d58f5 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -19,7 +19,7 @@ from synapse.api.errors import ( cs_exception, SynapseError, CodeMessageException, UnrecognizedRequestError, Codes ) from synapse.http.request_metrics import ( - RequestMetrics, requests_counter, + requests_counter, outgoing_responses_counter, ) from synapse.util.logcontext import LoggingContext, PreserveLoggingContext @@ -81,19 +81,18 @@ def wrap_request_handler(request_handler, include_metrics=False): with LoggingContext(request_id) as request_context: request_context.request = request_id with Measure(self.clock, "wrapped_request_handler"): - request_metrics = RequestMetrics() # we start the request metrics timer here with an initial stab # at the servlet name. For most requests that name will be # JsonResource (or a subclass), and JsonResource._async_render # will update it once it picks a servlet. servlet_name = self.__class__.__name__ - request_metrics.start(self.clock.time_msec(), name=servlet_name) - - with request.processing(): + with request.processing(servlet_name): try: with PreserveLoggingContext(request_context): if include_metrics: - yield request_handler(self, request, request_metrics) + yield request_handler( + self, request, request.request_metrics, + ) else: requests_counter.inc(request.method, servlet_name) yield request_handler(self, request) @@ -135,13 +134,7 @@ def wrap_request_handler(request_handler, include_metrics=False): pretty_print=_request_user_agent_is_curl(request), version_string=self.version_string, ) - finally: - try: - request_metrics.stop( - self.clock.time_msec(), request - ) - except Exception as e: - logger.warn("Failed to stop metrics: %r", e) + return wrapped_request_handler diff --git a/synapse/http/site.py b/synapse/http/site.py index 6af276e69a..bfd9832aa0 100644 --- a/synapse/http/site.py +++ b/synapse/http/site.py @@ -12,20 +12,38 @@ # See the License for the specific language governing permissions and # limitations under the License. -from synapse.util.logcontext import LoggingContext -from twisted.web.server import Site, Request - import contextlib import logging import re import time +from twisted.web.server import Site, Request + +from synapse.http.request_metrics import RequestMetrics +from synapse.util.logcontext import LoggingContext + +logger = logging.getLogger(__name__) + ACCESS_TOKEN_RE = re.compile(br'(\?.*access(_|%5[Ff])token=)[^&]*(.*)$') _next_request_seq = 0 class SynapseRequest(Request): + """Class which encapsulates an HTTP request to synapse. + + All of the requests processed in synapse are of this type. + + It extends twisted's twisted.web.server.Request, and adds: + * Unique request ID + * Redaction of access_token query-params in __repr__ + * Logging at start and end + * Metrics to record CPU, wallclock and DB time by endpoint. + + It provides a method `processing` which should be called by the Resource + which is handling the request, and returns a context manager. + + """ def __init__(self, site, *args, **kw): Request.__init__(self, *args, **kw) self.site = site @@ -59,7 +77,11 @@ class SynapseRequest(Request): def get_user_agent(self): return self.requestHeaders.getRawHeaders(b"User-Agent", [None])[-1] - def started_processing(self): + def _started_processing(self, servlet_name): + self.start_time = int(time.time() * 1000) + self.request_metrics = RequestMetrics() + self.request_metrics.start(self.start_time, name=servlet_name) + self.site.access_logger.info( "%s - %s - Received request: %s %s", self.getClientIP(), @@ -67,10 +89,8 @@ class SynapseRequest(Request): self.method, self.get_redacted_uri() ) - self.start_time = int(time.time() * 1000) - - def finished_processing(self): + def _finished_processing(self): try: context = LoggingContext.current_context() ru_utime, ru_stime = context.get_resource_usage() @@ -81,6 +101,8 @@ class SynapseRequest(Request): ru_utime, ru_stime = (0, 0) db_txn_count, db_txn_duration_ms = (0, 0) + end_time = int(time.time() * 1000) + self.site.access_logger.info( "%s - %s - {%s}" " Processed request: %dms (%dms, %dms) (%dms/%dms/%d)" @@ -88,7 +110,7 @@ class SynapseRequest(Request): self.getClientIP(), self.site.site_tag, self.authenticated_entity, - int(time.time() * 1000) - self.start_time, + end_time - self.start_time, int(ru_utime * 1000), int(ru_stime * 1000), db_sched_duration_ms, @@ -102,11 +124,36 @@ class SynapseRequest(Request): self.get_user_agent(), ) + try: + self.request_metrics.stop(end_time, self) + except Exception as e: + logger.warn("Failed to stop metrics: %r", e) + @contextlib.contextmanager - def processing(self): - self.started_processing() + def processing(self, servlet_name): + """Record the fact that we are processing this request. + + Returns a context manager; the correct way to use this is: + + @defer.inlineCallbacks + def handle_request(request): + with request.processing("FooServlet"): + yield really_handle_the_request() + + This will log the request's arrival. Once the context manager is + closed, the completion of the request will be logged, and the various + metrics will be updated. + + Args: + servlet_name (str): the name of the servlet which will be + processing this request. This is used in the metrics. + + It is possible to update this afterwards by updating + self.request_metrics.servlet_name. + """ + self._started_processing(servlet_name) yield - self.finished_processing() + self._finished_processing() class XForwardedForRequest(SynapseRequest): From b8700dd7d0482813beb9c780b411de5108ae078a Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 9 May 2018 18:03:04 +0100 Subject: [PATCH 65/84] Bump requests_counter in wrapped_request_handler less magic --- synapse/http/server.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/synapse/http/server.py b/synapse/http/server.py index 8e5d1d58f5..1d05b53873 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -90,12 +90,20 @@ def wrap_request_handler(request_handler, include_metrics=False): try: with PreserveLoggingContext(request_context): if include_metrics: - yield request_handler( + d = request_handler( self, request, request.request_metrics, ) else: - requests_counter.inc(request.method, servlet_name) - yield request_handler(self, request) + d = request_handler(self, request) + + # record the arrival of the request *after* + # dispatching to the handler, so that the handler + # can update the servlet name in the request + # metrics + requests_counter.inc(request.method, + request.request_metrics.name) + yield d + except CodeMessageException as e: code = e.code if isinstance(e, SynapseError): @@ -220,7 +228,6 @@ class JsonResource(HttpServer, resource.Resource): servlet_classname = "%r" % callback request_metrics.name = servlet_classname - requests_counter.inc(request.method, servlet_classname) # Now trigger the callback. If it returns a response, we send it # here. If it throws an exception, that is handled by the wrapper From 49e5a613f1ac53498b31150f78332fc64932e61b Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 9 May 2018 23:44:22 +0100 Subject: [PATCH 66/84] Move outgoing_responses_counter handling to RequestMetrics it's much neater there. --- synapse/http/request_metrics.py | 2 ++ synapse/http/server.py | 4 ---- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/synapse/http/request_metrics.py b/synapse/http/request_metrics.py index 4e8a5f5306..8c850bf23f 100644 --- a/synapse/http/request_metrics.py +++ b/synapse/http/request_metrics.py @@ -119,6 +119,8 @@ class RequestMetrics(object): ) return + outgoing_responses_counter.inc(request.method, str(request.code)) + response_count.inc(request.method, self.name, tag) response_timer.inc_by( diff --git a/synapse/http/server.py b/synapse/http/server.py index 1d05b53873..200b2c4837 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -20,7 +20,6 @@ from synapse.api.errors import ( ) from synapse.http.request_metrics import ( requests_counter, - outgoing_responses_counter, ) from synapse.util.logcontext import LoggingContext, PreserveLoggingContext from synapse.util.caches import intern_dict @@ -112,7 +111,6 @@ def wrap_request_handler(request_handler, include_metrics=False): ) else: logger.exception(e) - outgoing_responses_counter.inc(request.method, str(code)) respond_with_json( request, code, cs_exception(e), send_cors=True, pretty_print=_request_user_agent_is_curl(request), @@ -274,8 +272,6 @@ class JsonResource(HttpServer, resource.Resource): def _send_response(self, request, code, response_json_object, response_code_message=None): - outgoing_responses_counter.inc(request.method, str(code)) - # TODO: Only enable CORS for the requests that need it. respond_with_json( request, code, response_json_object, From 9589a1925e944360c68f7eb8a65f8ba94ec9bb84 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 9 May 2018 23:49:29 +0100 Subject: [PATCH 67/84] Remove include_metrics param The metrics are now available via the request, so this is redundant and can go away at last. --- synapse/http/server.py | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/synapse/http/server.py b/synapse/http/server.py index 200b2c4837..9598969de8 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -45,12 +45,12 @@ import simplejson logger = logging.getLogger(__name__) -def request_handler(include_metrics=False): +def request_handler(): """Decorator for ``wrap_request_handler``""" - return lambda request_handler: wrap_request_handler(request_handler, include_metrics) + return wrap_request_handler -def wrap_request_handler(request_handler, include_metrics=False): +def wrap_request_handler(request_handler): """Wraps a request handler method with the necessary logging and exception handling. @@ -88,12 +88,7 @@ def wrap_request_handler(request_handler, include_metrics=False): with request.processing(servlet_name): try: with PreserveLoggingContext(request_context): - if include_metrics: - d = request_handler( - self, request, request.request_metrics, - ) - else: - d = request_handler(self, request) + d = request_handler(self, request) # record the arrival of the request *after* # dispatching to the handler, so that the handler @@ -206,13 +201,9 @@ class JsonResource(HttpServer, resource.Resource): self._async_render(request) return server.NOT_DONE_YET - # Disable metric reporting because _async_render does its own metrics. - # It does its own metric reporting because _async_render dispatches to - # a callback and it's the class name of that callback we want to report - # against rather than the JsonResource itself. - @request_handler(include_metrics=True) + @request_handler() @defer.inlineCallbacks - def _async_render(self, request, request_metrics): + def _async_render(self, request): """ This gets called from render() every time someone sends us a request. This checks if anyone has registered a callback for that method and path. @@ -224,8 +215,7 @@ class JsonResource(HttpServer, resource.Resource): servlet_classname = servlet_instance.__class__.__name__ else: servlet_classname = "%r" % callback - - request_metrics.name = servlet_classname + request.request_metrics.name = servlet_classname # Now trigger the callback. If it returns a response, we send it # here. If it throws an exception, that is handled by the wrapper From 09f570b9357d40764a0716f7e287fa4dde44610a Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 10 May 2018 11:59:51 +0100 Subject: [PATCH 68/84] Factor wrap_request_handler_with_logging out of wrap_request_handler ... so that it can be used on non-JSON endpoints --- synapse/http/server.py | 118 +++++++++++++++++++++++------------------ 1 file changed, 65 insertions(+), 53 deletions(-) diff --git a/synapse/http/server.py b/synapse/http/server.py index 9598969de8..fd58e65c4b 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -50,9 +50,10 @@ def request_handler(): return wrap_request_handler -def wrap_request_handler(request_handler): - """Wraps a request handler method with the necessary logging and exception - handling. +def wrap_request_handler(h): + """Wraps a request handler method with exception handling. + + Also adds logging as per wrap_request_handler_with_logging. The handler method must have a signature of "handle_foo(self, request)", where "self" must have "version_string" and "clock" attributes (and @@ -62,12 +63,63 @@ def wrap_request_handler(request_handler): a response has been sent. If the deferred fails with a SynapseError we use it to send a JSON response with the appropriate HTTP reponse code. If the deferred fails with any other type of error we send a 500 reponse. + """ + + @defer.inlineCallbacks + def wrapped_request_handler(self, request): + try: + yield h(self, request) + except CodeMessageException as e: + code = e.code + if isinstance(e, SynapseError): + logger.info( + "%s SynapseError: %s - %s", request, code, e.msg + ) + else: + logger.exception(e) + respond_with_json( + request, code, cs_exception(e), send_cors=True, + pretty_print=_request_user_agent_is_curl(request), + version_string=self.version_string, + ) + + except Exception: + # failure.Failure() fishes the original Failure out + # of our stack, and thus gives us a sensible stack + # trace. + f = failure.Failure() + logger.error( + "Failed handle request via %r: %r: %s", + h, + request, + f.getTraceback().rstrip(), + ) + respond_with_json( + request, + 500, + { + "error": "Internal server error", + "errcode": Codes.UNKNOWN, + }, + send_cors=True, + pretty_print=_request_user_agent_is_curl(request), + version_string=self.version_string, + ) + + return wrap_request_handler_with_logging(wrapped_request_handler) + + +def wrap_request_handler_with_logging(h): + """Wraps a request handler to provide logging and metrics + + The handler method must have a signature of "handle_foo(self, request)", + where "self" must have a "clock" attribute (and "request" must be a + SynapseRequest). As well as calling `request.processing` (which will log the response and duration for this request), the wrapped request handler will insert the request id into the logging context. """ - @defer.inlineCallbacks def wrapped_request_handler(self, request): """ @@ -86,56 +138,16 @@ def wrap_request_handler(request_handler): # will update it once it picks a servlet. servlet_name = self.__class__.__name__ with request.processing(servlet_name): - try: - with PreserveLoggingContext(request_context): - d = request_handler(self, request) - - # record the arrival of the request *after* - # dispatching to the handler, so that the handler - # can update the servlet name in the request - # metrics - requests_counter.inc(request.method, - request.request_metrics.name) - yield d - - except CodeMessageException as e: - code = e.code - if isinstance(e, SynapseError): - logger.info( - "%s SynapseError: %s - %s", request, code, e.msg - ) - else: - logger.exception(e) - respond_with_json( - request, code, cs_exception(e), send_cors=True, - pretty_print=_request_user_agent_is_curl(request), - version_string=self.version_string, - ) - except Exception: - # failure.Failure() fishes the original Failure out - # of our stack, and thus gives us a sensible stack - # trace. - f = failure.Failure() - logger.error( - "Failed handle request %s.%s on %r: %r: %s", - request_handler.__module__, - request_handler.__name__, - self, - request, - f.getTraceback().rstrip(), - ) - respond_with_json( - request, - 500, - { - "error": "Internal server error", - "errcode": Codes.UNKNOWN, - }, - send_cors=True, - pretty_print=_request_user_agent_is_curl(request), - version_string=self.version_string, - ) + with PreserveLoggingContext(request_context): + d = h(self, request) + # record the arrival of the request *after* + # dispatching to the handler, so that the handler + # can update the servlet name in the request + # metrics + requests_counter.inc(request.method, + request.request_metrics.name) + yield d return wrapped_request_handler From 645cb4bf06deee1c4c10ecc3d7df2c914168f19a Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 10 May 2018 12:10:27 +0100 Subject: [PATCH 69/84] Remove redundant request_handler decorator This is needless complexity; we might as well use the wrapper directly. Also rename wrap_request_handler->wrap_json_request_handler. --- synapse/http/additional_resource.py | 4 ++-- synapse/http/server.py | 9 ++------ synapse/rest/key/v2/remote_key_resource.py | 8 ++++--- synapse/rest/media/v1/download_resource.py | 22 ++++++++++-------- synapse/rest/media/v1/preview_url_resource.py | 5 ++-- synapse/rest/media/v1/thumbnail_resource.py | 23 +++++++++++-------- synapse/rest/media/v1/upload_resource.py | 19 +++++++-------- 7 files changed, 47 insertions(+), 43 deletions(-) diff --git a/synapse/http/additional_resource.py b/synapse/http/additional_resource.py index 343e932cb1..d9e7f5dfb7 100644 --- a/synapse/http/additional_resource.py +++ b/synapse/http/additional_resource.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from synapse.http.server import wrap_request_handler +from synapse.http.server import wrap_json_request_handler from twisted.web.resource import Resource from twisted.web.server import NOT_DONE_YET @@ -50,6 +50,6 @@ class AdditionalResource(Resource): self._async_render(request) return NOT_DONE_YET - @wrap_request_handler + @wrap_json_request_handler def _async_render(self, request): return self._handler(request) diff --git a/synapse/http/server.py b/synapse/http/server.py index fd58e65c4b..f29e36f490 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -45,12 +45,7 @@ import simplejson logger = logging.getLogger(__name__) -def request_handler(): - """Decorator for ``wrap_request_handler``""" - return wrap_request_handler - - -def wrap_request_handler(h): +def wrap_json_request_handler(h): """Wraps a request handler method with exception handling. Also adds logging as per wrap_request_handler_with_logging. @@ -213,7 +208,7 @@ class JsonResource(HttpServer, resource.Resource): self._async_render(request) return server.NOT_DONE_YET - @request_handler() + @wrap_json_request_handler @defer.inlineCallbacks def _async_render(self, request): """ This gets called from render() every time someone sends us a request. diff --git a/synapse/rest/key/v2/remote_key_resource.py b/synapse/rest/key/v2/remote_key_resource.py index 17e6079cba..17b3077926 100644 --- a/synapse/rest/key/v2/remote_key_resource.py +++ b/synapse/rest/key/v2/remote_key_resource.py @@ -12,7 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -from synapse.http.server import request_handler, respond_with_json_bytes +from synapse.http.server import ( + respond_with_json_bytes, wrap_json_request_handler, +) from synapse.http.servlet import parse_integer, parse_json_object_from_request from synapse.api.errors import SynapseError, Codes from synapse.crypto.keyring import KeyLookupError @@ -99,7 +101,7 @@ class RemoteKey(Resource): self.async_render_GET(request) return NOT_DONE_YET - @request_handler() + @wrap_json_request_handler @defer.inlineCallbacks def async_render_GET(self, request): if len(request.postpath) == 1: @@ -124,7 +126,7 @@ class RemoteKey(Resource): self.async_render_POST(request) return NOT_DONE_YET - @request_handler() + @wrap_json_request_handler @defer.inlineCallbacks def async_render_POST(self, request): content = parse_json_object_from_request(request) diff --git a/synapse/rest/media/v1/download_resource.py b/synapse/rest/media/v1/download_resource.py index fe7e17596f..3fc3f64d62 100644 --- a/synapse/rest/media/v1/download_resource.py +++ b/synapse/rest/media/v1/download_resource.py @@ -12,17 +12,19 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import synapse.http.servlet - -from ._base import parse_media_id, respond_404 -from twisted.web.resource import Resource -from synapse.http.server import request_handler, set_cors_headers - -from twisted.web.server import NOT_DONE_YET -from twisted.internet import defer - import logging +from twisted.internet import defer +from twisted.web.resource import Resource +from twisted.web.server import NOT_DONE_YET + +from synapse.http.server import ( + set_cors_headers, + wrap_json_request_handler, +) +import synapse.http.servlet +from ._base import parse_media_id, respond_404 + logger = logging.getLogger(__name__) @@ -43,7 +45,7 @@ class DownloadResource(Resource): self._async_render_GET(request) return NOT_DONE_YET - @request_handler() + @wrap_json_request_handler @defer.inlineCallbacks def _async_render_GET(self, request): set_cors_headers(request) diff --git a/synapse/rest/media/v1/preview_url_resource.py b/synapse/rest/media/v1/preview_url_resource.py index 9290d7946f..6b089689b4 100644 --- a/synapse/rest/media/v1/preview_url_resource.py +++ b/synapse/rest/media/v1/preview_url_resource.py @@ -40,8 +40,9 @@ from synapse.util.stringutils import random_string from synapse.util.caches.expiringcache import ExpiringCache from synapse.http.client import SpiderHttpClient from synapse.http.server import ( - request_handler, respond_with_json_bytes, + respond_with_json_bytes, respond_with_json, + wrap_json_request_handler, ) from synapse.util.async import ObservableDeferred from synapse.util.stringutils import is_ascii @@ -90,7 +91,7 @@ class PreviewUrlResource(Resource): self._async_render_GET(request) return NOT_DONE_YET - @request_handler() + @wrap_json_request_handler @defer.inlineCallbacks def _async_render_GET(self, request): diff --git a/synapse/rest/media/v1/thumbnail_resource.py b/synapse/rest/media/v1/thumbnail_resource.py index 58ada49711..6c12d79f56 100644 --- a/synapse/rest/media/v1/thumbnail_resource.py +++ b/synapse/rest/media/v1/thumbnail_resource.py @@ -14,18 +14,21 @@ # limitations under the License. +import logging + +from twisted.internet import defer +from twisted.web.resource import Resource +from twisted.web.server import NOT_DONE_YET + +from synapse.http.server import ( + set_cors_headers, + wrap_json_request_handler, +) +from synapse.http.servlet import parse_integer, parse_string from ._base import ( - parse_media_id, respond_404, respond_with_file, FileInfo, + FileInfo, parse_media_id, respond_404, respond_with_file, respond_with_responder, ) -from twisted.web.resource import Resource -from synapse.http.servlet import parse_string, parse_integer -from synapse.http.server import request_handler, set_cors_headers - -from twisted.web.server import NOT_DONE_YET -from twisted.internet import defer - -import logging logger = logging.getLogger(__name__) @@ -48,7 +51,7 @@ class ThumbnailResource(Resource): self._async_render_GET(request) return NOT_DONE_YET - @request_handler() + @wrap_json_request_handler @defer.inlineCallbacks def _async_render_GET(self, request): set_cors_headers(request) diff --git a/synapse/rest/media/v1/upload_resource.py b/synapse/rest/media/v1/upload_resource.py index a31e75cb46..7d01c57fd1 100644 --- a/synapse/rest/media/v1/upload_resource.py +++ b/synapse/rest/media/v1/upload_resource.py @@ -13,16 +13,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -from synapse.http.server import respond_with_json, request_handler +import logging + +from twisted.internet import defer +from twisted.web.resource import Resource +from twisted.web.server import NOT_DONE_YET from synapse.api.errors import SynapseError - -from twisted.web.server import NOT_DONE_YET -from twisted.internet import defer - -from twisted.web.resource import Resource - -import logging +from synapse.http.server import ( + respond_with_json, + wrap_json_request_handler, +) logger = logging.getLogger(__name__) @@ -51,7 +52,7 @@ class UploadResource(Resource): respond_with_json(request, 200, {}, send_cors=True) return NOT_DONE_YET - @request_handler() + @wrap_json_request_handler @defer.inlineCallbacks def _async_render_POST(self, request): requester = yield self.auth.get_user_by_req(request) From 217bc53c981c03ce880d1fa6afad2970be3f1d78 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 10 May 2018 12:20:40 +0100 Subject: [PATCH 70/84] Many docstrings --- synapse/handlers/deactivate_account.py | 24 ++++++++++++++++++++++++ synapse/storage/registration.py | 12 ++++++++++++ 2 files changed, 36 insertions(+) diff --git a/synapse/handlers/deactivate_account.py b/synapse/handlers/deactivate_account.py index 78558dd309..c00ec18627 100644 --- a/synapse/handlers/deactivate_account.py +++ b/synapse/handlers/deactivate_account.py @@ -31,8 +31,11 @@ class DeactivateAccountHandler(BaseHandler): self._device_handler = hs.get_device_handler() self._room_member_handler = hs.get_room_member_handler() + # Flag that indicates whether the process to part users from rooms is running self._user_parter_running = False + # Start the user parter loop so it can resume parting users from rooms where + # it left off (if it has work left to do). reactor.callWhenRunning(self._start_user_parting) @defer.inlineCallbacks @@ -58,16 +61,32 @@ class DeactivateAccountHandler(BaseHandler): yield self.store.user_delete_threepids(user_id) yield self.store.user_set_password_hash(user_id, None) + # Add the user to a table of users penpding deactivation (ie. + # removal from all the rooms they're a member of) yield self.store.add_user_pending_deactivation(user_id) + # Now start the process that goes through that list and + # parts users from rooms (if it isn't already running) self._start_user_parting() def _start_user_parting(self): + """ + Start the process that goes through the table of users + pending deactivation, if it isn't already running. + + Returns: + None + """ if not self._user_parter_running: run_in_background(self._user_parter_loop) @defer.inlineCallbacks def _user_parter_loop(self): + """Loop that parts deactivated users from rooms + + Returns: + None + """ self._user_parter_running = True logger.info("Starting user parter") try: @@ -85,6 +104,11 @@ class DeactivateAccountHandler(BaseHandler): @defer.inlineCallbacks def _part_user(self, user_id): + """Causes the given user_id to leave all the rooms they're joined to + + Returns: + None + """ user = UserID.from_string(user_id) rooms_for_user = yield self.store.get_rooms_for_user(user_id) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index de068a55d2..c05ce4612f 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -528,6 +528,10 @@ class RegistrationStore(RegistrationWorkerStore, defer.returnValue(ret) def add_user_pending_deactivation(self, user_id): + """ + Adds a user to the table of users who need to be parted from all the rooms they're + in + """ return self._simple_insert( "users_pending_deactivation", values={ @@ -537,6 +541,10 @@ class RegistrationStore(RegistrationWorkerStore, ) def del_user_pending_deactivation(self, user_id): + """ + Removes the given user to the table of users who need to be parted from all the + rooms they're in, effectively marking that user as fully deactivated. + """ return self._simple_delete_one( "users_pending_deactivation", keyvalues={ @@ -546,6 +554,10 @@ class RegistrationStore(RegistrationWorkerStore, ) def get_user_pending_deactivation(self): + """ + Gets one user from the table of users waiting to be parted from all the rooms + they're in. + """ return self._simple_select_one_onecol( "users_pending_deactivation", keyvalues={}, From 6b49628e3bf18f6cc1a1347fef8e4180e854d245 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 10 May 2018 12:23:53 +0100 Subject: [PATCH 71/84] Catch failure to part user from room --- synapse/handlers/deactivate_account.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/synapse/handlers/deactivate_account.py b/synapse/handlers/deactivate_account.py index c00ec18627..4eb18775e8 100644 --- a/synapse/handlers/deactivate_account.py +++ b/synapse/handlers/deactivate_account.py @@ -114,10 +114,16 @@ class DeactivateAccountHandler(BaseHandler): rooms_for_user = yield self.store.get_rooms_for_user(user_id) for room_id in rooms_for_user: logger.info("User parter parting %r from %r", user_id, room_id) - yield self._room_member_handler.update_membership( - create_requester(user), - user, - room_id, - "leave", - ratelimit=False, - ) + try: + yield self._room_member_handler.update_membership( + create_requester(user), + user, + room_id, + "leave", + ratelimit=False, + ) + except Exception: + logger.exception( + "Failed to part user %r from room %r: ignoring and continuing", + user_id, room_id, + ) From 318711e1399da009910c3a9e5fa297c28a2d0a97 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 10 May 2018 18:46:59 +0100 Subject: [PATCH 72/84] Set Server header in SynapseRequest (instead of everywhere that writes a response. Or rather, the subset of places which write responses where we haven't forgotten it). This also means that we don't have to have the mysterious version_string attribute in anything with a request handler. Unfortunately it does mean that we have to pass the version string wherever we instantiate a SynapseSite, which has been c&ped 150 times, but that is code that ought to be cleaned up anyway really. --- synapse/app/appservice.py | 1 + synapse/app/client_reader.py | 1 + synapse/app/event_creator.py | 1 + synapse/app/federation_reader.py | 1 + synapse/app/federation_sender.py | 1 + synapse/app/frontend_proxy.py | 1 + synapse/app/homeserver.py | 2 ++ synapse/app/media_repository.py | 1 + synapse/app/pusher.py | 1 + synapse/app/synchrotron.py | 1 + synapse/app/user_dir.py | 1 + synapse/http/additional_resource.py | 3 +-- synapse/http/server.py | 14 ++++---------- synapse/http/site.py | 11 ++++++++++- synapse/rest/client/v1/pusher.py | 1 - synapse/rest/client/v2_alpha/auth.py | 2 -- synapse/rest/key/v1/server_key_resource.py | 2 -- synapse/rest/key/v2/local_key_resource.py | 2 -- synapse/rest/key/v2/remote_key_resource.py | 2 -- synapse/rest/media/v1/download_resource.py | 3 +-- synapse/rest/media/v1/preview_url_resource.py | 1 - synapse/rest/media/v1/thumbnail_resource.py | 1 - synapse/rest/media/v1/upload_resource.py | 1 - 23 files changed, 28 insertions(+), 27 deletions(-) diff --git a/synapse/app/appservice.py b/synapse/app/appservice.py index 58f2c9d68c..b1efacc9f8 100644 --- a/synapse/app/appservice.py +++ b/synapse/app/appservice.py @@ -74,6 +74,7 @@ class AppserviceServer(HomeServer): site_tag, listener_config, root_resource, + self.version_string, ) ) diff --git a/synapse/app/client_reader.py b/synapse/app/client_reader.py index 267d34c881..38b98382c6 100644 --- a/synapse/app/client_reader.py +++ b/synapse/app/client_reader.py @@ -98,6 +98,7 @@ class ClientReaderServer(HomeServer): site_tag, listener_config, root_resource, + self.version_string, ) ) diff --git a/synapse/app/event_creator.py b/synapse/app/event_creator.py index b915d12d53..bd7f3d5679 100644 --- a/synapse/app/event_creator.py +++ b/synapse/app/event_creator.py @@ -114,6 +114,7 @@ class EventCreatorServer(HomeServer): site_tag, listener_config, root_resource, + self.version_string, ) ) diff --git a/synapse/app/federation_reader.py b/synapse/app/federation_reader.py index c1dc66dd17..6e10b27b9e 100644 --- a/synapse/app/federation_reader.py +++ b/synapse/app/federation_reader.py @@ -87,6 +87,7 @@ class FederationReaderServer(HomeServer): site_tag, listener_config, root_resource, + self.version_string, ) ) diff --git a/synapse/app/federation_sender.py b/synapse/app/federation_sender.py index a08af83a4c..6f24e32d6d 100644 --- a/synapse/app/federation_sender.py +++ b/synapse/app/federation_sender.py @@ -101,6 +101,7 @@ class FederationSenderServer(HomeServer): site_tag, listener_config, root_resource, + self.version_string, ) ) diff --git a/synapse/app/frontend_proxy.py b/synapse/app/frontend_proxy.py index b349e3e3ce..0f700ee786 100644 --- a/synapse/app/frontend_proxy.py +++ b/synapse/app/frontend_proxy.py @@ -152,6 +152,7 @@ class FrontendProxyServer(HomeServer): site_tag, listener_config, root_resource, + self.version_string, ) ) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index a0e465d644..75f40fd5a4 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -140,6 +140,7 @@ class SynapseHomeServer(HomeServer): site_tag, listener_config, root_resource, + self.version_string, ), self.tls_server_context_factory, ) @@ -153,6 +154,7 @@ class SynapseHomeServer(HomeServer): site_tag, listener_config, root_resource, + self.version_string, ) ) logger.info("Synapse now listening on port %d", port) diff --git a/synapse/app/media_repository.py b/synapse/app/media_repository.py index fc8282bbc1..9c93195f0a 100644 --- a/synapse/app/media_repository.py +++ b/synapse/app/media_repository.py @@ -94,6 +94,7 @@ class MediaRepositoryServer(HomeServer): site_tag, listener_config, root_resource, + self.version_string, ) ) diff --git a/synapse/app/pusher.py b/synapse/app/pusher.py index 26930d1b3b..3912eae48c 100644 --- a/synapse/app/pusher.py +++ b/synapse/app/pusher.py @@ -104,6 +104,7 @@ class PusherServer(HomeServer): site_tag, listener_config, root_resource, + self.version_string, ) ) diff --git a/synapse/app/synchrotron.py b/synapse/app/synchrotron.py index 7152b1deb4..c6294a7a0c 100644 --- a/synapse/app/synchrotron.py +++ b/synapse/app/synchrotron.py @@ -281,6 +281,7 @@ class SynchrotronServer(HomeServer): site_tag, listener_config, root_resource, + self.version_string, ) ) diff --git a/synapse/app/user_dir.py b/synapse/app/user_dir.py index 5ba7e9b416..53eb3474da 100644 --- a/synapse/app/user_dir.py +++ b/synapse/app/user_dir.py @@ -126,6 +126,7 @@ class UserDirectoryServer(HomeServer): site_tag, listener_config, root_resource, + self.version_string, ) ) diff --git a/synapse/http/additional_resource.py b/synapse/http/additional_resource.py index d9e7f5dfb7..a797396ade 100644 --- a/synapse/http/additional_resource.py +++ b/synapse/http/additional_resource.py @@ -42,8 +42,7 @@ class AdditionalResource(Resource): Resource.__init__(self) self._handler = handler - # these are required by the request_handler wrapper - self.version_string = hs.version_string + # required by the request_handler wrapper self.clock = hs.get_clock() def render(self, request): diff --git a/synapse/http/server.py b/synapse/http/server.py index f29e36f490..b6e2ae14a2 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -51,8 +51,8 @@ def wrap_json_request_handler(h): Also adds logging as per wrap_request_handler_with_logging. The handler method must have a signature of "handle_foo(self, request)", - where "self" must have "version_string" and "clock" attributes (and - "request" must be a SynapseRequest). + where "self" must have a "clock" attribute (and "request" must be a + SynapseRequest). The handler must return a deferred. If the deferred succeeds we assume that a response has been sent. If the deferred fails with a SynapseError we use @@ -75,7 +75,6 @@ def wrap_json_request_handler(h): respond_with_json( request, code, cs_exception(e), send_cors=True, pretty_print=_request_user_agent_is_curl(request), - version_string=self.version_string, ) except Exception: @@ -98,7 +97,6 @@ def wrap_json_request_handler(h): }, send_cors=True, pretty_print=_request_user_agent_is_curl(request), - version_string=self.version_string, ) return wrap_request_handler_with_logging(wrapped_request_handler) @@ -192,7 +190,6 @@ class JsonResource(HttpServer, resource.Resource): self.canonical_json = canonical_json self.clock = hs.get_clock() self.path_regexs = {} - self.version_string = hs.version_string self.hs = hs def register_paths(self, method, path_patterns, callback): @@ -275,7 +272,6 @@ class JsonResource(HttpServer, resource.Resource): send_cors=True, response_code_message=response_code_message, pretty_print=_request_user_agent_is_curl(request), - version_string=self.version_string, canonical_json=self.canonical_json, ) @@ -326,7 +322,7 @@ class RootRedirect(resource.Resource): def respond_with_json(request, code, json_object, send_cors=False, response_code_message=None, pretty_print=False, - version_string="", canonical_json=True): + canonical_json=True): # could alternatively use request.notifyFinish() and flip a flag when # the Deferred fires, but since the flag is RIGHT THERE it seems like # a waste. @@ -348,12 +344,11 @@ def respond_with_json(request, code, json_object, send_cors=False, request, code, json_bytes, send_cors=send_cors, response_code_message=response_code_message, - version_string=version_string ) def respond_with_json_bytes(request, code, json_bytes, send_cors=False, - version_string="", response_code_message=None): + response_code_message=None): """Sends encoded JSON in response to the given request. Args: @@ -367,7 +362,6 @@ def respond_with_json_bytes(request, code, json_bytes, send_cors=False, request.setResponseCode(code, message=response_code_message) request.setHeader(b"Content-Type", b"application/json") - request.setHeader(b"Server", version_string) request.setHeader(b"Content-Length", b"%d" % (len(json_bytes),)) request.setHeader(b"Cache-Control", b"no-cache, no-store, must-revalidate") diff --git a/synapse/http/site.py b/synapse/http/site.py index bfd9832aa0..202a990508 100644 --- a/synapse/http/site.py +++ b/synapse/http/site.py @@ -77,6 +77,11 @@ class SynapseRequest(Request): def get_user_agent(self): return self.requestHeaders.getRawHeaders(b"User-Agent", [None])[-1] + def render(self, resrc): + # override the Server header which is set by twisted + self.setHeader("Server", self.site.server_version_string) + return Request.render(self, resrc) + def _started_processing(self, servlet_name): self.start_time = int(time.time() * 1000) self.request_metrics = RequestMetrics() @@ -151,6 +156,8 @@ class SynapseRequest(Request): It is possible to update this afterwards by updating self.request_metrics.servlet_name. """ + # TODO: we should probably just move this into render() and finish(), + # to save having to call a separate method. self._started_processing(servlet_name) yield self._finished_processing() @@ -191,7 +198,8 @@ class SynapseSite(Site): Subclass of a twisted http Site that does access logging with python's standard logging """ - def __init__(self, logger_name, site_tag, config, resource, *args, **kwargs): + def __init__(self, logger_name, site_tag, config, resource, + server_version_string, *args, **kwargs): Site.__init__(self, resource, *args, **kwargs) self.site_tag = site_tag @@ -199,6 +207,7 @@ class SynapseSite(Site): proxied = config.get("x_forwarded", False) self.requestFactory = SynapseRequestFactory(self, proxied) self.access_logger = logging.getLogger(logger_name) + self.server_version_string = server_version_string def log(self, request): pass diff --git a/synapse/rest/client/v1/pusher.py b/synapse/rest/client/v1/pusher.py index 0206e664c1..40e523cc5f 100644 --- a/synapse/rest/client/v1/pusher.py +++ b/synapse/rest/client/v1/pusher.py @@ -176,7 +176,6 @@ class PushersRemoveRestServlet(RestServlet): request.setResponseCode(200) request.setHeader(b"Content-Type", b"text/html; charset=utf-8") - request.setHeader(b"Server", self.hs.version_string) request.setHeader(b"Content-Length", b"%d" % ( len(PushersRemoveRestServlet.SUCCESS_HTML), )) diff --git a/synapse/rest/client/v2_alpha/auth.py b/synapse/rest/client/v2_alpha/auth.py index 8e5577148f..d6f3a19648 100644 --- a/synapse/rest/client/v2_alpha/auth.py +++ b/synapse/rest/client/v2_alpha/auth.py @@ -129,7 +129,6 @@ class AuthRestServlet(RestServlet): html_bytes = html.encode("utf8") request.setResponseCode(200) request.setHeader(b"Content-Type", b"text/html; charset=utf-8") - request.setHeader(b"Server", self.hs.version_string) request.setHeader(b"Content-Length", b"%d" % (len(html_bytes),)) request.write(html_bytes) @@ -175,7 +174,6 @@ class AuthRestServlet(RestServlet): html_bytes = html.encode("utf8") request.setResponseCode(200) request.setHeader(b"Content-Type", b"text/html; charset=utf-8") - request.setHeader(b"Server", self.hs.version_string) request.setHeader(b"Content-Length", b"%d" % (len(html_bytes),)) request.write(html_bytes) diff --git a/synapse/rest/key/v1/server_key_resource.py b/synapse/rest/key/v1/server_key_resource.py index bd4fea5774..1498d188c1 100644 --- a/synapse/rest/key/v1/server_key_resource.py +++ b/synapse/rest/key/v1/server_key_resource.py @@ -49,7 +49,6 @@ class LocalKey(Resource): """ def __init__(self, hs): - self.version_string = hs.version_string self.response_body = encode_canonical_json( self.response_json_object(hs.config) ) @@ -84,7 +83,6 @@ class LocalKey(Resource): def render_GET(self, request): return respond_with_json_bytes( request, 200, self.response_body, - version_string=self.version_string ) def getChild(self, name, request): diff --git a/synapse/rest/key/v2/local_key_resource.py b/synapse/rest/key/v2/local_key_resource.py index be68d9a096..04775b3c45 100644 --- a/synapse/rest/key/v2/local_key_resource.py +++ b/synapse/rest/key/v2/local_key_resource.py @@ -63,7 +63,6 @@ class LocalKey(Resource): isLeaf = True def __init__(self, hs): - self.version_string = hs.version_string self.config = hs.config self.clock = hs.clock self.update_response_body(self.clock.time_msec()) @@ -115,5 +114,4 @@ class LocalKey(Resource): self.update_response_body(time_now) return respond_with_json_bytes( request, 200, self.response_body, - version_string=self.version_string ) diff --git a/synapse/rest/key/v2/remote_key_resource.py b/synapse/rest/key/v2/remote_key_resource.py index 17b3077926..21b4c1175e 100644 --- a/synapse/rest/key/v2/remote_key_resource.py +++ b/synapse/rest/key/v2/remote_key_resource.py @@ -93,7 +93,6 @@ class RemoteKey(Resource): def __init__(self, hs): self.keyring = hs.get_keyring() self.store = hs.get_datastore() - self.version_string = hs.version_string self.clock = hs.get_clock() self.federation_domain_whitelist = hs.config.federation_domain_whitelist @@ -242,5 +241,4 @@ class RemoteKey(Resource): respond_with_json_bytes( request, 200, result_io.getvalue(), - version_string=self.version_string ) diff --git a/synapse/rest/media/v1/download_resource.py b/synapse/rest/media/v1/download_resource.py index 3fc3f64d62..8cf8820c31 100644 --- a/synapse/rest/media/v1/download_resource.py +++ b/synapse/rest/media/v1/download_resource.py @@ -37,9 +37,8 @@ class DownloadResource(Resource): self.media_repo = media_repo self.server_name = hs.hostname - # Both of these are expected by @request_handler() + # this is expected by @wrap_json_request_handler self.clock = hs.get_clock() - self.version_string = hs.version_string def render_GET(self, request): self._async_render_GET(request) diff --git a/synapse/rest/media/v1/preview_url_resource.py b/synapse/rest/media/v1/preview_url_resource.py index 6b089689b4..2839207abc 100644 --- a/synapse/rest/media/v1/preview_url_resource.py +++ b/synapse/rest/media/v1/preview_url_resource.py @@ -58,7 +58,6 @@ class PreviewUrlResource(Resource): self.auth = hs.get_auth() self.clock = hs.get_clock() - self.version_string = hs.version_string self.filepaths = media_repo.filepaths self.max_spider_size = hs.config.max_spider_size self.server_name = hs.hostname diff --git a/synapse/rest/media/v1/thumbnail_resource.py b/synapse/rest/media/v1/thumbnail_resource.py index 6c12d79f56..aae6e464e8 100644 --- a/synapse/rest/media/v1/thumbnail_resource.py +++ b/synapse/rest/media/v1/thumbnail_resource.py @@ -44,7 +44,6 @@ class ThumbnailResource(Resource): self.media_storage = media_storage self.dynamic_thumbnails = hs.config.dynamic_thumbnails self.server_name = hs.hostname - self.version_string = hs.version_string self.clock = hs.get_clock() def render_GET(self, request): diff --git a/synapse/rest/media/v1/upload_resource.py b/synapse/rest/media/v1/upload_resource.py index 7d01c57fd1..7567476fce 100644 --- a/synapse/rest/media/v1/upload_resource.py +++ b/synapse/rest/media/v1/upload_resource.py @@ -41,7 +41,6 @@ class UploadResource(Resource): self.server_name = hs.hostname self.auth = hs.get_auth() self.max_upload_size = hs.config.max_upload_size - self.version_string = hs.version_string self.clock = hs.get_clock() def render_POST(self, request): From db18d854cd8e197cb3d77441b2a7ebd6f81beb81 Mon Sep 17 00:00:00 2001 From: Damir Manapov Date: Thu, 10 May 2018 22:13:31 +0300 Subject: [PATCH 73/84] transaction_id, destination twice --- synapse/federation/units.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/synapse/federation/units.py b/synapse/federation/units.py index 3f645acc43..01c5b8fe17 100644 --- a/synapse/federation/units.py +++ b/synapse/federation/units.py @@ -74,8 +74,6 @@ class Transaction(JsonEncodedObject): "previous_ids", "pdus", "edus", - "transaction_id", - "destination", "pdu_failures", ] From f077e97914c9b5c82c94786130d98af52516cde0 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Mon, 14 May 2018 13:50:58 +0100 Subject: [PATCH 74/84] instead of inserting user daily visit data at the end of the day, instead insert incrementally through the day --- synapse/app/homeserver.py | 19 ++++++++++--- synapse/storage/__init__.py | 54 ++++++++++--------------------------- 2 files changed, 30 insertions(+), 43 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index f785a7a22b..bfc79a5e81 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -17,6 +17,7 @@ import gc import logging import os import sys +import datetime import synapse import synapse.config.logger @@ -475,9 +476,24 @@ def run(hs): " changes across releases." ) + # def recurring_user_daily_visit_stats(): + def generate_user_daily_visit_stats(): hs.get_datastore().generate_user_daily_visits() + # Since user daily stats are bucketed at midnight UTC, + # and user_ips.last_seen can be updated at any time, it is important to call + # generate_user_daily_visit_stats immediately prior to the day end. Assuming + # an hourly cadence, the simplist way is to allign all calls to the hour + # end + end_of_hour = datetime.datetime.now().replace(microsecond=0, second=0, minute=0) \ + + datetime.timedelta(hours=1) \ + - datetime.timedelta(seconds=10) # Ensure method fires before day transistion + + time_to_next_hour = end_of_hour - datetime.datetime.now() + clock.call_later(time_to_next_hour.seconds, + clock.looping_call(generate_user_daily_visit_stats, 60 * 60 * 1000)) + if hs.config.report_stats: logger.info("Scheduling stats reporting for 3 hour intervals") clock.looping_call(phone_stats_home, 3 * 60 * 60 * 1000) @@ -490,9 +506,6 @@ def run(hs): # be quite busy the first few minutes clock.call_later(5 * 60, phone_stats_home) - clock.looping_call(generate_user_daily_visit_stats, 10 * 60 * 1000) - clock.call_later(5 * 60, generate_user_daily_visit_stats) - if hs.config.daemonize and hs.config.print_pidfile: print (hs.config.pid_file) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index b51cf70336..6949876c13 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -353,48 +353,22 @@ class DataStore(RoomMemberStore, RoomStore, Generates daily visit data for use in cohort/ retention analysis """ def _generate_user_daily_visits(txn): - logger.info("Calling _generate_user_daily_visits") - # determine timestamp of previous days - yesterday = datetime.datetime.utcnow() - datetime.timedelta(days=1) - yesterday_start = datetime.datetime(yesterday.year, yesterday.month, - yesterday.day, tzinfo=tz.tzutc()) - yesterday_start_time = int(time.mktime(yesterday_start.timetuple())) * 1000 - # Check that this job has not already been completed + # determine timestamp of the day start + now = datetime.datetime.utcnow() + today_start = datetime.datetime(now.year, now.month, + now.day, tzinfo=tz.tzutc()) + today_start_time = int(time.mktime(today_start.timetuple())) * 1000 + logger.info(today_start_time) sql = """ - SELECT timestamp - FROM user_daily_visits - ORDER by timestamp desc limit 1 - """ - txn.execute(sql) - row = txn.fetchone() - - # Bail if the most recent time is yesterday - if row and row[0] == yesterday_start_time: - return - - # Not specificying an upper bound means that if the update is run at - # 10 mins past midnight and the user is active during a 30 min session - # that the user is still included in the previous days stats - # This does mean that if the update is run hours late, then it is possible - # to overstate the cohort, but this seems a reasonable trade off - # The alternative is to insert on every request - but prefer to avoid - # for performance reasons - sql = """ - SELECT user_id, device_id - FROM user_ips - WHERE last_seen > ? - """ - txn.execute(sql, (yesterday_start_time,)) - user_visits = txn.fetchall() - - sql = """ - INSERT INTO user_daily_visits (user_id, device_id, timestamp) - VALUES (?, ?, ?) - """ - - for visit in user_visits: - txn.execute(sql, (visit + (yesterday_start_time,))) + INSERT INTO user_daily_visits (user_id, device_id, timestamp) + SELECT user_id, device_id, ? + FROM user_ips AS u + LEFT JOIN user_daily_visits USING (user_id, device_id) + WHERE last_seen > ? AND timestamp IS NULL + GROUP BY user_id, device_id; + """ + txn.execute(sql, (today_start_time, today_start_time)) return self.runInteraction("generate_user_daily_visits", _generate_user_daily_visits) From aea80a0118270807feb6e679d3d65562e9c9e512 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Mon, 14 May 2018 15:50:57 +0100 Subject: [PATCH 75/84] v0.29.0-rc1: bump version and change log --- CHANGES.rst | 66 +++++++++++++++++++++++++++++++++++++++++++-- synapse/__init__.py | 2 +- 2 files changed, 65 insertions(+), 3 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 9d40b2ac1e..a5c3848bce 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,5 +1,9 @@ -Changes in synapse -=============================== +Changes in synapse v0.29.0-rc1 (2018-05-14) +=========================================== + +Notable changes, a docker file for running Synapse (Thanks to @kaiyou!) and a +closed spec bug in the Client Server API. Additionally further prep for Python 3 +migration. Potentially breaking change: @@ -12,6 +16,64 @@ Potentially breaking change: Thanks to @NotAFile for fixing this. +Features: + +* Add a Dockerfile for synapse (PR #2846) Thanks to @kaiyou! + +Changes: + +* nuke-room-from-db.sh: added postgresql option and help (PR #2337) Thanks to @rubo77! +* Make 'unexpected logging context' into warnings (PR #3007) +* Open certificate files as bytes (PR #3084) Thanks to @NotAFile! +* Open config file in non-bytes mode (PR #3085) Thanks to @NotAFile! +* Make event properties raise AttributeError instead (PR #3102) Thanks to @NotAFile! +* Use six.moves.urlparse (PR #3108) Thanks to @NotAFile! +* Use deferred.addTimeout instead of time_bound_deferred (PR #3127) +* remove duplicates from groups tables (PR #3129) +* Miscellaneous fixes to python_dependencies (PR #3136) +* Improve exception handling for background processes (PR #3138) +* Add missing consumeErrors to improve exception handling (PR #3139) +* Use run_in_background in preference to preserve_fn (PR #3140) +* Refactor event storage to prepare for changes in state calculations (PR #3141) +* reraise exceptions more carefully (PR #3142) +* Remove redundant call to preserve_fn (PR #3143) +* Trap exceptions thrown within run_in_background (PR #3144) +* Add py3 tests to tox with folders that work (PR #3145) Thanks to @NotAFile! +* Don't yield in list comprehensions (PR #3150) Thanks to @NotAFile! +* Move more xrange to six (PR #3151) Thanks to @NotAFile! +* make imports local (PR #3152) Thanks to @NotAFile! +* move httplib import to six (PR #3153) Thanks to @NotAFile! +* Replace stringIO imports with six (PR #3154) Thanks to @NotAFile! +* more bytes strings (PR #3155) Thanks to @NotAFile! +* Construct HMAC as bytes on py3 (PR #3156) Thanks to @NotAFile! +* run config tests on py3 (PR #3159) Thanks to @NotAFile! +* add guard for None on purge_history api (PR #3160) Thanks to @krombel! +* Refactor /context to reuse pagination storage functions (PR #3193) +* Refactor recent events func to use pagination func (PR #3195) +* Refactor pagination DB API to return concrete type (PR #3196) +* Refactor get_recent_events_for_room return type (PR #3198) +* Refactor sync APIs to reuse pagination API (PR #3199) +* Remove unused code path from member change DB func (PR #3200) +* Part user from rooms on account deactivate (PR #3201) +* Refactor request handling wrappers (PR #3203) +* transaction_id, destination defined twice (PR #3209) Thanks to @damir-manapov! + +Bug Fixes: + +* synapse fails to start under Twisted >= 18.4 (PR #3135) +* Fix incorrect reference to StringIO (PR #3168) +* Fix a class of logcontext leaks (PR #3170) +* Fix a couple of logcontext leaks in unit tests (PR #3172) +* Fix logcontext leak in media repo (PR #3174) +* Escape label values in prometheus metrics (PR #3175) +* fix http request timeout code (PR #3178) +* Fix 'Unhandled Error' logs with Twisted 18.4 (PR #3182) Thanks to @Half-Shot! +* Fix logcontext leaks in rate limiter (PR #3183) +* Fix metrics that have integer value labels (PR #3186) +* notifications: Convert next_token to string according to the spec (PR #3190) Thanks to @mujx! +* nuke-room-from-db.sh: fix deletion from search table (PR #3194) Thanks to @rubo77! +* Set Server header in SynapseRequest (PR #3208) + Changes in synapse v0.28.1 (2018-05-01) ======================================= diff --git a/synapse/__init__.py b/synapse/__init__.py index f31cb9a3cb..69c081eb37 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.28.1" +__version__ = "0.29.0-rc1" From e71fb118f4d57f0c77647aa6fcabe36c08e31b20 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Mon, 14 May 2018 17:39:22 +0100 Subject: [PATCH 76/84] rearrange and collect related PRs --- CHANGES.rst | 53 +++++++++++++++++++++++++++-------------------------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index a5c3848bce..aed8525f2d 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -20,60 +20,61 @@ Features: * Add a Dockerfile for synapse (PR #2846) Thanks to @kaiyou! -Changes: +Changes - General: * nuke-room-from-db.sh: added postgresql option and help (PR #2337) Thanks to @rubo77! +* Part user from rooms on account deactivate (PR #3201) * Make 'unexpected logging context' into warnings (PR #3007) -* Open certificate files as bytes (PR #3084) Thanks to @NotAFile! -* Open config file in non-bytes mode (PR #3085) Thanks to @NotAFile! -* Make event properties raise AttributeError instead (PR #3102) Thanks to @NotAFile! -* Use six.moves.urlparse (PR #3108) Thanks to @NotAFile! -* Use deferred.addTimeout instead of time_bound_deferred (PR #3127) +* Set Server header in SynapseRequest (PR #3208) +* Use deferred.addTimeout instead of time_bound_deferred (PR #3127, #3178) * remove duplicates from groups tables (PR #3129) -* Miscellaneous fixes to python_dependencies (PR #3136) * Improve exception handling for background processes (PR #3138) * Add missing consumeErrors to improve exception handling (PR #3139) * Use run_in_background in preference to preserve_fn (PR #3140) -* Refactor event storage to prepare for changes in state calculations (PR #3141) * reraise exceptions more carefully (PR #3142) * Remove redundant call to preserve_fn (PR #3143) * Trap exceptions thrown within run_in_background (PR #3144) -* Add py3 tests to tox with folders that work (PR #3145) Thanks to @NotAFile! -* Don't yield in list comprehensions (PR #3150) Thanks to @NotAFile! -* Move more xrange to six (PR #3151) Thanks to @NotAFile! -* make imports local (PR #3152) Thanks to @NotAFile! -* move httplib import to six (PR #3153) Thanks to @NotAFile! -* Replace stringIO imports with six (PR #3154) Thanks to @NotAFile! -* more bytes strings (PR #3155) Thanks to @NotAFile! -* Construct HMAC as bytes on py3 (PR #3156) Thanks to @NotAFile! -* run config tests on py3 (PR #3159) Thanks to @NotAFile! -* add guard for None on purge_history api (PR #3160) Thanks to @krombel! + +Changes - Refactors: + * Refactor /context to reuse pagination storage functions (PR #3193) * Refactor recent events func to use pagination func (PR #3195) * Refactor pagination DB API to return concrete type (PR #3196) * Refactor get_recent_events_for_room return type (PR #3198) * Refactor sync APIs to reuse pagination API (PR #3199) * Remove unused code path from member change DB func (PR #3200) -* Part user from rooms on account deactivate (PR #3201) * Refactor request handling wrappers (PR #3203) * transaction_id, destination defined twice (PR #3209) Thanks to @damir-manapov! +* Refactor event storage to prepare for changes in state calculations (PR #3141) + +Changes - Python 3 migration: + +* Construct HMAC as bytes on py3 (PR #3156) Thanks to @NotAFile! +* run config tests on py3 (PR #3159) Thanks to @NotAFile! +* Open certificate files as bytes (PR #3084) Thanks to @NotAFile! +* Open config file in non-bytes mode (PR #3085) Thanks to @NotAFile! +* Make event properties raise AttributeError instead (PR #3102) Thanks to @NotAFile! +* Use six.moves.urlparse (PR #3108) Thanks to @NotAFile! +* Add py3 tests to tox with folders that work (PR #3145) Thanks to @NotAFile! +* Don't yield in list comprehensions (PR #3150) Thanks to @NotAFile! +* Move more xrange to six (PR #3151) Thanks to @NotAFile! +* make imports local (PR #3152) Thanks to @NotAFile! +* move httplib import to six (PR #3153) Thanks to @NotAFile! +* Replace stringIO imports with six (PR #3154, #3168) Thanks to @NotAFile! +* more bytes strings (PR #3155) Thanks to @NotAFile! Bug Fixes: -* synapse fails to start under Twisted >= 18.4 (PR #3135) -* Fix incorrect reference to StringIO (PR #3168) +* synapse fails to start under Twisted >= 18.4 (PR #3157) * Fix a class of logcontext leaks (PR #3170) * Fix a couple of logcontext leaks in unit tests (PR #3172) * Fix logcontext leak in media repo (PR #3174) -* Escape label values in prometheus metrics (PR #3175) -* fix http request timeout code (PR #3178) +* Escape label values in prometheus metrics (PR #3175, #3186) * Fix 'Unhandled Error' logs with Twisted 18.4 (PR #3182) Thanks to @Half-Shot! * Fix logcontext leaks in rate limiter (PR #3183) -* Fix metrics that have integer value labels (PR #3186) * notifications: Convert next_token to string according to the spec (PR #3190) Thanks to @mujx! * nuke-room-from-db.sh: fix deletion from search table (PR #3194) Thanks to @rubo77! -* Set Server header in SynapseRequest (PR #3208) - +* add guard for None on purge_history api (PR #3160) Thanks to @krombel! Changes in synapse v0.28.1 (2018-05-01) ======================================= From 589ecc5b5800a86406a0d1464fd6974c029a1163 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Mon, 14 May 2018 17:49:59 +0100 Subject: [PATCH 77/84] further musical chairs --- CHANGES.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index aed8525f2d..78d015db9a 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -26,11 +26,9 @@ Changes - General: * Part user from rooms on account deactivate (PR #3201) * Make 'unexpected logging context' into warnings (PR #3007) * Set Server header in SynapseRequest (PR #3208) -* Use deferred.addTimeout instead of time_bound_deferred (PR #3127, #3178) * remove duplicates from groups tables (PR #3129) * Improve exception handling for background processes (PR #3138) * Add missing consumeErrors to improve exception handling (PR #3139) -* Use run_in_background in preference to preserve_fn (PR #3140) * reraise exceptions more carefully (PR #3142) * Remove redundant call to preserve_fn (PR #3143) * Trap exceptions thrown within run_in_background (PR #3144) @@ -46,6 +44,9 @@ Changes - Refactors: * Refactor request handling wrappers (PR #3203) * transaction_id, destination defined twice (PR #3209) Thanks to @damir-manapov! * Refactor event storage to prepare for changes in state calculations (PR #3141) +* Set Server header in SynapseRequest (PR #3208) +* Use deferred.addTimeout instead of time_bound_deferred (PR #3127, #3178) +* Use run_in_background in preference to preserve_fn (PR #3140) Changes - Python 3 migration: From 47815edcfae73c5b938f8354853a09c0b80ef27e Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 11 May 2018 00:17:11 +0100 Subject: [PATCH 78/84] ConsentResource to gather policy consent from users Hopefully there are enough comments and docs in this that it makes sense on its own. --- docs/privacy_policy_templates/README.md | 23 ++ docs/privacy_policy_templates/en/1.0.html | 17 ++ docs/privacy_policy_templates/en/success.html | 11 + synapse/app/homeserver.py | 9 + synapse/config/__init__.py | 6 + synapse/config/consent_config.py | 42 ++++ synapse/config/homeserver.py | 8 +- synapse/config/key.py | 10 + synapse/http/server.py | 76 ++++++- synapse/rest/consent/__init__.py | 0 synapse/rest/consent/consent_resource.py | 210 ++++++++++++++++++ synapse/server.py | 3 + synapse/storage/registration.py | 18 ++ .../schema/delta/48/add_user_consent.sql | 18 ++ 14 files changed, 446 insertions(+), 5 deletions(-) create mode 100644 docs/privacy_policy_templates/README.md create mode 100644 docs/privacy_policy_templates/en/1.0.html create mode 100644 docs/privacy_policy_templates/en/success.html create mode 100644 synapse/config/consent_config.py create mode 100644 synapse/rest/consent/__init__.py create mode 100644 synapse/rest/consent/consent_resource.py create mode 100644 synapse/storage/schema/delta/48/add_user_consent.sql diff --git a/docs/privacy_policy_templates/README.md b/docs/privacy_policy_templates/README.md new file mode 100644 index 0000000000..8e91c516b3 --- /dev/null +++ b/docs/privacy_policy_templates/README.md @@ -0,0 +1,23 @@ +If enabling the 'consent' resource in synapse, you will need some templates +for the HTML to be served to the user. This directory contains very simple +examples of the sort of thing that can be done. + +You'll need to add this sort of thing to your homeserver.yaml: + +``` +form_secret: + +user_consent: + template_dir: docs/privacy_policy_templates + default_version: 1.0 +``` + +You should then be able to enable the `consent` resource under a `listener` +entry. For example: + +``` +listeners: + - port: 8008 + resources: + - names: [client, consent] +``` diff --git a/docs/privacy_policy_templates/en/1.0.html b/docs/privacy_policy_templates/en/1.0.html new file mode 100644 index 0000000000..ab8666f0c3 --- /dev/null +++ b/docs/privacy_policy_templates/en/1.0.html @@ -0,0 +1,17 @@ + + + + Matrix.org Privacy policy + + +

+ All your base are belong to us. +

+
+ + + + +
+ + diff --git a/docs/privacy_policy_templates/en/success.html b/docs/privacy_policy_templates/en/success.html new file mode 100644 index 0000000000..d55e90c94f --- /dev/null +++ b/docs/privacy_policy_templates/en/success.html @@ -0,0 +1,11 @@ + + + + Matrix.org Privacy policy + + +

+ Sweet. +

+ + diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index a0e465d644..730271628e 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -41,6 +41,7 @@ from synapse.python_dependencies import CONDITIONAL_REQUIREMENTS, \ from synapse.replication.http import ReplicationRestResource, REPLICATION_PREFIX from synapse.replication.tcp.resource import ReplicationStreamProtocolFactory from synapse.rest import ClientRestResource +from synapse.rest.consent.consent_resource import ConsentResource from synapse.rest.key.v1.server_key_resource import LocalKey from synapse.rest.key.v2 import KeyApiV2Resource from synapse.rest.media.v0.content_repository import ContentRepoResource @@ -182,6 +183,14 @@ class SynapseHomeServer(HomeServer): "/_matrix/client/versions": client_resource, }) + if name == "consent": + consent_resource = ConsentResource(self) + if compress: + consent_resource = gz_wrap(consent_resource) + resources.update({ + "/_matrix/consent": consent_resource, + }) + if name == "federation": resources.update({ FEDERATION_PREFIX: TransportLayerServer(self), diff --git a/synapse/config/__init__.py b/synapse/config/__init__.py index bfebb0f644..f2a5a41e92 100644 --- a/synapse/config/__init__.py +++ b/synapse/config/__init__.py @@ -12,3 +12,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from ._base import ConfigError + +# export ConfigError if somebody does import * +# this is largely a fudge to stop PEP8 moaning about the import +__all__ = ["ConfigError"] diff --git a/synapse/config/consent_config.py b/synapse/config/consent_config.py new file mode 100644 index 0000000000..675fce0911 --- /dev/null +++ b/synapse/config/consent_config.py @@ -0,0 +1,42 @@ +# -*- coding: utf-8 -*- +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ._base import Config + +DEFAULT_CONFIG = """\ +# User Consent configuration +# +# uncomment and configure if enabling the 'consent' resource under 'listeners'. +# +# 'template_dir' gives the location of the templates for the HTML forms. +# This directory should contain one subdirectory per language (eg, 'en', 'fr'), +# and each language directory should contain the policy document (named as +# '.html') and a success page (success.html). +# +# 'default_version' gives the version of the policy document to serve up if +# there is no 'v' parameter. +# +# user_consent: +# template_dir: res/templates/privacy +# default_version: 1.0 +""" + + +class ConsentConfig(Config): + def read_config(self, config): + self.consent_config = config.get("user_consent") + + def default_config(self, **kwargs): + return DEFAULT_CONFIG diff --git a/synapse/config/homeserver.py b/synapse/config/homeserver.py index bf19cfee29..fb6bd3b421 100644 --- a/synapse/config/homeserver.py +++ b/synapse/config/homeserver.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Copyright 2014-2016 OpenMarket Ltd +# Copyright 2018 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,7 +13,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - from .tls import TlsConfig from .server import ServerConfig from .logger import LoggingConfig @@ -37,6 +37,7 @@ from .push import PushConfig from .spam_checker import SpamCheckerConfig from .groups import GroupsConfig from .user_directory import UserDirectoryConfig +from .consent_config import ConsentConfig class HomeServerConfig(TlsConfig, ServerConfig, DatabaseConfig, LoggingConfig, @@ -45,12 +46,13 @@ class HomeServerConfig(TlsConfig, ServerConfig, DatabaseConfig, LoggingConfig, AppServiceConfig, KeyConfig, SAML2Config, CasConfig, JWTConfig, PasswordConfig, EmailConfig, WorkerConfig, PasswordAuthProviderConfig, PushConfig, - SpamCheckerConfig, GroupsConfig, UserDirectoryConfig,): + SpamCheckerConfig, GroupsConfig, UserDirectoryConfig, + ConsentConfig): pass if __name__ == '__main__': import sys sys.stdout.write( - HomeServerConfig().generate_config(sys.argv[1], sys.argv[2])[0] + HomeServerConfig().generate_config(sys.argv[1], sys.argv[2], True)[0] ) diff --git a/synapse/config/key.py b/synapse/config/key.py index 4b8fc063d0..d1382ad9ac 100644 --- a/synapse/config/key.py +++ b/synapse/config/key.py @@ -59,14 +59,20 @@ class KeyConfig(Config): self.expire_access_token = config.get("expire_access_token", False) + # a secret which is used to calculate HMACs for form values, to stop + # falsification of values + self.form_secret = config.get("form_secret", None) + def default_config(self, config_dir_path, server_name, is_generating_file=False, **kwargs): base_key_name = os.path.join(config_dir_path, server_name) if is_generating_file: macaroon_secret_key = random_string_with_symbols(50) + form_secret = '"%s"' % random_string_with_symbols(50) else: macaroon_secret_key = None + form_secret = 'null' return """\ macaroon_secret_key: "%(macaroon_secret_key)s" @@ -74,6 +80,10 @@ class KeyConfig(Config): # Used to enable access token expiration. expire_access_token: False + # a secret which is used to calculate HMACs for form values, to stop + # falsification of values + form_secret: %(form_secret)s + ## Signing Keys ## # Path to the signing key to sign messages with diff --git a/synapse/http/server.py b/synapse/http/server.py index f29e36f490..a38209770d 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -13,7 +13,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +import cgi +from six.moves import http_client from synapse.api.errors import ( cs_exception, SynapseError, CodeMessageException, UnrecognizedRequestError, Codes @@ -44,6 +45,18 @@ import simplejson logger = logging.getLogger(__name__) +HTML_ERROR_TEMPLATE = """ + + + + Error {code} + + +

{msg}

+ + +""" + def wrap_json_request_handler(h): """Wraps a request handler method with exception handling. @@ -104,6 +117,65 @@ def wrap_json_request_handler(h): return wrap_request_handler_with_logging(wrapped_request_handler) +def wrap_html_request_handler(h): + """Wraps a request handler method with exception handling. + + Also adds logging as per wrap_request_handler_with_logging. + + The handler method must have a signature of "handle_foo(self, request)", + where "self" must have a "clock" attribute (and "request" must be a + SynapseRequest). + """ + def wrapped_request_handler(self, request): + d = defer.maybeDeferred(h, self, request) + d.addErrback(_return_html_error, request) + return d + + return wrap_request_handler_with_logging(wrapped_request_handler) + + +def _return_html_error(f, request): + """Sends an HTML error page corresponding to the given failure + + Args: + f (twisted.python.failure.Failure): + request (twisted.web.iweb.IRequest): + """ + if f.check(CodeMessageException): + cme = f.value + code = cme.code + msg = cme.msg + + if isinstance(cme, SynapseError): + logger.info( + "%s SynapseError: %s - %s", request, code, msg + ) + else: + logger.error( + "Failed handle request %r: %s", + request, + f.getTraceback().rstrip(), + ) + else: + code = http_client.INTERNAL_SERVER_ERROR + msg = "Internal server error" + + logger.error( + "Failed handle request %r: %s", + request, + f.getTraceback().rstrip(), + ) + + body = HTML_ERROR_TEMPLATE.format( + code=code, msg=cgi.escape(msg), + ).encode("utf-8") + request.setResponseCode(code) + request.setHeader(b"Content-Type", b"text/html; charset=utf-8") + request.setHeader(b"Content-Length", b"%i" % (len(body),)) + request.write(body) + finish_request(request) + + def wrap_request_handler_with_logging(h): """Wraps a request handler to provide logging and metrics @@ -134,7 +206,7 @@ def wrap_request_handler_with_logging(h): servlet_name = self.__class__.__name__ with request.processing(servlet_name): with PreserveLoggingContext(request_context): - d = h(self, request) + d = defer.maybeDeferred(h, self, request) # record the arrival of the request *after* # dispatching to the handler, so that the handler diff --git a/synapse/rest/consent/__init__.py b/synapse/rest/consent/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/synapse/rest/consent/consent_resource.py b/synapse/rest/consent/consent_resource.py new file mode 100644 index 0000000000..d791302278 --- /dev/null +++ b/synapse/rest/consent/consent_resource.py @@ -0,0 +1,210 @@ +# -*- coding: utf-8 -*- +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from hashlib import sha256 +import hmac +import logging +from os import path +from six.moves import http_client + +import jinja2 +from jinja2 import TemplateNotFound +from twisted.internet import defer +from twisted.web.resource import Resource +from twisted.web.server import NOT_DONE_YET + +from synapse.api.errors import NotFoundError, SynapseError, StoreError +from synapse.config import ConfigError +from synapse.http.server import ( + finish_request, + wrap_html_request_handler, +) +from synapse.http.servlet import parse_string +from synapse.types import UserID + + +# language to use for the templates. TODO: figure this out from Accept-Language +TEMPLATE_LANGUAGE = "en" + +logger = logging.getLogger(__name__) + +# use hmac.compare_digest if we have it (python 2.7.7), else just use equality +if hasattr(hmac, "compare_digest"): + compare_digest = hmac.compare_digest +else: + def compare_digest(a, b): + return a == b + + +class ConsentResource(Resource): + """A twisted Resource to display a privacy policy and gather consent to it + + When accessed via GET, returns the privacy policy via a template. + + When accessed via POST, records the user's consent in the database and + displays a success page. + + The config should include a template_dir setting which contains templates + for the HTML. The directory should contain one subdirectory per language + (eg, 'en', 'fr'), and each language directory should contain the policy + document (named as '.html') and a success page (success.html). + + Both forms take a set of parameters from the browser. For the POST form, + these are normally sent as form parameters (but may be query-params); for + GET requests they must be query params. These are: + + u: the complete mxid, or the localpart of the user giving their + consent. Required for both GET (where it is used as an input to the + template) and for POST (where it is used to find the row in the db + to update). + + h: hmac_sha256(secret, u), where 'secret' is the privacy_secret in the + config file. If it doesn't match, the request is 403ed. + + v: the version of the privacy policy being agreed to. + + For GET: optional, and defaults to whatever was set in the config + file. Used to choose the version of the policy to pick from the + templates directory. + + For POST: required; gives the value to be recorded in the database + against the user. + """ + def __init__(self, hs): + """ + Args: + hs (synapse.server.HomeServer): homeserver + """ + Resource.__init__(self) + + self.hs = hs + self.store = hs.get_datastore() + + # this is required by the request_handler wrapper + self.clock = hs.get_clock() + + consent_config = hs.config.consent_config + if consent_config is None: + raise ConfigError( + "Consent resource is enabled but user_consent section is " + "missing in config file.", + ) + + # daemonize changes the cwd to /, so make the path absolute now. + consent_template_directory = path.abspath( + consent_config["template_dir"], + ) + if not path.isdir(consent_template_directory): + raise ConfigError( + "Could not find template directory '%s'" % ( + consent_template_directory, + ), + ) + + loader = jinja2.FileSystemLoader(consent_template_directory) + self._jinja_env = jinja2.Environment(loader=loader) + + self._default_consent_verison = consent_config["default_version"] + + if hs.config.form_secret is None: + raise ConfigError( + "Consent resource is enabled but form_secret is not set in " + "config file. It should be set to an arbitrary secret string.", + ) + + self._hmac_secret = hs.config.form_secret.encode("utf-8") + + def render_GET(self, request): + self._async_render_GET(request) + return NOT_DONE_YET + + @wrap_html_request_handler + def _async_render_GET(self, request): + """ + Args: + request (twisted.web.http.Request): + """ + + version = parse_string(request, "v", + default=self._default_consent_verison) + username = parse_string(request, "u", required=True) + userhmac = parse_string(request, "h", required=True) + + self._check_hash(username, userhmac) + + try: + self._render_template( + request, "%s.html" % (version,), + user=username, userhmac=userhmac, version=version, + ) + except TemplateNotFound: + raise NotFoundError("Unknown policy version") + + def render_POST(self, request): + self._async_render_POST(request) + return NOT_DONE_YET + + @wrap_html_request_handler + @defer.inlineCallbacks + def _async_render_POST(self, request): + """ + Args: + request (twisted.web.http.Request): + """ + version = parse_string(request, "v", required=True) + username = parse_string(request, "u", required=True) + userhmac = parse_string(request, "h", required=True) + + self._check_hash(username, userhmac) + + if username.startswith('@'): + qualified_user_id = username + else: + qualified_user_id = UserID(username, self.hs.hostname).to_string() + + try: + yield self.store.user_set_consent_version(qualified_user_id, version) + except StoreError as e: + if e.code != 404: + raise + raise NotFoundError("Unknown user") + + try: + self._render_template(request, "success.html") + except TemplateNotFound: + raise NotFoundError("success.html not found") + + def _render_template(self, request, template_name, **template_args): + # get_template checks for ".." so we don't need to worry too much + # about path traversal here. + template_html = self._jinja_env.get_template( + path.join(TEMPLATE_LANGUAGE, template_name) + ) + html_bytes = template_html.render(**template_args).encode("utf8") + + request.setHeader(b"Content-Type", b"text/html; charset=utf-8") + request.setHeader(b"Content-Length", b"%i" % len(html_bytes)) + request.write(html_bytes) + finish_request(request) + + def _check_hash(self, userid, userhmac): + want_mac = hmac.new( + key=self._hmac_secret, + msg=userid, + digestmod=sha256, + ).hexdigest() + + if not compare_digest(want_mac, userhmac): + raise SynapseError(http_client.FORBIDDEN, "HMAC incorrect") diff --git a/synapse/server.py b/synapse/server.py index ebdea6b0c4..21cde5b6fc 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -97,6 +97,9 @@ class HomeServer(object): which must be implemented by the subclass. This code may call any of the required "get" methods on the instance to obtain the sub-dependencies that one requires. + + Attributes: + config (synapse.config.homeserver.HomeserverConfig): """ DEPENDENCIES = [ diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index a50717db2d..6ffc397861 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -286,6 +286,24 @@ class RegistrationStore(RegistrationWorkerStore, "user_set_password_hash", user_set_password_hash_txn ) + def user_set_consent_version(self, user_id, consent_version): + """Updates the user table to record privacy policy consent + + Args: + user_id (str): full mxid of the user to update + consent_version (str): version of the policy the user has consented + to + + Raises: + StoreError(404) if user not found + """ + return self._simple_update_one( + table='users', + keyvalues={'name': user_id, }, + updatevalues={'consent_version': consent_version, }, + desc="user_set_consent_version" + ) + def user_delete_access_tokens(self, user_id, except_token_id=None, device_id=None): """ diff --git a/synapse/storage/schema/delta/48/add_user_consent.sql b/synapse/storage/schema/delta/48/add_user_consent.sql new file mode 100644 index 0000000000..5237491506 --- /dev/null +++ b/synapse/storage/schema/delta/48/add_user_consent.sql @@ -0,0 +1,18 @@ +/* Copyright 2018 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* record the version of the privacy policy the user has consented to + */ +ALTER TABLE users ADD COLUMN consent_version TEXT; From 05ac15ae824cc538b869e3cc8db7af2ac22e6754 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Tue, 15 May 2018 17:01:33 +0100 Subject: [PATCH 79/84] Limit query load of generate_user_daily_visits The aim is to keep track of when it was last called and only query from that point in time --- synapse/app/homeserver.py | 19 ++++-------- synapse/storage/__init__.py | 60 +++++++++++++++++++++++++++++-------- 2 files changed, 53 insertions(+), 26 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index bfc79a5e81..f25eaf9ffc 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -476,23 +476,16 @@ def run(hs): " changes across releases." ) - # def recurring_user_daily_visit_stats(): - def generate_user_daily_visit_stats(): hs.get_datastore().generate_user_daily_visits() - # Since user daily stats are bucketed at midnight UTC, - # and user_ips.last_seen can be updated at any time, it is important to call - # generate_user_daily_visit_stats immediately prior to the day end. Assuming - # an hourly cadence, the simplist way is to allign all calls to the hour - # end - end_of_hour = datetime.datetime.now().replace(microsecond=0, second=0, minute=0) \ - + datetime.timedelta(hours=1) \ - - datetime.timedelta(seconds=10) # Ensure method fires before day transistion + def recurring_user_daily_visit_stats(): + clock.looping_call(generate_user_daily_visit_stats, 60 * 60 * 1000) - time_to_next_hour = end_of_hour - datetime.datetime.now() - clock.call_later(time_to_next_hour.seconds, - clock.looping_call(generate_user_daily_visit_stats, 60 * 60 * 1000)) + # Rather than update on per session basis, batch up the requests. + # If you increase the loop period, the accuracy of user_daily_visits + # table will decrease + clock.looping_call(generate_user_daily_visit_stats, 5 * 60 * 1000) if hs.config.report_stats: logger.info("Scheduling stats reporting for 3 hour intervals") diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 6949876c13..52f176a03c 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -214,6 +214,9 @@ class DataStore(RoomMemberStore, RoomStore, self._stream_order_on_start = self.get_room_max_stream_ordering() self._min_stream_order_on_start = self.get_room_min_stream_ordering() + # Used in _generate_user_daily_visits to keep track of progress + self._last_user_visit_update = self._get_start_of_day() + super(DataStore, self).__init__(db_conn, hs) def take_presence_startup_info(self): @@ -348,27 +351,58 @@ class DataStore(RoomMemberStore, RoomStore, return self.runInteraction("count_r30_users", _count_r30_users) + def _get_start_of_day(self): + """ + Returns millisecond unixtime for start of UTC day. + """ + now = datetime.datetime.utcnow() + today_start = datetime.datetime(now.year, now.month, + now.day, tzinfo=tz.tzutc()) + return int(time.mktime(today_start.timetuple())) * 1000 + def generate_user_daily_visits(self): """ Generates daily visit data for use in cohort/ retention analysis """ def _generate_user_daily_visits(txn): + logger.info("Calling _generate_user_daily_visits") + today_start = self._get_start_of_day() + a_day_in_milliseconds = 24 * 60 * 60 * 1000 - # determine timestamp of the day start - now = datetime.datetime.utcnow() - today_start = datetime.datetime(now.year, now.month, - now.day, tzinfo=tz.tzutc()) - today_start_time = int(time.mktime(today_start.timetuple())) * 1000 - logger.info(today_start_time) sql = """ INSERT INTO user_daily_visits (user_id, device_id, timestamp) - SELECT user_id, device_id, ? - FROM user_ips AS u - LEFT JOIN user_daily_visits USING (user_id, device_id) - WHERE last_seen > ? AND timestamp IS NULL - GROUP BY user_id, device_id; - """ - txn.execute(sql, (today_start_time, today_start_time)) + SELECT u.user_id, u.device_id, ? + FROM user_ips AS u + LEFT JOIN ( + SELECT user_id, device_id, timestamp FROM user_daily_visits + WHERE timestamp IS ? + ) udv + ON u.user_id = udv.user_id AND u.device_id=udv.device_id + WHERE last_seen > ? AND last_seen <= ? AND udv.timestamp IS NULL + """ + + # This means that the day has rolled over but there could still + # be entries from the previous day. There is an edge case + # where if the user logs in at 23:59 and overwrites their + # last_seen at 00:01 then they will not be counted in the + # previous day's stats - it is important that the query is run + # to minimise this case. + if today_start > self._last_user_visit_update: + yesterday_start = today_start - a_day_in_milliseconds + txn.execute(sql, (yesterday_start, yesterday_start, + self._last_user_visit_update, today_start)) + self._last_user_visit_update = today_start + + txn.execute(sql, (today_start, today_start, + self._last_user_visit_update, + today_start + a_day_in_milliseconds)) + # Update _last_user_visit_update to now. The reason to do this + # rather just clamping to the beginning of the day is to limit + # the size of the join - meaning that the query can be run more + # frequently + + now = datetime.datetime.utcnow() + self._last_user_visit_update = int(time.mktime(now.timetuple())) * 1000 return self.runInteraction("generate_user_daily_visits", _generate_user_daily_visits) From c92a8aa5780a71b5ab0227f204606004b16ff7fb Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Tue, 15 May 2018 17:31:11 +0100 Subject: [PATCH 80/84] pep8 --- synapse/app/homeserver.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index f25eaf9ffc..41ee8b0a69 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -17,7 +17,6 @@ import gc import logging import os import sys -import datetime import synapse import synapse.config.logger From 31c2502ca8d74797f1eae553690830b8c132aed9 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Wed, 16 May 2018 09:46:43 +0100 Subject: [PATCH 81/84] style and further contraining query --- synapse/storage/__init__.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 52f176a03c..6f324e075f 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -368,6 +368,7 @@ class DataStore(RoomMemberStore, RoomStore, logger.info("Calling _generate_user_daily_visits") today_start = self._get_start_of_day() a_day_in_milliseconds = 24 * 60 * 60 * 1000 + now = self.clock.time_msec() sql = """ INSERT INTO user_daily_visits (user_id, device_id, timestamp) @@ -386,23 +387,26 @@ class DataStore(RoomMemberStore, RoomStore, # where if the user logs in at 23:59 and overwrites their # last_seen at 00:01 then they will not be counted in the # previous day's stats - it is important that the query is run - # to minimise this case. + # often to minimise this case. if today_start > self._last_user_visit_update: yesterday_start = today_start - a_day_in_milliseconds - txn.execute(sql, (yesterday_start, yesterday_start, - self._last_user_visit_update, today_start)) + txn.execute(sql, ( + yesterday_start, yesterday_start, + self._last_user_visit_update, today_start + )) self._last_user_visit_update = today_start - txn.execute(sql, (today_start, today_start, - self._last_user_visit_update, - today_start + a_day_in_milliseconds)) + txn.execute(sql, ( + today_start, today_start, + self._last_user_visit_update, + now + )) # Update _last_user_visit_update to now. The reason to do this # rather just clamping to the beginning of the day is to limit # the size of the join - meaning that the query can be run more # frequently - now = datetime.datetime.utcnow() - self._last_user_visit_update = int(time.mktime(now.timetuple())) * 1000 + self._last_user_visit_update = now return self.runInteraction("generate_user_daily_visits", _generate_user_daily_visits) From a2204cc9cc3f9edb9ae6c1f564b8f315e62d1978 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Wed, 16 May 2018 09:47:20 +0100 Subject: [PATCH 82/84] remove unused method recurring_user_daily_visit_stats --- synapse/app/homeserver.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 41ee8b0a69..a4648b84d5 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -478,9 +478,6 @@ def run(hs): def generate_user_daily_visit_stats(): hs.get_datastore().generate_user_daily_visits() - def recurring_user_daily_visit_stats(): - clock.looping_call(generate_user_daily_visit_stats, 60 * 60 * 1000) - # Rather than update on per session basis, batch up the requests. # If you increase the loop period, the accuracy of user_daily_visits # table will decrease From be11a02c4f854452687f0f54492bc53b4827637d Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Wed, 16 May 2018 10:45:40 +0100 Subject: [PATCH 83/84] remove empty line --- synapse/storage/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 6f324e075f..4551cf8774 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -405,7 +405,6 @@ class DataStore(RoomMemberStore, RoomStore, # rather just clamping to the beginning of the day is to limit # the size of the join - meaning that the query can be run more # frequently - self._last_user_visit_update = now return self.runInteraction("generate_user_daily_visits", From 3c099219e0506f81fc333ff9d27cc3af5c365a2c Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Wed, 16 May 2018 15:44:59 +0100 Subject: [PATCH 84/84] bump version and changelog for 0.29.0 --- CHANGES.rst | 4 ++++ synapse/__init__.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 78d015db9a..b769b0f046 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,7 @@ +Changes in synapse v0.29.0 (2018-05-16) +=========================================== + + Changes in synapse v0.29.0-rc1 (2018-05-14) =========================================== diff --git a/synapse/__init__.py b/synapse/__init__.py index 69c081eb37..d94c20505e 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.29.0-rc1" +__version__ = "0.29.0"