Federated homeservers in Docker for e2e tests

package.json

@@ -1,7 +1,6 @@
 {
   "name": "matrix-public-archive",
   "version": "0.0.0",
-  "type": "module",
   "scripts": {
     "start-dev": "node server/start-dev.js",
     "lint": "eslint **/*.js",
@@ -29,6 +28,7 @@
     "express": "^4.17.2",
     "hydrogen-view-sdk": "^0.0.4",
     "linkedom": "^0.14.1",
+    "matrix-bot-sdk": "^0.5.19",
     "matrix-js-sdk": "^15.5.2",
     "matrix-public-archive-shared": "file:./shared/",
     "node-fetch": "^2.6.7",

server/fetch-events-for-timestamp.js

@@ -8,7 +8,8 @@ const fetchEndpoint = require('./lib/fetch-endpoint');
 const { matrixServerUrl } = require('../config.json');
 assert(matrixServerUrl);
 
-async function fetchEventsForTimestamp(roomId, ts) {
+async function fetchEventsForTimestamp(accessToken, roomId, ts) {
+  assert(accessToken);
   assert(roomId);
   assert(ts);
 
@@ -16,7 +17,9 @@ async function fetchEventsForTimestamp(roomId, ts) {
     matrixServerUrl,
     `_matrix/client/unstable/org.matrix.msc3030/rooms/${roomId}/timestamp_to_event?ts=${ts}&dir=b`
   );
-  const timestampToEventResData = await fetchEndpoint(timestampToEventEndpoint);
+  const timestampToEventResData = await fetchEndpoint(timestampToEventEndpoint, {
+    accessToken,
+  });
   const eventIdForTimestamp = timestampToEventResData.event_id;
   assert(eventIdForTimestamp);
   console.log('eventIdForTimestamp', eventIdForTimestamp);
@@ -25,14 +28,18 @@ async function fetchEventsForTimestamp(roomId, ts) {
     matrixServerUrl,
     `_matrix/client/r0/rooms/${roomId}/context/${eventIdForTimestamp}?limit=0`
   );
-  const contextResData = await fetchEndpoint(contextEndpoint);
+  const contextResData = await fetchEndpoint(contextEndpoint, {
+    accessToken,
+  });
   //console.log('contextResData', contextResData);
 
   const messagesEndpoint = urlJoin(
     matrixServerUrl,
     `_matrix/client/r0/rooms/${roomId}/messages?from=${contextResData.start}&limit=50&filter={"lazy_load_members":true,"include_redundant_members":true}`
   );
-  const messageResData = await fetchEndpoint(messagesEndpoint);
+  const messageResData = await fetchEndpoint(messagesEndpoint, {
+    accessToken,
+  });
 
   //console.log('messageResData.state', messageResData.state);
   const stateEventMap = {};

server/fetch-room-data.js

@@ -8,7 +8,10 @@ const fetchEndpoint = require('./lib/fetch-endpoint');
 const { matrixServerUrl } = require('../config.json');
 assert(matrixServerUrl);
 
-async function fetchRoomData(roomId) {
+async function fetchRoomData(accessToken, roomId) {
+  assert(accessToken);
+  assert(roomId);
+
   const stateNameEndpoint = urlJoin(
     matrixServerUrl,
     `_matrix/client/r0/rooms/${roomId}/state/m.room.name`
@@ -19,8 +22,12 @@ async function fetchRoomData(roomId) {
   );
 
   const [stateNameResDataOutcome, stateAvatarResDataOutcome] = await Promise.allSettled([
-    fetchEndpoint(stateNameEndpoint),
+    fetchEndpoint(stateNameEndpoint, {
-    fetchEndpoint(stateAvatarEndpoint),
+      accessToken,
+    }),
+    fetchEndpoint(stateAvatarEndpoint, {
+      accessToken,
+    }),
   ]);
 
   let name;

server/lib/fetch-endpoint.js

@@ -1,11 +1,7 @@
 'use strict';
 
-const assert = require('assert');
 const fetch = require('node-fetch');
 
-const { matrixAccessToken } = require('../../secrets.json');
-assert(matrixAccessToken);
-
 class HTTPResponseError extends Error {
   constructor(response, responseText, ...args) {
     super(
@@ -26,11 +22,11 @@ const checkResponseStatus = async (response) => {
   }
 };
 
-async function fetchEndpoint(endpoint) {
+async function fetchEndpoint(endpoint, { accessToken }) {
   const res = await fetch(endpoint, {
     headers: {
       'Content-Type': 'application/json',
-      Authorization: `Bearer ${matrixAccessToken}`,
+      Authorization: `Bearer ${accessToken}`,
     },
   });
   await checkResponseStatus(res);

server/routes/install-routes.js

@@ -15,6 +15,9 @@ const config = require('../../config.json');
 const basePath = config.basePath;
 assert(basePath);
 
+const { matrixAccessToken } = require('../../secrets.json');
+assert(matrixAccessToken);
+
 function parseArchiveRangeFromReq(req) {
   const yyyy = parseInt(req.params.yyyy, 10);
   // Month is the only zero-based index in this group
@@ -121,8 +124,8 @@ function installRoutes(app) {
       //const aroundId = req.query.at;
 
       const [roomData, { events, stateEventMap }] = await Promise.all([
-        fetchRoomData(roomIdOrAlias),
+        fetchRoomData(matrixAccessToken, roomIdOrAlias),
-        fetchEventsForTimestamp(roomIdOrAlias, fromTimestamp),
+        fetchEventsForTimestamp(matrixAccessToken, roomIdOrAlias, fromTimestamp),
       ]);
 
       const hydrogenHtmlOutput = await renderHydrogenToString({

test/README.md

@@ -0,0 +1,7 @@
+```
+docker build -t matrix-public-archive-test-homeserver -f test/dockerfiles/Synapse.Dockerfile test/dockerfiles/
+```
+
+```
+docker-compose -f test/docker-compose.yml up -d --no-recreate
+```

test/docker-compose.yml

@@ -0,0 +1,16 @@
+version: '3'
+
+services:
+  hs1:
+    image: matrix-public-archive-test-homeserver:latest
+    ports:
+      - '11008:8008'
+    environment:
+      - SERVER_NAME=hs1
+
+  hs2:
+    image: matrix-public-archive-test-homeserver:latest
+    ports:
+      - '12008:8008'
+    environment:
+      - SERVER_NAME=hs2

test/dockerfiles/Synapse.Dockerfile

@@ -0,0 +1,36 @@
+# A dockerfile which builds an image suitable for testing Synapse under
+# complement.
+#
+# Currently this is based on the published 'synapse:latest' image -- ie, the
+# most recent Synapse release.
+#
+# Also... none of the tests seem to pass yet. They do run though.
+#
+# To use it:
+#
+# (cd dockerfiles && docker build -t complement-synapse -f Synapse.Dockerfile .)
+# COMPLEMENT_BASE_IMAGE=complement-synapse go test -v ./tests
+
+ARG SYNAPSE_VERSION=latest
+
+FROM matrixdotorg/synapse:${SYNAPSE_VERSION}
+
+ENV SERVER_NAME=localhost
+
+COPY synapse/* /conf/
+COPY keys/* /ca/
+
+# SSL key for the server (can't make the cert until we know the server name)
+RUN openssl genrsa -out /conf/server.tls.key 2048
+
+# generate a signing key
+RUN generate_signing_key.py -o /conf/server.signing.key
+
+WORKDIR /data
+
+EXPOSE 8008 8448
+
+ENTRYPOINT ["/conf/start.sh"]
+
+HEALTHCHECK --start-period=5s --interval=1s --timeout=1s \
+  CMD curl -fSs http://localhost:8008/health || exit 1

test/dockerfiles/keys/README.md

@@ -0,0 +1,9 @@
+This directory contains a key and certificate for a dummy Certificate
+Authority. This is used to create certificates for the servers under test.
+
+The files were generated with:
+
+```
+openssl genrsa -out ca.key 2048
+openssl req -new -x509 -key ca.key -days 3650 -subj "/C=GB/ST=London/O=matrix.org/CN=Complement CA" -out ca.crt
+```

test/dockerfiles/keys/ca.crt

@@ -0,0 +1,21 @@
+-----BEGIN CERTIFICATE-----
+MIIDdzCCAl+gAwIBAgIUDZUl0q5DMzWkd/fev4gJurDpuH8wDQYJKoZIhvcNAQEL
+BQAwSzELMAkGA1UEBhMCR0IxDzANBgNVBAgMBkxvbmRvbjETMBEGA1UECgwKbWF0
+cml4Lm9yZzEWMBQGA1UEAwwNQ29tcGxlbWVudCBDQTAeFw0yMDA4MDUxMjAyMDRa
+Fw0zMDA4MDMxMjAyMDRaMEsxCzAJBgNVBAYTAkdCMQ8wDQYDVQQIDAZMb25kb24x
+EzARBgNVBAoMCm1hdHJpeC5vcmcxFjAUBgNVBAMMDUNvbXBsZW1lbnQgQ0EwggEi
+MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC9LfS/T0Y72b9j+M7/43Hry+Cf
+UUt7iym5z14su8cve545ceT3Y1SRwrGWRy8GuvhgQyVTzvgslwdsLYwVEaK1ZsWQ
+7l+UBD128HWCSo/OFMV3j+allMzj4PO3PnMcU1cq6HuTlYeAFlOelLB7FUGFmOzc
+c829XHWvDD/Hjz+SQ4E11nftHyu6Chw+Razfcftzp+qLjozjHeXMLI09peiy65zp
+DGYhiB78vTCjtioNlMZ5bTrI5PAG3O0nY0NAiAqrGgijpbJdYigf2vcJ8UQ6e0LH
+0nTIhITxRd+hN+FGbsLe+OtzaaEkgdtxQrtjAq7wjsxbchNDjnqh/eTJwBaPAgMB
+AAGjUzBRMB0GA1UdDgQWBBRNM7hzjkBvs7Rmw4I7g4wpEy53nDAfBgNVHSMEGDAW
+gBRNM7hzjkBvs7Rmw4I7g4wpEy53nDAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3
+DQEBCwUAA4IBAQAOfCmPTM9NFPbVwu003kbOPfbtHULlDyMvx4LRiYmpvxfpFjGy
+Xp2p/ZqzFHvyT5QcCFrkOoQlX8tIzr8KZqbemf2aWfdRzqT1VG7ADErNfQ2JOjLo
+jptliOWcnknmCi8elGjBxVmAqWfDyQKYeiK2AyrJywMLr45UlrxRh03d81TaTRYk
+rRHMMKJ3iKb8HQKlrfjQrYCCCSzNkp0RP/TyhpB4qIZDg9AbhUlXn24bQ/Gpq9Hw
+pGmYXObJ8Uid7SICxxeFm42Nc8Orke+yw77glcQEumLJM0TqzR1UV3aTyE0qrXeG
+R8e6WTemuXtcDEeo1MzCwW/75Sd/1exbs2dW
+-----END CERTIFICATE-----

test/dockerfiles/keys/ca.key

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpQIBAAKCAQEAvS30v09GO9m/Y/jO/+Nx68vgn1FLe4spuc9eLLvHL3ueOXHk
+92NUkcKxlkcvBrr4YEMlU874LJcHbC2MFRGitWbFkO5flAQ9dvB1gkqPzhTFd4/m
+pZTM4+Dztz5zHFNXKuh7k5WHgBZTnpSwexVBhZjs3HPNvVx1rww/x48/kkOBNdZ3
+7R8rugocPkWs33H7c6fqi46M4x3lzCyNPaXosuuc6QxmIYge/L0wo7YqDZTGeW06
+yOTwBtztJ2NDQIgKqxoIo6WyXWIoH9r3CfFEOntCx9J0yISE8UXfoTfhRm7C3vjr
+c2mhJIHbcUK7YwKu8I7MW3ITQ456of3kycAWjwIDAQABAoIBAQCt+ctw/IiMlg2q
+9K/5J8Qmo1IpcevG7A4tcQUcgnwFsNur3JBC6jhl6/TBTSl0ORIeBOIZFueoqiQL
+Wj/PSG6oAKGETZPcXaHg9fY7IjLpxLRs5gupDxT2JHQ9etMFo6cwMKkk6oi4RNAV
+sIw0RTICmMaLXnxAvv0DRHKLJbPC33gvJCwXg4PJ8JWkrfn6EdExWTT5V6OSBJW3
+W7hwUp7cZ03yXZg8KLt8A3ZwFA1jSNtgISbGojcjV++ylK8xMV7NOVvBwcGQHOCi
+pwvjoHMc2SqpssgsyHPg5s/ZvgNWqMBQms24Sb0tAS3HASiPjL72Oc69kuqTPfbN
+Pn24VDBBAoGBAOpAtH/kZOGBe/NEQWUrpnseteQE/482udWt9+3H3VPW8cCYzVWf
+xV/anj18wtx+nxZgMtu6p5NiU2Tx1OQLOJElKPqd+MOcYeDm+IutRnNETP4i+2P7
+8HXUAEySZCz4W2ettznnMGHKINV2LEv4v3yMtvVPx/GtTLk7m8axSfEfAoGBAM6+
+CCwpUQwKACQClwNkmrvpWmuj+XkiXJr6rnGNZ2akPSpz7YAbre9kDCGYyo3gsKmP
+xmRs6dbHWqbSyCxu+zhjgqGYKsyLcAUHxGDeH2q8Cf8Bf6Jw6dtW5CHka4QIbq7W
+hAFFCXTPQM59cOjxbCzQJwb8by7bChhpALVl/PyRAoGAHE4rQVTot7L8tj1mJYJr
+wG7Kjb3o33Y+aEp4Ldw7qzjKx3nvPcxrfhSdkFJ1/UyzjnGbU5+vRgjYiNcL4W5V
+rHwwAnB6MbDAxvtBxybt31m+b+rsM32q5aHzU01RG3n7GYENJcsAtqBE7WY1Hgz1
+nmgmZ2f3ggVtwugn/49GnwMCgYEAmnOr5Vokm2rpOf3ZVgaV9ubZz2rcWNwL9Rhs
+tJagdjUjIREkkL2dDyheS8c7JA2FzdeErsgTVorutYzpURDEjBcDo7fr2Y4ebROl
+aXzoWbQLf8Bd2zyh6WSb4JomnKg0EFhgnBY4f1TKpLlgXhbm6v/C+FwksAuiGiRw
+DBUakJECgYEAmHcPSXefohshJIflTs8u/M4FMt9scJMXZuOp6npLOKIZp1KuxTst
+Kh3GCI51kJI68XvXxPNrNAQ6ay1Ayp/CwCnlkGYTJeACno4u94fgbQBsRLksRM6g
+JvQZtOvi5OqrnVz2iTWJhtInV/dIbUtv+yJ9hYOa7AtP3AhiDVePvts=
+-----END RSA PRIVATE KEY-----

test/dockerfiles/synapse/homeserver.yaml

@@ -0,0 +1,103 @@
+## Server ##
+
+server_name: SERVER_NAME
+log_config: /conf/log_config.yaml
+report_stats: False
+signing_key_path: /conf/server.signing.key
+trusted_key_servers: []
+enable_registration: true
+
+## Listeners ##
+
+tls_certificate_path: /conf/server.tls.crt
+tls_private_key_path: /conf/server.tls.key
+bcrypt_rounds: 4
+registration_shared_secret: matrix_public_archive
+
+listeners:
+  - port: 8448
+    bind_addresses: ['::']
+    type: http
+    tls: true
+    resources:
+      - names: [federation]
+
+  - port: 8008
+    bind_addresses: ['::']
+    type: http
+
+    resources:
+      - names: [client]
+
+## Database ##
+
+database:
+  name: 'sqlite3'
+  args:
+    # We avoid /data, as it is a volume and is not transferred when the container is committed,
+    # which is a fundamental necessity in complement.
+    database: '/conf/homeserver.db'
+
+## Federation ##
+
+# disable verification of federation certificates
+#
+# TODO: this is temporary until https://github.com/matrix-org/complement/pull/28 lands and
+# allows homeservers spun up by complement access to the complement CA certificate to trust
+federation_verify_certificates: false
+
+# trust certs signed by the dummy CA
+federation_custom_ca_list:
+  - /ca/ca.crt
+
+# unblacklist RFC1918 addresses
+ip_range_blacklist: []
+
+# Disable server rate-limiting
+rc_federation:
+  window_size: 1000
+  sleep_limit: 10
+  sleep_delay: 500
+  reject_limit: 99999
+  concurrent: 3
+
+rc_message:
+  per_second: 9999
+  burst_count: 9999
+
+rc_registration:
+  per_second: 9999
+  burst_count: 9999
+
+rc_login:
+  address:
+    per_second: 9999
+    burst_count: 9999
+  account:
+    per_second: 9999
+    burst_count: 9999
+  failed_attempts:
+    per_second: 9999
+    burst_count: 9999
+
+rc_admin_redaction:
+  per_second: 9999
+  burst_count: 9999
+
+rc_joins:
+  local:
+    per_second: 9999
+    burst_count: 9999
+  remote:
+    per_second: 9999
+    burst_count: 9999
+
+federation_rr_transactions_per_room_per_second: 9999
+
+## Experimental Features ##
+
+experimental_features:
+  # Enable history backfilling support
+  msc2716_enabled: true
+  # Enable jump to date endpoint
+  msc3030_enabled: true

test/dockerfiles/synapse/log_config.yaml

@@ -0,0 +1,24 @@
+version: 1
+
+formatters:
+  precise:
+   format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s'
+
+filters:
+  context:
+    (): synapse.logging.context.LoggingContextFilter
+    request: ""
+
+handlers:
+  console:
+    class: logging.StreamHandler
+    formatter: precise
+    filters: [context]
+    # log to stdout, for easier use with 'docker logs'
+    stream: 'ext://sys.stdout'
+
+root:
+    level: INFO
+    handlers: [console]
+
+disable_existing_loggers: false

test/dockerfiles/synapse/start.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+
+set -e
+
+sed -i "s/SERVER_NAME/${SERVER_NAME}/g" /conf/homeserver.yaml
+
+# generate an ssl cert for the server, signed by our dummy CA
+openssl req -new -key /conf/server.tls.key -out /conf/server.tls.csr \
+  -subj "/CN=${SERVER_NAME}"
+openssl x509 -req -in /conf/server.tls.csr \
+  -CA /ca/ca.crt -CAkey /ca/ca.key -set_serial 1 \
+  -out /conf/server.tls.crt
+
+exec python -m synapse.app.homeserver -c /conf/homeserver.yaml "$@"
+

test/e2e-tests.js

@@ -1,15 +1,80 @@
 'use strict';
 
 const assert = require('assert');
-const sdk = require('matrix-js-sdk');
+const urlJoin = require('url-join');
+const { MatrixAuth } = require('matrix-bot-sdk');
+
+const fetchEndpoint = require('../server/lib/fetch-endpoint');
 
 const config = require('../config');
-assert(config.testMatrixServerUrl);
+assert(config.testMatrixServerUrl1);
+assert(config.testMatrixServerUrl2);
 
-const client = sdk.createClient(config.testMatrixServerUrl);
+async function getTestClientForHs(testMatrixServerUrl) {
+  const auth = new MatrixAuth(testMatrixServerUrl);
+
+  const client = await auth.passwordRegister(
+    `user-${Math.floor(Math.random() * 1000000000)}`,
+    'password'
+  );
+
+  return client;
+}
+
+async function createTestRoom(client) {
+  const roomId = await client.createRoom({
+    preset: 'public_chat',
+    name: 'the hangout spot',
+  });
+
+  let eventIds = [];
+  for (let i = 0; i < 100; i++) {
+    const eventId = await client.sendMessage(roomId, {
+      msgtype: 'm.text',
+      body: `${client.homeserverUrl} - message${i}`,
+    });
+    eventIds.push(eventId);
+  }
+
+  return {
+    roomId,
+    eventIds,
+  };
+}
 
 describe('matrix-public-archive', () => {
   it('asdf', async () => {
-    await client.register(`user-${Math.floor(Math.random() * 1000000000)}`, 'awfe');
+    const hs1Client = await getTestClientForHs(config.testMatrixServerUrl1);
+    const hs2Client = await getTestClientForHs(config.testMatrixServerUrl2);
+
+    const { roomId: hs1RoomId, eventIds: room1EventIds } = await createTestRoom(hs1Client);
+    const { roomId: hs2RoomId, eventIds: room2EventIds } = await createTestRoom(hs2Client);
+
+    console.log('hs1RoomId', hs1RoomId, room1EventIds);
+    console.log('hs2RoomId', hs2RoomId, room2EventIds);
+
+    try {
+      await hs1Client.joinRoom(hs2RoomId, 'hs2');
+    } catch (err) {
+      throw new Error(
+        `Stub error to stop matrix-bot-sdk from logging the response statusCode=${
+          err.statusCode
+        } body=${JSON.stringify(err.body)}`
+      );
+    }
+
+    const messagesEndpoint = urlJoin(
+      hs1Client.homeserverUrl,
+      `_matrix/client/r0/rooms/${hs2RoomId}/messages?limit=5&dir=b`
+    );
+    const messageResData = await fetchEndpoint(messagesEndpoint, {
+      accessToken: hs1Client.accessToken,
+    });
+    console.log(messageResData);
+    console.log(
+      messageResData.chunk.map((event) => {
+        return `${event.event_id} (${event.type}) - ${event.content.body}`;
+      })
+    );
   });
 });