Federated homeservers in Docker for e2e tests

This commit is contained in:
Eric Eastwood 2022-02-22 20:25:24 -06:00
parent aea382e4f8
commit 0f493a241e
16 changed files with 1020 additions and 67 deletions

711
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@ -1,7 +1,6 @@
{ {
"name": "matrix-public-archive", "name": "matrix-public-archive",
"version": "0.0.0", "version": "0.0.0",
"type": "module",
"scripts": { "scripts": {
"start-dev": "node server/start-dev.js", "start-dev": "node server/start-dev.js",
"lint": "eslint **/*.js", "lint": "eslint **/*.js",
@ -29,6 +28,7 @@
"express": "^4.17.2", "express": "^4.17.2",
"hydrogen-view-sdk": "^0.0.4", "hydrogen-view-sdk": "^0.0.4",
"linkedom": "^0.14.1", "linkedom": "^0.14.1",
"matrix-bot-sdk": "^0.5.19",
"matrix-js-sdk": "^15.5.2", "matrix-js-sdk": "^15.5.2",
"matrix-public-archive-shared": "file:./shared/", "matrix-public-archive-shared": "file:./shared/",
"node-fetch": "^2.6.7", "node-fetch": "^2.6.7",

View File

@ -8,7 +8,8 @@ const fetchEndpoint = require('./lib/fetch-endpoint');
const { matrixServerUrl } = require('../config.json'); const { matrixServerUrl } = require('../config.json');
assert(matrixServerUrl); assert(matrixServerUrl);
async function fetchEventsForTimestamp(roomId, ts) { async function fetchEventsForTimestamp(accessToken, roomId, ts) {
assert(accessToken);
assert(roomId); assert(roomId);
assert(ts); assert(ts);
@ -16,7 +17,9 @@ async function fetchEventsForTimestamp(roomId, ts) {
matrixServerUrl, matrixServerUrl,
`_matrix/client/unstable/org.matrix.msc3030/rooms/${roomId}/timestamp_to_event?ts=${ts}&dir=b` `_matrix/client/unstable/org.matrix.msc3030/rooms/${roomId}/timestamp_to_event?ts=${ts}&dir=b`
); );
const timestampToEventResData = await fetchEndpoint(timestampToEventEndpoint); const timestampToEventResData = await fetchEndpoint(timestampToEventEndpoint, {
accessToken,
});
const eventIdForTimestamp = timestampToEventResData.event_id; const eventIdForTimestamp = timestampToEventResData.event_id;
assert(eventIdForTimestamp); assert(eventIdForTimestamp);
console.log('eventIdForTimestamp', eventIdForTimestamp); console.log('eventIdForTimestamp', eventIdForTimestamp);
@ -25,14 +28,18 @@ async function fetchEventsForTimestamp(roomId, ts) {
matrixServerUrl, matrixServerUrl,
`_matrix/client/r0/rooms/${roomId}/context/${eventIdForTimestamp}?limit=0` `_matrix/client/r0/rooms/${roomId}/context/${eventIdForTimestamp}?limit=0`
); );
const contextResData = await fetchEndpoint(contextEndpoint); const contextResData = await fetchEndpoint(contextEndpoint, {
accessToken,
});
//console.log('contextResData', contextResData); //console.log('contextResData', contextResData);
const messagesEndpoint = urlJoin( const messagesEndpoint = urlJoin(
matrixServerUrl, matrixServerUrl,
`_matrix/client/r0/rooms/${roomId}/messages?from=${contextResData.start}&limit=50&filter={"lazy_load_members":true,"include_redundant_members":true}` `_matrix/client/r0/rooms/${roomId}/messages?from=${contextResData.start}&limit=50&filter={"lazy_load_members":true,"include_redundant_members":true}`
); );
const messageResData = await fetchEndpoint(messagesEndpoint); const messageResData = await fetchEndpoint(messagesEndpoint, {
accessToken,
});
//console.log('messageResData.state', messageResData.state); //console.log('messageResData.state', messageResData.state);
const stateEventMap = {}; const stateEventMap = {};

View File

@ -8,7 +8,10 @@ const fetchEndpoint = require('./lib/fetch-endpoint');
const { matrixServerUrl } = require('../config.json'); const { matrixServerUrl } = require('../config.json');
assert(matrixServerUrl); assert(matrixServerUrl);
async function fetchRoomData(roomId) { async function fetchRoomData(accessToken, roomId) {
assert(accessToken);
assert(roomId);
const stateNameEndpoint = urlJoin( const stateNameEndpoint = urlJoin(
matrixServerUrl, matrixServerUrl,
`_matrix/client/r0/rooms/${roomId}/state/m.room.name` `_matrix/client/r0/rooms/${roomId}/state/m.room.name`
@ -19,8 +22,12 @@ async function fetchRoomData(roomId) {
); );
const [stateNameResDataOutcome, stateAvatarResDataOutcome] = await Promise.allSettled([ const [stateNameResDataOutcome, stateAvatarResDataOutcome] = await Promise.allSettled([
fetchEndpoint(stateNameEndpoint), fetchEndpoint(stateNameEndpoint, {
fetchEndpoint(stateAvatarEndpoint), accessToken,
}),
fetchEndpoint(stateAvatarEndpoint, {
accessToken,
}),
]); ]);
let name; let name;

View File

@ -1,11 +1,7 @@
'use strict'; 'use strict';
const assert = require('assert');
const fetch = require('node-fetch'); const fetch = require('node-fetch');
const { matrixAccessToken } = require('../../secrets.json');
assert(matrixAccessToken);
class HTTPResponseError extends Error { class HTTPResponseError extends Error {
constructor(response, responseText, ...args) { constructor(response, responseText, ...args) {
super( super(
@ -26,11 +22,11 @@ const checkResponseStatus = async (response) => {
} }
}; };
async function fetchEndpoint(endpoint) { async function fetchEndpoint(endpoint, { accessToken }) {
const res = await fetch(endpoint, { const res = await fetch(endpoint, {
headers: { headers: {
'Content-Type': 'application/json', 'Content-Type': 'application/json',
Authorization: `Bearer ${matrixAccessToken}`, Authorization: `Bearer ${accessToken}`,
}, },
}); });
await checkResponseStatus(res); await checkResponseStatus(res);

View File

@ -15,6 +15,9 @@ const config = require('../../config.json');
const basePath = config.basePath; const basePath = config.basePath;
assert(basePath); assert(basePath);
const { matrixAccessToken } = require('../../secrets.json');
assert(matrixAccessToken);
function parseArchiveRangeFromReq(req) { function parseArchiveRangeFromReq(req) {
const yyyy = parseInt(req.params.yyyy, 10); const yyyy = parseInt(req.params.yyyy, 10);
// Month is the only zero-based index in this group // Month is the only zero-based index in this group
@ -121,8 +124,8 @@ function installRoutes(app) {
//const aroundId = req.query.at; //const aroundId = req.query.at;
const [roomData, { events, stateEventMap }] = await Promise.all([ const [roomData, { events, stateEventMap }] = await Promise.all([
fetchRoomData(roomIdOrAlias), fetchRoomData(matrixAccessToken, roomIdOrAlias),
fetchEventsForTimestamp(roomIdOrAlias, fromTimestamp), fetchEventsForTimestamp(matrixAccessToken, roomIdOrAlias, fromTimestamp),
]); ]);
const hydrogenHtmlOutput = await renderHydrogenToString({ const hydrogenHtmlOutput = await renderHydrogenToString({

7
test/README.md Normal file
View File

@ -0,0 +1,7 @@
```
docker build -t matrix-public-archive-test-homeserver -f test/dockerfiles/Synapse.Dockerfile test/dockerfiles/
```
```
docker-compose -f test/docker-compose.yml up -d --no-recreate
```

16
test/docker-compose.yml Normal file
View File

@ -0,0 +1,16 @@
version: '3'
services:
hs1:
image: matrix-public-archive-test-homeserver:latest
ports:
- '11008:8008'
environment:
- SERVER_NAME=hs1
hs2:
image: matrix-public-archive-test-homeserver:latest
ports:
- '12008:8008'
environment:
- SERVER_NAME=hs2

View File

@ -0,0 +1,36 @@
# A dockerfile which builds an image suitable for testing Synapse under
# complement.
#
# Currently this is based on the published 'synapse:latest' image -- ie, the
# most recent Synapse release.
#
# Also... none of the tests seem to pass yet. They do run though.
#
# To use it:
#
# (cd dockerfiles && docker build -t complement-synapse -f Synapse.Dockerfile .)
# COMPLEMENT_BASE_IMAGE=complement-synapse go test -v ./tests
ARG SYNAPSE_VERSION=latest
FROM matrixdotorg/synapse:${SYNAPSE_VERSION}
ENV SERVER_NAME=localhost
COPY synapse/* /conf/
COPY keys/* /ca/
# SSL key for the server (can't make the cert until we know the server name)
RUN openssl genrsa -out /conf/server.tls.key 2048
# generate a signing key
RUN generate_signing_key.py -o /conf/server.signing.key
WORKDIR /data
EXPOSE 8008 8448
ENTRYPOINT ["/conf/start.sh"]
HEALTHCHECK --start-period=5s --interval=1s --timeout=1s \
CMD curl -fSs http://localhost:8008/health || exit 1

View File

@ -0,0 +1,9 @@
This directory contains a key and certificate for a dummy Certificate
Authority. This is used to create certificates for the servers under test.
The files were generated with:
```
openssl genrsa -out ca.key 2048
openssl req -new -x509 -key ca.key -days 3650 -subj "/C=GB/ST=London/O=matrix.org/CN=Complement CA" -out ca.crt
```

View File

@ -0,0 +1,21 @@
-----BEGIN CERTIFICATE-----
MIIDdzCCAl+gAwIBAgIUDZUl0q5DMzWkd/fev4gJurDpuH8wDQYJKoZIhvcNAQEL
BQAwSzELMAkGA1UEBhMCR0IxDzANBgNVBAgMBkxvbmRvbjETMBEGA1UECgwKbWF0
cml4Lm9yZzEWMBQGA1UEAwwNQ29tcGxlbWVudCBDQTAeFw0yMDA4MDUxMjAyMDRa
Fw0zMDA4MDMxMjAyMDRaMEsxCzAJBgNVBAYTAkdCMQ8wDQYDVQQIDAZMb25kb24x
EzARBgNVBAoMCm1hdHJpeC5vcmcxFjAUBgNVBAMMDUNvbXBsZW1lbnQgQ0EwggEi
MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC9LfS/T0Y72b9j+M7/43Hry+Cf
UUt7iym5z14su8cve545ceT3Y1SRwrGWRy8GuvhgQyVTzvgslwdsLYwVEaK1ZsWQ
7l+UBD128HWCSo/OFMV3j+allMzj4PO3PnMcU1cq6HuTlYeAFlOelLB7FUGFmOzc
c829XHWvDD/Hjz+SQ4E11nftHyu6Chw+Razfcftzp+qLjozjHeXMLI09peiy65zp
DGYhiB78vTCjtioNlMZ5bTrI5PAG3O0nY0NAiAqrGgijpbJdYigf2vcJ8UQ6e0LH
0nTIhITxRd+hN+FGbsLe+OtzaaEkgdtxQrtjAq7wjsxbchNDjnqh/eTJwBaPAgMB
AAGjUzBRMB0GA1UdDgQWBBRNM7hzjkBvs7Rmw4I7g4wpEy53nDAfBgNVHSMEGDAW
gBRNM7hzjkBvs7Rmw4I7g4wpEy53nDAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3
DQEBCwUAA4IBAQAOfCmPTM9NFPbVwu003kbOPfbtHULlDyMvx4LRiYmpvxfpFjGy
Xp2p/ZqzFHvyT5QcCFrkOoQlX8tIzr8KZqbemf2aWfdRzqT1VG7ADErNfQ2JOjLo
jptliOWcnknmCi8elGjBxVmAqWfDyQKYeiK2AyrJywMLr45UlrxRh03d81TaTRYk
rRHMMKJ3iKb8HQKlrfjQrYCCCSzNkp0RP/TyhpB4qIZDg9AbhUlXn24bQ/Gpq9Hw
pGmYXObJ8Uid7SICxxeFm42Nc8Orke+yw77glcQEumLJM0TqzR1UV3aTyE0qrXeG
R8e6WTemuXtcDEeo1MzCwW/75Sd/1exbs2dW
-----END CERTIFICATE-----

View File

@ -0,0 +1,27 @@
-----BEGIN RSA PRIVATE KEY-----
MIIEpQIBAAKCAQEAvS30v09GO9m/Y/jO/+Nx68vgn1FLe4spuc9eLLvHL3ueOXHk
92NUkcKxlkcvBrr4YEMlU874LJcHbC2MFRGitWbFkO5flAQ9dvB1gkqPzhTFd4/m
pZTM4+Dztz5zHFNXKuh7k5WHgBZTnpSwexVBhZjs3HPNvVx1rww/x48/kkOBNdZ3
7R8rugocPkWs33H7c6fqi46M4x3lzCyNPaXosuuc6QxmIYge/L0wo7YqDZTGeW06
yOTwBtztJ2NDQIgKqxoIo6WyXWIoH9r3CfFEOntCx9J0yISE8UXfoTfhRm7C3vjr
c2mhJIHbcUK7YwKu8I7MW3ITQ456of3kycAWjwIDAQABAoIBAQCt+ctw/IiMlg2q
9K/5J8Qmo1IpcevG7A4tcQUcgnwFsNur3JBC6jhl6/TBTSl0ORIeBOIZFueoqiQL
Wj/PSG6oAKGETZPcXaHg9fY7IjLpxLRs5gupDxT2JHQ9etMFo6cwMKkk6oi4RNAV
sIw0RTICmMaLXnxAvv0DRHKLJbPC33gvJCwXg4PJ8JWkrfn6EdExWTT5V6OSBJW3
W7hwUp7cZ03yXZg8KLt8A3ZwFA1jSNtgISbGojcjV++ylK8xMV7NOVvBwcGQHOCi
pwvjoHMc2SqpssgsyHPg5s/ZvgNWqMBQms24Sb0tAS3HASiPjL72Oc69kuqTPfbN
Pn24VDBBAoGBAOpAtH/kZOGBe/NEQWUrpnseteQE/482udWt9+3H3VPW8cCYzVWf
xV/anj18wtx+nxZgMtu6p5NiU2Tx1OQLOJElKPqd+MOcYeDm+IutRnNETP4i+2P7
8HXUAEySZCz4W2ettznnMGHKINV2LEv4v3yMtvVPx/GtTLk7m8axSfEfAoGBAM6+
CCwpUQwKACQClwNkmrvpWmuj+XkiXJr6rnGNZ2akPSpz7YAbre9kDCGYyo3gsKmP
xmRs6dbHWqbSyCxu+zhjgqGYKsyLcAUHxGDeH2q8Cf8Bf6Jw6dtW5CHka4QIbq7W
hAFFCXTPQM59cOjxbCzQJwb8by7bChhpALVl/PyRAoGAHE4rQVTot7L8tj1mJYJr
wG7Kjb3o33Y+aEp4Ldw7qzjKx3nvPcxrfhSdkFJ1/UyzjnGbU5+vRgjYiNcL4W5V
rHwwAnB6MbDAxvtBxybt31m+b+rsM32q5aHzU01RG3n7GYENJcsAtqBE7WY1Hgz1
nmgmZ2f3ggVtwugn/49GnwMCgYEAmnOr5Vokm2rpOf3ZVgaV9ubZz2rcWNwL9Rhs
tJagdjUjIREkkL2dDyheS8c7JA2FzdeErsgTVorutYzpURDEjBcDo7fr2Y4ebROl
aXzoWbQLf8Bd2zyh6WSb4JomnKg0EFhgnBY4f1TKpLlgXhbm6v/C+FwksAuiGiRw
DBUakJECgYEAmHcPSXefohshJIflTs8u/M4FMt9scJMXZuOp6npLOKIZp1KuxTst
Kh3GCI51kJI68XvXxPNrNAQ6ay1Ayp/CwCnlkGYTJeACno4u94fgbQBsRLksRM6g
JvQZtOvi5OqrnVz2iTWJhtInV/dIbUtv+yJ9hYOa7AtP3AhiDVePvts=
-----END RSA PRIVATE KEY-----

View File

@ -0,0 +1,103 @@
## Server ##
server_name: SERVER_NAME
log_config: /conf/log_config.yaml
report_stats: False
signing_key_path: /conf/server.signing.key
trusted_key_servers: []
enable_registration: true
## Listeners ##
tls_certificate_path: /conf/server.tls.crt
tls_private_key_path: /conf/server.tls.key
bcrypt_rounds: 4
registration_shared_secret: matrix_public_archive
listeners:
- port: 8448
bind_addresses: ['::']
type: http
tls: true
resources:
- names: [federation]
- port: 8008
bind_addresses: ['::']
type: http
resources:
- names: [client]
## Database ##
database:
name: 'sqlite3'
args:
# We avoid /data, as it is a volume and is not transferred when the container is committed,
# which is a fundamental necessity in complement.
database: '/conf/homeserver.db'
## Federation ##
# disable verification of federation certificates
#
# TODO: this is temporary until https://github.com/matrix-org/complement/pull/28 lands and
# allows homeservers spun up by complement access to the complement CA certificate to trust
federation_verify_certificates: false
# trust certs signed by the dummy CA
federation_custom_ca_list:
- /ca/ca.crt
# unblacklist RFC1918 addresses
ip_range_blacklist: []
# Disable server rate-limiting
rc_federation:
window_size: 1000
sleep_limit: 10
sleep_delay: 500
reject_limit: 99999
concurrent: 3
rc_message:
per_second: 9999
burst_count: 9999
rc_registration:
per_second: 9999
burst_count: 9999
rc_login:
address:
per_second: 9999
burst_count: 9999
account:
per_second: 9999
burst_count: 9999
failed_attempts:
per_second: 9999
burst_count: 9999
rc_admin_redaction:
per_second: 9999
burst_count: 9999
rc_joins:
local:
per_second: 9999
burst_count: 9999
remote:
per_second: 9999
burst_count: 9999
federation_rr_transactions_per_room_per_second: 9999
## Experimental Features ##
experimental_features:
# Enable history backfilling support
msc2716_enabled: true
# Enable jump to date endpoint
msc3030_enabled: true

View File

@ -0,0 +1,24 @@
version: 1
formatters:
precise:
format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s'
filters:
context:
(): synapse.logging.context.LoggingContextFilter
request: ""
handlers:
console:
class: logging.StreamHandler
formatter: precise
filters: [context]
# log to stdout, for easier use with 'docker logs'
stream: 'ext://sys.stdout'
root:
level: INFO
handlers: [console]
disable_existing_loggers: false

View File

@ -0,0 +1,15 @@
#!/bin/sh
set -e
sed -i "s/SERVER_NAME/${SERVER_NAME}/g" /conf/homeserver.yaml
# generate an ssl cert for the server, signed by our dummy CA
openssl req -new -key /conf/server.tls.key -out /conf/server.tls.csr \
-subj "/CN=${SERVER_NAME}"
openssl x509 -req -in /conf/server.tls.csr \
-CA /ca/ca.crt -CAkey /ca/ca.key -set_serial 1 \
-out /conf/server.tls.crt
exec python -m synapse.app.homeserver -c /conf/homeserver.yaml "$@"

View File

@ -1,15 +1,80 @@
'use strict'; 'use strict';
const assert = require('assert'); const assert = require('assert');
const sdk = require('matrix-js-sdk'); const urlJoin = require('url-join');
const { MatrixAuth } = require('matrix-bot-sdk');
const fetchEndpoint = require('../server/lib/fetch-endpoint');
const config = require('../config'); const config = require('../config');
assert(config.testMatrixServerUrl); assert(config.testMatrixServerUrl1);
assert(config.testMatrixServerUrl2);
const client = sdk.createClient(config.testMatrixServerUrl); async function getTestClientForHs(testMatrixServerUrl) {
const auth = new MatrixAuth(testMatrixServerUrl);
const client = await auth.passwordRegister(
`user-${Math.floor(Math.random() * 1000000000)}`,
'password'
);
return client;
}
async function createTestRoom(client) {
const roomId = await client.createRoom({
preset: 'public_chat',
name: 'the hangout spot',
});
let eventIds = [];
for (let i = 0; i < 100; i++) {
const eventId = await client.sendMessage(roomId, {
msgtype: 'm.text',
body: `${client.homeserverUrl} - message${i}`,
});
eventIds.push(eventId);
}
return {
roomId,
eventIds,
};
}
describe('matrix-public-archive', () => { describe('matrix-public-archive', () => {
it('asdf', async () => { it('asdf', async () => {
await client.register(`user-${Math.floor(Math.random() * 1000000000)}`, 'awfe'); const hs1Client = await getTestClientForHs(config.testMatrixServerUrl1);
const hs2Client = await getTestClientForHs(config.testMatrixServerUrl2);
const { roomId: hs1RoomId, eventIds: room1EventIds } = await createTestRoom(hs1Client);
const { roomId: hs2RoomId, eventIds: room2EventIds } = await createTestRoom(hs2Client);
console.log('hs1RoomId', hs1RoomId, room1EventIds);
console.log('hs2RoomId', hs2RoomId, room2EventIds);
try {
await hs1Client.joinRoom(hs2RoomId, 'hs2');
} catch (err) {
throw new Error(
`Stub error to stop matrix-bot-sdk from logging the response statusCode=${
err.statusCode
} body=${JSON.stringify(err.body)}`
);
}
const messagesEndpoint = urlJoin(
hs1Client.homeserverUrl,
`_matrix/client/r0/rooms/${hs2RoomId}/messages?limit=5&dir=b`
);
const messageResData = await fetchEndpoint(messagesEndpoint, {
accessToken: hs1Client.accessToken,
});
console.log(messageResData);
console.log(
messageResData.chunk.map((event) => {
return `${event.event_id} (${event.type}) - ${event.content.body}`;
})
);
}); });
}); });