From e9d13db91169f4182bfb40563b7a621eca04802f Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Mon, 29 Aug 2022 18:56:31 -0500 Subject: [PATCH] Add test for joining a new federated room (#31) Add test for joining a new federated room and making sure the messages are available (homeserver should backfill). Synapse changes: https://github.com/matrix-org/synapse/pull/13205, https://github.com/matrix-org/synapse/pull/13320 --- server/ensure-room-joined.js | 32 +++++++++ server/fetch-events-in-range.js | 10 --- server/routes/install-routes.js | 16 ++++- shared/lib/url-creator.js | 9 ++- test/client-utils.js | 111 ++++++++++++++++++++++---------- test/e2e-tests.js | 50 +++++++++++++- 6 files changed, 179 insertions(+), 49 deletions(-) create mode 100644 server/ensure-room-joined.js diff --git a/server/ensure-room-joined.js b/server/ensure-room-joined.js new file mode 100644 index 0000000..cd0bd29 --- /dev/null +++ b/server/ensure-room-joined.js @@ -0,0 +1,32 @@ +'use strict'; + +const assert = require('assert'); +const urlJoin = require('url-join'); + +const { fetchEndpointAsJson } = require('./lib/fetch-endpoint'); + +const config = require('./lib/config'); +const matrixServerUrl = config.get('matrixServerUrl'); +assert(matrixServerUrl); + +async function ensureRoomJoined(accessToken, roomId, viaServers = []) { + let qs = new URLSearchParams(); + [].concat(viaServers).forEach((viaServer) => { + qs.append('server_name', viaServer); + }); + + // TODO: Only join world_readable rooms. Perhaps we want to serve public rooms + // where we have been invited. GET + // /_matrix/client/v3/directory/list/room/{roomId} (Gets the visibility of a + // given room on the server’s public room directory.) + const joinEndpoint = urlJoin( + matrixServerUrl, + `_matrix/client/r0/join/${roomId}?${qs.toString()}` + ); + await fetchEndpointAsJson(joinEndpoint, { + method: 'POST', + accessToken, + }); +} + +module.exports = ensureRoomJoined; diff --git a/server/fetch-events-in-range.js b/server/fetch-events-in-range.js index 36ae229..e92c14e 100644 --- a/server/fetch-events-in-range.js +++ b/server/fetch-events-in-range.js @@ -27,16 +27,6 @@ async function fetchEventsFromTimestampBackwards(accessToken, roomId, ts, limit) assert(ts); assert(limit); - // TODO: Only join world_readable rooms. Perhaps we want to serve public rooms - // where we have been invited. GET - // /_matrix/client/v3/directory/list/room/{roomId} (Gets the visibility of a - // given room on the server’s public room directory.) - const joinEndpoint = urlJoin(matrixServerUrl, `_matrix/client/r0/join/${roomId}`); - await fetchEndpointAsJson(joinEndpoint, { - method: 'POST', - accessToken, - }); - const timestampToEventEndpoint = urlJoin( matrixServerUrl, `_matrix/client/unstable/org.matrix.msc3030/rooms/${roomId}/timestamp_to_event?ts=${ts}&dir=b` diff --git a/server/routes/install-routes.js b/server/routes/install-routes.js index 031ee9a..71d51c6 100644 --- a/server/routes/install-routes.js +++ b/server/routes/install-routes.js @@ -12,6 +12,7 @@ const timeoutMiddleware = require('./timeout-middleware'); const fetchRoomData = require('../fetch-room-data'); const fetchEventsInRange = require('../fetch-events-in-range'); +const ensureRoomJoined = require('../ensure-room-joined'); const renderHydrogenToString = require('../hydrogen-render/1-render-hydrogen-to-string'); const sanitizeHtml = require('../lib/sanitize-html'); const safeJson = require('../lib/safe-json'); @@ -136,8 +137,15 @@ function installRoutes(app) { // If the hourRange is defined, we force the range to always be 1 hour. If // the format isn't correct, redirect to the correct hour range if (hourRange && toHour !== fromHour + 1) { + // Pass through the query parameters + let queryParamterUrlPiece = ''; + if (req.query) { + queryParamterUrlPiece = `?${new URLSearchParams(req.query).toString()}`; + } + res.redirect( - urlJoin( + // FIXME: Can we use the matrixPublicArchiveURLCreator here? + `${urlJoin( basePath, roomIdOrAlias, 'date', @@ -145,7 +153,7 @@ function installRoutes(app) { req.params.mm, req.params.dd, `${fromHour}-${fromHour + 1}` - ) + )}${queryParamterUrlPiece}` ); return; } @@ -153,6 +161,10 @@ function installRoutes(app) { // TODO: Highlight tile that matches ?at=$xxx //const aroundId = req.query.at; + // We have to wait for the room join to happen first before we can fetch + // any of the additional room info or messages. + await ensureRoomJoined(matrixAccessToken, roomIdOrAlias, req.query.via); + // Do these in parallel to avoid the extra time in sequential round-trips // (we want to display the archive page faster) const [roomData, { events, stateEventMap }] = await Promise.all([ diff --git a/shared/lib/url-creator.js b/shared/lib/url-creator.js index 7057a34..21bd4dc 100644 --- a/shared/lib/url-creator.js +++ b/shared/lib/url-creator.js @@ -7,12 +7,17 @@ class URLCreator { this._basePath = basePath; } - archiveUrlForDate(roomId, date) { + archiveUrlForDate(roomId, date, { viaServers = [] } = {}) { + let qs = new URLSearchParams(); + [].concat(viaServers).forEach((viaServer) => { + qs.append('via', viaServer); + }); + // Gives the date in YYYY/mm/dd format. // date.toISOString() -> 2022-02-16T23:20:04.709Z const urlDate = date.toISOString().split('T')[0].replaceAll('-', '/'); - return urlJoin(this._basePath, `${roomId}/date/${urlDate}`); + return `${urlJoin(this._basePath, `${roomId}/date/${urlDate}`)}?${qs.toString()}`; } } diff --git a/test/client-utils.js b/test/client-utils.js index 30dc2ef..a119aa4 100644 --- a/test/client-utils.js +++ b/test/client-utils.js @@ -1,13 +1,14 @@ 'use strict'; const assert = require('assert'); -const { URLSearchParams } = require('url'); const urlJoin = require('url-join'); const { fetchEndpointAsJson, fetchEndpoint } = require('../server/lib/fetch-endpoint'); const config = require('../server/lib/config'); const matrixAccessToken = config.get('matrixAccessToken'); assert(matrixAccessToken); +const testMatrixServerUrl1 = config.get('testMatrixServerUrl1'); +assert(testMatrixServerUrl1); let txnCount = 0; function getTxnId() { @@ -15,6 +16,30 @@ function getTxnId() { return `${new Date().getTime()}--${txnCount}`; } +async function ensureUserRegistered({ matrixServerUrl, username }) { + const registerResponse = await fetchEndpointAsJson( + urlJoin(matrixServerUrl, '/_matrix/client/v3/register'), + { + method: 'POST', + body: { + type: 'm.login.dummy', + username, + }, + } + ); + + const userId = registerResponse['user_id']; + assert(userId); +} + +async function getTestClientForAs() { + return { + homeserverUrl: testMatrixServerUrl1, + accessToken: matrixAccessToken, + userId: '@archiver:hs1', + }; +} + // Get client to act with for all of the client methods. This will use the // application service access token and client methods will append `?user_id` // for the specific user to act upon so we can use the `?ts` message timestamp @@ -92,13 +117,15 @@ async function joinRoom({ client, roomId, viaServers }) { qs.append('user_id', client.applicationServiceUserIdOverride); } - const joinRoomResponse = await fetchEndpointAsJson( - urlJoin(client.homeserverUrl, `/_matrix/client/v3/join/${roomId}?${qs.toString()}`), - { - method: 'POST', - accessToken: client.accessToken, - } + const joinRoomUrl = urlJoin( + client.homeserverUrl, + `/_matrix/client/v3/join/${roomId}?${qs.toString()}` ); + console.log('test client joinRoomUrl', joinRoomUrl); + const joinRoomResponse = await fetchEndpointAsJson(joinRoomUrl, { + method: 'POST', + accessToken: client.accessToken, + }); const joinedRoomId = joinRoomResponse['room_id']; assert(joinedRoomId); @@ -164,11 +191,19 @@ async function createMessagesInRoom({ client, roomId, numMessages, prefix, times msgtype: 'm.text', body: `${prefix} - message${i}`, }, - timestamp, + // We can't use the exact same timestamp for every message in the tests + // otherwise it's a toss up which event will be returned as the closest + // for `/timestamp_to_event`. As a note, we don't have to do this after + // https://github.com/matrix-org/synapse/pull/13658 merges but it still + // seems like a good idea to make the tests more clear. + timestamp: timestamp + i, }); eventIds.push(eventId); } + // Sanity check that we actually sent some messages + assert.strictEqual(eventIds.length, numMessages); + return eventIds; } @@ -178,33 +213,39 @@ async function updateProfile({ client, displayName, avatarUrl }) { qs.append('user_id', client.applicationServiceUserIdOverride); } - const updateDisplayNamePromise = fetchEndpointAsJson( - urlJoin( - client.homeserverUrl, - `/_matrix/client/v3/profile/${client.userId}/displayname?${qs.toString()}` - ), - { - method: 'PUT', - body: { - displayname: displayName, - }, - accessToken: client.accessToken, - } - ); + let updateDisplayNamePromise = Promise.resolve(); + if (displayName) { + updateDisplayNamePromise = fetchEndpointAsJson( + urlJoin( + client.homeserverUrl, + `/_matrix/client/v3/profile/${client.userId}/displayname?${qs.toString()}` + ), + { + method: 'PUT', + body: { + displayname: displayName, + }, + accessToken: client.accessToken, + } + ); + } - const updateAvatarUrlPromise = fetchEndpointAsJson( - urlJoin( - client.homeserverUrl, - `/_matrix/client/v3/profile/${client.userId}/avatar_url?${qs.toString()}` - ), - { - method: 'PUT', - body: { - avatar_url: avatarUrl, - }, - accessToken: client.accessToken, - } - ); + let updateAvatarUrlPromise = Promise.resolve(); + if (avatarUrl) { + updateAvatarUrlPromise = fetchEndpointAsJson( + urlJoin( + client.homeserverUrl, + `/_matrix/client/v3/profile/${client.userId}/avatar_url?${qs.toString()}` + ), + { + method: 'PUT', + body: { + avatar_url: avatarUrl, + }, + accessToken: client.accessToken, + } + ); + } await Promise.all([updateDisplayNamePromise, updateAvatarUrlPromise]); @@ -248,6 +289,8 @@ async function uploadContent({ client, roomId, data, fileName, contentType }) { } module.exports = { + ensureUserRegistered, + getTestClientForAs, getTestClientForHs, createTestRoom, joinRoom, diff --git a/test/e2e-tests.js b/test/e2e-tests.js index 8600056..c221160 100644 --- a/test/e2e-tests.js +++ b/test/e2e-tests.js @@ -14,6 +14,7 @@ const { fetchEndpointAsText, fetchEndpointAsJson } = require('../server/lib/fetc const config = require('../server/lib/config'); const { + getTestClientForAs, getTestClientForHs, createTestRoom, joinRoom, @@ -106,6 +107,20 @@ describe('matrix-public-archive', () => { }); describe('Archive', () => { + before(async () => { + // Make sure the application service archiver user itself has a profile + // set otherwise we run into 404, `Profile was not found` errors when + // joining a remote federated room from the archiver user, see + // https://github.com/matrix-org/synapse/issues/4778 + // + // FIXME: Remove after https://github.com/matrix-org/synapse/issues/4778 is resolved + const asClient = await getTestClientForAs(); + await updateProfile({ + client: asClient, + displayName: 'Archiver', + }); + }); + // Use a fixed date at the start of the UTC day so that the tests are // consistent. Otherwise, the tests could fail when they start close to // midnight and it rolls over to the next day. @@ -163,6 +178,7 @@ describe('matrix-public-archive', () => { `Coulomb's Law of Friction: Kinetic friction is independent of the sliding velocity.`, ]; + // TODO: Can we use `createMessagesInRoom` here instead? const eventIds = []; for (const messageText of messageTextList) { const eventId = await sendMessageOnArchiveDate({ @@ -375,7 +391,39 @@ describe('matrix-public-archive', () => { ); }); - it(`can render day back in time from room on remote homeserver we haven't backfilled from`); + it(`can render day back in time from room on remote homeserver we haven't backfilled from`, async () => { + const hs2Client = await getTestClientForHs(testMatrixServerUrl2); + + // Create a room on hs2 + const hs2RoomId = await createTestRoom(hs2Client); + const room2EventIds = await createMessagesInRoom({ + client: hs2Client, + roomId: hs2RoomId, + numMessages: 3, + prefix: HOMESERVER_URL_TO_PRETTY_NAME_MAP[hs2Client.homeserverUrl], + timestamp: archiveDate.getTime(), + }); + + archiveUrl = matrixPublicArchiveURLCreator.archiveUrlForDate(hs2RoomId, archiveDate, { + // Since hs1 doesn't know about this room on hs2 yet, we have to provide + // a via server to ask through. + viaServers: ['hs2'], + }); + + const archivePageHtml = await fetchEndpointAsText(archiveUrl); + + const dom = parseHTML(archivePageHtml); + + // Make sure the messages are visible + assert.deepStrictEqual( + room2EventIds.map((eventId) => { + return dom.document + .querySelector(`[data-event-id="${eventId}"]`) + ?.getAttribute('data-event-id'); + }), + room2EventIds + ); + }); it(`will redirect to hour pagination when there are too many messages`);