From db6d3797d74104ad7c93e572ed106f1a685a90d0 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 24 Feb 2022 03:27:53 -0600 Subject: [PATCH] Working e2e test --- config/config.default.json | 1 + package-lock.json | 1 + package.json | 2 + server/fetch-events-for-timestamp.js | 67 --------------- server/fetch-events-in-range.js | 124 +++++++++++++++++++++++++++ server/routes/install-routes.js | 19 +++- shared/hydrogen-vm-render-script.js | 1 - test/e2e-tests.js | 113 +++++++++++++++++++----- 8 files changed, 236 insertions(+), 92 deletions(-) delete mode 100644 server/fetch-events-for-timestamp.js create mode 100644 server/fetch-events-in-range.js diff --git a/config/config.default.json b/config/config.default.json index 2f6fda7..277e440 100644 --- a/config/config.default.json +++ b/config/config.default.json @@ -2,6 +2,7 @@ "basePort": "3050", "basePath": "http://localhost:3050", "matrixServerUrl": "http://localhost:8008/", + "archiveMessageLimit": 500, "testMatrixServerUrl1": "http://localhost:11008/", "testMatrixServerUrl2": "http://localhost:12008/", diff --git a/package-lock.json b/package-lock.json index cb000aa..ee8d676 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,6 +19,7 @@ "url-join": "^4.0.1" }, "devDependencies": { + "escape-string-regexp": "^4.0.0", "eslint": "^8.8.0", "eslint-config-prettier": "^8.3.0", "eslint-plugin-node": "^11.1.0", diff --git a/package.json b/package.json index 9570a9b..5bd3548 100644 --- a/package.json +++ b/package.json @@ -5,6 +5,7 @@ "start-dev": "node server/start-dev.js", "lint": "eslint **/*.js", "test": "npm run mocha -- test/e2e-tests.js --timeout 15000", + "test-interactive": "npm run mocha -- test/e2e-tests.js --timeout 15000 --interactive", "nodemon": "nodemon", "vite": "vite", "mocha": "mocha", @@ -15,6 +16,7 @@ "node": ">=16.0.0" }, "devDependencies": { + "escape-string-regexp": "^4.0.0", "eslint": "^8.8.0", "eslint-config-prettier": "^8.3.0", "eslint-plugin-node": "^11.1.0", diff --git a/server/fetch-events-for-timestamp.js b/server/fetch-events-for-timestamp.js deleted file mode 100644 index 96dcd79..0000000 --- a/server/fetch-events-for-timestamp.js +++ /dev/null @@ -1,67 +0,0 @@ -'use strict'; - -const assert = require('assert'); -const urlJoin = require('url-join'); - -const { fetchEndpointAsJson } = require('./lib/fetch-endpoint'); - -const config = require('./lib/config'); -const matrixServerUrl = config.get('matrixServerUrl'); -assert(matrixServerUrl); - -async function fetchEventsForTimestamp(accessToken, roomId, ts) { - assert(accessToken); - assert(roomId); - assert(ts); - - // TODO: Only join world_readable rooms - const joinEndpoint = urlJoin(matrixServerUrl, `_matrix/client/r0/join/${roomId}`); - await fetchEndpointAsJson(joinEndpoint, { - method: 'POST', - accessToken, - }); - - const timestampToEventEndpoint = urlJoin( - matrixServerUrl, - `_matrix/client/unstable/org.matrix.msc3030/rooms/${roomId}/timestamp_to_event?ts=${ts}&dir=b` - ); - const timestampToEventResData = await fetchEndpointAsJson(timestampToEventEndpoint, { - accessToken, - }); - const eventIdForTimestamp = timestampToEventResData.event_id; - assert(eventIdForTimestamp); - console.log('eventIdForTimestamp', eventIdForTimestamp); - - const contextEndpoint = urlJoin( - matrixServerUrl, - `_matrix/client/r0/rooms/${roomId}/context/${eventIdForTimestamp}?limit=0` - ); - const contextResData = await fetchEndpointAsJson(contextEndpoint, { - accessToken, - }); - //console.log('contextResData', contextResData); - - // Add filter={"lazy_load_members":true,"include_redundant_members":true} to get member state events included - const messagesEndpoint = urlJoin( - matrixServerUrl, - `_matrix/client/r0/rooms/${roomId}/messages?from=${contextResData.start}&limit=50&filter={"lazy_load_members":true,"include_redundant_members":true}` - ); - const messageResData = await fetchEndpointAsJson(messagesEndpoint, { - accessToken, - }); - - //console.log('messageResData.state', messageResData.state); - const stateEventMap = {}; - for (const stateEvent of messageResData.state || []) { - if (stateEvent.type === 'm.room.member') { - stateEventMap[stateEvent.state_key] = stateEvent; - } - } - - return { - stateEventMap, - events: messageResData.chunk, - }; -} - -module.exports = fetchEventsForTimestamp; diff --git a/server/fetch-events-in-range.js b/server/fetch-events-in-range.js new file mode 100644 index 0000000..8b70f84 --- /dev/null +++ b/server/fetch-events-in-range.js @@ -0,0 +1,124 @@ +'use strict'; + +const assert = require('assert'); +const urlJoin = require('url-join'); + +const { fetchEndpointAsJson } = require('./lib/fetch-endpoint'); + +const config = require('./lib/config'); +const matrixServerUrl = config.get('matrixServerUrl'); +assert(matrixServerUrl); + +// Find an event right ahead of where we are trying to look. Then paginate +// /messages backwards. This makes sure that we can get events for the day +// when the room started. +// +// Consider this scenario: dayStart(fromTs) <---- msg1 <- msg2 <-- msg3 <---- dayEnd(toTs) +// - ❌ If we start from dayStart and look backwards, we will find nothing. +// - ❌ If we start from dayStart and look forwards, we will find msg1, but federated backfill won't be able to paginate forwards +// - ✅ If we start from dayEnd and look backwards, we will msg3 +// - ❌ If we start from dayEnd and look forwards, we will find nothing +// +// Returns events in reverse-chronological order. +async function fetchEventsFromTimestampBackwards(accessToken, roomId, ts, limit) { + assert(accessToken); + assert(roomId); + assert(ts); + assert(limit); + + // TODO: Only join world_readable rooms. Perhaps we want to serve public rooms + // where we have been invited. GET + // /_matrix/client/v3/directory/list/room/{roomId} (Gets the visibility of a + // given room on the server’s public room directory.) + const joinEndpoint = urlJoin(matrixServerUrl, `_matrix/client/r0/join/${roomId}`); + await fetchEndpointAsJson(joinEndpoint, { + method: 'POST', + accessToken, + }); + + const timestampToEventEndpoint = urlJoin( + matrixServerUrl, + `_matrix/client/unstable/org.matrix.msc3030/rooms/${roomId}/timestamp_to_event?ts=${ts}&dir=b` + ); + const timestampToEventResData = await fetchEndpointAsJson(timestampToEventEndpoint, { + accessToken, + }); + const eventIdForTimestamp = timestampToEventResData.event_id; + assert(eventIdForTimestamp); + //console.log('eventIdForTimestamp', eventIdForTimestamp); + + const contextEndpoint = urlJoin( + matrixServerUrl, + `_matrix/client/r0/rooms/${roomId}/context/${eventIdForTimestamp}?limit=0` + ); + const contextResData = await fetchEndpointAsJson(contextEndpoint, { + accessToken, + }); + //console.log('contextResData', contextResData); + + // Add filter={"lazy_load_members":true,"include_redundant_members":true} to get member state events included + const messagesEndpoint = urlJoin( + matrixServerUrl, + `_matrix/client/r0/rooms/${roomId}/messages?dir=b&from=${contextResData.start}&limit=${limit}&filter={"lazy_load_members":true,"include_redundant_members":true}` + ); + const messageResData = await fetchEndpointAsJson(messagesEndpoint, { + accessToken, + }); + + //console.log('messageResData.state', messageResData.state); + const stateEventMap = {}; + for (const stateEvent of messageResData.state || []) { + if (stateEvent.type === 'm.room.member') { + stateEventMap[stateEvent.state_key] = stateEvent; + } + } + + return { + stateEventMap, + events: messageResData.chunk, + }; +} + +async function fetchEventsInRange(accessToken, roomId, startTs, endTs, limit) { + assert(accessToken); + assert(roomId); + assert(startTs); + assert(endTs); + assert(limit); + + // Fetch events from endTs and before + const { events, stateEventMap } = await fetchEventsFromTimestampBackwards( + accessToken, + roomId, + endTs, + limit + ); + + let eventsInRange = events; + // `events` are in reverse-chronological order. + // We only need to filter if the oldest message is before startTs + if (events[events.length - 1].origin_server_ts < startTs) { + eventsInRange = []; + + // Let's iterate until we see events before startTs + for (let i = 0; i < events.length; i++) { + const event = events[i]; + + // Once we found an event before startTs, the rest are outside of our range + if (event.origin_server_ts < startTs) { + break; + } + + eventsInRange.push(event); + } + } + + const chronologicalEventsInRange = eventsInRange.reverse(); + + return { + stateEventMap, + events: chronologicalEventsInRange, + }; +} + +module.exports = fetchEventsInRange; diff --git a/server/routes/install-routes.js b/server/routes/install-routes.js index dc41ace..ef7cbd1 100644 --- a/server/routes/install-routes.js +++ b/server/routes/install-routes.js @@ -8,7 +8,7 @@ const asyncHandler = require('../lib/express-async-handler'); const StatusError = require('../lib/status-error'); const fetchRoomData = require('../fetch-room-data'); -const fetchEventsForTimestamp = require('../fetch-events-for-timestamp'); +const fetchEventsInRange = require('../fetch-events-in-range'); const renderHydrogenToString = require('../render-hydrogen-to-string'); const config = require('../lib/config'); @@ -16,6 +16,8 @@ const basePath = config.get('basePath'); assert(basePath); const matrixAccessToken = config.get('matrixAccessToken'); assert(matrixAccessToken); +const archiveMessageLimit = config.get('archiveMessageLimit'); +assert(archiveMessageLimit); function parseArchiveRangeFromReq(req) { const yyyy = parseInt(req.params.yyyy, 10); @@ -100,7 +102,8 @@ function installRoutes(app) { const roomIdOrAlias = req.params.roomIdOrAlias; assert(roomIdOrAlias.startsWith('!') || roomIdOrAlias.startsWith('#')); - const { fromTimestamp, hourRange, fromHour, toHour } = parseArchiveRangeFromReq(req); + const { fromTimestamp, toTimestamp, hourRange, fromHour, toHour } = + parseArchiveRangeFromReq(req); // If the hourRange is defined, we force the range to always be 1 hour. If // the format isn't correct, redirect to the correct hour range @@ -124,10 +127,18 @@ function installRoutes(app) { const [roomData, { events, stateEventMap }] = await Promise.all([ fetchRoomData(matrixAccessToken, roomIdOrAlias), - fetchEventsForTimestamp(matrixAccessToken, roomIdOrAlias, fromTimestamp), + fetchEventsInRange( + matrixAccessToken, + roomIdOrAlias, + fromTimestamp, + toTimestamp, + archiveMessageLimit + ), ]); - console.log('events', JSON.stringify(events, null, 2)); + if (events.length >= archiveMessageLimit) { + throw new Error('TODO: Redirect user to smaller hour range'); + } const hydrogenHtmlOutput = await renderHydrogenToString({ fromTimestamp, diff --git a/shared/hydrogen-vm-render-script.js b/shared/hydrogen-vm-render-script.js index 82ee568..0834ce1 100644 --- a/shared/hydrogen-vm-render-script.js +++ b/shared/hydrogen-vm-render-script.js @@ -6,7 +6,6 @@ const { MediaRepository, createNavigation, createRouter, - Segment, TilesCollection, FragmentIdComparer, diff --git a/test/e2e-tests.js b/test/e2e-tests.js index b1b16c5..ec39873 100644 --- a/test/e2e-tests.js +++ b/test/e2e-tests.js @@ -4,9 +4,10 @@ process.env.NODE_ENV = 'test'; const assert = require('assert'); const urlJoin = require('url-join'); +const escapeStringRegexp = require('escape-string-regexp'); const { MatrixAuth } = require('matrix-bot-sdk'); +const { parseHTML } = require('linkedom'); -const server = require('../server/server'); const { fetchEndpointAsText, fetchEndpointAsJson } = require('../server/lib/fetch-endpoint'); const MatrixPublicArchiveURLCreator = require('matrix-public-archive-shared/lib/url-creator'); @@ -14,10 +15,11 @@ const MatrixPublicArchiveURLCreator = require('matrix-public-archive-shared/lib/ const config = require('../server/lib/config'); const testMatrixServerUrl1 = config.get('testMatrixServerUrl1'); const testMatrixServerUrl2 = config.get('testMatrixServerUrl2'); -const basePath = config.get('basePath'); assert(testMatrixServerUrl1); assert(testMatrixServerUrl2); +const basePath = config.get('basePath'); assert(basePath); +const interactive = config.get('interactive'); const matrixPublicArchiveURLCreator = new MatrixPublicArchiveURLCreator(basePath); @@ -69,10 +71,20 @@ async function createTestRoom(client) { } describe('matrix-public-archive', () => { - after(() => { - //server.close(); + let server; + before(() => { + // Start the archive server + server = require('../server/server'); }); + after(() => { + if (!interactive) { + server.close(); + } + }); + + // Sanity check that our test homeservers can actually federate with each + // other. The rest of the tests won't work properly if this isn't working. it('Test federation between fixture homeservers', async () => { try { const hs1Client = await getTestClientForHs(testMatrixServerUrl1); @@ -133,6 +145,7 @@ describe('matrix-public-archive', () => { } }); + // eslint-disable-next-line max-statements it('can render diverse messages', async () => { try { const client = await getTestClientForHs(testMatrixServerUrl1); @@ -146,68 +159,122 @@ describe('matrix-public-archive', () => { const mxcUri = await client.uploadContentFromUrl( 'https://en.wikipedia.org/wiki/Friction#/media/File:Friction_between_surfaces.jpg' ); - await client.sendMessage(roomId, { + const imageEventId = await client.sendMessage(roomId, { body: 'Friction_between_surfaces.jpeg', info: { size: 396644, mimetype: 'image/jpeg', + thumbnail_info: { + w: 800, + h: 390, + mimetype: 'image/jpeg', + size: 126496, + }, w: 1894, h: 925, 'xyz.amorgan.blurhash': 'LkR3G|IU?w%NbwbIemae_NxuD$M{', + // TODO: How to get a proper thumnail URL that will load? + thumbnail_url: mxcUri, }, msgtype: 'm.image', url: mxcUri, }); // A normal text message - await client.sendMessage(roomId, { + const normalMessageText1 = + '^ Figure 1: Simulated blocks with fractal rough surfaces, exhibiting static frictional interactions'; + const normalMessageEventId1 = await client.sendMessage(roomId, { msgtype: 'm.text', - body: '^ Figure 1: Simulated blocks with fractal rough surfaces, exhibiting static frictional interactions', + body: normalMessageText1, }); - // A normal text message - await client.sendMessage(roomId, { + // Another normal text message + const normalMessageText2 = + 'The topography of the Moon has been measured with laser altimetry and stereo image analysis.'; + const normalMessageEventId2 = await client.sendMessage(roomId, { msgtype: 'm.text', - body: 'The topography of the Moon has been measured with laser altimetry and stereo image analysis.', + body: normalMessageText2, }); // Test replies - const eventToReplyTo = await client.sendMessage(roomId, { + const replyMessageText = `The concentration of maria on the near side likely reflects the substantially thicker crust of the highlands of the Far Side, which may have formed in a slow-velocity impact of a second moon of Earth a few tens of millions of years after the Moon's formation.`; + const replyMessageEventId = await client.sendMessage(roomId, { 'org.matrix.msc1767.message': [ { - body: "> <@ericgittertester:my.synapse.server> The topography of the Moon has been measured with laser altimetry and stereo image analysis.\n\nThe concentration of maria on the near side likely reflects the substantially thicker crust of the highlands of the Far Side, which may have formed in a slow-velocity impact of a second moon of Earth a few tens of millions of years after the Moon's formation.", + body: '> <@ericgittertester:my.synapse.server> ${normalMessageText2}', mimetype: 'text/plain', }, { - body: '
In reply to @ericgittertester:my.synapse.server
The topography of the Moon has been measured with laser altimetry and stereo image analysis.
The concentration of maria on the near side likely reflects the substantially thicker crust of the highlands of the Far Side, which may have formed in a slow-velocity impact of a second moon of Earth a few tens of millions of years after the Moon\'s formation.', + body: `
In reply to @ericgittertester:my.synapse.server
${normalMessageText2}
${replyMessageText}`, mimetype: 'text/html', }, ], - body: "> <@ericgittertester:my.synapse.server> The topography of the Moon has been measured with laser altimetry and stereo image analysis.\n\nThe concentration of maria on the near side likely reflects the substantially thicker crust of the highlands of the Far Side, which may have formed in a slow-velocity impact of a second moon of Earth a few tens of millions of years after the Moon's formation.", + body: `> <@ericgittertester:my.synapse.server> ${normalMessageText2}\n\n${replyMessageText}`, msgtype: 'm.text', format: 'org.matrix.custom.html', - formatted_body: - '
In reply to @ericgittertester:my.synapse.server
The topography of the Moon has been measured with laser altimetry and stereo image analysis.
The concentration of maria on the near side likely reflects the substantially thicker crust of the highlands of the Far Side, which may have formed in a slow-velocity impact of a second moon of Earth a few tens of millions of years after the Moon\'s formation.', + formatted_body: `
In reply to @ericgittertester:my.synapse.server
${normalMessageText2}
${replyMessageText}`, 'm.relates_to': { 'm.in_reply_to': { - event_id: '$uEeScM2gfILkLpG8sOBTK7vcS0w_t3a9EVIAnSwqyiY', + event_id: normalMessageEventId2, }, }, }); // Test reactions + const reactionText = '😅'; await client.sendEvent(roomId, 'm.reaction', { 'm.relates_to': { rel_type: 'm.annotation', - event_id: eventToReplyTo, - key: '��', + event_id: replyMessageEventId, + key: reactionText, }, }); const archiveUrl = matrixPublicArchiveURLCreator.archiveUrlForDate(roomId, new Date()); + if (interactive) { + console.log('Interactive URL for test', archiveUrl); + } const archivePageHtml = await fetchEndpointAsText(archiveUrl); - console.log('archivePageHtml', archivePageHtml); + + const dom = parseHTML(archivePageHtml); + + // Make sure the image message is visible + const imageElement = dom.document.querySelector(`[data-event-id="${imageEventId}"] img`); + assert(imageElement); + assert.match(imageElement.getAttribute('src'), new RegExp(`^http://.*`)); + assert.strictEqual(imageElement.getAttribute('alt'), 'Friction_between_surfaces.jpeg'); + + // Make sure the normal message is visible + assert.match( + dom.document.querySelector(`[data-event-id="${normalMessageEventId1}"]`).outerHTML, + new RegExp(`.*${escapeStringRegexp(normalMessageText1)}.*`) + ); + + // Make sure the other normal message is visible + assert.match( + dom.document.querySelector(`[data-event-id="${normalMessageEventId2}"]`).outerHTML, + new RegExp(`.*${escapeStringRegexp(normalMessageText2)}.*`) + ); + + const replyMessageElement = dom.document.querySelector( + `[data-event-id="${replyMessageEventId}"]` + ); + // Make sure the reply text is there + assert.match( + replyMessageElement.outerHTML, + new RegExp(`.*${escapeStringRegexp(replyMessageText)}.*`) + ); + // Make sure it also includes the message we're replying to + assert.match( + replyMessageElement.outerHTML, + new RegExp(`.*${escapeStringRegexp(normalMessageEventId2)}.*`) + ); + // Make sure the reaction also exists + assert.match( + replyMessageElement.outerHTML, + new RegExp(`.*${escapeStringRegexp(reactionText)}.*`) + ); } catch (err) { if (err.body) { // FIXME: Remove this try/catch once the matrix-bot-sdk no longer throws @@ -227,4 +294,10 @@ describe('matrix-public-archive', () => { it(`can render day back in time from room on remote homeserver we haven't backfilled from`); it(`will redirect to hour pagination when there are too many messages`); + + it(`will render a room with only a day of messages`); + + it( + `will render a room with a sparse amount of messages (a few per day) with no contamination between days` + ); });