Mirrors
/
matrix-public-archive
mirror of https://github.com/matrix-org/matrix-public-archive.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Working e2e test

This commit is contained in:
Eric Eastwood 2022-02-24 03:27:53 -06:00
parent 839e31a35e
commit db6d3797d7
8 changed files with 236 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
config/config.default.json
View File

@ -2,6 +2,7 @@
"basePort": "3050",
"basePath": "http://localhost:3050",
"matrixServerUrl": "http://localhost:8008/",
"archiveMessageLimit": 500,
"testMatrixServerUrl1": "http://localhost:11008/",
"testMatrixServerUrl2": "http://localhost:12008/",

1
package-lock.json generated
View File

@ -19,6 +19,7 @@
"url-join": "^4.0.1"
},
"devDependencies": {
"escape-string-regexp": "^4.0.0",
"eslint": "^8.8.0",
"eslint-config-prettier": "^8.3.0",
"eslint-plugin-node": "^11.1.0",

2
package.json
View File

@ -5,6 +5,7 @@
"start-dev": "node server/start-dev.js",
"lint": "eslint **/*.js",
"test": "npm run mocha -- test/e2e-tests.js --timeout 15000",
"test-interactive": "npm run mocha -- test/e2e-tests.js --timeout 15000 --interactive",
"nodemon": "nodemon",
"vite": "vite",
"mocha": "mocha",
@ -15,6 +16,7 @@
"node": ">=16.0.0"
},
"devDependencies": {
"escape-string-regexp": "^4.0.0",
"eslint": "^8.8.0",
"eslint-config-prettier": "^8.3.0",
"eslint-plugin-node": "^11.1.0",

67
server/fetch-events-for-timestamp.js
View File

@ -1,67 +0,0 @@
'use strict';
const assert = require('assert');
const urlJoin = require('url-join');
const { fetchEndpointAsJson } = require('./lib/fetch-endpoint');
const config = require('./lib/config');
const matrixServerUrl = config.get('matrixServerUrl');
assert(matrixServerUrl);
async function fetchEventsForTimestamp(accessToken, roomId, ts) {
assert(accessToken);
assert(roomId);
assert(ts);
// TODO: Only join world_readable rooms
const joinEndpoint = urlJoin(matrixServerUrl, `_matrix/client/r0/join/${roomId}`);
await fetchEndpointAsJson(joinEndpoint, {
method: 'POST',
accessToken,
});
const timestampToEventEndpoint = urlJoin(
matrixServerUrl,
`_matrix/client/unstable/org.matrix.msc3030/rooms/${roomId}/timestamp_to_event?ts=${ts}&dir=b`
);
const timestampToEventResData = await fetchEndpointAsJson(timestampToEventEndpoint, {
accessToken,
});
const eventIdForTimestamp = timestampToEventResData.event_id;
assert(eventIdForTimestamp);
console.log('eventIdForTimestamp', eventIdForTimestamp);
const contextEndpoint = urlJoin(
matrixServerUrl,
`_matrix/client/r0/rooms/${roomId}/context/${eventIdForTimestamp}?limit=0`
);
const contextResData = await fetchEndpointAsJson(contextEndpoint, {
accessToken,
});
//console.log('contextResData', contextResData);
// Add filter={"lazy_load_members":true,"include_redundant_members":true} to get member state events included
const messagesEndpoint = urlJoin(
matrixServerUrl,
`_matrix/client/r0/rooms/${roomId}/messages?from=${contextResData.start}&limit=50&filter={"lazy_load_members":true,"include_redundant_members":true}`
);
const messageResData = await fetchEndpointAsJson(messagesEndpoint, {
accessToken,
});
//console.log('messageResData.state', messageResData.state);
const stateEventMap = {};
for (const stateEvent of messageResData.state || []) {
if (stateEvent.type === 'm.room.member') {
stateEventMap[stateEvent.state_key] = stateEvent;
}
}
return {
stateEventMap,
events: messageResData.chunk,
};
}
module.exports = fetchEventsForTimestamp;

124
server/fetch-events-in-range.js Normal file
View File

@ -0,0 +1,124 @@
'use strict';
const assert = require('assert');
const urlJoin = require('url-join');
const { fetchEndpointAsJson } = require('./lib/fetch-endpoint');
const config = require('./lib/config');
const matrixServerUrl = config.get('matrixServerUrl');
assert(matrixServerUrl);
// Find an event right ahead of where we are trying to look. Then paginate
// /messages backwards. This makes sure that we can get events for the day
// when the room started.
//
// Consider this scenario: dayStart(fromTs) <---- msg1 <- msg2 <-- msg3 <---- dayEnd(toTs)
// - ❌ If we start from dayStart and look backwards, we will find nothing.
// - ❌ If we start from dayStart and look forwards, we will find msg1, but federated backfill won't be able to paginate forwards
// - ✅ If we start from dayEnd and look backwards, we will msg3
// - ❌ If we start from dayEnd and look forwards, we will find nothing
//
// Returns events in reverse-chronological order.
async function fetchEventsFromTimestampBackwards(accessToken, roomId, ts, limit) {
assert(accessToken);
assert(roomId);
assert(ts);
assert(limit);
// TODO: Only join world_readable rooms. Perhaps we want to serve public rooms
// where we have been invited. GET
// /_matrix/client/v3/directory/list/room/{roomId} (Gets the visibility of a
// given room on the server’s public room directory.)
const joinEndpoint = urlJoin(matrixServerUrl, `_matrix/client/r0/join/${roomId}`);
await fetchEndpointAsJson(joinEndpoint, {
method: 'POST',
accessToken,
});
const timestampToEventEndpoint = urlJoin(
matrixServerUrl,
`_matrix/client/unstable/org.matrix.msc3030/rooms/${roomId}/timestamp_to_event?ts=${ts}&dir=b`
);
const timestampToEventResData = await fetchEndpointAsJson(timestampToEventEndpoint, {
accessToken,
});
const eventIdForTimestamp = timestampToEventResData.event_id;
assert(eventIdForTimestamp);
//console.log('eventIdForTimestamp', eventIdForTimestamp);
const contextEndpoint = urlJoin(
matrixServerUrl,
`_matrix/client/r0/rooms/${roomId}/context/${eventIdForTimestamp}?limit=0`
);
const contextResData = await fetchEndpointAsJson(contextEndpoint, {
accessToken,
});
//console.log('contextResData', contextResData);
// Add filter={"lazy_load_members":true,"include_redundant_members":true} to get member state events included
const messagesEndpoint = urlJoin(
matrixServerUrl,
`_matrix/client/r0/rooms/${roomId}/messages?dir=b&from=${contextResData.start}&limit=${limit}&filter={"lazy_load_members":true,"include_redundant_members":true}`
);
const messageResData = await fetchEndpointAsJson(messagesEndpoint, {
accessToken,
});
//console.log('messageResData.state', messageResData.state);
const stateEventMap = {};
for (const stateEvent of messageResData.state || []) {
if (stateEvent.type === 'm.room.member') {
stateEventMap[stateEvent.state_key] = stateEvent;
}
}
return {
stateEventMap,
events: messageResData.chunk,
};
}
async function fetchEventsInRange(accessToken, roomId, startTs, endTs, limit) {
assert(accessToken);
assert(roomId);
assert(startTs);
assert(endTs);
assert(limit);
// Fetch events from endTs and before
const { events, stateEventMap } = await fetchEventsFromTimestampBackwards(
accessToken,
roomId,
endTs,
limit
);
let eventsInRange = events;
// `events` are in reverse-chronological order.
// We only need to filter if the oldest message is before startTs
if (events[events.length - 1].origin_server_ts < startTs) {
eventsInRange = [];
// Let's iterate until we see events before startTs
for (let i = 0; i < events.length; i++) {
const event = events[i];
// Once we found an event before startTs, the rest are outside of our range
if (event.origin_server_ts < startTs) {
break;
}
eventsInRange.push(event);
}
}
const chronologicalEventsInRange = eventsInRange.reverse();
return {
stateEventMap,
events: chronologicalEventsInRange,
};
}
module.exports = fetchEventsInRange;

19
server/routes/install-routes.js
View File

@ -8,7 +8,7 @@ const asyncHandler = require('../lib/express-async-handler');
const StatusError = require('../lib/status-error');
const fetchRoomData = require('../fetch-room-data');
const fetchEventsForTimestamp = require('../fetch-events-for-timestamp');
const fetchEventsInRange = require('../fetch-events-in-range');
const renderHydrogenToString = require('../render-hydrogen-to-string');
const config = require('../lib/config');
@ -16,6 +16,8 @@ const basePath = config.get('basePath');
assert(basePath);
const matrixAccessToken = config.get('matrixAccessToken');
assert(matrixAccessToken);
const archiveMessageLimit = config.get('archiveMessageLimit');
assert(archiveMessageLimit);
function parseArchiveRangeFromReq(req) {
const yyyy = parseInt(req.params.yyyy, 10);
@ -100,7 +102,8 @@ function installRoutes(app) {
const roomIdOrAlias = req.params.roomIdOrAlias;
assert(roomIdOrAlias.startsWith('!') || roomIdOrAlias.startsWith('#'));
const { fromTimestamp, hourRange, fromHour, toHour } = parseArchiveRangeFromReq(req);
const { fromTimestamp, toTimestamp, hourRange, fromHour, toHour } =
parseArchiveRangeFromReq(req);
// If the hourRange is defined, we force the range to always be 1 hour. If
// the format isn't correct, redirect to the correct hour range
@ -124,10 +127,18 @@ function installRoutes(app) {
const [roomData, { events, stateEventMap }] = await Promise.all([
fetchRoomData(matrixAccessToken, roomIdOrAlias),
fetchEventsForTimestamp(matrixAccessToken, roomIdOrAlias, fromTimestamp),
fetchEventsInRange(
matrixAccessToken,
roomIdOrAlias,
fromTimestamp,
toTimestamp,
archiveMessageLimit
),
]);
console.log('events', JSON.stringify(events, null, 2));
if (events.length >= archiveMessageLimit) {
throw new Error('TODO: Redirect user to smaller hour range');
}
const hydrogenHtmlOutput = await renderHydrogenToString({
fromTimestamp,

1
shared/hydrogen-vm-render-script.js
View File

@ -6,7 +6,6 @@ const {
MediaRepository,
createNavigation,
createRouter,
Segment,
TilesCollection,
FragmentIdComparer,

113
test/e2e-tests.js
View File

@ -4,9 +4,10 @@ process.env.NODE_ENV = 'test';
const assert = require('assert');
const urlJoin = require('url-join');
const escapeStringRegexp = require('escape-string-regexp');
const { MatrixAuth } = require('matrix-bot-sdk');
const { parseHTML } = require('linkedom');
const server = require('../server/server');
const { fetchEndpointAsText, fetchEndpointAsJson } = require('../server/lib/fetch-endpoint');
const MatrixPublicArchiveURLCreator = require('matrix-public-archive-shared/lib/url-creator');
@ -14,10 +15,11 @@ const MatrixPublicArchiveURLCreator = require('matrix-public-archive-shared/lib/
const config = require('../server/lib/config');
const testMatrixServerUrl1 = config.get('testMatrixServerUrl1');
const testMatrixServerUrl2 = config.get('testMatrixServerUrl2');
const basePath = config.get('basePath');
assert(testMatrixServerUrl1);
assert(testMatrixServerUrl2);
const basePath = config.get('basePath');
assert(basePath);
const interactive = config.get('interactive');
const matrixPublicArchiveURLCreator = new MatrixPublicArchiveURLCreator(basePath);
@ -69,10 +71,20 @@ async function createTestRoom(client) {
}
describe('matrix-public-archive', () => {
after(() => {
//server.close();
let server;
before(() => {
// Start the archive server
server = require('../server/server');
});
after(() => {
if (!interactive) {
server.close();
}
});
// Sanity check that our test homeservers can actually federate with each
// other. The rest of the tests won't work properly if this isn't working.
it('Test federation between fixture homeservers', async () => {
try {
const hs1Client = await getTestClientForHs(testMatrixServerUrl1);
@ -133,6 +145,7 @@ describe('matrix-public-archive', () => {
}
});
// eslint-disable-next-line max-statements
it('can render diverse messages', async () => {
try {
const client = await getTestClientForHs(testMatrixServerUrl1);
@ -146,68 +159,122 @@ describe('matrix-public-archive', () => {
const mxcUri = await client.uploadContentFromUrl(
'https://en.wikipedia.org/wiki/Friction#/media/File:Friction_between_surfaces.jpg'
);
await client.sendMessage(roomId, {
const imageEventId = await client.sendMessage(roomId, {
body: 'Friction_between_surfaces.jpeg',
info: {
size: 396644,
mimetype: 'image/jpeg',
thumbnail_info: {
w: 800,
h: 390,
mimetype: 'image/jpeg',
size: 126496,
},
w: 1894,
h: 925,
'xyz.amorgan.blurhash': 'LkR3G|IU?w%NbwbIemae_NxuD$M{',
// TODO: How to get a proper thumnail URL that will load?
thumbnail_url: mxcUri,
},
msgtype: 'm.image',
url: mxcUri,
});
// A normal text message
await client.sendMessage(roomId, {
const normalMessageText1 =
'^ Figure 1: Simulated blocks with fractal rough surfaces, exhibiting static frictional interactions';
const normalMessageEventId1 = await client.sendMessage(roomId, {
msgtype: 'm.text',
body: '^ Figure 1: Simulated blocks with fractal rough surfaces, exhibiting static frictional interactions',
body: normalMessageText1,
});
// A normal text message
await client.sendMessage(roomId, {
// Another normal text message
const normalMessageText2 =
'The topography of the Moon has been measured with laser altimetry and stereo image analysis.';
const normalMessageEventId2 = await client.sendMessage(roomId, {
msgtype: 'm.text',
body: 'The topography of the Moon has been measured with laser altimetry and stereo image analysis.',
body: normalMessageText2,
});
// Test replies
const eventToReplyTo = await client.sendMessage(roomId, {
const replyMessageText = `The concentration of maria on the near side likely reflects the substantially thicker crust of the highlands of the Far Side, which may have formed in a slow-velocity impact of a second moon of Earth a few tens of millions of years after the Moon's formation.`;
const replyMessageEventId = await client.sendMessage(roomId, {
'org.matrix.msc1767.message': [
{
body: "> <@ericgittertester:my.synapse.server> The topography of the Moon has been measured with laser altimetry and stereo image analysis.\n\nThe concentration of maria on the near side likely reflects the substantially thicker crust of the highlands of the Far Side, which may have formed in a slow-velocity impact of a second moon of Earth a few tens of millions of years after the Moon's formation.",
body: '> <@ericgittertester:my.synapse.server> ${normalMessageText2}',
mimetype: 'text/plain',
},
{
body: '<mx-reply><blockquote><a href="https://matrix.to/#/!HBehERstyQBxyJDLfR:my.synapse.server/$uEeScM2gfILkLpG8sOBTK7vcS0w_t3a9EVIAnSwqyiY?via=my.synapse.server">In reply to</a> <a href="https://matrix.to/#/@ericgittertester:my.synapse.server">@ericgittertester:my.synapse.server</a><br>The topography of the Moon has been measured with laser altimetry and stereo image analysis.</blockquote></mx-reply>The concentration of maria on the near side likely reflects the substantially thicker crust of the highlands of the Far Side, which may have formed in a slow-velocity impact of a second moon of Earth a few tens of millions of years after the Moon\'s formation.',
body: `<mx-reply><blockquote><a href="https://matrix.to/#/${roomId}/${normalMessageEventId2}?via=my.synapse.server">In reply to</a> <a href="https://matrix.to/#/@ericgittertester:my.synapse.server">@ericgittertester:my.synapse.server</a><br>${normalMessageText2}</blockquote></mx-reply>${replyMessageText}`,
mimetype: 'text/html',
},
],
body: "> <@ericgittertester:my.synapse.server> The topography of the Moon has been measured with laser altimetry and stereo image analysis.\n\nThe concentration of maria on the near side likely reflects the substantially thicker crust of the highlands of the Far Side, which may have formed in a slow-velocity impact of a second moon of Earth a few tens of millions of years after the Moon's formation.",
body: `> <@ericgittertester:my.synapse.server> ${normalMessageText2}\n\n${replyMessageText}`,
msgtype: 'm.text',
format: 'org.matrix.custom.html',
formatted_body:
'<mx-reply><blockquote><a href="https://matrix.to/#/!HBehERstyQBxyJDLfR:my.synapse.server/$uEeScM2gfILkLpG8sOBTK7vcS0w_t3a9EVIAnSwqyiY?via=my.synapse.server">In reply to</a> <a href="https://matrix.to/#/@ericgittertester:my.synapse.server">@ericgittertester:my.synapse.server</a><br>The topography of the Moon has been measured with laser altimetry and stereo image analysis.</blockquote></mx-reply>The concentration of maria on the near side likely reflects the substantially thicker crust of the highlands of the Far Side, which may have formed in a slow-velocity impact of a second moon of Earth a few tens of millions of years after the Moon\'s formation.',
formatted_body: `<mx-reply><blockquote><a href="https://matrix.to/#/${roomId}/${normalMessageEventId2}?via=my.synapse.server">In reply to</a> <a href="https://matrix.to/#/@ericgittertester:my.synapse.server">@ericgittertester:my.synapse.server</a><br>${normalMessageText2}</blockquote></mx-reply>${replyMessageText}`,
'm.relates_to': {
'm.in_reply_to': {
event_id: '$uEeScM2gfILkLpG8sOBTK7vcS0w_t3a9EVIAnSwqyiY',
event_id: normalMessageEventId2,
},
},
});
// Test reactions
const reactionText = '😅';
await client.sendEvent(roomId, 'm.reaction', {
'm.relates_to': {
rel_type: 'm.annotation',
event_id: eventToReplyTo,
key: '<27><>',
event_id: replyMessageEventId,
key: reactionText,
},
});
const archiveUrl = matrixPublicArchiveURLCreator.archiveUrlForDate(roomId, new Date());
if (interactive) {
console.log('Interactive URL for test', archiveUrl);
}
const archivePageHtml = await fetchEndpointAsText(archiveUrl);
console.log('archivePageHtml', archivePageHtml);
const dom = parseHTML(archivePageHtml);
// Make sure the image message is visible
const imageElement = dom.document.querySelector(`[data-event-id="${imageEventId}"] img`);
assert(imageElement);
assert.match(imageElement.getAttribute('src'), new RegExp(`^http://.*`));
assert.strictEqual(imageElement.getAttribute('alt'), 'Friction_between_surfaces.jpeg');
// Make sure the normal message is visible
assert.match(
dom.document.querySelector(`[data-event-id="${normalMessageEventId1}"]`).outerHTML,
new RegExp(`.*${escapeStringRegexp(normalMessageText1)}.*`)
);
// Make sure the other normal message is visible
assert.match(
dom.document.querySelector(`[data-event-id="${normalMessageEventId2}"]`).outerHTML,
new RegExp(`.*${escapeStringRegexp(normalMessageText2)}.*`)
);
const replyMessageElement = dom.document.querySelector(
`[data-event-id="${replyMessageEventId}"]`
);
// Make sure the reply text is there
assert.match(
replyMessageElement.outerHTML,
new RegExp(`.*${escapeStringRegexp(replyMessageText)}.*`)
);
// Make sure it also includes the message we're replying to
assert.match(
replyMessageElement.outerHTML,
new RegExp(`.*${escapeStringRegexp(normalMessageEventId2)}.*`)
);
// Make sure the reaction also exists
assert.match(
replyMessageElement.outerHTML,
new RegExp(`.*${escapeStringRegexp(reactionText)}.*`)
);
} catch (err) {
if (err.body) {
// FIXME: Remove this try/catch once the matrix-bot-sdk no longer throws
@ -227,4 +294,10 @@ describe('matrix-public-archive', () => {
it(`can render day back in time from room on remote homeserver we haven't backfilled from`);
it(`will redirect to hour pagination when there are too many messages`);
it(`will render a room with only a day of messages`);
it(
`will render a room with a sparse amount of messages (a few per day) with no contamination between days`
);
});