From 551b4e72d19fcf4ba5ec1b4efba17dc9ca5faee7 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 19 Apr 2023 01:26:15 -0500 Subject: [PATCH] Follow tombstone and predecessor history (#167) Fix https://github.com/matrix-org/matrix-public-archive/issues/59 Other updates: - Update tests to use `/roomid/room1/date/2022/01/03` format instead of trying to retrofit the weird alias stuff on there. Which also makes the fancy to actual URL utilities much more simple. - Update to specify `archiveMessageLimit` in the test case because pages have different number of events depending on if we are against a boundary, hidden events, etc. --- package-lock.json | 11 +- package.json | 3 +- server/lib/matrix-utils/fetch-room-data.js | 207 +- ...erver-name-from-matrix-room-id-or-alias.js | 3 + server/routes/room-routes.js | 405 +++- shared/lib/reference-values.js | 6 + shared/lib/timestamp-utilities.js | 1 + shared/lib/url-creator.js | 31 +- shared/viewmodels/ArchiveRoomViewModel.js | 7 + .../JumpToNextActivitySummaryTileViewModel.js | 12 + ...pToPreviousActivitySummaryTileViewModel.js | 12 + test/e2e-tests.js | 1815 ++++++++++++----- test/test-utils/client-utils.js | 197 +- test/test-utils/parse-archive-url-for-room.js | 6 +- .../parse-room-day-message-structure.js | 10 - 15 files changed, 1988 insertions(+), 738 deletions(-) diff --git a/package-lock.json b/package-lock.json index dc7c058..3889035 100644 --- a/package-lock.json +++ b/package-lock.json @@ -31,6 +31,7 @@ "url-join": "^4.0.1" }, "devDependencies": { + "chalk": "^4.1.2", "escape-string-regexp": "^4.0.0", "eslint": "^8.37.0", "eslint-config-prettier": "^8.8.0", @@ -2463,8 +2464,9 @@ }, "node_modules/chalk": { "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", "dev": true, - "license": "MIT", "dependencies": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" @@ -5480,8 +5482,9 @@ }, "node_modules/supports-color": { "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", "dev": true, - "license": "MIT", "dependencies": { "has-flag": "^4.0.0" }, @@ -7494,6 +7497,8 @@ }, "chalk": { "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", "dev": true, "requires": { "ansi-styles": "^4.1.0", @@ -9371,6 +9376,8 @@ }, "supports-color": { "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", "dev": true, "requires": { "has-flag": "^4.0.0" diff --git a/package.json b/package.json index b161249..3d29a6f 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,7 @@ "start": "node server/server.js", "start-dev": "node server/start-dev.js", "test": "npm run mocha -- test/**/*-tests.js --timeout 15000", - "test-e2e-interactive": "npm run mocha -- test/e2e-tests.js --timeout 15000 --interactive", + "test-e2e-interactive": "npm run mocha -- test/e2e-tests.js --timeout 15000 --bail --interactive", "nodemon": "nodemon", "vite": "vite", "mocha": "mocha", @@ -23,6 +23,7 @@ "node": ">=16.0.0" }, "devDependencies": { + "chalk": "^4.1.2", "escape-string-regexp": "^4.0.0", "eslint": "^8.37.0", "eslint-config-prettier": "^8.8.0", diff --git a/server/lib/matrix-utils/fetch-room-data.js b/server/lib/matrix-utils/fetch-room-data.js index d6b73e6..055f50c 100644 --- a/server/lib/matrix-utils/fetch-room-data.js +++ b/server/lib/matrix-utils/fetch-room-data.js @@ -4,89 +4,204 @@ const assert = require('assert'); const urlJoin = require('url-join'); const { fetchEndpointAsJson } = require('../fetch-endpoint'); +const parseViaServersFromUserInput = require('../parse-via-servers-from-user-input'); const { traceFunction } = require('../../tracing/trace-utilities'); const config = require('../config'); const matrixServerUrl = config.get('matrixServerUrl'); assert(matrixServerUrl); -async function fetchRoomData(accessToken, roomId) { - assert(accessToken); - assert(roomId); +function getStateEndpointForRoomIdAndEventType(roomId, eventType) { + return urlJoin( + matrixServerUrl, + `_matrix/client/r0/rooms/${encodeURIComponent(roomId)}/state/${encodeURIComponent( + eventType + )}?format=event` + ); +} - const stateNameEndpoint = urlJoin( - matrixServerUrl, - `_matrix/client/r0/rooms/${encodeURIComponent(roomId)}/state/m.room.name` - ); - const canoncialAliasEndpoint = urlJoin( - matrixServerUrl, - `_matrix/client/r0/rooms/${encodeURIComponent(roomId)}/state/m.room.canonical_alias` - ); - const stateAvatarEndpoint = urlJoin( - matrixServerUrl, - `_matrix/client/r0/rooms/${encodeURIComponent(roomId)}/state/m.room.avatar` - ); - const stateHistoryVisibilityEndpoint = urlJoin( - matrixServerUrl, - `_matrix/client/r0/rooms/${encodeURIComponent(roomId)}/state/m.room.history_visibility` - ); - const stateJoinRulesEndpoint = urlJoin( - matrixServerUrl, - `_matrix/client/r0/rooms/${encodeURIComponent(roomId)}/state/m.room.join_rules` +// Unfortunately, we can't just get the event ID from the `/state?format=event` +// endpoint, so we have to do this trick. Related to +// https://github.com/matrix-org/synapse/issues/15454 +// +// TODO: Remove this when we have MSC3999 (because it's the only usage) +const removeMe_fetchRoomCreateEventId = traceFunction(async function (matrixAccessToken, roomId) { + const { data } = await fetchEndpointAsJson( + urlJoin( + matrixServerUrl, + `_matrix/client/r0/rooms/${encodeURIComponent(roomId)}/messages?dir=f&limit1` + ), + { + accessToken: matrixAccessToken, + } ); + const roomCreateEventId = data?.chunk?.[0]?.event_id; + + return roomCreateEventId; +}); + +const fetchRoomCreationInfo = traceFunction(async function (matrixAccessToken, roomId) { + const [stateCreateResDataOutcome] = await Promise.allSettled([ + fetchEndpointAsJson(getStateEndpointForRoomIdAndEventType(roomId, 'm.room.create'), { + accessToken: matrixAccessToken, + }), + ]); + + let roomCreationTs; + let predecessorRoomId; + let predecessorLastKnownEventId; + if (stateCreateResDataOutcome.reason === undefined) { + const { data } = stateCreateResDataOutcome.value; + roomCreationTs = data?.origin_server_ts; + predecessorLastKnownEventId = data?.content?.event_id; + predecessorRoomId = data?.content?.predecessor?.room_id; + } + + return { roomCreationTs, predecessorRoomId, predecessorLastKnownEventId }; +}); + +const fetchPredecessorInfo = traceFunction(async function (matrixAccessToken, roomId) { + const [roomCreationInfoOutcome, statePredecessorResDataOutcome] = await Promise.allSettled([ + fetchRoomCreationInfo(matrixAccessToken, roomId), + fetchEndpointAsJson( + getStateEndpointForRoomIdAndEventType(roomId, 'org.matrix.msc3946.room_predecessor'), + { + accessToken: matrixAccessToken, + } + ), + ]); + + let predecessorRoomId; + let predecessorLastKnownEventId; + let predecessorViaServers; + // Prefer the dynamic predecessor from the dedicated state event + if (statePredecessorResDataOutcome.reason === undefined) { + const { data } = statePredecessorResDataOutcome.value; + predecessorRoomId = data?.content?.predecessor_room_id; + predecessorLastKnownEventId = data?.content?.last_known_event_id; + predecessorViaServers = parseViaServersFromUserInput(data?.content?.via_servers); + } + // Then fallback to the predecessor defined by the room creation event + else if (roomCreationInfoOutcome.reason === undefined) { + ({ predecessorRoomId, predecessorLastKnownEventId } = roomCreationInfoOutcome.value); + } + + const { roomCreationTs: currentRoomCreationTs } = roomCreationInfoOutcome; + + return { + // This is prefixed with "current" so we don't get this confused with the + // predecessor room creation timestamp. + currentRoomCreationTs, + predecessorRoomId, + predecessorLastKnownEventId, + predecessorViaServers, + }; +}); + +const fetchSuccessorInfo = traceFunction(async function (matrixAccessToken, roomId) { + const [stateTombstoneResDataOutcome] = await Promise.allSettled([ + fetchEndpointAsJson(getStateEndpointForRoomIdAndEventType(roomId, 'm.room.tombstone'), { + accessToken: matrixAccessToken, + }), + ]); + + let successorRoomId; + let successorSetTs; + if (stateTombstoneResDataOutcome.reason === undefined) { + const { data } = stateTombstoneResDataOutcome.value; + successorRoomId = data?.content?.replacement_room; + successorSetTs = data?.origin_server_ts; + } + + return { + successorRoomId, + successorSetTs, + }; +}); + +// eslint-disable-next-line max-statements +const fetchRoomData = traceFunction(async function (matrixAccessToken, roomId) { + assert(matrixAccessToken); + assert(roomId); + const [ stateNameResDataOutcome, stateCanonicalAliasResDataOutcome, stateAvatarResDataOutcome, stateHistoryVisibilityResDataOutcome, stateJoinRulesResDataOutcome, + predecessorInfoOutcome, + successorInfoOutcome, ] = await Promise.allSettled([ - fetchEndpointAsJson(stateNameEndpoint, { - accessToken, + fetchEndpointAsJson(getStateEndpointForRoomIdAndEventType(roomId, 'm.room.name'), { + accessToken: matrixAccessToken, }), - fetchEndpointAsJson(canoncialAliasEndpoint, { - accessToken, + fetchEndpointAsJson(getStateEndpointForRoomIdAndEventType(roomId, 'm.room.canonical_alias'), { + accessToken: matrixAccessToken, }), - fetchEndpointAsJson(stateAvatarEndpoint, { - accessToken, + fetchEndpointAsJson(getStateEndpointForRoomIdAndEventType(roomId, 'm.room.avatar'), { + accessToken: matrixAccessToken, }), - fetchEndpointAsJson(stateHistoryVisibilityEndpoint, { - accessToken, - }), - fetchEndpointAsJson(stateJoinRulesEndpoint, { - accessToken, + fetchEndpointAsJson( + getStateEndpointForRoomIdAndEventType(roomId, 'm.room.history_visibility'), + { + accessToken: matrixAccessToken, + } + ), + fetchEndpointAsJson(getStateEndpointForRoomIdAndEventType(roomId, 'm.room.join_rules'), { + accessToken: matrixAccessToken, }), + fetchPredecessorInfo(matrixAccessToken, roomId), + fetchSuccessorInfo(matrixAccessToken, roomId), ]); let name; if (stateNameResDataOutcome.reason === undefined) { const { data } = stateNameResDataOutcome.value; - name = data.name; + name = data?.content?.name; } let canonicalAlias; if (stateCanonicalAliasResDataOutcome.reason === undefined) { const { data } = stateCanonicalAliasResDataOutcome.value; - canonicalAlias = data.alias; + canonicalAlias = data?.content?.alias; } let avatarUrl; if (stateAvatarResDataOutcome.reason === undefined) { const { data } = stateAvatarResDataOutcome.value; - avatarUrl = data.url; + avatarUrl = data?.content?.url; } let historyVisibility; if (stateHistoryVisibilityResDataOutcome.reason === undefined) { const { data } = stateHistoryVisibilityResDataOutcome.value; - historyVisibility = data.history_visibility; + historyVisibility = data?.content?.history_visibility; } let joinRule; if (stateJoinRulesResDataOutcome.reason === undefined) { const { data } = stateJoinRulesResDataOutcome.value; - joinRule = data.join_rule; + joinRule = data?.content?.join_rule; + } + + let roomCreationTs; + let predecessorRoomId; + let predecessorLastKnownEventId; + let predecessorViaServers; + if (predecessorInfoOutcome.reason === undefined) { + ({ + currentRoomCreationTs: roomCreationTs, + predecessorRoomId, + predecessorLastKnownEventId, + predecessorViaServers, + } = predecessorInfoOutcome.value); + } + let successorRoomId; + let successorSetTs; + if (successorInfoOutcome.reason === undefined) { + ({ successorRoomId, successorSetTs } = successorInfoOutcome.value); } return { @@ -96,7 +211,19 @@ async function fetchRoomData(accessToken, roomId) { avatarUrl, historyVisibility, joinRule, + roomCreationTs, + predecessorRoomId, + predecessorLastKnownEventId, + predecessorViaServers, + successorRoomId, + successorSetTs, }; -} +}); -module.exports = traceFunction(fetchRoomData); +module.exports = { + fetchRoomData, + fetchRoomCreationInfo, + fetchPredecessorInfo, + fetchSuccessorInfo, + removeMe_fetchRoomCreateEventId, +}; diff --git a/server/lib/matrix-utils/get-server-name-from-matrix-room-id-or-alias.js b/server/lib/matrix-utils/get-server-name-from-matrix-room-id-or-alias.js index 1ddcf5a..1f51545 100644 --- a/server/lib/matrix-utils/get-server-name-from-matrix-room-id-or-alias.js +++ b/server/lib/matrix-utils/get-server-name-from-matrix-room-id-or-alias.js @@ -4,6 +4,9 @@ const assert = require('assert'); // See https://spec.matrix.org/v1.5/appendices/#server-name function getServerNameFromMatrixRoomIdOrAlias(roomIdOrAlias) { + // `roomIdOrAlias` looks like `!foo:matrix.org` or `#foo:matrix.org` or even something + // as crazy as `!foo:[1234:5678::abcd]:1234` where `[1234:5678::abcd]:1234` is the + // server name part we're trying to parse out (see tests for more examples) assert(roomIdOrAlias); const pieces = roomIdOrAlias.split(':'); diff --git a/server/routes/room-routes.js b/server/routes/room-routes.js index 26faf50..16a4416 100644 --- a/server/routes/room-routes.js +++ b/server/routes/room-routes.js @@ -12,10 +12,15 @@ const redirectToCorrectArchiveUrlIfBadSigil = require('./redirect-to-correct-arc const { HTTPResponseError } = require('../lib/fetch-endpoint'); const parseViaServersFromUserInput = require('../lib/parse-via-servers-from-user-input'); -const fetchRoomData = require('../lib/matrix-utils/fetch-room-data'); +const { + fetchRoomData, + fetchPredecessorInfo, + fetchSuccessorInfo, +} = require('../lib/matrix-utils/fetch-room-data'); const fetchEventsFromTimestampBackwards = require('../lib/matrix-utils/fetch-events-from-timestamp-backwards'); const ensureRoomJoined = require('../lib/matrix-utils/ensure-room-joined'); const timestampToEvent = require('../lib/matrix-utils/timestamp-to-event'); +const { removeMe_fetchRoomCreateEventId } = require('../lib/matrix-utils/fetch-room-data'); const getMessagesResponseFromEventId = require('../lib/matrix-utils/get-messages-response-from-event-id'); const renderHydrogenVmRenderScriptToPageHtml = require('../hydrogen-render/render-hydrogen-vm-render-script-to-page-html'); const MatrixPublicArchiveURLCreator = require('matrix-public-archive-shared/lib/url-creator'); @@ -23,6 +28,7 @@ const { MS_LOOKUP, TIME_PRECISION_VALUES, DIRECTION, + VALID_ENTITY_DESCRIPTOR_TO_SIGIL_MAP, } = require('matrix-public-archive-shared/lib/reference-values'); const { ONE_DAY_IN_MS, ONE_HOUR_IN_MS, ONE_MINUTE_IN_MS, ONE_SECOND_IN_MS } = MS_LOOKUP; const { @@ -57,10 +63,6 @@ const router = express.Router({ mergeParams: true, }); -const VALID_ENTITY_DESCRIPTOR_TO_SIGIL_MAP = { - r: '#', - roomid: '!', -}; const validSigilList = Object.values(VALID_ENTITY_DESCRIPTOR_TO_SIGIL_MAP); const sigilRe = new RegExp(`^(${validSigilList.join('|')})`); @@ -231,31 +233,46 @@ router.get( '?dir query parameter must be [f|b]' ); + const timelineStartEventId = req.query.timelineStartEventId; + assert( + ['string', 'undefined'].includes(typeof timelineStartEventId), + `?timelineStartEventId must be a string or undefined but saw ${typeof timelineStartEventId}` + ); + const timelineEndEventId = req.query.timelineEndEventId; + assert( + ['string', 'undefined'].includes(typeof timelineStartEventId), + `?timelineEndEventId must be a string or undefined but saw ${typeof timelineStartEventId}` + ); + + // We have to wait for the room join to happen first before we can use the jump to + // date endpoint (or any other Matrix endpoint) + const viaServers = parseViaServersFromUserInput(req.query.via); + const roomId = await ensureRoomJoined(matrixAccessToken, roomIdOrAlias, viaServers); + let ts; + let fromCausalEventId; if (dir === DIRECTION.backward) { // We `- 1` so we don't jump to the same event because the endpoint is inclusive. // // XXX: This is probably an edge-case flaw when there could be multiple events at // the same timestamp + // + // TODO: Remove the `- 1` when we have the MSC3999 causal event ID support ts = currentRangeStartTs - 1; + fromCausalEventId = timelineStartEventId; } else if (dir === DIRECTION.forward) { // We `+ 1` so we don't jump to the same event because the endpoint is inclusive // // XXX: This is probably an edge-case flaw when there could be multiple events at // the same timestamp + // + // TODO: Remove the `+ 1` when we have the MSC3999 causal event ID support ts = currentRangeEndTs + 1; + fromCausalEventId = timelineEndEventId; } else { - throw new Error(`Unable to handle unknown dir=${dir} in /jump`); + throw new StatusError(400, `Unable to handle unknown dir=${dir} in /jump`); } - // We have to wait for the room join to happen first before we can use the jump to - // date endpoint - const roomId = await ensureRoomJoined( - matrixAccessToken, - roomIdOrAlias, - parseViaServersFromUserInput(req.query.via) - ); - let eventIdForClosestEvent; let tsForClosestEvent; let newOriginServerTs; @@ -265,14 +282,53 @@ router.get( // updated value between each e2e test const archiveMessageLimit = config.get('archiveMessageLimit'); + let roomCreateEventId; // Find the closest event to the given timestamp - ({ eventId: eventIdForClosestEvent, originServerTs: tsForClosestEvent } = - await timestampToEvent({ - accessToken: matrixAccessToken, - roomId, - ts: ts, - direction: dir, - })); + [{ eventId: eventIdForClosestEvent, originServerTs: tsForClosestEvent }, roomCreateEventId] = + await Promise.all([ + timestampToEvent({ + accessToken: matrixAccessToken, + roomId, + ts: ts, + direction: dir, + // Since timestamps are untrusted and can be crafted to make loops in the + // timeline. We use this as a signal to keep progressing from this event + // regardless of what timestamp shenanigans are going on. See MSC3999 + // (https://github.com/matrix-org/matrix-spec-proposals/pull/3999) + // + // TODO: Add tests for timestamp loops once Synapse supports MSC3999. We + // currently just have this set in case some server has this implemented in + // the future but there currently is no implementation (as of 2023-04-17) and + // we can't have passing tests without a server implementation first. + 'org.matrix.msc3999.event_id': fromCausalEventId, + }), + removeMe_fetchRoomCreateEventId(matrixAccessToken, roomId), + ]); + + // Without MSC3999, we currently only detect one kind of loop where the + // `m.room.create` has a timestamp that comes after the timestamp massaged events + // in the room. This is a common pattern for historical Gitter rooms where we + // created the room and then imported a bunch of messages at a time before the + // room was created. + // + // By nature of having an `timelineEndEventId`, we know we are already paginated + // past the `m.room.create` event which is always the first event in the room. So + // we can use that to detect the end of the room before we loop back around to the + // start of the room. + // + // XXX: Once we have MSC3999, we can remove this check in favor of that mechanism + if ( + dir === DIRECTION.forward && + timelineEndEventId && + eventIdForClosestEvent === roomCreateEventId + ) { + throw new StatusError( + 404, + `/jump?dir=${dir}: We detected a loop back to the beginning of the room so we can assume ` + + `we hit the end of the room instead of doing a loop. We throw a 404 error here we hit ` + + `the normal 404 no more /messages error handling below` + ); + } // Based on what we found was the closest, figure out the URL that will represent // the next chunk in the desired direction. @@ -280,24 +336,31 @@ router.get( // // When jumping backwards, since a given room archive URL represents the end of // the day/time-period looking backward (scroll is also anchored to the bottom), - // we just need to get the user to the previous time-period. + // we just need to move the user to the time-period just prior the current one. // // We are trying to avoid sending the user to the same time period they were just // viewing. i.e, if they were visiting `/2020/01/02T16:00:00` (displays messages // backwards from that time up to the limit), which had more messages than we // could display in that day, jumping backwards from the earliest displayed event - // in the displayed range, say `T12:00:05` would still give us the same day - // `/2020/01/02` and we want to redirect them to previous chunk from that same - // day, like `/2020/01/02T12:00:00` + // in the displayed range (say that occured on `T12:00:25`) would still give us + // the same day `/2020/01/02` and we want to redirect them to previous chunk from + // that same day that still encompasses the closest message looking backwards, + // like `/2020/01/02T13:00:00` if (dir === DIRECTION.backward) { + // We choose `currentRangeStartTs` instead of `ts` (the jump point) because + // TODO: why? and we don't choose `currentRangeEndTs` because TODO: why? - I + // feel like I can't justify this, see + // https://github.com/matrix-org/matrix-public-archive/pull/167#discussion_r1170850432 const fromSameDay = - tsForClosestEvent && areTimestampsFromSameUtcDay(currentRangeEndTs, tsForClosestEvent); + tsForClosestEvent && areTimestampsFromSameUtcDay(currentRangeStartTs, tsForClosestEvent); const fromSameHour = - tsForClosestEvent && areTimestampsFromSameUtcHour(currentRangeEndTs, tsForClosestEvent); + tsForClosestEvent && areTimestampsFromSameUtcHour(currentRangeStartTs, tsForClosestEvent); const fromSameMinute = - tsForClosestEvent && areTimestampsFromSameUtcMinute(currentRangeEndTs, tsForClosestEvent); + tsForClosestEvent && + areTimestampsFromSameUtcMinute(currentRangeStartTs, tsForClosestEvent); const fromSameSecond = - tsForClosestEvent && areTimestampsFromSameUtcSecond(currentRangeEndTs, tsForClosestEvent); + tsForClosestEvent && + areTimestampsFromSameUtcSecond(currentRangeStartTs, tsForClosestEvent); // The closest event is from the same second we tried to jump from. Since we // can't represent something smaller than a second in the URL yet (we could do @@ -355,13 +418,11 @@ router.get( // XXX: This is flawed in the fact that when we go `/messages?dir=b` later, it // could backfill messages which will fill up the response before we perfectly // connect and continue from the position they were jumping from before. When - // `/messages?dir=f` backfills, we won't have this problem anymore because any - // messages backfilled in the forwards direction would be picked up the same going - // backwards. - if (dir === DIRECTION.forward) { - // Use `/messages?dir=f` and get the `end` pagination token to paginate from. And - // then start the scroll from the top of the page so they can continue. - // + // `/messages?dir=f` backfills (forwards fill), we won't have this problem anymore + // because any messages backfilled in the forwards direction would be picked up + // the same going backwards. See MSC4000 + // (https://github.com/matrix-org/matrix-spec-proposals/pull/4000). + else if (dir === DIRECTION.forward) { // XXX: It would be cool to somehow cache this response and re-use our work here // for the actual room display that we redirect to from this route. No need for // us go out 100 messages, only for us to go backwards 100 messages again in the @@ -377,13 +438,50 @@ router.get( if (!messageResData.chunk?.length) { throw new StatusError( 404, - `/jump?dir=${dir}: /messages response didn't contain any more messages to jump to` + `/jump?dir=${dir}: /messages response didn't contain any more messages to jump to so we can assume we reached the end of the room.` ); } - const timestampOfLastMessage = - messageResData.chunk[messageResData.chunk.length - 1].origin_server_ts; - const dateOfLastMessage = new Date(timestampOfLastMessage); + const firstMessage = messageResData.chunk[0]; + const tsOfFirstMessage = firstMessage.origin_server_ts; + + const lastMessage = messageResData.chunk[messageResData.chunk.length - 1]; + const tsOfLastMessage = lastMessage.origin_server_ts; + + let msGapFromJumpPointToLastMessage; + // If someone is jumping from `0`, let's assume this is their first time + // navigating in the room and are just trying to get to the first messages in + // the room. Instead of using `0` which give us `moreThanDayGap=true` every time + // (unless someone sent messages in 1970 :P), and round us down to the nearest + // day before any of the messages in the room start, let's just use the start of + // the timeline as the start which will show us a page of content on the first + // try. For the backwards direction, we could have a similar check but with + // `currentRangeStartTs === Infinity` check but it's not necessary since we + // don't have to do any back-tracking extra work. + if (currentRangeEndTs === 0) { + msGapFromJumpPointToLastMessage = tsOfLastMessage - tsOfFirstMessage; + } + // Otherwise do the normal calculation: where we jumped to - where we jumped from + else { + // TODO: Should we use `ts` or `currentRangeStartTs` here? + msGapFromJumpPointToLastMessage = tsOfLastMessage - ts; + } + const moreThanDayGap = msGapFromJumpPointToLastMessage > ONE_DAY_IN_MS; + const moreThanHourGap = msGapFromJumpPointToLastMessage > ONE_HOUR_IN_MS; + const moreThanMinuteGap = msGapFromJumpPointToLastMessage > ONE_MINUTE_IN_MS; + const moreThanSecondGap = msGapFromJumpPointToLastMessage > ONE_SECOND_IN_MS; + + // If the first message is on different day than the last message, then we know + // there are messages on days before the last mesage and can safely round to the + // nearest day and still see new content. + // + // We use this information to handle situations where we jump over multiple-day + // gaps with no messages in between. In those cases, we don't want to round down + // to a day where there are no messages in the gap. + const hasMessagesOnDayBeforeDayOfLastMessage = !areTimestampsFromSameUtcDay( + tsOfFirstMessage, + tsOfLastMessage + ); // Back-track from the last message timestamp to the nearest date boundary. // Because we're back-tracking a couple events here, when we paginate back out @@ -395,29 +493,25 @@ router.get( // back-tracking but then we get ugly URL's every time you jump instead of being // able to back-track and round down to the nearest hour in a lot of cases. The // other reason not to return the exact date is maybe there multiple messages at - // the same timestamp and we will lose messages in the gap it displays more than - // we thought. - const msGapFromJumpPointToLastMessage = timestampOfLastMessage - ts; - const moreThanDayGap = msGapFromJumpPointToLastMessage > ONE_DAY_IN_MS; - const moreThanHourGap = msGapFromJumpPointToLastMessage > ONE_HOUR_IN_MS; - const moreThanMinuteGap = msGapFromJumpPointToLastMessage > ONE_MINUTE_IN_MS; - const moreThanSecondGap = msGapFromJumpPointToLastMessage > ONE_SECOND_IN_MS; - + // the same timestamp and we will lose messages in the gap because it displays + // more than we thought. + // // If the `/messages` response returns less than the `archiveMessageLimit` // looking forwards, it means we're looking at the latest events in the room. We - // can simply just display the day that the latest event occured on or given + // can simply just display the day that the latest event occured on or the given // rangeEnd (whichever is later). const haveReachedLatestMessagesInRoom = messageResData.chunk?.length < archiveMessageLimit; if (haveReachedLatestMessagesInRoom) { - const latestDesiredTs = Math.max(currentRangeEndTs, timestampOfLastMessage); + const latestDesiredTs = Math.max(currentRangeEndTs, tsOfLastMessage); const latestDesiredDate = new Date(latestDesiredTs); const utcMidnightTs = getUtcStartOfDayTs(latestDesiredDate); newOriginServerTs = utcMidnightTs; preferredPrecision = TIME_PRECISION_VALUES.none; } - // More than a day gap here, so we can just back-track to the nearest day - else if (moreThanDayGap) { - const utcMidnightOfDayBefore = getUtcStartOfDayTs(dateOfLastMessage); + // More than a day gap here, so we can just back-track to the nearest day as + // long as there are messages we haven't seen yet if we visit the nearest day. + else if (moreThanDayGap && hasMessagesOnDayBeforeDayOfLastMessage) { + const utcMidnightOfDayBefore = getUtcStartOfDayTs(tsOfLastMessage); // We `- 1` from UTC midnight to get the timestamp that is a millisecond // before the next day but we choose a no time precision so we jump to just // the bare date without a time. A bare date in the `/date/2022/12/16` @@ -429,19 +523,19 @@ router.get( } // More than a hour gap here, we will need to back-track to the nearest hour else if (moreThanHourGap) { - const utcTopOfHourBefore = getUtcStartOfHourTs(dateOfLastMessage); + const utcTopOfHourBefore = getUtcStartOfHourTs(tsOfLastMessage); newOriginServerTs = utcTopOfHourBefore; preferredPrecision = TIME_PRECISION_VALUES.minutes; } // More than a minute gap here, we will need to back-track to the nearest minute else if (moreThanMinuteGap) { - const utcTopOfMinuteBefore = getUtcStartOfMinuteTs(dateOfLastMessage); + const utcTopOfMinuteBefore = getUtcStartOfMinuteTs(tsOfLastMessage); newOriginServerTs = utcTopOfMinuteBefore; preferredPrecision = TIME_PRECISION_VALUES.minutes; } // More than a second gap here, we will need to back-track to the nearest second else if (moreThanSecondGap) { - const utcTopOfSecondBefore = getUtcStartOfSecondTs(dateOfLastMessage); + const utcTopOfSecondBefore = getUtcStartOfSecondTs(tsOfLastMessage); newOriginServerTs = utcTopOfSecondBefore; preferredPrecision = TIME_PRECISION_VALUES.seconds; } @@ -459,11 +553,134 @@ router.get( } } } catch (err) { - const is404Error = err instanceof HTTPResponseError && err.response.status === 404; + const is404HTTPResponseError = + err instanceof HTTPResponseError && err.response.status === 404; + const is404StatusError = err instanceof StatusError && err.status === 404; + const is404Error = is404HTTPResponseError || is404StatusError; + // A 404 error just means there is no more messages to paginate in that room and + // we should try to go to the predecessor/successor room appropriately. + if (is404Error) { + if (dir === DIRECTION.backward) { + const { + currentRoomCreationTs, + predecessorRoomId, + predecessorLastKnownEventId, + predecessorViaServers, + } = await fetchPredecessorInfo(matrixAccessToken, roomId); + + if (!predecessorRoomId) { + throw new StatusError( + 404, + `No predecessor room found for ${roomId} so we can't jump backwards to anywhere (you already reached the end of the room)` + ); + } + + // We have to join the predecessor room before we can fetch the successor info + // (this could be our first time seeing the room) + await ensureRoomJoined(matrixAccessToken, predecessorRoomId, viaServers); + const { + successorRoomId: successorRoomIdForPredecessor, + successorSetTs: successorSetTsForPredecessor, + } = await fetchSuccessorInfo(matrixAccessToken, predecessorRoomId); + + let tombstoneEventId; + if (!predecessorLastKnownEventId) { + // This is a hack because we can't get the tombstone event ID directly from + // `fetchSuccessorInfo(...)` and the `/state?format=event` + // endpoint, so we have to do this trick. Related to + // https://github.com/matrix-org/synapse/issues/15454 + // + // We just assume this is the tombstone event ID but in any case it gets us to + // an event that happened at the same time. + ({ eventId: tombstoneEventId } = await timestampToEvent({ + accessToken: matrixAccessToken, + roomId: predecessorRoomId, + ts: successorSetTsForPredecessor, + direction: DIRECTION.backward, + })); + } + + // Try to continue from the tombstone event in the predecessor room because + // that is the signal that the room admins gave to indicate the end of the + // room in favor of the other regardless of further activity that may have + // occured in the room. + // + // Make sure the the room that the predecessor specifies as the replacement + // room is the same as what the current room is. This is a good signal that + // the rooms are a true continuation of each other and the room admins agree. + let continueAtTsInPredecessorRoom; + if (successorRoomIdForPredecessor === roomId) { + continueAtTsInPredecessorRoom = successorSetTsForPredecessor; + } + // Fallback to the room creation event time if we can't find the predecessor + // room tombstone which will work just fine and as expected for normal room + // upgrade scenarios. + else { + continueAtTsInPredecessorRoom = currentRoomCreationTs; + } + + if ( + continueAtTsInPredecessorRoom === null || + continueAtTsInPredecessorRoom === undefined + ) { + throw new StatusError( + 500, + `You navigated past the end of the room and it has a predecessor set (${predecessorRoomId}) ` + + `but we were unable to find a suitable place to jump to and continue from. ` + + `We could just redirect you to that predecessor room but we decided to throw an error ` + + `instead because we should be able to fallback to the room creation time in any case. ` + + `In other words, there shouldn't be a reason why we can't fetch the \`m.room.create\`` + + `event for this room unless the server is just broken right now. You can try refreshing to try again.` + ); + } + + // Jump to the predecessor room at the appropriate timestamp to continue from. + // Since we're going backwards, we already know where to go so we can navigate + // straight there. + res.redirect( + matrixPublicArchiveURLCreator.archiveUrlForDate( + predecessorRoomId, + // XXX: We should probably go fetch and use the timestamp from + // `predecessorLastKnownEventId` here but that requires an extra + // `timestampToEvent(...)` lookup. We can assume it's close to the + // tombstone for now. + new Date(continueAtTsInPredecessorRoom), + { + viaServers: Array.from(predecessorViaServers || []), + scrollStartEventId: predecessorLastKnownEventId || tombstoneEventId, + // We can just visit a rough time where the tombstone is as we assume + // it's the last event in the room or at least the last event we care + // about. A given day should be good for most cases but it's possible + // that messages are sent after the tombstone and we end up missing the + // tombstone. + preferredPrecision: TIME_PRECISION_VALUES.none, + } + ) + ); + return; + } else if (dir === DIRECTION.forward) { + const { successorRoomId } = await fetchSuccessorInfo(matrixAccessToken, roomId); + if (successorRoomId) { + // Jump to the successor room and continue at the first event of the room + res.redirect( + matrixPublicArchiveURLCreator.archiveJumpUrlForRoom(successorRoomId, { + dir: DIRECTION.forward, + currentRangeStartTs: 0, + currentRangeEndTs: 0, + // We don't need to define + // `currentRangeStartEventId`/`currentRangeEndEventId` here because we're + // jumping to a completely new room so the event IDs won't pertain to the + // new room and we don't have any to use anyway. + }) + ); + return; + } + } + } // Only throw if it's something other than a 404 error. 404 errors are fine, they // just mean there is no more messages to paginate in that room and we were // already viewing the latest in the room. - if (!is404Error) { + else { throw err; } } @@ -511,6 +728,7 @@ router.get( timeoutMiddleware, // eslint-disable-next-line max-statements, complexity asyncHandler(async function (req, res) { + const nowTs = Date.now(); const roomIdOrAlias = getRoomIdOrAliasFromReq(req); // We pull this fresh from the config for each request to ensure we have an @@ -532,8 +750,8 @@ router.get( precisionFromUrl = TIME_PRECISION_VALUES.minutes; } - // Just 404 if anyone is trying to view the future, no need to waste resources on that - const nowTs = Date.now(); + // Just 404 if anyone is trying to view the future, no need to waste resources on + // that if (toTimestamp > roundUpTimestampToUtcDay(nowTs)) { throw new StatusError( 404, @@ -545,11 +763,12 @@ router.get( // We have to wait for the room join to happen first before we can fetch // any of the additional room info or messages. - const roomId = await ensureRoomJoined( - matrixAccessToken, - roomIdOrAlias, - parseViaServersFromUserInput(req.query.via) - ); + // + // XXX: It would be better if we just tried fetching first and assume that we are + // already joined and only join after we see a 403 Forbidden error (we should do + // this for all places we `ensureRoomJoined`) + const viaServers = parseViaServersFromUserInput(req.query.via); + const roomId = await ensureRoomJoined(matrixAccessToken, roomIdOrAlias, viaServers); // Do these in parallel to avoid the extra time in sequential round-trips // (we want to display the archive page faster) @@ -577,16 +796,65 @@ router.get( // Only `world_readable` or `shared` rooms that are `public` are viewable in the archive const allowedToViewRoom = - roomData?.historyVisibility === 'world_readable' || - (roomData?.historyVisibility === 'shared' && roomData?.joinRule === 'public'); + roomData.historyVisibility === 'world_readable' || + (roomData.historyVisibility === 'shared' && roomData.joinRule === 'public'); if (!allowedToViewRoom) { throw new StatusError( 403, - `Only \`world_readable\` or \`shared\` rooms that are \`public\` can be viewed in the archive. ${roomData.id} has m.room.history_visiblity=${roomData?.historyVisibility} m.room.join_rules=${roomData?.joinRule}` + `Only \`world_readable\` or \`shared\` rooms that are \`public\` can be viewed in the archive. ${roomData.id} has m.room.history_visiblity=${roomData.historyVisibility} m.room.join_rules=${roomData.joinRule}` ); } + // Since we're looking backwards from the given day, if we don't see any events, + // then we can assume that it's before the start of the room (it's the only way we + // would see no events). + const hasNavigatedBeforeStartOfRoom = events.length === 0; + // Check if we need to navigate backward to the predecessor room + if (hasNavigatedBeforeStartOfRoom && roomData.predecessorRoomId) { + // Jump to the predecessor room at the date/time the user is trying to visit at + res.redirect( + matrixPublicArchiveURLCreator.archiveUrlForDate( + roomData.predecessorRoomId, + new Date(toTimestamp), + { + preferredPrecision: precisionFromUrl, + // XXX: Should we also try combining `viaServers` we used to get to this room? + viaServers: Array.from(roomData.predecessorViaServers || []), + } + ) + ); + return; + } + + // We only care to navigate to the successor room if we're trying to view something + // past when the successor was set (it's an indicator that we need to go to the new + // room from this time forward). + const isNavigatedPastSuccessor = toTimestamp > roomData.successorSetTs; + // But if we're viewing the day when the successor was set, we want to allow viewing + // the room up until the successor was set. + const newestEvent = events[events.length - 1]; + const isNewestEventFromSameDay = + newestEvent && + newestEvent?.origin_server_ts && + areTimestampsFromSameUtcDay(toTimestamp, newestEvent?.origin_server_ts); + // Check if we need to navigate forward to the successor room + if (roomData.successorRoomId && isNavigatedPastSuccessor && !isNewestEventFromSameDay) { + // Jump to the successor room at the date/time the user is trying to visit at + res.redirect( + matrixPublicArchiveURLCreator.archiveUrlForDate( + roomData.successorRoomId, + new Date(toTimestamp), + { + preferredPrecision: precisionFromUrl, + // Just try to pass on the `viaServers` the user was using to get to this room + viaServers: Array.from(viaServers || []), + } + ) + ); + return; + } + // Default to no indexing (safe default) let shouldIndex = false; if (stopSearchEngineIndexing) { @@ -600,6 +868,7 @@ router.get( const stylesUrl = urlJoin(basePath, '/css/styles.css'); const jsBundleUrl = urlJoin(basePath, '/js/entry-client-hydrogen.es.js'); + // XXX: The `renderHydrogenVmRenderScriptToPageHtml` API surface is pretty awkward const pageHtml = await renderHydrogenVmRenderScriptToPageHtml( path.resolve(__dirname, '../../shared/hydrogen-vm-render-script.js'), { diff --git a/shared/lib/reference-values.js b/shared/lib/reference-values.js index d9fec80..caf08ac 100644 --- a/shared/lib/reference-values.js +++ b/shared/lib/reference-values.js @@ -28,9 +28,15 @@ const VALID_SIGIL_TO_ENTITY_DESCRIPTOR_MAP = { '!': 'roomid', }; +const VALID_ENTITY_DESCRIPTOR_TO_SIGIL_MAP = { + r: '#', + roomid: '!', +}; + module.exports = { MS_LOOKUP, TIME_PRECISION_VALUES, DIRECTION, VALID_SIGIL_TO_ENTITY_DESCRIPTOR_MAP, + VALID_ENTITY_DESCRIPTOR_TO_SIGIL_MAP, }; diff --git a/shared/lib/timestamp-utilities.js b/shared/lib/timestamp-utilities.js index a5ed40d..1990add 100644 --- a/shared/lib/timestamp-utilities.js +++ b/shared/lib/timestamp-utilities.js @@ -32,6 +32,7 @@ function roundUpTimestampToUtcSecond(ts) { return dateRountedUp.getTime(); } +// XXX: Should these just be renamed to `roundDownTimestampToUtcDay`? function getUtcStartOfDayTs(ts) { assert(typeof ts === 'number' || ts instanceof Date); const date = new Date(ts); diff --git a/shared/lib/url-creator.js b/shared/lib/url-creator.js index 02e5ca3..4285d21 100644 --- a/shared/lib/url-creator.js +++ b/shared/lib/url-creator.js @@ -58,6 +58,7 @@ class URLCreator { archiveUrlForRoom(roomIdOrAlias, { viaServers = [] } = {}) { assert(roomIdOrAlias); + assert(Array.isArray(viaServers)); let qs = new URLSearchParams(); [].concat(viaServers).forEach((viaServer) => { qs.append('via', viaServer); @@ -75,6 +76,7 @@ class URLCreator { ) { assert(roomIdOrAlias); assert(date); + assert(Array.isArray(viaServers)); // `preferredPrecision` is optional but if they gave a value, make sure it's something expected if (preferredPrecision) { assert( @@ -117,16 +119,39 @@ class URLCreator { )}${qsToUrlPiece(qs)}`; } - archiveJumpUrlForRoom(roomIdOrAlias, { dir, currentRangeStartTs, currentRangeEndTs }) { + archiveJumpUrlForRoom( + roomIdOrAlias, + { + dir, + currentRangeStartTs, + currentRangeEndTs, + timelineStartEventId, + timelineEndEventId, + viaServers = [], + } + ) { assert(roomIdOrAlias); assert(dir); - assert(currentRangeStartTs); - assert(currentRangeEndTs); + assert(typeof currentRangeStartTs === 'number'); + assert(typeof currentRangeEndTs === 'number'); + assert(Array.isArray(viaServers)); + // `timelineStartEventId` and `timelineEndEventId` are optional because the + // timeline could be showing 0 events or we could be jumping with no knowledge of + // what was shown before. let qs = new URLSearchParams(); qs.append('dir', dir); qs.append('currentRangeStartTs', currentRangeStartTs); qs.append('currentRangeEndTs', currentRangeEndTs); + if (timelineStartEventId) { + qs.append('timelineStartEventId', timelineStartEventId); + } + if (timelineEndEventId) { + qs.append('timelineEndEventId', timelineEndEventId); + } + [].concat(viaServers).forEach((viaServer) => { + qs.append('via', viaServer); + }); const urlPath = this._getArchiveUrlPathForRoomIdOrAlias(roomIdOrAlias); diff --git a/shared/viewmodels/ArchiveRoomViewModel.js b/shared/viewmodels/ArchiveRoomViewModel.js index 84b0d78..6ff0bd5 100644 --- a/shared/viewmodels/ArchiveRoomViewModel.js +++ b/shared/viewmodels/ArchiveRoomViewModel.js @@ -363,6 +363,9 @@ class ArchiveRoomViewModel extends ViewModel { // to paginate from const jumpRangeEndTimestamp = this._dayTimestampTo; + const timelineStartEventId = events[0]?.event_id; + const timelineEndEventId = events[events.length - 1]?.event_id; + // Check whether the given day represented in the URL has any events on the page // from that day. We only need to check the last event which would be closest to // `_dayTimestampTo` anyway. @@ -394,6 +397,8 @@ class ArchiveRoomViewModel extends ViewModel { canonicalAlias: this._room.canonicalAlias, jumpRangeStartTimestamp, jumpRangeEndTimestamp, + timelineStartEventId, + timelineEndEventId, // This is a bit cheating but I don't know how else to pass this kind of // info to the Tile viewmodel basePath: this._basePath, @@ -417,6 +422,8 @@ class ArchiveRoomViewModel extends ViewModel { dayTimestamp: this._dayTimestampTo, jumpRangeStartTimestamp, jumpRangeEndTimestamp, + timelineStartEventId, + timelineEndEventId, // This is a bit cheating but I don't know how else to pass this kind of // info to the Tile viewmodel basePath: this._basePath, diff --git a/shared/viewmodels/JumpToNextActivitySummaryTileViewModel.js b/shared/viewmodels/JumpToNextActivitySummaryTileViewModel.js index fd88a0b..7876985 100644 --- a/shared/viewmodels/JumpToNextActivitySummaryTileViewModel.js +++ b/shared/viewmodels/JumpToNextActivitySummaryTileViewModel.js @@ -38,6 +38,16 @@ class JumpToNextActivitySummaryTileViewModel extends SimpleTile { return this._entry?.content?.['jumpRangeEndTimestamp']; } + // The first event shown in the timeline. + get timelineStartEventId() { + return this._entry?.content?.['timelineStartEventId']; + } + + // The last event shown in the timeline. + get timelineEndEventId() { + return this._entry?.content?.['timelineEndEventId']; + } + get jumpToNextActivityUrl() { return this._matrixPublicArchiveURLCreator.archiveJumpUrlForRoom( this._entry?.content?.['canonicalAlias'] || this._entry.roomId, @@ -45,6 +55,8 @@ class JumpToNextActivitySummaryTileViewModel extends SimpleTile { dir: DIRECTION.forward, currentRangeStartTs: this.jumpRangeStartTimestamp, currentRangeEndTs: this.jumpRangeEndTimestamp, + timelineStartEventId: this.timelineStartEventId, + timelineEndEventId: this.timelineEndEventId, } ); } diff --git a/shared/viewmodels/JumpToPreviousActivitySummaryTileViewModel.js b/shared/viewmodels/JumpToPreviousActivitySummaryTileViewModel.js index db72b8e..6f2d684 100644 --- a/shared/viewmodels/JumpToPreviousActivitySummaryTileViewModel.js +++ b/shared/viewmodels/JumpToPreviousActivitySummaryTileViewModel.js @@ -30,6 +30,16 @@ class JumpToPreviousActivitySummaryTileViewModel extends SimpleTile { return this._entry?.content?.['jumpRangeEndTimestamp']; } + // The first event shown in the timeline. + get timelineStartEventId() { + return this._entry?.content?.['timelineStartEventId']; + } + + // The last event shown in the timeline. + get timelineEndEventId() { + return this._entry?.content?.['timelineEndEventId']; + } + get jumpToPreviousActivityUrl() { return this._matrixPublicArchiveURLCreator.archiveJumpUrlForRoom( this._entry?.content?.['canonicalAlias'] || this._entry.roomId, @@ -37,6 +47,8 @@ class JumpToPreviousActivitySummaryTileViewModel extends SimpleTile { dir: DIRECTION.backward, currentRangeStartTs: this.jumpRangeStartTimestamp, currentRangeEndTs: this.jumpRangeEndTimestamp, + timelineStartEventId: this.timelineStartEventId, + timelineEndEventId: this.timelineEndEventId, } ); } diff --git a/test/e2e-tests.js b/test/e2e-tests.js index 32ded28..ebd63df 100644 --- a/test/e2e-tests.js +++ b/test/e2e-tests.js @@ -9,6 +9,7 @@ const urlJoin = require('url-join'); const escapeStringRegexp = require('escape-string-regexp'); const { parseHTML } = require('linkedom'); const { readFile } = require('fs').promises; +const chalk = require('chalk'); const RethrownError = require('../server/lib/rethrown-error'); const MatrixPublicArchiveURLCreator = require('matrix-public-archive-shared/lib/url-creator'); @@ -25,11 +26,13 @@ const { getTestClientForAs, getTestClientForHs, createTestRoom, + upgradeTestRoom, getCanonicalAlias, joinRoom, sendEvent, sendMessage, createMessagesInRoom, + getMessagesInRoom, updateProfile, uploadContent, } = require('./test-utils/client-utils'); @@ -133,7 +136,7 @@ describe('matrix-public-archive', () => { true, `Expected ${event.event_id} (${event.type}: "${ event.content.body - }") to be in room on hs2=${JSON.stringify(room2EventIds)}` + }") to be in room on hs2=${JSON.stringify(room2EventIds, null, 2)}` ); }); }); @@ -147,7 +150,7 @@ describe('matrix-public-archive', () => { const archiveDate = new Date(Date.UTC(2022, 0, 15)); let archiveUrl; let numMessagesSent = 0; - afterEach(() => { + afterEach(function () { if (interactive) { // eslint-disable-next-line no-console console.log('Interactive URL for test', archiveUrl); @@ -159,8 +162,16 @@ describe('matrix-public-archive', () => { // changes in the UI). numMessagesSent = 0; - // Reset any custom modifications made for a particular test - config.reset(); + // Reset any custom modifications made for a particular test. + // + // We don't reset when interactive and there is a failure because we want to be + // able to preview the interactive URL with the same config as seen in the test + // and because we also set `--bail` when running `npm run test-e2e-interactive`, + // the tests stop running after the first failure so it doesn't leak state between + // tests. + if (!interactive && this.currentTest.state !== 'passed') { + config.reset(); + } }); // Sends a message and makes sure that a timestamp was provided @@ -757,497 +768,7 @@ describe('matrix-public-archive', () => { }); describe('Jump forwards and backwards', () => { - const jumpTestCases = [ - { - // In order to jump from the 1st page to the 2nd, we first jump forward 4 - // messages, then back-track to the first date boundary which is day3. We do - // this so that we don't start from day4 backwards which would miss messages - // because there are more than 5 messages in between day4 and day2. - // - // Even though there is overlap between the pages, our scroll continues from - // the event where the 1st page starts. - testName: 'can jump forward to the next activity', - // Create enough surround messages on nearby days that overflow the page - // limit but don't overflow the limit on a single day basis. We create 4 - // days of messages so we can see a seamless continuation from page1 to - // page2. - roomDayMessageStructureString: ` - [room1 ] - 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 - [day1 ] [day2 ] [day3 ] [day4 ] - [page1 ] - |--jump-fwd-4-messages-->| - [page2 ] - `, - startUrl: '/r/room1/date/2022/01/02', - page1: { - url: '/r/room1/date/2022/01/02', - action: 'next', - }, - page2: { - url: '/r/room1/date/2022/01/03?at=$event7', - action: null, - }, - }, - { - // This test is just slightly different and jumps further into day4 (just a - // slight variation to make sure it still does the correct thing) - // - // In order to jump from the 1st page to the 2nd, we first jump forward 4 - // messages, then back-track to the first date boundary which is day3. There - // is exactly 5 messages between day4 and day2 which would be a perfect next - // page but because we hit the middle of day4, we have no idea how many more - // messages are in day4. - // - // Even though there is overlap between the pages, our scroll continues from - // the event where the 1st page starts. - testName: 'can jump forward to the next activity2', - roomDayMessageStructureString: ` - [room1 ] - 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 - [day1 ] [day2 ] [day3 ] [day4 ] - [page1 ] - |--jump-fwd-4-messages-->| - [page2 ] - `, - startUrl: '/r/room1/date/2022/01/02', - page1: { - url: '/r/room1/date/2022/01/02', - action: 'next', - }, - page2: { - url: '/r/room1/date/2022/01/03?at=$event7', - action: null, - }, - }, - { - // In order to jump from the 1st page to the 2nd, we first "jump" forward 4 - // messages by paginating `/messages?limit=4` but it only returns 2x - // messages (event11 and event12) which is less than our limit of 4, so we - // know we reached the end and can simply display the day that the latest - // event occured on. - testName: 'can jump forward to the latest activity in the room (same day)', - roomDayMessageStructureString: ` - [room1 ] - 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 - [day1 ] [day2 ] [day3 ] [day4 ] - [page1 ] - |--jump-fwd-4-messages-->| - [page2 ] - `, - startUrl: '/r/room1/date/2022/01/04T01:00', - page1: { - url: '/r/room1/date/2022/01/04T01:00', - action: 'next', - }, - page2: { - url: '/r/room1/date/2022/01/04?at=$event11', - action: null, - }, - }, - { - // In order to jump from the 1st page to the 2nd, we first "jump" forward 4 - // messages by paginating `/messages?limit=4` but it only returns 3x - // messages (event10, event11, event12) which is less than our limit of 4, - // so we know we reached the end and can simply display the day that the - // latest event occured on. - testName: 'can jump forward to the latest activity in the room (different day)', - roomDayMessageStructureString: ` - [room1 ] - 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 - [day1 ] [day2 ] [day3 ] [day4 ] - [page1 ] - |---jump-fwd-4-messages--->| - [page2 ] - `, - startUrl: '/r/room1/date/2022/01/03', - page1: { - url: '/r/room1/date/2022/01/03', - action: 'next', - }, - page2: { - url: '/r/room1/date/2022/01/04?at=$event10', - action: null, - }, - }, - // This test currently doesn't work because it finds the primordial room - // creation events which are created in now time vs the timestamp massaging we - // do for the message fixtures. We can uncomment this once Synapse supports - // timestamp massaging for `/createRoom`, see - // https://github.com/matrix-org/synapse/issues/15346 - // - // { - // // In order to jump from the 1st page to the 2nd, we first "jump" forward 4 - // // messages by paginating `/messages?limit=4` but it returns no messages - // // which is less than our limit of 4, so we know we reached the end and can - // // simply TODO - // testName: - // 'can jump forward to the latest activity in the room (when already viewing the latest activity)', - // roomDayMessageStructureString: ` - // [room1 ] - // 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 - // [day1 ] [day2 ] [day3 ] [day4 ] - // [page1 ] - // |---jump-fwd-4-messages--->| - // [page2 ] - // `, - // startUrl: '/r/room1/date/2022/01/04', - // page1: { - // url: '/r/room1/date/2022/01/04', - // action: 'next', - // }, - // page2: { - // // If we can't find any more messages to paginate to, we just progress the - // // date forward by a day so we can display the empty view for that day. - // // - // // TODO: This page probably doesn't need a `?at=` continue event - // url: '/r/room1/date/2022/01/05?at=TODO', - // action: null, - // }, - // }, - { - // Test to make sure we can jump from the 1st page to the 2nd page forwards - // even when it exactly paginates to the last message of the next day. We're - // testing this specifically to ensure that you actually jump to the next - // day (previously failed with naive flawed code). - // - // In order to jump from the 1st page to the 2nd, we first jump forward 3 - // messages, then back-track to the first date boundary which is the nearest - // hour backwards from event9. We use the nearest hour because there is - // less than a day of gap between event6 and event9 and we fallback from - // nearest day to hour boundary. - // - testName: - 'can jump forward to the next activity even when it exactly goes to the end of the next day', - roomDayMessageStructureString: ` - [room1 ] - 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 - [day1 ] [day2 ] [day3 ] [day4 ] - [page1 ] - |-jump-fwd-3-msg->| - [page2 ] - `, - startUrl: '/r/room1/date/2022/01/02', - page1: { - url: '/r/room1/date/2022/01/02', - action: 'next', - }, - page2: { - // We expect the URL to look like `T02:00` because we're rendering part way - // through day3 and while we could get away with just hour precision, the - // default precision has hours and minutes. - url: '/r/room1/date/2022/01/03T02:00?at=$event7', - action: null, - }, - }, - { - // From the first page with too many messages, starting at event5(page1 - // rangeStart), we look backwards for the closest event. Because we find - // event4 as the closest, which is from a different day from event9(page1 - // rangeEnd), we can just display the day where event5 resides. - // - // Even though there is overlap between - // the pages, our scroll continues from the event where the 1st page starts. - testName: 'can jump backward to the previous activity', - roomDayMessageStructureString: ` - [room1 ] - 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 - [day1 ] [day2 ] [day3 ] [day4 ] - [page1 ] - [page2 ] - `, - startUrl: '/r/room1/date/2022/01/03', - page1: { - url: '/r/room1/date/2022/01/03', - action: 'previous', - }, - page2: { - // Continuing from the first event of day2 since we already saw the rest - // of day2 in the first page - url: '/r/room1/date/2022/01/02?at=$event4', - action: null, - }, - }, - { - // In order to jump from the 1st page to the 2nd, we first jump forward 8 - // messages, then back-track to the first date boundary which is the nearest - // day backwards from event20. We use the nearest day because there is more - // than a day of gap between event12 and event20. - testName: - 'can jump forward over many quiet days without losing any messages in the gap', - roomDayMessageStructureString: ` - [room1 ] - 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 <-- 15 <-- 16 <-- 17 <-- 18 <-- 19 <-- 20 <-- 21 - [day1 ] [day2 ] [day3 ] [day4 ] [day5 ] [day6 ] [day7 ] [day8 ] - [page1 ] - |------------------jump-fwd-8-msg---------------------->| - [page2 ] - `, - startUrl: '/r/room1/date/2022/01/04', - page1: { - url: '/r/room1/date/2022/01/04', - action: 'next', - }, - page2: { - url: '/r/room1/date/2022/01/07?at=$event13', - action: null, - }, - }, - { - // From the first page with too many messages, starting at event13 (page1 - // rangeStart), we look backwards for the closest event. Because we find - // event12 as the closest, which is from the a different day from event21 - // (page1 rangeEnd), we can just display the day where event12 resides. - testName: 'can jump backward to the previous activity with many small quiet days', - roomDayMessageStructureString: ` - [room1 ] - 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 <-- 15 <-- 16 <-- 17 <-- 18 <-- 19 <-- 20 <-- 21 - [day1 ] [day2 ] [day3 ] [day4 ] [day5 ] [day6 ] [day7 ] [day8 ] [day9 ] - [page1 ] - [page2 ] - `, - startUrl: '/r/room1/date/2022/01/09', - page1: { - url: '/r/room1/date/2022/01/09', - action: 'previous', - }, - page2: { - url: '/r/room1/date/2022/01/06?at=$event12', - action: null, - }, - }, - { - // Test to make sure we can jump forwards from the 1st page to the 2nd page - // with too many messages to display on a single day. - // - // We jump forward 4 messages (`archiveMessageLimit`), then back-track to - // the nearest hour which starts us from event9, and then we display 5 - // messages because we fetch one more than `archiveMessageLimit` to - // determine overflow. - testName: 'can jump forward to the next activity and land in too many messages', - roomDayMessageStructureString: ` - [room1 ] - 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 <-- 15 - [day1 ] [day2 ] [day3 ] [day4 ] - [page1 ] - |--jump-fwd-4-messages-->| - [page2 ] - `, - startUrl: '/r/room1/date/2022/01/02', - page1: { - url: '/r/room1/date/2022/01/02', - action: 'next', - }, - page2: { - url: '/r/room1/date/2022/01/03T03:00?at=$event7', - action: null, - }, - }, - { - // Test to make sure we can jump backwards from the 1st page to the 2nd page - // with too many messages to display on a single day. - // - // From the first page with too many messages, starting at event10 (page1 - // rangeStart), we look backwards for the closest event. Because we find - // event9 as the closest, which is from the a different day from event14 - // (page1 rangeEnd), we can just display the day where event9 resides. - testName: 'can jump backward to the previous activity and land in too many messages', - roomDayMessageStructureString: ` - [room1 ] - 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 - [day1 ] [day2 ] [day3 ] [day4 ] - [page1 ] - [page2 ] - `, - startUrl: '/r/room1/date/2022/01/04', - page1: { - url: '/r/room1/date/2022/01/04', - action: 'previous', - }, - page2: { - url: '/r/room1/date/2022/01/02?at=$event9', - action: null, - }, - }, - { - // We jump forward 4 messages (`archiveMessageLimit`) to event12, then - // back-track to the nearest hour which starts off at event11 and render the - // page with 5 messages because we fetch one more than `archiveMessageLimit` - // to determine overflow. - testName: - 'can jump forward from one day with too many messages into the next day with too many messages', - roomDayMessageStructureString: ` - [room1 ] - 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 - [day1 ] [day2 ] [day3 ] - [page1 ] - |---jump-fwd-4-messages--->| - [page2 ] - `, - startUrl: '/r/room1/date/2022/01/02', - page1: { - url: '/r/room1/date/2022/01/02', - action: 'next', - }, - page2: { - url: '/r/room1/date/2022/01/03T03:00?at=$event9', - action: null, - }, - }, - { - // From the first page with too many messages, starting at event10 (page1 - // rangeStart), we look backwards for the closest event. Because we find - // event9 as the closest, which is from the same day as event14 (page1 - // rangeEnd), we round up to the nearest hour so that the URL encompasses it - // when looking backwards. - testName: - 'can jump backward from one day with too many messages into the previous day with too many messages', - roomDayMessageStructureString: ` - [room1 ] - 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 - [day1 ] [day2 ] [day3 ] - [page1 ] - [page2 ] - `, - startUrl: '/r/room1/date/2022/01/03', - page1: { - url: '/r/room1/date/2022/01/03', - action: 'previous', - }, - page2: { - url: '/r/room1/date/2022/01/03T01:00?at=$event9', - action: null, - }, - }, - { - // We jump forward 4 messages (`archiveMessageLimit`) to event12, then - // back-track to the nearest hour which starts off at event11 and render the - // page with 5 messages because we fetch one more than `archiveMessageLimit` - // to determine overflow. - testName: - 'can jump forward from one day with too many messages into the same day with too many messages', - roomDayMessageStructureString: ` - [room1 ] - 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 - [day1 ] [day2 ] - [page1 ] - |---jump-fwd-4-messages--->| - [page2 ] - `, - startUrl: '/r/room1/date/2022/01/02T6:00', - page1: { - url: '/r/room1/date/2022/01/02T6:00', - action: 'next', - }, - page2: { - url: '/r/room1/date/2022/01/02T09:00?at=$event9', - action: null, - }, - }, - { - // From the first page with too many messages, starting at event10 (page1 - // rangeStart), we look backwards for the closest event. Because we find - // event9 as the closest, which is from the same day as event14 (page1 - // rangeEnd), we round up to the nearest hour so that the URL encompasses it - // when looking backwards. - testName: - 'can jump backward from one day with too many messages into the same day with too many messages', - roomDayMessageStructureString: ` - [room1 ] - 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 - [day1 ] [day2 ] - [page1 ] - [page2 ] - `, - startUrl: '/r/room1/date/2022/01/02T11:00', - page1: { - url: '/r/room1/date/2022/01/02T11:00', - action: 'previous', - }, - page2: { - url: '/r/room1/date/2022/01/02T06:00?at=$event8', - action: null, - }, - }, - { - // We jump forward 4 messages (`archiveMessageLimit`) to event12, then - // back-track to the nearest hour which starts off at event11 and render the - // page with 5 messages because we fetch one more than `archiveMessageLimit` - // to determine overflow. - testName: - 'can jump forward from the middle of one day with too many messages into the next day with too many messages', - roomDayMessageStructureString: ` - [room1 ] - 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 - [day1 ] [day2 ] [day3 ] - [page1 ] - |---jump-fwd-4-messages--->| - [page2 ] - `, - startUrl: '/r/room1/date/2022/01/02T06:00', - page1: { - url: '/r/room1/date/2022/01/02T06:00', - action: 'next', - }, - page2: { - // Continuing from the unseen event in day2 - url: '/r/room1/date/2022/01/03T02:00?at=$event9', - action: null, - }, - }, - { - // From the first page with too many messages, starting at event10 (page1 - // rangeStart), we look backwards for the closest event. Because we find - // event9 as the closest, which is from the same day as event14 (page1 - // rangeEnd), we round up to the nearest hour so that the URL encompasses it - // when looking backwards. - testName: - 'can jump backward from the middle of one day with too many messages into the previous day with too many messages', - roomDayMessageStructureString: ` - [room1 ] - 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 - [day1 ] [day2 ] [day3 ] - [page1 ] - [page2 ] - `, - startUrl: '/r/room1/date/2022/01/03T06:00', - page1: { - url: '/r/room1/date/2022/01/03T06:00', - action: 'previous', - }, - page2: { - url: '/r/room1/date/2022/01/03T01:00?at=$event8', - action: null, - }, - }, - { - // From the first page with too many messages, starting at event8 (page1 - // rangeStart), we look backwards for the closest event. Because we find - // event7 as the closest, which is from a different day than event12 (page1 - // rangeEnd), we can just display the day where event7 resides. - // - testName: - 'can jump backward from the start of one day with too many messages into the previous day with exactly the limit', - roomDayMessageStructureString: ` - [room1 ] - 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 - [day1 ] [day2 ] [day3 ] - [page1 ] - [page2 ] - `, - startUrl: '/r/room1/date/2022/01/03T05:00', - page1: { - url: '/r/room1/date/2022/01/03T05:00', - action: 'previous', - }, - page2: { - url: '/r/room1/date/2022/01/02?at=$event7', - action: null, - }, - }, - ]; - - jumpTestCases.forEach((testCase) => { + function runJumpTestCase(testCase) { // eslint-disable-next-line max-statements, complexity it(testCase.testName, async () => { // Setup @@ -1256,6 +777,35 @@ describe('matrix-public-archive', () => { const eventMap = new Map(); const fancyIdentifierToEventIdMap = new Map(); const eventIdToFancyIdentifierMap = new Map(); + const fancyIdentifierToRoomIdMap = new Map(); + const roomIdToFancyIdentifierMap = new Map(); + const fancyRoomIdToDebugEventsInRoom = new Map(); + + // String used to log out all possible events in the room + function getDebugStringForEventsInRoomsAndLookForEventId(eventIdToLookFor) { + return `For reference, here are all of the events in the rooms: ${JSON.stringify( + Object.fromEntries( + Array.from(fancyRoomIdToDebugEventsInRoom.entries()).map((entry) => { + const fancyRoomId = entry[0]; + const newKey = `${fancyRoomId} - ${fancyIdentifierToRoomIdMap.get( + fancyRoomId + )}`; + return [newKey, entry[1]]; + }) + ), + null, + 2 + ) + .split(/\r?\n/) + .map((line) => { + if (eventIdToLookFor && line.includes(eventIdToLookFor)) { + return chalk.yellow(line); + } + + return line; + }) + .join('\n')}`; + } function convertFancyIdentifierListToDebugEventIds(fancyEventIdentifiers) { // eslint-disable-next-line max-nested-callbacks @@ -1263,7 +813,7 @@ describe('matrix-public-archive', () => { const eventId = fancyIdentifierToEventIdMap.get(fancyId); if (!eventId) { throw new Error( - `Unable to find ${fancyId} in the fancyIdentifierToEventMap=${JSON.stringify( + `Could not find fancy ID ${fancyId} in the fancyIdentifierToEventMap=${JSON.stringify( Object.fromEntries(fancyIdentifierToEventIdMap.entries()), null, 2 @@ -1282,11 +832,11 @@ describe('matrix-public-archive', () => { const fancyEventId = eventIdToFancyIdentifierMap.get(eventId); if (!fancyEventId) { throw new Error( - `Unable to find ${eventId} in the eventIdToFancyIdentifierMap=${JSON.stringify( + `Could not find event ID for ${eventId} in the eventIdToFancyIdentifierMap=${JSON.stringify( Object.fromEntries(eventIdToFancyIdentifierMap.entries()), null, 2 - )}` + )}\n${getDebugStringForEventsInRoomsAndLookForEventId(eventId)}` ); } const ts = eventMap.get(eventId)?.originServerTs; @@ -1295,17 +845,91 @@ describe('matrix-public-archive', () => { }); } + function convertUrlBetween(inputUrl, roomMap, eventMap) { + const { + roomIdOrAliasUrlPart: inputRoomIdOrAliasUrlPart, + roomIdOrAlias: inputRoomIdOrAlias, + //urlDateTime: actualUrlDateTime, + continueAtEvent: inputContinueAtEventId, + } = parseArchiveUrlForRoom(inputUrl); + + let outputContinueAtEventId; + if (inputContinueAtEventId) { + outputContinueAtEventId = eventMap.get(inputContinueAtEventId); + assert( + outputContinueAtEventId, + `Could not find event ID for ${inputContinueAtEventId} in the map=${JSON.stringify( + Object.fromEntries(eventMap.entries()), + null, + 2 + )}\n${getDebugStringForEventsInRoomsAndLookForEventId(inputContinueAtEventId)}` + ); + } + + const outputRoomIdOrAlias = roomMap.get(inputRoomIdOrAlias); + assert( + outputRoomIdOrAlias, + `Could not find room ID for ${inputRoomIdOrAlias} in our map ${JSON.stringify( + Object.fromEntries(roomMap.entries()), + null, + 2 + )}` + ); + + return inputUrl + .replace( + `/roomid/${inputRoomIdOrAliasUrlPart}`, + // Slice to remove the sigil + `/roomid/${outputRoomIdOrAlias.slice(1)}` + ) + .replace(inputContinueAtEventId, outputContinueAtEventId); + } + + function convertUrlWithFancyIdsToActualUrl(urlWithFancyIds) { + return convertUrlBetween( + urlWithFancyIds, + fancyIdentifierToRoomIdMap, + fancyIdentifierToEventIdMap + ); + } + + function convertActualUrlToUrlWithFancyIds(urlWithFancyIds) { + return convertUrlBetween( + urlWithFancyIds, + roomIdToFancyIdentifierMap, + eventIdToFancyIdentifierMap + ); + } + const client = await getTestClientForHs(testMatrixServerUrl1); - const { rooms, archiveMessageLimit, pages } = parseRoomDayMessageStructure( + const { rooms, pages } = parseRoomDayMessageStructure( testCase.roomDayMessageStructureString ); - const fancyIdentifierToRoomIdMap = new Map(); - const roomIdToFancyIdentifierMap = new Map(); + let previousRoomId; + let lastEventTsUsedInPreviousRoom; for (const [roomIndex, room] of rooms.entries()) { - // TODO: upgradeRoom to link one room to another - const roomId = await createTestRoom(client); - const fancyRoomId = `#room${roomIndex + 1}`; + let roomId; + if (previousRoomId) { + const { newRoomid, tombstoneEventId } = await upgradeTestRoom({ + client, + oldRoomId: previousRoomId, + //useMsc3946DynamicPredecessor: TODO: Enable this when we have a way to configure it. + // We `+ 1` just to space out the tombstone from the last event so + // things are sequential `/timestamp_to_event` doesn't get confused. + timestamp: lastEventTsUsedInPreviousRoom + 1, + }); + roomId = newRoomid; + + const fancyEventId = `$tombstone`; + fancyIdentifierToEventIdMap.set(fancyEventId, tombstoneEventId); + eventIdToFancyIdentifierMap.set(tombstoneEventId, fancyEventId); + } else { + // TODO: Pass `timestamp` massaging option to `createTestRoom()` when it + // supports it, see https://github.com/matrix-org/matrix-public-archive/issues/169 + roomId = await createTestRoom(client); + } + const fancyRoomId = `!room${roomIndex + 1}`; fancyIdentifierToRoomIdMap.set(fancyRoomId, roomId); roomIdToFancyIdentifierMap.set(roomId, fancyRoomId); @@ -1321,7 +945,8 @@ describe('matrix-public-archive', () => { // Just spread things out a bit so the event times are more obvious // and stand out from each other while debugging and so we just have // to deal with hour time slicing - const eventSendTimeIncrement = ONE_HOUR_IN_MS; + const eventSendTimeIncrement = + testCase.timeIncrementBetweenMessages || ONE_HOUR_IN_MS; for (const eventMeta of room.events) { const archiveDate = new Date(Date.UTC(2022, 0, eventMeta.dayNumber, 0, 0, 0, 1)); @@ -1340,6 +965,7 @@ describe('matrix-public-archive', () => { timestamp: originServerTs, }); eventMap.set(eventId, { + type: 'm.room.message', roomId, originServerTs, content, @@ -1347,7 +973,45 @@ describe('matrix-public-archive', () => { const fancyEventId = `$event${eventMeta.eventNumber}`; fancyIdentifierToEventIdMap.set(fancyEventId, eventId); eventIdToFancyIdentifierMap.set(eventId, fancyEventId); + lastEventTsUsedInPreviousRoom = originServerTs; } + + previousRoomId = roomId; + } + + // Assemble a list of events to to reference and assist with debugging when + // some assertion fails + for (const [fancyRoomId, roomId] of fancyIdentifierToRoomIdMap.entries()) { + const archiveAppServiceUserClient = await getTestClientForAs(); + const eventsInRoom = await getMessagesInRoom({ + client: archiveAppServiceUserClient, + roomId: roomId, + // This is arbitrarily larger than any amount of messages we would ever + // send in the tests + limit: 1000, + }); + const eventDebugStrings = eventsInRoom.map((event) => { + let relevantContentString = ''; + if (event.type === 'm.room.message' && event.content.msgtype === 'm.text') { + relevantContentString = ` "${event.content.body}"`; + } else if (event.type === 'm.room.create') { + const predecessorRoomId = event.content?.predecessor?.room_id; + if (predecessorRoomId) { + relevantContentString = ` "predecessor=${predecessorRoomId}"`; + } + } else if (event.type === 'm.room.tombstone') { + const replacementRoomId = event.content?.replacement_room; + if (replacementRoomId) { + relevantContentString = ` "successor=${replacementRoomId}"`; + } + } + + return `${event.type}${event.state_key ? ` (${event.state_key})` : ''}: ${ + event.event_id + }${relevantContentString} - ${new Date(event.origin_server_ts).toISOString()}`; + }); + + fancyRoomIdToDebugEventsInRoom.set(fancyRoomId, eventDebugStrings); } // Now Test @@ -1355,7 +1019,8 @@ describe('matrix-public-archive', () => { // -------------------------------------- // Make sure the archive is configured as the test expects - config.set('archiveMessageLimit', archiveMessageLimit); + assert(testCase.archiveMessageLimit); + config.set('archiveMessageLimit', testCase.archiveMessageLimit); // eslint-disable-next-line max-nested-callbacks const pagesKeyList = Object.keys(testCase).filter((key) => { @@ -1392,7 +1057,9 @@ describe('matrix-public-archive', () => { assert( startRoomIdOrAlias, `Could not find room ID for ${startRoomFancyKey} in our list of known rooms ${JSON.stringify( - Object.fromEntries(fancyIdentifierToRoomIdMap.entries()) + Object.fromEntries(fancyIdentifierToRoomIdMap.entries()), + null, + 2 )}` ); archiveUrl = `${matrixPublicArchiveURLCreator.archiveUrlForRoom( @@ -1421,7 +1088,9 @@ describe('matrix-public-archive', () => { assert( expectedRoomId, `Could not find room ID for ${expectedRoomFancyId} in our list of known rooms ${JSON.stringify( - Object.fromEntries(fancyIdentifierToRoomIdMap.entries()) + Object.fromEntries(fancyIdentifierToRoomIdMap.entries()), + null, + 2 )}` ); @@ -1430,45 +1099,25 @@ describe('matrix-public-archive', () => { archiveUrl ); const pageDom = parseHTML(archivePageHtml); - const { - roomIdOrAliasUrlPart: actualRoomIdOrAliasUrlPart, - roomIdOrAlias: actualRoomId, - //urlDateTime: actualUrlDateTime, - continueAtEvent: actualContinueAtEventId, - } = parseArchiveUrlForRoom(pageRes.url); - const actualRoomFancyId = roomIdToFancyIdentifierMap.get(actualRoomId); - assert( - actualRoomFancyId, - `Could not find room ID for ${actualRoomId} in our list of known rooms ${JSON.stringify( - Object.fromEntries(roomIdToFancyIdentifierMap.entries()) - )}` - ); - let actualContinueAtEventFancyId; - if (actualContinueAtEventId) { - actualContinueAtEventFancyId = - eventIdToFancyIdentifierMap.get(actualContinueAtEventId); - assert( - actualContinueAtEventFancyId, - `Could not find event ID for ${actualContinueAtEventId} in our list of known events ${JSON.stringify( - Object.fromEntries(eventIdToFancyIdentifierMap.entries()) - )}` - ); - } - // Replace messy room ID's and event ID's that change with every test - // run with their fancy ID's which correlate with the test meta so it's - // easier to reason about things when the assertion fails. - let actualUrlWithFancyIdentifies = pageRes.url - .replace( - `/roomid/${actualRoomIdOrAliasUrlPart}`, - // Slice to remove the sigil - `/r/${actualRoomFancyId.slice(1)}` - ) - .replace(actualContinueAtEventId, actualContinueAtEventFancyId); + const eventIdsOnPage = [...pageDom.document.querySelectorAll(`[data-event-id]`)] + // eslint-disable-next-line max-nested-callbacks + .map((eventEl) => { + return eventEl.getAttribute('data-event-id'); + }); + // Assert the correct room and time precision in the URL + const actualUrlWithFancyIdentifies = convertActualUrlToUrlWithFancyIds(pageRes.url); assert.match( actualUrlWithFancyIdentifies, - new RegExp(`${escapeStringRegexp(pageTestMeta.url)}$`) + new RegExp(`${escapeStringRegexp(pageTestMeta.url)}$`), + `The actual URL (${actualUrlWithFancyIdentifies}) for the page did not match the expected URL (${ + pageTestMeta.url + }).\nFor reference, here are the events on the page: ${JSON.stringify( + eventIdsOnPage, + null, + 2 + )}\n${getDebugStringForEventsInRoomsAndLookForEventId()}` ); // If provided, assert that it's a smooth continuation to more messages. @@ -1490,11 +1139,20 @@ describe('matrix-public-archive', () => { assert.strictEqual(continationDebugEventId, expectedContinuationDebugEventId); } - const eventIdsOnPage = [...pageDom.document.querySelectorAll(`[data-event-id]`)] - // eslint-disable-next-line max-nested-callbacks - .map((eventEl) => { - return eventEl.getAttribute('data-event-id'); - }); + // We only care about messages for now (no easy way to specify the + // primordial room creation or member events in the test expectations) + const eventIdsOnPageWeCareAboutToAssert = eventIdsOnPage.filter((eventId) => { + const event = eventMap.get(eventId); + if (!event) { + return false; + } + + assert( + event?.type, + `Event should have a type: ${JSON.stringify(event, null, 2)}}` + ); + return event?.type === 'm.room.message'; + }); const pageNumber = pageKey.replace('page', ''); const page = pages[pageNumber - 1]; @@ -1506,17 +1164,22 @@ describe('matrix-public-archive', () => { // Assert that the page contains all expected events assert.deepEqual( - convertEventIdsToDebugEventIds(eventIdsOnPage), + convertEventIdsToDebugEventIds(eventIdsOnPageWeCareAboutToAssert), convertFancyIdentifierListToDebugEventIds(expectedFancyIdsOnPage), `Events on ${pageKey} should be as expected` ); // Follow the next activity link. Aka, fetch messages for the 2nd page let actionLinkSelector; + let nextPageLink; if (pageTestMeta.action === 'next') { actionLinkSelector = '[data-testid="jump-to-next-activity-link"]'; } else if (pageTestMeta.action === 'previous') { actionLinkSelector = '[data-testid="jump-to-previous-activity-link"]'; + } else if (pageTestMeta.action?.startsWith('navigate:')) { + const navigateUrlWithFancyIds = pageTestMeta.action.replace('navigate:', ''); + const fullNavigateUrlWithFancyIds = urlJoin(basePath, navigateUrlWithFancyIds); + nextPageLink = convertUrlWithFancyIdsToActualUrl(fullNavigateUrlWithFancyIds); } else if (pageTestMeta.action === null) { // No more pages to test ✅, move on alreadyEncounteredLastPage = true; @@ -1526,15 +1189,20 @@ describe('matrix-public-archive', () => { `Unexpected value for ${pageKey}.action=${pageTestMeta.action} that we don't know what to do with` ); } - const jumpToActivityLinkEl = pageDom.document.querySelector(actionLinkSelector); - const jumpToActivityLinkHref = jumpToActivityLinkEl.getAttribute('href'); + + if (actionLinkSelector) { + const jumpToActivityLinkEl = pageDom.document.querySelector(actionLinkSelector); + const jumpToActivityLinkHref = jumpToActivityLinkEl.getAttribute('href'); + nextPageLink = jumpToActivityLinkHref; + } + // Move to the next iteration of the loop // // Set this for debugging if the test fails here - archiveUrl = jumpToActivityLinkHref; + archiveUrl = nextPageLink; } catch (err) { const errorWithContext = new RethrownError( - `Encountered error while asserting ${pageKey}: ${err.message}`, + `Encountered error while asserting ${pageKey}: (see original error below)`, err ); // Copy these over so mocha generates a nice diff for us @@ -1546,6 +1214,1047 @@ describe('matrix-public-archive', () => { } } }); + } + + const jumpTestCases = [ + { + // In order to jump from the 1st page to the 2nd, we first jump forward 4 + // messages, then back-track to the first date boundary which is day3. We do + // this so that we don't start from day4 backwards which would miss messages + // because there are more than 5 messages in between day4 and day2. + // + // Even though there is overlap between the pages, our scroll continues from + // the event where the 1st page starts. + testName: 'can jump forward to the next activity', + // Create enough surround messages on nearby days that overflow the page + // limit but don't overflow the limit on a single day basis. We create 4 + // days of messages so we can see a seamless continuation from page1 to + // page2. + roomDayMessageStructureString: ` + [room1 ] + 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 + [day1 ] [day2 ] [day3 ] [day4 ] + [page1 ] + |--jump-fwd-4-messages-->| + [page2 ] + `, + archiveMessageLimit: 4, + startUrl: '/roomid/room1/date/2022/01/02', + page1: { + url: '/roomid/room1/date/2022/01/02', + action: 'next', + }, + page2: { + url: '/roomid/room1/date/2022/01/03?at=$event7', + action: null, + }, + }, + { + // This test is just slightly different and jumps further into day4 (just a + // slight variation to make sure it still does the correct thing) + // + // In order to jump from the 1st page to the 2nd, we first jump forward 4 + // messages, then back-track to the first date boundary which is day3. There + // is exactly 5 messages between day4 and day2 which would be a perfect next + // page but because we hit the middle of day4, we have no idea how many more + // messages are in day4. + // + // Even though there is overlap between the pages, our scroll continues from + // the event where the 1st page starts. + testName: 'can jump forward to the next activity2', + roomDayMessageStructureString: ` + [room1 ] + 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 + [day1 ] [day2 ] [day3 ] [day4 ] + [page1 ] + |--jump-fwd-4-messages-->| + [page2 ] + `, + archiveMessageLimit: 4, + startUrl: '/roomid/room1/date/2022/01/02', + page1: { + url: '/roomid/room1/date/2022/01/02', + action: 'next', + }, + page2: { + url: '/roomid/room1/date/2022/01/03?at=$event7', + action: null, + }, + }, + { + // In order to jump from the 1st page to the 2nd, we first "jump" forward 4 + // messages by paginating `/messages?limit=4` but it only returns 2x + // messages (event11 and event12) which is less than our limit of 4, so we + // know we reached the end and can simply display the day that the latest + // event occured on. + testName: 'can jump forward to the latest activity in the room (same day)', + roomDayMessageStructureString: ` + [room1 ] + 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 + [day1 ] [day2 ] [day3 ] [day4 ] + [page1 ] + |--jump-fwd-4-messages-->| + [page2 ] + `, + archiveMessageLimit: 4, + startUrl: '/roomid/room1/date/2022/01/04T01:00', + page1: { + url: '/roomid/room1/date/2022/01/04T01:00', + action: 'next', + }, + page2: { + url: '/roomid/room1/date/2022/01/04?at=$event11', + action: null, + }, + }, + { + // In order to jump from the 1st page to the 2nd, we first "jump" forward 4 + // messages by paginating `/messages?limit=4` but it only returns 3x + // messages (event10, event11, event12) which is less than our limit of 4, + // so we know we reached the end and can simply display the day that the + // latest event occured on. + testName: 'can jump forward to the latest activity in the room (different day)', + roomDayMessageStructureString: ` + [room1 ] + 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 + [day1 ] [day2 ] [day3 ] [day4 ] + [page1 ] + |---jump-fwd-4-messages--->| + [page2 ] + `, + archiveMessageLimit: 4, + startUrl: '/roomid/room1/date/2022/01/03', + page1: { + url: '/roomid/room1/date/2022/01/03', + action: 'next', + }, + page2: { + url: '/roomid/room1/date/2022/01/04?at=$event10', + action: null, + }, + }, + // This test currently doesn't work because it finds the primordial room + // creation events which are created in now time vs the timestamp massaging we + // do for the message fixtures. We can uncomment this once Synapse supports + // timestamp massaging for `/createRoom`, see + // https://github.com/matrix-org/matrix-public-archive/issues/169 + // + // { + // // In order to jump from the 1st page to the 2nd, we first "jump" forward 4 + // // messages by paginating `/messages?limit=4` but it returns no messages + // // which is less than our limit of 4, so we know we reached the end and can + // // simply TODO + // testName: + // 'can jump forward to the latest activity in the room (when already viewing the latest activity)', + // roomDayMessageStructureString: ` + // [room1 ] + // 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 + // [day1 ] [day2 ] [day3 ] [day4 ] + // [page1 ] + // |---jump-fwd-4-messages--->| + // [page2 ] + // `, + // archiveMessageLimit: 4, + // startUrl: '/roomid/room1/date/2022/01/04', + // page1: { + // url: '/roomid/room1/date/2022/01/04', + // action: 'next', + // }, + // page2: { + // // If we can't find any more messages to paginate to, we just progress the + // // date forward by a day so we can display the empty view for that day. + // // + // // TODO: This page probably doesn't need a `?at=` continue event + // url: '/roomid/room1/date/2022/01/05?at=TODO', + // action: null, + // }, + // }, + { + // Test to make sure we can jump from the 1st page to the 2nd page forwards + // even when it exactly paginates to the last message of the next day. We're + // testing this specifically to ensure that you actually jump to the next + // day (previously failed with naive flawed code). + // + // In order to jump from the 1st page to the 2nd, we first jump forward 3 + // messages, then back-track to the first date boundary which is the nearest + // hour backwards from event9. We use the nearest hour because there is + // less than a day of gap between event6 and event9 and we fallback from + // nearest day to hour boundary. + // + testName: + 'can jump forward to the next activity even when it exactly goes to the end of the next day', + roomDayMessageStructureString: ` + [room1 ] + 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 + [day1 ] [day2 ] [day3 ] [day4 ] + [page1 ] + |-jump-fwd-3-msg->| + [page2 ] + `, + archiveMessageLimit: 3, + startUrl: '/roomid/room1/date/2022/01/02', + page1: { + url: '/roomid/room1/date/2022/01/02', + action: 'next', + }, + page2: { + // We expect the URL to look like `T02:00` because we're rendering part way + // through day3 and while we could get away with just hour precision, the + // default precision has hours and minutes. + url: '/roomid/room1/date/2022/01/03T02:00?at=$event7', + action: null, + }, + }, + { + // From the first page with too many messages, starting at event5(page1 + // rangeStart), we look backwards for the closest event. Because we find + // event4 as the closest, which is from a different day from event9(page1 + // rangeEnd), we can just display the time where event4 resides. + testName: 'can jump backward to the previous activity', + roomDayMessageStructureString: ` + [room1 ] + 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 + [day1 ] [day2 ] [day3 ] [day4 ] + [page1 ] + [page2 ] + `, + archiveMessageLimit: 4, + startUrl: '/roomid/room1/date/2022/01/03', + page1: { + url: '/roomid/room1/date/2022/01/03', + action: 'previous', + }, + page2: { + // Continuing from the first event of day2 since we already saw the rest + // of day2 in the first page + url: '/roomid/room1/date/2022/01/02T01:00?at=$event4', + action: null, + }, + }, + { + // In order to jump from the 1st page to the 2nd, we first jump forward 8 + // messages, then back-track to the first date boundary which is the nearest + // day backwards from event20. We use the nearest day because there is more + // than a day of gap between event12 and event20. + testName: + 'can jump forward over many quiet days without losing any messages in the gap', + roomDayMessageStructureString: ` + [room1 ] + 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 <-- 15 <-- 16 <-- 17 <-- 18 <-- 19 <-- 20 <-- 21 + [day1 ] [day2 ] [day3 ] [day4 ] [day5 ] [day6 ] [day7 ] [day8 ] + [page1 ] + |------------------jump-fwd-8-msg---------------------->| + [page2 ] + `, + archiveMessageLimit: 8, + startUrl: '/roomid/room1/date/2022/01/04', + page1: { + url: '/roomid/room1/date/2022/01/04', + action: 'next', + }, + page2: { + url: '/roomid/room1/date/2022/01/07?at=$event13', + action: null, + }, + }, + { + // From the first page with too many messages, starting at event13 (page1 + // rangeStart), we look backwards for the closest event. Because we find + // event12 as the closest, which is from the a different day from event21 + // (page1 rangeEnd), we can just display the day where event12 resides. + testName: 'can jump backward to the previous activity with many small quiet days', + roomDayMessageStructureString: ` + [room1 ] + 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 <-- 15 <-- 16 <-- 17 <-- 18 <-- 19 <-- 20 <-- 21 + [day1 ] [day2 ] [day3 ] [day4 ] [day5 ] [day6 ] [day7 ] [day8 ] [day9 ] + [page1 ] + [page2 ] + `, + archiveMessageLimit: 8, + startUrl: '/roomid/room1/date/2022/01/09', + page1: { + url: '/roomid/room1/date/2022/01/09', + action: 'previous', + }, + page2: { + url: '/roomid/room1/date/2022/01/06?at=$event12', + action: null, + }, + }, + { + // Test to make sure we can jump forwards from the 1st page to the 2nd page + // with too many messages to display on a single day. + // + // We jump forward 4 messages (`archiveMessageLimit`), then back-track to + // the nearest hour which starts us from event9, and then we display 5 + // messages because we fetch one more than `archiveMessageLimit` to + // determine overflow. + testName: 'can jump forward to the next activity and land in too many messages', + roomDayMessageStructureString: ` + [room1 ] + 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 <-- 15 + [day1 ] [day2 ] [day3 ] [day4 ] + [page1 ] + |--jump-fwd-4-messages-->| + [page2 ] + `, + archiveMessageLimit: 4, + startUrl: '/roomid/room1/date/2022/01/02', + page1: { + url: '/roomid/room1/date/2022/01/02', + action: 'next', + }, + page2: { + url: '/roomid/room1/date/2022/01/03T03:00?at=$event7', + action: null, + }, + }, + { + // Test to make sure we can jump forwards from the 1st page to the 2nd page + // when there is a multiple-day gap between the end of the first page to the + // next messages. + // + // We jump forward 4 messages (`archiveMessageLimit`), then back-track to + // the nearest hour because even though there is more than a day gap in the + // jump, there aren't any mesages in between from another day. Because, we + // back-tracked to the nearest hour, this starts us from event9, and then we + // display 5 messages because we fetch one more than `archiveMessageLimit` + // to determine overflow. + testName: 'can jump forward to the next activity when there is a multiple day gap', + roomDayMessageStructureString: ` + [room1 ] + 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 + [day1 ] [day5 ] + [page1 ] + |--jump-fwd-4-messages-->| + [page2 ] + `, + archiveMessageLimit: 4, + startUrl: '/roomid/room1/date/2022/01/01', + page1: { + url: '/roomid/room1/date/2022/01/01', + action: 'next', + }, + page2: { + url: '/roomid/room1/date/2022/01/05T03:00?at=$event7', + action: null, + }, + }, + { + // Test to make sure we can jump backwards from the 1st page to the 2nd page + // with too many messages to display on a single day. + // + // From the first page with too many messages, starting at event10 (page1 + // rangeStart), we look backwards for the closest event. Because we find + // event9 as the closest, which is from the a different day from event14 + // (page1 rangeEnd), we can just display the day where event9 resides. + testName: 'can jump backward to the previous activity and land in too many messages', + roomDayMessageStructureString: ` + [room1 ] + 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 + [day1 ] [day2 ] [day3 ] [day4 ] + [page1 ] + [page2 ] + `, + archiveMessageLimit: 4, + startUrl: '/roomid/room1/date/2022/01/04', + page1: { + url: '/roomid/room1/date/2022/01/04', + action: 'previous', + }, + page2: { + url: '/roomid/room1/date/2022/01/02?at=$event9', + action: null, + }, + }, + { + // We jump forward 4 messages (`archiveMessageLimit`) to event12, then + // back-track to the nearest hour which starts off at event11 and render the + // page with 5 messages because we fetch one more than `archiveMessageLimit` + // to determine overflow. + testName: + 'can jump forward from one day with too many messages into the next day with too many messages', + roomDayMessageStructureString: ` + [room1 ] + 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 + [day1 ] [day2 ] [day3 ] + [page1 ] + |---jump-fwd-4-messages--->| + [page2 ] + `, + archiveMessageLimit: 4, + startUrl: '/roomid/room1/date/2022/01/02', + page1: { + url: '/roomid/room1/date/2022/01/02', + action: 'next', + }, + page2: { + url: '/roomid/room1/date/2022/01/03T03:00?at=$event9', + action: null, + }, + }, + { + // From the first page with too many messages, starting at event10 (page1 + // rangeStart), we look backwards for the closest event. Because we find + // event9 as the closest, which is from the same day as event14 (page1 + // rangeEnd), we round up to the nearest hour so that the URL encompasses it + // when looking backwards. + testName: + 'can jump backward from one day with too many messages into the previous day with too many messages', + roomDayMessageStructureString: ` + [room1 ] + 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 + [day1 ] [day2 ] [day3 ] + [page1 ] + [page2 ] + `, + archiveMessageLimit: 4, + startUrl: '/roomid/room1/date/2022/01/03', + page1: { + url: '/roomid/room1/date/2022/01/03', + action: 'previous', + }, + page2: { + url: '/roomid/room1/date/2022/01/03T01:00?at=$event9', + action: null, + }, + }, + { + // We jump forward 4 messages (`archiveMessageLimit`) to event12, then + // back-track to the nearest hour which starts off at event11 and render the + // page with 5 messages because we fetch one more than `archiveMessageLimit` + // to determine overflow. + testName: + 'can jump forward from one day with too many messages into the same day with too many messages', + roomDayMessageStructureString: ` + [room1 ] + 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 + [day1 ] [day2 ] + [page1 ] + |---jump-fwd-4-messages--->| + [page2 ] + `, + archiveMessageLimit: 4, + startUrl: '/roomid/room1/date/2022/01/02T6:00', + page1: { + url: '/roomid/room1/date/2022/01/02T6:00', + action: 'next', + }, + page2: { + url: '/roomid/room1/date/2022/01/02T09:00?at=$event9', + action: null, + }, + }, + { + // From the first page with too many messages, starting at event10 (page1 + // rangeStart), we look backwards for the closest event. Because we find + // event9 as the closest, which is from the same day as event14 (page1 + // rangeEnd), we round up to the nearest hour so that the URL encompasses it + // when looking backwards. + testName: + 'can jump backward from one day with too many messages into the same day with too many messages', + roomDayMessageStructureString: ` + [room1 ] + 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 + [day1 ] [day2 ] + [page1 ] + [page2 ] + `, + archiveMessageLimit: 4, + startUrl: '/roomid/room1/date/2022/01/02T11:00', + page1: { + url: '/roomid/room1/date/2022/01/02T11:00', + action: 'previous', + }, + page2: { + url: '/roomid/room1/date/2022/01/02T06:00?at=$event8', + action: null, + }, + }, + { + // We jump forward 4 messages (`archiveMessageLimit`) to event12, then + // back-track to the nearest hour which starts off at event11 and render the + // page with 5 messages because we fetch one more than `archiveMessageLimit` + // to determine overflow. + testName: + 'can jump forward from the middle of one day with too many messages into the next day with too many messages', + roomDayMessageStructureString: ` + [room1 ] + 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 + [day1 ] [day2 ] [day3 ] + [page1 ] + |---jump-fwd-4-messages--->| + [page2 ] + `, + archiveMessageLimit: 4, + startUrl: '/roomid/room1/date/2022/01/02T06:00', + page1: { + url: '/roomid/room1/date/2022/01/02T06:00', + action: 'next', + }, + page2: { + // Continuing from the unseen event in day2 + url: '/roomid/room1/date/2022/01/03T02:00?at=$event9', + action: null, + }, + }, + { + // From the first page with too many messages, starting at event10 (page1 + // rangeStart), we look backwards for the closest event. Because we find + // event9 as the closest, which is from the same day as event14 (page1 + // rangeEnd), we round up to the nearest hour so that the URL encompasses it + // when looking backwards. + testName: + 'can jump backward from the middle of one day with too many messages into the previous day with too many messages', + roomDayMessageStructureString: ` + [room1 ] + 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 + [day1 ] [day2 ] [day3 ] + [page1 ] + [page2 ] + `, + archiveMessageLimit: 4, + startUrl: '/roomid/room1/date/2022/01/03T06:00', + page1: { + url: '/roomid/room1/date/2022/01/03T06:00', + action: 'previous', + }, + page2: { + url: '/roomid/room1/date/2022/01/03T01:00?at=$event8', + action: null, + }, + }, + { + // From the first page with too many messages, starting at event8 (page1 + // rangeStart), we look backwards for the closest event. Because we find + // event7 as the closest, which is from a different day than event12 (page1 + // rangeEnd), we can just display the day where event7 resides. + testName: + 'can jump backward from the start of one day with too many messages into the previous day with exactly the limit', + roomDayMessageStructureString: ` + [room1 ] + 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 + [day1 ] [day2 ] [day3 ] + [page1 ] + [page2 ] + `, + archiveMessageLimit: 4, + startUrl: '/roomid/room1/date/2022/01/03T05:00', + page1: { + url: '/roomid/room1/date/2022/01/03T05:00', + action: 'previous', + }, + page2: { + url: '/roomid/room1/date/2022/01/02?at=$event7', + action: null, + }, + }, + // Tests for "less than X" for the forwards direction + // -------------------------------------------------- + { + // From the first page with too many messages, starting at event14 with + // minute precision in the URL, we look backwards for the closest event. + // Because we find event9 as the closest, where the page1 + // `currentRangeStartTs` is less than an hour away from event9, we have to + // round up to the nearest minute. + testName: + 'can jump backward to the previous activity when less than an hour between all messages', + roomDayMessageStructureString: ` + [room1 ] + 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 + [day1 ] + [page1 ] + [page2 ] + `, + // More than a minute for each but less than an hour when you multiply this + // across all of messages + timeIncrementBetweenMessages: 2 * ONE_MINUTE_IN_MS, + archiveMessageLimit: 4, + startUrl: '/roomid/room1/date/2022/01/01T01:00', + page1: { + url: '/roomid/room1/date/2022/01/01T01:00', + action: 'previous', + }, + page2: { + url: '/roomid/room1/date/2022/01/01T00:17?at=$event9', + action: null, + }, + }, + { + // From the first page with too many messages, starting at event14 with + // day precision in the URL, we look backwards for the closest event. + // Because we find event9 as the closest, where the page1 + // `currentRangeStartTs` is less than an hour away from event9, we have to + // round up to the nearest minute. + testName: + 'can jump backward to the previous activity when less than an hour between all messages (starting from day precision)', + roomDayMessageStructureString: ` + [room1 ] + 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 + [day1 ] + [page1 ] + [page2 ] + `, + // More than a minute for each but less than an hour when you multiply this + // across all of messages + timeIncrementBetweenMessages: 2 * ONE_MINUTE_IN_MS, + archiveMessageLimit: 4, + startUrl: '/roomid/room1/date/2022/01/01', + page1: { + url: '/roomid/room1/date/2022/01/01', + action: 'previous', + }, + page2: { + url: '/roomid/room1/date/2022/01/01T00:17?at=$event9', + action: null, + }, + }, + { + // From the first page with too many messages, starting at event14 with + // minute precision in the URL, we look backwards for the closest event. + // Because we find event9 as the closest, where the page1 + // `currentRangeStartTs` is less than an minute away from event9, we have to + // round up to the nearest second. + testName: + 'can jump backward to the previous activity when less than an minute between all messages', + roomDayMessageStructureString: ` + [room1 ] + 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 + [day1 ] + [page1 ] + [page2 ] + `, + // More than a second for each but less than an minute when you multiply + // this across all of messages + timeIncrementBetweenMessages: 2 * ONE_SECOND_IN_MS, + archiveMessageLimit: 4, + startUrl: '/roomid/room1/date/2022/01/01T00:01', + page1: { + url: '/roomid/room1/date/2022/01/01T00:01', + action: 'previous', + }, + page2: { + url: '/roomid/room1/date/2022/01/01T00:00:17?at=$event9', + action: null, + }, + }, + { + // From the first page with too many messages, starting at event14 with + // day precision in the URL, we look backwards for the closest event. + // Because we find event9 as the closest, where the page1 + // `currentRangeStartTs` is less than an minute away from event9, we have to + // round up to the nearest second. + testName: + 'can jump backward to the previous activity when less than an minute between all messages (starting from day precision)', + roomDayMessageStructureString: ` + [room1 ] + 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 + [day1 ] + [page1 ] + [page2 ] + `, + // More than a second for each but less than an minute when you multiply + // this across all of messages + timeIncrementBetweenMessages: 2 * ONE_SECOND_IN_MS, + archiveMessageLimit: 4, + startUrl: '/roomid/room1/date/2022/01/01', + page1: { + url: '/roomid/room1/date/2022/01/01', + action: 'previous', + }, + page2: { + url: '/roomid/room1/date/2022/01/01T00:00:17?at=$event9', + action: null, + }, + }, + // Tests for "less than X" for the forwards direction + // -------------------------------------------------- + // We can't do the `(start from day precision)` variants when jumping forwards + // because day precision starts off at `T23:59:59.999Z` and jumping forward + // will always land us in the next day. + { + // We jump forward 4 messages (`archiveMessageLimit`) to event10, then + // back-track to the nearest minute which starts off at event9 and render the + // page with 5 messages because we fetch one more than `archiveMessageLimit` + // to determine overflow. + testName: + 'can jump forward to the next activity when less than an hour between all messages', + roomDayMessageStructureString: ` + [room1 ] + 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 + [day1 ] + [page1 ] + |--jump-fwd-4-messages-->| + [page2 ] + `, + // More than a minute for each but less than an hour when you multiply this + // across all of messages + timeIncrementBetweenMessages: 2 * ONE_MINUTE_IN_MS, + archiveMessageLimit: 4, + startUrl: '/roomid/room1/date/2022/01/01T00:11', + page1: { + url: '/roomid/room1/date/2022/01/01T00:11', + action: 'next', + }, + page2: { + url: '/roomid/room1/date/2022/01/01T00:18?at=$event7', + action: null, + }, + }, + { + // We jump forward 4 messages (`archiveMessageLimit`) to event10, then + // back-track to the nearest second which starts off at event9 and render the + // page with 5 messages because we fetch one more than `archiveMessageLimit` + // to determine overflow. + testName: + 'can jump forward to the next activity when less than an minute between all messages', + roomDayMessageStructureString: ` + [room1 ] + 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 + [day1 ] + [page1 ] + |--jump-fwd-4-messages-->| + [page2 ] + `, + // More than a second for each but less than an minute when you multiply this + // across all of messages + timeIncrementBetweenMessages: 2 * ONE_SECOND_IN_MS, + archiveMessageLimit: 4, + startUrl: '/roomid/room1/date/2022/01/01T00:00:11', + page1: { + url: '/roomid/room1/date/2022/01/01T00:00:11', + action: 'next', + }, + page2: { + url: '/roomid/room1/date/2022/01/01T00:00:18?at=$event7', + action: null, + }, + }, + ]; + + const jumpBackwardPredecessorTestCases = [ + { + // Page2 only shows 4 messages ($event4-7) instead of 5 + // (`archiveMessageLimit` + 1) because it also has the tombstone event which + // is hidden + testName: 'can jump backward from one room to the predecessor room (different day)', + roomDayMessageStructureString: ` + [room1 ] [room2 ] + 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 + [day1 ] [day2 ] [day3 ] + [page1 ] + [page2 ] + `, + archiveMessageLimit: 4, + startUrl: '/roomid/room2/date/2022/01/03T05:00', + page1: { + url: '/roomid/room2/date/2022/01/03T05:00', + action: 'previous', + }, + page2: { + url: '/roomid/room1/date/2022/01/02?at=$tombstone', + action: null, + }, + }, + { + // Page2 only shows 4 messages ($event4-7) instead of 5 + // (`archiveMessageLimit` + 1) because it also has the tombstone event which + // is hidden + testName: 'can jump backward from one room to the predecessor room (same day)', + roomDayMessageStructureString: ` + [room1 ] [room2 ] + 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 + [day1 ] [day2 ] + [page1 ] + [page2 ] + `, + archiveMessageLimit: 4, + startUrl: '/roomid/room2/date/2022/01/02T10:00', + page1: { + url: '/roomid/room2/date/2022/01/02T10:00', + action: 'previous', + }, + page2: { + url: '/roomid/room1/date/2022/01/02?at=$tombstone', + action: null, + }, + }, + { + // Page2 only shows 3 messages ($event2-4) instead of 4 + // (`archiveMessageLimit` + 1) because it also has the tombstone event which + // is hidden + testName: 'jumping back before room was created will go down the predecessor chain', + roomDayMessageStructureString: ` + [room1 ] [room2 ] [room3 ] [room4 ] + 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 <-- 15 <-- 16 + [day1 ] [day2 ] [day3 ] [day4 ] [day5 ] [day6 ] [day7 ] [day8 ] + [page1 ] + [page2 ] + `, + archiveMessageLimit: 3, + startUrl: '/roomid/room4/date/2022/01/08', + page1: { + url: '/roomid/room4/date/2022/01/08', + action: 'navigate:/roomid/room4/date/2022/01/02', + }, + page2: { + url: '/roomid/room1/date/2022/01/02', + action: null, + }, + }, + // This doesn't work well because of the primordial create room events which + // we can't control the timestamp of or assert properly in this diagram. If we + // ever get timestamp massaging on the `/createRoom` endpoint (see + // https://github.com/matrix-org/matrix-public-archive/issues/169), we could + // make this work by increasing the `archiveMessageLimit` to something that + // would encompass all of the primordial events along with the last few + // messages. + // + // { + // testName: `will paginate to the oldest messages in the room (doesn't skip the last few) before jumping backward to the predecessor room`, + // roomDayMessageStructureString: ` + // [room1 ] [room2 ] + // 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 + // [day1 ] [day2 ] [day3 ] + // [page1 ] + // [page2] + // [page3 ] + // `, + // archiveMessageLimit: 4, + // startUrl: '/roomid/room2/date/2022/01/03', + // page1: { + // url: '/roomid/room2/date/2022/01/03', + // action: 'previous', + // }, + // page2: { + // url: '/roomid/room2/date/2022/01/03T02:00?at=$event9', + // action: 'previous', + // }, + // page3: { + // url: '/roomid/room1/date/2022/01/02', + // action: null, + // }, + // }, + ]; + + const jumpForwardSuccessorTestCases = [ + { + // We jump from event3 which is found as the closest event looking forward + // from the ts=0 in the successor room because the timestamp massaged events + // come before `m.room.create` and other primordial events here (related to + // https://github.com/matrix-org/matrix-public-archive/issues/169). From + // event3, we jump forward 10 messages (`archiveMessageLimit`) to event12, + // then back-track to the nearest hour which starts off at event11 and try + // to render the page with 11 messages because we fetch one more than + // `archiveMessageLimit` to determine overflow but there aren't enough + // messages. + testName: 'can jump forward from one room to the successor room (different day)', + roomDayMessageStructureString: ` + [room1] [room2 ] + 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 + [day1 ] [day2 ] + [page1] + |------------------jump-fwd-10-messages----------------->| + [page2 ] + `, + archiveMessageLimit: 10, + startUrl: '/roomid/room1/date/2022/01/01', + page1: { + url: '/roomid/room1/date/2022/01/01', + action: 'next', + }, + page2: { + url: '/roomid/room2/date/2022/01/02T09:00?at=$event3', + action: null, + }, + }, + { + // We jump from event8 which is found as the closest event looking forward + // from the ts=0 in the successor room because the timestamp massaged events + // come before `m.room.create` and other primordial events here (related to + // https://github.com/matrix-org/matrix-public-archive/issues/169). From + // event8, we jump forward 10 messages (`archiveMessageLimit`) to the end of + // the room, then go to the day of the last message which will show us all + // messages in room2 because we fetch one more than `archiveMessageLimit` to + // determine overflow which is more messages than room2 has. + testName: 'can jump forward from one room to the successor room (same day)', + roomDayMessageStructureString: ` + [room1 ] [room2 ] + 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 + [day1 ] [day2 ] + [page1 ] + |------------------jump-fwd-10-messages----------------------->| + [page2 ] + `, + archiveMessageLimit: 10, + startUrl: '/roomid/room1/date/2022/01/02T05:00', + page1: { + url: '/roomid/room1/date/2022/01/02T05:00', + action: 'next', + }, + page2: { + url: '/roomid/room2/date/2022/01/02?at=$event8', + action: null, + }, + }, + { + // We jump from event3 which is found as the closest event looking forward + // from the ts=0 in the successor room because the timestamp massaged events + // come before `m.room.create` and other primordial events here (related to + // https://github.com/matrix-org/matrix-public-archive/issues/169). From + // event3, we jump forward 10 messages (`archiveMessageLimit`) to event13, + // then back-track to the nearest hour which starts off at event11 and try + // to render the page with 11 messages because we fetch one more than + // `archiveMessageLimit` to determine overflow but there aren't enough + // messages. + testName: 'can jump forward from one room to the successor room (multiple day gap)', + roomDayMessageStructureString: ` + [room1] [room2 ] + 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 + [day1 ] [day5 ] + [page1] + |----------------jump-fwd-10-messages------------------->| + [page2 ] + `, + archiveMessageLimit: 10, + startUrl: '/roomid/room1/date/2022/01/01', + page1: { + url: '/roomid/room1/date/2022/01/01', + action: 'next', + }, + page2: { + url: '/roomid/room2/date/2022/01/05T09:00?at=$event3', + action: null, + }, + }, + { + // For the jump from page1 to page2, we jump forward 10 messages which gets + // us to the end of the room. + // + // For the jump from page2 to page3, since we see the end of room1, we jump + // to the successor room and find the closest event from ts=0 looking + // forward which is event5 because the timestamp massaged events come before + // `m.room.create` and other primordial events here (related to + // https://github.com/matrix-org/matrix-public-archive/issues/169). From + // event5, we jump forward 10 messages (`archiveMessageLimit`) to event14, + // then back-track to the *day* before the last message found which starts off + // at event6 and try to render the page with 11 messages because we fetch + // one more than `archiveMessageLimit` to determine overflow but there + // aren't enough messages. + // + // For the jump from page3 to page4, we jump forward 10 messages to event16, + // then back-track to the nearest hour which starts off at event15 and try + // to render the page with 11 messages because we fetch one more than + // `archiveMessageLimit`. + testName: 'can jump forward from one room to the successor room (across multiple days)', + roomDayMessageStructureString: ` + [room1 ] [room2 ] + 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 <-- 15 <-- 16 + [day1 ] [day2 ] [day3 ] + [page1] + |--jump-10->| + [page2 ] + |------------------jump-fwd-10-messages------------------->| + [page3] + |----------------------jump-fwd-10-messages----------------------->| + [page4 ] + `, + archiveMessageLimit: 10, + startUrl: '/roomid/room1/date/2022/01/01', + page1: { + url: '/roomid/room1/date/2022/01/01', + action: 'next', + }, + page2: { + url: '/roomid/room1/date/2022/01/02?at=$event3', + action: 'next', + }, + page3: { + url: '/roomid/room2/date/2022/01/02?at=$event5', + action: 'next', + }, + page4: { + url: '/roomid/room2/date/2022/01/03T09:00?at=$event7', + action: null, + }, + }, + { + // (same as the test above just with more day gaps) + testName: + 'can jump forward from one room to the successor room (across multiple days and day gaps)', + roomDayMessageStructureString: ` + [room1 ] [room2 ] + 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 <-- 15 <-- 16 + [day1 ] [day4 ] [day6 ] + [page1] + |--jump-10->| + [page2 ] + |------------------jump-fwd-10-messages------------------->| + [page3] + |----------------------jump-fwd-10-messages----------------------->| + [page4 ] + `, + archiveMessageLimit: 10, + startUrl: '/roomid/room1/date/2022/01/01', + page1: { + url: '/roomid/room1/date/2022/01/01', + action: 'next', + }, + page2: { + url: '/roomid/room1/date/2022/01/04?at=$event3', + action: 'next', + }, + page3: { + // You might expect `/date/2022/01/04?at=$event5` here but we just get the + // UTC day before the day of last message we jumped to (event14) + url: '/roomid/room2/date/2022/01/05?at=$event5', + action: 'next', + }, + page4: { + url: '/roomid/room2/date/2022/01/06T09:00?at=$event7', + action: null, + }, + }, + { + testName: 'jumping forward past the end of the room will go down the successor chain', + roomDayMessageStructureString: ` + [room1 ] [room2 ] [room3 ] [room4 ] + 1 <-- 2 <-- 3 <-- 4 <-- 5 <-- 6 <-- 7 <-- 8 <-- 9 <-- 10 <-- 11 <-- 12 <-- 13 <-- 14 <-- 15 <-- 16 + [day1 ] [day2 ] [day3 ] [day4 ] [day5 ] [day6 ] [day7 ] [day8 ] + [page1 ] + [page2 ] + `, + archiveMessageLimit: 3, + startUrl: '/roomid/room1/date/2022/01/02', + page1: { + url: '/roomid/room1/date/2022/01/02', + action: 'navigate:/roomid/room1/date/2022/01/08', + }, + page2: { + url: '/roomid/room4/date/2022/01/08', + action: null, + }, + }, + ]; + + jumpTestCases.forEach((testCase) => { + runJumpTestCase(testCase); + }); + + describe('with room upgrades', () => { + describe('jump backward into predecessor rooms', () => { + // eslint-disable-next-line max-nested-callbacks + jumpBackwardPredecessorTestCases.forEach((testCase) => { + runJumpTestCase(testCase); + }); + }); + + describe('jump forward from tombstone to replacement/successor rooms', () => { + // eslint-disable-next-line max-nested-callbacks + jumpForwardSuccessorTestCases.forEach((testCase) => { + runJumpTestCase(testCase); + }); + }); }); const jumpPrecisionTestCases = [ diff --git a/test/test-utils/client-utils.js b/test/test-utils/client-utils.js index 4abc50c..e101618 100644 --- a/test/test-utils/client-utils.js +++ b/test/test-utils/client-utils.js @@ -3,6 +3,7 @@ const assert = require('assert'); const urlJoin = require('url-join'); const { fetchEndpointAsJson, fetchEndpoint } = require('../../server/lib/fetch-endpoint'); +const getServerNameFromMatrixRoomIdOrAlias = require('../../server/lib/matrix-utils/get-server-name-from-matrix-room-id-or-alias'); const config = require('../../server/lib/config'); const matrixAccessToken = config.get('matrixAccessToken'); @@ -85,6 +86,62 @@ async function getTestClientForHs(testMatrixServerUrl) { }; } +async function sendEvent({ client, roomId, eventType, stateKey, content, timestamp }) { + assert(client); + assert(roomId); + assert(content); + + let qs = new URLSearchParams(); + if (timestamp) { + assert( + timestamp && client.applicationServiceUserIdOverride, + 'We can only do `?ts` massaging from an application service access token. ' + + 'Expected `client.applicationServiceUserIdOverride` to be defined so we can act on behalf of that user' + ); + + qs.append('ts', timestamp); + } + + if (client.applicationServiceUserIdOverride) { + qs.append('user_id', client.applicationServiceUserIdOverride); + } + + let url; + if (typeof stateKey === 'string') { + url = urlJoin( + client.homeserverUrl, + `/_matrix/client/v3/rooms/${encodeURIComponent( + roomId + )}/state/${eventType}/${stateKey}?${qs.toString()}` + ); + } else { + url = urlJoin( + client.homeserverUrl, + `/_matrix/client/v3/rooms/${encodeURIComponent( + roomId + )}/send/${eventType}/${getTxnId()}?${qs.toString()}` + ); + } + + const { data: sendResponse } = await fetchEndpointAsJson(url, { + method: 'PUT', + body: content, + accessToken: client.accessToken, + }); + + const eventId = sendResponse['event_id']; + assert(eventId); + return eventId; +} + +const WORLD_READABLE_STATE_EVENT = { + type: 'm.room.history_visibility', + state_key: '', + content: { + history_visibility: 'world_readable', + }, +}; + // Create a public room to test in async function createTestRoom(client, overrideCreateOptions = {}) { let qs = new URLSearchParams(); @@ -103,15 +160,7 @@ async function createTestRoom(client, overrideCreateOptions = {}) { preset: 'public_chat', name: roomName, room_alias_name: roomAlias, - initial_state: [ - { - type: 'm.room.history_visibility', - state_key: '', - content: { - history_visibility: 'world_readable', - }, - }, - ], + initial_state: [WORLD_READABLE_STATE_EVENT], visibility: 'public', ...overrideCreateOptions, }, @@ -124,6 +173,64 @@ async function createTestRoom(client, overrideCreateOptions = {}) { return roomId; } +async function upgradeTestRoom({ + client, + oldRoomId, + useMsc3946DynamicPredecessor = false, + overrideCreateOptions = {}, + timestamp, +}) { + assert(client); + assert(oldRoomId); + + const createOptions = { + ...overrideCreateOptions, + }; + // Setup the pointer from the new room to the old room + if (useMsc3946DynamicPredecessor) { + createOptions.initial_state = [ + WORLD_READABLE_STATE_EVENT, + { + type: 'org.matrix.msc3946.room_predecessor', + state_key: '', + content: { + predecessor_room_id: oldRoomId, + via_servers: [getServerNameFromMatrixRoomIdOrAlias(oldRoomId)], + }, + }, + ]; + } else { + createOptions.creation_content = { + predecessor: { + room_id: oldRoomId, + // The event ID of the last known event in the old room (supposedly required). + //event_id: TODO, + }, + }; + } + + // TODO: Pass `timestamp` massaging option to `createTestRoom()` when it supports it, + // see https://github.com/matrix-org/matrix-public-archive/issues/169 + const newRoomid = await createTestRoom(client, createOptions); + + // Now send the tombstone event pointing from the old room to the new room + const tombstoneEventId = await sendEvent({ + client, + roomId: oldRoomId, + eventType: 'm.room.tombstone', + stateKey: '', + content: { + replacement_room: newRoomid, + }, + timestamp, + }); + + return { + newRoomid, + tombstoneEventId, + }; +} + async function getCanonicalAlias({ client, roomId }) { const { data: stateCanonicalAliasRes } = await fetchEndpointAsJson( urlJoin( @@ -167,54 +274,6 @@ async function joinRoom({ client, roomId, viaServers }) { return joinedRoomId; } -async function sendEvent({ client, roomId, eventType, stateKey, content, timestamp }) { - assert(client); - assert(roomId); - assert(content); - - let qs = new URLSearchParams(); - if (timestamp) { - assert( - timestamp && client.applicationServiceUserIdOverride, - 'We can only do `?ts` massaging from an application service access token. ' + - 'Expected `client.applicationServiceUserIdOverride` to be defined so we can act on behalf of that user' - ); - - qs.append('ts', timestamp); - } - - if (client.applicationServiceUserIdOverride) { - qs.append('user_id', client.applicationServiceUserIdOverride); - } - - let url; - if (stateKey) { - url = urlJoin( - client.homeserverUrl, - `/_matrix/client/v3/rooms/${encodeURIComponent( - roomId - )}/state/${eventType}/${stateKey}?${qs.toString()}` - ); - } else { - url = urlJoin( - client.homeserverUrl, - `/_matrix/client/v3/rooms/${encodeURIComponent( - roomId - )}/send/${eventType}/${getTxnId()}?${qs.toString()}` - ); - } - - const { data: sendResponse } = await fetchEndpointAsJson(url, { - method: 'PUT', - body: content, - accessToken: client.accessToken, - }); - - const eventId = sendResponse['event_id']; - assert(eventId); - return eventId; -} - async function sendMessage({ client, roomId, content, timestamp }) { return sendEvent({ client, roomId, eventType: 'm.room.message', content, timestamp }); } @@ -259,6 +318,28 @@ async function createMessagesInRoom({ return { eventIds, eventMap }; } +async function getMessagesInRoom({ client, roomId, limit }) { + assert(client); + assert(roomId); + assert(limit); + + let qs = new URLSearchParams(); + qs.append('limit', limit); + + const { data } = await fetchEndpointAsJson( + urlJoin( + client.homeserverUrl, + `/_matrix/client/v3/rooms/${encodeURIComponent(roomId)}/messages?${qs.toString()}` + ), + { + method: 'GET', + accessToken: client.accessToken, + } + ); + + return data.chunk; +} + async function updateProfile({ client, displayName, avatarUrl }) { let qs = new URLSearchParams(); if (client.applicationServiceUserIdOverride) { @@ -345,11 +426,13 @@ module.exports = { getTestClientForAs, getTestClientForHs, createTestRoom, + upgradeTestRoom, getCanonicalAlias, joinRoom, sendEvent, sendMessage, createMessagesInRoom, + getMessagesInRoom, updateProfile, uploadContent, }; diff --git a/test/test-utils/parse-archive-url-for-room.js b/test/test-utils/parse-archive-url-for-room.js index 090c637..eada9b4 100644 --- a/test/test-utils/parse-archive-url-for-room.js +++ b/test/test-utils/parse-archive-url-for-room.js @@ -1,7 +1,7 @@ 'use strict'; const { - VALID_SIGIL_TO_ENTITY_DESCRIPTOR_MAP, + VALID_ENTITY_DESCRIPTOR_TO_SIGIL_MAP, } = require('matrix-public-archive-shared/lib/reference-values'); // http://archive.matrix.org/r/some-room:matrix.org/date/2022/11/16T23:59:59?at=$xxx @@ -13,9 +13,7 @@ function parseArchiveUrlForRoom(archiveUrlForRoom) { /\/(r|roomid)\/(.*?)\/date\/(.*)/ ); - const [sigil] = Object.entries(VALID_SIGIL_TO_ENTITY_DESCRIPTOR_MAP).find( - ([_sigil, entityDescriptor]) => roomIdOrAliasDescriptor === entityDescriptor - ); + const sigil = VALID_ENTITY_DESCRIPTOR_TO_SIGIL_MAP[roomIdOrAliasDescriptor]; const roomIdOrAlias = `${sigil}${roomIdOrAliasUrlPart}`; const continueAtEvent = urlObj.searchParams.get('at'); diff --git a/test/test-utils/parse-room-day-message-structure.js b/test/test-utils/parse-room-day-message-structure.js index b9c4ab8..2541668 100644 --- a/test/test-utils/parse-room-day-message-structure.js +++ b/test/test-utils/parse-room-day-message-structure.js @@ -191,19 +191,9 @@ function parseRoomDayMessageStructure(roomDayMessageStructureString) { events, }; }); - // Ensure that each page has the same number of events on it - const numEventsOnEachPage = pages.map((page) => page.events.length); - // The page limit is X but each page will display X + 1 messages because we fetch one - // extra to determine overflow. - const archiveMessageLimit = numEventsOnEachPage[0] - 1; - assert( - numEventsOnEachPage.every((numEvents) => numEvents === archiveMessageLimit + 1), - `Expected all pages to have the same number of events (archiveMessageLimit + 1) where archiveMessageLimit=${archiveMessageLimit} but found ${numEventsOnEachPage}` - ); return { rooms, - archiveMessageLimit, pages, }; }