From dd2cd9126da44f2701917e4ba7e5e10bb34856f8 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Fri, 30 Jun 2023 03:08:32 -0500 Subject: [PATCH] Only show `world_readable` rooms in the room directory (#276) Happens to address part of https://github.com/matrix-org/matrix-public-archive/issues/271 but made primarily as a follow-up to https://github.com/matrix-org/matrix-public-archive/pull/239 --- Only 42% rooms on the `matrix.org` room directory are `world_readable` which means we will get pages of rooms that are half-empty most of the time if we just naively fetch 9 rooms at a time. Ideally, we would be able to just add a filter directly to `/publicRooms` in order to only grab the `world_readable` rooms and still get full pages but the filter option doesn't allow us to slice by `world_readable` history visibility. Instead, we have to paginate until we get a full grid of 9 rooms, then make a final `/publicRooms` request to backtrack to the exact continuation point so next page won't skip any rooms in between. --- We had empty spaces in the grid before because some rooms in the room directory are private which we filtered out before. But that was a much more rare experience since only 2% of rooms were private . --- .../matrix-utils/fetch-accessible-rooms.js | 191 ++++++++++++++ server/lib/matrix-utils/fetch-public-rooms.js | 57 ----- server/routes/room-directory-routes.js | 18 +- shared/lib/url-creator.js | 19 +- shared/viewmodels/RoomDirectoryViewModel.js | 3 + shared/views/RoomDirectoryView.js | 15 +- test/dockerfiles/Synapse.Dockerfile | 7 +- test/e2e-tests.js | 232 +++++++++++++++++- test/test-utils/client-utils.js | 51 +++- 9 files changed, 515 insertions(+), 78 deletions(-) create mode 100644 server/lib/matrix-utils/fetch-accessible-rooms.js delete mode 100644 server/lib/matrix-utils/fetch-public-rooms.js diff --git a/server/lib/matrix-utils/fetch-accessible-rooms.js b/server/lib/matrix-utils/fetch-accessible-rooms.js new file mode 100644 index 0000000..cc24bd7 --- /dev/null +++ b/server/lib/matrix-utils/fetch-accessible-rooms.js @@ -0,0 +1,191 @@ +'use strict'; + +const assert = require('assert'); + +const urlJoin = require('url-join'); +const { DIRECTION } = require('matrix-public-archive-shared/lib/reference-values'); +const { fetchEndpointAsJson } = require('../fetch-endpoint'); +const { traceFunction } = require('../../tracing/trace-utilities'); + +const config = require('../config'); +const matrixServerUrl = config.get('matrixServerUrl'); +assert(matrixServerUrl); + +// The number of requests we should make to try to fill the limit before bailing out +const NUM_MAX_REQUESTS = 10; + +async function requestPublicRooms( + accessToken, + { server, searchTerm, paginationToken, limit, abortSignal } = {} +) { + let qs = new URLSearchParams(); + if (server) { + qs.append('server', server); + } + + const publicRoomsEndpoint = urlJoin( + matrixServerUrl, + `_matrix/client/v3/publicRooms?${qs.toString()}` + ); + + const { data: publicRoomsRes } = await fetchEndpointAsJson(publicRoomsEndpoint, { + method: 'POST', + body: { + include_all_networks: true, + filter: { + generic_search_term: searchTerm, + }, + since: paginationToken, + limit, + }, + accessToken, + abortSignal, + }); + + return publicRoomsRes; +} + +// eslint-disable-next-line complexity, max-statements +async function fetchAccessibleRooms( + accessToken, + { + server, + searchTerm, + // Direction is baked into the pagination token but we're unable to decipher it from + // the opaque token, we also have to pass it in explicitly. + paginationToken, + direction = DIRECTION.forward, + limit, + abortSignal, + } = {} +) { + assert(accessToken); + assert([DIRECTION.forward, DIRECTION.backward].includes(direction), 'direction must be [f|b]'); + + // Based off of the matrix.org room directory, only 42% of rooms are world_readable, + // which means our best bet to fill up the results to the limit is to request at least + // 2.4 times as many. I've doubled and rounded it up to 5 times as many so we can have + // less round-trips. + const bulkPaginationLimit = Math.ceil(5 * limit); + + let accessibleRooms = []; + + let firstResponse; + let lastResponse; + + let loopToken = paginationToken; + let lastLoopToken; + let continuationIndex; + let currentRequestCount = 0; + while ( + // Stop if we have reached the limit of rooms we want to fetch + accessibleRooms.length < limit && + // And bail if we're already gone through a bunch of pages to try to fill the limit + currentRequestCount < NUM_MAX_REQUESTS && + // And bail if we've reached the end of the pagination + // Always do the first request + (currentRequestCount === 0 || + // If we have a next token, we can do another request + (currentRequestCount > 0 && loopToken)) + ) { + const publicRoomsRes = await requestPublicRooms(accessToken, { + server, + searchTerm, + paginationToken: loopToken, + limit: bulkPaginationLimit, + abortSignal, + }); + lastLoopToken = loopToken; + lastResponse = publicRoomsRes; + + if (currentRequestCount === 0) { + firstResponse = publicRoomsRes; + } + + // Get the token ready for the next loop + loopToken = + direction === DIRECTION.forward ? publicRoomsRes.next_batch : publicRoomsRes.prev_batch; + + const fetchedRooms = publicRoomsRes.chunk; + const fetchedRoomsInDirection = + direction === DIRECTION.forward ? fetchedRooms : fetchedRooms.reverse(); + + // We only want to see world_readable rooms in the archive + let index = 0; + for (let room of fetchedRoomsInDirection) { + if (room.world_readable) { + if (direction === DIRECTION.forward) { + accessibleRooms.push(room); + } else if (direction === DIRECTION.backward) { + accessibleRooms.unshift(room); + } else { + throw new Error(`Invalid direction: ${direction}`); + } + } + + if (accessibleRooms.length === limit && !continuationIndex) { + continuationIndex = index; + } + + // Stop after we've reached the limit + if (accessibleRooms.length >= limit) { + break; + } + + index += 1; + } + + currentRequestCount += 1; + } + + // Back-track to get the perfect continuation point and show exactly the limit of + // rooms in the grid. + // + // Alternatively, we could just not worry about and show more than the limit of rooms + // + // XXX: Since the room directory order is not stable, this is slightly flawed as the + // results could have shifted slightly from when we made the last request to now but + // we assume it's good enough. + let nextPaginationToken; + let prevPaginationToken; + if (continuationIndex) { + const publicRoomsRes = await requestPublicRooms(accessToken, { + server, + searchTerm, + // Start from the last request + paginationToken: lastLoopToken, + // Then only go out as far out as the continuation index (the point when we filled + // the limit) + limit: continuationIndex + 1, + abortSignal, + }); + + if (direction === DIRECTION.forward) { + prevPaginationToken = firstResponse.prev_batch; + nextPaginationToken = publicRoomsRes.next_batch; + } else if (direction === DIRECTION.backward) { + prevPaginationToken = publicRoomsRes.prev_batch; + nextPaginationToken = firstResponse.next_batch; + } else { + throw new Error(`Invalid direction: ${direction}`); + } + } else { + if (direction === DIRECTION.forward) { + prevPaginationToken = firstResponse.prev_batch; + nextPaginationToken = lastResponse.next_batch; + } else if (direction === DIRECTION.backward) { + prevPaginationToken = lastResponse.prev_batch; + nextPaginationToken = firstResponse.next_batch; + } else { + throw new Error(`Invalid direction: ${direction}`); + } + } + + return { + rooms: accessibleRooms, + prevPaginationToken, + nextPaginationToken, + }; +} + +module.exports = traceFunction(fetchAccessibleRooms); diff --git a/server/lib/matrix-utils/fetch-public-rooms.js b/server/lib/matrix-utils/fetch-public-rooms.js deleted file mode 100644 index bc9e7c5..0000000 --- a/server/lib/matrix-utils/fetch-public-rooms.js +++ /dev/null @@ -1,57 +0,0 @@ -'use strict'; - -const assert = require('assert'); - -const urlJoin = require('url-join'); -const { fetchEndpointAsJson } = require('../fetch-endpoint'); -const { traceFunction } = require('../../tracing/trace-utilities'); - -const config = require('../config'); -const matrixServerUrl = config.get('matrixServerUrl'); -assert(matrixServerUrl); - -async function fetchPublicRooms( - accessToken, - { server, searchTerm, paginationToken, limit, abortSignal } = {} -) { - assert(accessToken); - - let qs = new URLSearchParams(); - if (server) { - qs.append('server', server); - } - - const publicRoomsEndpoint = urlJoin( - matrixServerUrl, - `_matrix/client/v3/publicRooms?${qs.toString()}` - ); - - const { data: publicRoomsRes } = await fetchEndpointAsJson(publicRoomsEndpoint, { - method: 'POST', - body: { - include_all_networks: true, - filter: { - generic_search_term: searchTerm, - }, - since: paginationToken, - limit, - }, - accessToken, - abortSignal, - }); - - // We only want to see public rooms in the archive - const accessibleRooms = publicRoomsRes.chunk.filter((room) => { - // `room.world_readable` is also accessible here but we only use history - // `world_readable` to determine search indexing. - return room.join_rule === 'public'; - }); - - return { - rooms: accessibleRooms, - nextPaginationToken: publicRoomsRes.next_batch, - prevPaginationToken: publicRoomsRes.prev_batch, - }; -} - -module.exports = traceFunction(fetchPublicRooms); diff --git a/server/routes/room-directory-routes.js b/server/routes/room-directory-routes.js index ad87681..fde1467 100644 --- a/server/routes/room-directory-routes.js +++ b/server/routes/room-directory-routes.js @@ -6,10 +6,11 @@ const urlJoin = require('url-join'); const express = require('express'); const asyncHandler = require('../lib/express-async-handler'); +const { DIRECTION } = require('matrix-public-archive-shared/lib/reference-values'); const RouteTimeoutAbortError = require('../lib/errors/route-timeout-abort-error'); const UserClosedConnectionAbortError = require('../lib/errors/user-closed-connection-abort-error'); const identifyRoute = require('../middleware/identify-route-middleware'); -const fetchPublicRooms = require('../lib/matrix-utils/fetch-public-rooms'); +const fetchAccessibleRooms = require('../lib/matrix-utils/fetch-accessible-rooms'); const renderHydrogenVmRenderScriptToPageHtml = require('../hydrogen-render/render-hydrogen-vm-render-script-to-page-html'); const setHeadersToPreloadAssets = require('../lib/set-headers-to-preload-assets'); @@ -33,9 +34,19 @@ router.get( '/', identifyRoute('app-room-directory-index'), asyncHandler(async function (req, res) { - const paginationToken = req.query.page; const searchTerm = req.query.search; const homeserver = req.query.homeserver; + const paginationToken = req.query.page; + const direction = req.query.dir; + + // You must provide both `paginationToken` and `direction` if either is defined + if (paginationToken || direction) { + assert( + [DIRECTION.forward, DIRECTION.backward].includes(direction), + '?dir query parameter must be [f|b]' + ); + assert(paginationToken, '?page query parameter must be defined if ?dir is defined'); + } // It would be good to grab more rooms than we display in case we need // to filter any out but then the pagination tokens with the homeserver @@ -48,12 +59,13 @@ router.get( let prevPaginationToken; let roomFetchError; try { - ({ rooms, nextPaginationToken, prevPaginationToken } = await fetchPublicRooms( + ({ rooms, nextPaginationToken, prevPaginationToken } = await fetchAccessibleRooms( matrixAccessToken, { server: homeserver, searchTerm, paginationToken, + direction, limit, abortSignal: req.abortSignal, } diff --git a/shared/lib/url-creator.js b/shared/lib/url-creator.js index 4285d21..f8b9b3d 100644 --- a/shared/lib/url-creator.js +++ b/shared/lib/url-creator.js @@ -3,7 +3,10 @@ const urlJoin = require('url-join'); const assert = require('matrix-public-archive-shared/lib/assert'); -const { TIME_PRECISION_VALUES } = require('matrix-public-archive-shared/lib/reference-values'); +const { + DIRECTION, + TIME_PRECISION_VALUES, +} = require('matrix-public-archive-shared/lib/reference-values'); function qsToUrlPiece(qs) { if (qs.toString()) { @@ -25,7 +28,16 @@ class URLCreator { return `https://matrix.to/#/${roomIdOrAlias}`; } - roomDirectoryUrl({ searchTerm, homeserver, paginationToken } = {}) { + roomDirectoryUrl({ searchTerm, homeserver, paginationToken, direction } = {}) { + // You must provide both `paginationToken` and `direction` if either is defined + if (paginationToken || direction) { + assert( + [DIRECTION.forward, DIRECTION.backward].includes(direction), + 'direction must be [f|b]' + ); + assert(paginationToken); + } + let qs = new URLSearchParams(); if (searchTerm) { qs.append('search', searchTerm); @@ -36,6 +48,9 @@ class URLCreator { if (paginationToken) { qs.append('page', paginationToken); } + if (direction) { + qs.append('dir', direction); + } return `${this._basePath}${qsToUrlPiece(qs)}`; } diff --git a/shared/viewmodels/RoomDirectoryViewModel.js b/shared/viewmodels/RoomDirectoryViewModel.js index 90c5c13..3268b6b 100644 --- a/shared/viewmodels/RoomDirectoryViewModel.js +++ b/shared/viewmodels/RoomDirectoryViewModel.js @@ -9,6 +9,7 @@ const ModalViewModel = require('matrix-public-archive-shared/viewmodels/ModalVie const HomeserverSelectionModalContentViewModel = require('matrix-public-archive-shared/viewmodels/HomeserverSelectionModalContentViewModel'); const RoomCardViewModel = require('matrix-public-archive-shared/viewmodels/RoomCardViewModel'); const checkTextForNsfw = require('matrix-public-archive-shared/lib/check-text-for-nsfw'); +const { DIRECTION } = require('../lib/reference-values'); const DEFAULT_SERVER_LIST = ['matrix.org', 'gitter.im']; @@ -304,6 +305,7 @@ class RoomDirectoryViewModel extends ViewModel { homeserver: this.homeserverSelection, searchTerm: this.searchTerm, paginationToken: this._nextPaginationToken, + direction: DIRECTION.forward, }); } @@ -316,6 +318,7 @@ class RoomDirectoryViewModel extends ViewModel { homeserver: this.homeserverSelection, searchTerm: this.searchTerm, paginationToken: this._prevPaginationToken, + direction: DIRECTION.backward, }); } diff --git a/shared/views/RoomDirectoryView.js b/shared/views/RoomDirectoryView.js index 3466322..61157d1 100644 --- a/shared/views/RoomDirectoryView.js +++ b/shared/views/RoomDirectoryView.js @@ -322,10 +322,21 @@ class RoomDirectoryView extends TemplateView { t.view(roomList), t.div({ className: 'RoomDirectoryView_paginationButtonCombo' }, [ t.a( - { className: 'RoomDirectoryView_paginationButton', href: vm.prevPageUrl }, + { + className: 'RoomDirectoryView_paginationButton', + href: vm.prevPageUrl, + 'data-testid': 'room-directory-prev-link', + }, 'Previous' ), - t.a({ className: 'RoomDirectoryView_paginationButton', href: vm.nextPageUrl }, 'Next'), + t.a( + { + className: 'RoomDirectoryView_paginationButton', + href: vm.nextPageUrl, + 'data-testid': 'room-directory-next-link', + }, + 'Next' + ), ]), ]), t.if( diff --git a/test/dockerfiles/Synapse.Dockerfile b/test/dockerfiles/Synapse.Dockerfile index 90407f3..dbfbf1c 100644 --- a/test/dockerfiles/Synapse.Dockerfile +++ b/test/dockerfiles/Synapse.Dockerfile @@ -3,12 +3,7 @@ # # Currently this is based on Complement Synapse images which are based on the # published 'synapse:latest' image -- ie, the most recent Synapse release. - -# FIXME: We're pinning the version to `v1.79.0` until -# https://github.com/matrix-org/synapse/issues/15526 is fixed. Feel free to update back -# to `latest` once that issue is resolved. More context: -# https://github.com/matrix-org/matrix-public-archive/pull/208#discussion_r1183294630 -ARG SYNAPSE_VERSION=v1.79.0 +ARG SYNAPSE_VERSION=latest FROM matrixdotorg/synapse:${SYNAPSE_VERSION} diff --git a/test/e2e-tests.js b/test/e2e-tests.js index b95425c..2e26ece 100644 --- a/test/e2e-tests.js +++ b/test/e2e-tests.js @@ -38,6 +38,7 @@ const { sendMessage, createMessagesInRoom, getMessagesInRoom, + waitForResultsInHomeserverRoomDirectory, updateProfile, uploadContent, } = require('./test-utils/client-utils'); @@ -2507,15 +2508,33 @@ describe('matrix-public-archive', () => { // test runs against the same homeserver const timeToken = Date.now(); const roomPlanetPrefix = `planet-${timeToken}`; + const roomSaturnName = `${roomPlanetPrefix}-saturn`; const roomSaturnId = await createTestRoom(client, { - name: `${roomPlanetPrefix}-saturn`, + name: roomSaturnName, }); + const roomMarsName = `${roomPlanetPrefix}-mars`; const roomMarsId = await createTestRoom(client, { - name: `${roomPlanetPrefix}-mars`, + name: roomMarsName, }); // Browse the room directory without search to see many rooms + // + // (we set this here in case we timeout while waiting for the test rooms to + // appear in the room directory) archiveUrl = matrixPublicArchiveURLCreator.roomDirectoryUrl(); + + // Try to avoid flakey tests where the homeserver hasn't added the rooms to the + // room directory yet. This isn't completely robust as it doesn't check that the + // random room at the start is in the directory but should be good enough. + await waitForResultsInHomeserverRoomDirectory({ + client, + searchTerm: roomSaturnName, + }); + await waitForResultsInHomeserverRoomDirectory({ + client, + searchTerm: roomMarsName, + }); + const { data: roomDirectoryPageHtml } = await fetchEndpointAsText(archiveUrl); const dom = parseHTML(roomDirectoryPageHtml); @@ -2556,17 +2575,33 @@ describe('matrix-public-archive', () => { // test runs against the same homeserver const timeToken = Date.now(); const roomPlanetPrefix = `remote-planet-${timeToken}`; + const roomXName = `${roomPlanetPrefix}-x`; const roomXId = await createTestRoom(hs2Client, { - name: `${roomPlanetPrefix}-x`, + name: roomXName, }); + const roomYname = `${roomPlanetPrefix}-y`; const roomYId = await createTestRoom(hs2Client, { - name: `${roomPlanetPrefix}-y`, + name: roomYname, }); + // (we set this here in case we timeout while waiting for the test rooms to + // appear in the room directory) archiveUrl = matrixPublicArchiveURLCreator.roomDirectoryUrl({ homeserver: HOMESERVER_URL_TO_PRETTY_NAME_MAP[testMatrixServerUrl2], searchTerm: roomPlanetPrefix, }); + + // Try to avoid flakey tests where the homeserver hasn't added the rooms to the + // room directory yet. + await waitForResultsInHomeserverRoomDirectory({ + client: hs2Client, + searchTerm: roomXName, + }); + await waitForResultsInHomeserverRoomDirectory({ + client: hs2Client, + searchTerm: roomYname, + }); + const { data: roomDirectoryWithSearchPageHtml } = await fetchEndpointAsText(archiveUrl); const domWithSearch = parseHTML(roomDirectoryWithSearchPageHtml); @@ -2601,21 +2636,39 @@ describe('matrix-public-archive', () => { // test runs against the same homeserver const timeToken = Date.now(); const roomPlanetPrefix = `planet-${timeToken}`; + const roomUranusName = `${roomPlanetPrefix}-uranus-nsfw`; const roomUranusId = await createTestRoom(client, { // NSFW in title - name: `${roomPlanetPrefix}-uranus-nsfw`, + name: roomUranusName, }); + const roomMarsName = `${roomPlanetPrefix}-mars`; const roomMarsId = await createTestRoom(client, { - name: `${roomPlanetPrefix}-mars`, + name: roomMarsName, // NSFW in room topic/description topic: 'Get your ass to mars (NSFW)', }); // Browse the room directory searching the room directory for those NSFW rooms // (narrowing down results). + // + // (we set this here in case we timeout while waiting for the test rooms to + // appear in the room directory) archiveUrl = matrixPublicArchiveURLCreator.roomDirectoryUrl({ searchTerm: roomPlanetPrefix, }); + + // Try to avoid flakey tests where the homeserver hasn't added the rooms to the + // room directory yet. This isn't completely robust as it doesn't check that the + // random room at the start is in the directory but should be good enough. + await waitForResultsInHomeserverRoomDirectory({ + client, + searchTerm: roomUranusName, + }); + await waitForResultsInHomeserverRoomDirectory({ + client, + searchTerm: roomMarsName, + }); + const { data: roomDirectoryWithSearchPageHtml } = await fetchEndpointAsText(archiveUrl); const domWithSearch = parseHTML(roomDirectoryWithSearchPageHtml); @@ -2644,6 +2697,173 @@ describe('matrix-public-archive', () => { ); }); }); + + it('pagination is seamless', async () => { + const client = await getTestClientForHs(testMatrixServerUrl1); + // We use a `timeToken` so that we can namespace these rooms away from other + // test runs against the same homeserver + const timeToken = Date.now(); + const roomPlanetPrefix = `planet-${timeToken}`; + + // Fill up the room room directory with multiple pages of rooms + const visibleRoomConfigurations = []; + const roomsConfigurationsToCreate = []; + for (let i = 0; i < 40; i++) { + const roomCreateOptions = { + name: `${roomPlanetPrefix}-room-${i}`, + }; + + // Sprinkle in some rooms every so often that should not appear in the room directory + if (i % 3 === 0) { + roomCreateOptions.name = `${roomPlanetPrefix}-room-not-world-readable-${i}`; + roomCreateOptions.initial_state = [ + { + type: 'm.room.history_visibility', + state_key: '', + content: { + history_visibility: 'joined', + }, + }, + { + type: 'm.room.topic', + state_key: '', + content: { + // Just a specific token we can search for in the DOM to make sure + // this room does not appear in the room directory. + topic: 'should-not-be-visible-in-archive-room-directory', + }, + }, + ]; + } else { + visibleRoomConfigurations.push(roomCreateOptions); + } + + roomsConfigurationsToCreate.push(roomCreateOptions); + } + + // Doing all of these create room requests in parallel is about 2x faster than + // doing them serially and the room directory doesn't return the rooms in any + // particular order so it doesn't make the test any more clear doing them + // serially anyway. + const createdRoomsIds = await Promise.all( + roomsConfigurationsToCreate.map((roomCreateOptions) => + createTestRoom(client, roomCreateOptions) + ) + ); + + function roomIdToRoomName(expectedRoomId) { + const roomIndex = createdRoomsIds.findIndex((roomId) => { + return roomId === expectedRoomId; + }); + assert( + roomIndex > 0, + `Expected to find expectedRoomId=${expectedRoomId} in the list of created rooms createdRoomsIds=${createdRoomsIds}` + ); + + const roomConfig = roomsConfigurationsToCreate[roomIndex]; + assert( + roomConfig, + `Expected to find room config for roomIndex=${roomIndex} in the list of roomsConfigurationsToCreate (length ${roomsConfigurationsToCreate.length})}` + ); + + return roomConfig.name; + } + + async function checkRoomsOnPage(archiveUrl) { + const { data: roomDirectoryWithSearchPageHtml } = await fetchEndpointAsText(archiveUrl); + const dom = parseHTML(roomDirectoryWithSearchPageHtml); + + const roomsCardsOnPageWithSearch = [ + ...dom.document.querySelectorAll(`[data-testid="room-card"]`), + ]; + + const roomsIdsOnPage = roomsCardsOnPageWithSearch.map((roomCardEl) => { + return roomCardEl.getAttribute('data-room-id'); + }); + + // Sanity check that we don't see any non-world_readable rooms. + roomsCardsOnPageWithSearch.forEach((roomCardEl) => { + assert.match( + roomCardEl.innerHTML, + /^((?!should-not-be-visible-in-archive-room-directory).)*$/, + `Expected not to see any non-world_readable rooms on the page but saw ${roomCardEl.getAttribute( + 'data-room-id' + )} which has "should-not-be-visible-in-archive-room-directory" in the room topic` + ); + }); + + // Find the pagination buttons and grab the links to the previous and next pages + const previousLinkElement = dom.document.querySelector( + `[data-testid="room-directory-prev-link"]` + ); + const nextLinkElement = dom.document.querySelector( + `[data-testid="room-directory-next-link"]` + ); + + const previousPaginationLink = previousLinkElement.getAttribute('href'); + const nextPaginationLink = nextLinkElement.getAttribute('href'); + + return { + archiveUrl, + roomsIdsOnPage, + previousPaginationLink, + nextPaginationLink, + }; + } + + // Browse the room directory with the search prefix so we only see rooms + // relevant to this test. + // + // (we set this here in case we timeout while waiting for the test rooms to + // appear in the room directory) + archiveUrl = matrixPublicArchiveURLCreator.roomDirectoryUrl({ + searchTerm: roomPlanetPrefix, + }); + + // Try to avoid flakey tests where the homeserver hasn't added the rooms + // to the room directory yet. This isn't completely robust as it doesn't check + // that all rooms are visible but it's better than nothing. + await waitForResultsInHomeserverRoomDirectory({ + client, + searchTerm: visibleRoomConfigurations[0].name, + }); + await waitForResultsInHomeserverRoomDirectory({ + client, + searchTerm: visibleRoomConfigurations[visibleRoomConfigurations.length - 1].name, + }); + + // Visit a sequence of pages using the pagination links: 1 -> 2 -> 3 -> 2 -> 1 + const firstPage = await checkRoomsOnPage(archiveUrl); + const secondPage = await checkRoomsOnPage(firstPage.nextPaginationLink); + const thirdPage = await checkRoomsOnPage(secondPage.nextPaginationLink); + const backtrackSecondPage = await checkRoomsOnPage(thirdPage.previousPaginationLink); + const backtrackFirstPage = await checkRoomsOnPage( + backtrackSecondPage.previousPaginationLink + ); + + // Ensure that we saw all of the visible rooms paginating through the directory + assert.deepStrictEqual( + [...firstPage.roomsIdsOnPage, ...secondPage.roomsIdsOnPage, ...thirdPage.roomsIdsOnPage] + .map(roomIdToRoomName) + .sort(), + visibleRoomConfigurations.map((roomConfig) => roomConfig.name).sort(), + 'Make sure we saw all visible rooms paginating through the directory' + ); + + // Ensure that we see the same rooms in the same order going backward that we saw going forward + archiveUrl = backtrackSecondPage.archiveUrl; + assert.deepStrictEqual( + backtrackSecondPage.roomsIdsOnPage.map(roomIdToRoomName), + secondPage.roomsIdsOnPage.map(roomIdToRoomName), + 'From the third page, going backward to second page should show the same rooms that we saw on the second page when going forward' + ); + archiveUrl = backtrackFirstPage.archiveUrl; + assert.deepStrictEqual( + backtrackFirstPage.roomsIdsOnPage.map(roomIdToRoomName), + firstPage.roomsIdsOnPage.map(roomIdToRoomName), + 'From the second page, going backward to first page should show the same rooms that we saw on first page when going forward' + ); + }); }); describe('access controls', () => { diff --git a/test/test-utils/client-utils.js b/test/test-utils/client-utils.js index e101618..f8d7ef3 100644 --- a/test/test-utils/client-utils.js +++ b/test/test-utils/client-utils.js @@ -4,6 +4,8 @@ const assert = require('assert'); const urlJoin = require('url-join'); const { fetchEndpointAsJson, fetchEndpoint } = require('../../server/lib/fetch-endpoint'); const getServerNameFromMatrixRoomIdOrAlias = require('../../server/lib/matrix-utils/get-server-name-from-matrix-room-id-or-alias'); +const { MS_LOOKUP } = require('matrix-public-archive-shared/lib/reference-values'); +const { ONE_SECOND_IN_MS } = MS_LOOKUP; const config = require('../../server/lib/config'); const matrixAccessToken = config.get('matrixAccessToken'); @@ -14,7 +16,7 @@ assert(testMatrixServerUrl1); let txnCount = 0; function getTxnId() { txnCount++; - return `${new Date().getTime()}--${txnCount}`; + return `txn${txnCount}-${new Date().getTime()}`; } // Basic slugify function, plenty of edge cases and should not be used for @@ -150,7 +152,7 @@ async function createTestRoom(client, overrideCreateOptions = {}) { } const roomName = overrideCreateOptions.name || 'the hangout spot'; - const roomAlias = slugify(roomName + getTxnId()); + const roomAlias = slugify(roomName + '-' + getTxnId()); const { data: createRoomResponse } = await fetchEndpointAsJson( urlJoin(client.homeserverUrl, `/_matrix/client/v3/createRoom?${qs.toString()}`), @@ -421,6 +423,50 @@ async function uploadContent({ client, roomId, data, fileName, contentType }) { return mxcUri; } +// This can be removed after https://github.com/matrix-org/synapse/issues/15526 is solved +async function waitForResultsInHomeserverRoomDirectory({ + client, + searchTerm, + timeoutMs = 10 * ONE_SECOND_IN_MS, +}) { + assert(client); + assert(searchTerm !== undefined); + + const roomDirectoryEndpoint = urlJoin(client.homeserverUrl, `_matrix/client/v3/publicRooms`); + + // eslint-disable-next-line no-async-promise-executor + await new Promise(async (resolve, reject) => { + try { + setTimeout(() => { + reject(new Error('Timed out waiting for rooms to appear in the room directory')); + }, timeoutMs); + + let foundResults = false; + while (!foundResults) { + const { data: publicRoomsRes } = await fetchEndpointAsJson(roomDirectoryEndpoint, { + method: 'POST', + body: { + include_all_networks: true, + filter: { + generic_search_term: searchTerm, + }, + limit: 1, + }, + accessToken: client.accessToken, + }); + + if (publicRoomsRes.chunk.length > 0) { + foundResults = true; + resolve(); + break; + } + } + } catch (err) { + reject(err); + } + }); +} + module.exports = { ensureUserRegistered, getTestClientForAs, @@ -433,6 +479,7 @@ module.exports = { sendMessage, createMessagesInRoom, getMessagesInRoom, + waitForResultsInHomeserverRoomDirectory, updateProfile, uploadContent, };