Remove unneeded `include_redundant_members` from `/messages` `filter` and test that member state is still visible (#29)

Follow-up to https://github.com/matrix-org/matrix-public-archive/pull/28#discussion_r909428366

server/fetch-events-in-range.js

@@ -47,13 +47,16 @@ async function fetchEventsFromTimestampBackwards(accessToken, roomId, ts, limit)
   assert(eventIdForTimestamp);
   //console.log('eventIdForTimestamp', eventIdForTimestamp);
 
-  // We only use this endpoint to get a pagination we can use with `/messages`.
+  // We only use this endpoint to get a pagination token we can use with
-  // 
+  // `/messages`.
-  // We add `limit=0` here because we want to grab
+  //
+  // We add `limit=0` here because we want to grab the pagination token right
+  // (before/after) the event.
   //
   // Add `filter={"lazy_load_members":true}` so that this endpoint responds
-  // without timing out. Otherwise, the homeserver returns all state in the room
+  // without timing out by returning just the state for the sender of the
-  // at that point in time which in big rooms, can be 100k member events that we
+  // included event. Otherwise, the homeserver returns all state in the room at
+  // that point in time which in big rooms, can be 100k member events that we
   // don't care about anyway. Synapse seems to timeout at about the ~5k state
   // event mark.
   const contextEndpoint = urlJoin(
@@ -65,13 +68,11 @@ async function fetchEventsFromTimestampBackwards(accessToken, roomId, ts, limit)
   });
   //console.log('contextResData', contextResData);
 
-  // TODO: Do we need `"include_redundant_members":true` here?
+  // Add `filter={"lazy_load_members":true}` to only get member state events for
-  //
+  // the messages included in the response
-  // Add `filter={"lazy_load_members":true,"include_redundant_members":true}` to
-  // get member state events included
   const messagesEndpoint = urlJoin(
     matrixServerUrl,
-    `_matrix/client/r0/rooms/${roomId}/messages?dir=b&from=${contextResData.end}&limit=${limit}&filter={"lazy_load_members":true,"include_redundant_members":true}`
+    `_matrix/client/r0/rooms/${roomId}/messages?dir=b&from=${contextResData.end}&limit=${limit}&filter={"lazy_load_members":true}`
   );
   const messageResData = await fetchEndpointAsJson(messagesEndpoint, {
     accessToken,

test/client-utils.js

@@ -34,23 +34,24 @@ async function getTestClientForHs(testMatrixServerUrl) {
     }
   );
 
-  const userId = registerResponse['user_id'];
+  const applicationServiceUserIdOverride = registerResponse['user_id'];
-  assert(userId);
+  assert(applicationServiceUserIdOverride);
 
   return {
     homeserverUrl: testMatrixServerUrl,
     // We use the application service AS token because we need to be able to use
     // the `?ts` timestamp massaging when sending events
     accessToken: matrixAccessToken,
-    userId: userId,
+    userId: applicationServiceUserIdOverride,
+    applicationServiceUserIdOverride,
   };
 }
 
 // Create a public room to test in
 async function createTestRoom(client) {
   let qs = new URLSearchParams();
-  if (client.userId) {
+  if (client.applicationServiceUserIdOverride) {
-    qs.append('user_id', client.userId);
+    qs.append('user_id', client.applicationServiceUserIdOverride);
   }
 
   const createRoomResponse = await fetchEndpointAsJson(
@@ -87,8 +88,8 @@ async function joinRoom({ client, roomId, viaServers }) {
     });
   }
 
-  if (client.userId) {
+  if (client.applicationServiceUserIdOverride) {
-    qs.append('user_id', client.userId);
+    qs.append('user_id', client.applicationServiceUserIdOverride);
   }
 
   const joinRoomResponse = await fetchEndpointAsJson(
@@ -104,7 +105,7 @@ async function joinRoom({ client, roomId, viaServers }) {
   return joinedRoomId;
 }
 
-async function sendEvent({ client, roomId, eventType, content, timestamp }) {
+async function sendEvent({ client, roomId, eventType, stateKey, content, timestamp }) {
   assert(client);
   assert(roomId);
   assert(content);
@@ -112,29 +113,36 @@ async function sendEvent({ client, roomId, eventType, content, timestamp }) {
   let qs = new URLSearchParams();
   if (timestamp) {
     assert(
-      timestamp && client.userId,
+      timestamp && client.applicationServiceUserIdOverride,
       'We can only do `?ts` massaging from an application service access token. ' +
-        'Expected `client.userId` to be defined so we can act on behalf of that user'
+        'Expected `client.applicationServiceUserIdOverride` to be defined so we can act on behalf of that user'
     );
 
     qs.append('ts', timestamp);
   }
 
-  if (client.userId) {
+  if (client.applicationServiceUserIdOverride) {
-    qs.append('user_id', client.userId);
+    qs.append('user_id', client.applicationServiceUserIdOverride);
   }
 
-  const sendResponse = await fetchEndpointAsJson(
+  let url;
-    urlJoin(
+  if (stateKey) {
+    url = urlJoin(
+      client.homeserverUrl,
+      `/_matrix/client/v3/rooms/${roomId}/state/${eventType}/${stateKey}?${qs.toString()}`
+    );
+  } else {
+    url = urlJoin(
       client.homeserverUrl,
       `/_matrix/client/v3/rooms/${roomId}/send/${eventType}/${getTxnId()}?${qs.toString()}`
-    ),
+    );
-    {
+  }
-      method: 'PUT',
+
-      body: content,
+  const sendResponse = await fetchEndpointAsJson(url, {
-      accessToken: client.accessToken,
+    method: 'PUT',
-    }
+    body: content,
-  );
+    accessToken: client.accessToken,
+  });
 
   const eventId = sendResponse['event_id'];
   assert(eventId);
@@ -164,6 +172,45 @@ async function createMessagesInRoom({ client, roomId, numMessages, prefix, times
   return eventIds;
 }
 
+async function updateProfile({ client, displayName, avatarUrl }) {
+  let qs = new URLSearchParams();
+  if (client.applicationServiceUserIdOverride) {
+    qs.append('user_id', client.applicationServiceUserIdOverride);
+  }
+
+  const updateDisplayNamePromise = fetchEndpointAsJson(
+    urlJoin(
+      client.homeserverUrl,
+      `/_matrix/client/v3/profile/${client.userId}/displayname?${qs.toString()}`
+    ),
+    {
+      method: 'PUT',
+      body: {
+        displayname: displayName,
+      },
+      accessToken: client.accessToken,
+    }
+  );
+
+  const updateAvatarUrlPromise = fetchEndpointAsJson(
+    urlJoin(
+      client.homeserverUrl,
+      `/_matrix/client/v3/profile/${client.userId}/avatar_url?${qs.toString()}`
+    ),
+    {
+      method: 'PUT',
+      body: {
+        avatar_url: avatarUrl,
+      },
+      accessToken: client.accessToken,
+    }
+  );
+
+  await Promise.all([updateDisplayNamePromise, updateAvatarUrlPromise]);
+
+  return null;
+}
+
 // Uploads the given data Buffer and returns the MXC URI of the uploaded content
 async function uploadContent({ client, roomId, data, fileName, contentType }) {
   assert(client);
@@ -171,8 +218,8 @@ async function uploadContent({ client, roomId, data, fileName, contentType }) {
   assert(data);
 
   let qs = new URLSearchParams();
-  if (client.userId) {
+  if (client.applicationServiceUserIdOverride) {
-    qs.append('user_id', client.userId);
+    qs.append('user_id', client.applicationServiceUserIdOverride);
   }
 
   if (fileName) {
@@ -207,5 +254,6 @@ module.exports = {
   sendEvent,
   sendMessage,
   createMessagesInRoom,
+  updateProfile,
   uploadContent,
 };

test/e2e-tests.js

@@ -20,6 +20,7 @@ const {
   sendEvent,
   sendMessage,
   createMessagesInRoom,
+  updateProfile,
   uploadContent,
 } = require('./client-utils');
 
@@ -199,7 +200,23 @@ describe('matrix-public-archive', () => {
       const client = await getTestClientForHs(testMatrixServerUrl1);
       const roomId = await createTestRoom(client);
 
-      // TODO: Set avatar of user
+      const userAvatarBuffer = Buffer.from(
+        // Purple PNG pixel
+        'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mPsD9j0HwAFmQKScbjOAwAAAABJRU5ErkJggg==',
+        'base64'
+      );
+      const userAvatarMxcUri = await uploadContent({
+        client,
+        roomId,
+        data: userAvatarBuffer,
+        fileName: 'client user avatar',
+      });
+      const displayName = `${client.userId}-some-display-name`;
+      await updateProfile({
+        client,
+        displayName,
+        avatarUrl: userAvatarMxcUri,
+      });
 
       // TODO: Set avatar of room
 
@@ -305,8 +322,27 @@ describe('matrix-public-archive', () => {
 
       const dom = parseHTML(archivePageHtml);
 
+      // Make sure the user display name is visible on the message
+      assert.match(
+        dom.document.querySelector(`[data-event-id="${imageEventId}"]`).outerHTML,
+        new RegExp(`.*${escapeStringRegexp(displayName)}.*`)
+      );
+
+      // Make sure the user avatar is visible on the message
+      const avatarImageElement = dom.document.querySelector(
+        // FIXME: Use more stable select here instead of `.avatar`,
+        // see https://github.com/vector-im/hydrogen-web/pull/773
+        `[data-event-id="${imageEventId}"] .avatar img`
+      );
+      assert(avatarImageElement);
+      assert.match(avatarImageElement.getAttribute('src'), new RegExp(`^http://.*`));
+
       // Make sure the image message is visible
-      const imageElement = dom.document.querySelector(`[data-event-id="${imageEventId}"] img`);
+      const imageElement = dom.document.querySelector(
+        // FIXME: Use more stable select here instead of `.media`,
+        // see https://github.com/vector-im/hydrogen-web/pull/773
+        `[data-event-id="${imageEventId}"] .media img`
+      );
       assert(imageElement);
       assert.match(imageElement.getAttribute('src'), new RegExp(`^http://.*`));
       assert.strictEqual(imageElement.getAttribute('alt'), imageFileName);