Fetch and render from homeserver

.eslintrc.json

@@ -0,0 +1,95 @@
+{
+  "env": {
+    "commonjs": true,
+    "node": true
+  },
+  "parserOptions": {
+    "ecmaVersion": 2018,
+    "sourceType": "script"
+  },
+  "plugins": ["node"],
+  "extends": ["eslint:recommended", "prettier"],
+  "rules": {
+    "indent": "off",
+    "comma-dangle": "off",
+    "quotes": "off",
+    "eqeqeq": ["warn", "allow-null"],
+    "strict": ["error", "safe"],
+    "no-unused-vars": ["error"],
+    "no-extra-boolean-cast": ["warn"],
+    "complexity": [
+      "error",
+      {
+        "max": 12
+      }
+    ],
+    "max-statements-per-line": [
+      "error",
+      {
+        "max": 3
+      }
+    ],
+    "no-debugger": "error",
+    "no-dupe-keys": "error",
+    "no-unsafe-finally": "error",
+    "no-with": "error",
+    "no-useless-call": "error",
+    "no-spaced-func": "error",
+    "no-useless-escape": "warn",
+    "max-statements": ["warn", 30],
+    "max-depth": ["error", 4],
+    "no-throw-literal": ["error"],
+    "no-sequences": "error",
+    "no-warning-comments": [
+      "warn",
+      {
+        "terms": ["fixme", "xxx"],
+        "location": "anywhere"
+      }
+    ],
+    "radix": "error",
+    "yoda": "error",
+    "no-nested-ternary": "warn",
+    "no-whitespace-before-property": "error",
+    "no-trailing-spaces": ["error"],
+    "space-in-parens": ["warn", "never"],
+    "max-nested-callbacks": ["error", 6],
+    "eol-last": "warn",
+    "no-mixed-spaces-and-tabs": "error",
+    "no-negated-condition": "warn",
+    "no-unneeded-ternary": "error",
+    "no-use-before-define": ["warn", { "variables": true, "functions": true, "classes": true }],
+    "no-undef": "error",
+    "no-param-reassign": "warn",
+    "no-multi-spaces": [
+      "warn",
+      {
+        "exceptions": {
+          "Property": true
+        }
+      }
+    ],
+    "key-spacing": [
+      "warn",
+      {
+        "singleLine": {
+          "beforeColon": false,
+          "afterColon": true
+        },
+        "multiLine": {
+          "beforeColon": false,
+          "afterColon": true,
+          "mode": "minimum"
+        }
+      }
+    ],
+    "node/no-missing-require": "error",
+    "node/no-missing-import": "error",
+    "node/no-unsupported-features": [
+      "error",
+      {
+        "version": 10
+      }
+    ]
+  }
+}

.gitignore

@@ -4,3 +4,4 @@ dist
 dist-ssr
 *.local
 secrets.json
+config.json

package.json

@@ -1,15 +1,18 @@
 {
   "name": "matrix-public-archive",
   "version": "0.0.0",
-  "scripts": {
-    
-  },
+  "scripts": {},
   "devDependencies": {
+    "eslint": "^8.8.0",
+    "eslint-config-prettier": "^8.3.0",
+    "eslint-plugin-node": "^11.1.0",
+    "eslint-plugin-prettier": "^4.0.0",
     "prettier": "^2.5.1"
   },
   "dependencies": {
     "express": "^4.17.2",
     "hydrogen-view-sdk": "^0.0.4",
-    "linkedom": "^0.13.2"
+    "linkedom": "^0.13.2",
+    "node-fetch": "^2.6.7"
   }
 }

server/express-async-handler.js

@@ -0,0 +1,12 @@
+// Simple middleware for handling exceptions inside of async express routes and
+// passing them to your express error handlers.
+//
+// via https://github.com/Abazhenov/express-async-handler
+const asyncUtil = (fn) =>
+  function asyncUtilWrap(...args) {
+    const fnReturn = fn(...args);
+    const next = args[args.length - 1];
+    return Promise.resolve(fnReturn).catch(next);
+  };
+
+module.exports = asyncUtil;

server/fetch-events-for-timestamp.js

@@ -0,0 +1,82 @@
+const assert = require('assert');
+const path = require('path');
+const fetch = require('node-fetch');
+const { matrixServerUrl } = require('../config.json');
+const secrets = require('../secrets.json');
+
+const matrixAccessToken = secrets['matrix-access-token'];
+assert(matrixAccessToken);
+
+class HTTPResponseError extends Error {
+  constructor(response, responseText, ...args) {
+    super(
+      `HTTP Error Response: ${response.status} ${response.statusText}: ${responseText}\n\tURL=${response.url}`,
+      ...args
+    );
+    this.response = response;
+  }
+}
+
+const checkStatus = async (response) => {
+  if (response.ok) {
+    // response.status >= 200 && response.status < 300
+    return response;
+  } else {
+    const responseText = await response.text();
+    throw new HTTPResponseError(response, responseText);
+  }
+};
+
+async function fetchEndpoint(endpoint) {
+  const res = await fetch(endpoint, {
+    headers: {
+      'Content-Type': 'application/json',
+      Authorization: `Bearer ${matrixAccessToken}`,
+    },
+  });
+  await checkStatus(res);
+  const data = await res.json();
+  return data;
+}
+
+async function fetchEventsForTimestamp(roomId, ts) {
+  const timestampToEventEndpoint = path.join(
+    matrixServerUrl,
+    `_matrix/client/unstable/org.matrix.msc3030/rooms/${roomId}/timestamp_to_event?ts=${ts}&dir=f`
+  );
+  console.log('timestampToEventEndpoint', timestampToEventEndpoint);
+
+  const timestampToEventResData = await fetchEndpoint(timestampToEventEndpoint);
+  //console.log('timestampToEventResData', timestampToEventResData);
+
+  const eventIdForTimestamp = timestampToEventResData.event_id;
+  assert(eventIdForTimestamp);
+
+  const contextEndpoint = path.join(
+    matrixServerUrl,
+    `_matrix/client/r0/rooms/${roomId}/context/${eventIdForTimestamp}?limit=0`
+  );
+  const contextResData = await fetchEndpoint(contextEndpoint);
+  //console.log('contextResData', contextResData);
+
+  const messagesEndpoint = path.join(
+    matrixServerUrl,
+    `_matrix/client/r0/rooms/${roomId}/messages?from=${contextResData.start}&limit=100&filter={"lazy_load_members":true,"include_redundant_members":true}`
+  );
+  const messageResData = await fetchEndpoint(messagesEndpoint);
+  //console.log('messageResData', messageResData);
+
+  const stateEventMap = {};
+  for (const stateEvent of messageResData.state) {
+    if (stateEvent.type === 'm.room.member') {
+      stateEventMap[stateEvent.state_key] = stateEventMap;
+    }
+  }
+
+  return {
+    stateEventMap,
+    events: messageResData.chunk,
+  };
+}
+
+module.exports = fetchEventsForTimestamp;

server/hydrogen-vm-render-script.js

@@ -1,3 +1,4 @@
+const assert = require('assert');
 const {
   Platform,
   MediaRepository,
@@ -12,14 +13,11 @@ const {
   TimelineView,
 } = require('hydrogen-view-sdk');
 
-const roomId = '!OWqptMTjnQfUWubCid:matrix.org';
-const eventsJson = require('../fixtures/events2.json');
-
 let eventIndexCounter = 0;
 const fragmentIdComparer = new FragmentIdComparer([]);
-function makeEventEntryFromEventJson(roomId, eventJson) {
-  console.assert(roomId);
-  console.assert(eventJson);
+function makeEventEntryFromEventJson(eventJson, memberEvent) {
+  assert(eventJson);
+  assert(memberEvent);
 
   const eventIndex = eventIndexCounter;
   const eventEntry = new EventEntry(
@@ -28,8 +26,8 @@ function makeEventEntryFromEventJson(roomId, eventJson) {
       eventIndex: eventIndex, // TODO: What should this be?
       roomId: roomId,
       event: eventJson,
-      displayName: 'todo',
-      avatarUrl: 'mxc://matrix.org/todo',
+      displayName: memberEvent.content && memberEvent.content.displayname,
+      avatarUrl: memberEvent.content && memberEvent.content.avatar_url,
       key: encodeKey(roomId, 0, eventIndex),
       eventIdKey: encodeEventIdKey(roomId, eventJson.event_id),
     },
@@ -42,6 +40,11 @@ function makeEventEntryFromEventJson(roomId, eventJson) {
 }
 
 async function mountHydrogen() {
+  const events = global.INPUT_EVENTS;
+  assert(events);
+  const stateEventMap = global.INPUT_STATE_EVENT_MAP;
+  assert(stateEventMap);
+
   const app = document.querySelector('#app');
 
   const config = {};
@@ -82,9 +85,9 @@ async function mountHydrogen() {
     },
   });
 
-  //console.log('eventsJson', eventsJson);
-  const eventEntries = eventsJson.map((eventJson) => {
-    return makeEventEntryFromEventJson(roomId, eventJson);
+  const eventEntries = events.map((event) => {
+    const memberEvent = stateEventMap[event.user_id];
+    return makeEventEntryFromEventJson(event, memberEvent);
   });
   //console.log('eventEntries', eventEntries);
 

server/render-hydrogen-to-string.js

@@ -0,0 +1,60 @@
+const assert = require('assert');
+const vm = require('vm');
+const path = require('path');
+const { readFile } = require('fs').promises;
+const crypto = require('crypto');
+const { parseHTML } = require('linkedom');
+
+async function renderToString(events, stateEventMap) {
+  assert(events);
+  assert(stateEventMap);
+
+  const dom = parseHTML(`
+    <!doctype html>
+    <html lang="en">
+      <body>
+        <div id="app" class="hydrogen"></div>
+      </body>
+    </html>
+  `);
+
+  if (!dom.requestAnimationFrame) {
+    dom.requestAnimationFrame = function (cb) {
+      setTimeout(cb, 0);
+    };
+  }
+
+  const vmContext = vm.createContext(dom);
+  // Make the dom properties available in sub-`require(...)` calls
+  vmContext.global.window = dom.window;
+  vmContext.global.document = dom.document;
+  vmContext.global.Node = dom.Node;
+  vmContext.global.navigator = dom.navigator;
+  vmContext.global.DOMParser = dom.DOMParser;
+  vmContext.global.crypto = crypto.webcrypto;
+
+  // So require(...) works in the vm
+  vmContext.global.require = require;
+  // So we can see logs from the underlying vm
+  vmContext.global.console = console;
+
+  vmContext.global.INPUT_EVENTS = events;
+  vmContext.global.INPUT_STATE_EVENT_MAP = stateEventMap;
+
+  const hydrogenRenderScriptCode = await readFile(
+    path.resolve(__dirname, './hydrogen-vm-render-script.js'),
+    'utf8'
+  );
+  const hydrogenRenderScript = new vm.Script(hydrogenRenderScriptCode, {
+    filename: 'hydrogen-vm-render-script.js',
+  });
+  const vmResult = hydrogenRenderScript.runInContext(vmContext);
+  // Wait for everything to render
+  // (waiting on the promise returned from `hydrogen-render-script.js`)
+  await vmResult;
+
+  const documentString = dom.document.querySelector('#app').toString();
+  return documentString;
+}
+
+module.exports = renderToString;

server/server1.js

@@ -1,74 +1,27 @@
-const vm = require('vm');
-const path = require('path');
-const { readFile } = require('fs').promises;
-const crypto = require('crypto');
-const { parseHTML } = require('linkedom');
 const express = require('express');
+const asyncHandler = require('./express-async-handler');
+
+const fetchEventsForTimestamp = require('./fetch-events-for-timestamp');
+const renderHydrogenToString = require('./render-hydrogen-to-string');
+
 const app = express();
 
-// const hsdk = require('hydrogen-view-sdk');
-// console.log(`require.resolve('hydrogen-view-sdk')`, require.resolve('hydrogen-view-sdk'));
-// console.log('hsdk', hsdk);
-// console.log('FragmentIdComparer', hsdk.FragmentIdComparer);
-
-async function renderToString() {
-  const dom = parseHTML(`
-    <!doctype html>
-    <html lang="en">
-      <body>
-        <div id="app" class="hydrogen"></div>
-      </body>
-    </html>
-  `);
-
-  if (!dom.crypto) {
-    dom.crypto = crypto.webcrypto;
-  }
-
-  if (!dom.requestAnimationFrame) {
-    dom.requestAnimationFrame = function (cb) {
-      setTimeout(cb, 0);
-    };
-  }
-
-  const vmContext = vm.createContext(dom);
-  // Make the dom properties available in sub-`require(...)` calls
-  vmContext.global.window = dom.window;
-  vmContext.global.document = dom.document;
-  vmContext.global.Node = dom.Node;
-  vmContext.global.navigator = dom.navigator;
-  vmContext.global.DOMParser = dom.DOMParser;
-
-  // So require(...) works in the vm
-  vmContext.global.require = require;
-  // So we can see logs from the underlying vm
-  vmContext.global.console = console;
-
-  const hydrogenRenderScriptCode = await readFile(
-    path.resolve(__dirname, './hydrogen-render-script.js'),
-    'utf8'
-  );
-  const hydrogenRenderScript = new vm.Script(hydrogenRenderScriptCode);
-  const vmResult = hydrogenRenderScript.runInContext(vmContext);
-  // Wait for everything to render
-  // (waiting on the promise returned from `hydrogen-render-script.js`)
-  await vmResult;
-
-  const documentString = dom.document.querySelector('#app').toString();
-  return documentString;
-}
-
 app.get('/style.css', async function (req, res) {
-  const htmlOutput = await renderToString();
-
   res.set('Content-Type', 'text/css');
   res.sendFile(require.resolve('hydrogen-view-sdk/style.css'));
 });
 
-app.get('/', async function (req, res) {
-  const hydrogenHtmlOutput = await renderToString();
+app.get(
+  '/',
+  asyncHandler(async function (req, res) {
+    const { events, stateEventMap } = await fetchEventsForTimestamp(
+      '!HBehERstyQBxyJDLfR:my.synapse.server',
+      new Date('2022-01-01').getTime()
+    );
 
-  const pageHtml = `
+    const hydrogenHtmlOutput = await renderHydrogenToString(events, stateEventMap);
+
+    const pageHtml = `
     <!doctype html>
     <html lang="en">
       <head>
@@ -80,8 +33,9 @@ app.get('/', async function (req, res) {
     </html>
   `;
 
-  res.set('Content-Type', 'text/html');
-  res.send(pageHtml);
-});
+    res.set('Content-Type', 'text/html');
+    res.send(pageHtml);
+  })
+);
 
 app.listen(3050);