Fetch and render from homeserver

This commit is contained in:
Eric Eastwood 2022-02-07 19:55:11 -06:00
parent 4ab26ef2d1
commit a49657f9e3
9 changed files with 2367 additions and 81 deletions

95
.eslintrc.json Normal file
View File

@ -0,0 +1,95 @@
{
"env": {
"commonjs": true,
"node": true
},
"parserOptions": {
"ecmaVersion": 2018,
"sourceType": "script"
},
"plugins": ["node"],
"extends": ["eslint:recommended", "prettier"],
"rules": {
"indent": "off",
"comma-dangle": "off",
"quotes": "off",
"eqeqeq": ["warn", "allow-null"],
"strict": ["error", "safe"],
"no-unused-vars": ["error"],
"no-extra-boolean-cast": ["warn"],
"complexity": [
"error",
{
"max": 12
}
],
"max-statements-per-line": [
"error",
{
"max": 3
}
],
"no-debugger": "error",
"no-dupe-keys": "error",
"no-unsafe-finally": "error",
"no-with": "error",
"no-useless-call": "error",
"no-spaced-func": "error",
"no-useless-escape": "warn",
"max-statements": ["warn", 30],
"max-depth": ["error", 4],
"no-throw-literal": ["error"],
"no-sequences": "error",
"no-warning-comments": [
"warn",
{
"terms": ["fixme", "xxx"],
"location": "anywhere"
}
],
"radix": "error",
"yoda": "error",
"no-nested-ternary": "warn",
"no-whitespace-before-property": "error",
"no-trailing-spaces": ["error"],
"space-in-parens": ["warn", "never"],
"max-nested-callbacks": ["error", 6],
"eol-last": "warn",
"no-mixed-spaces-and-tabs": "error",
"no-negated-condition": "warn",
"no-unneeded-ternary": "error",
"no-use-before-define": ["warn", { "variables": true, "functions": true, "classes": true }],
"no-undef": "error",
"no-param-reassign": "warn",
"no-multi-spaces": [
"warn",
{
"exceptions": {
"Property": true
}
}
],
"key-spacing": [
"warn",
{
"singleLine": {
"beforeColon": false,
"afterColon": true
},
"multiLine": {
"beforeColon": false,
"afterColon": true,
"mode": "minimum"
}
}
],
"node/no-missing-require": "error",
"node/no-missing-import": "error",
"node/no-unsupported-features": [
"error",
{
"version": 10
}
]
}
}

1
.gitignore vendored
View File

@ -4,3 +4,4 @@ dist
dist-ssr dist-ssr
*.local *.local
secrets.json secrets.json
config.json

2078
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@ -1,15 +1,18 @@
{ {
"name": "matrix-public-archive", "name": "matrix-public-archive",
"version": "0.0.0", "version": "0.0.0",
"scripts": { "scripts": {},
},
"devDependencies": { "devDependencies": {
"eslint": "^8.8.0",
"eslint-config-prettier": "^8.3.0",
"eslint-plugin-node": "^11.1.0",
"eslint-plugin-prettier": "^4.0.0",
"prettier": "^2.5.1" "prettier": "^2.5.1"
}, },
"dependencies": { "dependencies": {
"express": "^4.17.2", "express": "^4.17.2",
"hydrogen-view-sdk": "^0.0.4", "hydrogen-view-sdk": "^0.0.4",
"linkedom": "^0.13.2" "linkedom": "^0.13.2",
"node-fetch": "^2.6.7"
} }
} }

View File

@ -0,0 +1,12 @@
// Simple middleware for handling exceptions inside of async express routes and
// passing them to your express error handlers.
//
// via https://github.com/Abazhenov/express-async-handler
const asyncUtil = (fn) =>
function asyncUtilWrap(...args) {
const fnReturn = fn(...args);
const next = args[args.length - 1];
return Promise.resolve(fnReturn).catch(next);
};
module.exports = asyncUtil;

View File

@ -0,0 +1,82 @@
const assert = require('assert');
const path = require('path');
const fetch = require('node-fetch');
const { matrixServerUrl } = require('../config.json');
const secrets = require('../secrets.json');
const matrixAccessToken = secrets['matrix-access-token'];
assert(matrixAccessToken);
class HTTPResponseError extends Error {
constructor(response, responseText, ...args) {
super(
`HTTP Error Response: ${response.status} ${response.statusText}: ${responseText}\n\tURL=${response.url}`,
...args
);
this.response = response;
}
}
const checkStatus = async (response) => {
if (response.ok) {
// response.status >= 200 && response.status < 300
return response;
} else {
const responseText = await response.text();
throw new HTTPResponseError(response, responseText);
}
};
async function fetchEndpoint(endpoint) {
const res = await fetch(endpoint, {
headers: {
'Content-Type': 'application/json',
Authorization: `Bearer ${matrixAccessToken}`,
},
});
await checkStatus(res);
const data = await res.json();
return data;
}
async function fetchEventsForTimestamp(roomId, ts) {
const timestampToEventEndpoint = path.join(
matrixServerUrl,
`_matrix/client/unstable/org.matrix.msc3030/rooms/${roomId}/timestamp_to_event?ts=${ts}&dir=f`
);
console.log('timestampToEventEndpoint', timestampToEventEndpoint);
const timestampToEventResData = await fetchEndpoint(timestampToEventEndpoint);
//console.log('timestampToEventResData', timestampToEventResData);
const eventIdForTimestamp = timestampToEventResData.event_id;
assert(eventIdForTimestamp);
const contextEndpoint = path.join(
matrixServerUrl,
`_matrix/client/r0/rooms/${roomId}/context/${eventIdForTimestamp}?limit=0`
);
const contextResData = await fetchEndpoint(contextEndpoint);
//console.log('contextResData', contextResData);
const messagesEndpoint = path.join(
matrixServerUrl,
`_matrix/client/r0/rooms/${roomId}/messages?from=${contextResData.start}&limit=100&filter={"lazy_load_members":true,"include_redundant_members":true}`
);
const messageResData = await fetchEndpoint(messagesEndpoint);
//console.log('messageResData', messageResData);
const stateEventMap = {};
for (const stateEvent of messageResData.state) {
if (stateEvent.type === 'm.room.member') {
stateEventMap[stateEvent.state_key] = stateEventMap;
}
}
return {
stateEventMap,
events: messageResData.chunk,
};
}
module.exports = fetchEventsForTimestamp;

View File

@ -1,3 +1,4 @@
const assert = require('assert');
const { const {
Platform, Platform,
MediaRepository, MediaRepository,
@ -12,14 +13,11 @@ const {
TimelineView, TimelineView,
} = require('hydrogen-view-sdk'); } = require('hydrogen-view-sdk');
const roomId = '!OWqptMTjnQfUWubCid:matrix.org';
const eventsJson = require('../fixtures/events2.json');
let eventIndexCounter = 0; let eventIndexCounter = 0;
const fragmentIdComparer = new FragmentIdComparer([]); const fragmentIdComparer = new FragmentIdComparer([]);
function makeEventEntryFromEventJson(roomId, eventJson) { function makeEventEntryFromEventJson(eventJson, memberEvent) {
console.assert(roomId); assert(eventJson);
console.assert(eventJson); assert(memberEvent);
const eventIndex = eventIndexCounter; const eventIndex = eventIndexCounter;
const eventEntry = new EventEntry( const eventEntry = new EventEntry(
@ -28,8 +26,8 @@ function makeEventEntryFromEventJson(roomId, eventJson) {
eventIndex: eventIndex, // TODO: What should this be? eventIndex: eventIndex, // TODO: What should this be?
roomId: roomId, roomId: roomId,
event: eventJson, event: eventJson,
displayName: 'todo', displayName: memberEvent.content && memberEvent.content.displayname,
avatarUrl: 'mxc://matrix.org/todo', avatarUrl: memberEvent.content && memberEvent.content.avatar_url,
key: encodeKey(roomId, 0, eventIndex), key: encodeKey(roomId, 0, eventIndex),
eventIdKey: encodeEventIdKey(roomId, eventJson.event_id), eventIdKey: encodeEventIdKey(roomId, eventJson.event_id),
}, },
@ -42,6 +40,11 @@ function makeEventEntryFromEventJson(roomId, eventJson) {
} }
async function mountHydrogen() { async function mountHydrogen() {
const events = global.INPUT_EVENTS;
assert(events);
const stateEventMap = global.INPUT_STATE_EVENT_MAP;
assert(stateEventMap);
const app = document.querySelector('#app'); const app = document.querySelector('#app');
const config = {}; const config = {};
@ -82,9 +85,9 @@ async function mountHydrogen() {
}, },
}); });
//console.log('eventsJson', eventsJson); const eventEntries = events.map((event) => {
const eventEntries = eventsJson.map((eventJson) => { const memberEvent = stateEventMap[event.user_id];
return makeEventEntryFromEventJson(roomId, eventJson); return makeEventEntryFromEventJson(event, memberEvent);
}); });
//console.log('eventEntries', eventEntries); //console.log('eventEntries', eventEntries);

View File

@ -0,0 +1,60 @@
const assert = require('assert');
const vm = require('vm');
const path = require('path');
const { readFile } = require('fs').promises;
const crypto = require('crypto');
const { parseHTML } = require('linkedom');
async function renderToString(events, stateEventMap) {
assert(events);
assert(stateEventMap);
const dom = parseHTML(`
<!doctype html>
<html lang="en">
<body>
<div id="app" class="hydrogen"></div>
</body>
</html>
`);
if (!dom.requestAnimationFrame) {
dom.requestAnimationFrame = function (cb) {
setTimeout(cb, 0);
};
}
const vmContext = vm.createContext(dom);
// Make the dom properties available in sub-`require(...)` calls
vmContext.global.window = dom.window;
vmContext.global.document = dom.document;
vmContext.global.Node = dom.Node;
vmContext.global.navigator = dom.navigator;
vmContext.global.DOMParser = dom.DOMParser;
vmContext.global.crypto = crypto.webcrypto;
// So require(...) works in the vm
vmContext.global.require = require;
// So we can see logs from the underlying vm
vmContext.global.console = console;
vmContext.global.INPUT_EVENTS = events;
vmContext.global.INPUT_STATE_EVENT_MAP = stateEventMap;
const hydrogenRenderScriptCode = await readFile(
path.resolve(__dirname, './hydrogen-vm-render-script.js'),
'utf8'
);
const hydrogenRenderScript = new vm.Script(hydrogenRenderScriptCode, {
filename: 'hydrogen-vm-render-script.js',
});
const vmResult = hydrogenRenderScript.runInContext(vmContext);
// Wait for everything to render
// (waiting on the promise returned from `hydrogen-render-script.js`)
await vmResult;
const documentString = dom.document.querySelector('#app').toString();
return documentString;
}
module.exports = renderToString;

View File

@ -1,74 +1,27 @@
const vm = require('vm');
const path = require('path');
const { readFile } = require('fs').promises;
const crypto = require('crypto');
const { parseHTML } = require('linkedom');
const express = require('express'); const express = require('express');
const asyncHandler = require('./express-async-handler');
const fetchEventsForTimestamp = require('./fetch-events-for-timestamp');
const renderHydrogenToString = require('./render-hydrogen-to-string');
const app = express(); const app = express();
// const hsdk = require('hydrogen-view-sdk');
// console.log(`require.resolve('hydrogen-view-sdk')`, require.resolve('hydrogen-view-sdk'));
// console.log('hsdk', hsdk);
// console.log('FragmentIdComparer', hsdk.FragmentIdComparer);
async function renderToString() {
const dom = parseHTML(`
<!doctype html>
<html lang="en">
<body>
<div id="app" class="hydrogen"></div>
</body>
</html>
`);
if (!dom.crypto) {
dom.crypto = crypto.webcrypto;
}
if (!dom.requestAnimationFrame) {
dom.requestAnimationFrame = function (cb) {
setTimeout(cb, 0);
};
}
const vmContext = vm.createContext(dom);
// Make the dom properties available in sub-`require(...)` calls
vmContext.global.window = dom.window;
vmContext.global.document = dom.document;
vmContext.global.Node = dom.Node;
vmContext.global.navigator = dom.navigator;
vmContext.global.DOMParser = dom.DOMParser;
// So require(...) works in the vm
vmContext.global.require = require;
// So we can see logs from the underlying vm
vmContext.global.console = console;
const hydrogenRenderScriptCode = await readFile(
path.resolve(__dirname, './hydrogen-render-script.js'),
'utf8'
);
const hydrogenRenderScript = new vm.Script(hydrogenRenderScriptCode);
const vmResult = hydrogenRenderScript.runInContext(vmContext);
// Wait for everything to render
// (waiting on the promise returned from `hydrogen-render-script.js`)
await vmResult;
const documentString = dom.document.querySelector('#app').toString();
return documentString;
}
app.get('/style.css', async function (req, res) { app.get('/style.css', async function (req, res) {
const htmlOutput = await renderToString();
res.set('Content-Type', 'text/css'); res.set('Content-Type', 'text/css');
res.sendFile(require.resolve('hydrogen-view-sdk/style.css')); res.sendFile(require.resolve('hydrogen-view-sdk/style.css'));
}); });
app.get('/', async function (req, res) { app.get(
const hydrogenHtmlOutput = await renderToString(); '/',
asyncHandler(async function (req, res) {
const { events, stateEventMap } = await fetchEventsForTimestamp(
'!HBehERstyQBxyJDLfR:my.synapse.server',
new Date('2022-01-01').getTime()
);
const pageHtml = ` const hydrogenHtmlOutput = await renderHydrogenToString(events, stateEventMap);
const pageHtml = `
<!doctype html> <!doctype html>
<html lang="en"> <html lang="en">
<head> <head>
@ -80,8 +33,9 @@ app.get('/', async function (req, res) {
</html> </html>
`; `;
res.set('Content-Type', 'text/html'); res.set('Content-Type', 'text/html');
res.send(pageHtml); res.send(pageHtml);
}); })
);
app.listen(3050); app.listen(3050);