E2E test but still failing because fetching from start of day before test events happened

This commit is contained in:
Eric Eastwood 2022-02-23 21:25:05 -06:00
parent 40fa71df03
commit 839e31a35e
21 changed files with 455 additions and 123 deletions

7
.gitignore vendored
View File

@ -3,5 +3,10 @@ node_modules
dist
dist-ssr
*.local
secrets.json
config.user-overrides.json
config/config.dev.user-overrides.json
config/config.beta.user-overrides.json
config/config.prod.user-overrides.json
config.json
secrets.json

View File

@ -5,11 +5,12 @@ config/secrets, using a draft branch of Hydrogen, etc.
In the vein of [feature parity with
Gitter](https://github.com/vector-im/roadmap/issues/26), the goal is to make a
public archive site like Gitter's archives which search engines can index and
keep all of the content accessible/available. There is already
https://view.matrix.org/ (https://github.com/matrix-org/matrix-static) but there
is some desire to make something with more Element-feeling polish and loading
faster (avoid the slow 502's errors that are frequent on `view.matrix.org`).
public archive site for `world_readable` Matrix rooms like Gitter's archives
which search engines can index and keep all of the content accessible/available.
There is already https://view.matrix.org/
(https://github.com/matrix-org/matrix-static) but there is some desire to make
something with more Element-feeling polish and loading faster (avoid the slow
502's errors that are frequent on `view.matrix.org`).
## Plan summary

View File

@ -0,0 +1,10 @@
{
"basePort": "3050",
"basePath": "http://localhost:3050",
"matrixServerUrl": "http://localhost:8008/",
"testMatrixServerUrl1": "http://localhost:11008/",
"testMatrixServerUrl2": "http://localhost:12008/",
"Secrets": "Remove the 'xxx__' prefix and fill these in your config/config.user-overrides.json or by environment variable",
"xxx__matrixAccessToken": "xxx"
}

10
config/config.test.json Normal file
View File

@ -0,0 +1,10 @@
{
"basePort": "3051",
"basePath": "http://localhost:3051",
"matrixServerUrl": "http://localhost:11008/",
"testMatrixServerUrl1": "http://localhost:11008/",
"testMatrixServerUrl2": "http://localhost:12008/",
"Secrets": "xxx",
"matrixAccessToken": "as_token_8664700429a911bbbecf7d91b9e1a74716d669f40cf32259630e38439726e29d"
}

94
package-lock.json generated
View File

@ -14,6 +14,7 @@
"matrix-bot-sdk": "^0.5.19",
"matrix-js-sdk": "^15.5.2",
"matrix-public-archive-shared": "file:./shared/",
"nconf": "^0.11.3",
"node-fetch": "^2.6.7",
"url-join": "^4.0.1"
},
@ -324,7 +325,6 @@
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
"integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==",
"dev": true,
"engines": {
"node": ">=8"
}
@ -383,6 +383,11 @@
"node": ">=0.8"
}
},
"node_modules/async": {
"version": "1.5.2",
"resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz",
"integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo="
},
"node_modules/asynckit": {
"version": "0.4.0",
"resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
@ -704,7 +709,6 @@
"version": "7.0.4",
"resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz",
"integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==",
"dev": true,
"dependencies": {
"string-width": "^4.2.0",
"strip-ansi": "^6.0.0",
@ -1066,8 +1070,7 @@
"node_modules/emoji-regex": {
"version": "8.0.0",
"resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
"integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==",
"dev": true
"integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="
},
"node_modules/encodeurl": {
"version": "1.0.2",
@ -1436,7 +1439,6 @@
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz",
"integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==",
"dev": true,
"engines": {
"node": ">=6"
}
@ -2007,7 +2009,6 @@
"version": "2.0.5",
"resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz",
"integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==",
"dev": true,
"engines": {
"node": "6.* || 8.* || >= 10.*"
}
@ -2409,7 +2410,6 @@
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ini/-/ini-2.0.0.tgz",
"integrity": "sha512-7PnF4oN3CvZF23ADhA5wRaYEQpJ8qygSkbtTXWBeXWXmEVRXK+1ITciHWwHhsjv1TmW0MgacIv6hEi5pX5NQdA==",
"dev": true,
"engines": {
"node": ">=10"
}
@ -2471,7 +2471,6 @@
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
"integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==",
"dev": true,
"engines": {
"node": ">=8"
}
@ -3119,6 +3118,20 @@
"integrity": "sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=",
"dev": true
},
"node_modules/nconf": {
"version": "0.11.3",
"resolved": "https://registry.npmjs.org/nconf/-/nconf-0.11.3.tgz",
"integrity": "sha512-iYsAuDS9pzjVMGIzJrGE0Vk3Eh8r/suJanRAnWGBd29rVS2XtSgzcAo5l6asV3e4hH2idVONHirg1efoBOslBg==",
"dependencies": {
"async": "^1.4.0",
"ini": "^2.0.0",
"secure-keys": "^1.0.0",
"yargs": "^16.1.1"
},
"engines": {
"node": ">= 0.4.0"
}
},
"node_modules/negotiator": {
"version": "0.6.3",
"resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz",
@ -3788,7 +3801,6 @@
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
"integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=",
"dev": true,
"engines": {
"node": ">=0.10.0"
}
@ -3921,6 +3933,11 @@
"entities": "^2.0.0"
}
},
"node_modules/secure-keys": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/secure-keys/-/secure-keys-1.0.0.tgz",
"integrity": "sha1-8MgtmKOxOah3aogIBQuCRDEIf8o="
},
"node_modules/semver": {
"version": "6.3.0",
"resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz",
@ -4085,7 +4102,6 @@
"version": "4.2.3",
"resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
"integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==",
"dev": true,
"dependencies": {
"emoji-regex": "^8.0.0",
"is-fullwidth-code-point": "^3.0.0",
@ -4099,7 +4115,6 @@
"version": "6.0.1",
"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz",
"integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==",
"dev": true,
"dependencies": {
"ansi-regex": "^5.0.1"
},
@ -4512,7 +4527,6 @@
"version": "7.0.0",
"resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz",
"integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==",
"dev": true,
"dependencies": {
"ansi-styles": "^4.0.0",
"string-width": "^4.1.0",
@ -4556,7 +4570,6 @@
"version": "5.0.8",
"resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz",
"integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==",
"dev": true,
"engines": {
"node": ">=10"
}
@ -4570,7 +4583,6 @@
"version": "16.2.0",
"resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz",
"integrity": "sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==",
"dev": true,
"dependencies": {
"cliui": "^7.0.2",
"escalade": "^3.1.1",
@ -4588,7 +4600,6 @@
"version": "20.2.4",
"resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.4.tgz",
"integrity": "sha512-WOkpgNhPTlE73h4VFAFsOnomJVaovO8VqLDzy5saChRBFQFBoMYirowyW+Q9HB4HFF4Z7VZTiG3iSzJJA29yRA==",
"dev": true,
"engines": {
"node": ">=10"
}
@ -4864,8 +4875,7 @@
"ansi-regex": {
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
"integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==",
"dev": true
"integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="
},
"ansi-styles": {
"version": "4.3.0",
@ -4909,6 +4919,11 @@
"resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz",
"integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU="
},
"async": {
"version": "1.5.2",
"resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz",
"integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo="
},
"asynckit": {
"version": "0.4.0",
"resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
@ -5159,7 +5174,6 @@
"version": "7.0.4",
"resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz",
"integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==",
"dev": true,
"requires": {
"string-width": "^4.2.0",
"strip-ansi": "^6.0.0",
@ -5437,8 +5451,7 @@
"emoji-regex": {
"version": "8.0.0",
"resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
"integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==",
"dev": true
"integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="
},
"encodeurl": {
"version": "1.0.2",
@ -5622,8 +5635,7 @@
"escalade": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz",
"integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==",
"dev": true
"integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw=="
},
"escape-goat": {
"version": "2.1.1",
@ -6040,8 +6052,7 @@
"get-caller-file": {
"version": "2.0.5",
"resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz",
"integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==",
"dev": true
"integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg=="
},
"get-stream": {
"version": "4.1.0",
@ -6344,8 +6355,7 @@
"ini": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ini/-/ini-2.0.0.tgz",
"integrity": "sha512-7PnF4oN3CvZF23ADhA5wRaYEQpJ8qygSkbtTXWBeXWXmEVRXK+1ITciHWwHhsjv1TmW0MgacIv6hEi5pX5NQdA==",
"dev": true
"integrity": "sha512-7PnF4oN3CvZF23ADhA5wRaYEQpJ8qygSkbtTXWBeXWXmEVRXK+1ITciHWwHhsjv1TmW0MgacIv6hEi5pX5NQdA=="
},
"ipaddr.js": {
"version": "1.9.1",
@ -6388,8 +6398,7 @@
"is-fullwidth-code-point": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
"integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==",
"dev": true
"integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg=="
},
"is-glob": {
"version": "4.0.3",
@ -6875,6 +6884,17 @@
"integrity": "sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=",
"dev": true
},
"nconf": {
"version": "0.11.3",
"resolved": "https://registry.npmjs.org/nconf/-/nconf-0.11.3.tgz",
"integrity": "sha512-iYsAuDS9pzjVMGIzJrGE0Vk3Eh8r/suJanRAnWGBd29rVS2XtSgzcAo5l6asV3e4hH2idVONHirg1efoBOslBg==",
"requires": {
"async": "^1.4.0",
"ini": "^2.0.0",
"secure-keys": "^1.0.0",
"yargs": "^16.1.1"
}
},
"negotiator": {
"version": "0.6.3",
"resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz",
@ -7359,8 +7379,7 @@
"require-directory": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
"integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=",
"dev": true
"integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I="
},
"resolve": {
"version": "1.22.0",
@ -7447,6 +7466,11 @@
}
}
},
"secure-keys": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/secure-keys/-/secure-keys-1.0.0.tgz",
"integrity": "sha1-8MgtmKOxOah3aogIBQuCRDEIf8o="
},
"semver": {
"version": "6.3.0",
"resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz",
@ -7578,7 +7602,6 @@
"version": "4.2.3",
"resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
"integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==",
"dev": true,
"requires": {
"emoji-regex": "^8.0.0",
"is-fullwidth-code-point": "^3.0.0",
@ -7589,7 +7612,6 @@
"version": "6.0.1",
"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz",
"integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==",
"dev": true,
"requires": {
"ansi-regex": "^5.0.1"
}
@ -7886,7 +7908,6 @@
"version": "7.0.0",
"resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz",
"integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==",
"dev": true,
"requires": {
"ansi-styles": "^4.0.0",
"string-width": "^4.1.0",
@ -7920,8 +7941,7 @@
"y18n": {
"version": "5.0.8",
"resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz",
"integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==",
"dev": true
"integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA=="
},
"yallist": {
"version": "4.0.0",
@ -7932,7 +7952,6 @@
"version": "16.2.0",
"resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz",
"integrity": "sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==",
"dev": true,
"requires": {
"cliui": "^7.0.2",
"escalade": "^3.1.1",
@ -7946,8 +7965,7 @@
"yargs-parser": {
"version": "20.2.4",
"resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.4.tgz",
"integrity": "sha512-WOkpgNhPTlE73h4VFAFsOnomJVaovO8VqLDzy5saChRBFQFBoMYirowyW+Q9HB4HFF4Z7VZTiG3iSzJJA29yRA==",
"dev": true
"integrity": "sha512-WOkpgNhPTlE73h4VFAFsOnomJVaovO8VqLDzy5saChRBFQFBoMYirowyW+Q9HB4HFF4Z7VZTiG3iSzJJA29yRA=="
},
"yargs-unparser": {
"version": "2.0.0",

View File

@ -4,6 +4,7 @@
"scripts": {
"start-dev": "node server/start-dev.js",
"lint": "eslint **/*.js",
"test": "npm run mocha -- test/e2e-tests.js --timeout 15000",
"nodemon": "nodemon",
"vite": "vite",
"mocha": "mocha",
@ -31,6 +32,7 @@
"matrix-bot-sdk": "^0.5.19",
"matrix-js-sdk": "^15.5.2",
"matrix-public-archive-shared": "file:./shared/",
"nconf": "^0.11.3",
"node-fetch": "^2.6.7",
"url-join": "^4.0.1"
}

View File

@ -3,9 +3,10 @@
const assert = require('assert');
const urlJoin = require('url-join');
const fetchEndpoint = require('./lib/fetch-endpoint');
const { fetchEndpointAsJson } = require('./lib/fetch-endpoint');
const { matrixServerUrl } = require('../config.json');
const config = require('./lib/config');
const matrixServerUrl = config.get('matrixServerUrl');
assert(matrixServerUrl);
async function fetchEventsForTimestamp(accessToken, roomId, ts) {
@ -13,11 +14,18 @@ async function fetchEventsForTimestamp(accessToken, roomId, ts) {
assert(roomId);
assert(ts);
// TODO: Only join world_readable rooms
const joinEndpoint = urlJoin(matrixServerUrl, `_matrix/client/r0/join/${roomId}`);
await fetchEndpointAsJson(joinEndpoint, {
method: 'POST',
accessToken,
});
const timestampToEventEndpoint = urlJoin(
matrixServerUrl,
`_matrix/client/unstable/org.matrix.msc3030/rooms/${roomId}/timestamp_to_event?ts=${ts}&dir=b`
);
const timestampToEventResData = await fetchEndpoint(timestampToEventEndpoint, {
const timestampToEventResData = await fetchEndpointAsJson(timestampToEventEndpoint, {
accessToken,
});
const eventIdForTimestamp = timestampToEventResData.event_id;
@ -28,16 +36,17 @@ async function fetchEventsForTimestamp(accessToken, roomId, ts) {
matrixServerUrl,
`_matrix/client/r0/rooms/${roomId}/context/${eventIdForTimestamp}?limit=0`
);
const contextResData = await fetchEndpoint(contextEndpoint, {
const contextResData = await fetchEndpointAsJson(contextEndpoint, {
accessToken,
});
//console.log('contextResData', contextResData);
// Add filter={"lazy_load_members":true,"include_redundant_members":true} to get member state events included
const messagesEndpoint = urlJoin(
matrixServerUrl,
`_matrix/client/r0/rooms/${roomId}/messages?from=${contextResData.start}&limit=50&filter={"lazy_load_members":true,"include_redundant_members":true}`
);
const messageResData = await fetchEndpoint(messagesEndpoint, {
const messageResData = await fetchEndpointAsJson(messagesEndpoint, {
accessToken,
});

View File

@ -3,9 +3,10 @@
const assert = require('assert');
const urlJoin = require('url-join');
const fetchEndpoint = require('./lib/fetch-endpoint');
const { fetchEndpointAsJson } = require('./lib/fetch-endpoint');
const { matrixServerUrl } = require('../config.json');
const config = require('./lib/config');
const matrixServerUrl = config.get('matrixServerUrl');
assert(matrixServerUrl);
async function fetchRoomData(accessToken, roomId) {
@ -22,10 +23,10 @@ async function fetchRoomData(accessToken, roomId) {
);
const [stateNameResDataOutcome, stateAvatarResDataOutcome] = await Promise.allSettled([
fetchEndpoint(stateNameEndpoint, {
fetchEndpointAsJson(stateNameEndpoint, {
accessToken,
}),
fetchEndpoint(stateAvatarEndpoint, {
fetchEndpointAsJson(stateAvatarEndpoint, {
accessToken,
}),
]);

41
server/lib/config.js Normal file
View File

@ -0,0 +1,41 @@
'use strict';
// This file is based off the Gitter config, https://gitlab.com/gitlab-org/gitter/env/blob/master/lib/config.js
const path = require('path');
const nconf = require('nconf');
function configureNodeEnv() {
const nodeEnv = process.env.NODE_ENV;
if (nodeEnv === 'production') {
return 'prod';
} else if (nodeEnv === 'development') {
return 'dev';
}
if (nodeEnv) return nodeEnv;
// Default to NODE_ENV=dev
process.env.NODE_ENV = 'dev';
return 'dev';
}
const nodeEnv = configureNodeEnv();
console.log('nodeEnv', nodeEnv);
const configDir = path.join(__dirname, '../../config');
nconf.argv().env('__');
nconf.add('envUser', {
type: 'file',
file: path.join(configDir, 'config.' + nodeEnv + '.user-overrides.json'),
});
// Only use user-overrides in dev
if (nodeEnv === 'dev') {
nconf.add('user', { type: 'file', file: path.join(configDir, 'config.user-overrides.json') });
}
nconf.add('nodeEnv', { type: 'file', file: path.join(configDir, 'config.' + nodeEnv + '.json') });
nconf.add('defaults', { type: 'file', file: path.join(configDir, 'config.default.json') });
module.exports = nconf;

View File

@ -22,16 +22,45 @@ const checkResponseStatus = async (response) => {
}
};
async function fetchEndpoint(endpoint, { accessToken }) {
async function fetchEndpoint(endpoint, options = {}) {
const { method, accessToken } = options;
const headers = options.headers || {};
if (accessToken) {
headers.Authorization = `Bearer ${accessToken}`;
}
const res = await fetch(endpoint, {
headers: {
'Content-Type': 'application/json',
Authorization: `Bearer ${accessToken}`,
},
method,
headers,
});
await checkResponseStatus(res);
return res;
}
async function fetchEndpointAsText(endpoint, options) {
const res = await fetchEndpoint(endpoint, options);
const data = await res.text();
return data;
}
async function fetchEndpointAsJson(endpoint, options) {
const opts = {
...options,
headers: {
'Content-Type': 'application/json',
...(options.headers || {}),
},
};
const res = await fetchEndpoint(endpoint, opts);
const data = await res.json();
return data;
}
module.exports = fetchEndpoint;
module.exports = {
fetchEndpoint,
fetchEndpointAsText,
fetchEndpointAsJson,
};

View File

@ -7,7 +7,7 @@ const { readFile } = require('fs').promises;
const crypto = require('crypto');
const { parseHTML } = require('linkedom');
const config = require('../config.json');
const config = require('./lib/config');
async function renderToString({ fromTimestamp, roomData, events, stateEventMap }) {
assert(fromTimestamp);
@ -37,7 +37,11 @@ async function renderToString({ fromTimestamp, roomData, events, stateEventMap }
roomData,
events,
stateEventMap,
config,
config: {
basePort: config.get('basePort'),
basePath: config.get('basePath'),
matrixServerUrl: config.get('matrixServerUrl'),
},
};
// Serialize it for when we run this again client-side
dom.document.body.insertAdjacentHTML(

View File

@ -11,11 +11,10 @@ const fetchRoomData = require('../fetch-room-data');
const fetchEventsForTimestamp = require('../fetch-events-for-timestamp');
const renderHydrogenToString = require('../render-hydrogen-to-string');
const config = require('../../config.json');
const basePath = config.basePath;
const config = require('../lib/config');
const basePath = config.get('basePath');
assert(basePath);
const { matrixAccessToken } = require('../../secrets.json');
const matrixAccessToken = config.get('matrixAccessToken');
assert(matrixAccessToken);
function parseArchiveRangeFromReq(req) {
@ -128,6 +127,8 @@ function installRoutes(app) {
fetchEventsForTimestamp(matrixAccessToken, roomIdOrAlias, fromTimestamp),
]);
console.log('events', JSON.stringify(events, null, 2));
const hydrogenHtmlOutput = await renderHydrogenToString({
fromTimestamp,
roomData,

View File

@ -1,8 +1,16 @@
'use strict';
console.log('server process.env.NODE_ENV', process.env.NODE_ENV);
const express = require('express');
const installRoutes = require('./routes/install-routes');
const config = require('./lib/config');
const basePort = config.get('basePort');
const app = express();
installRoutes(app);
app.listen(3050);
const server = app.listen(basePort);
module.exports = server;

View File

@ -7,6 +7,8 @@ const mergeOptions = require('merge-options');
const viteConfig = require('../vite.config');
console.log('start-dev process.env.NODE_ENV', process.env.NODE_ENV);
// Listen for any changes to files and restart the Node.js server process
//
// For API docs, see
@ -14,7 +16,6 @@ const viteConfig = require('../vite.config');
nodemon({
script: path.join(__dirname, './server.js'),
ext: 'js json',
delay: 2500,
ignoreRoot: ['.git'],
ignore: [path.join(__dirname, '../dist/*')],
});
@ -40,3 +41,5 @@ build(
},
})
);
console.log('start-dev2 process.env.NODE_ENV', process.env.NODE_ENV);

View File

@ -20,10 +20,10 @@ const {
RoomViewModel,
ViewModel,
} = require('hydrogen-view-sdk');
const urlJoin = require('url-join');
const ArchiveView = require('matrix-public-archive-shared/ArchiveView');
const RightPanelContentView = require('matrix-public-archive-shared/RightPanelContentView');
const MatrixPublicArchiveURLCreator = require('matrix-public-archive-shared/lib/url-creator');
const fromTimestamp = window.matrixPublicArchiveContext.fromTimestamp;
assert(fromTimestamp);
@ -38,6 +38,8 @@ assert(config);
assert(config.matrixServerUrl);
assert(config.basePath);
const matrixPublicArchiveURLCreator = new MatrixPublicArchiveURLCreator(config.basePath);
function addSupportClasses() {
const input = document.createElement('input');
input.type = 'month';
@ -235,11 +237,7 @@ async function mountHydrogen() {
}
archiveUrlForDate(date) {
// Gives the date in YYYY-mm-dd format.
// date.toISOString() -> 2022-02-16T23:20:04.709Z
const urlDate = date.toISOString().split('T')[0].replaceAll('-', '/');
return urlJoin(config.basePath, `${room.id}/date/${urlDate}`);
return matrixPublicArchiveURLCreator.archiveUrlForDate(room.id, date);
}
prevMonth() {

19
shared/lib/url-creator.js Normal file
View File

@ -0,0 +1,19 @@
'use strict';
const urlJoin = require('url-join');
class URLCreator {
constructor(basePath) {
this._basePath = basePath;
}
archiveUrlForDate(roomId, date) {
// Gives the date in YYYY/mm/dd format.
// date.toISOString() -> 2022-02-16T23:20:04.709Z
const urlDate = date.toISOString().split('T')[0].replaceAll('-', '/');
return urlJoin(this._basePath, `${roomId}/date/${urlDate}`);
}
}
module.exports = URLCreator;

View File

@ -1,7 +1,17 @@
```
docker build -t matrix-public-archive-test-homeserver -f test/dockerfiles/Synapse.Dockerfile test/dockerfiles/
$ docker pull matrixdotorg/synapse:latest
$ docker build -t matrix-public-archive-test-homeserver -f test/dockerfiles/Synapse.Dockerfile test/dockerfiles/
```
```
docker-compose -f test/docker-compose.yml up -d --no-recreate
```
```
$ docker ps --all | grep test_hs
$ docker logs test_hs1_1
$ docker logs test_hs2_1
$ docker stop test_hs1_1 test_hs2_1
$ docker rm test_hs1_1 test_hs2_1
```

View File

@ -0,0 +1,18 @@
id: 6527ecdd6b8fe61c645d9d412222d75b
hs_token: hs_token_1f5d7675517072f0c0b5b684ca23f9ffad5a0cfc32e1ca824c0600ee74e105d7
# We use this as a consistent acccess token to use in our tests
as_token: as_token_8664700429a911bbbecf7d91b9e1a74716d669f40cf32259630e38439726e29d
# This doesn't relate to anything else
url: 'http://localhost:0000'
sender_localpart: archiver
namespaces:
users:
- exclusive: false
regex: .*
aliases:
- exclusive: false
regex: .*
rooms:
- exclusive: false
regex: .*
rate_limited: false

View File

@ -94,6 +94,14 @@ rc_joins:
federation_rr_transactions_per_room_per_second: 9999
## API Configuration ##
# A list of application service config files to use
#
app_service_config_files:
# We use this to provide a constant matrixAccessToken for the tests
- /conf/as_registration.yaml
## Experimental Features ##
experimental_features:

View File

@ -1,20 +1,36 @@
'use strict';
process.env.NODE_ENV = 'test';
const assert = require('assert');
const urlJoin = require('url-join');
const { MatrixAuth } = require('matrix-bot-sdk');
const fetchEndpoint = require('../server/lib/fetch-endpoint');
const server = require('../server/server');
const { fetchEndpointAsText, fetchEndpointAsJson } = require('../server/lib/fetch-endpoint');
const config = require('../config');
assert(config.testMatrixServerUrl1);
assert(config.testMatrixServerUrl2);
const MatrixPublicArchiveURLCreator = require('matrix-public-archive-shared/lib/url-creator');
const config = require('../server/lib/config');
const testMatrixServerUrl1 = config.get('testMatrixServerUrl1');
const testMatrixServerUrl2 = config.get('testMatrixServerUrl2');
const basePath = config.get('basePath');
assert(testMatrixServerUrl1);
assert(testMatrixServerUrl2);
assert(basePath);
const matrixPublicArchiveURLCreator = new MatrixPublicArchiveURLCreator(basePath);
const HOMESERVER_URL_TO_PRETTY_NAME_MAP = {
[testMatrixServerUrl1]: 'hs1',
[testMatrixServerUrl2]: 'hs2',
};
async function getTestClientForHs(testMatrixServerUrl) {
const auth = new MatrixAuth(testMatrixServerUrl);
const client = await auth.passwordRegister(
`user-${Math.floor(Math.random() * 1000000000)}`,
`user-t${new Date().getTime()}-r${Math.floor(Math.random() * 1000000000)}`,
'password'
);
@ -38,58 +54,175 @@ async function createTestRoom(client) {
const roomId = await client.createRoom({
preset: 'public_chat',
name: 'the hangout spot',
initial_state: [
{
type: 'm.room.history_visibility',
state_key: '',
content: {
history_visibility: 'world_readable',
},
},
],
});
return roomId;
}
describe('matrix-public-archive', () => {
it('Test federation between fixture homeservers', async () => {
const hs1Client = await getTestClientForHs(config.testMatrixServerUrl1);
const hs2Client = await getTestClientForHs(config.testMatrixServerUrl2);
// Create a room on hs2
const hs2RoomId = await createTestRoom(hs2Client);
const room2EventIds = await createMessagesInRoom(
hs2Client,
hs2RoomId,
10,
hs2Client.homeserverUrl
);
// Join hs1 to a room on hs2 (federation)
await hs1Client.joinRoom(hs2RoomId, 'hs2');
// From, hs1, make sure we can fetch messages from hs2
const messagesEndpoint = urlJoin(
hs1Client.homeserverUrl,
`_matrix/client/r0/rooms/${hs2RoomId}/messages?limit=5&dir=b&filter={"types":["m.room.message"]}`
);
const messageResData = await fetchEndpoint(messagesEndpoint, {
accessToken: hs1Client.accessToken,
});
// Make sure it returned some messages
assert.strictEqual(messageResData.chunk.length, 5);
// Make sure all of the messages belong to the room
messageResData.chunk.map((event) => {
const isEventInRoomFromHs2 = room2EventIds.some((room2EventId) => {
return room2EventId === event.event_id;
});
// Make sure the message belongs to the room on hs2
assert.strictEqual(
isEventInRoomFromHs2,
true,
`Expected ${event.event_id} (${event.type}: "${
event.content.body
}") to be in room on hs2=${JSON.stringify(room2EventIds)}`
);
});
after(() => {
//server.close();
});
it('can render diverse messages');
it('Test federation between fixture homeservers', async () => {
try {
const hs1Client = await getTestClientForHs(testMatrixServerUrl1);
const hs2Client = await getTestClientForHs(testMatrixServerUrl2);
// Create a room on hs2
const hs2RoomId = await createTestRoom(hs2Client);
const room2EventIds = await createMessagesInRoom(
hs2Client,
hs2RoomId,
10,
HOMESERVER_URL_TO_PRETTY_NAME_MAP[hs2Client.homeserverUrl]
);
// Join hs1 to a room on hs2 (federation)
await hs1Client.joinRoom(hs2RoomId, 'hs2');
// From, hs1, make sure we can fetch messages from hs2
const messagesEndpoint = urlJoin(
hs1Client.homeserverUrl,
`_matrix/client/r0/rooms/${hs2RoomId}/messages?limit=5&dir=b&filter={"types":["m.room.message"]}`
);
const messageResData = await fetchEndpointAsJson(messagesEndpoint, {
accessToken: hs1Client.accessToken,
});
// Make sure it returned some messages
assert.strictEqual(messageResData.chunk.length, 5);
// Make sure all of the messages belong to the room
messageResData.chunk.map((event) => {
const isEventInRoomFromHs2 = room2EventIds.some((room2EventId) => {
return room2EventId === event.event_id;
});
// Make sure the message belongs to the room on hs2
assert.strictEqual(
isEventInRoomFromHs2,
true,
`Expected ${event.event_id} (${event.type}: "${
event.content.body
}") to be in room on hs2=${JSON.stringify(room2EventIds)}`
);
});
} catch (err) {
if (err.body) {
// FIXME: Remove this try/catch once the matrix-bot-sdk no longer throws
// huge response objects as errors, see
// https://github.com/turt2live/matrix-bot-sdk/pull/158
throw new Error(
`Error occured in matrix-bot-sdk (this new error is to stop it from logging the huge response) statusCode=${
err.statusCode
} body=${JSON.stringify(err.body)}`
);
}
throw err;
}
});
it('can render diverse messages', async () => {
try {
const client = await getTestClientForHs(testMatrixServerUrl1);
const roomId = await createTestRoom(client);
// TODO: Set avatar of user
// TODO: Set avatar of room
// Test image
const mxcUri = await client.uploadContentFromUrl(
'https://en.wikipedia.org/wiki/Friction#/media/File:Friction_between_surfaces.jpg'
);
await client.sendMessage(roomId, {
body: 'Friction_between_surfaces.jpeg',
info: {
size: 396644,
mimetype: 'image/jpeg',
w: 1894,
h: 925,
'xyz.amorgan.blurhash': 'LkR3G|IU?w%NbwbIemae_NxuD$M{',
},
msgtype: 'm.image',
url: mxcUri,
});
// A normal text message
await client.sendMessage(roomId, {
msgtype: 'm.text',
body: '^ Figure 1: Simulated blocks with fractal rough surfaces, exhibiting static frictional interactions',
});
// A normal text message
await client.sendMessage(roomId, {
msgtype: 'm.text',
body: 'The topography of the Moon has been measured with laser altimetry and stereo image analysis.',
});
// Test replies
const eventToReplyTo = await client.sendMessage(roomId, {
'org.matrix.msc1767.message': [
{
body: "> <@ericgittertester:my.synapse.server> The topography of the Moon has been measured with laser altimetry and stereo image analysis.\n\nThe concentration of maria on the near side likely reflects the substantially thicker crust of the highlands of the Far Side, which may have formed in a slow-velocity impact of a second moon of Earth a few tens of millions of years after the Moon's formation.",
mimetype: 'text/plain',
},
{
body: '<mx-reply><blockquote><a href="https://matrix.to/#/!HBehERstyQBxyJDLfR:my.synapse.server/$uEeScM2gfILkLpG8sOBTK7vcS0w_t3a9EVIAnSwqyiY?via=my.synapse.server">In reply to</a> <a href="https://matrix.to/#/@ericgittertester:my.synapse.server">@ericgittertester:my.synapse.server</a><br>The topography of the Moon has been measured with laser altimetry and stereo image analysis.</blockquote></mx-reply>The concentration of maria on the near side likely reflects the substantially thicker crust of the highlands of the Far Side, which may have formed in a slow-velocity impact of a second moon of Earth a few tens of millions of years after the Moon\'s formation.',
mimetype: 'text/html',
},
],
body: "> <@ericgittertester:my.synapse.server> The topography of the Moon has been measured with laser altimetry and stereo image analysis.\n\nThe concentration of maria on the near side likely reflects the substantially thicker crust of the highlands of the Far Side, which may have formed in a slow-velocity impact of a second moon of Earth a few tens of millions of years after the Moon's formation.",
msgtype: 'm.text',
format: 'org.matrix.custom.html',
formatted_body:
'<mx-reply><blockquote><a href="https://matrix.to/#/!HBehERstyQBxyJDLfR:my.synapse.server/$uEeScM2gfILkLpG8sOBTK7vcS0w_t3a9EVIAnSwqyiY?via=my.synapse.server">In reply to</a> <a href="https://matrix.to/#/@ericgittertester:my.synapse.server">@ericgittertester:my.synapse.server</a><br>The topography of the Moon has been measured with laser altimetry and stereo image analysis.</blockquote></mx-reply>The concentration of maria on the near side likely reflects the substantially thicker crust of the highlands of the Far Side, which may have formed in a slow-velocity impact of a second moon of Earth a few tens of millions of years after the Moon\'s formation.',
'm.relates_to': {
'm.in_reply_to': {
event_id: '$uEeScM2gfILkLpG8sOBTK7vcS0w_t3a9EVIAnSwqyiY',
},
},
});
// Test reactions
await client.sendEvent(roomId, 'm.reaction', {
'm.relates_to': {
rel_type: 'm.annotation',
event_id: eventToReplyTo,
key: '<27><>',
},
});
const archiveUrl = matrixPublicArchiveURLCreator.archiveUrlForDate(roomId, new Date());
const archivePageHtml = await fetchEndpointAsText(archiveUrl);
console.log('archivePageHtml', archivePageHtml);
} catch (err) {
if (err.body) {
// FIXME: Remove this try/catch once the matrix-bot-sdk no longer throws
// huge response objects as errors, see
// https://github.com/turt2live/matrix-bot-sdk/pull/158
throw new Error(
`Error occured in matrix-bot-sdk (this new error is to stop it from logging the huge response) statusCode=${
err.statusCode
} body=${JSON.stringify(err.body)}`
);
}
throw err;
}
});
it(`can render day back in time from room on remote homeserver we haven't backfilled from`);

View File

@ -5,6 +5,10 @@ const path = require('path');
const { defineConfig } = require('vite');
module.exports = defineConfig({
// We have to specify this otherwise Vite will override NODE_ENV as
// `production` when we start the server and watch build in our `start-dev.js`.
mode: process.env.NODE_ENV || 'dev',
//root: './',
//base: './',
outDir: './dist',