diff --git a/.gitignore b/.gitignore index 9180642..c4b29a0 100644 --- a/.gitignore +++ b/.gitignore @@ -3,5 +3,10 @@ node_modules dist dist-ssr *.local -secrets.json + +config.user-overrides.json +config/config.dev.user-overrides.json +config/config.beta.user-overrides.json +config/config.prod.user-overrides.json config.json +secrets.json diff --git a/README.md b/README.md index aa39c8e..bfd7370 100644 --- a/README.md +++ b/README.md @@ -5,11 +5,12 @@ config/secrets, using a draft branch of Hydrogen, etc. In the vein of [feature parity with Gitter](https://github.com/vector-im/roadmap/issues/26), the goal is to make a -public archive site like Gitter's archives which search engines can index and -keep all of the content accessible/available. There is already -https://view.matrix.org/ (https://github.com/matrix-org/matrix-static) but there -is some desire to make something with more Element-feeling polish and loading -faster (avoid the slow 502's errors that are frequent on `view.matrix.org`). +public archive site for `world_readable` Matrix rooms like Gitter's archives +which search engines can index and keep all of the content accessible/available. +There is already https://view.matrix.org/ +(https://github.com/matrix-org/matrix-static) but there is some desire to make +something with more Element-feeling polish and loading faster (avoid the slow +502's errors that are frequent on `view.matrix.org`). ## Plan summary diff --git a/config/config.default.json b/config/config.default.json new file mode 100644 index 0000000..2f6fda7 --- /dev/null +++ b/config/config.default.json @@ -0,0 +1,10 @@ +{ + "basePort": "3050", + "basePath": "http://localhost:3050", + "matrixServerUrl": "http://localhost:8008/", + "testMatrixServerUrl1": "http://localhost:11008/", + "testMatrixServerUrl2": "http://localhost:12008/", + + "Secrets": "Remove the 'xxx__' prefix and fill these in your config/config.user-overrides.json or by environment variable", + "xxx__matrixAccessToken": "xxx" +} diff --git a/config/config.test.json b/config/config.test.json new file mode 100644 index 0000000..115dcf5 --- /dev/null +++ b/config/config.test.json @@ -0,0 +1,10 @@ +{ + "basePort": "3051", + "basePath": "http://localhost:3051", + "matrixServerUrl": "http://localhost:11008/", + "testMatrixServerUrl1": "http://localhost:11008/", + "testMatrixServerUrl2": "http://localhost:12008/", + + "Secrets": "xxx", + "matrixAccessToken": "as_token_8664700429a911bbbecf7d91b9e1a74716d669f40cf32259630e38439726e29d" +} diff --git a/package-lock.json b/package-lock.json index 4a7308f..cb000aa 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,6 +14,7 @@ "matrix-bot-sdk": "^0.5.19", "matrix-js-sdk": "^15.5.2", "matrix-public-archive-shared": "file:./shared/", + "nconf": "^0.11.3", "node-fetch": "^2.6.7", "url-join": "^4.0.1" }, @@ -324,7 +325,6 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true, "engines": { "node": ">=8" } @@ -383,6 +383,11 @@ "node": ">=0.8" } }, + "node_modules/async": { + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz", + "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=" + }, "node_modules/asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -704,7 +709,6 @@ "version": "7.0.4", "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz", "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==", - "dev": true, "dependencies": { "string-width": "^4.2.0", "strip-ansi": "^6.0.0", @@ -1066,8 +1070,7 @@ "node_modules/emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "dev": true + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" }, "node_modules/encodeurl": { "version": "1.0.2", @@ -1436,7 +1439,6 @@ "version": "3.1.1", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==", - "dev": true, "engines": { "node": ">=6" } @@ -2007,7 +2009,6 @@ "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", - "dev": true, "engines": { "node": "6.* || 8.* || >= 10.*" } @@ -2409,7 +2410,6 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/ini/-/ini-2.0.0.tgz", "integrity": "sha512-7PnF4oN3CvZF23ADhA5wRaYEQpJ8qygSkbtTXWBeXWXmEVRXK+1ITciHWwHhsjv1TmW0MgacIv6hEi5pX5NQdA==", - "dev": true, "engines": { "node": ">=10" } @@ -2471,7 +2471,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "dev": true, "engines": { "node": ">=8" } @@ -3119,6 +3118,20 @@ "integrity": "sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=", "dev": true }, + "node_modules/nconf": { + "version": "0.11.3", + "resolved": "https://registry.npmjs.org/nconf/-/nconf-0.11.3.tgz", + "integrity": "sha512-iYsAuDS9pzjVMGIzJrGE0Vk3Eh8r/suJanRAnWGBd29rVS2XtSgzcAo5l6asV3e4hH2idVONHirg1efoBOslBg==", + "dependencies": { + "async": "^1.4.0", + "ini": "^2.0.0", + "secure-keys": "^1.0.0", + "yargs": "^16.1.1" + }, + "engines": { + "node": ">= 0.4.0" + } + }, "node_modules/negotiator": { "version": "0.6.3", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", @@ -3788,7 +3801,6 @@ "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=", - "dev": true, "engines": { "node": ">=0.10.0" } @@ -3921,6 +3933,11 @@ "entities": "^2.0.0" } }, + "node_modules/secure-keys": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/secure-keys/-/secure-keys-1.0.0.tgz", + "integrity": "sha1-8MgtmKOxOah3aogIBQuCRDEIf8o=" + }, "node_modules/semver": { "version": "6.3.0", "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", @@ -4085,7 +4102,6 @@ "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dev": true, "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", @@ -4099,7 +4115,6 @@ "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, "dependencies": { "ansi-regex": "^5.0.1" }, @@ -4512,7 +4527,6 @@ "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "dev": true, "dependencies": { "ansi-styles": "^4.0.0", "string-width": "^4.1.0", @@ -4556,7 +4570,6 @@ "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", - "dev": true, "engines": { "node": ">=10" } @@ -4570,7 +4583,6 @@ "version": "16.2.0", "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz", "integrity": "sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==", - "dev": true, "dependencies": { "cliui": "^7.0.2", "escalade": "^3.1.1", @@ -4588,7 +4600,6 @@ "version": "20.2.4", "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.4.tgz", "integrity": "sha512-WOkpgNhPTlE73h4VFAFsOnomJVaovO8VqLDzy5saChRBFQFBoMYirowyW+Q9HB4HFF4Z7VZTiG3iSzJJA29yRA==", - "dev": true, "engines": { "node": ">=10" } @@ -4864,8 +4875,7 @@ "ansi-regex": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==" }, "ansi-styles": { "version": "4.3.0", @@ -4909,6 +4919,11 @@ "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" }, + "async": { + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz", + "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=" + }, "asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -5159,7 +5174,6 @@ "version": "7.0.4", "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz", "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==", - "dev": true, "requires": { "string-width": "^4.2.0", "strip-ansi": "^6.0.0", @@ -5437,8 +5451,7 @@ "emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "dev": true + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" }, "encodeurl": { "version": "1.0.2", @@ -5622,8 +5635,7 @@ "escalade": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", - "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==", - "dev": true + "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==" }, "escape-goat": { "version": "2.1.1", @@ -6040,8 +6052,7 @@ "get-caller-file": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", - "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", - "dev": true + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==" }, "get-stream": { "version": "4.1.0", @@ -6344,8 +6355,7 @@ "ini": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ini/-/ini-2.0.0.tgz", - "integrity": "sha512-7PnF4oN3CvZF23ADhA5wRaYEQpJ8qygSkbtTXWBeXWXmEVRXK+1ITciHWwHhsjv1TmW0MgacIv6hEi5pX5NQdA==", - "dev": true + "integrity": "sha512-7PnF4oN3CvZF23ADhA5wRaYEQpJ8qygSkbtTXWBeXWXmEVRXK+1ITciHWwHhsjv1TmW0MgacIv6hEi5pX5NQdA==" }, "ipaddr.js": { "version": "1.9.1", @@ -6388,8 +6398,7 @@ "is-fullwidth-code-point": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "dev": true + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==" }, "is-glob": { "version": "4.0.3", @@ -6875,6 +6884,17 @@ "integrity": "sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=", "dev": true }, + "nconf": { + "version": "0.11.3", + "resolved": "https://registry.npmjs.org/nconf/-/nconf-0.11.3.tgz", + "integrity": "sha512-iYsAuDS9pzjVMGIzJrGE0Vk3Eh8r/suJanRAnWGBd29rVS2XtSgzcAo5l6asV3e4hH2idVONHirg1efoBOslBg==", + "requires": { + "async": "^1.4.0", + "ini": "^2.0.0", + "secure-keys": "^1.0.0", + "yargs": "^16.1.1" + } + }, "negotiator": { "version": "0.6.3", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", @@ -7359,8 +7379,7 @@ "require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", - "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=", - "dev": true + "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=" }, "resolve": { "version": "1.22.0", @@ -7447,6 +7466,11 @@ } } }, + "secure-keys": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/secure-keys/-/secure-keys-1.0.0.tgz", + "integrity": "sha1-8MgtmKOxOah3aogIBQuCRDEIf8o=" + }, "semver": { "version": "6.3.0", "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", @@ -7578,7 +7602,6 @@ "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dev": true, "requires": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", @@ -7589,7 +7612,6 @@ "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, "requires": { "ansi-regex": "^5.0.1" } @@ -7886,7 +7908,6 @@ "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "dev": true, "requires": { "ansi-styles": "^4.0.0", "string-width": "^4.1.0", @@ -7920,8 +7941,7 @@ "y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", - "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", - "dev": true + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==" }, "yallist": { "version": "4.0.0", @@ -7932,7 +7952,6 @@ "version": "16.2.0", "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz", "integrity": "sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==", - "dev": true, "requires": { "cliui": "^7.0.2", "escalade": "^3.1.1", @@ -7946,8 +7965,7 @@ "yargs-parser": { "version": "20.2.4", "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.4.tgz", - "integrity": "sha512-WOkpgNhPTlE73h4VFAFsOnomJVaovO8VqLDzy5saChRBFQFBoMYirowyW+Q9HB4HFF4Z7VZTiG3iSzJJA29yRA==", - "dev": true + "integrity": "sha512-WOkpgNhPTlE73h4VFAFsOnomJVaovO8VqLDzy5saChRBFQFBoMYirowyW+Q9HB4HFF4Z7VZTiG3iSzJJA29yRA==" }, "yargs-unparser": { "version": "2.0.0", diff --git a/package.json b/package.json index aa374e5..9570a9b 100644 --- a/package.json +++ b/package.json @@ -4,6 +4,7 @@ "scripts": { "start-dev": "node server/start-dev.js", "lint": "eslint **/*.js", + "test": "npm run mocha -- test/e2e-tests.js --timeout 15000", "nodemon": "nodemon", "vite": "vite", "mocha": "mocha", @@ -31,6 +32,7 @@ "matrix-bot-sdk": "^0.5.19", "matrix-js-sdk": "^15.5.2", "matrix-public-archive-shared": "file:./shared/", + "nconf": "^0.11.3", "node-fetch": "^2.6.7", "url-join": "^4.0.1" } diff --git a/server/fetch-events-for-timestamp.js b/server/fetch-events-for-timestamp.js index bb6b53f..96dcd79 100644 --- a/server/fetch-events-for-timestamp.js +++ b/server/fetch-events-for-timestamp.js @@ -3,9 +3,10 @@ const assert = require('assert'); const urlJoin = require('url-join'); -const fetchEndpoint = require('./lib/fetch-endpoint'); +const { fetchEndpointAsJson } = require('./lib/fetch-endpoint'); -const { matrixServerUrl } = require('../config.json'); +const config = require('./lib/config'); +const matrixServerUrl = config.get('matrixServerUrl'); assert(matrixServerUrl); async function fetchEventsForTimestamp(accessToken, roomId, ts) { @@ -13,11 +14,18 @@ async function fetchEventsForTimestamp(accessToken, roomId, ts) { assert(roomId); assert(ts); + // TODO: Only join world_readable rooms + const joinEndpoint = urlJoin(matrixServerUrl, `_matrix/client/r0/join/${roomId}`); + await fetchEndpointAsJson(joinEndpoint, { + method: 'POST', + accessToken, + }); + const timestampToEventEndpoint = urlJoin( matrixServerUrl, `_matrix/client/unstable/org.matrix.msc3030/rooms/${roomId}/timestamp_to_event?ts=${ts}&dir=b` ); - const timestampToEventResData = await fetchEndpoint(timestampToEventEndpoint, { + const timestampToEventResData = await fetchEndpointAsJson(timestampToEventEndpoint, { accessToken, }); const eventIdForTimestamp = timestampToEventResData.event_id; @@ -28,16 +36,17 @@ async function fetchEventsForTimestamp(accessToken, roomId, ts) { matrixServerUrl, `_matrix/client/r0/rooms/${roomId}/context/${eventIdForTimestamp}?limit=0` ); - const contextResData = await fetchEndpoint(contextEndpoint, { + const contextResData = await fetchEndpointAsJson(contextEndpoint, { accessToken, }); //console.log('contextResData', contextResData); + // Add filter={"lazy_load_members":true,"include_redundant_members":true} to get member state events included const messagesEndpoint = urlJoin( matrixServerUrl, `_matrix/client/r0/rooms/${roomId}/messages?from=${contextResData.start}&limit=50&filter={"lazy_load_members":true,"include_redundant_members":true}` ); - const messageResData = await fetchEndpoint(messagesEndpoint, { + const messageResData = await fetchEndpointAsJson(messagesEndpoint, { accessToken, }); diff --git a/server/fetch-room-data.js b/server/fetch-room-data.js index 25a9fd2..7bf03b6 100644 --- a/server/fetch-room-data.js +++ b/server/fetch-room-data.js @@ -3,9 +3,10 @@ const assert = require('assert'); const urlJoin = require('url-join'); -const fetchEndpoint = require('./lib/fetch-endpoint'); +const { fetchEndpointAsJson } = require('./lib/fetch-endpoint'); -const { matrixServerUrl } = require('../config.json'); +const config = require('./lib/config'); +const matrixServerUrl = config.get('matrixServerUrl'); assert(matrixServerUrl); async function fetchRoomData(accessToken, roomId) { @@ -22,10 +23,10 @@ async function fetchRoomData(accessToken, roomId) { ); const [stateNameResDataOutcome, stateAvatarResDataOutcome] = await Promise.allSettled([ - fetchEndpoint(stateNameEndpoint, { + fetchEndpointAsJson(stateNameEndpoint, { accessToken, }), - fetchEndpoint(stateAvatarEndpoint, { + fetchEndpointAsJson(stateAvatarEndpoint, { accessToken, }), ]); diff --git a/server/lib/config.js b/server/lib/config.js new file mode 100644 index 0000000..e2d3cd0 --- /dev/null +++ b/server/lib/config.js @@ -0,0 +1,41 @@ +'use strict'; + +// This file is based off the Gitter config, https://gitlab.com/gitlab-org/gitter/env/blob/master/lib/config.js + +const path = require('path'); +const nconf = require('nconf'); + +function configureNodeEnv() { + const nodeEnv = process.env.NODE_ENV; + if (nodeEnv === 'production') { + return 'prod'; + } else if (nodeEnv === 'development') { + return 'dev'; + } + if (nodeEnv) return nodeEnv; + + // Default to NODE_ENV=dev + process.env.NODE_ENV = 'dev'; + return 'dev'; +} + +const nodeEnv = configureNodeEnv(); +console.log('nodeEnv', nodeEnv); +const configDir = path.join(__dirname, '../../config'); + +nconf.argv().env('__'); + +nconf.add('envUser', { + type: 'file', + file: path.join(configDir, 'config.' + nodeEnv + '.user-overrides.json'), +}); + +// Only use user-overrides in dev +if (nodeEnv === 'dev') { + nconf.add('user', { type: 'file', file: path.join(configDir, 'config.user-overrides.json') }); +} + +nconf.add('nodeEnv', { type: 'file', file: path.join(configDir, 'config.' + nodeEnv + '.json') }); +nconf.add('defaults', { type: 'file', file: path.join(configDir, 'config.default.json') }); + +module.exports = nconf; diff --git a/server/lib/fetch-endpoint.js b/server/lib/fetch-endpoint.js index 6dae615..b2358ba 100644 --- a/server/lib/fetch-endpoint.js +++ b/server/lib/fetch-endpoint.js @@ -22,16 +22,45 @@ const checkResponseStatus = async (response) => { } }; -async function fetchEndpoint(endpoint, { accessToken }) { +async function fetchEndpoint(endpoint, options = {}) { + const { method, accessToken } = options; + const headers = options.headers || {}; + + if (accessToken) { + headers.Authorization = `Bearer ${accessToken}`; + } + const res = await fetch(endpoint, { - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${accessToken}`, - }, + method, + headers, }); await checkResponseStatus(res); + + return res; +} + +async function fetchEndpointAsText(endpoint, options) { + const res = await fetchEndpoint(endpoint, options); + const data = await res.text(); + return data; +} + +async function fetchEndpointAsJson(endpoint, options) { + const opts = { + ...options, + headers: { + 'Content-Type': 'application/json', + ...(options.headers || {}), + }, + }; + + const res = await fetchEndpoint(endpoint, opts); const data = await res.json(); return data; } -module.exports = fetchEndpoint; +module.exports = { + fetchEndpoint, + fetchEndpointAsText, + fetchEndpointAsJson, +}; diff --git a/server/render-hydrogen-to-string.js b/server/render-hydrogen-to-string.js index 5a47c3d..e5fbaed 100644 --- a/server/render-hydrogen-to-string.js +++ b/server/render-hydrogen-to-string.js @@ -7,7 +7,7 @@ const { readFile } = require('fs').promises; const crypto = require('crypto'); const { parseHTML } = require('linkedom'); -const config = require('../config.json'); +const config = require('./lib/config'); async function renderToString({ fromTimestamp, roomData, events, stateEventMap }) { assert(fromTimestamp); @@ -37,7 +37,11 @@ async function renderToString({ fromTimestamp, roomData, events, stateEventMap } roomData, events, stateEventMap, - config, + config: { + basePort: config.get('basePort'), + basePath: config.get('basePath'), + matrixServerUrl: config.get('matrixServerUrl'), + }, }; // Serialize it for when we run this again client-side dom.document.body.insertAdjacentHTML( diff --git a/server/routes/install-routes.js b/server/routes/install-routes.js index c2d3d1a..dc41ace 100644 --- a/server/routes/install-routes.js +++ b/server/routes/install-routes.js @@ -11,11 +11,10 @@ const fetchRoomData = require('../fetch-room-data'); const fetchEventsForTimestamp = require('../fetch-events-for-timestamp'); const renderHydrogenToString = require('../render-hydrogen-to-string'); -const config = require('../../config.json'); -const basePath = config.basePath; +const config = require('../lib/config'); +const basePath = config.get('basePath'); assert(basePath); - -const { matrixAccessToken } = require('../../secrets.json'); +const matrixAccessToken = config.get('matrixAccessToken'); assert(matrixAccessToken); function parseArchiveRangeFromReq(req) { @@ -128,6 +127,8 @@ function installRoutes(app) { fetchEventsForTimestamp(matrixAccessToken, roomIdOrAlias, fromTimestamp), ]); + console.log('events', JSON.stringify(events, null, 2)); + const hydrogenHtmlOutput = await renderHydrogenToString({ fromTimestamp, roomData, diff --git a/server/server.js b/server/server.js index 075dc61..489492f 100644 --- a/server/server.js +++ b/server/server.js @@ -1,8 +1,16 @@ 'use strict'; +console.log('server process.env.NODE_ENV', process.env.NODE_ENV); + const express = require('express'); + const installRoutes = require('./routes/install-routes'); +const config = require('./lib/config'); +const basePort = config.get('basePort'); const app = express(); installRoutes(app); -app.listen(3050); + +const server = app.listen(basePort); + +module.exports = server; diff --git a/server/start-dev.js b/server/start-dev.js index fe2a17d..982bd2a 100644 --- a/server/start-dev.js +++ b/server/start-dev.js @@ -7,6 +7,8 @@ const mergeOptions = require('merge-options'); const viteConfig = require('../vite.config'); +console.log('start-dev process.env.NODE_ENV', process.env.NODE_ENV); + // Listen for any changes to files and restart the Node.js server process // // For API docs, see @@ -14,7 +16,6 @@ const viteConfig = require('../vite.config'); nodemon({ script: path.join(__dirname, './server.js'), ext: 'js json', - delay: 2500, ignoreRoot: ['.git'], ignore: [path.join(__dirname, '../dist/*')], }); @@ -40,3 +41,5 @@ build( }, }) ); + +console.log('start-dev2 process.env.NODE_ENV', process.env.NODE_ENV); diff --git a/shared/hydrogen-vm-render-script.js b/shared/hydrogen-vm-render-script.js index 2c916dc..82ee568 100644 --- a/shared/hydrogen-vm-render-script.js +++ b/shared/hydrogen-vm-render-script.js @@ -20,10 +20,10 @@ const { RoomViewModel, ViewModel, } = require('hydrogen-view-sdk'); -const urlJoin = require('url-join'); const ArchiveView = require('matrix-public-archive-shared/ArchiveView'); const RightPanelContentView = require('matrix-public-archive-shared/RightPanelContentView'); +const MatrixPublicArchiveURLCreator = require('matrix-public-archive-shared/lib/url-creator'); const fromTimestamp = window.matrixPublicArchiveContext.fromTimestamp; assert(fromTimestamp); @@ -38,6 +38,8 @@ assert(config); assert(config.matrixServerUrl); assert(config.basePath); +const matrixPublicArchiveURLCreator = new MatrixPublicArchiveURLCreator(config.basePath); + function addSupportClasses() { const input = document.createElement('input'); input.type = 'month'; @@ -235,11 +237,7 @@ async function mountHydrogen() { } archiveUrlForDate(date) { - // Gives the date in YYYY-mm-dd format. - // date.toISOString() -> 2022-02-16T23:20:04.709Z - const urlDate = date.toISOString().split('T')[0].replaceAll('-', '/'); - - return urlJoin(config.basePath, `${room.id}/date/${urlDate}`); + return matrixPublicArchiveURLCreator.archiveUrlForDate(room.id, date); } prevMonth() { diff --git a/shared/lib/url-creator.js b/shared/lib/url-creator.js new file mode 100644 index 0000000..7057a34 --- /dev/null +++ b/shared/lib/url-creator.js @@ -0,0 +1,19 @@ +'use strict'; + +const urlJoin = require('url-join'); + +class URLCreator { + constructor(basePath) { + this._basePath = basePath; + } + + archiveUrlForDate(roomId, date) { + // Gives the date in YYYY/mm/dd format. + // date.toISOString() -> 2022-02-16T23:20:04.709Z + const urlDate = date.toISOString().split('T')[0].replaceAll('-', '/'); + + return urlJoin(this._basePath, `${roomId}/date/${urlDate}`); + } +} + +module.exports = URLCreator; diff --git a/test/README.md b/test/README.md index e1cc5e8..344576e 100644 --- a/test/README.md +++ b/test/README.md @@ -1,7 +1,17 @@ ``` -docker build -t matrix-public-archive-test-homeserver -f test/dockerfiles/Synapse.Dockerfile test/dockerfiles/ +$ docker pull matrixdotorg/synapse:latest +$ docker build -t matrix-public-archive-test-homeserver -f test/dockerfiles/Synapse.Dockerfile test/dockerfiles/ ``` ``` docker-compose -f test/docker-compose.yml up -d --no-recreate ``` + +``` +$ docker ps --all | grep test_hs +$ docker logs test_hs1_1 +$ docker logs test_hs2_1 + +$ docker stop test_hs1_1 test_hs2_1 +$ docker rm test_hs1_1 test_hs2_1 +``` diff --git a/test/dockerfiles/synapse/as_registration.yaml b/test/dockerfiles/synapse/as_registration.yaml new file mode 100644 index 0000000..8b6a612 --- /dev/null +++ b/test/dockerfiles/synapse/as_registration.yaml @@ -0,0 +1,18 @@ +id: 6527ecdd6b8fe61c645d9d412222d75b +hs_token: hs_token_1f5d7675517072f0c0b5b684ca23f9ffad5a0cfc32e1ca824c0600ee74e105d7 +# We use this as a consistent acccess token to use in our tests +as_token: as_token_8664700429a911bbbecf7d91b9e1a74716d669f40cf32259630e38439726e29d +# This doesn't relate to anything else +url: 'http://localhost:0000' +sender_localpart: archiver +namespaces: + users: + - exclusive: false + regex: .* + aliases: + - exclusive: false + regex: .* + rooms: + - exclusive: false + regex: .* +rate_limited: false diff --git a/test/dockerfiles/synapse/homeserver.yaml b/test/dockerfiles/synapse/homeserver.yaml index 56b758a..a1b36e0 100644 --- a/test/dockerfiles/synapse/homeserver.yaml +++ b/test/dockerfiles/synapse/homeserver.yaml @@ -94,6 +94,14 @@ rc_joins: federation_rr_transactions_per_room_per_second: 9999 +## API Configuration ## + +# A list of application service config files to use +# +app_service_config_files: + # We use this to provide a constant matrixAccessToken for the tests + - /conf/as_registration.yaml + ## Experimental Features ## experimental_features: diff --git a/test/e2e-tests.js b/test/e2e-tests.js index dc19eca..b1b16c5 100644 --- a/test/e2e-tests.js +++ b/test/e2e-tests.js @@ -1,20 +1,36 @@ 'use strict'; +process.env.NODE_ENV = 'test'; + const assert = require('assert'); const urlJoin = require('url-join'); const { MatrixAuth } = require('matrix-bot-sdk'); -const fetchEndpoint = require('../server/lib/fetch-endpoint'); +const server = require('../server/server'); +const { fetchEndpointAsText, fetchEndpointAsJson } = require('../server/lib/fetch-endpoint'); -const config = require('../config'); -assert(config.testMatrixServerUrl1); -assert(config.testMatrixServerUrl2); +const MatrixPublicArchiveURLCreator = require('matrix-public-archive-shared/lib/url-creator'); + +const config = require('../server/lib/config'); +const testMatrixServerUrl1 = config.get('testMatrixServerUrl1'); +const testMatrixServerUrl2 = config.get('testMatrixServerUrl2'); +const basePath = config.get('basePath'); +assert(testMatrixServerUrl1); +assert(testMatrixServerUrl2); +assert(basePath); + +const matrixPublicArchiveURLCreator = new MatrixPublicArchiveURLCreator(basePath); + +const HOMESERVER_URL_TO_PRETTY_NAME_MAP = { + [testMatrixServerUrl1]: 'hs1', + [testMatrixServerUrl2]: 'hs2', +}; async function getTestClientForHs(testMatrixServerUrl) { const auth = new MatrixAuth(testMatrixServerUrl); const client = await auth.passwordRegister( - `user-${Math.floor(Math.random() * 1000000000)}`, + `user-t${new Date().getTime()}-r${Math.floor(Math.random() * 1000000000)}`, 'password' ); @@ -38,58 +54,175 @@ async function createTestRoom(client) { const roomId = await client.createRoom({ preset: 'public_chat', name: 'the hangout spot', + initial_state: [ + { + type: 'm.room.history_visibility', + state_key: '', + content: { + history_visibility: 'world_readable', + }, + }, + ], }); return roomId; } describe('matrix-public-archive', () => { - it('Test federation between fixture homeservers', async () => { - const hs1Client = await getTestClientForHs(config.testMatrixServerUrl1); - const hs2Client = await getTestClientForHs(config.testMatrixServerUrl2); - - // Create a room on hs2 - const hs2RoomId = await createTestRoom(hs2Client); - const room2EventIds = await createMessagesInRoom( - hs2Client, - hs2RoomId, - 10, - hs2Client.homeserverUrl - ); - - // Join hs1 to a room on hs2 (federation) - await hs1Client.joinRoom(hs2RoomId, 'hs2'); - - // From, hs1, make sure we can fetch messages from hs2 - const messagesEndpoint = urlJoin( - hs1Client.homeserverUrl, - `_matrix/client/r0/rooms/${hs2RoomId}/messages?limit=5&dir=b&filter={"types":["m.room.message"]}` - ); - const messageResData = await fetchEndpoint(messagesEndpoint, { - accessToken: hs1Client.accessToken, - }); - - // Make sure it returned some messages - assert.strictEqual(messageResData.chunk.length, 5); - - // Make sure all of the messages belong to the room - messageResData.chunk.map((event) => { - const isEventInRoomFromHs2 = room2EventIds.some((room2EventId) => { - return room2EventId === event.event_id; - }); - - // Make sure the message belongs to the room on hs2 - assert.strictEqual( - isEventInRoomFromHs2, - true, - `Expected ${event.event_id} (${event.type}: "${ - event.content.body - }") to be in room on hs2=${JSON.stringify(room2EventIds)}` - ); - }); + after(() => { + //server.close(); }); - it('can render diverse messages'); + it('Test federation between fixture homeservers', async () => { + try { + const hs1Client = await getTestClientForHs(testMatrixServerUrl1); + const hs2Client = await getTestClientForHs(testMatrixServerUrl2); + + // Create a room on hs2 + const hs2RoomId = await createTestRoom(hs2Client); + const room2EventIds = await createMessagesInRoom( + hs2Client, + hs2RoomId, + 10, + HOMESERVER_URL_TO_PRETTY_NAME_MAP[hs2Client.homeserverUrl] + ); + + // Join hs1 to a room on hs2 (federation) + await hs1Client.joinRoom(hs2RoomId, 'hs2'); + + // From, hs1, make sure we can fetch messages from hs2 + const messagesEndpoint = urlJoin( + hs1Client.homeserverUrl, + `_matrix/client/r0/rooms/${hs2RoomId}/messages?limit=5&dir=b&filter={"types":["m.room.message"]}` + ); + const messageResData = await fetchEndpointAsJson(messagesEndpoint, { + accessToken: hs1Client.accessToken, + }); + + // Make sure it returned some messages + assert.strictEqual(messageResData.chunk.length, 5); + + // Make sure all of the messages belong to the room + messageResData.chunk.map((event) => { + const isEventInRoomFromHs2 = room2EventIds.some((room2EventId) => { + return room2EventId === event.event_id; + }); + + // Make sure the message belongs to the room on hs2 + assert.strictEqual( + isEventInRoomFromHs2, + true, + `Expected ${event.event_id} (${event.type}: "${ + event.content.body + }") to be in room on hs2=${JSON.stringify(room2EventIds)}` + ); + }); + } catch (err) { + if (err.body) { + // FIXME: Remove this try/catch once the matrix-bot-sdk no longer throws + // huge response objects as errors, see + // https://github.com/turt2live/matrix-bot-sdk/pull/158 + throw new Error( + `Error occured in matrix-bot-sdk (this new error is to stop it from logging the huge response) statusCode=${ + err.statusCode + } body=${JSON.stringify(err.body)}` + ); + } + + throw err; + } + }); + + it('can render diverse messages', async () => { + try { + const client = await getTestClientForHs(testMatrixServerUrl1); + const roomId = await createTestRoom(client); + + // TODO: Set avatar of user + + // TODO: Set avatar of room + + // Test image + const mxcUri = await client.uploadContentFromUrl( + 'https://en.wikipedia.org/wiki/Friction#/media/File:Friction_between_surfaces.jpg' + ); + await client.sendMessage(roomId, { + body: 'Friction_between_surfaces.jpeg', + info: { + size: 396644, + mimetype: 'image/jpeg', + w: 1894, + h: 925, + 'xyz.amorgan.blurhash': 'LkR3G|IU?w%NbwbIemae_NxuD$M{', + }, + msgtype: 'm.image', + url: mxcUri, + }); + + // A normal text message + await client.sendMessage(roomId, { + msgtype: 'm.text', + body: '^ Figure 1: Simulated blocks with fractal rough surfaces, exhibiting static frictional interactions', + }); + + // A normal text message + await client.sendMessage(roomId, { + msgtype: 'm.text', + body: 'The topography of the Moon has been measured with laser altimetry and stereo image analysis.', + }); + + // Test replies + const eventToReplyTo = await client.sendMessage(roomId, { + 'org.matrix.msc1767.message': [ + { + body: "> <@ericgittertester:my.synapse.server> The topography of the Moon has been measured with laser altimetry and stereo image analysis.\n\nThe concentration of maria on the near side likely reflects the substantially thicker crust of the highlands of the Far Side, which may have formed in a slow-velocity impact of a second moon of Earth a few tens of millions of years after the Moon's formation.", + mimetype: 'text/plain', + }, + { + body: '
In reply to @ericgittertester:my.synapse.server
The topography of the Moon has been measured with laser altimetry and stereo image analysis.
The concentration of maria on the near side likely reflects the substantially thicker crust of the highlands of the Far Side, which may have formed in a slow-velocity impact of a second moon of Earth a few tens of millions of years after the Moon\'s formation.', + mimetype: 'text/html', + }, + ], + body: "> <@ericgittertester:my.synapse.server> The topography of the Moon has been measured with laser altimetry and stereo image analysis.\n\nThe concentration of maria on the near side likely reflects the substantially thicker crust of the highlands of the Far Side, which may have formed in a slow-velocity impact of a second moon of Earth a few tens of millions of years after the Moon's formation.", + msgtype: 'm.text', + format: 'org.matrix.custom.html', + formatted_body: + '
In reply to @ericgittertester:my.synapse.server
The topography of the Moon has been measured with laser altimetry and stereo image analysis.
The concentration of maria on the near side likely reflects the substantially thicker crust of the highlands of the Far Side, which may have formed in a slow-velocity impact of a second moon of Earth a few tens of millions of years after the Moon\'s formation.', + 'm.relates_to': { + 'm.in_reply_to': { + event_id: '$uEeScM2gfILkLpG8sOBTK7vcS0w_t3a9EVIAnSwqyiY', + }, + }, + }); + + // Test reactions + await client.sendEvent(roomId, 'm.reaction', { + 'm.relates_to': { + rel_type: 'm.annotation', + event_id: eventToReplyTo, + key: '��', + }, + }); + + const archiveUrl = matrixPublicArchiveURLCreator.archiveUrlForDate(roomId, new Date()); + + const archivePageHtml = await fetchEndpointAsText(archiveUrl); + console.log('archivePageHtml', archivePageHtml); + } catch (err) { + if (err.body) { + // FIXME: Remove this try/catch once the matrix-bot-sdk no longer throws + // huge response objects as errors, see + // https://github.com/turt2live/matrix-bot-sdk/pull/158 + throw new Error( + `Error occured in matrix-bot-sdk (this new error is to stop it from logging the huge response) statusCode=${ + err.statusCode + } body=${JSON.stringify(err.body)}` + ); + } + + throw err; + } + }); it(`can render day back in time from room on remote homeserver we haven't backfilled from`); diff --git a/vite.config.js b/vite.config.js index 467f942..40b9f0a 100644 --- a/vite.config.js +++ b/vite.config.js @@ -5,6 +5,10 @@ const path = require('path'); const { defineConfig } = require('vite'); module.exports = defineConfig({ + // We have to specify this otherwise Vite will override NODE_ENV as + // `production` when we start the server and watch build in our `start-dev.js`. + mode: process.env.NODE_ENV || 'dev', + //root: './', //base: './', outDir: './dist',