E2E test but still failing because fetching from start of day before test events happened

.gitignore

@@ -3,5 +3,10 @@ node_modules
 dist
 dist-ssr
 *.local
-secrets.json
+
+config.user-overrides.json
+config/config.dev.user-overrides.json
+config/config.beta.user-overrides.json
+config/config.prod.user-overrides.json
 config.json
+secrets.json

README.md

@@ -5,11 +5,12 @@ config/secrets, using a draft branch of Hydrogen, etc.
 
 In the vein of [feature parity with
 Gitter](https://github.com/vector-im/roadmap/issues/26), the goal is to make a
-public archive site like Gitter's archives which search engines can index and
+public archive site for `world_readable` Matrix rooms like Gitter's archives
-keep all of the content accessible/available. There is already
+which search engines can index and keep all of the content accessible/available.
-https://view.matrix.org/ (https://github.com/matrix-org/matrix-static) but there
+There is already https://view.matrix.org/
-is some desire to make something with more Element-feeling polish and loading
+(https://github.com/matrix-org/matrix-static) but there is some desire to make
-faster (avoid the slow 502's errors that are frequent on `view.matrix.org`).
+something with more Element-feeling polish and loading faster (avoid the slow
+502's errors that are frequent on `view.matrix.org`).
 
 ## Plan summary
 

config/config.default.json

@@ -0,0 +1,10 @@
+{
+  "basePort": "3050",
+  "basePath": "http://localhost:3050",
+  "matrixServerUrl": "http://localhost:8008/",
+  "testMatrixServerUrl1": "http://localhost:11008/",
+  "testMatrixServerUrl2": "http://localhost:12008/",
+
+  "Secrets": "Remove the 'xxx__' prefix and fill these in your config/config.user-overrides.json or by environment variable",
+  "xxx__matrixAccessToken": "xxx"
+}

config/config.test.json

@@ -0,0 +1,10 @@
+{
+  "basePort": "3051",
+  "basePath": "http://localhost:3051",
+  "matrixServerUrl": "http://localhost:11008/",
+  "testMatrixServerUrl1": "http://localhost:11008/",
+  "testMatrixServerUrl2": "http://localhost:12008/",
+
+  "Secrets": "xxx",
+  "matrixAccessToken": "as_token_8664700429a911bbbecf7d91b9e1a74716d669f40cf32259630e38439726e29d"
+}

package-lock.json

@@ -14,6 +14,7 @@
         "matrix-bot-sdk": "^0.5.19",
         "matrix-js-sdk": "^15.5.2",
         "matrix-public-archive-shared": "file:./shared/",
+        "nconf": "^0.11.3",
         "node-fetch": "^2.6.7",
         "url-join": "^4.0.1"
       },
@@ -324,7 +325,6 @@
       "version": "5.0.1",
       "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
       "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==",
-      "dev": true,
       "engines": {
         "node": ">=8"
       }
@@ -383,6 +383,11 @@
         "node": ">=0.8"
       }
     },
+    "node_modules/async": {
+      "version": "1.5.2",
+      "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz",
+      "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo="
+    },
     "node_modules/asynckit": {
       "version": "0.4.0",
       "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
@@ -704,7 +709,6 @@
       "version": "7.0.4",
       "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz",
       "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==",
-      "dev": true,
       "dependencies": {
         "string-width": "^4.2.0",
         "strip-ansi": "^6.0.0",
@@ -1066,8 +1070,7 @@
     "node_modules/emoji-regex": {
       "version": "8.0.0",
       "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
-      "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==",
+      "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="
-      "dev": true
     },
     "node_modules/encodeurl": {
       "version": "1.0.2",
@@ -1436,7 +1439,6 @@
       "version": "3.1.1",
       "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz",
       "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==",
-      "dev": true,
       "engines": {
         "node": ">=6"
       }
@@ -2007,7 +2009,6 @@
       "version": "2.0.5",
       "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz",
       "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==",
-      "dev": true,
       "engines": {
         "node": "6.* || 8.* || >= 10.*"
       }
@@ -2409,7 +2410,6 @@
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/ini/-/ini-2.0.0.tgz",
       "integrity": "sha512-7PnF4oN3CvZF23ADhA5wRaYEQpJ8qygSkbtTXWBeXWXmEVRXK+1ITciHWwHhsjv1TmW0MgacIv6hEi5pX5NQdA==",
-      "dev": true,
       "engines": {
         "node": ">=10"
       }
@@ -2471,7 +2471,6 @@
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
       "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==",
-      "dev": true,
       "engines": {
         "node": ">=8"
       }
@@ -3119,6 +3118,20 @@
       "integrity": "sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=",
       "dev": true
     },
+    "node_modules/nconf": {
+      "version": "0.11.3",
+      "resolved": "https://registry.npmjs.org/nconf/-/nconf-0.11.3.tgz",
+      "integrity": "sha512-iYsAuDS9pzjVMGIzJrGE0Vk3Eh8r/suJanRAnWGBd29rVS2XtSgzcAo5l6asV3e4hH2idVONHirg1efoBOslBg==",
+      "dependencies": {
+        "async": "^1.4.0",
+        "ini": "^2.0.0",
+        "secure-keys": "^1.0.0",
+        "yargs": "^16.1.1"
+      },
+      "engines": {
+        "node": ">= 0.4.0"
+      }
+    },
     "node_modules/negotiator": {
       "version": "0.6.3",
       "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz",
@@ -3788,7 +3801,6 @@
       "version": "2.1.1",
       "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
       "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=",
-      "dev": true,
       "engines": {
         "node": ">=0.10.0"
       }
@@ -3921,6 +3933,11 @@
         "entities": "^2.0.0"
       }
     },
+    "node_modules/secure-keys": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/secure-keys/-/secure-keys-1.0.0.tgz",
+      "integrity": "sha1-8MgtmKOxOah3aogIBQuCRDEIf8o="
+    },
     "node_modules/semver": {
       "version": "6.3.0",
       "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz",
@@ -4085,7 +4102,6 @@
       "version": "4.2.3",
       "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
       "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==",
-      "dev": true,
       "dependencies": {
         "emoji-regex": "^8.0.0",
         "is-fullwidth-code-point": "^3.0.0",
@@ -4099,7 +4115,6 @@
       "version": "6.0.1",
       "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz",
       "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==",
-      "dev": true,
       "dependencies": {
         "ansi-regex": "^5.0.1"
       },
@@ -4512,7 +4527,6 @@
       "version": "7.0.0",
       "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz",
       "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==",
-      "dev": true,
       "dependencies": {
         "ansi-styles": "^4.0.0",
         "string-width": "^4.1.0",
@@ -4556,7 +4570,6 @@
       "version": "5.0.8",
       "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz",
       "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==",
-      "dev": true,
       "engines": {
         "node": ">=10"
       }
@@ -4570,7 +4583,6 @@
       "version": "16.2.0",
       "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz",
       "integrity": "sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==",
-      "dev": true,
       "dependencies": {
         "cliui": "^7.0.2",
         "escalade": "^3.1.1",
@@ -4588,7 +4600,6 @@
       "version": "20.2.4",
       "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.4.tgz",
       "integrity": "sha512-WOkpgNhPTlE73h4VFAFsOnomJVaovO8VqLDzy5saChRBFQFBoMYirowyW+Q9HB4HFF4Z7VZTiG3iSzJJA29yRA==",
-      "dev": true,
       "engines": {
         "node": ">=10"
       }
@@ -4864,8 +4875,7 @@
     "ansi-regex": {
       "version": "5.0.1",
       "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
-      "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==",
+      "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="
-      "dev": true
     },
     "ansi-styles": {
       "version": "4.3.0",
@@ -4909,6 +4919,11 @@
       "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz",
       "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU="
     },
+    "async": {
+      "version": "1.5.2",
+      "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz",
+      "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo="
+    },
     "asynckit": {
       "version": "0.4.0",
       "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
@@ -5159,7 +5174,6 @@
       "version": "7.0.4",
       "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz",
       "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==",
-      "dev": true,
       "requires": {
         "string-width": "^4.2.0",
         "strip-ansi": "^6.0.0",
@@ -5437,8 +5451,7 @@
     "emoji-regex": {
       "version": "8.0.0",
       "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
-      "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==",
+      "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="
-      "dev": true
     },
     "encodeurl": {
       "version": "1.0.2",
@@ -5622,8 +5635,7 @@
     "escalade": {
       "version": "3.1.1",
       "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz",
-      "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==",
+      "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw=="
-      "dev": true
     },
     "escape-goat": {
       "version": "2.1.1",
@@ -6040,8 +6052,7 @@
     "get-caller-file": {
       "version": "2.0.5",
       "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz",
-      "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==",
+      "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg=="
-      "dev": true
     },
     "get-stream": {
       "version": "4.1.0",
@@ -6344,8 +6355,7 @@
     "ini": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/ini/-/ini-2.0.0.tgz",
-      "integrity": "sha512-7PnF4oN3CvZF23ADhA5wRaYEQpJ8qygSkbtTXWBeXWXmEVRXK+1ITciHWwHhsjv1TmW0MgacIv6hEi5pX5NQdA==",
+      "integrity": "sha512-7PnF4oN3CvZF23ADhA5wRaYEQpJ8qygSkbtTXWBeXWXmEVRXK+1ITciHWwHhsjv1TmW0MgacIv6hEi5pX5NQdA=="
-      "dev": true
     },
     "ipaddr.js": {
       "version": "1.9.1",
@@ -6388,8 +6398,7 @@
     "is-fullwidth-code-point": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
-      "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==",
+      "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg=="
-      "dev": true
     },
     "is-glob": {
       "version": "4.0.3",
@@ -6875,6 +6884,17 @@
       "integrity": "sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=",
       "dev": true
     },
+    "nconf": {
+      "version": "0.11.3",
+      "resolved": "https://registry.npmjs.org/nconf/-/nconf-0.11.3.tgz",
+      "integrity": "sha512-iYsAuDS9pzjVMGIzJrGE0Vk3Eh8r/suJanRAnWGBd29rVS2XtSgzcAo5l6asV3e4hH2idVONHirg1efoBOslBg==",
+      "requires": {
+        "async": "^1.4.0",
+        "ini": "^2.0.0",
+        "secure-keys": "^1.0.0",
+        "yargs": "^16.1.1"
+      }
+    },
     "negotiator": {
       "version": "0.6.3",
       "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz",
@@ -7359,8 +7379,7 @@
     "require-directory": {
       "version": "2.1.1",
       "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
-      "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=",
+      "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I="
-      "dev": true
     },
     "resolve": {
       "version": "1.22.0",
@@ -7447,6 +7466,11 @@
         }
       }
     },
+    "secure-keys": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/secure-keys/-/secure-keys-1.0.0.tgz",
+      "integrity": "sha1-8MgtmKOxOah3aogIBQuCRDEIf8o="
+    },
     "semver": {
       "version": "6.3.0",
       "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz",
@@ -7578,7 +7602,6 @@
       "version": "4.2.3",
       "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
       "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==",
-      "dev": true,
       "requires": {
         "emoji-regex": "^8.0.0",
         "is-fullwidth-code-point": "^3.0.0",
@@ -7589,7 +7612,6 @@
       "version": "6.0.1",
       "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz",
       "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==",
-      "dev": true,
       "requires": {
         "ansi-regex": "^5.0.1"
       }
@@ -7886,7 +7908,6 @@
       "version": "7.0.0",
       "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz",
       "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==",
-      "dev": true,
       "requires": {
         "ansi-styles": "^4.0.0",
         "string-width": "^4.1.0",
@@ -7920,8 +7941,7 @@
     "y18n": {
       "version": "5.0.8",
       "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz",
-      "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==",
+      "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA=="
-      "dev": true
     },
     "yallist": {
       "version": "4.0.0",
@@ -7932,7 +7952,6 @@
       "version": "16.2.0",
       "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz",
       "integrity": "sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==",
-      "dev": true,
       "requires": {
         "cliui": "^7.0.2",
         "escalade": "^3.1.1",
@@ -7946,8 +7965,7 @@
     "yargs-parser": {
       "version": "20.2.4",
       "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.4.tgz",
-      "integrity": "sha512-WOkpgNhPTlE73h4VFAFsOnomJVaovO8VqLDzy5saChRBFQFBoMYirowyW+Q9HB4HFF4Z7VZTiG3iSzJJA29yRA==",
+      "integrity": "sha512-WOkpgNhPTlE73h4VFAFsOnomJVaovO8VqLDzy5saChRBFQFBoMYirowyW+Q9HB4HFF4Z7VZTiG3iSzJJA29yRA=="
-      "dev": true
     },
     "yargs-unparser": {
       "version": "2.0.0",

package.json

@@ -4,6 +4,7 @@
   "scripts": {
     "start-dev": "node server/start-dev.js",
     "lint": "eslint **/*.js",
+    "test": "npm run mocha -- test/e2e-tests.js --timeout 15000",
     "nodemon": "nodemon",
     "vite": "vite",
     "mocha": "mocha",
@@ -31,6 +32,7 @@
     "matrix-bot-sdk": "^0.5.19",
     "matrix-js-sdk": "^15.5.2",
     "matrix-public-archive-shared": "file:./shared/",
+    "nconf": "^0.11.3",
     "node-fetch": "^2.6.7",
     "url-join": "^4.0.1"
   }

server/fetch-events-for-timestamp.js

@@ -3,9 +3,10 @@
 const assert = require('assert');
 const urlJoin = require('url-join');
 
-const fetchEndpoint = require('./lib/fetch-endpoint');
+const { fetchEndpointAsJson } = require('./lib/fetch-endpoint');
 
-const { matrixServerUrl } = require('../config.json');
+const config = require('./lib/config');
+const matrixServerUrl = config.get('matrixServerUrl');
 assert(matrixServerUrl);
 
 async function fetchEventsForTimestamp(accessToken, roomId, ts) {
@@ -13,11 +14,18 @@ async function fetchEventsForTimestamp(accessToken, roomId, ts) {
   assert(roomId);
   assert(ts);
 
+  // TODO: Only join world_readable rooms
+  const joinEndpoint = urlJoin(matrixServerUrl, `_matrix/client/r0/join/${roomId}`);
+  await fetchEndpointAsJson(joinEndpoint, {
+    method: 'POST',
+    accessToken,
+  });
+
   const timestampToEventEndpoint = urlJoin(
     matrixServerUrl,
     `_matrix/client/unstable/org.matrix.msc3030/rooms/${roomId}/timestamp_to_event?ts=${ts}&dir=b`
   );
-  const timestampToEventResData = await fetchEndpoint(timestampToEventEndpoint, {
+  const timestampToEventResData = await fetchEndpointAsJson(timestampToEventEndpoint, {
     accessToken,
   });
   const eventIdForTimestamp = timestampToEventResData.event_id;
@@ -28,16 +36,17 @@ async function fetchEventsForTimestamp(accessToken, roomId, ts) {
     matrixServerUrl,
     `_matrix/client/r0/rooms/${roomId}/context/${eventIdForTimestamp}?limit=0`
   );
-  const contextResData = await fetchEndpoint(contextEndpoint, {
+  const contextResData = await fetchEndpointAsJson(contextEndpoint, {
     accessToken,
   });
   //console.log('contextResData', contextResData);
 
+  // Add filter={"lazy_load_members":true,"include_redundant_members":true} to get member state events included
   const messagesEndpoint = urlJoin(
     matrixServerUrl,
     `_matrix/client/r0/rooms/${roomId}/messages?from=${contextResData.start}&limit=50&filter={"lazy_load_members":true,"include_redundant_members":true}`
   );
-  const messageResData = await fetchEndpoint(messagesEndpoint, {
+  const messageResData = await fetchEndpointAsJson(messagesEndpoint, {
     accessToken,
   });
 

server/fetch-room-data.js

@@ -3,9 +3,10 @@
 const assert = require('assert');
 
 const urlJoin = require('url-join');
-const fetchEndpoint = require('./lib/fetch-endpoint');
+const { fetchEndpointAsJson } = require('./lib/fetch-endpoint');
 
-const { matrixServerUrl } = require('../config.json');
+const config = require('./lib/config');
+const matrixServerUrl = config.get('matrixServerUrl');
 assert(matrixServerUrl);
 
 async function fetchRoomData(accessToken, roomId) {
@@ -22,10 +23,10 @@ async function fetchRoomData(accessToken, roomId) {
   );
 
   const [stateNameResDataOutcome, stateAvatarResDataOutcome] = await Promise.allSettled([
-    fetchEndpoint(stateNameEndpoint, {
+    fetchEndpointAsJson(stateNameEndpoint, {
       accessToken,
     }),
-    fetchEndpoint(stateAvatarEndpoint, {
+    fetchEndpointAsJson(stateAvatarEndpoint, {
       accessToken,
     }),
   ]);

server/lib/config.js

@@ -0,0 +1,41 @@
+'use strict';
+
+// This file is based off the Gitter config, https://gitlab.com/gitlab-org/gitter/env/blob/master/lib/config.js
+
+const path = require('path');
+const nconf = require('nconf');
+
+function configureNodeEnv() {
+  const nodeEnv = process.env.NODE_ENV;
+  if (nodeEnv === 'production') {
+    return 'prod';
+  } else if (nodeEnv === 'development') {
+    return 'dev';
+  }
+  if (nodeEnv) return nodeEnv;
+
+  // Default to NODE_ENV=dev
+  process.env.NODE_ENV = 'dev';
+  return 'dev';
+}
+
+const nodeEnv = configureNodeEnv();
+console.log('nodeEnv', nodeEnv);
+const configDir = path.join(__dirname, '../../config');
+
+nconf.argv().env('__');
+
+nconf.add('envUser', {
+  type: 'file',
+  file: path.join(configDir, 'config.' + nodeEnv + '.user-overrides.json'),
+});
+
+// Only use user-overrides in dev
+if (nodeEnv === 'dev') {
+  nconf.add('user', { type: 'file', file: path.join(configDir, 'config.user-overrides.json') });
+}
+
+nconf.add('nodeEnv', { type: 'file', file: path.join(configDir, 'config.' + nodeEnv + '.json') });
+nconf.add('defaults', { type: 'file', file: path.join(configDir, 'config.default.json') });
+
+module.exports = nconf;

server/lib/fetch-endpoint.js

@@ -22,16 +22,45 @@ const checkResponseStatus = async (response) => {
   }
 };
 
-async function fetchEndpoint(endpoint, { accessToken }) {
+async function fetchEndpoint(endpoint, options = {}) {
+  const { method, accessToken } = options;
+  const headers = options.headers || {};
+
+  if (accessToken) {
+    headers.Authorization = `Bearer ${accessToken}`;
+  }
+
   const res = await fetch(endpoint, {
-    headers: {
+    method,
-      'Content-Type': 'application/json',
+    headers,
-      Authorization: `Bearer ${accessToken}`,
-    },
   });
   await checkResponseStatus(res);
+
+  return res;
+}
+
+async function fetchEndpointAsText(endpoint, options) {
+  const res = await fetchEndpoint(endpoint, options);
+  const data = await res.text();
+  return data;
+}
+
+async function fetchEndpointAsJson(endpoint, options) {
+  const opts = {
+    ...options,
+    headers: {
+      'Content-Type': 'application/json',
+      ...(options.headers || {}),
+    },
+  };
+
+  const res = await fetchEndpoint(endpoint, opts);
   const data = await res.json();
   return data;
 }
 
-module.exports = fetchEndpoint;
+module.exports = {
+  fetchEndpoint,
+  fetchEndpointAsText,
+  fetchEndpointAsJson,
+};

server/render-hydrogen-to-string.js

@@ -7,7 +7,7 @@ const { readFile } = require('fs').promises;
 const crypto = require('crypto');
 const { parseHTML } = require('linkedom');
 
-const config = require('../config.json');
+const config = require('./lib/config');
 
 async function renderToString({ fromTimestamp, roomData, events, stateEventMap }) {
   assert(fromTimestamp);
@@ -37,7 +37,11 @@ async function renderToString({ fromTimestamp, roomData, events, stateEventMap }
     roomData,
     events,
     stateEventMap,
-    config,
+    config: {
+      basePort: config.get('basePort'),
+      basePath: config.get('basePath'),
+      matrixServerUrl: config.get('matrixServerUrl'),
+    },
   };
   // Serialize it for when we run this again client-side
   dom.document.body.insertAdjacentHTML(

server/routes/install-routes.js

@@ -11,11 +11,10 @@ const fetchRoomData = require('../fetch-room-data');
 const fetchEventsForTimestamp = require('../fetch-events-for-timestamp');
 const renderHydrogenToString = require('../render-hydrogen-to-string');
 
-const config = require('../../config.json');
+const config = require('../lib/config');
-const basePath = config.basePath;
+const basePath = config.get('basePath');
 assert(basePath);
-
+const matrixAccessToken = config.get('matrixAccessToken');
-const { matrixAccessToken } = require('../../secrets.json');
 assert(matrixAccessToken);
 
 function parseArchiveRangeFromReq(req) {
@@ -128,6 +127,8 @@ function installRoutes(app) {
         fetchEventsForTimestamp(matrixAccessToken, roomIdOrAlias, fromTimestamp),
       ]);
 
+      console.log('events', JSON.stringify(events, null, 2));
+
       const hydrogenHtmlOutput = await renderHydrogenToString({
         fromTimestamp,
         roomData,

server/server.js

@@ -1,8 +1,16 @@
 'use strict';
 
+console.log('server process.env.NODE_ENV', process.env.NODE_ENV);
+
 const express = require('express');
+
 const installRoutes = require('./routes/install-routes');
+const config = require('./lib/config');
+const basePort = config.get('basePort');
 
 const app = express();
 installRoutes(app);
-app.listen(3050);
+
+const server = app.listen(basePort);
+
+module.exports = server;

server/start-dev.js

@@ -7,6 +7,8 @@ const mergeOptions = require('merge-options');
 
 const viteConfig = require('../vite.config');
 
+console.log('start-dev process.env.NODE_ENV', process.env.NODE_ENV);
+
 // Listen for any changes to files and restart the Node.js server process
 //
 // For API docs, see
@@ -14,7 +16,6 @@ const viteConfig = require('../vite.config');
 nodemon({
   script: path.join(__dirname, './server.js'),
   ext: 'js json',
-  delay: 2500,
   ignoreRoot: ['.git'],
   ignore: [path.join(__dirname, '../dist/*')],
 });
@@ -40,3 +41,5 @@ build(
     },
   })
 );
+
+console.log('start-dev2 process.env.NODE_ENV', process.env.NODE_ENV);

shared/hydrogen-vm-render-script.js

@@ -20,10 +20,10 @@ const {
   RoomViewModel,
   ViewModel,
 } = require('hydrogen-view-sdk');
-const urlJoin = require('url-join');
 
 const ArchiveView = require('matrix-public-archive-shared/ArchiveView');
 const RightPanelContentView = require('matrix-public-archive-shared/RightPanelContentView');
+const MatrixPublicArchiveURLCreator = require('matrix-public-archive-shared/lib/url-creator');
 
 const fromTimestamp = window.matrixPublicArchiveContext.fromTimestamp;
 assert(fromTimestamp);
@@ -38,6 +38,8 @@ assert(config);
 assert(config.matrixServerUrl);
 assert(config.basePath);
 
+const matrixPublicArchiveURLCreator = new MatrixPublicArchiveURLCreator(config.basePath);
+
 function addSupportClasses() {
   const input = document.createElement('input');
   input.type = 'month';
@@ -235,11 +237,7 @@ async function mountHydrogen() {
     }
 
     archiveUrlForDate(date) {
-      // Gives the date in YYYY-mm-dd format.
+      return matrixPublicArchiveURLCreator.archiveUrlForDate(room.id, date);
-      // date.toISOString() -> 2022-02-16T23:20:04.709Z
-      const urlDate = date.toISOString().split('T')[0].replaceAll('-', '/');
-
-      return urlJoin(config.basePath, `${room.id}/date/${urlDate}`);
     }
 
     prevMonth() {

shared/lib/url-creator.js

@@ -0,0 +1,19 @@
+'use strict';
+
+const urlJoin = require('url-join');
+
+class URLCreator {
+  constructor(basePath) {
+    this._basePath = basePath;
+  }
+
+  archiveUrlForDate(roomId, date) {
+    // Gives the date in YYYY/mm/dd format.
+    // date.toISOString() -> 2022-02-16T23:20:04.709Z
+    const urlDate = date.toISOString().split('T')[0].replaceAll('-', '/');
+
+    return urlJoin(this._basePath, `${roomId}/date/${urlDate}`);
+  }
+}
+
+module.exports = URLCreator;

test/README.md

@@ -1,7 +1,17 @@
 ```
-docker build -t matrix-public-archive-test-homeserver -f test/dockerfiles/Synapse.Dockerfile test/dockerfiles/
+$ docker pull matrixdotorg/synapse:latest
+$ docker build -t matrix-public-archive-test-homeserver -f test/dockerfiles/Synapse.Dockerfile test/dockerfiles/
 ```
 
 ```
 docker-compose -f test/docker-compose.yml up -d --no-recreate
 ```
+
+```
+$ docker ps --all | grep test_hs
+$ docker logs test_hs1_1
+$ docker logs test_hs2_1
+
+$ docker stop test_hs1_1 test_hs2_1
+$ docker rm test_hs1_1 test_hs2_1
+```

test/dockerfiles/synapse/as_registration.yaml

@@ -0,0 +1,18 @@
+id: 6527ecdd6b8fe61c645d9d412222d75b
+hs_token: hs_token_1f5d7675517072f0c0b5b684ca23f9ffad5a0cfc32e1ca824c0600ee74e105d7
+# We use this as a consistent acccess token to use in our tests
+as_token: as_token_8664700429a911bbbecf7d91b9e1a74716d669f40cf32259630e38439726e29d
+# This doesn't relate to anything else
+url: 'http://localhost:0000'
+sender_localpart: archiver
+namespaces:
+  users:
+    - exclusive: false
+      regex: .*
+  aliases:
+    - exclusive: false
+      regex: .*
+  rooms:
+    - exclusive: false
+      regex: .*
+rate_limited: false

test/dockerfiles/synapse/homeserver.yaml

@@ -94,6 +94,14 @@ rc_joins:
 
 federation_rr_transactions_per_room_per_second: 9999
 
+## API Configuration ##
+
+# A list of application service config files to use
+#
+app_service_config_files:
+  # We use this to provide a constant matrixAccessToken for the tests
+  - /conf/as_registration.yaml
+
 ## Experimental Features ##
 
 experimental_features:

test/e2e-tests.js

@@ -1,20 +1,36 @@
 'use strict';
 
+process.env.NODE_ENV = 'test';
+
 const assert = require('assert');
 const urlJoin = require('url-join');
 const { MatrixAuth } = require('matrix-bot-sdk');
 
-const fetchEndpoint = require('../server/lib/fetch-endpoint');
+const server = require('../server/server');
+const { fetchEndpointAsText, fetchEndpointAsJson } = require('../server/lib/fetch-endpoint');
 
-const config = require('../config');
+const MatrixPublicArchiveURLCreator = require('matrix-public-archive-shared/lib/url-creator');
-assert(config.testMatrixServerUrl1);
+
-assert(config.testMatrixServerUrl2);
+const config = require('../server/lib/config');
+const testMatrixServerUrl1 = config.get('testMatrixServerUrl1');
+const testMatrixServerUrl2 = config.get('testMatrixServerUrl2');
+const basePath = config.get('basePath');
+assert(testMatrixServerUrl1);
+assert(testMatrixServerUrl2);
+assert(basePath);
+
+const matrixPublicArchiveURLCreator = new MatrixPublicArchiveURLCreator(basePath);
+
+const HOMESERVER_URL_TO_PRETTY_NAME_MAP = {
+  [testMatrixServerUrl1]: 'hs1',
+  [testMatrixServerUrl2]: 'hs2',
+};
 
 async function getTestClientForHs(testMatrixServerUrl) {
   const auth = new MatrixAuth(testMatrixServerUrl);
 
   const client = await auth.passwordRegister(
-    `user-${Math.floor(Math.random() * 1000000000)}`,
+    `user-t${new Date().getTime()}-r${Math.floor(Math.random() * 1000000000)}`,
     'password'
   );
 
@@ -38,15 +54,29 @@ async function createTestRoom(client) {
   const roomId = await client.createRoom({
     preset: 'public_chat',
     name: 'the hangout spot',
+    initial_state: [
+      {
+        type: 'm.room.history_visibility',
+        state_key: '',
+        content: {
+          history_visibility: 'world_readable',
+        },
+      },
+    ],
   });
 
   return roomId;
 }
 
 describe('matrix-public-archive', () => {
+  after(() => {
+    //server.close();
+  });
+
   it('Test federation between fixture homeservers', async () => {
-    const hs1Client = await getTestClientForHs(config.testMatrixServerUrl1);
+    try {
-    const hs2Client = await getTestClientForHs(config.testMatrixServerUrl2);
+      const hs1Client = await getTestClientForHs(testMatrixServerUrl1);
+      const hs2Client = await getTestClientForHs(testMatrixServerUrl2);
 
       // Create a room on hs2
       const hs2RoomId = await createTestRoom(hs2Client);
@@ -54,7 +84,7 @@ describe('matrix-public-archive', () => {
         hs2Client,
         hs2RoomId,
         10,
-      hs2Client.homeserverUrl
+        HOMESERVER_URL_TO_PRETTY_NAME_MAP[hs2Client.homeserverUrl]
       );
 
       // Join hs1 to a room on hs2 (federation)
@@ -65,7 +95,7 @@ describe('matrix-public-archive', () => {
         hs1Client.homeserverUrl,
         `_matrix/client/r0/rooms/${hs2RoomId}/messages?limit=5&dir=b&filter={"types":["m.room.message"]}`
       );
-    const messageResData = await fetchEndpoint(messagesEndpoint, {
+      const messageResData = await fetchEndpointAsJson(messagesEndpoint, {
         accessToken: hs1Client.accessToken,
       });
 
@@ -87,9 +117,112 @@ describe('matrix-public-archive', () => {
           }") to be in room on hs2=${JSON.stringify(room2EventIds)}`
         );
       });
+    } catch (err) {
+      if (err.body) {
+        // FIXME: Remove this try/catch once the matrix-bot-sdk no longer throws
+        // huge response objects as errors, see
+        // https://github.com/turt2live/matrix-bot-sdk/pull/158
+        throw new Error(
+          `Error occured in matrix-bot-sdk (this new error is to stop it from logging the huge response) statusCode=${
+            err.statusCode
+          } body=${JSON.stringify(err.body)}`
+        );
+      }
+
+      throw err;
+    }
   });
 
-  it('can render diverse messages');
+  it('can render diverse messages', async () => {
+    try {
+      const client = await getTestClientForHs(testMatrixServerUrl1);
+      const roomId = await createTestRoom(client);
+
+      // TODO: Set avatar of user
+
+      // TODO: Set avatar of room
+
+      // Test image
+      const mxcUri = await client.uploadContentFromUrl(
+        'https://en.wikipedia.org/wiki/Friction#/media/File:Friction_between_surfaces.jpg'
+      );
+      await client.sendMessage(roomId, {
+        body: 'Friction_between_surfaces.jpeg',
+        info: {
+          size: 396644,
+          mimetype: 'image/jpeg',
+          w: 1894,
+          h: 925,
+          'xyz.amorgan.blurhash': 'LkR3G|IU?w%NbwbIemae_NxuD$M{',
+        },
+        msgtype: 'm.image',
+        url: mxcUri,
+      });
+
+      // A normal text message
+      await client.sendMessage(roomId, {
+        msgtype: 'm.text',
+        body: '^ Figure 1: Simulated blocks with fractal rough surfaces, exhibiting static frictional interactions',
+      });
+
+      // A normal text message
+      await client.sendMessage(roomId, {
+        msgtype: 'm.text',
+        body: 'The topography of the Moon has been measured with laser altimetry and stereo image analysis.',
+      });
+
+      // Test replies
+      const eventToReplyTo = await client.sendMessage(roomId, {
+        'org.matrix.msc1767.message': [
+          {
+            body: "> <@ericgittertester:my.synapse.server> The topography of the Moon has been measured with laser altimetry and stereo image analysis.\n\nThe concentration of maria on the near side likely reflects the substantially thicker crust of the highlands of the Far Side, which may have formed in a slow-velocity impact of a second moon of Earth a few tens of millions of years after the Moon's formation.",
+            mimetype: 'text/plain',
+          },
+          {
+            body: '<mx-reply><blockquote><a href="https://matrix.to/#/!HBehERstyQBxyJDLfR:my.synapse.server/$uEeScM2gfILkLpG8sOBTK7vcS0w_t3a9EVIAnSwqyiY?via=my.synapse.server">In reply to</a> <a href="https://matrix.to/#/@ericgittertester:my.synapse.server">@ericgittertester:my.synapse.server</a><br>The topography of the Moon has been measured with laser altimetry and stereo image analysis.</blockquote></mx-reply>The concentration of maria on the near side likely reflects the substantially thicker crust of the highlands of the Far Side, which may have formed in a slow-velocity impact of a second moon of Earth a few tens of millions of years after the Moon\'s formation.',
+            mimetype: 'text/html',
+          },
+        ],
+        body: "> <@ericgittertester:my.synapse.server> The topography of the Moon has been measured with laser altimetry and stereo image analysis.\n\nThe concentration of maria on the near side likely reflects the substantially thicker crust of the highlands of the Far Side, which may have formed in a slow-velocity impact of a second moon of Earth a few tens of millions of years after the Moon's formation.",
+        msgtype: 'm.text',
+        format: 'org.matrix.custom.html',
+        formatted_body:
+          '<mx-reply><blockquote><a href="https://matrix.to/#/!HBehERstyQBxyJDLfR:my.synapse.server/$uEeScM2gfILkLpG8sOBTK7vcS0w_t3a9EVIAnSwqyiY?via=my.synapse.server">In reply to</a> <a href="https://matrix.to/#/@ericgittertester:my.synapse.server">@ericgittertester:my.synapse.server</a><br>The topography of the Moon has been measured with laser altimetry and stereo image analysis.</blockquote></mx-reply>The concentration of maria on the near side likely reflects the substantially thicker crust of the highlands of the Far Side, which may have formed in a slow-velocity impact of a second moon of Earth a few tens of millions of years after the Moon\'s formation.',
+        'm.relates_to': {
+          'm.in_reply_to': {
+            event_id: '$uEeScM2gfILkLpG8sOBTK7vcS0w_t3a9EVIAnSwqyiY',
+          },
+        },
+      });
+
+      // Test reactions
+      await client.sendEvent(roomId, 'm.reaction', {
+        'm.relates_to': {
+          rel_type: 'm.annotation',
+          event_id: eventToReplyTo,
+          key: '<27><>',
+        },
+      });
+
+      const archiveUrl = matrixPublicArchiveURLCreator.archiveUrlForDate(roomId, new Date());
+
+      const archivePageHtml = await fetchEndpointAsText(archiveUrl);
+      console.log('archivePageHtml', archivePageHtml);
+    } catch (err) {
+      if (err.body) {
+        // FIXME: Remove this try/catch once the matrix-bot-sdk no longer throws
+        // huge response objects as errors, see
+        // https://github.com/turt2live/matrix-bot-sdk/pull/158
+        throw new Error(
+          `Error occured in matrix-bot-sdk (this new error is to stop it from logging the huge response) statusCode=${
+            err.statusCode
+          } body=${JSON.stringify(err.body)}`
+        );
+      }
+
+      throw err;
+    }
+  });
 
   it(`can render day back in time from room on remote homeserver we haven't backfilled from`);
 

vite.config.js

@@ -5,6 +5,10 @@ const path = require('path');
 const { defineConfig } = require('vite');
 
 module.exports = defineConfig({
+  // We have to specify this otherwise Vite will override NODE_ENV as
+  // `production` when we start the server and watch build in our `start-dev.js`.
+  mode: process.env.NODE_ENV || 'dev',
+
   //root: './',
   //base: './',
   outDir: './dist',