Add config to disable search engine indexing (#127)
This commit is contained in:
parent
b3c553a863
commit
dc85e839a1
|
@ -8,7 +8,11 @@
|
||||||
"archiveMessageLimit": 100,
|
"archiveMessageLimit": 100,
|
||||||
"requestTimeoutMs": 25000,
|
"requestTimeoutMs": 25000,
|
||||||
"logOutputFromChildProcesses": false,
|
"logOutputFromChildProcesses": false,
|
||||||
|
//"stopSearchEngineIndexing": true,
|
||||||
|
// Tracing
|
||||||
//"jaegerTracesEndpoint": "http://localhost:14268/api/traces",
|
//"jaegerTracesEndpoint": "http://localhost:14268/api/traces",
|
||||||
|
|
||||||
|
// Testing
|
||||||
"testMatrixServerUrl1": "http://localhost:11008/",
|
"testMatrixServerUrl1": "http://localhost:11008/",
|
||||||
"testMatrixServerUrl2": "http://localhost:12008/",
|
"testMatrixServerUrl2": "http://localhost:12008/",
|
||||||
|
|
||||||
|
|
|
@ -18,6 +18,7 @@ const matrixServerName = config.get('matrixServerName');
|
||||||
assert(matrixServerName);
|
assert(matrixServerName);
|
||||||
const matrixAccessToken = config.get('matrixAccessToken');
|
const matrixAccessToken = config.get('matrixAccessToken');
|
||||||
assert(matrixAccessToken);
|
assert(matrixAccessToken);
|
||||||
|
const stopSearchEngineIndexing = config.get('stopSearchEngineIndexing');
|
||||||
|
|
||||||
const router = express.Router({
|
const router = express.Router({
|
||||||
caseSensitive: true,
|
caseSensitive: true,
|
||||||
|
@ -56,6 +57,9 @@ router.get(
|
||||||
roomFetchError = err;
|
roomFetchError = err;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// We index the room directory unless the config says we shouldn't index anything
|
||||||
|
const shouldIndex = !stopSearchEngineIndexing;
|
||||||
|
|
||||||
const hydrogenStylesUrl = urlJoin(basePath, '/hydrogen-styles.css');
|
const hydrogenStylesUrl = urlJoin(basePath, '/hydrogen-styles.css');
|
||||||
const stylesUrl = urlJoin(basePath, '/css/styles.css');
|
const stylesUrl = urlJoin(basePath, '/css/styles.css');
|
||||||
const roomDirectoryStylesUrl = urlJoin(basePath, '/css/room-directory.css');
|
const roomDirectoryStylesUrl = urlJoin(basePath, '/css/room-directory.css');
|
||||||
|
@ -89,6 +93,7 @@ router.get(
|
||||||
title: `Matrix Public Archive`,
|
title: `Matrix Public Archive`,
|
||||||
styles: [hydrogenStylesUrl, stylesUrl, roomDirectoryStylesUrl],
|
styles: [hydrogenStylesUrl, stylesUrl, roomDirectoryStylesUrl],
|
||||||
scripts: [jsBundleUrl],
|
scripts: [jsBundleUrl],
|
||||||
|
shouldIndex,
|
||||||
cspNonce: res.locals.cspNonce,
|
cspNonce: res.locals.cspNonce,
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
|
|
|
@ -26,6 +26,7 @@ const matrixServerUrl = config.get('matrixServerUrl');
|
||||||
assert(matrixServerUrl);
|
assert(matrixServerUrl);
|
||||||
const matrixAccessToken = config.get('matrixAccessToken');
|
const matrixAccessToken = config.get('matrixAccessToken');
|
||||||
assert(matrixAccessToken);
|
assert(matrixAccessToken);
|
||||||
|
const stopSearchEngineIndexing = config.get('stopSearchEngineIndexing');
|
||||||
|
|
||||||
const matrixPublicArchiveURLCreator = new MatrixPublicArchiveURLCreator(basePath);
|
const matrixPublicArchiveURLCreator = new MatrixPublicArchiveURLCreator(basePath);
|
||||||
|
|
||||||
|
@ -270,6 +271,7 @@ router.get(
|
||||||
router.get(
|
router.get(
|
||||||
'/date/:yyyy(\\d{4})/:mm(\\d{2})/:dd(\\d{2})/:hourRange(\\d\\d?-\\d\\d?)?',
|
'/date/:yyyy(\\d{4})/:mm(\\d{2})/:dd(\\d{2})/:hourRange(\\d\\d?-\\d\\d?)?',
|
||||||
timeoutMiddleware,
|
timeoutMiddleware,
|
||||||
|
// eslint-disable-next-line max-statements
|
||||||
asyncHandler(async function (req, res) {
|
asyncHandler(async function (req, res) {
|
||||||
const roomIdOrAlias = getRoomIdOrAliasFromReq(req);
|
const roomIdOrAlias = getRoomIdOrAliasFromReq(req);
|
||||||
|
|
||||||
|
@ -359,8 +361,14 @@ router.get(
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// We only allow search engines to index `world_readable` rooms
|
// Default to no indexing (safe default)
|
||||||
const shouldIndex = roomData?.historyVisibility === `world_readable`;
|
let shouldIndex = false;
|
||||||
|
if (stopSearchEngineIndexing) {
|
||||||
|
shouldIndex = false;
|
||||||
|
} else {
|
||||||
|
// Otherwise we only allow search engines to index `world_readable` rooms
|
||||||
|
shouldIndex = roomData?.historyVisibility === `world_readable`;
|
||||||
|
}
|
||||||
|
|
||||||
// If we have over the `archiveMessageLimit` number of messages fetching
|
// If we have over the `archiveMessageLimit` number of messages fetching
|
||||||
// from the given day, it's acceptable to have them be from surrounding
|
// from the given day, it's acceptable to have them be from surrounding
|
||||||
|
|
Loading…
Reference in New Issue