uBlock/platform/mv3/make-rulesets.js

285 lines
8.9 KiB
JavaScript

/*******************************************************************************
uBlock Origin - a browser extension to block requests.
Copyright (C) 2022-present Raymond Hill
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see {http://www.gnu.org/licenses/}.
Home: https://github.com/gorhill/uBlock
*/
'use strict';
/******************************************************************************/
import fs from 'fs/promises';
import https from 'https';
import process from 'process';
import rulesetConfigs from './ruleset-config.js';
import { dnrRulesetFromRawLists } from './js/static-dnr-filtering.js';
/******************************************************************************/
const commandLineArgs = (( ) => {
const args = new Map();
let name, value;
for ( const arg of process.argv.slice(2) ) {
const pos = arg.indexOf('=');
if ( pos === -1 ) {
name = arg;
value = '';
} else {
name = arg.slice(0, pos);
value = arg.slice(pos+1);
}
args.set(name, value);
}
return args;
})();
/******************************************************************************/
async function main() {
const writeOps = [];
const ruleResources = [];
const rulesetDetails = [];
const outputDir = commandLineArgs.get('output') || '.';
const output = [];
const log = (text, silent = false) => {
output.push(text);
if ( silent === false ) {
console.log(text);
}
};
// Get manifest content
const manifest = await fs.readFile(
`${outputDir}/manifest.json`,
{ encoding: 'utf8' }
).then(text =>
JSON.parse(text)
);
// Create unique version number according to build time
let version = manifest.version;
{
const now = new Date();
const yearPart = now.getUTCFullYear() - 2000;
const monthPart = (now.getUTCMonth() + 1) * 1000;
const dayPart = now.getUTCDate() * 10;
const hourPart = Math.floor(now.getUTCHours() / 3) + 1;
version += `.${yearPart}.${monthPart + dayPart + hourPart}`;
}
log(`Version: ${version}`);
let goodTotalCount = 0;
let maybeGoodTotalCount = 0;
const replacer = (k, v) => {
if ( k.startsWith('__') ) { return; }
if ( Array.isArray(v) ) {
return v.sort();
}
if ( v instanceof Object ) {
const sorted = {};
for ( const kk of Object.keys(v).sort() ) {
sorted[kk] = v[kk];
}
return sorted;
}
return v;
};
const isUnsupported = rule =>
rule._error !== undefined;
const isRegex = rule =>
rule.condition !== undefined &&
rule.condition.regexFilter !== undefined;
const isRedirect = rule =>
rule.action !== undefined &&
rule.action.type === 'redirect' &&
rule.action.redirect.extensionPath !== undefined;
const isCsp = rule =>
rule.action !== undefined &&
rule.action.type === 'modifyHeaders';
const isRemoveparam = rule =>
rule.action !== undefined &&
rule.action.type === 'redirect' &&
rule.action.redirect.transform !== undefined;
const isGood = rule =>
isUnsupported(rule) === false &&
isRedirect(rule) === false &&
isCsp(rule) === false &&
isRemoveparam(rule) === false
;
const rulesetDir = `${outputDir}/rulesets`;
const rulesetDirPromise = fs.mkdir(`${rulesetDir}`, { recursive: true });
const fetchList = url => {
return new Promise((resolve, reject) => {
https.get(url, response => {
const data = [];
response.on('data', chunk => {
data.push(chunk.toString());
});
response.on('end', ( ) => {
resolve({ name: url, text: data.join('') });
});
}).on('error', error => {
reject(error);
});
});
};
const readList = path =>
fs.readFile(path, { encoding: 'utf8' })
.then(text => ({ name: path, text }));
const writeFile = (path, data) =>
rulesetDirPromise.then(( ) =>
fs.writeFile(path, data));
for ( const ruleset of rulesetConfigs ) {
const lists = [];
log('============================');
log(`Listset for '${ruleset.id}':`);
if ( Array.isArray(ruleset.paths) ) {
for ( const path of ruleset.paths ) {
log(`\t${path}`);
lists.push(readList(`assets/${path}`));
}
}
if ( Array.isArray(ruleset.urls) ) {
for ( const url of ruleset.urls ) {
log(`\t${url}`);
lists.push(fetchList(url));
}
}
const details = await dnrRulesetFromRawLists(lists, {
env: [ 'chromium' ],
});
const { ruleset: rules } = details;
log(`Input filter count: ${details.filterCount}`);
log(`\tAccepted filter count: ${details.acceptedFilterCount}`);
log(`\tRejected filter count: ${details.rejectedFilterCount}`);
log(`Output rule count: ${rules.length}`);
const good = rules.filter(rule => isGood(rule) && isRegex(rule) === false);
log(`\tGood: ${good.length}`);
const regexes = rules.filter(rule => isGood(rule) && isRegex(rule));
log(`\tMaybe good (regexes): ${regexes.length}`);
const redirects = rules.filter(rule =>
isUnsupported(rule) === false &&
isRedirect(rule)
);
log(`\tredirect-rule= (discarded): ${redirects.length}`);
const headers = rules.filter(rule =>
isUnsupported(rule) === false &&
isCsp(rule)
);
log(`\tcsp= (discarded): ${headers.length}`);
const removeparams = rules.filter(rule =>
isUnsupported(rule) === false &&
isRemoveparam(rule)
);
log(`\tremoveparams= (discarded): ${removeparams.length}`);
const bad = rules.filter(rule =>
isUnsupported(rule)
);
log(`\tUnsupported: ${bad.length}`);
log(
bad.map(rule => rule._error.map(v => `\t\t${v}`)).join('\n'),
true
);
writeOps.push(
writeFile(
`${rulesetDir}/${ruleset.id}.json`,
`${JSON.stringify(good, replacer, 2)}\n`
)
);
rulesetDetails.push({
id: ruleset.id,
name: ruleset.name,
enabled: ruleset.enabled,
filters: {
total: details.filterCount,
accepted: details.acceptedFilterCount,
rejected: details.rejectedFilterCount,
},
rules: {
total: rules.length,
accepted: good.length,
discarded: redirects.length + headers.length + removeparams.length,
rejected: bad.length,
regexes,
},
});
ruleResources.push({
id: ruleset.id,
enabled: ruleset.enabled,
path: `/rulesets/${ruleset.id}.json`
});
goodTotalCount += good.length;
maybeGoodTotalCount += regexes.length;
}
writeOps.push(
writeFile(
`${rulesetDir}/ruleset-details.js`,
`export default ${JSON.stringify(rulesetDetails, replacer, 2)};\n`
)
);
await Promise.all(writeOps);
log(`Total good rules count: ${goodTotalCount}`);
log(`Total regex rules count: ${maybeGoodTotalCount}`);
// Patch manifest
manifest.declarative_net_request = { rule_resources: ruleResources };
const now = new Date();
const yearPart = now.getUTCFullYear() - 2000;
const monthPart = (now.getUTCMonth() + 1) * 1000;
const dayPart = now.getUTCDate() * 10;
const hourPart = Math.floor(now.getUTCHours() / 3) + 1;
manifest.version = manifest.version + `.${yearPart}.${monthPart + dayPart + hourPart}`;
await fs.writeFile(
`${outputDir}/manifest.json`,
JSON.stringify(manifest, null, 2) + '\n'
);
// Log results
await fs.writeFile(`${outputDir}/log.txt`, output.join('\n') + '\n');
}
main();
/******************************************************************************/