mirror of https://github.com/gorhill/uBlock.git
[mv3] Add support for converting `^responseheader()` filters to DNR
Additionally, finalize versioning scheme for uBOL. Since most updates will be simply related to update rulesets, the version will from now on reflects the date at which the extension package was created: year.month.day.minutes So for example: 2023.8.19.690
This commit is contained in:
Raymond Hill
parent
eb235404bd
commit
857abb380b
|
|
@ -320,18 +320,19 @@ async function processNetworkFilters(assetDetails, network) {
|
|||
log(`Output rule count: ${rules.length}`);
|
||||
|
||||
// https://developer.mozilla.org/en-US/docs/Mozilla/Add-ons/WebExtensions/API/declarativeNetRequest/RuleCondition#browser_compatibility
|
||||
// isUrlFilterCaseSensitive is false by default in Firefox. It will be
|
||||
// isUrlFilterCaseSensitive is true by default in Chromium. It will be
|
||||
// false by default in Chromium 118+.
|
||||
if ( platform === 'firefox' ) {
|
||||
if ( platform !== 'firefox' ) {
|
||||
for ( const rule of rules ) {
|
||||
if ( rule.condition === undefined ) { continue; }
|
||||
if ( rule.condition.urlFilter === undefined ) {
|
||||
if ( rule.condition.regexFilter === undefined ) { continue; }
|
||||
const { condition } = rule;
|
||||
if ( condition === undefined ) { continue; }
|
||||
if ( condition.urlFilter === undefined ) {
|
||||
if ( condition.regexFilter === undefined ) { continue; }
|
||||
}
|
||||
if ( rule.condition.isUrlFilterCaseSensitive === undefined ) {
|
||||
rule.condition.isUrlFilterCaseSensitive = true;
|
||||
} else if ( rule.condition.isUrlFilterCaseSensitive === false ) {
|
||||
rule.condition.isUrlFilterCaseSensitive = undefined;
|
||||
if ( condition.isUrlFilterCaseSensitive === undefined ) {
|
||||
condition.isUrlFilterCaseSensitive = false;
|
||||
} else if ( condition.isUrlFilterCaseSensitive === true ) {
|
||||
condition.isUrlFilterCaseSensitive = undefined;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1098,23 +1099,15 @@ async function rulesetFromURLs(assetDetails) {
|
|||
|
||||
async function main() {
|
||||
|
||||
// Get manifest content
|
||||
const manifest = await fs.readFile(
|
||||
`${outputDir}/manifest.json`,
|
||||
{ encoding: 'utf8' }
|
||||
).then(text =>
|
||||
JSON.parse(text)
|
||||
);
|
||||
|
||||
// Create unique version number according to build time
|
||||
let version = manifest.version;
|
||||
let version = '';
|
||||
{
|
||||
const now = new Date();
|
||||
const yearPart = now.getUTCFullYear() - 2000;
|
||||
const monthPart = (now.getUTCMonth() + 1) * 1000;
|
||||
const dayPart = now.getUTCDate() * 10;
|
||||
const hourPart = Math.floor(now.getUTCHours() / 3) + 1;
|
||||
version += `.${yearPart}.${monthPart + dayPart + hourPart}`;
|
||||
const yearPart = now.getUTCFullYear();
|
||||
const monthPart = now.getUTCMonth() + 1;
|
||||
const dayPart = now.getUTCDate();
|
||||
const hourPart = Math.floor(now.getUTCHours());
|
||||
const minutePart = Math.floor(now.getUTCMinutes());
|
||||
version = `${yearPart}.${monthPart}.${dayPart}.${hourPart * 60 + minutePart}`;
|
||||
}
|
||||
log(`Version: ${version}`);
|
||||
|
||||
|
|
@ -1300,6 +1293,13 @@ async function main() {
|
|||
await Promise.all(writeOps);
|
||||
|
||||
// Patch manifest
|
||||
// Get manifest content
|
||||
const manifest = await fs.readFile(
|
||||
`${outputDir}/manifest.json`,
|
||||
{ encoding: 'utf8' }
|
||||
).then(text =>
|
||||
JSON.parse(text)
|
||||
);
|
||||
// Patch declarative_net_request key
|
||||
manifest.declarative_net_request = { rule_resources: ruleResources };
|
||||
// Patch web_accessible_resources key
|
||||
|
|
@ -1312,13 +1312,8 @@ async function main() {
|
|||
}
|
||||
manifest.web_accessible_resources = [ web_accessible_resources ];
|
||||
|
||||
// Patch version key
|
||||
const now = new Date();
|
||||
const yearPart = now.getUTCFullYear() - 2000;
|
||||
const monthPart = (now.getUTCMonth() + 1) * 1000;
|
||||
const dayPart = now.getUTCDate() * 10;
|
||||
const hourPart = Math.floor(now.getUTCHours() / 3) + 1;
|
||||
manifest.version = manifest.version + `.${yearPart}.${monthPart + dayPart + hourPart}`;
|
||||
// Patch manifest version property
|
||||
manifest.version = version;
|
||||
// Commit changes
|
||||
await fs.writeFile(
|
||||
`${outputDir}/manifest.json`,
|
||||
|
|
|
|||
|
|
@ -134,7 +134,54 @@ function addExtendedToDNR(context, parser) {
|
|||
}
|
||||
|
||||
// Response header filtering
|
||||
if ( (parser.flavorBits & parser.BITFlavorExtResponseHeader) !== 0 ) {
|
||||
if ( parser.isResponseheaderFilter() ) {
|
||||
if ( parser.hasError() ) { return; }
|
||||
if ( parser.hasOptions() === false ) { return; }
|
||||
if ( parser.isException() ) { return; }
|
||||
const node = parser.getBranchFromType(sfp.NODE_TYPE_EXT_PATTERN_RESPONSEHEADER);
|
||||
if ( node === 0 ) { return; }
|
||||
const header = parser.getNodeString(node);
|
||||
if ( context.responseHeaderRules === undefined ) {
|
||||
context.responseHeaderRules = [];
|
||||
}
|
||||
const rule = {
|
||||
action: {
|
||||
responseHeaders: [
|
||||
{
|
||||
header,
|
||||
operation: 'remove',
|
||||
}
|
||||
],
|
||||
type: 'modifyHeaders'
|
||||
},
|
||||
condition: {
|
||||
resourceTypes: [
|
||||
'main_frame',
|
||||
'sub_frame'
|
||||
]
|
||||
},
|
||||
};
|
||||
for ( const { hn, not, bad } of parser.getExtFilterDomainIterator() ) {
|
||||
if ( bad ) { continue; }
|
||||
if ( not ) {
|
||||
if ( rule.condition.excludedInitiatorDomains === undefined ) {
|
||||
rule.condition.excludedInitiatorDomains = [];
|
||||
}
|
||||
rule.condition.excludedInitiatorDomains.push(hn);
|
||||
continue;
|
||||
}
|
||||
if ( hn === '*' ) {
|
||||
if ( rule.condition.initiatorDomains !== undefined ) {
|
||||
rule.condition.initiatorDomains = undefined;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if ( rule.condition.initiatorDomains === undefined ) {
|
||||
rule.condition.initiatorDomains = [];
|
||||
}
|
||||
rule.condition.initiatorDomains.push(hn);
|
||||
}
|
||||
context.responseHeaderRules.push(rule);
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -286,6 +333,129 @@ function addToDNR(context, list) {
|
|||
|
||||
/******************************************************************************/
|
||||
|
||||
function finalizeRuleset(context, network) {
|
||||
const ruleset = network.ruleset;
|
||||
|
||||
// Assign rule ids
|
||||
const rulesetMap = new Map();
|
||||
{
|
||||
let ruleId = 1;
|
||||
for ( const rule of ruleset ) {
|
||||
rulesetMap.set(ruleId++, rule);
|
||||
}
|
||||
}
|
||||
// Merge rules where possible by merging arrays of a specific property.
|
||||
//
|
||||
// https://github.com/uBlockOrigin/uBOL-issues/issues/10#issuecomment-1304822579
|
||||
// Do not merge rules which have errors.
|
||||
const mergeRules = (rulesetMap, mergeTarget) => {
|
||||
const mergeMap = new Map();
|
||||
const sorter = (_, v) => {
|
||||
if ( Array.isArray(v) ) {
|
||||
return typeof v[0] === 'string' ? v.sort() : v;
|
||||
}
|
||||
if ( v instanceof Object ) {
|
||||
const sorted = {};
|
||||
for ( const kk of Object.keys(v).sort() ) {
|
||||
sorted[kk] = v[kk];
|
||||
}
|
||||
return sorted;
|
||||
}
|
||||
return v;
|
||||
};
|
||||
const ruleHasher = (rule, target) => {
|
||||
return JSON.stringify(rule, (k, v) => {
|
||||
if ( k.startsWith('_') ) { return; }
|
||||
if ( k === target ) { return; }
|
||||
return sorter(k, v);
|
||||
});
|
||||
};
|
||||
const extractTargetValue = (obj, target) => {
|
||||
for ( const [ k, v ] of Object.entries(obj) ) {
|
||||
if ( Array.isArray(v) && k === target ) { return v; }
|
||||
if ( v instanceof Object ) {
|
||||
const r = extractTargetValue(v, target);
|
||||
if ( r !== undefined ) { return r; }
|
||||
}
|
||||
}
|
||||
};
|
||||
const extractTargetOwner = (obj, target) => {
|
||||
for ( const [ k, v ] of Object.entries(obj) ) {
|
||||
if ( Array.isArray(v) && k === target ) { return obj; }
|
||||
if ( v instanceof Object ) {
|
||||
const r = extractTargetOwner(v, target);
|
||||
if ( r !== undefined ) { return r; }
|
||||
}
|
||||
}
|
||||
};
|
||||
for ( const [ id, rule ] of rulesetMap ) {
|
||||
if ( rule._error !== undefined ) { continue; }
|
||||
const hash = ruleHasher(rule, mergeTarget);
|
||||
if ( mergeMap.has(hash) === false ) {
|
||||
mergeMap.set(hash, []);
|
||||
}
|
||||
mergeMap.get(hash).push(id);
|
||||
}
|
||||
for ( const ids of mergeMap.values() ) {
|
||||
if ( ids.length === 1 ) { continue; }
|
||||
const leftHand = rulesetMap.get(ids[0]);
|
||||
const leftHandSet = new Set(
|
||||
extractTargetValue(leftHand, mergeTarget) || []
|
||||
);
|
||||
for ( let i = 1; i < ids.length; i++ ) {
|
||||
const rightHandId = ids[i];
|
||||
const rightHand = rulesetMap.get(rightHandId);
|
||||
const rightHandArray = extractTargetValue(rightHand, mergeTarget);
|
||||
if ( rightHandArray !== undefined ) {
|
||||
if ( leftHandSet.size !== 0 ) {
|
||||
for ( const item of rightHandArray ) {
|
||||
leftHandSet.add(item);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
leftHandSet.clear();
|
||||
}
|
||||
rulesetMap.delete(rightHandId);
|
||||
}
|
||||
const leftHandOwner = extractTargetOwner(leftHand, mergeTarget);
|
||||
if ( leftHandSet.size > 1 ) {
|
||||
//if ( leftHandOwner === undefined ) { debugger; }
|
||||
leftHandOwner[mergeTarget] = Array.from(leftHandSet).sort();
|
||||
} else if ( leftHandSet.size === 0 ) {
|
||||
if ( leftHandOwner !== undefined ) {
|
||||
leftHandOwner[mergeTarget] = undefined;
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
mergeRules(rulesetMap, 'resourceTypes');
|
||||
mergeRules(rulesetMap, 'initiatorDomains');
|
||||
mergeRules(rulesetMap, 'requestDomains');
|
||||
mergeRules(rulesetMap, 'removeParams');
|
||||
mergeRules(rulesetMap, 'responseHeaders');
|
||||
|
||||
// Patch id
|
||||
const rulesetFinal = [];
|
||||
{
|
||||
let ruleId = 1;
|
||||
for ( const rule of rulesetMap.values() ) {
|
||||
if ( rule._error === undefined ) {
|
||||
rule.id = ruleId++;
|
||||
} else {
|
||||
rule.id = 0;
|
||||
}
|
||||
rulesetFinal.push(rule);
|
||||
}
|
||||
for ( const invalid of context.invalid ) {
|
||||
rulesetFinal.push({ _error: [ invalid ] });
|
||||
}
|
||||
}
|
||||
|
||||
network.ruleset = rulesetFinal;
|
||||
}
|
||||
|
||||
/******************************************************************************/
|
||||
|
||||
async function dnrRulesetFromRawLists(lists, options = {}) {
|
||||
const context = Object.assign({}, options);
|
||||
staticNetFilteringEngine.dnrFromCompiled('begin', context);
|
||||
|
|
@ -300,8 +470,7 @@ async function dnrRulesetFromRawLists(lists, options = {}) {
|
|||
}
|
||||
}
|
||||
await Promise.all(toLoad);
|
||||
|
||||
return {
|
||||
const result = {
|
||||
network: staticNetFilteringEngine.dnrFromCompiled('end', context),
|
||||
genericCosmetic: context.genericCosmeticFilters,
|
||||
genericHighCosmetic: context.genericHighCosmeticFilters,
|
||||
|
|
@ -309,6 +478,11 @@ async function dnrRulesetFromRawLists(lists, options = {}) {
|
|||
specificCosmetic: context.specificCosmeticFilters,
|
||||
scriptlet: context.scriptletFilters,
|
||||
};
|
||||
if ( context.responseHeaderRules ) {
|
||||
result.network.ruleset.push(...context.responseHeaderRules);
|
||||
}
|
||||
finalizeRuleset(context, result.network);
|
||||
return result;
|
||||
}
|
||||
|
||||
/******************************************************************************/
|
||||
|
|
|
|||
|
|
@ -1274,7 +1274,9 @@ class FilterRegex {
|
|||
dnrAddRuleError(rule, `regexFilter is not RE2-compatible: ${args[1]}`);
|
||||
}
|
||||
rule.condition.regexFilter = args[1];
|
||||
rule.condition.isUrlFilterCaseSensitive = args[2] === 1;
|
||||
if ( args[2] === 1 ) {
|
||||
rule.condition.isUrlFilterCaseSensitive = true;
|
||||
}
|
||||
}
|
||||
|
||||
static keyFromArgs(args) {
|
||||
|
|
@ -4349,7 +4351,7 @@ FilterContainer.prototype.dnrFromCompiled = function(op, context, ...args) {
|
|||
'ping',
|
||||
'other',
|
||||
]);
|
||||
let ruleset = [];
|
||||
const ruleset = [];
|
||||
for ( const [ realmBits, realmName ] of realms ) {
|
||||
for ( const [ partyBits, partyName ] of partyness ) {
|
||||
for ( const typeName in typeNameToTypeValue ) {
|
||||
|
|
@ -4521,141 +4523,8 @@ FilterContainer.prototype.dnrFromCompiled = function(op, context, ...args) {
|
|||
}
|
||||
}
|
||||
|
||||
// Assign rule ids
|
||||
const rulesetMap = new Map();
|
||||
{
|
||||
let ruleId = 1;
|
||||
for ( const rule of ruleset ) {
|
||||
rulesetMap.set(ruleId++, rule);
|
||||
}
|
||||
}
|
||||
|
||||
// Merge rules where possible by merging arrays of a specific property.
|
||||
//
|
||||
// https://github.com/uBlockOrigin/uBOL-issues/issues/10#issuecomment-1304822579
|
||||
// Do not merge rules which have errors.
|
||||
const mergeRules = (rulesetMap, mergeTarget) => {
|
||||
const mergeMap = new Map();
|
||||
const sorter = (_, v) => {
|
||||
if ( Array.isArray(v) ) {
|
||||
return typeof v[0] === 'string' ? v.sort() : v;
|
||||
}
|
||||
if ( v instanceof Object ) {
|
||||
const sorted = {};
|
||||
for ( const kk of Object.keys(v).sort() ) {
|
||||
sorted[kk] = v[kk];
|
||||
}
|
||||
return sorted;
|
||||
}
|
||||
return v;
|
||||
};
|
||||
const ruleHasher = (rule, target) => {
|
||||
return JSON.stringify(rule, (k, v) => {
|
||||
if ( k.startsWith('_') ) { return; }
|
||||
if ( k === target ) { return; }
|
||||
return sorter(k, v);
|
||||
});
|
||||
};
|
||||
const extractTargetValue = (obj, target) => {
|
||||
for ( const [ k, v ] of Object.entries(obj) ) {
|
||||
if ( Array.isArray(v) && k === target ) { return v; }
|
||||
if ( v instanceof Object ) {
|
||||
const r = extractTargetValue(v, target);
|
||||
if ( r !== undefined ) { return r; }
|
||||
}
|
||||
}
|
||||
};
|
||||
const extractTargetOwner = (obj, target) => {
|
||||
for ( const [ k, v ] of Object.entries(obj) ) {
|
||||
if ( Array.isArray(v) && k === target ) { return obj; }
|
||||
if ( v instanceof Object ) {
|
||||
const r = extractTargetOwner(v, target);
|
||||
if ( r !== undefined ) { return r; }
|
||||
}
|
||||
}
|
||||
};
|
||||
for ( const [ id, rule ] of rulesetMap ) {
|
||||
if ( rule._error !== undefined ) { continue; }
|
||||
const hash = ruleHasher(rule, mergeTarget);
|
||||
if ( mergeMap.has(hash) === false ) {
|
||||
mergeMap.set(hash, []);
|
||||
}
|
||||
mergeMap.get(hash).push(id);
|
||||
}
|
||||
for ( const ids of mergeMap.values() ) {
|
||||
if ( ids.length === 1 ) { continue; }
|
||||
const leftHand = rulesetMap.get(ids[0]);
|
||||
const leftHandSet = new Set(
|
||||
extractTargetValue(leftHand, mergeTarget) || []
|
||||
);
|
||||
for ( let i = 1; i < ids.length; i++ ) {
|
||||
const rightHandId = ids[i];
|
||||
const rightHand = rulesetMap.get(rightHandId);
|
||||
const rightHandArray = extractTargetValue(rightHand, mergeTarget);
|
||||
if ( rightHandArray !== undefined ) {
|
||||
if ( leftHandSet.size !== 0 ) {
|
||||
for ( const item of rightHandArray ) {
|
||||
leftHandSet.add(item);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
leftHandSet.clear();
|
||||
}
|
||||
rulesetMap.delete(rightHandId);
|
||||
}
|
||||
const leftHandOwner = extractTargetOwner(leftHand, mergeTarget);
|
||||
if ( leftHandSet.size > 1 ) {
|
||||
//if ( leftHandOwner === undefined ) { debugger; }
|
||||
leftHandOwner[mergeTarget] = Array.from(leftHandSet).sort();
|
||||
} else if ( leftHandSet.size === 0 ) {
|
||||
if ( leftHandOwner !== undefined ) {
|
||||
leftHandOwner[mergeTarget] = undefined;
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
mergeRules(rulesetMap, 'resourceTypes');
|
||||
mergeRules(rulesetMap, 'initiatorDomains');
|
||||
mergeRules(rulesetMap, 'requestDomains');
|
||||
mergeRules(rulesetMap, 'removeParams');
|
||||
mergeRules(rulesetMap, 'responseHeaders');
|
||||
|
||||
// Patch case-sensitiveness
|
||||
for ( const rule of rulesetMap.values() ) {
|
||||
const { condition } = rule;
|
||||
if (
|
||||
condition === undefined ||
|
||||
condition.urlFilter === undefined &&
|
||||
condition.regexFilter === undefined
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
if ( condition.isUrlFilterCaseSensitive === undefined ) {
|
||||
condition.isUrlFilterCaseSensitive = false;
|
||||
} else if ( condition.isUrlFilterCaseSensitive === true ) {
|
||||
condition.isUrlFilterCaseSensitive = undefined;
|
||||
}
|
||||
}
|
||||
|
||||
// Patch id
|
||||
const rulesetFinal = [];
|
||||
{
|
||||
let ruleId = 1;
|
||||
for ( const rule of rulesetMap.values() ) {
|
||||
if ( rule._error === undefined ) {
|
||||
rule.id = ruleId++;
|
||||
} else {
|
||||
rule.id = 0;
|
||||
}
|
||||
rulesetFinal.push(rule);
|
||||
}
|
||||
for ( const invalid of context.invalid ) {
|
||||
rulesetFinal.push({ _error: [ invalid ] });
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
ruleset: rulesetFinal,
|
||||
ruleset,
|
||||
filterCount: context.filterCount,
|
||||
acceptedFilterCount: context.acceptedFilterCount,
|
||||
rejectedFilterCount: context.rejectedFilterCount,
|
||||
|
|
|
|||
Loading…
Reference in New Issue