[mv3] Add support for converting `^responseheader()` filters to DNR

Additionally, finalize versioning scheme for uBOL. Since most updates
will be simply related to update rulesets, the version will from now
on reflects the date at which the extension package was created:

  year.month.day.minutes

So for example:

  2023.8.19.690
This commit is contained in:
Raymond Hill 2023-08-19 07:48:14 -04:00
parent eb235404bd
commit 857abb380b
No known key found for this signature in database
GPG Key ID: 25E1490B761470C2
3 changed files with 208 additions and 170 deletions

View File

@ -320,18 +320,19 @@ async function processNetworkFilters(assetDetails, network) {
log(`Output rule count: ${rules.length}`);
// https://developer.mozilla.org/en-US/docs/Mozilla/Add-ons/WebExtensions/API/declarativeNetRequest/RuleCondition#browser_compatibility
// isUrlFilterCaseSensitive is false by default in Firefox. It will be
// isUrlFilterCaseSensitive is true by default in Chromium. It will be
// false by default in Chromium 118+.
if ( platform === 'firefox' ) {
if ( platform !== 'firefox' ) {
for ( const rule of rules ) {
if ( rule.condition === undefined ) { continue; }
if ( rule.condition.urlFilter === undefined ) {
if ( rule.condition.regexFilter === undefined ) { continue; }
const { condition } = rule;
if ( condition === undefined ) { continue; }
if ( condition.urlFilter === undefined ) {
if ( condition.regexFilter === undefined ) { continue; }
}
if ( rule.condition.isUrlFilterCaseSensitive === undefined ) {
rule.condition.isUrlFilterCaseSensitive = true;
} else if ( rule.condition.isUrlFilterCaseSensitive === false ) {
rule.condition.isUrlFilterCaseSensitive = undefined;
if ( condition.isUrlFilterCaseSensitive === undefined ) {
condition.isUrlFilterCaseSensitive = false;
} else if ( condition.isUrlFilterCaseSensitive === true ) {
condition.isUrlFilterCaseSensitive = undefined;
}
}
}
@ -1098,23 +1099,15 @@ async function rulesetFromURLs(assetDetails) {
async function main() {
// Get manifest content
const manifest = await fs.readFile(
`${outputDir}/manifest.json`,
{ encoding: 'utf8' }
).then(text =>
JSON.parse(text)
);
// Create unique version number according to build time
let version = manifest.version;
let version = '';
{
const now = new Date();
const yearPart = now.getUTCFullYear() - 2000;
const monthPart = (now.getUTCMonth() + 1) * 1000;
const dayPart = now.getUTCDate() * 10;
const hourPart = Math.floor(now.getUTCHours() / 3) + 1;
version += `.${yearPart}.${monthPart + dayPart + hourPart}`;
const yearPart = now.getUTCFullYear();
const monthPart = now.getUTCMonth() + 1;
const dayPart = now.getUTCDate();
const hourPart = Math.floor(now.getUTCHours());
const minutePart = Math.floor(now.getUTCMinutes());
version = `${yearPart}.${monthPart}.${dayPart}.${hourPart * 60 + minutePart}`;
}
log(`Version: ${version}`);
@ -1300,6 +1293,13 @@ async function main() {
await Promise.all(writeOps);
// Patch manifest
// Get manifest content
const manifest = await fs.readFile(
`${outputDir}/manifest.json`,
{ encoding: 'utf8' }
).then(text =>
JSON.parse(text)
);
// Patch declarative_net_request key
manifest.declarative_net_request = { rule_resources: ruleResources };
// Patch web_accessible_resources key
@ -1312,13 +1312,8 @@ async function main() {
}
manifest.web_accessible_resources = [ web_accessible_resources ];
// Patch version key
const now = new Date();
const yearPart = now.getUTCFullYear() - 2000;
const monthPart = (now.getUTCMonth() + 1) * 1000;
const dayPart = now.getUTCDate() * 10;
const hourPart = Math.floor(now.getUTCHours() / 3) + 1;
manifest.version = manifest.version + `.${yearPart}.${monthPart + dayPart + hourPart}`;
// Patch manifest version property
manifest.version = version;
// Commit changes
await fs.writeFile(
`${outputDir}/manifest.json`,

View File

@ -134,7 +134,54 @@ function addExtendedToDNR(context, parser) {
}
// Response header filtering
if ( (parser.flavorBits & parser.BITFlavorExtResponseHeader) !== 0 ) {
if ( parser.isResponseheaderFilter() ) {
if ( parser.hasError() ) { return; }
if ( parser.hasOptions() === false ) { return; }
if ( parser.isException() ) { return; }
const node = parser.getBranchFromType(sfp.NODE_TYPE_EXT_PATTERN_RESPONSEHEADER);
if ( node === 0 ) { return; }
const header = parser.getNodeString(node);
if ( context.responseHeaderRules === undefined ) {
context.responseHeaderRules = [];
}
const rule = {
action: {
responseHeaders: [
{
header,
operation: 'remove',
}
],
type: 'modifyHeaders'
},
condition: {
resourceTypes: [
'main_frame',
'sub_frame'
]
},
};
for ( const { hn, not, bad } of parser.getExtFilterDomainIterator() ) {
if ( bad ) { continue; }
if ( not ) {
if ( rule.condition.excludedInitiatorDomains === undefined ) {
rule.condition.excludedInitiatorDomains = [];
}
rule.condition.excludedInitiatorDomains.push(hn);
continue;
}
if ( hn === '*' ) {
if ( rule.condition.initiatorDomains !== undefined ) {
rule.condition.initiatorDomains = undefined;
}
continue;
}
if ( rule.condition.initiatorDomains === undefined ) {
rule.condition.initiatorDomains = [];
}
rule.condition.initiatorDomains.push(hn);
}
context.responseHeaderRules.push(rule);
return;
}
@ -286,6 +333,129 @@ function addToDNR(context, list) {
/******************************************************************************/
function finalizeRuleset(context, network) {
const ruleset = network.ruleset;
// Assign rule ids
const rulesetMap = new Map();
{
let ruleId = 1;
for ( const rule of ruleset ) {
rulesetMap.set(ruleId++, rule);
}
}
// Merge rules where possible by merging arrays of a specific property.
//
// https://github.com/uBlockOrigin/uBOL-issues/issues/10#issuecomment-1304822579
// Do not merge rules which have errors.
const mergeRules = (rulesetMap, mergeTarget) => {
const mergeMap = new Map();
const sorter = (_, v) => {
if ( Array.isArray(v) ) {
return typeof v[0] === 'string' ? v.sort() : v;
}
if ( v instanceof Object ) {
const sorted = {};
for ( const kk of Object.keys(v).sort() ) {
sorted[kk] = v[kk];
}
return sorted;
}
return v;
};
const ruleHasher = (rule, target) => {
return JSON.stringify(rule, (k, v) => {
if ( k.startsWith('_') ) { return; }
if ( k === target ) { return; }
return sorter(k, v);
});
};
const extractTargetValue = (obj, target) => {
for ( const [ k, v ] of Object.entries(obj) ) {
if ( Array.isArray(v) && k === target ) { return v; }
if ( v instanceof Object ) {
const r = extractTargetValue(v, target);
if ( r !== undefined ) { return r; }
}
}
};
const extractTargetOwner = (obj, target) => {
for ( const [ k, v ] of Object.entries(obj) ) {
if ( Array.isArray(v) && k === target ) { return obj; }
if ( v instanceof Object ) {
const r = extractTargetOwner(v, target);
if ( r !== undefined ) { return r; }
}
}
};
for ( const [ id, rule ] of rulesetMap ) {
if ( rule._error !== undefined ) { continue; }
const hash = ruleHasher(rule, mergeTarget);
if ( mergeMap.has(hash) === false ) {
mergeMap.set(hash, []);
}
mergeMap.get(hash).push(id);
}
for ( const ids of mergeMap.values() ) {
if ( ids.length === 1 ) { continue; }
const leftHand = rulesetMap.get(ids[0]);
const leftHandSet = new Set(
extractTargetValue(leftHand, mergeTarget) || []
);
for ( let i = 1; i < ids.length; i++ ) {
const rightHandId = ids[i];
const rightHand = rulesetMap.get(rightHandId);
const rightHandArray = extractTargetValue(rightHand, mergeTarget);
if ( rightHandArray !== undefined ) {
if ( leftHandSet.size !== 0 ) {
for ( const item of rightHandArray ) {
leftHandSet.add(item);
}
}
} else {
leftHandSet.clear();
}
rulesetMap.delete(rightHandId);
}
const leftHandOwner = extractTargetOwner(leftHand, mergeTarget);
if ( leftHandSet.size > 1 ) {
//if ( leftHandOwner === undefined ) { debugger; }
leftHandOwner[mergeTarget] = Array.from(leftHandSet).sort();
} else if ( leftHandSet.size === 0 ) {
if ( leftHandOwner !== undefined ) {
leftHandOwner[mergeTarget] = undefined;
}
}
}
};
mergeRules(rulesetMap, 'resourceTypes');
mergeRules(rulesetMap, 'initiatorDomains');
mergeRules(rulesetMap, 'requestDomains');
mergeRules(rulesetMap, 'removeParams');
mergeRules(rulesetMap, 'responseHeaders');
// Patch id
const rulesetFinal = [];
{
let ruleId = 1;
for ( const rule of rulesetMap.values() ) {
if ( rule._error === undefined ) {
rule.id = ruleId++;
} else {
rule.id = 0;
}
rulesetFinal.push(rule);
}
for ( const invalid of context.invalid ) {
rulesetFinal.push({ _error: [ invalid ] });
}
}
network.ruleset = rulesetFinal;
}
/******************************************************************************/
async function dnrRulesetFromRawLists(lists, options = {}) {
const context = Object.assign({}, options);
staticNetFilteringEngine.dnrFromCompiled('begin', context);
@ -300,8 +470,7 @@ async function dnrRulesetFromRawLists(lists, options = {}) {
}
}
await Promise.all(toLoad);
return {
const result = {
network: staticNetFilteringEngine.dnrFromCompiled('end', context),
genericCosmetic: context.genericCosmeticFilters,
genericHighCosmetic: context.genericHighCosmeticFilters,
@ -309,6 +478,11 @@ async function dnrRulesetFromRawLists(lists, options = {}) {
specificCosmetic: context.specificCosmeticFilters,
scriptlet: context.scriptletFilters,
};
if ( context.responseHeaderRules ) {
result.network.ruleset.push(...context.responseHeaderRules);
}
finalizeRuleset(context, result.network);
return result;
}
/******************************************************************************/

View File

@ -1274,7 +1274,9 @@ class FilterRegex {
dnrAddRuleError(rule, `regexFilter is not RE2-compatible: ${args[1]}`);
}
rule.condition.regexFilter = args[1];
rule.condition.isUrlFilterCaseSensitive = args[2] === 1;
if ( args[2] === 1 ) {
rule.condition.isUrlFilterCaseSensitive = true;
}
}
static keyFromArgs(args) {
@ -4349,7 +4351,7 @@ FilterContainer.prototype.dnrFromCompiled = function(op, context, ...args) {
'ping',
'other',
]);
let ruleset = [];
const ruleset = [];
for ( const [ realmBits, realmName ] of realms ) {
for ( const [ partyBits, partyName ] of partyness ) {
for ( const typeName in typeNameToTypeValue ) {
@ -4521,141 +4523,8 @@ FilterContainer.prototype.dnrFromCompiled = function(op, context, ...args) {
}
}
// Assign rule ids
const rulesetMap = new Map();
{
let ruleId = 1;
for ( const rule of ruleset ) {
rulesetMap.set(ruleId++, rule);
}
}
// Merge rules where possible by merging arrays of a specific property.
//
// https://github.com/uBlockOrigin/uBOL-issues/issues/10#issuecomment-1304822579
// Do not merge rules which have errors.
const mergeRules = (rulesetMap, mergeTarget) => {
const mergeMap = new Map();
const sorter = (_, v) => {
if ( Array.isArray(v) ) {
return typeof v[0] === 'string' ? v.sort() : v;
}
if ( v instanceof Object ) {
const sorted = {};
for ( const kk of Object.keys(v).sort() ) {
sorted[kk] = v[kk];
}
return sorted;
}
return v;
};
const ruleHasher = (rule, target) => {
return JSON.stringify(rule, (k, v) => {
if ( k.startsWith('_') ) { return; }
if ( k === target ) { return; }
return sorter(k, v);
});
};
const extractTargetValue = (obj, target) => {
for ( const [ k, v ] of Object.entries(obj) ) {
if ( Array.isArray(v) && k === target ) { return v; }
if ( v instanceof Object ) {
const r = extractTargetValue(v, target);
if ( r !== undefined ) { return r; }
}
}
};
const extractTargetOwner = (obj, target) => {
for ( const [ k, v ] of Object.entries(obj) ) {
if ( Array.isArray(v) && k === target ) { return obj; }
if ( v instanceof Object ) {
const r = extractTargetOwner(v, target);
if ( r !== undefined ) { return r; }
}
}
};
for ( const [ id, rule ] of rulesetMap ) {
if ( rule._error !== undefined ) { continue; }
const hash = ruleHasher(rule, mergeTarget);
if ( mergeMap.has(hash) === false ) {
mergeMap.set(hash, []);
}
mergeMap.get(hash).push(id);
}
for ( const ids of mergeMap.values() ) {
if ( ids.length === 1 ) { continue; }
const leftHand = rulesetMap.get(ids[0]);
const leftHandSet = new Set(
extractTargetValue(leftHand, mergeTarget) || []
);
for ( let i = 1; i < ids.length; i++ ) {
const rightHandId = ids[i];
const rightHand = rulesetMap.get(rightHandId);
const rightHandArray = extractTargetValue(rightHand, mergeTarget);
if ( rightHandArray !== undefined ) {
if ( leftHandSet.size !== 0 ) {
for ( const item of rightHandArray ) {
leftHandSet.add(item);
}
}
} else {
leftHandSet.clear();
}
rulesetMap.delete(rightHandId);
}
const leftHandOwner = extractTargetOwner(leftHand, mergeTarget);
if ( leftHandSet.size > 1 ) {
//if ( leftHandOwner === undefined ) { debugger; }
leftHandOwner[mergeTarget] = Array.from(leftHandSet).sort();
} else if ( leftHandSet.size === 0 ) {
if ( leftHandOwner !== undefined ) {
leftHandOwner[mergeTarget] = undefined;
}
}
}
};
mergeRules(rulesetMap, 'resourceTypes');
mergeRules(rulesetMap, 'initiatorDomains');
mergeRules(rulesetMap, 'requestDomains');
mergeRules(rulesetMap, 'removeParams');
mergeRules(rulesetMap, 'responseHeaders');
// Patch case-sensitiveness
for ( const rule of rulesetMap.values() ) {
const { condition } = rule;
if (
condition === undefined ||
condition.urlFilter === undefined &&
condition.regexFilter === undefined
) {
continue;
}
if ( condition.isUrlFilterCaseSensitive === undefined ) {
condition.isUrlFilterCaseSensitive = false;
} else if ( condition.isUrlFilterCaseSensitive === true ) {
condition.isUrlFilterCaseSensitive = undefined;
}
}
// Patch id
const rulesetFinal = [];
{
let ruleId = 1;
for ( const rule of rulesetMap.values() ) {
if ( rule._error === undefined ) {
rule.id = ruleId++;
} else {
rule.id = 0;
}
rulesetFinal.push(rule);
}
for ( const invalid of context.invalid ) {
rulesetFinal.push({ _error: [ invalid ] });
}
}
return {
ruleset: rulesetFinal,
ruleset,
filterCount: context.filterCount,
acceptedFilterCount: context.acceptedFilterCount,
rejectedFilterCount: context.rejectedFilterCount,