[mv3] Add support for converting `^responseheader()` filters to DNR

Additionally, finalize versioning scheme for uBOL. Since most updates
will be simply related to update rulesets, the version will from now
on reflects the date at which the extension package was created:

  year.month.day.minutes

So for example:

  2023.8.19.690

platform/mv3/make-rulesets.js

@@ -320,18 +320,19 @@ async function processNetworkFilters(assetDetails, network) {
     log(`Output rule count: ${rules.length}`);
 
     // https://developer.mozilla.org/en-US/docs/Mozilla/Add-ons/WebExtensions/API/declarativeNetRequest/RuleCondition#browser_compatibility
-    // isUrlFilterCaseSensitive is false by default in Firefox. It will be
+    // isUrlFilterCaseSensitive is true by default in Chromium. It will be
     // false by default in Chromium 118+.
-    if ( platform === 'firefox' ) {
+    if ( platform !== 'firefox' ) {
         for ( const rule of rules ) {
-            if ( rule.condition === undefined ) { continue; }
+            const { condition } = rule;
-            if ( rule.condition.urlFilter === undefined ) {
+            if ( condition === undefined ) { continue; }
-                if ( rule.condition.regexFilter === undefined ) { continue; }
+            if ( condition.urlFilter === undefined ) {
+                if ( condition.regexFilter === undefined ) { continue; }
             }
-            if ( rule.condition.isUrlFilterCaseSensitive === undefined ) {
+            if ( condition.isUrlFilterCaseSensitive === undefined ) {
-                rule.condition.isUrlFilterCaseSensitive = true;
+                condition.isUrlFilterCaseSensitive = false;
-            } else if ( rule.condition.isUrlFilterCaseSensitive === false ) {
+            } else if ( condition.isUrlFilterCaseSensitive === true ) {
-                rule.condition.isUrlFilterCaseSensitive = undefined;
+                condition.isUrlFilterCaseSensitive = undefined;
             }
         }
     }
@@ -1098,23 +1099,15 @@ async function rulesetFromURLs(assetDetails) {
 
 async function main() {
 
-    // Get manifest content
+    let version = '';
-    const manifest = await fs.readFile(
-        `${outputDir}/manifest.json`,
-        { encoding: 'utf8' }
-    ).then(text =>
-        JSON.parse(text)
-    );
-
-    // Create unique version number according to build time
-    let version = manifest.version;
     {
         const now = new Date();
-        const yearPart = now.getUTCFullYear() - 2000;
+        const yearPart = now.getUTCFullYear();
-        const monthPart = (now.getUTCMonth() + 1) * 1000;
+        const monthPart = now.getUTCMonth() + 1;
-        const dayPart = now.getUTCDate() * 10;
+        const dayPart = now.getUTCDate();
-        const hourPart = Math.floor(now.getUTCHours() / 3) + 1;
+        const hourPart = Math.floor(now.getUTCHours());
-        version += `.${yearPart}.${monthPart + dayPart + hourPart}`;
+        const minutePart = Math.floor(now.getUTCMinutes());
+        version = `${yearPart}.${monthPart}.${dayPart}.${hourPart * 60 + minutePart}`;
     }
     log(`Version: ${version}`);
 
@@ -1300,6 +1293,13 @@ async function main() {
     await Promise.all(writeOps);
 
     // Patch manifest
+    // Get manifest content
+    const manifest = await fs.readFile(
+        `${outputDir}/manifest.json`,
+        { encoding: 'utf8' }
+    ).then(text =>
+        JSON.parse(text)
+    );
     // Patch declarative_net_request key
     manifest.declarative_net_request = { rule_resources: ruleResources };
     // Patch web_accessible_resources key
@@ -1312,13 +1312,8 @@ async function main() {
     }
     manifest.web_accessible_resources = [ web_accessible_resources ];
 
-    // Patch version key
+    // Patch manifest version property
-    const now = new Date();
+    manifest.version = version;
-    const yearPart = now.getUTCFullYear() - 2000;
-    const monthPart = (now.getUTCMonth() + 1) * 1000;
-    const dayPart = now.getUTCDate() * 10;
-    const hourPart = Math.floor(now.getUTCHours() / 3) + 1;
-    manifest.version = manifest.version + `.${yearPart}.${monthPart + dayPart + hourPart}`;
     // Commit changes
     await fs.writeFile(
         `${outputDir}/manifest.json`,

src/js/static-dnr-filtering.js

@@ -134,7 +134,54 @@ function addExtendedToDNR(context, parser) {
     }
 
     // Response header filtering
-    if ( (parser.flavorBits & parser.BITFlavorExtResponseHeader) !== 0 ) {
+    if ( parser.isResponseheaderFilter() ) {
+        if ( parser.hasError() ) { return; }
+        if ( parser.hasOptions() === false ) { return; }
+        if ( parser.isException() ) { return; }
+        const node = parser.getBranchFromType(sfp.NODE_TYPE_EXT_PATTERN_RESPONSEHEADER);
+        if ( node === 0 ) { return; }
+        const header = parser.getNodeString(node);
+        if ( context.responseHeaderRules === undefined ) {
+            context.responseHeaderRules = [];
+        }
+        const rule =  {
+            action: {
+                responseHeaders: [
+                    {
+                        header,
+                        operation: 'remove',
+                    }
+                ],
+                type: 'modifyHeaders'
+            },
+            condition: {
+                resourceTypes: [
+                    'main_frame',
+                    'sub_frame'
+                ]
+            },
+        };
+        for ( const { hn, not, bad } of parser.getExtFilterDomainIterator() ) {
+            if ( bad ) { continue; }
+            if ( not ) {
+                if ( rule.condition.excludedInitiatorDomains === undefined ) {
+                    rule.condition.excludedInitiatorDomains = [];
+                }
+                rule.condition.excludedInitiatorDomains.push(hn);
+                continue;
+            }
+            if ( hn === '*' ) {
+                if ( rule.condition.initiatorDomains !== undefined ) {
+                    rule.condition.initiatorDomains = undefined;
+                }
+                continue;
+            }
+            if ( rule.condition.initiatorDomains === undefined ) {
+                rule.condition.initiatorDomains = [];
+            }
+            rule.condition.initiatorDomains.push(hn);
+        }
+        context.responseHeaderRules.push(rule);
         return;
     }
 
@@ -286,6 +333,129 @@ function addToDNR(context, list) {
 
 /******************************************************************************/
 
+function finalizeRuleset(context, network) {
+    const ruleset = network.ruleset;
+
+    // Assign rule ids
+    const rulesetMap = new Map();
+    {
+        let ruleId = 1;
+        for ( const rule of ruleset ) {
+            rulesetMap.set(ruleId++, rule);
+        }
+    }
+    // Merge rules where possible by merging arrays of a specific property.
+    //
+    // https://github.com/uBlockOrigin/uBOL-issues/issues/10#issuecomment-1304822579
+    //   Do not merge rules which have errors.
+    const mergeRules = (rulesetMap, mergeTarget) => {
+        const mergeMap = new Map();
+        const sorter = (_, v) => {
+            if ( Array.isArray(v) ) {
+                return typeof v[0] === 'string' ? v.sort() : v;
+            }
+            if ( v instanceof Object ) {
+                const sorted = {};
+                for ( const kk of Object.keys(v).sort() ) {
+                    sorted[kk] = v[kk];
+                }
+                return sorted;
+            }
+            return v;
+        };
+        const ruleHasher = (rule, target) => {
+            return JSON.stringify(rule, (k, v) => {
+                if ( k.startsWith('_') ) { return; }
+                if ( k === target ) { return; }
+                return sorter(k, v);
+            });
+        };
+        const extractTargetValue = (obj, target) => {
+            for ( const [ k, v ] of Object.entries(obj) ) {
+                if ( Array.isArray(v) && k === target ) { return v; }
+                if ( v instanceof Object ) {
+                    const r = extractTargetValue(v, target);
+                    if ( r !== undefined ) { return r; }
+                }
+            }
+        };
+        const extractTargetOwner = (obj, target) => {
+            for ( const [ k, v ] of Object.entries(obj) ) {
+                if ( Array.isArray(v) && k === target ) { return obj; }
+                if ( v instanceof Object ) {
+                    const r = extractTargetOwner(v, target);
+                    if ( r !== undefined ) { return r; }
+                }
+            }
+        };
+        for ( const [ id, rule ] of rulesetMap ) {
+            if ( rule._error !== undefined ) { continue; }
+            const hash = ruleHasher(rule, mergeTarget);
+            if ( mergeMap.has(hash) === false ) {
+                mergeMap.set(hash, []);
+            }
+            mergeMap.get(hash).push(id);
+        }
+        for ( const ids of mergeMap.values() ) {
+            if ( ids.length === 1 ) { continue; }
+            const leftHand = rulesetMap.get(ids[0]);
+            const leftHandSet = new Set(
+                extractTargetValue(leftHand, mergeTarget) || []
+            );
+            for ( let i = 1; i < ids.length; i++ ) {
+                const rightHandId = ids[i];
+                const rightHand = rulesetMap.get(rightHandId);
+                const rightHandArray =  extractTargetValue(rightHand, mergeTarget);
+                if ( rightHandArray !== undefined ) {
+                    if ( leftHandSet.size !== 0 ) {
+                        for ( const item of rightHandArray ) {
+                            leftHandSet.add(item);
+                        }
+                    }
+                } else {
+                    leftHandSet.clear();
+                }
+                rulesetMap.delete(rightHandId);
+            }
+            const leftHandOwner = extractTargetOwner(leftHand, mergeTarget);
+            if ( leftHandSet.size > 1 ) {
+                //if ( leftHandOwner === undefined ) { debugger; }
+                leftHandOwner[mergeTarget] = Array.from(leftHandSet).sort();
+            } else if ( leftHandSet.size === 0 ) {
+                if ( leftHandOwner !== undefined ) {
+                    leftHandOwner[mergeTarget] = undefined;
+                }
+            }
+        }
+    };
+    mergeRules(rulesetMap, 'resourceTypes');
+    mergeRules(rulesetMap, 'initiatorDomains');
+    mergeRules(rulesetMap, 'requestDomains');
+    mergeRules(rulesetMap, 'removeParams');
+    mergeRules(rulesetMap, 'responseHeaders');
+
+    // Patch id
+    const rulesetFinal = [];
+    {
+        let ruleId = 1;
+        for ( const rule of rulesetMap.values() ) {
+            if ( rule._error === undefined ) {
+                rule.id = ruleId++;
+            } else {
+                rule.id = 0;
+            }
+            rulesetFinal.push(rule);
+        }
+        for ( const invalid of context.invalid ) {
+            rulesetFinal.push({ _error: [ invalid ] });
+        }
+    }
+
+    network.ruleset = rulesetFinal;
+}
+
+/******************************************************************************/
+
 async function dnrRulesetFromRawLists(lists, options = {}) {
     const context = Object.assign({}, options);
     staticNetFilteringEngine.dnrFromCompiled('begin', context);
@@ -300,8 +470,7 @@ async function dnrRulesetFromRawLists(lists, options = {}) {
         }
     }
     await Promise.all(toLoad);
-
+    const result = {
-    return {
         network: staticNetFilteringEngine.dnrFromCompiled('end', context),
         genericCosmetic: context.genericCosmeticFilters,
         genericHighCosmetic: context.genericHighCosmeticFilters,
@@ -309,6 +478,11 @@ async function dnrRulesetFromRawLists(lists, options = {}) {
         specificCosmetic: context.specificCosmeticFilters,
         scriptlet: context.scriptletFilters,
     };
+    if ( context.responseHeaderRules ) {
+        result.network.ruleset.push(...context.responseHeaderRules);
+    }
+    finalizeRuleset(context, result.network);
+    return result;
 }
 
 /******************************************************************************/

src/js/static-net-filtering.js

@@ -1274,7 +1274,9 @@ class FilterRegex {
             dnrAddRuleError(rule, `regexFilter is not RE2-compatible: ${args[1]}`);
         }
         rule.condition.regexFilter = args[1];
-        rule.condition.isUrlFilterCaseSensitive = args[2] === 1;
+        if ( args[2] === 1 ) {
+            rule.condition.isUrlFilterCaseSensitive = true;
+        }
     }
 
     static keyFromArgs(args) {
@@ -4349,7 +4351,7 @@ FilterContainer.prototype.dnrFromCompiled = function(op, context, ...args) {
         'ping',
         'other',
     ]);
-    let ruleset = [];
+    const ruleset = [];
     for ( const [ realmBits, realmName ] of realms ) {
         for ( const [ partyBits, partyName ] of partyness ) {
             for ( const typeName in typeNameToTypeValue ) {
@@ -4521,141 +4523,8 @@ FilterContainer.prototype.dnrFromCompiled = function(op, context, ...args) {
         }
     }
 
-    // Assign rule ids
-    const rulesetMap = new Map();
-    {
-        let ruleId = 1;
-        for ( const rule of ruleset ) {
-            rulesetMap.set(ruleId++, rule);
-        }
-    }
-
-    // Merge rules where possible by merging arrays of a specific property.
-    //
-    // https://github.com/uBlockOrigin/uBOL-issues/issues/10#issuecomment-1304822579
-    //   Do not merge rules which have errors.
-    const mergeRules = (rulesetMap, mergeTarget) => {
-        const mergeMap = new Map();
-        const sorter = (_, v) => {
-            if ( Array.isArray(v) ) {
-                return typeof v[0] === 'string' ? v.sort() : v;
-            }
-            if ( v instanceof Object ) {
-                const sorted = {};
-                for ( const kk of Object.keys(v).sort() ) {
-                    sorted[kk] = v[kk];
-                }
-                return sorted;
-            }
-            return v;
-        };
-        const ruleHasher = (rule, target) => {
-            return JSON.stringify(rule, (k, v) => {
-                if ( k.startsWith('_') ) { return; }
-                if ( k === target ) { return; }
-                return sorter(k, v);
-            });
-        };
-        const extractTargetValue = (obj, target) => {
-            for ( const [ k, v ] of Object.entries(obj) ) {
-                if ( Array.isArray(v) && k === target ) { return v; }
-                if ( v instanceof Object ) {
-                    const r = extractTargetValue(v, target);
-                    if ( r !== undefined ) { return r; }
-                }
-            }
-        };
-        const extractTargetOwner = (obj, target) => {
-            for ( const [ k, v ] of Object.entries(obj) ) {
-                if ( Array.isArray(v) && k === target ) { return obj; }
-                if ( v instanceof Object ) {
-                    const r = extractTargetOwner(v, target);
-                    if ( r !== undefined ) { return r; }
-                }
-            }
-        };
-        for ( const [ id, rule ] of rulesetMap ) {
-            if ( rule._error !== undefined ) { continue; }
-            const hash = ruleHasher(rule, mergeTarget);
-            if ( mergeMap.has(hash) === false ) {
-                mergeMap.set(hash, []);
-            }
-            mergeMap.get(hash).push(id);
-        }
-        for ( const ids of mergeMap.values() ) {
-            if ( ids.length === 1 ) { continue; }
-            const leftHand = rulesetMap.get(ids[0]);
-            const leftHandSet = new Set(
-                extractTargetValue(leftHand, mergeTarget) || []
-            );
-            for ( let i = 1; i < ids.length; i++ ) {
-                const rightHandId = ids[i];
-                const rightHand = rulesetMap.get(rightHandId);
-                const rightHandArray =  extractTargetValue(rightHand, mergeTarget);
-                if ( rightHandArray !== undefined ) {
-                    if ( leftHandSet.size !== 0 ) {
-                        for ( const item of rightHandArray ) {
-                            leftHandSet.add(item);
-                        }
-                    }
-                } else {
-                    leftHandSet.clear();
-                }
-                rulesetMap.delete(rightHandId);
-            }
-            const leftHandOwner = extractTargetOwner(leftHand, mergeTarget);
-            if ( leftHandSet.size > 1 ) {
-                //if ( leftHandOwner === undefined ) { debugger; }
-                leftHandOwner[mergeTarget] = Array.from(leftHandSet).sort();
-            } else if ( leftHandSet.size === 0 ) {
-                if ( leftHandOwner !== undefined ) {
-                    leftHandOwner[mergeTarget] = undefined;
-                }
-            }
-        }
-    };
-    mergeRules(rulesetMap, 'resourceTypes');
-    mergeRules(rulesetMap, 'initiatorDomains');
-    mergeRules(rulesetMap, 'requestDomains');
-    mergeRules(rulesetMap, 'removeParams');
-    mergeRules(rulesetMap, 'responseHeaders');
-
-    // Patch case-sensitiveness
-    for ( const rule of rulesetMap.values() ) {
-        const { condition } = rule;
-        if (
-            condition === undefined ||
-            condition.urlFilter === undefined &&
-            condition.regexFilter === undefined
-        ) {
-            continue;
-        }
-        if ( condition.isUrlFilterCaseSensitive === undefined ) {
-            condition.isUrlFilterCaseSensitive = false;
-        } else if ( condition.isUrlFilterCaseSensitive === true ) {
-            condition.isUrlFilterCaseSensitive = undefined;
-        }
-    }
-
-    // Patch id
-    const rulesetFinal = [];
-    {
-        let ruleId = 1;
-        for ( const rule of rulesetMap.values() ) {
-            if ( rule._error === undefined ) {
-                rule.id = ruleId++;
-            } else {
-                rule.id = 0;
-            }
-            rulesetFinal.push(rule);
-        }
-        for ( const invalid of context.invalid ) {
-            rulesetFinal.push({ _error: [ invalid ] });
-        }
-    }
-
     return {
-        ruleset: rulesetFinal,
+        ruleset,
         filterCount: context.filterCount,
         acceptedFilterCount: context.acceptedFilterCount,
         rejectedFilterCount: context.rejectedFilterCount,