[mv3] Fix procedural operator matches-media()

The failure was caused by the fact that there is no
window.matchMedia() API available in Nodejs. The validation
is now done using cssTree.

platform/mv3/extension/js/background.js

@@ -72,7 +72,7 @@ async function loadRulesetConfig() {
         return;
     }
 
-    const match = /^\|\|example.invalid\/([^\/]+)\/(?:([^\/]+)\/)?/.exec(
+    const match = /^\|\|(?:example|ubolite)\.invalid\/([^\/]+)\/(?:([^\/]+)\/)?/.exec(
         configRule.condition.urlFilter
     );
     if ( match === null ) { return; }
@@ -101,7 +101,7 @@ async function saveRulesetConfig() {
 
     const version = rulesetConfig.version;
     const enabledRulesets = encodeURIComponent(rulesetConfig.enabledRulesets.join(' '));
-    const urlFilter = `||example.invalid/${version}/${enabledRulesets}/`;
+    const urlFilter = `||ubolite.invalid/${version}/${enabledRulesets}/`;
     if ( urlFilter === configRule.condition.urlFilter ) { return; }
     configRule.condition.urlFilter = urlFilter;
 
@@ -115,26 +115,26 @@ async function saveRulesetConfig() {
 
 async function hasGreatPowers(origin) {
     return browser.permissions.contains({
-        origins: [ `${origin}/*` ]
+        origins: [ `${origin}/*` ],
     });
 }
 
 function grantGreatPowers(hostname) {
     return browser.permissions.request({
-        origins: [
+        origins: [ `*://${hostname}/*` ],
-            `*://${hostname}/*`,
-        ]
     });
 }
 
 function revokeGreatPowers(hostname) {
     return browser.permissions.remove({
-        origins: [
+        origins: [ `*://${hostname}/*` ],
-            `*://${hostname}/*`,
-        ]
     });
 }
 
+function onPermissionsChanged() {
+    registerInjectable();
+}
+
 /******************************************************************************/
 
 function onMessage(request, sender, callback) {
@@ -213,10 +213,6 @@ function onMessage(request, sender, callback) {
     }
 }
 
-async function onPermissionsChanged() {
-    await registerInjectable();
-}
-
 /******************************************************************************/
 
 async function start() {

platform/mv3/extension/js/ruleset-manager.js

@@ -205,26 +205,42 @@ async function defaultRulesetsFromLanguage() {
 async function enableRulesets(ids) {
     const afterIds = new Set(ids);
     const beforeIds = new Set(await dnr.getEnabledRulesets());
-    const enableRulesetIds = [];
+    const enableRulesetSet = new Set();
-    const disableRulesetIds = [];
+    const disableRulesetSet = new Set();
     for ( const id of afterIds ) {
         if ( beforeIds.has(id) ) { continue; }
-        enableRulesetIds.push(id);
+        enableRulesetSet.add(id);
     }
     for ( const id of beforeIds ) {
         if ( afterIds.has(id) ) { continue; }
-        disableRulesetIds.push(id);
+        disableRulesetSet.add(id);
     }
-    
+
+    if ( enableRulesetSet.size === 0 && disableRulesetSet.size === 0 ) {
+        return;
+    }
+
+    // Be sure the rulesets to enable/disable do exist in the current version,
+    // otherwise the API throws.
+    const rulesetDetails = await getRulesetDetails();
+    for ( const id of enableRulesetSet ) {
+        if ( rulesetDetails.has(id) ) { continue; }
+        enableRulesetSet.delete(id);
+    }
+    for ( const id of disableRulesetSet ) {
+        if ( rulesetDetails.has(id) ) { continue; }
+        disableRulesetSet.delete(id);
+    }
+    const enableRulesetIds = Array.from(enableRulesetSet);
+    const disableRulesetIds = Array.from(disableRulesetSet);
+
     if ( enableRulesetIds.length !== 0 ) {
         console.info(`Enable rulesets: ${enableRulesetIds}`);
     }
     if ( disableRulesetIds.length !== 0 ) {
         console.info(`Disable ruleset: ${disableRulesetIds}`);
     }
-    if ( enableRulesetIds.length !== 0 || disableRulesetIds.length !== 0 ) {
+    return dnr.updateEnabledRulesets({ enableRulesetIds, disableRulesetIds });
-        return dnr.updateEnabledRulesets({ enableRulesetIds, disableRulesetIds  });
-    }
 }
 
 /******************************************************************************/

platform/mv3/extension/js/scripting-manager.js

@@ -27,7 +27,7 @@
 
 import { browser, dnr } from './ext.js';
 import { fetchJSON } from './fetch.js';
-import { matchesTrustedSiteDirective } from './trusted-sites.js';
+import { getAllTrustedSiteDirectives } from './trusted-sites.js';
 
 import {
     parsedURLromOrigin,
@@ -94,7 +94,6 @@ const arrayEq = (a, b) => {
 const toRegisterable = (fname, entry) => {
     const directive = {
         id: fname,
-        allFrames: true,
     };
     if ( entry.matches ) {
         directive.matches = matchesFromHostnames(entry.matches);
@@ -173,37 +172,36 @@ async function getInjectableCount(origin) {
 
 async function registerInjectable() {
 
+    if ( browser.scripting === undefined ) { return false; }
+
     const [
         hostnames,
+        trustedSites,
         rulesetIds,
         registered,
         scriptingDetails,
     ] = await Promise.all([
         browser.permissions.getAll(),
+        getAllTrustedSiteDirectives(),
         dnr.getEnabledRulesets(),
         browser.scripting.getRegisteredContentScripts(),
         getScriptingDetails(),
     ]).then(results => {
-        results[0] = new Map(
+        results[0] = new Set(hostnamesFromMatches(results[0].origins));
-            hostnamesFromMatches(results[0].origins).map(hn => [ hn, false ])
+        results[1] = new Set(results[1]);
-        );
         return results;
     });
 
-    if ( hostnames.has('*') && hostnames.size > 1 ) {
-        hostnames.clear();
-        hostnames.set('*', false);
-    }
-
-    await Promise.all(
-        Array.from(hostnames.keys()).map(
-            hn => matchesTrustedSiteDirective({ hostname: hn })
-                .then(trusted => hostnames.set(hn, trusted))
-        )
-    );
-
     const toRegister = new Map();
 
+    const isTrustedHostname = hn => {
+        while ( hn ) {
+            if ( trustedSites.has(hn) ) { return true; }
+            hn = toBroaderHostname(hn);
+        }
+        return false;
+    };
+
     const checkMatches = (details, hn) => {
         let fids = details.matches?.get(hn);
         if ( fids === undefined ) { return; }
@@ -222,9 +220,9 @@ async function registerInjectable() {
     for ( const rulesetId of rulesetIds ) {
         const details = scriptingDetails.get(rulesetId);
         if ( details === undefined ) { continue; }
-        for ( let [ hn, trusted ] of hostnames ) {
+        for ( let hn of hostnames ) {
-            if ( trusted ) { continue; }
+            if ( isTrustedHostname(hn) ) { continue; }
-            while ( hn !== '' ) {
+            while ( hn ) {
                 checkMatches(details, hn);
                 hn = toBroaderHostname(hn);
             }
@@ -251,7 +249,7 @@ async function registerInjectable() {
         const details = scriptingDetails.get(rulesetId);
         if ( details === undefined ) { continue; }
         for ( let hn of hostnames.keys() ) {
-            while ( hn !== '' ) {
+            while ( hn ) {
                 checkExcludeMatches(details, hn);
                 hn = toBroaderHostname(hn);
             }

platform/mv3/extension/js/trusted-sites.js

@@ -39,6 +39,15 @@ import {
 
 /******************************************************************************/
 
+async function getAllTrustedSiteDirectives() {
+    const dynamicRuleMap = await getDynamicRules();
+    const rule = dynamicRuleMap.get(TRUSTED_DIRECTIVE_BASE_RULE_ID);
+    if ( rule === undefined ) { return []; }
+    return rule.condition.requestDomains;
+}
+
+/******************************************************************************/
+
 async function matchesTrustedSiteDirective(details) {
     const hostname =
         details.hostname ||
@@ -47,7 +56,7 @@ async function matchesTrustedSiteDirective(details) {
     if ( hostname === undefined ) { return false; }
     
     const dynamicRuleMap = await getDynamicRules();
-    let rule = dynamicRuleMap.get(TRUSTED_DIRECTIVE_BASE_RULE_ID);
+    const rule = dynamicRuleMap.get(TRUSTED_DIRECTIVE_BASE_RULE_ID);
     if ( rule === undefined ) { return false; }
 
     const domainSet = new Set(rule.condition.requestDomains);
@@ -155,6 +164,7 @@ async function toggleTrustedSiteDirective(details) {
 /******************************************************************************/
 
 export {
+    getAllTrustedSiteDirectives,
     matchesTrustedSiteDirective,
     toggleTrustedSiteDirective,
 };

platform/mv3/make-rulesets.js

@@ -74,35 +74,6 @@ const uidint32 = (s) => {
 
 /******************************************************************************/
 
-const isUnsupported = rule =>
-    rule._error !== undefined;
-
-const isRegex = rule =>
-    rule.condition !== undefined &&
-    rule.condition.regexFilter !== undefined;
-
-const isRedirect = rule =>
-    rule.action !== undefined &&
-    rule.action.type === 'redirect' &&
-    rule.action.redirect.extensionPath !== undefined;
-
-const isCsp = rule =>
-    rule.action !== undefined &&
-    rule.action.type === 'modifyHeaders';
-
-const isRemoveparam = rule =>
-    rule.action !== undefined &&
-    rule.action.type === 'redirect' &&
-    rule.action.redirect.transform !== undefined;
-
-const isGood = rule =>
-    isUnsupported(rule) === false &&
-    isRedirect(rule) === false &&
-    isCsp(rule) === false &&
-    isRemoveparam(rule) === false;
-
-/******************************************************************************/
-
 const stdOutput = [];
 
 const log = (text, silent = false) => {
@@ -217,6 +188,35 @@ async function fetchAsset(assetDetails) {
 
 /******************************************************************************/
 
+const isUnsupported = rule =>
+    rule._error !== undefined;
+
+const isRegex = rule =>
+    rule.condition !== undefined &&
+    rule.condition.regexFilter !== undefined;
+
+const isRedirect = rule =>
+    rule.action !== undefined &&
+    rule.action.type === 'redirect' &&
+    rule.action.redirect.extensionPath !== undefined;
+
+const isCsp = rule =>
+    rule.action !== undefined &&
+    rule.action.type === 'modifyHeaders';
+
+const isRemoveparam = rule =>
+    rule.action !== undefined &&
+    rule.action.type === 'redirect' &&
+    rule.action.redirect.transform !== undefined;
+
+const isGood = rule =>
+    isUnsupported(rule) === false &&
+    isRedirect(rule) === false &&
+    isCsp(rule) === false &&
+    isRemoveparam(rule) === false;
+
+/******************************************************************************/
+
 async function processNetworkFilters(assetDetails, network) {
     const replacer = (k, v) => {
         if ( k.startsWith('__') ) { return; }
@@ -993,7 +993,9 @@ async function main() {
     );
 
     // Log results
-    await fs.writeFile(`${outputDir}/log.txt`, stdOutput.join('\n') + '\n');
+    const logContent = stdOutput.join('\n') + '\n';
+    await fs.writeFile(`${outputDir}/log.txt`, logContent);
+    await fs.writeFile(`${cacheDir}/log.txt`, logContent);
 }
 
 main();

platform/mv3/scriptlets/css-specific-procedural.js

@@ -69,6 +69,15 @@ class PSelectorTask {
     }
 }
 
+class PSelectorVoidTask extends PSelectorTask {
+    constructor(task) {
+        super();
+        console.info(`uBO: :${task[0]}() operator does not exist`);
+    }
+    transpose() {
+    }
+}
+
 class PSelectorHasTextTask extends PSelectorTask {
     constructor(task) {
         super();
@@ -120,6 +129,19 @@ class PSelectorMatchesCSSTask extends PSelectorTask {
         }
     }
 }
+class PSelectorMatchesCSSAfterTask extends PSelectorMatchesCSSTask {
+    constructor(task) {
+        super(task);
+        this.pseudo = '::after';
+    }
+}
+
+class PSelectorMatchesCSSBeforeTask extends PSelectorMatchesCSSTask {
+    constructor(task) {
+        super(task);
+        this.pseudo = '::before';
+    }
+}
 
 class PSelectorMatchesMediaTask extends PSelectorTask {
     constructor(task) {
@@ -370,6 +392,8 @@ class PSelector {
                 [ 'if', PSelectorIfTask ],
                 [ 'if-not', PSelectorIfNotTask ],
                 [ 'matches-css', PSelectorMatchesCSSTask ],
+                [ 'matches-css-after', PSelectorMatchesCSSAfterTask ],
+                [ 'matches-css-before', PSelectorMatchesCSSBeforeTask ],
                 [ 'matches-media', PSelectorMatchesMediaTask ],
                 [ 'matches-path', PSelectorMatchesPathTask ],
                 [ 'min-text-length', PSelectorMinTextLengthTask ],
@@ -387,8 +411,7 @@ class PSelector {
         const tasks = [];
         if ( Array.isArray(o.tasks) === false ) { return; }
         for ( const task of o.tasks ) {
-            const ctor = this.operatorToTaskMap.get(task[0]);
+            const ctor = this.operatorToTaskMap.get(task[0]) || PSelectorVoidTask;
-            if ( ctor === undefined ) { return; }
             tasks.push(new ctor(task));
         }
         // Initialize only after all tasks have been successfully instantiated

src/js/static-filtering-parser.js

@@ -1499,46 +1499,33 @@ Parser.prototype.SelectorCompiler = class {
         case 'ClassSelector':
         case 'Combinator':
         case 'IdSelector':
+        case 'MediaFeature':
+        case 'Nth':
+        case 'Raw':
         case 'TypeSelector':
             out.push({ data });
             break;
-        case 'Declaration': {
+        case 'Declaration':
             if ( data.value ) {
                 this.astFlatten(data.value, args = []);
             }
             out.push({ data, args });
             args = undefined;
             break;
-        }
         case 'DeclarationList':
-            args = out;
-            out.push({ data });
-            break;
         case 'Identifier':
+        case 'MediaQueryList':
+        case 'Selector':
+        case 'SelectorList':
             args = out;
             out.push({ data });
             break;
-        case 'Nth': {
+        case 'MediaQuery':
-            out.push({ data });
-            break;
-        }
         case 'PseudoClassSelector':
         case 'PseudoElementSelector':
             if ( head ) { args = []; }
             out.push({ data, args });
             break;
-        case 'Raw':
-            if ( head ) { args = []; }
-            out.push({ data, args });
-            break;
-        case 'Selector':
-            args = out;
-            out.push({ data });
-            break;
-        case 'SelectorList':
-            args = out;
-            out.push({ data });
-            break;
         case 'Value':
             args = out;
             break;
@@ -1552,7 +1539,7 @@ Parser.prototype.SelectorCompiler = class {
             }
             let next = head.next;
             while ( next ) {
-                this.astFlatten(next.data, out);
+                this.astFlatten(next.data, args);
                 next = next.next;
             }
         }
@@ -1923,15 +1910,12 @@ Parser.prototype.SelectorCompiler = class {
     }
 
     compileMediaQuery(s) {
-        if ( typeof self !== 'object' ) { return; }
+        const parts = this.astFromRaw(s, 'mediaQueryList');
-        if ( self === null ) { return; }
+        if ( parts === undefined ) { return; }
-        if ( typeof self.matchMedia !== 'function' ) { return; }
+        if ( this.astHasType(parts, 'Raw') ) { return; }
-        try {
+        if ( this.astHasType(parts, 'MediaQuery') === false ) { return; }
-            const mql = self.matchMedia(s);
+        // TODO: normalize by serializing resulting AST
-            if ( mql instanceof self.MediaQueryList === false ) { return; }
+        return s;
-            if ( mql.media !== 'not all' ) { return s; }
-        } catch(ex) {
-        }
     }
 
     compileUpwardArgument(s) {

src/js/static-net-filtering.js

@@ -3903,65 +3903,65 @@ FilterContainer.prototype.dnrFromCompiled = function(op, context, ...args) {
         const bucket = buckets.get(bits);
 
         switch ( tokenHash ) {
-            case DOT_TOKEN_HASH: {
+        case DOT_TOKEN_HASH: {
-                if ( bucket.has(DOT_TOKEN_HASH) === false ) {
+            if ( bucket.has(DOT_TOKEN_HASH) === false ) {
-                    bucket.set(DOT_TOKEN_HASH, [{
+                bucket.set(DOT_TOKEN_HASH, [{
-                        condition: {
+                    condition: {
-                            requestDomains: []
+                        requestDomains: []
-                        }
+                    }
-                    }]);
+                }]);
-                }
-                const rule = bucket.get(DOT_TOKEN_HASH)[0];
-                rule.condition.requestDomains.push(fdata);
-                break;
             }
-            case ANY_TOKEN_HASH: {
+            const rule = bucket.get(DOT_TOKEN_HASH)[0];
-                if ( bucket.has(ANY_TOKEN_HASH) === false ) {
+            rule.condition.requestDomains.push(fdata);
-                    bucket.set(ANY_TOKEN_HASH, [{
+            break;
-                        condition: {
+        }
-                            initiatorDomains: []
+        case ANY_TOKEN_HASH: {
-                        }
+            if ( bucket.has(ANY_TOKEN_HASH) === false ) {
-                    }]);
+                bucket.set(ANY_TOKEN_HASH, [{
-                }
+                    condition: {
-                const rule = bucket.get(ANY_TOKEN_HASH)[0];
+                        initiatorDomains: []
-                rule.condition.initiatorDomains.push(fdata);
+                    }
-                break;
+                }]);
             }
-            case ANY_HTTPS_TOKEN_HASH: {
+            const rule = bucket.get(ANY_TOKEN_HASH)[0];
-                if ( bucket.has(ANY_HTTPS_TOKEN_HASH) === false ) {
+            rule.condition.initiatorDomains.push(fdata);
-                    bucket.set(ANY_HTTPS_TOKEN_HASH, [{
+            break;
-                        condition: {
+        }
-                            urlFilter: '|https://',
+        case ANY_HTTPS_TOKEN_HASH: {
-                            initiatorDomains: []
+            if ( bucket.has(ANY_HTTPS_TOKEN_HASH) === false ) {
-                        }
+                bucket.set(ANY_HTTPS_TOKEN_HASH, [{
-                    }]);
+                    condition: {
-                }
+                        urlFilter: '|https://',
-                const rule = bucket.get(ANY_HTTPS_TOKEN_HASH)[0];
+                        initiatorDomains: []
-                rule.condition.initiatorDomains.push(fdata);
+                    }
-                break;
+                }]);
             }
-            case ANY_HTTP_TOKEN_HASH: {
+            const rule = bucket.get(ANY_HTTPS_TOKEN_HASH)[0];
-                if ( bucket.has(ANY_HTTP_TOKEN_HASH) === false ) {
+            rule.condition.initiatorDomains.push(fdata);
-                    bucket.set(ANY_HTTP_TOKEN_HASH, [{
+            break;
-                        condition: {
+        }
-                            urlFilter: '|http://',
+        case ANY_HTTP_TOKEN_HASH: {
-                            initiatorDomains: []
+            if ( bucket.has(ANY_HTTP_TOKEN_HASH) === false ) {
-                        }
+                bucket.set(ANY_HTTP_TOKEN_HASH, [{
-                    }]);
+                    condition: {
-                }
+                        urlFilter: '|http://',
-                const rule = bucket.get(ANY_HTTP_TOKEN_HASH)[0];
+                        initiatorDomains: []
-                rule.condition.initiatorDomains.push(fdata);
+                    }
-                break;
+                }]);
             }
-            default: {
+            const rule = bucket.get(ANY_HTTP_TOKEN_HASH)[0];
-                if ( bucket.has(EMPTY_TOKEN_HASH) === false ) {
+            rule.condition.initiatorDomains.push(fdata);
-                    bucket.set(EMPTY_TOKEN_HASH, []);
+            break;
-                }
+        }
-                const rule = {};
+        default: {
-                dnrRuleFromCompiled(fdata, rule);
+            if ( bucket.has(EMPTY_TOKEN_HASH) === false ) {
-                bucket.get(EMPTY_TOKEN_HASH).push(rule);
+                bucket.set(EMPTY_TOKEN_HASH, []);
-                break;
             }
+            const rule = {};
+            dnrRuleFromCompiled(fdata, rule);
+            bucket.get(EMPTY_TOKEN_HASH).push(rule);
+            break;
+        }
         }
     }
 
@@ -4066,7 +4066,7 @@ FilterContainer.prototype.dnrFromCompiled = function(op, context, ...args) {
                         }
                     }
                 };
-                if ( /^\/.+\/$/.test(rule.__modifierValue) ) {
+                if ( /^~?\/.+\/$/.test(rule.__modifierValue) ) {
                     dnrAddRuleError(rule, `Unsupported regex-based removeParam: ${rule.__modifierValue}`);
                 }
             } else {
@@ -4076,6 +4076,17 @@ FilterContainer.prototype.dnrFromCompiled = function(op, context, ...args) {
                     }
                 };
             }
+            if ( rule.condition === undefined ) {
+                rule.condition = {
+                };
+            }
+            if ( rule.condition.resourceTypes === undefined ) {
+                rule.condition.resourceTypes = [
+                    'main_frame',
+                    'sub_frame',
+                    'xmlhttprequest',
+                ];
+            }
             if ( rule.__modifierAction === AllowAction ) {
                 dnrAddRuleError(rule, 'Unhandled modifier exception');
             }