[mv3] Fix procedural operator matches-media()

The failure was caused by the fact that there is no window.matchMedia() API available in Nodejs. The validation is now done using cssTree.
2022-09-27 07:46:24 -04:00 · 2022-09-27 07:46:24 -04:00 · 51c2e22c7a
parent ec83127f6c
commit 51c2e22c7a
8 changed files with 201 additions and 161 deletions
--- a/platform/mv3/extension/js/background.js
+++ b/platform/mv3/extension/js/background.js
@ -72,7 +72,7 @@ async function loadRulesetConfig() {
        return;
    }

-    const match = /^\|\|example.invalid\/([^\/]+)\/(?:([^\/]+)\/)?/.exec(
+    const match = /^\|\|(?:example|ubolite)\.invalid\/([^\/]+)\/(?:([^\/]+)\/)?/.exec(
        configRule.condition.urlFilter
    );
    if ( match === null ) { return; }
@ -101,7 +101,7 @@ async function saveRulesetConfig() {

    const version = rulesetConfig.version;
    const enabledRulesets = encodeURIComponent(rulesetConfig.enabledRulesets.join(' '));
-    const urlFilter = `||example.invalid/${version}/${enabledRulesets}/`;
+    const urlFilter = `||ubolite.invalid/${version}/${enabledRulesets}/`;
    if ( urlFilter === configRule.condition.urlFilter ) { return; }
    configRule.condition.urlFilter = urlFilter;

@ -115,26 +115,26 @@ async function saveRulesetConfig() {

 async function hasGreatPowers(origin) {
    return browser.permissions.contains({
-        origins: [ `${origin}/*` ]
+        origins: [ `${origin}/*` ],
    });
 }

 function grantGreatPowers(hostname) {
    return browser.permissions.request({
-        origins: [
-            `*://${hostname}/*`,
-        ]
+        origins: [ `*://${hostname}/*` ],
    });
 }

 function revokeGreatPowers(hostname) {
    return browser.permissions.remove({
-        origins: [
-            `*://${hostname}/*`,
-        ]
+        origins: [ `*://${hostname}/*` ],
    });
 }

+function onPermissionsChanged() {
+    registerInjectable();
+}
+
 /******************************************************************************/

 function onMessage(request, sender, callback) {
@ -213,10 +213,6 @@ function onMessage(request, sender, callback) {
    }
 }

-async function onPermissionsChanged() {
-    await registerInjectable();
-}
-
 /******************************************************************************/

 async function start() {
--- a/platform/mv3/extension/js/ruleset-manager.js
+++ b/platform/mv3/extension/js/ruleset-manager.js
@ -205,26 +205,42 @@ async function defaultRulesetsFromLanguage() {
 async function enableRulesets(ids) {
    const afterIds = new Set(ids);
    const beforeIds = new Set(await dnr.getEnabledRulesets());
-    const enableRulesetIds = [];
-    const disableRulesetIds = [];
+    const enableRulesetSet = new Set();
+    const disableRulesetSet = new Set();
    for ( const id of afterIds ) {
        if ( beforeIds.has(id) ) { continue; }
-        enableRulesetIds.push(id);
+        enableRulesetSet.add(id);
    }
    for ( const id of beforeIds ) {
        if ( afterIds.has(id) ) { continue; }
-        disableRulesetIds.push(id);
+        disableRulesetSet.add(id);
    }
-    
+
+    if ( enableRulesetSet.size === 0 && disableRulesetSet.size === 0 ) {
+        return;
+    }
+
+    // Be sure the rulesets to enable/disable do exist in the current version,
+    // otherwise the API throws.
+    const rulesetDetails = await getRulesetDetails();
+    for ( const id of enableRulesetSet ) {
+        if ( rulesetDetails.has(id) ) { continue; }
+        enableRulesetSet.delete(id);
+    }
+    for ( const id of disableRulesetSet ) {
+        if ( rulesetDetails.has(id) ) { continue; }
+        disableRulesetSet.delete(id);
+    }
+    const enableRulesetIds = Array.from(enableRulesetSet);
+    const disableRulesetIds = Array.from(disableRulesetSet);
+
    if ( enableRulesetIds.length !== 0 ) {
        console.info(`Enable rulesets: ${enableRulesetIds}`);
    }
    if ( disableRulesetIds.length !== 0 ) {
        console.info(`Disable ruleset: ${disableRulesetIds}`);
    }
-    if ( enableRulesetIds.length !== 0 || disableRulesetIds.length !== 0 ) {
-        return dnr.updateEnabledRulesets({ enableRulesetIds, disableRulesetIds  });
-    }
+    return dnr.updateEnabledRulesets({ enableRulesetIds, disableRulesetIds });
 }

 /******************************************************************************/
--- a/platform/mv3/extension/js/scripting-manager.js
+++ b/platform/mv3/extension/js/scripting-manager.js
@ -27,7 +27,7 @@

 import { browser, dnr } from './ext.js';
 import { fetchJSON } from './fetch.js';
-import { matchesTrustedSiteDirective } from './trusted-sites.js';
+import { getAllTrustedSiteDirectives } from './trusted-sites.js';

 import {
    parsedURLromOrigin,
@ -94,7 +94,6 @@ const arrayEq = (a, b) => {
 const toRegisterable = (fname, entry) => {
    const directive = {
        id: fname,
-        allFrames: true,
    };
    if ( entry.matches ) {
        directive.matches = matchesFromHostnames(entry.matches);
@ -173,37 +172,36 @@ async function getInjectableCount(origin) {

 async function registerInjectable() {

+    if ( browser.scripting === undefined ) { return false; }
+
    const [
        hostnames,
+        trustedSites,
        rulesetIds,
        registered,
        scriptingDetails,
    ] = await Promise.all([
        browser.permissions.getAll(),
+        getAllTrustedSiteDirectives(),
        dnr.getEnabledRulesets(),
        browser.scripting.getRegisteredContentScripts(),
        getScriptingDetails(),
    ]).then(results => {
-        results[0] = new Map(
-            hostnamesFromMatches(results[0].origins).map(hn => [ hn, false ])
-        );
+        results[0] = new Set(hostnamesFromMatches(results[0].origins));
+        results[1] = new Set(results[1]);
        return results;
    });

-    if ( hostnames.has('*') && hostnames.size > 1 ) {
-        hostnames.clear();
-        hostnames.set('*', false);
-    }
-
-    await Promise.all(
-        Array.from(hostnames.keys()).map(
-            hn => matchesTrustedSiteDirective({ hostname: hn })
-                .then(trusted => hostnames.set(hn, trusted))
-        )
-    );
-
    const toRegister = new Map();

+    const isTrustedHostname = hn => {
+        while ( hn ) {
+            if ( trustedSites.has(hn) ) { return true; }
+            hn = toBroaderHostname(hn);
+        }
+        return false;
+    };
+
    const checkMatches = (details, hn) => {
        let fids = details.matches?.get(hn);
        if ( fids === undefined ) { return; }
@ -222,9 +220,9 @@ async function registerInjectable() {
    for ( const rulesetId of rulesetIds ) {
        const details = scriptingDetails.get(rulesetId);
        if ( details === undefined ) { continue; }
-        for ( let [ hn, trusted ] of hostnames ) {
-            if ( trusted ) { continue; }
-            while ( hn !== '' ) {
+        for ( let hn of hostnames ) {
+            if ( isTrustedHostname(hn) ) { continue; }
+            while ( hn ) {
                checkMatches(details, hn);
                hn = toBroaderHostname(hn);
            }
@ -251,7 +249,7 @@ async function registerInjectable() {
        const details = scriptingDetails.get(rulesetId);
        if ( details === undefined ) { continue; }
        for ( let hn of hostnames.keys() ) {
-            while ( hn !== '' ) {
+            while ( hn ) {
                checkExcludeMatches(details, hn);
                hn = toBroaderHostname(hn);
            }
--- a/platform/mv3/extension/js/trusted-sites.js
+++ b/platform/mv3/extension/js/trusted-sites.js
@ -39,6 +39,15 @@ import {

 /******************************************************************************/

+async function getAllTrustedSiteDirectives() {
+    const dynamicRuleMap = await getDynamicRules();
+    const rule = dynamicRuleMap.get(TRUSTED_DIRECTIVE_BASE_RULE_ID);
+    if ( rule === undefined ) { return []; }
+    return rule.condition.requestDomains;
+}
+
+/******************************************************************************/
+
 async function matchesTrustedSiteDirective(details) {
    const hostname =
        details.hostname ||
@ -47,7 +56,7 @@ async function matchesTrustedSiteDirective(details) {
    if ( hostname === undefined ) { return false; }
    
    const dynamicRuleMap = await getDynamicRules();
-    let rule = dynamicRuleMap.get(TRUSTED_DIRECTIVE_BASE_RULE_ID);
+    const rule = dynamicRuleMap.get(TRUSTED_DIRECTIVE_BASE_RULE_ID);
    if ( rule === undefined ) { return false; }

    const domainSet = new Set(rule.condition.requestDomains);
@ -155,6 +164,7 @@ async function toggleTrustedSiteDirective(details) {
 /******************************************************************************/

 export {
+    getAllTrustedSiteDirectives,
    matchesTrustedSiteDirective,
    toggleTrustedSiteDirective,
 };
--- a/platform/mv3/make-rulesets.js
+++ b/platform/mv3/make-rulesets.js
@ -74,35 +74,6 @@ const uidint32 = (s) => {

 /******************************************************************************/

-const isUnsupported = rule =>
-    rule._error !== undefined;
-
-const isRegex = rule =>
-    rule.condition !== undefined &&
-    rule.condition.regexFilter !== undefined;
-
-const isRedirect = rule =>
-    rule.action !== undefined &&
-    rule.action.type === 'redirect' &&
-    rule.action.redirect.extensionPath !== undefined;
-
-const isCsp = rule =>
-    rule.action !== undefined &&
-    rule.action.type === 'modifyHeaders';
-
-const isRemoveparam = rule =>
-    rule.action !== undefined &&
-    rule.action.type === 'redirect' &&
-    rule.action.redirect.transform !== undefined;
-
-const isGood = rule =>
-    isUnsupported(rule) === false &&
-    isRedirect(rule) === false &&
-    isCsp(rule) === false &&
-    isRemoveparam(rule) === false;
-
-/******************************************************************************/
-
 const stdOutput = [];

 const log = (text, silent = false) => {
@ -217,6 +188,35 @@ async function fetchAsset(assetDetails) {

 /******************************************************************************/

+const isUnsupported = rule =>
+    rule._error !== undefined;
+
+const isRegex = rule =>
+    rule.condition !== undefined &&
+    rule.condition.regexFilter !== undefined;
+
+const isRedirect = rule =>
+    rule.action !== undefined &&
+    rule.action.type === 'redirect' &&
+    rule.action.redirect.extensionPath !== undefined;
+
+const isCsp = rule =>
+    rule.action !== undefined &&
+    rule.action.type === 'modifyHeaders';
+
+const isRemoveparam = rule =>
+    rule.action !== undefined &&
+    rule.action.type === 'redirect' &&
+    rule.action.redirect.transform !== undefined;
+
+const isGood = rule =>
+    isUnsupported(rule) === false &&
+    isRedirect(rule) === false &&
+    isCsp(rule) === false &&
+    isRemoveparam(rule) === false;
+
+/******************************************************************************/
+
 async function processNetworkFilters(assetDetails, network) {
    const replacer = (k, v) => {
        if ( k.startsWith('__') ) { return; }
@ -993,7 +993,9 @@ async function main() {
    );

    // Log results
-    await fs.writeFile(`${outputDir}/log.txt`, stdOutput.join('\n') + '\n');
+    const logContent = stdOutput.join('\n') + '\n';
+    await fs.writeFile(`${outputDir}/log.txt`, logContent);
+    await fs.writeFile(`${cacheDir}/log.txt`, logContent);
 }

 main();
--- a/platform/mv3/scriptlets/css-specific-procedural.js
+++ b/platform/mv3/scriptlets/css-specific-procedural.js
@ -69,6 +69,15 @@ class PSelectorTask {
    }
 }

+class PSelectorVoidTask extends PSelectorTask {
+    constructor(task) {
+        super();
+        console.info(`uBO: :${task[0]}() operator does not exist`);
+    }
+    transpose() {
+    }
+}
+
 class PSelectorHasTextTask extends PSelectorTask {
    constructor(task) {
        super();
@ -120,6 +129,19 @@ class PSelectorMatchesCSSTask extends PSelectorTask {
        }
    }
 }
+class PSelectorMatchesCSSAfterTask extends PSelectorMatchesCSSTask {
+    constructor(task) {
+        super(task);
+        this.pseudo = '::after';
+    }
+}
+
+class PSelectorMatchesCSSBeforeTask extends PSelectorMatchesCSSTask {
+    constructor(task) {
+        super(task);
+        this.pseudo = '::before';
+    }
+}

 class PSelectorMatchesMediaTask extends PSelectorTask {
    constructor(task) {
@ -370,6 +392,8 @@ class PSelector {
                [ 'if', PSelectorIfTask ],
                [ 'if-not', PSelectorIfNotTask ],
                [ 'matches-css', PSelectorMatchesCSSTask ],
+                [ 'matches-css-after', PSelectorMatchesCSSAfterTask ],
+                [ 'matches-css-before', PSelectorMatchesCSSBeforeTask ],
                [ 'matches-media', PSelectorMatchesMediaTask ],
                [ 'matches-path', PSelectorMatchesPathTask ],
                [ 'min-text-length', PSelectorMinTextLengthTask ],
@ -387,8 +411,7 @@ class PSelector {
        const tasks = [];
        if ( Array.isArray(o.tasks) === false ) { return; }
        for ( const task of o.tasks ) {
-            const ctor = this.operatorToTaskMap.get(task[0]);
-            if ( ctor === undefined ) { return; }
+            const ctor = this.operatorToTaskMap.get(task[0]) || PSelectorVoidTask;
            tasks.push(new ctor(task));
        }
        // Initialize only after all tasks have been successfully instantiated
--- a/src/js/static-filtering-parser.js
+++ b/src/js/static-filtering-parser.js
@ -1499,46 +1499,33 @@ Parser.prototype.SelectorCompiler = class {
        case 'ClassSelector':
        case 'Combinator':
        case 'IdSelector':
+        case 'MediaFeature':
+        case 'Nth':
+        case 'Raw':
        case 'TypeSelector':
            out.push({ data });
            break;
-        case 'Declaration': {
+        case 'Declaration':
            if ( data.value ) {
                this.astFlatten(data.value, args = []);
            }
            out.push({ data, args });
            args = undefined;
            break;
-        }
        case 'DeclarationList':
-            args = out;
-            out.push({ data });
-            break;
        case 'Identifier':
+        case 'MediaQueryList':
+        case 'Selector':
+        case 'SelectorList':
            args = out;
            out.push({ data });
            break;
-        case 'Nth': {
-            out.push({ data });
-            break;
-        }
+        case 'MediaQuery':
        case 'PseudoClassSelector':
        case 'PseudoElementSelector':
            if ( head ) { args = []; }
            out.push({ data, args });
            break;
-        case 'Raw':
-            if ( head ) { args = []; }
-            out.push({ data, args });
-            break;
-        case 'Selector':
-            args = out;
-            out.push({ data });
-            break;
-        case 'SelectorList':
-            args = out;
-            out.push({ data });
-            break;
        case 'Value':
            args = out;
            break;
@ -1552,7 +1539,7 @@ Parser.prototype.SelectorCompiler = class {
            }
            let next = head.next;
            while ( next ) {
-                this.astFlatten(next.data, out);
+                this.astFlatten(next.data, args);
                next = next.next;
            }
        }
@ -1923,15 +1910,12 @@ Parser.prototype.SelectorCompiler = class {
    }

    compileMediaQuery(s) {
-        if ( typeof self !== 'object' ) { return; }
-        if ( self === null ) { return; }
-        if ( typeof self.matchMedia !== 'function' ) { return; }
-        try {
-            const mql = self.matchMedia(s);
-            if ( mql instanceof self.MediaQueryList === false ) { return; }
-            if ( mql.media !== 'not all' ) { return s; }
-        } catch(ex) {
-        }
+        const parts = this.astFromRaw(s, 'mediaQueryList');
+        if ( parts === undefined ) { return; }
+        if ( this.astHasType(parts, 'Raw') ) { return; }
+        if ( this.astHasType(parts, 'MediaQuery') === false ) { return; }
+        // TODO: normalize by serializing resulting AST
+        return s;
    }

    compileUpwardArgument(s) {
--- a/src/js/static-net-filtering.js
+++ b/src/js/static-net-filtering.js
@ -3903,65 +3903,65 @@ FilterContainer.prototype.dnrFromCompiled = function(op, context, ...args) {
        const bucket = buckets.get(bits);

        switch ( tokenHash ) {
-            case DOT_TOKEN_HASH: {
-                if ( bucket.has(DOT_TOKEN_HASH) === false ) {
-                    bucket.set(DOT_TOKEN_HASH, [{
-                        condition: {
-                            requestDomains: []
-                        }
-                    }]);
-                }
-                const rule = bucket.get(DOT_TOKEN_HASH)[0];
-                rule.condition.requestDomains.push(fdata);
-                break;
+        case DOT_TOKEN_HASH: {
+            if ( bucket.has(DOT_TOKEN_HASH) === false ) {
+                bucket.set(DOT_TOKEN_HASH, [{
+                    condition: {
+                        requestDomains: []
+                    }
+                }]);
            }
-            case ANY_TOKEN_HASH: {
-                if ( bucket.has(ANY_TOKEN_HASH) === false ) {
-                    bucket.set(ANY_TOKEN_HASH, [{
-                        condition: {
-                            initiatorDomains: []
-                        }
-                    }]);
-                }
-                const rule = bucket.get(ANY_TOKEN_HASH)[0];
-                rule.condition.initiatorDomains.push(fdata);
-                break;
+            const rule = bucket.get(DOT_TOKEN_HASH)[0];
+            rule.condition.requestDomains.push(fdata);
+            break;
+        }
+        case ANY_TOKEN_HASH: {
+            if ( bucket.has(ANY_TOKEN_HASH) === false ) {
+                bucket.set(ANY_TOKEN_HASH, [{
+                    condition: {
+                        initiatorDomains: []
+                    }
+                }]);
            }
-            case ANY_HTTPS_TOKEN_HASH: {
-                if ( bucket.has(ANY_HTTPS_TOKEN_HASH) === false ) {
-                    bucket.set(ANY_HTTPS_TOKEN_HASH, [{
-                        condition: {
-                            urlFilter: '|https://',
-                            initiatorDomains: []
-                        }
-                    }]);
-                }
-                const rule = bucket.get(ANY_HTTPS_TOKEN_HASH)[0];
-                rule.condition.initiatorDomains.push(fdata);
-                break;
+            const rule = bucket.get(ANY_TOKEN_HASH)[0];
+            rule.condition.initiatorDomains.push(fdata);
+            break;
+        }
+        case ANY_HTTPS_TOKEN_HASH: {
+            if ( bucket.has(ANY_HTTPS_TOKEN_HASH) === false ) {
+                bucket.set(ANY_HTTPS_TOKEN_HASH, [{
+                    condition: {
+                        urlFilter: '|https://',
+                        initiatorDomains: []
+                    }
+                }]);
            }
-            case ANY_HTTP_TOKEN_HASH: {
-                if ( bucket.has(ANY_HTTP_TOKEN_HASH) === false ) {
-                    bucket.set(ANY_HTTP_TOKEN_HASH, [{
-                        condition: {
-                            urlFilter: '|http://',
-                            initiatorDomains: []
-                        }
-                    }]);
-                }
-                const rule = bucket.get(ANY_HTTP_TOKEN_HASH)[0];
-                rule.condition.initiatorDomains.push(fdata);
-                break;
+            const rule = bucket.get(ANY_HTTPS_TOKEN_HASH)[0];
+            rule.condition.initiatorDomains.push(fdata);
+            break;
+        }
+        case ANY_HTTP_TOKEN_HASH: {
+            if ( bucket.has(ANY_HTTP_TOKEN_HASH) === false ) {
+                bucket.set(ANY_HTTP_TOKEN_HASH, [{
+                    condition: {
+                        urlFilter: '|http://',
+                        initiatorDomains: []
+                    }
+                }]);
            }
-            default: {
-                if ( bucket.has(EMPTY_TOKEN_HASH) === false ) {
-                    bucket.set(EMPTY_TOKEN_HASH, []);
-                }
-                const rule = {};
-                dnrRuleFromCompiled(fdata, rule);
-                bucket.get(EMPTY_TOKEN_HASH).push(rule);
-                break;
+            const rule = bucket.get(ANY_HTTP_TOKEN_HASH)[0];
+            rule.condition.initiatorDomains.push(fdata);
+            break;
+        }
+        default: {
+            if ( bucket.has(EMPTY_TOKEN_HASH) === false ) {
+                bucket.set(EMPTY_TOKEN_HASH, []);
            }
+            const rule = {};
+            dnrRuleFromCompiled(fdata, rule);
+            bucket.get(EMPTY_TOKEN_HASH).push(rule);
+            break;
+        }
        }
    }

@ -4066,7 +4066,7 @@ FilterContainer.prototype.dnrFromCompiled = function(op, context, ...args) {
                        }
                    }
                };
-                if ( /^\/.+\/$/.test(rule.__modifierValue) ) {
+                if ( /^~?\/.+\/$/.test(rule.__modifierValue) ) {
                    dnrAddRuleError(rule, `Unsupported regex-based removeParam: ${rule.__modifierValue}`);
                }
            } else {
@ -4076,6 +4076,17 @@ FilterContainer.prototype.dnrFromCompiled = function(op, context, ...args) {
                    }
                };
            }
+            if ( rule.condition === undefined ) {
+                rule.condition = {
+                };
+            }
+            if ( rule.condition.resourceTypes === undefined ) {
+                rule.condition.resourceTypes = [
+                    'main_frame',
+                    'sub_frame',
+                    'xmlhttprequest',
+                ];
+            }
            if ( rule.__modifierAction === AllowAction ) {
                dnrAddRuleError(rule, 'Unhandled modifier exception');
            }