Add ability to uncloak CNAME records

Related issue: - https://github.com/uBlockOrigin/uBlock-issues/issues/780 New webext permission added: `dns`, which purpose is to allow an extension to fetch the DNS record of specific hostnames, reference documentation: https://developer.mozilla.org/en-US/docs/Mozilla/Add-ons/WebExtensions/API/dns The webext API `dns` is available in Firefox 60+ only. The new API will enable uBO to "uncloak" the actual hostname used in network requests. The ability is currently disabled by default for now -- this is only a first commit related to the above issue to allow advanced users to immediately use the new ability. Four advanced settings have been created to control the uncloaking of actual hostnames: cnameAliasList: a space-separated list of hostnames. Default value: unset => empty list. Special value: * => all hostnames. A space-separated list of hostnames => this tells uBO to "uncloak" the hostnames in the list will. cnameIgnoreList: a space-separated list of hostnames. Default value: unset => empty list. Special value: * => all hostnames. A space-separated list of hostnames => this tells uBO to NOT re-run the network request through uBO's filtering engine with the CNAME hostname. This is useful to exclude commonly used actual hostnames from being re-run through uBO's filtering engine, so as to avoid pointless overhead. cnameIgnore1stParty: boolean. Default value: true. Whether uBO should ignore to re-run a network request through the filtering engine when the CNAME hostname is 1st-party to the alias hostname. cnameMaxTTL: number of minutes. Default value: 120. This tells uBO to clear its CNAME cache after the specified time. For efficiency purpose, uBO will cache alias=>CNAME associations for reuse so as to reduce calls to `browser.dns.resolve`. All the associations will be cleared after the specified time to ensure the map does not grow too large and too ensure uBO uses up to date CNAME information.
2019-11-19 12:05:33 -05:00 · 2019-11-19 12:05:33 -05:00 · 3a564c1992
parent 60816b68a1
commit 3a564c1992
12 changed files with 226 additions and 92 deletions
--- a/platform/chromium/vapi-background.js
+++ b/platform/chromium/vapi-background.js
@ -1164,16 +1164,17 @@ vAPI.Net = class {
        browser.webRequest.onBeforeRequest.addListener(
            details => {
                this.normalizeDetails(details);
-                if ( this.suspendDepth === 0 || details.tabId < 0 ) {
-                    if ( this.suspendableListener === undefined ) { return; }
-                    return this.suspendableListener(details);
+                if ( this.suspendDepth !== 0 && details.tabId >= 0 ) {
+                    return this.suspendOneRequest(details);
                }
-                return this.suspendOneRequest(details);
+                return this.onBeforeSuspendableRequest(details);
            },
            this.denormalizeFilters({ urls: [ 'http://*/*', 'https://*/*' ] }),
            [ 'blocking' ]
        );
    }
+    setOptions(/* options */) {
+    }
    normalizeDetails(/* details */) {
    }
    denormalizeFilters(filters) {
@ -1208,6 +1209,10 @@ vAPI.Net = class {
            options
        );
    }
+    onBeforeSuspendableRequest(details) {
+        if ( this.suspendableListener === undefined ) { return; }
+        return this.suspendableListener(details);
+    }
    setSuspendableListener(listener) {
        this.suspendableListener = listener;
    }
@ -1242,7 +1247,7 @@ vAPI.Net = class {
            this.suspendDepth -= 1;
        }
        if ( this.suspendDepth !== 0 ) { return; }
-        this.unsuspendAllRequests(this.suspendableListener);
+        this.unsuspendAllRequests();
    }
    canSuspend() {
        return false;
--- a/platform/chromium/vapi-common.js
+++ b/platform/chromium/vapi-common.js
@ -116,6 +116,51 @@ vAPI.webextFlavor = {

 /******************************************************************************/

+{
+    const punycode = self.punycode;
+    const reCommonHostnameFromURL  = /^https?:\/\/([0-9a-z_][0-9a-z._-]*[0-9a-z])\//;
+    const reAuthorityFromURI       = /^(?:[^:\/?#]+:)?(\/\/[^\/?#]+)/;
+    const reHostFromNakedAuthority = /^[0-9a-z._-]+[0-9a-z]$/i;
+    const reHostFromAuthority      = /^(?:[^@]*@)?([^:]+)(?::\d*)?$/;
+    const reIPv6FromAuthority      = /^(?:[^@]*@)?(\[[0-9a-f:]+\])(?::\d*)?$/i;
+    const reMustNormalizeHostname  = /[^0-9a-z._-]/;
+
+    vAPI.hostnameFromURI = function(uri) {
+        let matches = reCommonHostnameFromURL.exec(uri);
+        if ( matches !== null ) { return matches[1]; }
+        matches = reAuthorityFromURI.exec(uri);
+        if ( matches === null ) { return ''; }
+        const authority = matches[1].slice(2);
+        if ( reHostFromNakedAuthority.test(authority) ) {
+            return authority.toLowerCase();
+        }
+        matches = reHostFromAuthority.exec(authority);
+        if ( matches === null ) {
+            matches = reIPv6FromAuthority.exec(authority);
+            if ( matches === null ) { return ''; }
+        }
+        let hostname = matches[1];
+        while ( hostname.endsWith('.') ) {
+            hostname = hostname.slice(0, -1);
+        }
+        if ( reMustNormalizeHostname.test(hostname) ) {
+            hostname = punycode.toASCII(hostname.toLowerCase());
+        }
+        return hostname;
+    };
+
+    const psl = self.publicSuffixList;
+    const reIPAddressNaive = /^\d+\.\d+\.\d+\.\d+$|^\[[\da-zA-Z:]+\]$/;
+
+    vAPI.domainFromHostname = function(hostname) {
+        return reIPAddressNaive.test(hostname)
+            ? hostname
+            : psl.getDomain(hostname);
+    };
+}
+
+/******************************************************************************/
+
 vAPI.download = function(details) {
    if ( !details.url ) { return; }
    const a = document.createElement('a');
--- a/platform/firefox/manifest.json
+++ b/platform/firefox/manifest.json
@ -75,6 +75,7 @@
    "open_in_tab": true
  },
  "permissions": [
+    "dns",
    "menus",
    "privacy",
    "storage",
--- a/platform/firefox/vapi-webrequest.js
+++ b/platform/firefox/vapi-webrequest.js
@ -60,6 +60,20 @@
        constructor() {
            super();
            this.pendingRequests = [];
+            this.cnames = new Map();
+            this.cnameAliasList = null;
+            this.cnameIgnoreList = null;
+            this.url = new URL(vAPI.getURL('/'));
+            this.cnameMaxTTL = 60;
+            this.cnameTimer = undefined;
+        }
+        setOptions(options) {
+            super.setOptions(options);
+            this.cnameAliasList = this.regexFromStrList(options.cnameAliasList);
+            this.cnameIgnoreList = this.regexFromStrList(options.cnameIgnoreList);
+            this.cnameIgnore1stParty = options.cnameIgnore1stParty === true;
+            this.cnameMaxTTL = options.cnameMaxTTL || 120;
+            this.cnames.clear();
        }
        normalizeDetails(details) {
            if ( mustPunycode && !reAsciiHostname.test(details.url) ) {
@ -109,6 +123,87 @@
            }
            return Array.from(out);
        }
+        processCanonicalName(cname, details) {
+            this.url.href = details.url;
+            details.cnameOf = this.url.hostname;
+            this.url.hostname = cname;
+            details.url = this.url.href;
+            return super.onBeforeSuspendableRequest(details);
+        }
+        recordCanonicalName(hn, record) {
+            let cname =
+                typeof record.canonicalName === 'string' &&
+                record.canonicalName !== hn
+                    ? record.canonicalName
+                    : '';
+            if (
+                cname !== '' &&
+                this.cnameIgnore1stParty &&
+                vAPI.domainFromHostname(cname) === vAPI.domainFromHostname(hn)
+            ) {
+                cname = '';
+            }
+            if (
+                cname !== '' &&
+                this.cnameIgnoreList !== null &&
+                this.cnameIgnoreList.test(cname)
+            ) {
+
+                cname = '';
+            }
+            this.cnames.set(hn, cname);
+            if ( this.cnameTimer === undefined ) {
+                this.cnameTimer = self.setTimeout(
+                    ( ) => {
+                        this.cnameTimer = undefined;
+                        this.cnames.clear();
+                    },
+                    this.cnameMaxTTL * 60000
+                );
+            }
+            return cname;
+        }
+        regexFromStrList(list) {
+            if (
+                typeof list !== 'string' ||
+                list.length === 0 ||
+                list === 'unset'
+            ) {
+                return null;
+            }
+            if ( list === '*' ) {
+                return /^./;
+            }
+            return new RegExp(
+                '(?:^|\.)(?:' +
+                list.trim()
+                    .split(/\s+/)
+                    .map(a => a.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'))
+                    .join('|') +
+                ')$'
+            );
+        }
+        onBeforeSuspendableRequest(details) {
+            let r = super.onBeforeSuspendableRequest(details);
+            if ( r !== undefined ) { return r; }
+            if ( this.cnameAliasList === null ) { return; }
+            const hn = vAPI.hostnameFromURI(details.url);
+            let cname = this.cnames.get(hn);
+            if ( cname === '' ) { return; }
+            if ( cname !== undefined ) {
+                return this.processCanonicalName(cname, details);
+            }
+            if ( this.cnameAliasList.test(hn) === false ) {
+                this.cnames.set(hn, '');
+                return;
+            }
+            return browser.dns.resolve(hn, [ 'canonical_name' ]).then(rec => {
+                const cname = this.recordCanonicalName(hn, rec);
+                if ( cname === '' ) { return; }
+                return this.processCanonicalName(cname, details);
+
+            });
+        }
        suspendOneRequest(details) {
            const pending = {
                details: Object.assign({}, details),
@ -121,11 +216,11 @@
            this.pendingRequests.push(pending);
            return pending.promise;
        }
-        unsuspendAllRequests(resolver) {
+        unsuspendAllRequests() {
            const pendingRequests = this.pendingRequests;
            this.pendingRequests = [];
            for ( const entry of pendingRequests ) {
-                entry.resolve(resolver(entry.details));
+                entry.resolve(this.onBeforeSuspendableRequest(entry.details));
            }
        }
        canSuspend() {
--- a/src/css/logger-ui.css
+++ b/src/css/logger-ui.css
@ -268,6 +268,9 @@ body.colorBlind #vwRenderer .logEntry > div.cosmeticRealm,
 body.colorBlind #vwRenderer .logEntry > div.redirect {
    background-color: rgba(0, 19, 110, 0.1);
    }
+#vwRenderer .logEntry > div[data-cnameof] {
+    color: mediumblue;
+    }
 #vwRenderer .logEntry > div[data-type="tabLoad"] {
    background-color: #666;
    color: white;
--- a/src/js/background.js
+++ b/src/js/background.js
@ -46,6 +46,10 @@ const µBlock = (( ) => { // jshint ignore:line
        cacheStorageAPI: 'unset',
        cacheStorageCompression: true,
        cacheControlForFirefox1376932: 'no-cache, no-store, must-revalidate',
+        cnameAliasList: 'unset',
+        cnameIgnoreList: 'unset',
+        cnameIgnore1stParty: true,
+        cnameMaxTTL: 120,
        consoleLogLevel: 'unset',
        debugScriptlets: false,
        debugScriptletInjector: false,
--- a/src/js/filtering-context.js
+++ b/src/js/filtering-context.js
@ -30,6 +30,7 @@
    this.tstamp = 0;
    this.realm = '';
    this.type = undefined;
+    this.cnameOf = undefined;
    this.url = undefined;
    this.hostname = undefined;
    this.domain = undefined;
@ -65,6 +66,7 @@
        this.realm = '';
        this.type = details.type;
        this.setURL(details.url);
+        this.cnameOf = details.cnameOf !== undefined ? details.cnameOf : '';
        this.docId = details.type !== 'sub_frame'
            ? details.frameId
            : details.parentFrameId;
--- a/src/js/logger-ui.js
+++ b/src/js/logger-ui.js
@ -44,6 +44,7 @@ let activeTabId = 0;
 let filterAuthorMode = false;
 let selectedTabId = 0;
 let netInspectorPaused = false;
+let cnameOfEnabled = false;

 /******************************************************************************/

@ -221,6 +222,7 @@ const LogEntry = function(details) {
    }
 };
 LogEntry.prototype = {
+    cnameOf: '',
    dead: false,
    docDomain: '',
    docHostname: '',
@ -292,7 +294,7 @@ const processLoggerEntries = function(response) {
            if ( autoDeleteVoidedRows ) { continue; }
            parsed.voided = true;
        }
-        if ( parsed.type === 'main_frame' ) {
+        if ( parsed.type === 'main_frame' && parsed.cnameOf === '' ) {
            const separator = createLogSeparator(parsed, unboxed.url);
            loggerEntries.unshift(separator);
            if ( rowFilterer.filterOne(separator) ) {
@ -302,6 +304,10 @@ const processLoggerEntries = function(response) {
                }
            }
        }
+        if ( cnameOfEnabled === false && parsed.cnameOf !== '' ) {
+            uDom.nodeFromId('filterExprCnameOf').style.display = '';
+            cnameOfEnabled = true;
+        }
        loggerEntries.unshift(parsed);
        if ( rowFilterer.filterOne(parsed) ) {
            filteredLoggerEntries.unshift(parsed);
@ -364,29 +370,28 @@ const parseLogEntry = function(details) {
    textContent.push(normalizeToStr(entry.docHostname));

    // Cell 4
-    if (
-        entry.realm === 'network' &&
-        typeof entry.domain === 'string' &&
-        entry.domain !== ''
-    ) {
-        let partyness = '';
-        if ( entry.tabDomain !== undefined ) {
-            if ( entry.tabId < 0 ) {
-                partyness += '0,';
-            }
-            partyness += entry.domain === entry.tabDomain ? '1' : '3';
-        } else {
-            partyness += '?';
-        }
-        if ( entry.docDomain !== entry.tabDomain ) {
-            partyness += ',';
-            if ( entry.docDomain !== undefined ) {
-                partyness += entry.domain === entry.docDomain ? '1' : '3';
+    if ( entry.realm === 'network' ) {
+        // partyness
+        if ( typeof entry.domain === 'string' && entry.domain !== '' ) {
+            let partyness = '';
+            if ( entry.tabDomain !== undefined ) {
+                if ( entry.tabId < 0 ) {
+                    partyness += '0,';
+                }
+                partyness += entry.domain === entry.tabDomain ? '1' : '3';
            } else {
                partyness += '?';
            }
+            if ( entry.docDomain !== entry.tabDomain ) {
+                partyness += ',';
+                if ( entry.docDomain !== undefined ) {
+                    partyness += entry.domain === entry.docDomain ? '1' : '3';
+                } else {
+                    partyness += '?';
+                }
+            }
+            textContent.push(partyness);
        }
-        textContent.push(partyness);
    } else {
        textContent.push('');
    }
@ -399,6 +404,11 @@ const parseLogEntry = function(details) {
    // Cell 6
    textContent.push(normalizeToStr(details.url));

+    // Hidden cells -- useful for row-filtering purpose
+    if ( entry.cnameOf !== '' ) {
+        textContent.push(`cnameOf=${entry.cnameOf}`);
+    }
+
    entry.textContent = textContent.join('\t');
    return entry;
 };
@ -721,6 +731,11 @@ const viewPort = (( ) => {
        }
        nodeFromURL(div.children[6], cells[6], re);

+        // Cname
+        if ( details.cnameOf !== '' ) {
+            div.setAttribute('data-cnameof', details.cnameOf);
+        }
+
        return div;
    };

@ -1608,6 +1623,13 @@ const reloadTab = function(ev) {
        } else {
            rows[7].style.display = 'none';
        }
+        // CNAME of
+        text = tr.getAttribute('data-cnameof') || '';
+        if ( text !== '' ) {
+            rows[8].children[1].textContent = text;
+        } else {
+            rows[8].style.display = 'none';
+        }
    };

    // Fill dynamic URL filtering pane
@ -1951,14 +1973,14 @@ const rowFilterer = (( ) => {
        );
    };

-    const onFilterChangedAsync = (function() {
+    const onFilterChangedAsync = (( ) => {
        let timer;
        const commit = ( ) => {
            timer = undefined;
            parseInput();
            filterAll();
        };
-        return function() {
+        return ( ) => {
            if ( timer !== undefined ) {
                clearTimeout(timer);
            }
--- a/src/js/storage.js
+++ b/src/js/storage.js
@ -112,7 +112,6 @@
                    : 'unset';
        }
    }
-    self.log.verbosity = this.hiddenSettings.consoleLogLevel;
    this.fireDOMEvent('hiddenSettingsChanged');
 };

@ -132,9 +131,18 @@
    }
    vAPI.storage.set(bin);
    this.saveImmediateHiddenSettings();
-    self.log.verbosity = this.hiddenSettings.consoleLogLevel;
 };

+self.addEventListener('hiddenSettingsChanged', ( ) => {
+    self.log.verbosity = µBlock.hiddenSettings.consoleLogLevel;
+    vAPI.net.setOptions({
+        cnameAliasList: µBlock.hiddenSettings.cnameAliasList,
+        cnameIgnoreList: µBlock.hiddenSettings.cnameIgnoreList,
+        cnameIgnore1stParty: µBlock.hiddenSettings.cnameIgnore1stParty,
+        cnameMaxTTL: µBlock.hiddenSettings.cnameMaxTTL,
+    });
+});
+
 /******************************************************************************/

 µBlock.hiddenSettingsFromString = function(raw) {
--- a/src/js/traffic.js
+++ b/src/js/traffic.js
@ -96,7 +96,11 @@ const onBeforeRequest = function(details) {

    // Not blocked
    if ( result !== 1 ) {
-        if ( details.parentFrameId !== -1 && details.type === 'sub_frame' ) {
+        if (
+            details.parentFrameId !== -1 &&
+            details.type === 'sub_frame' &&
+            details.cnameOf === undefined
+        ) {
            pageStore.setFrame(details.frameId, details.url);
        }
        return;
--- a/src/js/uritools.js
+++ b/src/js/uritools.js
@ -19,8 +19,6 @@
    Home: https://github.com/gorhill/uBlock
 */

-/* global publicSuffixList */
-
 'use strict';

 /*******************************************************************************
@ -33,12 +31,10 @@ Naming convention from https://en.wikipedia.org/wiki/URI_scheme#Examples

 /******************************************************************************/

-µBlock.URI = (function() {
+µBlock.URI = (( ) => {

 /******************************************************************************/

-const punycode = self.punycode;
-
 // Favorite regex tool: http://regex101.com/

 // Ref: <http://tools.ietf.org/html/rfc3986#page-50>
@ -50,11 +46,8 @@ const reRFC3986 = /^([^:\/?#]+:)?(\/\/[^\/?#]*)?([^?#]*)(\?[^#]*)?(#.*)?/;

 // Derived
 const reSchemeFromURI          = /^[^:\/?#]+:/;
-const reAuthorityFromURI       = /^(?:[^:\/?#]+:)?(\/\/[^\/?#]+)/;
 const reOriginFromURI          = /^(?:[^:\/?#]+:)\/\/[^\/?#]+/;
-const reCommonHostnameFromURL  = /^https?:\/\/([0-9a-z_][0-9a-z._-]*[0-9a-z])\//;
 const rePathFromURI            = /^(?:[^:\/?#]+:)?(?:\/\/[^\/?#]*)?([^?#]*)/;
-const reMustNormalizeHostname  = /[^0-9a-z._-]/;

 // These are to parse authority field, not parsed by above official regex
 // IPv6 is seen as an exception: a non-compatible IPv6 is first tried, and
@ -68,12 +61,9 @@ const reHostPortFromAuthority  = /^(?:[^@]*@)?([^:]*)(:\d*)?$/;
 const reIPv6PortFromAuthority  = /^(?:[^@]*@)?(\[[0-9a-f:]*\])(:\d*)?$/i;

 const reHostFromNakedAuthority = /^[0-9a-z._-]+[0-9a-z]$/i;
-const reHostFromAuthority      = /^(?:[^@]*@)?([^:]+)(?::\d*)?$/;
-const reIPv6FromAuthority      = /^(?:[^@]*@)?(\[[0-9a-f:]+\])(?::\d*)?$/i;

 // Coarse (but fast) tests
 const reValidHostname          = /^([a-z\d]+(-*[a-z\d]+)*)(\.[a-z\d]+(-*[a-z\d])*)*$/;
-const reIPAddressNaive         = /^\d+\.\d+\.\d+\.\d+$|^\[[\da-zA-Z:]+\]$/;

 /******************************************************************************/

@ -239,60 +229,13 @@ URI.schemeFromURI = function(uri) {

 /******************************************************************************/

-URI.authorityFromURI = function(uri) {
-    const matches = reAuthorityFromURI.exec(uri);
-    if ( !matches ) { return ''; }
-    return matches[1].slice(2).toLowerCase();
-};
-
-/******************************************************************************/
-
-// The most used function, so it better be fast.
-
-// https://github.com/gorhill/uBlock/issues/1559
-//   See http://en.wikipedia.org/wiki/FQDN
-// https://bugzilla.mozilla.org/show_bug.cgi?id=1360285
-//   Revisit punycode dependency when above issue is fixed in Firefox.
-
-URI.hostnameFromURI = function(uri) {
-    let matches = reCommonHostnameFromURL.exec(uri);
-    if ( matches !== null ) { return matches[1]; }
-    matches = reAuthorityFromURI.exec(uri);
-    if ( matches === null ) { return ''; }
-    const authority = matches[1].slice(2);
-    // Assume very simple authority (most common case for µBlock)
-    if ( reHostFromNakedAuthority.test(authority) ) {
-        return authority.toLowerCase();
-    }
-    matches = reHostFromAuthority.exec(authority);
-    if ( matches === null ) {
-        matches = reIPv6FromAuthority.exec(authority);
-        if ( matches === null ) { return ''; }
-    }
-    let hostname = matches[1];
-    while ( hostname.endsWith('.') ) {
-        hostname = hostname.slice(0, -1);
-    }
-    if ( reMustNormalizeHostname.test(hostname) ) {
-        hostname = punycode.toASCII(hostname.toLowerCase());
-    }
-    return hostname;
-};
-
-/******************************************************************************/
-
-URI.domainFromHostname = function(hostname) {
-    return reIPAddressNaive.test(hostname) ? hostname : psl.getDomain(hostname);
-};
+URI.hostnameFromURI = vAPI.hostnameFromURI;
+URI.domainFromHostname = vAPI.domainFromHostname;

 URI.domain = function() {
    return this.domainFromHostname(this.hostname);
 };

-// It is expected that there is higher-scoped `publicSuffixList` lingering
-// somewhere. Cache it. See <https://github.com/gorhill/publicsuffixlist.js>.
-const psl = publicSuffixList;
-
 /******************************************************************************/

 URI.entityFromDomain = function(domain) {
--- a/src/logger-ui.html
+++ b/src/logger-ui.html
@ -65,6 +65,7 @@
                                </span>
                            </div>
                            <div><span data-filtex="!" data-i18n="loggerRowFiltererBuiltinNot"></span><span data-filtex="\t(?:0,)?1\t" data-i18n="loggerRowFiltererBuiltin1p"></span><span data-filtex="\t(?:3(?:,\d)?|0,3)\t" data-i18n="loggerRowFiltererBuiltin3p"></span></div>
+                            <div id="filterExprCnameOf" style="display:none"><span data-filtex="!" data-i18n="loggerRowFiltererBuiltinNot"></span><span data-filtex="\tcnameOf=.">CNAME</span></div>
                        </div>
                    </span>
                </span>
@ -120,6 +121,7 @@
                <div><span data-i18n="loggerEntryDetailsPartyness"></span><span class="prose"></span></div>
                <div><span data-i18n="loggerEntryDetailsType"></span><span></span></div>
                <div><span data-i18n="loggerEntryDetailsURL"></span><span></span></div>
+                <div><span >CNAME of</span><span></span></div>
            </div>
            <div class="pane dynamic hide" data-pane="dynamic">
                <div class="toolbar row">