Firefox WebExt readiness (#622) + support more webRequest types when available (websocket, font, csp_report)

platform/chromium/vapi-background.js

@@ -1,7 +1,7 @@
 /*******************************************************************************
 
     uBlock Origin - a browser extension to block requests.
-    Copyright (C) 2014-2016 The uBlock Origin authors
+    Copyright (C) 2014-2017 The uBlock Origin authors
 
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -859,26 +859,36 @@ vAPI.net = {};
 /******************************************************************************/
 
 vAPI.net.registerListeners = function() {
-    var µb = µBlock;
+    var µb = µBlock,
-    var µburi = µb.URI;
+        µburi = µb.URI,
+        wrApi = chrome.webRequest;
 
     // https://bugs.chromium.org/p/chromium/issues/detail?id=410382
     // Between Chromium 38-48, plug-ins' network requests were reported as
     // type "other" instead of "object".
-    var is_v38_48 = /\bChrom[a-z]+\/(?:3[89]|4[0-8])\.[\d.]+\b/.test(navigator.userAgent),
+    var is_v38_48 = /\bChrom[a-z]+\/(?:3[89]|4[0-8])\.[\d.]+\b/.test(navigator.userAgent);
-        is_v49_55 = /\bChrom[a-z]+\/(?:49|5[012345])\b/.test(navigator.userAgent);
 
-    // Chromium-based browsers understand only these network request types.
+    // legacy Chromium understands only these network request types.
-    var validTypes = [
+    var validTypes = {
-        'main_frame',
+        main_frame: true,
-        'sub_frame',
+        sub_frame: true,
-        'stylesheet',
+        stylesheet: true,
-        'script',
+        script: true,
-        'image',
+        image: true,
-        'object',
+        object: true,
-        'xmlhttprequest',
+        xmlhttprequest: true,
-        'other'
+        other: true
-    ];
+    };
+    // modern Chromium/WebExtensions: more types available.
+    if ( wrApi.ResourceType ) {
+        (function() {
+            for ( var typeKey in wrApi.ResourceType ) {
+                if ( wrApi.ResourceType.hasOwnProperty(typeKey) ) {
+                    validTypes[wrApi.ResourceType[typeKey]] = true;
+                }
+            }
+        })();
+    }
 
     var extToTypeMap = new Map([
         ['eot','font'],['otf','font'],['svg','font'],['ttf','font'],['woff','font'],['woff2','font'],
@@ -888,7 +898,7 @@ vAPI.net.registerListeners = function() {
 
     var denormalizeTypes = function(aa) {
         if ( aa.length === 0 ) {
-            return validTypes;
+            return Object.keys(validTypes);
         }
         var out = [];
         var i = aa.length,
@@ -896,7 +906,7 @@ vAPI.net.registerListeners = function() {
             needOther = true;
         while ( i-- ) {
             type = aa[i];
-            if ( validTypes.indexOf(type) !== -1 ) {
+            if ( validTypes[type] ) {
                 out.push(type);
             }
             if ( type === 'other' ) {
@@ -922,23 +932,29 @@ vAPI.net.registerListeners = function() {
     var normalizeRequestDetails = function(details) {
         details.tabId = details.tabId.toString();
 
+        var type = details.type;
+
         // https://github.com/gorhill/uBlock/issues/1493
-        // Chromium 49+ support a new request type: `ping`, which is fired as
+        // Chromium 49+/WebExtensions support a new request type: `ping`,
-        // a result of using `navigator.sendBeacon`.
+        // which is fired as a result of using `navigator.sendBeacon`.
-        if ( details.type === 'ping' ) {
+        if ( type === 'ping' ) {
             details.type = 'beacon';
             return;
         }
 
+        if ( type === 'imageset' ) {
+            details.type = 'image';
+            return;
+        }
+
         // The rest of the function code is to normalize type
-        if ( details.type !== 'other' ) {
+        if ( type !== 'other' ) {
             return;
         }
 
         // Try to map known "extension" part of URL to request type.
         var path = µburi.pathFromURI(details.url),
-            pos = path.indexOf('.', path.length - 6),
+            pos = path.indexOf('.', path.length - 6);
-            type;
         if ( pos !== -1 && (type = extToTypeMap.get(path.slice(pos + 1))) ) {
             details.type = type;
             return;
@@ -1002,81 +1018,110 @@ vAPI.net.registerListeners = function() {
     };
 
     var onBeforeRequestClient = this.onBeforeRequest.callback;
-    var onBeforeRequest = function(details) {
+    var onBeforeRequest = validTypes.websocket
-        // https://github.com/gorhill/uBlock/issues/1497
+        // modern Chromium/WebExtensions: type 'websocket' is supported
-        if ( details.url.endsWith('ubofix=f41665f3028c7fd10eecf573336216d3') ) {
+        ? function(details) {
-            var r = onBeforeWebsocketRequest(details);
+            normalizeRequestDetails(details);
-            if ( r !== undefined ) { return r; }
-        }
-
-        normalizeRequestDetails(details);
-        return onBeforeRequestClient(details);
-    };
-
-    // This is needed for Chromium 49-55.
-    var onBeforeSendHeaders = function(details) {
-        if ( details.type !== 'ping' || details.method !== 'POST' ) { return; }
-        var type = headerValue(details.requestHeaders, 'content-type');
-        if ( type === '' ) { return; }
-        if ( type.endsWith('/csp-report') ) {
-            details.type = 'csp_report';
             return onBeforeRequestClient(details);
         }
-    };
+        // legacy Chromium
+        : function(details) {
+            // https://github.com/gorhill/uBlock/issues/1497
+            if ( details.url.endsWith('ubofix=f41665f3028c7fd10eecf573336216d3') ) {
+                var r = onBeforeWebsocketRequest(details);
+                if ( r !== undefined ) { return r; }
+            }
+            normalizeRequestDetails(details);
+            return onBeforeRequestClient(details);
+        };
+
+    // This is needed for Chromium 49-55.
+    var onBeforeSendHeaders = validTypes.csp_report
+        // modern Chromium/WebExtensions: type 'csp_report' is supported
+        ? null
+        // legacy Chromium
+        : function(details) {
+            if ( details.type !== 'ping' || details.method !== 'POST' ) { return; }
+            var type = headerValue(details.requestHeaders, 'content-type');
+            if ( type === '' ) { return; }
+            if ( type.endsWith('/csp-report') ) {
+                details.type = 'csp_report';
+                return onBeforeRequestClient(details);
+            }
+        };
 
     var onHeadersReceivedClient = this.onHeadersReceived.callback,
         onHeadersReceivedClientTypes = this.onHeadersReceived.types.slice(0),
         onHeadersReceivedTypes = denormalizeTypes(onHeadersReceivedClientTypes);
-    var onHeadersReceived = function(details) {
+    var onHeadersReceived = validTypes.font
-        normalizeRequestDetails(details);
+        // modern Chromium/WebExtensions: type 'font' is supported
-        // Hack to work around Chromium API limitations, where requests of
+        ? function(details) {
-        // type `font` are returned as `other`. For example, our normalization
+            normalizeRequestDetails(details);
-        // fail at transposing `other` into `font` for URLs which are outside
+            if (
-        // what is expected. At least when headers are received we can check
+                onHeadersReceivedClientTypes.length !== 0 &&
-        // for content type `font/*`. Blocking at onHeadersReceived time is
+                onHeadersReceivedClientTypes.indexOf(details.type) === -1
-        // less worse than not blocking at all. Also, due to Chromium bug,
+            ) {
-        // `other` always becomes `object` when it can't be normalized into
+                return;
-        // something else. Test case for "unfriendly" font URLs:
-        //   https://www.google.com/fonts
-        if ( details.type === 'font' ) {
-            var r = onBeforeRequestClient(details);
-            if ( typeof r === 'object' && r.cancel === true ) {
-                return { cancel: true };
             }
+            return onHeadersReceivedClient(details);
         }
-        if (
+        // legacy Chromium
-            onHeadersReceivedClientTypes.length !== 0 &&
+        : function(details) {
-            onHeadersReceivedClientTypes.indexOf(details.type) === -1
+            normalizeRequestDetails(details);
-        ) {
+            // Hack to work around Chromium API limitations, where requests of
-            return;
+            // type `font` are returned as `other`. For example, our normalization
-        }
+            // fail at transposing `other` into `font` for URLs which are outside
-        return onHeadersReceivedClient(details);
+            // what is expected. At least when headers are received we can check
-    };
+            // for content type `font/*`. Blocking at onHeadersReceived time is
+            // less worse than not blocking at all. Also, due to Chromium bug,
+            // `other` always becomes `object` when it can't be normalized into
+            // something else. Test case for "unfriendly" font URLs:
+            //   https://www.google.com/fonts
+            if ( details.type === 'font' ) {
+                var r = onBeforeRequestClient(details);
+                if ( typeof r === 'object' && r.cancel === true ) {
+                    return { cancel: true };
+                }
+            }
+            if (
+                onHeadersReceivedClientTypes.length !== 0 &&
+                onHeadersReceivedClientTypes.indexOf(details.type) === -1
+            ) {
+                return;
+            }
+            return onHeadersReceivedClient(details);
+        };
+
 
     var installListeners = (function() {
-        var crapi = chrome.webRequest;
-
         //listener = function(details) {
         //    quickProfiler.start('onBeforeRequest');
         //    var r = onBeforeRequest(details);
         //    quickProfiler.stop();
         //    return r;
         //};
-        if ( crapi.onBeforeRequest.hasListener(onBeforeRequest) === false ) {
+        var urls, types;
-            crapi.onBeforeRequest.addListener(
+
+        if (
+            onBeforeRequest &&
+            wrApi.onBeforeRequest.hasListener(onBeforeRequest) === false
+        ) {
+            urls = this.onBeforeRequest.urls || ['<all_urls>'];
+            types = this.onBeforeRequest.types || undefined;
+            wrApi.onBeforeRequest.addListener(
                 onBeforeRequest,
-                {
+                { urls: urls, types: types },
-                    'urls': this.onBeforeRequest.urls || ['<all_urls>'],
-                    'types': this.onBeforeRequest.types || undefined
-                },
                 this.onBeforeRequest.extra
             );
         }
 
         // Chromium 48 and lower does not support `ping` type.
         // Chromium 56 and higher does support `csp_report` stype.
-        if ( is_v49_55 && crapi.onBeforeSendHeaders.hasListener(onBeforeSendHeaders) === false ) {
+        if (
-            crapi.onBeforeSendHeaders.addListener(
+            onBeforeSendHeaders &&
+            wrApi.onBeforeSendHeaders.hasListener(onBeforeSendHeaders) === false
+        ) {
+            wrApi.onBeforeSendHeaders.addListener(
                 onBeforeSendHeaders,
                 {
                     'urls': [ '<all_urls>' ],
@@ -1086,13 +1131,15 @@ vAPI.net.registerListeners = function() {
             );
         }
 
-        if ( crapi.onHeadersReceived.hasListener(onHeadersReceived) === false ) {
+        if (
-            crapi.onHeadersReceived.addListener(
+            onHeadersReceived &&
+            wrApi.onHeadersReceived.hasListener(onHeadersReceived) === false
+        ) {
+            urls = this.onHeadersReceived.urls || ['<all_urls>'];
+            types = onHeadersReceivedTypes;
+            wrApi.onHeadersReceived.addListener(
                 onHeadersReceived,
-                {
+                { urls: urls, types: types },
-                    'urls': this.onHeadersReceived.urls || ['<all_urls>'],
-                    'types': onHeadersReceivedTypes
-                },
                 this.onHeadersReceived.extra
             );
         }

platform/webext/background.html

@@ -0,0 +1,37 @@
+<!DOCTYPE html>
+<html>
+<head>
+<meta charset="utf-8">
+<title>uBlock Origin</title>
+</head>
+<body>
+<script src="js/polyfill.js"></script>
+<script src="lib/punycode.js"></script>
+<script src="lib/publicsuffixlist.js"></script>
+<script src="js/vapi-common.js"></script>
+<script src="js/vapi-background.js"></script>
+<script src="js/background.js"></script>
+<script src="js/utils.js"></script>
+<script src="js/uritools.js"></script>
+<script src="js/assets.js"></script>
+<script src="js/redirect-engine.js"></script>
+<script src="js/dynamic-net-filtering.js"></script>
+<script src="js/static-net-filtering.js"></script>
+<script src="js/url-net-filtering.js"></script>
+<script src="js/cosmetic-filtering.js"></script>
+<script src="js/hnswitches.js"></script>
+<script src="js/ublock.js"></script>
+<script src="js/messaging.js"></script>
+<script src="js/profiler.js"></script>
+<script src="js/storage.js"></script>
+<script src="js/logger.js"></script>
+<script src="js/pagestore.js"></script>
+<script src="js/tab.js"></script>
+<script src="js/traffic.js"></script>
+<script src="js/contextmenu.js"></script>
+<script src="js/reverselookup.js"></script>
+<script src="js/rpcreceiver.js"></script>
+<script src="js/from-legacy.js"></script>
+<script src="js/start.js"></script>
+</body>
+</html>

platform/webext/bootstrap.js

@@ -0,0 +1,280 @@
+/*******************************************************************************
+
+    uBlock Origin - a browser extension to block requests.
+    Copyright (C) 2014-2017 The uBlock Origin authors
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see {http://www.gnu.org/licenses/}.
+
+    Home: https://github.com/gorhill/uBlock
+*/
+
+/* exported startup, shutdown, install, uninstall */
+
+'use strict';
+
+/******************************************************************************/
+
+const hostName = 'ublock0';
+
+/******************************************************************************/
+
+function startup({ webExtension }) {
+    webExtension.startup().then(api => {
+        let { browser } = api,
+            storageMigrator;
+        let onMessage = function(message, sender, callback) {
+            if ( message.what === 'webext:storageMigrateNext' ) {
+                storageMigrator = storageMigrator || getStorageMigrator();
+                storageMigrator.getNext((key, value) => {
+                    if ( key === undefined ) {
+                        storageMigrator.markAsDone();
+                        storageMigrator = undefined;
+                        browser.runtime.onMessage.removeListener(onMessage);
+                    }
+                    callback({ key: key, value: JSON.stringify(value) });
+                });
+                return true;
+            }
+            if ( typeof callback === 'function' ) {
+                callback();
+            }
+        };
+        browser.runtime.onMessage.addListener(onMessage);
+    });
+}
+
+function shutdown() {
+}
+
+function install() {
+}
+
+function uninstall() {
+}
+
+/******************************************************************************/
+
+var getStorageMigrator = function() {
+    var db = null;
+    var dbOpenError = '';
+
+    var close = function() {
+        if ( db !== null ) {
+            db.asyncClose();
+        }
+        db = null;
+    };
+
+    var open = function() {
+        if ( db !== null ) {
+            return db;
+        }
+
+        // Create path
+        var { Services } = Components.utils.import('resource://gre/modules/Services.jsm', null),
+            path = Services.dirsvc.get('ProfD', Components.interfaces.nsIFile);
+        path.append('extension-data');
+        path.append(hostName + '.sqlite');
+        if ( !path.exists() || !path.isFile() ) {
+            return null;
+        }
+
+        // Open database.
+        try {
+            db = Services.storage.openDatabase(path);
+            if ( db.connectionReady === false ) {
+                db.asyncClose();
+                db = null;
+            }
+        } catch (ex) {
+            if ( dbOpenError === '' ) {
+                dbOpenError = ex.name;
+                if ( ex.name === 'NS_ERROR_FILE_CORRUPTED' ) {
+                    close();
+                }
+            }
+        }
+
+        if ( db === null ) {
+            return null;
+        }
+
+        // Since database could be opened successfully, reset error flag (its
+        // purpose is to avoid spamming console with error messages).
+        dbOpenError = '';
+
+        return db;
+    };
+
+    // Execute a query
+    var runStatement = function(stmt, callback) {
+        var result = {};
+
+        stmt.executeAsync({
+            handleResult: function(rows) {
+                if ( !rows || typeof callback !== 'function' ) {
+                    return;
+                }
+
+                var row;
+
+                while ( (row = rows.getNextRow()) ) {
+                    // we assume that there will be two columns, since we're
+                    // using it only for preferences
+                    result[row.getResultByIndex(0)] = row.getResultByIndex(1);
+                }
+            },
+            handleCompletion: function(reason) {
+                if ( typeof callback === 'function' && reason === 0 ) {
+                    callback(result);
+                }
+                result = null;
+            },
+            handleError: function(error) {
+                // Caller expects an answer regardless of failure.
+                if ( typeof callback === 'function' ) {
+                    callback({});
+                }
+                result = null;
+                // https://github.com/gorhill/uBlock/issues/1768
+                // Error cases which warrant a removal of the SQL file, so far:
+                // - SQLLite error 11 database disk image is malformed
+                // Can't find doc on MDN about the type of error.result, so I
+                // force a string comparison.
+                if ( error.result.toString() === '11' ) {
+                    close();
+                }
+            }
+        });
+    };
+
+    var bindNames = function(stmt, names) {
+        if ( Array.isArray(names) === false || names.length === 0 ) {
+            return;
+        }
+        var params = stmt.newBindingParamsArray();
+        var i = names.length, bp;
+        while ( i-- ) {
+            bp = params.newBindingParams();
+            bp.bindByName('name', names[i]);
+            params.addParams(bp);
+        }
+        stmt.bindParameters(params);
+    };
+
+    var read = function(details, callback) {
+        if ( typeof callback !== 'function' ) {
+            return;
+        }
+
+        var prepareResult = function(result) {
+            var key;
+            for ( key in result ) {
+                if ( result.hasOwnProperty(key) === false ) {
+                    continue;
+                }
+                result[key] = JSON.parse(result[key]);
+            }
+            if ( typeof details === 'object' && details !== null ) {
+                for ( key in details ) {
+                    if ( result.hasOwnProperty(key) === false ) {
+                        result[key] = details[key];
+                    }
+                }
+            }
+            callback(result);
+        };
+
+        if ( open() === null ) {
+            prepareResult({});
+            return;
+        }
+
+        var names = [];
+        if ( details !== null ) {
+            if ( Array.isArray(details) ) {
+                names = details;
+            } else if ( typeof details === 'object' ) {
+                names = Object.keys(details);
+            } else {
+                names = [details.toString()];
+            }
+        }
+
+        var stmt;
+        if ( names.length === 0 ) {
+            stmt = db.createAsyncStatement('SELECT * FROM "settings"');
+        } else {
+            stmt = db.createAsyncStatement('SELECT * FROM "settings" WHERE "name" = :name');
+            bindNames(stmt, names);
+        }
+
+        runStatement(stmt, prepareResult);
+    };
+
+    let allKeys;
+
+    let readNext = function(key, callback) {
+        if ( key === undefined ) {
+            callback();
+            return;
+        }
+        read(key, bin => {
+            if ( bin instanceof Object && bin.hasOwnProperty(key) ) {
+                callback(key, bin[key]);
+            } else {
+                callback(key);
+            }
+        });
+    };
+
+    let getNext = function(callback) {
+        if ( Array.isArray(allKeys) ) {
+            readNext(allKeys.pop(), callback);
+            return;
+        }
+        if ( open() === null ) {
+            callback();
+            return;
+        }
+        let stmt = db.createAsyncStatement('SELECT "name",\'dummy\' FROM "settings"');
+        runStatement(stmt, result => {
+            allKeys = [];
+            for ( let key in result ) {
+                if ( result.hasOwnProperty(key) ) {
+                    allKeys.push(key);
+                }
+            }
+            readNext(allKeys.pop(), callback);
+        });
+    };
+
+    let markAsDone = function() {
+        close();
+        let { Services } = Components.utils.import('resource://gre/modules/Services.jsm', null),
+            path = Services.dirsvc.get('ProfD', Components.interfaces.nsIFile);
+        path.append('extension-data');
+        path.append(hostName + '.sqlite');
+        if ( path.exists() && path.isFile() ) {
+            path.renameTo(null, hostName + '.migrated.sqlite');
+        }
+    };
+
+    return {
+        getNext: getNext,
+        markAsDone: markAsDone,
+    };
+};
+
+/******************************************************************************/

platform/webext/chrome.manifest

@@ -0,0 +1 @@
+content ublock0 ./

platform/webext/from-legacy.js

@@ -0,0 +1,61 @@
+/*******************************************************************************
+
+    uBlock Origin - a browser extension to block requests.
+    Copyright (C) 2017 Raymond Hill
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see {http://www.gnu.org/licenses/}.
+
+    Home: https://github.com/gorhill/uBlock
+*/
+
+// For background page
+
+'use strict';
+
+/******************************************************************************/
+
+(function() {
+    let µb = µBlock;
+
+    let migrateAll = function(callback) {
+        let mustRestart = false;
+
+        let migrateKeyValue = function(details, callback) {
+            let bin = {};
+            bin[details.key] = JSON.parse(details.value);
+            self.browser.storage.local.set(bin, callback);
+            mustRestart = true;
+        };
+
+        let migrateNext = function() {
+            self.browser.runtime.sendMessage({ what: 'webext:storageMigrateNext' }, response => {
+                if ( response.key === undefined ) {
+                    if ( mustRestart ) {
+                        self.browser.runtime.reload();
+                    } else {
+                        callback();
+                    }
+                    return;
+                }
+                migrateKeyValue(response, migrateNext);
+            });
+        };
+
+        migrateNext();
+    };
+
+    µb.onBeforeStartQueue.push(migrateAll);
+})();
+
+/******************************************************************************/

platform/webext/install.rdf

@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="utf-8"?>
+<r:RDF xmlns:r="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns="http://www.mozilla.org/2004/em-rdf#">
+    <r:Description about="urn:mozilla:install-manifest">
+        <id>uBlock0@raymondhill.net</id>
+        <version>{version}</version>
+        <name>{name}</name>
+        <description>{description}</description>
+        <homepageURL>https://github.com/gorhill/uBlock</homepageURL>
+        <creator>{author}</creator>
+        <developer>Deathamns</developer>
+        <developer>Alex Vallat</developer>
+        <developer>Manuel Reimer</developer>
+        <type>2</type>
+        <bootstrap>true</bootstrap>
+        <multiprocessCompatible>true</multiprocessCompatible>
+        <hasEmbeddedWebExtension>true</hasEmbeddedWebExtension>
+{localized}
+
+        <!-- Firefox -->
+        <targetApplication>
+            <r:Description>
+                <id>{{ec8030f7-c20a-464f-9b0e-13a3a9e97384}}</id>
+                <minVersion>52.0a1</minVersion>
+                <maxVersion>*</maxVersion>
+            </r:Description>
+        </targetApplication>
+    </r:Description>
+</r:RDF>

src/js/background.js

@@ -19,54 +19,19 @@
     Home: https://github.com/gorhill/uBlock
 */
 
+
+/* global objectAssign */
+
 'use strict';
 
 /******************************************************************************/
 
 var µBlock = (function() { // jshint ignore:line
 
-/******************************************************************************/
+    var oneSecond = 1000,
+        oneMinute = 60 * oneSecond;
 
-var oneSecond = 1000;
+    var hiddenSettingsDefault = {
-var oneMinute = 60 * oneSecond;
-
-/******************************************************************************/
-
-var defaultExternalLists = [
-    '! Examples:',
-    '! https://easylist-downloads.adblockplus.org/fb_annoyances_full.txt',
-    '! https://easylist-downloads.adblockplus.org/yt_annoyances_full.txt',
-    ''
-].join('\n');
-
-/******************************************************************************/
-
-return {
-    firstInstall: false,
-
-    userSettings: {
-        advancedUserEnabled: false,
-        alwaysDetachLogger: false,
-        autoUpdate: true,
-        cloudStorageEnabled: false,
-        collapseBlocked: true,
-        colorBlindFriendly: false,
-        contextMenuEnabled: true,
-        dynamicFilteringEnabled: false,
-        externalLists: defaultExternalLists,
-        firewallPaneMinimized: true,
-        hyperlinkAuditingDisabled: true,
-        ignoreGenericCosmeticFilters: false,
-        largeMediaSize: 50,
-        parseAllABPHideFilters: true,
-        prefetchingDisabled: true,
-        requestLogMaxEntries: 1000,
-        showIconBadge: true,
-        tooltipsDisabled: false,
-        webrtcIPAddressHidden: false
-    },
-
-    hiddenSettingsDefault: {
         assetFetchTimeout: 30,
         autoUpdateAssetFetchPeriod: 120,
         autoUpdatePeriod: 7,
@@ -75,86 +40,131 @@ return {
         manualUpdateAssetFetchPeriod: 2000,
         popupFontSize: 'unset',
         suspendTabsUntilReady: false
-    },
+    };
-    // This will be filled ASAP:
-    hiddenSettings: {},
 
-    // Features detection.
+    return {
-    privacySettingsSupported: vAPI.browserSettings instanceof Object,
+        firstInstall: false,
-    cloudStorageSupported: vAPI.cloud instanceof Object,
 
-    // https://github.com/chrisaljoudi/uBlock/issues/180
+        onBeforeStartQueue: [],
-    // Whitelist directives need to be loaded once the PSL is available
+        onStartCompletedQueue: [],
-    netWhitelist: {},
-    netWhitelistModifyTime: 0,
-    netWhitelistDefault: [
-        'about-scheme',
-        'behind-the-scene',
-        'chrome-extension-scheme',
-        'chrome-scheme',
-        'loopconversation.about-scheme',
-        'moz-extension-scheme',
-        'opera-scheme',
-        'vivaldi-scheme',
-        ''
-    ].join('\n'),
 
-    localSettings: {
+        userSettings: {
-        blockedRequestCount: 0,
+            advancedUserEnabled: false,
-        allowedRequestCount: 0
+            alwaysDetachLogger: false,
-    },
+            autoUpdate: true,
-    localSettingsLastModified: 0,
+            cloudStorageEnabled: false,
-    localSettingsLastSaved: 0,
+            collapseBlocked: true,
+            colorBlindFriendly: false,
+            contextMenuEnabled: true,
+            dynamicFilteringEnabled: false,
+            externalLists: [],
+            firewallPaneMinimized: true,
+            hyperlinkAuditingDisabled: true,
+            ignoreGenericCosmeticFilters: false,
+            largeMediaSize: 50,
+            parseAllABPHideFilters: true,
+            prefetchingDisabled: true,
+            requestLogMaxEntries: 1000,
+            showIconBadge: true,
+            tooltipsDisabled: false,
+            webrtcIPAddressHidden: false
+        },
 
-    // read-only
+        hiddenSettingsDefault: hiddenSettingsDefault,
-    systemSettings: {
+        hiddenSettings: (function() {
-        compiledMagic: 'fxtcjjhbhyiw',
+            var out = objectAssign({}, hiddenSettingsDefault),
-        selfieMagic: 'fxtcjjhbhyiw'
+                json = vAPI.localStorage.getItem('hiddenSettings');
-    },
+            if ( typeof json === 'string' ) {
+                try {
+                    var o = JSON.parse(json);
+                    if ( o instanceof Object ) {
+                        for ( var k in o ) {
+                            if ( out.hasOwnProperty(k) ) {
+                                out[k] = o[k];
+                            }
+                        }
+                    }
+                }
+                catch(ex) {
+                }
+            }
+            return out;
+        })(),
 
-    restoreBackupSettings: {
+        // Features detection.
-        lastRestoreFile: '',
+        privacySettingsSupported: vAPI.browserSettings instanceof Object,
-        lastRestoreTime: 0,
+        cloudStorageSupported: vAPI.cloud instanceof Object,
-        lastBackupFile: '',
-        lastBackupTime: 0
-    },
 
-    // Allows to fully customize uBO's assets, typically set through admin
+        // https://github.com/chrisaljoudi/uBlock/issues/180
-    // settings. The content of 'assets.json' will also tell which filter
+        // Whitelist directives need to be loaded once the PSL is available
-    // lists to enable by default when uBO is first installed.
+        netWhitelist: {},
-    assetsBootstrapLocation: 'assets/assets.json',
+        netWhitelistModifyTime: 0,
+        netWhitelistDefault: [
+            'about-scheme',
+            'behind-the-scene',
+            'chrome-extension-scheme',
+            'chrome-scheme',
+            'loopconversation.about-scheme',
+            'moz-extension-scheme',
+            'opera-scheme',
+            'vivaldi-scheme',
+            ''
+        ].join('\n'),
 
-    userFiltersPath: 'user-filters',
+        localSettings: {
-    pslAssetKey: 'public_suffix_list.dat',
+            blockedRequestCount: 0,
+            allowedRequestCount: 0
+        },
+        localSettingsLastModified: 0,
+        localSettingsLastSaved: 0,
 
-    selectedFilterLists: [],
+        // read-only
-    availableFilterLists: {},
+        systemSettings: {
+            compiledMagic: 'fxtcjjhbhyiw',
+            selfieMagic: 'fxtcjjhbhyiw'
+        },
 
-    selfieAfter: 23 * oneMinute,
+        restoreBackupSettings: {
+            lastRestoreFile: '',
+            lastRestoreTime: 0,
+            lastBackupFile: '',
+            lastBackupTime: 0
+        },
 
-    pageStores: {},
+        // Allows to fully customize uBO's assets, typically set through admin
-    pageStoresToken: 0,
+        // settings. The content of 'assets.json' will also tell which filter
+        // lists to enable by default when uBO is first installed.
+        assetsBootstrapLocation: 'assets/assets.json',
 
-    storageQuota: vAPI.storage.QUOTA_BYTES,
+        userFiltersPath: 'user-filters',
-    storageUsed: 0,
+        pslAssetKey: 'public_suffix_list.dat',
 
-    noopFunc: function(){},
+        selectedFilterLists: [],
+        availableFilterLists: {},
 
-    apiErrorCount: 0,
+        selfieAfter: 23 * oneMinute,
-    mouseX: -1,
-    mouseY: -1,
-    mouseURL: '',
-    epickerTarget: '',
-    epickerEprom: null,
 
-    scriptlets: {
+        pageStores: {},
-    },
+        pageStoresToken: 0,
 
-    // so that I don't have to care for last comma
+        storageQuota: vAPI.storage.QUOTA_BYTES,
-    dummy: 0
+        storageUsed: 0,
-};
 
-/******************************************************************************/
+        noopFunc: function(){},
+
+        apiErrorCount: 0,
+        mouseX: -1,
+        mouseY: -1,
+        mouseURL: '',
+        epickerTarget: '',
+        epickerEprom: null,
+
+        scriptlets: {
+        },
+
+        // so that I don't have to care for last comma
+        dummy: 0
+    };
 
 })();
 

src/js/start.js

@@ -19,7 +19,7 @@
     Home: https://github.com/gorhill/uBlock
 */
 
-/* global objectAssign, publicSuffixList */
+/* global publicSuffixList */
 
 'use strict';
 
@@ -51,6 +51,20 @@ vAPI.app.onShutdown = function() {
 
 /******************************************************************************/
 
+var processCallbackQueue = function(queue, callback) {
+    var processOne = function() {
+        var fn = queue.pop();
+        if ( fn ) {
+            fn(processOne);
+        } else if ( typeof callback === 'function' ) {
+            callback();
+        }
+    };
+    processOne();
+};
+
+/******************************************************************************/
+
 // Final initialization steps after all needed assets are in memory.
 // - Initialize internal state with maybe already existing tabs.
 // - Schedule next update operation.
@@ -76,7 +90,7 @@ var onAllReady = function() {
     µb.contextMenu.update(null);
     µb.firstInstall = false;
 
-    vAPI.net.onReady();
+    processCallbackQueue(µb.onStartCompletedQueue);
 };
 
 /******************************************************************************/
@@ -278,31 +292,11 @@ var onAdminSettingsRestored = function() {
 
 /******************************************************************************/
 
-µb.hiddenSettings = (function() {
-    var out = objectAssign({}, µb.hiddenSettingsDefault),
-        json = vAPI.localStorage.getItem('hiddenSettings');
-    if ( typeof json === 'string' ) {
-        try {
-            var o = JSON.parse(json);
-            if ( o instanceof Object ) {
-                for ( var k in o ) {
-                    if ( out.hasOwnProperty(k) ) {
-                        out[k] = o[k];
-                    }
-                }
-            }
-        }
-        catch(ex) {
-        }
-    }
-    return out;
-})();
-
-/******************************************************************************/
-
 return function() {
-    // https://github.com/gorhill/uBlock/issues/531
+    processCallbackQueue(µb.onBeforeStartQueue, function() {
-    µb.restoreAdminSettings(onAdminSettingsRestored);
+        // https://github.com/gorhill/uBlock/issues/531
+        µb.restoreAdminSettings(onAdminSettingsRestored);
+    });
 };
 
 /******************************************************************************/

src/js/traffic.js

@@ -34,47 +34,41 @@ var exports = {};
 /******************************************************************************/
 
 // https://github.com/gorhill/uBlock/issues/2067
-//   Experimental: Suspend tabs until uBO is fully ready.
+//   Experimental: Block everything until uBO is fully ready.
+// TODO: re-work vAPI code to match more closely how listeners are
+//       registered with the webRequest API. This will simplify implementing
+//       the feature here: we could have a temporary onBeforeRequest listener
+//       which blocks everything until all is ready.
+//       This would allow to avoid the permanent special test at the top of
+//       the main onBeforeRequest just to implement this.
+var onBeforeReady = null;
 
-vAPI.net.onReady = function() {
+if ( µBlock.hiddenSettings.suspendTabsUntilReady ) {
-    if ( µBlock.hiddenSettings.suspendTabsUntilReady !== true ) {
+    onBeforeReady = (function() {
+        var suspendedTabs = new Set();
+        µBlock.onStartCompletedQueue.push(function(callback) {
+            onBeforeReady = null;
+            var iter = suspendedTabs.values(),
+                entry;
+            for (;;) {
+                entry = iter.next();
+                if ( entry.done ) { break; }
+                vAPI.tabs.reload(entry.value);
+            }
+            callback();
+        });
+        return function(tabId) {
+            if ( vAPI.isBehindTheSceneTabId(tabId) ) { return; }
+            suspendedTabs.add(tabId);
+            return true;
+        };
+    })();
+} else {
+    µBlock.onStartCompletedQueue.push(function(callback) {
         vAPI.onLoadAllCompleted();
-    }
+        callback();
-    var fn = onBeforeReady;
+    });
-    onBeforeReady = null;
+}
-    if ( fn !== null ) {
-        fn('ready');
-    }
-};
-
-var onBeforeReady = (function() {
-    var suspendedTabs = new Set();
-
-    var forceReloadSuspendedTabs = function() {
-        var iter = suspendedTabs.values(),
-            entry;
-        for (;;) {
-            entry = iter.next();
-            if ( entry.done ) { break; }
-            vAPI.tabs.reload(entry.value);
-        }
-    };
-
-    return function(tabId) {
-        if (
-            vAPI.isBehindTheSceneTabId(tabId) ||
-            µBlock.hiddenSettings.suspendTabsUntilReady !== true
-        ) {
-            return;
-        }
-        if ( tabId === 'ready' ) {
-            forceReloadSuspendedTabs();
-            return;
-        }
-        suspendedTabs.add(tabId);
-        return true;
-    };
-})();
 
 /******************************************************************************/
 

tools/make-firefox-meta.py

@@ -31,23 +31,17 @@ for alpha2 in sorted(os.listdir(source_locale_dir)):
     locale_path = pj(source_locale_dir, alpha2, 'messages.json')
     with open(locale_path, encoding='utf-8') as f:
         strings = json.load(f, object_pairs_hook=OrderedDict)
-
     alpha2 = alpha2.replace('_', '-')
     descriptions[alpha2] = strings['extShortDesc']['message']
     del strings['extShortDesc']
-
     language_codes.append(alpha2)
-
     mkdirs(pj(target_locale_dir, alpha2))
-
     locale_path = pj(target_locale_dir, alpha2, 'messages.properties')
     with open(locale_path, 'wt', encoding='utf-8', newline='\n') as f:
         for string_name in strings:
             string = strings[string_name]['message']
-
             if alpha2 == 'en' and string_name in title_case_strings:
                 string = string.title()
-
             f.write(string_name)
             f.write(u'=')
             f.write(string.replace('\n', r'\n'))
@@ -58,11 +52,9 @@ chrome_manifest = pj(build_dir, 'chrome.manifest')
 
 with open(chrome_manifest, 'at', encoding='utf-8', newline='\n') as f:
     f.write(u'\nlocale ublock0 en ./locale/en/\n')
-
     for alpha2 in language_codes:
         if alpha2 == 'en':
             continue
-
         f.write(u'locale ublock0 ' + alpha2 + ' ./locale/' + alpha2 + '/\n')
 
 rmtree(source_locale_dir)
@@ -89,15 +81,13 @@ if match:
 manifest['homepage'] = 'https://github.com/gorhill/uBlock'
 manifest['description'] = descriptions['en']
 del descriptions['en']
-manifest['localized'] = []
 
+manifest['localized'] = []
 t = '    '
 t3 = 3 * t
-
 for alpha2 in descriptions:
     if alpha2 == 'en':
         continue
-
     manifest['localized'].append(
         '\n' + t*2 + '<localized><r:Description>\n' +
         t3 + '<locale>' + alpha2 + '</locale>\n' +
@@ -108,12 +98,10 @@ for alpha2 in descriptions:
         t3 + '<homepageURL>' + manifest['homepage'] + '</homepageURL>\n' +
         t*2 + '</r:Description></localized>'
     )
-
 manifest['localized'] = '\n'.join(manifest['localized'])
 
 install_rdf = pj(build_dir, 'install.rdf')
 with open(install_rdf, 'r+t', encoding='utf-8', newline='\n') as f:
     install_rdf = f.read()
     f.seek(0)
-
     f.write(install_rdf.format(**manifest))

tools/make-firefox.sh

@@ -17,9 +17,6 @@ cp -R src/lib                           $DES/
 cp -R src/_locales                      $DES/
 cp    src/*.html                        $DES/
 
-# AMO review feedback: avoid "unnecessary files or folders" in package
-cat   src/background.html | sed -e '/vapi-polyfill\.js/d' > $DES/background.html
-
 mv    $DES/img/icon_128.png             $DES/icon.png
 cp    platform/firefox/css/*            $DES/css/
 cp    platform/firefox/polyfill.js      $DES/js/

tools/make-webext-meta.py

@@ -2,7 +2,10 @@
 
 import os
 import json
+import re
 import sys
+from io import open as uopen
+from collections import OrderedDict
 
 if len(sys.argv) == 1 or not sys.argv[1]:
     raise SystemExit('Build dir missing.')
@@ -10,7 +13,7 @@ if len(sys.argv) == 1 or not sys.argv[1]:
 proj_dir = os.path.join(os.path.split(os.path.abspath(__file__))[0], '..')
 build_dir = os.path.abspath(sys.argv[1])
 
-# Import version number from chromium platform
+# Import data from chromium platform
 chromium_manifest = {}
 webext_manifest = {}
 
@@ -18,7 +21,8 @@ chromium_manifest_file = os.path.join(proj_dir, 'platform', 'chromium', 'manifes
 with open(chromium_manifest_file) as f1:
     chromium_manifest = json.load(f1)
 
-webext_manifest_file = os.path.join(build_dir, 'manifest.json')
+# WebExtension part
+webext_manifest_file = os.path.join(build_dir, 'webextension', 'manifest.json')
 with open(webext_manifest_file) as f2:
     webext_manifest = json.load(f2)
 
@@ -27,3 +31,52 @@ webext_manifest['version'] = chromium_manifest['version']
 with open(webext_manifest_file, 'w') as f2:
     json.dump(webext_manifest, f2, indent=2, separators=(',', ': '), sort_keys=True)
     f2.write('\n')
+
+# Legacy part
+descriptions = OrderedDict({})
+source_locale_dir = os.path.join(build_dir, 'webextension', '_locales')
+for alpha2 in sorted(os.listdir(source_locale_dir)):
+    locale_path = os.path.join(source_locale_dir, alpha2, 'messages.json')
+    with uopen(locale_path, encoding='utf-8') as f:
+        strings = json.load(f, object_pairs_hook=OrderedDict)
+    alpha2 = alpha2.replace('_', '-')
+    descriptions[alpha2] = strings['extShortDesc']['message']
+
+webext_manifest['author'] = chromium_manifest['author'];
+webext_manifest['homepage'] = 'https://github.com/gorhill/uBlock'
+webext_manifest['description'] = descriptions['en']
+del descriptions['en']
+
+match = re.search('^(\d+\.\d+\.\d+)(\.\d+)$', chromium_manifest['version'])
+if match:
+    buildtype = int(match.group(2)[1:])
+    if buildtype < 100:
+        builttype = 'b' + str(buildtype)
+    else:
+        builttype = 'rc' + str(buildtype - 100)
+    webext_manifest['version'] = match.group(1) + builttype
+
+webext_manifest['localized'] = []
+t = '    '
+t3 = 3 * t
+for alpha2 in descriptions:
+    if alpha2 == 'en':
+        continue
+    webext_manifest['localized'].append(
+        '\n' + t*2 + '<localized><r:Description>\n' +
+        t3 + '<locale>' + alpha2 + '</locale>\n' +
+        t3 + '<name>' + webext_manifest['name'] + '</name>\n' +
+        t3 + '<description>' + descriptions[alpha2] + '</description>\n' +
+        t3 + '<creator>' + webext_manifest['author'] + '</creator>\n' +
+        # t3 + '<translator>' + ??? + '</translator>\n' +
+        t3 + '<homepageURL>' + webext_manifest['homepage'] + '</homepageURL>\n' +
+        t*2 + '</r:Description></localized>'
+    )
+webext_manifest['localized'] = '\n'.join(webext_manifest['localized'])
+
+install_rdf = os.path.join(build_dir, 'install.rdf')
+with uopen(install_rdf, 'r+t', encoding='utf-8', newline='\n') as f:
+    install_rdf = f.read()
+    f.seek(0)
+    f.write(install_rdf.format(**webext_manifest))
+    f.truncate()

tools/make-webext.sh

@@ -7,32 +7,39 @@ echo "*** uBlock0.webext: Copying files"
 
 DES=dist/build/uBlock0.webext
 rm -rf $DES
-mkdir -p $DES
+mkdir -p $DES/webextension
 
-bash ./tools/make-assets.sh $DES
+bash ./tools/make-assets.sh $DES/webextension
 
-cp -R src/css                    $DES/
+cp -R src/css                         $DES/webextension/
-cp -R src/img                    $DES/
+cp -R src/img                         $DES/webextension/
-cp -R src/js                     $DES/
+cp -R src/js                          $DES/webextension/
-cp -R src/lib                    $DES/
+cp -R src/lib                         $DES/webextension/
-cp -R src/_locales               $DES/
+cp -R src/_locales                    $DES/webextension/
-cp -R $DES/_locales/nb           $DES/_locales/no
+cp -R $DES/webextension/_locales/nb   $DES/webextension/_locales/no
-cp src/*.html                    $DES/
+cp src/*.html                         $DES/webextension/
-cp platform/chromium/*.js        $DES/js/
+cp platform/chromium/*.js             $DES/webextension/js/
-cp -R platform/chromium/img      $DES/
+cp -R platform/chromium/img           $DES/webextension/
-cp platform/chromium/*.html      $DES/
+cp platform/chromium/*.html           $DES/webextension/
-cp platform/chromium/*.json      $DES/
+cp platform/chromium/*.json           $DES/webextension/
-cp platform/webext/polyfill.js   $DES/js/
+cp platform/webext/polyfill.js        $DES/webextension/js/
-cp platform/webext/manifest.json $DES/
+cp LICENSE.txt                        $DES/webextension/
-cp LICENSE.txt                   $DES/
+                                  
+cp platform/webext/background.html    $DES/webextension/
+cp platform/webext/from-legacy.js     $DES/webextension/js/
+cp platform/webext/manifest.json      $DES/webextension/
+cp platform/webext/bootstrap.js       $DES/
+cp platform/webext/chrome.manifest    $DES/
+cp platform/webext/install.rdf        $DES/
+mv $DES/webextension/img/icon_128.png $DES/icon.png
 
 echo "*** uBlock0.webext: Generating meta..."
 python tools/make-webext-meta.py $DES/
 
 if [ "$1" = all ]; then
     echo "*** uBlock0.webext: Creating package..."
-    pushd $(dirname $DES/) > /dev/null
+    pushd $DES > /dev/null
-    zip uBlock0.webext.zip -qr $(basename $DES/)/*
+    zip ../$(basename $DES).zip -qr *
     popd > /dev/null
 fi