From 0d369cda21bbce23a7376e0f7b2847a3c7a6d3d8 Mon Sep 17 00:00:00 2001 From: Raymond Hill Date: Sun, 17 Feb 2019 15:40:09 -0500 Subject: [PATCH] Allow use of browser.storage.local as cache storage backend in Firefox Related issue: - https://github.com/uBlockOrigin/uBlock-issues/issues/409 By default `indexedDB` is used in Firefox for purpose of cache storage backend. This commit allows to force the use of `browser.storage.local` instead as cache storage backend. For this to happen, set `cacheStorageAPI` to `browser.storage.local` in advanced settings. Additionally, should `indexedDB` not be available for whatever reason, uBO will automatically fallback to `browser.storage.local`. --- platform/chromium/vapi-background.js | 1 - src/js/assets.js | 70 ++- src/js/cachestorage.js | 843 ++++++++++++++------------- src/js/messaging.js | 10 +- src/js/start.js | 26 +- src/js/storage.js | 83 +-- 6 files changed, 540 insertions(+), 493 deletions(-) diff --git a/platform/chromium/vapi-background.js b/platform/chromium/vapi-background.js index 1df622c57..02c091d8d 100644 --- a/platform/chromium/vapi-background.js +++ b/platform/chromium/vapi-background.js @@ -91,7 +91,6 @@ vAPI.app.restart = function() { // chrome.storage.local.get(null, function(bin){ console.debug('%o', bin); }); vAPI.storage = chrome.storage.local; -vAPI.cacheStorage = chrome.storage.local; /******************************************************************************/ /******************************************************************************/ diff --git a/src/js/assets.js b/src/js/assets.js index 3ccc7f8a5..d16f9579f 100644 --- a/src/js/assets.js +++ b/src/js/assets.js @@ -396,13 +396,17 @@ const updateAssetSourceRegistry = function(json, silent) { const getAssetSourceRegistry = function(callback) { if ( assetSourceRegistryPromise === undefined ) { - assetSourceRegistryPromise = new Promise(resolve => { - // start of executor - µBlock.cacheStorage.get('assetSourceRegistry', bin => { + assetSourceRegistryPromise = µBlock.cacheStorage.get( + 'assetSourceRegistry' + ).then(bin => { if ( - bin instanceof Object === false || - bin.assetSourceRegistry instanceof Object === false + bin instanceof Object && + bin.assetSourceRegistry instanceof Object ) { + assetSourceRegistry = bin.assetSourceRegistry; + return; + } + return new Promise(resolve => { api.fetchText( µBlock.assetsBootstrapLocation || 'assets/assets.json', details => { @@ -410,12 +414,7 @@ const getAssetSourceRegistry = function(callback) { resolve(); } ); - return; - } - assetSourceRegistry = bin.assetSourceRegistry; - resolve(); - }); - // end of executor + }); }); } @@ -451,16 +450,15 @@ let assetCacheRegistry = {}; const getAssetCacheRegistry = function() { if ( assetCacheRegistryPromise === undefined ) { - assetCacheRegistryPromise = new Promise(resolve => { - µBlock.cacheStorage.get('assetCacheRegistry', bin => { - if ( - bin instanceof Object && - bin.assetCacheRegistry instanceof Object - ) { - assetCacheRegistry = bin.assetCacheRegistry; - } - resolve(); - }); + assetCacheRegistryPromise = µBlock.cacheStorage.get( + 'assetCacheRegistry' + ).then(bin => { + if ( + bin instanceof Object && + bin.assetCacheRegistry instanceof Object + ) { + assetCacheRegistry = bin.assetCacheRegistry; + } }); } @@ -509,8 +507,11 @@ const assetCacheRead = function(assetKey, callback) { reportBack(bin[internalKey]); }; - getAssetCacheRegistry().then(( ) => { - µBlock.cacheStorage.get(internalKey, onAssetRead); + Promise.all([ + getAssetCacheRegistry(), + µBlock.cacheStorage.get(internalKey), + ]).then(results => { + onAssetRead(results[1]); }); }; @@ -537,17 +538,16 @@ const assetCacheWrite = function(assetKey, details, callback) { entry.remoteURL = details.url; } µBlock.cacheStorage.set( - { [internalKey]: content }, - details => { - if ( - details instanceof Object && - typeof details.bytesInUse === 'number' - ) { - entry.byteLength = details.bytesInUse; - } - saveAssetCacheRegistry(true); + { [internalKey]: content } + ).then(details => { + if ( + details instanceof Object && + typeof details.bytesInUse === 'number' + ) { + entry.byteLength = details.bytesInUse; } - ); + saveAssetCacheRegistry(true); + }); const result = { assetKey, content }; if ( typeof callback === 'function' ) { callback(result); @@ -556,9 +556,7 @@ const assetCacheWrite = function(assetKey, details, callback) { fireNotification('after-asset-updated', result); }; - getAssetCacheRegistry().then(( ) => { - µBlock.cacheStorage.get(internalKey, onReady); - }); + getAssetCacheRegistry().then(( ) => onReady()); }; const assetCacheRemove = function(pattern, callback) { diff --git a/src/js/cachestorage.js b/src/js/cachestorage.js index 47771fe52..6c5a6ca4c 100644 --- a/src/js/cachestorage.js +++ b/src/js/cachestorage.js @@ -34,129 +34,155 @@ // The original imported code has been subsequently modified as it was not // compatible with Firefox. // (a Promise thing, see https://github.com/dfahlander/Dexie.js/issues/317) -// Furthermore, code to migrate from browser.storage.local to vAPI.cacheStorage +// Furthermore, code to migrate from browser.storage.local to vAPI.storage // has been added, for seamless migration of cache-related entries into // indexedDB. +// https://bugzilla.mozilla.org/show_bug.cgi?id=1371255 +// Firefox-specific: we use indexedDB because chrome.storage.local() has +// poor performance in Firefox. +// https://github.com/uBlockOrigin/uBlock-issues/issues/328 +// Use IndexedDB for Chromium as well, to take advantage of LZ4 +// compression. +// https://github.com/uBlockOrigin/uBlock-issues/issues/399 +// Revert Chromium support of IndexedDB, use advanced setting to force +// IndexedDB. +// https://github.com/uBlockOrigin/uBlock-issues/issues/409 +// Allow forcing the use of webext storage on Firefox. + µBlock.cacheStorage = (function() { const STORAGE_NAME = 'uBlock0CacheStorage'; - // https://bugzilla.mozilla.org/show_bug.cgi?id=1371255 - // Firefox-specific: we use indexedDB because chrome.storage.local() has - // poor performance in Firefox. - // https://github.com/uBlockOrigin/uBlock-issues/issues/328 - // Use IndexedDB for Chromium as well, to take advantage of LZ4 - // compression. - // https://github.com/uBlockOrigin/uBlock-issues/issues/399 - // Revert Chromium support of IndexedDB, use advanced setting to force - // IndexedDB. - if ( - vAPI.webextFlavor.soup.has('firefox') === false && - µBlock.hiddenSettings.cacheStorageAPI !== 'indexedDB' - ) { - // In case IndexedDB was used as cache storage, remove it. - indexedDB.deleteDatabase(STORAGE_NAME); - return vAPI.cacheStorage; - } - - let db; - let pendingInitialization; - let dbBytesInUse; - - const get = function get(input, callback) { - if ( typeof callback !== 'function' ) { return; } - if ( input === null ) { - return getAllFromDb(callback); + // Default to webext storage. Wrapped into promises if the API does not + // support returning promises. + const promisified = (function() { + try { + return browser.storage.local.get('_') instanceof Promise; } - var toRead, output = {}; - if ( typeof input === 'string' ) { - toRead = [ input ]; - } else if ( Array.isArray(input) ) { - toRead = input; - } else /* if ( typeof input === 'object' ) */ { - toRead = Object.keys(input); - output = input; + catch(ex) { } - return getFromDb(toRead, output, callback); - }; - - const set = function set(input, callback) { - putToDb(input, callback); - }; - - const remove = function remove(key, callback) { - deleteFromDb(key, callback); - }; - - const clear = function clear(callback) { - clearDb(callback); - }; - - const getBytesInUse = function getBytesInUse(keys, callback) { - getDbSize(callback); - }; + return false; + })(); const api = { - get, - set, - remove, - clear, - getBytesInUse, + name: 'browser.storage.local', + get: promisified ? + browser.storage.local.get : + function(keys) { + return new Promise(resolve => { + browser.storage.local.get(keys, bin => { + resolve(bin); + }); + }); + }, + set: promisified ? + browser.storage.local.set : + function(keys) { + return new Promise(resolve => { + browser.storage.local.set(keys, ( ) => { + resolve(); + }); + }); + }, + remove: promisified ? + browser.storage.local.remove : + function(keys) { + return new Promise(resolve => { + browser.storage.local.remove(keys, ( ) => { + resolve(); + }); + }); + }, + clear: promisified ? + browser.storage.local.clear : + function() { + return new Promise(resolve => { + browser.storage.local.clear(( ) => { + resolve(); + }); + }); + }, + getBytesInUse: promisified ? + browser.storage.local.getBytesInUse : + function(keys) { + return new Promise(resolve => { + browser.storage.local.getBytesInUse(keys, count => { + resolve(count); + }); + }); + }, + select: function(backend) { + if ( backend === undefined || backend === 'unset' ) { + backend = vAPI.webextFlavor.soup.has('firefox') + ? 'indexedDB' + : 'browser.storage.local'; + } + if ( backend === 'indexedDB' ) { + return selectIDB().then(success => { + if ( success ) { + clearWebext(); + return 'indexedDB'; + } + clearIDB(); + return 'browser.storage.local'; + }); + } + if ( backend === 'browser.storage.local' ) { + clearIDB(); + } + return Promise.resolve('browser.storage.local'); + + }, error: undefined }; - const genericErrorHandler = function(ev) { - let error = ev.target && ev.target.error; - if ( error && error.name === 'QuotaExceededError' ) { - api.error = error.name; - } - console.error('[%s]', STORAGE_NAME, error && error.name); - }; + // Reassign API entries to that of indexedDB-based ones + const selectIDB = function() { + let dbPromise; + let dbTimer; - const noopfn = function () { - }; + const genericErrorHandler = function(ev) { + let error = ev.target && ev.target.error; + if ( error && error.name === 'QuotaExceededError' ) { + api.error = error.name; + } + console.error('[%s]', STORAGE_NAME, error && error.name); + }; - const disconnect = function() { - if ( dbTimer !== undefined ) { - clearTimeout(dbTimer); - dbTimer = undefined; - } - if ( db instanceof IDBDatabase ) { - db.close(); - db = undefined; - } - }; + const noopfn = function () { + }; - let dbTimer; - - const keepAlive = function() { - if ( dbTimer !== undefined ) { - clearTimeout(dbTimer); - } - dbTimer = vAPI.setTimeout( - ( ) => { + const disconnect = function() { + if ( dbTimer !== undefined ) { + clearTimeout(dbTimer); dbTimer = undefined; - disconnect(); - }, - Math.max( - µBlock.hiddenSettings.autoUpdateAssetFetchPeriod * 2 * 1000, - 180000 - ) - ); - }; + } + if ( dbPromise === undefined ) { return; } + dbPromise.then(db => { + if ( db instanceof IDBDatabase ) { + db.close(); + } + dbPromise = undefined; + }); + }; + + const keepAlive = function() { + if ( dbTimer !== undefined ) { + clearTimeout(dbTimer); + } + dbTimer = vAPI.setTimeout( + ( ) => { + dbTimer = undefined; + disconnect(); + }, + Math.max( + µBlock.hiddenSettings.autoUpdateAssetFetchPeriod * 2 * 1000, + 180000 + ) + ); + }; - const getDb = function getDb() { - if ( db === null ) { - return Promise.resolve(null); - } - keepAlive(); - if ( db instanceof IDBDatabase ) { - return Promise.resolve(db); - } - if ( pendingInitialization !== undefined ) { - return pendingInitialization; - } // https://github.com/gorhill/uBlock/issues/3156 // I have observed that no event was fired in Tor Browser 7.0.7 + // medium security level after the request to open the database was @@ -166,327 +192,320 @@ // necessary when reading the `error` property because we are not // allowed to read this propery outside of event handlers in newer // implementation of IDBRequest (my understanding). - pendingInitialization = new Promise(resolve => { - let req; - try { - req = indexedDB.open(STORAGE_NAME, 1); - if ( req.error ) { - console.log(req.error); + + const getDb = function() { + keepAlive(); + if ( dbPromise !== undefined ) { + return dbPromise; + } + dbPromise = new Promise(resolve => { + let req; + try { + req = indexedDB.open(STORAGE_NAME, 1); + if ( req.error ) { + console.log(req.error); + req = undefined; + } + } catch(ex) { + } + if ( req === undefined ) { + return resolve(null); + } + req.onupgradeneeded = function(ev) { + req = undefined; + const db = ev.target.result; + db.onerror = db.onabort = genericErrorHandler; + const table = db.createObjectStore( + STORAGE_NAME, + { keyPath: 'key' } + ); + table.createIndex('value', 'value', { unique: false }); + }; + req.onsuccess = function(ev) { + req = undefined; + const db = ev.target.result; + db.onerror = db.onabort = genericErrorHandler; + resolve(db); + }; + req.onerror = req.onblocked = function() { + req = undefined; + console.log(this.error); + resolve(null); + }; + }); + return dbPromise; + }; + + const getFromDb = function(keys, keyvalStore, callback) { + if ( typeof callback !== 'function' ) { return; } + if ( keys.length === 0 ) { return callback(keyvalStore); } + let promises = []; + let gotOne = function() { + if ( typeof this.result !== 'object' ) { return; } + keyvalStore[this.result.key] = this.result.value; + if ( this.result.value instanceof Blob === false ) { return; } + promises.push( + µBlock.lz4Codec.decode( + this.result.key, + this.result.value + ).then(result => { + keyvalStore[result.key] = result.data; + }) + ); + }; + getDb().then(db => { + if ( !db ) { return callback(); } + const transaction = db.transaction(STORAGE_NAME); + transaction.oncomplete = + transaction.onerror = + transaction.onabort = ( ) => { + Promise.all(promises).then(( ) => { + callback(keyvalStore); + }); + }; + const table = transaction.objectStore(STORAGE_NAME); + for ( const key of keys ) { + let req = table.get(key); + req.onsuccess = gotOne; + req.onerror = noopfn; req = undefined; } - } catch(ex) { - } - if ( req === undefined ) { - pendingInitialization = undefined; - db = null; - resolve(null); - return; - } - req.onupgradeneeded = function(ev) { - req = undefined; - let db = ev.target.result; - db.onerror = db.onabort = genericErrorHandler; - let table = db.createObjectStore(STORAGE_NAME, { keyPath: 'key' }); - table.createIndex('value', 'value', { unique: false }); - }; - req.onsuccess = function(ev) { - pendingInitialization = undefined; - req = undefined; - db = ev.target.result; - db.onerror = db.onabort = genericErrorHandler; - resolve(db); - }; - req.onerror = req.onblocked = function() { - pendingInitialization = undefined; - req = undefined; - db = null; - console.log(this.error); - resolve(null); - }; - }); - return pendingInitialization; - }; - - const getFromDb = function(keys, keyvalStore, callback) { - if ( typeof callback !== 'function' ) { return; } - if ( keys.length === 0 ) { return callback(keyvalStore); } - let promises = []; - let gotOne = function() { - if ( typeof this.result !== 'object' ) { return; } - keyvalStore[this.result.key] = this.result.value; - if ( this.result.value instanceof Blob === false ) { return; } - promises.push( - µBlock.lz4Codec.decode( - this.result.key, - this.result.value - ).then(result => { - keyvalStore[result.key] = result.data; - }) - ); - }; - getDb().then(( ) => { - if ( !db ) { return callback(); } - let transaction = db.transaction(STORAGE_NAME); - transaction.oncomplete = - transaction.onerror = - transaction.onabort = ( ) => { - Promise.all(promises).then(( ) => { - callback(keyvalStore); - }); - }; - let table = transaction.objectStore(STORAGE_NAME); - for ( let key of keys ) { - let req = table.get(key); - req.onsuccess = gotOne; - req.onerror = noopfn; - req = undefined; - } - }); - }; - - const visitAllFromDb = function(visitFn) { - getDb().then(( ) => { - if ( !db ) { return visitFn(); } - let transaction = db.transaction(STORAGE_NAME); - transaction.oncomplete = - transaction.onerror = - transaction.onabort = ( ) => visitFn(); - let table = transaction.objectStore(STORAGE_NAME); - let req = table.openCursor(); - req.onsuccess = function(ev) { - let cursor = ev.target && ev.target.result; - if ( !cursor ) { return; } - let entry = cursor.value; - visitFn(entry); - cursor.continue(); - }; - }); - }; - - const getAllFromDb = function(callback) { - if ( typeof callback !== 'function' ) { return; } - let promises = []; - let keyvalStore = {}; - visitAllFromDb(entry => { - if ( entry === undefined ) { - Promise.all(promises).then(( ) => { - callback(keyvalStore); - }); - return; - } - keyvalStore[entry.key] = entry.value; - if ( entry.value instanceof Blob === false ) { return; } - promises.push( - µBlock.lz4Codec.decode( - entry.key, - entry.value - ).then(result => { - keyvalStore[result.key] = result.value; - }) - ); - }); - }; - - const getDbSize = function(callback) { - if ( typeof callback !== 'function' ) { return; } - if ( typeof dbBytesInUse === 'number' ) { - return Promise.resolve().then(( ) => { - callback(dbBytesInUse); }); - } - const textEncoder = new TextEncoder(); - let totalByteLength = 0; - visitAllFromDb(entry => { - if ( entry === undefined ) { - dbBytesInUse = totalByteLength; - return callback(totalByteLength); - } - let value = entry.value; - if ( typeof value === 'string' ) { - totalByteLength += textEncoder.encode(value).byteLength; - } else if ( value instanceof Blob ) { - totalByteLength += value.size; - } else { - totalByteLength += textEncoder.encode(JSON.stringify(value)).byteLength; - } - if ( typeof entry.key === 'string' ) { - totalByteLength += textEncoder.encode(entry.key).byteLength; - } - }); - }; - - - // https://github.com/uBlockOrigin/uBlock-issues/issues/141 - // Mind that IDBDatabase.transaction() and IDBObjectStore.put() - // can throw: - // https://developer.mozilla.org/en-US/docs/Web/API/IDBDatabase/transaction - // https://developer.mozilla.org/en-US/docs/Web/API/IDBObjectStore/put - - const putToDb = function(keyvalStore, callback) { - if ( typeof callback !== 'function' ) { - callback = noopfn; - } - const keys = Object.keys(keyvalStore); - if ( keys.length === 0 ) { return callback(); } - const promises = [ getDb() ]; - const entries = []; - const dontCompress = µBlock.hiddenSettings.cacheStorageCompression !== true; - let bytesInUse = 0; - const handleEncodingResult = result => { - if ( typeof result.data === 'string' ) { - bytesInUse += result.data.length; - } else if ( result.data instanceof Blob ) { - bytesInUse += result.data.size; - } - entries.push({ key: result.key, value: result.data }); }; - for ( const key of keys ) { - const data = keyvalStore[key]; - const isString = typeof data === 'string'; - if ( isString === false || dontCompress ) { - if ( isString ) { - bytesInUse += data.length; - } - entries.push({ key, value: data }); - continue; - } - promises.push( - µBlock.lz4Codec.encode(key, data).then(handleEncodingResult) - ); - } - Promise.all(promises).then(( ) => { - if ( !db ) { return callback(); } - const finish = ( ) => { - dbBytesInUse = undefined; - if ( callback === undefined ) { return; } - let cb = callback; - callback = undefined; - cb({ bytesInUse }); - }; - try { - const transaction = db.transaction(STORAGE_NAME, 'readwrite'); + + const visitAllFromDb = function(visitFn) { + getDb().then(db => { + if ( !db ) { return visitFn(); } + const transaction = db.transaction(STORAGE_NAME); transaction.oncomplete = transaction.onerror = - transaction.onabort = finish; + transaction.onabort = ( ) => visitFn(); const table = transaction.objectStore(STORAGE_NAME); - for ( const entry of entries ) { - table.put(entry); - } - } catch (ex) { - finish(); - } - }); - }; + const req = table.openCursor(); + req.onsuccess = function(ev) { + let cursor = ev.target && ev.target.result; + if ( !cursor ) { return; } + let entry = cursor.value; + visitFn(entry); + cursor.continue(); + }; + }); + }; - const deleteFromDb = function(input, callback) { - if ( typeof callback !== 'function' ) { - callback = noopfn; - } - let keys = Array.isArray(input) ? input.slice() : [ input ]; - if ( keys.length === 0 ) { return callback(); } - getDb().then(db => { - if ( !db ) { return callback(); } - let finish = ( ) => { - dbBytesInUse = undefined; - if ( callback === undefined ) { return; } - let cb = callback; - callback = undefined; - cb(); - }; - try { - let transaction = db.transaction(STORAGE_NAME, 'readwrite'); - transaction.oncomplete = - transaction.onerror = - transaction.onabort = finish; - let table = transaction.objectStore(STORAGE_NAME); - for ( let key of keys ) { - table.delete(key); - } - } catch (ex) { - finish(); - } - }); - }; - - const clearDb = function(callback) { - if ( typeof callback !== 'function' ) { - callback = noopfn; - } - getDb().then(db => { - if ( !db ) { return callback(); } - let finish = ( ) => { - disconnect(); - indexedDB.deleteDatabase(STORAGE_NAME); - dbBytesInUse = 0; - if ( callback === undefined ) { return; } - let cb = callback; - callback = undefined; - cb(); - }; - try { - let req = db.transaction(STORAGE_NAME, 'readwrite') - .objectStore(STORAGE_NAME) - .clear(); - req.onsuccess = req.onerror = finish; - } catch (ex) { - finish(); - } - }); - }; - - // prime the db so that it's ready asap for next access. - getDb(noopfn); - - // https://github.com/uBlockOrigin/uBlock-issues/issues/328 - // Detect whether browser.storage.local was used as cache storage, - // and if so, move cache-related entries to the new storage. - { - const srcStorage = vAPI.cacheStorage; - const desStorage = api; - srcStorage.get( - [ 'assetCacheRegistry', 'assetSourceRegistry' ], - bin => { - if ( - bin instanceof Object === false || - bin.assetSourceRegistry instanceof Object === false - ) { + const getAllFromDb = function(callback) { + if ( typeof callback !== 'function' ) { return; } + const promises = []; + const keyvalStore = {}; + visitAllFromDb(entry => { + if ( entry === undefined ) { + Promise.all(promises).then(( ) => { + callback(keyvalStore); + }); return; } - desStorage.set(bin); - const toRemove = [ - 'assetCacheRegistry', - 'assetSourceRegistry', - 'resourcesSelfie', - 'selfie' - ]; - let toMigrate = 0; - const setEntry = function(assetKey, bin) { - if ( - bin instanceof Object && - bin[assetKey] !== undefined - ) { - desStorage.set(bin); - } - toMigrate -= 1; - if ( toMigrate === 0 ) { - srcStorage.remove(toRemove); - } - }; - for ( const key in bin.assetCacheRegistry ) { - if ( bin.assetCacheRegistry.hasOwnProperty(key) === false ) { - continue; - } - const assetKey = 'cache/' + key; - srcStorage.get(assetKey, setEntry.bind(null, assetKey)); - toMigrate += 1; - toRemove.push(assetKey); + keyvalStore[entry.key] = entry.value; + if ( entry.value instanceof Blob === false ) { return; } + promises.push( + µBlock.lz4Codec.decode( + entry.key, + entry.value + ).then(result => { + keyvalStore[result.key] = result.value; + }) + ); + }); + }; + + // https://github.com/uBlockOrigin/uBlock-issues/issues/141 + // Mind that IDBDatabase.transaction() and IDBObjectStore.put() + // can throw: + // https://developer.mozilla.org/en-US/docs/Web/API/IDBDatabase/transaction + // https://developer.mozilla.org/en-US/docs/Web/API/IDBObjectStore/put + + const putToDb = function(keyvalStore, callback) { + if ( typeof callback !== 'function' ) { + callback = noopfn; + } + const keys = Object.keys(keyvalStore); + if ( keys.length === 0 ) { return callback(); } + const promises = [ getDb() ]; + const entries = []; + const dontCompress = + µBlock.hiddenSettings.cacheStorageCompression !== true; + let bytesInUse = 0; + const handleEncodingResult = result => { + if ( typeof result.data === 'string' ) { + bytesInUse += result.data.length; + } else if ( result.data instanceof Blob ) { + bytesInUse += result.data.size; } - if ( toMigrate === 0 ) { - srcStorage.remove(toRemove); + entries.push({ key: result.key, value: result.data }); + }; + for ( const key of keys ) { + const data = keyvalStore[key]; + const isString = typeof data === 'string'; + if ( isString === false || dontCompress ) { + if ( isString ) { + bytesInUse += data.length; + } + entries.push({ key, value: data }); + continue; + } + promises.push( + µBlock.lz4Codec.encode(key, data).then(handleEncodingResult) + ); + } + Promise.all(promises).then(results => { + const db = results[0]; + if ( !db ) { return callback(); } + const finish = ( ) => { + if ( callback === undefined ) { return; } + let cb = callback; + callback = undefined; + cb({ bytesInUse }); + }; + try { + const transaction = db.transaction( + STORAGE_NAME, + 'readwrite' + ); + transaction.oncomplete = + transaction.onerror = + transaction.onabort = finish; + const table = transaction.objectStore(STORAGE_NAME); + for ( const entry of entries ) { + table.put(entry); + } + } catch (ex) { + finish(); + } + }); + }; + + const deleteFromDb = function(input, callback) { + if ( typeof callback !== 'function' ) { + callback = noopfn; + } + const keys = Array.isArray(input) ? input.slice() : [ input ]; + if ( keys.length === 0 ) { return callback(); } + getDb().then(db => { + if ( !db ) { return callback(); } + let finish = ( ) => { + if ( callback === undefined ) { return; } + let cb = callback; + callback = undefined; + cb(); + }; + try { + let transaction = db.transaction(STORAGE_NAME, 'readwrite'); + transaction.oncomplete = + transaction.onerror = + transaction.onabort = finish; + let table = transaction.objectStore(STORAGE_NAME); + for ( let key of keys ) { + table.delete(key); + } + } catch (ex) { + finish(); + } + }); + }; + + const clearDb = function(callback) { + if ( typeof callback !== 'function' ) { + callback = noopfn; + } + getDb().then(db => { + if ( !db ) { return callback(); } + const finish = ( ) => { + disconnect(); + indexedDB.deleteDatabase(STORAGE_NAME); + if ( callback === undefined ) { return; } + let cb = callback; + callback = undefined; + cb(); + }; + try { + const req = db.transaction(STORAGE_NAME, 'readwrite') + .objectStore(STORAGE_NAME) + .clear(); + req.onsuccess = req.onerror = finish; + } catch (ex) { + finish(); + } + }); + }; + + return getDb().then(db => { + if ( !db ) { return false; } + api.name = 'indexedDB'; + api.get = function get(keys) { + return new Promise(resolve => { + if ( keys === null ) { + return getAllFromDb(bin => resolve(bin)); + } + let toRead, output = {}; + if ( typeof keys === 'string' ) { + toRead = [ keys ]; + } else if ( Array.isArray(keys) ) { + toRead = keys; + } else /* if ( typeof keys === 'object' ) */ { + toRead = Object.keys(keys); + output = keys; + } + getFromDb(toRead, output, bin => resolve(bin)); + }); + }; + api.set = function set(keys) { + return new Promise(resolve => { + putToDb(keys, details => resolve(details)); + }); + }; + api.remove = function remove(keys) { + return new Promise(resolve => { + deleteFromDb(keys, ( ) => resolve()); + }); + }; + api.clear = function clear() { + return new Promise(resolve => { + clearDb(( ) => resolve()); + }); + }; + api.getBytesInUse = function getBytesInUse() { + return Promise.resolve(0); + }; + return true; + }); + }; + + // https://github.com/uBlockOrigin/uBlock-issues/issues/328 + // Delete cache-related entries from webext storage. + const clearWebext = function() { + browser.storage.local.get('assetCacheRegistry', bin => { + if ( + bin instanceof Object === false || + bin.assetCacheRegistry instanceof Object === false + ) { + return; + } + const toRemove = [ + 'assetCacheRegistry', + 'assetSourceRegistry', + 'resourcesSelfie', + 'selfie' + ]; + for ( const key in bin.assetCacheRegistry ) { + if ( bin.assetCacheRegistry.hasOwnProperty(key) ) { + toRemove.push('cache/' + key); } } - ); - } + browser.storage.local.remove(toRemove); + }); + }; + + const clearIDB = function() { + indexedDB.deleteDatabase(STORAGE_NAME); + }; return api; }()); diff --git a/src/js/messaging.js b/src/js/messaging.js index 877b3c9a3..9904df11a 100644 --- a/src/js/messaging.js +++ b/src/js/messaging.js @@ -848,17 +848,17 @@ var restoreUserData = function(request) { // Remove all stored data but keep global counts, people can become // quite attached to numbers -var resetUserData = function() { +const resetUserData = function() { let count = 3; - let countdown = ( ) => { + const countdown = ( ) => { count -= 1; if ( count === 0 ) { vAPI.app.restart(); } }; - µb.cacheStorage.clear(countdown); // 1 - vAPI.storage.clear(countdown); // 2 - µb.saveLocalSettings(countdown); // 3 + µb.cacheStorage.clear().then(( ) => countdown()); // 1 + vAPI.storage.clear(countdown); // 2 + µb.saveLocalSettings(countdown); // 3 vAPI.localStorage.removeItem('immediateHiddenSettings'); }; diff --git a/src/js/start.js b/src/js/start.js index 26cad1393..13b55182c 100644 --- a/src/js/start.js +++ b/src/js/start.js @@ -333,7 +333,7 @@ const fromFetch = function(to, fetched) { /******************************************************************************/ -const onSelectedFilterListsLoaded = function() { +const onSelectedFilterListsReady = function() { log.info(`List selection ready ${Date.now()-vAPI.T0} ms after launch`); const fetchableProps = { @@ -371,6 +371,16 @@ const onSelectedFilterListsLoaded = function() { /******************************************************************************/ +const onHiddenSettingsReady = function() { + return µb.cacheStorage.select( + µb.hiddenSettings.cacheStorageAPI + ).then(backend => { + log.info(`Backend storage for cache will be ${backend}`); + }); +}; + +/******************************************************************************/ + // TODO(seamless migration): // Eventually selected filter list keys will be loaded as a fetchable // property. Until then we need to handle backward and forward @@ -379,14 +389,24 @@ const onSelectedFilterListsLoaded = function() { const onAdminSettingsRestored = function() { log.info(`Admin settings ready ${Date.now()-vAPI.T0} ms after launch`); - µb.loadSelectedFilterLists(onSelectedFilterListsLoaded); + + Promise.all([ + µb.loadHiddenSettings().then(( ) => + onHiddenSettingsReady() + ), + µb.loadSelectedFilterLists(), + ]).then(( ) => + onSelectedFilterListsReady() + ); }; /******************************************************************************/ return function() { // https://github.com/gorhill/uBlock/issues/531 - µb.restoreAdminSettings(onAdminSettingsRestored); + µb.restoreAdminSettings().then(( ) => { + onAdminSettingsRestored(); + }); }; /******************************************************************************/ diff --git a/src/js/storage.js b/src/js/storage.js index 9076216f3..1216c8b6b 100644 --- a/src/js/storage.js +++ b/src/js/storage.js @@ -50,7 +50,7 @@ countdown += 1; vAPI.storage.getBytesInUse(null, process); } - if ( this.cacheStorage !== vAPI.storage ) { + if ( this.cacheStorage.name !== 'browser.storage.local' ) { countdown += 1; this.assets.getBytesInUse().then(count => { process(count); @@ -91,8 +91,13 @@ /******************************************************************************/ µBlock.loadHiddenSettings = function() { + return new Promise(resolve => { + // >>>> start of executor + vAPI.storage.get('hiddenSettings', bin => { - if ( bin instanceof Object === false ) { return; } + if ( bin instanceof Object === false ) { + return resolve(); + } const hs = bin.hiddenSettings; if ( hs instanceof Object ) { const hsDefault = this.hiddenSettingsDefault; @@ -110,6 +115,10 @@ this.saveImmediateHiddenSettings(); } self.log.verbosity = this.hiddenSettings.consoleLogLevel; + resolve(); + }); + + // <<<< end of executor }); }; @@ -195,9 +204,6 @@ ); }; -// Do this here to have these hidden settings loaded ASAP. -µBlock.loadHiddenSettings(); - /******************************************************************************/ µBlock.savePermanentFirewallRules = function() { @@ -240,24 +246,29 @@ **/ -µBlock.loadSelectedFilterLists = function(callback) { - var µb = this; - vAPI.storage.get('selectedFilterLists', function(bin) { +µBlock.loadSelectedFilterLists = function() { + return new Promise(resolve => { + // >>>> start of executor + + vAPI.storage.get('selectedFilterLists', bin => { // Select default filter lists if first-time launch. - if ( !bin || Array.isArray(bin.selectedFilterLists) === false ) { - µb.assets.metadata(function(availableLists) { - µb.saveSelectedFilterLists( - µb.autoSelectRegionalFilterLists(availableLists) + if ( + bin instanceof Object === false || + Array.isArray(bin.selectedFilterLists) === false + ) { + this.assets.metadata(function(availableLists) { + this.saveSelectedFilterLists( + this.autoSelectRegionalFilterLists(availableLists) ); - callback(); + resolve(); }); return; } - // TODO: Removes once 1.1.15 is in widespread use. - // https://github.com/gorhill/uBlock/issues/3383 - vAPI.storage.remove('remoteBlacklists'); - µb.selectedFilterLists = bin.selectedFilterLists; - callback(); + this.selectedFilterLists = bin.selectedFilterLists; + resolve(); + }); + + // <<<< end of executor }); }; @@ -1130,16 +1141,16 @@ // necessarily present, i.e. administrators may removed entries which // values are left to the user's choice. -µBlock.restoreAdminSettings = function(callback) { - // Support for vAPI.adminStorage is optional (webext). +µBlock.restoreAdminSettings = function() { + return new Promise(resolve => { + // >>>> start of executor + if ( vAPI.adminStorage instanceof Object === false ) { - callback(); - return; + return resolve(); } - var onRead = function(json) { - var µb = µBlock; - var data; + vAPI.adminStorage.getItem('adminSettings', json => { + let data; if ( typeof json === 'string' && json !== '' ) { try { data = JSON.parse(json); @@ -1148,13 +1159,12 @@ } } - if ( typeof data !== 'object' || data === null ) { - callback(); - return; + if ( data instanceof Object === false ) { + return resolve(); } - var bin = {}; - var binNotEmpty = false; + const bin = {}; + let binNotEmpty = false; // Allows an admin to set their own 'assets.json' file, with their own // set of stock assets. @@ -1164,8 +1174,8 @@ } if ( typeof data.userSettings === 'object' ) { - for ( var name in µb.userSettings ) { - if ( µb.userSettings.hasOwnProperty(name) === false ) { + for ( const name in this.userSettings ) { + if ( this.userSettings.hasOwnProperty(name) === false ) { continue; } if ( data.userSettings.hasOwnProperty(name) === false ) { @@ -1208,13 +1218,14 @@ } if ( typeof data.userFilters === 'string' ) { - µb.assets.put(µb.userFiltersPath, data.userFilters); + this.assets.put(this.userFiltersPath, data.userFilters); } - callback(); - }; + resolve(); + }); - vAPI.adminStorage.getItem('adminSettings', onRead); + // <<<< end of executor + }); }; /******************************************************************************/