From ed7e34fb07f956229b2290608e4500b35f431afe Mon Sep 17 00:00:00 2001 From: Raymond Hill Date: Thu, 14 Feb 2019 13:33:55 -0500 Subject: [PATCH] Refactor selfie generation into a more flexible persistence mechanism The motivation is to address the higher peak memory usage at launch time with 3rd-gen HNTrie when a selfie was present. The selfie generation prior to this change was to collect all filtering data into a single data structure, and then to serialize that whole structure at once into storage (using JSON.stringify). However, HNTrie serialization requires that a large UintArray32 be converted into a plain JS array, which itslef would be indirectly converted into a JSON string. This was the main reason why peak memory usage would be higher at launch from selfie, since the JSON string would need to be wholly unserialized into JS objects, which themselves would need to be converted into more specialized data structures (like that Uint32Array one). The solution to lower peak memory usage at launch is to refactor selfie generation to allow a more piecemeal approach: each filtering component is given the ability to serialize itself rather than to be forced to be embedded in the master selfie. With this approach, the HNTrie buffer can now serialize to its own storage by converting the buffer data directly into a string which can be directly sent to storage. This avoiding expensive intermediate steps such as converting into a JS array and then to a JSON string. As part of the refactoring, there was also opportunistic code upgrade to ES6 and Promise (eventually all of uBO's code will be proper ES6). Additionally, the polyfill to bring getBytesInUse() to Firefox has been revisited to replace the rather expensive previous implementation with an implementation with virtually no overhead. --- .jshintrc | 1 + platform/chromium/vapi-common.js | 4 + src/background.html | 1 + src/js/assets.js | 145 ++++++++++++++--------- src/js/background.js | 14 +-- src/js/cachestorage.js | 36 +++--- src/js/console.js | 34 ++++++ src/js/hntrie.js | 34 ++++-- src/js/redirect-engine.js | 115 ++++++++++-------- src/js/start.js | 73 ++++++------ src/js/static-ext-filtering.js | 32 +++-- src/js/static-net-filtering.js | 119 +++++++++++-------- src/js/storage.js | 193 ++++++++++++++++--------------- src/js/utils.js | 109 +++++++++++++++++ 14 files changed, 588 insertions(+), 322 deletions(-) create mode 100644 src/js/console.js diff --git a/.jshintrc b/.jshintrc index b0fe596b3..b82e4d8f8 100644 --- a/.jshintrc +++ b/.jshintrc @@ -7,6 +7,7 @@ "browser": false, // global variable in Firefox, Edge "chrome": false, // global variable in Chromium, Chrome, Opera "Components": false, // global variable in Firefox + "log": false, "safari": false, "self": false, "vAPI": false, diff --git a/platform/chromium/vapi-common.js b/platform/chromium/vapi-common.js index 8e577f441..0454086d3 100644 --- a/platform/chromium/vapi-common.js +++ b/platform/chromium/vapi-common.js @@ -30,6 +30,10 @@ /******************************************************************************/ +vAPI.T0 = Date.now(); + +/******************************************************************************/ + vAPI.setTimeout = vAPI.setTimeout || self.setTimeout.bind(self); /******************************************************************************/ diff --git a/src/background.html b/src/background.html index 104b94131..484d55944 100644 --- a/src/background.html +++ b/src/background.html @@ -5,6 +5,7 @@ uBlock Origin + diff --git a/src/js/assets.js b/src/js/assets.js index a1a9e1157..3ccc7f8a5 100644 --- a/src/js/assets.js +++ b/src/js/assets.js @@ -449,26 +449,22 @@ const assetCacheRegistryStartTime = Date.now(); let assetCacheRegistryPromise; let assetCacheRegistry = {}; -const getAssetCacheRegistry = function(callback) { +const getAssetCacheRegistry = function() { if ( assetCacheRegistryPromise === undefined ) { assetCacheRegistryPromise = new Promise(resolve => { - // start of executor - µBlock.cacheStorage.get('assetCacheRegistry', bin => { - if ( - bin instanceof Object && - bin.assetCacheRegistry instanceof Object - ) { - assetCacheRegistry = bin.assetCacheRegistry; - } - resolve(); - }); - // end of executor + µBlock.cacheStorage.get('assetCacheRegistry', bin => { + if ( + bin instanceof Object && + bin.assetCacheRegistry instanceof Object + ) { + assetCacheRegistry = bin.assetCacheRegistry; + } + resolve(); + }); }); } - assetCacheRegistryPromise.then(( ) => { - callback(assetCacheRegistry); - }); + return assetCacheRegistryPromise.then(( ) => assetCacheRegistry); }; const saveAssetCacheRegistry = (function() { @@ -513,11 +509,9 @@ const assetCacheRead = function(assetKey, callback) { reportBack(bin[internalKey]); }; - let onReady = function() { + getAssetCacheRegistry().then(( ) => { µBlock.cacheStorage.get(internalKey, onAssetRead); - }; - - getAssetCacheRegistry(onReady); + }); }; const assetCacheWrite = function(assetKey, details, callback) { @@ -542,7 +536,18 @@ const assetCacheWrite = function(assetKey, details, callback) { if ( details instanceof Object && typeof details.url === 'string' ) { entry.remoteURL = details.url; } - µBlock.cacheStorage.set({ assetCacheRegistry, [internalKey]: content }); + µBlock.cacheStorage.set( + { [internalKey]: content }, + details => { + if ( + details instanceof Object && + typeof details.bytesInUse === 'number' + ) { + entry.byteLength = details.bytesInUse; + } + saveAssetCacheRegistry(true); + } + ); const result = { assetKey, content }; if ( typeof callback === 'function' ) { callback(result); @@ -550,14 +555,16 @@ const assetCacheWrite = function(assetKey, details, callback) { // https://github.com/uBlockOrigin/uBlock-issues/issues/248 fireNotification('after-asset-updated', result); }; - getAssetCacheRegistry(onReady); + + getAssetCacheRegistry().then(( ) => { + µBlock.cacheStorage.get(internalKey, onReady); + }); }; const assetCacheRemove = function(pattern, callback) { - const onReady = function() { - const cacheDict = assetCacheRegistry, - removedEntries = [], - removedContent = []; + getAssetCacheRegistry().then(cacheDict => { + const removedEntries = []; + const removedContent = []; for ( const assetKey in cacheDict ) { if ( pattern instanceof RegExp && !pattern.test(assetKey) ) { continue; @@ -582,14 +589,15 @@ const assetCacheRemove = function(pattern, callback) { { assetKey: removedEntries[i] } ); } - }; - - getAssetCacheRegistry(onReady); + }); }; const assetCacheMarkAsDirty = function(pattern, exclude, callback) { - const onReady = function() { - const cacheDict = assetCacheRegistry; + if ( typeof exclude === 'function' ) { + callback = exclude; + exclude = undefined; + } + getAssetCacheRegistry().then(cacheDict => { let mustSave = false; for ( const assetKey in cacheDict ) { if ( pattern instanceof RegExp ) { @@ -617,12 +625,7 @@ const assetCacheMarkAsDirty = function(pattern, exclude, callback) { if ( typeof callback === 'function' ) { callback(); } - }; - if ( typeof exclude === 'function' ) { - callback = exclude; - exclude = undefined; - } - getAssetCacheRegistry(onReady); + }); }; /******************************************************************************/ @@ -642,12 +645,12 @@ const stringIsNotEmpty = function(s) { **/ -var readUserAsset = function(assetKey, callback) { - var reportBack = function(content) { +const readUserAsset = function(assetKey, callback) { + const reportBack = function(content) { callback({ assetKey: assetKey, content: content }); }; - var onLoaded = function(bin) { + const onLoaded = function(bin) { if ( !bin ) { return reportBack(''); } var content = ''; if ( typeof bin['cached_asset_content://assets/user/filters.txt'] === 'string' ) { @@ -671,7 +674,7 @@ var readUserAsset = function(assetKey, callback) { } return reportBack(content); }; - var toRead = assetKey; + let toRead = assetKey; if ( assetKey === µBlock.userFiltersPath ) { toRead = [ assetKey, @@ -682,7 +685,7 @@ var readUserAsset = function(assetKey, callback) { vAPI.storage.get(toRead, onLoaded); }; -var saveUserAsset = function(assetKey, content, callback) { +const saveUserAsset = function(assetKey, content, callback) { var bin = {}; bin[assetKey] = content; // TODO(seamless migration): @@ -711,27 +714,33 @@ api.get = function(assetKey, options, callback) { callback = noopfunc; } + return new Promise(resolve => { + // start of executor if ( assetKey === µBlock.userFiltersPath ) { - readUserAsset(assetKey, callback); + readUserAsset(assetKey, details => { + callback(details); + resolve(details); + }); return; } - var assetDetails = {}, + let assetDetails = {}, contentURLs, contentURL; - var reportBack = function(content, err) { - var details = { assetKey: assetKey, content: content }; + const reportBack = function(content, err) { + const details = { assetKey: assetKey, content: content }; if ( err ) { details.error = assetDetails.lastError = err; } else { assetDetails.lastError = undefined; } callback(details); + resolve(details); }; - var onContentNotLoaded = function() { - var isExternal; + const onContentNotLoaded = function() { + let isExternal; while ( (contentURL = contentURLs.shift()) ) { isExternal = reIsExternalPath.test(contentURL); if ( isExternal === false || assetDetails.hasLocalURL !== true ) { @@ -748,7 +757,7 @@ api.get = function(assetKey, options, callback) { } }; - var onContentLoaded = function(details) { + const onContentLoaded = function(details) { if ( stringIsNotEmpty(details.content) === false ) { onContentNotLoaded(); return; @@ -762,7 +771,7 @@ api.get = function(assetKey, options, callback) { reportBack(details.content); }; - var onCachedContentLoaded = function(details) { + const onCachedContentLoaded = function(details) { if ( details.content !== '' ) { return reportBack(details.content); } @@ -780,11 +789,13 @@ api.get = function(assetKey, options, callback) { }; assetCacheRead(assetKey, onCachedContentLoaded); + // end of executor + }); }; /******************************************************************************/ -var getRemote = function(assetKey, callback) { +const getRemote = function(assetKey, callback) { var assetDetails = {}, contentURLs, contentURL; @@ -852,10 +863,19 @@ var getRemote = function(assetKey, callback) { /******************************************************************************/ api.put = function(assetKey, content, callback) { - if ( reIsUserAsset.test(assetKey) ) { - return saveUserAsset(assetKey, content, callback); - } - assetCacheWrite(assetKey, content, callback); + return new Promise(resolve => { + const onDone = function(details) { + if ( typeof callback === 'function' ) { + callback(details); + } + resolve(details); + }; + if ( reIsUserAsset.test(assetKey) ) { + saveUserAsset(assetKey, content, onDone); + } else { + assetCacheWrite(assetKey, content, onDone); + } + }); }; /******************************************************************************/ @@ -895,7 +915,7 @@ api.metadata = function(callback) { if ( cacheRegistryReady ) { onReady(); } }); - getAssetCacheRegistry(function() { + getAssetCacheRegistry().then(( ) => { cacheRegistryReady = true; if ( assetRegistryReady ) { onReady(); } }); @@ -903,6 +923,19 @@ api.metadata = function(callback) { /******************************************************************************/ +api.getBytesInUse = function() { + return getAssetCacheRegistry().then(cacheDict => { + let bytesUsed = 0; + for ( const assetKey in cacheDict ) { + if ( cacheDict.hasOwnProperty(assetKey) === false ) { continue; } + bytesUsed += cacheDict[assetKey].byteLength || 0; + } + return bytesUsed; + }); +}; + +/******************************************************************************/ + api.purge = assetCacheMarkAsDirty; api.remove = function(pattern, callback) { @@ -1013,7 +1046,7 @@ var updateNext = function() { updateOne(); }); - getAssetCacheRegistry(function(dict) { + getAssetCacheRegistry().then(dict => { cacheDict = dict; if ( !assetDict ) { return; } updateOne(); diff --git a/src/js/background.js b/src/js/background.js index 37dff9d9f..52cbb67a3 100644 --- a/src/js/background.js +++ b/src/js/background.js @@ -46,6 +46,7 @@ const µBlock = (function() { // jshint ignore:line cacheStorageAPI: 'unset', cacheStorageCompression: true, cacheControlForFirefox1376932: 'no-cache, no-store, must-revalidate', + consoleLogLevel: 'unset', debugScriptlets: false, disableWebAssembly: false, ignoreRedirectFilters: false, @@ -53,6 +54,7 @@ const µBlock = (function() { // jshint ignore:line manualUpdateAssetFetchPeriod: 500, popupFontSize: 'unset', requestJournalProcessPeriod: 1000, + selfieAfter: 11, strictBlockingBypassDuration: 120, suspendTabsUntilReady: false, userResourcesLocation: 'unset' @@ -95,13 +97,13 @@ const µBlock = (function() { // jshint ignore:line hiddenSettingsDefault: hiddenSettingsDefault, hiddenSettings: (function() { - let out = Object.assign({}, hiddenSettingsDefault), + const out = Object.assign({}, hiddenSettingsDefault), json = vAPI.localStorage.getItem('immediateHiddenSettings'); if ( typeof json === 'string' ) { try { - let o = JSON.parse(json); + const o = JSON.parse(json); if ( o instanceof Object ) { - for ( let k in o ) { + for ( const k in o ) { if ( out.hasOwnProperty(k) ) { out[k] = o[k]; } @@ -111,8 +113,6 @@ const µBlock = (function() { // jshint ignore:line catch(ex) { } } - // Remove once 1.15.12+ is widespread. - vAPI.localStorage.removeItem('hiddenSettings'); return out; })(), @@ -138,7 +138,7 @@ const µBlock = (function() { // jshint ignore:line // Read-only systemSettings: { compiledMagic: 6, // Increase when compiled format changes - selfieMagic: 7 // Increase when selfie format changes + selfieMagic: 8 // Increase when selfie format changes }, restoreBackupSettings: { @@ -161,8 +161,6 @@ const µBlock = (function() { // jshint ignore:line selectedFilterLists: [], availableFilterLists: {}, - selfieAfter: 17 * oneMinute, - pageStores: new Map(), pageStoresToken: 0, diff --git a/src/js/cachestorage.js b/src/js/cachestorage.js index 9b5a0035f..47771fe52 100644 --- a/src/js/cachestorage.js +++ b/src/js/cachestorage.js @@ -326,17 +326,27 @@ if ( typeof callback !== 'function' ) { callback = noopfn; } - let keys = Object.keys(keyvalStore); + const keys = Object.keys(keyvalStore); if ( keys.length === 0 ) { return callback(); } - let promises = [ getDb() ]; - let entries = []; - let dontCompress = µBlock.hiddenSettings.cacheStorageCompression !== true; - let handleEncodingResult = result => { + const promises = [ getDb() ]; + const entries = []; + const dontCompress = µBlock.hiddenSettings.cacheStorageCompression !== true; + let bytesInUse = 0; + const handleEncodingResult = result => { + if ( typeof result.data === 'string' ) { + bytesInUse += result.data.length; + } else if ( result.data instanceof Blob ) { + bytesInUse += result.data.size; + } entries.push({ key: result.key, value: result.data }); }; - for ( let key of keys ) { - let data = keyvalStore[key]; - if ( typeof data !== 'string' || dontCompress ) { + for ( const key of keys ) { + const data = keyvalStore[key]; + const isString = typeof data === 'string'; + if ( isString === false || dontCompress ) { + if ( isString ) { + bytesInUse += data.length; + } entries.push({ key, value: data }); continue; } @@ -346,20 +356,20 @@ } Promise.all(promises).then(( ) => { if ( !db ) { return callback(); } - let finish = ( ) => { + const finish = ( ) => { dbBytesInUse = undefined; if ( callback === undefined ) { return; } let cb = callback; callback = undefined; - cb(); + cb({ bytesInUse }); }; try { - let transaction = db.transaction(STORAGE_NAME, 'readwrite'); + const transaction = db.transaction(STORAGE_NAME, 'readwrite'); transaction.oncomplete = transaction.onerror = transaction.onabort = finish; - let table = transaction.objectStore(STORAGE_NAME); - for ( let entry of entries ) { + const table = transaction.objectStore(STORAGE_NAME); + for ( const entry of entries ) { table.put(entry); } } catch (ex) { diff --git a/src/js/console.js b/src/js/console.js new file mode 100644 index 000000000..9e7fafb5c --- /dev/null +++ b/src/js/console.js @@ -0,0 +1,34 @@ +/******************************************************************************* + + uBlock Origin - a browser extension to block requests. + Copyright (C) 2019-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +self.log = (function() { + const noopFunc = function() {}; + const info = function(s) { console.log(`[uBO] ${s}`); }; + return { + get verbosity( ) { return; }, + set verbosity(level) { + this.info = console.info = level === 'info' ? info : noopFunc; + }, + info, + }; +})(); diff --git a/src/js/hntrie.js b/src/js/hntrie.js index 2ad411618..607d8ef8a 100644 --- a/src/js/hntrie.js +++ b/src/js/hntrie.js @@ -355,7 +355,13 @@ HNTrieContainer.prototype = { return trieRef; }, - serialize: function() { + serialize: function(encoder) { + if ( encoder instanceof Object ) { + return encoder.encode( + this.buf32.buffer, + this.buf32[HNTRIE_CHAR1_SLOT] + ); + } return Array.from( new Uint32Array( this.buf32.buffer, @@ -365,23 +371,29 @@ HNTrieContainer.prototype = { ); }, - unserialize: function(selfie) { - const len = (selfie.length << 2) + HNTRIE_PAGE_SIZE-1 & ~(HNTRIE_PAGE_SIZE-1); + unserialize: function(selfie, decoder) { + const shouldDecode = typeof selfie === 'string'; + let byteLength = shouldDecode + ? decoder.decodeSize(selfie) + : selfie.length << 2; + byteLength = byteLength + HNTRIE_PAGE_SIZE-1 & ~(HNTRIE_PAGE_SIZE-1); if ( this.wasmMemory !== null ) { const pageCountBefore = this.buf.length >>> 16; - const pageCountAfter = len >>> 16; + const pageCountAfter = byteLength >>> 16; if ( pageCountAfter > pageCountBefore ) { this.wasmMemory.grow(pageCountAfter - pageCountBefore); this.buf = new Uint8Array(this.wasmMemory.buffer); this.buf32 = new Uint32Array(this.buf.buffer); } - } else { - if ( len > this.buf.length ) { - this.buf = new Uint8Array(len); - this.buf32 = new Uint32Array(this.buf.buffer); - } + } else if ( byteLength > this.buf.length ) { + this.buf = new Uint8Array(byteLength); + this.buf32 = new Uint32Array(this.buf.buffer); + } + if ( shouldDecode ) { + decoder.decode(selfie, this.buf.buffer); + } else { + this.buf32.set(selfie); } - this.buf32.set(selfie); this.needle = ''; }, @@ -684,6 +696,6 @@ HNTrieContainer.prototype.HNTrieRef.prototype = { WebAssembly.compileStreaming ).catch(reason => { HNTrieContainer.wasmModulePromise = null; - console.info(reason); + log.info(reason); }); })(); diff --git a/src/js/redirect-engine.js b/src/js/redirect-engine.js index 2faf39cd5..419361216 100644 --- a/src/js/redirect-engine.js +++ b/src/js/redirect-engine.js @@ -29,12 +29,12 @@ /******************************************************************************/ const warResolve = (function() { - var warPairs = []; + let warPairs = []; - var onPairsReady = function() { - var reng = µBlock.redirectEngine; - for ( var i = 0; i < warPairs.length; i += 2 ) { - var resource = reng.resources.get(warPairs[i+0]); + const onPairsReady = function() { + const reng = µBlock.redirectEngine; + for ( let i = 0; i < warPairs.length; i += 2 ) { + const resource = reng.resources.get(warPairs[i+0]); if ( resource === undefined ) { continue; } resource.warURL = vAPI.getURL( '/web_accessible_resources/' + warPairs[i+1] @@ -48,15 +48,15 @@ const warResolve = (function() { return onPairsReady(); } - var onPairsLoaded = function(details) { - var marker = '>>>>>'; - var pos = details.content.indexOf(marker); + const onPairsLoaded = function(details) { + const marker = '>>>>>'; + const pos = details.content.indexOf(marker); if ( pos === -1 ) { return; } - var pairs = details.content.slice(pos + marker.length) + const pairs = details.content.slice(pos + marker.length) .trim() .split('\n'); if ( (pairs.length & 1) !== 0 ) { return; } - for ( var i = 0; i < pairs.length; i++ ) { + for ( let i = 0; i < pairs.length; i++ ) { pairs[i] = pairs[i].trim(); } warPairs = pairs; @@ -64,7 +64,7 @@ const warResolve = (function() { }; µBlock.assets.fetchText( - '/web_accessible_resources/imported.txt?secret=' + vAPI.warSecret, + `/web_accessible_resources/imported.txt?secret=${vAPI.warSecret}`, onPairsLoaded ); }; @@ -374,18 +374,17 @@ RedirectEngine.prototype.supportedTypes = new Map([ /******************************************************************************/ -RedirectEngine.prototype.toSelfie = function() { +RedirectEngine.prototype.toSelfie = function(path) { // Because rules may contains RegExp instances, we need to manually // convert it to a serializable format. The serialized format must be // suitable to be used as an argument to the Map() constructor. - var rules = [], - rule, entries, i, entry; - for ( var item of this.rules ) { - rule = [ item[0], [] ]; - entries = item[1]; - i = entries.length; + const rules = []; + for ( const item of this.rules ) { + const rule = [ item[0], [] ]; + const entries = item[1]; + let i = entries.length; while ( i-- ) { - entry = entries[i]; + const entry = entries[i]; rule[1].push({ tok: entry.tok, pat: entry.pat instanceof RegExp ? entry.pat.source : entry.pat @@ -393,23 +392,34 @@ RedirectEngine.prototype.toSelfie = function() { } rules.push(rule); } - return { - rules: rules, - ruleTypes: Array.from(this.ruleTypes), - ruleSources: Array.from(this.ruleSources), - ruleDestinations: Array.from(this.ruleDestinations) - }; + return µBlock.assets.put( + `${path}/main`, + JSON.stringify({ + rules: rules, + ruleTypes: Array.from(this.ruleTypes), + ruleSources: Array.from(this.ruleSources), + ruleDestinations: Array.from(this.ruleDestinations) + }) + ); }; /******************************************************************************/ -RedirectEngine.prototype.fromSelfie = function(selfie) { - this.rules = new Map(selfie.rules); - this.ruleTypes = new Set(selfie.ruleTypes); - this.ruleSources = new Set(selfie.ruleSources); - this.ruleDestinations = new Set(selfie.ruleDestinations); - this.modifyTime = Date.now(); - return true; +RedirectEngine.prototype.fromSelfie = function(path) { + return µBlock.assets.get(`${path}/main`).then(details => { + let selfie; + try { + selfie = JSON.parse(details.content); + } catch (ex) { + } + if ( selfie instanceof Object === false ) { return false; } + this.rules = new Map(selfie.rules); + this.ruleTypes = new Set(selfie.ruleTypes); + this.ruleSources = new Set(selfie.ruleSources); + this.ruleDestinations = new Set(selfie.ruleDestinations); + this.modifyTime = Date.now(); + return true; + }); }; /******************************************************************************/ @@ -494,41 +504,46 @@ RedirectEngine.prototype.resourcesFromString = function(text) { /******************************************************************************/ -let resourcesSelfieVersion = 3; +const resourcesSelfieVersion = 3; RedirectEngine.prototype.selfieFromResources = function() { - let selfie = { - version: resourcesSelfieVersion, - resources: Array.from(this.resources) - }; - µBlock.cacheStorage.set({ resourcesSelfie: JSON.stringify(selfie) }); + µBlock.assets.put( + 'compiled/redirectEngine/resources', + JSON.stringify({ + version: resourcesSelfieVersion, + resources: Array.from(this.resources) + }) + ); }; -RedirectEngine.prototype.resourcesFromSelfie = function(callback) { - µBlock.cacheStorage.get('resourcesSelfie', bin => { - let selfie = bin && bin.resourcesSelfie; - if ( typeof selfie === 'string' ) { - try { - selfie = JSON.parse(selfie); - } catch(ex) { - } +RedirectEngine.prototype.resourcesFromSelfie = function() { + return µBlock.assets.get( + 'compiled/redirectEngine/resources' + ).then(details => { + let selfie; + try { + selfie = JSON.parse(details.content); + } catch(ex) { } if ( selfie instanceof Object === false || selfie.version !== resourcesSelfieVersion || Array.isArray(selfie.resources) === false ) { - return callback(false); + return false; } this.resources = new Map(); - for ( let entry of selfie.resources ) { - this.resources.set(entry[0], RedirectEntry.fromSelfie(entry[1])); + for ( const [ token, entry ] of selfie.resources ) { + this.resources.set(token, RedirectEntry.fromSelfie(entry)); } - callback(true); + return true; }); }; RedirectEngine.prototype.invalidateResourcesSelfie = function() { + µBlock.assets.remove('compiled/redirectEngine/resources'); + + // TODO: obsolete, remove eventually µBlock.cacheStorage.remove('resourcesSelfie'); }; diff --git a/src/js/start.js b/src/js/start.js index ad7ba5e73..26cad1393 100644 --- a/src/js/start.js +++ b/src/js/start.js @@ -81,6 +81,8 @@ var onAllReady = function() { µb.contextMenu.update(null); µb.firstInstall = false; + + log.info(`All ready ${Date.now()-vAPI.T0} ms after launch`); }; /******************************************************************************/ @@ -137,22 +139,29 @@ let initializeTabs = function() { // Filtering engines dependencies: // - PSL -var onPSLReady = function() { - µb.selfieManager.load(function(valid) { +const onPSLReady = function() { + log.info(`PSL ready ${Date.now()-vAPI.T0} ms after launch`); + + µb.selfieManager.load().then(valid => { if ( valid === true ) { - return onAllReady(); + log.info(`Selfie ready ${Date.now()-vAPI.T0} ms after launch`); + onAllReady(); + return; } - µb.loadFilterLists(onAllReady); + µb.loadFilterLists(( ) => { + log.info(`Filter lists ready ${Date.now()-vAPI.T0} ms after launch`); + onAllReady(); + }); }); }; /******************************************************************************/ -var onCommandShortcutsReady = function(commandShortcuts) { +const onCommandShortcutsReady = function(commandShortcuts) { if ( Array.isArray(commandShortcuts) === false ) { return; } µb.commandShortcuts = new Map(commandShortcuts); if ( µb.canUpdateShortcuts === false ) { return; } - for ( let entry of commandShortcuts ) { + for ( const entry of commandShortcuts ) { vAPI.commands.update({ name: entry[0], shortcut: entry[1] }); } }; @@ -161,7 +170,7 @@ var onCommandShortcutsReady = function(commandShortcuts) { // To bring older versions up to date -var onVersionReady = function(lastVersion) { +const onVersionReady = function(lastVersion) { if ( lastVersion === vAPI.app.version ) { return; } // Since AMO does not allow updating resources.txt, force a reload when a @@ -176,7 +185,7 @@ var onVersionReady = function(lastVersion) { // If unused, just comment out for when we need to compare versions in the // future. - let intFromVersion = function(s) { + const intFromVersion = function(s) { let parts = s.match(/(?:^|\.|b|rc)\d+/g); if ( parts === null ) { return 0; } let vint = 0; @@ -223,7 +232,7 @@ var onVersionReady = function(lastVersion) { // Whitelist parser needs PSL to be ready. // gorhill 2014-12-15: not anymore -var onNetWhitelistReady = function(netWhitelistRaw) { +const onNetWhitelistReady = function(netWhitelistRaw) { µb.netWhitelist = µb.whitelistFromString(netWhitelistRaw); µb.netWhitelistModifyTime = Date.now(); }; @@ -232,8 +241,10 @@ var onNetWhitelistReady = function(netWhitelistRaw) { // User settings are in memory -var onUserSettingsReady = function(fetched) { - var userSettings = µb.userSettings; +const onUserSettingsReady = function(fetched) { + log.info(`User settings ready ${Date.now()-vAPI.T0} ms after launch`); + + const userSettings = µb.userSettings; fromFetch(userSettings, fetched); @@ -264,7 +275,7 @@ var onUserSettingsReady = function(fetched) { // Housekeeping, as per system setting changes -var onSystemSettingsReady = function(fetched) { +const onSystemSettingsReady = function(fetched) { var mustSaveSystemSettings = false; if ( fetched.compiledMagic !== µb.systemSettings.compiledMagic ) { µb.assets.remove(/^compiled\//); @@ -282,7 +293,9 @@ var onSystemSettingsReady = function(fetched) { /******************************************************************************/ -var onFirstFetchReady = function(fetched) { +const onFirstFetchReady = function(fetched) { + log.info(`First fetch ready ${Date.now()-vAPI.T0} ms after launch`); + // https://github.com/gorhill/uBlock/issues/747 µb.firstInstall = fetched.version === '0.0.0.0'; @@ -295,10 +308,7 @@ var onFirstFetchReady = function(fetched) { onVersionReady(fetched.version); onCommandShortcutsReady(fetched.commandShortcuts); - Promise.all([ - µb.loadPublicSuffixList(), - µb.staticNetFilteringEngine.readyToUse() - ]).then(( ) => { + µb.loadPublicSuffixList().then(( ) => { onPSLReady(); }); µb.loadRedirectResources(); @@ -306,31 +316,27 @@ var onFirstFetchReady = function(fetched) { /******************************************************************************/ -var toFetch = function(from, fetched) { - for ( var k in from ) { - if ( from.hasOwnProperty(k) === false ) { - continue; - } +const toFetch = function(from, fetched) { + for ( const k in from ) { + if ( from.hasOwnProperty(k) === false ) { continue; } fetched[k] = from[k]; } }; -var fromFetch = function(to, fetched) { - for ( var k in to ) { - if ( to.hasOwnProperty(k) === false ) { - continue; - } - if ( fetched.hasOwnProperty(k) === false ) { - continue; - } +const fromFetch = function(to, fetched) { + for ( const k in to ) { + if ( to.hasOwnProperty(k) === false ) { continue; } + if ( fetched.hasOwnProperty(k) === false ) { continue; } to[k] = fetched[k]; } }; /******************************************************************************/ -var onSelectedFilterListsLoaded = function() { - var fetchableProps = { +const onSelectedFilterListsLoaded = function() { + log.info(`List selection ready ${Date.now()-vAPI.T0} ms after launch`); + + const fetchableProps = { 'commandShortcuts': [], 'compiledMagic': 0, 'dynamicFilteringString': [ @@ -371,7 +377,8 @@ var onSelectedFilterListsLoaded = function() { // compatibility, this means a special asynchronous call to load selected // filter lists. -var onAdminSettingsRestored = function() { +const onAdminSettingsRestored = function() { + log.info(`Admin settings ready ${Date.now()-vAPI.T0} ms after launch`); µb.loadSelectedFilterLists(onSelectedFilterListsLoaded); }; diff --git a/src/js/static-ext-filtering.js b/src/js/static-ext-filtering.js index 135cb0980..8b2f533ef 100644 --- a/src/js/static-ext-filtering.js +++ b/src/js/static-ext-filtering.js @@ -821,18 +821,30 @@ µb.htmlFilteringEngine.fromCompiledContent(reader, options); }; - api.toSelfie = function() { - return { - cosmetic: µb.cosmeticFilteringEngine.toSelfie(), - scriptlets: µb.scriptletFilteringEngine.toSelfie(), - html: µb.htmlFilteringEngine.toSelfie() - }; + api.toSelfie = function(path) { + return µBlock.assets.put( + `${path}/main`, + JSON.stringify({ + cosmetic: µb.cosmeticFilteringEngine.toSelfie(), + scriptlets: µb.scriptletFilteringEngine.toSelfie(), + html: µb.htmlFilteringEngine.toSelfie() + }) + ); }; - api.fromSelfie = function(selfie) { - µb.cosmeticFilteringEngine.fromSelfie(selfie.cosmetic); - µb.scriptletFilteringEngine.fromSelfie(selfie.scriptlets); - µb.htmlFilteringEngine.fromSelfie(selfie.html); + api.fromSelfie = function(path) { + return µBlock.assets.get(`${path}/main`).then(details => { + let selfie; + try { + selfie = JSON.parse(details.content); + } catch (ex) { + } + if ( selfie instanceof Object === false ) { return false; } + µb.cosmeticFilteringEngine.fromSelfie(selfie.cosmetic); + µb.scriptletFilteringEngine.fromSelfie(selfie.scriptlets); + µb.htmlFilteringEngine.fromSelfie(selfie.html); + return true; + }); }; return api; diff --git a/src/js/static-net-filtering.js b/src/js/static-net-filtering.js index 59518f056..2d2874c9c 100644 --- a/src/js/static-net-filtering.js +++ b/src/js/static-net-filtering.js @@ -2105,21 +2105,21 @@ FilterContainer.prototype.readyToUse = function() { /******************************************************************************/ -FilterContainer.prototype.toSelfie = function() { - let categoriesToSelfie = function(categoryMap) { - let selfie = []; - for ( let categoryEntry of categoryMap ) { - let tokenEntries = []; - for ( let tokenEntry of categoryEntry[1] ) { - tokenEntries.push([ tokenEntry[0], tokenEntry[1].compile() ]); +FilterContainer.prototype.toSelfie = function(path) { + const categoriesToSelfie = function(categoryMap) { + const selfie = []; + for ( const [ catbits, bucket ] of categoryMap ) { + const tokenEntries = []; + for ( const [ token, filter ] of bucket ) { + tokenEntries.push([ token, filter.compile() ]); } - selfie.push([ categoryEntry[0], tokenEntries ]); + selfie.push([ catbits, tokenEntries ]); } return selfie; }; - let dataFiltersToSelfie = function(dataFilters) { - let selfie = []; + const dataFiltersToSelfie = function(dataFilters) { + const selfie = []; for ( let entry of dataFilters.values() ) { do { selfie.push(entry.compile()); @@ -2129,47 +2129,72 @@ FilterContainer.prototype.toSelfie = function() { return selfie; }; - return { - processedFilterCount: this.processedFilterCount, - acceptedCount: this.acceptedCount, - rejectedCount: this.rejectedCount, - allowFilterCount: this.allowFilterCount, - blockFilterCount: this.blockFilterCount, - discardedCount: this.discardedCount, - trieContainer: FilterHostnameDict.trieContainer.serialize(), - categories: categoriesToSelfie(this.categories), - dataFilters: dataFiltersToSelfie(this.dataFilters) - }; + return Promise.all([ + µBlock.assets.put( + `${path}/trieContainer`, + FilterHostnameDict.trieContainer.serialize(µBlock.base128) + ), + µBlock.assets.put( + `${path}/main`, + JSON.stringify({ + processedFilterCount: this.processedFilterCount, + acceptedCount: this.acceptedCount, + rejectedCount: this.rejectedCount, + allowFilterCount: this.allowFilterCount, + blockFilterCount: this.blockFilterCount, + discardedCount: this.discardedCount, + categories: categoriesToSelfie(this.categories), + dataFilters: dataFiltersToSelfie(this.dataFilters), + }) + ) + ]); }; /******************************************************************************/ -FilterContainer.prototype.fromSelfie = function(selfie) { - this.frozen = true; - this.processedFilterCount = selfie.processedFilterCount; - this.acceptedCount = selfie.acceptedCount; - this.rejectedCount = selfie.rejectedCount; - this.allowFilterCount = selfie.allowFilterCount; - this.blockFilterCount = selfie.blockFilterCount; - this.discardedCount = selfie.discardedCount; - FilterHostnameDict.trieContainer.unserialize(selfie.trieContainer); - - for ( let categoryEntry of selfie.categories ) { - let tokenMap = new Map(); - for ( let tokenEntry of categoryEntry[1] ) { - tokenMap.set(tokenEntry[0], filterFromCompiledData(tokenEntry[1])); - } - this.categories.set(categoryEntry[0], tokenMap); - } - - for ( let dataEntry of selfie.dataFilters ) { - let entry = FilterDataHolderEntry.load(dataEntry); - let bucket = this.dataFilters.get(entry.tokenHash); - if ( bucket !== undefined ) { - entry.next = bucket; - } - this.dataFilters.set(entry.tokenHash, entry); - } +FilterContainer.prototype.fromSelfie = function(path) { + return Promise.all([ + µBlock.assets.get(`${path}/trieContainer`).then(details => { + FilterHostnameDict.trieContainer.unserialize( + details.content, + µBlock.base128 + ); + return true; + }), + µBlock.assets.get(`${path}/main`).then(details => { + let selfie; + try { + selfie = JSON.parse(details.content); + } catch (ex) { + } + if ( selfie instanceof Object === false ) { return false; } + this.frozen = true; + this.processedFilterCount = selfie.processedFilterCount; + this.acceptedCount = selfie.acceptedCount; + this.rejectedCount = selfie.rejectedCount; + this.allowFilterCount = selfie.allowFilterCount; + this.blockFilterCount = selfie.blockFilterCount; + this.discardedCount = selfie.discardedCount; + for ( const [ catbits, bucket ] of selfie.categories ) { + const tokenMap = new Map(); + for ( const [ token, fdata ] of bucket ) { + tokenMap.set(token, filterFromCompiledData(fdata)); + } + this.categories.set(catbits, tokenMap); + } + for ( const dataEntry of selfie.dataFilters ) { + const entry = FilterDataHolderEntry.load(dataEntry); + const bucket = this.dataFilters.get(entry.tokenHash); + if ( bucket !== undefined ) { + entry.next = bucket; + } + this.dataFilters.set(entry.tokenHash, entry); + } + return true; + }), + ]).then(results => + results.reduce((acc, v) => acc && v, true) + ); }; /******************************************************************************/ diff --git a/src/js/storage.js b/src/js/storage.js index bd281decc..9076216f3 100644 --- a/src/js/storage.js +++ b/src/js/storage.js @@ -32,7 +32,7 @@ let bytesInUse; let countdown = 0; - let process = count => { + const process = count => { if ( typeof count === 'number' ) { if ( bytesInUse === undefined ) { bytesInUse = 0; @@ -50,12 +50,11 @@ countdown += 1; vAPI.storage.getBytesInUse(null, process); } - if ( - this.cacheStorage !== vAPI.storage && - this.cacheStorage.getBytesInUse instanceof Function - ) { + if ( this.cacheStorage !== vAPI.storage ) { countdown += 1; - this.cacheStorage.getBytesInUse(null, process); + this.assets.getBytesInUse().then(count => { + process(count); + }); } if ( countdown === 0 ) { callback(); @@ -94,10 +93,10 @@ µBlock.loadHiddenSettings = function() { vAPI.storage.get('hiddenSettings', bin => { if ( bin instanceof Object === false ) { return; } - let hs = bin.hiddenSettings; + const hs = bin.hiddenSettings; if ( hs instanceof Object ) { - let hsDefault = this.hiddenSettingsDefault; - for ( let key in hsDefault ) { + const hsDefault = this.hiddenSettingsDefault; + for ( const key in hsDefault ) { if ( hsDefault.hasOwnProperty(key) && hs.hasOwnProperty(key) && @@ -110,6 +109,7 @@ if ( vAPI.localStorage.getItem('immediateHiddenSettings') === null ) { this.saveImmediateHiddenSettings(); } + self.log.verbosity = this.hiddenSettings.consoleLogLevel; }); }; @@ -118,8 +118,8 @@ // which were not modified by the user. µBlock.saveHiddenSettings = function(callback) { - let bin = { hiddenSettings: {} }; - for ( let prop in this.hiddenSettings ) { + const bin = { hiddenSettings: {} }; + for ( const prop in this.hiddenSettings ) { if ( this.hiddenSettings.hasOwnProperty(prop) && this.hiddenSettings[prop] !== this.hiddenSettingsDefault[prop] @@ -129,6 +129,7 @@ } vAPI.storage.set(bin, callback); this.saveImmediateHiddenSettings(); + self.log.verbosity = this.hiddenSettings.consoleLogLevel; }; /******************************************************************************/ @@ -969,41 +970,41 @@ /******************************************************************************/ µBlock.loadRedirectResources = function(updatedContent) { - var µb = this, - content = ''; + let content = ''; - var onDone = function() { - µb.redirectEngine.resourcesFromString(content); + const onDone = ( ) => { + this.redirectEngine.resourcesFromString(content); }; - var onUserResourcesLoaded = function(details) { + const onUserResourcesLoaded = details => { if ( details.content !== '' ) { content += '\n\n' + details.content; } onDone(); }; - var onResourcesLoaded = function(details) { + const onResourcesLoaded = details => { if ( details.content !== '' ) { content = details.content; } - if ( µb.hiddenSettings.userResourcesLocation === 'unset' ) { + if ( this.hiddenSettings.userResourcesLocation === 'unset' ) { return onDone(); } - µb.assets.fetchText(µb.hiddenSettings.userResourcesLocation, onUserResourcesLoaded); + this.assets.fetchText( + this.hiddenSettings.userResourcesLocation, + onUserResourcesLoaded + ); }; if ( typeof updatedContent === 'string' && updatedContent.length !== 0 ) { return onResourcesLoaded({ content: updatedContent }); } - var onSelfieReady = function(success) { + this.redirectEngine.resourcesFromSelfie().then(success => { if ( success !== true ) { - µb.assets.get('ublock-resources', onResourcesLoaded); + this.assets.get('ublock-resources', onResourcesLoaded); } - }; - - µb.redirectEngine.resourcesFromSelfie(onSelfieReady); + }); }; /******************************************************************************/ @@ -1013,39 +1014,25 @@ publicSuffixList.enableWASM(); } - return new Promise(resolve => { - // start of executor - this.assets.get('compiled/' + this.pslAssetKey, details => { - let selfie; - try { - selfie = JSON.parse(details.content); - } catch (ex) { - } - if ( - selfie instanceof Object && - publicSuffixList.fromSelfie(selfie) - ) { - resolve(); - return; - } - this.assets.get(this.pslAssetKey, details => { + return this.assets.get( + 'compiled/' + this.pslAssetKey + ).then(details => + publicSuffixList.fromSelfie(details.content, µBlock.base128) + ).then(valid => { + if ( valid === true ) { return; } + return this.assets.get(this.pslAssetKey, details => { if ( details.content !== '' ) { this.compilePublicSuffixList(details.content); } - resolve(); }); }); - // end of executor - }); }; -/******************************************************************************/ - µBlock.compilePublicSuffixList = function(content) { publicSuffixList.parse(content, punycode.toASCII); this.assets.put( 'compiled/' + this.pslAssetKey, - JSON.stringify(publicSuffixList.toSelfie()) + publicSuffixList.toSelfie(µBlock.base128) ); }; @@ -1056,60 +1043,76 @@ // some set time. µBlock.selfieManager = (function() { - let µb = µBlock; - let timer = null; + const µb = µBlock; + let timer; // As of 2018-05-31: - // JSON.stringify-ing ourselves results in a better baseline - // memory usage at selfie-load time. For some reasons. + // JSON.stringify-ing ourselves results in a better baseline + // memory usage at selfie-load time. For some reasons. - let create = function() { - timer = null; - let selfie = JSON.stringify({ - magic: µb.systemSettings.selfieMagic, - availableFilterLists: µb.availableFilterLists, - staticNetFilteringEngine: µb.staticNetFilteringEngine.toSelfie(), - redirectEngine: µb.redirectEngine.toSelfie(), - staticExtFilteringEngine: µb.staticExtFilteringEngine.toSelfie() - }); - µb.cacheStorage.set({ selfie: selfie }); - µb.lz4Codec.relinquish(); - }; - - let load = function(callback) { - µb.cacheStorage.get('selfie', function(bin) { - if ( - bin instanceof Object === false || - typeof bin.selfie !== 'string' - ) { - return callback(false); - } - let selfie; - try { - selfie = JSON.parse(bin.selfie); - } catch(ex) { - } - if ( - selfie instanceof Object === false || - selfie.magic !== µb.systemSettings.selfieMagic - ) { - return callback(false); - } - µb.availableFilterLists = selfie.availableFilterLists; - µb.staticNetFilteringEngine.fromSelfie(selfie.staticNetFilteringEngine); - µb.redirectEngine.fromSelfie(selfie.redirectEngine); - µb.staticExtFilteringEngine.fromSelfie(selfie.staticExtFilteringEngine); - callback(true); + const create = function() { + Promise.all([ + µb.assets.put( + 'selfie/main', + JSON.stringify({ + magic: µb.systemSettings.selfieMagic, + availableFilterLists: µb.availableFilterLists, + }) + ), + µb.redirectEngine.toSelfie('selfie/redirectEngine'), + µb.staticExtFilteringEngine.toSelfie('selfie/staticExtFilteringEngine'), + µb.staticNetFilteringEngine.toSelfie('selfie/staticNetFilteringEngine'), + ]).then(( ) => { + µb.lz4Codec.relinquish(); }); }; - let destroy = function() { - if ( timer !== null ) { + const load = function() { + return Promise.all([ + µb.assets.get('selfie/main').then(details => { + if ( + details instanceof Object === false || + typeof details.content !== 'string' || + details.content === '' + ) { + return false; + } + let selfie; + try { + selfie = JSON.parse(details.content); + } catch(ex) { + } + if ( + selfie instanceof Object === false || + selfie.magic !== µb.systemSettings.selfieMagic + ) { + return false; + } + µb.availableFilterLists = selfie.availableFilterLists; + return true; + }), + µb.redirectEngine.fromSelfie('selfie/redirectEngine'), + µb.staticExtFilteringEngine.fromSelfie('selfie/staticExtFilteringEngine'), + µb.staticNetFilteringEngine.fromSelfie('selfie/staticNetFilteringEngine'), + ]).then(results => + results.reduce((acc, v) => acc && v, true) + ).catch(reason => { + log.info(reason); + return false; + }); + }; + + const destroy = function() { + if ( timer !== undefined ) { clearTimeout(timer); - timer = null; + timer = undefined; } - µb.cacheStorage.remove('selfie'); - timer = vAPI.setTimeout(create, µb.selfieAfter); + µb.cacheStorage.remove('selfie'); // TODO: obsolete, remove eventually. + µb.assets.remove(/^selfie\//); + timer = vAPI.setTimeout(( ) => { + timer = undefined; + create(); + }, µb.hiddenSettings.selfieAfter * 60000); }; return { @@ -1299,6 +1302,8 @@ // Compile the list while we have the raw version in memory if ( topic === 'after-asset-updated' ) { + // Skip selfie-related content. + if ( details.assetKey.startsWith('selfie/') ) { return; } var cached = typeof details.content === 'string' && details.content !== ''; if ( this.availableFilterLists.hasOwnProperty(details.assetKey) ) { if ( cached ) { @@ -1334,8 +1339,8 @@ cached: cached }); // https://github.com/gorhill/uBlock/issues/2585 - // Whenever an asset is overwritten, the current selfie is quite - // likely no longer valid. + // Whenever an asset is overwritten, the current selfie is quite + // likely no longer valid. this.selfieManager.destroy(); return; } diff --git a/src/js/utils.js b/src/js/utils.js index a9193e78b..dcee74c28 100644 --- a/src/js/utils.js +++ b/src/js/utils.js @@ -496,3 +496,112 @@ µBlock.orphanizeString = function(s) { return JSON.parse(JSON.stringify(s)); }; + +/******************************************************************************/ + +// Custom base128 encoder/decoder +// +// TODO: +// Could expand the LZ4 codec API to be able to return UTF8-safe string +// representation of a compressed buffer, and thus the code below could be +// moved LZ4 codec-side. + +µBlock.base128 = { + encode: function(arrbuf, arrlen) { + const inbuf = new Uint8Array(arrbuf, 0, arrlen); + const inputLength = arrlen; + let _7cnt = Math.floor(inputLength / 7); + let outputLength = _7cnt * 8; + let _7rem = inputLength % 7; + if ( _7rem !== 0 ) { + outputLength += 1 + _7rem; + } + const outbuf = new Uint8Array(outputLength); + let msbits, v; + let i = 0, j = 0; + while ( _7cnt-- ) { + v = inbuf[i+0]; + msbits = (v & 0x80) >>> 7; + outbuf[j+1] = v & 0x7F; + v = inbuf[i+1]; + msbits |= (v & 0x80) >>> 6; + outbuf[j+2] = v & 0x7F; + v = inbuf[i+2]; + msbits |= (v & 0x80) >>> 5; + outbuf[j+3] = v & 0x7F; + v = inbuf[i+3]; + msbits |= (v & 0x80) >>> 4; + outbuf[j+4] = v & 0x7F; + v = inbuf[i+4]; + msbits |= (v & 0x80) >>> 3; + outbuf[j+5] = v & 0x7F; + v = inbuf[i+5]; + msbits |= (v & 0x80) >>> 2; + outbuf[j+6] = v & 0x7F; + v = inbuf[i+6]; + msbits |= (v & 0x80) >>> 1; + outbuf[j+7] = v & 0x7F; + outbuf[j+0] = msbits; + i += 7; j += 8; + } + if ( _7rem > 0 ) { + msbits = 0; + for ( let ir = 0; ir < _7rem; ir++ ) { + v = inbuf[i+ir]; + msbits |= (v & 0x80) >>> (7 - ir); + outbuf[j+ir+1] = v & 0x7F; + } + outbuf[j+0] = msbits; + } + const textDecoder = new TextDecoder(); + return textDecoder.decode(outbuf); + }, + // TODO: + // Surprisingly, there does not seem to be any performance gain when + // first converting the input string into a Uint8Array through + // TextEncoder. Investigate again to confirm original findings and + // to find out whether results have changed. Not using TextEncoder() + // to create an intermediate input buffer lower peak memory usage + // at selfie load time. + // + // const textEncoder = new TextEncoder(); + // const inbuf = textEncoder.encode(instr); + // const inputLength = inbuf.byteLength; + decode: function(instr, arrbuf) { + const inputLength = instr.length; + let _8cnt = inputLength >>> 3; + let outputLength = _8cnt * 7; + let _8rem = inputLength % 8; + if ( _8rem !== 0 ) { + outputLength += _8rem - 1; + } + const outbuf = arrbuf instanceof ArrayBuffer === false + ? new Uint8Array(outputLength) + : new Uint8Array(arrbuf); + let msbits; + let i = 0, j = 0; + while ( _8cnt-- ) { + msbits = instr.charCodeAt(i+0); + outbuf[j+0] = msbits << 7 & 0x80 | instr.charCodeAt(i+1); + outbuf[j+1] = msbits << 6 & 0x80 | instr.charCodeAt(i+2); + outbuf[j+2] = msbits << 5 & 0x80 | instr.charCodeAt(i+3); + outbuf[j+3] = msbits << 4 & 0x80 | instr.charCodeAt(i+4); + outbuf[j+4] = msbits << 3 & 0x80 | instr.charCodeAt(i+5); + outbuf[j+5] = msbits << 2 & 0x80 | instr.charCodeAt(i+6); + outbuf[j+6] = msbits << 1 & 0x80 | instr.charCodeAt(i+7); + i += 8; j += 7; + } + if ( _8rem > 1 ) { + msbits = instr.charCodeAt(i+0); + for ( let ir = 1; ir < _8rem; ir++ ) { + outbuf[j+ir-1] = msbits << (8-ir) & 0x80 | instr.charCodeAt(i+ir); + } + } + return outbuf; + }, + decodeSize: function(instr) { + const size = (instr.length >>> 3) * 7; + const rem = instr.length & 7; + return rem === 0 ? size : size + rem - 1; + }, +};