From 4d482f91336f5273b69301ee13c73fe4490021ef Mon Sep 17 00:00:00 2001 From: Raymond Hill Date: Sun, 12 Dec 2021 10:32:49 -0500 Subject: [PATCH] Store regex filter pattern into bidi-trie buffer As was done with generic pattern-based filters, the source string of regex-based filters is now stored into the bidi-trie (pattern) buffer. Additionally, added a new "dev tools" page to more conveniently peer into uBO's internals at run time, without having to do so from the browser's dev console -- something which has become more difficult with the use of JS modules. The new page can be launched from the Support pane through the "More" button in the troubleshooting section. The benchmark button in the About pane has been moved to this new "dev tools" page. The new "dev tools" page is for development purpose only, do not open issues about it. --- src/about.html | 5 - src/css/about.css | 13 -- src/css/common.css | 7 +- src/css/devtools.css | 22 +++ src/devtools.html | 49 ++++++ src/js/about.js | 17 --- src/js/biditrie.js | 7 + src/js/devtools.js | 81 ++++++++++ src/js/hntrie.js | 7 + src/js/messaging.js | 4 + src/js/static-net-filtering.js | 264 +++++++++++++++++++++++++++------ src/support.html | 4 +- 12 files changed, 396 insertions(+), 84 deletions(-) create mode 100644 src/css/devtools.css create mode 100644 src/devtools.html create mode 100644 src/js/devtools.js diff --git a/src/about.html b/src/about.html index f852e5324..27e22b6b3 100644 --- a/src/about.html +++ b/src/about.html @@ -42,11 +42,6 @@
-
-
- -
-
diff --git a/src/css/about.css b/src/css/about.css index d731dbe96..8c3afcdc9 100644 --- a/src/css/about.css +++ b/src/css/about.css @@ -1,16 +1,3 @@ body { margin-bottom: 6rem; } -#dev { - align-items: flex-start; - display: none; - } -#dev.enabled { - display: flex; - } -#dev > * { - margin-inline-end: 1em; - } -#dev > div { - white-space: pre; - } diff --git a/src/css/common.css b/src/css/common.css index be3767653..4ff179059 100644 --- a/src/css/common.css +++ b/src/css/common.css @@ -125,8 +125,13 @@ button.iconifiable > .fa-icon { font-size: 120%; } body[dir="rtl"] button.iconifiable > .fa-icon { - padding-left: 0.5em; + padding-left: 0.4em; + padding-right: 0; } +body[dir] button.iconifiable > .fa-icon:last-child { + padding-left: 0; + padding-right: 0; +} label { align-items: center; display: inline-flex; diff --git a/src/css/devtools.css b/src/css/devtools.css new file mode 100644 index 000000000..425aac475 --- /dev/null +++ b/src/css/devtools.css @@ -0,0 +1,22 @@ +html { + height: 100vh; + overflow: hidden; + width: 100vw; + } +body { + display: flex; + flex-direction: column; + height: 100%; + justify-content: stretch; + overflow: hidden; + width: 100%; + } +.body { + flex-shrink: 0; + } +.codeMirrorContainer { + flex-grow: 1; + } +#console { + text-align: left; + } diff --git a/src/devtools.html b/src/devtools.html new file mode 100644 index 000000000..50a793829 --- /dev/null +++ b/src/devtools.html @@ -0,0 +1,49 @@ + + + + + +uBlock — Dev tools + + + + + + + + + + + + + + + +
+

+ + + +

+
+ + + + + + + + + + + + + + + + + + + + + diff --git a/src/js/about.js b/src/js/about.js index 6317d09f8..153794577 100644 --- a/src/js/about.js +++ b/src/js/about.js @@ -31,21 +31,4 @@ }); uDom('#aboutNameVer').text(appData.name + ' ' + appData.version); - - if ( appData.canBenchmark !== true ) { return; } - - document.getElementById('dev').classList.add('enabled'); - - document.getElementById('sfneBenchmark').addEventListener('click', ev => { - const button = ev.target; - button.setAttribute('disabled', ''); - vAPI.messaging.send('dashboard', { - what: 'sfneBenchmark', - }).then(result => { - document.getElementById('sfneBenchmarkResult').prepend( - document.createTextNode(result.trim() + '\n') - ); - button.removeAttribute('disabled'); - }); - }); })(); diff --git a/src/js/biditrie.js b/src/js/biditrie.js index 54362008a..38b780faf 100644 --- a/src/js/biditrie.js +++ b/src/js/biditrie.js @@ -792,6 +792,13 @@ class BidiTrieContainer { return true; } + dumpInfo() { + return [ + `Buffer size (Uint8Array): ${this.buf32[CHAR1_SLOT].toLocaleString('en')}`, + `WASM: ${this.wasmMemory === null ? 'disabled' : 'enabled'}`, + ].join('\n'); + } + //-------------------------------------------------------------------------- // Private methods //-------------------------------------------------------------------------- diff --git a/src/js/devtools.js b/src/js/devtools.js new file mode 100644 index 000000000..b3eccb43c --- /dev/null +++ b/src/js/devtools.js @@ -0,0 +1,81 @@ +/******************************************************************************* + + uBlock Origin - a browser extension to block requests. + Copyright (C) 2014-2018 Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +/* global CodeMirror, uDom, uBlockDashboard */ + +'use strict'; + +/******************************************************************************/ + +const cmEditor = new CodeMirror( + document.getElementById('console'), + { + autofocus: true, + lineNumbers: true, + lineWrapping: true, + styleActiveLine: true, + undoDepth: 5, + } +); + +uBlockDashboard.patchCodeMirrorEditor(cmEditor); + +/******************************************************************************/ + +function log(text) { + cmEditor.replaceRange(text.trim() + '\n\n', { line: 0, ch: 0 }); +} + +/******************************************************************************/ + +uDom.nodeFromId('console-clear').addEventListener('click', ( ) => { + cmEditor.setValue(''); +}); + +uDom.nodeFromId('snfe-dump').addEventListener('click', ev => { + const button = ev.target; + button.setAttribute('disabled', ''); + vAPI.messaging.send('dashboard', { + what: 'sfneDump', + }).then(result => { + log(result); + button.removeAttribute('disabled'); + }); +}); + +vAPI.messaging.send('dashboard', { + what: 'getAppData', +}).then(appData => { + if ( appData.canBenchmark !== true ) { return; } + uDom.nodeFromId('snfe-benchmark').removeAttribute('disabled'); + uDom.nodeFromId('snfe-benchmark').addEventListener('click', ev => { + const button = ev.target; + button.setAttribute('disabled', ''); + vAPI.messaging.send('dashboard', { + what: 'sfneBenchmark', + }).then(result => { + log(result); + button.removeAttribute('disabled'); + }); + }); +}); + +/******************************************************************************/ diff --git a/src/js/hntrie.js b/src/js/hntrie.js index 576f3b927..e83dfe2d9 100644 --- a/src/js/hntrie.js +++ b/src/js/hntrie.js @@ -529,6 +529,13 @@ class HNTrieContainer { return true; } + dumpInfo() { + return [ + `Buffer size (Uint8Array): ${this.buf32[CHAR1_SLOT].toLocaleString('en')}`, + `WASM: ${this.wasmMemory === null ? 'disabled' : 'enabled'}`, + ].join('\n'); + } + //-------------------------------------------------------------------------- // Private methods //-------------------------------------------------------------------------- diff --git a/src/js/messaging.js b/src/js/messaging.js index 5d56774b4..33dab03c2 100644 --- a/src/js/messaging.js +++ b/src/js/messaging.js @@ -239,6 +239,10 @@ const onMessage = function(request, sender, callback) { } break; + case 'sfneDump': + response = staticNetFilteringEngine.dump(); + break; + default: return vAPI.messaging.UNHANDLED; } diff --git a/src/js/static-net-filtering.js b/src/js/static-net-filtering.js index 17e7fc9fd..a1c4be5fb 100644 --- a/src/js/static-net-filtering.js +++ b/src/js/static-net-filtering.js @@ -418,20 +418,20 @@ function filterDataFromSelfie(selfie) { } const filterRefs = [ null ]; -let filterRefWritePtr = 1; +let filterRefsWritePtr = 1; const filterRefAdd = function(ref) { - const i = filterRefWritePtr; + const i = filterRefsWritePtr; filterRefs[i] = ref; - filterRefWritePtr += 1; + filterRefsWritePtr += 1; return i; }; function filterRefsReset() { filterRefs.fill(null); - filterRefWritePtr = 1; + filterRefsWritePtr = 1; } function filterRefsToSelfie() { const refs = []; - for ( let i = 0; i < filterRefWritePtr; i++ ) { + for ( let i = 0; i < filterRefsWritePtr; i++ ) { const v = filterRefs[i]; if ( v instanceof RegExp ) { refs.push({ t: 1, s: v.source, f: v.flags }); @@ -475,7 +475,7 @@ function filterRefsFromSelfie(selfie) { throw new Error('Unknown filter reference!'); } } - filterRefWritePtr = refs.length; + filterRefsWritePtr = refs.length; return true; } @@ -593,13 +593,13 @@ const filterLogData = (idata, details) => { /******************************************************************************/ -const FilterTrue = class { +const FilterPatternAny = class { static match() { return true; } static compile() { - return [ FilterTrue.fid ]; + return [ FilterPatternAny.fid ]; } static fromCompiled(args) { @@ -615,7 +615,7 @@ const FilterTrue = class { } }; -registerFilterClass(FilterTrue); +registerFilterClass(FilterPatternAny); /******************************************************************************/ @@ -709,6 +709,14 @@ const FilterPatternPlain = class { details.regex.push('(?![0-9A-Za-z%])'); } } + + static dumpInfo(idata) { + const pattern = bidiTrie.extractString( + filterData[idata+1], + filterData[idata+2] + ); + return `${pattern} ${filterData[idata+3]}`; + } }; FilterPatternPlain.isPatternPlain = true; @@ -823,6 +831,13 @@ const FilterPatternGeneric = class { details.regex.length = 0; details.regex.push(restrFromGenericPattern(s, anchor & ~0b100)); } + + static dumpInfo(idata) { + return bidiTrie.extractString( + filterData[idata+1], + filterData[idata+2] + ); + } }; FilterPatternGeneric.isSlow = true; @@ -1004,11 +1019,14 @@ registerFilterClass(FilterTrailingSeparator); const FilterRegex = class { static match(idata) { - const refs = filterRefs[filterData[idata+2]]; + const refs = filterRefs[filterData[idata+4]]; if ( refs.$re === null ) { refs.$re = new RegExp( - refs.s, - filterData[idata+1] === 0 ? '' : 'i' + bidiTrie.extractString( + filterData[idata+1], + filterData[idata+2] + ), + filterData[idata+3] === 0 ? '' : 'i' ); } if ( refs.$re.test($requestURLRaw) === false ) { return false; } @@ -1025,13 +1043,12 @@ const FilterRegex = class { } static fromCompiled(args) { - const idata = filterDataAllocLen(3); - filterData[idata+0] = args[0]; // fid - filterData[idata+1] = args[2]; // match-case - filterData[idata+2] = filterRefAdd({ - s: args[1], - $re: null, - }); + const idata = filterDataAllocLen(5); + filterData[idata+0] = args[0]; // fid + filterData[idata+1] = bidiTrie.storeString(args[1]); // i + filterData[idata+2] = args[1].length; // n + filterData[idata+3] = args[2]; // match-case + filterData[idata+4] = filterRefAdd({ $re: null }); return idata; } @@ -1040,14 +1057,29 @@ const FilterRegex = class { } static logData(idata, details) { - const refs = filterRefs[filterData[idata+2]]; - details.pattern.push('/', refs.s, '/'); - details.regex.push(refs.s); + const s = bidiTrie.extractString( + filterData[idata+1], + filterData[idata+2] + ); + details.pattern.push('/', s, '/'); + details.regex.push(s); details.isRegex = true; if ( filterData[idata+1] !== 0 ) { details.options.push('match-case'); } } + + static dumpInfo(idata) { + return [ + '/', + bidiTrie.extractString( + filterData[idata+1], + filterData[idata+2] + ), + '/', + filterData[idata+3] === 1 ? ' (match-case)' : '', + ].join(''); + } }; FilterRegex.isSlow = true; @@ -1092,6 +1124,10 @@ const FilterNotType = class { details.options.push(`~${typeValueToTypeName[i]}`); } } + + static dumpInfo(idata) { + return `0b${filterData[idata+1].toString(2)}`; + } }; registerFilterClass(FilterNotType); @@ -1256,6 +1292,10 @@ const FilterOriginHit = class { static logData(idata, details) { details.domains.push(this.getDomainOpt(idata)); } + + static dumpInfo(idata) { + return this.getDomainOpt(idata); + } }; registerFilterClass(FilterOriginHit); @@ -1368,6 +1408,10 @@ const FilterOriginHitSet = class { static logData(idata, details) { details.domains.push(this.getDomainOpt(idata)); } + + static dumpInfo(idata) { + return this.getDomainOpt(idata); + } }; registerFilterClass(FilterOriginHitSet); @@ -1441,6 +1485,10 @@ const FilterOriginEntityHit = class { static logData(idata, details) { details.domains.push(this.getDomainOpt(idata)); } + + static dumpInfo(idata) { + return this.getDomainOpt(idata); + } }; registerFilterClass(FilterOriginEntityHit); @@ -1493,6 +1541,10 @@ const FilterOriginHitSetTest = class extends FilterOriginHitSet { filterData[idata+3] = 0; // $lastResult return idata; } + + static dumpInfo(idata) { + return super.dumpInfo(filterData[idata+1]); + } }; registerFilterClass(FilterOriginHitSetTest); @@ -1546,6 +1598,13 @@ const FilterModifier = class { } details.options.push(opt); } + + static dumpInfo(idata) { + const s = StaticFilteringParser.netOptionTokenNames.get(filterData[idata+2]); + const refs = filterRefs[filterData[idata+3]]; + if ( refs.value === '' ) { return s; } + return `${s}=${refs.value}`; + } }; registerFilterClass(FilterModifier); @@ -1657,6 +1716,10 @@ const FilterCollection = class { filterLogData(iunit, details); }); } + + static dumpInfo(idata) { + return this.getCount(idata); + } }; registerFilterClass(FilterCollection); @@ -1829,6 +1892,10 @@ const FilterHostnameDict = class { restrSeparator ); } + + static dumpInfo(idata) { + return this.getCount(idata); + } }; registerFilterClass(FilterHostnameDict); @@ -1866,6 +1933,10 @@ const FilterDenyAllow = class { static logData(idata, details) { details.denyallow.push(filterRefs[filterData[idata+2]]); } + + static dumpInfo(idata) { + return filterRefs[filterData[idata+2]]; + } }; registerFilterClass(FilterDenyAllow); @@ -1910,6 +1981,10 @@ const FilterJustOrigin = class { details.regex.push('^'); details.domains.push(filterRefs[filterData[idata+2]]); } + + static dumpInfo(idata) { + return this.getCount(idata); + } }; registerFilterClass(FilterJustOrigin); @@ -2017,6 +2092,10 @@ const FilterPlainTrie = class { filterLogData(filterData[idata+2], details); } } + + static dumpInfo(idata) { + return `${Array.from(bidiTrie.trieIterator(filterData[idata+1])).length}`; + } }; registerFilterClass(FilterPlainTrie); @@ -2028,6 +2107,10 @@ const FilterBucket = class extends FilterCollection { return filterData[idata+2]; } + static forEach(idata, fn) { + return super.forEach(filterData[idata+1], fn); + } + static match(idata) { const icollection = filterData[idata+1]; let iseq = filterData[icollection+1]; @@ -2168,6 +2251,10 @@ const FilterBucket = class extends FilterCollection { const ioriginhitset = FilterOriginHitSetTest.create(domainOpts.join('|')); return FilterBucketOfOriginHits.create(ioriginhitset, idesbucket); } + + static dumpInfo(idata) { + return this.getCount(idata); + } }; registerFilterClass(FilterBucket); @@ -3247,7 +3334,7 @@ class FilterCompiler { return; } if ( this.pattern === '*' ) { - units.push(FilterTrue.compile()); + units.push(FilterPatternAny.compile()); return; } if ( this.tokenHash === NO_TOKEN_HASH ) { @@ -3321,7 +3408,7 @@ FilterCompiler.prototype.FILTER_UNSUPPORTED = 2; const FilterContainer = function() { this.compilerVersion = '5'; - this.selfieVersion = '5'; + this.selfieVersion = '6'; this.MAX_TOKEN_LENGTH = MAX_TOKEN_LENGTH; this.optimizeTaskId = undefined; @@ -3472,8 +3559,6 @@ FilterContainer.prototype.freeze = function() { this.goodFilters.clear(); filterArgsToUnit.clear(); - //this.filterClassHistogram(); - // Optimizing is not critical for the static network filtering engine to // work properly, so defer this until later to allow for reduced delay to // readiness when no valid selfie is available. @@ -3566,8 +3651,6 @@ FilterContainer.prototype.optimize = function(throttle = 0) { ); bidiTrieOptimize(); filterDataShrink(); - - //this.filterClassHistogram(); }; /******************************************************************************/ @@ -4388,28 +4471,115 @@ FilterContainer.prototype.bucketHistogram = function() { /******************************************************************************/ -FilterContainer.prototype.filterClassHistogram = function() { - const filterClassDetails = new Map(); - for ( const fclass of filterClasses ) { - filterClassDetails.set(fclass.fid, { name: fclass.name, count: 0, }); - } - const countFilter = idata => { - const fc = filterGetClass(idata); - filterClassDetails.get(fc.fid).count += 1; - if ( fc.forEach === undefined ) { return; } - fc.forEach(idata, iunit => { countFilter(iunit); }); +// Dump the internal state of the filtering engine to the console. +// Useful to make development decisions and investigate issues. + +FilterContainer.prototype.dump = function() { + const thConstants = new Map([ + [ NO_TOKEN_HASH, 'NO_TOKEN_HASH' ], + [ DOT_TOKEN_HASH, 'DOT_TOKEN_HASH' ], + [ ANY_TOKEN_HASH, 'ANY_TOKEN_HASH' ], + [ ANY_HTTPS_TOKEN_HASH, 'ANY_HTTPS_TOKEN_HASH' ], + [ ANY_HTTP_TOKEN_HASH, 'ANY_HTTP_TOKEN_HASH' ], + [ EMPTY_TOKEN_HASH, 'EMPTY_TOKEN_HASH' ], + ]); + + const dumpInfo = (idata, options) => { + const fc = filterClasses[filterData[idata+0]]; + if ( fc.dumpInfo === undefined ) { return; } + return fc.dumpInfo(idata, options); }; - for ( let bits = 0; bits < this.bitsToBucketIndices.length; bits++ ) { - const ibucket = this.bitsToBucketIndices[bits]; - if ( ibucket === 0 ) { continue; } - for ( const iunit of this.buckets[ibucket].values() ) { - countFilter(iunit); + + const out = []; + + const toOutput = (depth, line) => { + out.push(`${' '.repeat(depth*2)}${line}`); + }; + + // TODO: Also report filters "hidden" behind FilterPlainTrie + const dumpUnit = (idata, out, depth = 0) => { + const fc = filterGetClass(idata); + fcCounts.set(fc.name, (fcCounts.get(fc.name) || 0) + 1); + const info = dumpInfo(idata) || ''; + toOutput(depth, info !== '' ? `${fc.name}: ${info}` : fc.name); + switch ( fc ) { + case FilterBucket: + case FilterCompositeAll: + case FilterOriginHitAny: { + fc.forEach(idata, i => { + dumpUnit(i, out, depth+1); + }); + break; + } + case FilterBucketOfOriginHits: { + dumpUnit(filterData[idata+1], out, depth+1); + dumpUnit(filterData[idata+2], out, depth+1); + break; + } + default: + break; + } + }; + + const fcCounts = new Map(); + const thCounts = new Set(); + + const realms = new Map([ + [ BlockAction, 'block' ], + [ BlockImportant, 'block-important' ], + [ AllowAction, 'allow' ], + [ ModifyAction, 'modify' ], + ]); + const partyness = new Map([ + [ AnyParty, 'any-party' ], + [ FirstParty, '1st-party' ], + [ ThirdParty, '3rd-party' ], + ]); + for ( const [ realmBits, realmName ] of realms ) { + toOutput(0, `realm: ${realmName}`); + for ( const [ partyBits, partyName ] of partyness ) { + toOutput(1, `party: ${partyName}`); + for ( const typeName in typeNameToTypeValue ) { + const bits = realmBits | partyBits | typeNameToTypeValue[typeName]; + const ibucket = this.bitsToBucketIndices[bits]; + if ( ibucket === 0 ) { continue; } + toOutput(2, `type: ${typeName}`); + for ( const [ th, iunit ] of this.buckets[ibucket] ) { + thCounts.add(th); + const ths = thConstants.has(th) + ? thConstants.get(th) + : `0x${th.toString(16)}`; + toOutput(3, `th: ${ths}`); + dumpUnit(iunit, out, 4); + } + } } } - const results = Array.from(filterClassDetails.values()).sort((a, b) => { - return b.count - a.count; - }); - console.info(results); + + const knownTokens = + urlTokenizer.knownTokens + .reduce((a, b) => b !== 0 ? a+1 : a, 0); + + out.unshift([ + 'Static Network Filtering Engine internals:', + `Distinct token hashes: ${thCounts.size.toLocaleString('en')}`, + `Known-token sieve (Uint8Array): ${knownTokens.toLocaleString('en')} out of 65,536`, + `Filter data (Int32Array): ${filterDataWritePtr.toLocaleString('en')}`, + `Filter refs (JS array): ${filterRefsWritePtr.toLocaleString('en')}`, + 'Origin trie container:', + origHNTrieContainer.dumpInfo().split('\n').map(a => ` ${a}`).join('\n'), + 'Request trie container:', + destHNTrieContainer.dumpInfo().split('\n').map(a => ` ${a}`).join('\n'), + 'Pattern trie container:', + bidiTrie.dumpInfo().split('\n').map(a => ` ${a}`).join('\n'), + 'Filter class stats:', + Array.from(fcCounts) + .sort((a, b) => b[1] - a[1]) + .map(a => ` ${a[0]}: ${a[1].toLocaleString('en')}`) + .join('\n'), + 'Filter tree:', + ].join('\n')); + return out.join('\n'); }; /******************************************************************************/ diff --git a/src/support.html b/src/support.html index fcdee784d..dc2cc7366 100644 --- a/src/support.html +++ b/src/support.html @@ -96,7 +96,9 @@

-

+

+ +