Store regex filter pattern into bidi-trie buffer

As was done with generic pattern-based filters, the source
string of regex-based filters is now stored into the
bidi-trie (pattern) buffer.

Additionally, added a new "dev tools" page to more
conveniently peer into uBO's internals at run time, without
having to do so from the browser's dev console -- something
which has become more difficult with the use of JS modules.

The new page can be launched from the Support pane through
the "More" button in the troubleshooting section.

The benchmark button in the About pane has been moved to this
new "dev tools" page.

The new "dev tools" page is for development purpose only,
do not open issues about it.
This commit is contained in:
Raymond Hill 2021-12-12 10:32:49 -05:00
parent d6d80e60c0
commit 4d482f9133
No known key found for this signature in database
GPG Key ID: 25E1490B761470C2
12 changed files with 396 additions and 84 deletions

View File

@ -42,11 +42,6 @@
</div>
<div class="li" data-i18n="aboutCDNsInfo"></div>
</div>
<hr>
<div id="dev">
<button id="sfneBenchmark" type="button">Benchmark static filtering engine</button>
<div id="sfneBenchmarkResult"></div>
</div>
</div>
<script src="js/vapi.js"></script>

View File

@ -1,16 +1,3 @@
body {
margin-bottom: 6rem;
}
#dev {
align-items: flex-start;
display: none;
}
#dev.enabled {
display: flex;
}
#dev > * {
margin-inline-end: 1em;
}
#dev > div {
white-space: pre;
}

View File

@ -125,8 +125,13 @@ button.iconifiable > .fa-icon {
font-size: 120%;
}
body[dir="rtl"] button.iconifiable > .fa-icon {
padding-left: 0.5em;
padding-left: 0.4em;
padding-right: 0;
}
body[dir] button.iconifiable > .fa-icon:last-child {
padding-left: 0;
padding-right: 0;
}
label {
align-items: center;
display: inline-flex;

22
src/css/devtools.css Normal file
View File

@ -0,0 +1,22 @@
html {
height: 100vh;
overflow: hidden;
width: 100vw;
}
body {
display: flex;
flex-direction: column;
height: 100%;
justify-content: stretch;
overflow: hidden;
width: 100%;
}
.body {
flex-shrink: 0;
}
.codeMirrorContainer {
flex-grow: 1;
}
#console {
text-align: left;
}

49
src/devtools.html Normal file
View File

@ -0,0 +1,49 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>uBlock — Dev tools</title>
<link rel="stylesheet" href="lib/codemirror/lib/codemirror.css">
<link rel="stylesheet" href="lib/codemirror/addon/search/matchesonscrollbar.css">
<link rel="stylesheet" href="css/themes/default.css">
<link rel="stylesheet" href="css/common.css">
<link rel="stylesheet" href="css/fa-icons.css">
<link rel="stylesheet" href="css/dashboard-common.css">
<link rel="stylesheet" href="css/cloud-ui.css">
<link rel="stylesheet" href="css/devtools.css">
<link rel="stylesheet" href="css/codemirror.css">
</head>
<body>
<div class="body">
<p>
<button id="console-clear" class="iconifiable" type="button"><span class="fa-icon">trash-o</span></button>
<button id="snfe-dump" type="button">Dump SNFE</button>
<button id="snfe-benchmark" type="button" disabled>Benchmark SNFE</button>
</div>
<div id="console" class="codeMirrorContainer"></div>
<script src="lib/codemirror/lib/codemirror.js"></script>
<script src="lib/codemirror/addon/display/panel.js"></script>
<script src="lib/codemirror/addon/scroll/annotatescrollbar.js"></script>
<script src="lib/codemirror/addon/search/searchcursor.js"></script>
<script src="lib/codemirror/addon/selection/active-line.js"></script>
<script src="js/codemirror/search.js"></script>
<script src="js/codemirror/search-thread.js"></script>
<script src="js/fa-icons.js"></script>
<script src="js/vapi.js"></script>
<script src="js/vapi-common.js"></script>
<script src="js/vapi-client.js"></script>
<script src="js/udom.js"></script>
<script src="js/i18n.js"></script>
<script src="js/dashboard-common.js"></script>
<script src="js/devtools.js" type="module"></script>
</body>
</html>

View File

@ -31,21 +31,4 @@
});
uDom('#aboutNameVer').text(appData.name + ' ' + appData.version);
if ( appData.canBenchmark !== true ) { return; }
document.getElementById('dev').classList.add('enabled');
document.getElementById('sfneBenchmark').addEventListener('click', ev => {
const button = ev.target;
button.setAttribute('disabled', '');
vAPI.messaging.send('dashboard', {
what: 'sfneBenchmark',
}).then(result => {
document.getElementById('sfneBenchmarkResult').prepend(
document.createTextNode(result.trim() + '\n')
);
button.removeAttribute('disabled');
});
});
})();

View File

@ -792,6 +792,13 @@ class BidiTrieContainer {
return true;
}
dumpInfo() {
return [
`Buffer size (Uint8Array): ${this.buf32[CHAR1_SLOT].toLocaleString('en')}`,
`WASM: ${this.wasmMemory === null ? 'disabled' : 'enabled'}`,
].join('\n');
}
//--------------------------------------------------------------------------
// Private methods
//--------------------------------------------------------------------------

81
src/js/devtools.js Normal file
View File

@ -0,0 +1,81 @@
/*******************************************************************************
uBlock Origin - a browser extension to block requests.
Copyright (C) 2014-2018 Raymond Hill
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see {http://www.gnu.org/licenses/}.
Home: https://github.com/gorhill/uBlock
*/
/* global CodeMirror, uDom, uBlockDashboard */
'use strict';
/******************************************************************************/
const cmEditor = new CodeMirror(
document.getElementById('console'),
{
autofocus: true,
lineNumbers: true,
lineWrapping: true,
styleActiveLine: true,
undoDepth: 5,
}
);
uBlockDashboard.patchCodeMirrorEditor(cmEditor);
/******************************************************************************/
function log(text) {
cmEditor.replaceRange(text.trim() + '\n\n', { line: 0, ch: 0 });
}
/******************************************************************************/
uDom.nodeFromId('console-clear').addEventListener('click', ( ) => {
cmEditor.setValue('');
});
uDom.nodeFromId('snfe-dump').addEventListener('click', ev => {
const button = ev.target;
button.setAttribute('disabled', '');
vAPI.messaging.send('dashboard', {
what: 'sfneDump',
}).then(result => {
log(result);
button.removeAttribute('disabled');
});
});
vAPI.messaging.send('dashboard', {
what: 'getAppData',
}).then(appData => {
if ( appData.canBenchmark !== true ) { return; }
uDom.nodeFromId('snfe-benchmark').removeAttribute('disabled');
uDom.nodeFromId('snfe-benchmark').addEventListener('click', ev => {
const button = ev.target;
button.setAttribute('disabled', '');
vAPI.messaging.send('dashboard', {
what: 'sfneBenchmark',
}).then(result => {
log(result);
button.removeAttribute('disabled');
});
});
});
/******************************************************************************/

View File

@ -529,6 +529,13 @@ class HNTrieContainer {
return true;
}
dumpInfo() {
return [
`Buffer size (Uint8Array): ${this.buf32[CHAR1_SLOT].toLocaleString('en')}`,
`WASM: ${this.wasmMemory === null ? 'disabled' : 'enabled'}`,
].join('\n');
}
//--------------------------------------------------------------------------
// Private methods
//--------------------------------------------------------------------------

View File

@ -239,6 +239,10 @@ const onMessage = function(request, sender, callback) {
}
break;
case 'sfneDump':
response = staticNetFilteringEngine.dump();
break;
default:
return vAPI.messaging.UNHANDLED;
}

View File

@ -418,20 +418,20 @@ function filterDataFromSelfie(selfie) {
}
const filterRefs = [ null ];
let filterRefWritePtr = 1;
let filterRefsWritePtr = 1;
const filterRefAdd = function(ref) {
const i = filterRefWritePtr;
const i = filterRefsWritePtr;
filterRefs[i] = ref;
filterRefWritePtr += 1;
filterRefsWritePtr += 1;
return i;
};
function filterRefsReset() {
filterRefs.fill(null);
filterRefWritePtr = 1;
filterRefsWritePtr = 1;
}
function filterRefsToSelfie() {
const refs = [];
for ( let i = 0; i < filterRefWritePtr; i++ ) {
for ( let i = 0; i < filterRefsWritePtr; i++ ) {
const v = filterRefs[i];
if ( v instanceof RegExp ) {
refs.push({ t: 1, s: v.source, f: v.flags });
@ -475,7 +475,7 @@ function filterRefsFromSelfie(selfie) {
throw new Error('Unknown filter reference!');
}
}
filterRefWritePtr = refs.length;
filterRefsWritePtr = refs.length;
return true;
}
@ -593,13 +593,13 @@ const filterLogData = (idata, details) => {
/******************************************************************************/
const FilterTrue = class {
const FilterPatternAny = class {
static match() {
return true;
}
static compile() {
return [ FilterTrue.fid ];
return [ FilterPatternAny.fid ];
}
static fromCompiled(args) {
@ -615,7 +615,7 @@ const FilterTrue = class {
}
};
registerFilterClass(FilterTrue);
registerFilterClass(FilterPatternAny);
/******************************************************************************/
@ -709,6 +709,14 @@ const FilterPatternPlain = class {
details.regex.push('(?![0-9A-Za-z%])');
}
}
static dumpInfo(idata) {
const pattern = bidiTrie.extractString(
filterData[idata+1],
filterData[idata+2]
);
return `${pattern} ${filterData[idata+3]}`;
}
};
FilterPatternPlain.isPatternPlain = true;
@ -823,6 +831,13 @@ const FilterPatternGeneric = class {
details.regex.length = 0;
details.regex.push(restrFromGenericPattern(s, anchor & ~0b100));
}
static dumpInfo(idata) {
return bidiTrie.extractString(
filterData[idata+1],
filterData[idata+2]
);
}
};
FilterPatternGeneric.isSlow = true;
@ -1004,11 +1019,14 @@ registerFilterClass(FilterTrailingSeparator);
const FilterRegex = class {
static match(idata) {
const refs = filterRefs[filterData[idata+2]];
const refs = filterRefs[filterData[idata+4]];
if ( refs.$re === null ) {
refs.$re = new RegExp(
refs.s,
filterData[idata+1] === 0 ? '' : 'i'
bidiTrie.extractString(
filterData[idata+1],
filterData[idata+2]
),
filterData[idata+3] === 0 ? '' : 'i'
);
}
if ( refs.$re.test($requestURLRaw) === false ) { return false; }
@ -1025,13 +1043,12 @@ const FilterRegex = class {
}
static fromCompiled(args) {
const idata = filterDataAllocLen(3);
filterData[idata+0] = args[0]; // fid
filterData[idata+1] = args[2]; // match-case
filterData[idata+2] = filterRefAdd({
s: args[1],
$re: null,
});
const idata = filterDataAllocLen(5);
filterData[idata+0] = args[0]; // fid
filterData[idata+1] = bidiTrie.storeString(args[1]); // i
filterData[idata+2] = args[1].length; // n
filterData[idata+3] = args[2]; // match-case
filterData[idata+4] = filterRefAdd({ $re: null });
return idata;
}
@ -1040,14 +1057,29 @@ const FilterRegex = class {
}
static logData(idata, details) {
const refs = filterRefs[filterData[idata+2]];
details.pattern.push('/', refs.s, '/');
details.regex.push(refs.s);
const s = bidiTrie.extractString(
filterData[idata+1],
filterData[idata+2]
);
details.pattern.push('/', s, '/');
details.regex.push(s);
details.isRegex = true;
if ( filterData[idata+1] !== 0 ) {
details.options.push('match-case');
}
}
static dumpInfo(idata) {
return [
'/',
bidiTrie.extractString(
filterData[idata+1],
filterData[idata+2]
),
'/',
filterData[idata+3] === 1 ? ' (match-case)' : '',
].join('');
}
};
FilterRegex.isSlow = true;
@ -1092,6 +1124,10 @@ const FilterNotType = class {
details.options.push(`~${typeValueToTypeName[i]}`);
}
}
static dumpInfo(idata) {
return `0b${filterData[idata+1].toString(2)}`;
}
};
registerFilterClass(FilterNotType);
@ -1256,6 +1292,10 @@ const FilterOriginHit = class {
static logData(idata, details) {
details.domains.push(this.getDomainOpt(idata));
}
static dumpInfo(idata) {
return this.getDomainOpt(idata);
}
};
registerFilterClass(FilterOriginHit);
@ -1368,6 +1408,10 @@ const FilterOriginHitSet = class {
static logData(idata, details) {
details.domains.push(this.getDomainOpt(idata));
}
static dumpInfo(idata) {
return this.getDomainOpt(idata);
}
};
registerFilterClass(FilterOriginHitSet);
@ -1441,6 +1485,10 @@ const FilterOriginEntityHit = class {
static logData(idata, details) {
details.domains.push(this.getDomainOpt(idata));
}
static dumpInfo(idata) {
return this.getDomainOpt(idata);
}
};
registerFilterClass(FilterOriginEntityHit);
@ -1493,6 +1541,10 @@ const FilterOriginHitSetTest = class extends FilterOriginHitSet {
filterData[idata+3] = 0; // $lastResult
return idata;
}
static dumpInfo(idata) {
return super.dumpInfo(filterData[idata+1]);
}
};
registerFilterClass(FilterOriginHitSetTest);
@ -1546,6 +1598,13 @@ const FilterModifier = class {
}
details.options.push(opt);
}
static dumpInfo(idata) {
const s = StaticFilteringParser.netOptionTokenNames.get(filterData[idata+2]);
const refs = filterRefs[filterData[idata+3]];
if ( refs.value === '' ) { return s; }
return `${s}=${refs.value}`;
}
};
registerFilterClass(FilterModifier);
@ -1657,6 +1716,10 @@ const FilterCollection = class {
filterLogData(iunit, details);
});
}
static dumpInfo(idata) {
return this.getCount(idata);
}
};
registerFilterClass(FilterCollection);
@ -1829,6 +1892,10 @@ const FilterHostnameDict = class {
restrSeparator
);
}
static dumpInfo(idata) {
return this.getCount(idata);
}
};
registerFilterClass(FilterHostnameDict);
@ -1866,6 +1933,10 @@ const FilterDenyAllow = class {
static logData(idata, details) {
details.denyallow.push(filterRefs[filterData[idata+2]]);
}
static dumpInfo(idata) {
return filterRefs[filterData[idata+2]];
}
};
registerFilterClass(FilterDenyAllow);
@ -1910,6 +1981,10 @@ const FilterJustOrigin = class {
details.regex.push('^');
details.domains.push(filterRefs[filterData[idata+2]]);
}
static dumpInfo(idata) {
return this.getCount(idata);
}
};
registerFilterClass(FilterJustOrigin);
@ -2017,6 +2092,10 @@ const FilterPlainTrie = class {
filterLogData(filterData[idata+2], details);
}
}
static dumpInfo(idata) {
return `${Array.from(bidiTrie.trieIterator(filterData[idata+1])).length}`;
}
};
registerFilterClass(FilterPlainTrie);
@ -2028,6 +2107,10 @@ const FilterBucket = class extends FilterCollection {
return filterData[idata+2];
}
static forEach(idata, fn) {
return super.forEach(filterData[idata+1], fn);
}
static match(idata) {
const icollection = filterData[idata+1];
let iseq = filterData[icollection+1];
@ -2168,6 +2251,10 @@ const FilterBucket = class extends FilterCollection {
const ioriginhitset = FilterOriginHitSetTest.create(domainOpts.join('|'));
return FilterBucketOfOriginHits.create(ioriginhitset, idesbucket);
}
static dumpInfo(idata) {
return this.getCount(idata);
}
};
registerFilterClass(FilterBucket);
@ -3247,7 +3334,7 @@ class FilterCompiler {
return;
}
if ( this.pattern === '*' ) {
units.push(FilterTrue.compile());
units.push(FilterPatternAny.compile());
return;
}
if ( this.tokenHash === NO_TOKEN_HASH ) {
@ -3321,7 +3408,7 @@ FilterCompiler.prototype.FILTER_UNSUPPORTED = 2;
const FilterContainer = function() {
this.compilerVersion = '5';
this.selfieVersion = '5';
this.selfieVersion = '6';
this.MAX_TOKEN_LENGTH = MAX_TOKEN_LENGTH;
this.optimizeTaskId = undefined;
@ -3472,8 +3559,6 @@ FilterContainer.prototype.freeze = function() {
this.goodFilters.clear();
filterArgsToUnit.clear();
//this.filterClassHistogram();
// Optimizing is not critical for the static network filtering engine to
// work properly, so defer this until later to allow for reduced delay to
// readiness when no valid selfie is available.
@ -3566,8 +3651,6 @@ FilterContainer.prototype.optimize = function(throttle = 0) {
);
bidiTrieOptimize();
filterDataShrink();
//this.filterClassHistogram();
};
/******************************************************************************/
@ -4388,28 +4471,115 @@ FilterContainer.prototype.bucketHistogram = function() {
/******************************************************************************/
FilterContainer.prototype.filterClassHistogram = function() {
const filterClassDetails = new Map();
for ( const fclass of filterClasses ) {
filterClassDetails.set(fclass.fid, { name: fclass.name, count: 0, });
}
const countFilter = idata => {
const fc = filterGetClass(idata);
filterClassDetails.get(fc.fid).count += 1;
if ( fc.forEach === undefined ) { return; }
fc.forEach(idata, iunit => { countFilter(iunit); });
// Dump the internal state of the filtering engine to the console.
// Useful to make development decisions and investigate issues.
FilterContainer.prototype.dump = function() {
const thConstants = new Map([
[ NO_TOKEN_HASH, 'NO_TOKEN_HASH' ],
[ DOT_TOKEN_HASH, 'DOT_TOKEN_HASH' ],
[ ANY_TOKEN_HASH, 'ANY_TOKEN_HASH' ],
[ ANY_HTTPS_TOKEN_HASH, 'ANY_HTTPS_TOKEN_HASH' ],
[ ANY_HTTP_TOKEN_HASH, 'ANY_HTTP_TOKEN_HASH' ],
[ EMPTY_TOKEN_HASH, 'EMPTY_TOKEN_HASH' ],
]);
const dumpInfo = (idata, options) => {
const fc = filterClasses[filterData[idata+0]];
if ( fc.dumpInfo === undefined ) { return; }
return fc.dumpInfo(idata, options);
};
for ( let bits = 0; bits < this.bitsToBucketIndices.length; bits++ ) {
const ibucket = this.bitsToBucketIndices[bits];
if ( ibucket === 0 ) { continue; }
for ( const iunit of this.buckets[ibucket].values() ) {
countFilter(iunit);
const out = [];
const toOutput = (depth, line) => {
out.push(`${' '.repeat(depth*2)}${line}`);
};
// TODO: Also report filters "hidden" behind FilterPlainTrie
const dumpUnit = (idata, out, depth = 0) => {
const fc = filterGetClass(idata);
fcCounts.set(fc.name, (fcCounts.get(fc.name) || 0) + 1);
const info = dumpInfo(idata) || '';
toOutput(depth, info !== '' ? `${fc.name}: ${info}` : fc.name);
switch ( fc ) {
case FilterBucket:
case FilterCompositeAll:
case FilterOriginHitAny: {
fc.forEach(idata, i => {
dumpUnit(i, out, depth+1);
});
break;
}
case FilterBucketOfOriginHits: {
dumpUnit(filterData[idata+1], out, depth+1);
dumpUnit(filterData[idata+2], out, depth+1);
break;
}
default:
break;
}
};
const fcCounts = new Map();
const thCounts = new Set();
const realms = new Map([
[ BlockAction, 'block' ],
[ BlockImportant, 'block-important' ],
[ AllowAction, 'allow' ],
[ ModifyAction, 'modify' ],
]);
const partyness = new Map([
[ AnyParty, 'any-party' ],
[ FirstParty, '1st-party' ],
[ ThirdParty, '3rd-party' ],
]);
for ( const [ realmBits, realmName ] of realms ) {
toOutput(0, `realm: ${realmName}`);
for ( const [ partyBits, partyName ] of partyness ) {
toOutput(1, `party: ${partyName}`);
for ( const typeName in typeNameToTypeValue ) {
const bits = realmBits | partyBits | typeNameToTypeValue[typeName];
const ibucket = this.bitsToBucketIndices[bits];
if ( ibucket === 0 ) { continue; }
toOutput(2, `type: ${typeName}`);
for ( const [ th, iunit ] of this.buckets[ibucket] ) {
thCounts.add(th);
const ths = thConstants.has(th)
? thConstants.get(th)
: `0x${th.toString(16)}`;
toOutput(3, `th: ${ths}`);
dumpUnit(iunit, out, 4);
}
}
}
}
const results = Array.from(filterClassDetails.values()).sort((a, b) => {
return b.count - a.count;
});
console.info(results);
const knownTokens =
urlTokenizer.knownTokens
.reduce((a, b) => b !== 0 ? a+1 : a, 0);
out.unshift([
'Static Network Filtering Engine internals:',
`Distinct token hashes: ${thCounts.size.toLocaleString('en')}`,
`Known-token sieve (Uint8Array): ${knownTokens.toLocaleString('en')} out of 65,536`,
`Filter data (Int32Array): ${filterDataWritePtr.toLocaleString('en')}`,
`Filter refs (JS array): ${filterRefsWritePtr.toLocaleString('en')}`,
'Origin trie container:',
origHNTrieContainer.dumpInfo().split('\n').map(a => ` ${a}`).join('\n'),
'Request trie container:',
destHNTrieContainer.dumpInfo().split('\n').map(a => ` ${a}`).join('\n'),
'Pattern trie container:',
bidiTrie.dumpInfo().split('\n').map(a => ` ${a}`).join('\n'),
'Filter class stats:',
Array.from(fcCounts)
.sort((a, b) => b[1] - a[1])
.map(a => ` ${a[0]}: ${a[1].toLocaleString('en')}`)
.join('\n'),
'Filter tree:',
].join('\n'));
return out.join('\n');
};
/******************************************************************************/

View File

@ -96,7 +96,9 @@
<div class="a b c d">
<h3 data-i18n="supportS5H"></h3>
<p data-i18n="supportS5P1">
<p><button id="selectAllButton" type="button" data-i18n="genericSelectAll"></button>
<p>
<button id="selectAllButton" type="button" data-i18n="genericSelectAll"></button>
<button id="moreButton" type="button" data-i18n="popupMoreButton_v2" data-url="/devtools.html"></button>
<div id="supportData" class="codeMirrorContainer"></div>
</div>
</div>