Mirrors
/
uBlock
mirror of https://github.com/gorhill/uBlock.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Refactoring work in static network filtering engine 

The original motivation is to further speed up launch time
for either non-selfie-based and selfie-based initialization
of the static network filtering engine (SNFE).

As a result of the refactoring:

Filters are no longer instance-based, they are sequence-of-
integer-based. This eliminates the need to create instances
of filters at launch, and consequently eliminates all the
calls to class constructors, the resulting churning of memory,
and so forth.

All the properties defining filter instances are now as much
as possible 32-bit integer-based, and these are allocated in a
single module-scoped typed array -- this eliminates the need
to allocate memory for every filter being instantiated.

Not all filter properties can be represented as a 32-bit
integer, and in this case a filter class can allocate slots
into another module-scoped array of references.

As a result, this eliminates a lot of memory allocations when
the SNFE is populated with filters, and this makes the saving
and loading of selfie more straightforward, as the operation
is reduced to saving/loading two arrays, one of 32-bit
integers, and the other, much smaller, an array JSON-able
values.

All filter classes now only contain static methods, and all
of these methods are called with an index to the specific
filter data in the module-scoped array of 32-bit integers.

The filter sequences (used to avoid the use of JS arrays) are
also allocated in the single module-scoped array of 32-bit
integers -- they used to be stored in their own dedicated
array.

Additionally, some filters are now loaded more in a deferred
way, so as reduce uBO's time-to-readiness -- the outcome of
this still needs to be evaluated, time-to-readiness is
especially a concern in Firefox for Android or less powerful
computers.
This commit is contained in:
Raymond Hill 2021-12-04 11:16:44 -05:00
parent 64f427d0e5
commit 725e6931f5
No known key found for this signature in database
GPG Key ID: 25E1490B761470C2
6 changed files with 1477 additions and 1763 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
src/js/background.js
View File

@ -46,8 +46,8 @@ const hiddenSettingsDefault = {
allowGenericProceduralFilters: false,
assetFetchTimeout: 30,
autoCommentFilterTemplate: '{{date}} {{origin}}',
autoUpdateAssetFetchPeriod: 120,
autoUpdateDelayAfterLaunch: 180,
autoUpdateAssetFetchPeriod: 60,
autoUpdateDelayAfterLaunch: 105,
autoUpdatePeriod: 4,
benchmarkDatasetURL: 'unset',
blockingProfiles: '11111/#F00 11010/#C0F 11001/#00F 00001',
@ -78,7 +78,7 @@ const hiddenSettingsDefault = {
popupPanelLockedSections: 0,
popupPanelHeightMode: 0,
requestJournalProcessPeriod: 1000,
selfieAfter: 3,
selfieAfter: 2,
strictBlockingBypassDuration: 120,
suspendTabsUntilReady: 'unset',
uiPopupConfig: 'unset',
@ -175,8 +175,8 @@ const µBlock = { // jshint ignore:line
// Read-only
systemSettings: {
compiledMagic: 39, // Increase when compiled format changes
selfieMagic: 39, // Increase when selfie format changes
compiledMagic: 40, // Increase when compiled format changes
selfieMagic: 40, // Increase when selfie format changes
},
// https://github.com/uBlockOrigin/uBlock-issues/issues/759#issuecomment-546654501

204
src/js/biditrie.js
View File

@ -125,7 +125,7 @@ const toSegmentInfo = (aL, l, r) => ((r - l) << 24) | (aL + l);
const roundToPageSize = v => (v + PAGE_SIZE-1) & ~(PAGE_SIZE-1);
const BidiTrieContainer = class {
class BidiTrieContainer {
constructor(extraHandler) {
const len = PAGE_SIZE * 4;
@ -177,6 +177,19 @@ const BidiTrieContainer = class {
this.lastStoredLen = this.lastStoredIndex = 0;
}
createTrie() {
// grow buffer if needed
if ( (this.buf32[CHAR0_SLOT] - this.buf32[TRIE1_SLOT]) < CELL_BYTE_LENGTH ) {
this.growBuf(CELL_BYTE_LENGTH, 0);
}
const iroot = this.buf32[TRIE1_SLOT] >>> 2;
this.buf32[TRIE1_SLOT] += CELL_BYTE_LENGTH;
this.buf32[iroot+CELL_OR] = 0;
this.buf32[iroot+CELL_AND] = 0;
this.buf32[iroot+SEGMENT_INFO] = 0;
return iroot;
}
matches(icell, ai) {
const buf32 = this.buf32;
const buf8 = this.buf8;
@ -284,25 +297,9 @@ const BidiTrieContainer = class {
return 1;
}
createOne(args) {
if ( Array.isArray(args) ) {
return new this.STrieRef(this, args[0], args[1]);
}
// grow buffer if needed
if ( (this.buf32[CHAR0_SLOT] - this.buf32[TRIE1_SLOT]) < CELL_BYTE_LENGTH ) {
this.growBuf(CELL_BYTE_LENGTH, 0);
}
const iroot = this.buf32[TRIE1_SLOT] >>> 2;
this.buf32[TRIE1_SLOT] += CELL_BYTE_LENGTH;
this.buf32[iroot+CELL_OR] = 0;
this.buf32[iroot+CELL_AND] = 0;
this.buf32[iroot+SEGMENT_INFO] = 0;
return new this.STrieRef(this, iroot, 0);
}
compileOne(trieRef) {
return [ trieRef.iroot, trieRef.size ];
}
get $l() { return this.buf32[RESULT_L_SLOT] | 0; }
get $r() { return this.buf32[RESULT_R_SLOT] | 0; }
get $iu() { return this.buf32[RESULT_IU_SLOT] | 0; }
add(iroot, aL0, n, pivot = 0) {
const aR = n;
@ -561,6 +558,14 @@ const BidiTrieContainer = class {
}
}
getExtra(iboundary) {
return this.buf32[iboundary+BCELL_EXTRA];
}
setExtra(iboundary, v) {
this.buf32[iboundary+BCELL_EXTRA] = v;
}
optimize(shrink = false) {
if ( shrink ) {
this.shrinkBuf();
@ -693,6 +698,65 @@ const BidiTrieContainer = class {
return -1;
}
dumpTrie(iroot) {
for ( const s of this.trieIterator(iroot) ) {
console.log(s);
}
}
trieIterator(iroot) {
return {
value: undefined,
done: false,
next() {
if ( this.icell === 0 ) {
if ( this.forks.length === 0 ) {
this.value = undefined;
this.done = true;
return this;
}
this.charPtr = this.forks.pop();
this.icell = this.forks.pop();
}
for (;;) {
const idown = this.container.buf32[this.icell+CELL_OR];
if ( idown !== 0 ) {
this.forks.push(idown, this.charPtr);
}
const v = this.container.buf32[this.icell+SEGMENT_INFO];
let i0 = this.container.buf32[CHAR0_SLOT] + (v & 0x00FFFFFF);
const i1 = i0 + (v >>> 24);
while ( i0 < i1 ) {
this.charBuf[this.charPtr] = this.container.buf8[i0];
this.charPtr += 1;
i0 += 1;
}
this.icell = this.container.buf32[this.icell+CELL_AND];
if ( this.icell === 0 ) {
return this.toPattern();
}
if ( this.container.buf32[this.icell+SEGMENT_INFO] === 0 ) {
this.icell = this.container.buf32[this.icell+CELL_AND];
return this.toPattern();
}
}
},
toPattern() {
this.value = this.textDecoder.decode(
new Uint8Array(this.charBuf.buffer, 0, this.charPtr)
);
return this;
},
container: this,
icell: iroot,
charBuf: new Uint8Array(256),
charPtr: 0,
forks: [],
textDecoder: new TextDecoder(),
[Symbol.iterator]() { return this; },
};
}
async enableWASM(wasmModuleFetcher, path) {
if ( typeof WebAssembly !== 'object' ) { return false; }
if ( this.wasmMemory instanceof WebAssembly.Memory ) { return true; }
@ -816,103 +880,7 @@ const BidiTrieContainer = class {
HAYSTACK_START + HAYSTACK_SIZE
);
}
};
/*******************************************************************************
Class to hold reference to a specific trie
*/
BidiTrieContainer.prototype.STrieRef = class {
constructor(container, iroot, size) {
this.container = container;
this.iroot = iroot;
this.size = size;
}
add(i, n, pivot = 0) {
const iboundary = this.container.add(this.iroot, i, n, pivot);
if ( iboundary !== 0 ) {
this.size += 1;
}
return iboundary;
}
getExtra(iboundary) {
return this.container.buf32[iboundary+BCELL_EXTRA];
}
setExtra(iboundary, v) {
this.container.buf32[iboundary+BCELL_EXTRA] = v;
}
matches(i) {
return this.container.matches(this.iroot, i);
}
dump() {
for ( const s of this ) {
console.log(s);
}
}
get $l() { return this.container.buf32[RESULT_L_SLOT] | 0; }
get $r() { return this.container.buf32[RESULT_R_SLOT] | 0; }
get $iu() { return this.container.buf32[RESULT_IU_SLOT] | 0; }
[Symbol.iterator]() {
return {
value: undefined,
done: false,
next: function() {
if ( this.icell === 0 ) {
if ( this.forks.length === 0 ) {
this.value = undefined;
this.done = true;
return this;
}
this.charPtr = this.forks.pop();
this.icell = this.forks.pop();
}
for (;;) {
const idown = this.container.buf32[this.icell+CELL_OR];
if ( idown !== 0 ) {
this.forks.push(idown, this.charPtr);
}
const v = this.container.buf32[this.icell+SEGMENT_INFO];
let i0 = this.container.buf32[CHAR0_SLOT] + (v & 0x00FFFFFF);
const i1 = i0 + (v >>> 24);
while ( i0 < i1 ) {
this.charBuf[this.charPtr] = this.container.buf8[i0];
this.charPtr += 1;
i0 += 1;
}
this.icell = this.container.buf32[this.icell+CELL_AND];
if ( this.icell === 0 ) {
return this.toPattern();
}
if ( this.container.buf32[this.icell+SEGMENT_INFO] === 0 ) {
this.icell = this.container.buf32[this.icell+CELL_AND];
return this.toPattern();
}
}
},
toPattern: function() {
this.value = this.textDecoder.decode(
new Uint8Array(this.charBuf.buffer, 0, this.charPtr)
);
return this;
},
container: this.container,
icell: this.iroot,
charBuf: new Uint8Array(256),
charPtr: 0,
forks: [],
textDecoder: new TextDecoder()
};
}
};
}
/******************************************************************************/
@ -954,4 +922,4 @@ const getWasmModule = (( ) => {
/******************************************************************************/
export { BidiTrieContainer };
export default BidiTrieContainer;

224
src/js/hntrie.js
View File

@ -124,7 +124,7 @@ const TRIE0_START = TRIE0_SLOT + 4 << 2; // 272
const roundToPageSize = v => (v + PAGE_SIZE-1) & ~(PAGE_SIZE-1);
const HNTrieContainer = class {
class HNTrieContainer {
constructor() {
const len = PAGE_SIZE * 2;
@ -223,10 +223,7 @@ const HNTrieContainer = class {
return -1;
}
createOne(args) {
if ( Array.isArray(args) ) {
return new this.HNTrieRef(this, args[0], args[1]);
}
createTrie(hostnames = undefined) {
// grow buffer if needed
if ( (this.buf32[CHAR0_SLOT] - this.buf32[TRIE1_SLOT]) < 12 ) {
this.growBuf(12, 0);
@ -236,11 +233,75 @@ const HNTrieContainer = class {
this.buf32[iroot+0] = 0;
this.buf32[iroot+1] = 0;
this.buf32[iroot+2] = 0;
return new this.HNTrieRef(this, iroot, 0);
if ( hostnames !== undefined ) {
for ( const hn of hostnames ) {
this.setNeedle(hn).add(iroot);
}
}
return iroot;
}
compileOne(trieRef) {
return [ trieRef.iroot, trieRef.size ];
dumpTrie(iroot) {
let hostnames = Array.from(this.trieIterator(iroot));
if ( String.prototype.padStart instanceof Function ) {
const maxlen = Math.min(
hostnames.reduce((maxlen, hn) => Math.max(maxlen, hn.length), 0),
64
);
hostnames = hostnames.map(hn => hn.padStart(maxlen));
}
for ( const hn of hostnames ) {
console.log(hn);
}
}
trieIterator(iroot) {
return {
value: undefined,
done: false,
next() {
if ( this.icell === 0 ) {
if ( this.forks.length === 0 ) {
this.value = undefined;
this.done = true;
return this;
}
this.charPtr = this.forks.pop();
this.icell = this.forks.pop();
}
for (;;) {
const idown = this.container.buf32[this.icell+0];
if ( idown !== 0 ) {
this.forks.push(idown, this.charPtr);
}
const v = this.container.buf32[this.icell+2];
let i0 = this.container.buf32[CHAR0_SLOT] + (v >>> 8);
const i1 = i0 + (v & 0x7F);
while ( i0 < i1 ) {
this.charPtr -= 1;
this.charBuf[this.charPtr] = this.container.buf[i0];
i0 += 1;
}
this.icell = this.container.buf32[this.icell+1];
if ( (v & 0x80) !== 0 ) {
return this.toHostname();
}
}
},
toHostname() {
this.value = this.textDecoder.decode(
new Uint8Array(this.charBuf.buffer, this.charPtr)
);
return this;
},
container: this,
icell: this.buf32[iroot],
charBuf: new Uint8Array(256),
charPtr: 256,
forks: [],
textDecoder: new TextDecoder(),
[Symbol.iterator]() { return this; },
};
}
addJS(iroot) {
@ -348,15 +409,6 @@ const HNTrieContainer = class {
};
}
fromIterable(hostnames, add) {
if ( add === undefined ) { add = 'add'; }
const trieRef = this.createOne();
for ( const hn of hostnames ) {
trieRef[add](hn);
}
return trieRef;
}
serialize(encoder) {
if ( encoder instanceof Object ) {
return encoder.encode(
@ -612,7 +664,7 @@ const HNTrieContainer = class {
}
this.buf32 = new Uint32Array(this.buf.buffer);
}
};
}
HNTrieContainer.prototype.matches = HNTrieContainer.prototype.matchesJS;
HNTrieContainer.prototype.matchesWASM = null;
@ -620,142 +672,6 @@ HNTrieContainer.prototype.matchesWASM = null;
HNTrieContainer.prototype.add = HNTrieContainer.prototype.addJS;
HNTrieContainer.prototype.addWASM = null;
/*******************************************************************************
Class to hold reference to a specific trie
*/
HNTrieContainer.prototype.HNTrieRef = class {
constructor(container, iroot, size) {
this.container = container;
this.iroot = iroot;
this.size = size;
this.needle = '';
this.last = -1;
}
add(hn) {
if ( this.container.setNeedle(hn).add(this.iroot) > 0 ) {
this.last = -1;
this.needle = '';
this.size += 1;
return true;
}
return false;
}
addJS(hn) {
if ( this.container.setNeedle(hn).addJS(this.iroot) > 0 ) {
this.last = -1;
this.needle = '';
this.size += 1;
return true;
}
return false;
}
addWASM(hn) {
if ( this.container.setNeedle(hn).addWASM(this.iroot) > 0 ) {
this.last = -1;
this.needle = '';
this.size += 1;
return true;
}
return false;
}
matches(needle) {
if ( needle !== this.needle ) {
this.needle = needle;
this.last = this.container.setNeedle(needle).matches(this.iroot);
}
return this.last;
}
matchesJS(needle) {
if ( needle !== this.needle ) {
this.needle = needle;
this.last = this.container.setNeedle(needle).matchesJS(this.iroot);
}
return this.last;
}
matchesWASM(needle) {
if ( needle !== this.needle ) {
this.needle = needle;
this.last = this.container.setNeedle(needle).matchesWASM(this.iroot);
}
return this.last;
}
dump() {
let hostnames = Array.from(this);
if ( String.prototype.padStart instanceof Function ) {
const maxlen = Math.min(
hostnames.reduce((maxlen, hn) => Math.max(maxlen, hn.length), 0),
64
);
hostnames = hostnames.map(hn => hn.padStart(maxlen));
}
for ( const hn of hostnames ) {
console.log(hn);
}
}
[Symbol.iterator]() {
return {
value: undefined,
done: false,
next: function() {
if ( this.icell === 0 ) {
if ( this.forks.length === 0 ) {
this.value = undefined;
this.done = true;
return this;
}
this.charPtr = this.forks.pop();
this.icell = this.forks.pop();
}
for (;;) {
const idown = this.container.buf32[this.icell+0];
if ( idown !== 0 ) {
this.forks.push(idown, this.charPtr);
}
const v = this.container.buf32[this.icell+2];
let i0 = this.container.buf32[CHAR0_SLOT] + (v >>> 8);
const i1 = i0 + (v & 0x7F);
while ( i0 < i1 ) {
this.charPtr -= 1;
this.charBuf[this.charPtr] = this.container.buf[i0];
i0 += 1;
}
this.icell = this.container.buf32[this.icell+1];
if ( (v & 0x80) !== 0 ) {
return this.toHostname();
}
}
},
toHostname: function() {
this.value = this.textDecoder.decode(
new Uint8Array(this.charBuf.buffer, this.charPtr)
);
return this;
},
container: this.container,
icell: this.container.buf32[this.iroot],
charBuf: new Uint8Array(256),
charPtr: 256,
forks: [],
textDecoder: new TextDecoder()
};
}
};
HNTrieContainer.prototype.HNTrieRef.prototype.last = -1;
HNTrieContainer.prototype.HNTrieRef.prototype.needle = '';
/******************************************************************************/
// Code below is to attempt to load a WASM module which implements:

2796
src/js/static-net-filtering.js
View File

File diff suppressed because it is too large Load Diff

4
src/js/tasks.js
View File

@ -25,12 +25,12 @@
/******************************************************************************/
export function queueTask(func) {
export function queueTask(func, timeout = 5000) {
if ( typeof requestIdleCallback === 'undefined' ) {
return setTimeout(func, 1);
}
return requestIdleCallback(func, { timeout: 5000 });
return requestIdleCallback(func, { timeout });
}
export function dropTask(id) {

2
src/js/traffic.js
View File

@ -936,7 +936,7 @@ const injectCSP = function(fctxt, pageStore, responseHeaders) {
if ( staticDirectives !== undefined ) {
for ( const directive of staticDirectives ) {
if ( directive.result !== 1 ) { continue; }
cspSubsets.push(directive.modifier.value);
cspSubsets.push(directive.value);
}
}