Rewrite logger's "exceptor" feature

Related issue:
- https://github.com/uBlockOrigin/uBlock-issues/issues/1861

The "exceptor" feature has been rewritten, with the following
changes as a result:

- The excepted filters cease to exist when closing the logger
- It's now possible to temporary except network filters

When toggling on/off a temporary exception, filter lists are now
fully reloaded. This simplified managing temporary exceptions, and
made it easy to implement temporary exception for network filters,
but this also means there might be a perceptible delay when
adding/removing temporary exceptions. At this point I consider
this an acceptable side-effect just to bring the ability to easily
create temporary exception for network filters, while this
simplified the existing temporary exception code throughout.
This commit is contained in:
Raymond Hill 2022-12-14 16:04:45 -05:00
parent ce3f852bad
commit a91781a495
No known key found for this signature in database
GPG Key ID: 25E1490B761470C2
13 changed files with 127 additions and 255 deletions

View File

@ -919,6 +919,9 @@ vAPI.messaging = {
this.onPortDisconnect(port);
}
}
if ( this.defaultHandler ) {
this.defaultHandler(message, null, ( ) => { });
}
},
onFrameworkMessage: function(request, port, callback) {

View File

@ -678,6 +678,9 @@ body[dir="rtl"] #netFilteringDialog > .panes > .details > div > span:nth-of-type
background-color: rgb(var(--primary-50) / 50%);
}
#netFilteringDialog > .panes > .details .exceptor::before {
content: '@@';
}
#netFilteringDialog.extendedRealm > .panes > .details .exceptor::before {
content: '#@#';
}
#netFilteringDialog > div.panes > .dynamic > .toolbar {

View File

@ -211,6 +211,9 @@ const µBlock = { // jshint ignore:line
availableFilterLists: {},
badLists: new Map(),
inMemoryFilters: [],
inMemoryFiltersCompiled: '',
// https://github.com/uBlockOrigin/uBlock-issues/issues/974
// This can be used to defer filtering decision-making.
readyToFilter: false,

View File

@ -27,10 +27,7 @@ import './utils.js';
import logger from './logger.js';
import µb from './background.js';
import {
StaticExtFilteringHostnameDB,
StaticExtFilteringSessionDB,
} from './static-ext-filtering-db.js';
import { StaticExtFilteringHostnameDB } from './static-ext-filtering-db.js';
/******************************************************************************/
/******************************************************************************/
@ -236,10 +233,7 @@ const FilterContainer = function() {
// specific filters
this.specificFilters = new StaticExtFilteringHostnameDB(2);
// temporary filters
this.sessionFilterDB = new StaticExtFilteringSessionDB();
// low generic cosmetic filters: map of hash => array of selectors
// low generic cosmetic filters: map of hash => stringified selector list
this.lowlyGeneric = new Map();
// highly generic selectors sets
@ -478,15 +472,6 @@ FilterContainer.prototype.compileSpecificSelector = function(
/******************************************************************************/
FilterContainer.prototype.compileTemporary = function(parser) {
return {
session: this.sessionFilterDB,
selector: parser.result.compiled,
};
};
/******************************************************************************/
FilterContainer.prototype.fromCompiledContent = function(reader, options) {
if ( options.skipCosmetic ) {
this.skipCompiledContent(reader, 'SPECIFIC');
@ -697,12 +682,6 @@ FilterContainer.prototype.disableSurveyor = function(details) {
/******************************************************************************/
FilterContainer.prototype.getSession = function() {
return this.sessionFilterDB;
};
/******************************************************************************/
FilterContainer.prototype.cssRuleFromProcedural = function(json) {
const pfilter = JSON.parse(json);
if ( pfilter.cssable !== true ) { return; }
@ -831,11 +810,6 @@ FilterContainer.prototype.retrieveSpecificSelectors = function(
}
}
// Retrieve temporary filters
if ( this.sessionFilterDB.isNotEmpty ) {
this.sessionFilterDB.retrieve([ null, exceptionSet ]);
}
// Retrieve filters with a non-empty hostname
this.specificFilters.retrieve(
hostname,

View File

@ -27,10 +27,7 @@ import logger from './logger.js';
import µb from './background.js';
import { sessionFirewall } from './filtering-engines.js';
import {
StaticExtFilteringHostnameDB,
StaticExtFilteringSessionDB,
} from './static-ext-filtering-db.js';
import { StaticExtFilteringHostnameDB } from './static-ext-filtering-db.js';
/******************************************************************************/
@ -38,7 +35,6 @@ const pselectors = new Map();
const duplicates = new Set();
const filterDB = new StaticExtFilteringHostnameDB(2);
const sessionFilterDB = new StaticExtFilteringSessionDB();
let acceptedCount = 0;
let discardedCount = 0;
@ -347,13 +343,6 @@ htmlFilteringEngine.compile = function(parser, writer) {
}
};
htmlFilteringEngine.compileTemporary = function(parser) {
return {
session: sessionFilterDB,
selector: parser.result.compiled,
};
};
htmlFilteringEngine.fromCompiledContent = function(reader) {
// Don't bother loading filters if stream filtering is not supported.
if ( µb.canFilterResponseData === false ) { return; }
@ -373,10 +362,6 @@ htmlFilteringEngine.fromCompiledContent = function(reader) {
}
};
htmlFilteringEngine.getSession = function() {
return sessionFilterDB;
};
htmlFilteringEngine.retrieve = function(details) {
const hostname = details.hostname;
@ -384,9 +369,6 @@ htmlFilteringEngine.retrieve = function(details) {
const procedurals = new Set();
const exceptions = new Set();
if ( sessionFilterDB.isNotEmpty ) {
sessionFilterDB.retrieve([ null, exceptions ]);
}
filterDB.retrieve(
hostname,
[ plains, exceptions, procedurals, exceptions ]

View File

@ -28,16 +28,12 @@ import µb from './background.js';
import { entityFromDomain } from './uri-utils.js';
import { sessionFirewall } from './filtering-engines.js';
import {
StaticExtFilteringHostnameDB,
StaticExtFilteringSessionDB,
} from './static-ext-filtering-db.js';
import { StaticExtFilteringHostnameDB } from './static-ext-filtering-db.js';
/******************************************************************************/
const duplicates = new Set();
const filterDB = new StaticExtFilteringHostnameDB(1);
const sessionFilterDB = new StaticExtFilteringSessionDB();
const $headers = new Set();
const $exceptions = new Set();
@ -123,13 +119,6 @@ httpheaderFilteringEngine.compile = function(parser, writer) {
}
};
httpheaderFilteringEngine.compileTemporary = function(parser) {
return {
session: sessionFilterDB,
selector: parser.result.compiled.slice(15, -1),
};
};
// 01234567890123456789
// responseheader(name)
// ^ ^
@ -152,10 +141,6 @@ httpheaderFilteringEngine.fromCompiledContent = function(reader) {
}
};
httpheaderFilteringEngine.getSession = function() {
return sessionFilterDB;
};
httpheaderFilteringEngine.apply = function(fctxt, headers) {
if ( filterDB.size === 0 ) { return; }
@ -173,9 +158,6 @@ httpheaderFilteringEngine.apply = function(fctxt, headers) {
$headers.clear();
$exceptions.clear();
if ( sessionFilterDB.isNotEmpty ) {
sessionFilterDB.retrieve([ null, $exceptions ]);
}
filterDB.retrieve(hostname, [ $headers, $exceptions ]);
filterDB.retrieve(entity, [ $headers, $exceptions ], 1);
if ( $headers.size === 0 ) { return; }

View File

@ -1246,9 +1246,13 @@ const reloadTab = function(ev) {
// Toggle temporary exception filter
if ( tcl.contains('exceptor') ) {
ev.stopPropagation();
const filter = filterFromTargetRow();
const exceptedFilter = dom.cl.has(targetRow, 'extendedRealm')
? `#@#${filter.replace(/^.*?#@?#/, '')}`
: `@@${filter.replace(/^@@/, '')}`;
const status = await messaging.send('loggerUI', {
what: 'toggleTemporaryException',
filter: filterFromTargetRow(),
what: 'toggleInMemoryFilter',
filter: exceptedFilter,
});
const row = target.closest('div');
dom.cl.toggle(row, 'exceptored', status);
@ -1477,26 +1481,16 @@ const reloadTab = function(ev) {
const toSummaryPaneFilterNode = async function(receiver, filter) {
receiver.children[1].textContent = filter;
if ( filterAuthorMode !== true ) { return; }
const match = /#@?#/.exec(filter);
if ( match === null ) { return; }
const fragment = document.createDocumentFragment();
const pos = match.index + match[0].length;
fragment.appendChild(document.createTextNode(filter.slice(0, pos)));
const selector = filter.slice(pos);
const span = document.createElement('span');
span.className = 'filter';
span.textContent = selector;
fragment.appendChild(span);
if ( dom.cl.has(targetRow, 'canLookup') === false ) { return; }
const exceptedFilter = dom.cl.has(targetRow, 'extendedRealm')
? `#@#${filter.replace(/^.*?#@?#/, '')}`
: `@@${filter.replace(/^@@/, '')}`;
const isTemporaryException = await messaging.send('loggerUI', {
what: 'hasTemporaryException',
filter,
what: 'hasInMemoryFilter',
filter: exceptedFilter,
});
dom.cl.toggle(receiver, 'exceptored', isTemporaryException);
if ( match[0] === '##' || isTemporaryException ) {
receiver.children[2].style.visibility = '';
}
receiver.children[1].textContent = '';
receiver.children[1].appendChild(fragment);
};
const fillSummaryPaneFilterList = async function(rows) {

View File

@ -32,19 +32,15 @@ let writePtr = 0;
const logBufferObsoleteAfter = 30 * 1000;
const janitor = ( ) => {
if (
buffer !== null &&
lastReadTime < (Date.now() - logBufferObsoleteAfter)
) {
if ( buffer === null ) { return; }
if ( lastReadTime >= (Date.now() - logBufferObsoleteAfter) ) {
return vAPI.setTimeout(janitor, logBufferObsoleteAfter);
}
logger.enabled = false;
buffer = null;
writePtr = 0;
logger.ownerId = undefined;
vAPI.messaging.broadcast({ what: 'loggerDisabled' });
}
if ( buffer !== null ) {
vAPI.setTimeout(janitor, logBufferObsoleteAfter);
}
};
const boxEntry = function(details) {

View File

@ -35,7 +35,6 @@ import logger from './logger.js';
import lz4Codec from './lz4.js';
import io from './assets.js';
import scriptletFilteringEngine from './scriptlet-filtering.js';
import staticExtFilteringEngine from './static-ext-filtering.js';
import staticFilteringReverseLookup from './reverselookup.js';
import staticNetFilteringEngine from './static-net-filtering.js';
import µb from './background.js';
@ -305,6 +304,10 @@ const onMessage = function(request, sender, callback) {
µb.elementPickerExec(request.tabId, 0, request.targetURL, request.zap);
break;
case 'loggerDisabled':
µb.clearInMemoryFilters();
break;
case 'gotoURL':
µb.openNewTab(request.details);
break;
@ -1680,42 +1683,11 @@ const getURLFilteringData = function(details) {
return response;
};
const compileTemporaryException = function(filter) {
const parser = new StaticFilteringParser({
nativeCssHas: vAPI.webextFlavor.env.includes('native_css_has'),
});
parser.analyze(filter);
if ( parser.shouldDiscard() ) { return; }
return staticExtFilteringEngine.compileTemporary(parser);
};
const toggleTemporaryException = function(details) {
const result = compileTemporaryException(details.filter);
if ( result === undefined ) { return false; }
const { session, selector } = result;
if ( session.has(1, selector) ) {
session.remove(1, selector);
return false;
}
session.add(1, selector);
return true;
};
const hasTemporaryException = function(details) {
const result = compileTemporaryException(details.filter);
if ( result === undefined ) { return false; }
const { session, selector } = result;
return session && session.has(1, selector);
};
const onMessage = function(request, sender, callback) {
// Async
switch ( request.what ) {
case 'readAll':
if (
logger.ownerId !== undefined &&
logger.ownerId !== request.ownerId
) {
if ( logger.ownerId !== undefined && logger.ownerId !== request.ownerId ) {
return callback({ unavailable: true });
}
vAPI.tabs.getCurrent().then(tab => {
@ -1723,6 +1695,13 @@ const onMessage = function(request, sender, callback) {
});
return;
case 'toggleInMemoryFilter': {
const promise = µb.hasInMemoryFilter(request.filter)
? µb.removeInMemoryFilter(request.filter)
: µb.addInMemoryFilter(request.filter);
promise.then(status => { callback(status); });
return;
}
default:
break;
}
@ -1731,14 +1710,14 @@ const onMessage = function(request, sender, callback) {
let response;
switch ( request.what ) {
case 'hasTemporaryException':
response = hasTemporaryException(request);
case 'hasInMemoryFilter':
response = µb.hasInMemoryFilter(request.filter);
break;
case 'releaseView':
if ( request.ownerId === logger.ownerId ) {
if ( request.ownerId !== logger.ownerId ) { break; }
logger.ownerId = undefined;
}
µb.clearInMemoryFilters();
break;
case 'saveURLFilteringRules':
@ -1761,10 +1740,6 @@ const onMessage = function(request, sender, callback) {
response = getURLFilteringData(request);
break;
case 'toggleTemporaryException':
response = toggleTemporaryException(request);
break;
default:
return vAPI.messaging.UNHANDLED;
}

View File

@ -28,10 +28,7 @@ import µb from './background.js';
import { redirectEngine } from './redirect-engine.js';
import { sessionFirewall } from './filtering-engines.js';
import {
StaticExtFilteringHostnameDB,
StaticExtFilteringSessionDB,
} from './static-ext-filtering-db.js';
import { StaticExtFilteringHostnameDB } from './static-ext-filtering-db.js';
import {
domainFromHostname,
@ -46,7 +43,6 @@ const scriptletCache = new µb.MRUCache(32);
const reEscapeScriptArg = /[\\'"]/g;
const scriptletDB = new StaticExtFilteringHostnameDB(1);
const sessionScriptletDB = new StaticExtFilteringSessionDB();
let acceptedCount = 0;
let discardedCount = 0;
@ -262,13 +258,6 @@ scriptletFilteringEngine.compile = function(parser, writer) {
}
};
scriptletFilteringEngine.compileTemporary = function(parser) {
return {
session: sessionScriptletDB,
selector: parser.result.compiled,
};
};
// 01234567890123456789
// +js(token[, arg[, ...]])
// ^ ^
@ -291,10 +280,6 @@ scriptletFilteringEngine.fromCompiledContent = function(reader) {
}
};
scriptletFilteringEngine.getSession = function() {
return sessionScriptletDB;
};
const $scriptlets = new Set();
const $exceptions = new Set();
const $scriptletToCodeMap = new Map();
@ -307,9 +292,6 @@ scriptletFilteringEngine.retrieve = function(request, options = {}) {
$scriptlets.clear();
$exceptions.clear();
if ( sessionScriptletDB.isNotEmpty ) {
sessionScriptletDB.retrieve([ null, $exceptions ]);
}
scriptletDB.retrieve(hostname, [ $scriptlets, $exceptions ]);
const entity = request.entity !== ''
? `${hostname.slice(0, -request.domain.length)}${request.entity}`

View File

@ -170,55 +170,8 @@ const StaticExtFilteringHostnameDB = class {
/******************************************************************************/
const StaticExtFilteringSessionDB = class {
constructor() {
this.db = new Map();
}
compile(s) {
return s;
}
add(bits, s) {
const bucket = this.db.get(bits);
if ( bucket === undefined ) {
this.db.set(bits, new Set([ s ]));
} else {
bucket.add(s);
}
}
remove(bits, s) {
const bucket = this.db.get(bits);
if ( bucket === undefined ) { return; }
bucket.delete(s);
if ( bucket.size !== 0 ) { return; }
this.db.delete(bits);
}
retrieve(out) {
const mask = out.length - 1;
for ( const [ bits, bucket ] of this.db ) {
const i = bits & mask;
if ( out[i] instanceof Object === false ) { continue; }
for ( const s of bucket ) {
out[i].add(s);
}
}
}
has(bits, s) {
const selectors = this.db.get(bits);
return selectors !== undefined && selectors.has(s);
}
clear() {
this.db.clear();
}
get isNotEmpty() {
return this.db.size !== 0;
}
};
/******************************************************************************/
export {
StaticExtFilteringHostnameDB,
StaticExtFilteringSessionDB,
};
/******************************************************************************/

View File

@ -132,19 +132,6 @@ staticExtFilteringEngine.compile = function(parser, writer) {
return true;
};
staticExtFilteringEngine.compileTemporary = function(parser) {
if ( (parser.flavorBits & parser.BITFlavorExtScriptlet) !== 0 ) {
return scriptletFilteringEngine.compileTemporary(parser);
}
if ( (parser.flavorBits & parser.BITFlavorExtResponseHeader) !== 0 ) {
return httpheaderFilteringEngine.compileTemporary(parser);
}
if ( (parser.flavorBits & parser.BITFlavorExtHTML) !== 0 ) {
return htmlFilteringEngine.compileTemporary(parser);
}
return cosmeticFilteringEngine.compileTemporary(parser);
};
staticExtFilteringEngine.fromCompiledContent = function(reader, options) {
cosmeticFilteringEngine.fromCompiledContent(reader, options);
scriptletFilteringEngine.fromCompiledContent(reader, options);

View File

@ -618,6 +618,36 @@ self.addEventListener('hiddenSettingsChanged', ( ) => {
/******************************************************************************/
µb.hasInMemoryFilter = function(raw) {
return this.inMemoryFilters.includes(raw);
};
µb.addInMemoryFilter = async function(raw) {
if ( this.inMemoryFilters.includes(raw) ){ return true; }
this.inMemoryFilters.push(raw);
this.inMemoryFiltersCompiled = '';
await this.loadFilterLists();
return true;
};
µb.removeInMemoryFilter = async function(raw) {
const pos = this.inMemoryFilters.indexOf(raw);
if ( pos === -1 ) { return false; }
this.inMemoryFilters.splice(pos, 1);
this.inMemoryFiltersCompiled = '';
await this.loadFilterLists();
return false;
};
µb.clearInMemoryFilters = async function() {
if ( this.inMemoryFilters.length === 0 ) { return; }
this.inMemoryFilters = [];
this.inMemoryFiltersCompiled = '';
await this.loadFilterLists();
};
/******************************************************************************/
µb.getAvailableLists = async function() {
let oldAvailableLists = {},
newAvailableLists = {};
@ -760,12 +790,12 @@ self.addEventListener('hiddenSettingsChanged', ( ) => {
/******************************************************************************/
µb.loadFilterLists = (( ) => {
{
const loadedListKeys = [];
let loadingPromise;
let t0 = 0;
const onDone = function() {
const onDone = ( ) => {
ubolog(`loadFilterLists() took ${Date.now()-t0} ms`);
staticNetFilteringEngine.freeze();
@ -773,30 +803,30 @@ self.addEventListener('hiddenSettingsChanged', ( ) => {
redirectEngine.freeze();
vAPI.net.unsuspend();
vAPI.storage.set({ 'availableFilterLists': this.availableFilterLists });
vAPI.storage.set({ 'availableFilterLists': µb.availableFilterLists });
vAPI.messaging.broadcast({
what: 'staticFilteringDataChanged',
parseCosmeticFilters: this.userSettings.parseAllABPHideFilters,
ignoreGenericCosmeticFilters: this.userSettings.ignoreGenericCosmeticFilters,
parseCosmeticFilters: µb.userSettings.parseAllABPHideFilters,
ignoreGenericCosmeticFilters: µb.userSettings.ignoreGenericCosmeticFilters,
listKeys: loadedListKeys
});
this.selfieManager.destroy();
µb.selfieManager.destroy();
lz4Codec.relinquish();
this.compiledFormatChanged = false;
µb.compiledFormatChanged = false;
loadingPromise = undefined;
};
const applyCompiledFilters = function(assetKey, compiled) {
const applyCompiledFilters = (assetKey, compiled) => {
const snfe = staticNetFilteringEngine;
const sxfe = staticExtFilteringEngine;
let acceptedCount = snfe.acceptedCount + sxfe.acceptedCount,
discardedCount = snfe.discardedCount + sxfe.discardedCount;
this.applyCompiledFilters(compiled, assetKey === this.userFiltersPath);
if ( this.availableFilterLists.hasOwnProperty(assetKey) ) {
const entry = this.availableFilterLists[assetKey];
let acceptedCount = snfe.acceptedCount + sxfe.acceptedCount;
let discardedCount = snfe.discardedCount + sxfe.discardedCount;
µb.applyCompiledFilters(compiled, assetKey === µb.userFiltersPath);
if ( µb.availableFilterLists.hasOwnProperty(assetKey) ) {
const entry = µb.availableFilterLists[assetKey];
entry.entryCount = snfe.acceptedCount + sxfe.acceptedCount -
acceptedCount;
entry.entryUsedCount = entry.entryCount -
@ -805,8 +835,8 @@ self.addEventListener('hiddenSettingsChanged', ( ) => {
loadedListKeys.push(assetKey);
};
const onFilterListsReady = function(lists) {
this.availableFilterLists = lists;
const onFilterListsReady = lists => {
µb.availableFilterLists = lists;
if ( vAPI.Net.canSuspend() ) {
vAPI.net.suspend();
@ -814,7 +844,7 @@ self.addEventListener('hiddenSettingsChanged', ( ) => {
redirectEngine.reset();
staticExtFilteringEngine.reset();
staticNetFilteringEngine.reset();
this.selfieManager.destroy();
µb.selfieManager.destroy();
staticFilteringReverseLookup.resetLists();
// We need to build a complete list of assets to pull first: this is
@ -825,37 +855,44 @@ self.addEventListener('hiddenSettingsChanged', ( ) => {
for ( const assetKey in lists ) {
if ( lists.hasOwnProperty(assetKey) === false ) { continue; }
if ( lists[assetKey].off ) { continue; }
toLoad.push(
this.getCompiledFilterList(assetKey).then(details => {
applyCompiledFilters.call(
this,
details.assetKey,
details.content
);
µb.getCompiledFilterList(assetKey).then(details => {
applyCompiledFilters(details.assetKey, details.content);
})
);
}
if ( µb.inMemoryFilters.length !== 0 ) {
if ( µb.inMemoryFiltersCompiled === '' ) {
µb.inMemoryFiltersCompiled =
µb.compileFilters(
µb.inMemoryFilters.join('\n'),
{ assetKey: 'in-memory'}
);
}
if ( µb.inMemoryFiltersCompiled !== '' ) {
toLoad.push(
µb.applyCompiledFilters(µb.inMemoryFiltersCompiled, true)
);
}
}
return Promise.all(toLoad);
};
return function() {
if ( loadingPromise instanceof Promise === false ) {
µb.loadFilterLists = function() {
if ( loadingPromise instanceof Promise ) { return loadingPromise; }
t0 = Date.now();
loadedListKeys.length = 0;
loadingPromise = Promise.all([
this.getAvailableLists().then(lists =>
onFilterListsReady.call(this, lists)
),
this.getAvailableLists().then(lists => onFilterListsReady(lists)),
this.loadRedirectResources(),
]).then(( ) => {
onDone.call(this);
onDone();
});
}
return loadingPromise;
};
})();
}
/******************************************************************************/
@ -1158,6 +1195,7 @@ self.addEventListener('hiddenSettingsChanged', ( ) => {
// memory usage at selfie-load time. For some reasons.
const create = async function() {
if ( µb.inMemoryFilters.length !== 0 ) { return; }
await Promise.all([
io.put(
'selfie/main',