Remove "Purge all caches" button from "Filter lists" pane

Purging all the lists from cache storage is detrimental to
differential update, and cause filter lists to be updated less
often and consequently to be less up to date then when letting
differential updater do its work.
This commit is contained in:
Raymond Hill 2023-12-13 21:01:51 -05:00
parent 14926913f7
commit bd7ce41224
No known key found for this signature in database
GPG Key ID: 25E1490B761470C2
11 changed files with 276 additions and 205 deletions

View File

@ -20,7 +20,6 @@
<div id="actions">
<button id="buttonApply" class="preferred disabled iconified" type="button" data-i18n-title="3pApplyChanges"><span class="fa-icon">check</span><span data-i18n="3pApplyChanges">_</span><span class="hover"></span></button>
<button id="buttonUpdate" class="preferred disabled iconified" type="button" data-i18n-title="3pUpdateNow"><span class="fa-icon">refresh</span><span data-i18n="3pUpdateNow">_</span><span class="hover"></span></button>
<button id="buttonPurgeAll" class="disabled iconified" type="button" data-i18n-title="3pPurgeAll"><span class="fa-icon">clock-o</span><span data-i18n="3pPurgeAll">_</span><span class="hover"></span></button>
</div>
<div>

View File

@ -29,6 +29,7 @@
<button id="snfe-todnr" type="button">SNFE: DNR<span class="hover"></span></button>
<button id="snfe-benchmark" type="button" disabled>SNFE: Benchmark<span class="hover"></span></button>
<button id="cfe-dump" type="button">CFE: Dump<span class="hover"></span></button>
<button id="purge-all-caches" type="button" data-i18n-title="3pPurgeAll"><span data-i18n="3pPurgeAll">_</span><span class="hover"></span></button>
</div>
<div id="console" class="codeMirrorContainer"></div>

View File

@ -276,7 +276,7 @@ const renderFilterLists = ( ) => {
renderWidgets();
};
vAPI.messaging.send('dashboard', {
return vAPI.messaging.send('dashboard', {
what: 'getLists',
}).then(response => {
onListsReceived(response);
@ -295,9 +295,6 @@ const renderWidgets = ( ) => {
updating === false &&
qs$('#lists .listEntry.checked.obsolete:not(.toRemove)') === null
);
dom.cl.toggle('#buttonPurgeAll', 'disabled',
updating || qs$('#lists .listEntry.cached:not(.obsolete)') === null
);
};
/******************************************************************************/
@ -510,8 +507,9 @@ const onPurgeClicked = ev => {
}
vAPI.messaging.send('dashboard', {
what: 'purgeCaches',
what: 'listsUpdateNow',
assetKeys,
preferOrigin: ev.shiftKey,
});
// If the cached version is purged, the installed version must be assumed
@ -519,6 +517,7 @@ const onPurgeClicked = ev => {
// https://github.com/gorhill/uBlock/issues/1733
// An external filter list must not be marked as obsolete, they will
// always be fetched anyways if there is no cached copy.
dom.cl.add(dom.body, 'updating');
dom.cl.add(liEntry, 'obsolete');
if ( qs$(liEntry, 'input[type="checkbox"]').checked ) {
@ -608,25 +607,13 @@ const buttonUpdateHandler = async ( ) => {
await selectFilterLists();
dom.cl.add(dom.body, 'updating');
renderWidgets();
vAPI.messaging.send('dashboard', { what: 'forceUpdateAssets' });
vAPI.messaging.send('dashboard', { what: 'updateNow' });
};
dom.on('#buttonUpdate', 'click', ( ) => { buttonUpdateHandler(); });
/******************************************************************************/
const buttonPurgeAllHandler = async hard => {
await vAPI.messaging.send('dashboard', {
what: 'purgeAllCaches',
hard,
});
renderFilterLists(true);
};
dom.on('#buttonPurgeAll', 'click', ev => { buttonPurgeAllHandler(ev.shiftKey); });
/******************************************************************************/
const userSettingCheckboxChanged = ( ) => {
const target = event.target;
vAPI.messaging.send('dashboard', {
@ -863,6 +850,12 @@ self.hasUnsavedData = function() {
/******************************************************************************/
renderFilterLists();
renderFilterLists().then(( ) => {
const buttonUpdate = qs$('#buttonUpdate');
if ( dom.cl.has(buttonUpdate, 'active') ) { return; }
if ( dom.cl.has(buttonUpdate, 'disabled') ) { return; }
if ( listsetDetails.autoUpdate !== true ) { return; }
buttonUpdateHandler();
});
/******************************************************************************/

View File

@ -39,6 +39,7 @@ const reIsUserAsset = /^user-/;
const errorCantConnectTo = i18n$('errorCantConnectTo');
const MS_PER_HOUR = 60 * 60 * 1000;
const MS_PER_DAY = 24 * MS_PER_HOUR;
const MINUTES_PER_DAY = 24 * 60;
const EXPIRES_DEFAULT = 7;
const assets = {};
@ -175,6 +176,23 @@ const isDiffUpdatableAsset = content => {
data.diffPath.startsWith('%') === false;
};
const computedPatchUpdateTime = assetKey => {
const entry = assetCacheRegistry[assetKey];
if ( entry === undefined ) { return 0; }
if ( typeof entry.diffPath !== 'string' ) { return 0; }
if ( typeof entry.diffExpires !== 'number' ) { return 0; }
const match = /(\d+)\.(\d+)\.(\d+)\.(\d+)/.exec(entry.diffPath);
if ( match === null ) { return getWriteTime(); }
const date = new Date();
date.setUTCFullYear(
parseInt(match[1], 10),
parseInt(match[2], 10) - 1,
parseInt(match[3], 10)
);
date.setUTCHours(0, parseInt(match[4], 10) + entry.diffExpires * MINUTES_PER_DAY, 0, 0);
return date.getTime();
};
/******************************************************************************/
// favorLocal: avoid making network requests whenever possible
@ -1169,7 +1187,7 @@ assets.getUpdateAges = async function(conditions = {}) {
out.push({
assetKey,
age,
ageNormalized: age / getUpdateAfterTime(assetKey),
ageNormalized: age / Math.max(1, getUpdateAfterTime(assetKey)),
});
}
return out;
@ -1221,12 +1239,13 @@ async function diffUpdater() {
const assetDetails = getAssetDiffDetails(assetKey);
if ( assetDetails === undefined ) { continue; }
assetDetails.what = 'update';
if ( (getWriteTime(assetKey) + assetDetails.diffExpires) > now ) {
assetDetails.fetch = false;
toSoftUpdate.push(assetDetails);
} else {
const computedUpdateTime = computedPatchUpdateTime(assetKey);
if ( computedUpdateTime !== 0 && computedUpdateTime <= now ) {
assetDetails.fetch = true;
toHardUpdate.push(assetDetails);
} else {
assetDetails.fetch = false;
toSoftUpdate.push(assetDetails);
}
}
if ( toHardUpdate.length === 0 ) { return; }
@ -1432,8 +1451,8 @@ function updateDone() {
assets.updateStart = function(details) {
const oldUpdateDelay = updaterAssetDelay;
const newUpdateDelay = typeof details.delay === 'number'
? details.delay
const newUpdateDelay = typeof details.fetchDelay === 'number'
? details.fetchDelay
: updaterAssetDelayDefault;
updaterAssetDelay = Math.min(oldUpdateDelay, newUpdateDelay);
updaterAuto = details.auto === true;

View File

@ -134,7 +134,7 @@ dom.on('#console-unfold', 'click', ( ) => {
dom.on('#snfe-dump', 'click', ev => {
const button = ev.target;
dom.attr(button, 'disabled', '');
vAPI.messaging.send('dashboard', {
vAPI.messaging.send('devTools', {
what: 'snfeDump',
}).then(result => {
log(result);
@ -145,7 +145,7 @@ dom.on('#snfe-dump', 'click', ev => {
dom.on('#snfe-todnr', 'click', ev => {
const button = ev.target;
dom.attr(button, 'disabled', '');
vAPI.messaging.send('dashboard', {
vAPI.messaging.send('devTools', {
what: 'snfeToDNR',
}).then(result => {
log(result);
@ -153,6 +153,25 @@ dom.on('#snfe-todnr', 'click', ev => {
});
});
dom.on('#cfe-dump', 'click', ev => {
const button = ev.target;
dom.attr(button, 'disabled', '');
vAPI.messaging.send('devTools', {
what: 'cfeDump',
}).then(result => {
log(result);
dom.attr(button, 'disabled', null);
});
});
dom.on('#purge-all-caches', 'click', ( ) => {
vAPI.messaging.send('devTools', {
what: 'purgeAllCaches'
}).then(result => {
log(result);
});
});
vAPI.messaging.send('dashboard', {
what: 'getAppData',
}).then(appData => {
@ -161,7 +180,7 @@ vAPI.messaging.send('dashboard', {
dom.on('#snfe-benchmark', 'click', ev => {
const button = ev.target;
dom.attr(button, 'disabled', '');
vAPI.messaging.send('dashboard', {
vAPI.messaging.send('devTools', {
what: 'snfeBenchmark',
}).then(result => {
log(result);
@ -170,15 +189,4 @@ vAPI.messaging.send('dashboard', {
});
});
dom.on('#cfe-dump', 'click', ev => {
const button = ev.target;
dom.attr(button, 'disabled', '');
vAPI.messaging.send('dashboard', {
what: 'cfeDump',
}).then(result => {
log(result);
dom.attr(button, 'disabled', null);
});
});
/******************************************************************************/

View File

@ -44,7 +44,7 @@ let ttlCount = 0;
let ttlDelay = 60000;
const init = function() {
ttlDelay = µb.hiddenSettings.autoUpdateAssetFetchPeriod * 1000 + 15000;
ttlDelay = µb.hiddenSettings.autoUpdateAssetFetchPeriod * 2 * 1000;
if ( promisedInstance === undefined ) {
let flavor;
if ( µb.hiddenSettings.disableWebAssembly === true ) {

View File

@ -142,124 +142,6 @@ const onMessage = function(request, sender, callback) {
});
return;
case 'snfeBenchmark':
µb.benchmarkStaticNetFiltering({ redirectEngine }).then(result => {
callback(result);
});
return;
case 'snfeToDNR': {
const listPromises = [];
const listNames = [];
for ( const assetKey of µb.selectedFilterLists ) {
listPromises.push(
io.get(assetKey, { dontCache: true }).then(details => {
listNames.push(assetKey);
return { name: assetKey, text: details.content };
})
);
}
const options = {
extensionPaths: redirectEngine.getResourceDetails().filter(e =>
typeof e[1].extensionPath === 'string' && e[1].extensionPath !== ''
).map(e =>
[ e[0], e[1].extensionPath ]
),
env: vAPI.webextFlavor.env,
};
const t0 = Date.now();
dnrRulesetFromRawLists(listPromises, options).then(result => {
const { network } = result;
const replacer = (k, v) => {
if ( k.startsWith('__') ) { return; }
if ( Array.isArray(v) ) {
return v.sort();
}
if ( v instanceof Object ) {
const sorted = {};
for ( const kk of Object.keys(v).sort() ) {
sorted[kk] = v[kk];
}
return sorted;
}
return v;
};
const isUnsupported = rule =>
rule._error !== undefined;
const isRegex = rule =>
rule.condition !== undefined &&
rule.condition.regexFilter !== undefined;
const isRedirect = rule =>
rule.action !== undefined &&
rule.action.type === 'redirect' &&
rule.action.redirect.extensionPath !== undefined;
const isCsp = rule =>
rule.action !== undefined &&
rule.action.type === 'modifyHeaders';
const isRemoveparam = rule =>
rule.action !== undefined &&
rule.action.type === 'redirect' &&
rule.action.redirect.transform !== undefined;
const runtime = Date.now() - t0;
const { ruleset } = network;
const good = ruleset.filter(rule =>
isUnsupported(rule) === false &&
isRegex(rule) === false &&
isRedirect(rule) === false &&
isCsp(rule) === false &&
isRemoveparam(rule) === false
);
const unsupported = ruleset.filter(rule =>
isUnsupported(rule)
);
const regexes = ruleset.filter(rule =>
isUnsupported(rule) === false &&
isRegex(rule) &&
isRedirect(rule) === false &&
isCsp(rule) === false &&
isRemoveparam(rule) === false
);
const redirects = ruleset.filter(rule =>
isUnsupported(rule) === false &&
isRedirect(rule)
);
const headers = ruleset.filter(rule =>
isUnsupported(rule) === false &&
isCsp(rule)
);
const removeparams = ruleset.filter(rule =>
isUnsupported(rule) === false &&
isRemoveparam(rule)
);
const out = [
`dnrRulesetFromRawLists(${JSON.stringify(listNames, null, 2)})`,
`Run time: ${runtime} ms`,
`Filters count: ${network.filterCount}`,
`Accepted filter count: ${network.acceptedFilterCount}`,
`Rejected filter count: ${network.rejectedFilterCount}`,
`Un-DNR-able filter count: ${unsupported.length}`,
`Resulting DNR rule count: ${ruleset.length}`,
];
out.push(`+ Good filters (${good.length}): ${JSON.stringify(good, replacer, 2)}`);
out.push(`+ Regex-based filters (${regexes.length}): ${JSON.stringify(regexes, replacer, 2)}`);
out.push(`+ 'redirect=' filters (${redirects.length}): ${JSON.stringify(redirects, replacer, 2)}`);
out.push(`+ 'csp=' filters (${headers.length}): ${JSON.stringify(headers, replacer, 2)}`);
out.push(`+ 'removeparam=' filters (${removeparams.length}): ${JSON.stringify(removeparams, replacer, 2)}`);
out.push(`+ Unsupported filters (${unsupported.length}): ${JSON.stringify(unsupported, replacer, 2)}`);
out.push(`+ generichide exclusions (${network.generichideExclusions.length}): ${JSON.stringify(network.generichideExclusions, replacer, 2)}`);
if ( result.specificCosmetic ) {
out.push(`+ Cosmetic filters: ${result.specificCosmetic.size}`);
for ( const details of result.specificCosmetic ) {
out.push(` ${JSON.stringify(details)}`);
}
} else {
out.push(' Cosmetic filters: 0');
}
callback(out.join('\n'));
});
return;
}
default:
break;
}
@ -280,13 +162,6 @@ const onMessage = function(request, sender, callback) {
µb.createUserFilters(request);
break;
case 'forceUpdateAssets':
µb.scheduleAssetUpdater(0);
io.updateStart({
delay: µb.hiddenSettings.manualUpdateAssetFetchPeriod
});
break;
case 'getAppData':
response = {
name: browser.runtime.getManifest().name,
@ -383,14 +258,6 @@ const onMessage = function(request, sender, callback) {
}
break;
case 'snfeDump':
response = staticNetFilteringEngine.dump();
break;
case 'cfeDump':
response = cosmeticFilteringEngine.dump();
break;
default:
return vAPI.messaging.UNHANDLED;
}
@ -1630,19 +1497,25 @@ const onMessage = function(request, sender, callback) {
response = getRules();
break;
case 'purgeAllCaches':
if ( request.hard ) {
io.remove(/./);
} else {
io.purge(/./, 'public_suffix_list.dat');
}
break;
case 'purgeCaches':
for ( const assetKey of request.assetKeys ) {
case 'supportUpdateNow': {
const { assetKeys } = request;
if ( assetKeys.length === 0 ) { return; }
for ( const assetKey of assetKeys ) {
io.purge(assetKey);
}
µb.scheduleAssetUpdater({ now: true, fetchDelay: 100 });
break;
}
case 'listsUpdateNow': {
const { assetKeys, preferOrigin = false } = request;
if ( assetKeys.length === 0 ) { return; }
for ( const assetKey of assetKeys ) {
io.purge(assetKey);
}
µb.scheduleAssetUpdater({ now: true, fetchDelay: 100, auto: preferOrigin !== true });
break;
}
case 'readHiddenSettings':
response = {
@ -1660,6 +1533,10 @@ const onMessage = function(request, sender, callback) {
resetUserData();
break;
case 'updateNow':
µb.scheduleAssetUpdater({ now: true, fetchDelay: 100, auto: true });
break;
case 'writeHiddenSettings':
µb.changeHiddenSettings(µb.hiddenSettingsFromString(request.content));
break;
@ -1944,6 +1821,181 @@ vAPI.messaging.listen({
/******************************************************************************/
/******************************************************************************/
// Channel:
// devTools
// privileged
{
// >>>>> start of local scope
const onMessage = function(request, sender, callback) {
// Async
switch ( request.what ) {
case 'purgeAllCaches':
µb.getBytesInUse().then(bytesInUseBefore =>
io.remove(/./).then(( ) =>
µb.getBytesInUse().then(bytesInUseAfter => {
callback([
`Storage used before: ${µb.formatCount(bytesInUseBefore)}B`,
`Storage used after: ${µb.formatCount(bytesInUseAfter)}B`,
].join('\n'));
})
)
);
return;
case 'snfeBenchmark':
µb.benchmarkStaticNetFiltering({ redirectEngine }).then(result => {
callback(result);
});
return;
case 'snfeToDNR': {
const listPromises = [];
const listNames = [];
for ( const assetKey of µb.selectedFilterLists ) {
listPromises.push(
io.get(assetKey, { dontCache: true }).then(details => {
listNames.push(assetKey);
return { name: assetKey, text: details.content };
})
);
}
const options = {
extensionPaths: redirectEngine.getResourceDetails().filter(e =>
typeof e[1].extensionPath === 'string' && e[1].extensionPath !== ''
).map(e =>
[ e[0], e[1].extensionPath ]
),
env: vAPI.webextFlavor.env,
};
const t0 = Date.now();
dnrRulesetFromRawLists(listPromises, options).then(result => {
const { network } = result;
const replacer = (k, v) => {
if ( k.startsWith('__') ) { return; }
if ( Array.isArray(v) ) {
return v.sort();
}
if ( v instanceof Object ) {
const sorted = {};
for ( const kk of Object.keys(v).sort() ) {
sorted[kk] = v[kk];
}
return sorted;
}
return v;
};
const isUnsupported = rule =>
rule._error !== undefined;
const isRegex = rule =>
rule.condition !== undefined &&
rule.condition.regexFilter !== undefined;
const isRedirect = rule =>
rule.action !== undefined &&
rule.action.type === 'redirect' &&
rule.action.redirect.extensionPath !== undefined;
const isCsp = rule =>
rule.action !== undefined &&
rule.action.type === 'modifyHeaders';
const isRemoveparam = rule =>
rule.action !== undefined &&
rule.action.type === 'redirect' &&
rule.action.redirect.transform !== undefined;
const runtime = Date.now() - t0;
const { ruleset } = network;
const good = ruleset.filter(rule =>
isUnsupported(rule) === false &&
isRegex(rule) === false &&
isRedirect(rule) === false &&
isCsp(rule) === false &&
isRemoveparam(rule) === false
);
const unsupported = ruleset.filter(rule =>
isUnsupported(rule)
);
const regexes = ruleset.filter(rule =>
isUnsupported(rule) === false &&
isRegex(rule) &&
isRedirect(rule) === false &&
isCsp(rule) === false &&
isRemoveparam(rule) === false
);
const redirects = ruleset.filter(rule =>
isUnsupported(rule) === false &&
isRedirect(rule)
);
const headers = ruleset.filter(rule =>
isUnsupported(rule) === false &&
isCsp(rule)
);
const removeparams = ruleset.filter(rule =>
isUnsupported(rule) === false &&
isRemoveparam(rule)
);
const out = [
`dnrRulesetFromRawLists(${JSON.stringify(listNames, null, 2)})`,
`Run time: ${runtime} ms`,
`Filters count: ${network.filterCount}`,
`Accepted filter count: ${network.acceptedFilterCount}`,
`Rejected filter count: ${network.rejectedFilterCount}`,
`Un-DNR-able filter count: ${unsupported.length}`,
`Resulting DNR rule count: ${ruleset.length}`,
];
out.push(`+ Good filters (${good.length}): ${JSON.stringify(good, replacer, 2)}`);
out.push(`+ Regex-based filters (${regexes.length}): ${JSON.stringify(regexes, replacer, 2)}`);
out.push(`+ 'redirect=' filters (${redirects.length}): ${JSON.stringify(redirects, replacer, 2)}`);
out.push(`+ 'csp=' filters (${headers.length}): ${JSON.stringify(headers, replacer, 2)}`);
out.push(`+ 'removeparam=' filters (${removeparams.length}): ${JSON.stringify(removeparams, replacer, 2)}`);
out.push(`+ Unsupported filters (${unsupported.length}): ${JSON.stringify(unsupported, replacer, 2)}`);
out.push(`+ generichide exclusions (${network.generichideExclusions.length}): ${JSON.stringify(network.generichideExclusions, replacer, 2)}`);
if ( result.specificCosmetic ) {
out.push(`+ Cosmetic filters: ${result.specificCosmetic.size}`);
for ( const details of result.specificCosmetic ) {
out.push(` ${JSON.stringify(details)}`);
}
} else {
out.push(' Cosmetic filters: 0');
}
callback(out.join('\n'));
});
return;
}
default:
break;
}
// Sync
let response;
switch ( request.what ) {
case 'snfeDump':
response = staticNetFilteringEngine.dump();
break;
case 'cfeDump':
response = cosmeticFilteringEngine.dump();
break;
default:
return vAPI.messaging.UNHANDLED;
}
callback(response);
};
vAPI.messaging.listen({
name: 'devTools',
listener: onMessage,
privileged: true,
});
// <<<<< end of local scope
}
/******************************************************************************/
/******************************************************************************/
// Channel:
// scriptlets
// unprivileged
@ -2122,12 +2174,11 @@ const onMessage = function(request, sender, callback) {
io.purge(listkey);
}
}
µb.scheduleAssetUpdater(0);
µb.openNewTab({
url: 'dashboard.html#3p-filters.html',
select: true,
});
io.updateStart({ delay: 100, auto: request.manual !== true });
µb.scheduleAssetUpdater({ now: true, fetchDelay: 100, auto: request.manual !== true });
break;
default:

View File

@ -474,11 +474,7 @@ lz4Codec.relinquish();
// https://github.com/chrisaljoudi/uBlock/issues/184
// Check for updates not too far in the future.
io.addObserver(µb.assetObserver.bind(µb));
µb.scheduleAssetUpdater(
µb.userSettings.autoUpdate
? µb.hiddenSettings.autoUpdateDelayAfterLaunch * 1000
: 0
);
µb.scheduleAssetUpdater();
// Force an update of the context menu according to the currently
// active tab.

View File

@ -1497,16 +1497,27 @@ onBroadcast(msg => {
const launchTimer = vAPI.defer.create(fetchDelay => {
next = 0;
io.updateStart({ delay: fetchDelay, auto: true });
io.updateStart({ fetchDelay, auto: true });
});
µb.scheduleAssetUpdater = async function(updateDelay) {
µb.scheduleAssetUpdater = async function(details = {}) {
launchTimer.off();
if ( updateDelay === 0 ) {
if ( details.now ) {
next = 0;
io.updateStart(details);
return;
}
if ( µb.userSettings.autoUpdate === false ) {
if ( details.updateDelay === undefined ) {
next = 0;
return;
}
}
let updateDelay = details.updateDelay ||
this.hiddenSettings.autoUpdatePeriod * 3600000;
const now = Date.now();
let needEmergencyUpdate = false;
@ -1632,13 +1643,7 @@ onBroadcast(msg => {
}
this.loadFilterLists();
}
if ( this.userSettings.autoUpdate ) {
this.scheduleAssetUpdater(
this.hiddenSettings.autoUpdatePeriod * 3600000 || 25200000
);
} else {
this.scheduleAssetUpdater(0);
}
this.scheduleAssetUpdater();
broadcast({
what: 'assetsUpdated',
assetKeys: details.assetKeys

View File

@ -255,8 +255,7 @@ async function updateFilterLists() {
if ( dom.body.dataset.shouldUpdateLists === undefined ) { return false; }
dom.cl.add(dom.body, 'updating');
const assetKeys = JSON.parse(dom.body.dataset.shouldUpdateLists);
vAPI.messaging.send('dashboard', { what: 'purgeCaches', assetKeys });
vAPI.messaging.send('dashboard', { what: 'forceUpdateAssets' });
vAPI.messaging.send('dashboard', { what: 'supportUpdateNow', assetKeys });
return true;
}

View File

@ -345,7 +345,7 @@ const matchBucket = function(url, hostname, bucket, start) {
}
break;
case 'autoUpdate':
this.scheduleAssetUpdater(value ? 7 * 60 * 1000 : 0);
this.scheduleAssetUpdater({ updateDelay: value ? 2000 : 0 });
break;
case 'cnameUncloakEnabled':
if ( vAPI.net.canUncloakCnames === true ) {