From 69fce3aa6eaa2a34b7e9d2b974674d9da9d49514 Mon Sep 17 00:00:00 2001 From: Raymond Hill Date: Wed, 1 Nov 2023 10:45:31 -0400 Subject: [PATCH] Fine-tune details about when differential update should kick in Manual update of one or more lists will cause the most recent version of these lists to be fetched from the "origin" server, and since the lists from "origin" servers cannot be updated through differential update, the lists will be subsequently updated according to their `Expires` directive. When the lists are auto-updated, the "CDN" servers will be used, and as a result the lists will start to be updated trhough differential updates every 6-hour (currently). Thus it is recommended and optimal to let the lists auto-update, since you will benefit from a much shorter delay to get up-to-date lists (i.e. every 6-hour instead of every 6-day). You can force the auto-updater to fetch all the lists by clicking "Purge all caches", then restart uBO without clicking "Update". This will cause uBO to perform an emergency auto-update at restart time, after which you will have all the lists which are candidates for differential update. The "Update now" button in the "Support" pane will also cause lists to be fetched from their "origin" server. --- src/js/assets.js | 122 +++++++++++++++++++++---------------------- src/js/background.js | 5 +- src/js/storage.js | 5 +- 3 files changed, 66 insertions(+), 66 deletions(-) diff --git a/src/js/assets.js b/src/js/assets.js index 3b50b5273..dcbc1a051 100644 --- a/src/js/assets.js +++ b/src/js/assets.js @@ -26,9 +26,10 @@ import cacheStorage from './cachestorage.js'; import logger from './logger.js'; import µb from './background.js'; +import { ubolog } from './console.js'; import { i18n$ } from './i18n.js'; import * as sfp from './static-filtering-parser.js'; -import { ubolog } from './console.js'; +import { orphanizeString, } from './text-utils.js'; /******************************************************************************/ @@ -47,6 +48,8 @@ let remoteServerFriendly = false; /******************************************************************************/ +const stringIsNotEmpty = s => typeof s === 'string' && s !== ''; + const parseExpires = s => { const matches = s.match(/(\d+)\s*([dh])?/i); if ( matches === null ) { return 0; } @@ -71,7 +74,7 @@ const extractMetadataFromList = (content, fields) => { field = field.toLowerCase().replace( /-[a-z]/g, s => s.charAt(1).toUpperCase() ); - out[field] = value; + out[field] = value && orphanizeString(value); } // Pre-process known fields if ( out.lastModified ) { @@ -169,7 +172,44 @@ const isDiffUpdatableAsset = content => { /^[^%].*[^%]$/.test(data.diffPath); }; -const stringIsNotEmpty = s => typeof s === 'string' && s !== ''; +/******************************************************************************/ + +// favorLocal: avoid making network requests whenever possible +// favorOrigin: avoid using CDN URLs whenever possible + +const getContentURLs = (assetKey, options = {}) => { + const contentURLs = []; + const entry = assetSourceRegistry[assetKey]; + if ( entry instanceof Object === false ) { return contentURLs; } + if ( typeof entry.contentURL === 'string' ) { + contentURLs.push(entry.contentURL); + } else if ( Array.isArray(entry.contentURL) ) { + contentURLs.push(...entry.contentURL); + } else if ( reIsExternalPath.test(assetKey) ) { + contentURLs.push(assetKey); + } + if ( options.favorLocal ) { + contentURLs.sort((a, b) => { + if ( reIsExternalPath.test(a) ) { return 1; } + if ( reIsExternalPath.test(b) ) { return -1; } + return 0; + }); + } + if ( Array.isArray(entry.cdnURLs) ) { + const cdnURLs = entry.cdnURLs.slice(); + for ( let i = 0, n = cdnURLs.length; i < n; i++ ) { + const j = Math.floor(Math.random() * n); + if ( j === i ) { continue; } + [ cdnURLs[j], cdnURLs[i] ] = [ cdnURLs[i], cdnURLs[j] ]; + } + if ( options.favorLocal || options.favorOrigin ) { + contentURLs.push(...cdnURLs); + } else { + contentURLs.unshift(...cdnURLs); + } + } + return contentURLs; +}; /******************************************************************************/ @@ -917,28 +957,17 @@ assets.get = async function(assetKey, options = {}) { } const assetRegistry = await getAssetSourceRegistry(); + assetDetails = assetRegistry[assetKey] || {}; - const contentURLs = []; - if ( typeof assetDetails.contentURL === 'string' ) { - contentURLs.push(assetDetails.contentURL); - } else if ( Array.isArray(assetDetails.contentURL) ) { - contentURLs.push(...assetDetails.contentURL); - } else if ( reIsExternalPath.test(assetKey) ) { + + const contentURLs = getContentURLs(assetKey, options); + if ( contentURLs.length === 0 && reIsExternalPath.test(assetKey) ) { assetDetails.content = 'filters'; contentURLs.push(assetKey); } - // https://github.com/uBlockOrigin/uBlock-issues/issues/1566#issuecomment-826473517 - // Use CDN URLs as fall back URLs. - if ( Array.isArray(assetDetails.cdnURLs) ) { - contentURLs.push(...assetDetails.cdnURLs); - } - let error = 'ENOTFOUND'; for ( const contentURL of contentURLs ) { - if ( reIsExternalPath.test(contentURL) && assetDetails.hasLocalURL ) { - continue; - } const details = assetDetails.content === 'filters' ? await assets.fetchFilterList(contentURL) : await assets.fetchText(contentURL); @@ -966,7 +995,7 @@ assets.get = async function(assetKey, options = {}) { /******************************************************************************/ -async function getRemote(assetKey) { +async function getRemote(assetKey, options = {}) { const [ assetDetails = {}, cacheDetails = {}, @@ -978,9 +1007,9 @@ async function getRemote(assetKey) { let error; let stale = false; - const reportBack = function(content, err) { - const details = { assetKey, content }; - if ( err ) { + const reportBack = function(content, url = '', err = '') { + const details = { assetKey, content, url }; + if ( err !== '') { details.error = assetDetails.lastError = err; } else { assetDetails.lastError = undefined; @@ -988,46 +1017,9 @@ async function getRemote(assetKey) { return details; }; - const contentURLs = []; - if ( typeof assetDetails.contentURL === 'string' ) { - contentURLs.push(assetDetails.contentURL); - } else if ( Array.isArray(assetDetails.contentURL) ) { - contentURLs.push(...assetDetails.contentURL); - } - - // If asked to be gentle on remote servers, favour using dedicated CDN - // servers. If more than one CDN server is present, randomly shuffle the - // set of servers so as to spread the bandwidth burden. - // - // https://github.com/uBlockOrigin/uBlock-issues/issues/1566#issuecomment-826473517 - // In case of manual update, use CDNs URLs as fall back URLs. - if ( Array.isArray(assetDetails.cdnURLs) ) { - const cdnURLs = assetDetails.cdnURLs.slice(); - for ( let i = 0, n = cdnURLs.length; i < n; i++ ) { - const j = Math.floor(Math.random() * n); - if ( j === i ) { continue; } - [ cdnURLs[j], cdnURLs[i] ] = [ cdnURLs[i], cdnURLs[j] ]; - } - if ( remoteServerFriendly ) { - contentURLs.unshift(...cdnURLs); - } else { - contentURLs.push(...cdnURLs); - } - } - - for ( let contentURL of contentURLs ) { + for ( const contentURL of getContentURLs(assetKey, options) ) { if ( reIsExternalPath.test(contentURL) === false ) { continue; } - // This will force uBO to fetch the proper version according to whether - // the dev build is being used. This can be removed when execution of - // this code path is widespread for dev build revisions of uBO. - if ( assetKey === 'assets.json' ) { - contentURL = contentURL.replace( - /\/assets\/assets\.json$/, - µb.assetsJsonPath - ); - } - const result = assetDetails.content === 'filters' ? await assets.fetchFilterList(contentURL) : await assets.fetchText(contentURL); @@ -1066,12 +1058,12 @@ async function getRemote(assetKey) { } registerAssetSource(assetKey, { birthtime: undefined, error: undefined }); - return reportBack(result.content); + return reportBack(result.content, contentURL); } if ( error !== undefined ) { registerAssetSource(assetKey, { error: { time: Date.now(), error } }); - return reportBack('', 'ENOTFOUND'); + return reportBack('', '', 'ENOTFOUND'); } if ( stale ) { @@ -1194,6 +1186,8 @@ const getAssetDiffDetails = assetKey => { }; async function diffUpdater() { + if ( updaterAuto === false ) { return; } + if ( µb.hiddenSettings.differentialUpdate === false ) { return; } const toUpdate = await getUpdateCandidates(); const now = Date.now(); const toHardUpdate = []; @@ -1298,6 +1292,7 @@ async function diffUpdater() { function updateFirst() { ubolog('Updater: cycle start'); + ubolog('Updater: Fetch from ', updaterAuto ? 'CDNs' : 'origin'); updaterStatus = 'updating'; updaterFetched.clear(); updaterUpdated.length = 0; @@ -1367,7 +1362,7 @@ async function updateNext() { let result; if ( assetKey !== 'assets.json' || µb.hiddenSettings.debugAssetsJson !== true ) { - result = await getRemote(assetKey); + result = await getRemote(assetKey, { favorOrigin: updaterAuto === false }); } else { result = await assets.fetchText(µb.assetsJsonPath); result.assetKey = 'assets.json'; @@ -1396,6 +1391,7 @@ function updateDone() { updaterFetched.clear(); updaterUpdated.length = 0; updaterStatus = undefined; + updaterAuto = false; updaterAssetDelay = updaterAssetDelayDefault; ubolog('Updater: cycle end'); if ( assetKeys.length ) { diff --git a/src/js/background.js b/src/js/background.js index 2d1260ad8..7b8c31769 100644 --- a/src/js/background.js +++ b/src/js/background.js @@ -49,9 +49,9 @@ const hiddenSettingsDefault = { allowGenericProceduralFilters: false, assetFetchTimeout: 30, autoCommentFilterTemplate: '{{date}} {{origin}}', - autoUpdateAssetFetchPeriod: 60, + autoUpdateAssetFetchPeriod: 15, autoUpdateDelayAfterLaunch: 105, - autoUpdatePeriod: 2, + autoUpdatePeriod: 1, benchmarkDatasetURL: 'unset', blockingProfiles: '11111/#F00 11010/#C0F 11001/#00F 00001', cacheStorageAPI: 'unset', @@ -69,6 +69,7 @@ const hiddenSettingsDefault = { debugAssetsJson: false, debugScriptlets: false, debugScriptletInjector: false, + differentialUpdate: true, disableWebAssembly: false, extensionUpdateForceReload: false, filterAuthorMode: false, diff --git a/src/js/storage.js b/src/js/storage.js index 4def990e2..730338382 100644 --- a/src/js/storage.js +++ b/src/js/storage.js @@ -981,7 +981,10 @@ import { return { assetKey, content: '' }; } - const rawDetails = await io.get(assetKey, { silent: true }); + const rawDetails = await io.get(assetKey, { + favorLocal: this.readyToFilter !== true, + silent: true, + }); // Compiling an empty string results in an empty string. if ( rawDetails.content === '' ) { rawDetails.assetKey = assetKey;