diff --git a/platform/mv3/extension/css/popup.css b/platform/mv3/extension/css/popup.css index 4e0a6d2c8..20804f366 100644 --- a/platform/mv3/extension/css/popup.css +++ b/platform/mv3/extension/css/popup.css @@ -123,12 +123,13 @@ body.needSave #revertRules { #rulesetStats .rulesetDetails h1 { font-size: 1em; font-weight: normal; - margin: 0.5em 0; + margin: 0.5em 0 0.25em 0; text-transform: capitalize; } #rulesetStats .rulesetDetails p { + color: var(--ink-2); font-size: var(--font-size-smaller); - margin: 0.5em 0 0.5em var(--popup-gap-thin); + margin: 0.25em 0 0.5em 0.5em; } .itemRibbon { diff --git a/platform/mv3/extension/js/3p-filters.js b/platform/mv3/extension/js/3p-filters.js index d0aaec660..55973ad16 100644 --- a/platform/mv3/extension/js/3p-filters.js +++ b/platform/mv3/extension/js/3p-filters.js @@ -30,19 +30,35 @@ import { simpleStorage } from './storage.js'; /******************************************************************************/ +const rulesetMap = new Map(); let cachedRulesetData = {}; let filteringSettingsHash = ''; let hideUnusedSet = new Set([ 'regions' ]); /******************************************************************************/ -const renderNumber = function(value) { +function renderNumber(value) { return value.toLocaleString(); -}; +} /******************************************************************************/ -function renderFilterLists(soft) { +function rulesetStats(rulesetId) { + const canRemoveParams = cachedRulesetData.hasOmnipotence; + const rulesetDetails = rulesetMap.get(rulesetId); + if ( rulesetDetails === undefined ) { return; } + const { rules, filters } = rulesetDetails; + let ruleCount = rules.plain + rules.regexes; + if ( canRemoveParams ) { + ruleCount += rules.removeparams; + } + const filterCount = filters.accepted; + return { ruleCount, filterCount }; +} + +/******************************************************************************/ + +function renderFilterLists(soft = false) { const { enabledRulesets, rulesetDetails } = cachedRulesetData; const listGroupTemplate = qs$('#templates .groupEntry'); const listEntryTemplate = qs$('#templates .listEntry'); @@ -77,12 +93,19 @@ function renderFilterLists(soft) { dom.cl.toggle(li, 'unused', hideUnused && !on); } // https://github.com/gorhill/uBlock/issues/1429 - if ( !soft ) { + if ( soft !== true ) { qs$('input[type="checkbox"]', li).checked = on; } + const stats = rulesetStats(ruleset.id); li.title = listStatsTemplate - .replace('{{ruleCount}}', renderNumber(ruleset.rules.accepted)) - .replace('{{filterCount}}', renderNumber(ruleset.filters.accepted)); + .replace('{{ruleCount}}', renderNumber(stats.ruleCount)) + .replace('{{filterCount}}', renderNumber(stats.filterCount)); + dom.attr( + qs$('.input.checkbox', li), + 'disabled', + stats.ruleCount === 0 ? '' : null + ); + dom.cl.remove(li, 'discard'); return li; }; @@ -200,17 +223,14 @@ const renderWidgets = function() { ); // Compute total counts - const rulesetMap = new Map( - cachedRulesetData.rulesetDetails.map(rule => [ rule.id, rule ]) - ); let filterCount = 0; let ruleCount = 0; for ( const liEntry of qsa$('#lists .listEntry[data-listkey]') ) { if ( qs$('input[type="checkbox"]:checked', liEntry) === null ) { continue; } - const ruleset = rulesetMap.get(liEntry.dataset.listkey); - if ( ruleset === undefined ) { continue; } - filterCount += ruleset.filters.accepted; - ruleCount += ruleset.rules.accepted; + const stats = rulesetStats(liEntry.dataset.listkey); + if ( stats === undefined ) { continue; } + ruleCount += stats.ruleCount; + filterCount += stats.filterCount; } qs$('#listsOfBlockedHostsPrompt').textContent = i18n$('perRulesetStats') .replace('{{ruleCount}}', ruleCount.toLocaleString()) @@ -239,7 +259,10 @@ async function onOmnipotenceChanged(ev) { }) !== true; } + cachedRulesetData.hasOmnipotence = actualState; qs$('#omnipotenceWidget input').checked = actualState; + renderFilterLists(true); + renderWidgets(); } dom.on( @@ -385,6 +408,8 @@ sendMessage({ }).then(data => { if ( !data ) { return; } cachedRulesetData = data; + rulesetMap.clear(); + cachedRulesetData.rulesetDetails.forEach(rule => rulesetMap.set(rule.id, rule)); try { renderFilterLists(); } catch(ex) { diff --git a/platform/mv3/extension/js/background.js b/platform/mv3/extension/js/background.js index d835d6e4c..7ccab8f97 100644 --- a/platform/mv3/extension/js/background.js +++ b/platform/mv3/extension/js/background.js @@ -37,8 +37,8 @@ import { getDynamicRules, defaultRulesetsFromLanguage, enableRulesets, - getEnabledRulesetsStats, - updateRegexRules, + getEnabledRulesetsDetails, + updateDynamicRules, } from './ruleset-manager.js'; import { @@ -113,7 +113,8 @@ async function saveRulesetConfig() { /******************************************************************************/ -function hasGreatPowers(origin) { +async function hasGreatPowers(origin) { + if ( /^https?:\/\//.test(origin) === false ) { return false; } return browser.permissions.contains({ origins: [ `${origin}/*` ], }); @@ -126,10 +127,16 @@ function hasOmnipotence() { } function onPermissionsAdded(permissions) { + if ( permissions.origins?.includes('') ) { + updateDynamicRules(); + } registerInjectables(permissions.origins); } function onPermissionsRemoved(permissions) { + if ( permissions.origins?.includes('') ) { + updateDynamicRules(); + } registerInjectables(permissions.origins); } @@ -170,7 +177,7 @@ function onMessage(request, sender, callback) { matchesTrustedSiteDirective(request), hasOmnipotence(), hasGreatPowers(request.origin), - getEnabledRulesetsStats(), + getEnabledRulesetsDetails(), getInjectableCount(request.origin), ]).then(results => { callback({ @@ -208,7 +215,7 @@ async function start() { const currentVersion = getCurrentVersion(); if ( currentVersion !== rulesetConfig.version ) { console.log(`Version change: ${rulesetConfig.version} => ${currentVersion}`); - updateRegexRules().then(( ) => { + updateDynamicRules().then(( ) => { rulesetConfig.version = currentVersion; saveRulesetConfig(); }); diff --git a/platform/mv3/extension/js/dom.js b/platform/mv3/extension/js/dom.js index 625d0060b..4e2a4657c 100644 --- a/platform/mv3/extension/js/dom.js +++ b/platform/mv3/extension/js/dom.js @@ -62,7 +62,11 @@ class dom { if ( value === undefined ) { return elem.getAttribute(attr); } - elem.setAttribute(attr, value); + if ( value === null ) { + elem.removeAttribute(attr); + } else { + elem.setAttribute(attr, value); + } } } diff --git a/platform/mv3/extension/js/popup.js b/platform/mv3/extension/js/popup.js index f0c18a8f5..671fd30ea 100644 --- a/platform/mv3/extension/js/popup.js +++ b/platform/mv3/extension/js/popup.js @@ -237,10 +237,14 @@ async function init() { const div = qs$('#templates .rulesetDetails').cloneNode(true); dom.text(qs$('h1', div), details.name); const { rules, filters, css } = details; + let ruleCount = rules.plain + rules.regexes; + if ( popupPanelData.hasOmnipotence ) { + ruleCount += rules.removeparams; + } dom.text( qs$('p', div), i18n$('perRulesetStats') - .replace('{{ruleCount}}', rules.accepted.toLocaleString()) + .replace('{{ruleCount}}', ruleCount.toLocaleString()) .replace('{{filterCount}}', filters.accepted.toLocaleString()) .replace('{{cssSpecificCount}}', css.specific.toLocaleString()) ); diff --git a/platform/mv3/extension/js/ruleset-manager.js b/platform/mv3/extension/js/ruleset-manager.js index 933468226..1baeb58fd 100644 --- a/platform/mv3/extension/js/ruleset-manager.js +++ b/platform/mv3/extension/js/ruleset-manager.js @@ -25,7 +25,7 @@ /******************************************************************************/ -import { dnr, i18n } from './ext.js'; +import { browser, dnr, i18n } from './ext.js'; import { fetchJSON } from './fetch.js'; /******************************************************************************/ @@ -33,6 +33,8 @@ import { fetchJSON } from './fetch.js'; const RULE_REALM_SIZE = 1000000; const REGEXES_REALM_START = 1000000; const REGEXES_REALM_END = REGEXES_REALM_START + RULE_REALM_SIZE; +const REMOVEPARAMS_REALM_START = 2000000; +const REMOVEPARAMS_REALM_END = REMOVEPARAMS_REALM_START + RULE_REALM_SIZE; const TRUSTED_DIRECTIVE_BASE_RULE_ID = 8000000; const CURRENT_CONFIG_BASE_RULE_ID = 9000000; @@ -79,7 +81,7 @@ async function updateRegexRules() { rulesetDetails, dynamicRules ] = await Promise.all([ - getRulesetDetails(), + getEnabledRulesetsDetails(), dnr.getDynamicRules(), ]); @@ -96,8 +98,7 @@ async function updateRegexRules() { // Fetch regexes for all enabled rulesets const toFetch = []; - for ( const details of rulesetDetails.values() ) { - if ( details.enabled !== true ) { continue; } + for ( const details of rulesetDetails ) { if ( details.rules.regexes === 0 ) { continue; } toFetch.push(fetchJSON(`/rulesets/${details.id}.regexes`)); } @@ -131,7 +132,7 @@ async function updateRegexRules() { if ( result instanceof Object && result.isSupported ) { newRules.push(rule); } else { - console.info(`${result.reason}: ${rule.condition.regexFilter}`); + //console.info(`${result.reason}: ${rule.condition.regexFilter}`); } } console.info( @@ -139,11 +140,12 @@ async function updateRegexRules() { ); // Add validated regex rules to dynamic ruleset without affecting rules - // outside regex rule realm. + // outside regex rules realm. const dynamicRuleMap = await getDynamicRules(); const newRuleMap = new Map(newRules.map(rule => [ rule.id, rule ])); const addRules = []; const removeRuleIds = []; + for ( const oldRule of dynamicRuleMap.values() ) { if ( oldRule.id < REGEXES_REALM_START ) { continue; } if ( oldRule.id >= REGEXES_REALM_END ) { continue; } @@ -157,14 +159,104 @@ async function updateRegexRules() { dynamicRuleMap.set(oldRule.id, newRule); } } + for ( const newRule of newRuleMap.values() ) { if ( dynamicRuleMap.has(newRule.id) ) { continue; } addRules.push(newRule); dynamicRuleMap.set(newRule.id, newRule); } - if ( addRules.length !== 0 || removeRuleIds.length !== 0 ) { - return dnr.updateDynamicRules({ addRules, removeRuleIds }); + + if ( addRules.length === 0 && removeRuleIds.length === 0 ) { return; } + + if ( removeRuleIds.length !== 0 ) { + console.info(`Remove ${removeRuleIds.length} DNR regex rules`); } + if ( addRules.length !== 0 ) { + console.info(`Add ${addRules.length} DNR regex rules`); + } + + return dnr.updateDynamicRules({ addRules, removeRuleIds }); +} + +/******************************************************************************/ + +async function updateRemoveparamRules() { + const [ + hasOmnipotence, + rulesetDetails, + dynamicRuleMap, + ] = await Promise.all([ + browser.permissions.contains({ origins: [ '' ] }), + getEnabledRulesetsDetails(), + getDynamicRules(), + ]); + + // Fetch removeparam rules for all enabled rulesets + const toFetch = []; + for ( const details of rulesetDetails ) { + if ( details.rules.removeparams === 0 ) { continue; } + toFetch.push(fetchJSON(`/rulesets/${details.id}.removeparams`)); + } + const removeparamRulesets = await Promise.all(toFetch); + + // Removeparam rules can only be enforced with omnipotence + const newRules = []; + if ( hasOmnipotence ) { + let removeparamRuleId = REMOVEPARAMS_REALM_START; + for ( const rules of removeparamRulesets ) { + if ( Array.isArray(rules) === false ) { continue; } + for ( const rule of rules ) { + rule.id = removeparamRuleId++; + newRules.push(rule); + } + } + } + + // Add removeparam rules to dynamic ruleset without affecting rules + // outside removeparam rules realm. + const newRuleMap = new Map(newRules.map(rule => [ rule.id, rule ])); + const addRules = []; + const removeRuleIds = []; + + for ( const oldRule of dynamicRuleMap.values() ) { + if ( oldRule.id < REMOVEPARAMS_REALM_START ) { continue; } + if ( oldRule.id >= REMOVEPARAMS_REALM_END ) { continue; } + const newRule = newRuleMap.get(oldRule.id); + if ( newRule === undefined ) { + removeRuleIds.push(oldRule.id); + dynamicRuleMap.delete(oldRule.id); + } else if ( JSON.stringify(oldRule) !== JSON.stringify(newRule) ) { + removeRuleIds.push(oldRule.id); + addRules.push(newRule); + dynamicRuleMap.set(oldRule.id, newRule); + } + } + + for ( const newRule of newRuleMap.values() ) { + if ( dynamicRuleMap.has(newRule.id) ) { continue; } + addRules.push(newRule); + dynamicRuleMap.set(newRule.id, newRule); + } + + if ( addRules.length === 0 && removeRuleIds.length === 0 ) { return; } + + if ( removeRuleIds.length !== 0 ) { + console.info(`Remove ${removeRuleIds.length} DNR removeparam rules`); + } + if ( addRules.length !== 0 ) { + console.info(`Add ${addRules.length} DNR removeparam rules`); + } + + return dnr.updateDynamicRules({ addRules, removeRuleIds }); +} + +/******************************************************************************/ + +async function updateDynamicRules() { + return Promise.all([ + updateRegexRules(), + updateRemoveparamRules(), + ]); } /******************************************************************************/ @@ -240,18 +332,23 @@ async function enableRulesets(ids) { if ( disableRulesetIds.length !== 0 ) { console.info(`Disable ruleset: ${disableRulesetIds}`); } - return dnr.updateEnabledRulesets({ enableRulesetIds, disableRulesetIds }); + await dnr.updateEnabledRulesets({ enableRulesetIds, disableRulesetIds }); + + return Promise.all([ + updateRegexRules(), + updateRemoveparamRules(), + ]); } /******************************************************************************/ -async function getEnabledRulesetsStats() { +async function getEnabledRulesetsDetails() { const [ - rulesetDetails, ids, + rulesetDetails, ] = await Promise.all([ - getRulesetDetails(), dnr.getEnabledRulesets(), + getRulesetDetails(), ]); const out = []; for ( const id of ids ) { @@ -265,14 +362,12 @@ async function getEnabledRulesetsStats() { /******************************************************************************/ export { - REGEXES_REALM_START, - REGEXES_REALM_END, TRUSTED_DIRECTIVE_BASE_RULE_ID, CURRENT_CONFIG_BASE_RULE_ID, getRulesetDetails, getDynamicRules, enableRulesets, defaultRulesetsFromLanguage, - getEnabledRulesetsStats, - updateRegexRules, + getEnabledRulesetsDetails, + updateDynamicRules, }; diff --git a/platform/mv3/extension/js/scripting-manager.js b/platform/mv3/extension/js/scripting-manager.js index 0a92aa644..18c2e66fa 100644 --- a/platform/mv3/extension/js/scripting-manager.js +++ b/platform/mv3/extension/js/scripting-manager.js @@ -40,10 +40,9 @@ function getScriptingDetails() { return scriptingDetailsPromise; } scriptingDetailsPromise = fetchJSON('/rulesets/scripting-details').then(entries => { - const out = new Map(entries); - for ( const details of out.values() ) { - details.matches = new Map(details.matches); - details.excludeMatches = new Map(details.excludeMatches); + const out = new Map(); + for ( const entry of entries ) { + out.set(entry[0], new Map(entry[1])); } return out; }); @@ -52,32 +51,19 @@ function getScriptingDetails() { /******************************************************************************/ -const arrayEq = (a, b) => { - if ( a === undefined ) { return b === undefined; } - if ( b === undefined ) { return false; } - if ( a.length !== b.length ) { return false; } - for ( const i of a ) { - if ( b.includes(i) === false ) { return false; } - } - return true; -}; - -/******************************************************************************/ - -const toRegisterable = (fname, entry) => { +const toRegisterable = (fname, hostnames) => { const directive = { id: fname, + allFrames: true, + matchOriginAsFallback: true, }; - if ( entry.matches ) { - directive.matches = ut.matchesFromHostnames(entry.matches); + if ( hostnames ) { + directive.matches = ut.matchesFromHostnames(hostnames); } else { directive.matches = [ '' ]; } - if ( entry.excludeMatches ) { - directive.excludeMatches = ut.matchesFromHostnames(entry.excludeMatches); - } directive.js = [ `/rulesets/js/${fname.slice(0,2)}/${fname.slice(2)}.js` ]; - if ( (ut.fidFromFileName(fname) & RUN_AT_BIT) !== 0 ) { + if ( (ut.fidFromFileName(fname) & RUN_AT_END_BIT) !== 0 ) { directive.runAt = 'document_end'; } else { directive.runAt = 'document_start'; @@ -88,23 +74,28 @@ const toRegisterable = (fname, entry) => { return directive; }; -const RUN_AT_BIT = 0b10; +const RUN_AT_END_BIT = 0b10; const MAIN_WORLD_BIT = 0b01; /******************************************************************************/ -const shouldUpdate = (registered, candidate) => { - const matches = candidate.matches && - ut.matchesFromHostnames(candidate.matches); - if ( arrayEq(registered.matches, matches) === false ) { - return true; +// Important: We need to sort the arrays for fast comparison +const arrayEq = (a, b) => { + if ( a === undefined ) { return b === undefined; } + if ( b === undefined ) { return false; } + const alen = a.length; + if ( alen !== b.length ) { return false; } + a.sort(); b.sort(); + for ( let i = 0; i < alen; i++ ) { + if ( a[i] !== b[i] ) { return false; } } - const excludeMatches = candidate.excludeMatches && - ut.matchesFromHostnames(candidate.excludeMatches); - if ( arrayEq(registered.excludeMatches, excludeMatches) === false ) { - return true; - } - return false; + return true; +}; + +const shouldUpdate = (registered, candidateHostnames) => { + const registeredHostnames = registered.matches && + ut.hostnamesFromMatches(registered.matches); + return arrayEq(registeredHostnames, candidateHostnames) === false; }; const isTrustedHostname = (trustedSites, hn) => { @@ -133,11 +124,11 @@ async function getInjectableCount(origin) { let total = 0; for ( const rulesetId of rulesetIds ) { - if ( scriptingDetails.has(rulesetId) === false ) { continue; } - const details = scriptingDetails.get(rulesetId); + const hostnamesToFidsMap = scriptingDetails.get(rulesetId); + if ( hostnamesToFidsMap === undefined ) { continue; } let hn = url.hostname; while ( hn !== '' ) { - const fids = details.matches?.get(hn); + const fids = hostnamesToFidsMap.get(hn); if ( typeof fids === 'number' ) { total += 1; } else if ( Array.isArray(fids) ) { @@ -162,28 +153,33 @@ function registerSomeInjectables(args) { const toRegisterMap = new Map(); - const checkMatches = (details, hn) => { - let fids = details.matches?.get(hn); + const checkMatches = (hostnamesToFidsMap, hn) => { + let fids = hostnamesToFidsMap.get(hn); if ( fids === undefined ) { return; } if ( typeof fids === 'number' ) { fids = [ fids ]; } for ( const fid of fids ) { const fname = ut.fnameFromFileId(fid); - const existing = toRegisterMap.get(fname); + let existing = toRegisterMap.get(fname); if ( existing ) { - existing.matches.push(hn); + if ( existing[0] === '*' ) { continue; } + existing.push(hn); } else { - toRegisterMap.set(fname, { matches: [ hn ] }); + toRegisterMap.set(fname, existing = [ hn ]); } + if ( hn !== '*' ) { continue; } + existing.length = 0; + existing.push('*'); + break; } }; for ( const rulesetId of rulesetIds ) { - const details = scriptingDetails.get(rulesetId); - if ( details === undefined ) { continue; } + const hostnamesToFidsMap = scriptingDetails.get(rulesetId); + if ( hostnamesToFidsMap === undefined ) { continue; } for ( let hn of hostnamesSet ) { if ( isTrustedHostname(trustedSites, hn) ) { continue; } while ( hn ) { - checkMatches(details, hn); + checkMatches(hostnamesToFidsMap, hn); hn = ut.toBroaderHostname(hn); } } @@ -202,19 +198,24 @@ function registerAllInjectables(args) { const toRegisterMap = new Map(); for ( const rulesetId of rulesetIds ) { - const details = scriptingDetails.get(rulesetId); - if ( details === undefined ) { continue; } - for ( let [ hn, fids ] of details.matches ) { + const hostnamesToFidsMap = scriptingDetails.get(rulesetId); + if ( hostnamesToFidsMap === undefined ) { continue; } + for ( let [ hn, fids ] of hostnamesToFidsMap ) { if ( isTrustedHostname(trustedSites, hn) ) { continue; } if ( typeof fids === 'number' ) { fids = [ fids ]; } for ( const fid of fids ) { const fname = ut.fnameFromFileId(fid); - const existing = toRegisterMap.get(fname); + let existing = toRegisterMap.get(fname); if ( existing ) { - existing.matches.push(hn); + if ( existing[0] === '*' ) { continue; } + existing.push(hn); } else { - toRegisterMap.set(fname, { matches: [ hn ] }); + toRegisterMap.set(fname, existing = [ hn ]); } + if ( hn !== '*' ) { continue; } + existing.length = 0; + existing.push('*'); + break; } } } @@ -264,13 +265,13 @@ async function registerInjectables(origins) { const toAdd = []; const toUpdate = []; - for ( const [ fname, entry ] of toRegisterMap ) { + for ( const [ fname, hostnames ] of toRegisterMap ) { if ( before.has(fname) === false ) { - toAdd.push(toRegisterable(fname, entry)); + toAdd.push(toRegisterable(fname, hostnames)); continue; } - if ( shouldUpdate(before.get(fname), entry) ) { - toUpdate.push(toRegisterable(fname, entry)); + if ( shouldUpdate(before.get(fname), hostnames) ) { + toUpdate.push(toRegisterable(fname, hostnames)); } } @@ -282,16 +283,25 @@ async function registerInjectables(origins) { const todo = []; if ( toRemove.length !== 0 ) { - todo.push(browser.scripting.unregisterContentScripts({ ids: toRemove })); console.info(`Unregistered ${toRemove} content (css/js)`); + todo.push( + browser.scripting.unregisterContentScripts({ ids: toRemove }) + .catch(reason => { console.info(reason); }) + ); } if ( toAdd.length !== 0 ) { - todo.push(browser.scripting.registerContentScripts(toAdd)); console.info(`Registered ${toAdd.map(v => v.id)} content (css/js)`); + todo.push( + browser.scripting.registerContentScripts(toAdd) + .catch(reason => { console.info(reason); }) + ); } if ( toUpdate.length !== 0 ) { - todo.push(browser.scripting.updateContentScripts(toUpdate)); console.info(`Updated ${toUpdate.map(v => v.id)} content (css/js)`); + todo.push( + browser.scripting.updateContentScripts(toUpdate) + .catch(reason => { console.info(reason); }) + ); } if ( todo.length === 0 ) { return; } diff --git a/platform/mv3/extension/js/utils.js b/platform/mv3/extension/js/utils.js index 23ba6a0c5..3ec940d42 100644 --- a/platform/mv3/extension/js/utils.js +++ b/platform/mv3/extension/js/utils.js @@ -46,10 +46,11 @@ const matchesFromHostnames = hostnames => { const out = []; for ( const hn of hostnames ) { if ( hn === '*' ) { + out.length = 0; out.push(''); - } else { - out.push(`*://*.${hn}/*`); + break; } + out.push(`*://*.${hn}/*`); } return out; }; @@ -58,8 +59,9 @@ const hostnamesFromMatches = origins => { const out = []; for ( const origin of origins ) { if ( origin === '' ) { + out.length = 0; out.push('*'); - continue; + break; } const match = /^\*:\/\/(?:\*\.)?([^\/]+)\/\*/.exec(origin); if ( match === null ) { continue; } diff --git a/platform/mv3/make-rulesets.js b/platform/mv3/make-rulesets.js index 3307c146a..20145cf95 100644 --- a/platform/mv3/make-rulesets.js +++ b/platform/mv3/make-rulesets.js @@ -72,6 +72,11 @@ const uidint32 = (s) => { return parseInt(h,16) & 0x7FFFFFFF; }; +const hnSort = (a, b) => + a.split('.').reverse().join('.').localeCompare( + b.split('.').reverse().join('.') + ); + /******************************************************************************/ const stdOutput = []; @@ -239,8 +244,8 @@ async function processNetworkFilters(assetDetails, network) { log(`\tRejected filter count: ${network.rejectedFilterCount}`); log(`Output rule count: ${rules.length}`); - const good = rules.filter(rule => isGood(rule) && isRegex(rule) === false); - log(`\tGood: ${good.length}`); + const plainGood = rules.filter(rule => isGood(rule) && isRegex(rule) === false); + log(`\tPlain good: ${plainGood.length}`); const regexes = rules.filter(rule => isGood(rule) && isRegex(rule)); log(`\tMaybe good (regexes): ${regexes.length}`); @@ -257,24 +262,23 @@ async function processNetworkFilters(assetDetails, network) { ); log(`\tcsp= (discarded): ${headers.length}`); - const removeparams = rules.filter(rule => - isUnsupported(rule) === false && - isRemoveparam(rule) + const removeparamsGood = rules.filter(rule => + isUnsupported(rule) === false && isRemoveparam(rule) ); - log(`\tremoveparams= (discarded): ${removeparams.length}`); + const removeparamsBad = rules.filter(rule => + isUnsupported(rule) && isRemoveparam(rule) + ); + log(`\tremoveparams= (accepted/discarded): ${removeparamsGood.length}/${removeparamsBad.length}`); const bad = rules.filter(rule => isUnsupported(rule) ); log(`\tUnsupported: ${bad.length}`); - log( - bad.map(rule => rule._error.map(v => `\t\t${v}`)).join('\n'), - true - ); + log(bad.map(rule => rule._error.map(v => `\t\t${v}`)).join('\n'), true); writeFile( `${rulesetDir}/${assetDetails.id}.json`, - `${JSON.stringify(good, replacer)}\n` + `${JSON.stringify(plainGood, replacer)}\n` ); if ( regexes.length !== 0 ) { @@ -284,12 +288,20 @@ async function processNetworkFilters(assetDetails, network) { ); } + if ( removeparamsGood.length !== 0 ) { + writeFile( + `${rulesetDir}/${assetDetails.id}.removeparams.json`, + `${JSON.stringify(removeparamsGood, replacer)}\n` + ); + } + return { total: rules.length, - accepted: good.length, - discarded: redirects.length + headers.length + removeparams.length, + plain: plainGood.length, + discarded: redirects.length + headers.length + removeparamsBad.length, rejected: bad.length, regexes: regexes.length, + removeparams: removeparamsGood.length, }; } @@ -344,25 +356,22 @@ function loadAllSourceScriptlets() { const globalPatchedScriptletsSet = new Set(); -function addScriptingAPIResources(id, entry, prop, fid) { - if ( entry[prop] === undefined ) { return; } - for ( const hn of entry[prop] ) { - let details = scriptingDetails.get(id); - if ( details === undefined ) { - details = {}; - scriptingDetails.set(id, details); +function addScriptingAPIResources(id, hostnames, fid) { + if ( hostnames === undefined ) { return; } + for ( const hn of hostnames ) { + let hostnamesToFidMap = scriptingDetails.get(id); + if ( hostnamesToFidMap === undefined ) { + hostnamesToFidMap = new Map(); + scriptingDetails.set(id, hostnamesToFidMap); } - if ( details[prop] === undefined ) { - details[prop] = new Map(); - } - let fids = details[prop].get(hn); + let fids = hostnamesToFidMap.get(hn); if ( fids === undefined ) { - details[prop].set(hn, fid); + hostnamesToFidMap.set(hn, fid); } else if ( fids instanceof Set ) { fids.add(fid); } else if ( fid !== fids ) { fids = new Set([ fids, fid ]); - details[prop].set(hn, fids); + hostnamesToFidMap.set(hn, fids); } } } @@ -432,10 +441,6 @@ function groupCosmeticBySelectors(arrayin) { } } } - const hnSort = (a, b) => - a.split('.').reverse().join('.').localeCompare( - b.split('.').reverse().join('.') - ); const out = Array.from(contentMap).map(a => [ a[0], { a: a[1].a, @@ -530,18 +535,7 @@ async function processCosmeticFilters(assetDetails, mapin) { generatedFiles.push(fname); } for ( const entry of slice ) { - addScriptingAPIResources( - assetDetails.id, - { matches: entry[1].y }, - 'matches', - fid - ); - addScriptingAPIResources( - assetDetails.id, - { excludeMatches: entry[1].n }, - 'excludeMatches', - fid - ); + addScriptingAPIResources(assetDetails.id, entry[1].y, fid); } } @@ -609,18 +603,7 @@ async function processProceduralCosmeticFilters(assetDetails, mapin) { generatedFiles.push(fname); } for ( const entry of slice ) { - addScriptingAPIResources( - assetDetails.id, - { matches: entry[1].y }, - 'matches', - fid - ); - addScriptingAPIResources( - assetDetails.id, - { excludeMatches: entry[1].n }, - 'excludeMatches', - fid - ); + addScriptingAPIResources(assetDetails.id, entry[1].y, fid); } } @@ -749,18 +732,7 @@ async function processScriptletFilters(assetDetails, mapin) { generatedFiles.push(fname); } for ( const details of argsDetails.values() ) { - addScriptingAPIResources( - assetDetails.id, - { matches: details.y }, - 'matches', - fid - ); - addScriptingAPIResources( - assetDetails.id, - { excludeMatches: details.n }, - 'excludeMatches', - fid - ); + addScriptingAPIResources(assetDetails.id, details.y, fid); } } @@ -844,10 +816,11 @@ const rulesetFromURLS = async function(assetDetails) { }, rules: { total: netStats.total, - accepted: netStats.accepted, + plain: netStats.plain, + regexes: netStats.regexes, + removeparams: netStats.removeparams, discarded: netStats.discarded, rejected: netStats.rejected, - regexes: netStats.regexes, }, css: { specific: cosmeticStats, @@ -941,7 +914,7 @@ async function main() { } // Handpicked rulesets from assets.json - const handpicked = [ 'block-lan', 'dpollock-0' ]; + const handpicked = [ 'block-lan', 'dpollock-0', 'adguard-spyware-url' ]; for ( const id of handpicked ) { const asset = assets[id]; if ( asset.content !== 'filters' ) { continue; } @@ -972,6 +945,14 @@ async function main() { `${JSON.stringify(rulesetDetails, null, 1)}\n` ); + // We sort the hostnames for convenience/performance in the extension's + // script manager -- the scripting API does a sort() internally. + for ( const [ rulesetId, hostnamesToFidsMap ] of scriptingDetails ) { + scriptingDetails.set( + rulesetId, + Array.from(hostnamesToFidsMap).sort() + ); + } writeFile( `${rulesetDir}/scripting-details.json`, `${JSON.stringify(scriptingDetails, jsonSetMapReplacer)}\n` diff --git a/src/js/static-net-filtering.js b/src/js/static-net-filtering.js index 8485cd001..b61dd3415 100644 --- a/src/js/static-net-filtering.js +++ b/src/js/static-net-filtering.js @@ -633,11 +633,6 @@ const dnrAddRuleError = (rule, msg) => { rule._error.push(msg); }; -const dnrAddRuleWarning = (rule, msg) => { - rule._warning = rule._warning || []; - rule._warning.push(msg); -}; - /******************************************************************************* Filter classes @@ -4050,7 +4045,7 @@ FilterContainer.prototype.dnrFromCompiled = function(op, context, ...args) { value: rule.__modifierValue, }]; if ( rule.__modifierAction === AllowAction ) { - dnrAddRuleError(rule, 'Unhandled modifier exception'); + dnrAddRuleError(rule, 'Unsupported modifier exception'); } break; case 'redirect-rule': { @@ -4063,7 +4058,7 @@ FilterContainer.prototype.dnrFromCompiled = function(op, context, ...args) { } const resource = context.extensionPaths.get(token); if ( rule.__modifierValue !== '' && resource === undefined ) { - dnrAddRuleWarning(rule, `Unpatchable redirect filter: ${rule.__modifierValue}`); + dnrAddRuleError(rule, `Unpatchable redirect filter: ${rule.__modifierValue}`); } const extensionPath = resource && resource.extensionPath || token; if ( rule.__modifierAction !== AllowAction ) { @@ -4078,6 +4073,9 @@ FilterContainer.prototype.dnrFromCompiled = function(op, context, ...args) { } case 'removeparam': rule.action.type = 'redirect'; + if ( rule.__modifierValue === '|' ) { + rule.__modifierValue = ''; + } if ( rule.__modifierValue !== '' ) { rule.action.redirect = { transform: { @@ -4108,7 +4106,7 @@ FilterContainer.prototype.dnrFromCompiled = function(op, context, ...args) { ]; } if ( rule.__modifierAction === AllowAction ) { - dnrAddRuleError(rule, 'Unhandled modifier exception'); + dnrAddRuleError(rule, 'Unsupported modifier exception'); } break; default: