From 266ec4894b09e3686225f044daa29fe94b0b14cf Mon Sep 17 00:00:00 2001 From: Raymond Hill Date: Sun, 15 Sep 2024 09:17:19 -0400 Subject: [PATCH] New static network filter option `urlskip=` Related issue: https://github.com/uBlockOrigin/uBlock-issues/issues/3206 The main purpose is to bypass URLs designed to track whether a user visited a specific URL, typically used in click-tracking links. The `urlskip=` option ... - ... is valid only when used in a trusted filter list - ... is enforced only on top documents - ... is enforced on both blocked and non-blocked documents - ... is a modifier, i.e. it cannot be used along with other modifier options in a single filter The syntax is `urlskip=[steps]`, where steps is a space-separated list of extraction directives detailing what action to perform on the current URL. The only supported directive in this first commit is `?name`, which purpose is to extract the value of a named URL parameter and use the result as the new URL. Example: ||example.com/path/to/tracker$urlskip=?url The above filter will cause navigation to https://example.com/path/to/tracker?url=https://example.org/ to automatically bypass navigation to `example.com` and navigate directly to https://example.org/ It is possible to recursively extract URL parameters by using more than one directive, example: ||example.com/path/to/tracker$urlskip=?url ?to More extraction capabilities may be added in the future. --- src/js/filtering-context.js | 3 + src/js/logger-ui.js | 2 +- src/js/pagestore.js | 32 ++-- src/js/static-filtering-parser.js | 19 ++ src/js/static-net-filtering.js | 277 ++++++++++++++++++------------ src/js/traffic.js | 11 +- 6 files changed, 216 insertions(+), 128 deletions(-) diff --git a/src/js/filtering-context.js b/src/js/filtering-context.js index 933785f55..a24cdabc5 100644 --- a/src/js/filtering-context.js +++ b/src/js/filtering-context.js @@ -163,6 +163,9 @@ export const FilteringContext = class { this.stype = a; } + isRootDocument() { + return (this.itype & MAIN_FRAME) !== 0; + } isDocument() { return (this.itype & FRAME_ANY) !== 0; } diff --git a/src/js/logger-ui.js b/src/js/logger-ui.js index db95f2e11..e76586a50 100644 --- a/src/js/logger-ui.js +++ b/src/js/logger-ui.js @@ -331,7 +331,7 @@ const processLoggerEntries = function(response) { parsed.type === 'main_frame' && parsed.aliased === false && ( parsed.filter === undefined || - parsed.filter.modifier !== true + parsed.filter.modifier !== true && parsed.filter.source !== 'redirect' ) ) { const separator = createLogSeparator(parsed, unboxed.url); diff --git a/src/js/pagestore.js b/src/js/pagestore.js index d6f52b8cd..e20149636 100644 --- a/src/js/pagestore.js +++ b/src/js/pagestore.js @@ -933,19 +933,14 @@ const PageStore = class { } redirectNonBlockedRequest(fctxt) { - const transformDirectives = staticNetFilteringEngine.transformRequest(fctxt); - const pruneDirectives = fctxt.redirectURL === undefined && - staticNetFilteringEngine.hasQuery(fctxt) && - staticNetFilteringEngine.filterQuery(fctxt) || - undefined; - if ( transformDirectives === undefined && pruneDirectives === undefined ) { return; } + const directives = []; + staticNetFilteringEngine.transformRequest(fctxt, directives); + if ( staticNetFilteringEngine.hasQuery(fctxt) ) { + staticNetFilteringEngine.filterQuery(fctxt, directives); + } + if ( directives.length === 0 ) { return; } if ( logger.enabled !== true ) { return; } - if ( transformDirectives !== undefined ) { - fctxt.pushFilters(transformDirectives.map(a => a.logData())); - } - if ( pruneDirectives !== undefined ) { - fctxt.pushFilters(pruneDirectives.map(a => a.logData())); - } + fctxt.pushFilters(directives.map(a => a.logData())); if ( fctxt.redirectURL === undefined ) { return; } fctxt.pushFilter({ source: 'redirect', @@ -953,6 +948,19 @@ const PageStore = class { }); } + skipMainDocument(fctxt) { + const directives = staticNetFilteringEngine.urlSkip(fctxt); + if ( directives === undefined ) { return; } + if ( logger.enabled !== true ) { return; } + fctxt.pushFilters(directives.map(a => a.logData())); + if ( fctxt.redirectURL !== undefined ) { + fctxt.pushFilter({ + source: 'redirect', + raw: fctxt.redirectURL + }); + } + } + filterCSPReport(fctxt) { if ( sessionSwitches.evaluateZ( diff --git a/src/js/static-filtering-parser.js b/src/js/static-filtering-parser.js index ee3318240..db5b2bd13 100644 --- a/src/js/static-filtering-parser.js +++ b/src/js/static-filtering-parser.js @@ -191,6 +191,7 @@ export const NODE_TYPE_NET_OPTION_NAME_REPLACE = iota++; export const NODE_TYPE_NET_OPTION_NAME_SCRIPT = iota++; export const NODE_TYPE_NET_OPTION_NAME_SHIDE = iota++; export const NODE_TYPE_NET_OPTION_NAME_TO = iota++; +export const NODE_TYPE_NET_OPTION_NAME_URLSKIP = iota++; export const NODE_TYPE_NET_OPTION_NAME_URLTRANSFORM = iota++; export const NODE_TYPE_NET_OPTION_NAME_XHR = iota++; export const NODE_TYPE_NET_OPTION_NAME_WEBRTC = iota++; @@ -274,6 +275,7 @@ export const nodeTypeFromOptionName = new Map([ [ 'shide', NODE_TYPE_NET_OPTION_NAME_SHIDE ], /* synonym */ [ 'specifichide', NODE_TYPE_NET_OPTION_NAME_SHIDE ], [ 'to', NODE_TYPE_NET_OPTION_NAME_TO ], + [ 'urlskip', NODE_TYPE_NET_OPTION_NAME_URLSKIP ], [ 'uritransform', NODE_TYPE_NET_OPTION_NAME_URLTRANSFORM ], [ 'xhr', NODE_TYPE_NET_OPTION_NAME_XHR ], /* synonym */ [ 'xmlhttprequest', NODE_TYPE_NET_OPTION_NAME_XHR ], @@ -1441,6 +1443,7 @@ export class AstFilterParser { case NODE_TYPE_NET_OPTION_NAME_REDIRECT: case NODE_TYPE_NET_OPTION_NAME_REDIRECTRULE: case NODE_TYPE_NET_OPTION_NAME_REPLACE: + case NODE_TYPE_NET_OPTION_NAME_URLSKIP: case NODE_TYPE_NET_OPTION_NAME_URLTRANSFORM: realBad = isNegated || (isException || hasValue) === false || modifierType !== 0; @@ -1519,6 +1522,21 @@ export class AstFilterParser { } break; } + case NODE_TYPE_NET_OPTION_NAME_URLSKIP: { + realBad = abstractTypeCount || behaviorTypeCount || unredirectableTypeCount; + if ( realBad ) { break; } + if ( requiresTrustedSource() ) { + this.astError = AST_ERROR_UNTRUSTED_SOURCE; + realBad = true; + break; + } + const value = this.getNetOptionValue(NODE_TYPE_NET_OPTION_NAME_URLSKIP); + if ( value.startsWith('?') === false || value.length < 2 ) { + this.astError = AST_ERROR_OPTION_BADVALUE; + realBad = true; + } + break; + } case NODE_TYPE_NET_OPTION_NAME_URLTRANSFORM: { realBad = abstractTypeCount || behaviorTypeCount || unredirectableTypeCount; if ( realBad ) { break; } @@ -3139,6 +3157,7 @@ export const netOptionTokenDescriptors = new Map([ [ 'shide', { } ], /* synonym */ [ 'specifichide', { } ], [ 'to', { mustAssign: true } ], + [ 'urlskip', { mustAssign: true } ], [ 'uritransform', { mustAssign: true } ], [ 'xhr', { canNegate: true } ], /* synonym */ [ 'xmlhttprequest', { canNegate: true } ], diff --git a/src/js/static-net-filtering.js b/src/js/static-net-filtering.js index 780015209..87216156f 100644 --- a/src/js/static-net-filtering.js +++ b/src/js/static-net-filtering.js @@ -43,97 +43,99 @@ const keyvalStore = typeof vAPI !== 'undefined' /******************************************************************************/ -// 0fedcba9876543210 -// ||||||| | || | -// ||||||| | || | -// ||||||| | || | -// ||||||| | || | -// ||||||| | || +---- bit 0- 1: block=0, allow=1, block important=2 -// ||||||| | |+------ bit 2: unused -// ||||||| | +------- bit 3- 4: party [0-3] -// ||||||| +--------- bit 5- 9: type [0-31] -// ||||||+-------------- bit 10: headers-based filters -// |||||+--------------- bit 11: redirect filters -// ||||+---------------- bit 12: removeparam filters -// |||+----------------- bit 13: csp filters -// ||+------------------ bit 14: permissions filters -// |+------------------- bit 15: uritransform filters -// +-------------------- bit 16: replace filters -// TODO: bit 11-16 can be converted into 3-bit value, as these options are not +// 10fedcba9876543210 +// |||||||| | || | +// |||||||| | || | +// |||||||| | || | +// |||||||| | || | +// |||||||| | || +---- bit 0- 1: block=0, allow=1, block important=2 +// |||||||| | |+------ bit 2: unused +// |||||||| | +------- bit 3- 4: party [0-3] +// |||||||| +--------- bit 5- 9: type [0-31] +// |||||||+-------------- bit 10: headers-based filters +// ||||||+--------------- bit 11: redirect filters +// |||||+---------------- bit 12: removeparam filters +// ||||+----------------- bit 13: csp filters +// |||+------------------ bit 14: permissions filters +// ||+------------------- bit 15: uritransform filters +// |+-------------------- bit 16: replace filters +// +--------------------- bit 17: urlskip filters +// TODO: bit 11-17 could be converted into 3-bit value, as these options are not // meant to be combined. -const RealmBitsMask = 0b00000000111; -const ActionBitsMask = 0b00000000011; -const TypeBitsMask = 0b01111100000; -const TypeBitsOffset = 5; - -const BLOCK_REALM = 0b00000000000000000; -const ALLOW_REALM = 0b00000000000000001; -const IMPORTANT_REALM = 0b00000000000000010; +const BLOCK_REALM = 0b0000_0000_0000_0000_0000; +const ALLOW_REALM = 0b0000_0000_0000_0000_0001; +const IMPORTANT_REALM = 0b0000_0000_0000_0000_0010; +const BLOCKALLOW_REALM = BLOCK_REALM | ALLOW_REALM | IMPORTANT_REALM; const BLOCKIMPORTANT_REALM = BLOCK_REALM | IMPORTANT_REALM; -const ANYPARTY_REALM = 0b00000000000000000; -const FIRSTPARTY_REALM = 0b00000000000001000; -const THIRDPARTY_REALM = 0b00000000000010000; +const ANYPARTY_REALM = 0b0000_0000_0000_0000_0000; +const FIRSTPARTY_REALM = 0b0000_0000_0000_0000_1000; +const THIRDPARTY_REALM = 0b0000_0000_0000_0001_0000; const ALLPARTIES_REALM = FIRSTPARTY_REALM | THIRDPARTY_REALM; -const HEADERS_REALM = 0b00000010000000000; -const REDIRECT_REALM = 0b00000100000000000; -const REMOVEPARAM_REALM = 0b00001000000000000; -const CSP_REALM = 0b00010000000000000; -const PERMISSIONS_REALM = 0b00100000000000000; -const URLTRANSFORM_REALM = 0b01000000000000000; -const REPLACE_REALM = 0b10000000000000000; +const TYPE_REALM = 0b0000_0000_0011_1110_0000; +const HEADERS_REALM = 0b0000_0000_0100_0000_0000; +const REDIRECT_REALM = 0b0000_0000_1000_0000_0000; +const REMOVEPARAM_REALM = 0b0000_0001_0000_0000_0000; +const CSP_REALM = 0b0000_0010_0000_0000_0000; +const PERMISSIONS_REALM = 0b0000_0100_0000_0000_0000; +const URLTRANSFORM_REALM = 0b0000_1000_0000_0000_0000; +const REPLACE_REALM = 0b0001_0000_0000_0000_0000; +const URLSKIP_REALM = 0b0010_0000_0000_0000_0000; const MODIFY_REALMS = REDIRECT_REALM | CSP_REALM | REMOVEPARAM_REALM | PERMISSIONS_REALM | - URLTRANSFORM_REALM | REPLACE_REALM; + URLTRANSFORM_REALM | REPLACE_REALM | + URLSKIP_REALM; + +const TYPE_REALM_OFFSET = 5; const typeNameToTypeValue = { - 'no_type': 0 << TypeBitsOffset, - 'stylesheet': 1 << TypeBitsOffset, - 'image': 2 << TypeBitsOffset, - 'object': 3 << TypeBitsOffset, - 'object_subrequest': 3 << TypeBitsOffset, - 'script': 4 << TypeBitsOffset, - 'fetch': 5 << TypeBitsOffset, - 'xmlhttprequest': 5 << TypeBitsOffset, - 'sub_frame': 6 << TypeBitsOffset, - 'font': 7 << TypeBitsOffset, - 'media': 8 << TypeBitsOffset, - 'websocket': 9 << TypeBitsOffset, - 'beacon': 10 << TypeBitsOffset, - 'ping': 10 << TypeBitsOffset, - 'other': 11 << TypeBitsOffset, - 'popup': 12 << TypeBitsOffset, // start of behavioral filtering - 'popunder': 13 << TypeBitsOffset, - 'main_frame': 14 << TypeBitsOffset, // start of 1p behavioral filtering - 'generichide': 15 << TypeBitsOffset, - 'specifichide': 16 << TypeBitsOffset, - 'inline-font': 17 << TypeBitsOffset, - 'inline-script': 18 << TypeBitsOffset, - 'cname': 19 << TypeBitsOffset, - 'webrtc': 20 << TypeBitsOffset, - 'unsupported': 21 << TypeBitsOffset, + 'no_type': 0 << TYPE_REALM_OFFSET, + 'stylesheet': 1 << TYPE_REALM_OFFSET, + 'image': 2 << TYPE_REALM_OFFSET, + 'object': 3 << TYPE_REALM_OFFSET, + 'object_subrequest': 3 << TYPE_REALM_OFFSET, + 'script': 4 << TYPE_REALM_OFFSET, + 'fetch': 5 << TYPE_REALM_OFFSET, + 'xmlhttprequest': 5 << TYPE_REALM_OFFSET, + 'sub_frame': 6 << TYPE_REALM_OFFSET, + 'font': 7 << TYPE_REALM_OFFSET, + 'media': 8 << TYPE_REALM_OFFSET, + 'websocket': 9 << TYPE_REALM_OFFSET, + 'beacon': 10 << TYPE_REALM_OFFSET, + 'ping': 10 << TYPE_REALM_OFFSET, + 'other': 11 << TYPE_REALM_OFFSET, + 'popup': 12 << TYPE_REALM_OFFSET, // start of behavioral filtering + 'popunder': 13 << TYPE_REALM_OFFSET, + 'main_frame': 14 << TYPE_REALM_OFFSET, // start of 1p behavioral filtering + 'generichide': 15 << TYPE_REALM_OFFSET, + 'specifichide': 16 << TYPE_REALM_OFFSET, + 'inline-font': 17 << TYPE_REALM_OFFSET, + 'inline-script': 18 << TYPE_REALM_OFFSET, + 'cname': 19 << TYPE_REALM_OFFSET, + 'webrtc': 20 << TYPE_REALM_OFFSET, + 'unsupported': 21 << TYPE_REALM_OFFSET, }; const otherTypeBitValue = typeNameToTypeValue.other; const bitFromType = type => - 1 << ((typeNameToTypeValue[type] >>> TypeBitsOffset) - 1); + 1 << ((typeNameToTypeValue[type] >>> TYPE_REALM_OFFSET) - 1); // All network request types to bitmap -// bring origin to 0 (from TypeBitsOffset -- see typeNameToTypeValue) +// bring origin to 0 (from TYPE_REALM_OFFSET -- see typeNameToTypeValue) // left-shift 1 by the above-calculated value // subtract 1 to set all type bits const allNetworkTypesBits = - (1 << (otherTypeBitValue >>> TypeBitsOffset)) - 1; + (1 << (otherTypeBitValue >>> TYPE_REALM_OFFSET)) - 1; const allTypesBits = allNetworkTypesBits | - 1 << (typeNameToTypeValue['popup'] >>> TypeBitsOffset) - 1 | - 1 << (typeNameToTypeValue['main_frame'] >>> TypeBitsOffset) - 1 | - 1 << (typeNameToTypeValue['inline-font'] >>> TypeBitsOffset) - 1 | - 1 << (typeNameToTypeValue['inline-script'] >>> TypeBitsOffset) - 1; + 1 << (typeNameToTypeValue['popup'] >>> TYPE_REALM_OFFSET) - 1 | + 1 << (typeNameToTypeValue['main_frame'] >>> TYPE_REALM_OFFSET) - 1 | + 1 << (typeNameToTypeValue['inline-font'] >>> TYPE_REALM_OFFSET) - 1 | + 1 << (typeNameToTypeValue['inline-script'] >>> TYPE_REALM_OFFSET) - 1; const unsupportedTypeBit = - 1 << (typeNameToTypeValue['unsupported'] >>> TypeBitsOffset) - 1; + 1 << (typeNameToTypeValue['unsupported'] >>> TYPE_REALM_OFFSET) - 1; const typeValueToTypeName = [ '', @@ -186,6 +188,7 @@ const MODIFIER_TYPE_CSP = 4; const MODIFIER_TYPE_PERMISSIONS = 5; const MODIFIER_TYPE_URLTRANSFORM = 6; const MODIFIER_TYPE_REPLACE = 7; +const MODIFIER_TYPE_URLSKIP = 8; const modifierBitsFromType = new Map([ [ MODIFIER_TYPE_REDIRECT, REDIRECT_REALM ], @@ -195,6 +198,7 @@ const modifierBitsFromType = new Map([ [ MODIFIER_TYPE_PERMISSIONS, PERMISSIONS_REALM ], [ MODIFIER_TYPE_URLTRANSFORM, URLTRANSFORM_REALM ], [ MODIFIER_TYPE_REPLACE, REPLACE_REALM ], + [ MODIFIER_TYPE_URLSKIP, URLSKIP_REALM ], ]); const modifierTypeFromName = new Map([ @@ -205,6 +209,7 @@ const modifierTypeFromName = new Map([ [ 'permissions', MODIFIER_TYPE_PERMISSIONS ], [ 'uritransform', MODIFIER_TYPE_URLTRANSFORM ], [ 'replace', MODIFIER_TYPE_REPLACE ], + [ 'urlskip', MODIFIER_TYPE_URLSKIP ], ]); const modifierNameFromType = new Map([ @@ -215,22 +220,23 @@ const modifierNameFromType = new Map([ [ MODIFIER_TYPE_PERMISSIONS, 'permissions' ], [ MODIFIER_TYPE_URLTRANSFORM, 'uritransform' ], [ MODIFIER_TYPE_REPLACE, 'replace' ], + [ MODIFIER_TYPE_URLSKIP, 'urlskip' ], ]); -//const typeValueFromCatBits = catBits => (catBits >>> TypeBitsOffset) & 0b11111; +//const typeValueFromCatBits = catBits => (catBits >>> TYPE_REALM_OFFSET) & 0b11111; const MAX_TOKEN_LENGTH = 7; // Four upper bits of token hash are reserved for built-in predefined // token hashes, which should never end up being used when tokenizing // any arbitrary string. -const NO_TOKEN_HASH = 0x50000000; -const DOT_TOKEN_HASH = 0x10000000; -const ANY_TOKEN_HASH = 0x20000000; -const ANY_HTTPS_TOKEN_HASH = 0x30000000; -const ANY_HTTP_TOKEN_HASH = 0x40000000; -const EMPTY_TOKEN_HASH = 0xF0000000; -const INVALID_TOKEN_HASH = 0xFFFFFFFF; +const NO_TOKEN_HASH = 0x5000_0000; +const DOT_TOKEN_HASH = 0x1000_0000; +const ANY_TOKEN_HASH = 0x2000_0000; +const ANY_HTTPS_TOKEN_HASH = 0x3000_0000; +const ANY_HTTP_TOKEN_HASH = 0x4000_0000; +const EMPTY_TOKEN_HASH = 0xF000_0000; +const INVALID_TOKEN_HASH = 0xFFFF_FFFF; /******************************************************************************/ @@ -374,9 +380,9 @@ class LogData { } else if ( (categoryBits & FIRSTPARTY_REALM) !== 0 ) { logData.options.unshift('1p'); } - const type = categoryBits & TypeBitsMask; + const type = categoryBits & TYPE_REALM; if ( type !== 0 ) { - logData.options.unshift(typeValueToTypeName[type >>> TypeBitsOffset]); + logData.options.unshift(typeValueToTypeName[type >>> TYPE_REALM_OFFSET]); } let raw = logData.pattern.join(''); if ( @@ -2163,7 +2169,7 @@ class FilterModifierResult { this.refs = filterRefs[filterData[imodifierunit+3]]; this.ireportedunit = env.iunit; this.th = env.th; - this.bits = (env.bits & ~RealmBitsMask) | filterData[imodifierunit+1]; + this.bits = (env.bits & ~BLOCKALLOW_REALM) | filterData[imodifierunit+1]; } get result() { @@ -3276,6 +3282,7 @@ class FilterCompiler { [ sfp.NODE_TYPE_NET_OPTION_NAME_REMOVEPARAM, MODIFIER_TYPE_REMOVEPARAM ], [ sfp.NODE_TYPE_NET_OPTION_NAME_URLTRANSFORM, MODIFIER_TYPE_URLTRANSFORM ], [ sfp.NODE_TYPE_NET_OPTION_NAME_REPLACE, MODIFIER_TYPE_REPLACE ], + [ sfp.NODE_TYPE_NET_OPTION_NAME_URLSKIP, MODIFIER_TYPE_URLSKIP ], ]); // These top 100 "bad tokens" are collated using the "miss" histogram // from tokenHistograms(). The "score" is their occurrence among the @@ -3548,6 +3555,7 @@ class FilterCompiler { case sfp.NODE_TYPE_NET_OPTION_NAME_REDIRECTRULE: case sfp.NODE_TYPE_NET_OPTION_NAME_REMOVEPARAM: case sfp.NODE_TYPE_NET_OPTION_NAME_REPLACE: + case sfp.NODE_TYPE_NET_OPTION_NAME_URLSKIP: case sfp.NODE_TYPE_NET_OPTION_NAME_URLTRANSFORM: if ( this.processModifierOption(id, parser.getNetOptionValue(id)) === false ) { return false; @@ -3667,6 +3675,7 @@ class FilterCompiler { case sfp.NODE_TYPE_NET_OPTION_NAME_REMOVEPARAM: case sfp.NODE_TYPE_NET_OPTION_NAME_REPLACE: case sfp.NODE_TYPE_NET_OPTION_NAME_TO: + case sfp.NODE_TYPE_NET_OPTION_NAME_URLSKIP: case sfp.NODE_TYPE_NET_OPTION_NAME_URLTRANSFORM: if ( this.processOptionWithValue(parser, type) === false ) { return this.FILTER_INVALID; @@ -4054,7 +4063,7 @@ class FilterCompiler { // IMPORTANT: the modifier unit MUST always appear first in a sequence if ( this.modifyType !== undefined ) { units.unshift(FilterModifier.compile(this)); - this.action = (this.action & ~ActionBitsMask) | + this.action = (this.action & ~BLOCKALLOW_REALM) | modifierBitsFromType.get(this.modifyType); } @@ -4123,7 +4132,7 @@ class FilterCompiler { do { if ( typeBits & 1 ) { writer.push([ - catBits | (bitOffset << TypeBitsOffset), + catBits | (bitOffset << TYPE_REALM_OFFSET), this.tokenHash, fdata ]); @@ -4286,7 +4295,7 @@ StaticNetFilteringEngine.prototype.freeze = function() { // the block-important realm should be checked when and only when // there is a matched exception filter, which important filters are // meant to override. - if ( (bits & ActionBitsMask) === BLOCKIMPORTANT_REALM ) { + if ( (bits & BLOCKALLOW_REALM) === BLOCKIMPORTANT_REALM ) { this.addFilterUnit( bits & ~IMPORTANT_REALM, tokenHash, @@ -4446,6 +4455,7 @@ StaticNetFilteringEngine.prototype.dnrFromCompiled = function(op, context, ...ar [ PERMISSIONS_REALM, { type: 'permissions', priority: 0 } ], [ URLTRANSFORM_REALM, { type: 'uritransform', priority: 0 } ], [ HEADERS_REALM, { type: 'block', priority: 0 } ], + [ URLSKIP_REALM, { type: 'urlskip', priority: 0 } ], ]); const partyness = new Map([ [ ANYPARTY_REALM, '' ], @@ -4860,7 +4870,7 @@ StaticNetFilteringEngine.prototype.matchAndFetchModifiers = function( $docDomain = fctxt.getDocDomain(); $requestHostname = fctxt.getHostname(); $requestMethodBit = fctxt.method || 0; - $requestTypeValue = (typeBits & TypeBitsMask) >>> TypeBitsOffset; + $requestTypeValue = (typeBits & TYPE_REALM) >>> TYPE_REALM_OFFSET; $requestAddress = fctxt.getIPAddress(); const modifierType = modifierTypeFromName.get(modifierName); @@ -4955,7 +4965,7 @@ StaticNetFilteringEngine.prototype.matchAndFetchModifiers = function( const toRemove = new Map(); for ( const result of results ) { - const actionBits = result.bits & ActionBitsMask; + const actionBits = result.bits & BLOCKALLOW_REALM; const modifyValue = result.value; if ( actionBits === BLOCKIMPORTANT_REALM ) { toAddImportant.set(modifyValue, result); @@ -5158,7 +5168,7 @@ StaticNetFilteringEngine.prototype.matchRequestReverse = function(type, url) { $requestURL = urlTokenizer.setURL(url); $requestURLRaw = url; $requestMethodBit = 0; - $requestTypeValue = (typeBits & TypeBitsMask) >>> TypeBitsOffset; + $requestTypeValue = (typeBits & TYPE_REALM) >>> TYPE_REALM_OFFSET; $requestAddress = ''; $isBlockImportant = false; this.$filterUnit = 0; @@ -5227,7 +5237,7 @@ StaticNetFilteringEngine.prototype.matchRequest = function(fctxt, modifiers = 0) $docDomain = fctxt.getDocDomain(); $requestHostname = fctxt.getHostname(); $requestMethodBit = fctxt.method || 0; - $requestTypeValue = (typeBits & TypeBitsMask) >>> TypeBitsOffset; + $requestTypeValue = (typeBits & TYPE_REALM) >>> TYPE_REALM_OFFSET; $requestAddress = fctxt.getIPAddress(); $isBlockImportant = false; @@ -5263,7 +5273,7 @@ StaticNetFilteringEngine.prototype.matchHeaders = function(fctxt, headers) { $docDomain = fctxt.getDocDomain(); $requestHostname = fctxt.getHostname(); $requestMethodBit = fctxt.method || 0; - $requestTypeValue = (typeBits & TypeBitsMask) >>> TypeBitsOffset; + $requestTypeValue = (typeBits & TYPE_REALM) >>> TYPE_REALM_OFFSET; $requestAddress = fctxt.getIPAddress(); $httpHeaders.init(headers); @@ -5309,11 +5319,35 @@ StaticNetFilteringEngine.prototype.redirectRequest = function(redirectEngine, fc return directives; }; -StaticNetFilteringEngine.prototype.transformRequest = function(fctxt) { +function parseRedirectRequestValue(directive) { + if ( directive.cache === null ) { + directive.cache = sfp.parseRedirectValue(directive.value); + } + return directive.cache; +} + +function compareRedirectRequests(redirectEngine, a, b) { + const { token: atok, priority: aint, bits: abits } = + parseRedirectRequestValue(a); + if ( redirectEngine.hasToken(atok) === false ) { return -1; } + const { token: btok, priority: bint, bits: bbits } = + parseRedirectRequestValue(b); + if ( redirectEngine.hasToken(btok) === false ) { return 1; } + if ( abits !== bbits ) { + if ( (abits & IMPORTANT_REALM) !== 0 ) { return 1; } + if ( (bbits & IMPORTANT_REALM) !== 0 ) { return -1; } + if ( (abits & ALLOW_REALM) !== 0 ) { return -1; } + if ( (bbits & ALLOW_REALM) !== 0 ) { return 1; } + } + return aint - bint; +} + +/******************************************************************************/ + +StaticNetFilteringEngine.prototype.transformRequest = function(fctxt, out = []) { const directives = this.matchAndFetchModifiers(fctxt, 'uritransform'); if ( directives === undefined ) { return; } const redirectURL = new URL(fctxt.url); - const out = []; for ( const directive of directives ) { if ( (directive.bits & ALLOW_REALM) !== 0 ) { out.push(directive); @@ -5345,27 +5379,47 @@ StaticNetFilteringEngine.prototype.transformRequest = function(fctxt) { return out; }; -function parseRedirectRequestValue(directive) { - if ( directive.cache === null ) { - directive.cache = sfp.parseRedirectValue(directive.value); - } - return directive.cache; -} +/******************************************************************************/ -function compareRedirectRequests(redirectEngine, a, b) { - const { token: atok, priority: aint, bits: abits } = - parseRedirectRequestValue(a); - if ( redirectEngine.hasToken(atok) === false ) { return -1; } - const { token: btok, priority: bint, bits: bbits } = - parseRedirectRequestValue(b); - if ( redirectEngine.hasToken(btok) === false ) { return 1; } - if ( abits !== bbits ) { - if ( (abits & IMPORTANT_REALM) !== 0 ) { return 1; } - if ( (bbits & IMPORTANT_REALM) !== 0 ) { return -1; } - if ( (abits & ALLOW_REALM) !== 0 ) { return -1; } - if ( (bbits & ALLOW_REALM) !== 0 ) { return 1; } +StaticNetFilteringEngine.prototype.urlSkip = function(fctxt, out = []) { + if ( fctxt.redirectURL !== undefined ) { return; } + const directives = this.matchAndFetchModifiers(fctxt, 'urlskip'); + if ( directives === undefined ) { return; } + for ( const directive of directives ) { + if ( (directive.bits & ALLOW_REALM) !== 0 ) { + out.push(directive); + continue; + } + const urlin = fctxt.url; + const value = directive.value; + const steps = value.includes(' ') && value.split(/ +/) || [ value ]; + const urlout = urlSkip(urlin, steps); + if ( urlout === undefined ) { continue; } + if ( urlout === urlin ) { continue; } + fctxt.redirectURL = urlout; + out.push(directive); + break; + } + if ( out.length === 0 ) { return; } + return out; +}; + +function urlSkip(urlin, steps) { + try { + let urlout; + for ( const step of steps ) { + if ( step.startsWith('?') === false ) { return; } + urlout = (new URL(urlin)).searchParams.get(step.slice(1)); + if ( urlout === null ) { return; } + if ( urlout.includes(' ') ) { + urlout = urlout.replace(/ /g, '%20'); + } + urlin = urlout; + } + void new URL(urlout); + return urlout; + } catch(x) { } - return aint - bint; } /******************************************************************************/ @@ -5373,7 +5427,8 @@ function compareRedirectRequests(redirectEngine, a, b) { // https://github.com/uBlockOrigin/uBlock-issues/issues/1626 // Do not redirect when the number of query parameters does not change. -StaticNetFilteringEngine.prototype.filterQuery = function(fctxt) { +StaticNetFilteringEngine.prototype.filterQuery = function(fctxt, out = []) { + if ( fctxt.redirectURL !== undefined ) { return; } const directives = this.matchAndFetchModifiers(fctxt, 'removeparam'); if ( directives === undefined ) { return; } const url = fctxt.url; @@ -5396,7 +5451,6 @@ StaticNetFilteringEngine.prototype.filterQuery = function(fctxt) { } } const inParamCount = params.size; - const out = []; for ( const directive of directives ) { if ( params.size === 0 ) { break; } const isException = (directive.bits & ALLOW_REALM) !== 0; @@ -5664,6 +5718,7 @@ StaticNetFilteringEngine.prototype.dump = function() { [ PERMISSIONS_REALM, 'permissions' ], [ URLTRANSFORM_REALM, 'uritransform' ], [ REPLACE_REALM, 'replace' ], + [ URLSKIP_REALM, 'urlskip' ], ]); const partyness = new Map([ [ ANYPARTY_REALM, 'any-party' ], diff --git a/src/js/traffic.js b/src/js/traffic.js index 38ba7264b..55538fa7e 100644 --- a/src/js/traffic.js +++ b/src/js/traffic.js @@ -188,17 +188,20 @@ const onBeforeRootFrameRequest = function(fctxt) { } if ( logger.enabled ) { - fctxt.setFilter(logData); + fctxt.setRealm('network').setFilter(logData); } // https://github.com/uBlockOrigin/uBlock-issues/issues/760 // Redirect non-blocked request? - if ( result !== 1 && trusted === false && pageStore !== null ) { - pageStore.redirectNonBlockedRequest(fctxt); + if ( trusted === false && pageStore !== null ) { + if ( result !== 1 ) { + pageStore.redirectNonBlockedRequest(fctxt); + } + pageStore.skipMainDocument(fctxt); } if ( logger.enabled ) { - fctxt.setRealm('network').toLogger(); + fctxt.toLogger(); } // Redirected