From f3e6057e07d586d62d1ec405d7aa6465c41929d4 Mon Sep 17 00:00:00 2001 From: gorhill Date: Thu, 25 May 2017 17:46:59 -0400 Subject: [PATCH] fix #2598: refactor to address the cause rather than the symptoms --- src/js/background.js | 4 +- src/js/cosmetic-filtering.js | 515 +++++++++++----------------- src/js/reverselookup-worker.js | 203 ++++++----- src/js/reverselookup.js | 2 +- src/js/static-net-filtering.js | 596 ++++++++++++++------------------- src/js/storage.js | 30 +- src/js/utils.js | 87 ++--- 7 files changed, 618 insertions(+), 819 deletions(-) diff --git a/src/js/background.js b/src/js/background.js index 6102941da..d6140465c 100644 --- a/src/js/background.js +++ b/src/js/background.js @@ -121,8 +121,8 @@ var µBlock = (function() { // jshint ignore:line // read-only systemSettings: { - compiledMagic: 'alufjifllsxz', - selfieMagic: 'alufjifllsxz' + compiledMagic: 'ohszqbtqggmp', + selfieMagic: 'ohszqbtqggmp' }, restoreBackupSettings: { diff --git a/src/js/cosmetic-filtering.js b/src/js/cosmetic-filtering.js index 8b069369f..1f918cd0d 100644 --- a/src/js/cosmetic-filtering.js +++ b/src/js/cosmetic-filtering.js @@ -34,11 +34,6 @@ var µb = µBlock; /******************************************************************************/ -// Could be replaced with encodeURIComponent/decodeURIComponent, -// which seems faster on Firefox. -var encode = JSON.stringify; -var decode = JSON.parse; - var isValidCSSSelector = (function() { var div = document.createElement('div'), matchesFn; @@ -118,56 +113,23 @@ var histogram = function(label, buckets) { console.log('\tTotal buckets count: %d', total); }; */ -/******************************************************************************/ +/******************************************************************************* -// Pure id- and class-based filters -// Examples: -// #A9AdsMiddleBoxTop -// .AD-POST + Each filter class will register itself in the map. -var FilterPlain = function() { + IMPORTANT: any change which modifies the mapping will have to be + reflected with µBlock.systemSettings.compiledMagic. + +**/ + +var filterClasses = []; + +var registerFilterClass = function(ctor) { + filterClasses[ctor.prototype.fid] = ctor; }; -FilterPlain.prototype.retrieve = function(s, out) { - out.push(s); -}; - -FilterPlain.prototype.fid = '#'; - -FilterPlain.prototype.toSelfie = function() { -}; - -FilterPlain.fromSelfie = function() { - return filterPlain; -}; - -var filterPlain = new FilterPlain(); - -/******************************************************************************/ - -// Id- and class-based filters with extra selector stuff following. -// Examples: -// #center_col > div[style="font-size:14px;margin-right:0;min-height:5px"] ... -// #adframe:not(frameset) -// .l-container > #fishtank -// body #sliding-popup - -var FilterPlainMore = function(s) { - this.s = s; -}; - -FilterPlainMore.prototype.retrieve = function(s, out) { - out.push(this.s); -}; - -FilterPlainMore.prototype.fid = '#+'; - -FilterPlainMore.prototype.toSelfie = function() { - return this.s; -}; - -FilterPlainMore.fromSelfie = function(s) { - return new FilterPlainMore(s); +var filterFromCompiledData = function(args) { + return filterClasses[args[0]].load(args); }; /******************************************************************************/ @@ -187,23 +149,24 @@ var FilterHostname = function(s, hostname) { this.hostname = hostname; }; +FilterHostname.prototype.fid = 8; + FilterHostname.prototype.retrieve = function(hostname, out) { if ( hostname.endsWith(this.hostname) ) { out.push(this.s); } }; -FilterHostname.prototype.fid = 'h'; - -FilterHostname.prototype.toSelfie = function() { - return encode(this.s) + '\t' + this.hostname; +FilterHostname.prototype.compile = function() { + return [ this.fid, this.s, this.hostname ]; }; -FilterHostname.fromSelfie = function(s) { - var pos = s.indexOf('\t'); - return new FilterHostname(decode(s.slice(0, pos)), s.slice(pos + 1)); +FilterHostname.load = function(data) { + return new FilterHostname(data[1], data[2]); }; +registerFilterClass(FilterHostname); + /******************************************************************************/ var FilterBucket = function(a, b) { @@ -211,12 +174,12 @@ var FilterBucket = function(a, b) { this.filters = []; if ( a !== undefined ) { this.filters[0] = a; - if ( b !== undefined ) { - this.filters[1] = b; - } + this.filters[1] = b; } }; +FilterBucket.prototype.fid = 10; + FilterBucket.prototype.add = function(a) { this.filters.push(a); }; @@ -228,16 +191,26 @@ FilterBucket.prototype.retrieve = function(s, out) { } }; -FilterBucket.prototype.fid = '[]'; - -FilterBucket.prototype.toSelfie = function() { - return this.filters.length.toString(); +FilterBucket.prototype.compile = function() { + var out = [], + filters = this.filters; + for ( var i = 0, n = filters.length; i < n; i++ ) { + out[i] = filters[i].compile(); + } + return [ this.fid, out ]; }; -FilterBucket.fromSelfie = function() { - return new FilterBucket(); +FilterBucket.load = function(data) { + var bucket = new FilterBucket(), + entries = data[1]; + for ( var i = 0, n = entries.length; i < n; i++ ) { + bucket.filters[i] = filterFromCompiledData(entries[i]); + } + return bucket; }; +registerFilterClass(FilterBucket); + /******************************************************************************/ /******************************************************************************/ @@ -1029,7 +1002,7 @@ FilterContainer.prototype.keyFromSelector = function(selector) { /******************************************************************************/ -FilterContainer.prototype.compile = function(s, out) { +FilterContainer.prototype.compile = function(s, writer) { var parsed = this.parser.parse(s); if ( parsed.cosmetic === false ) { return false; @@ -1042,7 +1015,7 @@ FilterContainer.prototype.compile = function(s, out) { var hostnames = parsed.hostnames; var i = hostnames.length; if ( i === 0 ) { - this.compileGenericSelector(parsed, out); + this.compileGenericSelector(parsed, writer); return true; } @@ -1056,10 +1029,10 @@ FilterContainer.prototype.compile = function(s, out) { if ( hostname.startsWith('~') === false ) { applyGlobally = false; } - this.compileHostnameSelector(hostname, parsed, out); + this.compileHostnameSelector(hostname, parsed, writer); } if ( applyGlobally ) { - this.compileGenericSelector(parsed, out); + this.compileGenericSelector(parsed, writer); } return true; @@ -1067,17 +1040,17 @@ FilterContainer.prototype.compile = function(s, out) { /******************************************************************************/ -FilterContainer.prototype.compileGenericSelector = function(parsed, out) { +FilterContainer.prototype.compileGenericSelector = function(parsed, writer) { if ( parsed.unhide === 0 ) { - this.compileGenericHideSelector(parsed, out); + this.compileGenericHideSelector(parsed, writer); } else { - this.compileGenericUnhideSelector(parsed, out); + this.compileGenericUnhideSelector(parsed, writer); } }; /******************************************************************************/ -FilterContainer.prototype.compileGenericHideSelector = function(parsed, out) { +FilterContainer.prototype.compileGenericHideSelector = function(parsed, writer) { var selector = parsed.suffix, type = selector.charAt(0), key, matches; @@ -1089,12 +1062,12 @@ FilterContainer.prototype.compileGenericHideSelector = function(parsed, out) { // is valid, the regex took care of this. Most generic selector falls // into that category. if ( key === selector ) { - out.push(4, 'lg\v' + key); + writer.push([ 0 /* lg */, key ]); return; } // Composite CSS rule. if ( this.compileSelector(selector) !== undefined ) { - out.push(4, 'lg+\v' + key + '\v' + selector); + writer.push([ 1 /* lg+ */, key, selector ]); } return; } @@ -1105,21 +1078,14 @@ FilterContainer.prototype.compileGenericHideSelector = function(parsed, out) { // ["title"] and ["alt"] will go in high-low generic bin. if ( this.reHighLow.test(selector) ) { - out.push(4, 'hlg0\v' + selector); + writer.push([ 2 /* hlg0 */, selector ]); return; } // [href^="..."] will go in high-medium generic bin. matches = this.reHighMedium.exec(selector); if ( matches && matches.length === 2 ) { - out.push(4, 'hmg0\v' + matches[1] + '\v' + selector); - return; - } - - // script:contains(...) - // script:inject(...) - if ( this.reScriptSelector.test(selector) ) { - out.push(4, 'js\v0\v\v' + selector); + writer.push([ 3 /* hmg0 */, matches[1], selector ]); return; } @@ -1128,28 +1094,28 @@ FilterContainer.prototype.compileGenericHideSelector = function(parsed, out) { // as a low generic cosmetic filter. matches = this.rePlainSelectorEx.exec(selector); if ( matches && matches.length === 2 ) { - out.push(4, 'lg+\v' + matches[1] + '\v' + selector); + writer.push([ 1 /* lg+ */, matches[1], selector ]); return; } // All else: high-high generics. // Distinguish simple vs complex selectors. if ( selector.indexOf(' ') === -1 ) { - out.push(4, 'hhsg0\v' + selector); + writer.push([ 4 /* hhsg0 */, selector ]); } else { - out.push(4, 'hhcg0\v' + selector); + writer.push([ 5 /* hhcg0 */, selector ]); } }; /******************************************************************************/ -FilterContainer.prototype.compileGenericUnhideSelector = function(parsed, out) { +FilterContainer.prototype.compileGenericUnhideSelector = function(parsed, writer) { var selector = parsed.suffix; // script:contains(...) // script:inject(...) if ( this.reScriptSelector.test(selector) ) { - out.push(4, 'js\v1\v\v' + selector); + writer.push([ 6 /* js */, '!', '', selector ]); return; } @@ -1160,12 +1126,12 @@ FilterContainer.prototype.compileGenericUnhideSelector = function(parsed, out) { // https://github.com/chrisaljoudi/uBlock/issues/497 // All generic exception filters are put in the same bucket: they are // expected to be very rare. - out.push(4, 'g1\v' + compiled); + writer.push([ 7 /* g1 */, compiled ]); }; /******************************************************************************/ -FilterContainer.prototype.compileHostnameSelector = function(hostname, parsed, out) { +FilterContainer.prototype.compileHostnameSelector = function(hostname, parsed, writer) { // https://github.com/chrisaljoudi/uBlock/issues/145 var unhide = parsed.unhide; if ( hostname.startsWith('~') ) { @@ -1189,7 +1155,7 @@ FilterContainer.prototype.compileHostnameSelector = function(hostname, parsed, o if ( unhide ) { hash = '!' + hash; } - out.push(4, 'js\v' + hash + '\v' + hostname + '\v' + selector); + writer.push([ 6 /* js */, hash, hostname, selector ]); return; } @@ -1207,251 +1173,212 @@ FilterContainer.prototype.compileHostnameSelector = function(hostname, parsed, o hash = '!' + hash; } - out.push(4, 'h\v' + hash + '\v' + hostname + '\v' + compiled); + // h, hash, example.com, .promoted-tweet + // h, hash, example.*, .promoted-tweet + writer.push([ 8 /* h */, hash, hostname, compiled ]); }; /******************************************************************************/ FilterContainer.prototype.fromCompiledContent = function( - lineIter, + reader, skipGenericCosmetic, skipCosmetic ) { if ( skipCosmetic ) { - this.skipCompiledContent(lineIter); + this.skipCompiledContent(reader); return; } if ( skipGenericCosmetic ) { - this.skipGenericCompiledContent(lineIter); + this.skipGenericCompiledContent(reader); return; } - var lineBits, line, field0, field1, field2, field3, filter, bucket, - aCharCode = 'a'.charCodeAt(0), - fieldIter = new µb.FieldIterator('\v'); - - while ( lineIter.eot() === false ) { - lineBits = lineIter.charCodeAt(0) - aCharCode; - if ( (lineBits & 0x04) === 0 ) { - return; - } - line = lineIter.next(1); - if ( (lineBits & 0x02) !== 0 ) { - line = decodeURIComponent(line); - } + var fingerprint, args, filter, bucket; + while ( reader.next() === true ) { this.acceptedCount += 1; - if ( this.duplicateBuster.has(line) ) { + fingerprint = reader.fingerprint(); + if ( this.duplicateBuster.has(fingerprint) ) { this.discardedCount += 1; continue; } - this.duplicateBuster.add(line); + this.duplicateBuster.add(fingerprint); - field0 = fieldIter.first(line); - field1 = fieldIter.next(); + args = reader.args(); - // h [\v] hash [\v] example.com [\v] .promoted-tweet - // h [\v] hash [\v] example.* [\v] .promoted-tweet - if ( field0 === 'h' ) { - field2 = fieldIter.next(); - field3 = fieldIter.next(); - filter = new FilterHostname(field3, field2); - bucket = this.specificFilters.get(field1); + switch ( args[0] ) { + + // .largeAd + case 0: + bucket = this.lowGenericHideEx.get(args[1]); if ( bucket === undefined ) { - this.specificFilters.set(field1, filter); - } else if ( bucket instanceof FilterBucket ) { - bucket.add(filter); - } else { - this.specificFilters.set(field1, new FilterBucket(bucket, filter)); - } - continue; - } - - // lg [\v] .largeAd - if ( field0 === 'lg' ) { - bucket = this.lowGenericHideEx.get(field1); - if ( bucket === undefined ) { - this.lowGenericHide.add(field1); + this.lowGenericHide.add(args[1]); } else if ( Array.isArray(bucket) ) { - bucket.push(field1); + bucket.push(args[1]); } else { - this.lowGenericHideEx.set(field1, [ bucket, field1 ]); + this.lowGenericHideEx.set(args[1], [ bucket, args[1] ]); } this.lowGenericHideCount += 1; - continue; - } + break; - // lg+ [\v] .Mpopup [\v] .Mpopup + #Mad > #MadZone - if ( field0 === 'lg+' ) { - field2 = fieldIter.next(); - bucket = this.lowGenericHideEx.get(field1); + // .Mpopup, .Mpopup + #Mad > #MadZone + case 1: + bucket = this.lowGenericHideEx.get(args[1]); if ( bucket === undefined ) { - if ( this.lowGenericHide.has(field1) ) { - this.lowGenericHideEx.set(field1, [ field1, field2 ]); + if ( this.lowGenericHide.has(args[1]) ) { + this.lowGenericHideEx.set(args[1], [ args[1], args[2] ]); } else { - this.lowGenericHideEx.set(field1, field2); - this.lowGenericHide.add(field1); + this.lowGenericHideEx.set(args[1], args[2]); + this.lowGenericHide.add(args[1]); } } else if ( Array.isArray(bucket) ) { - bucket.push(field2); + bucket.push(args[2]); } else { - this.lowGenericHideEx.set(field1, [ bucket, field2 ]); + this.lowGenericHideEx.set(args[1], [ bucket, args[2] ]); } this.lowGenericHideCount += 1; - continue; - } + break; - if ( field0 === 'hlg0' ) { - this.highLowGenericHide[field1] = true; + // ["title"] + // ["alt"] + case 2: + this.highLowGenericHide[args[1]] = true; this.highLowGenericHideCount += 1; - continue; - } + break; - if ( field0 === 'hmg0' ) { - field2 = fieldIter.next(); - bucket = this.highMediumGenericHide[field1]; + // [href^="..."] + case 3: + bucket = this.highMediumGenericHide[args[1]]; if ( bucket === undefined ) { - this.highMediumGenericHide[field1] = field2; + this.highMediumGenericHide[args[1]] = args[2]; } else if ( Array.isArray(bucket) ) { - bucket.push(field2); + bucket.push(args[2]); } else { - this.highMediumGenericHide[field1] = [bucket, field2]; + this.highMediumGenericHide[args[1]] = [bucket, args[2]]; } this.highMediumGenericHideCount += 1; - continue; - } + break; - if ( field0 === 'hhsg0' ) { - this.highHighSimpleGenericHideArray.push(field1); + // High-high generic hide/simple selectors + // div[id^="allo"] + case 4: + this.highHighSimpleGenericHideArray.push(args[1]); this.highHighSimpleGenericHideCount += 1; - continue; - } + break; - if ( field0 === 'hhcg0' ) { - this.highHighComplexGenericHideArray.push(field1); + // High-high generic hide/complex selectors + // div[id^="allo"] > span + case 5: + this.highHighComplexGenericHideArray.push(args[1]); this.highHighComplexGenericHideCount += 1; - continue; - } + break; - // js [\v] hash [\v] example.com [\v] script:contains(...) - // js [\v] hash [\v] example.com [\v] script:inject(...) - if ( field0 === 'js' ) { - field2 = fieldIter.next(); - field3 = fieldIter.next(); - this.createScriptFilter(field1, field2, field3); - continue; - } + // js, hash, example.com, script:contains(...) + // js, hash, example.com, script:inject(...) + case 6: + this.createScriptFilter(args[1], args[2], args[3]); + break; // https://github.com/chrisaljoudi/uBlock/issues/497 // Generic exception filters: expected to be a rare occurrence. - if ( field0 === 'g1' ) { - this.genericDonthide.push(field1); - continue; - } + // #@#.tweet + case 7: + this.genericDonthide.push(args[1]); + break; - this.discardedCount += 1; + // h, hash, example.com, .promoted-tweet + // h, hash, example.*, .promoted-tweet + case 8: + filter = new FilterHostname(args[3], args[2]); + bucket = this.specificFilters.get(args[1]); + if ( bucket === undefined ) { + this.specificFilters.set(args[1], filter); + } else if ( bucket instanceof FilterBucket ) { + bucket.add(filter); + } else { + this.specificFilters.set(args[1], new FilterBucket(bucket, filter)); + } + break; + + default: + this.discardedCount += 1; + break; + } } }; /******************************************************************************/ -FilterContainer.prototype.skipGenericCompiledContent = function(lineIter) { - var lineBits, line, field0, field1, field2, field3, filter, bucket, - aCharCode = 'a'.charCodeAt(0), - fieldIter = new µb.FieldIterator('\v'); - - while ( lineIter.eot() === false ) { - lineBits = lineIter.charCodeAt(0) - aCharCode; - if ( (lineBits & 0x04) === 0 ) { - return; - } - line = lineIter.next(1); - if ( (lineBits & 0x02) !== 0 ) { - line = decodeURIComponent(line); - } +FilterContainer.prototype.skipGenericCompiledContent = function(reader) { + var fingerprint, args, filter, bucket; + while ( reader.next() === true ) { this.acceptedCount += 1; - if ( this.duplicateBuster.has(line) ) { + fingerprint = reader.fingerprint(); + if ( this.duplicateBuster.has(fingerprint) ) { this.discardedCount += 1; continue; } - field0 = fieldIter.first(line); - field1 = fieldIter.next(); + args = reader.args(); - // h [\v] hash [\v] example.com [\v] .promoted-tweet - // h [\v] hash [\v] example.* [\v] .promoted-tweet - if ( field0 === 'h' ) { - field2 = fieldIter.next(); - field3 = fieldIter.next(); - this.duplicateBuster.add(line); - filter = new FilterHostname(field3, field2); - bucket = this.specificFilters.get(field1); - if ( bucket === undefined ) { - this.specificFilters.set(field1, filter); - } else if ( bucket instanceof FilterBucket ) { - bucket.add(filter); - } else { - this.specificFilters.set(field1, new FilterBucket(bucket, filter)); - } - continue; - } + switch ( args[0] ) { - // js [\v] hash [\v] example.com [\v] script:contains(...) - // js [\v] hash [\v] example.com [\v] script:inject(...) - if ( field0 === 'js' ) { - field2 = fieldIter.next(); - field3 = fieldIter.next(); - this.duplicateBuster.add(line); - this.createScriptFilter(field1, field2, field3); - continue; - } + // js, hash, example.com, script:contains(...) + // js, hash, example.com, script:inject(...) + case 6: + this.duplicateBuster.add(fingerprint); + this.createScriptFilter(args[1], args[2], args[3]); + break; // https://github.com/chrisaljoudi/uBlock/issues/497 // Generic exception filters: expected to be a rare occurrence. - if ( field0 === 'g1' ) { - this.duplicateBuster.add(line); - this.genericDonthide.push(field1); - continue; - } + case 7: + this.duplicateBuster.add(fingerprint); + this.genericDonthide.push(args[1]); + break; - this.discardedCount += 1; + // h, hash, example.com, .promoted-tweet + // h, hash, example.*, .promoted-tweet + case 8: + this.duplicateBuster.add(fingerprint); + filter = new FilterHostname(args[3], args[2]); + bucket = this.specificFilters.get(args[1]); + if ( bucket === undefined ) { + this.specificFilters.set(args[1], filter); + } else if ( bucket instanceof FilterBucket ) { + bucket.add(filter); + } else { + this.specificFilters.set(args[1], new FilterBucket(bucket, filter)); + } + break; + + default: + this.discardedCount += 1; + break; + } } }; /******************************************************************************/ -FilterContainer.prototype.skipCompiledContent = function(lineIter) { - var lineBits, line, field0, field1, field2, field3, - aCharCode = 'a'.charCodeAt(0), - fieldIter = new µb.FieldIterator('\v'); - - while ( lineIter.eot() === false ) { - lineBits = lineIter.charCodeAt(0) - aCharCode; - if ( (lineBits & 0x04) === 0 ) { - return; - } - line = lineIter.next(1); - if ( (lineBits & 0x02) !== 0 ) { - line = decodeURIComponent(line); - } +FilterContainer.prototype.skipCompiledContent = function(reader) { + var fingerprint, args; + while ( reader.next() === true ) { this.acceptedCount += 1; - if ( this.duplicateBuster.has(line) ) { - this.discardedCount += 1; - continue; - } - field0 = fieldIter.first(line); + args = reader.args(); - // js [\v] hash [\v] example.com [\v] script:contains(...) - // js [\v] hash [\v] example.com [\v] script:inject(...) - if ( field0 === 'js' ) { - this.duplicateBuster.add(line); - field1 = fieldIter.next(); - field2 = fieldIter.next(); - field3 = fieldIter.next(); - this.createScriptFilter(field1, field2, field3); + // js, hash, example.com, script:contains(...) + // js, hash, example.com, script:inject(...) + if ( args[0] === 6 ) { + fingerprint = reader.fingerprint(); + if ( this.duplicateBuster.has(fingerprint) === false ) { + this.duplicateBuster.add(fingerprint); + this.createScriptFilter(args[1], args[2], args[3]); + } continue; } @@ -1665,21 +1592,12 @@ FilterContainer.prototype._reEscapeScriptArg = /[\\'"]/g; FilterContainer.prototype.toSelfie = function() { var selfieFromMap = function(map) { - var selfie = [], - bucket, ff, f; + var selfie = []; // Note: destructuring assignment not supported before Chromium 49. for ( var entry of map ) { - selfie.push('k\t' + entry[0]); - bucket = entry[1]; - selfie.push(bucket.fid + '\t' + bucket.toSelfie()); - if ( bucket.fid !== '[]' ) { continue; } - ff = bucket.filters; - for ( var j = 0, nj = ff.length; j < nj; j++ ) { - f = ff[j]; - selfie.push(f.fid + '\t' + f.toSelfie()); - } + selfie.push([ entry[0], entry[1].compile() ]); } - return selfie.join('\n'); + return JSON.stringify(selfie); }; return { @@ -1709,46 +1627,15 @@ FilterContainer.prototype.toSelfie = function() { /******************************************************************************/ FilterContainer.prototype.fromSelfie = function(selfie) { - var factories = { - '[]': FilterBucket, - '#': FilterPlain, - '#+': FilterPlainMore, - 'h': FilterHostname - }; - var mapFromSelfie = function(selfie) { - var map = new Map(), - key, - bucket = null, - rawText = selfie, - rawEnd = rawText.length, - lineBeg = 0, lineEnd, - line, pos, what, factory; - while ( lineBeg < rawEnd ) { - lineEnd = rawText.indexOf('\n', lineBeg); - if ( lineEnd < 0 ) { - lineEnd = rawEnd; - } - line = rawText.slice(lineBeg, lineEnd); - lineBeg = lineEnd + 1; - pos = line.indexOf('\t'); - what = line.slice(0, pos); - if ( what === 'k' ) { - key = line.slice(pos + 1); - bucket = null; - continue; - } - factory = factories[what]; - if ( bucket === null ) { - bucket = factory.fromSelfie(line.slice(pos + 1)); - map.set(key, bucket); - continue; - } - // When token key is reused, it can't be anything - // else than FilterBucket - bucket.add(factory.fromSelfie(line.slice(pos + 1))); + var entries = JSON.parse(selfie), + out = new Map(), + entry; + for ( var i = 0, n = entries.length; i < n; i++ ) { + entry = entries[i]; + out.set(entry[0], filterFromCompiledData(entry[1])); } - return map; + return out; }; this.acceptedCount = selfie.acceptedCount; @@ -1909,10 +1796,8 @@ FilterContainer.prototype.retrieveGenericSelectors = function(request) { selector, bucket; while ( i-- ) { selector = selectors[i]; - if ( this.lowGenericHide.has(selector) === false ) { - continue; - } - if ( (bucket = this.lowGenericHideEx.get(selector)) ) { + if ( this.lowGenericHide.has(selector) === false ) { continue; } + if ( (bucket = this.lowGenericHideEx.get(selector)) !== undefined ) { if ( Array.isArray(bucket) ) { hideSelectors = hideSelectors.concat(bucket); } else { diff --git a/src/js/reverselookup-worker.js b/src/js/reverselookup-worker.js index 242f41a72..a1bf8e4d6 100644 --- a/src/js/reverselookup-worker.js +++ b/src/js/reverselookup-worker.js @@ -25,7 +25,8 @@ /******************************************************************************/ -var listEntries = Object.create(null); +var listEntries = Object.create(null), + filterClassSeparator = '\n/* end of network - start of cosmetic */\n'; /******************************************************************************/ @@ -35,20 +36,19 @@ var reEscape = function(s) { return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); }; -var reSpecialNetworkChars = /[a-d]/; - /******************************************************************************/ var fromNetFilter = function(details) { - var lists = []; - var compiledFilter = details.compiledFilter; - var entry, content, pos, notFound; + var lists = [], + compiledFilter = details.compiledFilter, + entry, content, pos, notFound; for ( var assetKey in listEntries ) { entry = listEntries[assetKey]; - if ( entry === undefined ) { - continue; - } - content = entry.content; + if ( entry === undefined ) { continue; } + content = entry.content.slice( + 0, + entry.content.indexOf(filterClassSeparator) + ); pos = 0; for (;;) { pos = content.indexOf(compiledFilter, pos); @@ -56,19 +56,19 @@ var fromNetFilter = function(details) { // We need an exact match. // https://github.com/gorhill/uBlock/issues/1392 // https://github.com/gorhill/uBlock/issues/835 - pos -= 1; - notFound = - reSpecialNetworkChars.test(content.charAt(pos)) === false || - pos !== 0 && content.charCodeAt(pos - 1) !== 0x0A /* '\n' */; - pos += 1 + compiledFilter.length; - if ( notFound ) { continue; } - if ( pos === content.length || content.charCodeAt(pos) === 0x0A ) { - lists.push({ - title: entry.title, - supportURL: entry.supportURL - }); - break; + notFound = pos !== 0 && content.charCodeAt(pos - 1) !== 0x0A; + pos += compiledFilter.length; + if ( + notFound || + pos !== content.length && content.charCodeAt(pos) !== 0x0A + ) { + continue; } + lists.push({ + title: entry.title, + supportURL: entry.supportURL + }); + break; } } @@ -104,93 +104,92 @@ var fromNetFilter = function(details) { // the various compiled versions. var fromCosmeticFilter = function(details) { - var filter = details.rawFilter; - var exception = filter.startsWith('#@#'); + var match = /^#@?#/.exec(details.rawFilter), + prefix = match[0], + filter = details.rawFilter.slice(prefix.length); - filter = exception ? filter.slice(3) : filter.slice(2); + var compiled = JSON.stringify(filter), + reFilter = new RegExp('(^|\\n).*?' + reEscape(compiled) + '.*?(\\n|$)', 'g'); - var candidates = Object.create(null); - var response = Object.create(null); + var reHostname = new RegExp( + '^' + + details.hostname.split('.').reduce( + function(acc, item) { + return acc === '' + ? item + : '(' + acc + '\\.)?' + item; + }, + '' + ) + + '$' + ); - // First step: assuming the filter is generic, find out its compiled - // representation. - // Reference: FilterContainer.compileGenericSelector(). - var reStr = []; - var matches = rePlainSelector.exec(filter); - if ( matches ) { - if ( matches[0] === filter ) { // simple CSS selector - reStr.push('[e-h]lg', reEscape(filter)); - } else { // complex CSS selector - reStr.push('[e-h]lg\\+', reEscape(matches[0]), reEscape(filter)); - } - } else if ( reHighLow.test(filter) ) { // [alt] or [title] - reStr.push('[e-h]hlg0', reEscape(filter)); - } else if ( reHighMedium.test(filter) ) { // [href^="..."] - reStr.push('[e-h]hmg0', '[^"]{8}', '[a-z]*' + reEscape(filter)); - } else if ( filter.indexOf(' ') === -1 ) { // high-high-simple selector - reStr.push('[e-h]hhsg0', reEscape(filter)); - } else { // high-high-complex selector - reStr.push('[e-h]hhcg0', reEscape(filter)); - } - candidates[details.rawFilter] = new RegExp(reStr.join('\\v') + '(?:\\n|$)'); - - // Procedural filters, which are pre-compiled, make thing sort of - // complicated. We are going to also search for one portion of the - // compiled form of a filter. - var filterEx = '(' + - reEscape(filter) + - '|\{[^\\v]*' + - reEscape(JSON.stringify({ raw: filter }).slice(1,-1)) + - '[^\\v]*\})'; - - // Second step: find hostname-based versions. - // Reference: FilterContainer.compileHostnameSelector(). - var pos, - hostname = details.hostname; - if ( hostname !== '' ) { - for ( ;; ) { - candidates[hostname + '##' + filter] = new RegExp( - ['[e-h]h', '[^\\v]+', reEscape(hostname), filterEx].join('\\v') + - '(?:\\n|$)' - ); - pos = hostname.indexOf('.'); - if ( pos === -1 ) { - break; - } - hostname = hostname.slice(pos + 1); - } - } - - // Last step: find entity-based versions. - // Reference: FilterContainer.compileEntitySelector(). - var domain = details.domain; - pos = domain.indexOf('.'); + var reEntity, + domain = details.domain, + pos = domain.indexOf('.'); if ( pos !== -1 ) { - var entity = domain.slice(0, pos) + '.*'; - candidates[entity + '##' + filter] = new RegExp( - ['[e-h]h', '[^\\v]+', reEscape(entity), filterEx].join('\\v') + - '(?:\\n|$)' + reEntity = new RegExp( + '^' + + domain.slice(0, pos).split('.').reduce( + function(acc, item) { + return acc === '' + ? item + : '(' + acc + '\\.)?' + item; + }, + '' + ) + + '\\.\\*$' ); } + + var response = Object.create(null), + assetKey, entry, content, found, fargs; - var re, assetKey, entry; - for ( var candidate in candidates ) { - re = candidates[candidate]; - for ( assetKey in listEntries ) { - entry = listEntries[assetKey]; - if ( entry === undefined ) { - continue; + for ( assetKey in listEntries ) { + entry = listEntries[assetKey]; + if ( entry === undefined ) { continue; } + content = entry.content.slice( + entry.content.indexOf(filterClassSeparator) + + filterClassSeparator.length + ); + found = undefined; + while ( (match = reFilter.exec(content)) !== null ) { + fargs = JSON.parse(match[0]); + switch ( fargs[0] ) { + case 0: + case 2: + case 4: + case 5: + case 7: + found = prefix + filter; + break; + case 1: + case 3: + if ( fargs[2] === filter ) { + found = prefix + filter; + } + break; + case 6: + case 8: + if ( + fargs[2] === '' || + reHostname.test(fargs[2]) === true || + reEntity !== undefined && reEntity.test(fargs[2]) === true + ) { + found = fargs[2] + prefix + filter; + } + break; } - if ( re.test(entry.content) === false ) { - continue; + if ( found !== undefined ) { + if ( response[found] === undefined ) { + response[found] = []; + } + response[found].push({ + title: entry.title, + supportURL: entry.supportURL + }); + break; } - if ( response[candidate] === undefined ) { - response[candidate] = []; - } - response[candidate].push({ - title: entry.title, - supportURL: entry.supportURL - }); } } @@ -200,10 +199,6 @@ var fromCosmeticFilter = function(details) { }); }; -var rePlainSelector = /^([#.][\w-]+)/; -var reHighLow = /^[a-z]*\[(?:alt|title)="[^"]+"\]$/; -var reHighMedium = /^\[href\^="https?:\/\/([^"]{8})[^"]*"\]$/; - /******************************************************************************/ onmessage = function(e) { // jshint ignore:line diff --git a/src/js/reverselookup.js b/src/js/reverselookup.js index c18bfba66..ebe73d56c 100644 --- a/src/js/reverselookup.js +++ b/src/js/reverselookup.js @@ -28,7 +28,7 @@ /******************************************************************************/ var worker = null; -var workerTTL = 11 * 60 * 1000; +var workerTTL = 5 * 60 * 1000; var workerTTLTimer = null; var needLists = true; var messageId = 1; diff --git a/src/js/static-net-filtering.js b/src/js/static-net-filtering.js index a3ba6b5b7..b6d8a80fd 100644 --- a/src/js/static-net-filtering.js +++ b/src/js/static-net-filtering.js @@ -33,15 +33,16 @@ var µb = µBlock; // fedcba9876543210 -// | | ||| -// | | ||| -// | | ||| -// | | ||| -// | | ||+---- bit 0: [BlockAction | AllowAction] -// | | |+---- bit 1: `important` -// | | +---- bit 2-3: party [0 - 3] -// | +---- bit 4-8: type [0 - 31] -// +---- bit 9-15: unused +// | | | ||| +// | | | ||| +// | | | ||| +// | | | ||| +// | | | ||+---- bit 0: [BlockAction | AllowAction] +// | | | |+----- bit 1: `important` +// | | | +------ bit 2- 3: party [0 - 3] +// | | +-------- bit 4- 8: type [0 - 31] +// | +------------- bit 9-14: unused +// +------------------- bit 15: bad filter var BlockAction = 0 << 0; var AllowAction = 1 << 0; @@ -49,6 +50,7 @@ var Important = 1 << 1; var AnyParty = 0 << 2; var FirstParty = 1 << 2; var ThirdParty = 2 << 2; +var BadFilter = 1 << 15; var AnyType = 0 << 4; var typeNameToTypeValue = { @@ -68,7 +70,8 @@ var typeNameToTypeValue = { 'main_frame': 13 << 4, // start of 1st-party-only behavorial filtering 'generichide': 14 << 4, 'inline-script': 15 << 4, - 'data': 16 << 4 // special: a generic data holder + 'data': 16 << 4, // special: a generic data holder + 'redirect': 17 << 4 }; var otherTypeBitValue = typeNameToTypeValue.other; @@ -88,7 +91,8 @@ var typeValueToTypeName = { 13: 'document', 14: 'generichide', 15: 'inline-script', - 16: 'data' + 16: 'data', + 17: 'redirect' }; // All network request types to bitmap @@ -208,22 +212,12 @@ rawToRegexStr.escape2 = /\^/g; rawToRegexStr.escape3 = /^\*|\*$/g; rawToRegexStr.escape4 = /\*/g; -// If using native Map, we use numerical keys, otherwise for -// Object-based map we use string-based keys. -var exportInt = function(k) { - return k.toString(32); -}; - -var importInt = function(k) { - return parseInt(k,32); -}; +var filterFingerprinter = µb.CompiledLineWriter.fingerprint; var toLogDataInternal = function(categoryBits, tokenHash, filter) { if ( filter === null ) { return undefined; } var logData = filter.logData(); - logData.compiled = exportInt(categoryBits) + '\v' + - exportInt(tokenHash) + '\v' + - logData.compiled; + logData.compiled = filterFingerprinter([ categoryBits, tokenHash, logData.compiled ]); if ( categoryBits & 0x001 ) { logData.raw = '@@' + logData.raw; } @@ -275,16 +269,23 @@ var reURLPostHostnameAnchors = /[\/?#]/; **/ -var filterClasses = new Map(), +var filterClasses = [], filterClassIdGenerator = 0; var registerFilterClass = function(ctor) { var fid = filterClassIdGenerator++; - ctor.fidPrefix = ctor.prototype.fidPrefix = fid.toString(32) + '\t'; - filterClasses.set(fid, ctor); + ctor.fid = ctor.prototype.fid = fid; + filterClasses[fid] = ctor; //console.log(ctor.name, fid); }; +var filterFromCompiledData = function(args) { + //filterClassHistogram.set(fid, (filterClassHistogram.get(fid) || 0) + 1); + return filterClasses[args[0]].load(args); +}; + +//var filterClassHistogram = new Map(); + /******************************************************************************/ var FilterTrue = function() { @@ -303,11 +304,11 @@ FilterTrue.prototype.logData = function() { }; FilterTrue.prototype.compile = function() { - return this.fidPrefix; + return [ this.fid ]; }; FilterTrue.compile = function() { - return FilterTrue.fidPrefix; + return [ FilterTrue.fid ]; }; FilterTrue.load = function() { @@ -336,19 +337,15 @@ FilterPlain.prototype.logData = function() { }; FilterPlain.prototype.compile = function() { - return this.fidPrefix + this.s + '\t' + this.tokenBeg; + return [ this.fid, this.s, this.tokenBeg ]; }; FilterPlain.compile = function(details) { - return FilterPlain.fidPrefix + details.f + '\t' + details.tokenBeg; + return [ FilterPlain.fid, details.f, details.tokenBeg ]; }; -FilterPlain.load = function(s) { - var pos = s.indexOf('\t', 2); - return new FilterPlain( - s.slice(2, pos), - parseInt(s.slice(pos + 1), 10) - ); +FilterPlain.load = function(args) { + return new FilterPlain(args[1], args[2]); }; registerFilterClass(FilterPlain); @@ -372,15 +369,15 @@ FilterPlainPrefix0.prototype.logData = function() { }; FilterPlainPrefix0.prototype.compile = function() { - return this.fidPrefix + this.s; + return [ this.fid, this.s ]; }; FilterPlainPrefix0.compile = function(details) { - return FilterPlainPrefix0.fidPrefix + details.f; + return [ FilterPlainPrefix0.fid, details.f ]; }; -FilterPlainPrefix0.load = function(s) { - return new FilterPlainPrefix0(s.slice(2)); +FilterPlainPrefix0.load = function(args) { + return new FilterPlainPrefix0(args[1]); }; registerFilterClass(FilterPlainPrefix0); @@ -404,15 +401,15 @@ FilterPlainPrefix1.prototype.logData = function() { }; FilterPlainPrefix1.prototype.compile = function() { - return this.fidPrefix + this.s; + return [ this.fid, this.s ]; }; FilterPlainPrefix1.compile = function(details) { - return FilterPlainPrefix1.fidPrefix + details.f; + return [ FilterPlainPrefix1.fid, details.f ]; }; -FilterPlainPrefix1.load = function(s) { - return new FilterPlainPrefix1(s.slice(2)); +FilterPlainPrefix1.load = function(args) { + return new FilterPlainPrefix1(args[1]); }; registerFilterClass(FilterPlainPrefix1); @@ -439,15 +436,15 @@ FilterPlainHostname.prototype.logData = function() { }; FilterPlainHostname.prototype.compile = function() { - return this.fidPrefix + this.s; + return [ this.fid, this.s ]; }; FilterPlainHostname.compile = function(details) { - return FilterPlainHostname.fidPrefix + details.f; + return [ FilterPlainHostname.fid, details.f ]; }; -FilterPlainHostname.load = function(s) { - return new FilterPlainHostname(s.slice(2)); +FilterPlainHostname.load = function(args) { + return new FilterPlainHostname(args[1]); }; registerFilterClass(FilterPlainHostname); @@ -471,15 +468,15 @@ FilterPlainLeftAnchored.prototype.logData = function() { }; FilterPlainLeftAnchored.prototype.compile = function() { - return this.fidPrefix + this.s; + return [ this.fid, this.s ]; }; FilterPlainLeftAnchored.compile = function(details) { - return FilterPlainLeftAnchored.fidPrefix + details.f; + return [ FilterPlainLeftAnchored.fid, details.f ]; }; -FilterPlainLeftAnchored.load = function(s) { - return new FilterPlainLeftAnchored(s.slice(2)); +FilterPlainLeftAnchored.load = function(args) { + return new FilterPlainLeftAnchored(args[1]); }; registerFilterClass(FilterPlainLeftAnchored); @@ -503,15 +500,15 @@ FilterPlainRightAnchored.prototype.logData = function() { }; FilterPlainRightAnchored.prototype.compile = function() { - return this.fidPrefix + this.s; + return [ this.fid, this.s ]; }; FilterPlainRightAnchored.compile = function(details) { - return FilterPlainRightAnchored.fidPrefix + details.f; + return [ FilterPlainRightAnchored.fid, details.f ]; }; -FilterPlainRightAnchored.load = function(s) { - return new FilterPlainRightAnchored(s.slice(2)); +FilterPlainRightAnchored.load = function(args) { + return new FilterPlainRightAnchored(args[1]); }; registerFilterClass(FilterPlainRightAnchored); @@ -536,15 +533,15 @@ FilterPlainHnAnchored.prototype.logData = function() { }; FilterPlainHnAnchored.prototype.compile = function() { - return this.fidPrefix + this.s; + return [ this.fid, this.s ]; }; FilterPlainHnAnchored.compile = function(details) { - return FilterPlainHnAnchored.fidPrefix + details.f; + return [ FilterPlainHnAnchored.fid, details.f ]; }; -FilterPlainHnAnchored.load = function(s) { - return new FilterPlainHnAnchored(s.slice(2)); +FilterPlainHnAnchored.load = function(args) { + return new FilterPlainHnAnchored(args[1]); }; registerFilterClass(FilterPlainHnAnchored); @@ -581,19 +578,15 @@ FilterGeneric.prototype.logData = function() { }; FilterGeneric.prototype.compile = function() { - return this.fidPrefix + this.s + '\t' + this.anchor; + return [ this.fid, this.s, this.anchor ]; }; FilterGeneric.compile = function(details) { - return FilterGeneric.fidPrefix + details.f + '\t' + details.anchor; + return [ FilterGeneric.fid, details.f, details.anchor ]; }; -FilterGeneric.load = function(s) { - var pos = s.indexOf('\t', 2); - return new FilterGeneric( - s.slice(2, pos), - parseInt(s.slice(pos + 1), 10) - ); +FilterGeneric.load = function(args) { + return new FilterGeneric(args[1], args[2]); }; registerFilterClass(FilterGeneric); @@ -625,15 +618,15 @@ FilterGenericHnAnchored.prototype.logData = function() { }; FilterGenericHnAnchored.prototype.compile = function() { - return this.fidPrefix + this.s; + return [ this.fid, this.s ]; }; FilterGenericHnAnchored.compile = function(details) { - return FilterGenericHnAnchored.fidPrefix + details.f; + return [ FilterGenericHnAnchored.fid, details.f ]; }; -FilterGenericHnAnchored.load = function(s) { - return new FilterGenericHnAnchored(s.slice(2)); +FilterGenericHnAnchored.load = function(args) { + return new FilterGenericHnAnchored(args[1]); }; registerFilterClass(FilterGenericHnAnchored); @@ -644,33 +637,36 @@ var FilterGenericHnAndRightAnchored = function(s) { FilterGenericHnAnchored.call(this, s); }; -FilterGenericHnAndRightAnchored.prototype = Object.create(FilterGenericHnAnchored.prototype, { - constructor: { - value: FilterGenericHnAndRightAnchored - }, - anchor: { - value: 0x5 - }, - logData: { - value: function() { - var out = FilterGenericHnAnchored.prototype.logData.call(this); - out.raw += '|'; - return out; +FilterGenericHnAndRightAnchored.prototype = Object.create( + FilterGenericHnAnchored.prototype, + { + constructor: { + value: FilterGenericHnAndRightAnchored + }, + anchor: { + value: 0x5 + }, + logData: { + value: function() { + var out = FilterGenericHnAnchored.prototype.logData.call(this); + out.raw += '|'; + return out; + } + }, + compile: { + value: function() { + return [ this.fid, this.s ]; + } } - }, - compile: { - value: function() { - return this.fidPrefix + this.s; - } - }, -}); + } +); FilterGenericHnAndRightAnchored.compile = function(details) { - return FilterGenericHnAndRightAnchored.fidPrefix + details.f; + return [ FilterGenericHnAndRightAnchored.fid, details.f ]; }; -FilterGenericHnAndRightAnchored.load = function(s) { - return new FilterGenericHnAndRightAnchored(s.slice(2)); +FilterGenericHnAndRightAnchored.load = function(args) { + return new FilterGenericHnAndRightAnchored(args[1]); }; registerFilterClass(FilterGenericHnAndRightAnchored); @@ -694,15 +690,15 @@ FilterRegex.prototype.logData = function() { }; FilterRegex.prototype.compile = function() { - return this.fidPrefix + this.re.source; + return [ this.fid, this.re.source ]; }; FilterRegex.compile = function(details) { - return FilterRegex.fidPrefix + details.f; + return [ FilterRegex.fid, details.f ]; }; -FilterRegex.load = function(s) { - return new FilterRegex(s.slice(2)); +FilterRegex.load = function(args) { + return new FilterRegex(args[1]); }; registerFilterClass(FilterRegex); @@ -739,7 +735,7 @@ FilterOrigin.prototype.match = function(url, tokenBeg) { FilterOrigin.prototype.logData = function() { var out = this.wrapped.logData(), domainOpt = this.toDomainOpt(); - out.compiled = this.fidPrefix + domainOpt + '\v' + out.compiled; + out.compiled = [ this.fid, domainOpt, out.compiled ]; if ( out.opts === undefined ) { out.opts = 'domain=' + domainOpt; } else { @@ -749,7 +745,7 @@ FilterOrigin.prototype.logData = function() { }; FilterOrigin.prototype.compile = function() { - return this.fidPrefix + this.toDomainOpt() + '\v' + this.wrapped.compile(); + return [ this.fid, this.toDomainOpt(), this.wrapped.compile() ]; }; // *** start of specialized origin matchers @@ -949,13 +945,12 @@ FilterOrigin.matcherFactory = function(domainOpt) { FilterOrigin.reAllNegated = /^~(?:[^|~]+\|~)+[^|~]+$/; FilterOrigin.compile = function(details) { - return FilterOrigin.fidPrefix + details.domainOpt; + return [ FilterOrigin.fid, details.domainOpt ]; }; -FilterOrigin.load = function(s) { - var pos = s.indexOf('\v', 2), - f = FilterOrigin.matcherFactory(s.slice(2, pos)); - f.wrapped = filterFromCompiledData(s.slice(pos + 1)); +FilterOrigin.load = function(args) { + var f = FilterOrigin.matcherFactory(args[1]); + f.wrapped = filterFromCompiledData(args[2]); return f; }; @@ -975,7 +970,7 @@ FilterDataHolder.prototype.match = function(url, tokenBeg) { FilterDataHolder.prototype.logData = function() { var out = this.wrapped.logData(); - out.compiled = this.fidPrefix + this.dataType + '\t' + this.dataStr + '\v' + out.compiled; + out.compiled = [ this.fid, this.dataType, this.dataStr, out.compiled ]; var opt = this.dataType; if ( this.dataStr !== '' ) { opt += '=' + this.dataStr; @@ -989,18 +984,16 @@ FilterDataHolder.prototype.logData = function() { }; FilterDataHolder.prototype.compile = function() { - return this.fidPrefix + this.dataType + '\t' + this.dataStr + '\v' + this.wrapped.compile(); + return [ this.fid, this.dataType, this.dataStr, this.wrapped.compile() ]; }; FilterDataHolder.compile = function(details) { - return FilterDataHolder.fidPrefix + details.dataType + '\t' + details.dataStr; + return [ FilterDataHolder.fid, details.dataType, details.dataStr ]; }; -FilterDataHolder.load = function(s) { - var pos = s.indexOf('\t', 2), - end = s.indexOf('\v', pos), - f = new FilterDataHolder(s.slice(2, pos), s.slice(pos + 1, end)); - f.wrapped = filterFromCompiledData(s.slice(end + 1)); +FilterDataHolder.load = function(args) { + var f = new FilterDataHolder(args[1], args[2]); + f.wrapped = filterFromCompiledData(args[3]); return f; }; @@ -1020,19 +1013,11 @@ FilterDataHolderEntry.prototype.logData = function() { }; FilterDataHolderEntry.prototype.compile = function() { - return exportInt(this.categoryBits) + '\t' + - exportInt(this.tokenHash) + '\t' + - this.filter.compile(); + return [ this.categoryBits, this.tokenHash, this.filter.compile() ]; }; -FilterDataHolderEntry.load = function(s) { - var pos1 = s.indexOf('\t'), - pos2 = s.indexOf('\t', pos1 + 1); - return new FilterDataHolderEntry( - importInt(s), - importInt(s.slice(pos1 + 1, pos2)), - s.slice(pos2 + 1) - ); +FilterDataHolderEntry.load = function(data) { + return new FilterDataHolderEntry(data[0], data[1], data[2]); }; /******************************************************************************/ @@ -1051,9 +1036,7 @@ Object.defineProperty(FilterHostnameDict.prototype, 'size', { }); FilterHostnameDict.prototype.add = function(hn) { - if ( this.dict.has(hn) ) { - return false; - } + if ( this.dict.has(hn) === true ) { return false; } this.dict.add(hn); return true; }; @@ -1087,12 +1070,12 @@ FilterHostnameDict.prototype.logData = function() { }; FilterHostnameDict.prototype.compile = function() { - return this.fidPrefix + JSON.stringify(µb.setToArray(this.dict)); + return [ this.fid, µb.setToArray(this.dict) ]; }; -FilterHostnameDict.load = function(s) { +FilterHostnameDict.load = function(args) { var f = new FilterHostnameDict(); - f.dict = µb.setFromArray(JSON.parse(s.slice(2))); + f.dict = µb.setFromArray(args[1]); return f; }; @@ -1133,20 +1116,18 @@ registerFilterClass(FilterHostnameDict); /******************************************************************************/ var FilterBucket = function(a, b) { - this.promoted = 0; - this.vip = 16; - this.f = null; // short-lived register this.filters = []; + this.f = null; if ( a !== undefined ) { this.filters[0] = a; - if ( b !== undefined ) { - this.filters[1] = b; - } + this.filters[1] = b; } }; -FilterBucket.prototype.add = function(a) { - this.filters.push(a); +FilterBucket.prototype.promoted = 0; + +FilterBucket.prototype.add = function(fdata) { + this.filters[this.filters.length] = filterFromCompiledData(fdata); }; FilterBucket.prototype.remove = function(fdata) { @@ -1162,13 +1143,11 @@ FilterBucket.prototype.remove = function(fdata) { // Promote hit filters so they can be found faster next time. FilterBucket.prototype.promote = function(i) { - var filters = this.filters; - var pivot = filters.length >>> 1; + var filters = this.filters, + pivot = filters.length >>> 1; while ( i < pivot ) { pivot >>>= 1; - if ( pivot < this.vip ) { - break; - } + if ( pivot < 16 ) { break; } } if ( i <= pivot ) { return; } var j = this.promoted % pivot; @@ -1180,14 +1159,11 @@ FilterBucket.prototype.promote = function(i) { }; FilterBucket.prototype.match = function(url, tokenBeg) { - var filters = this.filters, - n = filters.length; - for ( var i = 0; i < n; i++ ) { - if ( filters[i].match(url, tokenBeg) ) { + var filters = this.filters; + for ( var i = 0, n = filters.length; i < n; i++ ) { + if ( filters[i].match(url, tokenBeg) === true ) { this.f = filters[i]; - if ( i >= this.vip ) { - this.promote(i); - } + if ( i >= 16 ) { this.promote(i); } return true; } } @@ -1204,15 +1180,15 @@ FilterBucket.prototype.compile = function() { for ( var i = 0, n = filters.length; i < n; i++ ) { compiled[i] = filters[i].compile(); } - return this.fidPrefix + JSON.stringify(compiled); + return [ this.fid, compiled ]; }; -FilterBucket.load = function(s) { +FilterBucket.load = function(args) { var f = new FilterBucket(), - compiled = JSON.parse(s.slice(2)), + compiledFilters = args[1], filters = f.filters; - for ( var i = 0, n = compiled.length; i < n; i++ ) { - filters[i] = filterFromCompiledData(compiled[i]); + for ( var i = 0, n = compiledFilters.length; i < n; i++ ) { + filters[i] = filterFromCompiledData(compiledFilters[i]); } return f; }; @@ -1222,25 +1198,6 @@ registerFilterClass(FilterBucket); /******************************************************************************/ /******************************************************************************/ -var filterFromCompiledData = function(compiled) { - if ( compiled === lastLoadedFilterString ) { - return lastLoadedFilter; - } - var fid = parseInt(compiled, 36), - f = filterClasses.get(fid).load(compiled); - //filterClassHistogram.set(fid, (filterClassHistogram.get(fid) || 0) + 1); - lastLoadedFilterString = compiled; - lastLoadedFilter = f; - return f; -}; - -var lastLoadedFilterString, - lastLoadedFilter; -//var filterClassHistogram = new Map(); - -/******************************************************************************/ -/******************************************************************************/ - var FilterParser = function() { this.cantWebsocket = vAPI.cantWebsocket; this.reBadDomainOptChars = /[*+?^${}()[\]\\]/; @@ -1293,7 +1250,7 @@ FilterParser.prototype.toNormalizedType = { FilterParser.prototype.reset = function() { this.action = BlockAction; this.anchor = 0; - this.badFilter = false; + this.badFilter = 0; this.dataType = undefined; this.dataStr = undefined; this.elemHiding = false; @@ -1419,13 +1376,17 @@ FilterParser.prototype.parseOptions = function(s) { this.unsupported = true; break; } + // Test before handling all other types. + if ( opt.startsWith('redirect=') ) { + if ( this.action === BlockAction ) { + this.redirect = true; + continue; + } + this.unsupported = true; + break; + } if ( this.toNormalizedType.hasOwnProperty(opt) ) { this.parseTypeOption(opt, not); - // Due to ABP categorizing `websocket` requests as `other`, we need - // to add `websocket` for when `other` is used. - if ( opt === 'other' ) { - this.parseTypeOption('websocket', not); - } continue; } // https://github.com/gorhill/uBlock/issues/2294 @@ -1447,14 +1408,6 @@ FilterParser.prototype.parseOptions = function(s) { this.parsePartyOption(true, not); continue; } - if ( opt.startsWith('redirect=') ) { - if ( this.action === BlockAction ) { - this.redirect = true; - continue; - } - this.unsupported = true; - break; - } if ( opt.startsWith('csp=') ) { if ( opt.length > 4 && this.reBadCSP.test(opt) === false ) { this.parseTypeOption('data', not); @@ -1475,7 +1428,7 @@ FilterParser.prototype.parseOptions = function(s) { } // https://github.com/uBlockOrigin/uAssets/issues/192 if ( opt === 'badfilter' ) { - this.badFilter = true; + this.badFilter = BadFilter; continue; } // Unrecognized filter option: ignore whole filter. @@ -1821,7 +1774,6 @@ var FilterContainer = function() { this.urlTokenizer = µb.urlTokenizer; this.noTokenHash = this.urlTokenizer.tokenHashFromString('*'); this.dotTokenHash = this.urlTokenizer.tokenHashFromString('.'); - this.exportedDotTokenHash = exportInt(this.dotTokenHash); this.reset(); }; @@ -1874,22 +1826,16 @@ FilterContainer.prototype.freeze = function() { /******************************************************************************/ FilterContainer.prototype.toSelfie = function() { - var categoryToSelfie = function(map) { - var selfie = []; - for ( var entry of map ) { - selfie.push('k2\t' + exportInt(entry[0])); // token hash - selfie.push(entry[1].compile()); + var categoriesToSelfie = function(categoryMap) { + var categoryEntries = []; + for ( var categoryEntry of categoryMap ) { + var tokenEntries = []; + for ( var tokenEntry of categoryEntry[1] ) { + tokenEntries.push([ tokenEntry[0], tokenEntry[1].compile() ]); + } + categoryEntries.push([ categoryEntry[0], tokenEntries ]); } - return selfie.join('\n'); - }; - - var categoriesToSelfie = function(map) { - var selfie = []; - for ( var entry of map ) { - selfie.push('k1\t' + exportInt(entry[0])); // category bits - selfie.push(categoryToSelfie(entry[1])); - } - return selfie.join('\n'); + return JSON.stringify(categoryEntries); }; var dataFiltersToSelfie = function(dataFilters) { @@ -1900,7 +1846,7 @@ FilterContainer.prototype.toSelfie = function() { entry = entry.next; } while ( entry !== undefined ); } - return selfie; + return JSON.stringify(selfie); }; return { @@ -1926,29 +1872,27 @@ FilterContainer.prototype.fromSelfie = function(selfie) { this.blockFilterCount = selfie.blockFilterCount; this.discardedCount = selfie.discardedCount; - var categoryBits, tokenHash, - map = this.categories, submap, - lineIter = new µb.LineIterator(selfie.categories), - line; - while ( lineIter.eot() === false ) { - line = lineIter.next(); - if ( line.startsWith('k1\t') ) { // category bits - categoryBits = importInt(line.slice(3)); - submap = new Map(); - map.set(categoryBits, submap); - continue; - } - if ( line.startsWith('k2\t') ) { // token hash - tokenHash = importInt(line.slice(3)); - continue; - } - submap.set(tokenHash, filterFromCompiledData(line)); - } + var entries; - var i = selfie.dataFilters.length, - entry, bucket; + var categoryMap = new Map(); + entries = JSON.parse(selfie.categories); + for ( var i = 0, ni = entries.length; i < ni; i++ ) { + var categoryEntry = entries[i], + tokenMap = new Map(); + var tokenEntries = categoryEntry[1]; + for ( var j = 0, nj = tokenEntries.length; j < nj; j++ ) { + var tokenEntry = tokenEntries[j]; + tokenMap.set(tokenEntry[0], filterFromCompiledData(tokenEntry[1])); + } + categoryMap.set(categoryEntry[0], tokenMap); + } + this.categories = categoryMap; + + entries = JSON.parse(selfie.dataFilters); + var entry, bucket; + i = entries.length; while ( i-- ) { - entry = FilterDataHolderEntry.load(selfie.dataFilters[i]); + entry = FilterDataHolderEntry.load(entries[i]); bucket = this.dataFilters.get(entry.tokenHash); if ( bucket !== undefined ) { entry.next = bucket; @@ -1959,7 +1903,7 @@ FilterContainer.prototype.fromSelfie = function(selfie) { /******************************************************************************/ -FilterContainer.prototype.compile = function(raw, out) { +FilterContainer.prototype.compile = function(raw, writer) { // ORDER OF TESTS IS IMPORTANT! // Ignore empty lines @@ -1988,65 +1932,69 @@ FilterContainer.prototype.compile = function(raw, out) { parsed.hostnamePure && parsed.domainOpt === '' && parsed.dataType === undefined && - this.compileHostnameOnlyFilter(parsed, out) + this.compileHostnameOnlyFilter(parsed, writer) ) { return true; } parsed.makeToken(); - var fdata = ''; - if ( parsed.dataType !== undefined ) { - if ( fdata !== '' ) { fdata += '\v'; } - fdata += FilterDataHolder.compile(parsed); - } - if ( parsed.domainOpt !== '' ) { - if ( fdata !== '' ) { fdata += '\v'; } - fdata += FilterOrigin.compile(parsed); - } - if ( fdata !== '' ) { fdata += '\v'; } + var fdata; if ( parsed.isRegex ) { - fdata += FilterRegex.compile(parsed); + fdata = FilterRegex.compile(parsed); } else if ( parsed.hostnamePure ) { - fdata += FilterPlainHostname.compile(parsed); + fdata = FilterPlainHostname.compile(parsed); } else if ( parsed.f === '*' ) { - fdata += FilterTrue.compile(); + fdata = FilterTrue.compile(); } else if ( parsed.anchor === 0x5 ) { // https://github.com/gorhill/uBlock/issues/1669 - fdata += FilterGenericHnAndRightAnchored.compile(parsed); + fdata = FilterGenericHnAndRightAnchored.compile(parsed); } else if ( this.reIsGeneric.test(parsed.f) || parsed.tokenHash === parsed.noTokenHash ) { if ( parsed.anchor === 0x4 ) { - fdata += FilterGenericHnAnchored.compile(parsed); + fdata = FilterGenericHnAnchored.compile(parsed); } else { - fdata += FilterGeneric.compile(parsed); + fdata = FilterGeneric.compile(parsed); } } else if ( parsed.anchor === 0x4 ) { - fdata += FilterPlainHnAnchored.compile(parsed); + fdata = FilterPlainHnAnchored.compile(parsed); } else if ( parsed.anchor === 0x2 ) { - fdata += FilterPlainLeftAnchored.compile(parsed); + fdata = FilterPlainLeftAnchored.compile(parsed); } else if ( parsed.anchor === 0x1 ) { - fdata += FilterPlainRightAnchored.compile(parsed); + fdata = FilterPlainRightAnchored.compile(parsed); } else if ( parsed.tokenBeg === 0 ) { - fdata += FilterPlainPrefix0.compile(parsed); + fdata = FilterPlainPrefix0.compile(parsed); } else if ( parsed.tokenBeg === 1 ) { - fdata += FilterPlainPrefix1.compile(parsed); + fdata = FilterPlainPrefix1.compile(parsed); } else { - fdata += FilterPlain.compile(parsed); + fdata = FilterPlain.compile(parsed); } - this.compileToAtomicFilter(fdata, parsed, out); + var fwrapped; + if ( parsed.domainOpt !== '' ) { + fwrapped = fdata; + fdata = FilterOrigin.compile(parsed); + fdata.push(fwrapped); + } + + if ( parsed.dataType !== undefined ) { + fwrapped = fdata; + fdata = FilterDataHolder.compile(parsed); + fdata.push(fwrapped); + } + + this.compileToAtomicFilter(fdata, parsed, writer); return true; }; /******************************************************************************/ -// Using fast/compact dictionary when filter is a (or portion of) pure hostname. +// Using fast/compact dictionary when filter is a pure hostname. -FilterContainer.prototype.compileHostnameOnlyFilter = function(parsed, out) { +FilterContainer.prototype.compileHostnameOnlyFilter = function(parsed, writer) { // Can't fit the filter in a pure hostname dictionary. // https://github.com/gorhill/uBlock/issues/1757 // This should no longer happen with fix to above issue. @@ -2054,29 +2002,18 @@ FilterContainer.prototype.compileHostnameOnlyFilter = function(parsed, out) { // return; //} - var route = parsed.badFilter ? 0x01 : 0x00, - categoryBits = parsed.action | parsed.important | parsed.party; + var descBits = parsed.action | parsed.important | parsed.party | parsed.badFilter; var type = parsed.types; if ( type === 0 ) { - out.push( - route, - exportInt(categoryBits) + '\v' + - this.exportedDotTokenHash + '\v' + - parsed.f - ); + writer.push([ descBits, this.dotTokenHash, parsed.f ]); return true; } var bitOffset = 1; do { if ( type & 1 ) { - out.push( - route, - exportInt(categoryBits | (bitOffset << 4)) + '\v' + - this.exportedDotTokenHash + '\v' + - parsed.f - ); + writer.push([ descBits | (bitOffset << 4), this.dotTokenHash, parsed.f ]); } bitOffset += 1; type >>>= 1; @@ -2086,28 +2023,17 @@ FilterContainer.prototype.compileHostnameOnlyFilter = function(parsed, out) { /******************************************************************************/ -FilterContainer.prototype.compileToAtomicFilter = function(fdata, parsed, out) { - var route = parsed.badFilter ? 0x01 : 0x00, - categoryBits = parsed.action | parsed.important | parsed.party, +FilterContainer.prototype.compileToAtomicFilter = function(fdata, parsed, writer) { + var descBits = parsed.action | parsed.important | parsed.party | parsed.badFilter, type = parsed.types; if ( type === 0 ) { - out.push( - route, - exportInt(categoryBits) + '\v' + - exportInt(parsed.tokenHash) + '\v' + - fdata - ); + writer.push([ descBits, parsed.tokenHash, fdata ]); return; } var bitOffset = 1; do { if ( type & 1 ) { - out.push( - route, - exportInt(categoryBits | (bitOffset << 4)) + '\v' + - exportInt(parsed.tokenHash) + '\v' + - fdata - ); + writer.push([ descBits | (bitOffset << 4), parsed.tokenHash, fdata ]); } bitOffset += 1; type >>>= 1; @@ -2125,62 +2051,56 @@ FilterContainer.prototype.compileToAtomicFilter = function(fdata, parsed, out) { var redirects = µb.redirectEngine.compileRuleFromStaticFilter(parsed.raw); if ( Array.isArray(redirects) === false ) { - return; } + descBits = typeNameToTypeValue.redirect; var i = redirects.length; while ( i-- ) { - out.push(0, '\v\v=>\t' + redirects[i]); + writer.push([ descBits, redirects[i] ]); } }; /******************************************************************************/ -FilterContainer.prototype.fromCompiledContent = function(lineIter) { - var line, lineBits, categoryBits, tokenHash, fdata, - bucket, entry, filter, - fieldIter = new µb.FieldIterator('\v'), - dataFilterFid = FilterDataHolder.fidPrefix, - buckerFilterFid = FilterBucket.fidPrefix, - aCharCode = 'a'.charCodeAt(0); +FilterContainer.prototype.fromCompiledContent = function(reader) { + var badFilterBit = BadFilter, + filterBucketId = FilterBucket.fid, + filterDataHolderId = FilterDataHolder.fid, + args, bits, bucket, entry, + tokenHash, fdata, fingerprint; - while ( lineIter.eot() === false ) { - lineBits = lineIter.charCodeAt(0) - aCharCode; - if ( (lineBits & 0x04) !== 0 ) { - return; - } - line = lineIter.next(1); - if ( (lineBits & 0x02) !== 0 ) { - line = decodeURIComponent(line); - } - if ( (lineBits & 0x01) !== 0 ) { - this.badFilters.add(line); + while ( reader.next() === true ) { + args = reader.args(); + bits = args[0]; + + if ( (bits & badFilterBit) !== 0 ) { + this.badFilters.add(args); continue; } - categoryBits = importInt(fieldIter.first(line)); - tokenHash = importInt(fieldIter.next()); - fdata = fieldIter.remainder(); - // Special cases: delegate to more specialized engines. // Redirect engine. - if ( fdata.startsWith('=>\t') ) { - µb.redirectEngine.fromCompiledRule(fdata.slice(3)); + if ( (bits >>> 4) === 17 ) { + µb.redirectEngine.fromCompiledRule(args[1]); continue; } - // Plain static filters. this.acceptedCount += 1; + // Plain static filters. + fingerprint = reader.fingerprint(); + tokenHash = args[1]; + fdata = args[2]; + // Special treatment: data-holding filters are stored separately // because they require special matching algorithm (unlike other // filters, ALL hits must be reported). - if ( fdata.startsWith(dataFilterFid) ) { - if ( this.duplicateBuster.has(line) ) { + if ( fdata[0] === filterDataHolderId ) { + if ( this.duplicateBuster.has(fingerprint) ) { this.discardedCount += 1; continue; } - this.duplicateBuster.add(line); - entry = new FilterDataHolderEntry(categoryBits, tokenHash, fdata); + this.duplicateBuster.add(fingerprint); + entry = new FilterDataHolderEntry(bits, tokenHash, fdata); bucket = this.dataFilters.get(tokenHash); if ( bucket !== undefined ) { entry.next = bucket; @@ -2189,10 +2109,10 @@ FilterContainer.prototype.fromCompiledContent = function(lineIter) { continue; } - bucket = this.categories.get(categoryBits); + bucket = this.categories.get(bits); if ( bucket === undefined ) { bucket = new Map(); - this.categories.set(categoryBits, bucket); + this.categories.set(bits, bucket); } entry = bucket.get(tokenHash); @@ -2207,48 +2127,36 @@ FilterContainer.prototype.fromCompiledContent = function(lineIter) { continue; } - if ( this.duplicateBuster.has(line) ) { + if ( this.duplicateBuster.has(fingerprint) ) { this.discardedCount += 1; continue; } - this.duplicateBuster.add(line); + this.duplicateBuster.add(fingerprint); - //this.tokenHistogram.set(tokenHash, (this.tokenHistogram.get(tokenHash) || 0) + 1); - - filter = filterFromCompiledData(fdata); if ( entry === undefined ) { - bucket.set(tokenHash, filter); + bucket.set(tokenHash, filterFromCompiledData(fdata)); continue; } - if ( entry.fidPrefix === buckerFilterFid ) { - entry.add(filter); + if ( entry.fid === filterBucketId ) { + entry.add(fdata); continue; } - bucket.set(tokenHash, new FilterBucket(entry, filter)); + bucket.set(tokenHash, new FilterBucket(entry, filterFromCompiledData(fdata))); } }; -//FilterContainer.prototype.tokenHistogram = new Map(); - /******************************************************************************/ FilterContainer.prototype.removeBadFilters = function() { - var lines = µb.setToArray(this.badFilters), - fieldIter = new µb.FieldIterator('\v'), - categoryBits, tokenHash, fdata, bucket, entry, - i = lines.length; - while ( i-- ) { - categoryBits = importInt(fieldIter.first(lines[i])); - bucket = this.categories.get(categoryBits); - if ( bucket === undefined ) { - continue; - } - tokenHash = importInt(fieldIter.next()); + var bits, tokenHash, fdata, bucket, entry; + for ( var args of this.badFilters ) { + bits = args[0] & ~BadFilter; + bucket = this.categories.get(bits); + if ( bucket === undefined ) { continue; } + tokenHash = args[1]; entry = bucket.get(tokenHash); - if ( entry === undefined ) { - continue; - } - fdata = fieldIter.remainder(); + if ( entry === undefined ) { continue; } + fdata = args[2]; if ( entry instanceof FilterBucket ) { entry.remove(fdata); if ( entry.filters.length === 1 ) { @@ -2261,7 +2169,7 @@ FilterContainer.prototype.removeBadFilters = function() { if ( entry.size === 0 ) { bucket.delete(tokenHash); if ( bucket.size === 0 ) { - this.categories.delete(categoryBits); + this.categories.delete(bits); } } continue; @@ -2269,7 +2177,7 @@ FilterContainer.prototype.removeBadFilters = function() { if ( entry.compile() === fdata ) { bucket.delete(tokenHash); if ( bucket.size === 0 ) { - this.categories.delete(categoryBits); + this.categories.delete(bits); } continue; } diff --git a/src/js/storage.js b/src/js/storage.js index 4a43eb0a9..435babe3a 100644 --- a/src/js/storage.js +++ b/src/js/storage.js @@ -737,7 +737,8 @@ /******************************************************************************/ µBlock.compileFilters = function(rawText) { - var compiledFilters = new this.CompiledOutput(); + var networkFilters = new this.CompiledLineWriter(), + cosmeticFilters = new this.CompiledLineWriter(); // Useful references: // https://adblockplus.org/en/filter-cheatsheet @@ -766,7 +767,7 @@ // Parse or skip cosmetic filters // All cosmetic filters are caught here - if ( cosmeticFilteringEngine.compile(line, compiledFilters) ) { + if ( cosmeticFilteringEngine.compile(line, cosmeticFilters) ) { continue; } @@ -800,10 +801,12 @@ if ( line.length === 0 ) { continue; } - staticNetFilteringEngine.compile(line, compiledFilters); + staticNetFilteringEngine.compile(line, networkFilters); } - return compiledFilters.toString(); + return networkFilters.toString() + + '\n/* end of network - start of cosmetic */\n' + + cosmeticFilters.toString(); }; /******************************************************************************/ @@ -813,15 +816,16 @@ // applying 1st-party filters. µBlock.applyCompiledFilters = function(rawText, firstparty) { - var skipCosmetic = !firstparty && !this.userSettings.parseAllABPHideFilters, - skipGenericCosmetic = this.userSettings.ignoreGenericCosmeticFilters, - staticNetFilteringEngine = this.staticNetFilteringEngine, - cosmeticFilteringEngine = this.cosmeticFilteringEngine, - lineIter = new this.LineIterator(rawText); - while ( lineIter.eot() === false ) { - cosmeticFilteringEngine.fromCompiledContent(lineIter, skipGenericCosmetic, skipCosmetic); - staticNetFilteringEngine.fromCompiledContent(lineIter); - } + if ( rawText === '' ) { return; } + var separator = '\n/* end of network - start of cosmetic */\n', + pos = rawText.indexOf(separator), + reader = new this.CompiledLineReader(rawText.slice(0, pos)); + this.staticNetFilteringEngine.fromCompiledContent(reader); + this.cosmeticFilteringEngine.fromCompiledContent( + reader.reset(rawText.slice(pos + separator.length)), + this.userSettings.ignoreGenericCosmeticFilters, + !firstparty && !this.userSettings.parseAllABPHideFilters + ); }; /******************************************************************************/ diff --git a/src/js/utils.js b/src/js/utils.js index 6ab0f9fb8..ec14bef2e 100644 --- a/src/js/utils.js +++ b/src/js/utils.js @@ -223,51 +223,58 @@ /******************************************************************************/ -µBlock.CompiledOutput = function() { - this.bufferLen = 8192; - this.buffer = new Uint8Array(this.bufferLen); - this.offset = 0; +µBlock.CompiledLineWriter = function() { + this.output = []; + this.stringifier = JSON.stringify; }; -µBlock.CompiledOutput.prototype.push = function(lineBits, line) { - var lineLen = line.length, - offset = this.offset, - need = offset + 2 + lineLen; // lineBits, line, \n - if ( need > this.bufferLen ) { - this.grow(need); +µBlock.CompiledLineWriter.fingerprint = function(args) { + return JSON.stringify(args); +}; + +µBlock.CompiledLineWriter.prototype = { + push: function(args) { + this.output[this.output.length] = this.stringifier(args); + }, + toString: function() { + return this.output.join('\n'); } - var buffer = this.buffer; - if ( offset !== 0 ) { - buffer[offset++] = 0x0A /* '\n' */; - } - buffer[offset++] = 0x61 /* 'a' */ + lineBits; - for ( var i = 0, c; i < lineLen; i++ ) { - c = line.charCodeAt(i); - if ( c > 0x7F ) { - return this.push(lineBits | 0x02, encodeURIComponent(line)); +}; + +µBlock.CompiledLineReader = function(raw) { + this.reset(raw); + this.parser = JSON.parse; +}; + +µBlock.CompiledLineReader.prototype = { + reset: function(raw) { + this.input = raw; + this.len = raw.length; + this.offset = 0; + this.s = ''; + return this; + }, + next: function() { + if ( this.offset === this.len ) { + this.s = ''; + return false; } - buffer[offset++] = c; + var pos = this.input.indexOf('\n', this.offset); + if ( pos !== -1 ) { + this.s = this.input.slice(this.offset, pos); + this.offset = pos + 1; + } else { + this.s = this.input.slice(this.offset); + this.offset = this.len; + } + return true; + }, + fingerprint: function() { + return this.s; + }, + args: function() { + return this.parser(this.s); } - this.offset = offset; -}; - -µBlock.CompiledOutput.prototype.grow = function(need) { - var newBufferLen = Math.min( - 2097152, - 1 << Math.ceil(Math.log(need) / Math.log(2)) - ); - while ( newBufferLen < need ) { - newBufferLen += 1048576; - } - var newBuffer = new Uint8Array(newBufferLen); - newBuffer.set(this.buffer); - this.buffer = newBuffer; - this.bufferLen = newBufferLen; -}; - -µBlock.CompiledOutput.prototype.toString = function() { - var decoder = new TextDecoder(); - return decoder.decode(new Uint8Array(this.buffer.buffer, 0, this.offset)); }; /******************************************************************************/