Mirrors
/
uBlock
mirror of https://github.com/gorhill/uBlock.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

this mitigates #520, #376

This commit is contained in:
gorhill 2015-02-23 18:31:29 -05:00
parent ae15aca35f
commit c7bab5502e
12 changed files with 977 additions and 579 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
platform/chromium/manifest.json
View File

@ -2,7 +2,7 @@
"manifest_version": 2,
"name": "µBlock",
"version": "0.8.8.4",
"version": "0.8.8.5",
"default_locale": "en",
"description": "__MSG_extShortDesc__",

2
src/js/3p-filters.js
View File

@ -43,7 +43,7 @@ var re3rdPartyExternalAsset = /^https?:\/\/[a-z0-9]+/;
var onMessage = function(msg) {
switch ( msg.what ) {
case 'loadUbiquitousBlacklistCompleted':
case 'allFilterListsReloaded':
renderBlacklists();
break;

157
src/js/assets.js
View File

@ -57,6 +57,7 @@ var thirdpartiesRepositoryRoot = 'https://raw.githubusercontent.com/gorhill/uAss
var nullFunc = function() {};
var reIsExternalPath = /^[a-z]+:\/\//;
var reIsUserPath = /^assets\/user\//;
var reIsCachePath = /^cache:\/\//;
var lastRepoMetaTimestamp = 0;
var lastRepoMetaIsRemote = false;
var refreshRepoMetaPeriod = 5 * oneHour;
@ -197,17 +198,26 @@ var cachedAssetsManager = (function() {
var cachedContentPath = cachedAssetPathPrefix + path;
var bin = {};
bin[cachedContentPath] = content;
var removedItems = [];
var onSaved = function() {
var lastError = vAPI.lastError();
if ( lastError ) {
details.error = 'Error: ' + lastError.message;
console.error('µBlock> cachedAssetsManager.save():', details.error);
cbError(details);
} else {
cbSuccess(details);
return;
}
// Saving over an existing item must be seen as removing an
// existing item and adding a new one.
if ( typeof exports.onRemovedListener === 'function' ) {
exports.onRemovedListener(removedItems);
}
cbSuccess(details);
};
var onEntries = function(entries) {
if ( entries.hasOwnProperty(path) ) {
removedItems.push(path);
}
entries[path] = Date.now();
bin.cached_asset_entries = entries;
vAPI.storage.set(bin, onSaved);
@ -218,6 +228,7 @@ var cachedAssetsManager = (function() {
exports.remove = function(pattern, before) {
var onEntries = function(entries) {
var keystoRemove = [];
var removedItems = [];
var paths = Object.keys(entries);
var i = paths.length;
var path;
@ -232,12 +243,16 @@ var cachedAssetsManager = (function() {
if ( typeof before === 'number' && entries[path] >= before ) {
continue;
}
removedItems.push(path);
keystoRemove.push(cachedAssetPathPrefix + path);
delete entries[path];
}
if ( keystoRemove.length ) {
vAPI.storage.remove(keystoRemove);
vAPI.storage.set({ 'cached_asset_entries': entries });
if ( typeof exports.onRemovedListener === 'function' ) {
exports.onRemovedListener(removedItems);
}
}
};
getEntries(onEntries);
@ -245,8 +260,10 @@ var cachedAssetsManager = (function() {
exports.removeAll = function(callback) {
var onEntries = function() {
// Careful! do not remove 'assets/user/'
exports.remove(/^https?:\/\/[a-z0-9]+/);
exports.remove(/^assets\/(ublock|thirdparties)\//);
exports.remove(/^cache:\/\//);
exports.remove('assets/checksums.txt');
if ( typeof callback === 'function' ) {
callback(null);
@ -255,6 +272,8 @@ var cachedAssetsManager = (function() {
getEntries(onEntries);
};
exports.onRemovedListener = null;
return exports;
})();
@ -263,15 +282,18 @@ var cachedAssetsManager = (function() {
var getTextFileFromURL = function(url, onLoad, onError) {
// https://github.com/gorhill/uMatrix/issues/15
var onResponseReceived = function() {
if ( this.status !== 0 && ( this.status < 200 || this.status >= 300 ) ) {
this.onload = this.onerror = this.ontimeout = null;
// xhr for local files gives status 0, but actually succeeds
var status = this.status || 200;
if ( status < 200 || status >= 300 ) {
return onError.call(this);
}
// xhr for local files gives status 0, but actually succeeds
if ( this.status === 0 && stringIsNotEmpty(this.responseText) === false ) {
// consider an empty result to be an error
if ( stringIsNotEmpty(this.responseText) === false ) {
return onError.call(this);
}
// we never download anything else than plain text: discard if response
// appears to be a HTML document: could happen when server returns
// appears to be a HTML document: could happen when server serves
// some kind of error page I suppose
var text = this.responseText.trim();
if ( text.charAt(0) === '<' && text.slice(-1) === '>' ) {
@ -279,15 +301,23 @@ var getTextFileFromURL = function(url, onLoad, onError) {
}
return onLoad.call(this);
};
var onErrorReceived = function() {
this.onload = this.onerror = this.ontimeout = null;
onError.call(this);
};
// console.log('µBlock> getTextFileFromURL("%s"):', url);
var xhr = new XMLHttpRequest();
xhr.open('get', url, true);
xhr.timeout = 30000;
xhr.onload = onResponseReceived;
xhr.onerror = onError;
xhr.ontimeout = onError;
xhr.responseType = 'text';
xhr.send();
try {
xhr.open('get', url, true);
xhr.timeout = 30000;
xhr.onload = onResponseReceived;
xhr.onerror = onErrorReceived;
xhr.ontimeout = onErrorReceived;
xhr.responseType = 'text';
xhr.send();
} catch (e) {
onErrorReceived.call(xhr);
}
};
/******************************************************************************/
@ -454,13 +484,11 @@ var readLocalFile = function(path, callback) {
};
var onInstallFileLoaded = function() {
this.onload = this.onerror = null;
//console.log('µBlock> readLocalFile("%s") / onInstallFileLoaded()', path);
reportBack(this.responseText);
};
var onInstallFileError = function() {
this.onload = this.onerror = null;
console.error('µBlock> readLocalFile("%s") / onInstallFileError()', path);
reportBack('', 'Error');
};
@ -512,7 +540,6 @@ var readRepoFile = function(path, callback) {
var repositoryURL = projectRepositoryRoot + path;
var onRepoFileLoaded = function() {
this.onload = this.onerror = null;
//console.log('µBlock> readRepoFile("%s") / onRepoFileLoaded()', path);
// https://github.com/gorhill/httpswitchboard/issues/263
if ( this.status === 200 ) {
@ -523,7 +550,6 @@ var readRepoFile = function(path, callback) {
};
var onRepoFileError = function() {
this.onload = this.onerror = null;
console.error(errorCantConnectTo.replace('{{url}}', repositoryURL));
reportBack('', 'Error');
};
@ -568,13 +594,11 @@ var readRepoCopyAsset = function(path, callback) {
};
var onInstallFileLoaded = function() {
this.onload = this.onerror = null;
//console.log('µBlock> readRepoCopyAsset("%s") / onInstallFileLoaded()', path);
reportBack(this.responseText);
};
var onInstallFileError = function() {
this.onload = this.onerror = null;
console.error('µBlock> readRepoCopyAsset("%s") / onInstallFileError():', path, this.statusText);
reportBack('', 'Error');
};
@ -593,7 +617,6 @@ var readRepoCopyAsset = function(path, callback) {
var repositoryURLSkipCache = repositoryURL + '?ublock=' + Date.now();
var onRepoFileLoaded = function() {
this.onload = this.onerror = null;
if ( stringIsNotEmpty(this.responseText) === false ) {
console.error('µBlock> readRepoCopyAsset("%s") / onRepoFileLoaded("%s"): error', path, repositoryURL);
cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError);
@ -605,13 +628,11 @@ var readRepoCopyAsset = function(path, callback) {
};
var onRepoFileError = function() {
this.onload = this.onerror = null;
console.error(errorCantConnectTo.replace('{{url}}', repositoryURL));
cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError);
};
var onHomeFileLoaded = function() {
this.onload = this.onerror = null;
if ( stringIsNotEmpty(this.responseText) === false ) {
console.error('µBlock> readRepoCopyAsset("%s") / onHomeFileLoaded("%s"): no response', path, homeURL);
// Fetch from repo only if obsolescence was due to repo checksum
@ -628,7 +649,6 @@ var readRepoCopyAsset = function(path, callback) {
};
var onHomeFileError = function() {
this.onload = this.onerror = null;
console.error(errorCantConnectTo.replace('{{url}}', homeURL));
// Fetch from repo only if obsolescence was due to repo checksum
if ( assetEntry.localChecksum !== assetEntry.repoChecksum ) {
@ -715,13 +735,11 @@ var readRepoOnlyAsset = function(path, callback) {
};
var onInstallFileLoaded = function() {
this.onload = this.onerror = null;
//console.log('µBlock> readRepoOnlyAsset("%s") / onInstallFileLoaded()', path);
reportBack(this.responseText);
};
var onInstallFileError = function() {
this.onload = this.onerror = null;
console.error('µBlock> readRepoOnlyAsset("%s") / onInstallFileError()', path);
reportBack('', 'Error');
};
@ -739,7 +757,6 @@ var readRepoOnlyAsset = function(path, callback) {
var repositoryURL = projectRepositoryRoot + path + '?ublock=' + Date.now();
var onRepoFileLoaded = function() {
this.onload = this.onerror = null;
if ( typeof this.responseText !== 'string' ) {
console.error('µBlock> readRepoOnlyAsset("%s") / onRepoFileLoaded("%s"): no response', path, repositoryURL);
cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError);
@ -757,7 +774,6 @@ var readRepoOnlyAsset = function(path, callback) {
};
var onRepoFileError = function() {
this.onload = this.onerror = null;
console.error(errorCantConnectTo.replace('{{url}}', repositoryURL));
cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError);
};
@ -827,7 +843,6 @@ var readExternalAsset = function(path, callback) {
};
var onExternalFileLoaded = function() {
this.onload = this.onerror = null;
// https://github.com/gorhill/uBlock/issues/708
// A successful download should never return an empty file: turn this
// into an error condition.
@ -841,7 +856,6 @@ var readExternalAsset = function(path, callback) {
};
var onExternalFileError = function() {
this.onload = this.onerror = null;
console.error(errorCantConnectTo.replace('{{url}}', path));
cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError);
};
@ -874,12 +888,33 @@ var readExternalAsset = function(path, callback) {
var readUserAsset = function(path, callback) {
var onCachedContentLoaded = function(details) {
//console.log('µBlock> readUserAsset("%s") / onCachedContentLoaded()', path);
//console.log('µBlock.assets/readUserAsset("%s")/onCachedContentLoaded()', path);
callback({ 'path': path, 'content': details.content });
};
var onCachedContentError = function() {
//console.log('µBlock> readUserAsset("%s") / onCachedContentError()', path);
//console.log('µBlock.assets/readUserAsset("%s")/onCachedContentError()', path);
callback({ 'path': path, 'content': '' });
};
cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError);
};
/******************************************************************************/
// Asset available only from the cache.
// Cache data:
// Path --> starts with 'cache://'
// Cache --> whatever
var readCacheAsset = function(path, callback) {
var onCachedContentLoaded = function(details) {
//console.log('µBlock.assets/readCacheAsset("%s")/onCachedContentLoaded()', path);
callback({ 'path': path, 'content': details.content });
};
var onCachedContentError = function() {
//console.log('µBlock.assets/readCacheAsset("%s")/onCachedContentError()', path);
callback({ 'path': path, 'content': '' });
};
@ -930,6 +965,11 @@ exports.get = function(path, callback) {
return;
}
if ( reIsCachePath.test(path) ) {
readCacheAsset(path, callback);
return;
}
if ( reIsExternalPath.test(path) ) {
readExternalAsset(path, callback);
return;
@ -1042,14 +1082,20 @@ exports.purge = function(pattern, before) {
cachedAssetsManager.remove(pattern, before);
};
/******************************************************************************/
exports.purgeAll = function(callback) {
cachedAssetsManager.removeAll(callback);
};
/******************************************************************************/
exports.onAssetCacheRemoved = {
addEventListener: function(callback) {
cachedAssetsManager.onRemovedListener = callback || null;
}
};
/******************************************************************************/
return exports;
})();
@ -1076,8 +1122,9 @@ var updated = {};
var updatedCount = 0;
var metadata = null;
var onStart = null;
var onCompleted = null;
var onStartListener = null;
var onCompletedListener = null;
var onAssetUpdatedListener = null;
var exports = {};
@ -1086,15 +1133,15 @@ var exports = {};
var onAssetUpdated = function(details) {
var path = details.path;
if ( details.error ) {
//console.debug('assets.js > µBlock.assetUpdater/onAssetUpdated: "%s" failed', path);
//console.debug('µBlock.assetUpdater/onAssetUpdated: "%s" failed', path);
return;
}
//console.debug('assets.js > µBlock.assetUpdater/onAssetUpdated: "%s"', path);
//console.debug('µBlock.assetUpdater/onAssetUpdated: "%s"', path);
updated[path] = true;
updatedCount += 1;
// New data available: selfie is now invalid
µb.destroySelfie();
if ( typeof onAssetUpdatedListener === 'function' ) {
onAssetUpdatedListener(details);
}
};
/******************************************************************************/
@ -1117,7 +1164,7 @@ var updateOne = function() {
if ( !metaEntry.cacheObsolete && !metaEntry.repoObsolete ) {
continue;
}
//console.debug('assets.js > µBlock.assetUpdater/updateOne: assets.get("%s")', path);
//console.debug('µBlock.assetUpdater/updateOne: assets.get("%s")', path);
µb.assets.get(path, onAssetUpdated);
break;
}
@ -1144,10 +1191,10 @@ var updateDaemon = function() {
// Start an update cycle?
if ( updateCycleTime !== 0 ) {
if ( Date.now() >= updateCycleTime ) {
//console.debug('assets.js > µBlock.assetUpdater/updateDaemon: update cycle started');
//console.debug('µBlock.assetUpdater/updateDaemon: update cycle started');
reset();
if ( onStart !== null ) {
onStart();
if ( typeof onStartListener === 'function' ) {
onStartListener();
}
}
return;
@ -1166,9 +1213,9 @@ var updateDaemon = function() {
// If anything was updated, notify listener
if ( updatedCount !== 0 ) {
if ( onCompleted !== null ) {
//console.debug('assets.js > µBlock.assetUpdater/updateDaemon: update cycle completed');
onCompleted({
if ( typeof onCompletedListener === 'function' ) {
//console.debug('µBlock.assetUpdater/updateDaemon: update cycle completed');
onCompletedListener({
updated: JSON.parse(JSON.stringify(updated)), // give callee its own safe copy
updatedCount: updatedCount
});
@ -1178,7 +1225,7 @@ var updateDaemon = function() {
// Schedule next update cycle
if ( updateCycleTime === 0 ) {
reset();
//console.debug('assets.js > µBlock.assetUpdater/updateDaemon: update cycle re-scheduled');
//console.debug('µBlock.assetUpdater/updateDaemon: update cycle re-scheduled');
updateCycleTime = Date.now() + updateCycleNextPeriod;
}
};
@ -1200,8 +1247,8 @@ var reset = function() {
exports.onStart = {
addEventListener: function(callback) {
onStart = callback || null;
if ( onStart !== null ) {
onStartListener = callback || null;
if ( typeof onStartListener === 'function' ) {
updateCycleTime = Date.now() + updateCycleFirstPeriod;
}
}
@ -1209,9 +1256,17 @@ exports.onStart = {
/******************************************************************************/
exports.onAssetUpdated = {
addEventListener: function(callback) {
onAssetUpdatedListener = callback || null;
}
};
/******************************************************************************/
exports.onCompleted = {
addEventListener: function(callback) {
onCompleted = callback || null;
onCompletedListener = callback || null;
}
};

7
src/js/background.js
View File

@ -83,6 +83,12 @@ return {
allowedRequestCount: 0
},
// read-only
systemSettings: {
compiledMagic: 'dgycowxrdjuf',
selfieMagic: 'dmakcrbecglp'
},
// EasyList, EasyPrivacy and many others have an 4-day update period,
// as per list headers.
updateAssetsEvery: 97 * oneHour,
@ -111,7 +117,6 @@ return {
remoteBlacklists: {
},
selfieMagic: 'bizhviclttie',
selfieAfter: 23 * oneMinute,
pageStores: {},

1
src/js/contentscript-end.js
View File

@ -334,7 +334,6 @@ var messager = vAPI.messaging.channel('contentscript-end.js');
hash = href.slice(pos + 3, pos + 11);
selectors = generics[hash];
if ( selectors === undefined ) { continue; }
selectors = selectors.split(',\n');
iSelector = selectors.length;
while ( iSelector-- ) {
selector = selectors[iSelector];

4
src/js/contentscript-start.js
View File

@ -71,8 +71,6 @@ var cosmeticFilters = function(details) {
var hide = details.cosmeticHide;
var i;
if ( donthide.length !== 0 ) {
donthide = donthide.length !== 1 ? donthide.join(',\n') : donthide[0];
donthide = donthide.split(',\n');
i = donthide.length;
while ( i-- ) {
donthideCosmeticFilters[donthide[i]] = true;
@ -80,8 +78,6 @@ var cosmeticFilters = function(details) {
}
// https://github.com/gorhill/uBlock/issues/143
if ( hide.length !== 0 ) {
hide = hide.length !== 1 ? hide.join(',\n') : hide[0];
hide = hide.split(',\n');
i = hide.length;
var selector;
while ( i-- ) {

475
src/js/cosmetic-filtering.js
View File

@ -481,7 +481,7 @@ var makeHash = function(unhide, token, mask) {
if ( unhide !== 0 ) {
hval |= 0x20000;
}
return String.fromCharCode(hval >>> 9, hval & 0x1FF);
return hval.toString(36);
};
/******************************************************************************/
@ -503,8 +503,8 @@ var makeHash = function(unhide, token, mask) {
// Specific filers can be enforced before the main document is loaded.
var FilterContainer = function() {
this.domainHashMask = (1 << 10) - 1;
this.genericHashMask = (1 << 15) - 1;
this.domainHashMask = (1 << 10) - 1; // 10 bits
this.genericHashMask = (1 << 15) - 1; // 15 bits
this.type0NoDomainHash = 'type0NoDomain';
this.type1NoDomainHash = 'type1NoDomain';
this.parser = new FilterParser();
@ -521,23 +521,15 @@ var FilterContainer = function() {
FilterContainer.prototype.reset = function() {
this.parser.reset();
this.µburi = µb.URI;
this.frozen = false;
this.acceptedCount = 0;
this.duplicateCount = 0;
this.duplicateBuster = {};
this.selectorCache = {};
this.selectorCacheCount = 0;
// temporary (at parse time)
this.lowGenericHide = {};
this.lowGenericDonthide = {};
this.highGenericHide = {};
this.highGenericDonthide = {};
this.hostnameHide = {};
this.hostnameDonthide = {};
this.entityHide = {};
this.entityDonthide = {};
// permanent
// [class], [id]
this.lowGenericFilters = {};
@ -555,9 +547,11 @@ FilterContainer.prototype.reset = function() {
this.highMediumGenericDonthideCount = 0;
// everything else
this.highHighGenericHideArray = [];
this.highHighGenericHide = '';
this.highHighGenericDonthide = '';
this.highHighGenericHideCount = 0;
this.highHighGenericDonthideArray = [];
this.highHighGenericDonthide = '';
this.highHighGenericDonthideCount = 0;
// hostname, entity-based filters
@ -567,7 +561,7 @@ FilterContainer.prototype.reset = function() {
/******************************************************************************/
FilterContainer.prototype.add = function(s) {
FilterContainer.prototype.compile = function(s, out) {
var parsed = this.parser.parse(s);
if ( parsed.invalid ) {
return false;
@ -576,271 +570,284 @@ FilterContainer.prototype.add = function(s) {
var hostnames = parsed.hostnames;
var i = hostnames.length;
if ( i === 0 ) {
this.addGenericSelector(parsed);
return true;
}
// https://github.com/gorhill/uBlock/issues/151
// Negated hostname means the filter applies to all non-negated hostnames
// of same filter OR globally if there is no non-negated hostnames.
var applyGlobally = true;
var hostname;
while ( i-- ) {
hostname = hostnames[i];
if ( hostname.charAt(0) !== '~' ) {
applyGlobally = false;
this.compileGenericSelector(parsed, out);
} else {
// https://github.com/gorhill/uBlock/issues/151
// Negated hostname means the filter applies to all non-negated hostnames
// of same filter OR globally if there is no non-negated hostnames.
var applyGlobally = true;
var hostname;
while ( i-- ) {
hostname = hostnames[i];
if ( hostname.charAt(0) !== '~' ) {
applyGlobally = false;
}
if ( hostname.slice(-2) === '.*' ) {
this.compileEntitySelector(hostname, parsed, out);
} else {
this.compileHostnameSelector(hostname, parsed, out);
}
}
if ( hostname.slice(-2) === '.*' ) {
this.addEntitySelector(hostname, parsed);
} else {
this.addHostnameSelector(hostname, parsed);
if ( applyGlobally ) {
this.compileGenericSelector(parsed, out);
}
}
if ( applyGlobally ) {
this.addGenericSelector(parsed);
}
return true;
};
/******************************************************************************/
FilterContainer.prototype.addGenericSelector = function(parsed) {
var entries;
var selectorType = parsed.suffix.charAt(0);
if ( selectorType === '#' || selectorType === '.' ) {
entries = parsed.unhide === 0 ?
this.lowGenericHide :
this.lowGenericDonthide;
} else {
entries = parsed.unhide === 0 ?
this.highGenericHide :
this.highGenericDonthide;
FilterContainer.prototype.compileGenericSelector = function(parsed, out) {
var selector = parsed.suffix;
var type = selector.charAt(0);
var matches;
if ( type === '#' || type === '.' ) {
matches = this.rePlainSelector.exec(selector);
if ( matches === null ) {
return;
}
out.push(
'c\v' +
(matches[1] === selector ? 'lg\v' : 'lg+\v') +
makeHash(parsed.unhide, matches[1], this.genericHashMask) + '\v' +
selector
);
return;
}
if ( entries[parsed.suffix] === undefined ) {
entries[parsed.suffix] = true;
} else {
//console.log('cosmetic-filtering.js > FilterContainer.addGenericSelector(): duplicate filter "%s"', parsed.suffix);
this.duplicateCount += 1;
// ["title"] and ["alt"] will go in high-low generic bin.
if ( this.reHighLow.test(selector) ) {
out.push(
'c\v' +
(parsed.unhide === 0 ? 'hlg0\v' : 'hlg1\v') +
selector
);
return;
}
this.acceptedCount += 1;
// [href^="..."] will go in high-medium generic bin.
matches = this.reHighMedium.exec(selector);
if ( matches && matches.length === 2 ) {
out.push(
'c\v' +
(parsed.unhide === 0 ? 'hmg0\v' : 'hmg1\v') +
matches[1] + '\v' +
selector
);
return;
}
// All else
out.push(
'c\v' +
(parsed.unhide === 0 ? 'hhg0\v' : 'hhg1\v') +
selector
);
};
FilterContainer.prototype.rePlainSelector = /^([#.][\w-]+)/;
FilterContainer.prototype.reHighLow = /^[a-z]*\[(?:alt|title)="[^"]+"\]$/;
FilterContainer.prototype.reHighMedium = /^\[href\^="https?:\/\/([^"]{8})[^"]*"\]$/;
/******************************************************************************/
FilterContainer.prototype.addHostnameSelector = function(hostname, parsed) {
FilterContainer.prototype.compileHostnameSelector = function(hostname, parsed, out) {
// https://github.com/gorhill/uBlock/issues/145
var unhide = parsed.unhide;
if ( hostname.charAt(0) === '~' ) {
hostname = hostname.slice(1);
unhide ^= 1;
}
var entries = unhide === 0 ?
this.hostnameHide :
this.hostnameDonthide;
var entry = entries[hostname];
if ( entry === undefined ) {
entry = entries[hostname] = {};
entry[parsed.suffix] = true;
} else if ( entry[parsed.suffix] === undefined ) {
entry[parsed.suffix] = true;
// https://github.com/gorhill/uBlock/issues/188
// If not a real domain as per PSL, assign a synthetic one
var hash;
var domain = this.µburi.domainFromHostname(hostname);
if ( domain === '' ) {
hash = unhide === 0 ? this.type0NoDomainHash : this.type1NoDomainHash;
} else {
//console.log('cosmetic-filtering.js > FilterContainer.addHostnameSelector(): duplicate filter "%s"', parsed.suffix);
this.duplicateCount += 1;
hash = makeHash(unhide, domain, this.domainHashMask);
}
this.acceptedCount += 1;
out.push(
'c\v' +
'h\v' +
hash + '\v' +
hostname + '\v' +
parsed.suffix
);
};
/******************************************************************************/
FilterContainer.prototype.addEntitySelector = function(hostname, parsed) {
var entries = parsed.unhide === 0 ?
this.entityHide :
this.entityDonthide;
FilterContainer.prototype.compileEntitySelector = function(hostname, parsed, out) {
var entity = hostname.slice(0, -2);
var entry = entries[entity];
if ( entry === undefined ) {
entry = entries[entity] = {};
entry[parsed.suffix] = true;
} else if ( entry[parsed.suffix] === undefined ) {
entry[parsed.suffix] = true;
} else {
//console.log('cosmetic-filtering.js > FilterContainer.addEntitySelector(): duplicate filter "%s"', parsed.suffix);
this.duplicateCount += 1;
}
this.acceptedCount += 1;
out.push(
'c\v' +
'e\v' +
entity + '\v' +
parsed.suffix
);
};
/******************************************************************************/
FilterContainer.prototype.freezeLowGenerics = function(what, type) {
var selectors = this[what];
var matches, selectorPrefix, f, hash, bucket;
for ( var selector in selectors ) {
if ( selectors.hasOwnProperty(selector) === false ) {
continue;
}
matches = this.rePlainSelector.exec(selector);
if ( !matches ) {
continue;
}
selectorPrefix = matches[1];
f = selectorPrefix === selector ?
new FilterPlain(selector) :
new FilterPlainMore(selector);
hash = makeHash(type, selectorPrefix, this.genericHashMask);
bucket = this.lowGenericFilters[hash];
if ( bucket === undefined ) {
this.lowGenericFilters[hash] = f;
} else if ( bucket instanceof FilterBucket ) {
bucket.add(f);
} else {
this.lowGenericFilters[hash] = new FilterBucket(bucket, f);
}
FilterContainer.prototype.fromCompiledContent = function(text, lineBeg, skip) {
if ( skip ) {
return this.skipCompiledContent(text, lineBeg);
}
this[what] = {};
};
FilterContainer.prototype.rePlainSelector = /^([#.][\w-]+)/;
var lineEnd;
var textEnd = text.length;
var line, fields, filter, bucket;
/******************************************************************************/
while ( lineBeg < textEnd ) {
if ( text.charAt(lineBeg) !== 'c' ) {
return lineBeg;
}
lineEnd = text.indexOf('\n', lineBeg);
if ( lineEnd === -1 ) {
lineEnd = textEnd;
}
line = text.slice(lineBeg + 2, lineEnd);
lineBeg = lineEnd + 1;
FilterContainer.prototype.freezeHostnameSpecifics = function(what, type) {
var µburi = µb.URI;
var entries = this[what];
var filters = this.hostnameFilters;
var f, domain, hash, bucket;
for ( var hostname in entries ) {
if ( entries.hasOwnProperty(hostname) === false ) {
this.acceptedCount += 1;
if ( this.duplicateBuster.hasOwnProperty(line) ) {
this.duplicateCount += 1;
continue;
}
f = new FilterHostname(Object.keys(entries[hostname]).join(',\n'), hostname);
// https://github.com/gorhill/uBlock/issues/188
// If not a real domain as per PSL, assign a synthetic one
domain = µburi.domainFromHostname(hostname);
if ( domain === '' ) {
hash = type === 0 ? this.type0NoDomainHash : this.type1NoDomainHash;
} else {
hash = makeHash(type, domain, this.domainHashMask);
}
bucket = filters[hash];
if ( bucket === undefined ) {
filters[hash] = f;
} else if ( bucket instanceof FilterBucket ) {
bucket.add(f);
} else {
filters[hash] = new FilterBucket(bucket, f);
}
}
this[what] = {};
};
this.duplicateBuster[line] = true;
/******************************************************************************/
fields = line.split('\v');
FilterContainer.prototype.freezeEntitySpecifics = function(what, type) {
var entries = this[what];
var filters = this.entityFilters;
var f, hash, bucket;
for ( var entity in entries ) {
if ( entries.hasOwnProperty(entity) === false ) {
continue;
}
f = new FilterEntity(Object.keys(entries[entity]).join(',\n'), entity);
hash = makeHash(type, entity, this.domainHashMask);
bucket = filters[hash];
if ( bucket === undefined ) {
filters[hash] = f;
} else if ( bucket instanceof FilterBucket ) {
bucket.add(f);
} else {
filters[hash] = new FilterBucket(bucket, f);
}
}
this[what] = {};
};
/******************************************************************************/
FilterContainer.prototype.freezeHighGenerics = function(what) {
var selectors = this['highGeneric' + what];
// ["title"] and ["alt"] will go in high-low generic bin.
var highLowGenericProp = 'highLowGeneric' + what;
var highLowGeneric = this[highLowGenericProp];
var highLowGenericCount = 0;
// [href^="..."] will go in high-medium generic bin.
var highMediumGenericProp = 'highMediumGeneric' + what;
var highMediumGeneric = this[highMediumGenericProp];
var highMediumGenericCount = 0;
// The rest will be put in the high-high generic bin.
// https://github.com/gorhill/uBlock/issues/236
// Insert whatever we already have
var highHighGeneric = [];
var highHighGenericProp = 'highHighGeneric' + what;
if ( this[highHighGenericProp] !== '' ) {
highHighGeneric.push(this[highHighGenericProp]);
}
var highHighGenericCount = 0;
// https://github.com/gorhill/uBlock/issues/456
// Include tag name, it's part of the filter
var reHighLow = /^[a-z]*\[(?:alt|title)="[^"]+"\]$/;
var reHighMedium = /^\[href\^="https?:\/\/([^"]{8})[^"]*"\]$/;
var matches, hash;
for ( var selector in selectors ) {
if ( selectors.hasOwnProperty(selector) === false ) {
continue;
}
// ["title"] and ["alt"] will go in high-low generic bin.
matches = reHighLow.exec(selector);
if ( matches && matches.length === 1 ) {
highLowGeneric[matches[0]] = true;
highLowGenericCount += 1;
continue;
}
// [href^="..."] will go in high-medium generic bin.
matches = reHighMedium.exec(selector);
if ( matches && matches.length === 2 ) {
hash = matches[1];
if ( highMediumGeneric[hash] === undefined ) {
highMediumGeneric[hash] = matches[0];
// h ir twitter.com .promoted-tweet
if ( fields[0] === 'h' ) {
filter = new FilterHostname(fields[3], fields[2]);
bucket = this.hostnameFilters[fields[1]];
if ( bucket === undefined ) {
this.hostnameFilters[fields[1]] = filter;
} else if ( bucket instanceof FilterBucket ) {
bucket.add(filter);
} else {
highMediumGeneric[hash] += ',\n' + matches[0];
this.hostnameFilters[fields[1]] = new FilterBucket(bucket, filter);
}
highMediumGenericCount += 1;
continue;
}
// All else
highHighGeneric.push(selector);
highHighGenericCount += 1;
// lg 105 .largeAd
// lg+ 2jx .Mpopup + #Mad > #MadZone
if ( fields[0] === 'lg' || fields[0] === 'lg+' ) {
filter = fields[0] === 'lg' ?
new FilterPlain(fields[2]) :
new FilterPlainMore(fields[2]);
bucket = this.lowGenericFilters[fields[1]];
if ( bucket === undefined ) {
this.lowGenericFilters[fields[1]] = filter;
} else if ( bucket instanceof FilterBucket ) {
bucket.add(filter);
} else {
this.lowGenericFilters[fields[1]] = new FilterBucket(bucket, filter);
}
continue;
}
// entity selector
if ( fields[0] === 'e' ) {
bucket = this.entityFilters[fields[1]];
if ( bucket === undefined ) {
this.entityFilters[fields[1]] = [fields[2]];
} else {
bucket.push(fields[2]);
}
continue;
}
if ( fields[0] === 'hlg0' ) {
this.highLowGenericHide[fields[1]] = true;
this.highLowGenericHideCount += 1;
continue;
}
if ( fields[0] === 'hlg1' ) {
this.highLowGenericDonthide[fields[1]] = true;
this.highLowGenericDonthideCount += 1;
continue;
}
if ( fields[0] === 'hmg0' ) {
if ( Array.isArray(this.highMediumGenericHide[fields[1]]) ) {
this.highMediumGenericHide[fields[1]].push(fields[2]);
} else {
this.highMediumGenericHide[fields[1]] = [fields[2]];
}
this.highMediumGenericHideCount += 1;
continue;
}
if ( fields[0] === 'hmg1' ) {
if ( Array.isArray(this.highMediumGenericDonthide[fields[1]]) ) {
this.highMediumGenericDonthide[fields[1]].push(fields[2]);
} else {
this.highMediumGenericDonthide[fields[1]] = [fields[2]];
}
this.highMediumGenericDonthideCount += 1;
continue;
}
if ( fields[0] === 'hhg0' ) {
this.highHighGenericHideArray.push(fields[1]);
this.highHighGenericHideCount += 1;
continue;
}
if ( fields[0] === 'hhg1' ) {
this.highHighGenericDonthideArray.push(fields[1]);
this.highHighGenericDonthideCount += 1;
continue;
}
}
return textEnd;
};
this[highLowGenericProp + 'Count'] += highLowGenericCount;
this[highMediumGenericProp + 'Count'] += highMediumGenericCount;
this[highHighGenericProp] = highHighGeneric.join(',\n');
this[highHighGenericProp + 'Count'] += highHighGenericCount;
/******************************************************************************/
// Empty cumulated selectors
this['highGeneric' + what] = {};
FilterContainer.prototype.skipCompiledContent = function(text, lineBeg) {
var lineEnd;
var textEnd = text.length;
while ( lineBeg < textEnd ) {
if ( text.charAt(lineBeg) !== 'c' ) {
return lineBeg;
}
lineEnd = text.indexOf('\n', lineBeg);
if ( lineEnd === -1 ) {
lineEnd = textEnd;
}
lineBeg = lineEnd + 1;
}
return textEnd;
};
/******************************************************************************/
FilterContainer.prototype.freeze = function() {
this.freezeLowGenerics('lowGenericHide', 0);
this.freezeLowGenerics('lowGenericDonthide', 1);
this.freezeHighGenerics('Hide');
this.freezeHighGenerics('Donthide');
this.freezeHostnameSpecifics('hostnameHide', 0);
this.freezeHostnameSpecifics('hostnameDonthide', 1);
this.freezeEntitySpecifics('entityHide', 0);
this.freezeEntitySpecifics('entityDonthide', 1);
this.duplicateBuster = {};
if ( this.highHighGenericHide !== '' ) {
this.highHighGenericHideArray.unshift(this.highHighGenericHide);
}
this.highHighGenericHide = this.highHighGenericHideArray.join(',\n');
this.highHighGenericHideArray = [];
if ( this.highHighGenericDonthide !== '' ) {
this.highHighGenericDonthideArray.unshift(this.highHighGenericDonthide);
}
this.highHighGenericDonthide = this.highHighGenericDonthideArray.join(',\n');
this.highHighGenericDonthideArray = [];
this.parser.reset();
this.frozen = true;
//histogram('lowGenericFilters', this.lowGenericFilters);
//histogram('hostnameFilters', this.hostnameFilters);
};
/******************************************************************************/
@ -875,7 +882,7 @@ FilterContainer.prototype.toSelfie = function() {
acceptedCount: this.acceptedCount,
duplicateCount: this.duplicateCount,
hostnameSpecificFilters: selfieFromDict(this.hostnameFilters),
entitySpecificFilters: selfieFromDict(this.entityFilters),
entitySpecificFilters: this.entityFilters,
lowGenericFilters: selfieFromDict(this.lowGenericFilters),
highLowGenericHide: this.highLowGenericHide,
highLowGenericDonthide: this.highLowGenericDonthide,
@ -940,7 +947,7 @@ FilterContainer.prototype.fromSelfie = function(selfie) {
this.acceptedCount = selfie.acceptedCount;
this.duplicateCount = selfie.duplicateCount;
this.hostnameFilters = dictFromSelfie(selfie.hostnameSpecificFilters);
this.entityFilters = dictFromSelfie(selfie.entitySpecificFilters);
this.entityFilters = selfie.entitySpecificFilters;
this.lowGenericFilters = dictFromSelfie(selfie.lowGenericFilters);
this.highLowGenericHide = selfie.highLowGenericHide;
this.highLowGenericDonthide = selfie.highLowGenericDonthide;
@ -1156,14 +1163,18 @@ FilterContainer.prototype.retrieveDomainSelectors = function(request) {
if ( bucket = this.hostnameFilters[this.type0NoDomainHash] ) {
bucket.retrieve(hostname, r.cosmeticHide);
}
hash = makeHash(0, r.entity, this.domainHashMask);
if ( bucket = this.entityFilters[hash] ) {
bucket.retrieve(pos === -1 ? domain : hostname.slice(0, pos - domain.length), r.cosmeticHide);
// entity filter buckets are always plain js array
if ( bucket = this.entityFilters[r.entity] ) {
r.cosmeticHide = r.cosmeticHide.concat(bucket);
}
// No entity exceptions as of now
hash = makeHash(1, domain, this.domainHashMask);
if ( bucket = this.hostnameFilters[hash] ) {
bucket.retrieve(hostname, r.cosmeticDonthide);
}
// https://github.com/gorhill/uBlock/issues/188
// Special bucket for those filters without a valid domain name as per PSL
if ( bucket = this.hostnameFilters[this.type1NoDomainHash] ) {

2
src/js/messaging.js
View File

@ -68,7 +68,7 @@ var onMessage = function(request, sender, callback) {
break;
case 'reloadAllFilters':
µb.reloadPresetBlacklists(request.switches, request.update);
µb.reloadFilterLists(request.switches, request.update);
break;
case 'reloadTab':

3
src/js/profiler.js
View File

@ -20,11 +20,12 @@
*/
/* exported quickProfiler */
'use strict';
/******************************************************************************/
var quickProfiler = (function() {
'use strict';
var timer = window.performance || Date;
var time = 0;
var count = 0;

37
src/js/start.js
View File

@ -27,6 +27,8 @@
(function() {
quickProfiler.start('start.js');
/******************************************************************************/
// Final initialization steps after all needed assets are in memory.
@ -40,11 +42,15 @@ var onAllReady = function() {
// Check for updates not too far in the future.
µb.assetUpdater.onStart.addEventListener(µb.updateStartHandler.bind(µb));
µb.assetUpdater.onCompleted.addEventListener(µb.updateCompleteHandler.bind(µb));
µb.assetUpdater.onAssetUpdated.addEventListener(µb.assetUpdatedHandler.bind(µb));
µb.assets.onAssetCacheRemoved.addEventListener(µb.assetCacheRemovedHandler.bind(µb));
// Important: remove barrier to remote fetching, this was useful only
// for launch time.
µb.assets.allowRemoteFetch = true;
quickProfiler.stop(0);
vAPI.onLoadAllCompleted();
};
@ -155,9 +161,34 @@ var onUserSettingsReady = function(userSettings) {
µb.XAL.keyvalRemoveOne('logRequests');
};
µBlock.loadUserSettings(onUserSettingsReady);
µBlock.loadWhitelist(onWhitelistReady);
µBlock.loadLocalSettings();
/******************************************************************************/
// Housekeeping, as per system setting changes
var onSystemSettingsReady = function(system) {
var µb = µBlock;
var mustSaveSystemSettings = false;
if ( system.compiledMagic !== µb.systemSettings.compiledMagic ) {
µb.assets.purge(/^cache:\/\/compiled-/);
mustSaveSystemSettings = true;
}
if ( system.selfieMagic !== µb.systemSettings.selfieMagic ) {
µb.destroySelfie();
mustSaveSystemSettings = true;
}
if ( mustSaveSystemSettings ) {
µb.saveSystemSettings();
}
µb.loadUserSettings(onUserSettingsReady);
µb.loadWhitelist(onWhitelistReady);
µb.loadLocalSettings();
};
/******************************************************************************/
µBlock.loadSystemSettings(onSystemSettingsReady);
/******************************************************************************/

483
src/js/static-net-filtering.js
View File

@ -227,7 +227,7 @@ FilterPlain.prototype.match = function(url, tokenBeg) {
return url.substr(tokenBeg - this.tokenBeg, this.s.length) === this.s;
};
FilterPlain.prototype.fid = 'a';
FilterPlain.fid = FilterPlain.prototype.fid = 'a';
FilterPlain.prototype.toString = function() {
return this.s;
@ -238,6 +238,10 @@ FilterPlain.prototype.toSelfie = function() {
this.tokenBeg;
};
FilterPlain.compile = function(details) {
return details.f + '\t' + details.tokenBeg;
};
FilterPlain.fromSelfie = function(s) {
var pos = s.indexOf('\t');
return new FilterPlain(s.slice(0, pos), atoi(s.slice(pos + 1)));
@ -256,7 +260,7 @@ FilterPlainHostname.prototype.match = function(url, tokenBeg) {
url.substr(tokenBeg - this.tokenBeg, this.s.length) === this.s;
};
FilterPlainHostname.prototype.fid = 'ah';
FilterPlainHostname.fid = FilterPlainHostname.prototype.fid = 'ah';
FilterPlainHostname.prototype.toString = function() {
return this.s + '$domain=' + this.hostname;
@ -268,6 +272,12 @@ FilterPlainHostname.prototype.toSelfie = function() {
this.hostname;
};
FilterPlainHostname.compile = function(details, hostname) {
return details.f + '\t' +
details.tokenBeg + '\t' +
hostname;
};
FilterPlainHostname.fromSelfie = function(s) {
var args = s.split('\t');
return new FilterPlainHostname(args[0], atoi(args[1]), args[2]);
@ -283,7 +293,7 @@ FilterPlainPrefix0.prototype.match = function(url, tokenBeg) {
return url.substr(tokenBeg, this.s.length) === this.s;
};
FilterPlainPrefix0.prototype.fid = '0a';
FilterPlainPrefix0.fid = FilterPlainPrefix0.prototype.fid = '0a';
FilterPlainPrefix0.prototype.toString = function() {
return this.s;
@ -293,6 +303,10 @@ FilterPlainPrefix0.prototype.toSelfie = function() {
return this.s;
};
FilterPlainPrefix0.compile = function(details) {
return details.f;
};
FilterPlainPrefix0.fromSelfie = function(s) {
return new FilterPlainPrefix0(s);
};
@ -309,7 +323,7 @@ FilterPlainPrefix0Hostname.prototype.match = function(url, tokenBeg) {
url.substr(tokenBeg, this.s.length) === this.s;
};
FilterPlainPrefix0Hostname.prototype.fid = '0ah';
FilterPlainPrefix0Hostname.fid = FilterPlainPrefix0Hostname.prototype.fid = '0ah';
FilterPlainPrefix0Hostname.prototype.toString = function() {
return this.s + '$domain=' + this.hostname;
@ -320,6 +334,10 @@ FilterPlainPrefix0Hostname.prototype.toSelfie = function() {
this.hostname;
};
FilterPlainPrefix0Hostname.compile = function(details, hostname) {
return details.f + '\t' + hostname;
};
FilterPlainPrefix0Hostname.fromSelfie = function(s) {
var pos = s.indexOf('\t');
return new FilterPlainPrefix0Hostname(s.slice(0, pos), s.slice(pos + 1));
@ -335,7 +353,7 @@ FilterPlainPrefix1.prototype.match = function(url, tokenBeg) {
return url.substr(tokenBeg - 1, this.s.length) === this.s;
};
FilterPlainPrefix1.prototype.fid = '1a';
FilterPlainPrefix1.fid = FilterPlainPrefix1.prototype.fid = '1a';
FilterPlainPrefix1.prototype.toString = function() {
return this.s;
@ -345,6 +363,10 @@ FilterPlainPrefix1.prototype.toSelfie = function() {
return this.s;
};
FilterPlainPrefix1.compile = function(details) {
return details.f;
};
FilterPlainPrefix1.fromSelfie = function(s) {
return new FilterPlainPrefix1(s);
};
@ -361,7 +383,7 @@ FilterPlainPrefix1Hostname.prototype.match = function(url, tokenBeg) {
url.substr(tokenBeg - 1, this.s.length) === this.s;
};
FilterPlainPrefix1Hostname.prototype.fid = '1ah';
FilterPlainPrefix1Hostname.fid = FilterPlainPrefix1Hostname.prototype.fid = '1ah';
FilterPlainPrefix1Hostname.prototype.toString = function() {
return this.s + '$domain=' + this.hostname;
@ -372,6 +394,10 @@ FilterPlainPrefix1Hostname.prototype.toSelfie = function() {
this.hostname;
};
FilterPlainPrefix1Hostname.compile = function(details, hostname) {
return details.f + '\t' + hostname;
};
FilterPlainPrefix1Hostname.fromSelfie = function(s) {
var pos = s.indexOf('\t');
return new FilterPlainPrefix1Hostname(s.slice(0, pos), s.slice(pos + 1));
@ -387,7 +413,7 @@ FilterPlainLeftAnchored.prototype.match = function(url) {
return url.slice(0, this.s.length) === this.s;
};
FilterPlainLeftAnchored.prototype.fid = '|a';
FilterPlainLeftAnchored.fid = FilterPlainLeftAnchored.prototype.fid = '|a';
FilterPlainLeftAnchored.prototype.toString = function() {
return '|' + this.s;
@ -397,6 +423,10 @@ FilterPlainLeftAnchored.prototype.toSelfie = function() {
return this.s;
};
FilterPlainLeftAnchored.compile = function(details) {
return details.f;
};
FilterPlainLeftAnchored.fromSelfie = function(s) {
return new FilterPlainLeftAnchored(s);
};
@ -413,7 +443,7 @@ FilterPlainLeftAnchoredHostname.prototype.match = function(url) {
url.slice(0, this.s.length) === this.s;
};
FilterPlainLeftAnchoredHostname.prototype.fid = '|ah';
FilterPlainLeftAnchoredHostname.fid = FilterPlainLeftAnchoredHostname.prototype.fid = '|ah';
FilterPlainLeftAnchoredHostname.prototype.toString = function() {
return '|' + this.s + '$domain=' + this.hostname;
@ -424,6 +454,10 @@ FilterPlainLeftAnchoredHostname.prototype.toSelfie = function() {
this.hostname;
};
FilterPlainLeftAnchoredHostname.compile = function(details, hostname) {
return details.f + '\t' + hostname;
};
FilterPlainLeftAnchoredHostname.fromSelfie = function(s) {
var pos = s.indexOf('\t');
return new FilterPlainLeftAnchoredHostname(s.slice(0, pos), s.slice(pos + 1));
@ -439,7 +473,7 @@ FilterPlainRightAnchored.prototype.match = function(url) {
return url.slice(-this.s.length) === this.s;
};
FilterPlainRightAnchored.prototype.fid = 'a|';
FilterPlainRightAnchored.fid = FilterPlainRightAnchored.prototype.fid = 'a|';
FilterPlainRightAnchored.prototype.toString = function() {
return this.s + '|';
@ -449,6 +483,10 @@ FilterPlainRightAnchored.prototype.toSelfie = function() {
return this.s;
};
FilterPlainRightAnchored.compile = function(details) {
return details.f;
};
FilterPlainRightAnchored.fromSelfie = function(s) {
return new FilterPlainRightAnchored(s);
};
@ -465,7 +503,7 @@ FilterPlainRightAnchoredHostname.prototype.match = function(url) {
url.slice(-this.s.length) === this.s;
};
FilterPlainRightAnchoredHostname.prototype.fid = 'a|h';
FilterPlainRightAnchoredHostname.fid = FilterPlainRightAnchoredHostname.prototype.fid = 'a|h';
FilterPlainRightAnchoredHostname.prototype.toString = function() {
return this.s + '|$domain=' + this.hostname;
@ -476,6 +514,10 @@ FilterPlainRightAnchoredHostname.prototype.toSelfie = function() {
this.hostname;
};
FilterPlainRightAnchoredHostname.compile = function(details, hostname) {
return details.f + '\t' + hostname;
};
FilterPlainRightAnchoredHostname.fromSelfie = function(s) {
var pos = s.indexOf('\t');
return new FilterPlainRightAnchoredHostname(s.slice(0, pos), s.slice(pos + 1));
@ -500,7 +542,7 @@ FilterPlainHnAnchored.prototype.match = function(url, tokenBeg) {
reURLPostHostnameAnchors.test(url.slice(pos + 3, tokenBeg)) === false;
};
FilterPlainHnAnchored.prototype.fid = 'h|a';
FilterPlainHnAnchored.fid = FilterPlainHnAnchored.prototype.fid = 'h|a';
FilterPlainHnAnchored.prototype.toString = function() {
return '||' + this.s;
@ -510,6 +552,10 @@ FilterPlainHnAnchored.prototype.toSelfie = function() {
return this.s;
};
FilterPlainHnAnchored.compile = function(details) {
return details.f;
};
FilterPlainHnAnchored.fromSelfie = function(s) {
return new FilterPlainHnAnchored(s);
};
@ -535,7 +581,7 @@ FilterSingleWildcard.prototype.match = function(url, tokenBeg) {
url.indexOf(this.rSegment, tokenBeg + this.lSegment.length) > 0;
};
FilterSingleWildcard.prototype.fid = '*';
FilterSingleWildcard.fid = FilterSingleWildcard.prototype.fid = '*';
FilterSingleWildcard.prototype.toString = function() {
return this.lSegment + '*' + this.rSegment;
@ -547,6 +593,14 @@ FilterSingleWildcard.prototype.toSelfie = function() {
this.tokenBeg;
};
FilterSingleWildcard.compile = function(details) {
var s = details.f;
var pos = s.indexOf('*');
return s.slice(0, pos) + '\t' +
s.slice(pos + 1) + '\t' +
details.tokenBeg;
};
FilterSingleWildcard.fromSelfie = function(s) {
var args = s.split('\t');
return new FilterSingleWildcard(args[0], args[1], atoi(args[2]));
@ -568,7 +622,7 @@ FilterSingleWildcardHostname.prototype.match = function(url, tokenBeg) {
url.indexOf(this.rSegment, tokenBeg + this.lSegment.length) > 0;
};
FilterSingleWildcardHostname.prototype.fid = '*h';
FilterSingleWildcardHostname.fid = FilterSingleWildcardHostname.prototype.fid = '*h';
FilterSingleWildcardHostname.prototype.toString = function() {
return this.lSegment + '*' + this.rSegment + '$domain=' + this.hostname;
@ -581,6 +635,15 @@ FilterSingleWildcardHostname.prototype.toSelfie = function() {
this.hostname;
};
FilterSingleWildcardHostname.compile = function(details, hostname) {
var s = details.f;
var pos = s.indexOf('*');
return s.slice(0, pos) + '\t' +
s.slice(pos + 1) + '\t' +
details.tokenBeg + '\t' +
hostname;
};
FilterSingleWildcardHostname.fromSelfie = function(s) {
var args = s.split('\t');
return new FilterSingleWildcardHostname(args[0], args[1], atoi(args[2]), args[3]);
@ -598,7 +661,7 @@ FilterSingleWildcardPrefix0.prototype.match = function(url, tokenBeg) {
url.indexOf(this.rSegment, tokenBeg + this.lSegment.length) > 0;
};
FilterSingleWildcardPrefix0.prototype.fid = '0*';
FilterSingleWildcardPrefix0.fid = FilterSingleWildcardPrefix0.prototype.fid = '0*';
FilterSingleWildcardPrefix0.prototype.toString = function() {
return this.lSegment + '*' + this.rSegment;
@ -609,6 +672,12 @@ FilterSingleWildcardPrefix0.prototype.toSelfie = function() {
this.rSegment;
};
FilterSingleWildcardPrefix0.compile = function(details) {
var s = details.f;
var pos = s.indexOf('*');
return s.slice(0, pos) + '\t' + s.slice(pos + 1);
};
FilterSingleWildcardPrefix0.fromSelfie = function(s) {
var pos = s.indexOf('\t');
return new FilterSingleWildcardPrefix0(s.slice(0, pos), s.slice(pos + 1));
@ -628,7 +697,7 @@ FilterSingleWildcardPrefix0Hostname.prototype.match = function(url, tokenBeg) {
url.indexOf(this.rSegment, tokenBeg + this.lSegment.length) > 0;
};
FilterSingleWildcardPrefix0Hostname.prototype.fid = '0*h';
FilterSingleWildcardPrefix0Hostname.fid = FilterSingleWildcardPrefix0Hostname.prototype.fid = '0*h';
FilterSingleWildcardPrefix0Hostname.prototype.toString = function() {
return this.lSegment + '*' + this.rSegment + '$domain=' + this.hostname;
@ -640,6 +709,14 @@ FilterSingleWildcardPrefix0Hostname.prototype.toSelfie = function() {
this.hostname;
};
FilterSingleWildcardPrefix0Hostname.compile = function(details, hostname) {
var s = details.f;
var pos = s.indexOf('*');
return s.slice(0, pos) + '\t' +
s.slice(pos + 1) + '\t' +
hostname;
};
FilterSingleWildcardPrefix0Hostname.fromSelfie = function(s) {
var args = s.split('\t');
return new FilterSingleWildcardPrefix0Hostname(args[0], args[1], args[2]);
@ -657,7 +734,7 @@ FilterSingleWildcardLeftAnchored.prototype.match = function(url) {
url.indexOf(this.rSegment, this.lSegment.length) > 0;
};
FilterSingleWildcardLeftAnchored.prototype.fid = '|*';
FilterSingleWildcardLeftAnchored.fid = FilterSingleWildcardLeftAnchored.prototype.fid = '|*';
FilterSingleWildcardLeftAnchored.prototype.toString = function() {
return '|' + this.lSegment + '*' + this.rSegment;
@ -668,6 +745,13 @@ FilterSingleWildcardLeftAnchored.prototype.toSelfie = function() {
this.rSegment;
};
FilterSingleWildcardLeftAnchored.compile = function(details) {
var s = details.f;
var pos = s.indexOf('*');
return s.slice(0, pos) + '\t' +
s.slice(pos + 1) + '\t';
};
FilterSingleWildcardLeftAnchored.fromSelfie = function(s) {
var pos = s.indexOf('\t');
return new FilterSingleWildcardLeftAnchored(s.slice(0, pos), s.slice(pos + 1));
@ -687,7 +771,7 @@ FilterSingleWildcardLeftAnchoredHostname.prototype.match = function(url) {
url.indexOf(this.rSegment, this.lSegment.length) > 0;
};
FilterSingleWildcardLeftAnchoredHostname.prototype.fid = '|*h';
FilterSingleWildcardLeftAnchoredHostname.fid = FilterSingleWildcardLeftAnchoredHostname.prototype.fid = '|*h';
FilterSingleWildcardLeftAnchoredHostname.prototype.toString = function() {
return '|' + this.lSegment + '*' + this.rSegment + '$domain=' + this.hostname;
@ -699,6 +783,14 @@ FilterSingleWildcardLeftAnchoredHostname.prototype.toSelfie = function() {
this.hostname;
};
FilterSingleWildcardLeftAnchoredHostname.compile = function(details, hostname) {
var s = details.f;
var pos = s.indexOf('*');
return s.slice(0, pos) + '\t' +
s.slice(pos + 1) + '\t' +
hostname;
};
FilterSingleWildcardLeftAnchoredHostname.fromSelfie = function(s) {
var args = s.split('\t');
return new FilterSingleWildcardLeftAnchoredHostname(args[0], args[1], args[2]);
@ -716,7 +808,7 @@ FilterSingleWildcardRightAnchored.prototype.match = function(url) {
url.lastIndexOf(this.lSegment, url.length - this.rSegment.length - this.lSegment.length) >= 0;
};
FilterSingleWildcardRightAnchored.prototype.fid = '*|';
FilterSingleWildcardRightAnchored.fid = FilterSingleWildcardRightAnchored.prototype.fid = '*|';
FilterSingleWildcardRightAnchored.prototype.toString = function() {
return this.lSegment + '*' + this.rSegment + '|';
@ -727,6 +819,13 @@ FilterSingleWildcardRightAnchored.prototype.toSelfie = function() {
this.rSegment;
};
FilterSingleWildcardRightAnchored.compile = function(details) {
var s = details.f;
var pos = s.indexOf('*');
return s.slice(0, pos) + '\t' +
s.slice(pos + 1) + '\t';
};
FilterSingleWildcardRightAnchored.fromSelfie = function(s) {
var pos = s.indexOf('\t');
return new FilterSingleWildcardRightAnchored(s.slice(0, pos), s.slice(pos + 1));
@ -746,7 +845,7 @@ FilterSingleWildcardRightAnchoredHostname.prototype.match = function(url) {
url.lastIndexOf(this.lSegment, url.length - this.rSegment.length - this.lSegment.length) >= 0;
};
FilterSingleWildcardRightAnchoredHostname.prototype.fid = '*|h';
FilterSingleWildcardRightAnchoredHostname.fid = FilterSingleWildcardRightAnchoredHostname.prototype.fid = '*|h';
FilterSingleWildcardRightAnchoredHostname.prototype.toString = function() {
return this.lSegment + '*' + this.rSegment + '|$domain=' + this.hostname;
@ -758,6 +857,14 @@ FilterSingleWildcardRightAnchoredHostname.prototype.toSelfie = function() {
this.hostname;
};
FilterSingleWildcardRightAnchoredHostname.compile = function(details, hostname) {
var s = details.f;
var pos = s.indexOf('*');
return s.slice(0, pos) + '\t' +
s.slice(pos + 1) + '\t' +
hostname;
};
FilterSingleWildcardRightAnchoredHostname.fromSelfie = function(s) {
var args = s.split('\t');
return new FilterSingleWildcardRightAnchoredHostname(args[0], args[1], args[2]);
@ -781,15 +888,18 @@ FilterManyWildcards.prototype.match = function(url, tokenBeg) {
return this.re.test(url.slice(tokenBeg - this.tokenBeg));
};
FilterManyWildcards.prototype.fid = '*+';
FilterManyWildcards.fid = FilterManyWildcards.prototype.fid = '*+';
FilterManyWildcards.prototype.toString = function() {
return this.s;
};
FilterManyWildcards.prototype.toSelfie = function() {
return this.s + '\t' +
this.tokenBeg;
return this.s + '\t' + this.tokenBeg;
};
FilterManyWildcards.compile = function(details) {
return details.f + '\t' + details.tokenBeg;
};
FilterManyWildcards.fromSelfie = function(s) {
@ -811,7 +921,7 @@ FilterManyWildcardsHostname.prototype.match = function(url, tokenBeg) {
this.re.test(url.slice(tokenBeg - this.tokenBeg));
};
FilterManyWildcardsHostname.prototype.fid = '*+h';
FilterManyWildcardsHostname.fid = FilterManyWildcardsHostname.prototype.fid = '*+h';
FilterManyWildcardsHostname.prototype.toString = function() {
return this.s + '$domain=' + this.hostname;
@ -823,6 +933,12 @@ FilterManyWildcardsHostname.prototype.toSelfie = function() {
this.hostname;
};
FilterManyWildcardsHostname.compile = function(details, hostname) {
return details.f + '\t' +
details.tokenBeg + '\t' +
hostname;
};
FilterManyWildcardsHostname.fromSelfie = function(s) {
var args = s.split('\t');
return new FilterManyWildcardsHostname(args[0], atoi(args[1]), args[2]);
@ -840,7 +956,7 @@ FilterRegex.prototype.match = function(url) {
return this.re.test(url);
};
FilterRegex.prototype.fid = '//';
FilterRegex.fid = FilterRegex.prototype.fid = '//';
FilterRegex.prototype.toString = function() {
return '/' + this.re.source + '/';
@ -850,6 +966,10 @@ FilterRegex.prototype.toSelfie = function() {
return this.re.source;
};
FilterRegex.compile = function(details) {
return details.f;
};
FilterRegex.fromSelfie = function(s) {
return new FilterRegex(s);
};
@ -867,15 +987,18 @@ FilterRegexHostname.prototype.match = function(url) {
this.re.test(url);
};
FilterRegexHostname.prototype.fid = '//h';
FilterRegexHostname.fid = FilterRegexHostname.prototype.fid = '//h';
FilterRegexHostname.prototype.toString = function() {
return '/' + this.re.source + '/$domain=' + this.hostname;
};
FilterRegexHostname.prototype.toSelfie = function() {
return this.re.source + '\t' +
this.hostname;
return this.re.source + '\t' + this.hostname;
};
FilterRegexHostname.compile = function(details, hostname) {
return details.f + '\t' + hostname;
};
FilterRegexHostname.fromSelfie = function(s) {
@ -981,12 +1104,12 @@ FilterHostnameDict.prototype.add = function(hn) {
if ( typeof bucket === 'string' ) {
bucket = this.dict[key] = this.meltBucket(hn.len, bucket);
}
if ( bucket[hn] === undefined ) {
bucket[hn] = true;
this.count += 1;
return true;
if ( bucket.hasOwnProperty(hn) ) {
return false;
}
return false;
bucket[hn] = true;
this.count += 1;
return true;
};
FilterHostnameDict.prototype.freeze = function() {
@ -1041,7 +1164,7 @@ FilterHostnameDict.prototype.matchesExactly = function(hn) {
return false;
};
FilterHostnameDict.prototype.match = function(hn) {
FilterHostnameDict.prototype.match = function() {
// TODO: mind IP addresses
var pos,
@ -1058,7 +1181,7 @@ FilterHostnameDict.prototype.match = function(hn) {
return this;
};
FilterHostnameDict.prototype.fid = '{h}';
FilterHostnameDict.fid = FilterHostnameDict.prototype.fid = '{h}';
FilterHostnameDict.prototype.toString = function() {
return this.h;
@ -1186,85 +1309,81 @@ FilterBucket.fromSelfie = function() {
/******************************************************************************/
var makeFilter = function(details) {
var s = details.f;
var getFilterClass = function(details) {
if ( details.isRegex ) {
return new FilterRegex(s);
return FilterRegex;
}
var s = details.f;
var wcOffset = s.indexOf('*');
if ( wcOffset !== -1 ) {
if ( s.indexOf('*', wcOffset + 1) !== -1 ) {
return details.anchor === 0 ? new FilterManyWildcards(s, details.tokenBeg) : null;
return details.anchor === 0 ? FilterManyWildcards : null;
}
var lSegment = s.slice(0, wcOffset);
var rSegment = s.slice(wcOffset + 1);
if ( details.anchor < 0 ) {
return new FilterSingleWildcardLeftAnchored(lSegment, rSegment);
return FilterSingleWildcardLeftAnchored;
}
if ( details.anchor > 0 ) {
return new FilterSingleWildcardRightAnchored(lSegment, rSegment);
return FilterSingleWildcardRightAnchored;
}
if ( details.tokenBeg === 0 ) {
return new FilterSingleWildcardPrefix0(lSegment, rSegment);
return FilterSingleWildcardPrefix0;
}
return new FilterSingleWildcard(lSegment, rSegment, details.tokenBeg);
return FilterSingleWildcard;
}
if ( details.anchor < 0 ) {
return new FilterPlainLeftAnchored(s);
return FilterPlainLeftAnchored;
}
if ( details.anchor > 0 ) {
return new FilterPlainRightAnchored(s);
return FilterPlainRightAnchored;
}
if ( details.hostnameAnchored ) {
return new FilterPlainHnAnchored(s);
return FilterPlainHnAnchored;
}
if ( details.tokenBeg === 0 ) {
return new FilterPlainPrefix0(s);
return FilterPlainPrefix0;
}
if ( details.tokenBeg === 1 ) {
return new FilterPlainPrefix1(s);
return FilterPlainPrefix1;
}
return new FilterPlain(s, details.tokenBeg);
return FilterPlain;
};
/******************************************************************************/
var makeHostnameFilter = function(details, hostname) {
var s = details.f;
var getHostnameBasedFilterClass = function(details) {
if ( details.isRegex ) {
return new FilterRegexHostname(s, hostname);
return FilterRegexHostname;
}
var s = details.f;
var wcOffset = s.indexOf('*');
if ( wcOffset !== -1 ) {
if ( s.indexOf('*', wcOffset + 1) !== -1 ) {
return details.anchor === 0 ? new FilterManyWildcardsHostname(s, details.tokenBeg, hostname) : null;
return details.anchor === 0 ? FilterManyWildcardsHostname : null;
}
var lSegment = s.slice(0, wcOffset);
var rSegment = s.slice(wcOffset + 1);
if ( details.anchor < 0 ) {
return new FilterSingleWildcardLeftAnchoredHostname(lSegment, rSegment, hostname);
return FilterSingleWildcardLeftAnchoredHostname;
}
if ( details.anchor > 0 ) {
return new FilterSingleWildcardRightAnchoredHostname(lSegment, rSegment, hostname);
return FilterSingleWildcardRightAnchoredHostname;
}
if ( details.tokenBeg === 0 ) {
return new FilterSingleWildcardPrefix0Hostname(lSegment, rSegment, hostname);
return FilterSingleWildcardPrefix0Hostname;
}
return new FilterSingleWildcardHostname(lSegment, rSegment, details.tokenBeg, hostname);
return FilterSingleWildcardHostname;
}
if ( details.anchor < 0 ) {
return new FilterPlainLeftAnchoredHostname(s, hostname);
return FilterPlainLeftAnchoredHostname;
}
if ( details.anchor > 0 ) {
return new FilterPlainRightAnchoredHostname(s, hostname);
return FilterPlainRightAnchoredHostname;
}
if ( details.tokenBeg === 0 ) {
return new FilterPlainPrefix0Hostname(s, hostname);
return FilterPlainPrefix0Hostname;
}
if ( details.tokenBeg === 1 ) {
return new FilterPlainPrefix1Hostname(s, hostname);
return FilterPlainPrefix1Hostname;
}
return new FilterPlainHostname(s, details.tokenBeg, hostname);
return FilterPlainHostname;
};
/******************************************************************************/
@ -1615,8 +1734,8 @@ FilterContainer.prototype.reset = function() {
this.allowFilterCount = 0;
this.blockFilterCount = 0;
this.duplicateCount = 0;
this.duplicateBuster = {};
this.categories = Object.create(null);
this.duplicates = Object.create(null);
this.filterParser.reset();
};
@ -1624,6 +1743,8 @@ FilterContainer.prototype.reset = function() {
FilterContainer.prototype.freeze = function() {
histogram('allFilters', this.categories);
this.duplicateBuster = {};
var categories = this.categories;
var bucket;
for ( var k in categories ) {
@ -1632,13 +1753,43 @@ FilterContainer.prototype.freeze = function() {
bucket.freeze();
}
}
this.duplicates = Object.create(null);
this.filterParser.reset();
this.frozen = true;
};
/******************************************************************************/
FilterContainer.prototype.factories = {
'[]': FilterBucket,
'a': FilterPlain,
'ah': FilterPlainHostname,
'0a': FilterPlainPrefix0,
'0ah': FilterPlainPrefix0Hostname,
'1a': FilterPlainPrefix1,
'1ah': FilterPlainPrefix1Hostname,
'|a': FilterPlainLeftAnchored,
'|ah': FilterPlainLeftAnchoredHostname,
'a|': FilterPlainRightAnchored,
'a|h': FilterPlainRightAnchoredHostname,
'h|a': FilterPlainHnAnchored,
'*': FilterSingleWildcard,
'*h': FilterSingleWildcardHostname,
'0*': FilterSingleWildcardPrefix0,
'0*h': FilterSingleWildcardPrefix0Hostname,
'|*': FilterSingleWildcardLeftAnchored,
'|*h': FilterSingleWildcardLeftAnchoredHostname,
'*|': FilterSingleWildcardRightAnchored,
'*|h': FilterSingleWildcardRightAnchoredHostname,
'*+': FilterManyWildcards,
'*+h': FilterManyWildcardsHostname,
'//': FilterRegex,
'//h': FilterRegexHostname,
'{h}': FilterHostnameDict
};
/******************************************************************************/
FilterContainer.prototype.toSelfie = function() {
var categoryToSelfie = function(dict) {
var selfie = [];
@ -1697,34 +1848,6 @@ FilterContainer.prototype.fromSelfie = function(selfie) {
this.blockFilterCount = selfie.blockFilterCount;
this.duplicateCount = selfie.duplicateCount;
var factories = {
'[]': FilterBucket,
'a': FilterPlain,
'ah': FilterPlainHostname,
'0a': FilterPlainPrefix0,
'0ah': FilterPlainPrefix0Hostname,
'1a': FilterPlainPrefix1,
'1ah': FilterPlainPrefix1Hostname,
'|a': FilterPlainLeftAnchored,
'|ah': FilterPlainLeftAnchoredHostname,
'a|': FilterPlainRightAnchored,
'a|h': FilterPlainRightAnchoredHostname,
'h|a': FilterPlainHnAnchored,
'*': FilterSingleWildcard,
'*h': FilterSingleWildcardHostname,
'0*': FilterSingleWildcardPrefix0,
'0*h': FilterSingleWildcardPrefix0Hostname,
'|*': FilterSingleWildcardLeftAnchored,
'|*h': FilterSingleWildcardLeftAnchoredHostname,
'*|': FilterSingleWildcardRightAnchored,
'*|h': FilterSingleWildcardRightAnchoredHostname,
'*+': FilterManyWildcards,
'*+h': FilterManyWildcardsHostname,
'//': FilterRegex,
'//h': FilterRegexHostname,
'{h}': FilterHostnameDict
};
var catKey, tokenKey;
var dict = this.categories, subdict;
var bucket = null;
@ -1752,7 +1875,7 @@ FilterContainer.prototype.fromSelfie = function(selfie) {
bucket = null;
continue;
}
factory = factories[what];
factory = this.factories[what];
if ( bucket === null ) {
bucket = subdict[tokenKey] = factory.fromSelfie(line.slice(pos + 1));
continue;
@ -1766,12 +1889,12 @@ FilterContainer.prototype.fromSelfie = function(selfie) {
/******************************************************************************/
FilterContainer.prototype.makeCategoryKey = function(category) {
return String.fromCharCode(category);
return category.toString(16);
};
/******************************************************************************/
FilterContainer.prototype.add = function(raw) {
FilterContainer.prototype.compile = function(raw, out) {
// ORDER OF TESTS IS IMPORTANT!
// Ignore empty lines
@ -1795,40 +1918,22 @@ FilterContainer.prototype.add = function(raw) {
// Ignore filters with unsupported options
if ( parsed.unsupported ) {
this.rejectedCount += 1;
//console.log('static-net-filtering.js > FilterContainer.add(): unsupported filter "%s"', raw);
return false;
}
this.processedFilterCount += 1;
this.acceptedCount += 1;
// Pure hostnames, use more efficient liquid dict
// https://github.com/gorhill/uBlock/issues/665
// Create a dict keyed on request type etc.
if ( parsed.hostnamePure && this.addHostnameOnlyFilter(parsed) ) {
if ( parsed.hostnamePure && this.compileHostnameOnlyFilter(parsed, out) ) {
return true;
}
if ( this.duplicates[s] ) {
//console.log('static-net-filtering.js > FilterContainer.add(): duplicate filter "%s"', raw);
this.duplicateCount++;
return false;
}
if ( this.frozen === false ) {
this.duplicates[s] = true;
}
var r = this.addFilter(parsed);
var r = this.compileFilter(parsed, out);
if ( r === false ) {
return false;
}
if ( parsed.action ) {
this.allowFilterCount += 1;
} else {
this.blockFilterCount += 1;
}
return true;
};
@ -1836,55 +1941,37 @@ FilterContainer.prototype.add = function(raw) {
// Using fast/compact dictionary when filter is a (or portion of) pure hostname.
FilterContainer.prototype.addHostnameOnlyFilter = function(parsed) {
FilterContainer.prototype.compileHostnameOnlyFilter = function(parsed, out) {
// Can't fit the filter in a pure hostname dictionary.
if ( parsed.hostnames.length !== 0 || parsed.notHostnames.length !== 0 ) {
return false;
return;
}
var isNewFilter = false;
var party = AnyParty;
if ( parsed.firstParty !== parsed.thirdParty ) {
party = parsed.firstParty ? FirstParty : ThirdParty;
}
var keyShard = parsed.action | parsed.important | party;
var key, bucket;
var type = parsed.types >>> 1 || 1; // bit 0 is unused; also, default to AnyType
var bitOffset = 1;
while ( type !== 0 ) {
if ( type & 1 ) {
key = this.makeCategoryKey(keyShard | (bitOffset << 4));
bucket = this.categories[key];
if ( bucket === undefined ) {
bucket = this.categories[key] = Object.create(null);
}
if ( bucket['.'] === undefined ) {
bucket['.'] = new FilterHostnameDict();
}
if ( bucket['.'].add(parsed.f) ) {
isNewFilter = true;
}
out.push(
'n\v' +
this.makeCategoryKey(keyShard | (bitOffset << 4)) + '\v' +
'.\v' +
parsed.f
);
}
bitOffset += 1;
type >>>= 1;
}
// https://github.com/gorhill/uBlock/issues/719
// Count whole filter, not its decomposed versions
if ( isNewFilter ) {
if ( parsed.action ) {
this.allowFilterCount += 1;
} else {
this.blockFilterCount += 1;
}
} else {
this.duplicateCount += 1;
}
return true;
};
/******************************************************************************/
FilterContainer.prototype.addFilter = function(parsed) {
FilterContainer.prototype.compileFilter = function(parsed, out) {
parsed.makeToken();
if ( parsed.token === '' ) {
console.error('static-net-filtering.js > FilterContainer.addFilter("%s"): can\'t tokenize', parsed.f);
@ -1896,28 +1983,28 @@ FilterContainer.prototype.addFilter = function(parsed) {
party = parsed.firstParty ? FirstParty : ThirdParty;
}
var filter;
var filterClass;
var i = parsed.hostnames.length;
var j = parsed.notHostnames.length;
// Applies to all domains without exceptions
if ( i === 0 && j === 0 ) {
filter = makeFilter(parsed);
if ( !filter ) {
filterClass = getFilterClass(parsed);
if ( filterClass === null ) {
return false;
}
this.addFilterEntry(filter, parsed, party);
this.compileToAtomicFilter(filterClass, parsed, party, out);
return true;
}
// Applies to specific domains
if ( i !== 0 ) {
while ( i-- ) {
filter = makeHostnameFilter(parsed, parsed.hostnames[i]);
if ( !filter ) {
filterClass = getHostnameBasedFilterClass(parsed);
if ( filterClass === null ) {
return false;
}
this.addFilterEntry(filter, parsed, party);
this.compileToAtomicFilter(filterClass, parsed, party, out, parsed.hostnames[i]);
}
}
// No exceptions
@ -1930,13 +2017,13 @@ FilterContainer.prototype.addFilter = function(parsed) {
// Example:
// - ||adm.fwmrm.net/p/msnbc_live/$object-subrequest,third-party,domain=~msnbc.msn.com|~www.nbcnews.com
if ( i === 0 ) {
filter = makeFilter(parsed);
if ( !filter ) {
filterClass = getFilterClass(parsed);
if ( filterClass === null ) {
return false;
}
// https://github.com/gorhill/uBlock/issues/251
// Apply third-party option if it is present
this.addFilterEntry(filter, parsed, party);
this.compileToAtomicFilter(filterClass, parsed, party, out);
}
// Cases:
@ -1948,8 +2035,8 @@ FilterContainer.prototype.addFilter = function(parsed) {
// Reverse purpose of filter
parsed.action ^= ToggleAction;
while ( j-- ) {
filter = makeHostnameFilter(parsed, parsed.notHostnames[j]);
if ( !filter ) {
filterClass = getHostnameBasedFilterClass(parsed);
if ( filterClass === null ) {
return false;
}
// https://github.com/gorhill/uBlock/issues/191#issuecomment-53654024
@ -1958,20 +2045,26 @@ FilterContainer.prototype.addFilter = function(parsed) {
if ( parsed.action === BlockAction ) {
parsed.important = Important;
}
this.addFilterEntry(filter, parsed, party);
this.compileToAtomicFilter(filterClass, parsed, party, out, parsed.notHostnames[j]);
}
return true;
};
/******************************************************************************/
FilterContainer.prototype.addFilterEntry = function(filter, parsed, party) {
FilterContainer.prototype.compileToAtomicFilter = function(filterClass, parsed, party, out, hostname) {
var bits = parsed.action | parsed.important | party;
var type = parsed.types >>> 1 || 1; // bit 0 is unused; also, default to AnyType
var bitOffset = 1;
while ( type !== 0 ) {
if ( type & 1 ) {
this.addToCategory(bits | (bitOffset << 4), parsed.token, filter);
out.push(
'n\v' +
this.makeCategoryKey(bits | (bitOffset << 4)) + '\v' +
parsed.token + '\v' +
filterClass.fid + '\v' +
filterClass.compile(parsed, hostname)
);
}
bitOffset += 1;
type >>>= 1;
@ -1980,22 +2073,60 @@ FilterContainer.prototype.addFilterEntry = function(filter, parsed, party) {
/******************************************************************************/
FilterContainer.prototype.addToCategory = function(category, tokenKey, filter) {
var categoryKey = this.makeCategoryKey(category);
var categoryBucket = this.categories[categoryKey];
if ( !categoryBucket ) {
categoryBucket = this.categories[categoryKey] = Object.create(null);
FilterContainer.prototype.fromCompiledContent = function(text, lineBeg) {
var lineEnd;
var textEnd = text.length;
var line, fields, bucket, entry, factory, filter;
while ( lineBeg < textEnd ) {
if ( text.charAt(lineBeg) !== 'n' ) {
return lineBeg;
}
lineEnd = text.indexOf('\n', lineBeg);
if ( lineEnd === -1 ) {
lineEnd = textEnd;
}
line = text.slice(lineBeg + 2, lineEnd);
fields = line.split('\v');
lineBeg = lineEnd + 1;
this.acceptedCount += 1;
bucket = this.categories[fields[0]];
if ( bucket === undefined ) {
bucket = this.categories[fields[0]] = Object.create(null);
}
entry = bucket[fields[1]];
if ( fields[1] === '.' ) {
if ( entry === undefined ) {
entry = bucket['.'] = new FilterHostnameDict();
}
if ( entry.add(fields[2]) === false ) {
this.duplicateCount += 1;
}
continue;
}
if ( this.duplicateBuster.hasOwnProperty(line) ) {
this.duplicateCount += 1;
continue;
}
this.duplicateBuster[line] = true;
factory = this.factories[fields[2]];
filter = factory.fromSelfie(fields[3]);
if ( entry === undefined ) {
bucket[fields[1]] = filter;
continue;
}
if ( entry.fid === '[]' ) {
entry.add(filter);
continue;
}
bucket[fields[1]] = new FilterBucket(entry, filter);
}
var filterEntry = categoryBucket[tokenKey];
if ( filterEntry === undefined ) {
categoryBucket[tokenKey] = filter;
return;
}
if ( filterEntry.fid === '[]' ) {
filterEntry.add(filter);
return;
}
categoryBucket[tokenKey] = new FilterBucket(filterEntry, filter);
return textEnd;
};
/******************************************************************************/
@ -2275,7 +2406,7 @@ FilterContainer.prototype.matchString = function(context) {
/******************************************************************************/
FilterContainer.prototype.getFilterCount = function() {
return this.blockFilterCount + this.allowFilterCount;
return this.acceptedCount - this.duplicateCount;
};
/******************************************************************************/

383
src/js/storage.js
View File

@ -19,7 +19,7 @@
Home: https://github.com/gorhill/uBlock
*/
/* global µBlock, vAPI, punycode, publicSuffixList */
/* global YaMD5, µBlock, vAPI, punycode, publicSuffixList */
'use strict';
@ -53,6 +53,21 @@
/******************************************************************************/
µBlock.saveSystemSettings = function() {
vAPI.storage.set(this.systemSettings, this.noopFunc);
};
/******************************************************************************/
µBlock.loadSystemSettings = function(callback) {
vAPI.storage.get({
compiledMagic: '',
selfieMagic: ''
}, callback);
};
/******************************************************************************/
// Save local settings regularly. Not critical.
µBlock.asyncJobs.add(
@ -133,17 +148,30 @@
/******************************************************************************/
µBlock.appendUserFilters = function(content) {
if ( content.length === 0 ) {
return;
}
var µb = this;
var onCompiledListLoaded = function(details) {
var snfe = µb.staticNetFilteringEngine;
var cfe = µb.cosmeticFilteringEngine;
var acceptedCount = snfe.acceptedCount + cfe.acceptedCount;
var duplicateCount = snfe.duplicateCount + cfe.duplicateCount;
µb.applyCompiledFilters(details.content);
var entry = µb.remoteBlacklists[µb.userFiltersPath];
entry.entryCount = snfe.acceptedCount + cfe.acceptedCount - acceptedCount;
entry.entryUsedCount = entry.entryCount - snfe.duplicateCount - cfe.duplicateCount + duplicateCount;
µb.staticNetFilteringEngine.freeze();
µb.cosmeticFilteringEngine.freeze();
};
var onSaved = function(details) {
if ( details.error ) {
return;
}
µb.mergeFilterText(content);
µb.staticNetFilteringEngine.freeze();
µb.cosmeticFilteringEngine.freeze();
µb.destroySelfie();
µb.toSelfieAsync();
µb.getCompiledFilterList(µb.userFiltersPath, onCompiledListLoaded);
};
var onLoaded = function(details) {
@ -153,12 +181,10 @@
if ( details.content.indexOf(content.trim()) !== -1 ) {
return;
}
µb.saveUserFilters(details.content + '\n' + content, onSaved);
µb.saveUserFilters(details.content.trim() + '\n' + content.trim(), onSaved);
};
if ( content.length > 0 ) {
this.loadUserFilters(onLoaded);
}
this.loadUserFilters(onLoaded);
};
/******************************************************************************/
@ -260,111 +286,152 @@
/******************************************************************************/
µBlock.createShortUniqueId = function(path) {
var md5 = YaMD5.hashStr(path);
return md5.slice(0, 4) + md5.slice(-4);
};
µBlock.createShortUniqueId.idLength = 8;
/******************************************************************************/
µBlock.loadFilterLists = function(callback) {
//quickProfiler.start('µBlock.loadFilterLists()');
var µb = this;
var filterlistCount;
var filterlistsCount = 0;
if ( typeof callback !== 'function' ) {
callback = this.noopFunc;
}
var loadBlacklistsEnd = function() {
var onDone = function() {
µb.staticNetFilteringEngine.freeze();
µb.cosmeticFilteringEngine.freeze();
vAPI.storage.set({ 'remoteBlacklists': µb.remoteBlacklists });
vAPI.messaging.broadcast({ what: 'loadUbiquitousBlacklistCompleted' });
µb.toSelfieAsync();
vAPI.messaging.broadcast({ what: 'allFilterListsReloaded' });
callback();
µb.toSelfieAsync();
//quickProfiler.stop(0);
};
var mergeBlacklist = function(details) {
µb.mergeFilterList(details);
filterlistCount -= 1;
if ( filterlistCount === 0 ) {
loadBlacklistsEnd();
var applyCompiledFilters = function(path, compiled) {
var snfe = µb.staticNetFilteringEngine;
var cfe = µb.cosmeticFilteringEngine;
var acceptedCount = snfe.acceptedCount + cfe.acceptedCount;
var duplicateCount = snfe.duplicateCount + cfe.duplicateCount;
µb.applyCompiledFilters(compiled);
if ( µb.remoteBlacklists.hasOwnProperty(path) ) {
var entry = µb.remoteBlacklists[path];
entry.entryCount = snfe.acceptedCount + cfe.acceptedCount - acceptedCount;
entry.entryUsedCount = entry.entryCount - snfe.duplicateCount - cfe.duplicateCount + duplicateCount;
}
};
var loadBlacklistsStart = function(lists) {
µb.remoteBlacklists = lists;
µb.staticNetFilteringEngine.reset();
µb.cosmeticFilteringEngine.reset();
µb.destroySelfie();
var locations = Object.keys(lists);
filterlistCount = locations.length;
var onCompiledListLoaded = function(details) {
applyCompiledFilters(details.path, details.content);
filterlistsCount -= 1;
if ( filterlistsCount === 0 ) {
onDone();
}
};
// Load all filter lists which are not disabled
var location;
while ( location = locations.pop() ) {
// rhill 2013-12-09:
// Ignore list if disabled
// https://github.com/gorhill/httpswitchboard/issues/78
if ( lists[location].off ) {
filterlistCount -= 1;
var onFilterListsReady = function(lists) {
µb.remoteBlacklists = lists;
µb.cosmeticFilteringEngine.reset();
µb.staticNetFilteringEngine.reset();
µb.destroySelfie();
// We need to build a complete list of assets to pull first: this is
// because it *may* happens that some load operations are synchronous:
// This happens for assets which do not exist, ot assets with no
// content.
var toLoad = [];
for ( var path in lists ) {
if ( lists.hasOwnProperty(path) === false ) {
continue;
}
µb.assets.get(location, mergeBlacklist);
if ( lists[path].off ) {
continue;
}
toLoad.push(path);
}
// https://github.com/gorhill/uBlock/issues/695
// It may happen not a single filter list is selected
if ( filterlistCount === 0 ) {
loadBlacklistsEnd();
filterlistsCount = toLoad.length;
if ( filterlistsCount === 0 ) {
onDone();
return;
}
var i = toLoad.length;
while ( i-- ) {
µb.getCompiledFilterList(toLoad[i], onCompiledListLoaded);
}
};
this.getAvailableLists(loadBlacklistsStart);
this.getAvailableLists(onFilterListsReady);
};
/******************************************************************************/
µBlock.mergeFilterList = function(details) {
// console.log('µBlock > mergeFilterList from "%s": "%s..."', details.path, details.content.slice(0, 40));
µBlock.getCompiledFilterListPath = function(path) {
return 'cache://compiled-filter-list:' + this.createShortUniqueId(path);
};
var staticNetFilteringEngine = this.staticNetFilteringEngine;
var cosmeticFilteringEngine = this.cosmeticFilteringEngine;
var duplicateCount = staticNetFilteringEngine.duplicateCount + cosmeticFilteringEngine.duplicateCount;
var acceptedCount = staticNetFilteringEngine.acceptedCount + cosmeticFilteringEngine.acceptedCount;
/******************************************************************************/
this.mergeFilterText(details.content);
µBlock.getCompiledFilterList = function(path, callback) {
var compiledPath = this.getCompiledFilterListPath(path);
var µb = this;
// For convenience, store the number of entries for this
// blacklist, user might be happy to know this information.
duplicateCount = staticNetFilteringEngine.duplicateCount + cosmeticFilteringEngine.duplicateCount - duplicateCount;
acceptedCount = staticNetFilteringEngine.acceptedCount + cosmeticFilteringEngine.acceptedCount - acceptedCount;
var filterListMeta = this.remoteBlacklists[details.path];
filterListMeta.entryCount = acceptedCount;
filterListMeta.entryUsedCount = acceptedCount - duplicateCount;
// Try to extract a human-friendly name (works only for
// ABP-compatible filter lists)
if ( filterListMeta.title === '' ) {
var matches = details.content.slice(0, 1024).match(/(?:^|\n)!\s*Title:([^\n]+)/i);
if ( matches !== null ) {
filterListMeta.title = matches[1].trim();
var onRawListLoaded = function(details) {
if ( details.content !== '' ) {
//console.debug('µBlock.getCompiledFilterList/onRawListLoaded: compiling "%s"', path);
details.content = µb.compileFilters(details.content);
µb.assets.put(compiledPath, details.content);
}
}
callback(details);
};
var onCompiledListLoaded = function(details) {
if ( details.content === '' ) {
//console.debug('µBlock.getCompiledFilterList/onCompiledListLoaded: no compiled version for "%s"', path);
µb.assets.get(path, onRawListLoaded);
return;
}
//console.debug('µBlock.getCompiledFilterList/onCompiledListLoaded: using compiled version for "%s"', path);
details.path = path;
callback(details);
};
this.assets.get(compiledPath, onCompiledListLoaded);
};
/******************************************************************************/
µBlock.mergeFilterText = function(rawText) {
µBlock.purgeCompiledFilterList = function(path) {
this.assets.purge(this.getCompiledFilterListPath(path));
};
/******************************************************************************/
µBlock.compileFilters = function(rawText) {
var rawEnd = rawText.length;
var compiledFilters = [];
// Useful references:
// https://adblockplus.org/en/filter-cheatsheet
// https://adblockplus.org/en/filters
var staticNetFilteringEngine = this.staticNetFilteringEngine;
var cosmeticFilteringEngine = this.cosmeticFilteringEngine;
var parseCosmeticFilters = this.userSettings.parseAllABPHideFilters;
var reIsCosmeticFilter = /#[@#]/;
var reIsWhitespaceChar = /\s/;
var reMaybeLocalIp = /^[\d:f]/;
var reIsLocalhostRedirect = /\s+(?:broadcasthost|local|localhost|localhost\.localdomain)(?=\s|$)/;
var reLocalIp = /^(?:0\.0\.0\.0|127\.0\.0\.1|::1|fe80::1%lo0)/;
//var reAsciiSegment = /^[\x21-\x7e]+$/;
var lineBeg = 0, lineEnd, currentLineBeg;
var line, lineRaw, c, pos;
@ -396,11 +463,7 @@
// Parse or skip cosmetic filters
// All cosmetic filters are caught here
if ( parseCosmeticFilters ) {
if ( cosmeticFilteringEngine.add(line) ) {
continue;
}
} else if ( reIsCosmeticFilter.test(line) ) {
if ( cosmeticFilteringEngine.compile(line, compiledFilters) ) {
continue;
}
@ -440,41 +503,46 @@
continue;
}
// The filter is whatever sequence of printable ascii character without
// whitespaces
//matches = reAsciiSegment.exec(line);
//if ( matches === null ) {
// console.debug('storage.js > µBlock.mergeFilterList(): skipping "%s"', lineRaw);
// continue;
//}
//staticNetFilteringEngine.add(line);
staticNetFilteringEngine.compile(line, compiledFilters);
}
// Bypass anomalies
// For example, when a filter contains whitespace characters, or
// whatever else outside the range of printable ascii characters.
//if ( matches[0] !== line ) {
// console.error('"%s" !== "%s"', matches[0], line);
// continue;
//}
return compiledFilters.join('\n');
};
staticNetFilteringEngine.add(line);
/******************************************************************************/
µBlock.applyCompiledFilters = function(rawText) {
var skipCosmetic = !this.userSettings.parseAllABPHideFilters;
var staticNetFilteringEngine = this.staticNetFilteringEngine;
var cosmeticFilteringEngine = this.cosmeticFilteringEngine;
var lineBeg = 0;
var rawEnd = rawText.length;
while ( lineBeg < rawEnd ) {
lineBeg = cosmeticFilteringEngine.fromCompiledContent(rawText, lineBeg, skipCosmetic);
lineBeg = staticNetFilteringEngine.fromCompiledContent(rawText, lineBeg);
}
};
/******************************************************************************/
// `switches` contains the preset blacklists for which the switch must be
// revisited.
// `switches` contains the filter lists for which the switch must be revisited.
µBlock.reloadPresetBlacklists = function(switches, update) {
var µb = µBlock;
µBlock.reloadFilterLists = function(switches, update) {
var µb = this;
var onFilterListsReady = function() {
µb.loadUpdatableAssets({ update: update, psl: update });
};
// Toggle switches, if any
if ( switches !== undefined ) {
var filterLists = this.remoteBlacklists;
var onPurgeDone = function() {
// Toggle switches, if any
if ( switches === undefined ) {
onFilterListsReady();
return;
}
var filterLists = µb.remoteBlacklists;
var i = switches.length;
while ( i-- ) {
if ( filterLists.hasOwnProperty(switches[i].location) === false ) {
@ -484,26 +552,76 @@
}
// Save switch states
vAPI.storage.set({ 'remoteBlacklists': filterLists }, onFilterListsReady);
} else {
onFilterListsReady();
};
// If we must update, we need to purge the compiled versions of
// obsolete assets.
if ( update !== true ) {
onPurgeDone();
return;
}
var onMetadataReady = function(metadata) {
var filterLists = µb.remoteBlacklists;
var entry;
// Purge obsolete filter lists
for ( var path in filterLists ) {
if ( filterLists.hasOwnProperty(path) === false ) {
continue;
}
if ( metadata.hasOwnProperty(path) === false ) {
continue;
}
entry = metadata[path];
if ( entry.repoObsolete !== true && entry.cacheObsolete !== true ) {
continue;
}
µb.purgeCompiledFilterList(path);
}
// Purge obsolete PSL
if ( metadata.hasOwnProperty(µb.pslPath) === false ) {
entry = metadata[µb.pslPath];
if ( entry.repoObsolete === true ) {
µb.assets.purge('cache://compiled-publicsuffixlist');
}
}
onPurgeDone();
};
this.assets.metadata(onMetadataReady);
};
/******************************************************************************/
µBlock.loadPublicSuffixList = function(callback) {
var µb = this;
var path = µb.pslPath;
var compiledPath = 'cache://compiled-publicsuffixlist';
if ( typeof callback !== 'function' ) {
callback = this.noopFunc;
}
var applyPublicSuffixList = function(details) {
// TODO: Not getting proper suffix list is a bit serious, I think
// the extension should be force-restarted if it occurs..
if ( !details.error ) {
var onRawListLoaded = function(details) {
if ( details.content !== '' ) {
//console.debug('µBlock.loadPublicSuffixList/onRawListLoaded: compiling "%s"', path);
publicSuffixList.parse(details.content, punycode.toASCII);
µb.assets.put(compiledPath, JSON.stringify(publicSuffixList.toSelfie()));
}
callback();
};
this.assets.get(this.pslPath, applyPublicSuffixList);
var onCompiledListLoaded = function(details) {
if ( details.content === '' ) {
//console.debug('µBlock.loadPublicSuffixList/onCompiledListLoaded: no compiled version for "%s"', path);
µb.assets.get(path, onRawListLoaded);
return;
}
//console.debug('µBlock.loadPublicSuffixList/onCompiledListLoaded: using compiled version for "%s"', path);
publicSuffixList.fromSelfie(JSON.parse(details.content));
callback();
};
this.assets.get(compiledPath, onCompiledListLoaded);
};
/******************************************************************************/
@ -540,7 +658,7 @@
µBlock.toSelfie = function() {
var selfie = {
magic: this.selfieMagic,
magic: this.systemSettings.selfieMagic,
publicSuffixList: publicSuffixList.toSelfie(),
filterLists: this.remoteBlacklists,
staticNetFilteringEngine: this.staticNetFilteringEngine.toSelfie(),
@ -578,7 +696,7 @@
var onSelfieReady = function(store) {
var selfie = store.selfie;
if ( typeof selfie !== 'object' || selfie.magic !== µb.selfieMagic ) {
if ( typeof selfie !== 'object' || selfie.magic !== µb.systemSettings.selfieMagic ) {
callback(false);
return;
}
@ -586,7 +704,7 @@
callback(false);
return;
}
// console.log('µBlock.fromSelfie> selfie looks good');
//console.log('µBlock.fromSelfie> selfie looks good');
µb.remoteBlacklists = selfie.filterLists;
µb.staticNetFilteringEngine.fromSelfie(selfie.staticNetFilteringEngine);
µb.cosmeticFilteringEngine.fromSelfie(selfie.cosmeticFilteringEngine);
@ -601,7 +719,7 @@
µBlock.destroySelfie = function() {
vAPI.storage.remove('selfie');
this.asyncJobs.remove('toSelfie');
//console.debug('storage.js > µBlock.destroySelfie()');
//console.debug('µBlock.destroySelfie()');
};
/******************************************************************************/
@ -626,6 +744,25 @@
/******************************************************************************/
µBlock.assetUpdatedHandler = function(details) {
var path = details.path || '';
if ( this.remoteBlacklists.hasOwnProperty(path) === false ) {
return;
}
var entry = this.remoteBlacklists[path];
if ( entry.off ) {
return;
}
// Compile the list while we have the raw version in memory
//console.debug('µBlock.getCompiledFilterList/onRawListLoaded: compiling "%s"', path);
this.assets.put(
this.getCompiledFilterListPath(path),
this.compileFilters(details.content)
);
};
/******************************************************************************/
µBlock.updateCompleteHandler = function(details) {
var µb = this;
var updatedCount = details.updatedCount;
@ -659,3 +796,35 @@
onPSLReady();
}
};
/******************************************************************************/
µBlock.assetCacheRemovedHandler = (function() {
var barrier = false;
var handler = function(paths) {
if ( barrier ) {
return;
}
barrier = true;
var i = paths.length;
var path;
while ( i-- ) {
path = paths[i];
if ( this.remoteBlacklists.hasOwnProperty(path) ) {
//console.debug('µBlock.assetCacheRemovedHandler: decompiling "%s"', path);
this.purgeCompiledFilterList(path);
continue;
}
if ( path === this.pslPath ) {
//console.debug('µBlock.assetCacheRemovedHandler: decompiling "%s"', path);
this.assets.purge('cache://compiled-publicsuffixlist');
continue;
}
}
this.destroySelfie();
barrier = false;
};
return handler;
})();