This commit is contained in:
gorhill 2015-02-13 12:10:10 -05:00
parent df8f98d82d
commit bca949748d
5 changed files with 368 additions and 199 deletions

View File

@ -1,7 +1,7 @@
/*******************************************************************************
µBlock - a Chromium browser extension to block requests.
Copyright (C) 2014 Raymond Hill
Copyright (C) 2014-2015 Raymond Hill
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@ -274,7 +274,7 @@ var getTextFileFromURL = function(url, onLoad, onError) {
// console.log('µBlock> getTextFileFromURL("%s"):', url);
var xhr = new XMLHttpRequest();
xhr.open('get', url, true);
xhr.timeout = 15000;
xhr.timeout = 30000;
xhr.onload = onResponseReceived;
xhr.onerror = onError;
xhr.ontimeout = onError;
@ -1044,7 +1044,194 @@ exports.purgeAll = function(callback) {
return exports;
})();
/******************************************************************************/
/******************************************************************************/
µBlock.assetUpdater = (function() {
'use strict';
/******************************************************************************/
var µb = µBlock;
var updateDaemonTimerPeriod = 11 * 60 * 1000; // 11 minutes
var updateCycleFirstPeriod = 7 * 60 * 1000; // 7 minutes
var updateCycleNextPeriod = 11 * 60 * 60 * 1000; // 11 hours
var updateCycleTime = 0;
var toUpdate = {};
var toUpdateCount = 0;
var updated = {};
var updatedCount = 0;
var metadata = null;
var onStart = null;
var onCompleted = null;
var exports = {};
/******************************************************************************/
var onAssetUpdated = function(details) {
var path = details.path;
if ( details.error ) {
console.debug('assets.js > µBlock.assetUpdater/onAssetUpdated: "%s" failed', path);
return;
}
console.debug('assets.js > µBlock.assetUpdater/onAssetUpdated: "%s"', path);
updated[path] = true;
updatedCount += 1;
// New data available: selfie is now invalid
µb.destroySelfie();
};
/******************************************************************************/
var updateOne = function() {
var metaEntry;
for ( var path in toUpdate ) {
if ( toUpdate.hasOwnProperty(path) === false ) {
continue;
}
if ( toUpdate[path] !== true ) {
continue;
}
toUpdate[path] = false;
toUpdateCount -= 1;
if ( metadata.hasOwnProperty(path) === false ) {
continue;
}
metaEntry = metadata[path];
if ( !metaEntry.cacheObsolete && !metaEntry.repoObsolete ) {
continue;
}
console.debug('assets.js > µBlock.assetUpdater/updateOne: assets.get("%s")', path);
µb.assets.get(path, onAssetUpdated);
break;
}
};
/******************************************************************************/
var onMetadataReady = function(response) {
metadata = response;
updateOne();
};
/******************************************************************************/
var updateDaemon = function() {
setTimeout(updateDaemon, updateDaemonTimerPeriod);
µb.assets.autoUpdate = µb.userSettings.autoUpdate;
if ( µb.assets.autoUpdate !== true ) {
return;
}
// Start an update cycle?
if ( updateCycleTime !== 0 ) {
if ( Date.now() >= updateCycleTime ) {
console.debug('assets.js > µBlock.assetUpdater/updateDaemon: update cycle started');
reset();
if ( onStart !== null ) {
onStart();
}
}
return;
}
// Any asset to update?
if ( toUpdateCount !== 0 ) {
if ( metadata === null ) {
µb.assets.metadata(onMetadataReady);
} else {
updateOne();
}
return;
}
// Nothing left to update
// If anything was updated, notify listener
if ( updatedCount !== 0 ) {
if ( onCompleted !== null ) {
console.debug('assets.js > µBlock.assetUpdater/updateDaemon: update cycle completed');
onCompleted({
updated: JSON.parse(JSON.stringify(updated)), // give callee its own safe copy
updatedCount: updatedCount
});
}
}
// Schedule next update cycle
if ( updateCycleTime === 0 ) {
reset();
console.debug('assets.js > µBlock.assetUpdater/updateDaemon: update cycle re-scheduled');
updateCycleTime = Date.now() + updateCycleNextPeriod;
}
};
setTimeout(updateDaemon, updateDaemonTimerPeriod);
/******************************************************************************/
var reset = function() {
toUpdate = {};
toUpdateCount = 0;
updated = {};
updatedCount = 0;
updateCycleTime = 0;
metadata = null;
};
/******************************************************************************/
exports.onStart = {
addEventListener: function(callback) {
onStart = callback || null;
if ( onStart !== null ) {
updateCycleTime = Date.now() + updateCycleFirstPeriod;
}
}
};
/******************************************************************************/
exports.onCompleted = {
addEventListener: function(callback) {
onCompleted = callback || null;
}
};
/******************************************************************************/
exports.add = function(path) {
if ( toUpdate.hasOwnProperty(path) ) {
return;
}
console.debug('assets.js > µBlock.assetUpdater.add("%s")', path);
toUpdate[path] = true;
toUpdateCount += 1;
};
/******************************************************************************/
// Typically called when an update has been forced.
exports.restart = function() {
reset();
updateCycleTime = Date.now() + updateCycleNextPeriod;
};
/******************************************************************************/
return exports;
})();

View File

@ -67,7 +67,6 @@ return {
// https://github.com/gorhill/uBlock/issues/180
// Whitelist directives need to be loaded once the PSL is available
netExceptionList: {}, // TODO: remove once all users are up to date
netWhitelist: {},
netWhitelistModifyTime: 0,
netWhitelistDefault: [
@ -85,7 +84,7 @@ return {
// EasyList, EasyPrivacy and many others have an 4-day update period,
// as per list headers.
updateAssetsEvery: 75 * oneHour + 23 * oneMinute + 53 * oneSecond + 605,
updateAssetsEvery: 97 * oneHour,
projectServerRoot: 'https://raw.githubusercontent.com/gorhill/uBlock/master/',
userFiltersPath: 'assets/user/filters.txt',
pslPath: 'assets/thirdparties/publicsuffix.org/list/effective_tld_names.dat',
@ -111,11 +110,8 @@ return {
remoteBlacklists: {
},
firstUpdateAfter: 5 * oneMinute,
nextUpdateAfter: 7 * oneHour,
selfieMagic: 'bizhviclttie',
selfieAfter: 7 * oneMinute,
selfieAfter: 23 * oneMinute,
pageStores: {},

View File

@ -498,6 +498,9 @@ var parseMirrorCandidates = function(rawText) {
/******************************************************************************/
var load = function() {
if ( loaded ) {
return;
}
loaded = true;
var onMirrorCandidatesReady = function(details) {

View File

@ -1,7 +1,7 @@
/*******************************************************************************
µBlock - a Chromium browser extension to block requests.
Copyright (C) 2014 Raymond Hill
Copyright (C) 2014-2015 Raymond Hill
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@ -19,97 +19,148 @@
Home: https://github.com/gorhill/uBlock
*/
/* global µBlock */
/* global vAPI, µBlock */
/******************************************************************************/
// Automatic update of non-user assets
// https://github.com/gorhill/httpswitchboard/issues/334
// Load all: executed once.
µBlock.updater = (function() {
'use strict';
(function() {
/******************************************************************************/
var µb = µBlock;
// Final initialization steps after all needed assets are in memory.
// - Initialize internal state with maybe already existing tabs.
// - Schedule next update operation.
var jobCallback = function() {
// Simpler to fire restart here, and safe given how far this will happen
// in the future.
restart();
var onAllReady = function() {
var µb = µBlock;
// If auto-update is disabled, check again in a while.
if ( µb.userSettings.autoUpdate !== true ) {
// https://github.com/gorhill/uBlock/issues/184
// Check for updates not too far in the future.
µb.assetUpdater.onStart.addEventListener(µb.updateStartHandler.bind(µb));
µb.assetUpdater.onCompleted.addEventListener(µb.updateCompleteHandler.bind(µb));
// Important: remove barrier to remote fetching, this was useful only
// for launch time.
µb.assets.allowRemoteFetch = true;
vAPI.onLoadAllCompleted();
};
/******************************************************************************/
// To bring older versions up to date
var onVersionReady = function(bin) {
var µb = µBlock;
var lastVersion = bin.version || '0.0.0.0';
// Whitelist some key scopes by default
if ( lastVersion.localeCompare('0.8.6.0') < 0 ) {
µb.netWhitelist = µb.whitelistFromString(
µb.stringFromWhitelist(µb.netWhitelist) +
'\n' +
µb.netWhitelistDefault
);
µb.saveWhitelist();
}
vAPI.storage.set({ version: vAPI.app.version });
onAllReady();
};
/******************************************************************************/
// Filter lists
// Whitelist
var countdown = 2;
var doCountdown = function() {
countdown -= 1;
if ( countdown !== 0 ) {
return;
}
var onMetadataReady = function(metadata) {
// Check PSL
var mdEntry = metadata[µb.pslPath];
if ( mdEntry.repoObsolete ) {
// console.log('µBlock.updater> updating all updatable assets');
µb.loadUpdatableAssets({ update: true });
return;
}
// Check used filter lists
var lists = µb.remoteBlacklists;
for ( var path in lists ) {
if ( lists.hasOwnProperty(path) === false ) {
continue;
}
if ( lists[path].off ) {
continue;
}
if ( metadata.hasOwnProperty(path) === false ) {
continue;
}
mdEntry = metadata[path];
if ( mdEntry.cacheObsolete || mdEntry.repoObsolete ) {
// console.log('µBlock.updater> updating only filter lists');
µb.loadUpdatableAssets({ update: true, psl: false });
return;
}
}
// console.log('µBlock.updater> all is up to date');
};
µb.assets.metadata(onMetadataReady);
// Last step: do whatever is necessary when version changes
vAPI.storage.get('version', onVersionReady);
};
// https://www.youtube.com/watch?v=cIrGQD84F1g
/******************************************************************************/
var restart = function(after) {
if ( after === undefined ) {
after = µb.nextUpdateAfter;
// Filters are in memory.
// Filter engines need PSL to be ready.
var onFiltersReady = function() {
doCountdown();
};
/******************************************************************************/
// https://github.com/gorhill/uBlock/issues/226
// Whitelist in memory.
// Whitelist parser needs PSL to be ready.
// gorhill 2014-12-15: not anymore
var onWhitelistReady = function() {
doCountdown();
};
/******************************************************************************/
// Load order because dependencies:
// User settings -> PSL -> [filter lists]
var onPSLReady = function() {
µBlock.loadFilterLists(onFiltersReady);
};
/******************************************************************************/
// If no selfie available, take the long way, i.e. load and parse
// raw data.
var onSelfieReady = function(success) {
if ( success === true ) {
onFiltersReady();
return;
}
µb.asyncJobs.add(
'autoUpdateAssets',
null,
jobCallback,
after,
false
);
µBlock.loadPublicSuffixList(onPSLReady);
};
/******************************************************************************/
return {
restart: restart
// User settings are in memory
var onUserSettingsReady = function(userSettings) {
var µb = µBlock;
// https://github.com/gorhill/uBlock/issues/426
// Important: block remote fetching for when loading assets at launch
// time.
µb.assets.allowRemoteFetch = false;
µb.assets.autoUpdate = userSettings.autoUpdate;
µb.fromSelfie(onSelfieReady);
// https://github.com/gorhill/uBlock/issues/540
// Disabling local mirroring for the time being
userSettings.experimentalEnabled = false;
µb.mirrors.toggle(false /* userSettings.experimentalEnabled */);
µb.contextMenu.toggle(userSettings.contextMenuEnabled);
µb.permanentFirewall.fromString(userSettings.dynamicFilteringString);
µb.sessionFirewall.assign(µb.permanentFirewall);
// Remove obsolete setting
delete userSettings.logRequests;
µb.XAL.keyvalRemoveOne('logRequests');
};
µBlock.loadUserSettings(onUserSettingsReady);
µBlock.loadWhitelist(onWhitelistReady);
µBlock.loadLocalSettings();
/******************************************************************************/
})();
/******************************************************************************/
// Load everything
µBlock.load();
/******************************************************************************/

View File

@ -1,7 +1,7 @@
/*******************************************************************************
µBlock - a Chromium browser extension to block requests.
Copyright (C) 2014 Raymond Hill
Copyright (C) 2014-2015 Raymond Hill
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@ -104,17 +104,6 @@
µBlock.loadWhitelist = function(callback) {
var onWhitelistLoaded = function(store) {
var µb = µBlock;
// Backward compatibility after fix to #5
// TODO: remove once all users are up to date with latest version.
if ( store.netExceptionList ) {
if ( store.netWhitelist === '' ) {
store.netWhitelist = Object.keys(store.netExceptionList).join('\n');
if ( store.netWhitelist !== '' ) {
vAPI.storage.set({ 'netWhitelist': store.netWhitelist });
}
}
vAPI.storage.remove('netExceptionList');
}
µb.netWhitelist = µb.whitelistFromString(store.netWhitelist);
µb.netWhitelistModifyTime = Date.now();
@ -124,8 +113,7 @@
};
var bin = {
'netWhitelist': this.netWhitelistDefault,
'netExceptionList': ''
'netWhitelist': this.netWhitelistDefault
};
vAPI.storage.get(bin, onWhitelistLoaded);
};
@ -377,7 +365,6 @@
var reIsLocalhostRedirect = /\s+(?:broadcasthost|local|localhost|localhost\.localdomain)(?=\s|$)/;
var reLocalIp = /^(?:0\.0\.0\.0|127\.0\.0\.1|::1|fe80::1%lo0)/;
//var reAsciiSegment = /^[\x21-\x7e]+$/;
var matches;
var lineBeg = 0, lineEnd, currentLineBeg;
var line, lineRaw, c, pos;
@ -534,7 +521,7 @@
var onFiltersReady = function() {
if ( update ) {
µb.updater.restart();
µb.assetUpdater.restart();
}
};
@ -560,7 +547,7 @@
cosmeticFilteringEngine: this.cosmeticFilteringEngine.toSelfie()
};
vAPI.storage.set({ selfie: selfie });
// console.log('µBlock.toSelfie> made a selfie!');
console.log('storage.js > µBlock.toSelfie()');
};
// This is to be sure the selfie is generated in a sane manner: the selfie will
@ -613,117 +600,62 @@
µBlock.destroySelfie = function() {
vAPI.storage.remove('selfie');
this.asyncJobs.remove('toSelfie');
console.log('storage.js > µBlock.destroySelfie()');
};
/******************************************************************************/
// Load all
µBlock.load = function() {
µBlock.updateStartHandler = function() {
var µb = this;
var fromSelfie = false;
// Final initialization steps after all needed assets are in memory.
// - Initialize internal state with maybe already existing tabs.
// - Schedule next update operation.
var onAllReady = function() {
// https://github.com/gorhill/uBlock/issues/426
// Important: remove barrier to remote fetching, this was useful only
// for launch time.
µb.assets.allowRemoteFetch = true;
// https://github.com/gorhill/uBlock/issues/184
// Check for updates not too far in the future.
µb.updater.restart(µb.firstUpdateAfter);
vAPI.onLoadAllCompleted();
};
// To bring older versions up to date
var onVersionReady = function(bin) {
var lastVersion = bin.version || '0.0.0.0';
// Whitelist some key scopes by default
if ( lastVersion.localeCompare('0.8.6.0') < 0 ) {
µb.netWhitelist = µb.whitelistFromString(
µb.stringFromWhitelist(µb.netWhitelist) +
'\n' +
µb.netWhitelistDefault
);
µb.saveWhitelist();
var onListsReady = function(lists) {
for ( var location in lists ) {
if ( lists.hasOwnProperty(location) === false ) {
continue;
}
if ( lists[location].off ) {
continue;
}
µb.assetUpdater.add(location);
}
vAPI.storage.set({ version: vAPI.app.version });
onAllReady();
};
// Filter lists
// Whitelist
var countdown = 2;
var doCountdown = function() {
countdown -= 1;
if ( countdown !== 0 ) {
return;
}
// Last step: do whatever is necessary when version changes
vAPI.storage.get('version', onVersionReady);
};
// Filters are in memory.
// Filter engines need PSL to be ready.
var onFiltersReady = function() {
doCountdown();
};
// https://github.com/gorhill/uBlock/issues/226
// Whitelist in memory.
// Whitelist parser needs PSL to be ready.
// gorhill 2014-12-15: not anymore
var onWhitelistReady = function() {
doCountdown();
};
// Load order because dependencies:
// User settings -> PSL -> [filter lists]
var onPSLReady = function() {
µb.loadFilterLists(onFiltersReady);
};
// If no selfie available, take the long way, i.e. load and parse
// raw data.
var onSelfieReady = function(success) {
if ( success === true ) {
fromSelfie = true;
onFiltersReady();
return;
}
µb.loadPublicSuffixList(onPSLReady);
};
// User settings are in memory
var onUserSettingsReady = function(userSettings) {
// https://github.com/gorhill/uBlock/issues/426
// Important: block remote fetching for when loading assets at launch
// time.
µb.assets.allowRemoteFetch = false;
µb.assets.autoUpdate = userSettings.autoUpdate;
µb.fromSelfie(onSelfieReady);
// https://github.com/gorhill/uBlock/issues/540
// Disabling local mirroring for the time being
userSettings.experimentalEnabled = false;
µb.mirrors.toggle(false /* userSettings.experimentalEnabled */);
µb.contextMenu.toggle(userSettings.contextMenuEnabled);
µb.permanentFirewall.fromString(userSettings.dynamicFilteringString);
µb.sessionFirewall.assign(µb.permanentFirewall);
// Remove obsolete setting
delete userSettings.logRequests;
µb.XAL.keyvalRemoveOne('logRequests');
};
this.loadUserSettings(onUserSettingsReady);
this.loadWhitelist(onWhitelistReady);
this.loadLocalSettings();
this.getAvailableLists(onListsReady);
this.assetUpdater.add(this.pslPath);
this.assetUpdater.add('assets/ublock/mirror-candidates.txt');
};
/******************************************************************************/
µBlock.updateCompleteHandler = function(details) {
var µb = this;
var updatedCount = details.updatedCount;
// Assets are supposed to have been all updated, avoid fetching from
// remote servers.
µb.assets.allowRemoteFetch = false;
var onFiltersReady = function() {
µb.assets.allowRemoteFetch = true;
};
var onPSLReady = function() {
if ( updatedCount !== 0 ) {
console.debug('storage.js > µBlock.updateCompleteHandler: reloading filter lists');
µb.loadFilterLists(onFiltersReady);
} else {
onFiltersReady();
}
};
if ( details.hasOwnProperty('assets/ublock/mirror-candidates.txt') ) {
/* TODO */
}
if ( details.hasOwnProperty(this.pslPath) ) {
console.debug('storage.js > µBlock.updateCompleteHandler: reloading PSL');
this.loadPublicSuffixList(onPSLReady);
updatedCount -= 1;
} else {
onPSLReady();
}
};