Mirrors
/
uBlock
mirror of https://github.com/gorhill/uBlock.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[mv3] Reuse rule ids across release where possible 

This is to reduce the diff size of rulesets in new
releases. Beside smaller diff size, this also makes it
easier to investigate rule changes across releases.
This commit is contained in:
Raymond Hill 2024-02-14 14:27:36 -05:00
parent d6b88d5d6e
commit f2d7413a42
No known key found for this signature in database
GPG Key ID: 25E1490B761470C2
3 changed files with 164 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
platform/mv3/make-rulesets.js
View File

@ -1152,10 +1152,10 @@ async function main() {
// Assemble all default lists as the default ruleset
const contentURLs = [
'https://ublockorigin.github.io/uAssets/filters/filters.min.txt',
'https://ublockorigin.github.io/uAssets/filters/badware.txt',
'https://ublockorigin.github.io/uAssets/filters/badware.min.txt',
'https://ublockorigin.github.io/uAssets/filters/privacy.min.txt',
'https://ublockorigin.github.io/uAssets/filters/unbreak.min.txt',
'https://ublockorigin.github.io/uAssets/filters/quick-fixes.txt',
'https://ublockorigin.github.io/uAssets/filters/quick-fixes.min.txt',
'https://ublockorigin.github.io/uAssets/filters/ubol-filters.txt',
'https://ublockorigin.github.io/uAssets/thirdparties/easylist.txt',
'https://ublockorigin.github.io/uAssets/thirdparties/easyprivacy.txt',

108
platform/mv3/salvage-ruleids.mjs Normal file
View File

@ -0,0 +1,108 @@
/*******************************************************************************
uBlock Origin - a comprehensive, efficient content blocker
Copyright (C) 2024-present Raymond Hill
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see {http://www.gnu.org/licenses/}.
Home: https://github.com/gorhill/uBlock
*/
'use strict';
/******************************************************************************/
import fs from 'fs/promises';
import process from 'process';
/******************************************************************************/
const commandLineArgs = (( ) => {
const args = new Map();
let name, value;
for ( const arg of process.argv.slice(2) ) {
const pos = arg.indexOf('=');
if ( pos === -1 ) {
name = arg;
value = '';
} else {
name = arg.slice(0, pos);
value = arg.slice(pos+1);
}
args.set(name, value);
}
return args;
})();
const beforeDir = commandLineArgs.get('before') || '';
const afterDir = commandLineArgs.get('after') || '';
if ( beforeDir === '' || afterDir === '' ) {
process.exit(0);
}
/******************************************************************************/
async function main() {
const afterFiles = await fs.readdir(`${afterDir}/rulesets/main`);
const writePromises = [];
for ( const file of afterFiles ) {
let raw = await fs.readFile(`${beforeDir}/rulesets/main/${file}`, 'utf-8').catch(( ) => '');
let beforeRules;
try { beforeRules = JSON.parse(raw); } catch(_) { }
if ( Array.isArray(beforeRules) === false ) { continue; }
raw = await fs.readFile(`${afterDir}/rulesets/main/${file}`, 'utf-8').catch(( ) => '');
let afterRules;
try { afterRules = JSON.parse(raw); } catch(_) { }
if ( Array.isArray(afterRules) === false ) { continue; }
const beforeMap = new Map(beforeRules.map(a => {
const id = a.id;
a.id = 0;
return [ JSON.stringify(a), id ];
}));
const usedIds = new Set();
for ( const afterRule of afterRules ) {
afterRule.id = 0;
const key = JSON.stringify(afterRule);
const beforeId = beforeMap.get(key);
if ( beforeId === undefined ) { continue; }
if ( usedIds.has(beforeId) ) { continue; }
afterRule.id = beforeId;
usedIds.add(beforeId);
}
// Assign new ids to unmatched rules
let ruleIdGenerator = 1;
for ( const afterRule of afterRules ) {
if ( afterRule.id !== 0 ) { continue; }
while ( usedIds.has(ruleIdGenerator) ) { ruleIdGenerator += 1; }
afterRule.id = ruleIdGenerator++;
}
afterRules.sort((a, b) => a.id - b.id);
const lines = [];
for ( const afterRule of afterRules ) {
lines.push(JSON.stringify(afterRule));
}
writePromises.push(
fs.writeFile(
`${afterDir}/rulesets/main/${file}`,
`[\n${lines.join(',\n')}\n]\n`
)
);
}
await Promise.all(writePromises);
}
main();
/******************************************************************************/

99
tools/make-mv3.sh
View File

@ -27,11 +27,15 @@ for i in "$@"; do
PLATFORM="chromium"
shift # past argument=value
;;
(uBOLite_+([0-9]).+([0-9]).+([0-9]).+([0-9]))
uBOLite_+([0-9]).+([0-9]).+([0-9]).+([0-9]))
TAGNAME="$i"
FULL="yes"
shift # past argument=value
;;
before=+([print]))
BEFORE="${i:7}"
shift # past argument=value
;;
esac
done
@ -46,9 +50,9 @@ cd $DES
DES=$(pwd)
cd - > /dev/null
mkdir -p $DES/css/fonts
mkdir -p $DES/js
mkdir -p $DES/img
mkdir -p "$DES"/css/fonts
mkdir -p "$DES"/js
mkdir -p "$DES"/img
if [ -n "$UBO_VERSION" ]; then
UBO_REPO="https://github.com/gorhill/uBlock.git"
@ -65,55 +69,60 @@ else
fi
echo "*** uBOLite.mv3: Copying common files"
cp -R $UBO_DIR/src/css/fonts/* $DES/css/fonts/
cp $UBO_DIR/src/css/themes/default.css $DES/css/
cp $UBO_DIR/src/css/common.css $DES/css/
cp $UBO_DIR/src/css/dashboard-common.css $DES/css/
cp $UBO_DIR/src/css/fa-icons.css $DES/css/
cp -R "$UBO_DIR"/src/css/fonts/* "$DES"/css/fonts/
cp "$UBO_DIR"/src/css/themes/default.css "$DES"/css/
cp "$UBO_DIR"/src/css/common.css "$DES"/css/
cp "$UBO_DIR"/src/css/dashboard-common.css "$DES"/css/
cp "$UBO_DIR"/src/css/fa-icons.css "$DES"/css/
cp $UBO_DIR/src/js/dom.js $DES/js/
cp $UBO_DIR/src/js/fa-icons.js $DES/js/
cp $UBO_DIR/src/js/i18n.js $DES/js/
cp $UBO_DIR/src/lib/punycode.js $DES/js/
cp "$UBO_DIR"/src/js/dom.js "$DES"/js/
cp "$UBO_DIR"/src/js/fa-icons.js "$DES"/js/
cp "$UBO_DIR"/src/js/i18n.js "$DES"/js/
cp "$UBO_DIR"/src/lib/punycode.js "$DES"/js/
cp -R $UBO_DIR/src/img/flags-of-the-world $DES/img
cp -R "$UBO_DIR/src/img/flags-of-the-world" "$DES"/img
cp LICENSE.txt $DES/
cp LICENSE.txt "$DES"/
echo "*** uBOLite.mv3: Copying mv3-specific files"
if [ "$PLATFORM" = "firefox" ]; then
cp platform/mv3/firefox/background.html $DES/
cp platform/mv3/firefox/background.html "$DES"/
fi
cp platform/mv3/extension/*.html $DES/
cp platform/mv3/extension/*.json $DES/
cp platform/mv3/extension/css/* $DES/css/
cp -R platform/mv3/extension/js/* $DES/js/
cp platform/mv3/extension/img/* $DES/img/
cp -R platform/mv3/extension/_locales $DES/
cp platform/mv3/README.md $DES/
cp platform/mv3/extension/*.html "$DES"/
cp platform/mv3/extension/*.json "$DES"/
cp platform/mv3/extension/css/* "$DES"/css/
cp -R platform/mv3/extension/js/* "$DES"/js/
cp platform/mv3/extension/img/* "$DES"/img/
cp -R platform/mv3/extension/_locales "$DES"/
cp platform/mv3/README.md "$DES/"
if [ "$QUICK" != "yes" ]; then
echo "*** uBOLite.mv3: Generating rulesets"
TMPDIR=$(mktemp -d)
mkdir -p $TMPDIR
mkdir -p "$TMPDIR"
if [ "$PLATFORM" = "chromium" ]; then
cp platform/mv3/chromium/manifest.json $DES/
cp platform/mv3/chromium/manifest.json "$DES"/
elif [ "$PLATFORM" = "firefox" ]; then
cp platform/mv3/firefox/manifest.json $DES/
cp platform/mv3/firefox/manifest.json "$DES"/
fi
./tools/make-nodejs.sh "$TMPDIR"
cp platform/mv3/package.json "$TMPDIR"/
cp platform/mv3/*.js "$TMPDIR"/
cp platform/mv3/*.mjs "$TMPDIR"/
cp platform/mv3/extension/js/utils.js "$TMPDIR"/js/
cp "$UBO_DIR"/assets/assets.json "$TMPDIR"/
cp "$UBO_DIR"/assets/resources/scriptlets.js "$TMPDIR"/
cp -R platform/mv3/scriptlets "$TMPDIR"/
mkdir -p "$TMPDIR"/web_accessible_resources
cp "$UBO_DIR"/src/web_accessible_resources/* "$TMPDIR"/web_accessible_resources/
cd "$TMPDIR"
node --no-warnings make-rulesets.js output="$DES" platform="$PLATFORM"
if [ -n "$BEFORE" ]; then
echo "*** uBOLite.mv3: salvaging rule ids to minimize diff size"
node --no-warnings salvage-ruleids.mjs before="$BEFORE"/"$PLATFORM" after="$DES"
fi
./tools/make-nodejs.sh $TMPDIR
cp platform/mv3/package.json $TMPDIR/
cp platform/mv3/*.js $TMPDIR/
cp platform/mv3/extension/js/utils.js $TMPDIR/js/
cp $UBO_DIR/assets/assets.json $TMPDIR/
cp $UBO_DIR/assets/resources/scriptlets.js $TMPDIR/
cp -R platform/mv3/scriptlets $TMPDIR/
mkdir -p $TMPDIR/web_accessible_resources
cp $UBO_DIR/src/web_accessible_resources/* $TMPDIR/web_accessible_resources/
cd $TMPDIR
node --no-warnings make-rulesets.js output=$DES platform="$PLATFORM"
cd - > /dev/null
rm -rf $TMPDIR
rm -rf "$TMPDIR"
fi
echo "*** uBOLite.mv3: extension ready"
@ -126,7 +135,7 @@ if [ "$FULL" = "yes" ]; then
fi
echo "*** uBOLite.mv3: Creating publishable package..."
if [ -z "$TAGNAME" ]; then
TAGNAME="uBOLite_$(jq -r .version $DES/manifest.json)"
TAGNAME="uBOLite_$(jq -r .version "$DES"/manifest.json)"
else
tmp=$(mktemp)
jq --arg version "${TAGNAME:8}" '.version = $version' "$DES/manifest.json" > "$tmp" \
@ -134,12 +143,12 @@ if [ "$FULL" = "yes" ]; then
fi
PACKAGENAME="$TAGNAME.$PLATFORM.mv3.$EXTENSION"
TMPDIR=$(mktemp -d)
mkdir -p $TMPDIR
cp -R $DES/* $TMPDIR/
cd $TMPDIR > /dev/null
zip $PACKAGENAME -qr ./*
mkdir -p "$TMPDIR"
cp -R "$DES"/* "$TMPDIR"/
cd "$TMPDIR" > /dev/null
zip "$PACKAGENAME" -qr ./*
cd - > /dev/null
cp $TMPDIR/$PACKAGENAME dist/build/
rm -rf $TMPDIR
cp "$TMPDIR"/"$PACKAGENAME" dist/build/
rm -rf "$TMPDIR"
echo "Package location: $(pwd)/dist/build/$PACKAGENAME"
fi