From f2d7413a421a676322c4a45041887dbf7718a86f Mon Sep 17 00:00:00 2001 From: Raymond Hill Date: Wed, 14 Feb 2024 14:27:36 -0500 Subject: [PATCH] [mv3] Reuse rule ids across release where possible This is to reduce the diff size of rulesets in new releases. Beside smaller diff size, this also makes it easier to investigate rule changes across releases. --- platform/mv3/make-rulesets.js | 4 +- platform/mv3/salvage-ruleids.mjs | 108 +++++++++++++++++++++++++++++++ tools/make-mv3.sh | 99 +++++++++++++++------------- 3 files changed, 164 insertions(+), 47 deletions(-) create mode 100644 platform/mv3/salvage-ruleids.mjs diff --git a/platform/mv3/make-rulesets.js b/platform/mv3/make-rulesets.js index 34966e421..59ed88f0f 100644 --- a/platform/mv3/make-rulesets.js +++ b/platform/mv3/make-rulesets.js @@ -1152,10 +1152,10 @@ async function main() { // Assemble all default lists as the default ruleset const contentURLs = [ 'https://ublockorigin.github.io/uAssets/filters/filters.min.txt', - 'https://ublockorigin.github.io/uAssets/filters/badware.txt', + 'https://ublockorigin.github.io/uAssets/filters/badware.min.txt', 'https://ublockorigin.github.io/uAssets/filters/privacy.min.txt', 'https://ublockorigin.github.io/uAssets/filters/unbreak.min.txt', - 'https://ublockorigin.github.io/uAssets/filters/quick-fixes.txt', + 'https://ublockorigin.github.io/uAssets/filters/quick-fixes.min.txt', 'https://ublockorigin.github.io/uAssets/filters/ubol-filters.txt', 'https://ublockorigin.github.io/uAssets/thirdparties/easylist.txt', 'https://ublockorigin.github.io/uAssets/thirdparties/easyprivacy.txt', diff --git a/platform/mv3/salvage-ruleids.mjs b/platform/mv3/salvage-ruleids.mjs new file mode 100644 index 000000000..cfbfd7ab6 --- /dev/null +++ b/platform/mv3/salvage-ruleids.mjs @@ -0,0 +1,108 @@ +/******************************************************************************* + + uBlock Origin - a comprehensive, efficient content blocker + Copyright (C) 2024-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +/******************************************************************************/ + +import fs from 'fs/promises'; +import process from 'process'; + +/******************************************************************************/ + +const commandLineArgs = (( ) => { + const args = new Map(); + let name, value; + for ( const arg of process.argv.slice(2) ) { + const pos = arg.indexOf('='); + if ( pos === -1 ) { + name = arg; + value = ''; + } else { + name = arg.slice(0, pos); + value = arg.slice(pos+1); + } + args.set(name, value); + } + return args; +})(); + +const beforeDir = commandLineArgs.get('before') || ''; +const afterDir = commandLineArgs.get('after') || ''; + +if ( beforeDir === '' || afterDir === '' ) { + process.exit(0); +} + +/******************************************************************************/ + +async function main() { + const afterFiles = await fs.readdir(`${afterDir}/rulesets/main`); + const writePromises = []; + for ( const file of afterFiles ) { + let raw = await fs.readFile(`${beforeDir}/rulesets/main/${file}`, 'utf-8').catch(( ) => ''); + let beforeRules; + try { beforeRules = JSON.parse(raw); } catch(_) { } + if ( Array.isArray(beforeRules) === false ) { continue; } + raw = await fs.readFile(`${afterDir}/rulesets/main/${file}`, 'utf-8').catch(( ) => ''); + let afterRules; + try { afterRules = JSON.parse(raw); } catch(_) { } + if ( Array.isArray(afterRules) === false ) { continue; } + const beforeMap = new Map(beforeRules.map(a => { + const id = a.id; + a.id = 0; + return [ JSON.stringify(a), id ]; + })); + const usedIds = new Set(); + for ( const afterRule of afterRules ) { + afterRule.id = 0; + const key = JSON.stringify(afterRule); + const beforeId = beforeMap.get(key); + if ( beforeId === undefined ) { continue; } + if ( usedIds.has(beforeId) ) { continue; } + afterRule.id = beforeId; + usedIds.add(beforeId); + } + // Assign new ids to unmatched rules + let ruleIdGenerator = 1; + for ( const afterRule of afterRules ) { + if ( afterRule.id !== 0 ) { continue; } + while ( usedIds.has(ruleIdGenerator) ) { ruleIdGenerator += 1; } + afterRule.id = ruleIdGenerator++; + } + afterRules.sort((a, b) => a.id - b.id); + const lines = []; + for ( const afterRule of afterRules ) { + lines.push(JSON.stringify(afterRule)); + } + writePromises.push( + fs.writeFile( + `${afterDir}/rulesets/main/${file}`, + `[\n${lines.join(',\n')}\n]\n` + ) + ); + } + await Promise.all(writePromises); +} + +main(); + +/******************************************************************************/ diff --git a/tools/make-mv3.sh b/tools/make-mv3.sh index 8b1b2abd2..a5598ff54 100755 --- a/tools/make-mv3.sh +++ b/tools/make-mv3.sh @@ -27,11 +27,15 @@ for i in "$@"; do PLATFORM="chromium" shift # past argument=value ;; - (uBOLite_+([0-9]).+([0-9]).+([0-9]).+([0-9])) + uBOLite_+([0-9]).+([0-9]).+([0-9]).+([0-9])) TAGNAME="$i" FULL="yes" shift # past argument=value ;; + before=+([print])) + BEFORE="${i:7}" + shift # past argument=value + ;; esac done @@ -46,9 +50,9 @@ cd $DES DES=$(pwd) cd - > /dev/null -mkdir -p $DES/css/fonts -mkdir -p $DES/js -mkdir -p $DES/img +mkdir -p "$DES"/css/fonts +mkdir -p "$DES"/js +mkdir -p "$DES"/img if [ -n "$UBO_VERSION" ]; then UBO_REPO="https://github.com/gorhill/uBlock.git" @@ -65,55 +69,60 @@ else fi echo "*** uBOLite.mv3: Copying common files" -cp -R $UBO_DIR/src/css/fonts/* $DES/css/fonts/ -cp $UBO_DIR/src/css/themes/default.css $DES/css/ -cp $UBO_DIR/src/css/common.css $DES/css/ -cp $UBO_DIR/src/css/dashboard-common.css $DES/css/ -cp $UBO_DIR/src/css/fa-icons.css $DES/css/ +cp -R "$UBO_DIR"/src/css/fonts/* "$DES"/css/fonts/ +cp "$UBO_DIR"/src/css/themes/default.css "$DES"/css/ +cp "$UBO_DIR"/src/css/common.css "$DES"/css/ +cp "$UBO_DIR"/src/css/dashboard-common.css "$DES"/css/ +cp "$UBO_DIR"/src/css/fa-icons.css "$DES"/css/ -cp $UBO_DIR/src/js/dom.js $DES/js/ -cp $UBO_DIR/src/js/fa-icons.js $DES/js/ -cp $UBO_DIR/src/js/i18n.js $DES/js/ -cp $UBO_DIR/src/lib/punycode.js $DES/js/ +cp "$UBO_DIR"/src/js/dom.js "$DES"/js/ +cp "$UBO_DIR"/src/js/fa-icons.js "$DES"/js/ +cp "$UBO_DIR"/src/js/i18n.js "$DES"/js/ +cp "$UBO_DIR"/src/lib/punycode.js "$DES"/js/ -cp -R $UBO_DIR/src/img/flags-of-the-world $DES/img +cp -R "$UBO_DIR/src/img/flags-of-the-world" "$DES"/img -cp LICENSE.txt $DES/ +cp LICENSE.txt "$DES"/ echo "*** uBOLite.mv3: Copying mv3-specific files" if [ "$PLATFORM" = "firefox" ]; then - cp platform/mv3/firefox/background.html $DES/ + cp platform/mv3/firefox/background.html "$DES"/ fi -cp platform/mv3/extension/*.html $DES/ -cp platform/mv3/extension/*.json $DES/ -cp platform/mv3/extension/css/* $DES/css/ -cp -R platform/mv3/extension/js/* $DES/js/ -cp platform/mv3/extension/img/* $DES/img/ -cp -R platform/mv3/extension/_locales $DES/ -cp platform/mv3/README.md $DES/ +cp platform/mv3/extension/*.html "$DES"/ +cp platform/mv3/extension/*.json "$DES"/ +cp platform/mv3/extension/css/* "$DES"/css/ +cp -R platform/mv3/extension/js/* "$DES"/js/ +cp platform/mv3/extension/img/* "$DES"/img/ +cp -R platform/mv3/extension/_locales "$DES"/ +cp platform/mv3/README.md "$DES/" if [ "$QUICK" != "yes" ]; then echo "*** uBOLite.mv3: Generating rulesets" TMPDIR=$(mktemp -d) - mkdir -p $TMPDIR + mkdir -p "$TMPDIR" if [ "$PLATFORM" = "chromium" ]; then - cp platform/mv3/chromium/manifest.json $DES/ + cp platform/mv3/chromium/manifest.json "$DES"/ elif [ "$PLATFORM" = "firefox" ]; then - cp platform/mv3/firefox/manifest.json $DES/ + cp platform/mv3/firefox/manifest.json "$DES"/ + fi + ./tools/make-nodejs.sh "$TMPDIR" + cp platform/mv3/package.json "$TMPDIR"/ + cp platform/mv3/*.js "$TMPDIR"/ + cp platform/mv3/*.mjs "$TMPDIR"/ + cp platform/mv3/extension/js/utils.js "$TMPDIR"/js/ + cp "$UBO_DIR"/assets/assets.json "$TMPDIR"/ + cp "$UBO_DIR"/assets/resources/scriptlets.js "$TMPDIR"/ + cp -R platform/mv3/scriptlets "$TMPDIR"/ + mkdir -p "$TMPDIR"/web_accessible_resources + cp "$UBO_DIR"/src/web_accessible_resources/* "$TMPDIR"/web_accessible_resources/ + cd "$TMPDIR" + node --no-warnings make-rulesets.js output="$DES" platform="$PLATFORM" + if [ -n "$BEFORE" ]; then + echo "*** uBOLite.mv3: salvaging rule ids to minimize diff size" + node --no-warnings salvage-ruleids.mjs before="$BEFORE"/"$PLATFORM" after="$DES" fi - ./tools/make-nodejs.sh $TMPDIR - cp platform/mv3/package.json $TMPDIR/ - cp platform/mv3/*.js $TMPDIR/ - cp platform/mv3/extension/js/utils.js $TMPDIR/js/ - cp $UBO_DIR/assets/assets.json $TMPDIR/ - cp $UBO_DIR/assets/resources/scriptlets.js $TMPDIR/ - cp -R platform/mv3/scriptlets $TMPDIR/ - mkdir -p $TMPDIR/web_accessible_resources - cp $UBO_DIR/src/web_accessible_resources/* $TMPDIR/web_accessible_resources/ - cd $TMPDIR - node --no-warnings make-rulesets.js output=$DES platform="$PLATFORM" cd - > /dev/null - rm -rf $TMPDIR + rm -rf "$TMPDIR" fi echo "*** uBOLite.mv3: extension ready" @@ -126,7 +135,7 @@ if [ "$FULL" = "yes" ]; then fi echo "*** uBOLite.mv3: Creating publishable package..." if [ -z "$TAGNAME" ]; then - TAGNAME="uBOLite_$(jq -r .version $DES/manifest.json)" + TAGNAME="uBOLite_$(jq -r .version "$DES"/manifest.json)" else tmp=$(mktemp) jq --arg version "${TAGNAME:8}" '.version = $version' "$DES/manifest.json" > "$tmp" \ @@ -134,12 +143,12 @@ if [ "$FULL" = "yes" ]; then fi PACKAGENAME="$TAGNAME.$PLATFORM.mv3.$EXTENSION" TMPDIR=$(mktemp -d) - mkdir -p $TMPDIR - cp -R $DES/* $TMPDIR/ - cd $TMPDIR > /dev/null - zip $PACKAGENAME -qr ./* + mkdir -p "$TMPDIR" + cp -R "$DES"/* "$TMPDIR"/ + cd "$TMPDIR" > /dev/null + zip "$PACKAGENAME" -qr ./* cd - > /dev/null - cp $TMPDIR/$PACKAGENAME dist/build/ - rm -rf $TMPDIR + cp "$TMPDIR"/"$PACKAGENAME" dist/build/ + rm -rf "$TMPDIR" echo "Package location: $(pwd)/dist/build/$PACKAGENAME" fi