[mv3] Reuse rule ids across release where possible

This is to reduce the diff size of rulesets in new
releases. Beside smaller diff size, this also makes it
easier to investigate rule changes across releases.

platform/mv3/make-rulesets.js

@@ -1152,10 +1152,10 @@ async function main() {
     // Assemble all default lists as the default ruleset
     const contentURLs = [
         'https://ublockorigin.github.io/uAssets/filters/filters.min.txt',
-        'https://ublockorigin.github.io/uAssets/filters/badware.txt',
+        'https://ublockorigin.github.io/uAssets/filters/badware.min.txt',
         'https://ublockorigin.github.io/uAssets/filters/privacy.min.txt',
         'https://ublockorigin.github.io/uAssets/filters/unbreak.min.txt',
-        'https://ublockorigin.github.io/uAssets/filters/quick-fixes.txt',
+        'https://ublockorigin.github.io/uAssets/filters/quick-fixes.min.txt',
         'https://ublockorigin.github.io/uAssets/filters/ubol-filters.txt',
         'https://ublockorigin.github.io/uAssets/thirdparties/easylist.txt',
         'https://ublockorigin.github.io/uAssets/thirdparties/easyprivacy.txt',

platform/mv3/salvage-ruleids.mjs

@@ -0,0 +1,108 @@
+/*******************************************************************************
+
+    uBlock Origin - a comprehensive, efficient content blocker
+    Copyright (C) 2024-present Raymond Hill
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see {http://www.gnu.org/licenses/}.
+
+    Home: https://github.com/gorhill/uBlock
+*/
+
+'use strict';
+
+/******************************************************************************/
+
+import fs from 'fs/promises';
+import process from 'process';
+
+/******************************************************************************/
+
+const commandLineArgs = (( ) => {
+    const args = new Map();
+    let name, value;
+    for ( const arg of process.argv.slice(2) ) {
+        const pos = arg.indexOf('=');
+        if ( pos === -1 ) {
+            name = arg;
+            value = '';
+        } else {
+            name = arg.slice(0, pos);
+            value = arg.slice(pos+1);
+        }
+        args.set(name, value);
+    }
+    return args;
+})();
+
+const beforeDir = commandLineArgs.get('before') || '';
+const afterDir = commandLineArgs.get('after') || '';
+
+if ( beforeDir === '' || afterDir === '' ) {
+    process.exit(0);
+}
+
+/******************************************************************************/
+
+async function main() {
+    const afterFiles = await fs.readdir(`${afterDir}/rulesets/main`);
+    const writePromises = [];
+    for ( const file of afterFiles ) {
+        let raw = await fs.readFile(`${beforeDir}/rulesets/main/${file}`, 'utf-8').catch(( ) => '');
+        let beforeRules;
+        try { beforeRules = JSON.parse(raw); } catch(_) { }
+        if ( Array.isArray(beforeRules) === false ) { continue; }
+        raw = await fs.readFile(`${afterDir}/rulesets/main/${file}`, 'utf-8').catch(( ) => '');
+        let afterRules;
+        try { afterRules = JSON.parse(raw); } catch(_) { }
+        if ( Array.isArray(afterRules) === false ) { continue; }
+        const beforeMap = new Map(beforeRules.map(a => {
+            const id = a.id;
+            a.id = 0;
+            return [ JSON.stringify(a), id ];
+        }));
+        const usedIds = new Set();
+        for ( const afterRule of afterRules ) {
+            afterRule.id = 0;
+            const key = JSON.stringify(afterRule);
+            const beforeId = beforeMap.get(key);
+            if ( beforeId === undefined ) { continue; }
+            if ( usedIds.has(beforeId) ) { continue; }
+            afterRule.id = beforeId;
+            usedIds.add(beforeId);
+        }
+        // Assign new ids to unmatched rules
+        let ruleIdGenerator = 1;
+        for ( const afterRule of afterRules ) {
+            if ( afterRule.id !== 0 ) { continue; }
+            while ( usedIds.has(ruleIdGenerator) ) { ruleIdGenerator += 1; }
+            afterRule.id = ruleIdGenerator++;
+        }
+        afterRules.sort((a, b) => a.id - b.id);
+        const lines = [];
+        for ( const afterRule of afterRules ) {
+            lines.push(JSON.stringify(afterRule));
+        }
+        writePromises.push(
+            fs.writeFile(
+                `${afterDir}/rulesets/main/${file}`,
+                `[\n${lines.join(',\n')}\n]\n`
+            )
+        );
+    }
+    await Promise.all(writePromises);
+}
+
+main();
+
+/******************************************************************************/

tools/make-mv3.sh

@@ -27,11 +27,15 @@ for i in "$@"; do
       PLATFORM="chromium"
       shift # past argument=value
       ;;
-    (uBOLite_+([0-9]).+([0-9]).+([0-9]).+([0-9]))
+    uBOLite_+([0-9]).+([0-9]).+([0-9]).+([0-9]))
       TAGNAME="$i"
       FULL="yes"
       shift # past argument=value
       ;;
+    before=+([print]))
+      BEFORE="${i:7}"
+      shift # past argument=value
+      ;;
   esac
 done
 
@@ -46,9 +50,9 @@ cd $DES
 DES=$(pwd)
 cd - > /dev/null
 
-mkdir -p $DES/css/fonts
+mkdir -p "$DES"/css/fonts
-mkdir -p $DES/js
+mkdir -p "$DES"/js
-mkdir -p $DES/img
+mkdir -p "$DES"/img
 
 if [ -n "$UBO_VERSION" ]; then
     UBO_REPO="https://github.com/gorhill/uBlock.git"
@@ -65,55 +69,60 @@ else
 fi
 
 echo "*** uBOLite.mv3: Copying common files"
-cp -R $UBO_DIR/src/css/fonts/* $DES/css/fonts/
+cp -R "$UBO_DIR"/src/css/fonts/* "$DES"/css/fonts/
-cp $UBO_DIR/src/css/themes/default.css $DES/css/
+cp "$UBO_DIR"/src/css/themes/default.css "$DES"/css/
-cp $UBO_DIR/src/css/common.css $DES/css/
+cp "$UBO_DIR"/src/css/common.css "$DES"/css/
-cp $UBO_DIR/src/css/dashboard-common.css $DES/css/
+cp "$UBO_DIR"/src/css/dashboard-common.css "$DES"/css/
-cp $UBO_DIR/src/css/fa-icons.css $DES/css/
+cp "$UBO_DIR"/src/css/fa-icons.css "$DES"/css/
 
-cp $UBO_DIR/src/js/dom.js $DES/js/
+cp "$UBO_DIR"/src/js/dom.js "$DES"/js/
-cp $UBO_DIR/src/js/fa-icons.js $DES/js/
+cp "$UBO_DIR"/src/js/fa-icons.js "$DES"/js/
-cp $UBO_DIR/src/js/i18n.js $DES/js/
+cp "$UBO_DIR"/src/js/i18n.js "$DES"/js/
-cp $UBO_DIR/src/lib/punycode.js $DES/js/
+cp "$UBO_DIR"/src/lib/punycode.js "$DES"/js/
 
-cp -R $UBO_DIR/src/img/flags-of-the-world $DES/img
+cp -R "$UBO_DIR/src/img/flags-of-the-world" "$DES"/img
 
-cp LICENSE.txt $DES/
+cp LICENSE.txt "$DES"/
 
 echo "*** uBOLite.mv3: Copying mv3-specific files"
 if [ "$PLATFORM" = "firefox" ]; then
-    cp platform/mv3/firefox/background.html $DES/
+    cp platform/mv3/firefox/background.html "$DES"/
 fi
-cp platform/mv3/extension/*.html $DES/
+cp platform/mv3/extension/*.html "$DES"/
-cp platform/mv3/extension/*.json $DES/
+cp platform/mv3/extension/*.json "$DES"/
-cp platform/mv3/extension/css/* $DES/css/
+cp platform/mv3/extension/css/* "$DES"/css/
-cp -R platform/mv3/extension/js/* $DES/js/
+cp -R platform/mv3/extension/js/* "$DES"/js/
-cp platform/mv3/extension/img/* $DES/img/
+cp platform/mv3/extension/img/* "$DES"/img/
-cp -R platform/mv3/extension/_locales $DES/
+cp -R platform/mv3/extension/_locales "$DES"/
-cp platform/mv3/README.md $DES/
+cp platform/mv3/README.md "$DES/"
 
 if [ "$QUICK" != "yes" ]; then
     echo "*** uBOLite.mv3: Generating rulesets"
     TMPDIR=$(mktemp -d)
-    mkdir -p $TMPDIR
+    mkdir -p "$TMPDIR"
     if [ "$PLATFORM" = "chromium" ]; then
-        cp platform/mv3/chromium/manifest.json $DES/
+        cp platform/mv3/chromium/manifest.json "$DES"/
     elif [ "$PLATFORM" = "firefox" ]; then
-        cp platform/mv3/firefox/manifest.json $DES/
+        cp platform/mv3/firefox/manifest.json "$DES"/
+    fi
+    ./tools/make-nodejs.sh "$TMPDIR"
+    cp platform/mv3/package.json "$TMPDIR"/
+    cp platform/mv3/*.js "$TMPDIR"/
+    cp platform/mv3/*.mjs "$TMPDIR"/
+    cp platform/mv3/extension/js/utils.js "$TMPDIR"/js/
+    cp "$UBO_DIR"/assets/assets.json "$TMPDIR"/
+    cp "$UBO_DIR"/assets/resources/scriptlets.js "$TMPDIR"/
+    cp -R platform/mv3/scriptlets "$TMPDIR"/
+    mkdir -p "$TMPDIR"/web_accessible_resources
+    cp "$UBO_DIR"/src/web_accessible_resources/* "$TMPDIR"/web_accessible_resources/
+    cd "$TMPDIR"
+    node --no-warnings make-rulesets.js output="$DES" platform="$PLATFORM"
+    if [ -n "$BEFORE" ]; then
+        echo "*** uBOLite.mv3: salvaging rule ids to minimize diff size"
+        node --no-warnings salvage-ruleids.mjs before="$BEFORE"/"$PLATFORM" after="$DES"
     fi
-    ./tools/make-nodejs.sh $TMPDIR
-    cp platform/mv3/package.json $TMPDIR/
-    cp platform/mv3/*.js $TMPDIR/
-    cp platform/mv3/extension/js/utils.js $TMPDIR/js/
-    cp $UBO_DIR/assets/assets.json $TMPDIR/
-    cp $UBO_DIR/assets/resources/scriptlets.js $TMPDIR/
-    cp -R platform/mv3/scriptlets $TMPDIR/
-    mkdir -p $TMPDIR/web_accessible_resources
-    cp $UBO_DIR/src/web_accessible_resources/* $TMPDIR/web_accessible_resources/
-    cd $TMPDIR
-    node --no-warnings make-rulesets.js output=$DES platform="$PLATFORM"
     cd - > /dev/null
-    rm -rf $TMPDIR
+    rm -rf "$TMPDIR"
 fi
 
 echo "*** uBOLite.mv3: extension ready"
@@ -126,7 +135,7 @@ if [ "$FULL" = "yes" ]; then
     fi
     echo "*** uBOLite.mv3: Creating publishable package..."
     if [ -z "$TAGNAME" ]; then
-        TAGNAME="uBOLite_$(jq -r .version $DES/manifest.json)"
+        TAGNAME="uBOLite_$(jq -r .version "$DES"/manifest.json)"
     else
         tmp=$(mktemp)
         jq --arg version "${TAGNAME:8}" '.version = $version' "$DES/manifest.json"  > "$tmp" \
@@ -134,12 +143,12 @@ if [ "$FULL" = "yes" ]; then
     fi
     PACKAGENAME="$TAGNAME.$PLATFORM.mv3.$EXTENSION"
     TMPDIR=$(mktemp -d)
-    mkdir -p $TMPDIR
+    mkdir -p "$TMPDIR"
-    cp -R $DES/* $TMPDIR/
+    cp -R "$DES"/* "$TMPDIR"/
-    cd $TMPDIR > /dev/null
+    cd "$TMPDIR" > /dev/null
-    zip $PACKAGENAME -qr ./*
+    zip "$PACKAGENAME" -qr ./*
     cd - > /dev/null
-    cp $TMPDIR/$PACKAGENAME dist/build/
+    cp "$TMPDIR"/"$PACKAGENAME" dist/build/
-    rm -rf $TMPDIR
+    rm -rf "$TMPDIR"
     echo "Package location: $(pwd)/dist/build/$PACKAGENAME"
 fi