Support for splitting sync storage items into chunks, to allow synchronization of big policies across devices.

src/common/Storage.js

@@ -1,64 +1,165 @@
-var Storage = {
+"use strict";
+var Storage = (() => {
 
-  async safeOp(op, type, keys) {
+  let chunksKey = k => `${k}/CHUNKS`;
+
+  async function safeOp(op, type, keys) {
     let sync = type === "sync";
-    if (sync && op === "get") {
+
-      let localFallback = await this.localFallback();
-      if (localFallback.size) {
-        for (let k of Array.isArray(keys) ? keys : [keys]) {
-          if (localFallback.has(k)) {
-            type = "local";
-            break;
-          }
-        }
-      }
-    }
     try {
+      if (sync) {
+        let remove = op === "remove";
+        if (remove || op === "get") {
+          keys = [].concat(keys); // don't touch the passed argument
+          let mergeResults = {};
+          let localFallback = await getLocalFallback();
+          if (localFallback.size) {
+            let localKeys = keys.filter(k => localFallback.has(k));
+            if (localKeys.length) {
+              if (remove) {
+                await browser.storage.local.remove(localKeys);
+                for (let k of localKeys) {
+                  localFallback.delete(k);
+                }
+                await setLocalFallback(localFallback);
+              } else {
+                mergeResults = await browser.storage.local.get(localKeys);
+              }
+              keys = keys.filter(k => !localFallback.has(k));
+            }
+          }
+
+          if (keys.length) { // we may not have non-fallback keys anymore
+            let chunkCounts = Object.entries(await browser.storage.sync.get(
+                keys.map(chunksKey)))
+                  .map(([k, count]) => [k.split("/")[0], count]);
+            if (chunkCounts.length) {
+              let chunkedKeys = [];
+              for (let [k, count] of chunkCounts) {
+                // prepare to fetch all the chunks at once
+                while (count-- > 0) chunkedKeys.push(`${k}/${count}`);
+              }
+              if (remove) {
+                let doomedKeys = keys
+                  .concat(chunkCounts.map(([k, count]) => chunksKey(k)))
+                  .concat(chunkedKeys);
+                return await browser.storage.sync.remove(doomedKeys);
+              } else {
+                let chunks = await browser.storage.sync.get(chunkedKeys);
+                for (let [k, count] of chunkCounts) {
+                  let orderedChunks = [];
+                  for (let j = 0; j < count; j++) {
+                    orderedChunks.push(chunks[`${k}/${j}`]);
+                  }
+                  let whole = orderedChunks.join('');
+                  try {
+                    mergeResults[k] = JSON.parse(whole);
+                    keys.splice(keys.indexOf(k), 1); // remove from "main" keys
+                  } catch (e) {
+                    error(e, "Could not parse chunked storage key %s (%s).", k, whole);
+                  }
+                }
+              }
+            }
+          }
+          return keys.length ?
+            Object.assign(mergeResults, await browser.storage.sync[op](keys))
+            : mergeResults;
+        } else if (op === "set") {
+          keys = Object.assign({}, keys); // don't touch the passed argument
+          const MAX_ITEM_SIZE = 4096;
+          // Firefox Sync's max object BYTEs size is 16384, Chrome's 8192.
+          // Rather than mesuring actual bytes, we play it safe by halving then
+          // lowest to cope with escapes / multibyte characters.
+          for (let k of Object.keys(keys)) {
+            let s = JSON.stringify(keys[k]);
+            if (s.length > MAX_ITEM_SIZE) {
+              let count = Math.ceil(s.length / MAX_ITEM_SIZE);
+              let chunksCountKey = chunksKey(k);
+              let oldCount = await browser.storage.sync.get(chunksCountKey);
+              let chunks = {
+                [chunksCountKey]: count
+              };
+              for(let j = 0, o = 0; j < count; ++j, o += MAX_ITEM_SIZE) {
+                chunks[`${k}/${j}`] = s.substr(o, MAX_ITEM_SIZE);
+              }
+              await browser.storage.sync.set(chunks);
+              keys[k] = "[CHUNKED]";
+              if (oldCount-- > count) {
+                let oldChunks = [];
+                do {
+                  oldChunks.push(`${k}${oldCount}`);
+                } while(oldCount-- > count);
+                await browser.storage.sync.remove(oldChunks);
+              }
+            }
+          }
+        }
+      }
+
       let ret = await browser.storage[type][op](keys);
       if (sync && op === "set") {
-        let localFallback = await this.localFallback();
+        let localFallback = await getLocalFallback();
         let size = localFallback.size;
         if (size > 0) {
           for (let k of Object.keys(keys)) {
             localFallback.delete(k);
           }
-          if (size > localFallback.size) this.localFallback(localFallback);
+          if (size > localFallback.size) {
+            await setLocalFallback(localFallback);
+          }
         }
       }
       return ret;
     } catch (e) {
+      error(e, "%s.%s(%o)", type, op, keys);
       if (sync) {
         debug("Sync disabled? Falling back to local storage (%s %o)", op, keys);
-        let localFallback = await this.localFallback();
+        let localFallback = await getLocalFallback();
         let failedKeys = Array.isArray(keys) ? keys
           : typeof keys === "string" ? [keys] : Object.keys(keys);
         for (let k of failedKeys) {
           localFallback.add(k);
         }
-        await this.localFallback(localFallback);
+        await setLocalFallback(localFallback);
       } else {
-        error(e);
         throw e;
       }
     }
 
     return await browser.storage.local[op](keys);
-  },
+  }
 
+  const LFK_NAME = "__fallbackKeys";
+  async function setLocalFallback(keys) {
+    return await browser.storage.local.set({[LFK_NAME]: [...keys]});
+  }
+  async function getLocalFallback() {
+    let keys = (await browser.storage.local.get(LFK_NAME))[LFK_NAME];
+    return new Set(Array.isArray(keys) ? keys : []);
+  }
+
+  return {
     async get(type, keys) {
-    return await this.safeOp("get", type, keys);
+      return await safeOp("get", type, keys);
     },
 
     async set(type, keys) {
-    return await this.safeOp("set", type, keys);
+      return await safeOp("set", type, keys);
     },
 
-  async localFallback(keys) {
+    async remove(type, keys) {
-    let name = "__fallbackKeys";
+      return await safeOp("remove", type, keys);
-    if (keys) {
+    },
-      return await browser.storage.local.set({[name]: [...keys]});
+
+    async hasLocalFallback(key) {
+      return (await getLocalFallback()).has(key);
+    },
+
+    async isChunked(key) {
+      let ccKey = chunksKey(key);
+      let data = await browser.storage.sync.get([key, ccKey]);
+      return data[key] === "[CHUNKED]" && parseInt(data[ccKey]);
     }
-    let fallbackKeys = (await browser.storage.local.get(name))[name];
+  };
-    return new Set(Array.isArray(fallbackKeys) ? fallbackKeys : []);
+})()
-  }
-}

src/test/Storage_test.js

@@ -0,0 +1,69 @@
+"use strict";
+{
+  let makeBigObj = propsNum => {
+    let bigObj = {};
+    for (let j = propsNum; j-- > 0;) {
+      let x = "0000".concat(j.toString(16)).slice(-4);
+      bigObj[`k${x}`] = `v${x}`;
+    }
+    log("[TEST] created bigObj %s JSON characters long.", JSON.stringify(bigObj).length)
+    return bigObj;
+  }
+  let HUGE_SIZE = 16000,
+      BIG_SIZE = 1000;
+  let bigObject = makeBigObj(BIG_SIZE);
+  let hugeObject = makeBigObj(HUGE_SIZE);
+  let items = {"small1": {x: 1, y: 2}, bigObject, "small2": {k:3, j: 4}};
+  let keys = Object.keys(items);
+  keys.push("hugeObject");
+
+  let eq = async (key, prop, val) => {
+    let current = (await Storage.get("sync", key))[key];
+    let ok = current[prop] === val;
+    log("[TEST] sync.%s.%s %s %s\n(%o)", key, prop, ok ? "==" : "!=", val, current);
+    return ok;
+  };
+
+  let fallbackOrChunked = async key => {
+    let fallback = await Storage.hasLocalFallback(key);
+    let chunked = await Storage.isChunked(key);
+    log("[TEST] %s fallback: %s, chunked: %s", key, fallback, chunked);
+    return fallback ? !chunked : chunked;
+  }
+
+  let checkSize = async (key, size) =>
+    Object.keys((await Storage.get("sync", key))[key]).length === size;
+
+  let all;
+
+  (async () => {
+    for(let t of [
+      async () => {
+        await Storage.set("sync", items)
+        await Storage.set("sync", {hugeObject});  // fallback to local
+        all = await Storage.get("sync", keys);
+        log("[TEST] Storage:\nsync %o\nlocal %o\nfiltered (%o) %o",
+              await browser.storage.sync.get(),
+              await browser.storage.local.get(),
+              keys, all);
+        return Object.keys(all).length === keys.length;
+      },
+      async () => checkSize("hugeObject", HUGE_SIZE),
+      async () => checkSize("bigObject", BIG_SIZE),
+      async () => await fallbackOrChunked("bigObject"),
+      async () => await fallbackOrChunked("hugeObject"),
+      async () => await eq("small1", "y", 2),
+      async () => await eq("small2", "k", 3),
+      async () => await eq("bigObject", "k0000", "v0000"),
+      async () => await eq("hugeObject", "k0001", "v0001"),
+      async  () => {
+        await Storage.remove("sync", keys);
+        let myItems = await Storage.get("sync", keys);
+        return Object.keys(myItems).length === 0;
+      },
+    ]) {
+      await Test.run(t);
+    }
+    Test.report();
+  })();
+}

src/test/Test.js

@@ -26,17 +26,16 @@ var Test = (() => {
         error(e);
       }
       this[r ? "passed" : "failed"]++;
-      log(`${r ? "PASSED" : "FAILED"} ${msg || test}`);
+      log(`[TEST] ${r ? "PASSED" : "FAILED"} ${msg || test}`);
       if (typeof callback === "function") try {
-        callback(r, test, msg);
+        await callback(r, test, msg);
       } catch(e) {
-        error(e);
+        error(e, "[TEST]");
       }
     },
-
     report() {
       let {passed, failed} = this;
-      log(`FAILED: ${failed}, PASSED: ${passed}, TOTAL ${passed + failed}.`);
+      log(`[TESTS] FAILED: ${failed}, PASSED: ${passed}, TOTAL ${passed + failed}.`);
     }
   };
 

src/test/run.js

@@ -2,6 +2,7 @@
   await include("/test/Test.js");
   Test.include([
     "Policy",
+    "Storage",
     "XSS",
     "embargoed/XSS",
   ]);