Chunk 1.9.0 db upgrade and remove observable tables

package.json

@@ -30,6 +30,7 @@
     "image-outline": "^0.1.0",
     "intersection-observer": "^0.12.0",
     "konva": "^7.2.5",
+    "lodash.chunk": "^4.2.0",
     "lodash.clonedeep": "^4.5.0",
     "lodash.get": "^4.4.2",
     "lodash.set": "^4.3.2",

src/upgrade.js

@@ -3,6 +3,7 @@ import Dexie, { Version } from "dexie";
 import shortid from "shortid";
 import { v4 as uuid } from "uuid";
 import Case from "case";
+import chunk from "lodash.chunk";
 
 import blobToBuffer from "./helpers/blobToBuffer";
 import { getGridDefaultInset } from "./helpers/grid";
@@ -475,120 +476,179 @@ export const versions = {
   },
   // v1.9.0 - Move map assets into new table
   24(v, onUpgrade) {
-    v.stores({ assets: "id, owner" }).upgrade((tx) => {
+    v.stores({ assets: "id, owner" }).upgrade(async (tx) => {
       onUpgrade?.(24);
-      tx.table("maps").each((map) => {
-        let assets = [];
-        assets.push({
-          id: uuid(),
-          owner: map.owner,
-          file: map.file,
-          width: map.width,
-          height: map.height,
-          mime: "",
-          prevId: map.id,
-          prevType: "map",
-        });
 
-        for (let resolution in map.resolutions) {
+      const primaryKeys = await Dexie.waitFor(
-          const mapRes = map.resolutions[resolution];
+        tx.table("maps").toCollection().primaryKeys()
+      );
+      const keyChunks = chunk(primaryKeys, 4);
+
+      for (let keys of keyChunks) {
+        let assets = [];
+        let maps = await Dexie.waitFor(tx.table("maps").bulkGet(keys));
+        while (maps.length > 0) {
+          const map = maps.pop();
           assets.push({
             id: uuid(),
             owner: map.owner,
-            file: mapRes.file,
+            file: map.file,
-            width: mapRes.width,
+            width: map.width,
-            height: mapRes.height,
+            height: map.height,
             mime: "",
             prevId: map.id,
-            prevType: "mapResolution",
+            prevType: "map",
-            resolution,
+          });
+
+          for (let resolution in map.resolutions) {
+            const mapRes = map.resolutions[resolution];
+            assets.push({
+              id: uuid(),
+              owner: map.owner,
+              file: mapRes.file,
+              width: mapRes.width,
+              height: mapRes.height,
+              mime: "",
+              prevId: map.id,
+              prevType: "mapResolution",
+              resolution,
+            });
+          }
+
+          assets.push({
+            id: uuid(),
+            owner: map.owner,
+            file: map.thumbnail.file,
+            width: map.thumbnail.width,
+            height: map.thumbnail.height,
+            mime: "",
+            prevId: map.id,
+            prevType: "mapThumbnail",
           });
         }
-
+        maps = null;
-        assets.push({
+        await tx.table("assets").bulkAdd(assets);
-          id: uuid(),
+        assets = null;
-          owner: map.owner,
+      }
-          file: map.thumbnail.file,
-          width: map.thumbnail.width,
-          height: map.thumbnail.height,
-          mime: "",
-          prevId: map.id,
-          prevType: "mapThumbnail",
-        });
-
-        tx.table("assets").bulkAdd(assets);
-      });
     });
   },
   // v1.9.0 - Move token assets into new table
   25(v, onUpgrade) {
-    v.stores({}).upgrade((tx) => {
+    v.stores({}).upgrade(async (tx) => {
       onUpgrade?.(25);
-      tx.table("tokens").each((token) => {
+
+      const primaryKeys = await Dexie.waitFor(
+        tx.table("tokens").toCollection().primaryKeys()
+      );
+      const keyChunks = chunk(primaryKeys, 4);
+
+      for (let keys of keyChunks) {
         let assets = [];
-        assets.push({
+        let tokens = await Dexie.waitFor(tx.table("tokens").bulkGet(keys));
-          id: uuid(),
+        while (tokens.length > 0) {
-          owner: token.owner,
+          let token = tokens.pop();
-          file: token.file,
+          assets.push({
-          width: token.width,
+            id: uuid(),
-          height: token.height,
+            owner: token.owner,
-          mime: "",
+            file: token.file,
-          prevId: token.id,
+            width: token.width,
-          prevType: "token",
+            height: token.height,
-        });
+            mime: "",
-        assets.push({
+            prevId: token.id,
-          id: uuid(),
+            prevType: "token",
-          owner: token.owner,
+          });
-          file: token.thumbnail.file,
+          assets.push({
-          width: token.thumbnail.width,
+            id: uuid(),
-          height: token.thumbnail.height,
+            owner: token.owner,
-          mime: "",
+            file: token.thumbnail.file,
-          prevId: token.id,
+            width: token.thumbnail.width,
-          prevType: "tokenThumbnail",
+            height: token.thumbnail.height,
-        });
+            mime: "",
-        tx.table("assets").bulkAdd(assets);
+            prevId: token.id,
-      });
+            prevType: "tokenThumbnail",
+          });
+        }
+        tokens = null;
+        await tx.table("assets").bulkAdd(assets);
+        assets = null;
+      }
     });
   },
   // v1.9.0 - Create foreign keys for assets
   26(v, onUpgrade) {
-    v.stores({}).upgrade((tx) => {
+    v.stores({}).upgrade(async (tx) => {
       onUpgrade?.(26);
-      tx.table("assets").each((asset) => {
+
-        if (asset.prevType === "map") {
+      let mapUpdates = {};
-          tx.table("maps").update(asset.prevId, {
+      let tokenUpdates = {};
-            file: asset.id,
+
-          });
+      const primaryKeys = await Dexie.waitFor(
-        } else if (asset.prevType === "token") {
+        tx.table("assets").toCollection().primaryKeys()
-          tx.table("tokens").update(asset.prevId, {
+      );
-            file: asset.id,
+      const keyChunks = chunk(primaryKeys, 4);
-          });
+
-        } else if (asset.prevType === "mapThumbnail") {
+      for (let keys of keyChunks) {
-          tx.table("maps").update(asset.prevId, { thumbnail: asset.id });
+        let assets = await Dexie.waitFor(tx.table("assets").bulkGet(keys));
-        } else if (asset.prevType === "tokenThumbnail") {
+        while (assets.length > 0) {
-          tx.table("tokens").update(asset.prevId, { thumbnail: asset.id });
+          const asset = assets.pop();
-        } else if (asset.prevType === "mapResolution") {
+          const { prevId, id, prevType, resolution } = asset;
-          tx.table("maps").update(asset.prevId, {
+          if (prevType === "token" || prevType === "tokenThumbnail") {
-            resolutions: undefined,
+            if (!(prevId in tokenUpdates)) {
-            [asset.resolution]: asset.id,
+              tokenUpdates[prevId] = {};
-          });
+            }
+          } else {
+            if (!(prevId in mapUpdates)) {
+              mapUpdates[prevId] = {};
+            }
+          }
+
+          if (prevType === "map") {
+            mapUpdates[prevId].file = id;
+          } else if (prevType === "token") {
+            tokenUpdates[prevId].file = id;
+          } else if (prevType === "mapThumbnail") {
+            mapUpdates[prevId].thumbnail = id;
+          } else if (prevType === "tokenThumbnail") {
+            tokenUpdates[prevId].thumbnail = id;
+          } else if (prevType === "mapResolution") {
+            mapUpdates[prevId][resolution] = id;
+          }
         }
-      });
+        assets = null;
+      }
+
+      await tx
+        .table("maps")
+        .toCollection()
+        .modify((map) => {
+          if (map.id in mapUpdates) {
+            for (let key in mapUpdates[map.id]) {
+              map[key] = mapUpdates[map.id][key];
+            }
+          }
+          delete map.resolutions;
+        });
+      await tx
+        .table("tokens")
+        .toCollection()
+        .modify((token) => {
+          if (token.id in tokenUpdates) {
+            for (let key in tokenUpdates[token.id]) {
+              token[key] = tokenUpdates[token.id][key];
+            }
+          }
+        });
     });
   },
   // v1.9.0 - Remove asset migration helpers
   27(v, onUpgrade) {
     v.stores({}).upgrade((tx) => {
       onUpgrade?.(27);
-      tx.table("assets")
+      tx.table("assets").toCollection().modify({
-        .toCollection()
+        prevId: undefined,
-        .modify((asset) => {
+        prevType: undefined,
-          delete asset.prevId;
+        resolution: undefined,
-          if (asset.prevType === "mapResolution") {
+      });
-            delete asset.resolution;
-          }
-          delete asset.prevType;
-        });
     });
   },
   // v1.9.0 - Remap map resolution assets
@@ -771,9 +831,17 @@ export const versions = {
         });
     });
   },
+  36(v) {
+    v.stores({
+      _changes: null,
+      _intercomm: null,
+      _syncNodes: null,
+      _uncommittedChanges: null,
+    });
+  },
 };
 
-export const latestVersion = 35;
+export const latestVersion = 36;
 
 /**
  * Load versions onto a database up to a specific version number

yarn.lock

@@ -8604,6 +8604,11 @@ lodash.camelcase@^4.3.0:
   resolved "https://registry.yarnpkg.com/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz#b28aa6288a2b9fc651035c7711f65ab6190331a6"
   integrity sha1-soqmKIorn8ZRA1x3EfZathkDMaY=
 
+lodash.chunk@^4.2.0:
+  version "4.2.0"
+  resolved "https://registry.yarnpkg.com/lodash.chunk/-/lodash.chunk-4.2.0.tgz#66e5ce1f76ed27b4303d8c6512e8d1216e8106bc"
+  integrity sha1-ZuXOH3btJ7QwPYxlEujRIW6BBrw=
+
 lodash.clonedeep@^4.5.0:
   version "4.5.0"
   resolved "https://registry.yarnpkg.com/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz#e23f3f9c4f8fbdde872529c1071857a086e5ccef"