Migrate localStorage data to IndexedDB (#2040)

* Migrate localStorage data to IndexedDB

This ports our localStorage data (identity numbers used & timestamps) to
IndexedDB. By doing this we follow the team's convention of storing only
UI related, short-lived data in localStorage, and all meaningful data in
IndexedDB. This also simplifies the storage handling by skipping the
JSON serialization.

For the time being, the data is still written to localStorage in case a
rollback in needed. That way, data written after the upgrade but before
the rollback will still be available to users after the rollback.

* Use idb.clear()

* Use explicit idb-keyval imports

src/frontend/src/components/authenticateBox.ts

@@ -289,7 +289,7 @@ export const authenticateBoxFlow = async <T, I>({
 
   // If there _are_ some anchors, then we show the "pick" screen, otherwise
   // we assume a new user and show the "firstTime" screen.
-  const anchors = getAnchors();
+  const anchors = await getAnchors();
   if (isNonEmptyArray(anchors)) {
     const result = await pages.pick({ anchors });
 
@@ -316,7 +316,7 @@ export const handleLoginFlowResult = async <T>(
 ): Promise<LoginData<T> | undefined> => {
   switch (result.tag) {
     case "ok":
-      setAnchorUsed(result.userNumber);
+      await setAnchorUsed(result.userNumber);
       return result;
     case "err":
       await displayError({

src/frontend/src/flows/addDevice/manage/addFIDODevice.ts

@@ -85,7 +85,7 @@ export const addFIDODevice = async (
     );
   }
 
-  setAnchorUsed(userNumber);
+  await setAnchorUsed(userNumber);
 };
 
 const unknownError = (): Error => {

src/frontend/src/flows/addDevice/welcomeView/registerTentativeDevice.ts

@@ -40,7 +40,7 @@ export const registerTentativeDevice = async (
     if (isWebAuthnDuplicateDevice(result)) {
       // Given that this is a remote device where we get the result that authentication should work,
       // let's help the user and fill in their anchor number.
-      setAnchorUsed(userNumber);
+      await setAnchorUsed(userNumber);
       await displayDuplicateDeviceError({ primaryButton: "Ok" });
     } else if (isWebAuthnCancel(result)) {
       await displayCancelError({ primaryButton: "Ok" });

src/frontend/src/flows/addDevice/welcomeView/showVerificationCode.ts

@@ -168,7 +168,7 @@ const handlePollResult = async ({
   result: "match" | "canceled" | typeof AsyncCountdown.timeout;
 }): Promise<"ok"> => {
   if (result === "match") {
-    setAnchorUsed(userNumber);
+    await setAnchorUsed(userNumber);
     return "ok";
   } else if (result === AsyncCountdown.timeout) {
     await displayError({

src/frontend/src/flows/recovery/useRecovery.ts

@@ -170,6 +170,6 @@ const enrollAuthenticator = async ({
     return "error";
   }
 
-  setAnchorUsed(userNumber);
+  await setAnchorUsed(userNumber);
   return "enrolled";
 };

src/frontend/src/flows/register/index.ts

@@ -174,7 +174,7 @@ export const registerFlow = async <T>({
   if (result.kind === "loginSuccess") {
     const userNumber = result.userNumber;
     await finalizeIdentity?.(userNumber);
-    setAnchorUsed(userNumber);
+    await setAnchorUsed(userNumber);
     await displayUserNumber({
       userNumber,
       stepper: finishStepper,

src/frontend/src/storage/index.test.ts

@@ -1,15 +1,27 @@
 import { nonNullish } from "@dfinity/utils";
+import { IDBFactory } from "fake-indexeddb";
+import {
+  clear as idbClear,
+  get as idbGet,
+  keys as idbKeys,
+  set as idbSet,
+} from "idb-keyval";
 import { MAX_SAVED_ANCHORS, getAnchors, setAnchorUsed } from ".";
 
-test("anchors default to nothing", () => {
-  expect(getAnchors()).toStrictEqual([]);
+beforeAll(() => {
+  // Initialize the IndexedDB global
+  global.indexedDB = new IDBFactory();
+});
+
+test("anchors default to nothing", async () => {
+  expect(await getAnchors()).toStrictEqual([]);
 });
 
 test(
   "old userNumber is recovered",
   withStorage(
-    () => {
-      expect(getAnchors()).toStrictEqual([BigInt(123456)]);
+    async () => {
+      expect(await getAnchors()).toStrictEqual([BigInt(123456)]);
     },
     { localStorage: { before: { userNumber: "123456" } } }
   )
@@ -18,8 +30,8 @@ test(
 test(
   "old userNumber is not deleted",
   withStorage(
-    () => {
-      getAnchors();
+    async () => {
+      await getAnchors();
     },
     {
       localStorage: {
@@ -32,11 +44,34 @@ test(
   )
 );
 
+test(
+  "old local storage anchors are not deleted",
+  withStorage(
+    async () => {
+      expect(await getAnchors()).toContain(BigInt("123456"));
+    },
+    {
+      localStorage: {
+        before: {
+          anchors: JSON.stringify({ "123456": { lastUsedTimestamp: 10 } }),
+        },
+        after: (storage) => {
+          const value = storage["anchors"];
+          expect(value).toBeDefined();
+          const anchors = JSON.parse(value);
+          expect(anchors).toBeTypeOf("object");
+          expect(anchors["123456"]).toBeDefined();
+        },
+      },
+    }
+  )
+);
+
 test(
   "reading old userNumber migrates anchors",
   withStorage(
-    () => {
-      getAnchors();
+    async () => {
+      await getAnchors();
     },
     {
       localStorage: {
@@ -49,88 +84,120 @@ test(
           expect(anchors["123456"]).toBeDefined();
         },
       },
+      indexeddb: {
+        after: (storage) => {
+          // eslint-disable-next-line @typescript-eslint/no-explicit-any
+          const anchors: any = storage["anchors"];
+          expect(anchors).toBeTypeOf("object");
+          expect(anchors["123456"]).toBeDefined();
+        },
+      },
     }
   )
 );
 
 test(
   "one anchor can be stored",
-  withStorage(() => {
-    setAnchorUsed(BigInt(10000));
-    expect(getAnchors()).toStrictEqual([BigInt(10000)]);
+  withStorage(async () => {
+    await setAnchorUsed(BigInt(10000));
+    expect(await getAnchors()).toStrictEqual([BigInt(10000)]);
   })
 );
 
 test(
   "multiple anchors can be stored",
-  withStorage(() => {
-    setAnchorUsed(BigInt(10000));
-    setAnchorUsed(BigInt(10001));
-    setAnchorUsed(BigInt(10003));
-    expect(getAnchors()).toContain(BigInt(10000));
-    expect(getAnchors()).toContain(BigInt(10001));
-    expect(getAnchors()).toContain(BigInt(10003));
+  withStorage(async () => {
+    await setAnchorUsed(BigInt(10000));
+    await setAnchorUsed(BigInt(10001));
+    await setAnchorUsed(BigInt(10003));
+    expect(await getAnchors()).toContain(BigInt(10000));
+    expect(await getAnchors()).toContain(BigInt(10001));
+    expect(await getAnchors()).toContain(BigInt(10003));
   })
 );
 
+test(
+  "anchors are also written to localstorage",
+  withStorage(
+    async () => {
+      await setAnchorUsed(BigInt(10000));
+      await setAnchorUsed(BigInt(10001));
+      await setAnchorUsed(BigInt(10003));
+    },
+    {
+      localStorage: {
+        after: (storage) => {
+          const value = storage["anchors"];
+          expect(value).toBeDefined();
+          const anchors = JSON.parse(value);
+          expect(anchors).toBeTypeOf("object");
+          expect(anchors["10000"]).toBeDefined();
+          expect(anchors["10001"]).toBeDefined();
+          expect(anchors["10003"]).toBeDefined();
+        },
+      },
+    }
+  )
+);
+
 test(
   "anchors are sorted",
-  withStorage(() => {
+  withStorage(async () => {
     const anchors = [BigInt(10400), BigInt(10001), BigInt(1011003)];
     for (const anchor of anchors) {
-      setAnchorUsed(anchor);
+      await setAnchorUsed(anchor);
     }
     anchors.sort();
-    expect(getAnchors()).toStrictEqual(anchors);
+    expect(await getAnchors()).toStrictEqual(anchors);
   })
 );
 
 test(
   "only N anchors are stored",
-  withStorage(() => {
+  withStorage(async () => {
     for (let i = 0; i < MAX_SAVED_ANCHORS + 5; i++) {
-      setAnchorUsed(BigInt(i));
+      await setAnchorUsed(BigInt(i));
     }
-    expect(getAnchors().length).toStrictEqual(MAX_SAVED_ANCHORS);
+    expect((await getAnchors()).length).toStrictEqual(MAX_SAVED_ANCHORS);
   })
 );
 
 test(
   "old anchors are dropped",
-  withStorage(() => {
+  withStorage(async () => {
     vi.useFakeTimers().setSystemTime(new Date(0));
-    setAnchorUsed(BigInt(10000));
+    await setAnchorUsed(BigInt(10000));
     vi.useFakeTimers().setSystemTime(new Date(1));
-    setAnchorUsed(BigInt(203000));
+    await setAnchorUsed(BigInt(203000));
     vi.useFakeTimers().setSystemTime(new Date(2));
     for (let i = 0; i < MAX_SAVED_ANCHORS; i++) {
-      setAnchorUsed(BigInt(i));
+      await setAnchorUsed(BigInt(i));
     }
-    expect(getAnchors()).not.toContain(BigInt(10000));
-    expect(getAnchors()).not.toContain(BigInt(203000));
+    expect(await getAnchors()).not.toContain(BigInt(10000));
+    expect(await getAnchors()).not.toContain(BigInt(203000));
     vi.useRealTimers();
   })
 );
 
 test(
   "unknown fields are not dropped",
   withStorage(
-    () => {
+    async () => {
       vi.useFakeTimers().setSystemTime(new Date(20));
-      setAnchorUsed(BigInt(10000));
+      await setAnchorUsed(BigInt(10000));
       vi.useRealTimers();
     },
     {
-      localStorage: {
+      indexeddb: {
         before: {
-          anchors: JSON.stringify({
+          anchors: {
             "10000": { lastUsedTimestamp: 10, hello: "world" },
-          }),
+          },
         },
         after: {
-          anchors: JSON.stringify({
+          anchors: {
             "10000": { lastUsedTimestamp: 20, hello: "world" },
-          }),
+          },
         },
       },
     }
@@ -144,33 +211,64 @@ test(
  * If `after` is a function, the function is called with the content of the storage.
  */
 function withStorage(
-  fn: () => void,
+  fn: () => void | Promise<void>,
   opts?: {
     localStorage?: {
       before?: LocalStorage;
       after?: LocalStorage | ((storage: LocalStorage) => void);
     };
+    indexeddb?: {
+      before?: IndexedDB;
+      after?: IndexedDB | ((storage: IndexedDB) => void);
+    };
   }
-): () => void {
-  return () => {
+): () => Promise<void> {
+  return async () => {
     localStorage.clear();
-    const before = opts?.localStorage?.before;
-    if (nonNullish(before)) {
-      setLocalStorage(before);
+    const lsBefore = opts?.localStorage?.before;
+    if (nonNullish(lsBefore)) {
+      setLocalStorage(lsBefore);
     }
-    fn();
-    const after = opts?.localStorage?.after;
-    if (nonNullish(after)) {
-      const actual: LocalStorage = readLocalStorage();
 
-      if (typeof after === "function") {
-        after(actual);
+    await idbClear();
+    const idbBefore = opts?.indexeddb?.before;
+    if (nonNullish(idbBefore)) {
+      await setIndexedDB(idbBefore);
+    }
+
+    await fn();
+
+    // Check the IndexedDB "after"
+
+    const idbAfter = opts?.indexeddb?.after;
+    if (nonNullish(idbAfter)) {
+      const actual: IndexedDB = await readIndexedDB();
+
+      if (typeof idbAfter === "function") {
+        idbAfter(actual);
       } else {
-        const expected: LocalStorage = after;
+        const expected: IndexedDB = idbAfter;
         expect(actual).toStrictEqual(expected);
       }
     }
 
+    // Remove all entries
+    // (cannot just reset global.indexeddb because idb-keyval stores a pointer to the DB)
+    await idbClear();
+
+    // Check the localStorage "after"
+
+    const lsAfter = opts?.localStorage?.after;
+    if (nonNullish(lsAfter)) {
+      const actual: LocalStorage = readLocalStorage();
+
+      if (typeof lsAfter === "function") {
+        lsAfter(actual);
+      } else {
+        const expected: LocalStorage = lsAfter;
+        expect(actual).toStrictEqual(expected);
+      }
+    }
     localStorage.clear();
   };
 }
@@ -196,3 +294,25 @@ function readLocalStorage(): LocalStorage {
 
   return ls;
 }
+
+/** Indexed DB */
+
+type IndexedDB = Record<string, unknown>;
+
+const setIndexedDB = async (db: IndexedDB) => {
+  for (const key in db) {
+    await idbSet(key, db[key]);
+  }
+};
+
+const readIndexedDB = async (): Promise<IndexedDB> => {
+  const db: IndexedDB = {};
+
+  for (const k of await idbKeys()) {
+    if (typeof k !== "string") {
+      throw new Error("Bad type");
+    }
+    db[k] = await idbGet(k);
+  }
+  return db;
+};