Merge pull request #483 from margelo/feat/split-up-in-memory-pr-part-2

feat: fallback to NoopProvider if we run into OOM [2/3]

lib/storage/__mocks__/index.ts

@@ -10,6 +10,7 @@ const set = jest.fn((key, value) => {
 });
 
 const idbKeyvalMock: StorageProvider = {
+    name: 'KeyValMockProvider',
     init: () => undefined,
     setItem(key, value) {
         return set(key, value);

lib/storage/index.ts

@@ -1,13 +1,52 @@
+import * as Logger from '../Logger';
+
 import PlatformStorage from './platforms';
 import InstanceSync from './InstanceSync';
+import NoopProvider from './providers/NoopProvider';
 import type StorageProvider from './providers/types';
 
-const provider = PlatformStorage;
+let provider = PlatformStorage;
 let shouldKeepInstancesSync = false;
+let finishInitalization: (value?: unknown) => void;
+const initPromise = new Promise((resolve) => {
+    finishInitalization = resolve;
+});
 
 type Storage = {
     getStorageProvider: () => StorageProvider;
-} & StorageProvider;
+} & Omit<StorageProvider, 'name'>;
+
+/**
+ * Degrade performance by removing the storage provider and only using cache
+ */
+function degradePerformance(error: Error) {
+    Logger.logAlert(`Error while using ${provider.name}. Falling back to only using cache and dropping storage.`);
+    console.error(error);
+    provider = NoopProvider;
+}
+
+/**
+ * Runs a piece of code and degrades performance if certain errors are thrown
+ */
+function tryOrDegradePerformance<T>(fn: () => Promise<T> | T): Promise<T> {
+    return new Promise<T>((resolve, reject) => {
+        initPromise.then(() => {
+            try {
+                resolve(fn());
+            } catch (error) {
+                // Test for known critical errors that the storage provider throws, e.g. when storage is full
+                if (error instanceof Error) {
+                    // IndexedDB error when storage is full (https://github.com/Expensify/App/issues/29403)
+                    if (error.message.includes('Internal error opening backing store for indexedDB.open')) {
+                        degradePerformance(error);
+                    }
+                }
+
+                reject(error);
+            }
+        });
+    });
+}
 
 const Storage: Storage = {
     /**
@@ -22,112 +61,119 @@ const Storage: Storage = {
      * and enables fallback providers if necessary
      */
     init() {
-        provider.init();
+        tryOrDegradePerformance(provider.init).finally(() => {
+            finishInitalization();
+        });
     },
 
     /**
      * Get the value of a given key or return `null` if it's not available
      */
-    getItem: (key) => provider.getItem(key),
+    getItem: (key) => tryOrDegradePerformance(() => provider.getItem(key)),
 
     /**
      * Get multiple key-value pairs for the give array of keys in a batch
      */
-    multiGet: (keys) => provider.multiGet(keys),
+    multiGet: (keys) => tryOrDegradePerformance(() => provider.multiGet(keys)),
 
     /**
      * Sets the value for a given key. The only requirement is that the value should be serializable to JSON string
      */
-    setItem: (key, value) => {
-        const promise = provider.setItem(key, value);
+    setItem: (key, value) =>
+        tryOrDegradePerformance(() => {
+            const promise = provider.setItem(key, value);
 
-        if (shouldKeepInstancesSync) {
-            return promise.then(() => InstanceSync.setItem(key));
-        }
+            if (shouldKeepInstancesSync) {
+                return promise.then(() => InstanceSync.setItem(key));
+            }
 
-        return promise;
-    },
+            return promise;
+        }),
 
     /**
      * Stores multiple key-value pairs in a batch
      */
-    multiSet: (pairs) => provider.multiSet(pairs),
+    multiSet: (pairs) => tryOrDegradePerformance(() => provider.multiSet(pairs)),
 
     /**
      * Merging an existing value with a new one
      */
-    mergeItem: (key, changes, modifiedData) => {
-        const promise = provider.mergeItem(key, changes, modifiedData);
+    mergeItem: (key, changes, modifiedData) =>
+        tryOrDegradePerformance(() => {
+            const promise = provider.mergeItem(key, changes, modifiedData);
 
-        if (shouldKeepInstancesSync) {
-            return promise.then(() => InstanceSync.mergeItem(key));
-        }
+            if (shouldKeepInstancesSync) {
+                return promise.then(() => InstanceSync.mergeItem(key));
+            }
 
-        return promise;
-    },
+            return promise;
+        }),
 
     /**
      * Multiple merging of existing and new values in a batch
      * This function also removes all nested null values from an object.
      */
-    multiMerge: (pairs) => provider.multiMerge(pairs),
+    multiMerge: (pairs) => tryOrDegradePerformance(() => provider.multiMerge(pairs)),
 
     /**
      * Removes given key and its value
      */
-    removeItem: (key) => {
-        const promise = provider.removeItem(key);
+    removeItem: (key) =>
+        tryOrDegradePerformance(() => {
+            const promise = provider.removeItem(key);
 
-        if (shouldKeepInstancesSync) {
-            return promise.then(() => InstanceSync.removeItem(key));
-        }
+            if (shouldKeepInstancesSync) {
+                return promise.then(() => InstanceSync.removeItem(key));
+            }
 
-        return promise;
-    },
+            return promise;
+        }),
 
     /**
      * Remove given keys and their values
      */
-    removeItems: (keys) => {
-        const promise = provider.removeItems(keys);
+    removeItems: (keys) =>
+        tryOrDegradePerformance(() => {
+            const promise = provider.removeItems(keys);
 
-        if (shouldKeepInstancesSync) {
-            return promise.then(() => InstanceSync.removeItems(keys));
-        }
+            if (shouldKeepInstancesSync) {
+                return promise.then(() => InstanceSync.removeItems(keys));
+            }
 
-        return promise;
-    },
+            return promise;
+        }),
 
     /**
      * Clears everything
      */
-    clear: () => {
-        if (shouldKeepInstancesSync) {
-            return InstanceSync.clear(() => provider.clear());
-        }
+    clear: () =>
+        tryOrDegradePerformance(() => {
+            if (shouldKeepInstancesSync) {
+                return InstanceSync.clear(() => provider.clear());
+            }
 
-        return provider.clear();
-    },
+            return provider.clear();
+        }),
 
     // This is a noop for now in order to keep clients from crashing see https://github.com/Expensify/Expensify/issues/312438
-    setMemoryOnlyKeys: () => provider.setMemoryOnlyKeys(),
+    setMemoryOnlyKeys: () => tryOrDegradePerformance(() => provider.setMemoryOnlyKeys()),
 
     /**
      * Returns all available keys
      */
-    getAllKeys: () => provider.getAllKeys(),
+    getAllKeys: () => tryOrDegradePerformance(() => provider.getAllKeys()),
 
     /**
      * Gets the total bytes of the store
      */
-    getDatabaseSize: () => provider.getDatabaseSize(),
+    getDatabaseSize: () => tryOrDegradePerformance(() => provider.getDatabaseSize()),
 
     /**
      * @param onStorageKeyChanged - Storage synchronization mechanism keeping all opened tabs in sync (web only)
      */
     keepInstancesSync(onStorageKeyChanged) {
         // If InstanceSync is null, it means we're on a native platform and we don't need to keep instances in sync
-        if (InstanceSync == null) return;
+        if (InstanceSync === null) return;
 
         shouldKeepInstancesSync = true;
         InstanceSync.init(onStorageKeyChanged);

lib/storage/providers/IDBKeyValProvider.ts

@@ -9,6 +9,10 @@ import type {Value} from './types';
 let idbKeyValStore: UseStore;
 
 const provider: StorageProvider = {
+    /**
+     * The name of the provider that can be printed to the logs
+     */
+    name: 'IDBKeyValProvider',
     /**
      * Initializes the storage provider
      */

lib/storage/providers/NoopProvider.ts

@@ -0,0 +1,103 @@
+import type StorageProvider from './types';
+
+const provider: StorageProvider = {
+    /**
+     * The name of the provider that can be printed to the logs
+     */
+    name: 'NoopProvider',
+
+    /**
+     * Initializes the storage provider
+     */
+    init() {
+        // do nothing
+    },
+
+    /**
+     * Get the value of a given key or return `null` if it's not available in memory
+     * @param {String} key
+     * @return {Promise<*>}
+     */
+    getItem() {
+        return Promise.resolve(null);
+    },
+
+    /**
+     * Get multiple key-value pairs for the give array of keys in a batch.
+     */
+    multiGet() {
+        return Promise.resolve([]);
+    },
+
+    /**
+     * Sets the value for a given key. The only requirement is that the value should be serializable to JSON string
+     */
+    setItem() {
+        return Promise.resolve();
+    },
+
+    /**
+     * Stores multiple key-value pairs in a batch
+     */
+    multiSet() {
+        return Promise.resolve();
+    },
+
+    /**
+     * Merging an existing value with a new one
+     */
+    mergeItem() {
+        return Promise.resolve();
+    },
+
+    /**
+     * Multiple merging of existing and new values in a batch
+     * This function also removes all nested null values from an object.
+     */
+    multiMerge() {
+        return Promise.resolve([]);
+    },
+
+    /**
+     * Remove given key and it's value from memory
+     */
+    removeItem() {
+        return Promise.resolve();
+    },
+
+    /**
+     * Remove given keys and their values from memory
+     */
+    removeItems() {
+        return Promise.resolve();
+    },
+
+    /**
+     * Clear everything from memory
+     */
+    clear() {
+        return Promise.resolve();
+    },
+
+    // This is a noop for now in order to keep clients from crashing see https://github.com/Expensify/Expensify/issues/312438
+    setMemoryOnlyKeys() {
+        // do nothing
+    },
+
+    /**
+     * Returns all keys available in memory
+     */
+    getAllKeys() {
+        return Promise.resolve([]);
+    },
+
+    /**
+     * Gets the total bytes of the store.
+     * `bytesRemaining` will always be `Number.POSITIVE_INFINITY` since we don't have a hard limit on memory.
+     */
+    getDatabaseSize() {
+        return Promise.resolve({bytesRemaining: Number.POSITIVE_INFINITY, bytesUsed: 0});
+    },
+};
+
+export default provider;

lib/storage/providers/SQLiteProvider.ts

@@ -13,6 +13,10 @@ const DB_NAME = 'OnyxDB';
 let db: QuickSQLiteConnection;
 
 const provider: StorageProvider = {
+    /**
+     * The name of the provider that can be printed to the logs
+     */
+    name: 'SQLiteProvider',
     /**
      * Initializes the storage provider
      */

lib/storage/providers/types.ts

@@ -9,6 +9,10 @@ type KeyValuePairList = KeyValuePair[];
 type OnStorageKeyChanged = (key: Key, value: Value | null) => void;
 
 type StorageProvider = {
+    /**
+     * The name of the provider that can be printed to the logs
+     */
+    name: string;
     /**
      * Initializes the storage provider
      */
@@ -82,4 +86,4 @@ type StorageProvider = {
 };
 
 export default StorageProvider;
-export type {Value, Key, KeyList, KeyValuePairList, OnStorageKeyChanged};
+export type {Value, Key, KeyList, KeyValuePair, KeyValuePairList, OnStorageKeyChanged};