diff --git a/jestSetup.js b/jestSetup.js index 2288af999..453ed737f 100644 --- a/jestSetup.js +++ b/jestSetup.js @@ -1,7 +1,7 @@ jest.mock('./lib/storage'); -jest.mock('./lib/storage/NativeStorage', () => require('./lib/storage/__mocks__')); -jest.mock('./lib/storage/WebStorage', () => require('./lib/storage/__mocks__')); -jest.mock('./lib/storage/providers/IDBKeyVal', () => require('./lib/storage/__mocks__')); +jest.mock('./lib/storage/platforms/NativeStorage', () => require('./lib/storage/__mocks__')); +jest.mock('./lib/storage/platforms/WebStorage', () => require('./lib/storage/__mocks__')); +jest.mock('./lib/storage/providers/IDBKeyValProvider', () => require('./lib/storage/__mocks__')); jest.mock('react-native-device-info', () => ({getFreeDiskStorage: () => {}})); jest.mock('react-native-quick-sqlite', () => ({ diff --git a/lib/Onyx.js b/lib/Onyx.js index d459bc546..bd178306f 100644 --- a/lib/Onyx.js +++ b/lib/Onyx.js @@ -1563,6 +1563,16 @@ function init({ shouldSyncMultipleInstances = Boolean(global.localStorage), debugSetState = false, } = {}) { + Storage.init(); + + if (shouldSyncMultipleInstances) { + Storage.keepInstancesSync((key, value) => { + const prevValue = cache.getValue(key, false); + cache.set(key, value); + keyChanged(key, value, prevValue); + }); + } + if (captureMetrics) { // The code here is only bundled and applied when the captureMetrics is set // eslint-disable-next-line no-use-before-define @@ -1599,14 +1609,6 @@ function init({ // Initialize all of our keys with data provided then give green light to any pending connections Promise.all([addAllSafeEvictionKeysToRecentlyAccessedList(), initializeWithDefaultKeyStates()]).then(deferredInitTask.resolve); - - if (shouldSyncMultipleInstances && _.isFunction(Storage.keepInstancesSync)) { - Storage.keepInstancesSync((key, value) => { - const prevValue = cache.getValue(key, false); - cache.set(key, value); - keyChanged(key, value, prevValue); - }); - } } const Onyx = { diff --git a/lib/storage/InstanceSync/index.js b/lib/storage/InstanceSync/index.js new file mode 100644 index 000000000..a31564523 --- /dev/null +++ b/lib/storage/InstanceSync/index.js @@ -0,0 +1,7 @@ +/** + * This is used to keep multiple browser tabs in sync, therefore only needed on web + * On native platforms, we omit this syncing logic by setting this to null. + */ +const InstanceSync = null; + +export default InstanceSync; diff --git a/lib/storage/InstanceSync/index.web.js b/lib/storage/InstanceSync/index.web.js new file mode 100644 index 000000000..f333051c9 --- /dev/null +++ b/lib/storage/InstanceSync/index.web.js @@ -0,0 +1,63 @@ +/* eslint-disable no-invalid-this */ +/** + * The InstancesSync object provides data-changed events like the ones that exist + * when using LocalStorage APIs in the browser. These events are great because multiple tabs can listen for when + * data changes and then stay up-to-date with everything happening in Onyx. + */ +import _ from 'underscore'; + +const SYNC_ONYX = 'SYNC_ONYX'; + +/** + * Raise an event through `localStorage` to let other tabs know a value changed + * @param {String} onyxKey + */ +function raiseStorageSyncEvent(onyxKey) { + global.localStorage.setItem(SYNC_ONYX, onyxKey); + global.localStorage.removeItem(SYNC_ONYX, onyxKey); +} + +function raiseStorageSyncManyKeysEvent(onyxKeys) { + _.each(onyxKeys, (onyxKey) => { + raiseStorageSyncEvent(onyxKey); + }); +} + +const InstanceSync = { + /** + * @param {Function} onStorageKeyChanged Storage synchronization mechanism keeping all opened tabs in sync + */ + init: (onStorageKeyChanged) => { + // This listener will only be triggered by events coming from other tabs + global.addEventListener('storage', (event) => { + // Ignore events that don't originate from the SYNC_ONYX logic + if (event.key !== SYNC_ONYX || !event.newValue) { + return; + } + + const onyxKey = event.newValue; + this.getItem(onyxKey).then((value) => onStorageKeyChanged(onyxKey, value)); + }); + }, + setItem: raiseStorageSyncEvent, + removeItem: raiseStorageSyncEvent, + removeItems: raiseStorageSyncManyKeysEvent, + mergeItem: raiseStorageSyncEvent, + clear: (clearImplementation) => { + let allKeys; + + // The keys must be retrieved before storage is cleared or else the list of keys would be empty + return this.getAllKeys() + .then((keys) => { + allKeys = keys; + }) + .then(() => clearImplementation()) + .then(() => { + // Now that storage is cleared, the storage sync event can happen which is a more atomic action + // for other browser tabs + raiseStorageSyncManyKeysEvent(allKeys) + }); + }, +}; + +export default InstanceSync; diff --git a/lib/storage/NativeStorage.js b/lib/storage/NativeStorage.js deleted file mode 100644 index 1473613fa..000000000 --- a/lib/storage/NativeStorage.js +++ /dev/null @@ -1,3 +0,0 @@ -import SQLiteStorage from './providers/SQLiteStorage'; - -export default SQLiteStorage; diff --git a/lib/storage/WebStorage.js b/lib/storage/WebStorage.js deleted file mode 100644 index 4b78bb2df..000000000 --- a/lib/storage/WebStorage.js +++ /dev/null @@ -1,74 +0,0 @@ -/** - * This file is here to wrap IDBKeyVal with a layer that provides data-changed events like the ones that exist - * when using LocalStorage APIs in the browser. These events are great because multiple tabs can listen for when - * data changes and then stay up-to-date with everything happening in Onyx. - */ -import _ from 'underscore'; -import Storage from './providers/IDBKeyVal'; - -const SYNC_ONYX = 'SYNC_ONYX'; - -/** - * Raise an event thorough `localStorage` to let other tabs know a value changed - * @param {String} onyxKey - */ -function raiseStorageSyncEvent(onyxKey) { - global.localStorage.setItem(SYNC_ONYX, onyxKey); - global.localStorage.removeItem(SYNC_ONYX, onyxKey); -} - -function raiseStorageSyncManyKeysEvent(onyxKeys) { - _.each(onyxKeys, (onyxKey) => { - raiseStorageSyncEvent(onyxKey); - }); -} - -const webStorage = { - ...Storage, - - /** - * @param {Function} onStorageKeyChanged Storage synchronization mechanism keeping all opened tabs in sync - */ - keepInstancesSync(onStorageKeyChanged) { - // Override set, remove and clear to raise storage events that we intercept in other tabs - this.setItem = (key, value) => Storage.setItem(key, value).then(() => raiseStorageSyncEvent(key)); - - this.removeItem = (key) => Storage.removeItem(key).then(() => raiseStorageSyncEvent(key)); - - this.removeItems = (keys) => Storage.removeItems(keys).then(() => raiseStorageSyncManyKeysEvent(keys)); - - this.mergeItem = (key, batchedChanges, modifiedData) => Storage.mergeItem(key, batchedChanges, modifiedData).then(() => raiseStorageSyncEvent(key)); - - // If we just call Storage.clear other tabs will have no idea which keys were available previously - // so that they can call keysChanged for them. That's why we iterate over every key and raise a storage sync - // event for each one - this.clear = () => { - let allKeys; - - // The keys must be retrieved before storage is cleared or else the list of keys would be empty - return Storage.getAllKeys() - .then((keys) => { - allKeys = keys; - }) - .then(() => Storage.clear()) - .then(() => { - // Now that storage is cleared, the storage sync event can happen which is a more atomic action - // for other browser tabs - _.each(allKeys, raiseStorageSyncEvent); - }); - }; - - // This listener will only be triggered by events coming from other tabs - global.addEventListener('storage', (event) => { - // Ignore events that don't originate from the SYNC_ONYX logic - if (event.key !== SYNC_ONYX || !event.newValue) { - return; - } - - const onyxKey = event.newValue; - Storage.getItem(onyxKey).then((value) => onStorageKeyChanged(onyxKey, value)); - }); - }, -}; - -export default webStorage; diff --git a/lib/storage/__mocks__/index.js b/lib/storage/__mocks__/index.js index 6ed13c52d..41b6c208f 100644 --- a/lib/storage/__mocks__/index.js +++ b/lib/storage/__mocks__/index.js @@ -9,6 +9,7 @@ const set = jest.fn((key, value) => { }); const idbKeyvalMock = { + init: () => {}, setItem(key, value) { return set(key, value); }, @@ -58,10 +59,12 @@ const idbKeyvalMock = { return Promise.resolve({bytesRemaining: 0, bytesUsed: 99999}); }, setMemoryOnlyKeys() {}, + keepInstancesSync: jest.fn(), }; const idbKeyvalMockSpy = { idbKeyvalSet: set, + init: jest.fn(idbKeyvalMock.init), setItem: jest.fn(idbKeyvalMock.setItem), getItem: jest.fn(idbKeyvalMock.getItem), removeItem: jest.fn(idbKeyvalMock.removeItem), @@ -79,6 +82,7 @@ const idbKeyvalMockSpy = { }), getDatabaseSize: jest.fn(idbKeyvalMock.getDatabaseSize), setMemoryOnlyKeys: jest.fn(idbKeyvalMock.setMemoryOnlyKeys), + keepInstancesSync: jest.fn(idbKeyvalMock.keepInstancesSync), }; export default idbKeyvalMockSpy; diff --git a/lib/storage/index.js b/lib/storage/index.js index 4ee520d20..54cbfdbe2 100644 --- a/lib/storage/index.js +++ b/lib/storage/index.js @@ -1,3 +1,158 @@ -import WebStorage from './WebStorage'; +import PlatformStorage from './platforms'; +import InstanceSync from './InstanceSync'; -export default WebStorage; +const provider = PlatformStorage; +let shouldKeepInstancesSync = false; + + +const Storage = { + /** + * Returns the storage provider currently in use + * @returns {Object} the current storage provider + */ + getStorageProvider() { + return provider; + }, + + /** + * Initializes all providers in the list of storage providers + * and enables fallback providers if necessary + */ + init() { + provider.init(); + }, + + /** + * Get the value of a given key or return `null` if it's not available in memory + * @param {String} key + * @return {Promise<*>} + */ + getItem: (key) => provider.getItem(key), + + /** + * Get multiple key-value pairs for the give array of keys in a batch. + * @param {String[]} keys + * @return {Promise>} + */ + multiGet: (keys) => provider.multiGet(keys), + + /** + * Sets the value for a given key. The only requirement is that the value should be serializable to JSON string + * @param {String} key + * @param {*} value + * @return {Promise} + */ + setItem: (key, value) => { + const promise = provider.setItem(key, value); + + if (shouldKeepInstancesSync) { + return promise.then(() => InstanceSync.setItem(key)); + } + + return promise; + }, + + /** + * Stores multiple key-value pairs in a batch + * @param {Array<[key, value]>} pairs + * @return {Promise} + */ + multiSet: (pairs) => provider.multiSet(pairs), + + /** + * Merging an existing value with a new one + * @param {String} key + * @param {*} changes - the delta for a specific key + * @param {any} modifiedData - the pre-merged data from `Onyx.applyMerge` + * @return {Promise} + */ + mergeItem: (key, changes, modifiedData) => { + const promise = provider.mergeItem(key, changes, modifiedData); + + if (shouldKeepInstancesSync) { + return promise.then(() => InstanceSync.mergeItem(key)); + } + + return promise; + }, + + /** + * Multiple merging of existing and new values in a batch + * @param {Array<[key, value]>} pairs + * This function also removes all nested null values from an object. + * @return {Promise} + */ + multiMerge: (pairs) => provider.multiMerge(pairs), + + /** + * Remove given key and it's value from memory + * @param {String} key + * @returns {Promise} + */ + removeItem: (key) => { + const promise = provider.removeItem(key); + + if (shouldKeepInstancesSync) { + return promise.then(() => InstanceSync.removeItem(key)); + } + + return promise; + }, + + /** + * Remove given keys and their values from memory + * + * @param {Array} keys + * @returns {Promise} + */ + removeItems: (keys) => { + const promise = provider.removeItems(keys); + + if (shouldKeepInstancesSync) { + return promise.then(() => InstanceSync.removeItems(keys)); + } + + return promise; + }, + + /** + * Clear everything from memory + * @returns {Promise} + */ + clear: () => { + if (shouldKeepInstancesSync) { + return InstanceSync.clear(() => provider.clear()); + } + + return provider.clear(); + }, + + // This is a noop for now in order to keep clients from crashing see https://github.com/Expensify/Expensify/issues/312438 + setMemoryOnlyKeys: () => provider.setMemoryOnlyKeys(), + + /** + * Returns all keys available in memory + * @returns {Promise} + */ + getAllKeys: () => provider.getAllKeys(), + + /** + * Gets the total bytes of the store. + * `bytesRemaining` will always be `Number.POSITIVE_INFINITY` since we don't have a hard limit on memory. + * @returns {Promise} + */ + getDatabaseSize: () => provider.getDatabaseSize(), + + /** + * @param {Function} onStorageKeyChanged Storage synchronization mechanism keeping all opened tabs in sync (web only) + */ + keepInstancesSync(onStorageKeyChanged) { + // If InstanceSync is null, it means we're on a native platform and we don't need to keep instances in sync + if (InstanceSync == null) return; + + shouldKeepInstancesSync = true; + InstanceSync.init(onStorageKeyChanged); + }, +}; + +export default Storage; \ No newline at end of file diff --git a/lib/storage/platforms/NativeStorage.js b/lib/storage/platforms/NativeStorage.js new file mode 100644 index 000000000..3ca4e4d98 --- /dev/null +++ b/lib/storage/platforms/NativeStorage.js @@ -0,0 +1,3 @@ +import SQLiteStorage from '../providers/SQLiteProvider'; + +export default SQLiteStorage; diff --git a/lib/storage/platforms/WebStorage.js b/lib/storage/platforms/WebStorage.js new file mode 100644 index 000000000..c24f98eb1 --- /dev/null +++ b/lib/storage/platforms/WebStorage.js @@ -0,0 +1,3 @@ +import IDBKeyValProvider from '../providers/IDBKeyValProvider'; + +export default IDBKeyValProvider; \ No newline at end of file diff --git a/lib/storage/index.native.js b/lib/storage/platforms/index.native.js similarity index 100% rename from lib/storage/index.native.js rename to lib/storage/platforms/index.native.js diff --git a/lib/storage/platforms/index.web.js b/lib/storage/platforms/index.web.js new file mode 100644 index 000000000..4ee520d20 --- /dev/null +++ b/lib/storage/platforms/index.web.js @@ -0,0 +1,3 @@ +import WebStorage from './WebStorage'; + +export default WebStorage; diff --git a/lib/storage/providers/IDBKeyVal.js b/lib/storage/providers/IDBKeyValProvider.js similarity index 83% rename from lib/storage/providers/IDBKeyVal.js rename to lib/storage/providers/IDBKeyValProvider.js index 4e7dd0f9a..9c01dc5cc 100644 --- a/lib/storage/providers/IDBKeyVal.js +++ b/lib/storage/providers/IDBKeyValProvider.js @@ -4,22 +4,27 @@ import utils from '../../utils'; // We don't want to initialize the store while the JS bundle loads as idb-keyval will try to use global.indexedDB // which might not be available in certain environments that load the bundle (e.g. electron main process). -let customStoreInstance; -const getCustomStore = () => { - if (!customStoreInstance) { - customStoreInstance = createStore('OnyxDB', 'keyvaluepairs'); - } - return customStoreInstance; -}; +let idbKeyValStore; const provider = { + /** + * Initializes the storage provider + */ + init() { + const newIdbKeyValStore = createStore('OnyxDB', 'keyvaluepairs'); + + if (newIdbKeyValStore == null) throw Error('IDBKeyVal store could not be created'); + + idbKeyValStore = newIdbKeyValStore; + }, + /** * Sets the value for a given key. The only requirement is that the value should be serializable to JSON string * @param {String} key * @param {*} value * @return {Promise} */ - setItem: (key, value) => set(key, value, getCustomStore()), + setItem: (key, value) => set(key, value, idbKeyValStore), /** * Get multiple key-value pairs for the give array of keys in a batch. @@ -27,7 +32,7 @@ const provider = { * @param {String[]} keysParam * @return {Promise>} */ - multiGet: (keysParam) => getMany(keysParam, getCustomStore()).then((values) => _.map(values, (value, index) => [keysParam[index], value])), + multiGet: (keysParam) => getMany(keysParam, idbKeyValStore).then((values) => _.map(values, (value, index) => [keysParam[index], value])), /** * Multiple merging of existing and new values in a batch @@ -36,7 +41,7 @@ const provider = { * @return {Promise} */ multiMerge: (pairs) => - getCustomStore()('readwrite', (store) => { + idbKeyValStore('readwrite', (store) => { // Note: we are using the manual store transaction here, to fit the read and update // of the items in one transaction to achieve best performance. @@ -69,13 +74,13 @@ const provider = { * @param {Array<[key, value]>} pairs * @return {Promise} */ - multiSet: (pairs) => setMany(pairs, getCustomStore()), + multiSet: (pairs) => setMany(pairs, idbKeyValStore), /** * Clear everything from storage and also stops the SyncQueue from adding anything more to storage * @returns {Promise} */ - clear: () => clear(getCustomStore()), + clear: () => clear(idbKeyValStore), // This is a noop for now in order to keep clients from crashing see https://github.com/Expensify/Expensify/issues/312438 setMemoryOnlyKeys: () => {}, @@ -84,7 +89,7 @@ const provider = { * Returns all keys available in storage * @returns {Promise} */ - getAllKeys: () => keys(getCustomStore()), + getAllKeys: () => keys(idbKeyValStore), /** * Get the value of a given key or return `null` if it's not available in storage @@ -92,7 +97,7 @@ const provider = { * @return {Promise<*>} */ getItem: (key) => - get(key, getCustomStore()) + get(key, idbKeyValStore) // idb-keyval returns undefined for missing items, but this needs to return null so that idb-keyval does the same thing as SQLiteStorage. .then((val) => (val === undefined ? null : val)), @@ -101,7 +106,7 @@ const provider = { * @param {String} key * @returns {Promise} */ - removeItem: (key) => del(key, getCustomStore()), + removeItem: (key) => del(key, idbKeyValStore), /** * Remove given keys and their values from storage @@ -109,7 +114,7 @@ const provider = { * @param {Array} keysParam * @returns {Promise} */ - removeItems: (keysParam) => delMany(keysParam, getCustomStore()), + removeItems: (keysParam) => delMany(keysParam, idbKeyValStore), /** * Gets the total bytes of the database file diff --git a/lib/storage/providers/SQLiteStorage.js b/lib/storage/providers/SQLiteProvider.js similarity index 90% rename from lib/storage/providers/SQLiteStorage.js rename to lib/storage/providers/SQLiteProvider.js index 1bfe60bf0..fa75337c4 100644 --- a/lib/storage/providers/SQLiteStorage.js +++ b/lib/storage/providers/SQLiteProvider.js @@ -7,17 +7,24 @@ import {getFreeDiskStorage} from 'react-native-device-info'; import _ from 'underscore'; const DB_NAME = 'OnyxDB'; -const db = open({name: DB_NAME}); -db.execute('CREATE TABLE IF NOT EXISTS keyvaluepairs (record_key TEXT NOT NULL PRIMARY KEY , valueJSON JSON NOT NULL) WITHOUT ROWID;'); - -// All of the 3 pragmas below were suggested by SQLite team. -// You can find more info about them here: https://www.sqlite.org/pragma.html -db.execute('PRAGMA CACHE_SIZE=-20000;'); -db.execute('PRAGMA synchronous=NORMAL;'); -db.execute('PRAGMA journal_mode=WAL;'); +let db; const provider = { + /** + * Initializes the storage provider + */ + init() { + db = open({name: DB_NAME}); + + db.execute('CREATE TABLE IF NOT EXISTS keyvaluepairs (record_key TEXT NOT NULL PRIMARY KEY , valueJSON JSON NOT NULL) WITHOUT ROWID;'); + + // All of the 3 pragmas below were suggested by SQLite team. + // You can find more info about them here: https://www.sqlite.org/pragma.html + db.execute('PRAGMA CACHE_SIZE=-20000;'); + db.execute('PRAGMA synchronous=NORMAL;'); + db.execute('PRAGMA journal_mode=WAL;'); + }, /** * Get the value of a given key or return `null` if it's not available in storage * @param {String} key @@ -159,11 +166,6 @@ const provider = { }; }); }, - - /** - * Noop on native - */ - keepInstancesSync: () => {}, }; export default provider; diff --git a/tests/unit/storage/providers/IDBKeyvalProviderTest.js b/tests/unit/storage/providers/IDBKeyvalProviderTest.js index ea4f5601a..9fa895d51 100644 --- a/tests/unit/storage/providers/IDBKeyvalProviderTest.js +++ b/tests/unit/storage/providers/IDBKeyvalProviderTest.js @@ -1,6 +1,6 @@ import _ from 'underscore'; -import IDBKeyValProviderMock from '../../../../lib/storage/providers/IDBKeyVal'; +import IDBKeyValProviderMock from '../../../../lib/storage/providers/IDBKeyValProvider'; import createDeferredTask from '../../../../lib/createDeferredTask'; import waitForPromisesToResolve from '../../../utils/waitForPromisesToResolve'; diff --git a/tests/unit/storage/providers/StorageProviderTest.js b/tests/unit/storage/providers/StorageProviderTest.js index 82aca46b5..2be2647c7 100644 --- a/tests/unit/storage/providers/StorageProviderTest.js +++ b/tests/unit/storage/providers/StorageProviderTest.js @@ -1,11 +1,11 @@ /* eslint-disable import/first */ -jest.unmock('../../../../lib/storage/NativeStorage'); -jest.unmock('../../../../lib/storage/WebStorage'); -jest.unmock('../../../../lib/storage/providers/IDBKeyVal'); +jest.unmock('../../../../lib/storage/platforms/NativeStorage'); +jest.unmock('../../../../lib/storage/platforms/WebStorage'); +jest.unmock('../../../../lib/storage/providers/IDBKeyValProvider'); import _ from 'underscore'; -import NativeStorage from '../../../../lib/storage/NativeStorage'; -import WebStorage from '../../../../lib/storage/WebStorage'; +import NativeStorage from '../../../../lib/storage/platforms/NativeStorage'; +import WebStorage from '../../../../lib/storage/platforms/WebStorage'; it('storage providers have same methods implemented', () => { const nativeMethods = _.keys(NativeStorage);