fix: adjust storage options for MapeoManager and MapeoProject

package.json

@@ -111,7 +111,7 @@
     "hyperdrive": "^11.5.3",
     "hyperswarm": "^4.4.1",
     "magic-bytes.js": "^1.0.14",
-    "multi-core-indexer": "^1.0.0-alpha.4",
+    "multi-core-indexer": "^1.0.0-alpha.7",
     "multicast-service-discovery": "^4.0.4",
     "p-defer": "^4.0.0",
     "private-ip": "^3.0.0",

src/mapeo-manager.js

@@ -5,36 +5,47 @@ import Database from 'better-sqlite3'
 import { eq } from 'drizzle-orm'
 import { drizzle } from 'drizzle-orm/better-sqlite3'
 import { migrate } from 'drizzle-orm/better-sqlite3/migrator'
+import Hypercore from 'hypercore'
 import { IndexWriter } from './index-writer/index.js'
 import { MapeoProject } from './mapeo-project.js'
 import { projectKeysTable, projectTable } from './schema/client.js'
 import { ProjectKeys } from './generated/keys.js'
 import { deNullify } from './utils.js'
+import { RandomAccessFilePool } from './core-manager/random-access-file-pool.js'
 
 /** @typedef {import("@mapeo/schema").ProjectValue} ProjectValue */
 
 const CLIENT_SQLITE_FILE_NAME = 'client.db'
 
+// Max file descriptors that RandomAccessFile should use for hypercore storage
+// and index bitfield persistence (used by MultiCoreIndexer). Android has a
+// limit of 1024 per process, so choosing 768 to leave 256 descriptors free for
+// other things e.g. SQLite and other parts of the app.
+const MAX_FILE_DESCRIPTORS = 768
+
 export class MapeoManager {
   #keyManager
   #projectSettingsIndexWriter
-  #storagePath
   #db
   /** @type {Map<string, MapeoProject>} */
   #activeProjects
+  /** @type {import('./types.js').CoreStorage} */
+  #coreStorage
+  #dbFolder
 
   /**
    * @param {Object} opts
    * @param {Buffer} opts.rootKey 16-bytes of random data that uniquely identify the device, used to derive a 32-byte master key, which is used to derive all the keypairs used for Mapeo
-   * @param {string} [opts.storagePath] Folder for all data storage (hypercores and sqlite db). Folder must exist. If not defined, everything is stored in-memory
+   * @param {string} opts.dbFolder Folder for sqlite Dbs. Folder must exist. Use ':memory:' to store everything in-memory
+   * @param {string | import('./types.js').CoreStorage} opts.coreStorage Folder for hypercore storage or a function that returns a RandomAccessStorage instance
    */
-  constructor({ rootKey, storagePath }) {
-    const dbPath =
-      storagePath !== undefined
-        ? path.join(storagePath, CLIENT_SQLITE_FILE_NAME)
-        : ':memory:'
-
-    const sqlite = new Database(dbPath)
+  constructor({ rootKey, dbFolder, coreStorage }) {
+    this.#dbFolder = dbFolder
+    const sqlite = new Database(
+      dbFolder === ':memory:'
+        ? ':memory:'
+        : path.join(dbFolder, CLIENT_SQLITE_FILE_NAME)
+    )
     this.#db = drizzle(sqlite)
     migrate(this.#db, { migrationsFolder: './drizzle/client' })
 
@@ -43,8 +54,15 @@ export class MapeoManager {
       tables: [projectTable],
       sqlite,
     })
-    this.#storagePath = storagePath
     this.#activeProjects = new Map()
+
+    if (typeof coreStorage === 'string') {
+      const pool = new RandomAccessFilePool(MAX_FILE_DESCRIPTORS)
+      // @ts-ignore
+      this.#coreStorage = Hypercore.createStorage(coreStorage, { pool })
+    } else {
+      this.#coreStorage = coreStorage
+    }
   }
 
   /**
@@ -58,19 +76,34 @@ export class MapeoManager {
     )
   }
 
+  /**
+   * @param {string} projectId
+   * @returns {Pick<ConstructorParameters<typeof MapeoProject>[0], 'dbPath' | 'coreStorage'>}
+   */
+  #projectStorage(projectId) {
+    return {
+      dbPath:
+        this.#dbFolder === ':memory:'
+          ? ':memory:'
+          : path.join(this.#dbFolder, projectId + '.db'),
+      coreStorage: (name) => this.#coreStorage(path.join(projectId, name)),
+    }
+  }
+
   /**
    * @param {Object} opts
    * @param {string} opts.projectId
    * @param {ProjectKeys} opts.projectKeys
    * @param {import('./generated/rpc.js').Invite_ProjectInfo} [opts.projectInfo]
    */
   #saveToProjectKeysTable({ projectId, projectKeys, projectInfo }) {
+    const encoded = ProjectKeys.encode(projectKeys).finish()
     this.#db
       .insert(projectKeysTable)
       .values({
         projectId,
         keysCipher: this.#keyManager.encryptLocalMessage(
-          Buffer.from(ProjectKeys.encode(projectKeys).finish().buffer),
+          Buffer.from(encoded.buffer, encoded.byteOffset, encoded.byteLength),
           projectId
         ),
         projectInfo,
@@ -112,7 +145,7 @@ export class MapeoManager {
 
     // 4. Create MapeoProject instance
     const project = new MapeoProject({
-      storagePath: this.#storagePath,
+      ...this.#projectStorage(projectId),
       encryptionKeys,
       keyManager: this.#keyManager,
       projectKey: projectKeypair.publicKey,
@@ -161,8 +194,8 @@ export class MapeoManager {
     )
 
     const project = new MapeoProject({
+      ...this.#projectStorage(projectId),
       ...projectKeys,
-      storagePath: this.#storagePath,
       keyManager: this.#keyManager,
       sharedDb: this.#db,
       sharedIndexWriter: this.#projectSettingsIndexWriter,

src/mapeo-project.js

@@ -1,4 +1,6 @@
 // @ts-check
+import path from 'path'
+import Database from 'better-sqlite3'
 import { decodeBlockPrefix } from '@mapeo/schema'
 import { drizzle } from 'drizzle-orm/better-sqlite3'
 import { migrate } from 'drizzle-orm/better-sqlite3/migrator'
@@ -9,23 +11,12 @@ import { DataType, kCreateWithDocId } from './datatype/index.js'
 import { IndexWriter } from './index-writer/index.js'
 import { projectTable } from './schema/client.js'
 import { fieldTable, observationTable, presetTable } from './schema/project.js'
-import RandomAccessFile from 'random-access-file'
-import RAM from 'random-access-memory'
-import Database from 'better-sqlite3'
-import path from 'path'
-import { RandomAccessFilePool } from './core-manager/random-access-file-pool.js'
 import { valueOf } from './utils.js'
 
 /** @typedef {Omit<import('@mapeo/schema').ProjectValue, 'schemaName'>} EditableProjectSettings */
 
-const PROJECT_SQLITE_FILE_NAME = 'project.db'
-const CORE_STORAGE_FOLDER_NAME = 'cores'
+const CORESTORE_STORAGE_FOLDER_NAME = 'corestore'
 const INDEXER_STORAGE_FOLDER_NAME = 'indexer'
-// Max file descriptors that RandomAccessFile should use for hypercore storage
-// and index bitfield persistence (used by MultiCoreIndexer). Android has a
-// limit of 1024 per process, so choosing 768 to leave 256 descriptors free for
-// other things e.g. SQLite and other parts of the app.
-const MAX_FILE_DESCRIPTORS = 768
 
 export class MapeoProject {
   #coreManager
@@ -35,57 +26,40 @@ export class MapeoProject {
 
   /**
    * @param {Object} opts
-   * @param {string} [opts.storagePath] Folder for all data storage (hypercores and sqlite db). Folder must exist. If not defined, everything is stored in-memory
+   * @param {string} opts.dbPath Path to store project sqlite db. Use `:memory:` for memory storage
    * @param {import('@mapeo/crypto').KeyManager} opts.keyManager mapeo/crypto KeyManager instance
    * @param {Buffer} opts.projectKey 32-byte public key of the project creator core
    * @param {Buffer} [opts.projectSecretKey] 32-byte secret key of the project creator core
    * @param {Partial<Record<import('./core-manager/index.js').Namespace, Buffer>>} [opts.encryptionKeys] Encryption keys for each namespace
    * @param {import('drizzle-orm/better-sqlite3').BetterSQLite3Database} opts.sharedDb
    * @param {IndexWriter} opts.sharedIndexWriter
+   * @param {import('./types.js').CoreStorage} opts.coreStorage Folder to store all hypercore data
+   *
    */
   constructor({
-    storagePath,
+    dbPath,
     sharedDb,
     sharedIndexWriter,
+    coreStorage,
     ...coreManagerOpts
   }) {
     // TODO: Update to use @mapeo/crypto when ready (https://github.com/digidem/mapeo-core-next/issues/171)
     this.#projectId = coreManagerOpts.projectKey.toString('hex')
 
     ///////// 1. Setup database
-
-    const dbPath =
-      storagePath !== undefined
-        ? path.join(storagePath, PROJECT_SQLITE_FILE_NAME)
-        : ':memory:'
     const sqlite = new Database(dbPath)
     const db = drizzle(sqlite)
     migrate(db, { migrationsFolder: './drizzle/project' })
 
     ///////// 2. Setup random-access-storage functions
 
-    const filePool =
-      storagePath !== undefined
-        ? new RandomAccessFilePool(MAX_FILE_DESCRIPTORS)
-        : undefined
     /** @type {ConstructorParameters<typeof CoreManager>[0]['storage']} */
-    const coreManagerStorage =
-      storagePath !== undefined
-        ? (name) =>
-            new RandomAccessFile(
-              path.join(storagePath, CORE_STORAGE_FOLDER_NAME, name),
-              { pool: filePool }
-            )
-        : () => new RAM()
+    const coreManagerStorage = (name) =>
+      coreStorage(path.join(CORESTORE_STORAGE_FOLDER_NAME, name))
+
     /** @type {ConstructorParameters<typeof DataStore>[0]['storage']} */
-    const indexerStorage =
-      storagePath !== undefined
-        ? (name) =>
-            new RandomAccessFile(
-              path.join(storagePath, INDEXER_STORAGE_FOLDER_NAME, name),
-              { pool: filePool }
-            )
-        : () => new RAM()
+    const indexerStorage = (name) =>
+      coreStorage(path.join(INDEXER_STORAGE_FOLDER_NAME, name))
 
     ///////// 3. Create instances
 

src/types.ts

@@ -14,6 +14,7 @@ import { Socket } from 'net'
 import MultiCoreIndexer from 'multi-core-indexer'
 import Corestore from 'corestore'
 import Hypercore from 'hypercore'
+import RandomAccessStorage from 'random-access-storage'
 
 type SupportedBlobVariants = typeof SUPPORTED_BLOB_VARIANTS
 type BlobType = keyof SupportedBlobVariants
@@ -168,3 +169,5 @@ export type ProtocolStream = NoiseStream & { userData: Protomux }
 export type Entries<T> = {
   [K in keyof T]: [K, T[K]]
 }[keyof T][]
+
+export type CoreStorage = (name: string) => RandomAccessStorage