diff --git a/app/common/DocumentSettings.ts b/app/common/DocumentSettings.ts index 54239cd678..ab7e40c126 100644 --- a/app/common/DocumentSettings.ts +++ b/app/common/DocumentSettings.ts @@ -2,6 +2,14 @@ export interface DocumentSettings { locale: string; currency?: string; engine?: EngineCode; + // Grist attachments can be stored within the document (embedded in the SQLite file), or held + // externally. The attachmentStoreId expresses a preference for which store should be used for + // attachments in this doc. This store will be used for new attachments when they're added, and a + // process can be triggered to start transferring all attachments that aren't already in this + // store over to it. A full id is stored, rather than something more convenient like a boolean or + // the string "external", after thinking carefully about how downloads/uploads and transferring + // files to other installations could work. + attachmentStoreId?: string; } /** diff --git a/app/server/generateInitialDocSql.ts b/app/server/generateInitialDocSql.ts index a3d3a2a352..661b4542d3 100644 --- a/app/server/generateInitialDocSql.ts +++ b/app/server/generateInitialDocSql.ts @@ -1,4 +1,5 @@ import { ActiveDoc } from 'app/server/lib/ActiveDoc'; +import { AttachmentStoreProvider } from 'app/server/lib/AttachmentStoreProvider'; import { create } from 'app/server/lib/create'; import { DocManager } from 'app/server/lib/DocManager'; import { makeExceptionalDocSession } from 'app/server/lib/DocSession'; @@ -33,7 +34,7 @@ export async function main(baseName: string) { if (await fse.pathExists(fname)) { await fse.remove(fname); } - const docManager = new DocManager(storageManager, pluginManager, null as any, { + const docManager = new DocManager(storageManager, pluginManager, null as any, new AttachmentStoreProvider([], ""), { create, getAuditLogger() { return createNullAuditLogger(); }, getTelemetry() { return createDummyTelemetry(); }, diff --git a/app/server/lib/ActiveDoc.ts b/app/server/lib/ActiveDoc.ts index 42ec507347..cd527eb266 100644 --- a/app/server/lib/ActiveDoc.ts +++ b/app/server/lib/ActiveDoc.ts @@ -13,7 +13,7 @@ import { UserActionBundle } from 'app/common/ActionBundle'; import {ActionGroup, MinimalActionGroup} from 'app/common/ActionGroup'; -import {ActionSummary} from "app/common/ActionSummary"; +import {ActionSummary} from 'app/common/ActionSummary'; import { AclResources, AclTableDescription, @@ -94,7 +94,6 @@ import {AssistanceSchemaPromptV1Context} from 'app/server/lib/Assistance'; import {AssistanceContext} from 'app/common/AssistancePrompts'; import {AuditEventAction} from 'app/server/lib/AuditEvent'; import {Authorizer, RequestWithLogin} from 'app/server/lib/Authorizer'; -import {checksumFile} from 'app/server/lib/checksumFile'; import {Client} from 'app/server/lib/Client'; import {getMetaTables} from 'app/server/lib/DocApi'; import {DEFAULT_CACHE_TTL, DocManager} from 'app/server/lib/DocManager'; @@ -105,7 +104,7 @@ import {makeForkIds} from 'app/server/lib/idUtils'; import {GRIST_DOC_SQL, GRIST_DOC_WITH_TABLE1_SQL} from 'app/server/lib/initialDocSql'; import {ISandbox} from 'app/server/lib/ISandbox'; import log from 'app/server/lib/log'; -import {LogMethods} from "app/server/lib/LogMethods"; +import {LogMethods} from 'app/server/lib/LogMethods'; import {ISandboxOptions} from 'app/server/lib/NSandbox'; import {NullSandbox, UnavailableSandboxMethodError} from 'app/server/lib/NullSandbox'; import {DocRequests} from 'app/server/lib/Requests'; @@ -121,12 +120,13 @@ import { } from 'app/server/lib/sessionUtils'; import {shortDesc} from 'app/server/lib/shortDesc'; import {TableMetadataLoader} from 'app/server/lib/TableMetadataLoader'; -import {DocTriggers} from "app/server/lib/Triggers"; +import {DocTriggers} from 'app/server/lib/Triggers'; import {fetchURL, FileUploadInfo, globalUploadSet, UploadInfo} from 'app/server/lib/uploads'; import assert from 'assert'; import {Mutex} from 'async-mutex'; import * as bluebird from 'bluebird'; import {EventEmitter} from 'events'; +import {readFile} from 'fs-extra'; import {IMessage, MsgType} from 'grain-rpc'; import imageSize from 'image-size'; import * as moment from 'moment-timezone'; @@ -137,6 +137,8 @@ import tmp from 'tmp'; import {ActionHistory} from './ActionHistory'; import {ActionHistoryImpl} from './ActionHistoryImpl'; import {ActiveDocImport, FileImportOptions} from './ActiveDocImport'; +import {AttachmentFileManager} from './AttachmentFileManager'; +import {IAttachmentStoreProvider} from './AttachmentStoreProvider'; import {DocClients} from './DocClients'; import {DocPluginManager} from './DocPluginManager'; import {DocSession, makeExceptionalDocSession, OptDocSession} from './DocSession'; @@ -265,6 +267,7 @@ export class ActiveDoc extends EventEmitter { private _onlyAllowMetaDataActionsOnDb: boolean = false; // Cache of which columns are attachment columns. private _attachmentColumns?: AttachmentColumns; + private _attachmentFileManager: AttachmentFileManager; // Client watching for 'product changed' event published by Billing to update usage private _redisSubscriber?: RedisClient; @@ -283,6 +286,7 @@ export class ActiveDoc extends EventEmitter { constructor( private readonly _docManager: DocManager, private _docName: string, + externalAttachmentStoreProvider?: IAttachmentStoreProvider, private _options?: ICreateActiveDocOptions ) { super(); @@ -388,6 +392,14 @@ export class ActiveDoc extends EventEmitter { loadTable: this._rawPyCall.bind(this, 'load_table'), }); + // This will throw errors if _options?.doc or externalAttachmentStoreProvider aren't provided, + // and ActiveDoc tries to use an external attachment store. + this._attachmentFileManager = new AttachmentFileManager( + this.docStorage, + externalAttachmentStoreProvider, + _options?.doc, + ); + // Our DataEngine is a separate sandboxed process (one sandbox per open document, // corresponding to one process for pynbox, more for gvisor). // The data engine runs user-defined python code including formula calculations. @@ -925,7 +937,7 @@ export class ActiveDoc extends EventEmitter { } } } - const data = await this.docStorage.getFileData(fileIdent); + const data = await this._attachmentFileManager.getFileData(fileIdent); if (!data) { throw new ApiError("Invalid attachment identifier", 404); } this._log.info(docSession, "getAttachment: %s -> %s bytes", fileIdent, data.length); return data; @@ -2344,13 +2356,16 @@ export class ActiveDoc extends EventEmitter { dimensions.height = 0; dimensions.width = 0; } - const checksum = await checksumFile(fileData.absPath); - const fileIdent = checksum + fileData.ext; - const ret: boolean = await this.docStorage.findOrAttachFile(fileData.absPath, fileIdent); - this._log.info(docSession, "addAttachment: file %s (image %sx%s) %s", fileIdent, - dimensions.width, dimensions.height, ret ? "attached" : "already exists"); + const attachmentStoreId = (await this._getDocumentSettings()).attachmentStoreId; + const addFileResult = await this._attachmentFileManager + .addFile(attachmentStoreId, fileData.ext, await readFile(fileData.absPath)); + this._log.info( + docSession, "addAttachment: store: '%s', file: '%s' (image %sx%s) %s", + attachmentStoreId ?? 'local document', addFileResult.fileIdent, dimensions.width, dimensions.height, + addFileResult.isNewFile ? "attached" : "already exists" + ); return ['AddRecord', '_grist_Attachments', null, { - fileIdent, + fileIdent: addFileResult.fileIdent, fileName: fileData.origName, // We used to set fileType, but it's not easily available for native types. Since it's // also entirely unused, we just skip it until it becomes relevant. @@ -2822,17 +2837,25 @@ export class ActiveDoc extends EventEmitter { return this._dataEngine; } + private async _getDocumentSettings(): Promise { + const docInfo = await this.docStorage.get('SELECT documentSettings FROM _grist_DocInfo'); + const docSettingsString = docInfo?.documentSettings; + const docSettings = docSettingsString ? safeJsonParse(docSettingsString, undefined) : undefined; + if (!docSettings) { + throw new Error("No document settings found"); + } + return docSettings; + } + private async _makeEngine(): Promise { // Figure out what kind of engine we need for this document. let preferredPythonVersion: '2' | '3' = process.env.PYTHON_VERSION === '2' ? '2' : '3'; // Careful, migrations may not have run on this document and it may not have a // documentSettings column. Failures are treated as lack of an engine preference. - const docInfo = await this.docStorage.get('SELECT documentSettings FROM _grist_DocInfo').catch(e => undefined); - const docSettingsString = docInfo?.documentSettings; - if (docSettingsString) { - const docSettings: DocumentSettings|undefined = safeJsonParse(docSettingsString, undefined); - const engine = docSettings?.engine; + const docSettings = await this._getDocumentSettings().catch(e => undefined); + if (docSettings) { + const engine = docSettings.engine; if (engine) { if (engine === 'python2') { preferredPythonVersion = '2'; diff --git a/app/server/lib/AttachmentFileManager.ts b/app/server/lib/AttachmentFileManager.ts new file mode 100644 index 0000000000..f406ee12bb --- /dev/null +++ b/app/server/lib/AttachmentFileManager.ts @@ -0,0 +1,220 @@ +import { + AttachmentStoreDocInfo, + DocPoolId, + getDocPoolIdFromDocInfo, + IAttachmentStore +} from 'app/server/lib/AttachmentStore'; +import {AttachmentStoreId, IAttachmentStoreProvider} from 'app/server/lib/AttachmentStoreProvider'; +import {checksumFileStream} from 'app/server/lib/checksumFile'; +import {DocStorage} from 'app/server/lib/DocStorage'; +import log from 'app/server/lib/log'; +import {LogMethods} from 'app/server/lib/LogMethods'; +import {MemoryWritableStream} from 'app/server/utils/MemoryWritableStream'; +import {Readable} from 'node:stream'; + +export interface IAttachmentFileManager { + addFile(storeId: AttachmentStoreId, fileExtension: string, fileData: Buffer): Promise; + getFileData(fileIdent: string): Promise; +} + +export interface AddFileResult { + fileIdent: string; + isNewFile: boolean; +} + +export class StoresNotConfiguredError extends Error { + constructor() { + super('Attempted to access a file store, but AttachmentFileManager was initialized without store access'); + } +} + +export class StoreNotAvailableError extends Error { + public readonly storeId: AttachmentStoreId; + + constructor(storeId: AttachmentStoreId) { + super(`Store '${storeId}' is not a valid and available store`); + this.storeId = storeId; + } +} + +export class MissingAttachmentError extends Error { + public readonly fileIdent: string; + + constructor(fileIdent: string) { + super(`Attachment file '${fileIdent}' could not be found in this document`); + this.fileIdent = fileIdent; + } +} + +export class AttachmentRetrievalError extends Error { + public readonly storeId: AttachmentStoreId; + public readonly fileId: string; + + constructor(storeId: AttachmentStoreId, fileId: string, cause?: any) { + const causeError = cause instanceof Error ? cause : undefined; + const causeDescriptor = causeError ? `: ${cause.message}` : ''; + super(`Unable to retrieve '${fileId}' from '${storeId}'${causeDescriptor}`); + this.storeId = storeId; + this.fileId = fileId; + this.cause = causeError; + } +} + + +interface AttachmentFileManagerLogInfo { + fileIdent?: string; + storeId?: string | null; +} + +/** + * Instantiated on a per-document basis to provide a document with access to its attachments. + * Handles attachment uploading / fetching, as well as trying to ensure consistency with the local + * document database, which tracks attachments and where they're stored. + * + * This class should prevent the document code from having to worry about accessing the underlying + * stores. + */ +export class AttachmentFileManager implements IAttachmentFileManager { + // _docPoolId is a critical point for security. Documents with a common pool id can access each others' attachments. + private readonly _docPoolId: DocPoolId | null; + private readonly _docName: string; + private _log = new LogMethods( + "AttachmentFileManager ", + (logInfo: AttachmentFileManagerLogInfo) => this._getLogMeta(logInfo) + ); + + /** + * @param _docStorage - Storage of this manager's document. + * @param _storeProvider - Allows instantiating of stores. Should be provided except in test + * scenarios. + * @param _docInfo - The document this manager is for. Should be provided except in test + * scenarios. + */ + constructor( + private _docStorage: DocStorage, + private _storeProvider: IAttachmentStoreProvider | undefined, + _docInfo: AttachmentStoreDocInfo | undefined, + ) { + this._docName = _docStorage.docName; + this._docPoolId = _docInfo ? getDocPoolIdFromDocInfo(_docInfo) : null; + } + + public async addFile( + storeId: AttachmentStoreId | undefined, + fileExtension: string, + fileData: Buffer + ): Promise { + const fileIdent = await this._getFileIdentifier(fileExtension, Readable.from(fileData)); + return this._addFile(storeId, fileIdent, fileData); + } + + public async _addFile( + storeId: AttachmentStoreId | undefined, + fileIdent: string, + fileData: Buffer + ): Promise { + this._log.info({ fileIdent, storeId }, `adding file to ${storeId ? "external" : "document"} storage`); + if (storeId === undefined) { + return this._addFileToLocalStorage(fileIdent, fileData); + } + const store = await this._getStore(storeId); + if (!store) { + this._log.info({ fileIdent, storeId }, "tried to fetch attachment from an unavailable store"); + throw new StoreNotAvailableError(storeId); + } + return this._addFileToAttachmentStore(store, fileIdent, fileData); + } + + public async getFileData(fileIdent: string): Promise { + const fileInfo = await this._docStorage.getFileInfo(fileIdent); + if (!fileInfo) { + this._log.error({ fileIdent }, "cannot find file metadata in document"); + throw new MissingAttachmentError(fileIdent); + } + this._log.debug( + { fileIdent, storeId: fileInfo.storageId }, + `fetching attachment from ${fileInfo.storageId ? "external" : "document "} storage` + ); + if (!fileInfo.storageId) { + return fileInfo.data; + } + const store = await this._getStore(fileInfo.storageId); + if (!store) { + this._log.warn({ fileIdent, storeId: fileInfo.storageId }, `unable to retrieve file, store is unavailable`); + throw new StoreNotAvailableError(fileInfo.storageId); + } + return this._getFileDataFromAttachmentStore(store, fileIdent); + } + + private async _addFileToLocalStorage(fileIdent: string, fileData: Buffer): Promise { + const isNewFile = await this._docStorage.findOrAttachFile(fileIdent, fileData); + + return { + fileIdent, + isNewFile, + }; + } + + private async _getStore(storeId: AttachmentStoreId): Promise { + if (!this._storeProvider) { + throw new StoresNotConfiguredError(); + } + return this._storeProvider.getStore(storeId); + } + + private _getDocPoolId(): string { + if (!this._docPoolId) { + throw new StoresNotConfiguredError(); + } + return this._docPoolId; + } + + private async _getFileIdentifier(fileExtension: string, fileData: Readable): Promise { + const checksum = await checksumFileStream(fileData); + return `${checksum}${fileExtension}`; + } + + private async _addFileToAttachmentStore( + store: IAttachmentStore, fileIdent: string, fileData: Buffer + ): Promise { + const isNewFile = await this._docStorage.findOrAttachFile(fileIdent, undefined, store.id); + + // Verify the file exists in the store. This allows for a second attempt to correct a failed upload. + const existsInRemoteStorage = !isNewFile && await store.exists(this._getDocPoolId(), fileIdent); + + if (!isNewFile && existsInRemoteStorage) { + return { + fileIdent, + isNewFile: false, + }; + } + + // Possible issue if this upload fails - we have the file tracked in the document, but not available in the store. + // TODO - Decide if we keep an entry in SQLite after an upload error or not. Probably not? + await store.upload(this._getDocPoolId(), fileIdent, Readable.from(fileData)); + + // TODO - Confirm in doc storage that it's successfully uploaded? Need to decide how to handle a failed upload. + return { + fileIdent, + isNewFile, + }; + } + + private async _getFileDataFromAttachmentStore(store: IAttachmentStore, fileIdent: string): Promise { + try { + const outputStream = new MemoryWritableStream(); + await store.download(this._getDocPoolId(), fileIdent, outputStream); + return outputStream.getBuffer(); + } catch(e) { + throw new AttachmentRetrievalError(store.id, fileIdent, e); + } + } + + private _getLogMeta(logInfo?: AttachmentFileManagerLogInfo): log.ILogMeta { + return { + docName: this._docName, + docPoolId: this._docPoolId, + ...logInfo, + }; + } +} diff --git a/app/server/lib/AttachmentStore.ts b/app/server/lib/AttachmentStore.ts new file mode 100644 index 0000000000..7b90625a05 --- /dev/null +++ b/app/server/lib/AttachmentStore.ts @@ -0,0 +1,178 @@ +import {joinKeySegments, StreamingExternalStorage} from 'app/server/lib/ExternalStorage'; +import * as fse from 'fs-extra'; +import * as stream from 'node:stream'; +import * as path from 'path'; + +export type DocPoolId = string; +type FileId = string; + + +// Minimum document info needed to know which document pool to use. +// Compatible with Document entity for ease of use +export interface AttachmentStoreDocInfo { + id: string; + // We explicitly make this a union type instead of making the attribute optional because the + // programmer must make a conscious choice to mark it as null or undefined, not merely omit it. + // Omission could easily result in invalid behaviour. + trunkId: string | null | undefined; +} + +/** + * Gets the correct pool id for a given document, given the document's id and trunk id. + * + * Attachments are stored in a "Document Pool", which is used to manage the attachments' lifecycle. + * Document pools are shared between snapshots and forks, but not between documents. This provides + * quick forking and snapshotting (not having to copy attachments), while avoiding more complex + * systems like reference tracking. + * + * Generally, the pool id of a document should be its trunk id if available (because it's a fork), + * or the document's id (if it isn't a fork). + * + * This means that attachments need copying to a new pool when a document is copied. + * Avoids other areas of the codebase having to understand how documents are mapped to pools. + * + * This is a key security measure, as only a specific document and its forks can access its + * attachments. This helps prevent malicious documents being uploaded, which might attempt to + * access another user's attachments. + * + * Therefore, it is CRITICAL that documents with different security domains (e.g from different + * teams) do not share a document pool. + * @param {AttachmentStoreDocInfo} docInfo - Document details needed to calculate the document + * pool. + * @returns {string} - ID of the pool the attachments will be stored in. + */ +export function getDocPoolIdFromDocInfo(docInfo: AttachmentStoreDocInfo): string { + return docInfo.trunkId ?? docInfo.id; +} + +/** + * Provides access to external storage, specifically for storing attachments. Each store represents + * a specific location to store attachments, e.g. "/srv/grist/attachments" on the filesystem. + * + * This is a general-purpose interface that should abstract over many different storage providers, + * so shouldn't have methods which rely on one the features of one specific provider. + * + * `IAttachmentStore` is distinct from `ExternalStorage` as it's specific to attachments, and can + * therefore not concern itself with some features ExternalStorage has (e.g versioning). This means + * it can present a more straightforward interface for components which need to access attachment + * files. + * + * A document pool needs specifying for all store operations, which should be calculated with + * `getDocPoolIdFromDocInfo` See {@link getDocPoolIdFromDocInfo} for more details. + */ +export interface IAttachmentStore { + // Universally unique id, such that no two Grist installations should have the same store ids, if + // they're for different stores. This allows for explicit detection of unavailable stores. + readonly id: string; + + // Check if attachment exists in the store. + exists(docPoolId: DocPoolId, fileId: FileId): Promise; + + // Upload attachment to the store. + upload(docPoolId: DocPoolId, fileId: FileId, fileData: stream.Readable): Promise; + + // Download attachment to an in-memory buffer. + // It's preferable to accept an output stream as a parameter, as it simplifies attachment store + // implementation and gives them control over local buffering. + download(docPoolId: DocPoolId, fileId: FileId, outputStream: stream.Writable): Promise; + + // Remove attachments for all documents in the given document pool. + removePool(docPoolId: DocPoolId): Promise; + + // Close the storage object. + close(): Promise; +} + +export class InvalidAttachmentExternalStorageError extends Error { + constructor(storeId: string, context?: string) { + const formattedContext = context ? `: ${context}` : ""; + super(`External Storage for store '${storeId}' is invalid` + formattedContext); + } +} + +export class AttachmentStoreCreationError extends Error { + constructor(storeBackend: string, storeId: string, context?: string) { + const formattedContext = context ? `: ${context}` : ""; + super(`Unable to create ${storeBackend} store '${storeId}'` + formattedContext); + } +} + +export class ExternalStorageAttachmentStore implements IAttachmentStore { + constructor( + public id: string, + private _storage: StreamingExternalStorage, + private _prefixParts: string[] + ) { + if (!_storage.removeAllWithPrefix) { + throw new InvalidAttachmentExternalStorageError("ExternalStorage does not support removeAllWithPrefix"); + } + } + + public exists(docPoolId: string, fileId: string): Promise { + return this._storage.exists(this._getKey(docPoolId, fileId)); + } + + public async upload(docPoolId: string, fileId: string, fileData: stream.Readable): Promise { + await this._storage.uploadStream(this._getKey(docPoolId, fileId), fileData); + } + + public async download(docPoolId: string, fileId: string, outputStream: stream.Writable): Promise { + await this._storage.downloadStream(this._getKey(docPoolId, fileId), outputStream); + } + + public async removePool(docPoolId: string): Promise { + // Null assertion is safe because this should be checked before this class is instantiated. + await this._storage.removeAllWithPrefix!(this._getPoolPrefix(docPoolId)); + } + + public async close(): Promise { + await this._storage.close(); + } + + private _getPoolPrefix(docPoolId: string): string { + return joinKeySegments([...this._prefixParts, docPoolId]); + } + + private _getKey(docPoolId: string, fileId: string): string { + return joinKeySegments([this._getPoolPrefix(docPoolId), fileId]); + } +} + +export class FilesystemAttachmentStore implements IAttachmentStore { + constructor(public readonly id: string, private _rootFolderPath: string) { + } + + public async exists(docPoolId: DocPoolId, fileId: FileId): Promise { + return fse.pathExists(this._createPath(docPoolId, fileId)) + .catch(() => false); + } + + public async upload(docPoolId: DocPoolId, fileId: FileId, fileData: stream.Readable): Promise { + const filePath = this._createPath(docPoolId, fileId); + await fse.ensureDir(path.dirname(filePath)); + const writeStream = fse.createWriteStream(filePath); + await stream.promises.pipeline( + fileData, + writeStream, + ); + } + + public async download(docPoolId: DocPoolId, fileId: FileId, output: stream.Writable): Promise { + await stream.promises.pipeline( + fse.createReadStream(this._createPath(docPoolId, fileId)), + output, + ); + } + + public async removePool(docPoolId: DocPoolId): Promise { + await fse.remove(this._createPath(docPoolId)); + } + + public async close(): Promise { + // Not needed here, no resources held. + } + + private _createPath(docPoolId: DocPoolId, fileId: FileId = ""): string { + return path.join(this._rootFolderPath, docPoolId, fileId); + } +} diff --git a/app/server/lib/AttachmentStoreProvider.ts b/app/server/lib/AttachmentStoreProvider.ts new file mode 100644 index 0000000000..1ae1d0f678 --- /dev/null +++ b/app/server/lib/AttachmentStoreProvider.ts @@ -0,0 +1,97 @@ +import {IAttachmentStore} from 'app/server/lib/AttachmentStore'; +import log from 'app/server/lib/log'; +import {ICreateAttachmentStoreOptions} from './ICreate'; + +export type AttachmentStoreId = string + +/** + * Creates an {@link IAttachmentStore} from a given store id, if the Grist installation is + * configured with that store's unique id. + * + * Each store represents a specific location to store attachments at, for example a "/attachments" + * bucket on MinIO, or "/srv/grist/attachments" on the filesystem. + * + * Attachments in Grist Documents are accompanied by the id of the store they're in, allowing Grist + * to store/retrieve them as long as that store exists on the document's installation. + */ +export interface IAttachmentStoreProvider { + // Returns the store associated with the given id, returning null if no store with that id exists. + getStore(id: AttachmentStoreId): Promise + + getAllStores(): Promise; + + storeExists(id: AttachmentStoreId): Promise; + + listAllStoreIds(): AttachmentStoreId[]; +} + +export interface IAttachmentStoreSpecification { + name: string, + create: (storeId: string) => Promise, +} + +interface IAttachmentStoreDetails { + id: string; + spec: IAttachmentStoreSpecification; +} + +export class AttachmentStoreProvider implements IAttachmentStoreProvider { + private _storeDetailsById: { [storeId: string]: IAttachmentStoreDetails } = {}; + + constructor( + _backends: IAttachmentStoreSpecification[], + _installationUuid: string + ) { + // In the current setup, we automatically generate store IDs based on the installation ID. + // The installation ID is guaranteed to be unique, and we only allow one store of each backend type. + // This gives us a way to reproducibly generate a unique ID for the stores. + _backends.forEach((storeSpec) => { + const storeId = `${_installationUuid}-${storeSpec.name}`; + this._storeDetailsById[storeId] = { + id: storeId, + spec: storeSpec, + }; + }); + + const storeIds = Object.values(this._storeDetailsById).map(storeDetails => storeDetails.id); + log.info(`AttachmentStoreProvider initialised with stores: ${storeIds}`); + } + + public async getStore(id: AttachmentStoreId): Promise { + const storeDetails = this._storeDetailsById[id]; + if (!storeDetails) { return null; } + return storeDetails.spec.create(id); + } + + public async getAllStores(): Promise { + return await Promise.all( + Object.values(this._storeDetailsById).map(storeDetails => storeDetails.spec.create(storeDetails.id)) + ); + } + + public async storeExists(id: AttachmentStoreId): Promise { + return id in this._storeDetailsById; + } + + public listAllStoreIds(): string[] { + return Object.keys(this._storeDetailsById); + } +} + +async function checkAvailabilityAttachmentStoreOption(option: ICreateAttachmentStoreOptions) { + try { + return await option.isAvailable(); + } catch (error) { + log.error(`Error checking availability of store option '${option}'`, error); + return false; + } +} + +export async function checkAvailabilityAttachmentStoreOptions(options: ICreateAttachmentStoreOptions[]) { + const availability = await Promise.all(options.map(checkAvailabilityAttachmentStoreOption)); + + return { + available: options.filter((option, index) => availability[index]), + unavailable: options.filter((option, index) => !availability[index]), + }; +} diff --git a/app/server/lib/DocApi.ts b/app/server/lib/DocApi.ts index cc6ca4aad8..28652552d9 100644 --- a/app/server/lib/DocApi.ts +++ b/app/server/lib/DocApi.ts @@ -46,6 +46,8 @@ import { import {ActiveDoc, colIdToRef as colIdToReference, getRealTableId, tableIdToRef} from "app/server/lib/ActiveDoc"; import {appSettings} from "app/server/lib/AppSettings"; import {sendForCompletion} from 'app/server/lib/Assistance'; +import {getDocPoolIdFromDocInfo} from 'app/server/lib/AttachmentStore'; +import {IAttachmentStoreProvider} from 'app/server/lib/AttachmentStoreProvider'; import { assertAccess, getAuthorizedUserId, @@ -179,7 +181,8 @@ export class DocWorkerApi { constructor(private _app: Application, private _docWorker: DocWorker, private _docWorkerMap: IDocWorkerMap, private _docManager: DocManager, - private _dbManager: HomeDBManager, private _grist: GristServer) {} + private _dbManager: HomeDBManager, private _attachmentStoreProvider: IAttachmentStoreProvider, + private _grist: GristServer) {} /** * Adds endpoints for the doc api. @@ -2027,6 +2030,16 @@ export class DocWorkerApi { ...forks.map((fork) => buildUrlId({forkId: fork.id, forkUserId: fork.createdBy!, trunkId: docId})), ]; + if (!forkId) { + // Delete all remote document attachments before the doc itself. + // This way we can re-attempt deletion if an error is thrown. + const attachmentStores = await this._attachmentStoreProvider.getAllStores(); + log.debug(`Deleting all attachments for ${docId} from ${attachmentStores.length} stores`); + const poolDeletions = attachmentStores.map( + store => store.removePool(getDocPoolIdFromDocInfo({ id: docId, trunkId: null })) + ); + await Promise.all(poolDeletions); + } await Promise.all(docsToDelete.map(docName => this._docManager.deleteDoc(null, docName, true))); // Permanently delete from database. result = await this._dbManager.deleteDocument(scope); @@ -2359,9 +2372,9 @@ export class DocWorkerApi { export function addDocApiRoutes( app: Application, docWorker: DocWorker, docWorkerMap: IDocWorkerMap, docManager: DocManager, dbManager: HomeDBManager, - grist: GristServer + attachmentStoreProvider: IAttachmentStoreProvider, grist: GristServer ) { - const api = new DocWorkerApi(app, docWorker, docWorkerMap, docManager, dbManager, grist); + const api = new DocWorkerApi(app, docWorker, docWorkerMap, docManager, dbManager, attachmentStoreProvider, grist); api.addEndpoints(); } diff --git a/app/server/lib/DocManager.ts b/app/server/lib/DocManager.ts index ac90ba318f..45d4dbd901 100644 --- a/app/server/lib/DocManager.ts +++ b/app/server/lib/DocManager.ts @@ -17,6 +17,7 @@ import {NEW_DOCUMENT_CODE} from 'app/common/UserAPI'; import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager'; import {assertAccess, Authorizer, DocAuthorizer, DummyAuthorizer, isSingleUserMode, RequestWithLogin} from 'app/server/lib/Authorizer'; +import {IAttachmentStoreProvider} from 'app/server/lib/AttachmentStoreProvider'; import {Client} from 'app/server/lib/Client'; import {makeExceptionalDocSession, makeOptDocSession, OptDocSession} from 'app/server/lib/DocSession'; import * as docUtils from 'app/server/lib/docUtils'; @@ -61,7 +62,8 @@ export class DocManager extends EventEmitter { public readonly storageManager: IDocStorageManager, public readonly pluginManager: PluginManager|null, private _homeDbManager: HomeDBManager|null, - public gristServer: GristServer + private _attachmentStoreProvider: IAttachmentStoreProvider, + public gristServer: GristServer, ) { super(); } @@ -607,7 +609,7 @@ export class DocManager extends EventEmitter { const doc = await this._getDoc(docSession, docName); // Get URL for document for use with SELF_HYPERLINK(). const docUrls = doc && await this._getDocUrls(doc); - const activeDoc = new ActiveDoc(this, docName, {...docUrls, safeMode, doc}); + const activeDoc = new ActiveDoc(this, docName, this._attachmentStoreProvider, {...docUrls, safeMode, doc}); // Restore the timing mode of the document. activeDoc.isTimingOn = this._inTimingOn.get(docName) || false; return activeDoc; diff --git a/app/server/lib/DocStorage.ts b/app/server/lib/DocStorage.ts index d43504a1a5..bca89e7f73 100644 --- a/app/server/lib/DocStorage.ts +++ b/app/server/lib/DocStorage.ts @@ -21,7 +21,6 @@ import {IDocStorageManager} from 'app/server/lib/IDocStorageManager'; import log from 'app/server/lib/log'; import assert from 'assert'; import * as bluebird from 'bluebird'; -import * as fse from 'fs-extra'; import * as _ from 'underscore'; import * as util from 'util'; import {v4 as uuidv4} from 'uuid'; @@ -73,7 +72,8 @@ export class DocStorage implements ISQLiteDB, OnDemandStorage { await db.exec(`CREATE TABLE _gristsys_Files ( id INTEGER PRIMARY KEY, ident TEXT UNIQUE, - data BLOB + data BLOB, + storageId TEXT )`); await db.exec(`CREATE TABLE _gristsys_Action ( id INTEGER PRIMARY KEY, @@ -394,7 +394,13 @@ export class DocStorage implements ISQLiteDB, OnDemandStorage { } await createAttachmentsIndex(db); }, - + async function(db: SQLiteDB): Promise { + // Storage version 9. + // Migration to add `storage` column to _gristsys_Files, which can optionally refer to an external storage + // where the file is stored. + // Default should be NULL. + await db.exec(`ALTER TABLE _gristsys_Files ADD COLUMN storageId TEXT`); + }, ] }; @@ -768,19 +774,23 @@ export class DocStorage implements ISQLiteDB, OnDemandStorage { * would be (very?) inefficient until node-sqlite3 adds support for incremental reading from a * blob: https://github.com/mapbox/node-sqlite3/issues/424. * - * @param {String} sourcePath: The path of the file containing the attachment data. - * @param {String} fileIdent: The unique identifier of the file in the database. ActiveDoc uses the + * @param {string} fileIdent - The unique identifier of the file in the database. ActiveDoc uses the * checksum of the file's contents with the original extension. + * @param {Buffer | undefined} fileData - Contents of the file. + * @param {string | undefined} storageId - Identifier of the store that file is stored in. * @returns {Promise[Boolean]} True if the file got attached; false if this ident already exists. */ - public findOrAttachFile(sourcePath: string, fileIdent: string): Promise { + public findOrAttachFile( + fileIdent: string, + fileData: Buffer | undefined, + storageId?: string, + ): Promise { return this.execTransaction(db => { // Try to insert a new record with the given ident. It'll fail UNIQUE constraint if exists. return db.run('INSERT INTO _gristsys_Files (ident) VALUES (?)', fileIdent) // Only if this succeeded, do the work of reading the file and inserting its data. - .then(() => fse.readFile(sourcePath)) - .then(data => - db.run('UPDATE _gristsys_Files SET data=? WHERE ident=?', data, fileIdent)) + .then(() => + db.run('UPDATE _gristsys_Files SET data=?, storageId=? WHERE ident=?', fileData, storageId, fileIdent)) .then(() => true) // If UNIQUE constraint failed, this ident must already exists, so return false. .catch(err => { @@ -794,12 +804,16 @@ export class DocStorage implements ISQLiteDB, OnDemandStorage { /** * Reads and returns the data for the given attachment. - * @param {String} fileIdent: The unique identifier of a file, as used by findOrAttachFile. + * @param {string} fileIdent - The unique identifier of a file, as used by findOrAttachFile. * @returns {Promise[Buffer]} The data buffer associated with fileIdent. */ - public getFileData(fileIdent: string): Promise { - return this.get('SELECT data FROM _gristsys_Files WHERE ident=?', fileIdent) - .then(row => row && row.data); + public getFileInfo(fileIdent: string): Promise { + return this.get('SELECT ident, storageId, data FROM _gristsys_Files WHERE ident=?', fileIdent) + .then(row => row ? ({ + ident: row.ident as string, + storageId: (row.storageId ?? null) as (string | null), + data: row.data as Buffer, + }) : null); } @@ -1403,6 +1417,7 @@ export class DocStorage implements ISQLiteDB, OnDemandStorage { /** * Delete attachments from _gristsys_Files that have no matching metadata row in _grist_Attachments. + * This leaves any attachment files in any remote attachment stores, which will be cleaned up separately. */ public async removeUnusedAttachments() { const result = await this._getDB().run(` @@ -1851,3 +1866,10 @@ export async function createAttachmentsIndex(db: ISQLiteDB) { function fixDefault(def: string) { return (def === '""') ? "''" : def; } + +// Information on an attached file from _gristsys_files +export interface FileInfo { + ident: string; + storageId: string | null; + data: Buffer; +} diff --git a/app/server/lib/ExternalStorage.ts b/app/server/lib/ExternalStorage.ts index 74394ba535..7004e2c642 100644 --- a/app/server/lib/ExternalStorage.ts +++ b/app/server/lib/ExternalStorage.ts @@ -5,6 +5,7 @@ import {createTmpDir} from 'app/server/lib/uploads'; import {delay} from 'bluebird'; import * as fse from 'fs-extra'; import * as path from 'path'; +import stream from 'node:stream'; // A special token representing a deleted document, used in places where a // checksum is expected otherwise. @@ -39,6 +40,9 @@ export interface ExternalStorage { // newest should be given first. remove(key: string, snapshotIds?: string[]): Promise; + // Removes all keys which start with the given prefix + removeAllWithPrefix?(prefix: string): Promise; + // List content versions that exist for the given key. More recent versions should // come earlier in the result list. versions(key: string): Promise; @@ -55,6 +59,11 @@ export interface ExternalStorage { close(): Promise; } +export interface StreamingExternalStorage extends ExternalStorage { + uploadStream(key: string, inStream: stream.Readable, metadata?: ObjMetadata): Promise; + downloadStream(key: string, outStream: stream.Writable, snapshotId?: string ): Promise; +} + /** * Convenience wrapper to transform keys for an external store. * E.g. this could convert "" to "v1/.grist" @@ -386,6 +395,27 @@ export interface ExternalStorageSettings { export type ExternalStorageCreator = (purpose: ExternalStorageSettings["purpose"], extraPrefix: string) => ExternalStorage | undefined; +function stripTrailingSlash(text: string): string { + return text.endsWith("/") ? text.slice(0, -1) : text; +} + +function stripLeadingSlash(text: string): string { + return text[0] === "/" ? text.slice(1) : text; +} + +export function joinKeySegments(keySegments: string[]): string { + if (keySegments.length < 1) { + return ""; + } + const firstPart = keySegments[0]; + const remainingParts = keySegments.slice(1); + const strippedParts = [ + stripTrailingSlash(firstPart), + ...remainingParts.map(stripTrailingSlash).map(stripLeadingSlash) + ]; + return strippedParts.join("/"); +} + /** * The storage mapping we use for our SaaS. A reasonable default, but relies * on appropriate lifecycle rules being set up in the bucket. diff --git a/app/server/lib/FlexServer.ts b/app/server/lib/FlexServer.ts index b0a5f5341d..b3ef8e689e 100644 --- a/app/server/lib/FlexServer.ts +++ b/app/server/lib/FlexServer.ts @@ -11,8 +11,8 @@ import {SandboxInfo} from 'app/common/SandboxInfo'; import {tbind} from 'app/common/tbind'; import * as version from 'app/common/version'; import {ApiServer, getOrgFromRequest} from 'app/gen-server/ApiServer'; -import {Document} from "app/gen-server/entity/Document"; -import {Organization} from "app/gen-server/entity/Organization"; +import {Document} from 'app/gen-server/entity/Document'; +import {Organization} from 'app/gen-server/entity/Organization'; import {User} from 'app/gen-server/entity/User'; import {Workspace} from 'app/gen-server/entity/Workspace'; import {Activations} from 'app/gen-server/lib/Activations'; @@ -27,11 +27,16 @@ import {createSandbox} from 'app/server/lib/ActiveDoc'; import {attachAppEndpoint} from 'app/server/lib/AppEndpoint'; import {appSettings} from 'app/server/lib/AppSettings'; import {attachEarlyEndpoints} from 'app/server/lib/attachEarlyEndpoints'; +import { + AttachmentStoreProvider, checkAvailabilityAttachmentStoreOptions, IAttachmentStoreProvider +} from 'app/server/lib/AttachmentStoreProvider'; import {addRequestUser, getTransitiveHeaders, getUser, getUserId, isAnonymousUser, isSingleUserMode, redirectToLoginUnconditionally} from 'app/server/lib/Authorizer'; import {redirectToLogin, RequestWithLogin, signInStatusMiddleware} from 'app/server/lib/Authorizer'; import {forceSessionChange} from 'app/server/lib/BrowserSession'; import {Comm} from 'app/server/lib/Comm'; +import {ConfigBackendAPI} from 'app/server/lib/ConfigBackendAPI'; +import {IGristCoreConfig} from 'app/server/lib/configCore'; import {create} from 'app/server/lib/create'; import {addDiscourseConnectEndpoints} from 'app/server/lib/DiscourseConnect'; import {addDocApiRoutes} from 'app/server/lib/DocApi'; @@ -40,7 +45,7 @@ import {DocWorker} from 'app/server/lib/DocWorker'; import {DocWorkerInfo, IDocWorkerMap} from 'app/server/lib/DocWorkerMap'; import {expressWrap, jsonErrorHandler, secureJsonErrorHandler} from 'app/server/lib/expressWrap'; import {Hosts, RequestWithOrg} from 'app/server/lib/extractOrg'; -import {addGoogleAuthEndpoint} from "app/server/lib/GoogleAuth"; +import {addGoogleAuthEndpoint} from 'app/server/lib/GoogleAuth'; import {GristBullMQJobs, GristJobs} from 'app/server/lib/GristJobs'; import {DocTemplate, GristLoginMiddleware, GristLoginSystem, GristServer, RequestWithGrist} from 'app/server/lib/GristServer'; @@ -59,6 +64,7 @@ import * as ProcessMonitor from 'app/server/lib/ProcessMonitor'; import {adaptServerUrl, getOrgUrl, getOriginUrl, getScope, integerParam, isParameterOn, optIntegerParam, optStringParam, RequestWithGristInfo, stringArrayParam, stringParam, TEST_HTTPS_OFFSET, trustOrigin} from 'app/server/lib/requestUtils'; +import {buildScimRouter} from 'app/server/lib/scim'; import {ISendAppPageOptions, makeGristConfig, makeMessagePage, makeSendAppPage} from 'app/server/lib/sendAppPage'; import {getDatabaseUrl, listenPromise, timeoutReached} from 'app/server/lib/serverUtils'; import {Sessions} from 'app/server/lib/Sessions'; @@ -78,17 +84,14 @@ import * as fse from 'fs-extra'; import * as http from 'http'; import * as https from 'https'; import {i18n} from 'i18next'; -import i18Middleware from "i18next-http-middleware"; +import i18Middleware from 'i18next-http-middleware'; import mapValues = require('lodash/mapValues'); import pick = require('lodash/pick'); import morganLogger from 'morgan'; import {AddressInfo} from 'net'; import fetch from 'node-fetch'; import * as path from 'path'; -import * as serveStatic from "serve-static"; -import {ConfigBackendAPI} from "app/server/lib/ConfigBackendAPI"; -import {IGristCoreConfig} from "app/server/lib/configCore"; -import {buildScimRouter} from 'app/server/lib/scim'; +import * as serveStatic from 'serve-static'; // Health checks are a little noisy in the logs, so we don't show them all. // We show the first N health checks: @@ -144,6 +147,7 @@ export class FlexServer implements GristServer { private _billing: IBilling; private _installAdmin: InstallAdmin; private _instanceRoot: string; + private _attachmentStoreProvider: IAttachmentStoreProvider; private _docManager: DocManager; private _docWorker: DocWorker; private _hosts: Hosts; @@ -1384,8 +1388,22 @@ export class FlexServer implements GristServer { } const pluginManager = await this._addPluginManager(); - this._docManager = this._docManager || new DocManager(this._storageManager, pluginManager, - this._dbManager, this); + + const storeOptions = await checkAvailabilityAttachmentStoreOptions(this.create.getAttachmentStoreOptions()); + log.info("Attachment store backend availability", { + available: storeOptions.available.map(option => option.name), + unavailable: storeOptions.unavailable.map(option => option.name), + }); + + this._attachmentStoreProvider = this._attachmentStoreProvider || new AttachmentStoreProvider( + storeOptions.available, + (await this.getActivations().current()).id, + ); + this._docManager = this._docManager || new DocManager(this._storageManager, + pluginManager, + this._dbManager, + this._attachmentStoreProvider, + this); const docManager = this._docManager; shutdown.addCleanupHandler(null, this._shutdown.bind(this), 25000, 'FlexServer._shutdown'); @@ -1415,7 +1433,8 @@ export class FlexServer implements GristServer { this._addSupportPaths(docAccessMiddleware); if (!isSingleUserMode()) { - addDocApiRoutes(this.app, docWorker, this._docWorkerMap, docManager, this._dbManager, this); + addDocApiRoutes(this.app, docWorker, this._docWorkerMap, docManager, this._dbManager, + this._attachmentStoreProvider, this); } } diff --git a/app/server/lib/ICreate.ts b/app/server/lib/ICreate.ts index 2242c7a257..38e3dcb3d0 100644 --- a/app/server/lib/ICreate.ts +++ b/app/server/lib/ICreate.ts @@ -3,10 +3,16 @@ import {getCoreLoginSystem} from 'app/server/lib/coreLogins'; import {getThemeBackgroundSnippet} from 'app/common/Themes'; import {Document} from 'app/gen-server/entity/Document'; import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager'; +import {IAttachmentStore} from 'app/server/lib/AttachmentStore'; +import {Comm} from 'app/server/lib/Comm'; +import {DocStorageManager} from 'app/server/lib/DocStorageManager'; +import {IDocWorkerMap} from 'app/server/lib/DocWorkerMap'; import {ExternalStorage, ExternalStorageCreator} from 'app/server/lib/ExternalStorage'; import {createDummyTelemetry, GristLoginSystem, GristServer} from 'app/server/lib/GristServer'; +import {HostedStorageManager, HostedStorageOptions} from 'app/server/lib/HostedStorageManager'; import {createNullAuditLogger, IAuditLogger} from 'app/server/lib/IAuditLogger'; import {IBilling} from 'app/server/lib/IBilling'; +import {IDocStorageManager} from 'app/server/lib/IDocStorageManager'; import {EmptyNotifier, INotifier} from 'app/server/lib/INotifier'; import {InstallAdmin, SimpleInstallAdmin} from 'app/server/lib/InstallAdmin'; import {ISandbox, ISandboxCreationOptions} from 'app/server/lib/ISandbox'; @@ -14,11 +20,6 @@ import {IShell} from 'app/server/lib/IShell'; import {createSandbox, SpawnFn} from 'app/server/lib/NSandbox'; import {SqliteVariant} from 'app/server/lib/SqliteCommon'; import {ITelemetry} from 'app/server/lib/Telemetry'; -import {IDocStorageManager} from './IDocStorageManager'; -import { Comm } from "./Comm"; -import { IDocWorkerMap } from "./DocWorkerMap"; -import { HostedStorageManager, HostedStorageOptions } from "./HostedStorageManager"; -import { DocStorageManager } from "./DocStorageManager"; // In the past, the session secret was used as an additional // protection passed on to expressjs-session for security when @@ -89,6 +90,7 @@ export interface ICreate { // static page. getExtraHeadHtml?(): string; getStorageOptions?(name: string): ICreateStorageOptions|undefined; + getAttachmentStoreOptions(): ICreateAttachmentStoreOptions[]; getSqliteVariant?(): SqliteVariant; getSandboxVariants?(): Record; @@ -125,6 +127,12 @@ export interface ICreateTelemetryOptions { create(dbManager: HomeDBManager, gristConfig: GristServer): ITelemetry|undefined; } +export interface ICreateAttachmentStoreOptions { + name: string; + isAvailable(): Promise; + create(storeId: string): Promise; +} + /** * This function returns a `create` object that defines various core * aspects of a Grist installation, such as what kind of billing or @@ -151,6 +159,7 @@ export function makeSimpleCreator(opts: { getLoginSystem?: () => Promise, createHostedDocStorageManager?: HostedDocStorageManagerCreator, createLocalDocStorageManager?: LocalDocStorageManagerCreator, + attachmentStoreOptions?: ICreateAttachmentStoreOptions[], }): ICreate { const {deploymentType, sessionSecret, storage, notifier, billing, auditLogger, telemetry} = opts; return { @@ -228,6 +237,9 @@ export function makeSimpleCreator(opts: { getStorageOptions(name: string) { return storage?.find(s => s.name === name); }, + getAttachmentStoreOptions(): ICreateAttachmentStoreOptions[] { + return opts.attachmentStoreOptions ?? []; + }, getSqliteVariant: opts.getSqliteVariant, getSandboxVariants: opts.getSandboxVariants, createInstallAdmin: opts.createInstallAdmin || (async (dbManager) => new SimpleInstallAdmin(dbManager)), diff --git a/app/server/lib/MinIOExternalStorage.ts b/app/server/lib/MinIOExternalStorage.ts index f5b5788616..bf06736813 100644 --- a/app/server/lib/MinIOExternalStorage.ts +++ b/app/server/lib/MinIOExternalStorage.ts @@ -1,9 +1,10 @@ import {ApiError} from 'app/common/ApiError'; import {ObjMetadata, ObjSnapshotWithMetadata, toExternalMetadata, toGristMetadata} from 'app/common/DocSnapshot'; -import {ExternalStorage} from 'app/server/lib/ExternalStorage'; +import {StreamingExternalStorage} from 'app/server/lib/ExternalStorage'; import {IncomingMessage} from 'http'; import * as fse from 'fs-extra'; import * as minio from 'minio'; +import * as stream from 'node:stream'; // The minio-js v8.0.0 typings are sometimes incorrect. Here are some workarounds. interface MinIOClient extends @@ -43,7 +44,7 @@ type RemoveObjectsResponse = null | undefined | { * An external store implemented using the MinIO client, which * will work with MinIO and other S3-compatible storage. */ -export class MinIOExternalStorage implements ExternalStorage { +export class MinIOExternalStorage implements StreamingExternalStorage { // Specify bucket to use, and optionally the max number of keys to request // in any call to listObjectVersions (used for testing) constructor( @@ -86,18 +87,21 @@ export class MinIOExternalStorage implements ExternalStorage { } } - public async upload(key: string, fname: string, metadata?: ObjMetadata) { - const stream = fse.createReadStream(fname); + public async uploadStream(key: string, inStream: stream.Readable, metadata?: ObjMetadata) { const result = await this._s3.putObject( - this.bucket, key, stream, undefined, + this.bucket, key, inStream, undefined, metadata ? {Metadata: toExternalMetadata(metadata)} : undefined ); // Empirically VersionId is available in result for buckets with versioning enabled. return result.versionId || null; } - public async download(key: string, fname: string, snapshotId?: string) { - const stream = fse.createWriteStream(fname); + public async upload(key: string, fname: string, metadata?: ObjMetadata) { + const filestream = fse.createReadStream(fname); + return this.uploadStream(key, filestream, metadata); + } + + public async downloadStream(key: string, outStream: stream.Writable, snapshotId?: string ) { const request = await this._s3.getObject( this.bucket, key, snapshotId ? {versionId: snapshotId} : {} @@ -114,20 +118,34 @@ export class MinIOExternalStorage implements ExternalStorage { return new Promise((resolve, reject) => { request .on('error', reject) // handle errors on the read stream - .pipe(stream) + .pipe(outStream) .on('error', reject) // handle errors on the write stream .on('finish', () => resolve(downloadedSnapshotId)); }); } + public async download(key: string, fname: string, snapshotId?: string) { + const fileStream = fse.createWriteStream(fname); + return this.downloadStream(key, fileStream, snapshotId); + } + public async remove(key: string, snapshotIds?: string[]) { if (snapshotIds) { - await this._deleteBatch(key, snapshotIds); + await this._deleteVersions(key, snapshotIds); } else { await this._deleteAllVersions(key); } } + public async removeAllWithPrefix(prefix: string) { + const objects = await this._listObjects(this.bucket, prefix, true, { IncludeVersion: true }); + const objectsToDelete = objects.filter(o => o.name !== undefined).map(o => ({ + name: o.name!, + versionId: (o as any).versionId as (string | undefined), + })); + await this._deleteObjects(objectsToDelete); + } + public async hasVersioning(): Promise { const versioning = await this._s3.getBucketVersioning(this.bucket); // getBucketVersioning() may return an empty string when versioning has never been enabled. @@ -136,18 +154,7 @@ export class MinIOExternalStorage implements ExternalStorage { } public async versions(key: string, options?: { includeDeleteMarkers?: boolean }) { - const results: minio.BucketItem[] = []; - await new Promise((resolve, reject) => { - const stream = this._s3.listObjects(this.bucket, key, false, {IncludeVersion: true}); - stream - .on('error', reject) - .on('end', () => { - resolve(results); - }) - .on('data', data => { - results.push(data); - }); - }); + const results = await this._listObjects(this.bucket, key, false, {IncludeVersion: true}); return results .filter(v => v.name === key && v.lastModified && (v as any).versionId && @@ -182,21 +189,38 @@ export class MinIOExternalStorage implements ExternalStorage { // Delete all versions of an object. public async _deleteAllVersions(key: string) { const vs = await this.versions(key, {includeDeleteMarkers: true}); - await this._deleteBatch(key, vs.map(v => v.snapshotId)); + await this._deleteVersions(key, vs.map(v => v.snapshotId)); } // Delete a batch of versions for an object. - private async _deleteBatch(key: string, versions: Array) { + private async _deleteVersions(key: string, versions: Array) { + return this._deleteObjects( + versions.filter(v => v).map(versionId => ({ + name: key, + versionId, + })) + ); + } + + // Delete an arbitrary number of objects, batched appropriately. + private async _deleteObjects(objects: { name: string, versionId?: string }[]): Promise { // Max number of keys per request for AWS S3 is 1000, see: // https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html // Stick to this maximum in case we are using this client to talk to AWS. const N = this._batchSize || 1000; - for (let i = 0; i < versions.length; i += N) { - const iVersions = versions.slice(i, i + N).filter(v => v) as string[]; - if (iVersions.length === 0) { continue; } - await this._s3.removeObjects(this.bucket, iVersions.map(versionId => { - return { name: key, versionId }; - })); + for (let i = 0; i < objects.length; i += N) { + const batch = objects.slice(i, i + N); + if (batch.length === 0) { continue; } + await this._s3.removeObjects(this.bucket, batch); + } + } + + private async _listObjects(...args: Parameters): Promise { + const bucketItemStream = this._s3.listObjects(...args); + const results: minio.BucketItem[] = []; + for await (const data of bucketItemStream) { + results.push(data); } + return results; } } diff --git a/app/server/lib/checksumFile.ts b/app/server/lib/checksumFile.ts index dfd6241af1..84af3247ed 100644 --- a/app/server/lib/checksumFile.ts +++ b/app/server/lib/checksumFile.ts @@ -1,4 +1,5 @@ import {createHash} from 'crypto'; +import {Readable} from 'node:stream'; import * as fs from 'fs'; /** @@ -6,8 +7,12 @@ import * as fs from 'fs'; * supported by crypto.createHash(). */ export async function checksumFile(filePath: string, algorithm: string = 'sha1'): Promise { - const shaSum = createHash(algorithm); const stream = fs.createReadStream(filePath); + return checksumFileStream(stream, algorithm); +} + +export async function checksumFileStream(stream: Readable, algorithm: string = 'sha1'): Promise { + const shaSum = createHash(algorithm); try { stream.on('data', (data) => shaSum.update(data)); await new Promise((resolve, reject) => { diff --git a/app/server/lib/coreCreator.ts b/app/server/lib/coreCreator.ts index 2eda4e9f9b..5a0da0039f 100644 --- a/app/server/lib/coreCreator.ts +++ b/app/server/lib/coreCreator.ts @@ -1,7 +1,15 @@ -import { checkMinIOBucket, checkMinIOExternalStorage, - configureMinIOExternalStorage } from 'app/server/lib/configureMinIOExternalStorage'; -import { makeSimpleCreator } from 'app/server/lib/ICreate'; -import { Telemetry } from 'app/server/lib/Telemetry'; +import { + AttachmentStoreCreationError, + ExternalStorageAttachmentStore +} from 'app/server/lib/AttachmentStore'; +import { + checkMinIOBucket, + checkMinIOExternalStorage, + configureMinIOExternalStorage +} from 'app/server/lib/configureMinIOExternalStorage'; +import {makeSimpleCreator} from 'app/server/lib/ICreate'; +import {MinIOExternalStorage} from 'app/server/lib/MinIOExternalStorage'; +import {Telemetry} from 'app/server/lib/Telemetry'; export const makeCoreCreator = () => makeSimpleCreator({ deploymentType: 'core', @@ -13,6 +21,23 @@ export const makeCoreCreator = () => makeSimpleCreator({ create: configureMinIOExternalStorage, }, ], + attachmentStoreOptions: [ + { + name: 'minio', + isAvailable: async () => checkMinIOExternalStorage() !== undefined, + create: async (storeId: string) => { + const options = checkMinIOExternalStorage(); + if (!options) { + throw new AttachmentStoreCreationError('minio', storeId, 'MinIO storage not configured'); + } + return new ExternalStorageAttachmentStore( + storeId, + new MinIOExternalStorage(options.bucket, options), + [options?.prefix || "", "attachments"] + ); + } + } + ], telemetry: { create: (dbManager, gristServer) => new Telemetry(dbManager, gristServer), } diff --git a/app/server/utils/MemoryWritableStream.ts b/app/server/utils/MemoryWritableStream.ts new file mode 100644 index 0000000000..fd0f81f78c --- /dev/null +++ b/app/server/utils/MemoryWritableStream.ts @@ -0,0 +1,21 @@ +import {Writable} from 'stream'; + +// Creates a writable stream that can be retrieved as a buffer. +// Sub-optimal implementation, as we end up with *at least* two copies in memory one in `buffers`, +// and one produced by `Buffer.concat` at the end. +export class MemoryWritableStream extends Writable { + private _buffers: Buffer[] = []; + + public getBuffer(): Buffer { + return Buffer.concat(this._buffers); + } + + public _write(chunk: any, encoding: BufferEncoding, callback: (error?: (Error | null)) => void) { + if (typeof (chunk) == "string") { + this._buffers.push(Buffer.from(chunk, encoding)); + } else { + this._buffers.push(chunk); + } + callback(); + } +} diff --git a/app/server/utils/gristify.ts b/app/server/utils/gristify.ts index 5a57a5b147..57a0c68fd1 100644 --- a/app/server/utils/gristify.ts +++ b/app/server/utils/gristify.ts @@ -1,10 +1,11 @@ -import { ColInfoWithId } from 'app/common/DocActions'; -import { ActiveDoc } from 'app/server/lib/ActiveDoc'; -import { DocManager } from 'app/server/lib/DocManager'; -import { makeExceptionalDocSession, OptDocSession } from 'app/server/lib/DocSession'; -import { createDummyGristServer } from 'app/server/lib/GristServer'; -import { TrivialDocStorageManager } from 'app/server/lib/IDocStorageManager'; -import { DBMetadata, quoteIdent, SQLiteDB } from 'app/server/lib/SQLiteDB'; +import {ColInfoWithId} from 'app/common/DocActions'; +import {ActiveDoc} from 'app/server/lib/ActiveDoc'; +import {AttachmentStoreProvider} from 'app/server/lib/AttachmentStoreProvider'; +import {DocManager} from 'app/server/lib/DocManager'; +import {makeExceptionalDocSession, OptDocSession} from 'app/server/lib/DocSession'; +import {createDummyGristServer} from 'app/server/lib/GristServer'; +import {TrivialDocStorageManager} from 'app/server/lib/IDocStorageManager'; +import {DBMetadata, quoteIdent, SQLiteDB} from 'app/server/lib/SQLiteDB'; /** * A utility class for modifying a SQLite file to be viewed/edited with Grist. @@ -52,7 +53,7 @@ export class Gristifier { // Open the file as an empty Grist document, creating Grist metadata // tables. const docManager = new DocManager( - new TrivialDocStorageManager(), null, null, createDummyGristServer() + new TrivialDocStorageManager(), null, null, new AttachmentStoreProvider([], ""), createDummyGristServer() ); const activeDoc = new ActiveDoc(docManager, this._filename); const docSession = makeExceptionalDocSession('system'); diff --git a/test/fixtures/docs/BlobMigrationV9.grist b/test/fixtures/docs/BlobMigrationV9.grist new file mode 100644 index 0000000000..3582467b17 Binary files /dev/null and b/test/fixtures/docs/BlobMigrationV9.grist differ diff --git a/test/fixtures/docs/DefaultValuesV9.grist b/test/fixtures/docs/DefaultValuesV9.grist new file mode 100644 index 0000000000..5a20dc7378 Binary files /dev/null and b/test/fixtures/docs/DefaultValuesV9.grist differ diff --git a/test/fixtures/docs/Hello.grist b/test/fixtures/docs/Hello.grist index ca9c56299e..5fa4942b4c 100644 Binary files a/test/fixtures/docs/Hello.grist and b/test/fixtures/docs/Hello.grist differ diff --git a/test/server/docTools.ts b/test/server/docTools.ts index 72043ff8d4..20576f1471 100644 --- a/test/server/docTools.ts +++ b/test/server/docTools.ts @@ -1,7 +1,9 @@ import {Role} from 'app/common/roles'; import {getDocWorkerMap} from 'app/gen-server/lib/DocWorkerMap'; import {ActiveDoc} from 'app/server/lib/ActiveDoc'; +import {AttachmentStoreProvider, IAttachmentStoreProvider} from 'app/server/lib/AttachmentStoreProvider'; import {DummyAuthorizer} from 'app/server/lib/Authorizer'; +import {create} from 'app/server/lib/create'; import {DocManager} from 'app/server/lib/DocManager'; import {DocSession, makeExceptionalDocSession} from 'app/server/lib/DocSession'; import {createDummyGristServer, GristServer} from 'app/server/lib/GristServer'; @@ -16,7 +18,6 @@ import * as fse from 'fs-extra'; import {tmpdir} from 'os'; import * as path from 'path'; import * as tmp from 'tmp'; -import {create} from "app/server/lib/create"; tmp.setGracefulCleanup(); @@ -135,15 +136,18 @@ export function createDocTools(options: {persistAcrossCases?: boolean, export async function createDocManager( options: {tmpDir?: string, pluginManager?: PluginManager, storageManager?: IDocStorageManager, - server?: GristServer} = {}): Promise { + server?: GristServer, + attachmentStoreProvider?: IAttachmentStoreProvider, + } = {}): Promise { // Set Grist home to a temporary directory, and wipe it out on exit. const tmpDir = options.tmpDir || await createTmpDir(); const docStorageManager = options.storageManager || await create.createLocalDocStorageManager(tmpDir); const pluginManager = options.pluginManager || await getGlobalPluginManager(); + const attachmentStoreProvider = options.attachmentStoreProvider || new AttachmentStoreProvider([], "TEST_INSTALL"); const store = getDocWorkerMap(); const internalPermitStore = store.getPermitStore('1'); const externalPermitStore = store.getPermitStore('2'); - return new DocManager(docStorageManager, pluginManager, null, options.server || { + return new DocManager(docStorageManager, pluginManager, null, attachmentStoreProvider, options.server || { ...createDummyGristServer(), getPermitStore() { return internalPermitStore; }, getExternalPermitStore() { return externalPermitStore; }, diff --git a/test/server/lib/ActiveDoc.ts b/test/server/lib/ActiveDoc.ts index 993590be58..8babb27b96 100644 --- a/test/server/lib/ActiveDoc.ts +++ b/test/server/lib/ActiveDoc.ts @@ -1,27 +1,28 @@ -import { getEnvContent } from 'app/common/ActionBundle'; -import { ServerQuery } from 'app/common/ActiveDocAPI'; -import { delay } from 'app/common/delay'; -import { BulkColValues, CellValue, fromTableDataAction } from 'app/common/DocActions'; +import {getEnvContent} from 'app/common/ActionBundle'; +import {ServerQuery} from 'app/common/ActiveDocAPI'; +import {delay} from 'app/common/delay'; +import {BulkColValues, CellValue, fromTableDataAction} from 'app/common/DocActions'; import * as gristTypes from 'app/common/gristTypes'; -import { GristObjCode } from 'app/plugin/GristData'; -import { TableData } from 'app/common/TableData'; -import { ActiveDoc } from 'app/server/lib/ActiveDoc'; -import { DummyAuthorizer } from 'app/server/lib/Authorizer'; -import { Client } from 'app/server/lib/Client'; -import { makeExceptionalDocSession, OptDocSession } from 'app/server/lib/DocSession'; +import {GristObjCode} from 'app/plugin/GristData'; +import {TableData} from 'app/common/TableData'; +import {ActiveDoc} from 'app/server/lib/ActiveDoc'; +import {AttachmentStoreProvider} from 'app/server/lib/AttachmentStoreProvider'; +import {DummyAuthorizer} from 'app/server/lib/Authorizer'; +import {Client} from 'app/server/lib/Client'; +import {makeExceptionalDocSession, OptDocSession} from 'app/server/lib/DocSession'; import log from 'app/server/lib/log'; -import { timeoutReached } from 'app/server/lib/serverUtils'; -import { Throttle } from 'app/server/lib/Throttle'; -import { promisify } from 'bluebird'; -import { assert } from 'chai'; +import {timeoutReached} from 'app/server/lib/serverUtils'; +import {Throttle} from 'app/server/lib/Throttle'; +import {promisify} from 'bluebird'; +import {assert} from 'chai'; import * as child_process from 'child_process'; import * as fse from 'fs-extra'; import * as _ from 'lodash'; -import { resolve } from 'path'; +import {resolve} from 'path'; import * as sinon from 'sinon'; -import { createDocTools } from 'test/server/docTools'; +import {createDocTools} from 'test/server/docTools'; import * as testUtils from 'test/server/testUtils'; -import { EnvironmentSnapshot } from 'test/server/testUtils'; +import {EnvironmentSnapshot} from 'test/server/testUtils'; import * as tmp from 'tmp'; const execFileAsync = promisify(child_process.execFile); @@ -1132,7 +1133,8 @@ describe('ActiveDoc', function() { 'https://templates!.getgrist.com/doc/lightweight-crm 8sJPiNkWZo68KFJkc5Ukbr~4' ] as const) { const activeDoc = new ActiveDoc(docTools.getDocManager(), 'docUrlTest' + docUrl.length, - { docUrl }); + new AttachmentStoreProvider([], "TEST-INSTALL-ID"), + { docUrl }); await activeDoc.createEmptyDoc(fakeSession); await activeDoc.applyUserActions(fakeSession, [ ["AddTable", "Info", [{id: 'Url', formula: 'SELF_HYPERLINK()'}]], diff --git a/test/server/lib/AttachmentFileManager.ts b/test/server/lib/AttachmentFileManager.ts new file mode 100644 index 0000000000..d7be3c5201 --- /dev/null +++ b/test/server/lib/AttachmentFileManager.ts @@ -0,0 +1,213 @@ +import { DocStorage, FileInfo } from "app/server/lib/DocStorage"; +import { + AttachmentFileManager, AttachmentRetrievalError, + StoreNotAvailableError, + StoresNotConfiguredError +} from "app/server/lib/AttachmentFileManager"; +import { AttachmentStoreProvider, IAttachmentStoreProvider } from "app/server/lib/AttachmentStoreProvider"; +import { makeTestingFilesystemStoreSpec } from "./FilesystemAttachmentStore"; +import { assert } from "chai"; +import * as sinon from "sinon"; + +// Minimum features of doc storage that are needed to make AttachmentFileManager work. +type IMinimalDocStorage = Pick + +// Implements the minimal functionality needed for the AttachmentFileManager to work. +class DocStorageFake implements IMinimalDocStorage { + private _files: { [key: string]: FileInfo } = {}; + + constructor(public docName: string) { + } + + public async getFileInfo(fileIdent: string): Promise { + return this._files[fileIdent] ?? null; + } + + // Return value is true if the file was newly added. + public async findOrAttachFile( + fileIdent: string, fileData: Buffer | undefined, storageId?: string | undefined + ): Promise { + if (fileIdent in this._files) { + return false; + } + this._files[fileIdent] = { + ident: fileIdent, + data: fileData ?? Buffer.alloc(0), + storageId: storageId ?? null, + }; + return true; + } +} + +const defaultTestDocName = "1234"; +const defaultTestFileContent = "Some content"; + +function createDocStorageFake(docName: string): DocStorage { + return new DocStorageFake(docName) as unknown as DocStorage; +} + +async function createFakeAttachmentStoreProvider(): Promise { + return new AttachmentStoreProvider( + [await makeTestingFilesystemStoreSpec("filesystem")], + "TEST-INSTALLATION-UUID" + ); +} + +describe("AttachmentFileManager", function() { + let defaultProvider: IAttachmentStoreProvider; + let defaultDocStorageFake: DocStorage; + + beforeEach(async function() { + defaultProvider = await createFakeAttachmentStoreProvider(); + defaultDocStorageFake = createDocStorageFake(defaultTestDocName); + }); + + it("should throw if uses an external store when no document pool id is available", async function () { + const manager = new AttachmentFileManager( + defaultDocStorageFake, + defaultProvider, + undefined, + ); + + const storeId = defaultProvider.listAllStoreIds()[0]; + + await assert.isRejected(manager.addFile(storeId, ".txt", Buffer.alloc(0)), StoresNotConfiguredError); + }); + + it("should throw if it tries to add a file to an unavailable store", async function () { + const manager = new AttachmentFileManager( + defaultDocStorageFake, + defaultProvider, + { id: "Unimportant", trunkId: null }, + ); + + await assert.isRejected(manager.addFile("BAD STORE ID", ".txt", Buffer.alloc(0)), StoreNotAvailableError); + }); + + it("should throw if it tries to get a file from an unavailable store", async function() { + const manager = new AttachmentFileManager( + defaultDocStorageFake, + defaultProvider, + { id: "Unimportant", trunkId: null }, + ); + + const fileId = "123456.png"; + await defaultDocStorageFake.findOrAttachFile(fileId, undefined, "SOME-STORE-ID"); + + await assert.isRejected(manager.getFileData(fileId), StoreNotAvailableError); + }); + + it("should add a file to local document storage if no store id is provided", async function() { + const manager = new AttachmentFileManager( + defaultDocStorageFake, + defaultProvider, + { id: "Unimportant", trunkId: null }, + ); + + const result = await manager.addFile(undefined, ".txt", Buffer.from(defaultTestFileContent)); + + // Checking the file is present in the document storage. + const fileInfo = await defaultDocStorageFake.getFileInfo(result.fileIdent); + assert.equal(fileInfo?.data.toString(), "Some content"); + }); + + it("should add a file to an available attachment store", async function() { + const docId = "12345"; + const manager = new AttachmentFileManager( + defaultDocStorageFake, + defaultProvider, + { id: docId, trunkId: null }, + ); + + const storeId = defaultProvider.listAllStoreIds()[0]; + const result = await manager.addFile(storeId, ".txt", Buffer.from(defaultTestFileContent)); + + const store = await defaultProvider.getStore(storeId); + assert.isTrue(await store!.exists(docId, result.fileIdent), "file does not exist in store"); + }); + + it("should get a file from local storage", async function() { + const docId = "12345"; + const manager = new AttachmentFileManager( + defaultDocStorageFake, + defaultProvider, + { id: docId, trunkId: null }, + ); + + const result = await manager.addFile(undefined, ".txt", Buffer.from(defaultTestFileContent)); + const fileData = await manager.getFileData(result.fileIdent); + + assert.equal(fileData?.toString(), defaultTestFileContent, "downloaded file contents do not match original file"); + }); + + it("should get a file from an attachment store", async function() { + const docId = "12345"; + const manager = new AttachmentFileManager( + defaultDocStorageFake, + defaultProvider, + { id: docId, trunkId: null }, + ); + + const storeIds = defaultProvider.listAllStoreIds(); + const result = await manager.addFile(storeIds[0], ".txt", Buffer.from(defaultTestFileContent)); + const fileData = await manager.getFileData(result.fileIdent); + const fileInfo = await defaultDocStorageFake.getFileInfo(result.fileIdent); + assert.equal(fileInfo?.storageId, storeIds[0]); + // Ideally this should be null, but the current fake returns an empty buffer. + assert.equal(fileInfo?.data.length, 0); + assert.equal(fileData?.toString(), defaultTestFileContent, "downloaded file contents do not match original file"); + }); + + it("should detect existing files and not upload them", async function () { + const docId = "12345"; + const manager = new AttachmentFileManager( + defaultDocStorageFake, + defaultProvider, + { id: docId, trunkId: null }, + ); + + const storeId = defaultProvider.listAllStoreIds()[0]; + const addFile = () => manager.addFile(storeId, ".txt", Buffer.from(defaultTestFileContent)); + + const addFileResult1 = await addFile(); + assert.isTrue(addFileResult1.isNewFile); + + // Makes the store's upload method throw an error, so we can detect if it gets called. + const originalGetStore = defaultProvider.getStore.bind(defaultProvider); + sinon.replace(defaultProvider, 'getStore', sinon.fake( + async function(...args: Parameters) { + const store = (await originalGetStore(...args))!; + sinon.replace(store, 'upload', () => { throw new Error("Upload should never be called"); }); + return store; + } + )); + + const addFileResult2 = await addFile(); + assert.isFalse(addFileResult2.isNewFile); + }); + + it("should check if an existing file is in the attachment store, and re-upload them if not", async function() { + const docId = "12345"; + const manager = new AttachmentFileManager( + defaultDocStorageFake, + defaultProvider, + { id: docId, trunkId: null }, + ); + + const storeId = defaultProvider.listAllStoreIds()[0]; + const store = (await defaultProvider.getStore(storeId))!; + const addFile = () => manager.addFile(storeId, ".txt", Buffer.from(defaultTestFileContent)); + + const addFileResult = await addFile(); + // This might be overkill, but it works. + await store.removePool(docId); + + await assert.isRejected(manager.getFileData(addFileResult.fileIdent), AttachmentRetrievalError); + + await addFile(); + + const fileData = await manager.getFileData(addFileResult.fileIdent); + assert(fileData); + assert.equal(fileData.toString(), defaultTestFileContent); + }); +}); diff --git a/test/server/lib/AttachmentStoreProvider.ts b/test/server/lib/AttachmentStoreProvider.ts new file mode 100644 index 0000000000..eb14db7647 --- /dev/null +++ b/test/server/lib/AttachmentStoreProvider.ts @@ -0,0 +1,37 @@ +import {assert} from 'chai'; +import {AttachmentStoreProvider} from 'app/server/lib/AttachmentStoreProvider'; +import {makeTestingFilesystemStoreSpec} from './FilesystemAttachmentStore'; + +const testInstallationUUID = "FAKE-UUID"; +function expectedStoreId(type: string) { + return `${testInstallationUUID}-${type}`; +} + +describe('AttachmentStoreProvider', () => { + it('constructs stores using the installations UUID and store type', async () => { + const filesystemType1 = await makeTestingFilesystemStoreSpec("filesystem1"); + const filesystemType2 = await makeTestingFilesystemStoreSpec("filesystem2"); + + const provider = new AttachmentStoreProvider([filesystemType1, filesystemType2], testInstallationUUID); + const allStores = await provider.getAllStores(); + const ids = allStores.map(store => store.id); + + assert.includeMembers(ids, [expectedStoreId("filesystem1"), expectedStoreId("filesystem2")]); + }); + + it("can retrieve a store if it exists", async () => { + const filesystemSpec = await makeTestingFilesystemStoreSpec("filesystem"); + const provider = new AttachmentStoreProvider([filesystemSpec], testInstallationUUID); + + assert.isNull(await provider.getStore("doesn't exist"), "store shouldn't exist"); + + assert(await provider.getStore(expectedStoreId("filesystem")), "store not present"); + }); + + it("can check if a store exists", async () => { + const filesystemSpec = await makeTestingFilesystemStoreSpec("filesystem"); + const provider = new AttachmentStoreProvider([filesystemSpec], testInstallationUUID); + + assert(await provider.storeExists(expectedStoreId("filesystem"))); + }); +}); diff --git a/test/server/lib/DocStorage.js b/test/server/lib/DocStorage.js index 78e9fa825e..7f09e5d38c 100644 --- a/test/server/lib/DocStorage.js +++ b/test/server/lib/DocStorage.js @@ -389,19 +389,18 @@ describe('DocStorage', function() { var docStorage; it("should create attachment blob", function() { docStorage = new DocStorage(docStorageManager, 'test_Attachments'); + const correctFileContents = "Hello, world!" return docStorage.createFile() - .then(() => testUtils.writeTmpFile("Hello, world!")) - .then(tmpPath => docStorage.findOrAttachFile(tmpPath, "hello_world.txt")) + .then(() => docStorage.findOrAttachFile( "hello_world.txt", Buffer.from(correctFileContents))) .then(result => assert.isTrue(result)) - .then(() => docStorage.getFileData("hello_world.txt")) - .then(data => assert.equal(data.toString('utf8'), "Hello, world!")) + .then(() => docStorage.getFileInfo("hello_world.txt")) + .then(fileInfo => assert.equal(fileInfo.data.toString('utf8'), correctFileContents)) // If we use the same fileIdent for another file, it should not get attached. - .then(() => testUtils.writeTmpFile("Another file")) - .then(tmpPath => docStorage.findOrAttachFile(tmpPath, "hello_world.txt")) + .then(() => docStorage.findOrAttachFile("hello_world.txt", Buffer.from("Another file"))) .then(result => assert.isFalse(result)) - .then(() => docStorage.getFileData("hello_world.txt")) - .then(data => assert.equal(data.toString('utf8'), "Hello, world!")); + .then(() => docStorage.getFileInfo("hello_world.txt")) + .then(fileInfo => assert.equal(fileInfo.data.toString('utf8'), correctFileContents)); }); }); diff --git a/test/server/lib/DocStorageMigrations.ts b/test/server/lib/DocStorageMigrations.ts index aca14982cf..68f887ca23 100644 --- a/test/server/lib/DocStorageMigrations.ts +++ b/test/server/lib/DocStorageMigrations.ts @@ -39,11 +39,11 @@ describe('DocStorageMigrations', function() { } it('should migrate from v1 correctly', function() { - return testMigration('BlobMigrationV1.grist', 'BlobMigrationV8.grist'); + return testMigration('BlobMigrationV1.grist', 'BlobMigrationV9.grist'); }); it('should migrate from v2 correctly', function() { - return testMigration('BlobMigrationV2.grist', 'BlobMigrationV8.grist'); + return testMigration('BlobMigrationV2.grist', 'BlobMigrationV9.grist'); }); it('should migrate from v3 correctly', async function() { @@ -69,11 +69,11 @@ describe('DocStorageMigrations', function() { // Also do the test to check out the full document against a saved copy. To know if the copy // makes sense, run in test/fixtures/docs: // diff -u <(sqlite3 BlobMigrationV3.grist .dump) <(sqlite3 BlobMigrationV4.grist .dump) - await testMigration('BlobMigrationV3.grist', 'BlobMigrationV8.grist'); + await testMigration('BlobMigrationV3.grist', 'BlobMigrationV9.grist'); }); it('should migrate from v4 correctly', function() { - return testMigration('BlobMigrationV4.grist', 'BlobMigrationV8.grist'); + return testMigration('BlobMigrationV4.grist', 'BlobMigrationV9.grist'); }); it('should migrate from v5 correctly', async function() { @@ -83,7 +83,7 @@ describe('DocStorageMigrations', function() { // // Verify correctness of these fixture files with: // diff -u <(sqlite3 DefaultValuesV5.grist .dump) <(sqlite3 DefaultValuesV7.grist .dump) - return testMigration('DefaultValuesV5.grist', 'DefaultValuesV8.grist'); + return testMigration('DefaultValuesV5.grist', 'DefaultValuesV9.grist'); }); it('should migrate from v6 correctly', async function() { @@ -93,7 +93,7 @@ describe('DocStorageMigrations', function() { // Verify correctness of updated fixture files with, for instance: // cd test/fixtures/docs ; \ // diff -u <(sqlite3 DefaultValuesV6.grist .dump) <(sqlite3 DefaultValuesV7.grist .dump) - await testMigration('BlobMigrationV6.grist', 'BlobMigrationV8.grist'); - await testMigration('DefaultValuesV6.grist', 'DefaultValuesV8.grist'); + await testMigration('BlobMigrationV6.grist', 'BlobMigrationV9.grist'); + await testMigration('DefaultValuesV6.grist', 'DefaultValuesV9.grist'); }); }); diff --git a/test/server/lib/FilesystemAttachmentStore.ts b/test/server/lib/FilesystemAttachmentStore.ts new file mode 100644 index 0000000000..7f2cd31ea9 --- /dev/null +++ b/test/server/lib/FilesystemAttachmentStore.ts @@ -0,0 +1,74 @@ +import {FilesystemAttachmentStore} from 'app/server/lib/AttachmentStore'; +import {MemoryWritableStream} from 'app/server/utils/MemoryWritableStream'; +import {createTmpDir} from 'test/server/docTools'; + +import {assert} from 'chai'; +import {mkdtemp, pathExists} from 'fs-extra'; +import * as stream from 'node:stream'; +import * as path from 'path'; + +const testingDocPoolId = "1234-5678"; +const testingFileId = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.grist"; +const testingFileContents = "Grist is the best tool ever."; + +function getTestingFileAsBuffer(contents: string = testingFileContents) { + return Buffer.from(contents, 'utf8'); +} + +function getTestingFileAsReadableStream(contents?: string): stream.Readable { + return stream.Readable.from(getTestingFileAsBuffer(contents)); +} + +export async function makeTestingFilesystemStoreSpec( + name: string = "filesystem" +) { + const tempFolder = await createTmpDir(); + const tempDir = await mkdtemp(path.join(tempFolder, 'filesystem-store-test-')); + return { + rootDirectory: tempDir, + name, + create: async (storeId: string) => (new FilesystemAttachmentStore(storeId, tempDir)) + }; +} + +describe('FilesystemAttachmentStore', () => { + it('can upload a file', async () => { + const spec = await makeTestingFilesystemStoreSpec(); + const store = await spec.create("test-filesystem-store"); + await store.upload(testingDocPoolId, testingFileId, getTestingFileAsReadableStream()); + + const exists = await pathExists(path.join(spec.rootDirectory, testingDocPoolId, testingFileId)); + assert.isTrue(exists, "uploaded file does not exist on the filesystem"); + }); + + it('can download a file', async () => { + const spec = await makeTestingFilesystemStoreSpec(); + const store = await spec.create("test-filesystem-store"); + await store.upload(testingDocPoolId, testingFileId, getTestingFileAsReadableStream()); + + const outputBuffer = new MemoryWritableStream(); + await store.download(testingDocPoolId, testingFileId, outputBuffer); + + assert.equal(outputBuffer.getBuffer().toString(), testingFileContents, "file contents do not match"); + }); + + it('can check if a file exists', async () => { + const spec = await makeTestingFilesystemStoreSpec(); + const store = await spec.create("test-filesystem-store"); + + assert.isFalse(await store.exists(testingDocPoolId, testingFileId)); + await store.upload(testingDocPoolId, testingFileId, getTestingFileAsReadableStream()); + assert.isTrue(await store.exists(testingDocPoolId, testingFileId)); + }); + + it('can remove an entire pool', async () => { + const spec = await makeTestingFilesystemStoreSpec(); + const store = await spec.create("test-filesystem-store"); + await store.upload(testingDocPoolId, testingFileId, getTestingFileAsReadableStream()); + assert.isTrue(await store.exists(testingDocPoolId, testingFileId)); + + await store.removePool(testingDocPoolId); + + assert.isFalse(await store.exists(testingDocPoolId, testingFileId)); + }); +}); diff --git a/test/server/lib/HostedStorageManager.ts b/test/server/lib/HostedStorageManager.ts index 8aa8fd5a86..44359259cb 100644 --- a/test/server/lib/HostedStorageManager.ts +++ b/test/server/lib/HostedStorageManager.ts @@ -4,6 +4,10 @@ import {SCHEMA_VERSION} from 'app/common/schema'; import {DocWorkerMap, getDocWorkerMap} from 'app/gen-server/lib/DocWorkerMap'; import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager'; import {ActiveDoc} from 'app/server/lib/ActiveDoc'; +import { + AttachmentStoreProvider, + IAttachmentStoreProvider +} from 'app/server/lib/AttachmentStoreProvider'; import {create} from 'app/server/lib/create'; import {DocManager} from 'app/server/lib/DocManager'; import {makeExceptionalDocSession} from 'app/server/lib/DocSession'; @@ -276,7 +280,8 @@ class TestStore { private _localDirectory: string, private _workerId: string, private _workers: IDocWorkerMap, - private _externalStorageCreate: ExternalStorageCreator) { + private _externalStorageCreate: ExternalStorageCreator, + private _attachmentStoreProvider?: IAttachmentStoreProvider) { } public async run(fn: () => Promise): Promise { @@ -310,6 +315,8 @@ class TestStore { return result; }; + const attachmentStoreProvider = this._attachmentStoreProvider ?? new AttachmentStoreProvider([], "TESTINSTALL"); + const storageManager = new HostedStorageManager(this._localDirectory, this._workerId, false, @@ -318,8 +325,8 @@ class TestStore { externalStorageCreator, options); this.storageManager = storageManager; - this.docManager = new DocManager(storageManager, await getGlobalPluginManager(), - dbManager, { + this.docManager = new DocManager(storageManager, await getGlobalPluginManager(), dbManager, attachmentStoreProvider, + { ...createDummyGristServer(), getStorageManager() { return storageManager; }, }); diff --git a/test/upgradeDocumentImpl.ts b/test/upgradeDocumentImpl.ts index 499c28a0e2..f4cb037e4d 100644 --- a/test/upgradeDocumentImpl.ts +++ b/test/upgradeDocumentImpl.ts @@ -4,15 +4,59 @@ * Usage: * test/upgradeDocument */ +import {DocStorage} from 'app/server/lib/DocStorage'; +import {DocStorageManager} from 'app/server/lib/DocStorageManager'; import {copyFile} from 'app/server/lib/docUtils'; import {createDocTools} from 'test/server/docTools'; import log from 'app/server/lib/log'; import * as fs from "fs"; +import * as fse from "fs-extra"; +import * as path from "path"; +import * as tmp from "tmp"; + +export async function upgradeDocuments(docPaths: string[]): Promise { + const docTools = createDocTools(); + await docTools.before(); + try { + for (const docPath of docPaths) { + console.log(`Upgrading ${docPath}`); + const activeDoc = await docTools.loadLocalDoc(docPath); + await activeDoc.waitForInitialization(); + await activeDoc.shutdown(); + await copyFile(docTools.getStorageManager().getPath(activeDoc.docName), docPath); + } + } finally { + await docTools.after(); + } +} + +export async function upgradeDocumentsDocStorageOnly(paths: string[]): Promise { + let tmpDir = await tmp.dirAsync({ prefix: 'grist_migrate_', unsafeCleanup: true }); + tmpDir = await fse.realpath(tmpDir); + const docStorageManager = new DocStorageManager(tmpDir); + + for (const docPath of paths) { + console.log(`Upgrading '${docPath}' (DocStorage migrations only)`); + const docName = path.basename(docPath); + const tempPath = docStorageManager.getPath(docName); + fs.copyFileSync(docPath, tempPath); + + const docStorage = new DocStorage(docStorageManager, docName); + await docStorage.openFile(); + await docStorage.shutdown(); + + fs.copyFileSync(tempPath, docPath); + } +} export async function main() { - const docPaths = process.argv.slice(2); + const params = process.argv.slice(2); + const onlyRunDocStorageMigrations = params.map((text) => text.toLowerCase()).includes("--doc-storage-only"); + const docPaths = params.filter((text) => text.trim()[0] != "-"); if (docPaths.length === 0) { console.log(`Usage:\n test/upgradeDocument path/to/doc.grist ...\n`); + console.log(`Parameters: `); + console.log(` --doc-storage-only - Only runs DocStorage migrations`); throw new Error("Document argument required"); } for (const docPath of docPaths) { @@ -26,18 +70,13 @@ export async function main() { const prevLogLevel = log.transports.file.level; log.transports.file.level = 'warn'; - const docTools = createDocTools(); - await docTools.before(); try { - for (const docPath of docPaths) { - console.log(`Upgrading ${docPath}`); - const activeDoc = await docTools.loadLocalDoc(docPath); - await activeDoc.waitForInitialization(); - await activeDoc.shutdown(); - await copyFile(docTools.getStorageManager().getPath(activeDoc.docName), docPath); + if (onlyRunDocStorageMigrations) { + await upgradeDocumentsDocStorageOnly(docPaths); + } else { + await upgradeDocuments(docPaths); } } finally { - await docTools.after(); log.transports.file.level = prevLogLevel; } }