Initial Loader and driver changes for Data Virtualization

.changeset/four-tables-smell.md

@@ -0,0 +1,12 @@
+---
+"@fluidframework/container-definitions": minor
+"@fluidframework/container-loader": minor
+"@fluidframework/driver-definitions": minor
+"@fluidframework/driver-utils": minor
+"@fluidframework/file-driver": minor
+"@fluidframework/odsp-driver": minor
+---
+
+Deprecate `ISnapshotContents`
+
+`ISnapshotContents` is deprecated. It has been replaced with `ISnapshot`.

packages/common/container-definitions/api-report/container-definitions.api.md

@@ -24,6 +24,7 @@ import { IResolvedUrl } from '@fluidframework/driver-definitions';
 import { ISequencedDocumentMessage } from '@fluidframework/protocol-definitions';
 import { ISequencedProposal } from '@fluidframework/protocol-definitions';
 import { ISignalMessage } from '@fluidframework/protocol-definitions';
+import { ISnapshot } from '@fluidframework/driver-definitions';
 import { ISnapshotTree } from '@fluidframework/protocol-definitions';
 import { ISummaryContent } from '@fluidframework/protocol-definitions';
 import { ISummaryTree } from '@fluidframework/protocol-definitions';
@@ -176,6 +177,7 @@ export interface IContainerContext {
     // (undocumented)
     readonly quorum: IQuorumClients;
     readonly scope: FluidObject;
+    readonly snapshotWithContents?: ISnapshot;
     // (undocumented)
     readonly storage: IDocumentStorageService;
     // (undocumented)

packages/common/container-definitions/src/runtime.ts

@@ -5,7 +5,7 @@
 
 import { ITelemetryBaseLogger, IDisposable, FluidObject } from "@fluidframework/core-interfaces";
 
-import { IDocumentStorageService } from "@fluidframework/driver-definitions";
+import { IDocumentStorageService, ISnapshot } from "@fluidframework/driver-definitions";
 import {
 	IClientDetails,
 	ISequencedDocumentMessage,
@@ -198,6 +198,11 @@ export interface IContainerContext {
 	 * @privateremarks Tracking in AB#5714
 	 */
 	readonly id: string;
+
+	/**
+	 * This contains all parts of a snapshot like blobContents, ops etc.
+	 */
+	readonly snapshotWithContents?: ISnapshot;
 }
 
 /**

packages/common/driver-definitions/api-report/driver-definitions.api.md

@@ -176,12 +176,14 @@ export interface IDocumentServiceFactory {
 export interface IDocumentServicePolicies {
     readonly storageOnly?: boolean;
     readonly summarizeProtocolTree?: boolean;
+    readonly supportGetSnapshotApi?: boolean;
 }
 
 // @alpha
 export interface IDocumentStorageService extends Partial<IDisposable> {
     createBlob(file: ArrayBufferLike): Promise<ICreateBlobResponse>;
     downloadSummary(handle: ISummaryHandle): Promise<ISummaryTree>;
+    getSnapshot?(snapshotFetchOptions?: ISnapshotFetchOptions): Promise<ISnapshot>;
     getSnapshotTree(version?: IVersion, scenarioName?: string): Promise<ISnapshotTree | null>;
     getVersions(versionId: string | null, count: number, scenarioName?: string, fetchSource?: FetchSource): Promise<IVersion[]>;
     readonly policies?: IDocumentStorageServicePolicies;
@@ -255,6 +257,27 @@ export interface IResolvedUrl {
     url: string;
 }
 
+// @alpha (undocumented)
+export interface ISnapshot {
+    // (undocumented)
+    blobContents: Map<string, ArrayBuffer>;
+    latestSequenceNumber: number | undefined;
+    // (undocumented)
+    ops: ISequencedDocumentMessage[];
+    sequenceNumber: number | undefined;
+    // (undocumented)
+    snapshotFormatV: 1;
+    // (undocumented)
+    snapshotTree: ISnapshotTree;
+}
+
+// @alpha
+export interface ISnapshotFetchOptions {
+    cacheSnapshot?: boolean;
+    scenarioName?: string;
+    versionId?: string;
+}
+
 // @alpha
 export interface IStream<T> {
     // (undocumented)

packages/common/driver-definitions/src/index.ts

@@ -28,6 +28,8 @@ export {
 	IDocumentServicePolicies,
 	IDocumentStorageService,
 	IDocumentStorageServicePolicies,
+	ISnapshot,
+	ISnapshotFetchOptions,
 	IStream,
 	IStreamResult,
 	ISummaryContext,

packages/common/driver-definitions/src/storage.ts

@@ -161,6 +161,14 @@ export interface IDocumentStorageService extends Partial<IDisposable> {
 	// eslint-disable-next-line @rushstack/no-new-null
 	getSnapshotTree(version?: IVersion, scenarioName?: string): Promise<ISnapshotTree | null>;
 
+	/**
+	 * Returns the snapshot which can contain other artifacts too like blob contents, ops etc. It is different from
+	 * `getSnapshotTree` api in that, that API only returns the snapshot tree from the snapshot.
+	 * @param snapshotFetchOptions - Options specified by the caller to specify and want certain behavior from the
+	 * driver when fetching the snapshot.
+	 */
+	getSnapshot?(snapshotFetchOptions?: ISnapshotFetchOptions): Promise<ISnapshot>;
+
 	/**
 	 * Retrieves all versions of the document starting at the specified versionId - or null if from the head
 	 * @param versionId - Version id of the requested version.
@@ -341,6 +349,13 @@ export interface IDocumentServicePolicies {
 	 * Summarizer uploads the protocol tree too when summarizing.
 	 */
 	readonly summarizeProtocolTree?: boolean;
+
+	/**
+	 * Whether the driver supports the new getSnapshot api which returns snapshot which
+	 * contains all contents along with the snapshot tree. Enable this by default when the
+	 * driver can fully support the api.
+	 */
+	readonly supportGetSnapshotApi?: boolean;
 }
 
 /**
@@ -450,3 +465,48 @@ export enum FetchSource {
 	default = "default",
 	noCache = "noCache",
 }
+
+/**
+ * @alpha
+ */
+export interface ISnapshot {
+	snapshotTree: ISnapshotTree;
+	blobContents: Map<string, ArrayBuffer>;
+	ops: ISequencedDocumentMessage[];
+
+	/**
+	 * Sequence number of the snapshot
+	 */
+	sequenceNumber: number | undefined;
+
+	/**
+	 * Sequence number for the latest op/snapshot for the file in ODSP
+	 */
+	latestSequenceNumber: number | undefined;
+
+	snapshotFormatV: 1;
+}
+
+/**
+ * Snapshot fetch options which are used to communicate different things to the driver
+ * when fetching the snapshot.
+ * @alpha
+ */
+export interface ISnapshotFetchOptions {
+	/**
+	 * Indicates scenario in which the snapshot is fetched. It is a free form string  mostly
+	 * used for telemetry purposes.
+	 */
+	scenarioName?: string;
+	/**
+	 * Tell driver to cache the fetched snapshot. Driver is supposed to cache the fetched snapshot if this is
+	 * set to true. If undefined, then it is upto the driver, to cache it or not.
+	 */
+	cacheSnapshot?: boolean;
+
+	/**
+	 * Version of the snapshot to be fetched. Certain storage services just keep 1 snapshot for the
+	 * container, so specifying version is not necessary for storage services.
+	 */
+	versionId?: string;
+}

packages/drivers/file-driver/api-report/file-driver.api.md

@@ -22,6 +22,8 @@ import { IResolvedUrl } from '@fluidframework/driver-definitions';
 import { ISequencedDocumentMessage } from '@fluidframework/protocol-definitions';
 import { ISignalClient } from '@fluidframework/protocol-definitions';
 import { ISignalMessage } from '@fluidframework/protocol-definitions';
+import { ISnapshot } from '@fluidframework/driver-definitions';
+import { ISnapshotFetchOptions } from '@fluidframework/driver-definitions';
 import { IStream } from '@fluidframework/driver-definitions';
 import { ISummaryContext } from '@fluidframework/driver-definitions';
 import { ISummaryTree } from '@fluidframework/protocol-definitions';
@@ -63,6 +65,7 @@ export const FileSnapshotWriterClassFactory: <TBase extends ReaderConstructor>(B
         buildTree(snapshotTree: api.ISnapshotTree): Promise<api.ITree>;
         repositoryUrl: string;
         readonly policies?: IDocumentStorageServicePolicies | undefined;
+        getSnapshot?(snapshotFetchOptions?: ISnapshotFetchOptions | undefined): Promise<ISnapshot>;
         createBlob(file: ArrayBufferLike): Promise<api.ICreateBlobResponse>;
         downloadSummary(handle: api.ISummaryHandle): Promise<api.ISummaryTree>;
         readonly disposed?: boolean | undefined;
@@ -99,6 +102,7 @@ export const FluidFetchReaderFileSnapshotWriter: {
         buildTree(snapshotTree: api.ISnapshotTree): Promise<api.ITree>;
         repositoryUrl: string;
         readonly policies?: IDocumentStorageServicePolicies | undefined;
+        getSnapshot?(snapshotFetchOptions?: ISnapshotFetchOptions | undefined): Promise<ISnapshot>;
         createBlob(file: ArrayBufferLike): Promise<api.ICreateBlobResponse>;
         downloadSummary(handle: api.ISummaryHandle): Promise<api.ISummaryTree>;
         readonly disposed?: boolean | undefined;

packages/drivers/odsp-driver/api-report/odsp-driver.api.md

@@ -20,6 +20,7 @@ import { IRequest } from '@fluidframework/core-interfaces';
 import { IResolvedUrl } from '@fluidframework/driver-definitions';
 import { ISequencedDocumentMessage } from '@fluidframework/protocol-definitions';
 import { ISharingLinkKind } from '@fluidframework/odsp-driver-definitions';
+import { ISnapshot } from '@fluidframework/driver-definitions';
 import { ISnapshotOptions } from '@fluidframework/odsp-driver-definitions';
 import { ISnapshotTree } from '@fluidframework/protocol-definitions';
 import { ISocketStorageDiscovery } from '@fluidframework/odsp-driver-definitions';
@@ -159,7 +160,7 @@ export interface IPersistedFileCache {
 }
 
 // @alpha (undocumented)
-export interface IPrefetchSnapshotContents extends ISnapshotContents {
+export interface IPrefetchSnapshotContents extends ISnapshot {
     // (undocumented)
     fluidEpoch: string;
     // (undocumented)
@@ -172,7 +173,7 @@ export interface ISharingLinkHeader {
     [SharingLinkHeader.isSharingLinkToRedeem]: boolean;
 }
 
-// @alpha (undocumented)
+// @alpha @deprecated (undocumented)
 export interface ISnapshotContents {
     // (undocumented)
     blobs: Map<string, ArrayBuffer>;
@@ -185,7 +186,7 @@ export interface ISnapshotContents {
 }
 
 // @internal
-export interface ISnapshotContentsWithProps extends ISnapshotContents {
+export interface ISnapshotContentsWithProps extends ISnapshot {
     // (undocumented)
     telemetryProps: Record<string, number>;
 }

packages/drivers/odsp-driver/package.json

@@ -120,6 +120,15 @@
 		}
 	},
 	"typeValidation": {
-		"broken": {}
+		"broken": {
+			"InterfaceDeclaration_ISnapshotContentsWithProps": {
+				"backCompat": false,
+				"forwardCompat": false
+			},
+			"InterfaceDeclaration_IPrefetchSnapshotContents": {
+				"backCompat": false,
+				"forwardCompat": false
+			}
+		}
 	}
 }

packages/drivers/odsp-driver/src/compactSnapshotParser.ts

@@ -6,7 +6,7 @@
 import { assert } from "@fluidframework/core-utils";
 import { ISequencedDocumentMessage, ISnapshotTree } from "@fluidframework/protocol-definitions";
 import { ITelemetryLoggerExt } from "@fluidframework/telemetry-utils";
-import { ISnapshotContents } from "./odspPublicUtils";
+import { ISnapshot } from "@fluidframework/driver-definitions";
 import { ReadBuffer } from "./ReadBufferUtils";
 import {
 	assertBlobCoreInstance,
@@ -29,7 +29,7 @@ export const currentReadVersion = "1.0";
  * represents how many times slower parsing path is executed. This will be then logged into telemetry.
  * @internal
  */
-export interface ISnapshotContentsWithProps extends ISnapshotContents {
+export interface ISnapshotContentsWithProps extends ISnapshot {
 	telemetryProps: Record<string, number>;
 }
 
@@ -40,7 +40,7 @@ export interface ISnapshotContentsWithProps extends ISnapshotContents {
 function readBlobSection(node: NodeTypes) {
 	assertNodeCoreInstance(node, "TreeBlobs should be of type NodeCore");
 	let slowBlobStructureCount = 0;
-	const blobs: Map<string, ArrayBuffer> = new Map();
+	const blobContents: Map<string, ArrayBuffer> = new Map();
 	for (const blob of node) {
 		assertNodeCoreInstance(blob, "blob should be node");
 
@@ -56,7 +56,7 @@ function readBlobSection(node: NodeTypes) {
 		) {
 			// "id": <node name>
 			// "data": <blob>
-			blobs.set(blob.getString(1), blob.getBlob(3).arrayBuffer);
+			blobContents.set(blob.getString(1), blob.getBlob(3).arrayBuffer);
 			continue;
 		}
 
@@ -67,9 +67,9 @@ function readBlobSection(node: NodeTypes) {
 		const records = getNodeProps(blob);
 		assertBlobCoreInstance(records.data, "data should be of BlobCore type");
 		const id = getStringInstance(records.id, "blob id should be string");
-		blobs.set(id, records.data.arrayBuffer);
+		blobContents.set(id, records.data.arrayBuffer);
 	}
-	return { blobs, slowBlobStructureCount };
+	return { blobContents, slowBlobStructureCount };
 }
 
 /**
@@ -249,19 +249,20 @@ export function parseCompactSnapshotResponse(
 	);
 
 	const [snapshot, durationSnapshotTree] = measure(() => readSnapshotSection(records.snapshot));
-	const [blobs, durationBlobs] = measure(() => readBlobSection(records.blobs));
+	const [blobContents, durationBlobs] = measure(() => readBlobSection(records.blobs));
 
 	return {
 		...snapshot,
-		...blobs,
+		...blobContents,
 		ops: records.deltas !== undefined ? readOpsSection(records.deltas) : [],
 		latestSequenceNumber: records.lsn,
+		snapshotFormatV: 1,
 		telemetryProps: {
 			...telemetryProps,
 			durationSnapshotTree,
 			durationBlobs,
 			slowTreeStructureCount: snapshot.slowTreeStructureCount,
-			slowBlobStructureCount: blobs.slowBlobStructureCount,
+			slowBlobStructureCount: blobContents.slowBlobStructureCount,
 		},
 	};
 }

packages/drivers/odsp-driver/src/compactSnapshotWriter.ts

@@ -10,8 +10,8 @@ import {
 	ISequencedDocumentMessage,
 	ISnapshotTree,
 } from "@fluidframework/protocol-definitions";
+import { ISnapshot } from "@fluidframework/driver-definitions";
 import { snapshotMinReadVersion } from "./compactSnapshotParser";
-import { ISnapshotContents } from "./odspPublicUtils";
 import { TreeBuilderSerializer } from "./WriteBufferUtils";
 import {
 	addBoolProperty,
@@ -144,7 +144,7 @@ function writeOpsSection(rootNode: NodeCore, ops: ISequencedDocumentMessage[]) {
  * @param snapshotContents - snapshot tree contents to serialize
  * @returns ReadBuffer - binary representation of the data.
  */
-export function convertToCompactSnapshot(snapshotContents: ISnapshotContents): Uint8Array {
+export function convertToCompactSnapshot(snapshotContents: ISnapshot): Uint8Array {
 	const builder = new TreeBuilderSerializer();
 	// Create the root node.
 	const rootNode = builder.addNode();
@@ -166,7 +166,7 @@ export function convertToCompactSnapshot(snapshotContents: ISnapshotContents): U
 	writeSnapshotSection(rootNode, snapshotContents.snapshotTree, snapshotContents.sequenceNumber);
 
 	// Add Blobs
-	writeBlobsSection(rootNode, snapshotContents.blobs);
+	writeBlobsSection(rootNode, snapshotContents.blobContents);
 
 	// Then write the ops node.
 	writeOpsSection(rootNode, snapshotContents.ops);

packages/drivers/odsp-driver/src/contracts.ts

@@ -5,6 +5,7 @@
 
 import * as api from "@fluidframework/protocol-definitions";
 import { HostStoragePolicy } from "@fluidframework/odsp-driver-definitions";
+import { ISnapshot } from "@fluidframework/driver-definitions";
 import { ISnapshotContents } from "./odspPublicUtils";
 
 /**
@@ -165,6 +166,8 @@ export interface IOdspSnapshot {
  */
 export interface HostStoragePolicyInternal extends HostStoragePolicy {
 	summarizerClient?: boolean;
+
+	supportGetSnapshotApi?: boolean;
 }
 
 export interface ICreateFileResponse {
@@ -207,11 +210,20 @@ export interface IFlushOpsResponse {
 
 /**
  * Represents the cached snapshot value.
+ * @deprecated - This will be replaced with ISnapshotCachedEntry2 which wraps the new ISnapshot interface.
+ * For now, to support back compat from cache, we need to keep it for now.
  */
 export interface ISnapshotCachedEntry extends ISnapshotContents {
 	cacheEntryTime: number;
 }
 
+/**
+ * Represents the cached snapshot value.
+ */
+export interface ISnapshotCachedEntry2 extends ISnapshot {
+	cacheEntryTime: number;
+}
+
 /**
  * Represents the type of signal containing the sensitivity policy labels for the container.
  */