Allow ID compressor to be enabled in existing files. (#20531)

This work builds on previous work:
- #19859
- #20174

While prior work gives us opportunity to generate short IDs (this is used, for example, to generate short IDs for data stores and DDSs), the current state of ID compressor settings does not allow us to change them through lifetime of the container.
I.e. if container is created with ID compressor off, then it stays off for duration of container lifetime.

This is not great, and ideally, we want (eventually) all files to use ID compressor, at least for generation of short IDs.
This work makes it possible, allowing off -> delayed mode transition.

It stops short from enabling off | delayed -> on transition, as currently this is not possible with async nature of ID compressor loading and synchronous op processing pipeline. It's possible we can make such transition work with some delay, but more work is required to make it happen.

packages/runtime/container-runtime/src/containerRuntime.ts

@@ -882,6 +882,19 @@ export class ContainerRuntime
 			}
 		}
 
+		let desiredIdCompressorMode: IdCompressorMode;
+		switch (mc.config.getBoolean("Fluid.ContainerRuntime.IdCompressorEnabled")) {
+			case true:
+				desiredIdCompressorMode = "on";
+				break;
+			case false:
+				desiredIdCompressorMode = undefined;
+				break;
+			default:
+				desiredIdCompressorMode = enableRuntimeIdCompressor;
+				break;
+		}
+
 		// Enabling the IdCompressor is a one-way operation and we only want to
 		// allow new containers to turn it on.
 		let idCompressorMode: IdCompressorMode;
@@ -893,23 +906,19 @@ export class ContainerRuntime
 			// 3) Same logic applies for "delayed" mode
 			// Maybe in the future we will need to enabled (and figure how to do it safely) "delayed" -> "on" change.
 			// We could do "off" -> "on" transition too, if all clients start loading compressor (but not using it initially) and
-			// do so for a while - this will allow clients to eventually to disregard "off" setting (when it's safe so) and start
+			// do so for a while - this will allow clients to eventually disregard "off" setting (when it's safe so) and start
 			// using compressor in future sessions.
 			// Everyting is possible, but it needs to be designed and executed carefully, when such need arises.
 			idCompressorMode = metadata?.documentSchema?.runtime
 				?.idCompressorMode as IdCompressorMode;
-		} else {
-			switch (mc.config.getBoolean("Fluid.ContainerRuntime.IdCompressorEnabled")) {
-				case true:
-					idCompressorMode = "on";
-					break;
-				case false:
-					idCompressorMode = undefined;
-					break;
-				default:
-					idCompressorMode = enableRuntimeIdCompressor;
-					break;
+
+			// This is the only exception to the rule above - we have proper plumbing to load ID compressor on schema change
+			// event. It is loaded async (relative to op processing), so this conversion is only safe for off -> delayed conversion!
+			if (idCompressorMode === undefined && desiredIdCompressorMode === "delayed") {
+				idCompressorMode = desiredIdCompressorMode;
 			}
+		} else {
+			idCompressorMode = desiredIdCompressorMode;
 		}
 
 		const createIdCompressorFn = async () => {
@@ -2642,6 +2651,10 @@ export class ContainerRuntime
 					// Some other client turned on the id compressor. If we have not turned it on,
 					// put it in a pending queue and delay finalization.
 					if (this._idCompressor === undefined) {
+						assert(
+							this.idCompressorMode !== undefined,
+							"id compressor should be enabled",
+						);
 						this.pendingIdCompressorOps.push(range);
 					} else {
 						this._idCompressor.finalizeCreationRange(range);

packages/test/test-end-to-end-tests/src/test/containerRuntime.spec.ts

@@ -5,7 +5,7 @@
 
 import { strict as assert } from "assert";
 
-import { describeCompat } from "@fluid-private/test-version-utils";
+import { describeCompat, ITestDataObject } from "@fluid-private/test-version-utils";
 import { IContainer } from "@fluidframework/container-definitions/internal";
 import { CompressionAlgorithms } from "@fluidframework/container-runtime/internal";
 import {
@@ -32,7 +32,7 @@ describeCompat("ContainerRuntime Document Schema", "FullCompat", (getTestObjectP
 		return provider.loadTestContainer(options);
 	}
 
-	async function getentryPoint(container: IContainer) {
+	async function getEntryPoint(container: IContainer) {
 		return getContainerEntryPointBackCompat<TestDataObject>(container);
 	}
 
@@ -121,7 +121,7 @@ describeCompat("ContainerRuntime Document Schema", "FullCompat", (getTestObjectP
 			},
 		};
 		const container = await provider.makeTestContainer(options);
-		entry = await getentryPoint(container);
+		entry = await getEntryPoint(container);
 
 		assert(entry);
 		entry.root.set("key", generateStringOfSize(10000));
@@ -134,7 +134,7 @@ describeCompat("ContainerRuntime Document Schema", "FullCompat", (getTestObjectP
 		}
 
 		const container2 = await loadContainer(options);
-		const entry2 = await getentryPoint(container2);
+		const entry2 = await getEntryPoint(container2);
 		assert(entry.root.get("key").length === 10000);
 
 		entry2.root.set("key2", generateStringOfSize(5000));
@@ -145,7 +145,7 @@ describeCompat("ContainerRuntime Document Schema", "FullCompat", (getTestObjectP
 		assert(crash || entry.root.get("key2").length === 5000);
 
 		const container3 = await loadContainer(options);
-		const entry3 = await getentryPoint(container3);
+		const entry3 = await getEntryPoint(container3);
 		await provider.ensureSynchronized();
 
 		assert.equal(crash, container.closed);
@@ -179,3 +179,74 @@ describeCompat("ContainerRuntime Document Schema", "FullCompat", (getTestObjectP
 		}
 	}
 });
+
+describeCompat("Id Compressor Schema change", "NoCompat", (getTestObjectProvider, apis) => {
+	let provider: ITestObjectProvider;
+
+	async function loadContainer(options: ITestContainerConfig) {
+		return provider.loadTestContainer(options);
+	}
+
+	async function getEntryPoint(container: IContainer) {
+		return getContainerEntryPointBackCompat<ITestDataObject>(container);
+	}
+
+	beforeEach("getTestObjectProvider", async () => {
+		provider = getTestObjectProvider();
+	});
+
+	it("upgrade", async () => {
+		const options: ITestContainerConfig = {
+			runtimeOptions: {
+				explicitSchemaControl: true,
+			},
+		};
+
+		const container = await provider.makeTestContainer({
+			runtimeOptions: {
+				explicitSchemaControl: false,
+				enableRuntimeIdCompressor: undefined,
+			},
+		});
+		const entry = await getEntryPoint(container);
+		entry._root.set("someKey", "someValue");
+
+		// ensure that old container is fully loaded (connected)
+		await provider.ensureSynchronized();
+
+		const container2 = await loadContainer({
+			runtimeOptions: {
+				explicitSchemaControl: true,
+				enableRuntimeIdCompressor: "delayed",
+			},
+		});
+		const entry2 = await getEntryPoint(container2);
+
+		// Send some ops, it will trigger schema change ops
+		// This will also trigger delay loading of ID compressor for both clients!
+		entry2._root.set("someKey2", "someValue");
+		await provider.ensureSynchronized();
+
+		// ID compressor loading is async. THere is no way to check when it's done.
+		// To be safe, make another round of sending-waiting
+		entry2._root.set("someKey2", "someValue");
+		await provider.ensureSynchronized();
+
+		// Now we should have new schema, ID compressor loaded, and be able to allocate ID range
+		// In order for ID compressor to produce short IDs, the following needs to happen:
+		// 1. Request unique ID (will initially get long ID)
+		// 2. Send any op (will trigger ID compressor to reserve short IDs)
+		entry._context.containerRuntime.generateDocumentUniqueId();
+		entry._root.set("someKey3", "someValue");
+		entry2._context.containerRuntime.generateDocumentUniqueId();
+		entry2._root.set("someKey4", "someValue");
+		await provider.ensureSynchronized();
+
+		// Now ID compressor should give us short IDs!
+		const id = entry2._context.containerRuntime.generateDocumentUniqueId();
+		assert(Number.isInteger(id));
+
+		const id2 = entry._context.containerRuntime.generateDocumentUniqueId();
+		assert(Number.isInteger(id2));
+	});
+});

packages/test/test-version-utils/src/compatUtils.ts

@@ -124,7 +124,9 @@ function filterRuntimeOptionsForVersion(
 		options = {
 			compressionOptions: compressorDisabled, // Can't use compression, need https://github.com/microsoft/FluidFramework/pull/20111 fix
 			enableGroupedBatching,
-			enableRuntimeIdCompressor,
+			// control over schema was generalized in RC3 - see https://github.com/microsoft/FluidFramework/pull/20174
+			// IdCompressor settings moved around - can't enable them across versions without tripping on asserts
+			enableRuntimeIdCompressor: undefined,
 			chunkSizeInBytes: Number.POSITIVE_INFINITY, // disabled, need https://github.com/microsoft/FluidFramework/pull/20115 fix
 			...options,
 		};