Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Container Rehydration from a snapshot taken from detached container. #3061

Closed
wants to merge 39 commits into from
Closed
Show file tree
Hide file tree
Changes from 10 commits
Commits
Show all changes
39 commits
Select commit Hold shift + click to select a range
59f1e16
final changes for rehydration
jatgarg Aug 4, 2020
4bffb77
final changes for rehydration
jatgarg Aug 5, 2020
5573ffa
merge conflict
jatgarg Aug 5, 2020
59d66b5
mertge conflict
jatgarg Aug 5, 2020
d240a3a
remove
jatgarg Aug 5, 2020
f99f68e
Merge branch 'master' of https://github.com/microsoft/FluidFramework …
jatgarg Aug 5, 2020
217aacc
fix tiny
jatgarg Aug 5, 2020
1bb9da0
remove unloaded map
jatgarg Aug 5, 2020
6fcce12
remove unrealized map
jatgarg Aug 5, 2020
3f903b7
add comment
jatgarg Aug 5, 2020
e255f54
remove loadlocal
jatgarg Aug 5, 2020
59a4ff2
pr sugg
jatgarg Aug 5, 2020
737111d
pr sugg
jatgarg Aug 5, 2020
cfde78c
pr sugg
jatgarg Aug 5, 2020
0a38dff
merge conflict
jatgarg Aug 5, 2020
acc535a
fix name
jatgarg Aug 5, 2020
918a5c4
merge conflict
jatgarg Aug 6, 2020
84bc42e
pr sugg
jatgarg Aug 6, 2020
808d530
merge conflict
jatgarg Aug 12, 2020
7150af4
separate the 2 apis
jatgarg Aug 12, 2020
60f977f
resolve m,erge
jatgarg Aug 12, 2020
4bd5d7e
merge conflict
jatgarg Aug 13, 2020
f93b1df
set existing as true if loaded from rehydration snapshopt
jatgarg Aug 17, 2020
79592b1
set existing as true if loaded from rehydration snapshopt
jatgarg Aug 17, 2020
2876424
merge conflict
jatgarg Aug 17, 2020
acd860d
remove
jatgarg Aug 17, 2020
e52d2ac
remove
jatgarg Aug 17, 2020
3f46d22
localdatastore
jatgarg Aug 17, 2020
a19116f
rename comp toi datta store
jatgarg Aug 19, 2020
eae3ec6
merge conflict
jatgarg Aug 19, 2020
562b29b
merge conflict
jatgarg Aug 19, 2020
55d76e2
cast
jatgarg Aug 20, 2020
d18fd5f
cast
jatgarg Aug 20, 2020
0525797
merge conflict
jatgarg Aug 24, 2020
7c3e821
merge conflict
jatgarg Aug 28, 2020
c395773
pr sugg
jatgarg Aug 28, 2020
cf97acd
merge conflict
jatgarg Sep 4, 2020
a1a07a8
pr sugg
jatgarg Sep 8, 2020
545f304
merge conflict
jatgarg Sep 8, 2020
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion packages/dds/cell/src/cell.ts
Original file line number Diff line number Diff line change
Expand Up @@ -202,7 +202,7 @@ export class SharedCell<T extends Serializable = any> extends SharedObject<IShar
* @returns - promise that resolved when the load is completed
*/
protected async loadCore(
branchId: string,
branchId: string | undefined,
jatgarg marked this conversation as resolved.
Show resolved Hide resolved
jatgarg marked this conversation as resolved.
Show resolved Hide resolved
storage: IChannelStorageService): Promise<void> {
const rawContent = await storage.read(snapshotFileName);

Expand Down
12 changes: 12 additions & 0 deletions packages/dds/cell/src/cellFactory.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import {
IFluidDataStoreRuntime,
IChannelServices,
IChannelFactory,
IChannelStorageService,
} from "@fluidframework/datastore-definitions";
import { SharedCell } from "./cell";
import { ISharedCell } from "./interfaces";
Expand Down Expand Up @@ -44,6 +45,17 @@ export class CellFactory implements IChannelFactory {
return cell;
}

public async loadLocal(
runtime: IFluidDataStoreRuntime,
id: string,
objectStorage: IChannelStorageService,
attributes: IChannelAttributes,
): Promise<ISharedCell> {
const cell = new SharedCell(id, runtime, attributes);
await cell.loadLocal(objectStorage);
return cell;
}

public create(document: IFluidDataStoreRuntime, id: string): ISharedCell {
const cell = new SharedCell(id, document, this.attributes);
cell.initializeLocal();
Expand Down
2 changes: 1 addition & 1 deletion packages/dds/counter/src/counter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,7 @@ export class SharedCounter extends SharedObject<ISharedCounterEvents> implements
* @returns - promise that resolved when the load is completed
*/
protected async loadCore(
branchId: string,
branchId: string | undefined,
storage: IChannelStorageService): Promise<void> {
const rawContent = await storage.read(snapshotFileName);

Expand Down
12 changes: 12 additions & 0 deletions packages/dds/counter/src/counterFactory.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import {
IFluidDataStoreRuntime,
IChannelServices,
IChannelFactory,
IChannelStorageService,
} from "@fluidframework/datastore-definitions";
import { SharedCounter } from "./counter";
import { ISharedCounter } from "./interfaces";
Expand Down Expand Up @@ -44,6 +45,17 @@ export class CounterFactory implements IChannelFactory {
return counter;
}

public async loadLocal(
runtime: IFluidDataStoreRuntime,
id: string,
objectStorage: IChannelStorageService,
attributes: IChannelAttributes,
): Promise<ISharedCounter> {
const counter = new SharedCounter(id, runtime, attributes);
await counter.loadLocal(objectStorage);
return counter;
}

public create(document: IFluidDataStoreRuntime, id: string): ISharedCounter {
const counter = new SharedCounter(id, document, this.attributes);
counter.initializeLocal();
Expand Down
2 changes: 1 addition & 1 deletion packages/dds/ink/src/ink.ts
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,7 @@ export class Ink extends SharedObject<IInkEvents> implements IInk {
* {@inheritDoc @fluidframework/shared-object-base#SharedObject.loadCore}
*/
protected async loadCore(
branchId: string,
branchId: string | undefined,
storage: IChannelStorageService,
): Promise<void> {
const header = await storage.read(snapshotFileName);
Expand Down
15 changes: 15 additions & 0 deletions packages/dds/ink/src/inkFactory.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import {
IFluidDataStoreRuntime,
IChannelServices,
IChannelFactory,
IChannelStorageService,
} from "@fluidframework/datastore-definitions";
import { ISharedObject } from "@fluidframework/shared-object-base";
import { Ink } from "./ink";
Expand Down Expand Up @@ -61,6 +62,20 @@ export class InkFactory implements IChannelFactory {
return ink;
}

/**
* {@inheritDoc @fluidframework/shared-object-base#IChannelFactory.loadLocal}
*/
public async loadLocal(
runtime: IFluidDataStoreRuntime,
id: string,
objectStorage: IChannelStorageService,
attributes: IChannelAttributes,
): Promise<ISharedObject> {
const ink = new Ink(runtime, id, attributes);
await ink.loadLocal(objectStorage);
return ink;
}

/**
* {@inheritDoc @fluidframework/shared-object-base#IChannelFactory.create}
*/
Expand Down
17 changes: 16 additions & 1 deletion packages/dds/map/src/directory.ts
Original file line number Diff line number Diff line change
Expand Up @@ -343,6 +343,21 @@ export class DirectoryFactory {
return directory;
}

/**
* {@inheritDoc @fluidframework/shared-object-base#IChannelFactory.loadLocal}
*/
public async loadLocal(
jatgarg marked this conversation as resolved.
Show resolved Hide resolved
runtime: IFluidDataStoreRuntime,
id: string,
objectStorage: IChannelStorageService,
attributes: IChannelAttributes,
): Promise<ISharedDirectory> {
const directory = new SharedDirectory(id, runtime, attributes);
await directory.loadLocal(objectStorage);

return directory;
}

/**
* {@inheritDoc @fluidframework/shared-object-base#IChannelFactory.create}
*/
Expand Down Expand Up @@ -653,7 +668,7 @@ export class SharedDirectory extends SharedObject<ISharedDirectoryEvents> implem
* {@inheritDoc @fluidframework/shared-object-base#SharedObject.loadCore}
*/
protected async loadCore(
branchId: string,
branchId: string | undefined,
storage: IChannelStorageService) {
const header = await storage.read(snapshotFileName);
const data = JSON.parse(fromBase64ToUtf8(header));
Expand Down
17 changes: 16 additions & 1 deletion packages/dds/map/src/map.ts
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,21 @@ export class MapFactory implements IChannelFactory {
return map;
}

/**
* {@inheritDoc @fluidframework/shared-object-base#IChannelFactory.loadLocal}
*/
public async loadLocal(
runtime: IFluidDataStoreRuntime,
id: string,
objectStorage: IChannelStorageService,
attributes: IChannelAttributes,
): Promise<ISharedMap> {
const map = new SharedMap(id, runtime, attributes);
await map.loadLocal(objectStorage);

return map;
}

/**
* {@inheritDoc @fluidframework/shared-object-base#IChannelFactory.create}
*/
Expand Down Expand Up @@ -321,7 +336,7 @@ export class SharedMap extends SharedObject<ISharedMapEvents> implements IShared
* {@inheritDoc @fluidframework/shared-object-base#SharedObject.loadCore}
*/
protected async loadCore(
branchId: string,
branchId: string | undefined,
storage: IChannelStorageService) {
const header = await storage.read(snapshotFileName);

Expand Down
2 changes: 1 addition & 1 deletion packages/dds/matrix/src/matrix.ts
Original file line number Diff line number Diff line change
Expand Up @@ -462,7 +462,7 @@ export class SharedMatrix<T extends Serializable = Serializable>
debug(`${this.id} is now disconnected`);
}

protected async loadCore(branchId: string, storage: IChannelStorageService) {
protected async loadCore(branchId: string | undefined, storage: IChannelStorageService) {
try {
await this.rows.load(this.runtime, new ObjectStoragePartition(storage, SnapshotPath.rows), branchId);
await this.cols.load(this.runtime, new ObjectStoragePartition(storage, SnapshotPath.cols), branchId);
Expand Down
12 changes: 12 additions & 0 deletions packages/dds/matrix/src/runtime.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import {
IChannelServices,
IChannel,
IChannelFactory,
IChannelStorageService,
} from "@fluidframework/datastore-definitions";
import { pkgVersion } from "./packageVersion";
import { SharedMatrix } from "./matrix";
Expand Down Expand Up @@ -42,6 +43,17 @@ export class SharedMatrixFactory implements IChannelFactory {
return matrix;
}

public async loadLocal(
runtime: IFluidDataStoreRuntime,
id: string,
objectStorage: IChannelStorageService,
attributes: IChannelAttributes,
): Promise<IChannel> {
const matrix = new SharedMatrix(runtime, id, attributes);
await matrix.loadLocal(objectStorage);
return matrix;
}

public create(document: IFluidDataStoreRuntime, id: string): IChannel {
const matrix = new SharedMatrix(document, id, this.attributes);
matrix.initializeLocal();
Expand Down
1 change: 1 addition & 0 deletions packages/dds/merge-tree/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@
"@fluidframework/common-definitions": "^0.18.1",
"@fluidframework/common-utils": "^0.21.0",
"@fluidframework/component-core-interfaces": "^0.25.0",
"@fluidframework/container-definitions": "^0.25.0",
"@fluidframework/datastore-definitions": "^0.25.0",
"@fluidframework/protocol-definitions": "^0.1010.0",
"@fluidframework/telemetry-utils": "^0.25.0"
Expand Down
8 changes: 4 additions & 4 deletions packages/dds/merge-tree/src/snapshotLoader.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import { ChildLogger } from "@fluidframework/telemetry-utils";
import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
import { IFluidDataStoreRuntime, IChannelStorageService } from "@fluidframework/datastore-definitions";
import { ITelemetryLogger } from "@fluidframework/common-definitions";
import { AttachState } from "@fluidframework/container-definitions";
jatgarg marked this conversation as resolved.
Show resolved Hide resolved
import { Client } from "./client";
import { NonCollabClient, UniversalSequenceNumber } from "./constants";
import { ISegment, MergeTree } from "./mergeTree";
Expand Down Expand Up @@ -139,11 +140,10 @@ export class SnapshotLoader {
// a snapshot in any case (summary or attach message)
// once we get a client id this will be called with that
// clientId in the connected event
// TODO: this won't support rehydrating a detached container
// we need to think more holistically about the dds state machine
// now that we differentiate attached vs local
// However if we load a detached container from snapshot, then we don't supply a default clientId
// because we don't want to start collaboration.
this.client.startOrUpdateCollaboration(
this.runtime.clientId ?? "snapshot",
this.runtime.attachState === AttachState.Detached ? undefined : this.runtime.clientId ?? "snapshot",
jatgarg marked this conversation as resolved.
Show resolved Hide resolved

// TODO: Make 'minSeq' non-optional once the new snapshot format becomes the default?
// (See https://github.com/microsoft/FluidFramework/issues/84)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -273,7 +273,7 @@ export class ConsensusOrderedCollection<T = any>
}

protected async loadCore(
branchId: string,
branchId: string | undefined,
storage: IChannelStorageService): Promise<void> {
assert(this.jobTracking.size === 0);
const rawContentTracking = await storage.read(snapshotFileNameTracking);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import {
IChannelAttributes,
IFluidDataStoreRuntime,
IChannelServices,
IChannelStorageService,
} from "@fluidframework/datastore-definitions";
import { ConsensusQueue } from "./consensusQueue";
import { IConsensusOrderedCollection, IConsensusOrderedCollectionFactory } from "./interfaces";
Expand Down Expand Up @@ -43,6 +44,17 @@ export class ConsensusQueueFactory implements IConsensusOrderedCollectionFactory
return collection;
}

public async loadLocal(
runtime: IFluidDataStoreRuntime,
id: string,
objectStorage: IChannelStorageService,
attributes: IChannelAttributes,
): Promise<IConsensusOrderedCollection> {
const collection = new ConsensusQueue(id, runtime, attributes);
await collection.loadLocal(objectStorage);
return collection;
}

public create(document: IFluidDataStoreRuntime, id: string): IConsensusOrderedCollection {
const collection = new ConsensusQueue(id, document, this.attributes);
collection.initializeLocal();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -210,7 +210,7 @@ export class ConsensusRegisterCollection<T>
}

protected async loadCore(
branchId: string,
branchId: string | undefined,
storage: IChannelStorageService,
): Promise<void> {
const header = await storage.read(snapshotFileName);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import {
IChannelAttributes,
IFluidDataStoreRuntime,
IChannelServices,
IChannelStorageService,
} from "@fluidframework/datastore-definitions";
import { ConsensusRegisterCollection } from "./consensusRegisterCollection";
import { IConsensusRegisterCollection, IConsensusRegisterCollectionFactory } from "./interfaces";
Expand Down Expand Up @@ -43,6 +44,17 @@ export class ConsensusRegisterCollectionFactory implements IConsensusRegisterCol
return collection;
}

public async loadLocal(
runtime: IFluidDataStoreRuntime,
id: string,
objectStorage: IChannelStorageService,
attributes: IChannelAttributes,
): Promise<IConsensusRegisterCollection> {
const collection = new ConsensusRegisterCollection(id, runtime, attributes);
await collection.loadLocal(objectStorage);
return collection;
}

public create(document: IFluidDataStoreRuntime, id: string): IConsensusRegisterCollection {
const collection = new ConsensusRegisterCollection(id, document, ConsensusRegisterCollectionFactory.Attributes);
collection.initializeLocal();
Expand Down
2 changes: 1 addition & 1 deletion packages/dds/sequence/src/sequence.ts
Original file line number Diff line number Diff line change
Expand Up @@ -487,7 +487,7 @@ export abstract class SharedSegmentSequence<T extends MergeTree.ISegment>
}

protected async loadCore(
branchId: string,
branchId: string | undefined,
storage: IChannelStorageService) {
const header = await storage.read(snapshotFileName);

Expand Down
34 changes: 34 additions & 0 deletions packages/dds/sequence/src/sequenceFactory.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import {
IFluidDataStoreRuntime,
IChannelServices,
IChannelFactory,
IChannelStorageService,
} from "@fluidframework/datastore-definitions";
import { ISharedObject } from "@fluidframework/shared-object-base";
import { pkgVersion } from "./packageVersion";
Expand Down Expand Up @@ -55,6 +56,17 @@ export class SharedStringFactory implements IChannelFactory {
return sharedString;
}

public async loadLocal(
runtime: IFluidDataStoreRuntime,
id: string,
objectStorage: IChannelStorageService,
attributes: IChannelAttributes,
): Promise<SharedString> {
const sharedString = new SharedString(runtime, id, attributes);
await sharedString.loadLocal(objectStorage);
return sharedString;
}

public create(document: IFluidDataStoreRuntime, id: string): SharedString {
const sharedString = new SharedString(document, id, this.attributes);
sharedString.initializeLocal();
Expand Down Expand Up @@ -101,6 +113,17 @@ export class SharedObjectSequenceFactory implements IChannelFactory {
return sharedSeq;
}

public async loadLocal(
runtime: IFluidDataStoreRuntime,
id: string,
objectStorage: IChannelStorageService,
attributes: IChannelAttributes,
): Promise<ISharedObject> {
const sharedSeq = new SharedObjectSequence<object>(runtime, id, attributes);
await sharedSeq.loadLocal(objectStorage);
return sharedSeq;
}

public create(document: IFluidDataStoreRuntime, id: string): ISharedObject {
const sharedString = new SharedObjectSequence(document, id, this.attributes);
sharedString.initializeLocal();
Expand Down Expand Up @@ -147,6 +170,17 @@ export class SharedNumberSequenceFactory implements IChannelFactory {
return sharedSeq;
}

public async loadLocal(
runtime: IFluidDataStoreRuntime,
id: string,
objectStorage: IChannelStorageService,
attributes: IChannelAttributes,
): Promise<ISharedObject> {
const sharedSeq = new SharedNumberSequence(runtime, id, attributes);
await sharedSeq.loadLocal(objectStorage);
return sharedSeq;
}

public create(document: IFluidDataStoreRuntime, id: string): ISharedObject {
const sharedString = new SharedNumberSequence(document, id, this.attributes);
sharedString.initializeLocal();
Expand Down
Loading