This repository has been archived by the owner on Feb 16, 2023. It is now read-only.
forked from airbytehq/airbyte
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Choose optimal sync modes by default on UI (airbytehq#12411)
* fix CatalogSectionInner component padding * fix React key error in StreamFieldTable component * Update choosing the optimal sync mode logic. Sync Mode priority: 1. Incremental(cursor defined) => Append Dedup 2. Full Refresh => Overwrite 3. Incremental => Append 4. Full Refresh => Append * Fixes after comments to PR: - replace func "isEmpty" with "[].length" - replace named export with direct export - update "config" type for updateStreamConfig func - rename "getOptimalSyncMode" to "setOptimalSyncMode" * remove fp-ts package * fix wrong condition * cover formConfigHelpers with tests * add npm script: run tests with coverage * tiny fix: test assert
- Loading branch information
Showing
6 changed files
with
217 additions
and
51 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
120 changes: 120 additions & 0 deletions
120
airbyte-webapp/src/views/Connection/ConnectionForm/formConfigHelpers.test.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,120 @@ | ||
import { DestinationSyncMode, SyncMode, SyncSchemaStream } from "../../../core/domain/catalog"; | ||
import { getOptimalSyncMode, verifyConfigCursorField, verifySupportedSyncModes } from "./formConfigHelpers"; | ||
|
||
const mockedStreamNode: SyncSchemaStream = { | ||
stream: { | ||
name: "test", | ||
supportedSyncModes: [], | ||
jsonSchema: {}, | ||
sourceDefinedCursor: null, | ||
sourceDefinedPrimaryKey: [], | ||
defaultCursorField: [], | ||
}, | ||
config: { | ||
cursorField: [], | ||
primaryKey: [], | ||
selected: true, | ||
syncMode: SyncMode.FullRefresh, | ||
destinationSyncMode: DestinationSyncMode.Append, | ||
aliasName: "", | ||
}, | ||
id: "1", | ||
}; | ||
|
||
describe("formConfigHelpers", () => { | ||
describe("verifySupportedSyncModes", () => { | ||
const streamNodeWithDefinedSyncMode: SyncSchemaStream = { | ||
...mockedStreamNode, | ||
stream: { ...mockedStreamNode.stream, supportedSyncModes: [SyncMode.Incremental] }, | ||
}; | ||
|
||
test("should not change supportedSyncModes if it's not empty", () => { | ||
const streamNode = verifySupportedSyncModes(streamNodeWithDefinedSyncMode); | ||
|
||
expect(streamNode.stream.supportedSyncModes).toStrictEqual([SyncMode.Incremental]); | ||
}); | ||
|
||
test("should set default supportedSyncModes if it's empty", () => { | ||
const streamNodeWithEmptySyncMode = { | ||
...streamNodeWithDefinedSyncMode, | ||
stream: { ...mockedStreamNode.stream, supportedSyncModes: [] }, | ||
}; | ||
const streamNode = verifySupportedSyncModes(streamNodeWithEmptySyncMode); | ||
|
||
expect(streamNode.stream.supportedSyncModes).toStrictEqual([SyncMode.FullRefresh]); | ||
}); | ||
}); | ||
|
||
describe("verifyConfigCursorField", () => { | ||
const streamWithDefinedCursorField: SyncSchemaStream = { | ||
...mockedStreamNode, | ||
config: { ...mockedStreamNode.config, cursorField: ["id"] }, | ||
stream: { ...mockedStreamNode.stream, defaultCursorField: ["name"] }, | ||
}; | ||
|
||
test("should leave cursorField value as is if it's defined", () => { | ||
const streamNode = verifyConfigCursorField(streamWithDefinedCursorField); | ||
|
||
expect(streamNode.config.cursorField).toStrictEqual(["id"]); | ||
}); | ||
|
||
test("should set defaultCursorField if cursorField is not defined", () => { | ||
const streamNodeWithoutDefinedCursor = { | ||
...streamWithDefinedCursorField, | ||
config: { ...mockedStreamNode.config, cursorField: [] }, | ||
}; | ||
const streamNode = verifyConfigCursorField(streamNodeWithoutDefinedCursor); | ||
|
||
expect(streamNode.config.cursorField).toStrictEqual(["name"]); | ||
}); | ||
|
||
test("should leave cursorField empty if defaultCursorField not defined", () => { | ||
const streamNode = verifyConfigCursorField(mockedStreamNode); | ||
|
||
expect(streamNode.config.cursorField).toStrictEqual([]); | ||
}); | ||
}); | ||
|
||
describe("getOptimalSyncMode", () => { | ||
test("should get 'Incremental(cursor defined) => Append dedup' mode", () => { | ||
const streamNodeWithIncrDedupMode = { | ||
...mockedStreamNode, | ||
stream: { ...mockedStreamNode.stream, supportedSyncModes: [SyncMode.Incremental], sourceDefinedCursor: true }, | ||
}; | ||
const nodeStream = getOptimalSyncMode(streamNodeWithIncrDedupMode, [DestinationSyncMode.Dedupted]); | ||
|
||
expect(nodeStream.config.syncMode).toBe(SyncMode.Incremental); | ||
expect(nodeStream.config.destinationSyncMode).toBe(DestinationSyncMode.Dedupted); | ||
}); | ||
|
||
test("should get 'FullRefresh => Overwrite' mode", () => { | ||
const nodeStream = getOptimalSyncMode(mockedStreamNode, [DestinationSyncMode.Overwrite]); | ||
|
||
expect(nodeStream.config.syncMode).toBe(SyncMode.FullRefresh); | ||
expect(nodeStream.config.destinationSyncMode).toBe(DestinationSyncMode.Overwrite); | ||
}); | ||
|
||
test("should get 'Incremental => Append' mode", () => { | ||
const streamNodeWithIncrAppendMode = { | ||
...mockedStreamNode, | ||
stream: { ...mockedStreamNode.stream, supportedSyncModes: [SyncMode.Incremental] }, | ||
}; | ||
const nodeStream = getOptimalSyncMode(streamNodeWithIncrAppendMode, [DestinationSyncMode.Append]); | ||
|
||
expect(nodeStream.config.syncMode).toBe(SyncMode.Incremental); | ||
expect(nodeStream.config.destinationSyncMode).toBe(DestinationSyncMode.Append); | ||
}); | ||
|
||
test("should get 'FullRefresh => Append' mode", () => { | ||
const nodeStream = getOptimalSyncMode(mockedStreamNode, [DestinationSyncMode.Append]); | ||
|
||
expect(nodeStream.config.syncMode).toBe(SyncMode.FullRefresh); | ||
expect(nodeStream.config.destinationSyncMode).toBe(DestinationSyncMode.Append); | ||
}); | ||
test("should return untouched nodeStream", () => { | ||
const nodeStream = getOptimalSyncMode(mockedStreamNode, []); | ||
|
||
expect(nodeStream).toBe(mockedStreamNode); | ||
}); | ||
}); | ||
}); |
86 changes: 86 additions & 0 deletions
86
airbyte-webapp/src/views/Connection/ConnectionForm/formConfigHelpers.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,86 @@ | ||
import { | ||
AirbyteStreamConfiguration, | ||
DestinationSyncMode, | ||
SyncMode, | ||
SyncSchemaStream, | ||
} from "../../../core/domain/catalog"; | ||
|
||
const getDefaultCursorField = (streamNode: SyncSchemaStream): string[] => { | ||
if (streamNode.stream.defaultCursorField.length) { | ||
return streamNode.stream.defaultCursorField; | ||
} | ||
return streamNode.config.cursorField; | ||
}; | ||
|
||
export const verifySupportedSyncModes = (streamNode: SyncSchemaStream): SyncSchemaStream => { | ||
const { | ||
stream: { supportedSyncModes }, | ||
} = streamNode; | ||
|
||
if (supportedSyncModes?.length) return streamNode; | ||
return { ...streamNode, stream: { ...streamNode.stream, supportedSyncModes: [SyncMode.FullRefresh] } }; | ||
}; | ||
|
||
export const verifyConfigCursorField = (streamNode: SyncSchemaStream): SyncSchemaStream => { | ||
const { config } = streamNode; | ||
|
||
return { | ||
...streamNode, | ||
config: { | ||
...config, | ||
cursorField: config.cursorField?.length ? config.cursorField : getDefaultCursorField(streamNode), | ||
}, | ||
}; | ||
}; | ||
|
||
export const getOptimalSyncMode = ( | ||
streamNode: SyncSchemaStream, | ||
supportedDestinationSyncModes: DestinationSyncMode[] | ||
): SyncSchemaStream => { | ||
const updateStreamConfig = ( | ||
config: Pick<AirbyteStreamConfiguration, "syncMode" | "destinationSyncMode"> | ||
): SyncSchemaStream => ({ | ||
...streamNode, | ||
config: { ...streamNode.config, ...config }, | ||
}); | ||
|
||
const { | ||
stream: { supportedSyncModes, sourceDefinedCursor }, | ||
} = streamNode; | ||
|
||
if ( | ||
supportedSyncModes.includes(SyncMode.Incremental) && | ||
supportedDestinationSyncModes.includes(DestinationSyncMode.Dedupted) && | ||
sourceDefinedCursor | ||
) { | ||
return updateStreamConfig({ | ||
syncMode: SyncMode.Incremental, | ||
destinationSyncMode: DestinationSyncMode.Dedupted, | ||
}); | ||
} | ||
|
||
if (supportedDestinationSyncModes.includes(DestinationSyncMode.Overwrite)) { | ||
return updateStreamConfig({ | ||
syncMode: SyncMode.FullRefresh, | ||
destinationSyncMode: DestinationSyncMode.Overwrite, | ||
}); | ||
} | ||
|
||
if ( | ||
supportedSyncModes.includes(SyncMode.Incremental) && | ||
supportedDestinationSyncModes.includes(DestinationSyncMode.Append) | ||
) { | ||
return updateStreamConfig({ | ||
syncMode: SyncMode.Incremental, | ||
destinationSyncMode: DestinationSyncMode.Append, | ||
}); | ||
} | ||
|
||
if (supportedDestinationSyncModes.includes(DestinationSyncMode.Append)) { | ||
return updateStreamConfig({ | ||
syncMode: SyncMode.FullRefresh, | ||
destinationSyncMode: DestinationSyncMode.Append, | ||
}); | ||
} | ||
return streamNode; | ||
}; |