From d822ea8e6d20cd01fb5233a6eb0b306b3d047814 Mon Sep 17 00:00:00 2001 From: "andrii.leonets" Date: Mon, 24 Jan 2022 16:04:28 +0200 Subject: [PATCH 1/7] #9554 PR --- .../debezium/internals/DebeziumConverterUtils.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/DebeziumConverterUtils.java b/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/DebeziumConverterUtils.java index a97694727ed3..efcfe6faed61 100644 --- a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/DebeziumConverterUtils.java +++ b/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/DebeziumConverterUtils.java @@ -11,6 +11,7 @@ import java.time.LocalDate; import java.time.LocalDateTime; import java.time.format.DateTimeParseException; +import java.sql.Date; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -42,7 +43,10 @@ public static String convertDate(final Object input) { } else if (input instanceof Number) { return DataTypeUtils.toISO8601String( new Timestamp(((Number) input).longValue()).toLocalDateTime()); - } else if (input instanceof String) { + } else if (input instanceof Date) { + return DataTypeUtils.toISO8601String((Date) input); + } + else if (input instanceof String) { try { return LocalDateTime.parse((String) input).toString(); } catch (final DateTimeParseException e) { From e683402813045bf4252700ebe22ad8319e9e319c Mon Sep 17 00:00:00 2001 From: "andrii.leonets" Date: Mon, 24 Jan 2022 16:39:02 +0200 Subject: [PATCH 2/7] format --- .../debezium/internals/DebeziumConverterUtils.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/DebeziumConverterUtils.java b/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/DebeziumConverterUtils.java index efcfe6faed61..7e978adbc890 100644 --- a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/DebeziumConverterUtils.java +++ b/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/DebeziumConverterUtils.java @@ -6,12 +6,12 @@ import io.airbyte.db.DataTypeUtils; import io.debezium.spi.converter.RelationalColumn; +import java.sql.Date; import java.sql.Timestamp; import java.time.Duration; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.format.DateTimeParseException; -import java.sql.Date; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,8 +45,7 @@ public static String convertDate(final Object input) { new Timestamp(((Number) input).longValue()).toLocalDateTime()); } else if (input instanceof Date) { return DataTypeUtils.toISO8601String((Date) input); - } - else if (input instanceof String) { + } else if (input instanceof String) { try { return LocalDateTime.parse((String) input).toString(); } catch (final DateTimeParseException e) { From 50a532a670fb623f4c2ac4e1321514310a835989 Mon Sep 17 00:00:00 2001 From: "andrii.leonets" Date: Mon, 24 Jan 2022 19:23:33 +0200 Subject: [PATCH 3/7] incr version for Postgres --- .../decd338e-5647-4c0b-adf4-da0e75f5a750.json | 2 +- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- airbyte-integrations/connectors/source-postgres/Dockerfile | 2 +- docs/integrations/sources/postgres.md | 1 + 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/decd338e-5647-4c0b-adf4-da0e75f5a750.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/decd338e-5647-4c0b-adf4-da0e75f5a750.json index a85e89d23cde..ab16cb434e1c 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/decd338e-5647-4c0b-adf4-da0e75f5a750.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/decd338e-5647-4c0b-adf4-da0e75f5a750.json @@ -2,7 +2,7 @@ "sourceDefinitionId": "decd338e-5647-4c0b-adf4-da0e75f5a750", "name": "Postgres", "dockerRepository": "airbyte/source-postgres", - "dockerImageTag": "0.4.2", + "dockerImageTag": "0.4.3", "documentationUrl": "https://docs.airbyte.io/integrations/sources/postgres", "icon": "postgresql.svg" } diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 429326d3775f..5bdc70b98e74 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -564,7 +564,7 @@ - name: Postgres sourceDefinitionId: decd338e-5647-4c0b-adf4-da0e75f5a750 dockerRepository: airbyte/source-postgres - dockerImageTag: 0.4.2 + dockerImageTag: 0.4.3 documentationUrl: https://docs.airbyte.io/integrations/sources/postgres icon: postgresql.svg sourceType: database diff --git a/airbyte-integrations/connectors/source-postgres/Dockerfile b/airbyte-integrations/connectors/source-postgres/Dockerfile index 98017bc25cc9..0f7ebc8867bb 100644 --- a/airbyte-integrations/connectors/source-postgres/Dockerfile +++ b/airbyte-integrations/connectors/source-postgres/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-postgres COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.4.2 +LABEL io.airbyte.version=0.4.3 LABEL io.airbyte.name=airbyte/source-postgres diff --git a/docs/integrations/sources/postgres.md b/docs/integrations/sources/postgres.md index 70b2f10d2dca..8daaa24fd9cf 100644 --- a/docs/integrations/sources/postgres.md +++ b/docs/integrations/sources/postgres.md @@ -257,6 +257,7 @@ According to Postgres [documentation](https://www.postgresql.org/docs/14/datatyp | Version | Date | Pull Request | Subject | |:--------|:-----------|:-------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------| +| 0.4.3 | 2022-01-24 | [9554](https://github.com/airbytehq/airbyte/pull/9554) | :bug: Source Postgres: allow handling of java sql date | | 0.4.2 | 2022-01-13 | [9360](https://github.com/airbytehq/airbyte/pull/9360) | Added schema selection | | 0.4.1 | 2022-01-05 | [9116](https://github.com/airbytehq/airbyte/pull/9116) | Added materialized views processing | | 0.4.0 | 2021-12-13 | [8726](https://github.com/airbytehq/airbyte/pull/8726) | Support all Postgres types | From 97c7ccfaf51d081b07da28908fd237ffd33655d8 Mon Sep 17 00:00:00 2001 From: Andrii Leonets <30464745+DoNotPanicUA@users.noreply.github.com> Date: Mon, 24 Jan 2022 20:00:02 +0200 Subject: [PATCH 4/7] Update docs/integrations/sources/postgres.md Co-authored-by: Marcos Marx --- docs/integrations/sources/postgres.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/sources/postgres.md b/docs/integrations/sources/postgres.md index 8daaa24fd9cf..3b5cdc41c78d 100644 --- a/docs/integrations/sources/postgres.md +++ b/docs/integrations/sources/postgres.md @@ -257,7 +257,7 @@ According to Postgres [documentation](https://www.postgresql.org/docs/14/datatyp | Version | Date | Pull Request | Subject | |:--------|:-----------|:-------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------| -| 0.4.3 | 2022-01-24 | [9554](https://github.com/airbytehq/airbyte/pull/9554) | :bug: Source Postgres: allow handling of java sql date | +| 0.4.3 | 2022-01-24 | [9554](https://github.com/airbytehq/airbyte/pull/9554) | Allow handling of java sql date in CDC | | 0.4.2 | 2022-01-13 | [9360](https://github.com/airbytehq/airbyte/pull/9360) | Added schema selection | | 0.4.1 | 2022-01-05 | [9116](https://github.com/airbytehq/airbyte/pull/9116) | Added materialized views processing | | 0.4.0 | 2021-12-13 | [8726](https://github.com/airbytehq/airbyte/pull/8726) | Support all Postgres types | From 03fbd8d697829cd9140655c9db811e2d7ae73b0b Mon Sep 17 00:00:00 2001 From: "andrii.leonets" Date: Mon, 24 Jan 2022 20:22:43 +0200 Subject: [PATCH 5/7] test upd --- .../integrations/source/mssql/CdcMssqlSourceDatatypeTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceDatatypeTest.java b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceDatatypeTest.java index ab31da097253..fb9c59dddbd4 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceDatatypeTest.java @@ -331,7 +331,7 @@ protected void initTests() { .sourceType("date") .airbyteType(JsonSchemaPrimitive.STRING) .addInsertValues("'0001-01-01'", "'9999-12-31'", "'1999-01-08'", "null") - .addExpectedValues("0001-01-01", "9999-12-31", "1999-01-08", null) + .addExpectedValues("0001-01-01T00:00:00Z", "9999-12-31T00:00:00Z", "1999-01-08T00:00:00Z", null) .createTablePatternSql(CREATE_TABLE_SQL) .build()); From d230e63909f86a024875810813df99bc5304a680 Mon Sep 17 00:00:00 2001 From: "andrii.leonets" Date: Tue, 25 Jan 2022 13:24:08 +0200 Subject: [PATCH 6/7] test upd --- .../sources/CdcPostgresSourceDatatypeTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceDatatypeTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceDatatypeTest.java index c4964892eac6..6fe8a70f0bec 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceDatatypeTest.java @@ -225,7 +225,7 @@ protected void initTests() { .sourceType("date") .airbyteType(JsonSchemaPrimitive.STRING) .addInsertValues("'January 7, 1999'", "'1999-01-08'", "'1/9/1999'", "'January 10, 99 BC'", "'January 11, 99 AD'", "null") - .addExpectedValues("1999-01-07", "1999-01-08", "1999-01-09", "0099-01-10", "1999-01-11", null) + .addExpectedValues("1999-01-07T00:00:00Z", "1999-01-08T00:00:00Z", "1999-01-09T00:00:00Z", "0099-01-10T00:00:00Z", "1999-01-11T00:00:00Z", null) .build()); addDataTypeTestData( From 57685194463942c3cba32092807e0461957ea462 Mon Sep 17 00:00:00 2001 From: "andrii.leonets" Date: Tue, 25 Jan 2022 14:52:07 +0200 Subject: [PATCH 7/7] incr ver for mssql as effected source --- .../b5ea17b1-f170-46dc-bc31-cc744ca984c1.json | 2 +- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- airbyte-integrations/connectors/source-mssql/Dockerfile | 2 +- docs/integrations/sources/mssql.md | 5 +++-- docs/integrations/sources/postgres.md | 2 +- 5 files changed, 7 insertions(+), 6 deletions(-) diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/b5ea17b1-f170-46dc-bc31-cc744ca984c1.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/b5ea17b1-f170-46dc-bc31-cc744ca984c1.json index d56bf69b98ac..9058351ef75b 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/b5ea17b1-f170-46dc-bc31-cc744ca984c1.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/b5ea17b1-f170-46dc-bc31-cc744ca984c1.json @@ -2,7 +2,7 @@ "sourceDefinitionId": "b5ea17b1-f170-46dc-bc31-cc744ca984c1", "name": "Microsoft SQL Server (MSSQL)", "dockerRepository": "airbyte/source-mssql", - "dockerImageTag": "0.3.13", + "dockerImageTag": "0.3.14", "documentationUrl": "https://docs.airbyte.io/integrations/sources/mssql", "icon": "mssql.svg" } diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 5bdc70b98e74..26a761cd126b 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -418,7 +418,7 @@ - name: Microsoft SQL Server (MSSQL) sourceDefinitionId: b5ea17b1-f170-46dc-bc31-cc744ca984c1 dockerRepository: airbyte/source-mssql - dockerImageTag: 0.3.13 + dockerImageTag: 0.3.14 documentationUrl: https://docs.airbyte.io/integrations/sources/mssql icon: mssql.svg sourceType: database diff --git a/airbyte-integrations/connectors/source-mssql/Dockerfile b/airbyte-integrations/connectors/source-mssql/Dockerfile index 5691347a99a6..5bde73f85b61 100644 --- a/airbyte-integrations/connectors/source-mssql/Dockerfile +++ b/airbyte-integrations/connectors/source-mssql/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-mssql COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.3.13 +LABEL io.airbyte.version=0.3.14 LABEL io.airbyte.name=airbyte/source-mssql diff --git a/docs/integrations/sources/mssql.md b/docs/integrations/sources/mssql.md index d20309068c25..b7579593579e 100644 --- a/docs/integrations/sources/mssql.md +++ b/docs/integrations/sources/mssql.md @@ -292,8 +292,9 @@ If you do not see a type in this list, assume that it is coerced into a string. ## Changelog -| Version | Date | Pull Request | Subject | | -|:--------| :--- | :--- | :--- | :-- | +| Version | Date | Pull Request | Subject | +|:------- | :--------- | :----------------------------------------------------- | :------------------------------------- | +| 0.3.14 | 2022-01-24 | [9554](https://github.com/airbytehq/airbyte/pull/9554) | Allow handling of java sql date in CDC | | 0.3.13 | 2022-01-07 | [9094](https://github.com/airbytehq/airbyte/pull/9094) | Added support for missed data types | | 0.3.12 | 2021-12-30 | [9206](https://github.com/airbytehq/airbyte/pull/9206) | Update connector fields title/description | | 0.3.11 | 2021-12-24 | [8958](https://github.com/airbytehq/airbyte/pull/8958) | Add support for JdbcType.ARRAY | diff --git a/docs/integrations/sources/postgres.md b/docs/integrations/sources/postgres.md index 3b5cdc41c78d..755a022e9abc 100644 --- a/docs/integrations/sources/postgres.md +++ b/docs/integrations/sources/postgres.md @@ -257,7 +257,7 @@ According to Postgres [documentation](https://www.postgresql.org/docs/14/datatyp | Version | Date | Pull Request | Subject | |:--------|:-----------|:-------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------| -| 0.4.3 | 2022-01-24 | [9554](https://github.com/airbytehq/airbyte/pull/9554) | Allow handling of java sql date in CDC | +| 0.4.3 | 2022-01-24 | [9554](https://github.com/airbytehq/airbyte/pull/9554) | Allow handling of java sql date in CDC | | 0.4.2 | 2022-01-13 | [9360](https://github.com/airbytehq/airbyte/pull/9360) | Added schema selection | | 0.4.1 | 2022-01-05 | [9116](https://github.com/airbytehq/airbyte/pull/9116) | Added materialized views processing | | 0.4.0 | 2021-12-13 | [8726](https://github.com/airbytehq/airbyte/pull/8726) | Support all Postgres types |