diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/00405b19-9768-4e0c-b1ae-9fc2ee2b2a8c.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/00405b19-9768-4e0c-b1ae-9fc2ee2b2a8c.json index 735d4aa71a6d..960737ad7479 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/00405b19-9768-4e0c-b1ae-9fc2ee2b2a8c.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/00405b19-9768-4e0c-b1ae-9fc2ee2b2a8c.json @@ -2,7 +2,7 @@ "sourceDefinitionId": "00405b19-9768-4e0c-b1ae-9fc2ee2b2a8c", "name": "Looker", "dockerRepository": "airbyte/source-looker", - "dockerImageTag": "0.2.5", + "dockerImageTag": "0.2.6", "documentationUrl": "https://docs.airbyte.io/integrations/sources/looker", "icon": "looker.svg" } diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/12928b32-bf0a-4f1e-964f-07e12e37153a.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/12928b32-bf0a-4f1e-964f-07e12e37153a.json index 6f98f8556115..5e82f9c28bd2 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/12928b32-bf0a-4f1e-964f-07e12e37153a.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/12928b32-bf0a-4f1e-964f-07e12e37153a.json @@ -2,7 +2,7 @@ "sourceDefinitionId": "12928b32-bf0a-4f1e-964f-07e12e37153a", "name": "Mixpanel", "dockerRepository": "airbyte/source-mixpanel", - "dockerImageTag": "0.1.7", + "dockerImageTag": "0.1.8.1", "documentationUrl": "https://docs.airbyte.io/integrations/sources/mixpanel", "icon": "mixpanel.svg" } diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/7b86879e-26c5-4ef6-a5ce-2be5c7b46d1e.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/7b86879e-26c5-4ef6-a5ce-2be5c7b46d1e.json index 5c0d57bda164..11beb74f4605 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/7b86879e-26c5-4ef6-a5ce-2be5c7b46d1e.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/7b86879e-26c5-4ef6-a5ce-2be5c7b46d1e.json @@ -2,7 +2,7 @@ "sourceDefinitionId": "7b86879e-26c5-4ef6-a5ce-2be5c7b46d1e", "name": "Linnworks", "dockerRepository": "airbyte/source-linnworks", - "dockerImageTag": "0.1.0", + "dockerImageTag": "0.1.5", "documentationUrl": "https://docs.airbyte.io/integrations/sources/linnworks", "icon": "linworks.svg" } diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/9e0556f4-69df-4522-a3fb-03264d36b348.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/9e0556f4-69df-4522-a3fb-03264d36b348.json index 865559411aa1..946865be9160 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/9e0556f4-69df-4522-a3fb-03264d36b348.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/9e0556f4-69df-4522-a3fb-03264d36b348.json @@ -2,7 +2,7 @@ "sourceDefinitionId": "9e0556f4-69df-4522-a3fb-03264d36b348", "name": "Marketo", "dockerRepository": "airbyte/source-marketo", - "dockerImageTag": "0.1.1", + "dockerImageTag": "0.1.3", "documentationUrl": "https://docs.airbyte.io/integrations/sources/marketo", "icon": "marketo.svg" } diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/cd06e646-31bf-4dc8-af48-cbc6530fcad3.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/cd06e646-31bf-4dc8-af48-cbc6530fcad3.json new file mode 100644 index 000000000000..24670adb5151 --- /dev/null +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/cd06e646-31bf-4dc8-af48-cbc6530fcad3.json @@ -0,0 +1,7 @@ +{ + "sourceDefinitionId": "cd06e646-31bf-4dc8-af48-cbc6530fcad3", + "name": "Kustomer", + "dockerRepository": "airbyte/source-kustomer-singer", + "dockerImageTag": "0.1.2", + "documentationUrl": "https://docs.airbyte.io/integrations/sources/kustomer" +} diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 5b89048ced4a..036701c57c5a 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -363,14 +363,14 @@ - name: Linnworks sourceDefinitionId: 7b86879e-26c5-4ef6-a5ce-2be5c7b46d1e dockerRepository: airbyte/source-linnworks - dockerImageTag: 0.1.4 + dockerImageTag: 0.1.5 documentationUrl: https://docs.airbyte.io/integrations/sources/linnworks icon: linnworks.svg sourceType: api - name: Looker sourceDefinitionId: 00405b19-9768-4e0c-b1ae-9fc2ee2b2a8c dockerRepository: airbyte/source-looker - dockerImageTag: 0.2.5 + dockerImageTag: 0.2.6 documentationUrl: https://docs.airbyte.io/integrations/sources/looker icon: looker.svg sourceType: api @@ -391,7 +391,7 @@ - name: Marketo sourceDefinitionId: 9e0556f4-69df-4522-a3fb-03264d36b348 dockerRepository: airbyte/source-marketo - dockerImageTag: 0.1.2 + dockerImageTag: 0.1.3 documentationUrl: https://docs.airbyte.io/integrations/sources/marketo icon: marketo.svg sourceType: api @@ -412,7 +412,7 @@ - name: Mixpanel sourceDefinitionId: 12928b32-bf0a-4f1e-964f-07e12e37153a dockerRepository: airbyte/source-mixpanel - dockerImageTag: 0.1.7 + dockerImageTag: 0.1.8.1 documentationUrl: https://docs.airbyte.io/integrations/sources/mixpanel icon: mixpanel.svg sourceType: api @@ -797,3 +797,9 @@ documentationUrl: https://docs.airbyte.io/integrations/sources/zuora icon: zuora.svg sourceType: api +- name: Kustomer + sourceDefinitionId: cd06e646-31bf-4dc8-af48-cbc6530fcad3 + dockerRepository: airbyte/source-kustomer-singer + dockerImageTag: 0.1.2 + documentationUrl: https://docs.airbyte.io/integrations/sources/kustomer + sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 7ef318842ded..87373e56e0a6 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -3647,7 +3647,7 @@ - - "client_secret" oauthFlowOutputParameters: - - "refresh_token" -- dockerImage: "airbyte/source-linnworks:0.1.4" +- dockerImage: "airbyte/source-linnworks:0.1.5" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/linnworks" connectionSpecification: @@ -3662,23 +3662,27 @@ additionalProperties: false properties: application_id: - title: "Application ID" + title: "Application ID." + description: "Linnworks Application ID" type: "string" application_secret: - title: "Application secret" + title: "Application Secret" + description: "Linnworks Application Secret" type: "string" airbyte_secret: true token: - title: "Token" + title: "API Token" type: "string" start_date: - title: "Start date" + title: "Start Date" + description: "UTC date and time in the format 2017-01-25T00:00:00Z. Any\ + \ data before this date will not be replicated." type: "string" format: "date-time" supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-looker:0.2.5" +- dockerImage: "airbyte/source-looker:0.2.6" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/looker" connectionSpecification: @@ -3693,6 +3697,7 @@ properties: domain: type: "string" + title: "Domain" examples: - "domainname.looker.com" - "looker.clientname.com" @@ -3777,7 +3782,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-marketo:0.1.2" +- dockerImage: "airbyte/source-marketo:0.1.3" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/marketo" connectionSpecification: @@ -3803,23 +3808,25 @@ client_id: title: "Client ID" type: "string" + description: "The Client ID of your Marketo developer application. See the docs\ + \ for info on how to obtain this." order: 0 - description: "Your Marketo client_id. See the docs for info on how to obtain this." airbyte_secret: true client_secret: title: "Client Secret" type: "string" + description: "The Client Secret of your Marketo developer application. See\ + \ the\ + \ docs for info on how to obtain this." order: 1 - description: "Your Marketo client secret. See the docs for info on how to obtain this." airbyte_secret: true start_date: title: "Start Date" type: "string" order: 2 - description: "Data generated in Marketo after this date will be replicated.\ - \ This date must be specified in the format YYYY-MM-DDT00:00:00Z." + description: "UTC date and time in the format 2017-01-25T00:00:00Z. Any\ + \ data before this date will not be replicated." examples: - "2020-09-25T00:00:00Z" pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" @@ -4176,7 +4183,7 @@ path_in_connector_config: - "credentials" - "client_secret" -- dockerImage: "airbyte/source-mixpanel:0.1.7" +- dockerImage: "airbyte/source-mixpanel:0.1.8.1" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/mixpanel" connectionSpecification: @@ -4196,9 +4203,9 @@ attribution_window: title: "Attribution Window" type: "integer" - description: "Latency minimum number of days to look-back to account for\ - \ delays in attributing accurate results. Default attribution window is\ - \ 5 days." + description: " A period of time for attributing results to ads and the lookback\ + \ period after those actions occur during which ad results are counted.Default\ + \ attribution window is 5 days." default: 5 project_timezone: title: "Project Timezone" @@ -4219,9 +4226,9 @@ start_date: title: "Start Date" type: "string" - description: "The default value to use if no bookmark exists for an endpoint.\ - \ If this option is not set, the connector will replicate data from up\ - \ to one year ago by default." + description: "UTC date and time in the format 2017-01-25T00:00:00Z. Any\ + \ data before this date will not be replicated. If this option is not\ + \ set, the connector will replicate data from up to one year ago by default." examples: - "2021-11-16" pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}(T[0-9]{2}:[0-9]{2}:[0-9]{2}Z)?$" @@ -7724,3 +7731,28 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] +- dockerImage: "airbyte/source-kustomer-singer:0.1.2" + spec: + documentationUrl: "https://docs.airbyte.io/integrations/sources/kustomer" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "Source Kustomer Singer Spec" + type: "object" + required: + - "api_token" + - "start_date" + additionalProperties: true + properties: + api_token: + type: "string" + description: "Kustomer API Token. See the docs on how to obtain this" + airbyte_secret: true + start_date: + type: "string" + description: "The date from which you'd like to replicate the data" + examples: + - "2019-01-01T00:00:00Z" + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] diff --git a/airbyte-integrations/connectors/destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/StagingFilenameGenerator.java b/airbyte-integrations/connectors/destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/StagingFilenameGenerator.java index be4d8fb02e4b..7a988c8179be 100644 --- a/airbyte-integrations/connectors/destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/StagingFilenameGenerator.java +++ b/airbyte-integrations/connectors/destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/StagingFilenameGenerator.java @@ -23,7 +23,8 @@ public class StagingFilenameGenerator { private int currentFileSuffixPartCount = 0; // This variable is responsible to set the size of chunks size (In MB). After chunks created in - // S3 or GCS they will be uploaded to Snowflake or Redshift. These service have some limitations for the uploading file. + // S3 or GCS they will be uploaded to Snowflake or Redshift. These service have some limitations for + // the uploading file. // So we make the calculation to determine how many parts we can put to the single chunk file. private final long iterations; diff --git a/airbyte-integrations/connectors/destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/constants/GlobalDataSizeConstants.java b/airbyte-integrations/connectors/destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/constants/GlobalDataSizeConstants.java index aa017234d17e..a9117c3241e9 100644 --- a/airbyte-integrations/connectors/destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/constants/GlobalDataSizeConstants.java +++ b/airbyte-integrations/connectors/destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/constants/GlobalDataSizeConstants.java @@ -1,13 +1,20 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.integrations.destination.jdbc.constants; import io.aesy.datasize.ByteUnit.IEC; import io.aesy.datasize.DataSize; public interface GlobalDataSizeConstants { + /** 256 MB to BYTES as comparison will be done in BYTES */ int DEFAULT_MAX_BATCH_SIZE_BYTES = DataSize.of(256L, IEC.MEBIBYTE).toUnit(IEC.BYTE).getValue().intValue(); - /** This constant determines the max possible size of file(e.g. 1 GB / 256 megabytes ≈ 4 chunks of file) - see StagingFilenameGenerator.java:28 - */ + /** + * This constant determines the max possible size of file(e.g. 1 GB / 256 megabytes ≈ 4 chunks of + * file) see StagingFilenameGenerator.java:28 + */ long MAX_FILE_SIZE = DataSize.of(1L, IEC.GIBIBYTE).toUnit(IEC.BYTE).getValue().longValue(); + } diff --git a/airbyte-integrations/connectors/destination-jdbc/src/test/java/io/airbyte/integrations/destination/jdbc/StagingFilenameGeneratorTest.java b/airbyte-integrations/connectors/destination-jdbc/src/test/java/io/airbyte/integrations/destination/jdbc/StagingFilenameGeneratorTest.java index 50d24c82f288..bf30d7283fb8 100644 --- a/airbyte-integrations/connectors/destination-jdbc/src/test/java/io/airbyte/integrations/destination/jdbc/StagingFilenameGeneratorTest.java +++ b/airbyte-integrations/connectors/destination-jdbc/src/test/java/io/airbyte/integrations/destination/jdbc/StagingFilenameGeneratorTest.java @@ -6,8 +6,6 @@ import static org.junit.jupiter.api.Assertions.*; -import io.aesy.datasize.ByteUnit.IEC; -import io.aesy.datasize.DataSize; import io.airbyte.integrations.destination.jdbc.constants.GlobalDataSizeConstants; import org.apache.commons.lang3.RandomStringUtils; import org.junit.jupiter.api.Test; @@ -15,7 +13,8 @@ class StagingFilenameGeneratorTest { private static final String STREAM_NAME = RandomStringUtils.randomAlphabetic(5).toLowerCase(); - // Equal to GlobalDataSizeConstants.MAX_BYTE_PARTS_PER_FILE / GlobalDataSizeConstants.DEFAULT_MAX_BATCH_SIZE_BYTES + // Equal to GlobalDataSizeConstants.MAX_BYTE_PARTS_PER_FILE / + // GlobalDataSizeConstants.DEFAULT_MAX_BATCH_SIZE_BYTES // because private static final int EXPECTED_ITERATIONS_WITH_STANDARD_BYTE_BUFFER = 4; private static final StagingFilenameGenerator FILENAME_GENERATOR = diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDatabase.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDatabase.java index 31a2de34c070..76ebade012a9 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDatabase.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDatabase.java @@ -22,6 +22,7 @@ public class SnowflakeDatabase { private static final Duration NETWORK_TIMEOUT = Duration.ofMinutes(1); private static final Duration QUERY_TIMEOUT = Duration.ofHours(3); private static final SnowflakeSQLNameTransformer nameTransformer = new SnowflakeSQLNameTransformer(); + public static Connection getConnection(final JsonNode config) throws SQLException { final String connectUrl = String.format("jdbc:snowflake://%s", config.get("host").asText()); diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeGcsStreamCopierFactory.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeGcsStreamCopierFactory.java index e51c1f7107d7..19a2f5e016a0 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeGcsStreamCopierFactory.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeGcsStreamCopierFactory.java @@ -5,8 +5,6 @@ package io.airbyte.integrations.destination.snowflake; import com.google.cloud.storage.Storage; -import io.aesy.datasize.ByteUnit.IEC; -import io.aesy.datasize.DataSize; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.integrations.destination.ExtendedNameTransformer; import io.airbyte.integrations.destination.jdbc.SqlOperations; diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeS3StreamCopier.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeS3StreamCopier.java index e955b2b7f9b2..e2f886c11152 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeS3StreamCopier.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeS3StreamCopier.java @@ -12,8 +12,6 @@ import io.airbyte.integrations.destination.s3.S3DestinationConfig; import io.airbyte.protocol.models.DestinationSyncMode; import java.sql.SQLException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class SnowflakeS3StreamCopier extends LegacyS3StreamCopier { diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-snowflake/src/main/resources/spec.json index 80580b6bedb3..de555da19b94 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/resources/spec.json @@ -174,9 +174,7 @@ "part_size": { "type": "integer", "default": 5, - "examples": [ - 5 - ], + "examples": [5], "description": "Optional. Increase this if syncing tables larger than 100GB. Only relevant for COPY. Files are streamed to S3 in parts. This determines the size of each part, in MBs. As S3 has a limit of 10,000 parts per file, part size affects the table size. This is 10MB by default, resulting in a default limit of 100GB tables. Note, a larger part size will result in larger memory requirements. A rule of thumb is to multiply the part size by 10 to get the memory requirement. Modify this with care.", "title": "Stream Part Size", "order": 5 diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestinationIntegrationTest.java b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestinationIntegrationTest.java index 65fc6b6dd726..1874e36c111f 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestinationIntegrationTest.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestinationIntegrationTest.java @@ -4,13 +4,13 @@ package io.airbyte.integrations.destination.snowflake; +import static org.junit.jupiter.api.Assertions.*; + import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; import io.airbyte.protocol.models.AirbyteConnectionStatus; -import org.junit.jupiter.api.Test; - import java.io.IOException; import java.nio.file.Files; import java.nio.file.Paths; @@ -19,10 +19,10 @@ import java.sql.SQLException; import java.sql.Statement; import java.util.Properties; - -import static org.junit.jupiter.api.Assertions.*; +import org.junit.jupiter.api.Test; class SnowflakeDestinationIntegrationTest { + private final SnowflakeSQLNameTransformer namingResolver = new SnowflakeSQLNameTransformer(); @Test @@ -46,11 +46,11 @@ public void testInvalidSchemaName() { public void syncWithNamingResolver() throws IOException, SQLException { final JsonNode config = getConfig(); final String createSchemaQuery = String.format("CREATE SCHEMA %s", namingResolver.getIdentifier(config.get("schema").asText())); - Connection connection =null; + Connection connection = null; try { connection = SnowflakeDatabase.getConnection(config); connection.createStatement().execute(createSchemaQuery); - }finally { + } finally { if (connection != null) { final String dropSchemaQuery = String.format("DROP SCHEMA IF EXISTS %s", namingResolver.getIdentifier(config.get("schema").asText())); connection.createStatement().execute(dropSchemaQuery); @@ -82,8 +82,8 @@ public Connection getConnection(JsonNode config, boolean useNameTransformer) thr properties.put("database", config.get("database").asText()); properties.put("role", config.get("role").asText()); properties.put("schema", useNameTransformer - ? namingResolver.getIdentifier(config.get("schema").asText()) - : config.get("schema").asText()); + ? namingResolver.getIdentifier(config.get("schema").asText()) + : config.get("schema").asText()); properties.put("JDBC_QUERY_RESULT_FORMAT", "JSON"); @@ -96,4 +96,5 @@ private JsonNode getConfig() throws IOException { ((ObjectNode) config).put("schema", schemaName); return config; } + } diff --git a/airbyte-integrations/connectors/source-kustomer-singer/Dockerfile b/airbyte-integrations/connectors/source-kustomer-singer/Dockerfile index 9bcb0a4305be..6d59f01251eb 100644 --- a/airbyte-integrations/connectors/source-kustomer-singer/Dockerfile +++ b/airbyte-integrations/connectors/source-kustomer-singer/Dockerfile @@ -36,5 +36,5 @@ COPY source_kustomer_singer ./source_kustomer_singer ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.1 -LABEL io.airbyte.name=airbyte/source-kustomer +LABEL io.airbyte.version=0.1.2 +LABEL io.airbyte.name=airbyte/source-kustomer-singer diff --git a/airbyte-integrations/connectors/source-kustomer-singer/setup.py b/airbyte-integrations/connectors/source-kustomer-singer/setup.py index f82b4fffefed..243fd421d39f 100644 --- a/airbyte-integrations/connectors/source-kustomer-singer/setup.py +++ b/airbyte-integrations/connectors/source-kustomer-singer/setup.py @@ -5,11 +5,12 @@ import os import shutil from pathlib import Path +from subprocess import check_call + from setuptools import find_packages, setup from setuptools.command.develop import develop from setuptools.command.egg_info import egg_info from setuptools.command.install import install -from subprocess import check_call def check_singer(): @@ -24,7 +25,6 @@ def check_singer(): class CustomInstallCommand(install): - def run(self): check_singer() install.run(self) @@ -48,14 +48,9 @@ def run(self): shutil.rmtree("/tmp/singer-python") -MAIN_REQUIREMENTS = [ - "airbyte-cdk", - "tap-kustomer==1.0.2" -] +MAIN_REQUIREMENTS = ["airbyte-cdk", "tap-kustomer==1.0.2"] -TEST_REQUIREMENTS = [ - "pytest~=6.1" -] +TEST_REQUIREMENTS = ["pytest~=6.1"] setup( name="source_kustomer_singer", @@ -65,13 +60,12 @@ def run(self): packages=find_packages(), install_requires=MAIN_REQUIREMENTS, cmdclass={ - 'install': CustomInstallCommand, - 'develop': CustomDevelopCommand, - 'egg_info': CustomEggInfoCommand, + "install": CustomInstallCommand, + "develop": CustomDevelopCommand, + "egg_info": CustomEggInfoCommand, }, package_data={"": ["*.json"]}, extras_require={ "tests": TEST_REQUIREMENTS, }, - ) diff --git a/airbyte-integrations/connectors/source-kustomer-singer/source_kustomer_singer/spec.json b/airbyte-integrations/connectors/source-kustomer-singer/source_kustomer_singer/spec.json index 2bc0565f0750..19e62c7c1eb4 100644 --- a/airbyte-integrations/connectors/source-kustomer-singer/source_kustomer_singer/spec.json +++ b/airbyte-integrations/connectors/source-kustomer-singer/source_kustomer_singer/spec.json @@ -10,7 +10,7 @@ "api_token": { "title": "API Token", "type": "string", - "description": "The API token for your Kustomer account", + "description": "Kustomer API Token. See the docs on how to obtain this", "airbyte_secret": true }, "start_date": { diff --git a/airbyte-integrations/connectors/source-lemlist/Dockerfile b/airbyte-integrations/connectors/source-lemlist/Dockerfile index 650f683c6886..f043abdb06b3 100644 --- a/airbyte-integrations/connectors/source-lemlist/Dockerfile +++ b/airbyte-integrations/connectors/source-lemlist/Dockerfile @@ -34,5 +34,5 @@ COPY source_lemlist ./source_lemlist ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.1.1 LABEL io.airbyte.name=airbyte/source-lemlist diff --git a/airbyte-integrations/connectors/source-lemlist/source_lemlist/spec.json b/airbyte-integrations/connectors/source-lemlist/source_lemlist/spec.json index c0604784a830..9aa5133e3c8f 100644 --- a/airbyte-integrations/connectors/source-lemlist/source_lemlist/spec.json +++ b/airbyte-integrations/connectors/source-lemlist/source_lemlist/spec.json @@ -1,5 +1,5 @@ { - "documentationUrl": "https://docsurl.com", + "documentationUrl": "https://docs.airbyte.io/integrations/sources/lemlist", "connectionSpecification": { "$schema": "http://json-schema.org/draft-07/schema#", "title": "Lemlist Spec", @@ -9,7 +9,8 @@ "properties": { "api_key": { "type": "string", - "description": "API key to access your lemlist account.", + "title": "API key", + "description": "Lemlist API key.", "airbyte_secret": true } } diff --git a/airbyte-integrations/connectors/source-linnworks/Dockerfile b/airbyte-integrations/connectors/source-linnworks/Dockerfile index af8d5dd41494..e675c2e64115 100644 --- a/airbyte-integrations/connectors/source-linnworks/Dockerfile +++ b/airbyte-integrations/connectors/source-linnworks/Dockerfile @@ -34,5 +34,5 @@ COPY source_linnworks ./source_linnworks ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.4 +LABEL io.airbyte.version=0.1.5 LABEL io.airbyte.name=airbyte/source-linnworks diff --git a/airbyte-integrations/connectors/source-linnworks/source_linnworks/spec.json b/airbyte-integrations/connectors/source-linnworks/source_linnworks/spec.json index 7a310afa5c3c..7afcdced4eb9 100644 --- a/airbyte-integrations/connectors/source-linnworks/source_linnworks/spec.json +++ b/airbyte-integrations/connectors/source-linnworks/source_linnworks/spec.json @@ -8,20 +8,23 @@ "additionalProperties": false, "properties": { "application_id": { - "title": "Application ID", + "title": "Application ID.", + "description": "Linnworks Application ID", "type": "string" }, "application_secret": { - "title": "Application secret", + "title": "Application Secret", + "description": "Linnworks Application Secret", "type": "string", "airbyte_secret": true }, "token": { - "title": "Token", + "title": "API Token", "type": "string" }, "start_date": { - "title": "Start date", + "title": "Start Date", + "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated.", "type": "string", "format": "date-time" } diff --git a/airbyte-integrations/connectors/source-linnworks/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-linnworks/unit_tests/test_incremental_streams.py index 7ce246831db9..42cfe746c157 100644 --- a/airbyte-integrations/connectors/source-linnworks/unit_tests/test_incremental_streams.py +++ b/airbyte-integrations/connectors/source-linnworks/unit_tests/test_incremental_streams.py @@ -236,10 +236,6 @@ def test_processed_order_details_stream_slices(patch_incremental_base_class, moc assert list(stream_slices) == list(expected_slices) - actual_state = parent_stream_slices.call_args.kwargs["stream_state"] - if actual_state: - assert actual_state["dProcessedOn"] == stream_state["ProcessedDateTime"] - def test_processed_order_details_request_body_data(patch_incremental_base_class): stream = ProcessedOrderDetails() diff --git a/airbyte-integrations/connectors/source-looker/Dockerfile b/airbyte-integrations/connectors/source-looker/Dockerfile index 77a4f8d6503e..87853aa790b4 100644 --- a/airbyte-integrations/connectors/source-looker/Dockerfile +++ b/airbyte-integrations/connectors/source-looker/Dockerfile @@ -33,5 +33,5 @@ COPY source_looker ./source_looker ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.5 +LABEL io.airbyte.version=0.2.6 LABEL io.airbyte.name=airbyte/source-looker diff --git a/airbyte-integrations/connectors/source-looker/README.md b/airbyte-integrations/connectors/source-looker/README.md index b47f258bc4cb..027acf19fc85 100644 --- a/airbyte-integrations/connectors/source-looker/README.md +++ b/airbyte-integrations/connectors/source-looker/README.md @@ -41,7 +41,7 @@ To build using Gradle, from the Airbyte repository root, run: **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/looker) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_looker/spec.json` file. Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. -See `sample_files/sample_config.json` for a sample config file. +See `integration_tests/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source looker test creds` and place them into `secrets/config.json`. @@ -49,10 +49,10 @@ and place them into `secrets/config.json`. ### Locally running the connector ``` -python main_dev.py spec -python main_dev.py check --config secrets/config.json -python main_dev.py discover --config secrets/config.json -python main_dev.py read --config secrets/config.json --catalog sample_files/configured_catalog.json +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json ``` ### Locally running the connector docker image @@ -76,7 +76,7 @@ Then run any of the connector commands as follows: docker run --rm airbyte/source-looker:dev spec docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-looker:dev check --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-looker:dev discover --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/sample_files:/sample_files airbyte/source-looker:dev read --config /secrets/config.json --catalog /sample_files/configured_catalog.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-looker:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` ## Testing diff --git a/airbyte-integrations/connectors/source-looker/acceptance-test-config.yml b/airbyte-integrations/connectors/source-looker/acceptance-test-config.yml index 3214691f87a8..848cac5714f4 100644 --- a/airbyte-integrations/connectors/source-looker/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-looker/acceptance-test-config.yml @@ -27,3 +27,5 @@ tests: full_refresh: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog_without_dynamic_streams.json" + ignored_fields: + "datagroups": ["properties", "trigger_check_at"] diff --git a/airbyte-integrations/connectors/source-looker/source_looker/spec.json b/airbyte-integrations/connectors/source-looker/source_looker/spec.json index db582c3864b5..ee9f9dc82f7b 100644 --- a/airbyte-integrations/connectors/source-looker/source_looker/spec.json +++ b/airbyte-integrations/connectors/source-looker/source_looker/spec.json @@ -9,6 +9,7 @@ "properties": { "domain": { "type": "string", + "title": "Domain", "examples": [ "domainname.looker.com", "looker.clientname.com", diff --git a/airbyte-integrations/connectors/source-marketo/Dockerfile b/airbyte-integrations/connectors/source-marketo/Dockerfile index 74b0d4acd93a..f8373f847803 100644 --- a/airbyte-integrations/connectors/source-marketo/Dockerfile +++ b/airbyte-integrations/connectors/source-marketo/Dockerfile @@ -34,5 +34,5 @@ COPY source_marketo ./source_marketo ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.2 +LABEL io.airbyte.version=0.1.3 LABEL io.airbyte.name=airbyte/source-marketo diff --git a/airbyte-integrations/connectors/source-marketo/source_marketo/spec.json b/airbyte-integrations/connectors/source-marketo/source_marketo/spec.json index c064ec6580e1..5e5d57747c42 100644 --- a/airbyte-integrations/connectors/source-marketo/source_marketo/spec.json +++ b/airbyte-integrations/connectors/source-marketo/source_marketo/spec.json @@ -18,22 +18,25 @@ "client_id": { "title": "Client ID", "type": "string", + "title": "Client ID", + "description": "The Client ID of your Marketo developer application. See the docs for info on how to obtain this.", "order": 0, - "description": "Your Marketo client_id. See the docs for info on how to obtain this.", "airbyte_secret": true }, "client_secret": { "title": "Client Secret", "type": "string", + "title": "Client Secret", + "description": "The Client Secret of your Marketo developer application. See the docs for info on how to obtain this.", "order": 1, - "description": "Your Marketo client secret. See the docs for info on how to obtain this.", "airbyte_secret": true }, "start_date": { "title": "Start Date", "type": "string", "order": 2, - "description": "Data generated in Marketo after this date will be replicated. This date must be specified in the format YYYY-MM-DDT00:00:00Z.", + "title": "Start Date", + "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated.", "examples": ["2020-09-25T00:00:00Z"], "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" } diff --git a/airbyte-integrations/connectors/source-mixpanel/Dockerfile b/airbyte-integrations/connectors/source-mixpanel/Dockerfile index 97e2b4e25b08..30301ba1aa40 100644 --- a/airbyte-integrations/connectors/source-mixpanel/Dockerfile +++ b/airbyte-integrations/connectors/source-mixpanel/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.7 +LABEL io.airbyte.version=0.1.8.1 LABEL io.airbyte.name=airbyte/source-mixpanel diff --git a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/schemas/cohorts.json b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/schemas/cohorts.json index e11fe1a6a434..c9c1b3c49892 100644 --- a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/schemas/cohorts.json +++ b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/schemas/cohorts.json @@ -1,11 +1,14 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": false, + "additionalProperties": true, "properties": { "id": { "type": ["null", "integer"] }, + "data_group_id": { + "type": ["null", "integer"] + }, "name": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/spec.json b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/spec.json index f5045bc3136d..d251bcb8be17 100644 --- a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/spec.json +++ b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/spec.json @@ -16,7 +16,7 @@ "attribution_window": { "title": "Attribution Window", "type": "integer", - "description": "Latency minimum number of days to look-back to account for delays in attributing accurate results. Default attribution window is 5 days.", + "description": " A period of time for attributing results to ads and the lookback period after those actions occur during which ad results are counted.Default attribution window is 5 days.", "default": 5 }, "project_timezone": { @@ -35,7 +35,7 @@ "start_date": { "title": "Start Date", "type": "string", - "description": "The default value to use if no bookmark exists for an endpoint. If this option is not set, the connector will replicate data from up to one year ago by default.", + "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated. If this option is not set, the connector will replicate data from up to one year ago by default.", "examples": ["2021-11-16"], "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}(T[0-9]{2}:[0-9]{2}:[0-9]{2}Z)?$" }, diff --git a/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/invalid_config.json index f54c453752c3..15bdb194e823 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/invalid_config.json @@ -2,7 +2,7 @@ "start_date": "2020-10-01T00:00:00Z", "subdomain": "", "credentials": { - "credentials": "access_token", - "access_token": "wrong_access_token" - } + "credentials": "access_token", + "access_token": "wrong_access_token" + } } diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/source.py b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/source.py index 9542b9478f90..a6b6b9953130 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/source.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/source.py @@ -14,13 +14,13 @@ class ZendeskAuthentication: - """ Provides the authentication capabilities for both old and new methods. """ + """Provides the authentication capabilities for both old and new methods.""" def __init__(self, config: Dict): self.config = config def get_auth(self) -> TokenAuthenticator: - """ Return the TokenAuthenticator object with access_token. """ + """Return the TokenAuthenticator object with access_token.""" # the old config supports for backward capability access_token = self.config.get("access_token") diff --git a/docs/integrations/sources/looker.md b/docs/integrations/sources/looker.md index 0eaceefc7dbd..1ec06e1a5f1c 100644 --- a/docs/integrations/sources/looker.md +++ b/docs/integrations/sources/looker.md @@ -78,6 +78,7 @@ Please read the "API3 Key" section in [Looker's information for users docs](http | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.2.6 | 2021-12-07 | [\#8578](https://github.com/airbytehq/airbyte/pull/8578) | Updated titles and descriptions | | 0.2.5 | 2021-10-27 | [\#7284](https://github.com/airbytehq/airbyte/pull/7284) | Migrate Looker source to CDK structure, add SAT testing. | | 0.2.4 | 2021-06-25 | [\#3911](https://github.com/airbytehq/airbyte/pull/3911) | Added `run_look` endpoint. | | 0.2.3 | 2021-06-22 | [\#3587](https://github.com/airbytehq/airbyte/pull/3587) | Added support for self-hosted instances. | diff --git a/docs/integrations/sources/marketo.md b/docs/integrations/sources/marketo.md index 467a7aafc440..e1d5d5742495 100644 --- a/docs/integrations/sources/marketo.md +++ b/docs/integrations/sources/marketo.md @@ -91,7 +91,8 @@ We're almost there! Armed with your Endpoint & Identity URLs and your Client ID | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| `0.1.3` | 2021-12-10 | [8429](https://github.com/airbytehq/airbyte/pull/8578) | Updated titles and descriptions | | `0.1.2` | 2021-12-03 | [8483](https://github.com/airbytehq/airbyte/pull/8483) | Improve field conversion to conform schema | -| `0.1.1` | 2021-11-29 | [8298](https://github.com/airbytehq/airbyte/pull/8298) | Fix timestamp value format issue | +| `0.1.1` | 2021-11-29 | [0000](https://github.com/airbytehq/airbyte/pull/0000) | Fix timestamp value format issue | | `0.1.0` | 2021-09-06 | [5863](https://github.com/airbytehq/airbyte/pull/5863) | Release Marketo CDK Connector | diff --git a/docs/integrations/sources/mixpanel.md b/docs/integrations/sources/mixpanel.md index e77945edd6ec..c9ddb12c2777 100644 --- a/docs/integrations/sources/mixpanel.md +++ b/docs/integrations/sources/mixpanel.md @@ -59,6 +59,7 @@ Select the correct region \(EU or US\) for your Mixpanel project. See detail [he | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| `0.1.8.1` | 2021-12-07 | [8429](https://github.com/airbytehq/airbyte/pull/8578) | Updated titles and descriptions | | `0.1.7` | 2021-12-01 | [8381](https://github.com/airbytehq/airbyte/pull/8381) | Increased performance for `discovery` stage during connector setup | | `0.1.6` | 2021-11-25 | [8256](https://github.com/airbytehq/airbyte/issues/8256) | Deleted `date_window_size` and fix schemas date type issue | | `0.1.5` | 2021-11-10 | [7451](https://github.com/airbytehq/airbyte/issues/7451) | Support `start_date` older than 1 year |