From 23401415b90c32f91f50aab8254b709c8e8515a1 Mon Sep 17 00:00:00 2001 From: Andrii Leonets Date: Sun, 30 May 2021 13:48:43 +0300 Subject: [PATCH 01/11] first test + abstract class for the comprehensive tests --- .../bases/standard-source-test/build.gradle | 1 + .../source/SourceComprehensiveTest.java | 284 ++++++++++++++++++ .../mysql/MySqlSourceComprehensiveTest.java | 102 +++++++ 3 files changed, 387 insertions(+) create mode 100644 airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceComprehensiveTest.java create mode 100644 airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceComprehensiveTest.java diff --git a/airbyte-integrations/bases/standard-source-test/build.gradle b/airbyte-integrations/bases/standard-source-test/build.gradle index 01c4c9bf8c388..4fcf44bb5f225 100644 --- a/airbyte-integrations/bases/standard-source-test/build.gradle +++ b/airbyte-integrations/bases/standard-source-test/build.gradle @@ -12,6 +12,7 @@ plugins { import org.jsoup.Jsoup; dependencies { + implementation project(':airbyte-db') implementation project(':airbyte-config:models') implementation project(':airbyte-protocol:models') implementation project(':airbyte-workers') diff --git a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceComprehensiveTest.java b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceComprehensiveTest.java new file mode 100644 index 0000000000000..966a8f29d2def --- /dev/null +++ b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceComprehensiveTest.java @@ -0,0 +1,284 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.standardtest.source; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.Lists; +import io.airbyte.commons.json.Jsons; +import io.airbyte.config.*; +import io.airbyte.protocol.models.*; +import io.airbyte.protocol.models.DestinationSyncMode; +import io.airbyte.protocol.models.AirbyteMessage.Type; +import io.airbyte.protocol.models.SyncMode; +import io.airbyte.workers.process.AirbyteIntegrationLauncher; +import io.airbyte.workers.process.DockerProcessFactory; +import io.airbyte.workers.process.ProcessFactory; +import io.airbyte.db.Database; +import io.airbyte.workers.protocols.airbyte.AirbyteSource; +import io.airbyte.workers.protocols.airbyte.DefaultAirbyteSource; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.*; +import java.util.stream.Collectors; + +import static io.airbyte.protocol.models.SyncMode.FULL_REFRESH; +import static org.junit.jupiter.api.Assertions.*; + +public abstract class SourceComprehensiveTest { + + private static final long JOB_ID = 0L; + private static final int JOB_ATTEMPT = 0; + + private static final Logger LOGGER = LoggerFactory.getLogger(SourceComprehensiveTest.class); + private static final String DEFAULT_CREATE_TABLE_SQL = "CREATE TABLE %1$s(test_column %2$s);"; + private static final String DEFAULT_INSERT_SQL = "INSERT INTO %1$s VALUES (%2$s);"; + + private TestDestinationEnv testEnv; + private JsonNode config; + private final List dataTypeTests = new ArrayList<>(); + + private Path jobRoot; + protected Path localRoot; + private ProcessFactory processFactory; + + /** + * Name of the docker image that the tests will run against. + * + * @return docker image name + */ + protected abstract String getImageName(); + + protected abstract JsonNode setupConfig(TestDestinationEnv testEnv) throws Exception; + + protected abstract Database setupDatabase(JsonNode config) throws Exception; + + protected abstract void initTests(); + + /** + * Function that performs any clean up of external resources required for the test. e.g. delete a + * postgres database. This function will be called after EACH test. It MUST remove all data in the + * destination so that there is no contamination across tests. + * + * @param testEnv - information about the test environment. + * @throws Exception - can throw any exception, test framework will handle. + */ + protected abstract void tearDown(TestDestinationEnv testEnv) throws Exception; + + public void addDataTypeTest(DataTypeTest test) { + dataTypeTests.add(test); + test.setTestNumber(dataTypeTests.stream().filter(t -> t.getSourceType().equals(test.getSourceType())).count()); + } + + @BeforeEach + public void setUpInternal() throws Exception { + final Path testDir = Path.of("/tmp/airbyte_tests/"); + Files.createDirectories(testDir); + final Path workspaceRoot = Files.createTempDirectory(testDir, "test"); + jobRoot = Files.createDirectories(Path.of(workspaceRoot.toString(), "job")); + localRoot = Files.createTempDirectory(testDir, "output"); + testEnv = new TestDestinationEnv(localRoot); + + config = setupConfig(testEnv); + setupDatabaseInternal(); + + processFactory = new DockerProcessFactory( + workspaceRoot, + workspaceRoot.toString(), + localRoot.toString(), + "host"); + } + + @AfterEach + public void tearDownInternal() throws Exception { + tearDown(testEnv); + } + + /** + * Configuring all streams in the input catalog to full refresh mode, verifies that a read operation + * produces some RECORD messages. + */ + @Test + public void testFullRefreshRead() throws Exception { + ConfiguredAirbyteCatalog catalog = withFullRefreshSyncModes(getConfiguredCatalog()); + List allMessages = runRead(catalog); + LOGGER.info("Size: " + allMessages.size()); + final List recordMessages = allMessages.stream().filter(m -> m.getType() == Type.RECORD).collect(Collectors.toList()); + + recordMessages.forEach(msg -> LOGGER.info(msg.toString())); + + assertFalse(recordMessages.isEmpty(), "Expected a full refresh sync to produce records"); + + allMessages = runRead(catalog); + LOGGER.info("Size: " + allMessages.size()); + recordMessages.forEach(msg -> LOGGER.info(msg.toString())); + } + + private void setupDatabaseInternal() throws Exception { + Database database = setupDatabase(config); + + initTests(); + + for (DataTypeTest test : dataTypeTests) { + database.query(ctx -> { + ctx.fetch(test.getCreateSQL()); + test.getInsertSQLs().forEach(ctx::fetch); + return null; + }); + } +// database.query(ctx -> { +// ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));"); +// ctx.fetch("INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');"); +// ctx.fetch("CREATE TABLE starships(id INTEGER, name VARCHAR(200));"); +// ctx.fetch("INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');"); +// return null; +// }); + + database.close(); + } + + private ConfiguredAirbyteCatalog getConfiguredCatalog() { + return new ConfiguredAirbyteCatalog().withStreams( + dataTypeTests + .stream() + .map(test -> new ConfiguredAirbyteStream() + .withSyncMode(SyncMode.FULL_REFRESH) + .withDestinationSyncMode(DestinationSyncMode.APPEND) + .withStream(CatalogHelpers.createAirbyteStream( + String.format("%s.%s", config.get("database").asText(), test.getName()), + Field.of("test_column", test.getAirbyteType())) + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH)))) + .collect(Collectors.toList()) + ); + } + + private ConfiguredAirbyteCatalog withFullRefreshSyncModes(ConfiguredAirbyteCatalog catalog) { + final ConfiguredAirbyteCatalog clone = Jsons.clone(catalog); + for (ConfiguredAirbyteStream configuredStream : clone.getStreams()) { + if (configuredStream.getStream().getSupportedSyncModes().contains(FULL_REFRESH)) { + configuredStream.setSyncMode(FULL_REFRESH); + configuredStream.setDestinationSyncMode(DestinationSyncMode.OVERWRITE); + } + } + return clone; + } + + private List runRead(ConfiguredAirbyteCatalog configuredCatalog) throws Exception { + final StandardTapConfig sourceConfig = new StandardTapConfig() + .withSourceConnectionConfiguration(config) + .withState(null) + .withCatalog(configuredCatalog); + + final AirbyteSource source = new DefaultAirbyteSource(new AirbyteIntegrationLauncher(JOB_ID, JOB_ATTEMPT, getImageName(), processFactory)); + final List messages = new ArrayList<>(); + source.start(sourceConfig, jobRoot); + while (!source.isFinished()) { + source.attemptRead().ifPresent(messages::add); + } + source.close(); + + return messages; + } + + public static class TestDestinationEnv { + + private final Path localRoot; + + public TestDestinationEnv(Path localRoot) { + this.localRoot = localRoot; + } + + public Path getLocalRoot() { + return localRoot; + } + + } + + public static class DataTypeTest { + + private final String sourceType; + private final Field.JsonSchemaPrimitive airbyteType; + private final List values = new ArrayList<>(); + private String createTablePatternSQL; + private String insertPatternSQL; + private long testNumber; + + public DataTypeTest(String sourceType, Field.JsonSchemaPrimitive airbyteType) { + this.sourceType = sourceType; + this.airbyteType = airbyteType; + setCreateTablePatternSQL(DEFAULT_CREATE_TABLE_SQL); + setInsertPatternSQL(DEFAULT_INSERT_SQL); + } + + public DataTypeTest addInsertValue(String insertValue) { + values.add(insertValue); + return this; + } + + public String getName() { + return "test_" + testNumber + "_" + sourceType; + } + + public String getSourceType() { + return sourceType; + } + + public Field.JsonSchemaPrimitive getAirbyteType() { + return airbyteType; + } + + public List getValues() { + return values; + } + + public DataTypeTest setCreateTablePatternSQL(String createTablePatternSQL) { + this.createTablePatternSQL = createTablePatternSQL; + return this; + } + + public DataTypeTest setInsertPatternSQL(String insertPatternSQL) { + this.insertPatternSQL = insertPatternSQL; + return this; + } + + public void setTestNumber(long testNumber) { + this.testNumber = testNumber; + } + + public String getCreateSQL() { + return String.format(createTablePatternSQL, getName(), getSourceType()); + } + + public List getInsertSQLs() { + return values.stream().map(value -> String.format(insertPatternSQL, getName(), value)).collect(Collectors.toList()); + } + } + +} diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceComprehensiveTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceComprehensiveTest.java new file mode 100644 index 0000000000000..7235c7948ed10 --- /dev/null +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceComprehensiveTest.java @@ -0,0 +1,102 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.source.mysql; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableMap; +import io.airbyte.commons.json.Jsons; +import io.airbyte.db.Database; +import io.airbyte.db.Databases; +import io.airbyte.integrations.standardtest.source.SourceComprehensiveTest; +import io.airbyte.protocol.models.Field.JsonSchemaPrimitive; +import org.jooq.SQLDialect; +import org.testcontainers.containers.MySQLContainer; + +public class MySqlSourceComprehensiveTest extends SourceComprehensiveTest { + + private MySQLContainer container; + private JsonNode config; + + @Override + protected JsonNode setupConfig(TestDestinationEnv testEnv) { + container = new MySQLContainer<>("mysql:8.0"); + container.start(); + + config = Jsons.jsonNode(ImmutableMap.builder() + .put("host", container.getHost()) + .put("port", container.getFirstMappedPort()) + .put("database", container.getDatabaseName()) + .put("username", container.getUsername()) + .put("password", container.getPassword()) + .build()); + + return config; + } + + @Override + protected void tearDown(TestDestinationEnv testEnv) { + container.close(); + } + + @Override + protected String getImageName() { + return "airbyte/source-mysql:dev"; + } + + @Override + protected Database setupDatabase(JsonNode config) { + return Databases.createDatabase( + config.get("username").asText(), + config.get("password").asText(), + String.format("jdbc:mysql://%s:%s/%s", + config.get("host").asText(), + config.get("port").asText(), + config.get("database").asText()), + "com.mysql.cj.jdbc.Driver", + SQLDialect.MYSQL); + } + + @Override + protected void initTests() { + addDataTypeTest( + new DataTypeTest("tinyint", JsonSchemaPrimitive.NUMBER) + .addInsertValue("null") + .addInsertValue("-128") + .addInsertValue("127") + ); + addDataTypeTest( + new DataTypeTest("smallint", JsonSchemaPrimitive.NUMBER) + .addInsertValue("null") + .addInsertValue("-32768") + .addInsertValue("32767") + ); + + addDataTypeTest( + new DataTypeTest("smallint", JsonSchemaPrimitive.NUMBER) + .setCreateTablePatternSQL("CREATE TABLE %1$s(test_column %2$s zerofill);") + .addInsertValue("1") + ); + } +} From 8332273ec60196bead2fa07c270fdfa64fd9a5ac Mon Sep 17 00:00:00 2001 From: Andrii Leonets Date: Sun, 30 May 2021 15:01:07 +0300 Subject: [PATCH 02/11] missing tests --- .../source/SourceComprehensiveTest.java | 24 ++- .../mysql/MySqlSourceComprehensiveTest.java | 171 +++++++++++++++++- 2 files changed, 182 insertions(+), 13 deletions(-) diff --git a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceComprehensiveTest.java b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceComprehensiveTest.java index 966a8f29d2def..73352fef11e6a 100644 --- a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceComprehensiveTest.java +++ b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceComprehensiveTest.java @@ -149,17 +149,12 @@ private void setupDatabaseInternal() throws Exception { for (DataTypeTest test : dataTypeTests) { database.query(ctx -> { ctx.fetch(test.getCreateSQL()); + LOGGER.info("Table " + test.getName() + " is created."); test.getInsertSQLs().forEach(ctx::fetch); + LOGGER.info("Values " + test.getValues() + " are inserted into " + test.getName()); return null; }); } -// database.query(ctx -> { -// ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));"); -// ctx.fetch("INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');"); -// ctx.fetch("CREATE TABLE starships(id INTEGER, name VARCHAR(200));"); -// ctx.fetch("INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');"); -// return null; -// }); database.close(); } @@ -229,12 +224,14 @@ public static class DataTypeTest { private String createTablePatternSQL; private String insertPatternSQL; private long testNumber; + private String fullSourceDataType; public DataTypeTest(String sourceType, Field.JsonSchemaPrimitive airbyteType) { this.sourceType = sourceType; this.airbyteType = airbyteType; setCreateTablePatternSQL(DEFAULT_CREATE_TABLE_SQL); setInsertPatternSQL(DEFAULT_INSERT_SQL); + setFullSourceDataType(sourceType); } public DataTypeTest addInsertValue(String insertValue) { @@ -268,17 +265,26 @@ public DataTypeTest setInsertPatternSQL(String insertPatternSQL) { return this; } - public void setTestNumber(long testNumber) { + public DataTypeTest setFullSourceDataType(String fullSourceDataType) { + this.fullSourceDataType = fullSourceDataType; + return this; + } + + private void setTestNumber(long testNumber) { this.testNumber = testNumber; } public String getCreateSQL() { - return String.format(createTablePatternSQL, getName(), getSourceType()); + return String.format(createTablePatternSQL, getName(), getFullSourceDataType()); } public List getInsertSQLs() { return values.stream().map(value -> String.format(insertPatternSQL, getName(), value)).collect(Collectors.toList()); } + + public String getFullSourceDataType() { + return fullSourceDataType; + } } } diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceComprehensiveTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceComprehensiveTest.java index 7235c7948ed10..7029560db1d60 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceComprehensiveTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceComprehensiveTest.java @@ -66,8 +66,8 @@ protected String getImageName() { } @Override - protected Database setupDatabase(JsonNode config) { - return Databases.createDatabase( + protected Database setupDatabase(JsonNode config) throws Exception { + final Database database = Databases.createDatabase( config.get("username").asText(), config.get("password").asText(), String.format("jdbc:mysql://%s:%s/%s", @@ -76,6 +76,12 @@ protected Database setupDatabase(JsonNode config) { config.get("database").asText()), "com.mysql.cj.jdbc.Driver", SQLDialect.MYSQL); + + // It disable strict mode in the DB and allows to insert specific values. + // For example, it's possible to insert date with zero values "2021-00-00" + database.query(ctx -> ctx.fetch("SET @@sql_mode=''")); + + return database; } @Override @@ -95,8 +101,165 @@ protected void initTests() { addDataTypeTest( new DataTypeTest("smallint", JsonSchemaPrimitive.NUMBER) - .setCreateTablePatternSQL("CREATE TABLE %1$s(test_column %2$s zerofill);") + .setFullSourceDataType("smallint zerofill") + .addInsertValue("1") + ); + + addDataTypeTest( + new DataTypeTest("mediumint", JsonSchemaPrimitive.NUMBER) + .addInsertValue("null") + .addInsertValue("-8388608") + .addInsertValue("8388607") + ); + + addDataTypeTest( + new DataTypeTest("mediumint", JsonSchemaPrimitive.NUMBER) + .setFullSourceDataType("mediumint zerofill") .addInsertValue("1") ); + + addDataTypeTest( + new DataTypeTest("int", JsonSchemaPrimitive.NUMBER) + .addInsertValue("null") + .addInsertValue("-2147483648") + .addInsertValue("2147483647") + ); + + addDataTypeTest( + new DataTypeTest("int", JsonSchemaPrimitive.NUMBER) + .setFullSourceDataType("int zerofill") + .addInsertValue("1") + ); + + addDataTypeTest( + new DataTypeTest("bigint", JsonSchemaPrimitive.NUMBER) + .addInsertValue("null") + .addInsertValue("9223372036854775807") + ); + + addDataTypeTest( + new DataTypeTest("float", JsonSchemaPrimitive.NUMBER) + .addInsertValue("null") + ); + + addDataTypeTest( + new DataTypeTest("double", JsonSchemaPrimitive.NUMBER) + .addInsertValue("null") + .addInsertValue("power(10, 308)") + .addInsertValue("1/power(10, 45)") + ); + + addDataTypeTest( + new DataTypeTest("decimal", JsonSchemaPrimitive.NUMBER) + .setFullSourceDataType("decimal(5,2)") + .addInsertValue("null") + ); + + addDataTypeTest( + new DataTypeTest("bit", JsonSchemaPrimitive.NUMBER) + .addInsertValue("null") + .addInsertValue("1") + .addInsertValue("0") + ); + + addDataTypeTest( + new DataTypeTest("date", JsonSchemaPrimitive.STRING) + .addInsertValue("null") +// .addInsertValue("'2021-01-00'") +// .addInsertValue("'2021-00-00'") +// .addInsertValue("'0000-00-00'") + ); + + addDataTypeTest( + new DataTypeTest("datetime", JsonSchemaPrimitive.STRING) + .addInsertValue("null") +// .addInsertValue("'0000-00-00 00:00:00'") + ); + + addDataTypeTest( + new DataTypeTest("timestamp", JsonSchemaPrimitive.STRING) + .addInsertValue("null") + ); + + addDataTypeTest( + new DataTypeTest("time", JsonSchemaPrimitive.STRING) + .addInsertValue("null") +// .addInsertValue("'-838:59:59.000000'") + ); + + addDataTypeTest( + new DataTypeTest("varchar", JsonSchemaPrimitive.STRING) + .setFullSourceDataType("varchar(256) character set cp1251") + .addInsertValue("null") + .addInsertValue("'тест'") + ); + + addDataTypeTest( + new DataTypeTest("varchar", JsonSchemaPrimitive.STRING) + .setFullSourceDataType("varchar(256) character set utf16") + .addInsertValue("null") + .addInsertValue("0xfffd") + ); + + addDataTypeTest( + new DataTypeTest("varchar", JsonSchemaPrimitive.STRING) + .setFullSourceDataType("varchar(256)") + .addInsertValue("null") + .addInsertValue("'!\"#$%&\\'()*+,-./:;<=>?\\@[\\]^_\\`{|}~'") + ); + + addDataTypeTest( + new DataTypeTest("varbinary", JsonSchemaPrimitive.STRING) + .setFullSourceDataType("varbinary(256)") + .addInsertValue("null") + .addInsertValue("'test'") + ); + + addDataTypeTest( + new DataTypeTest("blob", JsonSchemaPrimitive.STRING) + .addInsertValue("null") + .addInsertValue("'test'") + ); + + addDataTypeTest( + new DataTypeTest("mediumtext", JsonSchemaPrimitive.STRING) + .addInsertValue("null") + .addInsertValue("lpad('0', 16777214, '0')") + ); + + addDataTypeTest( + new DataTypeTest("tinytext", JsonSchemaPrimitive.STRING) + .addInsertValue("null") + ); + + addDataTypeTest( + new DataTypeTest("longtext", JsonSchemaPrimitive.STRING) + .addInsertValue("null") + ); + + addDataTypeTest( + new DataTypeTest("text", JsonSchemaPrimitive.STRING) + .addInsertValue("null") + ); + + addDataTypeTest( + new DataTypeTest("json", JsonSchemaPrimitive.STRING) + .addInsertValue("null") + .addInsertValue("'{\"a\" :10, \"b\": 15}'") + ); + + addDataTypeTest( + new DataTypeTest("point", JsonSchemaPrimitive.OBJECT) + .addInsertValue("null") + .addInsertValue("(ST_GeomFromText('POINT(1 1)'))") + ); + + addDataTypeTest( + new DataTypeTest("bool", JsonSchemaPrimitive.STRING) + .addInsertValue("null") + .addInsertValue("127") + .addInsertValue("-128") + ); + } -} +} \ No newline at end of file From 3675641c590508951be61d94a33fb318b1aa4f72 Mon Sep 17 00:00:00 2001 From: Andrii Leonets Date: Mon, 31 May 2021 23:38:57 +0300 Subject: [PATCH 03/11] + migrate common test part from SourceAcceptanceTest to a new abstract class SourceTest + add test comprehensive for MySQL CDC + add javadocs --- .../source/SourceAcceptanceTest.java | 139 +-------- .../source/SourceComprehensiveTest.java | 293 ++++++++---------- .../standardtest/source/SourceTest.java | 143 +++++++++ .../CdcMySqlSourceComprehensiveTest.java | 277 +++++++++++++++++ .../mysql/MySqlSourceComprehensiveTest.java | 164 +++++----- 5 files changed, 637 insertions(+), 379 deletions(-) create mode 100644 airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceTest.java create mode 100644 airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceComprehensiveTest.java diff --git a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceAcceptanceTest.java b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceAcceptanceTest.java index d7418bc6995bc..4cd2b6c6e7cac 100644 --- a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceAcceptanceTest.java +++ b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceAcceptanceTest.java @@ -35,14 +35,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.Sets; import io.airbyte.commons.json.Jsons; -import io.airbyte.config.JobGetSpecConfig; -import io.airbyte.config.StandardCheckConnectionInput; -import io.airbyte.config.StandardCheckConnectionOutput; import io.airbyte.config.StandardCheckConnectionOutput.Status; -import io.airbyte.config.StandardDiscoverCatalogInput; -import io.airbyte.config.StandardTapConfig; -import io.airbyte.config.State; -import io.airbyte.protocol.models.AirbyteCatalog; import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.AirbyteMessage.Type; import io.airbyte.protocol.models.AirbyteRecordMessage; @@ -51,30 +44,16 @@ import io.airbyte.protocol.models.ConfiguredAirbyteStream; import io.airbyte.protocol.models.ConnectorSpecification; import io.airbyte.protocol.models.DestinationSyncMode; -import io.airbyte.workers.DefaultCheckConnectionWorker; -import io.airbyte.workers.DefaultDiscoverCatalogWorker; -import io.airbyte.workers.DefaultGetSpecWorker; -import io.airbyte.workers.WorkerException; -import io.airbyte.workers.process.AirbyteIntegrationLauncher; -import io.airbyte.workers.process.DockerProcessFactory; -import io.airbyte.workers.process.ProcessFactory; -import io.airbyte.workers.protocols.airbyte.AirbyteSource; -import io.airbyte.workers.protocols.airbyte.DefaultAirbyteSource; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Set; import java.util.stream.Collectors; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public abstract class SourceAcceptanceTest { +public abstract class SourceAcceptanceTest extends SourceTest{ public static final String CDC_LSN = "_ab_cdc_lsn"; public static final String CDC_UPDATED_AT = "_ab_cdc_updated_at"; @@ -82,17 +61,8 @@ public abstract class SourceAcceptanceTest { public static final String CDC_LOG_FILE = "_ab_cdc_log_file"; public static final String CDC_LOG_POS = "_ab_cdc_log_pos"; - private static final long JOB_ID = 0L; - private static final int JOB_ATTEMPT = 0; - private static final Logger LOGGER = LoggerFactory.getLogger(SourceAcceptanceTest.class); - private TestDestinationEnv testEnv; - - private Path jobRoot; - protected Path localRoot; - private ProcessFactory processFactory; - /** * TODO hack: Various Singer integrations use cursor fields inclusively i.e: they output records * whose cursor field >= the provided cursor value. This leads to the last record in a sync to @@ -129,13 +99,6 @@ public abstract class SourceAcceptanceTest { private Set IMAGES_TO_SKIP_IDENTICAL_FULL_REFRESHES = Sets.newHashSet( "airbyte/source-google-workspace-admin-reports"); - /** - * Name of the docker image that the tests will run against. - * - * @return docker image name - */ - protected abstract String getImageName(); - /** * Specification for integration. Will be passed to integration where appropriate in each test. * Should be valid. @@ -144,14 +107,6 @@ public abstract class SourceAcceptanceTest { */ protected abstract ConnectorSpecification getSpec() throws Exception; - /** - * Configuration specific to the integration. Will be passed to integration where appropriate in - * each test. Should be valid. - * - * @return integration-specific configuration - */ - protected abstract JsonNode getConfig() throws Exception; - /** * The catalog to use to validate the output of read operations. This will be used as follows: *

@@ -178,48 +133,6 @@ public abstract class SourceAcceptanceTest { */ protected abstract List getRegexTests() throws Exception; - /** - * Function that performs any setup of external resources required for the test. e.g. instantiate a - * postgres database. This function will be called before EACH test. - * - * @param testEnv - information about the test environment. - * @throws Exception - can throw any exception, test framework will handle. - */ - protected abstract void setup(TestDestinationEnv testEnv) throws Exception; - - /** - * Function that performs any clean up of external resources required for the test. e.g. delete a - * postgres database. This function will be called after EACH test. It MUST remove all data in the - * destination so that there is no contamination across tests. - * - * @param testEnv - information about the test environment. - * @throws Exception - can throw any exception, test framework will handle. - */ - protected abstract void tearDown(TestDestinationEnv testEnv) throws Exception; - - @BeforeEach - public void setUpInternal() throws Exception { - final Path testDir = Path.of("/tmp/airbyte_tests/"); - Files.createDirectories(testDir); - final Path workspaceRoot = Files.createTempDirectory(testDir, "test"); - jobRoot = Files.createDirectories(Path.of(workspaceRoot.toString(), "job")); - localRoot = Files.createTempDirectory(testDir, "output"); - testEnv = new TestDestinationEnv(localRoot); - - setup(testEnv); - - processFactory = new DockerProcessFactory( - workspaceRoot, - workspaceRoot.toString(), - localRoot.toString(), - "host"); - } - - @AfterEach - public void tearDownInternal() throws Exception { - tearDown(testEnv); - } - /** * Verify that a spec operation issued to the connector returns a valid spec. */ @@ -419,42 +332,6 @@ private boolean sourceSupportsIncremental() throws Exception { return false; } - private ConnectorSpecification runSpec() throws WorkerException { - return new DefaultGetSpecWorker(new AirbyteIntegrationLauncher(JOB_ID, JOB_ATTEMPT, getImageName(), processFactory)) - .run(new JobGetSpecConfig().withDockerImage(getImageName()), jobRoot); - } - - private StandardCheckConnectionOutput runCheck() throws Exception { - return new DefaultCheckConnectionWorker(new AirbyteIntegrationLauncher(JOB_ID, JOB_ATTEMPT, getImageName(), processFactory)) - .run(new StandardCheckConnectionInput().withConnectionConfiguration(getConfig()), jobRoot); - } - - private AirbyteCatalog runDiscover() throws Exception { - return new DefaultDiscoverCatalogWorker(new AirbyteIntegrationLauncher(JOB_ID, JOB_ATTEMPT, getImageName(), processFactory)) - .run(new StandardDiscoverCatalogInput().withConnectionConfiguration(getConfig()), jobRoot); - } - - private List runRead(ConfiguredAirbyteCatalog configuredCatalog) throws Exception { - return runRead(configuredCatalog, null); - } - - // todo (cgardens) - assume no state since we are all full refresh right now. - private List runRead(ConfiguredAirbyteCatalog catalog, JsonNode state) throws Exception { - final StandardTapConfig sourceConfig = new StandardTapConfig() - .withSourceConnectionConfiguration(getConfig()) - .withState(state == null ? null : new State().withState(state)) - .withCatalog(catalog); - - final AirbyteSource source = new DefaultAirbyteSource(new AirbyteIntegrationLauncher(JOB_ID, JOB_ATTEMPT, getImageName(), processFactory)); - final List messages = new ArrayList<>(); - source.start(sourceConfig, jobRoot); - while (!source.isFinished()) { - source.attemptRead().ifPresent(messages::add); - } - source.close(); - - return messages; - } private void assertSameRecords(List expected, List actual, String message) { final List prunedExpected = expected.stream().map(this::pruneEmittedAt).collect(Collectors.toList()); @@ -482,18 +359,4 @@ private AirbyteRecordMessage pruneCdcMetadata(AirbyteRecordMessage m) { return clone; } - public static class TestDestinationEnv { - - private final Path localRoot; - - public TestDestinationEnv(Path localRoot) { - this.localRoot = localRoot; - } - - public Path getLocalRoot() { - return localRoot; - } - - } - } diff --git a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceComprehensiveTest.java b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceComprehensiveTest.java index 73352fef11e6a..ddc08fb6eed37 100644 --- a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceComprehensiveTest.java +++ b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceComprehensiveTest.java @@ -26,99 +26,43 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.Lists; -import io.airbyte.commons.json.Jsons; -import io.airbyte.config.*; import io.airbyte.protocol.models.*; import io.airbyte.protocol.models.DestinationSyncMode; import io.airbyte.protocol.models.AirbyteMessage.Type; import io.airbyte.protocol.models.SyncMode; -import io.airbyte.workers.process.AirbyteIntegrationLauncher; -import io.airbyte.workers.process.DockerProcessFactory; -import io.airbyte.workers.process.ProcessFactory; import io.airbyte.db.Database; -import io.airbyte.workers.protocols.airbyte.AirbyteSource; -import io.airbyte.workers.protocols.airbyte.DefaultAirbyteSource; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.nio.file.Files; -import java.nio.file.Path; import java.util.*; import java.util.stream.Collectors; -import static io.airbyte.protocol.models.SyncMode.FULL_REFRESH; import static org.junit.jupiter.api.Assertions.*; -public abstract class SourceComprehensiveTest { - - private static final long JOB_ID = 0L; - private static final int JOB_ATTEMPT = 0; +public abstract class SourceComprehensiveTest extends SourceTest { private static final Logger LOGGER = LoggerFactory.getLogger(SourceComprehensiveTest.class); - private static final String DEFAULT_CREATE_TABLE_SQL = "CREATE TABLE %1$s(test_column %2$s);"; - private static final String DEFAULT_INSERT_SQL = "INSERT INTO %1$s VALUES (%2$s);"; + private static final String DEFAULT_CREATE_TABLE_SQL = "CREATE TABLE %1$s(id integer primary key, test_column %2$s);"; + private static final String DEFAULT_INSERT_SQL = "INSERT INTO %1$s VALUES (%2$s, %3$s);"; - private TestDestinationEnv testEnv; - private JsonNode config; private final List dataTypeTests = new ArrayList<>(); - private Path jobRoot; - protected Path localRoot; - private ProcessFactory processFactory; - /** - * Name of the docker image that the tests will run against. - * - * @return docker image name + * Setup the test database. All tables and data described in the registered tests will be put there. + * @return configured test database + * @throws Exception - might throw any exception during initialization. */ - protected abstract String getImageName(); - - protected abstract JsonNode setupConfig(TestDestinationEnv testEnv) throws Exception; - - protected abstract Database setupDatabase(JsonNode config) throws Exception; - - protected abstract void initTests(); + protected abstract Database setupDatabase() throws Exception; /** - * Function that performs any clean up of external resources required for the test. e.g. delete a - * postgres database. This function will be called after EACH test. It MUST remove all data in the - * destination so that there is no contamination across tests. - * - * @param testEnv - information about the test environment. - * @throws Exception - can throw any exception, test framework will handle. + * Put all required tests here using method {@link #addDataTypeTest(DataTypeTest)} */ - protected abstract void tearDown(TestDestinationEnv testEnv) throws Exception; - - public void addDataTypeTest(DataTypeTest test) { - dataTypeTests.add(test); - test.setTestNumber(dataTypeTests.stream().filter(t -> t.getSourceType().equals(test.getSourceType())).count()); - } - - @BeforeEach - public void setUpInternal() throws Exception { - final Path testDir = Path.of("/tmp/airbyte_tests/"); - Files.createDirectories(testDir); - final Path workspaceRoot = Files.createTempDirectory(testDir, "test"); - jobRoot = Files.createDirectories(Path.of(workspaceRoot.toString(), "job")); - localRoot = Files.createTempDirectory(testDir, "output"); - testEnv = new TestDestinationEnv(localRoot); + protected abstract void initTests(); - config = setupConfig(testEnv); + @Override + protected void setup(TestDestinationEnv testEnv) throws Exception { setupDatabaseInternal(); - - processFactory = new DockerProcessFactory( - workspaceRoot, - workspaceRoot.toString(), - localRoot.toString(), - "host"); - } - - @AfterEach - public void tearDownInternal() throws Exception { - tearDown(testEnv); } /** @@ -126,8 +70,8 @@ public void tearDownInternal() throws Exception { * produces some RECORD messages. */ @Test - public void testFullRefreshRead() throws Exception { - ConfiguredAirbyteCatalog catalog = withFullRefreshSyncModes(getConfiguredCatalog()); + public void testDataTypes() throws Exception { + ConfiguredAirbyteCatalog catalog = getConfiguredCatalog(); List allMessages = runRead(catalog); LOGGER.info("Size: " + allMessages.size()); final List recordMessages = allMessages.stream().filter(m -> m.getType() == Type.RECORD).collect(Collectors.toList()); @@ -141,8 +85,12 @@ public void testFullRefreshRead() throws Exception { recordMessages.forEach(msg -> LOGGER.info(msg.toString())); } + /** + * Creates all tables and insert data described in the registered data type tests. + * @throws Exception might raise exception if configuration goes wrong or tables creation/insert scripts failed. + */ private void setupDatabaseInternal() throws Exception { - Database database = setupDatabase(config); + Database database = setupDatabase(); initTests(); @@ -151,7 +99,7 @@ private void setupDatabaseInternal() throws Exception { ctx.fetch(test.getCreateSQL()); LOGGER.info("Table " + test.getName() + " is created."); test.getInsertSQLs().forEach(ctx::fetch); - LOGGER.info("Values " + test.getValues() + " are inserted into " + test.getName()); + LOGGER.info("Values " + test.values + " are inserted into " + test.getName()); return null; }); } @@ -159,131 +107,150 @@ private void setupDatabaseInternal() throws Exception { database.close(); } - private ConfiguredAirbyteCatalog getConfiguredCatalog() { + /** + * Configures streams for all registered data type tests. + * @return configured catalog + */ + private ConfiguredAirbyteCatalog getConfiguredCatalog() throws Exception { + final JsonNode config = getConfig(); + return new ConfiguredAirbyteCatalog().withStreams( dataTypeTests .stream() - .map(test -> new ConfiguredAirbyteStream() - .withSyncMode(SyncMode.FULL_REFRESH) - .withDestinationSyncMode(DestinationSyncMode.APPEND) - .withStream(CatalogHelpers.createAirbyteStream( - String.format("%s.%s", config.get("database").asText(), test.getName()), - Field.of("test_column", test.getAirbyteType())) - .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH)))) + .map(test -> + new ConfiguredAirbyteStream() + .withSyncMode(SyncMode.INCREMENTAL) + .withCursorField(Lists.newArrayList("id")) + .withDestinationSyncMode(DestinationSyncMode.APPEND) + .withStream(CatalogHelpers.createAirbyteStream( + String.format("%s", test.getName()), + String.format("%s", config.get("database").asText()), + Field.of("id", Field.JsonSchemaPrimitive.NUMBER), + Field.of("test_column", test.airbyteType)) + .withSourceDefinedCursor(true) + .withSourceDefinedPrimaryKey(List.of(List.of("id"))) + .withSupportedSyncModes( + Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)))) .collect(Collectors.toList()) ); } - private ConfiguredAirbyteCatalog withFullRefreshSyncModes(ConfiguredAirbyteCatalog catalog) { - final ConfiguredAirbyteCatalog clone = Jsons.clone(catalog); - for (ConfiguredAirbyteStream configuredStream : clone.getStreams()) { - if (configuredStream.getStream().getSupportedSyncModes().contains(FULL_REFRESH)) { - configuredStream.setSyncMode(FULL_REFRESH); - configuredStream.setDestinationSyncMode(DestinationSyncMode.OVERWRITE); - } - } - return clone; - } - - private List runRead(ConfiguredAirbyteCatalog configuredCatalog) throws Exception { - final StandardTapConfig sourceConfig = new StandardTapConfig() - .withSourceConnectionConfiguration(config) - .withState(null) - .withCatalog(configuredCatalog); - - final AirbyteSource source = new DefaultAirbyteSource(new AirbyteIntegrationLauncher(JOB_ID, JOB_ATTEMPT, getImageName(), processFactory)); - final List messages = new ArrayList<>(); - source.start(sourceConfig, jobRoot); - while (!source.isFinished()) { - source.attemptRead().ifPresent(messages::add); - } - source.close(); - - return messages; + /** + * Register your test in the run scope. + * For each test will be created a table with one column of specified type. + * Note! If you register more than one test with the same type name, they will be run as independent tests with own + * streams. + * @param test comprehensive data type test built by {@link DataTypeTestBuilder} + */ + public void addDataTypeTest(DataTypeTest test) { + dataTypeTests.add(test); + test.testNumber = dataTypeTests.stream().filter(t -> t.sourceType.equals(test.sourceType)).count(); } - public static class TestDestinationEnv { - - private final Path localRoot; - - public TestDestinationEnv(Path localRoot) { - this.localRoot = localRoot; - } - - public Path getLocalRoot() { - return localRoot; - } - + /** + * The builder allows to setup any comprehensive data type test. + * @param sourceType name of the source data type. Duplicates by name will be tested independently from each others. + * Note that this name will be used for connector setup and table creation. + * If source syntax requires more details (E.g. "varchar" type requires length "varchar(50)"), + * you can additionally set custom data type syntax by {@link DataTypeTestBuilder#setFullSourceDataType(String)} method. + * @param airbyteType corresponding Airbyte data type. It requires for proper configuration {@link ConfiguredAirbyteStream} + * @return builder for setup comprehensive test + */ + public DataTypeTestBuilder dataTypeTestBuilder(String sourceType, Field.JsonSchemaPrimitive airbyteType) { + return new DataTypeTestBuilder(sourceType, airbyteType); } - public static class DataTypeTest { + public class DataTypeTestBuilder { - private final String sourceType; - private final Field.JsonSchemaPrimitive airbyteType; - private final List values = new ArrayList<>(); - private String createTablePatternSQL; - private String insertPatternSQL; - private long testNumber; - private String fullSourceDataType; + private final DataTypeTest dataTypeTest; - public DataTypeTest(String sourceType, Field.JsonSchemaPrimitive airbyteType) { - this.sourceType = sourceType; - this.airbyteType = airbyteType; - setCreateTablePatternSQL(DEFAULT_CREATE_TABLE_SQL); - setInsertPatternSQL(DEFAULT_INSERT_SQL); - setFullSourceDataType(sourceType); + private DataTypeTestBuilder(String sourceType, Field.JsonSchemaPrimitive airbyteType) { + this.dataTypeTest = new DataTypeTest(); + this.dataTypeTest.sourceType = sourceType; + this.dataTypeTest.airbyteType = airbyteType; + this.dataTypeTest.createTablePatternSQL = DEFAULT_CREATE_TABLE_SQL; + this.dataTypeTest.insertPatternSQL = DEFAULT_INSERT_SQL; + this.dataTypeTest.fullSourceDataType = sourceType; } - public DataTypeTest addInsertValue(String insertValue) { - values.add(insertValue); + /** + * Set custom the create table script pattern. Use it if you source uses untypical table creation sql. + * Default patter described {@link #DEFAULT_CREATE_TABLE_SQL} + * Note! The patter should contains two String place holders for the table name and data type. + * @param createTablePatternSQL creation table sql pattern + * @return builder + */ + public DataTypeTestBuilder setCreateTablePatternSQL(String createTablePatternSQL) { + this.dataTypeTest.createTablePatternSQL = createTablePatternSQL; return this; } - public String getName() { - return "test_" + testNumber + "_" + sourceType; - } - - public String getSourceType() { - return sourceType; - } - - public Field.JsonSchemaPrimitive getAirbyteType() { - return airbyteType; - } - - public List getValues() { - return values; - } - - public DataTypeTest setCreateTablePatternSQL(String createTablePatternSQL) { - this.createTablePatternSQL = createTablePatternSQL; + /** + * Set custom the insert record script pattern. Use it if you source uses untypical insert record sql. + * Default patter described {@link #DEFAULT_INSERT_SQL} + * Note! The patter should contains two String place holders for the table name and value. + * @param insertPatternSQL creation table sql pattern + * @return builder + */ + public DataTypeTestBuilder setInsertPatternSQL(String insertPatternSQL) { + this.dataTypeTest.insertPatternSQL = insertPatternSQL; return this; } - public DataTypeTest setInsertPatternSQL(String insertPatternSQL) { - this.insertPatternSQL = insertPatternSQL; + /** + * Allows to set extended data type for the table creation. + * E.g. The "varchar" type requires in MySQL requires length. In this case fullSourceDataType will be "varchar(50)". + * @param fullSourceDataType actual string for the column data type description + * @return builder + */ + public DataTypeTestBuilder setFullSourceDataType(String fullSourceDataType) { + this.dataTypeTest.fullSourceDataType = fullSourceDataType; return this; } - public DataTypeTest setFullSourceDataType(String fullSourceDataType) { - this.fullSourceDataType = fullSourceDataType; + /** + * Adds value(s) to the scope of a corresponding test. The values will be inserted into the created table. + * Note! The value will be inserted into the insert script without any transformations. Make sure that the value is + * in line with the source syntax. + * @param insertValue test value + * @return builder + */ + public DataTypeTestBuilder addInsertValue(String...insertValue) { + this.dataTypeTest.values.addAll(Arrays.asList(insertValue)); return this; } - private void setTestNumber(long testNumber) { - this.testNumber = testNumber; + public DataTypeTest build() { + return dataTypeTest; } - public String getCreateSQL() { - return String.format(createTablePatternSQL, getName(), getFullSourceDataType()); + } + + private class DataTypeTest { + + private String sourceType; + private Field.JsonSchemaPrimitive airbyteType; + private final List values = new ArrayList<>(); + private String createTablePatternSQL; + private String insertPatternSQL; + private long testNumber; + private String fullSourceDataType; + + private String getName() { + return "test_" + testNumber + "_" + sourceType; } - public List getInsertSQLs() { - return values.stream().map(value -> String.format(insertPatternSQL, getName(), value)).collect(Collectors.toList()); + private String getCreateSQL() { + return String.format(createTablePatternSQL, getName(), fullSourceDataType); } - public String getFullSourceDataType() { - return fullSourceDataType; + private List getInsertSQLs() { + List insertSQLs = new ArrayList<>(); + int rowId = 1; + for (String value : values) { + insertSQLs.add(String.format(insertPatternSQL, getName(), rowId++, value)); + } + return insertSQLs; } } diff --git a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceTest.java b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceTest.java new file mode 100644 index 0000000000000..e17cfcb384973 --- /dev/null +++ b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceTest.java @@ -0,0 +1,143 @@ +package io.airbyte.integrations.standardtest.source; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.config.*; +import io.airbyte.protocol.models.AirbyteCatalog; +import io.airbyte.protocol.models.AirbyteMessage; +import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.ConnectorSpecification; +import io.airbyte.workers.DefaultCheckConnectionWorker; +import io.airbyte.workers.DefaultDiscoverCatalogWorker; +import io.airbyte.workers.DefaultGetSpecWorker; +import io.airbyte.workers.WorkerException; +import io.airbyte.workers.process.AirbyteIntegrationLauncher; +import io.airbyte.workers.process.DockerProcessFactory; +import io.airbyte.workers.process.ProcessFactory; +import io.airbyte.workers.protocols.airbyte.AirbyteSource; +import io.airbyte.workers.protocols.airbyte.DefaultAirbyteSource; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +public abstract class SourceTest { + + private TestDestinationEnv testEnv; + private Path jobRoot; + protected Path localRoot; + private ProcessFactory processFactory; + + private static final long JOB_ID = 0L; + private static final int JOB_ATTEMPT = 0; + + /** + * Name of the docker image that the tests will run against. + * + * @return docker image name + */ + protected abstract String getImageName(); + + /** + * Configuration specific to the integration. Will be passed to integration where appropriate in + * each test. Should be valid. + * + * @return integration-specific configuration + */ + protected abstract JsonNode getConfig() throws Exception; + + /** + * Function that performs any setup of external resources required for the test. e.g. instantiate a + * postgres database. This function will be called before EACH test. + * + * @param testEnv - information about the test environment. + * @throws Exception - can throw any exception, test framework will handle. + */ + protected abstract void setup(TestDestinationEnv testEnv) throws Exception; + + /** + * Function that performs any clean up of external resources required for the test. e.g. delete a + * postgres database. This function will be called after EACH test. It MUST remove all data in the + * destination so that there is no contamination across tests. + * + * @param testEnv - information about the test environment. + * @throws Exception - can throw any exception, test framework will handle. + */ + protected abstract void tearDown(TestDestinationEnv testEnv) throws Exception; + + @BeforeEach + public void setUpInternal() throws Exception { + final Path testDir = Path.of("/tmp/airbyte_tests/"); + Files.createDirectories(testDir); + final Path workspaceRoot = Files.createTempDirectory(testDir, "test"); + jobRoot = Files.createDirectories(Path.of(workspaceRoot.toString(), "job")); + localRoot = Files.createTempDirectory(testDir, "output"); + testEnv = new TestDestinationEnv(localRoot); + + setup(testEnv); + + processFactory = new DockerProcessFactory( + workspaceRoot, + workspaceRoot.toString(), + localRoot.toString(), + "host"); + } + + @AfterEach + public void tearDownInternal() throws Exception { + tearDown(testEnv); + } + + protected ConnectorSpecification runSpec() throws WorkerException { + return new DefaultGetSpecWorker(new AirbyteIntegrationLauncher(JOB_ID, JOB_ATTEMPT, getImageName(), processFactory)) + .run(new JobGetSpecConfig().withDockerImage(getImageName()), jobRoot); + } + + protected StandardCheckConnectionOutput runCheck() throws Exception { + return new DefaultCheckConnectionWorker(new AirbyteIntegrationLauncher(JOB_ID, JOB_ATTEMPT, getImageName(), processFactory)) + .run(new StandardCheckConnectionInput().withConnectionConfiguration(getConfig()), jobRoot); + } + + protected AirbyteCatalog runDiscover() throws Exception { + return new DefaultDiscoverCatalogWorker(new AirbyteIntegrationLauncher(JOB_ID, JOB_ATTEMPT, getImageName(), processFactory)) + .run(new StandardDiscoverCatalogInput().withConnectionConfiguration(getConfig()), jobRoot); + } + + protected List runRead(ConfiguredAirbyteCatalog configuredCatalog) throws Exception { + return runRead(configuredCatalog, null); + } + + // todo (cgardens) - assume no state since we are all full refresh right now. + protected List runRead(ConfiguredAirbyteCatalog catalog, JsonNode state) throws Exception { + final StandardTapConfig sourceConfig = new StandardTapConfig() + .withSourceConnectionConfiguration(getConfig()) + .withState(state == null ? null : new State().withState(state)) + .withCatalog(catalog); + + final AirbyteSource source = new DefaultAirbyteSource(new AirbyteIntegrationLauncher(JOB_ID, JOB_ATTEMPT, getImageName(), processFactory)); + final List messages = new ArrayList<>(); + source.start(sourceConfig, jobRoot); + while (!source.isFinished()) { + source.attemptRead().ifPresent(messages::add); + } + source.close(); + + return messages; + } + + public static class TestDestinationEnv { + + private final Path localRoot; + + public TestDestinationEnv(Path localRoot) { + this.localRoot = localRoot; + } + + public Path getLocalRoot() { + return localRoot; + } + + } +} diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceComprehensiveTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceComprehensiveTest.java new file mode 100644 index 0000000000000..132edfa50b492 --- /dev/null +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceComprehensiveTest.java @@ -0,0 +1,277 @@ +package io.airbyte.integrations.source.mysql; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableMap; +import io.airbyte.commons.json.Jsons; +import io.airbyte.db.Database; +import io.airbyte.db.Databases; +import io.airbyte.integrations.standardtest.source.SourceComprehensiveTest; +import io.airbyte.integrations.standardtest.source.SourceTest; +import io.airbyte.protocol.models.Field; +import org.jooq.SQLDialect; +import org.testcontainers.containers.MySQLContainer; + +public class CdcMySqlSourceComprehensiveTest extends SourceComprehensiveTest { + + private MySQLContainer container; + private JsonNode config; + + @Override + protected JsonNode getConfig() { + return config; + } + + @Override + protected void tearDown(SourceTest.TestDestinationEnv testEnv) { + container.close(); + } + + @Override + protected String getImageName() { + return "airbyte/source-mysql:dev"; + } + + @Override + protected Database setupDatabase() throws Exception { + container = new MySQLContainer<>("mysql:8.0"); + container.start(); + + config = Jsons.jsonNode(ImmutableMap.builder() + .put("host", container.getHost()) + .put("port", container.getFirstMappedPort()) + .put("database", container.getDatabaseName()) + .put("username", container.getUsername()) + .put("password", container.getPassword()) + .put("replication_method", MySqlSource.ReplicationMethod.CDC) + .build()); + + final Database database = Databases.createDatabase( + config.get("username").asText(), + config.get("password").asText(), + String.format("jdbc:mysql://%s:%s/%s", + config.get("host").asText(), + config.get("port").asText(), + config.get("database").asText()), + "com.mysql.cj.jdbc.Driver", + SQLDialect.MYSQL); + + // It disable strict mode in the DB and allows to insert specific values. + // For example, it's possible to insert date with zero values "2021-00-00" + database.query(ctx -> ctx.fetch("SET @@sql_mode=''")); + + revokeAllPermissions(); + grantCorrectPermissions(); + + return database; + } + + private void revokeAllPermissions() { + executeQuery("REVOKE ALL PRIVILEGES, GRANT OPTION FROM " + container.getUsername() + "@'%';"); + } + + private void grantCorrectPermissions() { + executeQuery( + "GRANT SELECT, RELOAD, SHOW DATABASES, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO " + + container.getUsername() + "@'%';"); + } + + private void executeQuery(String query) { + try (Database database = Databases.createDatabase( + "root", + "test", + String.format("jdbc:mysql://%s:%s/%s", + container.getHost(), + container.getFirstMappedPort(), + container.getDatabaseName()), + MySqlSource.DRIVER_CLASS, + SQLDialect.MYSQL)) { + database.query( + ctx -> ctx + .execute(query)); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + protected void initTests() { + addDataTypeTest( + dataTypeTestBuilder("tinyint", Field.JsonSchemaPrimitive.NUMBER) + .addInsertValue("null") + .addInsertValue("-128", "127") + .build() + ); + + addDataTypeTest( + dataTypeTestBuilder("smallint", Field.JsonSchemaPrimitive.NUMBER) + .addInsertValue("null") + .addInsertValue("-32768", "32767") + .build() + ); + + addDataTypeTest( + dataTypeTestBuilder("smallint", Field.JsonSchemaPrimitive.NUMBER) + .setFullSourceDataType("smallint zerofill") + .addInsertValue("1") + .build() + ); + + addDataTypeTest( + dataTypeTestBuilder("mediumint", Field.JsonSchemaPrimitive.NUMBER) + .addInsertValue("null", "-8388608", "8388607") + .build() + ); + + addDataTypeTest( + dataTypeTestBuilder("mediumint", Field.JsonSchemaPrimitive.NUMBER) + .setFullSourceDataType("mediumint zerofill") + .addInsertValue("1") + .build() + ); + + addDataTypeTest( + dataTypeTestBuilder("int", Field.JsonSchemaPrimitive.NUMBER) + .addInsertValue("null", "-2147483648", "2147483647") + .build() + ); + + addDataTypeTest( + dataTypeTestBuilder("int", Field.JsonSchemaPrimitive.NUMBER) + .setFullSourceDataType("int zerofill") + .addInsertValue("1") + .build() + ); + + addDataTypeTest( + dataTypeTestBuilder("bigint", Field.JsonSchemaPrimitive.NUMBER) + .addInsertValue("null", "9223372036854775807") + .build() + ); + + addDataTypeTest( + dataTypeTestBuilder("float", Field.JsonSchemaPrimitive.NUMBER) + .addInsertValue("null") + .build() + ); + + addDataTypeTest( + dataTypeTestBuilder("double", Field.JsonSchemaPrimitive.NUMBER) + .addInsertValue("null", "power(10, 308)", "1/power(10, 45)") + .build() + ); + + addDataTypeTest( + dataTypeTestBuilder("decimal", Field.JsonSchemaPrimitive.NUMBER) + .setFullSourceDataType("decimal(5,2)") + .addInsertValue("null") + .build() + ); + + addDataTypeTest( + dataTypeTestBuilder("bit", Field.JsonSchemaPrimitive.NUMBER) + .addInsertValue("null", "1", "0") + .build() + ); + + addDataTypeTest( + dataTypeTestBuilder("date", Field.JsonSchemaPrimitive.STRING) + .addInsertValue("null", "'2021-01-00'", "'2021-00-00'", "'0000-00-00'") + .build() + ); + + addDataTypeTest( + dataTypeTestBuilder("datetime", Field.JsonSchemaPrimitive.STRING) + .addInsertValue("null", "'0000-00-00 00:00:00'") + .build() + ); + + addDataTypeTest( + dataTypeTestBuilder("timestamp", Field.JsonSchemaPrimitive.STRING) + .addInsertValue("null") + .build() + ); + + addDataTypeTest( + dataTypeTestBuilder("time", Field.JsonSchemaPrimitive.STRING) + .addInsertValue("null", "'-838:59:59.000000'") + .build() + ); + + addDataTypeTest( + dataTypeTestBuilder("varchar", Field.JsonSchemaPrimitive.STRING) + .setFullSourceDataType("varchar(256) character set cp1251") + .addInsertValue("null", "'тест'") + .build() + ); + + addDataTypeTest( + dataTypeTestBuilder("varchar", Field.JsonSchemaPrimitive.STRING) + .setFullSourceDataType("varchar(256) character set utf16") + .addInsertValue("null", "0xfffd") + .build() + ); + + addDataTypeTest( + dataTypeTestBuilder("varchar", Field.JsonSchemaPrimitive.STRING) + .setFullSourceDataType("varchar(256)") + .addInsertValue("null", "'!\"#$%&\\'()*+,-./:;<=>?\\@[\\]^_\\`{|}~'") + .build() + ); + + addDataTypeTest( + dataTypeTestBuilder("varbinary", Field.JsonSchemaPrimitive.STRING) + .setFullSourceDataType("varbinary(256)") + .addInsertValue("null", "'test'") + .build() + ); + + addDataTypeTest( + dataTypeTestBuilder("blob", Field.JsonSchemaPrimitive.STRING) + .addInsertValue("null", "'test'") + .build() + ); + + addDataTypeTest( + dataTypeTestBuilder("mediumtext", Field.JsonSchemaPrimitive.STRING) + .addInsertValue("null", "lpad('0', 16777214, '0')") + .build() + ); + + addDataTypeTest( + dataTypeTestBuilder("tinytext", Field.JsonSchemaPrimitive.STRING) + .addInsertValue("null") + .build() + ); + + addDataTypeTest( + dataTypeTestBuilder("longtext", Field.JsonSchemaPrimitive.STRING) + .addInsertValue("null") + .build() + ); + + addDataTypeTest( + dataTypeTestBuilder("text", Field.JsonSchemaPrimitive.STRING) + .addInsertValue("null") + .build() + ); + + addDataTypeTest( + dataTypeTestBuilder("json", Field.JsonSchemaPrimitive.STRING) + .addInsertValue("null", "'{\"a\" :10, \"b\": 15}'") + .build() + ); + + addDataTypeTest( + dataTypeTestBuilder("point", Field.JsonSchemaPrimitive.OBJECT) + .addInsertValue("null", "(ST_GeomFromText('POINT(1 1)'))") + .build() + ); + + addDataTypeTest( + dataTypeTestBuilder("bool", Field.JsonSchemaPrimitive.STRING) + .addInsertValue("null", "1", "127", "-128") + .build() + ); + + } +} diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceComprehensiveTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceComprehensiveTest.java index 7029560db1d60..e3b3dd1aa02f8 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceComprehensiveTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceComprehensiveTest.java @@ -40,18 +40,7 @@ public class MySqlSourceComprehensiveTest extends SourceComprehensiveTest { private JsonNode config; @Override - protected JsonNode setupConfig(TestDestinationEnv testEnv) { - container = new MySQLContainer<>("mysql:8.0"); - container.start(); - - config = Jsons.jsonNode(ImmutableMap.builder() - .put("host", container.getHost()) - .put("port", container.getFirstMappedPort()) - .put("database", container.getDatabaseName()) - .put("username", container.getUsername()) - .put("password", container.getPassword()) - .build()); - + protected JsonNode getConfig() { return config; } @@ -66,7 +55,18 @@ protected String getImageName() { } @Override - protected Database setupDatabase(JsonNode config) throws Exception { + protected Database setupDatabase() throws Exception { + container = new MySQLContainer<>("mysql:8.0"); + container.start(); + + config = Jsons.jsonNode(ImmutableMap.builder() + .put("host", container.getHost()) + .put("port", container.getFirstMappedPort()) + .put("database", container.getDatabaseName()) + .put("username", container.getUsername()) + .put("password", container.getPassword()) + .build()); + final Database database = Databases.createDatabase( config.get("username").asText(), config.get("password").asText(), @@ -87,178 +87,186 @@ protected Database setupDatabase(JsonNode config) throws Exception { @Override protected void initTests() { addDataTypeTest( - new DataTypeTest("tinyint", JsonSchemaPrimitive.NUMBER) + dataTypeTestBuilder("tinyint", JsonSchemaPrimitive.NUMBER) .addInsertValue("null") - .addInsertValue("-128") - .addInsertValue("127") + .addInsertValue("-128", "127") + .build() ); + addDataTypeTest( - new DataTypeTest("smallint", JsonSchemaPrimitive.NUMBER) + dataTypeTestBuilder("smallint", JsonSchemaPrimitive.NUMBER) .addInsertValue("null") - .addInsertValue("-32768") - .addInsertValue("32767") + .addInsertValue("-32768", "32767") + .build() ); addDataTypeTest( - new DataTypeTest("smallint", JsonSchemaPrimitive.NUMBER) + dataTypeTestBuilder("smallint", JsonSchemaPrimitive.NUMBER) .setFullSourceDataType("smallint zerofill") .addInsertValue("1") + .build() ); addDataTypeTest( - new DataTypeTest("mediumint", JsonSchemaPrimitive.NUMBER) - .addInsertValue("null") - .addInsertValue("-8388608") - .addInsertValue("8388607") + dataTypeTestBuilder("mediumint", JsonSchemaPrimitive.NUMBER) + .addInsertValue("null", "-8388608", "8388607") + .build() ); addDataTypeTest( - new DataTypeTest("mediumint", JsonSchemaPrimitive.NUMBER) + dataTypeTestBuilder("mediumint", JsonSchemaPrimitive.NUMBER) .setFullSourceDataType("mediumint zerofill") .addInsertValue("1") + .build() ); addDataTypeTest( - new DataTypeTest("int", JsonSchemaPrimitive.NUMBER) - .addInsertValue("null") - .addInsertValue("-2147483648") - .addInsertValue("2147483647") + dataTypeTestBuilder("int", JsonSchemaPrimitive.NUMBER) + .addInsertValue("null", "-2147483648", "2147483647") + .build() ); addDataTypeTest( - new DataTypeTest("int", JsonSchemaPrimitive.NUMBER) + dataTypeTestBuilder("int", JsonSchemaPrimitive.NUMBER) .setFullSourceDataType("int zerofill") .addInsertValue("1") + .build() ); addDataTypeTest( - new DataTypeTest("bigint", JsonSchemaPrimitive.NUMBER) - .addInsertValue("null") - .addInsertValue("9223372036854775807") + dataTypeTestBuilder("bigint", JsonSchemaPrimitive.NUMBER) + .addInsertValue("null", "9223372036854775807") + .build() ); addDataTypeTest( - new DataTypeTest("float", JsonSchemaPrimitive.NUMBER) + dataTypeTestBuilder("float", JsonSchemaPrimitive.NUMBER) .addInsertValue("null") + .build() ); addDataTypeTest( - new DataTypeTest("double", JsonSchemaPrimitive.NUMBER) - .addInsertValue("null") - .addInsertValue("power(10, 308)") - .addInsertValue("1/power(10, 45)") + dataTypeTestBuilder("double", JsonSchemaPrimitive.NUMBER) + .addInsertValue("null", "power(10, 308)", "1/power(10, 45)") + .build() ); addDataTypeTest( - new DataTypeTest("decimal", JsonSchemaPrimitive.NUMBER) + dataTypeTestBuilder("decimal", JsonSchemaPrimitive.NUMBER) .setFullSourceDataType("decimal(5,2)") .addInsertValue("null") + .build() ); addDataTypeTest( - new DataTypeTest("bit", JsonSchemaPrimitive.NUMBER) - .addInsertValue("null") - .addInsertValue("1") - .addInsertValue("0") + dataTypeTestBuilder("bit", JsonSchemaPrimitive.NUMBER) + .addInsertValue("null", "1", "0") + .build() ); addDataTypeTest( - new DataTypeTest("date", JsonSchemaPrimitive.STRING) + dataTypeTestBuilder("date", JsonSchemaPrimitive.STRING) .addInsertValue("null") // .addInsertValue("'2021-01-00'") // .addInsertValue("'2021-00-00'") // .addInsertValue("'0000-00-00'") + .build() ); addDataTypeTest( - new DataTypeTest("datetime", JsonSchemaPrimitive.STRING) + dataTypeTestBuilder("datetime", JsonSchemaPrimitive.STRING) .addInsertValue("null") // .addInsertValue("'0000-00-00 00:00:00'") + .build() ); addDataTypeTest( - new DataTypeTest("timestamp", JsonSchemaPrimitive.STRING) + dataTypeTestBuilder("timestamp", JsonSchemaPrimitive.STRING) .addInsertValue("null") + .build() ); addDataTypeTest( - new DataTypeTest("time", JsonSchemaPrimitive.STRING) + dataTypeTestBuilder("time", JsonSchemaPrimitive.STRING) .addInsertValue("null") // .addInsertValue("'-838:59:59.000000'") + .build() ); addDataTypeTest( - new DataTypeTest("varchar", JsonSchemaPrimitive.STRING) + dataTypeTestBuilder("varchar", JsonSchemaPrimitive.STRING) .setFullSourceDataType("varchar(256) character set cp1251") - .addInsertValue("null") - .addInsertValue("'тест'") + .addInsertValue("null", "'тест'") + .build() ); addDataTypeTest( - new DataTypeTest("varchar", JsonSchemaPrimitive.STRING) + dataTypeTestBuilder("varchar", JsonSchemaPrimitive.STRING) .setFullSourceDataType("varchar(256) character set utf16") - .addInsertValue("null") - .addInsertValue("0xfffd") + .addInsertValue("null", "0xfffd") + .build() ); addDataTypeTest( - new DataTypeTest("varchar", JsonSchemaPrimitive.STRING) + dataTypeTestBuilder("varchar", JsonSchemaPrimitive.STRING) .setFullSourceDataType("varchar(256)") - .addInsertValue("null") - .addInsertValue("'!\"#$%&\\'()*+,-./:;<=>?\\@[\\]^_\\`{|}~'") + .addInsertValue("null", "'!\"#$%&\\'()*+,-./:;<=>?\\@[\\]^_\\`{|}~'") + .build() ); addDataTypeTest( - new DataTypeTest("varbinary", JsonSchemaPrimitive.STRING) + dataTypeTestBuilder("varbinary", JsonSchemaPrimitive.STRING) .setFullSourceDataType("varbinary(256)") - .addInsertValue("null") - .addInsertValue("'test'") + .addInsertValue("null", "'test'") + .build() ); addDataTypeTest( - new DataTypeTest("blob", JsonSchemaPrimitive.STRING) - .addInsertValue("null") - .addInsertValue("'test'") + dataTypeTestBuilder("blob", JsonSchemaPrimitive.STRING) + .addInsertValue("null", "'test'") + .build() ); addDataTypeTest( - new DataTypeTest("mediumtext", JsonSchemaPrimitive.STRING) - .addInsertValue("null") - .addInsertValue("lpad('0', 16777214, '0')") + dataTypeTestBuilder("mediumtext", JsonSchemaPrimitive.STRING) + .addInsertValue("null", "lpad('0', 16777214, '0')") + .build() ); addDataTypeTest( - new DataTypeTest("tinytext", JsonSchemaPrimitive.STRING) + dataTypeTestBuilder("tinytext", JsonSchemaPrimitive.STRING) .addInsertValue("null") + .build() ); addDataTypeTest( - new DataTypeTest("longtext", JsonSchemaPrimitive.STRING) + dataTypeTestBuilder("longtext", JsonSchemaPrimitive.STRING) .addInsertValue("null") + .build() ); addDataTypeTest( - new DataTypeTest("text", JsonSchemaPrimitive.STRING) + dataTypeTestBuilder("text", JsonSchemaPrimitive.STRING) .addInsertValue("null") + .build() ); addDataTypeTest( - new DataTypeTest("json", JsonSchemaPrimitive.STRING) - .addInsertValue("null") - .addInsertValue("'{\"a\" :10, \"b\": 15}'") + dataTypeTestBuilder("json", JsonSchemaPrimitive.STRING) + .addInsertValue("null", "'{\"a\" :10, \"b\": 15}'") + .build() ); addDataTypeTest( - new DataTypeTest("point", JsonSchemaPrimitive.OBJECT) - .addInsertValue("null") - .addInsertValue("(ST_GeomFromText('POINT(1 1)'))") + dataTypeTestBuilder("point", JsonSchemaPrimitive.OBJECT) + .addInsertValue("null", "(ST_GeomFromText('POINT(1 1)'))") + .build() ); addDataTypeTest( - new DataTypeTest("bool", JsonSchemaPrimitive.STRING) - .addInsertValue("null") - .addInsertValue("127") - .addInsertValue("-128") + dataTypeTestBuilder("bool", JsonSchemaPrimitive.STRING) + .addInsertValue("null", "127", "-128") + .build() ); } From d307970e5cd2dc92db4cdff01ba8d94f0cef2277 Mon Sep 17 00:00:00 2001 From: Andrii Leonets Date: Mon, 31 May 2021 23:50:00 +0300 Subject: [PATCH 04/11] clear logging --- .../source/SourceComprehensiveTest.java | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceComprehensiveTest.java b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceComprehensiveTest.java index ddc08fb6eed37..6a8a036c87d87 100644 --- a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceComprehensiveTest.java +++ b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceComprehensiveTest.java @@ -73,16 +73,10 @@ protected void setup(TestDestinationEnv testEnv) throws Exception { public void testDataTypes() throws Exception { ConfiguredAirbyteCatalog catalog = getConfiguredCatalog(); List allMessages = runRead(catalog); - LOGGER.info("Size: " + allMessages.size()); final List recordMessages = allMessages.stream().filter(m -> m.getType() == Type.RECORD).collect(Collectors.toList()); - recordMessages.forEach(msg -> LOGGER.info(msg.toString())); - - assertFalse(recordMessages.isEmpty(), "Expected a full refresh sync to produce records"); - - allMessages = runRead(catalog); - LOGGER.info("Size: " + allMessages.size()); - recordMessages.forEach(msg -> LOGGER.info(msg.toString())); + recordMessages.forEach(msg -> LOGGER.debug(msg.toString())); + assertFalse(recordMessages.isEmpty(), "Expected records from source"); } /** @@ -97,9 +91,9 @@ private void setupDatabaseInternal() throws Exception { for (DataTypeTest test : dataTypeTests) { database.query(ctx -> { ctx.fetch(test.getCreateSQL()); - LOGGER.info("Table " + test.getName() + " is created."); + LOGGER.debug("Table " + test.getName() + " is created."); test.getInsertSQLs().forEach(ctx::fetch); - LOGGER.info("Values " + test.values + " are inserted into " + test.getName()); + LOGGER.debug("Values " + test.values + " are inserted into " + test.getName()); return null; }); } From ec15fc4493ea08267af61ad1d354ac8879f8c247 Mon Sep 17 00:00:00 2001 From: Andrii Leonets Date: Tue, 1 Jun 2021 18:54:34 +0300 Subject: [PATCH 05/11] Move common inner classes to public classes. + rename SourceTest to SourceAbstractTest + clean autogenerated imports + proper builder implementation according to lombok style. --- .../source/fs/ExecutableTestSource.java | 1 + .../standardtest/source/DataTypeTest.java | 151 ++++++++++++++ .../source/SourceAbstractTest.java | 158 +++++++++++++++ .../source/SourceAcceptanceTest.java | 3 +- .../source/SourceComprehensiveTest.java | 186 ++++-------------- .../standardtest/source/SourceTest.java | 143 -------------- .../source/TestDestinationEnv.java | 17 ++ .../ClickHouseSourceAcceptanceTest.java | 1 + .../jdbc/JdbcSourceSourceAcceptanceTest.java | 1 + .../mssql/MssqlSourceAcceptanceTest.java | 1 + .../mysql/CdcMySqlSourceAcceptanceTest.java | 1 + .../CdcMySqlSourceComprehensiveTest.java | 77 ++++---- .../mysql/MySqlSourceAcceptanceTest.java | 1 + .../mysql/MySqlSourceComprehensiveTest.java | 74 +++---- .../sources/OracleSourceAcceptanceTest.java | 1 + .../CdcPostgresSourceAcceptanceTest.java | 1 + .../sources/PostgresSourceAcceptanceTest.java | 1 + .../sources/RedshiftSourceAcceptanceTest.java | 1 + 18 files changed, 455 insertions(+), 364 deletions(-) create mode 100644 airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/DataTypeTest.java create mode 100644 airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceAbstractTest.java delete mode 100644 airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceTest.java create mode 100644 airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/TestDestinationEnv.java diff --git a/airbyte-integrations/bases/base-standard-source-test-file/src/main/java/io/airbyte/integrations/standardtest/source/fs/ExecutableTestSource.java b/airbyte-integrations/bases/base-standard-source-test-file/src/main/java/io/airbyte/integrations/standardtest/source/fs/ExecutableTestSource.java index 18622c369e923..5cbe73875c453 100644 --- a/airbyte-integrations/bases/base-standard-source-test-file/src/main/java/io/airbyte/integrations/standardtest/source/fs/ExecutableTestSource.java +++ b/airbyte-integrations/bases/base-standard-source-test-file/src/main/java/io/airbyte/integrations/standardtest/source/fs/ExecutableTestSource.java @@ -28,6 +28,7 @@ import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; +import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.ConnectorSpecification; import java.nio.file.Path; diff --git a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/DataTypeTest.java b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/DataTypeTest.java new file mode 100644 index 0000000000000..d173f52bf063f --- /dev/null +++ b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/DataTypeTest.java @@ -0,0 +1,151 @@ +package io.airbyte.integrations.standardtest.source; + +import io.airbyte.protocol.models.ConfiguredAirbyteStream; +import io.airbyte.protocol.models.Field; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class DataTypeTest { + + private static final String DEFAULT_CREATE_TABLE_SQL = "CREATE TABLE %1$s(id integer primary key, test_column %2$s);"; + private static final String DEFAULT_INSERT_SQL = "INSERT INTO %1$s VALUES (%2$s, %3$s);"; + + private final String sourceType; + private final Field.JsonSchemaPrimitive airbyteType; + private final List values; + private final String createTablePatternSQL; + private final String insertPatternSQL; + private final String fullSourceDataType; + private long testNumber; + + DataTypeTest(String sourceType, Field.JsonSchemaPrimitive airbyteType, List values, String createTablePatternSQL, String insertPatternSQL, String fullSourceDataType) { + this.sourceType = sourceType; + this.airbyteType = airbyteType; + this.values = values; + this.createTablePatternSQL = createTablePatternSQL; + this.insertPatternSQL = insertPatternSQL; + this.fullSourceDataType = fullSourceDataType; + } + + /** + * The builder allows to setup any comprehensive data type test. + * + * @param sourceType name of the source data type. Duplicates by name will be tested independently + * from each others. Note that this name will be used for connector setup and table creation. + * If source syntax requires more details (E.g. "varchar" type requires length + * "varchar(50)"), you can additionally set custom data type syntax by + * {@link DataTypeTestBuilder#fullSourceDataType(String)} method. + * @param airbyteType corresponding Airbyte data type. It requires for proper configuration + * {@link ConfiguredAirbyteStream} + * @return builder for setup comprehensive test + */ + public static DataTypeTestBuilder builder(String sourceType, Field.JsonSchemaPrimitive airbyteType) { + return new DataTypeTestBuilder(sourceType, airbyteType); + } + + public static class DataTypeTestBuilder { + + private final String sourceType; + private final Field.JsonSchemaPrimitive airbyteType; + private final List values = new ArrayList<>(); + private String createTablePatternSQL; + private String insertPatternSQL; + private String fullSourceDataType; + + DataTypeTestBuilder(String sourceType, Field.JsonSchemaPrimitive airbyteType) { + this.sourceType = sourceType; + this.airbyteType = airbyteType; + this.createTablePatternSQL = DEFAULT_CREATE_TABLE_SQL; + this.insertPatternSQL = DEFAULT_INSERT_SQL; + this.fullSourceDataType = sourceType; + } + + /** + * Set custom the create table script pattern. Use it if you source uses untypical table creation + * sql. Default patter described {@link #DEFAULT_CREATE_TABLE_SQL} Note! The patter should contains + * two String place holders for the table name and data type. + * + * @param createTablePatternSQL creation table sql pattern + * @return builder + */ + public DataTypeTestBuilder createTablePatternSQL(String createTablePatternSQL) { + this.createTablePatternSQL = createTablePatternSQL; + return this; + } + + /** + * Set custom the insert record script pattern. Use it if you source uses untypical insert record + * sql. Default patter described {@link #DEFAULT_INSERT_SQL} Note! The patter should contains two + * String place holders for the table name and value. + * + * @param insertPatternSQL creation table sql pattern + * @return builder + */ + public DataTypeTestBuilder insertPatternSQL(String insertPatternSQL) { + this.insertPatternSQL = insertPatternSQL; + return this; + } + + /** + * Allows to set extended data type for the table creation. E.g. The "varchar" type requires in + * MySQL requires length. In this case fullSourceDataType will be "varchar(50)". + * + * @param fullSourceDataType actual string for the column data type description + * @return builder + */ + public DataTypeTestBuilder fullSourceDataType(String fullSourceDataType) { + this.fullSourceDataType = fullSourceDataType; + return this; + } + + /** + * Adds value(s) to the scope of a corresponding test. The values will be inserted into the created + * table. Note! The value will be inserted into the insert script without any transformations. Make + * sure that the value is in line with the source syntax. + * + * @param insertValue test value + * @return builder + */ + public DataTypeTestBuilder addInsertValue(String... insertValue) { + this.values.addAll(Arrays.asList(insertValue)); + return this; + } + + public DataTypeTest build() { + return new DataTypeTest(sourceType, airbyteType, values, createTablePatternSQL, insertPatternSQL, fullSourceDataType); + } + + } + + public void setTestNumber(long testNumber) { + this.testNumber = testNumber; + } + + public String getSourceType() { + return sourceType; + } + + public Field.JsonSchemaPrimitive getAirbyteType() { + return airbyteType; + } + + public String getName() { + return "test_" + testNumber + "_" + sourceType; + } + + public String getCreateSQL() { + return String.format(createTablePatternSQL, getName(), fullSourceDataType); + } + + public List getInsertSQLs() { + List insertSQLs = new ArrayList<>(); + int rowId = 1; + for (String value : values) { + insertSQLs.add(String.format(insertPatternSQL, getName(), rowId++, value)); + } + return insertSQLs; + } + +} diff --git a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceAbstractTest.java b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceAbstractTest.java new file mode 100644 index 0000000000000..fe75814b9e857 --- /dev/null +++ b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceAbstractTest.java @@ -0,0 +1,158 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.standardtest.source; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.config.StandardCheckConnectionInput; +import io.airbyte.config.StandardCheckConnectionOutput; +import io.airbyte.config.StandardDiscoverCatalogInput; +import io.airbyte.config.StandardTapConfig; +import io.airbyte.config.State; +import io.airbyte.config.JobGetSpecConfig; +import io.airbyte.protocol.models.AirbyteCatalog; +import io.airbyte.protocol.models.AirbyteMessage; +import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.ConnectorSpecification; +import io.airbyte.workers.DefaultCheckConnectionWorker; +import io.airbyte.workers.DefaultDiscoverCatalogWorker; +import io.airbyte.workers.DefaultGetSpecWorker; +import io.airbyte.workers.WorkerException; +import io.airbyte.workers.process.AirbyteIntegrationLauncher; +import io.airbyte.workers.process.DockerProcessFactory; +import io.airbyte.workers.process.ProcessFactory; +import io.airbyte.workers.protocols.airbyte.AirbyteSource; +import io.airbyte.workers.protocols.airbyte.DefaultAirbyteSource; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; + +public abstract class SourceAbstractTest { + + private TestDestinationEnv testEnv; + private Path jobRoot; + protected Path localRoot; + private ProcessFactory processFactory; + + private static final long JOB_ID = 0L; + private static final int JOB_ATTEMPT = 0; + + /** + * Name of the docker image that the tests will run against. + * + * @return docker image name + */ + protected abstract String getImageName(); + + /** + * Configuration specific to the integration. Will be passed to integration where appropriate in + * each test. Should be valid. + * + * @return integration-specific configuration + */ + protected abstract JsonNode getConfig() throws Exception; + + /** + * Function that performs any setup of external resources required for the test. e.g. instantiate a + * postgres database. This function will be called before EACH test. + * + * @param testEnv - information about the test environment. + * @throws Exception - can throw any exception, test framework will handle. + */ + protected abstract void setup(TestDestinationEnv testEnv) throws Exception; + + /** + * Function that performs any clean up of external resources required for the test. e.g. delete a + * postgres database. This function will be called after EACH test. It MUST remove all data in the + * destination so that there is no contamination across tests. + * + * @param testEnv - information about the test environment. + * @throws Exception - can throw any exception, test framework will handle. + */ + protected abstract void tearDown(TestDestinationEnv testEnv) throws Exception; + + @BeforeEach + public void setUpInternal() throws Exception { + final Path testDir = Path.of("/tmp/airbyte_tests/"); + Files.createDirectories(testDir); + final Path workspaceRoot = Files.createTempDirectory(testDir, "test"); + jobRoot = Files.createDirectories(Path.of(workspaceRoot.toString(), "job")); + localRoot = Files.createTempDirectory(testDir, "output"); + testEnv = new TestDestinationEnv(localRoot); + + setup(testEnv); + + processFactory = new DockerProcessFactory( + workspaceRoot, + workspaceRoot.toString(), + localRoot.toString(), + "host"); + } + + @AfterEach + public void tearDownInternal() throws Exception { + tearDown(testEnv); + } + + protected ConnectorSpecification runSpec() throws WorkerException { + return new DefaultGetSpecWorker(new AirbyteIntegrationLauncher(JOB_ID, JOB_ATTEMPT, getImageName(), processFactory)) + .run(new JobGetSpecConfig().withDockerImage(getImageName()), jobRoot); + } + + protected StandardCheckConnectionOutput runCheck() throws Exception { + return new DefaultCheckConnectionWorker(new AirbyteIntegrationLauncher(JOB_ID, JOB_ATTEMPT, getImageName(), processFactory)) + .run(new StandardCheckConnectionInput().withConnectionConfiguration(getConfig()), jobRoot); + } + + protected AirbyteCatalog runDiscover() throws Exception { + return new DefaultDiscoverCatalogWorker(new AirbyteIntegrationLauncher(JOB_ID, JOB_ATTEMPT, getImageName(), processFactory)) + .run(new StandardDiscoverCatalogInput().withConnectionConfiguration(getConfig()), jobRoot); + } + + protected List runRead(ConfiguredAirbyteCatalog configuredCatalog) throws Exception { + return runRead(configuredCatalog, null); + } + + // todo (cgardens) - assume no state since we are all full refresh right now. + protected List runRead(ConfiguredAirbyteCatalog catalog, JsonNode state) throws Exception { + final StandardTapConfig sourceConfig = new StandardTapConfig() + .withSourceConnectionConfiguration(getConfig()) + .withState(state == null ? null : new State().withState(state)) + .withCatalog(catalog); + + final AirbyteSource source = new DefaultAirbyteSource(new AirbyteIntegrationLauncher(JOB_ID, JOB_ATTEMPT, getImageName(), processFactory)); + final List messages = new ArrayList<>(); + source.start(sourceConfig, jobRoot); + while (!source.isFinished()) { + source.attemptRead().ifPresent(messages::add); + } + source.close(); + + return messages; + } + +} diff --git a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceAcceptanceTest.java b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceAcceptanceTest.java index 4cd2b6c6e7cac..463c2a9e68db0 100644 --- a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceAcceptanceTest.java +++ b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceAcceptanceTest.java @@ -53,7 +53,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public abstract class SourceAcceptanceTest extends SourceTest{ +public abstract class SourceAcceptanceTest extends SourceAbstractTest { public static final String CDC_LSN = "_ab_cdc_lsn"; public static final String CDC_UPDATED_AT = "_ab_cdc_updated_at"; @@ -332,7 +332,6 @@ private boolean sourceSupportsIncremental() throws Exception { return false; } - private void assertSameRecords(List expected, List actual, String message) { final List prunedExpected = expected.stream().map(this::pruneEmittedAt).collect(Collectors.toList()); final List prunedActual = actual diff --git a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceComprehensiveTest.java b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceComprehensiveTest.java index 6a8a036c87d87..80f27e8fea127 100644 --- a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceComprehensiveTest.java +++ b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceComprehensiveTest.java @@ -24,33 +24,36 @@ package io.airbyte.integrations.standardtest.source; +import static org.junit.jupiter.api.Assertions.assertFalse; + import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.Lists; -import io.airbyte.protocol.models.*; -import io.airbyte.protocol.models.DestinationSyncMode; +import io.airbyte.db.Database; +import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.AirbyteMessage.Type; +import io.airbyte.protocol.models.CatalogHelpers; +import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.ConfiguredAirbyteStream; +import io.airbyte.protocol.models.DestinationSyncMode; +import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.SyncMode; -import io.airbyte.db.Database; +import java.util.List; +import java.util.ArrayList; +import java.util.stream.Collectors; import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.*; -import java.util.stream.Collectors; - -import static org.junit.jupiter.api.Assertions.*; - -public abstract class SourceComprehensiveTest extends SourceTest { +public abstract class SourceComprehensiveTest extends SourceAbstractTest { private static final Logger LOGGER = LoggerFactory.getLogger(SourceComprehensiveTest.class); - private static final String DEFAULT_CREATE_TABLE_SQL = "CREATE TABLE %1$s(id integer primary key, test_column %2$s);"; - private static final String DEFAULT_INSERT_SQL = "INSERT INTO %1$s VALUES (%2$s, %3$s);"; private final List dataTypeTests = new ArrayList<>(); /** * Setup the test database. All tables and data described in the registered tests will be put there. - * @return configured test database + * + * @return configured test database * @throws Exception - might throw any exception during initialization. */ protected abstract Database setupDatabase() throws Exception; @@ -81,7 +84,9 @@ public void testDataTypes() throws Exception { /** * Creates all tables and insert data described in the registered data type tests. - * @throws Exception might raise exception if configuration goes wrong or tables creation/insert scripts failed. + * + * @throws Exception might raise exception if configuration goes wrong or tables creation/insert + * scripts failed. */ private void setupDatabaseInternal() throws Exception { Database database = setupDatabase(); @@ -93,7 +98,6 @@ private void setupDatabaseInternal() throws Exception { ctx.fetch(test.getCreateSQL()); LOGGER.debug("Table " + test.getName() + " is created."); test.getInsertSQLs().forEach(ctx::fetch); - LOGGER.debug("Values " + test.values + " are inserted into " + test.getName()); return null; }); } @@ -103,149 +107,41 @@ private void setupDatabaseInternal() throws Exception { /** * Configures streams for all registered data type tests. + * * @return configured catalog */ private ConfiguredAirbyteCatalog getConfiguredCatalog() throws Exception { final JsonNode config = getConfig(); return new ConfiguredAirbyteCatalog().withStreams( - dataTypeTests - .stream() - .map(test -> - new ConfiguredAirbyteStream() - .withSyncMode(SyncMode.INCREMENTAL) - .withCursorField(Lists.newArrayList("id")) - .withDestinationSyncMode(DestinationSyncMode.APPEND) - .withStream(CatalogHelpers.createAirbyteStream( - String.format("%s", test.getName()), - String.format("%s", config.get("database").asText()), - Field.of("id", Field.JsonSchemaPrimitive.NUMBER), - Field.of("test_column", test.airbyteType)) - .withSourceDefinedCursor(true) - .withSourceDefinedPrimaryKey(List.of(List.of("id"))) - .withSupportedSyncModes( - Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)))) - .collect(Collectors.toList()) - ); + dataTypeTests + .stream() + .map(test -> new ConfiguredAirbyteStream() + .withSyncMode(SyncMode.INCREMENTAL) + .withCursorField(Lists.newArrayList("id")) + .withDestinationSyncMode(DestinationSyncMode.APPEND) + .withStream(CatalogHelpers.createAirbyteStream( + String.format("%s", test.getName()), + String.format("%s", config.get("database").asText()), + Field.of("id", Field.JsonSchemaPrimitive.NUMBER), + Field.of("test_column", test.getAirbyteType())) + .withSourceDefinedCursor(true) + .withSourceDefinedPrimaryKey(List.of(List.of("id"))) + .withSupportedSyncModes( + Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)))) + .collect(Collectors.toList())); } /** - * Register your test in the run scope. - * For each test will be created a table with one column of specified type. - * Note! If you register more than one test with the same type name, they will be run as independent tests with own - * streams. - * @param test comprehensive data type test built by {@link DataTypeTestBuilder} + * Register your test in the run scope. For each test will be created a table with one column of + * specified type. Note! If you register more than one test with the same type name, they will be + * run as independent tests with own streams. + * + * @param test comprehensive data type test */ public void addDataTypeTest(DataTypeTest test) { dataTypeTests.add(test); - test.testNumber = dataTypeTests.stream().filter(t -> t.sourceType.equals(test.sourceType)).count(); - } - - /** - * The builder allows to setup any comprehensive data type test. - * @param sourceType name of the source data type. Duplicates by name will be tested independently from each others. - * Note that this name will be used for connector setup and table creation. - * If source syntax requires more details (E.g. "varchar" type requires length "varchar(50)"), - * you can additionally set custom data type syntax by {@link DataTypeTestBuilder#setFullSourceDataType(String)} method. - * @param airbyteType corresponding Airbyte data type. It requires for proper configuration {@link ConfiguredAirbyteStream} - * @return builder for setup comprehensive test - */ - public DataTypeTestBuilder dataTypeTestBuilder(String sourceType, Field.JsonSchemaPrimitive airbyteType) { - return new DataTypeTestBuilder(sourceType, airbyteType); - } - - public class DataTypeTestBuilder { - - private final DataTypeTest dataTypeTest; - - private DataTypeTestBuilder(String sourceType, Field.JsonSchemaPrimitive airbyteType) { - this.dataTypeTest = new DataTypeTest(); - this.dataTypeTest.sourceType = sourceType; - this.dataTypeTest.airbyteType = airbyteType; - this.dataTypeTest.createTablePatternSQL = DEFAULT_CREATE_TABLE_SQL; - this.dataTypeTest.insertPatternSQL = DEFAULT_INSERT_SQL; - this.dataTypeTest.fullSourceDataType = sourceType; - } - - /** - * Set custom the create table script pattern. Use it if you source uses untypical table creation sql. - * Default patter described {@link #DEFAULT_CREATE_TABLE_SQL} - * Note! The patter should contains two String place holders for the table name and data type. - * @param createTablePatternSQL creation table sql pattern - * @return builder - */ - public DataTypeTestBuilder setCreateTablePatternSQL(String createTablePatternSQL) { - this.dataTypeTest.createTablePatternSQL = createTablePatternSQL; - return this; - } - - /** - * Set custom the insert record script pattern. Use it if you source uses untypical insert record sql. - * Default patter described {@link #DEFAULT_INSERT_SQL} - * Note! The patter should contains two String place holders for the table name and value. - * @param insertPatternSQL creation table sql pattern - * @return builder - */ - public DataTypeTestBuilder setInsertPatternSQL(String insertPatternSQL) { - this.dataTypeTest.insertPatternSQL = insertPatternSQL; - return this; - } - - /** - * Allows to set extended data type for the table creation. - * E.g. The "varchar" type requires in MySQL requires length. In this case fullSourceDataType will be "varchar(50)". - * @param fullSourceDataType actual string for the column data type description - * @return builder - */ - public DataTypeTestBuilder setFullSourceDataType(String fullSourceDataType) { - this.dataTypeTest.fullSourceDataType = fullSourceDataType; - return this; - } - - /** - * Adds value(s) to the scope of a corresponding test. The values will be inserted into the created table. - * Note! The value will be inserted into the insert script without any transformations. Make sure that the value is - * in line with the source syntax. - * @param insertValue test value - * @return builder - */ - public DataTypeTestBuilder addInsertValue(String...insertValue) { - this.dataTypeTest.values.addAll(Arrays.asList(insertValue)); - return this; - } - - public DataTypeTest build() { - return dataTypeTest; - } - - } - - private class DataTypeTest { - - private String sourceType; - private Field.JsonSchemaPrimitive airbyteType; - private final List values = new ArrayList<>(); - private String createTablePatternSQL; - private String insertPatternSQL; - private long testNumber; - private String fullSourceDataType; - - private String getName() { - return "test_" + testNumber + "_" + sourceType; - } - - private String getCreateSQL() { - return String.format(createTablePatternSQL, getName(), fullSourceDataType); - } - - private List getInsertSQLs() { - List insertSQLs = new ArrayList<>(); - int rowId = 1; - for (String value : values) { - insertSQLs.add(String.format(insertPatternSQL, getName(), rowId++, value)); - } - return insertSQLs; - } + test.setTestNumber(dataTypeTests.stream().filter(t -> t.getSourceType().equals(test.getSourceType())).count()); } } diff --git a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceTest.java b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceTest.java deleted file mode 100644 index e17cfcb384973..0000000000000 --- a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceTest.java +++ /dev/null @@ -1,143 +0,0 @@ -package io.airbyte.integrations.standardtest.source; - -import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.config.*; -import io.airbyte.protocol.models.AirbyteCatalog; -import io.airbyte.protocol.models.AirbyteMessage; -import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; -import io.airbyte.protocol.models.ConnectorSpecification; -import io.airbyte.workers.DefaultCheckConnectionWorker; -import io.airbyte.workers.DefaultDiscoverCatalogWorker; -import io.airbyte.workers.DefaultGetSpecWorker; -import io.airbyte.workers.WorkerException; -import io.airbyte.workers.process.AirbyteIntegrationLauncher; -import io.airbyte.workers.process.DockerProcessFactory; -import io.airbyte.workers.process.ProcessFactory; -import io.airbyte.workers.protocols.airbyte.AirbyteSource; -import io.airbyte.workers.protocols.airbyte.DefaultAirbyteSource; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; - -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.List; - -public abstract class SourceTest { - - private TestDestinationEnv testEnv; - private Path jobRoot; - protected Path localRoot; - private ProcessFactory processFactory; - - private static final long JOB_ID = 0L; - private static final int JOB_ATTEMPT = 0; - - /** - * Name of the docker image that the tests will run against. - * - * @return docker image name - */ - protected abstract String getImageName(); - - /** - * Configuration specific to the integration. Will be passed to integration where appropriate in - * each test. Should be valid. - * - * @return integration-specific configuration - */ - protected abstract JsonNode getConfig() throws Exception; - - /** - * Function that performs any setup of external resources required for the test. e.g. instantiate a - * postgres database. This function will be called before EACH test. - * - * @param testEnv - information about the test environment. - * @throws Exception - can throw any exception, test framework will handle. - */ - protected abstract void setup(TestDestinationEnv testEnv) throws Exception; - - /** - * Function that performs any clean up of external resources required for the test. e.g. delete a - * postgres database. This function will be called after EACH test. It MUST remove all data in the - * destination so that there is no contamination across tests. - * - * @param testEnv - information about the test environment. - * @throws Exception - can throw any exception, test framework will handle. - */ - protected abstract void tearDown(TestDestinationEnv testEnv) throws Exception; - - @BeforeEach - public void setUpInternal() throws Exception { - final Path testDir = Path.of("/tmp/airbyte_tests/"); - Files.createDirectories(testDir); - final Path workspaceRoot = Files.createTempDirectory(testDir, "test"); - jobRoot = Files.createDirectories(Path.of(workspaceRoot.toString(), "job")); - localRoot = Files.createTempDirectory(testDir, "output"); - testEnv = new TestDestinationEnv(localRoot); - - setup(testEnv); - - processFactory = new DockerProcessFactory( - workspaceRoot, - workspaceRoot.toString(), - localRoot.toString(), - "host"); - } - - @AfterEach - public void tearDownInternal() throws Exception { - tearDown(testEnv); - } - - protected ConnectorSpecification runSpec() throws WorkerException { - return new DefaultGetSpecWorker(new AirbyteIntegrationLauncher(JOB_ID, JOB_ATTEMPT, getImageName(), processFactory)) - .run(new JobGetSpecConfig().withDockerImage(getImageName()), jobRoot); - } - - protected StandardCheckConnectionOutput runCheck() throws Exception { - return new DefaultCheckConnectionWorker(new AirbyteIntegrationLauncher(JOB_ID, JOB_ATTEMPT, getImageName(), processFactory)) - .run(new StandardCheckConnectionInput().withConnectionConfiguration(getConfig()), jobRoot); - } - - protected AirbyteCatalog runDiscover() throws Exception { - return new DefaultDiscoverCatalogWorker(new AirbyteIntegrationLauncher(JOB_ID, JOB_ATTEMPT, getImageName(), processFactory)) - .run(new StandardDiscoverCatalogInput().withConnectionConfiguration(getConfig()), jobRoot); - } - - protected List runRead(ConfiguredAirbyteCatalog configuredCatalog) throws Exception { - return runRead(configuredCatalog, null); - } - - // todo (cgardens) - assume no state since we are all full refresh right now. - protected List runRead(ConfiguredAirbyteCatalog catalog, JsonNode state) throws Exception { - final StandardTapConfig sourceConfig = new StandardTapConfig() - .withSourceConnectionConfiguration(getConfig()) - .withState(state == null ? null : new State().withState(state)) - .withCatalog(catalog); - - final AirbyteSource source = new DefaultAirbyteSource(new AirbyteIntegrationLauncher(JOB_ID, JOB_ATTEMPT, getImageName(), processFactory)); - final List messages = new ArrayList<>(); - source.start(sourceConfig, jobRoot); - while (!source.isFinished()) { - source.attemptRead().ifPresent(messages::add); - } - source.close(); - - return messages; - } - - public static class TestDestinationEnv { - - private final Path localRoot; - - public TestDestinationEnv(Path localRoot) { - this.localRoot = localRoot; - } - - public Path getLocalRoot() { - return localRoot; - } - - } -} diff --git a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/TestDestinationEnv.java b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/TestDestinationEnv.java new file mode 100644 index 0000000000000..cdf2cfbe40d10 --- /dev/null +++ b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/TestDestinationEnv.java @@ -0,0 +1,17 @@ +package io.airbyte.integrations.standardtest.source; + +import java.nio.file.Path; + +public class TestDestinationEnv { + + private final Path localRoot; + + public TestDestinationEnv(Path localRoot) { + this.localRoot = localRoot; + } + + public Path getLocalRoot() { + return localRoot; + } + +} diff --git a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseSourceAcceptanceTest.java index 7ecb4e4fdf375..ede3c4ed42b50 100644 --- a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseSourceAcceptanceTest.java @@ -34,6 +34,7 @@ import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.source.clickhouse.ClickHouseSource; import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; +import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.protocol.models.CatalogHelpers; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.ConfiguredAirbyteStream; diff --git a/airbyte-integrations/connectors/source-jdbc/src/test-integration/java/io/airbyte/integrations/source/jdbc/JdbcSourceSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-jdbc/src/test-integration/java/io/airbyte/integrations/source/jdbc/JdbcSourceSourceAcceptanceTest.java index 95c0b9645e427..569f0e52d578a 100644 --- a/airbyte-integrations/connectors/source-jdbc/src/test-integration/java/io/airbyte/integrations/source/jdbc/JdbcSourceSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-jdbc/src/test-integration/java/io/airbyte/integrations/source/jdbc/JdbcSourceSourceAcceptanceTest.java @@ -31,6 +31,7 @@ import io.airbyte.db.Database; import io.airbyte.db.Databases; import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; +import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.protocol.models.CatalogHelpers; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.ConnectorSpecification; diff --git a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlSourceAcceptanceTest.java index d2af5e0e4917f..6a3911a12cd65 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlSourceAcceptanceTest.java @@ -32,6 +32,7 @@ import io.airbyte.db.Database; import io.airbyte.db.Databases; import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; +import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.protocol.models.CatalogHelpers; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.ConnectorSpecification; diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceAcceptanceTest.java index d5bdee740f220..c8162f01a7cd6 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceAcceptanceTest.java @@ -32,6 +32,7 @@ import io.airbyte.db.Database; import io.airbyte.db.Databases; import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; +import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.protocol.models.CatalogHelpers; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.ConfiguredAirbyteStream; diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceComprehensiveTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceComprehensiveTest.java index 132edfa50b492..bb3eac67da544 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceComprehensiveTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceComprehensiveTest.java @@ -5,8 +5,9 @@ import io.airbyte.commons.json.Jsons; import io.airbyte.db.Database; import io.airbyte.db.Databases; +import io.airbyte.integrations.standardtest.source.DataTypeTest; import io.airbyte.integrations.standardtest.source.SourceComprehensiveTest; -import io.airbyte.integrations.standardtest.source.SourceTest; +import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.protocol.models.Field; import org.jooq.SQLDialect; import org.testcontainers.containers.MySQLContainer; @@ -22,7 +23,7 @@ protected JsonNode getConfig() { } @Override - protected void tearDown(SourceTest.TestDestinationEnv testEnv) { + protected void tearDown(TestDestinationEnv testEnv) { container.close(); } @@ -96,179 +97,179 @@ private void executeQuery(String query) { @Override protected void initTests() { addDataTypeTest( - dataTypeTestBuilder("tinyint", Field.JsonSchemaPrimitive.NUMBER) + DataTypeTest.builder("tinyint", Field.JsonSchemaPrimitive.NUMBER) .addInsertValue("null") .addInsertValue("-128", "127") .build() ); addDataTypeTest( - dataTypeTestBuilder("smallint", Field.JsonSchemaPrimitive.NUMBER) + DataTypeTest.builder("smallint", Field.JsonSchemaPrimitive.NUMBER) .addInsertValue("null") .addInsertValue("-32768", "32767") .build() ); addDataTypeTest( - dataTypeTestBuilder("smallint", Field.JsonSchemaPrimitive.NUMBER) - .setFullSourceDataType("smallint zerofill") + DataTypeTest.builder("smallint", Field.JsonSchemaPrimitive.NUMBER) + .fullSourceDataType("smallint zerofill") .addInsertValue("1") .build() ); addDataTypeTest( - dataTypeTestBuilder("mediumint", Field.JsonSchemaPrimitive.NUMBER) + DataTypeTest.builder("mediumint", Field.JsonSchemaPrimitive.NUMBER) .addInsertValue("null", "-8388608", "8388607") .build() ); addDataTypeTest( - dataTypeTestBuilder("mediumint", Field.JsonSchemaPrimitive.NUMBER) - .setFullSourceDataType("mediumint zerofill") + DataTypeTest.builder("mediumint", Field.JsonSchemaPrimitive.NUMBER) + .fullSourceDataType("mediumint zerofill") .addInsertValue("1") .build() ); addDataTypeTest( - dataTypeTestBuilder("int", Field.JsonSchemaPrimitive.NUMBER) + DataTypeTest.builder("int", Field.JsonSchemaPrimitive.NUMBER) .addInsertValue("null", "-2147483648", "2147483647") .build() ); addDataTypeTest( - dataTypeTestBuilder("int", Field.JsonSchemaPrimitive.NUMBER) - .setFullSourceDataType("int zerofill") + DataTypeTest.builder("int", Field.JsonSchemaPrimitive.NUMBER) + .fullSourceDataType("int zerofill") .addInsertValue("1") .build() ); addDataTypeTest( - dataTypeTestBuilder("bigint", Field.JsonSchemaPrimitive.NUMBER) + DataTypeTest.builder("bigint", Field.JsonSchemaPrimitive.NUMBER) .addInsertValue("null", "9223372036854775807") .build() ); addDataTypeTest( - dataTypeTestBuilder("float", Field.JsonSchemaPrimitive.NUMBER) + DataTypeTest.builder("float", Field.JsonSchemaPrimitive.NUMBER) .addInsertValue("null") .build() ); addDataTypeTest( - dataTypeTestBuilder("double", Field.JsonSchemaPrimitive.NUMBER) + DataTypeTest.builder("double", Field.JsonSchemaPrimitive.NUMBER) .addInsertValue("null", "power(10, 308)", "1/power(10, 45)") .build() ); addDataTypeTest( - dataTypeTestBuilder("decimal", Field.JsonSchemaPrimitive.NUMBER) - .setFullSourceDataType("decimal(5,2)") + DataTypeTest.builder("decimal", Field.JsonSchemaPrimitive.NUMBER) + .fullSourceDataType("decimal(5,2)") .addInsertValue("null") .build() ); addDataTypeTest( - dataTypeTestBuilder("bit", Field.JsonSchemaPrimitive.NUMBER) + DataTypeTest.builder("bit", Field.JsonSchemaPrimitive.NUMBER) .addInsertValue("null", "1", "0") .build() ); addDataTypeTest( - dataTypeTestBuilder("date", Field.JsonSchemaPrimitive.STRING) + DataTypeTest.builder("date", Field.JsonSchemaPrimitive.STRING) .addInsertValue("null", "'2021-01-00'", "'2021-00-00'", "'0000-00-00'") .build() ); addDataTypeTest( - dataTypeTestBuilder("datetime", Field.JsonSchemaPrimitive.STRING) + DataTypeTest.builder("datetime", Field.JsonSchemaPrimitive.STRING) .addInsertValue("null", "'0000-00-00 00:00:00'") .build() ); addDataTypeTest( - dataTypeTestBuilder("timestamp", Field.JsonSchemaPrimitive.STRING) + DataTypeTest.builder("timestamp", Field.JsonSchemaPrimitive.STRING) .addInsertValue("null") .build() ); addDataTypeTest( - dataTypeTestBuilder("time", Field.JsonSchemaPrimitive.STRING) + DataTypeTest.builder("time", Field.JsonSchemaPrimitive.STRING) .addInsertValue("null", "'-838:59:59.000000'") .build() ); addDataTypeTest( - dataTypeTestBuilder("varchar", Field.JsonSchemaPrimitive.STRING) - .setFullSourceDataType("varchar(256) character set cp1251") + DataTypeTest.builder("varchar", Field.JsonSchemaPrimitive.STRING) + .fullSourceDataType("varchar(256) character set cp1251") .addInsertValue("null", "'тест'") .build() ); addDataTypeTest( - dataTypeTestBuilder("varchar", Field.JsonSchemaPrimitive.STRING) - .setFullSourceDataType("varchar(256) character set utf16") + DataTypeTest.builder("varchar", Field.JsonSchemaPrimitive.STRING) + .fullSourceDataType("varchar(256) character set utf16") .addInsertValue("null", "0xfffd") .build() ); addDataTypeTest( - dataTypeTestBuilder("varchar", Field.JsonSchemaPrimitive.STRING) - .setFullSourceDataType("varchar(256)") + DataTypeTest.builder("varchar", Field.JsonSchemaPrimitive.STRING) + .fullSourceDataType("varchar(256)") .addInsertValue("null", "'!\"#$%&\\'()*+,-./:;<=>?\\@[\\]^_\\`{|}~'") .build() ); addDataTypeTest( - dataTypeTestBuilder("varbinary", Field.JsonSchemaPrimitive.STRING) - .setFullSourceDataType("varbinary(256)") + DataTypeTest.builder("varbinary", Field.JsonSchemaPrimitive.STRING) + .fullSourceDataType("varbinary(256)") .addInsertValue("null", "'test'") .build() ); addDataTypeTest( - dataTypeTestBuilder("blob", Field.JsonSchemaPrimitive.STRING) + DataTypeTest.builder("blob", Field.JsonSchemaPrimitive.STRING) .addInsertValue("null", "'test'") .build() ); addDataTypeTest( - dataTypeTestBuilder("mediumtext", Field.JsonSchemaPrimitive.STRING) + DataTypeTest.builder("mediumtext", Field.JsonSchemaPrimitive.STRING) .addInsertValue("null", "lpad('0', 16777214, '0')") .build() ); addDataTypeTest( - dataTypeTestBuilder("tinytext", Field.JsonSchemaPrimitive.STRING) + DataTypeTest.builder("tinytext", Field.JsonSchemaPrimitive.STRING) .addInsertValue("null") .build() ); addDataTypeTest( - dataTypeTestBuilder("longtext", Field.JsonSchemaPrimitive.STRING) + DataTypeTest.builder("longtext", Field.JsonSchemaPrimitive.STRING) .addInsertValue("null") .build() ); addDataTypeTest( - dataTypeTestBuilder("text", Field.JsonSchemaPrimitive.STRING) + DataTypeTest.builder("text", Field.JsonSchemaPrimitive.STRING) .addInsertValue("null") .build() ); addDataTypeTest( - dataTypeTestBuilder("json", Field.JsonSchemaPrimitive.STRING) + DataTypeTest.builder("json", Field.JsonSchemaPrimitive.STRING) .addInsertValue("null", "'{\"a\" :10, \"b\": 15}'") .build() ); addDataTypeTest( - dataTypeTestBuilder("point", Field.JsonSchemaPrimitive.OBJECT) + DataTypeTest.builder("point", Field.JsonSchemaPrimitive.OBJECT) .addInsertValue("null", "(ST_GeomFromText('POINT(1 1)'))") .build() ); addDataTypeTest( - dataTypeTestBuilder("bool", Field.JsonSchemaPrimitive.STRING) + DataTypeTest.builder("bool", Field.JsonSchemaPrimitive.STRING) .addInsertValue("null", "1", "127", "-128") .build() ); diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceAcceptanceTest.java index abf197ee7fe2c..4163e0cf98f4f 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceAcceptanceTest.java @@ -32,6 +32,7 @@ import io.airbyte.db.Database; import io.airbyte.db.Databases; import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; +import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.protocol.models.CatalogHelpers; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.ConfiguredAirbyteStream; diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceComprehensiveTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceComprehensiveTest.java index e3b3dd1aa02f8..dd639c7482a22 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceComprehensiveTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceComprehensiveTest.java @@ -29,7 +29,9 @@ import io.airbyte.commons.json.Jsons; import io.airbyte.db.Database; import io.airbyte.db.Databases; +import io.airbyte.integrations.standardtest.source.DataTypeTest; import io.airbyte.integrations.standardtest.source.SourceComprehensiveTest; +import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.protocol.models.Field.JsonSchemaPrimitive; import org.jooq.SQLDialect; import org.testcontainers.containers.MySQLContainer; @@ -87,85 +89,85 @@ protected Database setupDatabase() throws Exception { @Override protected void initTests() { addDataTypeTest( - dataTypeTestBuilder("tinyint", JsonSchemaPrimitive.NUMBER) + DataTypeTest.builder("tinyint", JsonSchemaPrimitive.NUMBER) .addInsertValue("null") .addInsertValue("-128", "127") .build() ); addDataTypeTest( - dataTypeTestBuilder("smallint", JsonSchemaPrimitive.NUMBER) + DataTypeTest.builder("smallint", JsonSchemaPrimitive.NUMBER) .addInsertValue("null") .addInsertValue("-32768", "32767") .build() ); addDataTypeTest( - dataTypeTestBuilder("smallint", JsonSchemaPrimitive.NUMBER) - .setFullSourceDataType("smallint zerofill") + DataTypeTest.builder("smallint", JsonSchemaPrimitive.NUMBER) + .fullSourceDataType("smallint zerofill") .addInsertValue("1") .build() ); addDataTypeTest( - dataTypeTestBuilder("mediumint", JsonSchemaPrimitive.NUMBER) + DataTypeTest.builder("mediumint", JsonSchemaPrimitive.NUMBER) .addInsertValue("null", "-8388608", "8388607") .build() ); addDataTypeTest( - dataTypeTestBuilder("mediumint", JsonSchemaPrimitive.NUMBER) - .setFullSourceDataType("mediumint zerofill") + DataTypeTest.builder("mediumint", JsonSchemaPrimitive.NUMBER) + .fullSourceDataType("mediumint zerofill") .addInsertValue("1") .build() ); addDataTypeTest( - dataTypeTestBuilder("int", JsonSchemaPrimitive.NUMBER) + DataTypeTest.builder("int", JsonSchemaPrimitive.NUMBER) .addInsertValue("null", "-2147483648", "2147483647") .build() ); addDataTypeTest( - dataTypeTestBuilder("int", JsonSchemaPrimitive.NUMBER) - .setFullSourceDataType("int zerofill") + DataTypeTest.builder("int", JsonSchemaPrimitive.NUMBER) + .fullSourceDataType("int zerofill") .addInsertValue("1") .build() ); addDataTypeTest( - dataTypeTestBuilder("bigint", JsonSchemaPrimitive.NUMBER) + DataTypeTest.builder("bigint", JsonSchemaPrimitive.NUMBER) .addInsertValue("null", "9223372036854775807") .build() ); addDataTypeTest( - dataTypeTestBuilder("float", JsonSchemaPrimitive.NUMBER) + DataTypeTest.builder("float", JsonSchemaPrimitive.NUMBER) .addInsertValue("null") .build() ); addDataTypeTest( - dataTypeTestBuilder("double", JsonSchemaPrimitive.NUMBER) + DataTypeTest.builder("double", JsonSchemaPrimitive.NUMBER) .addInsertValue("null", "power(10, 308)", "1/power(10, 45)") .build() ); addDataTypeTest( - dataTypeTestBuilder("decimal", JsonSchemaPrimitive.NUMBER) - .setFullSourceDataType("decimal(5,2)") + DataTypeTest.builder("decimal", JsonSchemaPrimitive.NUMBER) + .fullSourceDataType("decimal(5,2)") .addInsertValue("null") .build() ); addDataTypeTest( - dataTypeTestBuilder("bit", JsonSchemaPrimitive.NUMBER) + DataTypeTest.builder("bit", JsonSchemaPrimitive.NUMBER) .addInsertValue("null", "1", "0") .build() ); addDataTypeTest( - dataTypeTestBuilder("date", JsonSchemaPrimitive.STRING) + DataTypeTest.builder("date", JsonSchemaPrimitive.STRING) .addInsertValue("null") // .addInsertValue("'2021-01-00'") // .addInsertValue("'2021-00-00'") @@ -174,97 +176,97 @@ protected void initTests() { ); addDataTypeTest( - dataTypeTestBuilder("datetime", JsonSchemaPrimitive.STRING) + DataTypeTest.builder("datetime", JsonSchemaPrimitive.STRING) .addInsertValue("null") // .addInsertValue("'0000-00-00 00:00:00'") .build() ); addDataTypeTest( - dataTypeTestBuilder("timestamp", JsonSchemaPrimitive.STRING) + DataTypeTest.builder("timestamp", JsonSchemaPrimitive.STRING) .addInsertValue("null") .build() ); addDataTypeTest( - dataTypeTestBuilder("time", JsonSchemaPrimitive.STRING) + DataTypeTest.builder("time", JsonSchemaPrimitive.STRING) .addInsertValue("null") // .addInsertValue("'-838:59:59.000000'") .build() ); addDataTypeTest( - dataTypeTestBuilder("varchar", JsonSchemaPrimitive.STRING) - .setFullSourceDataType("varchar(256) character set cp1251") + DataTypeTest.builder("varchar", JsonSchemaPrimitive.STRING) + .fullSourceDataType("varchar(256) character set cp1251") .addInsertValue("null", "'тест'") .build() ); addDataTypeTest( - dataTypeTestBuilder("varchar", JsonSchemaPrimitive.STRING) - .setFullSourceDataType("varchar(256) character set utf16") + DataTypeTest.builder("varchar", JsonSchemaPrimitive.STRING) + .fullSourceDataType("varchar(256) character set utf16") .addInsertValue("null", "0xfffd") .build() ); addDataTypeTest( - dataTypeTestBuilder("varchar", JsonSchemaPrimitive.STRING) - .setFullSourceDataType("varchar(256)") + DataTypeTest.builder("varchar", JsonSchemaPrimitive.STRING) + .fullSourceDataType("varchar(256)") .addInsertValue("null", "'!\"#$%&\\'()*+,-./:;<=>?\\@[\\]^_\\`{|}~'") .build() ); addDataTypeTest( - dataTypeTestBuilder("varbinary", JsonSchemaPrimitive.STRING) - .setFullSourceDataType("varbinary(256)") + DataTypeTest.builder("varbinary", JsonSchemaPrimitive.STRING) + .fullSourceDataType("varbinary(256)") .addInsertValue("null", "'test'") .build() ); addDataTypeTest( - dataTypeTestBuilder("blob", JsonSchemaPrimitive.STRING) + DataTypeTest.builder("blob", JsonSchemaPrimitive.STRING) .addInsertValue("null", "'test'") .build() ); addDataTypeTest( - dataTypeTestBuilder("mediumtext", JsonSchemaPrimitive.STRING) + DataTypeTest.builder("mediumtext", JsonSchemaPrimitive.STRING) .addInsertValue("null", "lpad('0', 16777214, '0')") .build() ); addDataTypeTest( - dataTypeTestBuilder("tinytext", JsonSchemaPrimitive.STRING) + DataTypeTest.builder("tinytext", JsonSchemaPrimitive.STRING) .addInsertValue("null") .build() ); addDataTypeTest( - dataTypeTestBuilder("longtext", JsonSchemaPrimitive.STRING) + DataTypeTest.builder("longtext", JsonSchemaPrimitive.STRING) .addInsertValue("null") .build() ); addDataTypeTest( - dataTypeTestBuilder("text", JsonSchemaPrimitive.STRING) + DataTypeTest.builder("text", JsonSchemaPrimitive.STRING) .addInsertValue("null") .build() ); addDataTypeTest( - dataTypeTestBuilder("json", JsonSchemaPrimitive.STRING) + DataTypeTest.builder("json", JsonSchemaPrimitive.STRING) .addInsertValue("null", "'{\"a\" :10, \"b\": 15}'") .build() ); addDataTypeTest( - dataTypeTestBuilder("point", JsonSchemaPrimitive.OBJECT) + DataTypeTest.builder("point", JsonSchemaPrimitive.OBJECT) .addInsertValue("null", "(ST_GeomFromText('POINT(1 1)'))") .build() ); addDataTypeTest( - dataTypeTestBuilder("bool", JsonSchemaPrimitive.STRING) + DataTypeTest.builder("bool", JsonSchemaPrimitive.STRING) .addInsertValue("null", "127", "-128") .build() ); diff --git a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/OracleSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/OracleSourceAcceptanceTest.java index ab194ef7cbe08..56cddb3bfd500 100644 --- a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/OracleSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/OracleSourceAcceptanceTest.java @@ -32,6 +32,7 @@ import io.airbyte.db.Databases; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; +import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.protocol.models.CatalogHelpers; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.ConfiguredAirbyteStream; diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceAcceptanceTest.java index 0760f59053c1d..aed17d762046d 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceAcceptanceTest.java @@ -32,6 +32,7 @@ import io.airbyte.db.Database; import io.airbyte.db.Databases; import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; +import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.protocol.models.CatalogHelpers; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.ConfiguredAirbyteStream; diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceAcceptanceTest.java index edcaeee547e2f..fd14f4c36d7ff 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceAcceptanceTest.java @@ -32,6 +32,7 @@ import io.airbyte.db.Database; import io.airbyte.db.Databases; import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; +import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.protocol.models.CatalogHelpers; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.ConfiguredAirbyteStream; diff --git a/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftSourceAcceptanceTest.java index 5e2f4caad8f2e..ec7c9bdbf647f 100644 --- a/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftSourceAcceptanceTest.java @@ -33,6 +33,7 @@ import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.source.redshift.RedshiftSource; import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; +import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.protocol.models.CatalogHelpers; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.ConnectorSpecification; From 76414fbac6696abfe9596846c943fbec011fce2f Mon Sep 17 00:00:00 2001 From: Andrii Leonets Date: Tue, 1 Jun 2021 19:55:51 +0300 Subject: [PATCH 06/11] code autoformatting --- .../standardtest/source/DataTypeTest.java | 264 +++++---- .../source/SourceAbstractTest.java | 2 +- .../source/SourceComprehensiveTest.java | 2 +- .../source/TestDestinationEnv.java | 38 +- .../CdcMySqlSourceComprehensiveTest.java | 515 +++++++++--------- .../mysql/MySqlSourceComprehensiveTest.java | 257 ++++----- 6 files changed, 550 insertions(+), 528 deletions(-) diff --git a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/DataTypeTest.java b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/DataTypeTest.java index d173f52bf063f..c10a62b800476 100644 --- a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/DataTypeTest.java +++ b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/DataTypeTest.java @@ -1,151 +1,179 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + package io.airbyte.integrations.standardtest.source; import io.airbyte.protocol.models.ConfiguredAirbyteStream; import io.airbyte.protocol.models.Field; - import java.util.ArrayList; import java.util.Arrays; import java.util.List; public class DataTypeTest { - private static final String DEFAULT_CREATE_TABLE_SQL = "CREATE TABLE %1$s(id integer primary key, test_column %2$s);"; - private static final String DEFAULT_INSERT_SQL = "INSERT INTO %1$s VALUES (%2$s, %3$s);"; + private static final String DEFAULT_CREATE_TABLE_SQL = "CREATE TABLE %1$s(id integer primary key, test_column %2$s);"; + private static final String DEFAULT_INSERT_SQL = "INSERT INTO %1$s VALUES (%2$s, %3$s);"; + + private final String sourceType; + private final Field.JsonSchemaPrimitive airbyteType; + private final List values; + private final String createTablePatternSQL; + private final String insertPatternSQL; + private final String fullSourceDataType; + private long testNumber; + + DataTypeTest(String sourceType, + Field.JsonSchemaPrimitive airbyteType, + List values, + String createTablePatternSQL, + String insertPatternSQL, + String fullSourceDataType) { + this.sourceType = sourceType; + this.airbyteType = airbyteType; + this.values = values; + this.createTablePatternSQL = createTablePatternSQL; + this.insertPatternSQL = insertPatternSQL; + this.fullSourceDataType = fullSourceDataType; + } + + /** + * The builder allows to setup any comprehensive data type test. + * + * @param sourceType name of the source data type. Duplicates by name will be tested independently + * from each others. Note that this name will be used for connector setup and table creation. + * If source syntax requires more details (E.g. "varchar" type requires length + * "varchar(50)"), you can additionally set custom data type syntax by + * {@link DataTypeTestBuilder#fullSourceDataType(String)} method. + * @param airbyteType corresponding Airbyte data type. It requires for proper configuration + * {@link ConfiguredAirbyteStream} + * @return builder for setup comprehensive test + */ + public static DataTypeTestBuilder builder(String sourceType, Field.JsonSchemaPrimitive airbyteType) { + return new DataTypeTestBuilder(sourceType, airbyteType); + } + + public static class DataTypeTestBuilder { private final String sourceType; private final Field.JsonSchemaPrimitive airbyteType; - private final List values; - private final String createTablePatternSQL; - private final String insertPatternSQL; - private final String fullSourceDataType; - private long testNumber; - - DataTypeTest(String sourceType, Field.JsonSchemaPrimitive airbyteType, List values, String createTablePatternSQL, String insertPatternSQL, String fullSourceDataType) { - this.sourceType = sourceType; - this.airbyteType = airbyteType; - this.values = values; - this.createTablePatternSQL = createTablePatternSQL; - this.insertPatternSQL = insertPatternSQL; - this.fullSourceDataType = fullSourceDataType; + private final List values = new ArrayList<>(); + private String createTablePatternSQL; + private String insertPatternSQL; + private String fullSourceDataType; + + DataTypeTestBuilder(String sourceType, Field.JsonSchemaPrimitive airbyteType) { + this.sourceType = sourceType; + this.airbyteType = airbyteType; + this.createTablePatternSQL = DEFAULT_CREATE_TABLE_SQL; + this.insertPatternSQL = DEFAULT_INSERT_SQL; + this.fullSourceDataType = sourceType; } /** - * The builder allows to setup any comprehensive data type test. + * Set custom the create table script pattern. Use it if you source uses untypical table creation + * sql. Default patter described {@link #DEFAULT_CREATE_TABLE_SQL} Note! The patter should contains + * two String place holders for the table name and data type. * - * @param sourceType name of the source data type. Duplicates by name will be tested independently - * from each others. Note that this name will be used for connector setup and table creation. - * If source syntax requires more details (E.g. "varchar" type requires length - * "varchar(50)"), you can additionally set custom data type syntax by - * {@link DataTypeTestBuilder#fullSourceDataType(String)} method. - * @param airbyteType corresponding Airbyte data type. It requires for proper configuration - * {@link ConfiguredAirbyteStream} - * @return builder for setup comprehensive test + * @param createTablePatternSQL creation table sql pattern + * @return builder */ - public static DataTypeTestBuilder builder(String sourceType, Field.JsonSchemaPrimitive airbyteType) { - return new DataTypeTestBuilder(sourceType, airbyteType); + public DataTypeTestBuilder createTablePatternSQL(String createTablePatternSQL) { + this.createTablePatternSQL = createTablePatternSQL; + return this; } - public static class DataTypeTestBuilder { - - private final String sourceType; - private final Field.JsonSchemaPrimitive airbyteType; - private final List values = new ArrayList<>(); - private String createTablePatternSQL; - private String insertPatternSQL; - private String fullSourceDataType; - - DataTypeTestBuilder(String sourceType, Field.JsonSchemaPrimitive airbyteType) { - this.sourceType = sourceType; - this.airbyteType = airbyteType; - this.createTablePatternSQL = DEFAULT_CREATE_TABLE_SQL; - this.insertPatternSQL = DEFAULT_INSERT_SQL; - this.fullSourceDataType = sourceType; - } - - /** - * Set custom the create table script pattern. Use it if you source uses untypical table creation - * sql. Default patter described {@link #DEFAULT_CREATE_TABLE_SQL} Note! The patter should contains - * two String place holders for the table name and data type. - * - * @param createTablePatternSQL creation table sql pattern - * @return builder - */ - public DataTypeTestBuilder createTablePatternSQL(String createTablePatternSQL) { - this.createTablePatternSQL = createTablePatternSQL; - return this; - } - - /** - * Set custom the insert record script pattern. Use it if you source uses untypical insert record - * sql. Default patter described {@link #DEFAULT_INSERT_SQL} Note! The patter should contains two - * String place holders for the table name and value. - * - * @param insertPatternSQL creation table sql pattern - * @return builder - */ - public DataTypeTestBuilder insertPatternSQL(String insertPatternSQL) { - this.insertPatternSQL = insertPatternSQL; - return this; - } - - /** - * Allows to set extended data type for the table creation. E.g. The "varchar" type requires in - * MySQL requires length. In this case fullSourceDataType will be "varchar(50)". - * - * @param fullSourceDataType actual string for the column data type description - * @return builder - */ - public DataTypeTestBuilder fullSourceDataType(String fullSourceDataType) { - this.fullSourceDataType = fullSourceDataType; - return this; - } - - /** - * Adds value(s) to the scope of a corresponding test. The values will be inserted into the created - * table. Note! The value will be inserted into the insert script without any transformations. Make - * sure that the value is in line with the source syntax. - * - * @param insertValue test value - * @return builder - */ - public DataTypeTestBuilder addInsertValue(String... insertValue) { - this.values.addAll(Arrays.asList(insertValue)); - return this; - } - - public DataTypeTest build() { - return new DataTypeTest(sourceType, airbyteType, values, createTablePatternSQL, insertPatternSQL, fullSourceDataType); - } - + /** + * Set custom the insert record script pattern. Use it if you source uses untypical insert record + * sql. Default patter described {@link #DEFAULT_INSERT_SQL} Note! The patter should contains two + * String place holders for the table name and value. + * + * @param insertPatternSQL creation table sql pattern + * @return builder + */ + public DataTypeTestBuilder insertPatternSQL(String insertPatternSQL) { + this.insertPatternSQL = insertPatternSQL; + return this; } - public void setTestNumber(long testNumber) { - this.testNumber = testNumber; + /** + * Allows to set extended data type for the table creation. E.g. The "varchar" type requires in + * MySQL requires length. In this case fullSourceDataType will be "varchar(50)". + * + * @param fullSourceDataType actual string for the column data type description + * @return builder + */ + public DataTypeTestBuilder fullSourceDataType(String fullSourceDataType) { + this.fullSourceDataType = fullSourceDataType; + return this; } - public String getSourceType() { - return sourceType; + /** + * Adds value(s) to the scope of a corresponding test. The values will be inserted into the created + * table. Note! The value will be inserted into the insert script without any transformations. Make + * sure that the value is in line with the source syntax. + * + * @param insertValue test value + * @return builder + */ + public DataTypeTestBuilder addInsertValue(String... insertValue) { + this.values.addAll(Arrays.asList(insertValue)); + return this; } - public Field.JsonSchemaPrimitive getAirbyteType() { - return airbyteType; + public DataTypeTest build() { + return new DataTypeTest(sourceType, airbyteType, values, createTablePatternSQL, insertPatternSQL, fullSourceDataType); } - public String getName() { - return "test_" + testNumber + "_" + sourceType; - } + } - public String getCreateSQL() { - return String.format(createTablePatternSQL, getName(), fullSourceDataType); - } + public void setTestNumber(long testNumber) { + this.testNumber = testNumber; + } + + public String getSourceType() { + return sourceType; + } + + public Field.JsonSchemaPrimitive getAirbyteType() { + return airbyteType; + } + + public String getName() { + return "test_" + testNumber + "_" + sourceType; + } + + public String getCreateSQL() { + return String.format(createTablePatternSQL, getName(), fullSourceDataType); + } - public List getInsertSQLs() { - List insertSQLs = new ArrayList<>(); - int rowId = 1; - for (String value : values) { - insertSQLs.add(String.format(insertPatternSQL, getName(), rowId++, value)); - } - return insertSQLs; + public List getInsertSQLs() { + List insertSQLs = new ArrayList<>(); + int rowId = 1; + for (String value : values) { + insertSQLs.add(String.format(insertPatternSQL, getName(), rowId++, value)); } + return insertSQLs; + } } diff --git a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceAbstractTest.java b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceAbstractTest.java index fe75814b9e857..52fa25407267f 100644 --- a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceAbstractTest.java +++ b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceAbstractTest.java @@ -25,12 +25,12 @@ package io.airbyte.integrations.standardtest.source; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.config.JobGetSpecConfig; import io.airbyte.config.StandardCheckConnectionInput; import io.airbyte.config.StandardCheckConnectionOutput; import io.airbyte.config.StandardDiscoverCatalogInput; import io.airbyte.config.StandardTapConfig; import io.airbyte.config.State; -import io.airbyte.config.JobGetSpecConfig; import io.airbyte.protocol.models.AirbyteCatalog; import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; diff --git a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceComprehensiveTest.java b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceComprehensiveTest.java index 80f27e8fea127..6e75d04d25586 100644 --- a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceComprehensiveTest.java +++ b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceComprehensiveTest.java @@ -37,8 +37,8 @@ import io.airbyte.protocol.models.DestinationSyncMode; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.SyncMode; -import java.util.List; import java.util.ArrayList; +import java.util.List; import java.util.stream.Collectors; import org.junit.jupiter.api.Test; import org.slf4j.Logger; diff --git a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/TestDestinationEnv.java b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/TestDestinationEnv.java index cdf2cfbe40d10..69a2980a7ec66 100644 --- a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/TestDestinationEnv.java +++ b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/TestDestinationEnv.java @@ -1,17 +1,41 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + package io.airbyte.integrations.standardtest.source; import java.nio.file.Path; public class TestDestinationEnv { - private final Path localRoot; + private final Path localRoot; - public TestDestinationEnv(Path localRoot) { - this.localRoot = localRoot; - } + public TestDestinationEnv(Path localRoot) { + this.localRoot = localRoot; + } - public Path getLocalRoot() { - return localRoot; - } + public Path getLocalRoot() { + return localRoot; + } } diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceComprehensiveTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceComprehensiveTest.java index bb3eac67da544..ca04a8124b64a 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceComprehensiveTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceComprehensiveTest.java @@ -1,3 +1,27 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + package io.airbyte.integrations.source.mysql; import com.fasterxml.jackson.databind.JsonNode; @@ -14,265 +38,238 @@ public class CdcMySqlSourceComprehensiveTest extends SourceComprehensiveTest { - private MySQLContainer container; - private JsonNode config; - - @Override - protected JsonNode getConfig() { - return config; - } - - @Override - protected void tearDown(TestDestinationEnv testEnv) { - container.close(); - } - - @Override - protected String getImageName() { - return "airbyte/source-mysql:dev"; - } - - @Override - protected Database setupDatabase() throws Exception { - container = new MySQLContainer<>("mysql:8.0"); - container.start(); - - config = Jsons.jsonNode(ImmutableMap.builder() - .put("host", container.getHost()) - .put("port", container.getFirstMappedPort()) - .put("database", container.getDatabaseName()) - .put("username", container.getUsername()) - .put("password", container.getPassword()) - .put("replication_method", MySqlSource.ReplicationMethod.CDC) - .build()); - - final Database database = Databases.createDatabase( - config.get("username").asText(), - config.get("password").asText(), - String.format("jdbc:mysql://%s:%s/%s", - config.get("host").asText(), - config.get("port").asText(), - config.get("database").asText()), - "com.mysql.cj.jdbc.Driver", - SQLDialect.MYSQL); - - // It disable strict mode in the DB and allows to insert specific values. - // For example, it's possible to insert date with zero values "2021-00-00" - database.query(ctx -> ctx.fetch("SET @@sql_mode=''")); - - revokeAllPermissions(); - grantCorrectPermissions(); - - return database; + private MySQLContainer container; + private JsonNode config; + + @Override + protected JsonNode getConfig() { + return config; + } + + @Override + protected void tearDown(TestDestinationEnv testEnv) { + container.close(); + } + + @Override + protected String getImageName() { + return "airbyte/source-mysql:dev"; + } + + @Override + protected Database setupDatabase() throws Exception { + container = new MySQLContainer<>("mysql:8.0"); + container.start(); + + config = Jsons.jsonNode(ImmutableMap.builder() + .put("host", container.getHost()) + .put("port", container.getFirstMappedPort()) + .put("database", container.getDatabaseName()) + .put("username", container.getUsername()) + .put("password", container.getPassword()) + .put("replication_method", MySqlSource.ReplicationMethod.CDC) + .build()); + + final Database database = Databases.createDatabase( + config.get("username").asText(), + config.get("password").asText(), + String.format("jdbc:mysql://%s:%s/%s", + config.get("host").asText(), + config.get("port").asText(), + config.get("database").asText()), + "com.mysql.cj.jdbc.Driver", + SQLDialect.MYSQL); + + // It disable strict mode in the DB and allows to insert specific values. + // For example, it's possible to insert date with zero values "2021-00-00" + database.query(ctx -> ctx.fetch("SET @@sql_mode=''")); + + revokeAllPermissions(); + grantCorrectPermissions(); + + return database; + } + + private void revokeAllPermissions() { + executeQuery("REVOKE ALL PRIVILEGES, GRANT OPTION FROM " + container.getUsername() + "@'%';"); + } + + private void grantCorrectPermissions() { + executeQuery( + "GRANT SELECT, RELOAD, SHOW DATABASES, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO " + + container.getUsername() + "@'%';"); + } + + private void executeQuery(String query) { + try (Database database = Databases.createDatabase( + "root", + "test", + String.format("jdbc:mysql://%s:%s/%s", + container.getHost(), + container.getFirstMappedPort(), + container.getDatabaseName()), + MySqlSource.DRIVER_CLASS, + SQLDialect.MYSQL)) { + database.query( + ctx -> ctx + .execute(query)); + } catch (Exception e) { + throw new RuntimeException(e); } + } + + @Override + protected void initTests() { + addDataTypeTest( + DataTypeTest.builder("tinyint", Field.JsonSchemaPrimitive.NUMBER) + .addInsertValue("null") + .addInsertValue("-128", "127") + .build()); + + addDataTypeTest( + DataTypeTest.builder("smallint", Field.JsonSchemaPrimitive.NUMBER) + .addInsertValue("null") + .addInsertValue("-32768", "32767") + .build()); + + addDataTypeTest( + DataTypeTest.builder("smallint", Field.JsonSchemaPrimitive.NUMBER) + .fullSourceDataType("smallint zerofill") + .addInsertValue("1") + .build()); + + addDataTypeTest( + DataTypeTest.builder("mediumint", Field.JsonSchemaPrimitive.NUMBER) + .addInsertValue("null", "-8388608", "8388607") + .build()); + + addDataTypeTest( + DataTypeTest.builder("mediumint", Field.JsonSchemaPrimitive.NUMBER) + .fullSourceDataType("mediumint zerofill") + .addInsertValue("1") + .build()); + + addDataTypeTest( + DataTypeTest.builder("int", Field.JsonSchemaPrimitive.NUMBER) + .addInsertValue("null", "-2147483648", "2147483647") + .build()); + + addDataTypeTest( + DataTypeTest.builder("int", Field.JsonSchemaPrimitive.NUMBER) + .fullSourceDataType("int zerofill") + .addInsertValue("1") + .build()); + + addDataTypeTest( + DataTypeTest.builder("bigint", Field.JsonSchemaPrimitive.NUMBER) + .addInsertValue("null", "9223372036854775807") + .build()); + + addDataTypeTest( + DataTypeTest.builder("float", Field.JsonSchemaPrimitive.NUMBER) + .addInsertValue("null") + .build()); + + addDataTypeTest( + DataTypeTest.builder("double", Field.JsonSchemaPrimitive.NUMBER) + .addInsertValue("null", "power(10, 308)", "1/power(10, 45)") + .build()); + + addDataTypeTest( + DataTypeTest.builder("decimal", Field.JsonSchemaPrimitive.NUMBER) + .fullSourceDataType("decimal(5,2)") + .addInsertValue("null") + .build()); + + addDataTypeTest( + DataTypeTest.builder("bit", Field.JsonSchemaPrimitive.NUMBER) + .addInsertValue("null", "1", "0") + .build()); + + addDataTypeTest( + DataTypeTest.builder("date", Field.JsonSchemaPrimitive.STRING) + .addInsertValue("null", "'2021-01-00'", "'2021-00-00'", "'0000-00-00'") + .build()); + + addDataTypeTest( + DataTypeTest.builder("datetime", Field.JsonSchemaPrimitive.STRING) + .addInsertValue("null", "'0000-00-00 00:00:00'") + .build()); + + addDataTypeTest( + DataTypeTest.builder("timestamp", Field.JsonSchemaPrimitive.STRING) + .addInsertValue("null") + .build()); + + addDataTypeTest( + DataTypeTest.builder("time", Field.JsonSchemaPrimitive.STRING) + .addInsertValue("null", "'-838:59:59.000000'") + .build()); + + addDataTypeTest( + DataTypeTest.builder("varchar", Field.JsonSchemaPrimitive.STRING) + .fullSourceDataType("varchar(256) character set cp1251") + .addInsertValue("null", "'тест'") + .build()); + + addDataTypeTest( + DataTypeTest.builder("varchar", Field.JsonSchemaPrimitive.STRING) + .fullSourceDataType("varchar(256) character set utf16") + .addInsertValue("null", "0xfffd") + .build()); + + addDataTypeTest( + DataTypeTest.builder("varchar", Field.JsonSchemaPrimitive.STRING) + .fullSourceDataType("varchar(256)") + .addInsertValue("null", "'!\"#$%&\\'()*+,-./:;<=>?\\@[\\]^_\\`{|}~'") + .build()); + + addDataTypeTest( + DataTypeTest.builder("varbinary", Field.JsonSchemaPrimitive.STRING) + .fullSourceDataType("varbinary(256)") + .addInsertValue("null", "'test'") + .build()); + + addDataTypeTest( + DataTypeTest.builder("blob", Field.JsonSchemaPrimitive.STRING) + .addInsertValue("null", "'test'") + .build()); + + addDataTypeTest( + DataTypeTest.builder("mediumtext", Field.JsonSchemaPrimitive.STRING) + .addInsertValue("null", "lpad('0', 16777214, '0')") + .build()); + + addDataTypeTest( + DataTypeTest.builder("tinytext", Field.JsonSchemaPrimitive.STRING) + .addInsertValue("null") + .build()); + + addDataTypeTest( + DataTypeTest.builder("longtext", Field.JsonSchemaPrimitive.STRING) + .addInsertValue("null") + .build()); + + addDataTypeTest( + DataTypeTest.builder("text", Field.JsonSchemaPrimitive.STRING) + .addInsertValue("null") + .build()); + + addDataTypeTest( + DataTypeTest.builder("json", Field.JsonSchemaPrimitive.STRING) + .addInsertValue("null", "'{\"a\" :10, \"b\": 15}'") + .build()); + + addDataTypeTest( + DataTypeTest.builder("point", Field.JsonSchemaPrimitive.OBJECT) + .addInsertValue("null", "(ST_GeomFromText('POINT(1 1)'))") + .build()); + + addDataTypeTest( + DataTypeTest.builder("bool", Field.JsonSchemaPrimitive.STRING) + .addInsertValue("null", "1", "127", "-128") + .build()); + + } - private void revokeAllPermissions() { - executeQuery("REVOKE ALL PRIVILEGES, GRANT OPTION FROM " + container.getUsername() + "@'%';"); - } - - private void grantCorrectPermissions() { - executeQuery( - "GRANT SELECT, RELOAD, SHOW DATABASES, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO " - + container.getUsername() + "@'%';"); - } - - private void executeQuery(String query) { - try (Database database = Databases.createDatabase( - "root", - "test", - String.format("jdbc:mysql://%s:%s/%s", - container.getHost(), - container.getFirstMappedPort(), - container.getDatabaseName()), - MySqlSource.DRIVER_CLASS, - SQLDialect.MYSQL)) { - database.query( - ctx -> ctx - .execute(query)); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - @Override - protected void initTests() { - addDataTypeTest( - DataTypeTest.builder("tinyint", Field.JsonSchemaPrimitive.NUMBER) - .addInsertValue("null") - .addInsertValue("-128", "127") - .build() - ); - - addDataTypeTest( - DataTypeTest.builder("smallint", Field.JsonSchemaPrimitive.NUMBER) - .addInsertValue("null") - .addInsertValue("-32768", "32767") - .build() - ); - - addDataTypeTest( - DataTypeTest.builder("smallint", Field.JsonSchemaPrimitive.NUMBER) - .fullSourceDataType("smallint zerofill") - .addInsertValue("1") - .build() - ); - - addDataTypeTest( - DataTypeTest.builder("mediumint", Field.JsonSchemaPrimitive.NUMBER) - .addInsertValue("null", "-8388608", "8388607") - .build() - ); - - addDataTypeTest( - DataTypeTest.builder("mediumint", Field.JsonSchemaPrimitive.NUMBER) - .fullSourceDataType("mediumint zerofill") - .addInsertValue("1") - .build() - ); - - addDataTypeTest( - DataTypeTest.builder("int", Field.JsonSchemaPrimitive.NUMBER) - .addInsertValue("null", "-2147483648", "2147483647") - .build() - ); - - addDataTypeTest( - DataTypeTest.builder("int", Field.JsonSchemaPrimitive.NUMBER) - .fullSourceDataType("int zerofill") - .addInsertValue("1") - .build() - ); - - addDataTypeTest( - DataTypeTest.builder("bigint", Field.JsonSchemaPrimitive.NUMBER) - .addInsertValue("null", "9223372036854775807") - .build() - ); - - addDataTypeTest( - DataTypeTest.builder("float", Field.JsonSchemaPrimitive.NUMBER) - .addInsertValue("null") - .build() - ); - - addDataTypeTest( - DataTypeTest.builder("double", Field.JsonSchemaPrimitive.NUMBER) - .addInsertValue("null", "power(10, 308)", "1/power(10, 45)") - .build() - ); - - addDataTypeTest( - DataTypeTest.builder("decimal", Field.JsonSchemaPrimitive.NUMBER) - .fullSourceDataType("decimal(5,2)") - .addInsertValue("null") - .build() - ); - - addDataTypeTest( - DataTypeTest.builder("bit", Field.JsonSchemaPrimitive.NUMBER) - .addInsertValue("null", "1", "0") - .build() - ); - - addDataTypeTest( - DataTypeTest.builder("date", Field.JsonSchemaPrimitive.STRING) - .addInsertValue("null", "'2021-01-00'", "'2021-00-00'", "'0000-00-00'") - .build() - ); - - addDataTypeTest( - DataTypeTest.builder("datetime", Field.JsonSchemaPrimitive.STRING) - .addInsertValue("null", "'0000-00-00 00:00:00'") - .build() - ); - - addDataTypeTest( - DataTypeTest.builder("timestamp", Field.JsonSchemaPrimitive.STRING) - .addInsertValue("null") - .build() - ); - - addDataTypeTest( - DataTypeTest.builder("time", Field.JsonSchemaPrimitive.STRING) - .addInsertValue("null", "'-838:59:59.000000'") - .build() - ); - - addDataTypeTest( - DataTypeTest.builder("varchar", Field.JsonSchemaPrimitive.STRING) - .fullSourceDataType("varchar(256) character set cp1251") - .addInsertValue("null", "'тест'") - .build() - ); - - addDataTypeTest( - DataTypeTest.builder("varchar", Field.JsonSchemaPrimitive.STRING) - .fullSourceDataType("varchar(256) character set utf16") - .addInsertValue("null", "0xfffd") - .build() - ); - - addDataTypeTest( - DataTypeTest.builder("varchar", Field.JsonSchemaPrimitive.STRING) - .fullSourceDataType("varchar(256)") - .addInsertValue("null", "'!\"#$%&\\'()*+,-./:;<=>?\\@[\\]^_\\`{|}~'") - .build() - ); - - addDataTypeTest( - DataTypeTest.builder("varbinary", Field.JsonSchemaPrimitive.STRING) - .fullSourceDataType("varbinary(256)") - .addInsertValue("null", "'test'") - .build() - ); - - addDataTypeTest( - DataTypeTest.builder("blob", Field.JsonSchemaPrimitive.STRING) - .addInsertValue("null", "'test'") - .build() - ); - - addDataTypeTest( - DataTypeTest.builder("mediumtext", Field.JsonSchemaPrimitive.STRING) - .addInsertValue("null", "lpad('0', 16777214, '0')") - .build() - ); - - addDataTypeTest( - DataTypeTest.builder("tinytext", Field.JsonSchemaPrimitive.STRING) - .addInsertValue("null") - .build() - ); - - addDataTypeTest( - DataTypeTest.builder("longtext", Field.JsonSchemaPrimitive.STRING) - .addInsertValue("null") - .build() - ); - - addDataTypeTest( - DataTypeTest.builder("text", Field.JsonSchemaPrimitive.STRING) - .addInsertValue("null") - .build() - ); - - addDataTypeTest( - DataTypeTest.builder("json", Field.JsonSchemaPrimitive.STRING) - .addInsertValue("null", "'{\"a\" :10, \"b\": 15}'") - .build() - ); - - addDataTypeTest( - DataTypeTest.builder("point", Field.JsonSchemaPrimitive.OBJECT) - .addInsertValue("null", "(ST_GeomFromText('POINT(1 1)'))") - .build() - ); - - addDataTypeTest( - DataTypeTest.builder("bool", Field.JsonSchemaPrimitive.STRING) - .addInsertValue("null", "1", "127", "-128") - .build() - ); - - } } diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceComprehensiveTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceComprehensiveTest.java index dd639c7482a22..942207d5a9fb4 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceComprehensiveTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceComprehensiveTest.java @@ -62,22 +62,22 @@ protected Database setupDatabase() throws Exception { container.start(); config = Jsons.jsonNode(ImmutableMap.builder() - .put("host", container.getHost()) - .put("port", container.getFirstMappedPort()) - .put("database", container.getDatabaseName()) - .put("username", container.getUsername()) - .put("password", container.getPassword()) - .build()); + .put("host", container.getHost()) + .put("port", container.getFirstMappedPort()) + .put("database", container.getDatabaseName()) + .put("username", container.getUsername()) + .put("password", container.getPassword()) + .build()); final Database database = Databases.createDatabase( - config.get("username").asText(), - config.get("password").asText(), - String.format("jdbc:mysql://%s:%s/%s", - config.get("host").asText(), - config.get("port").asText(), - config.get("database").asText()), - "com.mysql.cj.jdbc.Driver", - SQLDialect.MYSQL); + config.get("username").asText(), + config.get("password").asText(), + String.format("jdbc:mysql://%s:%s/%s", + config.get("host").asText(), + config.get("port").asText(), + config.get("database").asText()), + "com.mysql.cj.jdbc.Driver", + SQLDialect.MYSQL); // It disable strict mode in the DB and allows to insert specific values. // For example, it's possible to insert date with zero values "2021-00-00" @@ -89,187 +89,160 @@ protected Database setupDatabase() throws Exception { @Override protected void initTests() { addDataTypeTest( - DataTypeTest.builder("tinyint", JsonSchemaPrimitive.NUMBER) - .addInsertValue("null") - .addInsertValue("-128", "127") - .build() - ); + DataTypeTest.builder("tinyint", JsonSchemaPrimitive.NUMBER) + .addInsertValue("null") + .addInsertValue("-128", "127") + .build()); addDataTypeTest( - DataTypeTest.builder("smallint", JsonSchemaPrimitive.NUMBER) - .addInsertValue("null") - .addInsertValue("-32768", "32767") - .build() - ); + DataTypeTest.builder("smallint", JsonSchemaPrimitive.NUMBER) + .addInsertValue("null") + .addInsertValue("-32768", "32767") + .build()); addDataTypeTest( - DataTypeTest.builder("smallint", JsonSchemaPrimitive.NUMBER) - .fullSourceDataType("smallint zerofill") - .addInsertValue("1") - .build() - ); + DataTypeTest.builder("smallint", JsonSchemaPrimitive.NUMBER) + .fullSourceDataType("smallint zerofill") + .addInsertValue("1") + .build()); addDataTypeTest( - DataTypeTest.builder("mediumint", JsonSchemaPrimitive.NUMBER) - .addInsertValue("null", "-8388608", "8388607") - .build() - ); + DataTypeTest.builder("mediumint", JsonSchemaPrimitive.NUMBER) + .addInsertValue("null", "-8388608", "8388607") + .build()); addDataTypeTest( - DataTypeTest.builder("mediumint", JsonSchemaPrimitive.NUMBER) - .fullSourceDataType("mediumint zerofill") - .addInsertValue("1") - .build() - ); + DataTypeTest.builder("mediumint", JsonSchemaPrimitive.NUMBER) + .fullSourceDataType("mediumint zerofill") + .addInsertValue("1") + .build()); addDataTypeTest( - DataTypeTest.builder("int", JsonSchemaPrimitive.NUMBER) - .addInsertValue("null", "-2147483648", "2147483647") - .build() - ); + DataTypeTest.builder("int", JsonSchemaPrimitive.NUMBER) + .addInsertValue("null", "-2147483648", "2147483647") + .build()); addDataTypeTest( - DataTypeTest.builder("int", JsonSchemaPrimitive.NUMBER) - .fullSourceDataType("int zerofill") - .addInsertValue("1") - .build() - ); + DataTypeTest.builder("int", JsonSchemaPrimitive.NUMBER) + .fullSourceDataType("int zerofill") + .addInsertValue("1") + .build()); addDataTypeTest( - DataTypeTest.builder("bigint", JsonSchemaPrimitive.NUMBER) - .addInsertValue("null", "9223372036854775807") - .build() - ); + DataTypeTest.builder("bigint", JsonSchemaPrimitive.NUMBER) + .addInsertValue("null", "9223372036854775807") + .build()); addDataTypeTest( - DataTypeTest.builder("float", JsonSchemaPrimitive.NUMBER) - .addInsertValue("null") - .build() - ); + DataTypeTest.builder("float", JsonSchemaPrimitive.NUMBER) + .addInsertValue("null") + .build()); addDataTypeTest( - DataTypeTest.builder("double", JsonSchemaPrimitive.NUMBER) - .addInsertValue("null", "power(10, 308)", "1/power(10, 45)") - .build() - ); + DataTypeTest.builder("double", JsonSchemaPrimitive.NUMBER) + .addInsertValue("null", "power(10, 308)", "1/power(10, 45)") + .build()); addDataTypeTest( - DataTypeTest.builder("decimal", JsonSchemaPrimitive.NUMBER) - .fullSourceDataType("decimal(5,2)") - .addInsertValue("null") - .build() - ); + DataTypeTest.builder("decimal", JsonSchemaPrimitive.NUMBER) + .fullSourceDataType("decimal(5,2)") + .addInsertValue("null") + .build()); addDataTypeTest( - DataTypeTest.builder("bit", JsonSchemaPrimitive.NUMBER) - .addInsertValue("null", "1", "0") - .build() - ); + DataTypeTest.builder("bit", JsonSchemaPrimitive.NUMBER) + .addInsertValue("null", "1", "0") + .build()); addDataTypeTest( - DataTypeTest.builder("date", JsonSchemaPrimitive.STRING) - .addInsertValue("null") -// .addInsertValue("'2021-01-00'") -// .addInsertValue("'2021-00-00'") -// .addInsertValue("'0000-00-00'") - .build() - ); + DataTypeTest.builder("date", JsonSchemaPrimitive.STRING) + .addInsertValue("null") + // .addInsertValue("'2021-01-00'") + // .addInsertValue("'2021-00-00'") + // .addInsertValue("'0000-00-00'") + .build()); addDataTypeTest( - DataTypeTest.builder("datetime", JsonSchemaPrimitive.STRING) - .addInsertValue("null") -// .addInsertValue("'0000-00-00 00:00:00'") - .build() - ); + DataTypeTest.builder("datetime", JsonSchemaPrimitive.STRING) + .addInsertValue("null") + // .addInsertValue("'0000-00-00 00:00:00'") + .build()); addDataTypeTest( - DataTypeTest.builder("timestamp", JsonSchemaPrimitive.STRING) - .addInsertValue("null") - .build() - ); + DataTypeTest.builder("timestamp", JsonSchemaPrimitive.STRING) + .addInsertValue("null") + .build()); addDataTypeTest( - DataTypeTest.builder("time", JsonSchemaPrimitive.STRING) - .addInsertValue("null") -// .addInsertValue("'-838:59:59.000000'") - .build() - ); + DataTypeTest.builder("time", JsonSchemaPrimitive.STRING) + .addInsertValue("null") + // .addInsertValue("'-838:59:59.000000'") + .build()); addDataTypeTest( - DataTypeTest.builder("varchar", JsonSchemaPrimitive.STRING) - .fullSourceDataType("varchar(256) character set cp1251") - .addInsertValue("null", "'тест'") - .build() - ); + DataTypeTest.builder("varchar", JsonSchemaPrimitive.STRING) + .fullSourceDataType("varchar(256) character set cp1251") + .addInsertValue("null", "'тест'") + .build()); addDataTypeTest( - DataTypeTest.builder("varchar", JsonSchemaPrimitive.STRING) - .fullSourceDataType("varchar(256) character set utf16") - .addInsertValue("null", "0xfffd") - .build() - ); + DataTypeTest.builder("varchar", JsonSchemaPrimitive.STRING) + .fullSourceDataType("varchar(256) character set utf16") + .addInsertValue("null", "0xfffd") + .build()); addDataTypeTest( - DataTypeTest.builder("varchar", JsonSchemaPrimitive.STRING) - .fullSourceDataType("varchar(256)") - .addInsertValue("null", "'!\"#$%&\\'()*+,-./:;<=>?\\@[\\]^_\\`{|}~'") - .build() - ); + DataTypeTest.builder("varchar", JsonSchemaPrimitive.STRING) + .fullSourceDataType("varchar(256)") + .addInsertValue("null", "'!\"#$%&\\'()*+,-./:;<=>?\\@[\\]^_\\`{|}~'") + .build()); addDataTypeTest( - DataTypeTest.builder("varbinary", JsonSchemaPrimitive.STRING) - .fullSourceDataType("varbinary(256)") - .addInsertValue("null", "'test'") - .build() - ); + DataTypeTest.builder("varbinary", JsonSchemaPrimitive.STRING) + .fullSourceDataType("varbinary(256)") + .addInsertValue("null", "'test'") + .build()); addDataTypeTest( - DataTypeTest.builder("blob", JsonSchemaPrimitive.STRING) - .addInsertValue("null", "'test'") - .build() - ); + DataTypeTest.builder("blob", JsonSchemaPrimitive.STRING) + .addInsertValue("null", "'test'") + .build()); addDataTypeTest( - DataTypeTest.builder("mediumtext", JsonSchemaPrimitive.STRING) - .addInsertValue("null", "lpad('0', 16777214, '0')") - .build() - ); + DataTypeTest.builder("mediumtext", JsonSchemaPrimitive.STRING) + .addInsertValue("null", "lpad('0', 16777214, '0')") + .build()); addDataTypeTest( - DataTypeTest.builder("tinytext", JsonSchemaPrimitive.STRING) - .addInsertValue("null") - .build() - ); + DataTypeTest.builder("tinytext", JsonSchemaPrimitive.STRING) + .addInsertValue("null") + .build()); addDataTypeTest( - DataTypeTest.builder("longtext", JsonSchemaPrimitive.STRING) - .addInsertValue("null") - .build() - ); + DataTypeTest.builder("longtext", JsonSchemaPrimitive.STRING) + .addInsertValue("null") + .build()); addDataTypeTest( - DataTypeTest.builder("text", JsonSchemaPrimitive.STRING) - .addInsertValue("null") - .build() - ); + DataTypeTest.builder("text", JsonSchemaPrimitive.STRING) + .addInsertValue("null") + .build()); addDataTypeTest( - DataTypeTest.builder("json", JsonSchemaPrimitive.STRING) - .addInsertValue("null", "'{\"a\" :10, \"b\": 15}'") - .build() - ); + DataTypeTest.builder("json", JsonSchemaPrimitive.STRING) + .addInsertValue("null", "'{\"a\" :10, \"b\": 15}'") + .build()); addDataTypeTest( - DataTypeTest.builder("point", JsonSchemaPrimitive.OBJECT) - .addInsertValue("null", "(ST_GeomFromText('POINT(1 1)'))") - .build() - ); + DataTypeTest.builder("point", JsonSchemaPrimitive.OBJECT) + .addInsertValue("null", "(ST_GeomFromText('POINT(1 1)'))") + .build()); addDataTypeTest( - DataTypeTest.builder("bool", JsonSchemaPrimitive.STRING) - .addInsertValue("null", "127", "-128") - .build() - ); + DataTypeTest.builder("bool", JsonSchemaPrimitive.STRING) + .addInsertValue("null", "127", "-128") + .build()); } -} \ No newline at end of file + +} From aaabdfddb95b26245c67f0019ca5bd28827819bd Mon Sep 17 00:00:00 2001 From: Andrii Leonets Date: Wed, 2 Jun 2021 15:21:21 +0300 Subject: [PATCH 07/11] [internal review] rename some methods/variables for clarity. + move up `JsonSchemaPrimitive` enum from inner class level --- .../io/airbyte/config/DataTypeEnumTest.java | 2 +- .../java/io/airbyte/db/jdbc/JdbcUtils.java | 2 +- .../io/airbyte/db/jdbc/TestJdbcUtils.java | 2 +- .../BufferedStreamConsumerTest.java | 2 +- .../source/fs/ExecutableTestSource.java | 2 +- .../source/PythonSourceAcceptanceTest.java | 2 +- .../source/SourceAbstractTest.java | 12 +- .../source/SourceComprehensiveTest.java | 30 +-- ...DataTypeTest.java => TestDataWrapper.java} | 74 ++++---- .../bigquery/BigQueryDestinationTest.java | 2 +- .../destination/csv/CsvDestinationTest.java | 2 +- .../local_json/LocalJsonDestinationTest.java | 2 +- .../destination/PostgresDestinationTest.java | 2 +- .../ClickHouseSourceAcceptanceTest.java | 4 +- .../e2e_test/ExceptionAfterNSource.java | 2 +- .../source/e2e_test/InfiniteFeedSource.java | 2 +- .../source/jdbc/AbstractJdbcSource.java | 2 +- .../source/jdbc/IncrementalUtils.java | 2 +- .../source/jdbc/StateDecoratingIterator.java | 2 +- .../jdbc/JdbcSourceSourceAcceptanceTest.java | 4 +- .../source/jdbc/IncrementalUtilsTest.java | 2 +- .../jdbc/StateDecoratingIteratorTest.java | 2 +- .../jdbc/test/JdbcSourceAcceptanceTest.java | 2 +- .../source/jdbc/test/JdbcStressTest.java | 2 +- .../mssql/MssqlSourceAcceptanceTest.java | 4 +- .../source/mssql/MssqlSourceTest.java | 2 +- .../mysql/CdcMySqlSourceAcceptanceTest.java | 4 +- .../CdcMySqlSourceComprehensiveTest.java | 176 +++++++++--------- .../mysql/MySqlSourceAcceptanceTest.java | 4 +- .../mysql/MySqlSourceComprehensiveTest.java | 176 +++++++++--------- .../source/mysql/CdcMySqlSourceTest.java | 2 +- .../sources/OracleSourceAcceptanceTest.java | 4 +- .../source/oracle/OracleSourceTest.java | 2 +- .../CdcPostgresSourceAcceptanceTest.java | 4 +- .../sources/PostgresSourceAcceptanceTest.java | 4 +- .../postgres/CdcPostgresSourceTest.java | 2 +- .../postgres/PostgresSourceSSLTest.java | 2 +- .../source/postgres/PostgresSourceTest.java | 2 +- .../sources/RedshiftSourceAcceptanceTest.java | 9 +- .../io/airbyte/protocol/models/Field.java | 9 - .../protocol/models/JsonSchemaPrimitive.java | 34 ++++ .../protocol/models/CatalogHelpersTest.java | 1 - .../scheduler/app/JobSchedulerTest.java | 2 +- .../persistence/DefaultJobCreatorTest.java | 2 +- .../converters/ConfigurationUpdateTest.java | 2 +- .../server/handlers/SchedulerHandlerTest.java | 2 +- .../WebBackendConnectionsHandlerTest.java | 2 +- .../server/helpers/ConnectionHelpers.java | 2 +- .../DefaultDiscoverCatalogWorkerTest.java | 2 +- .../io/airbyte/workers/TestConfigHelpers.java | 2 +- .../airbyte/DefaultAirbyteSourceTest.java | 2 +- .../airbyte/NamespacingMapperTest.java | 2 +- 52 files changed, 324 insertions(+), 299 deletions(-) rename airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/{DataTypeTest.java => TestDataWrapper.java} (70%) create mode 100644 airbyte-protocol/models/src/main/java/io/airbyte/protocol/models/JsonSchemaPrimitive.java diff --git a/airbyte-config/models/src/test/java/io/airbyte/config/DataTypeEnumTest.java b/airbyte-config/models/src/test/java/io/airbyte/config/DataTypeEnumTest.java index 4ad40ef9391f9..de6eedf24fd80 100644 --- a/airbyte-config/models/src/test/java/io/airbyte/config/DataTypeEnumTest.java +++ b/airbyte-config/models/src/test/java/io/airbyte/config/DataTypeEnumTest.java @@ -27,7 +27,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; -import io.airbyte.protocol.models.Field.JsonSchemaPrimitive; +import io.airbyte.protocol.models.JsonSchemaPrimitive; import org.junit.jupiter.api.Test; public class DataTypeEnumTest { diff --git a/airbyte-db/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java b/airbyte-db/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java index ac6260b9add1d..fec8460395c13 100644 --- a/airbyte-db/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java +++ b/airbyte-db/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java @@ -28,7 +28,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import io.airbyte.commons.functional.CheckedFunction; import io.airbyte.commons.json.Jsons; -import io.airbyte.protocol.models.Field.JsonSchemaPrimitive; +import io.airbyte.protocol.models.JsonSchemaPrimitive; import java.math.BigDecimal; import java.sql.Connection; import java.sql.Date; diff --git a/airbyte-db/src/test/java/io/airbyte/db/jdbc/TestJdbcUtils.java b/airbyte-db/src/test/java/io/airbyte/db/jdbc/TestJdbcUtils.java index 69ae296d96f92..046c75e1632d3 100644 --- a/airbyte-db/src/test/java/io/airbyte/db/jdbc/TestJdbcUtils.java +++ b/airbyte-db/src/test/java/io/airbyte/db/jdbc/TestJdbcUtils.java @@ -34,7 +34,7 @@ import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.stream.MoreStreams; -import io.airbyte.protocol.models.Field.JsonSchemaPrimitive; +import io.airbyte.protocol.models.JsonSchemaPrimitive; import io.airbyte.test.utils.PostgreSQLContainerHelper; import java.math.BigDecimal; import java.sql.Connection; diff --git a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination/buffered_stream_consumer/BufferedStreamConsumerTest.java b/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination/buffered_stream_consumer/BufferedStreamConsumerTest.java index 24397cf5f2844..5857c36e4d9f7 100644 --- a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination/buffered_stream_consumer/BufferedStreamConsumerTest.java +++ b/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination/buffered_stream_consumer/BufferedStreamConsumerTest.java @@ -43,7 +43,7 @@ import io.airbyte.protocol.models.CatalogHelpers; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.Field; -import io.airbyte.protocol.models.Field.JsonSchemaPrimitive; +import io.airbyte.protocol.models.JsonSchemaPrimitive; import java.time.Instant; import java.util.Collection; import java.util.List; diff --git a/airbyte-integrations/bases/base-standard-source-test-file/src/main/java/io/airbyte/integrations/standardtest/source/fs/ExecutableTestSource.java b/airbyte-integrations/bases/base-standard-source-test-file/src/main/java/io/airbyte/integrations/standardtest/source/fs/ExecutableTestSource.java index 5cbe73875c453..167d637ee1dec 100644 --- a/airbyte-integrations/bases/base-standard-source-test-file/src/main/java/io/airbyte/integrations/standardtest/source/fs/ExecutableTestSource.java +++ b/airbyte-integrations/bases/base-standard-source-test-file/src/main/java/io/airbyte/integrations/standardtest/source/fs/ExecutableTestSource.java @@ -120,7 +120,7 @@ protected List getRegexTests() throws Exception { } @Override - protected void setup(TestDestinationEnv testEnv) throws Exception { + protected void setupEnvironment(TestDestinationEnv environment) throws Exception { // no-op, for now } diff --git a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/PythonSourceAcceptanceTest.java b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/PythonSourceAcceptanceTest.java index 5b5238f2c920d..5f3063659f885 100644 --- a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/PythonSourceAcceptanceTest.java +++ b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/PythonSourceAcceptanceTest.java @@ -90,7 +90,7 @@ protected List getRegexTests() throws IOException { } @Override - protected void setup(TestDestinationEnv testEnv) throws Exception { + protected void setupEnvironment(TestDestinationEnv environment) throws Exception { testRoot = Files.createTempDirectory(Files.createDirectories(Path.of("/tmp/standard_test")), "pytest"); runExecutableVoid(Command.SETUP); } diff --git a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceAbstractTest.java b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceAbstractTest.java index 52fa25407267f..0d0ab509803b3 100644 --- a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceAbstractTest.java +++ b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceAbstractTest.java @@ -53,7 +53,7 @@ public abstract class SourceAbstractTest { - private TestDestinationEnv testEnv; + private TestDestinationEnv environment; private Path jobRoot; protected Path localRoot; private ProcessFactory processFactory; @@ -80,10 +80,10 @@ public abstract class SourceAbstractTest { * Function that performs any setup of external resources required for the test. e.g. instantiate a * postgres database. This function will be called before EACH test. * - * @param testEnv - information about the test environment. + * @param environment - information about the test environment. * @throws Exception - can throw any exception, test framework will handle. */ - protected abstract void setup(TestDestinationEnv testEnv) throws Exception; + protected abstract void setupEnvironment(TestDestinationEnv environment) throws Exception; /** * Function that performs any clean up of external resources required for the test. e.g. delete a @@ -102,9 +102,9 @@ public void setUpInternal() throws Exception { final Path workspaceRoot = Files.createTempDirectory(testDir, "test"); jobRoot = Files.createDirectories(Path.of(workspaceRoot.toString(), "job")); localRoot = Files.createTempDirectory(testDir, "output"); - testEnv = new TestDestinationEnv(localRoot); + environment = new TestDestinationEnv(localRoot); - setup(testEnv); + setupEnvironment(environment); processFactory = new DockerProcessFactory( workspaceRoot, @@ -115,7 +115,7 @@ public void setUpInternal() throws Exception { @AfterEach public void tearDownInternal() throws Exception { - tearDown(testEnv); + tearDown(environment); } protected ConnectorSpecification runSpec() throws WorkerException { diff --git a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceComprehensiveTest.java b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceComprehensiveTest.java index 6e75d04d25586..fc21e1d81cf18 100644 --- a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceComprehensiveTest.java +++ b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceComprehensiveTest.java @@ -36,6 +36,7 @@ import io.airbyte.protocol.models.ConfiguredAirbyteStream; import io.airbyte.protocol.models.DestinationSyncMode; import io.airbyte.protocol.models.Field; +import io.airbyte.protocol.models.JsonSchemaPrimitive; import io.airbyte.protocol.models.SyncMode; import java.util.ArrayList; import java.util.List; @@ -48,7 +49,7 @@ public abstract class SourceComprehensiveTest extends SourceAbstractTest { private static final Logger LOGGER = LoggerFactory.getLogger(SourceComprehensiveTest.class); - private final List dataTypeTests = new ArrayList<>(); + private final List testDataWrappers = new ArrayList<>(); /** * Setup the test database. All tables and data described in the registered tests will be put there. @@ -59,18 +60,17 @@ public abstract class SourceComprehensiveTest extends SourceAbstractTest { protected abstract Database setupDatabase() throws Exception; /** - * Put all required tests here using method {@link #addDataTypeTest(DataTypeTest)} + * Put all required tests here using method {@link #addDataTypeTestData(TestDataWrapper)} */ protected abstract void initTests(); @Override - protected void setup(TestDestinationEnv testEnv) throws Exception { + protected void setupEnvironment(TestDestinationEnv environment) throws Exception { setupDatabaseInternal(); } /** - * Configuring all streams in the input catalog to full refresh mode, verifies that a read operation - * produces some RECORD messages. + * The test checks that connector can fetch prepared data without failure. */ @Test public void testDataTypes() throws Exception { @@ -93,11 +93,11 @@ private void setupDatabaseInternal() throws Exception { initTests(); - for (DataTypeTest test : dataTypeTests) { + for (TestDataWrapper test : testDataWrappers) { database.query(ctx -> { - ctx.fetch(test.getCreateSQL()); - LOGGER.debug("Table " + test.getName() + " is created."); - test.getInsertSQLs().forEach(ctx::fetch); + ctx.fetch(test.getCreateSqlQuery()); + LOGGER.debug("Table " + test.getNameWithTestPrefix() + " is created."); + test.getInsertSqlQueries().forEach(ctx::fetch); return null; }); } @@ -114,16 +114,16 @@ private ConfiguredAirbyteCatalog getConfiguredCatalog() throws Exception { final JsonNode config = getConfig(); return new ConfiguredAirbyteCatalog().withStreams( - dataTypeTests + testDataWrappers .stream() .map(test -> new ConfiguredAirbyteStream() .withSyncMode(SyncMode.INCREMENTAL) .withCursorField(Lists.newArrayList("id")) .withDestinationSyncMode(DestinationSyncMode.APPEND) .withStream(CatalogHelpers.createAirbyteStream( - String.format("%s", test.getName()), + String.format("%s", test.getNameWithTestPrefix()), String.format("%s", config.get("database").asText()), - Field.of("id", Field.JsonSchemaPrimitive.NUMBER), + Field.of("id", JsonSchemaPrimitive.NUMBER), Field.of("test_column", test.getAirbyteType())) .withSourceDefinedCursor(true) .withSourceDefinedPrimaryKey(List.of(List.of("id"))) @@ -139,9 +139,9 @@ private ConfiguredAirbyteCatalog getConfiguredCatalog() throws Exception { * * @param test comprehensive data type test */ - public void addDataTypeTest(DataTypeTest test) { - dataTypeTests.add(test); - test.setTestNumber(dataTypeTests.stream().filter(t -> t.getSourceType().equals(test.getSourceType())).count()); + public void addDataTypeTestData(TestDataWrapper test) { + testDataWrappers.add(test); + test.setTestNumber(testDataWrappers.stream().filter(t -> t.getSourceType().equals(test.getSourceType())).count()); } } diff --git a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/DataTypeTest.java b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/TestDataWrapper.java similarity index 70% rename from airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/DataTypeTest.java rename to airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/TestDataWrapper.java index c10a62b800476..aeae0a8efa032 100644 --- a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/DataTypeTest.java +++ b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/TestDataWrapper.java @@ -25,35 +25,35 @@ package io.airbyte.integrations.standardtest.source; import io.airbyte.protocol.models.ConfiguredAirbyteStream; -import io.airbyte.protocol.models.Field; +import io.airbyte.protocol.models.JsonSchemaPrimitive; import java.util.ArrayList; import java.util.Arrays; import java.util.List; -public class DataTypeTest { +public class TestDataWrapper { private static final String DEFAULT_CREATE_TABLE_SQL = "CREATE TABLE %1$s(id integer primary key, test_column %2$s);"; private static final String DEFAULT_INSERT_SQL = "INSERT INTO %1$s VALUES (%2$s, %3$s);"; private final String sourceType; - private final Field.JsonSchemaPrimitive airbyteType; + private final JsonSchemaPrimitive airbyteType; private final List values; - private final String createTablePatternSQL; - private final String insertPatternSQL; + private final String createTablePatternSql; + private final String insertPatternSql; private final String fullSourceDataType; private long testNumber; - DataTypeTest(String sourceType, - Field.JsonSchemaPrimitive airbyteType, - List values, - String createTablePatternSQL, - String insertPatternSQL, - String fullSourceDataType) { + TestDataWrapper(String sourceType, + JsonSchemaPrimitive airbyteType, + List values, + String createTablePatternSql, + String insertPatternSql, + String fullSourceDataType) { this.sourceType = sourceType; this.airbyteType = airbyteType; this.values = values; - this.createTablePatternSQL = createTablePatternSQL; - this.insertPatternSQL = insertPatternSQL; + this.createTablePatternSql = createTablePatternSql; + this.insertPatternSql = insertPatternSql; this.fullSourceDataType = fullSourceDataType; } @@ -69,24 +69,24 @@ public class DataTypeTest { * {@link ConfiguredAirbyteStream} * @return builder for setup comprehensive test */ - public static DataTypeTestBuilder builder(String sourceType, Field.JsonSchemaPrimitive airbyteType) { + public static DataTypeTestBuilder builder(String sourceType, JsonSchemaPrimitive airbyteType) { return new DataTypeTestBuilder(sourceType, airbyteType); } public static class DataTypeTestBuilder { private final String sourceType; - private final Field.JsonSchemaPrimitive airbyteType; + private final JsonSchemaPrimitive airbyteType; private final List values = new ArrayList<>(); - private String createTablePatternSQL; - private String insertPatternSQL; + private String createTablePatternSql; + private String insertPatternSql; private String fullSourceDataType; - DataTypeTestBuilder(String sourceType, Field.JsonSchemaPrimitive airbyteType) { + DataTypeTestBuilder(String sourceType, JsonSchemaPrimitive airbyteType) { this.sourceType = sourceType; this.airbyteType = airbyteType; - this.createTablePatternSQL = DEFAULT_CREATE_TABLE_SQL; - this.insertPatternSQL = DEFAULT_INSERT_SQL; + this.createTablePatternSql = DEFAULT_CREATE_TABLE_SQL; + this.insertPatternSql = DEFAULT_INSERT_SQL; this.fullSourceDataType = sourceType; } @@ -95,11 +95,11 @@ public static class DataTypeTestBuilder { * sql. Default patter described {@link #DEFAULT_CREATE_TABLE_SQL} Note! The patter should contains * two String place holders for the table name and data type. * - * @param createTablePatternSQL creation table sql pattern + * @param createTablePatternSql creation table sql pattern * @return builder */ - public DataTypeTestBuilder createTablePatternSQL(String createTablePatternSQL) { - this.createTablePatternSQL = createTablePatternSQL; + public DataTypeTestBuilder createTablePatternSql(String createTablePatternSql) { + this.createTablePatternSql = createTablePatternSql; return this; } @@ -108,11 +108,11 @@ public DataTypeTestBuilder createTablePatternSQL(String createTablePatternSQL) { * sql. Default patter described {@link #DEFAULT_INSERT_SQL} Note! The patter should contains two * String place holders for the table name and value. * - * @param insertPatternSQL creation table sql pattern + * @param insertPatternSql creation table sql pattern * @return builder */ - public DataTypeTestBuilder insertPatternSQL(String insertPatternSQL) { - this.insertPatternSQL = insertPatternSQL; + public DataTypeTestBuilder insertPatternSql(String insertPatternSql) { + this.insertPatternSql = insertPatternSql; return this; } @@ -136,13 +136,13 @@ public DataTypeTestBuilder fullSourceDataType(String fullSourceDataType) { * @param insertValue test value * @return builder */ - public DataTypeTestBuilder addInsertValue(String... insertValue) { + public DataTypeTestBuilder addInsertValues(String... insertValue) { this.values.addAll(Arrays.asList(insertValue)); return this; } - public DataTypeTest build() { - return new DataTypeTest(sourceType, airbyteType, values, createTablePatternSQL, insertPatternSQL, fullSourceDataType); + public TestDataWrapper build() { + return new TestDataWrapper(sourceType, airbyteType, values, createTablePatternSql, insertPatternSql, fullSourceDataType); } } @@ -155,25 +155,25 @@ public String getSourceType() { return sourceType; } - public Field.JsonSchemaPrimitive getAirbyteType() { + public JsonSchemaPrimitive getAirbyteType() { return airbyteType; } - public String getName() { + public String getNameWithTestPrefix() { return "test_" + testNumber + "_" + sourceType; } - public String getCreateSQL() { - return String.format(createTablePatternSQL, getName(), fullSourceDataType); + public String getCreateSqlQuery() { + return String.format(createTablePatternSql, getNameWithTestPrefix(), fullSourceDataType); } - public List getInsertSQLs() { - List insertSQLs = new ArrayList<>(); + public List getInsertSqlQueries() { + List insertSqls = new ArrayList<>(); int rowId = 1; for (String value : values) { - insertSQLs.add(String.format(insertPatternSQL, getName(), rowId++, value)); + insertSqls.add(String.format(insertPatternSql, getNameWithTestPrefix(), rowId++, value)); } - return insertSQLs; + return insertSqls; } } diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDestinationTest.java b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDestinationTest.java index a374bc809c05a..36e77dcfe99a2 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDestinationTest.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDestinationTest.java @@ -59,7 +59,7 @@ import io.airbyte.protocol.models.ConfiguredAirbyteStream; import io.airbyte.protocol.models.ConnectorSpecification; import io.airbyte.protocol.models.Field; -import io.airbyte.protocol.models.Field.JsonSchemaPrimitive; +import io.airbyte.protocol.models.JsonSchemaPrimitive; import java.io.ByteArrayInputStream; import java.io.IOException; import java.nio.file.Files; diff --git a/airbyte-integrations/connectors/destination-csv/src/test/java/io/airbyte/integrations/destination/csv/CsvDestinationTest.java b/airbyte-integrations/connectors/destination-csv/src/test/java/io/airbyte/integrations/destination/csv/CsvDestinationTest.java index 47cb7aefa9a71..a5f821d59fd2f 100644 --- a/airbyte-integrations/connectors/destination-csv/src/test/java/io/airbyte/integrations/destination/csv/CsvDestinationTest.java +++ b/airbyte-integrations/connectors/destination-csv/src/test/java/io/airbyte/integrations/destination/csv/CsvDestinationTest.java @@ -52,7 +52,7 @@ import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.ConnectorSpecification; import io.airbyte.protocol.models.Field; -import io.airbyte.protocol.models.Field.JsonSchemaPrimitive; +import io.airbyte.protocol.models.JsonSchemaPrimitive; import java.io.FileReader; import java.io.IOException; import java.io.Reader; diff --git a/airbyte-integrations/connectors/destination-local-json/src/test/java/io/airbyte/integrations/destination/local_json/LocalJsonDestinationTest.java b/airbyte-integrations/connectors/destination-local-json/src/test/java/io/airbyte/integrations/destination/local_json/LocalJsonDestinationTest.java index d5397ffa7dbba..6da1896e15ebe 100644 --- a/airbyte-integrations/connectors/destination-local-json/src/test/java/io/airbyte/integrations/destination/local_json/LocalJsonDestinationTest.java +++ b/airbyte-integrations/connectors/destination-local-json/src/test/java/io/airbyte/integrations/destination/local_json/LocalJsonDestinationTest.java @@ -52,7 +52,7 @@ import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.ConnectorSpecification; import io.airbyte.protocol.models.Field; -import io.airbyte.protocol.models.Field.JsonSchemaPrimitive; +import io.airbyte.protocol.models.JsonSchemaPrimitive; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; diff --git a/airbyte-integrations/connectors/destination-postgres/src/test/java/io/airbyte/integrations/destination/PostgresDestinationTest.java b/airbyte-integrations/connectors/destination-postgres/src/test/java/io/airbyte/integrations/destination/PostgresDestinationTest.java index a9c08d8f379ed..59fde37dc5049 100644 --- a/airbyte-integrations/connectors/destination-postgres/src/test/java/io/airbyte/integrations/destination/PostgresDestinationTest.java +++ b/airbyte-integrations/connectors/destination-postgres/src/test/java/io/airbyte/integrations/destination/PostgresDestinationTest.java @@ -41,7 +41,7 @@ import io.airbyte.protocol.models.CatalogHelpers; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.Field; -import io.airbyte.protocol.models.Field.JsonSchemaPrimitive; +import io.airbyte.protocol.models.JsonSchemaPrimitive; import io.airbyte.test.utils.PostgreSQLContainerHelper; import java.time.Instant; import java.util.List; diff --git a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseSourceAcceptanceTest.java index ede3c4ed42b50..c1edc17600f5e 100644 --- a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseSourceAcceptanceTest.java @@ -41,7 +41,7 @@ import io.airbyte.protocol.models.ConnectorSpecification; import io.airbyte.protocol.models.DestinationSyncMode; import io.airbyte.protocol.models.Field; -import io.airbyte.protocol.models.Field.JsonSchemaPrimitive; +import io.airbyte.protocol.models.JsonSchemaPrimitive; import io.airbyte.protocol.models.SyncMode; import java.util.Collections; import java.util.HashMap; @@ -105,7 +105,7 @@ protected List getRegexTests() { } @Override - protected void setup(TestDestinationEnv testEnv) throws Exception { + protected void setupEnvironment(TestDestinationEnv environment) throws Exception { db = new ClickHouseContainer("yandex/clickhouse-server:21.3.10.1-alpine"); db.start(); diff --git a/airbyte-integrations/connectors/source-e2e-test/src/main/java/io/airbyte/integrations/source/e2e_test/ExceptionAfterNSource.java b/airbyte-integrations/connectors/source-e2e-test/src/main/java/io/airbyte/integrations/source/e2e_test/ExceptionAfterNSource.java index 635362e1d2a7d..36025f26d1324 100644 --- a/airbyte-integrations/connectors/source-e2e-test/src/main/java/io/airbyte/integrations/source/e2e_test/ExceptionAfterNSource.java +++ b/airbyte-integrations/connectors/source-e2e-test/src/main/java/io/airbyte/integrations/source/e2e_test/ExceptionAfterNSource.java @@ -42,7 +42,7 @@ import io.airbyte.protocol.models.CatalogHelpers; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.Field; -import io.airbyte.protocol.models.Field.JsonSchemaPrimitive; +import io.airbyte.protocol.models.JsonSchemaPrimitive; import io.airbyte.protocol.models.SyncMode; import java.time.Instant; import java.util.List; diff --git a/airbyte-integrations/connectors/source-e2e-test/src/main/java/io/airbyte/integrations/source/e2e_test/InfiniteFeedSource.java b/airbyte-integrations/connectors/source-e2e-test/src/main/java/io/airbyte/integrations/source/e2e_test/InfiniteFeedSource.java index a44beb616f3f5..5470eaf3a2200 100644 --- a/airbyte-integrations/connectors/source-e2e-test/src/main/java/io/airbyte/integrations/source/e2e_test/InfiniteFeedSource.java +++ b/airbyte-integrations/connectors/source-e2e-test/src/main/java/io/airbyte/integrations/source/e2e_test/InfiniteFeedSource.java @@ -41,7 +41,7 @@ import io.airbyte.protocol.models.CatalogHelpers; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.Field; -import io.airbyte.protocol.models.Field.JsonSchemaPrimitive; +import io.airbyte.protocol.models.JsonSchemaPrimitive; import java.time.Instant; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Predicate; diff --git a/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java b/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java index 2913c6c8d386d..baef07f73fd6c 100644 --- a/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java +++ b/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java @@ -54,7 +54,7 @@ import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.ConfiguredAirbyteStream; import io.airbyte.protocol.models.Field; -import io.airbyte.protocol.models.Field.JsonSchemaPrimitive; +import io.airbyte.protocol.models.JsonSchemaPrimitive; import io.airbyte.protocol.models.SyncMode; import java.sql.JDBCType; import java.sql.PreparedStatement; diff --git a/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/IncrementalUtils.java b/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/IncrementalUtils.java index c4f24341037cf..d65c0291ee921 100644 --- a/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/IncrementalUtils.java +++ b/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/IncrementalUtils.java @@ -25,7 +25,7 @@ package io.airbyte.integrations.source.jdbc; import io.airbyte.protocol.models.ConfiguredAirbyteStream; -import io.airbyte.protocol.models.Field.JsonSchemaPrimitive; +import io.airbyte.protocol.models.JsonSchemaPrimitive; public class IncrementalUtils { diff --git a/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/StateDecoratingIterator.java b/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/StateDecoratingIterator.java index b45027dc32fd6..58b2c979c6553 100644 --- a/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/StateDecoratingIterator.java +++ b/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/StateDecoratingIterator.java @@ -29,7 +29,7 @@ import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.AirbyteMessage.Type; import io.airbyte.protocol.models.AirbyteStateMessage; -import io.airbyte.protocol.models.Field.JsonSchemaPrimitive; +import io.airbyte.protocol.models.JsonSchemaPrimitive; import java.util.Iterator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/airbyte-integrations/connectors/source-jdbc/src/test-integration/java/io/airbyte/integrations/source/jdbc/JdbcSourceSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-jdbc/src/test-integration/java/io/airbyte/integrations/source/jdbc/JdbcSourceSourceAcceptanceTest.java index 569f0e52d578a..8f76eefdedb22 100644 --- a/airbyte-integrations/connectors/source-jdbc/src/test-integration/java/io/airbyte/integrations/source/jdbc/JdbcSourceSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-jdbc/src/test-integration/java/io/airbyte/integrations/source/jdbc/JdbcSourceSourceAcceptanceTest.java @@ -36,7 +36,7 @@ import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.ConnectorSpecification; import io.airbyte.protocol.models.Field; -import io.airbyte.protocol.models.Field.JsonSchemaPrimitive; +import io.airbyte.protocol.models.JsonSchemaPrimitive; import java.sql.SQLException; import java.util.ArrayList; import java.util.HashMap; @@ -57,7 +57,7 @@ public class JdbcSourceSourceAcceptanceTest extends SourceAcceptanceTest { private JsonNode config; @Override - protected void setup(TestDestinationEnv testEnv) throws SQLException { + protected void setupEnvironment(TestDestinationEnv environment) throws SQLException { container = new PostgreSQLContainer<>("postgres:13-alpine"); container.start(); diff --git a/airbyte-integrations/connectors/source-jdbc/src/test/java/io/airbyte/integrations/source/jdbc/IncrementalUtilsTest.java b/airbyte-integrations/connectors/source-jdbc/src/test/java/io/airbyte/integrations/source/jdbc/IncrementalUtilsTest.java index 4aaf454f972c2..e5f97287db2ef 100644 --- a/airbyte-integrations/connectors/source-jdbc/src/test/java/io/airbyte/integrations/source/jdbc/IncrementalUtilsTest.java +++ b/airbyte-integrations/connectors/source-jdbc/src/test/java/io/airbyte/integrations/source/jdbc/IncrementalUtilsTest.java @@ -34,7 +34,7 @@ import io.airbyte.protocol.models.CatalogHelpers; import io.airbyte.protocol.models.ConfiguredAirbyteStream; import io.airbyte.protocol.models.Field; -import io.airbyte.protocol.models.Field.JsonSchemaPrimitive; +import io.airbyte.protocol.models.JsonSchemaPrimitive; import java.util.Collections; import org.junit.jupiter.api.Test; diff --git a/airbyte-integrations/connectors/source-jdbc/src/test/java/io/airbyte/integrations/source/jdbc/StateDecoratingIteratorTest.java b/airbyte-integrations/connectors/source-jdbc/src/test/java/io/airbyte/integrations/source/jdbc/StateDecoratingIteratorTest.java index e789c01fb6b35..f62efe27d9e8b 100644 --- a/airbyte-integrations/connectors/source-jdbc/src/test/java/io/airbyte/integrations/source/jdbc/StateDecoratingIteratorTest.java +++ b/airbyte-integrations/connectors/source-jdbc/src/test/java/io/airbyte/integrations/source/jdbc/StateDecoratingIteratorTest.java @@ -38,7 +38,7 @@ import io.airbyte.protocol.models.AirbyteMessage.Type; import io.airbyte.protocol.models.AirbyteRecordMessage; import io.airbyte.protocol.models.AirbyteStateMessage; -import io.airbyte.protocol.models.Field.JsonSchemaPrimitive; +import io.airbyte.protocol.models.JsonSchemaPrimitive; import java.util.Collections; import java.util.Iterator; import java.util.Optional; diff --git a/airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcSourceAcceptanceTest.java index f35b0a4a5fab8..6c3cfa37bad32 100644 --- a/airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcSourceAcceptanceTest.java @@ -59,7 +59,7 @@ import io.airbyte.protocol.models.ConnectorSpecification; import io.airbyte.protocol.models.DestinationSyncMode; import io.airbyte.protocol.models.Field; -import io.airbyte.protocol.models.Field.JsonSchemaPrimitive; +import io.airbyte.protocol.models.JsonSchemaPrimitive; import io.airbyte.protocol.models.SyncMode; import java.math.BigDecimal; import java.sql.SQLException; diff --git a/airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcStressTest.java b/airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcStressTest.java index 7dbd98040c3ea..437d6a9d861ea 100644 --- a/airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcStressTest.java +++ b/airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcStressTest.java @@ -44,7 +44,7 @@ import io.airbyte.protocol.models.ConfiguredAirbyteStream; import io.airbyte.protocol.models.DestinationSyncMode; import io.airbyte.protocol.models.Field; -import io.airbyte.protocol.models.Field.JsonSchemaPrimitive; +import io.airbyte.protocol.models.JsonSchemaPrimitive; import io.airbyte.protocol.models.SyncMode; import java.math.BigDecimal; import java.nio.ByteBuffer; diff --git a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlSourceAcceptanceTest.java index 6a3911a12cd65..ea1bcb0125097 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlSourceAcceptanceTest.java @@ -37,7 +37,7 @@ import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.ConnectorSpecification; import io.airbyte.protocol.models.Field; -import io.airbyte.protocol.models.Field.JsonSchemaPrimitive; +import io.airbyte.protocol.models.JsonSchemaPrimitive; import java.sql.SQLException; import java.util.Collections; import java.util.HashMap; @@ -53,7 +53,7 @@ public class MssqlSourceAcceptanceTest extends SourceAcceptanceTest { private JsonNode config; @Override - protected void setup(TestDestinationEnv testEnv) throws SQLException { + protected void setupEnvironment(TestDestinationEnv environment) throws SQLException { db = new MSSQLServerContainer<>("mcr.microsoft.com/mssql/server:2019-latest").acceptLicense(); db.start(); diff --git a/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlSourceTest.java b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlSourceTest.java index 0550b42bad128..bc3024c25644d 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlSourceTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlSourceTest.java @@ -36,7 +36,7 @@ import io.airbyte.protocol.models.AirbyteCatalog; import io.airbyte.protocol.models.CatalogHelpers; import io.airbyte.protocol.models.Field; -import io.airbyte.protocol.models.Field.JsonSchemaPrimitive; +import io.airbyte.protocol.models.JsonSchemaPrimitive; import io.airbyte.protocol.models.SyncMode; import java.sql.SQLException; import java.util.List; diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceAcceptanceTest.java index c8162f01a7cd6..cfc2715bb38f6 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceAcceptanceTest.java @@ -39,7 +39,7 @@ import io.airbyte.protocol.models.ConnectorSpecification; import io.airbyte.protocol.models.DestinationSyncMode; import io.airbyte.protocol.models.Field; -import io.airbyte.protocol.models.Field.JsonSchemaPrimitive; +import io.airbyte.protocol.models.JsonSchemaPrimitive; import io.airbyte.protocol.models.SyncMode; import java.util.Collections; import java.util.List; @@ -108,7 +108,7 @@ protected List getRegexTests() { } @Override - protected void setup(TestDestinationEnv testEnv) { + protected void setupEnvironment(TestDestinationEnv environment) { container = new MySQLContainer<>("mysql:8.0"); container.start(); diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceComprehensiveTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceComprehensiveTest.java index ca04a8124b64a..98440da09d708 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceComprehensiveTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceComprehensiveTest.java @@ -29,10 +29,10 @@ import io.airbyte.commons.json.Jsons; import io.airbyte.db.Database; import io.airbyte.db.Databases; -import io.airbyte.integrations.standardtest.source.DataTypeTest; import io.airbyte.integrations.standardtest.source.SourceComprehensiveTest; +import io.airbyte.integrations.standardtest.source.TestDataWrapper; import io.airbyte.integrations.standardtest.source.TestDestinationEnv; -import io.airbyte.protocol.models.Field; +import io.airbyte.protocol.models.JsonSchemaPrimitive; import org.jooq.SQLDialect; import org.testcontainers.containers.MySQLContainer; @@ -120,154 +120,154 @@ private void executeQuery(String query) { @Override protected void initTests() { - addDataTypeTest( - DataTypeTest.builder("tinyint", Field.JsonSchemaPrimitive.NUMBER) - .addInsertValue("null") - .addInsertValue("-128", "127") + addDataTypeTestData( + TestDataWrapper.builder("tinyint", JsonSchemaPrimitive.NUMBER) + .addInsertValues("null") + .addInsertValues("-128", "127") .build()); - addDataTypeTest( - DataTypeTest.builder("smallint", Field.JsonSchemaPrimitive.NUMBER) - .addInsertValue("null") - .addInsertValue("-32768", "32767") + addDataTypeTestData( + TestDataWrapper.builder("smallint", JsonSchemaPrimitive.NUMBER) + .addInsertValues("null") + .addInsertValues("-32768", "32767") .build()); - addDataTypeTest( - DataTypeTest.builder("smallint", Field.JsonSchemaPrimitive.NUMBER) + addDataTypeTestData( + TestDataWrapper.builder("smallint", JsonSchemaPrimitive.NUMBER) .fullSourceDataType("smallint zerofill") - .addInsertValue("1") + .addInsertValues("1") .build()); - addDataTypeTest( - DataTypeTest.builder("mediumint", Field.JsonSchemaPrimitive.NUMBER) - .addInsertValue("null", "-8388608", "8388607") + addDataTypeTestData( + TestDataWrapper.builder("mediumint", JsonSchemaPrimitive.NUMBER) + .addInsertValues("null", "-8388608", "8388607") .build()); - addDataTypeTest( - DataTypeTest.builder("mediumint", Field.JsonSchemaPrimitive.NUMBER) + addDataTypeTestData( + TestDataWrapper.builder("mediumint", JsonSchemaPrimitive.NUMBER) .fullSourceDataType("mediumint zerofill") - .addInsertValue("1") + .addInsertValues("1") .build()); - addDataTypeTest( - DataTypeTest.builder("int", Field.JsonSchemaPrimitive.NUMBER) - .addInsertValue("null", "-2147483648", "2147483647") + addDataTypeTestData( + TestDataWrapper.builder("int", JsonSchemaPrimitive.NUMBER) + .addInsertValues("null", "-2147483648", "2147483647") .build()); - addDataTypeTest( - DataTypeTest.builder("int", Field.JsonSchemaPrimitive.NUMBER) + addDataTypeTestData( + TestDataWrapper.builder("int", JsonSchemaPrimitive.NUMBER) .fullSourceDataType("int zerofill") - .addInsertValue("1") + .addInsertValues("1") .build()); - addDataTypeTest( - DataTypeTest.builder("bigint", Field.JsonSchemaPrimitive.NUMBER) - .addInsertValue("null", "9223372036854775807") + addDataTypeTestData( + TestDataWrapper.builder("bigint", JsonSchemaPrimitive.NUMBER) + .addInsertValues("null", "9223372036854775807") .build()); - addDataTypeTest( - DataTypeTest.builder("float", Field.JsonSchemaPrimitive.NUMBER) - .addInsertValue("null") + addDataTypeTestData( + TestDataWrapper.builder("float", JsonSchemaPrimitive.NUMBER) + .addInsertValues("null") .build()); - addDataTypeTest( - DataTypeTest.builder("double", Field.JsonSchemaPrimitive.NUMBER) - .addInsertValue("null", "power(10, 308)", "1/power(10, 45)") + addDataTypeTestData( + TestDataWrapper.builder("double", JsonSchemaPrimitive.NUMBER) + .addInsertValues("null", "power(10, 308)", "1/power(10, 45)") .build()); - addDataTypeTest( - DataTypeTest.builder("decimal", Field.JsonSchemaPrimitive.NUMBER) + addDataTypeTestData( + TestDataWrapper.builder("decimal", JsonSchemaPrimitive.NUMBER) .fullSourceDataType("decimal(5,2)") - .addInsertValue("null") + .addInsertValues("null") .build()); - addDataTypeTest( - DataTypeTest.builder("bit", Field.JsonSchemaPrimitive.NUMBER) - .addInsertValue("null", "1", "0") + addDataTypeTestData( + TestDataWrapper.builder("bit", JsonSchemaPrimitive.NUMBER) + .addInsertValues("null", "1", "0") .build()); - addDataTypeTest( - DataTypeTest.builder("date", Field.JsonSchemaPrimitive.STRING) - .addInsertValue("null", "'2021-01-00'", "'2021-00-00'", "'0000-00-00'") + addDataTypeTestData( + TestDataWrapper.builder("date", JsonSchemaPrimitive.STRING) + .addInsertValues("null", "'2021-01-00'", "'2021-00-00'", "'0000-00-00'") .build()); - addDataTypeTest( - DataTypeTest.builder("datetime", Field.JsonSchemaPrimitive.STRING) - .addInsertValue("null", "'0000-00-00 00:00:00'") + addDataTypeTestData( + TestDataWrapper.builder("datetime", JsonSchemaPrimitive.STRING) + .addInsertValues("null", "'0000-00-00 00:00:00'") .build()); - addDataTypeTest( - DataTypeTest.builder("timestamp", Field.JsonSchemaPrimitive.STRING) - .addInsertValue("null") + addDataTypeTestData( + TestDataWrapper.builder("timestamp", JsonSchemaPrimitive.STRING) + .addInsertValues("null") .build()); - addDataTypeTest( - DataTypeTest.builder("time", Field.JsonSchemaPrimitive.STRING) - .addInsertValue("null", "'-838:59:59.000000'") + addDataTypeTestData( + TestDataWrapper.builder("time", JsonSchemaPrimitive.STRING) + .addInsertValues("null", "'-838:59:59.000000'") .build()); - addDataTypeTest( - DataTypeTest.builder("varchar", Field.JsonSchemaPrimitive.STRING) + addDataTypeTestData( + TestDataWrapper.builder("varchar", JsonSchemaPrimitive.STRING) .fullSourceDataType("varchar(256) character set cp1251") - .addInsertValue("null", "'тест'") + .addInsertValues("null", "'тест'") .build()); - addDataTypeTest( - DataTypeTest.builder("varchar", Field.JsonSchemaPrimitive.STRING) + addDataTypeTestData( + TestDataWrapper.builder("varchar", JsonSchemaPrimitive.STRING) .fullSourceDataType("varchar(256) character set utf16") - .addInsertValue("null", "0xfffd") + .addInsertValues("null", "0xfffd") .build()); - addDataTypeTest( - DataTypeTest.builder("varchar", Field.JsonSchemaPrimitive.STRING) + addDataTypeTestData( + TestDataWrapper.builder("varchar", JsonSchemaPrimitive.STRING) .fullSourceDataType("varchar(256)") - .addInsertValue("null", "'!\"#$%&\\'()*+,-./:;<=>?\\@[\\]^_\\`{|}~'") + .addInsertValues("null", "'!\"#$%&\\'()*+,-./:;<=>?\\@[\\]^_\\`{|}~'") .build()); - addDataTypeTest( - DataTypeTest.builder("varbinary", Field.JsonSchemaPrimitive.STRING) + addDataTypeTestData( + TestDataWrapper.builder("varbinary", JsonSchemaPrimitive.STRING) .fullSourceDataType("varbinary(256)") - .addInsertValue("null", "'test'") + .addInsertValues("null", "'test'") .build()); - addDataTypeTest( - DataTypeTest.builder("blob", Field.JsonSchemaPrimitive.STRING) - .addInsertValue("null", "'test'") + addDataTypeTestData( + TestDataWrapper.builder("blob", JsonSchemaPrimitive.STRING) + .addInsertValues("null", "'test'") .build()); - addDataTypeTest( - DataTypeTest.builder("mediumtext", Field.JsonSchemaPrimitive.STRING) - .addInsertValue("null", "lpad('0', 16777214, '0')") + addDataTypeTestData( + TestDataWrapper.builder("mediumtext", JsonSchemaPrimitive.STRING) + .addInsertValues("null", "lpad('0', 16777214, '0')") .build()); - addDataTypeTest( - DataTypeTest.builder("tinytext", Field.JsonSchemaPrimitive.STRING) - .addInsertValue("null") + addDataTypeTestData( + TestDataWrapper.builder("tinytext", JsonSchemaPrimitive.STRING) + .addInsertValues("null") .build()); - addDataTypeTest( - DataTypeTest.builder("longtext", Field.JsonSchemaPrimitive.STRING) - .addInsertValue("null") + addDataTypeTestData( + TestDataWrapper.builder("longtext", JsonSchemaPrimitive.STRING) + .addInsertValues("null") .build()); - addDataTypeTest( - DataTypeTest.builder("text", Field.JsonSchemaPrimitive.STRING) - .addInsertValue("null") + addDataTypeTestData( + TestDataWrapper.builder("text", JsonSchemaPrimitive.STRING) + .addInsertValues("null") .build()); - addDataTypeTest( - DataTypeTest.builder("json", Field.JsonSchemaPrimitive.STRING) - .addInsertValue("null", "'{\"a\" :10, \"b\": 15}'") + addDataTypeTestData( + TestDataWrapper.builder("json", JsonSchemaPrimitive.STRING) + .addInsertValues("null", "'{\"a\" :10, \"b\": 15}'") .build()); - addDataTypeTest( - DataTypeTest.builder("point", Field.JsonSchemaPrimitive.OBJECT) - .addInsertValue("null", "(ST_GeomFromText('POINT(1 1)'))") + addDataTypeTestData( + TestDataWrapper.builder("point", JsonSchemaPrimitive.OBJECT) + .addInsertValues("null", "(ST_GeomFromText('POINT(1 1)'))") .build()); - addDataTypeTest( - DataTypeTest.builder("bool", Field.JsonSchemaPrimitive.STRING) - .addInsertValue("null", "1", "127", "-128") + addDataTypeTestData( + TestDataWrapper.builder("bool", JsonSchemaPrimitive.STRING) + .addInsertValues("null", "1", "127", "-128") .build()); } diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceAcceptanceTest.java index 4163e0cf98f4f..685a1362947ea 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceAcceptanceTest.java @@ -39,7 +39,7 @@ import io.airbyte.protocol.models.ConnectorSpecification; import io.airbyte.protocol.models.DestinationSyncMode; import io.airbyte.protocol.models.Field; -import io.airbyte.protocol.models.Field.JsonSchemaPrimitive; +import io.airbyte.protocol.models.JsonSchemaPrimitive; import io.airbyte.protocol.models.SyncMode; import java.util.Collections; import java.util.HashMap; @@ -56,7 +56,7 @@ public class MySqlSourceAcceptanceTest extends SourceAcceptanceTest { private JsonNode config; @Override - protected void setup(TestDestinationEnv testEnv) throws Exception { + protected void setupEnvironment(TestDestinationEnv environment) throws Exception { container = new MySQLContainer<>("mysql:8.0"); container.start(); diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceComprehensiveTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceComprehensiveTest.java index 942207d5a9fb4..973db5b77ec31 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceComprehensiveTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceComprehensiveTest.java @@ -29,10 +29,10 @@ import io.airbyte.commons.json.Jsons; import io.airbyte.db.Database; import io.airbyte.db.Databases; -import io.airbyte.integrations.standardtest.source.DataTypeTest; import io.airbyte.integrations.standardtest.source.SourceComprehensiveTest; +import io.airbyte.integrations.standardtest.source.TestDataWrapper; import io.airbyte.integrations.standardtest.source.TestDestinationEnv; -import io.airbyte.protocol.models.Field.JsonSchemaPrimitive; +import io.airbyte.protocol.models.JsonSchemaPrimitive; import org.jooq.SQLDialect; import org.testcontainers.containers.MySQLContainer; @@ -88,159 +88,159 @@ protected Database setupDatabase() throws Exception { @Override protected void initTests() { - addDataTypeTest( - DataTypeTest.builder("tinyint", JsonSchemaPrimitive.NUMBER) - .addInsertValue("null") - .addInsertValue("-128", "127") + addDataTypeTestData( + TestDataWrapper.builder("tinyint", JsonSchemaPrimitive.NUMBER) + .addInsertValues("null") + .addInsertValues("-128", "127") .build()); - addDataTypeTest( - DataTypeTest.builder("smallint", JsonSchemaPrimitive.NUMBER) - .addInsertValue("null") - .addInsertValue("-32768", "32767") + addDataTypeTestData( + TestDataWrapper.builder("smallint", JsonSchemaPrimitive.NUMBER) + .addInsertValues("null") + .addInsertValues("-32768", "32767") .build()); - addDataTypeTest( - DataTypeTest.builder("smallint", JsonSchemaPrimitive.NUMBER) + addDataTypeTestData( + TestDataWrapper.builder("smallint", JsonSchemaPrimitive.NUMBER) .fullSourceDataType("smallint zerofill") - .addInsertValue("1") + .addInsertValues("1") .build()); - addDataTypeTest( - DataTypeTest.builder("mediumint", JsonSchemaPrimitive.NUMBER) - .addInsertValue("null", "-8388608", "8388607") + addDataTypeTestData( + TestDataWrapper.builder("mediumint", JsonSchemaPrimitive.NUMBER) + .addInsertValues("null", "-8388608", "8388607") .build()); - addDataTypeTest( - DataTypeTest.builder("mediumint", JsonSchemaPrimitive.NUMBER) + addDataTypeTestData( + TestDataWrapper.builder("mediumint", JsonSchemaPrimitive.NUMBER) .fullSourceDataType("mediumint zerofill") - .addInsertValue("1") + .addInsertValues("1") .build()); - addDataTypeTest( - DataTypeTest.builder("int", JsonSchemaPrimitive.NUMBER) - .addInsertValue("null", "-2147483648", "2147483647") + addDataTypeTestData( + TestDataWrapper.builder("int", JsonSchemaPrimitive.NUMBER) + .addInsertValues("null", "-2147483648", "2147483647") .build()); - addDataTypeTest( - DataTypeTest.builder("int", JsonSchemaPrimitive.NUMBER) + addDataTypeTestData( + TestDataWrapper.builder("int", JsonSchemaPrimitive.NUMBER) .fullSourceDataType("int zerofill") - .addInsertValue("1") + .addInsertValues("1") .build()); - addDataTypeTest( - DataTypeTest.builder("bigint", JsonSchemaPrimitive.NUMBER) - .addInsertValue("null", "9223372036854775807") + addDataTypeTestData( + TestDataWrapper.builder("bigint", JsonSchemaPrimitive.NUMBER) + .addInsertValues("null", "9223372036854775807") .build()); - addDataTypeTest( - DataTypeTest.builder("float", JsonSchemaPrimitive.NUMBER) - .addInsertValue("null") + addDataTypeTestData( + TestDataWrapper.builder("float", JsonSchemaPrimitive.NUMBER) + .addInsertValues("null") .build()); - addDataTypeTest( - DataTypeTest.builder("double", JsonSchemaPrimitive.NUMBER) - .addInsertValue("null", "power(10, 308)", "1/power(10, 45)") + addDataTypeTestData( + TestDataWrapper.builder("double", JsonSchemaPrimitive.NUMBER) + .addInsertValues("null", "power(10, 308)", "1/power(10, 45)") .build()); - addDataTypeTest( - DataTypeTest.builder("decimal", JsonSchemaPrimitive.NUMBER) + addDataTypeTestData( + TestDataWrapper.builder("decimal", JsonSchemaPrimitive.NUMBER) .fullSourceDataType("decimal(5,2)") - .addInsertValue("null") + .addInsertValues("null") .build()); - addDataTypeTest( - DataTypeTest.builder("bit", JsonSchemaPrimitive.NUMBER) - .addInsertValue("null", "1", "0") + addDataTypeTestData( + TestDataWrapper.builder("bit", JsonSchemaPrimitive.NUMBER) + .addInsertValues("null", "1", "0") .build()); - addDataTypeTest( - DataTypeTest.builder("date", JsonSchemaPrimitive.STRING) - .addInsertValue("null") + addDataTypeTestData( + TestDataWrapper.builder("date", JsonSchemaPrimitive.STRING) + .addInsertValues("null") // .addInsertValue("'2021-01-00'") // .addInsertValue("'2021-00-00'") // .addInsertValue("'0000-00-00'") .build()); - addDataTypeTest( - DataTypeTest.builder("datetime", JsonSchemaPrimitive.STRING) - .addInsertValue("null") + addDataTypeTestData( + TestDataWrapper.builder("datetime", JsonSchemaPrimitive.STRING) + .addInsertValues("null") // .addInsertValue("'0000-00-00 00:00:00'") .build()); - addDataTypeTest( - DataTypeTest.builder("timestamp", JsonSchemaPrimitive.STRING) - .addInsertValue("null") + addDataTypeTestData( + TestDataWrapper.builder("timestamp", JsonSchemaPrimitive.STRING) + .addInsertValues("null") .build()); - addDataTypeTest( - DataTypeTest.builder("time", JsonSchemaPrimitive.STRING) - .addInsertValue("null") + addDataTypeTestData( + TestDataWrapper.builder("time", JsonSchemaPrimitive.STRING) + .addInsertValues("null") // .addInsertValue("'-838:59:59.000000'") .build()); - addDataTypeTest( - DataTypeTest.builder("varchar", JsonSchemaPrimitive.STRING) + addDataTypeTestData( + TestDataWrapper.builder("varchar", JsonSchemaPrimitive.STRING) .fullSourceDataType("varchar(256) character set cp1251") - .addInsertValue("null", "'тест'") + .addInsertValues("null", "'тест'") .build()); - addDataTypeTest( - DataTypeTest.builder("varchar", JsonSchemaPrimitive.STRING) + addDataTypeTestData( + TestDataWrapper.builder("varchar", JsonSchemaPrimitive.STRING) .fullSourceDataType("varchar(256) character set utf16") - .addInsertValue("null", "0xfffd") + .addInsertValues("null", "0xfffd") .build()); - addDataTypeTest( - DataTypeTest.builder("varchar", JsonSchemaPrimitive.STRING) + addDataTypeTestData( + TestDataWrapper.builder("varchar", JsonSchemaPrimitive.STRING) .fullSourceDataType("varchar(256)") - .addInsertValue("null", "'!\"#$%&\\'()*+,-./:;<=>?\\@[\\]^_\\`{|}~'") + .addInsertValues("null", "'!\"#$%&\\'()*+,-./:;<=>?\\@[\\]^_\\`{|}~'") .build()); - addDataTypeTest( - DataTypeTest.builder("varbinary", JsonSchemaPrimitive.STRING) + addDataTypeTestData( + TestDataWrapper.builder("varbinary", JsonSchemaPrimitive.STRING) .fullSourceDataType("varbinary(256)") - .addInsertValue("null", "'test'") + .addInsertValues("null", "'test'") .build()); - addDataTypeTest( - DataTypeTest.builder("blob", JsonSchemaPrimitive.STRING) - .addInsertValue("null", "'test'") + addDataTypeTestData( + TestDataWrapper.builder("blob", JsonSchemaPrimitive.STRING) + .addInsertValues("null", "'test'") .build()); - addDataTypeTest( - DataTypeTest.builder("mediumtext", JsonSchemaPrimitive.STRING) - .addInsertValue("null", "lpad('0', 16777214, '0')") + addDataTypeTestData( + TestDataWrapper.builder("mediumtext", JsonSchemaPrimitive.STRING) + .addInsertValues("null", "lpad('0', 16777214, '0')") .build()); - addDataTypeTest( - DataTypeTest.builder("tinytext", JsonSchemaPrimitive.STRING) - .addInsertValue("null") + addDataTypeTestData( + TestDataWrapper.builder("tinytext", JsonSchemaPrimitive.STRING) + .addInsertValues("null") .build()); - addDataTypeTest( - DataTypeTest.builder("longtext", JsonSchemaPrimitive.STRING) - .addInsertValue("null") + addDataTypeTestData( + TestDataWrapper.builder("longtext", JsonSchemaPrimitive.STRING) + .addInsertValues("null") .build()); - addDataTypeTest( - DataTypeTest.builder("text", JsonSchemaPrimitive.STRING) - .addInsertValue("null") + addDataTypeTestData( + TestDataWrapper.builder("text", JsonSchemaPrimitive.STRING) + .addInsertValues("null") .build()); - addDataTypeTest( - DataTypeTest.builder("json", JsonSchemaPrimitive.STRING) - .addInsertValue("null", "'{\"a\" :10, \"b\": 15}'") + addDataTypeTestData( + TestDataWrapper.builder("json", JsonSchemaPrimitive.STRING) + .addInsertValues("null", "'{\"a\" :10, \"b\": 15}'") .build()); - addDataTypeTest( - DataTypeTest.builder("point", JsonSchemaPrimitive.OBJECT) - .addInsertValue("null", "(ST_GeomFromText('POINT(1 1)'))") + addDataTypeTestData( + TestDataWrapper.builder("point", JsonSchemaPrimitive.OBJECT) + .addInsertValues("null", "(ST_GeomFromText('POINT(1 1)'))") .build()); - addDataTypeTest( - DataTypeTest.builder("bool", JsonSchemaPrimitive.STRING) - .addInsertValue("null", "127", "-128") + addDataTypeTestData( + TestDataWrapper.builder("bool", JsonSchemaPrimitive.STRING) + .addInsertValues("null", "127", "-128") .build()); } diff --git a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceTest.java b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceTest.java index 28982bb462f1b..c241fae2aa83e 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceTest.java @@ -59,7 +59,7 @@ import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.ConfiguredAirbyteStream; import io.airbyte.protocol.models.Field; -import io.airbyte.protocol.models.Field.JsonSchemaPrimitive; +import io.airbyte.protocol.models.JsonSchemaPrimitive; import io.airbyte.protocol.models.SyncMode; import java.sql.SQLException; import java.util.ArrayList; diff --git a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/OracleSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/OracleSourceAcceptanceTest.java index 56cddb3bfd500..cb081df2a9fc0 100644 --- a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/OracleSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/OracleSourceAcceptanceTest.java @@ -38,7 +38,7 @@ import io.airbyte.protocol.models.ConfiguredAirbyteStream; import io.airbyte.protocol.models.ConnectorSpecification; import io.airbyte.protocol.models.Field; -import io.airbyte.protocol.models.Field.JsonSchemaPrimitive; +import io.airbyte.protocol.models.JsonSchemaPrimitive; import io.airbyte.protocol.models.SyncMode; import java.util.Collections; import java.util.HashMap; @@ -54,7 +54,7 @@ public class OracleSourceAcceptanceTest extends SourceAcceptanceTest { private JsonNode config; @Override - protected void setup(TestDestinationEnv testEnv) throws Exception { + protected void setupEnvironment(TestDestinationEnv environment) throws Exception { container = new OracleContainer("epiclabs/docker-oracle-xe-11g"); container.start(); diff --git a/airbyte-integrations/connectors/source-oracle/src/test/java/io/airbyte/integrations/source/oracle/OracleSourceTest.java b/airbyte-integrations/connectors/source-oracle/src/test/java/io/airbyte/integrations/source/oracle/OracleSourceTest.java index 5735b418e0644..4ef8b98940dc4 100644 --- a/airbyte-integrations/connectors/source-oracle/src/test/java/io/airbyte/integrations/source/oracle/OracleSourceTest.java +++ b/airbyte-integrations/connectors/source-oracle/src/test/java/io/airbyte/integrations/source/oracle/OracleSourceTest.java @@ -41,7 +41,7 @@ import io.airbyte.protocol.models.CatalogHelpers; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.Field; -import io.airbyte.protocol.models.Field.JsonSchemaPrimitive; +import io.airbyte.protocol.models.JsonSchemaPrimitive; import io.airbyte.protocol.models.SyncMode; import java.math.BigDecimal; import java.util.HashMap; diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceAcceptanceTest.java index aed17d762046d..71f6a82703d12 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceAcceptanceTest.java @@ -39,7 +39,7 @@ import io.airbyte.protocol.models.ConnectorSpecification; import io.airbyte.protocol.models.DestinationSyncMode; import io.airbyte.protocol.models.Field; -import io.airbyte.protocol.models.Field.JsonSchemaPrimitive; +import io.airbyte.protocol.models.JsonSchemaPrimitive; import io.airbyte.protocol.models.SyncMode; import java.util.Collections; import java.util.HashMap; @@ -65,7 +65,7 @@ public class CdcPostgresSourceAcceptanceTest extends SourceAcceptanceTest { private JsonNode config; @Override - protected void setup(TestDestinationEnv testEnv) throws Exception { + protected void setupEnvironment(TestDestinationEnv environment) throws Exception { container = new PostgreSQLContainer<>("postgres:13-alpine") .withCopyFileToContainer(MountableFile.forClasspathResource("postgresql.conf"), "/etc/postgresql/postgresql.conf") .withCommand("postgres -c config_file=/etc/postgresql/postgresql.conf"); diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceAcceptanceTest.java index fd14f4c36d7ff..4c0e3ad88eddd 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceAcceptanceTest.java @@ -39,7 +39,7 @@ import io.airbyte.protocol.models.ConnectorSpecification; import io.airbyte.protocol.models.DestinationSyncMode; import io.airbyte.protocol.models.Field; -import io.airbyte.protocol.models.Field.JsonSchemaPrimitive; +import io.airbyte.protocol.models.JsonSchemaPrimitive; import io.airbyte.protocol.models.SyncMode; import java.util.Collections; import java.util.HashMap; @@ -56,7 +56,7 @@ public class PostgresSourceAcceptanceTest extends SourceAcceptanceTest { private JsonNode config; @Override - protected void setup(TestDestinationEnv testEnv) throws Exception { + protected void setupEnvironment(TestDestinationEnv environment) throws Exception { container = new PostgreSQLContainer<>("postgres:13-alpine"); container.start(); diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceTest.java index 30663033cdb85..4031d7366e4d0 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceTest.java @@ -56,7 +56,7 @@ import io.airbyte.protocol.models.CatalogHelpers; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.Field; -import io.airbyte.protocol.models.Field.JsonSchemaPrimitive; +import io.airbyte.protocol.models.JsonSchemaPrimitive; import io.airbyte.protocol.models.SyncMode; import io.airbyte.test.utils.PostgreSQLContainerHelper; import java.sql.SQLException; diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceSSLTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceSSLTest.java index 8cfdb0b471005..e8e8799795dc0 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceSSLTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceSSLTest.java @@ -46,7 +46,7 @@ import io.airbyte.protocol.models.CatalogHelpers; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.Field; -import io.airbyte.protocol.models.Field.JsonSchemaPrimitive; +import io.airbyte.protocol.models.JsonSchemaPrimitive; import io.airbyte.protocol.models.SyncMode; import io.airbyte.test.utils.PostgreSQLContainerHelper; import java.math.BigDecimal; diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceTest.java index 029524e966c71..010efdf9bf659 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceTest.java @@ -46,7 +46,7 @@ import io.airbyte.protocol.models.CatalogHelpers; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.Field; -import io.airbyte.protocol.models.Field.JsonSchemaPrimitive; +import io.airbyte.protocol.models.JsonSchemaPrimitive; import io.airbyte.protocol.models.SyncMode; import io.airbyte.test.utils.PostgreSQLContainerHelper; import java.math.BigDecimal; diff --git a/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftSourceAcceptanceTest.java index ec7c9bdbf647f..758bb978474a6 100644 --- a/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftSourceAcceptanceTest.java @@ -38,6 +38,7 @@ import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.ConnectorSpecification; import io.airbyte.protocol.models.Field; +import io.airbyte.protocol.models.JsonSchemaPrimitive; import java.nio.file.Path; import java.sql.SQLException; import java.util.Collections; @@ -58,7 +59,7 @@ private static JsonNode getStaticConfig() { } @Override - protected void setup(TestDestinationEnv testEnv) throws Exception { + protected void setupEnvironment(TestDestinationEnv environment) throws Exception { config = getStaticConfig(); database = Databases.createJdbcDatabase( @@ -116,9 +117,9 @@ protected ConfiguredAirbyteCatalog getConfiguredCatalog() { return CatalogHelpers.createConfiguredAirbyteCatalog( streamName, schemaName, - Field.of("c_custkey", Field.JsonSchemaPrimitive.NUMBER), - Field.of("c_name", Field.JsonSchemaPrimitive.STRING), - Field.of("c_nation", Field.JsonSchemaPrimitive.STRING)); + Field.of("c_custkey", JsonSchemaPrimitive.NUMBER), + Field.of("c_name", JsonSchemaPrimitive.STRING), + Field.of("c_nation", JsonSchemaPrimitive.STRING)); } @Override diff --git a/airbyte-protocol/models/src/main/java/io/airbyte/protocol/models/Field.java b/airbyte-protocol/models/src/main/java/io/airbyte/protocol/models/Field.java index 3c1eb8f7f51db..22b14d1f6efeb 100644 --- a/airbyte-protocol/models/src/main/java/io/airbyte/protocol/models/Field.java +++ b/airbyte-protocol/models/src/main/java/io/airbyte/protocol/models/Field.java @@ -28,15 +28,6 @@ public class Field { - public enum JsonSchemaPrimitive { - STRING, - NUMBER, - OBJECT, - ARRAY, - BOOLEAN, - NULL; - } - private final String name; private final JsonSchemaPrimitive type; diff --git a/airbyte-protocol/models/src/main/java/io/airbyte/protocol/models/JsonSchemaPrimitive.java b/airbyte-protocol/models/src/main/java/io/airbyte/protocol/models/JsonSchemaPrimitive.java new file mode 100644 index 0000000000000..d4aa8ed5c18ae --- /dev/null +++ b/airbyte-protocol/models/src/main/java/io/airbyte/protocol/models/JsonSchemaPrimitive.java @@ -0,0 +1,34 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.protocol.models; + +public enum JsonSchemaPrimitive { + STRING, + NUMBER, + OBJECT, + ARRAY, + BOOLEAN, + NULL; +} diff --git a/airbyte-protocol/models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java b/airbyte-protocol/models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java index 4153a852078ad..83eb342a3b4d1 100644 --- a/airbyte-protocol/models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java +++ b/airbyte-protocol/models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java @@ -31,7 +31,6 @@ import com.google.common.collect.Sets; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; -import io.airbyte.protocol.models.Field.JsonSchemaPrimitive; import java.io.IOException; import java.util.Set; import org.junit.jupiter.api.Test; diff --git a/airbyte-scheduler/app/src/test/java/io/airbyte/scheduler/app/JobSchedulerTest.java b/airbyte-scheduler/app/src/test/java/io/airbyte/scheduler/app/JobSchedulerTest.java index 1fd6202193f90..cd8c69b5e3ac8 100644 --- a/airbyte-scheduler/app/src/test/java/io/airbyte/scheduler/app/JobSchedulerTest.java +++ b/airbyte-scheduler/app/src/test/java/io/airbyte/scheduler/app/JobSchedulerTest.java @@ -40,7 +40,7 @@ import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.ConfiguredAirbyteStream; import io.airbyte.protocol.models.Field; -import io.airbyte.protocol.models.Field.JsonSchemaPrimitive; +import io.airbyte.protocol.models.JsonSchemaPrimitive; import io.airbyte.scheduler.models.Job; import io.airbyte.scheduler.persistence.JobPersistence; import io.airbyte.scheduler.persistence.job_factory.SyncJobFactory; diff --git a/airbyte-scheduler/persistence/src/test/java/io/airbyte/scheduler/persistence/DefaultJobCreatorTest.java b/airbyte-scheduler/persistence/src/test/java/io/airbyte/scheduler/persistence/DefaultJobCreatorTest.java index 9c98a462fd4c0..94162b010d377 100644 --- a/airbyte-scheduler/persistence/src/test/java/io/airbyte/scheduler/persistence/DefaultJobCreatorTest.java +++ b/airbyte-scheduler/persistence/src/test/java/io/airbyte/scheduler/persistence/DefaultJobCreatorTest.java @@ -52,7 +52,7 @@ import io.airbyte.protocol.models.ConfiguredAirbyteStream; import io.airbyte.protocol.models.DestinationSyncMode; import io.airbyte.protocol.models.Field; -import io.airbyte.protocol.models.Field.JsonSchemaPrimitive; +import io.airbyte.protocol.models.JsonSchemaPrimitive; import java.io.IOException; import java.util.Collections; import java.util.List; diff --git a/airbyte-server/src/test/java/io/airbyte/server/converters/ConfigurationUpdateTest.java b/airbyte-server/src/test/java/io/airbyte/server/converters/ConfigurationUpdateTest.java index 5536f69a7e626..836ed26c17c1c 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/converters/ConfigurationUpdateTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/converters/ConfigurationUpdateTest.java @@ -40,7 +40,7 @@ import io.airbyte.protocol.models.CatalogHelpers; import io.airbyte.protocol.models.ConnectorSpecification; import io.airbyte.protocol.models.Field; -import io.airbyte.protocol.models.Field.JsonSchemaPrimitive; +import io.airbyte.protocol.models.JsonSchemaPrimitive; import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; import java.util.UUID; diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/SchedulerHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/SchedulerHandlerTest.java index f1c7bb2dca419..6bced97687ca1 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/SchedulerHandlerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/SchedulerHandlerTest.java @@ -72,7 +72,7 @@ import io.airbyte.protocol.models.CatalogHelpers; import io.airbyte.protocol.models.ConnectorSpecification; import io.airbyte.protocol.models.Field; -import io.airbyte.protocol.models.Field.JsonSchemaPrimitive; +import io.airbyte.protocol.models.JsonSchemaPrimitive; import io.airbyte.scheduler.client.SchedulerJobClient; import io.airbyte.scheduler.client.SynchronousJobMetadata; import io.airbyte.scheduler.client.SynchronousResponse; diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/WebBackendConnectionsHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/WebBackendConnectionsHandlerTest.java index 808a6fec84f18..80316fd592274 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/WebBackendConnectionsHandlerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/WebBackendConnectionsHandlerTest.java @@ -79,7 +79,7 @@ import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.protocol.models.CatalogHelpers; import io.airbyte.protocol.models.Field; -import io.airbyte.protocol.models.Field.JsonSchemaPrimitive; +import io.airbyte.protocol.models.JsonSchemaPrimitive; import io.airbyte.server.helpers.ConnectionHelpers; import io.airbyte.server.helpers.DestinationDefinitionHelpers; import io.airbyte.server.helpers.DestinationHelpers; diff --git a/airbyte-server/src/test/java/io/airbyte/server/helpers/ConnectionHelpers.java b/airbyte-server/src/test/java/io/airbyte/server/helpers/ConnectionHelpers.java index 4a2d829be0790..8a612ae69744f 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/helpers/ConnectionHelpers.java +++ b/airbyte-server/src/test/java/io/airbyte/server/helpers/ConnectionHelpers.java @@ -45,7 +45,7 @@ import io.airbyte.protocol.models.ConfiguredAirbyteStream; import io.airbyte.protocol.models.DestinationSyncMode; import io.airbyte.protocol.models.Field; -import io.airbyte.protocol.models.Field.JsonSchemaPrimitive; +import io.airbyte.protocol.models.JsonSchemaPrimitive; import java.util.Collections; import java.util.List; import java.util.UUID; diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/DefaultDiscoverCatalogWorkerTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/DefaultDiscoverCatalogWorkerTest.java index 08d4b74f21a1f..f5e3f4484ee1d 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/DefaultDiscoverCatalogWorkerTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/DefaultDiscoverCatalogWorkerTest.java @@ -45,7 +45,7 @@ import io.airbyte.protocol.models.AirbyteMessage.Type; import io.airbyte.protocol.models.CatalogHelpers; import io.airbyte.protocol.models.Field; -import io.airbyte.protocol.models.Field.JsonSchemaPrimitive; +import io.airbyte.protocol.models.JsonSchemaPrimitive; import io.airbyte.workers.process.IntegrationLauncher; import io.airbyte.workers.protocols.airbyte.AirbyteStreamFactory; import java.io.ByteArrayInputStream; diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/TestConfigHelpers.java b/airbyte-workers/src/test/java/io/airbyte/workers/TestConfigHelpers.java index 499aba531a0f8..c8add3cfeec31 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/TestConfigHelpers.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/TestConfigHelpers.java @@ -42,7 +42,7 @@ import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.ConfiguredAirbyteStream; import io.airbyte.protocol.models.Field; -import io.airbyte.protocol.models.Field.JsonSchemaPrimitive; +import io.airbyte.protocol.models.JsonSchemaPrimitive; import java.util.Collections; import java.util.List; import java.util.Map; diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSourceTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSourceTest.java index f3ee84cc7c771..2d40c766d8000 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSourceTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSourceTest.java @@ -43,7 +43,7 @@ import io.airbyte.protocol.models.CatalogHelpers; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.Field; -import io.airbyte.protocol.models.Field.JsonSchemaPrimitive; +import io.airbyte.protocol.models.JsonSchemaPrimitive; import io.airbyte.workers.WorkerConstants; import io.airbyte.workers.WorkerException; import io.airbyte.workers.process.IntegrationLauncher; diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/protocols/airbyte/NamespacingMapperTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/protocols/airbyte/NamespacingMapperTest.java index 0c3e24df7c858..e61d0868394b0 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/protocols/airbyte/NamespacingMapperTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/protocols/airbyte/NamespacingMapperTest.java @@ -32,7 +32,7 @@ import io.airbyte.protocol.models.CatalogHelpers; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.Field; -import io.airbyte.protocol.models.Field.JsonSchemaPrimitive; +import io.airbyte.protocol.models.JsonSchemaPrimitive; import org.junit.jupiter.api.Test; class NamespacingMapperTest { From 02ad48cb8822996ab2d50c6d2e364e92f78e8f4e Mon Sep 17 00:00:00 2001 From: Andrii Leonets Date: Wed, 2 Jun 2021 17:08:06 +0300 Subject: [PATCH 08/11] [internal review] rename TestDataWrapper.java to TestDataHolder.java + make test builder without mandatory params (simple builder) --- .../source/SourceComprehensiveTest.java | 14 +-- ...stDataWrapper.java => TestDataHolder.java} | 76 +++++++----- .../CdcMySqlSourceComprehensiveTest.java | 114 +++++++++++++----- .../mysql/MySqlSourceComprehensiveTest.java | 114 +++++++++++++----- 4 files changed, 224 insertions(+), 94 deletions(-) rename airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/{TestDataWrapper.java => TestDataHolder.java} (71%) diff --git a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceComprehensiveTest.java b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceComprehensiveTest.java index fc21e1d81cf18..237a6c299560a 100644 --- a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceComprehensiveTest.java +++ b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceComprehensiveTest.java @@ -49,7 +49,7 @@ public abstract class SourceComprehensiveTest extends SourceAbstractTest { private static final Logger LOGGER = LoggerFactory.getLogger(SourceComprehensiveTest.class); - private final List testDataWrappers = new ArrayList<>(); + private final List testDataHolders = new ArrayList<>(); /** * Setup the test database. All tables and data described in the registered tests will be put there. @@ -60,7 +60,7 @@ public abstract class SourceComprehensiveTest extends SourceAbstractTest { protected abstract Database setupDatabase() throws Exception; /** - * Put all required tests here using method {@link #addDataTypeTestData(TestDataWrapper)} + * Put all required tests here using method {@link #addDataTypeTestData(TestDataHolder)} */ protected abstract void initTests(); @@ -93,7 +93,7 @@ private void setupDatabaseInternal() throws Exception { initTests(); - for (TestDataWrapper test : testDataWrappers) { + for (TestDataHolder test : testDataHolders) { database.query(ctx -> { ctx.fetch(test.getCreateSqlQuery()); LOGGER.debug("Table " + test.getNameWithTestPrefix() + " is created."); @@ -114,7 +114,7 @@ private ConfiguredAirbyteCatalog getConfiguredCatalog() throws Exception { final JsonNode config = getConfig(); return new ConfiguredAirbyteCatalog().withStreams( - testDataWrappers + testDataHolders .stream() .map(test -> new ConfiguredAirbyteStream() .withSyncMode(SyncMode.INCREMENTAL) @@ -139,9 +139,9 @@ private ConfiguredAirbyteCatalog getConfiguredCatalog() throws Exception { * * @param test comprehensive data type test */ - public void addDataTypeTestData(TestDataWrapper test) { - testDataWrappers.add(test); - test.setTestNumber(testDataWrappers.stream().filter(t -> t.getSourceType().equals(test.getSourceType())).count()); + public void addDataTypeTestData(TestDataHolder test) { + testDataHolders.add(test); + test.setTestNumber(testDataHolders.stream().filter(t -> t.getSourceType().equals(test.getSourceType())).count()); } } diff --git a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/TestDataWrapper.java b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/TestDataHolder.java similarity index 71% rename from airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/TestDataWrapper.java rename to airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/TestDataHolder.java index aeae0a8efa032..c685fa39d6057 100644 --- a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/TestDataWrapper.java +++ b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/TestDataHolder.java @@ -30,7 +30,7 @@ import java.util.Arrays; import java.util.List; -public class TestDataWrapper { +public class TestDataHolder { private static final String DEFAULT_CREATE_TABLE_SQL = "CREATE TABLE %1$s(id integer primary key, test_column %2$s);"; private static final String DEFAULT_INSERT_SQL = "INSERT INTO %1$s VALUES (%2$s, %3$s);"; @@ -43,12 +43,12 @@ public class TestDataWrapper { private final String fullSourceDataType; private long testNumber; - TestDataWrapper(String sourceType, - JsonSchemaPrimitive airbyteType, - List values, - String createTablePatternSql, - String insertPatternSql, - String fullSourceDataType) { + TestDataHolder(String sourceType, + JsonSchemaPrimitive airbyteType, + List values, + String createTablePatternSql, + String insertPatternSql, + String fullSourceDataType) { this.sourceType = sourceType; this.airbyteType = airbyteType; this.values = values; @@ -60,34 +60,52 @@ public class TestDataWrapper { /** * The builder allows to setup any comprehensive data type test. * - * @param sourceType name of the source data type. Duplicates by name will be tested independently - * from each others. Note that this name will be used for connector setup and table creation. - * If source syntax requires more details (E.g. "varchar" type requires length - * "varchar(50)"), you can additionally set custom data type syntax by - * {@link DataTypeTestBuilder#fullSourceDataType(String)} method. - * @param airbyteType corresponding Airbyte data type. It requires for proper configuration - * {@link ConfiguredAirbyteStream} * @return builder for setup comprehensive test */ - public static DataTypeTestBuilder builder(String sourceType, JsonSchemaPrimitive airbyteType) { - return new DataTypeTestBuilder(sourceType, airbyteType); + public static TestDataHolderBuilder builder() { + return new TestDataHolderBuilder(); } - public static class DataTypeTestBuilder { + public static class TestDataHolderBuilder { - private final String sourceType; - private final JsonSchemaPrimitive airbyteType; + private String sourceType; + private JsonSchemaPrimitive airbyteType; private final List values = new ArrayList<>(); private String createTablePatternSql; private String insertPatternSql; private String fullSourceDataType; - DataTypeTestBuilder(String sourceType, JsonSchemaPrimitive airbyteType) { - this.sourceType = sourceType; - this.airbyteType = airbyteType; + TestDataHolderBuilder() { this.createTablePatternSql = DEFAULT_CREATE_TABLE_SQL; this.insertPatternSql = DEFAULT_INSERT_SQL; - this.fullSourceDataType = sourceType; + } + + /** + * The name of the source data type. Duplicates by name will be tested independently from each + * others. Note that this name will be used for connector setup and table creation. If source syntax + * requires more details (E.g. "varchar" type requires length "varchar(50)"), you can additionally + * set custom data type syntax by {@link TestDataHolderBuilder#fullSourceDataType(String)} method. + * + * @param sourceType source data type name + * @return builder + */ + public TestDataHolderBuilder sourceType(String sourceType) { + this.sourceType = sourceType; + if (fullSourceDataType == null) + fullSourceDataType = sourceType; + return this; + } + + /** + * corresponding Airbyte data type. It requires for proper configuration + * {@link ConfiguredAirbyteStream} + * + * @param airbyteType Airbyte data type + * @return builder + */ + public TestDataHolderBuilder airbyteType(JsonSchemaPrimitive airbyteType) { + this.airbyteType = airbyteType; + return this; } /** @@ -98,7 +116,7 @@ public static class DataTypeTestBuilder { * @param createTablePatternSql creation table sql pattern * @return builder */ - public DataTypeTestBuilder createTablePatternSql(String createTablePatternSql) { + public TestDataHolderBuilder createTablePatternSql(String createTablePatternSql) { this.createTablePatternSql = createTablePatternSql; return this; } @@ -111,7 +129,7 @@ public DataTypeTestBuilder createTablePatternSql(String createTablePatternSql) { * @param insertPatternSql creation table sql pattern * @return builder */ - public DataTypeTestBuilder insertPatternSql(String insertPatternSql) { + public TestDataHolderBuilder insertPatternSql(String insertPatternSql) { this.insertPatternSql = insertPatternSql; return this; } @@ -123,7 +141,7 @@ public DataTypeTestBuilder insertPatternSql(String insertPatternSql) { * @param fullSourceDataType actual string for the column data type description * @return builder */ - public DataTypeTestBuilder fullSourceDataType(String fullSourceDataType) { + public TestDataHolderBuilder fullSourceDataType(String fullSourceDataType) { this.fullSourceDataType = fullSourceDataType; return this; } @@ -136,13 +154,13 @@ public DataTypeTestBuilder fullSourceDataType(String fullSourceDataType) { * @param insertValue test value * @return builder */ - public DataTypeTestBuilder addInsertValues(String... insertValue) { + public TestDataHolderBuilder addInsertValues(String... insertValue) { this.values.addAll(Arrays.asList(insertValue)); return this; } - public TestDataWrapper build() { - return new TestDataWrapper(sourceType, airbyteType, values, createTablePatternSql, insertPatternSql, fullSourceDataType); + public TestDataHolder build() { + return new TestDataHolder(sourceType, airbyteType, values, createTablePatternSql, insertPatternSql, fullSourceDataType); } } diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceComprehensiveTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceComprehensiveTest.java index 98440da09d708..012e16764f578 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceComprehensiveTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceComprehensiveTest.java @@ -30,7 +30,7 @@ import io.airbyte.db.Database; import io.airbyte.db.Databases; import io.airbyte.integrations.standardtest.source.SourceComprehensiveTest; -import io.airbyte.integrations.standardtest.source.TestDataWrapper; +import io.airbyte.integrations.standardtest.source.TestDataHolder; import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.protocol.models.JsonSchemaPrimitive; import org.jooq.SQLDialect; @@ -121,152 +121,208 @@ private void executeQuery(String query) { @Override protected void initTests() { addDataTypeTestData( - TestDataWrapper.builder("tinyint", JsonSchemaPrimitive.NUMBER) + TestDataHolder.builder() + .sourceType("tinyint") + .airbyteType(JsonSchemaPrimitive.NUMBER) .addInsertValues("null") .addInsertValues("-128", "127") .build()); addDataTypeTestData( - TestDataWrapper.builder("smallint", JsonSchemaPrimitive.NUMBER) + TestDataHolder.builder() + .sourceType("smallint") + .airbyteType(JsonSchemaPrimitive.NUMBER) .addInsertValues("null") .addInsertValues("-32768", "32767") .build()); addDataTypeTestData( - TestDataWrapper.builder("smallint", JsonSchemaPrimitive.NUMBER) + TestDataHolder.builder() + .sourceType("smallint") + .airbyteType(JsonSchemaPrimitive.NUMBER) .fullSourceDataType("smallint zerofill") .addInsertValues("1") .build()); addDataTypeTestData( - TestDataWrapper.builder("mediumint", JsonSchemaPrimitive.NUMBER) + TestDataHolder.builder() + .sourceType("mediumint") + .airbyteType(JsonSchemaPrimitive.NUMBER) .addInsertValues("null", "-8388608", "8388607") .build()); addDataTypeTestData( - TestDataWrapper.builder("mediumint", JsonSchemaPrimitive.NUMBER) + TestDataHolder.builder() + .sourceType("mediumint") + .airbyteType(JsonSchemaPrimitive.NUMBER) .fullSourceDataType("mediumint zerofill") .addInsertValues("1") .build()); addDataTypeTestData( - TestDataWrapper.builder("int", JsonSchemaPrimitive.NUMBER) + TestDataHolder.builder() + .sourceType("int") + .airbyteType(JsonSchemaPrimitive.NUMBER) .addInsertValues("null", "-2147483648", "2147483647") .build()); addDataTypeTestData( - TestDataWrapper.builder("int", JsonSchemaPrimitive.NUMBER) + TestDataHolder.builder() + .sourceType("int") + .airbyteType(JsonSchemaPrimitive.NUMBER) .fullSourceDataType("int zerofill") .addInsertValues("1") .build()); addDataTypeTestData( - TestDataWrapper.builder("bigint", JsonSchemaPrimitive.NUMBER) + TestDataHolder.builder() + .sourceType("bigint") + .airbyteType(JsonSchemaPrimitive.NUMBER) .addInsertValues("null", "9223372036854775807") .build()); addDataTypeTestData( - TestDataWrapper.builder("float", JsonSchemaPrimitive.NUMBER) + TestDataHolder.builder() + .sourceType("float") + .airbyteType(JsonSchemaPrimitive.NUMBER) .addInsertValues("null") .build()); addDataTypeTestData( - TestDataWrapper.builder("double", JsonSchemaPrimitive.NUMBER) + TestDataHolder.builder() + .sourceType("double") + .airbyteType(JsonSchemaPrimitive.NUMBER) .addInsertValues("null", "power(10, 308)", "1/power(10, 45)") .build()); addDataTypeTestData( - TestDataWrapper.builder("decimal", JsonSchemaPrimitive.NUMBER) + TestDataHolder.builder() + .sourceType("decimal") + .airbyteType(JsonSchemaPrimitive.NUMBER) .fullSourceDataType("decimal(5,2)") .addInsertValues("null") .build()); addDataTypeTestData( - TestDataWrapper.builder("bit", JsonSchemaPrimitive.NUMBER) + TestDataHolder.builder() + .sourceType("bit") + .airbyteType(JsonSchemaPrimitive.NUMBER) .addInsertValues("null", "1", "0") .build()); addDataTypeTestData( - TestDataWrapper.builder("date", JsonSchemaPrimitive.STRING) + TestDataHolder.builder() + .sourceType("date") + .airbyteType(JsonSchemaPrimitive.STRING) .addInsertValues("null", "'2021-01-00'", "'2021-00-00'", "'0000-00-00'") .build()); addDataTypeTestData( - TestDataWrapper.builder("datetime", JsonSchemaPrimitive.STRING) + TestDataHolder.builder() + .sourceType("datetime") + .airbyteType(JsonSchemaPrimitive.STRING) .addInsertValues("null", "'0000-00-00 00:00:00'") .build()); addDataTypeTestData( - TestDataWrapper.builder("timestamp", JsonSchemaPrimitive.STRING) + TestDataHolder.builder() + .sourceType("timestamp") + .airbyteType(JsonSchemaPrimitive.STRING) .addInsertValues("null") .build()); addDataTypeTestData( - TestDataWrapper.builder("time", JsonSchemaPrimitive.STRING) + TestDataHolder.builder() + .sourceType("time") + .airbyteType(JsonSchemaPrimitive.STRING) .addInsertValues("null", "'-838:59:59.000000'") .build()); addDataTypeTestData( - TestDataWrapper.builder("varchar", JsonSchemaPrimitive.STRING) + TestDataHolder.builder() + .sourceType("varchar") + .airbyteType(JsonSchemaPrimitive.STRING) .fullSourceDataType("varchar(256) character set cp1251") .addInsertValues("null", "'тест'") .build()); addDataTypeTestData( - TestDataWrapper.builder("varchar", JsonSchemaPrimitive.STRING) + TestDataHolder.builder() + .sourceType("varchar") + .airbyteType(JsonSchemaPrimitive.STRING) .fullSourceDataType("varchar(256) character set utf16") .addInsertValues("null", "0xfffd") .build()); addDataTypeTestData( - TestDataWrapper.builder("varchar", JsonSchemaPrimitive.STRING) + TestDataHolder.builder() + .sourceType("varchar") + .airbyteType(JsonSchemaPrimitive.STRING) .fullSourceDataType("varchar(256)") .addInsertValues("null", "'!\"#$%&\\'()*+,-./:;<=>?\\@[\\]^_\\`{|}~'") .build()); addDataTypeTestData( - TestDataWrapper.builder("varbinary", JsonSchemaPrimitive.STRING) + TestDataHolder.builder() + .sourceType("varbinary") + .airbyteType(JsonSchemaPrimitive.STRING) .fullSourceDataType("varbinary(256)") .addInsertValues("null", "'test'") .build()); addDataTypeTestData( - TestDataWrapper.builder("blob", JsonSchemaPrimitive.STRING) + TestDataHolder.builder() + .sourceType("blob") + .airbyteType(JsonSchemaPrimitive.STRING) .addInsertValues("null", "'test'") .build()); addDataTypeTestData( - TestDataWrapper.builder("mediumtext", JsonSchemaPrimitive.STRING) + TestDataHolder.builder() + .sourceType("mediumtext") + .airbyteType(JsonSchemaPrimitive.STRING) .addInsertValues("null", "lpad('0', 16777214, '0')") .build()); addDataTypeTestData( - TestDataWrapper.builder("tinytext", JsonSchemaPrimitive.STRING) + TestDataHolder.builder() + .sourceType("tinytext") + .airbyteType(JsonSchemaPrimitive.STRING) .addInsertValues("null") .build()); addDataTypeTestData( - TestDataWrapper.builder("longtext", JsonSchemaPrimitive.STRING) + TestDataHolder.builder() + .sourceType("longtext") + .airbyteType(JsonSchemaPrimitive.STRING) .addInsertValues("null") .build()); addDataTypeTestData( - TestDataWrapper.builder("text", JsonSchemaPrimitive.STRING) + TestDataHolder.builder() + .sourceType("text") + .airbyteType(JsonSchemaPrimitive.STRING) .addInsertValues("null") .build()); addDataTypeTestData( - TestDataWrapper.builder("json", JsonSchemaPrimitive.STRING) + TestDataHolder.builder() + .sourceType("json") + .airbyteType(JsonSchemaPrimitive.STRING) .addInsertValues("null", "'{\"a\" :10, \"b\": 15}'") .build()); addDataTypeTestData( - TestDataWrapper.builder("point", JsonSchemaPrimitive.OBJECT) + TestDataHolder.builder() + .sourceType("point") + .airbyteType(JsonSchemaPrimitive.OBJECT) .addInsertValues("null", "(ST_GeomFromText('POINT(1 1)'))") .build()); addDataTypeTestData( - TestDataWrapper.builder("bool", JsonSchemaPrimitive.STRING) + TestDataHolder.builder() + .sourceType("bool") + .airbyteType(JsonSchemaPrimitive.STRING) .addInsertValues("null", "1", "127", "-128") .build()); diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceComprehensiveTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceComprehensiveTest.java index 973db5b77ec31..9c69ec2c0d8f8 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceComprehensiveTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceComprehensiveTest.java @@ -30,7 +30,7 @@ import io.airbyte.db.Database; import io.airbyte.db.Databases; import io.airbyte.integrations.standardtest.source.SourceComprehensiveTest; -import io.airbyte.integrations.standardtest.source.TestDataWrapper; +import io.airbyte.integrations.standardtest.source.TestDataHolder; import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.protocol.models.JsonSchemaPrimitive; import org.jooq.SQLDialect; @@ -89,73 +89,99 @@ protected Database setupDatabase() throws Exception { @Override protected void initTests() { addDataTypeTestData( - TestDataWrapper.builder("tinyint", JsonSchemaPrimitive.NUMBER) + TestDataHolder.builder() + .sourceType("tinyint") + .airbyteType(JsonSchemaPrimitive.NUMBER) .addInsertValues("null") .addInsertValues("-128", "127") .build()); addDataTypeTestData( - TestDataWrapper.builder("smallint", JsonSchemaPrimitive.NUMBER) + TestDataHolder.builder() + .sourceType("smallint") + .airbyteType(JsonSchemaPrimitive.NUMBER) .addInsertValues("null") .addInsertValues("-32768", "32767") .build()); addDataTypeTestData( - TestDataWrapper.builder("smallint", JsonSchemaPrimitive.NUMBER) + TestDataHolder.builder() + .sourceType("smallint") + .airbyteType(JsonSchemaPrimitive.NUMBER) .fullSourceDataType("smallint zerofill") .addInsertValues("1") .build()); addDataTypeTestData( - TestDataWrapper.builder("mediumint", JsonSchemaPrimitive.NUMBER) + TestDataHolder.builder() + .sourceType("mediumint") + .airbyteType(JsonSchemaPrimitive.NUMBER) .addInsertValues("null", "-8388608", "8388607") .build()); addDataTypeTestData( - TestDataWrapper.builder("mediumint", JsonSchemaPrimitive.NUMBER) + TestDataHolder.builder() + .sourceType("mediumint") + .airbyteType(JsonSchemaPrimitive.NUMBER) .fullSourceDataType("mediumint zerofill") .addInsertValues("1") .build()); addDataTypeTestData( - TestDataWrapper.builder("int", JsonSchemaPrimitive.NUMBER) + TestDataHolder.builder() + .sourceType("int") + .airbyteType(JsonSchemaPrimitive.NUMBER) .addInsertValues("null", "-2147483648", "2147483647") .build()); addDataTypeTestData( - TestDataWrapper.builder("int", JsonSchemaPrimitive.NUMBER) + TestDataHolder.builder() + .sourceType("int") + .airbyteType(JsonSchemaPrimitive.NUMBER) .fullSourceDataType("int zerofill") .addInsertValues("1") .build()); addDataTypeTestData( - TestDataWrapper.builder("bigint", JsonSchemaPrimitive.NUMBER) + TestDataHolder.builder() + .sourceType("bigint") + .airbyteType(JsonSchemaPrimitive.NUMBER) .addInsertValues("null", "9223372036854775807") .build()); addDataTypeTestData( - TestDataWrapper.builder("float", JsonSchemaPrimitive.NUMBER) + TestDataHolder.builder() + .sourceType("float") + .airbyteType(JsonSchemaPrimitive.NUMBER) .addInsertValues("null") .build()); addDataTypeTestData( - TestDataWrapper.builder("double", JsonSchemaPrimitive.NUMBER) + TestDataHolder.builder() + .sourceType("double") + .airbyteType(JsonSchemaPrimitive.NUMBER) .addInsertValues("null", "power(10, 308)", "1/power(10, 45)") .build()); addDataTypeTestData( - TestDataWrapper.builder("decimal", JsonSchemaPrimitive.NUMBER) + TestDataHolder.builder() + .sourceType("decimal") + .airbyteType(JsonSchemaPrimitive.NUMBER) .fullSourceDataType("decimal(5,2)") .addInsertValues("null") .build()); addDataTypeTestData( - TestDataWrapper.builder("bit", JsonSchemaPrimitive.NUMBER) + TestDataHolder.builder() + .sourceType("bit") + .airbyteType(JsonSchemaPrimitive.NUMBER) .addInsertValues("null", "1", "0") .build()); addDataTypeTestData( - TestDataWrapper.builder("date", JsonSchemaPrimitive.STRING) + TestDataHolder.builder() + .sourceType("date") + .airbyteType(JsonSchemaPrimitive.STRING) .addInsertValues("null") // .addInsertValue("'2021-01-00'") // .addInsertValue("'2021-00-00'") @@ -163,83 +189,113 @@ protected void initTests() { .build()); addDataTypeTestData( - TestDataWrapper.builder("datetime", JsonSchemaPrimitive.STRING) + TestDataHolder.builder() + .sourceType("datetime") + .airbyteType(JsonSchemaPrimitive.STRING) .addInsertValues("null") // .addInsertValue("'0000-00-00 00:00:00'") .build()); addDataTypeTestData( - TestDataWrapper.builder("timestamp", JsonSchemaPrimitive.STRING) + TestDataHolder.builder() + .sourceType("timestamp") + .airbyteType(JsonSchemaPrimitive.STRING) .addInsertValues("null") .build()); addDataTypeTestData( - TestDataWrapper.builder("time", JsonSchemaPrimitive.STRING) + TestDataHolder.builder() + .sourceType("time") + .airbyteType(JsonSchemaPrimitive.STRING) .addInsertValues("null") // .addInsertValue("'-838:59:59.000000'") .build()); addDataTypeTestData( - TestDataWrapper.builder("varchar", JsonSchemaPrimitive.STRING) + TestDataHolder.builder() + .sourceType("varchar") + .airbyteType(JsonSchemaPrimitive.STRING) .fullSourceDataType("varchar(256) character set cp1251") .addInsertValues("null", "'тест'") .build()); addDataTypeTestData( - TestDataWrapper.builder("varchar", JsonSchemaPrimitive.STRING) + TestDataHolder.builder() + .sourceType("varchar") + .airbyteType(JsonSchemaPrimitive.STRING) .fullSourceDataType("varchar(256) character set utf16") .addInsertValues("null", "0xfffd") .build()); addDataTypeTestData( - TestDataWrapper.builder("varchar", JsonSchemaPrimitive.STRING) + TestDataHolder.builder() + .sourceType("varchar") + .airbyteType(JsonSchemaPrimitive.STRING) .fullSourceDataType("varchar(256)") .addInsertValues("null", "'!\"#$%&\\'()*+,-./:;<=>?\\@[\\]^_\\`{|}~'") .build()); addDataTypeTestData( - TestDataWrapper.builder("varbinary", JsonSchemaPrimitive.STRING) + TestDataHolder.builder() + .sourceType("varbinary") + .airbyteType(JsonSchemaPrimitive.STRING) .fullSourceDataType("varbinary(256)") .addInsertValues("null", "'test'") .build()); addDataTypeTestData( - TestDataWrapper.builder("blob", JsonSchemaPrimitive.STRING) + TestDataHolder.builder() + .sourceType("blob") + .airbyteType(JsonSchemaPrimitive.STRING) .addInsertValues("null", "'test'") .build()); addDataTypeTestData( - TestDataWrapper.builder("mediumtext", JsonSchemaPrimitive.STRING) + TestDataHolder.builder() + .sourceType("mediumtext") + .airbyteType(JsonSchemaPrimitive.STRING) .addInsertValues("null", "lpad('0', 16777214, '0')") .build()); addDataTypeTestData( - TestDataWrapper.builder("tinytext", JsonSchemaPrimitive.STRING) + TestDataHolder.builder() + .sourceType("tinytext") + .airbyteType(JsonSchemaPrimitive.STRING) .addInsertValues("null") .build()); addDataTypeTestData( - TestDataWrapper.builder("longtext", JsonSchemaPrimitive.STRING) + TestDataHolder.builder() + .sourceType("longtext") + .airbyteType(JsonSchemaPrimitive.STRING) .addInsertValues("null") .build()); addDataTypeTestData( - TestDataWrapper.builder("text", JsonSchemaPrimitive.STRING) + TestDataHolder.builder() + .sourceType("text") + .airbyteType(JsonSchemaPrimitive.STRING) .addInsertValues("null") .build()); addDataTypeTestData( - TestDataWrapper.builder("json", JsonSchemaPrimitive.STRING) + TestDataHolder.builder() + .sourceType("json") + .airbyteType(JsonSchemaPrimitive.STRING) .addInsertValues("null", "'{\"a\" :10, \"b\": 15}'") .build()); addDataTypeTestData( - TestDataWrapper.builder("point", JsonSchemaPrimitive.OBJECT) + TestDataHolder.builder() + .sourceType("point") + .airbyteType(JsonSchemaPrimitive.OBJECT) .addInsertValues("null", "(ST_GeomFromText('POINT(1 1)'))") .build()); addDataTypeTestData( - TestDataWrapper.builder("bool", JsonSchemaPrimitive.STRING) + TestDataHolder.builder() + .sourceType("bool") + .airbyteType(JsonSchemaPrimitive.STRING) .addInsertValues("null", "127", "-128") .build()); From 29b47637468d635de78fc9856897d146afa66591 Mon Sep 17 00:00:00 2001 From: Andrii Leonets Date: Fri, 4 Jun 2021 11:53:28 +0300 Subject: [PATCH 09/11] [3840] add test of unsigned integer value --- .../source/mysql/MySqlSourceComprehensiveTest.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceComprehensiveTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceComprehensiveTest.java index 9c69ec2c0d8f8..2e3bc3ec6d4f0 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceComprehensiveTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceComprehensiveTest.java @@ -134,6 +134,14 @@ protected void initTests() { .addInsertValues("null", "-2147483648", "2147483647") .build()); + addDataTypeTestData( + TestDataHolder.builder() + .sourceType("int") + .airbyteType(JsonSchemaPrimitive.NUMBER) + .fullSourceDataType("int unsigned") + .addInsertValues("3428724653") + .build()); + addDataTypeTestData( TestDataHolder.builder() .sourceType("int") From 57941301e3228a0f3f19b4e9a548a1a2dde6a440 Mon Sep 17 00:00:00 2001 From: Andrii Leonets Date: Sat, 5 Jun 2021 15:17:32 +0300 Subject: [PATCH 10/11] Add possibility to set expected values from a streamer. Implement validation of outcome data. --- .../source/SourceComprehensiveTest.java | 35 ++++++++++-- .../standardtest/source/TestDataHolder.java | 33 ++++++++++- .../CdcMySqlSourceComprehensiveTest.java | 57 ++++++++++++++++--- .../mysql/MySqlSourceComprehensiveTest.java | 54 ++++++++++++++---- 4 files changed, 156 insertions(+), 23 deletions(-) diff --git a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceComprehensiveTest.java b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceComprehensiveTest.java index 237a6c299560a..cc4825d82280f 100644 --- a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceComprehensiveTest.java +++ b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceComprehensiveTest.java @@ -24,7 +24,7 @@ package io.airbyte.integrations.standardtest.source; -import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.Lists; @@ -39,7 +39,9 @@ import io.airbyte.protocol.models.JsonSchemaPrimitive; import io.airbyte.protocol.models.SyncMode; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import org.junit.jupiter.api.Test; import org.slf4j.Logger; @@ -49,6 +51,7 @@ public abstract class SourceComprehensiveTest extends SourceAbstractTest { private static final Logger LOGGER = LoggerFactory.getLogger(SourceComprehensiveTest.class); + private final String TEST_COLUMN_NAME = "test_column"; private final List testDataHolders = new ArrayList<>(); /** @@ -78,8 +81,32 @@ public void testDataTypes() throws Exception { List allMessages = runRead(catalog); final List recordMessages = allMessages.stream().filter(m -> m.getType() == Type.RECORD).collect(Collectors.toList()); - recordMessages.forEach(msg -> LOGGER.debug(msg.toString())); - assertFalse(recordMessages.isEmpty(), "Expected records from source"); + Map> expectedValues = new HashMap<>(); + testDataHolders.forEach(testDataHolder -> { + if (!testDataHolder.getExpectedValues().isEmpty()) + expectedValues.put(testDataHolder.getNameWithTestPrefix(), testDataHolder.getExpectedValues()); + }); + + recordMessages.forEach(msg -> { + String streamName = msg.getRecord().getStream(); + List expectedValuesForStream = expectedValues.get(streamName); + if (expectedValuesForStream != null) { + String value = getValueFromJsonNode(msg.getRecord().getData().get(TEST_COLUMN_NAME)); + assertTrue(expectedValuesForStream.contains(value), + "Returned value '" + value + "' by streamer " + streamName + " should be in the expected list: " + expectedValuesForStream); + expectedValuesForStream.remove(value); + } + }); + + expectedValues.forEach((streamName, values) -> { + assertTrue(values.isEmpty(), "The streamer " + streamName + " should return all expected values. Missing values: " + values); + }); + } + + private String getValueFromJsonNode(JsonNode jsonNode) { + String value = (jsonNode != null ? jsonNode.asText() : null); + value = (value != null && value.equals("null") ? null : value); + return value; } /** @@ -124,7 +151,7 @@ private ConfiguredAirbyteCatalog getConfiguredCatalog() throws Exception { String.format("%s", test.getNameWithTestPrefix()), String.format("%s", config.get("database").asText()), Field.of("id", JsonSchemaPrimitive.NUMBER), - Field.of("test_column", test.getAirbyteType())) + Field.of(TEST_COLUMN_NAME, test.getAirbyteType())) .withSourceDefinedCursor(true) .withSourceDefinedPrimaryKey(List.of(List.of("id"))) .withSupportedSyncModes( diff --git a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/TestDataHolder.java b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/TestDataHolder.java index c685fa39d6057..9dc9626ecd1ce 100644 --- a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/TestDataHolder.java +++ b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/TestDataHolder.java @@ -38,6 +38,7 @@ public class TestDataHolder { private final String sourceType; private final JsonSchemaPrimitive airbyteType; private final List values; + private final List expectedValues; private final String createTablePatternSql; private final String insertPatternSql; private final String fullSourceDataType; @@ -46,12 +47,14 @@ public class TestDataHolder { TestDataHolder(String sourceType, JsonSchemaPrimitive airbyteType, List values, + List expectedValues, String createTablePatternSql, String insertPatternSql, String fullSourceDataType) { this.sourceType = sourceType; this.airbyteType = airbyteType; this.values = values; + this.expectedValues = expectedValues; this.createTablePatternSql = createTablePatternSql; this.insertPatternSql = insertPatternSql; this.fullSourceDataType = fullSourceDataType; @@ -71,6 +74,7 @@ public static class TestDataHolderBuilder { private String sourceType; private JsonSchemaPrimitive airbyteType; private final List values = new ArrayList<>(); + private final List expectedValues = new ArrayList<>(); private String createTablePatternSql; private String insertPatternSql; private String fullSourceDataType; @@ -159,8 +163,31 @@ public TestDataHolderBuilder addInsertValues(String... insertValue) { return this; } + /** + * Adds expected value(s) to the test scope. If you add at least one value, it will check that all + * values are provided by corresponding streamer. + * + * @param expectedValue value which should be provided by a streamer + * @return builder + */ + public TestDataHolderBuilder addExpectedValues(String... expectedValue) { + this.expectedValues.addAll(Arrays.asList(expectedValue)); + return this; + } + + /** + * Add NULL value to the expected value list. If you need to add only one value and it's NULL, you + * have to use this method instead of {@link #addExpectedValues(String...)} + * + * @return builder + */ + public TestDataHolderBuilder addNullExpectedValue() { + this.expectedValues.add(null); + return this; + } + public TestDataHolder build() { - return new TestDataHolder(sourceType, airbyteType, values, createTablePatternSql, insertPatternSql, fullSourceDataType); + return new TestDataHolder(sourceType, airbyteType, values, expectedValues, createTablePatternSql, insertPatternSql, fullSourceDataType); } } @@ -177,6 +204,10 @@ public JsonSchemaPrimitive getAirbyteType() { return airbyteType; } + public List getExpectedValues() { + return expectedValues; + } + public String getNameWithTestPrefix() { return "test_" + testNumber + "_" + sourceType; } diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceComprehensiveTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceComprehensiveTest.java index 012e16764f578..28be1557669e8 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceComprehensiveTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceComprehensiveTest.java @@ -124,16 +124,16 @@ protected void initTests() { TestDataHolder.builder() .sourceType("tinyint") .airbyteType(JsonSchemaPrimitive.NUMBER) - .addInsertValues("null") - .addInsertValues("-128", "127") + .addInsertValues("null", "-128", "127") + .addExpectedValues(null, "-128", "127") .build()); addDataTypeTestData( TestDataHolder.builder() .sourceType("smallint") .airbyteType(JsonSchemaPrimitive.NUMBER) - .addInsertValues("null") - .addInsertValues("-32768", "32767") + .addInsertValues("null", "-32768", "32767") + .addExpectedValues(null, "-32768", "32767") .build()); addDataTypeTestData( @@ -142,6 +142,7 @@ protected void initTests() { .airbyteType(JsonSchemaPrimitive.NUMBER) .fullSourceDataType("smallint zerofill") .addInsertValues("1") + .addExpectedValues("1") .build()); addDataTypeTestData( @@ -149,6 +150,7 @@ protected void initTests() { .sourceType("mediumint") .airbyteType(JsonSchemaPrimitive.NUMBER) .addInsertValues("null", "-8388608", "8388607") + .addExpectedValues(null, "-8388608", "8388607") .build()); addDataTypeTestData( @@ -157,6 +159,7 @@ protected void initTests() { .airbyteType(JsonSchemaPrimitive.NUMBER) .fullSourceDataType("mediumint zerofill") .addInsertValues("1") + .addExpectedValues("1") .build()); addDataTypeTestData( @@ -164,6 +167,16 @@ protected void initTests() { .sourceType("int") .airbyteType(JsonSchemaPrimitive.NUMBER) .addInsertValues("null", "-2147483648", "2147483647") + .addExpectedValues(null, "-2147483648", "2147483647") + .build()); + + addDataTypeTestData( + TestDataHolder.builder() + .sourceType("int") + .airbyteType(JsonSchemaPrimitive.NUMBER) + .fullSourceDataType("int unsigned") + .addInsertValues("3428724653") + .addExpectedValues("3428724653") .build()); addDataTypeTestData( @@ -172,6 +185,7 @@ protected void initTests() { .airbyteType(JsonSchemaPrimitive.NUMBER) .fullSourceDataType("int zerofill") .addInsertValues("1") + .addExpectedValues("1") .build()); addDataTypeTestData( @@ -179,6 +193,7 @@ protected void initTests() { .sourceType("bigint") .airbyteType(JsonSchemaPrimitive.NUMBER) .addInsertValues("null", "9223372036854775807") + .addExpectedValues(null, "9223372036854775807") .build()); addDataTypeTestData( @@ -186,6 +201,7 @@ protected void initTests() { .sourceType("float") .airbyteType(JsonSchemaPrimitive.NUMBER) .addInsertValues("null") + .addNullExpectedValue() .build()); addDataTypeTestData( @@ -193,6 +209,7 @@ protected void initTests() { .sourceType("double") .airbyteType(JsonSchemaPrimitive.NUMBER) .addInsertValues("null", "power(10, 308)", "1/power(10, 45)") + .addExpectedValues(null, String.valueOf(Math.pow(10, 308)), String.valueOf(1 / Math.pow(10, 45))) .build()); addDataTypeTestData( @@ -201,6 +218,7 @@ protected void initTests() { .airbyteType(JsonSchemaPrimitive.NUMBER) .fullSourceDataType("decimal(5,2)") .addInsertValues("null") + .addNullExpectedValue() .build()); addDataTypeTestData( @@ -208,6 +226,8 @@ protected void initTests() { .sourceType("bit") .airbyteType(JsonSchemaPrimitive.NUMBER) .addInsertValues("null", "1", "0") + // @TODO returns True/False instead of 1/0. + // .addExpectedValues(null, "1", "0") .build()); addDataTypeTestData( @@ -215,13 +235,17 @@ protected void initTests() { .sourceType("date") .airbyteType(JsonSchemaPrimitive.STRING) .addInsertValues("null", "'2021-01-00'", "'2021-00-00'", "'0000-00-00'") + .addExpectedValues(null, null, null, null) .build()); addDataTypeTestData( TestDataHolder.builder() .sourceType("datetime") .airbyteType(JsonSchemaPrimitive.STRING) - .addInsertValues("null", "'0000-00-00 00:00:00'") + .addInsertValues("null") + .addNullExpectedValue() + // @TODO stream fails when gets Zero date value + // .addInsertValues("'0000-00-00 00:00:00'") .build()); addDataTypeTestData( @@ -229,13 +253,15 @@ protected void initTests() { .sourceType("timestamp") .airbyteType(JsonSchemaPrimitive.STRING) .addInsertValues("null") + .addNullExpectedValue() .build()); addDataTypeTestData( TestDataHolder.builder() .sourceType("time") .airbyteType(JsonSchemaPrimitive.STRING) - .addInsertValues("null", "'-838:59:59.000000'") + .addInsertValues("null", "'-838:59:59.000000'", "'00:00:01.000000'") + .addExpectedValues(null, "-3020399000000", "1000000") .build()); addDataTypeTestData( @@ -244,6 +270,8 @@ protected void initTests() { .airbyteType(JsonSchemaPrimitive.STRING) .fullSourceDataType("varchar(256) character set cp1251") .addInsertValues("null", "'тест'") + // @TODO stream returns invalid text "тест" + // .addExpectedValues(null, "тест") .build()); addDataTypeTestData( @@ -252,6 +280,8 @@ protected void initTests() { .airbyteType(JsonSchemaPrimitive.STRING) .fullSourceDataType("varchar(256) character set utf16") .addInsertValues("null", "0xfffd") + // @TODO streamer returns invalid text "�" + // .addExpectedValues(null, "�") .build()); addDataTypeTestData( @@ -260,6 +290,7 @@ protected void initTests() { .airbyteType(JsonSchemaPrimitive.STRING) .fullSourceDataType("varchar(256)") .addInsertValues("null", "'!\"#$%&\\'()*+,-./:;<=>?\\@[\\]^_\\`{|}~'") + .addExpectedValues(null, "!\"#$%&'()*+,-./:;<=>?@[]^_`{|}~") .build()); addDataTypeTestData( @@ -268,6 +299,8 @@ protected void initTests() { .airbyteType(JsonSchemaPrimitive.STRING) .fullSourceDataType("varbinary(256)") .addInsertValues("null", "'test'") + // @TODO Returns binary value instead of text + // .addExpectedValues(null, "test") .build()); addDataTypeTestData( @@ -275,6 +308,8 @@ protected void initTests() { .sourceType("blob") .airbyteType(JsonSchemaPrimitive.STRING) .addInsertValues("null", "'test'") + // @TODO Returns binary value instead of text + // .addExpectedValues(null, "test") .build()); addDataTypeTestData( @@ -282,6 +317,8 @@ protected void initTests() { .sourceType("mediumtext") .airbyteType(JsonSchemaPrimitive.STRING) .addInsertValues("null", "lpad('0', 16777214, '0')") + // @TODO returns null instead of long text + // .addExpectedValues(null, StringUtils.leftPad("0", 16777214, "0")) .build()); addDataTypeTestData( @@ -289,6 +326,7 @@ protected void initTests() { .sourceType("tinytext") .airbyteType(JsonSchemaPrimitive.STRING) .addInsertValues("null") + .addNullExpectedValue() .build()); addDataTypeTestData( @@ -296,6 +334,7 @@ protected void initTests() { .sourceType("longtext") .airbyteType(JsonSchemaPrimitive.STRING) .addInsertValues("null") + .addNullExpectedValue() .build()); addDataTypeTestData( @@ -303,13 +342,15 @@ protected void initTests() { .sourceType("text") .airbyteType(JsonSchemaPrimitive.STRING) .addInsertValues("null") + .addNullExpectedValue() .build()); addDataTypeTestData( TestDataHolder.builder() .sourceType("json") .airbyteType(JsonSchemaPrimitive.STRING) - .addInsertValues("null", "'{\"a\" :10, \"b\": 15}'") + .addInsertValues("null", "'{\"a\": 10, \"b\": 15}'") + .addExpectedValues(null, "{\"a\": 10, \"b\": 15}") .build()); addDataTypeTestData( @@ -324,6 +365,8 @@ protected void initTests() { .sourceType("bool") .airbyteType(JsonSchemaPrimitive.STRING) .addInsertValues("null", "1", "127", "-128") + // @TODO returns number instead of boolean + // .addExpectedValues(null, "true", "false", "false") .build()); } diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceComprehensiveTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceComprehensiveTest.java index 2e3bc3ec6d4f0..cffc713f416b1 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceComprehensiveTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceComprehensiveTest.java @@ -92,16 +92,16 @@ protected void initTests() { TestDataHolder.builder() .sourceType("tinyint") .airbyteType(JsonSchemaPrimitive.NUMBER) - .addInsertValues("null") - .addInsertValues("-128", "127") + .addInsertValues("null", "-128", "127") + .addExpectedValues(null, "-128", "127") .build()); addDataTypeTestData( TestDataHolder.builder() .sourceType("smallint") .airbyteType(JsonSchemaPrimitive.NUMBER) - .addInsertValues("null") - .addInsertValues("-32768", "32767") + .addInsertValues("null", "-32768", "32767") + .addExpectedValues(null, "-32768", "32767") .build()); addDataTypeTestData( @@ -110,6 +110,7 @@ protected void initTests() { .airbyteType(JsonSchemaPrimitive.NUMBER) .fullSourceDataType("smallint zerofill") .addInsertValues("1") + .addExpectedValues("1") .build()); addDataTypeTestData( @@ -117,6 +118,7 @@ protected void initTests() { .sourceType("mediumint") .airbyteType(JsonSchemaPrimitive.NUMBER) .addInsertValues("null", "-8388608", "8388607") + .addExpectedValues(null, "-8388608", "8388607") .build()); addDataTypeTestData( @@ -125,6 +127,7 @@ protected void initTests() { .airbyteType(JsonSchemaPrimitive.NUMBER) .fullSourceDataType("mediumint zerofill") .addInsertValues("1") + .addExpectedValues("1") .build()); addDataTypeTestData( @@ -132,6 +135,7 @@ protected void initTests() { .sourceType("int") .airbyteType(JsonSchemaPrimitive.NUMBER) .addInsertValues("null", "-2147483648", "2147483647") + .addExpectedValues(null, "-2147483648", "2147483647") .build()); addDataTypeTestData( @@ -140,6 +144,7 @@ protected void initTests() { .airbyteType(JsonSchemaPrimitive.NUMBER) .fullSourceDataType("int unsigned") .addInsertValues("3428724653") + .addExpectedValues("3428724653") .build()); addDataTypeTestData( @@ -148,6 +153,7 @@ protected void initTests() { .airbyteType(JsonSchemaPrimitive.NUMBER) .fullSourceDataType("int zerofill") .addInsertValues("1") + .addExpectedValues("1") .build()); addDataTypeTestData( @@ -155,6 +161,7 @@ protected void initTests() { .sourceType("bigint") .airbyteType(JsonSchemaPrimitive.NUMBER) .addInsertValues("null", "9223372036854775807") + .addExpectedValues(null, "9223372036854775807") .build()); addDataTypeTestData( @@ -162,6 +169,7 @@ protected void initTests() { .sourceType("float") .airbyteType(JsonSchemaPrimitive.NUMBER) .addInsertValues("null") + .addNullExpectedValue() .build()); addDataTypeTestData( @@ -169,6 +177,7 @@ protected void initTests() { .sourceType("double") .airbyteType(JsonSchemaPrimitive.NUMBER) .addInsertValues("null", "power(10, 308)", "1/power(10, 45)") + .addExpectedValues(null, String.valueOf(Math.pow(10, 308)), String.valueOf(1 / Math.pow(10, 45))) .build()); addDataTypeTestData( @@ -177,6 +186,7 @@ protected void initTests() { .airbyteType(JsonSchemaPrimitive.NUMBER) .fullSourceDataType("decimal(5,2)") .addInsertValues("null") + .addNullExpectedValue() .build()); addDataTypeTestData( @@ -184,6 +194,8 @@ protected void initTests() { .sourceType("bit") .airbyteType(JsonSchemaPrimitive.NUMBER) .addInsertValues("null", "1", "0") + // @TODO returns True/False instead of 1/0. + // .addExpectedValues(null, "1", "0") .build()); addDataTypeTestData( @@ -191,9 +203,9 @@ protected void initTests() { .sourceType("date") .airbyteType(JsonSchemaPrimitive.STRING) .addInsertValues("null") - // .addInsertValue("'2021-01-00'") - // .addInsertValue("'2021-00-00'") - // .addInsertValue("'0000-00-00'") + .addNullExpectedValue() + // @TODO stream fails when gets Zero date value + // .addInsertValues("'2021-01-00'", "'2021-00-00'", "'0000-00-00'") .build()); addDataTypeTestData( @@ -201,7 +213,9 @@ protected void initTests() { .sourceType("datetime") .airbyteType(JsonSchemaPrimitive.STRING) .addInsertValues("null") - // .addInsertValue("'0000-00-00 00:00:00'") + .addNullExpectedValue() + // @TODO stream fails when gets Zero date value + // .addInsertValues("'0000-00-00 00:00:00'") .build()); addDataTypeTestData( @@ -209,6 +223,7 @@ protected void initTests() { .sourceType("timestamp") .airbyteType(JsonSchemaPrimitive.STRING) .addInsertValues("null") + .addNullExpectedValue() .build()); addDataTypeTestData( @@ -216,7 +231,9 @@ protected void initTests() { .sourceType("time") .airbyteType(JsonSchemaPrimitive.STRING) .addInsertValues("null") - // .addInsertValue("'-838:59:59.000000'") + .addNullExpectedValue() + // @TODO stream fails when gets Zero date value + // .addInsertValues("'-838:59:59.000000'") .build()); addDataTypeTestData( @@ -225,6 +242,7 @@ protected void initTests() { .airbyteType(JsonSchemaPrimitive.STRING) .fullSourceDataType("varchar(256) character set cp1251") .addInsertValues("null", "'тест'") + .addExpectedValues(null, "тест") .build()); addDataTypeTestData( @@ -233,6 +251,7 @@ protected void initTests() { .airbyteType(JsonSchemaPrimitive.STRING) .fullSourceDataType("varchar(256) character set utf16") .addInsertValues("null", "0xfffd") + .addExpectedValues(null, "�") .build()); addDataTypeTestData( @@ -241,6 +260,7 @@ protected void initTests() { .airbyteType(JsonSchemaPrimitive.STRING) .fullSourceDataType("varchar(256)") .addInsertValues("null", "'!\"#$%&\\'()*+,-./:;<=>?\\@[\\]^_\\`{|}~'") + .addExpectedValues(null, "!\"#$%&'()*+,-./:;<=>?@[]^_`{|}~") .build()); addDataTypeTestData( @@ -249,6 +269,8 @@ protected void initTests() { .airbyteType(JsonSchemaPrimitive.STRING) .fullSourceDataType("varbinary(256)") .addInsertValues("null", "'test'") + // @TODO Returns binary value instead of text + // .addExpectedValues(null, "test") .build()); addDataTypeTestData( @@ -256,6 +278,8 @@ protected void initTests() { .sourceType("blob") .airbyteType(JsonSchemaPrimitive.STRING) .addInsertValues("null", "'test'") + // @TODO Returns binary value instead of text + // .addExpectedValues(null, "test") .build()); addDataTypeTestData( @@ -263,6 +287,8 @@ protected void initTests() { .sourceType("mediumtext") .airbyteType(JsonSchemaPrimitive.STRING) .addInsertValues("null", "lpad('0', 16777214, '0')") + // @TODO returns null instead of long text + // .addExpectedValues(null, StringUtils.leftPad("0", 16777214, "0")) .build()); addDataTypeTestData( @@ -270,6 +296,7 @@ protected void initTests() { .sourceType("tinytext") .airbyteType(JsonSchemaPrimitive.STRING) .addInsertValues("null") + .addNullExpectedValue() .build()); addDataTypeTestData( @@ -277,6 +304,7 @@ protected void initTests() { .sourceType("longtext") .airbyteType(JsonSchemaPrimitive.STRING) .addInsertValues("null") + .addNullExpectedValue() .build()); addDataTypeTestData( @@ -284,13 +312,15 @@ protected void initTests() { .sourceType("text") .airbyteType(JsonSchemaPrimitive.STRING) .addInsertValues("null") + .addNullExpectedValue() .build()); addDataTypeTestData( TestDataHolder.builder() .sourceType("json") .airbyteType(JsonSchemaPrimitive.STRING) - .addInsertValues("null", "'{\"a\" :10, \"b\": 15}'") + .addInsertValues("null", "'{\"a\": 10, \"b\": 15}'") + .addExpectedValues(null, "{\"a\": 10, \"b\": 15}") .build()); addDataTypeTestData( @@ -304,7 +334,9 @@ protected void initTests() { TestDataHolder.builder() .sourceType("bool") .airbyteType(JsonSchemaPrimitive.STRING) - .addInsertValues("null", "127", "-128") + .addInsertValues("null", "1", "127", "-128") + // @TODO in MySQL boolean returns true only if value equals 1, all other not null values -> false + // .addExpectedValues(null, "true", "false", "false") .build()); } From c5845c563babd8aee5c3cfa213f73a72f9f5f652 Mon Sep 17 00:00:00 2001 From: Andrii Leonets Date: Sat, 5 Jun 2021 15:30:26 +0300 Subject: [PATCH 11/11] Add value coverage of zero date for CDC stream. --- .../source/mysql/CdcMySqlSourceComprehensiveTest.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceComprehensiveTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceComprehensiveTest.java index 28be1557669e8..d0f043f00f2b8 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceComprehensiveTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceComprehensiveTest.java @@ -242,10 +242,8 @@ protected void initTests() { TestDataHolder.builder() .sourceType("datetime") .airbyteType(JsonSchemaPrimitive.STRING) - .addInsertValues("null") - .addNullExpectedValue() - // @TODO stream fails when gets Zero date value - // .addInsertValues("'0000-00-00 00:00:00'") + .addInsertValues("null", "'0000-00-00 00:00:00'") + .addExpectedValues(null, null) .build()); addDataTypeTestData(