From fa92904076cabfae2b657acf034fe17ac553fe03 Mon Sep 17 00:00:00 2001 From: TaoZex <45089228+TaoZex@users.noreply.github.com> Date: Fri, 6 Jan 2023 16:28:22 +0800 Subject: [PATCH] [Improve][Connector-V2][DataHub] Refactor DataHub connector e2e test (#3866) * [Improve][Connector-V2][DataHub] Refactor DataHub connector e2e test * [Improve][Connector-V2][DataHub] change datahub e2e test --- .../connector-datahub-e2e}/pom.xml | 13 +--- .../e2e/connector/datahub/DatahubIT.java} | 30 +++++++--- .../resources}/fakesource_to_datahub.conf | 14 ----- .../seatunnel-connector-v2-e2e/pom.xml | 1 + .../v2/datahub/FakeSourceToDatahubIT.java | 37 ------------ .../seatunnel-flink-connector-v2-e2e/pom.xml | 1 - .../connector-datahub-spark-e2e/pom.xml | 53 ---------------- .../datahub/fakesource_to_datahub.conf | 60 ------------------- .../seatunnel-spark-connector-v2-e2e/pom.xml | 1 - 9 files changed, 26 insertions(+), 184 deletions(-) rename seatunnel-e2e/{seatunnel-flink-connector-v2-e2e/connector-datahub-flink-e2e => seatunnel-connector-v2-e2e/connector-datahub-e2e}/pom.xml (80%) rename seatunnel-e2e/{seatunnel-spark-connector-v2-e2e/connector-datahub-spark-e2e/src/test/java/org/apache/seatunnel/e2e/spark/v2/datahub/FakeSourceToDatahubIT.java => seatunnel-connector-v2-e2e/connector-datahub-e2e/src/test/java/org/apache/seatunnel/e2e/connector/datahub/DatahubIT.java} (58%) rename seatunnel-e2e/{seatunnel-flink-connector-v2-e2e/connector-datahub-flink-e2e/src/test/resources/datahub => seatunnel-connector-v2-e2e/connector-datahub-e2e/src/test/resources}/fakesource_to_datahub.conf (56%) delete mode 100644 seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-datahub-flink-e2e/src/test/java/org/apache/seatunnel/e2e/flink/v2/datahub/FakeSourceToDatahubIT.java delete mode 100644 seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-datahub-spark-e2e/pom.xml delete mode 100644 seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-datahub-spark-e2e/src/test/resources/datahub/fakesource_to_datahub.conf diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-datahub-flink-e2e/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-datahub-e2e/pom.xml similarity index 80% rename from seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-datahub-flink-e2e/pom.xml rename to seatunnel-e2e/seatunnel-connector-v2-e2e/connector-datahub-e2e/pom.xml index 79dc07e22a4..e7bbdadefac 100644 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-datahub-flink-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-datahub-e2e/pom.xml @@ -18,23 +18,14 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> org.apache.seatunnel - seatunnel-flink-connector-v2-e2e + seatunnel-connector-v2-e2e ${revision} 4.0.0 - connector-datahub-flink-e2e + connector-datahub-e2e - - org.apache.seatunnel - connector-flink-e2e-base - ${project.version} - tests - test-jar - test - - org.apache.seatunnel diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-datahub-spark-e2e/src/test/java/org/apache/seatunnel/e2e/spark/v2/datahub/FakeSourceToDatahubIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-datahub-e2e/src/test/java/org/apache/seatunnel/e2e/connector/datahub/DatahubIT.java similarity index 58% rename from seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-datahub-spark-e2e/src/test/java/org/apache/seatunnel/e2e/spark/v2/datahub/FakeSourceToDatahubIT.java rename to seatunnel-e2e/seatunnel-connector-v2-e2e/connector-datahub-e2e/src/test/java/org/apache/seatunnel/e2e/connector/datahub/DatahubIT.java index 3f1518b729f..fa841b08e56 100644 --- a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-datahub-spark-e2e/src/test/java/org/apache/seatunnel/e2e/spark/v2/datahub/FakeSourceToDatahubIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-datahub-e2e/src/test/java/org/apache/seatunnel/e2e/connector/datahub/DatahubIT.java @@ -15,23 +15,39 @@ * limitations under the License. */ -package org.apache.seatunnel.e2e.spark.v2.datahub; +package org.apache.seatunnel.e2e.connector.datahub; -import org.apache.seatunnel.e2e.spark.SparkContainer; +import org.apache.seatunnel.e2e.common.TestResource; +import org.apache.seatunnel.e2e.common.TestSuiteBase; +import org.apache.seatunnel.e2e.common.container.TestContainer; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestTemplate; import org.testcontainers.containers.Container; import java.io.IOException; @Disabled("Disabled because it needs user's personal datahub account to run this test") -public class FakeSourceToDatahubIT extends SparkContainer { +public class DatahubIT extends TestSuiteBase implements TestResource { - @Test - public void testFakeSourceToDatahub() throws IOException, InterruptedException { - Container.ExecResult execResult = executeSeaTunnelSparkJob("/datahub/fakesource_to_datahub.conf"); + @BeforeEach + @Override + public void startUp() throws Exception { + + } + + @AfterEach + @Override + public void tearDown() throws Exception { + + } + + @TestTemplate + public void testDatahub(TestContainer container) throws IOException, InterruptedException { + Container.ExecResult execResult = container.executeJob("/fakesource_to_datahub.conf"); Assertions.assertEquals(0, execResult.getExitCode()); } } diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-datahub-flink-e2e/src/test/resources/datahub/fakesource_to_datahub.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-datahub-e2e/src/test/resources/fakesource_to_datahub.conf similarity index 56% rename from seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-datahub-flink-e2e/src/test/resources/datahub/fakesource_to_datahub.conf rename to seatunnel-e2e/seatunnel-connector-v2-e2e/connector-datahub-e2e/src/test/resources/fakesource_to_datahub.conf index d183a6a8fba..7defe940551 100644 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-datahub-flink-e2e/src/test/resources/datahub/fakesource_to_datahub.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-datahub-e2e/src/test/resources/fakesource_to_datahub.conf @@ -14,20 +14,13 @@ # See the License for the specific language governing permissions and # limitations under the License. # -###### -###### This config file is a demonstration of streaming processing in seatunnel config -###### env { - # You can set flink configuration here execution.parallelism = 1 job.mode = "BATCH" - #execution.checkpoint.interval = 10000 - #execution.checkpoint.data-uri = "hdfs://localhost:9000/checkpoint" } source { - # This is a example source plugin **only for test and demonstrate the feature source plugin** FakeSource { result_table_name = "fake" schema = { @@ -37,13 +30,9 @@ source { } } } - # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source/FakeSource } transform { - # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/transform/sql } sink { @@ -56,7 +45,4 @@ sink { timeout=3000 retryTimes=3 } - - # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/pom.xml index 2f11b69ce8f..466e02041fa 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/pom.xml @@ -45,6 +45,7 @@ connector-cdc-mysql-e2e connector-iceberg-e2e connector-iceberg-hadoop3-e2e + connector-datahub-e2e seatunnel-connector-v2-e2e diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-datahub-flink-e2e/src/test/java/org/apache/seatunnel/e2e/flink/v2/datahub/FakeSourceToDatahubIT.java b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-datahub-flink-e2e/src/test/java/org/apache/seatunnel/e2e/flink/v2/datahub/FakeSourceToDatahubIT.java deleted file mode 100644 index 4ee66c96b8a..00000000000 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-datahub-flink-e2e/src/test/java/org/apache/seatunnel/e2e/flink/v2/datahub/FakeSourceToDatahubIT.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.e2e.flink.v2.datahub; - -import org.apache.seatunnel.e2e.flink.FlinkContainer; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; -import org.testcontainers.containers.Container; - -import java.io.IOException; - -@Disabled("Disabled because it needs user's personal datahub account to run this test") -public class FakeSourceToDatahubIT extends FlinkContainer { - - @Test - public void testFakeSourceToDatahub() throws IOException, InterruptedException { - Container.ExecResult execResult = executeSeaTunnelFlinkJob("/datahub/fakesource_to_datahub.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - } -} diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/pom.xml b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/pom.xml index 42ae59143f1..c5e966bca43 100644 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/pom.xml @@ -29,7 +29,6 @@ connector-flink-e2e-base connector-jdbc-flink-e2e - connector-datahub-flink-e2e connector-mongodb-flink-e2e diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-datahub-spark-e2e/pom.xml b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-datahub-spark-e2e/pom.xml deleted file mode 100644 index 79c3916e787..00000000000 --- a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-datahub-spark-e2e/pom.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - - - org.apache.seatunnel - seatunnel-spark-connector-v2-e2e - ${revision} - - 4.0.0 - - connector-datahub-spark-e2e - - - - org.apache.seatunnel - connector-spark-e2e-base - ${project.version} - tests - test-jar - test - - - - - org.apache.seatunnel - connector-fake - ${project.version} - test - - - org.apache.seatunnel - connector-datahub - ${project.version} - test - - - - \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-datahub-spark-e2e/src/test/resources/datahub/fakesource_to_datahub.conf b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-datahub-spark-e2e/src/test/resources/datahub/fakesource_to_datahub.conf deleted file mode 100644 index 888516aff0e..00000000000 --- a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-datahub-spark-e2e/src/test/resources/datahub/fakesource_to_datahub.conf +++ /dev/null @@ -1,60 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -###### -###### This config file is a demonstration of streaming processing in seatunnel config -###### - -env { - # You can set flink configuration here - execution.parallelism = 1 - job.mode = "BATCH" - #execution.checkpoint.interval = 10000 - #execution.checkpoint.data-uri = "hdfs://localhost:9000/checkpoint" -} - -source { - # This is a example source plugin **only for test and demonstrate the feature source plugin** - FakeSource { - result_table_name = "fake" - schema = { - fields { - name = "string" - age = "int" - } - } - } - # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source/FakeSource -} - -transform { -} - -sink { - DataHub { - endpoint="xxx" - accessId="xxx" - accessKey="xxx" - project="xxx" - topic="xxx" - timeout=3000 - retryTimes=3 - } - - # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink -} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/pom.xml b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/pom.xml index 32e46c8dce9..65b432d9008 100644 --- a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/pom.xml @@ -28,7 +28,6 @@ connector-spark-e2e-base - connector-datahub-spark-e2e connector-jdbc-spark-e2e connector-mongodb-spark-e2e