From 0e6d31d4459044c9abdd76ae2e8774f6953f28c7 Mon Sep 17 00:00:00 2001 From: He Wang Date: Wed, 20 Sep 2023 10:25:54 +0800 Subject: [PATCH] [oceanbase][ci] Move the OceanBase CI to free azure pipeline (#2506) Move OceanBase CI to free azure pipeline with new docker image of OceanBase and remove redundant maven profile. --- azure-pipelines.yml | 7 -- .../oceanbase/OceanBaseTestBase.java | 60 +++------ .../table/OceanBaseConnectorITCase.java | 16 +-- pom.xml | 89 ++++--------- .../jobs-template-for-self-hosted-agent.yml | 119 ------------------ tools/azure-pipelines/jobs-template.yml | 4 +- tools/ci/compile.sh | 2 +- 7 files changed, 55 insertions(+), 242 deletions(-) delete mode 100644 tools/azure-pipelines/jobs-template-for-self-hosted-agent.yml diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 562c3dc2985..4785ce64995 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -45,10 +45,3 @@ stages: vmImage: 'ubuntu-20.04' run_end_to_end: false jdk: 8 - - template: tools/azure-pipelines/jobs-template-for-self-hosted-agent.yml - parameters: # see template file for a definition of the parameters. - stage_name: ci_build_on_self_hosted_agent - test_pool_definition: - name: Flink_CDC_CI - run_end_to_end: false - jdk: 8 diff --git a/flink-connector-oceanbase-cdc/src/test/java/com/ververica/cdc/connectors/oceanbase/OceanBaseTestBase.java b/flink-connector-oceanbase-cdc/src/test/java/com/ververica/cdc/connectors/oceanbase/OceanBaseTestBase.java index 9014657d923..eeba1e0e9f3 100644 --- a/flink-connector-oceanbase-cdc/src/test/java/com/ververica/cdc/connectors/oceanbase/OceanBaseTestBase.java +++ b/flink-connector-oceanbase-cdc/src/test/java/com/ververica/cdc/connectors/oceanbase/OceanBaseTestBase.java @@ -56,37 +56,22 @@ public class OceanBaseTestBase extends TestLogger { private static final Pattern COMMENT_PATTERN = Pattern.compile("^(.*)--.*$"); private static final Duration CONTAINER_STARTUP_TIMEOUT = Duration.ofMinutes(4); - public static final int OB_SERVER_SQL_PORT = 2881; - public static final int OB_SERVER_RPC_PORT = 2882; - public static final int LOG_PROXY_PORT = 2983; - - public static final String OB_SYS_PASSWORD = "pswd"; - public static final String OB_TEST_PASSWORD = "test"; - public static final String NETWORK_MODE = "host"; // -------------------------------------------------------------------------------------------- // Attributes about host and port when network is on 'host' mode. // -------------------------------------------------------------------------------------------- - protected static String getObServerHost() { - return "127.0.0.1"; - } - - protected static String getLogProxyHost() { - return "127.0.0.1"; - } - protected static int getObServerSqlPort() { - return OB_SERVER_SQL_PORT; + return 2881; } - protected static int getObServerRpcPort() { - return OB_SERVER_RPC_PORT; + protected static int getLogProxyPort() { + return 2983; } - protected static int getLogProxyPort() { - return LOG_PROXY_PORT; + public static String getRsList() { + return "127.0.0.1:2882:2881"; } // -------------------------------------------------------------------------------------------- @@ -94,33 +79,34 @@ protected static int getLogProxyPort() { // From OceanBase 4.0.0.0 CE, we can only fetch the commit log of non-sys tenant. // -------------------------------------------------------------------------------------------- + public static final String OB_SYS_PASSWORD = "pswd"; + protected static String getTenant() { return "test"; } protected static String getUsername() { - return "root@test"; + return "root@" + getTenant(); } protected static String getPassword() { - return OB_TEST_PASSWORD; + return "test"; } @ClassRule public static final GenericContainer OB_SERVER = - new GenericContainer<>("oceanbase/oceanbase-ce:4.0.0.0") + new GenericContainer<>("oceanbase/oceanbase-ce:4.2.0.0") .withNetworkMode(NETWORK_MODE) - .withExposedPorts(OB_SERVER_SQL_PORT, OB_SERVER_RPC_PORT) + .withEnv("MODE", "slim") + .withEnv("OB_ROOT_PASSWORD", OB_SYS_PASSWORD) .waitingFor(Wait.forLogMessage(".*boot success!.*", 1)) .withStartupTimeout(CONTAINER_STARTUP_TIMEOUT) .withLogConsumer(new Slf4jLogConsumer(LOG)); @ClassRule public static final GenericContainer LOG_PROXY = - new GenericContainer<>("whhe/oblogproxy:1.1.0_4x") + new GenericContainer<>("whhe/oblogproxy:1.1.3_4x") .withNetworkMode(NETWORK_MODE) - .withExposedPorts(LOG_PROXY_PORT) - .withEnv("OB_SYS_USERNAME", "root") .withEnv("OB_SYS_PASSWORD", OB_SYS_PASSWORD) .waitingFor(Wait.forLogMessage(".*boot success!.*", 1)) .withStartupTimeout(CONTAINER_STARTUP_TIMEOUT) @@ -132,17 +118,12 @@ public static void startContainers() { Startables.deepStart(Stream.of(OB_SERVER, LOG_PROXY)).join(); LOG.info("Containers are started."); - setPassword("root@sys", OB_SYS_PASSWORD); - setPassword("root@test", OB_TEST_PASSWORD); - } - - private static void setPassword(String username, String password) { - try (Connection connection = DriverManager.getConnection(getJdbcUrl(""), username, ""); + try (Connection connection = + DriverManager.getConnection(getJdbcUrl(""), getUsername(), ""); Statement statement = connection.createStatement()) { - statement.execute(String.format("ALTER USER root IDENTIFIED BY '%s'", password)); - LOG.info("Set password of {} to {}", username, password); + statement.execute(String.format("ALTER USER root IDENTIFIED BY '%s'", getPassword())); } catch (SQLException e) { - LOG.error("Set password of {} failed.", username, e); + LOG.error("Set test user password failed.", e); throw new RuntimeException(e); } } @@ -156,7 +137,7 @@ public static void stopContainers() { public static String getJdbcUrl(String databaseName) { return "jdbc:mysql://" - + getObServerHost() + + OB_SERVER.getHost() + ":" + getObServerSqlPort() + "/" @@ -164,11 +145,6 @@ public static String getJdbcUrl(String databaseName) { + "?useSSL=false"; } - public static String getRsList() { - return String.format( - "%s:%s:%s", getObServerHost(), getObServerRpcPort(), getObServerSqlPort()); - } - protected static Connection getJdbcConnection(String databaseName) throws SQLException { return DriverManager.getConnection(getJdbcUrl(databaseName), getUsername(), getPassword()); } diff --git a/flink-connector-oceanbase-cdc/src/test/java/com/ververica/cdc/connectors/oceanbase/table/OceanBaseConnectorITCase.java b/flink-connector-oceanbase-cdc/src/test/java/com/ververica/cdc/connectors/oceanbase/table/OceanBaseConnectorITCase.java index 8e0005a5c06..80fbbddf3c3 100644 --- a/flink-connector-oceanbase-cdc/src/test/java/com/ververica/cdc/connectors/oceanbase/table/OceanBaseConnectorITCase.java +++ b/flink-connector-oceanbase-cdc/src/test/java/com/ververica/cdc/connectors/oceanbase/table/OceanBaseConnectorITCase.java @@ -104,9 +104,9 @@ public void testTableList() throws Exception { getPassword(), getTenant(), "inventory.products", - getObServerHost(), + OB_SERVER.getHost(), getObServerSqlPort(), - getLogProxyHost(), + LOG_PROXY.getHost(), getLogProxyPort(), getRsList()); @@ -230,9 +230,9 @@ public void testMetadataColumns() throws Exception { getTenant(), "^inventory_meta$", "^products$", - getObServerHost(), + OB_SERVER.getHost(), getObServerSqlPort(), - getLogProxyHost(), + LOG_PROXY.getHost(), getLogProxyPort(), getRsList()); @@ -382,9 +382,9 @@ public void testAllDataTypes() throws Exception { "^column_type_test$", "^full_types$", serverTimeZone, - getObServerHost(), + OB_SERVER.getHost(), getObServerSqlPort(), - getLogProxyHost(), + LOG_PROXY.getHost(), getLogProxyPort(), getRsList()); String sinkDDL = @@ -512,9 +512,9 @@ public void testTimeDataTypes(String serverTimeZone) throws Exception { "column_type_test", "full_types", serverTimeZone, - getObServerHost(), + OB_SERVER.getHost(), getObServerSqlPort(), - getLogProxyHost(), + LOG_PROXY.getHost(), getLogProxyPort(), getRsList()); diff --git a/pom.xml b/pom.xml index 1cf7969c7d2..fb518c28edd 100644 --- a/pom.xml +++ b/pom.xml @@ -30,6 +30,31 @@ under the License. 2.4-SNAPSHOT pom + + flink-cdc-base + flink-connector-db2-cdc + flink-connector-debezium + flink-connector-mongodb-cdc + flink-connector-mysql-cdc + flink-connector-oceanbase-cdc + flink-connector-oracle-cdc + flink-connector-postgres-cdc + flink-connector-sqlserver-cdc + flink-connector-test-util + flink-connector-tidb-cdc + flink-connector-vitess-cdc + flink-sql-connector-db2-cdc + flink-sql-connector-mongodb-cdc + flink-sql-connector-mysql-cdc + flink-sql-connector-oceanbase-cdc + flink-sql-connector-oracle-cdc + flink-sql-connector-postgres-cdc + flink-sql-connector-sqlserver-cdc + flink-sql-connector-tidb-cdc + flink-sql-connector-vitess-cdc + flink-cdc-e2e-tests + + The Apache Software License, Version 2.0 @@ -454,62 +479,8 @@ under the License. - - default - - true - - - flink-cdc-base - flink-connector-debezium - flink-connector-test-util - flink-connector-mysql-cdc - flink-connector-postgres-cdc - flink-connector-oracle-cdc - flink-connector-mongodb-cdc - flink-connector-oceanbase-cdc - flink-connector-sqlserver-cdc - flink-connector-tidb-cdc - flink-connector-db2-cdc - flink-connector-vitess-cdc - flink-sql-connector-mysql-cdc - flink-sql-connector-postgres-cdc - flink-sql-connector-mongodb-cdc - flink-sql-connector-oracle-cdc - flink-sql-connector-oceanbase-cdc - flink-sql-connector-sqlserver-cdc - flink-sql-connector-tidb-cdc - flink-sql-connector-db2-cdc - flink-sql-connector-vitess-cdc - flink-cdc-e2e-tests - - release - - flink-cdc-base - flink-connector-debezium - flink-connector-test-util - flink-connector-mysql-cdc - flink-connector-postgres-cdc - flink-connector-oracle-cdc - flink-connector-mongodb-cdc - flink-connector-oceanbase-cdc - flink-connector-sqlserver-cdc - flink-connector-tidb-cdc - flink-connector-db2-cdc - flink-connector-vitess-cdc - flink-sql-connector-mysql-cdc - flink-sql-connector-postgres-cdc - flink-sql-connector-mongodb-cdc - flink-sql-connector-oracle-cdc - flink-sql-connector-oceanbase-cdc - flink-sql-connector-sqlserver-cdc - flink-sql-connector-tidb-cdc - flink-sql-connector-db2-cdc - flink-sql-connector-vitess-cdc - flink-cdc-e2e-tests - @@ -612,15 +583,5 @@ under the License. - - self-hosted-pipeline - - flink-cdc-base - flink-connector-debezium - flink-connector-test-util - flink-connector-oceanbase-cdc - flink-sql-connector-oceanbase-cdc - - diff --git a/tools/azure-pipelines/jobs-template-for-self-hosted-agent.yml b/tools/azure-pipelines/jobs-template-for-self-hosted-agent.yml deleted file mode 100644 index 916d6b6ee92..00000000000 --- a/tools/azure-pipelines/jobs-template-for-self-hosted-agent.yml +++ /dev/null @@ -1,119 +0,0 @@ -# Copyright 2023 Ververica Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -parameters: - test_pool_definition: # defines the hardware pool for compilation and unit test execution. - stage_name: # defines a unique identifier for all jobs in a stage (in case the jobs are added multiple times to a stage) - run_end_to_end: # if set to 'true', the end to end tests will be executed - jdk: # the jdk version to use - -jobs: - - job: compile_${{parameters.stage_name}} - # succeeded() is needed to allow job cancellation - condition: and(succeeded(), not(eq(variables['MODE'], 'e2e'))) - pool: ${{parameters.test_pool_definition}} - timeoutInMinutes: 40 - cancelTimeoutInMinutes: 1 - workspace: - clean: all # this cleans the entire workspace directory before running a new job - # It is necessary because the custom build machines are reused for tests. - # See also https://docs.microsoft.com/en-us/azure/devops/pipelines/process/phases?view=azure-devops&tabs=yaml#workspace - - steps: - # The cache task is persisting the .m2 directory between builds, so that - # we do not have to re-download all dependencies from maven central for - # each build. The hope is that downloading the cache is faster than - # all dependencies individually. - # In this configuration, we use a hash over all committed (not generated) .pom files - # as a key for the build cache (CACHE_KEY). If we have a cache miss on the hash - # (usually because a pom file has changed), we'll fall back to a key without - # the pom files (CACHE_FALLBACK_KEY). - # Offical documentation of the Cache task: https://docs.microsoft.com/en-us/azure/devops/pipelines/caching/?view=azure-devops - - task: Cache@2 - inputs: - key: $(CACHE_KEY) - restoreKeys: $(CACHE_FALLBACK_KEY) - path: $(MAVEN_CACHE_FOLDER) - continueOnError: true # continue the build even if the cache fails. - displayName: Cache Maven local repo - - script: | - echo "##vso[task.setvariable variable=JAVA_HOME]$JAVA_HOME_${{parameters.jdk}}_X64" - echo "##vso[task.setvariable variable=PATH]$JAVA_HOME_${{parameters.jdk}}_X64/bin:$PATH" - displayName: "Set JDK" - # Compile - - script: | - ./tools/ci/compile.sh self-hosted-pipeline || exit $? - ./tools/azure-pipelines/create_build_artifact.sh - displayName: Compile - - # upload artifacts for next stage - - task: PublishPipelineArtifact@1 - inputs: - targetPath: $(FLINK_ARTIFACT_DIR) - artifact: FlinkCompileArtifact-${{parameters.stage_name}} - - - job: test_${{parameters.stage_name}} - dependsOn: compile_${{parameters.stage_name}} - condition: and(succeeded(), not(eq(variables['MODE'], 'e2e'))) - pool: ${{parameters.test_pool_definition}} - timeoutInMinutes: 40 - cancelTimeoutInMinutes: 1 - workspace: - clean: all - strategy: - matrix: - oceanbase: - module: oceanbase - steps: - # download artifact from compile stage - - task: DownloadPipelineArtifact@2 - inputs: - path: $(FLINK_ARTIFACT_DIR) - artifact: FlinkCompileArtifact-${{parameters.stage_name}} - - - script: ./tools/azure-pipelines/unpack_build_artifact.sh - displayName: "Unpack Build artifact" - - - task: Cache@2 - inputs: - key: $(CACHE_KEY) - restoreKeys: $(CACHE_FALLBACK_KEY) - path: $(MAVEN_CACHE_FOLDER) - continueOnError: true # continue the build even if the cache fails. - condition: not(eq('${{parameters.test_pool_definition.name}}', 'Default')) - displayName: Cache Maven local repo - - - script: | - echo "##vso[task.setvariable variable=JAVA_HOME]$JAVA_HOME_${{parameters.jdk}}_X64" - echo "##vso[task.setvariable variable=PATH]$JAVA_HOME_${{parameters.jdk}}_X64/bin:$PATH" - displayName: "Set JDK" - - - script: sudo sysctl -w kernel.core_pattern=core.%p - displayName: Set coredump pattern - - # Test - - script: ./tools/azure-pipelines/uploading_watchdog.sh ./tools/ci/test_controller.sh $(module) - displayName: Test - $(module) - - - task: PublishTestResults@2 - condition: succeededOrFailed() - inputs: - testResultsFormat: 'JUnit' - - # upload debug artifacts - - task: PublishPipelineArtifact@1 - condition: not(eq('$(DEBUG_FILES_OUTPUT_DIR)', '')) - displayName: Upload Logs - inputs: - targetPath: $(DEBUG_FILES_OUTPUT_DIR) - artifact: logs-${{parameters.stage_name}}-$(DEBUG_FILES_NAME) diff --git a/tools/azure-pipelines/jobs-template.yml b/tools/azure-pipelines/jobs-template.yml index 8025ede6fc7..eb23923a803 100644 --- a/tools/azure-pipelines/jobs-template.yml +++ b/tools/azure-pipelines/jobs-template.yml @@ -52,7 +52,7 @@ jobs: displayName: "Set JDK" # Compile - script: | - ./tools/ci/compile.sh default || exit $? + ./tools/ci/compile.sh || exit $? ./tools/azure-pipelines/create_build_artifact.sh displayName: Compile @@ -144,6 +144,8 @@ jobs: matrix: mysql: module: mysql + oceanbase: + module: oceanbase tidb: module: tidb e2e_2: diff --git a/tools/ci/compile.sh b/tools/ci/compile.sh index e9fa09c8e57..0cb5b8e5d3a 100755 --- a/tools/ci/compile.sh +++ b/tools/ci/compile.sh @@ -40,7 +40,7 @@ echo "========================================================================== EXIT_CODE=0 -run_mvn clean package -Dmaven.javadoc.skip=true -U -DskipTests -P$1 | tee $MVN_CLEAN_COMPILE_OUT +run_mvn clean package -Dmaven.javadoc.skip=true -U -DskipTests | tee $MVN_CLEAN_COMPILE_OUT EXIT_CODE=${PIPESTATUS[0]}