Skip to content

Commit

Permalink
[oceanbase][ci] Move the OceanBase CI to free azure pipeline (apache#…
Browse files Browse the repository at this point in the history 
…2506)

Move OceanBase CI to free azure pipeline with new docker image of OceanBase and remove redundant maven profile.
  • Loading branch information
@whhe
whhe committed Nov 14, 2023
1 parent 3162299 commit 0e6d31d
Show file tree
Hide file tree
Showing 7 changed files with 55 additions and 242 deletions.
7 changes: 0 additions & 7 deletions azure-pipelines.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,10 +45,3 @@ stages:
vmImage: 'ubuntu-20.04'
run_end_to_end: false
jdk: 8
- template: tools/azure-pipelines/jobs-template-for-self-hosted-agent.yml
parameters: # see template file for a definition of the parameters.
stage_name: ci_build_on_self_hosted_agent
test_pool_definition:
name: Flink_CDC_CI
run_end_to_end: false
jdk: 8
60 changes: 18 additions & 42 deletions ...oceanbase-cdc/src/test/java/com/ververica/cdc/connectors/oceanbase/OceanBaseTestBase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,71 +56,57 @@ public class OceanBaseTestBase extends TestLogger {
private static final Pattern COMMENT_PATTERN = Pattern.compile("^(.*)--.*$");
private static final Duration CONTAINER_STARTUP_TIMEOUT = Duration.ofMinutes(4);

public static final int OB_SERVER_SQL_PORT = 2881;
public static final int OB_SERVER_RPC_PORT = 2882;
public static final int LOG_PROXY_PORT = 2983;

public static final String OB_SYS_PASSWORD = "pswd";
public static final String OB_TEST_PASSWORD = "test";

public static final String NETWORK_MODE = "host";

// --------------------------------------------------------------------------------------------
// Attributes about host and port when network is on 'host' mode.
// --------------------------------------------------------------------------------------------

protected static String getObServerHost() {
return "127.0.0.1";
}

protected static String getLogProxyHost() {
return "127.0.0.1";
}

protected static int getObServerSqlPort() {
return OB_SERVER_SQL_PORT;
return 2881;
}

protected static int getObServerRpcPort() {
return OB_SERVER_RPC_PORT;
protected static int getLogProxyPort() {
return 2983;
}

protected static int getLogProxyPort() {
return LOG_PROXY_PORT;
public static String getRsList() {
return "127.0.0.1:2882:2881";
}

// --------------------------------------------------------------------------------------------
// Attributes about user.
// From OceanBase 4.0.0.0 CE, we can only fetch the commit log of non-sys tenant.
// --------------------------------------------------------------------------------------------

public static final String OB_SYS_PASSWORD = "pswd";

protected static String getTenant() {
return "test";
}

protected static String getUsername() {
return "root@test";
return "root@" + getTenant();
}

protected static String getPassword() {
return OB_TEST_PASSWORD;
return "test";
}

@ClassRule
public static final GenericContainer<?> OB_SERVER =
new GenericContainer<>("oceanbase/oceanbase-ce:4.0.0.0")
new GenericContainer<>("oceanbase/oceanbase-ce:4.2.0.0")
.withNetworkMode(NETWORK_MODE)
.withExposedPorts(OB_SERVER_SQL_PORT, OB_SERVER_RPC_PORT)
.withEnv("MODE", "slim")
.withEnv("OB_ROOT_PASSWORD", OB_SYS_PASSWORD)
.waitingFor(Wait.forLogMessage(".*boot success!.*", 1))
.withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)
.withLogConsumer(new Slf4jLogConsumer(LOG));

@ClassRule
public static final GenericContainer<?> LOG_PROXY =
new GenericContainer<>("whhe/oblogproxy:1.1.0_4x")
new GenericContainer<>("whhe/oblogproxy:1.1.3_4x")
.withNetworkMode(NETWORK_MODE)
.withExposedPorts(LOG_PROXY_PORT)
.withEnv("OB_SYS_USERNAME", "root")
.withEnv("OB_SYS_PASSWORD", OB_SYS_PASSWORD)
.waitingFor(Wait.forLogMessage(".*boot success!.*", 1))
.withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)
Expand All @@ -132,17 +118,12 @@ public static void startContainers() {
Startables.deepStart(Stream.of(OB_SERVER, LOG_PROXY)).join();
LOG.info("Containers are started.");

setPassword("root@sys", OB_SYS_PASSWORD);
setPassword("root@test", OB_TEST_PASSWORD);
}

private static void setPassword(String username, String password) {
try (Connection connection = DriverManager.getConnection(getJdbcUrl(""), username, "");
try (Connection connection =
DriverManager.getConnection(getJdbcUrl(""), getUsername(), "");
Statement statement = connection.createStatement()) {
statement.execute(String.format("ALTER USER root IDENTIFIED BY '%s'", password));
LOG.info("Set password of {} to {}", username, password);
statement.execute(String.format("ALTER USER root IDENTIFIED BY '%s'", getPassword()));
} catch (SQLException e) {
LOG.error("Set password of {} failed.", username, e);
LOG.error("Set test user password failed.", e);
throw new RuntimeException(e);
}
}
Expand All @@ -156,19 +137,14 @@ public static void stopContainers() {

public static String getJdbcUrl(String databaseName) {
return "jdbc:mysql://"
+ getObServerHost()
+ OB_SERVER.getHost()
+ ":"
+ getObServerSqlPort()
+ "/"
+ databaseName
+ "?useSSL=false";
}

public static String getRsList() {
return String.format(
"%s:%s:%s", getObServerHost(), getObServerRpcPort(), getObServerSqlPort());
}

protected static Connection getJdbcConnection(String databaseName) throws SQLException {
return DriverManager.getConnection(getJdbcUrl(databaseName), getUsername(), getPassword());
}
Expand Down
16 changes: 8 additions & 8 deletions .../src/test/java/com/ververica/cdc/connectors/oceanbase/table/OceanBaseConnectorITCase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,9 +104,9 @@ public void testTableList() throws Exception {
getPassword(),
getTenant(),
"inventory.products",
getObServerHost(),
OB_SERVER.getHost(),
getObServerSqlPort(),
getLogProxyHost(),
LOG_PROXY.getHost(),
getLogProxyPort(),
getRsList());

Expand Down Expand Up @@ -230,9 +230,9 @@ public void testMetadataColumns() throws Exception {
getTenant(),
"^inventory_meta$",
"^products$",
getObServerHost(),
OB_SERVER.getHost(),
getObServerSqlPort(),
getLogProxyHost(),
LOG_PROXY.getHost(),
getLogProxyPort(),
getRsList());

Expand Down Expand Up @@ -382,9 +382,9 @@ public void testAllDataTypes() throws Exception {
"^column_type_test$",
"^full_types$",
serverTimeZone,
getObServerHost(),
OB_SERVER.getHost(),
getObServerSqlPort(),
getLogProxyHost(),
LOG_PROXY.getHost(),
getLogProxyPort(),
getRsList());
String sinkDDL =
Expand Down Expand Up @@ -512,9 +512,9 @@ public void testTimeDataTypes(String serverTimeZone) throws Exception {
"column_type_test",
"full_types",
serverTimeZone,
getObServerHost(),
OB_SERVER.getHost(),
getObServerSqlPort(),
getLogProxyHost(),
LOG_PROXY.getHost(),
getLogProxyPort(),
getRsList());

Expand Down
89 changes: 25 additions & 64 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,31 @@ under the License.
<version>2.4-SNAPSHOT</version>
<packaging>pom</packaging>

<modules>
<module>flink-cdc-base</module>
<module>flink-connector-db2-cdc</module>
<module>flink-connector-debezium</module>
<module>flink-connector-mongodb-cdc</module>
<module>flink-connector-mysql-cdc</module>
<module>flink-connector-oceanbase-cdc</module>
<module>flink-connector-oracle-cdc</module>
<module>flink-connector-postgres-cdc</module>
<module>flink-connector-sqlserver-cdc</module>
<module>flink-connector-test-util</module>
<module>flink-connector-tidb-cdc</module>
<module>flink-connector-vitess-cdc</module>
<module>flink-sql-connector-db2-cdc</module>
<module>flink-sql-connector-mongodb-cdc</module>
<module>flink-sql-connector-mysql-cdc</module>
<module>flink-sql-connector-oceanbase-cdc</module>
<module>flink-sql-connector-oracle-cdc</module>
<module>flink-sql-connector-postgres-cdc</module>
<module>flink-sql-connector-sqlserver-cdc</module>
<module>flink-sql-connector-tidb-cdc</module>
<module>flink-sql-connector-vitess-cdc</module>
<module>flink-cdc-e2e-tests</module>
</modules>

<licenses>
<license>
<name>The Apache Software License, Version 2.0</name>
Expand Down Expand Up @@ -454,62 +479,8 @@ under the License.
</build>

<profiles>
<profile>
<id>default</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<modules>
<module>flink-cdc-base</module>
<module>flink-connector-debezium</module>
<module>flink-connector-test-util</module>
<module>flink-connector-mysql-cdc</module>
<module>flink-connector-postgres-cdc</module>
<module>flink-connector-oracle-cdc</module>
<module>flink-connector-mongodb-cdc</module>
<module>flink-connector-oceanbase-cdc</module>
<module>flink-connector-sqlserver-cdc</module>
<module>flink-connector-tidb-cdc</module>
<module>flink-connector-db2-cdc</module>
<module>flink-connector-vitess-cdc</module>
<module>flink-sql-connector-mysql-cdc</module>
<module>flink-sql-connector-postgres-cdc</module>
<module>flink-sql-connector-mongodb-cdc</module>
<module>flink-sql-connector-oracle-cdc</module>
<module>flink-sql-connector-oceanbase-cdc</module>
<module>flink-sql-connector-sqlserver-cdc</module>
<module>flink-sql-connector-tidb-cdc</module>
<module>flink-sql-connector-db2-cdc</module>
<module>flink-sql-connector-vitess-cdc</module>
<module>flink-cdc-e2e-tests</module>
</modules>
</profile>
<profile>
<id>release</id>
<modules>
<module>flink-cdc-base</module>
<module>flink-connector-debezium</module>
<module>flink-connector-test-util</module>
<module>flink-connector-mysql-cdc</module>
<module>flink-connector-postgres-cdc</module>
<module>flink-connector-oracle-cdc</module>
<module>flink-connector-mongodb-cdc</module>
<module>flink-connector-oceanbase-cdc</module>
<module>flink-connector-sqlserver-cdc</module>
<module>flink-connector-tidb-cdc</module>
<module>flink-connector-db2-cdc</module>
<module>flink-connector-vitess-cdc</module>
<module>flink-sql-connector-mysql-cdc</module>
<module>flink-sql-connector-postgres-cdc</module>
<module>flink-sql-connector-mongodb-cdc</module>
<module>flink-sql-connector-oracle-cdc</module>
<module>flink-sql-connector-oceanbase-cdc</module>
<module>flink-sql-connector-sqlserver-cdc</module>
<module>flink-sql-connector-tidb-cdc</module>
<module>flink-sql-connector-db2-cdc</module>
<module>flink-sql-connector-vitess-cdc</module>
<module>flink-cdc-e2e-tests</module>
</modules>
<build>
<plugins>
<plugin>
Expand Down Expand Up @@ -612,15 +583,5 @@ under the License.
</pluginManagement>
</build>
</profile>
<profile>
<id>self-hosted-pipeline</id>
<modules>
<module>flink-cdc-base</module>
<module>flink-connector-debezium</module>
<module>flink-connector-test-util</module>
<module>flink-connector-oceanbase-cdc</module>
<module>flink-sql-connector-oceanbase-cdc</module>
</modules>
</profile>
</profiles>
</project>
Loading

0 comments on commit 0e6d31d

Please sign in to comment.