From bc03d5a48849836faf5a0aab78d9ba01d4d2a699 Mon Sep 17 00:00:00 2001 From: mchades Date: Mon, 24 Jun 2024 12:17:14 +0800 Subject: [PATCH 1/2] add profile info --- .github/workflows/backend-integration-test.yml | 2 +- .../integration/test/container/DorisContainer.java | 8 ++++++++ .../integration/test/container/HiveContainer.java | 7 +++++++ .../integration/test/container/KafkaContainer.java | 8 ++++++++ .../integration/test/container/MySQLContainer.java | 7 +++++++ .../integration/test/container/PostgreSQLContainer.java | 7 +++++++ .../integration/test/container/RangerContainer.java | 8 ++++++++ .../integration/test/container/TrinoContainer.java | 8 ++++++++ .../integration/test/container/TrinoITContainers.java | 6 ++++++ .../gravitino/integration/test/util/AbstractIT.java | 6 ++++++ 10 files changed, 66 insertions(+), 1 deletion(-) diff --git a/.github/workflows/backend-integration-test.yml b/.github/workflows/backend-integration-test.yml index 2ba80aec413..e80855a4d8f 100644 --- a/.github/workflows/backend-integration-test.yml +++ b/.github/workflows/backend-integration-test.yml @@ -99,7 +99,7 @@ jobs: ./gradlew test -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -P${{ matrix.backend }} -PskipWebITs -x :web:test -x :clients:client-python:test -x :flink-connector:test -x :spark-connector:test -x :spark-connector:spark-common:test -x :spark-connector:spark-3.3:test -x :spark-connector:spark-3.4:test -x :spark-connector:spark-3.5:test - -x :spark-connector:spark-runtime-3.3:test -x :spark-connector:spark-runtime-3.4:test -x :spark-connector:spark-runtime-3.5:test + -x :spark-connector:spark-runtime-3.3:test -x :spark-connector:spark-runtime-3.4:test -x :spark-connector:spark-runtime-3.5:test --profile - name: Upload integrate tests reports uses: actions/upload-artifact@v3 diff --git a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/DorisContainer.java b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/DorisContainer.java index f173b060836..2db7147b709 100644 --- a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/DorisContainer.java +++ b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/DorisContainer.java @@ -60,9 +60,17 @@ protected void setupContainer() { @Override public void start() { + LOG.info("starting Doris container..."); + long currentTime = System.currentTimeMillis(); + super.start(); Preconditions.check("Doris container startup failed!", checkContainerStatus(5)); Preconditions.check("Doris container password change failed!", changePassword()); + + LOG.info( + "Doris[{}] start successfully in {}ms", + container.getDockerImageName(), + System.currentTimeMillis() - currentTime); } @Override diff --git a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/HiveContainer.java b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/HiveContainer.java index 76c74bc3ab5..8986693f43f 100644 --- a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/HiveContainer.java +++ b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/HiveContainer.java @@ -62,8 +62,15 @@ protected void setupContainer() { @Override public void start() { + LOG.info("starting Hive container..."); + long currentTime = System.currentTimeMillis(); + super.start(); Preconditions.check("Hive container startup failed!", checkContainerStatus(15)); + LOG.info( + "Hive container[{}] start successfully in {}ms", + container.getDockerImageName(), + System.currentTimeMillis() - currentTime); } @Override diff --git a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/KafkaContainer.java b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/KafkaContainer.java index ecd9ea5c6c4..dfc38dc640a 100644 --- a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/KafkaContainer.java +++ b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/KafkaContainer.java @@ -47,8 +47,16 @@ public static Builder builder() { @Override public void start() { + LOG.info("starting Kafka container..."); + long currentTime = System.currentTimeMillis(); + super.start(); Preconditions.checkArgument(checkContainerStatus(5), "Kafka container startup failed!"); + + LOG.info( + "Kafka container[{}] start successfully in {}ms", + container.getDockerImageName(), + System.currentTimeMillis() - currentTime); } @Override diff --git a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/MySQLContainer.java b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/MySQLContainer.java index a83d88f2fb0..375b32deda4 100644 --- a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/MySQLContainer.java +++ b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/MySQLContainer.java @@ -56,8 +56,15 @@ protected void setupContainer() { @Override public void start() { + LOG.info("starting MySQL container..."); + long currentTime = System.currentTimeMillis(); + super.start(); Preconditions.check("MySQL container startup failed!", checkContainerStatus(5)); + LOG.info( + "MySQL container[{}] start successfully in {}ms", + container.getDockerImageName(), + System.currentTimeMillis() - currentTime); } @Override diff --git a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/PostgreSQLContainer.java b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/PostgreSQLContainer.java index b7b50e5eb24..0ab70e9af66 100644 --- a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/PostgreSQLContainer.java +++ b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/PostgreSQLContainer.java @@ -54,8 +54,15 @@ protected void setupContainer() { @Override public void start() { + LOG.info("starting Ranger container..."); + long currentTime = System.currentTimeMillis(); + super.start(); Preconditions.check("PostgreSQL container startup failed!", checkContainerStatus(5)); + LOG.info( + "PostgreSQL container[{}] start successfully in {}ms", + container.getDockerImageName(), + System.currentTimeMillis() - currentTime); } @Override diff --git a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/RangerContainer.java b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/RangerContainer.java index 4eccbdfadb7..7568fa07e7a 100644 --- a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/RangerContainer.java +++ b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/RangerContainer.java @@ -58,12 +58,20 @@ protected void setupContainer() { @Override public void start() { + LOG.info("starting Ranger container..."); + long currentTime = System.currentTimeMillis(); + super.start(); rangerUrl = String.format("http://localhost:%s", this.getMappedPort(6080)); rangerClient = new RangerClient(rangerUrl, authType, username, password, null); Preconditions.check("Ranger container startup failed!", checkContainerStatus(10)); + + LOG.info( + "Ranger container[{}] start successfully in {}ms", + container.getDockerImageName(), + System.currentTimeMillis() - currentTime); } @Override diff --git a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/TrinoContainer.java b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/TrinoContainer.java index 354dfd3eb67..942507f7091 100644 --- a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/TrinoContainer.java +++ b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/TrinoContainer.java @@ -59,10 +59,18 @@ protected void setupContainer() { @Override public void start() { + LOG.info("starting Trino container..."); + long currentTime = System.currentTimeMillis(); + super.start(); Preconditions.check("Initialization Trino JDBC connect failed!", initTrinoJdbcConnection()); Preconditions.check("Trino container startup failed!", checkContainerStatus(5)); + + LOG.info( + "Trino container[{}] start successfully in {}ms", + container.getDockerImageName(), + System.currentTimeMillis() - currentTime); } @Override diff --git a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/TrinoITContainers.java b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/TrinoITContainers.java index 6c3805cd10d..68f645b15ce 100644 --- a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/TrinoITContainers.java +++ b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/TrinoITContainers.java @@ -35,6 +35,9 @@ public class TrinoITContainers implements AutoCloseable { } public void launch(int gravitinoServerPort) throws Exception { + LOG.info("starting Trino IT containers..."); + long currentTime = System.currentTimeMillis(); + shutdown(); Map env = new HashMap<>(); @@ -56,6 +59,9 @@ public void launch(int gravitinoServerPort) throws Exception { } resolveServerAddress(); + + LOG.info( + "Trino IT containers start successfully in {}ms", System.currentTimeMillis() - currentTime); } private void resolveServerAddress() throws Exception { diff --git a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/util/AbstractIT.java b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/util/AbstractIT.java index 623abda2d6c..5488b74af33 100644 --- a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/util/AbstractIT.java +++ b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/util/AbstractIT.java @@ -202,6 +202,7 @@ public static void startIntegrationTest() throws Exception { } serverConfig = new ServerConfig(); + long currentTime = System.currentTimeMillis(); if (testMode != null && testMode.equals(ITUtils.EMBEDDED_TEST_MODE)) { MiniGravitinoContext context = new MiniGravitinoContext(customConfigs, ignoreIcebergRestService); @@ -235,6 +236,11 @@ public static void startIntegrationTest() throws Exception { .until(() -> isHttpServerUp(checkServerUrl)); } + LOG.info( + "Gravitino {} server start successfully in {}ms", + testMode, + System.currentTimeMillis() - currentTime); + JettyServerConfig jettyServerConfig = JettyServerConfig.fromConfig(serverConfig, WEBSERVER_CONF_PREFIX); From 5acfe5300a5ad892ab80c8bc1500c6aaead6b601 Mon Sep 17 00:00:00 2001 From: mchades Date: Tue, 25 Jun 2024 18:01:59 +0800 Subject: [PATCH 2/2] add containers to docker compose --- integration-test/trino-it/docker-compose.yaml | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/integration-test/trino-it/docker-compose.yaml b/integration-test/trino-it/docker-compose.yaml index 305ad143aa1..ceb1c57464a 100644 --- a/integration-test/trino-it/docker-compose.yaml +++ b/integration-test/trino-it/docker-compose.yaml @@ -4,6 +4,37 @@ # version: '3.0' services: + hive-kerberos: + image: datastrato/gravitino-ci-kerberos-hive:0.1.2 + networks: + - trino-net + container_name: trino-ci-hive-kerberos + environment: + - HADOOP_USER_NAME=root + entrypoint: /bin/bash /tmp/hive/init.sh + volumes: + - ./init/hive:/tmp/hive + - ../build/trino-ci-container-log/hive-kerberos:/tmp/root + - ../build/trino-ci-container-log/hdfs-kerberos:/usr/local/hadoop/logs + healthcheck: + test: ["CMD", "/tmp/check-status.sh"] + interval: 10s + timeout: 60s + retries: 10 + + doris: + image: datastrato/gravitino-ci-doris:0.1.4 + networks: + - trino-net + container_name: trino-ci-doris + volumes: + - ../build/trino-ci-container-log/doris/fe:/opt/apache-doris/fe/log + - ../build/trino-ci-container-log/doris/be:/opt/apache-doris/be/log + healthcheck: + test: ["CMD", "mysql", "-uroot", "-P9030", "-h127.0.0.1", "-e", "select 1"] + interval: 10s + timeout: 60s + retries: 10 hive: image: datastrato/gravitino-ci-hive:0.1.12