[#3630] feat(trino-connector): Support insert and select operation on…

… Gravitino catalog (#3631) 1. Support insert and select operation on Gravitino catalog 2. Support query optimize 3. Upgrade CI docker environment Fix: #3630 NO UT IT
apache · May 31, 2024 · 25a45e5 · 25a45e5
1 parent 569b096
commit 25a45e5
Show file tree

Hide file tree

Showing 52 changed files with 1,487 additions and 451 deletions.
diff --git a/build.gradle.kts b/build.gradle.kts
@@ -407,8 +407,8 @@ subprojects {
     reports.html.outputLocation.set(file("${rootProject.projectDir}/build/reports/"))
     val skipTests = project.hasProperty("skipTests")
     if (!skipTests) {
+      jvmArgs = listOf("-Xmx2G")
       useJUnitPlatform()
-
       jvmArgs(project.property("extraJvmArgs") as List<*>)
       finalizedBy(tasks.getByName("jacocoTestReport"))
     }

diff --git a/...st/java/com/datastrato/gravitino/catalog/doris/integration/test/CatalogDorisDriverIT.java b/...st/java/com/datastrato/gravitino/catalog/doris/integration/test/CatalogDorisDriverIT.java
@@ -10,7 +10,5 @@
 public class CatalogDorisDriverIT extends CatalogDorisIT {
   public CatalogDorisDriverIT() {
     super();
-    mysqlDriverDownloadUrl =
-        "https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.11/mysql-connector-java-8.0.11.jar";
   }
 }
diff --git a/...src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/CatalogDorisIT.java b/...src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/CatalogDorisIT.java
@@ -18,7 +18,6 @@
 import com.datastrato.gravitino.integration.test.util.AbstractIT;
 import com.datastrato.gravitino.integration.test.util.GravitinoITUtils;
 import com.datastrato.gravitino.integration.test.util.ITUtils;
-import com.datastrato.gravitino.integration.test.util.JdbcDriverDownloader;
 import com.datastrato.gravitino.rel.Column;
 import com.datastrato.gravitino.rel.Schema;
 import com.datastrato.gravitino.rel.SupportsSchemas;
@@ -35,8 +34,6 @@
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Maps;
 import java.io.IOException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.Map;
@@ -58,8 +55,6 @@
 public class CatalogDorisIT extends AbstractIT {
 
   private static final String provider = "jdbc-doris";
-  private static final String DOWNLOAD_JDBC_DRIVER_URL =
-      "https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.27/mysql-connector-java-8.0.27.jar";
 
   private static final String DRIVER_CLASS_NAME = "com.mysql.jdbc.Driver";
 
@@ -88,17 +83,8 @@ public class CatalogDorisIT extends AbstractIT {
 
   protected Catalog catalog;
 
-  protected String mysqlDriverDownloadUrl = DOWNLOAD_JDBC_DRIVER_URL;
-
   @BeforeAll
   public void startup() throws IOException {
-
-    if (!ITUtils.EMBEDDED_TEST_MODE.equals(AbstractIT.testMode)) {
-      String gravitinoHome = System.getenv("GRAVITINO_HOME");
-      Path tmpPath = Paths.get(gravitinoHome, "/catalogs/jdbc-doris/libs");
-      JdbcDriverDownloader.downloadJdbcDriver(mysqlDriverDownloadUrl, tmpPath.toString());
-    }
-
     containerSuite.startDorisContainer();
 
     createMetalake();

diff --git a/...est/java/com/datastrato/gravitino/catalog/mysql/integration/test/AuditCatalogMysqlIT.java b/...est/java/com/datastrato/gravitino/catalog/mysql/integration/test/AuditCatalogMysqlIT.java
@@ -17,8 +17,6 @@
 import com.datastrato.gravitino.integration.test.container.MySQLContainer;
 import com.datastrato.gravitino.integration.test.util.AbstractIT;
 import com.datastrato.gravitino.integration.test.util.GravitinoITUtils;
-import com.datastrato.gravitino.integration.test.util.ITUtils;
-import com.datastrato.gravitino.integration.test.util.JdbcDriverDownloader;
 import com.datastrato.gravitino.integration.test.util.TestDatabaseName;
 import com.datastrato.gravitino.rel.Column;
 import com.datastrato.gravitino.rel.Schema;
@@ -27,8 +25,6 @@
 import com.datastrato.gravitino.rel.types.Types;
 import com.google.common.collect.Maps;
 import java.io.IOException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
 import java.sql.SQLException;
 import java.util.Collections;
 import java.util.Map;
@@ -44,8 +40,6 @@ public class AuditCatalogMysqlIT extends AbstractIT {
   private static final ContainerSuite containerSuite = ContainerSuite.getInstance();
   public static final String metalakeName = GravitinoITUtils.genRandomName("audit_mysql_metalake");
   private static final String expectUser = System.getProperty("user.name");
-  public static final String DOWNLOAD_JDBC_DRIVER_URL =
-      "https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.27/mysql-connector-java-8.0.27.jar";
   protected static TestDatabaseName TEST_DB_NAME;
   private static final String provider = "jdbc-mysql";
 
@@ -60,12 +54,6 @@ public static void startIntegrationTest() throws Exception {
     registerCustomConfigs(configs);
     AbstractIT.startIntegrationTest();
 
-    if (!ITUtils.EMBEDDED_TEST_MODE.equals(testMode)) {
-      String gravitinoHome = System.getenv("GRAVITINO_HOME");
-      Path tmpPath = Paths.get(gravitinoHome, "/catalogs/jdbc-mysql/libs");
-      JdbcDriverDownloader.downloadJdbcDriver(DOWNLOAD_JDBC_DRIVER_URL, tmpPath.toString());
-    }
-
     containerSuite.startMySQLContainer(TestDatabaseName.MYSQL_AUDIT_CATALOG_MYSQL_IT);
     MYSQL_CONTAINER = containerSuite.getMySQLContainer();
     TEST_DB_NAME = TestDatabaseName.MYSQL_AUDIT_CATALOG_MYSQL_IT;

diff --git a/...st/java/com/datastrato/gravitino/catalog/mysql/integration/test/CatalogMysqlDriverIT.java b/...st/java/com/datastrato/gravitino/catalog/mysql/integration/test/CatalogMysqlDriverIT.java
@@ -11,7 +11,5 @@
 public class CatalogMysqlDriverIT extends CatalogMysqlIT {
   public CatalogMysqlDriverIT() {
     super();
-    mysqlDriverDownloadUrl =
-        "https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.11/mysql-connector-java-8.0.11.jar";
   }
 }
diff --git a/...src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/CatalogMysqlIT.java b/...src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/CatalogMysqlIT.java
@@ -23,7 +23,6 @@
 import com.datastrato.gravitino.integration.test.util.AbstractIT;
 import com.datastrato.gravitino.integration.test.util.GravitinoITUtils;
 import com.datastrato.gravitino.integration.test.util.ITUtils;
-import com.datastrato.gravitino.integration.test.util.JdbcDriverDownloader;
 import com.datastrato.gravitino.integration.test.util.TestDatabaseName;
 import com.datastrato.gravitino.rel.Column;
 import com.datastrato.gravitino.rel.Column.ColumnImpl;
@@ -47,8 +46,6 @@
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Maps;
 import java.io.IOException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
 import java.sql.SQLException;
 import java.util.Arrays;
 import java.util.Collections;
@@ -72,8 +69,6 @@
 public class CatalogMysqlIT extends AbstractIT {
   private static final ContainerSuite containerSuite = ContainerSuite.getInstance();
   private static final String provider = "jdbc-mysql";
-  public static final String DOWNLOAD_JDBC_DRIVER_URL =
-      "https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.27/mysql-connector-java-8.0.27.jar";
 
   public String metalakeName = GravitinoITUtils.genRandomName("mysql_it_metalake");
   public String catalogName = GravitinoITUtils.genRandomName("mysql_it_catalog");
@@ -104,21 +99,12 @@ public class CatalogMysqlIT extends AbstractIT {
 
   protected String mysqlImageName = defaultMysqlImageName;
 
-  protected String mysqlDriverDownloadUrl = DOWNLOAD_JDBC_DRIVER_URL;
-
   boolean SupportColumnDefaultValueExpression() {
     return true;
   }
 
   @BeforeAll
   public void startup() throws IOException, SQLException {
-
-    if (!ITUtils.EMBEDDED_TEST_MODE.equals(testMode)) {
-      String gravitinoHome = System.getenv("GRAVITINO_HOME");
-      Path tmpPath = Paths.get(gravitinoHome, "/catalogs/jdbc-mysql/libs");
-      JdbcDriverDownloader.downloadJdbcDriver(mysqlDriverDownloadUrl, tmpPath.toString());
-    }
-
     TEST_DB_NAME = TestDatabaseName.MYSQL_CATALOG_MYSQL_IT;
 
     if (mysqlImageName.equals("mysql:5.7")) {

diff --git a/...ava/com/datastrato/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlIT.java b/...ava/com/datastrato/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlIT.java
@@ -22,7 +22,6 @@
 import com.datastrato.gravitino.integration.test.util.AbstractIT;
 import com.datastrato.gravitino.integration.test.util.GravitinoITUtils;
 import com.datastrato.gravitino.integration.test.util.ITUtils;
-import com.datastrato.gravitino.integration.test.util.JdbcDriverDownloader;
 import com.datastrato.gravitino.integration.test.util.TestDatabaseName;
 import com.datastrato.gravitino.rel.Column;
 import com.datastrato.gravitino.rel.Schema;
@@ -46,8 +45,6 @@
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Maps;
 import java.io.IOException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
 import java.sql.DriverManager;
 import java.sql.SQLException;
 import java.util.Arrays;
@@ -72,8 +69,6 @@
 public class CatalogPostgreSqlIT extends AbstractIT {
   private static final ContainerSuite containerSuite = ContainerSuite.getInstance();
   public static final PGImageName DEFAULT_POSTGRES_IMAGE = PGImageName.VERSION_13;
-  public static final String DOWNLOAD_JDBC_DRIVER_URL =
-      "https://jdbc.postgresql.org/download/postgresql-42.7.0.jar";
 
   public String metalakeName = GravitinoITUtils.genRandomName("postgresql_it_metalake");
   public String catalogName = GravitinoITUtils.genRandomName("postgresql_it_catalog");
@@ -101,12 +96,6 @@ public class CatalogPostgreSqlIT extends AbstractIT {
 
   @BeforeAll
   public void startup() throws IOException, SQLException {
-
-    if (!ITUtils.EMBEDDED_TEST_MODE.equals(testMode)) {
-      String gravitinoHome = System.getenv("GRAVITINO_HOME");
-      Path tmpPath = Paths.get(gravitinoHome, "/catalogs/jdbc-postgresql/libs");
-      JdbcDriverDownloader.downloadJdbcDriver(DOWNLOAD_JDBC_DRIVER_URL, tmpPath.toString());
-    }
     containerSuite.startPostgreSQLContainer(TEST_DB_NAME, postgreImageName);
     POSTGRESQL_CONTAINER = containerSuite.getPostgreSQLContainer(postgreImageName);
 

diff --git a/...va/com/datastrato/gravitino/catalog/postgresql/integration/test/TestMultipleJDBCLoad.java b/...va/com/datastrato/gravitino/catalog/postgresql/integration/test/TestMultipleJDBCLoad.java
@@ -14,17 +14,13 @@
 import com.datastrato.gravitino.integration.test.container.MySQLContainer;
 import com.datastrato.gravitino.integration.test.container.PostgreSQLContainer;
 import com.datastrato.gravitino.integration.test.util.AbstractIT;
-import com.datastrato.gravitino.integration.test.util.ITUtils;
-import com.datastrato.gravitino.integration.test.util.JdbcDriverDownloader;
 import com.datastrato.gravitino.integration.test.util.TestDatabaseName;
 import com.datastrato.gravitino.rel.Column;
 import com.datastrato.gravitino.rel.types.Types;
 import com.datastrato.gravitino.utils.RandomNameUtils;
 import com.google.common.collect.Maps;
 import java.io.IOException;
 import java.net.URISyntaxException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
 import java.sql.SQLException;
 import java.util.Collections;
 import java.util.Map;
@@ -42,38 +38,8 @@ public class TestMultipleJDBCLoad extends AbstractIT {
   private static MySQLContainer mySQLContainer;
   private static PostgreSQLContainer postgreSQLContainer;
 
-  private static final String DOWNLOAD_JDBC_DRIVER_URL =
-      "https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.27/mysql-connector-java-8.0.27.jar";
-
   @BeforeAll
   public static void startup() throws IOException {
-    String gravitinoHome = System.getenv("GRAVITINO_HOME");
-
-    // Deploy mode
-    if (!ITUtils.EMBEDDED_TEST_MODE.equals(testMode)) {
-      Path icebergLibsPath = Paths.get(gravitinoHome, "/catalogs/lakehouse-iceberg/libs");
-      Path pgDirPath = Paths.get(gravitinoHome, "/catalogs/jdbc-postgresql/libs");
-      JdbcDriverDownloader.downloadJdbcDriver(
-          CatalogPostgreSqlIT.DOWNLOAD_JDBC_DRIVER_URL,
-          pgDirPath.toString(),
-          icebergLibsPath.toString());
-
-      JdbcDriverDownloader.downloadJdbcDriver(
-          DOWNLOAD_JDBC_DRIVER_URL, pgDirPath.toString(), icebergLibsPath.toString());
-    } else {
-      // embedded mode
-      Path icebergLibsPath =
-          Paths.get(gravitinoHome, "/catalogs/catalog-lakehouse-iceberg/build/libs");
-      Path pgDirPath = Paths.get(gravitinoHome, "/catalogs/catalog-jdbc-postgresql/build/libs");
-      JdbcDriverDownloader.downloadJdbcDriver(
-          CatalogPostgreSqlIT.DOWNLOAD_JDBC_DRIVER_URL,
-          icebergLibsPath.toString(),
-          pgDirPath.toString());
-
-      JdbcDriverDownloader.downloadJdbcDriver(
-          DOWNLOAD_JDBC_DRIVER_URL, pgDirPath.toString(), icebergLibsPath.toString());
-    }
-
     containerSuite.startMySQLContainer(TEST_DB_NAME);
     mySQLContainer = containerSuite.getMySQLContainer();
     containerSuite.startPostgreSQLContainer(TEST_DB_NAME);

diff --git a/...datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/TestMultipleJDBCLoad.java b/...datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/TestMultipleJDBCLoad.java
@@ -16,17 +16,13 @@
 import com.datastrato.gravitino.integration.test.container.MySQLContainer;
 import com.datastrato.gravitino.integration.test.container.PostgreSQLContainer;
 import com.datastrato.gravitino.integration.test.util.AbstractIT;
-import com.datastrato.gravitino.integration.test.util.ITUtils;
-import com.datastrato.gravitino.integration.test.util.JdbcDriverDownloader;
 import com.datastrato.gravitino.integration.test.util.TestDatabaseName;
 import com.datastrato.gravitino.rel.Column;
 import com.datastrato.gravitino.rel.types.Types;
 import com.datastrato.gravitino.utils.RandomNameUtils;
 import com.google.common.collect.Maps;
 import java.io.IOException;
 import java.net.URISyntaxException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
 import java.sql.SQLException;
 import java.util.Collections;
 import java.util.Map;
@@ -43,34 +39,10 @@ public class TestMultipleJDBCLoad extends AbstractIT {
   private static MySQLContainer mySQLContainer;
   private static PostgreSQLContainer postgreSQLContainer;
 
-  private static final String DOWNLOAD_MYSQL_JDBC_DRIVER_URL =
-      "https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.27/mysql-connector-java-8.0.27.jar";
   public static final String DEFAULT_POSTGRES_IMAGE = "postgres:13";
-  public static final String DOWNLOAD_PG_JDBC_DRIVER_URL =
-      "https://jdbc.postgresql.org/download/postgresql-42.7.0.jar";
 
   @BeforeAll
   public static void startup() throws IOException {
-    String gravitinoHome = System.getenv("GRAVITINO_HOME");
-
-    // Deploy mode
-    if (!ITUtils.EMBEDDED_TEST_MODE.equals(testMode)) {
-      Path icebergLibsPath = Paths.get(gravitinoHome, "/catalogs/lakehouse-iceberg/libs");
-      JdbcDriverDownloader.downloadJdbcDriver(
-          DOWNLOAD_MYSQL_JDBC_DRIVER_URL, icebergLibsPath.toString());
-      JdbcDriverDownloader.downloadJdbcDriver(
-          DOWNLOAD_PG_JDBC_DRIVER_URL, icebergLibsPath.toString());
-    } else {
-      // embedded mode
-      Path icebergLibsPath =
-          Paths.get(gravitinoHome, "/catalogs/catalog-lakehouse-iceberg/build/libs");
-      JdbcDriverDownloader.downloadJdbcDriver(
-          DOWNLOAD_MYSQL_JDBC_DRIVER_URL, icebergLibsPath.toString());
-
-      JdbcDriverDownloader.downloadJdbcDriver(
-          DOWNLOAD_PG_JDBC_DRIVER_URL, icebergLibsPath.toString());
-    }
-
     containerSuite.startMySQLContainer(TEST_DB_NAME);
     mySQLContainer = containerSuite.getMySQLContainer();
     containerSuite.startPostgreSQLContainer(TEST_DB_NAME);

diff --git a/...-test-common/src/test/java/com/datastrato/gravitino/integration/test/util/AbstractIT.java b/...-test-common/src/test/java/com/datastrato/gravitino/integration/test/util/AbstractIT.java
@@ -61,9 +61,12 @@ public class AbstractIT {
 
   protected static boolean ignoreIcebergRestService = true;
 
-  private static final String DOWNLOAD_JDBC_DRIVER_URL =
+  public static final String DOWNLOAD_MYSQL_JDBC_DRIVER_URL =
       "https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.26/mysql-connector-java-8.0.26.jar";
 
+  public static final String DOWNLOAD_POSTGRESQL_JDBC_DRIVER_URL =
+      "https://jdbc.postgresql.org/download/postgresql-42.7.0.jar";
+
   private static TestDatabaseName META_DATA;
   private static MySQLContainer MYSQL_CONTAINER;
 
@@ -105,11 +108,24 @@ private static void recoverGravitinoServerConfig() throws IOException {
     Files.move(tmpPath, configPath);
   }
 
-  protected static void downLoadMySQLDriver(String relativeDeployLibsPath) throws IOException {
+  protected static void downLoadJDBCDriver() throws IOException {
+    String gravitinoHome = System.getenv("GRAVITINO_HOME");
     if (!ITUtils.EMBEDDED_TEST_MODE.equals(testMode)) {
-      String gravitinoHome = System.getenv("GRAVITINO_HOME");
-      java.nio.file.Path tmpPath = Paths.get(gravitinoHome, relativeDeployLibsPath);
-      JdbcDriverDownloader.downloadJdbcDriver(DOWNLOAD_JDBC_DRIVER_URL, tmpPath.toString());
+      String serverPath = ITUtils.joinPath(gravitinoHome, "libs");
+      String icebergCatalogPath =
+          ITUtils.joinPath(gravitinoHome, "catalogs", "lakehouse-iceberg", "libs");
+      JdbcDriverDownloader.downloadJdbcDriver(
+          DOWNLOAD_MYSQL_JDBC_DRIVER_URL, serverPath, icebergCatalogPath);
+      JdbcDriverDownloader.downloadJdbcDriver(
+          DOWNLOAD_POSTGRESQL_JDBC_DRIVER_URL, serverPath, icebergCatalogPath);
+    } else {
+      Path icebergLibsPath =
+          Paths.get(gravitinoHome, "catalogs", "catalog-lakehouse-iceberg", "build", "libs");
+      JdbcDriverDownloader.downloadJdbcDriver(
+          DOWNLOAD_MYSQL_JDBC_DRIVER_URL, icebergLibsPath.toString());
+
+      JdbcDriverDownloader.downloadJdbcDriver(
+          DOWNLOAD_POSTGRESQL_JDBC_DRIVER_URL, icebergLibsPath.toString());
     }
   }
 
@@ -185,7 +201,7 @@ public static void startIntegrationTest() throws Exception {
     } else {
       rewriteGravitinoServerConfig();
       serverConfig.loadFromFile(GravitinoServer.CONF_FILE);
-      downLoadMySQLDriver("/libs");
+      downLoadJDBCDriver();
       try {
         FileUtils.deleteDirectory(
             FileUtils.getFile(serverConfig.get(ENTRY_KV_ROCKSDB_BACKEND_PATH)));

diff --git a/...-test/src/test/java/com/datastrato/gravitino/integration/test/trino/TrinoConnectorIT.java b/...-test/src/test/java/com/datastrato/gravitino/integration/test/trino/TrinoConnectorIT.java
@@ -14,8 +14,6 @@
 import com.datastrato.gravitino.integration.test.container.TrinoContainer;
 import com.datastrato.gravitino.integration.test.util.AbstractIT;
 import com.datastrato.gravitino.integration.test.util.GravitinoITUtils;
-import com.datastrato.gravitino.integration.test.util.ITUtils;
-import com.datastrato.gravitino.integration.test.util.JdbcDriverDownloader;
 import com.datastrato.gravitino.rel.Column;
 import com.datastrato.gravitino.rel.Schema;
 import com.datastrato.gravitino.rel.Table;
@@ -105,26 +103,6 @@ public static void startDockerContainer() throws IOException, TException, Interr
         "Can not synchronize calatogs from gravitino");
 
     createSchema();
-
-    String testMode =
-        System.getProperty(ITUtils.TEST_MODE) == null
-            ? ITUtils.EMBEDDED_TEST_MODE
-            : System.getProperty(ITUtils.TEST_MODE);
-
-    // Deploy mode, you should download jars to the Gravitino server iceberg lib directory
-    if (!ITUtils.EMBEDDED_TEST_MODE.equals(testMode)) {
-      String gravitinoHome = System.getenv("GRAVITINO_HOME");
-      String destPath = ITUtils.joinPath(gravitinoHome, "catalogs", "lakehouse-iceberg", "libs");
-      String mysqlPath = ITUtils.joinPath(gravitinoHome, "catalogs", "jdbc-mysql", "libs");
-      String pgPath = ITUtils.joinPath(gravitinoHome, "catalogs", "jdbc-postgresql", "libs");
-
-      JdbcDriverDownloader.downloadJdbcDriver(
-          "https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.27/mysql-connector-java-8.0.27.jar",
-          destPath,
-          mysqlPath);
-      JdbcDriverDownloader.downloadJdbcDriver(
-          "https://jdbc.postgresql.org/download/postgresql-42.7.0.jar", destPath, pgPath);
-    }
   }
 
   @AfterAll