Revert "[apache#3524] feat(trino-connector): Load catalog by Trino dy…

…namic catalog feature" (apache#3677) Reverts apache#3664
diqiu50 · Jun 13, 2024 · 842fc5e · 842fc5e
1 parent b5af51d
commit 842fc5e
Show file tree

Hide file tree

Showing 83 changed files with 1,210 additions and 2,396 deletions.
diff --git a/build.gradle.kts b/build.gradle.kts
@@ -218,6 +218,10 @@ nexusPublishing {
   packageGroup.set("com.datastrato.gravitino")
 }
 
+dependencies {
+  testImplementation(libs.testng)
+}
+
 subprojects {
   // Gravitino Python client project didn't need to apply the java plugin
   if (project.name == "client-python") {
@@ -407,8 +411,13 @@ subprojects {
     reports.html.outputLocation.set(file("${rootProject.projectDir}/build/reports/"))
     val skipTests = project.hasProperty("skipTests")
     if (!skipTests) {
-      jvmArgs = listOf("-Xmx2G")
-      useJUnitPlatform()
+      if (project.name == "trino-connector") {
+        useTestNG()
+        maxHeapSize = "2G"
+      } else {
+        useJUnitPlatform()
+      }
+
       jvmArgs(project.property("extraJvmArgs") as List<*>)
       finalizedBy(tasks.getByName("jacocoTestReport"))
     }

diff --git a/...st/java/com/datastrato/gravitino/catalog/doris/integration/test/CatalogDorisDriverIT.java b/...st/java/com/datastrato/gravitino/catalog/doris/integration/test/CatalogDorisDriverIT.java
@@ -10,5 +10,7 @@
 public class CatalogDorisDriverIT extends CatalogDorisIT {
   public CatalogDorisDriverIT() {
     super();
+    mysqlDriverDownloadUrl =
+        "https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.11/mysql-connector-java-8.0.11.jar";
   }
 }
diff --git a/...src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/CatalogDorisIT.java b/...src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/CatalogDorisIT.java
@@ -19,6 +19,7 @@
 import com.datastrato.gravitino.integration.test.util.AbstractIT;
 import com.datastrato.gravitino.integration.test.util.GravitinoITUtils;
 import com.datastrato.gravitino.integration.test.util.ITUtils;
+import com.datastrato.gravitino.integration.test.util.JdbcDriverDownloader;
 import com.datastrato.gravitino.rel.Column;
 import com.datastrato.gravitino.rel.Table;
 import com.datastrato.gravitino.rel.TableCatalog;
@@ -33,6 +34,8 @@
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Maps;
 import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.Map;
@@ -54,6 +57,8 @@
 public class CatalogDorisIT extends AbstractIT {
 
   private static final String provider = "jdbc-doris";
+  private static final String DOWNLOAD_JDBC_DRIVER_URL =
+      "https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.27/mysql-connector-java-8.0.27.jar";
 
   private static final String DRIVER_CLASS_NAME = "com.mysql.jdbc.Driver";
 
@@ -82,8 +87,17 @@ public class CatalogDorisIT extends AbstractIT {
 
   protected Catalog catalog;
 
+  protected String mysqlDriverDownloadUrl = DOWNLOAD_JDBC_DRIVER_URL;
+
   @BeforeAll
   public void startup() throws IOException {
+
+    if (!ITUtils.EMBEDDED_TEST_MODE.equals(AbstractIT.testMode)) {
+      String gravitinoHome = System.getenv("GRAVITINO_HOME");
+      Path tmpPath = Paths.get(gravitinoHome, "/catalogs/jdbc-doris/libs");
+      JdbcDriverDownloader.downloadJdbcDriver(mysqlDriverDownloadUrl, tmpPath.toString());
+    }
+
     containerSuite.startDorisContainer();
 
     createMetalake();

diff --git a/...est/java/com/datastrato/gravitino/catalog/mysql/integration/test/AuditCatalogMysqlIT.java b/...est/java/com/datastrato/gravitino/catalog/mysql/integration/test/AuditCatalogMysqlIT.java
@@ -18,13 +18,17 @@
 import com.datastrato.gravitino.integration.test.container.MySQLContainer;
 import com.datastrato.gravitino.integration.test.util.AbstractIT;
 import com.datastrato.gravitino.integration.test.util.GravitinoITUtils;
+import com.datastrato.gravitino.integration.test.util.ITUtils;
+import com.datastrato.gravitino.integration.test.util.JdbcDriverDownloader;
 import com.datastrato.gravitino.integration.test.util.TestDatabaseName;
 import com.datastrato.gravitino.rel.Column;
 import com.datastrato.gravitino.rel.Table;
 import com.datastrato.gravitino.rel.TableChange;
 import com.datastrato.gravitino.rel.types.Types;
 import com.google.common.collect.Maps;
 import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.sql.SQLException;
 import java.util.Collections;
 import java.util.Map;
@@ -40,6 +44,8 @@ public class AuditCatalogMysqlIT extends AbstractIT {
   private static final ContainerSuite containerSuite = ContainerSuite.getInstance();
   public static final String metalakeName = GravitinoITUtils.genRandomName("audit_mysql_metalake");
   private static final String expectUser = System.getProperty("user.name");
+  public static final String DOWNLOAD_JDBC_DRIVER_URL =
+      "https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.27/mysql-connector-java-8.0.27.jar";
   protected static TestDatabaseName TEST_DB_NAME;
   private static final String provider = "jdbc-mysql";
 
@@ -54,6 +60,12 @@ public static void startIntegrationTest() throws Exception {
     registerCustomConfigs(configs);
     AbstractIT.startIntegrationTest();
 
+    if (!ITUtils.EMBEDDED_TEST_MODE.equals(testMode)) {
+      String gravitinoHome = System.getenv("GRAVITINO_HOME");
+      Path tmpPath = Paths.get(gravitinoHome, "/catalogs/jdbc-mysql/libs");
+      JdbcDriverDownloader.downloadJdbcDriver(DOWNLOAD_JDBC_DRIVER_URL, tmpPath.toString());
+    }
+
     containerSuite.startMySQLContainer(TestDatabaseName.MYSQL_AUDIT_CATALOG_MYSQL_IT);
     MYSQL_CONTAINER = containerSuite.getMySQLContainer();
     TEST_DB_NAME = TestDatabaseName.MYSQL_AUDIT_CATALOG_MYSQL_IT;

diff --git a/...st/java/com/datastrato/gravitino/catalog/mysql/integration/test/CatalogMysqlDriverIT.java b/...st/java/com/datastrato/gravitino/catalog/mysql/integration/test/CatalogMysqlDriverIT.java
@@ -11,5 +11,7 @@
 public class CatalogMysqlDriverIT extends CatalogMysqlIT {
   public CatalogMysqlDriverIT() {
     super();
+    mysqlDriverDownloadUrl =
+        "https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.11/mysql-connector-java-8.0.11.jar";
   }
 }
diff --git a/...src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/CatalogMysqlIT.java b/...src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/CatalogMysqlIT.java
@@ -25,6 +25,7 @@
 import com.datastrato.gravitino.integration.test.util.AbstractIT;
 import com.datastrato.gravitino.integration.test.util.GravitinoITUtils;
 import com.datastrato.gravitino.integration.test.util.ITUtils;
+import com.datastrato.gravitino.integration.test.util.JdbcDriverDownloader;
 import com.datastrato.gravitino.integration.test.util.TestDatabaseName;
 import com.datastrato.gravitino.rel.Column;
 import com.datastrato.gravitino.rel.Column.ColumnImpl;
@@ -46,6 +47,8 @@
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Maps;
 import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.sql.SQLException;
 import java.util.Arrays;
 import java.util.Collections;
@@ -69,6 +72,8 @@
 public class CatalogMysqlIT extends AbstractIT {
   private static final ContainerSuite containerSuite = ContainerSuite.getInstance();
   private static final String provider = "jdbc-mysql";
+  public static final String DOWNLOAD_JDBC_DRIVER_URL =
+      "https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.27/mysql-connector-java-8.0.27.jar";
 
   public String metalakeName = GravitinoITUtils.genRandomName("mysql_it_metalake");
   public String catalogName = GravitinoITUtils.genRandomName("mysql_it_catalog");
@@ -99,12 +104,21 @@ public class CatalogMysqlIT extends AbstractIT {
 
   protected String mysqlImageName = defaultMysqlImageName;
 
+  protected String mysqlDriverDownloadUrl = DOWNLOAD_JDBC_DRIVER_URL;
+
   boolean SupportColumnDefaultValueExpression() {
     return true;
   }
 
   @BeforeAll
   public void startup() throws IOException, SQLException {
+
+    if (!ITUtils.EMBEDDED_TEST_MODE.equals(testMode)) {
+      String gravitinoHome = System.getenv("GRAVITINO_HOME");
+      Path tmpPath = Paths.get(gravitinoHome, "/catalogs/jdbc-mysql/libs");
+      JdbcDriverDownloader.downloadJdbcDriver(mysqlDriverDownloadUrl, tmpPath.toString());
+    }
+
     TEST_DB_NAME = TestDatabaseName.MYSQL_CATALOG_MYSQL_IT;
 
     if (mysqlImageName.equals("mysql:5.7")) {

diff --git a/...ava/com/datastrato/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlIT.java b/...ava/com/datastrato/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlIT.java
@@ -24,6 +24,7 @@
 import com.datastrato.gravitino.integration.test.util.AbstractIT;
 import com.datastrato.gravitino.integration.test.util.GravitinoITUtils;
 import com.datastrato.gravitino.integration.test.util.ITUtils;
+import com.datastrato.gravitino.integration.test.util.JdbcDriverDownloader;
 import com.datastrato.gravitino.integration.test.util.TestDatabaseName;
 import com.datastrato.gravitino.rel.Column;
 import com.datastrato.gravitino.rel.Table;
@@ -45,6 +46,8 @@
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Maps;
 import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.sql.DriverManager;
 import java.sql.SQLException;
 import java.util.Arrays;
@@ -69,6 +72,8 @@
 public class CatalogPostgreSqlIT extends AbstractIT {
   private static final ContainerSuite containerSuite = ContainerSuite.getInstance();
   public static final PGImageName DEFAULT_POSTGRES_IMAGE = PGImageName.VERSION_13;
+  public static final String DOWNLOAD_JDBC_DRIVER_URL =
+      "https://jdbc.postgresql.org/download/postgresql-42.7.0.jar";
 
   public String metalakeName = GravitinoITUtils.genRandomName("postgresql_it_metalake");
   public String catalogName = GravitinoITUtils.genRandomName("postgresql_it_catalog");
@@ -96,6 +101,12 @@ public class CatalogPostgreSqlIT extends AbstractIT {
 
   @BeforeAll
   public void startup() throws IOException, SQLException {
+
+    if (!ITUtils.EMBEDDED_TEST_MODE.equals(testMode)) {
+      String gravitinoHome = System.getenv("GRAVITINO_HOME");
+      Path tmpPath = Paths.get(gravitinoHome, "/catalogs/jdbc-postgresql/libs");
+      JdbcDriverDownloader.downloadJdbcDriver(DOWNLOAD_JDBC_DRIVER_URL, tmpPath.toString());
+    }
     containerSuite.startPostgreSQLContainer(TEST_DB_NAME, postgreImageName);
     POSTGRESQL_CONTAINER = containerSuite.getPostgreSQLContainer(postgreImageName);
 

diff --git a/...va/com/datastrato/gravitino/catalog/postgresql/integration/test/TestMultipleJDBCLoad.java b/...va/com/datastrato/gravitino/catalog/postgresql/integration/test/TestMultipleJDBCLoad.java
@@ -13,13 +13,17 @@
 import com.datastrato.gravitino.integration.test.container.MySQLContainer;
 import com.datastrato.gravitino.integration.test.container.PostgreSQLContainer;
 import com.datastrato.gravitino.integration.test.util.AbstractIT;
+import com.datastrato.gravitino.integration.test.util.ITUtils;
+import com.datastrato.gravitino.integration.test.util.JdbcDriverDownloader;
 import com.datastrato.gravitino.integration.test.util.TestDatabaseName;
 import com.datastrato.gravitino.rel.Column;
 import com.datastrato.gravitino.rel.types.Types;
 import com.datastrato.gravitino.utils.RandomNameUtils;
 import com.google.common.collect.Maps;
 import java.io.IOException;
 import java.net.URISyntaxException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.sql.SQLException;
 import java.util.Collections;
 import java.util.Map;
@@ -37,8 +41,38 @@ public class TestMultipleJDBCLoad extends AbstractIT {
   private static MySQLContainer mySQLContainer;
   private static PostgreSQLContainer postgreSQLContainer;
 
+  private static final String DOWNLOAD_JDBC_DRIVER_URL =
+      "https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.27/mysql-connector-java-8.0.27.jar";
+
   @BeforeAll
   public static void startup() throws IOException {
+    String gravitinoHome = System.getenv("GRAVITINO_HOME");
+
+    // Deploy mode
+    if (!ITUtils.EMBEDDED_TEST_MODE.equals(testMode)) {
+      Path icebergLibsPath = Paths.get(gravitinoHome, "/catalogs/lakehouse-iceberg/libs");
+      Path pgDirPath = Paths.get(gravitinoHome, "/catalogs/jdbc-postgresql/libs");
+      JdbcDriverDownloader.downloadJdbcDriver(
+          CatalogPostgreSqlIT.DOWNLOAD_JDBC_DRIVER_URL,
+          pgDirPath.toString(),
+          icebergLibsPath.toString());
+
+      JdbcDriverDownloader.downloadJdbcDriver(
+          DOWNLOAD_JDBC_DRIVER_URL, pgDirPath.toString(), icebergLibsPath.toString());
+    } else {
+      // embedded mode
+      Path icebergLibsPath =
+          Paths.get(gravitinoHome, "/catalogs/catalog-lakehouse-iceberg/build/libs");
+      Path pgDirPath = Paths.get(gravitinoHome, "/catalogs/catalog-jdbc-postgresql/build/libs");
+      JdbcDriverDownloader.downloadJdbcDriver(
+          CatalogPostgreSqlIT.DOWNLOAD_JDBC_DRIVER_URL,
+          icebergLibsPath.toString(),
+          pgDirPath.toString());
+
+      JdbcDriverDownloader.downloadJdbcDriver(
+          DOWNLOAD_JDBC_DRIVER_URL, pgDirPath.toString(), icebergLibsPath.toString());
+    }
+
     containerSuite.startMySQLContainer(TEST_DB_NAME);
     mySQLContainer = containerSuite.getMySQLContainer();
     containerSuite.startPostgreSQLContainer(TEST_DB_NAME);

diff --git a/...datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/TestMultipleJDBCLoad.java b/...datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/TestMultipleJDBCLoad.java
@@ -15,13 +15,17 @@
 import com.datastrato.gravitino.integration.test.container.MySQLContainer;
 import com.datastrato.gravitino.integration.test.container.PostgreSQLContainer;
 import com.datastrato.gravitino.integration.test.util.AbstractIT;
+import com.datastrato.gravitino.integration.test.util.ITUtils;
+import com.datastrato.gravitino.integration.test.util.JdbcDriverDownloader;
 import com.datastrato.gravitino.integration.test.util.TestDatabaseName;
 import com.datastrato.gravitino.rel.Column;
 import com.datastrato.gravitino.rel.types.Types;
 import com.datastrato.gravitino.utils.RandomNameUtils;
 import com.google.common.collect.Maps;
 import java.io.IOException;
 import java.net.URISyntaxException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.sql.SQLException;
 import java.util.Collections;
 import java.util.Map;
@@ -38,10 +42,34 @@ public class TestMultipleJDBCLoad extends AbstractIT {
   private static MySQLContainer mySQLContainer;
   private static PostgreSQLContainer postgreSQLContainer;
 
+  private static final String DOWNLOAD_MYSQL_JDBC_DRIVER_URL =
+      "https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.27/mysql-connector-java-8.0.27.jar";
   public static final String DEFAULT_POSTGRES_IMAGE = "postgres:13";
+  public static final String DOWNLOAD_PG_JDBC_DRIVER_URL =
+      "https://jdbc.postgresql.org/download/postgresql-42.7.0.jar";
 
   @BeforeAll
   public static void startup() throws IOException {
+    String gravitinoHome = System.getenv("GRAVITINO_HOME");
+
+    // Deploy mode
+    if (!ITUtils.EMBEDDED_TEST_MODE.equals(testMode)) {
+      Path icebergLibsPath = Paths.get(gravitinoHome, "/catalogs/lakehouse-iceberg/libs");
+      JdbcDriverDownloader.downloadJdbcDriver(
+          DOWNLOAD_MYSQL_JDBC_DRIVER_URL, icebergLibsPath.toString());
+      JdbcDriverDownloader.downloadJdbcDriver(
+          DOWNLOAD_PG_JDBC_DRIVER_URL, icebergLibsPath.toString());
+    } else {
+      // embedded mode
+      Path icebergLibsPath =
+          Paths.get(gravitinoHome, "/catalogs/catalog-lakehouse-iceberg/build/libs");
+      JdbcDriverDownloader.downloadJdbcDriver(
+          DOWNLOAD_MYSQL_JDBC_DRIVER_URL, icebergLibsPath.toString());
+
+      JdbcDriverDownloader.downloadJdbcDriver(
+          DOWNLOAD_PG_JDBC_DRIVER_URL, icebergLibsPath.toString());
+    }
+
     containerSuite.startMySQLContainer(TEST_DB_NAME);
     mySQLContainer = containerSuite.getMySQLContainer();
     containerSuite.startPostgreSQLContainer(TEST_DB_NAME);

diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
@@ -13,9 +13,8 @@ log4j = "2.22.0"
 jetty = "9.4.51.v20230217"
 jersey = "2.41"
 mockito = "4.11.0"
-airlift-json = "237"
-airlift-log = "231"
 airlift-units = "1.8"
+airlift-log = "231"
 hive2 = "2.3.9"
 hadoop2 = "2.10.2"
 hadoop3 = "3.1.0"
@@ -30,17 +29,19 @@ caffeine = "2.9.3"
 rocksdbjni = "7.10.2"
 iceberg = '1.3.1' # used for Gravitino Iceberg catalog and Iceberg REST service
 iceberg4spark = "1.4.1" # used for compile spark connector
-spark33 = "3.3.4"
+trino = '426'
+spark33 = "3.3.4" 
 spark34 = "3.4.3"
 spark35 = "3.5.1"
 kyuubi4spark33 = "1.7.4"
 kyuubi4spark34 = "1.8.2"
 kyuubi4spark35 = "1.9.0"
-trino = '435'
 scala-collection-compat = "2.7.0"
 scala-java-compat = "1.0.2"
 sqlite-jdbc = "3.42.0.0"
+testng = "7.5.1"
 testcontainers = "1.19.0"
+trino-jdbc = "426"
 jwt = "0.11.1"
 jline = "3.21.0"
 okhttp3 = "4.11.0"
@@ -117,7 +118,6 @@ hadoop3-common = { group = "org.apache.hadoop", name = "hadoop-common", version.
 hadoop3-client = { group = "org.apache.hadoop", name = "hadoop-client", version.ref = "hadoop3"}
 hadoop3-mapreduce-client-core = { group = "org.apache.hadoop", name = "hadoop-mapreduce-client-core", version.ref = "hadoop3"}
 hadoop3-minicluster = { group = "org.apache.hadoop", name = "hadoop-minicluster", version.ref = "hadoop-minikdc"}
-airlift-json = { group = "io.airlift", name = "json", version.ref = "airlift-json"}
 airlift-units = { group = "io.airlift", name = "units", version.ref = "airlift-units"}
 airlift-log = { group = "io.airlift", name = "log", version.ref = "airlift-log"}
 httpclient5 = { group = "org.apache.httpcomponents.client5", name = "httpclient5", version.ref = "httpclient5" }
@@ -138,12 +138,13 @@ trino-memory= { group = "io.trino", name = "trino-memory", version.ref = "trino"
 trino-cli= { group = "io.trino", name = "trino-cli", version.ref = "trino" }
 trino-client= { group = "io.trino", name = "trino-client", version.ref = "trino" }
 sqlite-jdbc = { group = "org.xerial", name = "sqlite-jdbc", version.ref = "sqlite-jdbc" }
+testng = { group = "org.testng", name = "testng", version.ref = "testng" }
 commons-dbcp2 = { group = "org.apache.commons", name = "commons-dbcp2", version.ref = "commons-dbcp2" }
 testcontainers = { group = "org.testcontainers", name = "testcontainers", version.ref = "testcontainers" }
 testcontainers-mysql = { group = "org.testcontainers", name = "mysql", version.ref = "testcontainers" }
 testcontainers-postgresql = { group = "org.testcontainers", name = "postgresql", version.ref = "testcontainers" }
 testcontainers-junit-jupiter = { group = "org.testcontainers", name = "junit-jupiter", version.ref = "testcontainers" }
-trino-jdbc = { group = "io.trino", name = "trino-jdbc", version.ref = "trino" }
+trino-jdbc = { group = "io.trino", name = "trino-jdbc", version.ref = "trino-jdbc" }
 jwt-api = { group = "io.jsonwebtoken", name = "jjwt-api", version.ref = "jwt"}
 jwt-impl = { group = "io.jsonwebtoken", name = "jjwt-impl", version.ref = "jwt"}
 jwt-gson = { group = "io.jsonwebtoken", name = "jjwt-gson", version.ref = "jwt"}