From dd9cd76d1ec580ee0a1d3a2ead0d79453a650a76 Mon Sep 17 00:00:00 2001 From: Yuhui Date: Fri, 31 May 2024 10:56:25 +0800 Subject: [PATCH] =?UTF-8?q?Revert=20"[#3524]=20feat(trino-connector):=20Lo?= =?UTF-8?q?ad=20catalog=20by=20Trino=20dynamic=20catalog=20=E2=80=A6"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit d0699e7ec15687cd6672279d04970187b4b1a952. --- build.gradle.kts | 13 +- .../test/CatalogDorisDriverIT.java | 2 + .../integration/test/CatalogDorisIT.java | 14 + .../integration/test/AuditCatalogMysqlIT.java | 12 + .../test/CatalogMysqlDriverIT.java | 2 + .../integration/test/CatalogMysqlIT.java | 14 + .../integration/test/CatalogPostgreSqlIT.java | 11 + .../test/TestMultipleJDBCLoad.java | 34 ++ .../test/TestMultipleJDBCLoad.java | 28 ++ gradle/libs.versions.toml | 13 +- .../integration/test/util/AbstractIT.java | 28 +- .../test/trino/TrinoConnectorIT.java | 25 +- .../test/web/ui/CatalogsPageDorisTest.java | 15 + .../{bugs => }/00004_query_pushdown.sql | 0 .../{bugs => }/00004_query_pushdown.txt | 0 .../{bugs => }/00003_join_pushdown.sql | 0 .../{bugs => }/00003_join_pushdown.txt | 0 .../{bugs => }/00004_query_pushdown.sql | 0 .../{bugs => }/00004_query_pushdown.txt | 0 integration-test/trino-it/docker-compose.yaml | 2 +- .../trino/config/catalog/gravitino.properties | 7 - .../init/trino/config/config.properties | 10 - integration-test/trino-it/init/trino/init.sh | 21 +- trino-connector/build.gradle.kts | 10 +- .../connector/GravitinoColumnHandle.java | 76 ---- .../trino/connector/GravitinoConfig.java | 74 +--- .../trino/connector/GravitinoConnector.java | 5 +- .../connector/GravitinoConnectorFactory.java | 31 +- .../GravitinoConnectorPluginManager.java | 219 ----------- .../trino/connector/GravitinoConstraint.java | 58 --- .../GravitinoDataSourceProvider.java | 21 +- .../connector/GravitinoDynamicFilter.java | 45 --- .../trino/connector/GravitinoErrorCode.java | 2 - .../trino/connector/GravitinoHandle.java | 94 ----- .../connector/GravitinoInsertTableHandle.java | 26 +- .../trino/connector/GravitinoMetadata.java | 367 +++++++----------- .../connector/GravitinoPageSinkProvider.java | 12 +- .../trino/connector/GravitinoPredicate.java | 42 -- .../connector/GravitinoRecordSetProvider.java | 15 +- .../trino/connector/GravitinoSplit.java | 66 ---- .../connector/GravitinoSplitManager.java | 25 +- .../trino/connector/GravitinoSplitSource.java | 48 --- .../trino/connector/GravitinoTableHandle.java | 31 +- .../connector/GravitinoTransactionHandle.java | 27 +- .../catalog/CatalogConnectorAdapter.java | 5 +- .../catalog/CatalogConnectorContext.java | 36 +- .../catalog/CatalogConnectorFactory.java | 2 +- .../catalog/CatalogConnectorManager.java | 168 ++++---- .../catalog/CatalogConnectorMetadata.java | 6 +- .../connector/catalog/CatalogInjector.java | 356 +++++++++++++++++ .../connector/catalog/CatalogRegister.java | 235 ----------- .../catalog/hive/HiveConnectorAdapter.java | 28 +- .../iceberg/IcebergConnectorAdapter.java | 22 +- .../jdbc/mysql/MySQLConnectorAdapter.java | 22 +- .../PostgreSQLConnectorAdapter.java | 20 +- .../memory/MemoryConnectorAdapter.java | 22 +- .../connector/metadata/GravitinoCatalog.java | 64 +-- .../system/GravitinoSystemConnector.java | 4 +- .../AlterCatalogStoredProcedure.java | 2 +- .../util/json/AbstractTypedJacksonModule.java | 157 -------- .../connector/util/json/BlockJsonSerde.java | 101 ----- .../trino/connector/util/json/JsonCodec.java | 197 ---------- .../connector/util/json/TypeDeserializer.java | 45 --- .../util/json/TypeSignatureDeserializer.java | 50 --- .../trino/connector/GravitinoMockServer.java | 11 +- .../TestCreateGravitinoConnector.java | 4 +- .../trino/connector/TestGravitinoConfig.java | 12 +- .../connector/TestGravitinoConnector.java | 33 +- ...itinoConnectorWithMetalakeCatalogName.java | 74 ++-- .../trino/connector/TestGravitinoPlugin.java | 2 +- .../connector/TestGravitinoTableHandle.java | 6 +- .../TestHiveCatalogPropertyConverter.java | 33 +- .../hive/TestHiveDataTypeConverter.java | 23 +- .../TestIcebergCatalogPropertyConverter.java | 87 +++-- .../TestIcebergDataTypeTransformer.java | 14 +- .../TestJDBCCatalogPropertyConverter.java | 53 ++- .../mysql/TestMySQLDataTypeTransformer.java | 20 +- .../TestPostgreSQLDataTypeTransformer.java | 4 +- .../metadata/TestGravitinoCatalog.java | 4 +- .../metadata/TestGravitinoColumn.java | 4 +- .../metadata/TestGravitinoSchema.java | 4 +- .../metadata/TestGravitinoTable.java | 35 +- .../util/TestDataTypeTransformer.java | 96 ++--- 83 files changed, 1210 insertions(+), 2396 deletions(-) rename integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/{bugs => }/00004_query_pushdown.sql (100%) rename integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/{bugs => }/00004_query_pushdown.txt (100%) rename integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/{bugs => }/00003_join_pushdown.sql (100%) rename integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/{bugs => }/00003_join_pushdown.txt (100%) rename integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/{bugs => }/00004_query_pushdown.sql (100%) rename integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/{bugs => }/00004_query_pushdown.txt (100%) delete mode 100644 integration-test/trino-it/init/trino/config/catalog/gravitino.properties delete mode 100644 integration-test/trino-it/init/trino/config/config.properties delete mode 100644 trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoColumnHandle.java delete mode 100644 trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoConnectorPluginManager.java delete mode 100644 trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoConstraint.java delete mode 100644 trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoDynamicFilter.java delete mode 100644 trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoHandle.java delete mode 100644 trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoPredicate.java delete mode 100644 trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoSplit.java delete mode 100644 trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoSplitSource.java create mode 100644 trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogInjector.java delete mode 100644 trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogRegister.java delete mode 100644 trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/util/json/AbstractTypedJacksonModule.java delete mode 100644 trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/util/json/BlockJsonSerde.java delete mode 100644 trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/util/json/JsonCodec.java delete mode 100644 trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/util/json/TypeDeserializer.java delete mode 100644 trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/util/json/TypeSignatureDeserializer.java diff --git a/build.gradle.kts b/build.gradle.kts index 36affb2c006..7aaee41fc0e 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -218,6 +218,10 @@ nexusPublishing { packageGroup.set("com.datastrato.gravitino") } +dependencies { + testImplementation(libs.testng) +} + subprojects { // Gravitino Python client project didn't need to apply the java plugin if (project.name == "client-python") { @@ -407,8 +411,13 @@ subprojects { reports.html.outputLocation.set(file("${rootProject.projectDir}/build/reports/")) val skipTests = project.hasProperty("skipTests") if (!skipTests) { - jvmArgs = listOf("-Xmx2G") - useJUnitPlatform() + if (project.name == "trino-connector") { + useTestNG() + maxHeapSize = "2G" + } else { + useJUnitPlatform() + } + jvmArgs(project.property("extraJvmArgs") as List<*>) finalizedBy(tasks.getByName("jacocoTestReport")) } diff --git a/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/CatalogDorisDriverIT.java b/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/CatalogDorisDriverIT.java index 03a36e63e3d..d6144c1963e 100644 --- a/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/CatalogDorisDriverIT.java +++ b/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/CatalogDorisDriverIT.java @@ -10,5 +10,7 @@ public class CatalogDorisDriverIT extends CatalogDorisIT { public CatalogDorisDriverIT() { super(); + mysqlDriverDownloadUrl = + "https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.11/mysql-connector-java-8.0.11.jar"; } } diff --git a/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/CatalogDorisIT.java b/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/CatalogDorisIT.java index 18a2994d86b..12f4a5649c3 100644 --- a/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/CatalogDorisIT.java +++ b/catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/CatalogDorisIT.java @@ -19,6 +19,7 @@ import com.datastrato.gravitino.integration.test.util.AbstractIT; import com.datastrato.gravitino.integration.test.util.GravitinoITUtils; import com.datastrato.gravitino.integration.test.util.ITUtils; +import com.datastrato.gravitino.integration.test.util.JdbcDriverDownloader; import com.datastrato.gravitino.rel.Column; import com.datastrato.gravitino.rel.Table; import com.datastrato.gravitino.rel.TableCatalog; @@ -33,6 +34,8 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Arrays; import java.util.Collections; import java.util.Map; @@ -54,6 +57,8 @@ public class CatalogDorisIT extends AbstractIT { private static final String provider = "jdbc-doris"; + private static final String DOWNLOAD_JDBC_DRIVER_URL = + "https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.27/mysql-connector-java-8.0.27.jar"; private static final String DRIVER_CLASS_NAME = "com.mysql.jdbc.Driver"; @@ -82,8 +87,17 @@ public class CatalogDorisIT extends AbstractIT { protected Catalog catalog; + protected String mysqlDriverDownloadUrl = DOWNLOAD_JDBC_DRIVER_URL; + @BeforeAll public void startup() throws IOException { + + if (!ITUtils.EMBEDDED_TEST_MODE.equals(AbstractIT.testMode)) { + String gravitinoHome = System.getenv("GRAVITINO_HOME"); + Path tmpPath = Paths.get(gravitinoHome, "/catalogs/jdbc-doris/libs"); + JdbcDriverDownloader.downloadJdbcDriver(mysqlDriverDownloadUrl, tmpPath.toString()); + } + containerSuite.startDorisContainer(); createMetalake(); diff --git a/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/AuditCatalogMysqlIT.java b/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/AuditCatalogMysqlIT.java index 9929b2669d4..51b8d531d76 100644 --- a/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/AuditCatalogMysqlIT.java +++ b/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/AuditCatalogMysqlIT.java @@ -18,6 +18,8 @@ import com.datastrato.gravitino.integration.test.container.MySQLContainer; import com.datastrato.gravitino.integration.test.util.AbstractIT; import com.datastrato.gravitino.integration.test.util.GravitinoITUtils; +import com.datastrato.gravitino.integration.test.util.ITUtils; +import com.datastrato.gravitino.integration.test.util.JdbcDriverDownloader; import com.datastrato.gravitino.integration.test.util.TestDatabaseName; import com.datastrato.gravitino.rel.Column; import com.datastrato.gravitino.rel.Table; @@ -25,6 +27,8 @@ import com.datastrato.gravitino.rel.types.Types; import com.google.common.collect.Maps; import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; import java.sql.SQLException; import java.util.Collections; import java.util.Map; @@ -40,6 +44,8 @@ public class AuditCatalogMysqlIT extends AbstractIT { private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); public static final String metalakeName = GravitinoITUtils.genRandomName("audit_mysql_metalake"); private static final String expectUser = System.getProperty("user.name"); + public static final String DOWNLOAD_JDBC_DRIVER_URL = + "https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.27/mysql-connector-java-8.0.27.jar"; protected static TestDatabaseName TEST_DB_NAME; private static final String provider = "jdbc-mysql"; @@ -54,6 +60,12 @@ public static void startIntegrationTest() throws Exception { registerCustomConfigs(configs); AbstractIT.startIntegrationTest(); + if (!ITUtils.EMBEDDED_TEST_MODE.equals(testMode)) { + String gravitinoHome = System.getenv("GRAVITINO_HOME"); + Path tmpPath = Paths.get(gravitinoHome, "/catalogs/jdbc-mysql/libs"); + JdbcDriverDownloader.downloadJdbcDriver(DOWNLOAD_JDBC_DRIVER_URL, tmpPath.toString()); + } + containerSuite.startMySQLContainer(TestDatabaseName.MYSQL_AUDIT_CATALOG_MYSQL_IT); MYSQL_CONTAINER = containerSuite.getMySQLContainer(); TEST_DB_NAME = TestDatabaseName.MYSQL_AUDIT_CATALOG_MYSQL_IT; diff --git a/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/CatalogMysqlDriverIT.java b/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/CatalogMysqlDriverIT.java index 910f31a56e4..3bcb43eb880 100644 --- a/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/CatalogMysqlDriverIT.java +++ b/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/CatalogMysqlDriverIT.java @@ -11,5 +11,7 @@ public class CatalogMysqlDriverIT extends CatalogMysqlIT { public CatalogMysqlDriverIT() { super(); + mysqlDriverDownloadUrl = + "https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.11/mysql-connector-java-8.0.11.jar"; } } diff --git a/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/CatalogMysqlIT.java b/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/CatalogMysqlIT.java index cec6f30213e..cc7e79848d3 100644 --- a/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/CatalogMysqlIT.java +++ b/catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/CatalogMysqlIT.java @@ -25,6 +25,7 @@ import com.datastrato.gravitino.integration.test.util.AbstractIT; import com.datastrato.gravitino.integration.test.util.GravitinoITUtils; import com.datastrato.gravitino.integration.test.util.ITUtils; +import com.datastrato.gravitino.integration.test.util.JdbcDriverDownloader; import com.datastrato.gravitino.integration.test.util.TestDatabaseName; import com.datastrato.gravitino.rel.Column; import com.datastrato.gravitino.rel.Column.ColumnImpl; @@ -46,6 +47,8 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; import java.sql.SQLException; import java.util.Arrays; import java.util.Collections; @@ -69,6 +72,8 @@ public class CatalogMysqlIT extends AbstractIT { private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); private static final String provider = "jdbc-mysql"; + public static final String DOWNLOAD_JDBC_DRIVER_URL = + "https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.27/mysql-connector-java-8.0.27.jar"; public String metalakeName = GravitinoITUtils.genRandomName("mysql_it_metalake"); public String catalogName = GravitinoITUtils.genRandomName("mysql_it_catalog"); @@ -99,12 +104,21 @@ public class CatalogMysqlIT extends AbstractIT { protected String mysqlImageName = defaultMysqlImageName; + protected String mysqlDriverDownloadUrl = DOWNLOAD_JDBC_DRIVER_URL; + boolean SupportColumnDefaultValueExpression() { return true; } @BeforeAll public void startup() throws IOException, SQLException { + + if (!ITUtils.EMBEDDED_TEST_MODE.equals(testMode)) { + String gravitinoHome = System.getenv("GRAVITINO_HOME"); + Path tmpPath = Paths.get(gravitinoHome, "/catalogs/jdbc-mysql/libs"); + JdbcDriverDownloader.downloadJdbcDriver(mysqlDriverDownloadUrl, tmpPath.toString()); + } + TEST_DB_NAME = TestDatabaseName.MYSQL_CATALOG_MYSQL_IT; if (mysqlImageName.equals("mysql:5.7")) { diff --git a/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlIT.java b/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlIT.java index ac711366b1f..7b1050e06ee 100644 --- a/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlIT.java +++ b/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlIT.java @@ -24,6 +24,7 @@ import com.datastrato.gravitino.integration.test.util.AbstractIT; import com.datastrato.gravitino.integration.test.util.GravitinoITUtils; import com.datastrato.gravitino.integration.test.util.ITUtils; +import com.datastrato.gravitino.integration.test.util.JdbcDriverDownloader; import com.datastrato.gravitino.integration.test.util.TestDatabaseName; import com.datastrato.gravitino.rel.Column; import com.datastrato.gravitino.rel.Table; @@ -45,6 +46,8 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; import java.sql.DriverManager; import java.sql.SQLException; import java.util.Arrays; @@ -69,6 +72,8 @@ public class CatalogPostgreSqlIT extends AbstractIT { private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); public static final PGImageName DEFAULT_POSTGRES_IMAGE = PGImageName.VERSION_13; + public static final String DOWNLOAD_JDBC_DRIVER_URL = + "https://jdbc.postgresql.org/download/postgresql-42.7.0.jar"; public String metalakeName = GravitinoITUtils.genRandomName("postgresql_it_metalake"); public String catalogName = GravitinoITUtils.genRandomName("postgresql_it_catalog"); @@ -96,6 +101,12 @@ public class CatalogPostgreSqlIT extends AbstractIT { @BeforeAll public void startup() throws IOException, SQLException { + + if (!ITUtils.EMBEDDED_TEST_MODE.equals(testMode)) { + String gravitinoHome = System.getenv("GRAVITINO_HOME"); + Path tmpPath = Paths.get(gravitinoHome, "/catalogs/jdbc-postgresql/libs"); + JdbcDriverDownloader.downloadJdbcDriver(DOWNLOAD_JDBC_DRIVER_URL, tmpPath.toString()); + } containerSuite.startPostgreSQLContainer(TEST_DB_NAME, postgreImageName); POSTGRESQL_CONTAINER = containerSuite.getPostgreSQLContainer(postgreImageName); diff --git a/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/integration/test/TestMultipleJDBCLoad.java b/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/integration/test/TestMultipleJDBCLoad.java index 95f1c793cd0..be54d63e345 100644 --- a/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/integration/test/TestMultipleJDBCLoad.java +++ b/catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/integration/test/TestMultipleJDBCLoad.java @@ -13,6 +13,8 @@ import com.datastrato.gravitino.integration.test.container.MySQLContainer; import com.datastrato.gravitino.integration.test.container.PostgreSQLContainer; import com.datastrato.gravitino.integration.test.util.AbstractIT; +import com.datastrato.gravitino.integration.test.util.ITUtils; +import com.datastrato.gravitino.integration.test.util.JdbcDriverDownloader; import com.datastrato.gravitino.integration.test.util.TestDatabaseName; import com.datastrato.gravitino.rel.Column; import com.datastrato.gravitino.rel.types.Types; @@ -20,6 +22,8 @@ import com.google.common.collect.Maps; import java.io.IOException; import java.net.URISyntaxException; +import java.nio.file.Path; +import java.nio.file.Paths; import java.sql.SQLException; import java.util.Collections; import java.util.Map; @@ -37,8 +41,38 @@ public class TestMultipleJDBCLoad extends AbstractIT { private static MySQLContainer mySQLContainer; private static PostgreSQLContainer postgreSQLContainer; + private static final String DOWNLOAD_JDBC_DRIVER_URL = + "https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.27/mysql-connector-java-8.0.27.jar"; + @BeforeAll public static void startup() throws IOException { + String gravitinoHome = System.getenv("GRAVITINO_HOME"); + + // Deploy mode + if (!ITUtils.EMBEDDED_TEST_MODE.equals(testMode)) { + Path icebergLibsPath = Paths.get(gravitinoHome, "/catalogs/lakehouse-iceberg/libs"); + Path pgDirPath = Paths.get(gravitinoHome, "/catalogs/jdbc-postgresql/libs"); + JdbcDriverDownloader.downloadJdbcDriver( + CatalogPostgreSqlIT.DOWNLOAD_JDBC_DRIVER_URL, + pgDirPath.toString(), + icebergLibsPath.toString()); + + JdbcDriverDownloader.downloadJdbcDriver( + DOWNLOAD_JDBC_DRIVER_URL, pgDirPath.toString(), icebergLibsPath.toString()); + } else { + // embedded mode + Path icebergLibsPath = + Paths.get(gravitinoHome, "/catalogs/catalog-lakehouse-iceberg/build/libs"); + Path pgDirPath = Paths.get(gravitinoHome, "/catalogs/catalog-jdbc-postgresql/build/libs"); + JdbcDriverDownloader.downloadJdbcDriver( + CatalogPostgreSqlIT.DOWNLOAD_JDBC_DRIVER_URL, + icebergLibsPath.toString(), + pgDirPath.toString()); + + JdbcDriverDownloader.downloadJdbcDriver( + DOWNLOAD_JDBC_DRIVER_URL, pgDirPath.toString(), icebergLibsPath.toString()); + } + containerSuite.startMySQLContainer(TEST_DB_NAME); mySQLContainer = containerSuite.getMySQLContainer(); containerSuite.startPostgreSQLContainer(TEST_DB_NAME); diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/TestMultipleJDBCLoad.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/TestMultipleJDBCLoad.java index 71b87e19a93..4a0e4e3b53e 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/TestMultipleJDBCLoad.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/TestMultipleJDBCLoad.java @@ -15,6 +15,8 @@ import com.datastrato.gravitino.integration.test.container.MySQLContainer; import com.datastrato.gravitino.integration.test.container.PostgreSQLContainer; import com.datastrato.gravitino.integration.test.util.AbstractIT; +import com.datastrato.gravitino.integration.test.util.ITUtils; +import com.datastrato.gravitino.integration.test.util.JdbcDriverDownloader; import com.datastrato.gravitino.integration.test.util.TestDatabaseName; import com.datastrato.gravitino.rel.Column; import com.datastrato.gravitino.rel.types.Types; @@ -22,6 +24,8 @@ import com.google.common.collect.Maps; import java.io.IOException; import java.net.URISyntaxException; +import java.nio.file.Path; +import java.nio.file.Paths; import java.sql.SQLException; import java.util.Collections; import java.util.Map; @@ -38,10 +42,34 @@ public class TestMultipleJDBCLoad extends AbstractIT { private static MySQLContainer mySQLContainer; private static PostgreSQLContainer postgreSQLContainer; + private static final String DOWNLOAD_MYSQL_JDBC_DRIVER_URL = + "https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.27/mysql-connector-java-8.0.27.jar"; public static final String DEFAULT_POSTGRES_IMAGE = "postgres:13"; + public static final String DOWNLOAD_PG_JDBC_DRIVER_URL = + "https://jdbc.postgresql.org/download/postgresql-42.7.0.jar"; @BeforeAll public static void startup() throws IOException { + String gravitinoHome = System.getenv("GRAVITINO_HOME"); + + // Deploy mode + if (!ITUtils.EMBEDDED_TEST_MODE.equals(testMode)) { + Path icebergLibsPath = Paths.get(gravitinoHome, "/catalogs/lakehouse-iceberg/libs"); + JdbcDriverDownloader.downloadJdbcDriver( + DOWNLOAD_MYSQL_JDBC_DRIVER_URL, icebergLibsPath.toString()); + JdbcDriverDownloader.downloadJdbcDriver( + DOWNLOAD_PG_JDBC_DRIVER_URL, icebergLibsPath.toString()); + } else { + // embedded mode + Path icebergLibsPath = + Paths.get(gravitinoHome, "/catalogs/catalog-lakehouse-iceberg/build/libs"); + JdbcDriverDownloader.downloadJdbcDriver( + DOWNLOAD_MYSQL_JDBC_DRIVER_URL, icebergLibsPath.toString()); + + JdbcDriverDownloader.downloadJdbcDriver( + DOWNLOAD_PG_JDBC_DRIVER_URL, icebergLibsPath.toString()); + } + containerSuite.startMySQLContainer(TEST_DB_NAME); mySQLContainer = containerSuite.getMySQLContainer(); containerSuite.startPostgreSQLContainer(TEST_DB_NAME); diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 09f46d70a6a..47e6ddbbf92 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -13,9 +13,8 @@ log4j = "2.22.0" jetty = "9.4.51.v20230217" jersey = "2.41" mockito = "4.11.0" -airlift-json = "237" -airlift-log = "231" airlift-units = "1.8" +airlift-log = "231" hive2 = "2.3.9" hadoop2 = "2.10.2" hadoop3 = "3.1.0" @@ -30,17 +29,19 @@ caffeine = "2.9.3" rocksdbjni = "7.10.2" iceberg = '1.3.1' # used for Gravitino Iceberg catalog and Iceberg REST service iceberg4spark = "1.4.1" # used for compile spark connector -spark33 = "3.3.4" +trino = '426' +spark33 = "3.3.4" spark34 = "3.4.3" spark35 = "3.5.1" kyuubi4spark33 = "1.7.4" kyuubi4spark34 = "1.8.2" kyuubi4spark35 = "1.9.0" -trino = '435' scala-collection-compat = "2.7.0" scala-java-compat = "1.0.2" sqlite-jdbc = "3.42.0.0" +testng = "7.5.1" testcontainers = "1.19.0" +trino-jdbc = "426" jwt = "0.11.1" jline = "3.21.0" okhttp3 = "4.11.0" @@ -117,7 +118,6 @@ hadoop3-common = { group = "org.apache.hadoop", name = "hadoop-common", version. hadoop3-client = { group = "org.apache.hadoop", name = "hadoop-client", version.ref = "hadoop3"} hadoop3-mapreduce-client-core = { group = "org.apache.hadoop", name = "hadoop-mapreduce-client-core", version.ref = "hadoop3"} hadoop3-minicluster = { group = "org.apache.hadoop", name = "hadoop-minicluster", version.ref = "hadoop-minikdc"} -airlift-json = { group = "io.airlift", name = "json", version.ref = "airlift-json"} airlift-units = { group = "io.airlift", name = "units", version.ref = "airlift-units"} airlift-log = { group = "io.airlift", name = "log", version.ref = "airlift-log"} httpclient5 = { group = "org.apache.httpcomponents.client5", name = "httpclient5", version.ref = "httpclient5" } @@ -138,12 +138,13 @@ trino-memory= { group = "io.trino", name = "trino-memory", version.ref = "trino" trino-cli= { group = "io.trino", name = "trino-cli", version.ref = "trino" } trino-client= { group = "io.trino", name = "trino-client", version.ref = "trino" } sqlite-jdbc = { group = "org.xerial", name = "sqlite-jdbc", version.ref = "sqlite-jdbc" } +testng = { group = "org.testng", name = "testng", version.ref = "testng" } commons-dbcp2 = { group = "org.apache.commons", name = "commons-dbcp2", version.ref = "commons-dbcp2" } testcontainers = { group = "org.testcontainers", name = "testcontainers", version.ref = "testcontainers" } testcontainers-mysql = { group = "org.testcontainers", name = "mysql", version.ref = "testcontainers" } testcontainers-postgresql = { group = "org.testcontainers", name = "postgresql", version.ref = "testcontainers" } testcontainers-junit-jupiter = { group = "org.testcontainers", name = "junit-jupiter", version.ref = "testcontainers" } -trino-jdbc = { group = "io.trino", name = "trino-jdbc", version.ref = "trino" } +trino-jdbc = { group = "io.trino", name = "trino-jdbc", version.ref = "trino-jdbc" } jwt-api = { group = "io.jsonwebtoken", name = "jjwt-api", version.ref = "jwt"} jwt-impl = { group = "io.jsonwebtoken", name = "jjwt-impl", version.ref = "jwt"} jwt-gson = { group = "io.jsonwebtoken", name = "jjwt-gson", version.ref = "jwt"} diff --git a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/util/AbstractIT.java b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/util/AbstractIT.java index c55132ee7ac..155399b4477 100644 --- a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/util/AbstractIT.java +++ b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/util/AbstractIT.java @@ -61,12 +61,9 @@ public class AbstractIT { protected static boolean ignoreIcebergRestService = true; - public static final String DOWNLOAD_MYSQL_JDBC_DRIVER_URL = + private static final String DOWNLOAD_JDBC_DRIVER_URL = "https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.26/mysql-connector-java-8.0.26.jar"; - public static final String DOWNLOAD_POSTGRESQL_JDBC_DRIVER_URL = - "https://jdbc.postgresql.org/download/postgresql-42.7.0.jar"; - private static TestDatabaseName META_DATA; private static MySQLContainer MYSQL_CONTAINER; @@ -108,24 +105,11 @@ private static void recoverGravitinoServerConfig() throws IOException { Files.move(tmpPath, configPath); } - protected static void downLoadJDBCDriver() throws IOException { - String gravitinoHome = System.getenv("GRAVITINO_HOME"); + protected static void downLoadMySQLDriver(String relativeDeployLibsPath) throws IOException { if (!ITUtils.EMBEDDED_TEST_MODE.equals(testMode)) { - String serverPath = ITUtils.joinPath(gravitinoHome, "libs"); - String icebergCatalogPath = - ITUtils.joinPath(gravitinoHome, "catalogs", "lakehouse-iceberg", "libs"); - JdbcDriverDownloader.downloadJdbcDriver( - DOWNLOAD_MYSQL_JDBC_DRIVER_URL, serverPath, icebergCatalogPath); - JdbcDriverDownloader.downloadJdbcDriver( - DOWNLOAD_POSTGRESQL_JDBC_DRIVER_URL, serverPath, icebergCatalogPath); - } else { - Path icebergLibsPath = - Paths.get(gravitinoHome, "catalogs", "catalog-lakehouse-iceberg", "build", "libs"); - JdbcDriverDownloader.downloadJdbcDriver( - DOWNLOAD_MYSQL_JDBC_DRIVER_URL, icebergLibsPath.toString()); - - JdbcDriverDownloader.downloadJdbcDriver( - DOWNLOAD_POSTGRESQL_JDBC_DRIVER_URL, icebergLibsPath.toString()); + String gravitinoHome = System.getenv("GRAVITINO_HOME"); + java.nio.file.Path tmpPath = Paths.get(gravitinoHome, relativeDeployLibsPath); + JdbcDriverDownloader.downloadJdbcDriver(DOWNLOAD_JDBC_DRIVER_URL, tmpPath.toString()); } } @@ -201,7 +185,7 @@ public static void startIntegrationTest() throws Exception { } else { rewriteGravitinoServerConfig(); serverConfig.loadFromFile(GravitinoServer.CONF_FILE); - downLoadJDBCDriver(); + downLoadMySQLDriver("/libs"); try { FileUtils.deleteDirectory( FileUtils.getFile(serverConfig.get(ENTRY_KV_ROCKSDB_BACKEND_PATH))); diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/trino/TrinoConnectorIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/trino/TrinoConnectorIT.java index 792f4a80324..d314fa056cf 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/trino/TrinoConnectorIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/trino/TrinoConnectorIT.java @@ -15,6 +15,8 @@ import com.datastrato.gravitino.integration.test.container.TrinoContainer; import com.datastrato.gravitino.integration.test.util.AbstractIT; import com.datastrato.gravitino.integration.test.util.GravitinoITUtils; +import com.datastrato.gravitino.integration.test.util.ITUtils; +import com.datastrato.gravitino.integration.test.util.JdbcDriverDownloader; import com.datastrato.gravitino.rel.Column; import com.datastrato.gravitino.rel.Table; import com.datastrato.gravitino.rel.expressions.NamedReference; @@ -46,14 +48,11 @@ import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@Disabled -@Deprecated @Tag("gravitino-docker-it") public class TrinoConnectorIT extends AbstractIT { public static final Logger LOG = LoggerFactory.getLogger(TrinoConnectorIT.class); @@ -103,6 +102,26 @@ public static void startDockerContainer() throws IOException, TException, Interr "Can not synchronize calatogs from gravitino"); createSchema(); + + String testMode = + System.getProperty(ITUtils.TEST_MODE) == null + ? ITUtils.EMBEDDED_TEST_MODE + : System.getProperty(ITUtils.TEST_MODE); + + // Deploy mode, you should download jars to the Gravitino server iceberg lib directory + if (!ITUtils.EMBEDDED_TEST_MODE.equals(testMode)) { + String gravitinoHome = System.getenv("GRAVITINO_HOME"); + String destPath = ITUtils.joinPath(gravitinoHome, "catalogs", "lakehouse-iceberg", "libs"); + String mysqlPath = ITUtils.joinPath(gravitinoHome, "catalogs", "jdbc-mysql", "libs"); + String pgPath = ITUtils.joinPath(gravitinoHome, "catalogs", "jdbc-postgresql", "libs"); + + JdbcDriverDownloader.downloadJdbcDriver( + "https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.27/mysql-connector-java-8.0.27.jar", + destPath, + mysqlPath); + JdbcDriverDownloader.downloadJdbcDriver( + "https://jdbc.postgresql.org/download/postgresql-42.7.0.jar", destPath, pgPath); + } } @AfterAll diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/CatalogsPageDorisTest.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/CatalogsPageDorisTest.java index 20441168ceb..edaa286df4d 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/CatalogsPageDorisTest.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/CatalogsPageDorisTest.java @@ -12,6 +12,8 @@ import com.datastrato.gravitino.integration.test.container.ContainerSuite; import com.datastrato.gravitino.integration.test.container.DorisContainer; import com.datastrato.gravitino.integration.test.util.AbstractIT; +import com.datastrato.gravitino.integration.test.util.ITUtils; +import com.datastrato.gravitino.integration.test.util.JdbcDriverDownloader; import com.datastrato.gravitino.integration.test.web.ui.pages.CatalogsPage; import com.datastrato.gravitino.integration.test.web.ui.pages.MetalakePage; import com.datastrato.gravitino.integration.test.web.ui.utils.AbstractWebIT; @@ -21,6 +23,8 @@ import com.datastrato.gravitino.rel.expressions.sorts.SortOrder; import com.datastrato.gravitino.rel.types.Types; import com.google.common.collect.Maps; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -61,9 +65,20 @@ public class CatalogsPageDorisTest extends AbstractWebIT { private static final String COLUMN_NAME = "col1"; private static final String PROPERTIES_KEY1 = "key1"; private static final String PROPERTIES_VALUE1 = "val1"; + private static final String DOWNLOAD_JDBC_DRIVER_URL = + "https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.27/mysql-connector-java-8.0.27.jar"; @BeforeAll public static void before() throws Exception { + + String gravitinoHome = System.getenv("GRAVITINO_HOME"); + if (!ITUtils.EMBEDDED_TEST_MODE.equals(AbstractIT.testMode)) { + Path tmpPath = Paths.get(gravitinoHome, "/catalogs/jdbc-doris/libs"); + JdbcDriverDownloader.downloadJdbcDriver(DOWNLOAD_JDBC_DRIVER_URL, tmpPath.toString()); + } else { + Path tmpPath = Paths.get(gravitinoHome, "/catalogs/catalog-jdbc-doris/build/libs"); + JdbcDriverDownloader.downloadJdbcDriver(DOWNLOAD_JDBC_DRIVER_URL, tmpPath.toString()); + } gravitinoClient = AbstractIT.getGravitinoClient(); gravitinoUri = String.format("http://127.0.0.1:%d", AbstractIT.getGravitinoServerPort()); diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/bugs/00004_query_pushdown.sql b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00004_query_pushdown.sql similarity index 100% rename from integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/bugs/00004_query_pushdown.sql rename to integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00004_query_pushdown.sql diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/bugs/00004_query_pushdown.txt b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00004_query_pushdown.txt similarity index 100% rename from integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/bugs/00004_query_pushdown.txt rename to integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00004_query_pushdown.txt diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/bugs/00003_join_pushdown.sql b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/00003_join_pushdown.sql similarity index 100% rename from integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/bugs/00003_join_pushdown.sql rename to integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/00003_join_pushdown.sql diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/bugs/00003_join_pushdown.txt b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/00003_join_pushdown.txt similarity index 100% rename from integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/bugs/00003_join_pushdown.txt rename to integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/00003_join_pushdown.txt diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/bugs/00004_query_pushdown.sql b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/00004_query_pushdown.sql similarity index 100% rename from integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/bugs/00004_query_pushdown.sql rename to integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/00004_query_pushdown.sql diff --git a/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/bugs/00004_query_pushdown.txt b/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/00004_query_pushdown.txt similarity index 100% rename from integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/bugs/00004_query_pushdown.txt rename to integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-postgresql/00004_query_pushdown.txt diff --git a/integration-test/trino-it/docker-compose.yaml b/integration-test/trino-it/docker-compose.yaml index 7f405719d9c..83c8827e20d 100644 --- a/integration-test/trino-it/docker-compose.yaml +++ b/integration-test/trino-it/docker-compose.yaml @@ -66,7 +66,7 @@ services: retries: 5 trino: - image: trinodb/trino:435 + image: datastrato/gravitino-ci-trino:0.1.5 networks: - trino-net container_name: trino-ci-trino diff --git a/integration-test/trino-it/init/trino/config/catalog/gravitino.properties b/integration-test/trino-it/init/trino/config/catalog/gravitino.properties deleted file mode 100644 index 3ccb8f9377f..00000000000 --- a/integration-test/trino-it/init/trino/config/catalog/gravitino.properties +++ /dev/null @@ -1,7 +0,0 @@ -# -# Copyright 2024 Datastrato Pvt Ltd. -# This software is licensed under the Apache License version 2. -# -connector.name = gravitino -gravitino.uri = http://GRAVITINO_HOST_IP:GRAVITINO_HOST_PORT -gravitino.metalake = GRAVITINO_METALAKE_NAME diff --git a/integration-test/trino-it/init/trino/config/config.properties b/integration-test/trino-it/init/trino/config/config.properties deleted file mode 100644 index 80e63874c4b..00000000000 --- a/integration-test/trino-it/init/trino/config/config.properties +++ /dev/null @@ -1,10 +0,0 @@ -# -# Copyright 2024 Datastrato Pvt Ltd. -# This software is licensed under the Apache License version 2. -# - -coordinator=true -node-scheduler.include-coordinator=true -http-server.http.port=8080 -catalog.management=dynamic -discovery.uri=http://0.0.0.0:8080 diff --git a/integration-test/trino-it/init/trino/init.sh b/integration-test/trino-it/init/trino/init.sh index 62d9b0b4650..f638102ebf9 100644 --- a/integration-test/trino-it/init/trino/init.sh +++ b/integration-test/trino-it/init/trino/init.sh @@ -3,26 +3,7 @@ # This software is licensed under the Apache License version 2. # -set -ex -trino_conf_dir="$(dirname "${BASH_SOURCE-$0}")" -trino_conf_dir="$(cd "${trino_conf_dir}">/dev/null; pwd)" - -cp "$trino_conf_dir/config/config.properties" /etc/trino/config.properties -cp "$trino_conf_dir/config/catalog/gravitino.properties" /etc/trino/catalog/gravitino.properties -# -# Update `gravitino.uri = http://GRAVITINO_HOST_IP:GRAVITINO_HOST_PORT` in the `conf/catalog/gravitino.properties` -sed -i "s/GRAVITINO_HOST_IP:GRAVITINO_HOST_PORT/${GRAVITINO_HOST_IP}:${GRAVITINO_HOST_PORT}/g" /etc/trino/catalog/gravitino.properties -# Update `gravitino.metalake = GRAVITINO_METALAKE_NAME` in the `conf/catalog/gravitino.properties` -sed -i "s/GRAVITINO_METALAKE_NAME/${GRAVITINO_METALAKE_NAME}/g" /etc/trino/catalog/gravitino.properties - - -# Check the number of Gravitino connector plugins present in the Trino plugin directory -num_of_gravitino_connector=$(ls /usr/lib/trino/plugin/gravitino | grep gravitino-trino-connector-* | wc -l) -if [[ "${num_of_gravitino_connector}" -ne 1 ]]; then - echo "Multiple versions of the Gravitino connector plugin found or none present." - exit 1 -fi - +/etc/trino/update-trino-conf.sh nohup /usr/lib/trino/bin/run-trino & counter=0 diff --git a/trino-connector/build.gradle.kts b/trino-connector/build.gradle.kts index ecab766345a..38f75911c83 100644 --- a/trino-connector/build.gradle.kts +++ b/trino-connector/build.gradle.kts @@ -15,11 +15,12 @@ repositories { dependencies { implementation(project(":catalogs:bundled-catalog", configuration = "shadow")) implementation(project(":clients:client-java-runtime", configuration = "shadow")) - implementation(libs.airlift.json) - implementation(libs.bundles.log4j) - implementation(libs.commons.collections4) implementation(libs.commons.lang3) - implementation(libs.trino.jdbc) + implementation(libs.guava) + implementation(libs.httpclient5) + implementation(libs.jackson.annotations) + implementation(libs.jackson.databind) + implementation(libs.commons.collections4) compileOnly(libs.trino.spi) { exclude("org.apache.logging.log4j") } @@ -33,7 +34,6 @@ dependencies { testImplementation(libs.trino.testing) { exclude("org.apache.logging.log4j") } - testRuntimeOnly(libs.junit.jupiter.engine) } tasks.named("generateMetadataFileForMavenJavaPublication") { diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoColumnHandle.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoColumnHandle.java deleted file mode 100644 index 1da24118723..00000000000 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoColumnHandle.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. - */ -package com.datastrato.gravitino.trino.connector; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.base.Preconditions; -import io.trino.spi.connector.ColumnHandle; -import java.util.Objects; - -/** - * The GravitinoTableHandle is used to transform column information between Trino and Gravitino, as - * well as to wrap the inner connector column handle for data access. - */ -public final class GravitinoColumnHandle implements ColumnHandle, GravitinoHandle { - - private final String columnName; - private HandleWrapper handleWrapper = new HandleWrapper<>(ColumnHandle.class); - - @JsonCreator - public GravitinoColumnHandle( - @JsonProperty("columnName") String columnName, - @JsonProperty(HANDLE_STRING) String handleString) { - Preconditions.checkArgument(columnName != null, "schemaName is not null"); - Preconditions.checkArgument(handleString != null, "internalHandle is not null"); - - this.columnName = columnName; - this.handleWrapper = handleWrapper.fromJson(handleString); - } - - public GravitinoColumnHandle(String columnName, ColumnHandle internalColumnHandle) { - this.columnName = columnName; - this.handleWrapper = new HandleWrapper<>(internalColumnHandle); - } - - @JsonProperty - public String getColumnName() { - return columnName; - } - - @JsonProperty - @Override - public String getHandleString() { - return handleWrapper.toJson(); - } - - @Override - public ColumnHandle getInternalHandle() { - return handleWrapper.getHandle(); - } - - @Override - public int hashCode() { - return Objects.hash(columnName); - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if ((obj == null) || (getClass() != obj.getClass())) { - return false; - } - - GravitinoColumnHandle other = (GravitinoColumnHandle) obj; - return Objects.equals(this.columnName, other.columnName); - } - - @Override - public String toString() { - return columnName; - } -} diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoConfig.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoConfig.java index 42f869b9823..978e8d1f548 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoConfig.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoConfig.java @@ -7,17 +7,12 @@ import static com.datastrato.gravitino.trino.connector.GravitinoErrorCode.GRAVITINO_MISSING_CONFIG; import io.trino.spi.TrinoException; -import java.util.ArrayList; import java.util.HashMap; -import java.util.List; import java.util.Map; -import org.apache.logging.log4j.util.Strings; public class GravitinoConfig { - public static final String GRAVITINO_DYNAMIC_CONNECTOR = "__gravitino.dynamic.connector"; - public static final String GRAVITINO_DYNAMIC_CONNECTOR_CATALOG_CONFIG = - "__gravitino.dynamic.connector.catalog.config"; + public static String GRAVITINO_DYNAMIC_CONNECTOR = "__gravitino.dynamic.connector"; private static final Map CONFIG_DEFINITIONS = new HashMap<>(); private final Map config; @@ -36,37 +31,19 @@ public class GravitinoConfig { "true", false); - private static final ConfigEntry TRINO_JDBC_URI = - new ConfigEntry( - "trino.jdbc.uri", "The jdbc uri of Trino server", "jdbc:trino://localhost:8080", false); - - private static final ConfigEntry TRINO_CATALOG_STORE = - new ConfigEntry( - "trino.catalog.store", - "The directory stored the catalog configuration of Trino", - "etc/catalog", - false); - - private static final ConfigEntry TRINO_JDBC_USER = - new ConfigEntry("trino.jdbc.user", "The jdbc user name of Trino", "admin", false); - - private static final ConfigEntry TRINO_JDBC_PASSWORD = - new ConfigEntry("trino.jdbc.password", "The jdbc user password of Trino", "", false); - public GravitinoConfig(Map requiredConfig) { config = requiredConfig; - for (Map.Entry entry : CONFIG_DEFINITIONS.entrySet()) { - ConfigEntry configDefinition = entry.getValue(); - if (configDefinition.isRequired && !config.containsKey(configDefinition.key)) { - String message = - String.format("Missing gravitino config, %s is required", configDefinition.key); - throw new TrinoException(GRAVITINO_MISSING_CONFIG, message); + + if (!isDynamicConnector()) { + for (Map.Entry entry : CONFIG_DEFINITIONS.entrySet()) { + ConfigEntry configDefinition = entry.getValue(); + if (configDefinition.isRequired && !config.containsKey(configDefinition.key)) { + String message = + String.format("Missing gravitino config, %s is required", configDefinition.key); + throw new TrinoException(GRAVITINO_MISSING_CONFIG, message); + } } } - if (isDynamicConnector() && !config.containsKey(GRAVITINO_DYNAMIC_CONNECTOR_CATALOG_CONFIG)) { - throw new TrinoException( - GRAVITINO_MISSING_CONFIG, "Incomplete Dynamic catalog connector config"); - } } public String getURI() { @@ -94,37 +71,6 @@ boolean isDynamicConnector() { return config.getOrDefault(GRAVITINO_DYNAMIC_CONNECTOR, "false").equals("true"); } - public String getCatalogConfig() { - return config.get(GRAVITINO_DYNAMIC_CONNECTOR_CATALOG_CONFIG); - } - - public String getTrinoURI() { - return config.getOrDefault(TRINO_JDBC_URI.key, TRINO_JDBC_URI.defaultValue); - } - - public String getCatalogStoreDirectory() { - return config.getOrDefault(TRINO_CATALOG_STORE.key, TRINO_CATALOG_STORE.defaultValue); - } - - public String getTrinoUser() { - return config.getOrDefault(TRINO_JDBC_USER.key, TRINO_JDBC_USER.defaultValue); - } - - public String getTrinoPassword() { - return config.getOrDefault(TRINO_JDBC_PASSWORD.key, TRINO_JDBC_PASSWORD.defaultValue); - } - - public String toCatalogConfig() { - List stringList = new ArrayList<>(); - for (Map.Entry entry : CONFIG_DEFINITIONS.entrySet()) { - String value = config.get(entry.getKey()); - if (value != null) { - stringList.add(String.format("\"%s\"='%s'", entry.getKey(), value)); - } - } - return Strings.join(stringList, ','); - } - static class ConfigEntry { final String key; final String description; diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoConnector.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoConnector.java index 89be17cbf10..2d914c609b4 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoConnector.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoConnector.java @@ -60,7 +60,8 @@ public ConnectorMetadata getMetadata( Connector internalConnector = catalogConnectorContext.getInternalConnector(); ConnectorMetadata internalMetadata = - internalConnector.getMetadata(session, gravitinoTransactionHandle.getInternalHandle()); + internalConnector.getMetadata( + session, gravitinoTransactionHandle.getInternalTransactionHandle()); Preconditions.checkNotNull(internalMetadata); GravitinoMetalake metalake = catalogConnectorContext.getMetalake(); @@ -127,7 +128,7 @@ public void commit(ConnectorTransactionHandle transactionHandle) { GravitinoTransactionHandle gravitinoTransactionHandle = (GravitinoTransactionHandle) transactionHandle; Connector internalConnector = catalogConnectorContext.getInternalConnector(); - internalConnector.commit(gravitinoTransactionHandle.getInternalHandle()); + internalConnector.commit(gravitinoTransactionHandle.getInternalTransactionHandle()); } @Override diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoConnectorFactory.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoConnectorFactory.java index a1605b8e8cf..b6ee3e1740c 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoConnectorFactory.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoConnectorFactory.java @@ -6,12 +6,12 @@ import static com.datastrato.gravitino.trino.connector.GravitinoErrorCode.GRAVITINO_METALAKE_NOT_EXISTS; import static com.datastrato.gravitino.trino.connector.GravitinoErrorCode.GRAVITINO_MISSING_CONFIG; -import static com.datastrato.gravitino.trino.connector.GravitinoErrorCode.GRAVITINO_RUNTIME_ERROR; import com.datastrato.gravitino.client.GravitinoAdminClient; +import com.datastrato.gravitino.trino.connector.catalog.CatalogConnectorContext; import com.datastrato.gravitino.trino.connector.catalog.CatalogConnectorFactory; import com.datastrato.gravitino.trino.connector.catalog.CatalogConnectorManager; -import com.datastrato.gravitino.trino.connector.catalog.CatalogRegister; +import com.datastrato.gravitino.trino.connector.catalog.CatalogInjector; import com.datastrato.gravitino.trino.connector.system.GravitinoSystemConnector; import com.datastrato.gravitino.trino.connector.system.storedprocdure.GravitinoStoredProcedureFactory; import com.datastrato.gravitino.trino.connector.system.table.GravitinoSystemTableFactory; @@ -32,9 +32,6 @@ public class GravitinoConnectorFactory implements ConnectorFactory { private static final Logger LOG = LoggerFactory.getLogger(GravitinoConnectorFactory.class); private static final String DEFAULT_CONNECTOR_NAME = "gravitino"; - @SuppressWarnings("UnusedVariable") - private GravitinoSystemTableFactory gravitinoSystemTableFactory; - private CatalogConnectorManager catalogConnectorManager; @Override @@ -65,19 +62,20 @@ public Connector create( synchronized (this) { if (catalogConnectorManager == null) { try { - CatalogRegister catalogRegister = new CatalogRegister(); + CatalogInjector catalogInjector = new CatalogInjector(); + catalogInjector.init(context); CatalogConnectorFactory catalogConnectorFactory = new CatalogConnectorFactory(); catalogConnectorManager = - new CatalogConnectorManager(catalogRegister, catalogConnectorFactory); - catalogConnectorManager.config(config, clientProvider().get()); - catalogConnectorManager.start(context); + new CatalogConnectorManager(catalogInjector, catalogConnectorFactory); + catalogConnectorManager.config(config); + catalogConnectorManager.start(clientProvider().get()); + + new GravitinoSystemTableFactory(catalogConnectorManager); - gravitinoSystemTableFactory = new GravitinoSystemTableFactory(catalogConnectorManager); } catch (Exception e) { - String message = "Initialization of the GravitinoConnector failed" + e.getMessage(); - LOG.error(message); - throw new TrinoException(GRAVITINO_RUNTIME_ERROR, message, e); + LOG.error("Initialization of the GravitinoConnector failed.", e); + throw e; } } } @@ -85,7 +83,10 @@ public Connector create( if (config.isDynamicConnector()) { // The dynamic connector is an instance of GravitinoConnector. It is loaded from Gravitino // server. - return catalogConnectorManager.createConnector(catalogName, config, context); + CatalogConnectorContext catalogConnectorContext = + catalogConnectorManager.getCatalogConnector(catalogName); + Preconditions.checkNotNull(catalogConnectorContext, "catalogConnector is not null"); + return catalogConnectorContext.getConnector(); } else { // The static connector is an instance of GravitinoSystemConnector. It is loaded by Trino // using the connector configuration. @@ -101,6 +102,8 @@ public Connector create( catalogConnectorManager.addMetalake(metalake); GravitinoStoredProcedureFactory gravitinoStoredProcedureFactory = new GravitinoStoredProcedureFactory(catalogConnectorManager, metalake); + + catalogConnectorManager.loadMetalakeSync(); return new GravitinoSystemConnector(gravitinoStoredProcedureFactory); } } diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoConnectorPluginManager.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoConnectorPluginManager.java deleted file mode 100644 index 58fd0a7bb7b..00000000000 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoConnectorPluginManager.java +++ /dev/null @@ -1,219 +0,0 @@ -/* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. - */ -package com.datastrato.gravitino.trino.connector; - -import static com.datastrato.gravitino.trino.connector.GravitinoErrorCode.GRAVITINO_CREATE_INTERNAL_CONNECTOR_ERROR; -import static com.datastrato.gravitino.trino.connector.GravitinoErrorCode.GRAVITINO_RUNTIME_ERROR; - -import com.google.common.collect.ImmutableList; -import io.trino.spi.Plugin; -import io.trino.spi.TrinoException; -import io.trino.spi.classloader.ThreadContextClassLoader; -import io.trino.spi.connector.Connector; -import io.trino.spi.connector.ConnectorContext; -import io.trino.spi.connector.ConnectorFactory; -import java.io.File; -import java.lang.reflect.Constructor; -import java.net.URL; -import java.nio.file.Paths; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.ServiceLoader; -import java.util.Set; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** This class is mange the internal connector plugin and help to create the connector. */ -public class GravitinoConnectorPluginManager { - - private static final Logger LOG = LoggerFactory.getLogger(GravitinoConnectorPluginManager.class); - - public static final String APP_CLASS_LOADER_NAME = "app"; - - public static final String CONNECTOR_HIVE = "hive"; - public static final String CONNECTOR_ICEBERG = "iceberg"; - public static final String CONNECTOR_MYSQL = "mysql"; - public static final String CONNECTOR_POSTGRESQL = "postgresql"; - public static final String CONNECTOR_MEMORY = "memory"; - - private static final String PLUGIN_NAME_PREFIX = "gravitino-"; - private static final String PLUGIN_CLASSLOADER_CLASS_NAME = "io.trino.server.PluginClassLoader"; - - private static volatile GravitinoConnectorPluginManager instance; - - private final Class pluginLoaderClass; - - private static final Set usePlugins = - Set.of( - CONNECTOR_HIVE, - CONNECTOR_ICEBERG, - CONNECTOR_MYSQL, - CONNECTOR_POSTGRESQL, - CONNECTOR_MEMORY); - - private final Map connectorPlugins = new HashMap<>(); - private final ClassLoader appClassloader; - - public GravitinoConnectorPluginManager(ClassLoader classLoader) { - try { - // Retrieve plugin directory - // The Trino plugin director like: - // /data/trino/plugin/hive/**.jar - // /data/trino/plugin/gravitino/**.jar - // /data/trino/plugin/mysql/**.jar - String jarPath = - GravitinoConnectorPluginManager.class - .getProtectionDomain() - .getCodeSource() - .getLocation() - .toURI() - .getPath(); - String pluginDir = Paths.get(jarPath).getParent().getParent().toString(); - - this.appClassloader = classLoader; - pluginLoaderClass = appClassloader.loadClass(PLUGIN_CLASSLOADER_CLASS_NAME); - - // Load all plugins - for (String pluginName : usePlugins) { - loadPlugin(pluginDir, pluginName); - LOG.info("Load plugin {}/{} successful", pluginDir, pluginName); - } - } catch (Exception e) { - throw new TrinoException(GRAVITINO_RUNTIME_ERROR, "Error while loading plugins", e); - } - } - - public static GravitinoConnectorPluginManager instance(ClassLoader classLoader) { - if (instance != null) { - return instance; - } - synchronized (GravitinoConnectorPluginManager.class) { - if (instance == null) { - if (!APP_CLASS_LOADER_NAME.equals(classLoader.getName())) { - throw new TrinoException( - GRAVITINO_RUNTIME_ERROR, - "Can not initialize GravitinoConnectorPluginManager when classLoader is not appClassLoader"); - } - instance = new GravitinoConnectorPluginManager(classLoader); - } - return instance; - } - } - - public static GravitinoConnectorPluginManager instance() { - if (instance == null) { - throw new IllegalStateException("Need to call the function instance(ClassLoader) first"); - } - return instance; - } - - private void loadPlugin(String pluginPath, String pluginName) { - String dirName = pluginPath + "/" + pluginName; - File directory = new File(dirName); - if (!directory.exists()) { - LOG.warn("Can not found plugin {} in directory {}", pluginName, dirName); - return; - } - - File[] pluginFiles = directory.listFiles(); - if (pluginFiles == null || pluginFiles.length == 0) { - throw new TrinoException( - GRAVITINO_RUNTIME_ERROR, "Can not found any files plugin directory " + dirName); - } - List files = - Arrays.stream(pluginFiles) - .map(File::toURI) - .map( - uri -> { - try { - return uri.toURL(); - } catch (Exception e) { - throw new RuntimeException(e); - } - }) - .toList(); - - try { - Constructor constructor = - pluginLoaderClass.getConstructor(String.class, List.class, ClassLoader.class, List.class); - // The classloader name will use to serialize the Handle object - String classLoaderName = PLUGIN_NAME_PREFIX + pluginName; - // Load Trino SPI package and other dependencies refer to io.trino.server.PluginClassLoader - Object pluginClassLoader = - constructor.newInstance( - classLoaderName, - files, - appClassloader, - List.of( - "io.trino.spi.", - "com.fasterxml.jackson.annotation.", - "io.airlift.slice.", - "org.openjdk.jol.", - "io.opentelemetry.api.", - "io.opentelemetry.context.")); - - ServiceLoader serviceLoader = - ServiceLoader.load(Plugin.class, (ClassLoader) pluginClassLoader); - List pluginList = ImmutableList.copyOf(serviceLoader); - if (pluginList.isEmpty()) { - throw new TrinoException( - GRAVITINO_CREATE_INTERNAL_CONNECTOR_ERROR, - String.format("The %s plugin does not found connector SIP interface", pluginName)); - } - Plugin plugin = pluginList.get(0); - if (plugin.getConnectorFactories() == null - || !plugin.getConnectorFactories().iterator().hasNext()) { - throw new TrinoException( - GRAVITINO_CREATE_INTERNAL_CONNECTOR_ERROR, - String.format("The %s plugin does not contains any ConnectorFactories", pluginName)); - } - connectorPlugins.put(pluginName, pluginList.get(0)); - - } catch (Exception e) { - throw new TrinoException( - GRAVITINO_RUNTIME_ERROR, "Failed to create Plugin class loader " + pluginName, e); - } - } - - public void installPlugin(String pluginName, Plugin plugin) { - connectorPlugins.put(pluginName, plugin); - } - - public Connector createConnector( - String connectorName, Map config, ConnectorContext context) { - try { - Plugin plugin = connectorPlugins.get(connectorName); - try (ThreadContextClassLoader ignored = - new ThreadContextClassLoader(plugin.getClass().getClassLoader())) { - ConnectorFactory connectorFactory = plugin.getConnectorFactories().iterator().next(); - Connector connector = connectorFactory.create(connectorName, config, context); - LOG.info("create connector {} with config {} successful", connectorName, config); - return connector; - } - } catch (Exception e) { - throw new TrinoException( - GRAVITINO_RUNTIME_ERROR, "Failed to create connector " + connectorName, e); - } - } - - public ClassLoader getClassLoader(String classLoaderName) { - if (classLoaderName.equals(APP_CLASS_LOADER_NAME)) { - return appClassloader; - } - - Plugin plugin = connectorPlugins.get(classLoaderName.substring(PLUGIN_NAME_PREFIX.length())); - if (plugin == null) { - throw new TrinoException( - GRAVITINO_RUNTIME_ERROR, "Can not found class loader for " + classLoaderName); - } - return plugin.getClass().getClassLoader(); - } - - public ClassLoader getAppClassloader() { - return appClassloader; - } -} diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoConstraint.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoConstraint.java deleted file mode 100644 index 95b94f9a7ad..00000000000 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoConstraint.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. - */ -package com.datastrato.gravitino.trino.connector; - -import io.trino.spi.connector.ColumnHandle; -import io.trino.spi.connector.Constraint; -import io.trino.spi.expression.ConnectorExpression; -import io.trino.spi.predicate.NullableValue; -import io.trino.spi.predicate.TupleDomain; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.function.Predicate; -import java.util.stream.Collectors; - -/** The GravitinoConstraint is used to warp Constraint */ -public class GravitinoConstraint extends Constraint { - private final Constraint delegate; - - GravitinoConstraint(Constraint constraint) { - super(constraint.getSummary()); - this.delegate = constraint; - } - - @Override - public TupleDomain getSummary() { - return delegate.getSummary().transformKeys(GravitinoHandle::unWrap); - } - - @Override - public ConnectorExpression getExpression() { - return delegate.getExpression(); - } - - @Override - public Map getAssignments() { - return GravitinoHandle.unWrap(delegate.getAssignments()); - } - - @Override - public Optional>> predicate() { - return delegate.predicate().map(GravitinoPredicate::new); - } - - @Override - public Optional> getPredicateColumns() { - return delegate - .getPredicateColumns() - .map(result -> result.stream().map(GravitinoHandle::unWrap).collect(Collectors.toSet())); - } - - @Override - public String toString() { - return delegate.toString(); - } -} diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoDataSourceProvider.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoDataSourceProvider.java index dc6ec931eae..20153ed3937 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoDataSourceProvider.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoDataSourceProvider.java @@ -31,12 +31,23 @@ public ConnectorPageSource createPageSource( ConnectorTableHandle table, List columns, DynamicFilter dynamicFilter) { + if (!(table instanceof GravitinoTableHandle)) { + if (transaction instanceof GravitinoTransactionHandle) { + transaction = ((GravitinoTransactionHandle) transaction).getInternalTransactionHandle(); + } + return internalPageSourceProvider.createPageSource( + transaction, session, split, table, columns, dynamicFilter); + } + + GravitinoTableHandle gravitinoTableHandle = (GravitinoTableHandle) table; + GravitinoTransactionHandle gravitinoTransactionHandle = + (GravitinoTransactionHandle) transaction; return internalPageSourceProvider.createPageSource( - GravitinoHandle.unWrap(transaction), + gravitinoTransactionHandle.getInternalTransactionHandle(), session, - GravitinoHandle.unWrap(split), - GravitinoHandle.unWrap(table), - GravitinoHandle.unWrap(columns), - new GravitinoDynamicFilter(dynamicFilter)); + split, + gravitinoTableHandle.getInternalTableHandle(), + columns, + dynamicFilter); } } diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoDynamicFilter.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoDynamicFilter.java deleted file mode 100644 index 3dd677f41a8..00000000000 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoDynamicFilter.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. - */ -package com.datastrato.gravitino.trino.connector; - -import io.trino.spi.connector.ColumnHandle; -import io.trino.spi.connector.DynamicFilter; -import io.trino.spi.predicate.TupleDomain; -import java.util.Set; -import java.util.concurrent.CompletableFuture; - -/** The GravitinoDynamicFilter is used to warp DynamicFilter */ -class GravitinoDynamicFilter implements DynamicFilter { - private final DynamicFilter delegate; - - GravitinoDynamicFilter(DynamicFilter dynamicFilter) { - delegate = dynamicFilter; - } - - @Override - public Set getColumnsCovered() { - return delegate.getColumnsCovered(); - } - - @Override - public CompletableFuture isBlocked() { - return delegate.isBlocked(); - } - - @Override - public boolean isComplete() { - return delegate.isComplete(); - } - - @Override - public boolean isAwaitable() { - return delegate.isAwaitable(); - } - - @Override - public TupleDomain getCurrentPredicate() { - return delegate.getCurrentPredicate().transformKeys(GravitinoHandle::unWrap); - } -} diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoErrorCode.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoErrorCode.java index 3be2160d5c1..8e09d98f34f 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoErrorCode.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoErrorCode.java @@ -34,8 +34,6 @@ public enum GravitinoErrorCode implements ErrorCodeSupplier { GRAVITINO_CATALOG_ALREADY_EXISTS(20, EXTERNAL), GRAVITINO_METALAKE_ALREADY_EXISTS(21, EXTERNAL), GRAVITINO_OPERATION_FAILED(22, EXTERNAL), - GRAVITINO_RUNTIME_ERROR(23, EXTERNAL), - GRAVITINO_DUPLICATED_CATALOGS(24, EXTERNAL), ; // suppress ImmutableEnumChecker because ErrorCode is outside the project. diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoHandle.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoHandle.java deleted file mode 100644 index 051604efde7..00000000000 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoHandle.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. - */ -package com.datastrato.gravitino.trino.connector; - -import static com.datastrato.gravitino.trino.connector.GravitinoErrorCode.GRAVITINO_ILLEGAL_ARGUMENT; - -import com.datastrato.gravitino.trino.connector.util.json.JsonCodec; -import io.trino.spi.TrinoException; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.stream.Collectors; - -/** this interface for GravitinoXXXHande to communicate with Trino */ -interface GravitinoHandle { - String HANDLE_STRING = "handleString"; - - /** - * Serialize handle to json string - * - * @return the json string - */ - public String getHandleString(); - - /** - * Retrieve the internal handle - * - * @return the internal handle - */ - public T getInternalHandle(); - - public static T unWrap(T handle) { - return ((GravitinoHandle) handle).getInternalHandle(); - } - - public static List unWrap(List handles) { - return handles.stream() - .map(handle -> (((GravitinoHandle) handle).getInternalHandle())) - .collect(Collectors.toList()); - } - - public static Map unWrap(Map handleMap) { - return handleMap.entrySet().stream() - .collect( - Collectors.toMap( - Map.Entry::getKey, e -> ((GravitinoHandle) e.getValue()).getInternalHandle())); - } - - class HandleWrapper { - private final T handle; - private String valueString; - private final Class clazz; - - public HandleWrapper(Class clazz) { - this.handle = null; - this.clazz = clazz; - } - - public HandleWrapper(T handle) { - this.handle = Objects.requireNonNull(handle, "handle is not null"); - clazz = (Class) handle.getClass(); - } - - public HandleWrapper fromJson(String valueString) { - try { - T newHandle = - JsonCodec.getMapper(clazz.getClassLoader()).readerFor(clazz).readValue(valueString); - return new HandleWrapper<>(newHandle); - } catch (Exception e) { - throw new TrinoException(GRAVITINO_ILLEGAL_ARGUMENT, "Can not deserialize from json", e); - } - } - - public String toJson() { - if (valueString == null) { - try { - valueString = - JsonCodec.getMapper(clazz.getClassLoader()) - .writerFor(clazz) - .writeValueAsString(this.handle); - } catch (Exception e) { - throw new TrinoException(GRAVITINO_ILLEGAL_ARGUMENT, "Can not serialize to json", e); - } - } - return valueString; - } - - public T getHandle() { - return handle; - } - } -} diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoInsertTableHandle.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoInsertTableHandle.java index e37228e8f07..b86884a3710 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoInsertTableHandle.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoInsertTableHandle.java @@ -9,29 +9,23 @@ import io.trino.spi.connector.ConnectorInsertTableHandle; /** The GravitinoInsertTableHandle is used for handling insert operations. */ -public class GravitinoInsertTableHandle - implements ConnectorInsertTableHandle, GravitinoHandle { +public class GravitinoInsertTableHandle implements ConnectorInsertTableHandle { - private HandleWrapper handleWrapper = - new HandleWrapper<>(ConnectorInsertTableHandle.class); + private final ConnectorInsertTableHandle internalInsertTableHandle; @JsonCreator - public GravitinoInsertTableHandle(@JsonProperty(HANDLE_STRING) String handleString) { - this.handleWrapper = handleWrapper.fromJson(handleString); - } - - public GravitinoInsertTableHandle(ConnectorInsertTableHandle insertTableHandle) { - this.handleWrapper = new HandleWrapper<>(insertTableHandle); + public GravitinoInsertTableHandle( + @JsonProperty("internalInsertTableHandle") + ConnectorInsertTableHandle internalInsertTableHandle) { + this.internalInsertTableHandle = internalInsertTableHandle; } @JsonProperty - @Override - public String getHandleString() { - return handleWrapper.toJson(); + public ConnectorInsertTableHandle getInternalInsertTableHandle() { + return internalInsertTableHandle; } - @Override - public ConnectorInsertTableHandle getInternalHandle() { - return handleWrapper.getHandle(); + public ConnectorInsertTableHandle innerHandler() { + return internalInsertTableHandle; } } diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoMetadata.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoMetadata.java index 76699e4a6e4..a576223eee5 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoMetadata.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoMetadata.java @@ -18,7 +18,6 @@ import io.trino.spi.TrinoException; import io.trino.spi.connector.AggregateFunction; import io.trino.spi.connector.AggregationApplicationResult; -import io.trino.spi.connector.Assignment; import io.trino.spi.connector.ColumnHandle; import io.trino.spi.connector.ColumnMetadata; import io.trino.spi.connector.ConnectorInsertTableHandle; @@ -36,7 +35,6 @@ import io.trino.spi.connector.LimitApplicationResult; import io.trino.spi.connector.ProjectionApplicationResult; import io.trino.spi.connector.RetryMode; -import io.trino.spi.connector.SaveMode; import io.trino.spi.connector.SchemaTableName; import io.trino.spi.connector.SortItem; import io.trino.spi.connector.TopNApplicationResult; @@ -112,6 +110,9 @@ public GravitinoTableHandle getTableHandle( @Override public ConnectorTableMetadata getTableMetadata( ConnectorSession session, ConnectorTableHandle tableHandle) { + if (!(tableHandle instanceof GravitinoTableHandle)) { + return internalMetadata.getTableMetadata(session, tableHandle); + } GravitinoTableHandle gravitinoTableHandle = (GravitinoTableHandle) tableHandle; GravitinoTable table = catalogConnectorMetadata.getTable( @@ -121,7 +122,11 @@ public ConnectorTableMetadata getTableMetadata( @Override public SchemaTableName getTableName(ConnectorSession session, ConnectorTableHandle table) { - return getTableName(table); + if (!(table instanceof GravitinoTableHandle)) { + return internalMetadata.getTableName(session, table); + } + + return ((GravitinoTableHandle) table).toSchemaTableName(); } @Override @@ -145,28 +150,33 @@ public List listTables( @Override public Map getColumnHandles( ConnectorSession session, ConnectorTableHandle tableHandle) { + if (!(tableHandle instanceof GravitinoTableHandle)) { + return internalMetadata.getColumnHandles(session, tableHandle); + } + + GravitinoTableHandle gravitinoTableHandle = (GravitinoTableHandle) tableHandle; Map internalColumnHandles = - internalMetadata.getColumnHandles(session, GravitinoHandle.unWrap(tableHandle)); - return internalColumnHandles.entrySet().stream() - .collect( - Collectors.toMap( - Map.Entry::getKey, - (entry) -> new GravitinoColumnHandle(entry.getKey(), entry.getValue()))); + internalMetadata.getColumnHandles(session, gravitinoTableHandle.getInternalTableHandle()); + return internalColumnHandles; } @Override public ColumnMetadata getColumnMetadata( ConnectorSession session, ConnectorTableHandle tableHandle, ColumnHandle columnHandle) { + if (!(tableHandle instanceof GravitinoTableHandle)) { + return internalMetadata.getColumnMetadata(session, tableHandle, columnHandle); + } + + GravitinoTableHandle gravitinoTableHandle = (GravitinoTableHandle) tableHandle; return internalMetadata.getColumnMetadata( - session, GravitinoHandle.unWrap(tableHandle), GravitinoHandle.unWrap(columnHandle)); + session, gravitinoTableHandle.getInternalTableHandle(), columnHandle); } @Override public void createTable( - ConnectorSession session, ConnectorTableMetadata tableMetadata, SaveMode saveMode) { + ConnectorSession session, ConnectorTableMetadata tableMetadata, boolean ignoreExisting) { GravitinoTable table = metadataAdapter.createTable(tableMetadata); - // saveMode = SaveMode.IGNORE is used to ignore the table creation if it already exists - catalogConnectorMetadata.createTable(table, saveMode == SaveMode.IGNORE); + catalogConnectorMetadata.createTable(table); } @Override @@ -186,7 +196,8 @@ public void dropSchema(ConnectorSession session, String schemaName, boolean casc @Override public void dropTable(ConnectorSession session, ConnectorTableHandle tableHandle) { - catalogConnectorMetadata.dropTable(getTableName(tableHandle)); + GravitinoTableHandle gravitinoTableHandle = (GravitinoTableHandle) tableHandle; + catalogConnectorMetadata.dropTable(gravitinoTableHandle.toSchemaTableName()); } @Override @@ -205,12 +216,10 @@ public ConnectorInsertTableHandle beginInsert( ConnectorTableHandle tableHandle, List columns, RetryMode retryMode) { + GravitinoTableHandle gravitinoTableHandle = (GravitinoTableHandle) tableHandle; ConnectorInsertTableHandle insertTableHandle = internalMetadata.beginInsert( - session, - GravitinoHandle.unWrap(tableHandle), - GravitinoHandle.unWrap(columns), - retryMode); + session, gravitinoTableHandle.getInternalTableHandle(), columns, retryMode); return new GravitinoInsertTableHandle(insertTableHandle); } @@ -220,8 +229,14 @@ public Optional finishInsert( ConnectorInsertTableHandle insertHandle, Collection fragments, Collection computedStatistics) { + + GravitinoInsertTableHandle gravitinoInsertTableHandle = + (GravitinoInsertTableHandle) insertHandle; return internalMetadata.finishInsert( - session, GravitinoHandle.unWrap(insertHandle), fragments, computedStatistics); + session, + gravitinoInsertTableHandle.getInternalInsertTableHandle(), + fragments, + computedStatistics); } @Override @@ -232,13 +247,16 @@ public void renameSchema(ConnectorSession session, String source, String target) @Override public void renameTable( ConnectorSession session, ConnectorTableHandle tableHandle, SchemaTableName newTableName) { - catalogConnectorMetadata.renameTable(getTableName(tableHandle), newTableName); + GravitinoTableHandle gravitinoTableHandle = (GravitinoTableHandle) tableHandle; + catalogConnectorMetadata.renameTable(gravitinoTableHandle.toSchemaTableName(), newTableName); } @Override public void setTableComment( ConnectorSession session, ConnectorTableHandle tableHandle, Optional comment) { - catalogConnectorMetadata.setTableComment(getTableName(tableHandle), comment.orElse("")); + GravitinoTableHandle gravitinoTableHandle = (GravitinoTableHandle) tableHandle; + catalogConnectorMetadata.setTableComment( + gravitinoTableHandle.toSchemaTableName(), comment.orElse("")); } @Override @@ -246,26 +264,29 @@ public void setTableProperties( ConnectorSession session, ConnectorTableHandle tableHandle, Map> properties) { + GravitinoTableHandle gravitinoTableHandle = (GravitinoTableHandle) tableHandle; Map resultMap = properties.entrySet().stream() .filter(e -> e.getValue().isPresent()) .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().get())); Map allProps = metadataAdapter.toGravitinoTableProperties(resultMap); - catalogConnectorMetadata.setTableProperties(getTableName(tableHandle), allProps); + catalogConnectorMetadata.setTableProperties(gravitinoTableHandle.toSchemaTableName(), allProps); } @Override public void addColumn( ConnectorSession session, ConnectorTableHandle tableHandle, ColumnMetadata column) { + GravitinoTableHandle gravitinoTableHandle = (GravitinoTableHandle) tableHandle; GravitinoColumn gravitinoColumn = metadataAdapter.createColumn(column); - catalogConnectorMetadata.addColumn(getTableName(tableHandle), gravitinoColumn); + catalogConnectorMetadata.addColumn(gravitinoTableHandle.toSchemaTableName(), gravitinoColumn); } @Override public void dropColumn( ConnectorSession session, ConnectorTableHandle tableHandle, ColumnHandle column) { - String columnName = getColumnName(column); - catalogConnectorMetadata.dropColumn(getTableName(tableHandle), columnName); + GravitinoTableHandle gravitinoTableHandle = (GravitinoTableHandle) tableHandle; + String columnName = getColumnName(session, gravitinoTableHandle, column); + catalogConnectorMetadata.dropColumn(gravitinoTableHandle.toSchemaTableName(), columnName); } @Override @@ -274,16 +295,19 @@ public void renameColumn( ConnectorTableHandle tableHandle, ColumnHandle source, String target) { - String columnName = getColumnName(source); - catalogConnectorMetadata.renameColumn(getTableName(tableHandle), columnName, target); + GravitinoTableHandle gravitinoTableHandle = (GravitinoTableHandle) tableHandle; + String columnName = getColumnName(session, gravitinoTableHandle, source); + catalogConnectorMetadata.renameColumn( + gravitinoTableHandle.toSchemaTableName(), columnName, target); } @Override public void setColumnType( ConnectorSession session, ConnectorTableHandle tableHandle, ColumnHandle column, Type type) { - String columnName = getColumnName(column); + GravitinoTableHandle gravitinoTableHandle = (GravitinoTableHandle) tableHandle; + String columnName = getColumnName(session, gravitinoTableHandle, column); catalogConnectorMetadata.setColumnType( - getTableName(tableHandle), + gravitinoTableHandle.toSchemaTableName(), columnName, metadataAdapter.getDataTypeTransformer().getGravitinoType(type)); } @@ -294,13 +318,15 @@ public void setColumnComment( ConnectorTableHandle tableHandle, ColumnHandle column, Optional comment) { - String columnName = getColumnName(column); + GravitinoTableHandle gravitinoTableHandle = (GravitinoTableHandle) tableHandle; + String columnName = getColumnName(session, gravitinoTableHandle, column); String commentString = ""; if (comment.isPresent() && !StringUtils.isBlank(comment.get())) { commentString = comment.get(); } - catalogConnectorMetadata.setColumnComment(getTableName(tableHandle), columnName, commentString); + catalogConnectorMetadata.setColumnComment( + gravitinoTableHandle.toSchemaTableName(), columnName, commentString); } @Override @@ -313,60 +339,34 @@ public Optional> applyJoin( Map leftAssignments, Map rightAssignments, JoinStatistics statistics) { - return internalMetadata - .applyJoin( - session, - joinType, - GravitinoHandle.unWrap(left), - GravitinoHandle.unWrap(right), - joinCondition, - leftAssignments.entrySet().stream() - .collect( - Collectors.toMap( - Map.Entry::getKey, entry -> GravitinoHandle.unWrap(entry.getValue()))), - rightAssignments.entrySet().stream() - .collect( - Collectors.toMap( - Map.Entry::getKey, entry -> GravitinoHandle.unWrap(entry.getValue()))), - statistics) - .map( - result -> - new JoinApplicationResult<>( - new GravitinoTableHandle( - getTableName(left).getSchemaName(), - getTableName(left).getTableName(), - result.getTableHandle()), - result.getLeftColumnHandles().entrySet().stream() - .collect( - Collectors.toMap( - entry -> - new GravitinoColumnHandle( - getColumnName( - session, GravitinoHandle.unWrap(left), entry.getKey()), - entry.getKey()), - entry -> - new GravitinoColumnHandle( - getColumnName( - session, - GravitinoHandle.unWrap(left), - entry.getValue()), - entry.getValue()))), - result.getRightColumnHandles().entrySet().stream() - .collect( - Collectors.toMap( - entry -> - new GravitinoColumnHandle( - getColumnName( - session, GravitinoHandle.unWrap(right), entry.getKey()), - entry.getKey()), - entry -> - new GravitinoColumnHandle( - getColumnName( - session, - GravitinoHandle.unWrap(right), - entry.getValue()), - entry.getValue()))), - result.isPrecalculateStatistics())); + if (!(left instanceof GravitinoTableHandle) && !(right instanceof GravitinoTableHandle)) { + return internalMetadata.applyJoin( + session, + joinType, + left, + right, + joinCondition, + leftAssignments, + rightAssignments, + statistics); + } + + if (!(left instanceof GravitinoTableHandle) || !(right instanceof GravitinoTableHandle)) { + return Optional.empty(); + } + + GravitinoTableHandle gravitinoLeftTableHandle = (GravitinoTableHandle) left; + GravitinoTableHandle gravitinoRightTableHandle = (GravitinoTableHandle) right; + + return internalMetadata.applyJoin( + session, + joinType, + gravitinoLeftTableHandle.getInternalTableHandle(), + gravitinoRightTableHandle.getInternalTableHandle(), + joinCondition, + leftAssignments, + rightAssignments, + statistics); } @Override @@ -375,37 +375,13 @@ public Optional> applyProjecti ConnectorTableHandle handle, List projections, Map assignments) { - return internalMetadata - .applyProjection( - session, - GravitinoHandle.unWrap(handle), - projections, - assignments.entrySet().stream() - .collect( - Collectors.toMap( - Map.Entry::getKey, entry -> GravitinoHandle.unWrap(entry.getValue())))) - .map( - result -> - new ProjectionApplicationResult<>( - new GravitinoTableHandle( - getTableName(handle).getSchemaName(), - getTableName(handle).getTableName(), - result.getHandle()), - result.getProjections(), - result.getAssignments().stream() - .map( - entry -> - new Assignment( - entry.getVariable(), - new GravitinoColumnHandle( - getColumnName( - session, - GravitinoHandle.unWrap(handle), - entry.getColumn()), - entry.getColumn()), - entry.getType())) - .toList(), - result.isPrecalculateStatistics())); + if (!(handle instanceof GravitinoTableHandle)) { + return internalMetadata.applyProjection(session, handle, projections, assignments); + } + + GravitinoTableHandle gravitinoTableHandle = (GravitinoTableHandle) handle; + return internalMetadata.applyProjection( + session, gravitinoTableHandle.getInternalTableHandle(), projections, assignments); } @Override @@ -416,27 +392,14 @@ public ColumnHandle getMergeRowIdColumnHandle( @Override public Optional> applyFilter( - ConnectorSession session, ConnectorTableHandle tableHandle, Constraint constraint) { - return internalMetadata - .applyFilter( - session, GravitinoHandle.unWrap(tableHandle), new GravitinoConstraint(constraint)) - .map( - result -> - new ConstraintApplicationResult( - new GravitinoTableHandle( - getTableName(tableHandle).getSchemaName(), - getTableName(tableHandle).getTableName(), - result.getHandle()), - result - .getRemainingFilter() - .transformKeys( - (columnHandle) -> - new GravitinoColumnHandle( - getColumnName( - session, GravitinoHandle.unWrap(tableHandle), columnHandle), - columnHandle)), - result.getRemainingExpression().get(), - result.isPrecalculateStatistics())); + ConnectorSession session, ConnectorTableHandle handle, Constraint constraint) { + if (!(handle instanceof GravitinoTableHandle)) { + return internalMetadata.applyFilter(session, handle, constraint); + } + + GravitinoTableHandle gravitinoTableHandle = (GravitinoTableHandle) handle; + return internalMetadata.applyFilter( + session, gravitinoTableHandle.getInternalTableHandle(), constraint); } @Override @@ -446,77 +409,30 @@ public Optional> applyAggrega List aggregates, Map assignments, List> groupingSets) { - return internalMetadata - .applyAggregation( - session, - GravitinoHandle.unWrap(handle), - aggregates, - assignments.entrySet().stream() - .collect( - Collectors.toMap( - Map.Entry::getKey, entry -> GravitinoHandle.unWrap(entry.getValue()))), - groupingSets.stream() - .map( - innerList -> - innerList.stream() - .map(GravitinoHandle::unWrap) - .collect(Collectors.toList())) - .collect(Collectors.toList())) - .map( - result -> - new AggregationApplicationResult( - new GravitinoTableHandle( - getTableName(handle).getSchemaName(), - getTableName(handle).getTableName(), - result.getHandle()), - result.getProjections(), - result.getAssignments().stream() - .map( - entry -> - new Assignment( - entry.getVariable(), - new GravitinoColumnHandle( - getColumnName( - session, - GravitinoHandle.unWrap(handle), - entry.getColumn()), - entry.getColumn()), - entry.getType())) - .toList(), - result.getGroupingColumnMapping().entrySet().stream() - .collect( - Collectors.toMap( - entry -> - new GravitinoColumnHandle( - getColumnName( - session, - GravitinoHandle.unWrap(handle), - entry.getKey()), - entry.getKey()), - entry -> - new GravitinoColumnHandle( - getColumnName( - session, - GravitinoHandle.unWrap(handle), - entry.getValue()), - entry.getValue()))), - result.isPrecalculateStatistics())); + if (!(handle instanceof GravitinoTableHandle)) { + return internalMetadata.applyAggregation( + session, handle, aggregates, assignments, groupingSets); + } + + GravitinoTableHandle gravitinoTableHandle = (GravitinoTableHandle) handle; + return internalMetadata.applyAggregation( + session, + gravitinoTableHandle.getInternalTableHandle(), + aggregates, + assignments, + groupingSets); } @Override public Optional> applyLimit( ConnectorSession session, ConnectorTableHandle handle, long limit) { - return internalMetadata - .applyLimit(session, GravitinoHandle.unWrap(handle), limit) - .map( - result -> - new LimitApplicationResult( - new GravitinoTableHandle( - getTableName(handle).getSchemaName(), - getTableName(handle).getTableName(), - result.getHandle()), - result.isLimitGuaranteed(), - result.isPrecalculateStatistics())); + if (!(handle instanceof GravitinoTableHandle)) { + return internalMetadata.applyLimit(session, handle, limit); + } + + GravitinoTableHandle gravitinoTableHandle = (GravitinoTableHandle) handle; + return internalMetadata.applyLimit( + session, gravitinoTableHandle.getInternalTableHandle(), limit); } @Override @@ -526,45 +442,32 @@ public Optional> applyTopN( long topNCount, List sortItems, Map assignments) { - return internalMetadata - .applyTopN( - session, - GravitinoHandle.unWrap(handle), - topNCount, - sortItems, - assignments.entrySet().stream() - .collect( - Collectors.toMap( - Map.Entry::getKey, entry -> GravitinoHandle.unWrap(entry.getValue())))) - .map( - result -> - new TopNApplicationResult( - new GravitinoTableHandle( - getTableName(handle).getSchemaName(), - getTableName(handle).getTableName(), - result.getHandle()), - result.isTopNGuaranteed(), - result.isPrecalculateStatistics())); + if (!(handle instanceof GravitinoTableHandle)) { + return internalMetadata.applyTopN(session, handle, topNCount, sortItems, assignments); + } + + GravitinoTableHandle gravitinoTableHandle = (GravitinoTableHandle) handle; + return internalMetadata.applyTopN( + session, gravitinoTableHandle.getInternalTableHandle(), topNCount, sortItems, assignments); } @Override public TableStatistics getTableStatistics( ConnectorSession session, ConnectorTableHandle tableHandle) { - return internalMetadata.getTableStatistics(session, GravitinoHandle.unWrap(tableHandle)); - } - - private SchemaTableName getTableName(ConnectorTableHandle tableHandle) { - return ((GravitinoTableHandle) tableHandle).toSchemaTableName(); - } + if (!(tableHandle instanceof GravitinoTableHandle)) { + return internalMetadata.getTableStatistics(session, tableHandle); + } - private String getColumnName(ColumnHandle columnHandle) { - return ((GravitinoColumnHandle) columnHandle).getColumnName(); + GravitinoTableHandle gravitinoTableHandle = (GravitinoTableHandle) tableHandle; + return internalMetadata.getTableStatistics( + session, gravitinoTableHandle.getInternalTableHandle()); } private String getColumnName( - ConnectorSession session, ConnectorTableHandle tableHandle, ColumnHandle columnHandle) { + ConnectorSession session, GravitinoTableHandle tableHandle, ColumnHandle columnHandle) { ColumnMetadata internalMetadataColumnMetadata = - internalMetadata.getColumnMetadata(session, tableHandle, columnHandle); + internalMetadata.getColumnMetadata( + session, tableHandle.getInternalTableHandle(), columnHandle); if (internalMetadataColumnMetadata == null) { throw new TrinoException( GRAVITINO_COLUMN_NOT_EXISTS, diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoPageSinkProvider.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoPageSinkProvider.java index e39b981451a..0d627cb58d4 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoPageSinkProvider.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoPageSinkProvider.java @@ -11,7 +11,6 @@ import io.trino.spi.connector.ConnectorPageSinkProvider; import io.trino.spi.connector.ConnectorSession; import io.trino.spi.connector.ConnectorTransactionHandle; -import org.apache.commons.lang3.NotImplementedException; /** This class provides a ConnectorPageSink for trino to write data to internal connector. */ public class GravitinoPageSinkProvider implements ConnectorPageSinkProvider { @@ -28,7 +27,7 @@ public ConnectorPageSink createPageSink( ConnectorSession session, ConnectorOutputTableHandle outputTableHandle, ConnectorPageSinkId pageSinkId) { - throw new NotImplementedException(); + return null; } @Override @@ -37,10 +36,15 @@ public ConnectorPageSink createPageSink( ConnectorSession session, ConnectorInsertTableHandle insertTableHandle, ConnectorPageSinkId pageSinkId) { + GravitinoTransactionHandle gravitinoTransactionHandle = + (GravitinoTransactionHandle) transactionHandle; + GravitinoInsertTableHandle gravitinoInsertTableHandle = + (GravitinoInsertTableHandle) insertTableHandle; + return pageSinkProvider.createPageSink( - GravitinoHandle.unWrap(transactionHandle), + gravitinoTransactionHandle.getInternalTransactionHandle(), session, - GravitinoHandle.unWrap(insertTableHandle), + gravitinoInsertTableHandle.innerHandler(), pageSinkId); } } diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoPredicate.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoPredicate.java deleted file mode 100644 index db381b07873..00000000000 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoPredicate.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. - */ -package com.datastrato.gravitino.trino.connector; - -import io.trino.spi.connector.ColumnHandle; -import io.trino.spi.predicate.NullableValue; -import java.util.Map; -import java.util.function.Predicate; - -/** The GravitinoPredicate is used to warp Predicate */ -public class GravitinoPredicate implements Predicate> { - - private final Predicate delegate; - - GravitinoPredicate(Predicate> predicate) { - this.delegate = predicate; - } - - @Override - public boolean test(Map columnHandleNullableValueMap) { - return delegate.test(columnHandleNullableValueMap); - } - - @Override - public Predicate> and( - Predicate> other) { - return delegate.and(other); - } - - @Override - public Predicate> negate() { - return delegate.negate(); - } - - @Override - public Predicate> or( - Predicate> other) { - return delegate.or(other); - } -} diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoRecordSetProvider.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoRecordSetProvider.java index 30c41a349a8..94b222b6154 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoRecordSetProvider.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoRecordSetProvider.java @@ -29,11 +29,18 @@ public RecordSet getRecordSet( ConnectorSplit split, ConnectorTableHandle table, List columns) { + if (!(table instanceof GravitinoTableHandle)) { + return internalRecordSetProvider.getRecordSet(transaction, session, split, table, columns); + } + + GravitinoTableHandle gravitinoTableHandle = (GravitinoTableHandle) table; + GravitinoTransactionHandle gravitinoTransactionHandle = + (GravitinoTransactionHandle) transaction; return internalRecordSetProvider.getRecordSet( - GravitinoHandle.unWrap(transaction), + gravitinoTransactionHandle.getInternalTransactionHandle(), session, - GravitinoHandle.unWrap(split), - GravitinoHandle.unWrap(table), - GravitinoHandle.unWrap(columns)); + split, + gravitinoTableHandle.getInternalTableHandle(), + columns); } } diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoSplit.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoSplit.java deleted file mode 100644 index 80b6fc2dfc9..00000000000 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoSplit.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. - */ -package com.datastrato.gravitino.trino.connector; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import io.trino.spi.HostAddress; -import io.trino.spi.SplitWeight; -import io.trino.spi.connector.ConnectorSplit; -import java.util.List; - -/** - * The GravitinoFTransactionHandle is used to make Gravitino metadata operations transactional and - * wrap the inner connector transaction for data access. - */ -public class GravitinoSplit implements ConnectorSplit, GravitinoHandle { - - private HandleWrapper handleWrapper = new HandleWrapper<>(ConnectorSplit.class); - - @JsonCreator - public GravitinoSplit(@JsonProperty(HANDLE_STRING) String handleString) { - this.handleWrapper = handleWrapper.fromJson(handleString); - } - - public GravitinoSplit(ConnectorSplit split) { - this.handleWrapper = new HandleWrapper<>(split); - } - - @JsonProperty - @Override - public String getHandleString() { - return handleWrapper.toJson(); - } - - @Override - public ConnectorSplit getInternalHandle() { - return handleWrapper.getHandle(); - } - - @Override - public boolean isRemotelyAccessible() { - return handleWrapper.getHandle().isRemotelyAccessible(); - } - - @Override - public List getAddresses() { - return handleWrapper.getHandle().getAddresses(); - } - - @Override - public Object getInfo() { - return handleWrapper.getHandle().getInfo(); - } - - @Override - public SplitWeight getSplitWeight() { - return handleWrapper.getHandle().getSplitWeight(); - } - - @Override - public long getRetainedSizeInBytes() { - return handleWrapper.getHandle().getRetainedSizeInBytes(); - } -} diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoSplitManager.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoSplitManager.java index 0ad9657aa31..2fa6b286c3e 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoSplitManager.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoSplitManager.java @@ -27,13 +27,22 @@ public ConnectorSplitSource getSplits( ConnectorTableHandle connectorTableHandle, DynamicFilter dynamicFilter, Constraint constraint) { - ConnectorSplitSource splits = - internalSplitManager.getSplits( - GravitinoHandle.unWrap(transaction), - session, - GravitinoHandle.unWrap(connectorTableHandle), - new GravitinoDynamicFilter(dynamicFilter), - constraint); - return new GravitinoSplitSource(splits); + if (!(connectorTableHandle instanceof GravitinoTableHandle)) { + if (transaction instanceof GravitinoTransactionHandle) { + transaction = ((GravitinoTransactionHandle) transaction).getInternalTransactionHandle(); + } + return internalSplitManager.getSplits( + transaction, session, connectorTableHandle, dynamicFilter, constraint); + } + GravitinoTableHandle gravitinoTableHandle = (GravitinoTableHandle) connectorTableHandle; + GravitinoTransactionHandle gravitinoTransactionHandle = + (GravitinoTransactionHandle) transaction; + + return internalSplitManager.getSplits( + gravitinoTransactionHandle.getInternalTransactionHandle(), + session, + gravitinoTableHandle.getInternalTableHandle(), + dynamicFilter, + constraint); } } diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoSplitSource.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoSplitSource.java deleted file mode 100644 index 04f8ab19c56..00000000000 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoSplitSource.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. - */ -package com.datastrato.gravitino.trino.connector; - -import io.trino.spi.connector.ConnectorSplit; -import io.trino.spi.connector.ConnectorSplitSource; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.CompletableFuture; -import java.util.stream.Collectors; - -/** - * The GravitinoFTransactionHandle is used to make Gravitino metadata operations transactional and - * wrap the inner connector transaction for data access. - */ -public class GravitinoSplitSource implements ConnectorSplitSource { - - private final ConnectorSplitSource connectorSplitSource; - - public GravitinoSplitSource(ConnectorSplitSource connectorSplitSource) { - this.connectorSplitSource = connectorSplitSource; - } - - @Override - public CompletableFuture getNextBatch(int maxSize) { - ConnectorSplitBatch batch = connectorSplitSource.getNextBatch(maxSize).join(); - List list = - batch.getSplits().stream().map(GravitinoSplit::new).collect(Collectors.toList()); - return CompletableFuture.completedFuture(new ConnectorSplitBatch(list, batch.isNoMoreSplits())); - } - - @Override - public void close() { - connectorSplitSource.close(); - } - - @Override - public boolean isFinished() { - return connectorSplitSource.isFinished(); - } - - @Override - public Optional> getTableExecuteSplitsInfo() { - return connectorSplitSource.getTableExecuteSplitsInfo(); - } -} diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoTableHandle.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoTableHandle.java index b60282f893a..5d3c920eaa8 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoTableHandle.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoTableHandle.java @@ -15,38 +15,25 @@ * The GravitinoTableHandle is used to transform table information between Trino and Gravitino, as * well as to wrap the inner connector table handle for data access. */ -public final class GravitinoTableHandle - implements ConnectorTableHandle, GravitinoHandle { +public final class GravitinoTableHandle implements ConnectorTableHandle { private final String schemaName; private final String tableName; - private HandleWrapper handleWrapper = - new HandleWrapper<>(ConnectorTableHandle.class); + private final ConnectorTableHandle internalTableHandle; @JsonCreator public GravitinoTableHandle( @JsonProperty("schemaName") String schemaName, @JsonProperty("tableName") String tableName, - @JsonProperty(HANDLE_STRING) String handleString) { - Preconditions.checkArgument(schemaName != null, "schemaName is not null"); - Preconditions.checkArgument(tableName != null, "tableName is not null"); - Preconditions.checkArgument(handleString != null, "handleString is not null"); - - this.schemaName = schemaName; - this.tableName = tableName; - this.handleWrapper = handleWrapper.fromJson(handleString); - } - - public GravitinoTableHandle( - String schemaName, String tableName, ConnectorTableHandle internalTableHandle) { + @JsonProperty("internalTableHandle") ConnectorTableHandle internalTableHandle) { Preconditions.checkArgument(schemaName != null, "schemaName is not null"); Preconditions.checkArgument(tableName != null, "tableName is not null"); Preconditions.checkArgument(internalTableHandle != null, "internalTableHandle is not null"); this.schemaName = schemaName; this.tableName = tableName; - this.handleWrapper = new HandleWrapper<>(internalTableHandle); + this.internalTableHandle = internalTableHandle; } @JsonProperty @@ -60,14 +47,8 @@ public String getTableName() { } @JsonProperty - @Override - public String getHandleString() { - return handleWrapper.toJson(); - } - - @Override - public ConnectorTableHandle getInternalHandle() { - return handleWrapper.getHandle(); + public ConnectorTableHandle getInternalTableHandle() { + return internalTableHandle; } public SchemaTableName toSchemaTableName() { diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoTransactionHandle.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoTransactionHandle.java index 06fd46d3d42..9aa5f8e15cb 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoTransactionHandle.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoTransactionHandle.java @@ -12,29 +12,18 @@ * The GravitinoFTransactionHandle is used to make Gravitino metadata operations transactional and * wrap the inner connector transaction for data access. */ -public class GravitinoTransactionHandle - implements ConnectorTransactionHandle, GravitinoHandle { - - private HandleWrapper handleWrapper = - new HandleWrapper<>(ConnectorTransactionHandle.class); +public class GravitinoTransactionHandle implements ConnectorTransactionHandle { + ConnectorTransactionHandle internalTransactionHandle; @JsonCreator - public GravitinoTransactionHandle(@JsonProperty(HANDLE_STRING) String handleString) { - this.handleWrapper = handleWrapper.fromJson(handleString); - } - - public GravitinoTransactionHandle(ConnectorTransactionHandle internalTransactionHandle) { - this.handleWrapper = new HandleWrapper<>(internalTransactionHandle); + public GravitinoTransactionHandle( + @JsonProperty("internalTransactionHandle") + ConnectorTransactionHandle internalTransactionHandler) { + this.internalTransactionHandle = internalTransactionHandler; } @JsonProperty - @Override - public String getHandleString() { - return handleWrapper.toJson(); - } - - @Override - public ConnectorTransactionHandle getInternalHandle() { - return handleWrapper.getHandle(); + public ConnectorTransactionHandle getInternalTransactionHandle() { + return internalTransactionHandle; } } diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorAdapter.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorAdapter.java index 75eb0c488a7..0930d266584 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorAdapter.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorAdapter.java @@ -22,10 +22,7 @@ default List> getTableProperties() { } /** @return Return internal connector config with Trino. */ - Map buildInternalConnectorConfig(GravitinoCatalog catalog) throws Exception; - - /** @return Return internal connector name with Trino. */ - String internalConnectorName(); + Map buildInternalConnectorConfig(GravitinoCatalog catalog) throws Exception; /** @return SchemaProperties list that used to validate schema properties. */ default List> getSchemaProperties() { diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorContext.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorContext.java index 6f85df6c10c..838f99ab949 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorContext.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorContext.java @@ -6,11 +6,9 @@ import com.datastrato.gravitino.client.GravitinoMetalake; import com.datastrato.gravitino.trino.connector.GravitinoConnector; -import com.datastrato.gravitino.trino.connector.GravitinoConnectorPluginManager; import com.datastrato.gravitino.trino.connector.metadata.GravitinoCatalog; import com.google.common.base.Preconditions; import io.trino.spi.connector.Connector; -import io.trino.spi.connector.ConnectorContext; import io.trino.spi.session.PropertyMetadata; import java.util.List; import java.util.Map; @@ -84,21 +82,20 @@ public CatalogConnectorMetadataAdapter getMetadataAdapter() { static class Builder { private final CatalogConnectorAdapter connectorAdapter; - private GravitinoCatalog catalog; private GravitinoMetalake metalake; - private ConnectorContext context; + private Connector internalConnector; + private GravitinoCatalog catalog; Builder(CatalogConnectorAdapter connectorAdapter) { this.connectorAdapter = connectorAdapter; } - private Builder(CatalogConnectorAdapter connectorAdapter, GravitinoCatalog catalog) { - this.connectorAdapter = connectorAdapter; - this.catalog = catalog; + public Builder clone() { + return new Builder(connectorAdapter); } - public Builder clone(GravitinoCatalog catalog) { - return new Builder(connectorAdapter, catalog); + public Map buildConfig(GravitinoCatalog catalog) throws Exception { + return connectorAdapter.buildInternalConnectorConfig(catalog); } Builder withMetalake(GravitinoMetalake metalake) { @@ -106,22 +103,21 @@ Builder withMetalake(GravitinoMetalake metalake) { return this; } - Builder withContext(ConnectorContext context) { - this.context = context; + Builder withInternalConnector(Connector internalConnector) { + this.internalConnector = internalConnector; + return this; + } + + Builder withCatalog(GravitinoCatalog catalog) { + this.catalog = catalog; return this; } - CatalogConnectorContext build() throws Exception { + CatalogConnectorContext build() { Preconditions.checkArgument(metalake != null, "metalake is not null"); + Preconditions.checkArgument(internalConnector != null, "internalConnector is not null"); Preconditions.checkArgument(catalog != null, "catalog is not null"); - Preconditions.checkArgument(context != null, "context is not null"); - Map connectorConfig = connectorAdapter.buildInternalConnectorConfig(catalog); - String internalConnectorName = connectorAdapter.internalConnectorName(); - - Connector connector = - GravitinoConnectorPluginManager.instance(context.getClass().getClassLoader()) - .createConnector(internalConnectorName, connectorConfig, context); - return new CatalogConnectorContext(catalog, metalake, connector, connectorAdapter); + return new CatalogConnectorContext(catalog, metalake, internalConnector, connectorAdapter); } } } diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorFactory.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorFactory.java index cd1f8a053d5..1b8ab569247 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorFactory.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorFactory.java @@ -46,6 +46,6 @@ public CatalogConnectorContext.Builder createCatalogConnectorContextBuilder( } // Avoid using the same builder object to prevent catalog creation errors. - return builder.clone(catalog); + return builder.clone(); } } diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorManager.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorManager.java index 3ab755b4389..799f87f631f 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorManager.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorManager.java @@ -22,11 +22,11 @@ import com.datastrato.gravitino.exceptions.NoSuchMetalakeException; import com.datastrato.gravitino.trino.connector.GravitinoConfig; import com.datastrato.gravitino.trino.connector.metadata.GravitinoCatalog; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.util.concurrent.ThreadFactoryBuilder; import io.trino.spi.TrinoException; import io.trino.spi.connector.Connector; -import io.trino.spi.connector.ConnectorContext; import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; @@ -61,21 +61,19 @@ public class CatalogConnectorManager { private static final int LOAD_METALAKE_TIMEOUT = 30; private final ScheduledExecutorService executorService; - private final CatalogRegister catalogRegister; + private final CatalogInjector catalogInjector; private final CatalogConnectorFactory catalogConnectorFactory; private final ConcurrentHashMap catalogConnectors = new ConcurrentHashMap<>(); - private final Set usedMetalakes = new HashSet<>(); - private final Map metalakes = new ConcurrentHashMap<>(); - private GravitinoAdminClient gravitinoClient; private GravitinoConfig config; + private final Set usedMetalakes = new HashSet<>(); public CatalogConnectorManager( - CatalogRegister catalogRegister, CatalogConnectorFactory catalogFactory) { - this.catalogRegister = catalogRegister; + CatalogInjector catalogInjector, CatalogConnectorFactory catalogFactory) { + this.catalogInjector = catalogInjector; this.catalogConnectorFactory = catalogFactory; this.executorService = createScheduledThreadPoolExecutor(); } @@ -92,56 +90,54 @@ private static ScheduledThreadPoolExecutor createScheduledThreadPoolExecutor() { .build()); } - public void config(GravitinoConfig config, GravitinoAdminClient client) { + public void config(GravitinoConfig config) { this.config = Preconditions.checkNotNull(config, "config is not null"); + } + + public void start(GravitinoAdminClient client) { if (client == null) { this.gravitinoClient = GravitinoAdminClient.builder(config.getURI()).build(); } else { this.gravitinoClient = client; } + + LOG.info("Gravitino CatalogConnectorManager started."); } - public void start(ConnectorContext context) throws Exception { - catalogRegister.init(context, config); - if (catalogRegister.isCoordinator()) { + public void loadMetalakeSync() { + try { + Future future = executorService.submit(this::loadMetalakeImpl); + future.get(); + } catch (Exception e) { + LOG.error("Load metalake sync failed.", e); + } finally { + // Load metalake for handling catalog in the metalake updates. executorService.scheduleWithFixedDelay( - this::loadMetalake, + this::loadMetalakeImpl, CATALOG_LOAD_FREQUENCY_SECOND, CATALOG_LOAD_FREQUENCY_SECOND, TimeUnit.SECONDS); } - - LOG.info("Gravitino CatalogConnectorManager started."); } - private void loadMetalake() { - if (!catalogRegister.isTrinoStarted()) { - LOG.info("Waiting for the Trino started."); - return; - } - + private void loadMetalakeImpl() { for (String usedMetalake : usedMetalakes) { + GravitinoMetalake metalake; try { - GravitinoMetalake metalake = - metalakes.computeIfAbsent(usedMetalake, this::retrieveMetalake); + metalake = gravitinoClient.loadMetalake(usedMetalake); + LOG.info("Load metalake: {}", usedMetalake); loadCatalogs(metalake); + } catch (NoSuchMetalakeException noSuchMetalakeException) { + LOG.warn("Metalake {} does not exist.", usedMetalake); } catch (Exception e) { LOG.error("Load Metalake {} failed.", usedMetalake, e); } } } - private GravitinoMetalake retrieveMetalake(String metalakeName) { - try { - return gravitinoClient.loadMetalake(metalakeName); - } catch (NoSuchMetalakeException e) { - throw new TrinoException( - GRAVITINO_METALAKE_NOT_EXISTS, "Metalake " + metalakeName + " not exists."); - } - } - - private void loadCatalogs(GravitinoMetalake metalake) { + @VisibleForTesting + public void loadCatalogs(GravitinoMetalake metalake) { NameIdentifier[] catalogNames; try { catalogNames = metalake.listCatalogs(); @@ -150,19 +146,17 @@ private void loadCatalogs(GravitinoMetalake metalake) { return; } - LOG.info( - "Load metalake {}'s catalogs. catalogs: {}.", - metalake.name(), - Arrays.toString(catalogNames)); + if (LOG.isInfoEnabled()) { + LOG.info( + "Load metalake {}'s catalogs. catalogs: {}.", + metalake.name(), + Arrays.toString(catalogNames)); + } // Delete those catalogs that have been deleted in Gravitino server Set catalogNameStrings = Arrays.stream(catalogNames) - .map( - id -> - config.simplifyCatalogNames() - ? id.name() - : getTrinoCatalogName(metalake.name(), id.name())) + .map(config.simplifyCatalogNames() ? NameIdentifier::name : NameIdentifier::toString) .collect(Collectors.toSet()); for (Map.Entry entry : catalogConnectors.entrySet()) { @@ -171,7 +165,7 @@ private void loadCatalogs(GravitinoMetalake metalake) { // Skip the catalog doesn't belong to this metalake. entry.getValue().getMetalake().name().equals(metalake.name())) { try { - unloadCatalog(entry.getValue().getCatalog()); + unloadCatalog(metalake, entry.getKey()); } catch (Exception e) { LOG.error("Failed to remove catalog {}.", entry.getKey(), e); } @@ -187,10 +181,11 @@ private void loadCatalogs(GravitinoMetalake metalake) { GravitinoCatalog gravitinoCatalog = new GravitinoCatalog(metalake.name(), catalog); if (catalogConnectors.containsKey(getTrinoCatalogName(gravitinoCatalog))) { // Reload catalogs that have been updated in Gravitino server. - reloadCatalog(gravitinoCatalog); + reloadCatalog(metalake, gravitinoCatalog); + } else { if (catalog.type() == Catalog.Type.RELATIONAL) { - loadCatalog(gravitinoCatalog); + loadCatalog(metalake, gravitinoCatalog); } } } catch (Exception e) { @@ -200,44 +195,48 @@ private void loadCatalogs(GravitinoMetalake metalake) { }); } - private void reloadCatalog(GravitinoCatalog catalog) { - String catalogFullName = getTrinoCatalogName(catalog); - GravitinoCatalog oldCatalog = catalogConnectors.get(catalogFullName).getCatalog(); - if (catalog.getLastModifiedTime() <= oldCatalog.getLastModifiedTime()) { + private void reloadCatalog(GravitinoMetalake metalake, GravitinoCatalog catalog) { + GravitinoCatalog oldCatalog = catalogConnectors.get(getTrinoCatalogName(catalog)).getCatalog(); + if (!catalog.getLastModifiedTime().isAfter(oldCatalog.getLastModifiedTime())) { return; } - catalogRegister.unregisterCatalog(catalogFullName); - catalogConnectors.remove(catalogFullName); + catalogInjector.removeCatalogConnector((getTrinoCatalogName(catalog))); + catalogConnectors.remove(getTrinoCatalogName(catalog)); - loadCatalogImpl(catalog); - LOG.info("Update catalog '{}' in metalake {} successfully.", catalog, catalog.getMetalake()); + loadCatalogImpl(metalake, catalog); + LOG.info("Update catalog '{}' in metalake {} successfully.", catalog, metalake.name()); } - private void loadCatalog(GravitinoCatalog catalog) { - loadCatalogImpl(catalog); - LOG.info("Load catalog {} in metalake {} successfully.", catalog, catalog.getMetalake()); + private void loadCatalog(GravitinoMetalake metalake, GravitinoCatalog catalog) { + loadCatalogImpl(metalake, catalog); + LOG.info("Load catalog {} in metalake {} successfully.", catalog, metalake.name()); } - private void loadCatalogImpl(GravitinoCatalog catalog) { + private void loadCatalogImpl(GravitinoMetalake metalake, GravitinoCatalog catalog) { + CatalogConnectorContext.Builder builder = + catalogConnectorFactory.createCatalogConnectorContextBuilder(catalog); try { - catalogRegister.registerCatalog(getTrinoCatalogName(catalog), catalog); + Connector internalConnector = + catalogInjector.createConnector( + getTrinoCatalogName(catalog), builder.buildConfig(catalog)); + + builder.withMetalake(metalake).withCatalog(catalog).withInternalConnector(internalConnector); } catch (Exception e) { String message = String.format("Failed to create internal catalog connector. The catalog is: %s", catalog); LOG.error(message, e); throw new TrinoException(GRAVITINO_CREATE_INTERNAL_CONNECTOR_ERROR, message, e); } + + catalogConnectors.put(getTrinoCatalogName(catalog), builder.build()); + catalogInjector.injectCatalogConnector(getTrinoCatalogName(catalog)); } - private void unloadCatalog(GravitinoCatalog catalog) { - String catalogFullName = getTrinoCatalogName(catalog); - catalogRegister.unregisterCatalog(catalogFullName); + private void unloadCatalog(GravitinoMetalake metalake, String catalogFullName) { + catalogInjector.removeCatalogConnector(catalogFullName); catalogConnectors.remove(catalogFullName); - LOG.info( - "Remove catalog '{}' in metalake {} successfully.", - catalog.getName(), - catalog.getMetalake()); + LOG.info("Remove catalog '{}' in metalake {} successfully.", catalogFullName, metalake.name()); } public CatalogConnectorContext getCatalogConnector(String catalogName) { @@ -254,7 +253,7 @@ public void shutdown() { } public String getTrinoCatalogName(String metalake, String catalog) { - return config.simplifyCatalogNames() ? catalog : String.format("\"%s.%s\"", metalake, catalog); + return config.simplifyCatalogNames() ? catalog : metalake + "." + catalog; } public String getTrinoCatalogName(GravitinoCatalog catalog) { @@ -284,7 +283,7 @@ public void createCatalog( LOG.info("Create catalog {} in metalake {} successfully.", catalogName, metalake); - Future future = executorService.submit(this::loadMetalake); + Future future = executorService.submit(this::loadMetalakeImpl); future.get(LOAD_METALAKE_TIMEOUT, TimeUnit.SECONDS); if (!catalogConnectors.containsKey(getTrinoCatalogName(metalakeName, catalogName))) { @@ -309,6 +308,8 @@ public void createCatalog( public void dropCatalog(String metalakeName, String catalogName, boolean ignoreNotExist) { try { GravitinoMetalake metalake = gravitinoClient.loadMetalake(metalakeName); + + // NameIdentifier catalog = NameIdentifier.of(metalakeName, catalogName); if (!metalake.catalogExists(catalogName)) { if (ignoreNotExist) { return; @@ -321,11 +322,11 @@ public void dropCatalog(String metalakeName, String catalogName, boolean ignoreN boolean dropped = metalake.dropCatalog(catalogName); if (!dropped) { throw new TrinoException( - GRAVITINO_UNSUPPORTED_OPERATION, "Failed to drop no empty catalog " + catalogName); + GRAVITINO_UNSUPPORTED_OPERATION, "Drop catalog " + catalogName + " does not support."); } LOG.info("Drop catalog {} in metalake {} successfully.", catalogName, metalake); - Future future = executorService.submit(this::loadMetalake); + Future future = executorService.submit(this::loadMetalakeImpl); future.get(LOAD_METALAKE_TIMEOUT, TimeUnit.SECONDS); if (catalogConnectors.containsKey(getTrinoCatalogName(metalakeName, catalogName))) { @@ -383,14 +384,16 @@ public void alterCatalog( GravitinoMetalake metalake = gravitinoClient.loadMetalake(metalakeName); metalake.alterCatalog(catalogName, changes.toArray(changes.toArray(new CatalogChange[0]))); - Future future = executorService.submit(this::loadMetalake); + Future future = executorService.submit(this::loadMetalakeImpl); future.get(LOAD_METALAKE_TIMEOUT, TimeUnit.SECONDS); catalogConnectorContext = catalogConnectors.get(getTrinoCatalogName(metalakeName, catalogName)); if (catalogConnectorContext == null - || catalogConnectorContext.getCatalog().getLastModifiedTime() - == oldCatalog.getLastModifiedTime()) { + || catalogConnectorContext + .getCatalog() + .getLastModifiedTime() + .equals(oldCatalog.getLastModifiedTime())) { throw new TrinoException( GRAVITINO_OPERATION_FAILED, "Update catalog failed due to the reloading process fails"); } @@ -417,27 +420,4 @@ public void addMetalake(String metalake) { public Set getUsedMetalakes() { return usedMetalakes; } - - public Connector createConnector( - String connectorName, GravitinoConfig config, ConnectorContext context) { - try { - String catalogConfig = config.getCatalogConfig(); - - GravitinoCatalog catalog = GravitinoCatalog.fromJson(catalogConfig); - CatalogConnectorContext.Builder builder = - catalogConnectorFactory.createCatalogConnectorContextBuilder(catalog); - builder - .withMetalake(metalakes.computeIfAbsent(catalog.getMetalake(), this::retrieveMetalake)) - .withContext(context); - - CatalogConnectorContext connectorContext = builder.build(); - catalogConnectors.put(connectorName, connectorContext); - LOG.info("Create connector {} successful", connectorName); - return connectorContext.getConnector(); - } catch (Exception e) { - LOG.error("Failed to create connector: {}", connectorName, e); - throw new TrinoException( - GRAVITINO_OPERATION_FAILED, "Failed to create connector: " + connectorName, e); - } - } } diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorMetadata.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorMetadata.java index 597b684de1e..70b95395dfe 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorMetadata.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorMetadata.java @@ -112,7 +112,7 @@ public boolean tableExists(String schemaName, String tableName) { NameIdentifier.ofTable(metalake.name(), catalogName, schemaName, tableName)); } - public void createTable(GravitinoTable table, boolean ignoreExisting) { + public void createTable(GravitinoTable table) { NameIdentifier identifier = NameIdentifier.ofTable( metalake.name(), catalogName, table.getSchemaName(), table.getName()); @@ -128,9 +128,7 @@ public void createTable(GravitinoTable table, boolean ignoreExisting) { } catch (NoSuchSchemaException e) { throw new TrinoException(GRAVITINO_SCHEMA_NOT_EXISTS, SCHEMA_DOES_NOT_EXIST_MSG, e); } catch (TableAlreadyExistsException e) { - if (!ignoreExisting) { - throw new TrinoException(GRAVITINO_TABLE_ALREADY_EXISTS, "Table already exists", e); - } + throw new TrinoException(GRAVITINO_TABLE_ALREADY_EXISTS, "Table already exists", e); } } diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogInjector.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogInjector.java new file mode 100644 index 00000000000..a33a9b85984 --- /dev/null +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogInjector.java @@ -0,0 +1,356 @@ +/* + * Copyright 2023 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.trino.connector.catalog; + +import static com.datastrato.gravitino.trino.connector.GravitinoErrorCode.GRAVITINO_CREATE_INNER_CONNECTOR_FAILED; +import static com.datastrato.gravitino.trino.connector.GravitinoErrorCode.GRAVITINO_UNSUPPORTED_TRINO_VERSION; + +import com.datastrato.gravitino.trino.connector.GravitinoConfig; +import com.datastrato.gravitino.trino.connector.GravitinoErrorCode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Preconditions; +import io.trino.spi.TrinoException; +import io.trino.spi.connector.Connector; +import io.trino.spi.connector.ConnectorContext; +import io.trino.spi.connector.MetadataProvider; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This class dynamically injects the Catalog managed by Gravitino into Trino using reflection + * techniques. It allows it to be used in Trino like a regular Trino catalog. In Gravitino, the + * catalog name consists of the "metalake" and catalog name, for example, "user_0.hive_us." We can + * use it directly in Trino. + */ +public class CatalogInjector { + + private static final Logger LOG = LoggerFactory.getLogger(CatalogInjector.class); + + private static final int MIN_TRINO_SPI_VERSION = 360; + + // It is used to inject catalogs to trino + private InjectCatalogHandle injectHandle; + // It's used to remove catalogs from trino + private RemoveCatalogHandle removeHandle; + + // It is used to create internal catalogs. + private CreateCatalogHandle createHandle; + private String trinoVersion; + + private ConcurrentHashMap internalCatalogs = new ConcurrentHashMap<>(); + + private void checkTrinoSpiVersion(ConnectorContext context) { + this.trinoVersion = context.getSpiVersion(); + + int version = Integer.parseInt(context.getSpiVersion()); + if (version < MIN_TRINO_SPI_VERSION) { + String errmsg = + String.format( + "Unsupported trino-%s version. min support version is trino-%d", + trinoVersion, MIN_TRINO_SPI_VERSION); + throw new TrinoException(GravitinoErrorCode.GRAVITINO_UNSUPPORTED_TRINO_VERSION, errmsg); + } + } + + private static Field getField(Object targetObject, String fieldName) throws NoSuchFieldException { + Field field = targetObject.getClass().getDeclaredField(fieldName); + field.setAccessible(true); + return field; + } + + private static Object getFiledObject(Object targetObject, String fieldName) + throws NoSuchFieldException, IllegalAccessException { + return getField(targetObject, fieldName).get(targetObject); + } + + private static boolean isClassObject(Object targetObject, String className) { + return targetObject.getClass().getName().endsWith(className); + } + + private static Class getClass(ClassLoader classLoader, String className) + throws ClassNotFoundException { + return classLoader.loadClass(className); + } + + /** + * @param context + * 
+   *  This function does the following tasks by ConnectorContext:
+   *  1. Retrieve the DiscoveryNodeManager object.
+   *  2. To enable Trino to handle tables on every node,
+   *  set 'allCatalogsOnAllNodes' to 'true' and 'activeNodesByCatalogHandle' to empty.
+   *  3. Retrieve the catalogManager object.
+   *  4. Get createCatalog function in catalogFactory
+   *  5. Create a CreateCatalogHandle for the Gravitino connector's internal connector.
+   *  6. Create InjectCatalogHandle for injection catalogs to trino.
+   *
+   *  A runtime ConnectorContext hierarchy:
+   *  context (ConnectorContext)
+   *  --nodeManager (ConnectorAwareNodeManager)
+   *  ----nodeManager (DiscoveryNodeManager)
+   *  ------nodeManager (DiscoveryNodeManager)
+   *  ------allCatalogsOnAllNodes (boolean)
+   *  ------activeNodesByCatalogHandle (SetMultimap)
+   *  --metadataProvider(InternalMetadataProvider)
+   *  ----metadata (TracingMetadata)
+   *  ------delegate (MetadataManager)
+   *  --------transactionManager (InMemoryTransactionManager)
+   *  ----------catalogManager (StaticCatalogManager)
+   *  ------------catalogFactory (LazyCatalogFactory)
+   *  --------------createCatalog() (Function)
+   *  ------------catalogs (ConcurrentHashMap)
+   *
+ */ + public void init(ConnectorContext context) { + // Injector trino catalog need NodeManager support allCatalogsOnAllNodes; + checkTrinoSpiVersion(context); + + try { + // 1. Retrieve the DiscoveryNodeManager object. + Object nodeManager = context.getNodeManager(); + nodeManager = getFiledObject(nodeManager, "nodeManager"); + + if (isClassObject(nodeManager, "DiscoveryNodeManager")) { + // 2. To enable Trino to handle tables on every node + Field allCatalogsOnAllNodes = getField(nodeManager, "allCatalogsOnAllNodes"); + allCatalogsOnAllNodes.setBoolean(nodeManager, true); + + Field activeNodesByCatalogHandle = getField(nodeManager, "activeNodesByCatalogHandle"); + activeNodesByCatalogHandle.set(nodeManager, Optional.empty()); + } + + // 3. Retrieve the catalogManager object. + MetadataProvider metadataProvider = context.getMetadataProvider(); + + Object metadata = getFiledObject(metadataProvider, "metadata"); + Object metadataManager = metadata; + if (isClassObject(metadata, "TracingMetadata")) { + metadataManager = getFiledObject(metadata, "delegate"); + } + Preconditions.checkNotNull(metadataManager, "metadataManager should not be null"); + + Object transactionManager = getFiledObject(metadataManager, "transactionManager"); + Object catalogManager = getFiledObject(transactionManager, "catalogManager"); + Preconditions.checkNotNull(catalogManager, "catalogManager should not be null"); + + // 4. Get createCatalog function in catalogFactory + Object catalogFactory = getFiledObject(catalogManager, "catalogFactory"); + Preconditions.checkNotNull(catalogFactory, "catalogFactory should not be null"); + + Class catalogPropertiesClass = + getClass( + catalogManager.getClass().getClassLoader(), "io.trino.connector.CatalogProperties"); + Method createCatalogMethod = + catalogFactory.getClass().getDeclaredMethod("createCatalog", catalogPropertiesClass); + Preconditions.checkNotNull(createCatalogMethod, "createCatalogMethod should not be null"); + + // 5. Create a CreateCatalogHandle + createHandle = + (catalogName, catalogProperties) -> { + ObjectMapper objectMapper = new ObjectMapper(); + Object catalogPropertiesObject = + objectMapper.readValue(catalogProperties, catalogPropertiesClass); + + // Call catalogFactory.createCatalog() return CatalogConnector + Object catalogConnector = + createCatalogMethod.invoke(catalogFactory, catalogPropertiesObject); + internalCatalogs.put(catalogName, catalogConnector); + + // The catalogConnector hierarchy: + // --catalogConnector (CatalogConnector) + // ----catalogConnector (ConnectorServices) + // ------connector (Connector) + + // Get a connector object from trino CatalogConnector. + Object catalogConnectorObject = getFiledObject(catalogConnector, "catalogConnector"); + + return getFiledObject(catalogConnectorObject, "connector"); + }; + + // 6. Create InjectCatalogHandle + createInjectHandler( + catalogManager, catalogFactory, createCatalogMethod, catalogPropertiesClass); + + removeInjectHandle(catalogManager); + LOG.info("Bind Trino catalog manager successfully."); + } catch (Exception e) { + String message = + String.format( + "Bind Trino catalog manager failed, unsupported trino-%s version", trinoVersion); + LOG.error(message, e); + throw new TrinoException(GRAVITINO_UNSUPPORTED_TRINO_VERSION, message, e); + } + } + + private void removeInjectHandle(Object catalogManager) + throws NoSuchFieldException, IllegalAccessException { + if (isClassObject(catalogManager, "CoordinatorDynamicCatalogManager")) { + ConcurrentHashMap activeCatalogs = + (ConcurrentHashMap) getFiledObject(catalogManager, "activeCatalogs"); + Preconditions.checkNotNull(activeCatalogs, "activeCatalogs should not be null"); + + ConcurrentHashMap allCatalogs = + (ConcurrentHashMap) getFiledObject(catalogManager, "allCatalogs"); + Preconditions.checkNotNull(allCatalogs, "allCatalogs should not be null"); + + removeHandle = + (catalogName) -> { + activeCatalogs.remove(catalogName); + allCatalogs.remove(catalogName); + }; + } else { + // The catalogManager is an instance of StaticCatalogManager + ConcurrentHashMap catalogs = (ConcurrentHashMap) getFiledObject(catalogManager, "catalogs"); + Preconditions.checkNotNull(catalogs, "catalogs should not be null"); + removeHandle = + (catalogName) -> { + Object catalogConnector = catalogs.remove(catalogName); + if (catalogConnector != null) { + Method shutdown = catalogConnector.getClass().getDeclaredMethod("shutdown"); + shutdown.invoke(catalogConnector); + Object internalCatalogConnector = internalCatalogs.get(catalogName); + shutdown.invoke(internalCatalogConnector); + } + }; + } + } + + private void createInjectHandler( + Object catalogManager, + Object catalogFactory, + Method createCatalogMethod, + Class catalogPropertiesClass) + throws NoSuchFieldException, IllegalAccessException { + // The catalogManager is an instance of CoordinatorDynamicCatalogManager + if (isClassObject(catalogManager, "CoordinatorDynamicCatalogManager")) { + ConcurrentHashMap activeCatalogs = + (ConcurrentHashMap) getFiledObject(catalogManager, "activeCatalogs"); + Preconditions.checkNotNull(activeCatalogs, "activeCatalogs should not be null"); + + ConcurrentHashMap allCatalogs = + (ConcurrentHashMap) getFiledObject(catalogManager, "allCatalogs"); + Preconditions.checkNotNull(allCatalogs, "allCatalogs should not be null"); + + injectHandle = + (catalogName, catalogProperties) -> { + // Call CatalogFactory:createCatalog and add the catalog to + // CoordinatorDynamicCatalogManager + ObjectMapper objectMapper = new ObjectMapper(); + Object catalogPropertiesObject = + objectMapper.readValue(catalogProperties, catalogPropertiesClass); + Object catalogConnector = + createCatalogMethod.invoke(catalogFactory, catalogPropertiesObject); + + Field catalogField = catalogConnector.getClass().getDeclaredField("catalog"); + catalogField.setAccessible(true); + Object catalog = catalogField.get(catalogConnector); + activeCatalogs.put(catalogName, catalog); + + Field catelogHandleField = + catalogConnector.getClass().getDeclaredField("catalogHandle"); + catelogHandleField.setAccessible(true); + Object catalogHandle = catelogHandleField.get(catalogConnector); + allCatalogs.put(catalogHandle, catalogConnector); + }; + } else { + // The catalogManager is an instance of StaticCatalogManager + ConcurrentHashMap catalogs = (ConcurrentHashMap) getFiledObject(catalogManager, "catalogs"); + Preconditions.checkNotNull(catalogs, "catalogs should not be null"); + + injectHandle = + (catalogName, catalogProperties) -> { + // call CatalogFactory:createCatalog and add the catalog to StaticCatalogManager + ObjectMapper objectMapper = new ObjectMapper(); + Object catalogPropertiesObject = + objectMapper.readValue(catalogProperties, catalogPropertiesClass); + + Object catalogConnector = + createCatalogMethod.invoke(catalogFactory, catalogPropertiesObject); + if (catalogs.containsKey(catalogName)) { + String message = + String.format("Inject catalog failed. catalog %s already exists", catalogName); + LOG.error(message); + throw new TrinoException(GRAVITINO_CREATE_INNER_CONNECTOR_FAILED, message); + } + catalogs.put(catalogName, catalogConnector); + }; + } + } + + void removeCatalogConnector(String catalogName) { + try { + removeHandle.invoke(catalogName); + LOG.info("Remove trino catalog {} successfully.", catalogName); + } catch (Exception e) { + LOG.error("Remove trino catalog {} failed.", catalogName, e); + throw new TrinoException(GRAVITINO_CREATE_INNER_CONNECTOR_FAILED, e); + } + } + + void injectCatalogConnector(String catalogName) { + try { + String catalogProperties = createCatalogProperties(catalogName); + injectHandle.invoke(catalogName, catalogProperties); + + LOG.info("Inject trino catalog {} successfully.", catalogName); + } catch (Exception e) { + LOG.error("Inject trino catalog {} failed.", catalogName, e); + throw new TrinoException(GRAVITINO_CREATE_INNER_CONNECTOR_FAILED, e); + } + } + + String createCatalogProperties(String catalogName) { + String catalogPropertiesTemplate = + "{\"catalogHandle\": \"%s:normal:default\",\"connectorName\":\"gravitino\", \"properties\": " + + "{\"" + + GravitinoConfig.GRAVITINO_DYNAMIC_CONNECTOR + + "\": \"true\"}" + + "}"; + return String.format(catalogPropertiesTemplate, catalogName); + } + + Connector createConnector(String connectorName, Map properties) { + String connectorProperties; + try { + ObjectMapper objectMapper = new ObjectMapper(); + connectorProperties = objectMapper.writeValueAsString(properties); + if (LOG.isDebugEnabled()) { + LOG.debug( + "Create internal catalog connector {}. The config:{} .", + connectorName, + connectorProperties); + } + + Object catalogConnector = createHandle.invoke(connectorName, connectorProperties); + + LOG.info("Create internal catalog connector {} successfully.", connectorName); + return (Connector) catalogConnector; + } catch (Exception e) { + LOG.error( + "Create internal catalog connector {} failed. Connector properties: {} ", + connectorName, + properties, + e); + throw new TrinoException(GRAVITINO_CREATE_INNER_CONNECTOR_FAILED, e); + } + } + + interface InjectCatalogHandle { + void invoke(String name, String properties) throws Exception; + } + + interface CreateCatalogHandle { + Object invoke(String name, String properties) throws Exception; + } + + interface RemoveCatalogHandle { + void invoke(String catalogName) throws Exception; + } +} diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogRegister.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogRegister.java deleted file mode 100644 index ae12f299d77..00000000000 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogRegister.java +++ /dev/null @@ -1,235 +0,0 @@ -/* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. - */ -package com.datastrato.gravitino.trino.connector.catalog; - -import static com.datastrato.gravitino.trino.connector.GravitinoConfig.GRAVITINO_DYNAMIC_CONNECTOR; -import static com.datastrato.gravitino.trino.connector.GravitinoConfig.GRAVITINO_DYNAMIC_CONNECTOR_CATALOG_CONFIG; -import static com.datastrato.gravitino.trino.connector.GravitinoErrorCode.GRAVITINO_CATALOG_ALREADY_EXISTS; -import static com.datastrato.gravitino.trino.connector.GravitinoErrorCode.GRAVITINO_DUPLICATED_CATALOGS; -import static com.datastrato.gravitino.trino.connector.GravitinoErrorCode.GRAVITINO_MISSING_CONFIG; -import static com.datastrato.gravitino.trino.connector.GravitinoErrorCode.GRAVITINO_RUNTIME_ERROR; -import static com.datastrato.gravitino.trino.connector.GravitinoErrorCode.GRAVITINO_UNSUPPORTED_TRINO_VERSION; - -import com.datastrato.gravitino.trino.connector.GravitinoConfig; -import com.datastrato.gravitino.trino.connector.metadata.GravitinoCatalog; -import io.trino.jdbc.TrinoDriver; -import io.trino.spi.TrinoException; -import io.trino.spi.connector.ConnectorContext; -import java.io.File; -import java.nio.file.Files; -import java.nio.file.Path; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.Properties; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * This class dynamically register the Catalog managed by Gravitino into Trino using Trino CREATE - * CATALOG statement. It allows the catalog to be used in Trino like a regular Trino catalog. - */ -public class CatalogRegister { - - private static final Logger LOG = LoggerFactory.getLogger(CatalogRegister.class); - - private static final int MIN_TRINO_SPI_VERSION = 435; - private static final int EXECUTE_QUERY_MAX_RETRIES = 6; - private static final int EXECUTE_QUERY_BACKOFF_TIME_SECOND = 5; - - private String trinoVersion; - private Connection connection; - private boolean isCoordinator; - private boolean isStarted = false; - private String catalogStoreDirectory; - private GravitinoConfig config; - - private void checkTrinoSpiVersion(ConnectorContext context) { - this.trinoVersion = context.getSpiVersion(); - - int version = Integer.parseInt(context.getSpiVersion()); - if (version < MIN_TRINO_SPI_VERSION) { - String errmsg = - String.format( - "Unsupported Trino-%s version. min support version is Trino-%d", - trinoVersion, MIN_TRINO_SPI_VERSION); - throw new TrinoException(GRAVITINO_UNSUPPORTED_TRINO_VERSION, errmsg); - } - - isCoordinator = context.getNodeManager().getCurrentNode().isCoordinator(); - } - - boolean isCoordinator() { - return isCoordinator; - } - - boolean isTrinoStarted() { - if (isStarted) { - return true; - } - - String command = "SELECT 1"; - try (Statement statement = connection.createStatement()) { - isStarted = statement.execute(command); - return isStarted; - } catch (Exception e) { - LOG.warn("Trino server is not started: {}", e.getMessage()); - return false; - } - } - - public void init(ConnectorContext context, GravitinoConfig config) throws Exception { - this.config = config; - checkTrinoSpiVersion(context); - - TrinoDriver driver = new TrinoDriver(); - DriverManager.registerDriver(driver); - - Properties properties = new Properties(); - properties.put("user", config.getTrinoUser()); - properties.put("password", config.getTrinoPassword()); - try { - connection = driver.connect(config.getTrinoURI(), properties); - } catch (SQLException e) { - throw new TrinoException( - GRAVITINO_RUNTIME_ERROR, "Failed to initialize the Trino connection.", e); - } - - catalogStoreDirectory = config.getCatalogStoreDirectory(); - if (!Files.exists(Path.of(catalogStoreDirectory))) { - throw new TrinoException( - GRAVITINO_MISSING_CONFIG, - String.format( - "Error config for Trino catalog store directory %s, file not found", - catalogStoreDirectory)); - } - } - - private String generateCreateCatalogCommand(String name, GravitinoCatalog gravitinoCatalog) - throws Exception { - return String.format( - "CREATE CATALOG %s USING gravitino WITH ( \"%s\" = 'true', \"%s\" = '%s', %s)", - name, - GRAVITINO_DYNAMIC_CONNECTOR, - GRAVITINO_DYNAMIC_CONNECTOR_CATALOG_CONFIG, - GravitinoCatalog.toJson(gravitinoCatalog), - config.toCatalogConfig()); - } - - private String generateDropCatalogCommand(String name) throws Exception { - return String.format("DROP CATALOG %s", name); - } - - public void registerCatalog(String name, GravitinoCatalog catalog) { - try { - String catalogFileName = String.format("%s/%s.properties", catalogStoreDirectory, name); - File catalogFile = new File(catalogFileName); - if (catalogFile.exists()) { - String catalogContents = Files.readString(catalogFile.toPath()); - if (!catalogContents.contains(GRAVITINO_DYNAMIC_CONNECTOR + "=true")) { - throw new TrinoException( - GRAVITINO_DUPLICATED_CATALOGS, - "Catalog already exists, the catalog is not created by gravitino"); - } else { - throw new TrinoException( - GRAVITINO_CATALOG_ALREADY_EXISTS, - String.format( - "Catalog %s in metalake %s already exists", - catalog.getName(), catalog.getMetalake())); - } - } - - if (checkCatalogExist(name)) { - throw new TrinoException( - GRAVITINO_DUPLICATED_CATALOGS, "Catalog already exists with unknown reason"); - } - String createCatalogCommand = generateCreateCatalogCommand(name, catalog); - executeSql(createCatalogCommand); - LOG.info("Register catalog {} successfully: {}", name, createCatalogCommand); - } catch (Exception e) { - String message = String.format("Failed to register catalog %s", name); - LOG.error(message); - throw new TrinoException(GRAVITINO_RUNTIME_ERROR, message, e); - } - } - - private boolean checkCatalogExist(String name) { - String showCatalogCommand = String.format("SHOW CATALOGS like '%s'", name); - Exception failedException = null; - try { - int retries = EXECUTE_QUERY_MAX_RETRIES; - while (retries-- > 0) { - try (Statement statement = connection.createStatement()) { - // check the catalog is already created - statement.execute(showCatalogCommand); - ResultSet rs = statement.getResultSet(); - while (rs.next()) { - String catalogName = rs.getString(1); - if (catalogName.equals(name) || catalogName.equals("\"" + name + "\"")) { - return true; - } - } - return false; - } catch (Exception e) { - failedException = e; - LOG.warn("Execute command failed: {}, ", showCatalogCommand, e); - Thread.sleep(EXECUTE_QUERY_BACKOFF_TIME_SECOND * 1000); - } - } - throw failedException; - } catch (Exception e) { - throw new TrinoException(GRAVITINO_RUNTIME_ERROR, "Failed to check catalog exist", e); - } - } - - private void executeSql(String sql) { - try { - int retries = EXECUTE_QUERY_MAX_RETRIES; - Exception failedException = null; - while (retries-- > 0) { - try (Statement statement = connection.createStatement()) { - // check the catalog is already created - statement.execute(sql); - return; - } catch (Exception e) { - failedException = e; - LOG.warn("Execute command failed: {}, ", sql, e); - Thread.sleep(EXECUTE_QUERY_BACKOFF_TIME_SECOND * 1000); - } - } - throw failedException; - } catch (Exception e) { - throw new TrinoException(GRAVITINO_RUNTIME_ERROR, "Failed to execute query: " + sql, e); - } - } - - public void unregisterCatalog(String name) { - try { - if (!checkCatalogExist(name)) { - LOG.warn("Catalog {} does not exist", name); - return; - } - String dropCatalogCommand = generateDropCatalogCommand(name); - executeSql(dropCatalogCommand); - LOG.info("Unregister catalog {} successfully: {}", name, dropCatalogCommand); - } catch (Exception e) { - String message = String.format("Failed to unregister catalog %s", name); - LOG.error(message); - throw new TrinoException(GRAVITINO_RUNTIME_ERROR, message, e); - } - } - - public void close() { - try { - if (connection != null) { - connection.close(); - } - } catch (SQLException e) { - LOG.error("Failed to close connection", e); - } - } -} diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/hive/HiveConnectorAdapter.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/hive/HiveConnectorAdapter.java index 91a56ddc120..9d42ce80342 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/hive/HiveConnectorAdapter.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/hive/HiveConnectorAdapter.java @@ -4,8 +4,6 @@ */ package com.datastrato.gravitino.trino.connector.catalog.hive; -import static com.datastrato.gravitino.trino.connector.GravitinoConnectorPluginManager.CONNECTOR_HIVE; - import com.datastrato.gravitino.catalog.property.PropertyConverter; import com.datastrato.gravitino.trino.connector.catalog.CatalogConnectorAdapter; import com.datastrato.gravitino.trino.connector.catalog.CatalogConnectorMetadataAdapter; @@ -16,10 +14,13 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; /** Transforming Hive connector configuration and components into Gravitino connector. */ public class HiveConnectorAdapter implements CatalogConnectorAdapter { + private static final AtomicInteger VERSION = new AtomicInteger(0); + private final HasPropertyMeta propertyMetadata; private final PropertyConverter catalogConverter; @@ -29,20 +30,23 @@ public HiveConnectorAdapter() { } @Override - public Map buildInternalConnectorConfig(GravitinoCatalog catalog) + public Map buildInternalConnectorConfig(GravitinoCatalog catalog) throws Exception { - Map config = new HashMap<>(); - config.put("hive.metastore.uri", catalog.getRequiredProperty("metastore.uris")); - config.put("hive.security", "allow-all"); + Map config = new HashMap<>(); + config.put( + "catalogHandle", + String.format("%s_v%d:normal:default", catalog.getName(), VERSION.getAndIncrement())); + config.put("connectorName", "hive"); + + Map properties = new HashMap<>(); + properties.put("hive.metastore.uri", catalog.getRequiredProperty("metastore.uris")); + properties.put("hive.security", "allow-all"); Map trinoProperty = catalogConverter.gravitinoToEngineProperties(catalog.getProperties()); - config.putAll(trinoProperty); - return config; - } + properties.putAll(trinoProperty); - @Override - public String internalConnectorName() { - return CONNECTOR_HIVE; + config.put("properties", properties); + return config; } @Override diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/IcebergConnectorAdapter.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/IcebergConnectorAdapter.java index d4af5e25c6d..1fe2ac02131 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/IcebergConnectorAdapter.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/IcebergConnectorAdapter.java @@ -4,7 +4,6 @@ */ package com.datastrato.gravitino.trino.connector.catalog.iceberg; -import static com.datastrato.gravitino.trino.connector.GravitinoConnectorPluginManager.CONNECTOR_ICEBERG; import static java.util.Collections.emptyList; import com.datastrato.gravitino.catalog.property.PropertyConverter; @@ -12,12 +11,15 @@ import com.datastrato.gravitino.trino.connector.catalog.CatalogConnectorMetadataAdapter; import com.datastrato.gravitino.trino.connector.metadata.GravitinoCatalog; import io.trino.spi.session.PropertyMetadata; +import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; /** Transforming Iceberg connector configuration and components into Gravitino connector. */ public class IcebergConnectorAdapter implements CatalogConnectorAdapter { + private static final AtomicInteger VERSION = new AtomicInteger(0); private final IcebergPropertyMeta propertyMetadata; private final PropertyConverter catalogConverter; @@ -27,14 +29,18 @@ public IcebergConnectorAdapter() { } @Override - public Map buildInternalConnectorConfig(GravitinoCatalog catalog) + public Map buildInternalConnectorConfig(GravitinoCatalog catalog) throws Exception { - return catalogConverter.gravitinoToEngineProperties(catalog.getProperties()); - } - - @Override - public String internalConnectorName() { - return CONNECTOR_ICEBERG; + Map config = new HashMap<>(); + config.put( + "catalogHandle", + String.format("%s_v%d:normal:default", catalog.getName(), VERSION.getAndIncrement())); + config.put("connectorName", "iceberg"); + + Map properties = + catalogConverter.gravitinoToEngineProperties(catalog.getProperties()); + config.put("properties", properties); + return config; } @Override diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/mysql/MySQLConnectorAdapter.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/mysql/MySQLConnectorAdapter.java index fc7d7c94695..7d10d7f695e 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/mysql/MySQLConnectorAdapter.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/mysql/MySQLConnectorAdapter.java @@ -4,7 +4,6 @@ */ package com.datastrato.gravitino.trino.connector.catalog.jdbc.mysql; -import static com.datastrato.gravitino.trino.connector.GravitinoConnectorPluginManager.CONNECTOR_MYSQL; import static java.util.Collections.emptyList; import com.datastrato.gravitino.catalog.property.PropertyConverter; @@ -14,13 +13,16 @@ import com.datastrato.gravitino.trino.connector.catalog.jdbc.JDBCCatalogPropertyConverter; import com.datastrato.gravitino.trino.connector.metadata.GravitinoCatalog; import io.trino.spi.session.PropertyMetadata; +import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; /** Transforming MySQL connector configuration and components into Gravitino connector. */ public class MySQLConnectorAdapter implements CatalogConnectorAdapter { private final PropertyConverter catalogConverter; + private static final AtomicInteger VERSION = new AtomicInteger(0); private final HasPropertyMeta propertyMetadata; public MySQLConnectorAdapter() { @@ -29,14 +31,18 @@ public MySQLConnectorAdapter() { } @Override - public Map buildInternalConnectorConfig(GravitinoCatalog catalog) + public Map buildInternalConnectorConfig(GravitinoCatalog catalog) throws Exception { - return catalogConverter.gravitinoToEngineProperties(catalog.getProperties()); - } - - @Override - public String internalConnectorName() { - return CONNECTOR_MYSQL; + Map config = new HashMap<>(); + config.put( + "catalogHandle", + String.format("%s_v%d:normal:default", catalog.getName(), VERSION.getAndIncrement())); + config.put("connectorName", "mysql"); + + Map properties = + catalogConverter.gravitinoToEngineProperties(catalog.getProperties()); + config.put("properties", properties); + return config; } @Override diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/postgresql/PostgreSQLConnectorAdapter.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/postgresql/PostgreSQLConnectorAdapter.java index 9dac7d9dac1..b21ab031f20 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/postgresql/PostgreSQLConnectorAdapter.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/postgresql/PostgreSQLConnectorAdapter.java @@ -4,7 +4,6 @@ */ package com.datastrato.gravitino.trino.connector.catalog.jdbc.postgresql; -import static com.datastrato.gravitino.trino.connector.GravitinoConnectorPluginManager.CONNECTOR_POSTGRESQL; import static java.util.Collections.emptyList; import com.datastrato.gravitino.catalog.property.PropertyConverter; @@ -12,25 +11,32 @@ import com.datastrato.gravitino.trino.connector.catalog.CatalogConnectorMetadataAdapter; import com.datastrato.gravitino.trino.connector.catalog.jdbc.JDBCCatalogPropertyConverter; import com.datastrato.gravitino.trino.connector.metadata.GravitinoCatalog; +import java.util.HashMap; import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; /** Transforming PostgreSQL connector configuration and components into Gravitino connector. */ public class PostgreSQLConnectorAdapter implements CatalogConnectorAdapter { private final PropertyConverter catalogConverter; + private static final AtomicInteger VERSION = new AtomicInteger(0); public PostgreSQLConnectorAdapter() { this.catalogConverter = new JDBCCatalogPropertyConverter(); } @Override - public Map buildInternalConnectorConfig(GravitinoCatalog catalog) + public Map buildInternalConnectorConfig(GravitinoCatalog catalog) throws Exception { - return catalogConverter.gravitinoToEngineProperties(catalog.getProperties()); - } + Map config = new HashMap<>(); + config.put( + "catalogHandle", + String.format("%s_v%d:normal:default", catalog.getName(), VERSION.getAndIncrement())); + config.put("connectorName", "postgresql"); - @Override - public String internalConnectorName() { - return CONNECTOR_POSTGRESQL; + Map properties = + catalogConverter.gravitinoToEngineProperties(catalog.getProperties()); + config.put("properties", properties); + return config; } @Override diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/memory/MemoryConnectorAdapter.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/memory/MemoryConnectorAdapter.java index 043e1e3dd50..7b30df53d58 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/memory/MemoryConnectorAdapter.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/memory/MemoryConnectorAdapter.java @@ -4,16 +4,16 @@ */ package com.datastrato.gravitino.trino.connector.catalog.memory; -import static com.datastrato.gravitino.trino.connector.GravitinoConnectorPluginManager.CONNECTOR_MEMORY; - import com.datastrato.gravitino.trino.connector.catalog.CatalogConnectorAdapter; import com.datastrato.gravitino.trino.connector.catalog.CatalogConnectorMetadataAdapter; import com.datastrato.gravitino.trino.connector.catalog.HasPropertyMeta; import com.datastrato.gravitino.trino.connector.metadata.GravitinoCatalog; import io.trino.spi.session.PropertyMetadata; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; /** * Support trino Memory connector for testing. Transforming Memory connector configuration and @@ -21,6 +21,7 @@ */ public class MemoryConnectorAdapter implements CatalogConnectorAdapter { + private static final AtomicInteger VERSION = new AtomicInteger(0); private final HasPropertyMeta propertyMetadata; public MemoryConnectorAdapter() { @@ -28,13 +29,16 @@ public MemoryConnectorAdapter() { } @Override - public Map buildInternalConnectorConfig(GravitinoCatalog catalog) { - return Collections.emptyMap(); - } - - @Override - public String internalConnectorName() { - return CONNECTOR_MEMORY; + public Map buildInternalConnectorConfig(GravitinoCatalog catalog) { + Map config = new HashMap<>(); + config.put( + "catalogHandle", + String.format("%s_v%d:normal:default", catalog.getName(), VERSION.getAndIncrement())); + config.put("connectorName", "memory"); + + Map properties = new HashMap<>(); + config.put("properties", properties); + return config; } @Override diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/metadata/GravitinoCatalog.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/metadata/GravitinoCatalog.java index c489f62196b..291610b25b0 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/metadata/GravitinoCatalog.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/metadata/GravitinoCatalog.java @@ -8,10 +8,6 @@ import com.datastrato.gravitino.Catalog; import com.datastrato.gravitino.NameIdentifier; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; import io.trino.spi.TrinoException; import java.time.Instant; import java.util.Map; @@ -20,86 +16,48 @@ /** Help Gravitino connector access CatalogMetadata from gravitino client. */ public class GravitinoCatalog { - private static ObjectMapper objectMapper = new ObjectMapper(); - private final String metalake; - private final String provider; - private final String name; - private final Map properties; - private final long lastModifiedTime; + private final Catalog catalog; public GravitinoCatalog(String metalake, Catalog catalog) { this.metalake = metalake; - this.provider = catalog.provider(); - this.name = catalog.name(); - this.properties = catalog.properties(); - Instant time = - catalog.auditInfo().lastModifiedTime() == null - ? catalog.auditInfo().createTime() - : catalog.auditInfo().lastModifiedTime(); - lastModifiedTime = time.toEpochMilli(); - } - - @JsonCreator - public GravitinoCatalog( - @JsonProperty("metalake") String metalake, - @JsonProperty("provider") String provider, - @JsonProperty("name") String name, - @JsonProperty("properties") Map properties, - @JsonProperty("lastModifiedTime") long lastModifiedTime) { - this.metalake = metalake; - this.provider = provider; - this.name = name; - this.properties = properties; - this.lastModifiedTime = lastModifiedTime; + this.catalog = catalog; } - @JsonProperty public String getProvider() { - return provider; + return catalog.provider(); } - @JsonProperty public String getName() { - return name; + return catalog.name(); } - @JsonProperty public String getMetalake() { return metalake; } public NameIdentifier geNameIdentifier() { - return NameIdentifier.ofCatalog(metalake, name); + return NameIdentifier.ofCatalog(metalake, catalog.name()); } public String getProperty(String name, String defaultValue) { - return properties.getOrDefault(name, defaultValue); + return catalog.properties().getOrDefault(name, defaultValue); } public String getRequiredProperty(String name) throws Exception { - String value = properties.getOrDefault(name, ""); + String value = catalog.properties().getOrDefault(name, ""); if (StringUtils.isBlank(value)) { throw new TrinoException(GRAVITINO_MISSING_CONFIG, "Missing required config: " + name); } return value; } - @JsonProperty public Map getProperties() { - return properties; - } - - @JsonProperty - public long getLastModifiedTime() { - return lastModifiedTime; - } - - public static String toJson(GravitinoCatalog catalog) throws JsonProcessingException { - return objectMapper.writeValueAsString(catalog); + return catalog.properties(); } - public static GravitinoCatalog fromJson(String jsonString) throws JsonProcessingException { - return objectMapper.readValue(jsonString, GravitinoCatalog.class); + public Instant getLastModifiedTime() { + Instant time = catalog.auditInfo().lastModifiedTime(); + return time == null ? catalog.auditInfo().createTime() : time; } } diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/system/GravitinoSystemConnector.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/system/GravitinoSystemConnector.java index 4c8cb29167f..30d7c801142 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/system/GravitinoSystemConnector.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/system/GravitinoSystemConnector.java @@ -65,7 +65,7 @@ public ConnectorMetadata getMetadata( @Override public ConnectorSplitManager getSplitManager() { - return new SplitManager(); + return new SplitManger(); } @Override @@ -94,7 +94,7 @@ public ConnectorPageSource createPageSource( } } - public static class SplitManager implements ConnectorSplitManager { + public static class SplitManger implements ConnectorSplitManager { @Override public ConnectorSplitSource getSplits( diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/system/storedprocdure/AlterCatalogStoredProcedure.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/system/storedprocdure/AlterCatalogStoredProcedure.java index 35e2f8f6822..9565d40c6f9 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/system/storedprocdure/AlterCatalogStoredProcedure.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/system/storedprocdure/AlterCatalogStoredProcedure.java @@ -50,7 +50,7 @@ public Procedure createStoredProcedure() throws NoSuchMethodException, IllegalAc new ArrayType(VARCHAR), false, ArrayBlock.fromElementBlock( - 0, Optional.empty(), new int[1], VARCHAR.createBlockBuilder(null, 1).build()))); + 0, Optional.empty(), new int[1], VARCHAR.createBlockBuilder(null, 1)))); return new Procedure(SYSTEM_TABLE_SCHEMA_NAME, "alter_catalog", arguments, dropCatalog); } diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/util/json/AbstractTypedJacksonModule.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/util/json/AbstractTypedJacksonModule.java deleted file mode 100644 index 569d897201e..00000000000 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/util/json/AbstractTypedJacksonModule.java +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datastrato.gravitino.trino.connector.util.json; - -import static com.fasterxml.jackson.annotation.JsonTypeInfo.As.PROPERTY; -import static com.google.common.base.Preconditions.checkArgument; -import static java.util.Objects.requireNonNull; - -import com.fasterxml.jackson.annotation.JsonTypeInfo; -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.core.Version; -import com.fasterxml.jackson.databind.DatabindContext; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.JavaType; -import com.fasterxml.jackson.databind.JsonMappingException; -import com.fasterxml.jackson.databind.JsonSerializer; -import com.fasterxml.jackson.databind.SerializerProvider; -import com.fasterxml.jackson.databind.deser.std.StdDeserializer; -import com.fasterxml.jackson.databind.jsontype.TypeDeserializer; -import com.fasterxml.jackson.databind.jsontype.TypeIdResolver; -import com.fasterxml.jackson.databind.jsontype.TypeSerializer; -import com.fasterxml.jackson.databind.jsontype.impl.AsPropertyTypeDeserializer; -import com.fasterxml.jackson.databind.jsontype.impl.AsPropertyTypeSerializer; -import com.fasterxml.jackson.databind.jsontype.impl.TypeIdResolverBase; -import com.fasterxml.jackson.databind.module.SimpleModule; -import com.fasterxml.jackson.databind.ser.BeanSerializerFactory; -import com.fasterxml.jackson.databind.ser.std.StdSerializer; -import com.fasterxml.jackson.databind.type.TypeFactory; -import java.io.IOException; -import java.util.function.Function; - -/** - * This class is reference to Trino source code io.trino.metadata.AbstractTypedJacksonModule, It - * used to preform XXXHandle serialization - */ -public abstract class AbstractTypedJacksonModule extends SimpleModule { - private static final String TYPE_PROPERTY = "@gravitino-connector-type"; - - protected AbstractTypedJacksonModule( - Class baseClass, - Function nameResolver, - Function> classResolver) { - super(baseClass.getSimpleName() + "Module", Version.unknownVersion()); - - TypeIdResolver typeResolver = new InternalTypeResolver(nameResolver, classResolver); - - addSerializer(baseClass, new InternalTypeSerializer<>(baseClass, typeResolver)); - addDeserializer(baseClass, new InternalTypeDeserializer<>(baseClass, typeResolver)); - } - - private static class InternalTypeDeserializer extends StdDeserializer { - private final TypeDeserializer typeDeserializer; - - public InternalTypeDeserializer(Class baseClass, TypeIdResolver typeIdResolver) { - super(baseClass); - this.typeDeserializer = - new AsPropertyTypeDeserializer( - TypeFactory.defaultInstance().constructType(baseClass), - typeIdResolver, - TYPE_PROPERTY, - false, - null, - PROPERTY, - true); - } - - @SuppressWarnings("unchecked") - @Override - public T deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) - throws IOException { - return (T) typeDeserializer.deserializeTypedFromAny(jsonParser, deserializationContext); - } - } - - private static class InternalTypeSerializer extends StdSerializer { - private final TypeSerializer typeSerializer; - - public InternalTypeSerializer(Class baseClass, TypeIdResolver typeIdResolver) { - super(baseClass); - this.typeSerializer = new AsPropertyTypeSerializer(typeIdResolver, null, TYPE_PROPERTY); - } - - @Override - public void serialize(T value, JsonGenerator generator, SerializerProvider provider) - throws IOException { - if (value == null) { - provider.defaultSerializeNull(generator); - return; - } - - try { - Class type = value.getClass(); - JsonSerializer serializer = createSerializer(provider, type); - serializer.serializeWithType(value, generator, provider, typeSerializer); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - @SuppressWarnings("unchecked") - private static JsonSerializer createSerializer( - SerializerProvider provider, Class type) throws JsonMappingException { - JavaType javaType = provider.constructType(type); - return (JsonSerializer) - BeanSerializerFactory.instance.createSerializer(provider, javaType); - } - } - - private static class InternalTypeResolver extends TypeIdResolverBase { - private final Function nameResolver; - private final Function> classResolver; - - public InternalTypeResolver( - Function nameResolver, Function> classResolver) { - this.nameResolver = requireNonNull(nameResolver, "nameResolver is null"); - this.classResolver = requireNonNull(classResolver, "classResolver is null"); - } - - @Override - public String idFromValue(Object value) { - return idFromValueAndType(value, value.getClass()); - } - - @Override - public String idFromValueAndType(Object value, Class suggestedType) { - requireNonNull(value, "value is null"); - String type = nameResolver.apply(value); - checkArgument(type != null, "Unknown class: %s", value.getClass().getName()); - return type; - } - - @Override - public JavaType typeFromId(DatabindContext context, String id) { - requireNonNull(id, "id is null"); - Class typeClass = classResolver.apply(id); - checkArgument(typeClass != null, "Unknown type ID: %s", id); - return context.getTypeFactory().constructType(typeClass); - } - - @Override - public JsonTypeInfo.Id getMechanism() { - return JsonTypeInfo.Id.NAME; - } - } -} diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/util/json/BlockJsonSerde.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/util/json/BlockJsonSerde.java deleted file mode 100644 index b8bc95a7a36..00000000000 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/util/json/BlockJsonSerde.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datastrato.gravitino.trino.connector.util.json; - -import static java.lang.Math.toIntExact; - -import com.fasterxml.jackson.core.Base64Variants; -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.JsonDeserializer; -import com.fasterxml.jackson.databind.JsonSerializer; -import com.fasterxml.jackson.databind.SerializerProvider; -import io.airlift.slice.DynamicSliceOutput; -import io.airlift.slice.Slice; -import io.airlift.slice.SliceOutput; -import io.airlift.slice.Slices; -import io.trino.spi.block.Block; -import io.trino.spi.block.BlockEncodingSerde; -import java.io.IOException; -import java.lang.reflect.Method; - -/** - * This class is reference to Trino source code io.trino.block.BlockJsonSerde, use refactoring to - * call the key method to avoid class loader isolation - */ -public final class BlockJsonSerde { - private static final String BLOCK_SERDE_UTIL_CLASS_NAME = "io.trino.block.BlockSerdeUtil"; - - public static class Serializer extends JsonSerializer { - private final BlockEncodingSerde blockEncodingSerde; - private final Method writeBlock; - - public Serializer(BlockEncodingSerde blockEncodingSerde) throws Exception { - this.blockEncodingSerde = blockEncodingSerde; - Class clazz = - blockEncodingSerde.getClass().getClassLoader().loadClass(BLOCK_SERDE_UTIL_CLASS_NAME); - this.writeBlock = - clazz.getDeclaredMethod( - "writeBlock", BlockEncodingSerde.class, SliceOutput.class, Block.class); - } - - @Override - public void serialize( - Block block, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) - throws IOException { - // Encoding name is length prefixed as are many block encodings - SliceOutput output = - new DynamicSliceOutput( - toIntExact( - block.getSizeInBytes() + block.getEncodingName().length() + (2 * Integer.BYTES))); - - try { - writeBlock.invoke(null, blockEncodingSerde, output, block); - } catch (Exception e) { - throw new RuntimeException(e); - } - - Slice slice = output.slice(); - jsonGenerator.writeBinary( - Base64Variants.MIME_NO_LINEFEEDS, - slice.byteArray(), - slice.byteArrayOffset(), - slice.length()); - } - } - - public static class Deserializer extends JsonDeserializer { - private final BlockEncodingSerde blockEncodingSerde; - private final Method readBlock; - - public Deserializer(BlockEncodingSerde blockEncodingSerde) throws Exception { - this.blockEncodingSerde = blockEncodingSerde; - Class clazz = - blockEncodingSerde.getClass().getClassLoader().loadClass(BLOCK_SERDE_UTIL_CLASS_NAME); - readBlock = clazz.getDeclaredMethod("readBlock", BlockEncodingSerde.class, Slice.class); - } - - @Override - public Block deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) - throws IOException { - byte[] decoded = jsonParser.getBinaryValue(Base64Variants.MIME_NO_LINEFEEDS); - try { - return (Block) readBlock.invoke(null, blockEncodingSerde, Slices.wrappedBuffer(decoded)); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - } -} diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/util/json/JsonCodec.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/util/json/JsonCodec.java deleted file mode 100644 index a486abcadb9..00000000000 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/util/json/JsonCodec.java +++ /dev/null @@ -1,197 +0,0 @@ -/* - * Copyright 2024 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. - */ -package com.datastrato.gravitino.trino.connector.util.json; - -import static com.datastrato.gravitino.trino.connector.GravitinoErrorCode.GRAVITINO_RUNTIME_ERROR; - -import com.datastrato.gravitino.trino.connector.GravitinoConnectorPluginManager; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.MapperFeature; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; -import com.fasterxml.jackson.databind.module.SimpleModule; -import com.fasterxml.jackson.datatype.guava.GuavaModule; -import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; -import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; -import com.fasterxml.jackson.module.paramnames.ParameterNamesModule; -import io.airlift.json.RecordAutoDetectModule; -import io.trino.spi.TrinoException; -import io.trino.spi.block.Block; -import io.trino.spi.block.BlockEncodingSerde; -import io.trino.spi.connector.ColumnHandle; -import io.trino.spi.connector.ConnectorInsertTableHandle; -import io.trino.spi.connector.ConnectorSplit; -import io.trino.spi.connector.ConnectorTableHandle; -import io.trino.spi.connector.ConnectorTransactionHandle; -import io.trino.spi.type.Type; -import io.trino.spi.type.TypeManager; -import io.trino.spi.type.TypeSignature; -import java.lang.reflect.Field; -import java.util.function.Function; - -public class JsonCodec { - private static ObjectMapper mapper; - - private static ObjectMapper buildMapper(ClassLoader classLoader) { - try { - ClassLoader appClassLoader = - GravitinoConnectorPluginManager.instance(classLoader).getAppClassloader(); - TypeManager typeManager = createTypeManager(appClassLoader); - return createMapper(typeManager); - } catch (Exception e) { - throw new TrinoException(GRAVITINO_RUNTIME_ERROR, "Failed to build ObjectMapper", e); - } - } - - static TypeManager createTypeManager(ClassLoader classLoader) { - try { - Class internalTypeManagerClass = classLoader.loadClass("io.trino.type.InternalTypeManager"); - Class typeRegistryClass = classLoader.loadClass("io.trino.metadata.TypeRegistry"); - Class typeOperatorsClass = classLoader.loadClass("io.trino.spi.type.TypeOperators"); - Class featuresConfigClass = classLoader.loadClass("io.trino.FeaturesConfig"); - - Object typeRegistry = - typeRegistryClass - .getConstructor(typeOperatorsClass, featuresConfigClass) - .newInstance( - typeOperatorsClass.getConstructor().newInstance(), - featuresConfigClass.getConstructor().newInstance()); - return (TypeManager) - internalTypeManagerClass.getConstructor(typeRegistryClass).newInstance(typeRegistry); - } catch (Exception e) { - throw new RuntimeException("Failed to create TypeManager :" + e.getMessage(), e); - } - } - - static BlockEncodingSerde createBlockEncodingSerde(TypeManager typeManager) throws Exception { - ClassLoader classLoader = typeManager.getClass().getClassLoader(); - Class blockEncodingManagerClass = - classLoader.loadClass("io.trino.metadata.BlockEncodingManager"); - Class internalBlockEncodingSerdeClass = - classLoader.loadClass("io.trino.metadata.InternalBlockEncodingSerde"); - return (BlockEncodingSerde) - internalBlockEncodingSerdeClass - .getConstructor(blockEncodingManagerClass, TypeManager.class) - .newInstance(blockEncodingManagerClass.getConstructor().newInstance(), typeManager); - } - - static void registerHandleSerializationModule(ObjectMapper objectMapper) { - - GravitinoConnectorPluginManager pluginManager = GravitinoConnectorPluginManager.instance(); - Function nameResolver = - obj -> { - try { - ClassLoader pluginClassloader = obj.getClass().getClassLoader(); - if (pluginClassloader - == GravitinoConnectorPluginManager.instance().getAppClassloader()) { - return "app:" + obj.getClass().getName(); - } - - Field idField = pluginClassloader.getClass().getDeclaredField("pluginName"); - idField.setAccessible(true); - String classLoaderName = (String) idField.get(pluginClassloader); - return classLoaderName + ":" + obj.getClass().getName(); - } catch (Exception e) { - throw new RuntimeException("Gravitino connector serialize error " + e.getMessage(), e); - } - }; - - Function> classResolver = - name -> { - try { - String[] nameParts = name.split(":"); - String classLoaderName = nameParts[0]; - String className = nameParts[1]; - ClassLoader loader = pluginManager.getClassLoader(classLoaderName); - return loader.loadClass(className); - } catch (ClassNotFoundException e) { - throw new RuntimeException( - "Gravitino connector deserialize error " + e.getMessage(), e); - } - }; - - objectMapper.registerModule( - new AbstractTypedJacksonModule<>( - ConnectorTransactionHandle.class, nameResolver, classResolver) {}); - - objectMapper.registerModule( - new AbstractTypedJacksonModule<>( - ConnectorTableHandle.class, nameResolver, classResolver) {}); - - objectMapper.registerModule( - new AbstractTypedJacksonModule<>(ColumnHandle.class, nameResolver, classResolver) {}); - - objectMapper.registerModule( - new AbstractTypedJacksonModule<>(ConnectorSplit.class, nameResolver, classResolver) {}); - - objectMapper.registerModule( - new AbstractTypedJacksonModule<>( - ConnectorInsertTableHandle.class, nameResolver, classResolver) {}); - } - - @SuppressWarnings("deprecation") - private static ObjectMapper createMapper(TypeManager typeManager) { - try { - ObjectMapper objectMapper = new ObjectMapper(); - - objectMapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES); - objectMapper.disable(DeserializationFeature.ACCEPT_FLOAT_AS_INT); - objectMapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS); - objectMapper.setDefaultPropertyInclusion( - JsonInclude.Value.construct(JsonInclude.Include.NON_ABSENT, JsonInclude.Include.ALWAYS)); - - objectMapper.disable(MapperFeature.AUTO_DETECT_CREATORS); - objectMapper.disable(MapperFeature.AUTO_DETECT_FIELDS); - objectMapper.disable(MapperFeature.AUTO_DETECT_SETTERS); - objectMapper.disable(MapperFeature.AUTO_DETECT_GETTERS); - objectMapper.disable(MapperFeature.AUTO_DETECT_IS_GETTERS); - objectMapper.disable(MapperFeature.USE_GETTERS_AS_SETTERS); - objectMapper.disable(MapperFeature.CAN_OVERRIDE_ACCESS_MODIFIERS); - objectMapper.disable(MapperFeature.INFER_PROPERTY_MUTATORS); - objectMapper.disable(MapperFeature.ALLOW_FINAL_FIELDS_AS_MUTATORS); - - SimpleModule module = new SimpleModule(); - objectMapper.registerModule(new Jdk8Module()); - objectMapper.registerModule(new JavaTimeModule()); - objectMapper.registerModule(new GuavaModule()); - objectMapper.registerModule(new ParameterNamesModule()); - objectMapper.registerModule(new RecordAutoDetectModule()); - - // Handle serialization for plugin classes - registerHandleSerializationModule(objectMapper); - - // Type serialization for plugin classes - module.addDeserializer(Type.class, new TypeDeserializer(typeManager)); - module.addDeserializer( - TypeSignature.class, - new TypeSignatureDeserializer(typeManager.getClass().getClassLoader())); - - // Block serialization for plugin classes - BlockEncodingSerde blockEncodingSerde = createBlockEncodingSerde(typeManager); - module.addSerializer(Block.class, new BlockJsonSerde.Serializer(blockEncodingSerde)); - module.addDeserializer(Block.class, new BlockJsonSerde.Deserializer(blockEncodingSerde)); - - objectMapper.registerModule(module); - return objectMapper; - } catch (Exception e) { - throw new RuntimeException("Failed to create JsonMapper:" + e.getMessage(), e); - } - } - - public static ObjectMapper getMapper(ClassLoader appClassLoader) { - if (mapper != null) { - return mapper; - } - - synchronized (JsonCodec.class) { - if (mapper != null) { - return mapper; - } - mapper = buildMapper(appClassLoader); - return mapper; - } - } -} diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/util/json/TypeDeserializer.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/util/json/TypeDeserializer.java deleted file mode 100644 index 90c12101977..00000000000 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/util/json/TypeDeserializer.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datastrato.gravitino.trino.connector.util.json; - -import static java.util.Objects.requireNonNull; - -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.deser.std.FromStringDeserializer; -import io.trino.spi.type.Type; -import io.trino.spi.type.TypeId; -import io.trino.spi.type.TypeManager; -import java.util.function.Function; - -/** - * This class is reference to Trino source code io.trino.plugin.base.TypeDeserializer, It use to - * handle Type serialization - */ -public final class TypeDeserializer extends FromStringDeserializer { - private final Function typeLoader; - - public TypeDeserializer(TypeManager typeManager) { - this(typeManager::getType); - } - - public TypeDeserializer(Function typeLoader) { - super(Type.class); - this.typeLoader = requireNonNull(typeLoader, "typeLoader is null"); - } - - @Override - protected Type _deserialize(String value, DeserializationContext context) { - return typeLoader.apply(TypeId.of(value)); - } -} diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/util/json/TypeSignatureDeserializer.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/util/json/TypeSignatureDeserializer.java deleted file mode 100644 index d7973479f2e..00000000000 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/util/json/TypeSignatureDeserializer.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datastrato.gravitino.trino.connector.util.json; - -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.deser.std.FromStringDeserializer; -import com.google.common.collect.ImmutableSet; -import io.trino.spi.type.TypeSignature; -import java.lang.reflect.Method; -import java.util.Set; - -/** - * This class is reference to Trino source code io.trino.type.TypeSignatureDeserializer, use - * refactoring to call the key method to handle Type serialization - */ -public final class TypeSignatureDeserializer extends FromStringDeserializer { - private final Method parseTypeSignatureMethod; - - public TypeSignatureDeserializer(ClassLoader classLoader) { - super(TypeSignature.class); - try { - Class clazz = classLoader.loadClass("io.trino.sql.analyzer.TypeSignatureTranslator"); - parseTypeSignatureMethod = - clazz.getDeclaredMethod("parseTypeSignature", String.class, Set.class); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - @Override - protected TypeSignature _deserialize(String value, DeserializationContext context) { - try { - return (TypeSignature) - parseTypeSignatureMethod.invoke(null, "varchar(255)", ImmutableSet.of()); - } catch (Exception e) { - throw new RuntimeException(e); - } - } -} diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/GravitinoMockServer.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/GravitinoMockServer.java index 79d66bfe79e..7f98b532b35 100644 --- a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/GravitinoMockServer.java +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/GravitinoMockServer.java @@ -42,7 +42,6 @@ import io.trino.spi.connector.ConnectorMetadata; import io.trino.spi.connector.ConnectorTableHandle; import io.trino.spi.connector.ConnectorTableMetadata; -import io.trino.spi.connector.SaveMode; import io.trino.spi.connector.SchemaTableName; import io.trino.testing.ResourcePresence; import java.time.Instant; @@ -193,6 +192,14 @@ public Catalog answer(InvocationOnMock invocation) throws Throwable { return metaLake; } + void reloadCatalogs() { + GravitinoMetalake metaLake = mock(GravitinoMetalake.class); + when(metaLake.name()).thenReturn(testMetalake); + when(metaLake.listCatalogs()) + .thenReturn(new NameIdentifier[] {NameIdentifier.ofCatalog(testMetalake, testCatalog)}); + catalogConnectorManager.loadCatalogs(metaLake); + } + private Catalog createCatalog(String metalakeName, String catalogName) { Catalog catalog = mock(Catalog.class); when(catalog.name()).thenReturn(catalogName); @@ -362,7 +369,7 @@ public Table answer(InvocationOnMock invocation) throws Throwable { catalogConnectorManager.getTrinoCatalogName(catalog)) .getInternalConnector(); ConnectorMetadata metadata = memoryConnector.getMetadata(null, null); - metadata.createTable(null, tableMetadata, SaveMode.FAIL); + metadata.createTable(null, tableMetadata, false); return null; } }); diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestCreateGravitinoConnector.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestCreateGravitinoConnector.java index 6f4267f8b30..bd71b021bac 100644 --- a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestCreateGravitinoConnector.java +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestCreateGravitinoConnector.java @@ -12,10 +12,8 @@ import io.trino.testing.DistributedQueryRunner; import io.trino.testing.QueryRunner; import java.util.HashMap; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; +import org.testng.annotations.Test; -@Disabled public class TestCreateGravitinoConnector { GravitinoMockServer server; diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoConfig.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoConfig.java index 1a79f918791..c1f507d1c4e 100644 --- a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoConfig.java +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoConfig.java @@ -5,20 +5,20 @@ package com.datastrato.gravitino.trino.connector; import static com.datastrato.gravitino.trino.connector.GravitinoErrorCode.GRAVITINO_MISSING_CONFIG; -import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.testng.Assert.assertEquals; import com.google.common.collect.ImmutableMap; import io.trino.spi.TrinoException; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.jupiter.api.Test; +import org.testng.annotations.AfterTest; +import org.testng.annotations.BeforeTest; +import org.testng.annotations.Test; public class TestGravitinoConfig { - @BeforeClass + @BeforeTest public static void startup() throws Exception {} - @AfterClass + @AfterTest public static void shutdown() throws Exception {} @Test diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoConnector.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoConnector.java index db415a1ede2..4e6a364aeb4 100644 --- a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoConnector.java +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoConnector.java @@ -7,7 +7,8 @@ import static io.trino.testing.TestingSession.testSessionBuilder; import static java.lang.String.format; import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; import com.datastrato.gravitino.client.GravitinoAdminClient; import com.datastrato.gravitino.trino.connector.catalog.CatalogConnectorManager; @@ -20,13 +21,13 @@ import io.trino.testing.QueryRunner; import java.util.HashMap; import java.util.List; -import java.util.concurrent.TimeUnit; import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.awaitility.Awaitility; -import org.junit.jupiter.api.Test; import org.testcontainers.shaded.com.google.common.base.Preconditions; +import org.testng.annotations.Parameters; +import org.testng.annotations.Test; +@Parameters({"-Xmx4G"}) public class TestGravitinoConnector extends AbstractTestQueryFramework { GravitinoMockServer server; @@ -37,41 +38,31 @@ protected QueryRunner createQueryRunner() throws Exception { GravitinoAdminClient gravitinoClient = server.createGravitinoClient(); Session session = testSessionBuilder().setCatalog("gravitino").build(); + QueryRunner queryRunner = null; try { - - DistributedQueryRunner queryRunner = - DistributedQueryRunner.builder(session).setNodeCount(1).build(); + // queryRunner = LocalQueryRunner.builder(session).build(); + queryRunner = DistributedQueryRunner.builder(session).setNodeCount(1).build(); TestGravitinoPlugin gravitinoPlugin = new TestGravitinoPlugin(gravitinoClient); queryRunner.installPlugin(gravitinoPlugin); + queryRunner.installPlugin(new MemoryPlugin()); // create a gravitino connector named gravitino using metalake test HashMap properties = new HashMap<>(); properties.put("gravitino.metalake", "test"); properties.put("gravitino.uri", "http://127.0.0.1:8090"); - properties.put( - "trino.catalog.store", queryRunner.getCoordinator().getBaseDataDir().toString()); - properties.put( - "trino.jdbc.uri", - queryRunner.getCoordinator().getBaseUrl().toString().replace("http", "jdbc:trino")); queryRunner.createCatalog("gravitino", "gravitino", properties); - GravitinoConnectorPluginManager.instance(this.getClass().getClassLoader()) - .installPlugin("memory", new MemoryPlugin()); CatalogConnectorManager catalogConnectorManager = gravitinoPlugin.getCatalogConnectorManager(); server.setCatalogConnectorManager(catalogConnectorManager); - // Wait for the catalog to be created. Wait for at least 30 seconds. - Awaitility.await() - .atMost(30, TimeUnit.SECONDS) - .pollInterval(1, TimeUnit.SECONDS) - .until(() -> !catalogConnectorManager.getCatalogs().isEmpty()); - - return queryRunner; + // test the catalog has loaded + assertFalse(catalogConnectorManager.getCatalogs().isEmpty()); } catch (Exception e) { throw new RuntimeException("Create query runner failed", e); } + return queryRunner; } @Test diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoConnectorWithMetalakeCatalogName.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoConnectorWithMetalakeCatalogName.java index ea456e0a242..757a970722e 100644 --- a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoConnectorWithMetalakeCatalogName.java +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoConnectorWithMetalakeCatalogName.java @@ -7,6 +7,8 @@ import static io.trino.testing.TestingSession.testSessionBuilder; import static java.lang.String.format; import static org.assertj.core.api.Assertions.assertThat; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; import com.datastrato.gravitino.client.GravitinoAdminClient; import com.datastrato.gravitino.trino.connector.catalog.CatalogConnectorManager; @@ -20,13 +22,8 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; -import java.util.concurrent.TimeUnit; -import org.awaitility.Awaitility; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; +import org.testng.annotations.Test; -@Disabled public class TestGravitinoConnectorWithMetalakeCatalogName extends AbstractTestQueryFramework { GravitinoMockServer server; @@ -37,13 +34,13 @@ protected QueryRunner createQueryRunner() throws Exception { GravitinoAdminClient gravitinoClient = server.createGravitinoClient(); Session session = testSessionBuilder().setCatalog("gravitino").build(); - + QueryRunner queryRunner = null; try { - DistributedQueryRunner queryRunner = - DistributedQueryRunner.builder(session).setNodeCount(1).build(); + queryRunner = DistributedQueryRunner.builder(session).setNodeCount(1).build(); TestGravitinoPlugin gravitinoPlugin = new TestGravitinoPlugin(gravitinoClient); queryRunner.installPlugin(gravitinoPlugin); + queryRunner.installPlugin(new MemoryPlugin()); { // create a gravitino connector named gravitino using metalake test @@ -51,93 +48,74 @@ protected QueryRunner createQueryRunner() throws Exception { properties.put("gravitino.metalake", "test"); properties.put("gravitino.uri", "http://127.0.0.1:8090"); properties.put("gravitino.simplify-catalog-names", "false"); - properties.put( - "trino.catalog.store", queryRunner.getCoordinator().getBaseDataDir().toString()); - properties.put( - "trino.jdbc.uri", - queryRunner.getCoordinator().getBaseUrl().toString().replace("http", "jdbc:trino")); queryRunner.createCatalog("gravitino", "gravitino", properties); } { - // create a gravitino connector named test1 using metalake gravitino1 + // create a gravitino connector named test1 using metalake test1 HashMap properties = new HashMap<>(); properties.put("gravitino.metalake", "test1"); properties.put("gravitino.uri", "http://127.0.0.1:8090"); properties.put("gravitino.simplify-catalog-names", "false"); - properties.put( - "trino.catalog.store", queryRunner.getCoordinator().getBaseDataDir().toString()); - properties.put( - "trino.jdbc.uri", - queryRunner.getCoordinator().getBaseUrl().toString().replace("http", "jdbc:trino")); - queryRunner.createCatalog("gravitino1", "gravitino", properties); + queryRunner.createCatalog("test1", "gravitino", properties); } - GravitinoConnectorPluginManager.instance(this.getClass().getClassLoader()) - .installPlugin("memory", new MemoryPlugin()); CatalogConnectorManager catalogConnectorManager = gravitinoPlugin.getCatalogConnectorManager(); server.setCatalogConnectorManager(catalogConnectorManager); - // Wait for the catalog to be created. Wait for at least 30 seconds. - Awaitility.await() - .atMost(30, TimeUnit.SECONDS) - .pollInterval(1, TimeUnit.SECONDS) - .until(() -> !catalogConnectorManager.getCatalogs().isEmpty()); - return queryRunner; - + // test the catalog has loaded + assertFalse(catalogConnectorManager.getCatalogs().isEmpty()); } catch (Exception e) { throw new RuntimeException("Create query runner failed", e); } + return queryRunner; } @Test public void testSystemTable() throws Exception { MaterializedResult expectedResult = computeActual("select * from gravitino.system.catalog"); - Assertions.assertEquals(expectedResult.getRowCount(), 1); + assertEquals(expectedResult.getRowCount(), 1); List expectedRows = expectedResult.getMaterializedRows(); MaterializedRow row = expectedRows.get(0); - Assertions.assertEquals(row.getField(0), "memory"); - Assertions.assertEquals(row.getField(1), "memory"); - Assertions.assertEquals(row.getField(2), "{\"max_ttl\":\"10\"}"); + assertEquals(row.getField(0), "memory"); + assertEquals(row.getField(1), "memory"); + assertEquals(row.getField(2), "{\"max_ttl\":\"10\"}"); } @Test public void testCreateCatalog() throws Exception { // testing the catalogs assertThat(computeActual("show catalogs").getOnlyColumnAsSet()).contains("gravitino"); - assertThat(computeActual("show catalogs").getOnlyColumnAsSet()).contains("gravitino1"); - assertThat(computeActual("show catalogs").getOnlyColumnAsSet()).contains("\"test.memory\""); + assertThat(computeActual("show catalogs").getOnlyColumnAsSet()).contains("test1"); + assertThat(computeActual("show catalogs").getOnlyColumnAsSet()).contains("test.memory"); // testing the gravitino connector framework works. assertThat(computeActual("select * from system.jdbc.tables").getRowCount()).isGreaterThan(1); // test metalake named test. the connector name is gravitino assertUpdate("call gravitino.system.create_catalog('memory1', 'memory', Map())"); - assertThat(computeActual("show catalogs").getOnlyColumnAsSet()).contains("\"test.memory1\""); + assertThat(computeActual("show catalogs").getOnlyColumnAsSet()).contains("test.memory1"); assertUpdate("call gravitino.system.drop_catalog('memory1')"); - assertThat(computeActual("show catalogs").getOnlyColumnAsSet()) - .doesNotContain("\"test.memory1\""); + assertThat(computeActual("show catalogs").getOnlyColumnAsSet()).doesNotContain("test.memory1"); assertUpdate( "call gravitino.system.create_catalog(" + "catalog=>'memory1', provider=>'memory', properties => Map(array['max_ttl'], array['10']), ignore_exist => true)"); - assertThat(computeActual("show catalogs").getOnlyColumnAsSet()).contains("\"test.memory1\""); + assertThat(computeActual("show catalogs").getOnlyColumnAsSet()).contains("test.memory1"); assertUpdate( "call gravitino.system.drop_catalog(catalog => 'memory1', ignore_not_exist => true)"); - assertThat(computeActual("show catalogs").getOnlyColumnAsSet()) - .doesNotContain("\"test.memory1\""); + assertThat(computeActual("show catalogs").getOnlyColumnAsSet()).doesNotContain("test.memory1"); - // test metalake named test1. the connector name is gravitino1 + // test metalake named test1. the connnector name is test1 GravitinoAdminClient gravitinoClient = server.createGravitinoClient(); gravitinoClient.createMetalake("test1", "", Collections.emptyMap()); - assertUpdate("call gravitino1.system.create_catalog('memory1', 'memory', Map())"); - assertThat(computeActual("show catalogs").getOnlyColumnAsSet()).contains("\"test1.memory1\""); - assertUpdate("call gravitino1.system.drop_catalog('memory1')"); - assertThat(computeActual("show catalogs").getOnlyColumnAsSet()) - .doesNotContain("\"test1.memory1\""); + assertUpdate("call test1.system.create_catalog('memory1', 'memory', Map())"); + assertThat(computeActual("show catalogs").getOnlyColumnAsSet()).contains("test1.memory1"); + assertUpdate("call test1.system.drop_catalog('memory1')"); + assertThat(computeActual("show catalogs").getOnlyColumnAsSet()).doesNotContain("test1.memory1"); } @Test diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoPlugin.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoPlugin.java index 35db1765a8f..c6b6923db01 100644 --- a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoPlugin.java +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoPlugin.java @@ -12,7 +12,7 @@ public class TestGravitinoPlugin extends GravitinoPlugin { private TestGravitinoConnectorFactory factory; - private final GravitinoAdminClient gravitinoClient; + private GravitinoAdminClient gravitinoClient; public TestGravitinoPlugin(GravitinoAdminClient gravitinoClient) { this.gravitinoClient = gravitinoClient; diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoTableHandle.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoTableHandle.java index 4dbc7e25f73..aa12ab30220 100644 --- a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoTableHandle.java +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoTableHandle.java @@ -4,16 +4,14 @@ */ package com.datastrato.gravitino.trino.connector; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.testng.Assert.assertTrue; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import io.airlift.json.JsonCodec; import io.trino.spi.connector.ConnectorTableHandle; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; +import org.testng.annotations.Test; -@Disabled public class TestGravitinoTableHandle { private final JsonCodec codec = JsonCodec.jsonCodec(GravitinoTableHandle.class); diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/hive/TestHiveCatalogPropertyConverter.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/hive/TestHiveCatalogPropertyConverter.java index d6aa3b3fcaa..124762aefa5 100644 --- a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/hive/TestHiveCatalogPropertyConverter.java +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/hive/TestHiveCatalogPropertyConverter.java @@ -9,12 +9,12 @@ import com.datastrato.gravitino.catalog.hive.HiveTablePropertiesMetadata; import com.datastrato.gravitino.trino.connector.metadata.GravitinoCatalog; import com.datastrato.gravitino.trino.connector.metadata.TestGravitinoCatalog; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.Sets; import java.util.Map; import java.util.Set; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; +import org.testcontainers.shaded.com.google.common.collect.ImmutableMap; +import org.testng.Assert; +import org.testng.annotations.Test; public class TestHiveCatalogPropertyConverter { @@ -30,9 +30,9 @@ public void testConverter() { .build(); Map re = hiveCatalogPropertyConverter.gravitinoToEngineProperties(map); - Assertions.assertEquals(re.get("hive.immutable-partitions"), "true"); - Assertions.assertEquals(re.get("hive.compression-codec"), "ZSTD"); - Assertions.assertNull(re.get("hive.unknown-key")); + Assert.assertEquals(re.get("hive.immutable-partitions"), "true"); + Assert.assertEquals(re.get("hive.compression-codec"), "ZSTD"); + Assert.assertEquals(re.get("hive.unknown-key"), null); } // To test whether we load jar `bundled-catalog` successfully. @@ -45,7 +45,7 @@ public void testPropertyMetadata() { // Needs to confirm whether external should be a property key for Trino. gravitinoHiveKeys.remove("external"); - Assertions.assertTrue(actualGravitinoKeys.containsAll(gravitinoHiveKeys)); + Assert.assertTrue(actualGravitinoKeys.containsAll(gravitinoHiveKeys)); } @Test @@ -63,21 +63,26 @@ public void testBuildConnectorProperties() throws Exception { TestGravitinoCatalog.mockCatalog( name, "hive", "test catalog", Catalog.Type.RELATIONAL, properties); HiveConnectorAdapter adapter = new HiveConnectorAdapter(); - Map config = + Map stringObjectMap = adapter.buildInternalConnectorConfig(new GravitinoCatalog("test", mockCatalog)); + // test connector attributes + Assert.assertEquals(stringObjectMap.get("connectorName"), "hive"); + + Map propertiesMap = (Map) stringObjectMap.get("properties"); + // test converted properties - Assertions.assertEquals(config.get("hive.metastore.uri"), "thrift://localhost:9083"); + Assert.assertEquals(propertiesMap.get("hive.metastore.uri"), "thrift://localhost:9083"); // test fixed properties - Assertions.assertEquals(config.get("hive.security"), "allow-all"); + Assert.assertEquals(propertiesMap.get("hive.security"), "allow-all"); // test trino passing properties - Assertions.assertEquals( - config.get("hive.config.resources"), "/tmp/hive-site.xml, /tmp/core-site.xml"); + Assert.assertEquals( + propertiesMap.get("hive.config.resources"), "/tmp/hive-site.xml, /tmp/core-site.xml"); // test unknown properties - Assertions.assertNull(config.get("hive.unknown-key")); - Assertions.assertNull(config.get("trino.bypass.unknown-key")); + Assert.assertNull(propertiesMap.get("hive.unknown-key")); + Assert.assertNull(propertiesMap.get("trino.bypass.unknown-key")); } } diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/hive/TestHiveDataTypeConverter.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/hive/TestHiveDataTypeConverter.java index e3222a68199..9aaba6b9942 100644 --- a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/hive/TestHiveDataTypeConverter.java +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/hive/TestHiveDataTypeConverter.java @@ -8,9 +8,8 @@ import com.datastrato.gravitino.rel.types.Types; import com.datastrato.gravitino.trino.connector.util.GeneralDataTypeTransformer; import io.trino.spi.TrinoException; -import org.junit.Assert; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; +import org.testng.Assert; +import org.testng.annotations.Test; public class TestHiveDataTypeConverter { @@ -18,41 +17,41 @@ public class TestHiveDataTypeConverter { public void testTrinoTypeToGravitinoType() { GeneralDataTypeTransformer generalDataTypeTransformer = new HiveDataTypeTransformer(); io.trino.spi.type.Type charTypeWithLengthOne = io.trino.spi.type.CharType.createCharType(1); - Assertions.assertEquals( + Assert.assertEquals( generalDataTypeTransformer.getGravitinoType(charTypeWithLengthOne), Types.FixedCharType.of(1)); io.trino.spi.type.Type charTypeWithLength = io.trino.spi.type.CharType.createCharType(255); - Assertions.assertEquals( + Assert.assertEquals( generalDataTypeTransformer.getGravitinoType(charTypeWithLength), Types.FixedCharType.of(255)); io.trino.spi.type.Type charLengthIsOverflow = io.trino.spi.type.CharType.createCharType(256); Exception e = - Assert.assertThrows( + Assert.expectThrows( TrinoException.class, () -> generalDataTypeTransformer.getGravitinoType(charLengthIsOverflow)); - Assertions.assertTrue( + Assert.assertTrue( e.getMessage() .contains("Hive does not support the datatype CHAR with the length greater than 255")); io.trino.spi.type.Type varcharType = io.trino.spi.type.VarcharType.createVarcharType(1); - Assertions.assertEquals( + Assert.assertEquals( generalDataTypeTransformer.getGravitinoType(varcharType), Types.VarCharType.of(1)); io.trino.spi.type.Type varcharTypeWithLength = io.trino.spi.type.VarcharType.createVarcharType(65535); - Assertions.assertEquals( + Assert.assertEquals( generalDataTypeTransformer.getGravitinoType(varcharTypeWithLength), Types.VarCharType.of(65535)); io.trino.spi.type.Type varcharLengthIsOverflow = io.trino.spi.type.VarcharType.createVarcharType(65536); e = - Assert.assertThrows( + Assert.expectThrows( TrinoException.class, () -> generalDataTypeTransformer.getGravitinoType(varcharLengthIsOverflow)); - Assertions.assertTrue( + Assert.assertTrue( e.getMessage() .contains( "Hive does not support the datatype VARCHAR with the length greater than 65535")); @@ -60,7 +59,7 @@ public void testTrinoTypeToGravitinoType() { io.trino.spi.type.Type varcharTypeWithoutLength = io.trino.spi.type.VarcharType.createUnboundedVarcharType(); - Assertions.assertEquals( + Assert.assertEquals( generalDataTypeTransformer.getGravitinoType(varcharTypeWithoutLength), Types.StringType.get()); } diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/TestIcebergCatalogPropertyConverter.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/TestIcebergCatalogPropertyConverter.java index dc6bdfe0511..e4c072fef2b 100644 --- a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/TestIcebergCatalogPropertyConverter.java +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/TestIcebergCatalogPropertyConverter.java @@ -10,14 +10,15 @@ import com.datastrato.gravitino.catalog.property.PropertyConverter; import com.datastrato.gravitino.trino.connector.metadata.GravitinoCatalog; import com.datastrato.gravitino.trino.connector.metadata.TestGravitinoCatalog; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Maps; import com.google.common.collect.Sets; import io.trino.spi.TrinoException; import java.util.Map; import java.util.Set; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; +import org.assertj.core.api.Assertions; +import org.testcontainers.shaded.com.google.common.collect.ImmutableMap; +import org.testng.Assert; +import org.testng.annotations.Test; +import org.testng.collections.Maps; public class TestIcebergCatalogPropertyConverter { @@ -32,16 +33,15 @@ public void testHiveBackendProperty() { Map hiveBackendConfig = propertyConverter.gravitinoToEngineProperties(gravitinoIcebergConfig); - Assertions.assertEquals(hiveBackendConfig.get("iceberg.catalog.type"), "hive_metastore"); - Assertions.assertEquals(hiveBackendConfig.get("hive.metastore.uri"), "1111"); + Assert.assertEquals(hiveBackendConfig.get("iceberg.catalog.type"), "hive_metastore"); + Assert.assertEquals(hiveBackendConfig.get("hive.metastore.uri"), "1111"); Map wrongMap = Maps.newHashMap(gravitinoIcebergConfig); wrongMap.remove("uri"); - Assertions.assertThrows( - TrinoException.class, - () -> propertyConverter.gravitinoToEngineProperties(wrongMap), - "Missing required property for Hive backend: [uri]"); + Assertions.assertThatThrownBy(() -> propertyConverter.gravitinoToEngineProperties(wrongMap)) + .isInstanceOf(TrinoException.class) + .hasMessageContaining("Missing required property for Hive backend: [uri]"); } @Test @@ -60,25 +60,22 @@ public void testJDBCBackendProperty() { propertyConverter.gravitinoToEngineProperties(gravitinoIcebergConfig); // Test all properties are converted - Assertions.assertEquals( + Assert.assertEquals( hiveBackendConfig.get("iceberg.jdbc-catalog.connection-url"), "jdbc:mysql://127.0.0.1:3306/metastore_db?createDatabaseIfNotExist=true"); - Assertions.assertEquals( - hiveBackendConfig.get("iceberg.jdbc-catalog.connection-user"), "zhangsan"); - Assertions.assertEquals( - hiveBackendConfig.get("iceberg.jdbc-catalog.connection-password"), "lisi"); - Assertions.assertNull(hiveBackendConfig.get("other-key")); - Assertions.assertEquals(hiveBackendConfig.get("iceberg.catalog.type"), "jdbc"); - Assertions.assertEquals( + Assert.assertEquals(hiveBackendConfig.get("iceberg.jdbc-catalog.connection-user"), "zhangsan"); + Assert.assertEquals(hiveBackendConfig.get("iceberg.jdbc-catalog.connection-password"), "lisi"); + Assert.assertNull(hiveBackendConfig.get("other-key")); + Assert.assertEquals(hiveBackendConfig.get("iceberg.catalog.type"), "jdbc"); + Assert.assertEquals( hiveBackendConfig.get("iceberg.jdbc-catalog.driver-class"), "com.mysql.cj.jdbc.Driver"); Map wrongMap = Maps.newHashMap(gravitinoIcebergConfig); wrongMap.remove("jdbc-driver"); - Assertions.assertThrows( - TrinoException.class, - () -> propertyConverter.gravitinoToEngineProperties(wrongMap), - "Missing required property for JDBC backend: [jdbc-driver]"); + Assertions.assertThatThrownBy(() -> propertyConverter.gravitinoToEngineProperties(wrongMap)) + .isInstanceOf(TrinoException.class) + .hasMessageContaining("Missing required property for JDBC backend: [jdbc-driver]"); } // To test whether we load jar `bundled-catalog` successfully. @@ -89,7 +86,7 @@ public void testPropertyMetadata() { Set actualGravitinoKeys = Sets.newHashSet(new IcebergTablePropertiesMetadata().propertyEntries().keySet()); - Assertions.assertTrue(actualGravitinoKeys.containsAll(gravitinoHiveKeys)); + Assert.assertTrue(actualGravitinoKeys.containsAll(gravitinoHiveKeys)); } @Test @@ -110,19 +107,24 @@ public void testBuildConnectorPropertiesWithHiveBackend() throws Exception { name, "lakehouse-iceberg", "test catalog", Catalog.Type.RELATIONAL, properties); IcebergConnectorAdapter adapter = new IcebergConnectorAdapter(); - Map config = + Map stringObjectMap = adapter.buildInternalConnectorConfig(new GravitinoCatalog("test", mockCatalog)); + // test connector attributes + Assert.assertEquals(stringObjectMap.get("connectorName"), "iceberg"); + + Map propertiesMap = (Map) stringObjectMap.get("properties"); + // test converted properties - Assertions.assertEquals(config.get("hive.metastore.uri"), "thrift://localhost:9083"); - Assertions.assertEquals(config.get("iceberg.catalog.type"), "hive_metastore"); + Assert.assertEquals(propertiesMap.get("hive.metastore.uri"), "thrift://localhost:9083"); + Assert.assertEquals(propertiesMap.get("iceberg.catalog.type"), "hive_metastore"); // test trino passing properties - Assertions.assertEquals(config.get("iceberg.table-statistics-enabled"), "true"); + Assert.assertEquals(propertiesMap.get("iceberg.table-statistics-enabled"), "true"); // test unknown properties - Assertions.assertNull(config.get("hive.unknown-key")); - Assertions.assertNull(config.get("trino.bypass.unknown-key")); + Assert.assertNull(propertiesMap.get("hive.unknown-key")); + Assert.assertNull(propertiesMap.get("trino.bypass.unknown-key")); } @Test @@ -146,24 +148,29 @@ public void testBuildConnectorPropertiesWithMySqlBackEnd() throws Exception { name, "lakehouse-iceberg", "test catalog", Catalog.Type.RELATIONAL, properties); IcebergConnectorAdapter adapter = new IcebergConnectorAdapter(); - Map config = + Map stringObjectMap = adapter.buildInternalConnectorConfig(new GravitinoCatalog("test", mockCatalog)); + // test connector attributes + Assert.assertEquals(stringObjectMap.get("connectorName"), "iceberg"); + + Map propertiesMap = (Map) stringObjectMap.get("properties"); + // test converted properties - Assertions.assertEquals( - config.get("iceberg.jdbc-catalog.connection-url"), + Assert.assertEquals( + propertiesMap.get("iceberg.jdbc-catalog.connection-url"), "jdbc:mysql://%s:3306/metastore_db?createDatabaseIfNotExist=true"); - Assertions.assertEquals(config.get("iceberg.jdbc-catalog.connection-user"), "root"); - Assertions.assertEquals(config.get("iceberg.jdbc-catalog.connection-password"), "ds123"); - Assertions.assertEquals( - config.get("iceberg.jdbc-catalog.driver-class"), "com.mysql.cj.jdbc.Driver"); - Assertions.assertEquals(config.get("iceberg.catalog.type"), "jdbc"); + Assert.assertEquals(propertiesMap.get("iceberg.jdbc-catalog.connection-user"), "root"); + Assert.assertEquals(propertiesMap.get("iceberg.jdbc-catalog.connection-password"), "ds123"); + Assert.assertEquals( + propertiesMap.get("iceberg.jdbc-catalog.driver-class"), "com.mysql.cj.jdbc.Driver"); + Assert.assertEquals(propertiesMap.get("iceberg.catalog.type"), "jdbc"); // test trino passing properties - Assertions.assertEquals(config.get("iceberg.table-statistics-enabled"), "true"); + Assert.assertEquals(propertiesMap.get("iceberg.table-statistics-enabled"), "true"); // test unknown properties - Assertions.assertNull(config.get("hive.unknown-key")); - Assertions.assertNull(config.get("trino.bypass.unknown-key")); + Assert.assertNull(propertiesMap.get("hive.unknown-key")); + Assert.assertNull(propertiesMap.get("trino.bypass.unknown-key")); } } diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/TestIcebergDataTypeTransformer.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/TestIcebergDataTypeTransformer.java index e5f9adb0cec..b69c5d0a31d 100644 --- a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/TestIcebergDataTypeTransformer.java +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/TestIcebergDataTypeTransformer.java @@ -9,8 +9,8 @@ import com.datastrato.gravitino.trino.connector.util.GeneralDataTypeTransformer; import io.trino.spi.TrinoException; import io.trino.spi.type.VarcharType; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; +import org.testng.Assert; +import org.testng.annotations.Test; public class TestIcebergDataTypeTransformer { @@ -20,21 +20,21 @@ public void testTrinoTypeToGravitinoType() { io.trino.spi.type.Type charTypeWithLengthOne = io.trino.spi.type.CharType.createCharType(1); Exception e = - Assertions.assertThrows( + Assert.expectThrows( TrinoException.class, () -> generalDataTypeTransformer.getGravitinoType(charTypeWithLengthOne)); - Assertions.assertTrue(e.getMessage().contains("Iceberg does not support the datatype CHAR")); + Assert.assertTrue(e.getMessage().contains("Iceberg does not support the datatype CHAR")); io.trino.spi.type.Type varcharType = io.trino.spi.type.VarcharType.createVarcharType(1); e = - Assertions.assertThrows( + Assert.expectThrows( TrinoException.class, () -> generalDataTypeTransformer.getGravitinoType(varcharType)); - Assertions.assertTrue( + Assert.assertTrue( e.getMessage().contains("Iceberg does not support the datatype VARCHAR with length")); io.trino.spi.type.Type varcharTypeWithoutLength = VarcharType.VARCHAR; - Assertions.assertEquals( + Assert.assertEquals( generalDataTypeTransformer.getGravitinoType(varcharTypeWithoutLength), Types.StringType.get()); } diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/TestJDBCCatalogPropertyConverter.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/TestJDBCCatalogPropertyConverter.java index 0d43a5d2e72..02320854041 100644 --- a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/TestJDBCCatalogPropertyConverter.java +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/TestJDBCCatalogPropertyConverter.java @@ -15,10 +15,10 @@ import com.datastrato.gravitino.trino.connector.catalog.jdbc.postgresql.PostgreSQLConnectorAdapter; import com.datastrato.gravitino.trino.connector.metadata.GravitinoCatalog; import com.datastrato.gravitino.trino.connector.metadata.TestGravitinoCatalog; -import com.google.common.collect.ImmutableMap; import java.util.Map; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; +import org.testcontainers.shaded.com.google.common.collect.ImmutableMap; +import org.testng.Assert; +import org.testng.annotations.Test; public class TestJDBCCatalogPropertyConverter { @@ -33,17 +33,17 @@ public void testTrinoPropertyKeyToGravitino() { Map trinoProperties = propertyConverter.gravitinoToEngineProperties(gravitinoProperties); - Assertions.assertEquals( + Assert.assertEquals( trinoProperties.get(JDBC_CONNECTION_URL_KEY), "jdbc:mysql://localhost:3306"); - Assertions.assertEquals(trinoProperties.get(JDBC_CONNECTION_USER_KEY), "root"); - Assertions.assertEquals(trinoProperties.get(JDBC_CONNECTION_PASSWORD_KEY), "root"); + Assert.assertEquals(trinoProperties.get(JDBC_CONNECTION_USER_KEY), "root"); + Assert.assertEquals(trinoProperties.get(JDBC_CONNECTION_PASSWORD_KEY), "root"); Map gravitinoPropertiesWithoutPassword = ImmutableMap.of( "jdbc-url", "jdbc:mysql://localhost:3306", "jdbc-user", "root"); - Assertions.assertThrows( + Assert.assertThrows( IllegalArgumentException.class, () -> { propertyConverter.gravitinoToEngineProperties(gravitinoPropertiesWithoutPassword); @@ -68,20 +68,26 @@ public void testBuildPostgreSqlConnectorProperties() throws Exception { name, "jdbc-postgresql", "test catalog", Catalog.Type.RELATIONAL, properties); PostgreSQLConnectorAdapter adapter = new PostgreSQLConnectorAdapter(); - Map config = + Map stringObjectMap = adapter.buildInternalConnectorConfig(new GravitinoCatalog("test", mockCatalog)); + // test connector attributes + Assert.assertEquals(stringObjectMap.get("connectorName"), "postgresql"); + + Map propertiesMap = (Map) stringObjectMap.get("properties"); + // test converted properties - Assertions.assertEquals(config.get("connection-url"), "jdbc:postgresql://localhost:5432/test"); - Assertions.assertEquals(config.get("connection-user"), "test"); - Assertions.assertEquals(config.get("connection-password"), "test"); + Assert.assertEquals( + propertiesMap.get("connection-url"), "jdbc:postgresql://localhost:5432/test"); + Assert.assertEquals(propertiesMap.get("connection-user"), "test"); + Assert.assertEquals(propertiesMap.get("connection-password"), "test"); // test trino passing properties - Assertions.assertEquals(config.get("join-pushdown.strategy"), "EAGER"); + Assert.assertEquals(propertiesMap.get("join-pushdown.strategy"), "EAGER"); // test unknown properties - Assertions.assertNull(config.get("hive.unknown-key")); - Assertions.assertNull(config.get("trino.bypass.unknown-key")); + Assert.assertNull(propertiesMap.get("hive.unknown-key")); + Assert.assertNull(propertiesMap.get("trino.bypass.unknown-key")); } @Test @@ -102,19 +108,24 @@ public void testBuildMySqlConnectorProperties() throws Exception { name, "jdbc-postgresql", "test catalog", Catalog.Type.RELATIONAL, properties); MySQLConnectorAdapter adapter = new MySQLConnectorAdapter(); - Map config = + Map stringObjectMap = adapter.buildInternalConnectorConfig(new GravitinoCatalog("test", mockCatalog)); + // test connector attributes + Assert.assertEquals(stringObjectMap.get("connectorName"), "mysql"); + + Map propertiesMap = (Map) stringObjectMap.get("properties"); + // test converted properties - Assertions.assertEquals(config.get("connection-url"), "jdbc:mysql://localhost:5432/test"); - Assertions.assertEquals(config.get("connection-user"), "test"); - Assertions.assertEquals(config.get("connection-password"), "test"); + Assert.assertEquals(propertiesMap.get("connection-url"), "jdbc:mysql://localhost:5432/test"); + Assert.assertEquals(propertiesMap.get("connection-user"), "test"); + Assert.assertEquals(propertiesMap.get("connection-password"), "test"); // test trino passing properties - Assertions.assertEquals(config.get("join-pushdown.strategy"), "EAGER"); + Assert.assertEquals(propertiesMap.get("join-pushdown.strategy"), "EAGER"); // test unknown properties - Assertions.assertNull(config.get("hive.unknown-key")); - Assertions.assertNull(config.get("trino.bypass.unknown-key")); + Assert.assertNull(propertiesMap.get("hive.unknown-key")); + Assert.assertNull(propertiesMap.get("trino.bypass.unknown-key")); } } diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/mysql/TestMySQLDataTypeTransformer.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/mysql/TestMySQLDataTypeTransformer.java index 7e45850908f..2b75aa0f72b 100644 --- a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/mysql/TestMySQLDataTypeTransformer.java +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/mysql/TestMySQLDataTypeTransformer.java @@ -9,8 +9,8 @@ import com.datastrato.gravitino.rel.types.Types; import com.datastrato.gravitino.trino.connector.util.GeneralDataTypeTransformer; import io.trino.spi.TrinoException; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; +import org.testng.Assert; +import org.testng.annotations.Test; public class TestMySQLDataTypeTransformer { @@ -18,37 +18,37 @@ public class TestMySQLDataTypeTransformer { public void testTrinoTypeToGravitinoType() { GeneralDataTypeTransformer generalDataTypeTransformer = new MySQLDataTypeTransformer(); io.trino.spi.type.Type charTypeWithLengthOne = io.trino.spi.type.CharType.createCharType(1); - Assertions.assertEquals( + Assert.assertEquals( generalDataTypeTransformer.getGravitinoType(charTypeWithLengthOne), Types.FixedCharType.of(1)); io.trino.spi.type.Type charTypeWithLength = io.trino.spi.type.CharType.createCharType(256); Exception e = - Assertions.assertThrows( + Assert.expectThrows( TrinoException.class, () -> generalDataTypeTransformer.getGravitinoType(charTypeWithLength)); - Assertions.assertTrue( + Assert.assertTrue( e.getMessage() .contains("MySQL does not support the datatype CHAR with the length greater than 255")); io.trino.spi.type.Type varcharType = io.trino.spi.type.VarcharType.createVarcharType(1); - Assertions.assertEquals( + Assert.assertEquals( generalDataTypeTransformer.getGravitinoType(varcharType), Types.VarCharType.of(1)); io.trino.spi.type.Type varcharTypeWithLength = io.trino.spi.type.VarcharType.createVarcharType(16384); e = - Assertions.assertThrows( + Assert.expectThrows( TrinoException.class, () -> generalDataTypeTransformer.getGravitinoType(varcharTypeWithLength)); - Assertions.assertTrue( + Assert.assertTrue( e.getMessage() .contains( "MySQL does not support the datatype VARCHAR with the length greater than 16383")); io.trino.spi.type.Type varcharTypeWithLength2 = io.trino.spi.type.VarcharType.createUnboundedVarcharType(); - Assertions.assertEquals( + Assert.assertEquals( generalDataTypeTransformer.getGravitinoType(varcharTypeWithLength2), Types.StringType.get()); } @@ -58,7 +58,7 @@ public void testGravitinoCharToTrinoType() { GeneralDataTypeTransformer generalDataTypeTransformer = new MySQLDataTypeTransformer(); Type stringType = Types.StringType.get(); - Assertions.assertEquals( + Assert.assertEquals( generalDataTypeTransformer.getTrinoType(stringType), io.trino.spi.type.VarcharType.createUnboundedVarcharType()); } diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/postgresql/TestPostgreSQLDataTypeTransformer.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/postgresql/TestPostgreSQLDataTypeTransformer.java index 22b3adbc359..d4adcf31512 100644 --- a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/postgresql/TestPostgreSQLDataTypeTransformer.java +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/postgresql/TestPostgreSQLDataTypeTransformer.java @@ -8,8 +8,8 @@ import com.datastrato.gravitino.rel.types.Type; import com.datastrato.gravitino.rel.types.Types; import com.datastrato.gravitino.trino.connector.util.GeneralDataTypeTransformer; -import org.junit.Assert; -import org.junit.jupiter.api.Test; +import org.testng.Assert; +import org.testng.annotations.Test; public class TestPostgreSQLDataTypeTransformer { diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/metadata/TestGravitinoCatalog.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/metadata/TestGravitinoCatalog.java index 705995e2a62..58453993a63 100644 --- a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/metadata/TestGravitinoCatalog.java +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/metadata/TestGravitinoCatalog.java @@ -4,16 +4,16 @@ */ package com.datastrato.gravitino.trino.connector.metadata; -import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; import com.datastrato.gravitino.Audit; import com.datastrato.gravitino.Catalog; import java.time.Instant; import java.util.Collections; import java.util.Map; -import org.junit.jupiter.api.Test; +import org.testng.annotations.Test; public class TestGravitinoCatalog { diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/metadata/TestGravitinoColumn.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/metadata/TestGravitinoColumn.java index e9242e1db19..a3bd75dccf5 100644 --- a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/metadata/TestGravitinoColumn.java +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/metadata/TestGravitinoColumn.java @@ -4,11 +4,11 @@ */ package com.datastrato.gravitino.trino.connector.metadata; -import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.testng.Assert.assertEquals; import com.datastrato.gravitino.rel.Column; import com.datastrato.gravitino.rel.types.Types; -import org.junit.jupiter.api.Test; +import org.testng.annotations.Test; public class TestGravitinoColumn { diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/metadata/TestGravitinoSchema.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/metadata/TestGravitinoSchema.java index a34c3fc1ccc..49760279964 100644 --- a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/metadata/TestGravitinoSchema.java +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/metadata/TestGravitinoSchema.java @@ -4,16 +4,16 @@ */ package com.datastrato.gravitino.trino.connector.metadata; -import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; import com.datastrato.gravitino.Audit; import com.datastrato.gravitino.Schema; import java.time.Instant; import java.util.HashMap; import java.util.Map; -import org.junit.jupiter.api.Test; +import org.testng.annotations.Test; public class TestGravitinoSchema { diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/metadata/TestGravitinoTable.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/metadata/TestGravitinoTable.java index 7db924537c2..2cb89020957 100644 --- a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/metadata/TestGravitinoTable.java +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/metadata/TestGravitinoTable.java @@ -6,6 +6,9 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertTrue; import com.datastrato.gravitino.Audit; import com.datastrato.gravitino.rel.Column; @@ -22,8 +25,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; +import org.testng.annotations.Test; public class TestGravitinoTable { @@ -38,31 +40,30 @@ public void testGravitinoTable() { GravitinoTable table = new GravitinoTable("db1", "table1", mockTable); - Assertions.assertEquals(table.getName(), mockTable.name()); - Assertions.assertEquals(table.getSchemaName(), "db1"); - Assertions.assertEquals(table.getColumns().size(), mockTable.columns().length); + assertEquals(table.getName(), mockTable.name()); + assertEquals(table.getSchemaName(), "db1"); + assertEquals(table.getColumns().size(), mockTable.columns().length); for (int i = 0; i < table.getColumns().size(); i++) { - Assertions.assertEquals(table.getColumns().get(i).getName(), mockTable.columns()[i].name()); + assertEquals(table.getColumns().get(i).getName(), mockTable.columns()[i].name()); } - Assertions.assertEquals(table.getComment(), mockTable.comment()); - Assertions.assertEquals(table.getProperties(), mockTable.properties()); + assertEquals(table.getComment(), mockTable.comment()); + assertEquals(table.getProperties(), mockTable.properties()); CatalogConnectorMetadataAdapter adapter = new HiveMetadataAdapter( Collections.emptyList(), Collections.emptyList(), Collections.emptyList()); ConnectorTableMetadata tableMetadata = adapter.getTableMetadata(table); - Assertions.assertEquals(tableMetadata.getColumns().size(), table.getColumns().size()); - Assertions.assertEquals(tableMetadata.getTableSchema().getTable().getSchemaName(), "db1"); - Assertions.assertEquals( - tableMetadata.getTableSchema().getTable().getTableName(), table.getName()); + assertEquals(tableMetadata.getColumns().size(), table.getColumns().size()); + assertEquals(tableMetadata.getTableSchema().getTable().getSchemaName(), "db1"); + assertEquals(tableMetadata.getTableSchema().getTable().getTableName(), table.getName()); for (int i = 0; i < table.getColumns().size(); i++) { - Assertions.assertEquals( + assertEquals( tableMetadata.getColumns().get(i).getName(), table.getColumns().get(i).getName()); } - Assertions.assertTrue(tableMetadata.getComment().isPresent()); - Assertions.assertEquals(tableMetadata.getComment().get(), mockTable.comment()); + assertTrue(tableMetadata.getComment().isPresent()); + assertEquals(tableMetadata.getComment().get(), mockTable.comment()); } @Test @@ -76,13 +77,13 @@ public void testGravitinoTableWithOutComment() { Table mockTable = mockTable("table1", columns, null, properties); GravitinoTable table = new GravitinoTable("db1", "table1", mockTable); - Assertions.assertNull(table.getComment()); + assertNull(table.getComment()); CatalogConnectorMetadataAdapter adapter = new HiveMetadataAdapter( Collections.emptyList(), Collections.emptyList(), Collections.emptyList()); ConnectorTableMetadata tableMetadata = adapter.getTableMetadata(table); - Assertions.assertTrue(tableMetadata.getComment().isEmpty()); + assertTrue(tableMetadata.getComment().isEmpty()); } public static Table mockTable( diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/util/TestDataTypeTransformer.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/util/TestDataTypeTransformer.java index 319846dfb70..efbbd8e78f2 100644 --- a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/util/TestDataTypeTransformer.java +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/util/TestDataTypeTransformer.java @@ -6,6 +6,7 @@ import static com.datastrato.gravitino.trino.connector.GravitinoErrorCode.GRAVITINO_UNSUPPORTED_GRAVITINO_DATATYPE; import static com.datastrato.gravitino.trino.connector.GravitinoErrorCode.GRAVITINO_UNSUPPORTED_TRINO_DATATYPE; +import static org.testng.Assert.assertEquals; import com.datastrato.gravitino.rel.types.Types; import io.trino.spi.TrinoException; @@ -30,9 +31,8 @@ import io.trino.spi.type.UuidType; import io.trino.spi.type.VarcharType; import java.util.Optional; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; import org.testcontainers.shaded.com.google.common.collect.ImmutableList; +import org.testng.annotations.Test; public class TestDataTypeTransformer { @@ -40,53 +40,47 @@ public class TestDataTypeTransformer { @Test public void testGetGravitinoType() { - Assertions.assertEquals( + assertEquals( dataTypeTransformer.getGravitinoType(BooleanType.BOOLEAN), Types.BooleanType.get()); - Assertions.assertEquals( - dataTypeTransformer.getGravitinoType(TinyintType.TINYINT), Types.ByteType.get()); - Assertions.assertEquals( + assertEquals(dataTypeTransformer.getGravitinoType(TinyintType.TINYINT), Types.ByteType.get()); + assertEquals( dataTypeTransformer.getGravitinoType(SmallintType.SMALLINT), Types.ShortType.get()); - Assertions.assertEquals( + assertEquals( dataTypeTransformer.getGravitinoType(IntegerType.INTEGER), Types.IntegerType.get()); - Assertions.assertEquals( - dataTypeTransformer.getGravitinoType(BigintType.BIGINT), Types.LongType.get()); - - Assertions.assertEquals( - dataTypeTransformer.getGravitinoType(RealType.REAL), Types.FloatType.get()); - Assertions.assertEquals( - dataTypeTransformer.getGravitinoType(DoubleType.DOUBLE), Types.DoubleType.get()); - Assertions.assertEquals( + assertEquals(dataTypeTransformer.getGravitinoType(BigintType.BIGINT), Types.LongType.get()); + + assertEquals(dataTypeTransformer.getGravitinoType(RealType.REAL), Types.FloatType.get()); + assertEquals(dataTypeTransformer.getGravitinoType(DoubleType.DOUBLE), Types.DoubleType.get()); + assertEquals( dataTypeTransformer.getGravitinoType(DecimalType.createDecimalType(10, 2)), Types.DecimalType.of(10, 2)); - Assertions.assertEquals( + assertEquals( dataTypeTransformer.getGravitinoType(CharType.createCharType(10)), Types.FixedCharType.of(10)); - Assertions.assertEquals( + assertEquals( dataTypeTransformer.getGravitinoType(VarcharType.createVarcharType(10)), Types.VarCharType.of(10)); - Assertions.assertEquals( + assertEquals( dataTypeTransformer.getGravitinoType(VarcharType.createVarcharType(VarcharType.MAX_LENGTH)), Types.VarCharType.of(Integer.MAX_VALUE - 1)); - Assertions.assertEquals( - dataTypeTransformer.getGravitinoType(DateType.DATE), Types.DateType.get()); - Assertions.assertEquals( - dataTypeTransformer.getGravitinoType(TimeType.TIME_MILLIS), Types.TimeType.get()); - Assertions.assertEquals( + assertEquals(dataTypeTransformer.getGravitinoType(DateType.DATE), Types.DateType.get()); + assertEquals(dataTypeTransformer.getGravitinoType(TimeType.TIME_MILLIS), Types.TimeType.get()); + assertEquals( dataTypeTransformer.getGravitinoType(TimestampType.TIMESTAMP_MILLIS), Types.TimestampType.withoutTimeZone()); - Assertions.assertEquals( + assertEquals( dataTypeTransformer.getGravitinoType(new ArrayType(IntegerType.INTEGER)), Types.ListType.nullable(Types.IntegerType.get())); - Assertions.assertEquals( + assertEquals( dataTypeTransformer.getGravitinoType( new MapType(SmallintType.SMALLINT, IntegerType.INTEGER, new TypeOperators())), Types.MapType.of(Types.ShortType.get(), Types.IntegerType.get(), true)); - Assertions.assertEquals( + assertEquals( dataTypeTransformer.getGravitinoType( RowType.from( ImmutableList.of( @@ -96,8 +90,7 @@ public void testGetGravitinoType() { Types.StructType.Field.nullableField("a", Types.IntegerType.get()), Types.StructType.Field.nullableField("b", Types.VarCharType.of(10)))); - Assertions.assertEquals( - dataTypeTransformer.getGravitinoType(UuidType.UUID), Types.UUIDType.get()); + assertEquals(dataTypeTransformer.getGravitinoType(UuidType.UUID), Types.UUIDType.get()); try { dataTypeTransformer.getGravitinoType(HyperLogLogType.HYPER_LOG_LOG); @@ -110,51 +103,44 @@ public void testGetGravitinoType() { @Test public void testGetTrinoType() { - Assertions.assertEquals( - dataTypeTransformer.getTrinoType(Types.BooleanType.get()), BooleanType.BOOLEAN); - - Assertions.assertEquals( - dataTypeTransformer.getTrinoType(Types.ByteType.get()), TinyintType.TINYINT); - Assertions.assertEquals( - dataTypeTransformer.getTrinoType(Types.ShortType.get()), SmallintType.SMALLINT); - Assertions.assertEquals( - dataTypeTransformer.getTrinoType(Types.IntegerType.get()), IntegerType.INTEGER); - Assertions.assertEquals( - dataTypeTransformer.getTrinoType(Types.LongType.get()), BigintType.BIGINT); - - Assertions.assertEquals(dataTypeTransformer.getTrinoType(Types.FloatType.get()), RealType.REAL); - Assertions.assertEquals( - dataTypeTransformer.getTrinoType(Types.DoubleType.get()), DoubleType.DOUBLE); - Assertions.assertEquals( + assertEquals(dataTypeTransformer.getTrinoType(Types.BooleanType.get()), BooleanType.BOOLEAN); + + assertEquals(dataTypeTransformer.getTrinoType(Types.ByteType.get()), TinyintType.TINYINT); + assertEquals(dataTypeTransformer.getTrinoType(Types.ShortType.get()), SmallintType.SMALLINT); + assertEquals(dataTypeTransformer.getTrinoType(Types.IntegerType.get()), IntegerType.INTEGER); + assertEquals(dataTypeTransformer.getTrinoType(Types.LongType.get()), BigintType.BIGINT); + + assertEquals(dataTypeTransformer.getTrinoType(Types.FloatType.get()), RealType.REAL); + assertEquals(dataTypeTransformer.getTrinoType(Types.DoubleType.get()), DoubleType.DOUBLE); + assertEquals( dataTypeTransformer.getTrinoType(Types.DecimalType.of(10, 2)), DecimalType.createDecimalType(10, 2)); - Assertions.assertEquals( + assertEquals( dataTypeTransformer.getTrinoType(Types.FixedCharType.of(10)), CharType.createCharType(10)); - Assertions.assertEquals( + assertEquals( dataTypeTransformer.getTrinoType(Types.VarCharType.of(10)), VarcharType.createVarcharType(10)); - Assertions.assertEquals( + assertEquals( dataTypeTransformer.getTrinoType(Types.VarCharType.of(Integer.MAX_VALUE - 1)), VarcharType.createVarcharType(VarcharType.MAX_LENGTH)); - Assertions.assertEquals(dataTypeTransformer.getTrinoType(Types.DateType.get()), DateType.DATE); - Assertions.assertEquals( - dataTypeTransformer.getTrinoType(Types.TimeType.get()), TimeType.TIME_MILLIS); - Assertions.assertEquals( + assertEquals(dataTypeTransformer.getTrinoType(Types.DateType.get()), DateType.DATE); + assertEquals(dataTypeTransformer.getTrinoType(Types.TimeType.get()), TimeType.TIME_MILLIS); + assertEquals( dataTypeTransformer.getTrinoType(Types.TimestampType.withoutTimeZone()), TimestampType.TIMESTAMP_MILLIS); - Assertions.assertEquals( + assertEquals( dataTypeTransformer.getTrinoType(Types.ListType.nullable(Types.IntegerType.get())), new ArrayType(IntegerType.INTEGER)); - Assertions.assertEquals( + assertEquals( Types.MapType.of(Types.ShortType.get(), Types.IntegerType.get(), true), dataTypeTransformer.getGravitinoType( new MapType(SmallintType.SMALLINT, IntegerType.INTEGER, new TypeOperators()))); - Assertions.assertEquals( + assertEquals( dataTypeTransformer.getTrinoType( Types.StructType.of( Types.StructType.Field.nullableField("a", Types.IntegerType.get()), @@ -164,7 +150,7 @@ public void testGetTrinoType() { new Field(Optional.of("a"), IntegerType.INTEGER), new Field(Optional.of("b"), VarcharType.createVarcharType(10))))); - Assertions.assertEquals(dataTypeTransformer.getTrinoType(Types.UUIDType.get()), UuidType.UUID); + assertEquals(dataTypeTransformer.getTrinoType(Types.UUIDType.get()), UuidType.UUID); try { dataTypeTransformer.getTrinoType(Types.BinaryType.get());