[#1449] feat(catalogs): Introudce new module bundled-catalog for query engine.

build.gradle.kts

@@ -452,7 +452,7 @@ tasks {
     subprojects.forEach() {
       if (!it.name.startsWith("catalog") &&
         !it.name.startsWith("client") && it.name != "trino-connector" &&
-        it.name != "integration-test"
+        it.name != "integration-test" && !it.name.contains("bundled")
       ) {
         from(it.configurations.runtimeClasspath)
         into("distribution/package/libs")
@@ -465,7 +465,8 @@ tasks {
       if (!it.name.startsWith("catalog") &&
         !it.name.startsWith("client") &&
         it.name != "trino-connector" &&
-        it.name != "integration-test"
+        it.name != "integration-test" &&
+        !it.name.contains("bundled")
       ) {
         dependsOn("${it.name}:build")
         from("${it.name}/build/libs")

catalogs/bundled-catalog/build.gradle.kts

@@ -0,0 +1,81 @@
+/*
+ * Copyright 2024 Datastrato Pvt Ltd.
+ * This software is licensed under the Apache License version 2.
+ */
+
+import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
+
+plugins {
+  id("java")
+  alias(libs.plugins.shadow)
+  `maven-publish`
+}
+
+dependencies {
+  implementation(project(":core"))
+  implementation(project(":catalogs:catalog-hive"))
+  implementation(project(":catalogs:catalog-lakehouse-iceberg"))
+  implementation(project(":catalogs:catalog-jdbc-mysql"))
+  implementation(project(":catalogs:catalog-jdbc-postgresql"))
+}
+
+tasks.test {
+  useJUnitPlatform()
+}
+
+tasks.withType<ShadowJar>(ShadowJar::class.java) {
+  isZip64 = true
+  configurations = listOf(project.configurations.compileClasspath.get())
+  archiveClassifier.set("")
+
+  dependencies {
+    exclude("org.*")
+    exclude("javax.*")
+  }
+
+  exclude("**/package-info.class")
+  exclude("**/*.properties")
+  exclude("**/*.html")
+  exclude("org/**")
+  exclude("META-INF")
+  exclude("META-INF/**")
+  exclude("module-info.class")
+  exclude("com/google/**")
+  exclude("com/fasterxml/**")
+  exclude("javax/**")
+  exclude("schema/**")
+  exclude("fr/**")
+  exclude("google/**")
+  exclude("groovy/**")
+  exclude("images/**")
+  exclude("**/*.conf")
+  exclude("**/*.so")
+  exclude("**/*.sxd")
+  exclude("**/*.xsd")
+  exclude("*.ddl")
+  exclude("**/*.txt")
+  exclude("**/*.md")
+  exclude("**/*.dtd")
+  exclude("**/*.thrift")
+  exclude("**/*.jdo")
+  exclude("**/LICENSE")
+  exclude("**/*.MF")
+  exclude("**/*.xml")
+  exclude("*.proto")
+  exclude("*.template")
+  exclude("webapps")
+  exclude("license/*")
+  exclude("*.xml")
+  exclude("*.css")
+  exclude("*.jnilib")
+  exclude("*.dll")
+  exclude("*.jocl")
+  exclude("NOTICE")
+
+  minimize()
+}
+
+tasks.jar {
+  dependsOn(tasks.named("shadowJar"))
+  archiveClassifier.set("empty")
+}

catalogs/bundled-catalog/src/main/java/com/datastrato/catalog/common/ClassProvider.java

@@ -0,0 +1,79 @@
+/*
+ * Copyright 2024 Datastrato Pvt Ltd.
+ * This software is licensed under the Apache License version 2.
+ */
+
+package com.datastrato.catalog.common;
+
+import com.datastrato.gravitino.catalog.BasePropertiesMetadata;
+import com.datastrato.gravitino.catalog.PropertiesMetadata;
+import com.datastrato.gravitino.catalog.PropertyEntry;
+import com.datastrato.gravitino.catalog.hive.HiveCatalogPropertiesMeta;
+import com.datastrato.gravitino.catalog.hive.HiveSchemaPropertiesMetadata;
+import com.datastrato.gravitino.catalog.hive.HiveTablePropertiesMetadata;
+import com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergCatalogPropertiesMetadata;
+import com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergSchemaPropertiesMetadata;
+import com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergTablePropertiesMetadata;
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * The {@link ClassProvider} class serves as a container for the necessary classes used by the
+ * Gravitino query engine, with a primary focus on classes related to property metadata.
+ *
+ * <p>Purpose of this module and class:
+ *
+ * <pre>
+ * - Catalog-related classes are essential for the query engine to directly access catalog information.
+ * - The query engine should be able to detect catalog changes and automatically reload catalog-related
+ *   information to ensure synchronization.
+ * - Including catalog-related jar packages directly is suboptimal for query engines as it may introduce
+ *   unnecessary content.
+ * </pre>
+ *
+ * Therefore, this module is used to store the required classes for the query engine's
+ * functionality.
+ */
+public class ClassProvider {
+
+  private static final Set<Class<?>> BASE_CLASS =
+      new HashSet<Class<?>>() {
+        {
+          add(BasePropertiesMetadata.class);
+          add(PropertyEntry.class);
+          add(PropertiesMetadata.class);
+        }
+      };
+
+  private static final Set<Class<?>> HIVE_NEED_CLASS =
+      new HashSet<Class<?>>() {
+        {
+          add(HiveTablePropertiesMetadata.class);
+          add(HiveSchemaPropertiesMetadata.class);
+          add(HiveCatalogPropertiesMeta.class);
+        }
+      };
+
+  private static final Set<Class<?>> ICEBERG_NEED_CLASS =
+      new HashSet<Class<?>>() {
+        {
+          add(IcebergTablePropertiesMetadata.class);
+          add(IcebergSchemaPropertiesMetadata.class);
+          add(IcebergCatalogPropertiesMetadata.class);
+        }
+      };
+
+  private static final Set<Class<?>> MYSQL_NEED_CLASS =
+      new HashSet<Class<?>>() {
+        {
+          // TODO
+        }
+      };
+
+  private static final Set<Class<?>> PG_NEED_CLASS =
+      new HashSet<Class<?>>() {
+        {
+          // TODO
+        }
+      };
+}

catalogs/catalog-hive/build.gradle.kts

@@ -77,12 +77,12 @@ dependencies {
 tasks {
   val copyDepends by registering(Copy::class) {
     from(configurations.runtimeClasspath)
-    into("build/libs")
+    into("build/libs_all")
   }
 
   val copyCatalogLibs by registering(Copy::class) {
     dependsOn(copyDepends, "build")
-    from("build/libs")
+    from("build/libs_all", "build/libs")
     into("$rootDir/distribution/package/catalogs/hive/libs")
   }
 

catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveCatalogOperations.java

@@ -15,6 +15,7 @@
 import com.datastrato.gravitino.Namespace;
 import com.datastrato.gravitino.catalog.CatalogOperations;
 import com.datastrato.gravitino.catalog.PropertiesMetadata;
+import com.datastrato.gravitino.catalog.hive.HiveTablePropertiesMetadata.TableType;
 import com.datastrato.gravitino.catalog.hive.converter.ToHiveType;
 import com.datastrato.gravitino.exceptions.NoSuchCatalogException;
 import com.datastrato.gravitino.exceptions.NoSuchSchemaException;
@@ -51,7 +52,6 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.metastore.TableType;
 import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
 import org.apache.hadoop.hive.metastore.api.Database;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;

catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveTable.java

@@ -14,10 +14,11 @@
 import static com.datastrato.gravitino.catalog.hive.HiveTablePropertiesMetadata.SERDE_NAME;
 import static com.datastrato.gravitino.catalog.hive.HiveTablePropertiesMetadata.SERDE_PARAMETER_PREFIX;
 import static com.datastrato.gravitino.catalog.hive.HiveTablePropertiesMetadata.TABLE_TYPE;
+import static com.datastrato.gravitino.catalog.hive.HiveTablePropertiesMetadata.TableType.EXTERNAL_TABLE;
+import static com.datastrato.gravitino.catalog.hive.HiveTablePropertiesMetadata.TableType.MANAGED_TABLE;
 import static com.datastrato.gravitino.rel.expressions.transforms.Transforms.identity;
-import static org.apache.hadoop.hive.metastore.TableType.EXTERNAL_TABLE;
-import static org.apache.hadoop.hive.metastore.TableType.MANAGED_TABLE;
 
+import com.datastrato.gravitino.catalog.hive.HiveTablePropertiesMetadata.TableType;
 import com.datastrato.gravitino.catalog.hive.converter.FromHiveType;
 import com.datastrato.gravitino.catalog.hive.converter.ToHiveType;
 import com.datastrato.gravitino.catalog.rel.BaseTable;
@@ -45,7 +46,6 @@
 import lombok.ToString;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.lang3.ArrayUtils;
-import org.apache.hadoop.hive.metastore.TableType;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
 import org.apache.hadoop.hive.metastore.api.Order;
 import org.apache.hadoop.hive.metastore.api.SerDeInfo;

catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveTablePropertiesMetadata.java

@@ -9,7 +9,7 @@
 import static com.datastrato.gravitino.catalog.PropertyEntry.stringImmutablePropertyEntry;
 import static com.datastrato.gravitino.catalog.PropertyEntry.stringReservedPropertyEntry;
 import static com.datastrato.gravitino.catalog.hive.HiveTablePropertiesMetadata.StorageFormat.TEXTFILE;
-import static org.apache.hadoop.hive.metastore.TableType.MANAGED_TABLE;
+import static com.datastrato.gravitino.catalog.hive.HiveTablePropertiesMetadata.TableType.MANAGED_TABLE;
 
 import com.datastrato.gravitino.catalog.BasePropertiesMetadata;
 import com.datastrato.gravitino.catalog.PropertyEntry;
@@ -19,7 +19,6 @@
 import com.google.common.collect.Maps;
 import java.util.List;
 import java.util.Map;
-import org.apache.hadoop.hive.metastore.TableType;
 
 public class HiveTablePropertiesMetadata extends BasePropertiesMetadata {
   public static final String COMMENT = "comment";
@@ -82,6 +81,14 @@ public class HiveTablePropertiesMetadata extends BasePropertiesMetadata {
 
   private static final String REGEX_SERDE_CLASS = "org.apache.hadoop.hive.serde2.RegexSerDe";
 
+  public enum TableType {
+    MANAGED_TABLE,
+    EXTERNAL_TABLE,
+    VIRTUAL_VIEW,
+    INDEX_TABLE,
+    VIRTUAL_INDEX,
+  }
+
   enum StorageFormat {
     SEQUENCEFILE(
         SEQUENCEFILE_INPUT_FORMAT_CLASS, SEQUENCEFILE_OUTPUT_FORMAT_CLASS, LAZY_SIMPLE_SERDE_CLASS),

catalogs/catalog-jdbc-mysql/build.gradle.kts

@@ -24,11 +24,11 @@ dependencies {
 tasks {
   val copyDepends by registering(Copy::class) {
     from(configurations.runtimeClasspath)
-    into("build/libs")
+    into("build/libs_all")
   }
   val copyCatalogLibs by registering(Copy::class) {
     dependsOn(copyDepends, "build")
-    from("build/libs")
+    from("build/libs_all", "build/libs")
     into("$rootDir/distribution/package/catalogs/jdbc-mysql/libs")
   }
 

catalogs/catalog-jdbc-postgresql/build.gradle.kts

@@ -25,11 +25,11 @@ dependencies {
 tasks {
   val copyDepends by registering(Copy::class) {
     from(configurations.runtimeClasspath)
-    into("build/libs")
+    into("build/libs_all")
   }
   val copyCatalogLibs by registering(Copy::class) {
     dependsOn(copyDepends, "build")
-    from("build/libs")
+    from("build/libs_all", "build/libs")
     into("$rootDir/distribution/package/catalogs/jdbc-postgresql/libs")
   }
 

catalogs/catalog-lakehouse-iceberg/build.gradle.kts

@@ -79,11 +79,11 @@ dependencies {
 tasks {
   val copyDepends by registering(Copy::class) {
     from(configurations.runtimeClasspath)
-    into("build/libs")
+    into("build/libs_all")
   }
   val copyCatalogLibs by registering(Copy::class) {
     dependsOn(copyDepends, "build")
-    from("build/libs")
+    from("build/libs_all", "build/libs")
     into("$rootDir/distribution/package/catalogs/lakehouse-iceberg/libs")
   }
 

core/src/main/java/com/datastrato/gravitino/catalog/PropertyEntry.java

@@ -304,4 +304,19 @@ public static <T extends Enum<T>> PropertyEntry<T> enumImmutablePropertyEntry(
     return enumPropertyEntry(
         name, description, required, true, javaType, defaultValue, hidden, reserved);
   }
+
+  @Override
+  public String toString() {
+    final StringBuilder sb = new StringBuilder("PropertyEntry{");
+    sb.append("name='").append(name).append('\'');
+    sb.append(", description='").append(description).append('\'');
+    sb.append(", required=").append(required);
+    sb.append(", immutable=").append(immutable);
+    sb.append(", javaType=").append(javaType);
+    sb.append(", defaultValue=").append(defaultValue);
+    sb.append(", hidden=").append(hidden);
+    sb.append(", reserved=").append(reserved);
+    sb.append('}');
+    return sb.toString();
+  }
 }

integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/hive/CatalogHiveIT.java

@@ -19,7 +19,7 @@
 import static com.datastrato.gravitino.catalog.hive.HiveTablePropertiesMetadata.TEXT_INPUT_FORMAT_CLASS;
 import static com.datastrato.gravitino.catalog.hive.HiveTablePropertiesMetadata.TOTAL_SIZE;
 import static com.datastrato.gravitino.catalog.hive.HiveTablePropertiesMetadata.TRANSIENT_LAST_DDL_TIME;
-import static org.apache.hadoop.hive.metastore.TableType.EXTERNAL_TABLE;
+import static com.datastrato.gravitino.catalog.hive.HiveTablePropertiesMetadata.TableType.EXTERNAL_TABLE;
 import static org.apache.hadoop.hive.serde.serdeConstants.DATE_TYPE_NAME;
 import static org.apache.hadoop.hive.serde.serdeConstants.INT_TYPE_NAME;
 import static org.apache.hadoop.hive.serde.serdeConstants.STRING_TYPE_NAME;
@@ -34,6 +34,7 @@
 import com.datastrato.gravitino.catalog.hive.HiveClientPool;
 import com.datastrato.gravitino.catalog.hive.HiveSchemaPropertiesMetadata;
 import com.datastrato.gravitino.catalog.hive.HiveTablePropertiesMetadata;
+import com.datastrato.gravitino.catalog.hive.HiveTablePropertiesMetadata.TableType;
 import com.datastrato.gravitino.client.GravitinoMetaLake;
 import com.datastrato.gravitino.dto.rel.ColumnDTO;
 import com.datastrato.gravitino.dto.rel.expressions.FieldReferenceDTO;
@@ -75,7 +76,6 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.TableType;
 import org.apache.hadoop.hive.metastore.api.Database;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
 import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;

settings.gradle.kts

@@ -9,7 +9,7 @@ plugins {
 rootProject.name = "gravitino"
 
 include("api", "common", "core", "meta", "server", "integration-test", "server-common")
-include("catalogs:catalog-hive", "catalogs:catalog-lakehouse-iceberg", "catalogs:catalog-jdbc-common", "catalogs:catalog-jdbc-mysql", "catalogs:catalog-jdbc-postgresql")
+include("catalogs:bundled-catalog", "catalogs:catalog-hive", "catalogs:catalog-lakehouse-iceberg", "catalogs:catalog-jdbc-common", "catalogs:catalog-jdbc-mysql", "catalogs:catalog-jdbc-postgresql")
 include("clients:client-java", "clients:client-java-runtime")
 include("trino-connector")
 include("web")

trino-connector/build.gradle.kts

@@ -14,6 +14,7 @@ repositories {
 
 dependencies {
   implementation(project(":clients:client-java-runtime", configuration = "shadow"))
+  implementation(project(":catalogs:bundled-catalog", configuration = "shadow"))
   implementation(libs.jackson.databind)
   implementation(libs.jackson.annotations)
   implementation(libs.guava)

trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/hive/HiveTablePropertyConverter.java

@@ -6,13 +6,15 @@
 
 import com.datastrato.gravitino.shaded.org.apache.commons.collections4.bidimap.TreeBidiMap;
 import com.datastrato.gravitino.trino.connector.catalog.PropertyConverter;
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableMap;
 
 public class HiveTablePropertyConverter extends PropertyConverter {
 
   // Trino property key does not allow upper case character and '-', so we need to map it to
   // Gravitino
-  private static final TreeBidiMap<String, String> TRINO_KEY_TO_GRAVITINO_KEY =
+  @VisibleForTesting
+  static final TreeBidiMap<String, String> TRINO_KEY_TO_GRAVITINO_KEY =
       new TreeBidiMap<>(
           new ImmutableMap.Builder<String, String>()
               .put("format", "format")