[#1449] feat(catalogs): Introudce new module bundled-catalog for query engine.

catalogs/catalog-common/build.gradle.kts

@@ -0,0 +1,61 @@
+/*
+ * Copyright 2024 Datastrato Pvt Ltd.
+ * This software is licensed under the Apache License version 2.
+ */
+
+import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
+
+plugins {
+  id("java")
+  alias(libs.plugins.shadow)
+  `maven-publish`
+}
+
+dependencies {
+  implementation(project(":core"))
+  implementation(project(":catalogs:catalog-hive"))
+  implementation(project(":catalogs:catalog-lakehouse-iceberg"))
+  implementation(project(":catalogs:catalog-jdbc-mysql"))
+  implementation(project(":catalogs:catalog-jdbc-postgresql"))
+}
+
+tasks.test {
+  useJUnitPlatform()
+}
+
+tasks.withType<ShadowJar>(ShadowJar::class.java) {
+  isZip64 = true
+  configurations = listOf(project.configurations.runtimeClasspath.get())
+  archiveClassifier.set("")
+
+  dependencies {
+    exclude("org.*")
+    exclude("javax.*")
+  }
+
+  exclude("**/package-info.class")
+  exclude("**/*.properties")
+  exclude("**/*.html")
+  exclude("org/**")
+  exclude("META-INF/**")
+  exclude("module-info.class")
+  exclude("com/google/**")
+  exclude("com/fasterxml/**")
+  exclude("javax/**")
+  exclude("schema/**")
+  exclude("fr/**")
+  exclude("google/**")
+  exclude("groovy/**")
+  exclude("images/**")
+  exclude("**/*.conf")
+  exclude("**/*.so")
+  exclude("**/*.sxd")
+  exclude("**/*.ddl")
+
+  minimize()
+}
+
+tasks.jar {
+  dependsOn(tasks.named("shadowJar"))
+  archiveClassifier.set("empty")
+}

catalogs/catalog-common/src/main/java/com/datastrato/catalog/common/ClassProvider.java

@@ -0,0 +1,67 @@
+/*
+ * Copyright 2023 Datastrato Pvt Ltd.
+ * This software is licensed under the Apache License version 2.
+ */
+
+package com.datastrato.catalog.common;
+
+import com.datastrato.gravitino.catalog.BasePropertiesMetadata;
+import com.datastrato.gravitino.catalog.PropertiesMetadata;
+import com.datastrato.gravitino.catalog.PropertyEntry;
+import com.datastrato.gravitino.catalog.hive.HiveCatalogPropertiesMeta;
+import com.datastrato.gravitino.catalog.hive.HiveSchemaPropertiesMetadata;
+import com.datastrato.gravitino.catalog.hive.HiveTablePropertiesMetadata;
+import com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergCatalogPropertiesMetadata;
+import com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergSchemaPropertiesMetadata;
+import com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergTablePropertiesMetadata;
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * {@link ClassProvider} holds the set of classes that are required for the query engine to use
+ * Gravitino, currently, it mainly focuses on the classes that are required for the property
+ * metadata.
+ */
+public class ClassProvider {
+
+  private static final Set<Class<?>> BASE_CLASS =
+      new HashSet<Class<?>>() {
+        {
+          add(BasePropertiesMetadata.class);
+          add(PropertyEntry.class);
+          add(PropertiesMetadata.class);
+        }
+      };
+
+  private static final Set<Class<?>> HIVE_NEED_CLASS =
+      new HashSet<Class<?>>() {
+        {
+          add(HiveTablePropertiesMetadata.class);
+          add(HiveSchemaPropertiesMetadata.class);
+          add(HiveCatalogPropertiesMeta.class);
+        }
+      };
+
+  private static final Set<Class<?>> ICEBERG_NEED_CLASS =
+      new HashSet<Class<?>>() {
+        {
+          add(IcebergTablePropertiesMetadata.class);
+          add(IcebergSchemaPropertiesMetadata.class);
+          add(IcebergCatalogPropertiesMetadata.class);
+        }
+      };
+
+  private static final Set<Class<?>> MYSQL_NEED_CLASS =
+      new HashSet<Class<?>>() {
+        {
+          // TODO
+        }
+      };
+
+  private static final Set<Class<?>> PG_NEED_CLASS =
+      new HashSet<Class<?>>() {
+        {
+          // TODO
+        }
+      };
+}

catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveCatalogOperations.java

@@ -15,6 +15,7 @@
 import com.datastrato.gravitino.Namespace;
 import com.datastrato.gravitino.catalog.CatalogOperations;
 import com.datastrato.gravitino.catalog.PropertiesMetadata;
+import com.datastrato.gravitino.catalog.hive.HiveTablePropertiesMetadata.TableType;
 import com.datastrato.gravitino.catalog.hive.converter.ToHiveType;
 import com.datastrato.gravitino.exceptions.NoSuchCatalogException;
 import com.datastrato.gravitino.exceptions.NoSuchSchemaException;
@@ -51,7 +52,6 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.metastore.TableType;
 import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
 import org.apache.hadoop.hive.metastore.api.Database;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;

catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveSchemaPropertiesMetadata.java

@@ -15,7 +15,7 @@
 
 public class HiveSchemaPropertiesMetadata extends BasePropertiesMetadata {
   public static final String LOCATION = "location";
-  private static final Map<String, PropertyEntry<?>> propertiesMetadata;
+  public static final Map<String, PropertyEntry<?>> propertiesMetadata;
 
   static {
     List<PropertyEntry<?>> propertyEntries =
@@ -32,7 +32,7 @@ public class HiveSchemaPropertiesMetadata extends BasePropertiesMetadata {
   }
 
   @Override
-  protected Map<String, PropertyEntry<?>> specificPropertyEntries() {
+  public Map<String, PropertyEntry<?>> specificPropertyEntries() {
     return propertiesMetadata;
   }
 }

catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveTable.java

@@ -14,10 +14,11 @@
 import static com.datastrato.gravitino.catalog.hive.HiveTablePropertiesMetadata.SERDE_NAME;
 import static com.datastrato.gravitino.catalog.hive.HiveTablePropertiesMetadata.SERDE_PARAMETER_PREFIX;
 import static com.datastrato.gravitino.catalog.hive.HiveTablePropertiesMetadata.TABLE_TYPE;
+import static com.datastrato.gravitino.catalog.hive.HiveTablePropertiesMetadata.TableType.EXTERNAL_TABLE;
+import static com.datastrato.gravitino.catalog.hive.HiveTablePropertiesMetadata.TableType.MANAGED_TABLE;
 import static com.datastrato.gravitino.rel.expressions.transforms.Transforms.identity;
-import static org.apache.hadoop.hive.metastore.TableType.EXTERNAL_TABLE;
-import static org.apache.hadoop.hive.metastore.TableType.MANAGED_TABLE;
 
+import com.datastrato.gravitino.catalog.hive.HiveTablePropertiesMetadata.TableType;
 import com.datastrato.gravitino.catalog.hive.converter.FromHiveType;
 import com.datastrato.gravitino.catalog.hive.converter.ToHiveType;
 import com.datastrato.gravitino.catalog.rel.BaseTable;
@@ -45,7 +46,6 @@
 import lombok.ToString;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.lang3.ArrayUtils;
-import org.apache.hadoop.hive.metastore.TableType;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
 import org.apache.hadoop.hive.metastore.api.Order;
 import org.apache.hadoop.hive.metastore.api.SerDeInfo;

catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveTablePropertiesMetadata.java

@@ -9,7 +9,7 @@
 import static com.datastrato.gravitino.catalog.PropertyEntry.stringImmutablePropertyEntry;
 import static com.datastrato.gravitino.catalog.PropertyEntry.stringReservedPropertyEntry;
 import static com.datastrato.gravitino.catalog.hive.HiveTablePropertiesMetadata.StorageFormat.TEXTFILE;
-import static org.apache.hadoop.hive.metastore.TableType.MANAGED_TABLE;
+import static com.datastrato.gravitino.catalog.hive.HiveTablePropertiesMetadata.TableType.MANAGED_TABLE;
 
 import com.datastrato.gravitino.catalog.BasePropertiesMetadata;
 import com.datastrato.gravitino.catalog.PropertyEntry;
@@ -19,7 +19,6 @@
 import com.google.common.collect.Maps;
 import java.util.List;
 import java.util.Map;
-import org.apache.hadoop.hive.metastore.TableType;
 
 public class HiveTablePropertiesMetadata extends BasePropertiesMetadata {
   public static final String COMMENT = "comment";
@@ -82,6 +81,14 @@ public class HiveTablePropertiesMetadata extends BasePropertiesMetadata {
 
   private static final String REGEX_SERDE_CLASS = "org.apache.hadoop.hive.serde2.RegexSerDe";
 
+  public enum TableType {
+    MANAGED_TABLE,
+    EXTERNAL_TABLE,
+    VIRTUAL_VIEW,
+    INDEX_TABLE,
+    VIRTUAL_INDEX,
+  }
+
   enum StorageFormat {
     SEQUENCEFILE(
         SEQUENCEFILE_INPUT_FORMAT_CLASS, SEQUENCEFILE_OUTPUT_FORMAT_CLASS, LAZY_SIMPLE_SERDE_CLASS),
@@ -120,7 +127,7 @@ public String getSerde() {
     }
   }
 
-  private static final Map<String, PropertyEntry<?>> propertiesMetadata;
+  public static final Map<String, PropertyEntry<?>> propertiesMetadata;
 
   static {
     List<PropertyEntry<?>> propertyEntries =
@@ -182,7 +189,7 @@ public String getSerde() {
   }
 
   @Override
-  protected Map<String, PropertyEntry<?>> specificPropertyEntries() {
+  public Map<String, PropertyEntry<?>> specificPropertyEntries() {
     return propertiesMetadata;
   }
 }

catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergCatalogPropertiesMetadata.java

@@ -30,7 +30,7 @@ public class IcebergCatalogPropertiesMetadata extends BaseCatalogPropertiesMetad
   public static final String WAREHOUSE = "warehouse";
   public static final String URI = "uri";
 
-  private static final Map<String, PropertyEntry<?>> PROPERTIES_METADATA;
+  public static final Map<String, PropertyEntry<?>> PROPERTIES_METADATA;
 
   // Map that maintains the mapping of keys in Gravitino to that in Iceberg, for example, users
   // will only need to set the configuration 'catalog-backend' in Gravitino and Gravitino will

catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergSchemaPropertiesMetadata.java

@@ -16,7 +16,7 @@
 public class IcebergSchemaPropertiesMetadata extends BasePropertiesMetadata {
 
   public static final String COMMENT = "comment";
-  private static final Map<String, PropertyEntry<?>> PROPERTIES_METADATA;
+  public static final Map<String, PropertyEntry<?>> PROPERTIES_METADATA;
 
   static {
     List<PropertyEntry<?>> propertyEntries =

catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergTablePropertiesMetadata.java

@@ -22,7 +22,7 @@ public class IcebergTablePropertiesMetadata extends BasePropertiesMetadata {
   public static final String SORT_ORDER = "sort-order";
   public static final String IDENTIFIER_FIELDS = "identifier-fields";
 
-  private static final Map<String, PropertyEntry<?>> PROPERTIES_METADATA;
+  public static final Map<String, PropertyEntry<?>> PROPERTIES_METADATA;
 
   static {
     List<PropertyEntry<?>> propertyEntries =

core/src/main/java/com/datastrato/gravitino/catalog/PropertyEntry.java

@@ -304,4 +304,19 @@ public static <T extends Enum<T>> PropertyEntry<T> enumImmutablePropertyEntry(
     return enumPropertyEntry(
         name, description, required, true, javaType, defaultValue, hidden, reserved);
   }
+
+  @Override
+  public String toString() {
+    final StringBuilder sb = new StringBuilder("PropertyEntry{");
+    sb.append("name='").append(name).append('\'');
+    sb.append(", description='").append(description).append('\'');
+    sb.append(", required=").append(required);
+    sb.append(", immutable=").append(immutable);
+    sb.append(", javaType=").append(javaType);
+    sb.append(", defaultValue=").append(defaultValue);
+    sb.append(", hidden=").append(hidden);
+    sb.append(", reserved=").append(reserved);
+    sb.append('}');
+    return sb.toString();
+  }
 }

core/src/test/java/com/datastrato/gravitino/TestCatalogOperations.java

@@ -293,7 +293,7 @@ public PropertiesMetadata catalogPropertiesMetadata() throws UnsupportedOperatio
     if (config.containsKey("mock")) {
       return new BasePropertiesMetadata() {
         @Override
-        protected Map<String, PropertyEntry<?>> specificPropertyEntries() {
+        public Map<String, PropertyEntry<?>> specificPropertyEntries() {
           return ImmutableMap.<String, PropertyEntry<?>>builder()
               .put(
                   "key1",
@@ -345,7 +345,7 @@ protected Map<String, PropertyEntry<?>> specificPropertyEntries() {
     } else if (config.containsKey("hive")) {
       return new BasePropertiesMetadata() {
         @Override
-        protected Map<String, PropertyEntry<?>> specificPropertyEntries() {
+        public Map<String, PropertyEntry<?>> specificPropertyEntries() {
           return ImmutableMap.<String, PropertyEntry<?>>builder()
               .put(
                   "hive.metastore.uris",

integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/hive/CatalogHiveIT.java

@@ -19,7 +19,7 @@
 import static com.datastrato.gravitino.catalog.hive.HiveTablePropertiesMetadata.TEXT_INPUT_FORMAT_CLASS;
 import static com.datastrato.gravitino.catalog.hive.HiveTablePropertiesMetadata.TOTAL_SIZE;
 import static com.datastrato.gravitino.catalog.hive.HiveTablePropertiesMetadata.TRANSIENT_LAST_DDL_TIME;
-import static org.apache.hadoop.hive.metastore.TableType.EXTERNAL_TABLE;
+import static com.datastrato.gravitino.catalog.hive.HiveTablePropertiesMetadata.TableType.EXTERNAL_TABLE;
 import static org.apache.hadoop.hive.serde.serdeConstants.DATE_TYPE_NAME;
 import static org.apache.hadoop.hive.serde.serdeConstants.INT_TYPE_NAME;
 import static org.apache.hadoop.hive.serde.serdeConstants.STRING_TYPE_NAME;
@@ -34,6 +34,7 @@
 import com.datastrato.gravitino.catalog.hive.HiveClientPool;
 import com.datastrato.gravitino.catalog.hive.HiveSchemaPropertiesMetadata;
 import com.datastrato.gravitino.catalog.hive.HiveTablePropertiesMetadata;
+import com.datastrato.gravitino.catalog.hive.HiveTablePropertiesMetadata.TableType;
 import com.datastrato.gravitino.client.GravitinoMetaLake;
 import com.datastrato.gravitino.dto.rel.ColumnDTO;
 import com.datastrato.gravitino.dto.rel.expressions.FieldReferenceDTO;
@@ -75,7 +76,6 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.TableType;
 import org.apache.hadoop.hive.metastore.api.Database;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
 import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;

settings.gradle.kts

@@ -14,3 +14,4 @@ include("clients:client-java", "clients:client-java-runtime")
 include("trino-connector")
 include("web")
 include("docs")
+include("catalogs:catalog-common")

trino-connector/build.gradle.kts

@@ -14,6 +14,7 @@ repositories {
 
 dependencies {
   implementation(project(":clients:client-java-runtime", configuration = "shadow"))
+  implementation(project(":catalogs:catalog-common", configuration = "shadow"))
   implementation(libs.jackson.databind)
   implementation(libs.jackson.annotations)
   implementation(libs.guava)

trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/mysql/MySQLDataTypeTransformer.java

@@ -5,16 +5,24 @@
 
 package com.datastrato.gravitino.trino.connector.catalog.jdbc.mysql;
 
+import com.datastrato.gravitino.catalog.PropertyEntry;
+import com.datastrato.gravitino.catalog.hive.HiveTablePropertiesMetadata;
 import com.datastrato.gravitino.rel.types.Type;
 import com.datastrato.gravitino.rel.types.Type.Name;
 import com.datastrato.gravitino.rel.types.Types;
 import com.datastrato.gravitino.trino.connector.util.GeneralDataTypeTransformer;
+import java.util.Map;
 
 /** Type transformer between MySQL and Trino */
 public class MySQLDataTypeTransformer extends GeneralDataTypeTransformer {
 
   @Override
   public Type getGravitinoType(io.trino.spi.type.Type type) {
+    for (Map.Entry<String, PropertyEntry<?>> entryEntry :
+        HiveTablePropertiesMetadata.propertiesMetadata.entrySet()) {
+      System.out.println(entryEntry.getKey() + " " + entryEntry.getValue());
+    }
+
     Type gravitinoType = super.getGravitinoType(type);
     if (gravitinoType.name() == Name.VARCHAR || gravitinoType.name() == Name.FIXEDCHAR) {
       return Types.StringType.get();

trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/hive/TestHiveCatalogPropertyConverter.java

@@ -5,6 +5,8 @@
 
 package com.datastrato.gravitino.trino.connector.catalog.hive;
 
+import com.datastrato.gravitino.catalog.PropertyEntry;
+import com.datastrato.gravitino.catalog.hive.HiveTablePropertiesMetadata;
 import java.util.Map;
 import org.testcontainers.shaded.com.google.common.collect.ImmutableMap;
 import org.testng.Assert;
@@ -28,4 +30,13 @@ public void testConverter() {
     Assert.assertEquals(re.get("hive.compression-codec"), "ZSTD");
     Assert.assertEquals(re.get("hive.unknown-key"), null);
   }
+
+  @Test
+  // To test whether we can load property metadata from HiveTablePropertiesMetadata successfully.
+  public void testPropertyMetadata() {
+    for (Map.Entry<String, PropertyEntry<?>> entryEntry :
+        HiveTablePropertiesMetadata.propertiesMetadata.entrySet()) {
+      System.out.println(entryEntry.getKey() + " " + entryEntry.getValue());
+    }
+  }
 }