diff --git a/catalogs/bundled-catalog/build.gradle.kts b/catalogs/bundled-catalog/build.gradle.kts index 76beebf4c8e..2c8b03f4ed4 100644 --- a/catalogs/bundled-catalog/build.gradle.kts +++ b/catalogs/bundled-catalog/build.gradle.kts @@ -16,6 +16,7 @@ dependencies { implementation(project(":catalogs:catalog-lakehouse-iceberg")) implementation(project(":catalogs:catalog-jdbc-mysql")) implementation(project(":catalogs:catalog-jdbc-postgresql")) + implementation(libs.slf4j.api) } tasks.withType(ShadowJar::class.java) { diff --git a/catalogs/bundled-catalog/src/main/java/com/datastrato/catalog/property/PropertyConverter.java b/catalogs/bundled-catalog/src/main/java/com/datastrato/catalog/property/PropertyConverter.java new file mode 100644 index 00000000000..38ee3a3b94b --- /dev/null +++ b/catalogs/bundled-catalog/src/main/java/com/datastrato/catalog/property/PropertyConverter.java @@ -0,0 +1,94 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.catalog.property; + +import com.datastrato.gravitino.catalog.PropertyEntry; +import com.datastrato.gravitino.catalog.hive.HiveTablePropertiesMetadata; +import java.util.HashMap; +import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Transforming between gravitino schema/table/column property and engine property. */ +public abstract class PropertyConverter { + + private static final Logger LOG = LoggerFactory.getLogger(PropertyConverter.class); + /** + * Mapping that maps engine properties to Gravitino properties. It will return a map that holds + * the mapping between engine and gravitino properties. + * + * @return a map that holds the mapping from engine to Gravitino properties. + */ + public abstract Map engineToGravitinoMapping(); + + /** + * Get the property metadata for the catalog. for more please see {@link + * HiveTablePropertiesMetadata#propertyEntries()} or {@link + * com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergTablePropertiesMetadata#propertyEntries()} + */ + public abstract Map> gravitinoPropertyMeta(); + + Map revsereMap(Map map) { + Map res = new HashMap<>(); + for (Map.Entry entry : map.entrySet()) { + res.put(entry.getValue(), entry.getKey()); + } + + return res; + } + + /** Convert Gravitino properties to engine properties. */ + public Map gravitinoToEngineProperties(Map gravitinoProperties) { + Map engineProperties = new HashMap<>(); + Map gravitinoToEngineMapping = revsereMap(engineToGravitinoMapping()); + for (Map.Entry entry : gravitinoProperties.entrySet()) { + String engineKey = gravitinoToEngineMapping.get(entry.getKey()); + if (engineKey != null) { + engineProperties.put(engineKey, entry.getValue()); + } else { + LOG.info("Property {} is not supported by engine", entry.getKey()); + } + } + return engineProperties; + } + + /** + * Convert engine properties to Gravitino properties. + * + *

If different engine has different behavior about error handling, you can override this + * method. + */ + public Map engineToGravitinoProperties(Map engineProperties) { + Map gravitinoProperties = new HashMap<>(); + Map engineToGravitinoMapping = engineToGravitinoMapping(); + + Map> propertyEntryMap = gravitinoPropertyMeta(); + for (Map.Entry entry : engineProperties.entrySet()) { + String gravitinoKey = engineToGravitinoMapping.get(entry.getKey()); + if (gravitinoKey != null) { + PropertyEntry propertyEntry = propertyEntryMap.get(gravitinoKey); + if (propertyEntry != null) { + // Check value is valid. + propertyEntry.decode(entry.getValue().toString()); + } + gravitinoProperties.put(gravitinoKey, entry.getValue()); + } else { + LOG.info("Property {} is not supported by Gravitino", entry.getKey()); + } + } + + // Check the required properties. + for (Map.Entry> propertyEntry : propertyEntryMap.entrySet()) { + if (propertyEntry.getValue().isRequired() + && !gravitinoProperties.containsKey(propertyEntry.getKey())) { + throw new IllegalArgumentException( + "Property " + propertyEntry.getKey() + " is required, you should "); + } + } + + return gravitinoProperties; + } +} diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/trino/TrinoConnectorIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/trino/TrinoConnectorIT.java index b545c2e8358..e3e040a999c 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/trino/TrinoConnectorIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/trino/TrinoConnectorIT.java @@ -388,6 +388,14 @@ void testHiveTableCreatedByTrino() { Assertions.assertEquals( "hdfs://localhost:9000/user/hive/warehouse/hive_schema.db/hive_table", table.properties().get("location")); + Assertions.assertEquals("MANAGED_TABLE", table.properties().get("table-type")); + Assertions.assertEquals( + "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", table.properties().get("serde-lib")); + Assertions.assertEquals( + "org.apache.hadoop.mapred.TextInputFormat", table.properties().get("input-format")); + Assertions.assertEquals( + "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", + table.properties().get("output-format")); } @Test @@ -728,6 +736,9 @@ void testHiveTableCreatedByGravitino() throws InterruptedException { Assertions.assertTrue( data.contains( "location = 'hdfs://localhost:9000/user/hive/warehouse/hive_schema.db/hive_table'")); + Assertions.assertTrue( + data.contains("input_format = 'org.apache.hadoop.hive.ql.io.orc.OrcInputFormat'")); + Assertions.assertTrue(data.contains("serde_lib = 'org.apache.hadoop.hive.ql.io.orc.OrcSerde'")); } @Test diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/PropertyConverter.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/PropertyConverter.java index 848e73601a1..476b2cfc913 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/PropertyConverter.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/PropertyConverter.java @@ -5,6 +5,7 @@ package com.datastrato.gravitino.trino.connector.catalog; +import com.datastrato.gravitino.catalog.PropertyEntry; import com.datastrato.gravitino.shaded.org.apache.commons.collections4.bidimap.TreeBidiMap; import java.util.HashMap; import java.util.Map; @@ -12,6 +13,7 @@ import org.slf4j.LoggerFactory; /** Transforming between gravitino schema/table/column property and Trino property. */ +@Deprecated public abstract class PropertyConverter { private static final Logger LOG = LoggerFactory.getLogger(PropertyConverter.class); @@ -22,12 +24,12 @@ public abstract class PropertyConverter { * * @return a map that holds the mapping from Trino to Gravitino properties. */ - public abstract TreeBidiMap trinoPropertyKeyToGravitino(); + public abstract TreeBidiMap engineToGravitino(); /** Convert Trino properties to Gravitino properties. */ - public Map toTrinoProperties(Map properties) { + public Map fromGravitinoProperties(Map properties) { Map trinoProperties = new HashMap<>(); - Map gravitinoToTrinoMapping = trinoPropertyKeyToGravitino().inverseBidiMap(); + Map gravitinoToTrinoMapping = engineToGravitino().inverseBidiMap(); for (Map.Entry entry : properties.entrySet()) { String trinoKey = gravitinoToTrinoMapping.get(entry.getKey()); if (trinoKey != null) { @@ -42,7 +44,7 @@ public Map toTrinoProperties(Map properties) { /** Convert Gravitino properties to Trino properties. */ public Map toGravitinoProperties(Map properties) { Map gravitinoProperties = new HashMap<>(); - Map trinoToGravitinoMapping = trinoPropertyKeyToGravitino(); + Map trinoToGravitinoMapping = engineToGravitino(); for (Map.Entry entry : properties.entrySet()) { String gravitinoKey = trinoToGravitinoMapping.get(entry.getKey()); if (gravitinoKey != null) { @@ -53,4 +55,6 @@ public Map toGravitinoProperties(Map properties) } return gravitinoProperties; } + + public abstract Map> gravitinoPropertyMeta(); } diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/hive/HiveCatalogPropertyConverter.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/hive/HiveCatalogPropertyConverter.java index f5c32ea7a6a..71812d5f647 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/hive/HiveCatalogPropertyConverter.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/hive/HiveCatalogPropertyConverter.java @@ -5,9 +5,11 @@ package com.datastrato.gravitino.trino.connector.catalog.hive; +import com.datastrato.catalog.property.PropertyConverter; +import com.datastrato.gravitino.catalog.PropertyEntry; import com.datastrato.gravitino.shaded.org.apache.commons.collections4.bidimap.TreeBidiMap; -import com.datastrato.gravitino.trino.connector.catalog.PropertyConverter; import com.google.common.collect.ImmutableMap; +import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -23,7 +25,7 @@ public class HiveCatalogPropertyConverter extends PropertyConverter { .put("hive.storage-format", "hive.storage-format") .put("hive.compression-codec", "hive.compression-codec") .put("hive.config.resources", "hive.config.resources") - .put("hive.recursive-directories", "hive.ignore-absent-partitions") + .put("hive.recursive-directories", "hive.recursive-directories") .put("hive.ignore-absent-partitions", "hive.ignore-absent-partitions") .put("hive.force-local-scheduling", "hive.force-local-scheduling") .put("hive.respect-table-format", "hive.respect-table-format") @@ -41,7 +43,12 @@ public class HiveCatalogPropertyConverter extends PropertyConverter { .build()); @Override - public TreeBidiMap trinoPropertyKeyToGravitino() { + public TreeBidiMap engineToGravitinoMapping() { return TRINO_KEY_TO_GRAVITINO_KEY; } + + @Override + public Map> gravitinoPropertyMeta() { + return ImmutableMap.of(); + } } diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/hive/HiveConnectorAdapter.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/hive/HiveConnectorAdapter.java index 2d5f729d997..433e7df6002 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/hive/HiveConnectorAdapter.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/hive/HiveConnectorAdapter.java @@ -4,10 +4,10 @@ */ package com.datastrato.gravitino.trino.connector.catalog.hive; +import com.datastrato.catalog.property.PropertyConverter; import com.datastrato.gravitino.trino.connector.catalog.CatalogConnectorAdapter; import com.datastrato.gravitino.trino.connector.catalog.CatalogConnectorMetadataAdapter; import com.datastrato.gravitino.trino.connector.catalog.HasPropertyMeta; -import com.datastrato.gravitino.trino.connector.catalog.PropertyConverter; import com.datastrato.gravitino.trino.connector.metadata.GravitinoCatalog; import com.google.common.collect.Maps; import io.trino.spi.session.PropertyMetadata; @@ -38,7 +38,8 @@ public Map buildInternalConnectorConfig(GravitinoCatalog catalog Map properties = new HashMap<>(); properties.put("hive.metastore.uri", catalog.getRequiredProperty("metastore.uris")); - Map trinoProperty = catalogConverter.toTrinoProperties(catalog.getProperties()); + Map trinoProperty = + catalogConverter.gravitinoToEngineProperties(catalog.getProperties()); // Trino only supports properties that define in catalogPropertyMeta, the name of entries in // catalogPropertyMeta is in the format of "catalogName_propertyName", so we need to replace diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/hive/HiveMetadataAdapter.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/hive/HiveMetadataAdapter.java index 060837e4d9d..a5be430f364 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/hive/HiveMetadataAdapter.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/hive/HiveMetadataAdapter.java @@ -4,8 +4,8 @@ */ package com.datastrato.gravitino.trino.connector.catalog.hive; +import com.datastrato.catalog.property.PropertyConverter; import com.datastrato.gravitino.trino.connector.catalog.CatalogConnectorMetadataAdapter; -import com.datastrato.gravitino.trino.connector.catalog.PropertyConverter; import io.trino.spi.session.PropertyMetadata; import java.util.List; import java.util.Map; @@ -27,25 +27,25 @@ public HiveMetadataAdapter( @Override public Map toTrinoTableProperties(Map properties) { - Map objectMap = tableConverter.toTrinoProperties(properties); + Map objectMap = tableConverter.gravitinoToEngineProperties(properties); return super.toTrinoTableProperties(objectMap); } @Override public Map toTrinoSchemaProperties(Map properties) { - Map objectMap = schemaConverter.toTrinoProperties(properties); + Map objectMap = schemaConverter.gravitinoToEngineProperties(properties); return super.toTrinoSchemaProperties(objectMap); } @Override public Map toGravitinoTableProperties(Map properties) { - Map stringMap = tableConverter.toGravitinoProperties(properties); + Map stringMap = tableConverter.engineToGravitinoProperties(properties); return super.toGravitinoTableProperties(stringMap); } @Override public Map toGravitinoSchemaProperties(Map properties) { - Map stringMap = schemaConverter.toGravitinoProperties(properties); + Map stringMap = schemaConverter.engineToGravitinoProperties(properties); return super.toGravitinoSchemaProperties(stringMap); } } diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/hive/HiveSchemaPropertyConverter.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/hive/HiveSchemaPropertyConverter.java index d76c2a9b240..83540731877 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/hive/HiveSchemaPropertyConverter.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/hive/HiveSchemaPropertyConverter.java @@ -4,20 +4,33 @@ */ package com.datastrato.gravitino.trino.connector.catalog.hive; +import com.datastrato.catalog.property.PropertyConverter; +import com.datastrato.gravitino.catalog.BasePropertiesMetadata; +import com.datastrato.gravitino.catalog.PropertyEntry; +import com.datastrato.gravitino.catalog.hive.HiveSchemaPropertiesMetadata; import com.datastrato.gravitino.shaded.org.apache.commons.collections4.bidimap.TreeBidiMap; -import com.datastrato.gravitino.trino.connector.catalog.PropertyConverter; import com.google.common.collect.ImmutableMap; +import java.util.Map; public class HiveSchemaPropertyConverter extends PropertyConverter { + private final BasePropertiesMetadata hiveSchemaPropertiesMetadata = + new HiveSchemaPropertiesMetadata(); // Trino property key does not allow upper case character and '-', so we need to map it to // Gravitino private static final TreeBidiMap TRINO_KEY_TO_GRAVITINO_KEY = new TreeBidiMap<>( - new ImmutableMap.Builder().put("location", "location").build()); + new ImmutableMap.Builder() + .put("location", HiveSchemaPropertiesMetadata.LOCATION) + .build()); @Override - public TreeBidiMap trinoPropertyKeyToGravitino() { + public TreeBidiMap engineToGravitinoMapping() { return TRINO_KEY_TO_GRAVITINO_KEY; } + + @Override + public Map> gravitinoPropertyMeta() { + return hiveSchemaPropertiesMetadata.propertyEntries(); + } } diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/hive/HiveTablePropertyConverter.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/hive/HiveTablePropertyConverter.java index cd9a070420c..48271e874ab 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/hive/HiveTablePropertyConverter.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/hive/HiveTablePropertyConverter.java @@ -4,33 +4,43 @@ */ package com.datastrato.gravitino.trino.connector.catalog.hive; +import com.datastrato.catalog.property.PropertyConverter; +import com.datastrato.gravitino.catalog.BasePropertiesMetadata; +import com.datastrato.gravitino.catalog.PropertyEntry; +import com.datastrato.gravitino.catalog.hive.HiveTablePropertiesMetadata; import com.datastrato.gravitino.shaded.org.apache.commons.collections4.bidimap.TreeBidiMap; -import com.datastrato.gravitino.trino.connector.catalog.PropertyConverter; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableMap; +import java.util.Map; public class HiveTablePropertyConverter extends PropertyConverter { - + private final BasePropertiesMetadata hiveTablePropertiesMetadata = + new HiveTablePropertiesMetadata(); // Trino property key does not allow upper case character and '-', so we need to map it to // Gravitino @VisibleForTesting static final TreeBidiMap TRINO_KEY_TO_GRAVITINO_KEY = new TreeBidiMap<>( new ImmutableMap.Builder() - .put("format", "format") - .put("total_size", "totalSize") - .put("num_files", "numFiles") - .put("external", "external") - .put("location", "location") - .put("table_type", "table-type") - .put("input_format", "input-format") - .put("output_format", "output-format") - .put("serde_lib", "serde-lib") - .put("serde_name", "serde-name") + .put("format", HiveTablePropertiesMetadata.FORMAT) + .put("total_size", HiveTablePropertiesMetadata.TOTAL_SIZE) + .put("num_files", HiveTablePropertiesMetadata.NUM_FILES) + .put("external", HiveTablePropertiesMetadata.EXTERNAL) + .put("location", HiveTablePropertiesMetadata.LOCATION) + .put("table_type", HiveTablePropertiesMetadata.TABLE_TYPE) + .put("input_format", HiveTablePropertiesMetadata.INPUT_FORMAT) + .put("output_format", HiveTablePropertiesMetadata.OUTPUT_FORMAT) + .put("serde_lib", HiveTablePropertiesMetadata.SERDE_LIB) + .put("serde_name", HiveTablePropertiesMetadata.SERDE_NAME) .build()); @Override - public TreeBidiMap trinoPropertyKeyToGravitino() { + public TreeBidiMap engineToGravitinoMapping() { return TRINO_KEY_TO_GRAVITINO_KEY; } + + @Override + public Map> gravitinoPropertyMeta() { + return hiveTablePropertiesMetadata.propertyEntries(); + } } diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/IcebergCatalogPropertyConverter.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/IcebergCatalogPropertyConverter.java index 723e9fc7fd2..e7bd2ed3a26 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/IcebergCatalogPropertyConverter.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/IcebergCatalogPropertyConverter.java @@ -5,9 +5,10 @@ package com.datastrato.gravitino.trino.connector.catalog.iceberg; +import com.datastrato.catalog.property.PropertyConverter; +import com.datastrato.gravitino.catalog.PropertyEntry; import com.datastrato.gravitino.shaded.org.apache.commons.collections4.bidimap.TreeBidiMap; import com.datastrato.gravitino.trino.connector.GravitinoErrorCode; -import com.datastrato.gravitino.trino.connector.catalog.PropertyConverter; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Sets; import io.trino.spi.TrinoException; @@ -26,12 +27,12 @@ public class IcebergCatalogPropertyConverter extends PropertyConverter { private static final Set HIVE_BACKEND_REQUIRED_PROPERTIES = Set.of("uri"); @Override - public TreeBidiMap trinoPropertyKeyToGravitino() { + public TreeBidiMap engineToGravitinoMapping() { return TRINO_ICEBERG_TO_GRAVITINO_ICEBERG; } @Override - public Map toTrinoProperties(Map properties) { + public Map gravitinoToEngineProperties(Map properties) { String backend = properties.get("catalog-backend"); switch (backend) { case "hive": @@ -80,4 +81,9 @@ private Map buildJDBCBackendProperties(Map prope return jdbcProperties; } + + @Override + public Map> gravitinoPropertyMeta() { + return ImmutableMap.of(); + } } diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/IcebergConnectorAdapter.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/IcebergConnectorAdapter.java index d5e4926a85a..ac1fb696bd1 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/IcebergConnectorAdapter.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/IcebergConnectorAdapter.java @@ -6,9 +6,9 @@ import static java.util.Collections.emptyList; +import com.datastrato.catalog.property.PropertyConverter; import com.datastrato.gravitino.trino.connector.catalog.CatalogConnectorAdapter; import com.datastrato.gravitino.trino.connector.catalog.CatalogConnectorMetadataAdapter; -import com.datastrato.gravitino.trino.connector.catalog.PropertyConverter; import com.datastrato.gravitino.trino.connector.metadata.GravitinoCatalog; import io.trino.spi.session.PropertyMetadata; import java.util.HashMap; @@ -32,7 +32,8 @@ public Map buildInternalConnectorConfig(GravitinoCatalog catalog config.put("catalogHandle", catalog.getName() + ":normal:default"); config.put("connectorName", "iceberg"); - Map properties = catalogConverter.toTrinoProperties(catalog.getProperties()); + Map properties = + catalogConverter.gravitinoToEngineProperties(catalog.getProperties()); config.put("properties", properties); return config; } diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/IcebergMetadataAdapter.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/IcebergMetadataAdapter.java index 39fdc8ee930..3c7ecfc15c9 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/IcebergMetadataAdapter.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/IcebergMetadataAdapter.java @@ -4,8 +4,8 @@ */ package com.datastrato.gravitino.trino.connector.catalog.iceberg; +import com.datastrato.catalog.property.PropertyConverter; import com.datastrato.gravitino.trino.connector.catalog.CatalogConnectorMetadataAdapter; -import com.datastrato.gravitino.trino.connector.catalog.PropertyConverter; import io.trino.spi.session.PropertyMetadata; import java.util.List; import java.util.Map; @@ -28,25 +28,25 @@ public IcebergMetadataAdapter( @Override public Map toTrinoTableProperties(Map properties) { - Map objectMap = tableConverter.toTrinoProperties(properties); + Map objectMap = tableConverter.gravitinoToEngineProperties(properties); return super.toTrinoTableProperties(objectMap); } @Override public Map toTrinoSchemaProperties(Map properties) { - Map objectMap = schemaConverter.toTrinoProperties(properties); + Map objectMap = schemaConverter.gravitinoToEngineProperties(properties); return super.toTrinoSchemaProperties(objectMap); } @Override public Map toGravitinoTableProperties(Map properties) { - Map stringMap = tableConverter.toGravitinoProperties(properties); + Map stringMap = tableConverter.engineToGravitinoProperties(properties); return super.toGravitinoTableProperties(stringMap); } @Override public Map toGravitinoSchemaProperties(Map properties) { - Map stringMap = schemaConverter.toGravitinoProperties(properties); + Map stringMap = schemaConverter.engineToGravitinoProperties(properties); return super.toGravitinoSchemaProperties(stringMap); } } diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/IcebergSchemaPropertyConverter.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/IcebergSchemaPropertyConverter.java index 519ec9572b4..a1dee3ee757 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/IcebergSchemaPropertyConverter.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/IcebergSchemaPropertyConverter.java @@ -5,20 +5,31 @@ package com.datastrato.gravitino.trino.connector.catalog.iceberg; +import com.datastrato.catalog.property.PropertyConverter; +import com.datastrato.gravitino.catalog.BasePropertiesMetadata; +import com.datastrato.gravitino.catalog.PropertyEntry; +import com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergSchemaPropertiesMetadata; import com.datastrato.gravitino.shaded.org.apache.commons.collections4.bidimap.TreeBidiMap; -import com.datastrato.gravitino.trino.connector.catalog.PropertyConverter; import com.google.common.collect.ImmutableMap; +import java.util.Map; public class IcebergSchemaPropertyConverter extends PropertyConverter { + private final BasePropertiesMetadata icebergSchemaPropertiesMetadata = + new IcebergSchemaPropertiesMetadata(); // TODO (yuqi) add more properties private static final TreeBidiMap TRINO_KEY_TO_GRAVITINO_KEY = new TreeBidiMap<>(new ImmutableMap.Builder().build()); @Override - public TreeBidiMap trinoPropertyKeyToGravitino() { + public TreeBidiMap engineToGravitinoMapping() { // Note: As the properties for Iceberg table loaded from Gravitino are always empty currently, // no matter what the mapping is, the properties will be empty. return TRINO_KEY_TO_GRAVITINO_KEY; } + + @Override + public Map> gravitinoPropertyMeta() { + return icebergSchemaPropertiesMetadata.propertyEntries(); + } } diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/IcebergTablePropertyConverter.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/IcebergTablePropertyConverter.java index 78f3229151c..00c525a6552 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/IcebergTablePropertyConverter.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/IcebergTablePropertyConverter.java @@ -5,22 +5,34 @@ package com.datastrato.gravitino.trino.connector.catalog.iceberg; +import com.datastrato.catalog.property.PropertyConverter; +import com.datastrato.gravitino.catalog.BasePropertiesMetadata; +import com.datastrato.gravitino.catalog.PropertyEntry; +import com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergTablePropertiesMetadata; import com.datastrato.gravitino.shaded.org.apache.commons.collections4.bidimap.TreeBidiMap; -import com.datastrato.gravitino.trino.connector.catalog.PropertyConverter; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableMap; +import java.util.Map; public class IcebergTablePropertyConverter extends PropertyConverter { + private final BasePropertiesMetadata icebergTablePropertiesMetadata = + new IcebergTablePropertiesMetadata(); + // TODO (yuqi) add more properties @VisibleForTesting static final TreeBidiMap TRINO_KEY_TO_GRAVITINO_KEY = new TreeBidiMap<>(new ImmutableMap.Builder().build()); @Override - public TreeBidiMap trinoPropertyKeyToGravitino() { + public TreeBidiMap engineToGravitinoMapping() { // Note: As the properties for Iceberg table loaded from Gravitino are always empty currently, // no matter what the mapping is, the properties will be empty. return TRINO_KEY_TO_GRAVITINO_KEY; } + + @Override + public Map> gravitinoPropertyMeta() { + return icebergTablePropertiesMetadata.propertyEntries(); + } } diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/JDBCCatalogPropertyConverter.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/JDBCCatalogPropertyConverter.java index c9c228ba47a..6f296fb362b 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/JDBCCatalogPropertyConverter.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/JDBCCatalogPropertyConverter.java @@ -5,8 +5,9 @@ package com.datastrato.gravitino.trino.connector.catalog.jdbc; +import com.datastrato.catalog.property.PropertyConverter; +import com.datastrato.gravitino.catalog.PropertyEntry; import com.datastrato.gravitino.shaded.org.apache.commons.collections4.bidimap.TreeBidiMap; -import com.datastrato.gravitino.trino.connector.catalog.PropertyConverter; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Sets; import java.util.Map; @@ -32,13 +33,13 @@ public class JDBCCatalogPropertyConverter extends PropertyConverter { JDBC_CONNECTION_PASSWORD_KEY, JDBC_CONNECTION_USER_KEY, JDBC_CONNECTION_PASSWORD_KEY); @Override - public TreeBidiMap trinoPropertyKeyToGravitino() { + public TreeBidiMap engineToGravitinoMapping() { return TRINO_KEY_TO_GRAVITINO_KEY; } @Override - public Map toTrinoProperties(Map properties) { - Map trinoProperties = super.toTrinoProperties(properties); + public Map gravitinoToEngineProperties(Map properties) { + Map trinoProperties = super.gravitinoToEngineProperties(properties); for (String requiredProperty : REQUIRED_PROPERTIES) { if (!trinoProperties.containsKey(requiredProperty)) { throw new IllegalArgumentException("Missing required property: " + requiredProperty); @@ -47,4 +48,9 @@ public Map toTrinoProperties(Map properties) { return trinoProperties; } + + @Override + public Map> gravitinoPropertyMeta() { + return ImmutableMap.of(); + } } diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/mysql/MySQLConnectorAdapter.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/mysql/MySQLConnectorAdapter.java index 80e9773f4a1..78d08ec0ec1 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/mysql/MySQLConnectorAdapter.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/mysql/MySQLConnectorAdapter.java @@ -6,9 +6,9 @@ import static java.util.Collections.emptyList; +import com.datastrato.catalog.property.PropertyConverter; import com.datastrato.gravitino.trino.connector.catalog.CatalogConnectorAdapter; import com.datastrato.gravitino.trino.connector.catalog.CatalogConnectorMetadataAdapter; -import com.datastrato.gravitino.trino.connector.catalog.PropertyConverter; import com.datastrato.gravitino.trino.connector.catalog.jdbc.JDBCCatalogPropertyConverter; import com.datastrato.gravitino.trino.connector.metadata.GravitinoCatalog; import java.util.HashMap; @@ -28,7 +28,8 @@ public Map buildInternalConnectorConfig(GravitinoCatalog catalog config.put("catalogHandle", catalog.getName() + ":normal:default"); config.put("connectorName", "mysql"); - Map properties = catalogConverter.toTrinoProperties(catalog.getProperties()); + Map properties = + catalogConverter.gravitinoToEngineProperties(catalog.getProperties()); config.put("properties", properties); return config; } diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/postgresql/PostgreSQLConnectorAdapter.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/postgresql/PostgreSQLConnectorAdapter.java index 63c0620682a..09be47bd513 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/postgresql/PostgreSQLConnectorAdapter.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/postgresql/PostgreSQLConnectorAdapter.java @@ -6,9 +6,9 @@ import static java.util.Collections.emptyList; +import com.datastrato.catalog.property.PropertyConverter; import com.datastrato.gravitino.trino.connector.catalog.CatalogConnectorAdapter; import com.datastrato.gravitino.trino.connector.catalog.CatalogConnectorMetadataAdapter; -import com.datastrato.gravitino.trino.connector.catalog.PropertyConverter; import com.datastrato.gravitino.trino.connector.catalog.jdbc.JDBCCatalogPropertyConverter; import com.datastrato.gravitino.trino.connector.metadata.GravitinoCatalog; import java.util.HashMap; @@ -28,7 +28,8 @@ public Map buildInternalConnectorConfig(GravitinoCatalog catalog config.put("catalogHandle", catalog.getName() + ":normal:default"); config.put("connectorName", "postgresql"); - Map properties = catalogConverter.toTrinoProperties(catalog.getProperties()); + Map properties = + catalogConverter.gravitinoToEngineProperties(catalog.getProperties()); config.put("properties", properties); return config; } diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/hive/TestHiveCatalogPropertyConverter.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/hive/TestHiveCatalogPropertyConverter.java index f0bdd6e61f7..d07f7661f9d 100644 --- a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/hive/TestHiveCatalogPropertyConverter.java +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/hive/TestHiveCatalogPropertyConverter.java @@ -26,7 +26,7 @@ public void testConverter() { .put("hive.unknown-key", "1") .build(); - Map re = hiveCatalogPropertyConverter.toTrinoProperties(map); + Map re = hiveCatalogPropertyConverter.gravitinoToEngineProperties(map); Assert.assertEquals(re.get("hive.immutable-partitions"), "true"); Assert.assertEquals(re.get("hive.compression-codec"), "ZSTD"); Assert.assertEquals(re.get("hive.unknown-key"), null); diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/TestIcebergCatalogPropertyConverter.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/TestIcebergCatalogPropertyConverter.java index a42d75802f9..ce96b6abc7f 100644 --- a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/TestIcebergCatalogPropertyConverter.java +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/TestIcebergCatalogPropertyConverter.java @@ -5,8 +5,8 @@ package com.datastrato.gravitino.trino.connector.catalog.iceberg; +import com.datastrato.catalog.property.PropertyConverter; import com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergTablePropertiesMetadata; -import com.datastrato.gravitino.trino.connector.catalog.PropertyConverter; import com.google.common.collect.Sets; import io.trino.spi.TrinoException; import java.util.Map; @@ -28,7 +28,7 @@ public void testHiveBackendProperty() { .put("catalog-backend", "hive") .build(); Map hiveBackendConfig = - propertyConverter.toTrinoProperties(gravitinoIcebergConfig); + propertyConverter.gravitinoToEngineProperties(gravitinoIcebergConfig); Assert.assertEquals(hiveBackendConfig.get("iceberg.catalog.type"), "hive_metastore"); Assert.assertEquals(hiveBackendConfig.get("hive.metastore.uri"), "1111"); @@ -36,7 +36,7 @@ public void testHiveBackendProperty() { Map wrongMap = Maps.newHashMap(gravitinoIcebergConfig); wrongMap.remove("uri"); - Assertions.assertThatThrownBy(() -> propertyConverter.toTrinoProperties(wrongMap)) + Assertions.assertThatThrownBy(() -> propertyConverter.gravitinoToEngineProperties(wrongMap)) .isInstanceOf(TrinoException.class) .hasMessageContaining("Missing required property for Hive backend: [uri]"); } @@ -54,7 +54,7 @@ public void testJDBCBackendProperty() { .put("other-key", "other") .build(); Map hiveBackendConfig = - propertyConverter.toTrinoProperties(gravitinoIcebergConfig); + propertyConverter.gravitinoToEngineProperties(gravitinoIcebergConfig); // Test all properties are converted Assert.assertEquals( @@ -70,7 +70,7 @@ public void testJDBCBackendProperty() { Map wrongMap = Maps.newHashMap(gravitinoIcebergConfig); wrongMap.remove("jdbc-driver"); - Assertions.assertThatThrownBy(() -> propertyConverter.toTrinoProperties(wrongMap)) + Assertions.assertThatThrownBy(() -> propertyConverter.gravitinoToEngineProperties(wrongMap)) .isInstanceOf(TrinoException.class) .hasMessageContaining("Missing required property for JDBC backend: [jdbc-driver]"); } diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/TestJDBCCatalogPropertyConverter.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/TestJDBCCatalogPropertyConverter.java index ef42823dfe9..b8c3f81ae4c 100644 --- a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/TestJDBCCatalogPropertyConverter.java +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/TestJDBCCatalogPropertyConverter.java @@ -9,7 +9,7 @@ import static com.datastrato.gravitino.trino.connector.catalog.jdbc.JDBCCatalogPropertyConverter.JDBC_CONNECTION_URL_KEY; import static com.datastrato.gravitino.trino.connector.catalog.jdbc.JDBCCatalogPropertyConverter.JDBC_CONNECTION_USER_KEY; -import com.datastrato.gravitino.trino.connector.catalog.PropertyConverter; +import com.datastrato.catalog.property.PropertyConverter; import java.util.Map; import org.testcontainers.shaded.com.google.common.collect.ImmutableMap; import org.testng.Assert; @@ -26,7 +26,8 @@ public void testTrinoPropertyKeyToGravitino() { "jdbc-user", "root", "jdbc-password", "root"); - Map trinoProperties = propertyConverter.toTrinoProperties(gravitinoProperties); + Map trinoProperties = + propertyConverter.gravitinoToEngineProperties(gravitinoProperties); Assert.assertEquals( trinoProperties.get(JDBC_CONNECTION_URL_KEY), "jdbc:mysql://localhost:3306"); Assert.assertEquals(trinoProperties.get(JDBC_CONNECTION_USER_KEY), "root"); @@ -40,7 +41,7 @@ public void testTrinoPropertyKeyToGravitino() { Assert.assertThrows( IllegalArgumentException.class, () -> { - propertyConverter.toTrinoProperties(gravitinoPropertiesWithoutPassword); + propertyConverter.gravitinoToEngineProperties(gravitinoPropertiesWithoutPassword); }); } }