diff --git a/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergConstants.java b/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergConstants.java index 6e4aae37ab1..d07a6c48707 100644 --- a/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergConstants.java +++ b/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergConstants.java @@ -21,6 +21,7 @@ public class IcebergConstants { // Iceberg catalog properties constants public static final String CATALOG_BACKEND = "catalog-backend"; + public static final String CATALOG_BACKEND_IMPL = "catalog-backend-impl"; public static final String GRAVITINO_JDBC_USER = "jdbc-user"; public static final String ICEBERG_JDBC_USER = "jdbc.user"; diff --git a/docs/iceberg-rest-service.md b/docs/iceberg-rest-service.md index c1a4a8111d6..1753cc9496e 100644 --- a/docs/iceberg-rest-service.md +++ b/docs/iceberg-rest-service.md @@ -162,6 +162,14 @@ If you have a JDBC Iceberg catalog prior, you must set `catalog-backend-name` to You must download the corresponding JDBC driver to the `iceberg-rest-server/libs` directory. ::: +#### Custom backend configuration +| Configuration item | Description | Default value | Required | Since Version | +|------------------------------------------------|---------------------------------------------------------------------------------------------------------------------|------------------|----------|---------------| +| `gravitino.iceberg-rest.catalog-backend` | The Catalog backend of the Gravitino Iceberg REST catalog service. Use the value **`custom`** for a Custom catalog. | `memory` | Yes | 0.2.0 | +| `gravitino.iceberg-rest.catalog-backend-impl` | The fully-qualified class name of a custom catalog implementation, only worked if `catalog-backend` is `custom`. | (none) | No | 0.7.0 | + +If you want to use a custom Iceberg Catalog as `catalog-backend`, you can add a corresponding jar file to the classpath and load a custom Iceberg Catalog implementation by specifying the `catalog-backend-impl` property. + #### Multi catalog support The Gravitino Iceberg REST server supports multiple catalogs and offers a configuration-based catalog management system. diff --git a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergCatalogBackend.java b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergCatalogBackend.java index 63fb07605c9..4cdedc826e7 100644 --- a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergCatalogBackend.java +++ b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergCatalogBackend.java @@ -22,5 +22,6 @@ public enum IcebergCatalogBackend { HIVE, JDBC, MEMORY, - REST + REST, + CUSTOM } diff --git a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergConfig.java b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergConfig.java index b75fc88d672..bc1e47a0652 100644 --- a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergConfig.java +++ b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergConfig.java @@ -43,6 +43,14 @@ public class IcebergConfig extends Config implements OverwriteDefaultConfig { .stringConf() .createWithDefault("memory"); + public static final ConfigEntry CATALOG_BACKEND_IMPL = + new ConfigBuilder(IcebergConstants.CATALOG_BACKEND_IMPL) + .doc( + "The fully-qualified class name of a custom catalog implementation, only worked if `catalog-backend` is `custom`") + .version(ConfigConstants.VERSION_0_7_0) + .stringConf() + .create(); + public static final ConfigEntry CATALOG_WAREHOUSE = new ConfigBuilder(IcebergConstants.WAREHOUSE) .doc("Warehouse directory of catalog") diff --git a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/utils/IcebergCatalogUtil.java b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/utils/IcebergCatalogUtil.java index f6e8b4e36b6..dfbc574be29 100644 --- a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/utils/IcebergCatalogUtil.java +++ b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/utils/IcebergCatalogUtil.java @@ -39,6 +39,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.iceberg.CatalogProperties; +import org.apache.iceberg.CatalogUtil; import org.apache.iceberg.catalog.Catalog; import org.apache.iceberg.hive.HiveCatalog; import org.apache.iceberg.inmemory.InMemoryCatalog; @@ -149,6 +150,16 @@ private static Catalog loadRestCatalog(IcebergConfig icebergConfig) { return restCatalog; } + private static Catalog loadCustomCatalog(IcebergConfig icebergConfig) { + String customCatalogName = icebergConfig.getCatalogBackendName("custom"); + String className = icebergConfig.get(IcebergConfig.CATALOG_BACKEND_IMPL); + return CatalogUtil.loadCatalog( + className, + customCatalogName, + icebergConfig.getIcebergCatalogProperties(), + new HdfsConfiguration()); + } + @VisibleForTesting static Catalog loadCatalogBackend(String catalogType) { return loadCatalogBackend( @@ -168,6 +179,8 @@ public static Catalog loadCatalogBackend( return loadJdbcCatalog(icebergConfig); case REST: return loadRestCatalog(icebergConfig); + case CUSTOM: + return loadCustomCatalog(icebergConfig); default: throw new RuntimeException( catalogBackend diff --git a/iceberg/iceberg-common/src/test/java/org/apache/gravitino/iceberg/common/utils/CustomCatalogForTest.java b/iceberg/iceberg-common/src/test/java/org/apache/gravitino/iceberg/common/utils/CustomCatalogForTest.java new file mode 100644 index 00000000000..6c6492b7d6a --- /dev/null +++ b/iceberg/iceberg-common/src/test/java/org/apache/gravitino/iceberg/common/utils/CustomCatalogForTest.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.iceberg.common.utils; + +import java.util.List; +import org.apache.iceberg.Table; +import org.apache.iceberg.catalog.Catalog; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; + +public class CustomCatalogForTest implements Catalog { + @Override + public List listTables(Namespace namespace) { + return null; + } + + @Override + public boolean dropTable(TableIdentifier identifier, boolean purge) { + return false; + } + + @Override + public void renameTable(TableIdentifier from, TableIdentifier to) {} + + @Override + public Table loadTable(TableIdentifier identifier) { + return null; + } +} diff --git a/iceberg/iceberg-common/src/test/java/org/apache/gravitino/iceberg/common/utils/TestIcebergCatalogUtil.java b/iceberg/iceberg-common/src/test/java/org/apache/gravitino/iceberg/common/utils/TestIcebergCatalogUtil.java index 289858ae377..29ec0cc1507 100644 --- a/iceberg/iceberg-common/src/test/java/org/apache/gravitino/iceberg/common/utils/TestIcebergCatalogUtil.java +++ b/iceberg/iceberg-common/src/test/java/org/apache/gravitino/iceberg/common/utils/TestIcebergCatalogUtil.java @@ -78,4 +78,38 @@ void testLoadCatalog() { IcebergCatalogUtil.loadCatalogBackend("other"); }); } + + @Test + void testValidLoadCustomCatalog() { + Catalog catalog; + Map config = new HashMap<>(); + + config.put( + "catalog-backend-impl", "org.apache.gravitino.iceberg.common.utils.CustomCatalogForTest"); + catalog = + IcebergCatalogUtil.loadCatalogBackend( + IcebergCatalogBackend.valueOf("CUSTOM"), new IcebergConfig(config)); + Assertions.assertTrue(catalog instanceof CustomCatalogForTest); + } + + @Test + void testInvalidLoadCustomCatalog() { + Assertions.assertThrowsExactly( + NullPointerException.class, + () -> + IcebergCatalogUtil.loadCatalogBackend( + IcebergCatalogBackend.valueOf("CUSTOM"), new IcebergConfig(new HashMap<>()))); + + Assertions.assertThrowsExactly( + IllegalArgumentException.class, + () -> + IcebergCatalogUtil.loadCatalogBackend( + IcebergCatalogBackend.valueOf("CUSTOM"), + new IcebergConfig( + new HashMap() { + { + put("catalog-backend-impl", "org.apache."); + } + }))); + } }