From d321ae08e665819bb0c530bcf5f614fa90689082 Mon Sep 17 00:00:00 2001 From: "halo.kim" Date: Mon, 8 Apr 2024 11:07:24 +0900 Subject: [PATCH] [Improve][Connector-V2][Iceberg] Support for S3 in hadoop catalog # modify doc iceberg source --- docs/en/connector-v2/source/Iceberg.md | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/docs/en/connector-v2/source/Iceberg.md b/docs/en/connector-v2/source/Iceberg.md index 8fb296467a88..d5281016cf51 100644 --- a/docs/en/connector-v2/source/Iceberg.md +++ b/docs/en/connector-v2/source/Iceberg.md @@ -41,7 +41,7 @@ Source connector for Apache Iceberg. It can support batch and stream mode. ## Database Dependency -> In order to be compatible with different versions of Hadoop and Hive, the scope of hive-exec in the project pom file are provided, so if you use the Flink engine, first you may need to add the following Jar packages to /lib directory, if you are using the Spark engine and integrated with Hadoop, then you do not need to add the following Jar packages. +> In order to be compatible with different versions of Hadoop and Hive, the scope of hive-exec in the project pom file are provided, so if you use the Flink engine, first you may need to add the following Jar packages to /lib directory, if you are using the Spark engine and integrated with Hadoop, then you do not need to add the following Jar packages. If you are using the hadoop s3 catalog, you need to add the hadoop-aws,aws-java-sdk jars for your Flink and Spark engine versions. (Additional locations: /lib, /jars) ``` hive-exec-xxx.jar @@ -141,6 +141,30 @@ sink { } ``` +### Hadoop S3 Catalog: + +```hocon +source { + iceberg { + catalog_name = "seatunnel" + iceberg.catalog.config={ + "type"="hadoop" + "warehouse"="s3a://your_bucket/spark/warehouse/" + } + hadoop.config={ + "fs.s3a.aws.credentials.provider" = "org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider" + "fs.s3a.endpoint" = "s3.cn-north-1.amazonaws.com.cn" + "fs.s3a.access.key" = "xxxxxxxxxxxxxxxxx" + "fs.s3a.secret.key" = "xxxxxxxxxxxxxxxxx" + "fs.defaultFS" = "s3a://your_bucket" + } + namespace = "your_iceberg_database" + table = "your_iceberg_table" + result_table_name = "iceberg_test" + } +} +``` + ### Hive Catalog: ```hocon