[Hotfix][Connector-V2][Hive] Fix hive unknownhost (#4141)

* [Hotfix][Connector-V2][Hive] Fix the bug that can not load hdfs-site.xml
apache · Feb 16, 2023 · f1a1dfe · f1a1dfe
1 parent 32e1f91
commit f1a1dfe
Show file tree

Hide file tree

Showing 3 changed files with 18 additions and 2 deletions.
diff --git a/docs/en/connector-v2/sink/Hive.md b/docs/en/connector-v2/sink/Hive.md
@@ -35,6 +35,7 @@ By default, we use 2PC commit to ensure `exactly-once`
 | table_name           | string | yes      | -             |
 | metastore_uri        | string | yes      | -             |
 | compress_codec       | string | no       | none          |
+| hdfs_site_path       | string | no       | -             |
 | kerberos_principal   | string | no       | -             |
 | kerberos_keytab_path | string | no       | -             |
 | common-options       |        | no       | -             |
@@ -47,6 +48,10 @@ Target Hive table name eg: db1.table1
 
 Hive metastore uri
 
+### hdfs_site_path [string]
+
+The path of `hdfs-site.xml`, used to load ha configuration of namenodes
+
 ### kerberos_principal [string]
 
 The principal of kerberos

diff --git a/docs/en/connector-v2/source/Hive.md b/docs/en/connector-v2/source/Hive.md
@@ -39,6 +39,7 @@ Read all the data in a split in a pollNext call. What splits are read will be sa
 | metastore_uri        | string | yes      | -             |
 | kerberos_principal   | string | no       | -             |
 | kerberos_keytab_path | string | no       | -             |
+| hdfs_site_path       | string | no       | -             |
 | read_partitions      | list   | no       | -             |
 | common-options       |        | no       | -             |
 
@@ -50,6 +51,10 @@ Target Hive table name eg: db1.table1
 
 Hive metastore uri
 
+### hdfs_site_path [string]
+
+The path of `hdfs-site.xml`, used to load ha configuration of namenodes
+
 ### read_partitions [list]
 
 The target partitions that user want to read from hive table, if user does not set this parameter, it will read all the data from hive table.

diff --git a/...ctor-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/sink/HiveSink.java b/...ctor-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/sink/HiveSink.java
@@ -53,6 +53,7 @@
 import java.util.Optional;
 import java.util.stream.Collectors;
 
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY;
 import static org.apache.seatunnel.connectors.seatunnel.file.config.BaseSinkConfig.FIELD_DELIMITER;
 import static org.apache.seatunnel.connectors.seatunnel.file.config.BaseSinkConfig.FILE_FORMAT;
 import static org.apache.seatunnel.connectors.seatunnel.file.config.BaseSinkConfig.FILE_NAME_EXPRESSION;
@@ -157,9 +158,13 @@ public void prepare(Config pluginConfig) throws PrepareFailException {
         try {
             URI uri = new URI(hdfsLocation);
             String path = uri.getPath();
-            pluginConfig =
-                    pluginConfig.withValue(FILE_PATH.key(), ConfigValueFactory.fromAnyRef(path));
             hadoopConf = new HadoopConf(hdfsLocation.replace(path, ""));
+            pluginConfig =
+                    pluginConfig
+                            .withValue(FILE_PATH.key(), ConfigValueFactory.fromAnyRef(path))
+                            .withValue(
+                                    FS_DEFAULT_NAME_KEY,
+                                    ConfigValueFactory.fromAnyRef(hadoopConf.getHdfsNameKey()));
         } catch (URISyntaxException e) {
             String errorMsg =
                     String.format(
@@ -170,6 +175,7 @@ public void prepare(Config pluginConfig) throws PrepareFailException {
                     HiveConnectorErrorCode.GET_HDFS_NAMENODE_HOST_FAILED, errorMsg, e);
         }
         this.pluginConfig = pluginConfig;
+        super.prepare(pluginConfig);
     }
 
     @Override