[Feature][Connector][Elasticsearch] Support https protocol

* support https protocol * compatible with opensearch
apache · Jan 30, 2023 · 23df106 · 23df106
1 parent 6a4a850
commit 23df106
Show file tree

Hide file tree

Showing 14 changed files with 565 additions and 67 deletions.
diff --git a/docs/en/connector-v2/sink/Elasticsearch.md b/docs/en/connector-v2/sink/Elasticsearch.md
@@ -19,18 +19,24 @@ Engine Supported
 
 ## Options
 
-| name            | type   | required | default value | 
-|-----------------|--------|----------|---------------|
-| hosts           | array  | yes      | -             |
-| index           | string | yes      | -             |
-| index_type      | string | no       |               |
-| primary_keys    | list   | no       |               |
-| key_delimiter   | string | no       | `_`           |
-| username        | string | no       |               |
-| password        | string | no       |               | 
-| max_retry_count | int    | no       | 3             |
-| max_batch_size  | int    | no       | 10            |
-| common-options  |        | no       | -             |
+| name                    | type    | required | default value | 
+|-------------------------|---------|----------|---------------|
+| hosts                   | array   | yes      | -             |
+| index                   | string  | yes      | -             |
+| index_type              | string  | no       |               |
+| primary_keys            | list    | no       |               |
+| key_delimiter           | string  | no       | `_`           |
+| username                | string  | no       |               |
+| password                | string  | no       |               | 
+| max_retry_count         | int     | no       | 3             |
+| max_batch_size          | int     | no       | 10            |
+| tls_verify_certificate  | boolean | no       | true          |
+| tls_verify_hostnames    | boolean | no       | true          |
+| tls_keystore_path       | string  | no       | -             |
+| tls_keystore_password   | string  | no       | -             |
+| tls_truststore_path     | string  | no       | -             |
+| tls_truststore_password | string  | no       | -             |
+| common-options          |         | no       | -             |
 
 
 ### hosts [array]
@@ -61,6 +67,30 @@ one bulk request max try size
 ### max_batch_size [int]
 batch bulk doc max size
 
+### tls_verify_certificate [boolean]
+
+Enable certificates validation for HTTPS endpoints
+
+### tls_verify_hostname [boolean]
+
+Enable hostname validation for HTTPS endpoints
+
+### tls_keystore_path [string]
+
+The path to the PEM or JKS key store. This file must be readable by the operating system user running SeaTunnel.
+
+### tls_keystore_password [string]
+
+The key password for the key store specified
+
+### tls_truststore_path [string]
+
+The path to PEM or JKS trust store. This file must be readable by the operating system user running SeaTunnel.
+
+### tls_truststore_password [string]
+
+The key password for the trust store specified
+
 ### common options
 
 Sink plugin common parameters, please refer to [Sink Common Options](common-options.md) for details
@@ -92,6 +122,49 @@ sink {
 }
 ```
 
+SSL (Disable certificates validation)
+
+```hocon
+sink {
+    Elasticsearch {
+        hosts = ["https://localhost:9200"]
+        username = "elastic"
+        password = "elasticsearch"
+        
+        tls_verify_certificate = false
+    }
+}
+```
+
+SSL (Disable hostname validation)
+
+```hocon
+sink {
+    Elasticsearch {
+        hosts = ["https://localhost:9200"]
+        username = "elastic"
+        password = "elasticsearch"
+        
+        tls_verify_hostname = false
+    }
+}
+```
+
+SSL (Enable certificates validation)
+
+```hocon
+sink {
+    Elasticsearch {
+        hosts = ["https://localhost:9200"]
+        username = "elastic"
+        password = "elasticsearch"
+        
+        tls_keystore_path = "${your elasticsearch home}/config/certs/http.p12"
+        tls_keystore_password = "${your password}"
+    }
+}
+```
+
 ## Changelog
 
 ### 2.2.0-beta 2022-09-26

diff --git a/docs/en/connector-v2/source/Elasticsearch.md b/docs/en/connector-v2/source/Elasticsearch.md
@@ -19,16 +19,23 @@ support version >= 2.x and < 8.x.
 
 ## Options
 
-| name        | type   | required | default value | 
-|-------------|--------|----------|---------------|
-| hosts       | array  | yes      | -             |
-| username    | string | no       | -             |
-| password    | string | no       | -             |
-| index       | string | yes      | -             |
-| source      | array  | no       | -             |
-| scroll_time | string | no       | 1m            |
-| scroll_size | int    | no       | 100           |
-| schema      |        | no       | -             |
+| name                    | type    | required | default value | 
+|-------------------------|---------|----------|---------------|
+| hosts                   | array   | yes      | -             |
+| username                | string  | no       | -             |
+| password                | string  | no       | -             |
+| index                   | string  | yes      | -             |
+| source                  | array   | no       | -             |
+| scroll_time             | string  | no       | 1m            |
+| scroll_size             | int     | no       | 100           |
+| schema                  |         | no       | -             |
+| tls_verify_certificate  | boolean | no       | true          |
+| tls_verify_hostnames    | boolean | no       | true          |
+| tls_keystore_path       | string  | no       | -             |
+| tls_keystore_password   | string  | no       | -             |
+| tls_truststore_path     | string  | no       | -             |
+| tls_truststore_password | string  | no       | -             |
+| common-options          |         | no       | -             |
 
 
 
@@ -59,8 +66,38 @@ Maximum number of hits to be returned with each Elasticsearch scroll request.
 The structure of the data, including field names and field types.
 If you don't config schema, you must config `source`.
 
+### tls_verify_certificate [boolean]
+
+Enable certificates validation for HTTPS endpoints
+
+### tls_verify_hostname [boolean]
+
+Enable hostname validation for HTTPS endpoints
+
+### tls_keystore_path [string]
+
+The path to the PEM or JKS key store. This file must be readable by the operating system user running SeaTunnel.
+
+### tls_keystore_password [string]
+
+The key password for the key store specified
+
+### tls_truststore_path [string]
+
+The path to PEM or JKS trust store. This file must be readable by the operating system user running SeaTunnel.
+
+### tls_truststore_password [string]
+
+The key password for the trust store specified
+
+### common options
+
+Source plugin common parameters, please refer to [Source Common Options](common-options.md) for details
+
 ## Examples
+
 simple
+
 ```hocon
 Elasticsearch {
     hosts = ["localhost:9200"]
@@ -69,6 +106,7 @@ Elasticsearch {
 }
 ```
 complex
+
 ```hocon
 Elasticsearch {
     hosts = ["elasticsearch:9200"]
@@ -94,6 +132,49 @@ Elasticsearch {
 }
 ```
 
+SSL (Disable certificates validation)
+
+```hocon
+source {
+    Elasticsearch {
+        hosts = ["https://localhost:9200"]
+        username = "elastic"
+        password = "elasticsearch"
+        
+        tls_verify_certificate = false
+    }
+}
+```
+
+SSL (Disable hostname validation)
+
+```hocon
+source {
+    Elasticsearch {
+        hosts = ["https://localhost:9200"]
+        username = "elastic"
+        password = "elasticsearch"
+        
+        tls_verify_hostname = false
+    }
+}
+```
+
+SSL (Enable certificates validation)
+
+```hocon
+source {
+    Elasticsearch {
+        hosts = ["https://localhost:9200"]
+        username = "elastic"
+        password = "elasticsearch"
+        
+        tls_keystore_path = "${your elasticsearch home}/config/certs/http.p12"
+        tls_keystore_password = "${your password}"
+    }
+}
+```
+
 ## Changelog
 
 ### next version

diff --git a/seatunnel-connectors-v2/connector-elasticsearch/pom.xml b/seatunnel-connectors-v2/connector-elasticsearch/pom.xml
@@ -49,5 +49,10 @@
             <artifactId>seatunnel-format-json</artifactId>
             <version>${project.version}</version>
         </dependency>
+        <dependency>
+            <groupId>io.airlift</groupId>
+            <artifactId>security</artifactId>
+            <version>206</version>
+        </dependency>
     </dependencies>
 </project>