Merge branch 'dev' into timeplus

jovezhong · Sep 5, 2024 · 1882aae · 1882aae
2 parents 33797a8 + 90cd46f
commit 1882aae
Show file tree

Hide file tree

Showing 444 changed files with 23,503 additions and 1,903 deletions.
diff --git a/.github/workflows/labeler/label-scope-conf.yml b/.github/workflows/labeler/label-scope-conf.yml
@@ -257,6 +257,19 @@ activemq:
       - changed-files:
           - any-glob-to-any-file: seatunnel-connectors-v2/connector-activemq/**
           - all-globs-to-all-files: '!seatunnel-connectors-v2/connector-!(activemq)/**'
+
+qdrant:
+  - all:
+      - changed-files:
+          - any-glob-to-any-file: seatunnel-connectors-v2/connector-qdrant/**
+          - all-globs-to-all-files: '!seatunnel-connectors-v2/connector-!(qdrant)/**'
+
+typesense:
+  - all:
+      - changed-files:
+          - any-glob-to-any-file: seatunnel-connectors-v2/connector-typesense/**
+          - all-globs-to-all-files: '!seatunnel-connectors-v2/connector-!(typesense)/**'
+
 Zeta Rest API:
   - changed-files:
       - any-glob-to-any-file: seatunnel-engine/**/server/rest/**
@@ -284,4 +297,4 @@ timeplus:
   - all:
       - changed-files:
           - any-glob-to-any-file: seatunnel-connectors-v2/connector-timeplus/**
-          - all-globs-to-all-files: '!seatunnel-connectors-v2/connector-!(timeplus)/**'
+          - all-globs-to-all-files: '!seatunnel-connectors-v2/connector-!(timeplus)/**'
diff --git a/.github/workflows/publish-docker.yaml b/.github/workflows/publish-docker.yaml
@@ -18,16 +18,15 @@ name: publish-docker
 
 on:
   push:
-    branches:
-      - dev
-      - docker
+    tags:
+      - '*'
     paths-ignore:
       - 'docs/**'
       - '**/*.md'
-      - 'seatunnel-ui/**'
 
 env:
-  HUB: ghcr.io/${{ github.repository }}
+  DOCKER_USERNAME: ${{ secrets.DOCKERHUB_USER }}
+  DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
 
 jobs:
   build:
@@ -36,11 +35,16 @@ jobs:
     permissions:
       contents: read
       packages: write
-    timeout-minutes: 30
+    timeout-minutes: 60
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+      - name: free disk space
+        run: tools/github/free_disk_space.sh
+      - uses: actions/checkout@v4
       - name: Cache local Maven repository
-        uses: actions/cache@v2
+        uses: actions/cache@v4
         with:
           path: ~/.m2/repository
           key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
@@ -51,26 +55,27 @@ jobs:
         with:
           java-version:  8
           distribution: 'adopt'
+
       - name: Log in to the Container registry
-        uses: docker/login-action@v1.10.0
+        uses: docker/login-action@v3
         with:
-          registry: ${{ env.HUB }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
+          username: ${{ env.DOCKER_USERNAME }}
+          password: ${{ env.DOCKER_PASSWORD }}
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
       - name: Build and push docker images
         env:
-          MAVEN_OPTS: -Xmx2G -Xms2G
+          MAVEN_OPTS: -Xmx4096m
         run: |
-          ./mvnw -B clean deploy \
-          -Dmaven.test.skip \
-          -Dmaven.javadoc.skip \
+          ./mvnw -B clean install \
+          -Dmaven.test.skip=true \
+          -Dmaven.javadoc.skip=true \
           -Dlicense.skipAddThirdParty=true \
+          -D"docker.build.skip"=false \
+          -D"docker.verify.skip"=false \
+          -D"docker.push.skip"=false \
           -Dmaven.deploy.skip \
-          -Ddocker.tag=${{ github.sha }} \
-          -Ddocker.hub=${{ env.HUB }} \
-          -Pdocker \
-          --no-snapshot-updates
+          --no-snapshot-updates \
+          -Pdocker,seatunnel
diff --git a/config/log4j2.properties b/config/log4j2.properties
@@ -45,7 +45,7 @@ appender.consoleStdout.name = consoleStdoutAppender
 appender.consoleStdout.type = CONSOLE
 appender.consoleStdout.target = SYSTEM_OUT
 appender.consoleStdout.layout.type = PatternLayout
-appender.consoleStdout.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p [%-30.30c{1.}] [%t] - %m%n
+appender.consoleStdout.layout.pattern = [%X{ST-JID}] %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p [%-30.30c{1.}] [%t] - %m%n
 appender.consoleStdout.filter.acceptLtWarn.type = ThresholdFilter
 appender.consoleStdout.filter.acceptLtWarn.level = WARN
 appender.consoleStdout.filter.acceptLtWarn.onMatch = DENY
@@ -55,7 +55,7 @@ appender.consoleStderr.name = consoleStderrAppender
 appender.consoleStderr.type = CONSOLE
 appender.consoleStderr.target = SYSTEM_ERR
 appender.consoleStderr.layout.type = PatternLayout
-appender.consoleStderr.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p [%-30.30c{1.}] [%t] - %m%n
+appender.consoleStderr.layout.pattern = [%X{ST-JID}] %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p [%-30.30c{1.}] [%t] - %m%n
 appender.consoleStderr.filter.acceptGteWarn.type = ThresholdFilter
 appender.consoleStderr.filter.acceptGteWarn.level = WARN
 appender.consoleStderr.filter.acceptGteWarn.onMatch = ACCEPT
@@ -67,7 +67,7 @@ appender.file.fileName = ${file_path}/${file_name}.log
 appender.file.filePattern = ${file_path}/${file_name}.log.%d{yyyy-MM-dd}-%i
 appender.file.append = true
 appender.file.layout.type = PatternLayout
-appender.file.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p [%-30.30c{1.}] [%t] - %m%n
+appender.file.layout.pattern = [%X{ST-JID}] %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p [%-30.30c{1.}] [%t] - %m%n
 appender.file.policies.type = Policies
 appender.file.policies.time.type = TimeBasedTriggeringPolicy
 appender.file.policies.time.modulate = true

diff --git a/config/plugin_config b/config/plugin_config
@@ -88,5 +88,7 @@ connector-web3j
 connector-milvus
 connector-activemq
 connector-sls
-connector-timeplus
---end--
+connector-qdrant
+connector-typesense
+connector-cdc-opengauss
+connector-timeplus
diff --git a/config/seatunnel.yaml b/config/seatunnel.yaml
@@ -33,4 +33,4 @@ seatunnel:
         plugin-config:
           namespace: /tmp/seatunnel/checkpoint_snapshot
           storage.type: hdfs
-          fs.defaultFS: file:///tmp/ # Ensure that the directory has written permission
+          fs.defaultFS: file:///tmp/ # Ensure that the directory has written permission
diff --git a/docs/en/connector-v2/sink/Clickhouse.md b/docs/en/connector-v2/sink/Clickhouse.md
@@ -24,9 +24,9 @@ Used to write data to Clickhouse.
 In order to use the Clickhouse connector, the following dependencies are required.
 They can be downloaded via install-plugin.sh or from the Maven central repository.
 
-| Datasource | Supported Versions |                                                    Dependency                                                    |
-|------------|--------------------|------------------------------------------------------------------------------------------------------------------|
-| Clickhouse | universal          | [Download](https://mvnrepository.com/artifact/org.apache.seatunnel/seatunnel-connectors-v2/connector-clickhouse) |
+| Datasource | Supported Versions | Dependency                                                                               |
+|------------|--------------------|------------------------------------------------------------------------------------------|
+| Clickhouse | universal          | [Download](https://mvnrepository.com/artifact/org.apache.seatunnel/connector-clickhouse) |
 
 ## Data Type Mapping
 

diff --git a/docs/en/connector-v2/sink/Hbase.md b/docs/en/connector-v2/sink/Hbase.md
@@ -116,6 +116,79 @@ Hbase {
     all_columns = seatunnel
   }
 }
+
+```
+
+### Multiple Table
+
+```hocon
+env {
+  # You can set engine configuration here
+  execution.parallelism = 1
+  job.mode = "BATCH"
+}
+
+source {
+  FakeSource {
+    tables_configs = [
+       {
+        schema = {
+          table = "hbase_sink_1"
+         fields {
+                    name = STRING
+                    c_string = STRING
+                    c_double = DOUBLE
+                    c_bigint = BIGINT
+                    c_float = FLOAT
+                    c_int = INT
+                    c_smallint = SMALLINT
+                    c_boolean = BOOLEAN
+                    time = BIGINT
+           }
+        }
+            rows = [
+              {
+                kind = INSERT
+                fields = ["label_1", "sink_1", 4.3, 200, 2.5, 2, 5, true, 1627529632356]
+              }
+              ]
+       },
+       {
+       schema = {
+         table = "hbase_sink_2"
+              fields {
+                    name = STRING
+                    c_string = STRING
+                    c_double = DOUBLE
+                    c_bigint = BIGINT
+                    c_float = FLOAT
+                    c_int = INT
+                    c_smallint = SMALLINT
+                    c_boolean = BOOLEAN
+                    time = BIGINT
+              }
+       }
+           rows = [
+             {
+               kind = INSERT
+               fields = ["label_2", "sink_2", 4.3, 200, 2.5, 2, 5, true, 1627529632357]
+             }
+             ]
+      }
+    ]
+  }
+}
+
+sink {
+  Hbase {
+    zookeeper_quorum = "hadoop001:2181,hadoop002:2181,hadoop003:2181"
+    table = "${table_name}"
+    rowkey_column = ["name"]
+    family_name {
+      all_columns = info
+    }
+  }
+}
 ```
 
 ## Writes To The Specified Column Family

diff --git a/docs/en/connector-v2/sink/Http.md b/docs/en/connector-v2/sink/Http.md
@@ -27,9 +27,9 @@ Used to launch web hooks using data.
 In order to use the Http connector, the following dependencies are required.
 They can be downloaded via install-plugin.sh or from the Maven central repository.
 
-| Datasource | Supported Versions |                                                 Dependency                                                 |
-|------------|--------------------|------------------------------------------------------------------------------------------------------------|
-| Http       | universal          | [Download](https://mvnrepository.com/artifact/org.apache.seatunnel/seatunnel-connectors-v2/connector-http) |
+| Datasource | Supported Versions | Dependency                                                                         |
+|------------|--------------------|------------------------------------------------------------------------------------|
+| Http       | universal          | [Download](https://mvnrepository.com/artifact/org.apache.seatunnel/connector-http) |
 
 ## Sink Options
 

diff --git a/docs/en/connector-v2/sink/Jdbc.md b/docs/en/connector-v2/sink/Jdbc.md
@@ -235,7 +235,7 @@ there are some reference value for params above.
 | SQL Server        | com.microsoft.sqlserver.jdbc.SQLServerDriver | jdbc:sqlserver://localhost:1433                                    | com.microsoft.sqlserver.jdbc.SQLServerXADataSource | https://mvnrepository.com/artifact/com.microsoft.sqlserver/mssql-jdbc                                                         |
 | Oracle            | oracle.jdbc.OracleDriver                     | jdbc:oracle:thin:@localhost:1521/xepdb1                            | oracle.jdbc.xa.OracleXADataSource                  | https://mvnrepository.com/artifact/com.oracle.database.jdbc/ojdbc8                                                            |
 | sqlite            | org.sqlite.JDBC                              | jdbc:sqlite:test.db                                                | /                                                  | https://mvnrepository.com/artifact/org.xerial/sqlite-jdbc                                                                     |
-| GBase8a           | com.gbase.jdbc.Driver                        | jdbc:gbase://e2e_gbase8aDb:5258/test                               | /                                                  | https://www.gbase8.cn/wp-content/uploads/2020/10/gbase-connector-java-8.3.81.53-build55.5.7-bin_min_mix.jar                   |
+| GBase8a           | com.gbase.jdbc.Driver                        | jdbc:gbase://e2e_gbase8aDb:5258/test                               | /                                                  | https://cdn.gbase.cn/products/30/p5CiVwXBKQYIUGN8ecHvk/gbase-connector-java-9.5.0.7-build1-bin.jar                   |
 | StarRocks         | com.mysql.cj.jdbc.Driver                     | jdbc:mysql://localhost:3306/test                                   | /                                                  | https://mvnrepository.com/artifact/mysql/mysql-connector-java                                                                 |
 | db2               | com.ibm.db2.jcc.DB2Driver                    | jdbc:db2://localhost:50000/testdb                                  | com.ibm.db2.jcc.DB2XADataSource                    | https://mvnrepository.com/artifact/com.ibm.db2.jcc/db2jcc/db2jcc4                                                             |
 | saphana           | com.sap.db.jdbc.Driver                       | jdbc:sap://localhost:39015                                         | /                                                  | https://mvnrepository.com/artifact/com.sap.cloud.db.jdbc/ngdbc                                                                |