From 9ebe6ace5bf5bd0f03ca40eecb15e2df0f3c9e36 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 30 Aug 2024 20:07:00 +0800 Subject: [PATCH] [#3514] improvement(flink-connector): add flink-connector-runtime to build flink connector (#4815) ### What changes were proposed in this pull request? - add flink runtime connector ### Why are the changes needed? Fix: #3514 ### Does this PR introduce _any_ user-facing change? - no ### How was this patch tested? - local test Co-authored-by: Peidian li <38486782+coolderli@users.noreply.github.com> Co-authored-by: fanng --- .github/workflows/flink-integration-test.yml | 10 +- build.gradle.kts | 6 +- docs/flink-connector/flink-connector.md | 2 +- flink-connector/build.gradle.kts | 157 +------------- .../flink-runtime/build.gradle.kts | 73 +++++++ flink-connector/flink/build.gradle.kts | 202 ++++++++++++++++++ .../connector/DefaultPartitionConverter.java | 0 .../flink/connector/PartitionConverter.java | 0 .../flink/connector/PropertiesConverter.java | 0 .../flink/connector/catalog/BaseCatalog.java | 0 .../catalog/GravitinoCatalogManager.java | 0 .../connector/hive/GravitinoHiveCatalog.java | 0 .../hive/GravitinoHiveCatalogFactory.java | 0 .../GravitinoHiveCatalogFactoryOptions.java | 0 .../hive/HivePropertiesConverter.java | 0 .../store/GravitinoCatalogStore.java | 0 .../store/GravitinoCatalogStoreFactory.java | 0 .../GravitinoCatalogStoreFactoryOptions.java | 0 .../flink/connector/utils/FactoryUtils.java | 0 .../flink/connector/utils/PropertyUtils.java | 0 .../flink/connector/utils/TableUtils.java | 0 .../flink/connector/utils/TypeUtils.java | 0 .../org.apache.flink.table.factories.Factory | 0 .../connector/catalog/TestBaseCatalog.java | 0 .../hive/TestHivePropertiesConverter.java | 0 .../integration/test/FlinkCommonIT.java | 0 .../integration/test/FlinkEnvIT.java | 0 .../test/hive/FlinkHiveCatalogIT.java | 0 .../integration/test/utils/TestUtils.java | 0 .../flink/connector/utils/TestTypeUtils.java | 0 .../test/resources/flink-tests/hive-site.xml | 0 .../src/test/resources/log4j2.properties | 0 settings.gradle.kts | 3 +- 33 files changed, 289 insertions(+), 164 deletions(-) create mode 100644 flink-connector/flink-runtime/build.gradle.kts create mode 100644 flink-connector/flink/build.gradle.kts rename flink-connector/{ => flink}/src/main/java/org/apache/gravitino/flink/connector/DefaultPartitionConverter.java (100%) rename flink-connector/{ => flink}/src/main/java/org/apache/gravitino/flink/connector/PartitionConverter.java (100%) rename flink-connector/{ => flink}/src/main/java/org/apache/gravitino/flink/connector/PropertiesConverter.java (100%) rename flink-connector/{ => flink}/src/main/java/org/apache/gravitino/flink/connector/catalog/BaseCatalog.java (100%) rename flink-connector/{ => flink}/src/main/java/org/apache/gravitino/flink/connector/catalog/GravitinoCatalogManager.java (100%) rename flink-connector/{ => flink}/src/main/java/org/apache/gravitino/flink/connector/hive/GravitinoHiveCatalog.java (100%) rename flink-connector/{ => flink}/src/main/java/org/apache/gravitino/flink/connector/hive/GravitinoHiveCatalogFactory.java (100%) rename flink-connector/{ => flink}/src/main/java/org/apache/gravitino/flink/connector/hive/GravitinoHiveCatalogFactoryOptions.java (100%) rename flink-connector/{ => flink}/src/main/java/org/apache/gravitino/flink/connector/hive/HivePropertiesConverter.java (100%) rename flink-connector/{ => flink}/src/main/java/org/apache/gravitino/flink/connector/store/GravitinoCatalogStore.java (100%) rename flink-connector/{ => flink}/src/main/java/org/apache/gravitino/flink/connector/store/GravitinoCatalogStoreFactory.java (100%) rename flink-connector/{ => flink}/src/main/java/org/apache/gravitino/flink/connector/store/GravitinoCatalogStoreFactoryOptions.java (100%) rename flink-connector/{ => flink}/src/main/java/org/apache/gravitino/flink/connector/utils/FactoryUtils.java (100%) rename flink-connector/{ => flink}/src/main/java/org/apache/gravitino/flink/connector/utils/PropertyUtils.java (100%) rename flink-connector/{ => flink}/src/main/java/org/apache/gravitino/flink/connector/utils/TableUtils.java (100%) rename flink-connector/{ => flink}/src/main/java/org/apache/gravitino/flink/connector/utils/TypeUtils.java (100%) rename flink-connector/{ => flink}/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory (100%) rename flink-connector/{ => flink}/src/test/java/org/apache/gravitino/flink/connector/catalog/TestBaseCatalog.java (100%) rename flink-connector/{ => flink}/src/test/java/org/apache/gravitino/flink/connector/hive/TestHivePropertiesConverter.java (100%) rename flink-connector/{ => flink}/src/test/java/org/apache/gravitino/flink/connector/integration/test/FlinkCommonIT.java (100%) rename flink-connector/{ => flink}/src/test/java/org/apache/gravitino/flink/connector/integration/test/FlinkEnvIT.java (100%) rename flink-connector/{ => flink}/src/test/java/org/apache/gravitino/flink/connector/integration/test/hive/FlinkHiveCatalogIT.java (100%) rename flink-connector/{ => flink}/src/test/java/org/apache/gravitino/flink/connector/integration/test/utils/TestUtils.java (100%) rename flink-connector/{ => flink}/src/test/java/org/apache/gravitino/flink/connector/utils/TestTypeUtils.java (100%) rename flink-connector/{ => flink}/src/test/resources/flink-tests/hive-site.xml (100%) rename flink-connector/{ => flink}/src/test/resources/log4j2.properties (100%) diff --git a/.github/workflows/flink-integration-test.yml b/.github/workflows/flink-integration-test.yml index c59c0fd23f7..6e1227664cb 100644 --- a/.github/workflows/flink-integration-test.yml +++ b/.github/workflows/flink-integration-test.yml @@ -86,8 +86,8 @@ jobs: - name: Flink Integration Test id: integrationTest run: | - ./gradlew -PskipTests -PtestMode=embedded -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :flink-connector:test --tests "org.apache.gravitino.flink.connector.integration.test.**" - ./gradlew -PskipTests -PtestMode=deploy -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :flink-connector:test --tests "org.apache.gravitino.flink.connector.integration.test.**" + ./gradlew -PskipTests -PtestMode=embedded -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :flink-connector:flink:test --tests "org.apache.gravitino.flink.connector.integration.test.**" + ./gradlew -PskipTests -PtestMode=deploy -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :flink-connector:flink:test --tests "org.apache.gravitino.flink.connector.integration.test.**" - name: Upload integrate tests reports uses: actions/upload-artifact@v3 @@ -96,9 +96,9 @@ jobs: name: flink-connector-integrate-test-reports-${{ matrix.java-version }} path: | build/reports - flink-connector/build/flink-connector-integration-test.log - flink-connector/build/*.tar + flink-connector/flink/build/*.log + flink-connector/flink/build/*.tar distribution/package/logs/gravitino-server.out distribution/package/logs/gravitino-server.log catalogs/**/*.log - catalogs/**/*.tar \ No newline at end of file + catalogs/**/*.tar diff --git a/build.gradle.kts b/build.gradle.kts index a6307aa2795..50e62de3b62 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -736,7 +736,7 @@ tasks { if (!it.name.startsWith("catalog") && !it.name.startsWith("authorization") && !it.name.startsWith("client") && !it.name.startsWith("filesystem") && !it.name.startsWith("spark") && !it.name.startsWith("iceberg") && it.name != "trino-connector" && - it.name != "integration-test" && it.name != "bundled-catalog" && it.name != "flink-connector" + it.name != "integration-test" && it.name != "bundled-catalog" && !it.name.startsWith("flink") ) { from(it.configurations.runtimeClasspath) into("distribution/package/libs") @@ -753,9 +753,9 @@ tasks { !it.name.startsWith("spark") && !it.name.startsWith("iceberg") && !it.name.startsWith("integration-test") && + !it.name.startsWith("flink") && it.name != "trino-connector" && - it.name != "bundled-catalog" && - it.name != "flink-connector" + it.name != "bundled-catalog" ) { dependsOn("${it.name}:build") from("${it.name}/build/libs") diff --git a/docs/flink-connector/flink-connector.md b/docs/flink-connector/flink-connector.md index 6a323776e41..639dd0d682a 100644 --- a/docs/flink-connector/flink-connector.md +++ b/docs/flink-connector/flink-connector.md @@ -23,7 +23,7 @@ This capability allows users to perform federation queries, accessing data from ## How to use it -1. [Build](../how-to-build.md) or [download](https://mvnrepository.com/artifact/org.apache.gravitino/gravitino-flink-connector) the Gravitino flink connector jar, and place it to the classpath of Flink. +1. [Build](../how-to-build.md) or [download](https://mvnrepository.com/artifact/org.apache.gravitino/gravitino-flink-connector-runtime-1.18) the Gravitino flink connector runtime jar, and place it to the classpath of Flink. 2. Configure the Flink configuration to use the Gravitino flink connector. | Property | Type | Default Value | Description | Required | Since Version | diff --git a/flink-connector/build.gradle.kts b/flink-connector/build.gradle.kts index c2b13f40843..043fbfec673 100644 --- a/flink-connector/build.gradle.kts +++ b/flink-connector/build.gradle.kts @@ -16,158 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -plugins { - `maven-publish` - id("java") - id("idea") -} -repositories { - mavenCentral() -} - -val flinkVersion: String = libs.versions.flink.get() - -// The Flink only support scala 2.12, and all scala api will be removed in a future version. -// You can find more detail at the following issues: -// https://issues.apache.org/jira/browse/FLINK-23986, -// https://issues.apache.org/jira/browse/FLINK-20845, -// https://issues.apache.org/jira/browse/FLINK-13414. -val scalaVersion: String = "2.12" -val artifactName = "gravitino-${project.name}_$scalaVersion" - -dependencies { - implementation(project(":api")) - implementation(project(":catalogs:catalog-common")) - implementation(project(":common")) - implementation(project(":core")) - implementation(project(":clients:client-java")) - - implementation(libs.bundles.log4j) - implementation(libs.commons.lang3) - implementation(libs.guava) - implementation(libs.httpclient5) - implementation(libs.jackson.databind) - implementation(libs.jackson.annotations) - implementation(libs.jackson.datatype.jdk8) - implementation(libs.jackson.datatype.jsr310) - - implementation("org.apache.flink:flink-connector-hive_$scalaVersion:$flinkVersion") - implementation("org.apache.flink:flink-table-common:$flinkVersion") - implementation("org.apache.flink:flink-table-api-java:$flinkVersion") - - implementation(libs.hive2.exec) { - artifact { - classifier = "core" - } - exclude("com.fasterxml.jackson.core") - exclude("com.google.code.findbugs", "jsr305") - exclude("com.google.protobuf") - exclude("org.apache.avro") - exclude("org.apache.calcite") - exclude("org.apache.calcite.avatica") - exclude("org.apache.curator") - exclude("org.apache.hadoop", "hadoop-yarn-server-resourcemanager") - exclude("org.apache.logging.log4j") - exclude("org.apache.zookeeper") - exclude("org.eclipse.jetty.aggregate", "jetty-all") - exclude("org.eclipse.jetty.orbit", "javax.servlet") - exclude("org.openjdk.jol") - exclude("org.pentaho") - exclude("org.slf4j") - } - - testAnnotationProcessor(libs.lombok) - - testCompileOnly(libs.lombok) - testImplementation(project(":integration-test-common", "testArtifacts")) - testImplementation(project(":server")) - testImplementation(project(":server-common")) - testImplementation(libs.junit.jupiter.api) - testImplementation(libs.junit.jupiter.params) - testImplementation(libs.mockito.core) - testImplementation(libs.mysql.driver) - testImplementation(libs.sqlite.jdbc) - testImplementation(libs.testcontainers) - testImplementation(libs.testcontainers.junit.jupiter) - testImplementation(libs.testcontainers.mysql) - - testImplementation(libs.hadoop2.common) { - exclude("*") - } - testImplementation(libs.hadoop2.hdfs) { - exclude("com.sun.jersey") - exclude("commons-cli", "commons-cli") - exclude("commons-io", "commons-io") - exclude("commons-codec", "commons-codec") - exclude("commons-logging", "commons-logging") - exclude("javax.servlet", "servlet-api") - exclude("org.mortbay.jetty") - } - testImplementation(libs.hadoop2.mapreduce.client.core) { - exclude("*") - } - testImplementation(libs.hive2.common) { - exclude("org.eclipse.jetty.aggregate", "jetty-all") - exclude("org.eclipse.jetty.orbit", "javax.servlet") - } - testImplementation(libs.hive2.metastore) { - exclude("co.cask.tephra") - exclude("com.github.joshelser") - exclude("com.google.code.findbugs", "jsr305") - exclude("com.google.code.findbugs", "sr305") - exclude("com.tdunning", "json") - exclude("com.zaxxer", "HikariCP") - exclude("io.dropwizard.metricss") - exclude("javax.transaction", "transaction-api") - exclude("org.apache.avro") - exclude("org.apache.curator") - exclude("org.apache.hbase") - exclude("org.apache.hadoop", "hadoop-yarn-server-resourcemanager") - exclude("org.apache.logging.log4j") - exclude("org.apache.parquet", "parquet-hadoop-bundle") - exclude("org.apache.zookeeper") - exclude("org.eclipse.jetty.aggregate", "jetty-all") - exclude("org.eclipse.jetty.orbit", "javax.servlet") - exclude("org.slf4j") - } - testImplementation("org.apache.flink:flink-table-api-bridge-base:$flinkVersion") { - exclude("commons-cli", "commons-cli") - exclude("commons-io", "commons-io") - exclude("com.google.code.findbugs", "jsr305") - } - testImplementation("org.apache.flink:flink-table-planner_$scalaVersion:$flinkVersion") - testImplementation("org.apache.flink:flink-test-utils:$flinkVersion") - - testRuntimeOnly(libs.junit.jupiter.engine) -} - -tasks.test { - val skipUTs = project.hasProperty("skipTests") - if (skipUTs) { - // Only run integration tests - include("**/integration/**") - } - - val skipITs = project.hasProperty("skipITs") - val skipFlinkITs = project.hasProperty("skipFlinkITs") - if (skipITs || skipFlinkITs) { - // Exclude integration tests - exclude("**/integration/**") - } else { - dependsOn(tasks.jar) - dependsOn(":catalogs:catalog-hive:jar") - } -} - -tasks.withType { - archiveBaseName.set(artifactName) -} - -publishing { - publications { - withType().configureEach { - artifactId = artifactName - } - } -} +tasks.all { + enabled = false +} \ No newline at end of file diff --git a/flink-connector/flink-runtime/build.gradle.kts b/flink-connector/flink-runtime/build.gradle.kts new file mode 100644 index 00000000000..63349ba690a --- /dev/null +++ b/flink-connector/flink-runtime/build.gradle.kts @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar + +plugins { + `maven-publish` + id("java") + id("idea") + alias(libs.plugins.shadow) +} + +repositories { + mavenCentral() +} + +val flinkVersion: String = libs.versions.flink.get() +val flinkMajorVersion: String = flinkVersion.substringBeforeLast(".") + +// The Flink only support scala 2.12, and all scala api will be removed in a future version. +// You can find more detail at the following issues: +// https://issues.apache.org/jira/browse/FLINK-23986, +// https://issues.apache.org/jira/browse/FLINK-20845, +// https://issues.apache.org/jira/browse/FLINK-13414. +val scalaVersion: String = "2.12" +val artifactName = "gravitino-${project.name}_$scalaVersion" +val baseName = "${rootProject.name}-flink-connector-runtime-${flinkMajorVersion}_$scalaVersion" + +dependencies { + implementation(project(":clients:client-java-runtime", configuration = "shadow")) + implementation(project(":flink-connector:flink")) +} + +tasks.withType(ShadowJar::class.java) { + isZip64 = true + configurations = listOf(project.configurations.runtimeClasspath.get()) + archiveFileName.set("$baseName-$version.jar") + archiveClassifier.set("") + + // Relocate dependencies to avoid conflicts + relocate("com.google", "org.apache.gravitino.shaded.com.google") + relocate("google", "org.apache.gravitino.shaded.google") + relocate("org.apache.hc", "org.apache.gravitino.shaded.org.apache.hc") + relocate("org.apache.commons", "org.apache.gravitino.shaded.org.apache.commons") +} + +publishing { + publications { + withType().configureEach { + artifactId = baseName + } + } +} + +tasks.jar { + dependsOn(tasks.named("shadowJar")) + archiveClassifier.set("empty") +} diff --git a/flink-connector/flink/build.gradle.kts b/flink-connector/flink/build.gradle.kts new file mode 100644 index 00000000000..34b09a7f0a3 --- /dev/null +++ b/flink-connector/flink/build.gradle.kts @@ -0,0 +1,202 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +plugins { + `maven-publish` + id("java") + id("idea") +} + +repositories { + mavenCentral() +} + +val flinkVersion: String = libs.versions.flink.get() +val flinkMajorVersion: String = flinkVersion.substringBeforeLast(".") + +// The Flink only support scala 2.12, and all scala api will be removed in a future version. +// You can find more detail at the following issues: +// https://issues.apache.org/jira/browse/FLINK-23986, +// https://issues.apache.org/jira/browse/FLINK-20845, +// https://issues.apache.org/jira/browse/FLINK-13414. +val scalaVersion: String = "2.12" +val artifactName = "${rootProject.name}-flink-${flinkMajorVersion}_$scalaVersion" + +dependencies { + implementation(project(":api")) + implementation(project(":catalogs:catalog-common")) + implementation(project(":common")) + implementation(project(":core")) + + compileOnly(libs.bundles.log4j) + implementation(libs.commons.lang3) + implementation(libs.guava) + implementation(libs.httpclient5) + implementation(libs.jackson.databind) + implementation(libs.jackson.annotations) + implementation(libs.jackson.datatype.jdk8) + implementation(libs.jackson.datatype.jsr310) + + compileOnly("org.apache.flink:flink-connector-hive_$scalaVersion:$flinkVersion") + compileOnly("org.apache.flink:flink-table-common:$flinkVersion") + compileOnly("org.apache.flink:flink-table-api-java:$flinkVersion") + + compileOnly(libs.hive2.exec) { + artifact { + classifier = "core" + } + exclude("com.fasterxml.jackson.core") + exclude("com.google.code.findbugs", "jsr305") + exclude("com.google.protobuf") + exclude("org.apache.avro") + exclude("org.apache.calcite") + exclude("org.apache.calcite.avatica") + exclude("org.apache.curator") + exclude("org.apache.hadoop", "hadoop-yarn-server-resourcemanager") + exclude("org.apache.logging.log4j") + exclude("org.apache.zookeeper") + exclude("org.eclipse.jetty.aggregate", "jetty-all") + exclude("org.eclipse.jetty.orbit", "javax.servlet") + exclude("org.openjdk.jol") + exclude("org.pentaho") + exclude("org.slf4j") + } + + // replace with client-java-runtime in flink connector runtime + compileOnly(project(":clients:client-java")) + + testAnnotationProcessor(libs.lombok) + + testCompileOnly(libs.lombok) + testImplementation(project(":clients:client-java")) + testImplementation(project(":integration-test-common", "testArtifacts")) + testImplementation(project(":server")) + testImplementation(project(":server-common")) + testImplementation(libs.junit.jupiter.api) + testImplementation(libs.junit.jupiter.params) + testImplementation(libs.mockito.core) + testImplementation(libs.mysql.driver) + testImplementation(libs.sqlite.jdbc) + testImplementation(libs.testcontainers) + testImplementation(libs.testcontainers.junit.jupiter) + testImplementation(libs.testcontainers.mysql) + + testImplementation("org.apache.flink:flink-connector-hive_$scalaVersion:$flinkVersion") + testImplementation("org.apache.flink:flink-table-common:$flinkVersion") + testImplementation("org.apache.flink:flink-table-api-java:$flinkVersion") + + testImplementation(libs.hive2.exec) { + artifact { + classifier = "core" + } + exclude("com.fasterxml.jackson.core") + exclude("com.google.code.findbugs", "jsr305") + exclude("com.google.protobuf") + exclude("org.apache.avro") + exclude("org.apache.calcite") + exclude("org.apache.calcite.avatica") + exclude("org.apache.curator") + exclude("org.apache.hadoop", "hadoop-yarn-server-resourcemanager") + exclude("org.apache.logging.log4j") + exclude("org.apache.zookeeper") + exclude("org.eclipse.jetty.aggregate", "jetty-all") + exclude("org.eclipse.jetty.orbit", "javax.servlet") + exclude("org.openjdk.jol") + exclude("org.pentaho") + exclude("org.slf4j") + } + + testImplementation(libs.hadoop2.common) { + exclude("*") + } + testImplementation(libs.hadoop2.hdfs) { + exclude("com.sun.jersey") + exclude("commons-cli", "commons-cli") + exclude("commons-io", "commons-io") + exclude("commons-codec", "commons-codec") + exclude("commons-logging", "commons-logging") + exclude("javax.servlet", "servlet-api") + exclude("org.mortbay.jetty") + } + testImplementation(libs.hadoop2.mapreduce.client.core) { + exclude("*") + } + testImplementation(libs.hive2.common) { + exclude("org.eclipse.jetty.aggregate", "jetty-all") + exclude("org.eclipse.jetty.orbit", "javax.servlet") + } + testImplementation(libs.hive2.metastore) { + exclude("co.cask.tephra") + exclude("com.github.joshelser") + exclude("com.google.code.findbugs", "jsr305") + exclude("com.google.code.findbugs", "sr305") + exclude("com.tdunning", "json") + exclude("com.zaxxer", "HikariCP") + exclude("io.dropwizard.metricss") + exclude("javax.transaction", "transaction-api") + exclude("org.apache.avro") + exclude("org.apache.curator") + exclude("org.apache.hbase") + exclude("org.apache.hadoop", "hadoop-yarn-server-resourcemanager") + exclude("org.apache.logging.log4j") + exclude("org.apache.parquet", "parquet-hadoop-bundle") + exclude("org.apache.zookeeper") + exclude("org.eclipse.jetty.aggregate", "jetty-all") + exclude("org.eclipse.jetty.orbit", "javax.servlet") + exclude("org.slf4j") + } + testImplementation("org.apache.flink:flink-table-api-bridge-base:$flinkVersion") { + exclude("commons-cli", "commons-cli") + exclude("commons-io", "commons-io") + exclude("com.google.code.findbugs", "jsr305") + } + testImplementation("org.apache.flink:flink-table-planner_$scalaVersion:$flinkVersion") + testImplementation("org.apache.flink:flink-test-utils:$flinkVersion") + + testRuntimeOnly(libs.junit.jupiter.engine) +} + +tasks.test { + val skipUTs = project.hasProperty("skipTests") + if (skipUTs) { + // Only run integration tests + include("**/integration/**") + } + + val skipITs = project.hasProperty("skipITs") + val skipFlinkITs = project.hasProperty("skipFlinkITs") + if (skipITs || skipFlinkITs) { + // Exclude integration tests + exclude("**/integration/**") + } else { + dependsOn(tasks.jar) + dependsOn(":catalogs:catalog-hive:jar") + } +} + +tasks.withType { + archiveBaseName.set(artifactName) +} + +publishing { + publications { + withType().configureEach { + artifactId = artifactName + } + } +} diff --git a/flink-connector/src/main/java/org/apache/gravitino/flink/connector/DefaultPartitionConverter.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/DefaultPartitionConverter.java similarity index 100% rename from flink-connector/src/main/java/org/apache/gravitino/flink/connector/DefaultPartitionConverter.java rename to flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/DefaultPartitionConverter.java diff --git a/flink-connector/src/main/java/org/apache/gravitino/flink/connector/PartitionConverter.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/PartitionConverter.java similarity index 100% rename from flink-connector/src/main/java/org/apache/gravitino/flink/connector/PartitionConverter.java rename to flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/PartitionConverter.java diff --git a/flink-connector/src/main/java/org/apache/gravitino/flink/connector/PropertiesConverter.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/PropertiesConverter.java similarity index 100% rename from flink-connector/src/main/java/org/apache/gravitino/flink/connector/PropertiesConverter.java rename to flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/PropertiesConverter.java diff --git a/flink-connector/src/main/java/org/apache/gravitino/flink/connector/catalog/BaseCatalog.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/catalog/BaseCatalog.java similarity index 100% rename from flink-connector/src/main/java/org/apache/gravitino/flink/connector/catalog/BaseCatalog.java rename to flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/catalog/BaseCatalog.java diff --git a/flink-connector/src/main/java/org/apache/gravitino/flink/connector/catalog/GravitinoCatalogManager.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/catalog/GravitinoCatalogManager.java similarity index 100% rename from flink-connector/src/main/java/org/apache/gravitino/flink/connector/catalog/GravitinoCatalogManager.java rename to flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/catalog/GravitinoCatalogManager.java diff --git a/flink-connector/src/main/java/org/apache/gravitino/flink/connector/hive/GravitinoHiveCatalog.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/hive/GravitinoHiveCatalog.java similarity index 100% rename from flink-connector/src/main/java/org/apache/gravitino/flink/connector/hive/GravitinoHiveCatalog.java rename to flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/hive/GravitinoHiveCatalog.java diff --git a/flink-connector/src/main/java/org/apache/gravitino/flink/connector/hive/GravitinoHiveCatalogFactory.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/hive/GravitinoHiveCatalogFactory.java similarity index 100% rename from flink-connector/src/main/java/org/apache/gravitino/flink/connector/hive/GravitinoHiveCatalogFactory.java rename to flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/hive/GravitinoHiveCatalogFactory.java diff --git a/flink-connector/src/main/java/org/apache/gravitino/flink/connector/hive/GravitinoHiveCatalogFactoryOptions.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/hive/GravitinoHiveCatalogFactoryOptions.java similarity index 100% rename from flink-connector/src/main/java/org/apache/gravitino/flink/connector/hive/GravitinoHiveCatalogFactoryOptions.java rename to flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/hive/GravitinoHiveCatalogFactoryOptions.java diff --git a/flink-connector/src/main/java/org/apache/gravitino/flink/connector/hive/HivePropertiesConverter.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/hive/HivePropertiesConverter.java similarity index 100% rename from flink-connector/src/main/java/org/apache/gravitino/flink/connector/hive/HivePropertiesConverter.java rename to flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/hive/HivePropertiesConverter.java diff --git a/flink-connector/src/main/java/org/apache/gravitino/flink/connector/store/GravitinoCatalogStore.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/store/GravitinoCatalogStore.java similarity index 100% rename from flink-connector/src/main/java/org/apache/gravitino/flink/connector/store/GravitinoCatalogStore.java rename to flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/store/GravitinoCatalogStore.java diff --git a/flink-connector/src/main/java/org/apache/gravitino/flink/connector/store/GravitinoCatalogStoreFactory.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/store/GravitinoCatalogStoreFactory.java similarity index 100% rename from flink-connector/src/main/java/org/apache/gravitino/flink/connector/store/GravitinoCatalogStoreFactory.java rename to flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/store/GravitinoCatalogStoreFactory.java diff --git a/flink-connector/src/main/java/org/apache/gravitino/flink/connector/store/GravitinoCatalogStoreFactoryOptions.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/store/GravitinoCatalogStoreFactoryOptions.java similarity index 100% rename from flink-connector/src/main/java/org/apache/gravitino/flink/connector/store/GravitinoCatalogStoreFactoryOptions.java rename to flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/store/GravitinoCatalogStoreFactoryOptions.java diff --git a/flink-connector/src/main/java/org/apache/gravitino/flink/connector/utils/FactoryUtils.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/utils/FactoryUtils.java similarity index 100% rename from flink-connector/src/main/java/org/apache/gravitino/flink/connector/utils/FactoryUtils.java rename to flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/utils/FactoryUtils.java diff --git a/flink-connector/src/main/java/org/apache/gravitino/flink/connector/utils/PropertyUtils.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/utils/PropertyUtils.java similarity index 100% rename from flink-connector/src/main/java/org/apache/gravitino/flink/connector/utils/PropertyUtils.java rename to flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/utils/PropertyUtils.java diff --git a/flink-connector/src/main/java/org/apache/gravitino/flink/connector/utils/TableUtils.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/utils/TableUtils.java similarity index 100% rename from flink-connector/src/main/java/org/apache/gravitino/flink/connector/utils/TableUtils.java rename to flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/utils/TableUtils.java diff --git a/flink-connector/src/main/java/org/apache/gravitino/flink/connector/utils/TypeUtils.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/utils/TypeUtils.java similarity index 100% rename from flink-connector/src/main/java/org/apache/gravitino/flink/connector/utils/TypeUtils.java rename to flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/utils/TypeUtils.java diff --git a/flink-connector/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory b/flink-connector/flink/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory similarity index 100% rename from flink-connector/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory rename to flink-connector/flink/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory diff --git a/flink-connector/src/test/java/org/apache/gravitino/flink/connector/catalog/TestBaseCatalog.java b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/catalog/TestBaseCatalog.java similarity index 100% rename from flink-connector/src/test/java/org/apache/gravitino/flink/connector/catalog/TestBaseCatalog.java rename to flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/catalog/TestBaseCatalog.java diff --git a/flink-connector/src/test/java/org/apache/gravitino/flink/connector/hive/TestHivePropertiesConverter.java b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/hive/TestHivePropertiesConverter.java similarity index 100% rename from flink-connector/src/test/java/org/apache/gravitino/flink/connector/hive/TestHivePropertiesConverter.java rename to flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/hive/TestHivePropertiesConverter.java diff --git a/flink-connector/src/test/java/org/apache/gravitino/flink/connector/integration/test/FlinkCommonIT.java b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/FlinkCommonIT.java similarity index 100% rename from flink-connector/src/test/java/org/apache/gravitino/flink/connector/integration/test/FlinkCommonIT.java rename to flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/FlinkCommonIT.java diff --git a/flink-connector/src/test/java/org/apache/gravitino/flink/connector/integration/test/FlinkEnvIT.java b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/FlinkEnvIT.java similarity index 100% rename from flink-connector/src/test/java/org/apache/gravitino/flink/connector/integration/test/FlinkEnvIT.java rename to flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/FlinkEnvIT.java diff --git a/flink-connector/src/test/java/org/apache/gravitino/flink/connector/integration/test/hive/FlinkHiveCatalogIT.java b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/hive/FlinkHiveCatalogIT.java similarity index 100% rename from flink-connector/src/test/java/org/apache/gravitino/flink/connector/integration/test/hive/FlinkHiveCatalogIT.java rename to flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/hive/FlinkHiveCatalogIT.java diff --git a/flink-connector/src/test/java/org/apache/gravitino/flink/connector/integration/test/utils/TestUtils.java b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/utils/TestUtils.java similarity index 100% rename from flink-connector/src/test/java/org/apache/gravitino/flink/connector/integration/test/utils/TestUtils.java rename to flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/utils/TestUtils.java diff --git a/flink-connector/src/test/java/org/apache/gravitino/flink/connector/utils/TestTypeUtils.java b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/utils/TestTypeUtils.java similarity index 100% rename from flink-connector/src/test/java/org/apache/gravitino/flink/connector/utils/TestTypeUtils.java rename to flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/utils/TestTypeUtils.java diff --git a/flink-connector/src/test/resources/flink-tests/hive-site.xml b/flink-connector/flink/src/test/resources/flink-tests/hive-site.xml similarity index 100% rename from flink-connector/src/test/resources/flink-tests/hive-site.xml rename to flink-connector/flink/src/test/resources/flink-tests/hive-site.xml diff --git a/flink-connector/src/test/resources/log4j2.properties b/flink-connector/flink/src/test/resources/log4j2.properties similarity index 100% rename from flink-connector/src/test/resources/log4j2.properties rename to flink-connector/flink/src/test/resources/log4j2.properties diff --git a/settings.gradle.kts b/settings.gradle.kts index f62d5512546..1bf0d50832b 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -56,7 +56,8 @@ if (scalaVersion == "2.12") { project(":spark-connector:spark-3.3").projectDir = file("spark-connector/v3.3/spark") project(":spark-connector:spark-runtime-3.3").projectDir = file("spark-connector/v3.3/spark-runtime") // flink only support scala 2.12 - include("flink-connector") + include("flink-connector:flink") + include("flink-connector:flink-runtime") } include("spark-connector:spark-3.4", "spark-connector:spark-runtime-3.4", "spark-connector:spark-3.5", "spark-connector:spark-runtime-3.5") project(":spark-connector:spark-3.4").projectDir = file("spark-connector/v3.4/spark")