fix

apache · Dec 6, 2024 · 659338f · 659338f
1 parent dae7ccf
commit 659338f
Show file tree

Hide file tree

Showing 12 changed files with 39 additions and 42 deletions.
diff --git a/spark-connector/v3.3/spark/build.gradle.kts b/spark-connector/v3.3/spark/build.gradle.kts
@@ -44,7 +44,9 @@ dependencies {
     exclude("com.fasterxml.jackson")
   }
   compileOnly("org.apache.iceberg:iceberg-spark-runtime-${sparkMajorVersion}_$scalaVersion:$icebergVersion")
-  compileOnly("org.apache.paimon:paimon-spark-$sparkMajorVersion:$paimonVersion")
+  compileOnly("org.apache.paimon:paimon-spark-$sparkMajorVersion:$paimonVersion") {
+    exclude("org.apache.spark")
+  }
 
   testImplementation(project(":api")) {
     exclude("org.apache.logging.log4j")
@@ -124,7 +126,9 @@ dependencies {
   testImplementation("org.apache.iceberg:iceberg-core:$icebergVersion")
   testImplementation("org.apache.iceberg:iceberg-spark-runtime-${sparkMajorVersion}_$scalaVersion:$icebergVersion")
   testImplementation("org.apache.iceberg:iceberg-hive-metastore:$icebergVersion")
-  testImplementation("org.apache.paimon:paimon-spark-$sparkMajorVersion:$paimonVersion")
+  testImplementation("org.apache.paimon:paimon-spark-$sparkMajorVersion:$paimonVersion") {
+    exclude("org.apache.spark")
+  }
   testImplementation("org.apache.kyuubi:kyuubi-spark-connector-hive_$scalaVersion:$kyuubiVersion")
   // include spark-sql,spark-catalyst,hive-common,hdfs-client
   testImplementation("org.apache.spark:spark-hive_$scalaVersion:$sparkVersion") {
@@ -137,6 +141,9 @@ dependencies {
     exclude("com.fasterxml.jackson.core")
   }
   testImplementation("org.scala-lang.modules:scala-collection-compat_$scalaVersion:$scalaCollectionCompatVersion")
+  testImplementation("org.apache.spark:spark-catalyst_$scalaVersion:$sparkVersion")
+  testImplementation("org.apache.spark:spark-core_$scalaVersion:$sparkVersion")
+  testImplementation("org.apache.spark:spark-sql_$scalaVersion:$sparkVersion")
 
   testRuntimeOnly(libs.junit.jupiter.engine)
 }
@@ -155,6 +162,7 @@ tasks.test {
     dependsOn(":catalogs:catalog-lakehouse-iceberg:jar")
     dependsOn(":catalogs:catalog-hive:jar")
     dependsOn(":iceberg:iceberg-rest-server:jar")
+    dependsOn(":catalogs:catalog-lakehouse-paimon:jar")
   }
 }
 

diff --git a/...java/org/apache/gravitino/spark/connector/integration/test/hive/SparkHiveCatalogIT33.java b/...java/org/apache/gravitino/spark/connector/integration/test/hive/SparkHiveCatalogIT33.java
@@ -26,10 +26,7 @@ public class SparkHiveCatalogIT33 extends SparkHiveCatalogIT {
   @Test
   void testCatalogClassName() {
     String catalogClass =
-        getSparkSession()
-            .sessionState()
-            .conf()
-            .getConfString("spark.sql.catalog." + getCatalogName());
+        getSparkSession().sparkContext().conf().get("spark.sql.catalog." + getCatalogName());
     Assertions.assertEquals(GravitinoHiveCatalogSpark33.class.getName(), catalogClass);
   }
 }
diff --git a/...ravitino/spark/connector/integration/test/iceberg/SparkIcebergCatalogHiveBackendIT33.java b/...ravitino/spark/connector/integration/test/iceberg/SparkIcebergCatalogHiveBackendIT33.java
@@ -26,10 +26,7 @@ public class SparkIcebergCatalogHiveBackendIT33 extends SparkIcebergCatalogHiveB
   @Test
   void testCatalogClassName() {
     String catalogClass =
-        getSparkSession()
-            .sessionState()
-            .conf()
-            .getConfString("spark.sql.catalog." + getCatalogName());
+        getSparkSession().sparkContext().conf().get("spark.sql.catalog." + getCatalogName());
     Assertions.assertEquals(GravitinoIcebergCatalogSpark33.class.getName(), catalogClass);
   }
 }
diff --git a/...tino/spark/connector/integration/test/paimon/SparkPaimonCatalogFilesystemBackendIT33.java b/...tino/spark/connector/integration/test/paimon/SparkPaimonCatalogFilesystemBackendIT33.java
@@ -26,10 +26,7 @@ public class SparkPaimonCatalogFilesystemBackendIT33 extends SparkPaimonCatalogF
   @Test
   void testCatalogClassName() {
     String catalogClass =
-        getSparkSession()
-            .sessionState()
-            .conf()
-            .getConfString("spark.sql.catalog." + getCatalogName());
+        getSparkSession().sparkContext().conf().get("spark.sql.catalog." + getCatalogName());
     Assertions.assertEquals(GravitinoPaimonCatalogSpark33.class.getName(), catalogClass);
   }
 }
diff --git a/spark-connector/v3.4/spark/build.gradle.kts b/spark-connector/v3.4/spark/build.gradle.kts
@@ -45,7 +45,9 @@ dependencies {
   }
   compileOnly(project(":clients:client-java-runtime", configuration = "shadow"))
   compileOnly("org.apache.iceberg:iceberg-spark-runtime-${sparkMajorVersion}_$scalaVersion:$icebergVersion")
-  compileOnly("org.apache.paimon:paimon-spark-$sparkMajorVersion:$paimonVersion")
+  compileOnly("org.apache.paimon:paimon-spark-$sparkMajorVersion:$paimonVersion") {
+    exclude("org.apache.spark")
+  }
 
   testImplementation(project(":api")) {
     exclude("org.apache.logging.log4j")
@@ -124,7 +126,9 @@ dependencies {
   testImplementation("org.apache.iceberg:iceberg-core:$icebergVersion")
   testImplementation("org.apache.iceberg:iceberg-spark-runtime-${sparkMajorVersion}_$scalaVersion:$icebergVersion")
   testImplementation("org.apache.iceberg:iceberg-hive-metastore:$icebergVersion")
-  testImplementation("org.apache.paimon:paimon-spark-$sparkMajorVersion:$paimonVersion")
+  testImplementation("org.apache.paimon:paimon-spark-$sparkMajorVersion:$paimonVersion") {
+    exclude("org.apache.spark")
+  }
   testImplementation("org.apache.kyuubi:kyuubi-spark-connector-hive_$scalaVersion:$kyuubiVersion")
   // include spark-sql,spark-catalyst,hive-common,hdfs-client
   testImplementation("org.apache.spark:spark-hive_$scalaVersion:$sparkVersion") {
@@ -137,6 +141,9 @@ dependencies {
     exclude("com.fasterxml.jackson.core")
   }
   testImplementation("org.scala-lang.modules:scala-collection-compat_$scalaVersion:$scalaCollectionCompatVersion")
+  testImplementation("org.apache.spark:spark-catalyst_$scalaVersion:$sparkVersion")
+  testImplementation("org.apache.spark:spark-core_$scalaVersion:$sparkVersion")
+  testImplementation("org.apache.spark:spark-sql_$scalaVersion:$sparkVersion")
 
   testRuntimeOnly(libs.junit.jupiter.engine)
 }
@@ -155,6 +162,7 @@ tasks.test {
     dependsOn(":catalogs:catalog-lakehouse-iceberg:jar")
     dependsOn(":catalogs:catalog-hive:jar")
     dependsOn(":iceberg:iceberg-rest-server:jar")
+    dependsOn(":catalogs:catalog-lakehouse-paimon:jar")
   }
 }
 

diff --git a/...java/org/apache/gravitino/spark/connector/integration/test/hive/SparkHiveCatalogIT34.java b/...java/org/apache/gravitino/spark/connector/integration/test/hive/SparkHiveCatalogIT34.java
@@ -26,10 +26,7 @@ public class SparkHiveCatalogIT34 extends SparkHiveCatalogIT {
   @Test
   void testCatalogClassName() {
     String catalogClass =
-        getSparkSession()
-            .sessionState()
-            .conf()
-            .getConfString("spark.sql.catalog." + getCatalogName());
+        getSparkSession().sparkContext().conf().get("spark.sql.catalog." + getCatalogName());
     Assertions.assertEquals(GravitinoHiveCatalogSpark34.class.getName(), catalogClass);
   }
 }
diff --git a/...ravitino/spark/connector/integration/test/iceberg/SparkIcebergCatalogHiveBackendIT34.java b/...ravitino/spark/connector/integration/test/iceberg/SparkIcebergCatalogHiveBackendIT34.java
@@ -27,10 +27,7 @@ public class SparkIcebergCatalogHiveBackendIT34 extends SparkIcebergCatalogHiveB
   @Test
   void testCatalogClassName() {
     String catalogClass =
-        getSparkSession()
-            .sessionState()
-            .conf()
-            .getConfString("spark.sql.catalog." + getCatalogName());
+        getSparkSession().sparkContext().conf().get("spark.sql.catalog." + getCatalogName());
     Assertions.assertEquals(GravitinoIcebergCatalogSpark34.class.getName(), catalogClass);
   }
 }
diff --git a/...tino/spark/connector/integration/test/paimon/SparkPaimonCatalogFilesystemBackendIT34.java b/...tino/spark/connector/integration/test/paimon/SparkPaimonCatalogFilesystemBackendIT34.java
@@ -27,10 +27,7 @@ public class SparkPaimonCatalogFilesystemBackendIT34 extends SparkPaimonCatalogF
   @Test
   void testCatalogClassName() {
     String catalogClass =
-        getSparkSession()
-            .sessionState()
-            .conf()
-            .getConfString("spark.sql.catalog." + getCatalogName());
+        getSparkSession().sparkContext().conf().get("spark.sql.catalog." + getCatalogName());
     Assertions.assertEquals(GravitinoPaimonCatalogSpark34.class.getName(), catalogClass);
   }
 }
diff --git a/spark-connector/v3.5/spark/build.gradle.kts b/spark-connector/v3.5/spark/build.gradle.kts
@@ -46,7 +46,9 @@ dependencies {
   }
   compileOnly(project(":clients:client-java-runtime", configuration = "shadow"))
   compileOnly("org.apache.iceberg:iceberg-spark-runtime-${sparkMajorVersion}_$scalaVersion:$icebergVersion")
-  compileOnly("org.apache.paimon:paimon-spark-$sparkMajorVersion:$paimonVersion")
+  compileOnly("org.apache.paimon:paimon-spark-$sparkMajorVersion:$paimonVersion") {
+    exclude("org.apache.spark")
+  }
 
   testImplementation(project(":api")) {
     exclude("org.apache.logging.log4j")
@@ -126,7 +128,9 @@ dependencies {
   testImplementation("org.apache.iceberg:iceberg-core:$icebergVersion")
   testImplementation("org.apache.iceberg:iceberg-spark-runtime-${sparkMajorVersion}_$scalaVersion:$icebergVersion")
   testImplementation("org.apache.iceberg:iceberg-hive-metastore:$icebergVersion")
-  testImplementation("org.apache.paimon:paimon-spark-$sparkMajorVersion:$paimonVersion")
+  testImplementation("org.apache.paimon:paimon-spark-$sparkMajorVersion:$paimonVersion") {
+    exclude("org.apache.spark")
+  }
   testImplementation("org.apache.kyuubi:kyuubi-spark-connector-hive_$scalaVersion:$kyuubiVersion")
   // include spark-sql,spark-catalyst,hive-common,hdfs-client
   testImplementation("org.apache.spark:spark-hive_$scalaVersion:$sparkVersion") {
@@ -139,6 +143,9 @@ dependencies {
     exclude("com.fasterxml.jackson.core")
   }
   testImplementation("org.scala-lang.modules:scala-collection-compat_$scalaVersion:$scalaCollectionCompatVersion")
+  testImplementation("org.apache.spark:spark-catalyst_$scalaVersion:$sparkVersion")
+  testImplementation("org.apache.spark:spark-core_$scalaVersion:$sparkVersion")
+  testImplementation("org.apache.spark:spark-sql_$scalaVersion:$sparkVersion")
 
   testRuntimeOnly(libs.junit.jupiter.engine)
 }
@@ -157,6 +164,7 @@ tasks.test {
     dependsOn(":catalogs:catalog-lakehouse-iceberg:jar")
     dependsOn(":catalogs:catalog-hive:jar")
     dependsOn(":iceberg:iceberg-rest-server:jar")
+    dependsOn(":catalogs:catalog-lakehouse-paimon:jar")
   }
 }
 

diff --git a/...java/org/apache/gravitino/spark/connector/integration/test/hive/SparkHiveCatalogIT35.java b/...java/org/apache/gravitino/spark/connector/integration/test/hive/SparkHiveCatalogIT35.java
@@ -26,10 +26,7 @@ public class SparkHiveCatalogIT35 extends SparkHiveCatalogIT {
   @Test
   void testCatalogClassName() {
     String catalogClass =
-        getSparkSession()
-            .sessionState()
-            .conf()
-            .getConfString("spark.sql.catalog." + getCatalogName());
+        getSparkSession().sparkContext().conf().get("spark.sql.catalog." + getCatalogName());
     Assertions.assertEquals(GravitinoHiveCatalogSpark35.class.getName(), catalogClass);
   }
 }
diff --git a/...ravitino/spark/connector/integration/test/iceberg/SparkIcebergCatalogHiveBackendIT35.java b/...ravitino/spark/connector/integration/test/iceberg/SparkIcebergCatalogHiveBackendIT35.java
@@ -27,10 +27,7 @@ public class SparkIcebergCatalogHiveBackendIT35 extends SparkIcebergCatalogHiveB
   @Test
   void testCatalogClassName() {
     String catalogClass =
-        getSparkSession()
-            .sessionState()
-            .conf()
-            .getConfString("spark.sql.catalog." + getCatalogName());
+        getSparkSession().sparkContext().conf().get("spark.sql.catalog." + getCatalogName());
     Assertions.assertEquals(GravitinoIcebergCatalogSpark35.class.getName(), catalogClass);
   }
 }
diff --git a/...tino/spark/connector/integration/test/paimon/SparkPaimonCatalogFilesystemBackendIT35.java b/...tino/spark/connector/integration/test/paimon/SparkPaimonCatalogFilesystemBackendIT35.java
@@ -27,10 +27,7 @@ public class SparkPaimonCatalogFilesystemBackendIT35 extends SparkPaimonCatalogF
   @Test
   void testCatalogClassName() {
     String catalogClass =
-        getSparkSession()
-            .sessionState()
-            .conf()
-            .getConfString("spark.sql.catalog." + getCatalogName());
+        getSparkSession().sparkContext().conf().get("spark.sql.catalog." + getCatalogName());
     Assertions.assertEquals(GravitinoIcebergCatalogSpark35.class.getName(), catalogClass);
   }
 }