Fix local gradle build (#551)

Signed-off-by: Thomas Farr <[email protected]>
opensearch-project · Feb 12, 2025 · 9224a9c · 9224a9c
1 parent e219c16
commit 9224a9c
Show file tree

Hide file tree

Showing 47 changed files with 37 additions and 30 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -21,6 +21,9 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Bumps `com.google.protobuf:protobuf-java` from 4.28.2 to 4.29.3
 - Bumps `commons-codec:commons-codec` from 1.17.1 to 1.18.0
 - Bumps `commons-logging:commons-logging` from 1.3.4 to 1.3.5
+- Bumps Spark3 version from 3.2.4 to 3.4.4 ([#551](https://github.com/opensearch-project/opensearch-hadoop/pull/551))
+- Bumps Scala2.12 version from 2.12.8 to 2.12.20 ([#551](https://github.com/opensearch-project/opensearch-hadoop/pull/551))
+- Bumps Scala2.13 version from 2.13.6 to 2.13.16 ([#551](https://github.com/opensearch-project/opensearch-hadoop/pull/551))
 
 ## [1.3.0]
 ### Added

diff --git a/buildSrc/opensearch-hadoop-version.properties b/buildSrc/opensearch-hadoop-version.properties
@@ -1,4 +1,4 @@
 opensearch_hadoop   = 1.3.0
-opensearch          = 2.13.0
+opensearch          = 2.19.0
 lucene              = 9.10.0
 build-tools         = 2.11.1
diff --git a/...roovy/org/opensearch/hadoop/gradle/fixture/hadoop/services/HadoopServiceDescriptor.groovy b/...roovy/org/opensearch/hadoop/gradle/fixture/hadoop/services/HadoopServiceDescriptor.groovy
@@ -72,7 +72,7 @@ class HadoopServiceDescriptor implements ServiceDescriptor {
 
     @Override
     Version defaultVersion() {
-        return new Version(3, 3, 2)
+        return new Version(3, 3, 6)
     }
 
     @Override

diff --git a/.../groovy/org/opensearch/hadoop/gradle/fixture/hadoop/services/HiveServiceDescriptor.groovy b/.../groovy/org/opensearch/hadoop/gradle/fixture/hadoop/services/HiveServiceDescriptor.groovy
@@ -69,7 +69,7 @@ class HiveServiceDescriptor implements ServiceDescriptor {
 
     @Override
     Version defaultVersion() {
-        return new Version(3, 1, 2)
+        return new Version(4, 0, 1)
     }
 
     @Override

diff --git a/...vy/org/opensearch/hadoop/gradle/fixture/hadoop/services/SparkYarnServiceDescriptor.groovy b/...vy/org/opensearch/hadoop/gradle/fixture/hadoop/services/SparkYarnServiceDescriptor.groovy
@@ -66,13 +66,13 @@ class SparkYarnServiceDescriptor implements ServiceDescriptor {
 
     @Override
     Version defaultVersion() {
-        return new Version(3, 2, 4)
+        return new Version(3, 4, 4)
     }
 
     String hadoopVersionCompatibility() {
         // The spark artifacts that interface with Hadoop have a hadoop version in their names.
         // This version is not always a version that Hadoop still distributes.
-        return "3.2"
+        return "3"
     }
 
     @Override

diff --git a/gradle.properties b/gradle.properties
@@ -27,16 +27,16 @@ jacksonVersion = 1.8.8
 spark20Version = 2.3.0
 spark22Version = 2.2.3
 spark24Version = 2.4.4
-spark30Version = 3.2.4
+spark30Version = 3.4.4
 
 # same as Spark's
 scala210Version = 2.10.7
 scala210MajorVersion = 2.10
 scala211Version = 2.11.12
 scala211MajorVersion = 2.11
-scala212Version = 2.12.8
+scala212Version = 2.12.20
 scala212MajorVersion = 2.12
-scala213Version = 2.13.6
+scala213Version = 2.13.16
 scala213MajorVersion = 2.13
 
 # Testing

diff --git a/licenses/scala-library-2.12.20.jar.sha1 b/licenses/scala-library-2.12.20.jar.sha1
@@ -0,0 +1 @@
+562dc9629943eec010e27898c3be65ffa9210558
diff --git a/licenses/scala-library-2.12.8.jar.sha1 b/licenses/scala-library-2.12.8.jar.sha1
diff --git a/licenses/scala-reflect-2.12.20.jar.sha1 b/licenses/scala-reflect-2.12.20.jar.sha1
@@ -0,0 +1 @@
+f923933d1c168535e66ca825c18207a8db2b20a6
diff --git a/licenses/scala-reflect-2.12.8.jar.sha1 b/licenses/scala-reflect-2.12.8.jar.sha1
diff --git a/licenses/spark-catalyst_2.12-3.2.4.jar.sha1 b/licenses/spark-catalyst_2.12-3.2.4.jar.sha1
diff --git a/licenses/spark-catalyst_2.12-3.4.4.jar.sha1 b/licenses/spark-catalyst_2.12-3.4.4.jar.sha1
@@ -0,0 +1 @@
+5439fd3d28164f4486ee18bd682f2faaddc28514
diff --git a/licenses/spark-core_2.12-3.2.4.jar.sha1 b/licenses/spark-core_2.12-3.2.4.jar.sha1
diff --git a/licenses/spark-core_2.12-3.4.4.jar.sha1 b/licenses/spark-core_2.12-3.4.4.jar.sha1
@@ -0,0 +1 @@
+cf0a6995855ef82900e1e8e3cc3cd97bfcd2948c
diff --git a/licenses/spark-sql_2.12-3.2.4.jar.sha1 b/licenses/spark-sql_2.12-3.2.4.jar.sha1
diff --git a/licenses/spark-sql_2.12-3.4.4.jar.sha1 b/licenses/spark-sql_2.12-3.4.4.jar.sha1
@@ -0,0 +1 @@
+e7e720be91173317bffabf4132f664f9912aec53
diff --git a/licenses/spark-streaming_2.12-3.2.4.jar.sha1 b/licenses/spark-streaming_2.12-3.2.4.jar.sha1
diff --git a/licenses/spark-streaming_2.12-3.4.4.jar.sha1 b/licenses/spark-streaming_2.12-3.4.4.jar.sha1
@@ -0,0 +1 @@
+5c6d2acb3d60294ef95335b00f76fad4903c8cdd
diff --git a/licenses/spark-yarn_2.12-3.2.4.jar.sha1 b/licenses/spark-yarn_2.12-3.2.4.jar.sha1
diff --git a/licenses/spark-yarn_2.12-3.4.4.jar.sha1 b/licenses/spark-yarn_2.12-3.4.4.jar.sha1
@@ -0,0 +1 @@
+d76d44a3c0e58606afc6bd5850bb6e34c184e51f
diff --git a/mr/build.gradle b/mr/build.gradle
@@ -51,9 +51,11 @@ dependencies {
         // thirdparty submodule. Downstream users of the library can pull it in as needed
         // for tests.
         exclude group: 'com.fasterxml.jackson.core'
+        exclude group: 'org.slf4j', module: 'slf4j-reload4j'
     }
     api("org.apache.hadoop:hadoop-mapreduce-client-core:${project.ext.hadoopVersion}") {
         exclude group: 'com.fasterxml.jackson.core'
+        exclude group: 'org.slf4j', module: 'slf4j-reload4j'
     }
 
     embedded(project(path: ":thirdparty", configuration: "shadow"))

diff --git a/qa/kerberos/build.gradle b/qa/kerberos/build.gradle
@@ -85,7 +85,9 @@ dependencies {
     compileOnly("org.json4s:json4s-jackson_2.12:4.0.7")
     compileOnly("org.slf4j:slf4j-api:2.0.12")
 
-    implementation("org.apache.hadoop:hadoop-client:${HadoopClusterConfiguration.HADOOP.defaultVersion()}")
+    implementation("org.apache.hadoop:hadoop-client:${HadoopClusterConfiguration.HADOOP.defaultVersion()}") {
+        exclude group: 'org.slf4j', module: 'slf4j-reload4j'
+    }
     implementation("org.apache.spark:spark-sql_2.12:$project.ext.spark30Version")
 
     implementation( project(path: ':opensearch-hadoop-mr', configuration: 'itestJarConfig'))

diff --git a/spark/core/build.gradle b/spark/core/build.gradle
@@ -99,7 +99,7 @@ sparkVariants {
             String generatedJavaDirectory = "$buildDir/generated/java/${variant.name}"
             Configuration scalaCompilerPlugin = project.configurations.maybeCreate(variant.configuration('scalaCompilerPlugin'))
             scalaCompilerPlugin.defaultDependencies { dependencies ->
-                dependencies.add(project.dependencies.create("com.typesafe.genjavadoc:genjavadoc-plugin_${variant.scalaVersion}:0.18"))
+                dependencies.add(project.dependencies.create("com.typesafe.genjavadoc:genjavadoc-plugin_${variant.scalaVersion}:${variant.scalaMajorVersion != '2.11' ? '0.19' : '0.18'}"))
             }
 
             ScalaCompile compileScala = tasks.getByName(scalaCompileTaskName) as ScalaCompile

diff --git a/spark/core/licenses/scala-library-2.12.20.jar.sha1 b/spark/core/licenses/scala-library-2.12.20.jar.sha1
@@ -0,0 +1 @@
+562dc9629943eec010e27898c3be65ffa9210558
diff --git a/spark/core/licenses/scala-library-2.12.8.jar.sha1 b/spark/core/licenses/scala-library-2.12.8.jar.sha1
diff --git a/spark/core/licenses/scala-reflect-2.12.20.jar.sha1 b/spark/core/licenses/scala-reflect-2.12.20.jar.sha1
@@ -0,0 +1 @@
+f923933d1c168535e66ca825c18207a8db2b20a6
diff --git a/spark/core/licenses/scala-reflect-2.12.8.jar.sha1 b/spark/core/licenses/scala-reflect-2.12.8.jar.sha1
diff --git a/spark/sql-20/build.gradle b/spark/sql-20/build.gradle
@@ -144,7 +144,7 @@ sparkVariants {
             String generatedJavaDirectory = "$buildDir/generated/java/${variant.name}"
             Configuration scalaCompilerPlugin = project.configurations.maybeCreate(variant.configuration('scalaCompilerPlugin'))
             scalaCompilerPlugin.defaultDependencies { dependencies ->
-                dependencies.add(project.dependencies.create("com.typesafe.genjavadoc:genjavadoc-plugin_${variant.scalaVersion}:0.18"))
+                dependencies.add(project.dependencies.create("com.typesafe.genjavadoc:genjavadoc-plugin_${variant.scalaVersion}:${variant.scalaMajorVersion != '2.11' ? '0.19' : '0.18'}"))
             }
 
             ScalaCompile compileScala = tasks.getByName(scalaCompileTaskName) as ScalaCompile

diff --git a/spark/sql-20/licenses/scala-library-2.12.20.jar.sha1 b/spark/sql-20/licenses/scala-library-2.12.20.jar.sha1
@@ -0,0 +1 @@
+562dc9629943eec010e27898c3be65ffa9210558
diff --git a/spark/sql-20/licenses/scala-library-2.12.8.jar.sha1 b/spark/sql-20/licenses/scala-library-2.12.8.jar.sha1
diff --git a/spark/sql-20/licenses/scala-reflect-2.12.20.jar.sha1 b/spark/sql-20/licenses/scala-reflect-2.12.20.jar.sha1
@@ -0,0 +1 @@
+f923933d1c168535e66ca825c18207a8db2b20a6
diff --git a/spark/sql-20/licenses/scala-reflect-2.12.8.jar.sha1 b/spark/sql-20/licenses/scala-reflect-2.12.8.jar.sha1
diff --git a/spark/sql-30/build.gradle b/spark/sql-30/build.gradle
@@ -131,7 +131,7 @@ sparkVariants {
         String generatedJavaDirectory = "$buildDir/generated/java/${variant.name}"
         Configuration scalaCompilerPlugin = project.configurations.maybeCreate(variant.configuration('scalaCompilerPlugin'))
         scalaCompilerPlugin.defaultDependencies { dependencies ->
-            dependencies.add(project.dependencies.create("com.typesafe.genjavadoc:genjavadoc-plugin_${variant.scalaVersion}:0.18"))
+            dependencies.add(project.dependencies.create("com.typesafe.genjavadoc:genjavadoc-plugin_${variant.scalaVersion}:${variant.scalaMajorVersion != '2.11' ? '0.19' : '0.18'}"))
         }
 
         ScalaCompile compileScala = tasks.getByName(scalaCompileTaskName) as ScalaCompile

diff --git a/spark/sql-30/licenses/scala-library-2.12.20.jar.sha1 b/spark/sql-30/licenses/scala-library-2.12.20.jar.sha1
@@ -0,0 +1 @@
+562dc9629943eec010e27898c3be65ffa9210558
diff --git a/spark/sql-30/licenses/scala-library-2.12.8.jar.sha1 b/spark/sql-30/licenses/scala-library-2.12.8.jar.sha1
diff --git a/spark/sql-30/licenses/scala-reflect-2.12.20.jar.sha1 b/spark/sql-30/licenses/scala-reflect-2.12.20.jar.sha1
@@ -0,0 +1 @@
+f923933d1c168535e66ca825c18207a8db2b20a6
diff --git a/spark/sql-30/licenses/scala-reflect-2.12.8.jar.sha1 b/spark/sql-30/licenses/scala-reflect-2.12.8.jar.sha1
diff --git a/spark/sql-30/licenses/spark-catalyst_2.12-3.2.4.jar.sha1 b/spark/sql-30/licenses/spark-catalyst_2.12-3.2.4.jar.sha1
diff --git a/spark/sql-30/licenses/spark-catalyst_2.12-3.4.4.jar.sha1 b/spark/sql-30/licenses/spark-catalyst_2.12-3.4.4.jar.sha1
@@ -0,0 +1 @@
+5439fd3d28164f4486ee18bd682f2faaddc28514
diff --git a/spark/sql-30/licenses/spark-core_2.12-3.2.4.jar.sha1 b/spark/sql-30/licenses/spark-core_2.12-3.2.4.jar.sha1
diff --git a/spark/sql-30/licenses/spark-core_2.12-3.4.4.jar.sha1 b/spark/sql-30/licenses/spark-core_2.12-3.4.4.jar.sha1
@@ -0,0 +1 @@
+cf0a6995855ef82900e1e8e3cc3cd97bfcd2948c
diff --git a/spark/sql-30/licenses/spark-sql_2.12-3.2.4.jar.sha1 b/spark/sql-30/licenses/spark-sql_2.12-3.2.4.jar.sha1
diff --git a/spark/sql-30/licenses/spark-sql_2.12-3.4.4.jar.sha1 b/spark/sql-30/licenses/spark-sql_2.12-3.4.4.jar.sha1
@@ -0,0 +1 @@
+e7e720be91173317bffabf4132f664f9912aec53
diff --git a/spark/sql-30/licenses/spark-streaming_2.12-3.2.4.jar.sha1 b/spark/sql-30/licenses/spark-streaming_2.12-3.2.4.jar.sha1
diff --git a/spark/sql-30/licenses/spark-streaming_2.12-3.4.4.jar.sha1 b/spark/sql-30/licenses/spark-streaming_2.12-3.4.4.jar.sha1
@@ -0,0 +1 @@
+5c6d2acb3d60294ef95335b00f76fad4903c8cdd
diff --git a/spark/sql-30/licenses/spark-yarn_2.12-3.2.4.jar.sha1 b/spark/sql-30/licenses/spark-yarn_2.12-3.2.4.jar.sha1
diff --git a/spark/sql-30/licenses/spark-yarn_2.12-3.4.4.jar.sha1 b/spark/sql-30/licenses/spark-yarn_2.12-3.4.4.jar.sha1
@@ -0,0 +1 @@
+d76d44a3c0e58606afc6bd5850bb6e34c184e51f