From 5b48c98fba644eaf7b0dce08eb0a0a05ed677691 Mon Sep 17 00:00:00 2001 From: Piotr Findeisen Date: Mon, 10 Jun 2024 10:46:32 +0200 Subject: [PATCH] Support building with Java 21 This is currently the latest Java LTS version, so we should support building with it. --- .github/workflows/delta-conversion-ci.yml | 4 ++-- .github/workflows/flink-ci.yml | 5 ++++- .github/workflows/hive-ci.yml | 4 ++-- .github/workflows/java-ci.yml | 6 +++--- .github/workflows/spark-ci.yml | 8 +++++++- README.md | 2 +- baseline.gradle | 5 ++++- build.gradle | 25 ++++++++++++++++++++++- jmh.gradle | 4 ++-- site/docs/contribute.md | 2 +- spark/v3.3/build.gradle | 8 ++++++++ spark/v3.4/build.gradle | 8 ++++++++ spark/v3.5/build.gradle | 8 ++++++++ 13 files changed, 74 insertions(+), 15 deletions(-) diff --git a/.github/workflows/delta-conversion-ci.yml b/.github/workflows/delta-conversion-ci.yml index c89fbb6060af..36ec5e998ad9 100644 --- a/.github/workflows/delta-conversion-ci.yml +++ b/.github/workflows/delta-conversion-ci.yml @@ -69,7 +69,7 @@ jobs: runs-on: ubuntu-22.04 strategy: matrix: - jvm: [8, 11, 17] + jvm: [8, 11, 17, 21] env: SPARK_LOCAL_IP: localhost steps: @@ -98,7 +98,7 @@ jobs: runs-on: ubuntu-22.04 strategy: matrix: - jvm: [8, 11, 17] + jvm: [8, 11, 17, 21] env: SPARK_LOCAL_IP: localhost steps: diff --git a/.github/workflows/flink-ci.yml b/.github/workflows/flink-ci.yml index e360e0f2934b..81329937acac 100644 --- a/.github/workflows/flink-ci.yml +++ b/.github/workflows/flink-ci.yml @@ -71,12 +71,15 @@ jobs: runs-on: ubuntu-22.04 strategy: matrix: - jvm: [8, 11, 17] + jvm: [8, 11, 17, 21] flink: ['1.17', '1.18', '1.19'] exclude: # Flink 1.17 does not support Java 17. - jvm: 17 flink: '1.17' + # Flink 1.17 does not support Java 21. + - jvm: 21 + flink: '1.17' env: SPARK_LOCAL_IP: localhost steps: diff --git a/.github/workflows/hive-ci.yml b/.github/workflows/hive-ci.yml index cd201a09d0fa..b93aabc246ed 100644 --- a/.github/workflows/hive-ci.yml +++ b/.github/workflows/hive-ci.yml @@ -67,7 +67,7 @@ jobs: runs-on: ubuntu-22.04 strategy: matrix: - jvm: [8, 11, 17] + jvm: [8, 11, 17, 21] env: SPARK_LOCAL_IP: localhost steps: @@ -96,7 +96,7 @@ jobs: runs-on: ubuntu-22.04 strategy: matrix: - jvm: [8, 11, 17] + jvm: [8, 11, 17, 21] env: SPARK_LOCAL_IP: localhost steps: diff --git a/.github/workflows/java-ci.yml b/.github/workflows/java-ci.yml index 22cc60d2262e..6e8b11b3d95f 100644 --- a/.github/workflows/java-ci.yml +++ b/.github/workflows/java-ci.yml @@ -63,7 +63,7 @@ jobs: runs-on: ubuntu-22.04 strategy: matrix: - jvm: [8, 11, 17] + jvm: [8, 11, 17, 21] env: SPARK_LOCAL_IP: localhost steps: @@ -92,7 +92,7 @@ jobs: runs-on: ubuntu-22.04 strategy: matrix: - jvm: [8, 11, 17] + jvm: [8, 11, 17, 21] steps: - uses: actions/checkout@v4 - uses: actions/setup-java@v4 @@ -105,7 +105,7 @@ jobs: runs-on: ubuntu-22.04 strategy: matrix: - jvm: [8, 11, 17] + jvm: [8, 11, 17, 21] steps: - uses: actions/checkout@v4 - uses: actions/setup-java@v4 diff --git a/.github/workflows/spark-ci.yml b/.github/workflows/spark-ci.yml index 5ddff7a506dd..20c93abc9045 100644 --- a/.github/workflows/spark-ci.yml +++ b/.github/workflows/spark-ci.yml @@ -69,9 +69,15 @@ jobs: runs-on: ubuntu-22.04 strategy: matrix: - jvm: [8, 11, 17] + jvm: [8, 11, 17, 21] spark: ['3.3', '3.4', '3.5'] scala: ['2.12', '2.13'] + exclude: + # Spark 3.5 is the first version supporting Java 21 (https://issues.apache.org/jira/browse/SPARK-42369) + - jvm: 21 + spark: '3.3' + - jvm: 21 + spark: '3.4' env: SPARK_LOCAL_IP: localhost steps: diff --git a/README.md b/README.md index 8200a23ed63b..fe0d2b94c308 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ Community discussions happen primarily on the [dev mailing list][dev-list] or on ### Building -Iceberg is built using Gradle with Java 8, 11, or 17. +Iceberg is built using Gradle with Java 8, 11, 17, or 21. * To invoke a build and run tests: `./gradlew build` * To skip tests: `./gradlew build -x test -x integrationTest` diff --git a/baseline.gradle b/baseline.gradle index c890ada8de57..a8c47bfaeb5d 100644 --- a/baseline.gradle +++ b/baseline.gradle @@ -45,7 +45,10 @@ subprojects { apply plugin: 'com.palantir.baseline-reproducibility' apply plugin: 'com.palantir.baseline-exact-dependencies' apply plugin: 'com.palantir.baseline-release-compatibility' - apply plugin: 'com.diffplug.spotless' + // We need to update Google Java Format to 1.17.0+ to run spotless on JDK 8, but that requires dropping support for JDK 8. + if (JavaVersion.current() != JavaVersion.VERSION_21) { + apply plugin: 'com.diffplug.spotless' + } pluginManager.withPlugin('com.palantir.baseline-checkstyle') { checkstyle { diff --git a/build.gradle b/build.gradle index 8118b7a4af5b..d75c46bebb1a 100644 --- a/build.gradle +++ b/build.gradle @@ -85,8 +85,31 @@ if (JavaVersion.current() == JavaVersion.VERSION_1_8) { "--add-opens", "java.base/sun.nio.cs=ALL-UNNAMED", "--add-opens", "java.base/sun.security.action=ALL-UNNAMED", "--add-opens", "java.base/sun.util.calendar=ALL-UNNAMED"] +} else if (JavaVersion.current() == JavaVersion.VERSION_21) { + project.ext.jdkVersion = '21' + project.ext.extraJvmArgs = ["--add-opens", "java.base/java.io=ALL-UNNAMED", + "--add-opens", "java.base/java.lang.invoke=ALL-UNNAMED", + "--add-opens", "java.base/java.lang.reflect=ALL-UNNAMED", + "--add-opens", "java.base/java.lang=ALL-UNNAMED", + "--add-opens", "java.base/java.math=ALL-UNNAMED", + "--add-opens", "java.base/java.net=ALL-UNNAMED", + "--add-opens", "java.base/java.nio=ALL-UNNAMED", + "--add-opens", "java.base/java.text=ALL-UNNAMED", + "--add-opens", "java.base/java.time=ALL-UNNAMED", + "--add-opens", "java.base/java.util.concurrent.atomic=ALL-UNNAMED", + "--add-opens", "java.base/java.util.concurrent=ALL-UNNAMED", + "--add-opens", "java.base/java.util.regex=ALL-UNNAMED", + "--add-opens", "java.base/java.util=ALL-UNNAMED", + "--add-opens", "java.base/jdk.internal.ref=ALL-UNNAMED", + "--add-opens", "java.base/jdk.internal.reflect=ALL-UNNAMED", + "--add-opens", "java.sql/java.sql=ALL-UNNAMED", + "--add-opens", "java.base/sun.util.calendar=ALL-UNNAMED", + "--add-opens", "java.base/sun.nio.ch=ALL-UNNAMED", + "--add-opens", "java.base/sun.nio.cs=ALL-UNNAMED", + "--add-opens", "java.base/sun.security.action=ALL-UNNAMED", + "--add-opens", "java.base/sun.util.calendar=ALL-UNNAMED"] } else { - throw new GradleException("This build must be run with JDK 8 or 11 or 17 but was executed with JDK " + JavaVersion.current()) + throw new GradleException("This build must be run with JDK 8 or 11 or 17 or 21 but was executed with JDK " + JavaVersion.current()) } tasks.withType(AbstractArchiveTask).configureEach { diff --git a/jmh.gradle b/jmh.gradle index de50162cb099..80f5f8d0ea63 100644 --- a/jmh.gradle +++ b/jmh.gradle @@ -17,8 +17,8 @@ * under the License. */ -if (jdkVersion != '8' && jdkVersion != '11' && jdkVersion != '17') { - throw new GradleException("The JMH benchmarks must be run with JDK 8 or JDK 11 or JDK 17") +if (jdkVersion != '8' && jdkVersion != '11' && jdkVersion != '17' && jdkVersion != '21') { + throw new GradleException("The JMH benchmarks must be run with JDK 8 or JDK 11 or JDK 17 or JDK 21") } def flinkVersions = (System.getProperty("flinkVersions") != null ? System.getProperty("flinkVersions") : System.getProperty("defaultFlinkVersions")).split(",") diff --git a/site/docs/contribute.md b/site/docs/contribute.md index d1e16e1c9939..17099c56bd79 100644 --- a/site/docs/contribute.md +++ b/site/docs/contribute.md @@ -84,7 +84,7 @@ settle disagreements or to force a decision. ## Building the Project Locally -Iceberg is built using Gradle with Java 8, 11, or 17. +Iceberg is built using Gradle with Java 8, 11, 17, or 21. * To invoke a build and run tests: `./gradlew build` * To skip tests: `./gradlew build -x test -x integrationTest` diff --git a/spark/v3.3/build.gradle b/spark/v3.3/build.gradle index 2f1ddef0a325..a30f2a60929f 100644 --- a/spark/v3.3/build.gradle +++ b/spark/v3.3/build.gradle @@ -59,6 +59,10 @@ project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}") { implementation project(':iceberg-parquet') implementation project(':iceberg-arrow') implementation("org.scala-lang.modules:scala-collection-compat_${scalaVersion}:${libs.versions.scala.collection.compat.get()}") + if (scalaVersion == '2.12') { + // scala-collection-compat_2.12 pulls scala 2.12.17 and we need 2.12.18 for JDK 21 support + implementation 'org.scala-lang:scala-library:2.12.18' + } compileOnly libs.errorprone.annotations compileOnly libs.avro.avro @@ -136,6 +140,10 @@ project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVer dependencies { implementation "org.scala-lang.modules:scala-collection-compat_${scalaVersion}:${libs.versions.scala.collection.compat.get()}" + if (scalaVersion == '2.12') { + // scala-collection-compat_2.12 pulls scala 2.12.17 and we need 2.12.18 for JDK 21 support + implementation 'org.scala-lang:scala-library:2.12.18' + } implementation libs.roaringbitmap compileOnly "org.scala-lang:scala-library" diff --git a/spark/v3.4/build.gradle b/spark/v3.4/build.gradle index 1da53ecfc61d..960cd1ef13bc 100644 --- a/spark/v3.4/build.gradle +++ b/spark/v3.4/build.gradle @@ -59,6 +59,10 @@ project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}") { implementation project(':iceberg-parquet') implementation project(':iceberg-arrow') implementation("org.scala-lang.modules:scala-collection-compat_${scalaVersion}:${libs.versions.scala.collection.compat.get()}") + if (scalaVersion == '2.12') { + // scala-collection-compat_2.12 pulls scala 2.12.17 and we need 2.12.18 for JDK 21 support + implementation 'org.scala-lang:scala-library:2.12.18' + } compileOnly libs.errorprone.annotations compileOnly libs.avro.avro @@ -137,6 +141,10 @@ project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVer dependencies { implementation("org.scala-lang.modules:scala-collection-compat_${scalaVersion}:${libs.versions.scala.collection.compat.get()}") + if (scalaVersion == '2.12') { + // scala-collection-compat_2.12 pulls scala 2.12.17 and we need 2.12.18 for JDK 21 support + implementation 'org.scala-lang:scala-library:2.12.18' + } implementation libs.roaringbitmap compileOnly "org.scala-lang:scala-library" diff --git a/spark/v3.5/build.gradle b/spark/v3.5/build.gradle index f1af2120ef8c..2ba5d493c6cd 100644 --- a/spark/v3.5/build.gradle +++ b/spark/v3.5/build.gradle @@ -59,6 +59,10 @@ project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}") { implementation project(':iceberg-parquet') implementation project(':iceberg-arrow') implementation("org.scala-lang.modules:scala-collection-compat_${scalaVersion}:${libs.versions.scala.collection.compat.get()}") + if (scalaVersion == '2.12') { + // scala-collection-compat_2.12 pulls scala 2.12.17 and we need 2.12.18 for JDK 21 support + implementation 'org.scala-lang:scala-library:2.12.18' + } compileOnly libs.errorprone.annotations compileOnly libs.avro.avro @@ -137,6 +141,10 @@ project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVer dependencies { implementation("org.scala-lang.modules:scala-collection-compat_${scalaVersion}:${libs.versions.scala.collection.compat.get()}") + if (scalaVersion == '2.12') { + // scala-collection-compat_2.12 pulls scala 2.12.17 and we need 2.12.18 for JDK 21 support + implementation 'org.scala-lang:scala-library:2.12.18' + } implementation libs.roaringbitmap compileOnly "org.scala-lang:scala-library"