diff --git a/.github/workflows/delta-conversion-ci.yml b/.github/workflows/delta-conversion-ci.yml index c89fbb6060af..36ec5e998ad9 100644 --- a/.github/workflows/delta-conversion-ci.yml +++ b/.github/workflows/delta-conversion-ci.yml @@ -69,7 +69,7 @@ jobs: runs-on: ubuntu-22.04 strategy: matrix: - jvm: [8, 11, 17] + jvm: [8, 11, 17, 21] env: SPARK_LOCAL_IP: localhost steps: @@ -98,7 +98,7 @@ jobs: runs-on: ubuntu-22.04 strategy: matrix: - jvm: [8, 11, 17] + jvm: [8, 11, 17, 21] env: SPARK_LOCAL_IP: localhost steps: diff --git a/.github/workflows/flink-ci.yml b/.github/workflows/flink-ci.yml index e360e0f2934b..81329937acac 100644 --- a/.github/workflows/flink-ci.yml +++ b/.github/workflows/flink-ci.yml @@ -71,12 +71,15 @@ jobs: runs-on: ubuntu-22.04 strategy: matrix: - jvm: [8, 11, 17] + jvm: [8, 11, 17, 21] flink: ['1.17', '1.18', '1.19'] exclude: # Flink 1.17 does not support Java 17. - jvm: 17 flink: '1.17' + # Flink 1.17 does not support Java 21. + - jvm: 21 + flink: '1.17' env: SPARK_LOCAL_IP: localhost steps: diff --git a/.github/workflows/hive-ci.yml b/.github/workflows/hive-ci.yml index cd201a09d0fa..b93aabc246ed 100644 --- a/.github/workflows/hive-ci.yml +++ b/.github/workflows/hive-ci.yml @@ -67,7 +67,7 @@ jobs: runs-on: ubuntu-22.04 strategy: matrix: - jvm: [8, 11, 17] + jvm: [8, 11, 17, 21] env: SPARK_LOCAL_IP: localhost steps: @@ -96,7 +96,7 @@ jobs: runs-on: ubuntu-22.04 strategy: matrix: - jvm: [8, 11, 17] + jvm: [8, 11, 17, 21] env: SPARK_LOCAL_IP: localhost steps: diff --git a/.github/workflows/java-ci.yml b/.github/workflows/java-ci.yml index 22cc60d2262e..6e8b11b3d95f 100644 --- a/.github/workflows/java-ci.yml +++ b/.github/workflows/java-ci.yml @@ -63,7 +63,7 @@ jobs: runs-on: ubuntu-22.04 strategy: matrix: - jvm: [8, 11, 17] + jvm: [8, 11, 17, 21] env: SPARK_LOCAL_IP: localhost steps: @@ -92,7 +92,7 @@ jobs: runs-on: ubuntu-22.04 strategy: matrix: - jvm: [8, 11, 17] + jvm: [8, 11, 17, 21] steps: - uses: actions/checkout@v4 - uses: actions/setup-java@v4 @@ -105,7 +105,7 @@ jobs: runs-on: ubuntu-22.04 strategy: matrix: - jvm: [8, 11, 17] + jvm: [8, 11, 17, 21] steps: - uses: actions/checkout@v4 - uses: actions/setup-java@v4 diff --git a/.github/workflows/spark-ci.yml b/.github/workflows/spark-ci.yml index 5ddff7a506dd..51e6d594ba62 100644 --- a/.github/workflows/spark-ci.yml +++ b/.github/workflows/spark-ci.yml @@ -69,9 +69,16 @@ jobs: runs-on: ubuntu-22.04 strategy: matrix: - jvm: [8, 11, 17] + jvm: [8, 11, 17, 21] spark: ['3.3', '3.4', '3.5'] scala: ['2.12', '2.13'] + exclude: + # Spark 3.5 is the first version not failing on Java 21 (https://issues.apache.org/jira/browse/SPARK-42369) + # Full Java 21 support is coming in Spark 4 (https://issues.apache.org/jira/browse/SPARK-43831) + - jvm: 21 + spark: '3.3' + - jvm: 21 + spark: '3.4' env: SPARK_LOCAL_IP: localhost steps: diff --git a/README.md b/README.md index 8200a23ed63b..fe0d2b94c308 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ Community discussions happen primarily on the [dev mailing list][dev-list] or on ### Building -Iceberg is built using Gradle with Java 8, 11, or 17. +Iceberg is built using Gradle with Java 8, 11, 17, or 21. * To invoke a build and run tests: `./gradlew build` * To skip tests: `./gradlew build -x test -x integrationTest` diff --git a/baseline.gradle b/baseline.gradle index c890ada8de57..62ade9a632f4 100644 --- a/baseline.gradle +++ b/baseline.gradle @@ -45,7 +45,16 @@ subprojects { apply plugin: 'com.palantir.baseline-reproducibility' apply plugin: 'com.palantir.baseline-exact-dependencies' apply plugin: 'com.palantir.baseline-release-compatibility' - apply plugin: 'com.diffplug.spotless' + // We need to update Google Java Format to 1.17.0+ to run spotless on JDK 8, but that requires dropping support for JDK 8. + if (JavaVersion.current() == JavaVersion.VERSION_21) { + task spotlessApply { + doLast { + throw new GradleException("Spotless plugin is currently disabled when running on JDK 21 (until we drop JDK 8). To run spotlessApply please use a different JDK version.") + } + } + } else { + apply plugin: 'com.diffplug.spotless' + } pluginManager.withPlugin('com.palantir.baseline-checkstyle') { checkstyle { diff --git a/build.gradle b/build.gradle index 8118b7a4af5b..27504aba7dcb 100644 --- a/build.gradle +++ b/build.gradle @@ -62,8 +62,8 @@ if (JavaVersion.current() == JavaVersion.VERSION_1_8) { } else if (JavaVersion.current() == JavaVersion.VERSION_11) { project.ext.jdkVersion = '11' project.ext.extraJvmArgs = [] -} else if (JavaVersion.current() == JavaVersion.VERSION_17) { - project.ext.jdkVersion = '17' +} else if (JavaVersion.current() == JavaVersion.VERSION_17 || JavaVersion.current() == JavaVersion.VERSION_21) { + project.ext.jdkVersion = JavaVersion.current().getMajorVersion().toString() project.ext.extraJvmArgs = ["--add-opens", "java.base/java.io=ALL-UNNAMED", "--add-opens", "java.base/java.lang.invoke=ALL-UNNAMED", "--add-opens", "java.base/java.lang.reflect=ALL-UNNAMED", @@ -86,7 +86,7 @@ if (JavaVersion.current() == JavaVersion.VERSION_1_8) { "--add-opens", "java.base/sun.security.action=ALL-UNNAMED", "--add-opens", "java.base/sun.util.calendar=ALL-UNNAMED"] } else { - throw new GradleException("This build must be run with JDK 8 or 11 or 17 but was executed with JDK " + JavaVersion.current()) + throw new GradleException("This build must be run with JDK 8 or 11 or 17 or 21 but was executed with JDK " + JavaVersion.current()) } tasks.withType(AbstractArchiveTask).configureEach { diff --git a/jmh.gradle b/jmh.gradle index de50162cb099..80f5f8d0ea63 100644 --- a/jmh.gradle +++ b/jmh.gradle @@ -17,8 +17,8 @@ * under the License. */ -if (jdkVersion != '8' && jdkVersion != '11' && jdkVersion != '17') { - throw new GradleException("The JMH benchmarks must be run with JDK 8 or JDK 11 or JDK 17") +if (jdkVersion != '8' && jdkVersion != '11' && jdkVersion != '17' && jdkVersion != '21') { + throw new GradleException("The JMH benchmarks must be run with JDK 8 or JDK 11 or JDK 17 or JDK 21") } def flinkVersions = (System.getProperty("flinkVersions") != null ? System.getProperty("flinkVersions") : System.getProperty("defaultFlinkVersions")).split(",") diff --git a/site/docs/contribute.md b/site/docs/contribute.md index d1e16e1c9939..17099c56bd79 100644 --- a/site/docs/contribute.md +++ b/site/docs/contribute.md @@ -84,7 +84,7 @@ settle disagreements or to force a decision. ## Building the Project Locally -Iceberg is built using Gradle with Java 8, 11, or 17. +Iceberg is built using Gradle with Java 8, 11, 17, or 21. * To invoke a build and run tests: `./gradlew build` * To skip tests: `./gradlew build -x test -x integrationTest` diff --git a/spark/v3.3/build.gradle b/spark/v3.3/build.gradle index 2f1ddef0a325..a30f2a60929f 100644 --- a/spark/v3.3/build.gradle +++ b/spark/v3.3/build.gradle @@ -59,6 +59,10 @@ project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}") { implementation project(':iceberg-parquet') implementation project(':iceberg-arrow') implementation("org.scala-lang.modules:scala-collection-compat_${scalaVersion}:${libs.versions.scala.collection.compat.get()}") + if (scalaVersion == '2.12') { + // scala-collection-compat_2.12 pulls scala 2.12.17 and we need 2.12.18 for JDK 21 support + implementation 'org.scala-lang:scala-library:2.12.18' + } compileOnly libs.errorprone.annotations compileOnly libs.avro.avro @@ -136,6 +140,10 @@ project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVer dependencies { implementation "org.scala-lang.modules:scala-collection-compat_${scalaVersion}:${libs.versions.scala.collection.compat.get()}" + if (scalaVersion == '2.12') { + // scala-collection-compat_2.12 pulls scala 2.12.17 and we need 2.12.18 for JDK 21 support + implementation 'org.scala-lang:scala-library:2.12.18' + } implementation libs.roaringbitmap compileOnly "org.scala-lang:scala-library" diff --git a/spark/v3.4/build.gradle b/spark/v3.4/build.gradle index 1da53ecfc61d..960cd1ef13bc 100644 --- a/spark/v3.4/build.gradle +++ b/spark/v3.4/build.gradle @@ -59,6 +59,10 @@ project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}") { implementation project(':iceberg-parquet') implementation project(':iceberg-arrow') implementation("org.scala-lang.modules:scala-collection-compat_${scalaVersion}:${libs.versions.scala.collection.compat.get()}") + if (scalaVersion == '2.12') { + // scala-collection-compat_2.12 pulls scala 2.12.17 and we need 2.12.18 for JDK 21 support + implementation 'org.scala-lang:scala-library:2.12.18' + } compileOnly libs.errorprone.annotations compileOnly libs.avro.avro @@ -137,6 +141,10 @@ project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVer dependencies { implementation("org.scala-lang.modules:scala-collection-compat_${scalaVersion}:${libs.versions.scala.collection.compat.get()}") + if (scalaVersion == '2.12') { + // scala-collection-compat_2.12 pulls scala 2.12.17 and we need 2.12.18 for JDK 21 support + implementation 'org.scala-lang:scala-library:2.12.18' + } implementation libs.roaringbitmap compileOnly "org.scala-lang:scala-library" diff --git a/spark/v3.5/build.gradle b/spark/v3.5/build.gradle index f1af2120ef8c..2ba5d493c6cd 100644 --- a/spark/v3.5/build.gradle +++ b/spark/v3.5/build.gradle @@ -59,6 +59,10 @@ project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}") { implementation project(':iceberg-parquet') implementation project(':iceberg-arrow') implementation("org.scala-lang.modules:scala-collection-compat_${scalaVersion}:${libs.versions.scala.collection.compat.get()}") + if (scalaVersion == '2.12') { + // scala-collection-compat_2.12 pulls scala 2.12.17 and we need 2.12.18 for JDK 21 support + implementation 'org.scala-lang:scala-library:2.12.18' + } compileOnly libs.errorprone.annotations compileOnly libs.avro.avro @@ -137,6 +141,10 @@ project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVer dependencies { implementation("org.scala-lang.modules:scala-collection-compat_${scalaVersion}:${libs.versions.scala.collection.compat.get()}") + if (scalaVersion == '2.12') { + // scala-collection-compat_2.12 pulls scala 2.12.17 and we need 2.12.18 for JDK 21 support + implementation 'org.scala-lang:scala-library:2.12.18' + } implementation libs.roaringbitmap compileOnly "org.scala-lang:scala-library"