From 7f52ae62dd01ff7245a750cfcf7a442e02a7733d Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Mon, 17 Jun 2024 18:21:00 +0200 Subject: [PATCH] [SPARK-48177][BUILD] Upgrade Apache Parquet to 1.14.1 --- dev/deps/spark-deps-hadoop-3-hive-2.3 | 13 +++++++------ pom.xml | 2 +- .../parquet/ParquetVectorizedSuite.scala | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3 b/dev/deps/spark-deps-hadoop-3-hive-2.3 index c74482eb2fdb7..418530c0629eb 100644 --- a/dev/deps/spark-deps-hadoop-3-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-3-hive-2.3 @@ -108,6 +108,7 @@ jackson-core/2.17.1//jackson-core-2.17.1.jar jackson-databind/2.17.1//jackson-databind-2.17.1.jar jackson-dataformat-cbor/2.17.1//jackson-dataformat-cbor-2.17.1.jar jackson-dataformat-yaml/2.17.1//jackson-dataformat-yaml-2.17.1.jar +jackson-datatype-jdk8/2.17.0//jackson-datatype-jdk8-2.17.0.jar jackson-datatype-jsr310/2.17.1//jackson-datatype-jsr310-2.17.1.jar jackson-mapper-asl/1.9.13//jackson-mapper-asl-1.9.13.jar jackson-module-scala_2.13/2.17.1//jackson-module-scala_2.13-2.17.1.jar @@ -235,12 +236,12 @@ orc-shims/2.0.1//orc-shims-2.0.1.jar oro/2.0.8//oro-2.0.8.jar osgi-resource-locator/1.0.3//osgi-resource-locator-1.0.3.jar paranamer/2.8//paranamer-2.8.jar -parquet-column/1.13.1//parquet-column-1.13.1.jar -parquet-common/1.13.1//parquet-common-1.13.1.jar -parquet-encoding/1.13.1//parquet-encoding-1.13.1.jar -parquet-format-structures/1.13.1//parquet-format-structures-1.13.1.jar -parquet-hadoop/1.13.1//parquet-hadoop-1.13.1.jar -parquet-jackson/1.13.1//parquet-jackson-1.13.1.jar +parquet-column/1.14.1//parquet-column-1.14.1.jar +parquet-common/1.14.1//parquet-common-1.14.1.jar +parquet-encoding/1.14.1//parquet-encoding-1.14.1.jar +parquet-format-structures/1.14.1//parquet-format-structures-1.14.1.jar +parquet-hadoop/1.14.1//parquet-hadoop-1.14.1.jar +parquet-jackson/1.14.1//parquet-jackson-1.14.1.jar pickle/1.5//pickle-1.5.jar py4j/0.10.9.7//py4j-0.10.9.7.jar remotetea-oncrpc/1.1.2//remotetea-oncrpc-1.1.2.jar diff --git a/pom.xml b/pom.xml index 0c2fa604902f8..8434cc710dc4b 100644 --- a/pom.xml +++ b/pom.xml @@ -137,7 +137,7 @@ 3.7.0 10.16.1.1 - 1.13.1 + 1.14.1 2.0.1 shaded-protobuf 11.0.21 diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetVectorizedSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetVectorizedSuite.scala index 35e1a38376dd8..f2d04a9c28f2a 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetVectorizedSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetVectorizedSuite.scala @@ -502,7 +502,7 @@ class ParquetVectorizedSuite extends QueryTest with ParquetTest with SharedSpark val ty = parquetSchema.asGroupType().getType("a").asPrimitiveType() val cd = new ColumnDescriptor(Seq("a").toArray, ty, 0, maxDef) val repetitionLevels = Array.fill[Int](inputValues.length)(0) - val definitionLevels = inputValues.map(v => if (v == null) 0 else 1) + val definitionLevels = inputValues.map(v => if (v == null) 0 else maxDef) val memPageStore = new MemPageStore(expectedValues.length)