Upgrade Parquet to 1.12.0

apache · Feb 25, 2021 · 799364e · 799364e
1 parent 4a3200b
commit 799364e
Show file tree

Hide file tree

Showing 4 changed files with 28 additions and 14 deletions.
diff --git a/dev/deps/spark-deps-hadoop-2.7-hive-2.3 b/dev/deps/spark-deps-hadoop-2.7-hive-2.3
@@ -203,12 +203,12 @@ orc-shims/1.6.7//orc-shims-1.6.7.jar
 oro/2.0.8//oro-2.0.8.jar
 osgi-resource-locator/1.0.3//osgi-resource-locator-1.0.3.jar
 paranamer/2.8//paranamer-2.8.jar
-parquet-column/1.11.1//parquet-column-1.11.1.jar
-parquet-common/1.11.1//parquet-common-1.11.1.jar
-parquet-encoding/1.11.1//parquet-encoding-1.11.1.jar
-parquet-format-structures/1.11.1//parquet-format-structures-1.11.1.jar
-parquet-hadoop/1.11.1//parquet-hadoop-1.11.1.jar
-parquet-jackson/1.11.1//parquet-jackson-1.11.1.jar
+parquet-column/1.12.0//parquet-column-1.12.0.jar
+parquet-common/1.12.0//parquet-common-1.12.0.jar
+parquet-encoding/1.12.0//parquet-encoding-1.12.0.jar
+parquet-format-structures/1.12.0//parquet-format-structures-1.12.0.jar
+parquet-hadoop/1.12.0//parquet-hadoop-1.12.0.jar
+parquet-jackson/1.12.0//parquet-jackson-1.12.0.jar
 protobuf-java/2.5.0//protobuf-java-2.5.0.jar
 py4j/0.10.9.1//py4j-0.10.9.1.jar
 pyrolite/4.30//pyrolite-4.30.jar

diff --git a/dev/deps/spark-deps-hadoop-3.2-hive-2.3 b/dev/deps/spark-deps-hadoop-3.2-hive-2.3
@@ -173,12 +173,12 @@ orc-shims/1.6.7//orc-shims-1.6.7.jar
 oro/2.0.8//oro-2.0.8.jar
 osgi-resource-locator/1.0.3//osgi-resource-locator-1.0.3.jar
 paranamer/2.8//paranamer-2.8.jar
-parquet-column/1.11.1//parquet-column-1.11.1.jar
-parquet-common/1.11.1//parquet-common-1.11.1.jar
-parquet-encoding/1.11.1//parquet-encoding-1.11.1.jar
-parquet-format-structures/1.11.1//parquet-format-structures-1.11.1.jar
-parquet-hadoop/1.11.1//parquet-hadoop-1.11.1.jar
-parquet-jackson/1.11.1//parquet-jackson-1.11.1.jar
+parquet-column/1.12.0//parquet-column-1.12.0.jar
+parquet-common/1.12.0//parquet-common-1.12.0.jar
+parquet-encoding/1.12.0//parquet-encoding-1.12.0.jar
+parquet-format-structures/1.12.0//parquet-format-structures-1.12.0.jar
+parquet-hadoop/1.12.0//parquet-hadoop-1.12.0.jar
+parquet-jackson/1.12.0//parquet-jackson-1.12.0.jar
 protobuf-java/2.5.0//protobuf-java-2.5.0.jar
 py4j/0.10.9.1//py4j-0.10.9.1.jar
 pyrolite/4.30//pyrolite-4.30.jar

diff --git a/pom.xml b/pom.xml
@@ -136,7 +136,7 @@
     <kafka.version>2.6.0</kafka.version>
     <!-- After 10.15.1.3, the minimum required version is JDK9 -->
     <derby.version>10.14.2.0</derby.version>
-    <parquet.version>1.11.1</parquet.version>
+    <parquet.version>1.12.0</parquet.version>
     <orc.version>1.6.7</orc.version>
     <jetty.version>9.4.36.v20210114</jetty.version>
     <jakartaservlet.version>4.0.3</jakartaservlet.version>
@@ -291,6 +291,17 @@
         <enabled>false</enabled>
       </snapshots>
     </repository>
+    <repository>
+      <id>staging</id>
+      <name>apache-staging-releases</name>
+      <url>https://repository.apache.org/content/repositories/staging/</url>
+      <releases>
+        <enabled>true</enabled>
+      </releases>
+      <snapshots>
+        <enabled>true</enabled>
+      </snapshots>
+    </repository>
   </repositories>
   <pluginRepositories>
     <pluginRepository>
@@ -2061,7 +2072,7 @@
             <groupId>${hive.group}</groupId>
             <artifactId>hive-service-rpc</artifactId>
           </exclusion>
-          <!-- parquet-hadoop-bundle:1.8.1 conflict with 1.10.1 -->
+          <!-- parquet-hadoop-bundle:1.8.1 conflict with 1.12.0 -->
           <exclusion>
             <groupId>org.apache.parquet</groupId>
             <artifactId>parquet-hadoop-bundle</artifactId>

diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
@@ -271,6 +271,9 @@ object SparkBuild extends PomBuild {
       DefaultMavenRepository,
       Resolver.mavenLocal,
       Resolver.file("ivyLocal", file(Path.userHome.absolutePath + "/.ivy2/local"))(Resolver.ivyStylePatterns)
+    ) ++ Seq(
+      "apache-staging-releases" at "https://repository.apache.org/content/repositories/staging/",
+      Resolver.file("local", file(Path.userHome.absolutePath + "/.ivy2/local"))(Resolver.ivyStylePatterns)
     ),
     externalResolvers := resolvers.value,
     otherResolvers := SbtPomKeys.mvnLocalRepository(dotM2 => Seq(Resolver.file("dotM2", dotM2))).value,