Merge branch 'master' of github.com:apache/spark

.github/workflows/benchmark.yml

@@ -91,7 +91,7 @@ jobs:
         # To keep the directory structure and file permissions, tar them
         # See also https://github.com/actions/upload-artifact#maintaining-file-permissions-and-case-sensitive-files
         echo "Preparing the benchmark results:"
-        tar -cvf benchmark-results-${{ github.event.inputs.jdk }}.tar `git diff --name-only`
+        tar -cvf benchmark-results-${{ github.event.inputs.jdk }}.tar `git diff --name-only` `git ls-files --others --exclude-standard`
     - name: Upload benchmark results
       uses: actions/upload-artifact@v2
       with:

.github/workflows/build_and_test.yml

@@ -493,19 +493,6 @@ jobs:
     steps:
     - name: Checkout Spark repository
       uses: actions/checkout@v2
-    - name: Cache TPC-DS generated data
-      id: cache-tpcds-sf-1
-      uses: actions/cache@v2
-      with:
-        path: ./tpcds-sf-1
-        key: tpcds-556111e35d400f56cb0625dc16e9063d54628320
-    - name: Checkout TPC-DS (SF=1) generated data repository
-      if: steps.cache-tpcds-sf-1.outputs.cache-hit != 'true'
-      uses: actions/checkout@v2
-      with:
-        repository: maropu/spark-tpcds-sf-1
-        ref: 556111e35d400f56cb0625dc16e9063d54628320
-        path: ./tpcds-sf-1
     - name: Cache Scala, SBT and Maven
       uses: actions/cache@v2
       with:
@@ -528,6 +515,24 @@ jobs:
       uses: actions/setup-java@v1
       with:
         java-version: 8
+    - name: Cache TPC-DS generated data
+      id: cache-tpcds-sf-1
+      uses: actions/cache@v2
+      with:
+        path: ./tpcds-sf-1
+        key: tpcds-${{ hashFiles('.github/workflows/build_and_test.yml', 'sql/core/src/test/scala/org/apache/spark/sql/TPCDSSchema.scala') }}
+    - name: Checkout tpcds-kit repository
+      if: steps.cache-tpcds-sf-1.outputs.cache-hit != 'true'
+      uses: actions/checkout@v2
+      with:
+        repository: databricks/tpcds-kit
+        path: ./tpcds-kit
+    - name: Build tpcds-kit
+      if: steps.cache-tpcds-sf-1.outputs.cache-hit != 'true'
+      run: cd tpcds-kit/tools && make OS=LINUX
+    - name: Generate TPC-DS (SF=1) table data
+      if: steps.cache-tpcds-sf-1.outputs.cache-hit != 'true'
+      run: build/sbt "sql/test:runMain org.apache.spark.sql.GenTPCDSData --dsdgenDir `pwd`/tpcds-kit/tools --location `pwd`/tpcds-sf-1 --scaleFactor 1 --numPartitions 1 --overwrite"
     - name: Run TPC-DS queries
       run: |
         SPARK_TPCDS_DATA=`pwd`/tpcds-sf-1 build/sbt "sql/testOnly org.apache.spark.sql.TPCDSQueryTestSuite"

LICENSE-binary

@@ -218,7 +218,6 @@ javax.jdo:jdo-api
 joda-time:joda-time
 net.sf.opencsv:opencsv
 org.apache.derby:derby
-org.ehcache:ehcache
 org.objenesis:objenesis
 org.roaringbitmap:RoaringBitmap
 org.scalanlp:breeze-macros_2.12
@@ -261,7 +260,6 @@ net.sf.supercsv:super-csv
 org.apache.arrow:arrow-format
 org.apache.arrow:arrow-memory
 org.apache.arrow:arrow-vector
-org.apache.commons:commons-configuration2
 org.apache.commons:commons-crypto
 org.apache.commons:commons-lang3
 org.apache.hadoop:hadoop-annotations
@@ -296,7 +294,6 @@ org.apache.kerby:kerby-config
 org.apache.kerby:kerby-pkix
 org.apache.kerby:kerby-util
 org.apache.kerby:kerby-xdr
-org.apache.kerby:token-provider
 org.apache.orc:orc-core
 org.apache.orc:orc-mapreduce
 org.mortbay.jetty:jetty
@@ -316,19 +313,15 @@ com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider
 com.fasterxml.jackson.module:jackson-module-jaxb-annotations
 com.fasterxml.jackson.module:jackson-module-paranamer
 com.fasterxml.jackson.module:jackson-module-scala_2.12
-com.fasterxml.woodstox:woodstox-core
 com.github.mifmif:generex
-com.github.stephenc.jcip:jcip-annotations
 com.google.code.findbugs:jsr305
 com.google.code.gson:gson
 com.google.flatbuffers:flatbuffers-java
 com.google.guava:guava
 com.google.inject:guice
 com.google.inject.extensions:guice-servlet
-com.nimbusds:nimbus-jose-jwt
 com.twitter:parquet-hadoop-bundle
 commons-cli:commons-cli
-commons-daemon:commons-daemon
 commons-dbcp:commons-dbcp
 commons-io:commons-io
 commons-lang:commons-lang
@@ -340,8 +333,6 @@ javax.inject:javax.inject
 javax.validation:validation-api
 log4j:apache-log4j-extras
 log4j:log4j
-net.minidev:accessors-smart
-net.minidev:json-smart
 net.sf.jpam:jpam
 org.apache.avro:avro
 org.apache.avro:avro-ipc
@@ -357,7 +348,6 @@ org.apache.directory.server:apacheds-i18n
 org.apache.directory.server:apacheds-kerberos-codec
 org.apache.htrace:htrace-core
 org.apache.ivy:ivy
-org.apache.geronimo.specs:geronimo-jcache_1.0_spec
 org.apache.mesos:mesos
 org.apache.parquet:parquet-column
 org.apache.parquet:parquet-common
@@ -432,15 +422,13 @@ BSD 2-Clause
 ------------
 
 com.github.luben:zstd-jni
-dnsjava:dnsjava
 javolution:javolution
 com.esotericsoftware:kryo-shaded
 com.esotericsoftware:minlog
 com.esotericsoftware:reflectasm
 com.google.protobuf:protobuf-java
 org.codehaus.janino:commons-compiler
 org.codehaus.janino:janino
-org.codehaus.woodstox:stax2-api
 jline:jline
 org.jodd:jodd-core
 com.github.wendykierp:JTransforms
@@ -456,7 +444,6 @@ org.antlr:ST4
 org.antlr:stringtemplate
 org.antlr:antlr4-runtime
 antlr:antlr
-com.google.re2j:re2j
 com.thoughtworks.paranamer:paranamer
 org.scala-lang:scala-compiler
 org.scala-lang:scala-library

core/src/test/scala/org/apache/spark/benchmark/BenchmarkBase.scala

@@ -19,6 +19,8 @@ package org.apache.spark.benchmark
 
 import java.io.{File, FileOutputStream, OutputStream}
 
+import org.apache.spark.internal.config.Tests.IS_TESTING
+
 /**
  * A base class for generate benchmark results to a file.
  * For JDK9+, JDK major version number is added to the file names to distinguish the results.
@@ -42,14 +44,25 @@ abstract class BenchmarkBase {
   }
 
   def main(args: Array[String]): Unit = {
+    // turning this on so the behavior between running benchmark via `spark-submit` or SBT will
+    // be consistent, also allow users to turn on/off certain behavior such as
+    // `spark.sql.codegen.factoryMode`
+    System.setProperty(IS_TESTING.key, "true")
     val regenerateBenchmarkFiles: Boolean = System.getenv("SPARK_GENERATE_BENCHMARK_FILES") == "1"
     if (regenerateBenchmarkFiles) {
       val version = System.getProperty("java.version").split("\\D+")(0).toInt
       val jdkString = if (version > 8) s"-jdk$version" else ""
       val resultFileName =
         s"${this.getClass.getSimpleName.replace("$", "")}$jdkString$suffix-results.txt"
       val prefix = Benchmarks.currentProjectRoot.map(_ + "/").getOrElse("")
-      val file = new File(s"${prefix}benchmarks/$resultFileName")
+      val dir = new File(s"${prefix}benchmarks/")
+      if (!dir.exists()) {
+        // scalastyle:off println
+        println(s"Creating ${dir.getAbsolutePath} for benchmark results.")
+        // scalastyle:on println
+        dir.mkdirs()
+      }
+      val file = new File(dir, resultFileName)
       if (!file.exists()) {
         file.createNewFile()
       }

dev/deps/spark-deps-hadoop-2.7-hive-2.3

@@ -131,13 +131,12 @@ jaxb-api/2.2.11//jaxb-api-2.2.11.jar
 jaxb-runtime/2.3.2//jaxb-runtime-2.3.2.jar
 jcl-over-slf4j/1.7.30//jcl-over-slf4j-1.7.30.jar
 jdo-api/3.0.1//jdo-api-3.0.1.jar
-jersey-client/2.30//jersey-client-2.30.jar
-jersey-common/2.30//jersey-common-2.30.jar
-jersey-container-servlet-core/2.30//jersey-container-servlet-core-2.30.jar
-jersey-container-servlet/2.30//jersey-container-servlet-2.30.jar
-jersey-hk2/2.30//jersey-hk2-2.30.jar
-jersey-media-jaxb/2.30//jersey-media-jaxb-2.30.jar
-jersey-server/2.30//jersey-server-2.30.jar
+jersey-client/2.34//jersey-client-2.34.jar
+jersey-common/2.34//jersey-common-2.34.jar
+jersey-container-servlet-core/2.34//jersey-container-servlet-core-2.34.jar
+jersey-container-servlet/2.34//jersey-container-servlet-2.34.jar
+jersey-hk2/2.34//jersey-hk2-2.34.jar
+jersey-server/2.34//jersey-server-2.34.jar
 jetty-sslengine/6.1.26//jetty-sslengine-6.1.26.jar
 jetty-util/6.1.26//jetty-util-6.1.26.jar
 jetty/6.1.26//jetty-6.1.26.jar
@@ -155,26 +154,26 @@ jsr305/3.0.0//jsr305-3.0.0.jar
 jta/1.1//jta-1.1.jar
 jul-to-slf4j/1.7.30//jul-to-slf4j-1.7.30.jar
 kryo-shaded/4.0.2//kryo-shaded-4.0.2.jar
-kubernetes-client/5.3.0//kubernetes-client-5.3.0.jar
-kubernetes-model-admissionregistration/5.3.0//kubernetes-model-admissionregistration-5.3.0.jar
-kubernetes-model-apiextensions/5.3.0//kubernetes-model-apiextensions-5.3.0.jar
-kubernetes-model-apps/5.3.0//kubernetes-model-apps-5.3.0.jar
-kubernetes-model-autoscaling/5.3.0//kubernetes-model-autoscaling-5.3.0.jar
-kubernetes-model-batch/5.3.0//kubernetes-model-batch-5.3.0.jar
-kubernetes-model-certificates/5.3.0//kubernetes-model-certificates-5.3.0.jar
-kubernetes-model-common/5.3.0//kubernetes-model-common-5.3.0.jar
-kubernetes-model-coordination/5.3.0//kubernetes-model-coordination-5.3.0.jar
-kubernetes-model-core/5.3.0//kubernetes-model-core-5.3.0.jar
-kubernetes-model-discovery/5.3.0//kubernetes-model-discovery-5.3.0.jar
-kubernetes-model-events/5.3.0//kubernetes-model-events-5.3.0.jar
-kubernetes-model-extensions/5.3.0//kubernetes-model-extensions-5.3.0.jar
-kubernetes-model-metrics/5.3.0//kubernetes-model-metrics-5.3.0.jar
-kubernetes-model-networking/5.3.0//kubernetes-model-networking-5.3.0.jar
-kubernetes-model-node/5.3.0//kubernetes-model-node-5.3.0.jar
-kubernetes-model-policy/5.3.0//kubernetes-model-policy-5.3.0.jar
-kubernetes-model-rbac/5.3.0//kubernetes-model-rbac-5.3.0.jar
-kubernetes-model-scheduling/5.3.0//kubernetes-model-scheduling-5.3.0.jar
-kubernetes-model-storageclass/5.3.0//kubernetes-model-storageclass-5.3.0.jar
+kubernetes-client/5.3.1//kubernetes-client-5.3.1.jar
+kubernetes-model-admissionregistration/5.3.1//kubernetes-model-admissionregistration-5.3.1.jar
+kubernetes-model-apiextensions/5.3.1//kubernetes-model-apiextensions-5.3.1.jar
+kubernetes-model-apps/5.3.1//kubernetes-model-apps-5.3.1.jar
+kubernetes-model-autoscaling/5.3.1//kubernetes-model-autoscaling-5.3.1.jar
+kubernetes-model-batch/5.3.1//kubernetes-model-batch-5.3.1.jar
+kubernetes-model-certificates/5.3.1//kubernetes-model-certificates-5.3.1.jar
+kubernetes-model-common/5.3.1//kubernetes-model-common-5.3.1.jar
+kubernetes-model-coordination/5.3.1//kubernetes-model-coordination-5.3.1.jar
+kubernetes-model-core/5.3.1//kubernetes-model-core-5.3.1.jar
+kubernetes-model-discovery/5.3.1//kubernetes-model-discovery-5.3.1.jar
+kubernetes-model-events/5.3.1//kubernetes-model-events-5.3.1.jar
+kubernetes-model-extensions/5.3.1//kubernetes-model-extensions-5.3.1.jar
+kubernetes-model-metrics/5.3.1//kubernetes-model-metrics-5.3.1.jar
+kubernetes-model-networking/5.3.1//kubernetes-model-networking-5.3.1.jar
+kubernetes-model-node/5.3.1//kubernetes-model-node-5.3.1.jar
+kubernetes-model-policy/5.3.1//kubernetes-model-policy-5.3.1.jar
+kubernetes-model-rbac/5.3.1//kubernetes-model-rbac-5.3.1.jar
+kubernetes-model-scheduling/5.3.1//kubernetes-model-scheduling-5.3.1.jar
+kubernetes-model-storageclass/5.3.1//kubernetes-model-storageclass-5.3.1.jar
 lapack/2.0.0//lapack-2.0.0.jar
 leveldbjni-all/1.8//leveldbjni-all-1.8.jar
 libfb303/0.9.3//libfb303-0.9.3.jar

dev/deps/spark-deps-hadoop-3.2-hive-2.3

@@ -106,13 +106,12 @@ jaxb-api/2.2.11//jaxb-api-2.2.11.jar
 jaxb-runtime/2.3.2//jaxb-runtime-2.3.2.jar
 jcl-over-slf4j/1.7.30//jcl-over-slf4j-1.7.30.jar
 jdo-api/3.0.1//jdo-api-3.0.1.jar
-jersey-client/2.30//jersey-client-2.30.jar
-jersey-common/2.30//jersey-common-2.30.jar
-jersey-container-servlet-core/2.30//jersey-container-servlet-core-2.30.jar
-jersey-container-servlet/2.30//jersey-container-servlet-2.30.jar
-jersey-hk2/2.30//jersey-hk2-2.30.jar
-jersey-media-jaxb/2.30//jersey-media-jaxb-2.30.jar
-jersey-server/2.30//jersey-server-2.30.jar
+jersey-client/2.34//jersey-client-2.34.jar
+jersey-common/2.34//jersey-common-2.34.jar
+jersey-container-servlet-core/2.34//jersey-container-servlet-core-2.34.jar
+jersey-container-servlet/2.34//jersey-container-servlet-2.34.jar
+jersey-hk2/2.34//jersey-hk2-2.34.jar
+jersey-server/2.34//jersey-server-2.34.jar
 jline/2.14.6//jline-2.14.6.jar
 joda-time/2.10.5//joda-time-2.10.5.jar
 jodd-core/3.5.2//jodd-core-3.5.2.jar
@@ -126,26 +125,26 @@ jsr305/3.0.0//jsr305-3.0.0.jar
 jta/1.1//jta-1.1.jar
 jul-to-slf4j/1.7.30//jul-to-slf4j-1.7.30.jar
 kryo-shaded/4.0.2//kryo-shaded-4.0.2.jar
-kubernetes-client/5.3.0//kubernetes-client-5.3.0.jar
-kubernetes-model-admissionregistration/5.3.0//kubernetes-model-admissionregistration-5.3.0.jar
-kubernetes-model-apiextensions/5.3.0//kubernetes-model-apiextensions-5.3.0.jar
-kubernetes-model-apps/5.3.0//kubernetes-model-apps-5.3.0.jar
-kubernetes-model-autoscaling/5.3.0//kubernetes-model-autoscaling-5.3.0.jar
-kubernetes-model-batch/5.3.0//kubernetes-model-batch-5.3.0.jar
-kubernetes-model-certificates/5.3.0//kubernetes-model-certificates-5.3.0.jar
-kubernetes-model-common/5.3.0//kubernetes-model-common-5.3.0.jar
-kubernetes-model-coordination/5.3.0//kubernetes-model-coordination-5.3.0.jar
-kubernetes-model-core/5.3.0//kubernetes-model-core-5.3.0.jar
-kubernetes-model-discovery/5.3.0//kubernetes-model-discovery-5.3.0.jar
-kubernetes-model-events/5.3.0//kubernetes-model-events-5.3.0.jar
-kubernetes-model-extensions/5.3.0//kubernetes-model-extensions-5.3.0.jar
-kubernetes-model-metrics/5.3.0//kubernetes-model-metrics-5.3.0.jar
-kubernetes-model-networking/5.3.0//kubernetes-model-networking-5.3.0.jar
-kubernetes-model-node/5.3.0//kubernetes-model-node-5.3.0.jar
-kubernetes-model-policy/5.3.0//kubernetes-model-policy-5.3.0.jar
-kubernetes-model-rbac/5.3.0//kubernetes-model-rbac-5.3.0.jar
-kubernetes-model-scheduling/5.3.0//kubernetes-model-scheduling-5.3.0.jar
-kubernetes-model-storageclass/5.3.0//kubernetes-model-storageclass-5.3.0.jar
+kubernetes-client/5.3.1//kubernetes-client-5.3.1.jar
+kubernetes-model-admissionregistration/5.3.1//kubernetes-model-admissionregistration-5.3.1.jar
+kubernetes-model-apiextensions/5.3.1//kubernetes-model-apiextensions-5.3.1.jar
+kubernetes-model-apps/5.3.1//kubernetes-model-apps-5.3.1.jar
+kubernetes-model-autoscaling/5.3.1//kubernetes-model-autoscaling-5.3.1.jar
+kubernetes-model-batch/5.3.1//kubernetes-model-batch-5.3.1.jar
+kubernetes-model-certificates/5.3.1//kubernetes-model-certificates-5.3.1.jar
+kubernetes-model-common/5.3.1//kubernetes-model-common-5.3.1.jar
+kubernetes-model-coordination/5.3.1//kubernetes-model-coordination-5.3.1.jar
+kubernetes-model-core/5.3.1//kubernetes-model-core-5.3.1.jar
+kubernetes-model-discovery/5.3.1//kubernetes-model-discovery-5.3.1.jar
+kubernetes-model-events/5.3.1//kubernetes-model-events-5.3.1.jar
+kubernetes-model-extensions/5.3.1//kubernetes-model-extensions-5.3.1.jar
+kubernetes-model-metrics/5.3.1//kubernetes-model-metrics-5.3.1.jar
+kubernetes-model-networking/5.3.1//kubernetes-model-networking-5.3.1.jar
+kubernetes-model-node/5.3.1//kubernetes-model-node-5.3.1.jar
+kubernetes-model-policy/5.3.1//kubernetes-model-policy-5.3.1.jar
+kubernetes-model-rbac/5.3.1//kubernetes-model-rbac-5.3.1.jar
+kubernetes-model-scheduling/5.3.1//kubernetes-model-scheduling-5.3.1.jar
+kubernetes-model-storageclass/5.3.1//kubernetes-model-storageclass-5.3.1.jar
 lapack/2.0.0//lapack-2.0.0.jar
 leveldbjni-all/1.8//leveldbjni-all-1.8.jar
 libfb303/0.9.3//libfb303-0.9.3.jar

docs/sql-ref-ansi-compliance.md

@@ -254,6 +254,7 @@ Below is a list of all the keywords in Spark SQL.
 |DATA|non-reserved|non-reserved|non-reserved|
 |DATABASE|non-reserved|non-reserved|non-reserved|
 |DATABASES|non-reserved|non-reserved|non-reserved|
+|DAY|non-reserved|non-reserved|non-reserved|
 |DBPROPERTIES|non-reserved|non-reserved|non-reserved|
 |DEFINED|non-reserved|non-reserved|non-reserved|
 |DELETE|non-reserved|non-reserved|reserved|
@@ -336,6 +337,7 @@ Below is a list of all the keywords in Spark SQL.
 |MATCHED|non-reserved|non-reserved|non-reserved|
 |MERGE|non-reserved|non-reserved|non-reserved|
 |MINUS|non-reserved|strict-non-reserved|non-reserved|
+|MONTH|non-reserved|non-reserved|non-reserved|
 |MSCK|non-reserved|non-reserved|non-reserved|
 |NAMESPACE|non-reserved|non-reserved|non-reserved|
 |NAMESPACES|non-reserved|non-reserved|non-reserved|
@@ -396,6 +398,7 @@ Below is a list of all the keywords in Spark SQL.
 |ROWS|non-reserved|non-reserved|reserved|
 |SCHEMA|non-reserved|non-reserved|non-reserved|
 |SCHEMAS|non-reserved|non-reserved|not a keyword|
+|SECOND|non-reserved|non-reserved|non-reserved|
 |SELECT|reserved|non-reserved|reserved|
 |SEMI|non-reserved|strict-non-reserved|non-reserved|
 |SEPARATED|non-reserved|non-reserved|non-reserved|
@@ -456,4 +459,5 @@ Below is a list of all the keywords in Spark SQL.
 |WHERE|reserved|non-reserved|reserved|
 |WINDOW|non-reserved|non-reserved|reserved|
 |WITH|reserved|non-reserved|reserved|
+|YEAR|non-reserved|non-reserved|non-reserved|
 |ZONE|non-reserved|non-reserved|non-reserved|