diff --git a/assembly/pom.xml b/assembly/pom.xml index 5650f42587517..57205f187f7fa 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -21,7 +21,7 @@ com.apple.pie.spark spark-parent_2.11 - 2.3.2-pie1.0.3-SNAPSHOT + 2.3.2-pie1.0.4-SNAPSHOT ../pom.xml diff --git a/common/kvstore/pom.xml b/common/kvstore/pom.xml index e535fdef4c42b..42c0e392e7c98 100644 --- a/common/kvstore/pom.xml +++ b/common/kvstore/pom.xml @@ -22,7 +22,7 @@ com.apple.pie.spark spark-parent_2.11 - 2.3.2-pie1.0.3-SNAPSHOT + 2.3.2-pie1.0.4-SNAPSHOT ../../pom.xml diff --git a/common/network-common/pom.xml b/common/network-common/pom.xml index 65d7e95271bf8..f97f48adbccb3 100644 --- a/common/network-common/pom.xml +++ b/common/network-common/pom.xml @@ -22,7 +22,7 @@ com.apple.pie.spark spark-parent_2.11 - 2.3.2-pie1.0.3-SNAPSHOT + 2.3.2-pie1.0.4-SNAPSHOT ../../pom.xml diff --git a/common/network-shuffle/pom.xml b/common/network-shuffle/pom.xml index 584d5ff0ce275..48d870a5a6657 100644 --- a/common/network-shuffle/pom.xml +++ b/common/network-shuffle/pom.xml @@ -22,7 +22,7 @@ com.apple.pie.spark spark-parent_2.11 - 2.3.2-pie1.0.3-SNAPSHOT + 2.3.2-pie1.0.4-SNAPSHOT ../../pom.xml diff --git a/common/network-yarn/pom.xml b/common/network-yarn/pom.xml index 3245ba25011e0..599697294e311 100644 --- a/common/network-yarn/pom.xml +++ b/common/network-yarn/pom.xml @@ -22,7 +22,7 @@ com.apple.pie.spark spark-parent_2.11 - 2.3.2-pie1.0.3-SNAPSHOT + 2.3.2-pie1.0.4-SNAPSHOT ../../pom.xml diff --git a/common/sketch/pom.xml b/common/sketch/pom.xml index 8f0803243dc7a..df5bdf29b389f 100644 --- a/common/sketch/pom.xml +++ b/common/sketch/pom.xml @@ -22,7 +22,7 @@ com.apple.pie.spark spark-parent_2.11 - 2.3.2-pie1.0.3-SNAPSHOT + 2.3.2-pie1.0.4-SNAPSHOT ../../pom.xml diff --git a/common/tags/pom.xml b/common/tags/pom.xml index 55e3a66e33870..f4d7ad805aaf3 100644 --- a/common/tags/pom.xml +++ b/common/tags/pom.xml @@ -22,7 +22,7 @@ com.apple.pie.spark spark-parent_2.11 - 2.3.2-pie1.0.3-SNAPSHOT + 2.3.2-pie1.0.4-SNAPSHOT ../../pom.xml diff --git a/common/unsafe/pom.xml b/common/unsafe/pom.xml index 98edd19bf3130..a45a45e1ae69c 100644 --- a/common/unsafe/pom.xml +++ b/common/unsafe/pom.xml @@ -22,7 +22,7 @@ com.apple.pie.spark spark-parent_2.11 - 2.3.2-pie1.0.3-SNAPSHOT + 2.3.2-pie1.0.4-SNAPSHOT ../../pom.xml diff --git a/core/pom.xml b/core/pom.xml index 92445322c0c19..c86995617ae7b 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -21,7 +21,7 @@ com.apple.pie.spark spark-parent_2.11 - 2.3.2-pie1.0.3-SNAPSHOT + 2.3.2-pie1.0.4-SNAPSHOT ../pom.xml diff --git a/examples/pom.xml b/examples/pom.xml index c4a1ffb77e651..9878bf1986422 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -21,7 +21,7 @@ com.apple.pie.spark spark-parent_2.11 - 2.3.2-pie1.0.3-SNAPSHOT + 2.3.2-pie1.0.4-SNAPSHOT ../pom.xml diff --git a/external/docker-integration-tests/pom.xml b/external/docker-integration-tests/pom.xml index 3b01008d75913..7cb41d5029390 100644 --- a/external/docker-integration-tests/pom.xml +++ b/external/docker-integration-tests/pom.xml @@ -22,7 +22,7 @@ com.apple.pie.spark spark-parent_2.11 - 2.3.2-pie1.0.3-SNAPSHOT + 2.3.2-pie1.0.4-SNAPSHOT ../../pom.xml diff --git a/external/flume-assembly/pom.xml b/external/flume-assembly/pom.xml index 078903d72cd7a..e9535d989e358 100644 --- a/external/flume-assembly/pom.xml +++ b/external/flume-assembly/pom.xml @@ -21,7 +21,7 @@ com.apple.pie.spark spark-parent_2.11 - 2.3.2-pie1.0.3-SNAPSHOT + 2.3.2-pie1.0.4-SNAPSHOT ../../pom.xml diff --git a/external/flume-sink/pom.xml b/external/flume-sink/pom.xml index 1450180fcfdaf..8098ea18a85be 100644 --- a/external/flume-sink/pom.xml +++ b/external/flume-sink/pom.xml @@ -21,7 +21,7 @@ com.apple.pie.spark spark-parent_2.11 - 2.3.2-pie1.0.3-SNAPSHOT + 2.3.2-pie1.0.4-SNAPSHOT ../../pom.xml diff --git a/external/flume/pom.xml b/external/flume/pom.xml index 9a60e21930cd8..6ab231f603c5a 100644 --- a/external/flume/pom.xml +++ b/external/flume/pom.xml @@ -21,7 +21,7 @@ com.apple.pie.spark spark-parent_2.11 - 2.3.2-pie1.0.3-SNAPSHOT + 2.3.2-pie1.0.4-SNAPSHOT ../../pom.xml diff --git a/external/kafka-0-10-assembly/pom.xml b/external/kafka-0-10-assembly/pom.xml index 6ab066bd70404..6290ab14e303d 100644 --- a/external/kafka-0-10-assembly/pom.xml +++ b/external/kafka-0-10-assembly/pom.xml @@ -21,7 +21,7 @@ com.apple.pie.spark spark-parent_2.11 - 2.3.2-pie1.0.3-SNAPSHOT + 2.3.2-pie1.0.4-SNAPSHOT ../../pom.xml diff --git a/external/kafka-0-10-sql/pom.xml b/external/kafka-0-10-sql/pom.xml index 550b65b1a0552..105fd4d2d2ebf 100644 --- a/external/kafka-0-10-sql/pom.xml +++ b/external/kafka-0-10-sql/pom.xml @@ -21,7 +21,7 @@ com.apple.pie.spark spark-parent_2.11 - 2.3.2-pie1.0.3-SNAPSHOT + 2.3.2-pie1.0.4-SNAPSHOT ../../pom.xml diff --git a/external/kafka-0-10/pom.xml b/external/kafka-0-10/pom.xml index ed99f00c1d5bf..219a932ed9954 100644 --- a/external/kafka-0-10/pom.xml +++ b/external/kafka-0-10/pom.xml @@ -21,7 +21,7 @@ com.apple.pie.spark spark-parent_2.11 - 2.3.2-pie1.0.3-SNAPSHOT + 2.3.2-pie1.0.4-SNAPSHOT ../../pom.xml diff --git a/external/kafka-0-8-assembly/pom.xml b/external/kafka-0-8-assembly/pom.xml index 41f9b8e4bab1e..230c0c94a1144 100644 --- a/external/kafka-0-8-assembly/pom.xml +++ b/external/kafka-0-8-assembly/pom.xml @@ -21,7 +21,7 @@ com.apple.pie.spark spark-parent_2.11 - 2.3.2-pie1.0.3-SNAPSHOT + 2.3.2-pie1.0.4-SNAPSHOT ../../pom.xml diff --git a/external/kafka-0-8/pom.xml b/external/kafka-0-8/pom.xml index e230ae4566a56..d2c8f8f2cf891 100644 --- a/external/kafka-0-8/pom.xml +++ b/external/kafka-0-8/pom.xml @@ -21,7 +21,7 @@ com.apple.pie.spark spark-parent_2.11 - 2.3.2-pie1.0.3-SNAPSHOT + 2.3.2-pie1.0.4-SNAPSHOT ../../pom.xml diff --git a/external/kinesis-asl-assembly/pom.xml b/external/kinesis-asl-assembly/pom.xml index 09090b24b8476..1f4469277be80 100644 --- a/external/kinesis-asl-assembly/pom.xml +++ b/external/kinesis-asl-assembly/pom.xml @@ -21,7 +21,7 @@ com.apple.pie.spark spark-parent_2.11 - 2.3.2-pie1.0.3-SNAPSHOT + 2.3.2-pie1.0.4-SNAPSHOT ../../pom.xml diff --git a/external/kinesis-asl/pom.xml b/external/kinesis-asl/pom.xml index 68a6224cde957..f245fe8c4354c 100644 --- a/external/kinesis-asl/pom.xml +++ b/external/kinesis-asl/pom.xml @@ -20,7 +20,7 @@ com.apple.pie.spark spark-parent_2.11 - 2.3.2-pie1.0.3-SNAPSHOT + 2.3.2-pie1.0.4-SNAPSHOT ../../pom.xml diff --git a/external/spark-ganglia-lgpl/pom.xml b/external/spark-ganglia-lgpl/pom.xml index 14999a01160ad..4c0d58e742f45 100644 --- a/external/spark-ganglia-lgpl/pom.xml +++ b/external/spark-ganglia-lgpl/pom.xml @@ -20,7 +20,7 @@ com.apple.pie.spark spark-parent_2.11 - 2.3.2-pie1.0.3-SNAPSHOT + 2.3.2-pie1.0.4-SNAPSHOT ../../pom.xml diff --git a/graphx/pom.xml b/graphx/pom.xml index 0f5e29ecd54ce..26b15e832da57 100644 --- a/graphx/pom.xml +++ b/graphx/pom.xml @@ -21,7 +21,7 @@ com.apple.pie.spark spark-parent_2.11 - 2.3.2-pie1.0.3-SNAPSHOT + 2.3.2-pie1.0.4-SNAPSHOT ../pom.xml diff --git a/hadoop-cloud/pom.xml b/hadoop-cloud/pom.xml index 5d1f1eedab309..b1f1aa7b42949 100644 --- a/hadoop-cloud/pom.xml +++ b/hadoop-cloud/pom.xml @@ -22,7 +22,7 @@ com.apple.pie.spark spark-parent_2.11 - 2.3.2-pie1.0.3-SNAPSHOT + 2.3.2-pie1.0.4-SNAPSHOT ../pom.xml diff --git a/launcher/pom.xml b/launcher/pom.xml index cc3c8e6d2fe57..c871d5f97de59 100644 --- a/launcher/pom.xml +++ b/launcher/pom.xml @@ -22,7 +22,7 @@ com.apple.pie.spark spark-parent_2.11 - 2.3.2-pie1.0.3-SNAPSHOT + 2.3.2-pie1.0.4-SNAPSHOT ../pom.xml diff --git a/mllib-local/pom.xml b/mllib-local/pom.xml index 81819bd42d7a9..85944cad8a70a 100644 --- a/mllib-local/pom.xml +++ b/mllib-local/pom.xml @@ -21,7 +21,7 @@ com.apple.pie.spark spark-parent_2.11 - 2.3.2-pie1.0.3-SNAPSHOT + 2.3.2-pie1.0.4-SNAPSHOT ../pom.xml diff --git a/mllib/pom.xml b/mllib/pom.xml index a90e4d7a0903f..9e9a373c85e24 100644 --- a/mllib/pom.xml +++ b/mllib/pom.xml @@ -21,7 +21,7 @@ com.apple.pie.spark spark-parent_2.11 - 2.3.2-pie1.0.3-SNAPSHOT + 2.3.2-pie1.0.4-SNAPSHOT ../pom.xml diff --git a/pom.xml b/pom.xml index 5a7feddc2601a..555f17ceae3e9 100644 --- a/pom.xml +++ b/pom.xml @@ -26,7 +26,7 @@ com.apple.pie.spark spark-parent_2.11 - 2.3.2-pie1.0.3-SNAPSHOT + 2.3.2-pie1.0.4-SNAPSHOT pom Spark Project Parent POM http://spark.apache.org/ diff --git a/repl/pom.xml b/repl/pom.xml index 8cb47eb360d1e..b75c5f840d5dd 100644 --- a/repl/pom.xml +++ b/repl/pom.xml @@ -21,7 +21,7 @@ com.apple.pie.spark spark-parent_2.11 - 2.3.2-pie1.0.3-SNAPSHOT + 2.3.2-pie1.0.4-SNAPSHOT ../pom.xml diff --git a/resource-managers/kubernetes/core/pom.xml b/resource-managers/kubernetes/core/pom.xml index 361c406b58f9d..1a3882ac9f8b2 100644 --- a/resource-managers/kubernetes/core/pom.xml +++ b/resource-managers/kubernetes/core/pom.xml @@ -20,7 +20,7 @@ com.apple.pie.spark spark-parent_2.11 - 2.3.2-pie1.0.3-SNAPSHOT + 2.3.2-pie1.0.4-SNAPSHOT ../../../pom.xml diff --git a/resource-managers/mesos/pom.xml b/resource-managers/mesos/pom.xml index 97edd5c54a328..c34f4a8d0dfb0 100644 --- a/resource-managers/mesos/pom.xml +++ b/resource-managers/mesos/pom.xml @@ -20,7 +20,7 @@ com.apple.pie.spark spark-parent_2.11 - 2.3.2-pie1.0.3-SNAPSHOT + 2.3.2-pie1.0.4-SNAPSHOT ../../pom.xml diff --git a/resource-managers/yarn/pom.xml b/resource-managers/yarn/pom.xml index c3ae6df1c0198..4b2fea8c66331 100644 --- a/resource-managers/yarn/pom.xml +++ b/resource-managers/yarn/pom.xml @@ -20,7 +20,7 @@ com.apple.pie.spark spark-parent_2.11 - 2.3.2-pie1.0.3-SNAPSHOT + 2.3.2-pie1.0.4-SNAPSHOT ../../pom.xml diff --git a/sql/catalyst/pom.xml b/sql/catalyst/pom.xml index c2aa7949f400f..473b228f0e11c 100644 --- a/sql/catalyst/pom.xml +++ b/sql/catalyst/pom.xml @@ -22,7 +22,7 @@ com.apple.pie.spark spark-parent_2.11 - 2.3.2-pie1.0.3-SNAPSHOT + 2.3.2-pie1.0.4-SNAPSHOT ../../pom.xml diff --git a/sql/core/pom.xml b/sql/core/pom.xml index 67081930896d7..88b39c1b202b6 100644 --- a/sql/core/pom.xml +++ b/sql/core/pom.xml @@ -22,7 +22,7 @@ com.apple.pie.spark spark-parent_2.11 - 2.3.2-pie1.0.3-SNAPSHOT + 2.3.2-pie1.0.4-SNAPSHOT ../../pom.xml diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala index baea4ceebf8e3..bef3a56ca04f1 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala @@ -42,6 +42,19 @@ import org.apache.spark.util.{MutableURLClassLoader, Utils} */ private[sql] class SharedState(val sparkContext: SparkContext) extends Logging { + private var initialized = false + + initialize() + + private def initialize() { + synchronized { + if (!initialized) { + SharedState.setUrlStreamHanderFactoryIfNeeded(sparkContext.conf) + initialized = true + } + } + } + // Load hive-site.xml into hadoopConf and determine the warehouse path we want to use, based on // the config from both hive and Spark SQL. Finally set the warehouse config value to sparkConf. val warehousePath: String = { @@ -153,11 +166,16 @@ private[sql] class SharedState(val sparkContext: SparkContext) extends Logging { } object SharedState extends Logging { - try { - URL.setURLStreamHandlerFactory(new FsUrlStreamHandlerFactory()) - } catch { - case e: Error => - logWarning("URL.setURLStreamHandlerFactory failed to set FsUrlStreamHandlerFactory") + + private def setUrlStreamHanderFactoryIfNeeded(conf: SparkConf) = { + try { + if (conf.getBoolean("spark.FsUrlStreamHandlerFactory.enabled", true)) { + URL.setURLStreamHandlerFactory(new FsUrlStreamHandlerFactory()) + } + } catch { + case e: Error => + logWarning("URL.setURLStreamHandlerFactory failed to set FsUrlStreamHandlerFactory") + } } private val HIVE_EXTERNAL_CATALOG_CLASS_NAME = "org.apache.spark.sql.hive.HiveExternalCatalog" diff --git a/sql/hive-thriftserver/pom.xml b/sql/hive-thriftserver/pom.xml index 24f2331503924..146c1872ed0bd 100644 --- a/sql/hive-thriftserver/pom.xml +++ b/sql/hive-thriftserver/pom.xml @@ -22,7 +22,7 @@ com.apple.pie.spark spark-parent_2.11 - 2.3.2-pie1.0.3-SNAPSHOT + 2.3.2-pie1.0.4-SNAPSHOT ../../pom.xml diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml index d1213e40322df..3dfa1cec17b84 100644 --- a/sql/hive/pom.xml +++ b/sql/hive/pom.xml @@ -22,7 +22,7 @@ com.apple.pie.spark spark-parent_2.11 - 2.3.2-pie1.0.3-SNAPSHOT + 2.3.2-pie1.0.4-SNAPSHOT ../../pom.xml diff --git a/streaming/pom.xml b/streaming/pom.xml index 8c19a34ff6d33..7db7e5534fda2 100644 --- a/streaming/pom.xml +++ b/streaming/pom.xml @@ -21,7 +21,7 @@ com.apple.pie.spark spark-parent_2.11 - 2.3.2-pie1.0.3-SNAPSHOT + 2.3.2-pie1.0.4-SNAPSHOT ../pom.xml diff --git a/tools/pom.xml b/tools/pom.xml index 68bc106e40a37..c354090021b0f 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -20,7 +20,7 @@ com.apple.pie.spark spark-parent_2.11 - 2.3.2-pie1.0.3-SNAPSHOT + 2.3.2-pie1.0.4-SNAPSHOT ../pom.xml