Fix hadoop config may override spark config

apache · Mar 9, 2018 · 9745ec3 · 9745ec3
1 parent 2ce37b5
commit 9745ec3
Showing 1 changed file with 7 additions and 3 deletions.
diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala b/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala
@@ -490,9 +490,13 @@ object SparkHadoopUtil {
   }
 
   private def appendSparkHadoopConfigs(conf: SparkConf, hadoopConf: Configuration): Unit = {
-    // Copy any "spark.hadoop.foo=bar" spark properties into conf as "foo=bar"
-    for ((key, value) <- conf.getAll if key.startsWith("spark.hadoop.")) {
-      hadoopConf.set(key.substring("spark.hadoop.".length), value)
+    for ((key, value) <- conf.getAll if key.startsWith("spark.")) {
+      if (key.startsWith("spark.hadoop.")) {
+        // Copy any "spark.hadoop.foo=bar" system properties into conf as "foo=bar"
+        hadoopConf.set(key.substring("spark.hadoop.".length), value)
+      } else {
+        hadoopConf.set(key, value)
+      }
     }
   }
 }