Change Utils.getSparkOrYarnConfig

apache · Mar 10, 2018 · 0034a58 · 0034a58
1 parent 9745ec3
commit 0034a58
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 8 deletions.
diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala b/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala
@@ -490,13 +490,9 @@ object SparkHadoopUtil {
   }
 
   private def appendSparkHadoopConfigs(conf: SparkConf, hadoopConf: Configuration): Unit = {
-    for ((key, value) <- conf.getAll if key.startsWith("spark.")) {
-      if (key.startsWith("spark.hadoop.")) {
-        // Copy any "spark.hadoop.foo=bar" system properties into conf as "foo=bar"
-        hadoopConf.set(key.substring("spark.hadoop.".length), value)
-      } else {
-        hadoopConf.set(key, value)
-      }
+    // Copy any "spark.hadoop.foo=bar" spark properties into conf as "foo=bar"
+    for ((key, value) <- conf.getAll if key.startsWith("spark.hadoop.")) {
+      hadoopConf.set(key.substring("spark.hadoop.".length), value)
     }
   }
 }
diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala
@@ -2434,7 +2434,8 @@ private[spark] object Utils extends Logging {
    */
   def getSparkOrYarnConfig(conf: SparkConf, key: String, default: String): String = {
     val sparkValue = conf.get(key, default)
-    if (conf.get(SparkLauncher.SPARK_MASTER, null) == "yarn") {
+    if (conf.get(SparkLauncher.SPARK_MASTER, null) == "yarn"
+      && (key.startsWith("spark.hadoop.") || !key.startsWith("spark."))) {
       new YarnConfiguration(SparkHadoopUtil.get.newConfiguration(conf)).get(key, sparkValue)
     } else {
       sparkValue