Make config yarn specific

apache · Apr 14, 2014 · 446122d · 446122d
1 parent f5d5373
commit 446122d
Show file tree

Hide file tree

Showing 4 changed files with 3 additions and 9 deletions.
diff --git a/docs/configuration.md b/docs/configuration.md
@@ -627,13 +627,6 @@ Apart from these, the following properties are also available, and may be useful
     Number of cores to allocate for each task.
   </td>
 </tr>
-<tr>
-  <td>spark.historyserver.address</td>
-  <td>localhost:18080</td>
-  <td>
-    The URL of the Spark history server. The history server is optional. 
-  </td>
-</tr>
 </table>
 
 ## Viewing Spark Properties

diff --git a/docs/running-on-yarn.md b/docs/running-on-yarn.md
@@ -42,6 +42,7 @@ System Properties:
 * `spark.yarn.preserve.staging.files`, set to true to preserve the staged files(spark jar, app jar, distributed cache files) at the end of the job rather then delete them.
 * `spark.yarn.scheduler.heartbeat.interval-ms`, the interval in ms in which the Spark application master heartbeats into the YARN ResourceManager. Default is 5 seconds. 
 * `spark.yarn.max.executor.failures`, the maximum number of executor failures before failing the application. Default is the number of executors requested times 2 with minimum of 3.
+* `spark.yarn.historyServer.address`, the address of the Spark history server (i.e. host.com:18080). The address should not contain a scheme (http://). Defaults to not being set since the history server is an optional service. This address is given to the Yarn ResourceManager when the Spark application finishes to link the application from the ResourceManaer UI to the Spark history server UI. 
 
 # Launching Spark on YARN
 

diff --git a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
@@ -366,7 +366,7 @@ class ApplicationMaster(args: ApplicationMasterArguments, conf: Configuration,
         finishReq.setAppAttemptId(appAttemptId)
         finishReq.setFinishApplicationStatus(status)
         finishReq.setDiagnostics(diagnostics)
-        finishReq.setTrackingUrl(sparkConf.get("spark.historyserver.address", ""))
+        finishReq.setTrackingUrl(sparkConf.get("spark.yarn.historyServer.address", ""))
         resourceManager.finishApplicationMaster(finishReq)
       }
     }

diff --git a/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala b/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
@@ -347,7 +347,7 @@ class ApplicationMaster(args: ApplicationMasterArguments, conf: Configuration,
 
       logInfo("finishApplicationMaster with " + status)
       if (registered) {
-        val trackingUrl = sparkConf.get("spark.historyserver.address", "")
+        val trackingUrl = sparkConf.get("spark.yarn.historyServer.address", "")
         amClient.unregisterApplicationMaster(status, diagnostics, trackingUrl)
       }
     }