also ignore it in standalone cluster mode

apache · Nov 11, 2014 · ff8d5f7 · ff8d5f7
1 parent 2286e6b
commit ff8d5f7
Show file tree

Hide file tree

Showing 3 changed files with 10 additions and 2 deletions.
diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala
@@ -203,7 +203,7 @@ class SparkContext(config: SparkConf) extends SparkStatusAPI with Logging {
   if (master == "yarn-client") System.setProperty("SPARK_YARN_MODE", "true")
 
   // Set Spark driver host and port system properties. Ignore host setting in yarn-cluster mode.
-  if (master == "yarn-cluster") {
+  if (master.contains("cluster")) {
     conf.set("spark.driver.host", Utils.localHostName())
   } else {
     conf.setIfMissing("spark.driver.host", Utils.localHostName())

diff --git a/core/src/main/scala/org/apache/spark/deploy/Client.scala b/core/src/main/scala/org/apache/spark/deploy/Client.scala
@@ -155,6 +155,14 @@ object Client {
     }
     conf.set("spark.akka.askTimeout", "10")
     conf.set("akka.loglevel", driverArgs.logLevel.toString.replace("WARN", "WARNING"))
+
+    // Set the web ui port to be ephemeral so we don't conflict with other spark processes
+    // running on the same box
+    conf.set("spark.ui.port", "0")
+
+    // Set the master property to match the requested mode.
+    conf.set("spark.master", "standalone-cluster")
+
     Logger.getRootLogger.setLevel(driverArgs.logLevel)
 
     val (actorSystem, _) = AkkaUtils.createActorSystem(

diff --git a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMasterArguments.scala b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMasterArguments.scala
@@ -36,7 +36,7 @@ class ApplicationMasterArguments(val args: Array[String]) {
 
     var args = inputArgs
 
-    while (! args.isEmpty) {
+    while (!args.isEmpty) {
       // --num-workers, --worker-memory, and --worker-cores are deprecated since 1.0,
       // the properties with executor in their names are preferred.
       args match {