Remove shouldHostShuffleServiceIfEnabled

jeanlyn · May 8, 2015 · a5d9432 · a5d9432
1 parent e901eb2
commit a5d9432
Show file tree

Hide file tree

Showing 3 changed files with 4 additions and 8 deletions.
diff --git a/core/src/main/scala/org/apache/spark/deploy/ExternalShuffleService.scala b/core/src/main/scala/org/apache/spark/deploy/ExternalShuffleService.scala
@@ -40,10 +40,7 @@ private[deploy]
 class ExternalShuffleService(sparkConf: SparkConf, securityManager: SecurityManager)
   extends Logging {
 
-  // Check both if shuffle service is enabled, and that the worker should actually host the
-  // shuffle service in that case. (The latter is currently only used for testing.)
-  private val enabled = sparkConf.getBoolean("spark.shuffle.service.enabled", false) &&
-    sparkConf.getBoolean("spark.worker.shouldHostShuffleServiceIfEnabled", true)
+  private val enabled = sparkConf.getBoolean("spark.shuffle.service.enabled", false)
   private val port = sparkConf.getInt("spark.shuffle.service.port", 7337)
   private val useSasl: Boolean = securityManager.isAuthenticationEnabled()
 

diff --git a/core/src/main/scala/org/apache/spark/deploy/LocalSparkCluster.scala b/core/src/main/scala/org/apache/spark/deploy/LocalSparkCluster.scala
@@ -48,7 +48,9 @@ class LocalSparkCluster(
     logInfo("Starting a local Spark cluster with " + numWorkers + " workers.")
 
     // Disable REST server on Master in this mode unless otherwise specified
-    val _conf = conf.clone().setIfMissing("spark.master.rest.enabled", "false")
+    val _conf = conf.clone()
+      .setIfMissing("spark.master.rest.enabled", "false")
+      .setIfMissing("spark.shuffle.service.enabled", "false")
 
     /* Start the Master */
     val (masterSystem, masterPort, _, _) = Master.startSystemAndActor(localHostname, 0, 0, _conf)

diff --git a/core/src/test/scala/org/apache/spark/ExternalShuffleServiceSuite.scala b/core/src/test/scala/org/apache/spark/ExternalShuffleServiceSuite.scala
@@ -43,9 +43,6 @@ class ExternalShuffleServiceSuite extends ShuffleSuite with BeforeAndAfterAll {
     conf.set("spark.shuffle.manager", "sort")
     conf.set("spark.shuffle.service.enabled", "true")
     conf.set("spark.shuffle.service.port", server.getPort.toString)
-
-    // local-cluster mode starts a Worker which would start its own shuffle service without this:
-    conf.set("spark.worker.shouldHostShuffleServiceIfEnabled", "false")
   }
 
   override def afterAll() {