fix local-cluster mode for ExternalShuffleServiceSuite

nemccarthy · May 8, 2015 · e901eb2 · e901eb2
1 parent 59e5e38
commit e901eb2
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 1 deletion.
diff --git a/core/src/main/scala/org/apache/spark/deploy/ExternalShuffleService.scala b/core/src/main/scala/org/apache/spark/deploy/ExternalShuffleService.scala
@@ -40,7 +40,10 @@ private[deploy]
 class ExternalShuffleService(sparkConf: SparkConf, securityManager: SecurityManager)
   extends Logging {
 
-  private val enabled = sparkConf.getBoolean("spark.shuffle.service.enabled", false)
+  // Check both if shuffle service is enabled, and that the worker should actually host the
+  // shuffle service in that case. (The latter is currently only used for testing.)
+  private val enabled = sparkConf.getBoolean("spark.shuffle.service.enabled", false) &&
+    sparkConf.getBoolean("spark.worker.shouldHostShuffleServiceIfEnabled", true)
   private val port = sparkConf.getInt("spark.shuffle.service.port", 7337)
   private val useSasl: Boolean = securityManager.isAuthenticationEnabled()
 

diff --git a/core/src/test/scala/org/apache/spark/ExternalShuffleServiceSuite.scala b/core/src/test/scala/org/apache/spark/ExternalShuffleServiceSuite.scala
@@ -43,6 +43,9 @@ class ExternalShuffleServiceSuite extends ShuffleSuite with BeforeAndAfterAll {
     conf.set("spark.shuffle.manager", "sort")
     conf.set("spark.shuffle.service.enabled", "true")
     conf.set("spark.shuffle.service.port", server.getPort.toString)
+
+    // local-cluster mode starts a Worker which would start its own shuffle service without this:
+    conf.set("spark.worker.shouldHostShuffleServiceIfEnabled", "false")
   }
 
   override def afterAll() {