Throw exception when memoryFracton is out of range

apache · May 9, 2014 · a59d76b · a59d76b
1 parent 7b899c2
commit a59d76b
Show file tree

Hide file tree

Showing 2 changed files with 11 additions and 1 deletion.
diff --git a/core/src/main/scala/org/apache/spark/storage/BlockManager.scala b/core/src/main/scala/org/apache/spark/storage/BlockManager.scala
@@ -1046,7 +1046,7 @@ private[spark] object BlockManager extends Logging {
   def getMaxMemory(conf: SparkConf): Long = {
     val memoryFraction = conf.getDouble("spark.storage.memoryFraction", 0.6)
     if (memoryFraction > 1 && memoryFraction <= 0) {
-      logWarning("spark.storage.memoryFraction should be between 0 and 1.")
+      throw new Exception("spark.storage.memoryFraction should be between 0 and 1.")
     }
     (Runtime.getRuntime.maxMemory * memoryFraction).toLong
   }

diff --git a/core/src/main/scala/org/apache/spark/util/collection/ExternalAppendOnlyMap.scala b/core/src/main/scala/org/apache/spark/util/collection/ExternalAppendOnlyMap.scala
@@ -76,6 +76,16 @@ class ExternalAppendOnlyMap[K, V, C](
   private val maxMemoryThreshold = {
     val memoryFraction = sparkConf.getDouble("spark.shuffle.memoryFraction", 0.3)
     val safetyFraction = sparkConf.getDouble("spark.shuffle.safetyFraction", 0.8)
+
+    if (memoryFraction > 1 && memoryFraction <= 0) {
+      throw new Exception("spark.shuffle.memoryFraction should be between 0 and 1.")
+    }
+
+    if (safetyFraction > 1 && safetyFraction <= 0) {
+      throw new Exception("spark.shuffle.safetyFraction should be between 0 and 1.")
+    }
+
+
     (Runtime.getRuntime.maxMemory * memoryFraction * safetyFraction).toLong
   }