Skip to content

Commit

Permalink
Filter out JVM memory in Scala rather than Bash (minor)
Browse files Browse the repository at this point in the history
  • Loading branch information
andrewor14 committed Aug 20, 2014
1 parent 9a778f6 commit 51aeb01
Show file tree
Hide file tree
Showing 2 changed files with 6 additions and 2 deletions.
2 changes: 1 addition & 1 deletion bin/spark-class
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,7 @@ fi

# Set JAVA_OPTS to be able to load native libraries and to set heap size
JAVA_OPTS="-XX:MaxPermSize=128m $OUR_JAVA_OPTS"
JAVA_OPTS="$JAVA_OPTS -Xms$OUR_JAVA_MEM -Xmx$OUR_JAVA_MEM"

# Load extra JAVA_OPTS from conf/java-opts, if it exists
if [ -e "$FWDIR/conf/java-opts" ] ; then
Expand Down Expand Up @@ -169,7 +170,6 @@ if [ -n "$SPARK_SUBMIT_BOOTSTRAP_DRIVER" ]; then
shift
exec "$RUNNER" org.apache.spark.deploy.SparkSubmitDriverBootstrapper "$@"
else
JAVA_OPTS="$JAVA_OPTS -Xms$OUR_JAVA_MEM -Xmx$OUR_JAVA_MEM"
if [ -n "$SPARK_PRINT_LAUNCH_COMMAND" ]; then
echo -n "Spark Command: " 1>&2
echo "$RUNNER" -cp "$CLASSPATH" $JAVA_OPTS "$@" 1>&2
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -99,12 +99,16 @@ private[spark] object SparkSubmitDriverBootstrapper {
javaOpts + confJavaOpts.map(" " + _).getOrElse("")
}

val filteredJavaOpts = Utils.splitCommandString(newJavaOpts)
.filterNot(_.startsWith("-Xms"))
.filterNot(_.startsWith("-Xmx"))

// Build up command
val command: Seq[String] =
Seq(runner) ++
Seq("-cp", newClasspath) ++
Seq(newLibraryPath) ++
Utils.splitCommandString(newJavaOpts) ++
filteredJavaOpts ++
Seq(s"-Xms$newDriverMemory", s"-Xmx$newDriverMemory") ++
Seq("org.apache.spark.deploy.SparkSubmit") ++
submitArgs
Expand Down

0 comments on commit 51aeb01

Please sign in to comment.