diff --git a/bin/spark-class b/bin/spark-class index 700615d051204..a61d6a369cf80 100755 --- a/bin/spark-class +++ b/bin/spark-class @@ -30,6 +30,9 @@ FWDIR="$(cd `dirname $0`/..; pwd)" # Export this as SPARK_HOME export SPARK_HOME="$FWDIR" +# Load utility functions +. "$SPARK_HOME/bin/utils.sh" + . $FWDIR/bin/load-spark-env.sh if [ -z "$1" ]; then @@ -77,7 +80,7 @@ case "$1" in 'org.apache.spark.deploy.SparkSubmit') OUR_JAVA_OPTS="$SPARK_JAVA_OPTS $SPARK_SUBMIT_OPTS" if [ -n "$SPARK_SUBMIT_LIBRARY_PATH" ]; then - OUR_JAVA_OPTS="$OUT_JAVA_OPTS -Djava.library.path=$SPARK_SUBMIT_LIBRARY_PATH" + OUR_JAVA_OPTS="$OUR_JAVA_OPTS -Djava.library.path=$SPARK_SUBMIT_LIBRARY_PATH" fi OUR_JAVA_MEM=${SPARK_DRIVER_MEMORY:-$DEFAULT_MEM} ;; @@ -103,11 +106,16 @@ fi # Set JAVA_OPTS to be able to load native libraries and to set heap size JAVA_OPTS="-XX:MaxPermSize=128m $OUR_JAVA_OPTS" JAVA_OPTS="$JAVA_OPTS -Xms$OUR_JAVA_MEM -Xmx$OUR_JAVA_MEM" + # Load extra JAVA_OPTS from conf/java-opts, if it exists if [ -e "$FWDIR/conf/java-opts" ] ; then JAVA_OPTS="$JAVA_OPTS `cat $FWDIR/conf/java-opts`" fi -export JAVA_OPTS + +# Escape JAVA_OPTS properly to handle whitespace, double quotes and backslashes +# This exports the escaped java options into ESCAPED_JAVA_OPTS +escape_java_options "$JAVA_OPTS" + # Attention: when changing the way the JAVA_OPTS are assembled, the change must be reflected in CommandUtils.scala! TOOLS_DIR="$FWDIR"/tools @@ -148,37 +156,6 @@ if $cygwin; then fi export CLASSPATH -# Properly escape java options, dealing with whitespace, double quotes and backslashes -# This accepts a string, and returns the escaped list through ESCAPED_JAVA_OPTS -escape_java_options() { - ESCAPED_JAVA_OPTS=() # return value - option_buffer="" # buffer for collecting parts of an option - opened_quotes=0 # whether we are expecting a closing double quotes - for word in $1; do - contains_quote=$(echo "$word" | grep \" | grep -v \\\\\") - if [ -n "$contains_quote" ]; then - # Flip the bit - opened_quotes=$(((opened_quotes + 1) % 2)) - fi - if [[ $opened_quotes == 0 ]]; then - ESCAPED_JAVA_OPTS+=("$(echo "$option_buffer $word" | sed "s/^[[:space:]]*//" | sed "s/\([^\\]\)\"/\1/g")") - option_buffer="" - else - option_buffer="$option_buffer $word" - fi - done - # Something is wrong if we ended with open double quotes - if [[ $opened_quotes == 1 ]]; then - echo "Java options parse error! Expecting closing double quotes." 1>&2 - exit 1 - fi -} - -escape_java_options "$JAVA_OPTS" -for option in "${ESCAPED_JAVA_OPTS[@]}"; do - echo "$option" -done - if [ "$SPARK_PRINT_LAUNCH_COMMAND" == "1" ]; then echo -n "Spark Command: " 1>&2 echo "$RUNNER" -cp "$CLASSPATH" "${ESCAPED_JAVA_OPTS[@]}" "$@" 1>&2 diff --git a/bin/spark-submit b/bin/spark-submit index 3889ae79f6dc0..6d7f139fdc3cc 100755 --- a/bin/spark-submit +++ b/bin/spark-submit @@ -20,6 +20,14 @@ export SPARK_HOME="$(cd `dirname $0`/..; pwd)" ORIG_ARGS=("$@") +# Load utility functions +. "$SPARK_HOME/bin/utils.sh" + +# For client mode, the driver will be launched in the JVM that launches +# SparkSubmit, so we need to handle the class paths, java options, and +# memory pre-emptively in bash. Otherwise, it will be too late by the +# time the JVM has started. + while (($#)); do if [ "$1" = "--deploy-mode" ]; then DEPLOY_MODE=$2 @@ -40,34 +48,14 @@ done DEPLOY_MODE=${DEPLOY_MODE:-"client"} PROPERTIES_FILE=${PROPERTIES_FILE:-"$SPARK_HOME/conf/spark-defaults.conf"} -# For client mode, the driver will be launched in the JVM that launches -# SparkSubmit, so we need to handle the class paths, java options, and -# memory pre-emptively in bash. Otherwise, it will be too late by the -# time the JVM has started. - if [ $DEPLOY_MODE == "client" ]; then - # We parse the default properties file here, assuming each line is - # a key value pair delimited either by white space or "=" sign. All - # spark.driver.* configs must be processed now before it's too late. + # Parse the default properties file here for spark.driver.* configs if [ -f "$PROPERTIES_FILE" ]; then echo "Using properties file $PROPERTIES_FILE." 1>&2 - - # Parse the value of the given config according to the specifications outlined in - # http://docs.oracle.com/javase/7/docs/api/java/util/Properties.html#load(java.io.Reader) - parse_config() { - result=$( \ - sed "/^[#!]/ d" "conf/spark-defaults.conf" | \ - grep "$1" | \ - sed "s/$1//" | \ - sed "s/^[[:space:]]*[:=]\{0,1\}//" | \ - sed "s/^[[:space:]]*\(.*\)[[:space:]]*$/\1/g" \ - ) - } - parse_config "spark.driver.memory"; DRIVER_MEMORY_CONF="$result" - parse_config "spark.driver.extraJavaOptions"; DRIVER_EXTRA_JAVA_OPTS="$result" - parse_config "spark.driver.extraClassPath"; DRIVER_EXTRA_CLASSPATH="$result" - parse_config "spark.driver.extraLibraryPath"; DRIVER_EXTRA_LIBRARY_PATH="$result" - + parse_java_property "spark.driver.memory"; DRIVER_MEMORY_CONF="$JAVA_PROPERTY_VALUE" + parse_java_property "spark.driver.extraJavaOptions"; DRIVER_EXTRA_JAVA_OPTS="$JAVA_PROPERTY_VALUE" + parse_java_property "spark.driver.extraClassPath"; DRIVER_EXTRA_CLASSPATH="$JAVA_PROPERTY_VALUE" + parse_java_property "spark.driver.extraLibraryPath"; DRIVER_EXTRA_LIBRARY_PATH="$JAVA_PROPERTY_VALUE" if [ -n "$DRIVER_EXTRA_JAVA_OPTS" ]; then export SPARK_SUBMIT_OPTS="$SPARK_SUBMIT_OPTS $DRIVER_EXTRA_JAVA_OPTS" fi @@ -80,7 +68,6 @@ if [ $DEPLOY_MODE == "client" ]; then else echo "Warning: properties file $PROPERTIES_FILE does not exist!" 1>&2 fi - # Favor command line memory over config memory DRIVER_MEMORY=${DRIVER_MEMORY:-"$DRIVER_MEMORY_CONF"} if [ -n "$DRIVER_MEMORY" ]; then diff --git a/bin/utils.sh b/bin/utils.sh new file mode 100755 index 0000000000000..613b72a49cf4b --- /dev/null +++ b/bin/utils.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Parse the value of a config from a java properties file according to the specifications in +# http://docs.oracle.com/javase/7/docs/api/java/util/Properties.html#load(java.io.Reader). +# This accepts the name of the config and returns the value through JAVA_PROPERTY_VALUE. +parse_java_property() { + JAVA_PROPERTY_VALUE=$( \ + sed "/^[#!]/ d" "conf/spark-defaults.conf" | \ + grep "$1" | \ + sed "s/$1//" | \ + sed "s/^[[:space:]]*[:=]\{0,1\}//" | \ + sed "s/^[[:space:]]*\(.*\)[[:space:]]*$/\1/g" \ + ) + export JAVA_PROPERTY_VALUE +} + +# Properly escape java options, dealing with whitespace, double quotes and backslashes +# This accepts a string, and returns the escaped list through ESCAPED_JAVA_OPTS. +escape_java_options() { + ESCAPED_JAVA_OPTS=() # return value + option_buffer="" # buffer for collecting parts of an option + opened_quotes=0 # whether we are expecting a closing double quotes + for word in $1; do + contains_quote=$(echo "$word" | grep \" | grep -v \\\\\") + if [ -n "$contains_quote" ]; then + # Flip the bit + opened_quotes=$(((opened_quotes + 1) % 2)) + fi + if [[ $opened_quotes == 0 ]]; then + ESCAPED_JAVA_OPTS+=("$(echo "$option_buffer $word" | sed "s/^[[:space:]]*//" | sed "s/\([^\\]\)\"/\1/g")") + option_buffer="" + else + option_buffer="$option_buffer $word" + fi + done + # Something is wrong if we ended with open double quotes + if [[ $opened_quotes == 1 ]]; then + echo "Java options parse error! Expecting closing double quotes." 1>&2 + exit 1 + fi + export ESCAPED_JAVA_OPTS +} +