Skip to content

Commit

Permalink
Review feedback.
Browse files Browse the repository at this point in the history
  • Loading branch information
Marcelo Vanzin committed Jun 19, 2014
1 parent 93c3f85 commit b2e377f
Show file tree
Hide file tree
Showing 2 changed files with 21 additions and 11 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -395,7 +395,15 @@ object ClientBase extends Logging {
val APP_JAR: String = "__app__.jar"
val LOCAL_SCHEME = "local"
val CONF_SPARK_JAR = "spark.yarn.jar"
/**
* This is an internal config used to propagate the location of the user's jar file to the
* driver/executors.
*/
val CONF_SPARK_USER_JAR = "spark.yarn.user.jar"
/**
* This is an internal config used to propagate the list of extra jars to add to the classpath
* of executors.
*/
val CONF_SPARK_YARN_SECONDARY_JARS = "spark.yarn.secondary.jars"
val ENV_SPARK_JAR = "SPARK_JAR"

Expand Down Expand Up @@ -506,8 +514,8 @@ object ClientBase extends Logging {
addUserClasspath(args, sparkConf, env)
}

// Append all class files and jar files under the working directory to the classpath.
addFileToClasspath("*", null, env)
// Append all jar files under the working directory to the classpath.
addClasspathEntry(Environment.PWD.$() + Path.SEPARATOR + "*", env);
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.mockito.Matchers._
import org.mockito.Mockito._
import org.scalatest.FunSuite
import org.scalatest.matchers.ShouldMatchers._
import org.scalatest.Matchers

import scala.collection.JavaConversions._
import scala.collection.mutable.{ HashMap => MutableHashMap }
Expand All @@ -41,7 +41,7 @@ import scala.util.Try
import org.apache.spark.SparkConf
import org.apache.spark.util.Utils

class ClientBaseSuite extends FunSuite {
class ClientBaseSuite extends FunSuite with Matchers {

test("default Yarn application classpath") {
ClientBase.getDefaultYarnApplicationClasspath should be(Some(Fixtures.knownDefYarnAppCP))
Expand Down Expand Up @@ -89,17 +89,19 @@ class ClientBaseSuite extends FunSuite {

ClientBase.populateClasspath(args, conf, sparkConf, env, None)

val jars = env("CLASSPATH").split(File.pathSeparator)
s"$SPARK,$USER,$ADDED".split(",").foreach({ jar =>
val uri = new URI(jar)
val cp = env("CLASSPATH").split(File.pathSeparator)
s"$SPARK,$USER,$ADDED".split(",").foreach({ entry =>
val uri = new URI(entry)
if (ClientBase.LOCAL_SCHEME.equals(uri.getScheme())) {
jars should contain (uri.getPath())
cp should contain (uri.getPath())
} else {
jars should not contain (uri.getPath())
cp should not contain (uri.getPath())
}
})
jars should not contain (ClientBase.SPARK_JAR)
jars should not contain (ClientBase.APP_JAR)
cp should contain (s"$$PWD")
cp should contain (s"$$PWD${File.separator}*")
cp should not contain (ClientBase.SPARK_JAR)
cp should not contain (ClientBase.APP_JAR)
}

test("Jar path propagation through SparkConf") {
Expand Down

0 comments on commit b2e377f

Please sign in to comment.