Skip to content

Commit

Permalink
Add jars to classpath if they exist during testing.
Browse files Browse the repository at this point in the history
This allows pyspark tests to run.
  • Loading branch information
Marcelo Vanzin committed Mar 12, 2016
1 parent 40cd2e4 commit cc28749
Showing 1 changed file with 7 additions and 8 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -173,11 +173,9 @@ List<String> buildClassPath(String appClassPath) throws IOException {
// Add Spark jars to the classpath. For the testing case, we rely on the test code to set and
// propagate the test classpath appropriately. For normal invocation, look for the jars
// directory under SPARK_HOME.
if (!isTesting) {
String jarsDir = findJarsDir();
if (jarsDir != null) {
addToClassPath(cp, join(File.separator, jarsDir, "*"));
}
String jarsDir = findJarsDir(!isTesting);
if (jarsDir != null) {
addToClassPath(cp, join(File.separator, jarsDir, "*"));
}

// Datanucleus jars must be included on the classpath. Datanucleus jars do not work if only
Expand Down Expand Up @@ -312,18 +310,19 @@ private Properties loadPropertiesFile() throws IOException {
return props;
}

private String findJarsDir() {
private String findJarsDir(boolean failIfNotFound) {
// TODO: change to the correct directory once the assembly build is changed.
String sparkHome = getSparkHome();
File libdir;
if (new File(sparkHome, "RELEASE").isFile()) {
libdir = new File(sparkHome, "lib");
checkState(libdir.isDirectory(), "Library directory '%s' does not exist.",
checkState(!failIfNotFound || libdir.isDirectory(),
"Library directory '%s' does not exist.",
libdir.getAbsolutePath());
} else {
libdir = new File(sparkHome, String.format("assembly/target/scala-%s", getScalaVersion()));
if (!libdir.isDirectory()) {
checkState(!isEmpty(getenv("SPARK_PREPEND_CLASSES")),
checkState(!failIfNotFound,
"Library directory '%s' does not exist; make sure Spark is built.",
libdir.getAbsolutePath());
libdir = null;
Expand Down

0 comments on commit cc28749

Please sign in to comment.