From 94bdf30e550cb71315a36c900d3101ba97c5db82 Mon Sep 17 00:00:00 2001 From: Jongyoul Lee Date: Thu, 25 Jun 2015 01:33:14 +0900 Subject: [PATCH] [ZEPPELIN-18] Running pyspark without deploying python libraries to every yarn node - Fixed checkstyle --- .../org/apache/zeppelin/spark/SparkInterpreter.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java index 7d2359063bb..bf229cadecc 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java @@ -269,20 +269,20 @@ public SparkContext createSparkContext() { conf.set(key, val); } } - + //TODO(jongyoul): Move these codes into PySparkInterpreter.java String zeppelinHome = getSystemDefault("ZEPPELIN_HOME", "zeppelin.home", "../"); File zeppelinPythonLibPath = new File(zeppelinHome, "python/lib"); - String[] pythonLibs = new String[] {"pyspark.zip", "py4j-0.8.2.1-src.zip"}; - ArrayList pythonLibUris = new ArrayList<>(); - for (String lib: pythonLibs) { + String[] pythonLibs = new String[]{"pyspark.zip", "py4j-0.8.2.1-src.zip"}; + ArrayList pythonLibUris = new ArrayList<>(); + for (String lib : pythonLibs) { File libFile = new File(zeppelinPythonLibPath, lib); - if(libFile.exists()) { + if (libFile.exists()) { pythonLibUris.add(libFile.toURI().toString()); } } pythonLibUris.trimToSize(); - if(pythonLibs.length == pythonLibUris.size()) { + if (pythonLibs.length == pythonLibUris.size()) { conf.set("spark.yarn.dist.files", Joiner.on(",").join(pythonLibUris)); conf.set("spark.files", conf.get("spark.yarn.dist.files")); conf.set("spark.submit.pyArchives", Joiner.on(":").join(pythonLibs));