#6569 maven cell magic (#6665)

* #6565 additional logging for other clojure kernel tests * #6569 mvn cell magic part 1 * #6569 mvn cell magic part 2 * Revert "#6565 additional logging for other clojure kernel tests" This reverts commit 4cb0cc3 * #6569 cr changes - banners for new files, fixed typo * #6569 added examples and description for mvn cell magic * #6569 fixed mvn cell magic additional repos bug * #6569 removed dependecy conflict message * #6569 tablesaw and spark demos classpath example update
twosigma · Jan 24, 2018 · ab94835 · ab94835
1 parent 2ba3187
commit ab94835
Show file tree

Hide file tree

Showing 17 changed files with 488 additions and 102 deletions.
diff --git a/doc/groovy/ClasspathMagicCommands.ipynb b/doc/groovy/ClasspathMagicCommands.ipynb
@@ -106,7 +106,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "## Loading Dependencies with Grapes and Maven\n",
+    "## Loading single Dependency with Grapes and Maven\n",
     "\n",
     "Groovy has a dependency manager called [Grape](http://docs.groovy-lang.org/latest/html/documentation/grape.html) built-in and you can access it as follows:"
    ]
@@ -185,6 +185,26 @@
     "%classpath add mvn org.springframework spring-context 5.0.3.BUILD-SNAPSHOT"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Loading multiple Dependencies with Maven\n",
+    "\n",
+    "You can use `%%classpath` cell magic to load multiple dependencies at once. Cell magic will create POM file that will contain all dependencies listed in cell."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%classpath add mvn\n",
+    "org.slf4j slf4j-api 1.7.25\n",
+    "org.slf4j slf4j-nop 1.7.25"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},

diff --git a/doc/groovy/DataVec.ipynb b/doc/groovy/DataVec.ipynb
@@ -15,19 +15,18 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "%classpath add mvn org.datavec datavec-api 0.9.1\n",
-    "%classpath add mvn org.datavec datavec-local 0.9.1\n",
-    "%classpath add mvn org.datavec datavec-dataframe 0.9.1\n",
-    "%classpath add mvn org.deeplearning4j deeplearning4j-core 0.9.1\n",
-    "%classpath add mvn org.nd4j nd4j-native-platform 0.9.1"
+    "%%classpath add mvn\n",
+    "org.datavec datavec-api 0.9.1\n",
+    "org.datavec datavec-local 0.9.1\n",
+    "org.datavec datavec-dataframe 0.9.1\n",
+    "org.deeplearning4j deeplearning4j-core 0.9.1\n",
+    "org.nd4j nd4j-native-platform 0.9.1"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {
-    "collapsed": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "%import org.nd4j.linalg.api.ndarray.INDArray\n",
@@ -345,9 +344,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {
-    "collapsed": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": []
   }

diff --git a/doc/groovy/Tablesaw.ipynb b/doc/groovy/Tablesaw.ipynb
@@ -18,9 +18,18 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "%classpath add mvn tech.tablesaw tablesaw-plot 0.11.4\n",
-    "%classpath add mvn tech.tablesaw tablesaw-smile 0.11.4\n",
-    "%classpath add mvn tech.tablesaw tablesaw-beakerx 0.11.4\n",
+    "%%classpath add mvn\n",
+    "tech.tablesaw tablesaw-plot 0.11.4\n",
+    "tech.tablesaw tablesaw-smile 0.11.4\n",
+    "tech.tablesaw tablesaw-beakerx 0.11.4"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
     "%import tech.tablesaw.aggregate.*\n",
     "%import tech.tablesaw.api.*\n",
     "%import tech.tablesaw.api.ml.clustering.*\n",
@@ -369,9 +378,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {
-    "collapsed": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": []
   }

diff --git a/doc/scala/Flint.ipynb b/doc/scala/Flint.ipynb
@@ -20,11 +20,20 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "%classpath config resolver jitpack.io https://jitpack.io\n",
-    "%classpath add mvn com.github.twosigma flint master-SNAPSHOT\n",
-    "%classpath add mvn org.apache.spark spark-sql_2.11 2.2.1\n",
-    "%classpath add mvn org.apache.spark spark-mllib_2.11 2.2.1\n",
-    "%classpath add mvn org.scalanlp breeze_2.10 0.13.2"
+    "%classpath config resolver jitpack.io https://jitpack.io"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%classpath add mvn\n",
+    "com.github.twosigma flint master-SNAPSHOT\n",
+    "org.apache.spark spark-sql_2.11 2.2.1\n",
+    "org.apache.spark spark-mllib_2.11 2.2.1\n",
+    "org.scalanlp breeze_2.10 0.13.2"
    ]
   },
   {

diff --git a/doc/scala/Spark.ipynb b/doc/scala/Spark.ipynb
@@ -17,7 +17,8 @@
    },
    "outputs": [],
    "source": [
-    "%classpath add mvn org.apache.spark spark-sql_2.11 2.2.1"
+    "%%classpath add mvn\n",
+    "org.apache.spark spark-sql_2.11 2.2.1"
    ]
   },
   {

diff --git a/...ase/src/main/java/com/twosigma/beakerx/kernel/magic/command/MagicCommandTypesFactory.java b/...ase/src/main/java/com/twosigma/beakerx/kernel/magic/command/MagicCommandTypesFactory.java
@@ -20,15 +20,16 @@
 import com.twosigma.beakerx.kernel.magic.command.functionality.AddStaticImportMagicCommand;
 import com.twosigma.beakerx.kernel.magic.command.functionality.BashMagicCommand;
 import com.twosigma.beakerx.kernel.magic.command.functionality.ClasspathAddDynamicMagicCommand;
+import com.twosigma.beakerx.kernel.magic.command.functionality.ClassPathAddMvnCellMagicCommand;
 import com.twosigma.beakerx.kernel.magic.command.functionality.ClasspathAddJarMagicCommand;
 import com.twosigma.beakerx.kernel.magic.command.functionality.ClasspathAddMvnMagicCommand;
 import com.twosigma.beakerx.kernel.magic.command.functionality.ClasspathAddRepoMagicCommand;
 import com.twosigma.beakerx.kernel.magic.command.functionality.ClasspathRemoveMagicCommand;
 import com.twosigma.beakerx.kernel.magic.command.functionality.ClasspathShowMagicCommand;
 import com.twosigma.beakerx.kernel.magic.command.functionality.HtmlAliasMagicCommand;
 import com.twosigma.beakerx.kernel.magic.command.functionality.HtmlMagicCommand;
-import com.twosigma.beakerx.kernel.magic.command.functionality.JavaScriptMagicCommand;
 import com.twosigma.beakerx.kernel.magic.command.functionality.JSMagicCommand;
+import com.twosigma.beakerx.kernel.magic.command.functionality.JavaScriptMagicCommand;
 import com.twosigma.beakerx.kernel.magic.command.functionality.LoadMagicMagicCommand;
 import com.twosigma.beakerx.kernel.magic.command.functionality.LsMagicCommand;
 import com.twosigma.beakerx.kernel.magic.command.functionality.TimeCellModeMagicCommand;
@@ -57,6 +58,7 @@ public static List<MagicCommandType> createDefaults(KernelFunctionality kernel)
                     lsmagic(magicCommandTypes),
                     addJar(kernel),
                     addJarByMvn(kernel),
+                    addJarByMvnCell(kernel),
                     addDynamic(kernel),
                     addRepo(kernel),
                     removeJar(kernel),
@@ -123,6 +125,13 @@ private static MagicCommandType addJarByMvn(KernelFunctionality kernel) {
                     kernel.getTempFolder().toString() + MVN_DIR), kernel));
   }
 
+  private static MagicCommandType addJarByMvnCell(KernelFunctionality kernel) {
+    return new MagicCommandType(ClassPathAddMvnCellMagicCommand.CLASSPATH_ADD_MVN_CELL, "<group name version>",
+            new ClassPathAddMvnCellMagicCommand(new MavenJarResolver.ResolverParams(
+                    kernel.getCacheFolder().toString() + "/maven/cache",
+                    kernel.getTempFolder().toString() + MVN_DIR), kernel));
+  }
+
   private static MagicCommandType addJar(KernelFunctionality kernel) {
     return new MagicCommandType(ClasspathAddJarMagicCommand.CLASSPATH_ADD_JAR, "<jar path>", new ClasspathAddJarMagicCommand(kernel));
   }

diff --git a/kernel/base/src/main/java/com/twosigma/beakerx/kernel/magic/command/MavenJarResolver.java b/kernel/base/src/main/java/com/twosigma/beakerx/kernel/magic/command/MavenJarResolver.java
@@ -15,27 +15,26 @@
  */
 package com.twosigma.beakerx.kernel.magic.command;
 
-import static com.google.common.base.Preconditions.checkNotNull;
-
+import com.google.common.base.Preconditions;
 import com.twosigma.beakerx.kernel.commands.MavenInvocationSilentOutputHandler;
 import com.twosigma.beakerx.kernel.commands.MavenJarResolverSilentLogger;
+import com.twosigma.beakerx.kernel.magic.command.functionality.MvnLoggerWidget;
+import org.apache.commons.io.FileUtils;
+import org.apache.maven.shared.invoker.DefaultInvocationRequest;
+import org.apache.maven.shared.invoker.DefaultInvoker;
+import org.apache.maven.shared.invoker.InvocationRequest;
+import org.apache.maven.shared.invoker.InvocationResult;
+import org.apache.maven.shared.invoker.Invoker;
 
 import java.io.File;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.UUID;
 
-import com.twosigma.beakerx.kernel.magic.command.functionality.MvnLoggerWidget;
-import org.apache.commons.io.FileUtils;
-import org.apache.maven.shared.invoker.DefaultInvocationRequest;
-import org.apache.maven.shared.invoker.DefaultInvoker;
-import org.apache.maven.shared.invoker.InvocationRequest;
-import org.apache.maven.shared.invoker.InvocationResult;
-import org.apache.maven.shared.invoker.Invoker;
-
 public class MavenJarResolver {
 
   public static final String MVN_DIR = File.separator + "mvnJars";
@@ -49,17 +48,21 @@ public class MavenJarResolver {
 
   public MavenJarResolver(final ResolverParams commandParams,
                           PomFactory pomFactory) {
-    this.commandParams = checkNotNull(commandParams);
+    this.commandParams = Preconditions.checkNotNull(commandParams);
     this.pathToMavenRepo = getOrCreateFile(commandParams.getPathToNotebookJars()).getAbsolutePath();
     this.pomFactory = pomFactory;
   }
 
   public AddMvnCommandResult retrieve(String groupId, String artifactId, String version, MvnLoggerWidget progress) {
+    List<Dependency> dependencies = Arrays.asList(new Dependency(groupId, artifactId, version));
+    return retrieve(dependencies, progress);
+  }
+
+  public AddMvnCommandResult retrieve(List<Dependency> dependencies, MvnLoggerWidget progress) {
     File finalPom = null;
     try {
-      Dependency dependency = new Dependency(groupId, artifactId, version);
-      String pomAsString = pomFactory.createPom(pathToMavenRepo, dependency, commandParams.getRepos());
-      finalPom = saveToFile(commandParams.getPathToNotebookJars(), dependency, pomAsString);
+      String pomAsString = pomFactory.createPom(pathToMavenRepo, dependencies, commandParams.getRepos());
+      finalPom = saveToFile(commandParams.getPathToNotebookJars(), pomAsString);
       InvocationRequest request = createInvocationRequest();
       request.setOffline(commandParams.getOffline());
       request.setPomFile(finalPom);
@@ -68,18 +71,17 @@ public AddMvnCommandResult retrieve(String groupId, String artifactId, String ve
       progress.display();
       InvocationResult invocationResult = invoker.execute(request);
       progress.close();
-      return getResult(invocationResult, groupId, artifactId, version);
+      return getResult(invocationResult, dependencies);
     } catch (Exception e) {
       return AddMvnCommandResult.error(e.getMessage());
     } finally {
       deletePomFolder(finalPom);
     }
   }
 
-  private File saveToFile(String pathToNotebookJars, Dependency dependency, String pomAsString)
+  private File saveToFile(String pathToNotebookJars, String pomAsString)
           throws IOException {
-    File finalPom = new File(pathToNotebookJars + "/poms/pom-" + UUID.randomUUID() + "-" +
-            dependency.getGroupId() + dependency.getArtifactId() + dependency.getVersion() + "xml");
+    File finalPom = new File(pathToNotebookJars + "/poms/pom-" + UUID.randomUUID() + "-" + "xml");
 
     FileUtils.writeStringToFile(finalPom, pomAsString, StandardCharsets.UTF_8);
     return finalPom;
@@ -115,12 +117,19 @@ public String findMvn() {
     return mavenLocation;
   }
 
-  private AddMvnCommandResult getResult(InvocationResult invocationResult, String groupId, String artifactId, String version) {
+  private AddMvnCommandResult getResult(InvocationResult invocationResult, List<Dependency> dependencies) {
     if (invocationResult.getExitCode() != 0) {
       if (invocationResult.getExecutionException() != null) {
         return AddMvnCommandResult.error(invocationResult.getExecutionException().getMessage());
       }
-      return AddMvnCommandResult.error("Could not resolve dependencies for: " + groupId + " : " + artifactId + " : " + version);
+      StringBuilder errorMsgBuilder = new StringBuilder("Could not resolve dependencies for:");
+      for (Dependency dependency : dependencies) {
+        errorMsgBuilder
+                .append("\n").append(dependency.groupId).append(" : ")
+                .append(dependency.artifactId).append(" : ")
+                .append(dependency.version);
+      }
+      return AddMvnCommandResult.error(errorMsgBuilder.toString());
     }
 
     return AddMvnCommandResult.SUCCESS;
@@ -207,8 +216,8 @@ public static class ResolverParams {
     private Map<String, String> repos = Collections.emptyMap();
 
     public ResolverParams(String pathToCache, String pathToNotebookJars, boolean offline) {
-      this.pathToCache = checkNotNull(pathToCache);
-      this.pathToNotebookJars = checkNotNull(pathToNotebookJars);
+      this.pathToCache = Preconditions.checkNotNull(pathToCache);
+      this.pathToNotebookJars = Preconditions.checkNotNull(pathToNotebookJars);
       this.offline = offline;
     }
 

diff --git a/kernel/base/src/main/java/com/twosigma/beakerx/kernel/magic/command/PomFactory.java b/kernel/base/src/main/java/com/twosigma/beakerx/kernel/magic/command/PomFactory.java
@@ -15,38 +15,42 @@
  */
 package com.twosigma.beakerx.kernel.magic.command;
 
-import static com.twosigma.beakerx.kernel.magic.command.MavenJarResolver.POM_XML;
-
 import com.twosigma.beakerx.kernel.magic.command.MavenJarResolver.Dependency;
+import org.apache.commons.io.IOUtils;
 
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.charset.StandardCharsets;
+import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 
-import org.apache.commons.io.IOUtils;
-
 public class PomFactory {
 
-  public String createPom(String pathToMavenRepo, Dependency dependency, Map<String, String> repos) throws IOException {
-    InputStream pom = getClass().getClassLoader().getResourceAsStream(POM_XML);
+  public String createPom(String pathToMavenRepo, List<Dependency> dependencies, Map<String, String> repos) throws IOException {
+    InputStream pom = getClass().getClassLoader().getResourceAsStream(MavenJarResolver.POM_XML);
     String pomAsString = IOUtils.toString(pom, StandardCharsets.UTF_8);
     pomAsString = configureOutputDir(pathToMavenRepo, pomAsString);
-    pomAsString = configureDependencies(dependency.getGroupId(), dependency.getArtifactId(), dependency.getVersion(), pomAsString);
+    pomAsString = configureDependencies(dependencies, pomAsString);
     pomAsString = configureRepos(repos, pomAsString);
     return pomAsString;
   }
 
-  private String configureDependencies(String groupId, String artifactId, String version, String pomAsString) {
+  private String configureDependencies(List<Dependency> dependencies, String pomAsString) {
+    for (Dependency dependency : dependencies) {
+      pomAsString = configureDependency(dependency, pomAsString);
+    }
+    return pomAsString;
+  }
+
+  private String configureDependency(Dependency dependency, String pomAsString) {
     return pomAsString.replace(
-            "<dependencies></dependencies>",
-            "<dependencies>\n" +
+            "</dependencies>",
                     "  <dependency>\n" +
-                    "    <groupId>" + groupId + "</groupId>\n" +
-                    "    <artifactId>" + artifactId + "</artifactId>\n" +
-                    "    <version>" + version + "</version>\n" +
+                    "    <groupId>" + dependency.groupId + "</groupId>\n" +
+                    "    <artifactId>" + dependency.artifactId + "</artifactId>\n" +
+                    "    <version>" + dependency.version + "</version>\n" +
                     "  </dependency>\n" +
                     "</dependencies>");
   }