Feature: TensorflowOnSpark python plugin (#525)

* initial commit

* update

* update

* add gpu support

* remove comment

* change class to function

* fix merge issue

* add some docs

aztk/models/plugins/internal/plugin_manager.py

@@ -23,6 +23,7 @@ class PluginManager:
         hdfs=plugins.HDFSPlugin,
         simple=plugins.SimplePlugin,
         spark_ui_proxy=plugins.SparkUIProxyPlugin,
+        tensorflow_on_spark=plugins.TensorflowOnSparkPlugin,
         openblas=plugins.OpenBLASPlugin,
         nvblas=plugins.NvBLASPlugin,
     )

aztk/spark/models/plugins/__init__.py

@@ -5,5 +5,6 @@
 from .rstudio_server import RStudioServerPlugin
 from .simple import SimplePlugin
 from .spark_ui_proxy import SparkUIProxyPlugin
+from .tensorflow_on_spark import TensorflowOnSparkPlugin
 from .openblas import OpenBLASPlugin
 from .nvblas import NvBLASPlugin

aztk/spark/models/plugins/tensorflow_on_spark/__init__.py

@@ -0,0 +1 @@
+from .configuration import *

aztk/spark/models/plugins/tensorflow_on_spark/configuration.py

@@ -0,0 +1,17 @@
+import os
+from aztk.models.plugins.plugin_configuration import PluginConfiguration, PluginPort, PluginTargetRole
+from aztk.models.plugins.plugin_file import PluginFile
+from aztk.utils import constants
+
+dir_path = os.path.dirname(os.path.realpath(__file__))
+
+
+def TensorflowOnSparkPlugin():
+    return PluginConfiguration(
+        name="tensorflow_on_spark",
+        target_role=PluginTargetRole.Master,
+        execute="tensorflow_on_spark.sh",
+        files=[
+            PluginFile("tensorflow_on_spark.sh", os.path.join(dir_path, "tensorflow_on_spark.sh")),
+        ],
+    )

aztk/spark/models/plugins/tensorflow_on_spark/tensorflow_on_spark.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# This plugin requires HDFS to be enabled and on the path
+
+# setup TensorFlowOnSpark
+git clone https://github.com/yahoo/TensorFlowOnSpark.git
+cd TensorFlowOnSpark
+export TFoS_HOME=$(pwd)
+export TFoS_HOME=~/TensorFlowOnSpark >> ~/.bashrc
+
+if  [ "$AZTK_GPU_ENABLED" = "true" ]; then
+    pip install tensorflow-gpu
+    pip install tensorflowonspark
+else
+    pip install tensorflow-cpu
+    pip install tensorflowonspark
+fi
+
+# add libhdfs.so to path
+echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$HADOOP_HOME/lib/native/libhdfs.so" >> ~/.bashrc

aztk_cli/config/cluster.yaml

@@ -44,6 +44,7 @@ plugins:
   # - name: hdfs
   # - name: rstudio_server
   # - name: spark_ui_proxy
+  # - name: tensorflow_on_spark
   # - name: openblas
   # - name: nvblas
 

docs/15-plugins.md

@@ -13,7 +13,9 @@ Currently the following plugins are supported:
 - Jupyter
 - HDFS
 - RStudioServer
-- Spark UI Proxy
+- TensorflowOnSpark
+- OpenBLAS
+- mvBLAS
 
 ### Enable a plugin using the CLI
 If you are uing the `aztk` CLI and wish to enable a supported plugin, you need to update you `.aztk/cluster.yaml` configuration file.
@@ -38,7 +40,6 @@ cluster_config = ClusterConfiguration(
   ...# Other config,
   plugins=[
     JupyterPlugin(),
-    RStudioServerPlugin(version="1.1.383"),
     HDFSPlugin(),
   ]
 )