Feature: spark move jars on docker image

README.md

@@ -27,7 +27,7 @@ This toolkit is built on top of Azure Batch but does not require any Azure Batch
 ```
 3. Login or register for an [Azure Account](https://azure.microsoft.com), navigate to [Azure Cloud Shell](https://shell.azure.com), and run:
 ```sh
-wget -q https://raw.githubusercontent.com/Azure/aztk/master/account_setup.sh -O account_setup.sh &&
+wget -q https://raw.githubusercontent.com/Azure/aztk/v0.7.0/account_setup.sh -O account_setup.sh &&
 chmod 755 account_setup.sh &&
 /bin/bash account_setup.sh
 ```

account_setup.sh

@@ -4,7 +4,7 @@ echo "Installing depdendencies..." &&
 pip install --force-reinstall --upgrade --user pyyaml==3.12 azure==3.0.0 azure-cli-core==2.0.30 msrestazure==0.4.25 > /dev/null 2>&1 &&
 echo "Finished installing depdencies." &&
 echo "Getting account setup script..." &&
-wget -q https://raw.githubusercontent.com/Azure/aztk/master/account_setup.py -O account_setup.py &&
+wget -q https://raw.githubusercontent.com/Azure/aztk/v0.7.0/account_setup.py -O account_setup.py &&
 chmod 755 account_setup.py &&
 echo "Finished getting account setup script." &&
 echo "Running account setup script..." &&

aztk/node_scripts/Pipfile

@@ -0,0 +1,17 @@
+[[source]]
+url = "https://pypi.python.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+azure-batch = "==4.1.3"
+azure-mgmt-batch = "==5.0.0"
+azure-mgmt-storage = "==1.5.0"
+azure-storage-blob = "==1.1.0"
+pycryptodome = "==3.4.7"
+PyYAML = "==3.12"
+
+[dev-packages]
+
+[requires]
+python_version = "3.5"

aztk/node_scripts/docker_main.sh

@@ -3,6 +3,7 @@
 # This file is the entry point of the docker container.
 
 set -e
+source ~/.bashrc
 echo "Initializing spark container"
 
 # --------------------
@@ -25,15 +26,14 @@ done
 # ----------------------------
 # Run aztk setup python scripts
 # ----------------------------
-# use python v3.5.4 to run aztk software
+# setup docker container
 echo "Starting setup using Docker"
 
-$(pyenv root)/versions/$AZTK_PYTHON_VERSION/bin/pip install -r $(dirname $0)/requirements.txt
 export PYTHONPATH=$PYTHONPATH:$AZTK_WORKING_DIR
 echo 'export PYTHONPATH=$PYTHONPATH:$AZTK_WORKING_DIR' >> ~/.bashrc
 
 echo "Running main.py script"
-$(pyenv root)/versions/$AZTK_PYTHON_VERSION/bin/python $(dirname $0)/main.py setup-spark-container
+$AZTK_WORKING_DIR/.aztk-env/.venv/bin/python $(dirname $0)/main.py setup-spark-container
 
 # sleep to keep container running
 while true; do sleep 1; done

aztk/node_scripts/requirements.txt

@@ -4,3 +4,4 @@ azure-mgmt-storage==1.5.0
 azure-storage-blob==1.1.0
 pyyaml==3.12
 pycryptodome==3.4.7
+