Multi tenancy support

bin/README.md

@@ -0,0 +1,29 @@
+# Description of CI process
+
+The wonderful AMPlab Jenkins is responsible for running our integration test.
+
+## How does the CI process work
+
+
+0. Jenkins pull the PR.
+1. Jenkins inject environment variables configured in admin page. Currently, we set the following environment variables 
+in Pull Request Builder:
+    - AWS_ACCESS_KEY_ID
+    - AWS_SECRET_ACCESS_KEY
+    - CLIPPER_K8S_CERT_AUTH
+    - CLIPPER_K8S_CLIENT_CERT
+    - CLIPPER_K8S_CLIENT_KEY
+    - CLIPPER_K8S_PASSWORD
+    - CLIPPER_TESTING_DOCKERHUB_PASSWORD
+2. Jenkins will call `run_ci.sh`. It does three things:
+    - It calls `build_docker_images.sh` to build all the docker images
+    - Then it runs unittests docker container for python2 and python3
+    - Each unittest cotnainer will run `ci_checks.sh`
+3. `ci_checks.sh` will run two things:
+    - (Only in Python2) It runs `check_foramt.sh` to run the linter for C++ and Python
+    - It runs `run_unnitests.sh` to run all tests. 
+        - (Only in Python3) It will only run the integration test part, which contains all the python tests and R tests
+
+## Note on Minikube (WIP)
+- We are under the process of moving away from AWS EKS to Minikube in our CI process. Once the PR is in, there will be 
+more detail here. 

bin/build_docker_images.sh

@@ -14,7 +14,7 @@
 # pin their docker images to the minor version and get updates with new patches
 # automatically.
 
-
+set -x
 set -e
 set -u
 set -o pipefail
@@ -31,7 +31,7 @@ cd $CLIPPER_ROOT
 
 # Initialize tags
 version_tag=$(<VERSION.txt)
-sha_tag=`git rev-parse --verify --short HEAD`
+sha_tag=`git rev-parse --verify --short=10 HEAD`
 
 ######## Utilities for managing versioning ############
 # From https://github.com/cloudflare/semver_bash/blob/c1133faf0efe17767b654b213f212c326df73fa3/semver.sh
@@ -189,7 +189,11 @@ set_version_tag () {
 
 set_version_tag
 
-namespace="clipper"
+namespace=$(docker info | grep Username | awk '{ print $2 }')
+
+# Clear clipper_docker_images.txt for future write
+rm -f $CLIPPER_ROOT/bin/clipper_docker_images.txt
+touch $CLIPPER_ROOT/bin/clipper_docker_images.txt
 
 # We build images with the SHA tag to try to prevent clobbering other images
 # being built from different branches on the same machine. This is particularly
@@ -211,11 +215,15 @@ create_image () {
 
 
     echo "Building $namespace/$image:$sha_tag from file $dockerfile"
-    time docker build --build-arg CODE_VERSION=$sha_tag $rpc_version -t $namespace/$image:$sha_tag \
+    time docker build --build-arg CODE_VERSION=$sha_tag --build-arg REGISTRY=$namespace $rpc_version -t $namespace/$image:$sha_tag \
         -f dockerfiles/$dockerfile $CLIPPER_ROOT
-    docker tag $namespace/$image:$sha_tag $namespace/$image:$version_tag
+
+    echo "Image tag appended to CLIPPER_ROOT/bin/clipper_docker_images.txt"
+    echo "$namespace/$image:$sha_tag" >> $CLIPPER_ROOT/bin/clipper_docker_images.txt
 
     if [ "$publish" = true ] && [ "$public" = true ] ; then
+        docker tag $namespace/$image:$sha_tag $namespace/$image:$version_tag
+
         echo "Publishing $namespace/$image:$sha_tag"
         docker push $namespace/$image:$sha_tag
         echo "Publishing $namespace/$image:$version_tag"
@@ -285,7 +293,6 @@ build_images () {
     create_image pyspark-container PySparkContainerDockerfile $public py &
     create_image pyspark35-container PySparkContainerDockerfile $public py35 &
     create_image pyspark36-container PySparkContainerDockerfile $public py36 &
-    wait
 
     create_image tf-container TensorFlowDockerfile $public py &
     create_image tf35-container TensorFlowDockerfile $public py35 &
@@ -295,7 +302,6 @@ build_images () {
     create_image pytorch-container PyTorchContainerDockerfile $public py &
     create_image pytorch35-container PyTorchContainerDockerfile $public py35 &
     create_image pytorch36-container PyTorchContainerDockerfile $public py36 &
-    wait
 
     # See issue #475
     # create_image caffe2-onnx-container Caffe2OnnxDockerfile $public py

bin/ci_checks.sh

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 
+set -x
 set -e
 set -u
 set -o pipefail
@@ -26,10 +27,10 @@ tag=$(<VERSION.txt)
 cd -
 
 # Log in to Kubernetes Docker repo
-$DIR/aws_docker_repo_login.sh
+# $DIR/aws_docker_repo_login.sh
 
 # Test docker login
-docker pull 568959175238.dkr.ecr.us-west-1.amazonaws.com/clipper/query_frontend:$tag
+# docker pull 568959175238.dkr.ecr.us-west-1.amazonaws.com/clipper/query_frontend:$tag
 
 # Set up credentials for K8s testing cluster.
 export KUBECONFIG=~/kubeconfig_$(date +"%Y%m%d%H%M%S")
@@ -51,6 +52,9 @@ kubectl get nodes
 # Set kubectl proxy for k8s tests later
 kubectl proxy --port 8080 &
 
+# Login to clippertesting dockerhub here
+docker login --username="clippertesting" --password=$CLIPPER_TESTING_DOCKERHUB_PASSWORD
+
 if [[ $run_all = "true" ]]; then
     $DIR/check_format.sh
     $DIR/run_unittests.sh

bin/run_ci.sh

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 
+set -x
 set -e
 set -u
 set -o pipefail
@@ -21,12 +22,15 @@ tag=$(<VERSION.txt)
 # Build docker images
 ./bin/build_docker_images.sh
 
-# Tag and push the latest version of the Clipper Docker images to the container registry
-# for the Kubernetes testing cluster
-docker tag clipper/query_frontend:$tag 568959175238.dkr.ecr.us-west-1.amazonaws.com/clipper/query_frontend:$tag
-docker push 568959175238.dkr.ecr.us-west-1.amazonaws.com/clipper/query_frontend:$tag
-docker tag clipper/management_frontend:$tag 568959175238.dkr.ecr.us-west-1.amazonaws.com/clipper/management_frontend:$tag
-docker push 568959175238.dkr.ecr.us-west-1.amazonaws.com/clipper/management_frontend:$tag
+echo "Pushing the following images"
+cat ./bin/clipper_docker_images.txt
+
+# Push docker images
+while read in; do docker push "$in"; done < ./bin/clipper_docker_images.txt
+
+
+CLIPPER_REGISTRY=$(docker info | grep Username | awk '{ print $2 }')
+sha_tag=$(git rev-parse --verify --short=10 HEAD)
 
 # Run tests
 docker run --rm --network=host -v /var/run/docker.sock:/var/run/docker.sock -v /tmp:/tmp \
@@ -36,7 +40,9 @@ docker run --rm --network=host -v /var/run/docker.sock:/var/run/docker.sock -v /
     -e CLIPPER_K8S_PASSWORD=$CLIPPER_K8S_PASSWORD \
     -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
     -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
-    clipper/unittests:$tag
+    -e CLIPPER_REGISTRY=$CLIPPER_REGISTRY \
+    -e CLIPPER_TESTING_DOCKERHUB_PASSWORD=$CLIPPER_TESTING_DOCKERHUB_PASSWORD \
+    $CLIPPER_REGISTRY/unittests:$sha_tag
 
 # Python 3 unittests
 docker run --rm --network=host -v /var/run/docker.sock:/var/run/docker.sock -v /tmp:/tmp \
@@ -46,4 +52,6 @@ docker run --rm --network=host -v /var/run/docker.sock:/var/run/docker.sock -v /
     -e CLIPPER_K8S_PASSWORD=$CLIPPER_K8S_PASSWORD \
     -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
     -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
-    clipper/py35tests:$tag
+    -e CLIPPER_REGISTRY=$CLIPPER_REGISTRY \
+    -e CLIPPER_TESTING_DOCKERHUB_PASSWORD=$CLIPPER_TESTING_DOCKERHUB_PASSWORD \
+    $CLIPPER_REGISTRY/py35tests:$sha_tag

bin/run_unittests.sh

@@ -91,7 +91,7 @@ function run_jvm_container_tests {
   echo "Running JVM container tests..."
   cd $DIR
   cd ../containers/jvm
-  mvn test
+  mvn test -q
 }
 
 function run_r_container_tests {
@@ -129,6 +129,10 @@ function run_frontend_tests {
 function run_integration_tests {
   echo -e "\nRunning integration tests\n\n"
   cd $DIR
+
+  echo "GREPTHIS Docker State before:"
+  docker ps
+
   python ../integration-tests/clipper_admin_tests.py
   python ../integration-tests/many_apps_many_models.py 2 3
   python ../integration-tests/deploy_pyspark_models.py
@@ -137,13 +141,20 @@ function run_integration_tests {
   python ../integration-tests/kubernetes_integration_test.py
   python ../integration-tests/kubernetes_multi_frontend.py
   python ../integration-tests/deploy_tensorflow_models.py
-  python ../integration-tests/deploy_mxnet_models.py 
-  python ../integration-tests/deploy_pytorch_models.py 
+  python ../integration-tests/deploy_mxnet_models.py
+  python ../integration-tests/deploy_pytorch_models.py
   # See issue #475
   # python ../integration-tests/deploy_pytorch_to_caffe2_with_onnx.py
   ../integration-tests/r_integration_test/rclipper_test.sh
-  python ../integration-tests/clipper_metric_docker.py 
-  python ../integration-tests/clipper_metric_kube.py 
+  python ../integration-tests/clipper_metric_docker.py
+  python ../integration-tests/clipper_metric_kube.py
+  python ../integration-tests/multi_tenancy_test.py
+  python ../integration-tests/multi_tenancy_test.py --kubernetes
+  python ../integration-tests/kubernetes_namespace.py
+
+  echo "GREPTHIS Docker State After"
+  docker ps
+
   echo "Exit code: $?"
   echo "GREPTHIS Done running unit tests"
 }

clipper_admin/clipper_admin/__init__.py

@@ -4,5 +4,5 @@
 from .kubernetes.kubernetes_container_manager import KubernetesContainerManager
 from .clipper_admin import *
 from . import deployers
-from .version import __version__
+from .version import __version__, __registry__
 from .exceptions import ClipperException, UnconnectedException