Add smoke test errand job to verify the deployment

add golang package
add busybox package
add smoke-test package
add smoke-test job
use bosh link to get harbor app conn
provide deployment manifests for errand job running with harbor job or on separate node

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "src/harbor-api-testing"]
+	path = src/harbor-api-testing
+	url = https://gitlab.eng.vmware.com/harbor/harbor-api-testing.git

config/blobs.yml

@@ -1,12 +1,20 @@
 docker/docker-17.06.2-ce.tgz:
   size: 29704322
+  object_id: 1855e6b5-8861-46c6-5be7-74f7cdc192bf
   sha: 2a4206fea5d3d3f4373e13569538522417e2c9d3
 docker/docker-compose-Linux-x86_64-1.16.1:
   size: 8855632
+  object_id: 9466444f-ff50-4196-7e24-1fe7c0a178cc
   sha: e2c7c848e1fa388a2e5b8945fdb2660bf8d8adb1
+go/go1.9.2.linux-amd64.tar.gz:
+  size: 104247844
+  object_id: 91c263af-00db-4297-4014-15159b1edc6b
+  sha: 94c889e039e3d2e94ed95e8f8cb747c5bc1c2b58
 harbor/harbor-offline-installer-latest.tgz:
-  size: 817725606
-  sha: eeeea982e6d515a96b6e74f1f525ea843db820d9
+  size: 941077808
+  object_id: 1deeb0cd-4176-4c70-621c-656f8682e303
+  sha: 4f2f4f7fb2a85fd4c5aad0ef1703c101c50bbb9d
 library/openssl-1.0.2l.tar.gz:
   size: 5365054
+  object_id: 436e629e-41bf-4607-7ebe-f810dac74ac2
   sha: b58d5d0e9cea20e571d903aafa853e2ccd914138

jobs/harbor/spec

@@ -18,6 +18,18 @@ packages:
 - harbor-app
 - openssl
 
+provides:
+- name: harbor
+  type: harbor_conn
+  properties:
+  - hostname
+  - ui_url_protocol
+  - admin_password
+  - ssl.cert
+  - ssl.key
+  - ssl.ca
+  - populate_etc_hosts
+
 properties:
   hostname:
     description: "The IP address or hostname to access admin UI and registry service"

jobs/smoke-test/spec

@@ -0,0 +1,19 @@
+---
+name: smoke-test
+
+templates:
+  run.sh: bin/run
+  config/ca.crt: config/ca.crt
+  config/cert.crt: config/cert.crt
+  config/key.crt: config/key.crt
+
+packages:
+  - smoke-test
+  - docker
+  - busybox
+
+consumes:
+- name: harbor_reference
+  type: harbor_conn
+
+properties: {}

jobs/smoke-test/templates/config/ca.crt

@@ -0,0 +1 @@
+<%= link('harbor_reference').p('ssl.ca') %>

jobs/smoke-test/templates/config/cert.crt

@@ -0,0 +1 @@
+<%= link('harbor_reference').p('ssl.cert') %>

jobs/smoke-test/templates/config/key.crt

@@ -0,0 +1 @@
+<%= link('harbor_reference').p('ssl.key') %>

jobs/smoke-test/templates/run.sh

@@ -0,0 +1,149 @@
+#!/bin/bash
+
+set -e -u -x
+
+JOB_NAME=smoke-test
+
+# Prepare directories
+RUN_DIR="/var/vcap/sys/run/docker" # For docker daemon
+mkdir -p ${RUN_DIR}
+chown -R vcap:vcap ${RUN_DIR}
+chmod 755 ${RUN_DIR}
+
+# Set package dependencies paths
+PACKAGES_DIR=${BOSH_PACKAGES_DIR:-/var/vcap/packages}
+SMOKE_TEST_JOB_DIR=/var/vcap/jobs/$JOB_NAME
+
+DOCKER_PACKAGE_DIR=$PACKAGES_DIR/docker
+DOCKERD=$DOCKER_PACKAGE_DIR/bin/dockerd
+export PATH=$PATH:${DOCKER_PACKAGE_DIR}/bin
+
+CASE_BASE_DIR=${PACKAGES_DIR}/smoke-test
+TEST_ENTRYPOINT=$CASE_BASE_DIR/bin/smoke-test
+
+# Set docker runtime files
+DOCKER_DAEMON_PIDFILE=$RUN_DIR/dockerd.pid
+DOCKER_DAEMON_SOCK=$RUN_DIR/dockerd.sock
+DATA_ROOT_DIR="/var/vcap/store"
+DOCKER_HOST="unix://$DOCKER_DAEMON_SOCK"
+
+#### Utility functions
+# Prepare docker environment
+prepareDockerEnv() {
+  ulimit -n 8192
+
+  if grep -v '^#' /etc/fstab | grep -q cgroup || [ ! -e /proc/cgroups ] || [ ! -d /sys/fs/cgroup ]; then
+    mkdir -p /sys/fs/cgroup
+  fi
+  if ! mountpoint -q /sys/fs/cgroup; then
+    mount -t tmpfs -o uid=0,gid=0,mode=0755 cgroup /sys/fs/cgroup
+  fi
+  (cd /sys/fs/cgroup
+  for sys in $(awk '!/^#/ { if ($4 == 1) print $1 }' /proc/cgroups); do
+    mkdir -p $sys
+    if ! mountpoint -q $sys; then
+      if ! mount -n -t cgroup -o $sys cgroup $sys; then
+        rmdir $sys || true
+      fi
+    fi
+  done)
+}
+
+# Start docker daemon
+startDockerd() {
+  OPT="--data-root ${DATA_ROOT_DIR} --host $DOCKER_HOST"
+
+  /sbin/start-stop-daemon \
+  --pidfile $DOCKER_DAEMON_PIDFILE \
+  --make-pidfile \
+  --background \
+  --exec $DOCKERD \
+  --oknodo \
+  --start \
+  -- $OPT
+
+  echo "Docker daemon started"
+}
+
+# Stop the dockerd process
+stopDockerd() {
+  if /sbin/start-stop-daemon --pidfile $DOCKER_DAEMON_PIDFILE --retry TERM/30/QUIT/5/KILL --oknodo --stop; then
+    rm -f $DOCKER_DAEMON_PIDFILE
+    rm -f $DOCKER_DAEMON_SOCK
+  fi
+
+  echo "Docker daemon stopped"
+}
+
+# Waiting for docker daemon started
+waitForDockerd() {
+  sleep_time=3
+  timeout=60
+  count=0
+  while ! docker -H $DOCKER_HOST version >/dev/null
+  do
+    echo "Docker daemon is not running. Waiting for $sleep_time seconds then check again."
+    sleep $sleep_time
+    count=$((count + sleep_time));
+    if [ $count -ge $timeout ]; then
+      echo "Error: Docker daemon is still not running after $timeout seconds."
+      exit 1
+    fi
+  done
+  echo "Docker daemon is running"
+}
+
+# Start docker daemon for testing if it is not running (for run on a separate node)
+SHOULD_STOP="NO"
+if ! docker -H $DOCKER_HOST version >/dev/null; then
+  prepareDockerEnv
+  startDockerd
+  #Check and wait
+  waitForDockerd
+
+  SHOULD_STOP="YES"
+fi
+
+# Load testing image : busybox
+BUSYBOX_PATH=$PACKAGES_DIR/busybox/busybox.tar
+docker -H $DOCKER_HOST load -i $BUSYBOX_PATH
+
+# Set testing environment
+export APP_HOST_IP="<%= link('harbor_reference').instances[0].address %>"
+export HTTP_PROTOCOL="<%= link('harbor_reference').p('ui_url_protocol') %>"
+export TESTING_ENV_HOSTNAME="<%= link('harbor_reference').p('hostname') %>"
+export TESTING_ENV_ADMIN_PASS="<%= link('harbor_reference').p('admin_password') %>"
+export TESTING_ENV_PASSWORD="<%= link('harbor_reference').p('admin_password') %>"
+export POPULATE_ETC_HOSTS="<%= link('harbor_reference').p('populate_etc_hosts') %>"
+export CA_FILE_PATH=$SMOKE_TEST_JOB_DIR/config/ca.crt
+export CERT_FILE_PATH=$SMOKE_TEST_JOB_DIR/config/cert.crt
+export KEY_FILE_PATH=$SMOKE_TEST_JOB_DIR/config/key.crt
+export TESTING_DOCKER_HOST=$DOCKER_HOST
+export TESTING_IMAGE_NAME="busybox"
+export TESTING_IMAGE_TAG="latest"
+
+if [ "$POPULATE_ETC_HOSTS" = "true" ]; then
+  if [ "$APP_HOST_IP" != "$TESTING_ENV_HOSTNAME" ]; then
+    sed -i -e "/$APP_HOST_IP/d" /etc/hosts
+    sed -i -e "/$TESTING_ENV_HOSTNAME/d" /etc/hosts
+
+    echo "$APP_HOST_IP    $TESTING_ENV_HOSTNAME" >> /etc/hosts
+  fi
+fi
+
+mkdir -p /etc/docker/certs.d/${TESTING_ENV_HOSTNAME}/
+cp $CA_FILE_PATH /etc/docker/certs.d/${TESTING_ENV_HOSTNAME}/
+
+# Run smoke test
+set +e
+$TEST_ENTRYPOINT
+RET=$?
+set -e
+
+# Stop docker daemon
+if [ SHOULD_STOP = "YES" ]; then
+  stopDockerd
+fi
+
+# Return result
+exit $RET

packages/busybox/packaging

@@ -0,0 +1,3 @@
+set -e
+
+cp -a busybox/*.tar ${BOSH_INSTALL_TARGET}

packages/busybox/spec

@@ -0,0 +1,7 @@
+---
+name: busybox
+
+dependencies: []
+
+files:
+- busybox/busybox.tar

packages/golang/packaging

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -e -u
+
+GO_VERSION=1.9.2
+tar xzvf "${BOSH_COMPILE_TARGET}/go/go${GO_VERSION}.linux-amd64.tar.gz"
+
+cp -a ${BOSH_COMPILE_TARGET}/go/* "${BOSH_INSTALL_TARGET}"

packages/golang/spec

@@ -0,0 +1,5 @@
+---
+name: golang
+
+files:
+  - go/go1.9.2.linux-amd64.tar.gz

packages/smoke-test/packaging

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -e -u
+
+# Set package dependencies directory
+PACKAGES_DIR=${BOSH_PACKAGES_DIR:-/var/vcap/packages}
+
+# Set Golang dependency
+export GOROOT=$(cd "${PACKAGES_DIR}/golang" && pwd -P)
+export PATH=${GOROOT}/bin:${PATH}
+
+# Setup Golang compile workspace
+mkdir -p go_workspace/src
+cp -R harbor-api-testing go_workspace/src
+export GOPATH=$(pwd -P)/go_workspace
+
+mkdir -p $BOSH_INSTALL_TARGET/bin/
+go build -o $BOSH_INSTALL_TARGET/bin/smoke-test $GOPATH/src/harbor-api-testing/main.go
+
+

packages/smoke-test/spec

@@ -0,0 +1,8 @@
+---
+name: smoke-test
+
+dependencies:
+  - golang
+
+files:
+  - harbor-api-testing/** #git submodule add https://gitlab.eng.vmware.com/harbor/harbor-api-testing.git

templates/deployment_vsphere-dynamical.yml

@@ -2,7 +2,7 @@
 name: harbor-deployment-third-try
 
 releases:
-- name: harbor
+- name: harbor-container-registry
   version: latest
 
 stemcells:
@@ -16,7 +16,7 @@ instance_groups:
   instances: 1
   jobs:
   - name: harbor
-    release: harbor
+    release: harbor-container-registry
     properties:
       hostname: ((hostname))
       db_password: ((harbor_db_password))

templates/deployment_vsphere-errand.embedded.yml

@@ -0,0 +1,68 @@
+---
+name: harbor-deployment-4th-try
+
+releases:
+- name: harbor-container-registry
+  version: latest
+
+stemcells:
+- alias: default
+  os: ubuntu-trusty
+  version: latest
+
+instance_groups:
+- name: harbor
+  azs: [az1]
+  instances: 1
+  jobs:
+  - name: harbor
+    release: harbor-container-registry
+    properties:
+      hostname: ((hostname))
+      db_password: ((harbor_db_password))
+      admin_password: ((harbor_admin_password))
+      clair_db_password: ((clair_db_password))
+      ssl:
+        cert: ((harbor_ssl.certificate))
+        key: ((harbor_ssl.private_key))
+        ca: ((default_ca.ca))
+      with_clair: true
+      with_notary: true
+    provides:
+      harbor: {as: harbor_app_link}
+
+  - name: docker
+    release: harbor-container-registry
+
+  - name: smoke-test
+    release: harbor-container-registry
+    properties: {}
+    consumes:
+      harbor_reference: {from: harbor_app_link}
+  vm_type: standard
+  stemcell: default
+  persistent_disk_type: 20G
+  networks:
+  - name: default
+    static_ips:
+    - 10.112.122.1
+
+update:
+  canaries: 2
+  max_in_flight: 1
+  canary_watch_time: 30000-600000
+  update_watch_time: 30000-600000
+
+variables:
+- name: harbor_admin_password
+  type: password
+- name: clair_db_password
+  type: password
+- name: harbor_db_password
+  type: password
+- name: harbor_ssl
+  type: certificate
+  options:
+    ca: default_ca
+    common_name: ((hostname))
+    alternative_names: [((hostname))]