Load harbor docker images in harbor job prestart instead of job start

This can solve the canary watch timeout issue #3735, because the time
of prestart execution is not included in the canary_watch_time.
This patch moves docker related utils and bosh env vars to
src/common/utils.sh, and the key code for loading images is
in jobs/harbor/templates/bin/pre-start.erb.

Issue: goharbor/harbor#3735

jobs/docker/spec

@@ -7,6 +7,7 @@ templates:
   bin/properties.sh.erb: bin/properties.sh
 
 packages:
+- common
 - docker
 
 properties:

jobs/docker/templates/bin/ctl.erb

@@ -4,69 +4,49 @@ set -e # exit immediately if a simple command exits with a non-zero status
 set -u # report the usage of uninitialized variables
 set -x
 
+source /var/vcap/packages/common/utils.sh
+
 JOB_NAME=docker
-RUN_DIR="/var/vcap/sys/run/${JOB_NAME}"
-LOG_DIR="/var/vcap/sys/log/${JOB_NAME}"
-JOB_DIR="/var/vcap/jobs/${JOB_NAME}"
-PACKAGE_DIR=/var/vcap/packages
-DOCKER_PACKAGE_DIR=${PACKAGE_DIR}/docker
-DAEMON_PIDFILE=${RUN_DIR}/dockerd.pid
-DAEMON_SOCK=${RUN_DIR}/dockerd.sock
+DOCKER_RUN_DIR=$RUN_DIR/$JOB_NAME
+DOCKER_LOG_DIR=$LOG_DIR/$JOB_NAME
+DOCKER_JOB_DIR=$JOB_DIR/$JOB_NAME
+DOCKER_PACKAGE_DIR=$PACKAGE_DIR/docker
+DOCKER_DAEMON_PIDFILE=$DOCKER_RUN_DIR/dockerd.pid
+DOCKER_DAEMON_SOCK=$DOCKER_RUN_DIR/dockerd.sock
 
 export PATH=$PATH:${DOCKER_PACKAGE_DIR}/bin
 
 CTL_CMD=/sbin/start-stop-daemon
 DOCKERD_CMD=${DOCKER_PACKAGE_DIR}/bin/dockerd
-DOCKER_HOST="unix://$DAEMON_SOCK"
-
-exec 1>> $LOG_DIR/ctl.stdout.log
-exec 2>> $LOG_DIR/ctl.stderr.log
+DOCKER_HOST="unix://$DOCKER_DAEMON_SOCK"
 
-source $JOB_DIR/bin/properties.sh
+exec 1>> $DOCKER_LOG_DIR/ctl.stdout.log
+exec 2>> $DOCKER_LOG_DIR/ctl.stderr.log
 
-log() {
-  echo [`date`] $*
-}
+source $DOCKER_JOB_DIR/bin/properties.sh
 
 #Start docker daemon
 startDockerd() {
   OPT="--data-root ${DATA_ROOT_DIR} --host $DOCKER_HOST"
 
-  $CTL_CMD --pidfile $DAEMON_PIDFILE \
+  $CTL_CMD --pidfile $DOCKER_DAEMON_PIDFILE \
   --make-pidfile \
   --background \
   --exec $DOCKERD_CMD \
-  --start \
+  --start --oknodo \
   -- $OPT \
-  >> $LOG_DIR/dockerd.stdout.log \
-  2>> $LOG_DIR/dockerd.stderr.log
+  >> $DOCKER_LOG_DIR/dockerd.stdout.log \
+  2>> $DOCKER_LOG_DIR/dockerd.stderr.log
 }
 
 #Stop the dockerd process
 stopDockerd() {
-  if $CTL_CMD --pidfile $DAEMON_PIDFILE --retry TERM/30/QUIT/5/KILL --oknodo --stop; then
-    rm -f $DAEMON_PIDFILE
-    rm -f $DAEMON_SOCK
+  if $CTL_CMD --pidfile $DOCKER_DAEMON_PIDFILE --retry TERM/30/QUIT/5/KILL --oknodo --stop; then
+    rm -f $DOCKER_DAEMON_PIDFILE
+    rm -f $DOCKER_DAEMON_SOCK
   fi
 }
 
-waitForDockerd() {
-  sleep_time=2
-  timeout=20
-  count=0
-  while ! docker -H $DOCKER_HOST version 2>&1
-  do
-    log "Docker daemon is not running. Waiting for $sleep_time seconds then check again."
-    sleep $sleep_time
-    count=$((count + sleep_time));
-    if [ $count -ge $timeout ]; then
-      log "Error: Docker daemon is still not running after $timeout seconds."
-      exit 1
-    fi
-  done
-  log "Docker daemon is running"
-}
-
 case $1 in
 
   start)

jobs/harbor/spec

@@ -12,10 +12,11 @@ templates:
   config/ca.crt: config/ca.crt
 
 packages:
+- common
+- harbor-common
 - docker-compose
 - harbor-app
 - openssl
-- harbor-common
 
 properties:
   hostname:

jobs/harbor/templates/bin/ctl.erb

@@ -4,27 +4,23 @@ set -e # exit immediately if a simple command exits with a non-zero status
 set -u # report the usage of uninitialized variables
 set -o pipefail
 
+source /var/vcap/packages/common/utils.sh
+
 JOB_NAME=harbor
-RUN_DIR=/var/vcap/sys/run
 HARBOR_RUN_DIR=$RUN_DIR/$JOB_NAME
-LOG_DIR=/var/vcap/sys/log/$JOB_NAME
-HARBOR_JOB_DIR=/var/vcap/jobs/$JOB_NAME
+HARBOR_LOG_DIR=$LOG_DIR/$JOB_NAME
+HARBOR_JOB_DIR=$JOB_DIR/$JOB_NAME
 PIDFILE=${HARBOR_RUN_DIR}/harbor.pid
-PACKAGE_DIR=/var/vcap/packages
 HARBOR_PACKAGE_DIR=${PACKAGE_DIR}/harbor-app
 COMPOSE_PACKAGE_DIR=${PACKAGE_DIR}/docker-compose
-DOCKER_PACKAGE_DIR=${PACKAGE_DIR}/docker
 HARBOR_YAML=${HARBOR_PACKAGE_DIR}/docker-compose.yml
 HARBOR_CLAIR_YAML=${HARBOR_PACKAGE_DIR}/docker-compose.clair.yml
 HARBOR_NOTARY_YAML=${HARBOR_PACKAGE_DIR}/docker-compose.notary.yml
-IMAGES_TAR_PATH=${HARBOR_PACKAGE_DIR}/harbor*.tar
 
 CTL_CMD=/sbin/start-stop-daemon
 COMPOSE_CMD=${COMPOSE_PACKAGE_DIR}/bin/docker-compose
 DAEMON_SOCK=${RUN_DIR}/docker/dockerd.sock
 DAEMON_PID=${RUN_DIR}/docker/dockerd.pid
-DOCKER_HOST="unix://$DAEMON_SOCK"
-DOCKER_CMD="${DOCKER_PACKAGE_DIR}/bin/docker -H $DOCKER_HOST"
 CRON_PATH=/etc/cron.d/$JOB_NAME
 CRON_JOB_INTERVAL=2
 CHECK_SCRIPT_PATH=${HARBOR_JOB_DIR}/bin/status_check
@@ -33,8 +29,8 @@ HARBOR_DATA=/data
 HARBOR_DB_BACKUP_DIR=$HARBOR_DATA/db_backup
 COMMAND_NAME=$1
 
-exec 1>> $LOG_DIR/ctl.stdout.log
-exec 2>> $LOG_DIR/ctl.stderr.log
+exec 1>> $HARBOR_LOG_DIR/ctl.stdout.log
+exec 2>> $HARBOR_LOG_DIR/ctl.stderr.log
 
 source $PACKAGE_DIR/harbor-common/common.sh
 source $HARBOR_JOB_DIR/bin/properties.sh
@@ -48,8 +44,8 @@ startHarbor() {
   --exec $COMPOSE_CMD \
   --start --oknodo \
   -- $1 \
-  1>> $LOG_DIR/ctl.stdout.log \
-  2>> $LOG_DIR/ctl.stderr.log
+  1>> $HARBOR_LOG_DIR/ctl.stdout.log \
+  2>> $HARBOR_LOG_DIR/ctl.stderr.log
 }
 
 #Stop the harbor process
@@ -105,29 +101,6 @@ checkDockerdStatus() {
   pgrep -f dockerd >/dev/null 2>&1
 }
 
-#Make sure dockerd is started
-waitForDockerd() {
-  sleep_time=5
-  timeout=120
-  count=0
-  while ! checkDockerdStatus
-  do
-    log "Docker daemon is not running. Waiting for $sleep_time seconds then check again."
-    sleep $sleep_time
-    count=$((count + sleep_time));
-    if [ $count -ge $timeout ]; then
-      log "Error: Docker daemon is still not running after $timeout seconds."
-      exit 1
-    fi
-  done
-  log "Docker daemon is running"
-}
-
-#Load images
-loadImages() {
-  $DOCKER_CMD load -i ${IMAGES_TAR_PATH} 2>&1
-}
-
 #Upgrade Harbor if higher version of Harbor Bosh Release to be deployed
 upgradeHarbor() {
   log "Backing up Harbor database"
@@ -139,7 +112,7 @@ upgradeHarbor() {
 #Add cron job to check Harbor service availability.
 #If harbor service is not running well, remove the harbor pid file, then monit will restart it.
 cronJobUp() {
-  echo "*/$CRON_JOB_INTERVAL * * * * root ${CHECK_SCRIPT_PATH} > $LOG_DIR/cron.log 2>&1" > $CRON_PATH
+  echo "*/$CRON_JOB_INTERVAL * * * * root ${CHECK_SCRIPT_PATH} > $HARBOR_LOG_DIR/cron.log 2>&1" > $CRON_PATH
 }
 
 #Stop the cron job
@@ -161,12 +134,8 @@ case $COMMAND_NAME in
 
   start)
     log "Starting Harbor $HARBOR_FULL_VERSION at ${HARBOR_PROTOCOL}://${HARBOR_HOSTNAME}"
-
-    waitForDockerd
 
-    #TODO: Add image cleaning here if do migration
-    log "Loading docker images ..."
-    loadImages
+    waitForDockerd
 
     if $ENABLE_MIGRATION; then
       #The 1st Harbor Bosh Release for PKS does not need to upgrade from old version.

jobs/harbor/templates/bin/pre-start.erb

@@ -4,13 +4,16 @@ set -e # exit immediately if a simple command exits with a non-zero status
 
 [ -z "${DEBUG:-}" ] || set -x
 
+source /var/vcap/packages/common/utils.sh
+
 PACKAGE_DIR=/var/vcap/packages
 JOB_NAME=harbor
 RUN_DIR=/var/vcap/sys/run/$JOB_NAME
 LOG_DIR=/var/vcap/sys/log/$JOB_NAME
 HARBOR_JOB_DIR=/var/vcap/jobs/$JOB_NAME
 HARBOR_PACKAGE_DIR=${PACKAGE_DIR}/harbor-app
 HARBOR_PERSISTED_DATA=/var/vcap/store/$JOB_NAME
+HARBOR_IMAGES_TAR_PATH=${HARBOR_PACKAGE_DIR}/harbor*.tar
 HARBOR_DATA=/data
 CFG_FILE=${HARBOR_JOB_DIR}/config/harbor.cfg
 CRON_PATH=/etc/cron.d/$JOB_NAME
@@ -81,5 +84,22 @@ ${HARBOR_PACKAGE_DIR}/prepare ${PREPARE_OPTS}
 #Workaround to resolve the docker-compose libz issue
 sudo mount /tmp -o remount,exec
 
+#Load Harbor images
+loadImages() {
+  #Start dockerd
+  /var/vcap/jobs/docker/bin/ctl start
+  waitForDockerd
+  #TODO(szou): Add image cleaning here if do migration
+  #Load images
+  log "Loading docker images ..."
+  $DOCKER_CMD load -i $HARBOR_IMAGES_TAR_PATH 2>&1
+  /var/vcap/jobs/docker/bin/ctl stop
+  # wait for 3 seconds to avoid possible race condition
+  sleep 3
+}
+
+waitForDockerJobPrestart
+loadImages
+
 log "Successfully done!"
 exit 0

packages/common/packaging

@@ -0,0 +1,4 @@
+set -e
+
+cp -a common/* ${BOSH_INSTALL_TARGET}
+chmod +x ${BOSH_INSTALL_TARGET}/*.sh

packages/common/spec

@@ -0,0 +1,7 @@
+---
+name: common
+
+dependencies: []
+
+files:
+- common/utils.sh

packages/docker/spec

@@ -1,7 +1,7 @@
 ---
 name: docker
 
-dependencies: []
+dependencies: [common]
 
 files:
 - docker/docker-17.06.2-ce.tgz #https://download.docker.com/linux/static/stable/x86_64/docker-17.06.2-ce.tgz

packages/harbor-common/spec

@@ -1,7 +1,7 @@
 ---
 name: harbor-common
 
-dependencies: []
+dependencies: [common]
 
 files:
 - harbor-common/common.sh

src/common/utils.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+
+RUN_DIR=/var/vcap/sys/run
+LOG_DIR=/var/vcap/sys/log
+JOB_DIR=/var/vcap/jobs
+PACKAGE_DIR=/var/vcap/packages
+
+DOCKER_RUN_DIR=$RUN_DIR/docker
+DOCKER_PACKAGE_DIR=${PACKAGE_DIR}/docker
+DOCKER_DAEMON_SOCK=${DOCKER_RUN_DIR}/dockerd.sock
+DOCKER_HOST="unix://$DOCKER_DAEMON_SOCK"
+DOCKER_CMD="${DOCKER_PACKAGE_DIR}/bin/docker -H $DOCKER_HOST"
+
+log() {
+  echo [`date`] $*
+}
+
+waitForDockerd() {
+  sleep_time=3
+  timeout=60
+  count=0
+  while ! $DOCKER_CMD version 2>&1
+  do
+    log "Docker daemon is not running. Waiting for $sleep_time seconds then check again."
+    sleep $sleep_time
+    count=$((count + sleep_time));
+    if [ $count -ge $timeout ]; then
+      log "Error: Docker daemon is still not running after $timeout seconds."
+      exit 1
+    fi
+  done
+  log "Docker daemon is running"
+}
+
+waitForDockerJobPrestart() {
+  # All prestart scripts run in parallel.
+  # The docker job prestart (jobs/docker/templates/bin/pre-start.erb
+  # should be able to complete in 5 seconds.
+  sleep 5
+
+  sleep_time=2
+  timeout=60
+  count=0
+  while ! grep -q "^docker:" /etc/group
+  do
+    sleep $sleep_time
+    count=$((count + sleep_time));
+    if [ $count -ge $timeout ]; then
+      log "Error: docker job prestart doesn't exit in $timeout seconds."
+      exit 1
+    fi
+  done
+  log "docker job prestart completed"
+}

src/harbor-common/common.sh

@@ -5,7 +5,3 @@ HARBOR_FULL_VERSION=1.2.0
 HARBOR_MAJOR_VERSION=1
 HARBOR_MINOR_VERSION=2
 HARBOR_PATCH_VERSION=0
-
-log() {
-  echo [`date`] $*
-}