diff --git a/jobs/docker/spec b/jobs/docker/spec index aef49c0..34c7bdb 100644 --- a/jobs/docker/spec +++ b/jobs/docker/spec @@ -7,6 +7,7 @@ templates: bin/properties.sh.erb: bin/properties.sh packages: +- common - docker properties: diff --git a/jobs/docker/templates/bin/ctl.erb b/jobs/docker/templates/bin/ctl.erb index 4709f77..b3543f8 100644 --- a/jobs/docker/templates/bin/ctl.erb +++ b/jobs/docker/templates/bin/ctl.erb @@ -4,69 +4,49 @@ set -e # exit immediately if a simple command exits with a non-zero status set -u # report the usage of uninitialized variables set -x +source /var/vcap/packages/common/utils.sh + JOB_NAME=docker -RUN_DIR="/var/vcap/sys/run/${JOB_NAME}" -LOG_DIR="/var/vcap/sys/log/${JOB_NAME}" -JOB_DIR="/var/vcap/jobs/${JOB_NAME}" -PACKAGE_DIR=/var/vcap/packages -DOCKER_PACKAGE_DIR=${PACKAGE_DIR}/docker -DAEMON_PIDFILE=${RUN_DIR}/dockerd.pid -DAEMON_SOCK=${RUN_DIR}/dockerd.sock +DOCKER_RUN_DIR=$RUN_DIR/$JOB_NAME +DOCKER_LOG_DIR=$LOG_DIR/$JOB_NAME +DOCKER_JOB_DIR=$JOB_DIR/$JOB_NAME +DOCKER_PACKAGE_DIR=$PACKAGE_DIR/docker +DOCKER_DAEMON_PIDFILE=$DOCKER_RUN_DIR/dockerd.pid +DOCKER_DAEMON_SOCK=$DOCKER_RUN_DIR/dockerd.sock export PATH=$PATH:${DOCKER_PACKAGE_DIR}/bin CTL_CMD=/sbin/start-stop-daemon DOCKERD_CMD=${DOCKER_PACKAGE_DIR}/bin/dockerd -DOCKER_HOST="unix://$DAEMON_SOCK" - -exec 1>> $LOG_DIR/ctl.stdout.log -exec 2>> $LOG_DIR/ctl.stderr.log +DOCKER_HOST="unix://$DOCKER_DAEMON_SOCK" -source $JOB_DIR/bin/properties.sh +exec 1>> $DOCKER_LOG_DIR/ctl.stdout.log +exec 2>> $DOCKER_LOG_DIR/ctl.stderr.log -log() { - echo [`date`] $* -} +source $DOCKER_JOB_DIR/bin/properties.sh #Start docker daemon startDockerd() { OPT="--data-root ${DATA_ROOT_DIR} --host $DOCKER_HOST" - $CTL_CMD --pidfile $DAEMON_PIDFILE \ + $CTL_CMD --pidfile $DOCKER_DAEMON_PIDFILE \ --make-pidfile \ --background \ --exec $DOCKERD_CMD \ - --start \ + --start --oknodo \ -- $OPT \ - >> $LOG_DIR/dockerd.stdout.log \ - 2>> $LOG_DIR/dockerd.stderr.log + >> $DOCKER_LOG_DIR/dockerd.stdout.log \ + 2>> $DOCKER_LOG_DIR/dockerd.stderr.log } #Stop the dockerd process stopDockerd() { - if $CTL_CMD --pidfile $DAEMON_PIDFILE --retry TERM/30/QUIT/5/KILL --oknodo --stop; then - rm -f $DAEMON_PIDFILE - rm -f $DAEMON_SOCK + if $CTL_CMD --pidfile $DOCKER_DAEMON_PIDFILE --retry TERM/30/QUIT/5/KILL --oknodo --stop; then + rm -f $DOCKER_DAEMON_PIDFILE + rm -f $DOCKER_DAEMON_SOCK fi } -waitForDockerd() { - sleep_time=2 - timeout=20 - count=0 - while ! docker -H $DOCKER_HOST version 2>&1 - do - log "Docker daemon is not running. Waiting for $sleep_time seconds then check again." - sleep $sleep_time - count=$((count + sleep_time)); - if [ $count -ge $timeout ]; then - log "Error: Docker daemon is still not running after $timeout seconds." - exit 1 - fi - done - log "Docker daemon is running" -} - case $1 in start) diff --git a/jobs/harbor/spec b/jobs/harbor/spec index 4b7707d..e90d2c7 100644 --- a/jobs/harbor/spec +++ b/jobs/harbor/spec @@ -12,10 +12,11 @@ templates: config/ca.crt: config/ca.crt packages: +- common +- harbor-common - docker-compose - harbor-app - openssl -- harbor-common properties: hostname: diff --git a/jobs/harbor/templates/bin/ctl.erb b/jobs/harbor/templates/bin/ctl.erb index 199a711..511e0bb 100644 --- a/jobs/harbor/templates/bin/ctl.erb +++ b/jobs/harbor/templates/bin/ctl.erb @@ -4,27 +4,23 @@ set -e # exit immediately if a simple command exits with a non-zero status set -u # report the usage of uninitialized variables set -o pipefail +source /var/vcap/packages/common/utils.sh + JOB_NAME=harbor -RUN_DIR=/var/vcap/sys/run HARBOR_RUN_DIR=$RUN_DIR/$JOB_NAME -LOG_DIR=/var/vcap/sys/log/$JOB_NAME -HARBOR_JOB_DIR=/var/vcap/jobs/$JOB_NAME +HARBOR_LOG_DIR=$LOG_DIR/$JOB_NAME +HARBOR_JOB_DIR=$JOB_DIR/$JOB_NAME PIDFILE=${HARBOR_RUN_DIR}/harbor.pid -PACKAGE_DIR=/var/vcap/packages HARBOR_PACKAGE_DIR=${PACKAGE_DIR}/harbor-app COMPOSE_PACKAGE_DIR=${PACKAGE_DIR}/docker-compose -DOCKER_PACKAGE_DIR=${PACKAGE_DIR}/docker HARBOR_YAML=${HARBOR_PACKAGE_DIR}/docker-compose.yml HARBOR_CLAIR_YAML=${HARBOR_PACKAGE_DIR}/docker-compose.clair.yml HARBOR_NOTARY_YAML=${HARBOR_PACKAGE_DIR}/docker-compose.notary.yml -IMAGES_TAR_PATH=${HARBOR_PACKAGE_DIR}/harbor*.tar CTL_CMD=/sbin/start-stop-daemon COMPOSE_CMD=${COMPOSE_PACKAGE_DIR}/bin/docker-compose DAEMON_SOCK=${RUN_DIR}/docker/dockerd.sock DAEMON_PID=${RUN_DIR}/docker/dockerd.pid -DOCKER_HOST="unix://$DAEMON_SOCK" -DOCKER_CMD="${DOCKER_PACKAGE_DIR}/bin/docker -H $DOCKER_HOST" CRON_PATH=/etc/cron.d/$JOB_NAME CRON_JOB_INTERVAL=2 CHECK_SCRIPT_PATH=${HARBOR_JOB_DIR}/bin/status_check @@ -33,8 +29,8 @@ HARBOR_DATA=/data HARBOR_DB_BACKUP_DIR=$HARBOR_DATA/db_backup COMMAND_NAME=$1 -exec 1>> $LOG_DIR/ctl.stdout.log -exec 2>> $LOG_DIR/ctl.stderr.log +exec 1>> $HARBOR_LOG_DIR/ctl.stdout.log +exec 2>> $HARBOR_LOG_DIR/ctl.stderr.log source $PACKAGE_DIR/harbor-common/common.sh source $HARBOR_JOB_DIR/bin/properties.sh @@ -48,8 +44,8 @@ startHarbor() { --exec $COMPOSE_CMD \ --start --oknodo \ -- $1 \ - 1>> $LOG_DIR/ctl.stdout.log \ - 2>> $LOG_DIR/ctl.stderr.log + 1>> $HARBOR_LOG_DIR/ctl.stdout.log \ + 2>> $HARBOR_LOG_DIR/ctl.stderr.log } #Stop the harbor process @@ -105,29 +101,6 @@ checkDockerdStatus() { pgrep -f dockerd >/dev/null 2>&1 } -#Make sure dockerd is started -waitForDockerd() { - sleep_time=5 - timeout=120 - count=0 - while ! checkDockerdStatus - do - log "Docker daemon is not running. Waiting for $sleep_time seconds then check again." - sleep $sleep_time - count=$((count + sleep_time)); - if [ $count -ge $timeout ]; then - log "Error: Docker daemon is still not running after $timeout seconds." - exit 1 - fi - done - log "Docker daemon is running" -} - -#Load images -loadImages() { - $DOCKER_CMD load -i ${IMAGES_TAR_PATH} 2>&1 -} - #Upgrade Harbor if higher version of Harbor Bosh Release to be deployed upgradeHarbor() { log "Backing up Harbor database" @@ -139,7 +112,7 @@ upgradeHarbor() { #Add cron job to check Harbor service availability. #If harbor service is not running well, remove the harbor pid file, then monit will restart it. cronJobUp() { - echo "*/$CRON_JOB_INTERVAL * * * * root ${CHECK_SCRIPT_PATH} > $LOG_DIR/cron.log 2>&1" > $CRON_PATH + echo "*/$CRON_JOB_INTERVAL * * * * root ${CHECK_SCRIPT_PATH} > $HARBOR_LOG_DIR/cron.log 2>&1" > $CRON_PATH } #Stop the cron job @@ -161,12 +134,8 @@ case $COMMAND_NAME in start) log "Starting Harbor $HARBOR_FULL_VERSION at ${HARBOR_PROTOCOL}://${HARBOR_HOSTNAME}" - - waitForDockerd - #TODO: Add image cleaning here if do migration - log "Loading docker images ..." - loadImages + waitForDockerd if $ENABLE_MIGRATION; then #The 1st Harbor Bosh Release for PKS does not need to upgrade from old version. diff --git a/jobs/harbor/templates/bin/pre-start.erb b/jobs/harbor/templates/bin/pre-start.erb index f22b64f..c9fbde8 100644 --- a/jobs/harbor/templates/bin/pre-start.erb +++ b/jobs/harbor/templates/bin/pre-start.erb @@ -4,6 +4,8 @@ set -e # exit immediately if a simple command exits with a non-zero status [ -z "${DEBUG:-}" ] || set -x +source /var/vcap/packages/common/utils.sh + PACKAGE_DIR=/var/vcap/packages JOB_NAME=harbor RUN_DIR=/var/vcap/sys/run/$JOB_NAME @@ -11,6 +13,7 @@ LOG_DIR=/var/vcap/sys/log/$JOB_NAME HARBOR_JOB_DIR=/var/vcap/jobs/$JOB_NAME HARBOR_PACKAGE_DIR=${PACKAGE_DIR}/harbor-app HARBOR_PERSISTED_DATA=/var/vcap/store/$JOB_NAME +HARBOR_IMAGES_TAR_PATH=${HARBOR_PACKAGE_DIR}/harbor*.tar HARBOR_DATA=/data CFG_FILE=${HARBOR_JOB_DIR}/config/harbor.cfg CRON_PATH=/etc/cron.d/$JOB_NAME @@ -81,5 +84,22 @@ ${HARBOR_PACKAGE_DIR}/prepare ${PREPARE_OPTS} #Workaround to resolve the docker-compose libz issue sudo mount /tmp -o remount,exec +#Load Harbor images +loadImages() { + #Start dockerd + /var/vcap/jobs/docker/bin/ctl start + waitForDockerd + #TODO(szou): Add image cleaning here if do migration + #Load images + log "Loading docker images ..." + $DOCKER_CMD load -i $HARBOR_IMAGES_TAR_PATH 2>&1 + /var/vcap/jobs/docker/bin/ctl stop + # wait for 3 seconds to avoid possible race condition + sleep 3 +} + +waitForDockerJobPrestart +loadImages + log "Successfully done!" exit 0 \ No newline at end of file diff --git a/packages/common/packaging b/packages/common/packaging new file mode 100644 index 0000000..22e16f7 --- /dev/null +++ b/packages/common/packaging @@ -0,0 +1,4 @@ +set -e + +cp -a common/* ${BOSH_INSTALL_TARGET} +chmod +x ${BOSH_INSTALL_TARGET}/*.sh \ No newline at end of file diff --git a/packages/common/spec b/packages/common/spec new file mode 100644 index 0000000..4b32312 --- /dev/null +++ b/packages/common/spec @@ -0,0 +1,7 @@ +--- +name: common + +dependencies: [] + +files: +- common/utils.sh diff --git a/packages/docker/spec b/packages/docker/spec index edd934d..bc69160 100644 --- a/packages/docker/spec +++ b/packages/docker/spec @@ -1,7 +1,7 @@ --- name: docker -dependencies: [] +dependencies: [common] files: - docker/docker-17.06.2-ce.tgz #https://download.docker.com/linux/static/stable/x86_64/docker-17.06.2-ce.tgz \ No newline at end of file diff --git a/packages/harbor-common/spec b/packages/harbor-common/spec index 77686f6..279d1c9 100644 --- a/packages/harbor-common/spec +++ b/packages/harbor-common/spec @@ -1,7 +1,7 @@ --- name: harbor-common -dependencies: [] +dependencies: [common] files: - harbor-common/common.sh diff --git a/src/common/utils.sh b/src/common/utils.sh new file mode 100644 index 0000000..f78cba2 --- /dev/null +++ b/src/common/utils.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +RUN_DIR=/var/vcap/sys/run +LOG_DIR=/var/vcap/sys/log +JOB_DIR=/var/vcap/jobs +PACKAGE_DIR=/var/vcap/packages + +DOCKER_RUN_DIR=$RUN_DIR/docker +DOCKER_PACKAGE_DIR=${PACKAGE_DIR}/docker +DOCKER_DAEMON_SOCK=${DOCKER_RUN_DIR}/dockerd.sock +DOCKER_HOST="unix://$DOCKER_DAEMON_SOCK" +DOCKER_CMD="${DOCKER_PACKAGE_DIR}/bin/docker -H $DOCKER_HOST" + +log() { + echo [`date`] $* +} + +waitForDockerd() { + sleep_time=3 + timeout=60 + count=0 + while ! $DOCKER_CMD version 2>&1 + do + log "Docker daemon is not running. Waiting for $sleep_time seconds then check again." + sleep $sleep_time + count=$((count + sleep_time)); + if [ $count -ge $timeout ]; then + log "Error: Docker daemon is still not running after $timeout seconds." + exit 1 + fi + done + log "Docker daemon is running" +} + +waitForDockerJobPrestart() { + # All prestart scripts run in parallel. + # The docker job prestart (jobs/docker/templates/bin/pre-start.erb + # should be able to complete in 5 seconds. + sleep 5 + + sleep_time=2 + timeout=60 + count=0 + while ! grep -q "^docker:" /etc/group + do + sleep $sleep_time + count=$((count + sleep_time)); + if [ $count -ge $timeout ]; then + log "Error: docker job prestart doesn't exit in $timeout seconds." + exit 1 + fi + done + log "docker job prestart completed" +} diff --git a/src/harbor-common/common.sh b/src/harbor-common/common.sh index c1bb3b3..aebaacd 100644 --- a/src/harbor-common/common.sh +++ b/src/harbor-common/common.sh @@ -5,7 +5,3 @@ HARBOR_FULL_VERSION=1.2.0 HARBOR_MAJOR_VERSION=1 HARBOR_MINOR_VERSION=2 HARBOR_PATCH_VERSION=0 - -log() { - echo [`date`] $* -}