Add wavefront to monitor Harbor VM

jobs/harbor/monit

@@ -3,4 +3,5 @@ check process harbor with pidfile /var/vcap/sys/run/harbor/harbor.pid
   start program "/var/vcap/jobs/harbor/bin/ctl start" with timeout 300 seconds
   stop program "/var/vcap/jobs/harbor/bin/ctl stop" with timeout 180 seconds
   depends on dockerd
-  depends on harbor-enable-bosh-dns
+  depends on harbor-enable-bosh-dns
+

jobs/harbor/templates/bin/pre-start.erb.sh

@@ -86,10 +86,11 @@ function prepareCert() {
     cp ${HARBOR_JOB_DIR}/config/server.crt $HARBOR_DATA/cert/
     cp ${HARBOR_JOB_DIR}/config/server.key $HARBOR_DATA/cert/
     cp ${HARBOR_JOB_DIR}/config/uaa_ca.crt $HARBOR_DATA/cert/
-    cp ${HARBOR_JOB_DIR}/config/ca.crt $HARBOR_DATA/ca_download
-    chmod 644  $HARBOR_DATA/ca_download/ca.crt
     cp ${HARBOR_JOB_DIR}/config/trusted_certificates.crt $HARBOR_DATA/cert/
+    chmod 644 $HARBOR_DATA/cert/*
 
+    cp ${HARBOR_JOB_DIR}/config/ca.crt $HARBOR_DATA/ca_download
+    chmod 644  $HARBOR_DATA/ca_download/ca.crt
     #For status checking script usage
     CERT_PATH=$CERTS_D/${HARBOR_HOSTNAME}
     mkdir -p $CERT_PATH

jobs/wavefront/monit

@@ -0,0 +1,8 @@
+<% if_p('vm_monitor') do |monitor| %>
+  <% if monitor == 'wavefront_monitor' %>
+check process wavefront with pidfile /var/vcap/sys/run/wavefront/wavefront.pid
+  group vcap
+  start program "/var/vcap/jobs/wavefront/bin/ctl start" with timeout 180 seconds
+  stop program "/var/vcap/jobs/wavefront/bin/ctl stop" with timeout 180 seconds
+  depends on dockerd
+<%end%><%end%>

jobs/wavefront/spec

@@ -0,0 +1,28 @@
+---
+name: wavefront
+
+templates:
+  bin/ctl.sh.erb: bin/ctl
+  bin/pre-start.sh.erb: bin/pre-start
+  bin/status_check.sh.erb: bin/status_check
+  config/docker-compose.yml.erb: config/docker-compose.yml
+  config/telegraf.config.erb: config/telegraf.config
+
+packages:
+- common
+- docker-compose
+- wavefront
+
+properties:
+  wavefront_url:
+    description: "the url that telegraf send data to"
+    default: "https://longboard.wavefront.com/api"
+  wavefront_token:
+    description: "The token used to access url"
+    default: "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
+  wavefront_hostname:
+    description: "hostname used by telegraf, if empty use os.Hostname()"
+    default: ""
+  vm_monitor:
+    description: "Set this to wavefront_monitor to enable wavefront"
+    default: ""

jobs/wavefront/templates/bin/ctl.sh.erb

@@ -0,0 +1,126 @@
+#!/bin/bash
+
+set -e # exit immediately if a simple command exits with a non-zero status
+set -u # report the usage of uninitialized variables
+set -o pipefail
+
+source /var/vcap/packages/common/utils.sh
+
+JOB_NAME=wavefront
+WAVEFRONT_RUN_DIR=$RUN_DIR/$JOB_NAME
+WAVEFRONT_LOG_DIR=$LOG_DIR/$JOB_NAME
+WAVEFRONT_JOB_DIR=$JOB_DIR/$JOB_NAME
+PIDFILE=${WAVEFRONT_RUN_DIR}/wavefront.pid
+TEMP_PIDFILE=${WAVEFRONT_RUN_DIR}/wavefront.tmp.pid
+WAVEFRONT_PACKAGE_DIR=${PACKAGE_DIR}/wavefront
+COMPOSE_PACKAGE_DIR=${PACKAGE_DIR}/docker-compose
+WAVEFRONT_YAML=${WAVEFRONT_JOB_DIR}/config/docker-compose.yml
+
+CTL_CMD=/sbin/start-stop-daemon
+COMPOSE_CMD=${COMPOSE_PACKAGE_DIR}/bin/docker-compose
+DAEMON_SOCK=${RUN_DIR}/docker/dockerd.sock
+DAEMON_PID=${RUN_DIR}/docker/dockerd.pid
+CRON_PATH=/etc/cron.d/$JOB_NAME
+CRON_JOB_INTERVAL=2
+CHECK_SCRIPT_PATH=${WAVEFRONT_JOB_DIR}/bin/status_check
+
+COMMAND_NAME=$1
+
+#Make sure folders are ready
+for dir in $WAVEFRONT_RUN_DIR $WAVEFRONT_LOG_DIR ; do
+  mkdir -p ${dir}
+  chown vcap:vcap ${dir}
+  chmod 755 ${dir}
+done
+
+#Start the wavefront process
+#Require options parameter for $COMPOSE_CMD
+startWavefront() {
+  $CTL_CMD --pidfile $TEMP_PIDFILE \
+  --make-pidfile \
+  --background \
+  --start --oknodo \
+  --startas /bin/bash \
+  -- -c "$COMPOSE_CMD $1 1>> $WAVEFRONT_LOG_DIR/ctl.stdout.log 2>&1"
+  #Wait for a while to let docker-compose initialize
+  sleep 5
+}
+
+#Stop the Wavefront process
+stopWavefront() {
+  $COMPOSE_CMD $1 2>&1 &
+  pid=$!
+  # monit will use pid in $PIDFILE to check if Wavefront job is stopped,
+  # then stop dockerd job if running 'monit stop all'.
+  echo $pid > $PIDFILE
+  wait $pid
+  rm -f $PIDFILE $TEMP_PIDFILE
+}
+
+waitForWavefront() {
+  sleep_time=5
+  timeout=180
+  count=0
+  log "Waiting for Wavefront Service to be ready ..."
+  while ! $CHECK_SCRIPT_PATH >> $WAVEFRONT_LOG_DIR/cron.log 2>&1
+  do
+    log "Wavefront service is not ready. Waiting for $sleep_time seconds then check again."
+    sleep $sleep_time
+    count=$((count + sleep_time));
+    if [ $count -ge $timeout ]; then
+      log "Error: Wavefront Service failed to start in $timeout seconds."
+      exit 1
+    fi
+  done
+  log "Wavefront Service is ready"
+}
+
+#Check docker daemon status
+checkDockerdStatus() {
+  [[ -e $DAEMON_SOCK ]] && \
+  [[ -e $DAEMON_PID ]] && \
+  pgrep -f dockerd >/dev/null 2>&1
+}
+
+#Build compose options
+COMPOSE_OPTS="-H $DOCKER_HOST -f ${WAVEFRONT_YAML}"
+
+case $COMMAND_NAME in
+
+  start)
+    log "Starting wavefront ..."
+    waitForDockerd
+
+    #Dead Wavefront containers will prevent Wavefront service to start.
+    log "Remove dead docker containers if exist"
+    $DOCKER_CMD ps -aq -f 'status=dead'
+
+    log "Launching 'docker-compose up' ..."
+    COMPOSE_OPTS="${COMPOSE_OPTS} up"
+    startWavefront "$COMPOSE_OPTS"
+
+    #Wait for Wavefront Service
+    waitForWavefront
+    #Now let monit detect the real wavefront pid. Keep the TEMP_PIDFILE to be detected by startWavefront().
+    cp $TEMP_PIDFILE $PIDFILE
+
+    ;;
+
+  stop)
+    log "Stopping Wavefront ..."
+
+    #TODO: Add instance clean work here if migration is enabled
+
+    log "Launching 'docker-compose down' ..."
+    COMPOSE_OPTS="${COMPOSE_OPTS} down"
+    stopWavefront "$COMPOSE_OPTS"
+    ;;
+
+  *)
+    log -n "Usage: ctl {start|stop}"
+    ;;
+
+esac
+
+log "ctl $COMMAND_NAME is successfully done!"
+exit 0

jobs/wavefront/templates/bin/pre-start.sh.erb

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+set -e # exit immediately if a simple command exits with a non-zero status
+set -u # report the usage of uninitialized variables
+set -o pipefail
+
+source /var/vcap/packages/common/utils.sh
+
+WAVEFRONT_PACKAGE_DIR=${PACKAGE_DIR}/wavefront
+WAVEFRONT_IMAGES_TAR_PATH=$WAVEFRONT_PACKAGE_DIR/harbor-wavefront-bundle-1.8.0.tar
+#Load Wavefront images
+loadImages() {
+  waitForDockerd
+
+  # clean previous images
+  log "clean up wavefront images before load"
+  $DOCKER_CMD image rm wavefronthq/proxy -f
+  $DOCKER_CMD image rm telegraf -f
+  $DOCKER_CMD image prune -f
+
+  #Load images
+  log "Loading wavefront images ..."
+  $DOCKER_CMD load -i $WAVEFRONT_IMAGES_TAR_PATH 2>&1
+}
+
+log "load wavefront images"
+
+loadImages
+log "Successfully done!"
+exit 0

jobs/wavefront/templates/bin/status_check.sh.erb

@@ -0,0 +1,58 @@
+#!/bin/bash
+
+set -e # exit immediately if a simple command exits with a non-zero status
+set -u # report the usage of uninitialized variables
+
+source /var/vcap/packages/common/utils.sh
+
+JOB_NAME=wavefront
+WAVEFRONT_JOB_DIR=/var/vcap/jobs/$JOB_NAME
+DAEMON_SOCK=${RUN_DIR}/docker/dockerd.sock
+
+#Exit function with pid file deletion
+myExit() {
+  log "Wavefront status check: [FAILED]"
+  parseOpts
+  if [ "$RESTART_WAVEFRONT" = true ]; then
+    log "Now restart wavefront job"
+    rm -f /etc/cron.d/wavefront
+    /var/vcap/bosh/bin/monit restart wavefront
+  fi
+  exit $1
+}
+
+parseOpts() {
+  RESTART_WAVEFRONT=false
+  while getopts "r" opt; do
+    case $opt in
+      r)
+        RESTART_WAVEFRONT=true
+        ;;
+    esac
+  done
+}
+
+parseOpts $@
+
+#Check if containers existing
+wavefront_container=$($DOCKER_CMD ps | awk '{print $2}' | grep wavefront)
+telegraf_container=$($DOCKER_CMD ps | awk '{print $2}' | grep telegraf)
+
+if [ -z "$wavefront_container" ]; then
+  myExit 1
+fi
+
+if [ -z "$telegraf_container" ]; then
+  myExit 1
+fi
+
+#Check the status of wavefront containers
+if $DOCKER_CMD ps --filter "status=restarting" | grep 'wavefront'; then
+  myExit 2
+fi
+if $DOCKER_CMD ps --filter "status=restarting" | grep 'telegraf'; then
+  myExit 2
+fi
+
+log "Wavefront status check: [PASSED]"
+exit 0

jobs/wavefront/templates/config/docker-compose.yml.erb

@@ -0,0 +1,20 @@
+version: '3.7'
+services:
+    telegraf:
+       image: telegraf:latest
+       command: ["telegraf", "--config", "/telegraf.config"]
+       volumes:
+           - "/:/hostfs:ro"
+           - "./telegraf.config:/telegraf.config"
+       environment:
+           - HOST_MOUNT_PREFIX=/hostfs
+           - HOST_PROC=/hostfs/proc
+    wavefront-proxy:
+        image: wavefronthq/proxy:latest
+        environment:
+            - WAVEFRONT_URL=<%= p("wavefront_url") %> 
+            - WAVEFRONT_TOKEN=<%= p("wavefront_token") %> 
+            - JAVA_HEAP_USAGE=512m
+        ports:
+            - "2878:2878"
+            - "4242:4242"