deploy_data_specific_image

#!/bin/bash
# Deploy data from git repo(s) to local folder(s).
# This script is run directly on the specific images deriving from the pavics-jupyter-base image.
# It will be used to download and update the different tutorial notebooks associated with a specific image.
#
# This is meant to be run on the same host running PAVICS.
#
# The data details is specified in a yaml config file, using this format :
# - repo_url: path to the github repo (ex.: https://github.com/crim-ca/pavics-jupyter-images)
#   branch: name of the branch containing the required version of the data
#   source_dir: path of the desired notebook folder in the source repo
#   dest_sub_dir: Data is sent to a tutorial-notebooks directory with the same name as the image running the script.
#            Use this parameter to specify a subdirectory path (without a prefix '/') in this directory where the data will be sent.
#            If the parameter is absent or left empty, data is sent directly to the image's tutorial-notebooks folder
#
# One or more entries respecting this format can be added to the yaml file, in order to download multiple file/folders.
#
# Setting environment variable DEPLOY_DATA_LOGFILE='/path/to/logfile.log'
# will redirect all STDOUT and STDERR to that logfile so this script will be
# completely silent.

if [ ! -z "$DEPLOY_DATA_LOGFILE" ]; then
    exec >>$DEPLOY_DATA_LOGFILE 2>&1
fi

if [ -z "$DEPLOY_DATA_JOB_NOTEBOOK_DEST_DIR" ]; then
    echo "ERROR: DEPLOY_DATA_JOB_NOTEBOOK_DEST_DIR variable has not been set" 1>&2
    exit 2
fi

START_TIME="`date -Isecond`"
echo "==========
deploy-data-specific-image START_TIME=$START_TIME"

CONFIG_YML_PATH="$1"
if [ -z "$CONFIG_YML_PATH" ]; then
    echo "ERROR: missing config.yml file" 1>&2
    exit 2
fi

# Checks if the config has a required parameter value, takes the string name of the env variable to evaluate as input.
# Empty value could mean typo in the keys in the config file.
ensure_not_null() {
    if [ "${!1}" = null ]; then
        echo "ERROR: empty $1 value found in the config file" 1>&2
        exit 1
    fi
}

# Check installation of required package to read yaml files
if ! command -v yq &> /dev/null; then
    echo "ERROR: missing yq package installation" 1>&2
    exit 1
fi

# Find how many entries are in the config file
CONFIG_LIST_LENGTH="`yq -r '. | length' $CONFIG_YML_PATH`"

if [ -z $CONFIG_LIST_LENGTH ]; then
    echo "ERROR: empty config file" 1>&2
    exit 1
fi

for ((i=0;i<$CONFIG_LIST_LENGTH;i++)); do
    # Extract data from config
    GIT_REPO="`yq -r .[$i].repo_url $CONFIG_YML_PATH`"
    ensure_not_null "GIT_REPO"

    BRANCH="`yq -r .[$i].branch $CONFIG_YML_PATH`"
    ensure_not_null "BRANCH"

    SOURCE_DIR="`yq -r .[$i].source_dir $CONFIG_YML_PATH`"
    ensure_not_null "SOURCE_DIR"

    DEST_SUB_DIR="`yq -r .[$i].dest_sub_dir $CONFIG_YML_PATH`"
    if [ "${DEST_SUB_DIR}" = null ]; then
        DEST_SUB_DIR=""
    fi

    FULL_URL=$GIT_REPO/branches/$BRANCH/$SOURCE_DIR
    FULL_DEST_DIR=$DEPLOY_DATA_JOB_NOTEBOOK_DEST_DIR/$DEST_SUB_DIR
    FULL_TMP_DIR="/tmp/notebooks/item_$i"

    echo "Extracting ${FULL_URL} to ${FULL_DEST_DIR}"

    # Download the data from github into tmp directory
    svn export --force $FULL_URL $FULL_TMP_DIR

    # Synchronize FULL_TMP_DIR with FULL_DEST_DIR
    # --delete : delete extraneous files from destination dirs
    rsync --recursive --delete --times --checksum --protect-args --verbose $FULL_TMP_DIR/ $FULL_DEST_DIR/
done

# Remove /tmp/notebooks directory
rm --recursive --force $(dirname $FULL_TMP_DIR)

echo "notebookdeploy finished   END_TIME=`date -Isecond`"

# vi: tabstop=8 expandtab shiftwidth=4 softtabstop=4