eessi-upload-to-staging

#!/bin/bash
#
# This file is part of the EESSI infrastructure,
# see https://github.com/EESSI/infrastructure
#
# author: Bob Droege (@bedroge)
# author: Terje Kvernes (@terjekv)
# author: Thomas Roeblitz (@trz42)
#
# license: GPLv2
#

function upload_to_staging_bucket
{
  _file=$1
  _bucket=$2
  _path=$3
  _endpoint_url=$4

  _options=
  if [[ ! -z "${_endpoint_url}" ]]; then
    _options="--endpoint-url ${_endpoint_url}"
  fi
  aws ${_options} s3 cp "${_file}" s3://${_bucket}/${_path}
}

# This needs expanding etc.
function check_file_name
{
  filename=$1
  if ( echo ${filename} | grep ^eessi > /dev/null &&
    echo ${filename} | grep -E '(compat|init|software)' > /dev/null ); then
    return 0
  else
    return 1
  fi
}

function create_metadata_file
{
  _artefact=$1
  _url=$2
  _repository=$3
  _pull_request_number=$4
  _pull_request_comment_id=$5

  _tmpfile=$(mktemp)

  jq -n \
    --arg un $(whoami) \
    --arg ip $(curl -s https://checkip.amazonaws.com) \
    --arg hn "$(hostname -f)" \
    --arg fn "$(basename ${_artefact})" \
    --arg sz "$(du -b "${_artefact}" | awk '{print $1}')" \
    --arg ct "$(date -r "${_artefact}")" \
    --arg sha256 "$(sha256sum "${_artefact}" | awk '{print $1}')" \
    --arg url "${_url}" \
    --arg repo "${_repository}" \
    --arg pr "${_pull_request_number}" \
    --arg pr_comment_id "${_pull_request_comment_id}" \
    '{
       uploader: {username: $un, ip: $ip, hostname: $hn},
       payload: {filename: $fn, size: $sz, ctime: $ct, sha256sum: $sha256, url: $url},
       link2pr: {repo: $repo, pr: $pr, pr_comment_id: $pr_comment_id},
     }' > "${_tmpfile}"

  echo "${_tmpfile}"
}

function display_help
{
  echo "Usage: $0 [OPTIONS] <filenames>"                                           >&2
  echo "  -a | --artefact-prefix PREFIX  -  a directory to which the artefact"     >&2
  echo "                                    shall be uploaded; BASH variable"      >&2
  echo "                                    expansion will be applied; arg '-l'"   >&2
  echo "                                    lists variables that are defined at"   >&2
  echo "                                    the time of expansion"                 >&2
  echo "  -e | --endpoint-url URL        -  endpoint url (needed for non AWS S3)"  >&2
  echo "  -h | --help                    -  display this usage information"        >&2
  echo "  -i | --pr-comment-id           -  identifier of a PR comment; may be"    >&2
  echo "                                    used to efficiently determine the PR"  >&2
  echo "                                    comment to be updated during the"      >&2
  echo "                                    ingestion procedure"                   >&2
  echo "  -l | --list-variables          -  list variables that are available"     >&2
  echo "                                    for expansion"                         >&2
  echo "  -m | --metadata-prefix PREFIX  -  a directory to which the metadata"     >&2
  echo "                                    file shall be uploaded; BASH variable" >&2
  echo "                                    expansion will be applied; arg '-l'"   >&2
  echo "                                    lists variables that are defined at"   >&2
  echo "                                    the time of expansion"                 >&2
  echo "  -n | --bucket-name BUCKET      -  bucket name (same as BUCKET above)"    >&2
  echo "  -p | --pull-request-number INT -  a pull request number (INT); used to"  >&2
  echo "                                    link the upload to a PR"               >&2
  echo "  -r | --repository FULL_NAME    -  a repository name ACCOUNT/REPONAME;"   >&2
  echo "                                    used to link the upload to a PR"       >&2
}

if [[ $# -lt 1 ]]; then
    display_help
    exit 1
fi


# process command line args
POSITIONAL_ARGS=()

# depends on which service hosts the bucket
#   minio: https://MINIO_SERVER:MINIO_PORT/{bucket_name}/
#   s3aws: https://{bucket_name}.s3.amazonaws.com/
# should be contructable from endpoint_url and bucket_name
bucket_base=

# default bucket is eessi-staging
bucket_name="eessi-staging"

# provided via options in the bot's config file app.cfg
endpoint_url=

# provided via command line arguments
pr_comment_id="none"
pull_request_number="none"
github_repository="EESSI/software-layer"

# provided via options in the bot's config file app.cfg and/or command line argument
metadata_prefix=
artefact_prefix=

# other variables
legacy_aws_path=
variables="github_repository legacy_aws_path pull_request_number"

while [[ $# -gt 0 ]]; do
  case $1 in
    -a|--artefact-prefix)
      artefact_prefix="$2"
      shift 2
      ;;
    -e|--endpoint-url)
      endpoint_url="$2"
      shift 2
      ;;
    -h|--help)
      display_help
      exit 0
      ;;
    -l|--list-variables)
      echo "variables that will be expanded: name (default value)"
      for var in ${variables}
      do
        echo "    ${var} (${!var:-unset})"
      done
      exit 0
      ;;
    -i|--pr-comment-id)
      pr_comment_id="$2"
      shift 2
      ;;
    -m|--metadata-prefix)
      metadata_prefix="$2"
      shift 2
      ;;
    -n|--bucket-name)
      bucket_name="$2"
      shift 2
      ;;
    -p|--pull-request-number)
      pull_request_number="$2"
      shift 2
      ;;
    -r|--repository)
      github_repository="$2"
      shift 2
      ;;
    -*|--*)
      echo "Error: Unknown option: $1" >&2
      exit 1
      ;;
    *)  # No more options
      POSITIONAL_ARGS+=("$1") # save positional arg
      shift
      ;;
  esac
done

# restore potentially parsed filename(s) into $*
set -- "${POSITIONAL_ARGS[@]}"

# infer bucket_base:
#   if endpoint_url is not set (assume AWS S3 is used),
#     bucket_base=https://${bucket_name}.s3.amazonaws.com/
#   if endpoint_url is set (assume non AWS S3, eg minio, is used),
#     bucket_base=${endpoint_url}/${bucket_name}/
# check if endpoint_url is not set
if [[ -z "${endpoint_url}" ]]; then
  # assume AWS S3 being used
  bucket_base=https://${bucket_name}.s3.amazonaws.com
else
  # assume non AWS S3 being used or AWS S3 with bucket not in DNS
  bucket_base=${endpoint_url}/${bucket_name}
fi

for file in "$*"; do
  if [[ -r "${file}" && -f "${file}" &&  -s "${file}" ]]; then
    basefile=$( basename ${file} )
    if check_file_name ${basefile}; then
      if tar tf "${file}" | head -n1 > /dev/null; then
        # 'legacy_aws_path' might be used in artefact_prefix or metadata_prefix
        # its purpose is to support the old/legacy method to derive the location
        # where to store the artefact and metadata file
        export legacy_aws_path=$(basename ${file} | tr -s '-' '/' \
                 | perl -pe 's/^eessi.//;' | perl -pe 's/\.tar\.gz$//;' )
        if [ -z ${artefact_prefix} ]; then
          aws_path=${legacy_aws_path}
        else
          export pull_request_number
          export github_repository
          aws_path=$(envsubst <<< "${artefact_prefix}")
        fi
        aws_file=$(basename ${file})
        echo "Creating metadata file"
        url="${bucket_base}/${aws_path}/${aws_file}"
        echo "create_metadata_file file=${file} \
                                   url=${url} \
                                   github_repository=${github_repository} \
                                   pull_request_number=${pull_request_number} \
                                   pr_comment_id=${pr_comment_id}"
        metadata_file=$(create_metadata_file "${file}" \
                                             "${url}" \
                                             "${github_repository}" \
                                             "${pull_request_number}" \
                                             "${pr_comment_id}")
        echo "metadata:"
        cat ${metadata_file}

        echo Uploading to "${url}"
        echo "  store artefact at ${aws_path}/${aws_file}"
        upload_to_staging_bucket \
                "${file}" \
                "${bucket_name}" \
                "${aws_path}/${aws_file}" \
                "${endpoint_url}"

        if [ -z ${metadata_prefix} ]; then
          aws_path=${legacy_aws_path}
        else
          export pull_request_number
          export github_repository
          aws_path=$(envsubst <<< "${metadata_prefix}")
        fi
        echo "  store metadata file at ${aws_path}/${aws_file}.meta.txt"
        upload_to_staging_bucket \
                "${metadata_file}" \
                "${bucket_name}" \
                "${aws_path}/${aws_file}.meta.txt" \
                "${endpoint_url}"
      else
        echo "'${file}' is not a tar file."
        exit 1
      fi
    else
      echo "${file} does not look like an eessi layer filename!"
      exit 1
    fi
  else
      echo "'${file}' is not a readable non zero-sized file."
      exit 1
  fi
done