Prepend ingestion base directory with "versions" directory (+ some fixes for Ansible CI workflows)

.github/workflows/Dockerfile-ubuntu-18.04

@@ -3,9 +3,9 @@ FROM ubuntu:18.04
 USER root
 
 RUN apt-get update
-RUN apt-get install -y cron gpg python3-pip systemd
+RUN apt-get install -y cron gpg python3.8 python3-pip systemd
 
-RUN pip3 install ansible
+RUN python3.8 -m pip install ansible
 
 COPY ./.github/workflows/test-playbook.sh /test-playbook.sh
 

.github/workflows/build-test-release-client-packages.yml

@@ -34,7 +34,7 @@ jobs:
         run: ln -s inventory/group_vars
 
       - name: Prepare package source
-        uses: roles-ansible/check-ansible-debian-stretch-action@master
+        uses: roles-ansible/check-ansible-debian-stable-action@master
         with:
           targets: "./prepare-client-packages.yml"
           hosts: "localhost"

scripts/ingest-tarball.sh

@@ -5,7 +5,15 @@
 # nested catalogs in a separate transaction.
 # This script has to be run on a CVMFS publisher node.
 
+# This script assumes that the given tarball is named like:
+# eessi-<version>-{compat,init,software}-[additional information]-<timestamp>.tar.gz
+# It also assumes, and verifies, that the  name of the top-level directory of the contents of the
+# of the tarball matches <version>, and that name of the second level should is either compat, init, or software.
+
+# Only if it passes these checks, the tarball gets ingested to the base dir in the repository specified below.
+
 repo=pilot.eessi-hpc.org
+basedir=versions
 decompress="gunzip -c"
 
 function echo_green() {
@@ -33,33 +41,40 @@ if [ ! -f "${tar_file}" ]; then
 fi
 
 tar_file_basename=$(basename "${tar_file}")
-version=$(echo ${tar_file} | cut -d- -f2)
-top_level_dir=$(echo ${tar_file} | cut -d- -f3)
+version=$(echo ${tar_file_basename} | cut -d- -f2)
+contents_type_dir=$(echo ${tar_file_basename} | cut -d- -f3)
 tar_top_level_dir=$(tar tf "${tar_file}" | head -n 1 | cut -d/ -f1)
+# Use the 2nd file/dir in the tarball, as the first one may be just "<version>/"
+tar_contents_type_dir=$(tar tf "${tar_file}" | head -n 2 | tail -n 1 | cut -d/ -f2)
 
-# Check if the top level dir encoded in the filename
-# matches the top lever dir inside the tarball
-if [ "${top_level_dir}" != "${tar_top_level_dir}" ]
+# Check of the EESSI version number encoded in the filename
+# is a valid, i.e. matches the format YYYY.DD
+if ! echo "${version}" | egrep -q '^20[0-9][0-9]\.(0[0-9]|1[0-2])$'
 then
-    error "the top level directory in the filename (${top_level_dir}) does not match the top level directory in the tar ball (${tar_top_level_dir})."
+    error "${version} is not a valid EESSI version."
 fi
 
-# We assume that the top level dir must be compat, software, or init
-if [ "${tar_top_level_dir}" != "compat" ] && [ "${tar_top_level_dir}" != "software" ] && [ "${tar_top_level_dir}" != "init" ]
+# Check if the version encoded in the filename matches the top-level dir inside the tarball
+if [ "${version}" != "${tar_top_level_dir}" ]
 then
-    error "the top level directory in the tar ball should be either compat, software, or init!"
+    error "the version in the filename (${version}) does not match the top-level directory in the tarball (${tar_top_level_dir})."
 fi
 
-# Check of the EESSI version number encoded in the filename
-# is a valid, i.e. matches the format YYYY.DD
-if ! echo "${version}" | egrep -q '^20[0-9][0-9]\.(0[0-9]|1[0-2])$'
+# Check if the second-level dir in the tarball is compat, software, or init
+if [ "${tar_contents_type_dir}" != "compat" ] && [ "${tar_contents_type_dir}" != "software" ] && [ "${tar_contents_type_dir}" != "init" ]
 then
-    error "${version} is not a valid EESSI version."
+    error "the second directory level of the tarball contents should be either compat, software, or init."
+fi
+
+# Check if the name of the second-level dir in the tarball matches to what is specified in the filename
+if [ "${contents_type_dir}" != "${tar_contents_type_dir}" ]
+then
+    error "the contents type in the filename (${contents_type_dir}) does not match the contents type in the tarball (${tar_contents_type_dir})."
 fi
 
-# Ingest the tarball to the repository
+# Ingest the tarball to the repository, use "versions" as base dir for the ingestion
 echo "Ingesting tarball ${tar_file} to ${repo}..."
-${decompress} "${tar_file}" | cvmfs_server ingest -t - -b "${version}" "${repo}"
+${decompress} "${tar_file}" | cvmfs_server ingest -t - -b "${basedir}" "${repo}"
 ec=$?
 if [ $ec -eq 0 ]
 then