Delete unpublished RHEL images before publishing [DI-401][5.5.z] (#869)

Backport #868

Also merged `.github/scripts/logging.functions.sh` from master to avoid
changing code

Fixes: [DI-401](https://hazelcast.atlassian.net/browse/DI-401)

.github/scripts/logging.functions.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+# https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions
+
+# Prints the given message to stderr
+function echoerr() {
+  # https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions#setting-an-error-message
+  echo "::error::ERROR - $*" 1>&2;
+}
+
+# Create group
+function echo_group() {
+  # https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions#grouping-log-lines
+  local TITLE=$1
+  echo "::group::${TITLE}"
+}
+
+# Ends group after calling echo_group()
+function echo_group_end() {
+  # https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions#grouping-log-lines
+  echo "::endgroup::"
+}

.github/scripts/publish-rhel.sh

@@ -2,6 +2,9 @@
 
 set -o errexit -o nounset -o pipefail ${RUNNER_DEBUG:+-x}
 
+# shellcheck source=../.github/scripts/logging.functions.sh
+. .github/scripts/logging.functions.sh
+
 get_image()
 {
     local PUBLISHED=$1
@@ -206,7 +209,72 @@ wait_for_container_publish()
     done
 }
 
-# Prints the given message to stderr
-function echoerr() {
-  echo "::error::ERROR - $*" 1>&2;
+# Marks unpublished images as deleted for given version and then verifies if they were truly deleted
+function delete_unpublished_images() {
+    local RHEL_PROJECT_ID=$1
+    local VERSION=$2
+    local RHEL_API_KEY=$3
+
+    local IMAGE
+    local IS_PUBLISHED
+
+    UNPUBLISHED_IMAGES=$(get_image not_published "${RHEL_PROJECT_ID}" "${VERSION}" "${RHEL_API_KEY}")
+    UNPUBLISHED_COUNT=$(echo "${UNPUBLISHED_IMAGES}" | jq -r '.total')
+
+    echo "Found '${UNPUBLISHED_COUNT}' unpublished images for '${VERSION}'"
+
+    # mark images as deleted
+    for ((idx = 0 ; idx < $((UNPUBLISHED_COUNT)) ; idx++));
+    do
+        local IMAGE_ID=$(echo "${UNPUBLISHED_IMAGES}" | jq -r .data[${idx}]._id)
+        delete_image "${RHEL_API_KEY}" "${IMAGE_ID}"
+    done
+
+    # verify we have actually deleted the images
+    if [[ ${UNPUBLISHED_COUNT} -gt 0 ]]; then
+        verify_no_unpublished_images "${RHEL_PROJECT_ID}" "{$VERSION}" "${RHEL_API_KEY}"
+    fi
+}
+
+# this will actually send request to delete a single unpublished image
+function delete_image() {
+    local RHEL_API_KEY=$1
+    local IMAGE_ID=$2
+
+    echo "Marking image with ID=${IMAGE_ID} as deleted"
+
+    # https://catalog.redhat.com/api/containers/docs/endpoints/RESTPatchImage.html
+    RESPONSE=$( \
+        curl --silent \
+            --retry 5 --retry-all-errors \
+            --request PATCH \
+            --header "accept: application/json" \
+            --header "Content-Type: application/json" \
+            --header "X-API-KEY: ${RHEL_API_KEY}" \
+            --data '{"deleted": true}' \
+            "https://catalog.redhat.com/api/containers/v1/images/id/${IMAGE_ID}")
+
+    echo "::debug::HTTP response after image deletion"
+    echo "::debug::${RESPONSE}"
+}
+
+# verifies there are no unblished images for given version
+function verify_no_unpublished_images() {
+    local RHEL_PROJECT_ID=$1
+    local VERSION=$2
+    local RHEL_API_KEY=$3
+
+    UNPUBLISHED_IMAGES=$(get_image not_published "${RHEL_PROJECT_ID}" "${VERSION}" "${RHEL_API_KEY}")
+    UNPUBLISHED_COUNT=$(echo "${UNPUBLISHED_IMAGES}" | jq -r '.total')
+
+    if [[ ${UNPUBLISHED_COUNT} == "0" ]]; then
+        echo "No unpublished images found for '${VERSION}' after cleanup"
+        return 0
+    else
+        echoerr "Exiting as found '${UNPUBLISHED_COUNT}' unpublished images for '${VERSION}'"
+        echo_group "Unpublished images"
+        echoerr "${UNPUBLISHED_IMAGES}"
+        echo_group_end
+        return 1
+    fi
 }

.github/workflows/tag_image_push_rhel.yml

@@ -26,7 +26,7 @@ jobs:
     env:
       REQUIRED_HZ_MAJOR_VERSION: 5
       SCAN_REGISTRY: "quay.io"
-      TIMEOUT_IN_MINS: 60
+      TIMEOUT_IN_MINS: 240
       RHEL_API_KEY: ${{ secrets.RHEL_API_KEY }}
       HZ_VERSION: ${{ github.event.inputs.HZ_VERSION }}
       RELEASE_VERSION: ${{ github.event.inputs.RELEASE_VERSION }}
@@ -125,6 +125,14 @@ jobs:
         run: |
           docker login ${SCAN_REGISTRY} -u ${SCAN_REGISTRY_USER} -p ${SCAN_REGISTRY_PASSWORD}
 
+      - name: Delete unpublished images
+        if: inputs.DRY_RUN != 'true'
+        run: |
+          VERSION=${RELEASE_VERSION}-jdk${{ matrix.jdk }}
+          source .github/scripts/publish-rhel.sh
+
+          delete_unpublished_images "${RHEL_PROJECT_ID}" "${VERSION}" "${RHEL_API_KEY}"
+
       - name: Build the Hazelcast Enterprise image
         run: |
           . .github/scripts/get-tags-to-push.sh