Allow use of version qualifier for staging artifacts (#42304)

This commits adds support for an optional $VERSION_QUALIFIER env var/build option in the packaging pipeline to allow building prerelease artifacts from staging.

Closes elastic/ingest-dev#4857

.buildkite/packaging.pipeline.yml

@@ -24,7 +24,9 @@ steps:
     key: start-gate-snapshot
 
   - name: Start of concurrency group for DRA Staging
-    if: build.branch =~ /^[0-9]+\.[0-9x]+\$/
+    # exceptionally allow building staging from main when VERSION_QUALIFIER is set, to allow prerelease testing
+    # TODO remove OR clause below and above comment, and only allow matching /^[0-9]+\.[0-9x]+\$/ for build.branch
+    if: build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.env('VERSION_QUALIFIER') != null
     command: echo "--> Start of concurrency gate dra-staging"
     concurrency_group: "dra-gate-staging-$BUILDKITE_BRANCH"
     concurrency: 1
@@ -58,7 +60,8 @@ steps:
           - build/distributions/**/*
 
       - label: Staging dashboards
-        if: build.branch =~ /^[0-9]+\.[0-9x]+\$/
+        # TODO remove OR clause below (see earlier comment)
+        if: build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.env('VERSION_QUALIFIER') != null
         depends_on: start-gate-staging
         key: dashboards-staging
         # TODO: container with go and make
@@ -170,8 +173,8 @@ steps:
   - group: Packaging Staging
     key: packaging-staging
     depends_on: start-gate-staging
-    ## Only for release
-    if: build.branch =~ /^[0-9]+\.[0-9x]+\$/
+    # TODO remove OR clause below (see earlier comment)
+    if: build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.env('VERSION_QUALIFIER') != null
     steps:
       - label: "STAGING: {{matrix}}"
         env:
@@ -276,8 +279,8 @@ steps:
           machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
 
       - label: DRA Staging
-        ## Only for release branches
-        if: build.branch =~ /^[0-9]+\.[0-9x]+\$/
+        # TODO remove OR clause below (see earlier comment)
+        if: build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.env('VERSION_QUALIFIER') != null
         key: dra-staging
         env:
           DRA_WORKFLOW: staging

.buildkite/scripts/dra.sh

@@ -15,7 +15,7 @@ set -euo pipefail
 BRANCH="${DRA_BRANCH:="${BUILDKITE_BRANCH:=""}"}"
 
 BEAT_VERSION=$(make get-version)
-
+VERSION_QUALIFIER="${VERSION_QUALIFIER:=""}"
 CI_DRA_ROLE_PATH="kv/ci-shared/release/dra-role"
 
 function release_manager_login {
@@ -49,7 +49,9 @@ docker run --rm \
         --commit "${BUILDKITE_COMMIT}" \
         --workflow "${DRA_WORKFLOW}" \
         --version "${BEAT_VERSION}" \
-        --artifact-set "main"
+        --artifact-set "main" \
+        --qualifier "${VERSION_QUALIFIER}"
+
 
 echo "+++ :hammer_and_pick: Publishing DRA artifacts for version [$BEAT_VERSION], branch [$BRANCH], workflow [$DRA_WORKFLOW] and DRY_RUN: [$DRY_RUN]"
 
@@ -68,6 +70,7 @@ docker run --rm \
         --workflow "${DRA_WORKFLOW}" \
         --version "${BEAT_VERSION}" \
         --artifact-set "main" \
+        --qualifier "${VERSION_QUALIFIER}" \
         ${DRY_RUN} | tee rm-output.txt
 
 

.buildkite/scripts/packaging/prepare-release-manager.sh

@@ -8,13 +8,21 @@
 set -ueo pipefail
 
 readonly TYPE=${1:-snapshot}
+readonly VERSION_QUALIFIER="${VERSION_QUALIFIER:-""}"
 
 # rename dependencies.csv to the name expected by release-manager.
 VERSION=$(make get-version)
 FINAL_VERSION=$VERSION-SNAPSHOT
 if [ "$TYPE" != "snapshot" ] ; then
   FINAL_VERSION=$VERSION
 fi
+
+set +e
+if [[ -n "$VERSION_QUALIFIER" ]]; then
+  FINAL_VERSION="$FINAL_VERSION-${VERSION_QUALIFIER}"
+fi
+set -e
+
 echo "Rename dependencies to $FINAL_VERSION"
 mv build/distributions/dependencies.csv \
    build/distributions/dependencies-"$FINAL_VERSION".csv