[8.15] Add Independent Agent alternate upload step in pipeline (#287) (…

…#288)

Backport of #287

This PR adds an alternate step in the elastic-stack-installers that the Independent Agent Release will use in order to get only the Elastic Agent windows installer (.msi) built.

The new pipeline step uses a new script that does the following:

Retrieve the MSI artifacts from the Build step
Moves the MSI artifacts to a new directory
Creates a .sha512 file for the MSI file
Saves the MSI and .sha512 artifacts using Buildkite's built-in upload functionality
Sets metadata in the calling pipeline if the TRIGGER_JOB_ID is set
The calling pipeline will then use this metadata to download the saved artifacts

.buildkite/pipeline.yml

@@ -59,3 +59,27 @@ steps:
           provider: gcp
         env:
           DRA_WORKFLOW: "staging"
+  - group: ":package: Stack installers Independent Agent Staging"
+    key: "independent-agent-staging"
+    if: |
+      // This should only run when triggered from Independent Agent Staging
+        (build.env("BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG") == "independent-agent-release-staging")
+    steps:
+      - label: ":construction_worker: Build stack installers / Independent Agent Staging"
+        command: ".buildkite/scripts/build.ps1"
+        key: "build-staging-independent-agent"
+        artifact_paths: "c:/users/buildkite/esi/bin/out/**/*.msi"
+        agents:
+          provider: gcp
+          image: family/core-windows-2022
+        env:
+          DRA_WORKFLOW: "staging"
+      - label: ":package: Publishing via BK API for Independent Agent Release"
+        command: ".buildkite/scripts/ind-agent-publish.sh"
+        key: "publish-staging-independent-agent"
+        artifact_paths: "stack-installers-output/**/*"
+        depends_on: "build-staging-independent-agent"
+        agents:
+          provider: gcp
+        env:
+          DRA_WORKFLOW: "staging"

.buildkite/scripts/build.ps1

@@ -55,7 +55,7 @@ if (-not (Test-Path env:MANIFEST_URL) -and (Test-Path env:BUILDKITE_PULL_REQUEST
 
     setManifestUrl -targetBranch $targetBranch
 }
-elseif (-not (Test-Path env:MANIFEST_URL) -and ($env:BUILDKITE_SOURCE -ne "trigger_job") -and ($env:BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG -notlike "unified-release*")) {
+elseif (-not (Test-Path env:MANIFEST_URL) -and ($env:BUILDKITE_SOURCE -ne "trigger_job") -and (($env:BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG -notlike "unified-release*") -or ($env:BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG -notlike "independent-agent-release*"))) {
     # we got triggered by a branch push
     Write-Host "~~~ Running in branch push mode"
 

.buildkite/scripts/ind-agent-publish.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# Download artifacts from Buildkite "Build stack installers" step
+echo "+++ Downloading artifacts..."
+# it's possible to also use unix style paths for download (users/buildkites/...)
+# but it's an experimental feature that needs an agent flag set: https://buildkite.com/docs/agent/v3#experimental-features-normalised-upload-paths
+buildkite-agent artifact download 'users\buildkite\esi\bin\out\**\*.msi' . --step "build-${DRA_WORKFLOW}-independent-agent"
+
+echo "--- Creating output dir and moving files there"
+OUTPUT_DIR="stack-installers-output"
+mkdir "$OUTPUT_DIR"
+mv users/buildkite/esi/bin "$OUTPUT_DIR"
+chmod -R 777 "$OUTPUT_DIR"
+
+echo "--- ls -alR"
+ls -alR "${OUTPUT_DIR}"
+
+# Create sha512 for msi file(s)
+# We can leave it as this loop in order to account for a possible future ARM msi
+echo "--- Creating sha512 files"
+pushd "${OUTPUT_DIR}/bin/out/elastic"
+for f in *.msi; do 
+    sha512sum "${f}" > "${f}.sha512"
+done
+
+# The defined "artifacts_path" will pick up "stack-installers-output/**/*"
+
+# Check and set trigger id
+if [[ -z "${TRIGGER_JOB_ID-}" ]]; then
+    echo "--- TRIGGER_JOB_ID is not set, so not setting metadata"
+else
+    # If a pipeline that triggered this build passes in a "TRIGGER_JOB_ID" env var, that
+    # is an indicator that it will want us to set some metadata in that calling build
+    # so that it can reference this specific build ID in order to easily download
+    # artifacts saved off in this build.
+    #
+    # This is a much easier way to pull back artifacts from a triggered build than using
+    # a Buildkite token that we then have to manage via vault, etc.
+    # See: https://forum.buildkite.community/t/how-to-download-artifacts-back-from-triggered-pipeline/3480/2
+    echo "--- Setting metadata for job that trigger this one"
+    buildkite-agent meta-data set "triggered_build_id" "$BUILDKITE_BUILD_ID" --job $TRIGGER_JOB_ID
+    buildkite-agent meta-data set "triggered_commit_hash" "$BUILDKITE_COMMIT" --job $TRIGGER_JOB_ID
+fi

src/installer/BeatPackageCompiler/BeatPackageCompiler.cs

@@ -40,6 +40,7 @@ static void Main(string[] args)
             // Generate UUID v5 from product properties.
             // This UUID *must* be stable and unique between Beats.
             var upgradeCode = Uuid5.FromString(ap.CanonicalTargetName);
+            string version = SupportBuildVersionForAgent(ap.Version);
 
             var project = new Project(displayName)
             {
@@ -52,7 +53,7 @@ static void Main(string[] args)
                 Description = pc.Description,
 
                 OutFileName = Path.Combine(opts.PackageOutDir, opts.PackageName),
-                Version = new Version(ap.Version),
+                Version = new Version(version),
 
                 // We massage LICENSE.txt into .rtf below
                 LicenceFile = Path.Combine(
@@ -291,6 +292,22 @@ private static void HandleOpenExplorer(ArtifactPackage ap, Project project, stri
 </UI>"));
         }
 
+        private static string SupportBuildVersionForAgent(string originalVersion)
+        {
+            if (!originalVersion.Contains("+build"))
+                return originalVersion;
+
+            string version = originalVersion.Replace("+build", ".");
+            string[] arr = version.Split(new char[] { '.' });
+            if (arr.Length == 4 && arr[3].Length > 4)
+            {
+                arr[3] = arr[3].Substring(4);
+                return arr.Join(".");
+            }
+
+            return originalVersion;
+        }
+
         private static void RenameConfigFile(CmdLineOptions opts, ArtifactPackage ap, List<WixEntity> packageContents, string beatConfigExampleFileName, string beatConfigExampleFileId)
         {
             var dataContents = new DirectoryInfo(opts.PackageInDir)

src/installer/BeatPackageCompiler/Properties/launchSettings.json

@@ -29,6 +29,11 @@
       "commandName": "Project",
       "commandLineArgs": "--package=winlogbeat-8.12.1-windows-x86_64 -v --keep-temp-files",
       "workingDirectory": "$(SolutionDir)"
+    },
+    "Agent with custom build": {
+      "commandName": "Project",
+      "commandLineArgs": "--package=elastic-agent-8.14.0+build202407021002-windows-x86_64 -v --keep-temp-files",
+      "workingDirectory": "$(SolutionDir)"
     }
   }
 }

src/shared/ArtifactPackage.cs

@@ -38,7 +38,7 @@ public static bool FromFilename(string fileName, out ArtifactPackage artifactPac
         static readonly Regex rx = new Regex(
             /* 0 full capture, 7 groups total */
             /* 1 */ @$"(?<target>[^-]+({MagicStrings.Files.DashOssSuffix})?)" +
-            /* 2 */ @$"-(?<version>\d+\.\d+\.\d+)" +
+            /* 2 */ @$"-(?<version>\d+\.\d+\.\d+(?:\+build\d+)?)" +
             /* 3 */ @$"(-(?<qualifier>(?!\b(?:{MagicStrings.Ver.Snapshot})\b)[^-]+))?" +
             /* 4 */ @$"(-(?<snapshot>{MagicStrings.Ver.Snapshot}))?" +
             /* 5 */ @$"-(?<os>[^-]+)" +

src/shared/QualifiedVersion.cs

@@ -32,7 +32,7 @@ public static bool FromString(string value, out QualifiedVersion containerProper
 
         static readonly Regex rx = new Regex(
             /* 0 full capture, 4 groups total */
-            /* 1 */ @$"(?<version>\d+\.\d+(\.\d+)?)" +
+            /* 1 */ @$"(?<version>\d+\.\d+(\.\d+)?(?:\+build\d+)?)" +
             /* 2 */ @$"(-(?<qualifier>(?!\b(?:{MagicStrings.Ver.Snapshot})\b)[^-]+))?" +
             /* 3 */ @$"(-(?<snapshot>{MagicStrings.Ver.Snapshot}))?",
             RegexOptions.Compiled | RegexOptions.ExplicitCapture | RegexOptions.IgnoreCase);