From 372cbc75cdb9ea3fe925980c23a3876f717aa85f Mon Sep 17 00:00:00 2001 From: Doug W <55553826+dwhyrock@users.noreply.github.com> Date: Fri, 12 Jul 2024 11:44:09 -0400 Subject: [PATCH] Add Independent Agent alternate upload step in pipeline (#287) * Add Independent Agent alternate upload step in pipeline * empty commit to trigger CI --- .buildkite/pipeline.yml | 24 ++++++++++ .buildkite/scripts/build.ps1 | 2 +- .buildkite/scripts/ind-agent-publish.sh | 45 +++++++++++++++++++ .../BeatPackageCompiler.cs | 19 +++++++- .../Properties/launchSettings.json | 5 +++ src/shared/ArtifactPackage.cs | 2 +- src/shared/QualifiedVersion.cs | 2 +- 7 files changed, 95 insertions(+), 4 deletions(-) create mode 100644 .buildkite/scripts/ind-agent-publish.sh diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 765d10aa..add45359 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -59,3 +59,27 @@ steps: provider: gcp env: DRA_WORKFLOW: "staging" + - group: ":package: Stack installers Independent Agent Staging" + key: "independent-agent-staging" + if: | + // This should only run when triggered from Independent Agent Staging + (build.env("BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG") == "independent-agent-release-staging") + steps: + - label: ":construction_worker: Build stack installers / Independent Agent Staging" + command: ".buildkite/scripts/build.ps1" + key: "build-staging-independent-agent" + artifact_paths: "c:/users/buildkite/esi/bin/out/**/*.msi" + agents: + provider: gcp + image: family/core-windows-2022 + env: + DRA_WORKFLOW: "staging" + - label: ":package: Publishing via BK API for Independent Agent Release" + command: ".buildkite/scripts/ind-agent-publish.sh" + key: "publish-staging-independent-agent" + artifact_paths: "stack-installers-output/**/*" + depends_on: "build-staging-independent-agent" + agents: + provider: gcp + env: + DRA_WORKFLOW: "staging" diff --git a/.buildkite/scripts/build.ps1 b/.buildkite/scripts/build.ps1 index a0d25611..9a0dbc25 100755 --- a/.buildkite/scripts/build.ps1 +++ b/.buildkite/scripts/build.ps1 @@ -55,7 +55,7 @@ if (-not (Test-Path env:MANIFEST_URL) -and (Test-Path env:BUILDKITE_PULL_REQUEST setManifestUrl -targetBranch $targetBranch } -elseif (-not (Test-Path env:MANIFEST_URL) -and ($env:BUILDKITE_SOURCE -ne "trigger_job") -and ($env:BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG -notlike "unified-release*")) { +elseif (-not (Test-Path env:MANIFEST_URL) -and ($env:BUILDKITE_SOURCE -ne "trigger_job") -and (($env:BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG -notlike "unified-release*") -or ($env:BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG -notlike "independent-agent-release*"))) { # we got triggered by a branch push Write-Host "~~~ Running in branch push mode" diff --git a/.buildkite/scripts/ind-agent-publish.sh b/.buildkite/scripts/ind-agent-publish.sh new file mode 100644 index 00000000..ace17fe3 --- /dev/null +++ b/.buildkite/scripts/ind-agent-publish.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +set -euo pipefail + +# Download artifacts from Buildkite "Build stack installers" step +echo "+++ Downloading artifacts..." +# it's possible to also use unix style paths for download (users/buildkites/...) +# but it's an experimental feature that needs an agent flag set: https://buildkite.com/docs/agent/v3#experimental-features-normalised-upload-paths +buildkite-agent artifact download 'users\buildkite\esi\bin\out\**\*.msi' . --step "build-${DRA_WORKFLOW}-independent-agent" + +echo "--- Creating output dir and moving files there" +OUTPUT_DIR="stack-installers-output" +mkdir "$OUTPUT_DIR" +mv users/buildkite/esi/bin "$OUTPUT_DIR" +chmod -R 777 "$OUTPUT_DIR" + +echo "--- ls -alR" +ls -alR "${OUTPUT_DIR}" + +# Create sha512 for msi file(s) +# We can leave it as this loop in order to account for a possible future ARM msi +echo "--- Creating sha512 files" +pushd "${OUTPUT_DIR}/bin/out/elastic" +for f in *.msi; do + sha512sum "${f}" > "${f}.sha512" +done + +# The defined "artifacts_path" will pick up "stack-installers-output/**/*" + +# Check and set trigger id +if [[ -z "${TRIGGER_JOB_ID-}" ]]; then + echo "--- TRIGGER_JOB_ID is not set, so not setting metadata" +else + # If a pipeline that triggered this build passes in a "TRIGGER_JOB_ID" env var, that + # is an indicator that it will want us to set some metadata in that calling build + # so that it can reference this specific build ID in order to easily download + # artifacts saved off in this build. + # + # This is a much easier way to pull back artifacts from a triggered build than using + # a Buildkite token that we then have to manage via vault, etc. + # See: https://forum.buildkite.community/t/how-to-download-artifacts-back-from-triggered-pipeline/3480/2 + echo "--- Setting metadata for job that trigger this one" + buildkite-agent meta-data set "triggered_build_id" "$BUILDKITE_BUILD_ID" --job $TRIGGER_JOB_ID + buildkite-agent meta-data set "triggered_commit_hash" "$BUILDKITE_COMMIT" --job $TRIGGER_JOB_ID +fi diff --git a/src/installer/BeatPackageCompiler/BeatPackageCompiler.cs b/src/installer/BeatPackageCompiler/BeatPackageCompiler.cs index 3f4998b7..2fc3ee26 100644 --- a/src/installer/BeatPackageCompiler/BeatPackageCompiler.cs +++ b/src/installer/BeatPackageCompiler/BeatPackageCompiler.cs @@ -40,6 +40,7 @@ static void Main(string[] args) // Generate UUID v5 from product properties. // This UUID *must* be stable and unique between Beats. var upgradeCode = Uuid5.FromString(ap.CanonicalTargetName); + string version = SupportBuildVersionForAgent(ap.Version); var project = new Project(displayName) { @@ -52,7 +53,7 @@ static void Main(string[] args) Description = pc.Description, OutFileName = Path.Combine(opts.PackageOutDir, opts.PackageName), - Version = new Version(ap.Version), + Version = new Version(version), // We massage LICENSE.txt into .rtf below LicenceFile = Path.Combine( @@ -291,6 +292,22 @@ private static void HandleOpenExplorer(ArtifactPackage ap, Project project, stri ")); } + private static string SupportBuildVersionForAgent(string originalVersion) + { + if (!originalVersion.Contains("+build")) + return originalVersion; + + string version = originalVersion.Replace("+build", "."); + string[] arr = version.Split(new char[] { '.' }); + if (arr.Length == 4 && arr[3].Length > 4) + { + arr[3] = arr[3].Substring(4); + return arr.Join("."); + } + + return originalVersion; + } + private static void RenameConfigFile(CmdLineOptions opts, ArtifactPackage ap, List packageContents, string beatConfigExampleFileName, string beatConfigExampleFileId) { var dataContents = new DirectoryInfo(opts.PackageInDir) diff --git a/src/installer/BeatPackageCompiler/Properties/launchSettings.json b/src/installer/BeatPackageCompiler/Properties/launchSettings.json index 2734691f..062adec2 100644 --- a/src/installer/BeatPackageCompiler/Properties/launchSettings.json +++ b/src/installer/BeatPackageCompiler/Properties/launchSettings.json @@ -29,6 +29,11 @@ "commandName": "Project", "commandLineArgs": "--package=winlogbeat-8.12.1-windows-x86_64 -v --keep-temp-files", "workingDirectory": "$(SolutionDir)" + }, + "Agent with custom build": { + "commandName": "Project", + "commandLineArgs": "--package=elastic-agent-8.14.0+build202407021002-windows-x86_64 -v --keep-temp-files", + "workingDirectory": "$(SolutionDir)" } } } \ No newline at end of file diff --git a/src/shared/ArtifactPackage.cs b/src/shared/ArtifactPackage.cs index c7a18395..cd3a9efb 100644 --- a/src/shared/ArtifactPackage.cs +++ b/src/shared/ArtifactPackage.cs @@ -38,7 +38,7 @@ public static bool FromFilename(string fileName, out ArtifactPackage artifactPac static readonly Regex rx = new Regex( /* 0 full capture, 7 groups total */ /* 1 */ @$"(?[^-]+({MagicStrings.Files.DashOssSuffix})?)" + - /* 2 */ @$"-(?\d+\.\d+\.\d+)" + + /* 2 */ @$"-(?\d+\.\d+\.\d+(?:\+build\d+)?)" + /* 3 */ @$"(-(?(?!\b(?:{MagicStrings.Ver.Snapshot})\b)[^-]+))?" + /* 4 */ @$"(-(?{MagicStrings.Ver.Snapshot}))?" + /* 5 */ @$"-(?[^-]+)" + diff --git a/src/shared/QualifiedVersion.cs b/src/shared/QualifiedVersion.cs index b7c9be4f..19103de7 100644 --- a/src/shared/QualifiedVersion.cs +++ b/src/shared/QualifiedVersion.cs @@ -32,7 +32,7 @@ public static bool FromString(string value, out QualifiedVersion containerProper static readonly Regex rx = new Regex( /* 0 full capture, 4 groups total */ - /* 1 */ @$"(?\d+\.\d+(\.\d+)?)" + + /* 1 */ @$"(?\d+\.\d+(\.\d+)?(?:\+build\d+)?)" + /* 2 */ @$"(-(?(?!\b(?:{MagicStrings.Ver.Snapshot})\b)[^-]+))?" + /* 3 */ @$"(-(?{MagicStrings.Ver.Snapshot}))?", RegexOptions.Compiled | RegexOptions.ExplicitCapture | RegexOptions.IgnoreCase);