Selectively skip and fix tests after PR#212 (#235)

PR #212 made some changes on `main` that require the presence of `MANIFEST_URL`. However, the pipeline is still configured
to run the same script when a PR (from an upstream branch) is raised and it currently fails.

This commit skips tests when triggered from another pipeline (unified release), and allows running tests when triggered by PR creation
by picking up a suitable `MANIFEST_URL`. DRA publishing is skipped unless we are triggered from the unified release.

Closes elastic/ingest-dev#2895

.buildkite/pipeline.yml

@@ -1,7 +1,5 @@
 ###
 ### The following environment variables can be set for testing purposes via the buildkite UI or CLI
-###  RUN_SNAPSHOT: "true" - will run the snapshot workflow
-###  RUN_STAGING: "true" - will run the staging workflow
 ###  ONLY_AGENT: "true" - will build only the elastic-agent msi artifact
 ###
 
@@ -19,7 +17,9 @@ steps:
         env:
           DRA_WORKFLOW: "snapshot"
       - label: ":package: DRA Publish Snapshot"
-        if: build.branch == 'main' || build.branch =~ /^[0-9]+\.[0-9]+\$/ || build.env("RUN_SNAPSHOT") == "true"
+        if: |
+          // Only run when triggered from Unified Release
+          ( (build.branch == 'main' || build.branch =~ /^[0-9]+\.[0-9]+\$/) && build.env("BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG") == "unified-release-snapshot" )
         command: ".buildkite/scripts/dra-publish.sh"
         key: "publish-snapshot"
         depends_on: "build-snapshot"
@@ -31,7 +31,10 @@ steps:
     key: "dra-staging"
     steps:
       - label: ":construction_worker: Build stack installers / Staging"
-        if: build.branch =~ /^[0-9]+\.[0-9]+\$/ || build.env("RUN_STAGING") == "true"
+        if: |
+          // Only run when triggered from Unified Release or via PRs targetting non-main, since there is no valid MANIFEST_URL for staging/main branch.
+          ( build.env("BUILDKITE_PULL_REQUEST") != "false" && build.env("BUILDKITE_PULL_REQUEST_BASE_BRANCH") =~ /^[0-9]+\.[0-9]+\$/ ) ||
+            ( build.branch =~ /^[0-9]+\.[0-9]+\$/ && build.env("BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG") == "unified-release-staging" )
         command: ".buildkite/scripts/build.ps1"
         key: "build-staging"
         artifact_paths: "c:/users/buildkite/esi/bin/out/**/*.msi"
@@ -41,15 +44,13 @@ steps:
         env:
           DRA_WORKFLOW: "staging"
       - label: ":package: DRA Publish staging"
-        if: build.branch =~ /^[0-9]+\.[0-9]+\$/ || build.env("RUN_STAGING") == "true"
+        if: |
+          // Only run when triggered from Unified Release
+            ( build.branch =~ /^[0-9]+\.[0-9]+\$/ && build.env("BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG") == "unified-release-staging" )
         command: ".buildkite/scripts/dra-publish.sh"
         key: "publish-staging"
         depends_on: "build-staging"
         agents:
           provider: gcp
         env:
           DRA_WORKFLOW: "staging"
-
-notify:
-  - slack: "#ingest-notifications"
-    if: build.branch == 'main'

.buildkite/scripts/build.ps1

@@ -1,11 +1,84 @@
 $ErrorActionPreference = "Stop"
 Set-Strictmode -version 3
 
-if (-not (Test-Path env:MANIFEST_URL)) {
+$eligibleReleaseBranchesMajor = "^[89]"
+$runTests = $true
+
+
+if (-not (Test-Path env:DRA_WORKFLOW)) {
+    $errorMessage = "Error: Required environment variable [DRA_WORKFLOW] is missing."
+    Write-Host $errorMessage
+    throw $errorMessage
+}
+
+function setManifestUrl {
+    param (
+        [string]$targetBranch
+    )
+
+    # the API url (snapshot or staging) expects master where we normally use main
+    $ApiTargetBranch = if ($targetBranch -eq "main") { "master" } else { $targetBranch }
+
+    if ($env:DRA_WORKFLOW -like "*snapshot*") {
+       $artifactsUrl = "https://snapshots.elastic.co/latest/${ApiTargetBranch}.json"
+    } elseif ($env:DRA_WORKFLOW -like "*staging*") {
+       $artifactsUrl = "https://staging.elastic.co/latest/${ApiTargetBranch}.json"
+    } else {
+        $errorMessage = "Error: Required environment variable [DRA_WORKFLOW] must be snapshot or staging but it was [${env:DRA_WORKFLOW}]."
+        Write-Host $errorMessage
+        throw $errorMessage
+    }
+
+    Write-Host "Artifacts url is: $artifactsUrl"
+    try {
+        $response = Invoke-WebRequest -UseBasicParsing -Uri $artifactsUrl
+        $responseJson = $response.Content | ConvertFrom-Json
+        $env:MANIFEST_URL = $responseJson.manifest_url
+        Write-Host "Using MANIFEST_URL=$env:MANIFEST_URL"
+    } catch {
+        $errorMessage = "There was an error parsing manifest_url from $artifactsUrl. Exiting."
+        throw $errorMessage
+    }
+}
+
+if (-not (Test-Path env:MANIFEST_URL) -and (Test-Path env:BUILDKITE_PULL_REQUEST) -and ($env:BUILDKITE_PULL_REQUEST -ne "false")) {
+    # we are called via a PR
+    Write-Host "~~~ Running in pull request mode"
+
+    $targetBranch = $env:BUILDKITE_PULL_REQUEST_BASE_BRANCH
+    if ( ($targetBranch -ne "main") -and -not ($targetBranch -like $eligibleReleaseBranchesMajor)) {
+        Write-Host "^^^ +++"
+        $errorMessage = "This PR is targetting the [$targetBranch] branch, but running tests is only supported against `main` or $eligibleReleaseBranchesMajor major branches. Exiting."
+        Write-Host $errorMessage
+        throw $errorMessage
+    }
+
+    setManifestUrl -targetBranch $targetBranch
+}
+elseif (-not (Test-Path env:MANIFEST_URL) -and ($env:BUILDKITE_SOURCE -ne "trigger_job") -and ($env:BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG -notlike "unified-release*")) {
+    # we got triggered by a branch push
+    Write-Host "~~~ Running in branch push mode"
+
+    $targetBranch = $env:BUILDKITE_PIPELINE_DEFAULT_BRANCH
+    if ( ($targetBranch -ne "main") -and -not ($targetBranch -like $eligibleReleaseBranchesMajor)) {
+        Write-Host "^^^ +++"
+        $errorMessage = "Tests triggered by branch pushes but running tests is only supported against `main` or $eligibleReleaseBranchesMajor major branches. Exiting."
+        Write-Host $errorMessage
+        throw $errorMessage
+    }
+
+    setManifestUrl -targetBranch $targetBranch
+}
+elseif (-not (Test-Path env:MANIFEST_URL)) {
+    # any other invocation of this script (e.g. from unified release) must supply MANIFEST_URL
     $errorMessage = "Error: Required environment variable [MANIFEST_URL] is missing."
     Write-Host $errorMessage
     throw $errorMessage
 }
+else {
+    Write-Host "~~~ Running via a Buildkite trigger: [$env:BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG], MANIFEST_URL=[$env:MANIFEST_URL]"
+    $runTests = $false
+}
 
 # workaround path limitation for max 248 characters
 # example: https://buildkite.com/elastic/elastic-stack-installers/builds/3104#018c5e1b-23a7-4330-ad5d-4acc69157822/74-180
@@ -126,12 +199,13 @@ if ($msiCount -ne $expected) {
     Write-Output "Success, found $msiCount artifacts in bin/out."
 }
 
-# temporarily disabling tests; they'll be re added via https://github.com/elastic/elastic-stack-installers/pull/225
-# try {
-#     & (Join-Path $PSScriptRoot "test.ps1")
-#     write-host "Testing Completed"
-# } catch {
-#     write-host "Testing Failed"
-#     write-error $_
-#     exit 1
-# }
+if ($runTests) {
+    try {
+        & (Join-Path $PSScriptRoot "test.ps1")
+        write-host "Testing Completed"
+    } catch {
+        write-host "Testing Failed"
+        write-error $_
+        exit 1
+    }
+}

.buildkite/scripts/test.ps1

@@ -1,6 +1,7 @@
 $ErrorActionPreference = "Stop"
 Set-Strictmode -version 3
 
+Write-Host "~~~ Running Tests"
 write-host (ConvertTo-Json $PSVersiontable -Compress)
 write-host "Running as: $([Environment]::UserName)"
 

catalog-info.yaml

@@ -58,6 +58,11 @@ spec:
           access_level: BUILD_AND_READ
         ingest-fp: {}
         release-eng: {}
+      env:
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
+        SLACK_NOTIFICATIONS_CHANNEL: '#ingest-notifications'
+        SLACK_NOTIFICATIONS_ON_SUCCESS: 'false'
+
 ---
 # yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json
 apiVersion: backstage.io/v1alpha1

src/agent-qa/helpers.ps1

@@ -160,13 +160,19 @@ Function Has-AgentInstallerProcess {
 }
 
 Function Has-AgentLogged {
-    try {
-        $LogFile = Get-AgentLogFile -Latest $True
-        return $true
-    }
-    catch {
-        return $false
+
+    foreach ($i in 0..5) {
+
+        try {
+            $LogFile = Get-AgentLogFile -Latest $True
+            return $true
+        } catch {}
+
+        write-host "Waiting for agent to log message"
+        start-sleep -seconds 3
     }
+
+    return $false
 }
 
 Function Is-AgentLogGrowing {
@@ -196,36 +202,46 @@ Function Is-AgentLogGrowing {
 }
 
 Function Has-AgentStandaloneLog {
-    try {
+    if (-not (Has-AgentLogged)) { return $false }
+
+    foreach ($i in 0..5) {
         $LogFile = Get-AgentLogFile -Latest $True
-    }
-    catch {
-        return $false
-    }
 
-    $Content = Get-Content -raw $LogFile
+        $Content = Get-Content -raw $LogFile
+
+        if ($Content -like "*Parsed configuration and determined agent is managed locally*") {
+            return $True
+        }
 
-    if ($Content -like "*Parsed configuration and determined agent is managed locally*") {
-        return $True
+        write-host "Waiting for agent to log standalone message"
+        start-sleep -seconds 3
     }
 
+    write-host "Agent did not log standalone message"
+    write-host "Agent Logged to $LogFile :"
+    write-host $Content
     return $false
 }
 
 Function Has-AgentFleetEnrollmentAttempt {
-    try {
+    if (-not (Has-AgentLogged)) { return $false }
+
+    foreach ($i in 0..5) {
         $LogFile = Get-AgentLogFile -Latest $True
-    }
-    catch {
-        return $false
-    }
 
-    $Content = Get-Content -raw $LogFile
+        $Content = Get-Content -raw $LogFile
+
+        if ($Content -like "*failed to perform delayed enrollment: fail to enroll: fail to execute request to fleet-server: lookup placeholder: no such host*") {
+            return $True
+        }
 
-    if ($Content -like "*failed to perform delayed enrollment: fail to enroll: fail to execute request to fleet-server: lookup placeholder: no such host*") {
-        return $True
+        write-host "Waiting for agent to log fleet message"
+        start-sleep -seconds 3
     }
 
+    write-host "Agent did not log fleet message"
+    write-host "Agent Logged to $LogFile :"
+    write-host $Content
     return $false
 }
 
@@ -345,7 +361,7 @@ Function Clean-ElasticAgentDirectory {
 Function Clean-ElasticAgentUninstallKeys {
     $Keys = Get-AgentUninstallRegistryKey -Passthru
     if ($Keys) {
-        Remove-Item -Path $Keys
+        $Keys | Remove-Item -force -verbose
     }
 }
 
@@ -375,7 +391,7 @@ Function Invoke-MSIExec {
 
     if ($LogToDir) {
         $LoggingDestination = (New-MSIVerboseLoggingDestination -Destination $LogToDir -Suffix $Action)
-        $arglist += " /l*v " + $LoggingDestination
+        $arglist += " /l*v " + """" + $LoggingDestination + """"
     }
 
     write-verbose "Invoking msiexec.exe $arglist"
@@ -425,7 +441,7 @@ Function Install-MSI {
         $LogToDir
     )
 
-    $msiArgs = @(@($Path) + $Interactive + $Flags)
+    $msiArgs = @(@("""$Path""") + $Interactive + $Flags)
 
     Invoke-MSIExec -Action i -Arguments $msiArgs -LogToDir $LogToDir
 }
@@ -444,7 +460,17 @@ Function Uninstall-MSI {
         throw "Uninstall-msi called without path to an MSI or a GUID"
     }
 
-    $msiArgs = @($Path,$Guid).Where{$_} + $Interactive + $Flags
+    $msiargs = @()
+    if (-not [string]::IsNullOrWhiteSpace($Path)) {
+        $msiargs += @("""$Path""")
+    }
+
+    if (-not [string]::IsNullOrWhiteSpace($Guid)) {
+        $msiargs += @($Guid)
+    }
+
+    $msiArgs += $Interactive
+    $msiArgs += $Flags
 
     $OpenFiles = @(Find-OpenFile | Where-Object {$_.Name -like "*Elastic\Agent*" -or $_.Name -like "*Elastic\Beats*"})
     foreach ($OpenFile in $OpenFiles) {