Selectively skip and fix tests after PR#212

.buildkite/pipeline.yml

@@ -1,7 +1,5 @@
 ###
 ### The following environment variables can be set for testing purposes via the buildkite UI or CLI
-###  RUN_SNAPSHOT: "true" - will run the snapshot workflow
-###  RUN_STAGING: "true" - will run the staging workflow
 ###  ONLY_AGENT: "true" - will build only the elastic-agent msi artifact
 ###
 
@@ -19,7 +17,9 @@ steps:
         env:
           DRA_WORKFLOW: "snapshot"
       - label: ":package: DRA Publish Snapshot"
-        if: build.branch == 'main' || build.branch =~ /^[0-9]+\.[0-9]+\$/ || build.env("RUN_SNAPSHOT") == "true"
+        if: |
+          // Only run when triggered from Unified Release
+          ( (build.branch == 'main' || build.branch =~ /^[0-9]+\.[0-9]+\$/) && build.env("BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG") == "unified-release-snapshot" )
         command: ".buildkite/scripts/dra-publish.sh"
         key: "publish-snapshot"
         depends_on: "build-snapshot"
@@ -31,7 +31,10 @@ steps:
     key: "dra-staging"
     steps:
       - label: ":construction_worker: Build stack installers / Staging"
-        if: build.branch =~ /^[0-9]+\.[0-9]+\$/ || build.env("RUN_STAGING") == "true"
+        if: |
+          // Only run when triggered from Unified Release or via PRs targetting non-main, since there is no valid MANIFEST_URL for staging/main branch.
+          ( build.env("BUILDKITE_PULL_REQUEST") != "false" && build.env("BUILDKITE_PULL_REQUEST_BASE_BRANCH") =~ /^[0-9]+\.[0-9]+\$/ ) ||
+            ( build.branch =~ /^[0-9]+\.[0-9]+\$/ && build.env("BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG") == "unified-release-staging" )
         command: ".buildkite/scripts/build.ps1"
         key: "build-staging"
         artifact_paths: "c:/users/buildkite/esi/bin/out/**/*.msi"
@@ -41,15 +44,13 @@ steps:
         env:
           DRA_WORKFLOW: "staging"
       - label: ":package: DRA Publish staging"
-        if: build.branch =~ /^[0-9]+\.[0-9]+\$/ || build.env("RUN_STAGING") == "true"
+        if: |
+          // Only run when triggered from Unified Release
+            ( build.branch =~ /^[0-9]+\.[0-9]+\$/ && build.env("BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG") == "unified-release-staging" )
         command: ".buildkite/scripts/dra-publish.sh"
         key: "publish-staging"
         depends_on: "build-staging"
         agents:
           provider: gcp
         env:
           DRA_WORKFLOW: "staging"
-
-notify:
-  - slack: "#ingest-notifications"
-    if: build.branch == 'main'

.buildkite/scripts/build.ps1

@@ -1,11 +1,84 @@
 $ErrorActionPreference = "Stop"
 Set-Strictmode -version 3
 
-if (-not (Test-Path env:MANIFEST_URL)) {
+$eligibleReleaseBranchesMajor = "^[89]"
+$runTests = $true
+
+
+if (-not (Test-Path env:DRA_WORKFLOW)) {
+    $errorMessage = "Error: Required environment variable [DRA_WORKFLOW] is missing."
+    Write-Host $errorMessage
+    throw $errorMessage
+}
+
+function setManifestUrl {
+    param (
+        [string]$targetBranch
+    )
+
+    # the API url (snapshot or staging) expects master where we normally use main
+    $ApiTargetBranch = if ($targetBranch -eq "main") { "master" } else { $targetBranch }
+
+    if ($env:DRA_WORKFLOW -like "*snapshot*") {
+       $artifactsUrl = "https://snapshots.elastic.co/latest/${ApiTargetBranch}.json"
+    } elseif ($env:DRA_WORKFLOW -like "*staging*") {
+       $artifactsUrl = "https://staging.elastic.co/latest/${ApiTargetBranch}.json"
+    } else {
+        $errorMessage = "Error: Required environment variable [DRA_WORKFLOW] must be snapshot or staging but it was [${env:DRA_WORKFLOW}]."
+        Write-Host $errorMessage
+        throw $errorMessage
+    }
+
+    Write-Host "Artifacts url is: $artifactsUrl"
+    try {
+        $response = Invoke-WebRequest -UseBasicParsing -Uri $artifactsUrl
+        $responseJson = $response.Content | ConvertFrom-Json
+        $env:MANIFEST_URL = $responseJson.manifest_url
+        Write-Host "Using MANIFEST_URL=$env:MANIFEST_URL"
+    } catch {
+        $errorMessage = "There was an error parsing manifest_url from $artifactsUrl. Exiting."
+        throw $errorMessage
+    }
+}
+
+if (-not (Test-Path env:MANIFEST_URL) -and (Test-Path env:BUILDKITE_PULL_REQUEST) -and ($env:BUILDKITE_PULL_REQUEST -ne "false")) {
+    # we are called via a PR
+    Write-Host "~~~ Running in pull request mode"
+
+    $targetBranch = $env:BUILDKITE_PULL_REQUEST_BASE_BRANCH
+    if ( ($targetBranch -ne "main") -and -not ($targetBranch -like $eligibleReleaseBranchesMajor)) {
+        Write-Host "^^^ +++"
+        $errorMessage = "This PR is targetting the [$targetBranch] branch, but running tests is only supported against `main` or $eligibleReleaseBranchesMajor major branches. Exiting."
+        Write-Host $errorMessage
+        throw $errorMessage
+    }
+
+    setManifestUrl -targetBranch $targetBranch
+}
+elseif (-not (Test-Path env:MANIFEST_URL) -and ($env:BUILDKITE_SOURCE -ne "trigger_job") -and ($env:BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG -notlike "unified-release*")) {
+    # we got triggered by a branch push
+    Write-Host "~~~ Running in branch push mode"
+
+    $targetBranch = $env:BUILDKITE_PIPELINE_DEFAULT_BRANCH
+    if ( ($targetBranch -ne "main") -and -not ($targetBranch -like $eligibleReleaseBranchesMajor)) {
+        Write-Host "^^^ +++"
+        $errorMessage = "Tests triggered by branch pushes but running tests is only supported against `main` or $eligibleReleaseBranchesMajor major branches. Exiting."
+        Write-Host $errorMessage
+        throw $errorMessage
+    }
+
+    setManifestUrl -targetBranch $targetBranch
+}
+elseif (-not (Test-Path env:MANIFEST_URL)) {
+    # any other invocation of this script (e.g. from unified release) must supply MANIFEST_URL
     $errorMessage = "Error: Required environment variable [MANIFEST_URL] is missing."
     Write-Host $errorMessage
     throw $errorMessage
 }
+else {
+    Write-Host "~~~ Running via a Buildkite trigger: [$env:BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG], MANIFEST_URL=[$env:MANIFEST_URL]"
+    $runTests = $false
+}
 
 # workaround path limitation for max 248 characters
 # example: https://buildkite.com/elastic/elastic-stack-installers/builds/3104#018c5e1b-23a7-4330-ad5d-4acc69157822/74-180
@@ -126,12 +199,13 @@ if ($msiCount -ne $expected) {
     Write-Output "Success, found $msiCount artifacts in bin/out."
 }
 
-# temporarily disabling tests; they'll be re added via https://github.com/elastic/elastic-stack-installers/pull/225
-# try {
-#     & (Join-Path $PSScriptRoot "test.ps1")
-#     write-host "Testing Completed"
-# } catch {
-#     write-host "Testing Failed"
-#     write-error $_
-#     exit 1
-# }
+if ($runTests) {
+    try {
+        & (Join-Path $PSScriptRoot "test.ps1")
+        write-host "Testing Completed"
+    } catch {
+        write-host "Testing Failed"
+        write-error $_
+        exit 1
+    }
+}

.buildkite/scripts/test.ps1

@@ -1,6 +1,7 @@
 $ErrorActionPreference = "Stop"
 Set-Strictmode -version 3
 
+Write-Host "~~~ Running Tests"
 write-host (ConvertTo-Json $PSVersiontable -Compress)
 write-host "Running as: $([Environment]::UserName)"
 

catalog-info.yaml

@@ -58,6 +58,11 @@ spec:
           access_level: BUILD_AND_READ
         ingest-fp: {}
         release-eng: {}
+      env:
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
+        SLACK_NOTIFICATIONS_CHANNEL: '#ingest-notifications'
+        SLACK_NOTIFICATIONS_ON_SUCCESS: 'false'
+
 ---
 # yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json
 apiVersion: backstage.io/v1alpha1

src/agent-qa/helpers.ps1

@@ -160,13 +160,19 @@ Function Has-AgentInstallerProcess {
 }
 
 Function Has-AgentLogged {
-    try {
-        $LogFile = Get-AgentLogFile -Latest $True
-        return $true
-    }
-    catch {
-        return $false
+
+    foreach ($i in 0..5) {
+
+        try {
+            $LogFile = Get-AgentLogFile -Latest $True
+            return $true
+        } catch {}
+
+        write-host "Waiting for agent to log message"
+        start-sleep -seconds 3
     }
+
+    return $false
 }
 
 Function Is-AgentLogGrowing {
@@ -196,36 +202,46 @@ Function Is-AgentLogGrowing {
 }
 
 Function Has-AgentStandaloneLog {
-    try {
+    if (-not (Has-AgentLogged)) { return $false }
+
+    foreach ($i in 0..5) {
         $LogFile = Get-AgentLogFile -Latest $True
-    }
-    catch {
-        return $false
-    }
 
-    $Content = Get-Content -raw $LogFile
+        $Content = Get-Content -raw $LogFile
+
+        if ($Content -like "*Parsed configuration and determined agent is managed locally*") {
+            return $True
+        }
 
-    if ($Content -like "*Parsed configuration and determined agent is managed locally*") {
-        return $True
+        write-host "Waiting for agent to log standalone message"
+        start-sleep -seconds 3
     }
 
+    write-host "Agent did not log standalone message"
+    write-host "Agent Logged to $LogFile :"
+    write-host $Content
     return $false
 }
 
 Function Has-AgentFleetEnrollmentAttempt {
-    try {
+    if (-not (Has-AgentLogged)) { return $false }
+
+    foreach ($i in 0..5) {
         $LogFile = Get-AgentLogFile -Latest $True
-    }
-    catch {
-        return $false
-    }
 
-    $Content = Get-Content -raw $LogFile
+        $Content = Get-Content -raw $LogFile
+
+        if ($Content -like "*failed to perform delayed enrollment: fail to enroll: fail to execute request to fleet-server: lookup placeholder: no such host*") {
+            return $True
+        }
 
-    if ($Content -like "*failed to perform delayed enrollment: fail to enroll: fail to execute request to fleet-server: lookup placeholder: no such host*") {
-        return $True
+        write-host "Waiting for agent to log fleet message"
+        start-sleep -seconds 3
     }
 
+    write-host "Agent did not log fleet message"
+    write-host "Agent Logged to $LogFile :"
+    write-host $Content
     return $false
 }
 
@@ -345,7 +361,7 @@ Function Clean-ElasticAgentDirectory {
 Function Clean-ElasticAgentUninstallKeys {
     $Keys = Get-AgentUninstallRegistryKey -Passthru
     if ($Keys) {
-        Remove-Item -Path $Keys
+        $Keys | Remove-Item -force -verbose
     }
 }
 
@@ -375,7 +391,7 @@ Function Invoke-MSIExec {
 
     if ($LogToDir) {
         $LoggingDestination = (New-MSIVerboseLoggingDestination -Destination $LogToDir -Suffix $Action)
-        $arglist += " /l*v " + $LoggingDestination
+        $arglist += " /l*v " + """" + $LoggingDestination + """"
     }
 
     write-verbose "Invoking msiexec.exe $arglist"
@@ -425,7 +441,7 @@ Function Install-MSI {
         $LogToDir
     )
 
-    $msiArgs = @(@($Path) + $Interactive + $Flags)
+    $msiArgs = @(@("""$Path""") + $Interactive + $Flags)
 
     Invoke-MSIExec -Action i -Arguments $msiArgs -LogToDir $LogToDir
 }
@@ -444,7 +460,17 @@ Function Uninstall-MSI {
         throw "Uninstall-msi called without path to an MSI or a GUID"
     }
 
-    $msiArgs = @($Path,$Guid).Where{$_} + $Interactive + $Flags
+    $msiargs = @()
+    if (-not [string]::IsNullOrWhiteSpace($Path)) {
+        $msiargs += @("""$Path""")
+    }
+
+    if (-not [string]::IsNullOrWhiteSpace($Guid)) {
+        $msiargs += @($Guid)
+    }
+
+    $msiArgs += $Interactive
+    $msiArgs += $Flags
 
     $OpenFiles = @(Find-OpenFile | Where-Object {$_.Name -like "*Elastic\Agent*" -or $_.Name -like "*Elastic\Beats*"})
     foreach ($OpenFile in $OpenFiles) {