Fixed issues with Heartbeat pipeline (#39045)

* fixed issues in heartbeat pipeline

* pr fixes

(cherry picked from commit b4ff53c)

.buildkite/heartbeat/heartbeat-pipeline.yml

@@ -24,28 +24,39 @@ env:
 steps:
   - group: "Heartbeat Mandatory Testing"
     key: "heartbeat-mandatory-tests"
+
     steps:
-      - label: ":ubuntu: Unit Tests"
-        command: "cd heartbeat && mage build unitTest"
+      - label: ":ubuntu: Heartbeat Unit Tests"
+        command: |
+          cd heartbeat
+          mage build unitTest
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
           machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
         artifact_paths:
           - "heartbeat/build/*.xml"
           - "heartbeat/build/*.json"
+        notify:
+          - github_commit_status:
+              context: "Heartbeat: Ubuntu Unit Tests"
 
       - label: ":rhel: Heartbeat Rhel9 Unit Tests"
-        command: "cd heartbeat && mage build unitTest"
+        command: |
+          cd heartbeat
+          mage build unitTest
         agents:
           provider: "gcp"
           image: "${IMAGE_RHEL9}"
           machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
         artifact_paths:
           - "heartbeat/build/*.xml"
           - "heartbeat/build/*.json"
+        notify:
+          - github_commit_status:
+              context: "Heartbeat: Rhel9 Unit Tests"
 
-      - label: ":windows: Heartbeat Win-2016 Unit Test"
+      - label: ":windows: Heartbeat Win-2016 Unit Tests"
         key: "windows-2016"
         command: |
           Set-Location -Path heartbeat
@@ -58,6 +69,9 @@ steps:
         artifact_paths:
           - "heartbeat/build/*.xml"
           - "heartbeat/build/*.json"
+        notify:
+          - github_commit_status:
+              context: "Heartbeat: Win-2016 Unit Tests"
 
       - label: ":windows: Heartbeat Win-2022 Unit Test"
         key: "windows-2022"
@@ -72,53 +86,97 @@ steps:
         artifact_paths:
           - "heartbeat/build/*.xml"
           - "heartbeat/build/*.json"
+        notify:
+          - github_commit_status:
+              context: "Heartbeat: Win-2022 Unit Tests"
 
       - label: ":ubuntu: Heartbeat Go Integration Tests"
-        command: "cd heartbeat && mage goIntegTest"
+        command:  |
+          cd heartbeat
+          mage goIntegTest
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
           machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
         artifact_paths:
           - "heartbeat/build/*.xml"
           - "heartbeat/build/*.json"
+        notify:
+          - github_commit_status:
+              context: "Heartbeat: Go Integration Tests"
 
       - label: ":ubuntu: Heartbeat Python Integration Tests"
-        command: "cd heartbeat && mage pythonIntegTest"
+        command: |
+          cd heartbeat
+          mage pythonIntegTest
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
           machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
         artifact_paths:
           - "heartbeat/build/*.xml"
           - "heartbeat/build/*.json"
+        notify:
+          - github_commit_status:
+              context: "Heartbeat: Python Integration Tests"
 
   - group: "Heartbeat ARM Tests"
     key: "heartbeat-extended-tests-arm"
     if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
+
     steps:
       - label: ":linux: Heartbeat ARM64 Unit Tests"
         key: "arm-extended"
-        if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat for arm" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
-        command: "cd heartbeat && mage build unitTest"
+        command: |
+          cd heartbeat
+          mage build unitTest
         agents:
           provider: "aws"
           imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
           instanceType: "${AWS_ARM_INSTANCE_TYPE}"
         artifact_paths: "heartbeat/build/*.xml"
+        notify:
+          - github_commit_status:
+              context: "Heartbeat: Ubuntu ARM64 Unit Tests"
 
   - group: "Heartbeat Extended Testing MacOS"
     key: "heartbeat-extended-tests-macos"
     if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
+
     steps:
       - label: ":mac: Heartbeat MacOS Unit Tests"
         key: "macos-extended"
-        if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
-        command: "cd ${BEATS_PROJECT_NAME} && mage build unitTest"
+        command: |
+          set -euo pipefail
+          source .buildkite/scripts/install_macos_tools.sh
+          cd heartbeat
+          mage build unitTest
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_X86_64}"
-        artifact_paths: "heartbeat/build/*.xml"
+        artifact_paths:
+          - "metricbeat/build/*.xml"
+          - "metricbeat/build/*.json"
+        notify:
+          - github_commit_status:
+              context: "Heartbeat: MacOS Unit Tests"
+
+      - label: ":mac: Heartbeat MacOS ARM Unit Tests"
+        key: "macos-arm-extended"
+        command: |
+          set -euo pipefail
+          source .buildkite/scripts/install_macos_tools.sh
+          cd heartbeat
+          mage build unitTest
+        agents:
+          provider: "orka"
+          imagePrefix: "${IMAGE_MACOS_ARM}"
+        artifact_paths:
+          - "metricbeat/build/*.xml"
+          - "metricbeat/build/*.json"
+        notify:
+          - github_commit_status:
+              context: "Heartbeat: MacOS ARM Unit Tests"
 
   - group: "Heartbeat Windows Extended Testing"
     key: "heartbeat-extended-tests-win"
@@ -127,7 +185,9 @@ steps:
     steps:
       - label: ":windows: Heartbeat Win-2019 Unit Tests"
         key: "heartbeat-win-extended-2019"
-        command: "mage -d ${BEATS_PROJECT_NAME} unitTest"
+        command:  |
+          Set-Location -Path heartbeat
+          mage build unitTest
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2019}"
@@ -136,6 +196,9 @@ steps:
         artifact_paths:
           - "heartbeat/build/*.xml"
           - "heartbeat/build/*.json"
+        notify:
+          - github_commit_status:
+              context: "Heartbeat: Win-2019 Unit Tests"
 
       - label: ":windows: Heartbeat Win-11 Unit Tests"
         key: "heartbeat-windows-extended-11"
@@ -150,6 +213,9 @@ steps:
         artifact_paths:
           - "heartbeat/build/*.xml"
           - "heartbeat/build/*.json"
+        notify:
+          - github_commit_status:
+              context: "Heartbeat: Win-11 Unit Tests"
 
       - label: ":windows: Heartbeat Win-10 Unit Tests"
         key: "heartbeat-windows-extended-10"
@@ -164,32 +230,46 @@ steps:
         artifact_paths:
           - "heartbeat/build/*.xml"
           - "heartbeat/build/*.json"
+        notify:
+          - github_commit_status:
+              context: "Heartbeat: Win-10 Unit Tests"
+
+  - wait: ~
+    if: build.env("BUILDKITE_PULL_REQUEST") != "false"
+    depends_on: "heartbeat-mandatory-tests"
 
   - group: "Heartbeat Packaging"
     key: "heartbeat-packaging"
-    if: build.env("BUILDKITE_PULL_REQUEST") != "false"
-    depends_on:
-      - "heartbeat-mandatory-tests"
     steps:
       - label: ":ubuntu: Heartbeat Packaging Linux X86"
         key: "heartbeat-package-linux-x86"
         env:
           PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
           SNAPSHOT: true
-        command: "cd heartbeat && mage package"
+        command: |
+          cd heartbeat
+          mage package
         agents:
           provider: gcp
           image: "${IMAGE_UBUNTU_X86_64}"
           machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+        notify:
+          - github_commit_status:
+              context: "Heartbeat: Packaging Ubuntu x86_64"
 
       - label: ":linux: Heartbeat Packaging Linux ARM"
         key: "heartbeat-package-linux-arm"
         env:
           PLATFORMS: "linux/arm64"
           PACKAGES: "docker"
           SNAPSHOT: true
-        command: "cd heartbeat && mage package"
+        command: |
+          cd heartbeat
+          mage package
         agents:
           provider: "aws"
           imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
           instanceType: "${AWS_ARM_INSTANCE_TYPE}"
+        notify:
+          - github_commit_status:
+              context: "Heartbeat: Packaging Ubuntu ARM"

.buildkite/hooks/pre-command

@@ -11,7 +11,7 @@ PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/gcp-platform-inges
 DOCKER_REGISTRY_SECRET_PATH="kv/ci-shared/platform-ingest/docker_registry_prod"
 GITHUB_TOKEN_VAULT_PATH="kv/ci-shared/platform-ingest/github_token"
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats" || "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "heartbeat" ]]; then
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats" || "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" ]]; then
   source .buildkite/env-scripts/env.sh
   if [[ -z "${GO_VERSION-""}" ]]; then
     export GO_VERSION=$(cat "${WORKSPACE}/.go-version")
@@ -56,6 +56,7 @@ fi
 
 ENABLED_BEATS_PIPELINES_SLUGS=(
   "auditbeat"
+  "heartbeat"
   "filebeat"
   "beats-metricbeat"
   "beats-packetbeat"

.buildkite/hooks/scripts/util.sh

@@ -25,10 +25,11 @@ google_cloud_logout_active_account() {
 }
 
 cleanup() {
-  # TODO: disable for upload pipeline
+  if [[ "$BUILDKITE_COMMAND" != *"buildkite-agent pipeline upload"* ]]; then
   echo "Deleting temporary files..."
-  if [[ -e "${BIN}/${TMP_FOLDER}" ]]; then
-    rm -rf "${BIN}/${TMP_FOLDER}.*"
-  fi
+    if [[ -n "${BIN:-}" ]] && [[ -e "${BIN}/${TMP_FOLDER}" ]]; then
+      rm -rf "${BIN}/${TMP_FOLDER}.*"
+    fi
   echo "Done."
+  fi
 }