From 9c76f8d339aec1f6ba0b4465e3d5f5d04579319f Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Fri, 19 Apr 2024 12:02:33 +0300 Subject: [PATCH] Fixed issues with Heartbeat pipeline (#39045) (#39048) * fixed issues in heartbeat pipeline * pr fixes (cherry picked from commit b4ff53ce2b2d0037faa58d88afa2afbdc82d3033) Co-authored-by: Olga Naydyonock --- .buildkite/heartbeat/heartbeat-pipeline.yml | 114 +++++++++++++++++--- .buildkite/hooks/pre-command | 3 +- .buildkite/hooks/scripts/util.sh | 9 +- 3 files changed, 104 insertions(+), 22 deletions(-) diff --git a/.buildkite/heartbeat/heartbeat-pipeline.yml b/.buildkite/heartbeat/heartbeat-pipeline.yml index 7429d7eef4aa..707645a62899 100644 --- a/.buildkite/heartbeat/heartbeat-pipeline.yml +++ b/.buildkite/heartbeat/heartbeat-pipeline.yml @@ -24,9 +24,12 @@ env: steps: - group: "Heartbeat Mandatory Testing" key: "heartbeat-mandatory-tests" + steps: - - label: ":ubuntu: Unit Tests" - command: "cd heartbeat && mage build unitTest" + - label: ":ubuntu: Heartbeat Unit Tests" + command: | + cd heartbeat + mage build unitTest agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -34,9 +37,14 @@ steps: artifact_paths: - "heartbeat/build/*.xml" - "heartbeat/build/*.json" + notify: + - github_commit_status: + context: "Heartbeat: Ubuntu Unit Tests" - label: ":rhel: Heartbeat Rhel9 Unit Tests" - command: "cd heartbeat && mage build unitTest" + command: | + cd heartbeat + mage build unitTest agents: provider: "gcp" image: "${IMAGE_RHEL9}" @@ -44,8 +52,11 @@ steps: artifact_paths: - "heartbeat/build/*.xml" - "heartbeat/build/*.json" + notify: + - github_commit_status: + context: "Heartbeat: Rhel9 Unit Tests" - - label: ":windows: Heartbeat Win-2016 Unit Test" + - label: ":windows: Heartbeat Win-2016 Unit Tests" key: "windows-2016" command: | Set-Location -Path heartbeat @@ -58,6 +69,9 @@ steps: artifact_paths: - "heartbeat/build/*.xml" - "heartbeat/build/*.json" + notify: + - github_commit_status: + context: "Heartbeat: Win-2016 Unit Tests" - label: ":windows: Heartbeat Win-2022 Unit Test" key: "windows-2022" @@ -72,9 +86,14 @@ steps: artifact_paths: - "heartbeat/build/*.xml" - "heartbeat/build/*.json" + notify: + - github_commit_status: + context: "Heartbeat: Win-2022 Unit Tests" - label: ":ubuntu: Heartbeat Go Integration Tests" - command: "cd heartbeat && mage goIntegTest" + command: | + cd heartbeat + mage goIntegTest agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -82,9 +101,14 @@ steps: artifact_paths: - "heartbeat/build/*.xml" - "heartbeat/build/*.json" + notify: + - github_commit_status: + context: "Heartbeat: Go Integration Tests" - label: ":ubuntu: Heartbeat Python Integration Tests" - command: "cd heartbeat && mage pythonIntegTest" + command: | + cd heartbeat + mage pythonIntegTest agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -92,33 +116,67 @@ steps: artifact_paths: - "heartbeat/build/*.xml" - "heartbeat/build/*.json" + notify: + - github_commit_status: + context: "Heartbeat: Python Integration Tests" - group: "Heartbeat ARM Tests" key: "heartbeat-extended-tests-arm" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ + steps: - label: ":linux: Heartbeat ARM64 Unit Tests" key: "arm-extended" - if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat for arm" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ - command: "cd heartbeat && mage build unitTest" + command: | + cd heartbeat + mage build unitTest agents: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" artifact_paths: "heartbeat/build/*.xml" + notify: + - github_commit_status: + context: "Heartbeat: Ubuntu ARM64 Unit Tests" - group: "Heartbeat Extended Testing MacOS" key: "heartbeat-extended-tests-macos" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ + steps: - label: ":mac: Heartbeat MacOS Unit Tests" key: "macos-extended" - if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ - command: "cd ${BEATS_PROJECT_NAME} && mage build unitTest" + command: | + set -euo pipefail + source .buildkite/scripts/install_macos_tools.sh + cd heartbeat + mage build unitTest agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" - artifact_paths: "heartbeat/build/*.xml" + artifact_paths: + - "metricbeat/build/*.xml" + - "metricbeat/build/*.json" + notify: + - github_commit_status: + context: "Heartbeat: MacOS Unit Tests" + + - label: ":mac: Heartbeat MacOS ARM Unit Tests" + key: "macos-arm-extended" + command: | + set -euo pipefail + source .buildkite/scripts/install_macos_tools.sh + cd heartbeat + mage build unitTest + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_ARM}" + artifact_paths: + - "metricbeat/build/*.xml" + - "metricbeat/build/*.json" + notify: + - github_commit_status: + context: "Heartbeat: MacOS ARM Unit Tests" - group: "Heartbeat Windows Extended Testing" key: "heartbeat-extended-tests-win" @@ -127,7 +185,9 @@ steps: steps: - label: ":windows: Heartbeat Win-2019 Unit Tests" key: "heartbeat-win-extended-2019" - command: "mage -d ${BEATS_PROJECT_NAME} unitTest" + command: | + Set-Location -Path heartbeat + mage build unitTest agents: provider: "gcp" image: "${IMAGE_WIN_2019}" @@ -136,6 +196,9 @@ steps: artifact_paths: - "heartbeat/build/*.xml" - "heartbeat/build/*.json" + notify: + - github_commit_status: + context: "Heartbeat: Win-2019 Unit Tests" - label: ":windows: Heartbeat Win-11 Unit Tests" key: "heartbeat-windows-extended-11" @@ -150,6 +213,9 @@ steps: artifact_paths: - "heartbeat/build/*.xml" - "heartbeat/build/*.json" + notify: + - github_commit_status: + context: "Heartbeat: Win-11 Unit Tests" - label: ":windows: Heartbeat Win-10 Unit Tests" key: "heartbeat-windows-extended-10" @@ -164,23 +230,32 @@ steps: artifact_paths: - "heartbeat/build/*.xml" - "heartbeat/build/*.json" + notify: + - github_commit_status: + context: "Heartbeat: Win-10 Unit Tests" + + - wait: ~ + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + depends_on: "heartbeat-mandatory-tests" - group: "Heartbeat Packaging" key: "heartbeat-packaging" - if: build.env("BUILDKITE_PULL_REQUEST") != "false" - depends_on: - - "heartbeat-mandatory-tests" steps: - label: ":ubuntu: Heartbeat Packaging Linux X86" key: "heartbeat-package-linux-x86" env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64" SNAPSHOT: true - command: "cd heartbeat && mage package" + command: | + cd heartbeat + mage package agents: provider: gcp image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + notify: + - github_commit_status: + context: "Heartbeat: Packaging Ubuntu x86_64" - label: ":linux: Heartbeat Packaging Linux ARM" key: "heartbeat-package-linux-arm" @@ -188,8 +263,13 @@ steps: PLATFORMS: "linux/arm64" PACKAGES: "docker" SNAPSHOT: true - command: "cd heartbeat && mage package" + command: | + cd heartbeat + mage package agents: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" + notify: + - github_commit_status: + context: "Heartbeat: Packaging Ubuntu ARM" diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 7e5feabe74cd..92c349d4d7ee 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -11,7 +11,7 @@ PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/gcp-platform-inges DOCKER_REGISTRY_SECRET_PATH="kv/ci-shared/platform-ingest/docker_registry_prod" GITHUB_TOKEN_VAULT_PATH="kv/ci-shared/platform-ingest/github_token" -if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats" || "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "heartbeat" ]]; then +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats" || "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" ]]; then source .buildkite/env-scripts/env.sh # Images with prefix "platform-ingest-beats-*" has Go pre-setup. @@ -53,6 +53,7 @@ fi ENABLED_BEATS_PIPELINES_SLUGS=( "auditbeat" + "heartbeat" "filebeat" "beats-metricbeat" "beats-packetbeat" diff --git a/.buildkite/hooks/scripts/util.sh b/.buildkite/hooks/scripts/util.sh index 29ffb3899143..8ef932725c6e 100755 --- a/.buildkite/hooks/scripts/util.sh +++ b/.buildkite/hooks/scripts/util.sh @@ -25,10 +25,11 @@ google_cloud_logout_active_account() { } cleanup() { - # TODO: disable for upload pipeline + if [[ "$BUILDKITE_COMMAND" != *"buildkite-agent pipeline upload"* ]]; then echo "Deleting temporary files..." - if [[ -e "${BIN}/${TMP_FOLDER}" ]]; then - rm -rf "${BIN}/${TMP_FOLDER}.*" - fi + if [[ -n "${BIN:-}" ]] && [[ -e "${BIN}/${TMP_FOLDER}" ]]; then + rm -rf "${BIN}/${TMP_FOLDER}.*" + fi echo "Done." + fi }