From befd3ef7fc2705da447c0d09e26ec32394530b7d Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka Date: Thu, 18 Apr 2024 14:50:21 +0300 Subject: [PATCH 1/2] fixed issues in heartbeat pipeline --- .buildkite/heartbeat/heartbeat-pipeline.yml | 54 +++++++++++++++------ 1 file changed, 40 insertions(+), 14 deletions(-) diff --git a/.buildkite/heartbeat/heartbeat-pipeline.yml b/.buildkite/heartbeat/heartbeat-pipeline.yml index 0ab91594c25..f0ac285dd53 100644 --- a/.buildkite/heartbeat/heartbeat-pipeline.yml +++ b/.buildkite/heartbeat/heartbeat-pipeline.yml @@ -24,9 +24,12 @@ env: steps: - group: "Heartbeat Mandatory Testing" key: "heartbeat-mandatory-tests" + steps: - label: ":ubuntu: Unit Tests" - command: "cd heartbeat && mage build unitTest" + command: | + cd heartbeat + mage build unitTest agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -36,7 +39,9 @@ steps: - "heartbeat/build/*.json" - label: ":rhel: Heartbeat Rhel9 Unit Tests" - command: "cd heartbeat && mage build unitTest" + command: | + cd heartbeat + mage build unitTest agents: provider: "gcp" image: "${IMAGE_RHEL9}" @@ -74,7 +79,9 @@ steps: - "heartbeat/build/*.json" - label: ":ubuntu: Heartbeat Go Integration Tests" - command: "cd heartbeat && mage goIntegTest" + command: | + cd heartbeat + mage goIntegTest agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -84,7 +91,9 @@ steps: - "heartbeat/build/*.json" - label: ":ubuntu: Heartbeat Python Integration Tests" - command: "cd heartbeat && mage pythonIntegTest" + command: | + cd heartbeat + mage pythonIntegTest agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -96,11 +105,13 @@ steps: - group: "Heartbeat ARM Tests" key: "heartbeat-extended-tests-arm" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ + steps: - label: ":linux: Heartbeat ARM64 Unit Tests" key: "arm-extended" - if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat for arm" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ - command: "cd heartbeat && mage build unitTest" + command: | + cd heartbeat + mage build unitTest agents: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" @@ -110,11 +121,13 @@ steps: - group: "Heartbeat Extended Testing MacOS" key: "heartbeat-extended-tests-macos" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ + steps: - label: ":mac: Heartbeat MacOS Unit Tests" key: "macos-extended" - if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ - command: "cd ${BEATS_PROJECT_NAME} && mage build unitTest" + command: | + cd heartbeat + mage build unitTest agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" @@ -127,7 +140,9 @@ steps: steps: - label: ":windows: Heartbeat Win-2019 Unit Tests" key: "heartbeat-win-extended-2019" - command: "mage -d ${BEATS_PROJECT_NAME} unitTest" + command: | + Set-Location -Path heartbeat + mage build unitTest agents: provider: "gcp" image: "${IMAGE_WIN_2019}" @@ -165,22 +180,28 @@ steps: - "heartbeat/build/*.xml" - "heartbeat/build/*.json" + - wait: ~ + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + depends_on: "heartbeat-mandatory-tests" + - group: "Heartbeat Packaging" key: "heartbeat-packaging" - if: build.env("BUILDKITE_PULL_REQUEST") != "false" - depends_on: - - "heartbeat-mandatory-tests" steps: - label: ":ubuntu: Heartbeat Packaging Linux X86" key: "heartbeat-package-linux-x86" env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" SNAPSHOT: true - command: "cd heartbeat && mage package" + command: | + cd heartbeat + mage package agents: provider: gcp image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + notify: + - github_commit_status: + context: "Heartbeat: Packaging Ubuntu x86_64" - label: ":linux: Heartbeat Packaging Linux ARM" key: "heartbeat-package-linux-arm" @@ -188,8 +209,13 @@ steps: PLATFORMS: "linux/arm64" PACKAGES: "docker" SNAPSHOT: true - command: "cd heartbeat && mage package" + command: | + cd heartbeat + mage package agents: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" + notify: + - github_commit_status: + context: "Heartbeat: Packaging Ubuntu ARM" From 229105a2c79ee8b9bc05c3d0fe86fefba3c02b67 Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka Date: Thu, 18 Apr 2024 15:24:12 +0300 Subject: [PATCH 2/2] pr fixes --- .buildkite/heartbeat/heartbeat-pipeline.yml | 60 +++++++++++++++++++-- .buildkite/hooks/pre-command | 3 +- .buildkite/hooks/scripts/util.sh | 9 ++-- 3 files changed, 64 insertions(+), 8 deletions(-) diff --git a/.buildkite/heartbeat/heartbeat-pipeline.yml b/.buildkite/heartbeat/heartbeat-pipeline.yml index f0ac285dd53..8091b2eead1 100644 --- a/.buildkite/heartbeat/heartbeat-pipeline.yml +++ b/.buildkite/heartbeat/heartbeat-pipeline.yml @@ -26,7 +26,7 @@ steps: key: "heartbeat-mandatory-tests" steps: - - label: ":ubuntu: Unit Tests" + - label: ":ubuntu: Heartbeat Unit Tests" command: | cd heartbeat mage build unitTest @@ -37,6 +37,9 @@ steps: artifact_paths: - "heartbeat/build/*.xml" - "heartbeat/build/*.json" + notify: + - github_commit_status: + context: "Heartbeat: Ubuntu Unit Tests" - label: ":rhel: Heartbeat Rhel9 Unit Tests" command: | @@ -49,8 +52,11 @@ steps: artifact_paths: - "heartbeat/build/*.xml" - "heartbeat/build/*.json" + notify: + - github_commit_status: + context: "Heartbeat: Rhel9 Unit Tests" - - label: ":windows: Heartbeat Win-2016 Unit Test" + - label: ":windows: Heartbeat Win-2016 Unit Tests" key: "windows-2016" command: | Set-Location -Path heartbeat @@ -63,6 +69,9 @@ steps: artifact_paths: - "heartbeat/build/*.xml" - "heartbeat/build/*.json" + notify: + - github_commit_status: + context: "Heartbeat: Win-2016 Unit Tests" - label: ":windows: Heartbeat Win-2022 Unit Test" key: "windows-2022" @@ -77,6 +86,9 @@ steps: artifact_paths: - "heartbeat/build/*.xml" - "heartbeat/build/*.json" + notify: + - github_commit_status: + context: "Heartbeat: Win-2022 Unit Tests" - label: ":ubuntu: Heartbeat Go Integration Tests" command: | @@ -89,6 +101,9 @@ steps: artifact_paths: - "heartbeat/build/*.xml" - "heartbeat/build/*.json" + notify: + - github_commit_status: + context: "Heartbeat: Go Integration Tests" - label: ":ubuntu: Heartbeat Python Integration Tests" command: | @@ -101,6 +116,9 @@ steps: artifact_paths: - "heartbeat/build/*.xml" - "heartbeat/build/*.json" + notify: + - github_commit_status: + context: "Heartbeat: Python Integration Tests" - group: "Heartbeat ARM Tests" key: "heartbeat-extended-tests-arm" @@ -117,6 +135,9 @@ steps: imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" artifact_paths: "heartbeat/build/*.xml" + notify: + - github_commit_status: + context: "Heartbeat: Ubuntu ARM64 Unit Tests" - group: "Heartbeat Extended Testing MacOS" key: "heartbeat-extended-tests-macos" @@ -126,12 +147,36 @@ steps: - label: ":mac: Heartbeat MacOS Unit Tests" key: "macos-extended" command: | + set -euo pipefail + source .buildkite/scripts/install_macos_tools.sh cd heartbeat mage build unitTest agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" - artifact_paths: "heartbeat/build/*.xml" + artifact_paths: + - "metricbeat/build/*.xml" + - "metricbeat/build/*.json" + notify: + - github_commit_status: + context: "Heartbeat: MacOS Unit Tests" + + - label: ":mac: Heartbeat MacOS ARM Unit Tests" + key: "macos-arm-extended" + command: | + set -euo pipefail + source .buildkite/scripts/install_macos_tools.sh + cd heartbeat + mage build unitTest + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_ARM}" + artifact_paths: + - "metricbeat/build/*.xml" + - "metricbeat/build/*.json" + notify: + - github_commit_status: + context: "Heartbeat: MacOS ARM Unit Tests" - group: "Heartbeat Windows Extended Testing" key: "heartbeat-extended-tests-win" @@ -151,6 +196,9 @@ steps: artifact_paths: - "heartbeat/build/*.xml" - "heartbeat/build/*.json" + notify: + - github_commit_status: + context: "Heartbeat: Win-2019 Unit Tests" - label: ":windows: Heartbeat Win-11 Unit Tests" key: "heartbeat-windows-extended-11" @@ -165,6 +213,9 @@ steps: artifact_paths: - "heartbeat/build/*.xml" - "heartbeat/build/*.json" + notify: + - github_commit_status: + context: "Heartbeat: Win-11 Unit Tests" - label: ":windows: Heartbeat Win-10 Unit Tests" key: "heartbeat-windows-extended-10" @@ -179,6 +230,9 @@ steps: artifact_paths: - "heartbeat/build/*.xml" - "heartbeat/build/*.json" + notify: + - github_commit_status: + context: "Heartbeat: Win-10 Unit Tests" - wait: ~ if: build.env("BUILDKITE_PULL_REQUEST") != "false" diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index c6a37908e23..8fc0e914278 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -11,7 +11,7 @@ PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/gcp-platform-inges DOCKER_REGISTRY_SECRET_PATH="kv/ci-shared/platform-ingest/docker_registry_prod" GITHUB_TOKEN_VAULT_PATH="kv/ci-shared/platform-ingest/github_token" -if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats" || "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "heartbeat" ]]; then +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats" || "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" ]]; then source .buildkite/env-scripts/env.sh if [[ -z "${GO_VERSION-""}" ]]; then export GO_VERSION=$(cat "${WORKSPACE}/.go-version") @@ -56,6 +56,7 @@ fi ENABLED_BEATS_PIPELINES_SLUGS=( "auditbeat" + "heartbeat" "filebeat" "beats-metricbeat" "beats-packetbeat" diff --git a/.buildkite/hooks/scripts/util.sh b/.buildkite/hooks/scripts/util.sh index 29ffb389914..8ef932725c6 100755 --- a/.buildkite/hooks/scripts/util.sh +++ b/.buildkite/hooks/scripts/util.sh @@ -25,10 +25,11 @@ google_cloud_logout_active_account() { } cleanup() { - # TODO: disable for upload pipeline + if [[ "$BUILDKITE_COMMAND" != *"buildkite-agent pipeline upload"* ]]; then echo "Deleting temporary files..." - if [[ -e "${BIN}/${TMP_FOLDER}" ]]; then - rm -rf "${BIN}/${TMP_FOLDER}.*" - fi + if [[ -n "${BIN:-}" ]] && [[ -e "${BIN}/${TMP_FOLDER}" ]]; then + rm -rf "${BIN}/${TMP_FOLDER}.*" + fi echo "Done." + fi }