From 6b214bee9ab599fa71d091e572406384f5254072 Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka Date: Mon, 27 May 2024 13:18:12 +0300 Subject: [PATCH 01/12] updated namings and enabled tests --- .buildkite/x-pack/pipeline.xpack.metricbeat.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml index 693b10c86837..df89c721c5e5 100644 --- a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml @@ -257,8 +257,8 @@ steps: context: "x-pack/metricbeat: macOS x86_64 Unit Tests" - label: ":mac: x-pack/metricbeat: macOS arm64 Unit Tests" - skip: "Skipping due to elastic/beats#33036 & elastic/ingest-dev#3400" - # https://github.com/elastic/beats/issues/33036 https://github.com/elastic/ingest-dev/issues/3400 +# skip: "Skipping due to elastic/beats#33036 & elastic/ingest-dev#3400" +# # https://github.com/elastic/beats/issues/33036 https://github.com/elastic/ingest-dev/issues/3400 if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ command: | set -euo pipefail From b92dc261bc435a9e76236f1619f0a83d149e9ad6 Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka Date: Mon, 27 May 2024 16:21:09 +0300 Subject: [PATCH 02/12] disabled tests and updated skip info --- .buildkite/x-pack/pipeline.xpack.metricbeat.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml index df89c721c5e5..693b10c86837 100644 --- a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml @@ -257,8 +257,8 @@ steps: context: "x-pack/metricbeat: macOS x86_64 Unit Tests" - label: ":mac: x-pack/metricbeat: macOS arm64 Unit Tests" -# skip: "Skipping due to elastic/beats#33036 & elastic/ingest-dev#3400" -# # https://github.com/elastic/beats/issues/33036 https://github.com/elastic/ingest-dev/issues/3400 + skip: "Skipping due to elastic/beats#33036 & elastic/ingest-dev#3400" + # https://github.com/elastic/beats/issues/33036 https://github.com/elastic/ingest-dev/issues/3400 if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ command: | set -euo pipefail From 8fe8e963bc6aec2316708deb8d7846ae9cbe234b Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka Date: Tue, 18 Jun 2024 13:15:54 +0300 Subject: [PATCH 03/12] updated env vars --- .buildkite/auditbeat/auditbeat-pipeline.yml | 37 ++++++++++++-- .../deploy/kubernetes/deploy-k8s-pipeline.yml | 3 +- .../kubernetes/scripts/setup-k8s-env.sh | 1 + .buildkite/filebeat/filebeat-pipeline.yml | 31 ++++++++++- .buildkite/heartbeat/heartbeat-pipeline.yml | 34 ++++++++++++- .buildkite/libbeat/pipeline.libbeat.yml | 9 ++++ .buildkite/scripts/packaging/package-util.sh | 22 -------- .buildkite/scripts/packaging/package.sh | 51 ------------------- .buildkite/scripts/setenv.sh | 18 +++---- 9 files changed, 115 insertions(+), 91 deletions(-) delete mode 100755 .buildkite/scripts/packaging/package-util.sh delete mode 100755 .buildkite/scripts/packaging/package.sh diff --git a/.buildkite/auditbeat/auditbeat-pipeline.yml b/.buildkite/auditbeat/auditbeat-pipeline.yml index e7444e338c94..3fc7975579df 100644 --- a/.buildkite/auditbeat/auditbeat-pipeline.yml +++ b/.buildkite/auditbeat/auditbeat-pipeline.yml @@ -19,6 +19,10 @@ env: IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" + ARCH_UNIX_X86_64: "-arch amd64" + ARCH_MINGW_X86_64: "-arch 386" + ARCH_UNIX_ARM_64: "-arch arm" + # Other deps ASDF_MAGE_VERSION: 1.15.0 @@ -60,6 +64,8 @@ steps: steps: - label: ":ubuntu: Auditbeat: Ubuntu x86_64 Unit Tests" + env: + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | set -euo pipefail cd auditbeat @@ -79,6 +85,8 @@ steps: context: "auditbeat: Ubuntu x86_64 Unit Tests" - label: ":rhel: Auditbeat: RHEL9 Unit Tests" + env: + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | set -euo pipefail cd auditbeat @@ -98,6 +106,8 @@ steps: context: "auditbeat: RHEL9 Unit Tests" - label: ":windows: Auditbeat: Win 2016 Unit Tests" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path auditbeat mage build unitTest @@ -118,6 +128,8 @@ steps: context: "auditbeat: Win 2016 Unit Tests" - label: ":windows: Auditbeat: Win 2022 Unit Tests" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path auditbeat mage build unitTest @@ -138,10 +150,10 @@ steps: context: "auditbeat: Win 2022 Unit Tests" - label: ":ubuntu: Auditbeat: Crosscompile" + env: + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | make -C auditbeat crosscompile - env: - GOX_FLAGS: "-arch amd64" retry: automatic: - limit: 3 @@ -161,6 +173,8 @@ steps: - label: ":ubuntu: Auditbeat: Ubuntu x86_64 Integration Tests" key: "auditbeat-extended-integ-tests" if: build.env("GITHUB_PR_LABELS") =~ /.*integrations.*/ + env: + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | set -euo pipefail cd auditbeat @@ -182,6 +196,8 @@ steps: - label: ":ubuntu: Auditbeat: Ubuntu arm64 Integration Tests" key: "auditbeat-extended-arm64-integ-tests" if: build.env("GITHUB_PR_LABELS") =~ /.*integrations.*/ + env: + GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | set -euo pipefail cd auditbeat @@ -203,6 +219,8 @@ steps: - label: ":ubuntu: Auditbeat: Ubuntu arm64 Unit Tests" key: "auditbeat-extended-arm64-unit-tests" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ + env: + GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | set -euo pipefail cd auditbeat @@ -223,6 +241,8 @@ steps: - label: ":mac: Auditbeat: macOS x86_64 Unit Tests" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ + env: + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh @@ -243,6 +263,8 @@ steps: - label: ":mac: Auditbeat: macOS arm64 Unit Tests" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ + env: + GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh @@ -268,6 +290,8 @@ steps: steps: - label: ":windows: Auditbeat: Win 2019 Unit Tests" key: "auditbeat-extended-win-2019-unit-tests" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path auditbeat mage build unitTest @@ -289,6 +313,8 @@ steps: - label: ":windows: Auditbeat: Win 10 Unit Tests" key: "auditbeat-extended-win-10-unit-tests" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path auditbeat mage build unitTest @@ -310,6 +336,8 @@ steps: - label: ":windows: Auditbeat: Win 11 Unit Tests" key: "auditbeat-extended-win-11-unit-tests" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path auditbeat mage build unitTest @@ -338,12 +366,13 @@ steps: - group: "Packaging" key: "auditbeat-packaging" + steps: - label: ":ubuntu: Auditbeat: Packaging Linux" key: "auditbeat-package-linux-x86" env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - SNAPSHOT: true + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | set -euo pipefail cd auditbeat @@ -365,7 +394,7 @@ steps: env: PLATFORMS: "linux/arm64" PACKAGES: "docker" - SNAPSHOT: true + GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | set -euo pipefail cd auditbeat diff --git a/.buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml b/.buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml index c0005309457e..b7aa72dc0710 100644 --- a/.buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml +++ b/.buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml @@ -4,6 +4,7 @@ env: IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" MODULE: "kubernetes" + GOX_FLAGS: "-arch amd64" # Other deps ASDF_KIND_VERSION: "0.20.0" @@ -51,7 +52,7 @@ steps: - label: "K8S Test/K8S version: v1.28.0" key: "k8s-test-128" env: - K8S_VERSION: "v1.28.0" + K8S_VERSION: "v1.29.0" MODULE: "${MODULE}" commands: | set -euo pipefail diff --git a/.buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh b/.buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh index d5d8c690891a..012bf92f9184 100755 --- a/.buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh +++ b/.buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh @@ -2,6 +2,7 @@ set -euo pipefail +WORKSPACE=$(pwd) export KUBECONFIG="${WORKSPACE}/kubecfg" export BIN="${WORKSPACE}/bin" diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml index 053dc24e080b..86b5deb37305 100644 --- a/.buildkite/filebeat/filebeat-pipeline.yml +++ b/.buildkite/filebeat/filebeat-pipeline.yml @@ -18,6 +18,10 @@ env: IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" + ARCH_UNIX_X86_64: "-arch amd64" + ARCH_MINGW_X86_64: "-arch 386" + ARCH_UNIX_ARM_64: "-arch arm" + #Deps ASDF_MAGE_VERSION: 1.15.0 # Integration Tests @@ -60,6 +64,8 @@ steps: key: "filebeat-mandatory-tests" steps: - label: ":ubuntu: Filebeat: Ubuntu x86_64 Unit Tests" + env: + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd filebeat mage build unitTest @@ -78,6 +84,8 @@ steps: context: "filebeat: Ubuntu x86_64 Unit Tests" - label: ":ubuntu: Filebeat: Go Integration Tests" + env: + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd filebeat mage goIntegTest @@ -96,6 +104,8 @@ steps: context: "filebeat: Go Integration Tests" - label: ":ubuntu: Filebeat: Python Integration Tests" + env: + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd filebeat mage pythonIntegTest @@ -115,6 +125,8 @@ steps: - label: ":windows: Filebeat: Win 2016 Unit Tests" key: "windows-2016-unit-tests" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path filebeat mage build unitTest @@ -136,6 +148,8 @@ steps: - label: ":windows: Filebeat: Win 2022 Unit Tests" key: "windows-2022-unit-tests" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path filebeat mage build unitTest @@ -163,6 +177,8 @@ steps: - label: ":mac: Filebeat: macOS x86_64 Unit Tests" key: "macos-unit-tests-extended" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ + env: + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh @@ -184,6 +200,8 @@ steps: - label: ":mac: Filebeat: macOS arm64 Unit Tests" key: "macos-arm64-unit-tests-extended" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ + env: + GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh @@ -205,6 +223,8 @@ steps: - label: ":ubuntu: Filebeat: Ubuntu arm64 Unit Tests" key: "extended-arm64-unit-test" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ + env: + GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | cd filebeat mage build unitTest @@ -229,6 +249,8 @@ steps: steps: - label: ":windows: Filebeat: Win 2019 Unit Tests" key: "windows-extended-2019" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path filebeat mage build unitTest @@ -250,6 +272,8 @@ steps: - label: ":windows: Filebeat: Win 11 Unit Tests" key: "windows-extended-11" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path filebeat mage build unitTest @@ -271,6 +295,8 @@ steps: - label: ":windows: Filebeat: Win 10 Unit Tests" key: "windows-extended-10" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path filebeat mage build unitTest @@ -302,6 +328,9 @@ steps: steps: - label: ":ubuntu: Filebeat: Packaging Linux" key: "packaging-linux" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd filebeat mage package @@ -315,8 +344,6 @@ steps: machineType: "${GCP_HI_PERF_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" - env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" notify: - github_commit_status: context: "filebeat: Packaging Linux" diff --git a/.buildkite/heartbeat/heartbeat-pipeline.yml b/.buildkite/heartbeat/heartbeat-pipeline.yml index 8da785028589..7831f3d42df1 100644 --- a/.buildkite/heartbeat/heartbeat-pipeline.yml +++ b/.buildkite/heartbeat/heartbeat-pipeline.yml @@ -18,8 +18,13 @@ env: IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" + ARCH_UNIX_X86_64: "-arch amd64" + ARCH_MINGW_X86_64: "-arch 386" + ARCH_UNIX_ARM_64: "-arch arm" + # Other deps ASDF_MAGE_VERSION: 1.15.0 + ASDF_NODEJS_VERSION: 18.17.1 # Unit tests RACE_DETECTOR: "true" @@ -59,6 +64,8 @@ steps: steps: - label: ":ubuntu: Heartbeat: Ubuntu x86_64 Unit Tests" + env: + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd heartbeat mage build unitTest @@ -77,6 +84,8 @@ steps: context: "heartbeat: Ubuntu x86_64 Unit Tests" - label: ":rhel: Heartbeat: RHEL9 Unit Tests" + env: + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd heartbeat mage build unitTest @@ -96,6 +105,8 @@ steps: - label: ":windows: Heartbeat: Win 2016 Unit Tests" key: "windows-2016" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path heartbeat mage build unitTest @@ -116,6 +127,8 @@ steps: - label: ":windows: Heartbeat: Win 2022 Unit Test" key: "windows-2022" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path heartbeat mage build unitTest @@ -135,6 +148,8 @@ steps: context: "heartbeat: Win 2022 Unit Tests" - label: ":ubuntu: Heartbeat: Go Integration Tests" + env: + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | set -euo pipefail echo "~~~ Installing @elastic/synthetics" @@ -162,6 +177,8 @@ steps: context: "heartbeat: Go Integration Tests" - label: ":ubuntu: Heartbeat: Python Integration Tests" + env: + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | set -euo pipefail # defines the MODULE env var based on what's changed in a PR @@ -193,6 +210,8 @@ steps: - label: ":ubuntu: Heartbeat: Ubuntu arm64 Unit Tests" key: "arm-extended" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ + env: + GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | cd heartbeat mage build unitTest @@ -211,6 +230,8 @@ steps: - label: ":mac: Heartbeat: macOS x86_64 Unit Tests" key: "macos-extended" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ + env: + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh @@ -232,6 +253,8 @@ steps: - label: ":mac: Heartbeat: macOS arm64 Unit Tests" key: "macos-arm-extended" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ + env: + GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh @@ -257,6 +280,8 @@ steps: steps: - label: ":windows: Heartbeat: Win 2019 Unit Tests" key: "heartbeat-win-extended-2019" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path heartbeat mage build unitTest @@ -277,6 +302,8 @@ steps: - label: ":windows: Heartbeat: Win 11 Unit Tests" key: "heartbeat-windows-extended-11" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path heartbeat mage build unitTest @@ -297,6 +324,8 @@ steps: - label: ":windows: Heartbeat: Win 10 Unit Tests" key: "heartbeat-windows-extended-10" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path heartbeat mage build unitTest @@ -324,12 +353,13 @@ steps: - group: "Packaging" key: "heartbeat-packaging" + steps: - label: ":ubuntu: Heartbeat: Packaging Linux" key: "heartbeat-package-linux" env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - SNAPSHOT: true + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd heartbeat mage package @@ -350,7 +380,7 @@ steps: env: PLATFORMS: "linux/arm64" PACKAGES: "docker" - SNAPSHOT: true + GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | cd heartbeat mage package diff --git a/.buildkite/libbeat/pipeline.libbeat.yml b/.buildkite/libbeat/pipeline.libbeat.yml index f48370b0653d..baf190467562 100644 --- a/.buildkite/libbeat/pipeline.libbeat.yml +++ b/.buildkite/libbeat/pipeline.libbeat.yml @@ -9,6 +9,9 @@ env: GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + ARCH_UNIX_X86_64: "-arch amd64" + ARCH_UNIX_ARM_64: "-arch arm" + #Deps ASDF_MAGE_VERSION: 1.15.0 @@ -19,6 +22,7 @@ env: steps: - group: "Check/Update" key: "libbeat-check-update" + steps: - label: "Libbeat: Run check/update" command: | @@ -46,9 +50,12 @@ steps: - group: "Mandatory Tests" key: "mandatory-tests" + steps: - label: ":ubuntu: Libbeat: Ubuntu x86_64 Unit Tests" key: "mandatory-linux-unit-test" + env: + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | set -euo pipefail cd libbeat @@ -69,6 +76,8 @@ steps: - label: ":ubuntu: Libbeat: Go Integration Tests" key: "mandatory-int-test" + env: + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | set -euo pipefail cd libbeat diff --git a/.buildkite/scripts/packaging/package-util.sh b/.buildkite/scripts/packaging/package-util.sh deleted file mode 100755 index 4a50457cc9c5..000000000000 --- a/.buildkite/scripts/packaging/package-util.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -is_pr() { - if [[ $BUILDKITE_PULL_REQUEST != false ]]; then - return 0 - else - return 1 - fi -} - -define_tags() { - aliasVersion="${VERSION%.*}${IMG_POSTFIX}" - tags=("${BUILDKITE_COMMIT}") - - if is_pr; then - tags+=("pr-${GITHUB_PR_NUMBER}") - else - tags+=("${SOURCE_TAG}" "${aliasVersion}") - fi -} diff --git a/.buildkite/scripts/packaging/package.sh b/.buildkite/scripts/packaging/package.sh deleted file mode 100755 index 5744ee0776b2..000000000000 --- a/.buildkite/scripts/packaging/package.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -source .buildkite/scripts/packaging/package-util.sh - -IMG_POSTFIX="-SNAPSHOT" -VARIANTS=("" "-ubi" "-oss") -VERSION="$(make get-version)" -SOURCE_TAG+="${VERSION}${IMG_POSTFIX}" -TARGET="observability-ci/${BEATS_PROJECT_NAME}" - -echo "--- Creating package" -mage -d "${BEATS_PROJECT_NAME}" package - -echo "--- Distribution list" -dir="${BEATS_PROJECT_NAME}/build/distributions" -buildkite-agent artifact upload "$dir/*.tar.gz;$dir/*.tar.gz.sha512" - -echo "--- Docker image list" -docker images - -define_tags - -targetSuffix="" -if [[ ${HW_TYPE} == "aarch64" || ${HW_TYPE} == "arm64" ]]; then - targetSuffix="-arm64" -fi - -for variant in "${VARIANTS[@]}"; do - source="beats/${BEATS_PROJECT_NAME}${variant}" - - for tag in "${tags[@]}"; do - targetTag=$tag${targetSuffix} - - sourceName="${DOCKER_REGISTRY}/${source}:${SOURCE_TAG}" - targetName="${DOCKER_REGISTRY}/${TARGET}:${targetTag}" - #TODO Remove following line once beats fully migrated to Buildkite and Jenkins builds will be disabled - #Avoid clashing with the Jenkins produced images - targetName="${targetName}-buildkite" - - if docker image inspect "${sourceName}" &>/dev/null; then - echo "--- Tag & Push with target: $targetName" - echo "Source name: $sourceName" - docker tag "$sourceName" "$targetName" - docker push "$targetName" - else - echo "Docker image ${sourceName} does not exist" - fi - done -done diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh index 56f8d7257d60..4e5f4da7d5ea 100755 --- a/.buildkite/scripts/setenv.sh +++ b/.buildkite/scripts/setenv.sh @@ -5,17 +5,17 @@ set -euo pipefail WORKSPACE=${WORKSPACE:-"$(pwd)"} GO_VERSION=$(cat .go-version) -export REPO="beats" -export DOCKER_REGISTRY="docker.elastic.co" -export SETUP_GVM_VERSION="v0.5.1" -export DOCKER_COMPOSE_VERSION="1.21.0" -export DOCKER_COMPOSE_VERSION_AARCH64="v2.21.0" +#export REPO="beats" +#export DOCKER_REGISTRY="docker.elastic.co" +#export SETUP_GVM_VERSION="v0.5.1" +#export DOCKER_COMPOSE_VERSION="1.21.0" +#export DOCKER_COMPOSE_VERSION_AARCH64="v2.21.0" -export ASDF_NODEJS_VERSION="18.17.1" -export AWS_REGION="eu-central-1" +#export ASDF_NODEJS_VERSION="18.17.1" +#export AWS_REGION="eu-central-1" -export WORKSPACE -export GO_VERSION +#export WORKSPACE +#export GO_VERSION exportVars() { local platform_type="$(uname)" From 3548ddb991971e4ebc444d9e81b5ad183435f648 Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka Date: Thu, 20 Jun 2024 10:55:30 +0300 Subject: [PATCH 04/12] added GOX_FLAGS env vars --- .buildkite/auditbeat/auditbeat-pipeline.yml | 46 ++++++++-------- .../deploy/kubernetes/deploy-k8s-pipeline.yml | 2 +- .buildkite/filebeat/filebeat-pipeline.yml | 54 +++++++++---------- .buildkite/heartbeat/heartbeat-pipeline.yml | 42 +++++++-------- .buildkite/libbeat/pipeline.libbeat.yml | 12 ++++- .buildkite/metricbeat/pipeline.yml | 38 +++++++++++-- .buildkite/packetbeat/pipeline.packetbeat.yml | 36 +++++++++++-- .buildkite/scripts/setenv.sh | 12 ++--- .buildkite/winlogbeat/pipeline.winlogbeat.yml | 20 ++++++- .../x-pack/pipeline.xpack.auditbeat.yml | 36 +++++++++++-- .../x-pack/pipeline.xpack.dockerlogbeat.yml | 11 +++- .buildkite/x-pack/pipeline.xpack.filebeat.yml | 51 ++++++++++++++---- .../x-pack/pipeline.xpack.heartbeat.yml | 34 ++++++++++-- .buildkite/x-pack/pipeline.xpack.libbeat.yml | 22 ++++++++ .../x-pack/pipeline.xpack.metricbeat.yml | 49 +++++++++++++---- .../x-pack/pipeline.xpack.osquerybeat.yml | 27 +++++++++- .../x-pack/pipeline.xpack.packetbeat.yml | 42 +++++++++++++-- .../x-pack/pipeline.xpack.winlogbeat.yml | 20 ++++++- 18 files changed, 418 insertions(+), 136 deletions(-) diff --git a/.buildkite/auditbeat/auditbeat-pipeline.yml b/.buildkite/auditbeat/auditbeat-pipeline.yml index 3fc7975579df..29fcfadfa589 100644 --- a/.buildkite/auditbeat/auditbeat-pipeline.yml +++ b/.buildkite/auditbeat/auditbeat-pipeline.yml @@ -31,11 +31,11 @@ env: TEST_COVERAGE: "true" steps: - - group: "Check/Update" + - group: "Auditbeat Check/Update" key: "auditbeat-check-update" steps: - - label: "Auditbeat: Run check/update" + - label: "Run check/update" command: | set -eo pipefail make -C auditbeat check update @@ -59,7 +59,7 @@ steps: if: build.env("BUILDKITE_PULL_REQUEST") != "false" depends_on: "auditbeat-check-update" - - group: "Mandatory Testing" + - group: "Auditbeat Mandatory Testing" key: "auditbeat-mandatory-tests" steps: @@ -84,7 +84,7 @@ steps: - github_commit_status: context: "auditbeat: Ubuntu x86_64 Unit Tests" - - label: ":rhel: Auditbeat: RHEL9 Unit Tests" + - label: ":rhel: Auditbeat Unit Tests" env: GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | @@ -103,9 +103,9 @@ steps: - "auditbeat/build/*.json" notify: - github_commit_status: - context: "auditbeat: RHEL9 Unit Tests" + context: "auditbeat: RHEL Unit Tests" - - label: ":windows: Auditbeat: Win 2016 Unit Tests" + - label: ":windows: Auditbeat Win-2016 Unit Tests" env: GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | @@ -125,9 +125,9 @@ steps: - "auditbeat/build/*.json" notify: - github_commit_status: - context: "auditbeat: Win 2016 Unit Tests" + context: "auditbeat: Windows 2016 Unit Tests" - - label: ":windows: Auditbeat: Win 2022 Unit Tests" + - label: ":windows: Auditbeat Win-2022 Unit Tests" env: GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | @@ -147,9 +147,9 @@ steps: - "auditbeat/build/*.json" notify: - github_commit_status: - context: "auditbeat: Win 2022 Unit Tests" + context: "auditbeat: Windows 2022 Unit Tests" - - label: ":ubuntu: Auditbeat: Crosscompile" + - label: ":linux: Auditbeat Crosscompile" env: GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | @@ -165,7 +165,7 @@ steps: - github_commit_status: context: "auditbeat: Crosscompile" - - group: "Extended Tests" + - group: "Auditbeat Extended Tests" key: "auditbeat-extended-tests" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*(macOS|arm|integrations).*/ @@ -239,7 +239,7 @@ steps: - github_commit_status: context: "auditbeat: Ubuntu arm64 Unit Tests" - - label: ":mac: Auditbeat: macOS x86_64 Unit Tests" + - label: ":mac: Auditbeat macOS x86_64 Unit Tests" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ env: GOX_FLAGS: "${ARCH_UNIX_X86_64}" @@ -261,7 +261,7 @@ steps: - github_commit_status: context: "auditbeat: macOS x86_64 Unit Tests" - - label: ":mac: Auditbeat: macOS arm64 Unit Tests" + - label: ":mac: Auditbeat macOS arm64 ARM Unit Tests" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ env: GOX_FLAGS: "${ARCH_UNIX_ARM_64}" @@ -283,12 +283,12 @@ steps: - github_commit_status: context: "auditbeat: macOS arm64 Unit Tests" - - group: "Extended Windows Tests" + - group: "Auditbeat Windows Extended Testing" key: "auditbeat-extended-tests-win" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/ steps: - - label: ":windows: Auditbeat: Win 2019 Unit Tests" + - label: ":windows: Auditbeat Win-2019 Unit Tests" key: "auditbeat-extended-win-2019-unit-tests" env: GOX_FLAGS: "${ARCH_MINGW_X86_64}" @@ -309,9 +309,9 @@ steps: - "auditbeat/build/*.json" notify: - github_commit_status: - context: "auditbeat: Win 2019 Unit Tests" + context: "auditbeat: Extended Windows 2019 Unit Tests" - - label: ":windows: Auditbeat: Win 10 Unit Tests" + - label: ":windows: Auditbeat Win-10 Unit Tests" key: "auditbeat-extended-win-10-unit-tests" env: GOX_FLAGS: "${ARCH_MINGW_X86_64}" @@ -332,9 +332,9 @@ steps: - "auditbeat/build/*.json" notify: - github_commit_status: - context: "auditbeat: Win 10 Unit Tests" + context: "auditbeat: Extended Windows 10 Unit Tests" - - label: ":windows: Auditbeat: Win 11 Unit Tests" + - label: ":windows: Auditbeat Win-11 Unit Tests" key: "auditbeat-extended-win-11-unit-tests" env: GOX_FLAGS: "${ARCH_MINGW_X86_64}" @@ -355,7 +355,7 @@ steps: - "auditbeat/build/*.json" notify: - github_commit_status: - context: "auditbeat: Win 11 Unit Tests" + context: "auditbeat: Extended Windows 11 Unit Tests" - wait: ~ # with PRs, we want to run packaging only if mandatory tests succeed @@ -364,11 +364,11 @@ steps: if: build.env("BUILDKITE_PULL_REQUEST") != "false" depends_on: "auditbeat-mandatory-tests" - - group: "Packaging" + - group: "Auditbeat Packaging" key: "auditbeat-packaging" steps: - - label: ":ubuntu: Auditbeat: Packaging Linux" + - label: ":ubuntu: Auditbeat/Packaging Linux" key: "auditbeat-package-linux-x86" env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" @@ -389,7 +389,7 @@ steps: - github_commit_status: context: "auditbeat: Packaging Linux" - - label: ":ubuntu: Auditbeat: Packaging Linux arm64" + - label: ":linux: Auditbeat/Packaging Linux arm64" key: "auditbeat-package-linux-arm64" env: PLATFORMS: "linux/arm64" diff --git a/.buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml b/.buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml index b7aa72dc0710..b31a8facd848 100644 --- a/.buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml +++ b/.buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml @@ -52,7 +52,7 @@ steps: - label: "K8S Test/K8S version: v1.28.0" key: "k8s-test-128" env: - K8S_VERSION: "v1.29.0" + K8S_VERSION: "v1.28.0" MODULE: "${MODULE}" commands: | set -euo pipefail diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml index 86b5deb37305..02571d7d53a2 100644 --- a/.buildkite/filebeat/filebeat-pipeline.yml +++ b/.buildkite/filebeat/filebeat-pipeline.yml @@ -4,11 +4,9 @@ name: "beats-filebeat" env: AWS_ARM_INSTANCE_TYPE: "m6g.xlarge" AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" - GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" - IMAGE_MACOS_ARM: "generic-13-ventura-arm" IMAGE_MACOS_X86_64: "generic-13-ventura-x64" IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" @@ -33,10 +31,10 @@ env: TEST_COVERAGE: "true" steps: - - group: "Check/Update" + - group: "Filebeat Check/Update" key: "filebeat-check-update" steps: - - label: "Filebeat: Run check/update" + - label: "Run check/update" command: | set -eo pipefail make -C filebeat check update @@ -60,7 +58,7 @@ steps: if: build.env("BUILDKITE_PULL_REQUEST") != "false" depends_on: "filebeat-check-update" - - group: "Mandatory Tests" + - group: "Filebeat Mandatory Tests" key: "filebeat-mandatory-tests" steps: - label: ":ubuntu: Filebeat: Ubuntu x86_64 Unit Tests" @@ -83,7 +81,7 @@ steps: - github_commit_status: context: "filebeat: Ubuntu x86_64 Unit Tests" - - label: ":ubuntu: Filebeat: Go Integration Tests" + - label: ":ubuntu: Filebeat Go Integration Tests" env: GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | @@ -101,9 +99,9 @@ steps: - "filebeat/build/*.json" notify: - github_commit_status: - context: "filebeat: Go Integration Tests" + context: "filebeat: Filebeat Go Integration Tests" - - label: ":ubuntu: Filebeat: Python Integration Tests" + - label: ":ubuntu: Filebeat Python Integration Tests" env: GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | @@ -123,7 +121,7 @@ steps: - github_commit_status: context: "filebeat: Python Integration Tests" - - label: ":windows: Filebeat: Win 2016 Unit Tests" + - label: ":windows: Filebeat Windows 2016 Unit Tests" key: "windows-2016-unit-tests" env: GOX_FLAGS: "${ARCH_MINGW_X86_64}" @@ -144,9 +142,9 @@ steps: - "filebeat/build/*.json" notify: - github_commit_status: - context: "filebeat: Win 2016 Unit Tests" + context: "filebeat: Windows 2016 Unit Tests" - - label: ":windows: Filebeat: Win 2022 Unit Tests" + - label: ":windows: Filebeat Windows 2022 Unit Tests" key: "windows-2022-unit-tests" env: GOX_FLAGS: "${ARCH_MINGW_X86_64}" @@ -167,14 +165,14 @@ steps: - "filebeat/build/*.json" notify: - github_commit_status: - context: "filebeat: Win 2022 Unit Tests" + context: "filebeat: Windows 2022 Unit Tests" - - group: "Extended Tests" + - group: "Filebeat Extended Tests" key: "filebeat-extended-tests" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*(macOS|arm).*/ steps: - - label: ":mac: Filebeat: macOS x86_64 Unit Tests" + - label: ":mac: Filebeat macOS x86_64 Unit Tests" key: "macos-unit-tests-extended" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ env: @@ -197,7 +195,7 @@ steps: - github_commit_status: context: "filebeat: macOS x86_64 Unit Tests" - - label: ":mac: Filebeat: macOS arm64 Unit Tests" + - label: ":mac: Filebeat macOS arm64 Unit Tests" key: "macos-arm64-unit-tests-extended" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ env: @@ -242,12 +240,12 @@ steps: - github_commit_status: context: "filebeat: Ubuntu arm64 Unit Tests" - - group: "Extended Windows Tests" + - group: "Filebeat Extended Windows Tests" key: "filebeat-extended-win-tests" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/ steps: - - label: ":windows: Filebeat: Win 2019 Unit Tests" + - label: ":windows: Filebeat Windows 2019 Unit Tests" key: "windows-extended-2019" env: GOX_FLAGS: "${ARCH_MINGW_X86_64}" @@ -268,9 +266,9 @@ steps: - "filebeat/build/*.json" notify: - github_commit_status: - context: "filebeat: Win 2019 Unit Tests" + context: "filebeat: Windows 2019 Unit Tests" - - label: ":windows: Filebeat: Win 11 Unit Tests" + - label: ":windows: Filebeat Windows 11 Unit Tests" key: "windows-extended-11" env: GOX_FLAGS: "${ARCH_MINGW_X86_64}" @@ -291,9 +289,9 @@ steps: - "filebeat/build/*.json" notify: - github_commit_status: - context: "filebeat: Win 11 Unit Tests" + context: "filebeat: Windows 11 Unit Tests" - - label: ":windows: Filebeat: Win 10 Unit Tests" + - label: ":windows: Filebeat Windows 10 Unit Tests" key: "windows-extended-10" env: GOX_FLAGS: "${ARCH_MINGW_X86_64}" @@ -314,7 +312,7 @@ steps: - "filebeat/build/*.json" notify: - github_commit_status: - context: "filebeat: Win 10 Unit Tests" + context: "filebeat: Windows 10 Unit Tests" - wait: ~ # with PRs, we want to run packaging only if mandatory tests succeed @@ -323,10 +321,10 @@ steps: if: build.env("BUILDKITE_PULL_REQUEST") != "false" depends_on: "filebeat-mandatory-tests" - - group: "Packaging" + - group: "Filebeat Packaging" key: "packaging" steps: - - label: ":ubuntu: Filebeat: Packaging Linux" + - label: ":linux: Filebeat Packaging Linux" key: "packaging-linux" env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" @@ -348,8 +346,11 @@ steps: - github_commit_status: context: "filebeat: Packaging Linux" - - label: ":ubuntu: Filebeat: Packaging arm64" + - label: ":linux: Filebeat Packaging arm64" key: "packaging-arm" + env: + PLATFORMS: "linux/arm64" + PACKAGES: "docker" command: | cd filebeat mage package @@ -361,9 +362,6 @@ steps: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" - env: - PLATFORMS: "linux/arm64" - PACKAGES: "docker" notify: - github_commit_status: context: "filebeat: Packaging Linux arm64" diff --git a/.buildkite/heartbeat/heartbeat-pipeline.yml b/.buildkite/heartbeat/heartbeat-pipeline.yml index 7831f3d42df1..bb54409bfc5c 100644 --- a/.buildkite/heartbeat/heartbeat-pipeline.yml +++ b/.buildkite/heartbeat/heartbeat-pipeline.yml @@ -83,7 +83,7 @@ steps: - github_commit_status: context: "heartbeat: Ubuntu x86_64 Unit Tests" - - label: ":rhel: Heartbeat: RHEL9 Unit Tests" + - label: ":rhel: Heartbeat Rhel9 Unit Tests" env: GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | @@ -101,9 +101,9 @@ steps: - "heartbeat/build/*.json" notify: - github_commit_status: - context: "heartbeat: RHEL9 Unit Tests" + context: "heartbeat: Rhel9 Unit Tests" - - label: ":windows: Heartbeat: Win 2016 Unit Tests" + - label: ":windows: Win-2016 Unit Tests" key: "windows-2016" env: GOX_FLAGS: "${ARCH_MINGW_X86_64}" @@ -123,9 +123,9 @@ steps: - "heartbeat/build/*.json" notify: - github_commit_status: - context: "heartbeat: Win 2016 Unit Tests" + context: "heartbeat: Win-2016 Unit Tests" - - label: ":windows: Heartbeat: Win 2022 Unit Test" + - label: ":windows: Heartbeat: Win-2022 Unit Test" key: "windows-2022" env: GOX_FLAGS: "${ARCH_MINGW_X86_64}" @@ -145,9 +145,9 @@ steps: - "heartbeat/build/*.json" notify: - github_commit_status: - context: "heartbeat: Win 2022 Unit Tests" + context: "heartbeat: Win-2022 Unit Tests" - - label: ":ubuntu: Heartbeat: Go Integration Tests" + - label: ":ubuntu: Heartbeat Go (Module) Integration Tests" env: GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | @@ -176,7 +176,7 @@ steps: - github_commit_status: context: "heartbeat: Go Integration Tests" - - label: ":ubuntu: Heartbeat: Python Integration Tests" + - label: ":ubuntu: Heartbeat Python (Module) Integration Tests" env: GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | @@ -202,7 +202,7 @@ steps: - github_commit_status: context: "heartbeat: Python Integration Tests" - - group: "Extended Tests" + - group: "Heartbeat Extended Tests" key: "heartbeat-extended-tests" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*(macOS|arm).*/ @@ -227,7 +227,7 @@ steps: - github_commit_status: context: "heartbeat: Ubuntu arm64 Unit Tests" - - label: ":mac: Heartbeat: macOS x86_64 Unit Tests" + - label: ":mac: macOS x86_64 Unit Tests" key: "macos-extended" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ env: @@ -250,7 +250,7 @@ steps: - github_commit_status: context: "heartbeat: macOS x86_64 Unit Tests" - - label: ":mac: Heartbeat: macOS arm64 Unit Tests" + - label: ":mac: macOS arm64 Unit Tests" key: "macos-arm-extended" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ env: @@ -273,12 +273,12 @@ steps: - github_commit_status: context: "heartbeat: macOS arm64 Unit Tests" - - group: "Extended Windows Tests" + - group: "Heartbeat Windows Extended Testing" key: "heartbeat-extended-tests-win" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/ steps: - - label: ":windows: Heartbeat: Win 2019 Unit Tests" + - label: ":windows: Win-2019 Unit Tests" key: "heartbeat-win-extended-2019" env: GOX_FLAGS: "${ARCH_MINGW_X86_64}" @@ -298,9 +298,9 @@ steps: - "heartbeat/build/*.json" notify: - github_commit_status: - context: "heartbeat: Win 2019 Unit Tests" + context: "heartbeat: Win-2019 Unit Tests" - - label: ":windows: Heartbeat: Win 11 Unit Tests" + - label: ":windows: Win-11 Unit Tests" key: "heartbeat-windows-extended-11" env: GOX_FLAGS: "${ARCH_MINGW_X86_64}" @@ -320,9 +320,9 @@ steps: - "heartbeat/build/*.json" notify: - github_commit_status: - context: "heartbeat: Win 11 Unit Tests" + context: "heartbeat: Win-11 Unit Tests" - - label: ":windows: Heartbeat: Win 10 Unit Tests" + - label: ":windows: Win-10 Unit Tests" key: "heartbeat-windows-extended-10" env: GOX_FLAGS: "${ARCH_MINGW_X86_64}" @@ -342,7 +342,7 @@ steps: - "heartbeat/build/*.json" notify: - github_commit_status: - context: "heartbeat: Win 10 Unit Tests" + context: "heartbeat: Win-10 Unit Tests" - wait: ~ # with PRs, we want to run packaging only if mandatory tests succeed @@ -351,11 +351,11 @@ steps: if: build.env("BUILDKITE_PULL_REQUEST") != "false" depends_on: "heartbeat-mandatory-tests" - - group: "Packaging" + - group: "Heartbeat Packaging" key: "heartbeat-packaging" steps: - - label: ":ubuntu: Heartbeat: Packaging Linux" + - label: ":ubuntu: Heartbeat Packaging Linux" key: "heartbeat-package-linux" env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" @@ -375,7 +375,7 @@ steps: - github_commit_status: context: "heartbeat: Packaging Linux" - - label: ":ubuntu: Heartbeat: Packaging Linux arm64" + - label: ":linux: Heartbeat Packaging Linux arm64" key: "heartbeat-package-linux-arm" env: PLATFORMS: "linux/arm64" diff --git a/.buildkite/libbeat/pipeline.libbeat.yml b/.buildkite/libbeat/pipeline.libbeat.yml index baf190467562..99f7b0ed221a 100644 --- a/.buildkite/libbeat/pipeline.libbeat.yml +++ b/.buildkite/libbeat/pipeline.libbeat.yml @@ -20,11 +20,11 @@ env: TEST_COVERAGE: "true" steps: - - group: "Check/Update" + - group: "libbeat Check/Update" key: "libbeat-check-update" steps: - - label: "Libbeat: Run check/update" + - label: "Run check/update" command: | set -eo pipefail make -C libbeat check update @@ -98,6 +98,8 @@ steps: - label: ":ubuntu: Libbeat: Python Integration Tests" key: "mandatory-python-int-test" + env: + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | set -euo pipefail cd libbeat @@ -118,6 +120,8 @@ steps: - label: ":ubuntu: Libbeat: Crosscompile" key: "mandatory-cross-compile" + env: + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | set -euo pipefail cd libbeat @@ -138,6 +142,8 @@ steps: - label: ":ubuntu: Libbeat: Stress Tests" key: "mandatory-stress-test" + env: + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | set -euo pipefail cd libbeat @@ -161,6 +167,8 @@ steps: steps: - label: ":ubuntu: Libbeat: Ubuntu arm64 Unit Tests" key: "extended-arm64-unit-tests" + env: + GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | set -euo pipefail cd libbeat diff --git a/.buildkite/metricbeat/pipeline.yml b/.buildkite/metricbeat/pipeline.yml index 9090867c5637..5378c2c4218d 100644 --- a/.buildkite/metricbeat/pipeline.yml +++ b/.buildkite/metricbeat/pipeline.yml @@ -18,6 +18,10 @@ env: IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" + ARCH_UNIX_X86_64: "-arch amd64" + ARCH_MINGW_X86_64: "-arch 386" + ARCH_UNIX_ARM_64: "-arch arm" + # Integration Tests K8S_VERSION: "v1.29.0" ASDF_KIND_VERSION: "0.20.0" @@ -63,6 +67,8 @@ steps: steps: - label: ":ubuntu: Metricbeat: Ubuntu x86_64 Unit Tests" key: "mandatory-linux-unit-test" + env: + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd metricbeat mage build unitTest @@ -82,6 +88,8 @@ steps: - label: ":ubuntu: Metricbeat: Go Integration Tests" key: "mandatory-int-test" + env: + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | set -euo pipefail # defines the MODULE env var based on what's changed in a PR @@ -108,6 +116,8 @@ steps: - label: ":ubuntu: Metricbeat: Python Integration Tests" key: "mandatory-python-int-test" + env: + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | set -euo pipefail # defines the MODULE env var based on what's changed in a PR @@ -134,6 +144,8 @@ steps: - label: ":ubuntu: Metricbeat: Crosscompile" key: "mandatory-cross-compile" + env: + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: "make -C metricbeat crosscompile" retry: automatic: @@ -150,6 +162,8 @@ steps: context: "metricbeat: Crosscompile" - label: ":windows: Metricbeat: Win 2016 Unit Tests" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path metricbeat mage build unitTest @@ -171,6 +185,8 @@ steps: context: "metricbeat: Win 2016 Unit Tests" - label: ":windows: Metricbeat: Win 2022 Unit Tests" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path metricbeat mage build unitTest @@ -197,6 +213,8 @@ steps: steps: - label: ":windows: Metricbeat: Win 10 Unit Tests" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path metricbeat mage build unitTest @@ -218,6 +236,8 @@ steps: context: "metricbeat: Win 10 Unit Tests" - label: ":windows: Metricbeat: Win 11 Unit Tests" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path metricbeat mage build unitTest @@ -239,6 +259,8 @@ steps: context: "metricbeat: Win 11 Unit Tests" - label: ":windows: Metricbeat: Win 2019 Unit Tests" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path metricbeat mage build unitTest @@ -266,6 +288,8 @@ steps: steps: - label: ":mac: Metricbeat: macOS x64_64 Unit Tests" key: "extended-macos-x64-64-unit-tests" + env: + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh @@ -288,6 +312,8 @@ steps: key: "extended-macos-arm64-unit-tests" skip: "Skipping due to elastic/beats#33035" # https://github.com/elastic/beats/issues/33035 + env: + GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh @@ -318,6 +344,9 @@ steps: steps: - label: ":ubuntu: Metricbeat: Packaging Linux" key: "packaging-linux" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd metricbeat mage package @@ -331,14 +360,16 @@ steps: machineType: "${GCP_HI_PERF_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" - env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" notify: - github_commit_status: context: "metricbeat: Packaging Linux" - label: ":ubuntu: Metricbeat: Packaging Linux arm64" key: "packaging-arm" + env: + PLATFORMS: "linux/arm64" + PACKAGES: "docker" + GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | cd metricbeat mage package @@ -350,9 +381,6 @@ steps: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" - env: - PLATFORMS: "linux/arm64" - PACKAGES: "docker" notify: - github_commit_status: context: "metricbeat: Packaging Linux arm64" diff --git a/.buildkite/packetbeat/pipeline.packetbeat.yml b/.buildkite/packetbeat/pipeline.packetbeat.yml index 91c32e8f27f5..88601c2fe748 100644 --- a/.buildkite/packetbeat/pipeline.packetbeat.yml +++ b/.buildkite/packetbeat/pipeline.packetbeat.yml @@ -19,6 +19,10 @@ env: IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" + ARCH_UNIX_X86_64: "-arch amd64" + ARCH_MINGW_X86_64: "-arch 386" + ARCH_UNIX_ARM_64: "-arch arm" + #Deps ASDF_MAGE_VERSION: 1.15.0 @@ -60,6 +64,8 @@ steps: steps: - label: ":ubuntu: Packetbeat: Ubuntu x86_64 Unit Tests" + env: + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd packetbeat mage build unitTest @@ -78,6 +84,8 @@ steps: context: "packetbeat: Ubuntu x86_64 Unit Tests" - label: ":rhel: Packetbeat: RHEL9 Unit Tests" + env: + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd packetbeat mage build unitTest @@ -96,6 +104,8 @@ steps: context: "packetbeat: RHEL9 Unit Tests" - label: ":windows: Packetbeat: Win 2016 Unit Tests" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path packetbeat mage build unitTest @@ -116,6 +126,8 @@ steps: context: "packetbeat: Win 2016 Unit Tests" - label: ":windows: Packetbeat: Win 2022 Unit Tests" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path packetbeat mage build unitTest @@ -141,6 +153,8 @@ steps: steps: - label: ":windows: Packetbeat: Win 10 Unit Tests" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path packetbeat mage build unitTest @@ -161,6 +175,8 @@ steps: context: "packetbeat: Win 10 Unit Tests" - label: ":windows: Packetbeat: Win 11 Unit Tests" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path packetbeat mage build unitTest @@ -182,6 +198,8 @@ steps: context: "packetbeat: Win 11 Unit Tests" - label: ":windows: Packetbeat: Win 2019 Unit Tests" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path packetbeat mage build unitTest @@ -209,6 +227,8 @@ steps: - label: ":mac: Packetbeat: macOS x86_64 Unit Tests" key: "macos-x86-64-unit-tests-extended" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ + env: + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh @@ -230,6 +250,8 @@ steps: - label: ":mac: Packetbeat: macOS arm64 Unit Tests" key: "macos-arm64-unit-tests-extended" if: build.env("GITHUB_PR_LABELS") =~ /.*(macOS|arm).*/ + env: + GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh @@ -251,6 +273,8 @@ steps: - label: ":ubuntu: Packetbeat: Ubuntu arm64 Unit Tests" key: "linux-arm64-unit-tests-extended" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ + env: + GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | cd packetbeat mage build unitTest @@ -280,6 +304,9 @@ steps: steps: - label: ":ubuntu: Packetbeat: Packaging Linux" key: "packaging-linux" + env: + GOX_FLAGS: "${ARCH_UNIX_X86_64}" + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" command: | cd packetbeat mage package @@ -293,14 +320,16 @@ steps: machineType: "${GCP_HI_PERF_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" - env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" notify: - github_commit_status: context: "packetbeat: Packaging Linux" - label: ":ubuntu: Packetbeat: Packaging Linux arm64" key: "packaging-arm" + env: + PLATFORMS: "linux/arm64" + PACKAGES: "docker" + GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | cd packetbeat mage package @@ -312,9 +341,6 @@ steps: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" - env: - PLATFORMS: "linux/arm64" - PACKAGES: "docker" notify: - github_commit_status: context: "packetbeat: Packaging Linux arm64" diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh index 4e5f4da7d5ea..7a414208bbc8 100755 --- a/.buildkite/scripts/setenv.sh +++ b/.buildkite/scripts/setenv.sh @@ -43,12 +43,12 @@ exportVars() { } if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-winlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-auditbeat" ]]; then - exportVars +# exportVars export TEST_TAGS="${TEST_TAGS:+$TEST_TAGS,}oracle" fi -if [[ "$BUILDKITE_STEP_KEY" == "xpack-winlogbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-metricbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "metricbeat-pipeline" ]]; then - source .buildkite/scripts/common.sh - # Set the MODULE env variable if possible, it should be defined before generating pipeline's steps. It is used in multiple pipelines. - defineModuleFromTheChangeSet "${BEATS_PROJECT_NAME}" -fi +#if [[ "$BUILDKITE_STEP_KEY" == "xpack-winlogbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-metricbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "metricbeat-pipeline" ]]; then +# source .buildkite/scripts/common.sh +# # Set the MODULE env variable if possible, it should be defined before generating pipeline's steps. It is used in multiple pipelines. +# defineModuleFromTheChangeSet "${BEATS_PROJECT_NAME}" +#fi diff --git a/.buildkite/winlogbeat/pipeline.winlogbeat.yml b/.buildkite/winlogbeat/pipeline.winlogbeat.yml index b3122c23168d..c592bc118f66 100644 --- a/.buildkite/winlogbeat/pipeline.winlogbeat.yml +++ b/.buildkite/winlogbeat/pipeline.winlogbeat.yml @@ -13,6 +13,9 @@ env: IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" + ARCH_UNIX_X86_64: "-arch amd64" + ARCH_MINGW_X86_64: "-arch 386" + # Other deps ASDF_MAGE_VERSION: 1.15.0 @@ -57,6 +60,8 @@ steps: steps: - label: ":ubuntu: Winlogbeat: Crossccompile" key: "mandatory-cross-compile" + env: + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: "make -C winlogbeat crosscompile" retry: automatic: @@ -73,6 +78,8 @@ steps: context: "winlogbeat: Crosscompile" - label: ":windows: Winlogbeat: Win 2016 Unit Tests" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path winlogbeat mage build unitTest @@ -94,6 +101,8 @@ steps: context: "winlogbeat: Win 2016 Unit Tests" - label: ":windows: Winlogbeat: Win 2019 Unit Tests" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path winlogbeat mage build unitTest @@ -115,6 +124,8 @@ steps: context: "winlogbeat: Win 2019 Unit Tests" - label: ":windows: Winlogbeat: Win 2022 Unit Tests" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path winlogbeat mage build unitTest @@ -141,6 +152,8 @@ steps: steps: - label: ":windows: Winlogbeat: Win 10 Unit Tests" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path winlogbeat mage build unitTest @@ -162,6 +175,8 @@ steps: context: "winlogbeat: Win 10 Unit Tests" - label: ":windows: Winlogbeat: Win 11 Unit Tests" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path winlogbeat mage build unitTest @@ -194,6 +209,9 @@ steps: steps: - label: ":ubuntu: Winlogbeat: Packaging Linux" key: "packaging-linux" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd winlogbeat mage package @@ -205,8 +223,6 @@ steps: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" notify: - github_commit_status: context: "winlogbeat: Packaging Linux" diff --git a/.buildkite/x-pack/pipeline.xpack.auditbeat.yml b/.buildkite/x-pack/pipeline.xpack.auditbeat.yml index 1c7e675a57ec..6426db5f7752 100644 --- a/.buildkite/x-pack/pipeline.xpack.auditbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.auditbeat.yml @@ -19,6 +19,10 @@ env: IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" + ARCH_UNIX_X86_64: "-arch amd64" + ARCH_MINGW_X86_64: "-arch 386" + ARCH_UNIX_ARM_64: "-arch arm" + # Other deps ASDF_MAGE_VERSION: 1.15.0 @@ -61,6 +65,8 @@ steps: steps: - label: ":ubuntu: x-pack/auditbeat: Build Tests" key: "mandatory-linux-unit-test" + env: + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | set -euo pipefail # defines the MODULE env var based on what's changed in a PR @@ -85,6 +91,8 @@ steps: - label: ":rhel: x-pack/auditbeat: RHEL9 Unit Tests" key: "mandatory-rhel9-unit-test" + env: + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd x-pack/auditbeat mage build unitTest @@ -104,6 +112,8 @@ steps: - label: ":windows: x-pack/auditbeat: Win 2022 Unit Tests" key: "mandatory-win-2022-unit-tests" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/auditbeat mage build unitTest @@ -124,6 +134,8 @@ steps: context: "x-pack/auditbeat: Win 2022 Unit Tests" - label: ":windows: x-pack/auditbeat: Win 2016 Unit Tests" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/auditbeat mage build unitTest @@ -150,6 +162,8 @@ steps: steps: - label: ":windows: x-pack/auditbeat: Win 2019 Unit Tests" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/auditbeat mage build unitTest @@ -171,6 +185,8 @@ steps: context: "x-pack/auditbeat: Win 2019 Unit Tests" - label: ":windows: x-pack/auditbeat: Win 10 Unit Tests" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/auditbeat mage build unitTest @@ -192,6 +208,8 @@ steps: context: "x-pack/auditbeat: Win 10 Unit Tests" - label: ":windows: x-pack/auditbeat: Win 11 Unit Tests" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/auditbeat mage build unitTest @@ -219,6 +237,8 @@ steps: steps: - label: ":mac: x-pack/auditbeat: macOS x86_64 Unit Tests" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ + env: + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh @@ -239,6 +259,8 @@ steps: - label: ":mac: x-pack/auditbeat: macOS arm64 Unit Tests" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ + env: + GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh @@ -259,6 +281,8 @@ steps: - label: ":ubuntu: x-pack/auditbeat: Ubuntu arm64 Unit Tests" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ + env: + GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | cd x-pack/auditbeat mage build unitTest @@ -290,6 +314,9 @@ steps: steps: - label: ":ubuntu: x-pack/auditbeat: Packaging Linux" key: "packaging-linux" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd x-pack/auditbeat mage package @@ -303,14 +330,16 @@ steps: machineType: "${GCP_HI_PERF_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" - env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" notify: - github_commit_status: context: "x-pack/auditbeat: Packaging Linux" - label: ":ubuntu: x-pack/auditbeat: Packaging Linux arm64" key: "packaging-arm" + env: + PLATFORMS: "linux/arm64" + PACKAGES: "docker" + GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | cd x-pack/auditbeat mage package @@ -322,9 +351,6 @@ steps: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" - env: - PLATFORMS: "linux/arm64" - PACKAGES: "docker" notify: - github_commit_status: context: "x-pack/auditbeat: Packaging Linux arm64" diff --git a/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml b/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml index 526785e61ace..166fcde893ce 100644 --- a/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml @@ -10,6 +10,9 @@ env: IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + ARCH_UNIX_X86_64: "-arch amd64" + ARCH_UNIX_ARM_64: "-arch arm" + # Other deps ASDF_MAGE_VERSION: 1.15.0 @@ -51,6 +54,8 @@ steps: steps: - label: ":ubuntu: x-pack/dockerlogbeat: Ubuntu x86_64 Unit Tests" key: "mandatory-linux-unit-test" + env: + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd x-pack/dockerlogbeat mage build unitTest @@ -70,6 +75,8 @@ steps: - label: ":ubuntu: x-pack/dockerlogbeat: Go Integration Tests" key: "mandatory-int-test" + env: + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | set -euo pipefail # defines the MODULE env var based on what's changed in a PR @@ -108,7 +115,7 @@ steps: key: "auditbeat-package-linux-x86" env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - SNAPSHOT: true + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd x-pack/dockerlogbeat mage package @@ -129,7 +136,7 @@ steps: env: PLATFORMS: "linux/arm64" PACKAGES: "docker" - SNAPSHOT: true + GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | cd x-pack/dockerlogbeat mage package diff --git a/.buildkite/x-pack/pipeline.xpack.filebeat.yml b/.buildkite/x-pack/pipeline.xpack.filebeat.yml index a641390727c9..29fda52a09fc 100644 --- a/.buildkite/x-pack/pipeline.xpack.filebeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.filebeat.yml @@ -18,6 +18,10 @@ env: IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" + ARCH_UNIX_X86_64: "-arch amd64" + ARCH_MINGW_X86_64: "-arch 386" + ARCH_UNIX_ARM_64: "-arch arm" + # Other deps ASDF_MAGE_VERSION: 1.15.0 @@ -61,6 +65,8 @@ steps: steps: - label: ":ubuntu: x-pack/filebeat: Ubuntu x86_64 Unit Tests" key: "x-pack-filebeat-mandatory-linux-unit-test" + env: + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd x-pack/filebeat mage build unitTest @@ -80,6 +86,8 @@ steps: - label: ":ubuntu: x-pack/filebeat: Go Integration Tests" key: "x-pack-filebeat-mandatory-int-test" + env: + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd x-pack/filebeat mage goIntegTest @@ -99,6 +107,8 @@ steps: - label: ":ubuntu: x-pack/filebeat: Python Integration Tests" key: "x-pack-filebeat-mandatory-python-int-test" + env: + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd x-pack/filebeat mage pythonIntegTest @@ -117,6 +127,8 @@ steps: context: "x-pack/filebeat: Python Integration Tests" - label: ":windows: x-pack/filebeat: Win 2022 Unit Tests" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/filebeat mage build unitTest @@ -138,6 +150,8 @@ steps: context: "x-pack/filebeat: Win 2022 Unit Tests" - label: ":windows: x-pack/filebeat: Win 2016 Unit Tests" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/filebeat mage build unitTest @@ -164,6 +178,8 @@ steps: steps: - label: ":windows: x-pack/filebeat: Win 2019 Unit Tests" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/filebeat mage build unitTest @@ -185,6 +201,8 @@ steps: context: "x-pack/filebeat: Win 2019 Unit Tests" - label: ":windows: x-pack/filebeat: Win 10 Unit Tests" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/filebeat mage build unitTest @@ -206,6 +224,8 @@ steps: context: "x-pack/filebeat: Win 10 Unit Tests" - label: ":windows: x-pack/filebeat: Win 11 Unit Tests" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/filebeat mage build unitTest @@ -233,6 +253,8 @@ steps: - label: ":ubuntu: x-pack/filebeat: Ubuntu arm64 Unit Tests" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ key: "x-pack-filebeat-mandatory-linux-arm-unit-test" + env: + GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | cd x-pack/filebeat mage build unitTest @@ -252,6 +274,8 @@ steps: - label: ":mac: x-pack/filebeat: macOS x86_64 Unit Tests" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ + env: + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh @@ -272,6 +296,8 @@ steps: - label: ":mac: x-pack/filebeat: macOS arm64 Unit Tests" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ + env: + GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh @@ -295,18 +321,19 @@ steps: skip: "skipping as it was on Jenkins: elastic/ingest-dev#3467" # Related issue: https://github.com/elastic/ingest-dev/issues/3467 if: build.env("GITHUB_PR_LABELS") =~ /.*aws.*/ - command: | - set -euo pipefail - source .buildkite/scripts/initCloudEnv.sh - echo "~~~ Running tests" - cd x-pack/filebeat - mage build test goIntegTest env: ASDF_TERRAFORM_VERSION: 1.0.2 MODULE_DIR: "x-pack/filebeat/input/awss3/_meta/terraform" + GOX_FLAGS: "${ARCH_UNIX_ARM_64}" MODULE: "aws" # TEST_TAGS should be reviewed and updated: https://github.com/elastic/ingest-dev/issues/3476 TEST_TAGS: "aws" + command: | + set -euo pipefail + source .buildkite/scripts/initCloudEnv.sh + echo "~~~ Running tests" + cd x-pack/filebeat + mage build test goIntegTest agents: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" @@ -332,6 +359,9 @@ steps: steps: - label: ":linux: x-pack/filebeat: Packaging Linux" key: "packaging-linux" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd x-pack/filebeat mage package @@ -345,14 +375,16 @@ steps: machineType: "${GCP_HI_PERF_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" - env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" notify: - github_commit_status: context: "x-pack/filebeat: Packaging Linux" - label: ":linux: x-pack/filebeat: Packaging arm64" key: "packaging-arm" + env: + PLATFORMS: "linux/arm64" + PACKAGES: "docker" + GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | cd x-pack/filebeat mage package @@ -364,9 +396,6 @@ steps: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" - env: - PLATFORMS: "linux/arm64" - PACKAGES: "docker" notify: - github_commit_status: context: "x-pack/filebeat: Packaging Linux arm64" diff --git a/.buildkite/x-pack/pipeline.xpack.heartbeat.yml b/.buildkite/x-pack/pipeline.xpack.heartbeat.yml index 1227b8e7570e..4bf38d5c0883 100644 --- a/.buildkite/x-pack/pipeline.xpack.heartbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.heartbeat.yml @@ -19,6 +19,10 @@ env: IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" + ARCH_UNIX_X86_64: "-arch amd64" + ARCH_MINGW_X86_64: "-arch 386" + ARCH_UNIX_ARM_64: "-arch arm" + # pipeline specific ELASTIC_SYNTHETICS_CAPABLE: "true" @@ -65,6 +69,8 @@ steps: steps: - label: ":ubuntu: x-pack/heartbeat: Ubuntu x86_64 Unit Tests" key: "mandatory-linux-unit-test" + env: + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | set -euo pipefail echo "~~~ Installing @elastic/synthetics" @@ -88,6 +94,8 @@ steps: - label: ":ubuntu: x-pack/heartbeat: Go Integration Tests" key: "mandatory-int-test" + env: + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | set -euo pipefail echo "~~~ Installing @elastic/synthetics" @@ -112,6 +120,8 @@ steps: - label: ":windows: x-pack/heartbeat: Win 2016 Unit Tests" key: "mandatory-win-2016-unit-tests" skip: "skipping due to elastic/beats#23957 and elastic/beats#23958" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/heartbeat mage build test @@ -135,6 +145,8 @@ steps: - label: ":windows: x-pack/heartbeat: Win 2022 Unit Tests" key: "mandatory-win-2022-unit-tests" skip: "skipping due to elastic/beats#23957 and elastic/beats#23958" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/heartbeat mage build unitTest @@ -161,6 +173,8 @@ steps: steps: - label: ":windows: x-pack/heartbeat: Win 10 Unit Tests" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/heartbeat mage build test @@ -183,6 +197,8 @@ steps: # Doesn't exist in Jenkins - label: ":windows: x-pack/heartbeat: Win 11 Unit Tests" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/heartbeat mage build test @@ -204,6 +220,8 @@ steps: context: "x-pack/heartbeat: Win 11 Unit Tests" - label: ":windows: x-pack/heartbeat: Win 2019 Unit Tests" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/heartbeat mage build test @@ -230,6 +248,8 @@ steps: steps: - label: ":mac: x-pack/heartbeat: macOS x86_64 Unit Tests" + env: + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh @@ -251,6 +271,8 @@ steps: context: "x-pack/heartbeat: macOS x86_64 Extended Tests" - label: ":mac: x-pack/heartbeat: macOS arm64 Unit Tests" + env: + GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh @@ -286,6 +308,9 @@ steps: steps: - label: ":ubuntu: x-pack/heartbeat: Packaging Linux" key: "packaging-linux" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd x-pack/heartbeat mage package @@ -299,14 +324,16 @@ steps: machineType: "${GCP_HI_PERF_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" - env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" notify: - github_commit_status: context: "x-pack/heartbeat: Packaging Linux" - label: ":ubuntu: x-pack/heartbeat: Packaging Linux arm64" key: "packaging-arm" + env: + PLATFORMS: "linux/arm64" + PACKAGES: "docker" + GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | cd x-pack/heartbeat mage package @@ -318,9 +345,6 @@ steps: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" - env: - PLATFORMS: "linux/arm64" - PACKAGES: "docker" notify: - github_commit_status: context: "x-pack/heartbeat: Packaging Linux arm64" diff --git a/.buildkite/x-pack/pipeline.xpack.libbeat.yml b/.buildkite/x-pack/pipeline.xpack.libbeat.yml index a34c8ac732d0..744195953421 100644 --- a/.buildkite/x-pack/pipeline.xpack.libbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.libbeat.yml @@ -16,6 +16,10 @@ env: GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" + ARCH_UNIX_X86_64: "-arch amd64" + ARCH_MINGW_X86_64: "-arch 386" + ARCH_UNIX_ARM_64: "-arch arm" + #Deps ASDF_MAGE_VERSION: 1.15.0 @@ -57,6 +61,8 @@ steps: steps: - label: ":ubuntu: x-pack/libbeat: Ubuntu x86_64 Unit Tests" key: "mandatory-linux-unit-test" + env: + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd x-pack/libbeat mage build unitTest @@ -76,6 +82,8 @@ steps: - label: ":ubuntu: x-pack/libbeat: Go Integration Tests" key: "mandatory-int-test" + env: + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd x-pack/libbeat mage goIntegTest @@ -95,6 +103,8 @@ steps: - label: ":ubuntu: x-pack/libbeat: Python Integration Tests" key: "mandatory-python-int-test" + env: + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd x-pack/libbeat mage pythonIntegTest @@ -113,6 +123,8 @@ steps: context: "x-pack/libbeat: Python Integration Tests" - label: ":windows: x-pack/libbeat: Win 2016 Unit Tests" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/libbeat mage -w reader\etw build goUnitTest @@ -134,6 +146,8 @@ steps: context: "x-pack/libbeat: Win 2016 Unit Tests" - label: ":windows: x-pack/libbeat: Win 2022 Unit Tests" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/libbeat mage -w reader\etw build goUnitTest @@ -160,6 +174,8 @@ steps: steps: - label: ":windows: x-pack/libbeat: Win 10 Unit Tests" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/libbeat mage -w reader\etw build goUnitTest @@ -181,6 +197,8 @@ steps: context: "x-pack/libbeat: Win 10 Unit Tests" - label: ":windows: x-pack/libbeat: Win 11 Unit Tests" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/libbeat mage -w reader\etw build goUnitTest @@ -202,6 +220,8 @@ steps: context: "x-pack/libbeat: Win 11 Unit Tests" - label: ":windows: x-pack/libbeat: Win 2019 Unit Tests" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/libbeat mage -w reader\etw build goUnitTest @@ -229,6 +249,8 @@ steps: steps: - label: ":ubuntu: x-pack/libbeat: Ubuntu arm64 Unit Tests" key: "extended-arm64-unit-tests" + env: + GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | cd x-pack/libbeat mage build unitTest diff --git a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml index 693b10c86837..93588244a610 100644 --- a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml @@ -18,6 +18,10 @@ env: IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" + ARCH_UNIX_X86_64: "-arch amd64" + ARCH_MINGW_X86_64: "-arch 386" + ARCH_UNIX_ARM_64: "-arch arm" + # Other deps ASDF_MAGE_VERSION: 1.15.0 @@ -59,6 +63,8 @@ steps: steps: - label: ":ubuntu: x-pack/metricbeat: Ubuntu x86_64 Unit Tests" key: "mandatory-linux-unit-test" + env: + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd x-pack/metricbeat mage build unitTest @@ -78,6 +84,8 @@ steps: - label: ":ubuntu: x-pack/metricbeat: Go (MODULE) Integration Tests" key: "mandatory-int-test" + env: + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | set -euo pipefail # defines the MODULE env var based on what's changed in a PR @@ -101,6 +109,8 @@ steps: - label: ":ubuntu: x-pack/metricbeat: Python (MODULE) Integration Tests" key: "mandatory-python-int-test" + env: + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | set -euo pipefail # defines the MODULE env var based on what's changed in a PR @@ -123,6 +133,8 @@ steps: context: "x-pack/metricbeat: Python (MODULE) Integration Tests" - label: ":windows: x-pack/metricbeat: Win 2016 Unit Tests" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/metricbeat mage build unitTest @@ -144,6 +156,8 @@ steps: context: "x-pack/metricbeat: Win 2016 Unit Tests" - label: ":windows: x-pack/metricbeat: Win 2022 Unit Tests" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/metricbeat mage build unitTest @@ -170,6 +184,8 @@ steps: steps: - label: ":windows: x-pack/metricbeat: Win 10 Unit Tests" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/metricbeat mage build unitTest @@ -191,6 +207,8 @@ steps: context: "x-pack/metricbeat: Win 10 Unit Tests" - label: ":windows: x-pack/metricbeat: Win 11 Unit Tests" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/metricbeat mage build unitTest @@ -212,6 +230,8 @@ steps: context: "x-pack/metricbeat: Win 11 Unit Tests" - label: ":windows: x-pack/metricbeat: Win 2019 Unit Tests" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/metricbeat mage build unitTest @@ -239,6 +259,8 @@ steps: steps: - label: ":mac: x-pack/metricbeat: macOS x86_64 Unit Tests" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ + env: + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh @@ -260,6 +282,8 @@ steps: skip: "Skipping due to elastic/beats#33036 & elastic/ingest-dev#3400" # https://github.com/elastic/beats/issues/33036 https://github.com/elastic/ingest-dev/issues/3400 if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ + env: + GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh @@ -280,18 +304,19 @@ steps: - label: ":ubuntu: x-pack/metricbeat: AWS Tests" key: "x-pack-metricbeat-extended-cloud-test" if: build.env("GITHUB_PR_LABELS") =~ /.*aws.*/ - command: | - set -euo pipefail - source .buildkite/scripts/initCloudEnv.sh - echo "~~~ Running tests" - cd x-pack/metricbeat - mage build test goIntegTest env: ASDF_TERRAFORM_VERSION: 1.0.2 MODULE_DIR: "x-pack/metricbeat/module/aws" + GOX_FLAGS: "${ARCH_UNIX_ARM_64}" MODULE: "aws" # TEST_TAGS should be reviewed and updated: https://github.com/elastic/ingest-dev/issues/3476 TEST_TAGS: "aws" + command: | + set -euo pipefail + source .buildkite/scripts/initCloudEnv.sh + echo "~~~ Running tests" + cd x-pack/metricbeat + mage build test goIntegTest agents: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" @@ -317,6 +342,9 @@ steps: steps: - label: ":ubuntu: x-pack/metricbeat: Packaging Linux" key: "packaging-linux" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd x-pack/metricbeat mage package @@ -330,14 +358,16 @@ steps: machineType: "${GCP_HI_PERF_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" - env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" notify: - github_commit_status: context: "x-pack/metricbeat: Packaging Linux" - label: ":ubuntu: x-pack/metricbeat: Packaging linux arm64" key: "packaging-arm" + env: + PLATFORMS: "linux/arm64" + PACKAGES: "docker" + GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | cd x-pack/metricbeat mage package @@ -349,9 +379,6 @@ steps: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" - env: - PLATFORMS: "linux/arm64" - PACKAGES: "docker" notify: - github_commit_status: context: "x-pack/metricbeat: Packaging Linux arm64" diff --git a/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml b/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml index 18ac6cac9a75..5e02cfe89d26 100644 --- a/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml @@ -15,6 +15,10 @@ env: IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" + ARCH_UNIX_X86_64: "-arch amd64" + ARCH_MINGW_X86_64: "-arch 386" + ARCH_UNIX_ARM_64: "-arch arm" + # Other deps ASDF_MAGE_VERSION: 1.15.0 @@ -57,6 +61,8 @@ steps: steps: - label: ":ubuntu: x-pack/osquerybeat: Ubuntu x86_64 Unit Tests" key: "mandatory-linux-unit-test" + env: + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd x-pack/osquerybeat mage build unitTest @@ -76,6 +82,8 @@ steps: - label: ":ubuntu: x-pack/osquerybeat: Go Integration Tests" key: "mandatory-int-test" + env: + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd x-pack/osquerybeat mage goIntegTest @@ -94,6 +102,8 @@ steps: context: "x-pack/osquerybeat: Go Integration Tests" - label: ":windows: x-pack/osquerybeat: Win 2016 Unit Tests" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/osquerybeat mage build unitTest @@ -115,6 +125,8 @@ steps: context: "x-pack/osquerybeat: Win 2016 Unit Tests" - label: ":windows: x-pack/osquerybeat: Win 2022 Unit Tests" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/osquerybeat mage build unitTest @@ -141,6 +153,8 @@ steps: steps: - label: ":windows: x-pack/osquerybeat: Win 10 Unit Tests" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/osquerybeat mage build unitTest @@ -162,6 +176,8 @@ steps: context: "x-pack/osquerybeat: Win 10 Unit Tests" - label: ":windows: x-pack/osquerybeat: Win 11 Unit Tests" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/osquerybeat mage build unitTest @@ -183,6 +199,8 @@ steps: context: "x-pack/osquerybeat: Win 11 Unit Tests" - label: ":windows: x-pack/osquerybeat: Win 2019 Unit Tests" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/osquerybeat mage build unitTest @@ -209,6 +227,8 @@ steps: steps: - label: ":mac: x-pack/osquerybeat: macOS x86_64 Unit Tests" + env: + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh @@ -227,6 +247,8 @@ steps: context: "x-pack/osquerybeat: macOS x86_64 Unit Tests" - label: ":mac: x-pack/osquerybeat: macOS arm64 Unit Tests" + env: + GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh @@ -257,6 +279,9 @@ steps: steps: - label: ":ubuntu: Packaging Linux" key: "packaging-linux" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd x-pack/osquerybeat mage package @@ -270,8 +295,6 @@ steps: machineType: "${GCP_HI_PERF_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" - env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" notify: - github_commit_status: context: "x-pack/osquerybeat: Packaging Linux" diff --git a/.buildkite/x-pack/pipeline.xpack.packetbeat.yml b/.buildkite/x-pack/pipeline.xpack.packetbeat.yml index 55f944352aa5..da700494da35 100644 --- a/.buildkite/x-pack/pipeline.xpack.packetbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.packetbeat.yml @@ -19,6 +19,10 @@ env: IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" + ARCH_UNIX_X86_64: "-arch amd64" + ARCH_MINGW_X86_64: "-arch 386" + ARCH_UNIX_ARM_64: "-arch arm" + #Deps ASDF_MAGE_VERSION: 1.15.0 @@ -61,6 +65,8 @@ steps: steps: - label: ":ubuntu: x-pack/packetbeat: Ubuntu x86_64 Unit Tests" key: "mandatory-linux-unit-test" + env: + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd x-pack/packetbeat mage build unitTest @@ -80,6 +86,8 @@ steps: - label: ":ubuntu: x-pack/packetbeat: Ubuntu x86_64 System Tests" key: "mandatory-linux-system-test" + env: + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd x-pack/packetbeat mage systemTest @@ -99,6 +107,8 @@ steps: - label: ":rhel: x-pack/packetbeat: RHEL9 Unit Tests" key: "mandatory-rhel9-unit-test" + env: + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd x-pack/packetbeat mage build unitTest @@ -117,6 +127,8 @@ steps: context: "x-pack/packetbeat: RHEL9 Unit Tests" - label: ":windows: x-pack/packetbeat: Win 2016 Unit Tests" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/packetbeat mage build unitTest @@ -138,6 +150,8 @@ steps: context: "x-pack/packetbeat: Win 2016 Unit Tests" - label: ":windows: x-pack/packetbeat: Win 2022 Unit Tests" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/packetbeat mage build unitTest @@ -161,6 +175,8 @@ steps: - label: ":windows: x-pack/packetbeat: Win 2022 System Tests" key: "mandatory-win-2022-system-tests" skip: "skipping due to elastic/beats#38142" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/packetbeat mage systemTest @@ -186,6 +202,8 @@ steps: steps: - label: ":windows: x-pack/packetbeat: Win 10 Unit Tests" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/packetbeat mage build unitTest @@ -207,6 +225,8 @@ steps: context: "x-pack/packetbeat: Win 10 Unit Tests" - label: ":windows: x-pack/packetbeat: Win 11 Unit Tests" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/packetbeat mage build unitTest @@ -228,6 +248,8 @@ steps: context: "x-pack/packetbeat: Win 11 Unit Tests" - label: ":windows: x-pack/packetbeat: Win 2019 Unit Tests" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/packetbeat mage build unitTest @@ -251,6 +273,8 @@ steps: - label: ":windows: x-pack/packetbeat: Win 10 System Tests" key: "extended-win-10-system-tests" skip: "skipping due to elastic/beats#38142" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/packetbeat mage systemTest @@ -278,6 +302,8 @@ steps: - label: ":ubuntu: x-pack/packetbeat: Ubuntu arm64 Unit Tests" key: "extended-arm64-unit-test" if: build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ + env: + GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | cd x-pack/packetbeat mage build unitTest @@ -298,6 +324,8 @@ steps: - label: ":mac: x-pack/packetbeat: macOS x86_64 Unit Tests" key: "extended-macos-unit-tests" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ + env: + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh @@ -319,6 +347,8 @@ steps: - label: ":mac: x-pack/packetbeat: macOS arm64 Unit Tests" key: "macos-arm64-unit-tests-extended" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ + env: + GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh @@ -347,6 +377,9 @@ steps: steps: - label: ":ubuntu: x-pack/packetbeat: Packaging Linux" key: "packaging-linux" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd x-pack/packetbeat mage package @@ -360,14 +393,16 @@ steps: machineType: "${GCP_HI_PERF_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" - env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" notify: - github_commit_status: context: "x-pack/packetbeat: Packaging Linux" - label: ":ubuntu: x-pack/packetbeat: Packaging arm64" key: "packaging-arm" + env: + PLATFORMS: "linux/arm64" + PACKAGES: "docker" + GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | cd x-pack/packetbeat mage package @@ -379,9 +414,6 @@ steps: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" - env: - PLATFORMS: "linux/arm64" - PACKAGES: "docker" notify: - github_commit_status: context: "x-pack/packetbeat: Packaging Linux arm64" diff --git a/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml b/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml index fc561ef2faee..c63bfed23fe4 100644 --- a/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml @@ -11,6 +11,9 @@ env: GCP_WIN_MACHINE_TYPE: "n2-standard-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" + ARCH_UNIX_X86_64: "-arch amd64" + ARCH_MINGW_X86_64: "-arch 386" + # Other deps ASDF_MAGE_VERSION: 1.15.0 @@ -52,6 +55,8 @@ steps: steps: - label: ":windows: x-pack/winlogbeat Win 2019 Unit (MODULE) Tests" key: "mandatory-win-2019-module-unit-tests" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Import-Module ./.buildkite/scripts/changesets.psm1 defineModuleFromTheChangeSet 'x-pack/winlogbeat' @@ -75,6 +80,8 @@ steps: context: "x-pack/winlogbeat Win 2019 Unit (MODULE) Tests" - label: ":windows: x-pack/winlogbeat: Win 2016 Unit Tests" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/winlogbeat mage build unitTest @@ -96,6 +103,8 @@ steps: context: "x-pack/winlogbeat: Win 2016 Unit Tests" - label: ":windows: x-pack/winlogbeat: Win 2022 Unit Tests" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/winlogbeat mage build unitTest @@ -122,6 +131,8 @@ steps: steps: - label: ":windows: x-pack/winlogbeat: Win 10 Unit Tests" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/winlogbeat mage build unitTest @@ -143,6 +154,8 @@ steps: context: "x-pack/winlogbeat: Win 10 Unit Tests" - label: ":windows: x-pack/winlogbeat: Win 11 Unit Tests" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/winlogbeat mage build unitTest @@ -164,6 +177,8 @@ steps: context: "x-pack/winlogbeat: Win 11 Unit Tests" - label: ":windows: x-pack/winlogbeat: Win 2019 Unit Tests" + env: + GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/winlogbeat mage build unitTest @@ -198,6 +213,9 @@ steps: steps: - label: ":ubuntu: x-pack/winlogbeat: Packaging Linux" key: "packaging-linux" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: "cd x-pack/winlogbeat && mage package" retry: automatic: @@ -209,8 +227,6 @@ steps: machineType: "${GCP_HI_PERF_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" - env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" notify: - github_commit_status: context: "x-pack/winlogbeat: Packaging Linux" From 982938d1b65789acd1a483d94fcff13ca1727a70 Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka Date: Thu, 20 Jun 2024 14:28:18 +0300 Subject: [PATCH 05/12] added test tags --- .buildkite/filebeat/filebeat-pipeline.yml | 1 + .buildkite/scripts/setenv.sh | 58 +++++++++---------- .../x-pack/pipeline.xpack.metricbeat.yml | 2 + 3 files changed, 32 insertions(+), 29 deletions(-) diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml index 02571d7d53a2..e6b9f64ab7e8 100644 --- a/.buildkite/filebeat/filebeat-pipeline.yml +++ b/.buildkite/filebeat/filebeat-pipeline.yml @@ -22,6 +22,7 @@ env: #Deps ASDF_MAGE_VERSION: 1.15.0 + # Integration Tests K8S_VERSION: "v1.29.0" ASDF_KIND_VERSION: "0.20.0" diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh index 7a414208bbc8..bb759d87664f 100755 --- a/.buildkite/scripts/setenv.sh +++ b/.buildkite/scripts/setenv.sh @@ -17,35 +17,35 @@ GO_VERSION=$(cat .go-version) #export WORKSPACE #export GO_VERSION -exportVars() { - local platform_type="$(uname)" - local arch_type="$(uname -m)" - if [ "${arch_type}" == "x86_64" ]; then - case "${platform_type}" in - Linux|Darwin) - export GOX_FLAGS="-arch amd64" - export testResults="**/build/TEST*.xml" - export artifacts="**/build/TEST*.out" - ;; - MINGW*) - export GOX_FLAGS="-arch 386" - export testResults="**\\build\\TEST*.xml" - export artifacts="**\\build\\TEST*.out" - ;; - esac - elif [[ "${arch_type}" == "aarch64" || "${arch_type}" == "arm64" ]]; then - export GOX_FLAGS="-arch arm" - export testResults="**/build/TEST*.xml" - export artifacts="**/build/TEST*.out" - else - echo "Unsupported OS" - fi -} - -if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-winlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-auditbeat" ]]; then -# exportVars - export TEST_TAGS="${TEST_TAGS:+$TEST_TAGS,}oracle" -fi +#exportVars() { +# local platform_type="$(uname)" +# local arch_type="$(uname -m)" +# if [ "${arch_type}" == "x86_64" ]; then +# case "${platform_type}" in +# Linux|Darwin) +# export GOX_FLAGS="-arch amd64" +# export testResults="**/build/TEST*.xml" +# export artifacts="**/build/TEST*.out" +# ;; +# MINGW*) +# export GOX_FLAGS="-arch 386" +# export testResults="**\\build\\TEST*.xml" +# export artifacts="**\\build\\TEST*.out" +# ;; +# esac +# elif [[ "${arch_type}" == "aarch64" || "${arch_type}" == "arm64" ]]; then +# export GOX_FLAGS="-arch arm" +# export testResults="**/build/TEST*.xml" +# export artifacts="**/build/TEST*.out" +# else +# echo "Unsupported OS" +# fi +#} + +#if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-winlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-auditbeat" ]]; then +## exportVars +# export TEST_TAGS="${TEST_TAGS:+$TEST_TAGS,}oracle" +#fi #if [[ "$BUILDKITE_STEP_KEY" == "xpack-winlogbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-metricbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "metricbeat-pipeline" ]]; then # source .buildkite/scripts/common.sh diff --git a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml index 93588244a610..24d6b3acab6e 100644 --- a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml @@ -86,6 +86,7 @@ steps: key: "mandatory-int-test" env: GOX_FLAGS: "${ARCH_UNIX_X86_64}" + TEST_TAGS: "oracle" command: | set -euo pipefail # defines the MODULE env var based on what's changed in a PR @@ -111,6 +112,7 @@ steps: key: "mandatory-python-int-test" env: GOX_FLAGS: "${ARCH_UNIX_X86_64}" + TEST_TAGS: "oracle" command: | set -euo pipefail # defines the MODULE env var based on what's changed in a PR From 92fd980181a69fc87de578c848e6933d7701b7fa Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka Date: Wed, 26 Jun 2024 16:54:49 +0300 Subject: [PATCH 06/12] deleted setenv.sh --- .buildkite/auditbeat/auditbeat-pipeline.yml | 46 ++++++++-------- .buildkite/filebeat/filebeat-pipeline.yml | 49 +++++++++-------- .buildkite/heartbeat/heartbeat-pipeline.yml | 42 +++++++-------- .buildkite/hooks/pre-command | 21 -------- .buildkite/libbeat/pipeline.libbeat.yml | 4 +- .buildkite/scripts/initCloudEnv.sh | 6 +-- .buildkite/scripts/setenv.sh | 54 ------------------- .../x-pack/pipeline.xpack.metricbeat.yml | 2 +- 8 files changed, 76 insertions(+), 148 deletions(-) delete mode 100755 .buildkite/scripts/setenv.sh diff --git a/.buildkite/auditbeat/auditbeat-pipeline.yml b/.buildkite/auditbeat/auditbeat-pipeline.yml index 29fcfadfa589..3fc7975579df 100644 --- a/.buildkite/auditbeat/auditbeat-pipeline.yml +++ b/.buildkite/auditbeat/auditbeat-pipeline.yml @@ -31,11 +31,11 @@ env: TEST_COVERAGE: "true" steps: - - group: "Auditbeat Check/Update" + - group: "Check/Update" key: "auditbeat-check-update" steps: - - label: "Run check/update" + - label: "Auditbeat: Run check/update" command: | set -eo pipefail make -C auditbeat check update @@ -59,7 +59,7 @@ steps: if: build.env("BUILDKITE_PULL_REQUEST") != "false" depends_on: "auditbeat-check-update" - - group: "Auditbeat Mandatory Testing" + - group: "Mandatory Testing" key: "auditbeat-mandatory-tests" steps: @@ -84,7 +84,7 @@ steps: - github_commit_status: context: "auditbeat: Ubuntu x86_64 Unit Tests" - - label: ":rhel: Auditbeat Unit Tests" + - label: ":rhel: Auditbeat: RHEL9 Unit Tests" env: GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | @@ -103,9 +103,9 @@ steps: - "auditbeat/build/*.json" notify: - github_commit_status: - context: "auditbeat: RHEL Unit Tests" + context: "auditbeat: RHEL9 Unit Tests" - - label: ":windows: Auditbeat Win-2016 Unit Tests" + - label: ":windows: Auditbeat: Win 2016 Unit Tests" env: GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | @@ -125,9 +125,9 @@ steps: - "auditbeat/build/*.json" notify: - github_commit_status: - context: "auditbeat: Windows 2016 Unit Tests" + context: "auditbeat: Win 2016 Unit Tests" - - label: ":windows: Auditbeat Win-2022 Unit Tests" + - label: ":windows: Auditbeat: Win 2022 Unit Tests" env: GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | @@ -147,9 +147,9 @@ steps: - "auditbeat/build/*.json" notify: - github_commit_status: - context: "auditbeat: Windows 2022 Unit Tests" + context: "auditbeat: Win 2022 Unit Tests" - - label: ":linux: Auditbeat Crosscompile" + - label: ":ubuntu: Auditbeat: Crosscompile" env: GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | @@ -165,7 +165,7 @@ steps: - github_commit_status: context: "auditbeat: Crosscompile" - - group: "Auditbeat Extended Tests" + - group: "Extended Tests" key: "auditbeat-extended-tests" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*(macOS|arm|integrations).*/ @@ -239,7 +239,7 @@ steps: - github_commit_status: context: "auditbeat: Ubuntu arm64 Unit Tests" - - label: ":mac: Auditbeat macOS x86_64 Unit Tests" + - label: ":mac: Auditbeat: macOS x86_64 Unit Tests" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ env: GOX_FLAGS: "${ARCH_UNIX_X86_64}" @@ -261,7 +261,7 @@ steps: - github_commit_status: context: "auditbeat: macOS x86_64 Unit Tests" - - label: ":mac: Auditbeat macOS arm64 ARM Unit Tests" + - label: ":mac: Auditbeat: macOS arm64 Unit Tests" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ env: GOX_FLAGS: "${ARCH_UNIX_ARM_64}" @@ -283,12 +283,12 @@ steps: - github_commit_status: context: "auditbeat: macOS arm64 Unit Tests" - - group: "Auditbeat Windows Extended Testing" + - group: "Extended Windows Tests" key: "auditbeat-extended-tests-win" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/ steps: - - label: ":windows: Auditbeat Win-2019 Unit Tests" + - label: ":windows: Auditbeat: Win 2019 Unit Tests" key: "auditbeat-extended-win-2019-unit-tests" env: GOX_FLAGS: "${ARCH_MINGW_X86_64}" @@ -309,9 +309,9 @@ steps: - "auditbeat/build/*.json" notify: - github_commit_status: - context: "auditbeat: Extended Windows 2019 Unit Tests" + context: "auditbeat: Win 2019 Unit Tests" - - label: ":windows: Auditbeat Win-10 Unit Tests" + - label: ":windows: Auditbeat: Win 10 Unit Tests" key: "auditbeat-extended-win-10-unit-tests" env: GOX_FLAGS: "${ARCH_MINGW_X86_64}" @@ -332,9 +332,9 @@ steps: - "auditbeat/build/*.json" notify: - github_commit_status: - context: "auditbeat: Extended Windows 10 Unit Tests" + context: "auditbeat: Win 10 Unit Tests" - - label: ":windows: Auditbeat Win-11 Unit Tests" + - label: ":windows: Auditbeat: Win 11 Unit Tests" key: "auditbeat-extended-win-11-unit-tests" env: GOX_FLAGS: "${ARCH_MINGW_X86_64}" @@ -355,7 +355,7 @@ steps: - "auditbeat/build/*.json" notify: - github_commit_status: - context: "auditbeat: Extended Windows 11 Unit Tests" + context: "auditbeat: Win 11 Unit Tests" - wait: ~ # with PRs, we want to run packaging only if mandatory tests succeed @@ -364,11 +364,11 @@ steps: if: build.env("BUILDKITE_PULL_REQUEST") != "false" depends_on: "auditbeat-mandatory-tests" - - group: "Auditbeat Packaging" + - group: "Packaging" key: "auditbeat-packaging" steps: - - label: ":ubuntu: Auditbeat/Packaging Linux" + - label: ":ubuntu: Auditbeat: Packaging Linux" key: "auditbeat-package-linux-x86" env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" @@ -389,7 +389,7 @@ steps: - github_commit_status: context: "auditbeat: Packaging Linux" - - label: ":linux: Auditbeat/Packaging Linux arm64" + - label: ":ubuntu: Auditbeat: Packaging Linux arm64" key: "auditbeat-package-linux-arm64" env: PLATFORMS: "linux/arm64" diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml index e6b9f64ab7e8..464ab8ba8ff8 100644 --- a/.buildkite/filebeat/filebeat-pipeline.yml +++ b/.buildkite/filebeat/filebeat-pipeline.yml @@ -4,9 +4,11 @@ name: "beats-filebeat" env: AWS_ARM_INSTANCE_TYPE: "m6g.xlarge" AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" + GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" + IMAGE_MACOS_ARM: "generic-13-ventura-arm" IMAGE_MACOS_X86_64: "generic-13-ventura-x64" IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" @@ -32,10 +34,10 @@ env: TEST_COVERAGE: "true" steps: - - group: "Filebeat Check/Update" + - group: "Check/Update" key: "filebeat-check-update" steps: - - label: "Run check/update" + - label: "Filebeat: Run check/update" command: | set -eo pipefail make -C filebeat check update @@ -59,7 +61,7 @@ steps: if: build.env("BUILDKITE_PULL_REQUEST") != "false" depends_on: "filebeat-check-update" - - group: "Filebeat Mandatory Tests" + - group: "Mandatory Tests" key: "filebeat-mandatory-tests" steps: - label: ":ubuntu: Filebeat: Ubuntu x86_64 Unit Tests" @@ -82,7 +84,7 @@ steps: - github_commit_status: context: "filebeat: Ubuntu x86_64 Unit Tests" - - label: ":ubuntu: Filebeat Go Integration Tests" + - label: ":ubuntu: Filebeat: Go Integration Tests" env: GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | @@ -100,9 +102,9 @@ steps: - "filebeat/build/*.json" notify: - github_commit_status: - context: "filebeat: Filebeat Go Integration Tests" + context: "filebeat: Go Integration Tests" - - label: ":ubuntu: Filebeat Python Integration Tests" + - label: ":ubuntu: Filebeat: Python Integration Tests" env: GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | @@ -122,7 +124,7 @@ steps: - github_commit_status: context: "filebeat: Python Integration Tests" - - label: ":windows: Filebeat Windows 2016 Unit Tests" + - label: ":windows: Filebeat: Win 2016 Unit Tests" key: "windows-2016-unit-tests" env: GOX_FLAGS: "${ARCH_MINGW_X86_64}" @@ -143,9 +145,9 @@ steps: - "filebeat/build/*.json" notify: - github_commit_status: - context: "filebeat: Windows 2016 Unit Tests" + context: "filebeat: Win 2016 Unit Tests" - - label: ":windows: Filebeat Windows 2022 Unit Tests" + - label: ":windows: Filebeat: Win 2022 Unit Tests" key: "windows-2022-unit-tests" env: GOX_FLAGS: "${ARCH_MINGW_X86_64}" @@ -166,14 +168,14 @@ steps: - "filebeat/build/*.json" notify: - github_commit_status: - context: "filebeat: Windows 2022 Unit Tests" + context: "filebeat: Win 2022 Unit Tests" - - group: "Filebeat Extended Tests" + - group: "Extended Tests" key: "filebeat-extended-tests" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*(macOS|arm).*/ steps: - - label: ":mac: Filebeat macOS x86_64 Unit Tests" + - label: ":mac: Filebeat: macOS x86_64 Unit Tests" key: "macos-unit-tests-extended" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ env: @@ -196,7 +198,7 @@ steps: - github_commit_status: context: "filebeat: macOS x86_64 Unit Tests" - - label: ":mac: Filebeat macOS arm64 Unit Tests" + - label: ":mac: Filebeat: macOS arm64 Unit Tests" key: "macos-arm64-unit-tests-extended" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ env: @@ -241,12 +243,12 @@ steps: - github_commit_status: context: "filebeat: Ubuntu arm64 Unit Tests" - - group: "Filebeat Extended Windows Tests" + - group: "Extended Windows Tests" key: "filebeat-extended-win-tests" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/ steps: - - label: ":windows: Filebeat Windows 2019 Unit Tests" + - label: ":windows: Filebeat: Win 2019 Unit Tests" key: "windows-extended-2019" env: GOX_FLAGS: "${ARCH_MINGW_X86_64}" @@ -267,9 +269,9 @@ steps: - "filebeat/build/*.json" notify: - github_commit_status: - context: "filebeat: Windows 2019 Unit Tests" + context: "filebeat: Win 2019 Unit Tests" - - label: ":windows: Filebeat Windows 11 Unit Tests" + - label: ":windows: Filebeat: Win 11 Unit Tests" key: "windows-extended-11" env: GOX_FLAGS: "${ARCH_MINGW_X86_64}" @@ -290,9 +292,9 @@ steps: - "filebeat/build/*.json" notify: - github_commit_status: - context: "filebeat: Windows 11 Unit Tests" + context: "filebeat: Win 11 Unit Tests" - - label: ":windows: Filebeat Windows 10 Unit Tests" + - label: ":windows: Filebeat: Win 10 Unit Tests" key: "windows-extended-10" env: GOX_FLAGS: "${ARCH_MINGW_X86_64}" @@ -313,7 +315,7 @@ steps: - "filebeat/build/*.json" notify: - github_commit_status: - context: "filebeat: Windows 10 Unit Tests" + context: "filebeat: Win 10 Unit Tests" - wait: ~ # with PRs, we want to run packaging only if mandatory tests succeed @@ -322,10 +324,10 @@ steps: if: build.env("BUILDKITE_PULL_REQUEST") != "false" depends_on: "filebeat-mandatory-tests" - - group: "Filebeat Packaging" + - group: "Packaging" key: "packaging" steps: - - label: ":linux: Filebeat Packaging Linux" + - label: ":ubuntu: Filebeat: Packaging Linux" key: "packaging-linux" env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" @@ -347,11 +349,12 @@ steps: - github_commit_status: context: "filebeat: Packaging Linux" - - label: ":linux: Filebeat Packaging arm64" + - label: ":ubuntu: Filebeat: Packaging arm64" key: "packaging-arm" env: PLATFORMS: "linux/arm64" PACKAGES: "docker" + GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | cd filebeat mage package diff --git a/.buildkite/heartbeat/heartbeat-pipeline.yml b/.buildkite/heartbeat/heartbeat-pipeline.yml index bb54409bfc5c..7831f3d42df1 100644 --- a/.buildkite/heartbeat/heartbeat-pipeline.yml +++ b/.buildkite/heartbeat/heartbeat-pipeline.yml @@ -83,7 +83,7 @@ steps: - github_commit_status: context: "heartbeat: Ubuntu x86_64 Unit Tests" - - label: ":rhel: Heartbeat Rhel9 Unit Tests" + - label: ":rhel: Heartbeat: RHEL9 Unit Tests" env: GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | @@ -101,9 +101,9 @@ steps: - "heartbeat/build/*.json" notify: - github_commit_status: - context: "heartbeat: Rhel9 Unit Tests" + context: "heartbeat: RHEL9 Unit Tests" - - label: ":windows: Win-2016 Unit Tests" + - label: ":windows: Heartbeat: Win 2016 Unit Tests" key: "windows-2016" env: GOX_FLAGS: "${ARCH_MINGW_X86_64}" @@ -123,9 +123,9 @@ steps: - "heartbeat/build/*.json" notify: - github_commit_status: - context: "heartbeat: Win-2016 Unit Tests" + context: "heartbeat: Win 2016 Unit Tests" - - label: ":windows: Heartbeat: Win-2022 Unit Test" + - label: ":windows: Heartbeat: Win 2022 Unit Test" key: "windows-2022" env: GOX_FLAGS: "${ARCH_MINGW_X86_64}" @@ -145,9 +145,9 @@ steps: - "heartbeat/build/*.json" notify: - github_commit_status: - context: "heartbeat: Win-2022 Unit Tests" + context: "heartbeat: Win 2022 Unit Tests" - - label: ":ubuntu: Heartbeat Go (Module) Integration Tests" + - label: ":ubuntu: Heartbeat: Go Integration Tests" env: GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | @@ -176,7 +176,7 @@ steps: - github_commit_status: context: "heartbeat: Go Integration Tests" - - label: ":ubuntu: Heartbeat Python (Module) Integration Tests" + - label: ":ubuntu: Heartbeat: Python Integration Tests" env: GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | @@ -202,7 +202,7 @@ steps: - github_commit_status: context: "heartbeat: Python Integration Tests" - - group: "Heartbeat Extended Tests" + - group: "Extended Tests" key: "heartbeat-extended-tests" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*(macOS|arm).*/ @@ -227,7 +227,7 @@ steps: - github_commit_status: context: "heartbeat: Ubuntu arm64 Unit Tests" - - label: ":mac: macOS x86_64 Unit Tests" + - label: ":mac: Heartbeat: macOS x86_64 Unit Tests" key: "macos-extended" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ env: @@ -250,7 +250,7 @@ steps: - github_commit_status: context: "heartbeat: macOS x86_64 Unit Tests" - - label: ":mac: macOS arm64 Unit Tests" + - label: ":mac: Heartbeat: macOS arm64 Unit Tests" key: "macos-arm-extended" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ env: @@ -273,12 +273,12 @@ steps: - github_commit_status: context: "heartbeat: macOS arm64 Unit Tests" - - group: "Heartbeat Windows Extended Testing" + - group: "Extended Windows Tests" key: "heartbeat-extended-tests-win" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/ steps: - - label: ":windows: Win-2019 Unit Tests" + - label: ":windows: Heartbeat: Win 2019 Unit Tests" key: "heartbeat-win-extended-2019" env: GOX_FLAGS: "${ARCH_MINGW_X86_64}" @@ -298,9 +298,9 @@ steps: - "heartbeat/build/*.json" notify: - github_commit_status: - context: "heartbeat: Win-2019 Unit Tests" + context: "heartbeat: Win 2019 Unit Tests" - - label: ":windows: Win-11 Unit Tests" + - label: ":windows: Heartbeat: Win 11 Unit Tests" key: "heartbeat-windows-extended-11" env: GOX_FLAGS: "${ARCH_MINGW_X86_64}" @@ -320,9 +320,9 @@ steps: - "heartbeat/build/*.json" notify: - github_commit_status: - context: "heartbeat: Win-11 Unit Tests" + context: "heartbeat: Win 11 Unit Tests" - - label: ":windows: Win-10 Unit Tests" + - label: ":windows: Heartbeat: Win 10 Unit Tests" key: "heartbeat-windows-extended-10" env: GOX_FLAGS: "${ARCH_MINGW_X86_64}" @@ -342,7 +342,7 @@ steps: - "heartbeat/build/*.json" notify: - github_commit_status: - context: "heartbeat: Win-10 Unit Tests" + context: "heartbeat: Win 10 Unit Tests" - wait: ~ # with PRs, we want to run packaging only if mandatory tests succeed @@ -351,11 +351,11 @@ steps: if: build.env("BUILDKITE_PULL_REQUEST") != "false" depends_on: "heartbeat-mandatory-tests" - - group: "Heartbeat Packaging" + - group: "Packaging" key: "heartbeat-packaging" steps: - - label: ":ubuntu: Heartbeat Packaging Linux" + - label: ":ubuntu: Heartbeat: Packaging Linux" key: "heartbeat-package-linux" env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" @@ -375,7 +375,7 @@ steps: - github_commit_status: context: "heartbeat: Packaging Linux" - - label: ":linux: Heartbeat Packaging Linux arm64" + - label: ":ubuntu: Heartbeat: Packaging Linux arm64" key: "heartbeat-package-linux-arm" env: PLATFORMS: "linux/arm64" diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index f860e5fcff75..fdc855d09a7e 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -6,27 +6,6 @@ set -euo pipefail # https://buildkite.com/docs/pipelines/managing-log-output#redacted-environment-variables PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/gcp-platform-ingest-ci-service-account" -ENABLED_BEATS_PIPELINES_SLUGS=( - "auditbeat" - "heartbeat" - "filebeat" - "beats-metricbeat" - "beats-winlogbeat" - "beats-xpack-winlogbeat" - "beats-xpack-dockerlogbeat" - "beats-xpack-auditbeat" - "beats-xpack-metricbeat" - "beats-xpack-heartbeat" - "deploy-k8s" -) - -for slug in "${ENABLED_BEATS_PIPELINES_SLUGS[@]}"; do - if [[ "$BUILDKITE_PIPELINE_SLUG" == "$slug" ]]; then - source .buildkite/scripts/setenv.sh - break - fi -done - if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" && "$BUILDKITE_STEP_KEY" == *"system-tests"* ]]; then PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry -t 5 -- vault kv get -field plaintext -format=json ${PRIVATE_CI_GCS_CREDENTIALS_PATH}) export PRIVATE_CI_GCS_CREDENTIALS_SECRET diff --git a/.buildkite/libbeat/pipeline.libbeat.yml b/.buildkite/libbeat/pipeline.libbeat.yml index 99f7b0ed221a..dcb6d2817f68 100644 --- a/.buildkite/libbeat/pipeline.libbeat.yml +++ b/.buildkite/libbeat/pipeline.libbeat.yml @@ -20,11 +20,11 @@ env: TEST_COVERAGE: "true" steps: - - group: "libbeat Check/Update" + - group: "Check/Update" key: "libbeat-check-update" steps: - - label: "Run check/update" + - label: "Libbeat: Run check/update" command: | set -eo pipefail make -C libbeat check update diff --git a/.buildkite/scripts/initCloudEnv.sh b/.buildkite/scripts/initCloudEnv.sh index ac6cb1857883..e6c15a15966f 100755 --- a/.buildkite/scripts/initCloudEnv.sh +++ b/.buildkite/scripts/initCloudEnv.sh @@ -21,7 +21,7 @@ exportAwsSecrets() { } terraformApply() { - echo "Exporting Terraform Env Vars" + echo "~~~ Exporting Terraform Env Vars" TF_VAR_BRANCH=$(echo "${BUILDKITE_BRANCH}" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9-]/-/g') TF_VAR_CREATED_DATE=$(date +%s) export TF_VAR_BUILD_ID="${BUILDKITE_BUILD_ID}" @@ -30,10 +30,10 @@ terraformApply() { export TF_VAR_BRANCH export TF_VAR_CREATED_DATE - echo "Terraform Init on $MODULE_DIR" + echo "~~~ Terraform Init on $MODULE_DIR" terraform -chdir="$MODULE_DIR" init - echo "Terraform Apply on $MODULE_DIR" + echo "~~~ Terraform Apply on $MODULE_DIR" terraform -chdir="$MODULE_DIR" apply -auto-approve } diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh deleted file mode 100755 index bb759d87664f..000000000000 --- a/.buildkite/scripts/setenv.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -WORKSPACE=${WORKSPACE:-"$(pwd)"} -GO_VERSION=$(cat .go-version) - -#export REPO="beats" -#export DOCKER_REGISTRY="docker.elastic.co" -#export SETUP_GVM_VERSION="v0.5.1" -#export DOCKER_COMPOSE_VERSION="1.21.0" -#export DOCKER_COMPOSE_VERSION_AARCH64="v2.21.0" - -#export ASDF_NODEJS_VERSION="18.17.1" -#export AWS_REGION="eu-central-1" - -#export WORKSPACE -#export GO_VERSION - -#exportVars() { -# local platform_type="$(uname)" -# local arch_type="$(uname -m)" -# if [ "${arch_type}" == "x86_64" ]; then -# case "${platform_type}" in -# Linux|Darwin) -# export GOX_FLAGS="-arch amd64" -# export testResults="**/build/TEST*.xml" -# export artifacts="**/build/TEST*.out" -# ;; -# MINGW*) -# export GOX_FLAGS="-arch 386" -# export testResults="**\\build\\TEST*.xml" -# export artifacts="**\\build\\TEST*.out" -# ;; -# esac -# elif [[ "${arch_type}" == "aarch64" || "${arch_type}" == "arm64" ]]; then -# export GOX_FLAGS="-arch arm" -# export testResults="**/build/TEST*.xml" -# export artifacts="**/build/TEST*.out" -# else -# echo "Unsupported OS" -# fi -#} - -#if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-winlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-auditbeat" ]]; then -## exportVars -# export TEST_TAGS="${TEST_TAGS:+$TEST_TAGS,}oracle" -#fi - -#if [[ "$BUILDKITE_STEP_KEY" == "xpack-winlogbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-metricbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "metricbeat-pipeline" ]]; then -# source .buildkite/scripts/common.sh -# # Set the MODULE env variable if possible, it should be defined before generating pipeline's steps. It is used in multiple pipelines. -# defineModuleFromTheChangeSet "${BEATS_PROJECT_NAME}" -#fi diff --git a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml index 24d6b3acab6e..d393dc587fd5 100644 --- a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml @@ -309,7 +309,7 @@ steps: env: ASDF_TERRAFORM_VERSION: 1.0.2 MODULE_DIR: "x-pack/metricbeat/module/aws" - GOX_FLAGS: "${ARCH_UNIX_ARM_64}" +# GOX_FLAGS: "${ARCH_UNIX_ARM_64}" MODULE: "aws" # TEST_TAGS should be reviewed and updated: https://github.com/elastic/ingest-dev/issues/3476 TEST_TAGS: "aws" From 907b1fc43f816c962a6a2b9039ed754bb7885368 Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka Date: Tue, 9 Jul 2024 12:14:49 +0300 Subject: [PATCH 07/12] moved GOX_FLAGS to hook --- .buildkite/auditbeat/auditbeat-pipeline.yml | 32 ------------------- .../deploy/kubernetes/deploy-k8s-pipeline.yml | 1 - .buildkite/filebeat/filebeat-pipeline.yml | 28 ---------------- .buildkite/heartbeat/heartbeat-pipeline.yml | 30 ----------------- .buildkite/hooks/pre-command | 18 +++++++++++ .buildkite/libbeat/pipeline.libbeat.yml | 15 --------- .buildkite/metricbeat/pipeline.yml | 28 ---------------- .buildkite/packetbeat/pipeline.packetbeat.yml | 26 --------------- .buildkite/scripts/install_macos_tools.sh | 16 ---------- .buildkite/winlogbeat/pipeline.winlogbeat.yml | 16 ---------- .../x-pack/pipeline.xpack.auditbeat.yml | 26 --------------- .../x-pack/pipeline.xpack.dockerlogbeat.yml | 9 ------ .buildkite/x-pack/pipeline.xpack.filebeat.yml | 29 ----------------- .../x-pack/pipeline.xpack.heartbeat.yml | 24 -------------- .buildkite/x-pack/pipeline.xpack.libbeat.yml | 22 ------------- .../x-pack/pipeline.xpack.metricbeat.yml | 26 --------------- .../x-pack/pipeline.xpack.osquerybeat.yml | 23 ------------- .../x-pack/pipeline.xpack.packetbeat.yml | 32 ------------------- .../x-pack/pipeline.xpack.winlogbeat.yml | 20 ++---------- 19 files changed, 21 insertions(+), 400 deletions(-) diff --git a/.buildkite/auditbeat/auditbeat-pipeline.yml b/.buildkite/auditbeat/auditbeat-pipeline.yml index 3fc7975579df..55085f1401fc 100644 --- a/.buildkite/auditbeat/auditbeat-pipeline.yml +++ b/.buildkite/auditbeat/auditbeat-pipeline.yml @@ -19,10 +19,6 @@ env: IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" - ARCH_UNIX_X86_64: "-arch amd64" - ARCH_MINGW_X86_64: "-arch 386" - ARCH_UNIX_ARM_64: "-arch arm" - # Other deps ASDF_MAGE_VERSION: 1.15.0 @@ -64,8 +60,6 @@ steps: steps: - label: ":ubuntu: Auditbeat: Ubuntu x86_64 Unit Tests" - env: - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | set -euo pipefail cd auditbeat @@ -85,8 +79,6 @@ steps: context: "auditbeat: Ubuntu x86_64 Unit Tests" - label: ":rhel: Auditbeat: RHEL9 Unit Tests" - env: - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | set -euo pipefail cd auditbeat @@ -106,8 +98,6 @@ steps: context: "auditbeat: RHEL9 Unit Tests" - label: ":windows: Auditbeat: Win 2016 Unit Tests" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path auditbeat mage build unitTest @@ -128,8 +118,6 @@ steps: context: "auditbeat: Win 2016 Unit Tests" - label: ":windows: Auditbeat: Win 2022 Unit Tests" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path auditbeat mage build unitTest @@ -150,8 +138,6 @@ steps: context: "auditbeat: Win 2022 Unit Tests" - label: ":ubuntu: Auditbeat: Crosscompile" - env: - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | make -C auditbeat crosscompile retry: @@ -173,8 +159,6 @@ steps: - label: ":ubuntu: Auditbeat: Ubuntu x86_64 Integration Tests" key: "auditbeat-extended-integ-tests" if: build.env("GITHUB_PR_LABELS") =~ /.*integrations.*/ - env: - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | set -euo pipefail cd auditbeat @@ -196,8 +180,6 @@ steps: - label: ":ubuntu: Auditbeat: Ubuntu arm64 Integration Tests" key: "auditbeat-extended-arm64-integ-tests" if: build.env("GITHUB_PR_LABELS") =~ /.*integrations.*/ - env: - GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | set -euo pipefail cd auditbeat @@ -219,8 +201,6 @@ steps: - label: ":ubuntu: Auditbeat: Ubuntu arm64 Unit Tests" key: "auditbeat-extended-arm64-unit-tests" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ - env: - GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | set -euo pipefail cd auditbeat @@ -241,8 +221,6 @@ steps: - label: ":mac: Auditbeat: macOS x86_64 Unit Tests" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ - env: - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh @@ -263,8 +241,6 @@ steps: - label: ":mac: Auditbeat: macOS arm64 Unit Tests" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ - env: - GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh @@ -290,8 +266,6 @@ steps: steps: - label: ":windows: Auditbeat: Win 2019 Unit Tests" key: "auditbeat-extended-win-2019-unit-tests" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path auditbeat mage build unitTest @@ -313,8 +287,6 @@ steps: - label: ":windows: Auditbeat: Win 10 Unit Tests" key: "auditbeat-extended-win-10-unit-tests" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path auditbeat mage build unitTest @@ -336,8 +308,6 @@ steps: - label: ":windows: Auditbeat: Win 11 Unit Tests" key: "auditbeat-extended-win-11-unit-tests" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path auditbeat mage build unitTest @@ -372,7 +342,6 @@ steps: key: "auditbeat-package-linux-x86" env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | set -euo pipefail cd auditbeat @@ -394,7 +363,6 @@ steps: env: PLATFORMS: "linux/arm64" PACKAGES: "docker" - GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | set -euo pipefail cd auditbeat diff --git a/.buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml b/.buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml index b31a8facd848..c0005309457e 100644 --- a/.buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml +++ b/.buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml @@ -4,7 +4,6 @@ env: IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" MODULE: "kubernetes" - GOX_FLAGS: "-arch amd64" # Other deps ASDF_KIND_VERSION: "0.20.0" diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml index 464ab8ba8ff8..9dde05ccc81c 100644 --- a/.buildkite/filebeat/filebeat-pipeline.yml +++ b/.buildkite/filebeat/filebeat-pipeline.yml @@ -18,10 +18,6 @@ env: IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" - ARCH_UNIX_X86_64: "-arch amd64" - ARCH_MINGW_X86_64: "-arch 386" - ARCH_UNIX_ARM_64: "-arch arm" - #Deps ASDF_MAGE_VERSION: 1.15.0 @@ -65,8 +61,6 @@ steps: key: "filebeat-mandatory-tests" steps: - label: ":ubuntu: Filebeat: Ubuntu x86_64 Unit Tests" - env: - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd filebeat mage build unitTest @@ -85,8 +79,6 @@ steps: context: "filebeat: Ubuntu x86_64 Unit Tests" - label: ":ubuntu: Filebeat: Go Integration Tests" - env: - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd filebeat mage goIntegTest @@ -105,8 +97,6 @@ steps: context: "filebeat: Go Integration Tests" - label: ":ubuntu: Filebeat: Python Integration Tests" - env: - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd filebeat mage pythonIntegTest @@ -126,8 +116,6 @@ steps: - label: ":windows: Filebeat: Win 2016 Unit Tests" key: "windows-2016-unit-tests" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path filebeat mage build unitTest @@ -149,8 +137,6 @@ steps: - label: ":windows: Filebeat: Win 2022 Unit Tests" key: "windows-2022-unit-tests" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path filebeat mage build unitTest @@ -178,8 +164,6 @@ steps: - label: ":mac: Filebeat: macOS x86_64 Unit Tests" key: "macos-unit-tests-extended" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ - env: - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh @@ -201,8 +185,6 @@ steps: - label: ":mac: Filebeat: macOS arm64 Unit Tests" key: "macos-arm64-unit-tests-extended" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ - env: - GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh @@ -224,8 +206,6 @@ steps: - label: ":ubuntu: Filebeat: Ubuntu arm64 Unit Tests" key: "extended-arm64-unit-test" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ - env: - GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | cd filebeat mage build unitTest @@ -250,8 +230,6 @@ steps: steps: - label: ":windows: Filebeat: Win 2019 Unit Tests" key: "windows-extended-2019" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path filebeat mage build unitTest @@ -273,8 +251,6 @@ steps: - label: ":windows: Filebeat: Win 11 Unit Tests" key: "windows-extended-11" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path filebeat mage build unitTest @@ -296,8 +272,6 @@ steps: - label: ":windows: Filebeat: Win 10 Unit Tests" key: "windows-extended-10" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path filebeat mage build unitTest @@ -331,7 +305,6 @@ steps: key: "packaging-linux" env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd filebeat mage package @@ -354,7 +327,6 @@ steps: env: PLATFORMS: "linux/arm64" PACKAGES: "docker" - GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | cd filebeat mage package diff --git a/.buildkite/heartbeat/heartbeat-pipeline.yml b/.buildkite/heartbeat/heartbeat-pipeline.yml index 7831f3d42df1..27cf5a0d847f 100644 --- a/.buildkite/heartbeat/heartbeat-pipeline.yml +++ b/.buildkite/heartbeat/heartbeat-pipeline.yml @@ -18,10 +18,6 @@ env: IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" - ARCH_UNIX_X86_64: "-arch amd64" - ARCH_MINGW_X86_64: "-arch 386" - ARCH_UNIX_ARM_64: "-arch arm" - # Other deps ASDF_MAGE_VERSION: 1.15.0 ASDF_NODEJS_VERSION: 18.17.1 @@ -64,8 +60,6 @@ steps: steps: - label: ":ubuntu: Heartbeat: Ubuntu x86_64 Unit Tests" - env: - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd heartbeat mage build unitTest @@ -84,8 +78,6 @@ steps: context: "heartbeat: Ubuntu x86_64 Unit Tests" - label: ":rhel: Heartbeat: RHEL9 Unit Tests" - env: - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd heartbeat mage build unitTest @@ -105,8 +97,6 @@ steps: - label: ":windows: Heartbeat: Win 2016 Unit Tests" key: "windows-2016" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path heartbeat mage build unitTest @@ -127,8 +117,6 @@ steps: - label: ":windows: Heartbeat: Win 2022 Unit Test" key: "windows-2022" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path heartbeat mage build unitTest @@ -148,8 +136,6 @@ steps: context: "heartbeat: Win 2022 Unit Tests" - label: ":ubuntu: Heartbeat: Go Integration Tests" - env: - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | set -euo pipefail echo "~~~ Installing @elastic/synthetics" @@ -177,8 +163,6 @@ steps: context: "heartbeat: Go Integration Tests" - label: ":ubuntu: Heartbeat: Python Integration Tests" - env: - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | set -euo pipefail # defines the MODULE env var based on what's changed in a PR @@ -210,8 +194,6 @@ steps: - label: ":ubuntu: Heartbeat: Ubuntu arm64 Unit Tests" key: "arm-extended" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ - env: - GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | cd heartbeat mage build unitTest @@ -230,8 +212,6 @@ steps: - label: ":mac: Heartbeat: macOS x86_64 Unit Tests" key: "macos-extended" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ - env: - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh @@ -253,8 +233,6 @@ steps: - label: ":mac: Heartbeat: macOS arm64 Unit Tests" key: "macos-arm-extended" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ - env: - GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh @@ -280,8 +258,6 @@ steps: steps: - label: ":windows: Heartbeat: Win 2019 Unit Tests" key: "heartbeat-win-extended-2019" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path heartbeat mage build unitTest @@ -302,8 +278,6 @@ steps: - label: ":windows: Heartbeat: Win 11 Unit Tests" key: "heartbeat-windows-extended-11" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path heartbeat mage build unitTest @@ -324,8 +298,6 @@ steps: - label: ":windows: Heartbeat: Win 10 Unit Tests" key: "heartbeat-windows-extended-10" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path heartbeat mage build unitTest @@ -359,7 +331,6 @@ steps: key: "heartbeat-package-linux" env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd heartbeat mage package @@ -380,7 +351,6 @@ steps: env: PLATFORMS: "linux/arm64" PACKAGES: "docker" - GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | cd heartbeat mage package diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index fdc855d09a7e..9f554b312020 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -11,3 +11,21 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" && "$BUILDKITE_STEP export PRIVATE_CI_GCS_CREDENTIALS_SECRET fi +CPU_ARCH=$(uname -m) +PLATFORM_TYPE=$(uname) + +if [ "${CPU_ARCH}" == "x86_64" ]; then + case "${PLATFORM_TYPE}" in + Linux|Darwin) + export GOX_FLAGS="-arch amd64" + ;; + MINGW*) + export GOX_FLAGS="-arch 386" + ;; + esac +elif [[ "${CPU_ARCH}" == "aarch64" || "${CPU_ARCH}" == "arm64" ]]; then + export GOX_FLAGS="-arch arm" +else + echo "Unsupported OS" + exit 1 +fi diff --git a/.buildkite/libbeat/pipeline.libbeat.yml b/.buildkite/libbeat/pipeline.libbeat.yml index dcb6d2817f68..0efad7805236 100644 --- a/.buildkite/libbeat/pipeline.libbeat.yml +++ b/.buildkite/libbeat/pipeline.libbeat.yml @@ -9,9 +9,6 @@ env: GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" - ARCH_UNIX_X86_64: "-arch amd64" - ARCH_UNIX_ARM_64: "-arch arm" - #Deps ASDF_MAGE_VERSION: 1.15.0 @@ -54,8 +51,6 @@ steps: steps: - label: ":ubuntu: Libbeat: Ubuntu x86_64 Unit Tests" key: "mandatory-linux-unit-test" - env: - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | set -euo pipefail cd libbeat @@ -76,8 +71,6 @@ steps: - label: ":ubuntu: Libbeat: Go Integration Tests" key: "mandatory-int-test" - env: - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | set -euo pipefail cd libbeat @@ -98,8 +91,6 @@ steps: - label: ":ubuntu: Libbeat: Python Integration Tests" key: "mandatory-python-int-test" - env: - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | set -euo pipefail cd libbeat @@ -120,8 +111,6 @@ steps: - label: ":ubuntu: Libbeat: Crosscompile" key: "mandatory-cross-compile" - env: - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | set -euo pipefail cd libbeat @@ -142,8 +131,6 @@ steps: - label: ":ubuntu: Libbeat: Stress Tests" key: "mandatory-stress-test" - env: - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | set -euo pipefail cd libbeat @@ -167,8 +154,6 @@ steps: steps: - label: ":ubuntu: Libbeat: Ubuntu arm64 Unit Tests" key: "extended-arm64-unit-tests" - env: - GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | set -euo pipefail cd libbeat diff --git a/.buildkite/metricbeat/pipeline.yml b/.buildkite/metricbeat/pipeline.yml index 5378c2c4218d..c6081bd4c29a 100644 --- a/.buildkite/metricbeat/pipeline.yml +++ b/.buildkite/metricbeat/pipeline.yml @@ -18,10 +18,6 @@ env: IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" - ARCH_UNIX_X86_64: "-arch amd64" - ARCH_MINGW_X86_64: "-arch 386" - ARCH_UNIX_ARM_64: "-arch arm" - # Integration Tests K8S_VERSION: "v1.29.0" ASDF_KIND_VERSION: "0.20.0" @@ -67,8 +63,6 @@ steps: steps: - label: ":ubuntu: Metricbeat: Ubuntu x86_64 Unit Tests" key: "mandatory-linux-unit-test" - env: - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd metricbeat mage build unitTest @@ -88,8 +82,6 @@ steps: - label: ":ubuntu: Metricbeat: Go Integration Tests" key: "mandatory-int-test" - env: - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | set -euo pipefail # defines the MODULE env var based on what's changed in a PR @@ -116,8 +108,6 @@ steps: - label: ":ubuntu: Metricbeat: Python Integration Tests" key: "mandatory-python-int-test" - env: - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | set -euo pipefail # defines the MODULE env var based on what's changed in a PR @@ -144,8 +134,6 @@ steps: - label: ":ubuntu: Metricbeat: Crosscompile" key: "mandatory-cross-compile" - env: - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: "make -C metricbeat crosscompile" retry: automatic: @@ -162,8 +150,6 @@ steps: context: "metricbeat: Crosscompile" - label: ":windows: Metricbeat: Win 2016 Unit Tests" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path metricbeat mage build unitTest @@ -185,8 +171,6 @@ steps: context: "metricbeat: Win 2016 Unit Tests" - label: ":windows: Metricbeat: Win 2022 Unit Tests" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path metricbeat mage build unitTest @@ -213,8 +197,6 @@ steps: steps: - label: ":windows: Metricbeat: Win 10 Unit Tests" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path metricbeat mage build unitTest @@ -236,8 +218,6 @@ steps: context: "metricbeat: Win 10 Unit Tests" - label: ":windows: Metricbeat: Win 11 Unit Tests" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path metricbeat mage build unitTest @@ -259,8 +239,6 @@ steps: context: "metricbeat: Win 11 Unit Tests" - label: ":windows: Metricbeat: Win 2019 Unit Tests" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path metricbeat mage build unitTest @@ -288,8 +266,6 @@ steps: steps: - label: ":mac: Metricbeat: macOS x64_64 Unit Tests" key: "extended-macos-x64-64-unit-tests" - env: - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh @@ -312,8 +288,6 @@ steps: key: "extended-macos-arm64-unit-tests" skip: "Skipping due to elastic/beats#33035" # https://github.com/elastic/beats/issues/33035 - env: - GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh @@ -346,7 +320,6 @@ steps: key: "packaging-linux" env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd metricbeat mage package @@ -369,7 +342,6 @@ steps: env: PLATFORMS: "linux/arm64" PACKAGES: "docker" - GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | cd metricbeat mage package diff --git a/.buildkite/packetbeat/pipeline.packetbeat.yml b/.buildkite/packetbeat/pipeline.packetbeat.yml index 88601c2fe748..9550c7fae92e 100644 --- a/.buildkite/packetbeat/pipeline.packetbeat.yml +++ b/.buildkite/packetbeat/pipeline.packetbeat.yml @@ -19,10 +19,6 @@ env: IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" - ARCH_UNIX_X86_64: "-arch amd64" - ARCH_MINGW_X86_64: "-arch 386" - ARCH_UNIX_ARM_64: "-arch arm" - #Deps ASDF_MAGE_VERSION: 1.15.0 @@ -64,8 +60,6 @@ steps: steps: - label: ":ubuntu: Packetbeat: Ubuntu x86_64 Unit Tests" - env: - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd packetbeat mage build unitTest @@ -84,8 +78,6 @@ steps: context: "packetbeat: Ubuntu x86_64 Unit Tests" - label: ":rhel: Packetbeat: RHEL9 Unit Tests" - env: - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd packetbeat mage build unitTest @@ -104,8 +96,6 @@ steps: context: "packetbeat: RHEL9 Unit Tests" - label: ":windows: Packetbeat: Win 2016 Unit Tests" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path packetbeat mage build unitTest @@ -126,8 +116,6 @@ steps: context: "packetbeat: Win 2016 Unit Tests" - label: ":windows: Packetbeat: Win 2022 Unit Tests" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path packetbeat mage build unitTest @@ -153,8 +141,6 @@ steps: steps: - label: ":windows: Packetbeat: Win 10 Unit Tests" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path packetbeat mage build unitTest @@ -175,8 +161,6 @@ steps: context: "packetbeat: Win 10 Unit Tests" - label: ":windows: Packetbeat: Win 11 Unit Tests" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path packetbeat mage build unitTest @@ -198,8 +182,6 @@ steps: context: "packetbeat: Win 11 Unit Tests" - label: ":windows: Packetbeat: Win 2019 Unit Tests" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path packetbeat mage build unitTest @@ -227,8 +209,6 @@ steps: - label: ":mac: Packetbeat: macOS x86_64 Unit Tests" key: "macos-x86-64-unit-tests-extended" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ - env: - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh @@ -250,8 +230,6 @@ steps: - label: ":mac: Packetbeat: macOS arm64 Unit Tests" key: "macos-arm64-unit-tests-extended" if: build.env("GITHUB_PR_LABELS") =~ /.*(macOS|arm).*/ - env: - GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh @@ -273,8 +251,6 @@ steps: - label: ":ubuntu: Packetbeat: Ubuntu arm64 Unit Tests" key: "linux-arm64-unit-tests-extended" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ - env: - GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | cd packetbeat mage build unitTest @@ -305,7 +281,6 @@ steps: - label: ":ubuntu: Packetbeat: Packaging Linux" key: "packaging-linux" env: - GOX_FLAGS: "${ARCH_UNIX_X86_64}" PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" command: | cd packetbeat @@ -329,7 +304,6 @@ steps: env: PLATFORMS: "linux/arm64" PACKAGES: "docker" - GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | cd packetbeat mage package diff --git a/.buildkite/scripts/install_macos_tools.sh b/.buildkite/scripts/install_macos_tools.sh index 4572afb4cdc5..3d94e8aeabcc 100755 --- a/.buildkite/scripts/install_macos_tools.sh +++ b/.buildkite/scripts/install_macos_tools.sh @@ -27,21 +27,6 @@ retry() { return 0 } -define_arch() { - local platform_type="$(uname)" - local arch_type="$(uname -m)" - if [ "${arch_type}" == "x86_64" ]; then - export GOX_FLAGS="-arch amd64" - go_arch_type="amd64" - elif [[ "${arch_type}" == "aarch64" || "${arch_type}" == "arm64" ]]; then - export GOX_FLAGS="-arch arm" - go_arch_type="arm64" - else - echo "+++ Unsupported OS archictecture; uname: $platform_type and uname -m: $arch_type" - exit 1 - fi -} - create_workspace() { if [[ ! -d "${BIN}" ]]; then mkdir -p "${BIN}" @@ -83,7 +68,6 @@ with_mage() { with_go() { echo "Setting up the Go environment..." create_workspace - define_arch retry 5 curl -sL -o "${BIN}/gvm" "https://github.com/andrewkroh/gvm/releases/download/${SETUP_GVM_VERSION}/gvm-${PLATFORM_TYPE_LOWERCASE}-${go_arch_type}" chmod +x "${BIN}/gvm" eval "$(gvm $GO_VERSION)" diff --git a/.buildkite/winlogbeat/pipeline.winlogbeat.yml b/.buildkite/winlogbeat/pipeline.winlogbeat.yml index c592bc118f66..2e79a4bcf289 100644 --- a/.buildkite/winlogbeat/pipeline.winlogbeat.yml +++ b/.buildkite/winlogbeat/pipeline.winlogbeat.yml @@ -13,9 +13,6 @@ env: IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" - ARCH_UNIX_X86_64: "-arch amd64" - ARCH_MINGW_X86_64: "-arch 386" - # Other deps ASDF_MAGE_VERSION: 1.15.0 @@ -60,8 +57,6 @@ steps: steps: - label: ":ubuntu: Winlogbeat: Crossccompile" key: "mandatory-cross-compile" - env: - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: "make -C winlogbeat crosscompile" retry: automatic: @@ -78,8 +73,6 @@ steps: context: "winlogbeat: Crosscompile" - label: ":windows: Winlogbeat: Win 2016 Unit Tests" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path winlogbeat mage build unitTest @@ -101,8 +94,6 @@ steps: context: "winlogbeat: Win 2016 Unit Tests" - label: ":windows: Winlogbeat: Win 2019 Unit Tests" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path winlogbeat mage build unitTest @@ -124,8 +115,6 @@ steps: context: "winlogbeat: Win 2019 Unit Tests" - label: ":windows: Winlogbeat: Win 2022 Unit Tests" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path winlogbeat mage build unitTest @@ -152,8 +141,6 @@ steps: steps: - label: ":windows: Winlogbeat: Win 10 Unit Tests" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path winlogbeat mage build unitTest @@ -175,8 +162,6 @@ steps: context: "winlogbeat: Win 10 Unit Tests" - label: ":windows: Winlogbeat: Win 11 Unit Tests" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path winlogbeat mage build unitTest @@ -211,7 +196,6 @@ steps: key: "packaging-linux" env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd winlogbeat mage package diff --git a/.buildkite/x-pack/pipeline.xpack.auditbeat.yml b/.buildkite/x-pack/pipeline.xpack.auditbeat.yml index 6426db5f7752..3e3baa4b1381 100644 --- a/.buildkite/x-pack/pipeline.xpack.auditbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.auditbeat.yml @@ -19,10 +19,6 @@ env: IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" - ARCH_UNIX_X86_64: "-arch amd64" - ARCH_MINGW_X86_64: "-arch 386" - ARCH_UNIX_ARM_64: "-arch arm" - # Other deps ASDF_MAGE_VERSION: 1.15.0 @@ -65,8 +61,6 @@ steps: steps: - label: ":ubuntu: x-pack/auditbeat: Build Tests" key: "mandatory-linux-unit-test" - env: - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | set -euo pipefail # defines the MODULE env var based on what's changed in a PR @@ -91,8 +85,6 @@ steps: - label: ":rhel: x-pack/auditbeat: RHEL9 Unit Tests" key: "mandatory-rhel9-unit-test" - env: - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd x-pack/auditbeat mage build unitTest @@ -112,8 +104,6 @@ steps: - label: ":windows: x-pack/auditbeat: Win 2022 Unit Tests" key: "mandatory-win-2022-unit-tests" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/auditbeat mage build unitTest @@ -134,8 +124,6 @@ steps: context: "x-pack/auditbeat: Win 2022 Unit Tests" - label: ":windows: x-pack/auditbeat: Win 2016 Unit Tests" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/auditbeat mage build unitTest @@ -162,8 +150,6 @@ steps: steps: - label: ":windows: x-pack/auditbeat: Win 2019 Unit Tests" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/auditbeat mage build unitTest @@ -185,8 +171,6 @@ steps: context: "x-pack/auditbeat: Win 2019 Unit Tests" - label: ":windows: x-pack/auditbeat: Win 10 Unit Tests" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/auditbeat mage build unitTest @@ -208,8 +192,6 @@ steps: context: "x-pack/auditbeat: Win 10 Unit Tests" - label: ":windows: x-pack/auditbeat: Win 11 Unit Tests" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/auditbeat mage build unitTest @@ -237,8 +219,6 @@ steps: steps: - label: ":mac: x-pack/auditbeat: macOS x86_64 Unit Tests" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ - env: - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh @@ -259,8 +239,6 @@ steps: - label: ":mac: x-pack/auditbeat: macOS arm64 Unit Tests" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ - env: - GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh @@ -281,8 +259,6 @@ steps: - label: ":ubuntu: x-pack/auditbeat: Ubuntu arm64 Unit Tests" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ - env: - GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | cd x-pack/auditbeat mage build unitTest @@ -316,7 +292,6 @@ steps: key: "packaging-linux" env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd x-pack/auditbeat mage package @@ -339,7 +314,6 @@ steps: env: PLATFORMS: "linux/arm64" PACKAGES: "docker" - GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | cd x-pack/auditbeat mage package diff --git a/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml b/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml index 166fcde893ce..30264774b633 100644 --- a/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml @@ -10,9 +10,6 @@ env: IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" - ARCH_UNIX_X86_64: "-arch amd64" - ARCH_UNIX_ARM_64: "-arch arm" - # Other deps ASDF_MAGE_VERSION: 1.15.0 @@ -54,8 +51,6 @@ steps: steps: - label: ":ubuntu: x-pack/dockerlogbeat: Ubuntu x86_64 Unit Tests" key: "mandatory-linux-unit-test" - env: - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd x-pack/dockerlogbeat mage build unitTest @@ -75,8 +70,6 @@ steps: - label: ":ubuntu: x-pack/dockerlogbeat: Go Integration Tests" key: "mandatory-int-test" - env: - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | set -euo pipefail # defines the MODULE env var based on what's changed in a PR @@ -115,7 +108,6 @@ steps: key: "auditbeat-package-linux-x86" env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd x-pack/dockerlogbeat mage package @@ -136,7 +128,6 @@ steps: env: PLATFORMS: "linux/arm64" PACKAGES: "docker" - GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | cd x-pack/dockerlogbeat mage package diff --git a/.buildkite/x-pack/pipeline.xpack.filebeat.yml b/.buildkite/x-pack/pipeline.xpack.filebeat.yml index 29fda52a09fc..df7b39f3f03c 100644 --- a/.buildkite/x-pack/pipeline.xpack.filebeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.filebeat.yml @@ -18,10 +18,6 @@ env: IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" - ARCH_UNIX_X86_64: "-arch amd64" - ARCH_MINGW_X86_64: "-arch 386" - ARCH_UNIX_ARM_64: "-arch arm" - # Other deps ASDF_MAGE_VERSION: 1.15.0 @@ -65,8 +61,6 @@ steps: steps: - label: ":ubuntu: x-pack/filebeat: Ubuntu x86_64 Unit Tests" key: "x-pack-filebeat-mandatory-linux-unit-test" - env: - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd x-pack/filebeat mage build unitTest @@ -86,8 +80,6 @@ steps: - label: ":ubuntu: x-pack/filebeat: Go Integration Tests" key: "x-pack-filebeat-mandatory-int-test" - env: - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd x-pack/filebeat mage goIntegTest @@ -107,8 +99,6 @@ steps: - label: ":ubuntu: x-pack/filebeat: Python Integration Tests" key: "x-pack-filebeat-mandatory-python-int-test" - env: - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd x-pack/filebeat mage pythonIntegTest @@ -127,8 +117,6 @@ steps: context: "x-pack/filebeat: Python Integration Tests" - label: ":windows: x-pack/filebeat: Win 2022 Unit Tests" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/filebeat mage build unitTest @@ -150,8 +138,6 @@ steps: context: "x-pack/filebeat: Win 2022 Unit Tests" - label: ":windows: x-pack/filebeat: Win 2016 Unit Tests" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/filebeat mage build unitTest @@ -178,8 +164,6 @@ steps: steps: - label: ":windows: x-pack/filebeat: Win 2019 Unit Tests" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/filebeat mage build unitTest @@ -201,8 +185,6 @@ steps: context: "x-pack/filebeat: Win 2019 Unit Tests" - label: ":windows: x-pack/filebeat: Win 10 Unit Tests" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/filebeat mage build unitTest @@ -224,8 +206,6 @@ steps: context: "x-pack/filebeat: Win 10 Unit Tests" - label: ":windows: x-pack/filebeat: Win 11 Unit Tests" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/filebeat mage build unitTest @@ -253,8 +233,6 @@ steps: - label: ":ubuntu: x-pack/filebeat: Ubuntu arm64 Unit Tests" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ key: "x-pack-filebeat-mandatory-linux-arm-unit-test" - env: - GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | cd x-pack/filebeat mage build unitTest @@ -274,8 +252,6 @@ steps: - label: ":mac: x-pack/filebeat: macOS x86_64 Unit Tests" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ - env: - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh @@ -296,8 +272,6 @@ steps: - label: ":mac: x-pack/filebeat: macOS arm64 Unit Tests" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ - env: - GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh @@ -324,7 +298,6 @@ steps: env: ASDF_TERRAFORM_VERSION: 1.0.2 MODULE_DIR: "x-pack/filebeat/input/awss3/_meta/terraform" - GOX_FLAGS: "${ARCH_UNIX_ARM_64}" MODULE: "aws" # TEST_TAGS should be reviewed and updated: https://github.com/elastic/ingest-dev/issues/3476 TEST_TAGS: "aws" @@ -361,7 +334,6 @@ steps: key: "packaging-linux" env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd x-pack/filebeat mage package @@ -384,7 +356,6 @@ steps: env: PLATFORMS: "linux/arm64" PACKAGES: "docker" - GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | cd x-pack/filebeat mage package diff --git a/.buildkite/x-pack/pipeline.xpack.heartbeat.yml b/.buildkite/x-pack/pipeline.xpack.heartbeat.yml index 4bf38d5c0883..53dbab37716f 100644 --- a/.buildkite/x-pack/pipeline.xpack.heartbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.heartbeat.yml @@ -19,10 +19,6 @@ env: IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" - ARCH_UNIX_X86_64: "-arch amd64" - ARCH_MINGW_X86_64: "-arch 386" - ARCH_UNIX_ARM_64: "-arch arm" - # pipeline specific ELASTIC_SYNTHETICS_CAPABLE: "true" @@ -69,8 +65,6 @@ steps: steps: - label: ":ubuntu: x-pack/heartbeat: Ubuntu x86_64 Unit Tests" key: "mandatory-linux-unit-test" - env: - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | set -euo pipefail echo "~~~ Installing @elastic/synthetics" @@ -94,8 +88,6 @@ steps: - label: ":ubuntu: x-pack/heartbeat: Go Integration Tests" key: "mandatory-int-test" - env: - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | set -euo pipefail echo "~~~ Installing @elastic/synthetics" @@ -120,8 +112,6 @@ steps: - label: ":windows: x-pack/heartbeat: Win 2016 Unit Tests" key: "mandatory-win-2016-unit-tests" skip: "skipping due to elastic/beats#23957 and elastic/beats#23958" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/heartbeat mage build test @@ -145,8 +135,6 @@ steps: - label: ":windows: x-pack/heartbeat: Win 2022 Unit Tests" key: "mandatory-win-2022-unit-tests" skip: "skipping due to elastic/beats#23957 and elastic/beats#23958" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/heartbeat mage build unitTest @@ -173,8 +161,6 @@ steps: steps: - label: ":windows: x-pack/heartbeat: Win 10 Unit Tests" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/heartbeat mage build test @@ -197,8 +183,6 @@ steps: # Doesn't exist in Jenkins - label: ":windows: x-pack/heartbeat: Win 11 Unit Tests" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/heartbeat mage build test @@ -220,8 +204,6 @@ steps: context: "x-pack/heartbeat: Win 11 Unit Tests" - label: ":windows: x-pack/heartbeat: Win 2019 Unit Tests" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/heartbeat mage build test @@ -248,8 +230,6 @@ steps: steps: - label: ":mac: x-pack/heartbeat: macOS x86_64 Unit Tests" - env: - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh @@ -271,8 +251,6 @@ steps: context: "x-pack/heartbeat: macOS x86_64 Extended Tests" - label: ":mac: x-pack/heartbeat: macOS arm64 Unit Tests" - env: - GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh @@ -310,7 +288,6 @@ steps: key: "packaging-linux" env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd x-pack/heartbeat mage package @@ -333,7 +310,6 @@ steps: env: PLATFORMS: "linux/arm64" PACKAGES: "docker" - GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | cd x-pack/heartbeat mage package diff --git a/.buildkite/x-pack/pipeline.xpack.libbeat.yml b/.buildkite/x-pack/pipeline.xpack.libbeat.yml index 744195953421..a34c8ac732d0 100644 --- a/.buildkite/x-pack/pipeline.xpack.libbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.libbeat.yml @@ -16,10 +16,6 @@ env: GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" - ARCH_UNIX_X86_64: "-arch amd64" - ARCH_MINGW_X86_64: "-arch 386" - ARCH_UNIX_ARM_64: "-arch arm" - #Deps ASDF_MAGE_VERSION: 1.15.0 @@ -61,8 +57,6 @@ steps: steps: - label: ":ubuntu: x-pack/libbeat: Ubuntu x86_64 Unit Tests" key: "mandatory-linux-unit-test" - env: - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd x-pack/libbeat mage build unitTest @@ -82,8 +76,6 @@ steps: - label: ":ubuntu: x-pack/libbeat: Go Integration Tests" key: "mandatory-int-test" - env: - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd x-pack/libbeat mage goIntegTest @@ -103,8 +95,6 @@ steps: - label: ":ubuntu: x-pack/libbeat: Python Integration Tests" key: "mandatory-python-int-test" - env: - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd x-pack/libbeat mage pythonIntegTest @@ -123,8 +113,6 @@ steps: context: "x-pack/libbeat: Python Integration Tests" - label: ":windows: x-pack/libbeat: Win 2016 Unit Tests" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/libbeat mage -w reader\etw build goUnitTest @@ -146,8 +134,6 @@ steps: context: "x-pack/libbeat: Win 2016 Unit Tests" - label: ":windows: x-pack/libbeat: Win 2022 Unit Tests" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/libbeat mage -w reader\etw build goUnitTest @@ -174,8 +160,6 @@ steps: steps: - label: ":windows: x-pack/libbeat: Win 10 Unit Tests" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/libbeat mage -w reader\etw build goUnitTest @@ -197,8 +181,6 @@ steps: context: "x-pack/libbeat: Win 10 Unit Tests" - label: ":windows: x-pack/libbeat: Win 11 Unit Tests" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/libbeat mage -w reader\etw build goUnitTest @@ -220,8 +202,6 @@ steps: context: "x-pack/libbeat: Win 11 Unit Tests" - label: ":windows: x-pack/libbeat: Win 2019 Unit Tests" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/libbeat mage -w reader\etw build goUnitTest @@ -249,8 +229,6 @@ steps: steps: - label: ":ubuntu: x-pack/libbeat: Ubuntu arm64 Unit Tests" key: "extended-arm64-unit-tests" - env: - GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | cd x-pack/libbeat mage build unitTest diff --git a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml index d393dc587fd5..bb986cbbd852 100644 --- a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml @@ -18,10 +18,6 @@ env: IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" - ARCH_UNIX_X86_64: "-arch amd64" - ARCH_MINGW_X86_64: "-arch 386" - ARCH_UNIX_ARM_64: "-arch arm" - # Other deps ASDF_MAGE_VERSION: 1.15.0 @@ -63,8 +59,6 @@ steps: steps: - label: ":ubuntu: x-pack/metricbeat: Ubuntu x86_64 Unit Tests" key: "mandatory-linux-unit-test" - env: - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd x-pack/metricbeat mage build unitTest @@ -84,8 +78,6 @@ steps: - label: ":ubuntu: x-pack/metricbeat: Go (MODULE) Integration Tests" key: "mandatory-int-test" - env: - GOX_FLAGS: "${ARCH_UNIX_X86_64}" TEST_TAGS: "oracle" command: | set -euo pipefail @@ -111,7 +103,6 @@ steps: - label: ":ubuntu: x-pack/metricbeat: Python (MODULE) Integration Tests" key: "mandatory-python-int-test" env: - GOX_FLAGS: "${ARCH_UNIX_X86_64}" TEST_TAGS: "oracle" command: | set -euo pipefail @@ -135,8 +126,6 @@ steps: context: "x-pack/metricbeat: Python (MODULE) Integration Tests" - label: ":windows: x-pack/metricbeat: Win 2016 Unit Tests" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/metricbeat mage build unitTest @@ -158,8 +147,6 @@ steps: context: "x-pack/metricbeat: Win 2016 Unit Tests" - label: ":windows: x-pack/metricbeat: Win 2022 Unit Tests" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/metricbeat mage build unitTest @@ -186,8 +173,6 @@ steps: steps: - label: ":windows: x-pack/metricbeat: Win 10 Unit Tests" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/metricbeat mage build unitTest @@ -209,8 +194,6 @@ steps: context: "x-pack/metricbeat: Win 10 Unit Tests" - label: ":windows: x-pack/metricbeat: Win 11 Unit Tests" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/metricbeat mage build unitTest @@ -232,8 +215,6 @@ steps: context: "x-pack/metricbeat: Win 11 Unit Tests" - label: ":windows: x-pack/metricbeat: Win 2019 Unit Tests" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/metricbeat mage build unitTest @@ -261,8 +242,6 @@ steps: steps: - label: ":mac: x-pack/metricbeat: macOS x86_64 Unit Tests" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ - env: - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh @@ -284,8 +263,6 @@ steps: skip: "Skipping due to elastic/beats#33036 & elastic/ingest-dev#3400" # https://github.com/elastic/beats/issues/33036 https://github.com/elastic/ingest-dev/issues/3400 if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ - env: - GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh @@ -309,7 +286,6 @@ steps: env: ASDF_TERRAFORM_VERSION: 1.0.2 MODULE_DIR: "x-pack/metricbeat/module/aws" -# GOX_FLAGS: "${ARCH_UNIX_ARM_64}" MODULE: "aws" # TEST_TAGS should be reviewed and updated: https://github.com/elastic/ingest-dev/issues/3476 TEST_TAGS: "aws" @@ -346,7 +322,6 @@ steps: key: "packaging-linux" env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd x-pack/metricbeat mage package @@ -369,7 +344,6 @@ steps: env: PLATFORMS: "linux/arm64" PACKAGES: "docker" - GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | cd x-pack/metricbeat mage package diff --git a/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml b/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml index 5e02cfe89d26..3072c7b5bda1 100644 --- a/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml @@ -15,10 +15,6 @@ env: IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" - ARCH_UNIX_X86_64: "-arch amd64" - ARCH_MINGW_X86_64: "-arch 386" - ARCH_UNIX_ARM_64: "-arch arm" - # Other deps ASDF_MAGE_VERSION: 1.15.0 @@ -61,8 +57,6 @@ steps: steps: - label: ":ubuntu: x-pack/osquerybeat: Ubuntu x86_64 Unit Tests" key: "mandatory-linux-unit-test" - env: - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd x-pack/osquerybeat mage build unitTest @@ -82,8 +76,6 @@ steps: - label: ":ubuntu: x-pack/osquerybeat: Go Integration Tests" key: "mandatory-int-test" - env: - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd x-pack/osquerybeat mage goIntegTest @@ -102,8 +94,6 @@ steps: context: "x-pack/osquerybeat: Go Integration Tests" - label: ":windows: x-pack/osquerybeat: Win 2016 Unit Tests" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/osquerybeat mage build unitTest @@ -125,8 +115,6 @@ steps: context: "x-pack/osquerybeat: Win 2016 Unit Tests" - label: ":windows: x-pack/osquerybeat: Win 2022 Unit Tests" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/osquerybeat mage build unitTest @@ -153,8 +141,6 @@ steps: steps: - label: ":windows: x-pack/osquerybeat: Win 10 Unit Tests" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/osquerybeat mage build unitTest @@ -176,8 +162,6 @@ steps: context: "x-pack/osquerybeat: Win 10 Unit Tests" - label: ":windows: x-pack/osquerybeat: Win 11 Unit Tests" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/osquerybeat mage build unitTest @@ -199,8 +183,6 @@ steps: context: "x-pack/osquerybeat: Win 11 Unit Tests" - label: ":windows: x-pack/osquerybeat: Win 2019 Unit Tests" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/osquerybeat mage build unitTest @@ -227,8 +209,6 @@ steps: steps: - label: ":mac: x-pack/osquerybeat: macOS x86_64 Unit Tests" - env: - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh @@ -247,8 +227,6 @@ steps: context: "x-pack/osquerybeat: macOS x86_64 Unit Tests" - label: ":mac: x-pack/osquerybeat: macOS arm64 Unit Tests" - env: - GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh @@ -281,7 +259,6 @@ steps: key: "packaging-linux" env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd x-pack/osquerybeat mage package diff --git a/.buildkite/x-pack/pipeline.xpack.packetbeat.yml b/.buildkite/x-pack/pipeline.xpack.packetbeat.yml index da700494da35..6b96973d1225 100644 --- a/.buildkite/x-pack/pipeline.xpack.packetbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.packetbeat.yml @@ -19,10 +19,6 @@ env: IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" - ARCH_UNIX_X86_64: "-arch amd64" - ARCH_MINGW_X86_64: "-arch 386" - ARCH_UNIX_ARM_64: "-arch arm" - #Deps ASDF_MAGE_VERSION: 1.15.0 @@ -65,8 +61,6 @@ steps: steps: - label: ":ubuntu: x-pack/packetbeat: Ubuntu x86_64 Unit Tests" key: "mandatory-linux-unit-test" - env: - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd x-pack/packetbeat mage build unitTest @@ -86,8 +80,6 @@ steps: - label: ":ubuntu: x-pack/packetbeat: Ubuntu x86_64 System Tests" key: "mandatory-linux-system-test" - env: - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd x-pack/packetbeat mage systemTest @@ -107,8 +99,6 @@ steps: - label: ":rhel: x-pack/packetbeat: RHEL9 Unit Tests" key: "mandatory-rhel9-unit-test" - env: - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd x-pack/packetbeat mage build unitTest @@ -127,8 +117,6 @@ steps: context: "x-pack/packetbeat: RHEL9 Unit Tests" - label: ":windows: x-pack/packetbeat: Win 2016 Unit Tests" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/packetbeat mage build unitTest @@ -150,8 +138,6 @@ steps: context: "x-pack/packetbeat: Win 2016 Unit Tests" - label: ":windows: x-pack/packetbeat: Win 2022 Unit Tests" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/packetbeat mage build unitTest @@ -175,8 +161,6 @@ steps: - label: ":windows: x-pack/packetbeat: Win 2022 System Tests" key: "mandatory-win-2022-system-tests" skip: "skipping due to elastic/beats#38142" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/packetbeat mage systemTest @@ -202,8 +186,6 @@ steps: steps: - label: ":windows: x-pack/packetbeat: Win 10 Unit Tests" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/packetbeat mage build unitTest @@ -225,8 +207,6 @@ steps: context: "x-pack/packetbeat: Win 10 Unit Tests" - label: ":windows: x-pack/packetbeat: Win 11 Unit Tests" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/packetbeat mage build unitTest @@ -248,8 +228,6 @@ steps: context: "x-pack/packetbeat: Win 11 Unit Tests" - label: ":windows: x-pack/packetbeat: Win 2019 Unit Tests" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/packetbeat mage build unitTest @@ -273,8 +251,6 @@ steps: - label: ":windows: x-pack/packetbeat: Win 10 System Tests" key: "extended-win-10-system-tests" skip: "skipping due to elastic/beats#38142" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/packetbeat mage systemTest @@ -302,8 +278,6 @@ steps: - label: ":ubuntu: x-pack/packetbeat: Ubuntu arm64 Unit Tests" key: "extended-arm64-unit-test" if: build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ - env: - GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | cd x-pack/packetbeat mage build unitTest @@ -324,8 +298,6 @@ steps: - label: ":mac: x-pack/packetbeat: macOS x86_64 Unit Tests" key: "extended-macos-unit-tests" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ - env: - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh @@ -347,8 +319,6 @@ steps: - label: ":mac: x-pack/packetbeat: macOS arm64 Unit Tests" key: "macos-arm64-unit-tests-extended" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ - env: - GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh @@ -379,7 +349,6 @@ steps: key: "packaging-linux" env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - GOX_FLAGS: "${ARCH_UNIX_X86_64}" command: | cd x-pack/packetbeat mage package @@ -402,7 +371,6 @@ steps: env: PLATFORMS: "linux/arm64" PACKAGES: "docker" - GOX_FLAGS: "${ARCH_UNIX_ARM_64}" command: | cd x-pack/packetbeat mage package diff --git a/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml b/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml index c63bfed23fe4..15308a1b0634 100644 --- a/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml @@ -11,9 +11,6 @@ env: GCP_WIN_MACHINE_TYPE: "n2-standard-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" - ARCH_UNIX_X86_64: "-arch amd64" - ARCH_MINGW_X86_64: "-arch 386" - # Other deps ASDF_MAGE_VERSION: 1.15.0 @@ -55,8 +52,6 @@ steps: steps: - label: ":windows: x-pack/winlogbeat Win 2019 Unit (MODULE) Tests" key: "mandatory-win-2019-module-unit-tests" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Import-Module ./.buildkite/scripts/changesets.psm1 defineModuleFromTheChangeSet 'x-pack/winlogbeat' @@ -80,8 +75,6 @@ steps: context: "x-pack/winlogbeat Win 2019 Unit (MODULE) Tests" - label: ":windows: x-pack/winlogbeat: Win 2016 Unit Tests" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/winlogbeat mage build unitTest @@ -103,8 +96,6 @@ steps: context: "x-pack/winlogbeat: Win 2016 Unit Tests" - label: ":windows: x-pack/winlogbeat: Win 2022 Unit Tests" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/winlogbeat mage build unitTest @@ -131,8 +122,6 @@ steps: steps: - label: ":windows: x-pack/winlogbeat: Win 10 Unit Tests" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/winlogbeat mage build unitTest @@ -154,8 +143,6 @@ steps: context: "x-pack/winlogbeat: Win 10 Unit Tests" - label: ":windows: x-pack/winlogbeat: Win 11 Unit Tests" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/winlogbeat mage build unitTest @@ -177,8 +164,6 @@ steps: context: "x-pack/winlogbeat: Win 11 Unit Tests" - label: ":windows: x-pack/winlogbeat: Win 2019 Unit Tests" - env: - GOX_FLAGS: "${ARCH_MINGW_X86_64}" command: | Set-Location -Path x-pack/winlogbeat mage build unitTest @@ -215,8 +200,9 @@ steps: key: "packaging-linux" env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - GOX_FLAGS: "${ARCH_UNIX_X86_64}" - command: "cd x-pack/winlogbeat && mage package" + command: | + cd x-pack/winlogbeat + mage package" retry: automatic: - limit: 3 From 751ecf1e6c8e2cc750c7753a02885f96b39e0907 Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka Date: Tue, 9 Jul 2024 12:18:52 +0300 Subject: [PATCH 08/12] added missing key to x-pack/metricbeat pipeline --- .buildkite/x-pack/pipeline.xpack.metricbeat.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml index bb986cbbd852..64ca511350d0 100644 --- a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml @@ -78,6 +78,7 @@ steps: - label: ":ubuntu: x-pack/metricbeat: Go (MODULE) Integration Tests" key: "mandatory-int-test" + env: TEST_TAGS: "oracle" command: | set -euo pipefail From 0a2fa91a37f629f68ef22bd89097baa3b20a7f94 Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka Date: Tue, 9 Jul 2024 13:01:22 +0300 Subject: [PATCH 09/12] adjusted install_macos_tools.sh and pre-command hook --- .buildkite/hooks/pre-command | 2 ++ .buildkite/scripts/install_macos_tools.sh | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 9f554b312020..77bf22f1c35b 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -18,6 +18,7 @@ if [ "${CPU_ARCH}" == "x86_64" ]; then case "${PLATFORM_TYPE}" in Linux|Darwin) export GOX_FLAGS="-arch amd64" + export GO_ARCH_TYPE="amd64" ;; MINGW*) export GOX_FLAGS="-arch 386" @@ -25,6 +26,7 @@ if [ "${CPU_ARCH}" == "x86_64" ]; then esac elif [[ "${CPU_ARCH}" == "aarch64" || "${CPU_ARCH}" == "arm64" ]]; then export GOX_FLAGS="-arch arm" + export GO_ARCH_TYPE="arm64" else echo "Unsupported OS" exit 1 diff --git a/.buildkite/scripts/install_macos_tools.sh b/.buildkite/scripts/install_macos_tools.sh index 3d94e8aeabcc..a29606a922d8 100755 --- a/.buildkite/scripts/install_macos_tools.sh +++ b/.buildkite/scripts/install_macos_tools.sh @@ -68,7 +68,7 @@ with_mage() { with_go() { echo "Setting up the Go environment..." create_workspace - retry 5 curl -sL -o "${BIN}/gvm" "https://github.com/andrewkroh/gvm/releases/download/${SETUP_GVM_VERSION}/gvm-${PLATFORM_TYPE_LOWERCASE}-${go_arch_type}" + retry 5 curl -sL -o "${BIN}/gvm" "https://github.com/andrewkroh/gvm/releases/download/${SETUP_GVM_VERSION}/gvm-${PLATFORM_TYPE_LOWERCASE}-${GO_ARCH_TYPE}" chmod +x "${BIN}/gvm" eval "$(gvm $GO_VERSION)" go version From a21c63169e64eadb8af4a9071841cdb3507d189e Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka Date: Tue, 9 Jul 2024 16:16:59 +0300 Subject: [PATCH 10/12] updated x-pack/winlogbeat pipeline --- .buildkite/x-pack/pipeline.xpack.winlogbeat.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml b/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml index 15308a1b0634..7853f651670e 100644 --- a/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml @@ -202,7 +202,7 @@ steps: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" command: | cd x-pack/winlogbeat - mage package" + mage package retry: automatic: - limit: 3 From c52c4031edc54c302b4751e8f008112cb057ac7c Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka Date: Wed, 10 Jul 2024 12:02:13 +0300 Subject: [PATCH 11/12] removed common.sh --- .buildkite/scripts/common.sh | 481 ----------------------------------- 1 file changed, 481 deletions(-) delete mode 100755 .buildkite/scripts/common.sh diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh deleted file mode 100755 index 55e97890481f..000000000000 --- a/.buildkite/scripts/common.sh +++ /dev/null @@ -1,481 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -WORKSPACE=${WORKSPACE:-"$(pwd)"} -BIN="${WORKSPACE}/bin" -platform_type="$(uname)" -platform_type_lowercase=$(echo "$platform_type" | tr '[:upper:]' '[:lower:]') -arch_type="$(uname -m)" -GITHUB_PR_TRIGGER_COMMENT=${GITHUB_PR_TRIGGER_COMMENT:-""} -GITHUB_PR_LABELS=${GITHUB_PR_LABELS:-""} -ONLY_DOCS=${ONLY_DOCS:-"true"} -OSS_MODULE_PATTERN="^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" -XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" -# define if needed run the whole pipeline for the particular beat -[ -z "${run_filebeat+x}" ] && run_filebeat="$(buildkite-agent meta-data get run_filebeat --default "false")" -[ -z "${run_xpack_metricbeat+x}" ] && run_xpack_metricbeat="$(buildkite-agent meta-data get run_xpack_metricbeat --default "false")" - -# define if needed run ARM platform-specific tests for the particular beat -[ -z "${run_filebeat_arm_tests+x}" ] && run_filebeat_arm_tests="$(buildkite-agent meta-data get run_filebeat_arm_tests --default "false")" - -# define if needed run MacOS platform-specific tests for the particular beat -[ -z "${run_xpack_metricbeat_macos_tests+x}" ] && run_xpack_metricbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_macos_tests --default "false")" - -# define if needed run cloud-specific tests for the particular beat -[ -z "${run_xpack_metricbeat_aws_tests+x}" ] && run_xpack_metricbeat_aws_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_aws_tests --default "false")" - -ci_changeset=( - "^.buildkite/.*" - ) - -go_mod_changeset=( - "^go.mod" - ) - -oss_changeset=( - "^go.mod" - "^pytest.ini" - "^dev-tools/.*" - "^libbeat/.*" - "^testing/.*" -) - -xpack_changeset=( - "${oss_changeset[@]}" -) - -docs_changeset=( - ".*\\.(asciidoc|md)" - "deploy/kubernetes/.*-kubernetes\\.yaml" - ) - -packaging_changeset=( - "^dev-tools/packaging/.*" - ".go-version" - ) - -case "${BUILDKITE_PIPELINE_SLUG}" in - "beats-xpack-metricbeat") - BEAT_CHANGESET_REFERENCE=${xpack_metricbeat_changeset[@]} - ;; - *) - echo "~~~ The changeset for the ${BUILDKITE_PIPELINE_SLUG} pipeline hasn't been defined yet." - ;; -esac - -check_and_set_beat_vars() { - local BEATS_PROJECT_NAME=${BEATS_PROJECT_NAME:=""} - if [[ "${BEATS_PROJECT_NAME:=""}" == *"x-pack/"* ]]; then - BEATS_XPACK_PROJECT_NAME=${BEATS_PROJECT_NAME//-/} #remove - - BEATS_XPACK_PROJECT_NAME=${BEATS_XPACK_PROJECT_NAME//\//_} #replace / to _ - BEATS_XPACK_LABEL_PROJECT_NAME=${BEATS_PROJECT_NAME//\//-} #replace / to - for labels - BEATS_GH_LABEL=${BEATS_XPACK_LABEL_PROJECT_NAME} - TRIGGER_SPECIFIC_BEAT="run_${BEATS_XPACK_PROJECT_NAME}" - TRIGGER_SPECIFIC_ARM_TESTS="run_${BEATS_XPACK_PROJECT_NAME}_arm_tests" - TRIGGER_SPECIFIC_AWS_TESTS="run_${BEATS_XPACK_PROJECT_NAME}_aws_tests" - TRIGGER_SPECIFIC_MACOS_TESTS="run_${BEATS_XPACK_PROJECT_NAME}_macos_tests" - TRIGGER_SPECIFIC_WIN_TESTS="run_${BEATS_XPACK_PROJECT_NAME}_win_tests" - echo "--- Beats project name is $BEATS_XPACK_PROJECT_NAME" - mandatory_changeset=( - "${BEAT_CHANGESET_REFERENCE[@]}" - "${xpack_changeset[@]}" - "${ci_changeset[@]}" - ) - else - BEATS_GH_LABEL=${BEATS_PROJECT_NAME} - TRIGGER_SPECIFIC_BEAT="run_${BEATS_PROJECT_NAME}" - TRIGGER_SPECIFIC_ARM_TESTS="run_${BEATS_PROJECT_NAME}_arm_tests" - TRIGGER_SPECIFIC_AWS_TESTS="run_${BEATS_PROJECT_NAME}_aws_tests" - TRIGGER_SPECIFIC_MACOS_TESTS="run_${BEATS_PROJECT_NAME}_macos_tests" - TRIGGER_SPECIFIC_WIN_TESTS="run_${BEATS_PROJECT_NAME}_win_tests" - echo "--- Beats project name is $BEATS_PROJECT_NAME" - mandatory_changeset=( - "${BEAT_CHANGESET_REFERENCE[@]}" - "${oss_changeset[@]}" - "${ci_changeset[@]}" - ) - fi - BEATS_GH_COMMENT="/test ${BEATS_PROJECT_NAME}" - BEATS_GH_MACOS_COMMENT="${BEATS_GH_COMMENT} for macos" - BEATS_GH_ARM_COMMENT="${BEATS_GH_COMMENT} for arm" - BEATS_GH_AWS_COMMENT="${BEATS_GH_COMMENT} for aws cloud" - BEATS_GH_WIN_COMMENT="${BEATS_GH_COMMENT} for windows" - BEATS_GH_MACOS_LABEL="macOS" - BEATS_GH_ARM_LABEL="arm" - BEATS_GH_AWS_LABEL="aws" - BEATS_GH_WIN_LABEL="windows" -} - -with_docker_compose() { - local version=$1 - echo "Setting up the Docker-compose environment..." - create_workspace - retry 3 curl -sSL -o ${BIN}/docker-compose "https://github.com/docker/compose/releases/download/${version}/docker-compose-${platform_type_lowercase}-${arch_type}" - chmod +x ${BIN}/docker-compose - export PATH="${BIN}:${PATH}" - docker-compose version -} - -create_workspace() { - if [[ ! -d "${BIN}" ]]; then - mkdir -p "${BIN}" - fi -} - -add_bin_path() { - echo "Adding PATH to the environment variables..." - create_workspace - export PATH="${BIN}:${PATH}" -} - -check_platform_architeture() { - case "${arch_type}" in - "x86_64") - go_arch_type="amd64" - ;; - "aarch64") - go_arch_type="arm64" - ;; - "arm64") - go_arch_type="arm64" - ;; - *) - echo "The current platform or OS type is unsupported yet" - ;; - esac -} - -with_mage() { - local install_packages=( - "github.com/magefile/mage" - "github.com/elastic/go-licenser" - "golang.org/x/tools/cmd/goimports" - "github.com/jstemmer/go-junit-report" - "gotest.tools/gotestsum" - ) - create_workspace - for pkg in "${install_packages[@]}"; do - go install "${pkg}@latest" - done - echo "Download modules to local cache" - retry 3 go mod download -} - -with_go() { - echo "Setting up the Go environment..." - create_workspace - check_platform_architeture - retry 5 curl -sL -o "${BIN}/gvm" "https://github.com/andrewkroh/gvm/releases/download/${SETUP_GVM_VERSION}/gvm-${platform_type_lowercase}-${go_arch_type}" - chmod +x "${BIN}/gvm" - eval "$(gvm $GO_VERSION)" - go version - which go - local go_path="$(go env GOPATH):$(go env GOPATH)/bin" - export PATH="${go_path}:${PATH}" -} - -checkLinuxType() { - if [ "${platform_type}" == "Linux" ]; then - if grep -q 'ubuntu' /etc/os-release; then - echo "ubuntu" - elif grep -q 'rhel' /etc/os-release; then - echo "rhel" - else - echo "Unsupported Linux" - fi - else - echo "This is not a Linux" - fi -} - -with_python() { - local linuxType="$(checkLinuxType)" - echo "${linuxType}" - if [ "${platform_type}" == "Linux" ]; then - if [ "${linuxType}" = "ubuntu" ]; then - sudo apt-get update - sudo apt-get install -y python3-pip python3-venv - elif [ "${linuxType}" = "rhel" ]; then - sudo dnf update -y - sudo dnf install -y python3 python3-pip - pip3 install virtualenv - fi - elif [ "${platform_type}" == "Darwin" ]; then - brew update - pip3 install virtualenv - ulimit -Sn 10000 - fi -} - -with_dependencies() { - local linuxType="$(checkLinuxType)" - echo "${linuxType}" - if [ "${platform_type}" == "Linux" ]; then - if [ "${linuxType}" = "ubuntu" ]; then - sudo apt-get update - sudo apt-get install -y libsystemd-dev libpcap-dev librpm-dev - elif [ "${linuxType}" = "rhel" ]; then - # sudo dnf update -y - sudo dnf install -y systemd-devel rpm-devel - wget https://mirror.stream.centos.org/9-stream/CRB/${arch_type}/os/Packages/libpcap-devel-1.10.0-4.el9.${arch_type}.rpm #TODO: move this step to our own image - sudo dnf install -y libpcap-devel-1.10.0-4.el9.${arch_type}.rpm #TODO: move this step to our own image - fi - elif [ "${platform_type}" == "Darwin" ]; then - pip3 install libpcap - fi -} - -config_git() { - if [ -z "$(git config --get user.email)" ]; then - git config --global user.email "beatsmachine@users.noreply.github.com" - git config --global user.name "beatsmachine" - fi -} - -retry() { - local retries=$1 - shift - local count=0 - until "$@"; do - exit=$? - wait=$((2 ** count)) - count=$((count + 1)) - if [ $count -lt "$retries" ]; then - >&2 echo "Retry $count/$retries exited $exit, retrying in $wait seconds..." - sleep $wait - else - >&2 echo "Retry $count/$retries exited $exit, no more retries left." - return $exit - fi - done - return 0 -} - -are_paths_changed() { - local patterns=("${@}") - local changelist=() - for pattern in "${patterns[@]}"; do - changed_files=($(git diff --name-only HEAD@{1} HEAD | grep -E "$pattern")) - if [ "${#changed_files[@]}" -gt 0 ]; then - changelist+=("${changed_files[@]}") - fi - done - - if [ "${#changelist[@]}" -gt 0 ]; then - echo "Files changed:" - echo "${changelist[*]}" - return 0 - else - echo "No files changed within specified changeset:" - echo "${patterns[*]}" - return 1 - fi -} - -are_changed_only_paths() { - local patterns=("${@}") - local changed_files=($(git diff --name-only HEAD@{1} HEAD)) - local matched_files=() - for pattern in "${patterns[@]}"; do - local matched=($(grep -E "${pattern}" <<< "${changed_files[@]}")) - if [ "${#matched[@]}" -gt 0 ]; then - matched_files+=("${matched[@]}") - fi - done - if [ "${#matched_files[@]}" -eq "${#changed_files[@]}" ] || [ "${#changed_files[@]}" -eq 0 ]; then - return 0 - fi - return 1 -} - -are_conditions_met_mandatory_tests() { - if are_paths_changed "${mandatory_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_COMMENT}" || "${GITHUB_PR_LABELS}" =~ /(?i)${BEATS_GH_LABEL}/ || "${!TRIGGER_SPECIFIC_BEAT}" == "true" ]]; then - return 0 - fi - return 1 -} - -are_conditions_met_macos_tests() { - if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 - if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then - if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_MACOS_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BEATS_GH_MACOS_LABEL} || "${!TRIGGER_SPECIFIC_MACOS_TESTS}" == "true" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 - return 0 - fi - fi - fi - return 1 -} - -are_conditions_met_aws_tests() { - if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 - if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then - if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_AWS_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BEATS_GH_AWS_LABEL} || "${!TRIGGER_SPECIFIC_AWS_TESTS}" == "true" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 - return 0 - fi - fi - fi - return 1 -} - -are_conditions_met_packaging() { - if are_conditions_met_mandatory_tests; then - if [[ "${BUILDKITE_TAG}" == "" || "${BUILDKITE_PULL_REQUEST}" != "false" ]]; then - return 0 - fi - fi - return 1 -} - -defineModuleFromTheChangeSet() { - # This method gathers the module name, if required, in order to run the ITs only if the changeset affects a specific module. - # For such, it's required to look for changes under the module folder and exclude anything else such as asciidoc and png files. - # This method defines and exports the MODULE variable with a particular module name or '' if changeset doesn't affect a specific module - local project_path=$1 - local project_path_transformed=$(echo "$project_path" | sed 's/\//\\\//g') - local project_path_exclussion="((?!^${project_path_transformed}\\/).)*\$" - local exclude=("^(${project_path_exclussion}|((?!\\/module\\/).)*\$|.*\\.asciidoc|.*\\.png)") - - if [[ "$project_path" == *"x-pack/"* ]]; then - local pattern=("$XPACK_MODULE_PATTERN") - else - local pattern=("$OSS_MODULE_PATTERN") - fi - local changed_modules="" - local module_dirs=$(find "$project_path/module" -mindepth 1 -maxdepth 1 -type d) - for module_dir in $module_dirs; do - if are_paths_changed $module_dir && ! are_changed_only_paths "${exclude[@]}"; then - if [[ -z "$changed_modules" ]]; then - changed_modules=$(basename "$module_dir") - else - changed_modules+=",$(basename "$module_dir")" - fi - fi - done - if [[ -z "$changed_modules" ]]; then # TODO: remove this condition and uncomment the line below when the issue https://github.com/elastic/ingest-dev/issues/2993 is solved - if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then - export MODULE="aws" - else - export MODULE="kubernetes" - fi - else - export MODULE="${changed_modules}" # TODO: remove this line and uncomment the line below when the issue https://github.com/elastic/ingest-dev/issues/2993 is solved - # export MODULE="${changed_modules}" # TODO: uncomment the line when the issue https://github.com/elastic/ingest-dev/issues/2993 is solved - fi -} - -terraformInit() { - local dir=$1 - echo "Terraform Init on $dir" - pushd "${dir}" > /dev/null - terraform init - popd > /dev/null -} - -withAWS() { - # This method gathers the masked AWS credentials from pre-command hook and sets the right AWS variable names. - export AWS_ACCESS_KEY_ID=$BEATS_AWS_ACCESS_KEY - export AWS_SECRET_ACCESS_KEY=$BEATS_AWS_SECRET_KEY - export TEST_TAGS="${TEST_TAGS:+$TEST_TAGS,}aws" -} - -startCloudTestEnv() { - local dir=$1 - withAWS - echo "--- Run docker-compose services for emulated cloud env" - docker-compose -f .buildkite/deploy/docker/docker-compose.yml up -d - with_Terraform - terraformInit "$dir" - export TF_VAR_BRANCH=$(echo "${BUILDKITE_BRANCH}" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9-]/-/g') - export TF_VAR_BUILD_ID="${BUILDKITE_BUILD_ID}" - export TF_VAR_CREATED_DATE=$(date +%s) - export TF_VAR_ENVIRONMENT="ci" - export TF_VAR_REPO="${REPO}" - pushd "${dir}" > /dev/null - terraform apply -auto-approve - popd > /dev/null -} - -withNodeJSEnv() { - # HOME="${WORKSPACE}" - local version=$1 - # local nvmPath="${HOME}/.nvm/versions/node/${version}/bin" - echo "Installing nvm" - curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.1/install.sh | bash - export NVM_DIR="$HOME/.nvm" - [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" - echo "Installing the NodeJs version $version" - nvm install "$version" - # export PATH="${nvmPath}:${PATH}" - nvm use "$version" - node --version -} - -installNodeJsDependencies() { - # Install dependencies to run browsers - if [ "${platform_type}" == "Linux" ]; then - sudo apt-get install -y \ - libatk1.0-0 \ - libatk-bridge2.0-0 \ - libcups2 \ - libxkbcommon0 \ - libatspi2.0-0 \ - libxcomposite1 \ - libxdamage1 \ - libxfixes3 \ - libxrandr2 \ - libgbm1 \ - libpango-1.0-0 \ - libcairo2 \ - libasound2 - if [ $? -ne 0 ]; then - echo "Error: Failed to install dependencies." - exit 1 - else - echo "Dependencies installed successfully." - fi - elif [ "${platform_type}" == "Darwin" ]; then - echo "TBD" - else - echo "Unsupported platform type." - exit 1 - fi -} - -teardown() { - # Teardown resources after using them - echo "---Terraform Cleanup" - .buildkite/scripts/terraform-cleanup.sh "${MODULE_DIR}" - - echo "---Docker Compose Cleanup" - docker-compose -f .buildkite/deploy/docker/docker-compose.yml down -v -} - -unset_secrets () { - for var in $(printenv | sed 's;=.*;;' | sort); do - if [[ "$var" == AWS_* || "$var" == BEATS_AWS_* ]]; then - unset "$var" - fi - done -} - -if ! are_changed_only_paths "${docs_changeset[@]}" ; then - export ONLY_DOCS="false" - echo "Changes include files outside the docs_changeset vairiabe. ONLY_DOCS=$ONLY_DOCS." -else - echo "All changes are related to DOCS. ONLY_DOCS=$ONLY_DOCS." -fi - -if are_paths_changed "${go_mod_changeset[@]}" ; then - export GO_MOD_CHANGES="true" -fi - -if are_paths_changed "${packaging_changeset[@]}" ; then - export PACKAGING_CHANGES="true" -fi - -if [[ "$BUILDKITE_STEP_KEY" == "xpack-metricbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "metricbeat-pipeline" ]]; then - # Set the MODULE env variable if possible, it should be defined before generating pipeline's steps. It is used in multiple pipelines. - defineModuleFromTheChangeSet "${BEATS_PROJECT_NAME}" -fi - -check_and_set_beat_vars From f5a79db4161150cf493c4e8ea0892aae09299c97 Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka Date: Wed, 10 Jul 2024 13:41:26 +0300 Subject: [PATCH 12/12] updated brackets --- .buildkite/hooks/pre-command | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 77bf22f1c35b..6468046facc9 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -14,7 +14,7 @@ fi CPU_ARCH=$(uname -m) PLATFORM_TYPE=$(uname) -if [ "${CPU_ARCH}" == "x86_64" ]; then +if [[ "${CPU_ARCH}" == "x86_64" ]]; then case "${PLATFORM_TYPE}" in Linux|Darwin) export GOX_FLAGS="-arch amd64"