From fcd4be0560bd041665ce446f3ca64fc6bf30f85b Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Thu, 18 Apr 2024 11:58:15 +0300 Subject: [PATCH] Migrate X-PACK Filebeat pipeline to main Beat pipeline (#38941) Migrate x-pack filebeat to single pipeline Signed-off-by: Alexandros Sapranidis --- .buildkite/hooks/pre-command | 1 - .buildkite/pipeline.yml | 27 ++ .buildkite/scripts/cloud_tests.sh | 14 +- .buildkite/scripts/common.sh | 20 +- .../generate_xpack_filebeat_pipeline.sh | 284 -------------- .buildkite/scripts/setenv.sh | 2 +- .buildkite/x-pack/pipeline.xpack.filebeat.yml | 359 +++++++++++++++--- 7 files changed, 342 insertions(+), 365 deletions(-) delete mode 100755 .buildkite/scripts/generate_xpack_filebeat_pipeline.sh diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index d8862d9cc94..c6a37908e23 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -66,7 +66,6 @@ ENABLED_BEATS_PIPELINES_SLUGS=( "beats-xpack-winlogbeat" "beats-xpack-dockerlogbeat" "beats-xpack-auditbeat" - "beats-xpack-filebeat" "beats-xpack-metricbeat" "beats-xpack-heartbeat" "deploy-k8s" diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 7f7c1f107f4..ddca6e022d7 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -80,6 +80,33 @@ steps: - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH} - GITHUB_PR_LABELS=${GITHUB_PR_LABELS} + - label: "Trigger x-pack/filebeat" + plugins: + - monorepo-diff#v1.0.1: + diff: "git diff --name-only origin/${GITHUB_PR_TARGET_BRANCH}...HEAD" + watch: + - path: + - x-pack/filebeat/ + - x-pack/libbeat/ + - .buildkite/x-pack/pipeline.xpack.filebeat.yml + - .buildkite/scripts + - .buildkite/hooks/ + #OSS + - go.mod + - pytest.ini + - dev-tools/ + - libbeat/** + - testing/** + config: + trigger: "beats-xpack-filebeat" + build: + commit: "${BUILDKITE_COMMIT}" + branch: "${BUILDKITE_BRANCH}" + env: + - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST} + - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH} + - GITHUB_PR_LABELS=${GITHUB_PR_LABELS} + - label: "Trigger Xpack/Dockerlogbeat" plugins: - monorepo-diff#v1.0.1: diff --git a/.buildkite/scripts/cloud_tests.sh b/.buildkite/scripts/cloud_tests.sh index 3deeb89671f..c8f3737e13e 100755 --- a/.buildkite/scripts/cloud_tests.sh +++ b/.buildkite/scripts/cloud_tests.sh @@ -1,13 +1,12 @@ #!/usr/bin/env bash set -euo pipefail -# What Terraform Module will run -if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-filebeat" ]]; then - export MODULE_DIR="x-pack/filebeat/input/awss3/_meta/terraform" -fi + +REPO_DIR=$(pwd) teardown() { - popd + # reset the directory to the root of the project + cd $REPO_DIR # Teardown resources after using them echo "~~~ Terraform Cleanup" tf_cleanup "${MODULE_DIR}" #TODO: move all docker-compose files from the .ci to .buildkite folder before switching to BK @@ -43,15 +42,16 @@ set -o xtrace echo "~~~ Run docker-compose services for emulated cloud env" docker-compose -f .ci/jobs/docker-compose.yml up -d #TODO: move all docker-compose files from the .ci to .buildkite folder before switching to BK echo "~~~ Initialize TF cloud resources" -pushd "$MODULE_DIR" +cd "$MODULE_DIR" export TF_VAR_BRANCH=$(echo "${BUILDKITE_BRANCH}" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9-]/-/g') export TF_VAR_BUILD_ID="${BUILDKITE_BUILD_ID}" export TF_VAR_CREATED_DATE=$(date +%s) export TF_VAR_ENVIRONMENT="ci" export TF_VAR_REPO="${REPO}" terraform init && terraform apply -auto-approve +cd - # Run tests echo "~~~ Run Cloud Tests for $BEATS_PROJECT_NAME" -pushd "${BEATS_PROJECT_NAME}" +cd "${BEATS_PROJECT_NAME}" mage build test diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index ba076194bb7..c99eb6ec998 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -18,27 +18,20 @@ XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" [ -z "${run_xpack_libbeat+x}" ] && run_xpack_libbeat="$(buildkite-agent meta-data get run_xpack_libbeat --default "false")" [ -z "${run_xpack_metricbeat+x}" ] && run_xpack_metricbeat="$(buildkite-agent meta-data get run_xpack_metricbeat --default "false")" [ -z "${run_xpack_packetbeat+x}" ] && run_xpack_packetbeat="$(buildkite-agent meta-data get run_xpack_packetbeat --default "false")" -[ -z "${run_xpack_filebeat+x}" ] && run_xpack_filebeat="$(buildkite-agent meta-data get run_xpack_filebeat --default "false")" [ -z "${run_xpack_heartbeat+x}" ] && run_xpack_heartbeat="$(buildkite-agent meta-data get run_xpack_heartbeat --default "false")" - # define if needed run ARM platform-specific tests for the particular beat [ -z "${run_filebeat_arm_tests+x}" ] && run_filebeat_arm_tests="$(buildkite-agent meta-data get run_filebeat_arm_tests --default "false")" [ -z "${run_packetbeat_arm_tests+x}" ] && run_packetbeat_arm_tests="$(buildkite-agent meta-data get run_packetbeat_arm_tests --default "false")" -[ -z "${run_xpack_filebeat_arm_tests+x}" ] && run_xpack_filebeat_arm_tests="$(buildkite-agent meta-data get run_xpack_filebeat_arm_tests --default "false")" [ -z "${run_xpack_libbeat_arm_tests+x}" ] && run_xpack_libbeat_arm_tests="$(buildkite-agent meta-data get run_xpack_libbeat_arm_tests --default "false")" [ -z "${run_xpack_packetbeat_arm_tests+x}" ] && run_xpack_packetbeat_arm_tests="$(buildkite-agent meta-data get run_xpack_packetbeat_arm_tests --default "false")" - # define if needed run MacOS platform-specific tests for the particular beat [ -z "${run_packetbeat_macos_tests+x}" ] && run_packetbeat_macos_tests="$(buildkite-agent meta-data get run_packetbeat_macos_tests --default "false")" -[ -z "${run_xpack_filebeat_macos_tests+x}" ] && run_xpack_filebeat_macos_tests="$(buildkite-agent meta-data get run_xpack_filebeat_macos_tests --default "false")" [ -z "${run_xpack_metricbeat_macos_tests+x}" ] && run_xpack_metricbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_macos_tests --default "false")" [ -z "${run_xpack_packetbeat_macos_tests+x}" ] && run_xpack_packetbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_packetbeat_macos_tests --default "false")" [ -z "${run_xpack_heartbeat_macos_tests+x}" ] && run_xpack_heartbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_heartbeat_macos_tests --default "false")" # define if needed run cloud-specific tests for the particular beat [ -z "${run_xpack_metricbeat_aws_tests+x}" ] && run_xpack_metricbeat_aws_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_aws_tests --default "false")" -[ -z "${run_xpack_filebeat_aws_tests+x}" ] && run_xpack_filebeat_aws_tests="$(buildkite-agent meta-data get run_xpack_filebeat_aws_tests --default "false")" - libbeat_changeset=( "^libbeat/.*" @@ -60,10 +53,6 @@ xpack_heartbeat_changeset=( "^x-pack/heartbeat/.*" ) -xpack_filebeat_changeset=( - "^x-pack/filebeat/.*" - ) - xpack_libbeat_changeset=( "^x-pack/libbeat/.*" ) @@ -117,9 +106,6 @@ case "${BUILDKITE_PIPELINE_SLUG}" in "beats-winlogbeat") BEAT_CHANGESET_REFERENCE=${winlogbeat_changeset[@]} ;; - "beats-xpack-filebeat") - BEAT_CHANGESET_REFERENCE=${xpack_filebeat_changeset[@]} - ;; "beats-xpack-heartbeat") BEAT_CHANGESET_REFERENCE=${xpack_heartbeat_changeset[@]} ;; @@ -371,7 +357,7 @@ are_conditions_met_mandatory_tests() { are_conditions_met_arm_tests() { if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 - if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-filebeat" ]]; then + if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_ARM_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BEATS_GH_ARM_LABEL} || "${!TRIGGER_SPECIFIC_ARM_TESTS}" == "true" ]]; then return 0 fi @@ -382,7 +368,7 @@ are_conditions_met_arm_tests() { are_conditions_met_macos_tests() { if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 - if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-heartbeat"; then + if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-heartbeat" ]] ; then if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_MACOS_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BEATS_GH_MACOS_LABEL} || "${!TRIGGER_SPECIFIC_MACOS_TESTS}" == "true" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 return 0 fi @@ -393,7 +379,7 @@ are_conditions_met_macos_tests() { are_conditions_met_aws_tests() { if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 - if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-filebeat" ]]; then + if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_AWS_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BEATS_GH_AWS_LABEL} || "${!TRIGGER_SPECIFIC_AWS_TESTS}" == "true" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 return 0 fi diff --git a/.buildkite/scripts/generate_xpack_filebeat_pipeline.sh b/.buildkite/scripts/generate_xpack_filebeat_pipeline.sh deleted file mode 100755 index 7e3e4aa5f39..00000000000 --- a/.buildkite/scripts/generate_xpack_filebeat_pipeline.sh +++ /dev/null @@ -1,284 +0,0 @@ -#!/usr/bin/env bash - -source .buildkite/scripts/common.sh - -set -euo pipefail - -pipelineName="pipeline.xpack-filebeat-dynamic.yml" - -echo "Add the mandatory and extended tests without additional conditions into the pipeline" -if are_conditions_met_mandatory_tests; then - cat > $pipelineName <<- YAML - -steps: - - - group: "Mandatory Tests" - key: "mandatory-tests" - steps: - - label: ":linux: Ubuntu Unit Tests" - key: "mandatory-linux-unit-test" - command: "cd $BEATS_PROJECT_NAME && mage build unitTest" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Ubuntu Unit Tests" - - - label: ":go: Go Integration Tests" - key: "mandatory-int-test" - command: "cd $BEATS_PROJECT_NAME && mage goIntegTest" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Go Integration Tests" - - - label: ":python: Python Integration Tests" - key: "mandatory-python-int-test" - command: "cd $BEATS_PROJECT_NAME && mage pythonIntegTest" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Python Integration Tests" - - - label: ":windows: Windows 2016 Unit Tests" - command: | - Set-Location -Path $BEATS_PROJECT_NAME - mage build unitTest - key: "mandatory-win-2016-unit-tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2016}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Windows 2016 Unit Tests" - - - label: ":windows: Windows 2022 Unit Tests" - command: | - Set-Location -Path $BEATS_PROJECT_NAME - mage build unitTest - key: "mandatory-win-2022-unit-tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2022}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Windows 2022 Unit Tests" - -## TODO: this condition will be changed in the Phase 3 of the Migration Plan https://docs.google.com/document/d/1IPNprVtcnHlem-uyGZM0zGzhfUuFAh4LeSl9JFHMSZQ/edit#heading=h.sltz78yy249h - - - group: "Extended Windows Tests" - key: "extended-win-tests" - steps: - - label: ":windows: Windows 10 Unit Tests" - command: | - Set-Location -Path $BEATS_PROJECT_NAME - mage build unitTest - key: "extended-win-10-unit-tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_10}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Windows 10 Unit Tests" - - - label: ":windows: Windows 11 Unit Tests" - command: | - Set-Location -Path $BEATS_PROJECT_NAME - mage build unitTest - key: "extended-win-11-unit-tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_11}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Windows 11 Unit Tests" - - - label: ":windows: Windows 2019 Unit Tests" - command: | - Set-Location -Path $BEATS_PROJECT_NAME - mage build unitTest - key: "extended-win-2019-unit-tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2019}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Windows 2019 Unit Tests" - -YAML -else - echo "The conditions don't match to requirements for generating pipeline steps." - exit 0 -fi - -if are_conditions_met_arm_tests || are_conditions_met_macos_tests || are_conditions_met_aws_tests; then - cat >> $pipelineName <<- YAML - - - group: "Extended Tests" - key: "extended-tests" - steps: - -YAML -fi - -if are_conditions_met_macos_tests; then - cat >> $pipelineName <<- YAML - - - label: ":mac: MacOS Unit Tests" - key: "extended-macos-unit-tests" - command: ".buildkite/scripts/unit_tests.sh" - agents: - provider: "orka" - imagePrefix: "${IMAGE_MACOS_X86_64}" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: MacOS Unit Tests" - -YAML -fi - -if are_conditions_met_arm_tests; then - cat >> $pipelineName <<- YAML - - label: ":linux: Ubuntu ARM Unit Tests" - key: "extended-arm64-unit-test" - command: "cd $BEATS_PROJECT_NAME && mage build unitTest" - agents: - provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" - instanceType: "${AWS_ARM_INSTANCE_TYPE}" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Ubuntu ARM Unit Tests" - -YAML -fi - -if are_conditions_met_aws_tests; then - cat >> $pipelineName <<- YAML - - label: ":linux: Cloud Tests" - key: "extended-cloud-test" - command: ".buildkite/scripts/cloud_tests.sh" - skip: "Temporary disable, additional conditions required, we hit AWS limits" - env: - MODULE: $MODULE - ASDF_TERRAFORM_VERSION: 1.0.2 - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Cloud Tests" - -YAML -fi - -echo "Check and add the Packaging into the pipeline" -if are_conditions_met_packaging; then - cat >> $pipelineName <<- YAML - - - wait: ~ - depends_on: - - step: "mandatory-tests" - allow_failure: false - - - group: "Packaging" # TODO: check conditions for future the main pipeline migration: https://github.com/elastic/beats/pull/28589 - key: "packaging" - steps: - - label: ":linux: Packaging Linux" - key: "packaging-linux" - command: "cd $BEATS_PROJECT_NAME && mage package" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - env: - PLATFORMS: "${PACKAGING_PLATFORMS}" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Packaging Linux" - - - label: ":linux: Packaging ARM" - key: "packaging-arm" - command: "cd $BEATS_PROJECT_NAME && mage package" - agents: - provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" - instanceType: "${AWS_ARM_INSTANCE_TYPE}" - env: - PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" - PACKAGES: "docker" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Packaging Linux ARM" - -YAML -fi - -echo "+++ Printing dynamic steps" -cat $pipelineName | yq . -P - -echo "--- Loading dynamic steps" -buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh index f6b03f53f9b..81d00eb5dea 100755 --- a/.buildkite/scripts/setenv.sh +++ b/.buildkite/scripts/setenv.sh @@ -49,7 +49,7 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_S export TEST_TAGS="${TEST_TAGS:+$TEST_TAGS,}oracle" fi -if [[ "$BUILDKITE_STEP_KEY" == "xpack-winlogbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-metricbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-dockerlogbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-filebeat-pipeline" || "$BUILDKITE_STEP_KEY" == "metricbeat-pipeline" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-heartbeat" ]]; then +if [[ "$BUILDKITE_STEP_KEY" == "xpack-winlogbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-metricbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-dockerlogbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "metricbeat-pipeline" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-heartbeat" ]]; then source .buildkite/scripts/common.sh if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-heartbeat" ]]; then export ELASTIC_SYNTHETICS_CAPABLE=true diff --git a/.buildkite/x-pack/pipeline.xpack.filebeat.yml b/.buildkite/x-pack/pipeline.xpack.filebeat.yml index 02b1cf0ddea..795302bc2d9 100644 --- a/.buildkite/x-pack/pipeline.xpack.filebeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.filebeat.yml @@ -3,13 +3,13 @@ name: "beats-xpack-filebeat" env: AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" - BEATS_PROJECT_NAME: "x-pack/filebeat" + GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" + IMAGE_MACOS_ARM: "generic-13-ventura-arm" IMAGE_MACOS_X86_64: "generic-13-ventura-x64" - IMAGE_RHEL9_X86_64: "family/platform-ingest-beats-rhel-9" IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" @@ -18,62 +18,311 @@ env: IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" - #Packaging - PACKAGING_ARM_PLATFORMS: "linux/arm64" - PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - - #Deps + # Other deps ASDF_MAGE_VERSION: 1.15.0 steps: + - group: "x-pack/filebeat Mandatory Tests" + key: "x-pack-filebeat-mandatory-tests" + steps: + - label: ":linux: Ubuntu Unit Tests" + key: "x-pack-filebeat-mandatory-linux-unit-test" + command: | + cd x-pack/filebeat + mage build unitTest + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: + - "x-pack/filebeat/build/*.xml" + - "x-pack/filebeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/filebeat: Ubuntu Unit Tests" + + - label: ":go: Go (MODULE) Integration Tests" + key: "x-pack-filebeat-mandatory-int-test" + command: | + set -euo pipefail + # defines the MODULE env var based on what's changed in a PR + source .buildkite/scripts/changesets.sh + defineModuleFromTheChangeSet x-pack/filebeat + echo "~~~ Will run tests with env var MODULE=$$MODULE" + cd x-pack/filebeat && mage goIntegTest + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: + - "x-pack/filebeat/build/*.xml" + - "x-pack/filebeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/filebeat: Go (MODULE) Integration Tests" + + - label: ":python: Python (MODULE) Integration Tests" + key: "x-pack-filebeat-mandatory-python-int-test" + command: | + set -euo pipefail + # defines the MODULE env var based on what's changed in a PR + source .buildkite/scripts/changesets.sh + defineModuleFromTheChangeSet x-pack/filebeat + echo "~~~ Running tests with env var MODULE=$$MODULE" + cd x-pack/filebeat && mage pythonIntegTest + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: + - "x-pack/filebeat/build/*.xml" + - "x-pack/filebeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/filebeat: Python (MODULE) Integration Tests" + + - label: ":windows: Windows 2022 Unit Tests" + command: | + Set-Location -Path x-pack/filebeat + mage build unitTest + key: "x-pack-filebeat-mandatory-win-2022-unit-tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2022}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "x-pack/filebeat/build/*.xml" + - "x-pack/filebeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/filebeat: Windows 2022 Unit Tests" + + - label: ":windows: Windows 2016 Unit Tests" + command: | + Set-Location -Path x-pack/filebeat + mage build unitTest + key: "x-pack-filebeat-mandatory-win-2016-unit-tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2016}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "x-pack/filebeat/build/*.xml" + - "x-pack/filebeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/filebeat: Windows 2016 Unit Tests" + + - label: ":linux: Ubuntu ARM Unit Tests" + key: "x-pack-filebeat-mandatory-linux-arm-unit-test" + command: | + cd x-pack/filebeat + mage build unitTest + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + artifact_paths: + - "x-pack/filebeat/build/*.xml" + - "x-pack/filebeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/filebeat: Ubuntu ARM Unit Tests" + + - group: "x-pack/filebeat Extended Windows Tests" + key: "x-pack-filebeat-extended-win-tests" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/ + steps: + - label: ":windows: Windows 2019 Unit Tests" + command: | + Set-Location -Path x-pack/filebeat + mage build unitTest + key: "x-pack-filebeat-extended-win-2019-unit-tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2019}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "x-pack/filebeat/build/*.xml" + - "x-pack/filebeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/filebeat: Windows 2019 Unit Tests" - - input: "Input Parameters" - key: "force-run-stages" - fields: - - select: "Filebeat - run_xpack_filebeat" - key: "run_xpack_filebeat" - options: - - label: "True" - value: "true" - - label: "False" - value: "false" - default: "false" - - select: "Filebeat - run_xpack_filebeat_macos_tests" - key: "run_xpack_filebeat_macos_tests" - options: - - label: "True" - value: "true" - - label: "False" - value: "false" - default: "false" - - select: "Filebeat - run_xpack_filebeat_arm_tests" - key: "run_xpack_filebeat_arm_tests" - options: - - label: "True" - value: "true" - - label: "False" - value: "false" - default: "false" - - select: "Filebeat - run_xpack_filebeat_aws_tests" - key: "run_xpack_filebeat_aws_tests" - options: - - label: "True" - value: "true" - - label: "False" - value: "false" - default: "false" - - if: "build.source == 'ui'" + - label: ":windows: Windows 10 Unit Tests" + command: | + Set-Location -Path x-pack/filebeat + mage build unitTest + key: "x-pack-filebeat-extended-win-10-unit-tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_10}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "x-pack/filebeat/build/*.xml" + - "x-pack/filebeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/filebeat: Windows 10 Unit Tests" + + - label: ":windows: Windows 11 Unit Tests" + command: | + Set-Location -Path x-pack/filebeat + mage build unitTest + key: "x-pack-filebeat-extended-win-11-unit-tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_11}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "x-pack/filebeat/build/*.xml" + - "x-pack/filebeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/filebeat: Windows 11 Unit Tests" + + - group: "x-pack/filebeat Extended Tests" + key: "x-pack-filebeat-extended-tests" + steps: + - label: ":mac: MacOS x86_64 Unit Tests" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*(macOS).*/ + command: | + set -euo pipefail + source .buildkite/scripts/install_macos_tools.sh + cd x-pack/filebeat && mage build unitTest + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_X86_64}" + artifact_paths: + - "x-pack/filebeat/build/*.xml" + - "x-pack/filebeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/filebeat: MacOS x86_64 Unit Tests" + + - label: ":mac: MacOS arm64 Unit Tests" + skip: "https://github.com/elastic/beats/issues/33036" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*(macOS).*/ + command: | + set -euo pipefail + source .buildkite/scripts/install_macos_tools.sh + cd x-pack/filebeat && mage build unitTest + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_ARM}" + artifact_paths: + - "x-pack/filebeat/build/*.xml" + - "x-pack/filebeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/filebeat: MacOS arm64 Unit Tests" + + - label: ":linux: Cloud (MODULE) Tests" + key: "x-pack-filebeat-extended-cloud-test" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*aws.*/ + command: | + set -euo pipefail + # defines the MODULE env var based on what's changed in a PR + source .buildkite/scripts/changesets.sh + defineModuleFromTheChangeSet x-pack/filebeat + echo "~~~ Running tests with env var MODULE=$$MODULE" + + .buildkite/scripts/cloud_tests.sh + env: + ASDF_TERRAFORM_VERSION: 1.0.2 + AWS_REGION: "eu-central-1" + BEATS_PROJECT_NAME: x-pack/filebeat + MODULE_DIR: "x-pack/filebeat/input/awss3/_meta/terraform" + REPO: beats + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: + - "x-pack/filebeat/build/*.xml" + - "x-pack/filebeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/filebeat: Cloud (MODULE) Tests" + + - label: ":linux: Cloud AWS (MODULE) Tests" + key: "x-pack-filebeat-extended-cloud-test-aws" + skip: "Skip test until fixed https://github.com/elastic/beats/issues/36425" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*aws.*/ + command: | + set -euo pipefail + # defines the MODULE env var based on what's changed in a PR + source .buildkite/scripts/changesets.sh + defineModuleFromTheChangeSet x-pack/filebeat + echo "~~~ Running tests with env var MODULE=$$MODULE" + .buildkite/scripts/cloud_tests.sh + env: + ASDF_TERRAFORM_VERSION: 1.0.2 + AWS_REGION: "eu-central-1" + BEATS_PROJECT_NAME: x-pack/filebeat + MODULE_DIR: "x-pack/filebeat/input/awss3/_meta/terraform" + REPO: beats + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + artifact_paths: + - "x-pack/filebeat/build/*.xml" + - "x-pack/filebeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/filebeat: Cloud AWS (MODULE) Tests" - wait: ~ - if: "build.source == 'ui'" - allow_dependency_failure: false - - - label: ":linux: Load dynamic x-pack filebeat pipeline" - key: "xpack-filebeat-pipeline" - command: ".buildkite/scripts/generate_xpack_filebeat_pipeline.sh" - agents: - image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest" - notify: - - github_commit_status: - context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" + # with PRs, we want to run packaging only if mandatory tests succeed + # for other cases, e.g. merge commits, we want to run packaging (and publish) independently of other tests + # this allows building DRA artifacts even if there is flakiness in mandatory tests + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + depends_on: + - step: "x-pack-filebeat-mandatory-tests" + + - group: "x-pack/filebeat Packaging" + key: "x-pack-filebeat-packaging" + steps: + - label: ":linux: Packaging Linux" + key: "packaging-linux" + command: | + cd x-pack/filebeat + mage package + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + notify: + - github_commit_status: + context: "x-pack/filebeat: Packaging Linux" + + - label: ":linux: Packaging ARM" + key: "packaging-arm" + command: | + cd x-pack/filebeat + mage package + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + env: + PLATFORMS: "linux/arm64" + PACKAGES: "docker" + notify: + - github_commit_status: + context: "x-pack/filebeat: Packaging Linux ARM"