From 4a844aa204ee3ea58a419aa77bc105387cb5b630 Mon Sep 17 00:00:00 2001
From: Olga Naydyonock <olga.naidjonoka@elastic.co>
Date: Fri, 17 May 2024 10:13:18 +0300
Subject: [PATCH 1/7] Update xpack/filebeat & xpack/metricbeat pipelines to
 match Jenkins steps (#39211)

Updated x-pack/filebeat and x-pack/metricbeat pipelines to match same steps & commands as in Jenkins

(cherry picked from commit bf138189db06df45c9ef3f4a8313cafbb6dd5583)

# Conflicts:
#	.buildkite/x-pack/pipeline.xpack.filebeat.yml
#	.buildkite/x-pack/pipeline.xpack.metricbeat.yml
---
 .../{cloud_tests.sh => setup_cloud_env.sh}    |   7 -
 .buildkite/x-pack/pipeline.xpack.filebeat.yml | 142 +++++++++---------
 .../x-pack/pipeline.xpack.metricbeat.yml      |  36 +++--
 3 files changed, 96 insertions(+), 89 deletions(-)
 rename .buildkite/scripts/{cloud_tests.sh => setup_cloud_env.sh} (92%)
 mode change 100755 => 100644

diff --git a/.buildkite/scripts/cloud_tests.sh b/.buildkite/scripts/setup_cloud_env.sh
old mode 100755
new mode 100644
similarity index 92%
rename from .buildkite/scripts/cloud_tests.sh
rename to .buildkite/scripts/setup_cloud_env.sh
index c8f3737e13e..3c23d71573f
--- a/.buildkite/scripts/cloud_tests.sh
+++ b/.buildkite/scripts/setup_cloud_env.sh
@@ -1,7 +1,6 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-
 REPO_DIR=$(pwd)
 
 teardown() {
@@ -31,7 +30,6 @@ tf_cleanup() {
 trap 'teardown' EXIT
 
 # Prepare the cloud resources using Terraform
-#startCloudTestEnv "${MODULE_DIR}"
 echo "~~~ Loading creds"
 set +o xtrace
 export AWS_ACCESS_KEY_ID=$BEATS_AWS_ACCESS_KEY
@@ -50,8 +48,3 @@ export TF_VAR_ENVIRONMENT="ci"
 export TF_VAR_REPO="${REPO}"
 terraform init && terraform apply -auto-approve
 cd -
-
-# Run tests
-echo "~~~ Run Cloud Tests for $BEATS_PROJECT_NAME"
-cd "${BEATS_PROJECT_NAME}"
-mage build test
diff --git a/.buildkite/x-pack/pipeline.xpack.filebeat.yml b/.buildkite/x-pack/pipeline.xpack.filebeat.yml
index f9c5b821429..a7a443ed757 100644
--- a/.buildkite/x-pack/pipeline.xpack.filebeat.yml
+++ b/.buildkite/x-pack/pipeline.xpack.filebeat.yml
@@ -2,7 +2,8 @@
 name: "beats-xpack-filebeat"
 
 env:
-  AWS_ARM_INSTANCE_TYPE: "t4g.xlarge"
+  AWS_ARM_INSTANCE_TYPE: "m6g.xlarge"
+  AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64"
 
   GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
   GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
@@ -10,7 +11,6 @@ env:
 
   IMAGE_MACOS_ARM: "generic-13-ventura-arm"
   IMAGE_MACOS_X86_64: "generic-13-ventura-x64"
-  IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64"
   IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
   IMAGE_WIN_10: "family/platform-ingest-beats-windows-10"
   IMAGE_WIN_11: "family/platform-ingest-beats-windows-11"
@@ -29,7 +29,7 @@ steps:
   - group: "x-pack/filebeat Mandatory Tests"
     key: "x-pack-filebeat-mandatory-tests"
     steps:
-      - label: ":linux: Ubuntu Unit Tests"
+      - label: ":ubuntu: x-pack/filebeat Unit Tests"
         key: "x-pack-filebeat-mandatory-linux-unit-test"
         command: |
           cd x-pack/filebeat
@@ -46,17 +46,13 @@ steps:
           - "x-pack/filebeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "x-pack/filebeat: Ubuntu Unit Tests"
+              context: "x-pack/filebeat: Unit Tests"
 
-      - label: ":go: Go (MODULE) Integration Tests"
+      - label: ":ubuntu: x-pack/filebeat Go Integration Tests"
         key: "x-pack-filebeat-mandatory-int-test"
         command: |
-          set -euo pipefail
-          # defines the MODULE env var based on what's changed in a PR
-          source .buildkite/scripts/changesets.sh
-          defineModuleFromTheChangeSet x-pack/filebeat
-          echo "~~~ Running tests"
-          cd x-pack/filebeat && mage goIntegTest
+          cd x-pack/filebeat
+          mage goIntegTest
         retry:
           automatic:
             - limit: 3
@@ -69,17 +65,13 @@ steps:
           - "x-pack/filebeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "x-pack/filebeat: Go (MODULE) Integration Tests"
+              context: "x-pack/filebeat: Go Integration Tests"
 
-      - label: ":python: Python (MODULE) Integration Tests"
+      - label: ":ubuntu: x-pack/filebeat Python Integration Tests"
         key: "x-pack-filebeat-mandatory-python-int-test"
         command: |
-          set -euo pipefail
-          # defines the MODULE env var based on what's changed in a PR
-          source .buildkite/scripts/changesets.sh
-          defineModuleFromTheChangeSet x-pack/filebeat
-          echo "~~~ Running tests"
-          cd x-pack/filebeat && mage pythonIntegTest
+          cd x-pack/filebeat
+          mage pythonIntegTest
         retry:
           automatic:
             - limit: 3
@@ -92,9 +84,9 @@ steps:
           - "x-pack/filebeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "x-pack/filebeat: Python (MODULE) Integration Tests"
+              context: "x-pack/filebeat: Python Integration Tests"
 
-      - label: ":windows: Windows 2022 Unit Tests"
+      - label: ":windows: x-pack/filebeat Win-2022 Unit Tests"
         command: |
           Set-Location -Path x-pack/filebeat
           mage build unitTest
@@ -113,9 +105,9 @@ steps:
           - "x-pack/filebeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "x-pack/filebeat: Windows 2022 Unit Tests"
+              context: "x-pack/filebeat: Win-2022 Unit Tests"
 
-      - label: ":windows: Windows 2016 Unit Tests"
+      - label: ":windows: x-pack/filebeat Win-2016 Unit Tests"
         command: |
           Set-Location -Path x-pack/filebeat
           mage build unitTest
@@ -134,32 +126,14 @@ steps:
           - "x-pack/filebeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "x-pack/filebeat: Windows 2016 Unit Tests"
-
-      - label: ":linux: Ubuntu ARM Unit Tests"
-        key: "x-pack-filebeat-mandatory-linux-arm-unit-test"
-        command: |
-          cd x-pack/filebeat
-          mage build unitTest
-        retry:
-          automatic:
-            - limit: 3
-        agents:
-          provider: "aws"
-          imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
-          instanceType: "${AWS_ARM_INSTANCE_TYPE}"
-        artifact_paths:
-          - "x-pack/filebeat/build/*.xml"
-          - "x-pack/filebeat/build/*.json"
-        notify:
-          - github_commit_status:
-              context: "x-pack/filebeat: Ubuntu ARM Unit Tests"
+              context: "x-pack/filebeat: Win-2016 Unit Tests"
 
   - group: "x-pack/filebeat Extended Windows Tests"
     key: "x-pack-filebeat-extended-win-tests"
     if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/
+
     steps:
-      - label: ":windows: Windows 2019 Unit Tests"
+      - label: ":windows: x-pack/filebeat Win-2019 Unit Tests"
         command: |
           Set-Location -Path x-pack/filebeat
           mage build unitTest
@@ -178,9 +152,9 @@ steps:
           - "x-pack/filebeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "x-pack/filebeat: Windows 2019 Unit Tests"
+              context: "x-pack/filebeat: Win-2019 Unit Tests"
 
-      - label: ":windows: Windows 10 Unit Tests"
+      - label: ":windows: x-pack/filebeat Win 10 Unit Tests"
         command: |
           Set-Location -Path x-pack/filebeat
           mage build unitTest
@@ -199,9 +173,9 @@ steps:
           - "x-pack/filebeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "x-pack/filebeat: Windows 10 Unit Tests"
+              context: "x-pack/filebeat: Win 10 Unit Tests"
 
-      - label: ":windows: Windows 11 Unit Tests"
+      - label: ":windows: x-pack/filebeat Win 11 Unit Tests"
         command: |
           Set-Location -Path x-pack/filebeat
           mage build unitTest
@@ -220,17 +194,41 @@ steps:
           - "x-pack/filebeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "x-pack/filebeat: Windows 11 Unit Tests"
+              context: "x-pack/filebeat: Win 11 Unit Tests"
 
   - group: "x-pack/filebeat Extended Tests"
     key: "x-pack-filebeat-extended-tests"
+
     steps:
+<<<<<<< HEAD
       - label: ":mac: MacOS x86_64 Unit Tests"
         if: build.env("GITHUB_PR_LABELS") =~ /.*(macOS).*/
+=======
+      - label: ":ubuntu: x-pack/filebeat Linux arm64 Unit Tests"
+        if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
+        key: "x-pack-filebeat-mandatory-linux-arm-unit-test"
+        command: |
+          cd x-pack/filebeat
+          mage build unitTest
+        agents:
+          provider: "aws"
+          imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
+          instanceType: "${AWS_ARM_INSTANCE_TYPE}"
+        artifact_paths:
+          - "x-pack/filebeat/build/*.xml"
+          - "x-pack/filebeat/build/*.json"
+        notify:
+          - github_commit_status:
+              context: "x-pack/filebeat: Linux arm64 Unit Tests"
+
+      - label: ":mac: x-pack/filebeat macOS x86_64 Unit Tests"
+        if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
+>>>>>>> bf138189db (Update xpack/filebeat & xpack/metricbeat pipelines to match Jenkins steps (#39211))
         command: |
           set -euo pipefail
           source .buildkite/scripts/install_macos_tools.sh
-          cd x-pack/filebeat && mage build unitTest
+          cd x-pack/filebeat
+          mage build unitTest
         retry:
           automatic:
             - limit: 3
@@ -242,18 +240,26 @@ steps:
           - "x-pack/filebeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "x-pack/filebeat: MacOS x86_64 Unit Tests"
+              context: "x-pack/filebeat: macOS x86_64 Unit Tests"
 
+<<<<<<< HEAD
       - label: ":mac: MacOS arm64 Unit Tests"
         skip: "https://github.com/elastic/beats/issues/33036"
         if: build.env("GITHUB_PR_LABELS") =~ /.*(macOS).*/
+=======
+      - label: ":mac: x-pack/filebeat macOS arm64 Unit Tests"
+        skip: "Skipping due to elastic/beats#33036"
+        # https://github.com/elastic/beats/issues/33036
+        if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
+>>>>>>> bf138189db (Update xpack/filebeat & xpack/metricbeat pipelines to match Jenkins steps (#39211))
         command: |
           set -euo pipefail
           source .buildkite/scripts/install_macos_tools.sh
-          cd x-pack/filebeat && mage build unitTest
+          cd x-pack/filebeat
+          mage build unitTest
         retry:
           automatic:
-            - limit: 3
+          - limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_ARM}"
@@ -262,9 +268,9 @@ steps:
           - "x-pack/filebeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "x-pack/filebeat: MacOS arm64 Unit Tests"
+              context: "x-pack/filebeat: macOS arm64 Unit Tests"
 
-      - label: ":linux: Cloud (MODULE) Tests"
+      - label: ":ubuntu: x-pack/filebeat Cloud (MODULE) Tests"
         key: "x-pack-filebeat-extended-cloud-test"
         if: build.env("GITHUB_PR_LABELS") =~ /.*aws.*/
         command: |
@@ -273,12 +279,12 @@ steps:
           source .buildkite/scripts/changesets.sh
           defineModuleFromTheChangeSet x-pack/filebeat
           echo "~~~ Running tests"
-
-          .buildkite/scripts/cloud_tests.sh
+          source .buildkite/scripts/setup_cloud_env.sh
+          cd x-pack/filebeat
+          mage build test
         env:
           ASDF_TERRAFORM_VERSION: 1.0.2
           AWS_REGION: "eu-central-1"
-          BEATS_PROJECT_NAME: x-pack/filebeat
           MODULE_DIR: "x-pack/filebeat/input/awss3/_meta/terraform"
           REPO: beats
         agents:
@@ -292,9 +298,10 @@ steps:
           - github_commit_status:
               context: "x-pack/filebeat: Cloud (MODULE) Tests"
 
-      - label: ":linux: Cloud AWS (MODULE) Tests"
+      - label: ":ubuntu: x-pack/filebeat Cloud AWS (MODULE) Tests"
         key: "x-pack-filebeat-extended-cloud-test-aws"
-        skip: "Skip test until fixed https://github.com/elastic/beats/issues/36425"
+        skip: "Skipping due to elastic/beats#36425"
+        # https://github.com/elastic/beats/issues/36425
         if: build.env("BUILDKITE_PULL_REQUEST") == "false"  || build.env("GITHUB_PR_LABELS") =~ /.*aws.*/
         command: |
           set -euo pipefail
@@ -302,16 +309,17 @@ steps:
           source .buildkite/scripts/changesets.sh
           defineModuleFromTheChangeSet x-pack/filebeat
           echo "~~~ Running tests"
-          .buildkite/scripts/cloud_tests.sh
+          source .buildkite/scripts/setup_cloud_env.sh
+          cd x-pack/filebeat
+          mage build test goIntegTest
         env:
           ASDF_TERRAFORM_VERSION: 1.0.2
           AWS_REGION: "eu-central-1"
-          BEATS_PROJECT_NAME: x-pack/filebeat
           MODULE_DIR: "x-pack/filebeat/input/awss3/_meta/terraform"
           REPO: beats
         agents:
           provider: "aws"
-          imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
+          imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
           instanceType: "${AWS_ARM_INSTANCE_TYPE}"
         artifact_paths:
           - "x-pack/filebeat/build/*.xml"
@@ -331,7 +339,7 @@ steps:
   - group: "x-pack/filebeat Packaging"
     key: "x-pack-filebeat-packaging"
     steps:
-      - label: ":linux: Packaging Linux"
+      - label: ":linux: x-pack/filebeat Packaging Linux"
         key: "packaging-linux"
         command: |
           cd x-pack/filebeat
@@ -352,7 +360,7 @@ steps:
           - github_commit_status:
               context: "x-pack/filebeat: Packaging Linux"
 
-      - label: ":linux: Packaging ARM"
+      - label: ":linux: x-pack/filebeat Packaging arm64"
         key: "packaging-arm"
         command: |
           cd x-pack/filebeat
@@ -363,11 +371,11 @@ steps:
         timeout_in_minutes: 20
         agents:
           provider: "aws"
-          imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
+          imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
           instanceType: "${AWS_ARM_INSTANCE_TYPE}"
         env:
           PLATFORMS: "linux/arm64"
           PACKAGES: "docker"
         notify:
           - github_commit_status:
-              context: "x-pack/filebeat: Packaging Linux ARM"
+              context: "x-pack/filebeat: Packaging Linux arm64"
diff --git a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml
index 46bd2d71cf7..71f7d993ba9 100644
--- a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml
+++ b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml
@@ -2,7 +2,8 @@
 name: "beats-xpack-metricbeat"
 
 env:
-  AWS_ARM_INSTANCE_TYPE: "t4g.xlarge"
+  AWS_ARM_INSTANCE_TYPE: "m6g.xlarge"
+  AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64"
 
   GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
   GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
@@ -10,7 +11,6 @@ env:
 
   IMAGE_MACOS_ARM: "generic-13-ventura-arm"
   IMAGE_MACOS_X86_64: "generic-13-ventura-x64"
-  IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64"
   IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
   IMAGE_WIN_10: "family/platform-ingest-beats-windows-10"
   IMAGE_WIN_11: "family/platform-ingest-beats-windows-11"
@@ -227,8 +227,14 @@ steps:
               context: "x-pack/metricbeat: MacOS x86_64 Unit Tests"
 
       - label: ":mac: MacOS arm64 Unit Tests"
+<<<<<<< HEAD
         skip: "https://github.com/elastic/beats/issues/33036"
         if: build.env("GITHUB_PR_LABELS") =~ /.*macOS.**/
+=======
+        skip: "Skipping due to elastic/beats#33036"
+        # https://github.com/elastic/beats/issues/33036
+        if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
+>>>>>>> bf138189db (Update xpack/filebeat & xpack/metricbeat pipelines to match Jenkins steps (#39211))
         command: |
           set -euo pipefail
           source .buildkite/scripts/install_macos_tools.sh
@@ -248,22 +254,21 @@ steps:
 
       - label: ":linux: Cloud (MODULE) Tests"
         key: "x-pack-metricbeat-extended-cloud-test"
-        skip: "doesn't belong in a stage in Jenkins, thus skipped"
+        skip: "Not a stage on Jenkins & elastic/ingest-dev#3170"
+        # https://github.com/elastic/ingest-dev/issues/3170
         if: build.env("GITHUB_PR_LABELS") =~ /.*aws.*/
-        # see link in Jenkins: https://github.com/elastic/beats/blob/ccd7b135df70358f8a02393d9bd8b716428b8048/x-pack/metricbeat/Jenkinsfile.yml#L39
-        # additionally skipping due to https://github.com/elastic/ingest-dev/issues/3170
         command: |
           set -euo pipefail
           # defines the MODULE env var based on what's changed in a PR
           source .buildkite/scripts/changesets.sh
           defineModuleFromTheChangeSet x-pack/metricbeat
           echo "~~~ Running tests"
-
-          .buildkite/scripts/cloud_tests.sh
+          source .buildkite/scripts/setup_cloud_env.sh
+          cd x-pack/metricbeat
+          mage build test
         env:
           ASDF_TERRAFORM_VERSION: 1.0.2
           AWS_REGION: "eu-central-1"
-          BEATS_PROJECT_NAME: x-pack/metricbeat
           MODULE_DIR: x-pack/metricbeat/module/aws
           REPO: beats
         agents:
@@ -279,26 +284,27 @@ steps:
 
       - label: ":linux: Cloud AWS (MODULE) Tests"
         key: "x-pack-metricbeat-extended-cloud-test-aws"
-        skip: "https://github.com/elastic/beats/issues/36425"
-        # see commented out section in Jenkins: https://github.com/elastic/beats/blob/main/x-pack/metricbeat/Jenkinsfile.yml#L41-L52
-        # additionally skipping due to https://github.com/elastic/ingest-dev/issues/3170
+        skip: "Skipping due elastic/beats#36425 & elastic/ingest-dev#3170"
+        # https://github.com/elastic/beats/issues/36425 & https://github.com/elastic/ingest-dev/issues/3170
         if: build.env("GITHUB_PR_LABELS") =~ /.*aws.*/
         command: |
           set -euo pipefail
           # defines the MODULE env var based on what's changed in a PR
           source .buildkite/scripts/changesets.sh
           defineModuleFromTheChangeSet x-pack/metricbeat
+
           echo "~~~ Running tests"
-          .buildkite/scripts/cloud_tests.sh
+          source .buildkite/scripts/setup_cloud_env.sh
+          cd x-pack/metricbeat
+          mage build test goIntegTest
         env:
           ASDF_TERRAFORM_VERSION: 1.0.2
           AWS_REGION: "eu-central-1"
-          BEATS_PROJECT_NAME: x-pack/metricbeat
           MODULE_DIR: x-pack/metricbeat/module/aws
           REPO: beats
         agents:
           provider: "aws"
-          imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
+          imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
           instanceType: "${AWS_ARM_INSTANCE_TYPE}"
         artifact_paths:
           - "x-pack/metricbeat/build/*.xml"
@@ -350,7 +356,7 @@ steps:
         timeout_in_minutes: 20
         agents:
           provider: "aws"
-          imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
+          imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
           instanceType: "${AWS_ARM_INSTANCE_TYPE}"
         env:
           PLATFORMS: "linux/arm64"

From 00e46fd14296dd5df9c30e73b13e744c4551b480 Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Fri, 17 May 2024 12:17:41 +0300
Subject: [PATCH 2/7] resolved conlicts

---
 .buildkite/x-pack/pipeline.xpack.filebeat.yml   | 11 -----------
 .buildkite/x-pack/pipeline.xpack.metricbeat.yml |  5 -----
 2 files changed, 16 deletions(-)

diff --git a/.buildkite/x-pack/pipeline.xpack.filebeat.yml b/.buildkite/x-pack/pipeline.xpack.filebeat.yml
index a7a443ed757..7dff1991d13 100644
--- a/.buildkite/x-pack/pipeline.xpack.filebeat.yml
+++ b/.buildkite/x-pack/pipeline.xpack.filebeat.yml
@@ -200,10 +200,6 @@ steps:
     key: "x-pack-filebeat-extended-tests"
 
     steps:
-<<<<<<< HEAD
-      - label: ":mac: MacOS x86_64 Unit Tests"
-        if: build.env("GITHUB_PR_LABELS") =~ /.*(macOS).*/
-=======
       - label: ":ubuntu: x-pack/filebeat Linux arm64 Unit Tests"
         if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
         key: "x-pack-filebeat-mandatory-linux-arm-unit-test"
@@ -223,7 +219,6 @@ steps:
 
       - label: ":mac: x-pack/filebeat macOS x86_64 Unit Tests"
         if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
->>>>>>> bf138189db (Update xpack/filebeat & xpack/metricbeat pipelines to match Jenkins steps (#39211))
         command: |
           set -euo pipefail
           source .buildkite/scripts/install_macos_tools.sh
@@ -242,16 +237,10 @@ steps:
           - github_commit_status:
               context: "x-pack/filebeat: macOS x86_64 Unit Tests"
 
-<<<<<<< HEAD
-      - label: ":mac: MacOS arm64 Unit Tests"
-        skip: "https://github.com/elastic/beats/issues/33036"
-        if: build.env("GITHUB_PR_LABELS") =~ /.*(macOS).*/
-=======
       - label: ":mac: x-pack/filebeat macOS arm64 Unit Tests"
         skip: "Skipping due to elastic/beats#33036"
         # https://github.com/elastic/beats/issues/33036
         if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
->>>>>>> bf138189db (Update xpack/filebeat & xpack/metricbeat pipelines to match Jenkins steps (#39211))
         command: |
           set -euo pipefail
           source .buildkite/scripts/install_macos_tools.sh
diff --git a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml
index 71f7d993ba9..6d004913c67 100644
--- a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml
+++ b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml
@@ -227,14 +227,9 @@ steps:
               context: "x-pack/metricbeat: MacOS x86_64 Unit Tests"
 
       - label: ":mac: MacOS arm64 Unit Tests"
-<<<<<<< HEAD
-        skip: "https://github.com/elastic/beats/issues/33036"
-        if: build.env("GITHUB_PR_LABELS") =~ /.*macOS.**/
-=======
         skip: "Skipping due to elastic/beats#33036"
         # https://github.com/elastic/beats/issues/33036
         if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
->>>>>>> bf138189db (Update xpack/filebeat & xpack/metricbeat pipelines to match Jenkins steps (#39211))
         command: |
           set -euo pipefail
           source .buildkite/scripts/install_macos_tools.sh

From ec4644577c38f1af2fffca7c6ae2cad12424504e Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Fri, 17 May 2024 14:38:03 +0300
Subject: [PATCH 3/7] adjusted xpack/filebeat pipeline

---
 .buildkite/x-pack/pipeline.xpack.filebeat.yml | 97 ++++++-------------
 1 file changed, 28 insertions(+), 69 deletions(-)

diff --git a/.buildkite/x-pack/pipeline.xpack.filebeat.yml b/.buildkite/x-pack/pipeline.xpack.filebeat.yml
index 7dff1991d13..51fb1d70a6e 100644
--- a/.buildkite/x-pack/pipeline.xpack.filebeat.yml
+++ b/.buildkite/x-pack/pipeline.xpack.filebeat.yml
@@ -28,7 +28,25 @@ env:
 steps:
   - group: "x-pack/filebeat Mandatory Tests"
     key: "x-pack-filebeat-mandatory-tests"
+
     steps:
+      - label: ":ubuntu: x-pack/filebeat Linux arm64 Unit Tests"
+        if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
+        key: "x-pack-filebeat-mandatory-linux-arm-unit-test"
+        command: |
+          cd x-pack/filebeat
+          mage build unitTest
+        agents:
+          provider: "aws"
+          imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
+          instanceType: "${AWS_ARM_INSTANCE_TYPE}"
+        artifact_paths:
+          - "x-pack/filebeat/build/*.xml"
+          - "x-pack/filebeat/build/*.json"
+        notify:
+          - github_commit_status:
+              context: "x-pack/filebeat: Linux arm64 Unit Tests"
+
       - label: ":ubuntu: x-pack/filebeat Unit Tests"
         key: "x-pack-filebeat-mandatory-linux-unit-test"
         command: |
@@ -86,39 +104,18 @@ steps:
           - github_commit_status:
               context: "x-pack/filebeat: Python Integration Tests"
 
-      - label: ":windows: x-pack/filebeat Win-2022 Unit Tests"
-        command: |
-          Set-Location -Path x-pack/filebeat
-          mage build unitTest
-        key: "x-pack-filebeat-mandatory-win-2022-unit-tests"
-        retry:
-          automatic:
-            - limit: 3
-        agents:
-          provider: "gcp"
-          image: "${IMAGE_WIN_2022}"
-          machine_type: "${GCP_WIN_MACHINE_TYPE}"
-          disk_size: 100
-          disk_type: "pd-ssd"
-        artifact_paths:
-          - "x-pack/filebeat/build/*.xml"
-          - "x-pack/filebeat/build/*.json"
-        notify:
-          - github_commit_status:
-              context: "x-pack/filebeat: Win-2022 Unit Tests"
-
-      - label: ":windows: x-pack/filebeat Win-2016 Unit Tests"
+      - label: ":windows: x-pack/filebeat Win-2019 Unit Tests"
         command: |
           Set-Location -Path x-pack/filebeat
           mage build unitTest
-        key: "x-pack-filebeat-mandatory-win-2016-unit-tests"
+        key: "x-pack-filebeat-extended-win-2019-unit-tests"
         retry:
           automatic:
             - limit: 3
         agents:
           provider: "gcp"
-          image: "${IMAGE_WIN_2016}"
-          machine_type: "${GCP_WIN_MACHINE_TYPE}"
+          image: "${IMAGE_WIN_2019}"
+          machineType: "${GCP_WIN_MACHINE_TYPE}"
           disk_size: 100
           disk_type: "pd-ssd"
         artifact_paths:
@@ -126,25 +123,25 @@ steps:
           - "x-pack/filebeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "x-pack/filebeat: Win-2016 Unit Tests"
+              context: "x-pack/filebeat: Win-2019 Unit Tests"
 
   - group: "x-pack/filebeat Extended Windows Tests"
     key: "x-pack-filebeat-extended-win-tests"
     if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/
 
     steps:
-      - label: ":windows: x-pack/filebeat Win-2019 Unit Tests"
+      - label: ":windows: x-pack/filebeat Win-2016 Unit Tests"
         command: |
           Set-Location -Path x-pack/filebeat
           mage build unitTest
-        key: "x-pack-filebeat-extended-win-2019-unit-tests"
+        key: "x-pack-filebeat-mandatory-win-2016-unit-tests"
         retry:
           automatic:
             - limit: 3
         agents:
           provider: "gcp"
-          image: "${IMAGE_WIN_2019}"
-          machineType: "${GCP_WIN_MACHINE_TYPE}"
+          image: "${IMAGE_WIN_2016}"
+          machine_type: "${GCP_WIN_MACHINE_TYPE}"
           disk_size: 100
           disk_type: "pd-ssd"
         artifact_paths:
@@ -152,7 +149,7 @@ steps:
           - "x-pack/filebeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "x-pack/filebeat: Win-2019 Unit Tests"
+              context: "x-pack/filebeat: Win-2016 Unit Tests"
 
       - label: ":windows: x-pack/filebeat Win 10 Unit Tests"
         command: |
@@ -175,48 +172,10 @@ steps:
           - github_commit_status:
               context: "x-pack/filebeat: Win 10 Unit Tests"
 
-      - label: ":windows: x-pack/filebeat Win 11 Unit Tests"
-        command: |
-          Set-Location -Path x-pack/filebeat
-          mage build unitTest
-        key: "x-pack-filebeat-extended-win-11-unit-tests"
-        retry:
-          automatic:
-            - limit: 3
-        agents:
-          provider: "gcp"
-          image: "${IMAGE_WIN_11}"
-          machineType: "${GCP_WIN_MACHINE_TYPE}"
-          disk_size: 100
-          disk_type: "pd-ssd"
-        artifact_paths:
-          - "x-pack/filebeat/build/*.xml"
-          - "x-pack/filebeat/build/*.json"
-        notify:
-          - github_commit_status:
-              context: "x-pack/filebeat: Win 11 Unit Tests"
-
   - group: "x-pack/filebeat Extended Tests"
     key: "x-pack-filebeat-extended-tests"
 
     steps:
-      - label: ":ubuntu: x-pack/filebeat Linux arm64 Unit Tests"
-        if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
-        key: "x-pack-filebeat-mandatory-linux-arm-unit-test"
-        command: |
-          cd x-pack/filebeat
-          mage build unitTest
-        agents:
-          provider: "aws"
-          imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
-          instanceType: "${AWS_ARM_INSTANCE_TYPE}"
-        artifact_paths:
-          - "x-pack/filebeat/build/*.xml"
-          - "x-pack/filebeat/build/*.json"
-        notify:
-          - github_commit_status:
-              context: "x-pack/filebeat: Linux arm64 Unit Tests"
-
       - label: ":mac: x-pack/filebeat macOS x86_64 Unit Tests"
         if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
         command: |

From 6df5384f5ef121153519c3515858095dd9859e63 Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Fri, 17 May 2024 14:42:50 +0300
Subject: [PATCH 4/7] updated xpack/metricbeat pipeline

---
 .../x-pack/pipeline.xpack.metricbeat.yml      | 92 +++----------------
 1 file changed, 15 insertions(+), 77 deletions(-)

diff --git a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml
index 447332f4a56..6646c681711 100644
--- a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml
+++ b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml
@@ -94,18 +94,18 @@ steps:
           - github_commit_status:
               context: "x-pack/metricbeat: Python (MODULE) Integration Tests"
 
-      - label: ":windows: Windows 2016 Unit Tests"
+      - label: ":windows: Windows 2019 Unit Tests"
         command: |
           Set-Location -Path x-pack/metricbeat
           mage build unitTest
-        key: "mandatory-win-2016-unit-tests"
+        key: "extended-win-2019-unit-tests"
         retry:
           automatic:
             - limit: 3
         agents:
           provider: "gcp"
-          image: "${IMAGE_WIN_2016}"
-          machine_type: "${GCP_WIN_MACHINE_TYPE}"
+          image: "${IMAGE_WIN_2019}"
+          machineType: "${GCP_WIN_MACHINE_TYPE}"
           disk_size: 100
           disk_type: "pd-ssd"
         artifact_paths:
@@ -113,19 +113,24 @@ steps:
           - "x-pack/metricbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "x-pack/metricbeat: Windows 2016 Unit Tests"
+              context: "x-pack/metricbeat: Windows 2019 Unit Tests"
 
-      - label: ":windows: Windows 2022 Unit Tests"
+  - group: "x-pack/metricbeat Extended Windows Tests"
+    key: "x-pack-metricbeat-extended-win-tests"
+    if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/
+
+    steps:
+      - label: ":windows: Windows 2016 Unit Tests"
         command: |
           Set-Location -Path x-pack/metricbeat
           mage build unitTest
-        key: "mandatory-win-2022-unit-tests"
+        key: "mandatory-win-2016-unit-tests"
         retry:
           automatic:
             - limit: 3
         agents:
           provider: "gcp"
-          image: "${IMAGE_WIN_2022}"
+          image: "${IMAGE_WIN_2016}"
           machine_type: "${GCP_WIN_MACHINE_TYPE}"
           disk_size: 100
           disk_type: "pd-ssd"
@@ -134,12 +139,8 @@ steps:
           - "x-pack/metricbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "x-pack/metricbeat: Windows 2022 Unit Tests"
+              context: "x-pack/metricbeat: Windows 2016 Unit Tests"
 
-  - group: "x-pack/metricbeat Extended Windows Tests"
-    key: "x-pack-metricbeat-extended-win-tests"
-    if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/
-    steps:
       - label: ":windows: Windows 10 Unit Tests"
         command: |
           Set-Location -Path x-pack/metricbeat
@@ -161,48 +162,6 @@ steps:
           - github_commit_status:
               context: "x-pack/metricbeat: Windows 10 Unit Tests"
 
-      - label: ":windows: Windows 11 Unit Tests"
-        command: |
-          Set-Location -Path x-pack/metricbeat
-          mage build unitTest
-        key: "extended-win-11-unit-tests"
-        retry:
-          automatic:
-            - limit: 3
-        agents:
-          provider: "gcp"
-          image: "${IMAGE_WIN_11}"
-          machineType: "${GCP_WIN_MACHINE_TYPE}"
-          disk_size: 100
-          disk_type: "pd-ssd"
-        artifact_paths:
-          - "x-pack/metricbeat/build/*.xml"
-          - "x-pack/metricbeat/build/*.json"
-        notify:
-          - github_commit_status:
-              context: "x-pack/metricbeat: Windows 11 Unit Tests"
-
-      - label: ":windows: Windows 2019 Unit Tests"
-        command: |
-          Set-Location -Path x-pack/metricbeat
-          mage build unitTest
-        key: "extended-win-2019-unit-tests"
-        retry:
-          automatic:
-            - limit: 3
-        agents:
-          provider: "gcp"
-          image: "${IMAGE_WIN_2019}"
-          machineType: "${GCP_WIN_MACHINE_TYPE}"
-          disk_size: 100
-          disk_type: "pd-ssd"
-        artifact_paths:
-          - "x-pack/metricbeat/build/*.xml"
-          - "x-pack/metricbeat/build/*.json"
-        notify:
-          - github_commit_status:
-              context: "x-pack/metricbeat: Windows 2019 Unit Tests"
-
   - group: "x-pack/metricbeat Extended Tests"
     key: "x-pack-metricbeat-extended-tests"
     if: build.env("BUILDKITE_PULL_REQUEST") == "false"  || build.env("GITHUB_PR_LABELS") =~ /.*(macOS|aws).*/
@@ -226,27 +185,6 @@ steps:
           - github_commit_status:
               context: "x-pack/metricbeat: MacOS x86_64 Unit Tests"
 
-      - label: ":mac: MacOS arm64 Unit Tests"
-        skip: "Skipping due to elastic/beats#33036"
-        # https://github.com/elastic/beats/issues/33036
-        if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
-        command: |
-          set -euo pipefail
-          source .buildkite/scripts/install_macos_tools.sh
-          cd x-pack/metricbeat && mage build unitTest
-        retry:
-          automatic:
-            - limit: 3
-        agents:
-          provider: "orka"
-          imagePrefix: "${IMAGE_MACOS_ARM}"
-        artifact_paths:
-          - "x-pack/metricbeat/build/*.xml"
-          - "x-pack/metricbeat/build/*.json"
-        notify:
-          - github_commit_status:
-              context: "x-pack/metricbeat: MacOS arm64 Unit Tests"
-
       - label: ":linux: Cloud (MODULE) Tests"
         key: "x-pack-metricbeat-extended-cloud-test"
         skip: "Not a stage on Jenkins & elastic/ingest-dev#3170"
@@ -291,7 +229,7 @@ steps:
           echo "~~~ Running tests"
           source .buildkite/scripts/setup_cloud_env.sh
           cd x-pack/metricbeat
-          mage build test goIntegTest
+          mage build test
         env:
           ASDF_TERRAFORM_VERSION: 1.0.2
           AWS_REGION: "eu-central-1"

From fd1e688309d68960798835a753958655cedc9658 Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Fri, 17 May 2024 14:44:11 +0300
Subject: [PATCH 5/7] removed macos arm tests

---
 .buildkite/x-pack/pipeline.xpack.filebeat.yml | 22 -------------------
 1 file changed, 22 deletions(-)

diff --git a/.buildkite/x-pack/pipeline.xpack.filebeat.yml b/.buildkite/x-pack/pipeline.xpack.filebeat.yml
index 51fb1d70a6e..188529d1468 100644
--- a/.buildkite/x-pack/pipeline.xpack.filebeat.yml
+++ b/.buildkite/x-pack/pipeline.xpack.filebeat.yml
@@ -196,28 +196,6 @@ steps:
           - github_commit_status:
               context: "x-pack/filebeat: macOS x86_64 Unit Tests"
 
-      - label: ":mac: x-pack/filebeat macOS arm64 Unit Tests"
-        skip: "Skipping due to elastic/beats#33036"
-        # https://github.com/elastic/beats/issues/33036
-        if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
-        command: |
-          set -euo pipefail
-          source .buildkite/scripts/install_macos_tools.sh
-          cd x-pack/filebeat
-          mage build unitTest
-        retry:
-          automatic:
-          - limit: 3
-        agents:
-          provider: "orka"
-          imagePrefix: "${IMAGE_MACOS_ARM}"
-        artifact_paths:
-          - "x-pack/filebeat/build/*.xml"
-          - "x-pack/filebeat/build/*.json"
-        notify:
-          - github_commit_status:
-              context: "x-pack/filebeat: macOS arm64 Unit Tests"
-
       - label: ":ubuntu: x-pack/filebeat Cloud (MODULE) Tests"
         key: "x-pack-filebeat-extended-cloud-test"
         if: build.env("GITHUB_PR_LABELS") =~ /.*aws.*/

From 057305ba5d7d24f2ad988b7a9faab48fb8948c02 Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Fri, 17 May 2024 14:48:36 +0300
Subject: [PATCH 6/7] added missng module compat tests

---
 .buildkite/x-pack/pipeline.xpack.filebeat.yml | 27 +++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/.buildkite/x-pack/pipeline.xpack.filebeat.yml b/.buildkite/x-pack/pipeline.xpack.filebeat.yml
index 188529d1468..f4b23e4af6a 100644
--- a/.buildkite/x-pack/pipeline.xpack.filebeat.yml
+++ b/.buildkite/x-pack/pipeline.xpack.filebeat.yml
@@ -104,6 +104,33 @@ steps:
           - github_commit_status:
               context: "x-pack/filebeat: Python Integration Tests"
 
+      - label: "Filebeat: Module compat tests - previous minor"
+        # Run module integration tests under previous minor of Elastic stack.
+        env:
+          STACK_ENVIRONMENT: "prev-minor"
+          TESTING_FILEBEAT_SKIP_DIFF: 1
+          PYTEST_ADDOPTS: "-k test_xpack_modules"
+        command: |
+          set -euo pipefail
+          source .buildkite/scripts/changesets.sh
+          defineModuleFromTheChangeSet x-pack/filebeat
+          echo "~~~ Running tests"
+          cd filebeat
+          mage pythonIntegTest
+        retry:
+          automatic:
+            - limit: 3
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+        artifact_paths:
+          - "filebeat/build/*.xml"
+          - "filebeat/build/*.json"
+        notify:
+          - github_commit_status:
+              context: "filebeat: Module compat tests / previous minor"
+
       - label: ":windows: x-pack/filebeat Win-2019 Unit Tests"
         command: |
           Set-Location -Path x-pack/filebeat

From d0d571e85988cdec8d3a054456f5cab7a45990b0 Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Fri, 17 May 2024 16:27:26 +0300
Subject: [PATCH 7/7] changed module compat tests dir

---
 .buildkite/x-pack/pipeline.xpack.filebeat.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/x-pack/pipeline.xpack.filebeat.yml b/.buildkite/x-pack/pipeline.xpack.filebeat.yml
index f4b23e4af6a..ed249879a8f 100644
--- a/.buildkite/x-pack/pipeline.xpack.filebeat.yml
+++ b/.buildkite/x-pack/pipeline.xpack.filebeat.yml
@@ -115,7 +115,7 @@ steps:
           source .buildkite/scripts/changesets.sh
           defineModuleFromTheChangeSet x-pack/filebeat
           echo "~~~ Running tests"
-          cd filebeat
+          cd x-pack/filebeat
           mage pythonIntegTest
         retry:
           automatic: