From 0ea0b6bbee60f030dfca53d2e203d4649a703b6c Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Wed, 2 Oct 2024 11:50:20 +0300
Subject: [PATCH 01/17] adjusted pipeline

---
 .buildkite/scripts/agentbeat/prepare_env.py   |  57 +++++
 .../x-pack/pipeline.xpack.agentbeat.yml       | 202 +++++++++++-------
 2 files changed, 184 insertions(+), 75 deletions(-)
 create mode 100755 .buildkite/scripts/agentbeat/prepare_env.py

diff --git a/.buildkite/scripts/agentbeat/prepare_env.py b/.buildkite/scripts/agentbeat/prepare_env.py
new file mode 100755
index 000000000000..50f7d8123299
--- /dev/null
+++ b/.buildkite/scripts/agentbeat/prepare_env.py
@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+
+import os
+import platform
+import subprocess
+
+
+def get_os() -> str:
+    return platform.system()
+
+
+def get_arch() -> str:
+    return platform.machine()
+
+
+def get_cwd() -> str:
+    return os.getcwd()
+
+
+def download_agentbeat_artifact(os, arch):
+    pattern = "x-pack/agentbeat/build/distributions/agentbeat-9.0.0-SNAPSHOT-linux-x86_64.tar.gz"
+    # pattern = "x-pack/agentbeat/build/distributions/**"
+    # command = f"buildkite-agent artifact download \"{pattern}\" . --step 'agentbeat-package-linux'"
+
+    try:
+        print("--- Downloading agentbeat artifact")
+        result = subprocess.run(
+            ["buildkite-agent", "artifact", "download", pattern, ".",
+             "--build", "01924d2b-b061-45ae-a106-e885584ff26f",
+             "--step", "agentbeat-package-linux"],
+            check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        print(result.stdout.decode())
+    except subprocess.CalledProcessError as e:
+        print("--- Error occurred while downloading agentbeat\n" + e.stderr)
+        exit(1)
+
+
+def install_synthetics():
+    try:
+        print("--- Installing @elastic/synthetics")
+        subprocess.run(
+            ["npm install -g @elastic/synthetics"],
+            check=True
+        )
+    except subprocess.CalledProcessError:
+        print("Failed to install @elastic/synthetics")
+        exit(1)
+
+
+# print("--- OS: " + get_os())
+#
+# print("--- ARCH: " + get_arch())
+#
+# print("--- CWD: " + get_cwd())
+
+download_agentbeat_artifact(get_os(), get_arch())
+# install_synthetics()
diff --git a/.buildkite/x-pack/pipeline.xpack.agentbeat.yml b/.buildkite/x-pack/pipeline.xpack.agentbeat.yml
index ef7cb1598aa4..0af413435cbc 100644
--- a/.buildkite/x-pack/pipeline.xpack.agentbeat.yml
+++ b/.buildkite/x-pack/pipeline.xpack.agentbeat.yml
@@ -1,101 +1,153 @@
 env:
   ASDF_MAGE_VERSION: 1.15.0
+  ASDF_NODEJS_VERSION: 18.17.1
+
   GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
   IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
 
+  IMAGE_MACOS_ARM: "generic-13-ventura-arm"
+  IMAGE_MACOS_X86_64: "generic-13-ventura-x64"
+
+  IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022"
+
   IMAGE_BEATS_WITH_HOOKS_LATEST: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest"
 
 steps:
-  - group: "Check/Update"
-    key: "x-pack-agentbeat-check-update"
-
-    steps:
-      - label: "agentbeat: Run pre-commit"
-        command: "pre-commit run --all-files"
-        agents:
-          image: "${IMAGE_BEATS_WITH_HOOKS_LATEST}"
-          memory: "2Gi"
-          useCustomGlobalHooks: true
-        notify:
-          - github_commit_status:
-              context: "agentbeat: pre-commit"
+#  - group: "Check/Update"
+#    key: "x-pack-agentbeat-check-update"
+#
+#    steps:
+#      - label: "agentbeat: Run pre-commit"
+#        command: "pre-commit run --all-files"
+#        agents:
+#          image: "${IMAGE_BEATS_WITH_HOOKS_LATEST}"
+#          memory: "2Gi"
+#          useCustomGlobalHooks: true
+#        notify:
+#          - github_commit_status:
+#              context: "agentbeat: pre-commit"
 
-  - wait: ~
-    # with PRs, we want to run mandatory tests only if check/update step succeed
-    # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests
-    # this allows building DRA artifacts even if there is flakiness in check/update step
-    if: build.env("BUILDKITE_PULL_REQUEST") != "false"
-    depends_on: "x-pack-agentbeat-check-update"
+#  - wait: ~
+#    # with PRs, we want to run mandatory tests only if check/update step succeed
+#    # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests
+#    # this allows building DRA artifacts even if there is flakiness in check/update step
+#    if: build.env("BUILDKITE_PULL_REQUEST") != "false"
+#    depends_on: "x-pack-agentbeat-check-update"
 
   - group: "Agentbeat tests"
     key: "agentbeat-mandatory-tests"
 
     steps:
-      - label: ":linux: Agentbeat packaging Linux"
-        key: "agentbeat-package-linux"
-        env:
-          PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
-          SNAPSHOT: true
-        command: |
-          set -euo pipefail
-          cd x-pack/agentbeat
-          mage package
-        artifact_paths:
-          - x-pack/agentbeat/build/distributions/**/*
-          - "x-pack/agentbeat/build/*.xml"
-          - "x-pack/agentbeat/build/*.json"
-        retry:
-          automatic:
-            - limit: 2
-        timeout_in_minutes: 60
-        agents:
-          provider: "gcp"
-          image: "${IMAGE_UBUNTU_X86_64}"
-          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
-          disk_size: 100
-          disk_type: "pd-ssd"
-        notify:
-          - github_commit_status:
-              context: "agentbeat: Packaging"
+#      - label: ":linux: Agentbeat packaging Linux"
+#        key: "agentbeat-package-linux"
+#        env:
+#          PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
+#          SNAPSHOT: true
+#        command: |
+#          set -euo pipefail
+#          cd x-pack/agentbeat
+#          mage package
+#        artifact_paths:
+#          - x-pack/agentbeat/build/distributions/**/*
+#          - "x-pack/agentbeat/build/*.xml"
+#          - "x-pack/agentbeat/build/*.json"
+#        retry:
+#          automatic:
+#            - limit: 2
+#        timeout_in_minutes: 60
+#        agents:
+#          provider: "gcp"
+#          image: "${IMAGE_UBUNTU_X86_64}"
+#          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+#          disk_size: 100
+#          disk_type: "pd-ssd"
+#        notify:
+#          - github_commit_status:
+#              context: "agentbeat: Packaging"
+
+      #      - label: ":linux: Agentbeat/Integration tests Linux"
+      #        key: "agentbeat-it-linux"
+      #        depends_on:
+      #          - agentbeat-package-linux
+      #        env:
+      #          ASDF_NODEJS_VERSION: 18.17.1
+      #          PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
+      #          SNAPSHOT: true
+      #        command: |
+      #          set -euo pipefail
+      #          echo "~~~ Downloading artifacts"
+      #          buildkite-agent artifact download x-pack/agentbeat/build/distributions/** . --step 'agentbeat-package-linux'
+      #          ls -lah x-pack/agentbeat/build/distributions/
+      #          echo "~~~ Installing @elastic/synthetics with npm"
+      #          npm install -g @elastic/synthetics
+      #          echo "~~~ Running tests"
+      #          cd x-pack/agentbeat
+      #          mage goIntegTest
+      #        artifact_paths:
+      #          - x-pack/agentbeat/build/distributions/**/*
+      #          - "x-pack/agentbeat/build/*.xml"
+      #          - "x-pack/agentbeat/build/*.json"
+      #        retry:
+      #          automatic:
+      #            - limit: 1
+      #        timeout_in_minutes: 60
+      #        agents:
+      #          provider: "gcp"
+      #          image: "${IMAGE_UBUNTU_X86_64}"
+      #          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+      #          disk_size: 100
+      #          disk_type: "pd-ssd"
+      #        notify:
+      #          - github_commit_status:
+      #              context: "agentbeat: Integration tests"
 
       - label: ":linux: Agentbeat/Integration tests Linux"
         key: "agentbeat-it-linux"
-        depends_on:
-          - agentbeat-package-linux
-        env:
-          ASDF_NODEJS_VERSION: 18.17.1
-          PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
-          SNAPSHOT: true
+#        depends_on:
+#          - agentbeat-package-linux
         command: |
           set -euo pipefail
-          echo "~~~ Downloading artifacts"
-          buildkite-agent artifact download x-pack/agentbeat/build/distributions/** . --step 'agentbeat-package-linux'
+          # buildkite-agent artifact download x-pack/agentbeat/build/distributions/** . --build 01924d2b-b061-45ae-a106-e885584ff26f --step agentbeat-package-linux
+          ./.buildkite/scripts/agentbeat/prepare_env.py
           ls -lah x-pack/agentbeat/build/distributions/
-          echo "~~~ Installing @elastic/synthetics with npm"
-          npm install -g @elastic/synthetics
-          echo "~~~ Running tests"
-          cd x-pack/agentbeat
-          mage goIntegTest
-        artifact_paths:
-          - x-pack/agentbeat/build/distributions/**/*
-          - "x-pack/agentbeat/build/*.xml"
-          - "x-pack/agentbeat/build/*.json"
-        plugins:
-          - test-collector#v1.10.2:
-              files: "x-pack/agentbeat/build/TEST-*.xml"
-              format: "junit"
-              branches: "main"
-              debug: true
-        retry:
-          automatic:
-            - limit: 1
-        timeout_in_minutes: 60
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
           machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
           disk_size: 100
           disk_type: "pd-ssd"
-        notify:
-          - github_commit_status:
-              context: "agentbeat: Integration tests"
+
+#      - label: ":linux: Agentbeat/Integration tests Windows"
+#        key: "agentbeat-it-windows"
+#        depends_on:
+#          - agentbeat-package-linux
+#        commands:
+#          - ./.buildkite/scripts/agentbeat/prepare_env.py
+#        agents:
+#          provider: "gcp"
+#          image: "${IMAGE_WIN_2022}"
+#          machine_type: "${GCP_WIN_MACHINE_TYPE}"
+#          disk_size: 200
+#          disk_type: "pd-ssd"
+#
+#      - label: ":linux: Agentbeat/Integration tests macOS"
+#        key: "agentbeat-it-macos"
+#        depends_on:
+#          - agentbeat-package-linux
+#        command: |
+#          set -euo pipefail
+#          ./.buildkite/scripts/agentbeat/prepare_env.py
+#        agents:
+#          provider: "orka"
+#          imagePrefix: "${IMAGE_MACOS_X86_64}"
+#
+#      - label: ":linux: Agentbeat/Integration tests macOS arm64"
+#        key: "agentbeat-it-macos-arm"
+#        depends_on:
+#          - agentbeat-package-linux
+#        command: |
+#          set -euo pipefail
+#          ./.buildkite/scripts/agentbeat/prepare_env.py
+#        agents:
+#          provider: "orka"
+#          imagePrefix: "${IMAGE_MACOS_ARM}"

From 953d4f5af0a15cf517eae92fde944068734a0ce6 Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Wed, 2 Oct 2024 17:55:27 +0300
Subject: [PATCH 02/17] updated artifact pattern

---
 .buildkite/scripts/agentbeat/prepare_env.py   | 42 +++++++++----------
 .../x-pack/pipeline.xpack.agentbeat.yml       | 22 +++++-----
 2 files changed, 30 insertions(+), 34 deletions(-)

diff --git a/.buildkite/scripts/agentbeat/prepare_env.py b/.buildkite/scripts/agentbeat/prepare_env.py
index 50f7d8123299..11d11bc56cda 100755
--- a/.buildkite/scripts/agentbeat/prepare_env.py
+++ b/.buildkite/scripts/agentbeat/prepare_env.py
@@ -1,43 +1,47 @@
 #!/usr/bin/env python3
 
-import os
 import platform
 import subprocess
+import sys
 
 
 def get_os() -> str:
-    return platform.system()
+    return platform.system().lower()
 
 
 def get_arch() -> str:
-    return platform.machine()
+    arch = platform.machine().lower()
 
+    if arch == "amd64":
+        return "x86_64"
+    else:
+        return arch
 
-def get_cwd() -> str:
-    return os.getcwd()
 
+def download_agentbeat_artifact(agent_os, agent_arch):
+    pattern = f"x-pack/agentbeat/build/distributions/agentbeat-*-{agent_os}-{agent_arch}.tar.gz"
 
-def download_agentbeat_artifact(os, arch):
-    pattern = "x-pack/agentbeat/build/distributions/agentbeat-9.0.0-SNAPSHOT-linux-x86_64.tar.gz"
-    # pattern = "x-pack/agentbeat/build/distributions/**"
-    # command = f"buildkite-agent artifact download \"{pattern}\" . --step 'agentbeat-package-linux'"
+    print("--- Downloading agentbeat artifact")
 
     try:
-        print("--- Downloading agentbeat artifact")
-        result = subprocess.run(
+        subprocess.run(
             ["buildkite-agent", "artifact", "download", pattern, ".",
              "--build", "01924d2b-b061-45ae-a106-e885584ff26f",
              "--step", "agentbeat-package-linux"],
-            check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        print(result.stdout.decode())
+            check=True, stdout=sys.stdout, stderr=subprocess.PIPE, text=True)
     except subprocess.CalledProcessError as e:
-        print("--- Error occurred while downloading agentbeat\n" + e.stderr)
+        print("--- Error occurred. Failed to download agentbeat: \n" + e.stderr)
         exit(1)
 
 
+def unzip_agentbeat():
+     print("todo unzip")
+
+
 def install_synthetics():
+    print("--- Installing @elastic/synthetics")
+
     try:
-        print("--- Installing @elastic/synthetics")
         subprocess.run(
             ["npm install -g @elastic/synthetics"],
             check=True
@@ -46,12 +50,6 @@ def install_synthetics():
         print("Failed to install @elastic/synthetics")
         exit(1)
 
-
-# print("--- OS: " + get_os())
-#
-# print("--- ARCH: " + get_arch())
-#
-# print("--- CWD: " + get_cwd())
-
+print("--- OS Data: " + get_os() + " " + get_arch())
 download_agentbeat_artifact(get_os(), get_arch())
 # install_synthetics()
diff --git a/.buildkite/x-pack/pipeline.xpack.agentbeat.yml b/.buildkite/x-pack/pipeline.xpack.agentbeat.yml
index 0af413435cbc..cfed290ffba7 100644
--- a/.buildkite/x-pack/pipeline.xpack.agentbeat.yml
+++ b/.buildkite/x-pack/pipeline.xpack.agentbeat.yml
@@ -107,9 +107,7 @@ steps:
 #          - agentbeat-package-linux
         command: |
           set -euo pipefail
-          # buildkite-agent artifact download x-pack/agentbeat/build/distributions/** . --build 01924d2b-b061-45ae-a106-e885584ff26f --step agentbeat-package-linux
           ./.buildkite/scripts/agentbeat/prepare_env.py
-          ls -lah x-pack/agentbeat/build/distributions/
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -117,18 +115,18 @@ steps:
           disk_size: 100
           disk_type: "pd-ssd"
 
-#      - label: ":linux: Agentbeat/Integration tests Windows"
-#        key: "agentbeat-it-windows"
+      - label: ":linux: Agentbeat/Integration tests Windows"
+        key: "agentbeat-it-windows"
 #        depends_on:
 #          - agentbeat-package-linux
-#        commands:
-#          - ./.buildkite/scripts/agentbeat/prepare_env.py
-#        agents:
-#          provider: "gcp"
-#          image: "${IMAGE_WIN_2022}"
-#          machine_type: "${GCP_WIN_MACHINE_TYPE}"
-#          disk_size: 200
-#          disk_type: "pd-ssd"
+        commands:
+          - ./.buildkite/scripts/agentbeat/prepare_env.py
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_WIN_2022}"
+          machine_type: "${GCP_WIN_MACHINE_TYPE}"
+          disk_size: 200
+          disk_type: "pd-ssd"
 #
 #      - label: ":linux: Agentbeat/Integration tests macOS"
 #        key: "agentbeat-it-macos"

From 69713ba9c37a076eddde084c9a27f76200a464b8 Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Thu, 3 Oct 2024 12:35:46 +0300
Subject: [PATCH 03/17] added stderr

---
 .buildkite/scripts/agentbeat/prepare_env.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.buildkite/scripts/agentbeat/prepare_env.py b/.buildkite/scripts/agentbeat/prepare_env.py
index 11d11bc56cda..028f76476fbd 100755
--- a/.buildkite/scripts/agentbeat/prepare_env.py
+++ b/.buildkite/scripts/agentbeat/prepare_env.py
@@ -28,9 +28,9 @@ def download_agentbeat_artifact(agent_os, agent_arch):
             ["buildkite-agent", "artifact", "download", pattern, ".",
              "--build", "01924d2b-b061-45ae-a106-e885584ff26f",
              "--step", "agentbeat-package-linux"],
-            check=True, stdout=sys.stdout, stderr=subprocess.PIPE, text=True)
+            check=True, stdout=sys.stdout, stderr=sys.stderr, text=True)
     except subprocess.CalledProcessError as e:
-        print("--- Error occurred. Failed to download agentbeat: \n" + e.stderr)
+        print("Error occurred. Failed to download agentbeat: \n" + e.stderr)
         exit(1)
 
 

From c4ae8438604b30803470bea5cd84c84ebe332650 Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Thu, 3 Oct 2024 12:42:42 +0300
Subject: [PATCH 04/17] added stderr

---
 .buildkite/scripts/agentbeat/prepare_env.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.buildkite/scripts/agentbeat/prepare_env.py b/.buildkite/scripts/agentbeat/prepare_env.py
index 028f76476fbd..c3dbfd0977dd 100755
--- a/.buildkite/scripts/agentbeat/prepare_env.py
+++ b/.buildkite/scripts/agentbeat/prepare_env.py
@@ -28,14 +28,14 @@ def download_agentbeat_artifact(agent_os, agent_arch):
             ["buildkite-agent", "artifact", "download", pattern, ".",
              "--build", "01924d2b-b061-45ae-a106-e885584ff26f",
              "--step", "agentbeat-package-linux"],
-            check=True, stdout=sys.stdout, stderr=sys.stderr, text=True)
+            check=True, stdout=sys.stdout, stderr=subprocess.PIPE, text=True)
     except subprocess.CalledProcessError as e:
         print("Error occurred. Failed to download agentbeat: \n" + e.stderr)
         exit(1)
 
 
 def unzip_agentbeat():
-     print("todo unzip")
+    print("todo unzip")
 
 
 def install_synthetics():

From 69bd82f9717f292dcb8ee4c5e09caa479e977565 Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Thu, 3 Oct 2024 13:00:13 +0300
Subject: [PATCH 05/17] added stderr

---
 .buildkite/scripts/agentbeat/prepare_env.py | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/.buildkite/scripts/agentbeat/prepare_env.py b/.buildkite/scripts/agentbeat/prepare_env.py
index c3dbfd0977dd..6a5c0fd487c9 100755
--- a/.buildkite/scripts/agentbeat/prepare_env.py
+++ b/.buildkite/scripts/agentbeat/prepare_env.py
@@ -18,6 +18,13 @@ def get_arch() -> str:
         return arch
 
 
+def get_artifact_extension(agent_os) -> str:
+    if agent_os == "windows":
+        return "zip"
+    else:
+        return "tar.gz"
+
+
 def download_agentbeat_artifact(agent_os, agent_arch):
     pattern = f"x-pack/agentbeat/build/distributions/agentbeat-*-{agent_os}-{agent_arch}.tar.gz"
 
@@ -28,9 +35,9 @@ def download_agentbeat_artifact(agent_os, agent_arch):
             ["buildkite-agent", "artifact", "download", pattern, ".",
              "--build", "01924d2b-b061-45ae-a106-e885584ff26f",
              "--step", "agentbeat-package-linux"],
-            check=True, stdout=sys.stdout, stderr=subprocess.PIPE, text=True)
-    except subprocess.CalledProcessError as e:
-        print("Error occurred. Failed to download agentbeat: \n" + e.stderr)
+            check=True, stdout=sys.stdout, stderr=sys.stderr, text=True)
+    except subprocess.CalledProcessError:
+        print("Failed to download agentbeat")
         exit(1)
 
 

From 618f2d7159dd38e87de9596d91c5a298fa2d7486 Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Thu, 3 Oct 2024 13:21:00 +0300
Subject: [PATCH 06/17] test error otput

---
 .buildkite/scripts/agentbeat/prepare_env.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/.buildkite/scripts/agentbeat/prepare_env.py b/.buildkite/scripts/agentbeat/prepare_env.py
index 6a5c0fd487c9..db42e64ee8fc 100755
--- a/.buildkite/scripts/agentbeat/prepare_env.py
+++ b/.buildkite/scripts/agentbeat/prepare_env.py
@@ -26,9 +26,9 @@ def get_artifact_extension(agent_os) -> str:
 
 
 def download_agentbeat_artifact(agent_os, agent_arch):
-    pattern = f"x-pack/agentbeat/build/distributions/agentbeat-*-{agent_os}-{agent_arch}.tar.gz"
-
-    print("--- Downloading agentbeat artifact")
+    print(" ")
+    extension = get_artifact_extension(agent_os)
+    pattern = f"x-pack/agentbeat/build/distributions/agentbeat-*-{agent_os}-{agent_arch}.{extension}"
 
     try:
         subprocess.run(
@@ -37,12 +37,17 @@ def download_agentbeat_artifact(agent_os, agent_arch):
              "--step", "agentbeat-package-linux"],
             check=True, stdout=sys.stdout, stderr=sys.stderr, text=True)
     except subprocess.CalledProcessError:
-        print("Failed to download agentbeat")
         exit(1)
 
 
 def unzip_agentbeat():
     print("todo unzip")
+    try:
+        subprocess.run(
+            ["unzip"],
+            check=True, stdout=sys.stdout, stderr=sys.stderr, text=True)
+    except subprocess.CalledProcessError:
+        exit(1)
 
 
 def install_synthetics():

From 68925207cd5dbd996a0aa7e706bf9402d4a51ed4 Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Thu, 3 Oct 2024 13:51:23 +0300
Subject: [PATCH 07/17] added unzip

---
 .buildkite/scripts/agentbeat/prepare_env.py   | 107 ++++++++++++++----
 .../x-pack/pipeline.xpack.agentbeat.yml       |   4 +
 dev-tools/mage/parse_spec.go                  |  98 ++++++++++++++++
 x-pack/agentbeat/magefile.go                  |   4 +
 4 files changed, 192 insertions(+), 21 deletions(-)
 create mode 100644 dev-tools/mage/parse_spec.go

diff --git a/.buildkite/scripts/agentbeat/prepare_env.py b/.buildkite/scripts/agentbeat/prepare_env.py
index db42e64ee8fc..4bda517516ba 100755
--- a/.buildkite/scripts/agentbeat/prepare_env.py
+++ b/.buildkite/scripts/agentbeat/prepare_env.py
@@ -3,7 +3,20 @@
 import platform
 import subprocess
 import sys
+import tarfile
+import os
+import re
 
+PATH = 'x-pack/agentbeat/build/distributions'
+
+
+def log(msg):
+    sys.stdout.write(f'{msg}\n')
+    sys.stdout.flush()
+
+def log_err(msg):
+    sys.stderr.write(f'{msg}\n')
+    sys.stderr.flush()
 
 def get_os() -> str:
     return platform.system().lower()
@@ -12,56 +25,108 @@ def get_os() -> str:
 def get_arch() -> str:
     arch = platform.machine().lower()
 
-    if arch == "amd64":
-        return "x86_64"
+    if arch == 'amd64':
+        return 'x86_64'
     else:
         return arch
 
 
 def get_artifact_extension(agent_os) -> str:
-    if agent_os == "windows":
-        return "zip"
+    if agent_os == 'windows':
+        return 'zip'
     else:
-        return "tar.gz"
+        return 'tar.gz'
 
 
-def download_agentbeat_artifact(agent_os, agent_arch):
-    print(" ")
+def get_artifact_pattern() -> str:
+    agent_os = get_os()
+    agent_arch = get_arch()
     extension = get_artifact_extension(agent_os)
-    pattern = f"x-pack/agentbeat/build/distributions/agentbeat-*-{agent_os}-{agent_arch}.{extension}"
+    return f'{PATH}/agentbeat-*-{agent_os}-{agent_arch}.{extension}'
 
+
+def download_agentbeat(pattern, path) -> str:
+    log('--- Downloading agentbeat')
     try:
         subprocess.run(
-            ["buildkite-agent", "artifact", "download", pattern, ".",
-             "--build", "01924d2b-b061-45ae-a106-e885584ff26f",
-             "--step", "agentbeat-package-linux"],
+            ['buildkite-agent', 'artifact', 'download', pattern, '.',
+             '--build', '01924d2b-b061-45ae-a106-e885584ff26f',
+             '--step', 'agentbeat-package-linux'],
             check=True, stdout=sys.stdout, stderr=sys.stderr, text=True)
     except subprocess.CalledProcessError:
         exit(1)
 
+    return get_filename(path)
+
+
+def get_filename(path) -> str:
+    try:
+        out = subprocess.run(
+            ['ls', '-p', path],
+            check=True, capture_output=True, text=True)
+        return out.stdout.strip()
+    except subprocess.CalledProcessError:
+        exit(1)
+
+
+def extract_agentbeat(filename):
+    log('~~~ Extracting agentbeat')
+    filepath = PATH + '/' + filename
+
+    if filepath.endswith('.zip'):
+        unzip_agentbeat(filepath)
+    else:
+        untar_agentbeat(filepath)
+    log('Successfully extracted agentbeat')
+
 
-def unzip_agentbeat():
-    print("todo unzip")
+def unzip_agentbeat(filepath):
     try:
         subprocess.run(
-            ["unzip"],
+            ['unzip', filepath],
             check=True, stdout=sys.stdout, stderr=sys.stderr, text=True)
-    except subprocess.CalledProcessError:
+    except subprocess.CalledProcessError as e:
+        log_err(e)
+        exit(1)
+
+
+def untar_agentbeat(filepath):
+    try:
+        with tarfile.open(filepath, 'r:gz') as tar:
+            tar.list()
+            tar.extractall()
+    except Exception as e:
+        log_err(e)
+        exit(1)
+
+
+def add_to_path(filepath):
+    pattern = r'(.*)(?=\.zip|.tar\.gz)'
+    match = re.match(pattern, filepath)
+    if match:
+        path = f'../build/distributions/{match.group(1)}/agentbeat'
+        log('--- AGENTBEAT_PATH: ' + path)
+        os.environ['AGENTBEAT_PATH'] = path
+    else:
+        log_err("No agentbeat executable found")
         exit(1)
 
 
 def install_synthetics():
-    print("--- Installing @elastic/synthetics")
+    log('--- Installing @elastic/synthetics')
 
     try:
         subprocess.run(
-            ["npm install -g @elastic/synthetics"],
+            ['npm', 'install', '-g', '@elastic/synthetics'],
             check=True
         )
     except subprocess.CalledProcessError:
-        print("Failed to install @elastic/synthetics")
+        log_err('Failed to install @elastic/synthetics')
         exit(1)
 
-print("--- OS Data: " + get_os() + " " + get_arch())
-download_agentbeat_artifact(get_os(), get_arch())
-# install_synthetics()
+
+artifact_pattern = get_artifact_pattern()
+archive = download_agentbeat(artifact_pattern, PATH)
+extract_agentbeat(archive)
+add_to_path(archive)
+install_synthetics()
diff --git a/.buildkite/x-pack/pipeline.xpack.agentbeat.yml b/.buildkite/x-pack/pipeline.xpack.agentbeat.yml
index cfed290ffba7..71607aefb250 100644
--- a/.buildkite/x-pack/pipeline.xpack.agentbeat.yml
+++ b/.buildkite/x-pack/pipeline.xpack.agentbeat.yml
@@ -12,6 +12,8 @@ env:
 
   IMAGE_BEATS_WITH_HOOKS_LATEST: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest"
 
+  AGENTBEAT_SPEC: "../agentbeat.spec.yml"
+
 steps:
 #  - group: "Check/Update"
 #    key: "x-pack-agentbeat-check-update"
@@ -103,6 +105,8 @@ steps:
 
       - label: ":linux: Agentbeat/Integration tests Linux"
         key: "agentbeat-it-linux"
+        env:
+          PLATFORM: "linux/amd64"
 #        depends_on:
 #          - agentbeat-package-linux
         command: |
diff --git a/dev-tools/mage/parse_spec.go b/dev-tools/mage/parse_spec.go
new file mode 100644
index 000000000000..5dc20199288d
--- /dev/null
+++ b/dev-tools/mage/parse_spec.go
@@ -0,0 +1,98 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package mage
+
+import (
+	"gopkg.in/yaml.v2"
+	"log"
+	"os"
+)
+
+type spec struct {
+	Inputs []input
+}
+
+type input struct {
+	Name        string
+	Description string
+	Platforms   []string
+	Command     command
+}
+
+type command struct {
+	Name string
+	Args []string
+}
+
+// ParseSpec parses agent.beat.spec.yml and generates test command
+func ParseSpec() {
+	specPath := os.Getenv("AGENTBEAT_SPEC")
+	if specPath == "" {
+		log.Fatal("AGENTBEAT_SPEC is not defined")
+	}
+
+	platform := os.Getenv("PLATFORM")
+	if platform == "" {
+		log.Fatal("PLATFORM is not defined")
+	}
+
+	spec, err := parseToObj()
+	if err != nil {
+		log.Fatalf("Error parsing agentbeat.spec.yml: %v", err)
+	}
+
+	inputList := filter(spec.Inputs, func(input input) bool {
+		return contains(input.Platforms, platform)
+	})
+
+	log.Print(inputList)
+}
+
+func parseToObj() (spec, error) {
+	specFile, err := os.ReadFile("../agentbeat.spec.yml")
+	if err != nil {
+		log.Fatalf("Error opening agentbeat.spec.yml: %v", err)
+		return spec{}, err
+	}
+	var spec spec
+	err = yaml.Unmarshal(specFile, &spec)
+	if err != nil {
+		log.Fatalf("Error parsing agentbeat.spec.yml: %v", err)
+		return spec, err
+	}
+	return spec, nil
+}
+
+func filter[T any](slice []T, condition func(T) bool) []T {
+	var result []T
+	for _, v := range slice {
+		if condition(v) {
+			result = append(result, v)
+		}
+	}
+	return result
+}
+
+func contains(slice []string, item string) bool {
+	for _, v := range slice {
+		if v == item {
+			return true
+		}
+	}
+	return false
+}
diff --git a/x-pack/agentbeat/magefile.go b/x-pack/agentbeat/magefile.go
index b65a3e59af36..097695552a7c 100644
--- a/x-pack/agentbeat/magefile.go
+++ b/x-pack/agentbeat/magefile.go
@@ -213,3 +213,7 @@ func PythonIntegTest(ctx context.Context) error {
 	mg.Deps(BuildSystemTestBinary)
 	return devtools.PythonIntegTestFromHost(devtools.DefaultPythonTestIntegrationFromHostArgs())
 }
+
+func TestWithSpecs(ctx context.Context) {
+	mg.Deps(devtools.ParseSpec)
+}

From 4b59d7d3382a5a44367a5c9dde8124bd875fefaa Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Tue, 8 Oct 2024 16:47:32 +0300
Subject: [PATCH 08/17] added TestWithSpec to magefile

---
 .buildkite/scripts/agentbeat/prepare_env.py   |  4 +-
 .../x-pack/pipeline.xpack.agentbeat.yml       | 10 ++-
 dev-tools/mage/{parse_spec.go => spec.go}     | 31 ++++++---
 x-pack/agentbeat/magefile.go                  | 66 ++++++++++++++++++-
 4 files changed, 93 insertions(+), 18 deletions(-)
 rename dev-tools/mage/{parse_spec.go => spec.go} (75%)

diff --git a/.buildkite/scripts/agentbeat/prepare_env.py b/.buildkite/scripts/agentbeat/prepare_env.py
index 4bda517516ba..d17c3cdd0e10 100755
--- a/.buildkite/scripts/agentbeat/prepare_env.py
+++ b/.buildkite/scripts/agentbeat/prepare_env.py
@@ -105,8 +105,8 @@ def add_to_path(filepath):
     match = re.match(pattern, filepath)
     if match:
         path = f'../build/distributions/{match.group(1)}/agentbeat'
-        log('--- AGENTBEAT_PATH: ' + path)
-        os.environ['AGENTBEAT_PATH'] = path
+        log("--- PATH: " + str(path))
+        os.environ['AGENTBEAT_PATH'] = str(path)
     else:
         log_err("No agentbeat executable found")
         exit(1)
diff --git a/.buildkite/x-pack/pipeline.xpack.agentbeat.yml b/.buildkite/x-pack/pipeline.xpack.agentbeat.yml
index 71607aefb250..b6225577b846 100644
--- a/.buildkite/x-pack/pipeline.xpack.agentbeat.yml
+++ b/.buildkite/x-pack/pipeline.xpack.agentbeat.yml
@@ -12,7 +12,7 @@ env:
 
   IMAGE_BEATS_WITH_HOOKS_LATEST: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest"
 
-  AGENTBEAT_SPEC: "../agentbeat.spec.yml"
+  AGENTBEAT_SPEC: "./agentbeat.spec.yml"
 
 steps:
 #  - group: "Check/Update"
@@ -112,6 +112,8 @@ steps:
         command: |
           set -euo pipefail
           ./.buildkite/scripts/agentbeat/prepare_env.py
+          cd x-pack/agentbeat
+          mage -v testWithSpec
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -123,8 +125,10 @@ steps:
         key: "agentbeat-it-windows"
 #        depends_on:
 #          - agentbeat-package-linux
-        commands:
-          - ./.buildkite/scripts/agentbeat/prepare_env.py
+        command: |
+          ./.buildkite/scripts/agentbeat/prepare_env.py
+          cd x-pack/agentbeat
+          mage -v testWithSpec
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2022}"
diff --git a/dev-tools/mage/parse_spec.go b/dev-tools/mage/spec.go
similarity index 75%
rename from dev-tools/mage/parse_spec.go
rename to dev-tools/mage/spec.go
index 5dc20199288d..4afca22d5253 100644
--- a/dev-tools/mage/parse_spec.go
+++ b/dev-tools/mage/spec.go
@@ -21,6 +21,7 @@ import (
 	"gopkg.in/yaml.v2"
 	"log"
 	"os"
+	"strings"
 )
 
 type spec struct {
@@ -34,13 +35,17 @@ type input struct {
 	Command     command
 }
 
+func (i *input) GetCommand() string {
+	return strings.Join(i.Command.Args, " ")
+}
+
 type command struct {
 	Name string
 	Args []string
 }
 
-// ParseSpec parses agent.beat.spec.yml and generates test command
-func ParseSpec() {
+// SpecCommands parses agent.beat.spec.yml and collects commands for tests
+func SpecCommands() []string {
 	specPath := os.Getenv("AGENTBEAT_SPEC")
 	if specPath == "" {
 		log.Fatal("AGENTBEAT_SPEC is not defined")
@@ -51,20 +56,26 @@ func ParseSpec() {
 		log.Fatal("PLATFORM is not defined")
 	}
 
-	spec, err := parseToObj()
-	if err != nil {
-		log.Fatalf("Error parsing agentbeat.spec.yml: %v", err)
-	}
+	spec, _ := parseToObj(specPath)
 
-	inputList := filter(spec.Inputs, func(input input) bool {
+	filteredInputs := filter(spec.Inputs, func(input input) bool {
 		return contains(input.Platforms, platform)
 	})
 
-	log.Print(inputList)
+	commands := make(map[string]interface{})
+	for _, i := range filteredInputs {
+		commands[i.GetCommand()] = nil
+	}
+	keys := make([]string, 0, len(commands))
+	for k := range commands {
+		keys = append(keys, k)
+	}
+
+	return keys
 }
 
-func parseToObj() (spec, error) {
-	specFile, err := os.ReadFile("../agentbeat.spec.yml")
+func parseToObj(path string) (spec, error) {
+	specFile, err := os.ReadFile(path)
 	if err != nil {
 		log.Fatalf("Error opening agentbeat.spec.yml: %v", err)
 		return spec{}, err
diff --git a/x-pack/agentbeat/magefile.go b/x-pack/agentbeat/magefile.go
index 097695552a7c..c52e122efdb0 100644
--- a/x-pack/agentbeat/magefile.go
+++ b/x-pack/agentbeat/magefile.go
@@ -10,6 +10,7 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"time"
 
@@ -142,7 +143,7 @@ func Package() error {
 	return nil
 }
 
-// TestPackages tests the generated packages (i.e. file modes, owners, groups).
+// TestPackages tests the generated packages (i.agentbeatCmd. file modes, owners, groups).
 func TestPackages() error {
 	return devtools.TestPackages()
 }
@@ -214,6 +215,65 @@ func PythonIntegTest(ctx context.Context) error {
 	return devtools.PythonIntegTestFromHost(devtools.DefaultPythonTestIntegrationFromHostArgs())
 }
 
-func TestWithSpecs(ctx context.Context) {
-	mg.Deps(devtools.ParseSpec)
+// TestWithSpec executes unique commands from agentbeat.spec.yml and validates that app haven't exited with non-zero
+func TestWithSpec(ctx context.Context) {
+	var commands = devtools.SpecCommands()
+
+	agentbeatPath := os.Getenv("AGENTBEAT_PATH")
+	fmt.Printf("--- AGENTBEAT_PATH: %s", agentbeatPath)
+
+	cmdResults := make(map[string]bool)
+
+	for _, command := range commands {
+		cmdResults[command] = agentbeatCmd(agentbeatPath, command)
+	}
+
+	hasFailures := false
+	for cmd, res := range cmdResults {
+		if res {
+			fmt.Printf("Command [%s] succeeded", cmd)
+		} else {
+			fmt.Printf("Command [%s] failed", cmd)
+			hasFailures = true
+		}
+	}
+
+	if hasFailures {
+		fmt.Printf("Some inputs failed. Exiting with error")
+		os.Exit(1)
+	}
+}
+
+func agentbeatCmd(agentbeatPath string, command string) bool {
+	cmd := exec.Command(agentbeatPath, command)
+	fmt.Printf("Running command: %v", cmd)
+
+	if err := cmd.Start(); err != nil {
+		_ = fmt.Errorf("failed to start command: %v", err)
+	}
+
+	defer func() {
+		if err := cmd.Process.Kill(); err != nil {
+			_ = fmt.Errorf("failed to kill process: %v", err)
+		} else {
+			_ = fmt.Errorf("command process killed")
+		}
+	}()
+
+	done := make(chan error, 1)
+	go func() {
+		done <- cmd.Wait()
+	}()
+	timeout := 2 * time.Second
+	deadline := time.After(timeout)
+
+	select {
+	case err := <-done:
+		_ = fmt.Errorf("command exited before %s: %v", timeout.String(), err)
+		return false
+
+	case <-deadline:
+		_ = fmt.Errorf("%s", cmd.Stdout)
+		return true
+	}
 }

From 41110cf58127b7357ebfadc93c7311c935748bf0 Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Wed, 9 Oct 2024 11:00:12 +0300
Subject: [PATCH 09/17] updated scripts

---
 .buildkite/hooks/pre-command                  |  5 +++
 .../{prepare_env.py => setup_agentbeat.py}    | 30 +++-------------
 .../x-pack/pipeline.xpack.agentbeat.yml       | 10 +++---
 dev-tools/mage/spec.go                        | 12 +------
 x-pack/agentbeat/magefile.go                  | 35 ++++++++++++-------
 5 files changed, 38 insertions(+), 54 deletions(-)
 rename .buildkite/scripts/agentbeat/{prepare_env.py => setup_agentbeat.py} (76%)

diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index 5718d97879e1..8b84984a2823 100644
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -35,6 +35,11 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || \
   export BUILDKITE_ANALYTICS_TOKEN
 fi
 
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-agentbeat" ]]; then
+    AGENTBEAT_PATH=$(.buildkite/scripts/agentbeat/setup_agentbeat.py)
+    export AGENTBEAT_PATH
+fi
+
 CPU_ARCH=$(uname -m)
 PLATFORM_TYPE=$(uname)
 
diff --git a/.buildkite/scripts/agentbeat/prepare_env.py b/.buildkite/scripts/agentbeat/setup_agentbeat.py
similarity index 76%
rename from .buildkite/scripts/agentbeat/prepare_env.py
rename to .buildkite/scripts/agentbeat/setup_agentbeat.py
index d17c3cdd0e10..62e8c3bbd2cd 100755
--- a/.buildkite/scripts/agentbeat/prepare_env.py
+++ b/.buildkite/scripts/agentbeat/setup_agentbeat.py
@@ -4,7 +4,6 @@
 import subprocess
 import sys
 import tarfile
-import os
 import re
 
 PATH = 'x-pack/agentbeat/build/distributions'
@@ -46,7 +45,6 @@ def get_artifact_pattern() -> str:
 
 
 def download_agentbeat(pattern, path) -> str:
-    log('--- Downloading agentbeat')
     try:
         subprocess.run(
             ['buildkite-agent', 'artifact', 'download', pattern, '.',
@@ -70,20 +68,18 @@ def get_filename(path) -> str:
 
 
 def extract_agentbeat(filename):
-    log('~~~ Extracting agentbeat')
     filepath = PATH + '/' + filename
 
     if filepath.endswith('.zip'):
         unzip_agentbeat(filepath)
     else:
         untar_agentbeat(filepath)
-    log('Successfully extracted agentbeat')
 
 
 def unzip_agentbeat(filepath):
     try:
         subprocess.run(
-            ['unzip', filepath],
+            ['unzip', '-qq', filepath],
             check=True, stdout=sys.stdout, stderr=sys.stderr, text=True)
     except subprocess.CalledProcessError as e:
         log_err(e)
@@ -93,40 +89,24 @@ def unzip_agentbeat(filepath):
 def untar_agentbeat(filepath):
     try:
         with tarfile.open(filepath, 'r:gz') as tar:
-            tar.list()
             tar.extractall()
     except Exception as e:
         log_err(e)
         exit(1)
 
 
-def add_to_path(filepath):
+def get_path_to_executable(filepath) -> str:
     pattern = r'(.*)(?=\.zip|.tar\.gz)'
     match = re.match(pattern, filepath)
     if match:
-        path = f'../build/distributions/{match.group(1)}/agentbeat'
-        log("--- PATH: " + str(path))
-        os.environ['AGENTBEAT_PATH'] = str(path)
+        path = f'../../{match.group(1)}/agentbeat'
+        return path
     else:
         log_err("No agentbeat executable found")
         exit(1)
 
 
-def install_synthetics():
-    log('--- Installing @elastic/synthetics')
-
-    try:
-        subprocess.run(
-            ['npm', 'install', '-g', '@elastic/synthetics'],
-            check=True
-        )
-    except subprocess.CalledProcessError:
-        log_err('Failed to install @elastic/synthetics')
-        exit(1)
-
-
 artifact_pattern = get_artifact_pattern()
 archive = download_agentbeat(artifact_pattern, PATH)
 extract_agentbeat(archive)
-add_to_path(archive)
-install_synthetics()
+log(get_path_to_executable(archive))
diff --git a/.buildkite/x-pack/pipeline.xpack.agentbeat.yml b/.buildkite/x-pack/pipeline.xpack.agentbeat.yml
index b6225577b846..57618c671e40 100644
--- a/.buildkite/x-pack/pipeline.xpack.agentbeat.yml
+++ b/.buildkite/x-pack/pipeline.xpack.agentbeat.yml
@@ -110,9 +110,8 @@ steps:
 #        depends_on:
 #          - agentbeat-package-linux
         command: |
-          set -euo pipefail
-          ./.buildkite/scripts/agentbeat/prepare_env.py
           cd x-pack/agentbeat
+          ls -la ../../agentbeat-9.0.0-SNAPSHOT-linux-x86_64
           mage -v testWithSpec
         agents:
           provider: "gcp"
@@ -123,10 +122,11 @@ steps:
 
       - label: ":linux: Agentbeat/Integration tests Windows"
         key: "agentbeat-it-windows"
+        env:
+          PLATFORM: "windows/amd64"
 #        depends_on:
 #          - agentbeat-package-linux
         command: |
-          ./.buildkite/scripts/agentbeat/prepare_env.py
           cd x-pack/agentbeat
           mage -v testWithSpec
         agents:
@@ -142,7 +142,7 @@ steps:
 #          - agentbeat-package-linux
 #        command: |
 #          set -euo pipefail
-#          ./.buildkite/scripts/agentbeat/prepare_env.py
+#          ./.buildkite/scripts/agentbeat/setup_agentbeat.py
 #        agents:
 #          provider: "orka"
 #          imagePrefix: "${IMAGE_MACOS_X86_64}"
@@ -153,7 +153,7 @@ steps:
 #          - agentbeat-package-linux
 #        command: |
 #          set -euo pipefail
-#          ./.buildkite/scripts/agentbeat/prepare_env.py
+#          ./.buildkite/scripts/agentbeat/setup_agentbeat.py
 #        agents:
 #          provider: "orka"
 #          imagePrefix: "${IMAGE_MACOS_ARM}"
diff --git a/dev-tools/mage/spec.go b/dev-tools/mage/spec.go
index 4afca22d5253..af0527fd6196 100644
--- a/dev-tools/mage/spec.go
+++ b/dev-tools/mage/spec.go
@@ -45,17 +45,7 @@ type command struct {
 }
 
 // SpecCommands parses agent.beat.spec.yml and collects commands for tests
-func SpecCommands() []string {
-	specPath := os.Getenv("AGENTBEAT_SPEC")
-	if specPath == "" {
-		log.Fatal("AGENTBEAT_SPEC is not defined")
-	}
-
-	platform := os.Getenv("PLATFORM")
-	if platform == "" {
-		log.Fatal("PLATFORM is not defined")
-	}
-
+func SpecCommands(specPath string, platform string) []string {
 	spec, _ := parseToObj(specPath)
 
 	filteredInputs := filter(spec.Inputs, func(input input) bool {
diff --git a/x-pack/agentbeat/magefile.go b/x-pack/agentbeat/magefile.go
index c52e122efdb0..0b29c6c5d747 100644
--- a/x-pack/agentbeat/magefile.go
+++ b/x-pack/agentbeat/magefile.go
@@ -9,6 +9,7 @@ package main
 import (
 	"context"
 	"fmt"
+	"log"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -217,10 +218,19 @@ func PythonIntegTest(ctx context.Context) error {
 
 // TestWithSpec executes unique commands from agentbeat.spec.yml and validates that app haven't exited with non-zero
 func TestWithSpec(ctx context.Context) {
-	var commands = devtools.SpecCommands()
+	specPath := os.Getenv("AGENTBEAT_SPEC")
+	if specPath == "" {
+		log.Fatal("AGENTBEAT_SPEC is not defined\n")
+	}
+
+	platform := os.Getenv("PLATFORM")
+	if platform == "" {
+		log.Fatal("PLATFORM is not defined\n")
+	}
+
+	var commands = devtools.SpecCommands(specPath, platform)
 
 	agentbeatPath := os.Getenv("AGENTBEAT_PATH")
-	fmt.Printf("--- AGENTBEAT_PATH: %s", agentbeatPath)
 
 	cmdResults := make(map[string]bool)
 
@@ -230,33 +240,32 @@ func TestWithSpec(ctx context.Context) {
 
 	hasFailures := false
 	for cmd, res := range cmdResults {
-		if res {
-			fmt.Printf("Command [%s] succeeded", cmd)
-		} else {
-			fmt.Printf("Command [%s] failed", cmd)
+		if !res {
+			fmt.Printf("~~~ Failed: [%s]\n", cmd)
+			fmt.Print(res)
 			hasFailures = true
 		}
 	}
 
 	if hasFailures {
-		fmt.Printf("Some inputs failed. Exiting with error")
+		fmt.Printf("Some inputs failed. Exiting with error\n")
 		os.Exit(1)
 	}
 }
 
 func agentbeatCmd(agentbeatPath string, command string) bool {
 	cmd := exec.Command(agentbeatPath, command)
-	fmt.Printf("Running command: %v", cmd)
+	fmt.Printf("Running command: %v\n", cmd)
 
 	if err := cmd.Start(); err != nil {
-		_ = fmt.Errorf("failed to start command: %v", err)
+		fmt.Printf("failed to start command: %v\n", err)
 	}
 
 	defer func() {
 		if err := cmd.Process.Kill(); err != nil {
-			_ = fmt.Errorf("failed to kill process: %v", err)
+			fmt.Printf("failed to kill process: %v\n", err)
 		} else {
-			_ = fmt.Errorf("command process killed")
+			fmt.Print("command process killed\n")
 		}
 	}()
 
@@ -269,11 +278,11 @@ func agentbeatCmd(agentbeatPath string, command string) bool {
 
 	select {
 	case err := <-done:
-		_ = fmt.Errorf("command exited before %s: %v", timeout.String(), err)
+		fmt.Printf("command exited before %s: %v\n", timeout.String(), err)
 		return false
 
 	case <-deadline:
-		_ = fmt.Errorf("%s", cmd.Stdout)
+		fmt.Printf("%s\n", cmd.Stdout)
 		return true
 	}
 }

From a4e129cb35d75bc171a84ea72650d79df4fe3282 Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Thu, 10 Oct 2024 11:00:03 +0300
Subject: [PATCH 10/17] debug linux

---
 .buildkite/hooks/pre-command                  | 10 +++----
 .../x-pack/pipeline.xpack.agentbeat.yml       | 16 ++++++++----
 x-pack/agentbeat/magefile.go                  | 26 ++++++++++++++-----
 3 files changed, 35 insertions(+), 17 deletions(-)

diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index 8b84984a2823..a6e5cd8750ef 100644
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -18,6 +18,11 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" && "$BUILDKITE_STEP
     export PRIVATE_CI_GCS_CREDENTIALS_SECRET
 fi
 
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-agentbeat" ]]; then
+    AGENTBEAT_PATH=$(.buildkite/scripts/agentbeat/setup_agentbeat.py)
+    export AGENTBEAT_PATH
+fi
+
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || \
   "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || \
   "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || \
@@ -35,11 +40,6 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || \
   export BUILDKITE_ANALYTICS_TOKEN
 fi
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-agentbeat" ]]; then
-    AGENTBEAT_PATH=$(.buildkite/scripts/agentbeat/setup_agentbeat.py)
-    export AGENTBEAT_PATH
-fi
-
 CPU_ARCH=$(uname -m)
 PLATFORM_TYPE=$(uname)
 
diff --git a/.buildkite/x-pack/pipeline.xpack.agentbeat.yml b/.buildkite/x-pack/pipeline.xpack.agentbeat.yml
index 57618c671e40..1b4f68992631 100644
--- a/.buildkite/x-pack/pipeline.xpack.agentbeat.yml
+++ b/.buildkite/x-pack/pipeline.xpack.agentbeat.yml
@@ -89,6 +89,12 @@ steps:
       #          - x-pack/agentbeat/build/distributions/**/*
       #          - "x-pack/agentbeat/build/*.xml"
       #          - "x-pack/agentbeat/build/*.json"
+#              plugins:
+#                - test-collector#v1.10.2:
+#                    files: "x-pack/agentbeat/build/TEST-*.xml"
+#                    format: "junit"
+#                    branches: "main"
+#                    debug: true
       #        retry:
       #          automatic:
       #            - limit: 1
@@ -103,7 +109,7 @@ steps:
       #          - github_commit_status:
       #              context: "agentbeat: Integration tests"
 
-      - label: ":linux: Agentbeat/Integration tests Linux"
+      - label: ":linux: x-pack/agentbeat: Ubuntu x86_64 Spec tests"
         key: "agentbeat-it-linux"
         env:
           PLATFORM: "linux/amd64"
@@ -111,8 +117,8 @@ steps:
 #          - agentbeat-package-linux
         command: |
           cd x-pack/agentbeat
-          ls -la ../../agentbeat-9.0.0-SNAPSHOT-linux-x86_64
-          mage -v testWithSpec
+          ../../agentbeat-9.0.0-SNAPSHOT-linux-x86_64/agentbeat filebeat -E setup.ilm.enabled=false -E setup.template.enabled=false -E management.enabled=true -E management.restart_on_output_change=true -E logging.level=info -E logging.to_stderr=true -E gc_percent=${FILEBEAT_GOGC:100} -E filebeat.config.modules.enabled=false -E logging.event_data.to_stderr=true -E logging.event_data.to_files=false
+          # mage -v testWithSpec
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -120,14 +126,14 @@ steps:
           disk_size: 100
           disk_type: "pd-ssd"
 
-      - label: ":linux: Agentbeat/Integration tests Windows"
+      - label: ":windows: x-pack/agentbeat: Windows x86_64 Spec tests"
         key: "agentbeat-it-windows"
         env:
           PLATFORM: "windows/amd64"
 #        depends_on:
 #          - agentbeat-package-linux
         command: |
-          cd x-pack/agentbeat
+          Set-Location -Path x-pack/agentbeat
           mage -v testWithSpec
         agents:
           provider: "gcp"
diff --git a/x-pack/agentbeat/magefile.go b/x-pack/agentbeat/magefile.go
index 0b29c6c5d747..429c4a2049fd 100644
--- a/x-pack/agentbeat/magefile.go
+++ b/x-pack/agentbeat/magefile.go
@@ -7,6 +7,7 @@
 package main
 
 import (
+	"bufio"
 	"context"
 	"fmt"
 	"log"
@@ -144,7 +145,7 @@ func Package() error {
 	return nil
 }
 
-// TestPackages tests the generated packages (i.agentbeatCmd. file modes, owners, groups).
+// TestPackages tests the generated packages (i.runCmd. file modes, owners, groups).
 func TestPackages() error {
 	return devtools.TestPackages()
 }
@@ -235,14 +236,15 @@ func TestWithSpec(ctx context.Context) {
 	cmdResults := make(map[string]bool)
 
 	for _, command := range commands {
-		cmdResults[command] = agentbeatCmd(agentbeatPath, command)
+		cmdResults[command] = runCmd(agentbeatPath, command)
 	}
 
 	hasFailures := false
 	for cmd, res := range cmdResults {
-		if !res {
-			fmt.Printf("~~~ Failed: [%s]\n", cmd)
-			fmt.Print(res)
+		if res {
+			fmt.Printf("--- :large_green_circle: Succeeded: [%s.10s...]\n", cmd)
+		} else {
+			fmt.Printf("--- :bangbang: Failed: [%s.10s...]\n", cmd)
 			hasFailures = true
 		}
 	}
@@ -253,9 +255,14 @@ func TestWithSpec(ctx context.Context) {
 	}
 }
 
-func agentbeatCmd(agentbeatPath string, command string) bool {
+func runCmd(agentbeatPath string, command string) bool {
 	cmd := exec.Command(agentbeatPath, command)
-	fmt.Printf("Running command: %v\n", cmd)
+	fmt.Printf("Executing: %s\n", cmd.String())
+
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		fmt.Printf("Error creating stdout pipe: %v\n", err)
+	}
 
 	if err := cmd.Start(); err != nil {
 		fmt.Printf("failed to start command: %v\n", err)
@@ -279,6 +286,11 @@ func agentbeatCmd(agentbeatPath string, command string) bool {
 	select {
 	case err := <-done:
 		fmt.Printf("command exited before %s: %v\n", timeout.String(), err)
+		fmt.Println("printing command stdout")
+		scanner := bufio.NewScanner(stdout)
+		for scanner.Scan() {
+			fmt.Println(scanner.Text())
+		}
 		return false
 
 	case <-deadline:

From 380f54c63d2d725826487a7fcd443b91b5d2ca44 Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Mon, 14 Oct 2024 11:00:48 +0300
Subject: [PATCH 11/17] updated std's outputs

---
 .../x-pack/pipeline.xpack.agentbeat.yml       |  3 +--
 x-pack/agentbeat/magefile.go                  | 20 +++++++------------
 2 files changed, 8 insertions(+), 15 deletions(-)

diff --git a/.buildkite/x-pack/pipeline.xpack.agentbeat.yml b/.buildkite/x-pack/pipeline.xpack.agentbeat.yml
index 1b4f68992631..95c091fe0530 100644
--- a/.buildkite/x-pack/pipeline.xpack.agentbeat.yml
+++ b/.buildkite/x-pack/pipeline.xpack.agentbeat.yml
@@ -117,8 +117,7 @@ steps:
 #          - agentbeat-package-linux
         command: |
           cd x-pack/agentbeat
-          ../../agentbeat-9.0.0-SNAPSHOT-linux-x86_64/agentbeat filebeat -E setup.ilm.enabled=false -E setup.template.enabled=false -E management.enabled=true -E management.restart_on_output_change=true -E logging.level=info -E logging.to_stderr=true -E gc_percent=${FILEBEAT_GOGC:100} -E filebeat.config.modules.enabled=false -E logging.event_data.to_stderr=true -E logging.event_data.to_files=false
-          # mage -v testWithSpec
+          mage -v testWithSpec
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
diff --git a/x-pack/agentbeat/magefile.go b/x-pack/agentbeat/magefile.go
index 429c4a2049fd..d02b5f9c83a8 100644
--- a/x-pack/agentbeat/magefile.go
+++ b/x-pack/agentbeat/magefile.go
@@ -7,13 +7,13 @@
 package main
 
 import (
-	"bufio"
 	"context"
 	"fmt"
 	"log"
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/magefile/mage/sh"
@@ -236,7 +236,7 @@ func TestWithSpec(ctx context.Context) {
 	cmdResults := make(map[string]bool)
 
 	for _, command := range commands {
-		cmdResults[command] = runCmd(agentbeatPath, command)
+		cmdResults[command] = runCmd(agentbeatPath, strings.Split(command, " "))
 	}
 
 	hasFailures := false
@@ -255,14 +255,13 @@ func TestWithSpec(ctx context.Context) {
 	}
 }
 
-func runCmd(agentbeatPath string, command string) bool {
-	cmd := exec.Command(agentbeatPath, command)
+func runCmd(agentbeatPath string, command []string) bool {
+	cmd := exec.Command(agentbeatPath, command...)
 	fmt.Printf("Executing: %s\n", cmd.String())
 
-	stdout, err := cmd.StdoutPipe()
-	if err != nil {
-		fmt.Printf("Error creating stdout pipe: %v\n", err)
-	}
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	cmd.Stdin = os.Stdin
 
 	if err := cmd.Start(); err != nil {
 		fmt.Printf("failed to start command: %v\n", err)
@@ -286,11 +285,6 @@ func runCmd(agentbeatPath string, command string) bool {
 	select {
 	case err := <-done:
 		fmt.Printf("command exited before %s: %v\n", timeout.String(), err)
-		fmt.Println("printing command stdout")
-		scanner := bufio.NewScanner(stdout)
-		for scanner.Scan() {
-			fmt.Println(scanner.Text())
-		}
 		return false
 
 	case <-deadline:

From bc2cc099f2db651be57f0e4fc3e44685f11a7ec4 Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Mon, 14 Oct 2024 11:46:05 +0300
Subject: [PATCH 12/17] updated scripts && enbaled macos step

---
 .buildkite/hooks/pre-command                  |   5 +-
 .../scripts/agentbeat/setup_agentbeat.py      |   2 +-
 .../x-pack/pipeline.xpack.agentbeat.yml       | 110 +++++++++---------
 3 files changed, 60 insertions(+), 57 deletions(-)

diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index a6e5cd8750ef..a6755d9df3c1 100644
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -18,7 +18,10 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" && "$BUILDKITE_STEP
     export PRIVATE_CI_GCS_CREDENTIALS_SECRET
 fi
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-agentbeat" ]]; then
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-agentbeat" && "$BUILDKITE_STEP_KEY" == *"agentbeat-it"* ]]; then
+  if [[ "$BUILDKITE_STEP_KEY" == *"macos"* ]]; then
+      ulimit -Sn 10000
+  fi
     AGENTBEAT_PATH=$(.buildkite/scripts/agentbeat/setup_agentbeat.py)
     export AGENTBEAT_PATH
 fi
diff --git a/.buildkite/scripts/agentbeat/setup_agentbeat.py b/.buildkite/scripts/agentbeat/setup_agentbeat.py
index 62e8c3bbd2cd..0c82c16021f5 100755
--- a/.buildkite/scripts/agentbeat/setup_agentbeat.py
+++ b/.buildkite/scripts/agentbeat/setup_agentbeat.py
@@ -48,7 +48,7 @@ def download_agentbeat(pattern, path) -> str:
     try:
         subprocess.run(
             ['buildkite-agent', 'artifact', 'download', pattern, '.',
-             '--build', '01924d2b-b061-45ae-a106-e885584ff26f',
+             # '--build', '01924d2b-b061-45ae-a106-e885584ff26f',
              '--step', 'agentbeat-package-linux'],
             check=True, stdout=sys.stdout, stderr=sys.stderr, text=True)
     except subprocess.CalledProcessError:
diff --git a/.buildkite/x-pack/pipeline.xpack.agentbeat.yml b/.buildkite/x-pack/pipeline.xpack.agentbeat.yml
index 95c091fe0530..4cf23fdffd0c 100644
--- a/.buildkite/x-pack/pipeline.xpack.agentbeat.yml
+++ b/.buildkite/x-pack/pipeline.xpack.agentbeat.yml
@@ -15,19 +15,19 @@ env:
   AGENTBEAT_SPEC: "./agentbeat.spec.yml"
 
 steps:
-#  - group: "Check/Update"
-#    key: "x-pack-agentbeat-check-update"
-#
-#    steps:
-#      - label: "agentbeat: Run pre-commit"
-#        command: "pre-commit run --all-files"
-#        agents:
-#          image: "${IMAGE_BEATS_WITH_HOOKS_LATEST}"
-#          memory: "2Gi"
-#          useCustomGlobalHooks: true
-#        notify:
-#          - github_commit_status:
-#              context: "agentbeat: pre-commit"
+  - group: "Check/Update"
+    key: "x-pack-agentbeat-check-update"
+
+    steps:
+      - label: "agentbeat: Run pre-commit"
+        command: "pre-commit run --all-files"
+        agents:
+          image: "${IMAGE_BEATS_WITH_HOOKS_LATEST}"
+          memory: "2Gi"
+          useCustomGlobalHooks: true
+        notify:
+          - github_commit_status:
+              context: "agentbeat: pre-commit"
 
 #  - wait: ~
 #    # with PRs, we want to run mandatory tests only if check/update step succeed
@@ -40,32 +40,32 @@ steps:
     key: "agentbeat-mandatory-tests"
 
     steps:
-#      - label: ":linux: Agentbeat packaging Linux"
-#        key: "agentbeat-package-linux"
-#        env:
-#          PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
-#          SNAPSHOT: true
-#        command: |
-#          set -euo pipefail
-#          cd x-pack/agentbeat
-#          mage package
-#        artifact_paths:
-#          - x-pack/agentbeat/build/distributions/**/*
-#          - "x-pack/agentbeat/build/*.xml"
-#          - "x-pack/agentbeat/build/*.json"
-#        retry:
-#          automatic:
-#            - limit: 2
-#        timeout_in_minutes: 60
-#        agents:
-#          provider: "gcp"
-#          image: "${IMAGE_UBUNTU_X86_64}"
-#          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
-#          disk_size: 100
-#          disk_type: "pd-ssd"
-#        notify:
-#          - github_commit_status:
-#              context: "agentbeat: Packaging"
+      - label: ":linux: Agentbeat packaging Linux"
+        key: "agentbeat-package-linux"
+        env:
+          PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
+          SNAPSHOT: true
+        command: |
+          set -euo pipefail
+          cd x-pack/agentbeat
+          mage package
+        artifact_paths:
+          - x-pack/agentbeat/build/distributions/**/*
+          - "x-pack/agentbeat/build/*.xml"
+          - "x-pack/agentbeat/build/*.json"
+        retry:
+          automatic:
+            - limit: 2
+        timeout_in_minutes: 60
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+          disk_size: 100
+          disk_type: "pd-ssd"
+        notify:
+          - github_commit_status:
+              context: "agentbeat: Packaging"
 
       #      - label: ":linux: Agentbeat/Integration tests Linux"
       #        key: "agentbeat-it-linux"
@@ -113,8 +113,8 @@ steps:
         key: "agentbeat-it-linux"
         env:
           PLATFORM: "linux/amd64"
-#        depends_on:
-#          - agentbeat-package-linux
+        depends_on:
+          - agentbeat-package-linux
         command: |
           cd x-pack/agentbeat
           mage -v testWithSpec
@@ -129,8 +129,8 @@ steps:
         key: "agentbeat-it-windows"
         env:
           PLATFORM: "windows/amd64"
-#        depends_on:
-#          - agentbeat-package-linux
+        depends_on:
+          - agentbeat-package-linux
         command: |
           Set-Location -Path x-pack/agentbeat
           mage -v testWithSpec
@@ -140,18 +140,18 @@ steps:
           machine_type: "${GCP_WIN_MACHINE_TYPE}"
           disk_size: 200
           disk_type: "pd-ssd"
-#
-#      - label: ":linux: Agentbeat/Integration tests macOS"
-#        key: "agentbeat-it-macos"
-#        depends_on:
-#          - agentbeat-package-linux
-#        command: |
-#          set -euo pipefail
-#          ./.buildkite/scripts/agentbeat/setup_agentbeat.py
-#        agents:
-#          provider: "orka"
-#          imagePrefix: "${IMAGE_MACOS_X86_64}"
-#
+
+      - label: ":macos: x-pack/agentbeat: macOS x86_64 Spec tests"
+        key: "agentbeat-it-macos"
+        depends_on:
+          - agentbeat-package-linux
+        command: |
+          cd x-pack/agentbeat
+          mage -v testWithSpec
+        agents:
+          provider: "orka"
+          imagePrefix: "${IMAGE_MACOS_X86_64}"
+
 #      - label: ":linux: Agentbeat/Integration tests macOS arm64"
 #        key: "agentbeat-it-macos-arm"
 #        depends_on:

From 557b3186557741fb9785fa69af16c25e6dc16e77 Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Mon, 14 Oct 2024 15:26:15 +0300
Subject: [PATCH 13/17] debug macos

---
 .../scripts/agentbeat/setup_agentbeat.py      |   5 +-
 .../x-pack/pipeline.xpack.agentbeat.yml       | 146 +++++++++---------
 2 files changed, 77 insertions(+), 74 deletions(-)

diff --git a/.buildkite/scripts/agentbeat/setup_agentbeat.py b/.buildkite/scripts/agentbeat/setup_agentbeat.py
index 0c82c16021f5..818e048b999d 100755
--- a/.buildkite/scripts/agentbeat/setup_agentbeat.py
+++ b/.buildkite/scripts/agentbeat/setup_agentbeat.py
@@ -48,7 +48,7 @@ def download_agentbeat(pattern, path) -> str:
     try:
         subprocess.run(
             ['buildkite-agent', 'artifact', 'download', pattern, '.',
-             # '--build', '01924d2b-b061-45ae-a106-e885584ff26f',
+             '--build', '01924d2b-b061-45ae-a106-e885584ff26f',
              '--step', 'agentbeat-package-linux'],
             check=True, stdout=sys.stdout, stderr=sys.stderr, text=True)
     except subprocess.CalledProcessError:
@@ -58,6 +58,7 @@ def download_agentbeat(pattern, path) -> str:
 
 
 def get_filename(path) -> str:
+    print("--- Getting filename")
     try:
         out = subprocess.run(
             ['ls', '-p', path],
@@ -108,5 +109,7 @@ def get_path_to_executable(filepath) -> str:
 
 artifact_pattern = get_artifact_pattern()
 archive = download_agentbeat(artifact_pattern, PATH)
+print("--- Extracting")
 extract_agentbeat(archive)
+print("--- Getting path to exec")
 log(get_path_to_executable(archive))
diff --git a/.buildkite/x-pack/pipeline.xpack.agentbeat.yml b/.buildkite/x-pack/pipeline.xpack.agentbeat.yml
index 4cf23fdffd0c..8873ffe29b8e 100644
--- a/.buildkite/x-pack/pipeline.xpack.agentbeat.yml
+++ b/.buildkite/x-pack/pipeline.xpack.agentbeat.yml
@@ -29,43 +29,43 @@ steps:
           - github_commit_status:
               context: "agentbeat: pre-commit"
 
-#  - wait: ~
-#    # with PRs, we want to run mandatory tests only if check/update step succeed
-#    # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests
-#    # this allows building DRA artifacts even if there is flakiness in check/update step
-#    if: build.env("BUILDKITE_PULL_REQUEST") != "false"
-#    depends_on: "x-pack-agentbeat-check-update"
+  - wait: ~
+    # with PRs, we want to run mandatory tests only if check/update step succeed
+    # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests
+    # this allows building DRA artifacts even if there is flakiness in check/update step
+    if: build.env("BUILDKITE_PULL_REQUEST") != "false"
+    depends_on: "x-pack-agentbeat-check-update"
 
   - group: "Agentbeat tests"
     key: "agentbeat-mandatory-tests"
 
     steps:
-      - label: ":linux: Agentbeat packaging Linux"
-        key: "agentbeat-package-linux"
-        env:
-          PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
-          SNAPSHOT: true
-        command: |
-          set -euo pipefail
-          cd x-pack/agentbeat
-          mage package
-        artifact_paths:
-          - x-pack/agentbeat/build/distributions/**/*
-          - "x-pack/agentbeat/build/*.xml"
-          - "x-pack/agentbeat/build/*.json"
-        retry:
-          automatic:
-            - limit: 2
-        timeout_in_minutes: 60
-        agents:
-          provider: "gcp"
-          image: "${IMAGE_UBUNTU_X86_64}"
-          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
-          disk_size: 100
-          disk_type: "pd-ssd"
-        notify:
-          - github_commit_status:
-              context: "agentbeat: Packaging"
+#      - label: ":linux: Agentbeat packaging Linux"
+#        key: "agentbeat-package-linux"
+#        env:
+#          PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
+#          SNAPSHOT: true
+#        command: |
+#          set -euo pipefail
+#          cd x-pack/agentbeat
+#          mage package
+#        artifact_paths:
+#          - x-pack/agentbeat/build/distributions/**/*
+#          - "x-pack/agentbeat/build/*.xml"
+#          - "x-pack/agentbeat/build/*.json"
+#        retry:
+#          automatic:
+#            - limit: 2
+#        timeout_in_minutes: 60
+#        agents:
+#          provider: "gcp"
+#          image: "${IMAGE_UBUNTU_X86_64}"
+#          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+#          disk_size: 100
+#          disk_type: "pd-ssd"
+#        notify:
+#          - github_commit_status:
+#              context: "agentbeat: Packaging"
 
       #      - label: ":linux: Agentbeat/Integration tests Linux"
       #        key: "agentbeat-it-linux"
@@ -109,42 +109,42 @@ steps:
       #          - github_commit_status:
       #              context: "agentbeat: Integration tests"
 
-      - label: ":linux: x-pack/agentbeat: Ubuntu x86_64 Spec tests"
-        key: "agentbeat-it-linux"
-        env:
-          PLATFORM: "linux/amd64"
-        depends_on:
-          - agentbeat-package-linux
-        command: |
-          cd x-pack/agentbeat
-          mage -v testWithSpec
-        agents:
-          provider: "gcp"
-          image: "${IMAGE_UBUNTU_X86_64}"
-          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
-          disk_size: 100
-          disk_type: "pd-ssd"
-
-      - label: ":windows: x-pack/agentbeat: Windows x86_64 Spec tests"
-        key: "agentbeat-it-windows"
-        env:
-          PLATFORM: "windows/amd64"
-        depends_on:
-          - agentbeat-package-linux
-        command: |
-          Set-Location -Path x-pack/agentbeat
-          mage -v testWithSpec
-        agents:
-          provider: "gcp"
-          image: "${IMAGE_WIN_2022}"
-          machine_type: "${GCP_WIN_MACHINE_TYPE}"
-          disk_size: 200
-          disk_type: "pd-ssd"
+#      - label: ":linux: x-pack/agentbeat: Ubuntu x86_64 Spec tests"
+#        key: "agentbeat-it-linux"
+#        env:
+#          PLATFORM: "linux/amd64"
+#        depends_on:
+#          - agentbeat-package-linux
+#        command: |
+#          cd x-pack/agentbeat
+#          mage -v testWithSpec
+#        agents:
+#          provider: "gcp"
+#          image: "${IMAGE_UBUNTU_X86_64}"
+#          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+#          disk_size: 100
+#          disk_type: "pd-ssd"
+#
+#      - label: ":windows: x-pack/agentbeat: Windows x86_64 Spec tests"
+#        key: "agentbeat-it-windows"
+#        env:
+#          PLATFORM: "windows/amd64"
+#        depends_on:
+#          - agentbeat-package-linux
+#        command: |
+#          Set-Location -Path x-pack/agentbeat
+#          mage -v testWithSpec
+#        agents:
+#          provider: "gcp"
+#          image: "${IMAGE_WIN_2022}"
+#          machine_type: "${GCP_WIN_MACHINE_TYPE}"
+#          disk_size: 200
+#          disk_type: "pd-ssd"
 
       - label: ":macos: x-pack/agentbeat: macOS x86_64 Spec tests"
         key: "agentbeat-it-macos"
-        depends_on:
-          - agentbeat-package-linux
+#        depends_on:
+#          - agentbeat-package-linux
         command: |
           cd x-pack/agentbeat
           mage -v testWithSpec
@@ -152,13 +152,13 @@ steps:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_X86_64}"
 
-#      - label: ":linux: Agentbeat/Integration tests macOS arm64"
-#        key: "agentbeat-it-macos-arm"
+      - label: ":linux: Agentbeat/Integration tests macOS arm64"
+        key: "agentbeat-it-macos-arm"
 #        depends_on:
 #          - agentbeat-package-linux
-#        command: |
-#          set -euo pipefail
-#          ./.buildkite/scripts/agentbeat/setup_agentbeat.py
-#        agents:
-#          provider: "orka"
-#          imagePrefix: "${IMAGE_MACOS_ARM}"
+        command: |
+          cd x-pack/agentbeat
+          mage -v testWithSpec
+        agents:
+          provider: "orka"
+          imagePrefix: "${IMAGE_MACOS_ARM}"

From b1e9fa57ff06545db780b40efd0988ea5bb74a98 Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Mon, 14 Oct 2024 15:32:30 +0300
Subject: [PATCH 14/17] debug macos

---
 .buildkite/hooks/pre-command                  |   6 +-
 .../scripts/agentbeat/setup_agentbeat.py      |  18 +-
 .../x-pack/pipeline.xpack.agentbeat.yml       | 178 +++++++++---------
 dev-tools/mage/spec.go                        |   1 +
 4 files changed, 102 insertions(+), 101 deletions(-)

diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index a6755d9df3c1..f1f9bd540294 100644
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -19,10 +19,8 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" && "$BUILDKITE_STEP
 fi
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-agentbeat" && "$BUILDKITE_STEP_KEY" == *"agentbeat-it"* ]]; then
-  if [[ "$BUILDKITE_STEP_KEY" == *"macos"* ]]; then
-      ulimit -Sn 10000
-  fi
-    AGENTBEAT_PATH=$(.buildkite/scripts/agentbeat/setup_agentbeat.py)
+#    .buildkite/scripts/agentbeat/setup_agentbeat.py
+    AGENTBEAT_PATH=$(.buildkite/scripts/agentbeat/setup_agentbeat.py | tail -n 1)
     export AGENTBEAT_PATH
 fi
 
diff --git a/.buildkite/scripts/agentbeat/setup_agentbeat.py b/.buildkite/scripts/agentbeat/setup_agentbeat.py
index 818e048b999d..b5f148a18ccd 100755
--- a/.buildkite/scripts/agentbeat/setup_agentbeat.py
+++ b/.buildkite/scripts/agentbeat/setup_agentbeat.py
@@ -1,10 +1,9 @@
 #!/usr/bin/env python3
-
 import platform
+import re
 import subprocess
 import sys
 import tarfile
-import re
 
 PATH = 'x-pack/agentbeat/build/distributions'
 
@@ -48,9 +47,10 @@ def download_agentbeat(pattern, path) -> str:
     try:
         subprocess.run(
             ['buildkite-agent', 'artifact', 'download', pattern, '.',
-             '--build', '01924d2b-b061-45ae-a106-e885584ff26f',
+             # '--build', '01928f55-8452-41c6-89ba-fe21f019f53c',
              '--step', 'agentbeat-package-linux'],
             check=True, stdout=sys.stdout, stderr=sys.stderr, text=True)
+
     except subprocess.CalledProcessError:
         exit(1)
 
@@ -58,11 +58,11 @@ def download_agentbeat(pattern, path) -> str:
 
 
 def get_filename(path) -> str:
-    print("--- Getting filename")
     try:
         out = subprocess.run(
             ['ls', '-p', path],
             check=True, capture_output=True, text=True)
+        print("--- ls -p: " + out.stdout)
         return out.stdout.strip()
     except subprocess.CalledProcessError:
         exit(1)
@@ -89,9 +89,10 @@ def unzip_agentbeat(filepath):
 
 def untar_agentbeat(filepath):
     try:
-        with tarfile.open(filepath, 'r:gz') as tar:
-            tar.extractall()
-    except Exception as e:
+        subprocess.run(
+            ['tar', '-xvf', filepath],
+            check=True, stdout=sys.stdout, stderr=sys.stderr, text=True)
+    except subprocess.CalledProcessError as e:
         log_err(e)
         exit(1)
 
@@ -106,10 +107,7 @@ def get_path_to_executable(filepath) -> str:
         log_err("No agentbeat executable found")
         exit(1)
 
-
 artifact_pattern = get_artifact_pattern()
 archive = download_agentbeat(artifact_pattern, PATH)
-print("--- Extracting")
 extract_agentbeat(archive)
-print("--- Getting path to exec")
 log(get_path_to_executable(archive))
diff --git a/.buildkite/x-pack/pipeline.xpack.agentbeat.yml b/.buildkite/x-pack/pipeline.xpack.agentbeat.yml
index 8873ffe29b8e..da66c7186589 100644
--- a/.buildkite/x-pack/pipeline.xpack.agentbeat.yml
+++ b/.buildkite/x-pack/pipeline.xpack.agentbeat.yml
@@ -15,57 +15,57 @@ env:
   AGENTBEAT_SPEC: "./agentbeat.spec.yml"
 
 steps:
-  - group: "Check/Update"
-    key: "x-pack-agentbeat-check-update"
-
-    steps:
-      - label: "agentbeat: Run pre-commit"
-        command: "pre-commit run --all-files"
-        agents:
-          image: "${IMAGE_BEATS_WITH_HOOKS_LATEST}"
-          memory: "2Gi"
-          useCustomGlobalHooks: true
-        notify:
-          - github_commit_status:
-              context: "agentbeat: pre-commit"
-
-  - wait: ~
-    # with PRs, we want to run mandatory tests only if check/update step succeed
-    # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests
-    # this allows building DRA artifacts even if there is flakiness in check/update step
-    if: build.env("BUILDKITE_PULL_REQUEST") != "false"
-    depends_on: "x-pack-agentbeat-check-update"
+#  - group: "Check/Update"
+#    key: "x-pack-agentbeat-check-update"
+#
+#    steps:
+#      - label: "agentbeat: Run pre-commit"
+#        command: "pre-commit run --all-files"
+#        agents:
+#          image: "${IMAGE_BEATS_WITH_HOOKS_LATEST}"
+#          memory: "2Gi"
+#          useCustomGlobalHooks: true
+#        notify:
+#          - github_commit_status:
+#              context: "agentbeat: pre-commit"
+#
+#  - wait: ~
+#    # with PRs, we want to run mandatory tests only if check/update step succeed
+#    # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests
+#    # this allows building DRA artifacts even if there is flakiness in check/update step
+#    if: build.env("BUILDKITE_PULL_REQUEST") != "false"
+#    depends_on: "x-pack-agentbeat-check-update"
 
   - group: "Agentbeat tests"
     key: "agentbeat-mandatory-tests"
 
     steps:
-#      - label: ":linux: Agentbeat packaging Linux"
-#        key: "agentbeat-package-linux"
-#        env:
-#          PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
-#          SNAPSHOT: true
-#        command: |
-#          set -euo pipefail
-#          cd x-pack/agentbeat
-#          mage package
-#        artifact_paths:
-#          - x-pack/agentbeat/build/distributions/**/*
-#          - "x-pack/agentbeat/build/*.xml"
-#          - "x-pack/agentbeat/build/*.json"
-#        retry:
-#          automatic:
-#            - limit: 2
-#        timeout_in_minutes: 60
-#        agents:
-#          provider: "gcp"
-#          image: "${IMAGE_UBUNTU_X86_64}"
-#          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
-#          disk_size: 100
-#          disk_type: "pd-ssd"
-#        notify:
-#          - github_commit_status:
-#              context: "agentbeat: Packaging"
+      - label: ":linux: Agentbeat packaging Linux"
+        key: "agentbeat-package-linux"
+        env:
+          PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
+          SNAPSHOT: true
+        command: |
+          set -euo pipefail
+          cd x-pack/agentbeat
+          mage package
+        artifact_paths:
+          - x-pack/agentbeat/build/distributions/**/*
+          - "x-pack/agentbeat/build/*.xml"
+          - "x-pack/agentbeat/build/*.json"
+        retry:
+          automatic:
+            - limit: 2
+        timeout_in_minutes: 60
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+          disk_size: 100
+          disk_type: "pd-ssd"
+        notify:
+          - github_commit_status:
+              context: "agentbeat: Packaging"
 
       #      - label: ":linux: Agentbeat/Integration tests Linux"
       #        key: "agentbeat-it-linux"
@@ -109,56 +109,60 @@ steps:
       #          - github_commit_status:
       #              context: "agentbeat: Integration tests"
 
-#      - label: ":linux: x-pack/agentbeat: Ubuntu x86_64 Spec tests"
-#        key: "agentbeat-it-linux"
-#        env:
-#          PLATFORM: "linux/amd64"
-#        depends_on:
-#          - agentbeat-package-linux
-#        command: |
-#          cd x-pack/agentbeat
-#          mage -v testWithSpec
-#        agents:
-#          provider: "gcp"
-#          image: "${IMAGE_UBUNTU_X86_64}"
-#          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
-#          disk_size: 100
-#          disk_type: "pd-ssd"
-#
-#      - label: ":windows: x-pack/agentbeat: Windows x86_64 Spec tests"
-#        key: "agentbeat-it-windows"
-#        env:
-#          PLATFORM: "windows/amd64"
-#        depends_on:
-#          - agentbeat-package-linux
-#        command: |
-#          Set-Location -Path x-pack/agentbeat
-#          mage -v testWithSpec
-#        agents:
-#          provider: "gcp"
-#          image: "${IMAGE_WIN_2022}"
-#          machine_type: "${GCP_WIN_MACHINE_TYPE}"
-#          disk_size: 200
-#          disk_type: "pd-ssd"
+      - label: ":linux: x-pack/agentbeat: Ubuntu x86_64 Spec tests"
+        key: "agentbeat-it-linux"
+        env:
+          PLATFORM: "linux/amd64"
+        depends_on:
+          - agentbeat-package-linux
+        command: |
+          cd x-pack/agentbeat
+          mage -v testWithSpec
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+          disk_size: 100
+          disk_type: "pd-ssd"
+
+      - label: ":windows: x-pack/agentbeat: Windows x86_64 Spec tests"
+        key: "agentbeat-it-windows"
+        env:
+          PLATFORM: "windows/amd64"
+        depends_on:
+          - agentbeat-package-linux
+        command: |
+          Set-Location -Path x-pack/agentbeat
+          mage -v testWithSpec
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_WIN_2022}"
+          machine_type: "${GCP_WIN_MACHINE_TYPE}"
+          disk_size: 200
+          disk_type: "pd-ssd"
 
       - label: ":macos: x-pack/agentbeat: macOS x86_64 Spec tests"
         key: "agentbeat-it-macos"
-#        depends_on:
-#          - agentbeat-package-linux
+        env:
+          PLATFORM: "darwin/amd64"
+        depends_on:
+          - agentbeat-package-linux
         command: |
+          set -euo pipefail
+          source .buildkite/scripts/install_macos_tools.sh
           cd x-pack/agentbeat
           mage -v testWithSpec
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_X86_64}"
 
-      - label: ":linux: Agentbeat/Integration tests macOS arm64"
-        key: "agentbeat-it-macos-arm"
+#      - label: ":linux: Agentbeat/Integration tests macOS arm64"
+#        key: "agentbeat-it-macos-arm"
 #        depends_on:
 #          - agentbeat-package-linux
-        command: |
-          cd x-pack/agentbeat
-          mage -v testWithSpec
-        agents:
-          provider: "orka"
-          imagePrefix: "${IMAGE_MACOS_ARM}"
+#        command: |
+#          cd x-pack/agentbeat
+#          mage -v testWithSpec
+#        agents:
+#          provider: "orka"
+#          imagePrefix: "${IMAGE_MACOS_ARM}"
diff --git a/dev-tools/mage/spec.go b/dev-tools/mage/spec.go
index af0527fd6196..03c733f1dd6d 100644
--- a/dev-tools/mage/spec.go
+++ b/dev-tools/mage/spec.go
@@ -19,6 +19,7 @@ package mage
 
 import (
 	"gopkg.in/yaml.v2"
+
 	"log"
 	"os"
 	"strings"

From adf4233e061384e7fbfa0ad4e13d96657d84a4a3 Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Thu, 17 Oct 2024 12:33:04 +0300
Subject: [PATCH 15/17] added macos arm step

---
 .buildkite/hooks/pre-command                  |  1 -
 .../scripts/agentbeat/setup_agentbeat.py      |  9 +-
 .../x-pack/pipeline.xpack.agentbeat.yml       | 93 +++++++------------
 3 files changed, 43 insertions(+), 60 deletions(-)

diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index f1f9bd540294..cb5bb51eb803 100644
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -19,7 +19,6 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" && "$BUILDKITE_STEP
 fi
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-agentbeat" && "$BUILDKITE_STEP_KEY" == *"agentbeat-it"* ]]; then
-#    .buildkite/scripts/agentbeat/setup_agentbeat.py
     AGENTBEAT_PATH=$(.buildkite/scripts/agentbeat/setup_agentbeat.py | tail -n 1)
     export AGENTBEAT_PATH
 fi
diff --git a/.buildkite/scripts/agentbeat/setup_agentbeat.py b/.buildkite/scripts/agentbeat/setup_agentbeat.py
index b5f148a18ccd..35139ec22787 100755
--- a/.buildkite/scripts/agentbeat/setup_agentbeat.py
+++ b/.buildkite/scripts/agentbeat/setup_agentbeat.py
@@ -12,10 +12,12 @@ def log(msg):
     sys.stdout.write(f'{msg}\n')
     sys.stdout.flush()
 
+
 def log_err(msg):
     sys.stderr.write(f'{msg}\n')
     sys.stderr.flush()
 
+
 def get_os() -> str:
     return platform.system().lower()
 
@@ -26,7 +28,10 @@ def get_arch() -> str:
     if arch == 'amd64':
         return 'x86_64'
     else:
-        return arch
+        if get_os() == 'darwin':
+            return 'aarch64'
+        else:
+            return arch
 
 
 def get_artifact_extension(agent_os) -> str:
@@ -40,6 +45,7 @@ def get_artifact_pattern() -> str:
     agent_os = get_os()
     agent_arch = get_arch()
     extension = get_artifact_extension(agent_os)
+    print('Artifact params: ' + agent_os + ' ' + agent_arch + ' ' + extension)
     return f'{PATH}/agentbeat-*-{agent_os}-{agent_arch}.{extension}'
 
 
@@ -47,7 +53,6 @@ def download_agentbeat(pattern, path) -> str:
     try:
         subprocess.run(
             ['buildkite-agent', 'artifact', 'download', pattern, '.',
-             # '--build', '01928f55-8452-41c6-89ba-fe21f019f53c',
              '--step', 'agentbeat-package-linux'],
             check=True, stdout=sys.stdout, stderr=sys.stderr, text=True)
 
diff --git a/.buildkite/x-pack/pipeline.xpack.agentbeat.yml b/.buildkite/x-pack/pipeline.xpack.agentbeat.yml
index da66c7186589..a5375bd1e3d3 100644
--- a/.buildkite/x-pack/pipeline.xpack.agentbeat.yml
+++ b/.buildkite/x-pack/pipeline.xpack.agentbeat.yml
@@ -5,6 +5,9 @@ env:
   GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
   IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
 
+  AWS_ARM_INSTANCE_TYPE: "m6g.xlarge"
+  AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64"
+
   IMAGE_MACOS_ARM: "generic-13-ventura-arm"
   IMAGE_MACOS_X86_64: "generic-13-ventura-x64"
 
@@ -67,50 +70,8 @@ steps:
           - github_commit_status:
               context: "agentbeat: Packaging"
 
-      #      - label: ":linux: Agentbeat/Integration tests Linux"
-      #        key: "agentbeat-it-linux"
-      #        depends_on:
-      #          - agentbeat-package-linux
-      #        env:
-      #          ASDF_NODEJS_VERSION: 18.17.1
-      #          PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
-      #          SNAPSHOT: true
-      #        command: |
-      #          set -euo pipefail
-      #          echo "~~~ Downloading artifacts"
-      #          buildkite-agent artifact download x-pack/agentbeat/build/distributions/** . --step 'agentbeat-package-linux'
-      #          ls -lah x-pack/agentbeat/build/distributions/
-      #          echo "~~~ Installing @elastic/synthetics with npm"
-      #          npm install -g @elastic/synthetics
-      #          echo "~~~ Running tests"
-      #          cd x-pack/agentbeat
-      #          mage goIntegTest
-      #        artifact_paths:
-      #          - x-pack/agentbeat/build/distributions/**/*
-      #          - "x-pack/agentbeat/build/*.xml"
-      #          - "x-pack/agentbeat/build/*.json"
-#              plugins:
-#                - test-collector#v1.10.2:
-#                    files: "x-pack/agentbeat/build/TEST-*.xml"
-#                    format: "junit"
-#                    branches: "main"
-#                    debug: true
-      #        retry:
-      #          automatic:
-      #            - limit: 1
-      #        timeout_in_minutes: 60
-      #        agents:
-      #          provider: "gcp"
-      #          image: "${IMAGE_UBUNTU_X86_64}"
-      #          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
-      #          disk_size: 100
-      #          disk_type: "pd-ssd"
-      #        notify:
-      #          - github_commit_status:
-      #              context: "agentbeat: Integration tests"
-
-      - label: ":linux: x-pack/agentbeat: Ubuntu x86_64 Spec tests"
-        key: "agentbeat-it-linux"
+      - label: ":ubuntu: x-pack/agentbeat: Ubuntu x86_64 Spec tests"
+        key: "agentbeat-it-linux-x86-64"
         env:
           PLATFORM: "linux/amd64"
         depends_on:
@@ -125,6 +86,20 @@ steps:
           disk_size: 100
           disk_type: "pd-ssd"
 
+      - label: ":ubuntu: x-pack/agentbeat: Ubuntu arm64 Spec tests"
+        key: "agentbeat-it-linux-arm64"
+        env:
+          PLATFORM: "linux/arm64"
+        depends_on:
+          - agentbeat-package-linux
+        command: |
+          cd x-pack/agentbeat
+          mage -v testWithSpec
+        agents:
+          provider: "aws"
+          imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
+          instanceType: "${AWS_ARM_INSTANCE_TYPE}"
+
       - label: ":windows: x-pack/agentbeat: Windows x86_64 Spec tests"
         key: "agentbeat-it-windows"
         env:
@@ -142,11 +117,11 @@ steps:
           disk_type: "pd-ssd"
 
       - label: ":macos: x-pack/agentbeat: macOS x86_64 Spec tests"
-        key: "agentbeat-it-macos"
-        env:
-          PLATFORM: "darwin/amd64"
+        key: "agentbeat-it-macos-x86-64"
         depends_on:
           - agentbeat-package-linux
+        env:
+          PLATFORM: "darwin/amd64"
         command: |
           set -euo pipefail
           source .buildkite/scripts/install_macos_tools.sh
@@ -156,13 +131,17 @@ steps:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_X86_64}"
 
-#      - label: ":linux: Agentbeat/Integration tests macOS arm64"
-#        key: "agentbeat-it-macos-arm"
-#        depends_on:
-#          - agentbeat-package-linux
-#        command: |
-#          cd x-pack/agentbeat
-#          mage -v testWithSpec
-#        agents:
-#          provider: "orka"
-#          imagePrefix: "${IMAGE_MACOS_ARM}"
+      - label: ":macos: Agentbeat/Integration tests macOS arm64"
+        key: "agentbeat-it-macos-arm64"
+        depends_on:
+          - agentbeat-package-linux
+        env:
+          PLATFORM: "darwin/arm64"
+        command: |
+          set -euo pipefail
+          source .buildkite/scripts/install_macos_tools.sh
+          cd x-pack/agentbeat
+          mage -v testWithSpec
+        agents:
+          provider: "orka"
+          imagePrefix: "${IMAGE_MACOS_ARM}"

From 4740fbe18decc35456f9c06e81092042926a68cc Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Thu, 17 Oct 2024 16:08:45 +0300
Subject: [PATCH 16/17] refactor

---
 .buildkite/hooks/pre-command                  |  4 +-
 .../scripts/agentbeat/setup_agentbeat.py      | 69 +++++++++++--------
 .../x-pack/pipeline.xpack.agentbeat.yml       | 57 +++++++++------
 3 files changed, 79 insertions(+), 51 deletions(-)

diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index cb5bb51eb803..5cb0722edd84 100644
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -19,7 +19,9 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" && "$BUILDKITE_STEP
 fi
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-agentbeat" && "$BUILDKITE_STEP_KEY" == *"agentbeat-it"* ]]; then
-    AGENTBEAT_PATH=$(.buildkite/scripts/agentbeat/setup_agentbeat.py | tail -n 1)
+    out=$(.buildkite/scripts/agentbeat/setup_agentbeat.py)
+    echo "$out"
+    AGENTBEAT_PATH=$(echo "$out" | tail -n 1)
     export AGENTBEAT_PATH
 fi
 
diff --git a/.buildkite/scripts/agentbeat/setup_agentbeat.py b/.buildkite/scripts/agentbeat/setup_agentbeat.py
index 35139ec22787..02634649e787 100755
--- a/.buildkite/scripts/agentbeat/setup_agentbeat.py
+++ b/.buildkite/scripts/agentbeat/setup_agentbeat.py
@@ -6,6 +6,26 @@
 import tarfile
 
 PATH = 'x-pack/agentbeat/build/distributions'
+PLATFORMS = {
+                'windows': {
+                    'amd64': 'x86_64',
+                },
+                'linux': {
+                    'x86_64': 'x86_64',
+                    'aarch64': 'arm64',
+                },
+                'darwin': {
+                    'x86_64': 'x86_64',
+                    'arm64': 'aarch64',
+                }
+            }
+
+
+class Archive:
+    def __init__(self, os, arch, ext):
+            self.os = os
+            self.arch = arch
+            self.ext = ext
 
 
 def log(msg):
@@ -18,38 +38,29 @@ def log_err(msg):
     sys.stderr.flush()
 
 
-def get_os() -> str:
-    return platform.system().lower()
+def get_archive_params() -> Archive:
+    system = platform.system().lower()
+    machine = platform.machine().lower()
+    arch = PLATFORMS.get(system, {}).get(machine)
+    ext = get_artifact_extension(system)
 
+    return Archive(system, arch, ext)
 
-def get_arch() -> str:
-    arch = platform.machine().lower()
 
-    if arch == 'amd64':
-        return 'x86_64'
-    else:
-        if get_os() == 'darwin':
-            return 'aarch64'
-        else:
-            return arch
-
-
-def get_artifact_extension(agent_os) -> str:
-    if agent_os == 'windows':
+def get_artifact_extension(system) -> str:
+    if system == 'windows':
         return 'zip'
     else:
         return 'tar.gz'
 
 
-def get_artifact_pattern() -> str:
-    agent_os = get_os()
-    agent_arch = get_arch()
-    extension = get_artifact_extension(agent_os)
-    print('Artifact params: ' + agent_os + ' ' + agent_arch + ' ' + extension)
-    return f'{PATH}/agentbeat-*-{agent_os}-{agent_arch}.{extension}'
+def get_artifact_pattern(archive_obj) -> str:
+    return f'{PATH}/agentbeat-*-{archive_obj.os}-{archive_obj.arch}.{archive_obj.ext}'
 
 
-def download_agentbeat(pattern, path) -> str:
+def download_agentbeat(archive_obj) -> str:
+    pattern = get_artifact_pattern(archive_obj)
+    log('--- Downloading Agentbeat artifact by pattern: ' + pattern)
     try:
         subprocess.run(
             ['buildkite-agent', 'artifact', 'download', pattern, '.',
@@ -59,15 +70,14 @@ def download_agentbeat(pattern, path) -> str:
     except subprocess.CalledProcessError:
         exit(1)
 
-    return get_filename(path)
+    return get_full_filename()
 
 
-def get_filename(path) -> str:
+def get_full_filename() -> str:
     try:
         out = subprocess.run(
-            ['ls', '-p', path],
+            ['ls', '-p', PATH],
             check=True, capture_output=True, text=True)
-        print("--- ls -p: " + out.stdout)
         return out.stdout.strip()
     except subprocess.CalledProcessError:
         exit(1)
@@ -75,6 +85,7 @@ def get_filename(path) -> str:
 
 def extract_agentbeat(filename):
     filepath = PATH + '/' + filename
+    log('Extracting Agentbeat artifact: ' + filepath)
 
     if filepath.endswith('.zip'):
         unzip_agentbeat(filepath)
@@ -109,10 +120,10 @@ def get_path_to_executable(filepath) -> str:
         path = f'../../{match.group(1)}/agentbeat'
         return path
     else:
-        log_err("No agentbeat executable found")
+        log_err('No agentbeat executable found')
         exit(1)
 
-artifact_pattern = get_artifact_pattern()
-archive = download_agentbeat(artifact_pattern, PATH)
+archive_params = get_archive_params()
+archive = download_agentbeat(archive_params)
 extract_agentbeat(archive)
 log(get_path_to_executable(archive))
diff --git a/.buildkite/x-pack/pipeline.xpack.agentbeat.yml b/.buildkite/x-pack/pipeline.xpack.agentbeat.yml
index a5375bd1e3d3..1687aa25d922 100644
--- a/.buildkite/x-pack/pipeline.xpack.agentbeat.yml
+++ b/.buildkite/x-pack/pipeline.xpack.agentbeat.yml
@@ -18,26 +18,26 @@ env:
   AGENTBEAT_SPEC: "./agentbeat.spec.yml"
 
 steps:
-#  - group: "Check/Update"
-#    key: "x-pack-agentbeat-check-update"
-#
-#    steps:
-#      - label: "agentbeat: Run pre-commit"
-#        command: "pre-commit run --all-files"
-#        agents:
-#          image: "${IMAGE_BEATS_WITH_HOOKS_LATEST}"
-#          memory: "2Gi"
-#          useCustomGlobalHooks: true
-#        notify:
-#          - github_commit_status:
-#              context: "agentbeat: pre-commit"
-#
-#  - wait: ~
-#    # with PRs, we want to run mandatory tests only if check/update step succeed
-#    # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests
-#    # this allows building DRA artifacts even if there is flakiness in check/update step
-#    if: build.env("BUILDKITE_PULL_REQUEST") != "false"
-#    depends_on: "x-pack-agentbeat-check-update"
+  - group: "Check/Update"
+    key: "x-pack-agentbeat-check-update"
+
+    steps:
+      - label: "agentbeat: Run pre-commit"
+        command: "pre-commit run --all-files"
+        agents:
+          image: "${IMAGE_BEATS_WITH_HOOKS_LATEST}"
+          memory: "2Gi"
+          useCustomGlobalHooks: true
+        notify:
+          - github_commit_status:
+              context: "agentbeat: pre-commit"
+
+  - wait: ~
+    # with PRs, we want to run mandatory tests only if check/update step succeed
+    # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests
+    # this allows building DRA artifacts even if there is flakiness in check/update step
+    if: build.env("BUILDKITE_PULL_REQUEST") != "false"
+    depends_on: "x-pack-agentbeat-check-update"
 
   - group: "Agentbeat tests"
     key: "agentbeat-mandatory-tests"
@@ -85,6 +85,9 @@ steps:
           machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
           disk_size: 100
           disk_type: "pd-ssd"
+        notify:
+          - github_commit_status:
+              context: "agentbeat: Ubuntu x86_64 Spec tests"
 
       - label: ":ubuntu: x-pack/agentbeat: Ubuntu arm64 Spec tests"
         key: "agentbeat-it-linux-arm64"
@@ -99,6 +102,9 @@ steps:
           provider: "aws"
           imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
           instanceType: "${AWS_ARM_INSTANCE_TYPE}"
+        notify:
+          - github_commit_status:
+              context: "agentbeat: Ubuntu arm64 Spec tests"
 
       - label: ":windows: x-pack/agentbeat: Windows x86_64 Spec tests"
         key: "agentbeat-it-windows"
@@ -115,6 +121,9 @@ steps:
           machine_type: "${GCP_WIN_MACHINE_TYPE}"
           disk_size: 200
           disk_type: "pd-ssd"
+        notify:
+          - github_commit_status:
+              context: "agentbeat: Windows x86_64 Spec tests"
 
       - label: ":macos: x-pack/agentbeat: macOS x86_64 Spec tests"
         key: "agentbeat-it-macos-x86-64"
@@ -130,8 +139,11 @@ steps:
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_X86_64}"
+        notify:
+          - github_commit_status:
+              context: "agentbeat: macOS x86_64 Spec tests"
 
-      - label: ":macos: Agentbeat/Integration tests macOS arm64"
+      - label: ":macos: x-pack/agentbeat: macOS arm64 Spec tests"
         key: "agentbeat-it-macos-arm64"
         depends_on:
           - agentbeat-package-linux
@@ -145,3 +157,6 @@ steps:
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_ARM}"
+        notify:
+          - github_commit_status:
+              context: "agentbeat: macOS arm64 Spec tests"

From 22190d441297146cce3cf27c20930567e0dc93f1 Mon Sep 17 00:00:00 2001
From: Olga Naidjonoka <olga_naidjonoka@epam.com>
Date: Fri, 18 Oct 2024 14:58:40 +0300
Subject: [PATCH 17/17] restored testPackages annotation

---
 x-pack/agentbeat/magefile.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/x-pack/agentbeat/magefile.go b/x-pack/agentbeat/magefile.go
index d02b5f9c83a8..bd72a558ba39 100644
--- a/x-pack/agentbeat/magefile.go
+++ b/x-pack/agentbeat/magefile.go
@@ -145,7 +145,7 @@ func Package() error {
 	return nil
 }
 
-// TestPackages tests the generated packages (i.runCmd. file modes, owners, groups).
+// TestPackages tests the generated packages (i.e. file modes, owners, groups).
 func TestPackages() error {
 	return devtools.TestPackages()
 }