From 0ea0b6bbee60f030dfca53d2e203d4649a703b6c Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka <olga_naidjonoka@epam.com> Date: Wed, 2 Oct 2024 11:50:20 +0300 Subject: [PATCH 01/17] adjusted pipeline --- .buildkite/scripts/agentbeat/prepare_env.py | 57 +++++ .../x-pack/pipeline.xpack.agentbeat.yml | 202 +++++++++++------- 2 files changed, 184 insertions(+), 75 deletions(-) create mode 100755 .buildkite/scripts/agentbeat/prepare_env.py diff --git a/.buildkite/scripts/agentbeat/prepare_env.py b/.buildkite/scripts/agentbeat/prepare_env.py new file mode 100755 index 000000000000..50f7d8123299 --- /dev/null +++ b/.buildkite/scripts/agentbeat/prepare_env.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 + +import os +import platform +import subprocess + + +def get_os() -> str: + return platform.system() + + +def get_arch() -> str: + return platform.machine() + + +def get_cwd() -> str: + return os.getcwd() + + +def download_agentbeat_artifact(os, arch): + pattern = "x-pack/agentbeat/build/distributions/agentbeat-9.0.0-SNAPSHOT-linux-x86_64.tar.gz" + # pattern = "x-pack/agentbeat/build/distributions/**" + # command = f"buildkite-agent artifact download \"{pattern}\" . --step 'agentbeat-package-linux'" + + try: + print("--- Downloading agentbeat artifact") + result = subprocess.run( + ["buildkite-agent", "artifact", "download", pattern, ".", + "--build", "01924d2b-b061-45ae-a106-e885584ff26f", + "--step", "agentbeat-package-linux"], + check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + print(result.stdout.decode()) + except subprocess.CalledProcessError as e: + print("--- Error occurred while downloading agentbeat\n" + e.stderr) + exit(1) + + +def install_synthetics(): + try: + print("--- Installing @elastic/synthetics") + subprocess.run( + ["npm install -g @elastic/synthetics"], + check=True + ) + except subprocess.CalledProcessError: + print("Failed to install @elastic/synthetics") + exit(1) + + +# print("--- OS: " + get_os()) +# +# print("--- ARCH: " + get_arch()) +# +# print("--- CWD: " + get_cwd()) + +download_agentbeat_artifact(get_os(), get_arch()) +# install_synthetics() diff --git a/.buildkite/x-pack/pipeline.xpack.agentbeat.yml b/.buildkite/x-pack/pipeline.xpack.agentbeat.yml index ef7cb1598aa4..0af413435cbc 100644 --- a/.buildkite/x-pack/pipeline.xpack.agentbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.agentbeat.yml @@ -1,101 +1,153 @@ env: ASDF_MAGE_VERSION: 1.15.0 + ASDF_NODEJS_VERSION: 18.17.1 + GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + IMAGE_MACOS_ARM: "generic-13-ventura-arm" + IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + + IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" + IMAGE_BEATS_WITH_HOOKS_LATEST: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest" steps: - - group: "Check/Update" - key: "x-pack-agentbeat-check-update" - - steps: - - label: "agentbeat: Run pre-commit" - command: "pre-commit run --all-files" - agents: - image: "${IMAGE_BEATS_WITH_HOOKS_LATEST}" - memory: "2Gi" - useCustomGlobalHooks: true - notify: - - github_commit_status: - context: "agentbeat: pre-commit" +# - group: "Check/Update" +# key: "x-pack-agentbeat-check-update" +# +# steps: +# - label: "agentbeat: Run pre-commit" +# command: "pre-commit run --all-files" +# agents: +# image: "${IMAGE_BEATS_WITH_HOOKS_LATEST}" +# memory: "2Gi" +# useCustomGlobalHooks: true +# notify: +# - github_commit_status: +# context: "agentbeat: pre-commit" - - wait: ~ - # with PRs, we want to run mandatory tests only if check/update step succeed - # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests - # this allows building DRA artifacts even if there is flakiness in check/update step - if: build.env("BUILDKITE_PULL_REQUEST") != "false" - depends_on: "x-pack-agentbeat-check-update" +# - wait: ~ +# # with PRs, we want to run mandatory tests only if check/update step succeed +# # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests +# # this allows building DRA artifacts even if there is flakiness in check/update step +# if: build.env("BUILDKITE_PULL_REQUEST") != "false" +# depends_on: "x-pack-agentbeat-check-update" - group: "Agentbeat tests" key: "agentbeat-mandatory-tests" steps: - - label: ":linux: Agentbeat packaging Linux" - key: "agentbeat-package-linux" - env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - SNAPSHOT: true - command: | - set -euo pipefail - cd x-pack/agentbeat - mage package - artifact_paths: - - x-pack/agentbeat/build/distributions/**/* - - "x-pack/agentbeat/build/*.xml" - - "x-pack/agentbeat/build/*.json" - retry: - automatic: - - limit: 2 - timeout_in_minutes: 60 - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - notify: - - github_commit_status: - context: "agentbeat: Packaging" +# - label: ":linux: Agentbeat packaging Linux" +# key: "agentbeat-package-linux" +# env: +# PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" +# SNAPSHOT: true +# command: | +# set -euo pipefail +# cd x-pack/agentbeat +# mage package +# artifact_paths: +# - x-pack/agentbeat/build/distributions/**/* +# - "x-pack/agentbeat/build/*.xml" +# - "x-pack/agentbeat/build/*.json" +# retry: +# automatic: +# - limit: 2 +# timeout_in_minutes: 60 +# agents: +# provider: "gcp" +# image: "${IMAGE_UBUNTU_X86_64}" +# machineType: "${GCP_HI_PERF_MACHINE_TYPE}" +# disk_size: 100 +# disk_type: "pd-ssd" +# notify: +# - github_commit_status: +# context: "agentbeat: Packaging" + + # - label: ":linux: Agentbeat/Integration tests Linux" + # key: "agentbeat-it-linux" + # depends_on: + # - agentbeat-package-linux + # env: + # ASDF_NODEJS_VERSION: 18.17.1 + # PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + # SNAPSHOT: true + # command: | + # set -euo pipefail + # echo "~~~ Downloading artifacts" + # buildkite-agent artifact download x-pack/agentbeat/build/distributions/** . --step 'agentbeat-package-linux' + # ls -lah x-pack/agentbeat/build/distributions/ + # echo "~~~ Installing @elastic/synthetics with npm" + # npm install -g @elastic/synthetics + # echo "~~~ Running tests" + # cd x-pack/agentbeat + # mage goIntegTest + # artifact_paths: + # - x-pack/agentbeat/build/distributions/**/* + # - "x-pack/agentbeat/build/*.xml" + # - "x-pack/agentbeat/build/*.json" + # retry: + # automatic: + # - limit: 1 + # timeout_in_minutes: 60 + # agents: + # provider: "gcp" + # image: "${IMAGE_UBUNTU_X86_64}" + # machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + # disk_size: 100 + # disk_type: "pd-ssd" + # notify: + # - github_commit_status: + # context: "agentbeat: Integration tests" - label: ":linux: Agentbeat/Integration tests Linux" key: "agentbeat-it-linux" - depends_on: - - agentbeat-package-linux - env: - ASDF_NODEJS_VERSION: 18.17.1 - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - SNAPSHOT: true +# depends_on: +# - agentbeat-package-linux command: | set -euo pipefail - echo "~~~ Downloading artifacts" - buildkite-agent artifact download x-pack/agentbeat/build/distributions/** . --step 'agentbeat-package-linux' + # buildkite-agent artifact download x-pack/agentbeat/build/distributions/** . --build 01924d2b-b061-45ae-a106-e885584ff26f --step agentbeat-package-linux + ./.buildkite/scripts/agentbeat/prepare_env.py ls -lah x-pack/agentbeat/build/distributions/ - echo "~~~ Installing @elastic/synthetics with npm" - npm install -g @elastic/synthetics - echo "~~~ Running tests" - cd x-pack/agentbeat - mage goIntegTest - artifact_paths: - - x-pack/agentbeat/build/distributions/**/* - - "x-pack/agentbeat/build/*.xml" - - "x-pack/agentbeat/build/*.json" - plugins: - - test-collector#v1.10.2: - files: "x-pack/agentbeat/build/TEST-*.xml" - format: "junit" - branches: "main" - debug: true - retry: - automatic: - - limit: 1 - timeout_in_minutes: 60 agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" - notify: - - github_commit_status: - context: "agentbeat: Integration tests" + +# - label: ":linux: Agentbeat/Integration tests Windows" +# key: "agentbeat-it-windows" +# depends_on: +# - agentbeat-package-linux +# commands: +# - ./.buildkite/scripts/agentbeat/prepare_env.py +# agents: +# provider: "gcp" +# image: "${IMAGE_WIN_2022}" +# machine_type: "${GCP_WIN_MACHINE_TYPE}" +# disk_size: 200 +# disk_type: "pd-ssd" +# +# - label: ":linux: Agentbeat/Integration tests macOS" +# key: "agentbeat-it-macos" +# depends_on: +# - agentbeat-package-linux +# command: | +# set -euo pipefail +# ./.buildkite/scripts/agentbeat/prepare_env.py +# agents: +# provider: "orka" +# imagePrefix: "${IMAGE_MACOS_X86_64}" +# +# - label: ":linux: Agentbeat/Integration tests macOS arm64" +# key: "agentbeat-it-macos-arm" +# depends_on: +# - agentbeat-package-linux +# command: | +# set -euo pipefail +# ./.buildkite/scripts/agentbeat/prepare_env.py +# agents: +# provider: "orka" +# imagePrefix: "${IMAGE_MACOS_ARM}" From 953d4f5af0a15cf517eae92fde944068734a0ce6 Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka <olga_naidjonoka@epam.com> Date: Wed, 2 Oct 2024 17:55:27 +0300 Subject: [PATCH 02/17] updated artifact pattern --- .buildkite/scripts/agentbeat/prepare_env.py | 42 +++++++++---------- .../x-pack/pipeline.xpack.agentbeat.yml | 22 +++++----- 2 files changed, 30 insertions(+), 34 deletions(-) diff --git a/.buildkite/scripts/agentbeat/prepare_env.py b/.buildkite/scripts/agentbeat/prepare_env.py index 50f7d8123299..11d11bc56cda 100755 --- a/.buildkite/scripts/agentbeat/prepare_env.py +++ b/.buildkite/scripts/agentbeat/prepare_env.py @@ -1,43 +1,47 @@ #!/usr/bin/env python3 -import os import platform import subprocess +import sys def get_os() -> str: - return platform.system() + return platform.system().lower() def get_arch() -> str: - return platform.machine() + arch = platform.machine().lower() + if arch == "amd64": + return "x86_64" + else: + return arch -def get_cwd() -> str: - return os.getcwd() +def download_agentbeat_artifact(agent_os, agent_arch): + pattern = f"x-pack/agentbeat/build/distributions/agentbeat-*-{agent_os}-{agent_arch}.tar.gz" -def download_agentbeat_artifact(os, arch): - pattern = "x-pack/agentbeat/build/distributions/agentbeat-9.0.0-SNAPSHOT-linux-x86_64.tar.gz" - # pattern = "x-pack/agentbeat/build/distributions/**" - # command = f"buildkite-agent artifact download \"{pattern}\" . --step 'agentbeat-package-linux'" + print("--- Downloading agentbeat artifact") try: - print("--- Downloading agentbeat artifact") - result = subprocess.run( + subprocess.run( ["buildkite-agent", "artifact", "download", pattern, ".", "--build", "01924d2b-b061-45ae-a106-e885584ff26f", "--step", "agentbeat-package-linux"], - check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - print(result.stdout.decode()) + check=True, stdout=sys.stdout, stderr=subprocess.PIPE, text=True) except subprocess.CalledProcessError as e: - print("--- Error occurred while downloading agentbeat\n" + e.stderr) + print("--- Error occurred. Failed to download agentbeat: \n" + e.stderr) exit(1) +def unzip_agentbeat(): + print("todo unzip") + + def install_synthetics(): + print("--- Installing @elastic/synthetics") + try: - print("--- Installing @elastic/synthetics") subprocess.run( ["npm install -g @elastic/synthetics"], check=True @@ -46,12 +50,6 @@ def install_synthetics(): print("Failed to install @elastic/synthetics") exit(1) - -# print("--- OS: " + get_os()) -# -# print("--- ARCH: " + get_arch()) -# -# print("--- CWD: " + get_cwd()) - +print("--- OS Data: " + get_os() + " " + get_arch()) download_agentbeat_artifact(get_os(), get_arch()) # install_synthetics() diff --git a/.buildkite/x-pack/pipeline.xpack.agentbeat.yml b/.buildkite/x-pack/pipeline.xpack.agentbeat.yml index 0af413435cbc..cfed290ffba7 100644 --- a/.buildkite/x-pack/pipeline.xpack.agentbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.agentbeat.yml @@ -107,9 +107,7 @@ steps: # - agentbeat-package-linux command: | set -euo pipefail - # buildkite-agent artifact download x-pack/agentbeat/build/distributions/** . --build 01924d2b-b061-45ae-a106-e885584ff26f --step agentbeat-package-linux ./.buildkite/scripts/agentbeat/prepare_env.py - ls -lah x-pack/agentbeat/build/distributions/ agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -117,18 +115,18 @@ steps: disk_size: 100 disk_type: "pd-ssd" -# - label: ":linux: Agentbeat/Integration tests Windows" -# key: "agentbeat-it-windows" + - label: ":linux: Agentbeat/Integration tests Windows" + key: "agentbeat-it-windows" # depends_on: # - agentbeat-package-linux -# commands: -# - ./.buildkite/scripts/agentbeat/prepare_env.py -# agents: -# provider: "gcp" -# image: "${IMAGE_WIN_2022}" -# machine_type: "${GCP_WIN_MACHINE_TYPE}" -# disk_size: 200 -# disk_type: "pd-ssd" + commands: + - ./.buildkite/scripts/agentbeat/prepare_env.py + agents: + provider: "gcp" + image: "${IMAGE_WIN_2022}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 200 + disk_type: "pd-ssd" # # - label: ":linux: Agentbeat/Integration tests macOS" # key: "agentbeat-it-macos" From 69713ba9c37a076eddde084c9a27f76200a464b8 Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka <olga_naidjonoka@epam.com> Date: Thu, 3 Oct 2024 12:35:46 +0300 Subject: [PATCH 03/17] added stderr --- .buildkite/scripts/agentbeat/prepare_env.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/scripts/agentbeat/prepare_env.py b/.buildkite/scripts/agentbeat/prepare_env.py index 11d11bc56cda..028f76476fbd 100755 --- a/.buildkite/scripts/agentbeat/prepare_env.py +++ b/.buildkite/scripts/agentbeat/prepare_env.py @@ -28,9 +28,9 @@ def download_agentbeat_artifact(agent_os, agent_arch): ["buildkite-agent", "artifact", "download", pattern, ".", "--build", "01924d2b-b061-45ae-a106-e885584ff26f", "--step", "agentbeat-package-linux"], - check=True, stdout=sys.stdout, stderr=subprocess.PIPE, text=True) + check=True, stdout=sys.stdout, stderr=sys.stderr, text=True) except subprocess.CalledProcessError as e: - print("--- Error occurred. Failed to download agentbeat: \n" + e.stderr) + print("Error occurred. Failed to download agentbeat: \n" + e.stderr) exit(1) From c4ae8438604b30803470bea5cd84c84ebe332650 Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka <olga_naidjonoka@epam.com> Date: Thu, 3 Oct 2024 12:42:42 +0300 Subject: [PATCH 04/17] added stderr --- .buildkite/scripts/agentbeat/prepare_env.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/scripts/agentbeat/prepare_env.py b/.buildkite/scripts/agentbeat/prepare_env.py index 028f76476fbd..c3dbfd0977dd 100755 --- a/.buildkite/scripts/agentbeat/prepare_env.py +++ b/.buildkite/scripts/agentbeat/prepare_env.py @@ -28,14 +28,14 @@ def download_agentbeat_artifact(agent_os, agent_arch): ["buildkite-agent", "artifact", "download", pattern, ".", "--build", "01924d2b-b061-45ae-a106-e885584ff26f", "--step", "agentbeat-package-linux"], - check=True, stdout=sys.stdout, stderr=sys.stderr, text=True) + check=True, stdout=sys.stdout, stderr=subprocess.PIPE, text=True) except subprocess.CalledProcessError as e: print("Error occurred. Failed to download agentbeat: \n" + e.stderr) exit(1) def unzip_agentbeat(): - print("todo unzip") + print("todo unzip") def install_synthetics(): From 69bd82f9717f292dcb8ee4c5e09caa479e977565 Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka <olga_naidjonoka@epam.com> Date: Thu, 3 Oct 2024 13:00:13 +0300 Subject: [PATCH 05/17] added stderr --- .buildkite/scripts/agentbeat/prepare_env.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.buildkite/scripts/agentbeat/prepare_env.py b/.buildkite/scripts/agentbeat/prepare_env.py index c3dbfd0977dd..6a5c0fd487c9 100755 --- a/.buildkite/scripts/agentbeat/prepare_env.py +++ b/.buildkite/scripts/agentbeat/prepare_env.py @@ -18,6 +18,13 @@ def get_arch() -> str: return arch +def get_artifact_extension(agent_os) -> str: + if agent_os == "windows": + return "zip" + else: + return "tar.gz" + + def download_agentbeat_artifact(agent_os, agent_arch): pattern = f"x-pack/agentbeat/build/distributions/agentbeat-*-{agent_os}-{agent_arch}.tar.gz" @@ -28,9 +35,9 @@ def download_agentbeat_artifact(agent_os, agent_arch): ["buildkite-agent", "artifact", "download", pattern, ".", "--build", "01924d2b-b061-45ae-a106-e885584ff26f", "--step", "agentbeat-package-linux"], - check=True, stdout=sys.stdout, stderr=subprocess.PIPE, text=True) - except subprocess.CalledProcessError as e: - print("Error occurred. Failed to download agentbeat: \n" + e.stderr) + check=True, stdout=sys.stdout, stderr=sys.stderr, text=True) + except subprocess.CalledProcessError: + print("Failed to download agentbeat") exit(1) From 618f2d7159dd38e87de9596d91c5a298fa2d7486 Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka <olga_naidjonoka@epam.com> Date: Thu, 3 Oct 2024 13:21:00 +0300 Subject: [PATCH 06/17] test error otput --- .buildkite/scripts/agentbeat/prepare_env.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.buildkite/scripts/agentbeat/prepare_env.py b/.buildkite/scripts/agentbeat/prepare_env.py index 6a5c0fd487c9..db42e64ee8fc 100755 --- a/.buildkite/scripts/agentbeat/prepare_env.py +++ b/.buildkite/scripts/agentbeat/prepare_env.py @@ -26,9 +26,9 @@ def get_artifact_extension(agent_os) -> str: def download_agentbeat_artifact(agent_os, agent_arch): - pattern = f"x-pack/agentbeat/build/distributions/agentbeat-*-{agent_os}-{agent_arch}.tar.gz" - - print("--- Downloading agentbeat artifact") + print(" ") + extension = get_artifact_extension(agent_os) + pattern = f"x-pack/agentbeat/build/distributions/agentbeat-*-{agent_os}-{agent_arch}.{extension}" try: subprocess.run( @@ -37,12 +37,17 @@ def download_agentbeat_artifact(agent_os, agent_arch): "--step", "agentbeat-package-linux"], check=True, stdout=sys.stdout, stderr=sys.stderr, text=True) except subprocess.CalledProcessError: - print("Failed to download agentbeat") exit(1) def unzip_agentbeat(): print("todo unzip") + try: + subprocess.run( + ["unzip"], + check=True, stdout=sys.stdout, stderr=sys.stderr, text=True) + except subprocess.CalledProcessError: + exit(1) def install_synthetics(): From 68925207cd5dbd996a0aa7e706bf9402d4a51ed4 Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka <olga_naidjonoka@epam.com> Date: Thu, 3 Oct 2024 13:51:23 +0300 Subject: [PATCH 07/17] added unzip --- .buildkite/scripts/agentbeat/prepare_env.py | 107 ++++++++++++++---- .../x-pack/pipeline.xpack.agentbeat.yml | 4 + dev-tools/mage/parse_spec.go | 98 ++++++++++++++++ x-pack/agentbeat/magefile.go | 4 + 4 files changed, 192 insertions(+), 21 deletions(-) create mode 100644 dev-tools/mage/parse_spec.go diff --git a/.buildkite/scripts/agentbeat/prepare_env.py b/.buildkite/scripts/agentbeat/prepare_env.py index db42e64ee8fc..4bda517516ba 100755 --- a/.buildkite/scripts/agentbeat/prepare_env.py +++ b/.buildkite/scripts/agentbeat/prepare_env.py @@ -3,7 +3,20 @@ import platform import subprocess import sys +import tarfile +import os +import re +PATH = 'x-pack/agentbeat/build/distributions' + + +def log(msg): + sys.stdout.write(f'{msg}\n') + sys.stdout.flush() + +def log_err(msg): + sys.stderr.write(f'{msg}\n') + sys.stderr.flush() def get_os() -> str: return platform.system().lower() @@ -12,56 +25,108 @@ def get_os() -> str: def get_arch() -> str: arch = platform.machine().lower() - if arch == "amd64": - return "x86_64" + if arch == 'amd64': + return 'x86_64' else: return arch def get_artifact_extension(agent_os) -> str: - if agent_os == "windows": - return "zip" + if agent_os == 'windows': + return 'zip' else: - return "tar.gz" + return 'tar.gz' -def download_agentbeat_artifact(agent_os, agent_arch): - print(" ") +def get_artifact_pattern() -> str: + agent_os = get_os() + agent_arch = get_arch() extension = get_artifact_extension(agent_os) - pattern = f"x-pack/agentbeat/build/distributions/agentbeat-*-{agent_os}-{agent_arch}.{extension}" + return f'{PATH}/agentbeat-*-{agent_os}-{agent_arch}.{extension}' + +def download_agentbeat(pattern, path) -> str: + log('--- Downloading agentbeat') try: subprocess.run( - ["buildkite-agent", "artifact", "download", pattern, ".", - "--build", "01924d2b-b061-45ae-a106-e885584ff26f", - "--step", "agentbeat-package-linux"], + ['buildkite-agent', 'artifact', 'download', pattern, '.', + '--build', '01924d2b-b061-45ae-a106-e885584ff26f', + '--step', 'agentbeat-package-linux'], check=True, stdout=sys.stdout, stderr=sys.stderr, text=True) except subprocess.CalledProcessError: exit(1) + return get_filename(path) + + +def get_filename(path) -> str: + try: + out = subprocess.run( + ['ls', '-p', path], + check=True, capture_output=True, text=True) + return out.stdout.strip() + except subprocess.CalledProcessError: + exit(1) + + +def extract_agentbeat(filename): + log('~~~ Extracting agentbeat') + filepath = PATH + '/' + filename + + if filepath.endswith('.zip'): + unzip_agentbeat(filepath) + else: + untar_agentbeat(filepath) + log('Successfully extracted agentbeat') + -def unzip_agentbeat(): - print("todo unzip") +def unzip_agentbeat(filepath): try: subprocess.run( - ["unzip"], + ['unzip', filepath], check=True, stdout=sys.stdout, stderr=sys.stderr, text=True) - except subprocess.CalledProcessError: + except subprocess.CalledProcessError as e: + log_err(e) + exit(1) + + +def untar_agentbeat(filepath): + try: + with tarfile.open(filepath, 'r:gz') as tar: + tar.list() + tar.extractall() + except Exception as e: + log_err(e) + exit(1) + + +def add_to_path(filepath): + pattern = r'(.*)(?=\.zip|.tar\.gz)' + match = re.match(pattern, filepath) + if match: + path = f'../build/distributions/{match.group(1)}/agentbeat' + log('--- AGENTBEAT_PATH: ' + path) + os.environ['AGENTBEAT_PATH'] = path + else: + log_err("No agentbeat executable found") exit(1) def install_synthetics(): - print("--- Installing @elastic/synthetics") + log('--- Installing @elastic/synthetics') try: subprocess.run( - ["npm install -g @elastic/synthetics"], + ['npm', 'install', '-g', '@elastic/synthetics'], check=True ) except subprocess.CalledProcessError: - print("Failed to install @elastic/synthetics") + log_err('Failed to install @elastic/synthetics') exit(1) -print("--- OS Data: " + get_os() + " " + get_arch()) -download_agentbeat_artifact(get_os(), get_arch()) -# install_synthetics() + +artifact_pattern = get_artifact_pattern() +archive = download_agentbeat(artifact_pattern, PATH) +extract_agentbeat(archive) +add_to_path(archive) +install_synthetics() diff --git a/.buildkite/x-pack/pipeline.xpack.agentbeat.yml b/.buildkite/x-pack/pipeline.xpack.agentbeat.yml index cfed290ffba7..71607aefb250 100644 --- a/.buildkite/x-pack/pipeline.xpack.agentbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.agentbeat.yml @@ -12,6 +12,8 @@ env: IMAGE_BEATS_WITH_HOOKS_LATEST: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest" + AGENTBEAT_SPEC: "../agentbeat.spec.yml" + steps: # - group: "Check/Update" # key: "x-pack-agentbeat-check-update" @@ -103,6 +105,8 @@ steps: - label: ":linux: Agentbeat/Integration tests Linux" key: "agentbeat-it-linux" + env: + PLATFORM: "linux/amd64" # depends_on: # - agentbeat-package-linux command: | diff --git a/dev-tools/mage/parse_spec.go b/dev-tools/mage/parse_spec.go new file mode 100644 index 000000000000..5dc20199288d --- /dev/null +++ b/dev-tools/mage/parse_spec.go @@ -0,0 +1,98 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mage + +import ( + "gopkg.in/yaml.v2" + "log" + "os" +) + +type spec struct { + Inputs []input +} + +type input struct { + Name string + Description string + Platforms []string + Command command +} + +type command struct { + Name string + Args []string +} + +// ParseSpec parses agent.beat.spec.yml and generates test command +func ParseSpec() { + specPath := os.Getenv("AGENTBEAT_SPEC") + if specPath == "" { + log.Fatal("AGENTBEAT_SPEC is not defined") + } + + platform := os.Getenv("PLATFORM") + if platform == "" { + log.Fatal("PLATFORM is not defined") + } + + spec, err := parseToObj() + if err != nil { + log.Fatalf("Error parsing agentbeat.spec.yml: %v", err) + } + + inputList := filter(spec.Inputs, func(input input) bool { + return contains(input.Platforms, platform) + }) + + log.Print(inputList) +} + +func parseToObj() (spec, error) { + specFile, err := os.ReadFile("../agentbeat.spec.yml") + if err != nil { + log.Fatalf("Error opening agentbeat.spec.yml: %v", err) + return spec{}, err + } + var spec spec + err = yaml.Unmarshal(specFile, &spec) + if err != nil { + log.Fatalf("Error parsing agentbeat.spec.yml: %v", err) + return spec, err + } + return spec, nil +} + +func filter[T any](slice []T, condition func(T) bool) []T { + var result []T + for _, v := range slice { + if condition(v) { + result = append(result, v) + } + } + return result +} + +func contains(slice []string, item string) bool { + for _, v := range slice { + if v == item { + return true + } + } + return false +} diff --git a/x-pack/agentbeat/magefile.go b/x-pack/agentbeat/magefile.go index b65a3e59af36..097695552a7c 100644 --- a/x-pack/agentbeat/magefile.go +++ b/x-pack/agentbeat/magefile.go @@ -213,3 +213,7 @@ func PythonIntegTest(ctx context.Context) error { mg.Deps(BuildSystemTestBinary) return devtools.PythonIntegTestFromHost(devtools.DefaultPythonTestIntegrationFromHostArgs()) } + +func TestWithSpecs(ctx context.Context) { + mg.Deps(devtools.ParseSpec) +} From 4b59d7d3382a5a44367a5c9dde8124bd875fefaa Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka <olga_naidjonoka@epam.com> Date: Tue, 8 Oct 2024 16:47:32 +0300 Subject: [PATCH 08/17] added TestWithSpec to magefile --- .buildkite/scripts/agentbeat/prepare_env.py | 4 +- .../x-pack/pipeline.xpack.agentbeat.yml | 10 ++- dev-tools/mage/{parse_spec.go => spec.go} | 31 ++++++--- x-pack/agentbeat/magefile.go | 66 ++++++++++++++++++- 4 files changed, 93 insertions(+), 18 deletions(-) rename dev-tools/mage/{parse_spec.go => spec.go} (75%) diff --git a/.buildkite/scripts/agentbeat/prepare_env.py b/.buildkite/scripts/agentbeat/prepare_env.py index 4bda517516ba..d17c3cdd0e10 100755 --- a/.buildkite/scripts/agentbeat/prepare_env.py +++ b/.buildkite/scripts/agentbeat/prepare_env.py @@ -105,8 +105,8 @@ def add_to_path(filepath): match = re.match(pattern, filepath) if match: path = f'../build/distributions/{match.group(1)}/agentbeat' - log('--- AGENTBEAT_PATH: ' + path) - os.environ['AGENTBEAT_PATH'] = path + log("--- PATH: " + str(path)) + os.environ['AGENTBEAT_PATH'] = str(path) else: log_err("No agentbeat executable found") exit(1) diff --git a/.buildkite/x-pack/pipeline.xpack.agentbeat.yml b/.buildkite/x-pack/pipeline.xpack.agentbeat.yml index 71607aefb250..b6225577b846 100644 --- a/.buildkite/x-pack/pipeline.xpack.agentbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.agentbeat.yml @@ -12,7 +12,7 @@ env: IMAGE_BEATS_WITH_HOOKS_LATEST: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest" - AGENTBEAT_SPEC: "../agentbeat.spec.yml" + AGENTBEAT_SPEC: "./agentbeat.spec.yml" steps: # - group: "Check/Update" @@ -112,6 +112,8 @@ steps: command: | set -euo pipefail ./.buildkite/scripts/agentbeat/prepare_env.py + cd x-pack/agentbeat + mage -v testWithSpec agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -123,8 +125,10 @@ steps: key: "agentbeat-it-windows" # depends_on: # - agentbeat-package-linux - commands: - - ./.buildkite/scripts/agentbeat/prepare_env.py + command: | + ./.buildkite/scripts/agentbeat/prepare_env.py + cd x-pack/agentbeat + mage -v testWithSpec agents: provider: "gcp" image: "${IMAGE_WIN_2022}" diff --git a/dev-tools/mage/parse_spec.go b/dev-tools/mage/spec.go similarity index 75% rename from dev-tools/mage/parse_spec.go rename to dev-tools/mage/spec.go index 5dc20199288d..4afca22d5253 100644 --- a/dev-tools/mage/parse_spec.go +++ b/dev-tools/mage/spec.go @@ -21,6 +21,7 @@ import ( "gopkg.in/yaml.v2" "log" "os" + "strings" ) type spec struct { @@ -34,13 +35,17 @@ type input struct { Command command } +func (i *input) GetCommand() string { + return strings.Join(i.Command.Args, " ") +} + type command struct { Name string Args []string } -// ParseSpec parses agent.beat.spec.yml and generates test command -func ParseSpec() { +// SpecCommands parses agent.beat.spec.yml and collects commands for tests +func SpecCommands() []string { specPath := os.Getenv("AGENTBEAT_SPEC") if specPath == "" { log.Fatal("AGENTBEAT_SPEC is not defined") @@ -51,20 +56,26 @@ func ParseSpec() { log.Fatal("PLATFORM is not defined") } - spec, err := parseToObj() - if err != nil { - log.Fatalf("Error parsing agentbeat.spec.yml: %v", err) - } + spec, _ := parseToObj(specPath) - inputList := filter(spec.Inputs, func(input input) bool { + filteredInputs := filter(spec.Inputs, func(input input) bool { return contains(input.Platforms, platform) }) - log.Print(inputList) + commands := make(map[string]interface{}) + for _, i := range filteredInputs { + commands[i.GetCommand()] = nil + } + keys := make([]string, 0, len(commands)) + for k := range commands { + keys = append(keys, k) + } + + return keys } -func parseToObj() (spec, error) { - specFile, err := os.ReadFile("../agentbeat.spec.yml") +func parseToObj(path string) (spec, error) { + specFile, err := os.ReadFile(path) if err != nil { log.Fatalf("Error opening agentbeat.spec.yml: %v", err) return spec{}, err diff --git a/x-pack/agentbeat/magefile.go b/x-pack/agentbeat/magefile.go index 097695552a7c..c52e122efdb0 100644 --- a/x-pack/agentbeat/magefile.go +++ b/x-pack/agentbeat/magefile.go @@ -10,6 +10,7 @@ import ( "context" "fmt" "os" + "os/exec" "path/filepath" "time" @@ -142,7 +143,7 @@ func Package() error { return nil } -// TestPackages tests the generated packages (i.e. file modes, owners, groups). +// TestPackages tests the generated packages (i.agentbeatCmd. file modes, owners, groups). func TestPackages() error { return devtools.TestPackages() } @@ -214,6 +215,65 @@ func PythonIntegTest(ctx context.Context) error { return devtools.PythonIntegTestFromHost(devtools.DefaultPythonTestIntegrationFromHostArgs()) } -func TestWithSpecs(ctx context.Context) { - mg.Deps(devtools.ParseSpec) +// TestWithSpec executes unique commands from agentbeat.spec.yml and validates that app haven't exited with non-zero +func TestWithSpec(ctx context.Context) { + var commands = devtools.SpecCommands() + + agentbeatPath := os.Getenv("AGENTBEAT_PATH") + fmt.Printf("--- AGENTBEAT_PATH: %s", agentbeatPath) + + cmdResults := make(map[string]bool) + + for _, command := range commands { + cmdResults[command] = agentbeatCmd(agentbeatPath, command) + } + + hasFailures := false + for cmd, res := range cmdResults { + if res { + fmt.Printf("Command [%s] succeeded", cmd) + } else { + fmt.Printf("Command [%s] failed", cmd) + hasFailures = true + } + } + + if hasFailures { + fmt.Printf("Some inputs failed. Exiting with error") + os.Exit(1) + } +} + +func agentbeatCmd(agentbeatPath string, command string) bool { + cmd := exec.Command(agentbeatPath, command) + fmt.Printf("Running command: %v", cmd) + + if err := cmd.Start(); err != nil { + _ = fmt.Errorf("failed to start command: %v", err) + } + + defer func() { + if err := cmd.Process.Kill(); err != nil { + _ = fmt.Errorf("failed to kill process: %v", err) + } else { + _ = fmt.Errorf("command process killed") + } + }() + + done := make(chan error, 1) + go func() { + done <- cmd.Wait() + }() + timeout := 2 * time.Second + deadline := time.After(timeout) + + select { + case err := <-done: + _ = fmt.Errorf("command exited before %s: %v", timeout.String(), err) + return false + + case <-deadline: + _ = fmt.Errorf("%s", cmd.Stdout) + return true + } } From 41110cf58127b7357ebfadc93c7311c935748bf0 Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka <olga_naidjonoka@epam.com> Date: Wed, 9 Oct 2024 11:00:12 +0300 Subject: [PATCH 09/17] updated scripts --- .buildkite/hooks/pre-command | 5 +++ .../{prepare_env.py => setup_agentbeat.py} | 30 +++------------- .../x-pack/pipeline.xpack.agentbeat.yml | 10 +++--- dev-tools/mage/spec.go | 12 +------ x-pack/agentbeat/magefile.go | 35 ++++++++++++------- 5 files changed, 38 insertions(+), 54 deletions(-) rename .buildkite/scripts/agentbeat/{prepare_env.py => setup_agentbeat.py} (76%) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 5718d97879e1..8b84984a2823 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -35,6 +35,11 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || \ export BUILDKITE_ANALYTICS_TOKEN fi +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-agentbeat" ]]; then + AGENTBEAT_PATH=$(.buildkite/scripts/agentbeat/setup_agentbeat.py) + export AGENTBEAT_PATH +fi + CPU_ARCH=$(uname -m) PLATFORM_TYPE=$(uname) diff --git a/.buildkite/scripts/agentbeat/prepare_env.py b/.buildkite/scripts/agentbeat/setup_agentbeat.py similarity index 76% rename from .buildkite/scripts/agentbeat/prepare_env.py rename to .buildkite/scripts/agentbeat/setup_agentbeat.py index d17c3cdd0e10..62e8c3bbd2cd 100755 --- a/.buildkite/scripts/agentbeat/prepare_env.py +++ b/.buildkite/scripts/agentbeat/setup_agentbeat.py @@ -4,7 +4,6 @@ import subprocess import sys import tarfile -import os import re PATH = 'x-pack/agentbeat/build/distributions' @@ -46,7 +45,6 @@ def get_artifact_pattern() -> str: def download_agentbeat(pattern, path) -> str: - log('--- Downloading agentbeat') try: subprocess.run( ['buildkite-agent', 'artifact', 'download', pattern, '.', @@ -70,20 +68,18 @@ def get_filename(path) -> str: def extract_agentbeat(filename): - log('~~~ Extracting agentbeat') filepath = PATH + '/' + filename if filepath.endswith('.zip'): unzip_agentbeat(filepath) else: untar_agentbeat(filepath) - log('Successfully extracted agentbeat') def unzip_agentbeat(filepath): try: subprocess.run( - ['unzip', filepath], + ['unzip', '-qq', filepath], check=True, stdout=sys.stdout, stderr=sys.stderr, text=True) except subprocess.CalledProcessError as e: log_err(e) @@ -93,40 +89,24 @@ def unzip_agentbeat(filepath): def untar_agentbeat(filepath): try: with tarfile.open(filepath, 'r:gz') as tar: - tar.list() tar.extractall() except Exception as e: log_err(e) exit(1) -def add_to_path(filepath): +def get_path_to_executable(filepath) -> str: pattern = r'(.*)(?=\.zip|.tar\.gz)' match = re.match(pattern, filepath) if match: - path = f'../build/distributions/{match.group(1)}/agentbeat' - log("--- PATH: " + str(path)) - os.environ['AGENTBEAT_PATH'] = str(path) + path = f'../../{match.group(1)}/agentbeat' + return path else: log_err("No agentbeat executable found") exit(1) -def install_synthetics(): - log('--- Installing @elastic/synthetics') - - try: - subprocess.run( - ['npm', 'install', '-g', '@elastic/synthetics'], - check=True - ) - except subprocess.CalledProcessError: - log_err('Failed to install @elastic/synthetics') - exit(1) - - artifact_pattern = get_artifact_pattern() archive = download_agentbeat(artifact_pattern, PATH) extract_agentbeat(archive) -add_to_path(archive) -install_synthetics() +log(get_path_to_executable(archive)) diff --git a/.buildkite/x-pack/pipeline.xpack.agentbeat.yml b/.buildkite/x-pack/pipeline.xpack.agentbeat.yml index b6225577b846..57618c671e40 100644 --- a/.buildkite/x-pack/pipeline.xpack.agentbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.agentbeat.yml @@ -110,9 +110,8 @@ steps: # depends_on: # - agentbeat-package-linux command: | - set -euo pipefail - ./.buildkite/scripts/agentbeat/prepare_env.py cd x-pack/agentbeat + ls -la ../../agentbeat-9.0.0-SNAPSHOT-linux-x86_64 mage -v testWithSpec agents: provider: "gcp" @@ -123,10 +122,11 @@ steps: - label: ":linux: Agentbeat/Integration tests Windows" key: "agentbeat-it-windows" + env: + PLATFORM: "windows/amd64" # depends_on: # - agentbeat-package-linux command: | - ./.buildkite/scripts/agentbeat/prepare_env.py cd x-pack/agentbeat mage -v testWithSpec agents: @@ -142,7 +142,7 @@ steps: # - agentbeat-package-linux # command: | # set -euo pipefail -# ./.buildkite/scripts/agentbeat/prepare_env.py +# ./.buildkite/scripts/agentbeat/setup_agentbeat.py # agents: # provider: "orka" # imagePrefix: "${IMAGE_MACOS_X86_64}" @@ -153,7 +153,7 @@ steps: # - agentbeat-package-linux # command: | # set -euo pipefail -# ./.buildkite/scripts/agentbeat/prepare_env.py +# ./.buildkite/scripts/agentbeat/setup_agentbeat.py # agents: # provider: "orka" # imagePrefix: "${IMAGE_MACOS_ARM}" diff --git a/dev-tools/mage/spec.go b/dev-tools/mage/spec.go index 4afca22d5253..af0527fd6196 100644 --- a/dev-tools/mage/spec.go +++ b/dev-tools/mage/spec.go @@ -45,17 +45,7 @@ type command struct { } // SpecCommands parses agent.beat.spec.yml and collects commands for tests -func SpecCommands() []string { - specPath := os.Getenv("AGENTBEAT_SPEC") - if specPath == "" { - log.Fatal("AGENTBEAT_SPEC is not defined") - } - - platform := os.Getenv("PLATFORM") - if platform == "" { - log.Fatal("PLATFORM is not defined") - } - +func SpecCommands(specPath string, platform string) []string { spec, _ := parseToObj(specPath) filteredInputs := filter(spec.Inputs, func(input input) bool { diff --git a/x-pack/agentbeat/magefile.go b/x-pack/agentbeat/magefile.go index c52e122efdb0..0b29c6c5d747 100644 --- a/x-pack/agentbeat/magefile.go +++ b/x-pack/agentbeat/magefile.go @@ -9,6 +9,7 @@ package main import ( "context" "fmt" + "log" "os" "os/exec" "path/filepath" @@ -217,10 +218,19 @@ func PythonIntegTest(ctx context.Context) error { // TestWithSpec executes unique commands from agentbeat.spec.yml and validates that app haven't exited with non-zero func TestWithSpec(ctx context.Context) { - var commands = devtools.SpecCommands() + specPath := os.Getenv("AGENTBEAT_SPEC") + if specPath == "" { + log.Fatal("AGENTBEAT_SPEC is not defined\n") + } + + platform := os.Getenv("PLATFORM") + if platform == "" { + log.Fatal("PLATFORM is not defined\n") + } + + var commands = devtools.SpecCommands(specPath, platform) agentbeatPath := os.Getenv("AGENTBEAT_PATH") - fmt.Printf("--- AGENTBEAT_PATH: %s", agentbeatPath) cmdResults := make(map[string]bool) @@ -230,33 +240,32 @@ func TestWithSpec(ctx context.Context) { hasFailures := false for cmd, res := range cmdResults { - if res { - fmt.Printf("Command [%s] succeeded", cmd) - } else { - fmt.Printf("Command [%s] failed", cmd) + if !res { + fmt.Printf("~~~ Failed: [%s]\n", cmd) + fmt.Print(res) hasFailures = true } } if hasFailures { - fmt.Printf("Some inputs failed. Exiting with error") + fmt.Printf("Some inputs failed. Exiting with error\n") os.Exit(1) } } func agentbeatCmd(agentbeatPath string, command string) bool { cmd := exec.Command(agentbeatPath, command) - fmt.Printf("Running command: %v", cmd) + fmt.Printf("Running command: %v\n", cmd) if err := cmd.Start(); err != nil { - _ = fmt.Errorf("failed to start command: %v", err) + fmt.Printf("failed to start command: %v\n", err) } defer func() { if err := cmd.Process.Kill(); err != nil { - _ = fmt.Errorf("failed to kill process: %v", err) + fmt.Printf("failed to kill process: %v\n", err) } else { - _ = fmt.Errorf("command process killed") + fmt.Print("command process killed\n") } }() @@ -269,11 +278,11 @@ func agentbeatCmd(agentbeatPath string, command string) bool { select { case err := <-done: - _ = fmt.Errorf("command exited before %s: %v", timeout.String(), err) + fmt.Printf("command exited before %s: %v\n", timeout.String(), err) return false case <-deadline: - _ = fmt.Errorf("%s", cmd.Stdout) + fmt.Printf("%s\n", cmd.Stdout) return true } } From a4e129cb35d75bc171a84ea72650d79df4fe3282 Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka <olga_naidjonoka@epam.com> Date: Thu, 10 Oct 2024 11:00:03 +0300 Subject: [PATCH 10/17] debug linux --- .buildkite/hooks/pre-command | 10 +++---- .../x-pack/pipeline.xpack.agentbeat.yml | 16 ++++++++---- x-pack/agentbeat/magefile.go | 26 ++++++++++++++----- 3 files changed, 35 insertions(+), 17 deletions(-) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 8b84984a2823..a6e5cd8750ef 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -18,6 +18,11 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" && "$BUILDKITE_STEP export PRIVATE_CI_GCS_CREDENTIALS_SECRET fi +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-agentbeat" ]]; then + AGENTBEAT_PATH=$(.buildkite/scripts/agentbeat/setup_agentbeat.py) + export AGENTBEAT_PATH +fi + if [[ "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || \ "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || \ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || \ @@ -35,11 +40,6 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || \ export BUILDKITE_ANALYTICS_TOKEN fi -if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-agentbeat" ]]; then - AGENTBEAT_PATH=$(.buildkite/scripts/agentbeat/setup_agentbeat.py) - export AGENTBEAT_PATH -fi - CPU_ARCH=$(uname -m) PLATFORM_TYPE=$(uname) diff --git a/.buildkite/x-pack/pipeline.xpack.agentbeat.yml b/.buildkite/x-pack/pipeline.xpack.agentbeat.yml index 57618c671e40..1b4f68992631 100644 --- a/.buildkite/x-pack/pipeline.xpack.agentbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.agentbeat.yml @@ -89,6 +89,12 @@ steps: # - x-pack/agentbeat/build/distributions/**/* # - "x-pack/agentbeat/build/*.xml" # - "x-pack/agentbeat/build/*.json" +# plugins: +# - test-collector#v1.10.2: +# files: "x-pack/agentbeat/build/TEST-*.xml" +# format: "junit" +# branches: "main" +# debug: true # retry: # automatic: # - limit: 1 @@ -103,7 +109,7 @@ steps: # - github_commit_status: # context: "agentbeat: Integration tests" - - label: ":linux: Agentbeat/Integration tests Linux" + - label: ":linux: x-pack/agentbeat: Ubuntu x86_64 Spec tests" key: "agentbeat-it-linux" env: PLATFORM: "linux/amd64" @@ -111,8 +117,8 @@ steps: # - agentbeat-package-linux command: | cd x-pack/agentbeat - ls -la ../../agentbeat-9.0.0-SNAPSHOT-linux-x86_64 - mage -v testWithSpec + ../../agentbeat-9.0.0-SNAPSHOT-linux-x86_64/agentbeat filebeat -E setup.ilm.enabled=false -E setup.template.enabled=false -E management.enabled=true -E management.restart_on_output_change=true -E logging.level=info -E logging.to_stderr=true -E gc_percent=${FILEBEAT_GOGC:100} -E filebeat.config.modules.enabled=false -E logging.event_data.to_stderr=true -E logging.event_data.to_files=false + # mage -v testWithSpec agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -120,14 +126,14 @@ steps: disk_size: 100 disk_type: "pd-ssd" - - label: ":linux: Agentbeat/Integration tests Windows" + - label: ":windows: x-pack/agentbeat: Windows x86_64 Spec tests" key: "agentbeat-it-windows" env: PLATFORM: "windows/amd64" # depends_on: # - agentbeat-package-linux command: | - cd x-pack/agentbeat + Set-Location -Path x-pack/agentbeat mage -v testWithSpec agents: provider: "gcp" diff --git a/x-pack/agentbeat/magefile.go b/x-pack/agentbeat/magefile.go index 0b29c6c5d747..429c4a2049fd 100644 --- a/x-pack/agentbeat/magefile.go +++ b/x-pack/agentbeat/magefile.go @@ -7,6 +7,7 @@ package main import ( + "bufio" "context" "fmt" "log" @@ -144,7 +145,7 @@ func Package() error { return nil } -// TestPackages tests the generated packages (i.agentbeatCmd. file modes, owners, groups). +// TestPackages tests the generated packages (i.runCmd. file modes, owners, groups). func TestPackages() error { return devtools.TestPackages() } @@ -235,14 +236,15 @@ func TestWithSpec(ctx context.Context) { cmdResults := make(map[string]bool) for _, command := range commands { - cmdResults[command] = agentbeatCmd(agentbeatPath, command) + cmdResults[command] = runCmd(agentbeatPath, command) } hasFailures := false for cmd, res := range cmdResults { - if !res { - fmt.Printf("~~~ Failed: [%s]\n", cmd) - fmt.Print(res) + if res { + fmt.Printf("--- :large_green_circle: Succeeded: [%s.10s...]\n", cmd) + } else { + fmt.Printf("--- :bangbang: Failed: [%s.10s...]\n", cmd) hasFailures = true } } @@ -253,9 +255,14 @@ func TestWithSpec(ctx context.Context) { } } -func agentbeatCmd(agentbeatPath string, command string) bool { +func runCmd(agentbeatPath string, command string) bool { cmd := exec.Command(agentbeatPath, command) - fmt.Printf("Running command: %v\n", cmd) + fmt.Printf("Executing: %s\n", cmd.String()) + + stdout, err := cmd.StdoutPipe() + if err != nil { + fmt.Printf("Error creating stdout pipe: %v\n", err) + } if err := cmd.Start(); err != nil { fmt.Printf("failed to start command: %v\n", err) @@ -279,6 +286,11 @@ func agentbeatCmd(agentbeatPath string, command string) bool { select { case err := <-done: fmt.Printf("command exited before %s: %v\n", timeout.String(), err) + fmt.Println("printing command stdout") + scanner := bufio.NewScanner(stdout) + for scanner.Scan() { + fmt.Println(scanner.Text()) + } return false case <-deadline: From 380f54c63d2d725826487a7fcd443b91b5d2ca44 Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka <olga_naidjonoka@epam.com> Date: Mon, 14 Oct 2024 11:00:48 +0300 Subject: [PATCH 11/17] updated std's outputs --- .../x-pack/pipeline.xpack.agentbeat.yml | 3 +-- x-pack/agentbeat/magefile.go | 20 +++++++------------ 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/.buildkite/x-pack/pipeline.xpack.agentbeat.yml b/.buildkite/x-pack/pipeline.xpack.agentbeat.yml index 1b4f68992631..95c091fe0530 100644 --- a/.buildkite/x-pack/pipeline.xpack.agentbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.agentbeat.yml @@ -117,8 +117,7 @@ steps: # - agentbeat-package-linux command: | cd x-pack/agentbeat - ../../agentbeat-9.0.0-SNAPSHOT-linux-x86_64/agentbeat filebeat -E setup.ilm.enabled=false -E setup.template.enabled=false -E management.enabled=true -E management.restart_on_output_change=true -E logging.level=info -E logging.to_stderr=true -E gc_percent=${FILEBEAT_GOGC:100} -E filebeat.config.modules.enabled=false -E logging.event_data.to_stderr=true -E logging.event_data.to_files=false - # mage -v testWithSpec + mage -v testWithSpec agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" diff --git a/x-pack/agentbeat/magefile.go b/x-pack/agentbeat/magefile.go index 429c4a2049fd..d02b5f9c83a8 100644 --- a/x-pack/agentbeat/magefile.go +++ b/x-pack/agentbeat/magefile.go @@ -7,13 +7,13 @@ package main import ( - "bufio" "context" "fmt" "log" "os" "os/exec" "path/filepath" + "strings" "time" "github.com/magefile/mage/sh" @@ -236,7 +236,7 @@ func TestWithSpec(ctx context.Context) { cmdResults := make(map[string]bool) for _, command := range commands { - cmdResults[command] = runCmd(agentbeatPath, command) + cmdResults[command] = runCmd(agentbeatPath, strings.Split(command, " ")) } hasFailures := false @@ -255,14 +255,13 @@ func TestWithSpec(ctx context.Context) { } } -func runCmd(agentbeatPath string, command string) bool { - cmd := exec.Command(agentbeatPath, command) +func runCmd(agentbeatPath string, command []string) bool { + cmd := exec.Command(agentbeatPath, command...) fmt.Printf("Executing: %s\n", cmd.String()) - stdout, err := cmd.StdoutPipe() - if err != nil { - fmt.Printf("Error creating stdout pipe: %v\n", err) - } + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Stdin = os.Stdin if err := cmd.Start(); err != nil { fmt.Printf("failed to start command: %v\n", err) @@ -286,11 +285,6 @@ func runCmd(agentbeatPath string, command string) bool { select { case err := <-done: fmt.Printf("command exited before %s: %v\n", timeout.String(), err) - fmt.Println("printing command stdout") - scanner := bufio.NewScanner(stdout) - for scanner.Scan() { - fmt.Println(scanner.Text()) - } return false case <-deadline: From bc2cc099f2db651be57f0e4fc3e44685f11a7ec4 Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka <olga_naidjonoka@epam.com> Date: Mon, 14 Oct 2024 11:46:05 +0300 Subject: [PATCH 12/17] updated scripts && enbaled macos step --- .buildkite/hooks/pre-command | 5 +- .../scripts/agentbeat/setup_agentbeat.py | 2 +- .../x-pack/pipeline.xpack.agentbeat.yml | 110 +++++++++--------- 3 files changed, 60 insertions(+), 57 deletions(-) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index a6e5cd8750ef..a6755d9df3c1 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -18,7 +18,10 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" && "$BUILDKITE_STEP export PRIVATE_CI_GCS_CREDENTIALS_SECRET fi -if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-agentbeat" ]]; then +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-agentbeat" && "$BUILDKITE_STEP_KEY" == *"agentbeat-it"* ]]; then + if [[ "$BUILDKITE_STEP_KEY" == *"macos"* ]]; then + ulimit -Sn 10000 + fi AGENTBEAT_PATH=$(.buildkite/scripts/agentbeat/setup_agentbeat.py) export AGENTBEAT_PATH fi diff --git a/.buildkite/scripts/agentbeat/setup_agentbeat.py b/.buildkite/scripts/agentbeat/setup_agentbeat.py index 62e8c3bbd2cd..0c82c16021f5 100755 --- a/.buildkite/scripts/agentbeat/setup_agentbeat.py +++ b/.buildkite/scripts/agentbeat/setup_agentbeat.py @@ -48,7 +48,7 @@ def download_agentbeat(pattern, path) -> str: try: subprocess.run( ['buildkite-agent', 'artifact', 'download', pattern, '.', - '--build', '01924d2b-b061-45ae-a106-e885584ff26f', + # '--build', '01924d2b-b061-45ae-a106-e885584ff26f', '--step', 'agentbeat-package-linux'], check=True, stdout=sys.stdout, stderr=sys.stderr, text=True) except subprocess.CalledProcessError: diff --git a/.buildkite/x-pack/pipeline.xpack.agentbeat.yml b/.buildkite/x-pack/pipeline.xpack.agentbeat.yml index 95c091fe0530..4cf23fdffd0c 100644 --- a/.buildkite/x-pack/pipeline.xpack.agentbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.agentbeat.yml @@ -15,19 +15,19 @@ env: AGENTBEAT_SPEC: "./agentbeat.spec.yml" steps: -# - group: "Check/Update" -# key: "x-pack-agentbeat-check-update" -# -# steps: -# - label: "agentbeat: Run pre-commit" -# command: "pre-commit run --all-files" -# agents: -# image: "${IMAGE_BEATS_WITH_HOOKS_LATEST}" -# memory: "2Gi" -# useCustomGlobalHooks: true -# notify: -# - github_commit_status: -# context: "agentbeat: pre-commit" + - group: "Check/Update" + key: "x-pack-agentbeat-check-update" + + steps: + - label: "agentbeat: Run pre-commit" + command: "pre-commit run --all-files" + agents: + image: "${IMAGE_BEATS_WITH_HOOKS_LATEST}" + memory: "2Gi" + useCustomGlobalHooks: true + notify: + - github_commit_status: + context: "agentbeat: pre-commit" # - wait: ~ # # with PRs, we want to run mandatory tests only if check/update step succeed @@ -40,32 +40,32 @@ steps: key: "agentbeat-mandatory-tests" steps: -# - label: ":linux: Agentbeat packaging Linux" -# key: "agentbeat-package-linux" -# env: -# PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" -# SNAPSHOT: true -# command: | -# set -euo pipefail -# cd x-pack/agentbeat -# mage package -# artifact_paths: -# - x-pack/agentbeat/build/distributions/**/* -# - "x-pack/agentbeat/build/*.xml" -# - "x-pack/agentbeat/build/*.json" -# retry: -# automatic: -# - limit: 2 -# timeout_in_minutes: 60 -# agents: -# provider: "gcp" -# image: "${IMAGE_UBUNTU_X86_64}" -# machineType: "${GCP_HI_PERF_MACHINE_TYPE}" -# disk_size: 100 -# disk_type: "pd-ssd" -# notify: -# - github_commit_status: -# context: "agentbeat: Packaging" + - label: ":linux: Agentbeat packaging Linux" + key: "agentbeat-package-linux" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + SNAPSHOT: true + command: | + set -euo pipefail + cd x-pack/agentbeat + mage package + artifact_paths: + - x-pack/agentbeat/build/distributions/**/* + - "x-pack/agentbeat/build/*.xml" + - "x-pack/agentbeat/build/*.json" + retry: + automatic: + - limit: 2 + timeout_in_minutes: 60 + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + notify: + - github_commit_status: + context: "agentbeat: Packaging" # - label: ":linux: Agentbeat/Integration tests Linux" # key: "agentbeat-it-linux" @@ -113,8 +113,8 @@ steps: key: "agentbeat-it-linux" env: PLATFORM: "linux/amd64" -# depends_on: -# - agentbeat-package-linux + depends_on: + - agentbeat-package-linux command: | cd x-pack/agentbeat mage -v testWithSpec @@ -129,8 +129,8 @@ steps: key: "agentbeat-it-windows" env: PLATFORM: "windows/amd64" -# depends_on: -# - agentbeat-package-linux + depends_on: + - agentbeat-package-linux command: | Set-Location -Path x-pack/agentbeat mage -v testWithSpec @@ -140,18 +140,18 @@ steps: machine_type: "${GCP_WIN_MACHINE_TYPE}" disk_size: 200 disk_type: "pd-ssd" -# -# - label: ":linux: Agentbeat/Integration tests macOS" -# key: "agentbeat-it-macos" -# depends_on: -# - agentbeat-package-linux -# command: | -# set -euo pipefail -# ./.buildkite/scripts/agentbeat/setup_agentbeat.py -# agents: -# provider: "orka" -# imagePrefix: "${IMAGE_MACOS_X86_64}" -# + + - label: ":macos: x-pack/agentbeat: macOS x86_64 Spec tests" + key: "agentbeat-it-macos" + depends_on: + - agentbeat-package-linux + command: | + cd x-pack/agentbeat + mage -v testWithSpec + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_X86_64}" + # - label: ":linux: Agentbeat/Integration tests macOS arm64" # key: "agentbeat-it-macos-arm" # depends_on: From 557b3186557741fb9785fa69af16c25e6dc16e77 Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka <olga_naidjonoka@epam.com> Date: Mon, 14 Oct 2024 15:26:15 +0300 Subject: [PATCH 13/17] debug macos --- .../scripts/agentbeat/setup_agentbeat.py | 5 +- .../x-pack/pipeline.xpack.agentbeat.yml | 146 +++++++++--------- 2 files changed, 77 insertions(+), 74 deletions(-) diff --git a/.buildkite/scripts/agentbeat/setup_agentbeat.py b/.buildkite/scripts/agentbeat/setup_agentbeat.py index 0c82c16021f5..818e048b999d 100755 --- a/.buildkite/scripts/agentbeat/setup_agentbeat.py +++ b/.buildkite/scripts/agentbeat/setup_agentbeat.py @@ -48,7 +48,7 @@ def download_agentbeat(pattern, path) -> str: try: subprocess.run( ['buildkite-agent', 'artifact', 'download', pattern, '.', - # '--build', '01924d2b-b061-45ae-a106-e885584ff26f', + '--build', '01924d2b-b061-45ae-a106-e885584ff26f', '--step', 'agentbeat-package-linux'], check=True, stdout=sys.stdout, stderr=sys.stderr, text=True) except subprocess.CalledProcessError: @@ -58,6 +58,7 @@ def download_agentbeat(pattern, path) -> str: def get_filename(path) -> str: + print("--- Getting filename") try: out = subprocess.run( ['ls', '-p', path], @@ -108,5 +109,7 @@ def get_path_to_executable(filepath) -> str: artifact_pattern = get_artifact_pattern() archive = download_agentbeat(artifact_pattern, PATH) +print("--- Extracting") extract_agentbeat(archive) +print("--- Getting path to exec") log(get_path_to_executable(archive)) diff --git a/.buildkite/x-pack/pipeline.xpack.agentbeat.yml b/.buildkite/x-pack/pipeline.xpack.agentbeat.yml index 4cf23fdffd0c..8873ffe29b8e 100644 --- a/.buildkite/x-pack/pipeline.xpack.agentbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.agentbeat.yml @@ -29,43 +29,43 @@ steps: - github_commit_status: context: "agentbeat: pre-commit" -# - wait: ~ -# # with PRs, we want to run mandatory tests only if check/update step succeed -# # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests -# # this allows building DRA artifacts even if there is flakiness in check/update step -# if: build.env("BUILDKITE_PULL_REQUEST") != "false" -# depends_on: "x-pack-agentbeat-check-update" + - wait: ~ + # with PRs, we want to run mandatory tests only if check/update step succeed + # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests + # this allows building DRA artifacts even if there is flakiness in check/update step + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + depends_on: "x-pack-agentbeat-check-update" - group: "Agentbeat tests" key: "agentbeat-mandatory-tests" steps: - - label: ":linux: Agentbeat packaging Linux" - key: "agentbeat-package-linux" - env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - SNAPSHOT: true - command: | - set -euo pipefail - cd x-pack/agentbeat - mage package - artifact_paths: - - x-pack/agentbeat/build/distributions/**/* - - "x-pack/agentbeat/build/*.xml" - - "x-pack/agentbeat/build/*.json" - retry: - automatic: - - limit: 2 - timeout_in_minutes: 60 - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - notify: - - github_commit_status: - context: "agentbeat: Packaging" +# - label: ":linux: Agentbeat packaging Linux" +# key: "agentbeat-package-linux" +# env: +# PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" +# SNAPSHOT: true +# command: | +# set -euo pipefail +# cd x-pack/agentbeat +# mage package +# artifact_paths: +# - x-pack/agentbeat/build/distributions/**/* +# - "x-pack/agentbeat/build/*.xml" +# - "x-pack/agentbeat/build/*.json" +# retry: +# automatic: +# - limit: 2 +# timeout_in_minutes: 60 +# agents: +# provider: "gcp" +# image: "${IMAGE_UBUNTU_X86_64}" +# machineType: "${GCP_HI_PERF_MACHINE_TYPE}" +# disk_size: 100 +# disk_type: "pd-ssd" +# notify: +# - github_commit_status: +# context: "agentbeat: Packaging" # - label: ":linux: Agentbeat/Integration tests Linux" # key: "agentbeat-it-linux" @@ -109,42 +109,42 @@ steps: # - github_commit_status: # context: "agentbeat: Integration tests" - - label: ":linux: x-pack/agentbeat: Ubuntu x86_64 Spec tests" - key: "agentbeat-it-linux" - env: - PLATFORM: "linux/amd64" - depends_on: - - agentbeat-package-linux - command: | - cd x-pack/agentbeat - mage -v testWithSpec - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - - - label: ":windows: x-pack/agentbeat: Windows x86_64 Spec tests" - key: "agentbeat-it-windows" - env: - PLATFORM: "windows/amd64" - depends_on: - - agentbeat-package-linux - command: | - Set-Location -Path x-pack/agentbeat - mage -v testWithSpec - agents: - provider: "gcp" - image: "${IMAGE_WIN_2022}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 200 - disk_type: "pd-ssd" +# - label: ":linux: x-pack/agentbeat: Ubuntu x86_64 Spec tests" +# key: "agentbeat-it-linux" +# env: +# PLATFORM: "linux/amd64" +# depends_on: +# - agentbeat-package-linux +# command: | +# cd x-pack/agentbeat +# mage -v testWithSpec +# agents: +# provider: "gcp" +# image: "${IMAGE_UBUNTU_X86_64}" +# machineType: "${GCP_HI_PERF_MACHINE_TYPE}" +# disk_size: 100 +# disk_type: "pd-ssd" +# +# - label: ":windows: x-pack/agentbeat: Windows x86_64 Spec tests" +# key: "agentbeat-it-windows" +# env: +# PLATFORM: "windows/amd64" +# depends_on: +# - agentbeat-package-linux +# command: | +# Set-Location -Path x-pack/agentbeat +# mage -v testWithSpec +# agents: +# provider: "gcp" +# image: "${IMAGE_WIN_2022}" +# machine_type: "${GCP_WIN_MACHINE_TYPE}" +# disk_size: 200 +# disk_type: "pd-ssd" - label: ":macos: x-pack/agentbeat: macOS x86_64 Spec tests" key: "agentbeat-it-macos" - depends_on: - - agentbeat-package-linux +# depends_on: +# - agentbeat-package-linux command: | cd x-pack/agentbeat mage -v testWithSpec @@ -152,13 +152,13 @@ steps: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" -# - label: ":linux: Agentbeat/Integration tests macOS arm64" -# key: "agentbeat-it-macos-arm" + - label: ":linux: Agentbeat/Integration tests macOS arm64" + key: "agentbeat-it-macos-arm" # depends_on: # - agentbeat-package-linux -# command: | -# set -euo pipefail -# ./.buildkite/scripts/agentbeat/setup_agentbeat.py -# agents: -# provider: "orka" -# imagePrefix: "${IMAGE_MACOS_ARM}" + command: | + cd x-pack/agentbeat + mage -v testWithSpec + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_ARM}" From b1e9fa57ff06545db780b40efd0988ea5bb74a98 Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka <olga_naidjonoka@epam.com> Date: Mon, 14 Oct 2024 15:32:30 +0300 Subject: [PATCH 14/17] debug macos --- .buildkite/hooks/pre-command | 6 +- .../scripts/agentbeat/setup_agentbeat.py | 18 +- .../x-pack/pipeline.xpack.agentbeat.yml | 178 +++++++++--------- dev-tools/mage/spec.go | 1 + 4 files changed, 102 insertions(+), 101 deletions(-) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index a6755d9df3c1..f1f9bd540294 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -19,10 +19,8 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" && "$BUILDKITE_STEP fi if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-agentbeat" && "$BUILDKITE_STEP_KEY" == *"agentbeat-it"* ]]; then - if [[ "$BUILDKITE_STEP_KEY" == *"macos"* ]]; then - ulimit -Sn 10000 - fi - AGENTBEAT_PATH=$(.buildkite/scripts/agentbeat/setup_agentbeat.py) +# .buildkite/scripts/agentbeat/setup_agentbeat.py + AGENTBEAT_PATH=$(.buildkite/scripts/agentbeat/setup_agentbeat.py | tail -n 1) export AGENTBEAT_PATH fi diff --git a/.buildkite/scripts/agentbeat/setup_agentbeat.py b/.buildkite/scripts/agentbeat/setup_agentbeat.py index 818e048b999d..b5f148a18ccd 100755 --- a/.buildkite/scripts/agentbeat/setup_agentbeat.py +++ b/.buildkite/scripts/agentbeat/setup_agentbeat.py @@ -1,10 +1,9 @@ #!/usr/bin/env python3 - import platform +import re import subprocess import sys import tarfile -import re PATH = 'x-pack/agentbeat/build/distributions' @@ -48,9 +47,10 @@ def download_agentbeat(pattern, path) -> str: try: subprocess.run( ['buildkite-agent', 'artifact', 'download', pattern, '.', - '--build', '01924d2b-b061-45ae-a106-e885584ff26f', + # '--build', '01928f55-8452-41c6-89ba-fe21f019f53c', '--step', 'agentbeat-package-linux'], check=True, stdout=sys.stdout, stderr=sys.stderr, text=True) + except subprocess.CalledProcessError: exit(1) @@ -58,11 +58,11 @@ def download_agentbeat(pattern, path) -> str: def get_filename(path) -> str: - print("--- Getting filename") try: out = subprocess.run( ['ls', '-p', path], check=True, capture_output=True, text=True) + print("--- ls -p: " + out.stdout) return out.stdout.strip() except subprocess.CalledProcessError: exit(1) @@ -89,9 +89,10 @@ def unzip_agentbeat(filepath): def untar_agentbeat(filepath): try: - with tarfile.open(filepath, 'r:gz') as tar: - tar.extractall() - except Exception as e: + subprocess.run( + ['tar', '-xvf', filepath], + check=True, stdout=sys.stdout, stderr=sys.stderr, text=True) + except subprocess.CalledProcessError as e: log_err(e) exit(1) @@ -106,10 +107,7 @@ def get_path_to_executable(filepath) -> str: log_err("No agentbeat executable found") exit(1) - artifact_pattern = get_artifact_pattern() archive = download_agentbeat(artifact_pattern, PATH) -print("--- Extracting") extract_agentbeat(archive) -print("--- Getting path to exec") log(get_path_to_executable(archive)) diff --git a/.buildkite/x-pack/pipeline.xpack.agentbeat.yml b/.buildkite/x-pack/pipeline.xpack.agentbeat.yml index 8873ffe29b8e..da66c7186589 100644 --- a/.buildkite/x-pack/pipeline.xpack.agentbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.agentbeat.yml @@ -15,57 +15,57 @@ env: AGENTBEAT_SPEC: "./agentbeat.spec.yml" steps: - - group: "Check/Update" - key: "x-pack-agentbeat-check-update" - - steps: - - label: "agentbeat: Run pre-commit" - command: "pre-commit run --all-files" - agents: - image: "${IMAGE_BEATS_WITH_HOOKS_LATEST}" - memory: "2Gi" - useCustomGlobalHooks: true - notify: - - github_commit_status: - context: "agentbeat: pre-commit" - - - wait: ~ - # with PRs, we want to run mandatory tests only if check/update step succeed - # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests - # this allows building DRA artifacts even if there is flakiness in check/update step - if: build.env("BUILDKITE_PULL_REQUEST") != "false" - depends_on: "x-pack-agentbeat-check-update" +# - group: "Check/Update" +# key: "x-pack-agentbeat-check-update" +# +# steps: +# - label: "agentbeat: Run pre-commit" +# command: "pre-commit run --all-files" +# agents: +# image: "${IMAGE_BEATS_WITH_HOOKS_LATEST}" +# memory: "2Gi" +# useCustomGlobalHooks: true +# notify: +# - github_commit_status: +# context: "agentbeat: pre-commit" +# +# - wait: ~ +# # with PRs, we want to run mandatory tests only if check/update step succeed +# # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests +# # this allows building DRA artifacts even if there is flakiness in check/update step +# if: build.env("BUILDKITE_PULL_REQUEST") != "false" +# depends_on: "x-pack-agentbeat-check-update" - group: "Agentbeat tests" key: "agentbeat-mandatory-tests" steps: -# - label: ":linux: Agentbeat packaging Linux" -# key: "agentbeat-package-linux" -# env: -# PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" -# SNAPSHOT: true -# command: | -# set -euo pipefail -# cd x-pack/agentbeat -# mage package -# artifact_paths: -# - x-pack/agentbeat/build/distributions/**/* -# - "x-pack/agentbeat/build/*.xml" -# - "x-pack/agentbeat/build/*.json" -# retry: -# automatic: -# - limit: 2 -# timeout_in_minutes: 60 -# agents: -# provider: "gcp" -# image: "${IMAGE_UBUNTU_X86_64}" -# machineType: "${GCP_HI_PERF_MACHINE_TYPE}" -# disk_size: 100 -# disk_type: "pd-ssd" -# notify: -# - github_commit_status: -# context: "agentbeat: Packaging" + - label: ":linux: Agentbeat packaging Linux" + key: "agentbeat-package-linux" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + SNAPSHOT: true + command: | + set -euo pipefail + cd x-pack/agentbeat + mage package + artifact_paths: + - x-pack/agentbeat/build/distributions/**/* + - "x-pack/agentbeat/build/*.xml" + - "x-pack/agentbeat/build/*.json" + retry: + automatic: + - limit: 2 + timeout_in_minutes: 60 + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + notify: + - github_commit_status: + context: "agentbeat: Packaging" # - label: ":linux: Agentbeat/Integration tests Linux" # key: "agentbeat-it-linux" @@ -109,56 +109,60 @@ steps: # - github_commit_status: # context: "agentbeat: Integration tests" -# - label: ":linux: x-pack/agentbeat: Ubuntu x86_64 Spec tests" -# key: "agentbeat-it-linux" -# env: -# PLATFORM: "linux/amd64" -# depends_on: -# - agentbeat-package-linux -# command: | -# cd x-pack/agentbeat -# mage -v testWithSpec -# agents: -# provider: "gcp" -# image: "${IMAGE_UBUNTU_X86_64}" -# machineType: "${GCP_HI_PERF_MACHINE_TYPE}" -# disk_size: 100 -# disk_type: "pd-ssd" -# -# - label: ":windows: x-pack/agentbeat: Windows x86_64 Spec tests" -# key: "agentbeat-it-windows" -# env: -# PLATFORM: "windows/amd64" -# depends_on: -# - agentbeat-package-linux -# command: | -# Set-Location -Path x-pack/agentbeat -# mage -v testWithSpec -# agents: -# provider: "gcp" -# image: "${IMAGE_WIN_2022}" -# machine_type: "${GCP_WIN_MACHINE_TYPE}" -# disk_size: 200 -# disk_type: "pd-ssd" + - label: ":linux: x-pack/agentbeat: Ubuntu x86_64 Spec tests" + key: "agentbeat-it-linux" + env: + PLATFORM: "linux/amd64" + depends_on: + - agentbeat-package-linux + command: | + cd x-pack/agentbeat + mage -v testWithSpec + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + + - label: ":windows: x-pack/agentbeat: Windows x86_64 Spec tests" + key: "agentbeat-it-windows" + env: + PLATFORM: "windows/amd64" + depends_on: + - agentbeat-package-linux + command: | + Set-Location -Path x-pack/agentbeat + mage -v testWithSpec + agents: + provider: "gcp" + image: "${IMAGE_WIN_2022}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 200 + disk_type: "pd-ssd" - label: ":macos: x-pack/agentbeat: macOS x86_64 Spec tests" key: "agentbeat-it-macos" -# depends_on: -# - agentbeat-package-linux + env: + PLATFORM: "darwin/amd64" + depends_on: + - agentbeat-package-linux command: | + set -euo pipefail + source .buildkite/scripts/install_macos_tools.sh cd x-pack/agentbeat mage -v testWithSpec agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" - - label: ":linux: Agentbeat/Integration tests macOS arm64" - key: "agentbeat-it-macos-arm" +# - label: ":linux: Agentbeat/Integration tests macOS arm64" +# key: "agentbeat-it-macos-arm" # depends_on: # - agentbeat-package-linux - command: | - cd x-pack/agentbeat - mage -v testWithSpec - agents: - provider: "orka" - imagePrefix: "${IMAGE_MACOS_ARM}" +# command: | +# cd x-pack/agentbeat +# mage -v testWithSpec +# agents: +# provider: "orka" +# imagePrefix: "${IMAGE_MACOS_ARM}" diff --git a/dev-tools/mage/spec.go b/dev-tools/mage/spec.go index af0527fd6196..03c733f1dd6d 100644 --- a/dev-tools/mage/spec.go +++ b/dev-tools/mage/spec.go @@ -19,6 +19,7 @@ package mage import ( "gopkg.in/yaml.v2" + "log" "os" "strings" From adf4233e061384e7fbfa0ad4e13d96657d84a4a3 Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka <olga_naidjonoka@epam.com> Date: Thu, 17 Oct 2024 12:33:04 +0300 Subject: [PATCH 15/17] added macos arm step --- .buildkite/hooks/pre-command | 1 - .../scripts/agentbeat/setup_agentbeat.py | 9 +- .../x-pack/pipeline.xpack.agentbeat.yml | 93 +++++++------------ 3 files changed, 43 insertions(+), 60 deletions(-) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index f1f9bd540294..cb5bb51eb803 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -19,7 +19,6 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" && "$BUILDKITE_STEP fi if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-agentbeat" && "$BUILDKITE_STEP_KEY" == *"agentbeat-it"* ]]; then -# .buildkite/scripts/agentbeat/setup_agentbeat.py AGENTBEAT_PATH=$(.buildkite/scripts/agentbeat/setup_agentbeat.py | tail -n 1) export AGENTBEAT_PATH fi diff --git a/.buildkite/scripts/agentbeat/setup_agentbeat.py b/.buildkite/scripts/agentbeat/setup_agentbeat.py index b5f148a18ccd..35139ec22787 100755 --- a/.buildkite/scripts/agentbeat/setup_agentbeat.py +++ b/.buildkite/scripts/agentbeat/setup_agentbeat.py @@ -12,10 +12,12 @@ def log(msg): sys.stdout.write(f'{msg}\n') sys.stdout.flush() + def log_err(msg): sys.stderr.write(f'{msg}\n') sys.stderr.flush() + def get_os() -> str: return platform.system().lower() @@ -26,7 +28,10 @@ def get_arch() -> str: if arch == 'amd64': return 'x86_64' else: - return arch + if get_os() == 'darwin': + return 'aarch64' + else: + return arch def get_artifact_extension(agent_os) -> str: @@ -40,6 +45,7 @@ def get_artifact_pattern() -> str: agent_os = get_os() agent_arch = get_arch() extension = get_artifact_extension(agent_os) + print('Artifact params: ' + agent_os + ' ' + agent_arch + ' ' + extension) return f'{PATH}/agentbeat-*-{agent_os}-{agent_arch}.{extension}' @@ -47,7 +53,6 @@ def download_agentbeat(pattern, path) -> str: try: subprocess.run( ['buildkite-agent', 'artifact', 'download', pattern, '.', - # '--build', '01928f55-8452-41c6-89ba-fe21f019f53c', '--step', 'agentbeat-package-linux'], check=True, stdout=sys.stdout, stderr=sys.stderr, text=True) diff --git a/.buildkite/x-pack/pipeline.xpack.agentbeat.yml b/.buildkite/x-pack/pipeline.xpack.agentbeat.yml index da66c7186589..a5375bd1e3d3 100644 --- a/.buildkite/x-pack/pipeline.xpack.agentbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.agentbeat.yml @@ -5,6 +5,9 @@ env: GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + AWS_ARM_INSTANCE_TYPE: "m6g.xlarge" + AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" + IMAGE_MACOS_ARM: "generic-13-ventura-arm" IMAGE_MACOS_X86_64: "generic-13-ventura-x64" @@ -67,50 +70,8 @@ steps: - github_commit_status: context: "agentbeat: Packaging" - # - label: ":linux: Agentbeat/Integration tests Linux" - # key: "agentbeat-it-linux" - # depends_on: - # - agentbeat-package-linux - # env: - # ASDF_NODEJS_VERSION: 18.17.1 - # PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - # SNAPSHOT: true - # command: | - # set -euo pipefail - # echo "~~~ Downloading artifacts" - # buildkite-agent artifact download x-pack/agentbeat/build/distributions/** . --step 'agentbeat-package-linux' - # ls -lah x-pack/agentbeat/build/distributions/ - # echo "~~~ Installing @elastic/synthetics with npm" - # npm install -g @elastic/synthetics - # echo "~~~ Running tests" - # cd x-pack/agentbeat - # mage goIntegTest - # artifact_paths: - # - x-pack/agentbeat/build/distributions/**/* - # - "x-pack/agentbeat/build/*.xml" - # - "x-pack/agentbeat/build/*.json" -# plugins: -# - test-collector#v1.10.2: -# files: "x-pack/agentbeat/build/TEST-*.xml" -# format: "junit" -# branches: "main" -# debug: true - # retry: - # automatic: - # - limit: 1 - # timeout_in_minutes: 60 - # agents: - # provider: "gcp" - # image: "${IMAGE_UBUNTU_X86_64}" - # machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - # disk_size: 100 - # disk_type: "pd-ssd" - # notify: - # - github_commit_status: - # context: "agentbeat: Integration tests" - - - label: ":linux: x-pack/agentbeat: Ubuntu x86_64 Spec tests" - key: "agentbeat-it-linux" + - label: ":ubuntu: x-pack/agentbeat: Ubuntu x86_64 Spec tests" + key: "agentbeat-it-linux-x86-64" env: PLATFORM: "linux/amd64" depends_on: @@ -125,6 +86,20 @@ steps: disk_size: 100 disk_type: "pd-ssd" + - label: ":ubuntu: x-pack/agentbeat: Ubuntu arm64 Spec tests" + key: "agentbeat-it-linux-arm64" + env: + PLATFORM: "linux/arm64" + depends_on: + - agentbeat-package-linux + command: | + cd x-pack/agentbeat + mage -v testWithSpec + agents: + provider: "aws" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + - label: ":windows: x-pack/agentbeat: Windows x86_64 Spec tests" key: "agentbeat-it-windows" env: @@ -142,11 +117,11 @@ steps: disk_type: "pd-ssd" - label: ":macos: x-pack/agentbeat: macOS x86_64 Spec tests" - key: "agentbeat-it-macos" - env: - PLATFORM: "darwin/amd64" + key: "agentbeat-it-macos-x86-64" depends_on: - agentbeat-package-linux + env: + PLATFORM: "darwin/amd64" command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh @@ -156,13 +131,17 @@ steps: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" -# - label: ":linux: Agentbeat/Integration tests macOS arm64" -# key: "agentbeat-it-macos-arm" -# depends_on: -# - agentbeat-package-linux -# command: | -# cd x-pack/agentbeat -# mage -v testWithSpec -# agents: -# provider: "orka" -# imagePrefix: "${IMAGE_MACOS_ARM}" + - label: ":macos: Agentbeat/Integration tests macOS arm64" + key: "agentbeat-it-macos-arm64" + depends_on: + - agentbeat-package-linux + env: + PLATFORM: "darwin/arm64" + command: | + set -euo pipefail + source .buildkite/scripts/install_macos_tools.sh + cd x-pack/agentbeat + mage -v testWithSpec + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_ARM}" From 4740fbe18decc35456f9c06e81092042926a68cc Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka <olga_naidjonoka@epam.com> Date: Thu, 17 Oct 2024 16:08:45 +0300 Subject: [PATCH 16/17] refactor --- .buildkite/hooks/pre-command | 4 +- .../scripts/agentbeat/setup_agentbeat.py | 69 +++++++++++-------- .../x-pack/pipeline.xpack.agentbeat.yml | 57 +++++++++------ 3 files changed, 79 insertions(+), 51 deletions(-) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index cb5bb51eb803..5cb0722edd84 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -19,7 +19,9 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" && "$BUILDKITE_STEP fi if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-agentbeat" && "$BUILDKITE_STEP_KEY" == *"agentbeat-it"* ]]; then - AGENTBEAT_PATH=$(.buildkite/scripts/agentbeat/setup_agentbeat.py | tail -n 1) + out=$(.buildkite/scripts/agentbeat/setup_agentbeat.py) + echo "$out" + AGENTBEAT_PATH=$(echo "$out" | tail -n 1) export AGENTBEAT_PATH fi diff --git a/.buildkite/scripts/agentbeat/setup_agentbeat.py b/.buildkite/scripts/agentbeat/setup_agentbeat.py index 35139ec22787..02634649e787 100755 --- a/.buildkite/scripts/agentbeat/setup_agentbeat.py +++ b/.buildkite/scripts/agentbeat/setup_agentbeat.py @@ -6,6 +6,26 @@ import tarfile PATH = 'x-pack/agentbeat/build/distributions' +PLATFORMS = { + 'windows': { + 'amd64': 'x86_64', + }, + 'linux': { + 'x86_64': 'x86_64', + 'aarch64': 'arm64', + }, + 'darwin': { + 'x86_64': 'x86_64', + 'arm64': 'aarch64', + } + } + + +class Archive: + def __init__(self, os, arch, ext): + self.os = os + self.arch = arch + self.ext = ext def log(msg): @@ -18,38 +38,29 @@ def log_err(msg): sys.stderr.flush() -def get_os() -> str: - return platform.system().lower() +def get_archive_params() -> Archive: + system = platform.system().lower() + machine = platform.machine().lower() + arch = PLATFORMS.get(system, {}).get(machine) + ext = get_artifact_extension(system) + return Archive(system, arch, ext) -def get_arch() -> str: - arch = platform.machine().lower() - if arch == 'amd64': - return 'x86_64' - else: - if get_os() == 'darwin': - return 'aarch64' - else: - return arch - - -def get_artifact_extension(agent_os) -> str: - if agent_os == 'windows': +def get_artifact_extension(system) -> str: + if system == 'windows': return 'zip' else: return 'tar.gz' -def get_artifact_pattern() -> str: - agent_os = get_os() - agent_arch = get_arch() - extension = get_artifact_extension(agent_os) - print('Artifact params: ' + agent_os + ' ' + agent_arch + ' ' + extension) - return f'{PATH}/agentbeat-*-{agent_os}-{agent_arch}.{extension}' +def get_artifact_pattern(archive_obj) -> str: + return f'{PATH}/agentbeat-*-{archive_obj.os}-{archive_obj.arch}.{archive_obj.ext}' -def download_agentbeat(pattern, path) -> str: +def download_agentbeat(archive_obj) -> str: + pattern = get_artifact_pattern(archive_obj) + log('--- Downloading Agentbeat artifact by pattern: ' + pattern) try: subprocess.run( ['buildkite-agent', 'artifact', 'download', pattern, '.', @@ -59,15 +70,14 @@ def download_agentbeat(pattern, path) -> str: except subprocess.CalledProcessError: exit(1) - return get_filename(path) + return get_full_filename() -def get_filename(path) -> str: +def get_full_filename() -> str: try: out = subprocess.run( - ['ls', '-p', path], + ['ls', '-p', PATH], check=True, capture_output=True, text=True) - print("--- ls -p: " + out.stdout) return out.stdout.strip() except subprocess.CalledProcessError: exit(1) @@ -75,6 +85,7 @@ def get_filename(path) -> str: def extract_agentbeat(filename): filepath = PATH + '/' + filename + log('Extracting Agentbeat artifact: ' + filepath) if filepath.endswith('.zip'): unzip_agentbeat(filepath) @@ -109,10 +120,10 @@ def get_path_to_executable(filepath) -> str: path = f'../../{match.group(1)}/agentbeat' return path else: - log_err("No agentbeat executable found") + log_err('No agentbeat executable found') exit(1) -artifact_pattern = get_artifact_pattern() -archive = download_agentbeat(artifact_pattern, PATH) +archive_params = get_archive_params() +archive = download_agentbeat(archive_params) extract_agentbeat(archive) log(get_path_to_executable(archive)) diff --git a/.buildkite/x-pack/pipeline.xpack.agentbeat.yml b/.buildkite/x-pack/pipeline.xpack.agentbeat.yml index a5375bd1e3d3..1687aa25d922 100644 --- a/.buildkite/x-pack/pipeline.xpack.agentbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.agentbeat.yml @@ -18,26 +18,26 @@ env: AGENTBEAT_SPEC: "./agentbeat.spec.yml" steps: -# - group: "Check/Update" -# key: "x-pack-agentbeat-check-update" -# -# steps: -# - label: "agentbeat: Run pre-commit" -# command: "pre-commit run --all-files" -# agents: -# image: "${IMAGE_BEATS_WITH_HOOKS_LATEST}" -# memory: "2Gi" -# useCustomGlobalHooks: true -# notify: -# - github_commit_status: -# context: "agentbeat: pre-commit" -# -# - wait: ~ -# # with PRs, we want to run mandatory tests only if check/update step succeed -# # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests -# # this allows building DRA artifacts even if there is flakiness in check/update step -# if: build.env("BUILDKITE_PULL_REQUEST") != "false" -# depends_on: "x-pack-agentbeat-check-update" + - group: "Check/Update" + key: "x-pack-agentbeat-check-update" + + steps: + - label: "agentbeat: Run pre-commit" + command: "pre-commit run --all-files" + agents: + image: "${IMAGE_BEATS_WITH_HOOKS_LATEST}" + memory: "2Gi" + useCustomGlobalHooks: true + notify: + - github_commit_status: + context: "agentbeat: pre-commit" + + - wait: ~ + # with PRs, we want to run mandatory tests only if check/update step succeed + # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests + # this allows building DRA artifacts even if there is flakiness in check/update step + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + depends_on: "x-pack-agentbeat-check-update" - group: "Agentbeat tests" key: "agentbeat-mandatory-tests" @@ -85,6 +85,9 @@ steps: machineType: "${GCP_HI_PERF_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" + notify: + - github_commit_status: + context: "agentbeat: Ubuntu x86_64 Spec tests" - label: ":ubuntu: x-pack/agentbeat: Ubuntu arm64 Spec tests" key: "agentbeat-it-linux-arm64" @@ -99,6 +102,9 @@ steps: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" + notify: + - github_commit_status: + context: "agentbeat: Ubuntu arm64 Spec tests" - label: ":windows: x-pack/agentbeat: Windows x86_64 Spec tests" key: "agentbeat-it-windows" @@ -115,6 +121,9 @@ steps: machine_type: "${GCP_WIN_MACHINE_TYPE}" disk_size: 200 disk_type: "pd-ssd" + notify: + - github_commit_status: + context: "agentbeat: Windows x86_64 Spec tests" - label: ":macos: x-pack/agentbeat: macOS x86_64 Spec tests" key: "agentbeat-it-macos-x86-64" @@ -130,8 +139,11 @@ steps: agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" + notify: + - github_commit_status: + context: "agentbeat: macOS x86_64 Spec tests" - - label: ":macos: Agentbeat/Integration tests macOS arm64" + - label: ":macos: x-pack/agentbeat: macOS arm64 Spec tests" key: "agentbeat-it-macos-arm64" depends_on: - agentbeat-package-linux @@ -145,3 +157,6 @@ steps: agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_ARM}" + notify: + - github_commit_status: + context: "agentbeat: macOS arm64 Spec tests" From 22190d441297146cce3cf27c20930567e0dc93f1 Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka <olga_naidjonoka@epam.com> Date: Fri, 18 Oct 2024 14:58:40 +0300 Subject: [PATCH 17/17] restored testPackages annotation --- x-pack/agentbeat/magefile.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/agentbeat/magefile.go b/x-pack/agentbeat/magefile.go index d02b5f9c83a8..bd72a558ba39 100644 --- a/x-pack/agentbeat/magefile.go +++ b/x-pack/agentbeat/magefile.go @@ -145,7 +145,7 @@ func Package() error { return nil } -// TestPackages tests the generated packages (i.runCmd. file modes, owners, groups). +// TestPackages tests the generated packages (i.e. file modes, owners, groups). func TestPackages() error { return devtools.TestPackages() }