Agentbeats: added servrless tests step

.buildkite/hooks/pre-command

@@ -18,6 +18,13 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" && "$BUILDKITE_STEP
     export PRIVATE_CI_GCS_CREDENTIALS_SECRET
 fi
 
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-agentbeat" && "$BUILDKITE_STEP_KEY" == *"agentbeat-it"* ]]; then
+    out=$(.buildkite/scripts/agentbeat/setup_agentbeat.py)
+    echo "$out"
+    AGENT_BUILD_DIR=$(echo "$out" | tail -n 1)
+    export AGENT_BUILD_DIR
+fi
+
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || \
   "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || \
   "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || \

.buildkite/scripts/agentbeat/setup_agentbeat.py

@@ -0,0 +1,129 @@
+#!/usr/bin/env python3
+import platform
+import re
+import subprocess
+import sys
+import tarfile
+
+PATH = 'x-pack/agentbeat/build/distributions'
+PLATFORMS = {
+                'windows': {
+                    'amd64': 'x86_64',
+                },
+                'linux': {
+                    'x86_64': 'x86_64',
+                    'aarch64': 'arm64',
+                },
+                'darwin': {
+                    'x86_64': 'x86_64',
+                    'arm64': 'aarch64',
+                }
+            }
+
+
+class Archive:
+    def __init__(self, os, arch, ext):
+            self.os = os
+            self.arch = arch
+            self.ext = ext
+
+
+def log(msg):
+    sys.stdout.write(f'{msg}\n')
+    sys.stdout.flush()
+
+
+def log_err(msg):
+    sys.stderr.write(f'{msg}\n')
+    sys.stderr.flush()
+
+
+def get_archive_params() -> Archive:
+    system = platform.system().lower()
+    machine = platform.machine().lower()
+    arch = PLATFORMS.get(system, {}).get(machine)
+    ext = get_artifact_extension(system)
+
+    return Archive(system, arch, ext)
+
+
+def get_artifact_extension(system) -> str:
+    if system == 'windows':
+        return 'zip'
+    else:
+        return 'tar.gz'
+
+
+def get_artifact_pattern(archive_obj) -> str:
+    return f'{PATH}/agentbeat-*-{archive_obj.os}-{archive_obj.arch}.{archive_obj.ext}'
+
+
+def download_agentbeat(archive_obj) -> str:
+    pattern = get_artifact_pattern(archive_obj)
+    log('--- Downloading Agentbeat artifact by pattern: ' + pattern)
+    try:
+        subprocess.run(
+            ['buildkite-agent', 'artifact', 'download', pattern, '.',
+             '--step', 'agentbeat-package-linux'],
+            check=True, stdout=sys.stdout, stderr=sys.stderr, text=True)
+
+    except subprocess.CalledProcessError:
+        exit(1)
+
+    return get_full_filename()
+
+
+def get_full_filename() -> str:
+    try:
+        out = subprocess.run(
+            ['ls', '-p', PATH],
+            check=True, capture_output=True, text=True)
+        return out.stdout.strip()
+    except subprocess.CalledProcessError:
+        exit(1)
+
+
+def extract_agentbeat(filename):
+    filepath = PATH + '/' + filename
+    log('Extracting Agentbeat artifact: ' + filepath)
+
+    if filepath.endswith('.zip'):
+        unzip_agentbeat(filepath)
+    else:
+        untar_agentbeat(filepath)
+
+
+def unzip_agentbeat(filepath):
+    try:
+        subprocess.run(
+            ['unzip', '-qq', filepath],
+            check=True, stdout=sys.stdout, stderr=sys.stderr, text=True)
+    except subprocess.CalledProcessError as e:
+        log_err(e)
+        exit(1)
+
+
+def untar_agentbeat(filepath):
+    try:
+        subprocess.run(
+            ['tar', '-xvf', filepath],
+            check=True, stdout=sys.stdout, stderr=sys.stderr, text=True)
+    except subprocess.CalledProcessError as e:
+        log_err(e)
+        exit(1)
+
+
+def get_path_to_executable(filepath) -> str:
+    pattern = r'(.*)(?=\.zip|.tar\.gz)'
+    match = re.match(pattern, filepath)
+    if match:
+        path = f'../../{match.group(1)}/agentbeat'
+        return path
+    else:
+        log_err('No agentbeat executable found')
+        exit(1)
+
+archive_params = get_archive_params()
+archive = download_agentbeat(archive_params)
+extract_agentbeat(archive)
+log(get_path_to_executable(archive))

.buildkite/x-pack/pipeline.xpack.agentbeat.yml

@@ -6,26 +6,26 @@ env:
   IMAGE_BEATS_WITH_HOOKS_LATEST: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest"
 
 steps:
-  - group: "Check/Update"
-    key: "x-pack-agentbeat-check-update"
-
-    steps:
-      - label: "agentbeat: Run pre-commit"
-        command: "pre-commit run --all-files"
-        agents:
-          image: "${IMAGE_BEATS_WITH_HOOKS_LATEST}"
-          memory: "2Gi"
-          useCustomGlobalHooks: true
-        notify:
-          - github_commit_status:
-              context: "agentbeat: pre-commit"
-
-  - wait: ~
-    # with PRs, we want to run mandatory tests only if check/update step succeed
-    # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests
-    # this allows building DRA artifacts even if there is flakiness in check/update step
-    if: build.env("BUILDKITE_PULL_REQUEST") != "false"
-    depends_on: "x-pack-agentbeat-check-update"
+#  - group: "Check/Update"
+#    key: "x-pack-agentbeat-check-update"
+#
+#    steps:
+#      - label: "agentbeat: Run pre-commit"
+#        command: "pre-commit run --all-files"
+#        agents:
+#          image: "${IMAGE_BEATS_WITH_HOOKS_LATEST}"
+#          memory: "2Gi"
+#          useCustomGlobalHooks: true
+#        notify:
+#          - github_commit_status:
+#              context: "agentbeat: pre-commit"
+#
+#  - wait: ~
+#    # with PRs, we want to run mandatory tests only if check/update step succeed
+#    # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests
+#    # this allows building DRA artifacts even if there is flakiness in check/update step
+#    if: build.env("BUILDKITE_PULL_REQUEST") != "false"
+#    depends_on: "x-pack-agentbeat-check-update"
 
   - group: "Agentbeat tests"
     key: "agentbeat-mandatory-tests"
@@ -35,6 +35,7 @@ steps:
         key: "agentbeat-package-linux"
         env:
           PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
+          PACKAGES: tar.gz,zip
           SNAPSHOT: true
         command: |
           set -euo pipefail
@@ -58,28 +59,69 @@ steps:
           - github_commit_status:
               context: "agentbeat: Packaging"
 
-      - label: ":linux: Agentbeat/Integration tests Linux"
-        key: "agentbeat-it-linux"
+#      - label: ":linux: Agentbeat/Integration tests Linux"
+#        key: "agentbeat-it-linux"
+#        depends_on:
+#          - agentbeat-package-linux
+#        env:
+#          ASDF_NODEJS_VERSION: 18.17.1
+#          PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
+#          SNAPSHOT: true
+#        command: |
+#          set -euo pipefail
+#          echo "~~~ Downloading artifacts"
+#          buildkite-agent artifact download x-pack/agentbeat/build/distributions/** . --step 'agentbeat-package-linux'
+#          ls -lah x-pack/agentbeat/build/distributions/
+#          echo "~~~ Installing @elastic/synthetics with npm"
+#          npm install -g @elastic/synthetics
+#          echo "~~~ Running tests"
+#          cd x-pack/agentbeat
+#          mage goIntegTest
+#        artifact_paths:
+#          - x-pack/agentbeat/build/distributions/**/*
+#          - "x-pack/agentbeat/build/*.xml"
+#          - "x-pack/agentbeat/build/*.json"
+#        plugins:
+#          - test-collector#v1.10.2:
+#              files: "x-pack/agentbeat/build/TEST-*.xml"
+#              format: "junit"
+#              branches: "main"
+#              debug: true
+#        retry:
+#          automatic:
+#            - limit: 1
+#        timeout_in_minutes: 60
+#        agents:
+#          provider: "gcp"
+#          image: "${IMAGE_UBUNTU_X86_64}"
+#          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+#          disk_size: 100
+#          disk_type: "pd-ssd"
+#        notify:
+#          - github_commit_status:
+#              context: "agentbeat: Integration tests"
+
+  - group: "Agentbeat: Servelress Tests"
+    key: "agentbeat-serverless-tests"
+
+    steps:
+      - label: ":ubuntu: Serverless tests"
+        key: "agentbeat-it-serverless"
         depends_on:
           - agentbeat-package-linux
         env:
-          ASDF_NODEJS_VERSION: 18.17.1
-          PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
+          AGENT_STACK_VERSION: "8.16.0-SNAPSHOT"
+          TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-a"
+          GOFLAGS: "-buildvcs=false"
+          TEST_INTEG_CLEAN_ON_EXIT: true
+          TEST_PLATFORMS: "linux/amd64"
           SNAPSHOT: true
         command: |
-          set -euo pipefail
-          echo "~~~ Downloading artifacts"
-          buildkite-agent artifact download x-pack/agentbeat/build/distributions/** . --step 'agentbeat-package-linux'
-          ls -lah x-pack/agentbeat/build/distributions/
-          echo "~~~ Installing @elastic/synthetics with npm"
-          npm install -g @elastic/synthetics
-          echo "~~~ Running tests"
           cd x-pack/agentbeat
-          mage goIntegTest
+          mage serverlessTest metricbeat
         artifact_paths:
-          - x-pack/agentbeat/build/distributions/**/*
-          - "x-pack/agentbeat/build/*.xml"
-          - "x-pack/agentbeat/build/*.json"
+          - x-pack/agentbeat/build/TEST-**
+          - x-pack/agentbeat/build/diagnostics/*
         plugins:
           - test-collector#v1.10.2:
               files: "x-pack/agentbeat/build/TEST-*.xml"
@@ -90,12 +132,14 @@ steps:
           automatic:
             - limit: 1
         timeout_in_minutes: 60
+        concurrency_group: elastic-agent-extended-testing/serverless-integration
+        concurrency: 8
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
-          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+          machineType: "${GCP_STANDARD_MACHINE_TYPE}"
           disk_size: 100
           disk_type: "pd-ssd"
         notify:
           - github_commit_status:
-              context: "agentbeat: Integration tests"
+              context: "agentbeat: Serverless tests"

dev-tools/mage/agentbeat-serverless.go

@@ -0,0 +1,56 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package mage
+
+import (
+	"fmt"
+	"log"
+	"os"
+)
+
+// TestBeatServerless todo description
+func TestBeatServerless(beat string) {
+	if beat == "" {
+		log.Fatal("Beat is not defined")
+	}
+
+	if os.Getenv("AGENT_BUILD_DIR") == "" {
+		log.Fatal("AGENT_BUILD_DIR is not defined")
+	}
+
+	setStackProvisioner()
+	setTestBinaryName(beat)
+
+}
+
+func setStackProvisioner() {
+	stackProvisioner := os.Getenv("STACK_PROVISIONER")
+	if stackProvisioner == "" {
+		if err := os.Setenv("STACK_PROVISIONER", "serverless"); err != nil {
+			log.Fatal("error setting serverless stack var: %w", err)
+		}
+	} else if stackProvisioner == "stateful" {
+		fmt.Println("--- Warning: running TestBeatServerless as stateful")
+	}
+}
+
+func setTestBinaryName(beat string) {
+	if err := os.Setenv("TEST_BINARY_NAME", beat); err != nil {
+		log.Fatal("error setting binary name: %w", err)
+	}
+}

dev-tools/mage/gotest.go

@@ -428,3 +428,30 @@ func BuildSystemTestGoBinary(binArgs TestBinaryArgs) error {
 	}()
 	return sh.RunV("go", args...)
 }
+
+func GoTestBuild(ctx context.Context, params GoTestArgs) error {
+	if params.OutputFile == "" {
+		return fmt.Errorf("missing output file")
+	}
+
+	fmt.Println(">> go test:", params.TestName, "Building Test Binary")
+
+	args := []string{"test", "-c", "-o", params.OutputFile}
+
+	if len(params.Tags) > 0 {
+		params := strings.Join(params.Tags, " ")
+		if params != "" {
+			args = append(args, "-tags", params)
+		}
+	}
+
+	args = append(args, params.Packages...)
+
+	goTestBuild := makeCommand(ctx, params.Env, "go", args...)
+
+	err := goTestBuild.Run()
+	if err != nil {
+		return err
+	}
+	return nil
+}