Update x-pack/metricbeat & x-pack/filebeat pipellines to match Jenkins on 7.17

.buildkite/hooks/pre-command

@@ -6,7 +6,6 @@ source .buildkite/env-scripts/util.sh
 
 # Secrets must be redacted
 # https://buildkite.com/docs/pipelines/managing-log-output#redacted-environment-variables
-AWS_SERVICE_ACCOUNT_SECRET_PATH="kv/ci-shared/platform-ingest/aws_account_auth"
 PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/gcp-platform-ingest-ci-service-account"
 DOCKER_REGISTRY_SECRET_PATH="kv/ci-shared/platform-ingest/docker_registry_prod"
 GITHUB_TOKEN_VAULT_PATH="kv/ci-shared/platform-ingest/github_token"
@@ -72,15 +71,6 @@ for slug in "${ENABLED_BEATS_PIPELINES_SLUGS[@]}"; do
   fi
 done
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == *"xpack-metricbeat"* || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-filebeat" ]]; then
-  if [[ "$BUILDKITE_STEP_KEY" == *"extended-cloud-test"* ]]; then
-    BEATS_AWS_SECRET_KEY=$(retry_with_count 5 vault kv get -field secret_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
-    export BEATS_AWS_SECRET_KEY
-    BEATS_AWS_ACCESS_KEY=$(retry_with_count 5 vault kv get -field access_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
-    export BEATS_AWS_ACCESS_KEY
-  fi
-fi
-
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" ]]; then
   if [[ "$BUILDKITE_STEP_KEY" == "extended-win-10-system-tests" || "$BUILDKITE_STEP_KEY" == "mandatory-win-2022-system-tests" ]]; then
     PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry_with_count 5 vault kv get -field plaintext -format=json ${PRIVATE_CI_GCS_CREDENTIALS_PATH})

.buildkite/scripts/initCloudEnv.sh

@@ -0,0 +1,85 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+REPO_DIR=$(pwd)
+AWS_SERVICE_ACCOUNT_SECRET_PATH="kv/ci-shared/platform-ingest/aws_account_auth"
+
+exportAwsSecrets() {
+  local awsSecretKey
+  local awsAccessKey
+
+  awsSecretKey=$(retry -t 5 -- vault kv get -field secret_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
+  awsAccessKey=$(retry -t 5 -- vault kv get -field access_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
+
+  echo "~~~ Exporting AWS secrets"
+  export AWS_ACCESS_KEY_ID=$awsAccessKey
+  export AWS_SECRET_ACCESS_KEY=$awsSecretKey
+
+  # AWS_REGION is not set here, since AWS region is taken from *.tf file:
+  # - x-pack/metricbeat/module/aws/terraform.tf
+  # - x-pack/filebeat/input/awscloudwatch/_meta/terraform/variables.tf
+}
+
+terraformApply() {
+  TF_VAR_BRANCH=$(echo "${BUILDKITE_BRANCH}" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9-]/-/g')
+  TF_VAR_CREATED_DATE=$(date +%s)
+  export TF_VAR_BUILD_ID="${BUILDKITE_BUILD_ID}"
+  export TF_VAR_ENVIRONMENT="ci"
+  export TF_VAR_REPO="${REPO}"
+  export TF_VAR_BRANCH
+  export TF_VAR_CREATED_DATE
+
+  echo "Terraform Init on $MODULE_DIR"
+  terraform -chdir="$MODULE_DIR" init
+
+  echo "Terraform Apply on $MODULE_DIR"
+  terraform -chdir="$MODULE_DIR" apply -auto-approve
+}
+
+terraformDestroy() {
+  echo "--- Terraform Cleanup"
+  cd $REPO_DIR
+  find "$MODULE_DIR" -name terraform.tfstate -print0 | while IFS= read -r -d '' tfstate; do
+    cd "$(dirname "$tfstate")"
+    echo "Uploading terraform.tfstate to Buildkite artifacts"
+    buildkite-agent artifact upload "**/terraform.tfstate"
+    if ! terraform destroy -auto-approve; then
+      return 1
+    fi
+  cd -
+  done
+  return 0
+}
+
+trap 'terraformDestroy' EXIT
+exportAwsSecrets
+
+max_retries=2
+timeout=5
+retries=0
+
+while true; do
+  echo "~~~ Setting up Terraform"
+  out=$(terraformApply 2>&1)
+  exit_code=$?
+
+  echo "$out"
+
+  if [ $exit_code -eq 0 ]; then
+    break
+  else
+    retries=$((retries + 1))
+
+    if [ $retries -gt $max_retries ]; then
+      terraformDestroy
+      echo "+++ Terraform init & apply failed: $out"
+      exit 1
+    fi
+
+    terraformDestroy
+
+    sleep_time=$((timeout * retries))
+    echo "~~~~ Retry #$retries failed. Retrying after ${sleep_time}s..."
+    sleep $sleep_time
+  fi
+done

.buildkite/x-pack/pipeline.xpack.filebeat.yml

@@ -43,7 +43,7 @@ steps:
         notify:
           - github_commit_status:
               context: "x-pack-filebeat: check/update"
-      
+
   - wait: ~
     # with PRs, we want to run mandatory tests only if check/update step succeed
     # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests
@@ -129,7 +129,7 @@ steps:
           - github_commit_status:
               context: "x-pack/filebeat: Python Integration Tests"
 
-      - label: "Filebeat: Module compat tests - previous minor"
+      - label: ":ubuntu: x-pack/filebeat Module compat tests - previous minor"
         # Run module integration tests under previous minor of Elastic stack.
         env:
           STACK_ENVIRONMENT: "prev-minor"
@@ -154,7 +154,7 @@ steps:
           - "filebeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "filebeat: Module compat tests / previous minor"
+              context: "x-pack/filebeat: Module compat tests / previous minor"
 
       - label: ":windows: x-pack/filebeat Win-2019 Unit Tests"
         command: |
@@ -248,53 +248,23 @@ steps:
           - github_commit_status:
               context: "x-pack/filebeat: macOS x86_64 Unit Tests"
 
-      - label: ":ubuntu: x-pack/filebeat Cloud (MODULE) Tests"
+      - label: ":ubuntu: x-pack/filebeat AWS Tests"
         key: "x-pack-filebeat-extended-cloud-test"
+        skip: "skipping due to elastic/ingest-dev#3467"
+        # https://github.com/elastic/ingest-dev/issues/3467
         if: build.env("GITHUB_PR_LABELS") =~ /.*aws.*/
         command: |
           set -euo pipefail
-          # defines the MODULE env var based on what's changed in a PR
-          source .buildkite/scripts/changesets.sh
-          defineModuleFromTheChangeSet x-pack/filebeat
-          echo "~~~ Running tests"
-          source .buildkite/scripts/setup_cloud_env.sh
-          cd x-pack/filebeat
-          mage build test
-        env:
-          ASDF_TERRAFORM_VERSION: 1.0.2
-          AWS_REGION: "eu-central-1"
-          MODULE_DIR: "x-pack/filebeat/input/awss3/_meta/terraform"
-          REPO: beats
-        agents:
-          provider: "gcp"
-          image: "${IMAGE_UBUNTU_X86_64}"
-          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
-        artifact_paths:
-          - "x-pack/filebeat/build/*.xml"
-          - "x-pack/filebeat/build/*.json"
-        notify:
-          - github_commit_status:
-              context: "x-pack/filebeat: Cloud (MODULE) Tests"
-
-      - label: ":ubuntu: x-pack/filebeat Cloud AWS (MODULE) Tests"
-        key: "x-pack-filebeat-extended-cloud-test-aws"
-        skip: "Skipping due to elastic/beats#36425"
-        # https://github.com/elastic/beats/issues/36425
-        if: build.env("BUILDKITE_PULL_REQUEST") == "false"  || build.env("GITHUB_PR_LABELS") =~ /.*aws.*/
-        command: |
-          set -euo pipefail
-          # defines the MODULE env var based on what's changed in a PR
-          source .buildkite/scripts/changesets.sh
-          defineModuleFromTheChangeSet x-pack/filebeat
+          source .buildkite/scripts/initCloudEnv.sh
           echo "~~~ Running tests"
-          source .buildkite/scripts/setup_cloud_env.sh
           cd x-pack/filebeat
           mage build test
         env:
           ASDF_TERRAFORM_VERSION: 1.0.2
-          AWS_REGION: "eu-central-1"
           MODULE_DIR: "x-pack/filebeat/input/awss3/_meta/terraform"
-          REPO: beats
+          MODULE: "aws"
+          #  TEST_TAGS should be reviewed and updated: https://github.com/elastic/ingest-dev/issues/3476
+          TEST_TAGS: "aws"
         agents:
           provider: "aws"
           imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
@@ -304,7 +274,7 @@ steps:
           - "x-pack/filebeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "x-pack/filebeat: Cloud AWS (MODULE) Tests"
+              context: "x-pack/filebeat: AWS Tests"
 
   - wait: ~
     # with PRs, we want to run packaging only if mandatory tests succeed
@@ -317,7 +287,7 @@ steps:
   - group: "x-pack/filebeat Packaging"
     key: "x-pack-filebeat-packaging"
     steps:
-      - label: ":linux: x-pack/filebeat Packaging Linux"
+      - label: ":ubuntu: x-pack/filebeat Packaging Linux"
         key: "packaging-linux"
         command: |
           cd x-pack/filebeat
@@ -338,7 +308,7 @@ steps:
           - github_commit_status:
               context: "x-pack/filebeat: Packaging Linux"
 
-      - label: ":linux: x-pack/filebeat Packaging arm64"
+      - label: ":ubuntu: x-pack/filebeat Packaging arm64"
         key: "packaging-arm"
         command: |
           cd x-pack/filebeat