Update xpack/filebeat & xpack/metricbeat pipelines to match Jenkins s…

…teps (elastic#39211)

Updated x-pack/filebeat and x-pack/metricbeat pipelines to match same steps & commands as in Jenkins

.buildkite/scripts/changesets.sh

@@ -1,77 +1,62 @@
 #!/usr/bin/env bash
 
-# This script contains helper functions related to what should be run depending on Git changes
+set -euo pipefail
 
 OSS_MODULE_PATTERN="^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*"
 XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*"
 
-are_paths_changed() {
-  local patterns=("${@}")
-  local changelist=()
-  for pattern in "${patterns[@]}"; do
-    changed_files=($(git diff --name-only HEAD@{1} HEAD | grep -E "$pattern"))
-    if [ "${#changed_files[@]}" -gt 0 ]; then
-      changelist+=("${changed_files[@]}")
-    fi
-  done
+definePattern() {
+  pattern="${OSS_MODULE_PATTERN}"
 
-  if [ "${#changelist[@]}" -gt 0 ]; then
-    echo "Files changed:"
-    echo "${changelist[*]}"
-    return 0
-  else
-    echo "No files changed within specified changeset:"
-    echo "${patterns[*]}"
-    return 1
+  if [[ "$beatPath" == *"x-pack/"* ]]; then
+    pattern="${XPACK_MODULE_PATTERN}"
   fi
 }
 
-are_changed_only_paths() {
-  local patterns=("${@}")
-  local changed_files=($(git diff --name-only HEAD@{1} HEAD))
-  local matched_files=()
-  for pattern in "${patterns[@]}"; do
-    local matched=($(grep -E "${pattern}" <<< "${changed_files[@]}"))
-    if [ "${#matched[@]}" -gt 0 ]; then
-      matched_files+=("${matched[@]}")
-    fi
-  done
-  if [ "${#matched_files[@]}" -eq "${#changed_files[@]}" ] || [ "${#changed_files[@]}" -eq 0 ]; then
-    return 0
-  fi
-  return 1
+defineExclusions() {
+  exclude="^$beatPath\/module\/(.*(?<!\.asciidoc|\.png))$"
 }
 
-defineModuleFromTheChangeSet() {
-  # This function sets a `MODULE` env var, required by IT tests, containing a comma separated list of modules for a given beats project (specified via the first argument).
-  # The list is built depending on directories that have changed under `modules/` excluding anything else such as asciidoc and png files.
-  # `MODULE` will empty if no changes apply.
-  local project_path=$1
-  local project_path_transformed=$(echo "$project_path" | sed 's/\//\\\//g')
-  local project_path_exclussion="((?!^${project_path_transformed}\\/).)*\$"
-  local exclude=("^(${project_path_exclussion}|((?!\\/module\\/).)*\$|.*\\.asciidoc|.*\\.png)")
+defineFromCommit() {
+  local previousCommit
+  local changeTarget=${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-$BUILDKITE_BRANCH}
 
-  if [[ "$project_path" == *"x-pack/"* ]]; then
-    local pattern=("$XPACK_MODULE_PATTERN")
-  else
-    local pattern=("$OSS_MODULE_PATTERN")
-  fi
-  local changed_modules=""
-  local module_dirs=$(find "$project_path/module" -mindepth 1 -maxdepth 1 -type d)
-  for module_dir in $module_dirs; do
-    if are_paths_changed $module_dir && ! are_changed_only_paths "${exclude[@]}"; then
-      if [[ -z "$changed_modules" ]]; then
-        changed_modules=$(basename "$module_dir")
-      else
-        changed_modules+=",$(basename "$module_dir")"
-      fi
+  previousCommit=$(git rev-parse HEAD^)
+
+  from=${changeTarget:+"origin/$changeTarget"}
+  from=${from:-$previousCommit}
+  from=${from:-$BUILDKITE_COMMIT}
+}
+
+getMatchingModules() {
+  local changedPaths
+  mapfile -t changedPaths < <(git diff --name-only "$from"..."$BUILDKITE_COMMIT" | grep -P "$exclude" | grep -oP "$pattern")
+  mapfile -t modulesMatched < <(printf "%s\n" "${changedPaths[@]}" | grep -o 'module/.*' | awk -F '/' '{print $2}' | sort -u)
+}
+
+addToModuleEnvVar() {
+  local module
+  for module in "${modulesMatched[@]}"; do
+    if [[ -z ${modules+x} ]]; then
+      modules="$module"
+    else
+      modules+=",$module"
     fi
   done
+}
+
+defineModule() {
+  beatPath=$1
 
-  # export MODULE="" leads to an infinite loop https://github.com/elastic/ingest-dev/issues/2993
-  if [[ ! -z $changed_modules ]]; then 
-    export MODULE="${changed_modules}"
-    echo "~~~ Set env var MODULE to [$MODULE]"
-    echo "~~~ Resuming commands"
+  definePattern
+  defineExclusions
+  defineFromCommit
+  getMatchingModules
+
+  if [ "${#modulesMatched[@]}" -gt 0 ]; then
+    addToModuleEnvVar
+    export MODULE=$modules
+  else
+    echo "~~~ No changes in modules for $beatPath"
   fi
 }

.buildkite/scripts/cloud_tests.sh → .buildkite/scripts/setup_cloud_env.sh

@@ -1,7 +1,6 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-
 REPO_DIR=$(pwd)
 
 teardown() {
@@ -31,7 +30,6 @@ tf_cleanup() {
 trap 'teardown' EXIT
 
 # Prepare the cloud resources using Terraform
-#startCloudTestEnv "${MODULE_DIR}"
 echo "~~~ Loading creds"
 set +o xtrace
 export AWS_ACCESS_KEY_ID=$BEATS_AWS_ACCESS_KEY
@@ -50,8 +48,3 @@ export TF_VAR_ENVIRONMENT="ci"
 export TF_VAR_REPO="${REPO}"
 terraform init && terraform apply -auto-approve
 cd -
-
-# Run tests
-echo "~~~ Run Cloud Tests for $BEATS_PROJECT_NAME"
-cd "${BEATS_PROJECT_NAME}"
-mage build test