Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[8.14](backport #38941) Migrate X-PACK Filebeat pipeline to main Beat pipeline #39044

Merged
merged 2 commits into from
Apr 19, 2024
Merged

[8.14](backport #38941) Migrate X-PACK Filebeat pipeline to main Beat pipeline #39044

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a3242f3
Migrate X-PACK Filebeat pipeline to main Beat pipeline (#38941)
alexsapran Apr 18, 2024
0201f9b
Merge remote-tracking branch 'upstream/8.14' into mergify/bp/8.14/pr-…
alexsapran Apr 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion .buildkite/hooks/pre-command
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,6 @@ ENABLED_BEATS_PIPELINES_SLUGS=(
"beats-xpack-winlogbeat"
"beats-xpack-dockerlogbeat"
"beats-xpack-auditbeat"
"beats-xpack-filebeat"
"beats-xpack-metricbeat"
"beats-xpack-heartbeat"
"deploy-k8s"
Expand Down
27 changes: 27 additions & 0 deletions .buildkite/pipeline.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,33 @@ steps:
- BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
- GITHUB_PR_LABELS=${GITHUB_PR_LABELS}

- label: "Trigger x-pack/filebeat"
plugins:
- monorepo-diff#v1.0.1:
diff: "git diff --name-only origin/${GITHUB_PR_TARGET_BRANCH}...HEAD"
watch:
- path:
- x-pack/filebeat/
- x-pack/libbeat/
- .buildkite/x-pack/pipeline.xpack.filebeat.yml
- .buildkite/scripts
- .buildkite/hooks/
#OSS
- go.mod
- pytest.ini
- dev-tools/
- libbeat/**
- testing/**
config:
trigger: "beats-xpack-filebeat"
build:
commit: "${BUILDKITE_COMMIT}"
branch: "${BUILDKITE_BRANCH}"
env:
- BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST}
- BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
- GITHUB_PR_LABELS=${GITHUB_PR_LABELS}

- label: "Trigger Xpack/Dockerlogbeat"
plugins:
- monorepo-diff#v1.0.1:
Expand Down
14 changes: 7 additions & 7 deletions .buildkite/scripts/cloud_tests.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,12 @@
#!/usr/bin/env bash
set -euo pipefail

# What Terraform Module will run
if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-filebeat" ]]; then
export MODULE_DIR="x-pack/filebeat/input/awss3/_meta/terraform"
fi

REPO_DIR=$(pwd)

teardown() {
popd
# reset the directory to the root of the project
cd $REPO_DIR
# Teardown resources after using them
echo "~~~ Terraform Cleanup"
tf_cleanup "${MODULE_DIR}" #TODO: move all docker-compose files from the .ci to .buildkite folder before switching to BK
Expand Down Expand Up @@ -43,15 +42,16 @@ set -o xtrace
echo "~~~ Run docker-compose services for emulated cloud env"
docker-compose -f .ci/jobs/docker-compose.yml up -d #TODO: move all docker-compose files from the .ci to .buildkite folder before switching to BK
echo "~~~ Initialize TF cloud resources"
pushd "$MODULE_DIR"
cd "$MODULE_DIR"
export TF_VAR_BRANCH=$(echo "${BUILDKITE_BRANCH}" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9-]/-/g')
export TF_VAR_BUILD_ID="${BUILDKITE_BUILD_ID}"
export TF_VAR_CREATED_DATE=$(date +%s)
export TF_VAR_ENVIRONMENT="ci"
export TF_VAR_REPO="${REPO}"
terraform init && terraform apply -auto-approve
cd -

# Run tests
echo "~~~ Run Cloud Tests for $BEATS_PROJECT_NAME"
pushd "${BEATS_PROJECT_NAME}"
cd "${BEATS_PROJECT_NAME}"
mage build test
20 changes: 3 additions & 17 deletions .buildkite/scripts/common.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,27 +18,20 @@ XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*"
[ -z "${run_xpack_libbeat+x}" ] && run_xpack_libbeat="$(buildkite-agent meta-data get run_xpack_libbeat --default "false")"
[ -z "${run_xpack_metricbeat+x}" ] && run_xpack_metricbeat="$(buildkite-agent meta-data get run_xpack_metricbeat --default "false")"
[ -z "${run_xpack_packetbeat+x}" ] && run_xpack_packetbeat="$(buildkite-agent meta-data get run_xpack_packetbeat --default "false")"
[ -z "${run_xpack_filebeat+x}" ] && run_xpack_filebeat="$(buildkite-agent meta-data get run_xpack_filebeat --default "false")"
[ -z "${run_xpack_heartbeat+x}" ] && run_xpack_heartbeat="$(buildkite-agent meta-data get run_xpack_heartbeat --default "false")"
alexsapran marked this conversation as resolved.
Show resolved Hide resolved

# define if needed run ARM platform-specific tests for the particular beat
[ -z "${run_filebeat_arm_tests+x}" ] && run_filebeat_arm_tests="$(buildkite-agent meta-data get run_filebeat_arm_tests --default "false")"
[ -z "${run_packetbeat_arm_tests+x}" ] && run_packetbeat_arm_tests="$(buildkite-agent meta-data get run_packetbeat_arm_tests --default "false")"
[ -z "${run_xpack_filebeat_arm_tests+x}" ] && run_xpack_filebeat_arm_tests="$(buildkite-agent meta-data get run_xpack_filebeat_arm_tests --default "false")"
[ -z "${run_xpack_libbeat_arm_tests+x}" ] && run_xpack_libbeat_arm_tests="$(buildkite-agent meta-data get run_xpack_libbeat_arm_tests --default "false")"
[ -z "${run_xpack_packetbeat_arm_tests+x}" ] && run_xpack_packetbeat_arm_tests="$(buildkite-agent meta-data get run_xpack_packetbeat_arm_tests --default "false")"

# define if needed run MacOS platform-specific tests for the particular beat
[ -z "${run_packetbeat_macos_tests+x}" ] && run_packetbeat_macos_tests="$(buildkite-agent meta-data get run_packetbeat_macos_tests --default "false")"
[ -z "${run_xpack_filebeat_macos_tests+x}" ] && run_xpack_filebeat_macos_tests="$(buildkite-agent meta-data get run_xpack_filebeat_macos_tests --default "false")"
[ -z "${run_xpack_metricbeat_macos_tests+x}" ] && run_xpack_metricbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_macos_tests --default "false")"
[ -z "${run_xpack_packetbeat_macos_tests+x}" ] && run_xpack_packetbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_packetbeat_macos_tests --default "false")"
[ -z "${run_xpack_heartbeat_macos_tests+x}" ] && run_xpack_heartbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_heartbeat_macos_tests --default "false")"
alexsapran marked this conversation as resolved.
Show resolved Hide resolved

# define if needed run cloud-specific tests for the particular beat
[ -z "${run_xpack_metricbeat_aws_tests+x}" ] && run_xpack_metricbeat_aws_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_aws_tests --default "false")"
[ -z "${run_xpack_filebeat_aws_tests+x}" ] && run_xpack_filebeat_aws_tests="$(buildkite-agent meta-data get run_xpack_filebeat_aws_tests --default "false")"


libbeat_changeset=(
"^libbeat/.*"
Expand All @@ -60,10 +53,6 @@ xpack_heartbeat_changeset=(
"^x-pack/heartbeat/.*"
)

alexsapran marked this conversation as resolved.
Show resolved Hide resolved
xpack_filebeat_changeset=(
"^x-pack/filebeat/.*"
)

xpack_libbeat_changeset=(
"^x-pack/libbeat/.*"
)
Expand Down Expand Up @@ -117,9 +106,6 @@ case "${BUILDKITE_PIPELINE_SLUG}" in
"beats-winlogbeat")
BEAT_CHANGESET_REFERENCE=${winlogbeat_changeset[@]}
;;
"beats-xpack-filebeat")
BEAT_CHANGESET_REFERENCE=${xpack_filebeat_changeset[@]}
;;
"beats-xpack-heartbeat")
BEAT_CHANGESET_REFERENCE=${xpack_heartbeat_changeset[@]}
alexsapran marked this conversation as resolved.
Show resolved Hide resolved
;;
Expand Down Expand Up @@ -371,7 +357,7 @@ are_conditions_met_mandatory_tests() {

are_conditions_met_arm_tests() {
if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171
if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-filebeat" ]]; then
if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then
if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_ARM_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BEATS_GH_ARM_LABEL} || "${!TRIGGER_SPECIFIC_ARM_TESTS}" == "true" ]]; then
return 0
fi
Expand All @@ -382,7 +368,7 @@ are_conditions_met_arm_tests() {

are_conditions_met_macos_tests() {
if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171
if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-heartbeat"; then
if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-heartbeat" ]] ; then
alexsapran marked this conversation as resolved.
Show resolved Hide resolved
if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_MACOS_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BEATS_GH_MACOS_LABEL} || "${!TRIGGER_SPECIFIC_MACOS_TESTS}" == "true" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12
return 0
fi
Expand All @@ -393,7 +379,7 @@ are_conditions_met_macos_tests() {

are_conditions_met_aws_tests() {
if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171
if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-filebeat" ]]; then
if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then
if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_AWS_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BEATS_GH_AWS_LABEL} || "${!TRIGGER_SPECIFIC_AWS_TESTS}" == "true" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12
return 0
fi
Expand Down
Loading