Merge branch 'main' into enable-terraform-output

.buildkite/hooks/post-checkout

@@ -47,7 +47,7 @@ MERGE_BRANCH="pr_merge_${PR_ID}"
 checkout_merge "${TARGET_BRANCH}" "${PR_COMMIT}" "${MERGE_BRANCH}"
 
 echo "Commit information"
-git log --format=%B -n 1
+git --no-pager log --format=%B -n 1
 
 # Ensure buildkite groups are rendered
 echo ""

.buildkite/pipeline.trigger.integration.tests.sh

@@ -35,20 +35,20 @@ CHECK_PACKAGES_TESTS=(
     test-check-packages-with-custom-agent
     test-check-packages-benchmarks
 )
-# for test in ${CHECK_PACKAGES_TESTS[@]}; do
-#     echo "      - label: \":go: Running integration test: ${test}\""
-#     echo "        command: ./.buildkite/scripts/integration_tests.sh -t ${test}"
-#     echo "        agents:"
-#     echo "          provider: \"gcp\""
-#     echo "        artifact_paths:"
-#     echo "          - build/test-results/*.xml"
-#     echo "          - build/elastic-stack-dump/stack/check-*/logs/*.log"
-#     echo "          - build/elastic-stack-dump/stack/check-*/logs/fleet-server-internal/**/*"
-#     echo "          - build/elastic-stack-status/*/*"
-#     if [[ $test =~ with-kind$ ]]; then
-#         echo "          - build/kubectl-dump.txt"
-#     fi
-# done
+
+for test in ${CHECK_PACKAGES_TESTS[@]}; do
+    echo "      - label: \":go: Running integration test: ${test}\""
+    echo "        command: ./.buildkite/scripts/integration_tests.sh -t ${test}"
+    echo "        agents:"
+    echo "          provider: \"gcp\""
+    echo "        artifact_paths:"
+    echo "          - build/test-results/*.xml"
+    echo "          - build/elastic-stack-dump/check-*/logs/*.log"
+    echo "          - build/elastic-stack-dump/check-*/logs/fleet-server-internal/**/*"
+    if [[ $test =~ with-kind$ ]]; then
+        echo "          - build/kubectl-dump.txt"
+    fi
+done
 
 pushd test/packages/parallel > /dev/null
 for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do
@@ -62,8 +62,6 @@ for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do
     echo "          provider: \"gcp\""
     echo "        artifact_paths:"
     echo "          - build/test-results/*.xml"
-    echo "          - build/elastic-stack-dump/stack/check-*/logs/*.log"
-    echo "          - build/elastic-stack-dump/stack/check-*/logs/fleet-server-internal/**/*"
 done
 
 popd > /dev/null

.buildkite/pipeline.yml

@@ -2,6 +2,7 @@ env:
   SETUP_GVM_VERSION: 'v0.5.0' # https://github.com/andrewkroh/gvm/issues/44#issuecomment-1013231151
   DOCKER_COMPOSE_VERSION: "v2.17.2"
   ELASTIC_PACKAGE_COMPOSE_DISABLE_ANSI: "true"
+  ELASTIC_PACKAGE_COMPOSE_DISABLE_PULL_PROGRESS_INFORMATION: "true"
   KIND_VERSION: 'v0.17.0'
   K8S_VERSION: 'v1.26.0'
 

.buildkite/scripts/install_deps.sh

@@ -6,7 +6,7 @@ source .buildkite/scripts/tooling.sh
 
 add_bin_path(){
     mkdir -p ${WORKSPACE}/bin
-    export PATH="${PATH}:${WORKSPACE}/bin"
+    export PATH="${WORKSPACE}/bin:${PATH}"
 }
 
 with_kubernetes() {

.buildkite/scripts/integration_tests.sh

@@ -30,9 +30,8 @@ GOOGLE_CREDENTIALS_FILENAME="google-cloud-credentials.json"
 
 JOB_GCS_BUCKET_INTERNAL="fleet-ci-temp-internal"
 
-REPO_NAME=$(repoName "${BUILDKITE_REPO}")
-REPO_BUILD_TAG="${REPO_NAME}/${BUILDKITE_BUILD_NUMBER}"
-
+REPO_NAME=$(repo_name "${BUILDKITE_REPO}")
+REPO_BUILD_TAG="${REPO_NAME}/$(buildkite_pr_branch_build_id)"
 TARGET=""
 PACKAGE=""
 while getopts ":t:p:h" o; do

.buildkite/scripts/signAndPublishPackage.sh

@@ -16,7 +16,7 @@ cleanup() {
 
 trap cleanup EXIT
 
-isAlreadyPublished() {
+is_already_published() {
     local packageZip=$1
 
     if curl -s --head https://package-storage.elastic.co/artifacts/packages/${packageZip} | grep -q "HTTP/2 200" ; then
@@ -34,7 +34,7 @@ if ! command -v gsutil &> /dev/null ; then
 fi
 
 
-REPO_NAME=$(repoName "${BUILDKITE_REPO}")
+REPO_NAME=$(repo_name "${BUILDKITE_REPO}")
 BUILD_TAG="buildkite-${BUILDKITE_PIPELINE_SLUG}-${BUILDKITE_BUILD_NUMBER}"
 
 REPO_BUILD_TAG="${REPO_NAME}/${BUILD_TAG}"
@@ -76,7 +76,7 @@ google_cloud_auth_publishing() {
     echo "${gsUtilLocation}"
 }
 
-signPackage() {
+sign_package() {
     local package=${1}
     local packageZip=$(basename ${package})
 
@@ -110,7 +110,7 @@ signPackage() {
     rm -r "${gsUtilLocation}"
 }
 
-publishPackage() {
+publish_package() {
     local package=$1
     local packageZip=$(basename ${package})
 
@@ -154,14 +154,14 @@ ls -l "${BUILD_PACKAGES_PATH}"
 for package in $(ls ${BUILD_PACKAGES_PATH}/*.zip); do
     echo "isAlreadyInstalled ${package}?"
     packageZip=$(basename ${package})
-    if isAlreadyPublished ${packageZip} ; then
+    if is_already_published ${packageZip} ; then
         echo "Skipping. ${packageZip} already published"
         continue
     fi
 
     echo "Signing package ${packageZip}"
-    signPackage "${package}"
+    sign_package "${package}"
 
     echo "Publishing package ${packageZip}"
-    publishPackage "${package}"
+    publish_package "${package}"
 done

.buildkite/scripts/tooling.sh

@@ -1,14 +1,22 @@
 #!/bin/bash
 set -euo pipefail
 
-repoName() {
+repo_name() {
     # Example of URL: [email protected]:acme-inc/my-project.git
     local repoUrl=$1
 
     orgAndRepo=$(echo $repoUrl | cut -d':' -f 2)
     echo "$(basename ${orgAndRepo} .git)"
 }
 
+buildkite_pr_branch_build_id() {
+    if [ "${BUILDKITE_PULL_REQUEST}" == "false" ]; then
+        echo "${BUILDKITE_BRANCH}-${BUILDKITE_BUILD_NUMBER}"
+        return
+    fi
+    echo "PR-${BUILDKITE_PULL_REQUEST}-${BUILDKITE_BUILD_NUMBER}"
+}
+
 google_cloud_auth() {
     local keyFile=$1
 

Makefile

@@ -65,7 +65,7 @@ test-stack-command-86:
 	./scripts/test-stack-command.sh 8.6.2
 
 test-stack-command-8x:
-	./scripts/test-stack-command.sh 8.8.0-SNAPSHOT
+	./scripts/test-stack-command.sh 8.9.0-SNAPSHOT
 
 test-stack-command: test-stack-command-default test-stack-command-7x test-stack-command-800 test-stack-command-8x
 
@@ -89,15 +89,11 @@ test-check-packages-with-custom-agent:
 test-build-zip:
 	./scripts/test-build-zip.sh
 
-# TODO remove version once default one is 8.7.0 or higher
-# Setting environment variables manually
 test-install-zip:
-	./scripts/test-install-zip.sh -v 8.7.0-SNAPSHOT
+	./scripts/test-install-zip.sh
 
-# TODO remove version once default one is 8.7.0 or higher
-# Using elastic-package stack shellinit to populate environment variables
 test-install-zip-shellinit:
-	./scripts/test-install-zip.sh -v 8.7.0-SNAPSHOT -s
+	./scripts/test-install-zip.sh -s
 
 test-profiles-command:
 	./scripts/test-profiles-command.sh

README.md

@@ -100,6 +100,12 @@ These benchmarks allow you to benchmark any Ingest Node Pipelines defined by you
 
 For details on how to configure pipeline benchmarks for a package, review the [HOWTO guide](./docs/howto/pipeline_benchmarking.md).
 
+#### System Benchmarks
+
+These benchmarks allow you to benchmark an integration end to end.
+
+For details on how to configure system benchmarks for a package, review the [HOWTO guide](./docs/howto/system_benchmarking.md).
+
 ### `elastic-package benchmark generate-corpus`
 
 _Context: package_
@@ -116,6 +122,12 @@ _Context: package_
 
 Run pipeline benchmarks for the package.
 
+### `elastic-package benchmark system`
+
+_Context: package_
+
+Run system benchmarks for the package.
+
 ### `elastic-package build`
 
 _Context: package_
@@ -350,6 +362,8 @@ _Context: global_
 
 Use this command to spin up a Docker-based Elastic Stack consisting of Elasticsearch, Kibana, and the Package Registry. By default the latest released version of the stack is spun up but it is possible to specify a different version, including SNAPSHOT versions by appending --version <version>.
 
+You can run your own custom images for Elasticsearch, Kibana or Elastic Agent, see [this document](./docs/howto/custom_images.md).
+
 Be aware that a common issue while trying to boot up the stack is that your Docker environments settings are too low in terms of memory threshold.
 
 For details on how to connect the service with the Elastic stack, see the [service command](https://github.com/elastic/elastic-package/blob/main/README.md#elastic-package-service).
@@ -386,9 +400,11 @@ Use this command to boot up the stack locally.
 
 By default the latest released version of the stack is spun up but it is possible to specify a different version, including SNAPSHOT versions by appending --version <version>.
 
+You can run your own custom images for Elasticsearch, Kibana or Elastic Agent, see [this document](./docs/howto/custom_images.md).
+
 Be aware that a common issue while trying to boot up the stack is that your Docker environments settings are too low in terms of memory threshold.
 
-To expose local packages in the Package Registry, build them first and boot up the stack from inside of the Git repository containing the package (e.g. elastic/integrations). They will be copied to the development stack (~/.elastic-package/stack/development) and used to build a custom Docker image of the Package Registry.
+To expose local packages in the Package Registry, build them first and boot up the stack from inside of the Git repository containing the package (e.g. elastic/integrations). They will be copied to the development stack (~/.elastic-package/stack/development) and used to build a custom Docker image of the Package Registry. Starting with Elastic stack version >= 8.7.0, it is not mandatory to be available local packages in the Package Registry to run the tests.
 
 For details on how to connect the service with the Elastic stack, see the [service command](https://github.com/elastic/elastic-package/blob/main/README.md#elastic-package-service).
 

catalog-info.yaml

@@ -46,7 +46,7 @@ spec:
         build_pull_requests: true # requires filter_enabled and filter_condition settings as below when used with buildkite-pr-bot
         build_tags: true
         filter_enabled: true
-        filter_condition: |
+        filter_condition: >-
           build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null)
       repository: elastic/elastic-package
       cancel_intermediate_builds: true
@@ -92,7 +92,7 @@ spec:
         build_pull_requests: true # requires filter_enabled and filter_condition settings as below when used with buildkite-pr-bot
         build_tags: true
         filter_enabled: true
-        filter_condition: |
+        filter_condition: >-
           build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null)
       repository: elastic/elastic-package
       cancel_intermediate_builds: true
@@ -138,7 +138,7 @@ spec:
         build_pull_request_forks: false
         build_pull_requests: true # requires filter_enabled and filter_condition settings as below when used with buildkite-pr-bot
         filter_enabled: true
-        filter_condition: |
+        filter_condition: >-
           build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null)
       repository: elastic/elastic-package
       cancel_intermediate_builds: true