Merge branch 'main' into osquery-all-means-all

.buildkite/package.json

@@ -3,6 +3,6 @@
   "version": "1.0.0",
   "private": true,
   "dependencies": {
-    "kibana-buildkite-library": "elastic/kibana-buildkite-library"
+    "kibana-buildkite-library": "git+https://[email protected]/elastic/kibana-buildkite-library#0f95579ace8100de9f1ad4cc16976b9ec6d5841e"
   }
 }

.buildkite/pipelines/es_snapshots/verify.yml

@@ -29,7 +29,7 @@ steps:
 
   - command: .buildkite/scripts/steps/functional/xpack_cigroup.sh
     label: 'Default CI Group'
-    parallelism: 30
+    parallelism: 31
     agents:
       queue: n2-4
     depends_on: build
@@ -42,7 +42,7 @@ steps:
 
   - command: .buildkite/scripts/steps/functional/oss_cigroup.sh
     label: 'OSS CI Group'
-    parallelism: 11
+    parallelism: 12
     agents:
       queue: ci-group-4d
     depends_on: build
@@ -53,13 +53,12 @@ steps:
         - exit_status: '*'
           limit: 1
 
-  - command: .buildkite/scripts/steps/test/jest_integration.sh
-    label: 'Jest Integration Tests'
-    parallelism: 3
+  - command: .buildkite/scripts/steps/test/pick_jest_config_run_order.sh
+    label: 'Pick Jest Config Run Order'
     agents:
-      queue: n2-4
-    timeout_in_minutes: 120
-    key: jest-integration
+      queue: kibana-default
+    env:
+      FILTER_JEST_CONFIG_TYPE: integration
     retry:
       automatic:
         - exit_status: '*'
@@ -79,7 +78,6 @@ steps:
       queue: kibana-default
     depends_on:
       - default-cigroup
-      - default-cigroup-docker
       - oss-cigroup
       - jest-integration
       - api-integration

.buildkite/pipelines/on_merge.yml

@@ -51,7 +51,7 @@ steps:
 
   - command: .buildkite/scripts/steps/functional/xpack_cigroup.sh
     label: 'Default CI Group'
-    parallelism: 30
+    parallelism: 31
     agents:
       queue: n2-4-spot
     depends_on: build
@@ -66,7 +66,7 @@ steps:
 
   - command: .buildkite/scripts/steps/functional/oss_cigroup.sh
     label: 'OSS CI Group'
-    parallelism: 11
+    parallelism: 12
     agents:
       queue: n2-4-spot
     depends_on: build
@@ -157,29 +157,14 @@ steps:
         - exit_status: '*'
           limit: 1
 
-  - command: .buildkite/scripts/steps/test/jest.sh
-    label: 'Jest Tests'
-    parallelism: 8
+  - command: .buildkite/scripts/steps/test/pick_jest_config_run_order.sh
+    label: 'Pick Jest Config Run Order'
     agents:
-      queue: n2-4-spot
-    timeout_in_minutes: 90
-    key: jest
-    retry:
-      automatic:
-        - exit_status: '-1'
-          limit: 3
-
-  - command: .buildkite/scripts/steps/test/jest_integration.sh
-    label: 'Jest Integration Tests'
-    parallelism: 3
-    agents:
-      queue: n2-4-spot
-    timeout_in_minutes: 120
-    key: jest-integration
+      queue: kibana-default
     retry:
       automatic:
-        - exit_status: '-1'
-          limit: 3
+        - exit_status: '*'
+          limit: 1
 
   - command: .buildkite/scripts/steps/test/api_integration.sh
     label: 'API Integration Tests'

.buildkite/pipelines/pull_request/base.yml

@@ -17,7 +17,7 @@ steps:
 
   - command: .buildkite/scripts/steps/functional/xpack_cigroup.sh
     label: 'Default CI Group'
-    parallelism: 30
+    parallelism: 31
     agents:
       queue: n2-4-spot-2
     depends_on: build
@@ -32,7 +32,7 @@ steps:
 
   - command: .buildkite/scripts/steps/functional/oss_cigroup.sh
     label: 'OSS CI Group'
-    parallelism: 11
+    parallelism: 12
     agents:
       queue: n2-4-spot
     depends_on: build
@@ -123,29 +123,14 @@ steps:
         - exit_status: '*'
           limit: 1
 
-  - command: .buildkite/scripts/steps/test/jest.sh
-    label: 'Jest Tests'
-    parallelism: 8
+  - command: .buildkite/scripts/steps/test/pick_jest_config_run_order.sh
+    label: 'Pick Jest Config Run Order'
     agents:
-      queue: n2-4-spot
-    timeout_in_minutes: 90
-    key: jest
-    retry:
-      automatic:
-        - exit_status: '-1'
-          limit: 3
-
-  - command: .buildkite/scripts/steps/test/jest_integration.sh
-    label: 'Jest Integration Tests'
-    parallelism: 3
-    agents:
-      queue: n2-4-spot
-    timeout_in_minutes: 120
-    key: jest-integration
+      queue: kibana-default
     retry:
       automatic:
-        - exit_status: '-1'
-          limit: 3
+        - exit_status: '*'
+          limit: 1
 
   - command: .buildkite/scripts/steps/test/api_integration.sh
     label: 'API Integration Tests'

.buildkite/pipelines/pull_request/response_ops.yml

@@ -1,6 +1,6 @@
 steps:
-  - command: .buildkite/scripts/steps/functional/response_ops_cases.sh
-    label: 'Cases Cypress Tests on Security Solution'
+  - command: .buildkite/scripts/steps/functional/response_ops.sh
+    label: 'Rules, Alerts and Exceptions ResponseOps Cypress Tests on Security Solution'
     agents:
       queue: ci-group-6
     depends_on: build

.buildkite/pipelines/pull_request/response_ops_cases.yml

@@ -0,0 +1,11 @@
+steps:
+  - command: .buildkite/scripts/steps/functional/response_ops_cases.sh
+    label: 'Cases Cypress Tests on Security Solution'
+    agents:
+      queue: ci-group-6
+    depends_on: build
+    timeout_in_minutes: 120
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1

.buildkite/scripts/common/setup_bazel.sh

@@ -11,8 +11,29 @@ cat <<EOF > $KIBANA_DIR/.bazelrc
   build --build_metadata=ROLE=CI
 EOF
 
-if [[ "${BAZEL_CACHE_MODE:-none}" == read* ]]; then
-  echo "[bazel] enabling caching"
+BAZEL_CACHE_MODE=${BAZEL_CACHE_MODE:-gcs}
+
+if [[ "$BAZEL_CACHE_MODE" == "gcs" ]]; then
+  echo "[bazel] enabling caching with GCS buckets"
+
+  BAZEL_REGION="us-central1"
+  if [[ "$(curl -is metadata.google.internal || true)" ]]; then
+    # projects/1003139005402/zones/us-central1-a -> us-central1-a -> us-central1
+    BAZEL_REGION=$(curl -sH Metadata-Flavor:Google http://metadata.google.internal/computeMetadata/v1/instance/zone | rev | cut -d'/' -f1 | cut -c3- | rev)
+  fi
+
+  BAZEL_BUCKET="kibana-ci-bazel_$BAZEL_REGION"
+
+  echo "[bazel] using GCS bucket: $BAZEL_BUCKET"
+
+cat <<EOF >> $KIBANA_DIR/.bazelrc
+  build --remote_cache=https://storage.googleapis.com/$BAZEL_BUCKET
+  build --google_default_credentials
+EOF
+fi
+
+if [[ "$BAZEL_CACHE_MODE" == "buildbuddy" ]]; then
+  echo "[bazel] enabling caching with Buildbuddy"
 cat <<EOF >> $KIBANA_DIR/.bazelrc
   build --bes_results_url=https://app.buildbuddy.io/invocation/
   build --bes_backend=grpcs://remote.buildbuddy.io
@@ -22,14 +43,7 @@ cat <<EOF >> $KIBANA_DIR/.bazelrc
 EOF
 fi
 
-if [[ "${BAZEL_CACHE_MODE:-none}" == "read" ]]; then
-  echo "[bazel] cache set to read-only"
-cat <<EOF >> $KIBANA_DIR/.bazelrc
-  build --noremote_upload_local_results
-EOF
-fi
-
-if [[ "${BAZEL_CACHE_MODE:-none}" != @(read|read-write|none|) ]]; then
-  echo "invalid value for BAZEL_CACHE_MODE received ($BAZEL_CACHE_MODE), expected one of [read,read-write,none]"
+if [[ "$BAZEL_CACHE_MODE" != @(gcs|buildbuddy|none|) ]]; then
+  echo "invalid value for BAZEL_CACHE_MODE received ($BAZEL_CACHE_MODE), expected one of [gcs,buildbuddy,none]"
   exit 1
 fi

.buildkite/scripts/lifecycle/pre_command.sh

@@ -9,21 +9,7 @@ export BUILDKITE_TOKEN
 
 echo '--- Install buildkite dependencies'
 cd '.buildkite'
-
-# If this yarn install is terminated early, e.g. if the build is cancelled in buildkite,
-# A node module could end up in a bad state that can cause all future builds to fail
-# So, let's cache clean and try again to make sure that's not what caused the error
-install_deps() {
-  yarn install --production --pure-lockfile
-  EXIT=$?
-  if [[ "$EXIT" != "0" ]]; then
-    yarn cache clean
-  fi
-  return $EXIT
-}
-
-retry 5 15 install_deps
-
+retry 5 15 npm ci
 cd ..
 
 echo '--- Agent Debug/SSH Info'

.buildkite/scripts/pipelines/pull_request/pipeline.js

@@ -20,6 +20,7 @@ const SKIPPABLE_PATHS = [
   /^\.github\//,
   /\.md$/,
   /^\.backportrc\.json$/,
+  /^nav-kibana-dev\.docnav\.json$/,
 ];
 
 const REQUIRED_PATHS = [
@@ -64,25 +65,39 @@ const uploadPipeline = (pipelineContent) => {
 
     if (
       (await doAnyChangesMatch([
-        /^x-pack\/plugins\/security_solution/,
         /^x-pack\/plugins\/lists/,
+        /^x-pack\/plugins\/security_solution/,
         /^x-pack\/plugins\/timelines/,
-        /^x-pack\/test\/security_solution_cypress/,
         /^x-pack\/plugins\/triggers_actions_ui\/public\/application\/sections\/action_connector_form/,
         /^x-pack\/plugins\/triggers_actions_ui\/public\/application\/context\/actions_connectors_context\.tsx/,
+        /^x-pack\/test\/security_solution_cypress/,
       ])) ||
       process.env.GITHUB_PR_LABELS.includes('ci:all-cypress-suites')
     ) {
       pipeline.push(getPipeline('.buildkite/pipelines/pull_request/security_solution.yml'));
     }
 
     if (
-      (await doAnyChangesMatch([/^x-pack\/plugins\/cases/])) ||
+      (await doAnyChangesMatch([
+        /^src\/plugins\/data/,
+        /^x-pack\/plugins\/actions/,
+        /^x-pack\/plugins\/alerting/,
+        /^x-pack\/plugins\/event_log/,
+        /^x-pack\/plugins\/rule_registry/,
+        /^x-pack\/plugins\/task_manager/,
+      ])) ||
       process.env.GITHUB_PR_LABELS.includes('ci:all-cypress-suites')
     ) {
       pipeline.push(getPipeline('.buildkite/pipelines/pull_request/response_ops.yml'));
     }
 
+    if (
+      (await doAnyChangesMatch([/^x-pack\/plugins\/cases/])) ||
+      process.env.GITHUB_PR_LABELS.includes('ci:all-cypress-suites')
+    ) {
+      pipeline.push(getPipeline('.buildkite/pipelines/pull_request/response_ops_cases.yml'));
+    }
+
     if (
       (await doAnyChangesMatch([/^x-pack\/plugins\/apm/])) ||
       process.env.GITHUB_PR_LABELS.includes('ci:all-cypress-suites')

.buildkite/scripts/steps/artifacts/publish.sh

@@ -26,6 +26,7 @@ download "kibana-$FULL_VERSION-x86_64.rpm"
 download "kibana-$FULL_VERSION-aarch64.rpm"
 
 download "kibana-$FULL_VERSION-docker-build-context.tar.gz"
+download "kibana-cloud-$FULL_VERSION-docker-build-context.tar.gz"
 download "kibana-ironbank-$FULL_VERSION-docker-build-context.tar.gz"
 download "kibana-ubi8-$FULL_VERSION-docker-build-context.tar.gz"
 

.buildkite/scripts/steps/bazel_cache/bootstrap_mac.sh

@@ -2,7 +2,7 @@
 
 set -euo pipefail
 
-export BAZEL_CACHE_MODE=read-write
+export BAZEL_CACHE_MODE=buildbuddy
 export DISABLE_BOOTSTRAP_VALIDATION=true
 
 # Since our Mac agents are currently static,

.buildkite/scripts/steps/checks.sh

@@ -8,6 +8,7 @@ export DISABLE_BOOTSTRAP_VALIDATION=false
 .buildkite/scripts/steps/checks/commit/commit.sh
 .buildkite/scripts/steps/checks/bazel_packages.sh
 .buildkite/scripts/steps/checks/telemetry.sh
+.buildkite/scripts/steps/checks/validate_ci_groups.sh
 .buildkite/scripts/steps/checks/ts_projects.sh
 .buildkite/scripts/steps/checks/jest_configs.sh
 .buildkite/scripts/steps/checks/doc_api_changes.sh

.buildkite/scripts/steps/checks/validate_ci_groups.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+
+echo --- Ensure that all tests are in a CI Group
+checks-reporter-with-killswitch "Ensure that all tests are in a CI Group" \
+  node scripts/ensure_all_tests_in_ci_group

.buildkite/scripts/steps/cloud/deploy.json

@@ -207,7 +207,7 @@
               "zone_count": 1,
               "size": {
                 "resource": "memory",
-                "value": 1024
+                "value": 2048
               }
             }
           ],

.buildkite/scripts/steps/functional/response_ops.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/steps/functional/common.sh
+
+export JOB=kibana-security-solution-chrome
+
+echo "--- Response Ops Cypress Tests on Security Solution"
+
+cd "$XPACK_DIR"
+
+checks-reporter-with-killswitch "Response Ops Cypress Tests on Security Solution" \
+ node scripts/functional_tests \
+   --debug --bail \
+   --kibana-install-dir "$KIBANA_BUILD_LOCATION" \
+   --config test/security_solution_cypress/response_ops_cli_config.ts

.buildkite/scripts/steps/functional/synthetics.sh

@@ -14,4 +14,4 @@ echo "--- Uptime @elastic/synthetics Tests"
 cd "$XPACK_DIR"
 
 checks-reporter-with-killswitch "Uptime @elastic/synthetics Tests" \
-  node plugins/uptime/scripts/e2e.js --kibana-install-dir "$KIBANA_BUILD_LOCATION" --grep "MonitorManagement-monitor*"
+  node plugins/synthetics/scripts/e2e.js --kibana-install-dir "$KIBANA_BUILD_LOCATION" --grep "MonitorManagement-monitor*"

.buildkite/scripts/steps/functional/uptime.sh

@@ -13,5 +13,5 @@ echo "--- Uptime @elastic/synthetics Tests"
 
 cd "$XPACK_DIR"
 
-checks-reporter-with-killswitch "Uptime @elastic/synthetics Tests" \
-  node plugins/uptime/scripts/e2e.js --kibana-install-dir "$KIBANA_BUILD_LOCATION" ${GREP:+--grep \"${GREP}\"}
+checks-reporter-with-killswitch "uptime plugin @elastic/synthetics Tests" \
+  node plugins/synthetics/scripts/e2e.js --kibana-install-dir "$KIBANA_BUILD_LOCATION" ${GREP:+--grep \"${GREP}\"}

.buildkite/scripts/steps/on_merge_ts_refs_api_docs.sh

@@ -2,7 +2,7 @@
 
 set -euo pipefail
 
-export BAZEL_CACHE_MODE=read-write # Populate bazel remote cache for linux
+export BAZEL_CACHE_MODE=buildbuddy # Populate Buildbuddy bazel remote cache for linux
 export BUILD_TS_REFS_CACHE_ENABLE=true
 export BUILD_TS_REFS_CACHE_CAPTURE=true
 export DISABLE_BOOTSTRAP_VALIDATION=true

.buildkite/scripts/steps/storybooks/build_and_upload.js

@@ -20,7 +20,7 @@ const STORYBOOKS = [
   'custom_integrations',
   'dashboard_enhanced',
   'dashboard',
-  'data_enhanced',
+  'data',
   'embeddable',
   'expression_error',
   'expression_image',