Merge branch 'main' into issue-10111-awshealth

.buildkite/hooks/pre-command

@@ -142,11 +142,18 @@ fi
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "integrations-backport" ]]; then
   if [[ "$BUILDKITE_STEP_KEY" == "create-backport-branch" ]]; then
-    GITHUB_USERNAME_SECRET=$(retry 5 vault kv get -field username ${GITHUB_TOKEN_VAULT_PATH})
+    GITHUB_USERNAME_SECRET=$(retry 5 vault kv get -field username "${GITHUB_TOKEN_VAULT_PATH}")
     export GITHUB_USERNAME_SECRET
-    GITHUB_EMAIL_SECRET=$(retry 5 vault kv get -field email ${GITHUB_TOKEN_VAULT_PATH})
+    GITHUB_EMAIL_SECRET=$(retry 5 vault kv get -field email "${GITHUB_TOKEN_VAULT_PATH}")
     export GITHUB_EMAIL_SECRET
     GITHUB_TOKEN=$(retry 5 vault kv get -field token "${GITHUB_TOKEN_VAULT_PATH}")
     export GITHUB_TOKEN
   fi
 fi
+
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "integrations" || "$BUILDKITE_PIPELINE_SLUG" == "integrations-serverless" ]]; then
+  if [[ "$BUILDKITE_STEP_KEY" == "report-failed-tests" ]]; then
+    export GITHUB_TOKEN="${VAULT_GITHUB_TOKEN}"
+  fi
+fi
+

.buildkite/pipeline.schedule-daily.yml

@@ -2,7 +2,7 @@
 name: integrations-schedule-daily
 
 env:
-  SETUP_GVM_VERSION: "v0.5.1"
+  SETUP_GVM_VERSION: "v0.5.2"
   LINUX_AGENT_IMAGE: "golang:${GO_VERSION}"
 
 # The pipeline is triggered by the scheduler every day
@@ -28,14 +28,14 @@ steps:
       - step: "check"
         allow_failure: false
 
-  - label: "Check integrations local stacks - Stack Version v8.14"
+  - label: "Check integrations local stacks - Stack Version v8.15"
     trigger: "integrations"
     build:
       env:
         SERVERLESS: "false"
         SKIP_PUBLISHING: "true"
         FORCE_CHECK_ALL: "true"
-        STACK_VERSION: 8.14.0-SNAPSHOT
+        STACK_VERSION: 8.15.0-SNAPSHOT
         PUBLISH_COVERAGE_REPORTS: "true"
     depends_on:
       - step: "check"

.buildkite/pipeline.serverless.yml

@@ -4,11 +4,14 @@ env:
   SETUP_GVM_VERSION: "v0.5.2"
   LINUX_AGENT_IMAGE: "golang:${GO_VERSION}"
   DOCKER_COMPOSE_VERSION: "v2.24.1"
-  DOCKER_VERSION: "false"
+  DOCKER_VERSION: "false" # not required to set since system tests are not running yet
   KIND_VERSION: 'v0.20.0'
-  K8S_VERSION: 'v1.29.0'
+  K8S_VERSION: 'v1.30.0'
   YQ_VERSION: 'v4.35.2'
   IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204"
+  GH_CLI_VERSION: "2.29.0"
+  # This pipeline is intended to test packages with Elastic Serverless
+  SERVERLESS: true
 
   # Elastic package settings
   # Manage docker output/logs
@@ -49,9 +52,8 @@ steps:
   - label: "Check integrations in serverless"
     key: "test-integrations-serverless-project"
     command: ".buildkite/scripts/test_integrations_with_serverless.sh"
-    timeout_in_minutes: 120
+    timeout_in_minutes: 240
     env:
-      SERVERLESS: true
       FORCE_CHECK_ALL: true
       UPLOAD_SAFE_LOGS: 1
     agents:
@@ -74,3 +76,16 @@ steps:
           artifacts: "build/test-results/*.xml"
     agents:
       provider: "gcp"  # junit plugin requires docker
+
+  - label: ":github: Report failed tests"
+    key: report-failed-tests
+    command: ".buildkite/scripts/report_issues.sh"
+    agents:
+      image: "${LINUX_AGENT_IMAGE}"
+      cpu: "8"
+      memory: "4G"
+    # not fail build if this step fails
+    soft_fail: true
+    # run this step when if it is triggered by the daily job
+    if: >
+      build.source == "trigger_job" &&  build.env('BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG') == "integrations-schedule-daily"

.buildkite/pipeline.yml

@@ -5,7 +5,7 @@ env:
   DOCKER_COMPOSE_VERSION: "v2.24.1"
   DOCKER_VERSION: "26.1.2"
   KIND_VERSION: 'v0.20.0'
-  K8S_VERSION: 'v1.29.0'
+  K8S_VERSION: 'v1.30.0'
   YQ_VERSION: 'v4.35.2'
   JQ_VERSION: '1.7'
   GH_CLI_VERSION: "2.29.0"
@@ -53,6 +53,7 @@ steps:
       memory: "4G"
 
   - label: ":sonarqube: Continuous Code Inspection"
+    soft_fail: true # FIXME: Coverage is failing, remove this after solving the issue
     env:
       VAULT_SONAR_TOKEN_PATH: "kv/ci-shared/platform-ingest/elastic/integrations/sonar-analyze-token"
     agents:
@@ -68,3 +69,16 @@ steps:
           failed-download-exit-code: 0 # Not fail the build in case there are no XML files
     agents:
       provider: "gcp"  # junit plugin requires docker
+
+  - label: ":github: Report failed tests"
+    key: report-failed-tests
+    command: ".buildkite/scripts/report_issues.sh"
+    agents:
+      image: "${LINUX_AGENT_IMAGE}"
+      cpu: "8"
+      memory: "4G"
+    # not fail build if this step fails
+    soft_fail: true
+    # run this step when if it is triggered by the daily job
+    if: >
+      build.source == "trigger_job" &&  build.env('BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG') == "integrations-schedule-daily"

.buildkite/scripts/check_sources.sh

@@ -7,6 +7,6 @@ set -euo pipefail
 add_bin_path
 with_mage
 
-mage -debug check
+mage -v check
 
 check_git_diff

.buildkite/scripts/report_issues.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+source .buildkite/scripts/common.sh
+
+set -euo pipefail
+
+junit_folder="test-results"
+test_results_folder="build/${junit_folder}"
+buildkite_pattern="${test_results_folder}/*.xml"
+
+download_test_results() {
+    mkdir -p "${test_results_folder}"
+
+    if ! buildkite-agent artifact download "${buildkite_pattern}" . ; then
+        message="Could not download XML artifacts. Skip creating issues."
+        echo "--- :boom: ${message}"
+        buildkite-agent annotate \
+            "[Report Failed Tests] ${message}" \
+            --context "ctx-report-failed-tests-no-files" \
+            --style "warning"
+        return 1
+    fi
+    return 0
+}
+
+if running_on_buildkite ; then
+    echo "--- Installing tools"
+    add_bin_path
+    with_mage
+    with_github_cli # to list, create and update issues
+
+    echo "--- Download Test Results"
+    if ! download_test_results ; then
+        exit 0
+    fi
+fi
+
+echo "--- Create GitHub Issues for failed tests"
+mage -v ReportFailedTests build/test-results
+

.buildkite/scripts/run_sonar_scanner.sh

@@ -1,6 +1,12 @@
 #!/bin/bash
+
+source .buildkite/scripts/common.sh
+
 set -euo pipefail
 
+add_bin_path
+with_mage
+
 run_sonar_scanner() {
     local message=""
     echo "--- Download coverage reports and merge them"
@@ -15,7 +21,7 @@ run_sonar_scanner() {
     fi
 
     echo "Merge all coverage reports"
-    .buildkite/scripts/merge_xml.sh
+    mage mergeCoverage
 
     echo "--- Execute sonar scanner CLI"
     /scan-source-code.sh

.github/CODEOWNERS

@@ -320,6 +320,7 @@
 /packages/system_audit @elastic/sec-linux-platform
 /packages/tanium @elastic/security-service-integrations
 /packages/tcp @elastic/sec-deployment-and-devices
+/packages/teleport @elastic/security-service-integrations
 /packages/tenable_io @elastic/security-service-integrations
 /packages/tenable_sc @elastic/security-service-integrations
 /packages/thycotic_ss @elastic/security-service-integrations

CONTRIBUTING.md

@@ -1,15 +1,10 @@
 # Contributing Guide
 
-This page is intended for contributors to the [Package Registry](https://github.com/elastic/package-registry/) and Elastic Integrations.
+Ready to dive into the world of integrations? See the [Integrations Developer Guide](https://www.elastic.co/guide/en/integrations-developer/current/index.html) to get started. Topics include:
 
-## Table of Contents
-
-* [Definitions](./docs/definitions.md) - learn basic terms used in the universe of integrations, packages, data streams
-* [Generic guidelines](./docs/generic_guidelines.md) - generic guidelines for developing integrations, compliance with ECS, docs, dashboards
-* [Sample package: Nginx](./packages/nginx) - use as an inspiration for new packages, look at files, folders, test resources
-* [Developer workflow: build and test integration](./docs/developer_workflow_design_build_test_integration.md) - step-by-step guide on how to build and test an integration
-* [Developer workflow: bug fix an old package version](./docs/developer_workflow_bug_fix_older_package_version.md) - step-by-step guide on how to release a new fix for an old package version.
-* [Fine-tune integration](./docs/fine_tune_integration.md) - fill missing items, correct structure, review manifests
-* [Testing and validation](./docs/testing_and_validation.md) - run the Elastic stack, use test runners, review test coverage
-* [Tips for building integrations](./docs/tips_for_building_integrations.md) - see your local changes in Kibana, follow best practices
-* [Documentation Guidelines](./docs/documentation_guidelines.md) - guidelines for contributing to the docs, the Generic guidelines also has lots of docs tips
+* [Quick start: Sample integration](https://www.elastic.co/guide/en/integrations-developer/current/quick-start.html)
+* [Build an integration](https://www.elastic.co/guide/en/integrations-developer/current/build-a-new-integration.html)
+* [Upload an integration](https://www.elastic.co/guide/en/integrations-developer/current/upload-a-new-integration.html)
+* [Test an integration](https://www.elastic.co/guide/en/integrations-developer/current/testing.html)
+* [Publish an integration](https://www.elastic.co/guide/en/integrations-developer/current/_publish_an_integration.html)
+* [Developer workflows](https://www.elastic.co/guide/en/integrations-developer/current/developer-workflows.html)

dev/codeowners/codeowners.go

@@ -12,7 +12,7 @@ import (
 	"path/filepath"
 	"strings"
 
-	"gopkg.in/yaml.v2"
+	"gopkg.in/yaml.v3"
 )
 
 const (
@@ -41,6 +41,33 @@ func Check() error {
 	})
 }
 
+func PackageOwners(packageName, dataStream string) ([]string, error) {
+	return PackageOwnersCustomCodeowners(packageName, dataStream, codeownersPath)
+}
+
+func PackageOwnersCustomCodeowners(packageName, dataStream, codeownersPath string) ([]string, error) {
+	owners, err := readGithubOwners(codeownersPath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read CODEOWNERS file: %w", err)
+	}
+	packagePath := fmt.Sprintf("/packages/%s", packageName)
+	packageTeams, found := owners.owners[packagePath]
+	if !found {
+		return nil, fmt.Errorf("no owner found for package %s", packageName)
+	}
+
+	if dataStream == "" {
+		return packageTeams, nil
+	}
+
+	dataStreamPath := fmt.Sprintf("/packages/%s/data_stream/%s", packageName, dataStream)
+	dataStreamTeams, found := owners.owners[dataStreamPath]
+	if !found {
+		return packageTeams, nil
+	}
+	return dataStreamTeams, nil
+}
+
 type githubOwners struct {
 	owners map[string][]string
 	path   string

dev/codeowners/codeowners_test.go

@@ -116,3 +116,67 @@ func TestReadGithubOwners(t *testing.T) {
 		})
 	}
 }
+
+func TestReturnPackageOwners(t *testing.T) {
+	cases := []struct {
+		title          string
+		codeownersPath string
+		packageName    string
+		datastream     string
+		expected       []string
+		expectedError  bool
+	}{
+		{
+			title:          "just package",
+			codeownersPath: "testdata/CODEOWNERS-owners-packages-datastreams",
+			packageName:    "aws",
+			datastream:     "",
+			expected:       []string{"@elastic/obs-infraobs-integrations", "@elastic/obs-ds-hosted-services", "@elastic/security-service-integrations"},
+			expectedError:  false,
+		},
+		{
+			title:          "package and datastream",
+			codeownersPath: "testdata/CODEOWNERS-owners-packages-datastreams",
+			packageName:    "aws",
+			datastream:     "cloudtrail",
+			expected:       []string{"@elastic/obs-infraobs-integrations"},
+			expectedError:  false,
+		},
+		{
+			title:          "package and other datastream",
+			codeownersPath: "testdata/CODEOWNERS-owners-packages-datastreams",
+			packageName:    "aws",
+			datastream:     "cloudwatch_logs",
+			expected:       []string{"@elastic/obs-ds-hosted-services"},
+			expectedError:  false,
+		},
+		{
+			title:          "package not found",
+			codeownersPath: "testdata/CODEOWNERS-owners-packages-datastreams",
+			packageName:    "other",
+			datastream:     "",
+			expected:       []string{},
+			expectedError:  true,
+		},
+		{
+			title:          "package found but not data stream defined",
+			codeownersPath: "testdata/CODEOWNERS-owners-packages-datastreams",
+			packageName:    "aws",
+			datastream:     "other",
+			expected:       []string{"@elastic/obs-infraobs-integrations", "@elastic/obs-ds-hosted-services", "@elastic/security-service-integrations"},
+			expectedError:  false,
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.title, func(t *testing.T) {
+			owners, err := PackageOwnersCustomCodeowners(c.packageName, c.datastream, c.codeownersPath)
+			if c.expectedError {
+				assert.Error(t, err)
+				return
+			}
+			require.NoError(t, err)
+			assert.Equal(t, c.expected, owners)
+		})
+	}
+}

dev/codeowners/testdata/CODEOWNERS-owners-packages-datastreams

@@ -0,0 +1,10 @@
+# This is a valid test file
+* @elastic/ecosystem
+
+/packages/aws @elastic/obs-infraobs-integrations @elastic/obs-ds-hosted-services @elastic/security-service-integrations
+/packages/aws/data_stream/apigateway_logs @elastic/obs-infraobs-integrations
+/packages/aws/data_stream/apigateway_metrics @elastic/obs-infraobs-integrations
+/packages/aws/data_stream/billing @elastic/obs-infraobs-integrations
+/packages/aws/data_stream/cloudfront_logs @elastic/obs-infraobs-integrations
+/packages/aws/data_stream/cloudtrail @elastic/obs-infraobs-integrations
+/packages/aws/data_stream/cloudwatch_logs @elastic/obs-ds-hosted-services