Merge remote-tracking branch 'upstream/main' into system-benchmarks-i…

…mprovements

.buildkite/pipeline.yml

@@ -3,8 +3,8 @@ env:
   DOCKER_COMPOSE_VERSION: "v2.17.2"
   ELASTIC_PACKAGE_COMPOSE_DISABLE_ANSI: "true"
   ELASTIC_PACKAGE_COMPOSE_DISABLE_PULL_PROGRESS_INFORMATION: "true"
-  KIND_VERSION: 'v0.17.0'
-  K8S_VERSION: 'v1.26.0'
+  KIND_VERSION: 'v0.20.0'
+  K8S_VERSION: 'v1.27.3'
 
 steps:
   - label: ":go: Run check-static"

README.md

@@ -38,7 +38,26 @@ elastic-package help
 
 ## Development
 
-Download and build the latest main of `elastic-package` binary:
+Even though the project is "go-gettable", there is the [`Makefile`](./Makefile) present, which can be used to build,
+install, format the source code among others. Some examples of the available targets are:
+
+`make build` - build the tool source
+
+`make clean` - delete elastic-package binary and build folder
+
+`make format` - format the Go code
+
+`make check` - one-liner, used by CI to verify if source code is ready to be pushed to the repository
+
+`make install` - build the tool source and move binary to `$GOBIN`
+
+`make gomod` - ensure go.mod and go.sum are up to date
+
+`make update` - update README.md file
+
+`make licenser` - add the Elastic license header in the source code
+
+To start developing, download and build the latest main of `elastic-package` binary:
 
 ```bash
 git clone https://github.com/elastic/elastic-package.git
@@ -61,6 +80,40 @@ git rm --cached -r .
 git reset --hard
 ```
 
+### Testing with integrations repository
+
+While working on a new branch, it is interesting to test these changes
+with all the packages defined in the [integrations repository](https://github.com/elastic/integrations).
+This allows to test a much wider scenarios than the test packages that are defined in this repository.
+
+This test can be triggered automatically directly from your Pull Request by adding a comment `test integrations`. Example:
+- Comment: https://github.com/elastic/elastic-package/pull/1335#issuecomment-1619721861
+- Pull Request created in integrations repository: https://github.com/elastic/integrations/pull/6756
+
+This comment triggers this [Buildkite pipeline](https://github.com/elastic/elastic-package/blob/6f084e21561105ac9773acab00c3439251f111a0/.buildkite/pipeline.test-with-integrations-repo.yml) ([Buildkite job](https://buildkite.com/elastic/elastic-package-test-with-integrations)).
+
+This pipeline creates a new draft Pull Request in integration updating the required dependencies to test your own changes. As a new pull request is created, a CI
+job will be triggered to test all the packages defined in this repository. A new comment with the link to this new Pull Request will be posted in your package-spec Pull Request.
+
+**IMPORTANT**: Remember to close this PR in the integrations repository once you close the package-spec Pull Request.
+
+Usually, this process would require the following manual steps:
+1. Create your elastic-package pull request and push all your commits
+2. Get the SHA of the latest changeset of your PR:
+   ```bash
+    $ git show -s --pretty=format:%H
+   1131866bcff98c29e2c84bcc1c772fff4307aaca
+   ```
+3. Go to the integrations repository, and update go.mod and go.sum with that changeset:
+   ```bash
+   cd /path/to/integrations/repostiory
+   go mod edit -replace github.com/elastic/elastic-package=github.com/<your_github_user>/elastic-package@1131866bcff98c29e2c84bcc1c772fff4307aaca
+   go mod tidy
+   ```
+4. Push these changes into a branch and create a Pull Request
+    - Creating this PR would automatically trigger a new Jenkins pipeline.
+
+
 ## Commands
 
 `elastic-package` currently offers the commands listed below.
@@ -512,21 +565,6 @@ The following settings are available per profile:
   Elasticsearch in stacks managed by elastic-package. It is recommended to use
   an absolute path, out of the `.elastic-package` directory.
 
-## Development
-
-Even though the project is "go-gettable", there is the `Makefile` present, which can be used to build, format or vendor
-source code:
-
-`make build` - build the tool source
-
-`make format` - format the Go code
-
-`make install` - build the tool source and move binary to `$GOBIN`
-
-`make vendor` - vendor code of dependencies
-
-`make check` - one-liner, used by CI to verify if source code is ready to be pushed to the repository
-
 ## Release process
 
 This project uses [GoReleaser](https://goreleaser.com/) to release a new version of the application (semver). Release publishing

cmd/benchmark.go

@@ -15,6 +15,7 @@ import (
 	"github.com/dustin/go-humanize"
 
 	"github.com/elastic/elastic-package/internal/corpusgenerator"
+	"github.com/elastic/elastic-package/internal/install"
 	"github.com/elastic/elastic-package/internal/kibana"
 	"github.com/elastic/elastic-package/internal/logger"
 
@@ -60,6 +61,8 @@ func setupBenchmarkCommand() *cobraext.Command {
 		Long:  benchLongDescription,
 	}
 
+	cmd.PersistentFlags().StringP(cobraext.ProfileFlagName, "p", "", fmt.Sprintf(cobraext.ProfileFlagDescription, install.ProfileNameEnvVar))
+
 	pipelineCmd := getPipelineCommand()
 	cmd.AddCommand(pipelineCmd)
 
@@ -257,6 +260,11 @@ func systemCommandAction(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("locating package root failed: %w", err)
 	}
 
+	profile, err := cobraext.GetProfileFlag(cmd)
+	if err != nil {
+		return err
+	}
+
 	signal.Enable()
 
 	esClient, err := elasticsearch.NewClient()
@@ -282,6 +290,7 @@ func systemCommandAction(cmd *cobra.Command, args []string) error {
 		system.WithPackageRootPath(packageRootPath),
 		system.WithESAPI(esClient.API),
 		system.WithKibanaClient(kc),
+		system.WithProfile(profile),
 	}
 
 	esMetricsClient, err := initializeESMetricsClient(cmd.Context())

cmd/service.go

@@ -12,6 +12,7 @@ import (
 	"github.com/spf13/cobra"
 
 	"github.com/elastic/elastic-package/internal/cobraext"
+	"github.com/elastic/elastic-package/internal/install"
 	"github.com/elastic/elastic-package/internal/packages"
 	"github.com/elastic/elastic-package/internal/service"
 	"github.com/elastic/elastic-package/internal/testrunner/runners/system"
@@ -36,6 +37,7 @@ func setupServiceCommand() *cobraext.Command {
 		Long:  serviceLongDescription,
 	}
 	cmd.AddCommand(upCommand)
+	cmd.PersistentFlags().StringP(cobraext.ProfileFlagName, "p", "", fmt.Sprintf(cobraext.ProfileFlagDescription, install.ProfileNameEnvVar))
 
 	return cobraext.NewCommand(cmd, cobraext.ContextPackage)
 }
@@ -59,8 +61,14 @@ func upCommandAction(cmd *cobra.Command, args []string) error {
 
 	variantFlag, _ := cmd.Flags().GetString(cobraext.VariantFlagName)
 
+	profile, err := cobraext.GetProfileFlag(cmd)
+	if err != nil {
+		return err
+	}
+
 	_, serviceName := filepath.Split(packageRoot)
 	err = service.BootUp(service.Options{
+		Profile:            profile,
 		ServiceName:        serviceName,
 		PackageRootPath:    packageRoot,
 		DevDeployDir:       system.DevDeployDir,

cmd/testrunner.go

@@ -226,14 +226,14 @@ func testTypeCommandActionFactory(runner testrunner.TestRunner) cobraext.Command
 		var results []testrunner.TestResult
 		for _, folder := range testFolders {
 			r, err := testrunner.Run(testType, testrunner.TestOptions{
+				Profile:            profile,
 				TestFolder:         folder,
 				PackageRootPath:    packageRootPath,
 				GenerateTestResult: generateTestResult,
 				API:                esClient.API,
 				DeferCleanup:       deferCleanup,
 				ServiceVariant:     variantFlag,
 				WithCoverage:       testCoverage,
-				Profile:            profile,
 			})
 
 			results = append(results, r...)

go.mod

@@ -5,7 +5,7 @@ go 1.20
 require (
 	github.com/AlecAivazis/survey/v2 v2.3.7
 	github.com/Masterminds/semver/v3 v3.2.1
-	github.com/ProtonMail/gopenpgp/v2 v2.7.1
+	github.com/ProtonMail/gopenpgp/v2 v2.7.2
 	github.com/aymerick/raymond v2.0.2+incompatible
 	github.com/boumenot/gocover-cobertura v1.2.0
 	github.com/cespare/xxhash/v2 v2.2.0
@@ -15,10 +15,10 @@ require (
 	github.com/elastic/go-licenser v0.4.1
 	github.com/elastic/go-resource v0.1.1
 	github.com/elastic/go-ucfg v0.8.6
-	github.com/elastic/package-spec/v2 v2.9.0
+	github.com/elastic/package-spec/v2 v2.10.0
 	github.com/fatih/color v1.15.0
 	github.com/go-git/go-billy/v5 v5.4.1
-	github.com/go-git/go-git/v5 v5.7.0
+	github.com/go-git/go-git/v5 v5.8.1
 	github.com/google/go-cmp v0.5.9
 	github.com/google/go-github/v32 v32.1.0
 	github.com/google/go-querystring v1.1.0
@@ -28,30 +28,31 @@ require (
 	github.com/mholt/archiver/v3 v3.5.1
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/pmezard/go-difflib v1.0.0
-	github.com/shirou/gopsutil/v3 v3.23.6
+	github.com/shirou/gopsutil/v3 v3.23.7
 	github.com/spf13/cobra v1.7.0
 	github.com/stretchr/testify v1.8.4
-	golang.org/x/tools v0.11.0
+	golang.org/x/tools v0.12.0
 	gopkg.in/yaml.v3 v3.0.1
 	gotest.tools/gotestsum v1.10.1
-	helm.sh/helm/v3 v3.12.1
+	helm.sh/helm/v3 v3.12.3
 	honnef.co/go/tools v0.4.3
-	k8s.io/apimachinery v0.27.3
-	k8s.io/cli-runtime v0.27.3
-	k8s.io/client-go v0.27.3
+	k8s.io/apimachinery v0.27.4
+	k8s.io/cli-runtime v0.27.4
+	k8s.io/client-go v0.27.4
 )
 
 require (
+	dario.cat/mergo v1.0.0 // indirect
 	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
 	github.com/BurntSushi/toml v1.2.1 // indirect
 	github.com/MakeNowJust/heredoc v1.0.0 // indirect
 	github.com/Masterminds/goutils v1.1.1 // indirect
 	github.com/Masterminds/sprig/v3 v3.2.3 // indirect
-	github.com/Microsoft/go-winio v0.6.0 // indirect
+	github.com/Microsoft/go-winio v0.6.1 // indirect
 	github.com/PaesslerAG/gval v1.2.1 // indirect
 	github.com/PaesslerAG/jsonpath v0.1.1 // indirect
 	github.com/Pallinder/go-randomdata v1.2.0 // indirect
-	github.com/ProtonMail/go-crypto v0.0.0-20230518184743-7afd39499903 // indirect
+	github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95 // indirect
 	github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f // indirect
 	github.com/acomagu/bufpipe v1.0.4 // indirect
 	github.com/andybalholm/brotli v1.0.4 // indirect
@@ -130,7 +131,7 @@ require (
 	github.com/sergi/go-diff v1.2.0 // indirect
 	github.com/shoenig/go-m1cpu v0.1.6 // indirect
 	github.com/shopspring/decimal v1.3.1 // indirect
-	github.com/skeema/knownhosts v1.1.1 // indirect
+	github.com/skeema/knownhosts v1.2.0 // indirect
 	github.com/spf13/cast v1.5.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/tklauser/go-sysconf v0.3.11 // indirect
@@ -144,27 +145,27 @@ require (
 	github.com/yusufpapurcu/wmi v1.2.3 // indirect
 	go.mongodb.org/mongo-driver v1.11.1 // indirect
 	go.starlark.net v0.0.0-20221205180719-3fd0dac74452 // indirect
-	golang.org/x/crypto v0.11.0 // indirect
+	golang.org/x/crypto v0.12.0 // indirect
 	golang.org/x/exp/typeparams v0.0.0-20221208152030-732eee02a75a // indirect
 	golang.org/x/mod v0.12.0 // indirect
-	golang.org/x/net v0.12.0 // indirect
+	golang.org/x/net v0.14.0 // indirect
 	golang.org/x/oauth2 v0.4.0 // indirect
 	golang.org/x/sync v0.3.0 // indirect
-	golang.org/x/sys v0.10.0 // indirect
-	golang.org/x/term v0.10.0 // indirect
-	golang.org/x/text v0.11.0 // indirect
+	golang.org/x/sys v0.11.0 // indirect
+	golang.org/x/term v0.11.0 // indirect
+	golang.org/x/text v0.12.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/protobuf v1.28.1 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	k8s.io/api v0.27.3 // indirect
-	k8s.io/apiextensions-apiserver v0.27.2 // indirect
-	k8s.io/component-base v0.27.2 // indirect
-	k8s.io/klog/v2 v2.90.1 // indirect
+	k8s.io/api v0.27.4 // indirect
+	k8s.io/apiextensions-apiserver v0.27.3 // indirect
+	k8s.io/component-base v0.27.3 // indirect
+	k8s.io/klog/v2 v2.100.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect
-	k8s.io/kubectl v0.27.2 // indirect
+	k8s.io/kubectl v0.27.3 // indirect
 	k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/kustomize/api v0.13.2 // indirect