[Elastic Agent] Fix merging of fleet.yml. Add --staging to enroll cmd. (

#20026) (#20031)

* Fix usage of merged elastic-agent.yml and fleet.yml. Add --staging command line option to enroll.

* Add to docs.

* Add changelog.

* Fix import sorting.

(cherry picked from commit cfbd81b)

x-pack/elastic-agent/CHANGELOG.asciidoc

@@ -51,6 +51,7 @@
 - Forward revision number of the configuration to the endpoint. {pull}19759[19759]
 - Remove support for logs type and use logfile {pull}19761[19761]
 - Avoid comparing uncomparable types on enroll {issue}19976[19976]
+- Fix issues with merging of elastic-agent.yml and fleet.yml {pull}20026[20026]
 
 ==== New features
 
@@ -89,3 +90,4 @@
 - Agent now sends its own logs to elasticsearch {pull}19811[19811]
 - Add --insecure option to enroll command {pull}19900[19900]
 - Will retry to enroll if the server return a 429. {pull}19918[19811]
+- Add --staging option to enroll command {pull}20026[20026]

x-pack/elastic-agent/docs/elastic-agent-command-line.asciidoc

@@ -38,3 +38,6 @@ Force overwrite the current and do not prompt for confirmation.
 
 `--insecure`::
 Allow insecure connection to Kibana.
+
+`--staging`::
+Configures agent to download artifacts from a staging build.

x-pack/elastic-agent/pkg/agent/application/enroll_cmd.go

@@ -22,6 +22,7 @@ import (
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/fleetapi"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/kibana"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release"
 )
 
 type store interface {
@@ -64,6 +65,7 @@ type EnrollCmdOption struct {
 	Insecure             bool
 	UserProvidedMetadata map[string]interface{}
 	EnrollAPIKey         string
+	Staging              string
 }
 
 func (e *EnrollCmdOption) kibanaConfig() (*kibana.Config, error) {
@@ -173,11 +175,19 @@ func (c *EnrollCmd) Execute() error {
 	}
 
 	fleetConfig, err := createFleetConfigFromEnroll(resp.Item.AccessAPIKey, c.kibanaConfig)
+	agentConfig := map[string]interface{}{
+		"id": resp.Item.ID,
+	}
+	if c.options.Staging != "" {
+		staging := fmt.Sprintf("https://staging.elastic.co/%s-%s/downloads/", release.Version(), c.options.Staging[:8])
+		agentConfig["download"] = map[string]interface{}{
+			"sourceURI": staging,
+		}
+	}
+
 	configToStore := map[string]interface{}{
 		"fleet": fleetConfig,
-		"agent": map[string]interface{}{
-			"id": resp.Item.ID,
-		},
+		"agent": agentConfig,
 	}
 
 	reader, err := yamlToReader(configToStore)

x-pack/elastic-agent/pkg/agent/application/managed_mode.go

@@ -85,9 +85,15 @@ func newManaged(
 	}
 
 	// merge local configuration and configuration persisted from fleet.
-	rawConfig.Merge(config)
+	err = rawConfig.Merge(config)
+	if err != nil {
+		return nil, errors.New(err,
+			fmt.Sprintf("fail to merge configuration with %s for the elastic-agent", path),
+			errors.TypeConfig,
+			errors.M(errors.MetaKeyPath, path))
+	}
 
-	cfg, err := configuration.NewFromConfig(config)
+	cfg, err := configuration.NewFromConfig(rawConfig)
 	if err != nil {
 		return nil, errors.New(err,
 			fmt.Sprintf("fail to unpack configuration from %s", path),

x-pack/elastic-agent/pkg/agent/cmd/enroll.go

@@ -44,6 +44,7 @@ func newEnrollCommandWithArgs(flags *globalFlags, _ []string, streams *cli.IOStr
 	cmd.Flags().StringP("ca-sha256", "p", "", "Comma separated list of certificate authorities hash pins used for certificate verifications")
 	cmd.Flags().BoolP("force", "f", false, "Force overwrite the current and do not prompt for confirmation")
 	cmd.Flags().BoolP("insecure", "i", false, "Allow insecure connection to Kibana")
+	cmd.Flags().StringP("staging", "", "", "Configures agent to download artifacts from a staging build")
 
 	return cmd
 }
@@ -67,6 +68,13 @@ func enroll(streams *cli.IOStreams, cmd *cobra.Command, flags *globalFlags, args
 			errors.M(errors.MetaKeyPath, pathConfigFile))
 	}
 
+	staging, _ := cmd.Flags().GetString("staging")
+	if staging != "" {
+		if len(staging) < 8 {
+			return errors.New(fmt.Errorf("invalid staging build hash; must be at least 8 characters"), "Error")
+		}
+	}
+
 	force, _ := cmd.Flags().GetBool("force")
 	if !force {
 		confirm, err := c.Confirm("This will replace your current settings. Do you want to continue?", true)
@@ -105,6 +113,7 @@ func enroll(streams *cli.IOStreams, cmd *cobra.Command, flags *globalFlags, args
 		CASha256:             caSHA256,
 		Insecure:             insecure,
 		UserProvidedMetadata: make(map[string]interface{}),
+		Staging:              staging,
 	}
 
 	c, err := application.NewEnrollCmd(