feat: support running kibana with different configs (#1644)

* chore: match elastic-package's kibana config

* chore: bring back kibana's configuration file

This will enable having multiple config files per test suite, using the
profile abstraction: one dir per config with the profile name

* chore: rename configurations to default

It will represent the default profile

* chore: make kibana profile configurable

Default profile will be 'default'

* feat: support bootstrapping the stack with a new kibana config

The bootstrap method is called before the test suite (only once) AND before
each test scenario, as it has been defined as a Background clause. If an
scenario wants to configure a new config file for Kibana, a new profile file
with the new kibana config must be created at the compose level, using
the ProfileName as directory name.

* feat: create an on-the-fly elastic-package's profile from compose

* chore: override default kibana config for elastic-package

* fix: remove unused code

* docs: document kibana config and profiles

* chore: add a profile for preconfgured-policies

* elas

* chore: add placeholder for preconfigured policies

* chore: support passing custom kibana ref to elastic-package

* fix: honour current env

Instead of replacing it, we want to append!

* chore: unit test that we are not overriding the env

(cherry picked from commit a207500)

.ci/.e2e-tests.yaml

@@ -33,6 +33,9 @@ SUITES:
       - name: "Linux Integration"
         pullRequestFilter: " && ~debian"
         tags: "linux_integration"
+      - name: "Policies"
+        pullRequestFilter: " && ~debian"
+        tags: "policies"
       - name: "System Integration"
         platforms:
           - labels: "ubuntu-18.04"

cli/config/compose/profiles/fleet/configurations/kibana.config.yml → cli/config/compose/profiles/fleet/default/kibana.config.yml

@@ -7,14 +7,14 @@ telemetry.enabled: false
 elasticsearch.hosts: [ "http://elasticsearch:9200" ]
 elasticsearch.username: elastic
 elasticsearch.password: changeme
-monitoring.ui.container.elasticsearch.enabled: true
-
-xpack.encryptedSavedObjects.encryptionKey: "12345678901234567890123456789012"
+xpack.monitoring.ui.container.elasticsearch.enabled: true
 
 xpack.fleet.enabled: true
 xpack.fleet.registryUrl: https://epr-staging.elastic.co
 xpack.fleet.agents.enabled: true
 xpack.fleet.agents.elasticsearch.host: http://elasticsearch:9200
 xpack.fleet.agents.fleet_server.hosts:
-  - http://fleet-server:8220
+  ["http://fleet-server:8220"]
+
+xpack.encryptedSavedObjects.encryptionKey: "12345678901234567890123456789012"
 xpack.fleet.agents.tlsCheckDisabled: true

cli/config/compose/profiles/fleet/docker-compose.yml

@@ -32,21 +32,8 @@ services:
     platform: ${stackPlatform:-linux/amd64}
     ports:
       - "5601:5601"
-    environment:
-      - "SERVER_NAME=kibana"
-      - "SERVER_HOST=0.0.0.0"
-      - "TELEMETRY_ENABLED=false"
-      - "ELASTICSEARCH_HOSTS=http://elasticsearch:9200"
-      - "ELASTICSEARCH_USERNAME=elastic"
-      - "ELASTICSEARCH_PASSWORD=changeme"
-      - "MONITORING_UI_CONTAINER_ELASTICSEARCH_ENABLED=true"
-      - "XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY='12345678901234567890123456789012'"
-      - "XPACK_FLEET_ENABLED=true"
-      - "XPACK_FLEET_REGISTRYURL=https://epr-staging.elastic.co"
-      - "XPACK_FLEET_AGENTS_ENABLED=true"
-      - "XPACK_FLEET_AGENTS_ELASTICSEARCH_HOST=http://elasticsearch:9200"
-      - "XPACK_FLEET_AGENTS_FLEET_SERVER_HOSTS=[\"http://fleet-server:8220\"]"
-      - "XPACK_FLEET_AGENTS_TLSCHECKDISABLED=true"
+    volumes:
+      - ./${kibanaProfile:-default}/kibana.config.yml:/usr/share/kibana/config/kibana.yml
   fleet-server:
     image: "docker.elastic.co/beats/elastic-agent:${stackVersion:-7.15-SNAPSHOT}"
     depends_on:

cli/config/compose/profiles/fleet/preconfigured-policies/kibana.config.yml

@@ -0,0 +1,20 @@
+---
+server.name: kibana
+server.host: "0.0.0.0"
+
+telemetry.enabled: false
+
+elasticsearch.hosts: [ "http://elasticsearch:9200" ]
+elasticsearch.username: elastic
+elasticsearch.password: changeme
+xpack.monitoring.ui.container.elasticsearch.enabled: true
+
+xpack.fleet.enabled: true
+xpack.fleet.registryUrl: https://epr-staging.elastic.co
+xpack.fleet.agents.enabled: true
+xpack.fleet.agents.elasticsearch.host: http://elasticsearch:9200
+xpack.fleet.agents.fleet_server.hosts:
+  ["http://fleet-server:8220"]
+
+xpack.encryptedSavedObjects.encryptionKey: "12345678901234567890123456789012"
+xpack.fleet.agents.tlsCheckDisabled: true

cli/config/config.go

@@ -61,11 +61,6 @@ func AvailableProfiles() map[string]Profile {
 	return Op.Profiles
 }
 
-// FileExists checks if a configuration file exists
-func FileExists(configFile string) (bool, error) {
-	return io.Exists(configFile)
-}
-
 // GetServiceConfig configuration of a service
 func GetServiceConfig(service string) (Service, bool) {
 	return Op.GetServiceConfig(service)

e2e/_suites/fleet/README.md

@@ -121,6 +121,26 @@ This is an example of the optional configuration:
    OP_LOG_LEVEL=DEBUG go test -timeout 60m -v --godog.tags='@fleet_mode_agent'
    ```
 
+### Running Kibana with different configuration file
+In the case you need to run Kibana with a different set of properties, it's possible to do so simply using the `kibana uses "my-custom-profile" profile` step. This step, if executed at the very beginning of an scenario, or as a `Background` step for all scenarios, will execute the _Bootstrap_ code with the configuration located at `my-custom-profile`. As a reminder, `Bootstrap` will reevaluate the state of the runtime dependencies, recreating those that changed.
+
+In order to achieve that you have to:
+
+1. create a `kibana.config.yml` at `$E2E_ROOT_DIR/cli/config/compose/profiles/fleet/my-custom-profile/` file with your own properties. _You need to commit this file to the repository_.
+2. add the `kibana uses "my-custom-profile" profile` step in any of the following cases:
+   a. for an entire test suite adding a `Background` step like this:
+```gherkin
+Background: Setting up kibana instance with my custom profile
+  Given kibana uses "my-custom-profile" profile
+```
+
+   b. for a single test scenario adding a `Given` clause at the beginning. In this case, make sure you set the `default` profile as a `Background` so that it restores the Kibana state at the beginning of the next scenario.
+```gherkin
+Background: Setting up kibana instance with default profile
+  Given kibana uses "default" profile
+```
+3. Run the tests! Kibana will be recreated with the profile configuration in those scenarios using the new step.
+
 ### Need help?
 
 Please open an issue here: https://github.com/elastic/e2e-testing/issues/new

e2e/_suites/fleet/features/fleet_mode_agent.feature

@@ -2,6 +2,9 @@
 Feature: Fleet Mode Agent
   Scenarios for the Agent in Fleet mode connecting to Fleet application.
 
+Background: Setting up kibana instance with the default profile
+  Given kibana uses "default" profile
+
 @install
 Scenario Outline: Deploying the <os> agent
   Given a "<os>" agent is deployed to Fleet with "tar" installer

e2e/_suites/fleet/features/preconfigured_policies.feature

@@ -0,0 +1,37 @@
+@policies
+Feature: Preconfigured Policies
+  Scenarios for Preconfigured Policies
+
+Scenario Outline: Example using Kibana with custom policy
+  Given kibana uses "preconfigured-policies" profile
+    And a "<os>" agent is deployed to Fleet with "tar" installer
+  When the "elastic-agent" process is in the "started" state on the host
+  Then the agent is listed in Fleet as "online"
+    And system package dashboards are listed in Fleet
+
+@centos
+Examples: Centos
+| os     |
+| centos |
+
+@debian
+Examples: Debian
+| os     |
+| debian |
+
+Scenario Outline: Example using Kibana with default config
+  Given kibana uses "default" profile
+    And a "<os>" agent is deployed to Fleet with "tar" installer
+  When the "elastic-agent" process is in the "started" state on the host
+  Then the agent is listed in Fleet as "online"
+    And system package dashboards are listed in Fleet
+
+@centos
+Examples: Centos
+| os     |
+| centos |
+
+@debian
+Examples: Debian
+| os     |
+| debian |

e2e/_suites/fleet/fleet.go

@@ -147,6 +147,7 @@ func (fts *FleetTestSuite) beforeScenario() {
 }
 
 func (fts *FleetTestSuite) contributeSteps(s *godog.ScenarioContext) {
+	s.Step(`^kibana uses "([^"]*)" profile$`, fts.kibanaUsesProfile)
 	s.Step(`^a "([^"]*)" agent is deployed to Fleet$`, fts.anAgentIsDeployedToFleet)
 	s.Step(`^a "([^"]*)" agent is deployed to Fleet on top of "([^"]*)"$`, fts.anAgentIsDeployedToFleetOnTopOfBeat)
 	s.Step(`^a "([^"]*)" agent is deployed to Fleet with "([^"]*)" installer$`, fts.anAgentIsDeployedToFleetWithInstaller)
@@ -481,6 +482,50 @@ func (fts *FleetTestSuite) processStateChangedOnTheHost(process string, state st
 	return CheckProcessState(fts.deployer, manifest.Name, process, "stopped", 0)
 }
 
+// bootstrapFleet this method creates the runtime dependencies for the Fleet test suite, being of special
+// interest kibana profile passed as part of the environment variables to bootstrap the dependencies.
+func bootstrapFleet(ctx context.Context, env map[string]string) error {
+	deployer := deploy.New(common.Provider)
+
+	if profile, ok := env["kibanaProfile"]; ok {
+		log.Infof("Running kibana with %s profile", profile)
+	}
+
+	// the runtime dependencies must be started only in non-remote executions
+	return deployer.Bootstrap(ctx, deploy.NewServiceRequest(common.FleetProfileName), env, func() error {
+		kibanaClient, err := kibana.NewClient()
+		if err != nil {
+			log.WithFields(log.Fields{
+				"error": err,
+				"env":   env,
+			}).Fatal("Unable to create kibana client")
+		}
+		err = kibanaClient.WaitForFleet(ctx)
+		if err != nil {
+			log.WithFields(log.Fields{
+				"error": err,
+				"env":   env,
+			}).Fatal("Fleet could not be initialized")
+		}
+		return nil
+	})
+}
+
+// kibanaUsesProfile this step should be ideally called as a Background or a Given clause, so that it
+// is executed before any other in the test scenario. It will configure the Kibana profile to be used
+// in the scenario, changing the configuration file to be used.
+func (fts *FleetTestSuite) kibanaUsesProfile(profile string) error {
+	// copy the current profile environment, overriding the kibana profile with the one passed in the step
+	env := map[string]string{}
+	for k, v := range common.ProfileEnv {
+		env[k] = v
+	}
+
+	env["kibanaProfile"] = profile
+
+	return bootstrapFleet(context.Background(), env)
+}
+
 func (fts *FleetTestSuite) setup() error {
 	log.Trace("Creating Fleet setup")
 

e2e/_suites/fleet/ingest_manager_test.go

@@ -153,25 +153,19 @@ func InitializeIngestManagerTestSuite(ctx *godog.TestSuiteContext) {
 			"stackVersion":  common.StackVersion,
 		}
 
+		common.ProfileEnv["kibanaProfile"] = "default"
 		common.ProfileEnv["kibanaDockerNamespace"] = "kibana"
 		if strings.HasPrefix(common.KibanaVersion, "pr") || utils.IsCommit(common.KibanaVersion) {
 			// because it comes from a PR
 			common.ProfileEnv["kibanaDockerNamespace"] = "observability-ci"
+			common.ProfileEnv["KIBANA_IMAGE_REF_CUSTOM"] = "docker.elastic.co/observability-ci/kibana:" + common.KibanaVersion
 		}
 
 		if common.Provider != "remote" {
-			// the runtime dependencies must be started only in non-remote executions
-			deployer.Bootstrap(suiteContext, deploy.NewServiceRequest(common.FleetProfileName), common.ProfileEnv, func() error {
-				kibanaClient, err := kibana.NewClient()
-				if err != nil {
-					log.WithField("error", err).Fatal("Unable to create kibana client")
-				}
-				err = kibanaClient.WaitForFleet(suiteContext)
-				if err != nil {
-					log.WithField("error", err).Fatal("Fleet could not be initialized")
-				}
-				return nil
-			})
+			err := bootstrapFleet(suiteContext, common.ProfileEnv)
+			if err != nil {
+				log.WithError(err).Fatal("Could not bootstrap Fleet runtime dependencies")
+			}
 		}
 
 		imts.Fleet.Version = common.BeatVersionBase

internal/deploy/elastic_package.go

@@ -10,9 +10,12 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/elastic/e2e-testing/cli/config"
 	"github.com/elastic/e2e-testing/internal/common"
+	"github.com/elastic/e2e-testing/internal/io"
 	"github.com/elastic/e2e-testing/internal/shell"
 	"github.com/google/uuid"
+	"github.com/mitchellh/go-homedir"
 	log "github.com/sirupsen/logrus"
 	"github.com/testcontainers/testcontainers-go"
 	tc "github.com/testcontainers/testcontainers-go"
@@ -71,13 +74,78 @@ func (ep *EPServiceManager) Add(ctx context.Context, profile ServiceRequest, ser
 	return nil
 }
 
+func checkElasticPackageProfile(ctx context.Context, kibanaProfile string) error {
+	// check compose profile
+	kibanaProfileFile := filepath.Join(config.OpDir(), "compose", "profiles", "fleet", kibanaProfile, "kibana.config.yml")
+	found, err := io.Exists(kibanaProfileFile)
+	if !found || err != nil {
+		return err
+	}
+
+	args := append(elasticPackageBaseCommand, "profiles", "create", kibanaProfile, "--from", "default")
+
+	span, _ := apm.StartSpanOptions(ctx, "Copying Elastic Package profile", "elastic-package.profile.create", apm.SpanOptions{
+		Parent: apm.SpanFromContext(ctx).TraceContext(),
+	})
+	span.Context.SetLabel("args", args)
+	span.Context.SetLabel("kibanaProfile", kibanaProfile)
+
+	home, err := homedir.Dir()
+	if err != nil {
+		return err
+	}
+
+	elasticPackageProfile := filepath.Join(home, ".elastic-package", "profiles", kibanaProfile)
+	found, err = io.Exists(elasticPackageProfile)
+	if err != nil {
+		return err
+	}
+
+	if !found {
+		_, err = shell.Execute(ctx, ".", "go", args...)
+		if err != nil {
+			return err
+		}
+	} else {
+		log.Trace("Not creating a new Elastic Package profile for " + kibanaProfile + ". Kibana config will be overriden")
+	}
+
+	elasticPackageProfileFile := filepath.Join(elasticPackageProfile, "stack", "kibana.config.yml")
+
+	// copy compose's kibana's config to elastic-package's config
+	err = io.CopyFile(kibanaProfileFile, elasticPackageProfileFile, 10000)
+	if err != nil {
+		log.WithFields(log.Fields{
+			"error": err,
+		}).Fatal("Impossible to copy file")
+		return err
+	}
+
+	log.WithFields(log.Fields{
+		"src":    kibanaProfileFile,
+		"target": elasticPackageProfileFile,
+	}).Debug("Kibana profile copied")
+
+	return err
+}
+
 // Bootstrap sets up environment with docker compose
 func (ep *EPServiceManager) Bootstrap(ctx context.Context, profile ServiceRequest, env map[string]string, waitCB func() error) error {
 	services := "elasticsearch,fleet-server,kibana,package-registry"
 
 	version := common.StackVersion
 
-	args := append(elasticPackageBaseCommand, "stack", "up", "--daemon", "--verbose", "--version", version, "--services", services)
+	elasticPackageProfile := "default"
+	if kibanaProfile, ok := env["kibanaProfile"]; ok {
+		elasticPackageProfile = kibanaProfile
+	}
+
+	err := checkElasticPackageProfile(ctx, elasticPackageProfile)
+	if err != nil {
+		return err
+	}
+
+	args := append(elasticPackageBaseCommand, "stack", "up", "--daemon", "--verbose", "--version", version, "--services", services, "-p", elasticPackageProfile)
 
 	span, _ := apm.StartSpanOptions(ctx, "Bootstrapping Elastic Package deployment", "elastic-package.manifest.bootstrap", apm.SpanOptions{
 		Parent: apm.SpanFromContext(ctx).TraceContext(),
@@ -92,7 +160,7 @@ func (ep *EPServiceManager) Bootstrap(ctx context.Context, profile ServiceReques
 		return fmt.Errorf("profile %s not supported in elastic-package provisioner. Services: %v", profile.Name, services)
 	}
 
-	_, err := shell.Execute(ctx, ".", "go", args...)
+	_, err = shell.ExecuteWithEnv(ctx, ".", "go", env, args...)
 	return err
 }
 

internal/kubernetes/kubernetes.go

@@ -106,7 +106,7 @@ func (c Control) RunWithStdin(ctx context.Context, stdin io.Reader, runArgs ...s
 		args = append(args, "--namespace", c.Namespace)
 	}
 	args = append(args, runArgs...)
-	return shell.ExecuteWithStdin(ctx, ".", stdin, "kubectl", args...)
+	return shell.ExecuteWithStdin(ctx, ".", stdin, "kubectl", map[string]string{}, args...)
 }
 
 // Cluster kind structure definition