feat: support using Beats CI snapshots

.ci/Jenkinsfile

@@ -27,6 +27,7 @@ pipeline {
     choice(name: 'runTestsSuite', choices: ['all', 'helm', 'ingest-manager', 'metricbeat'], description: 'Choose which test suite to run (default: all)')
     booleanParam(name: "forceSkipGitChecks", defaultValue: false, description: "If it's needed to check for Git changes to filter by modified sources")
     string(name: 'ELASTIC_AGENT_DOWNLOAD_URL', defaultValue: '', description: 'If present, it will override the download URL for the Elastic agent artifact. (I.e. https://snapshots.elastic.co/8.0.0-59098054/downloads/beats/elastic-agent/elastic-agent-8.0.0-SNAPSHOT-linux-x86_64.tar.gz')
+    booleanParam(name: "ELASTIC_AGENT_USE_CI_SNAPSHOTS", defaultValue: false, description: "If it's needed to use the snapshots produced by Beats CI instead of the official releases")
     choice(name: 'LOG_LEVEL', choices: ['INFO', 'DEBUG'], description: 'Log level to be used')
     choice(name: 'RETRY_TIMEOUT', choices: ['3', '5', '7', '11'], description: 'Max number of minutes for timeout backoff strategies')
     string(name: 'STACK_VERSION_INGEST_MANAGER', defaultValue: '8.0.0-SNAPSHOT', description: 'SemVer version of the stack to be used for Ingest Manager tests.')
@@ -46,6 +47,7 @@ pipeline {
         PATH = "${env.PATH}:${env.WORKSPACE}/bin:${env.WORKSPACE}/${env.BASE_DIR}/.ci/scripts"
         GO111MODULE = 'on'
         ELASTIC_AGENT_DOWNLOAD_URL = "${params.ELASTIC_AGENT_DOWNLOAD_URL.trim()}"
+        ELASTIC_AGENT_USE_CI_SNAPSHOTS = "${params.ELASTIC_AGENT_USE_CI_SNAPSHOTS}"
         METRICBEAT_VERSION = "${params.METRICBEAT_VERSION.trim()}"
         STACK_VERSION_INGEST_MANAGER = "${params.STACK_VERSION_INGEST_MANAGER.trim()}"
         STACK_VERSION_METRICBEAT = "${params.STACK_VERSION_METRICBEAT.trim()}"

cli/config/compose/services/elastic-agent/docker-compose.yml

@@ -1,7 +1,7 @@
 version: '2.3'
 services:
   elastic-agent:
-    image: docker.elastic.co/beats/elastic-agent:${elasticAgentTag:-8.0.0-SNAPSHOT}
+    image: docker.elastic.co/observability-ci/elastic-agent:${elasticAgentTag:-8.0.0-SNAPSHOT}
     depends_on:
       elasticsearch:
         condition: service_healthy

cli/shell/shell.go

@@ -6,7 +6,9 @@ package shell
 
 import (
 	"bytes"
+	"os"
 	"os/exec"
+	"strconv"
 	"strings"
 
 	log "github.com/sirupsen/logrus"
@@ -54,6 +56,42 @@ func Execute(workspace string, command string, args ...string) (string, error) {
 	return strings.Trim(out.String(), "\n"), nil
 }
 
+// GetEnv returns an environment variable as string
+func GetEnv(envVar string, defaultValue string) string {
+	if value, exists := os.LookupEnv(envVar); exists {
+		return value
+	}
+
+	return defaultValue
+}
+
+// GetEnvBool returns an environment variable as boolean
+func GetEnvBool(key string) bool {
+	s := os.Getenv(key)
+	if s == "" {
+		return false
+	}
+
+	v, err := strconv.ParseBool(s)
+	if err != nil {
+		return false
+	}
+
+	return v
+}
+
+// GetEnvInteger returns an environment variable as integer, including a default value
+func GetEnvInteger(envVar string, defaultValue int) int {
+	if value, exists := os.LookupEnv(envVar); exists {
+		v, err := strconv.Atoi(value)
+		if err == nil {
+			return v
+		}
+	}
+
+	return defaultValue
+}
+
 // which checks if software is installed, else it aborts the execution
 func which(binary string) error {
 	path, err := exec.LookPath(binary)

e2e/_suites/ingest-manager/README.md

@@ -49,6 +49,9 @@ This is an example of the optional configuration:
    # This environment variable will use a fixed version of the Elastic agent binary, obtained from
    # https://artifacts-api.elastic.co/v1/search/8.0.0-SNAPSHOT/elastic-agent
    export ELASTIC_AGENT_DOWNLOAD_URL="https://snapshots.elastic.co/8.0.0-59098054/downloads/beats/elastic-agent/elastic-agent-8.0.0-SNAPSHOT-linux-x86_64.tar.gz"
+   # This environment variable will use the snapshots produced by Beats CI. If the above variable
+   # is set, this variable will take no effect
+   export ELASTIC_AGENT_USE_CI_SNAPSHOTS="true"
    ```
 
 3. Define the proper Docker images to be used in tests (Optional).

e2e/_suites/ingest-manager/ingest-manager_test.go

@@ -16,6 +16,7 @@ import (
 	"github.com/elastic/e2e-testing/cli/config"
 	"github.com/elastic/e2e-testing/cli/docker"
 	"github.com/elastic/e2e-testing/cli/services"
+	"github.com/elastic/e2e-testing/cli/shell"
 	"github.com/elastic/e2e-testing/e2e"
 	log "github.com/sirupsen/logrus"
 )
@@ -38,8 +39,8 @@ const kibanaBaseURL = "http://localhost:5601"
 func init() {
 	config.Init()
 
-	queryRetryTimeout = e2e.GetIntegerFromEnv("OP_RETRY_TIMEOUT", queryRetryTimeout)
-	stackVersion = e2e.GetEnv("OP_STACK_VERSION", stackVersion)
+	queryRetryTimeout = shell.GetEnvInteger("OP_RETRY_TIMEOUT", queryRetryTimeout)
+	stackVersion = shell.GetEnv("OP_STACK_VERSION", stackVersion)
 }
 
 func IngestManagerFeatureContext(s *godog.Suite) {

e2e/_suites/ingest-manager/stand-alone.go

@@ -49,6 +49,7 @@ func (sats *StandAloneTestSuite) aStandaloneAgentIsDeployed() error {
 	sats.AgentConfigFilePath = configurationFilePath
 
 	profileEnv["elasticAgentConfigFile"] = sats.AgentConfigFilePath
+	profileEnv["elasticAgentTag"] = stackVersion
 
 	err = serviceManager.AddServicesToCompose(profile, []string{serviceName}, profileEnv)
 	if err != nil {

e2e/_suites/metricbeat/metricbeat_test.go

@@ -16,6 +16,7 @@ import (
 	messages "github.com/cucumber/messages-go/v10"
 	"github.com/elastic/e2e-testing/cli/config"
 	"github.com/elastic/e2e-testing/cli/services"
+	"github.com/elastic/e2e-testing/cli/shell"
 	"github.com/elastic/e2e-testing/e2e"
 	log "github.com/sirupsen/logrus"
 )
@@ -37,9 +38,9 @@ var stackVersion = "7.8.0"
 func init() {
 	config.Init()
 
-	metricbeatVersion = e2e.GetEnv("OP_METRICBEAT_VERSION", metricbeatVersion)
-	queryRetryTimeout = e2e.GetIntegerFromEnv("OP_RETRY_TIMEOUT", queryRetryTimeout)
-	stackVersion = e2e.GetEnv("OP_STACK_VERSION", stackVersion)
+	metricbeatVersion = shell.GetEnv("OP_METRICBEAT_VERSION", metricbeatVersion)
+	queryRetryTimeout = shell.GetEnvInteger("OP_RETRY_TIMEOUT", queryRetryTimeout)
+	stackVersion = shell.GetEnv("OP_STACK_VERSION", stackVersion)
 
 	serviceManager = services.NewServiceManager()
 }

e2e/utils.go

@@ -21,6 +21,7 @@ import (
 	backoff "github.com/cenkalti/backoff/v4"
 	"github.com/elastic/e2e-testing/cli/docker"
 	curl "github.com/elastic/e2e-testing/cli/shell"
+	shell "github.com/elastic/e2e-testing/cli/shell"
 	log "github.com/sirupsen/logrus"
 )
 
@@ -31,27 +32,6 @@ const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
 var seededRand *rand.Rand = rand.New(
 	rand.NewSource(time.Now().UnixNano()))
 
-// GetEnv returns an environment variable as string
-func GetEnv(envVar string, defaultValue string) string {
-	if value, exists := os.LookupEnv(envVar); exists {
-		return value
-	}
-
-	return defaultValue
-}
-
-// GetIntegerFromEnv returns an environment variable as integer, including a default value
-func GetIntegerFromEnv(envVar string, defaultValue int) int {
-	if value, exists := os.LookupEnv(envVar); exists {
-		v, err := strconv.Atoi(value)
-		if err == nil {
-			return v
-		}
-	}
-
-	return defaultValue
-}
-
 // GetExponentialBackOff returns a preconfigured exponential backoff instance
 func GetExponentialBackOff(elapsedTime time.Duration) *backoff.ExponentialBackOff {
 	var (
@@ -72,18 +52,29 @@ func GetExponentialBackOff(elapsedTime time.Duration) *backoff.ExponentialBackOf
 	return exp
 }
 
-// GetElasticArtifactURL returns the URL of a released artifact from
-// Elastic's artifact repository, bbuilding the JSON path query based
-// on the desired OS, architecture and file extension
+// GetElasticArtifactURL returns the URL of a released artifact from two possible sources
+// on the desired OS, architecture and file extension:
+// 1. Observability CI Storage bucket
+// 2. Elastic's artifact repository, building the JSON path query based
 // i.e. GetElasticArtifactURL("elastic-agent", "8.0.0-SNAPSHOT", "linux", "x86_64", "tar.gz")
 // If the environment variable ELASTIC_AGENT_DOWNLOAD_URL exists, then the artifact to be downloaded will
 // be defined by that value
+// Else, if the environment variable ELASTIC_AGENT_USE_CI_SNAPSHOTS is set, then the artifact
+// to be downloaded will be defined by the latest snapshot produced by the Beats CI.
 func GetElasticArtifactURL(artifact string, version string, OS string, arch string, extension string) (string, error) {
 	downloadURL := os.Getenv("ELASTIC_AGENT_DOWNLOAD_URL")
 	if downloadURL != "" {
 		return downloadURL, nil
 	}
 
+	useCISnapshots := shell.GetEnvBool("ELASTIC_AGENT_USE_CI_SNAPSHOTS")
+	if useCISnapshots {
+		// We will use the snapshots produced by Beats CI
+		bucket := "beats-ci-artifacts"
+		object := fmt.Sprintf("%s-%s-%s-%s.%s", artifact, version, OS, arch, extension)
+		return GetObjectURLFromBucket(bucket, object)
+	}
+
 	exp := GetExponentialBackOff(1 * time.Minute)
 
 	retryCount := 1
@@ -150,6 +141,72 @@ func GetElasticArtifactURL(artifact string, version string, OS string, arch stri
 	return downloadURL, nil
 }
 
+// GetObjectURLFromBucket extracts the media URL for the desired artifact from the
+// Google Cloud Storage bucket used by the CI to push snapshots
+func GetObjectURLFromBucket(bucket string, object string) (string, error) {
+	exp := GetExponentialBackOff(1 * time.Minute)
+
+	retryCount := 1
+
+	body := ""
+
+	storageAPI := func() error {
+		r := curl.HTTPRequest{
+			URL: fmt.Sprintf("https://storage.googleapis.com/storage/v1/b/%s/o", bucket),
+		}
+
+		response, err := curl.Get(r)
+		if err != nil {
+			log.WithFields(log.Fields{
+				"bucket":         bucket,
+				"elapsedTime":    exp.GetElapsedTime(),
+				"error":          err,
+				"object":         object,
+				"retry":          retryCount,
+				"statusEndpoint": r.URL,
+			}).Warn("Google Cloud Storage API is not available yet")
+
+			retryCount++
+
+			return err
+		}
+
+		log.WithFields(log.Fields{
+			"bucket":         bucket,
+			"elapsedTime":    exp.GetElapsedTime(),
+			"object":         object,
+			"retries":        retryCount,
+			"statusEndpoint": r.URL,
+		}).Debug("Google Cloud Storage API is available")
+
+		body = response
+		return nil
+	}
+
+	err := backoff.Retry(storageAPI, exp)
+	if err != nil {
+		return "", err
+	}
+
+	jsonParsed, err := gabs.ParseJSON([]byte(body))
+	if err != nil {
+		log.WithFields(log.Fields{
+			"bucket": bucket,
+			"object": object,
+		}).Error("Could not parse the response body for the object")
+		return "", err
+	}
+
+	for _, item := range jsonParsed.Path("items").Children() {
+		itemID := item.Path("id").Data().(string)
+		if strings.Contains(itemID, object) {
+			return item.Path("mediaLink").Data().(string), nil
+		}
+	}
+
+	return "", fmt.Errorf("The %s object could not be found in the %s bucket", object, bucket)
+}
+
 // DownloadFile will download a url and store it in a temporary path.
 // It writes to the destination file as it downloads it, without
 // loading the entire file into memory.