feat: added new helper function to allow debugging e2e tests

CHANGELOG.md

@@ -21,12 +21,14 @@
 * [ENHANCEMENT] Added a boolean flag to enable or disable dualstack mode on Storage block config for S3 [#3721](https://github.com/grafana/tempo/pull/3721) (@sid-jar, @mapno)
 * [ENHANCEMENT] Add caching to query range queries [#3796](https://github.com/grafana/tempo/pull/3796) (@mapno)
 * [ENHANCEMENT] Add data quality metric to measure traces without a root [#3812](https://github.com/grafana/tempo/pull/3812) (@mapno)
+* [ENHANCEMENT] Add a new helper method to allow debugging e2e tests
 * [BUGFIX] Fix metrics queries when grouping by attributes that may not exist [#3734](https://github.com/grafana/tempo/pull/3734) (@mdisibio)
 * [BUGFIX] Fix frontend parsing error on cached responses [#3759](https://github.com/grafana/tempo/pull/3759) (@mdisibio)
 * [BUGFIX] max_global_traces_per_user: take into account ingestion.tenant_shard_size when converting to local limit [#3618](https://github.com/grafana/tempo/pull/3618) (@kvrhdn)
 * [BUGFIX] Fix http connection reuse on GCP and AWS by reading io.EOF through the http body. [#3760](https://github.com/grafana/tempo/pull/3760) (@bmteller)
 * [BUGFIX] Improved handling of complete blocks in localblocks processor after enabling flusing [#3805](https://github.com/grafana/tempo/pull/3805) (@mapno)
 
+
 ## v2.5.0
 
 * [CHANGE] Align metrics query time ranges to the step parameter [#3490](https://github.com/grafana/tempo/pull/3490) (@mdisibio)

integration/e2e/README.md

@@ -1,9 +1,9 @@
-To run the integration tests, use the following commands
+**Running the integration tests**
 
 
 `-count=1` is passed to disable cache during test runs.
 
-```
+```sh
 # build latest image
 make docker-tempo
 make docker-tempo-query
@@ -23,3 +23,20 @@ go test -timeout 3m -count=1 -v ./integration/e2e/... -run ^TestMultiTenantSearc
 # follow and watch logs while tests are running (assuming e2e test container is named tempo_e2e-tempo)
 docker logs $(docker container ls -f name=tempo_e2e-tempo -q) -f
 ```
+
+**How to debug Tempo while running an integration test**
+
+1. Build latest debug image
+```sh
+    make docker-tempo-debug
+```
+2. Use the function ``NewTempoAllInOneDebug`` in your test to spin a Tempo instance with debug capabilities
+3. Set a breakpoint after ``require.NoError(t, s.StartAndWaitReady(tempo))`` and before the action you want debug
+4. Get the port of Delve debugger inside the container
+```sh
+    docker ps --format '{{.Ports}}'  
+    # 0.0.0.0:53467->2345
+```
+5. Run the debugger against that port as is specified [here](https://github.com/grafana/tempo/tree/main/example/docker-compose/debug)
+
+

integration/util.go

@@ -39,6 +39,7 @@ import (
 
 const (
 	image      = "tempo:latest"
+	debugImage = "tempo-debug:latest"
 	queryImage = "tempo-query:latest"
 )
 
@@ -68,6 +69,34 @@ func NewTempoAllInOne(extraArgs ...string) *e2e.HTTPService {
 	return NewTempoAllInOneWithReadinessProbe(e2e.NewHTTPReadinessProbe(3200, "/ready", 200, 299), extraArgs...)
 }
 
+func NewTempoAllInOneDebug(extraArgs ...string) *e2e.HTTPService {
+	rp := e2e.NewHTTPReadinessProbe(3200, "/ready", 200, 299)
+	args := []string{"-config.file=" + filepath.Join(e2e.ContainerSharedDir, "config.yaml")}
+	args = buildArgsWithExtra(args, extraArgs)
+
+	s := e2e.NewHTTPService(
+		"tempo",
+		debugImage,
+		e2e.NewCommand("", args...),
+		rp,
+		3200,  // http all things
+		3201,  // http all things
+		9095,  // grpc tempo
+		14250, // jaeger grpc ingest
+		9411,  // zipkin ingest (used by load)
+		4317,  // otlp grpc
+		4318,  // OTLP HTTP
+		2345,  // delve port
+	)
+	env := map[string]string{
+		"DEBUG_BLOCK": "0",
+	}
+	s.SetEnvVars(env)
+
+	s.SetBackoff(TempoBackoff())
+	return s
+}
+
 func NewTempoAllInOneWithReadinessProbe(rp e2e.ReadinessProbe, extraArgs ...string) *e2e.HTTPService {
 	args := []string{"-config.file=" + filepath.Join(e2e.ContainerSharedDir, "config.yaml")}
 	args = buildArgsWithExtra(args, extraArgs)