From 35ad51847e097b40da17ae0f5a6e082c9c3d43d5 Mon Sep 17 00:00:00 2001 From: javiermolinar Date: Wed, 3 Jul 2024 12:05:23 +0200 Subject: [PATCH 1/5] added new helper method for allow debugging e2e tests --- CHANGELOG.md | 1 + integration/util.go | 29 +++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 299cc75faf3..42c8902c7fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ * [BUGFIX] max_global_traces_per_user: take into account ingestion.tenant_shard_size when converting to local limit [#3618](https://github.com/grafana/tempo/pull/3618) (@kvrhdn) * [BUGFIX] Fix http connection reuse on GCP and AWS by reading io.EOF through the http body. [#3760](https://github.com/grafana/tempo/pull/3760) (@bmteller) * [BUGFIX] Improved handling of complete blocks in localblocks processor after enabling flusing [#3805](https://github.com/grafana/tempo/pull/3805) (@mapno) +* [ENHANCEMENT] Add a new helper method to allow debugging e2e tests ## v2.5.0 diff --git a/integration/util.go b/integration/util.go index 1f01d729e7f..43778252d7a 100644 --- a/integration/util.go +++ b/integration/util.go @@ -39,6 +39,7 @@ import ( const ( image = "tempo:latest" + debugImage = "tempo-debug:latest" queryImage = "tempo-query:latest" ) @@ -68,6 +69,34 @@ func NewTempoAllInOne(extraArgs ...string) *e2e.HTTPService { return NewTempoAllInOneWithReadinessProbe(e2e.NewHTTPReadinessProbe(3200, "/ready", 200, 299), extraArgs...) } +func NewTempoAllInOnDebug(extraArgs ...string) *e2e.HTTPService { + rp := e2e.NewHTTPReadinessProbe(3200, "/ready", 200, 299) + args := []string{"-config.file=" + filepath.Join(e2e.ContainerSharedDir, "config.yaml")} + args = buildArgsWithExtra(args, extraArgs) + + s := e2e.NewHTTPService( + "tempo", + debugImage, + e2e.NewCommand("", args...), + rp, + 3200, // http all things + 3201, // http all things + 9095, // grpc tempo + 14250, // jaeger grpc ingest + 9411, // zipkin ingest (used by load) + 4317, // otlp grpc + 4318, // OTLP HTTP + 2345, // delve port + ) + env := map[string]string{ + "DEBUG_BLOCK": "0", + } + s.SetEnvVars(env) + + s.SetBackoff(TempoBackoff()) + return s +} + func NewTempoAllInOneWithReadinessProbe(rp e2e.ReadinessProbe, extraArgs ...string) *e2e.HTTPService { args := []string{"-config.file=" + filepath.Join(e2e.ContainerSharedDir, "config.yaml")} args = buildArgsWithExtra(args, extraArgs) From 885b04c2944dc0e63a8de3f3ecaf5f2a1dc64b44 Mon Sep 17 00:00:00 2001 From: javiermolinar Date: Wed, 3 Jul 2024 12:07:49 +0200 Subject: [PATCH 2/5] reorder changelog --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 42c8902c7fd..46a8e7e1e0d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,12 +21,13 @@ * [ENHANCEMENT] Added a boolean flag to enable or disable dualstack mode on Storage block config for S3 [#3721](https://github.com/grafana/tempo/pull/3721) (@sid-jar, @mapno) * [ENHANCEMENT] Add caching to query range queries [#3796](https://github.com/grafana/tempo/pull/3796) (@mapno) * [ENHANCEMENT] Add data quality metric to measure traces without a root [#3812](https://github.com/grafana/tempo/pull/3812) (@mapno) +* [ENHANCEMENT] Add a new helper method to allow debugging e2e tests * [BUGFIX] Fix metrics queries when grouping by attributes that may not exist [#3734](https://github.com/grafana/tempo/pull/3734) (@mdisibio) * [BUGFIX] Fix frontend parsing error on cached responses [#3759](https://github.com/grafana/tempo/pull/3759) (@mdisibio) * [BUGFIX] max_global_traces_per_user: take into account ingestion.tenant_shard_size when converting to local limit [#3618](https://github.com/grafana/tempo/pull/3618) (@kvrhdn) * [BUGFIX] Fix http connection reuse on GCP and AWS by reading io.EOF through the http body. [#3760](https://github.com/grafana/tempo/pull/3760) (@bmteller) * [BUGFIX] Improved handling of complete blocks in localblocks processor after enabling flusing [#3805](https://github.com/grafana/tempo/pull/3805) (@mapno) -* [ENHANCEMENT] Add a new helper method to allow debugging e2e tests + ## v2.5.0 From dff3910288eda944151caa804620df33e58c46ca Mon Sep 17 00:00:00 2001 From: javiermolinar Date: Wed, 3 Jul 2024 12:27:14 +0200 Subject: [PATCH 3/5] fix typo --- integration/util.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/util.go b/integration/util.go index 43778252d7a..bb820a11873 100644 --- a/integration/util.go +++ b/integration/util.go @@ -69,7 +69,7 @@ func NewTempoAllInOne(extraArgs ...string) *e2e.HTTPService { return NewTempoAllInOneWithReadinessProbe(e2e.NewHTTPReadinessProbe(3200, "/ready", 200, 299), extraArgs...) } -func NewTempoAllInOnDebug(extraArgs ...string) *e2e.HTTPService { +func NewTempoAllInOneDebug(extraArgs ...string) *e2e.HTTPService { rp := e2e.NewHTTPReadinessProbe(3200, "/ready", 200, 299) args := []string{"-config.file=" + filepath.Join(e2e.ContainerSharedDir, "config.yaml")} args = buildArgsWithExtra(args, extraArgs) From daeab2b0b86a5a5ed305e9622db463722dd3037a Mon Sep 17 00:00:00 2001 From: javiermolinar Date: Thu, 4 Jul 2024 10:56:49 +0200 Subject: [PATCH 4/5] added how to debug an e2e to the documentation --- integration/e2e/README.md | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/integration/e2e/README.md b/integration/e2e/README.md index 5f373f3291f..f9765fae13c 100644 --- a/integration/e2e/README.md +++ b/integration/e2e/README.md @@ -1,9 +1,9 @@ -To run the integration tests, use the following commands +**Running the integration tests** `-count=1` is passed to disable cache during test runs. -``` +```sh # build latest image make docker-tempo make docker-tempo-query @@ -23,3 +23,20 @@ go test -timeout 3m -count=1 -v ./integration/e2e/... -run ^TestMultiTenantSearc # follow and watch logs while tests are running (assuming e2e test container is named tempo_e2e-tempo) docker logs $(docker container ls -f name=tempo_e2e-tempo -q) -f ``` + +**How to debug Tempo while running an integration test** + +1. Build latest debug image +```sh + make docker-tempo-debug +``` +2. Use the function ``NewTempoAllInOneDebug`` in your test to spin a Tempo instance with debug capabilities +3. Set a breakpoint after ``require.NoError(t, s.StartAndWaitReady(tempo))`` and before the action you want debug +4. Get the port of Delve debugger inside the container +```sh + docker ps --format '{{.Ports}}' + # 0.0.0.0:53467->2345 +``` +5. Run the debugger against that port as is specified [here](https://github.com/grafana/tempo/tree/main/example/docker-compose/debug) +  + From d36e9bb84f01b44e1ee48c1a1116b4bdb262ca08 Mon Sep 17 00:00:00 2001 From: Mario Date: Thu, 4 Jul 2024 15:54:42 +0200 Subject: [PATCH 5/5] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 46a8e7e1e0d..8c9f63db406 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,7 +21,7 @@ * [ENHANCEMENT] Added a boolean flag to enable or disable dualstack mode on Storage block config for S3 [#3721](https://github.com/grafana/tempo/pull/3721) (@sid-jar, @mapno) * [ENHANCEMENT] Add caching to query range queries [#3796](https://github.com/grafana/tempo/pull/3796) (@mapno) * [ENHANCEMENT] Add data quality metric to measure traces without a root [#3812](https://github.com/grafana/tempo/pull/3812) (@mapno) -* [ENHANCEMENT] Add a new helper method to allow debugging e2e tests +* [ENHANCEMENT] Add a new helper method to allow debugging e2e tests [#3836](https://github.com/grafana/tempo/pull/3836) (@javiermolinar) * [BUGFIX] Fix metrics queries when grouping by attributes that may not exist [#3734](https://github.com/grafana/tempo/pull/3734) (@mdisibio) * [BUGFIX] Fix frontend parsing error on cached responses [#3759](https://github.com/grafana/tempo/pull/3759) (@mdisibio) * [BUGFIX] max_global_traces_per_user: take into account ingestion.tenant_shard_size when converting to local limit [#3618](https://github.com/grafana/tempo/pull/3618) (@kvrhdn)