From 735465609b1b1adc5ebed2e7619a9f13ed95ced2 Mon Sep 17 00:00:00 2001 From: Andre Baptista Aguas Date: Tue, 21 Feb 2023 13:37:28 +0100 Subject: [PATCH 1/5] Include traceID in `slow query detected` log line Signed-off-by: Andre Baptista Aguas --- internal/cortex/frontend/transport/handler.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/internal/cortex/frontend/transport/handler.go b/internal/cortex/frontend/transport/handler.go index e0b2075e0f..9a341b8f8e 100644 --- a/internal/cortex/frontend/transport/handler.go +++ b/internal/cortex/frontend/transport/handler.go @@ -154,7 +154,7 @@ func (f *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { } if shouldReportSlowQuery { - f.reportSlowQuery(r, queryString, queryResponseTime) + f.reportSlowQuery(r, hs, queryString, queryResponseTime) } if f.cfg.QueryStatsEnabled { f.reportQueryStats(r, queryString, queryResponseTime, stats) @@ -162,7 +162,7 @@ func (f *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { } // reportSlowQuery reports slow queries. -func (f *Handler) reportSlowQuery(r *http.Request, queryString url.Values, queryResponseTime time.Duration) { +func (f *Handler) reportSlowQuery(r *http.Request, responseHeaders http.Header, queryString url.Values, queryResponseTime time.Duration) { // NOTE(GiedriusS): see https://github.com/grafana/grafana/pull/60301 for more info. grafanaDashboardUID := "-" if dashboardUID := r.Header.Get("X-Dashboard-Uid"); dashboardUID != "" { @@ -172,6 +172,12 @@ func (f *Handler) reportSlowQuery(r *http.Request, queryString url.Values, query if panelID := r.Header.Get("X-Panel-Id"); panelID != "" { grafanaPanelID = panelID } + thanosTraceID := "-" + if traceID := responseHeaders.Get("X-Thanos-Trace-Id"); traceID != "" { + thanosTraceID = traceID + } + + level.Info(util_log.WithContext(r.Context(), f.log)).Log("aba trace id", thanosTraceID) logMessage := append([]interface{}{ "msg", "slow query detected", @@ -181,6 +187,7 @@ func (f *Handler) reportSlowQuery(r *http.Request, queryString url.Values, query "time_taken", queryResponseTime.String(), "grafana_dashboard_uid", grafanaDashboardUID, "grafana_panel_id", grafanaPanelID, + "trace_id", thanosTraceID, }, formatQueryString(queryString)...) level.Info(util_log.WithContext(r.Context(), f.log)).Log(logMessage...) From ed73d6711d6f792bc7475ced270d677b6f33384d Mon Sep 17 00:00:00 2001 From: Andre Baptista Aguas Date: Tue, 21 Feb 2023 13:50:06 +0100 Subject: [PATCH 2/5] Update changelog Signed-off-by: Andre Baptista Aguas --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fc5be175c3..48b3954d18 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,8 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#6074](https://github.com/thanos-io/thanos/pull/6074) *: Add histogram metrics `thanos_store_server_series_requested` and `thanos_store_server_chunks_requested` to all Stores. - [#6074](https://github.com/thanos-io/thanos/pull/6074) *: Allow configuring series and sample limits per `Series` request for all Stores. - [#6104](https://github.com/thanos-io/thanos/pull/6104) Objstore: Support S3 session token. -- [#5548](https://github.com/thanos-io/thanos/pull/5548) Query: Added experimental support for load balancing across multiple Store endpoints. + +- [#6148](https://github.com/thanos-io/thanos/pull/6148) Query-frontend: add traceID to slow query detected log line ### Fixed From 42afbdfc39839de7ca05a5fe237435c8345c6026 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20=C3=81guas?= Date: Tue, 21 Feb 2023 13:55:06 +0100 Subject: [PATCH 3/5] Update handler.go MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: André Águas --- internal/cortex/frontend/transport/handler.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/internal/cortex/frontend/transport/handler.go b/internal/cortex/frontend/transport/handler.go index 9a341b8f8e..e4cad320cc 100644 --- a/internal/cortex/frontend/transport/handler.go +++ b/internal/cortex/frontend/transport/handler.go @@ -177,8 +177,6 @@ func (f *Handler) reportSlowQuery(r *http.Request, responseHeaders http.Header, thanosTraceID = traceID } - level.Info(util_log.WithContext(r.Context(), f.log)).Log("aba trace id", thanosTraceID) - logMessage := append([]interface{}{ "msg", "slow query detected", "method", r.Method, From 2f53282b7b08d379eb8531a16ac8def8cc72aa98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20=C3=81guas?= Date: Tue, 21 Feb 2023 14:01:56 +0100 Subject: [PATCH 4/5] Update CHANGELOG.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: André Águas --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 48b3954d18..dba1d0be72 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,6 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#6074](https://github.com/thanos-io/thanos/pull/6074) *: Add histogram metrics `thanos_store_server_series_requested` and `thanos_store_server_chunks_requested` to all Stores. - [#6074](https://github.com/thanos-io/thanos/pull/6074) *: Allow configuring series and sample limits per `Series` request for all Stores. - [#6104](https://github.com/thanos-io/thanos/pull/6104) Objstore: Support S3 session token. - - [#6148](https://github.com/thanos-io/thanos/pull/6148) Query-frontend: add traceID to slow query detected log line ### Fixed From c26163ff829857c27975ecf79c9a2d5ab6ef893e Mon Sep 17 00:00:00 2001 From: Andre Baptista Aguas Date: Tue, 21 Feb 2023 14:45:38 +0100 Subject: [PATCH 5/5] CHANGELOG Signed-off-by: Andre Baptista Aguas --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index dba1d0be72..7cd9e3d815 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#6074](https://github.com/thanos-io/thanos/pull/6074) *: Add histogram metrics `thanos_store_server_series_requested` and `thanos_store_server_chunks_requested` to all Stores. - [#6074](https://github.com/thanos-io/thanos/pull/6074) *: Allow configuring series and sample limits per `Series` request for all Stores. - [#6104](https://github.com/thanos-io/thanos/pull/6104) Objstore: Support S3 session token. +- [#5548](https://github.com/thanos-io/thanos/pull/5548) Query: Added experimental support for load balancing across multiple Store endpoints. - [#6148](https://github.com/thanos-io/thanos/pull/6148) Query-frontend: add traceID to slow query detected log line ### Fixed