From 6f0f8db3b3ba86016b50f4c058ac052561839727 Mon Sep 17 00:00:00 2001 From: Jeanette Tan Date: Fri, 3 Nov 2023 00:12:22 +0800 Subject: [PATCH] Exclude vector queries from being counted in metric for rules with zero fetched series --- CHANGELOG.md | 1 + pkg/ruler/compat.go | 7 ++++++- pkg/ruler/compat_test.go | 8 ++++++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ae1b23f18c..b023e4e0159 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -59,6 +59,7 @@ * [ENHANCEMENT] Add connection-string option, `-.azure.connection-string`, for Azure Blob Storage. #6487 * [ENHANCEMENT] Ingester: Add `-ingester.instance-limits.max-inflight-push-requests-bytes`. This limit protects the ingester against requests that together may cause an OOM. #6492 * [ENHANCEMENT] Ingester: add new per-tenant `cortex_ingester_local_limits` metric to expose the calculated local per-tenant limits seen at each ingester. Exports the local per-tenant series limit with label `{limit="max_global_series_per_user"}` #6403 +* [ENHANCEMENT] Ruler: exclude vector queries from being tracked in `cortex_ruler_queries_zero_fetched_series_total`. #6544 * [BUGFIX] Ring: Ensure network addresses used for component hash rings are formatted correctly when using IPv6. #6068 * [BUGFIX] Query-scheduler: don't retain connections from queriers that have shut down, leading to gradually increasing enqueue latency over time. #6100 #6145 * [BUGFIX] Ingester: prevent query logic from continuing to execute after queries are canceled. #6085 diff --git a/pkg/ruler/compat.go b/pkg/ruler/compat.go index d28daeb3480..41f51d930ec 100644 --- a/pkg/ruler/compat.go +++ b/pkg/ruler/compat.go @@ -15,6 +15,7 @@ import ( "github.com/gogo/status" "github.com/grafana/dskit/httpgrpc" "github.com/grafana/dskit/user" + "github.com/grafana/regexp" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/prometheus/model/exemplar" @@ -32,6 +33,8 @@ import ( util_log "github.com/grafana/mimir/pkg/util/log" ) +var rVector = regexp.MustCompile(`vector\([\d.e+/ ]+\)`) + // Pusher is an ingester server that accepts pushes. type Pusher interface { Push(context.Context, *mimirpb.WriteRequest) (*mimirpb.WriteResponse, error) @@ -208,7 +211,9 @@ func RecordAndReportRuleQueryMetrics(qf rules.QueryFunc, queryTime, zeroFetchedS shardedQueries := stats.LoadShardedQueries() queryTime.Add(wallTime.Seconds()) - if err == nil && numSeries == 0 { // Do not count queries with errors for zero fetched series. + // Do not count queries with errors for zero fetched series, or vector queries that are not + // meant to fetch any series. + if err == nil && numSeries == 0 && !rVector.MatchString(qs) { zeroFetchedSeriesCount.Add(1) } diff --git a/pkg/ruler/compat_test.go b/pkg/ruler/compat_test.go index 6e0e39b1e92..a78bb35852f 100644 --- a/pkg/ruler/compat_test.go +++ b/pkg/ruler/compat_test.go @@ -361,6 +361,14 @@ func TestRecordAndReportRuleQueryMetrics(t *testing.T) { _, _ = qf(context.Background(), "test", time.Now()) require.LessOrEqual(t, float64(2), testutil.ToFloat64(queryTime.WithLabelValues("userID"))) require.Equal(t, float64(2), testutil.ToFloat64(zeroFetchedSeriesCount.WithLabelValues("userID"))) + + _, _ = qf(context.Background(), "vector(0.995)", time.Now()) + require.LessOrEqual(t, float64(2), testutil.ToFloat64(queryTime.WithLabelValues("userID"))) + require.Equal(t, float64(2), testutil.ToFloat64(zeroFetchedSeriesCount.WithLabelValues("userID"))) + + _, _ = qf(context.Background(), "vector(2.4192e+15 / 1e+09)", time.Now()) + require.LessOrEqual(t, float64(2), testutil.ToFloat64(queryTime.WithLabelValues("userID"))) + require.Equal(t, float64(2), testutil.ToFloat64(zeroFetchedSeriesCount.WithLabelValues("userID"))) } // TestManagerFactory_CorrectQueryableUsed ensures that when evaluating a group with non-empty SourceTenants