Exclude vector queries from being counted in metric for rules with zero fetched series

CHANGELOG.md

@@ -6,6 +6,7 @@
 
 * [CHANGE] Ingester: Increase default value of `-blocks-storage.tsdb.head-postings-for-matchers-cache-max-bytes` and `-blocks-storage.tsdb.block-postings-for-matchers-cache-max-bytes` to 100 MiB (previous default value was 10 MiB). #6764
 * [ENHANCEMENT] PromQL: ignore small errors for bucketQuantile #6766
+* [ENHANCEMENT] Ruler: exclude vector queries from being tracked in `cortex_ruler_queries_zero_fetched_series_total`. #6544
 * [BUGFIX] Ingester: don't ignore errors encountered while iterating through chunks or samples in response to a query request. #6451
 * [BUGFIX] Fix issue where queries can fail or omit OOO samples if OOO head compaction occurs between creating a querier and reading chunks #6766
 * [BUGFIX] Fix issue where concatenatingChunkIterator can obscure errors #6766

integration/ruler_test.go

@@ -694,10 +694,58 @@ func TestRulerMetricsForInvalidQueriesAndNoFetchedSeries(t *testing.T) {
 
 		deleteRuleAndWait(groupName)
 
-		const groupName2 = "good_rule_with_no_fetched_series_and_no_samples"
-		const expression2 = `sum(metric{foo="10000"})`
+		const groupName2 = "good_rule_with_fetched_series_and_samples"
+		const expression2 = `sum(metric{foo=~"1|2"})`
 		addNewRuleAndWait(groupName2, expression2, false)
 
+		// Ensure that samples were returned.
+		sum, err = ruler.SumMetrics([]string{"cortex_prometheus_last_evaluation_samples"})
+		require.NoError(t, err)
+		require.Less(t, float64(0), sum[0])
+
+		// Ensure that the metric for no fetched series was not incremented.
+		sum, err = ruler.SumMetrics([]string{"cortex_ruler_queries_zero_fetched_series_total"})
+		require.NoError(t, err)
+		require.Equal(t, float64(0), sum[0])
+
+		deleteRuleAndWait(groupName2)
+
+		const groupName3 = "good_rule_with_no_series_selector"
+		const expression3 = `vector(1.2345)`
+		addNewRuleAndWait(groupName3, expression3, false)
+
+		// Ensure that samples were returned.
+		sum, err = ruler.SumMetrics([]string{"cortex_prometheus_last_evaluation_samples"})
+		require.NoError(t, err)
+		require.Less(t, float64(0), sum[0])
+
+		// Ensure that the metric for no fetched series was not incremented.
+		sum, err = ruler.SumMetrics([]string{"cortex_ruler_queries_zero_fetched_series_total"})
+		require.NoError(t, err)
+		require.Equal(t, float64(0), sum[0])
+
+		deleteRuleAndWait(groupName3)
+
+		const groupName4 = "good_rule_with_fetched_series_and_samples_and_non_series_selector"
+		const expression4 = `sum(metric{foo=~"1|2"}) + vector(1.2345)`
+		addNewRuleAndWait(groupName4, expression4, false)
+
+		// Ensure that samples were returned.
+		sum, err = ruler.SumMetrics([]string{"cortex_prometheus_last_evaluation_samples"})
+		require.NoError(t, err)
+		require.Less(t, float64(0), sum[0])
+
+		// Ensure that the metric for no fetched series was not incremented.
+		sum, err = ruler.SumMetrics([]string{"cortex_ruler_queries_zero_fetched_series_total"})
+		require.NoError(t, err)
+		require.Equal(t, float64(0), sum[0])
+
+		deleteRuleAndWait(groupName4)
+
+		const groupName5 = "good_rule_with_no_fetched_series_and_no_samples_and_non_series_selector"
+		const expression5 = `sum(metric{foo="10000"}) + vector(1.2345)`
+		addNewRuleAndWait(groupName5, expression5, false)
+
 		// Ensure that no samples were returned.
 		sum, err = ruler.SumMetrics([]string{"cortex_prometheus_last_evaluation_samples"})
 		require.NoError(t, err)
@@ -707,6 +755,22 @@ func TestRulerMetricsForInvalidQueriesAndNoFetchedSeries(t *testing.T) {
 		sum, err = ruler.SumMetrics([]string{"cortex_ruler_queries_zero_fetched_series_total"})
 		require.NoError(t, err)
 		require.Equal(t, float64(1), sum[0])
+
+		deleteRuleAndWait(groupName5)
+
+		const groupName6 = "good_rule_with_no_fetched_series_and_no_samples"
+		const expression6 = `sum(metric{foo="10000"})`
+		addNewRuleAndWait(groupName6, expression6, false)
+
+		// Ensure that no samples were returned.
+		sum, err = ruler.SumMetrics([]string{"cortex_prometheus_last_evaluation_samples"})
+		require.NoError(t, err)
+		require.Equal(t, float64(0), sum[0])
+
+		// Ensure that the metric for no fetched series was incremented.
+		sum, err = ruler.SumMetrics([]string{"cortex_ruler_queries_zero_fetched_series_total"})
+		require.NoError(t, err)
+		require.Equal(t, float64(2), sum[0])
 	})
 
 	// Now let's stop ingester, and recheck metrics. This should increase cortex_ruler_queries_failed_total failures.

pkg/ruler/compat.go

@@ -23,6 +23,7 @@ import (
 	"github.com/prometheus/prometheus/model/metadata"
 	"github.com/prometheus/prometheus/notifier"
 	"github.com/prometheus/prometheus/promql"
+	"github.com/prometheus/prometheus/promql/parser"
 	"github.com/prometheus/prometheus/rules"
 	"github.com/prometheus/prometheus/storage"
 
@@ -208,8 +209,14 @@ func RecordAndReportRuleQueryMetrics(qf rules.QueryFunc, queryTime, zeroFetchedS
 			shardedQueries := stats.LoadShardedQueries()
 
 			queryTime.Add(wallTime.Seconds())
-			if err == nil && numSeries == 0 { // Do not count queries with errors for zero fetched series.
-				zeroFetchedSeriesCount.Add(1)
+			// Do not count queries with errors for zero fetched series, or queries
+			// with no selectors that are not meant to fetch any series.
+			if err == nil && numSeries == 0 {
+				if expr, err := parser.ParseExpr(qs); err == nil {
+					if len(parser.ExtractSelectors(expr)) > 0 {
+						zeroFetchedSeriesCount.Add(1)
+					}
+				}
 			}
 
 			// Log ruler query stats.

pkg/ruler/compat_test.go

@@ -354,13 +354,39 @@ func TestRecordAndReportRuleQueryMetrics(t *testing.T) {
 	}
 	qf := RecordAndReportRuleQueryMetrics(mockFunc, queryTime.WithLabelValues("userID"), zeroFetchedSeriesCount.WithLabelValues("userID"), log.NewNopLogger())
 
+	// Ensure we start with counters at 0.
+	require.LessOrEqual(t, float64(0), testutil.ToFloat64(queryTime.WithLabelValues("userID")))
+	require.Equal(t, float64(0), testutil.ToFloat64(zeroFetchedSeriesCount.WithLabelValues("userID")))
+
+	// Increment zeroFetchedSeriesCount for non-existent series.
 	_, _ = qf(context.Background(), "test", time.Now())
 	require.LessOrEqual(t, float64(1), testutil.ToFloat64(queryTime.WithLabelValues("userID")))
 	require.Equal(t, float64(1), testutil.ToFloat64(zeroFetchedSeriesCount.WithLabelValues("userID")))
 
-	_, _ = qf(context.Background(), "test", time.Now())
+	// Increment zeroFetchedSeriesCount for another non-existent series.
+	_, _ = qf(context.Background(), "test2", time.Now())
 	require.LessOrEqual(t, float64(2), testutil.ToFloat64(queryTime.WithLabelValues("userID")))
 	require.Equal(t, float64(2), testutil.ToFloat64(zeroFetchedSeriesCount.WithLabelValues("userID")))
+
+	// Don't increment zeroFetchedSeriesCount for query without series selectors.
+	_, _ = qf(context.Background(), "vector(0.995)", time.Now())
+	require.LessOrEqual(t, float64(3), testutil.ToFloat64(queryTime.WithLabelValues("userID")))
+	require.Equal(t, float64(2), testutil.ToFloat64(zeroFetchedSeriesCount.WithLabelValues("userID")))
+
+	// Don't increment zeroFetchedSeriesCount for another query without series selectors.
+	_, _ = qf(context.Background(), "vector(2.4192e+15 / 1e+09)", time.Now())
+	require.LessOrEqual(t, float64(4), testutil.ToFloat64(queryTime.WithLabelValues("userID")))
+	require.Equal(t, float64(2), testutil.ToFloat64(zeroFetchedSeriesCount.WithLabelValues("userID")))
+
+	// Increment zeroFetchedSeriesCount for non-existent series even when combined with a non-series selector.
+	_, _ = qf(context.Background(), "test + vector(0.995)", time.Now())
+	require.LessOrEqual(t, float64(5), testutil.ToFloat64(queryTime.WithLabelValues("userID")))
+	require.Equal(t, float64(3), testutil.ToFloat64(zeroFetchedSeriesCount.WithLabelValues("userID")))
+
+	// Don't increment zeroFetchedSeriesCount for queries with errors.
+	_, _ = qf(context.Background(), "rate(test)", time.Now())
+	require.LessOrEqual(t, float64(6), testutil.ToFloat64(queryTime.WithLabelValues("userID")))
+	require.Equal(t, float64(3), testutil.ToFloat64(zeroFetchedSeriesCount.WithLabelValues("userID")))
 }
 
 // TestManagerFactory_CorrectQueryableUsed ensures that when evaluating a group with non-empty SourceTenants