diff --git a/pkg/api/query/v1.go b/pkg/api/query/v1.go index 1aea3f12a2..f8a446d599 100644 --- a/pkg/api/query/v1.go +++ b/pkg/api/query/v1.go @@ -23,6 +23,7 @@ import ( "context" "math" "net/http" + "sort" "strconv" "strings" "time" @@ -439,16 +440,39 @@ func (qapi *QueryAPI) labelValues(r *http.Request) (interface{}, []error, *api.A return nil, nil, apiErr } - q, err := qapi.queryableCreate(true, nil, storeDebugMatchers, 0, enablePartialResponse, false). + var matcherSets [][]*labels.Matcher + for _, s := range r.Form[MatcherParam] { + matchers, err := parser.ParseMetricSelector(s) + if err != nil { + return nil, nil, &api.ApiError{Typ: api.ErrorBadData, Err: err} + } + matcherSets = append(matcherSets, matchers) + } + + q, err := qapi.queryableCreate(true, nil, storeDebugMatchers, 0, enablePartialResponse, true). Querier(ctx, timestamp.FromTime(start), timestamp.FromTime(end)) if err != nil { return nil, nil, &api.ApiError{Typ: api.ErrorExec, Err: err} } defer runutil.CloseWithLogOnErr(qapi.logger, q, "queryable labelValues") - // TODO(fabxc): add back request context. + var ( + vals []string + warnings storage.Warnings + ) + // TODO(yeya24): push down matchers to Store level. + if len(matcherSets) > 0 { + // Get all series which match matchers. + var sets []storage.SeriesSet + for _, mset := range matcherSets { + s := q.Select(false, nil, mset...) + sets = append(sets, s) + } + vals, warnings, err = labelValuesByMatchers(sets, name) + } else { + vals, warnings, err = q.LabelValues(name) + } - vals, warnings, err := q.LabelValues(name) if err != nil { return nil, nil, &api.ApiError{Typ: api.ErrorExec, Err: err} } @@ -544,14 +568,39 @@ func (qapi *QueryAPI) labelNames(r *http.Request) (interface{}, []error, *api.Ap return nil, nil, apiErr } - q, err := qapi.queryableCreate(true, nil, storeDebugMatchers, 0, enablePartialResponse, false). + var matcherSets [][]*labels.Matcher + for _, s := range r.Form[MatcherParam] { + matchers, err := parser.ParseMetricSelector(s) + if err != nil { + return nil, nil, &api.ApiError{Typ: api.ErrorBadData, Err: err} + } + matcherSets = append(matcherSets, matchers) + } + + q, err := qapi.queryableCreate(true, nil, storeDebugMatchers, 0, enablePartialResponse, true). Querier(r.Context(), timestamp.FromTime(start), timestamp.FromTime(end)) if err != nil { return nil, nil, &api.ApiError{Typ: api.ErrorExec, Err: err} } defer runutil.CloseWithLogOnErr(qapi.logger, q, "queryable labelNames") - names, warnings, err := q.LabelNames() + var ( + names []string + warnings storage.Warnings + ) + // TODO(yeya24): push down matchers to Store level. + if len(matcherSets) > 0 { + // Get all series which match matchers. + var sets []storage.SeriesSet + for _, mset := range matcherSets { + s := q.Select(false, nil, mset...) + sets = append(sets, s) + } + names, warnings, err = labelNamesByMatchers(sets) + } else { + names, warnings, err = q.LabelNames() + } + if err != nil { return nil, nil, &api.ApiError{Typ: api.ErrorExec, Err: err} } @@ -673,3 +722,52 @@ func parseDuration(s string) (time.Duration, error) { } return 0, errors.Errorf("cannot parse %q to a valid duration", s) } + +// Modified from https://github.com/eklockare/prometheus/blob/6178-matchers-with-label-values/web/api/v1/api.go#L571-L591. +// labelNamesByMatchers uses matchers to filter out matching series, then label names are extracted. +func labelNamesByMatchers(sets []storage.SeriesSet) ([]string, storage.Warnings, error) { + set := storage.NewMergeSeriesSet(sets, storage.ChainedSeriesMerge) + labelNamesSet := make(map[string]struct{}) + for set.Next() { + series := set.At() + for _, lb := range series.Labels() { + labelNamesSet[lb.Name] = struct{}{} + } + } + + warnings := set.Warnings() + if set.Err() != nil { + return nil, warnings, set.Err() + } + // Convert the map to an array. + labelNames := make([]string, 0, len(labelNamesSet)) + for key := range labelNamesSet { + labelNames = append(labelNames, key) + } + sort.Strings(labelNames) + return labelNames, warnings, nil +} + +// Modified from https://github.com/eklockare/prometheus/blob/6178-matchers-with-label-values/web/api/v1/api.go#L571-L591. +// LabelValuesByMatchers uses matchers to filter out matching series, then label values are extracted. +func labelValuesByMatchers(sets []storage.SeriesSet, name string) ([]string, storage.Warnings, error) { + set := storage.NewMergeSeriesSet(sets, storage.ChainedSeriesMerge) + labelValuesSet := make(map[string]struct{}) + for set.Next() { + series := set.At() + labelValue := series.Labels().Get(name) + labelValuesSet[labelValue] = struct{}{} + } + + warnings := set.Warnings() + if set.Err() != nil { + return nil, warnings, set.Err() + } + // Convert the map to an array. + labelValues := make([]string, 0, len(labelValuesSet)) + for key := range labelValuesSet { + labelValues = append(labelValues, key) + } + sort.Strings(labelValues) + return labelValues, warnings, nil +} diff --git a/pkg/api/query/v1_test.go b/pkg/api/query/v1_test.go index c41a0e28e1..b5b12f3960 100644 --- a/pkg/api/query/v1_test.go +++ b/pkg/api/query/v1_test.go @@ -745,9 +745,6 @@ func TestMetadataEndpoints(t *testing.T) { }, { endpoint: api.labelNames, - params: map[string]string{ - "name": "__name__", - }, response: []string{ "__name__", "foo", @@ -757,9 +754,6 @@ func TestMetadataEndpoints(t *testing.T) { }, { endpoint: apiWithLabelLookback.labelNames, - params: map[string]string{ - "name": "foo", - }, response: []string{ "__name__", "foo", @@ -773,9 +767,6 @@ func TestMetadataEndpoints(t *testing.T) { "start": []string{"1970-01-01T00:00:00Z"}, "end": []string{"1970-01-01T00:09:00Z"}, }, - params: map[string]string{ - "name": "foo", - }, response: []string{ "__name__", "foo", @@ -787,14 +778,78 @@ func TestMetadataEndpoints(t *testing.T) { "start": []string{"1970-01-01T00:00:00Z"}, "end": []string{"1970-01-01T00:09:00Z"}, }, - params: map[string]string{ - "name": "foo", - }, response: []string{ "__name__", "foo", }, }, + // Failed, to parse matchers. + { + endpoint: api.labelNames, + query: url.Values{ + "match[]": []string{`{xxxx`}, + }, + errType: baseAPI.ErrorBadData, + }, + // Failed to parse matchers. + { + endpoint: api.labelValues, + query: url.Values{ + "match[]": []string{`{xxxx`}, + }, + params: map[string]string{ + "name": "__name__", + }, + errType: baseAPI.ErrorBadData, + }, + { + endpoint: api.labelNames, + query: url.Values{ + "match[]": []string{`test_metric_replica2`}, + }, + response: []string{"__name__", "foo", "replica1"}, + }, + { + endpoint: api.labelValues, + query: url.Values{ + "match[]": []string{`test_metric_replica2`}, + }, + params: map[string]string{ + "name": "__name__", + }, + response: []string{"test_metric_replica2"}, + }, + { + endpoint: api.labelValues, + query: url.Values{ + "match[]": []string{`{foo="bar"}`, `{foo="boo"}`}, + }, + params: map[string]string{ + "name": "__name__", + }, + response: []string{"test_metric1", "test_metric2", "test_metric_replica1", "test_metric_replica2"}, + }, + // No matched series. + { + endpoint: api.labelValues, + query: url.Values{ + "match[]": []string{`{foo="yolo"}`}, + }, + params: map[string]string{ + "name": "__name__", + }, + response: []string{}, + }, + { + endpoint: api.labelValues, + query: url.Values{ + "match[]": []string{`test_metric_replica2`}, + }, + params: map[string]string{ + "name": "replica1", + }, + response: []string{"a"}, + }, // Bad name parameter. { endpoint: api.labelValues, diff --git a/pkg/promclient/promclient.go b/pkg/promclient/promclient.go index 0ce00060a5..89d1673e4b 100644 --- a/pkg/promclient/promclient.go +++ b/pkg/promclient/promclient.go @@ -672,11 +672,14 @@ func (c *Client) SeriesInGRPC(ctx context.Context, base *url.URL, matchers []sto // LabelNames returns all known label names. It uses gRPC errors. // NOTE: This method is tested in pkg/store/prometheus_test.go against Prometheus. -func (c *Client) LabelNamesInGRPC(ctx context.Context, base *url.URL, startTime, endTime int64) ([]string, error) { +func (c *Client) LabelNamesInGRPC(ctx context.Context, base *url.URL, matchers []storepb.LabelMatcher, startTime, endTime int64) ([]string, error) { u := *base u.Path = path.Join(u.Path, "/api/v1/labels") q := u.Query() + if len(matchers) > 0 { + q.Add("match[]", storepb.MatchersToString(matchers...)) + } q.Add("start", formatTime(timestamp.Time(startTime))) q.Add("end", formatTime(timestamp.Time(endTime))) u.RawQuery = q.Encode() @@ -689,11 +692,14 @@ func (c *Client) LabelNamesInGRPC(ctx context.Context, base *url.URL, startTime, // LabelValuesInGRPC returns all known label values for a given label name. It uses gRPC errors. // NOTE: This method is tested in pkg/store/prometheus_test.go against Prometheus. -func (c *Client) LabelValuesInGRPC(ctx context.Context, base *url.URL, label string, startTime, endTime int64) ([]string, error) { +func (c *Client) LabelValuesInGRPC(ctx context.Context, base *url.URL, label string, matchers []storepb.LabelMatcher, startTime, endTime int64) ([]string, error) { u := *base u.Path = path.Join(u.Path, "/api/v1/label/", label, "/values") q := u.Query() + if len(matchers) > 0 { + q.Add("match[]", storepb.MatchersToString(matchers...)) + } q.Add("start", formatTime(timestamp.Time(startTime))) q.Add("end", formatTime(timestamp.Time(endTime))) u.RawQuery = q.Encode() diff --git a/pkg/store/prometheus.go b/pkg/store/prometheus.go index 6d885e5c5b..81458f318c 100644 --- a/pkg/store/prometheus.go +++ b/pkg/store/prometheus.go @@ -483,7 +483,7 @@ func (p *PrometheusStore) encodeChunk(ss []prompb.Sample) (storepb.Chunk_Encodin // LabelNames returns all known label names. func (p *PrometheusStore) LabelNames(ctx context.Context, r *storepb.LabelNamesRequest) (*storepb.LabelNamesResponse, error) { - lbls, err := p.client.LabelNamesInGRPC(ctx, p.base, r.Start, r.End) + lbls, err := p.client.LabelNamesInGRPC(ctx, p.base, nil, r.Start, r.End) if err != nil { return nil, err } @@ -499,7 +499,7 @@ func (p *PrometheusStore) LabelValues(ctx context.Context, r *storepb.LabelValue return &storepb.LabelValuesResponse{Values: []string{l}}, nil } - vals, err := p.client.LabelValuesInGRPC(ctx, p.base, r.Label, r.Start, r.End) + vals, err := p.client.LabelValuesInGRPC(ctx, p.base, r.Label, nil, r.Start, r.End) if err != nil { return nil, err } diff --git a/test/e2e/query_frontend_test.go b/test/e2e/query_frontend_test.go index 7ee97b4acf..54840459a9 100644 --- a/test/e2e/query_frontend_test.go +++ b/test/e2e/query_frontend_test.go @@ -222,7 +222,7 @@ func TestQueryFrontend(t *testing.T) { t.Run("query frontend splitting works for labels names API", func(t *testing.T) { // LabelNames and LabelValues API should still work via query frontend. - labelNames(t, ctx, queryFrontend.HTTPEndpoint(), timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), func(res []string) bool { + labelNames(t, ctx, queryFrontend.HTTPEndpoint(), nil, timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), func(res []string) bool { return len(res) > 0 }) testutil.Ok(t, q.WaitSumMetricsWithOptions( @@ -241,7 +241,7 @@ func TestQueryFrontend(t *testing.T) { e2e.WithLabelMatchers(labels.MustNewMatcher(labels.MatchEqual, "tripperware", "labels"))), ) - labelNames(t, ctx, queryFrontend.HTTPEndpoint(), timestamp.FromTime(now.Add(-24*time.Hour)), timestamp.FromTime(now.Add(time.Hour)), func(res []string) bool { + labelNames(t, ctx, queryFrontend.HTTPEndpoint(), nil, timestamp.FromTime(now.Add(-24*time.Hour)), timestamp.FromTime(now.Add(time.Hour)), func(res []string) bool { return len(res) > 0 }) testutil.Ok(t, q.WaitSumMetricsWithOptions( @@ -262,7 +262,7 @@ func TestQueryFrontend(t *testing.T) { }) t.Run("query frontend splitting works for labels values API", func(t *testing.T) { - labelValues(t, ctx, queryFrontend.HTTPEndpoint(), "instance", timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), func(res []string) bool { + labelValues(t, ctx, queryFrontend.HTTPEndpoint(), "instance", nil, timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), func(res []string) bool { return len(res) == 1 && res[0] == "localhost:9090" }) testutil.Ok(t, q.WaitSumMetricsWithOptions( @@ -281,7 +281,7 @@ func TestQueryFrontend(t *testing.T) { e2e.WithLabelMatchers(labels.MustNewMatcher(labels.MatchEqual, "tripperware", "labels"))), ) - labelValues(t, ctx, queryFrontend.HTTPEndpoint(), "instance", timestamp.FromTime(now.Add(-24*time.Hour)), timestamp.FromTime(now.Add(time.Hour)), func(res []string) bool { + labelValues(t, ctx, queryFrontend.HTTPEndpoint(), "instance", nil, timestamp.FromTime(now.Add(-24*time.Hour)), timestamp.FromTime(now.Add(time.Hour)), func(res []string) bool { return len(res) == 1 && res[0] == "localhost:9090" }) testutil.Ok(t, q.WaitSumMetricsWithOptions( diff --git a/test/e2e/query_test.go b/test/e2e/query_test.go index 1658ebcfb6..103e008358 100644 --- a/test/e2e/query_test.go +++ b/test/e2e/query_test.go @@ -283,14 +283,28 @@ func TestQueryLabelNames(t *testing.T) { t.Cleanup(cancel) now := time.Now() - labelNames(t, ctx, q.HTTPEndpoint(), timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), func(res []string) bool { + labelNames(t, ctx, q.HTTPEndpoint(), nil, timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), func(res []string) bool { return len(res) > 0 }) // Outside time range. - labelNames(t, ctx, q.HTTPEndpoint(), timestamp.FromTime(now.Add(-24*time.Hour)), timestamp.FromTime(now.Add(-23*time.Hour)), func(res []string) bool { + labelNames(t, ctx, q.HTTPEndpoint(), nil, timestamp.FromTime(now.Add(-24*time.Hour)), timestamp.FromTime(now.Add(-23*time.Hour)), func(res []string) bool { return len(res) == 0 }) + + labelNames(t, ctx, q.HTTPEndpoint(), []storepb.LabelMatcher{{Type: storepb.LabelMatcher_EQ, Name: "__name__", Value: "up"}}, + timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), func(res []string) bool { + // Expected result: [__name__, instance, job, prometheus, replica] + return len(res) == 7 + }, + ) + + // There is no matched series. + labelNames(t, ctx, q.HTTPEndpoint(), []storepb.LabelMatcher{{Type: storepb.LabelMatcher_EQ, Name: "__name__", Value: "foobar"}}, + timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), func(res []string) bool { + return len(res) == 0 + }, + ) } func TestQueryLabelValues(t *testing.T) { @@ -327,14 +341,26 @@ func TestQueryLabelValues(t *testing.T) { t.Cleanup(cancel) now := time.Now() - labelValues(t, ctx, q.HTTPEndpoint(), "instance", timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), func(res []string) bool { + labelValues(t, ctx, q.HTTPEndpoint(), "instance", nil, timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), func(res []string) bool { return len(res) == 1 && res[0] == "localhost:9090" }) // Outside time range. - labelValues(t, ctx, q.HTTPEndpoint(), "instance", timestamp.FromTime(now.Add(-24*time.Hour)), timestamp.FromTime(now.Add(-23*time.Hour)), func(res []string) bool { + labelValues(t, ctx, q.HTTPEndpoint(), "instance", nil, timestamp.FromTime(now.Add(-24*time.Hour)), timestamp.FromTime(now.Add(-23*time.Hour)), func(res []string) bool { return len(res) == 0 }) + + labelValues(t, ctx, q.HTTPEndpoint(), "__name__", []storepb.LabelMatcher{{Type: storepb.LabelMatcher_EQ, Name: "__name__", Value: "up"}}, + timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), func(res []string) bool { + return len(res) == 1 && res[0] == "up" + }, + ) + + labelValues(t, ctx, q.HTTPEndpoint(), "__name__", []storepb.LabelMatcher{{Type: storepb.LabelMatcher_EQ, Name: "__name__", Value: "foobar"}}, + timestamp.FromTime(now.Add(-time.Hour)), timestamp.FromTime(now.Add(time.Hour)), func(res []string) bool { + return len(res) == 0 + }, + ) } func checkNetworkRequests(t *testing.T, addr string) { @@ -423,13 +449,13 @@ func queryAndAssert(t *testing.T, ctx context.Context, addr string, q string, op testutil.Equals(t, expected, result) } -func labelNames(t *testing.T, ctx context.Context, addr string, start, end int64, check func(res []string) bool) { +func labelNames(t *testing.T, ctx context.Context, addr string, matchers []storepb.LabelMatcher, start, end int64, check func(res []string) bool) { t.Helper() logger := log.NewLogfmtLogger(os.Stdout) logger = log.With(logger, "ts", log.DefaultTimestampUTC) testutil.Ok(t, runutil.RetryWithLog(logger, 2*time.Second, ctx.Done(), func() error { - res, err := promclient.NewDefaultClient().LabelNamesInGRPC(ctx, mustURLParse(t, "http://"+addr), start, end) + res, err := promclient.NewDefaultClient().LabelNamesInGRPC(ctx, mustURLParse(t, "http://"+addr), matchers, start, end) if err != nil { return err } @@ -442,13 +468,13 @@ func labelNames(t *testing.T, ctx context.Context, addr string, start, end int64 } //nolint:unparam -func labelValues(t *testing.T, ctx context.Context, addr, label string, start, end int64, check func(res []string) bool) { +func labelValues(t *testing.T, ctx context.Context, addr, label string, matchers []storepb.LabelMatcher, start, end int64, check func(res []string) bool) { t.Helper() logger := log.NewLogfmtLogger(os.Stdout) logger = log.With(logger, "ts", log.DefaultTimestampUTC) testutil.Ok(t, runutil.RetryWithLog(logger, 2*time.Second, ctx.Done(), func() error { - res, err := promclient.NewDefaultClient().LabelValuesInGRPC(ctx, mustURLParse(t, "http://"+addr), label, start, end) + res, err := promclient.NewDefaultClient().LabelValuesInGRPC(ctx, mustURLParse(t, "http://"+addr), label, matchers, start, end) if err != nil { return err }