Query: fix exemplar proxying for receivers with multiple tenants

CHANGELOG.md

@@ -13,6 +13,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re
 ### Fixed
 
 - [#7323](https://github.com/thanos-io/thanos/pull/7323) Sidecar: wait for prometheus on startup
+- [#7326](https://github.com/thanos-io/thanos/pull/7326) Query: fixing exemplars proxy when querying stores with multiple tenants.
 
 ### Added
 

pkg/exemplars/proxy.go

@@ -19,6 +19,7 @@ import (
 	"google.golang.org/grpc/status"
 
 	"github.com/thanos-io/thanos/pkg/exemplars/exemplarspb"
+	"github.com/thanos-io/thanos/pkg/store"
 	"github.com/thanos-io/thanos/pkg/store/storepb"
 	"github.com/thanos-io/thanos/pkg/tracing"
 )
@@ -85,28 +86,21 @@ func (s *Proxy) Exemplars(req *exemplarspb.ExemplarsRequest, srv exemplarspb.Exe
 	for _, st := range s.exemplars() {
 		queryParts = queryParts[:0]
 
-	Matchers:
-		for _, matchers := range selectors {
-			matcherSet := make(map[string]struct{})
-			for _, m := range matchers {
-				for _, ls := range st.LabelSets {
-					if lv := ls.Get(m.Name); lv != "" {
-						if !m.Matches(lv) {
-							continue Matchers
-						} else {
-							// If the current matcher matches one external label,
-							// we don't add it to the current metric selector
-							// as Prometheus' Exemplars API cannot handle external labels.
-							continue
-						}
-					}
-					matcherSet[m.String()] = struct{}{}
-				}
+		for _, matcherSet := range selectors {
+			extLbls := st.LabelSets
+			if !store.LabelSetsMatch(matcherSet, extLbls...) {
+				continue
 			}
 
 			labelMatchers = labelMatchers[:0]
-			for m := range matcherSet {
-				labelMatchers = append(labelMatchers, m)
+			for _, m := range matcherSet {
+				if isExternalLabel(m.Name, extLbls) {
+					// If the current matcher matches one external label,
+					// we don't add it to the current metric selector
+					// as Prometheus' Exemplars API cannot handle external labels.
+					continue
+				}
+				labelMatchers = append(labelMatchers, m.String())
 			}
 
 			queryParts = append(queryParts, "{"+strings.Join(labelMatchers, ", ")+"}")
@@ -164,6 +158,15 @@ func (s *Proxy) Exemplars(req *exemplarspb.ExemplarsRequest, srv exemplarspb.Exe
 	return nil
 }
 
+func isExternalLabel(name string, sets []labels.Labels) bool {
+	for _, ls := range sets {
+		if ls.Get(name) != "" {
+			return true
+		}
+	}
+	return false
+}
+
 func (stream *exemplarsStream) receive(ctx context.Context) error {
 	exemplars, err := stream.client.Exemplars(ctx, stream.request)
 	if err != nil {

pkg/exemplars/proxy_test.go

@@ -22,6 +22,7 @@ import (
 	"google.golang.org/grpc/status"
 
 	"github.com/efficientgo/core/testutil"
+
 	"github.com/thanos-io/thanos/pkg/exemplars/exemplarspb"
 	"github.com/thanos-io/thanos/pkg/store/labelpb"
 	"github.com/thanos-io/thanos/pkg/store/storepb"
@@ -243,6 +244,31 @@ func TestProxy(t *testing.T) {
 				}),
 			},
 		},
+		{
+			name: "one external label matches one of the selector labels",
+			request: &exemplarspb.ExemplarsRequest{
+				Query:                   `http_request_duration_bucket{cluster="A"}`,
+				PartialResponseStrategy: storepb.PartialResponseStrategy_WARN,
+			},
+			clients: []*exemplarspb.ExemplarStore{
+				{
+					ExemplarsClient: &testExemplarClient{
+						response: exemplarspb.NewExemplarsResponse(&exemplarspb.ExemplarData{
+							SeriesLabels: labelpb.ZLabelSet{Labels: labelpb.ZLabelsFromPromLabels(labels.FromMap(map[string]string{"__name__": "http_request_duration_bucket"}))},
+							Exemplars:    []*exemplarspb.Exemplar{{Value: 1}},
+						}),
+					},
+					LabelSets: []labels.Labels{labels.FromMap(map[string]string{"cluster": "non-matching"}), labels.FromMap(map[string]string{"cluster": "A"})},
+				},
+			},
+			server: &testExemplarServer{},
+			wantResponses: []*exemplarspb.ExemplarsResponse{
+				exemplarspb.NewExemplarsResponse(&exemplarspb.ExemplarData{
+					SeriesLabels: labelpb.ZLabelSet{Labels: labelpb.ZLabelsFromPromLabels(labels.FromMap(map[string]string{"__name__": "http_request_duration_bucket"}))},
+					Exemplars:    []*exemplarspb.Exemplar{{Value: 1}},
+				}),
+			},
+		},
 		{
 			name: "external label selects stores",
 			request: &exemplarspb.ExemplarsRequest{
@@ -341,12 +367,13 @@ func TestProxy(t *testing.T) {
 			// for matched response for simplicity.
 		Outer:
 			for _, exp := range tc.wantResponses {
+				var lres *exemplarspb.ExemplarsResponse
 				for _, res := range tc.server.responses {
 					if reflect.DeepEqual(exp, res) {
 						continue Outer
 					}
 				}
-				t.Errorf("miss expected response %v", exp)
+				t.Errorf("miss expected response %v. got: %v", exp, lres)
 			}
 		})
 	}

pkg/store/proxy.go

@@ -427,7 +427,7 @@ func storeMatches(ctx context.Context, s Client, debugLogging bool, mint, maxt i
 	}
 
 	extLset := s.LabelSets()
-	if !labelSetsMatch(matchers, extLset...) {
+	if !LabelSetsMatch(matchers, extLset...) {
 		if debugLogging {
 			reason = fmt.Sprintf("external labels %v does not match request label matchers: %v", extLset, matchers)
 		}
@@ -449,16 +449,16 @@ func storeMatchDebugMetadata(s Client, storeDebugMatchers [][]*labels.Matcher) (
 
 	match := false
 	for _, sm := range storeDebugMatchers {
-		match = match || labelSetsMatch(sm, labels.FromStrings("__address__", addr))
+		match = match || LabelSetsMatch(sm, labels.FromStrings("__address__", addr))
 	}
 	if !match {
 		return false, fmt.Sprintf("__address__ %v does not match debug store metadata matchers: %v", addr, storeDebugMatchers)
 	}
 	return true, ""
 }
 
-// labelSetsMatch returns false if all label-set do not match the matchers (aka: OR is between all label-sets).
-func labelSetsMatch(matchers []*labels.Matcher, lset ...labels.Labels) bool {
+// LabelSetsMatch returns false if all label-set do not match the matchers (aka: OR is between all label-sets).
+func LabelSetsMatch(matchers []*labels.Matcher, lset ...labels.Labels) bool {
 	if len(lset) == 0 {
 		return true
 	}