From 9ea595776df70ee238044fc758e7246be11211fd Mon Sep 17 00:00:00 2001 From: Hitanshu Mehta Date: Thu, 8 Apr 2021 13:51:25 +0530 Subject: [PATCH 01/16] Add Info server in Sidecar and StoreGW Signed-off-by: Hitanshu Mehta --- cmd/thanos/sidecar.go | 30 ++++++++++++++++- cmd/thanos/store.go | 20 +++++++++++ pkg/info/info.go | 54 +++++++++++++++++++++++++++++ pkg/store/bucket.go | 18 ++++++---- pkg/store/prometheus.go | 10 +++--- pkg/store/prometheus_test.go | 4 +-- scripts/insecure_grpcurl_info.sh | 32 ++++++++++++++++++ test/e2e/info_api_test.go | 58 ++++++++++++++++++++++++++++++++ 8 files changed, 211 insertions(+), 15 deletions(-) create mode 100644 pkg/info/info.go create mode 100755 scripts/insecure_grpcurl_info.sh create mode 100644 test/e2e/info_api_test.go diff --git a/cmd/thanos/sidecar.go b/cmd/thanos/sidecar.go index 8584492b4f..82967d19a2 100644 --- a/cmd/thanos/sidecar.go +++ b/cmd/thanos/sidecar.go @@ -30,6 +30,8 @@ import ( "github.com/thanos-io/thanos/pkg/extkingpin" "github.com/thanos-io/thanos/pkg/extprom" "github.com/thanos-io/thanos/pkg/httpconfig" + "github.com/thanos-io/thanos/pkg/info" + "github.com/thanos-io/thanos/pkg/info/infopb" "github.com/thanos-io/thanos/pkg/logging" meta "github.com/thanos-io/thanos/pkg/metadata" thanosmodel "github.com/thanos-io/thanos/pkg/model" @@ -43,6 +45,7 @@ import ( httpserver "github.com/thanos-io/thanos/pkg/server/http" "github.com/thanos-io/thanos/pkg/shipper" "github.com/thanos-io/thanos/pkg/store" + "github.com/thanos-io/thanos/pkg/store/labelpb" "github.com/thanos-io/thanos/pkg/targets" "github.com/thanos-io/thanos/pkg/tls" "github.com/thanos-io/thanos/pkg/tracing" @@ -241,12 +244,37 @@ func runSidecar( return errors.Wrap(err, "setup gRPC server") } + examplarSrv := exemplars.NewPrometheus(conf.prometheus.url, c, m.Labels) + + infoSrv := info.NewInfoServer( + infopb.ComponentType_SIDECAR, + func() []labelpb.ZLabelSet { + return promStore.LabelSet() + }, + func() *infopb.StoreInfo { + mint, maxt := promStore.Timestamps() + return &infopb.StoreInfo{ + MinTime: mint, + MaxTime: maxt, + } + }, + func() *infopb.ExemplarsInfo { + // Currently Exemplars API does not expose metadata such as min/max time, + // so we are using default minimum and maximum possible values as min/max time. + return &infopb.ExemplarsInfo{ + MinTime: time.Unix(math.MinInt64/1000+62135596801, 0).UTC().Unix(), + MaxTime: time.Unix(math.MaxInt64/1000-62135596801, 999999999).UTC().Unix(), + } + }, + ) + s := grpcserver.New(logger, reg, tracer, grpcLogOpts, tagOpts, comp, grpcProbe, grpcserver.WithServer(store.RegisterStoreServer(promStore)), grpcserver.WithServer(rules.RegisterRulesServer(rules.NewPrometheus(conf.prometheus.url, c, m.Labels))), grpcserver.WithServer(targets.RegisterTargetsServer(targets.NewPrometheus(conf.prometheus.url, c, m.Labels))), grpcserver.WithServer(meta.RegisterMetadataServer(meta.NewPrometheus(conf.prometheus.url, c))), - grpcserver.WithServer(exemplars.RegisterExemplarsServer(exemplars.NewPrometheus(conf.prometheus.url, c, m.Labels))), + grpcserver.WithServer(exemplars.RegisterExemplarsServer(examplarSrv)), + grpcserver.WithServer(info.RegisterInfoServer(infoSrv)), grpcserver.WithListen(conf.grpc.bindAddress), grpcserver.WithGracePeriod(time.Duration(conf.grpc.gracePeriod)), grpcserver.WithTLSConfig(tlsCfg), diff --git a/cmd/thanos/store.go b/cmd/thanos/store.go index 25310f9ef3..748fba715d 100644 --- a/cmd/thanos/store.go +++ b/cmd/thanos/store.go @@ -31,6 +31,8 @@ import ( "github.com/thanos-io/thanos/pkg/extprom" extpromhttp "github.com/thanos-io/thanos/pkg/extprom/http" "github.com/thanos-io/thanos/pkg/gate" + "github.com/thanos-io/thanos/pkg/info" + "github.com/thanos-io/thanos/pkg/info/infopb" "github.com/thanos-io/thanos/pkg/logging" "github.com/thanos-io/thanos/pkg/model" "github.com/thanos-io/thanos/pkg/objstore/client" @@ -40,6 +42,7 @@ import ( httpserver "github.com/thanos-io/thanos/pkg/server/http" "github.com/thanos-io/thanos/pkg/store" storecache "github.com/thanos-io/thanos/pkg/store/cache" + "github.com/thanos-io/thanos/pkg/store/labelpb" "github.com/thanos-io/thanos/pkg/tls" "github.com/thanos-io/thanos/pkg/ui" ) @@ -382,6 +385,22 @@ func runStore( cancel() }) } + + infoSrv := info.NewInfoServer( + infopb.ComponentType_STORE, + func() []labelpb.ZLabelSet { + return bs.LabelSet() + }, + func() *infopb.StoreInfo { + mint, maxt := bs.TimeRange() + return &infopb.StoreInfo{ + MinTime: mint, + MaxTime: maxt, + } + }, + func() *infopb.ExemplarsInfo { return nil }, + ) + // Start query (proxy) gRPC StoreAPI. { tlsCfg, err := tls.NewServerConfig(log.With(logger, "protocol", "gRPC"), conf.grpcConfig.tlsSrvCert, conf.grpcConfig.tlsSrvKey, conf.grpcConfig.tlsSrvClientCA) @@ -391,6 +410,7 @@ func runStore( s := grpcserver.New(logger, reg, tracer, grpcLogOpts, tagOpts, conf.component, grpcProbe, grpcserver.WithServer(store.RegisterStoreServer(bs)), + grpcserver.WithServer(info.RegisterInfoServer(infoSrv)), grpcserver.WithListen(conf.grpcConfig.bindAddress), grpcserver.WithGracePeriod(time.Duration(conf.grpcConfig.gracePeriod)), grpcserver.WithTLSConfig(tlsCfg), diff --git a/pkg/info/info.go b/pkg/info/info.go new file mode 100644 index 0000000000..6c78135f28 --- /dev/null +++ b/pkg/info/info.go @@ -0,0 +1,54 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + +package info + +import ( + "context" + + "github.com/go-kit/kit/log" + "github.com/thanos-io/thanos/pkg/info/infopb" + "github.com/thanos-io/thanos/pkg/store/labelpb" + "google.golang.org/grpc" +) + +type InfoServer struct { + infopb.UnimplementedInfoServer + + logger log.Logger + component infopb.ComponentType + + getLabelSet func() []labelpb.ZLabelSet + getStoreInfo func() *infopb.StoreInfo + getExemplarsInfo func() *infopb.ExemplarsInfo +} + +func NewInfoServer( + component infopb.ComponentType, + getLabelSet func() []labelpb.ZLabelSet, + getStoreInfo func() *infopb.StoreInfo, + getExemplarsInfo func() *infopb.ExemplarsInfo, +) *InfoServer { + return &InfoServer{ + component: component, + getLabelSet: getLabelSet, + getStoreInfo: getStoreInfo, + getExemplarsInfo: getExemplarsInfo, + } +} + +// RegisterInfoServer register info server. +func RegisterInfoServer(infoSrv infopb.InfoServer) func(*grpc.Server) { + return func(s *grpc.Server) { + infopb.RegisterInfoServer(s, infoSrv) + } +} + +func (srv *InfoServer) Info(ctx context.Context, req *infopb.InfoRequest) (*infopb.InfoResponse, error) { + return &infopb.InfoResponse{ + LabelSets: srv.getLabelSet(), + ComponentType: srv.component, + StoreInfo: srv.getStoreInfo(), + ExemplarsInfo: srv.getExemplarsInfo(), + }, nil +} diff --git a/pkg/store/bucket.go b/pkg/store/bucket.go index 0d8066e2bf..206820cddb 100644 --- a/pkg/store/bucket.go +++ b/pkg/store/bucket.go @@ -681,6 +681,16 @@ func (s *BucketStore) TimeRange() (mint, maxt int64) { return mint, maxt } +func (s *BucketStore) LabelSet() []labelpb.ZLabelSet { + labelSets := s.advLabelSets + + if s.enableCompatibilityLabel && len(labelSets) > 0 { + labelSets = append(labelSets, labelpb.ZLabelSet{Labels: []labelpb.ZLabel{{Name: CompatibilityTypeLabelName, Value: "store"}}}) + } + + return labelSets +} + // Info implements the storepb.StoreServer interface. func (s *BucketStore) Info(context.Context, *storepb.InfoRequest) (*storepb.InfoResponse, error) { mint, maxt := s.TimeRange() @@ -691,14 +701,8 @@ func (s *BucketStore) Info(context.Context, *storepb.InfoRequest) (*storepb.Info } s.mtx.RLock() - res.LabelSets = s.advLabelSets + res.LabelSets = s.LabelSet() s.mtx.RUnlock() - - if s.enableCompatibilityLabel && len(res.LabelSets) > 0 { - // This is for compatibility with Querier v0.7.0. - // See query.StoreCompatibilityTypeLabelName comment for details. - res.LabelSets = append(res.LabelSets, labelpb.ZLabelSet{Labels: []labelpb.ZLabel{{Name: CompatibilityTypeLabelName, Value: "store"}}}) - } return res, nil } diff --git a/pkg/store/prometheus.go b/pkg/store/prometheus.go index ec3525ad65..be55694c44 100644 --- a/pkg/store/prometheus.go +++ b/pkg/store/prometheus.go @@ -48,8 +48,8 @@ type PrometheusStore struct { buffers sync.Pool component component.StoreAPI externalLabelsFn func() labels.Labels - timestamps func() (mint int64, maxt int64) promVersion func() string + Timestamps func() (mint int64, maxt int64) remoteReadAcceptableResponses []prompb.ReadRequest_ResponseType @@ -72,7 +72,7 @@ func NewPrometheusStore( baseURL *url.URL, component component.StoreAPI, externalLabelsFn func() labels.Labels, - timestamps func() (mint int64, maxt int64), + Timestamps func() (mint int64, maxt int64), promVersion func() string, ) (*PrometheusStore, error) { if logger == nil { @@ -84,8 +84,8 @@ func NewPrometheusStore( client: client, component: component, externalLabelsFn: externalLabelsFn, - timestamps: timestamps, promVersion: promVersion, + Timestamps: Timestamps, remoteReadAcceptableResponses: []prompb.ReadRequest_ResponseType{prompb.ReadRequest_STREAMED_XOR_CHUNKS, prompb.ReadRequest_SAMPLES}, buffers: sync.Pool{New: func() interface{} { b := make([]byte, 0, initialBufSize) @@ -107,7 +107,7 @@ func NewPrometheusStore( // This is fine for now, but might be needed in future. func (p *PrometheusStore) Info(_ context.Context, _ *storepb.InfoRequest) (*storepb.InfoResponse, error) { lset := p.externalLabelsFn() - mint, maxt := p.timestamps() + mint, maxt := p.Timestamps() res := &storepb.InfoResponse{ Labels: make([]labelpb.ZLabel, 0, len(lset)), @@ -153,7 +153,7 @@ func (p *PrometheusStore) Series(r *storepb.SeriesRequest, s storepb.Store_Serie } // Don't ask for more than available time. This includes potential `minTime` flag limit. - availableMinTime, _ := p.timestamps() + availableMinTime, _ := p.Timestamps() if r.MinTime < availableMinTime { r.MinTime = availableMinTime } diff --git a/pkg/store/prometheus_test.go b/pkg/store/prometheus_test.go index a532a70db6..07090a16b0 100644 --- a/pkg/store/prometheus_test.go +++ b/pkg/store/prometheus_test.go @@ -312,8 +312,8 @@ func TestPrometheusStore_SeriesLabels_e2e(t *testing.T) { Matchers: []storepb.LabelMatcher{ {Type: storepb.LabelMatcher_EQ, Name: "job", Value: "test"}, }, - MinTime: func() int64 { minTime, _ := promStore.timestamps(); return minTime }(), - MaxTime: func() int64 { _, maxTime := promStore.timestamps(); return maxTime }(), + MinTime: func() int64 { minTime, _ := promStore.Timestamps(); return minTime }(), + MaxTime: func() int64 { _, maxTime := promStore.Timestamps(); return maxTime }(), }, expected: []storepb.Series{ { diff --git a/scripts/insecure_grpcurl_info.sh b/scripts/insecure_grpcurl_info.sh new file mode 100755 index 0000000000..0d75b02053 --- /dev/null +++ b/scripts/insecure_grpcurl_info.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +HELP=' +insecure_grpcurl_info.sh allows you to use call StoreAPI.Series gRPC method and receive streamed series in JSON format. + +Usage: + # Start some example Thanos component that exposes gRPC or use existing one. To start example one run: `thanos query &` + bash scripts/insecure_grpcurl_info.sh localhost:10901 '"'"'[{"type": 0, "name": "__name__", "value":"go_goroutines"}]'"'"' 0 10 +' + +INFO_API_HOSTPORT=$1 +if [ -z "${INFO_API_HOSTPORT}" ]; then + echo '$1 is missing (INFO_API_HOSTPORT). Expected host:port string for the target StoreAPI to grpcurl against, e.g. localhost:10901' + echo "${HELP}" + exit 1 +fi + +go install github.com/fullstorydev/grpcurl/cmd/grpcurl + +INFO_REQUEST='{}' + +GOGOPROTO_ROOT="$(GO111MODULE=on go list -f '{{ .Dir }}' -m github.com/gogo/protobuf)" + +cd $DIR/../pkg/ || exit +grpcurl \ + -import-path="${GOGOPROTO_ROOT}" \ + -import-path=. \ + -proto=info/infopb/rpc.proto \ + -plaintext \ + -d="${INFO_REQUEST}" "${INFO_API_HOSTPORT}" thanos.Info/Info diff --git a/test/e2e/info_api_test.go b/test/e2e/info_api_test.go new file mode 100644 index 0000000000..208ef2ee3a --- /dev/null +++ b/test/e2e/info_api_test.go @@ -0,0 +1,58 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + +package e2e_test + +import ( + "context" + "testing" + + "github.com/cortexproject/cortex/integration/e2e" + "github.com/thanos-io/thanos/pkg/info/infopb" + "github.com/thanos-io/thanos/pkg/store/labelpb" + "github.com/thanos-io/thanos/pkg/testutil" + "github.com/thanos-io/thanos/test/e2e/e2ethanos" + "google.golang.org/grpc" +) + +func TestInfoAPI_WithSidecar(t *testing.T) { + t.Parallel() + + netName := "e2e_test_info_with_sidecar" + + s, err := e2e.NewScenario(netName) + testutil.Ok(t, err) + t.Cleanup(e2ethanos.CleanScenario(t, s)) + + prom, sidecar, err := e2ethanos.NewPrometheusWithSidecar( + s.SharedDir(), + netName, + "prom", + defaultPromConfig("ha", 0, "", ""), + e2ethanos.DefaultPrometheusImage(), + ) + testutil.Ok(t, err) + testutil.Ok(t, s.StartAndWaitReady(prom, sidecar)) + + // Create grpc Client + conn, err := grpc.Dial(sidecar.GRPCEndpoint(), grpc.WithInsecure()) + testutil.Ok(t, err) + defer conn.Close() + + client := infopb.NewInfoClient(conn) + + res, err := client.Info(context.Background(), &infopb.InfoRequest{}) + testutil.Ok(t, err) + testutil.Equals(t, res, infopb.InfoResponse{ + LabelSets: []labelpb.ZLabelSet{}, + ComponentType: infopb.ComponentType_SIDECAR, + StoreInfo: &infopb.StoreInfo{ + MinTime: -62167219200000, + MaxTime: 9223372036854775807, + }, + ExemplarsInfo: &infopb.ExemplarsInfo{ + MinTime: -9223309901257974, + MaxTime: 9223309901257974, + }, + }) +} From a076da0188ae8c6b371798bcda939e56ada06ba3 Mon Sep 17 00:00:00 2001 From: Hitanshu Mehta Date: Thu, 8 Apr 2021 13:51:25 +0530 Subject: [PATCH 02/16] Add `--endpoint` in querier Signed-off-by: Hitanshu Mehta --- cmd/thanos/query.go | 22 + cmd/thanos/sidecar.go | 11 +- cmd/thanos/store.go | 5 +- pkg/info/info.go | 59 +- pkg/query/storeset.go | 874 ++++++++++++++++++++ pkg/query/storeset_test.go | 1331 ++++++++++++++++++++++++++++++ scripts/insecure_grpcurl_info.sh | 32 - test/e2e/info_api_test.go | 58 -- 8 files changed, 2283 insertions(+), 109 deletions(-) create mode 100644 pkg/query/storeset.go create mode 100644 pkg/query/storeset_test.go delete mode 100755 scripts/insecure_grpcurl_info.sh delete mode 100644 test/e2e/info_api_test.go diff --git a/cmd/thanos/query.go b/cmd/thanos/query.go index 2a071884bb..f0c3e9835d 100644 --- a/cmd/thanos/query.go +++ b/cmd/thanos/query.go @@ -101,6 +101,9 @@ func registerQuery(app *extkingpin.App) { selectorLabels := cmd.Flag("selector-label", "Query selector labels that will be exposed in info endpoint (repeated)."). PlaceHolder("=\"\"").Strings() + endpoints := cmd.Flag("endpoint", "Addresses of statically configured Thanos API servers (repeatable). The scheme may be prefixed with 'dns+' or 'dnssrv+' to detect Thanos API servers through respective DNS lookups."). + PlaceHolder("").Strings() + stores := cmd.Flag("store", "Addresses of statically configured store API servers (repeatable). The scheme may be prefixed with 'dns+' or 'dnssrv+' to detect store API servers through respective DNS lookups."). PlaceHolder("").Strings() @@ -264,6 +267,7 @@ func registerQuery(app *extkingpin.App) { *queryReplicaLabels, selectorLset, getFlagsMap(cmd.Flags()), + *endpoints, *stores, *ruleEndpoints, *targetEndpoints, @@ -329,6 +333,7 @@ func runQuery( queryReplicaLabels []string, selectorLset labels.Labels, flagsMap map[string]string, + endpoints []string, storeAddrs []string, ruleAddrs []string, targetAddrs []string, @@ -376,6 +381,12 @@ func runQuery( } } + dnsInfoProvider := dns.NewProvider( + logger, + extprom.WrapRegistererWithPrefix("thanos_query_info_apis_", reg), + dns.ResolverType(dnsSDResolver), + ) + dnsRuleProvider := dns.NewProvider( logger, extprom.WrapRegistererWithPrefix("thanos_query_rule_apis_", reg), @@ -422,6 +433,13 @@ func runQuery( return specs }, + func() (specs []query.InfoSpec) { + for _, addr := range dnsInfoProvider.Addresses() { + specs = append(specs, query.NewGRPCStoreSpec(addr, false)) + } + + return specs + }, dialOpts, unhealthyStoreTimeout, ) @@ -527,6 +545,10 @@ func runQuery( if err := dnsExemplarProvider.Resolve(resolveCtx, exemplarAddrs); err != nil { level.Error(logger).Log("msg", "failed to resolve addresses for exemplarsAPI", "err", err) } + if err := dnsInfoProvider.Resolve(resolveCtx, endpoints); err != nil { + level.Error(logger).Log("msg", "failed to resolve addresses passed using endpoint flag", "err", err) + + } return nil }) }, func(error) { diff --git a/cmd/thanos/sidecar.go b/cmd/thanos/sidecar.go index 82967d19a2..269e08ede4 100644 --- a/cmd/thanos/sidecar.go +++ b/cmd/thanos/sidecar.go @@ -247,7 +247,7 @@ func runSidecar( examplarSrv := exemplars.NewPrometheus(conf.prometheus.url, c, m.Labels) infoSrv := info.NewInfoServer( - infopb.ComponentType_SIDECAR, + component.Sidecar.String(), func() []labelpb.ZLabelSet { return promStore.LabelSet() }, @@ -266,6 +266,15 @@ func runSidecar( MaxTime: time.Unix(math.MaxInt64/1000-62135596801, 999999999).UTC().Unix(), } }, + func() *infopb.RulesInfo { + return &infopb.RulesInfo{} + }, + func() *infopb.TargetsInfo { + return &infopb.TargetsInfo{} + }, + func() *infopb.MetricMetadataInfo { + return &infopb.MetricMetadataInfo{} + }, ) s := grpcserver.New(logger, reg, tracer, grpcLogOpts, tagOpts, comp, grpcProbe, diff --git a/cmd/thanos/store.go b/cmd/thanos/store.go index 748fba715d..591c2a96ae 100644 --- a/cmd/thanos/store.go +++ b/cmd/thanos/store.go @@ -387,7 +387,7 @@ func runStore( } infoSrv := info.NewInfoServer( - infopb.ComponentType_STORE, + component.Store.String(), func() []labelpb.ZLabelSet { return bs.LabelSet() }, @@ -399,6 +399,9 @@ func runStore( } }, func() *infopb.ExemplarsInfo { return nil }, + nil, + nil, + nil, ) // Start query (proxy) gRPC StoreAPI. diff --git a/pkg/info/info.go b/pkg/info/info.go index 6c78135f28..f2408edce4 100644 --- a/pkg/info/info.go +++ b/pkg/info/info.go @@ -6,7 +6,6 @@ package info import ( "context" - "github.com/go-kit/kit/log" "github.com/thanos-io/thanos/pkg/info/infopb" "github.com/thanos-io/thanos/pkg/store/labelpb" "google.golang.org/grpc" @@ -15,25 +14,33 @@ import ( type InfoServer struct { infopb.UnimplementedInfoServer - logger log.Logger - component infopb.ComponentType + component string - getLabelSet func() []labelpb.ZLabelSet - getStoreInfo func() *infopb.StoreInfo - getExemplarsInfo func() *infopb.ExemplarsInfo + getLabelSet func() []labelpb.ZLabelSet + getStoreInfo func() *infopb.StoreInfo + getExemplarsInfo func() *infopb.ExemplarsInfo + getRulesInfo func() *infopb.RulesInfo + getTargetsInfo func() *infopb.TargetsInfo + getMetricMetadataInfo func() *infopb.MetricMetadataInfo } func NewInfoServer( - component infopb.ComponentType, + component string, getLabelSet func() []labelpb.ZLabelSet, getStoreInfo func() *infopb.StoreInfo, getExemplarsInfo func() *infopb.ExemplarsInfo, + getRulesInfo func() *infopb.RulesInfo, + getTargetsInfo func() *infopb.TargetsInfo, + getMetricMetadataInfo func() *infopb.MetricMetadataInfo, ) *InfoServer { return &InfoServer{ - component: component, - getLabelSet: getLabelSet, - getStoreInfo: getStoreInfo, - getExemplarsInfo: getExemplarsInfo, + component: component, + getLabelSet: getLabelSet, + getStoreInfo: getStoreInfo, + getExemplarsInfo: getExemplarsInfo, + getRulesInfo: getRulesInfo, + getTargetsInfo: getTargetsInfo, + getMetricMetadataInfo: getMetricMetadataInfo, } } @@ -45,10 +52,28 @@ func RegisterInfoServer(infoSrv infopb.InfoServer) func(*grpc.Server) { } func (srv *InfoServer) Info(ctx context.Context, req *infopb.InfoRequest) (*infopb.InfoResponse, error) { - return &infopb.InfoResponse{ - LabelSets: srv.getLabelSet(), - ComponentType: srv.component, - StoreInfo: srv.getStoreInfo(), - ExemplarsInfo: srv.getExemplarsInfo(), - }, nil + + if srv.getRulesInfo == nil { + srv.getRulesInfo = func() *infopb.RulesInfo { return nil } + } + + if srv.getTargetsInfo == nil { + srv.getTargetsInfo = func() *infopb.TargetsInfo { return nil } + } + + if srv.getMetricMetadataInfo == nil { + srv.getMetricMetadataInfo = func() *infopb.MetricMetadataInfo { return nil } + } + + resp := &infopb.InfoResponse{ + LabelSets: srv.getLabelSet(), + ComponentType: srv.component, + Store: srv.getStoreInfo(), + Exemplars: srv.getExemplarsInfo(), + Rules: srv.getRulesInfo(), + Targets: srv.getTargetsInfo(), + MetricMetadata: srv.getMetricMetadataInfo(), + } + + return resp, nil } diff --git a/pkg/query/storeset.go b/pkg/query/storeset.go new file mode 100644 index 0000000000..5568f1fa0a --- /dev/null +++ b/pkg/query/storeset.go @@ -0,0 +1,874 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + +package query + +import ( + "context" + "encoding/json" + "fmt" + "math" + "sort" + "sync" + "time" + + "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" + "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/prometheus/pkg/labels" + "github.com/thanos-io/thanos/pkg/exemplars/exemplarspb" + "github.com/thanos-io/thanos/pkg/info/infopb" + "google.golang.org/grpc" + + "github.com/thanos-io/thanos/pkg/component" + "github.com/thanos-io/thanos/pkg/metadata/metadatapb" + "github.com/thanos-io/thanos/pkg/rules/rulespb" + "github.com/thanos-io/thanos/pkg/runutil" + "github.com/thanos-io/thanos/pkg/store" + "github.com/thanos-io/thanos/pkg/store/labelpb" + "github.com/thanos-io/thanos/pkg/store/storepb" + "github.com/thanos-io/thanos/pkg/targets/targetspb" +) + +const ( + unhealthyStoreMessage = "removing store because it's unhealthy or does not exist" +) + +type StoreSpec interface { + // Addr returns StoreAPI Address for the store spec. It is used as ID for store. + Addr() string + // Metadata returns current labels, store type and min, max ranges for store. + // It can change for every call for this method. + // If metadata call fails we assume that store is no longer accessible and we should not use it. + // NOTE: It is implementation responsibility to retry until context timeout, but a caller responsibility to manage + // given store connection. + Metadata(ctx context.Context, client storepb.StoreClient) (labelSets []labels.Labels, mint int64, maxt int64, storeType component.StoreAPI, err error) + + // StrictStatic returns true if the StoreAPI has been statically defined and it is under a strict mode. + StrictStatic() bool +} + +type RuleSpec interface { + // Addr returns RulesAPI Address for the rules spec. It is used as its ID. + Addr() string +} + +type TargetSpec interface { + // Addr returns TargetsAPI Address for the targets spec. It is used as its ID. + Addr() string +} + +type MetadataSpec interface { + // Addr returns MetadataAPI Address for the metadata spec. It is used as its ID. + Addr() string +} + +type ExemplarSpec interface { + // Addr returns ExemplarsAPI Address for the exemplars spec. It is used as its ID. + Addr() string +} + +type InfoSpec interface { + // Addr returns InfoAPI Address for the info spec. It is used as its ID. + Addr() string +} + +// stringError forces the error to be a string +// when marshaled into a JSON. +type stringError struct { + originalErr error +} + +// MarshalJSON marshals the error into a string form. +func (e *stringError) MarshalJSON() ([]byte, error) { + return json.Marshal(e.originalErr.Error()) +} + +// Error returns the original underlying error. +func (e *stringError) Error() string { + return e.originalErr.Error() +} + +type StoreStatus struct { + Name string `json:"name"` + LastCheck time.Time `json:"lastCheck"` + LastError *stringError `json:"lastError"` + LabelSets []labels.Labels `json:"labelSets"` + StoreType component.StoreAPI `json:"-"` + MinTime int64 `json:"minTime"` + MaxTime int64 `json:"maxTime"` +} + +type grpcStoreSpec struct { + addr string + strictstatic bool +} + +// NewGRPCStoreSpec creates store pure gRPC spec. +// It uses Info gRPC call to get Metadata. +func NewGRPCStoreSpec(addr string, strictstatic bool) StoreSpec { + return &grpcStoreSpec{addr: addr, strictstatic: strictstatic} +} + +// StrictStatic returns true if the StoreAPI has been statically defined and it is under a strict mode. +func (s *grpcStoreSpec) StrictStatic() bool { + return s.strictstatic +} + +func (s *grpcStoreSpec) Addr() string { + // API addr should not change between state changes. + return s.addr +} + +// Metadata method for gRPC store API tries to reach host Info method until context timeout. If we are unable to get metadata after +// that time, we assume that the host is unhealthy and return error. +func (s *grpcStoreSpec) Metadata(ctx context.Context, client storepb.StoreClient) (labelSets []labels.Labels, mint int64, maxt int64, Type component.StoreAPI, err error) { + resp, err := client.Info(ctx, &storepb.InfoRequest{}, grpc.WaitForReady(true)) + if err != nil { + return nil, 0, 0, nil, errors.Wrapf(err, "fetching store info from %s", s.addr) + } + if len(resp.LabelSets) == 0 && len(resp.Labels) > 0 { + resp.LabelSets = []labelpb.ZLabelSet{{Labels: resp.Labels}} + } + + labelSets = make([]labels.Labels, 0, len(resp.LabelSets)) + for _, ls := range resp.LabelSets { + labelSets = append(labelSets, ls.PromLabels()) + } + return labelSets, resp.MinTime, resp.MaxTime, component.FromProto(resp.StoreType), nil +} + +// storeSetNodeCollector is a metric collector reporting the number of available storeAPIs for Querier. +// A Collector is required as we want atomic updates for all 'thanos_store_nodes_grpc_connections' series. +type storeSetNodeCollector struct { + mtx sync.Mutex + storeNodes map[component.StoreAPI]map[string]int + storePerExtLset map[string]int + + connectionsDesc *prometheus.Desc +} + +func newStoreSetNodeCollector() *storeSetNodeCollector { + return &storeSetNodeCollector{ + storeNodes: map[component.StoreAPI]map[string]int{}, + connectionsDesc: prometheus.NewDesc( + "thanos_store_nodes_grpc_connections", + "Number of gRPC connection to Store APIs. Opened connection means healthy store APIs available for Querier.", + []string{"external_labels", "store_type"}, nil, + ), + } +} + +func (c *storeSetNodeCollector) Update(nodes map[component.StoreAPI]map[string]int) { + storeNodes := make(map[component.StoreAPI]map[string]int, len(nodes)) + storePerExtLset := map[string]int{} + + for k, v := range nodes { + storeNodes[k] = make(map[string]int, len(v)) + for kk, vv := range v { + storePerExtLset[kk] += vv + storeNodes[k][kk] = vv + } + } + + c.mtx.Lock() + defer c.mtx.Unlock() + c.storeNodes = storeNodes + c.storePerExtLset = storePerExtLset +} + +func (c *storeSetNodeCollector) Describe(ch chan<- *prometheus.Desc) { + ch <- c.connectionsDesc +} + +func (c *storeSetNodeCollector) Collect(ch chan<- prometheus.Metric) { + c.mtx.Lock() + defer c.mtx.Unlock() + + for storeType, occurrencesPerExtLset := range c.storeNodes { + for externalLabels, occurrences := range occurrencesPerExtLset { + var storeTypeStr string + if storeType != nil { + storeTypeStr = storeType.String() + } + ch <- prometheus.MustNewConstMetric(c.connectionsDesc, prometheus.GaugeValue, float64(occurrences), externalLabels, storeTypeStr) + } + } +} + +// StoreSet maintains a set of active stores. It is backed up by Store Specifications that are dynamically fetched on +// every Update() call. +type StoreSet struct { + logger log.Logger + + // Store specifications can change dynamically. If some store is missing from the list, we assuming it is no longer + // accessible and we close gRPC client for it. + infoSpec func() []InfoSpec + storeSpecs func() []StoreSpec + ruleSpecs func() []RuleSpec + targetSpecs func() []TargetSpec + metadataSpecs func() []MetadataSpec + exemplarSpecs func() []ExemplarSpec + dialOpts []grpc.DialOption + gRPCInfoCallTimeout time.Duration + + updateMtx sync.Mutex + storesMtx sync.RWMutex + storesStatusesMtx sync.RWMutex + + // Main map of stores currently used for fanout. + stores map[string]*storeRef + storesMetric *storeSetNodeCollector + + // Map of statuses used only by UI. + storeStatuses map[string]*StoreStatus + unhealthyStoreTimeout time.Duration +} + +// NewStoreSet returns a new set of store APIs and potentially Rules APIs from given specs. +func NewStoreSet( + logger log.Logger, + reg *prometheus.Registry, + storeSpecs func() []StoreSpec, + ruleSpecs func() []RuleSpec, + targetSpecs func() []TargetSpec, + metadataSpecs func() []MetadataSpec, + exemplarSpecs func() []ExemplarSpec, + infoSpecs func() []InfoSpec, + dialOpts []grpc.DialOption, + unhealthyStoreTimeout time.Duration, +) *StoreSet { + storesMetric := newStoreSetNodeCollector() + if reg != nil { + reg.MustRegister(storesMetric) + } + + if logger == nil { + logger = log.NewNopLogger() + } + + if infoSpecs == nil { + infoSpecs = func() []InfoSpec { return nil } + } + + if storeSpecs == nil { + storeSpecs = func() []StoreSpec { return nil } + } + if ruleSpecs == nil { + ruleSpecs = func() []RuleSpec { return nil } + } + if targetSpecs == nil { + targetSpecs = func() []TargetSpec { return nil } + } + if metadataSpecs == nil { + metadataSpecs = func() []MetadataSpec { return nil } + } + if exemplarSpecs == nil { + exemplarSpecs = func() []ExemplarSpec { return nil } + } + + ss := &StoreSet{ + logger: log.With(logger, "component", "storeset"), + infoSpec: infoSpecs, + storeSpecs: storeSpecs, + ruleSpecs: ruleSpecs, + targetSpecs: targetSpecs, + metadataSpecs: metadataSpecs, + exemplarSpecs: exemplarSpecs, + dialOpts: dialOpts, + storesMetric: storesMetric, + gRPCInfoCallTimeout: 5 * time.Second, + stores: make(map[string]*storeRef), + storeStatuses: make(map[string]*StoreStatus), + unhealthyStoreTimeout: unhealthyStoreTimeout, + } + return ss +} + +// TODO(bwplotka): Consider moving storeRef out of this package and renaming it, as it also supports rules API. +type storeRef struct { + storepb.StoreClient + + mtx sync.RWMutex + cc *grpc.ClientConn + addr string + // If rule is not nil, then this store also supports rules API. + rule rulespb.RulesClient + metadata metadatapb.MetadataClient + + // If exemplar is not nil, then this store also support exemplars API. + exemplar exemplarspb.ExemplarsClient + + // If target is not nil, then this store also supports targets API. + target targetspb.TargetsClient + + // If info is not nil, then this store also supports Info API. + info infopb.InfoClient + + // Meta (can change during runtime). + labelSets []labels.Labels + storeType component.StoreAPI + minTime int64 + maxTime int64 + + logger log.Logger +} + +func (s *storeRef) Update(labelSets []labels.Labels, minTime int64, maxTime int64, storeType component.StoreAPI, rule rulespb.RulesClient, target targetspb.TargetsClient, metadata metadatapb.MetadataClient, exemplar exemplarspb.ExemplarsClient) { + s.mtx.Lock() + defer s.mtx.Unlock() + + s.storeType = storeType + s.labelSets = labelSets + s.minTime = minTime + s.maxTime = maxTime + s.rule = rule + s.target = target + s.metadata = metadata + s.exemplar = exemplar +} + +func (s *storeRef) UpdateWithStore(labelSets []labels.Labels, minTime int64, maxTime int64, storeType component.StoreAPI, store storepb.StoreClient, rule rulespb.RulesClient, target targetspb.TargetsClient, metadata metadatapb.MetadataClient, exemplar exemplarspb.ExemplarsClient) { + s.mtx.Lock() + defer s.mtx.Unlock() + + s.storeType = storeType + s.labelSets = labelSets + s.minTime = minTime + s.maxTime = maxTime + s.StoreClient = store + s.rule = rule + s.target = target + s.metadata = metadata + s.exemplar = exemplar +} + +func (s *storeRef) StoreType() component.StoreAPI { + s.mtx.RLock() + defer s.mtx.RUnlock() + + return s.storeType +} + +func (s *storeRef) HasRulesAPI() bool { + s.mtx.RLock() + defer s.mtx.RUnlock() + + return s.rule != nil +} + +func (s *storeRef) HasTargetsAPI() bool { + s.mtx.RLock() + defer s.mtx.RUnlock() + + return s.target != nil +} + +func (s *storeRef) HasMetadataAPI() bool { + s.mtx.RLock() + defer s.mtx.RUnlock() + + return s.metadata != nil +} + +func (s *storeRef) HasExemplarsAPI() bool { + s.mtx.RLock() + defer s.mtx.RUnlock() + + return s.exemplar != nil +} + +func (s *storeRef) LabelSets() []labels.Labels { + s.mtx.RLock() + defer s.mtx.RUnlock() + + labelSet := make([]labels.Labels, 0, len(s.labelSets)) + for _, ls := range s.labelSets { + if len(ls) == 0 { + continue + } + // Compatibility label for Queriers pre 0.8.1. Filter it out now. + if ls[0].Name == store.CompatibilityTypeLabelName { + continue + } + labelSet = append(labelSet, ls.Copy()) + } + return labelSet +} + +func (s *storeRef) TimeRange() (mint int64, maxt int64) { + s.mtx.RLock() + defer s.mtx.RUnlock() + + return s.minTime, s.maxTime +} + +func (s *storeRef) String() string { + mint, maxt := s.TimeRange() + return fmt.Sprintf("Addr: %s LabelSets: %v Mint: %d Maxt: %d", s.addr, labelpb.PromLabelSetsToString(s.LabelSets()), mint, maxt) +} + +func (s *storeRef) Addr() string { + return s.addr +} + +func (s *storeRef) Close() { + runutil.CloseWithLogOnErr(s.logger, s.cc, fmt.Sprintf("store %v connection close", s.addr)) +} + +func newStoreAPIStats() map[component.StoreAPI]map[string]int { + nodes := make(map[component.StoreAPI]map[string]int, len(storepb.StoreType_name)) + for i := range storepb.StoreType_name { + nodes[component.FromProto(storepb.StoreType(i))] = map[string]int{} + } + return nodes +} + +// Update updates the store set. It fetches current list of store specs from function and updates the fresh metadata +// from all stores. Keeps around statically defined nodes that were defined with the strict mode. +func (s *StoreSet) Update(ctx context.Context) { + s.updateMtx.Lock() + defer s.updateMtx.Unlock() + + s.storesMtx.RLock() + stores := make(map[string]*storeRef, len(s.stores)) + for addr, st := range s.stores { + stores[addr] = st + } + s.storesMtx.RUnlock() + + level.Debug(s.logger).Log("msg", "starting updating storeAPIs", "cachedStores", len(stores)) + + activeStores := s.getActiveStores(ctx, stores) + level.Debug(s.logger).Log("msg", "checked requested storeAPIs", "activeStores", len(activeStores), "cachedStores", len(stores)) + + stats := newStoreAPIStats() + + // Close stores that where not active this time (are not in active stores map). + for addr, st := range stores { + if _, ok := activeStores[addr]; ok { + stats[st.StoreType()][labelpb.PromLabelSetsToString(st.LabelSets())]++ + continue + } + + st.Close() + delete(stores, addr) + s.updateStoreStatus(st, errors.New(unhealthyStoreMessage)) + level.Info(s.logger).Log("msg", unhealthyStoreMessage, "address", addr, "extLset", labelpb.PromLabelSetsToString(st.LabelSets())) + } + + // Add stores that are not yet in stores. + for addr, st := range activeStores { + if _, ok := stores[addr]; ok { + continue + } + + extLset := labelpb.PromLabelSetsToString(st.LabelSets()) + + // All producers should have unique external labels. While this does not check only StoreAPIs connected to + // this querier this allows to notify early user about misconfiguration. Warn only. This is also detectable from metric. + if st.StoreType() != nil && + (st.StoreType() == component.Sidecar || st.StoreType() == component.Rule) && + stats[component.Sidecar][extLset]+stats[component.Rule][extLset] > 0 { + + level.Warn(s.logger).Log("msg", "found duplicate storeAPI producer (sidecar or ruler). This is not advices as it will malform data in in the same bucket", + "address", addr, "extLset", extLset, "duplicates", fmt.Sprintf("%v", stats[component.Sidecar][extLset]+stats[component.Rule][extLset]+1)) + } + stats[st.StoreType()][extLset]++ + + stores[addr] = st + s.updateStoreStatus(st, nil) + + if st.HasRulesAPI() { + level.Info(s.logger).Log("msg", "adding new rulesAPI to query storeset", "address", addr) + } + + if st.HasExemplarsAPI() { + level.Info(s.logger).Log("msg", "adding new exemplarsAPI to query storeset", "address", addr) + } + + if st.HasTargetsAPI() { + level.Info(s.logger).Log("msg", "adding new targetsAPI to query storeset", "address", addr) + } + + level.Info(s.logger).Log("msg", "adding new storeAPI to query storeset", "address", addr, "extLset", extLset) + } + + s.storesMetric.Update(stats) + s.storesMtx.Lock() + s.stores = stores + s.storesMtx.Unlock() + + s.cleanUpStoreStatuses(stores) +} + +func (s *StoreSet) getActiveStores(ctx context.Context, stores map[string]*storeRef) map[string]*storeRef { + var ( + // UNIQUE? + activeStores = make(map[string]*storeRef, len(stores)) + mtx sync.Mutex + wg sync.WaitGroup + + storeAddrSet = make(map[string]struct{}) + ruleAddrSet = make(map[string]struct{}) + targetAddrSet = make(map[string]struct{}) + metadataAddrSet = make(map[string]struct{}) + exemplarAddrSet = make(map[string]struct{}) + infoAddrSet = make(map[string]struct{}) + ) + + // Gather active stores map concurrently. Build new store if does not exist already. + for _, ruleSpec := range s.ruleSpecs() { + ruleAddrSet[ruleSpec.Addr()] = struct{}{} + } + + // Gather active targets map concurrently. Add a new target if it does not exist already. + for _, targetSpec := range s.targetSpecs() { + targetAddrSet[targetSpec.Addr()] = struct{}{} + } + + // Gather active stores map concurrently. Build new store if does not exist already. + for _, metadataSpec := range s.metadataSpecs() { + metadataAddrSet[metadataSpec.Addr()] = struct{}{} + } + + // Gather active stores map concurrently. Build new store if does not exist already. + for _, exemplarSpec := range s.exemplarSpecs() { + exemplarAddrSet[exemplarSpec.Addr()] = struct{}{} + } + + // Gather healthy stores map concurrently. Build new store if does not exist already. + for _, storeSpec := range s.storeSpecs() { + if _, ok := storeAddrSet[storeSpec.Addr()]; ok { + level.Warn(s.logger).Log("msg", "duplicated address in store nodes", "address", storeSpec.Addr()) + continue + } + storeAddrSet[storeSpec.Addr()] = struct{}{} + + wg.Add(1) + go func(spec StoreSpec) { + defer wg.Done() + + addr := spec.Addr() + + ctx, cancel := context.WithTimeout(ctx, s.gRPCInfoCallTimeout) + defer cancel() + + st, seenAlready := stores[addr] + if !seenAlready { + // New store or was unactive and was removed in the past - create new one. + conn, err := grpc.DialContext(ctx, addr, s.dialOpts...) + if err != nil { + s.updateStoreStatus(&storeRef{addr: addr}, err) + level.Warn(s.logger).Log("msg", "update of store node failed", "err", errors.Wrap(err, "dialing connection"), "address", addr) + return + } + + st = &storeRef{StoreClient: storepb.NewStoreClient(conn), storeType: component.UnknownStoreAPI, cc: conn, addr: addr, logger: s.logger} + if spec.StrictStatic() { + st.maxTime = math.MaxInt64 + } + } + + var rule rulespb.RulesClient + if _, ok := ruleAddrSet[addr]; ok { + rule = rulespb.NewRulesClient(st.cc) + } + + var target targetspb.TargetsClient + if _, ok := targetAddrSet[addr]; ok { + target = targetspb.NewTargetsClient(st.cc) + } + + var metadata metadatapb.MetadataClient + if _, ok := metadataAddrSet[addr]; ok { + metadata = metadatapb.NewMetadataClient(st.cc) + } + + var exemplar exemplarspb.ExemplarsClient + if _, ok := exemplarAddrSet[addr]; ok { + exemplar = exemplarspb.NewExemplarsClient(st.cc) + } + + // Check existing or new store. Is it healthy? What are current metadata? + labelSets, minTime, maxTime, storeType, err := spec.Metadata(ctx, st.StoreClient) + if err != nil { + if !seenAlready && !spec.StrictStatic() { + // Close only if new and not a strict static node. + // Unactive `s.stores` will be closed later on. + st.Close() + } + s.updateStoreStatus(st, err) + level.Warn(s.logger).Log("msg", "update of store node failed", "err", errors.Wrap(err, "getting metadata"), "address", addr) + + if !spec.StrictStatic() { + return + } + + // Still keep it around if static & strict mode enabled. + mtx.Lock() + defer mtx.Unlock() + + activeStores[addr] = st + return + } + + s.updateStoreStatus(st, nil) + st.Update(labelSets, minTime, maxTime, storeType, rule, target, metadata, exemplar) + + mtx.Lock() + defer mtx.Unlock() + + activeStores[addr] = st + }(storeSpec) + } + wg.Wait() + + for ruleAddr := range ruleAddrSet { + if _, ok := storeAddrSet[ruleAddr]; !ok { + level.Warn(s.logger).Log("msg", "ignored rule store", "address", ruleAddr) + } + } + + // Gather healthy stores map concurrently using info addresses. Build new store if does not exist already. + for _, infoSpec := range s.infoSpec() { + if _, ok := infoAddrSet[infoSpec.Addr()]; ok { + level.Warn(s.logger).Log("msg", "duplicated address in info nodes", "address", infoSpec.Addr()) + continue + } + infoAddrSet[infoSpec.Addr()] = struct{}{} + + wg.Add(1) + go func(spec InfoSpec) { + defer wg.Done() + + addr := spec.Addr() + + ctx, cancel := context.WithTimeout(ctx, s.gRPCInfoCallTimeout) + defer cancel() + + st, seenAlready := stores[addr] + if !seenAlready { + // New store or was unactive and was removed in the past - create the new one. + conn, err := grpc.DialContext(ctx, addr, s.dialOpts...) + if err != nil { + s.updateStoreStatus(&storeRef{addr: addr}, err) + level.Warn(s.logger).Log("msg", "update of store node failed", "err", errors.Wrap(err, "dialing connection"), "address", addr) + return + } + + st = &storeRef{ + info: infopb.NewInfoClient(conn), + storeType: component.UnknownStoreAPI, + cc: conn, + addr: addr, + logger: s.logger, + } + } + + info, err := st.info.Info(ctx, &infopb.InfoRequest{}, grpc.WaitForReady(true)) + if err != nil { + if !seenAlready { + // Close only if new + // Unactive `s.stores` will be closed later on. + st.Close() + } + + s.updateStoreStatus(st, err) + level.Warn(s.logger).Log("msg", "update of node failed", "err", errors.Wrap(err, "getting metadata"), "address", addr) + + return + } + + s.updateStoreStatus(st, nil) + + labelSets := make([]labels.Labels, 0, len(info.LabelSets)) + for _, ls := range info.LabelSets { + labelSets = append(labelSets, ls.PromLabels()) + } + + var minTime, maxTime int64 + var store storepb.StoreClient + if info.Store != nil { + store = storepb.NewStoreClient(st.cc) + minTime = info.Store.MinTime + maxTime = info.Store.MaxTime + } + + var rule rulespb.RulesClient + if info.Rules != nil { + rule = rulespb.NewRulesClient(st.cc) + } + + var target targetspb.TargetsClient + if info.Targets != nil { + target = targetspb.NewTargetsClient(st.cc) + } + + var metadata metadatapb.MetadataClient + if info.MetricMetadata != nil { + metadata = metadatapb.NewMetadataClient(st.cc) + } + + var exemplar exemplarspb.ExemplarsClient + if info.Exemplars != nil { + // min/max range is also provided by in the response of Info rpc call + // but we are not using this metadata anywhere right now so ignoring. + exemplar = exemplarspb.NewExemplarsClient(st.cc) + } + + s.updateStoreStatus(st, nil) + st.UpdateWithStore(labelSets, minTime, maxTime, component.Sidecar, store, rule, target, metadata, exemplar) + + mtx.Lock() + defer mtx.Unlock() + + activeStores[addr] = st + }(infoSpec) + } + wg.Wait() + + return activeStores +} + +func (s *StoreSet) updateStoreStatus(store *storeRef, err error) { + s.storesStatusesMtx.Lock() + defer s.storesStatusesMtx.Unlock() + + status := StoreStatus{Name: store.addr} + prev, ok := s.storeStatuses[store.addr] + if ok { + status = *prev + } else { + mint, maxt := store.TimeRange() + status.MinTime = mint + status.MaxTime = maxt + } + + if err == nil { + status.LastCheck = time.Now() + mint, maxt := store.TimeRange() + status.LabelSets = store.LabelSets() + status.StoreType = store.StoreType() + status.MinTime = mint + status.MaxTime = maxt + status.LastError = nil + } else { + status.LastError = &stringError{originalErr: err} + } + + s.storeStatuses[store.addr] = &status +} + +func (s *StoreSet) GetStoreStatus() []StoreStatus { + s.storesStatusesMtx.RLock() + defer s.storesStatusesMtx.RUnlock() + + statuses := make([]StoreStatus, 0, len(s.storeStatuses)) + for _, v := range s.storeStatuses { + statuses = append(statuses, *v) + } + + sort.Slice(statuses, func(i, j int) bool { + return statuses[i].Name < statuses[j].Name + }) + return statuses +} + +// Get returns a list of all active stores. +func (s *StoreSet) Get() []store.Client { + s.storesMtx.RLock() + defer s.storesMtx.RUnlock() + + stores := make([]store.Client, 0, len(s.stores)) + for _, st := range s.stores { + stores = append(stores, st) + } + return stores +} + +// GetRulesClients returns a list of all active rules clients. +func (s *StoreSet) GetRulesClients() []rulespb.RulesClient { + s.storesMtx.RLock() + defer s.storesMtx.RUnlock() + + rules := make([]rulespb.RulesClient, 0, len(s.stores)) + for _, st := range s.stores { + if st.HasRulesAPI() { + rules = append(rules, st.rule) + } + } + return rules +} + +// GetTargetsClients returns a list of all active targets clients. +func (s *StoreSet) GetTargetsClients() []targetspb.TargetsClient { + s.storesMtx.RLock() + defer s.storesMtx.RUnlock() + + targets := make([]targetspb.TargetsClient, 0, len(s.stores)) + for _, st := range s.stores { + if st.HasTargetsAPI() { + targets = append(targets, st.target) + } + } + return targets +} + +// GetMetadataClients returns a list of all active metadata clients. +func (s *StoreSet) GetMetadataClients() []metadatapb.MetadataClient { + s.storesMtx.RLock() + defer s.storesMtx.RUnlock() + + metadataClients := make([]metadatapb.MetadataClient, 0, len(s.stores)) + for _, st := range s.stores { + if st.HasMetadataAPI() { + metadataClients = append(metadataClients, st.metadata) + } + } + return metadataClients +} + +// GetExemplarsStores returns a list of all active exemplars stores. +func (s *StoreSet) GetExemplarsStores() []*exemplarspb.ExemplarStore { + s.storesMtx.RLock() + defer s.storesMtx.RUnlock() + + exemplarStores := make([]*exemplarspb.ExemplarStore, 0, len(s.stores)) + for _, st := range s.stores { + if st.HasExemplarsAPI() { + exemplarStores = append(exemplarStores, &exemplarspb.ExemplarStore{ + ExemplarsClient: st.exemplar, + LabelSets: st.labelSets, + }) + } + } + return exemplarStores +} + +func (s *StoreSet) Close() { + s.storesMtx.Lock() + defer s.storesMtx.Unlock() + + for _, st := range s.stores { + st.Close() + } + s.stores = map[string]*storeRef{} +} + +func (s *StoreSet) cleanUpStoreStatuses(stores map[string]*storeRef) { + s.storesStatusesMtx.Lock() + defer s.storesStatusesMtx.Unlock() + + now := time.Now() + for addr, status := range s.storeStatuses { + if _, ok := stores[addr]; ok { + continue + } + + if now.Sub(status.LastCheck) >= s.unhealthyStoreTimeout { + delete(s.storeStatuses, addr) + } + } +} diff --git a/pkg/query/storeset_test.go b/pkg/query/storeset_test.go new file mode 100644 index 0000000000..922a9fd89b --- /dev/null +++ b/pkg/query/storeset_test.go @@ -0,0 +1,1331 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + +package query + +import ( + "context" + "encoding/json" + "fmt" + "math" + "net" + "testing" + "time" + + "github.com/pkg/errors" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/thanos-io/thanos/pkg/component" + "github.com/thanos-io/thanos/pkg/info/infopb" + "github.com/thanos-io/thanos/pkg/store" + "github.com/thanos-io/thanos/pkg/store/labelpb" + "github.com/thanos-io/thanos/pkg/store/storepb" + "github.com/thanos-io/thanos/pkg/testutil" +) + +var testGRPCOpts = []grpc.DialOption{ + grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(math.MaxInt32)), + grpc.WithInsecure(), +} + +type mockedStore struct { + infoDelay time.Duration + info storepb.InfoResponse +} + +func (s *mockedStore) Info(ctx context.Context, r *storepb.InfoRequest) (*storepb.InfoResponse, error) { + if s.infoDelay > 0 { + time.Sleep(s.infoDelay) + } + return &s.info, nil +} + +func (s *mockedStore) Series(r *storepb.SeriesRequest, srv storepb.Store_SeriesServer) error { + return status.Error(codes.Unimplemented, "not implemented") +} + +func (s *mockedStore) LabelNames(ctx context.Context, r *storepb.LabelNamesRequest) ( + *storepb.LabelNamesResponse, error, +) { + return nil, status.Error(codes.Unimplemented, "not implemented") +} + +func (s *mockedStore) LabelValues(ctx context.Context, r *storepb.LabelValuesRequest) ( + *storepb.LabelValuesResponse, error, +) { + return nil, status.Error(codes.Unimplemented, "not implemented") +} + +type testStoreMeta struct { + extlsetFn func(addr string) []labelpb.ZLabelSet + storeType component.StoreAPI + minTime, maxTime int64 + infoDelay time.Duration +} + +type testStores struct { + srvs map[string]*grpc.Server + orderAddrs []string +} + +func startTestStores(storeMetas []testStoreMeta) (*testStores, error) { + st := &testStores{ + srvs: map[string]*grpc.Server{}, + } + + for _, meta := range storeMetas { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + // Close so far started servers. + st.Close() + return nil, err + } + + srv := grpc.NewServer() + + storeSrv := &mockedStore{ + info: storepb.InfoResponse{ + LabelSets: meta.extlsetFn(listener.Addr().String()), + MaxTime: meta.maxTime, + MinTime: meta.minTime, + }, + infoDelay: meta.infoDelay, + } + if meta.storeType != nil { + storeSrv.info.StoreType = meta.storeType.ToProto() + } + storepb.RegisterStoreServer(srv, storeSrv) + go func() { + _ = srv.Serve(listener) + }() + + st.srvs[listener.Addr().String()] = srv + st.orderAddrs = append(st.orderAddrs, listener.Addr().String()) + } + + return st, nil +} + +func (s *testStores) StoreAddresses() []string { + var stores []string + stores = append(stores, s.orderAddrs...) + return stores +} + +func (s *testStores) Close() { + for _, srv := range s.srvs { + srv.Stop() + } + s.srvs = nil +} + +func (s *testStores) CloseOne(addr string) { + srv, ok := s.srvs[addr] + if !ok { + return + } + + srv.Stop() + delete(s.srvs, addr) +} + +type mockedInfo struct { + info infopb.InfoResponse +} + +func (s *mockedInfo) Info(ctx context.Context, r *infopb.InfoRequest) (*infopb.InfoResponse, error) { + return &s.info, nil +} + +type testInfoMeta struct { + extlsetFn func(add string) []labelpb.ZLabelSet + storeType component.StoreAPI + store infopb.StoreInfo + rule infopb.RulesInfo + metadata infopb.MetricMetadataInfo + target infopb.TargetsInfo + exemplar infopb.ExemplarsInfo +} + +type testInfoSrvs struct { + srvs map[string]*grpc.Server + orderAddrs []string +} + +func startInfoSrvs(infoMetas []testInfoMeta) (*testInfoSrvs, error) { + info := &testInfoSrvs{ + srvs: map[string]*grpc.Server{}, + } + + for _, meta := range infoMetas { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + // Close the servers + info.Close() + return nil, err + } + + srv := grpc.NewServer() + + infoSrv := &mockedInfo{ + info: infopb.InfoResponse{ + LabelSets: meta.extlsetFn(listener.Addr().String()), + Store: &meta.store, + MetricMetadata: &meta.metadata, + Rules: &meta.rule, + Targets: &meta.target, + Exemplars: &meta.exemplar, + }, + } + + if meta.storeType != nil { + infoSrv.info.ComponentType = meta.storeType.String() + } + infopb.RegisterInfoServer(srv, infoSrv) + go func() { + _ = srv.Serve(listener) + }() + + info.srvs[listener.Addr().String()] = srv + info.orderAddrs = append(info.orderAddrs, listener.Addr().String()) + } + + return info, nil +} + +func (s *testInfoSrvs) Close() { + for _, srv := range s.srvs { + srv.Stop() + } + s.srvs = nil +} + +func (s *testInfoSrvs) CloseOne(addr string) { + srv, ok := s.srvs[addr] + if !ok { + return + } + + srv.Stop() + delete(s.srvs, addr) +} + +func (s *testInfoSrvs) InfoAddresses() []string { + var stores []string + stores = append(stores, s.orderAddrs...) + return stores +} + +func TestStoreSet_Update(t *testing.T) { + stores, err := startTestStores([]testStoreMeta{ + { + storeType: component.Sidecar, + extlsetFn: func(addr string) []labelpb.ZLabelSet { + return []labelpb.ZLabelSet{ + { + Labels: []labelpb.ZLabel{ + {Name: "addr", Value: addr}, + }, + }, + { + Labels: []labelpb.ZLabel{ + {Name: "a", Value: "b"}, + }, + }, + } + }, + }, + { + storeType: component.Sidecar, + extlsetFn: func(addr string) []labelpb.ZLabelSet { + return []labelpb.ZLabelSet{ + { + Labels: []labelpb.ZLabel{ + {Name: "addr", Value: addr}, + }, + }, + { + Labels: []labelpb.ZLabel{ + {Name: "a", Value: "b"}, + }, + }, + } + }, + }, + { + storeType: component.Query, + extlsetFn: func(addr string) []labelpb.ZLabelSet { + return []labelpb.ZLabelSet{ + { + Labels: []labelpb.ZLabel{ + {Name: "a", Value: "broken"}, + }, + }, + } + }, + }, + }) + testutil.Ok(t, err) + defer stores.Close() + + discoveredStoreAddr := stores.StoreAddresses() + + infoSrvs, err := startInfoSrvs([]testInfoMeta{ + { + storeType: component.Sidecar, + extlsetFn: func(addr string) []labelpb.ZLabelSet { + return []labelpb.ZLabelSet{ + { + Labels: []labelpb.ZLabel{ + {Name: "addr", Value: addr}, + }, + }, + { + Labels: []labelpb.ZLabel{ + {Name: "a", Value: "b"}, + }, + }, + } + }, + store: infopb.StoreInfo{ + MinTime: math.MaxInt64, + MaxTime: math.MinInt64, + }, + exemplar: infopb.ExemplarsInfo{ + MinTime: math.MaxInt64, + MaxTime: math.MinInt64, + }, + rule: infopb.RulesInfo{}, + metadata: infopb.MetricMetadataInfo{}, + target: infopb.TargetsInfo{}, + }, + { + storeType: component.Sidecar, + extlsetFn: func(addr string) []labelpb.ZLabelSet { + return []labelpb.ZLabelSet{ + { + Labels: []labelpb.ZLabel{ + {Name: "addr", Value: addr}, + }, + }, + { + Labels: []labelpb.ZLabel{ + {Name: "a", Value: "b"}, + }, + }, + } + }, + store: infopb.StoreInfo{ + MinTime: math.MaxInt64, + MaxTime: math.MinInt64, + }, + exemplar: infopb.ExemplarsInfo{ + MinTime: math.MaxInt64, + MaxTime: math.MinInt64, + }, + rule: infopb.RulesInfo{}, + metadata: infopb.MetricMetadataInfo{}, + target: infopb.TargetsInfo{}, + }, + { + storeType: component.Query, + extlsetFn: func(addr string) []labelpb.ZLabelSet { + return []labelpb.ZLabelSet{ + { + Labels: []labelpb.ZLabel{ + {Name: "addr", Value: "broken"}, + }, + }, + } + }, + store: infopb.StoreInfo{ + MinTime: math.MaxInt64, + MaxTime: math.MinInt64, + }, + exemplar: infopb.ExemplarsInfo{ + MinTime: math.MaxInt64, + MaxTime: math.MinInt64, + }, + rule: infopb.RulesInfo{}, + metadata: infopb.MetricMetadataInfo{}, + target: infopb.TargetsInfo{}, + }, + }) + testutil.Ok(t, err) + defer infoSrvs.Close() + + discoveredInfoAddr := infoSrvs.InfoAddresses() + + // Testing if duplicates can cause weird results. + discoveredStoreAddr = append(discoveredStoreAddr, discoveredStoreAddr[0]) + discoveredInfoAddr = append(discoveredInfoAddr, discoveredInfoAddr[0]) + storeSet := NewStoreSet(nil, nil, + func() (specs []StoreSpec) { + for _, addr := range discoveredStoreAddr { + specs = append(specs, NewGRPCStoreSpec(addr, false)) + } + return specs + }, + func() (specs []RuleSpec) { + return nil + }, + func() (specs []TargetSpec) { + return nil + }, + func() (specs []MetadataSpec) { + return nil + }, + func() (specs []ExemplarSpec) { + return nil + }, + func() (specs []InfoSpec) { + for _, addr := range discoveredInfoAddr { + specs = append(specs, NewGRPCStoreSpec(addr, false)) + } + return specs + }, + testGRPCOpts, time.Minute) + storeSet.gRPCInfoCallTimeout = 2 * time.Second + defer storeSet.Close() + + // Initial update. + storeSet.Update(context.Background()) + + // Start with one not available. + stores.CloseOne(discoveredStoreAddr[2]) + + // Make one address discovered by Info Servers unavailable. + infoSrvs.CloseOne(discoveredInfoAddr[2]) + + // Should not matter how many of these we run. + storeSet.Update(context.Background()) + storeSet.Update(context.Background()) + testutil.Equals(t, 4, len(storeSet.stores)) + testutil.Equals(t, 6, len(storeSet.storeStatuses)) + + for addr, st := range storeSet.stores { + testutil.Equals(t, addr, st.addr) + + lset := st.LabelSets() + testutil.Equals(t, 2, len(lset)) + testutil.Equals(t, "addr", lset[0][0].Name) + testutil.Equals(t, addr, lset[0][0].Value) + testutil.Equals(t, "a", lset[1][0].Name) + testutil.Equals(t, "b", lset[1][0].Value) + } + + // Check stats. + expected := newStoreAPIStats() + expected[component.Sidecar] = map[string]int{ + fmt.Sprintf("{a=\"b\"},{addr=\"%s\"}", discoveredStoreAddr[0]): 1, + fmt.Sprintf("{a=\"b\"},{addr=\"%s\"}", discoveredStoreAddr[1]): 1, + fmt.Sprintf("{a=\"b\"},{addr=\"%s\"}", discoveredInfoAddr[0]): 1, + fmt.Sprintf("{a=\"b\"},{addr=\"%s\"}", discoveredInfoAddr[1]): 1, + } + testutil.Equals(t, expected, storeSet.storesMetric.storeNodes) + + // Remove address from discovered and reset last check, which should ensure cleanup of status on next update. + storeSet.storeStatuses[discoveredStoreAddr[2]].LastCheck = time.Now().Add(-4 * time.Minute) + discoveredStoreAddr = discoveredStoreAddr[:len(discoveredStoreAddr)-2] + storeSet.storeStatuses[discoveredInfoAddr[2]].LastCheck = time.Now().Add(-4 * time.Minute) + discoveredInfoAddr = discoveredInfoAddr[:len(discoveredInfoAddr)-2] + storeSet.Update(context.Background()) + testutil.Equals(t, 4, len(storeSet.storeStatuses)) + + stores.CloseOne(discoveredStoreAddr[0]) + delete(expected[component.Sidecar], fmt.Sprintf("{a=\"b\"},{addr=\"%s\"}", discoveredStoreAddr[0])) + infoSrvs.CloseOne(discoveredInfoAddr[0]) + delete(expected[component.Sidecar], fmt.Sprintf("{a=\"b\"},{addr=\"%s\"}", discoveredInfoAddr[0])) + + // We expect Update to tear down store client for closed store server. + storeSet.Update(context.Background()) + testutil.Equals(t, 2, len(storeSet.stores), "only two service should respond just fine, so we expect one client to be ready.") + testutil.Equals(t, 4, len(storeSet.storeStatuses)) + + addr := discoveredStoreAddr[1] + st, ok := storeSet.stores[addr] + testutil.Assert(t, ok, "addr exist") + testutil.Equals(t, addr, st.addr) + + lset := st.LabelSets() + testutil.Equals(t, 2, len(lset)) + testutil.Equals(t, "addr", lset[0][0].Name) + testutil.Equals(t, addr, lset[0][0].Value) + testutil.Equals(t, "a", lset[1][0].Name) + testutil.Equals(t, "b", lset[1][0].Value) + testutil.Equals(t, expected, storeSet.storesMetric.storeNodes) + + // New big batch of storeAPIs. + stores2, err := startTestStores([]testStoreMeta{ + { + storeType: component.Query, + extlsetFn: func(addr string) []labelpb.ZLabelSet { + return []labelpb.ZLabelSet{ + { + Labels: []labelpb.ZLabel{ + {Name: "l1", Value: "v2"}, + {Name: "l2", Value: "v3"}, + }, + }, + { + Labels: []labelpb.ZLabel{ + {Name: "l3", Value: "v4"}, + }, + }, + } + }, + }, + { + // Duplicated Querier, in previous versions it would be deduplicated. Now it should be not. + storeType: component.Query, + extlsetFn: func(addr string) []labelpb.ZLabelSet { + return []labelpb.ZLabelSet{ + { + Labels: []labelpb.ZLabel{ + {Name: "l1", Value: "v2"}, + {Name: "l2", Value: "v3"}, + }, + }, + { + Labels: []labelpb.ZLabel{ + {Name: "l3", Value: "v4"}, + }, + }, + } + }, + }, + { + storeType: component.Sidecar, + extlsetFn: func(addr string) []labelpb.ZLabelSet { + return []labelpb.ZLabelSet{ + { + Labels: []labelpb.ZLabel{ + {Name: "l1", Value: "v2"}, + {Name: "l2", Value: "v3"}, + }, + }, + } + }, + }, + { + // Duplicated Sidecar, in previous versions it would be deduplicated. Now it should be not. + storeType: component.Sidecar, + extlsetFn: func(addr string) []labelpb.ZLabelSet { + return []labelpb.ZLabelSet{ + { + Labels: []labelpb.ZLabel{ + {Name: "l1", Value: "v2"}, + {Name: "l2", Value: "v3"}, + }, + }, + } + }, + }, + { + // Querier that duplicates with sidecar, in previous versions it would be deduplicated. Now it should be not. + storeType: component.Query, + extlsetFn: func(addr string) []labelpb.ZLabelSet { + return []labelpb.ZLabelSet{ + { + Labels: []labelpb.ZLabel{ + {Name: "l1", Value: "v2"}, + {Name: "l2", Value: "v3"}, + }, + }, + } + }, + }, + { + // Ruler that duplicates with sidecar, in previous versions it would be deduplicated. Now it should be not. + // Warning should be produced. + storeType: component.Rule, + extlsetFn: func(addr string) []labelpb.ZLabelSet { + return []labelpb.ZLabelSet{ + { + Labels: []labelpb.ZLabel{ + {Name: "l1", Value: "v2"}, + {Name: "l2", Value: "v3"}, + }, + }, + } + }, + }, + { + // Duplicated Rule, in previous versions it would be deduplicated. Now it should be not. Warning should be produced. + storeType: component.Rule, + extlsetFn: func(addr string) []labelpb.ZLabelSet { + return []labelpb.ZLabelSet{ + { + Labels: []labelpb.ZLabel{ + {Name: "l1", Value: "v2"}, + {Name: "l2", Value: "v3"}, + }, + }, + } + }, + }, + { + // No storeType. + extlsetFn: func(addr string) []labelpb.ZLabelSet { + return []labelpb.ZLabelSet{ + { + Labels: []labelpb.ZLabel{ + {Name: "l1", Value: "no-store-type"}, + {Name: "l2", Value: "v3"}, + }, + }, + } + }, + }, + // Two pre v0.8.0 store gateway nodes, they don't have ext labels set. + { + storeType: component.Store, + extlsetFn: func(addr string) []labelpb.ZLabelSet { + return []labelpb.ZLabelSet{} + }, + }, + { + storeType: component.Store, + extlsetFn: func(addr string) []labelpb.ZLabelSet { + return []labelpb.ZLabelSet{} + }, + }, + // Regression tests against https://github.com/thanos-io/thanos/issues/1632: From v0.8.0 stores advertise labels. + // If the object storage handled by store gateway has only one sidecar we used to hitting issue. + { + storeType: component.Store, + extlsetFn: func(addr string) []labelpb.ZLabelSet { + return []labelpb.ZLabelSet{ + { + Labels: []labelpb.ZLabel{ + {Name: "l1", Value: "v2"}, + {Name: "l2", Value: "v3"}, + }, + }, + { + Labels: []labelpb.ZLabel{ + {Name: "l3", Value: "v4"}, + }, + }, + } + }, + }, + // Stores v0.8.1 has compatibility labels. Check if they are correctly removed. + { + storeType: component.Store, + extlsetFn: func(addr string) []labelpb.ZLabelSet { + return []labelpb.ZLabelSet{ + { + Labels: []labelpb.ZLabel{ + {Name: "l1", Value: "v2"}, + {Name: "l2", Value: "v3"}, + }, + }, + { + Labels: []labelpb.ZLabel{ + {Name: "l3", Value: "v4"}, + }, + }, + { + Labels: []labelpb.ZLabel{ + {Name: store.CompatibilityTypeLabelName, Value: "store"}, + }, + }, + } + }, + }, + // Duplicated store, in previous versions it would be deduplicated. Now it should be not. + { + storeType: component.Store, + extlsetFn: func(addr string) []labelpb.ZLabelSet { + return []labelpb.ZLabelSet{ + { + Labels: []labelpb.ZLabel{ + {Name: "l1", Value: "v2"}, + {Name: "l2", Value: "v3"}, + }, + }, + { + Labels: []labelpb.ZLabel{ + {Name: "l3", Value: "v4"}, + }, + }, + { + Labels: []labelpb.ZLabel{ + {Name: store.CompatibilityTypeLabelName, Value: "store"}, + }, + }, + } + }, + }, + }) + testutil.Ok(t, err) + defer stores2.Close() + + discoveredStoreAddr = append(discoveredStoreAddr, stores2.StoreAddresses()...) + + // New stores should be loaded. + storeSet.Update(context.Background()) + testutil.Equals(t, 2+len(stores2.srvs), len(storeSet.stores)) + + // Check stats. + expected = newStoreAPIStats() + expected[component.UnknownStoreAPI] = map[string]int{ + "{l1=\"no-store-type\", l2=\"v3\"}": 1, + } + expected[component.Query] = map[string]int{ + "{l1=\"v2\", l2=\"v3\"}": 1, + "{l1=\"v2\", l2=\"v3\"},{l3=\"v4\"}": 2, + } + expected[component.Rule] = map[string]int{ + "{l1=\"v2\", l2=\"v3\"}": 2, + } + expected[component.Sidecar] = map[string]int{ + fmt.Sprintf("{a=\"b\"},{addr=\"%s\"}", discoveredStoreAddr[1]): 1, + fmt.Sprintf("{a=\"b\"},{addr=\"%s\"}", discoveredInfoAddr[1]): 1, + "{l1=\"v2\", l2=\"v3\"}": 2, + } + expected[component.Store] = map[string]int{ + "": 2, + "{l1=\"v2\", l2=\"v3\"},{l3=\"v4\"}": 3, + } + testutil.Equals(t, expected, storeSet.storesMetric.storeNodes) + + // Check statuses. + testutil.Equals(t, 4+len(stores2.srvs), len(storeSet.storeStatuses)) +} + +func TestStoreSet_Update_NoneAvailable(t *testing.T) { + st, err := startTestStores([]testStoreMeta{ + { + extlsetFn: func(addr string) []labelpb.ZLabelSet { + return []labelpb.ZLabelSet{ + { + Labels: []labelpb.ZLabel{ + { + Name: "addr", + Value: addr, + }, + }, + }, + } + }, + storeType: component.Sidecar, + }, + { + extlsetFn: func(addr string) []labelpb.ZLabelSet { + return []labelpb.ZLabelSet{ + { + Labels: []labelpb.ZLabel{ + { + Name: "addr", + Value: addr, + }, + }, + }, + } + }, + storeType: component.Sidecar, + }, + }) + testutil.Ok(t, err) + defer st.Close() + + infoSrvs, err := startInfoSrvs([]testInfoMeta{ + { + storeType: component.Sidecar, + extlsetFn: func(addr string) []labelpb.ZLabelSet { + return []labelpb.ZLabelSet{ + { + Labels: []labelpb.ZLabel{ + {Name: "addr", Value: addr}, + }, + }, + } + }, + }, + { + storeType: component.Sidecar, + extlsetFn: func(addr string) []labelpb.ZLabelSet { + return []labelpb.ZLabelSet{ + { + Labels: []labelpb.ZLabel{ + {Name: "addr", Value: addr}, + }, + }, + } + }, + }, + }) + testutil.Ok(t, err) + defer infoSrvs.Close() + + initialStoreAddr := st.StoreAddresses() + st.CloseOne(initialStoreAddr[0]) + st.CloseOne(initialStoreAddr[1]) + + initialInfoAddr := infoSrvs.InfoAddresses() + infoSrvs.CloseOne(initialInfoAddr[0]) + infoSrvs.CloseOne(initialInfoAddr[1]) + + storeSet := NewStoreSet(nil, nil, + func() (specs []StoreSpec) { + for _, addr := range initialStoreAddr { + specs = append(specs, NewGRPCStoreSpec(addr, false)) + } + return specs + }, + func() (specs []RuleSpec) { return nil }, + func() (specs []TargetSpec) { return nil }, + func() (specs []MetadataSpec) { return nil }, + func() (specs []ExemplarSpec) { return nil }, + func() (specs []InfoSpec) { + for _, addr := range initialInfoAddr { + specs = append(specs, NewGRPCStoreSpec(addr, false)) + } + return specs + }, + testGRPCOpts, time.Minute) + storeSet.gRPCInfoCallTimeout = 2 * time.Second + + // Should not matter how many of these we run. + storeSet.Update(context.Background()) + storeSet.Update(context.Background()) + testutil.Equals(t, 0, len(storeSet.stores), "none of services should respond just fine, so we expect no client to be ready.") + + // Leak test will ensure that we don't keep client connection around. + + expected := newStoreAPIStats() + testutil.Equals(t, expected, storeSet.storesMetric.storeNodes) +} + +// TestQuerierStrict tests what happens when the strict mode is enabled/disabled. +func TestQuerierStrict(t *testing.T) { + st, err := startTestStores([]testStoreMeta{ + { + minTime: 12345, + maxTime: 54321, + extlsetFn: func(addr string) []labelpb.ZLabelSet { + return []labelpb.ZLabelSet{ + { + Labels: []labelpb.ZLabel{ + { + Name: "addr", + Value: addr, + }, + }, + }, + } + }, + storeType: component.Sidecar, + }, + { + minTime: 66666, + maxTime: 77777, + extlsetFn: func(addr string) []labelpb.ZLabelSet { + return []labelpb.ZLabelSet{ + { + Labels: []labelpb.ZLabel{ + { + Name: "addr", + Value: addr, + }, + }, + }, + } + }, + storeType: component.Sidecar, + }, + // Slow store. + { + minTime: 65644, + maxTime: 77777, + extlsetFn: func(addr string) []labelpb.ZLabelSet { + return []labelpb.ZLabelSet{ + { + Labels: []labelpb.ZLabel{ + { + Name: "addr", + Value: addr, + }, + }, + }, + } + }, + storeType: component.Sidecar, + infoDelay: 2 * time.Second, + }, + }) + + testutil.Ok(t, err) + defer st.Close() + + staticStoreAddr := st.StoreAddresses()[0] + slowStaticStoreAddr := st.StoreAddresses()[2] + storeSet := NewStoreSet(nil, nil, func() (specs []StoreSpec) { + return []StoreSpec{ + NewGRPCStoreSpec(st.StoreAddresses()[0], true), + NewGRPCStoreSpec(st.StoreAddresses()[1], false), + NewGRPCStoreSpec(st.StoreAddresses()[2], true), + } + }, func() []RuleSpec { + return nil + }, func() []TargetSpec { + return nil + }, func() (specs []MetadataSpec) { + return nil + }, func() []ExemplarSpec { + return nil + }, func() []InfoSpec { + return nil + }, testGRPCOpts, time.Minute) + defer storeSet.Close() + storeSet.gRPCInfoCallTimeout = 1 * time.Second + + // Initial update. + storeSet.Update(context.Background()) + testutil.Equals(t, 3, len(storeSet.stores), "three clients must be available for running store nodes") + + // The store has not responded to the info call and is assumed to cover everything. + curMin, curMax := storeSet.stores[slowStaticStoreAddr].minTime, storeSet.stores[slowStaticStoreAddr].maxTime + testutil.Assert(t, storeSet.stores[slowStaticStoreAddr].cc.GetState().String() != "SHUTDOWN", "slow store's connection should not be closed") + testutil.Equals(t, int64(0), curMin) + testutil.Equals(t, int64(math.MaxInt64), curMax) + + // The store is statically defined + strict mode is enabled + // so its client + information must be retained. + curMin, curMax = storeSet.stores[staticStoreAddr].minTime, storeSet.stores[staticStoreAddr].maxTime + testutil.Equals(t, int64(12345), curMin, "got incorrect minimum time") + testutil.Equals(t, int64(54321), curMax, "got incorrect minimum time") + + // Successfully retrieve the information and observe minTime/maxTime updating. + storeSet.gRPCInfoCallTimeout = 3 * time.Second + storeSet.Update(context.Background()) + updatedCurMin, updatedCurMax := storeSet.stores[slowStaticStoreAddr].minTime, storeSet.stores[slowStaticStoreAddr].maxTime + testutil.Equals(t, int64(65644), updatedCurMin) + testutil.Equals(t, int64(77777), updatedCurMax) + storeSet.gRPCInfoCallTimeout = 1 * time.Second + + // Turn off the stores. + st.Close() + + // Update again many times. Should not matter WRT the static one. + storeSet.Update(context.Background()) + storeSet.Update(context.Background()) + storeSet.Update(context.Background()) + + // Check that the information is the same. + testutil.Equals(t, 2, len(storeSet.stores), "two static clients must remain available") + testutil.Equals(t, curMin, storeSet.stores[staticStoreAddr].minTime, "minimum time reported by the store node is different") + testutil.Equals(t, curMax, storeSet.stores[staticStoreAddr].maxTime, "minimum time reported by the store node is different") + testutil.NotOk(t, storeSet.storeStatuses[staticStoreAddr].LastError.originalErr) + + testutil.Equals(t, updatedCurMin, storeSet.stores[slowStaticStoreAddr].minTime, "minimum time reported by the store node is different") + testutil.Equals(t, updatedCurMax, storeSet.stores[slowStaticStoreAddr].maxTime, "minimum time reported by the store node is different") +} + +func TestStoreSet_Update_Rules(t *testing.T) { + stores, err := startTestStores([]testStoreMeta{ + { + extlsetFn: func(addr string) []labelpb.ZLabelSet { + return []labelpb.ZLabelSet{} + }, + storeType: component.Sidecar, + }, + { + extlsetFn: func(addr string) []labelpb.ZLabelSet { + return []labelpb.ZLabelSet{} + }, + storeType: component.Rule, + }, + }) + testutil.Ok(t, err) + defer stores.Close() + + for _, tc := range []struct { + name string + storeSpecs func() []StoreSpec + ruleSpecs func() []RuleSpec + exemplarSpecs func() []ExemplarSpec + expectedStores int + expectedRules int + }{ + { + name: "stores, no rules", + storeSpecs: func() []StoreSpec { + return []StoreSpec{ + NewGRPCStoreSpec(stores.orderAddrs[0], false), + NewGRPCStoreSpec(stores.orderAddrs[1], false), + } + }, + expectedStores: 2, + expectedRules: 0, + }, + { + name: "rules, no stores", + ruleSpecs: func() []RuleSpec { + return []RuleSpec{ + NewGRPCStoreSpec(stores.orderAddrs[0], false), + } + }, + expectedStores: 0, + expectedRules: 0, + }, + { + name: "one store, different rule", + storeSpecs: func() []StoreSpec { + return []StoreSpec{ + NewGRPCStoreSpec(stores.orderAddrs[0], false), + } + }, + ruleSpecs: func() []RuleSpec { + return []RuleSpec{ + NewGRPCStoreSpec(stores.orderAddrs[1], false), + } + }, + expectedStores: 1, + expectedRules: 0, + }, + { + name: "two stores, one rule", + storeSpecs: func() []StoreSpec { + return []StoreSpec{ + NewGRPCStoreSpec(stores.orderAddrs[0], false), + NewGRPCStoreSpec(stores.orderAddrs[1], false), + } + }, + ruleSpecs: func() []RuleSpec { + return []RuleSpec{ + NewGRPCStoreSpec(stores.orderAddrs[0], false), + } + }, + expectedStores: 2, + expectedRules: 1, + }, + { + name: "two stores, two rules", + storeSpecs: func() []StoreSpec { + return []StoreSpec{ + NewGRPCStoreSpec(stores.orderAddrs[0], false), + NewGRPCStoreSpec(stores.orderAddrs[1], false), + } + }, + ruleSpecs: func() []RuleSpec { + return []RuleSpec{ + NewGRPCStoreSpec(stores.orderAddrs[0], false), + NewGRPCStoreSpec(stores.orderAddrs[1], false), + } + }, + exemplarSpecs: func() []ExemplarSpec { + return []ExemplarSpec{ + NewGRPCStoreSpec(stores.orderAddrs[0], false), + NewGRPCStoreSpec(stores.orderAddrs[1], false), + } + }, + expectedStores: 2, + expectedRules: 2, + }, + } { + storeSet := NewStoreSet(nil, nil, + tc.storeSpecs, + tc.ruleSpecs, + func() []TargetSpec { return nil }, + func() []MetadataSpec { return nil }, + tc.exemplarSpecs, + func() []InfoSpec { return nil }, + testGRPCOpts, time.Minute) + + t.Run(tc.name, func(t *testing.T) { + defer storeSet.Close() + storeSet.Update(context.Background()) + testutil.Equals(t, tc.expectedStores, len(storeSet.stores)) + + gotRules := 0 + for _, ref := range storeSet.stores { + if ref.HasRulesAPI() { + gotRules += 1 + } + } + + testutil.Equals(t, tc.expectedRules, gotRules) + }) + } +} + +func TestStoreSet_Rules_Discovery(t *testing.T) { + stores, err := startTestStores([]testStoreMeta{ + { + extlsetFn: func(addr string) []labelpb.ZLabelSet { + return []labelpb.ZLabelSet{} + }, + storeType: component.Sidecar, + }, + { + extlsetFn: func(addr string) []labelpb.ZLabelSet { + return []labelpb.ZLabelSet{} + }, + storeType: component.Rule, + }, + }) + testutil.Ok(t, err) + defer stores.Close() + + type discoveryState struct { + name string + storeSpecs func() []StoreSpec + ruleSpecs func() []RuleSpec + expectedStores int + expectedRules int + } + + for _, tc := range []struct { + states []discoveryState + name string + }{ + { + name: "StoreAPI and RulesAPI concurrent discovery", + states: []discoveryState{ + { + name: "no stores", + storeSpecs: nil, + ruleSpecs: nil, + expectedRules: 0, + expectedStores: 0, + }, + { + name: "RulesAPI discovered", + storeSpecs: func() []StoreSpec { + return []StoreSpec{ + NewGRPCStoreSpec(stores.orderAddrs[0], false), + } + }, + ruleSpecs: func() []RuleSpec { + return []RuleSpec{ + NewGRPCStoreSpec(stores.orderAddrs[0], false), + } + }, + expectedRules: 1, + expectedStores: 1, + }, + }, + }, + + { + name: "StoreAPI discovery first, eventually discovered RulesAPI", + states: []discoveryState{ + { + name: "no stores", + storeSpecs: nil, + ruleSpecs: nil, + expectedRules: 0, + expectedStores: 0, + }, + { + name: "StoreAPI discovered, no RulesAPI discovered", + storeSpecs: func() []StoreSpec { + return []StoreSpec{ + NewGRPCStoreSpec(stores.orderAddrs[0], false), + } + }, + expectedStores: 1, + expectedRules: 0, + }, + { + name: "RulesAPI discovered", + storeSpecs: func() []StoreSpec { + return []StoreSpec{ + NewGRPCStoreSpec(stores.orderAddrs[0], false), + } + }, + ruleSpecs: func() []RuleSpec { + return []RuleSpec{ + NewGRPCStoreSpec(stores.orderAddrs[0], false), + } + }, + expectedStores: 1, + expectedRules: 1, + }, + }, + }, + + { + name: "RulesAPI discovery first, eventually discovered StoreAPI", + states: []discoveryState{ + { + name: "no stores", + storeSpecs: nil, + ruleSpecs: nil, + expectedRules: 0, + expectedStores: 0, + }, + { + name: "RulesAPI discovered, no StoreAPI discovered", + storeSpecs: nil, + ruleSpecs: func() []RuleSpec { + return []RuleSpec{ + NewGRPCStoreSpec(stores.orderAddrs[0], false), + } + }, + expectedStores: 0, + expectedRules: 0, + }, + { + name: "StoreAPI discovered", + storeSpecs: func() []StoreSpec { + return []StoreSpec{ + NewGRPCStoreSpec(stores.orderAddrs[0], false), + } + }, + ruleSpecs: func() []RuleSpec { + return []RuleSpec{ + NewGRPCStoreSpec(stores.orderAddrs[0], false), + } + }, + expectedStores: 1, + expectedRules: 1, + }, + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + currentState := 0 + + storeSet := NewStoreSet(nil, nil, + func() []StoreSpec { + if tc.states[currentState].storeSpecs == nil { + return nil + } + + return tc.states[currentState].storeSpecs() + }, + func() []RuleSpec { + if tc.states[currentState].ruleSpecs == nil { + return nil + } + + return tc.states[currentState].ruleSpecs() + }, + func() []TargetSpec { return nil }, + func() []MetadataSpec { + return nil + }, + func() []ExemplarSpec { return nil }, + func() []InfoSpec { + return nil + }, + testGRPCOpts, time.Minute) + + defer storeSet.Close() + + for { + storeSet.Update(context.Background()) + testutil.Equals( + t, + tc.states[currentState].expectedStores, + len(storeSet.stores), + "unexepected discovered stores in state %q", + tc.states[currentState].name, + ) + + gotRules := 0 + for _, ref := range storeSet.stores { + if ref.HasRulesAPI() { + gotRules += 1 + } + } + testutil.Equals( + t, + tc.states[currentState].expectedRules, + gotRules, + "unexpected discovered rules in state %q", + tc.states[currentState].name, + ) + + currentState = currentState + 1 + if len(tc.states) == currentState { + break + } + } + }) + } +} + +type errThatMarshalsToEmptyDict struct { + msg string +} + +// MarshalJSON marshals the error and returns and empty dict, not the error string. +func (e *errThatMarshalsToEmptyDict) MarshalJSON() ([]byte, error) { + return json.Marshal(map[string]string{}) +} + +// Error returns the original, underlying string. +func (e *errThatMarshalsToEmptyDict) Error() string { + return e.msg +} + +// Test highlights that without wrapping the error, it is marshaled to empty dict {}, not its message. +func TestStringError(t *testing.T) { + dictErr := &errThatMarshalsToEmptyDict{msg: "Error message"} + stringErr := &stringError{originalErr: dictErr} + + storestatusMock := map[string]error{} + storestatusMock["dictErr"] = dictErr + storestatusMock["stringErr"] = stringErr + + b, err := json.Marshal(storestatusMock) + + testutil.Ok(t, err) + testutil.Equals(t, []byte(`{"dictErr":{},"stringErr":"Error message"}`), b, "expected to get proper results") +} + +// Errors that usually marshal to empty dict should return the original error string. +func TestUpdateStoreStateLastError(t *testing.T) { + tcs := []struct { + InputError error + ExpectedLastErr string + }{ + {errors.New("normal_err"), `"normal_err"`}, + {nil, `null`}, + {&errThatMarshalsToEmptyDict{"the error message"}, `"the error message"`}, + } + + for _, tc := range tcs { + mockStoreSet := &StoreSet{ + storeStatuses: map[string]*StoreStatus{}, + } + mockStoreRef := &storeRef{ + addr: "mockedStore", + } + + mockStoreSet.updateStoreStatus(mockStoreRef, tc.InputError) + + b, err := json.Marshal(mockStoreSet.storeStatuses["mockedStore"].LastError) + testutil.Ok(t, err) + testutil.Equals(t, tc.ExpectedLastErr, string(b)) + } +} + +func TestUpdateStoreStateForgetsPreviousErrors(t *testing.T) { + mockStoreSet := &StoreSet{ + storeStatuses: map[string]*StoreStatus{}, + } + mockStoreRef := &storeRef{ + addr: "mockedStore", + } + + mockStoreSet.updateStoreStatus(mockStoreRef, errors.New("test err")) + + b, err := json.Marshal(mockStoreSet.storeStatuses["mockedStore"].LastError) + testutil.Ok(t, err) + testutil.Equals(t, `"test err"`, string(b)) + + // updating status without and error should clear the previous one. + mockStoreSet.updateStoreStatus(mockStoreRef, nil) + + b, err = json.Marshal(mockStoreSet.storeStatuses["mockedStore"].LastError) + testutil.Ok(t, err) + testutil.Equals(t, `null`, string(b)) +} diff --git a/scripts/insecure_grpcurl_info.sh b/scripts/insecure_grpcurl_info.sh deleted file mode 100755 index 0d75b02053..0000000000 --- a/scripts/insecure_grpcurl_info.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash - -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - -HELP=' -insecure_grpcurl_info.sh allows you to use call StoreAPI.Series gRPC method and receive streamed series in JSON format. - -Usage: - # Start some example Thanos component that exposes gRPC or use existing one. To start example one run: `thanos query &` - bash scripts/insecure_grpcurl_info.sh localhost:10901 '"'"'[{"type": 0, "name": "__name__", "value":"go_goroutines"}]'"'"' 0 10 -' - -INFO_API_HOSTPORT=$1 -if [ -z "${INFO_API_HOSTPORT}" ]; then - echo '$1 is missing (INFO_API_HOSTPORT). Expected host:port string for the target StoreAPI to grpcurl against, e.g. localhost:10901' - echo "${HELP}" - exit 1 -fi - -go install github.com/fullstorydev/grpcurl/cmd/grpcurl - -INFO_REQUEST='{}' - -GOGOPROTO_ROOT="$(GO111MODULE=on go list -f '{{ .Dir }}' -m github.com/gogo/protobuf)" - -cd $DIR/../pkg/ || exit -grpcurl \ - -import-path="${GOGOPROTO_ROOT}" \ - -import-path=. \ - -proto=info/infopb/rpc.proto \ - -plaintext \ - -d="${INFO_REQUEST}" "${INFO_API_HOSTPORT}" thanos.Info/Info diff --git a/test/e2e/info_api_test.go b/test/e2e/info_api_test.go deleted file mode 100644 index 208ef2ee3a..0000000000 --- a/test/e2e/info_api_test.go +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright (c) The Thanos Authors. -// Licensed under the Apache License 2.0. - -package e2e_test - -import ( - "context" - "testing" - - "github.com/cortexproject/cortex/integration/e2e" - "github.com/thanos-io/thanos/pkg/info/infopb" - "github.com/thanos-io/thanos/pkg/store/labelpb" - "github.com/thanos-io/thanos/pkg/testutil" - "github.com/thanos-io/thanos/test/e2e/e2ethanos" - "google.golang.org/grpc" -) - -func TestInfoAPI_WithSidecar(t *testing.T) { - t.Parallel() - - netName := "e2e_test_info_with_sidecar" - - s, err := e2e.NewScenario(netName) - testutil.Ok(t, err) - t.Cleanup(e2ethanos.CleanScenario(t, s)) - - prom, sidecar, err := e2ethanos.NewPrometheusWithSidecar( - s.SharedDir(), - netName, - "prom", - defaultPromConfig("ha", 0, "", ""), - e2ethanos.DefaultPrometheusImage(), - ) - testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(prom, sidecar)) - - // Create grpc Client - conn, err := grpc.Dial(sidecar.GRPCEndpoint(), grpc.WithInsecure()) - testutil.Ok(t, err) - defer conn.Close() - - client := infopb.NewInfoClient(conn) - - res, err := client.Info(context.Background(), &infopb.InfoRequest{}) - testutil.Ok(t, err) - testutil.Equals(t, res, infopb.InfoResponse{ - LabelSets: []labelpb.ZLabelSet{}, - ComponentType: infopb.ComponentType_SIDECAR, - StoreInfo: &infopb.StoreInfo{ - MinTime: -62167219200000, - MaxTime: 9223372036854775807, - }, - ExemplarsInfo: &infopb.ExemplarsInfo{ - MinTime: -9223309901257974, - MaxTime: 9223309901257974, - }, - }) -} From 61c68cbbe84926ad3935577bd2938c00da3239ca Mon Sep 17 00:00:00 2001 From: Hitanshu Mehta Date: Sun, 6 Jun 2021 00:22:37 +0530 Subject: [PATCH 03/16] Add E2E test for Info API Signed-off-by: Hitanshu Mehta --- pkg/query/storeset.go | 2 +- test/e2e/e2ethanos/services.go | 5 + test/e2e/info_api_test.go | 172 +++++++++++++++++++++++++++++++++ 3 files changed, 178 insertions(+), 1 deletion(-) create mode 100644 test/e2e/info_api_test.go diff --git a/pkg/query/storeset.go b/pkg/query/storeset.go index 5568f1fa0a..c20a75a311 100644 --- a/pkg/query/storeset.go +++ b/pkg/query/storeset.go @@ -719,7 +719,7 @@ func (s *StoreSet) getActiveStores(ctx context.Context, stores map[string]*store } s.updateStoreStatus(st, nil) - st.UpdateWithStore(labelSets, minTime, maxTime, component.Sidecar, store, rule, target, metadata, exemplar) + st.UpdateWithStore(labelSets, minTime, maxTime, component.FromString(info.ComponentType), store, rule, target, metadata, exemplar) mtx.Lock() defer mtx.Unlock() diff --git a/test/e2e/e2ethanos/services.go b/test/e2e/e2ethanos/services.go index be8a56b3f3..64836f7a2b 100644 --- a/test/e2e/e2ethanos/services.go +++ b/test/e2e/e2ethanos/services.go @@ -149,6 +149,7 @@ type QuerierBuilder struct { targetAddresses []string exemplarAddresses []string enableFeatures []string + endpoints []string tracingConfig string } @@ -296,6 +297,10 @@ func (q *QuerierBuilder) collectArgs() ([]string, error) { args = append(args, "--enable-feature="+feature) } + for _, addr := range q.endpoints { + args = append(args, "--endpoint="+addr) + } + if len(q.fileSDStoreAddresses) > 0 { queryFileSDDir := filepath.Join(q.sharedDir, "data", "querier", q.name) container := filepath.Join(ContainerSharedDir, "data", "querier", q.name) diff --git a/test/e2e/info_api_test.go b/test/e2e/info_api_test.go new file mode 100644 index 0000000000..c44224a2ef --- /dev/null +++ b/test/e2e/info_api_test.go @@ -0,0 +1,172 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + +package e2e_test + +import ( + "context" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "path" + "testing" + "time" + + "github.com/cortexproject/cortex/integration/e2e" + e2edb "github.com/cortexproject/cortex/integration/e2e/db" + "github.com/prometheus/prometheus/pkg/labels" + "github.com/thanos-io/thanos/pkg/objstore/client" + "github.com/thanos-io/thanos/pkg/objstore/s3" + "github.com/thanos-io/thanos/pkg/query" + "github.com/thanos-io/thanos/pkg/runutil" + "github.com/thanos-io/thanos/pkg/testutil" + "github.com/thanos-io/thanos/test/e2e/e2ethanos" +) + +func TestInfo(t *testing.T) { + t.Parallel() + + s, err := e2e.NewScenario("e2e_test_info") + testutil.Ok(t, err) + t.Cleanup(e2ethanos.CleanScenario(t, s)) + + prom1, sidecar1, err := e2ethanos.NewPrometheusWithSidecar(s.SharedDir(), "e2e_test_info", "alone1", defaultPromConfig("prom-alone1", 0, "", ""), e2ethanos.DefaultPrometheusImage()) + testutil.Ok(t, err) + prom2, sidecar2, err := e2ethanos.NewPrometheusWithSidecar(s.SharedDir(), "e2e_test_info", "alone2", defaultPromConfig("prom-alone2", 0, "", ""), e2ethanos.DefaultPrometheusImage()) + testutil.Ok(t, err) + prom3, sidecar3, err := e2ethanos.NewPrometheusWithSidecar(s.SharedDir(), "e2e_test_info", "alone3", defaultPromConfig("prom-alone3", 0, "", ""), e2ethanos.DefaultPrometheusImage()) + testutil.Ok(t, err) + testutil.Ok(t, s.StartAndWaitReady(prom1, sidecar1, prom2, sidecar2, prom3, sidecar3)) + + m := e2edb.NewMinio(8080, "thanos") + testutil.Ok(t, s.StartAndWaitReady(m)) + str, err := e2ethanos.NewStoreGW(s.SharedDir(), "1", client.BucketConfig{ + Type: client.S3, + Config: s3.Config{ + Bucket: "thanos", + AccessKey: e2edb.MinioAccessKey, + SecretKey: e2edb.MinioSecretKey, + Endpoint: m.NetworkHTTPEndpoint(), + Insecure: true, + }, + }) + testutil.Ok(t, err) + testutil.Ok(t, s.StartAndWaitReady(str)) + + // Register 1 sidecar using `--store`. + // Register 2 sidecars and 1 storeGW using `--endpoint`. + // Register `sidecar3` twice to verify it is deduplicated. + q, err := e2ethanos.NewQuerierBuilder(s.SharedDir(), "1", []string{sidecar1.GRPCNetworkEndpoint()}). + WithEndpoints([]string{ + sidecar2.GRPCNetworkEndpoint(), + sidecar3.GRPCNetworkEndpoint(), + sidecar3.GRPCNetworkEndpoint(), + str.GRPCNetworkEndpoint(), + }). + Build() + testutil.Ok(t, err) + testutil.Ok(t, s.StartAndWaitReady(q)) + + expected := map[string][]query.StoreStatus{ + "sidecar": { + { + Name: "e2e_test_info-sidecar-alone1:9091", + LabelSets: []labels.Labels{{ + { + Name: "prometheus", + Value: "prom-alone1", + }, + { + Name: "replica", + Value: "0", + }, + }}, + }, + { + Name: "e2e_test_info-sidecar-alone2:9091", + LabelSets: []labels.Labels{{ + { + Name: "prometheus", + Value: "prom-alone2", + }, + { + Name: "replica", + Value: "0", + }, + }}, + }, + { + Name: "e2e_test_info-sidecar-alone3:9091", + LabelSets: []labels.Labels{{ + { + Name: "prometheus", + Value: "prom-alone3", + }, + { + Name: "replica", + Value: "0", + }, + }}, + }, + }, + "store": { + { + Name: "e2e_test_info-store-gw-1:9091", + LabelSets: []labels.Labels{}, + }, + }, + } + + url := "http://" + path.Join(q.HTTPEndpoint(), "/api/v1/stores") + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*15) + defer cancel() + + err = runutil.Retry(time.Second, ctx.Done(), func() error { + + resp, err := http.Get(url) + testutil.Ok(t, err) + + body, err := ioutil.ReadAll(resp.Body) + testutil.Ok(t, err) + + var res struct { + Data map[string][]query.StoreStatus `json:"data"` + } + + if err = json.Unmarshal(body, &res); err != nil { + t.Fatalf("Error unmarshaling JSON body: %s", err) + } + + if err = assertStoreStatus(t, "sidecar", res.Data, expected); err != nil { + return err + } + + if err = assertStoreStatus(t, "store", res.Data, expected); err != nil { + return err + } + + return nil + }) + testutil.Ok(t, err) +} + +func assertStoreStatus(t *testing.T, component string, res map[string][]query.StoreStatus, expected map[string][]query.StoreStatus) error { + t.Helper() + + if len(res[component]) != len(expected[component]) { + return fmt.Errorf("Expected %d %s, got: %d", len(expected[component]), component, len(res[component])) + } + + for i, v := range res[component] { + // Set value of the fields which keep changing in every test run to their default value. + v.MaxTime = 0 + v.MinTime = 0 + v.LastCheck = time.Time{} + + testutil.Equals(t, expected[component][i], v) + } + + return nil +} From 95b1ce524188f161fa6fff4f922a31771feb8027 Mon Sep 17 00:00:00 2001 From: Hitanshu Mehta Date: Tue, 8 Jun 2021 23:48:39 +0530 Subject: [PATCH 04/16] Make min/max times global vaiables and some other minor changes Signed-off-by: Hitanshu Mehta --- cmd/thanos/sidecar.go | 5 +++-- cmd/thanos/store.go | 2 +- pkg/api/query/v1.go | 6 ++++++ pkg/info/info.go | 8 ++++++++ 4 files changed, 18 insertions(+), 3 deletions(-) diff --git a/cmd/thanos/sidecar.go b/cmd/thanos/sidecar.go index 269e08ede4..08b2303e01 100644 --- a/cmd/thanos/sidecar.go +++ b/cmd/thanos/sidecar.go @@ -23,6 +23,7 @@ import ( "github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/common/model" "github.com/prometheus/prometheus/pkg/labels" + v1 "github.com/thanos-io/thanos/pkg/api/query" "github.com/thanos-io/thanos/pkg/block/metadata" "github.com/thanos-io/thanos/pkg/component" "github.com/thanos-io/thanos/pkg/exemplars" @@ -262,8 +263,8 @@ func runSidecar( // Currently Exemplars API does not expose metadata such as min/max time, // so we are using default minimum and maximum possible values as min/max time. return &infopb.ExemplarsInfo{ - MinTime: time.Unix(math.MinInt64/1000+62135596801, 0).UTC().Unix(), - MaxTime: time.Unix(math.MaxInt64/1000-62135596801, 999999999).UTC().Unix(), + MinTime: v1.MinTime.Unix(), + MaxTime: v1.MaxTime.Unix(), } }, func() *infopb.RulesInfo { diff --git a/cmd/thanos/store.go b/cmd/thanos/store.go index 591c2a96ae..02cd4f5c24 100644 --- a/cmd/thanos/store.go +++ b/cmd/thanos/store.go @@ -398,7 +398,7 @@ func runStore( MaxTime: maxt, } }, - func() *infopb.ExemplarsInfo { return nil }, + nil, nil, nil, nil, diff --git a/pkg/api/query/v1.go b/pkg/api/query/v1.go index fd9ee99b68..5532013962 100644 --- a/pkg/api/query/v1.go +++ b/pkg/api/query/v1.go @@ -72,6 +72,12 @@ const ( Stats = "stats" ) +var ( + // Default minimum and maximum time values used by Prometheus when they are not passed as query parameter. + MinTime = time.Unix(math.MinInt64/1000+62135596801, 0).UTC() + MaxTime = time.Unix(math.MaxInt64/1000-62135596801, 999999999).UTC() +) + // QueryAPI is an API used by Thanos Querier. type QueryAPI struct { baseAPI *api.BaseAPI diff --git a/pkg/info/info.go b/pkg/info/info.go index f2408edce4..0043722209 100644 --- a/pkg/info/info.go +++ b/pkg/info/info.go @@ -53,6 +53,14 @@ func RegisterInfoServer(infoSrv infopb.InfoServer) func(*grpc.Server) { func (srv *InfoServer) Info(ctx context.Context, req *infopb.InfoRequest) (*infopb.InfoResponse, error) { + if srv.getStoreInfo == nil { + srv.getStoreInfo = func() *infopb.StoreInfo { return nil } + } + + if srv.getExemplarsInfo == nil { + srv.getExemplarsInfo = func() *infopb.ExemplarsInfo { return nil } + } + if srv.getRulesInfo == nil { srv.getRulesInfo = func() *infopb.RulesInfo { return nil } } From 245e9c4583a17a974a4c06a89dd5c8bd2237aac3 Mon Sep 17 00:00:00 2001 From: Hitanshu Mehta Date: Wed, 9 Jun 2021 00:37:50 +0530 Subject: [PATCH 05/16] Enhance e2e test Signed-off-by: Hitanshu Mehta --- test/e2e/info_api_test.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/test/e2e/info_api_test.go b/test/e2e/info_api_test.go index c44224a2ef..85d4f8146d 100644 --- a/test/e2e/info_api_test.go +++ b/test/e2e/info_api_test.go @@ -54,11 +54,17 @@ func TestInfo(t *testing.T) { testutil.Ok(t, err) testutil.Ok(t, s.StartAndWaitReady(str)) - // Register 1 sidecar using `--store`. - // Register 2 sidecars and 1 storeGW using `--endpoint`. + // Register `sidecar1` in all flags (i.e. '--store', '--rule', '--target', '--metadata', '--exemplar', '--endpoint') to verify + // '--endpoint' flag works properly works together with other flags ('--target', '--metadata' etc.). + // Register 2 sidecars and 1 storeGW using '--endpoint'. // Register `sidecar3` twice to verify it is deduplicated. q, err := e2ethanos.NewQuerierBuilder(s.SharedDir(), "1", []string{sidecar1.GRPCNetworkEndpoint()}). + WithTargetAddresses([]string{sidecar1.GRPCNetworkEndpoint()}). + WithMetadataAddresses([]string{sidecar1.GRPCNetworkEndpoint()}). + WithExemplarAddresses([]string{sidecar1.GRPCNetworkEndpoint()}). + WithRuleAddresses([]string{sidecar1.GRPCNetworkEndpoint()}). WithEndpoints([]string{ + sidecar1.GRPCNetworkEndpoint(), sidecar2.GRPCNetworkEndpoint(), sidecar3.GRPCNetworkEndpoint(), sidecar3.GRPCNetworkEndpoint(), From 27b9a217033383e32db67d1fb02f3ffb58e4306c Mon Sep 17 00:00:00 2001 From: Hitanshu Mehta Date: Sun, 13 Jun 2021 21:40:46 +0530 Subject: [PATCH 06/16] Rename `InfoRequest` and `InfoResponse` to `InfoReq` and `InfoResp` Signed-off-by: Hitanshu Mehta --- pkg/info/info.go | 4 +- pkg/info/infopb/rpc.pb.go | 94 +++++++++++++++++++------------------- pkg/info/infopb/rpc.proto | 6 +-- pkg/query/storeset.go | 2 +- pkg/query/storeset_test.go | 6 +-- 5 files changed, 56 insertions(+), 56 deletions(-) diff --git a/pkg/info/info.go b/pkg/info/info.go index 0043722209..2634e74940 100644 --- a/pkg/info/info.go +++ b/pkg/info/info.go @@ -51,7 +51,7 @@ func RegisterInfoServer(infoSrv infopb.InfoServer) func(*grpc.Server) { } } -func (srv *InfoServer) Info(ctx context.Context, req *infopb.InfoRequest) (*infopb.InfoResponse, error) { +func (srv *InfoServer) Info(ctx context.Context, req *infopb.InfoReq) (*infopb.InfoResp, error) { if srv.getStoreInfo == nil { srv.getStoreInfo = func() *infopb.StoreInfo { return nil } @@ -73,7 +73,7 @@ func (srv *InfoServer) Info(ctx context.Context, req *infopb.InfoRequest) (*info srv.getMetricMetadataInfo = func() *infopb.MetricMetadataInfo { return nil } } - resp := &infopb.InfoResponse{ + resp := &infopb.InfoResp{ LabelSets: srv.getLabelSet(), ComponentType: srv.component, Store: srv.getStoreInfo(), diff --git a/pkg/info/infopb/rpc.pb.go b/pkg/info/infopb/rpc.pb.go index 452ef93b22..dcfc4cb3a8 100644 --- a/pkg/info/infopb/rpc.pb.go +++ b/pkg/info/infopb/rpc.pb.go @@ -29,21 +29,21 @@ var _ = math.Inf // proto package needs to be updated. const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package -type InfoRequest struct { +type InfoReq struct { } -func (m *InfoRequest) Reset() { *m = InfoRequest{} } -func (m *InfoRequest) String() string { return proto.CompactTextString(m) } -func (*InfoRequest) ProtoMessage() {} -func (*InfoRequest) Descriptor() ([]byte, []int) { +func (m *InfoReq) Reset() { *m = InfoReq{} } +func (m *InfoReq) String() string { return proto.CompactTextString(m) } +func (*InfoReq) ProtoMessage() {} +func (*InfoReq) Descriptor() ([]byte, []int) { return fileDescriptor_a1214ec45d2bf952, []int{0} } -func (m *InfoRequest) XXX_Unmarshal(b []byte) error { +func (m *InfoReq) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } -func (m *InfoRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { +func (m *InfoReq) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { - return xxx_messageInfo_InfoRequest.Marshal(b, m, deterministic) + return xxx_messageInfo_InfoReq.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) @@ -53,19 +53,19 @@ func (m *InfoRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) return b[:n], nil } } -func (m *InfoRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_InfoRequest.Merge(m, src) +func (m *InfoReq) XXX_Merge(src proto.Message) { + xxx_messageInfo_InfoReq.Merge(m, src) } -func (m *InfoRequest) XXX_Size() int { +func (m *InfoReq) XXX_Size() int { return m.Size() } -func (m *InfoRequest) XXX_DiscardUnknown() { - xxx_messageInfo_InfoRequest.DiscardUnknown(m) +func (m *InfoReq) XXX_DiscardUnknown() { + xxx_messageInfo_InfoReq.DiscardUnknown(m) } -var xxx_messageInfo_InfoRequest proto.InternalMessageInfo +var xxx_messageInfo_InfoReq proto.InternalMessageInfo -type InfoResponse struct { +type InfoResp struct { LabelSets []labelpb.ZLabelSet `protobuf:"bytes,1,rep,name=label_sets,json=labelSets,proto3" json:"label_sets"` ComponentType string `protobuf:"bytes,2,opt,name=ComponentType,proto3" json:"ComponentType,omitempty"` /// StoreInfo holds the metadata related to Store API if exposed by the component otherwise it will be null. @@ -80,18 +80,18 @@ type InfoResponse struct { Exemplars *ExemplarsInfo `protobuf:"bytes,7,opt,name=exemplars,proto3" json:"exemplars,omitempty"` } -func (m *InfoResponse) Reset() { *m = InfoResponse{} } -func (m *InfoResponse) String() string { return proto.CompactTextString(m) } -func (*InfoResponse) ProtoMessage() {} -func (*InfoResponse) Descriptor() ([]byte, []int) { +func (m *InfoResp) Reset() { *m = InfoResp{} } +func (m *InfoResp) String() string { return proto.CompactTextString(m) } +func (*InfoResp) ProtoMessage() {} +func (*InfoResp) Descriptor() ([]byte, []int) { return fileDescriptor_a1214ec45d2bf952, []int{1} } -func (m *InfoResponse) XXX_Unmarshal(b []byte) error { +func (m *InfoResp) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } -func (m *InfoResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { +func (m *InfoResp) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { - return xxx_messageInfo_InfoResponse.Marshal(b, m, deterministic) + return xxx_messageInfo_InfoResp.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) @@ -101,17 +101,17 @@ func (m *InfoResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) return b[:n], nil } } -func (m *InfoResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_InfoResponse.Merge(m, src) +func (m *InfoResp) XXX_Merge(src proto.Message) { + xxx_messageInfo_InfoResp.Merge(m, src) } -func (m *InfoResponse) XXX_Size() int { +func (m *InfoResp) XXX_Size() int { return m.Size() } -func (m *InfoResponse) XXX_DiscardUnknown() { - xxx_messageInfo_InfoResponse.DiscardUnknown(m) +func (m *InfoResp) XXX_DiscardUnknown() { + xxx_messageInfo_InfoResp.DiscardUnknown(m) } -var xxx_messageInfo_InfoResponse proto.InternalMessageInfo +var xxx_messageInfo_InfoResp proto.InternalMessageInfo /// StoreInfo holds the metadata related to Store API exposed by the component. type StoreInfo struct { @@ -359,7 +359,7 @@ const _ = grpc.SupportPackageIsVersion4 // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. type InfoClient interface { /// Info returns the metadata (Eg. LabelSets, Min/Max time) about all the APIs the component supports. - Info(ctx context.Context, in *InfoRequest, opts ...grpc.CallOption) (*InfoResponse, error) + Info(ctx context.Context, in *InfoReq, opts ...grpc.CallOption) (*InfoResp, error) } type infoClient struct { @@ -382,14 +382,14 @@ func (c *infoClient) Info(ctx context.Context, in *InfoRequest, opts ...grpc.Cal // InfoServer is the server API for Info service. type InfoServer interface { /// Info returns the metadata (Eg. LabelSets, Min/Max time) about all the APIs the component supports. - Info(context.Context, *InfoRequest) (*InfoResponse, error) + Info(context.Context, *InfoReq) (*InfoResp, error) } // UnimplementedInfoServer can be embedded to have forward compatible implementations. type UnimplementedInfoServer struct { } -func (*UnimplementedInfoServer) Info(ctx context.Context, req *InfoRequest) (*InfoResponse, error) { +func (*UnimplementedInfoServer) Info(ctx context.Context, req *InfoReq) (*InfoResp, error) { return nil, status.Errorf(codes.Unimplemented, "method Info not implemented") } @@ -398,7 +398,7 @@ func RegisterInfoServer(s *grpc.Server, srv InfoServer) { } func _Info_Info_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(InfoRequest) + in := new(InfoReq) if err := dec(in); err != nil { return nil, err } @@ -410,7 +410,7 @@ func _Info_Info_Handler(srv interface{}, ctx context.Context, dec func(interface FullMethod: "/thanos.info.Info/Info", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(InfoServer).Info(ctx, req.(*InfoRequest)) + return srv.(InfoServer).Info(ctx, req.(*InfoReq)) } return interceptor(ctx, in, info, handler) } @@ -428,7 +428,7 @@ var _Info_serviceDesc = grpc.ServiceDesc{ Metadata: "info/infopb/rpc.proto", } -func (m *InfoRequest) Marshal() (dAtA []byte, err error) { +func (m *InfoReq) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) @@ -438,12 +438,12 @@ func (m *InfoRequest) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *InfoRequest) MarshalTo(dAtA []byte) (int, error) { +func (m *InfoReq) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } -func (m *InfoRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { +func (m *InfoReq) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int @@ -451,7 +451,7 @@ func (m *InfoRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } -func (m *InfoResponse) Marshal() (dAtA []byte, err error) { +func (m *InfoResp) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) @@ -461,12 +461,12 @@ func (m *InfoResponse) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *InfoResponse) MarshalTo(dAtA []byte) (int, error) { +func (m *InfoResp) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } -func (m *InfoResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { +func (m *InfoResp) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int @@ -701,7 +701,7 @@ func encodeVarintRpc(dAtA []byte, offset int, v uint64) int { dAtA[offset] = uint8(v) return base } -func (m *InfoRequest) Size() (n int) { +func (m *InfoReq) Size() (n int) { if m == nil { return 0 } @@ -710,7 +710,7 @@ func (m *InfoRequest) Size() (n int) { return n } -func (m *InfoResponse) Size() (n int) { +func (m *InfoResp) Size() (n int) { if m == nil { return 0 } @@ -812,7 +812,7 @@ func sovRpc(x uint64) (n int) { func sozRpc(x uint64) (n int) { return sovRpc(uint64((x << 1) ^ uint64((int64(x) >> 63)))) } -func (m *InfoRequest) Unmarshal(dAtA []byte) error { +func (m *InfoReq) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -835,10 +835,10 @@ func (m *InfoRequest) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: InfoRequest: wiretype end group for non-group") + return fmt.Errorf("proto: InfoReq: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: InfoRequest: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: InfoReq: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { default: @@ -862,7 +862,7 @@ func (m *InfoRequest) Unmarshal(dAtA []byte) error { } return nil } -func (m *InfoResponse) Unmarshal(dAtA []byte) error { +func (m *InfoResp) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -885,10 +885,10 @@ func (m *InfoResponse) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: InfoResponse: wiretype end group for non-group") + return fmt.Errorf("proto: InfoResp: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: InfoResponse: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: InfoResp: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: diff --git a/pkg/info/infopb/rpc.proto b/pkg/info/infopb/rpc.proto index 7d19c5c12d..1d6aca6336 100644 --- a/pkg/info/infopb/rpc.proto +++ b/pkg/info/infopb/rpc.proto @@ -23,12 +23,12 @@ option (gogoproto.goproto_sizecache_all) = false; /// Info represents the API that is responsible for gathering metadata about the all APIs supported by the component. service Info { /// Info returns the metadata (Eg. LabelSets, Min/Max time) about all the APIs the component supports. - rpc Info(InfoRequest) returns (InfoResponse); + rpc Info(InfoReq) returns (InfoResp); } -message InfoRequest {} +message InfoReq {} -message InfoResponse { +message InfoResp { repeated ZLabelSet label_sets = 1 [(gogoproto.nullable) = false]; string ComponentType = 2; diff --git a/pkg/query/storeset.go b/pkg/query/storeset.go index c20a75a311..14c9c28c8b 100644 --- a/pkg/query/storeset.go +++ b/pkg/query/storeset.go @@ -667,7 +667,7 @@ func (s *StoreSet) getActiveStores(ctx context.Context, stores map[string]*store } } - info, err := st.info.Info(ctx, &infopb.InfoRequest{}, grpc.WaitForReady(true)) + info, err := st.info.Info(ctx, &infopb.InfoReq{}, grpc.WaitForReady(true)) if err != nil { if !seenAlready { // Close only if new diff --git a/pkg/query/storeset_test.go b/pkg/query/storeset_test.go index 922a9fd89b..38dc2bbe62 100644 --- a/pkg/query/storeset_test.go +++ b/pkg/query/storeset_test.go @@ -132,10 +132,10 @@ func (s *testStores) CloseOne(addr string) { } type mockedInfo struct { - info infopb.InfoResponse + info infopb.InfoResp } -func (s *mockedInfo) Info(ctx context.Context, r *infopb.InfoRequest) (*infopb.InfoResponse, error) { +func (s *mockedInfo) Info(ctx context.Context, r *infopb.InfoReq) (*infopb.InfoResp, error) { return &s.info, nil } @@ -170,7 +170,7 @@ func startInfoSrvs(infoMetas []testInfoMeta) (*testInfoSrvs, error) { srv := grpc.NewServer() infoSrv := &mockedInfo{ - info: infopb.InfoResponse{ + info: infopb.InfoResp{ LabelSets: meta.extlsetFn(listener.Addr().String()), Store: &meta.store, MetricMetadata: &meta.metadata, From 7f941770807c3acf62ee0a037f263616dafc3f35 Mon Sep 17 00:00:00 2001 From: Hitanshu Mehta Date: Thu, 1 Jul 2021 23:05:57 +0530 Subject: [PATCH 07/16] Minor fixes and change package name for proto file for info service Signed-off-by: Hitanshu Mehta --- pkg/info/info.go | 4 +- pkg/info/infopb/rpc.pb.go | 94 +-- pkg/info/infopb/rpc.proto | 6 +- pkg/query/storeset.go | 874 ----------------------- pkg/query/storeset_test.go | 1331 ------------------------------------ test/e2e/info_api_test.go | 12 +- 6 files changed, 59 insertions(+), 2262 deletions(-) delete mode 100644 pkg/query/storeset.go delete mode 100644 pkg/query/storeset_test.go diff --git a/pkg/info/info.go b/pkg/info/info.go index 2634e74940..0043722209 100644 --- a/pkg/info/info.go +++ b/pkg/info/info.go @@ -51,7 +51,7 @@ func RegisterInfoServer(infoSrv infopb.InfoServer) func(*grpc.Server) { } } -func (srv *InfoServer) Info(ctx context.Context, req *infopb.InfoReq) (*infopb.InfoResp, error) { +func (srv *InfoServer) Info(ctx context.Context, req *infopb.InfoRequest) (*infopb.InfoResponse, error) { if srv.getStoreInfo == nil { srv.getStoreInfo = func() *infopb.StoreInfo { return nil } @@ -73,7 +73,7 @@ func (srv *InfoServer) Info(ctx context.Context, req *infopb.InfoReq) (*infopb.I srv.getMetricMetadataInfo = func() *infopb.MetricMetadataInfo { return nil } } - resp := &infopb.InfoResp{ + resp := &infopb.InfoResponse{ LabelSets: srv.getLabelSet(), ComponentType: srv.component, Store: srv.getStoreInfo(), diff --git a/pkg/info/infopb/rpc.pb.go b/pkg/info/infopb/rpc.pb.go index dcfc4cb3a8..452ef93b22 100644 --- a/pkg/info/infopb/rpc.pb.go +++ b/pkg/info/infopb/rpc.pb.go @@ -29,21 +29,21 @@ var _ = math.Inf // proto package needs to be updated. const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package -type InfoReq struct { +type InfoRequest struct { } -func (m *InfoReq) Reset() { *m = InfoReq{} } -func (m *InfoReq) String() string { return proto.CompactTextString(m) } -func (*InfoReq) ProtoMessage() {} -func (*InfoReq) Descriptor() ([]byte, []int) { +func (m *InfoRequest) Reset() { *m = InfoRequest{} } +func (m *InfoRequest) String() string { return proto.CompactTextString(m) } +func (*InfoRequest) ProtoMessage() {} +func (*InfoRequest) Descriptor() ([]byte, []int) { return fileDescriptor_a1214ec45d2bf952, []int{0} } -func (m *InfoReq) XXX_Unmarshal(b []byte) error { +func (m *InfoRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } -func (m *InfoReq) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { +func (m *InfoRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { - return xxx_messageInfo_InfoReq.Marshal(b, m, deterministic) + return xxx_messageInfo_InfoRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) @@ -53,19 +53,19 @@ func (m *InfoReq) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return b[:n], nil } } -func (m *InfoReq) XXX_Merge(src proto.Message) { - xxx_messageInfo_InfoReq.Merge(m, src) +func (m *InfoRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_InfoRequest.Merge(m, src) } -func (m *InfoReq) XXX_Size() int { +func (m *InfoRequest) XXX_Size() int { return m.Size() } -func (m *InfoReq) XXX_DiscardUnknown() { - xxx_messageInfo_InfoReq.DiscardUnknown(m) +func (m *InfoRequest) XXX_DiscardUnknown() { + xxx_messageInfo_InfoRequest.DiscardUnknown(m) } -var xxx_messageInfo_InfoReq proto.InternalMessageInfo +var xxx_messageInfo_InfoRequest proto.InternalMessageInfo -type InfoResp struct { +type InfoResponse struct { LabelSets []labelpb.ZLabelSet `protobuf:"bytes,1,rep,name=label_sets,json=labelSets,proto3" json:"label_sets"` ComponentType string `protobuf:"bytes,2,opt,name=ComponentType,proto3" json:"ComponentType,omitempty"` /// StoreInfo holds the metadata related to Store API if exposed by the component otherwise it will be null. @@ -80,18 +80,18 @@ type InfoResp struct { Exemplars *ExemplarsInfo `protobuf:"bytes,7,opt,name=exemplars,proto3" json:"exemplars,omitempty"` } -func (m *InfoResp) Reset() { *m = InfoResp{} } -func (m *InfoResp) String() string { return proto.CompactTextString(m) } -func (*InfoResp) ProtoMessage() {} -func (*InfoResp) Descriptor() ([]byte, []int) { +func (m *InfoResponse) Reset() { *m = InfoResponse{} } +func (m *InfoResponse) String() string { return proto.CompactTextString(m) } +func (*InfoResponse) ProtoMessage() {} +func (*InfoResponse) Descriptor() ([]byte, []int) { return fileDescriptor_a1214ec45d2bf952, []int{1} } -func (m *InfoResp) XXX_Unmarshal(b []byte) error { +func (m *InfoResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } -func (m *InfoResp) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { +func (m *InfoResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { - return xxx_messageInfo_InfoResp.Marshal(b, m, deterministic) + return xxx_messageInfo_InfoResponse.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) @@ -101,17 +101,17 @@ func (m *InfoResp) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return b[:n], nil } } -func (m *InfoResp) XXX_Merge(src proto.Message) { - xxx_messageInfo_InfoResp.Merge(m, src) +func (m *InfoResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_InfoResponse.Merge(m, src) } -func (m *InfoResp) XXX_Size() int { +func (m *InfoResponse) XXX_Size() int { return m.Size() } -func (m *InfoResp) XXX_DiscardUnknown() { - xxx_messageInfo_InfoResp.DiscardUnknown(m) +func (m *InfoResponse) XXX_DiscardUnknown() { + xxx_messageInfo_InfoResponse.DiscardUnknown(m) } -var xxx_messageInfo_InfoResp proto.InternalMessageInfo +var xxx_messageInfo_InfoResponse proto.InternalMessageInfo /// StoreInfo holds the metadata related to Store API exposed by the component. type StoreInfo struct { @@ -359,7 +359,7 @@ const _ = grpc.SupportPackageIsVersion4 // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. type InfoClient interface { /// Info returns the metadata (Eg. LabelSets, Min/Max time) about all the APIs the component supports. - Info(ctx context.Context, in *InfoReq, opts ...grpc.CallOption) (*InfoResp, error) + Info(ctx context.Context, in *InfoRequest, opts ...grpc.CallOption) (*InfoResponse, error) } type infoClient struct { @@ -382,14 +382,14 @@ func (c *infoClient) Info(ctx context.Context, in *InfoRequest, opts ...grpc.Cal // InfoServer is the server API for Info service. type InfoServer interface { /// Info returns the metadata (Eg. LabelSets, Min/Max time) about all the APIs the component supports. - Info(context.Context, *InfoReq) (*InfoResp, error) + Info(context.Context, *InfoRequest) (*InfoResponse, error) } // UnimplementedInfoServer can be embedded to have forward compatible implementations. type UnimplementedInfoServer struct { } -func (*UnimplementedInfoServer) Info(ctx context.Context, req *InfoReq) (*InfoResp, error) { +func (*UnimplementedInfoServer) Info(ctx context.Context, req *InfoRequest) (*InfoResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method Info not implemented") } @@ -398,7 +398,7 @@ func RegisterInfoServer(s *grpc.Server, srv InfoServer) { } func _Info_Info_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(InfoReq) + in := new(InfoRequest) if err := dec(in); err != nil { return nil, err } @@ -410,7 +410,7 @@ func _Info_Info_Handler(srv interface{}, ctx context.Context, dec func(interface FullMethod: "/thanos.info.Info/Info", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(InfoServer).Info(ctx, req.(*InfoReq)) + return srv.(InfoServer).Info(ctx, req.(*InfoRequest)) } return interceptor(ctx, in, info, handler) } @@ -428,7 +428,7 @@ var _Info_serviceDesc = grpc.ServiceDesc{ Metadata: "info/infopb/rpc.proto", } -func (m *InfoReq) Marshal() (dAtA []byte, err error) { +func (m *InfoRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) @@ -438,12 +438,12 @@ func (m *InfoReq) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *InfoReq) MarshalTo(dAtA []byte) (int, error) { +func (m *InfoRequest) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } -func (m *InfoReq) MarshalToSizedBuffer(dAtA []byte) (int, error) { +func (m *InfoRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int @@ -451,7 +451,7 @@ func (m *InfoReq) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } -func (m *InfoResp) Marshal() (dAtA []byte, err error) { +func (m *InfoResponse) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) @@ -461,12 +461,12 @@ func (m *InfoResp) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *InfoResp) MarshalTo(dAtA []byte) (int, error) { +func (m *InfoResponse) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } -func (m *InfoResp) MarshalToSizedBuffer(dAtA []byte) (int, error) { +func (m *InfoResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int @@ -701,7 +701,7 @@ func encodeVarintRpc(dAtA []byte, offset int, v uint64) int { dAtA[offset] = uint8(v) return base } -func (m *InfoReq) Size() (n int) { +func (m *InfoRequest) Size() (n int) { if m == nil { return 0 } @@ -710,7 +710,7 @@ func (m *InfoReq) Size() (n int) { return n } -func (m *InfoResp) Size() (n int) { +func (m *InfoResponse) Size() (n int) { if m == nil { return 0 } @@ -812,7 +812,7 @@ func sovRpc(x uint64) (n int) { func sozRpc(x uint64) (n int) { return sovRpc(uint64((x << 1) ^ uint64((int64(x) >> 63)))) } -func (m *InfoReq) Unmarshal(dAtA []byte) error { +func (m *InfoRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -835,10 +835,10 @@ func (m *InfoReq) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: InfoReq: wiretype end group for non-group") + return fmt.Errorf("proto: InfoRequest: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: InfoReq: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: InfoRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { default: @@ -862,7 +862,7 @@ func (m *InfoReq) Unmarshal(dAtA []byte) error { } return nil } -func (m *InfoResp) Unmarshal(dAtA []byte) error { +func (m *InfoResponse) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -885,10 +885,10 @@ func (m *InfoResp) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: InfoResp: wiretype end group for non-group") + return fmt.Errorf("proto: InfoResponse: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: InfoResp: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: InfoResponse: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: diff --git a/pkg/info/infopb/rpc.proto b/pkg/info/infopb/rpc.proto index 1d6aca6336..7d19c5c12d 100644 --- a/pkg/info/infopb/rpc.proto +++ b/pkg/info/infopb/rpc.proto @@ -23,12 +23,12 @@ option (gogoproto.goproto_sizecache_all) = false; /// Info represents the API that is responsible for gathering metadata about the all APIs supported by the component. service Info { /// Info returns the metadata (Eg. LabelSets, Min/Max time) about all the APIs the component supports. - rpc Info(InfoReq) returns (InfoResp); + rpc Info(InfoRequest) returns (InfoResponse); } -message InfoReq {} +message InfoRequest {} -message InfoResp { +message InfoResponse { repeated ZLabelSet label_sets = 1 [(gogoproto.nullable) = false]; string ComponentType = 2; diff --git a/pkg/query/storeset.go b/pkg/query/storeset.go deleted file mode 100644 index 14c9c28c8b..0000000000 --- a/pkg/query/storeset.go +++ /dev/null @@ -1,874 +0,0 @@ -// Copyright (c) The Thanos Authors. -// Licensed under the Apache License 2.0. - -package query - -import ( - "context" - "encoding/json" - "fmt" - "math" - "sort" - "sync" - "time" - - "github.com/go-kit/kit/log" - "github.com/go-kit/kit/log/level" - "github.com/pkg/errors" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/prometheus/pkg/labels" - "github.com/thanos-io/thanos/pkg/exemplars/exemplarspb" - "github.com/thanos-io/thanos/pkg/info/infopb" - "google.golang.org/grpc" - - "github.com/thanos-io/thanos/pkg/component" - "github.com/thanos-io/thanos/pkg/metadata/metadatapb" - "github.com/thanos-io/thanos/pkg/rules/rulespb" - "github.com/thanos-io/thanos/pkg/runutil" - "github.com/thanos-io/thanos/pkg/store" - "github.com/thanos-io/thanos/pkg/store/labelpb" - "github.com/thanos-io/thanos/pkg/store/storepb" - "github.com/thanos-io/thanos/pkg/targets/targetspb" -) - -const ( - unhealthyStoreMessage = "removing store because it's unhealthy or does not exist" -) - -type StoreSpec interface { - // Addr returns StoreAPI Address for the store spec. It is used as ID for store. - Addr() string - // Metadata returns current labels, store type and min, max ranges for store. - // It can change for every call for this method. - // If metadata call fails we assume that store is no longer accessible and we should not use it. - // NOTE: It is implementation responsibility to retry until context timeout, but a caller responsibility to manage - // given store connection. - Metadata(ctx context.Context, client storepb.StoreClient) (labelSets []labels.Labels, mint int64, maxt int64, storeType component.StoreAPI, err error) - - // StrictStatic returns true if the StoreAPI has been statically defined and it is under a strict mode. - StrictStatic() bool -} - -type RuleSpec interface { - // Addr returns RulesAPI Address for the rules spec. It is used as its ID. - Addr() string -} - -type TargetSpec interface { - // Addr returns TargetsAPI Address for the targets spec. It is used as its ID. - Addr() string -} - -type MetadataSpec interface { - // Addr returns MetadataAPI Address for the metadata spec. It is used as its ID. - Addr() string -} - -type ExemplarSpec interface { - // Addr returns ExemplarsAPI Address for the exemplars spec. It is used as its ID. - Addr() string -} - -type InfoSpec interface { - // Addr returns InfoAPI Address for the info spec. It is used as its ID. - Addr() string -} - -// stringError forces the error to be a string -// when marshaled into a JSON. -type stringError struct { - originalErr error -} - -// MarshalJSON marshals the error into a string form. -func (e *stringError) MarshalJSON() ([]byte, error) { - return json.Marshal(e.originalErr.Error()) -} - -// Error returns the original underlying error. -func (e *stringError) Error() string { - return e.originalErr.Error() -} - -type StoreStatus struct { - Name string `json:"name"` - LastCheck time.Time `json:"lastCheck"` - LastError *stringError `json:"lastError"` - LabelSets []labels.Labels `json:"labelSets"` - StoreType component.StoreAPI `json:"-"` - MinTime int64 `json:"minTime"` - MaxTime int64 `json:"maxTime"` -} - -type grpcStoreSpec struct { - addr string - strictstatic bool -} - -// NewGRPCStoreSpec creates store pure gRPC spec. -// It uses Info gRPC call to get Metadata. -func NewGRPCStoreSpec(addr string, strictstatic bool) StoreSpec { - return &grpcStoreSpec{addr: addr, strictstatic: strictstatic} -} - -// StrictStatic returns true if the StoreAPI has been statically defined and it is under a strict mode. -func (s *grpcStoreSpec) StrictStatic() bool { - return s.strictstatic -} - -func (s *grpcStoreSpec) Addr() string { - // API addr should not change between state changes. - return s.addr -} - -// Metadata method for gRPC store API tries to reach host Info method until context timeout. If we are unable to get metadata after -// that time, we assume that the host is unhealthy and return error. -func (s *grpcStoreSpec) Metadata(ctx context.Context, client storepb.StoreClient) (labelSets []labels.Labels, mint int64, maxt int64, Type component.StoreAPI, err error) { - resp, err := client.Info(ctx, &storepb.InfoRequest{}, grpc.WaitForReady(true)) - if err != nil { - return nil, 0, 0, nil, errors.Wrapf(err, "fetching store info from %s", s.addr) - } - if len(resp.LabelSets) == 0 && len(resp.Labels) > 0 { - resp.LabelSets = []labelpb.ZLabelSet{{Labels: resp.Labels}} - } - - labelSets = make([]labels.Labels, 0, len(resp.LabelSets)) - for _, ls := range resp.LabelSets { - labelSets = append(labelSets, ls.PromLabels()) - } - return labelSets, resp.MinTime, resp.MaxTime, component.FromProto(resp.StoreType), nil -} - -// storeSetNodeCollector is a metric collector reporting the number of available storeAPIs for Querier. -// A Collector is required as we want atomic updates for all 'thanos_store_nodes_grpc_connections' series. -type storeSetNodeCollector struct { - mtx sync.Mutex - storeNodes map[component.StoreAPI]map[string]int - storePerExtLset map[string]int - - connectionsDesc *prometheus.Desc -} - -func newStoreSetNodeCollector() *storeSetNodeCollector { - return &storeSetNodeCollector{ - storeNodes: map[component.StoreAPI]map[string]int{}, - connectionsDesc: prometheus.NewDesc( - "thanos_store_nodes_grpc_connections", - "Number of gRPC connection to Store APIs. Opened connection means healthy store APIs available for Querier.", - []string{"external_labels", "store_type"}, nil, - ), - } -} - -func (c *storeSetNodeCollector) Update(nodes map[component.StoreAPI]map[string]int) { - storeNodes := make(map[component.StoreAPI]map[string]int, len(nodes)) - storePerExtLset := map[string]int{} - - for k, v := range nodes { - storeNodes[k] = make(map[string]int, len(v)) - for kk, vv := range v { - storePerExtLset[kk] += vv - storeNodes[k][kk] = vv - } - } - - c.mtx.Lock() - defer c.mtx.Unlock() - c.storeNodes = storeNodes - c.storePerExtLset = storePerExtLset -} - -func (c *storeSetNodeCollector) Describe(ch chan<- *prometheus.Desc) { - ch <- c.connectionsDesc -} - -func (c *storeSetNodeCollector) Collect(ch chan<- prometheus.Metric) { - c.mtx.Lock() - defer c.mtx.Unlock() - - for storeType, occurrencesPerExtLset := range c.storeNodes { - for externalLabels, occurrences := range occurrencesPerExtLset { - var storeTypeStr string - if storeType != nil { - storeTypeStr = storeType.String() - } - ch <- prometheus.MustNewConstMetric(c.connectionsDesc, prometheus.GaugeValue, float64(occurrences), externalLabels, storeTypeStr) - } - } -} - -// StoreSet maintains a set of active stores. It is backed up by Store Specifications that are dynamically fetched on -// every Update() call. -type StoreSet struct { - logger log.Logger - - // Store specifications can change dynamically. If some store is missing from the list, we assuming it is no longer - // accessible and we close gRPC client for it. - infoSpec func() []InfoSpec - storeSpecs func() []StoreSpec - ruleSpecs func() []RuleSpec - targetSpecs func() []TargetSpec - metadataSpecs func() []MetadataSpec - exemplarSpecs func() []ExemplarSpec - dialOpts []grpc.DialOption - gRPCInfoCallTimeout time.Duration - - updateMtx sync.Mutex - storesMtx sync.RWMutex - storesStatusesMtx sync.RWMutex - - // Main map of stores currently used for fanout. - stores map[string]*storeRef - storesMetric *storeSetNodeCollector - - // Map of statuses used only by UI. - storeStatuses map[string]*StoreStatus - unhealthyStoreTimeout time.Duration -} - -// NewStoreSet returns a new set of store APIs and potentially Rules APIs from given specs. -func NewStoreSet( - logger log.Logger, - reg *prometheus.Registry, - storeSpecs func() []StoreSpec, - ruleSpecs func() []RuleSpec, - targetSpecs func() []TargetSpec, - metadataSpecs func() []MetadataSpec, - exemplarSpecs func() []ExemplarSpec, - infoSpecs func() []InfoSpec, - dialOpts []grpc.DialOption, - unhealthyStoreTimeout time.Duration, -) *StoreSet { - storesMetric := newStoreSetNodeCollector() - if reg != nil { - reg.MustRegister(storesMetric) - } - - if logger == nil { - logger = log.NewNopLogger() - } - - if infoSpecs == nil { - infoSpecs = func() []InfoSpec { return nil } - } - - if storeSpecs == nil { - storeSpecs = func() []StoreSpec { return nil } - } - if ruleSpecs == nil { - ruleSpecs = func() []RuleSpec { return nil } - } - if targetSpecs == nil { - targetSpecs = func() []TargetSpec { return nil } - } - if metadataSpecs == nil { - metadataSpecs = func() []MetadataSpec { return nil } - } - if exemplarSpecs == nil { - exemplarSpecs = func() []ExemplarSpec { return nil } - } - - ss := &StoreSet{ - logger: log.With(logger, "component", "storeset"), - infoSpec: infoSpecs, - storeSpecs: storeSpecs, - ruleSpecs: ruleSpecs, - targetSpecs: targetSpecs, - metadataSpecs: metadataSpecs, - exemplarSpecs: exemplarSpecs, - dialOpts: dialOpts, - storesMetric: storesMetric, - gRPCInfoCallTimeout: 5 * time.Second, - stores: make(map[string]*storeRef), - storeStatuses: make(map[string]*StoreStatus), - unhealthyStoreTimeout: unhealthyStoreTimeout, - } - return ss -} - -// TODO(bwplotka): Consider moving storeRef out of this package and renaming it, as it also supports rules API. -type storeRef struct { - storepb.StoreClient - - mtx sync.RWMutex - cc *grpc.ClientConn - addr string - // If rule is not nil, then this store also supports rules API. - rule rulespb.RulesClient - metadata metadatapb.MetadataClient - - // If exemplar is not nil, then this store also support exemplars API. - exemplar exemplarspb.ExemplarsClient - - // If target is not nil, then this store also supports targets API. - target targetspb.TargetsClient - - // If info is not nil, then this store also supports Info API. - info infopb.InfoClient - - // Meta (can change during runtime). - labelSets []labels.Labels - storeType component.StoreAPI - minTime int64 - maxTime int64 - - logger log.Logger -} - -func (s *storeRef) Update(labelSets []labels.Labels, minTime int64, maxTime int64, storeType component.StoreAPI, rule rulespb.RulesClient, target targetspb.TargetsClient, metadata metadatapb.MetadataClient, exemplar exemplarspb.ExemplarsClient) { - s.mtx.Lock() - defer s.mtx.Unlock() - - s.storeType = storeType - s.labelSets = labelSets - s.minTime = minTime - s.maxTime = maxTime - s.rule = rule - s.target = target - s.metadata = metadata - s.exemplar = exemplar -} - -func (s *storeRef) UpdateWithStore(labelSets []labels.Labels, minTime int64, maxTime int64, storeType component.StoreAPI, store storepb.StoreClient, rule rulespb.RulesClient, target targetspb.TargetsClient, metadata metadatapb.MetadataClient, exemplar exemplarspb.ExemplarsClient) { - s.mtx.Lock() - defer s.mtx.Unlock() - - s.storeType = storeType - s.labelSets = labelSets - s.minTime = minTime - s.maxTime = maxTime - s.StoreClient = store - s.rule = rule - s.target = target - s.metadata = metadata - s.exemplar = exemplar -} - -func (s *storeRef) StoreType() component.StoreAPI { - s.mtx.RLock() - defer s.mtx.RUnlock() - - return s.storeType -} - -func (s *storeRef) HasRulesAPI() bool { - s.mtx.RLock() - defer s.mtx.RUnlock() - - return s.rule != nil -} - -func (s *storeRef) HasTargetsAPI() bool { - s.mtx.RLock() - defer s.mtx.RUnlock() - - return s.target != nil -} - -func (s *storeRef) HasMetadataAPI() bool { - s.mtx.RLock() - defer s.mtx.RUnlock() - - return s.metadata != nil -} - -func (s *storeRef) HasExemplarsAPI() bool { - s.mtx.RLock() - defer s.mtx.RUnlock() - - return s.exemplar != nil -} - -func (s *storeRef) LabelSets() []labels.Labels { - s.mtx.RLock() - defer s.mtx.RUnlock() - - labelSet := make([]labels.Labels, 0, len(s.labelSets)) - for _, ls := range s.labelSets { - if len(ls) == 0 { - continue - } - // Compatibility label for Queriers pre 0.8.1. Filter it out now. - if ls[0].Name == store.CompatibilityTypeLabelName { - continue - } - labelSet = append(labelSet, ls.Copy()) - } - return labelSet -} - -func (s *storeRef) TimeRange() (mint int64, maxt int64) { - s.mtx.RLock() - defer s.mtx.RUnlock() - - return s.minTime, s.maxTime -} - -func (s *storeRef) String() string { - mint, maxt := s.TimeRange() - return fmt.Sprintf("Addr: %s LabelSets: %v Mint: %d Maxt: %d", s.addr, labelpb.PromLabelSetsToString(s.LabelSets()), mint, maxt) -} - -func (s *storeRef) Addr() string { - return s.addr -} - -func (s *storeRef) Close() { - runutil.CloseWithLogOnErr(s.logger, s.cc, fmt.Sprintf("store %v connection close", s.addr)) -} - -func newStoreAPIStats() map[component.StoreAPI]map[string]int { - nodes := make(map[component.StoreAPI]map[string]int, len(storepb.StoreType_name)) - for i := range storepb.StoreType_name { - nodes[component.FromProto(storepb.StoreType(i))] = map[string]int{} - } - return nodes -} - -// Update updates the store set. It fetches current list of store specs from function and updates the fresh metadata -// from all stores. Keeps around statically defined nodes that were defined with the strict mode. -func (s *StoreSet) Update(ctx context.Context) { - s.updateMtx.Lock() - defer s.updateMtx.Unlock() - - s.storesMtx.RLock() - stores := make(map[string]*storeRef, len(s.stores)) - for addr, st := range s.stores { - stores[addr] = st - } - s.storesMtx.RUnlock() - - level.Debug(s.logger).Log("msg", "starting updating storeAPIs", "cachedStores", len(stores)) - - activeStores := s.getActiveStores(ctx, stores) - level.Debug(s.logger).Log("msg", "checked requested storeAPIs", "activeStores", len(activeStores), "cachedStores", len(stores)) - - stats := newStoreAPIStats() - - // Close stores that where not active this time (are not in active stores map). - for addr, st := range stores { - if _, ok := activeStores[addr]; ok { - stats[st.StoreType()][labelpb.PromLabelSetsToString(st.LabelSets())]++ - continue - } - - st.Close() - delete(stores, addr) - s.updateStoreStatus(st, errors.New(unhealthyStoreMessage)) - level.Info(s.logger).Log("msg", unhealthyStoreMessage, "address", addr, "extLset", labelpb.PromLabelSetsToString(st.LabelSets())) - } - - // Add stores that are not yet in stores. - for addr, st := range activeStores { - if _, ok := stores[addr]; ok { - continue - } - - extLset := labelpb.PromLabelSetsToString(st.LabelSets()) - - // All producers should have unique external labels. While this does not check only StoreAPIs connected to - // this querier this allows to notify early user about misconfiguration. Warn only. This is also detectable from metric. - if st.StoreType() != nil && - (st.StoreType() == component.Sidecar || st.StoreType() == component.Rule) && - stats[component.Sidecar][extLset]+stats[component.Rule][extLset] > 0 { - - level.Warn(s.logger).Log("msg", "found duplicate storeAPI producer (sidecar or ruler). This is not advices as it will malform data in in the same bucket", - "address", addr, "extLset", extLset, "duplicates", fmt.Sprintf("%v", stats[component.Sidecar][extLset]+stats[component.Rule][extLset]+1)) - } - stats[st.StoreType()][extLset]++ - - stores[addr] = st - s.updateStoreStatus(st, nil) - - if st.HasRulesAPI() { - level.Info(s.logger).Log("msg", "adding new rulesAPI to query storeset", "address", addr) - } - - if st.HasExemplarsAPI() { - level.Info(s.logger).Log("msg", "adding new exemplarsAPI to query storeset", "address", addr) - } - - if st.HasTargetsAPI() { - level.Info(s.logger).Log("msg", "adding new targetsAPI to query storeset", "address", addr) - } - - level.Info(s.logger).Log("msg", "adding new storeAPI to query storeset", "address", addr, "extLset", extLset) - } - - s.storesMetric.Update(stats) - s.storesMtx.Lock() - s.stores = stores - s.storesMtx.Unlock() - - s.cleanUpStoreStatuses(stores) -} - -func (s *StoreSet) getActiveStores(ctx context.Context, stores map[string]*storeRef) map[string]*storeRef { - var ( - // UNIQUE? - activeStores = make(map[string]*storeRef, len(stores)) - mtx sync.Mutex - wg sync.WaitGroup - - storeAddrSet = make(map[string]struct{}) - ruleAddrSet = make(map[string]struct{}) - targetAddrSet = make(map[string]struct{}) - metadataAddrSet = make(map[string]struct{}) - exemplarAddrSet = make(map[string]struct{}) - infoAddrSet = make(map[string]struct{}) - ) - - // Gather active stores map concurrently. Build new store if does not exist already. - for _, ruleSpec := range s.ruleSpecs() { - ruleAddrSet[ruleSpec.Addr()] = struct{}{} - } - - // Gather active targets map concurrently. Add a new target if it does not exist already. - for _, targetSpec := range s.targetSpecs() { - targetAddrSet[targetSpec.Addr()] = struct{}{} - } - - // Gather active stores map concurrently. Build new store if does not exist already. - for _, metadataSpec := range s.metadataSpecs() { - metadataAddrSet[metadataSpec.Addr()] = struct{}{} - } - - // Gather active stores map concurrently. Build new store if does not exist already. - for _, exemplarSpec := range s.exemplarSpecs() { - exemplarAddrSet[exemplarSpec.Addr()] = struct{}{} - } - - // Gather healthy stores map concurrently. Build new store if does not exist already. - for _, storeSpec := range s.storeSpecs() { - if _, ok := storeAddrSet[storeSpec.Addr()]; ok { - level.Warn(s.logger).Log("msg", "duplicated address in store nodes", "address", storeSpec.Addr()) - continue - } - storeAddrSet[storeSpec.Addr()] = struct{}{} - - wg.Add(1) - go func(spec StoreSpec) { - defer wg.Done() - - addr := spec.Addr() - - ctx, cancel := context.WithTimeout(ctx, s.gRPCInfoCallTimeout) - defer cancel() - - st, seenAlready := stores[addr] - if !seenAlready { - // New store or was unactive and was removed in the past - create new one. - conn, err := grpc.DialContext(ctx, addr, s.dialOpts...) - if err != nil { - s.updateStoreStatus(&storeRef{addr: addr}, err) - level.Warn(s.logger).Log("msg", "update of store node failed", "err", errors.Wrap(err, "dialing connection"), "address", addr) - return - } - - st = &storeRef{StoreClient: storepb.NewStoreClient(conn), storeType: component.UnknownStoreAPI, cc: conn, addr: addr, logger: s.logger} - if spec.StrictStatic() { - st.maxTime = math.MaxInt64 - } - } - - var rule rulespb.RulesClient - if _, ok := ruleAddrSet[addr]; ok { - rule = rulespb.NewRulesClient(st.cc) - } - - var target targetspb.TargetsClient - if _, ok := targetAddrSet[addr]; ok { - target = targetspb.NewTargetsClient(st.cc) - } - - var metadata metadatapb.MetadataClient - if _, ok := metadataAddrSet[addr]; ok { - metadata = metadatapb.NewMetadataClient(st.cc) - } - - var exemplar exemplarspb.ExemplarsClient - if _, ok := exemplarAddrSet[addr]; ok { - exemplar = exemplarspb.NewExemplarsClient(st.cc) - } - - // Check existing or new store. Is it healthy? What are current metadata? - labelSets, minTime, maxTime, storeType, err := spec.Metadata(ctx, st.StoreClient) - if err != nil { - if !seenAlready && !spec.StrictStatic() { - // Close only if new and not a strict static node. - // Unactive `s.stores` will be closed later on. - st.Close() - } - s.updateStoreStatus(st, err) - level.Warn(s.logger).Log("msg", "update of store node failed", "err", errors.Wrap(err, "getting metadata"), "address", addr) - - if !spec.StrictStatic() { - return - } - - // Still keep it around if static & strict mode enabled. - mtx.Lock() - defer mtx.Unlock() - - activeStores[addr] = st - return - } - - s.updateStoreStatus(st, nil) - st.Update(labelSets, minTime, maxTime, storeType, rule, target, metadata, exemplar) - - mtx.Lock() - defer mtx.Unlock() - - activeStores[addr] = st - }(storeSpec) - } - wg.Wait() - - for ruleAddr := range ruleAddrSet { - if _, ok := storeAddrSet[ruleAddr]; !ok { - level.Warn(s.logger).Log("msg", "ignored rule store", "address", ruleAddr) - } - } - - // Gather healthy stores map concurrently using info addresses. Build new store if does not exist already. - for _, infoSpec := range s.infoSpec() { - if _, ok := infoAddrSet[infoSpec.Addr()]; ok { - level.Warn(s.logger).Log("msg", "duplicated address in info nodes", "address", infoSpec.Addr()) - continue - } - infoAddrSet[infoSpec.Addr()] = struct{}{} - - wg.Add(1) - go func(spec InfoSpec) { - defer wg.Done() - - addr := spec.Addr() - - ctx, cancel := context.WithTimeout(ctx, s.gRPCInfoCallTimeout) - defer cancel() - - st, seenAlready := stores[addr] - if !seenAlready { - // New store or was unactive and was removed in the past - create the new one. - conn, err := grpc.DialContext(ctx, addr, s.dialOpts...) - if err != nil { - s.updateStoreStatus(&storeRef{addr: addr}, err) - level.Warn(s.logger).Log("msg", "update of store node failed", "err", errors.Wrap(err, "dialing connection"), "address", addr) - return - } - - st = &storeRef{ - info: infopb.NewInfoClient(conn), - storeType: component.UnknownStoreAPI, - cc: conn, - addr: addr, - logger: s.logger, - } - } - - info, err := st.info.Info(ctx, &infopb.InfoReq{}, grpc.WaitForReady(true)) - if err != nil { - if !seenAlready { - // Close only if new - // Unactive `s.stores` will be closed later on. - st.Close() - } - - s.updateStoreStatus(st, err) - level.Warn(s.logger).Log("msg", "update of node failed", "err", errors.Wrap(err, "getting metadata"), "address", addr) - - return - } - - s.updateStoreStatus(st, nil) - - labelSets := make([]labels.Labels, 0, len(info.LabelSets)) - for _, ls := range info.LabelSets { - labelSets = append(labelSets, ls.PromLabels()) - } - - var minTime, maxTime int64 - var store storepb.StoreClient - if info.Store != nil { - store = storepb.NewStoreClient(st.cc) - minTime = info.Store.MinTime - maxTime = info.Store.MaxTime - } - - var rule rulespb.RulesClient - if info.Rules != nil { - rule = rulespb.NewRulesClient(st.cc) - } - - var target targetspb.TargetsClient - if info.Targets != nil { - target = targetspb.NewTargetsClient(st.cc) - } - - var metadata metadatapb.MetadataClient - if info.MetricMetadata != nil { - metadata = metadatapb.NewMetadataClient(st.cc) - } - - var exemplar exemplarspb.ExemplarsClient - if info.Exemplars != nil { - // min/max range is also provided by in the response of Info rpc call - // but we are not using this metadata anywhere right now so ignoring. - exemplar = exemplarspb.NewExemplarsClient(st.cc) - } - - s.updateStoreStatus(st, nil) - st.UpdateWithStore(labelSets, minTime, maxTime, component.FromString(info.ComponentType), store, rule, target, metadata, exemplar) - - mtx.Lock() - defer mtx.Unlock() - - activeStores[addr] = st - }(infoSpec) - } - wg.Wait() - - return activeStores -} - -func (s *StoreSet) updateStoreStatus(store *storeRef, err error) { - s.storesStatusesMtx.Lock() - defer s.storesStatusesMtx.Unlock() - - status := StoreStatus{Name: store.addr} - prev, ok := s.storeStatuses[store.addr] - if ok { - status = *prev - } else { - mint, maxt := store.TimeRange() - status.MinTime = mint - status.MaxTime = maxt - } - - if err == nil { - status.LastCheck = time.Now() - mint, maxt := store.TimeRange() - status.LabelSets = store.LabelSets() - status.StoreType = store.StoreType() - status.MinTime = mint - status.MaxTime = maxt - status.LastError = nil - } else { - status.LastError = &stringError{originalErr: err} - } - - s.storeStatuses[store.addr] = &status -} - -func (s *StoreSet) GetStoreStatus() []StoreStatus { - s.storesStatusesMtx.RLock() - defer s.storesStatusesMtx.RUnlock() - - statuses := make([]StoreStatus, 0, len(s.storeStatuses)) - for _, v := range s.storeStatuses { - statuses = append(statuses, *v) - } - - sort.Slice(statuses, func(i, j int) bool { - return statuses[i].Name < statuses[j].Name - }) - return statuses -} - -// Get returns a list of all active stores. -func (s *StoreSet) Get() []store.Client { - s.storesMtx.RLock() - defer s.storesMtx.RUnlock() - - stores := make([]store.Client, 0, len(s.stores)) - for _, st := range s.stores { - stores = append(stores, st) - } - return stores -} - -// GetRulesClients returns a list of all active rules clients. -func (s *StoreSet) GetRulesClients() []rulespb.RulesClient { - s.storesMtx.RLock() - defer s.storesMtx.RUnlock() - - rules := make([]rulespb.RulesClient, 0, len(s.stores)) - for _, st := range s.stores { - if st.HasRulesAPI() { - rules = append(rules, st.rule) - } - } - return rules -} - -// GetTargetsClients returns a list of all active targets clients. -func (s *StoreSet) GetTargetsClients() []targetspb.TargetsClient { - s.storesMtx.RLock() - defer s.storesMtx.RUnlock() - - targets := make([]targetspb.TargetsClient, 0, len(s.stores)) - for _, st := range s.stores { - if st.HasTargetsAPI() { - targets = append(targets, st.target) - } - } - return targets -} - -// GetMetadataClients returns a list of all active metadata clients. -func (s *StoreSet) GetMetadataClients() []metadatapb.MetadataClient { - s.storesMtx.RLock() - defer s.storesMtx.RUnlock() - - metadataClients := make([]metadatapb.MetadataClient, 0, len(s.stores)) - for _, st := range s.stores { - if st.HasMetadataAPI() { - metadataClients = append(metadataClients, st.metadata) - } - } - return metadataClients -} - -// GetExemplarsStores returns a list of all active exemplars stores. -func (s *StoreSet) GetExemplarsStores() []*exemplarspb.ExemplarStore { - s.storesMtx.RLock() - defer s.storesMtx.RUnlock() - - exemplarStores := make([]*exemplarspb.ExemplarStore, 0, len(s.stores)) - for _, st := range s.stores { - if st.HasExemplarsAPI() { - exemplarStores = append(exemplarStores, &exemplarspb.ExemplarStore{ - ExemplarsClient: st.exemplar, - LabelSets: st.labelSets, - }) - } - } - return exemplarStores -} - -func (s *StoreSet) Close() { - s.storesMtx.Lock() - defer s.storesMtx.Unlock() - - for _, st := range s.stores { - st.Close() - } - s.stores = map[string]*storeRef{} -} - -func (s *StoreSet) cleanUpStoreStatuses(stores map[string]*storeRef) { - s.storesStatusesMtx.Lock() - defer s.storesStatusesMtx.Unlock() - - now := time.Now() - for addr, status := range s.storeStatuses { - if _, ok := stores[addr]; ok { - continue - } - - if now.Sub(status.LastCheck) >= s.unhealthyStoreTimeout { - delete(s.storeStatuses, addr) - } - } -} diff --git a/pkg/query/storeset_test.go b/pkg/query/storeset_test.go deleted file mode 100644 index 38dc2bbe62..0000000000 --- a/pkg/query/storeset_test.go +++ /dev/null @@ -1,1331 +0,0 @@ -// Copyright (c) The Thanos Authors. -// Licensed under the Apache License 2.0. - -package query - -import ( - "context" - "encoding/json" - "fmt" - "math" - "net" - "testing" - "time" - - "github.com/pkg/errors" - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - - "github.com/thanos-io/thanos/pkg/component" - "github.com/thanos-io/thanos/pkg/info/infopb" - "github.com/thanos-io/thanos/pkg/store" - "github.com/thanos-io/thanos/pkg/store/labelpb" - "github.com/thanos-io/thanos/pkg/store/storepb" - "github.com/thanos-io/thanos/pkg/testutil" -) - -var testGRPCOpts = []grpc.DialOption{ - grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(math.MaxInt32)), - grpc.WithInsecure(), -} - -type mockedStore struct { - infoDelay time.Duration - info storepb.InfoResponse -} - -func (s *mockedStore) Info(ctx context.Context, r *storepb.InfoRequest) (*storepb.InfoResponse, error) { - if s.infoDelay > 0 { - time.Sleep(s.infoDelay) - } - return &s.info, nil -} - -func (s *mockedStore) Series(r *storepb.SeriesRequest, srv storepb.Store_SeriesServer) error { - return status.Error(codes.Unimplemented, "not implemented") -} - -func (s *mockedStore) LabelNames(ctx context.Context, r *storepb.LabelNamesRequest) ( - *storepb.LabelNamesResponse, error, -) { - return nil, status.Error(codes.Unimplemented, "not implemented") -} - -func (s *mockedStore) LabelValues(ctx context.Context, r *storepb.LabelValuesRequest) ( - *storepb.LabelValuesResponse, error, -) { - return nil, status.Error(codes.Unimplemented, "not implemented") -} - -type testStoreMeta struct { - extlsetFn func(addr string) []labelpb.ZLabelSet - storeType component.StoreAPI - minTime, maxTime int64 - infoDelay time.Duration -} - -type testStores struct { - srvs map[string]*grpc.Server - orderAddrs []string -} - -func startTestStores(storeMetas []testStoreMeta) (*testStores, error) { - st := &testStores{ - srvs: map[string]*grpc.Server{}, - } - - for _, meta := range storeMetas { - listener, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - // Close so far started servers. - st.Close() - return nil, err - } - - srv := grpc.NewServer() - - storeSrv := &mockedStore{ - info: storepb.InfoResponse{ - LabelSets: meta.extlsetFn(listener.Addr().String()), - MaxTime: meta.maxTime, - MinTime: meta.minTime, - }, - infoDelay: meta.infoDelay, - } - if meta.storeType != nil { - storeSrv.info.StoreType = meta.storeType.ToProto() - } - storepb.RegisterStoreServer(srv, storeSrv) - go func() { - _ = srv.Serve(listener) - }() - - st.srvs[listener.Addr().String()] = srv - st.orderAddrs = append(st.orderAddrs, listener.Addr().String()) - } - - return st, nil -} - -func (s *testStores) StoreAddresses() []string { - var stores []string - stores = append(stores, s.orderAddrs...) - return stores -} - -func (s *testStores) Close() { - for _, srv := range s.srvs { - srv.Stop() - } - s.srvs = nil -} - -func (s *testStores) CloseOne(addr string) { - srv, ok := s.srvs[addr] - if !ok { - return - } - - srv.Stop() - delete(s.srvs, addr) -} - -type mockedInfo struct { - info infopb.InfoResp -} - -func (s *mockedInfo) Info(ctx context.Context, r *infopb.InfoReq) (*infopb.InfoResp, error) { - return &s.info, nil -} - -type testInfoMeta struct { - extlsetFn func(add string) []labelpb.ZLabelSet - storeType component.StoreAPI - store infopb.StoreInfo - rule infopb.RulesInfo - metadata infopb.MetricMetadataInfo - target infopb.TargetsInfo - exemplar infopb.ExemplarsInfo -} - -type testInfoSrvs struct { - srvs map[string]*grpc.Server - orderAddrs []string -} - -func startInfoSrvs(infoMetas []testInfoMeta) (*testInfoSrvs, error) { - info := &testInfoSrvs{ - srvs: map[string]*grpc.Server{}, - } - - for _, meta := range infoMetas { - listener, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - // Close the servers - info.Close() - return nil, err - } - - srv := grpc.NewServer() - - infoSrv := &mockedInfo{ - info: infopb.InfoResp{ - LabelSets: meta.extlsetFn(listener.Addr().String()), - Store: &meta.store, - MetricMetadata: &meta.metadata, - Rules: &meta.rule, - Targets: &meta.target, - Exemplars: &meta.exemplar, - }, - } - - if meta.storeType != nil { - infoSrv.info.ComponentType = meta.storeType.String() - } - infopb.RegisterInfoServer(srv, infoSrv) - go func() { - _ = srv.Serve(listener) - }() - - info.srvs[listener.Addr().String()] = srv - info.orderAddrs = append(info.orderAddrs, listener.Addr().String()) - } - - return info, nil -} - -func (s *testInfoSrvs) Close() { - for _, srv := range s.srvs { - srv.Stop() - } - s.srvs = nil -} - -func (s *testInfoSrvs) CloseOne(addr string) { - srv, ok := s.srvs[addr] - if !ok { - return - } - - srv.Stop() - delete(s.srvs, addr) -} - -func (s *testInfoSrvs) InfoAddresses() []string { - var stores []string - stores = append(stores, s.orderAddrs...) - return stores -} - -func TestStoreSet_Update(t *testing.T) { - stores, err := startTestStores([]testStoreMeta{ - { - storeType: component.Sidecar, - extlsetFn: func(addr string) []labelpb.ZLabelSet { - return []labelpb.ZLabelSet{ - { - Labels: []labelpb.ZLabel{ - {Name: "addr", Value: addr}, - }, - }, - { - Labels: []labelpb.ZLabel{ - {Name: "a", Value: "b"}, - }, - }, - } - }, - }, - { - storeType: component.Sidecar, - extlsetFn: func(addr string) []labelpb.ZLabelSet { - return []labelpb.ZLabelSet{ - { - Labels: []labelpb.ZLabel{ - {Name: "addr", Value: addr}, - }, - }, - { - Labels: []labelpb.ZLabel{ - {Name: "a", Value: "b"}, - }, - }, - } - }, - }, - { - storeType: component.Query, - extlsetFn: func(addr string) []labelpb.ZLabelSet { - return []labelpb.ZLabelSet{ - { - Labels: []labelpb.ZLabel{ - {Name: "a", Value: "broken"}, - }, - }, - } - }, - }, - }) - testutil.Ok(t, err) - defer stores.Close() - - discoveredStoreAddr := stores.StoreAddresses() - - infoSrvs, err := startInfoSrvs([]testInfoMeta{ - { - storeType: component.Sidecar, - extlsetFn: func(addr string) []labelpb.ZLabelSet { - return []labelpb.ZLabelSet{ - { - Labels: []labelpb.ZLabel{ - {Name: "addr", Value: addr}, - }, - }, - { - Labels: []labelpb.ZLabel{ - {Name: "a", Value: "b"}, - }, - }, - } - }, - store: infopb.StoreInfo{ - MinTime: math.MaxInt64, - MaxTime: math.MinInt64, - }, - exemplar: infopb.ExemplarsInfo{ - MinTime: math.MaxInt64, - MaxTime: math.MinInt64, - }, - rule: infopb.RulesInfo{}, - metadata: infopb.MetricMetadataInfo{}, - target: infopb.TargetsInfo{}, - }, - { - storeType: component.Sidecar, - extlsetFn: func(addr string) []labelpb.ZLabelSet { - return []labelpb.ZLabelSet{ - { - Labels: []labelpb.ZLabel{ - {Name: "addr", Value: addr}, - }, - }, - { - Labels: []labelpb.ZLabel{ - {Name: "a", Value: "b"}, - }, - }, - } - }, - store: infopb.StoreInfo{ - MinTime: math.MaxInt64, - MaxTime: math.MinInt64, - }, - exemplar: infopb.ExemplarsInfo{ - MinTime: math.MaxInt64, - MaxTime: math.MinInt64, - }, - rule: infopb.RulesInfo{}, - metadata: infopb.MetricMetadataInfo{}, - target: infopb.TargetsInfo{}, - }, - { - storeType: component.Query, - extlsetFn: func(addr string) []labelpb.ZLabelSet { - return []labelpb.ZLabelSet{ - { - Labels: []labelpb.ZLabel{ - {Name: "addr", Value: "broken"}, - }, - }, - } - }, - store: infopb.StoreInfo{ - MinTime: math.MaxInt64, - MaxTime: math.MinInt64, - }, - exemplar: infopb.ExemplarsInfo{ - MinTime: math.MaxInt64, - MaxTime: math.MinInt64, - }, - rule: infopb.RulesInfo{}, - metadata: infopb.MetricMetadataInfo{}, - target: infopb.TargetsInfo{}, - }, - }) - testutil.Ok(t, err) - defer infoSrvs.Close() - - discoveredInfoAddr := infoSrvs.InfoAddresses() - - // Testing if duplicates can cause weird results. - discoveredStoreAddr = append(discoveredStoreAddr, discoveredStoreAddr[0]) - discoveredInfoAddr = append(discoveredInfoAddr, discoveredInfoAddr[0]) - storeSet := NewStoreSet(nil, nil, - func() (specs []StoreSpec) { - for _, addr := range discoveredStoreAddr { - specs = append(specs, NewGRPCStoreSpec(addr, false)) - } - return specs - }, - func() (specs []RuleSpec) { - return nil - }, - func() (specs []TargetSpec) { - return nil - }, - func() (specs []MetadataSpec) { - return nil - }, - func() (specs []ExemplarSpec) { - return nil - }, - func() (specs []InfoSpec) { - for _, addr := range discoveredInfoAddr { - specs = append(specs, NewGRPCStoreSpec(addr, false)) - } - return specs - }, - testGRPCOpts, time.Minute) - storeSet.gRPCInfoCallTimeout = 2 * time.Second - defer storeSet.Close() - - // Initial update. - storeSet.Update(context.Background()) - - // Start with one not available. - stores.CloseOne(discoveredStoreAddr[2]) - - // Make one address discovered by Info Servers unavailable. - infoSrvs.CloseOne(discoveredInfoAddr[2]) - - // Should not matter how many of these we run. - storeSet.Update(context.Background()) - storeSet.Update(context.Background()) - testutil.Equals(t, 4, len(storeSet.stores)) - testutil.Equals(t, 6, len(storeSet.storeStatuses)) - - for addr, st := range storeSet.stores { - testutil.Equals(t, addr, st.addr) - - lset := st.LabelSets() - testutil.Equals(t, 2, len(lset)) - testutil.Equals(t, "addr", lset[0][0].Name) - testutil.Equals(t, addr, lset[0][0].Value) - testutil.Equals(t, "a", lset[1][0].Name) - testutil.Equals(t, "b", lset[1][0].Value) - } - - // Check stats. - expected := newStoreAPIStats() - expected[component.Sidecar] = map[string]int{ - fmt.Sprintf("{a=\"b\"},{addr=\"%s\"}", discoveredStoreAddr[0]): 1, - fmt.Sprintf("{a=\"b\"},{addr=\"%s\"}", discoveredStoreAddr[1]): 1, - fmt.Sprintf("{a=\"b\"},{addr=\"%s\"}", discoveredInfoAddr[0]): 1, - fmt.Sprintf("{a=\"b\"},{addr=\"%s\"}", discoveredInfoAddr[1]): 1, - } - testutil.Equals(t, expected, storeSet.storesMetric.storeNodes) - - // Remove address from discovered and reset last check, which should ensure cleanup of status on next update. - storeSet.storeStatuses[discoveredStoreAddr[2]].LastCheck = time.Now().Add(-4 * time.Minute) - discoveredStoreAddr = discoveredStoreAddr[:len(discoveredStoreAddr)-2] - storeSet.storeStatuses[discoveredInfoAddr[2]].LastCheck = time.Now().Add(-4 * time.Minute) - discoveredInfoAddr = discoveredInfoAddr[:len(discoveredInfoAddr)-2] - storeSet.Update(context.Background()) - testutil.Equals(t, 4, len(storeSet.storeStatuses)) - - stores.CloseOne(discoveredStoreAddr[0]) - delete(expected[component.Sidecar], fmt.Sprintf("{a=\"b\"},{addr=\"%s\"}", discoveredStoreAddr[0])) - infoSrvs.CloseOne(discoveredInfoAddr[0]) - delete(expected[component.Sidecar], fmt.Sprintf("{a=\"b\"},{addr=\"%s\"}", discoveredInfoAddr[0])) - - // We expect Update to tear down store client for closed store server. - storeSet.Update(context.Background()) - testutil.Equals(t, 2, len(storeSet.stores), "only two service should respond just fine, so we expect one client to be ready.") - testutil.Equals(t, 4, len(storeSet.storeStatuses)) - - addr := discoveredStoreAddr[1] - st, ok := storeSet.stores[addr] - testutil.Assert(t, ok, "addr exist") - testutil.Equals(t, addr, st.addr) - - lset := st.LabelSets() - testutil.Equals(t, 2, len(lset)) - testutil.Equals(t, "addr", lset[0][0].Name) - testutil.Equals(t, addr, lset[0][0].Value) - testutil.Equals(t, "a", lset[1][0].Name) - testutil.Equals(t, "b", lset[1][0].Value) - testutil.Equals(t, expected, storeSet.storesMetric.storeNodes) - - // New big batch of storeAPIs. - stores2, err := startTestStores([]testStoreMeta{ - { - storeType: component.Query, - extlsetFn: func(addr string) []labelpb.ZLabelSet { - return []labelpb.ZLabelSet{ - { - Labels: []labelpb.ZLabel{ - {Name: "l1", Value: "v2"}, - {Name: "l2", Value: "v3"}, - }, - }, - { - Labels: []labelpb.ZLabel{ - {Name: "l3", Value: "v4"}, - }, - }, - } - }, - }, - { - // Duplicated Querier, in previous versions it would be deduplicated. Now it should be not. - storeType: component.Query, - extlsetFn: func(addr string) []labelpb.ZLabelSet { - return []labelpb.ZLabelSet{ - { - Labels: []labelpb.ZLabel{ - {Name: "l1", Value: "v2"}, - {Name: "l2", Value: "v3"}, - }, - }, - { - Labels: []labelpb.ZLabel{ - {Name: "l3", Value: "v4"}, - }, - }, - } - }, - }, - { - storeType: component.Sidecar, - extlsetFn: func(addr string) []labelpb.ZLabelSet { - return []labelpb.ZLabelSet{ - { - Labels: []labelpb.ZLabel{ - {Name: "l1", Value: "v2"}, - {Name: "l2", Value: "v3"}, - }, - }, - } - }, - }, - { - // Duplicated Sidecar, in previous versions it would be deduplicated. Now it should be not. - storeType: component.Sidecar, - extlsetFn: func(addr string) []labelpb.ZLabelSet { - return []labelpb.ZLabelSet{ - { - Labels: []labelpb.ZLabel{ - {Name: "l1", Value: "v2"}, - {Name: "l2", Value: "v3"}, - }, - }, - } - }, - }, - { - // Querier that duplicates with sidecar, in previous versions it would be deduplicated. Now it should be not. - storeType: component.Query, - extlsetFn: func(addr string) []labelpb.ZLabelSet { - return []labelpb.ZLabelSet{ - { - Labels: []labelpb.ZLabel{ - {Name: "l1", Value: "v2"}, - {Name: "l2", Value: "v3"}, - }, - }, - } - }, - }, - { - // Ruler that duplicates with sidecar, in previous versions it would be deduplicated. Now it should be not. - // Warning should be produced. - storeType: component.Rule, - extlsetFn: func(addr string) []labelpb.ZLabelSet { - return []labelpb.ZLabelSet{ - { - Labels: []labelpb.ZLabel{ - {Name: "l1", Value: "v2"}, - {Name: "l2", Value: "v3"}, - }, - }, - } - }, - }, - { - // Duplicated Rule, in previous versions it would be deduplicated. Now it should be not. Warning should be produced. - storeType: component.Rule, - extlsetFn: func(addr string) []labelpb.ZLabelSet { - return []labelpb.ZLabelSet{ - { - Labels: []labelpb.ZLabel{ - {Name: "l1", Value: "v2"}, - {Name: "l2", Value: "v3"}, - }, - }, - } - }, - }, - { - // No storeType. - extlsetFn: func(addr string) []labelpb.ZLabelSet { - return []labelpb.ZLabelSet{ - { - Labels: []labelpb.ZLabel{ - {Name: "l1", Value: "no-store-type"}, - {Name: "l2", Value: "v3"}, - }, - }, - } - }, - }, - // Two pre v0.8.0 store gateway nodes, they don't have ext labels set. - { - storeType: component.Store, - extlsetFn: func(addr string) []labelpb.ZLabelSet { - return []labelpb.ZLabelSet{} - }, - }, - { - storeType: component.Store, - extlsetFn: func(addr string) []labelpb.ZLabelSet { - return []labelpb.ZLabelSet{} - }, - }, - // Regression tests against https://github.com/thanos-io/thanos/issues/1632: From v0.8.0 stores advertise labels. - // If the object storage handled by store gateway has only one sidecar we used to hitting issue. - { - storeType: component.Store, - extlsetFn: func(addr string) []labelpb.ZLabelSet { - return []labelpb.ZLabelSet{ - { - Labels: []labelpb.ZLabel{ - {Name: "l1", Value: "v2"}, - {Name: "l2", Value: "v3"}, - }, - }, - { - Labels: []labelpb.ZLabel{ - {Name: "l3", Value: "v4"}, - }, - }, - } - }, - }, - // Stores v0.8.1 has compatibility labels. Check if they are correctly removed. - { - storeType: component.Store, - extlsetFn: func(addr string) []labelpb.ZLabelSet { - return []labelpb.ZLabelSet{ - { - Labels: []labelpb.ZLabel{ - {Name: "l1", Value: "v2"}, - {Name: "l2", Value: "v3"}, - }, - }, - { - Labels: []labelpb.ZLabel{ - {Name: "l3", Value: "v4"}, - }, - }, - { - Labels: []labelpb.ZLabel{ - {Name: store.CompatibilityTypeLabelName, Value: "store"}, - }, - }, - } - }, - }, - // Duplicated store, in previous versions it would be deduplicated. Now it should be not. - { - storeType: component.Store, - extlsetFn: func(addr string) []labelpb.ZLabelSet { - return []labelpb.ZLabelSet{ - { - Labels: []labelpb.ZLabel{ - {Name: "l1", Value: "v2"}, - {Name: "l2", Value: "v3"}, - }, - }, - { - Labels: []labelpb.ZLabel{ - {Name: "l3", Value: "v4"}, - }, - }, - { - Labels: []labelpb.ZLabel{ - {Name: store.CompatibilityTypeLabelName, Value: "store"}, - }, - }, - } - }, - }, - }) - testutil.Ok(t, err) - defer stores2.Close() - - discoveredStoreAddr = append(discoveredStoreAddr, stores2.StoreAddresses()...) - - // New stores should be loaded. - storeSet.Update(context.Background()) - testutil.Equals(t, 2+len(stores2.srvs), len(storeSet.stores)) - - // Check stats. - expected = newStoreAPIStats() - expected[component.UnknownStoreAPI] = map[string]int{ - "{l1=\"no-store-type\", l2=\"v3\"}": 1, - } - expected[component.Query] = map[string]int{ - "{l1=\"v2\", l2=\"v3\"}": 1, - "{l1=\"v2\", l2=\"v3\"},{l3=\"v4\"}": 2, - } - expected[component.Rule] = map[string]int{ - "{l1=\"v2\", l2=\"v3\"}": 2, - } - expected[component.Sidecar] = map[string]int{ - fmt.Sprintf("{a=\"b\"},{addr=\"%s\"}", discoveredStoreAddr[1]): 1, - fmt.Sprintf("{a=\"b\"},{addr=\"%s\"}", discoveredInfoAddr[1]): 1, - "{l1=\"v2\", l2=\"v3\"}": 2, - } - expected[component.Store] = map[string]int{ - "": 2, - "{l1=\"v2\", l2=\"v3\"},{l3=\"v4\"}": 3, - } - testutil.Equals(t, expected, storeSet.storesMetric.storeNodes) - - // Check statuses. - testutil.Equals(t, 4+len(stores2.srvs), len(storeSet.storeStatuses)) -} - -func TestStoreSet_Update_NoneAvailable(t *testing.T) { - st, err := startTestStores([]testStoreMeta{ - { - extlsetFn: func(addr string) []labelpb.ZLabelSet { - return []labelpb.ZLabelSet{ - { - Labels: []labelpb.ZLabel{ - { - Name: "addr", - Value: addr, - }, - }, - }, - } - }, - storeType: component.Sidecar, - }, - { - extlsetFn: func(addr string) []labelpb.ZLabelSet { - return []labelpb.ZLabelSet{ - { - Labels: []labelpb.ZLabel{ - { - Name: "addr", - Value: addr, - }, - }, - }, - } - }, - storeType: component.Sidecar, - }, - }) - testutil.Ok(t, err) - defer st.Close() - - infoSrvs, err := startInfoSrvs([]testInfoMeta{ - { - storeType: component.Sidecar, - extlsetFn: func(addr string) []labelpb.ZLabelSet { - return []labelpb.ZLabelSet{ - { - Labels: []labelpb.ZLabel{ - {Name: "addr", Value: addr}, - }, - }, - } - }, - }, - { - storeType: component.Sidecar, - extlsetFn: func(addr string) []labelpb.ZLabelSet { - return []labelpb.ZLabelSet{ - { - Labels: []labelpb.ZLabel{ - {Name: "addr", Value: addr}, - }, - }, - } - }, - }, - }) - testutil.Ok(t, err) - defer infoSrvs.Close() - - initialStoreAddr := st.StoreAddresses() - st.CloseOne(initialStoreAddr[0]) - st.CloseOne(initialStoreAddr[1]) - - initialInfoAddr := infoSrvs.InfoAddresses() - infoSrvs.CloseOne(initialInfoAddr[0]) - infoSrvs.CloseOne(initialInfoAddr[1]) - - storeSet := NewStoreSet(nil, nil, - func() (specs []StoreSpec) { - for _, addr := range initialStoreAddr { - specs = append(specs, NewGRPCStoreSpec(addr, false)) - } - return specs - }, - func() (specs []RuleSpec) { return nil }, - func() (specs []TargetSpec) { return nil }, - func() (specs []MetadataSpec) { return nil }, - func() (specs []ExemplarSpec) { return nil }, - func() (specs []InfoSpec) { - for _, addr := range initialInfoAddr { - specs = append(specs, NewGRPCStoreSpec(addr, false)) - } - return specs - }, - testGRPCOpts, time.Minute) - storeSet.gRPCInfoCallTimeout = 2 * time.Second - - // Should not matter how many of these we run. - storeSet.Update(context.Background()) - storeSet.Update(context.Background()) - testutil.Equals(t, 0, len(storeSet.stores), "none of services should respond just fine, so we expect no client to be ready.") - - // Leak test will ensure that we don't keep client connection around. - - expected := newStoreAPIStats() - testutil.Equals(t, expected, storeSet.storesMetric.storeNodes) -} - -// TestQuerierStrict tests what happens when the strict mode is enabled/disabled. -func TestQuerierStrict(t *testing.T) { - st, err := startTestStores([]testStoreMeta{ - { - minTime: 12345, - maxTime: 54321, - extlsetFn: func(addr string) []labelpb.ZLabelSet { - return []labelpb.ZLabelSet{ - { - Labels: []labelpb.ZLabel{ - { - Name: "addr", - Value: addr, - }, - }, - }, - } - }, - storeType: component.Sidecar, - }, - { - minTime: 66666, - maxTime: 77777, - extlsetFn: func(addr string) []labelpb.ZLabelSet { - return []labelpb.ZLabelSet{ - { - Labels: []labelpb.ZLabel{ - { - Name: "addr", - Value: addr, - }, - }, - }, - } - }, - storeType: component.Sidecar, - }, - // Slow store. - { - minTime: 65644, - maxTime: 77777, - extlsetFn: func(addr string) []labelpb.ZLabelSet { - return []labelpb.ZLabelSet{ - { - Labels: []labelpb.ZLabel{ - { - Name: "addr", - Value: addr, - }, - }, - }, - } - }, - storeType: component.Sidecar, - infoDelay: 2 * time.Second, - }, - }) - - testutil.Ok(t, err) - defer st.Close() - - staticStoreAddr := st.StoreAddresses()[0] - slowStaticStoreAddr := st.StoreAddresses()[2] - storeSet := NewStoreSet(nil, nil, func() (specs []StoreSpec) { - return []StoreSpec{ - NewGRPCStoreSpec(st.StoreAddresses()[0], true), - NewGRPCStoreSpec(st.StoreAddresses()[1], false), - NewGRPCStoreSpec(st.StoreAddresses()[2], true), - } - }, func() []RuleSpec { - return nil - }, func() []TargetSpec { - return nil - }, func() (specs []MetadataSpec) { - return nil - }, func() []ExemplarSpec { - return nil - }, func() []InfoSpec { - return nil - }, testGRPCOpts, time.Minute) - defer storeSet.Close() - storeSet.gRPCInfoCallTimeout = 1 * time.Second - - // Initial update. - storeSet.Update(context.Background()) - testutil.Equals(t, 3, len(storeSet.stores), "three clients must be available for running store nodes") - - // The store has not responded to the info call and is assumed to cover everything. - curMin, curMax := storeSet.stores[slowStaticStoreAddr].minTime, storeSet.stores[slowStaticStoreAddr].maxTime - testutil.Assert(t, storeSet.stores[slowStaticStoreAddr].cc.GetState().String() != "SHUTDOWN", "slow store's connection should not be closed") - testutil.Equals(t, int64(0), curMin) - testutil.Equals(t, int64(math.MaxInt64), curMax) - - // The store is statically defined + strict mode is enabled - // so its client + information must be retained. - curMin, curMax = storeSet.stores[staticStoreAddr].minTime, storeSet.stores[staticStoreAddr].maxTime - testutil.Equals(t, int64(12345), curMin, "got incorrect minimum time") - testutil.Equals(t, int64(54321), curMax, "got incorrect minimum time") - - // Successfully retrieve the information and observe minTime/maxTime updating. - storeSet.gRPCInfoCallTimeout = 3 * time.Second - storeSet.Update(context.Background()) - updatedCurMin, updatedCurMax := storeSet.stores[slowStaticStoreAddr].minTime, storeSet.stores[slowStaticStoreAddr].maxTime - testutil.Equals(t, int64(65644), updatedCurMin) - testutil.Equals(t, int64(77777), updatedCurMax) - storeSet.gRPCInfoCallTimeout = 1 * time.Second - - // Turn off the stores. - st.Close() - - // Update again many times. Should not matter WRT the static one. - storeSet.Update(context.Background()) - storeSet.Update(context.Background()) - storeSet.Update(context.Background()) - - // Check that the information is the same. - testutil.Equals(t, 2, len(storeSet.stores), "two static clients must remain available") - testutil.Equals(t, curMin, storeSet.stores[staticStoreAddr].minTime, "minimum time reported by the store node is different") - testutil.Equals(t, curMax, storeSet.stores[staticStoreAddr].maxTime, "minimum time reported by the store node is different") - testutil.NotOk(t, storeSet.storeStatuses[staticStoreAddr].LastError.originalErr) - - testutil.Equals(t, updatedCurMin, storeSet.stores[slowStaticStoreAddr].minTime, "minimum time reported by the store node is different") - testutil.Equals(t, updatedCurMax, storeSet.stores[slowStaticStoreAddr].maxTime, "minimum time reported by the store node is different") -} - -func TestStoreSet_Update_Rules(t *testing.T) { - stores, err := startTestStores([]testStoreMeta{ - { - extlsetFn: func(addr string) []labelpb.ZLabelSet { - return []labelpb.ZLabelSet{} - }, - storeType: component.Sidecar, - }, - { - extlsetFn: func(addr string) []labelpb.ZLabelSet { - return []labelpb.ZLabelSet{} - }, - storeType: component.Rule, - }, - }) - testutil.Ok(t, err) - defer stores.Close() - - for _, tc := range []struct { - name string - storeSpecs func() []StoreSpec - ruleSpecs func() []RuleSpec - exemplarSpecs func() []ExemplarSpec - expectedStores int - expectedRules int - }{ - { - name: "stores, no rules", - storeSpecs: func() []StoreSpec { - return []StoreSpec{ - NewGRPCStoreSpec(stores.orderAddrs[0], false), - NewGRPCStoreSpec(stores.orderAddrs[1], false), - } - }, - expectedStores: 2, - expectedRules: 0, - }, - { - name: "rules, no stores", - ruleSpecs: func() []RuleSpec { - return []RuleSpec{ - NewGRPCStoreSpec(stores.orderAddrs[0], false), - } - }, - expectedStores: 0, - expectedRules: 0, - }, - { - name: "one store, different rule", - storeSpecs: func() []StoreSpec { - return []StoreSpec{ - NewGRPCStoreSpec(stores.orderAddrs[0], false), - } - }, - ruleSpecs: func() []RuleSpec { - return []RuleSpec{ - NewGRPCStoreSpec(stores.orderAddrs[1], false), - } - }, - expectedStores: 1, - expectedRules: 0, - }, - { - name: "two stores, one rule", - storeSpecs: func() []StoreSpec { - return []StoreSpec{ - NewGRPCStoreSpec(stores.orderAddrs[0], false), - NewGRPCStoreSpec(stores.orderAddrs[1], false), - } - }, - ruleSpecs: func() []RuleSpec { - return []RuleSpec{ - NewGRPCStoreSpec(stores.orderAddrs[0], false), - } - }, - expectedStores: 2, - expectedRules: 1, - }, - { - name: "two stores, two rules", - storeSpecs: func() []StoreSpec { - return []StoreSpec{ - NewGRPCStoreSpec(stores.orderAddrs[0], false), - NewGRPCStoreSpec(stores.orderAddrs[1], false), - } - }, - ruleSpecs: func() []RuleSpec { - return []RuleSpec{ - NewGRPCStoreSpec(stores.orderAddrs[0], false), - NewGRPCStoreSpec(stores.orderAddrs[1], false), - } - }, - exemplarSpecs: func() []ExemplarSpec { - return []ExemplarSpec{ - NewGRPCStoreSpec(stores.orderAddrs[0], false), - NewGRPCStoreSpec(stores.orderAddrs[1], false), - } - }, - expectedStores: 2, - expectedRules: 2, - }, - } { - storeSet := NewStoreSet(nil, nil, - tc.storeSpecs, - tc.ruleSpecs, - func() []TargetSpec { return nil }, - func() []MetadataSpec { return nil }, - tc.exemplarSpecs, - func() []InfoSpec { return nil }, - testGRPCOpts, time.Minute) - - t.Run(tc.name, func(t *testing.T) { - defer storeSet.Close() - storeSet.Update(context.Background()) - testutil.Equals(t, tc.expectedStores, len(storeSet.stores)) - - gotRules := 0 - for _, ref := range storeSet.stores { - if ref.HasRulesAPI() { - gotRules += 1 - } - } - - testutil.Equals(t, tc.expectedRules, gotRules) - }) - } -} - -func TestStoreSet_Rules_Discovery(t *testing.T) { - stores, err := startTestStores([]testStoreMeta{ - { - extlsetFn: func(addr string) []labelpb.ZLabelSet { - return []labelpb.ZLabelSet{} - }, - storeType: component.Sidecar, - }, - { - extlsetFn: func(addr string) []labelpb.ZLabelSet { - return []labelpb.ZLabelSet{} - }, - storeType: component.Rule, - }, - }) - testutil.Ok(t, err) - defer stores.Close() - - type discoveryState struct { - name string - storeSpecs func() []StoreSpec - ruleSpecs func() []RuleSpec - expectedStores int - expectedRules int - } - - for _, tc := range []struct { - states []discoveryState - name string - }{ - { - name: "StoreAPI and RulesAPI concurrent discovery", - states: []discoveryState{ - { - name: "no stores", - storeSpecs: nil, - ruleSpecs: nil, - expectedRules: 0, - expectedStores: 0, - }, - { - name: "RulesAPI discovered", - storeSpecs: func() []StoreSpec { - return []StoreSpec{ - NewGRPCStoreSpec(stores.orderAddrs[0], false), - } - }, - ruleSpecs: func() []RuleSpec { - return []RuleSpec{ - NewGRPCStoreSpec(stores.orderAddrs[0], false), - } - }, - expectedRules: 1, - expectedStores: 1, - }, - }, - }, - - { - name: "StoreAPI discovery first, eventually discovered RulesAPI", - states: []discoveryState{ - { - name: "no stores", - storeSpecs: nil, - ruleSpecs: nil, - expectedRules: 0, - expectedStores: 0, - }, - { - name: "StoreAPI discovered, no RulesAPI discovered", - storeSpecs: func() []StoreSpec { - return []StoreSpec{ - NewGRPCStoreSpec(stores.orderAddrs[0], false), - } - }, - expectedStores: 1, - expectedRules: 0, - }, - { - name: "RulesAPI discovered", - storeSpecs: func() []StoreSpec { - return []StoreSpec{ - NewGRPCStoreSpec(stores.orderAddrs[0], false), - } - }, - ruleSpecs: func() []RuleSpec { - return []RuleSpec{ - NewGRPCStoreSpec(stores.orderAddrs[0], false), - } - }, - expectedStores: 1, - expectedRules: 1, - }, - }, - }, - - { - name: "RulesAPI discovery first, eventually discovered StoreAPI", - states: []discoveryState{ - { - name: "no stores", - storeSpecs: nil, - ruleSpecs: nil, - expectedRules: 0, - expectedStores: 0, - }, - { - name: "RulesAPI discovered, no StoreAPI discovered", - storeSpecs: nil, - ruleSpecs: func() []RuleSpec { - return []RuleSpec{ - NewGRPCStoreSpec(stores.orderAddrs[0], false), - } - }, - expectedStores: 0, - expectedRules: 0, - }, - { - name: "StoreAPI discovered", - storeSpecs: func() []StoreSpec { - return []StoreSpec{ - NewGRPCStoreSpec(stores.orderAddrs[0], false), - } - }, - ruleSpecs: func() []RuleSpec { - return []RuleSpec{ - NewGRPCStoreSpec(stores.orderAddrs[0], false), - } - }, - expectedStores: 1, - expectedRules: 1, - }, - }, - }, - } { - t.Run(tc.name, func(t *testing.T) { - currentState := 0 - - storeSet := NewStoreSet(nil, nil, - func() []StoreSpec { - if tc.states[currentState].storeSpecs == nil { - return nil - } - - return tc.states[currentState].storeSpecs() - }, - func() []RuleSpec { - if tc.states[currentState].ruleSpecs == nil { - return nil - } - - return tc.states[currentState].ruleSpecs() - }, - func() []TargetSpec { return nil }, - func() []MetadataSpec { - return nil - }, - func() []ExemplarSpec { return nil }, - func() []InfoSpec { - return nil - }, - testGRPCOpts, time.Minute) - - defer storeSet.Close() - - for { - storeSet.Update(context.Background()) - testutil.Equals( - t, - tc.states[currentState].expectedStores, - len(storeSet.stores), - "unexepected discovered stores in state %q", - tc.states[currentState].name, - ) - - gotRules := 0 - for _, ref := range storeSet.stores { - if ref.HasRulesAPI() { - gotRules += 1 - } - } - testutil.Equals( - t, - tc.states[currentState].expectedRules, - gotRules, - "unexpected discovered rules in state %q", - tc.states[currentState].name, - ) - - currentState = currentState + 1 - if len(tc.states) == currentState { - break - } - } - }) - } -} - -type errThatMarshalsToEmptyDict struct { - msg string -} - -// MarshalJSON marshals the error and returns and empty dict, not the error string. -func (e *errThatMarshalsToEmptyDict) MarshalJSON() ([]byte, error) { - return json.Marshal(map[string]string{}) -} - -// Error returns the original, underlying string. -func (e *errThatMarshalsToEmptyDict) Error() string { - return e.msg -} - -// Test highlights that without wrapping the error, it is marshaled to empty dict {}, not its message. -func TestStringError(t *testing.T) { - dictErr := &errThatMarshalsToEmptyDict{msg: "Error message"} - stringErr := &stringError{originalErr: dictErr} - - storestatusMock := map[string]error{} - storestatusMock["dictErr"] = dictErr - storestatusMock["stringErr"] = stringErr - - b, err := json.Marshal(storestatusMock) - - testutil.Ok(t, err) - testutil.Equals(t, []byte(`{"dictErr":{},"stringErr":"Error message"}`), b, "expected to get proper results") -} - -// Errors that usually marshal to empty dict should return the original error string. -func TestUpdateStoreStateLastError(t *testing.T) { - tcs := []struct { - InputError error - ExpectedLastErr string - }{ - {errors.New("normal_err"), `"normal_err"`}, - {nil, `null`}, - {&errThatMarshalsToEmptyDict{"the error message"}, `"the error message"`}, - } - - for _, tc := range tcs { - mockStoreSet := &StoreSet{ - storeStatuses: map[string]*StoreStatus{}, - } - mockStoreRef := &storeRef{ - addr: "mockedStore", - } - - mockStoreSet.updateStoreStatus(mockStoreRef, tc.InputError) - - b, err := json.Marshal(mockStoreSet.storeStatuses["mockedStore"].LastError) - testutil.Ok(t, err) - testutil.Equals(t, tc.ExpectedLastErr, string(b)) - } -} - -func TestUpdateStoreStateForgetsPreviousErrors(t *testing.T) { - mockStoreSet := &StoreSet{ - storeStatuses: map[string]*StoreStatus{}, - } - mockStoreRef := &storeRef{ - addr: "mockedStore", - } - - mockStoreSet.updateStoreStatus(mockStoreRef, errors.New("test err")) - - b, err := json.Marshal(mockStoreSet.storeStatuses["mockedStore"].LastError) - testutil.Ok(t, err) - testutil.Equals(t, `"test err"`, string(b)) - - // updating status without and error should clear the previous one. - mockStoreSet.updateStoreStatus(mockStoreRef, nil) - - b, err = json.Marshal(mockStoreSet.storeStatuses["mockedStore"].LastError) - testutil.Ok(t, err) - testutil.Equals(t, `null`, string(b)) -} diff --git a/test/e2e/info_api_test.go b/test/e2e/info_api_test.go index 85d4f8146d..af4b6cf1f5 100644 --- a/test/e2e/info_api_test.go +++ b/test/e2e/info_api_test.go @@ -131,7 +131,10 @@ func TestInfo(t *testing.T) { err = runutil.Retry(time.Second, ctx.Done(), func() error { - resp, err := http.Get(url) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + testutil.Ok(t, err) + + resp, err := http.DefaultClient.Do(req) testutil.Ok(t, err) body, err := ioutil.ReadAll(resp.Body) @@ -141,9 +144,8 @@ func TestInfo(t *testing.T) { Data map[string][]query.StoreStatus `json:"data"` } - if err = json.Unmarshal(body, &res); err != nil { - t.Fatalf("Error unmarshaling JSON body: %s", err) - } + err = json.Unmarshal(body, &res) + testutil.Ok(t, err) if err = assertStoreStatus(t, "sidecar", res.Data, expected); err != nil { return err @@ -162,7 +164,7 @@ func assertStoreStatus(t *testing.T, component string, res map[string][]query.St t.Helper() if len(res[component]) != len(expected[component]) { - return fmt.Errorf("Expected %d %s, got: %d", len(expected[component]), component, len(res[component])) + return fmt.Errorf("expected %d %s, got: %d", len(expected[component]), component, len(res[component])) } for i, v := range res[component] { From daff1e32d5eebd4812086090aed0f0caebca5d9a Mon Sep 17 00:00:00 2001 From: Hitanshu Mehta Date: Fri, 24 Sep 2021 09:25:53 +0530 Subject: [PATCH 08/16] resolve merge conflicts Signed-off-by: Hitanshu Mehta --- cmd/thanos/query.go | 26 +++++++++++++------------- cmd/thanos/sidecar.go | 6 +++--- pkg/api/query/v1.go | 6 ------ pkg/store/prometheus.go | 18 ++++++++++++++++++ test/e2e/e2ethanos/services.go | 5 +++++ test/e2e/info_api_test.go | 25 ++++++++++++++----------- 6 files changed, 53 insertions(+), 33 deletions(-) diff --git a/cmd/thanos/query.go b/cmd/thanos/query.go index f0c3e9835d..4777f30c14 100644 --- a/cmd/thanos/query.go +++ b/cmd/thanos/query.go @@ -333,7 +333,7 @@ func runQuery( queryReplicaLabels []string, selectorLset labels.Labels, flagsMap map[string]string, - endpoints []string, + endpointAddrs []string, storeAddrs []string, ruleAddrs []string, targetAddrs []string, @@ -381,9 +381,9 @@ func runQuery( } } - dnsInfoProvider := dns.NewProvider( + dnsEndpointProvider := dns.NewProvider( logger, - extprom.WrapRegistererWithPrefix("thanos_query_info_apis_", reg), + extprom.WrapRegistererWithPrefix("thanos_query_endpoints_", reg), dns.ResolverType(dnsSDResolver), ) @@ -421,7 +421,14 @@ func runQuery( specs = append(specs, query.NewGRPCEndpointSpec(addr, true)) } - for _, dnsProvider := range []*dns.Provider{dnsStoreProvider, dnsRuleProvider, dnsExemplarProvider, dnsMetadataProvider, dnsTargetProvider} { + for _, dnsProvider := range []*dns.Provider{ + dnsStoreProvider, + dnsRuleProvider, + dnsExemplarProvider, + dnsMetadataProvider, + dnsTargetProvider, + dnsEndpointProvider, + } { var tmpSpecs []query.EndpointSpec for _, addr := range dnsProvider.Addresses() { @@ -433,13 +440,6 @@ func runQuery( return specs }, - func() (specs []query.InfoSpec) { - for _, addr := range dnsInfoProvider.Addresses() { - specs = append(specs, query.NewGRPCStoreSpec(addr, false)) - } - - return specs - }, dialOpts, unhealthyStoreTimeout, ) @@ -545,8 +545,8 @@ func runQuery( if err := dnsExemplarProvider.Resolve(resolveCtx, exemplarAddrs); err != nil { level.Error(logger).Log("msg", "failed to resolve addresses for exemplarsAPI", "err", err) } - if err := dnsInfoProvider.Resolve(resolveCtx, endpoints); err != nil { - level.Error(logger).Log("msg", "failed to resolve addresses passed using endpoint flag", "err", err) + if err := dnsEndpointProvider.Resolve(resolveCtx, endpointAddrs); err != nil { + level.Error(logger).Log("msg", "failed to resolve addresses passed using endpoint flag", "err", err) } return nil diff --git a/cmd/thanos/sidecar.go b/cmd/thanos/sidecar.go index 08b2303e01..537ed2acbf 100644 --- a/cmd/thanos/sidecar.go +++ b/cmd/thanos/sidecar.go @@ -23,7 +23,6 @@ import ( "github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/common/model" "github.com/prometheus/prometheus/pkg/labels" - v1 "github.com/thanos-io/thanos/pkg/api/query" "github.com/thanos-io/thanos/pkg/block/metadata" "github.com/thanos-io/thanos/pkg/component" "github.com/thanos-io/thanos/pkg/exemplars" @@ -39,6 +38,7 @@ import ( "github.com/thanos-io/thanos/pkg/objstore/client" "github.com/thanos-io/thanos/pkg/prober" "github.com/thanos-io/thanos/pkg/promclient" + "github.com/thanos-io/thanos/pkg/query" "github.com/thanos-io/thanos/pkg/reloader" "github.com/thanos-io/thanos/pkg/rules" "github.com/thanos-io/thanos/pkg/runutil" @@ -263,8 +263,8 @@ func runSidecar( // Currently Exemplars API does not expose metadata such as min/max time, // so we are using default minimum and maximum possible values as min/max time. return &infopb.ExemplarsInfo{ - MinTime: v1.MinTime.Unix(), - MaxTime: v1.MaxTime.Unix(), + MinTime: query.MinTime, + MaxTime: query.MaxTime, } }, func() *infopb.RulesInfo { diff --git a/pkg/api/query/v1.go b/pkg/api/query/v1.go index 5532013962..fd9ee99b68 100644 --- a/pkg/api/query/v1.go +++ b/pkg/api/query/v1.go @@ -72,12 +72,6 @@ const ( Stats = "stats" ) -var ( - // Default minimum and maximum time values used by Prometheus when they are not passed as query parameter. - MinTime = time.Unix(math.MinInt64/1000+62135596801, 0).UTC() - MaxTime = time.Unix(math.MaxInt64/1000-62135596801, 999999999).UTC() -) - // QueryAPI is an API used by Thanos Querier. type QueryAPI struct { baseAPI *api.BaseAPI diff --git a/pkg/store/prometheus.go b/pkg/store/prometheus.go index be55694c44..32b50dbd19 100644 --- a/pkg/store/prometheus.go +++ b/pkg/store/prometheus.go @@ -607,3 +607,21 @@ func (p *PrometheusStore) LabelValues(ctx context.Context, r *storepb.LabelValue sort.Strings(vals) return &storepb.LabelValuesResponse{Values: vals}, nil } + +func (p *PrometheusStore) LabelSet() []labelpb.ZLabelSet { + lset := p.externalLabelsFn() + + labels := make([]labelpb.ZLabel, 0, len(lset)) + labels = append(labels, labelpb.ZLabelsFromPromLabels(lset)...) + + // Until we deprecate the single labels in the reply, we just duplicate + // them here for migration/compatibility purposes. + labelset := []labelpb.ZLabelSet{} + if len(labels) > 0 { + labelset = append(labelset, labelpb.ZLabelSet{ + Labels: labels, + }) + } + + return labelset +} diff --git a/test/e2e/e2ethanos/services.go b/test/e2e/e2ethanos/services.go index 64836f7a2b..ff7cac7632 100644 --- a/test/e2e/e2ethanos/services.go +++ b/test/e2e/e2ethanos/services.go @@ -199,6 +199,11 @@ func (q *QuerierBuilder) WithMetadataAddresses(metadataAddresses ...string) *Que return q } +func (q *QuerierBuilder) WithEndpoints(endpoints ...string) *QuerierBuilder { + q.endpoints = endpoints + return q +} + func (q *QuerierBuilder) WithRoutePrefix(routePrefix string) *QuerierBuilder { q.routePrefix = routePrefix return q diff --git a/test/e2e/info_api_test.go b/test/e2e/info_api_test.go index af4b6cf1f5..eb84d839f0 100644 --- a/test/e2e/info_api_test.go +++ b/test/e2e/info_api_test.go @@ -58,23 +58,23 @@ func TestInfo(t *testing.T) { // '--endpoint' flag works properly works together with other flags ('--target', '--metadata' etc.). // Register 2 sidecars and 1 storeGW using '--endpoint'. // Register `sidecar3` twice to verify it is deduplicated. - q, err := e2ethanos.NewQuerierBuilder(s.SharedDir(), "1", []string{sidecar1.GRPCNetworkEndpoint()}). - WithTargetAddresses([]string{sidecar1.GRPCNetworkEndpoint()}). - WithMetadataAddresses([]string{sidecar1.GRPCNetworkEndpoint()}). - WithExemplarAddresses([]string{sidecar1.GRPCNetworkEndpoint()}). - WithRuleAddresses([]string{sidecar1.GRPCNetworkEndpoint()}). - WithEndpoints([]string{ + q, err := e2ethanos.NewQuerierBuilder(s.SharedDir(), "1", sidecar1.GRPCNetworkEndpoint()). + WithTargetAddresses(sidecar1.GRPCNetworkEndpoint()). + WithMetadataAddresses(sidecar1.GRPCNetworkEndpoint()). + WithExemplarAddresses(sidecar1.GRPCNetworkEndpoint()). + WithRuleAddresses(sidecar1.GRPCNetworkEndpoint()). + WithEndpoints( sidecar1.GRPCNetworkEndpoint(), sidecar2.GRPCNetworkEndpoint(), sidecar3.GRPCNetworkEndpoint(), sidecar3.GRPCNetworkEndpoint(), str.GRPCNetworkEndpoint(), - }). + ). Build() testutil.Ok(t, err) testutil.Ok(t, s.StartAndWaitReady(q)) - expected := map[string][]query.StoreStatus{ + expected := map[string][]query.EndpointStatus{ "sidecar": { { Name: "e2e_test_info-sidecar-alone1:9091", @@ -126,7 +126,7 @@ func TestInfo(t *testing.T) { url := "http://" + path.Join(q.HTTPEndpoint(), "/api/v1/stores") - ctx, cancel := context.WithTimeout(context.Background(), time.Second*15) + ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) defer cancel() err = runutil.Retry(time.Second, ctx.Done(), func() error { @@ -141,12 +141,15 @@ func TestInfo(t *testing.T) { testutil.Ok(t, err) var res struct { - Data map[string][]query.StoreStatus `json:"data"` + Data map[string][]query.EndpointStatus `json:"data"` } err = json.Unmarshal(body, &res) testutil.Ok(t, err) + fmt.Println(res) + fmt.Println() + if err = assertStoreStatus(t, "sidecar", res.Data, expected); err != nil { return err } @@ -160,7 +163,7 @@ func TestInfo(t *testing.T) { testutil.Ok(t, err) } -func assertStoreStatus(t *testing.T, component string, res map[string][]query.StoreStatus, expected map[string][]query.StoreStatus) error { +func assertStoreStatus(t *testing.T, component string, res map[string][]query.EndpointStatus, expected map[string][]query.EndpointStatus) error { t.Helper() if len(res[component]) != len(expected[component]) { From 829af656bf4d715e9627dbafca00ef4456661607 Mon Sep 17 00:00:00 2001 From: Hitanshu Mehta Date: Fri, 24 Sep 2021 10:23:27 +0530 Subject: [PATCH 09/16] add deprecation warning in description of old flags Signed-off-by: Hitanshu Mehta --- cmd/thanos/query.go | 10 +++++----- docs/components/query.md | 16 +++++++++++----- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/cmd/thanos/query.go b/cmd/thanos/query.go index 4777f30c14..07628a94b3 100644 --- a/cmd/thanos/query.go +++ b/cmd/thanos/query.go @@ -104,21 +104,21 @@ func registerQuery(app *extkingpin.App) { endpoints := cmd.Flag("endpoint", "Addresses of statically configured Thanos API servers (repeatable). The scheme may be prefixed with 'dns+' or 'dnssrv+' to detect Thanos API servers through respective DNS lookups."). PlaceHolder("").Strings() - stores := cmd.Flag("store", "Addresses of statically configured store API servers (repeatable). The scheme may be prefixed with 'dns+' or 'dnssrv+' to detect store API servers through respective DNS lookups."). + stores := cmd.Flag("store", "Deprecation Warning - This flag is deprecated and replaced with `endpoints`. Addresses of statically configured store API servers (repeatable). The scheme may be prefixed with 'dns+' or 'dnssrv+' to detect store API servers through respective DNS lookups."). PlaceHolder("").Strings() // TODO(bwplotka): Hidden because we plan to extract discovery to separate API: https://github.com/thanos-io/thanos/issues/2600. - ruleEndpoints := cmd.Flag("rule", "Experimental: Addresses of statically configured rules API servers (repeatable). The scheme may be prefixed with 'dns+' or 'dnssrv+' to detect rule API servers through respective DNS lookups."). + ruleEndpoints := cmd.Flag("rule", "Deprecation Warning - This flag is deprecated and replaced with `endpoints`. Experimental: Addresses of statically configured rules API servers (repeatable). The scheme may be prefixed with 'dns+' or 'dnssrv+' to detect rule API servers through respective DNS lookups."). Hidden().PlaceHolder("").Strings() - metadataEndpoints := cmd.Flag("metadata", "Experimental: Addresses of statically configured metadata API servers (repeatable). The scheme may be prefixed with 'dns+' or 'dnssrv+' to detect metadata API servers through respective DNS lookups."). + metadataEndpoints := cmd.Flag("metadata", "Deprecation Warning - This flag is deprecated and replaced with `endpoints`. Experimental: Addresses of statically configured metadata API servers (repeatable). The scheme may be prefixed with 'dns+' or 'dnssrv+' to detect metadata API servers through respective DNS lookups."). Hidden().PlaceHolder("").Strings() - exemplarEndpoints := cmd.Flag("exemplar", "Experimental: Addresses of statically configured exemplars API servers (repeatable). The scheme may be prefixed with 'dns+' or 'dnssrv+' to detect exemplars API servers through respective DNS lookups."). + exemplarEndpoints := cmd.Flag("exemplar", "Deprecation Warning - This flag is deprecated and replaced with `endpoints`. Experimental: Addresses of statically configured exemplars API servers (repeatable). The scheme may be prefixed with 'dns+' or 'dnssrv+' to detect exemplars API servers through respective DNS lookups."). Hidden().PlaceHolder("").Strings() // TODO(atunik): Hidden because we plan to extract discovery to separate API: https://github.com/thanos-io/thanos/issues/2600. - targetEndpoints := cmd.Flag("target", "Experimental: Addresses of statically configured target API servers (repeatable). The scheme may be prefixed with 'dns+' or 'dnssrv+' to detect target API servers through respective DNS lookups."). + targetEndpoints := cmd.Flag("target", "Deprecation Warning - This flag is deprecated and replaced with `endpoints`. Experimental: Addresses of statically configured target API servers (repeatable). The scheme may be prefixed with 'dns+' or 'dnssrv+' to detect target API servers through respective DNS lookups."). Hidden().PlaceHolder("").Strings() strictStores := cmd.Flag("store-strict", "Addresses of only statically configured store API servers that are always used, even if the health check fails. Useful if you have a caching layer on top."). diff --git a/docs/components/query.md b/docs/components/query.md index 41e2cccf89..875ea9cd5f 100644 --- a/docs/components/query.md +++ b/docs/components/query.md @@ -255,6 +255,11 @@ Flags: --enable-feature= ... Comma separated experimental feature names to enable.The current list of features is promql-negative-offset and promql-at-modifier. + --endpoint= ... Addresses of statically configured Thanos API + servers (repeatable). The scheme may be + prefixed with 'dns+' or 'dnssrv+' to detect + Thanos API servers through respective DNS + lookups. --grpc-address="0.0.0.0:10901" Listen ip:port address for gRPC endpoints (StoreAPI). Make sure this address is routable @@ -367,11 +372,12 @@ Flags: --selector-label=="" ... Query selector labels that will be exposed in info endpoint (repeated). - --store= ... Addresses of statically configured store API - servers (repeatable). The scheme may be - prefixed with 'dns+' or 'dnssrv+' to detect - store API servers through respective DNS - lookups. + --store= ... Deprecation Warning - This flag is deprecated + and replaced with `endpoints`. Addresses of + statically configured store API servers + (repeatable). The scheme may be prefixed with + 'dns+' or 'dnssrv+' to detect store API servers + through respective DNS lookups. --store-strict= ... Addresses of only statically configured store API servers that are always used, even if the From 797f3a8260d50d049ac78776e453096e605a4255 Mon Sep 17 00:00:00 2001 From: Hitanshu Mehta Date: Tue, 19 Oct 2021 09:33:38 +0530 Subject: [PATCH 10/16] fix typo Signed-off-by: Hitanshu Mehta --- cmd/thanos/sidecar.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/thanos/sidecar.go b/cmd/thanos/sidecar.go index 537ed2acbf..c335b5b23a 100644 --- a/cmd/thanos/sidecar.go +++ b/cmd/thanos/sidecar.go @@ -245,7 +245,7 @@ func runSidecar( return errors.Wrap(err, "setup gRPC server") } - examplarSrv := exemplars.NewPrometheus(conf.prometheus.url, c, m.Labels) + exemplarSrv := exemplars.NewPrometheus(conf.prometheus.url, c, m.Labels) infoSrv := info.NewInfoServer( component.Sidecar.String(), @@ -283,7 +283,7 @@ func runSidecar( grpcserver.WithServer(rules.RegisterRulesServer(rules.NewPrometheus(conf.prometheus.url, c, m.Labels))), grpcserver.WithServer(targets.RegisterTargetsServer(targets.NewPrometheus(conf.prometheus.url, c, m.Labels))), grpcserver.WithServer(meta.RegisterMetadataServer(meta.NewPrometheus(conf.prometheus.url, c))), - grpcserver.WithServer(exemplars.RegisterExemplarsServer(examplarSrv)), + grpcserver.WithServer(exemplars.RegisterExemplarsServer(exemplarSrv)), grpcserver.WithServer(info.RegisterInfoServer(infoSrv)), grpcserver.WithListen(conf.grpc.bindAddress), grpcserver.WithGracePeriod(time.Duration(conf.grpc.gracePeriod)), From b9f677b3fa15b1ec1f223f49cc468933d5038c10 Mon Sep 17 00:00:00 2001 From: Hitanshu Mehta Date: Tue, 19 Oct 2021 18:25:35 +0530 Subject: [PATCH 11/16] export new Timestamps method Signed-off-by: Hitanshu Mehta --- pkg/store/prometheus.go | 14 +++++++++----- pkg/store/prometheus_test.go | 4 ++-- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/pkg/store/prometheus.go b/pkg/store/prometheus.go index 32b50dbd19..64e0b833c7 100644 --- a/pkg/store/prometheus.go +++ b/pkg/store/prometheus.go @@ -49,7 +49,7 @@ type PrometheusStore struct { component component.StoreAPI externalLabelsFn func() labels.Labels promVersion func() string - Timestamps func() (mint int64, maxt int64) + timestamps func() (mint int64, maxt int64) remoteReadAcceptableResponses []prompb.ReadRequest_ResponseType @@ -72,7 +72,7 @@ func NewPrometheusStore( baseURL *url.URL, component component.StoreAPI, externalLabelsFn func() labels.Labels, - Timestamps func() (mint int64, maxt int64), + timestamps func() (mint int64, maxt int64), promVersion func() string, ) (*PrometheusStore, error) { if logger == nil { @@ -85,7 +85,7 @@ func NewPrometheusStore( component: component, externalLabelsFn: externalLabelsFn, promVersion: promVersion, - Timestamps: Timestamps, + timestamps: timestamps, remoteReadAcceptableResponses: []prompb.ReadRequest_ResponseType{prompb.ReadRequest_STREAMED_XOR_CHUNKS, prompb.ReadRequest_SAMPLES}, buffers: sync.Pool{New: func() interface{} { b := make([]byte, 0, initialBufSize) @@ -107,7 +107,7 @@ func NewPrometheusStore( // This is fine for now, but might be needed in future. func (p *PrometheusStore) Info(_ context.Context, _ *storepb.InfoRequest) (*storepb.InfoResponse, error) { lset := p.externalLabelsFn() - mint, maxt := p.Timestamps() + mint, maxt := p.timestamps() res := &storepb.InfoResponse{ Labels: make([]labelpb.ZLabel, 0, len(lset)), @@ -153,7 +153,7 @@ func (p *PrometheusStore) Series(r *storepb.SeriesRequest, s storepb.Store_Serie } // Don't ask for more than available time. This includes potential `minTime` flag limit. - availableMinTime, _ := p.Timestamps() + availableMinTime, _ := p.timestamps() if r.MinTime < availableMinTime { r.MinTime = availableMinTime } @@ -625,3 +625,7 @@ func (p *PrometheusStore) LabelSet() []labelpb.ZLabelSet { return labelset } + +func (p *PrometheusStore) Timestamps() (mint int64, maxt int64) { + return p.timestamps() +} diff --git a/pkg/store/prometheus_test.go b/pkg/store/prometheus_test.go index 07090a16b0..a532a70db6 100644 --- a/pkg/store/prometheus_test.go +++ b/pkg/store/prometheus_test.go @@ -312,8 +312,8 @@ func TestPrometheusStore_SeriesLabels_e2e(t *testing.T) { Matchers: []storepb.LabelMatcher{ {Type: storepb.LabelMatcher_EQ, Name: "job", Value: "test"}, }, - MinTime: func() int64 { minTime, _ := promStore.Timestamps(); return minTime }(), - MaxTime: func() int64 { _, maxTime := promStore.Timestamps(); return maxTime }(), + MinTime: func() int64 { minTime, _ := promStore.timestamps(); return minTime }(), + MaxTime: func() int64 { _, maxTime := promStore.timestamps(); return maxTime }(), }, expected: []storepb.Series{ { From ceaf2b4af83f7294b939ee6116ed24929ad4d961 Mon Sep 17 00:00:00 2001 From: Hitanshu Mehta Date: Tue, 19 Oct 2021 18:27:09 +0530 Subject: [PATCH 12/16] add functional options in intializer of info server Signed-off-by: Hitanshu Mehta --- cmd/thanos/sidecar.go | 24 ++++++++--------- cmd/thanos/store.go | 12 +++------ pkg/info/info.go | 63 +++++++++++++++++++++++++++++++++---------- 3 files changed, 65 insertions(+), 34 deletions(-) diff --git a/cmd/thanos/sidecar.go b/cmd/thanos/sidecar.go index c335b5b23a..f454dd1d41 100644 --- a/cmd/thanos/sidecar.go +++ b/cmd/thanos/sidecar.go @@ -249,33 +249,33 @@ func runSidecar( infoSrv := info.NewInfoServer( component.Sidecar.String(), - func() []labelpb.ZLabelSet { + info.WithLabelSet(func() []labelpb.ZLabelSet { return promStore.LabelSet() - }, - func() *infopb.StoreInfo { + }), + info.WithStoreInfo(func() *infopb.StoreInfo { mint, maxt := promStore.Timestamps() return &infopb.StoreInfo{ MinTime: mint, MaxTime: maxt, } - }, - func() *infopb.ExemplarsInfo { + }), + info.WithExemplarsInfo(func() *infopb.ExemplarsInfo { // Currently Exemplars API does not expose metadata such as min/max time, // so we are using default minimum and maximum possible values as min/max time. return &infopb.ExemplarsInfo{ MinTime: query.MinTime, MaxTime: query.MaxTime, } - }, - func() *infopb.RulesInfo { + }), + info.WithRulesInfo(func() *infopb.RulesInfo { return &infopb.RulesInfo{} - }, - func() *infopb.TargetsInfo { + }), + info.WithTargetInfo(func() *infopb.TargetsInfo { return &infopb.TargetsInfo{} - }, - func() *infopb.MetricMetadataInfo { + }), + info.WithMetricMetadataInfo(func() *infopb.MetricMetadataInfo { return &infopb.MetricMetadataInfo{} - }, + }), ) s := grpcserver.New(logger, reg, tracer, grpcLogOpts, tagOpts, comp, grpcProbe, diff --git a/cmd/thanos/store.go b/cmd/thanos/store.go index 02cd4f5c24..14c37af08d 100644 --- a/cmd/thanos/store.go +++ b/cmd/thanos/store.go @@ -388,20 +388,16 @@ func runStore( infoSrv := info.NewInfoServer( component.Store.String(), - func() []labelpb.ZLabelSet { + info.WithLabelSet(func() []labelpb.ZLabelSet { return bs.LabelSet() - }, - func() *infopb.StoreInfo { + }), + info.WithStoreInfo(func() *infopb.StoreInfo { mint, maxt := bs.TimeRange() return &infopb.StoreInfo{ MinTime: mint, MaxTime: maxt, } - }, - nil, - nil, - nil, - nil, + }), ) // Start query (proxy) gRPC StoreAPI. diff --git a/pkg/info/info.go b/pkg/info/info.go index 0043722209..795b6feb9b 100644 --- a/pkg/info/info.go +++ b/pkg/info/info.go @@ -26,21 +26,52 @@ type InfoServer struct { func NewInfoServer( component string, - getLabelSet func() []labelpb.ZLabelSet, - getStoreInfo func() *infopb.StoreInfo, - getExemplarsInfo func() *infopb.ExemplarsInfo, - getRulesInfo func() *infopb.RulesInfo, - getTargetsInfo func() *infopb.TargetsInfo, - getMetricMetadataInfo func() *infopb.MetricMetadataInfo, + options ...func(*InfoServer), ) *InfoServer { - return &InfoServer{ - component: component, - getLabelSet: getLabelSet, - getStoreInfo: getStoreInfo, - getExemplarsInfo: getExemplarsInfo, - getRulesInfo: getRulesInfo, - getTargetsInfo: getTargetsInfo, - getMetricMetadataInfo: getMetricMetadataInfo, + srv := &InfoServer{ + component: component, + } + + for _, o := range options { + o(srv) + } + + return srv +} + +func WithLabelSet(getLabelSet func() []labelpb.ZLabelSet) func(*InfoServer) { + return func(s *InfoServer) { + s.getLabelSet = getLabelSet + } +} + +func WithStoreInfo(getStoreInfo func() *infopb.StoreInfo) func(*InfoServer) { + return func(s *InfoServer) { + s.getStoreInfo = getStoreInfo + } +} + +func WithRulesInfo(getRulesInfo func() *infopb.RulesInfo) func(*InfoServer) { + return func(s *InfoServer) { + s.getRulesInfo = getRulesInfo + } +} + +func WithExemplarsInfo(getExemplarsInfo func() *infopb.ExemplarsInfo) func(*InfoServer) { + return func(s *InfoServer) { + s.getExemplarsInfo = getExemplarsInfo + } +} + +func WithTargetInfo(getTargetsInfo func() *infopb.TargetsInfo) func(*InfoServer) { + return func(s *InfoServer) { + s.getTargetsInfo = getTargetsInfo + } +} + +func WithMetricMetadataInfo(getMetricMetadataInfo func() *infopb.MetricMetadataInfo) func(*InfoServer) { + return func(s *InfoServer) { + s.getMetricMetadataInfo = getMetricMetadataInfo } } @@ -53,6 +84,10 @@ func RegisterInfoServer(infoSrv infopb.InfoServer) func(*grpc.Server) { func (srv *InfoServer) Info(ctx context.Context, req *infopb.InfoRequest) (*infopb.InfoResponse, error) { + if srv.getLabelSet == nil { + srv.getLabelSet = func() []labelpb.ZLabelSet { return nil } + } + if srv.getStoreInfo == nil { srv.getStoreInfo = func() *infopb.StoreInfo { return nil } } From c3baaea6eddb5584dbba1eb7dc37bfca98754b95 Mon Sep 17 00:00:00 2001 From: Hitanshu Mehta Date: Tue, 19 Oct 2021 22:02:13 +0530 Subject: [PATCH 13/16] change e2e tests to use efficientgo/e2e Signed-off-by: Hitanshu Mehta --- test/e2e/info_api_test.go | 73 ++++++++++++++++++++------------------- 1 file changed, 38 insertions(+), 35 deletions(-) diff --git a/test/e2e/info_api_test.go b/test/e2e/info_api_test.go index eb84d839f0..f3521f61e8 100644 --- a/test/e2e/info_api_test.go +++ b/test/e2e/info_api_test.go @@ -13,8 +13,8 @@ import ( "testing" "time" - "github.com/cortexproject/cortex/integration/e2e" e2edb "github.com/cortexproject/cortex/integration/e2e/db" + "github.com/efficientgo/e2e" "github.com/prometheus/prometheus/pkg/labels" "github.com/thanos-io/thanos/pkg/objstore/client" "github.com/thanos-io/thanos/pkg/objstore/s3" @@ -27,52 +27,58 @@ import ( func TestInfo(t *testing.T) { t.Parallel() - s, err := e2e.NewScenario("e2e_test_info") + e, err := e2e.NewDockerEnvironment("e2e_test_info") testutil.Ok(t, err) - t.Cleanup(e2ethanos.CleanScenario(t, s)) + t.Cleanup(e2ethanos.CleanScenario(t, e)) - prom1, sidecar1, err := e2ethanos.NewPrometheusWithSidecar(s.SharedDir(), "e2e_test_info", "alone1", defaultPromConfig("prom-alone1", 0, "", ""), e2ethanos.DefaultPrometheusImage()) + prom1, sidecar1, err := e2ethanos.NewPrometheusWithSidecar(e, "alone1", defaultPromConfig("prom-alone1", 0, "", ""), e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) - prom2, sidecar2, err := e2ethanos.NewPrometheusWithSidecar(s.SharedDir(), "e2e_test_info", "alone2", defaultPromConfig("prom-alone2", 0, "", ""), e2ethanos.DefaultPrometheusImage()) + prom2, sidecar2, err := e2ethanos.NewPrometheusWithSidecar(e, "alone2", defaultPromConfig("prom-alone2", 0, "", ""), e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) - prom3, sidecar3, err := e2ethanos.NewPrometheusWithSidecar(s.SharedDir(), "e2e_test_info", "alone3", defaultPromConfig("prom-alone3", 0, "", ""), e2ethanos.DefaultPrometheusImage()) + prom3, sidecar3, err := e2ethanos.NewPrometheusWithSidecar(e, "alone3", defaultPromConfig("prom-alone3", 0, "", ""), e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(prom1, sidecar1, prom2, sidecar2, prom3, sidecar3)) - - m := e2edb.NewMinio(8080, "thanos") - testutil.Ok(t, s.StartAndWaitReady(m)) - str, err := e2ethanos.NewStoreGW(s.SharedDir(), "1", client.BucketConfig{ - Type: client.S3, - Config: s3.Config{ - Bucket: "thanos", - AccessKey: e2edb.MinioAccessKey, - SecretKey: e2edb.MinioSecretKey, - Endpoint: m.NetworkHTTPEndpoint(), - Insecure: true, + testutil.Ok(t, e2e.StartAndWaitReady(prom1, sidecar1, prom2, sidecar2, prom3, sidecar3)) + + const bucket = "info-api-test" + m := e2ethanos.NewMinio(e, "thanos-minio", bucket) + testutil.Ok(t, e2e.StartAndWaitReady(m)) + str, err := e2ethanos.NewStoreGW( + e, + "1", + client.BucketConfig{ + Type: client.S3, + Config: s3.Config{ + Bucket: bucket, + AccessKey: e2edb.MinioAccessKey, + SecretKey: e2edb.MinioSecretKey, + Endpoint: m.InternalEndpoint("http"), + Insecure: true, + }, }, - }) + "", + ) testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(str)) + testutil.Ok(t, e2e.StartAndWaitReady(str)) // Register `sidecar1` in all flags (i.e. '--store', '--rule', '--target', '--metadata', '--exemplar', '--endpoint') to verify // '--endpoint' flag works properly works together with other flags ('--target', '--metadata' etc.). // Register 2 sidecars and 1 storeGW using '--endpoint'. // Register `sidecar3` twice to verify it is deduplicated. - q, err := e2ethanos.NewQuerierBuilder(s.SharedDir(), "1", sidecar1.GRPCNetworkEndpoint()). - WithTargetAddresses(sidecar1.GRPCNetworkEndpoint()). - WithMetadataAddresses(sidecar1.GRPCNetworkEndpoint()). - WithExemplarAddresses(sidecar1.GRPCNetworkEndpoint()). - WithRuleAddresses(sidecar1.GRPCNetworkEndpoint()). + q, err := e2ethanos.NewQuerierBuilder(e, "1", sidecar1.InternalEndpoint("grpc")). + WithTargetAddresses(sidecar1.InternalEndpoint("grpc")). + WithMetadataAddresses(sidecar1.InternalEndpoint("grpc")). + WithExemplarAddresses(sidecar1.InternalEndpoint("grpc")). + WithRuleAddresses(sidecar1.InternalEndpoint("grpc")). WithEndpoints( - sidecar1.GRPCNetworkEndpoint(), - sidecar2.GRPCNetworkEndpoint(), - sidecar3.GRPCNetworkEndpoint(), - sidecar3.GRPCNetworkEndpoint(), - str.GRPCNetworkEndpoint(), + sidecar1.InternalEndpoint("grpc"), + sidecar2.InternalEndpoint("grpc"), + sidecar3.InternalEndpoint("grpc"), + sidecar3.InternalEndpoint("grpc"), + str.InternalEndpoint("grpc"), ). Build() testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(q)) + testutil.Ok(t, e2e.StartAndWaitReady(q)) expected := map[string][]query.EndpointStatus{ "sidecar": { @@ -124,7 +130,7 @@ func TestInfo(t *testing.T) { }, } - url := "http://" + path.Join(q.HTTPEndpoint(), "/api/v1/stores") + url := "http://" + path.Join(q.Endpoint("http"), "/api/v1/stores") ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) defer cancel() @@ -147,9 +153,6 @@ func TestInfo(t *testing.T) { err = json.Unmarshal(body, &res) testutil.Ok(t, err) - fmt.Println(res) - fmt.Println() - if err = assertStoreStatus(t, "sidecar", res.Data, expected); err != nil { return err } From 9b9b68c80bff317396bcdb57e6ff098c887d63ed Mon Sep 17 00:00:00 2001 From: Hitanshu Mehta Date: Thu, 21 Oct 2021 23:41:20 +0530 Subject: [PATCH 14/16] nits in info api e2e tests Signed-off-by: Hitanshu Mehta --- test/e2e/info_api_test.go | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/test/e2e/info_api_test.go b/test/e2e/info_api_test.go index f3521f61e8..c2a8805c3b 100644 --- a/test/e2e/info_api_test.go +++ b/test/e2e/info_api_test.go @@ -42,7 +42,7 @@ func TestInfo(t *testing.T) { const bucket = "info-api-test" m := e2ethanos.NewMinio(e, "thanos-minio", bucket) testutil.Ok(t, e2e.StartAndWaitReady(m)) - str, err := e2ethanos.NewStoreGW( + store, err := e2ethanos.NewStoreGW( e, "1", client.BucketConfig{ @@ -58,7 +58,7 @@ func TestInfo(t *testing.T) { "", ) testutil.Ok(t, err) - testutil.Ok(t, e2e.StartAndWaitReady(str)) + testutil.Ok(t, e2e.StartAndWaitReady(store)) // Register `sidecar1` in all flags (i.e. '--store', '--rule', '--target', '--metadata', '--exemplar', '--endpoint') to verify // '--endpoint' flag works properly works together with other flags ('--target', '--metadata' etc.). @@ -74,7 +74,7 @@ func TestInfo(t *testing.T) { sidecar2.InternalEndpoint("grpc"), sidecar3.InternalEndpoint("grpc"), sidecar3.InternalEndpoint("grpc"), - str.InternalEndpoint("grpc"), + store.InternalEndpoint("grpc"), ). Build() testutil.Ok(t, err) @@ -138,20 +138,28 @@ func TestInfo(t *testing.T) { err = runutil.Retry(time.Second, ctx.Done(), func() error { req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) - testutil.Ok(t, err) + if err != nil { + return err + } resp, err := http.DefaultClient.Do(req) - testutil.Ok(t, err) + if err != nil { + return err + } body, err := ioutil.ReadAll(resp.Body) - testutil.Ok(t, err) + if err != nil { + return err + } var res struct { Data map[string][]query.EndpointStatus `json:"data"` } err = json.Unmarshal(body, &res) - testutil.Ok(t, err) + if err != nil { + return err + } if err = assertStoreStatus(t, "sidecar", res.Data, expected); err != nil { return err From fec2e8bc14079af447c76a9400450756842e7888 Mon Sep 17 00:00:00 2001 From: Hitanshu Mehta Date: Fri, 5 Nov 2021 15:56:31 +0530 Subject: [PATCH 15/16] minor nits Signed-off-by: Hitanshu Mehta --- cmd/thanos/query.go | 10 +++++----- pkg/store/prometheus.go | 2 -- test/e2e/info_api_test.go | 4 +--- 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/cmd/thanos/query.go b/cmd/thanos/query.go index 07628a94b3..7a4a54f505 100644 --- a/cmd/thanos/query.go +++ b/cmd/thanos/query.go @@ -104,21 +104,21 @@ func registerQuery(app *extkingpin.App) { endpoints := cmd.Flag("endpoint", "Addresses of statically configured Thanos API servers (repeatable). The scheme may be prefixed with 'dns+' or 'dnssrv+' to detect Thanos API servers through respective DNS lookups."). PlaceHolder("").Strings() - stores := cmd.Flag("store", "Deprecation Warning - This flag is deprecated and replaced with `endpoints`. Addresses of statically configured store API servers (repeatable). The scheme may be prefixed with 'dns+' or 'dnssrv+' to detect store API servers through respective DNS lookups."). + stores := cmd.Flag("store", "Deprecation Warning - This flag is deprecated and replaced with `endpoint`. Addresses of statically configured store API servers (repeatable). The scheme may be prefixed with 'dns+' or 'dnssrv+' to detect store API servers through respective DNS lookups."). PlaceHolder("").Strings() // TODO(bwplotka): Hidden because we plan to extract discovery to separate API: https://github.com/thanos-io/thanos/issues/2600. - ruleEndpoints := cmd.Flag("rule", "Deprecation Warning - This flag is deprecated and replaced with `endpoints`. Experimental: Addresses of statically configured rules API servers (repeatable). The scheme may be prefixed with 'dns+' or 'dnssrv+' to detect rule API servers through respective DNS lookups."). + ruleEndpoints := cmd.Flag("rule", "Deprecation Warning - This flag is deprecated and replaced with `endpoint`. Experimental: Addresses of statically configured rules API servers (repeatable). The scheme may be prefixed with 'dns+' or 'dnssrv+' to detect rule API servers through respective DNS lookups."). Hidden().PlaceHolder("").Strings() - metadataEndpoints := cmd.Flag("metadata", "Deprecation Warning - This flag is deprecated and replaced with `endpoints`. Experimental: Addresses of statically configured metadata API servers (repeatable). The scheme may be prefixed with 'dns+' or 'dnssrv+' to detect metadata API servers through respective DNS lookups."). + metadataEndpoints := cmd.Flag("metadata", "Deprecation Warning - This flag is deprecated and replaced with `endpoint`. Experimental: Addresses of statically configured metadata API servers (repeatable). The scheme may be prefixed with 'dns+' or 'dnssrv+' to detect metadata API servers through respective DNS lookups."). Hidden().PlaceHolder("").Strings() - exemplarEndpoints := cmd.Flag("exemplar", "Deprecation Warning - This flag is deprecated and replaced with `endpoints`. Experimental: Addresses of statically configured exemplars API servers (repeatable). The scheme may be prefixed with 'dns+' or 'dnssrv+' to detect exemplars API servers through respective DNS lookups."). + exemplarEndpoints := cmd.Flag("exemplar", "Deprecation Warning - This flag is deprecated and replaced with `endpoint`. Experimental: Addresses of statically configured exemplars API servers (repeatable). The scheme may be prefixed with 'dns+' or 'dnssrv+' to detect exemplars API servers through respective DNS lookups."). Hidden().PlaceHolder("").Strings() // TODO(atunik): Hidden because we plan to extract discovery to separate API: https://github.com/thanos-io/thanos/issues/2600. - targetEndpoints := cmd.Flag("target", "Deprecation Warning - This flag is deprecated and replaced with `endpoints`. Experimental: Addresses of statically configured target API servers (repeatable). The scheme may be prefixed with 'dns+' or 'dnssrv+' to detect target API servers through respective DNS lookups."). + targetEndpoints := cmd.Flag("target", "Deprecation Warning - This flag is deprecated and replaced with `endpoint`. Experimental: Addresses of statically configured target API servers (repeatable). The scheme may be prefixed with 'dns+' or 'dnssrv+' to detect target API servers through respective DNS lookups."). Hidden().PlaceHolder("").Strings() strictStores := cmd.Flag("store-strict", "Addresses of only statically configured store API servers that are always used, even if the health check fails. Useful if you have a caching layer on top."). diff --git a/pkg/store/prometheus.go b/pkg/store/prometheus.go index 64e0b833c7..3403f6e607 100644 --- a/pkg/store/prometheus.go +++ b/pkg/store/prometheus.go @@ -614,8 +614,6 @@ func (p *PrometheusStore) LabelSet() []labelpb.ZLabelSet { labels := make([]labelpb.ZLabel, 0, len(lset)) labels = append(labels, labelpb.ZLabelsFromPromLabels(lset)...) - // Until we deprecate the single labels in the reply, we just duplicate - // them here for migration/compatibility purposes. labelset := []labelpb.ZLabelSet{} if len(labels) > 0 { labelset = append(labelset, labelpb.ZLabelSet{ diff --git a/test/e2e/info_api_test.go b/test/e2e/info_api_test.go index c2a8805c3b..1da5a95987 100644 --- a/test/e2e/info_api_test.go +++ b/test/e2e/info_api_test.go @@ -148,9 +148,7 @@ func TestInfo(t *testing.T) { } body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return err - } + defer runutil.CloseWithErrCapture(&err, resp.Body, "response body close") var res struct { Data map[string][]query.EndpointStatus `json:"data"` From a330d23b0f2b56017ed589393d895e6c67714203 Mon Sep 17 00:00:00 2001 From: Hitanshu Mehta Date: Tue, 9 Nov 2021 09:30:42 +0530 Subject: [PATCH 16/16] fix docs Signed-off-by: Hitanshu Mehta --- docs/components/query.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/components/query.md b/docs/components/query.md index 875ea9cd5f..ef3ca3e3ca 100644 --- a/docs/components/query.md +++ b/docs/components/query.md @@ -373,7 +373,7 @@ Flags: Query selector labels that will be exposed in info endpoint (repeated). --store= ... Deprecation Warning - This flag is deprecated - and replaced with `endpoints`. Addresses of + and replaced with `endpoint`. Addresses of statically configured store API servers (repeatable). The scheme may be prefixed with 'dns+' or 'dnssrv+' to detect store API servers