querier: per-endpoint configuration

CHANGELOG.md

@@ -16,6 +16,8 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re
 - [#4679](https://github.com/thanos-io/thanos/pull/4679) Added `enable-feature` flag to enable negative offsets and @ modifier, similar to Prometheus.
 - [#4696](https://github.com/thanos-io/thanos/pull/4696) Query: add cache name to tracing spans.
 - [#4764](https://github.com/thanos-io/thanos/pull/4764) Compactor: add `block-viewer.global.sync-block-timeout` flag to set the timeout of synchronization block metas.
+- [#4389](https://github.com/thanos-io/thanos/pull/4389) Querier: add `endpoint.configuration` and `endpoint.configuration-file` for granular endpoint configuration YAML content or file. 
+  - *:warning:* This also deprecates the following flags `store.sd-interval`, `store.sd-dns-interval`, `store.sd-dns-resolver`, `store.sd-files` and all `grpc-client-.*`. They will be removed in v0.27.0.
 
 ### Fixed
 

cmd/thanos/receive.go

@@ -22,6 +22,7 @@ import (
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/prometheus/pkg/labels"
 	"github.com/prometheus/prometheus/tsdb"
+	"github.com/thanos-io/thanos/pkg/exthttp"
 
 	"github.com/thanos-io/thanos/pkg/block/metadata"
 	"github.com/thanos-io/thanos/pkg/exemplars"
@@ -120,16 +121,21 @@ func runReceive(
 		return err
 	}
 
+	TLSConfig := exthttp.TLSConfig{
+		CertFile:   conf.rwClientCert,
+		KeyFile:    conf.rwClientKey,
+		CAFile:     conf.rwClientServerCA,
+		ServerName: conf.rwClientServerName,
+	}
+
 	dialOpts, err := extgrpc.StoreClientGRPCOpts(
 		logger,
 		reg,
 		tracer,
+		"",
 		*conf.grpcCert != "",
 		*conf.grpcClientCA == "",
-		conf.rwClientCert,
-		conf.rwClientKey,
-		conf.rwClientServerCA,
-		conf.rwClientServerName,
+		TLSConfig,
 	)
 	if err != nil {
 		return err

cmd/thanos/rule.go

@@ -33,8 +33,8 @@ import (
 	"github.com/prometheus/prometheus/tsdb"
 	"github.com/prometheus/prometheus/util/strutil"
 	"github.com/thanos-io/thanos/pkg/errutil"
+	"github.com/thanos-io/thanos/pkg/exthttp"
 	"github.com/thanos-io/thanos/pkg/extkingpin"
-	"github.com/thanos-io/thanos/pkg/httpconfig"
 
 	extflag "github.com/efficientgo/tools/extkingpin"
 	"github.com/thanos-io/thanos/pkg/alert"
@@ -265,29 +265,29 @@ func runRule(
 ) error {
 	metrics := newRuleMetrics(reg)
 
-	var queryCfg []httpconfig.Config
+	var queryCfg []exthttp.Config
 	var err error
 	if len(conf.queryConfigYAML) > 0 {
-		queryCfg, err = httpconfig.LoadConfigs(conf.queryConfigYAML)
+		queryCfg, err = exthttp.LoadConfigs(conf.queryConfigYAML)
 		if err != nil {
 			return err
 		}
 	} else {
-		queryCfg, err = httpconfig.BuildConfig(conf.query.addrs)
+		queryCfg, err = exthttp.BuildConfig(conf.query.addrs)
 		if err != nil {
 			return errors.Wrap(err, "query configuration")
 		}
 
 		// Build the query configuration from the legacy query flags.
-		var fileSDConfigs []httpconfig.FileSDConfig
+		var fileSDConfigs []exthttp.FileSDConfig
 		if len(conf.query.sdFiles) > 0 {
-			fileSDConfigs = append(fileSDConfigs, httpconfig.FileSDConfig{
+			fileSDConfigs = append(fileSDConfigs, exthttp.FileSDConfig{
 				Files:           conf.query.sdFiles,
 				RefreshInterval: model.Duration(conf.query.sdInterval),
 			})
 			queryCfg = append(queryCfg,
-				httpconfig.Config{
-					EndpointsConfig: httpconfig.EndpointsConfig{
+				exthttp.Config{
+					EndpointsConfig: exthttp.EndpointsConfig{
 						Scheme:        "http",
 						FileSDConfigs: fileSDConfigs,
 					},
@@ -301,22 +301,21 @@ func runRule(
 		extprom.WrapRegistererWithPrefix("thanos_rule_query_apis_", reg),
 		dns.ResolverType(conf.query.dnsSDResolver),
 	)
-	var queryClients []*httpconfig.Client
+	var queryClients []*exthttp.Client
 	queryClientMetrics := extpromhttp.NewClientMetrics(extprom.WrapRegistererWith(prometheus.Labels{"client": "query"}, reg))
 	for _, cfg := range queryCfg {
-		cfg.HTTPClientConfig.ClientMetrics = queryClientMetrics
-		c, err := httpconfig.NewHTTPClient(cfg.HTTPClientConfig, "query")
+		c, err := exthttp.NewHTTPClient(cfg.HTTPClientConfig, "query", queryClientMetrics)
 		if err != nil {
 			return err
 		}
 		c.Transport = tracing.HTTPTripperware(logger, c.Transport)
-		queryClient, err := httpconfig.NewClient(logger, cfg.EndpointsConfig, c, queryProvider.Clone())
+		queryClient, err := exthttp.NewClient(logger, cfg.EndpointsConfig, c, queryProvider.Clone())
 		if err != nil {
 			return err
 		}
 		queryClients = append(queryClients, queryClient)
 		// Discover and resolve query addresses.
-		addDiscoveryGroups(g, queryClient, conf.query.dnsSDInterval)
+		addDiscoveryGroups(g, queryClient.Discoverer, conf.query.dnsSDInterval)
 	}
 
 	db, err := tsdb.Open(conf.dataDir, log.With(logger, "component", "tsdb"), reg, tsdbOpts, nil)
@@ -379,19 +378,18 @@ func runRule(
 		extprom.WrapRegistererWith(prometheus.Labels{"client": "alertmanager"}, reg),
 	)
 	for _, cfg := range alertingCfg.Alertmanagers {
-		cfg.HTTPClientConfig.ClientMetrics = amClientMetrics
-		c, err := httpconfig.NewHTTPClient(cfg.HTTPClientConfig, "alertmanager")
+		c, err := exthttp.NewHTTPClient(cfg.HTTPClientConfig, "alertmanager", amClientMetrics)
 		if err != nil {
 			return err
 		}
 		c.Transport = tracing.HTTPTripperware(logger, c.Transport)
 		// Each Alertmanager client has a different list of targets thus each needs its own DNS provider.
-		amClient, err := httpconfig.NewClient(logger, cfg.EndpointsConfig, c, amProvider.Clone())
+		amClient, err := exthttp.NewClient(logger, cfg.EndpointsConfig, c, amProvider.Clone())
 		if err != nil {
 			return err
 		}
 		// Discover and resolve Alertmanager addresses.
-		addDiscoveryGroups(g, amClient, conf.alertmgr.alertmgrsDNSSDInterval)
+		addDiscoveryGroups(g, amClient.Discoverer, conf.alertmgr.alertmgrsDNSSDInterval)
 
 		alertmgrs = append(alertmgrs, alert.NewAlertmanager(logger, amClient, time.Duration(cfg.Timeout), cfg.APIVersion))
 	}
@@ -705,7 +703,7 @@ func removeDuplicateQueryEndpoints(logger log.Logger, duplicatedQueriers prometh
 
 func queryFuncCreator(
 	logger log.Logger,
-	queriers []*httpconfig.Client,
+	queriers []*exthttp.Client,
 	duplicatedQuery prometheus.Counter,
 	ruleEvalWarnings *prometheus.CounterVec,
 	httpMethod string,
@@ -761,18 +759,18 @@ func queryFuncCreator(
 	}
 }
 
-func addDiscoveryGroups(g *run.Group, c *httpconfig.Client, interval time.Duration) {
+func addDiscoveryGroups(g *run.Group, d *exthttp.Discoverer, interval time.Duration) {
 	ctx, cancel := context.WithCancel(context.Background())
 	g.Add(func() error {
-		c.Discover(ctx)
+		d.Discover(ctx)
 		return nil
 	}, func(error) {
 		cancel()
 	})
 
 	g.Add(func() error {
 		return runutil.Repeat(interval, ctx.Done(), func() error {
-			return c.Resolve(ctx)
+			return d.Resolve(ctx)
 		})
 	}, func(error) {
 		cancel()

cmd/thanos/sidecar.go

@@ -29,7 +29,6 @@ import (
 	"github.com/thanos-io/thanos/pkg/exthttp"
 	"github.com/thanos-io/thanos/pkg/extkingpin"
 	"github.com/thanos-io/thanos/pkg/extprom"
-	"github.com/thanos-io/thanos/pkg/httpconfig"
 	"github.com/thanos-io/thanos/pkg/logging"
 	meta "github.com/thanos-io/thanos/pkg/metadata"
 	thanosmodel "github.com/thanos-io/thanos/pkg/model"
@@ -228,7 +227,7 @@ func runSidecar(
 		t := exthttp.NewTransport()
 		t.MaxIdleConnsPerHost = conf.connection.maxIdleConnsPerHost
 		t.MaxIdleConns = conf.connection.maxIdleConns
-		c := promclient.NewClient(&http.Client{Transport: tracing.HTTPTripperware(logger, t)}, logger, httpconfig.ThanosUserAgent)
+		c := promclient.NewClient(&http.Client{Transport: tracing.HTTPTripperware(logger, t)}, logger, exthttp.ThanosUserAgent)
 
 		promStore, err := store.NewPrometheusStore(logger, reg, c, conf.prometheus.url, component.Sidecar, m.Labels, m.Timestamps, m.Version)
 		if err != nil {

docs/components/query.md

@@ -255,6 +255,18 @@ Flags:
       --enable-feature= ...      Comma separated experimental feature names to
                                  enable.The current list of features is
                                  promql-negative-offset and promql-at-modifier.
+      --endpoint.config=<content>  
+                                 Alternative to 'endpoint.config-file' flag
+                                 (mutually exclusive). Content of YAML file that
+                                 contains set of endpoints (e.g Store API) with
+                                 optional TLS options. To enable TLS either use
+                                 this option or deprecated ones
+                                 --grpc-client-tls* .
+      --endpoint.config-file=<file-path>  
+                                 Path to YAML file that contains set of
+                                 endpoints (e.g Store API) with optional TLS
+                                 options. To enable TLS either use this option
+                                 or deprecated ones --grpc-client-tls* .
       --grpc-address="0.0.0.0:10901"  
                                  Listen ip:port address for gRPC endpoints
                                  (StoreAPI). Make sure this address is routable

pkg/alert/config.go

@@ -13,7 +13,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/prometheus/pkg/relabel"
-	"github.com/thanos-io/thanos/pkg/httpconfig"
+	"github.com/thanos-io/thanos/pkg/exthttp"
 	"gopkg.in/yaml.v2"
 
 	"github.com/thanos-io/thanos/pkg/discovery/dns"
@@ -25,10 +25,10 @@ type AlertingConfig struct {
 
 // AlertmanagerConfig represents a client to a cluster of Alertmanager endpoints.
 type AlertmanagerConfig struct {
-	HTTPClientConfig httpconfig.ClientConfig    `yaml:"http_config"`
-	EndpointsConfig  httpconfig.EndpointsConfig `yaml:",inline"`
-	Timeout          model.Duration             `yaml:"timeout"`
-	APIVersion       APIVersion                 `yaml:"api_version"`
+	HTTPClientConfig exthttp.ClientConfig    `yaml:"http_config"`
+	EndpointsConfig  exthttp.EndpointsConfig `yaml:",inline"`
+	Timeout          model.Duration          `yaml:"timeout"`
+	APIVersion       APIVersion              `yaml:"api_version"`
 }
 
 // APIVersion represents the API version of the Alertmanager endpoint.
@@ -61,10 +61,10 @@ func (v *APIVersion) UnmarshalYAML(unmarshal func(interface{}) error) error {
 
 func DefaultAlertmanagerConfig() AlertmanagerConfig {
 	return AlertmanagerConfig{
-		EndpointsConfig: httpconfig.EndpointsConfig{
-			Scheme:          "http",
-			StaticAddresses: []string{},
-			FileSDConfigs:   []httpconfig.FileSDConfig{},
+		EndpointsConfig: exthttp.EndpointsConfig{
+			Scheme:        "http",
+			Addresses:     []string{},
+			FileSDConfigs: []exthttp.FileSDConfig{},
 		},
 		Timeout:    model.Duration(time.Second * 10),
 		APIVersion: APIv1,
@@ -111,21 +111,21 @@ func BuildAlertmanagerConfig(address string, timeout time.Duration) (Alertmanage
 			break
 		}
 	}
-	var basicAuth httpconfig.BasicAuth
+	var basicAuth exthttp.BasicAuth
 	if parsed.User != nil && parsed.User.String() != "" {
 		basicAuth.Username = parsed.User.Username()
 		pw, _ := parsed.User.Password()
 		basicAuth.Password = pw
 	}
 
 	return AlertmanagerConfig{
-		HTTPClientConfig: httpconfig.ClientConfig{
+		HTTPClientConfig: exthttp.ClientConfig{
 			BasicAuth: basicAuth,
 		},
-		EndpointsConfig: httpconfig.EndpointsConfig{
-			PathPrefix:      parsed.Path,
-			Scheme:          scheme,
-			StaticAddresses: []string{host},
+		EndpointsConfig: exthttp.EndpointsConfig{
+			PathPrefix: parsed.Path,
+			Scheme:     scheme,
+			Addresses:  []string{host},
 		},
 		Timeout:    model.Duration(timeout),
 		APIVersion: APIv1,

pkg/extgrpc/config.go

@@ -0,0 +1,53 @@
+// Copyright (c) The Thanos Authors.
+// Licensed under the Apache License 2.0.
+
+package extgrpc
+
+import (
+	"strings"
+
+	"github.com/pkg/errors"
+	"github.com/thanos-io/thanos/pkg/exthttp"
+)
+
+// Config is a structure that allows pointing to various gRPC endpoint, e.g Querier connecting to StoreAPI.
+type Config struct {
+	GRPCClientConfig exthttp.ClientConfig `yaml:"grpc_config"`
+	EndpointsConfig  EndpointsConfig      `yaml:",inline"`
+}
+
+func DefaultConfig() Config {
+	return Config{
+		EndpointsConfig: EndpointsConfig{
+			Addresses:     []string{},
+			FileSDConfigs: []exthttp.FileSDConfig{},
+		},
+	}
+}
+
+// UnmarshalYAML implements the yaml.Unmarshaler interface.
+func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	*c = DefaultConfig()
+	type plain Config
+	return unmarshal((*plain)(c))
+}
+
+// BuildConfig returns a configuration from a static addresses.
+func BuildConfig(addrs []string) ([]Config, error) {
+	configs := make([]Config, 0, len(addrs))
+	for i, addr := range addrs {
+		if addr == "" {
+			return nil, errors.Errorf("static address cannot be empty, but was at index %d", i)
+		}
+		if strings.Contains(addr, "/") {
+			return nil, errors.Errorf("gRPC address either has HTTP scheme or path. We expect only host+port with optional dns+ dnssrv+ prefix in it. Got %v", addr)
+		}
+
+		configs = append(configs, Config{
+			EndpointsConfig: EndpointsConfig{
+				Addresses: []string{addr},
+			},
+		})
+	}
+	return configs, nil
+}