Sidecar: mark as unqueryable if prometheus is down

If the prometheus that belongs to a sidecar is down we dont need to query the sidecar. This PR makes it so that we take the sidecar out of the endpoint set then. Signed-off-by: Michael Hoffmann <[email protected]>
thanos-io · Apr 22, 2024 · 1df444c · 1df444c
1 parent c3cd031
commit 1df444c
Show file tree

Hide file tree

Showing 7 changed files with 144 additions and 49 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -35,6 +35,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re
 - [#7233](https://github.com/thanos-io/thanos/pull/7233): UI: Showing Block Size Stats
 - [#7280](https://github.com/thanos-io/thanos/pull/7281): Adding User-Agent to request logs
 - [#7219](https://github.com/thanos-io/thanos/pull/7219): Receive: add `--remote-write.client-tls-secure` and `--remote-write.client-tls-skip-verify` flags to stop relying on grpc server config to determine grpc client secure/skipVerify.
+- [#7297](https://github.com/thanos-io/thanos/pull/7297): Sidecar: mark as not queryable if prometheus is down.
 
 ### Changed
 

diff --git a/cmd/thanos/sidecar.go b/cmd/thanos/sidecar.go
@@ -291,7 +291,7 @@ func runSidecar(
 						TsdbInfos:                    promStore.TSDBInfos(),
 					}
 				}
-				return nil
+				return &infopb.StoreInfo{PrometheusDown: true}
 			}),
 			info.WithExemplarsInfoFunc(),
 			info.WithRulesInfoFunc(),

diff --git a/pkg/info/info.go b/pkg/info/info.go
@@ -6,9 +6,10 @@ package info
 import (
 	"context"
 
+	"google.golang.org/grpc"
+
 	"github.com/thanos-io/thanos/pkg/info/infopb"
 	"github.com/thanos-io/thanos/pkg/store/labelpb"
-	"google.golang.org/grpc"
 )
 
 // InfoServer implements the corresponding protobuf interface

diff --git a/pkg/info/infopb/rpc.pb.go b/pkg/info/infopb/rpc.pb.go
diff --git a/pkg/info/infopb/rpc.proto b/pkg/info/infopb/rpc.proto
@@ -64,6 +64,10 @@ message StoreInfo {
 
     // TSDBInfos holds metadata for all TSDBs exposed by the store.
     repeated TSDBInfo tsdb_infos = 6 [(gogoproto.nullable) = false];
+
+    // Valid for Sidecars. We can use this to indicate that the store is available
+    // but gRPC calls would fail since its downstream prometheus is not available.
+    bool prometheus_down = 7;
 }
 
 // RulesInfo holds the metadata related to Rules API exposed by the component.

diff --git a/pkg/query/endpointset.go b/pkg/query/endpointset.go
@@ -13,15 +13,14 @@ import (
 	"time"
 	"unicode/utf8"
 
-	"github.com/thanos-io/thanos/pkg/api/query/querypb"
-
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/prometheus/model/labels"
 	"google.golang.org/grpc"
 
+	"github.com/thanos-io/thanos/pkg/api/query/querypb"
 	"github.com/thanos-io/thanos/pkg/component"
 	"github.com/thanos-io/thanos/pkg/exemplars/exemplarspb"
 	"github.com/thanos-io/thanos/pkg/info/infopb"
@@ -716,7 +715,20 @@ func (er *endpointRef) isQueryable() bool {
 	er.mtx.RLock()
 	defer er.mtx.RUnlock()
 
-	return er.isStrict || er.status.LastError == nil
+	if er.isStrict {
+		return true
+	}
+	if er.status.LastError != nil {
+		return false
+	}
+
+	// Dont query sidecars whose prometheus is known to be unreachable.
+	if er.metadata != nil && er.metadata.ComponentType == component.Sidecar.String() {
+		if er.metadata.Store != nil {
+			return !er.metadata.Store.PrometheusDown
+		}
+	}
+	return true
 }
 
 func (er *endpointRef) ComponentType() component.Component {

diff --git a/pkg/query/endpointset_test.go b/pkg/query/endpointset_test.go
@@ -14,20 +14,19 @@ import (
 	"testing"
 	"time"
 
+	"github.com/efficientgo/core/testutil"
+	"github.com/pkg/errors"
 	"github.com/stretchr/testify/require"
-
-	"github.com/prometheus/prometheus/model/labels"
-	"github.com/thanos-io/thanos/pkg/store"
-
 	"golang.org/x/sync/errgroup"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/insecure"
 
-	"github.com/efficientgo/core/testutil"
-	"github.com/pkg/errors"
 	promtestutil "github.com/prometheus/client_golang/prometheus/testutil"
+	"github.com/prometheus/prometheus/model/labels"
+
 	"github.com/thanos-io/thanos/pkg/component"
 	"github.com/thanos-io/thanos/pkg/info/infopb"
+	"github.com/thanos-io/thanos/pkg/store"
 	"github.com/thanos-io/thanos/pkg/store/labelpb"
 	"github.com/thanos-io/thanos/pkg/store/storepb"
 )
@@ -497,6 +496,46 @@ func TestEndpointSetUpdate_EndpointGoingAway(t *testing.T) {
 	testutil.Equals(t, 0, len(endpointSet.GetStoreClients()))
 }
 
+func TestEndpointSetUpdate_SidecarPrometheusBecomingUnreachable(t *testing.T) {
+	storeInfo := &infopb.InfoResponse{
+		ComponentType: component.Sidecar.String(),
+		Store: &infopb.StoreInfo{
+			MinTime: math.MinInt64,
+			MaxTime: math.MaxInt64,
+		},
+		Exemplars:      &infopb.ExemplarsInfo{},
+		Rules:          &infopb.RulesInfo{},
+		MetricMetadata: &infopb.MetricMetadataInfo{},
+		Targets:        &infopb.TargetsInfo{},
+	}
+	endpoints, err := startTestEndpoints([]testEndpointMeta{
+		{
+			InfoResponse: storeInfo,
+			extlsetFn: func(addr string) []labelpb.ZLabelSet {
+				return labelpb.ZLabelSetsFromPromLabels(
+					labels.FromStrings("addr", addr, "a", "b"),
+				)
+			},
+		},
+	})
+	testutil.Ok(t, err)
+	defer endpoints.Close()
+
+	discoveredEndpointAddr := endpoints.EndpointAddresses()
+	endpointSet := makeEndpointSet(discoveredEndpointAddr, false, time.Now)
+	defer endpointSet.Close()
+
+	// Initial update.
+	endpointSet.Update(context.Background())
+	testutil.Equals(t, 1, len(endpointSet.GetEndpointStatus()))
+	testutil.Equals(t, 1, len(endpointSet.GetStoreClients()))
+
+	storeInfo.Store.PrometheusDown = true
+	endpointSet.Update(context.Background())
+	testutil.Equals(t, 1, len(endpointSet.GetEndpointStatus()))
+	testutil.Equals(t, 0, len(endpointSet.GetStoreClients()))
+}
+
 func TestEndpointSetUpdate_EndpointComingOnline(t *testing.T) {
 	endpoints, err := startTestEndpoints([]testEndpointMeta{
 		{