From 406d8724764bc36a30dbf472474ffa7dbd045317 Mon Sep 17 00:00:00 2001 From: Simon Pasquier Date: Sun, 11 Jun 2023 06:23:08 +0200 Subject: [PATCH] Report gRPC connnection errors to the caller (#6428) By default `grpc.DialContext()` is non-blocking so any connection issue will not be surfaced to the user. This change makes it blocking and configures the gRPC dialer to report the underlying error if any happens. Signed-off-by: Simon Pasquier --- CHANGELOG.md | 1 + pkg/query/endpointset.go | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 48c05c12a6..133982ad94 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#6342](https://github.com/thanos-io/thanos/pull/6342) Cache/Redis: Upgrade `rueidis` to v1.0.2 to to improve error handling while shrinking a redis cluster. - [#6325](https://github.com/thanos-io/thanos/pull/6325) Store: return gRPC resource exhausted error for byte limiter. - [#6399](https://github.com/thanos-io/thanos/pull/6399) *: Fix double-counting bug in http_request_duration metric +- [#6428](https://github.com/thanos-io/thanos/pull/6428) Report gRPC connnection errors in the logs. ### Changed - [#6168](https://github.com/thanos-io/thanos/pull/6168) Receiver: Make ketama hashring fail early when configured with number of nodes lower than the replication factor. diff --git a/pkg/query/endpointset.go b/pkg/query/endpointset.go index ad963edd3f..a92bcf39b6 100644 --- a/pkg/query/endpointset.go +++ b/pkg/query/endpointset.go @@ -636,6 +636,11 @@ type endpointRef struct { // The call to newEndpointRef will return an error if establishing the channel fails. func (e *EndpointSet) newEndpointRef(ctx context.Context, spec *GRPCEndpointSpec) (*endpointRef, error) { dialOpts := append(e.dialOpts, spec.dialOpts...) + // By default DialContext is non-blocking which means that any connection + // failure won't be reported/logged. Instead block until the connection is + // successfully established and return the details of the connection error + // if any. + dialOpts = append(dialOpts, grpc.WithReturnConnectionError()) conn, err := grpc.DialContext(ctx, spec.Addr(), dialOpts...) if err != nil { return nil, errors.Wrap(err, "dialing connection")