From 7ab399fa4288560280bbee58cd13762383344bc5 Mon Sep 17 00:00:00 2001 From: Yuri Nikolic Date: Thu, 23 Nov 2023 15:02:36 +0100 Subject: [PATCH 1/7] Allow ingester's read path to return gRPC errors Signed-off-by: Yuri Nikolic --- CHANGELOG.md | 7 +- pkg/distributor/errors.go | 7 +- pkg/ingester/active_series.go | 5 +- pkg/ingester/errors.go | 55 ++++++++- pkg/ingester/errors_test.go | 139 +++++++++++++++++++---- pkg/ingester/ingester.go | 93 ++++++++------- pkg/mimirpb/mimir.proto | 1 + pkg/querier/error_translate_queryable.go | 35 +++--- pkg/util/http.go | 7 ++ 9 files changed, 264 insertions(+), 85 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f0fc124adc1..60d60f00440 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,9 +26,10 @@ * `http.StatusAccepted` (202) code is replaced with `codes.AlreadyExists`. * `http.BadRequest` (400) code is replaced with `codes.FailedPrecondition`. * `http.StatusTooManyRequests` (429) and the non-standard `529` (The service is overloaded) codes are replaced with `codes.ResourceExhausted`. -* [CHANGE] Ingester: by setting the newly introduced experimental CLI flag `-ingester.return-only-grpc-errors` to true, `Push()` will return only gRPC errors. This feature changes the following status codes: #6443 #6723 - * `http.StatusBadRequest` (400) is replaced with `codes.FailedPrecondition`. - * `http.StatusServiceUnavailable` (503) and `codes.Unknown` are replaced with `codes.Internal`. +* [CHANGE] Ingester: by setting the newly introduced experimental CLI flag `-ingester.return-only-grpc-errors` to true, ingester will return only gRPC errors. This feature changes the following status codes: #6443 #6680 #6723 + * `http.StatusBadRequest` (400) is replaced with `codes.FailedPrecondition` on the write path. + * `http.StatusServiceUnavailable` (503) is replaced with `codes.Internal` on the write path, and with `codes.ResourceExhausted` on the read path. + * `codes.Unknown` are replaced with `codes.Internal` on both write and read path. * [CHANGE] Upgrade Node.js to v20. #6540 * [CHANGE] Querier: `cortex_querier_blocks_consistency_checks_failed_total` is now incremented when a block couldn't be queried from any attempted store-gateway as opposed to incremented after each attempt. Also `cortex_querier_blocks_consistency_checks_total` is incremented once per query as opposed to once per attempt (with 3 attempts). #6590 * [CHANGE] Ingester: Modify utilization based read path limiter to base memory usage on Go heap size. #6584 diff --git a/pkg/distributor/errors.go b/pkg/distributor/errors.go index 9230673334d..ca186b993e3 100644 --- a/pkg/distributor/errors.go +++ b/pkg/distributor/errors.go @@ -12,6 +12,7 @@ import ( "google.golang.org/grpc/codes" "github.com/grafana/mimir/pkg/mimirpb" + "github.com/grafana/mimir/pkg/util" "github.com/grafana/mimir/pkg/util/globalerror" "github.com/grafana/mimir/pkg/util/validation" ) @@ -239,7 +240,7 @@ func handleIngesterPushError(err error) error { return errors.Wrap(err, failedPushingToIngesterMessage) } statusCode := stat.Code() - if isHTTPStatusCode(statusCode) { + if util.IsHTTPStatusCode(statusCode) { // TODO This code is needed for backwards compatibility, since ingesters may still return // errors created by httpgrpc.Errorf(). If pushErr is one of those errors, we just propagate // it. This code should be removed in mimir 2.12.0. @@ -250,10 +251,6 @@ func handleIngesterPushError(err error) error { return newIngesterPushError(stat) } -func isHTTPStatusCode(statusCode codes.Code) bool { - return int(statusCode) >= 100 && int(statusCode) < 600 -} - func isClientError(err error) bool { var ingesterPushErr ingesterPushError if errors.As(err, &ingesterPushErr) { diff --git a/pkg/ingester/active_series.go b/pkg/ingester/active_series.go index b238643687e..35809ab3fe4 100644 --- a/pkg/ingester/active_series.go +++ b/pkg/ingester/active_series.go @@ -21,8 +21,9 @@ const activeSeriesMaxSizeBytes = 1 * 1024 * 1024 // ActiveSeries implements the ActiveSeries RPC. It returns a stream of active // series that match the given matchers. -func (i *Ingester) ActiveSeries(request *client.ActiveSeriesRequest, stream client.Ingester_ActiveSeriesServer) error { - if err := i.checkRunning(); err != nil { +func (i *Ingester) ActiveSeries(request *client.ActiveSeriesRequest, stream client.Ingester_ActiveSeriesServer) (asErr error) { + defer func() { asErr = i.handleReadError(asErr) }() + if err := i.checkAvailable(); err != nil { return err } if err := i.checkReadOverloaded(); err != nil { diff --git a/pkg/ingester/errors.go b/pkg/ingester/errors.go index 6fa369294a3..9d99d966d9e 100644 --- a/pkg/ingester/errors.go +++ b/pkg/ingester/errors.go @@ -28,6 +28,11 @@ import ( const ( integerUnavailableMsgFormat = "ingester is unavailable (current state: %s)" + tooBusyErrorMsg = "the ingester is currently too busy to process queries, try again later" +) + +var ( + tooBusyError = ingesterTooBusyError{} ) // errorWithStatus is used for wrapping errors returned by ingester. @@ -509,6 +514,19 @@ func (e tsdbUnavailableError) errorCause() mimirpb.ErrorCause { // Ensure that tsdbUnavailableError is an ingesterError. var _ ingesterError = tsdbUnavailableError{} +type ingesterTooBusyError struct{} + +func (e ingesterTooBusyError) Error() string { + return tooBusyErrorMsg +} + +func (e ingesterTooBusyError) errorCause() mimirpb.ErrorCause { + return mimirpb.TOO_BUSY +} + +// Ensure that ingesterTooBusyError is an ingesterError. +var _ ingesterError = ingesterTooBusyError{} + type ingesterErrSamplers struct { sampleTimestampTooOld *log.Sampler sampleTimestampTooOldOOOEnabled *log.Sampler @@ -535,7 +553,7 @@ func newIngesterErrSamplers(freq int64) ingesterErrSamplers { } } -func handlePushErrorWithGRPC(err error) error { +func handlePushError(err error) error { var ( ingesterErr ingesterError errCode = codes.Internal @@ -577,3 +595,38 @@ func handlePushErrorWithHTTPGRPC(err error) error { } return err } + +func handleReadError(err error) error { + var ( + ingesterErr ingesterError + errCode = codes.Internal + ) + if errors.As(err, &ingesterErr) { + switch ingesterErr.errorCause() { + case mimirpb.TOO_BUSY: + errCode = codes.ResourceExhausted + case mimirpb.SERVICE_UNAVAILABLE: + errCode = codes.Unavailable + } + } + return newErrorWithStatus(err, errCode) +} + +// handleReadErrorWithHTTPGRPC maps ingesterError objects to an appropriate +// errorWithStatus, which may contain both HTTP and gRPC error codes. +// TODO this method is needed only for the backwards compatibility, +// and should be removed in mimir 2.12.0. +func handleReadErrorWithHTTPGRPC(err error) error { + var ( + ingesterErr ingesterError + ) + if errors.As(err, &ingesterErr) { + switch ingesterErr.errorCause() { + case mimirpb.TOO_BUSY: + return newErrorWithHTTPStatus(err, http.StatusServiceUnavailable) + case mimirpb.SERVICE_UNAVAILABLE: + return newErrorWithStatus(err, codes.Unavailable) + } + } + return err +} diff --git a/pkg/ingester/errors_test.go b/pkg/ingester/errors_test.go index ebf83d40f7b..cc204530a63 100644 --- a/pkg/ingester/errors_test.go +++ b/pkg/ingester/errors_test.go @@ -237,6 +237,18 @@ func TestNewTSDBIngestExemplarErr(t *testing.T) { checkIngesterError(t, wrappedErr, mimirpb.BAD_DATA, true) } +func TestTooBusyError(t *testing.T) { + require.Error(t, tooBusyError) + require.Equal(t, tooBusyErrorMsg, tooBusyError.Error()) + checkIngesterError(t, tooBusyError, mimirpb.TOO_BUSY, false) + + wrappedErr := wrapOrAnnotateWithUser(tooBusyError, userID) + require.ErrorIs(t, wrappedErr, tooBusyError) + var anotherIngesterTooBusyErr ingesterTooBusyError + require.ErrorAs(t, wrappedErr, &anotherIngesterTooBusyErr) + checkIngesterError(t, wrappedErr, mimirpb.TOO_BUSY, false) +} + func TestErrorWithStatus(t *testing.T) { errMsg := "this is an error" ingesterErr := mockIngesterErr(errMsg) @@ -288,7 +300,7 @@ func TestErrorWithStatus(t *testing.T) { require.Equal(t, codes.Unimplemented, st.Code()) require.Equal(t, st.Message(), data.expectedErrorMessage) - // Ensure httpgrpc's HTTPResponseFromError recognizes errWithStatus. + // Ensure httpgrpc's HTTPResponseFromError doesn't recognize errWithStatus. resp, ok := httpgrpc.HTTPResponseFromError(errWithStatus) require.False(t, ok) require.Nil(t, resp) @@ -347,7 +359,7 @@ func TestWrapOrAnnotateWithUser(t *testing.T) { require.Equal(t, wrappingErr, errors.Unwrap(wrappedSafeErr)) } -func TestHandlePushErrorWithGRPC(t *testing.T) { +func TestHandlePushError(t *testing.T) { originalMsg := "this is an error" originalErr := errors.New(originalMsg) labelAdapters := []mimirpb.LabelAdapter{{Name: labels.MetricName, Value: "testmetric"}, {Name: "foo", Value: "biz"}} @@ -355,12 +367,11 @@ func TestHandlePushErrorWithGRPC(t *testing.T) { timestamp := model.Time(1) testCases := map[string]struct { - err error - doNotLogExpected bool - expectedCode codes.Code - expectedMessage string - expectedDetails *mimirpb.ErrorDetails - expectedTranslation error + err error + doNotLogExpected bool + expectedCode codes.Code + expectedMessage string + expectedDetails *mimirpb.ErrorDetails }{ "a generic error gets translated into an Internal gRPC error without details": { err: originalErr, @@ -381,7 +392,7 @@ func TestHandlePushErrorWithGRPC(t *testing.T) { expectedMessage: newUnavailableError(services.Stopping).Error(), expectedDetails: &mimirpb.ErrorDetails{Cause: mimirpb.SERVICE_UNAVAILABLE}, }, - "a wrapped unavailableError gets translated into a non-loggable errorWithStatus Unavailable error": { + "a wrapped unavailableError gets translated into an errorWithStatus Unavailable error": { err: fmt.Errorf("wrapped: %w", newUnavailableError(services.Stopping)), expectedCode: codes.Unavailable, expectedMessage: fmt.Sprintf("wrapped: %s", newUnavailableError(services.Stopping).Error()), @@ -394,7 +405,7 @@ func TestHandlePushErrorWithGRPC(t *testing.T) { expectedDetails: &mimirpb.ErrorDetails{Cause: mimirpb.INSTANCE_LIMIT}, doNotLogExpected: true, }, - "a wrapped instanceLimitReachedError gets translated into a non-loggable errorWithStatus Unavailable error with details": { + "a wrapped instanceLimitReachedError gets translated into an errorWithStatus Unavailable error with details": { err: fmt.Errorf("wrapped: %w", newInstanceLimitReachedError("instance limit reached")), expectedCode: codes.Unavailable, expectedMessage: fmt.Sprintf("wrapped: %s", newInstanceLimitReachedError("instance limit reached").Error()), @@ -407,7 +418,7 @@ func TestHandlePushErrorWithGRPC(t *testing.T) { expectedMessage: newTSDBUnavailableError("tsdb stopping").Error(), expectedDetails: &mimirpb.ErrorDetails{Cause: mimirpb.TSDB_UNAVAILABLE}, }, - "a wrapped tsdbUnavailableError gets translated into a non-loggable errorWithStatus Internal error with details": { + "a wrapped tsdbUnavailableError gets translated into an errorWithStatus Internal error with details": { err: fmt.Errorf("wrapped: %w", newTSDBUnavailableError("tsdb stopping")), expectedCode: codes.Internal, expectedMessage: fmt.Sprintf("wrapped: %s", newTSDBUnavailableError("tsdb stopping").Error()), @@ -419,7 +430,7 @@ func TestHandlePushErrorWithGRPC(t *testing.T) { expectedMessage: newSampleError("id", "sample error", timestamp, labelAdapters).Error(), expectedDetails: &mimirpb.ErrorDetails{Cause: mimirpb.BAD_DATA}, }, - "a wrapped sampleError gets translated into a non-loggable errorWithStatus FailedPrecondition error with details": { + "a wrapped sampleError gets translated into an errorWithStatus FailedPrecondition error with details": { err: fmt.Errorf("wrapped: %w", newSampleError("id", "sample error", timestamp, labelAdapters)), expectedCode: codes.FailedPrecondition, expectedMessage: fmt.Sprintf("wrapped: %s", newSampleError("id", "sample error", timestamp, labelAdapters).Error()), @@ -431,7 +442,7 @@ func TestHandlePushErrorWithGRPC(t *testing.T) { expectedMessage: newExemplarError("id", "exemplar error", timestamp, labelAdapters, labelAdapters).Error(), expectedDetails: &mimirpb.ErrorDetails{Cause: mimirpb.BAD_DATA}, }, - "a wrapped exemplarError gets translated into a non-loggable errorWithStatus FailedPrecondition error with details": { + "a wrapped exemplarError gets translated into an errorWithStatus FailedPrecondition error with details": { err: fmt.Errorf("wrapped: %w", newExemplarError("id", "exemplar error", timestamp, labelAdapters, labelAdapters)), expectedCode: codes.FailedPrecondition, expectedMessage: fmt.Sprintf("wrapped: %s", newExemplarError("id", "exemplar error", timestamp, labelAdapters, labelAdapters).Error()), @@ -443,7 +454,7 @@ func TestHandlePushErrorWithGRPC(t *testing.T) { expectedMessage: newTSDBIngestExemplarErr(originalErr, timestamp, labelAdapters, labelAdapters).Error(), expectedDetails: &mimirpb.ErrorDetails{Cause: mimirpb.BAD_DATA}, }, - "a wrapped tsdbIngestExemplarErr gets translated into a non-loggable errorWithStatus FailedPrecondition error with details": { + "a wrapped tsdbIngestExemplarErr gets translated into an errorWithStatus FailedPrecondition error with details": { err: fmt.Errorf("wrapped: %w", newTSDBIngestExemplarErr(originalErr, timestamp, labelAdapters, labelAdapters)), expectedCode: codes.FailedPrecondition, expectedMessage: fmt.Sprintf("wrapped: %s", newTSDBIngestExemplarErr(originalErr, timestamp, labelAdapters, labelAdapters).Error()), @@ -455,7 +466,7 @@ func TestHandlePushErrorWithGRPC(t *testing.T) { expectedMessage: newPerUserSeriesLimitReachedError(10).Error(), expectedDetails: &mimirpb.ErrorDetails{Cause: mimirpb.BAD_DATA}, }, - "a wrapped perUserSeriesLimitReachedError gets translated into a non-loggable errorWithStatus FailedPrecondition error with details": { + "a wrapped perUserSeriesLimitReachedError gets translated into an errorWithStatus FailedPrecondition error with details": { err: fmt.Errorf("wrapped: %w", newPerUserSeriesLimitReachedError(10)), expectedCode: codes.FailedPrecondition, expectedMessage: fmt.Sprintf("wrapped: %s", newPerUserSeriesLimitReachedError(10).Error()), @@ -467,7 +478,7 @@ func TestHandlePushErrorWithGRPC(t *testing.T) { expectedMessage: newPerUserMetadataLimitReachedError(10).Error(), expectedDetails: &mimirpb.ErrorDetails{Cause: mimirpb.BAD_DATA}, }, - "a wrapped perUserMetadataLimitReachedError gets translated into a non-loggable errorWithStatus FailedPrecondition error with details": { + "a wrapped perUserMetadataLimitReachedError gets translated into an errorWithStatus FailedPrecondition error with details": { err: fmt.Errorf("wrapped: %w", newPerUserMetadataLimitReachedError(10)), expectedCode: codes.FailedPrecondition, expectedMessage: fmt.Sprintf("wrapped: %s", newPerUserMetadataLimitReachedError(10).Error()), @@ -479,7 +490,7 @@ func TestHandlePushErrorWithGRPC(t *testing.T) { expectedMessage: newPerMetricSeriesLimitReachedError(10, labels).Error(), expectedDetails: &mimirpb.ErrorDetails{Cause: mimirpb.BAD_DATA}, }, - "a wrapped perMetricSeriesLimitReachedError gets translated into a non-loggable errorWithStatus FailedPrecondition error with details": { + "a wrapped perMetricSeriesLimitReachedError gets translated into an errorWithStatus FailedPrecondition error with details": { err: fmt.Errorf("wrapped: %w", newPerMetricSeriesLimitReachedError(10, labels)), expectedCode: codes.FailedPrecondition, expectedMessage: fmt.Sprintf("wrapped: %s", newPerMetricSeriesLimitReachedError(10, labels).Error()), @@ -491,7 +502,7 @@ func TestHandlePushErrorWithGRPC(t *testing.T) { expectedMessage: newPerMetricMetadataLimitReachedError(10, labels).Error(), expectedDetails: &mimirpb.ErrorDetails{Cause: mimirpb.BAD_DATA}, }, - "a wrapped perMetricMetadataLimitReachedError gets translated into a non-loggable errorWithStatus FailedPrecondition error with details": { + "a wrapped perMetricMetadataLimitReachedError gets translated into an errorWithStatus FailedPrecondition error with details": { err: fmt.Errorf("wrapped: %w", newPerMetricMetadataLimitReachedError(10, labels)), expectedCode: codes.FailedPrecondition, expectedMessage: fmt.Sprintf("wrapped: %s", newPerMetricMetadataLimitReachedError(10, labels).Error()), @@ -501,7 +512,7 @@ func TestHandlePushErrorWithGRPC(t *testing.T) { for name, tc := range testCases { t.Run(name, func(t *testing.T) { - handledErr := handlePushErrorWithGRPC(tc.err) + handledErr := handlePushError(tc.err) stat, ok := status.FromError(handledErr) require.True(t, ok) require.Equal(t, tc.expectedCode, stat.Code()) @@ -677,6 +688,96 @@ func TestHandlePushErrorWithHTTPGRPC(t *testing.T) { } } +func TestHandleReadError(t *testing.T) { + originalMsg := "this is an error" + originalErr := errors.New(originalMsg) + + testCases := map[string]struct { + err error + expectedCode codes.Code + expectedMessage string + expectedDetails *mimirpb.ErrorDetails + }{ + "a generic error gets translated into an Internal gRPC error without details": { + err: originalErr, + expectedCode: codes.Internal, + expectedMessage: originalMsg, + expectedDetails: nil, + }, + "an unavailableError gets translated into an errorWithStatus Unavailable error with details": { + err: newUnavailableError(services.Stopping), + expectedCode: codes.Unavailable, + expectedMessage: newUnavailableError(services.Stopping).Error(), + expectedDetails: &mimirpb.ErrorDetails{Cause: mimirpb.SERVICE_UNAVAILABLE}, + }, + "a wrapped unavailableError gets translated into an errorWithStatus Unavailable error": { + err: fmt.Errorf("wrapped: %w", newUnavailableError(services.Stopping)), + expectedCode: codes.Unavailable, + expectedMessage: fmt.Sprintf("wrapped: %s", newUnavailableError(services.Stopping).Error()), + expectedDetails: &mimirpb.ErrorDetails{Cause: mimirpb.SERVICE_UNAVAILABLE}, + }, + "tooBusyError gets translated into an errorWithStatus ResourceExhausted error with details": { + err: tooBusyError, + expectedCode: codes.ResourceExhausted, + expectedMessage: tooBusyErrorMsg, + expectedDetails: &mimirpb.ErrorDetails{Cause: mimirpb.TOO_BUSY}, + }, + "a wrapped tooBusyError gets translated into an errorWithStatus ResourceExhausted error with details": { + err: fmt.Errorf("wrapped: %w", tooBusyError), + expectedCode: codes.ResourceExhausted, + expectedMessage: fmt.Sprintf("wrapped: %s", tooBusyErrorMsg), + expectedDetails: &mimirpb.ErrorDetails{Cause: mimirpb.TOO_BUSY}, + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + handledErr := handleReadError(tc.err) + stat, ok := status.FromError(handledErr) + require.True(t, ok) + require.Equal(t, tc.expectedCode, stat.Code()) + require.Equal(t, tc.expectedMessage, stat.Message()) + checkErrorWithStatusDetails(t, stat.Details(), tc.expectedDetails) + }) + } +} + +func TestHandleReadErrorWithHTTPGRPC(t *testing.T) { + originalMsg := "this is an error" + originalErr := errors.New(originalMsg) + + testCases := map[string]struct { + err error + expectedTranslation error + }{ + "a generic error is not translated": { + err: originalErr, + expectedTranslation: originalErr, + }, + "an unavailableError gets translated into an errorWithStatus Unavailable error with details": { + err: newUnavailableError(services.Stopping), + expectedTranslation: newErrorWithStatus(newUnavailableError(services.Stopping), codes.Unavailable), + }, + "a wrapped unavailableError gets translated into an errorWithStatus Unavailable error": { + err: fmt.Errorf("wrapped: %w", newUnavailableError(services.Stopping)), + expectedTranslation: newErrorWithStatus(fmt.Errorf("wrapped: %w", newUnavailableError(services.Stopping)), codes.Unavailable), + }, + "tooBusyError gets translated into an errorWithHTTPStatus with status code 503": { + err: tooBusyError, + expectedTranslation: newErrorWithHTTPStatus(tooBusyError, http.StatusServiceUnavailable), + }, + "a wrapped tooBusyError gets translated into an errorWithStatus with status code 503": { + err: fmt.Errorf("wrapped: %w", tooBusyError), + expectedTranslation: newErrorWithHTTPStatus(fmt.Errorf("wrapped: %w", tooBusyError), http.StatusServiceUnavailable), + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + handledErr := handleReadErrorWithHTTPGRPC(tc.err) + require.Equal(t, tc.expectedTranslation, handledErr) + }) + } +} + type mockIngesterErr string func (e mockIngesterErr) Error() string { diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 29508b75e34..342af885949 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -24,7 +24,6 @@ import ( "github.com/go-kit/log" "github.com/go-kit/log/level" - "github.com/gogo/status" "github.com/grafana/dskit/concurrency" "github.com/grafana/dskit/middleware" "github.com/grafana/dskit/ring" @@ -50,7 +49,6 @@ import ( "go.uber.org/atomic" "golang.org/x/exp/slices" "golang.org/x/sync/errgroup" - "google.golang.org/grpc/codes" "github.com/grafana/mimir/pkg/ingester/activeseries" "github.com/grafana/mimir/pkg/ingester/client" @@ -116,7 +114,6 @@ const ( // Value used to track the limit between sequential and concurrent TSDB opernings. // Below this value, TSDBs of different tenants are opened sequentially, otherwise concurrently. maxTSDBOpenWithoutConcurrency = 10 - tooBusyErrorMsg = "the ingester is currently too busy to process queries, try again later" ) var ( @@ -125,11 +122,6 @@ var ( reasonIngesterMaxInMemorySeries = globalerror.IngesterMaxInMemorySeries.LabelValue() reasonIngesterMaxInflightPushRequests = globalerror.IngesterMaxInflightPushRequests.LabelValue() reasonIngesterMaxInflightPushRequestsBytes = globalerror.IngesterMaxInflightPushRequestsBytes.LabelValue() - // This is the closest fitting Prometheus API error code for requests rejected due to limiting. - tooBusyError = newErrorWithHTTPStatus( - errors.New(tooBusyErrorMsg), - http.StatusServiceUnavailable, - ) ) // BlocksUploader interface is used to have an easy way to mock it in tests. @@ -1278,8 +1270,9 @@ func (i *Ingester) pushSamplesToAppender(userID string, timeseries []mimirpb.Pre return nil } -func (i *Ingester) QueryExemplars(ctx context.Context, req *client.ExemplarQueryRequest) (*client.ExemplarQueryResponse, error) { - if err := i.checkRunning(); err != nil { +func (i *Ingester) QueryExemplars(ctx context.Context, req *client.ExemplarQueryRequest) (resp *client.ExemplarQueryResponse, qeErr error) { + defer func() { qeErr = i.handleReadError(qeErr) }() + if err := i.checkAvailable(); err != nil { return nil, err } if err := i.checkReadOverloaded(); err != nil { @@ -1335,8 +1328,9 @@ func (i *Ingester) QueryExemplars(ctx context.Context, req *client.ExemplarQuery return result, nil } -func (i *Ingester) LabelValues(ctx context.Context, req *client.LabelValuesRequest) (*client.LabelValuesResponse, error) { - if err := i.checkRunning(); err != nil { +func (i *Ingester) LabelValues(ctx context.Context, req *client.LabelValuesRequest) (resp *client.LabelValuesResponse, lvErr error) { + defer func() { lvErr = i.handleReadError(lvErr) }() + if err := i.checkAvailable(); err != nil { return nil, err } if err := i.checkReadOverloaded(); err != nil { @@ -1374,8 +1368,9 @@ func (i *Ingester) LabelValues(ctx context.Context, req *client.LabelValuesReque }, nil } -func (i *Ingester) LabelNames(ctx context.Context, req *client.LabelNamesRequest) (*client.LabelNamesResponse, error) { - if err := i.checkRunning(); err != nil { +func (i *Ingester) LabelNames(ctx context.Context, req *client.LabelNamesRequest) (resp *client.LabelNamesResponse, lnErr error) { + defer func() { lnErr = i.handleReadError(lnErr) }() + if err := i.checkAvailable(); err != nil { return nil, err } if err := i.checkReadOverloaded(); err != nil { @@ -1414,8 +1409,9 @@ func (i *Ingester) LabelNames(ctx context.Context, req *client.LabelNamesRequest } // MetricsForLabelMatchers implements IngesterServer. -func (i *Ingester) MetricsForLabelMatchers(ctx context.Context, req *client.MetricsForLabelMatchersRequest) (*client.MetricsForLabelMatchersResponse, error) { - if err := i.checkRunning(); err != nil { +func (i *Ingester) MetricsForLabelMatchers(ctx context.Context, req *client.MetricsForLabelMatchersRequest) (resp *client.MetricsForLabelMatchersResponse, mlErr error) { + defer func() { mlErr = i.handleReadError(mlErr) }() + if err := i.checkAvailable(); err != nil { return nil, err } if err := i.checkReadOverloaded(); err != nil { @@ -1484,8 +1480,9 @@ func (i *Ingester) MetricsForLabelMatchers(ctx context.Context, req *client.Metr return result, nil } -func (i *Ingester) UserStats(ctx context.Context, req *client.UserStatsRequest) (*client.UserStatsResponse, error) { - if err := i.checkRunning(); err != nil { +func (i *Ingester) UserStats(ctx context.Context, req *client.UserStatsRequest) (resp *client.UserStatsResponse, usErr error) { + defer func() { usErr = i.handleReadError(usErr) }() + if err := i.checkAvailable(); err != nil { return nil, err } if err := i.checkReadOverloaded(); err != nil { @@ -1505,8 +1502,9 @@ func (i *Ingester) UserStats(ctx context.Context, req *client.UserStatsRequest) return createUserStats(db, req) } -func (i *Ingester) AllUserStats(_ context.Context, req *client.UserStatsRequest) (*client.UsersStatsResponse, error) { - if err := i.checkRunning(); err != nil { +func (i *Ingester) AllUserStats(_ context.Context, req *client.UserStatsRequest) (resp *client.UsersStatsResponse, auErr error) { + defer func() { auErr = i.handleReadError(auErr) }() + if err := i.checkAvailable(); err != nil { return nil, err } @@ -1535,8 +1533,9 @@ func (i *Ingester) AllUserStats(_ context.Context, req *client.UserStatsRequest) // So, 1 MB limit will prevent reaching the limit and won't affect performance significantly. const labelNamesAndValuesTargetSizeBytes = 1 * 1024 * 1024 -func (i *Ingester) LabelNamesAndValues(request *client.LabelNamesAndValuesRequest, stream client.Ingester_LabelNamesAndValuesServer) error { - if err := i.checkRunning(); err != nil { +func (i *Ingester) LabelNamesAndValues(request *client.LabelNamesAndValuesRequest, stream client.Ingester_LabelNamesAndValuesServer) (lnvErr error) { + defer func() { lnvErr = i.handleReadError(lnvErr) }() + if err := i.checkAvailable(); err != nil { return err } if err := i.checkReadOverloaded(); err != nil { @@ -1567,8 +1566,9 @@ func (i *Ingester) LabelNamesAndValues(request *client.LabelNamesAndValuesReques // We arbitrarily set it to 1mb to avoid reaching the actual gRPC default limit (4mb). const labelValuesCardinalityTargetSizeBytes = 1 * 1024 * 1024 -func (i *Ingester) LabelValuesCardinality(req *client.LabelValuesCardinalityRequest, srv client.Ingester_LabelValuesCardinalityServer) error { - if err := i.checkRunning(); err != nil { +func (i *Ingester) LabelValuesCardinality(req *client.LabelValuesCardinalityRequest, srv client.Ingester_LabelValuesCardinalityServer) (lvcErr error) { + defer func() { lvcErr = i.handleReadError(lvcErr) }() + if err := i.checkAvailable(); err != nil { return err } if err := i.checkReadOverloaded(); err != nil { @@ -1647,8 +1647,9 @@ func createUserStats(db *userTSDB, req *client.UserStatsRequest) (*client.UserSt const queryStreamBatchMessageSize = 1 * 1024 * 1024 // QueryStream streams metrics from a TSDB. This implements the client.IngesterServer interface -func (i *Ingester) QueryStream(req *client.QueryRequest, stream client.Ingester_QueryStreamServer) error { - if err := i.checkRunning(); err != nil { +func (i *Ingester) QueryStream(req *client.QueryRequest, stream client.Ingester_QueryStreamServer) (qsErr error) { + defer func() { qsErr = i.handleReadError(qsErr) }() + if err := i.checkAvailable(); err != nil { return err } if err := i.checkReadOverloaded(); err != nil { @@ -3165,18 +3166,6 @@ func (i *Ingester) checkAvailable() error { return newUnavailableError(s) } -// TODO checkRunning should be removed once the error handling improvement is completed. -// TODO return gRPC errors only at the topmost level. -// Using block store, the ingester is only available when it is in a Running state. The ingester is not available -// when stopping to prevent any read or writes to the TSDB after the ingester has closed them. -func (i *Ingester) checkRunning() error { - s := i.State() - if s == services.Running { - return nil - } - return status.Error(codes.Unavailable, s.String()) -} - // Push implements client.IngesterServer func (i *Ingester) Push(ctx context.Context, req *mimirpb.WriteRequest) (*mimirpb.WriteResponse, error) { err := i.PushWithCleanup(ctx, req, func() { mimirpb.ReuseSlice(req.Timeseries) }) @@ -3189,11 +3178,30 @@ func (i *Ingester) Push(ctx context.Context, req *mimirpb.WriteRequest) (*mimirp func (i *Ingester) handlePushError(err error) error { if i.cfg.ReturnOnlyGRPCErrors { - return handlePushErrorWithGRPC(err) + return handlePushError(err) } return handlePushErrorWithHTTPGRPC(err) } +func (i *Ingester) handleReadError(err error) error { + if err == nil { + return nil + } + + if errors.Is(err, context.Canceled) { + return err + } + + if errors.Is(err, context.DeadlineExceeded) { + return err + } + + if i.cfg.ReturnOnlyGRPCErrors { + return handleReadError(err) + } + return handleReadErrorWithHTTPGRPC(err) +} + // pushMetadata returns number of ingested metadata. func (i *Ingester) pushMetadata(ctx context.Context, userID string, metadata []*mimirpb.MetricMetadata) int { ingestedMetadata := 0 @@ -3293,8 +3301,9 @@ func (i *Ingester) purgeUserMetricsMetadata() { } // MetricsMetadata returns all the metrics metadata of a user. -func (i *Ingester) MetricsMetadata(ctx context.Context, req *client.MetricsMetadataRequest) (*client.MetricsMetadataResponse, error) { - if err := i.checkRunning(); err != nil { +func (i *Ingester) MetricsMetadata(ctx context.Context, req *client.MetricsMetadataRequest) (resp *client.MetricsMetadataResponse, mmErr error) { + defer func() { mmErr = i.handleReadError(mmErr) }() + if err := i.checkAvailable(); err != nil { return nil, err } @@ -3315,7 +3324,7 @@ func (i *Ingester) MetricsMetadata(ctx context.Context, req *client.MetricsMetad // CheckReady is the readiness handler used to indicate to k8s when the ingesters // are ready for the addition or removal of another ingester. func (i *Ingester) CheckReady(ctx context.Context) error { - if err := i.checkRunning(); err != nil { + if err := i.checkAvailable(); err != nil { return fmt.Errorf("ingester not ready: %v", err) } return i.lifecycler.CheckReady(ctx) diff --git a/pkg/mimirpb/mimir.proto b/pkg/mimirpb/mimir.proto index 22389d9ebbd..1ae86f02e3d 100644 --- a/pkg/mimirpb/mimir.proto +++ b/pkg/mimirpb/mimir.proto @@ -44,6 +44,7 @@ enum ErrorCause { INSTANCE_LIMIT = 6; SERVICE_UNAVAILABLE = 7; TSDB_UNAVAILABLE = 8; + TOO_BUSY = 9; } message ErrorDetails { diff --git a/pkg/querier/error_translate_queryable.go b/pkg/querier/error_translate_queryable.go index 2e55e118ee4..ee045e34f42 100644 --- a/pkg/querier/error_translate_queryable.go +++ b/pkg/querier/error_translate_queryable.go @@ -9,13 +9,15 @@ import ( "context" "net/http" - "github.com/gogo/status" + "github.com/grafana/dskit/grpcutil" "github.com/pkg/errors" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/promql" "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/util/annotations" + "github.com/grafana/mimir/pkg/mimirpb" + "github.com/grafana/mimir/pkg/util" "github.com/grafana/mimir/pkg/util/validation" ) @@ -51,20 +53,27 @@ func TranslateToPromqlAPIError(err error) error { return err // 499 } - s, ok := status.FromError(err) - if !ok { - s, ok = status.FromError(errors.Cause(err)) - } - - if ok { + if s, ok := grpcutil.ErrorToStatus(err); ok { code := s.Code() - // Treat these as HTTP status codes, even though they are supposed to be grpc codes. - if code >= 400 && code < 500 { - // Return directly, will be mapped to 422 - return err - } else if code == http.StatusServiceUnavailable { - return promql.ErrQueryTimeout(s.Message()) + if util.IsHTTPStatusCode(code) { + // Treat these as HTTP status codes, even though they are supposed to be grpc codes. + if code >= 400 && code < 500 { + // Return directly, will be mapped to 422 + return err + } else if code == http.StatusServiceUnavailable { + return promql.ErrQueryTimeout(s.Message()) + } + } + + details := s.Details() + if len(details) == 1 { + if errorDetails, ok := details[0].(*mimirpb.WriteErrorDetails); ok { + switch errorDetails.GetCause() { + case mimirpb.TOO_BUSY: + return promql.ErrQueryTimeout(s.Message()) + } + } } } diff --git a/pkg/util/http.go b/pkg/util/http.go index 0797fea08b0..e5a828d2cc5 100644 --- a/pkg/util/http.go +++ b/pkg/util/http.go @@ -24,6 +24,7 @@ import ( "github.com/grafana/dskit/flagext" "github.com/opentracing/opentracing-go" otlog "github.com/opentracing/opentracing-go/log" + "google.golang.org/grpc/codes" "gopkg.in/yaml.v3" ) @@ -342,3 +343,9 @@ func copyValues(src url.Values) url.Values { } return dst } + +// IsHTTPStatusCode returns true if the given code is a valid HTTP status code, or false otherwise. +func IsHTTPStatusCode(code codes.Code) bool { + httpStatus := http.StatusText(int(code)) + return httpStatus != "" +} From de22b89a736ae62f27a216f2658241129b1175b4 Mon Sep 17 00:00:00 2001 From: Yuri Nikolic Date: Thu, 23 Nov 2023 18:44:00 +0100 Subject: [PATCH 2/7] Allow ingester's read path to return gRPC errors Signed-off-by: Yuri Nikolic --- pkg/mimirpb/mimir.pb.go | 248 ++++++++++++++++++++-------------------- 1 file changed, 126 insertions(+), 122 deletions(-) diff --git a/pkg/mimirpb/mimir.pb.go b/pkg/mimirpb/mimir.pb.go index 468e4f845e3..4eba2ddb262 100644 --- a/pkg/mimirpb/mimir.pb.go +++ b/pkg/mimirpb/mimir.pb.go @@ -41,6 +41,7 @@ const ( INSTANCE_LIMIT ErrorCause = 6 SERVICE_UNAVAILABLE ErrorCause = 7 TSDB_UNAVAILABLE ErrorCause = 8 + TOO_BUSY ErrorCause = 9 ) var ErrorCause_name = map[int32]string{ @@ -53,6 +54,7 @@ var ErrorCause_name = map[int32]string{ 6: "INSTANCE_LIMIT", 7: "SERVICE_UNAVAILABLE", 8: "TSDB_UNAVAILABLE", + 9: "TOO_BUSY", } var ErrorCause_value = map[string]int32{ @@ -65,6 +67,7 @@ var ErrorCause_value = map[string]int32{ "INSTANCE_LIMIT": 6, "SERVICE_UNAVAILABLE": 7, "TSDB_UNAVAILABLE": 8, + "TOO_BUSY": 9, } func (ErrorCause) EnumDescriptor() ([]byte, []int) { @@ -1882,128 +1885,129 @@ func init() { func init() { proto.RegisterFile("mimir.proto", fileDescriptor_86d4d7485f544059) } var fileDescriptor_86d4d7485f544059 = []byte{ - // 1930 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0xcf, 0x73, 0xdc, 0x58, - 0x11, 0x1e, 0xcd, 0x68, 0x7e, 0xa8, 0x3d, 0x63, 0x2b, 0x2f, 0x26, 0x3b, 0xeb, 0xda, 0x8c, 0x1d, - 0x51, 0x2c, 0x26, 0x05, 0x0e, 0xb5, 0x0b, 0xd9, 0xda, 0x54, 0x28, 0xd0, 0xcc, 0x28, 0xf1, 0x78, - 0x3d, 0x33, 0xde, 0x27, 0x4d, 0x96, 0x70, 0x51, 0xc9, 0xe3, 0x67, 0x5b, 0xb5, 0xa3, 0xd1, 0x20, - 0x69, 0xb2, 0x31, 0x27, 0x2e, 0x50, 0x14, 0x27, 0x2e, 0x5c, 0x28, 0x6e, 0x5c, 0xf8, 0x0b, 0xf8, - 0x07, 0xb8, 0xa4, 0x8a, 0xa2, 0x2a, 0xc7, 0x85, 0x43, 0x8a, 0x38, 0x97, 0x3d, 0xee, 0x81, 0x13, - 0x27, 0xea, 0xf5, 0xd3, 0x8f, 0xd1, 0xd8, 0x86, 0xc0, 0xe6, 0xa6, 0xee, 0xfe, 0x5e, 0xeb, 0x53, - 0xbf, 0xaf, 0x9f, 0x5a, 0x82, 0x15, 0xcf, 0xf5, 0xdc, 0x60, 0x67, 0x16, 0xf8, 0x91, 0x4f, 0x6a, - 0x63, 0x3f, 0x88, 0xd8, 0xd3, 0xd9, 0xe1, 0xc6, 0x77, 0x4e, 0xdc, 0xe8, 0x74, 0x7e, 0xb8, 0x33, - 0xf6, 0xbd, 0x3b, 0x27, 0xfe, 0x89, 0x7f, 0x07, 0x01, 0x87, 0xf3, 0x63, 0xb4, 0xd0, 0xc0, 0x2b, - 0xb1, 0x50, 0xfb, 0x53, 0x11, 0xea, 0x9f, 0x04, 0x6e, 0xc4, 0x28, 0xfb, 0xe9, 0x9c, 0x85, 0x11, - 0x39, 0x00, 0x88, 0x5c, 0x8f, 0x85, 0x2c, 0x70, 0x59, 0xd8, 0x94, 0xb6, 0x4a, 0xdb, 0x2b, 0xef, - 0xad, 0xef, 0x24, 0xe9, 0x77, 0x2c, 0xd7, 0x63, 0x26, 0xc6, 0xda, 0x1b, 0xcf, 0x5e, 0x6c, 0x16, - 0xfe, 0xfe, 0x62, 0x93, 0x1c, 0x04, 0xcc, 0x99, 0x4c, 0xfc, 0xb1, 0x95, 0xae, 0xa3, 0x0b, 0x39, - 0xc8, 0x87, 0x50, 0x31, 0xfd, 0x79, 0x30, 0x66, 0xcd, 0xe2, 0x96, 0xb4, 0xbd, 0xfa, 0xde, 0xad, - 0x2c, 0xdb, 0xe2, 0x9d, 0x77, 0x04, 0xc8, 0x98, 0xce, 0x3d, 0x1a, 0x2f, 0x20, 0xf7, 0xa0, 0xe6, - 0xb1, 0xc8, 0x39, 0x72, 0x22, 0xa7, 0x59, 0x42, 0x2a, 0xcd, 0x6c, 0x71, 0x9f, 0x45, 0x81, 0x3b, - 0xee, 0xc7, 0xf1, 0xb6, 0xfc, 0xec, 0xc5, 0xa6, 0x44, 0x53, 0x3c, 0xb9, 0x0f, 0x1b, 0xe1, 0xa7, - 0xee, 0xcc, 0x9e, 0x38, 0x87, 0x6c, 0x62, 0x4f, 0x1d, 0x8f, 0xd9, 0x4f, 0x9c, 0x89, 0x7b, 0xe4, - 0x44, 0xae, 0x3f, 0x6d, 0x7e, 0x51, 0xdd, 0x92, 0xb6, 0x6b, 0xf4, 0x2d, 0x0e, 0xd9, 0xe7, 0x88, - 0x81, 0xe3, 0xb1, 0x47, 0x69, 0x5c, 0xdb, 0x04, 0xc8, 0xf8, 0x90, 0x2a, 0x94, 0xf4, 0x83, 0x9e, - 0x5a, 0x20, 0x35, 0x90, 0xe9, 0x68, 0xdf, 0x50, 0x25, 0x6d, 0x0d, 0x1a, 0x31, 0xfb, 0x70, 0xe6, - 0x4f, 0x43, 0xa6, 0xdd, 0x83, 0xba, 0x11, 0x04, 0x7e, 0xd0, 0x65, 0x91, 0xe3, 0x4e, 0x42, 0x72, - 0x1b, 0xca, 0x1d, 0x67, 0x1e, 0xb2, 0xa6, 0x84, 0x4f, 0xbd, 0x50, 0x43, 0x84, 0x61, 0x8c, 0x0a, - 0x88, 0xf6, 0x4f, 0x09, 0x20, 0xab, 0x2c, 0xd1, 0xa1, 0x82, 0xac, 0x93, 0xfa, 0x5f, 0xcf, 0xd6, - 0x22, 0xd7, 0x03, 0xc7, 0x0d, 0xda, 0xeb, 0x71, 0xf9, 0xeb, 0xe8, 0xd2, 0x8f, 0x9c, 0x59, 0xc4, - 0x02, 0x1a, 0x2f, 0x24, 0xdf, 0x85, 0x6a, 0xe8, 0x78, 0xb3, 0x09, 0x0b, 0x9b, 0x45, 0xcc, 0xa1, - 0x66, 0x39, 0x4c, 0x0c, 0x60, 0xc1, 0x0a, 0x34, 0x81, 0x91, 0xbb, 0xa0, 0xb0, 0xa7, 0xcc, 0x9b, - 0x4d, 0x9c, 0x20, 0x8c, 0x8b, 0x4d, 0x16, 0x38, 0xc7, 0xa1, 0x78, 0x55, 0x06, 0x25, 0x1f, 0x02, - 0x9c, 0xba, 0x61, 0xe4, 0x9f, 0x04, 0x8e, 0x17, 0x36, 0xe5, 0x65, 0xc2, 0xbb, 0x49, 0x2c, 0x5e, - 0xb9, 0x00, 0xd6, 0xbe, 0x0f, 0x4a, 0xfa, 0x3c, 0x84, 0x80, 0xcc, 0x37, 0x09, 0xcb, 0x55, 0xa7, - 0x78, 0x4d, 0xd6, 0xa1, 0xfc, 0xc4, 0x99, 0xcc, 0x85, 0x72, 0xea, 0x54, 0x18, 0x9a, 0x0e, 0x15, - 0xf1, 0x08, 0xe4, 0x16, 0xd4, 0x51, 0x68, 0x91, 0xe3, 0xcd, 0x6c, 0x2f, 0x44, 0x58, 0x89, 0xae, - 0xa4, 0xbe, 0x7e, 0x98, 0xa5, 0xe0, 0x79, 0xa5, 0x24, 0xc5, 0xef, 0x8a, 0xb0, 0x9a, 0xd7, 0x0f, - 0xf9, 0x00, 0xe4, 0xe8, 0x6c, 0x96, 0x6c, 0xd7, 0xd7, 0xaf, 0xd2, 0x59, 0x6c, 0x5a, 0x67, 0x33, - 0x46, 0x71, 0x01, 0xf9, 0x36, 0x10, 0x0f, 0x7d, 0xf6, 0xb1, 0xe3, 0xb9, 0x93, 0x33, 0xd4, 0x1a, - 0x52, 0x51, 0xa8, 0x2a, 0x22, 0x0f, 0x30, 0xc0, 0x25, 0xc6, 0x1f, 0xf3, 0x94, 0x4d, 0x66, 0x4d, - 0x19, 0xe3, 0x78, 0xcd, 0x7d, 0xf3, 0xa9, 0x1b, 0x35, 0xcb, 0xc2, 0xc7, 0xaf, 0xb5, 0x33, 0x80, - 0xec, 0x4e, 0x64, 0x05, 0xaa, 0xa3, 0xc1, 0x47, 0x83, 0xe1, 0x27, 0x03, 0xb5, 0xc0, 0x8d, 0xce, - 0x70, 0x34, 0xb0, 0x0c, 0xaa, 0x4a, 0x44, 0x81, 0xf2, 0x43, 0x7d, 0xf4, 0xd0, 0x50, 0x8b, 0xa4, - 0x01, 0xca, 0x6e, 0xcf, 0xb4, 0x86, 0x0f, 0xa9, 0xde, 0x57, 0x4b, 0x84, 0xc0, 0x2a, 0x46, 0x32, - 0x9f, 0xcc, 0x97, 0x9a, 0xa3, 0x7e, 0x5f, 0xa7, 0x8f, 0xd5, 0x32, 0x17, 0x73, 0x6f, 0xf0, 0x60, - 0xa8, 0x56, 0x48, 0x1d, 0x6a, 0xa6, 0xa5, 0x5b, 0x86, 0x69, 0x58, 0x6a, 0x55, 0xfb, 0x08, 0x2a, - 0xe2, 0xd6, 0x6f, 0x40, 0x88, 0xda, 0x2f, 0x25, 0xa8, 0x25, 0xe2, 0x79, 0x13, 0xc2, 0xce, 0x49, - 0x22, 0xd9, 0xcf, 0x0b, 0x42, 0x28, 0x5d, 0x10, 0x82, 0xf6, 0x97, 0x32, 0x28, 0xa9, 0x18, 0xc9, - 0x4d, 0x50, 0xc6, 0xfe, 0x7c, 0x1a, 0xd9, 0xee, 0x34, 0xc2, 0x2d, 0x97, 0x77, 0x0b, 0xb4, 0x86, - 0xae, 0xde, 0x34, 0x22, 0xb7, 0x60, 0x45, 0x84, 0x8f, 0x27, 0xbe, 0x13, 0x89, 0x7b, 0xed, 0x16, - 0x28, 0xa0, 0xf3, 0x01, 0xf7, 0x11, 0x15, 0x4a, 0xe1, 0xdc, 0xc3, 0x3b, 0x49, 0x94, 0x5f, 0x92, - 0x1b, 0x50, 0x09, 0xc7, 0xa7, 0xcc, 0x73, 0x70, 0x73, 0xaf, 0xd1, 0xd8, 0x22, 0xdf, 0x80, 0xd5, - 0x9f, 0xb1, 0xc0, 0xb7, 0xa3, 0xd3, 0x80, 0x85, 0xa7, 0xfe, 0xe4, 0x08, 0x37, 0x5a, 0xa2, 0x0d, - 0xee, 0xb5, 0x12, 0x27, 0x79, 0x37, 0x86, 0x65, 0xbc, 0x2a, 0xc8, 0x4b, 0xa2, 0x75, 0xee, 0xef, - 0x24, 0xdc, 0x6e, 0x83, 0xba, 0x80, 0x13, 0x04, 0xab, 0x48, 0x50, 0xa2, 0xab, 0x29, 0x52, 0x90, - 0xd4, 0x61, 0x75, 0xca, 0x4e, 0x9c, 0xc8, 0x7d, 0xc2, 0xec, 0x70, 0xe6, 0x4c, 0xc3, 0x66, 0x6d, - 0xf9, 0x44, 0x6f, 0xcf, 0xc7, 0x9f, 0xb2, 0xc8, 0x9c, 0x39, 0xd3, 0xb8, 0x43, 0x1b, 0xc9, 0x0a, - 0xee, 0x0b, 0xc9, 0x37, 0x61, 0x2d, 0x4d, 0x71, 0xc4, 0x26, 0x91, 0x13, 0x36, 0x95, 0xad, 0xd2, - 0x36, 0xa1, 0x69, 0xe6, 0x2e, 0x7a, 0x73, 0x40, 0xe4, 0x16, 0x36, 0x61, 0xab, 0xb4, 0x2d, 0x65, - 0x40, 0x24, 0xc6, 0x8f, 0xb7, 0xd5, 0x99, 0x1f, 0xba, 0x0b, 0xa4, 0x56, 0xfe, 0x3b, 0xa9, 0x64, - 0x45, 0x4a, 0x2a, 0x4d, 0x11, 0x93, 0xaa, 0x0b, 0x52, 0x89, 0x3b, 0x23, 0x95, 0x02, 0x63, 0x52, - 0x0d, 0x41, 0x2a, 0x71, 0xc7, 0xa4, 0xee, 0x03, 0x04, 0x2c, 0x64, 0x91, 0x7d, 0xca, 0x2b, 0xbf, - 0x8a, 0x87, 0xc0, 0xcd, 0x4b, 0x8e, 0xb1, 0x1d, 0xca, 0x51, 0xbb, 0xee, 0x34, 0xa2, 0x4a, 0x90, - 0x5c, 0x92, 0x77, 0x40, 0x49, 0xb5, 0xd6, 0x5c, 0x43, 0xf1, 0x65, 0x0e, 0xed, 0x1e, 0x28, 0xe9, - 0xaa, 0x7c, 0x2b, 0x57, 0xa1, 0xf4, 0xd8, 0x30, 0x55, 0x89, 0x54, 0xa0, 0x38, 0x18, 0xaa, 0xc5, - 0xac, 0x9d, 0x4b, 0x1b, 0xf2, 0xaf, 0xfe, 0xd0, 0x92, 0xda, 0x55, 0x28, 0x23, 0xef, 0x76, 0x1d, - 0x20, 0xdb, 0x76, 0xed, 0xaf, 0x32, 0xac, 0xe2, 0x16, 0x67, 0x92, 0x0e, 0x81, 0x60, 0x8c, 0x05, - 0xf6, 0xd2, 0x93, 0x34, 0xda, 0xc6, 0xbf, 0x5e, 0x6c, 0xea, 0x0b, 0x93, 0xc1, 0x2c, 0xf0, 0x3d, - 0x16, 0x9d, 0xb2, 0x79, 0xb8, 0x78, 0xe9, 0xf9, 0x47, 0x6c, 0x72, 0x27, 0x3d, 0xa0, 0x77, 0x3a, - 0x22, 0x5d, 0xf6, 0xc4, 0xea, 0x78, 0xc9, 0xf3, 0x55, 0x35, 0x7f, 0x73, 0xf1, 0xa1, 0x84, 0x8a, - 0xa9, 0x92, 0x6a, 0x98, 0x37, 0xbb, 0x88, 0xc4, 0xcd, 0x8e, 0xc6, 0x25, 0x9d, 0xf7, 0x06, 0x14, - 0xf5, 0x06, 0x3a, 0xe5, 0x5b, 0xa0, 0xa6, 0x2c, 0x0e, 0x11, 0x9b, 0x88, 0x2d, 0xd5, 0xa0, 0x48, - 0x81, 0xd0, 0xf4, 0x6e, 0x09, 0x54, 0x34, 0x4b, 0xda, 0x43, 0x31, 0x74, 0x4f, 0xae, 0x49, 0x6a, - 0x71, 0x4f, 0xae, 0x55, 0xd4, 0xea, 0x9e, 0x5c, 0x53, 0x54, 0xd8, 0x93, 0x6b, 0x75, 0xb5, 0xb1, - 0x27, 0xd7, 0xd6, 0x54, 0x95, 0x66, 0xa7, 0x18, 0x5d, 0x3a, 0x3d, 0xe8, 0x72, 0xdb, 0xd2, 0xe5, - 0x96, 0x59, 0x94, 0xe8, 0x7d, 0x80, 0xec, 0xf1, 0xf8, 0xae, 0xfa, 0xc7, 0xc7, 0x21, 0x13, 0x47, - 0xe3, 0x35, 0x1a, 0x5b, 0xdc, 0x3f, 0x61, 0xd3, 0x93, 0xe8, 0x14, 0x37, 0xa4, 0x41, 0x63, 0x4b, - 0x9b, 0x03, 0xc9, 0x8b, 0x11, 0xdf, 0xe8, 0xaf, 0xf1, 0x76, 0xbe, 0x0f, 0x4a, 0x2a, 0x37, 0xbc, - 0x57, 0x6e, 0xc2, 0xcb, 0xe7, 0x8c, 0x27, 0xbc, 0x6c, 0x81, 0x36, 0x85, 0x35, 0x31, 0x08, 0x64, - 0x4d, 0x90, 0x2a, 0x46, 0xba, 0x44, 0x31, 0xc5, 0x4c, 0x31, 0xef, 0x43, 0x35, 0xa9, 0xbb, 0x98, - 0x75, 0xde, 0xbe, 0x6c, 0x64, 0x41, 0x04, 0x4d, 0x90, 0x5a, 0x08, 0x6b, 0x4b, 0x31, 0xd2, 0x02, - 0x38, 0xf4, 0xe7, 0xd3, 0x23, 0x27, 0x1e, 0x97, 0xa5, 0xed, 0x32, 0x5d, 0xf0, 0x70, 0x3e, 0x13, - 0xff, 0x33, 0x16, 0x24, 0x0a, 0x46, 0x83, 0x7b, 0xe7, 0xb3, 0x19, 0x0b, 0x62, 0x0d, 0x0b, 0x23, - 0xe3, 0x2e, 0x2f, 0x70, 0xd7, 0x26, 0x70, 0x7d, 0xe9, 0x21, 0xb1, 0xb8, 0xb9, 0x13, 0xa7, 0xb8, - 0x74, 0xe2, 0x90, 0x0f, 0x2e, 0xd6, 0xf5, 0xed, 0xe5, 0x01, 0x30, 0xcd, 0xb7, 0x58, 0xd2, 0x3f, - 0xcb, 0xd0, 0xf8, 0x78, 0xce, 0x82, 0xb3, 0x64, 0xae, 0x25, 0x77, 0xa1, 0x12, 0x46, 0x4e, 0x34, - 0x0f, 0xe3, 0xc9, 0xa8, 0x95, 0xe5, 0xc9, 0x01, 0x77, 0x4c, 0x44, 0xd1, 0x18, 0x4d, 0x7e, 0x04, - 0xc0, 0xf8, 0xa0, 0x6b, 0xe3, 0x54, 0x75, 0x61, 0xf4, 0xcf, 0xaf, 0xc5, 0x91, 0x18, 0x67, 0x2a, - 0x85, 0x25, 0x97, 0xbc, 0x1e, 0x68, 0x60, 0x95, 0x14, 0x2a, 0x0c, 0xb2, 0xc3, 0xf9, 0x04, 0xee, - 0xf4, 0x04, 0xcb, 0x94, 0x6b, 0x50, 0x13, 0xfd, 0x5d, 0x27, 0x72, 0x76, 0x0b, 0x34, 0x46, 0x71, - 0xfc, 0x13, 0x36, 0x8e, 0xfc, 0x00, 0x4f, 0xa0, 0x1c, 0xfe, 0x11, 0xfa, 0x13, 0xbc, 0x40, 0x61, - 0xfe, 0xb1, 0x33, 0x71, 0x02, 0x7c, 0xfd, 0xe6, 0xf3, 0xa3, 0x3f, 0xcd, 0x8f, 0x16, 0xc7, 0x7b, - 0x4e, 0x14, 0xb8, 0x4f, 0xf1, 0xf8, 0xca, 0xe1, 0xfb, 0xe8, 0x4f, 0xf0, 0x02, 0x45, 0x36, 0xa0, - 0xf6, 0x99, 0x13, 0x4c, 0xdd, 0xe9, 0x89, 0x38, 0x62, 0x14, 0x9a, 0xda, 0xda, 0xbb, 0x50, 0x11, - 0x55, 0xe4, 0xef, 0x01, 0x83, 0xd2, 0x21, 0x15, 0xe3, 0x9e, 0x39, 0xea, 0x74, 0x0c, 0xd3, 0x54, - 0x25, 0xf1, 0x52, 0xd0, 0x7e, 0x2b, 0x81, 0x92, 0x96, 0x8c, 0xcf, 0x71, 0x83, 0xe1, 0xc0, 0x10, - 0x50, 0xab, 0xd7, 0x37, 0x86, 0x23, 0x4b, 0x95, 0xf8, 0x50, 0xd7, 0xd1, 0x07, 0x1d, 0x63, 0xdf, - 0xe8, 0x8a, 0xe1, 0xd0, 0xf8, 0xb1, 0xd1, 0x19, 0x59, 0xbd, 0xe1, 0x40, 0x2d, 0xf1, 0x60, 0x5b, - 0xef, 0xda, 0x5d, 0xdd, 0xd2, 0x55, 0x99, 0x5b, 0x3d, 0x3e, 0x4f, 0x0e, 0xf4, 0x7d, 0xb5, 0x4c, - 0xd6, 0x60, 0x65, 0x34, 0xd0, 0x1f, 0xe9, 0xbd, 0x7d, 0xbd, 0xbd, 0x6f, 0xa8, 0x15, 0xbe, 0x76, - 0x30, 0xb4, 0xec, 0x07, 0xc3, 0xd1, 0xa0, 0xab, 0x56, 0xf9, 0x60, 0xc9, 0x4d, 0xbd, 0xd3, 0x31, - 0x0e, 0x2c, 0x84, 0xd4, 0xe2, 0x97, 0x55, 0x05, 0x64, 0x3e, 0x23, 0x6b, 0x06, 0x40, 0xb6, 0x17, - 0xf9, 0x11, 0x5c, 0xb9, 0x6a, 0x64, 0xbb, 0x78, 0x3a, 0x68, 0xbf, 0x90, 0x00, 0xb2, 0x3d, 0x22, - 0x77, 0xb3, 0x6f, 0x1a, 0x31, 0x3e, 0xde, 0x58, 0xde, 0xca, 0xcb, 0xbf, 0x6c, 0x7e, 0x98, 0xfb, - 0x42, 0x29, 0x2e, 0xb7, 0xbb, 0x58, 0xfa, 0x9f, 0xbe, 0x53, 0x6c, 0xa8, 0x2f, 0xe6, 0xe7, 0xc7, - 0xa0, 0x98, 0xeb, 0x91, 0x87, 0x42, 0x63, 0xeb, 0xff, 0x9f, 0x4d, 0x7f, 0x2d, 0xc1, 0xda, 0x12, - 0x8d, 0x2b, 0x6f, 0x92, 0x3b, 0x32, 0x8b, 0xaf, 0x71, 0x64, 0x16, 0x16, 0xfa, 0xfb, 0x75, 0xc8, - 0xf0, 0xcd, 0x4b, 0x85, 0x7e, 0xf9, 0xf7, 0xd3, 0xeb, 0x6c, 0x5e, 0x1b, 0x20, 0xd3, 0x3f, 0xf9, - 0x1e, 0x54, 0x72, 0xbf, 0x14, 0x6e, 0x2c, 0x77, 0x49, 0xfc, 0x53, 0x41, 0x10, 0x8e, 0xb1, 0xda, - 0xef, 0x25, 0xa8, 0x2f, 0x86, 0xaf, 0x2c, 0xca, 0xff, 0xfe, 0xb9, 0xdb, 0xce, 0x89, 0x42, 0xbc, - 0x03, 0xde, 0xb9, 0xaa, 0x8e, 0xf8, 0x5d, 0x72, 0x41, 0x17, 0xb7, 0xff, 0x26, 0x01, 0x64, 0x1f, - 0xf3, 0xe4, 0x1a, 0x34, 0xe2, 0xc9, 0xce, 0xee, 0xe8, 0x23, 0x93, 0x37, 0xe4, 0x06, 0xdc, 0xa0, - 0xc6, 0xc1, 0x7e, 0xaf, 0xa3, 0x9b, 0x76, 0xb7, 0xd7, 0xb5, 0x79, 0xdf, 0xf4, 0x75, 0xab, 0xb3, - 0xab, 0x4a, 0xe4, 0x6b, 0x70, 0xcd, 0x1a, 0x0e, 0xed, 0xbe, 0x3e, 0x78, 0x6c, 0x77, 0xf6, 0x47, - 0xa6, 0x65, 0x50, 0x53, 0x2d, 0xe6, 0x3a, 0xb3, 0xc4, 0x13, 0xf4, 0x06, 0x0f, 0x0d, 0x93, 0xb7, - 0xad, 0x4d, 0x75, 0xcb, 0xb0, 0xf7, 0x7b, 0xfd, 0x9e, 0x65, 0x74, 0x55, 0x99, 0x34, 0x61, 0x9d, - 0x1a, 0x1f, 0x8f, 0x0c, 0xd3, 0xca, 0x47, 0xca, 0xbc, 0x43, 0x7b, 0x03, 0xd3, 0xe2, 0xdd, 0x2f, - 0xbc, 0x6a, 0x85, 0xbc, 0x05, 0xd7, 0x4d, 0x83, 0x3e, 0xea, 0x75, 0x0c, 0x7b, 0xb1, 0xbb, 0xab, - 0x64, 0x1d, 0x54, 0xcb, 0xec, 0xb6, 0x73, 0xde, 0x5a, 0xfb, 0x07, 0xcf, 0x5f, 0xb6, 0x0a, 0x9f, - 0xbf, 0x6c, 0x15, 0xbe, 0x7c, 0xd9, 0x92, 0x7e, 0x7e, 0xde, 0x92, 0xfe, 0x78, 0xde, 0x92, 0x9e, - 0x9d, 0xb7, 0xa4, 0xe7, 0xe7, 0x2d, 0xe9, 0x1f, 0xe7, 0x2d, 0xe9, 0x8b, 0xf3, 0x56, 0xe1, 0xcb, - 0xf3, 0x96, 0xf4, 0x9b, 0x57, 0xad, 0xc2, 0xf3, 0x57, 0xad, 0xc2, 0xe7, 0xaf, 0x5a, 0x85, 0x9f, - 0x54, 0xf1, 0xa7, 0xd4, 0xec, 0xf0, 0xb0, 0x82, 0xbf, 0x97, 0xde, 0xff, 0x77, 0x00, 0x00, 0x00, - 0xff, 0xff, 0x30, 0xdf, 0x5e, 0x2d, 0xa6, 0x12, 0x00, 0x00, + // 1939 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0xcf, 0x6f, 0xdb, 0xd8, + 0x11, 0x16, 0x25, 0xea, 0x07, 0xc7, 0x92, 0xcd, 0xbc, 0xb8, 0x59, 0xad, 0xb1, 0x91, 0x1d, 0x16, + 0xdd, 0xba, 0x41, 0xeb, 0x14, 0xbb, 0x6d, 0x16, 0x1b, 0xa4, 0x68, 0x29, 0x89, 0x89, 0xe5, 0xb5, + 0x24, 0xef, 0x23, 0x95, 0x6d, 0x7a, 0x21, 0x68, 0xf9, 0xd9, 0x26, 0x56, 0x14, 0x55, 0x92, 0xca, + 0xc6, 0x3d, 0xf5, 0xd2, 0xa2, 0xe8, 0xa9, 0x97, 0x5e, 0x8a, 0xde, 0x7a, 0xe9, 0x5f, 0xd0, 0x7f, + 0xa0, 0x97, 0x00, 0x45, 0x81, 0x1c, 0x17, 0x3d, 0x04, 0x8d, 0x73, 0xe8, 0x1e, 0xf7, 0xd0, 0x53, + 0x4f, 0xc5, 0x9b, 0xc7, 0x1f, 0xa2, 0x6c, 0xb7, 0x69, 0x37, 0x37, 0xce, 0xcc, 0xf7, 0x86, 0x1f, + 0xe7, 0x7d, 0xf3, 0x38, 0x24, 0xac, 0x78, 0xae, 0xe7, 0x06, 0x3b, 0xb3, 0xc0, 0x8f, 0x7c, 0x52, + 0x1b, 0xfb, 0x41, 0xc4, 0x9e, 0xce, 0x0e, 0x37, 0xbe, 0x73, 0xe2, 0x46, 0xa7, 0xf3, 0xc3, 0x9d, + 0xb1, 0xef, 0xdd, 0x39, 0xf1, 0x4f, 0xfc, 0x3b, 0x08, 0x38, 0x9c, 0x1f, 0xa3, 0x85, 0x06, 0x5e, + 0x89, 0x85, 0xda, 0x9f, 0x8a, 0x50, 0xff, 0x24, 0x70, 0x23, 0x46, 0xd9, 0x4f, 0xe7, 0x2c, 0x8c, + 0xc8, 0x01, 0x40, 0xe4, 0x7a, 0x2c, 0x64, 0x81, 0xcb, 0xc2, 0xa6, 0xb4, 0x55, 0xda, 0x5e, 0x79, + 0x6f, 0x7d, 0x27, 0x49, 0xbf, 0x63, 0xb9, 0x1e, 0x33, 0x31, 0xd6, 0xde, 0x78, 0xf6, 0x62, 0xb3, + 0xf0, 0xb7, 0x17, 0x9b, 0xe4, 0x20, 0x60, 0xce, 0x64, 0xe2, 0x8f, 0xad, 0x74, 0x1d, 0x5d, 0xc8, + 0x41, 0x3e, 0x84, 0x8a, 0xe9, 0xcf, 0x83, 0x31, 0x6b, 0x16, 0xb7, 0xa4, 0xed, 0xd5, 0xf7, 0x6e, + 0x65, 0xd9, 0x16, 0xef, 0xbc, 0x23, 0x40, 0xc6, 0x74, 0xee, 0xd1, 0x78, 0x01, 0xb9, 0x07, 0x35, + 0x8f, 0x45, 0xce, 0x91, 0x13, 0x39, 0xcd, 0x12, 0x52, 0x69, 0x66, 0x8b, 0xfb, 0x2c, 0x0a, 0xdc, + 0x71, 0x3f, 0x8e, 0xb7, 0xe5, 0x67, 0x2f, 0x36, 0x25, 0x9a, 0xe2, 0xc9, 0x7d, 0xd8, 0x08, 0x3f, + 0x75, 0x67, 0xf6, 0xc4, 0x39, 0x64, 0x13, 0x7b, 0xea, 0x78, 0xcc, 0x7e, 0xe2, 0x4c, 0xdc, 0x23, + 0x27, 0x72, 0xfd, 0x69, 0xf3, 0x8b, 0xea, 0x96, 0xb4, 0x5d, 0xa3, 0x6f, 0x71, 0xc8, 0x3e, 0x47, + 0x0c, 0x1c, 0x8f, 0x3d, 0x4a, 0xe3, 0xda, 0x26, 0x40, 0xc6, 0x87, 0x54, 0xa1, 0xa4, 0x1f, 0xf4, + 0xd4, 0x02, 0xa9, 0x81, 0x4c, 0x47, 0xfb, 0x86, 0x2a, 0x69, 0x6b, 0xd0, 0x88, 0xd9, 0x87, 0x33, + 0x7f, 0x1a, 0x32, 0xed, 0x1e, 0xd4, 0x8d, 0x20, 0xf0, 0x83, 0x2e, 0x8b, 0x1c, 0x77, 0x12, 0x92, + 0xdb, 0x50, 0xee, 0x38, 0xf3, 0x90, 0x35, 0x25, 0x7c, 0xea, 0x85, 0x1a, 0x22, 0x0c, 0x63, 0x54, + 0x40, 0xb4, 0x7f, 0x4a, 0x00, 0x59, 0x65, 0x89, 0x0e, 0x15, 0x64, 0x9d, 0xd4, 0xff, 0x7a, 0xb6, + 0x16, 0xb9, 0x1e, 0x38, 0x6e, 0xd0, 0x5e, 0x8f, 0xcb, 0x5f, 0x47, 0x97, 0x7e, 0xe4, 0xcc, 0x22, + 0x16, 0xd0, 0x78, 0x21, 0xf9, 0x2e, 0x54, 0x43, 0xc7, 0x9b, 0x4d, 0x58, 0xd8, 0x2c, 0x62, 0x0e, + 0x35, 0xcb, 0x61, 0x62, 0x00, 0x0b, 0x56, 0xa0, 0x09, 0x8c, 0xdc, 0x05, 0x85, 0x3d, 0x65, 0xde, + 0x6c, 0xe2, 0x04, 0x61, 0x5c, 0x6c, 0xb2, 0xc0, 0x39, 0x0e, 0xc5, 0xab, 0x32, 0x28, 0xf9, 0x10, + 0xe0, 0xd4, 0x0d, 0x23, 0xff, 0x24, 0x70, 0xbc, 0xb0, 0x29, 0x2f, 0x13, 0xde, 0x4d, 0x62, 0xf1, + 0xca, 0x05, 0xb0, 0xf6, 0x7d, 0x50, 0xd2, 0xe7, 0x21, 0x04, 0x64, 0xbe, 0x49, 0x58, 0xae, 0x3a, + 0xc5, 0x6b, 0xb2, 0x0e, 0xe5, 0x27, 0xce, 0x64, 0x2e, 0x94, 0x53, 0xa7, 0xc2, 0xd0, 0x74, 0xa8, + 0x88, 0x47, 0x20, 0xb7, 0xa0, 0x8e, 0x42, 0x8b, 0x1c, 0x6f, 0x66, 0x7b, 0x21, 0xc2, 0x4a, 0x74, + 0x25, 0xf5, 0xf5, 0xc3, 0x2c, 0x05, 0xcf, 0x2b, 0x25, 0x29, 0x7e, 0x57, 0x84, 0xd5, 0xbc, 0x7e, + 0xc8, 0x07, 0x20, 0x47, 0x67, 0xb3, 0x64, 0xbb, 0xbe, 0x7e, 0x95, 0xce, 0x62, 0xd3, 0x3a, 0x9b, + 0x31, 0x8a, 0x0b, 0xc8, 0xb7, 0x81, 0x78, 0xe8, 0xb3, 0x8f, 0x1d, 0xcf, 0x9d, 0x9c, 0xa1, 0xd6, + 0x90, 0x8a, 0x42, 0x55, 0x11, 0x79, 0x80, 0x01, 0x2e, 0x31, 0xfe, 0x98, 0xa7, 0x6c, 0x32, 0x6b, + 0xca, 0x18, 0xc7, 0x6b, 0xee, 0x9b, 0x4f, 0xdd, 0xa8, 0x59, 0x16, 0x3e, 0x7e, 0xad, 0x9d, 0x01, + 0x64, 0x77, 0x22, 0x2b, 0x50, 0x1d, 0x0d, 0x3e, 0x1a, 0x0c, 0x3f, 0x19, 0xa8, 0x05, 0x6e, 0x74, + 0x86, 0xa3, 0x81, 0x65, 0x50, 0x55, 0x22, 0x0a, 0x94, 0x1f, 0xea, 0xa3, 0x87, 0x86, 0x5a, 0x24, + 0x0d, 0x50, 0x76, 0x7b, 0xa6, 0x35, 0x7c, 0x48, 0xf5, 0xbe, 0x5a, 0x22, 0x04, 0x56, 0x31, 0x92, + 0xf9, 0x64, 0xbe, 0xd4, 0x1c, 0xf5, 0xfb, 0x3a, 0x7d, 0xac, 0x96, 0xb9, 0x98, 0x7b, 0x83, 0x07, + 0x43, 0xb5, 0x42, 0xea, 0x50, 0x33, 0x2d, 0xdd, 0x32, 0x4c, 0xc3, 0x52, 0xab, 0xda, 0x47, 0x50, + 0x11, 0xb7, 0x7e, 0x03, 0x42, 0xd4, 0x7e, 0x29, 0x41, 0x2d, 0x11, 0xcf, 0x9b, 0x10, 0x76, 0x4e, + 0x12, 0xc9, 0x7e, 0x5e, 0x10, 0x42, 0xe9, 0x82, 0x10, 0xb4, 0xbf, 0x94, 0x41, 0x49, 0xc5, 0x48, + 0x6e, 0x82, 0x32, 0xf6, 0xe7, 0xd3, 0xc8, 0x76, 0xa7, 0x11, 0x6e, 0xb9, 0xbc, 0x5b, 0xa0, 0x35, + 0x74, 0xf5, 0xa6, 0x11, 0xb9, 0x05, 0x2b, 0x22, 0x7c, 0x3c, 0xf1, 0x9d, 0x48, 0xdc, 0x6b, 0xb7, + 0x40, 0x01, 0x9d, 0x0f, 0xb8, 0x8f, 0xa8, 0x50, 0x0a, 0xe7, 0x1e, 0xde, 0x49, 0xa2, 0xfc, 0x92, + 0xdc, 0x80, 0x4a, 0x38, 0x3e, 0x65, 0x9e, 0x83, 0x9b, 0x7b, 0x8d, 0xc6, 0x16, 0xf9, 0x06, 0xac, + 0xfe, 0x8c, 0x05, 0xbe, 0x1d, 0x9d, 0x06, 0x2c, 0x3c, 0xf5, 0x27, 0x47, 0xb8, 0xd1, 0x12, 0x6d, + 0x70, 0xaf, 0x95, 0x38, 0xc9, 0xbb, 0x31, 0x2c, 0xe3, 0x55, 0x41, 0x5e, 0x12, 0xad, 0x73, 0x7f, + 0x27, 0xe1, 0x76, 0x1b, 0xd4, 0x05, 0x9c, 0x20, 0x58, 0x45, 0x82, 0x12, 0x5d, 0x4d, 0x91, 0x82, + 0xa4, 0x0e, 0xab, 0x53, 0x76, 0xe2, 0x44, 0xee, 0x13, 0x66, 0x87, 0x33, 0x67, 0x1a, 0x36, 0x6b, + 0xcb, 0x27, 0x7a, 0x7b, 0x3e, 0xfe, 0x94, 0x45, 0xe6, 0xcc, 0x99, 0xc6, 0x1d, 0xda, 0x48, 0x56, + 0x70, 0x5f, 0x48, 0xbe, 0x09, 0x6b, 0x69, 0x8a, 0x23, 0x36, 0x89, 0x9c, 0xb0, 0xa9, 0x6c, 0x95, + 0xb6, 0x09, 0x4d, 0x33, 0x77, 0xd1, 0x9b, 0x03, 0x22, 0xb7, 0xb0, 0x09, 0x5b, 0xa5, 0x6d, 0x29, + 0x03, 0x22, 0x31, 0x7e, 0xbc, 0xad, 0xce, 0xfc, 0xd0, 0x5d, 0x20, 0xb5, 0xf2, 0xdf, 0x49, 0x25, + 0x2b, 0x52, 0x52, 0x69, 0x8a, 0x98, 0x54, 0x5d, 0x90, 0x4a, 0xdc, 0x19, 0xa9, 0x14, 0x18, 0x93, + 0x6a, 0x08, 0x52, 0x89, 0x3b, 0x26, 0x75, 0x1f, 0x20, 0x60, 0x21, 0x8b, 0xec, 0x53, 0x5e, 0xf9, + 0x55, 0x3c, 0x04, 0x6e, 0x5e, 0x72, 0x8c, 0xed, 0x50, 0x8e, 0xda, 0x75, 0xa7, 0x11, 0x55, 0x82, + 0xe4, 0x92, 0xbc, 0x03, 0x4a, 0xaa, 0xb5, 0xe6, 0x1a, 0x8a, 0x2f, 0x73, 0x68, 0xf7, 0x40, 0x49, + 0x57, 0xe5, 0x5b, 0xb9, 0x0a, 0xa5, 0xc7, 0x86, 0xa9, 0x4a, 0xa4, 0x02, 0xc5, 0xc1, 0x50, 0x2d, + 0x66, 0xed, 0x5c, 0xda, 0x90, 0x7f, 0xf5, 0x87, 0x96, 0xd4, 0xae, 0x42, 0x19, 0x79, 0xb7, 0xeb, + 0x00, 0xd9, 0xb6, 0x6b, 0x7f, 0x95, 0x61, 0x15, 0xb7, 0x38, 0x93, 0x74, 0x08, 0x04, 0x63, 0x2c, + 0xb0, 0x97, 0x9e, 0xa4, 0xd1, 0x36, 0xfe, 0xf5, 0x62, 0x53, 0x5f, 0x98, 0x0c, 0x66, 0x81, 0xef, + 0xb1, 0xe8, 0x94, 0xcd, 0xc3, 0xc5, 0x4b, 0xcf, 0x3f, 0x62, 0x93, 0x3b, 0xe9, 0x01, 0xbd, 0xd3, + 0x11, 0xe9, 0xb2, 0x27, 0x56, 0xc7, 0x4b, 0x9e, 0xaf, 0xaa, 0xf9, 0x9b, 0x8b, 0x0f, 0x25, 0x54, + 0x4c, 0x95, 0x54, 0xc3, 0xbc, 0xd9, 0x45, 0x24, 0x6e, 0x76, 0x34, 0x2e, 0xe9, 0xbc, 0x37, 0xa0, + 0xa8, 0x37, 0xd0, 0x29, 0xdf, 0x02, 0x35, 0x65, 0x71, 0x88, 0xd8, 0x44, 0x6c, 0xa9, 0x06, 0x45, + 0x0a, 0x84, 0xa6, 0x77, 0x4b, 0xa0, 0xa2, 0x59, 0xd2, 0x1e, 0x8a, 0xa1, 0x7b, 0x72, 0x4d, 0x52, + 0x8b, 0x7b, 0x72, 0xad, 0xa2, 0x56, 0xf7, 0xe4, 0x9a, 0xa2, 0xc2, 0x9e, 0x5c, 0xab, 0xab, 0x8d, + 0x3d, 0xb9, 0xb6, 0xa6, 0xaa, 0x34, 0x3b, 0xc5, 0xe8, 0xd2, 0xe9, 0x41, 0x97, 0xdb, 0x96, 0x2e, + 0xb7, 0xcc, 0xa2, 0x44, 0xef, 0x03, 0x64, 0x8f, 0xc7, 0x77, 0xd5, 0x3f, 0x3e, 0x0e, 0x99, 0x38, + 0x1a, 0xaf, 0xd1, 0xd8, 0xe2, 0xfe, 0x09, 0x9b, 0x9e, 0x44, 0xa7, 0xb8, 0x21, 0x0d, 0x1a, 0x5b, + 0xda, 0x1c, 0x48, 0x5e, 0x8c, 0xf8, 0x46, 0x7f, 0x8d, 0xb7, 0xf3, 0x7d, 0x50, 0x52, 0xb9, 0xe1, + 0xbd, 0x72, 0x13, 0x5e, 0x3e, 0x67, 0x3c, 0xe1, 0x65, 0x0b, 0xb4, 0x29, 0xac, 0x89, 0x41, 0x20, + 0x6b, 0x82, 0x54, 0x31, 0xd2, 0x25, 0x8a, 0x29, 0x66, 0x8a, 0x79, 0x1f, 0xaa, 0x49, 0xdd, 0xc5, + 0xac, 0xf3, 0xf6, 0x65, 0x23, 0x0b, 0x22, 0x68, 0x82, 0xd4, 0x42, 0x58, 0x5b, 0x8a, 0x91, 0x16, + 0xc0, 0xa1, 0x3f, 0x9f, 0x1e, 0x39, 0xf1, 0xb8, 0x2c, 0x6d, 0x97, 0xe9, 0x82, 0x87, 0xf3, 0x99, + 0xf8, 0x9f, 0xb1, 0x20, 0x51, 0x30, 0x1a, 0xdc, 0x3b, 0x9f, 0xcd, 0x58, 0x10, 0x6b, 0x58, 0x18, + 0x19, 0x77, 0x79, 0x81, 0xbb, 0x36, 0x81, 0xeb, 0x4b, 0x0f, 0x89, 0xc5, 0xcd, 0x9d, 0x38, 0xc5, + 0xa5, 0x13, 0x87, 0x7c, 0x70, 0xb1, 0xae, 0x6f, 0x2f, 0x0f, 0x80, 0x69, 0xbe, 0xc5, 0x92, 0xfe, + 0x59, 0x86, 0xc6, 0xc7, 0x73, 0x16, 0x9c, 0x25, 0x73, 0x2d, 0xb9, 0x0b, 0x95, 0x30, 0x72, 0xa2, + 0x79, 0x18, 0x4f, 0x46, 0xad, 0x2c, 0x4f, 0x0e, 0xb8, 0x63, 0x22, 0x8a, 0xc6, 0x68, 0xf2, 0x23, + 0x00, 0xc6, 0x07, 0x5d, 0x1b, 0xa7, 0xaa, 0x0b, 0xa3, 0x7f, 0x7e, 0x2d, 0x8e, 0xc4, 0x38, 0x53, + 0x29, 0x2c, 0xb9, 0xe4, 0xf5, 0x40, 0x03, 0xab, 0xa4, 0x50, 0x61, 0x90, 0x1d, 0xce, 0x27, 0x70, + 0xa7, 0x27, 0x58, 0xa6, 0x5c, 0x83, 0x9a, 0xe8, 0xef, 0x3a, 0x91, 0xb3, 0x5b, 0xa0, 0x31, 0x8a, + 0xe3, 0x9f, 0xb0, 0x71, 0xe4, 0x07, 0x78, 0x02, 0xe5, 0xf0, 0x8f, 0xd0, 0x9f, 0xe0, 0x05, 0x0a, + 0xf3, 0x8f, 0x9d, 0x89, 0x13, 0xe0, 0xeb, 0x37, 0x9f, 0x1f, 0xfd, 0x69, 0x7e, 0xb4, 0x38, 0xde, + 0x73, 0xa2, 0xc0, 0x7d, 0x8a, 0xc7, 0x57, 0x0e, 0xdf, 0x47, 0x7f, 0x82, 0x17, 0x28, 0xb2, 0x01, + 0xb5, 0xcf, 0x9c, 0x60, 0xea, 0x4e, 0x4f, 0xc4, 0x11, 0xa3, 0xd0, 0xd4, 0xd6, 0xde, 0x85, 0x8a, + 0xa8, 0x22, 0x7f, 0x0f, 0x18, 0x94, 0x0e, 0xa9, 0x18, 0xf7, 0xcc, 0x51, 0xa7, 0x63, 0x98, 0xa6, + 0x2a, 0x89, 0x97, 0x82, 0xf6, 0x5b, 0x09, 0x94, 0xb4, 0x64, 0x7c, 0x8e, 0x1b, 0x0c, 0x07, 0x86, + 0x80, 0x5a, 0xbd, 0xbe, 0x31, 0x1c, 0x59, 0xaa, 0xc4, 0x87, 0xba, 0x8e, 0x3e, 0xe8, 0x18, 0xfb, + 0x46, 0x57, 0x0c, 0x87, 0xc6, 0x8f, 0x8d, 0xce, 0xc8, 0xea, 0x0d, 0x07, 0x6a, 0x89, 0x07, 0xdb, + 0x7a, 0xd7, 0xee, 0xea, 0x96, 0xae, 0xca, 0xdc, 0xea, 0xf1, 0x79, 0x72, 0xa0, 0xef, 0xab, 0x65, + 0xb2, 0x06, 0x2b, 0xa3, 0x81, 0xfe, 0x48, 0xef, 0xed, 0xeb, 0xed, 0x7d, 0x43, 0xad, 0xf0, 0xb5, + 0x83, 0xa1, 0x65, 0x3f, 0x18, 0x8e, 0x06, 0x5d, 0xb5, 0xca, 0x07, 0x4b, 0x6e, 0xea, 0x9d, 0x8e, + 0x71, 0x60, 0x21, 0xa4, 0x16, 0xbf, 0xac, 0x2a, 0x20, 0xf3, 0x19, 0x59, 0x33, 0x00, 0xb2, 0xbd, + 0xc8, 0x8f, 0xe0, 0xca, 0x55, 0x23, 0xdb, 0xc5, 0xd3, 0x41, 0xfb, 0x85, 0x04, 0x90, 0xed, 0x11, + 0xb9, 0x9b, 0x7d, 0xd3, 0x88, 0xf1, 0xf1, 0xc6, 0xf2, 0x56, 0x5e, 0xfe, 0x65, 0xf3, 0xc3, 0xdc, + 0x17, 0x4a, 0x71, 0xb9, 0xdd, 0xc5, 0xd2, 0xff, 0xf4, 0x9d, 0x62, 0x43, 0x7d, 0x31, 0x3f, 0x3f, + 0x06, 0xc5, 0x5c, 0x8f, 0x3c, 0x14, 0x1a, 0x5b, 0xff, 0xff, 0x6c, 0xfa, 0x6b, 0x09, 0xd6, 0x96, + 0x68, 0x5c, 0x79, 0x93, 0xdc, 0x91, 0x59, 0x7c, 0x8d, 0x23, 0xb3, 0xb0, 0xd0, 0xdf, 0xaf, 0x43, + 0x86, 0x6f, 0x5e, 0x2a, 0xf4, 0xcb, 0xbf, 0x9f, 0x5e, 0x67, 0xf3, 0xda, 0x00, 0x99, 0xfe, 0xc9, + 0xf7, 0xa0, 0x92, 0xfb, 0xa5, 0x70, 0x63, 0xb9, 0x4b, 0xe2, 0x9f, 0x0a, 0x82, 0x70, 0x8c, 0xd5, + 0x7e, 0x2f, 0x41, 0x7d, 0x31, 0x7c, 0x65, 0x51, 0xfe, 0xf7, 0xcf, 0xdd, 0x76, 0x4e, 0x14, 0xe2, + 0x1d, 0xf0, 0xce, 0x55, 0x75, 0xc4, 0xef, 0x92, 0x0b, 0xba, 0xb8, 0xfd, 0x0f, 0x09, 0x20, 0xfb, + 0x98, 0x27, 0xd7, 0xa0, 0x11, 0x4f, 0x76, 0x76, 0x47, 0x1f, 0x99, 0xbc, 0x21, 0x37, 0xe0, 0x06, + 0x35, 0x0e, 0xf6, 0x7b, 0x1d, 0xdd, 0xb4, 0xbb, 0xbd, 0xae, 0xcd, 0xfb, 0xa6, 0xaf, 0x5b, 0x9d, + 0x5d, 0x55, 0x22, 0x5f, 0x83, 0x6b, 0xd6, 0x70, 0x68, 0xf7, 0xf5, 0xc1, 0x63, 0xbb, 0xb3, 0x3f, + 0x32, 0x2d, 0x83, 0x9a, 0x6a, 0x31, 0xd7, 0x99, 0x25, 0x9e, 0xa0, 0x37, 0x78, 0x68, 0x98, 0xbc, + 0x6d, 0x6d, 0xaa, 0x5b, 0x86, 0xbd, 0xdf, 0xeb, 0xf7, 0x2c, 0xa3, 0xab, 0xca, 0xa4, 0x09, 0xeb, + 0xd4, 0xf8, 0x78, 0x64, 0x98, 0x56, 0x3e, 0x52, 0xe6, 0x1d, 0xda, 0x1b, 0x98, 0x16, 0xef, 0x7e, + 0xe1, 0x55, 0x2b, 0xe4, 0x2d, 0xb8, 0x6e, 0x1a, 0xf4, 0x51, 0xaf, 0x63, 0xd8, 0x8b, 0xdd, 0x5d, + 0x25, 0xeb, 0xa0, 0x5a, 0x66, 0xb7, 0x9d, 0xf3, 0xd6, 0x38, 0x0d, 0xce, 0xae, 0x3d, 0x32, 0x1f, + 0xab, 0x4a, 0xfb, 0x07, 0xcf, 0x5f, 0xb6, 0x0a, 0x9f, 0xbf, 0x6c, 0x15, 0xbe, 0x7c, 0xd9, 0x92, + 0x7e, 0x7e, 0xde, 0x92, 0xfe, 0x78, 0xde, 0x92, 0x9e, 0x9d, 0xb7, 0xa4, 0xe7, 0xe7, 0x2d, 0xe9, + 0xef, 0xe7, 0x2d, 0xe9, 0x8b, 0xf3, 0x56, 0xe1, 0xcb, 0xf3, 0x96, 0xf4, 0x9b, 0x57, 0xad, 0xc2, + 0xf3, 0x57, 0xad, 0xc2, 0xe7, 0xaf, 0x5a, 0x85, 0x9f, 0x54, 0xf1, 0x17, 0xd5, 0xec, 0xf0, 0xb0, + 0x82, 0x3f, 0x9b, 0xde, 0xff, 0x77, 0x00, 0x00, 0x00, 0xff, 0xff, 0xef, 0xbc, 0x47, 0xbe, 0xb4, + 0x12, 0x00, 0x00, } func (x ErrorCause) String() string { From f463f1296d7c2a3cab3c6717452c739df31a40e0 Mon Sep 17 00:00:00 2001 From: Yuri Nikolic Date: Thu, 23 Nov 2023 19:02:57 +0100 Subject: [PATCH 3/7] Fixing review findings Signed-off-by: Yuri Nikolic --- CHANGELOG.md | 2 +- pkg/ingester/active_series.go | 4 +-- pkg/ingester/errors_test.go | 10 +++--- pkg/ingester/ingester.go | 46 +++++++++++------------- pkg/querier/error_translate_queryable.go | 2 +- 5 files changed, 30 insertions(+), 34 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 60d60f00440..9168cef155e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,7 +29,7 @@ * [CHANGE] Ingester: by setting the newly introduced experimental CLI flag `-ingester.return-only-grpc-errors` to true, ingester will return only gRPC errors. This feature changes the following status codes: #6443 #6680 #6723 * `http.StatusBadRequest` (400) is replaced with `codes.FailedPrecondition` on the write path. * `http.StatusServiceUnavailable` (503) is replaced with `codes.Internal` on the write path, and with `codes.ResourceExhausted` on the read path. - * `codes.Unknown` are replaced with `codes.Internal` on both write and read path. + * `codes.Unknown` is replaced with `codes.Internal` on both write and read path. * [CHANGE] Upgrade Node.js to v20. #6540 * [CHANGE] Querier: `cortex_querier_blocks_consistency_checks_failed_total` is now incremented when a block couldn't be queried from any attempted store-gateway as opposed to incremented after each attempt. Also `cortex_querier_blocks_consistency_checks_total` is incremented once per query as opposed to once per attempt (with 3 attempts). #6590 * [CHANGE] Ingester: Modify utilization based read path limiter to base memory usage on Go heap size. #6584 diff --git a/pkg/ingester/active_series.go b/pkg/ingester/active_series.go index 35809ab3fe4..6c466a5f800 100644 --- a/pkg/ingester/active_series.go +++ b/pkg/ingester/active_series.go @@ -21,8 +21,8 @@ const activeSeriesMaxSizeBytes = 1 * 1024 * 1024 // ActiveSeries implements the ActiveSeries RPC. It returns a stream of active // series that match the given matchers. -func (i *Ingester) ActiveSeries(request *client.ActiveSeriesRequest, stream client.Ingester_ActiveSeriesServer) (asErr error) { - defer func() { asErr = i.handleReadError(asErr) }() +func (i *Ingester) ActiveSeries(request *client.ActiveSeriesRequest, stream client.Ingester_ActiveSeriesServer) (err error) { + defer func() { err = i.handleReadError(err) }() if err := i.checkAvailable(); err != nil { return err } diff --git a/pkg/ingester/errors_test.go b/pkg/ingester/errors_test.go index cc204530a63..04c2dced534 100644 --- a/pkg/ingester/errors_test.go +++ b/pkg/ingester/errors_test.go @@ -239,7 +239,7 @@ func TestNewTSDBIngestExemplarErr(t *testing.T) { func TestTooBusyError(t *testing.T) { require.Error(t, tooBusyError) - require.Equal(t, tooBusyErrorMsg, tooBusyError.Error()) + require.Equal(t, "the ingester is currently too busy to process queries, try again later", tooBusyError.Error()) checkIngesterError(t, tooBusyError, mimirpb.TOO_BUSY, false) wrappedErr := wrapOrAnnotateWithUser(tooBusyError, userID) @@ -360,7 +360,7 @@ func TestWrapOrAnnotateWithUser(t *testing.T) { } func TestHandlePushError(t *testing.T) { - originalMsg := "this is an error" + const originalMsg = "this is an error" originalErr := errors.New(originalMsg) labelAdapters := []mimirpb.LabelAdapter{{Name: labels.MetricName, Value: "testmetric"}, {Name: "foo", Value: "biz"}} labels := mimirpb.FromLabelAdaptersToLabels(labelAdapters) @@ -528,7 +528,7 @@ func TestHandlePushError(t *testing.T) { } func TestHandlePushErrorWithHTTPGRPC(t *testing.T) { - originalMsg := "this is an error" + const originalMsg = "this is an error" originalErr := errors.New(originalMsg) labelAdapters := []mimirpb.LabelAdapter{{Name: labels.MetricName, Value: "testmetric"}, {Name: "foo", Value: "biz"}} labels := mimirpb.FromLabelAdaptersToLabels(labelAdapters) @@ -689,7 +689,7 @@ func TestHandlePushErrorWithHTTPGRPC(t *testing.T) { } func TestHandleReadError(t *testing.T) { - originalMsg := "this is an error" + const originalMsg = "this is an error" originalErr := errors.New(originalMsg) testCases := map[string]struct { @@ -742,7 +742,7 @@ func TestHandleReadError(t *testing.T) { } func TestHandleReadErrorWithHTTPGRPC(t *testing.T) { - originalMsg := "this is an error" + const originalMsg = "this is an error" originalErr := errors.New(originalMsg) testCases := map[string]struct { diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 342af885949..242e83df96c 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -1270,8 +1270,8 @@ func (i *Ingester) pushSamplesToAppender(userID string, timeseries []mimirpb.Pre return nil } -func (i *Ingester) QueryExemplars(ctx context.Context, req *client.ExemplarQueryRequest) (resp *client.ExemplarQueryResponse, qeErr error) { - defer func() { qeErr = i.handleReadError(qeErr) }() +func (i *Ingester) QueryExemplars(ctx context.Context, req *client.ExemplarQueryRequest) (resp *client.ExemplarQueryResponse, err error) { + defer func() { err = i.handleReadError(err) }() if err := i.checkAvailable(); err != nil { return nil, err } @@ -1328,8 +1328,8 @@ func (i *Ingester) QueryExemplars(ctx context.Context, req *client.ExemplarQuery return result, nil } -func (i *Ingester) LabelValues(ctx context.Context, req *client.LabelValuesRequest) (resp *client.LabelValuesResponse, lvErr error) { - defer func() { lvErr = i.handleReadError(lvErr) }() +func (i *Ingester) LabelValues(ctx context.Context, req *client.LabelValuesRequest) (resp *client.LabelValuesResponse, err error) { + defer func() { err = i.handleReadError(err) }() if err := i.checkAvailable(); err != nil { return nil, err } @@ -1368,8 +1368,8 @@ func (i *Ingester) LabelValues(ctx context.Context, req *client.LabelValuesReque }, nil } -func (i *Ingester) LabelNames(ctx context.Context, req *client.LabelNamesRequest) (resp *client.LabelNamesResponse, lnErr error) { - defer func() { lnErr = i.handleReadError(lnErr) }() +func (i *Ingester) LabelNames(ctx context.Context, req *client.LabelNamesRequest) (resp *client.LabelNamesResponse, err error) { + defer func() { err = i.handleReadError(err) }() if err := i.checkAvailable(); err != nil { return nil, err } @@ -1409,8 +1409,8 @@ func (i *Ingester) LabelNames(ctx context.Context, req *client.LabelNamesRequest } // MetricsForLabelMatchers implements IngesterServer. -func (i *Ingester) MetricsForLabelMatchers(ctx context.Context, req *client.MetricsForLabelMatchersRequest) (resp *client.MetricsForLabelMatchersResponse, mlErr error) { - defer func() { mlErr = i.handleReadError(mlErr) }() +func (i *Ingester) MetricsForLabelMatchers(ctx context.Context, req *client.MetricsForLabelMatchersRequest) (resp *client.MetricsForLabelMatchersResponse, err error) { + defer func() { err = i.handleReadError(err) }() if err := i.checkAvailable(); err != nil { return nil, err } @@ -1480,8 +1480,8 @@ func (i *Ingester) MetricsForLabelMatchers(ctx context.Context, req *client.Metr return result, nil } -func (i *Ingester) UserStats(ctx context.Context, req *client.UserStatsRequest) (resp *client.UserStatsResponse, usErr error) { - defer func() { usErr = i.handleReadError(usErr) }() +func (i *Ingester) UserStats(ctx context.Context, req *client.UserStatsRequest) (resp *client.UserStatsResponse, err error) { + defer func() { err = i.handleReadError(err) }() if err := i.checkAvailable(); err != nil { return nil, err } @@ -1502,8 +1502,8 @@ func (i *Ingester) UserStats(ctx context.Context, req *client.UserStatsRequest) return createUserStats(db, req) } -func (i *Ingester) AllUserStats(_ context.Context, req *client.UserStatsRequest) (resp *client.UsersStatsResponse, auErr error) { - defer func() { auErr = i.handleReadError(auErr) }() +func (i *Ingester) AllUserStats(_ context.Context, req *client.UserStatsRequest) (resp *client.UsersStatsResponse, err error) { + defer func() { err = i.handleReadError(err) }() if err := i.checkAvailable(); err != nil { return nil, err } @@ -1533,8 +1533,8 @@ func (i *Ingester) AllUserStats(_ context.Context, req *client.UserStatsRequest) // So, 1 MB limit will prevent reaching the limit and won't affect performance significantly. const labelNamesAndValuesTargetSizeBytes = 1 * 1024 * 1024 -func (i *Ingester) LabelNamesAndValues(request *client.LabelNamesAndValuesRequest, stream client.Ingester_LabelNamesAndValuesServer) (lnvErr error) { - defer func() { lnvErr = i.handleReadError(lnvErr) }() +func (i *Ingester) LabelNamesAndValues(request *client.LabelNamesAndValuesRequest, stream client.Ingester_LabelNamesAndValuesServer) (err error) { + defer func() { err = i.handleReadError(err) }() if err := i.checkAvailable(); err != nil { return err } @@ -1566,8 +1566,8 @@ func (i *Ingester) LabelNamesAndValues(request *client.LabelNamesAndValuesReques // We arbitrarily set it to 1mb to avoid reaching the actual gRPC default limit (4mb). const labelValuesCardinalityTargetSizeBytes = 1 * 1024 * 1024 -func (i *Ingester) LabelValuesCardinality(req *client.LabelValuesCardinalityRequest, srv client.Ingester_LabelValuesCardinalityServer) (lvcErr error) { - defer func() { lvcErr = i.handleReadError(lvcErr) }() +func (i *Ingester) LabelValuesCardinality(req *client.LabelValuesCardinalityRequest, srv client.Ingester_LabelValuesCardinalityServer) (err error) { + defer func() { err = i.handleReadError(err) }() if err := i.checkAvailable(); err != nil { return err } @@ -1647,8 +1647,8 @@ func createUserStats(db *userTSDB, req *client.UserStatsRequest) (*client.UserSt const queryStreamBatchMessageSize = 1 * 1024 * 1024 // QueryStream streams metrics from a TSDB. This implements the client.IngesterServer interface -func (i *Ingester) QueryStream(req *client.QueryRequest, stream client.Ingester_QueryStreamServer) (qsErr error) { - defer func() { qsErr = i.handleReadError(qsErr) }() +func (i *Ingester) QueryStream(req *client.QueryRequest, stream client.Ingester_QueryStreamServer) (err error) { + defer func() { err = i.handleReadError(err) }() if err := i.checkAvailable(); err != nil { return err } @@ -3188,11 +3188,7 @@ func (i *Ingester) handleReadError(err error) error { return nil } - if errors.Is(err, context.Canceled) { - return err - } - - if errors.Is(err, context.DeadlineExceeded) { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { return err } @@ -3301,8 +3297,8 @@ func (i *Ingester) purgeUserMetricsMetadata() { } // MetricsMetadata returns all the metrics metadata of a user. -func (i *Ingester) MetricsMetadata(ctx context.Context, req *client.MetricsMetadataRequest) (resp *client.MetricsMetadataResponse, mmErr error) { - defer func() { mmErr = i.handleReadError(mmErr) }() +func (i *Ingester) MetricsMetadata(ctx context.Context, req *client.MetricsMetadataRequest) (resp *client.MetricsMetadataResponse, err error) { + defer func() { err = i.handleReadError(err) }() if err := i.checkAvailable(); err != nil { return nil, err } diff --git a/pkg/querier/error_translate_queryable.go b/pkg/querier/error_translate_queryable.go index ee045e34f42..79fa2eb64cf 100644 --- a/pkg/querier/error_translate_queryable.go +++ b/pkg/querier/error_translate_queryable.go @@ -68,7 +68,7 @@ func TranslateToPromqlAPIError(err error) error { details := s.Details() if len(details) == 1 { - if errorDetails, ok := details[0].(*mimirpb.WriteErrorDetails); ok { + if errorDetails, ok := details[0].(*mimirpb.ErrorDetails); ok { switch errorDetails.GetCause() { case mimirpb.TOO_BUSY: return promql.ErrQueryTimeout(s.Message()) From c288faaf844ab01f381f1bd3deccb98ea2db25f5 Mon Sep 17 00:00:00 2001 From: Yuri Nikolic Date: Fri, 24 Nov 2023 10:53:36 +0100 Subject: [PATCH 4/7] Fixing review findings 2 Signed-off-by: Yuri Nikolic --- pkg/distributor/distributor.go | 2 +- pkg/distributor/errors.go | 4 ++-- pkg/distributor/errors_test.go | 2 +- pkg/ingester/active_series.go | 2 +- pkg/ingester/errors.go | 20 +++++++++++--------- pkg/ingester/errors_test.go | 16 ++++++++-------- pkg/ingester/ingester.go | 34 +++++++++++++++++----------------- pkg/ingester/ingester_test.go | 2 +- 8 files changed, 42 insertions(+), 40 deletions(-) diff --git a/pkg/distributor/distributor.go b/pkg/distributor/distributor.go index 186dd6a3759..aeae71fb41e 100644 --- a/pkg/distributor/distributor.go +++ b/pkg/distributor/distributor.go @@ -1247,7 +1247,7 @@ func (d *Distributor) handlePushError(ctx context.Context, pushErr error) error // TODO This code is needed for backwards compatibility, since ingesters may still return // errors created by httpgrpc.Errorf(). If pushErr is one of those errors, we just propagate - // it. This code should be removed in mimir 2.12.0. + // it. This code should be removed in mimir 2.14.0. _, ok := httpgrpc.HTTPResponseFromError(pushErr) if ok { return pushErr diff --git a/pkg/distributor/errors.go b/pkg/distributor/errors.go index ca186b993e3..703f2448282 100644 --- a/pkg/distributor/errors.go +++ b/pkg/distributor/errors.go @@ -243,7 +243,7 @@ func handleIngesterPushError(err error) error { if util.IsHTTPStatusCode(statusCode) { // TODO This code is needed for backwards compatibility, since ingesters may still return // errors created by httpgrpc.Errorf(). If pushErr is one of those errors, we just propagate - // it. This code should be removed in mimir 2.12.0. + // it. This code should be removed in mimir 2.14.0. // Wrap HTTP gRPC error with more explanatory message. return httpgrpc.Errorf(int(statusCode), "%s: %s", failedPushingToIngesterMessage, stat.Message()) } @@ -259,7 +259,7 @@ func isClientError(err error) bool { // TODO This code is needed for backwards compatibility, since ingesters may still return // errors with HTTP status code created by httpgrpc.Errorf(). If err is one of those errors, - // we treat 4xx errors as client errors. This code should be removed in mimir 2.12.0. + // we treat 4xx errors as client errors. This code should be removed in mimir 2.14.0. if code := grpcutil.ErrorToStatusCode(err); code/100 == 4 { return true diff --git a/pkg/distributor/errors_test.go b/pkg/distributor/errors_test.go index 289c001ddd6..3ac2cf926ad 100644 --- a/pkg/distributor/errors_test.go +++ b/pkg/distributor/errors_test.go @@ -358,7 +358,7 @@ func TestHandleIngesterPushError(t *testing.T) { // other errors created by httpgrpc with the same code, and with // a more explanatory message. // TODO: this is needed for backwards compatibility and will be removed - // in mimir 2.12.0. + // in mimir 2.14.0. httpgrpcTests := map[string]struct { ingesterPushError error expectedStatus int32 diff --git a/pkg/ingester/active_series.go b/pkg/ingester/active_series.go index 6c466a5f800..890b19aa826 100644 --- a/pkg/ingester/active_series.go +++ b/pkg/ingester/active_series.go @@ -22,7 +22,7 @@ const activeSeriesMaxSizeBytes = 1 * 1024 * 1024 // ActiveSeries implements the ActiveSeries RPC. It returns a stream of active // series that match the given matchers. func (i *Ingester) ActiveSeries(request *client.ActiveSeriesRequest, stream client.Ingester_ActiveSeriesServer) (err error) { - defer func() { err = i.handleReadError(err) }() + defer func() { err = i.mapReadErrorToErrorWithStatus(err) }() if err := i.checkAvailable(); err != nil { return err } diff --git a/pkg/ingester/errors.go b/pkg/ingester/errors.go index 9d99d966d9e..8e2506d0eac 100644 --- a/pkg/ingester/errors.go +++ b/pkg/ingester/errors.go @@ -80,7 +80,7 @@ func newErrorWithStatus(originalErr error, code codes.Code) errorWithStatus { // newErrorWithHTTPStatus creates a new errorWithStatus backed by the given error, // and containing the given HTTP status code. // TODO this is needed for backwards compatibility only and should be removed -// in mimir 2.12.0. +// in mimir 2.14.0. func newErrorWithHTTPStatus(err error, code int) errorWithStatus { errWithHTTPStatus := httpgrpc.Errorf(code, err.Error()) stat, _ := status.FromError(errWithHTTPStatus) @@ -553,7 +553,8 @@ func newIngesterErrSamplers(freq int64) ingesterErrSamplers { } } -func handlePushError(err error) error { +// mapPushErrorToErrorWithStatus maps the given error to the corresponding error of type errorWithStatus. +func mapPushErrorToErrorWithStatus(err error) error { var ( ingesterErr ingesterError errCode = codes.Internal @@ -575,11 +576,11 @@ func handlePushError(err error) error { return newErrorWithStatus(wrappedErr, errCode) } -// handlePushErrorWithHTTPGRPC maps ingesterError objects to an appropriate +// mapPushErrorToErrorWithHTTPOrGRPCStatus maps ingesterError objects to an appropriate // errorWithStatus, which may contain both HTTP and gRPC error codes. // TODO this method is needed only for the backwards compatibility, -// and should be removed in mimir 2.12.0. -func handlePushErrorWithHTTPGRPC(err error) error { +// and should be removed in mimir 2.14.0. +func mapPushErrorToErrorWithHTTPOrGRPCStatus(err error) error { var ingesterErr ingesterError if errors.As(err, &ingesterErr) { switch ingesterErr.errorCause() { @@ -596,7 +597,8 @@ func handlePushErrorWithHTTPGRPC(err error) error { return err } -func handleReadError(err error) error { +// mapReadErrorToErrorWithStatus maps the given error to the corresponding error of type errorWithStatus. +func mapReadErrorToErrorWithStatus(err error) error { var ( ingesterErr ingesterError errCode = codes.Internal @@ -612,11 +614,11 @@ func handleReadError(err error) error { return newErrorWithStatus(err, errCode) } -// handleReadErrorWithHTTPGRPC maps ingesterError objects to an appropriate +// mapReadErrorToErrorWithHTTPOrGRPCStatus maps ingesterError objects to an appropriate // errorWithStatus, which may contain both HTTP and gRPC error codes. // TODO this method is needed only for the backwards compatibility, -// and should be removed in mimir 2.12.0. -func handleReadErrorWithHTTPGRPC(err error) error { +// and should be removed in mimir 2.14.0. +func mapReadErrorToErrorWithHTTPOrGRPCStatus(err error) error { var ( ingesterErr ingesterError ) diff --git a/pkg/ingester/errors_test.go b/pkg/ingester/errors_test.go index 04c2dced534..5dc8bb883c8 100644 --- a/pkg/ingester/errors_test.go +++ b/pkg/ingester/errors_test.go @@ -359,7 +359,7 @@ func TestWrapOrAnnotateWithUser(t *testing.T) { require.Equal(t, wrappingErr, errors.Unwrap(wrappedSafeErr)) } -func TestHandlePushError(t *testing.T) { +func TestMapPushErrorToErrorWithStatus(t *testing.T) { const originalMsg = "this is an error" originalErr := errors.New(originalMsg) labelAdapters := []mimirpb.LabelAdapter{{Name: labels.MetricName, Value: "testmetric"}, {Name: "foo", Value: "biz"}} @@ -512,7 +512,7 @@ func TestHandlePushError(t *testing.T) { for name, tc := range testCases { t.Run(name, func(t *testing.T) { - handledErr := handlePushError(tc.err) + handledErr := mapPushErrorToErrorWithStatus(tc.err) stat, ok := status.FromError(handledErr) require.True(t, ok) require.Equal(t, tc.expectedCode, stat.Code()) @@ -527,7 +527,7 @@ func TestHandlePushError(t *testing.T) { } } -func TestHandlePushErrorWithHTTPGRPC(t *testing.T) { +func TestMapPushErrorToErrorWithHTTPOrGRPCStatus(t *testing.T) { const originalMsg = "this is an error" originalErr := errors.New(originalMsg) labelAdapters := []mimirpb.LabelAdapter{{Name: labels.MetricName, Value: "testmetric"}, {Name: "foo", Value: "biz"}} @@ -677,7 +677,7 @@ func TestHandlePushErrorWithHTTPGRPC(t *testing.T) { for name, tc := range testCases { t.Run(name, func(t *testing.T) { - handledErr := handlePushErrorWithHTTPGRPC(tc.err) + handledErr := mapPushErrorToErrorWithHTTPOrGRPCStatus(tc.err) require.Equal(t, tc.expectedTranslation, handledErr) if tc.doNotLogExpected { var doNotLogError middleware.DoNotLogError @@ -688,7 +688,7 @@ func TestHandlePushErrorWithHTTPGRPC(t *testing.T) { } } -func TestHandleReadError(t *testing.T) { +func TestMapReadErrorToErrorWithStatus(t *testing.T) { const originalMsg = "this is an error" originalErr := errors.New(originalMsg) @@ -731,7 +731,7 @@ func TestHandleReadError(t *testing.T) { } for name, tc := range testCases { t.Run(name, func(t *testing.T) { - handledErr := handleReadError(tc.err) + handledErr := mapReadErrorToErrorWithStatus(tc.err) stat, ok := status.FromError(handledErr) require.True(t, ok) require.Equal(t, tc.expectedCode, stat.Code()) @@ -741,7 +741,7 @@ func TestHandleReadError(t *testing.T) { } } -func TestHandleReadErrorWithHTTPGRPC(t *testing.T) { +func TestMapReadErrorToErrorWithHTTPOrGRPCStatus(t *testing.T) { const originalMsg = "this is an error" originalErr := errors.New(originalMsg) @@ -772,7 +772,7 @@ func TestHandleReadErrorWithHTTPGRPC(t *testing.T) { } for name, tc := range testCases { t.Run(name, func(t *testing.T) { - handledErr := handleReadErrorWithHTTPGRPC(tc.err) + handledErr := mapReadErrorToErrorWithHTTPOrGRPCStatus(tc.err) require.Equal(t, tc.expectedTranslation, handledErr) }) } diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 242e83df96c..62eb147c8e2 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -1271,7 +1271,7 @@ func (i *Ingester) pushSamplesToAppender(userID string, timeseries []mimirpb.Pre } func (i *Ingester) QueryExemplars(ctx context.Context, req *client.ExemplarQueryRequest) (resp *client.ExemplarQueryResponse, err error) { - defer func() { err = i.handleReadError(err) }() + defer func() { err = i.mapReadErrorToErrorWithStatus(err) }() if err := i.checkAvailable(); err != nil { return nil, err } @@ -1329,7 +1329,7 @@ func (i *Ingester) QueryExemplars(ctx context.Context, req *client.ExemplarQuery } func (i *Ingester) LabelValues(ctx context.Context, req *client.LabelValuesRequest) (resp *client.LabelValuesResponse, err error) { - defer func() { err = i.handleReadError(err) }() + defer func() { err = i.mapReadErrorToErrorWithStatus(err) }() if err := i.checkAvailable(); err != nil { return nil, err } @@ -1369,7 +1369,7 @@ func (i *Ingester) LabelValues(ctx context.Context, req *client.LabelValuesReque } func (i *Ingester) LabelNames(ctx context.Context, req *client.LabelNamesRequest) (resp *client.LabelNamesResponse, err error) { - defer func() { err = i.handleReadError(err) }() + defer func() { err = i.mapReadErrorToErrorWithStatus(err) }() if err := i.checkAvailable(); err != nil { return nil, err } @@ -1410,7 +1410,7 @@ func (i *Ingester) LabelNames(ctx context.Context, req *client.LabelNamesRequest // MetricsForLabelMatchers implements IngesterServer. func (i *Ingester) MetricsForLabelMatchers(ctx context.Context, req *client.MetricsForLabelMatchersRequest) (resp *client.MetricsForLabelMatchersResponse, err error) { - defer func() { err = i.handleReadError(err) }() + defer func() { err = i.mapReadErrorToErrorWithStatus(err) }() if err := i.checkAvailable(); err != nil { return nil, err } @@ -1481,7 +1481,7 @@ func (i *Ingester) MetricsForLabelMatchers(ctx context.Context, req *client.Metr } func (i *Ingester) UserStats(ctx context.Context, req *client.UserStatsRequest) (resp *client.UserStatsResponse, err error) { - defer func() { err = i.handleReadError(err) }() + defer func() { err = i.mapReadErrorToErrorWithStatus(err) }() if err := i.checkAvailable(); err != nil { return nil, err } @@ -1503,7 +1503,7 @@ func (i *Ingester) UserStats(ctx context.Context, req *client.UserStatsRequest) } func (i *Ingester) AllUserStats(_ context.Context, req *client.UserStatsRequest) (resp *client.UsersStatsResponse, err error) { - defer func() { err = i.handleReadError(err) }() + defer func() { err = i.mapReadErrorToErrorWithStatus(err) }() if err := i.checkAvailable(); err != nil { return nil, err } @@ -1534,7 +1534,7 @@ func (i *Ingester) AllUserStats(_ context.Context, req *client.UserStatsRequest) const labelNamesAndValuesTargetSizeBytes = 1 * 1024 * 1024 func (i *Ingester) LabelNamesAndValues(request *client.LabelNamesAndValuesRequest, stream client.Ingester_LabelNamesAndValuesServer) (err error) { - defer func() { err = i.handleReadError(err) }() + defer func() { err = i.mapReadErrorToErrorWithStatus(err) }() if err := i.checkAvailable(); err != nil { return err } @@ -1567,7 +1567,7 @@ func (i *Ingester) LabelNamesAndValues(request *client.LabelNamesAndValuesReques const labelValuesCardinalityTargetSizeBytes = 1 * 1024 * 1024 func (i *Ingester) LabelValuesCardinality(req *client.LabelValuesCardinalityRequest, srv client.Ingester_LabelValuesCardinalityServer) (err error) { - defer func() { err = i.handleReadError(err) }() + defer func() { err = i.mapReadErrorToErrorWithStatus(err) }() if err := i.checkAvailable(); err != nil { return err } @@ -1648,7 +1648,7 @@ const queryStreamBatchMessageSize = 1 * 1024 * 1024 // QueryStream streams metrics from a TSDB. This implements the client.IngesterServer interface func (i *Ingester) QueryStream(req *client.QueryRequest, stream client.Ingester_QueryStreamServer) (err error) { - defer func() { err = i.handleReadError(err) }() + defer func() { err = i.mapReadErrorToErrorWithStatus(err) }() if err := i.checkAvailable(); err != nil { return err } @@ -3172,18 +3172,18 @@ func (i *Ingester) Push(ctx context.Context, req *mimirpb.WriteRequest) (*mimirp if err == nil { return &mimirpb.WriteResponse{}, nil } - handledErr := i.handlePushError(err) + handledErr := i.mapPushErrorToErrorWithStatus(err) return nil, handledErr } -func (i *Ingester) handlePushError(err error) error { +func (i *Ingester) mapPushErrorToErrorWithStatus(err error) error { if i.cfg.ReturnOnlyGRPCErrors { - return handlePushError(err) + return mapPushErrorToErrorWithStatus(err) } - return handlePushErrorWithHTTPGRPC(err) + return mapPushErrorToErrorWithHTTPOrGRPCStatus(err) } -func (i *Ingester) handleReadError(err error) error { +func (i *Ingester) mapReadErrorToErrorWithStatus(err error) error { if err == nil { return nil } @@ -3193,9 +3193,9 @@ func (i *Ingester) handleReadError(err error) error { } if i.cfg.ReturnOnlyGRPCErrors { - return handleReadError(err) + return mapReadErrorToErrorWithStatus(err) } - return handleReadErrorWithHTTPGRPC(err) + return mapReadErrorToErrorWithHTTPOrGRPCStatus(err) } // pushMetadata returns number of ingested metadata. @@ -3298,7 +3298,7 @@ func (i *Ingester) purgeUserMetricsMetadata() { // MetricsMetadata returns all the metrics metadata of a user. func (i *Ingester) MetricsMetadata(ctx context.Context, req *client.MetricsMetadataRequest) (resp *client.MetricsMetadataResponse, err error) { - defer func() { err = i.handleReadError(err) }() + defer func() { err = i.mapReadErrorToErrorWithStatus(err) }() if err := i.checkAvailable(); err != nil { return nil, err } diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index 633a304dce3..1f27731db79 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -1826,7 +1826,7 @@ func TestIngester_Push(t *testing.T) { if testData.expectedErr == nil { assert.NoError(t, err) } else { - handledErr := i.handlePushError(err) + handledErr := i.mapPushErrorToErrorWithStatus(err) errWithStatus, ok := handledErr.(errorWithStatus) assert.True(t, ok) assert.True(t, errWithStatus.equals(testData.expectedErr)) From 6527484d6fc2dd2a4e065261ec5382ba247ba2d2 Mon Sep 17 00:00:00 2001 From: Yuri Nikolic Date: Fri, 24 Nov 2023 11:31:18 +0100 Subject: [PATCH 5/7] Remove mimir version's mentions from TODOs Signed-off-by: Yuri Nikolic --- pkg/distributor/distributor.go | 2 +- pkg/distributor/errors.go | 5 +++-- pkg/distributor/errors_test.go | 4 ++-- pkg/ingester/errors.go | 12 ++++++------ 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/pkg/distributor/distributor.go b/pkg/distributor/distributor.go index aeae71fb41e..48641bd1b2d 100644 --- a/pkg/distributor/distributor.go +++ b/pkg/distributor/distributor.go @@ -1247,7 +1247,7 @@ func (d *Distributor) handlePushError(ctx context.Context, pushErr error) error // TODO This code is needed for backwards compatibility, since ingesters may still return // errors created by httpgrpc.Errorf(). If pushErr is one of those errors, we just propagate - // it. This code should be removed in mimir 2.14.0. + // it. This code should be removed together with the removal of `-ingester.return-only-grpc-errors`. _, ok := httpgrpc.HTTPResponseFromError(pushErr) if ok { return pushErr diff --git a/pkg/distributor/errors.go b/pkg/distributor/errors.go index 703f2448282..3fe6a366c45 100644 --- a/pkg/distributor/errors.go +++ b/pkg/distributor/errors.go @@ -243,7 +243,7 @@ func handleIngesterPushError(err error) error { if util.IsHTTPStatusCode(statusCode) { // TODO This code is needed for backwards compatibility, since ingesters may still return // errors created by httpgrpc.Errorf(). If pushErr is one of those errors, we just propagate - // it. This code should be removed in mimir 2.14.0. + // it. This code should be removed together with the removal of `-ingester.return-only-grpc-errors`. // Wrap HTTP gRPC error with more explanatory message. return httpgrpc.Errorf(int(statusCode), "%s: %s", failedPushingToIngesterMessage, stat.Message()) } @@ -259,7 +259,8 @@ func isClientError(err error) bool { // TODO This code is needed for backwards compatibility, since ingesters may still return // errors with HTTP status code created by httpgrpc.Errorf(). If err is one of those errors, - // we treat 4xx errors as client errors. This code should be removed in mimir 2.14.0. + // we treat 4xx errors as client errors. This code should be removed together with the removal + // of `-ingester.return-only-grpc-errors`. if code := grpcutil.ErrorToStatusCode(err); code/100 == 4 { return true diff --git a/pkg/distributor/errors_test.go b/pkg/distributor/errors_test.go index 3ac2cf926ad..88ea098187f 100644 --- a/pkg/distributor/errors_test.go +++ b/pkg/distributor/errors_test.go @@ -357,8 +357,8 @@ func TestHandleIngesterPushError(t *testing.T) { // Ensure that the errors created by httpgrpc get translated into // other errors created by httpgrpc with the same code, and with // a more explanatory message. - // TODO: this is needed for backwards compatibility and will be removed - // in mimir 2.14.0. + // TODO this method is needed only for the backwards compatibility, and should be removed + // together with the removal of `-ingester.return-only-grpc-errors`. httpgrpcTests := map[string]struct { ingesterPushError error expectedStatus int32 diff --git a/pkg/ingester/errors.go b/pkg/ingester/errors.go index 8e2506d0eac..943181ee708 100644 --- a/pkg/ingester/errors.go +++ b/pkg/ingester/errors.go @@ -79,8 +79,8 @@ func newErrorWithStatus(originalErr error, code codes.Code) errorWithStatus { // newErrorWithHTTPStatus creates a new errorWithStatus backed by the given error, // and containing the given HTTP status code. -// TODO this is needed for backwards compatibility only and should be removed -// in mimir 2.14.0. +// TODO this method is needed only for the backwards compatibility, and should be removed +// together with the removal of `-ingester.return-only-grpc-errors`. func newErrorWithHTTPStatus(err error, code int) errorWithStatus { errWithHTTPStatus := httpgrpc.Errorf(code, err.Error()) stat, _ := status.FromError(errWithHTTPStatus) @@ -578,8 +578,8 @@ func mapPushErrorToErrorWithStatus(err error) error { // mapPushErrorToErrorWithHTTPOrGRPCStatus maps ingesterError objects to an appropriate // errorWithStatus, which may contain both HTTP and gRPC error codes. -// TODO this method is needed only for the backwards compatibility, -// and should be removed in mimir 2.14.0. +// TODO this method is needed only for the backwards compatibility, and should be removed +// together with the removal of `-ingester.return-only-grpc-errors`. func mapPushErrorToErrorWithHTTPOrGRPCStatus(err error) error { var ingesterErr ingesterError if errors.As(err, &ingesterErr) { @@ -616,8 +616,8 @@ func mapReadErrorToErrorWithStatus(err error) error { // mapReadErrorToErrorWithHTTPOrGRPCStatus maps ingesterError objects to an appropriate // errorWithStatus, which may contain both HTTP and gRPC error codes. -// TODO this method is needed only for the backwards compatibility, -// and should be removed in mimir 2.14.0. +// TODO this method is needed only for the backwards compatibility, and should be removed +// together with the removal of `-ingester.return-only-grpc-errors`. func mapReadErrorToErrorWithHTTPOrGRPCStatus(err error) error { var ( ingesterErr ingesterError From 496f03fe501d718bb37ee8b8a698f425b65887ea Mon Sep 17 00:00:00 2001 From: Yuri Nikolic Date: Fri, 24 Nov 2023 11:58:07 +0100 Subject: [PATCH 6/7] Remove mimir version's mentions from TODOs Signed-off-by: Yuri Nikolic --- pkg/distributor/distributor.go | 5 ++--- pkg/distributor/errors.go | 12 ++++-------- pkg/distributor/errors_test.go | 2 -- pkg/ingester/errors.go | 6 ------ 4 files changed, 6 insertions(+), 19 deletions(-) diff --git a/pkg/distributor/distributor.go b/pkg/distributor/distributor.go index 48641bd1b2d..6e738da786f 100644 --- a/pkg/distributor/distributor.go +++ b/pkg/distributor/distributor.go @@ -1245,9 +1245,8 @@ func (d *Distributor) handlePushError(ctx context.Context, pushErr error) error return pushErr } - // TODO This code is needed for backwards compatibility, since ingesters may still return - // errors created by httpgrpc.Errorf(). If pushErr is one of those errors, we just propagate - // it. This code should be removed together with the removal of `-ingester.return-only-grpc-errors`. + // This code is needed for backwards compatibility, since ingesters may still return errors + // created by httpgrpc.Errorf(). If pushErr is one of those errors, we just propagate it. _, ok := httpgrpc.HTTPResponseFromError(pushErr) if ok { return pushErr diff --git a/pkg/distributor/errors.go b/pkg/distributor/errors.go index 3fe6a366c45..79379745061 100644 --- a/pkg/distributor/errors.go +++ b/pkg/distributor/errors.go @@ -241,9 +241,8 @@ func handleIngesterPushError(err error) error { } statusCode := stat.Code() if util.IsHTTPStatusCode(statusCode) { - // TODO This code is needed for backwards compatibility, since ingesters may still return - // errors created by httpgrpc.Errorf(). If pushErr is one of those errors, we just propagate - // it. This code should be removed together with the removal of `-ingester.return-only-grpc-errors`. + // This code is needed for backwards compatibility, since ingesters may still return errors + // created by httpgrpc.Errorf(). If pushErr is one of those errors, we just propagate it. // Wrap HTTP gRPC error with more explanatory message. return httpgrpc.Errorf(int(statusCode), "%s: %s", failedPushingToIngesterMessage, stat.Message()) } @@ -257,11 +256,8 @@ func isClientError(err error) bool { return ingesterPushErr.errorCause() == mimirpb.BAD_DATA } - // TODO This code is needed for backwards compatibility, since ingesters may still return - // errors with HTTP status code created by httpgrpc.Errorf(). If err is one of those errors, - // we treat 4xx errors as client errors. This code should be removed together with the removal - // of `-ingester.return-only-grpc-errors`. - + // This code is needed for backwards compatibility, since ingesters may still return errors with HTTP status + // code created by httpgrpc.Errorf(). If err is one of those errors, we treat 4xx errors as client errors. if code := grpcutil.ErrorToStatusCode(err); code/100 == 4 { return true } diff --git a/pkg/distributor/errors_test.go b/pkg/distributor/errors_test.go index 88ea098187f..0757f1be3b6 100644 --- a/pkg/distributor/errors_test.go +++ b/pkg/distributor/errors_test.go @@ -357,8 +357,6 @@ func TestHandleIngesterPushError(t *testing.T) { // Ensure that the errors created by httpgrpc get translated into // other errors created by httpgrpc with the same code, and with // a more explanatory message. - // TODO this method is needed only for the backwards compatibility, and should be removed - // together with the removal of `-ingester.return-only-grpc-errors`. httpgrpcTests := map[string]struct { ingesterPushError error expectedStatus int32 diff --git a/pkg/ingester/errors.go b/pkg/ingester/errors.go index 943181ee708..2c64b61f48b 100644 --- a/pkg/ingester/errors.go +++ b/pkg/ingester/errors.go @@ -79,8 +79,6 @@ func newErrorWithStatus(originalErr error, code codes.Code) errorWithStatus { // newErrorWithHTTPStatus creates a new errorWithStatus backed by the given error, // and containing the given HTTP status code. -// TODO this method is needed only for the backwards compatibility, and should be removed -// together with the removal of `-ingester.return-only-grpc-errors`. func newErrorWithHTTPStatus(err error, code int) errorWithStatus { errWithHTTPStatus := httpgrpc.Errorf(code, err.Error()) stat, _ := status.FromError(errWithHTTPStatus) @@ -578,8 +576,6 @@ func mapPushErrorToErrorWithStatus(err error) error { // mapPushErrorToErrorWithHTTPOrGRPCStatus maps ingesterError objects to an appropriate // errorWithStatus, which may contain both HTTP and gRPC error codes. -// TODO this method is needed only for the backwards compatibility, and should be removed -// together with the removal of `-ingester.return-only-grpc-errors`. func mapPushErrorToErrorWithHTTPOrGRPCStatus(err error) error { var ingesterErr ingesterError if errors.As(err, &ingesterErr) { @@ -616,8 +612,6 @@ func mapReadErrorToErrorWithStatus(err error) error { // mapReadErrorToErrorWithHTTPOrGRPCStatus maps ingesterError objects to an appropriate // errorWithStatus, which may contain both HTTP and gRPC error codes. -// TODO this method is needed only for the backwards compatibility, and should be removed -// together with the removal of `-ingester.return-only-grpc-errors`. func mapReadErrorToErrorWithHTTPOrGRPCStatus(err error) error { var ( ingesterErr ingesterError From 04eaa1130d71fcbd64dda52fc1bda4ffe0a27bed Mon Sep 17 00:00:00 2001 From: Yuri Nikolic Date: Fri, 24 Nov 2023 12:18:24 +0100 Subject: [PATCH 7/7] Remove mimir version's mentions from TODOs Signed-off-by: Yuri Nikolic --- pkg/util/http.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/util/http.go b/pkg/util/http.go index e5a828d2cc5..0b5c6a418c5 100644 --- a/pkg/util/http.go +++ b/pkg/util/http.go @@ -346,6 +346,5 @@ func copyValues(src url.Values) url.Values { // IsHTTPStatusCode returns true if the given code is a valid HTTP status code, or false otherwise. func IsHTTPStatusCode(code codes.Code) bool { - httpStatus := http.StatusText(int(code)) - return httpStatus != "" + return int(code) >= 100 && int(code) < 600 }