Introduce cortex_ingest_storage_reader_records_failed_total and corte…

…x_ingest_storage_reader_records_total

Signed-off-by: Dimitar Dimitrov <[email protected]>

pkg/storage/ingest/pusher.go

@@ -28,6 +28,7 @@ type pusherConsumer struct {
 
 	processingTimeSeconds prometheus.Observer
 	clientErrRequests     prometheus.Counter
+	serverErrRequests     prometheus.Counter
 	totalRequests         prometheus.Counter
 	l                     log.Logger
 }
@@ -39,6 +40,11 @@ type parsedRecord struct {
 }
 
 func newPusherConsumer(p Pusher, reg prometheus.Registerer, l log.Logger) *pusherConsumer {
+	errRequestsCounter := promauto.With(reg).NewCounterVec(prometheus.CounterOpts{
+		Name: "cortex_ingest_storage_reader_records_failed_total",
+		Help: "Number of records (write requests) which caused errors while processing. Client errors are errors such as tenant limits and samples out of bounds. Server errors indicate internal recoverable errors.",
+	}, []string{"cause"})
+
 	return &pusherConsumer{
 		p: p,
 		l: l,
@@ -49,13 +55,11 @@ func newPusherConsumer(p Pusher, reg prometheus.Registerer, l log.Logger) *pushe
 			MaxAge:     time.Minute,
 			AgeBuckets: 10,
 		}),
-		clientErrRequests: promauto.With(reg).NewCounter(prometheus.CounterOpts{
-			Name: "cortex_ingest_storage_reader_client_error_requests_total",
-			Help: "Number of write requests which caused client errors (e.g. tenant limits, samples out of bounds).",
-		}),
+		clientErrRequests: errRequestsCounter.WithLabelValues("client"),
+		serverErrRequests: errRequestsCounter.WithLabelValues("server"),
 		totalRequests: promauto.With(reg).NewCounter(prometheus.CounterOpts{
-			Name: "cortex_ingest_storage_reader_requests_total",
-			Help: "Number of attempted write requests.",
+			Name: "cortex_ingest_storage_reader_records_total",
+			Help: "Number of attempted records (write requests).",
 		}),
 	}
 }
@@ -91,6 +95,7 @@ func (c pusherConsumer) pushRequests(ctx context.Context, reqC <-chan parsedReco
 		c.totalRequests.Inc()
 		if err != nil {
 			if !isClientIngesterError(err) {
+				c.serverErrRequests.Inc()
 				return fmt.Errorf("consuming record at index %d for tenant %s: %w", recordIdx, wr.tenantID, err)
 			}
 			c.clientErrRequests.Inc()