Instrumentation: Add histogram for request duration on gRPC client to…

… Ingesters (#3409) * Instrumentation: Add histogram for request duration on gRPC client to Ingesters Signed-off-by: gotjosh <[email protected]> * fix lint Co-authored-by: Edward Welch <[email protected]>
grafana · Mar 1, 2021 · 144786e · 144786e
1 parent e57af51
commit 144786e
Showing 1 changed file with 12 additions and 0 deletions.
diff --git a/pkg/ingester/client/client.go b/pkg/ingester/client/client.go
@@ -9,13 +9,23 @@ import (
 	"github.com/cortexproject/cortex/pkg/util/grpcclient"
 	"github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc"
 	"github.com/opentracing/opentracing-go"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 	"github.com/weaveworks/common/middleware"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/health/grpc_health_v1"
 
+	cortex_middleware "github.com/cortexproject/cortex/pkg/util/middleware"
+
 	"github.com/grafana/loki/pkg/logproto"
 )
 
+var ingesterClientRequestDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
+	Name:    "loki_ingester_client_request_duration_seconds",
+	Help:    "Time spent doing Ingester requests.",
+	Buckets: prometheus.ExponentialBuckets(0.001, 4, 6),
+}, []string{"operation", "status_code"})
+
 type HealthAndIngesterClient interface {
 	logproto.IngesterClient
 	grpc_health_v1.HealthClient
@@ -76,8 +86,10 @@ func instrumentation() ([]grpc.UnaryClientInterceptor, []grpc.StreamClientInterc
 	return []grpc.UnaryClientInterceptor{
 			otgrpc.OpenTracingClientInterceptor(opentracing.GlobalTracer()),
 			middleware.ClientUserHeaderInterceptor,
+			cortex_middleware.PrometheusGRPCUnaryInstrumentation(ingesterClientRequestDuration),
 		}, []grpc.StreamClientInterceptor{
 			otgrpc.OpenTracingStreamClientInterceptor(opentracing.GlobalTracer()),
 			middleware.StreamClientUserHeaderInterceptor,
+			cortex_middleware.PrometheusGRPCStreamInstrumentation(ingesterClientRequestDuration),
 		}
 }