Add error count prom metric (#14670)

* add error count prom metric

* address review comments

* add comment for response writer

* update changelog

CHANGELOG.md

@@ -29,6 +29,7 @@ The format is based on Keep a Changelog, and this project adheres to Semantic Ve
 - PeerDAS: Added proto for `DataColumnIdentifier`, `DataColumnSidecar`, `DataColumnSidecarsByRangeRequest` and `MetadataV2`.
 - Better attestation packing for Electra. [PR](https://github.com/prysmaticlabs/prysm/pull/14534)
 - P2P: Add logs when a peer is (dis)connected. Add the reason of the disconnection when we initiate it.
+- Added a Prometheus error counter metric for HTTP requests to track beacon node requests.
 
 ### Changed
 

beacon-chain/rpc/BUILD.bazel

@@ -5,6 +5,7 @@ go_library(
     srcs = [
         "endpoints.go",
         "log.go",
+        "metrics.go",
         "service.go",
     ],
     importpath = "github.com/prysmaticlabs/prysm/v5/beacon-chain/rpc",

beacon-chain/rpc/endpoints.go

@@ -34,18 +34,41 @@ type endpoint struct {
 	methods    []string
 }
 
+// responseWriter is the wrapper to http Response writer.
+type responseWriter struct {
+	http.ResponseWriter
+	statusCode int
+}
+
+// WriteHeader wraps the WriteHeader method of the underlying http.ResponseWriter to capture the status code.
+// Refer for WriteHeader doc: https://pkg.go.dev/net/[email protected]#ResponseWriter.
+func (w *responseWriter) WriteHeader(statusCode int) {
+	w.statusCode = statusCode
+	w.ResponseWriter.WriteHeader(statusCode)
+}
+
 func (e *endpoint) handlerWithMiddleware() http.HandlerFunc {
 	handler := http.Handler(e.handler)
 	for _, m := range e.middleware {
 		handler = m(handler)
 	}
-	return promhttp.InstrumentHandlerDuration(
+
+	handler = promhttp.InstrumentHandlerDuration(
 		httpRequestLatency.MustCurryWith(prometheus.Labels{"endpoint": e.name}),
 		promhttp.InstrumentHandlerCounter(
 			httpRequestCount.MustCurryWith(prometheus.Labels{"endpoint": e.name}),
 			handler,
 		),
 	)
+
+	return func(w http.ResponseWriter, r *http.Request) {
+		rw := &responseWriter{ResponseWriter: w, statusCode: http.StatusOK}
+		handler.ServeHTTP(rw, r)
+
+		if rw.statusCode >= 400 {
+			httpErrorCount.WithLabelValues(r.URL.Path, http.StatusText(rw.statusCode), r.Method).Inc()
+		}
+	}
 }
 
 func (s *Service) endpoints(

beacon-chain/rpc/metrics.go

@@ -0,0 +1,31 @@
+package rpc
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+var (
+	httpRequestLatency = promauto.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name:    "http_request_latency_seconds",
+			Help:    "Latency of HTTP requests in seconds",
+			Buckets: []float64{0.001, 0.01, 0.025, 0.1, 0.25, 1, 2.5, 10},
+		},
+		[]string{"endpoint", "code", "method"},
+	)
+	httpRequestCount = promauto.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "http_request_count",
+			Help: "Number of HTTP requests",
+		},
+		[]string{"endpoint", "code", "method"},
+	)
+	httpErrorCount = promauto.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "http_error_count",
+			Help: "Total HTTP errors for beacon node requests",
+		},
+		[]string{"endpoint", "code", "method"},
+	)
+)

beacon-chain/rpc/service.go

@@ -14,8 +14,6 @@ import (
 	grpcopentracing "github.com/grpc-ecosystem/go-grpc-middleware/tracing/opentracing"
 	grpcprometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
 	"github.com/pkg/errors"
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto"
 	"github.com/prysmaticlabs/prysm/v5/beacon-chain/blockchain"
 	"github.com/prysmaticlabs/prysm/v5/beacon-chain/builder"
 	"github.com/prysmaticlabs/prysm/v5/beacon-chain/cache"
@@ -57,24 +55,6 @@ import (
 
 const attestationBufferSize = 100
 
-var (
-	httpRequestLatency = promauto.NewHistogramVec(
-		prometheus.HistogramOpts{
-			Name:    "http_request_latency_seconds",
-			Help:    "Latency of HTTP requests in seconds",
-			Buckets: []float64{0.001, 0.01, 0.025, 0.1, 0.25, 1, 2.5, 10},
-		},
-		[]string{"endpoint", "code", "method"},
-	)
-	httpRequestCount = promauto.NewCounterVec(
-		prometheus.CounterOpts{
-			Name: "http_request_count",
-			Help: "Number of HTTP requests",
-		},
-		[]string{"endpoint", "code", "method"},
-	)
-)
-
 // Service defining an RPC server for a beacon node.
 type Service struct {
 	cfg                  *Config