Add querier metrics for requests executed (#3524)

* Add metrics for concurrency and querier requests executed * update changelog * turn metric into log line * update
grafana · Mar 27, 2024 · 6cafca9 · 6cafca9
1 parent 56eb0cb
commit 6cafca9
Show file tree

Hide file tree

Showing 4 changed files with 20 additions and 0 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,6 @@
 ## main / unreleased
 
+* [ENHANCEMENT] Add querier metrics for requests executed [#3524](https://github.com/grafana/tempo/pull/3524) (@electron0zero)
 * [FEATURE] Added gRPC streaming endpoints for all tag queries. [#3460](https://github.com/grafana/tempo/pull/3460) (@joe-elliott)
 * [CHANGE] Align metrics query time ranges to the step parameter [#3490](https://github.com/grafana/tempo/pull/3490) (@mdisibio)
 * [ENHANCEMENT] Add string interning to TraceQL queries [#3411](https://github.com/grafana/tempo/pull/3411) (@mapno)

diff --git a/modules/querier/worker/frontend_processor.go b/modules/querier/worker/frontend_processor.go
@@ -15,6 +15,8 @@ import (
 	"github.com/grafana/dskit/httpgrpc"
 	"github.com/grafana/tempo/pkg/util/httpgrpcutil"
 	"github.com/opentracing/opentracing-go"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
@@ -27,6 +29,12 @@ var processorBackoffConfig = backoff.Config{
 	MaxBackoff: 1 * time.Second,
 }
 
+var metricWorkerRequests = promauto.NewCounter(prometheus.CounterOpts{
+	Namespace: "tempo",
+	Name:      "querier_worker_request_executed_total",
+	Help:      "The total number of requests executed by the querier worker.",
+})
+
 func newFrontendProcessor(cfg Config, handler RequestHandler, log log.Logger) processor {
 	return &frontendProcessor{
 		log:            log,
@@ -185,6 +193,8 @@ func (fp *frontendProcessor) runRequest(ctx context.Context, request *httpgrpc.H
 		level.Error(fp.log).Log("msg", "error processing query", "err", errMsg)
 	}
 
+	metricWorkerRequests.Inc()
+
 	return response
 }
 

diff --git a/modules/querier/worker/frontend_processor_test.go b/modules/querier/worker/frontend_processor_test.go
@@ -11,6 +11,7 @@ import (
 	"github.com/go-kit/log"
 	"github.com/grafana/dskit/grpcclient"
 	"github.com/grafana/dskit/httpgrpc"
+	dto "github.com/prometheus/client_model/go"
 	"github.com/stretchr/testify/require"
 )
 
@@ -46,6 +47,12 @@ func TestRunRequests(t *testing.T) {
 	for i, resp := range resps {
 		require.Equal(t, []byte{byte(i)}, resp.Body)
 	}
+
+	// check that counter metric is working
+	m := &dto.Metric{}
+	err := metricWorkerRequests.Write(m)
+	require.NoError(t, err)
+	require.Equal(t, float64(totalRequests), m.Counter.GetValue())
 }
 
 func TestHandleSendError(t *testing.T) {

diff --git a/modules/querier/worker/worker.go b/modules/querier/worker/worker.go
@@ -242,6 +242,8 @@ func (w *querierWorker) resetConcurrency() {
 	if totalConcurrency > w.cfg.MaxConcurrentRequests {
 		level.Warn(w.log).Log("msg", "total worker concurrency is greater than promql max concurrency. Queries may be queued in the querier which reduces QOS")
 	}
+
+	level.Info(w.log).Log("msg", "total worker concurrency updated", "totalConcurrency", totalConcurrency)
 }
 
 func (w *querierWorker) connect(ctx context.Context, address string) (*grpc.ClientConn, error) {