From f72a6e1415e88c1b015c78a53a7a0d163b7b4b16 Mon Sep 17 00:00:00 2001 From: Goutham Veeramachaneni Date: Mon, 17 Dec 2018 12:20:14 +0530 Subject: [PATCH 1/4] Fix stream tracking metric Signed-off-by: Goutham Veeramachaneni --- pkg/ingester/flush.go | 1 + pkg/ingester/instance.go | 18 ++++++++++-------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/pkg/ingester/flush.go b/pkg/ingester/flush.go index c8870dbcd908b..8e6b8aab3201a 100644 --- a/pkg/ingester/flush.go +++ b/pkg/ingester/flush.go @@ -195,6 +195,7 @@ func (i *Ingester) removeFlushedChunks(instance *instance, stream *stream) { if len(stream.chunks) == 0 { delete(instance.streams, stream.fp) + instance.streamsRemovedTotal.Inc() } } diff --git a/pkg/ingester/instance.go b/pkg/ingester/instance.go index 99b74a39d98a1..7de6f6ebac809 100644 --- a/pkg/ingester/instance.go +++ b/pkg/ingester/instance.go @@ -6,6 +6,7 @@ import ( "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/common/model" "github.com/prometheus/prometheus/pkg/labels" @@ -28,37 +29,37 @@ var ( ) var ( - streamsCreatedTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + streamsCreatedTotal = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: "loki", Name: "ingester_streams_created_total", Help: "The total number of streams created in the ingester.", }, []string{"org"}) - streamsRemovedTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + streamsRemovedTotal = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: "loki", Name: "ingester_streams_removed_total", Help: "The total number of streams removed by the ingester.", }, []string{"org"}) ) -func init() { - prometheus.MustRegister(streamsCreatedTotal) - prometheus.MustRegister(streamsRemovedTotal) -} - type instance struct { streamsMtx sync.RWMutex streams map[model.Fingerprint]*stream index *index.InvertedIndex instanceID string + + streamsCreatedTotal prometheus.Counter + streamsRemovedTotal prometheus.Counter } func newInstance(instanceID string) *instance { - streamsCreatedTotal.WithLabelValues(instanceID).Inc() return &instance{ streams: map[model.Fingerprint]*stream{}, index: index.New(), instanceID: instanceID, + + streamsCreatedTotal: streamsCreatedTotal.WithLabelValues(instanceID), + streamsRemovedTotal: streamsRemovedTotal.WithLabelValues(instanceID), } } @@ -78,6 +79,7 @@ func (i *instance) Push(ctx context.Context, req *logproto.PushRequest) error { stream = newStream(fp, labels) i.index.Add(labels, fp) i.streams[fp] = stream + i.streamsCreatedTotal.Inc() } if err := stream.Push(ctx, s.Entries); err != nil { From f272a889223a5dc7391da0669a27cd99c64261a6 Mon Sep 17 00:00:00 2001 From: Goutham Veeramachaneni Date: Mon, 17 Dec 2018 12:25:35 +0530 Subject: [PATCH 2/4] Track lines and bytes received per user Signed-off-by: Goutham Veeramachaneni --- pkg/distributor/distributor.go | 36 +++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/pkg/distributor/distributor.go b/pkg/distributor/distributor.go index f22d498c03235..e0b7f45c7cd61 100644 --- a/pkg/distributor/distributor.go +++ b/pkg/distributor/distributor.go @@ -11,6 +11,7 @@ import ( "github.com/cortexproject/cortex/pkg/util" opentracing "github.com/opentracing/opentracing-go" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" "github.com/weaveworks/common/user" "google.golang.org/grpc/health/grpc_health_v1" @@ -19,29 +20,34 @@ import ( ) var ( - sendDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + sendDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ Namespace: "loki", Name: "distributor_send_duration_seconds", Help: "Time spent sending a sample batch to multiple replicated ingesters.", Buckets: []float64{.001, .0025, .005, .01, .025, .05, .1, .25, .5, 1}, }, []string{"method", "status_code"}) - ingesterAppends = prometheus.NewCounterVec(prometheus.CounterOpts{ + ingesterAppends = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: "loki", Name: "distributor_ingester_appends_total", Help: "The total number of batch appends sent to ingesters.", }, []string{"ingester"}) - ingesterAppendFailures = prometheus.NewCounterVec(prometheus.CounterOpts{ + ingesterAppendFailures = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: "loki", Name: "distributor_ingester_append_failures_total", Help: "The total number of failed batch appends sent to ingesters.", }, []string{"ingester"}) -) -func init() { - prometheus.MustRegister(sendDuration) - prometheus.MustRegister(ingesterAppends) - prometheus.MustRegister(ingesterAppendFailures) -} + bytesIngested = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "loki", + Name: "distributor_bytes_received_total", + Help: "The total number of uncompressed bytes received per user", + }, []string{"org"}) + linesIngested = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "loki", + Name: "distributor_lines_received_total", + Help: "The total number of lines received per user", + }, []string{"org"}) +) // Config for a Distributor. type Config struct { @@ -97,6 +103,18 @@ func (d *Distributor) Push(ctx context.Context, req *logproto.PushRequest) (*log return nil, err } + // Track metrics. + bytesCount := 0 + lineCount := 0 + for _, stream := range req.Streams { + for _, entry := range stream.Entries { + bytesCount += len(entry.Line) + lineCount++ + } + } + bytesIngested.WithLabelValues(userID).Add(float64(bytesCount)) + linesIngested.WithLabelValues(userID).Add(float64(lineCount)) + // First we flatten out the request into a list of samples. // We use the heuristic of 1 sample per TS to size the array. // We also work out the hash value at the same time. From 9d0b98b2f30a6c3ddb0a38cff5ece4a325a59931 Mon Sep 17 00:00:00 2001 From: Goutham Veeramachaneni Date: Mon, 17 Dec 2018 16:40:26 +0530 Subject: [PATCH 3/4] Use instance instead of org Signed-off-by: Goutham Veeramachaneni --- pkg/distributor/distributor.go | 8 ++++---- pkg/ingester/instance.go | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pkg/distributor/distributor.go b/pkg/distributor/distributor.go index e0b7f45c7cd61..56aec50688542 100644 --- a/pkg/distributor/distributor.go +++ b/pkg/distributor/distributor.go @@ -40,13 +40,13 @@ var ( bytesIngested = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: "loki", Name: "distributor_bytes_received_total", - Help: "The total number of uncompressed bytes received per user", - }, []string{"org"}) + Help: "The total number of uncompressed bytes received per instance", + }, []string{"instance"}) linesIngested = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: "loki", Name: "distributor_lines_received_total", - Help: "The total number of lines received per user", - }, []string{"org"}) + Help: "The total number of lines received per instance", + }, []string{"instance"}) ) // Config for a Distributor. diff --git a/pkg/ingester/instance.go b/pkg/ingester/instance.go index 7de6f6ebac809..90204ac628b59 100644 --- a/pkg/ingester/instance.go +++ b/pkg/ingester/instance.go @@ -32,13 +32,13 @@ var ( streamsCreatedTotal = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: "loki", Name: "ingester_streams_created_total", - Help: "The total number of streams created in the ingester.", - }, []string{"org"}) + Help: "The total number of streams created per instance in the ingester.", + }, []string{"instance"}) streamsRemovedTotal = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: "loki", Name: "ingester_streams_removed_total", - Help: "The total number of streams removed by the ingester.", - }, []string{"org"}) + Help: "The total number of streams removed per instance by the ingester.", + }, []string{"instance"}) ) type instance struct { From b93fcad725e14857fd6731f91e03bd573cac6eb0 Mon Sep 17 00:00:00 2001 From: Goutham Veeramachaneni Date: Tue, 18 Dec 2018 17:20:21 +0530 Subject: [PATCH 4/4] Nuke dead code Usage has been removed as part of perf-optimisation: https://github.com/cortexproject/cortex/commit/3006e3901d607f25fa2e213ae05d00380e80a8bc#diff-180f56d9aaf5a9aa079e6fc9cfcf1bc8L365 Signed-off-by: Goutham Veeramachaneni --- pkg/distributor/distributor.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pkg/distributor/distributor.go b/pkg/distributor/distributor.go index 56aec50688542..41ca3fd8a44e9 100644 --- a/pkg/distributor/distributor.go +++ b/pkg/distributor/distributor.go @@ -20,12 +20,6 @@ import ( ) var ( - sendDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: "loki", - Name: "distributor_send_duration_seconds", - Help: "Time spent sending a sample batch to multiple replicated ingesters.", - Buckets: []float64{.001, .0025, .005, .01, .025, .05, .1, .25, .5, 1}, - }, []string{"method", "status_code"}) ingesterAppends = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: "loki", Name: "distributor_ingester_appends_total",