Add metric for chunk fetch time

Signed-off-by: Filip Petkovski <[email protected]>
thanos-io · Nov 12, 2022 · 8946647 · 8946647
1 parent f2c4ecc
commit 8946647
Show file tree

Hide file tree

Showing 2 changed files with 29 additions and 13 deletions.
diff --git a/pkg/store/bucket.go b/pkg/store/bucket.go
@@ -142,6 +142,7 @@ type bucketStoreMetrics struct {
 
 	seriesFetchDuration   prometheus.Histogram
 	postingsFetchDuration prometheus.Histogram
+	chunkFetchDuration    prometheus.Histogram
 }
 
 func newBucketStoreMetrics(reg prometheus.Registerer) *bucketStoreMetrics {
@@ -276,6 +277,12 @@ func newBucketStoreMetrics(reg prometheus.Registerer) *bucketStoreMetrics {
 		Buckets: []float64{0.001, 0.01, 0.1, 0.3, 0.6, 1, 3, 6, 9, 20, 30, 60, 90, 120},
 	})
 
+	m.chunkFetchDuration = promauto.With(reg).NewHistogram(prometheus.HistogramOpts{
+		Name:    "thanos_bucket_store_chunks_fetch_duration_seconds",
+		Help:    "The total time spent fetching chunks within a single request a store gateway.",
+		Buckets: []float64{0.001, 0.01, 0.1, 0.3, 0.6, 1, 3, 6, 9, 20, 30, 60, 90, 120},
+	})
+
 	m.emptyPostingCount = promauto.With(reg).NewCounter(prometheus.CounterOpts{
 		Name: "thanos_bucket_store_empty_postings_total",
 		Help: "Total number of empty postings when fetching block series.",
@@ -814,6 +821,7 @@ type blockSeriesClient struct {
 	skipChunks         bool
 	shardMatcher       *storepb.ShardMatcher
 	calculateChunkHash bool
+	chunkFetchDuration prometheus.Histogram
 
 	// Internal state.
 	i               int
@@ -836,23 +844,25 @@ func newBlockSeriesClient(
 	shardMatcher *storepb.ShardMatcher,
 	calculateChunkHash bool,
 	batchSize int,
+	chunkFetchDuration prometheus.Histogram,
 ) *blockSeriesClient {
 	var chunkr *bucketChunkReader
 	if !req.SkipChunks {
 		chunkr = b.chunkReader()
 	}
 
 	return &blockSeriesClient{
-		ctx:           ctx,
-		logger:        logger,
-		extLset:       b.extLset,
-		mint:          req.MinTime,
-		maxt:          req.MaxTime,
-		indexr:        b.indexReader(),
-		chunkr:        chunkr,
-		chunksLimiter: limiter,
-		bytesLimiter:  bytesLimiter,
-		skipChunks:    req.SkipChunks,
+		ctx:                ctx,
+		logger:             logger,
+		extLset:            b.extLset,
+		mint:               req.MinTime,
+		maxt:               req.MaxTime,
+		indexr:             b.indexReader(),
+		chunkr:             chunkr,
+		chunksLimiter:      limiter,
+		bytesLimiter:       bytesLimiter,
+		skipChunks:         req.SkipChunks,
+		chunkFetchDuration: chunkFetchDuration,
 
 		loadAggregates:     req.Aggregates,
 		shardMatcher:       shardMatcher,
@@ -911,6 +921,9 @@ func (b *blockSeriesClient) Recv() (*storepb.SeriesResponse, error) {
 	}
 
 	if len(b.entries) == 0 {
+		if b.chunkr != nil {
+			b.chunkFetchDuration.Observe(float64(b.chunkr.stats.ChunksFetchDurationSum))
+		}
 		return nil, io.EOF
 	}
 
@@ -1201,6 +1214,7 @@ func (s *BucketStore) Series(req *storepb.SeriesRequest, srv storepb.Store_Serie
 				shardMatcher,
 				s.enableChunkHashCalculation,
 				s.seriesBatchSize,
+				s.metrics.chunkFetchDuration,
 			)
 			defer blockClient.Close()
 
@@ -1445,7 +1459,7 @@ func (s *BucketStore) LabelNames(ctx context.Context, req *storepb.LabelNamesReq
 					MaxTime:    req.End,
 					SkipChunks: true,
 				}
-				blockClient := newBlockSeriesClient(newCtx, s.logger, b, seriesReq, nil, bytesLimiter, nil, true, SeriesBatchSize)
+				blockClient := newBlockSeriesClient(newCtx, s.logger, b, seriesReq, nil, bytesLimiter, nil, true, SeriesBatchSize, s.metrics.chunkFetchDuration)
 
 				if err := blockClient.ExpandPostings(
 					reqSeriesMatchersNoExtLabels,
@@ -1619,7 +1633,7 @@ func (s *BucketStore) LabelValues(ctx context.Context, req *storepb.LabelValuesR
 					MaxTime:    req.End,
 					SkipChunks: true,
 				}
-				blockClient := newBlockSeriesClient(newCtx, s.logger, b, seriesReq, nil, bytesLimiter, nil, true, SeriesBatchSize)
+				blockClient := newBlockSeriesClient(newCtx, s.logger, b, seriesReq, nil, bytesLimiter, nil, true, SeriesBatchSize, s.metrics.chunkFetchDuration)
 
 				if err := blockClient.ExpandPostings(
 					reqSeriesMatchersNoExtLabels,

diff --git a/pkg/store/bucket_test.go b/pkg/store/bucket_test.go
@@ -23,6 +23,7 @@ import (
 	"time"
 
 	"github.com/cespare/xxhash"
+	"github.com/prometheus/client_golang/prometheus"
 
 	"github.com/go-kit/log"
 	"github.com/gogo/protobuf/proto"
@@ -2438,7 +2439,8 @@ func benchmarkBlockSeriesWithConcurrency(b *testing.B, concurrency int, blockMet
 				// must be called only from the goroutine running the Benchmark function.
 				testutil.Ok(b, err)
 
-				blockClient := newBlockSeriesClient(ctx, nil, blk, req, chunksLimiter, NewBytesLimiterFactory(0)(nil), nil, false, SeriesBatchSize)
+				dummyHistogram := prometheus.NewHistogram(prometheus.HistogramOpts{})
+				blockClient := newBlockSeriesClient(ctx, nil, blk, req, chunksLimiter, NewBytesLimiterFactory(0)(nil), nil, false, SeriesBatchSize, dummyHistogram)
 				testutil.Ok(b, blockClient.ExpandPostings(matchers, seriesLimiter))
 				defer blockClient.Close()