diff --git a/pkg/store/bucket.go b/pkg/store/bucket.go index 907dd56a8c..e2c68576ce 100644 --- a/pkg/store/bucket.go +++ b/pkg/store/bucket.go @@ -142,6 +142,7 @@ type bucketStoreMetrics struct { seriesFetchDuration prometheus.Histogram postingsFetchDuration prometheus.Histogram + chunkFetchDuration prometheus.Histogram } func newBucketStoreMetrics(reg prometheus.Registerer) *bucketStoreMetrics { @@ -276,6 +277,12 @@ func newBucketStoreMetrics(reg prometheus.Registerer) *bucketStoreMetrics { Buckets: []float64{0.001, 0.01, 0.1, 0.3, 0.6, 1, 3, 6, 9, 20, 30, 60, 90, 120}, }) + m.chunkFetchDuration = promauto.With(reg).NewHistogram(prometheus.HistogramOpts{ + Name: "thanos_bucket_store_chunks_fetch_duration_seconds", + Help: "The total time spent fetching chunks within a single request a store gateway.", + Buckets: []float64{0.001, 0.01, 0.1, 0.3, 0.6, 1, 3, 6, 9, 20, 30, 60, 90, 120}, + }) + m.emptyPostingCount = promauto.With(reg).NewCounter(prometheus.CounterOpts{ Name: "thanos_bucket_store_empty_postings_total", Help: "Total number of empty postings when fetching block series.", @@ -814,6 +821,7 @@ type blockSeriesClient struct { skipChunks bool shardMatcher *storepb.ShardMatcher calculateChunkHash bool + chunkFetchDuration prometheus.Histogram // Internal state. i int @@ -836,6 +844,7 @@ func newBlockSeriesClient( shardMatcher *storepb.ShardMatcher, calculateChunkHash bool, batchSize int, + chunkFetchDuration prometheus.Histogram, ) *blockSeriesClient { var chunkr *bucketChunkReader if !req.SkipChunks { @@ -843,16 +852,17 @@ func newBlockSeriesClient( } return &blockSeriesClient{ - ctx: ctx, - logger: logger, - extLset: b.extLset, - mint: req.MinTime, - maxt: req.MaxTime, - indexr: b.indexReader(), - chunkr: chunkr, - chunksLimiter: limiter, - bytesLimiter: bytesLimiter, - skipChunks: req.SkipChunks, + ctx: ctx, + logger: logger, + extLset: b.extLset, + mint: req.MinTime, + maxt: req.MaxTime, + indexr: b.indexReader(), + chunkr: chunkr, + chunksLimiter: limiter, + bytesLimiter: bytesLimiter, + skipChunks: req.SkipChunks, + chunkFetchDuration: chunkFetchDuration, loadAggregates: req.Aggregates, shardMatcher: shardMatcher, @@ -911,6 +921,9 @@ func (b *blockSeriesClient) Recv() (*storepb.SeriesResponse, error) { } if len(b.entries) == 0 { + if b.chunkr != nil { + b.chunkFetchDuration.Observe(float64(b.chunkr.stats.ChunksFetchDurationSum)) + } return nil, io.EOF } @@ -1201,6 +1214,7 @@ func (s *BucketStore) Series(req *storepb.SeriesRequest, srv storepb.Store_Serie shardMatcher, s.enableChunkHashCalculation, s.seriesBatchSize, + s.metrics.chunkFetchDuration, ) defer blockClient.Close() @@ -1445,7 +1459,7 @@ func (s *BucketStore) LabelNames(ctx context.Context, req *storepb.LabelNamesReq MaxTime: req.End, SkipChunks: true, } - blockClient := newBlockSeriesClient(newCtx, s.logger, b, seriesReq, nil, bytesLimiter, nil, true, SeriesBatchSize) + blockClient := newBlockSeriesClient(newCtx, s.logger, b, seriesReq, nil, bytesLimiter, nil, true, SeriesBatchSize, s.metrics.chunkFetchDuration) if err := blockClient.ExpandPostings( reqSeriesMatchersNoExtLabels, @@ -1619,7 +1633,7 @@ func (s *BucketStore) LabelValues(ctx context.Context, req *storepb.LabelValuesR MaxTime: req.End, SkipChunks: true, } - blockClient := newBlockSeriesClient(newCtx, s.logger, b, seriesReq, nil, bytesLimiter, nil, true, SeriesBatchSize) + blockClient := newBlockSeriesClient(newCtx, s.logger, b, seriesReq, nil, bytesLimiter, nil, true, SeriesBatchSize, s.metrics.chunkFetchDuration) if err := blockClient.ExpandPostings( reqSeriesMatchersNoExtLabels, diff --git a/pkg/store/bucket_test.go b/pkg/store/bucket_test.go index 72c0c01a89..aa05bd07ea 100644 --- a/pkg/store/bucket_test.go +++ b/pkg/store/bucket_test.go @@ -23,6 +23,7 @@ import ( "time" "github.com/cespare/xxhash" + "github.com/prometheus/client_golang/prometheus" "github.com/go-kit/log" "github.com/gogo/protobuf/proto" @@ -2438,7 +2439,8 @@ func benchmarkBlockSeriesWithConcurrency(b *testing.B, concurrency int, blockMet // must be called only from the goroutine running the Benchmark function. testutil.Ok(b, err) - blockClient := newBlockSeriesClient(ctx, nil, blk, req, chunksLimiter, NewBytesLimiterFactory(0)(nil), nil, false, SeriesBatchSize) + dummyHistogram := prometheus.NewHistogram(prometheus.HistogramOpts{}) + blockClient := newBlockSeriesClient(ctx, nil, blk, req, chunksLimiter, NewBytesLimiterFactory(0)(nil), nil, false, SeriesBatchSize, dummyHistogram) testutil.Ok(b, blockClient.ExpandPostings(matchers, seriesLimiter)) defer blockClient.Close()