add stddev

Signed-off-by: Ben Ye <[email protected]> resolve conflict Signed-off-by: Ben Ye <[email protected]>
thanos-io · Jan 6, 2025 · 9627fd6 · 9627fd6
1 parent 647bce2
commit 9627fd6
Show file tree

Hide file tree

Showing 5 changed files with 39 additions and 17 deletions.
diff --git a/cmd/thanos/downsample.go b/cmd/thanos/downsample.go
@@ -415,6 +415,7 @@ func processDownsampling(
 		meta.Thanos.IndexStats.SeriesP99Size = stats.SeriesP99Size
 		meta.Thanos.IndexStats.SeriesP999Size = stats.SeriesP999Size
 		meta.Thanos.IndexStats.SeriesP9999Size = stats.SeriesP9999Size
+		meta.Thanos.IndexStats.SeriesSizeStdDev = stats.SeriesSizeStdDev
 	}
 	if err := meta.WriteToDir(logger, resdir); err != nil {
 		return errors.Wrap(err, "write meta")

diff --git a/go.mod b/go.mod
@@ -102,6 +102,8 @@ require (
 	gopkg.in/yaml.v3 v3.0.1
 )
 
+require github.com/DataDog/sketches-go v1.4.6
+
 require (
 	cloud.google.com/go v0.115.1 // indirect
 	cloud.google.com/go/auth v0.9.3 // indirect

diff --git a/pkg/block/index.go b/pkg/block/index.go
@@ -86,13 +86,14 @@ type HealthStats struct {
 	ChunkAvgSize int64
 	ChunkMaxSize int64
 
-	SeriesMinSize   int64
-	SeriesAvgSize   int64
-	SeriesMaxSize   int64
-	SeriesP9999Size int64
-	SeriesP999Size  int64
-	SeriesP99Size   int64
-	SeriesP90Size   int64
+	SeriesMinSize    int64
+	SeriesAvgSize    int64
+	SeriesMaxSize    int64
+	SeriesP9999Size  int64
+	SeriesP999Size   int64
+	SeriesP99Size    int64
+	SeriesP90Size    int64
+	SeriesSizeStdDev int64
 
 	SingleSampleSeries int64
 	SingleSampleChunks int64
@@ -216,8 +217,10 @@ func (n *minMaxSumInt64) Avg() int64 {
 
 // sketch is a wrapper for DDSketch which allows to calculate quantile values with a relative accuracy.
 type sketch struct {
-	cnt int64
-	s   *ddsketch.DDSketch
+	cnt  int64
+	sum  int64
+	sum2 int64
+	s    *ddsketch.DDSketch
 }
 
 func newSketch() *sketch {
@@ -229,6 +232,8 @@ func newSketch() *sketch {
 
 func (s *sketch) Add(v int64) {
 	s.cnt++
+	s.sum += v
+	s.sum2 += v * v
 	// Impossible to happen since v should > 0.
 	_ = s.s.Add(float64(v))
 }
@@ -237,8 +242,7 @@ func (s *sketch) Avg() int64 {
 	if s.cnt == 0 {
 		return 0
 	}
-	// Impossible to happen if sketch is not empty.
-	return int64(s.s.GetSum()) / s.cnt
+	return s.sum / s.cnt
 }
 
 func (s *sketch) Max() int64 {
@@ -259,6 +263,18 @@ func (s *sketch) Min() int64 {
 	return int64(v)
 }
 
+func (s *sketch) StdDev() float64 {
+	if s.cnt == 0 {
+		return 0
+	}
+	mean := s.sum / s.cnt
+	return math.Sqrt(float64(s.sum2/s.cnt - mean*mean))
+}
+
+func (s *sketch) ZScore(z float64) int64 {
+	return int64(s.StdDev()*z) + s.Avg()
+}
+
 func (s *sketch) Quantile(quantile float64) int64 {
 	if s.cnt == 0 {
 		return 0
@@ -457,6 +473,7 @@ func GatherIndexHealthStats(ctx context.Context, logger log.Logger, fn string, m
 	stats.SeriesP99Size = seriesSize.Quantile(0.99)
 	stats.SeriesP999Size = seriesSize.Quantile(0.999)
 	stats.SeriesP9999Size = seriesSize.Quantile(0.9999)
+	stats.SeriesSizeStdDev = int64(seriesSize.StdDev())
 
 	stats.ChunkMaxDuration = time.Duration(chunkDuration.max) * time.Millisecond
 	stats.ChunkAvgDuration = time.Duration(chunkDuration.Avg()) * time.Millisecond

diff --git a/pkg/block/metadata/meta.go b/pkg/block/metadata/meta.go
@@ -100,12 +100,13 @@ type Thanos struct {
 }
 
 type IndexStats struct {
-	SeriesMaxSize   int64 `json:"series_max_size,omitempty"`
-	SeriesP90Size   int64 `json:"series_p90_size,omitempty"`
-	SeriesP99Size   int64 `json:"series_p99_size,omitempty"`
-	SeriesP999Size  int64 `json:"series_p999_size,omitempty"`
-	SeriesP9999Size int64 `json:"series_p9999_size,omitempty"`
-	ChunkMaxSize    int64 `json:"chunk_max_size,omitempty"`
+	SeriesMaxSize    int64 `json:"series_max_size,omitempty"`
+	SeriesP90Size    int64 `json:"series_p90_size,omitempty"`
+	SeriesP99Size    int64 `json:"series_p99_size,omitempty"`
+	SeriesP999Size   int64 `json:"series_p999_size,omitempty"`
+	SeriesP9999Size  int64 `json:"series_p9999_size,omitempty"`
+	SeriesSizeStdDev int64 `json:"series_size_stddev,omitempty"`
+	ChunkMaxSize     int64 `json:"chunk_max_size,omitempty"`
 }
 
 func (m *Thanos) ParseExtensions(v any) (any, error) {

diff --git a/pkg/compact/compact.go b/pkg/compact/compact.go
@@ -1307,6 +1307,7 @@ func (cg *Group) compact(ctx context.Context, dir string, planner Planner, comp
 			thanosMeta.IndexStats.SeriesP99Size = stats.SeriesP99Size
 			thanosMeta.IndexStats.SeriesP999Size = stats.SeriesP999Size
 			thanosMeta.IndexStats.SeriesP9999Size = stats.SeriesP9999Size
+			thanosMeta.IndexStats.SeriesSizeStdDev = stats.SeriesSizeStdDev
 		}
 		newMeta, err = metadata.InjectThanos(cg.logger, bdir, thanosMeta, nil)
 		if err != nil {