Do not reuse labels, samples and histograms slices in the write request if there are more than 10x entries than the pre-allocated size

CHANGELOG.md

@@ -76,6 +76,7 @@
 * [ENHANCEMENT] Ruler: Support OAuth2 and proxies in Alertmanager client #9945
 * [ENHANCEMENT] Ingester: Add `-blocks-storage.tsdb.bigger-out-of-order-blocks-for-old-samples` to build 24h blocks for out-of-order data belonging to the previous days instead of building smaller 2h blocks. This reduces pressure on compactors and ingesters when the out-of-order samples span multiple days in the past. #9844 #10033 #10035
 * [ENHANCEMENT] Distributor: allow a different limit for info series (series ending in `_info`) label count, via `-validation.max-label-names-per-info-series`. #10028
+* [ENHANCEMENT] Ingester: do not reuse labels, samples and histograms slices in the write request if there are more than 10x entries than the pre-allocated size. This should help to reduce the in-use memory in case of few requests with a very large number of labels, samples or histograms. #10040
 * [BUGFIX] Fix issue where functions such as `rate()` over native histograms could return incorrect values if a float stale marker was present in the selected range. #9508
 * [BUGFIX] Fix issue where negation of native histograms (eg. `-some_native_histogram_series`) did nothing. #9508
 * [BUGFIX] Fix issue where `metric might not be a counter, name does not end in _total/_sum/_count/_bucket` annotation would be emitted even if `rate` or `increase` did not have enough samples to compute a result. #9508

pkg/mimirpb/timeseries.go

@@ -19,11 +19,14 @@ import (
 )
 
 const (
-	minPreallocatedTimeseries         = 100
-	minPreallocatedLabels             = 20
-	minPreallocatedSamplesPerSeries   = 10
-	minPreallocatedExemplarsPerSeries = 1
-	maxPreallocatedExemplarsPerSeries = 10
+	minPreallocatedTimeseries          = 100
+	minPreallocatedLabels              = 20
+	maxPreallocatedLabels              = 200
+	minPreallocatedSamplesPerSeries    = 10
+	maxPreallocatedSamplesPerSeries    = 100
+	maxPreallocatedHistogramsPerSeries = 100
+	minPreallocatedExemplarsPerSeries  = 1
+	maxPreallocatedExemplarsPerSeries  = 10
 )
 
 var (
@@ -476,9 +479,28 @@ func ReuseTimeseries(ts *TimeSeries) {
 		ts.Labels[i].Name = ""
 		ts.Labels[i].Value = ""
 	}
-	ts.Labels = ts.Labels[:0]
-	ts.Samples = ts.Samples[:0]
-	ts.Histograms = ts.Histograms[:0]
+
+	// Retain the slices only if their capacity is not bigger than the desired max pre-allocated size.
+	// This allows us to ensure we don't put very large slices back to the pool (e.g. a few requests with
+	// a huge number of samples may cause in-use heap memory to significantly increase, because the slices
+	// allocated by such poison requests would be reused by other requests with a normal number of samples).
+	if cap(ts.Labels) > maxPreallocatedLabels {
+		ts.Labels = nil
+	} else {
+		ts.Labels = ts.Labels[:0]
+	}
+
+	if cap(ts.Samples) > maxPreallocatedSamplesPerSeries {
+		ts.Samples = nil
+	} else {
+		ts.Samples = ts.Samples[:0]
+	}
+
+	if cap(ts.Histograms) > maxPreallocatedHistogramsPerSeries {
+		ts.Histograms = nil
+	} else {
+		ts.Histograms = ts.Histograms[:0]
+	}
 
 	ClearExemplars(ts)
 	timeSeriesPool.Put(ts)