Record tbs disk usage stats in benchtest (#14995)

Record TBS lsm size and vlog size in benchtest to facilitate TBS improvements when running benchmarks using gh actions following #14985

systemtest/benchtest/expvar/expvar.go

@@ -36,6 +36,7 @@ type expvar struct {
 	LibbeatStats
 	ElasticResponseStats
 	OTLPResponseStats
+	TailSamplingStats
 
 	// UncompressedBytes holds the number of bytes of uncompressed
 	// data that the server has read from the Elastic APM events
@@ -72,6 +73,11 @@ type LibbeatStats struct {
 	RSSMemoryBytes int64 `json:"beat.memstats.rss"`
 }
 
+type TailSamplingStats struct {
+	TBSLsmSize  int64 `json:"apm-server.sampling.tail.storage.lsm_size"`
+	TBSVlogSize int64 `json:"apm-server.sampling.tail.storage.value_log_size"`
+}
+
 func queryExpvar(ctx context.Context, out *expvar, srv string) error {
 	req, err := http.NewRequest("GET", srv+"/debug/vars", nil)
 	if err != nil {
@@ -113,6 +119,7 @@ func queryExpvar(ctx context.Context, out *expvar, srv string) error {
 		aggregateResponseStats(s.ElasticResponseStats, &result.ElasticResponseStats)
 		aggregateOTLPResponseStats(s.OTLPResponseStats, &result.OTLPResponseStats)
 		aggregateLibbeatStats(s.LibbeatStats, &result.LibbeatStats)
+		aggregateTailSamplingStats(s.TailSamplingStats, &result.TailSamplingStats)
 		result.UncompressedBytes += s.UncompressedBytes
 		result.AvailableBulkRequests += s.AvailableBulkRequests
 	}
@@ -205,3 +212,8 @@ func aggregateOTLPResponseStats(from OTLPResponseStats, to *OTLPResponseStats) {
 	to.ErrorOTLPTracesResponses += from.ErrorOTLPTracesResponses
 	to.ErrorOTLPMetricsResponses += from.ErrorOTLPMetricsResponses
 }
+
+func aggregateTailSamplingStats(from TailSamplingStats, to *TailSamplingStats) {
+	to.TBSLsmSize += from.TBSLsmSize
+	to.TBSVlogSize += from.TBSVlogSize
+}

systemtest/benchtest/expvar/metrics.go

@@ -47,6 +47,8 @@ const (
 	ErrorElasticResponses
 	ErrorOTLPTracesResponses
 	ErrorOTLPMetricsResponses
+	TBSLsmSize
+	TBSVlogSize
 )
 
 type AggregateStats struct {
@@ -164,6 +166,8 @@ func (c *Collector) accumulate(e expvar) {
 	c.processMetric(MemBytes, int64(e.TotalAlloc))
 	c.processMetric(HeapAlloc, int64(e.HeapAlloc))
 	c.processMetric(HeapObjects, int64(e.HeapObjects))
+	c.processMetric(TBSLsmSize, e.TBSLsmSize)
+	c.processMetric(TBSVlogSize, e.TBSVlogSize)
 }
 
 func (c *Collector) processMetric(m Metric, val int64) {

systemtest/benchtest/main.go

@@ -122,6 +122,8 @@ func addExpvarMetrics(result *testing.BenchmarkResult, collector *expvar.Collect
 		result.Extra["max_heap_alloc"] = float64(collector.Get(expvar.HeapAlloc).Max)
 		result.Extra["max_heap_objects"] = float64(collector.Get(expvar.HeapObjects).Max)
 		result.Extra["mean_available_indexers"] = float64(collector.Get(expvar.AvailableBulkRequests).Mean)
+		result.Extra["tbs_lsm_size"] = float64(collector.Get(expvar.TBSLsmSize).Max)
+		result.Extra["tbs_vlog_size"] = float64(collector.Get(expvar.TBSVlogSize).Max)
 	}
 
 	// Record the number of error responses returned by the server: lower is better.

systemtest/benchtest/main_test.go

@@ -141,6 +141,8 @@ func TestAddExpvarMetrics(t *testing.T) {
 				`"apm-server.processor.span.transformations": 5`,
 				`"apm-server.processor.metric.transformations": 9`,
 				`"apm-server.processor.error.transformations": 3`,
+				`"apm-server.sampling.tail.storage.lsm_size": 10`,
+				`"apm-server.sampling.tail.storage.value_log_size": 11`,
 				`"beat.runtime.goroutines": 4`,
 				`"beat.memstats.rss": 1048576`,
 				`"output.elasticsearch.bulk_requests.available": 0`,
@@ -165,6 +167,8 @@ func TestAddExpvarMetrics(t *testing.T) {
 				"max_heap_objects":        102,
 				"mean_available_indexers": 0,
 				"error_responses/sec":     1,
+				"tbs_lsm_size":            10,
+				"tbs_vlog_size":           11,
 			},
 		},
 	}