From 1a44981699cf630ab503b3d5c59e3ecbe2cb4495 Mon Sep 17 00:00:00 2001 From: Sean Porter Date: Mon, 13 Jan 2025 14:26:27 -0800 Subject: [PATCH 1/3] late span age histogram should include sampled traces (anything w/ a decision time) Signed-off-by: Sean Porter --- processor/tailsamplingprocessor/processor.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/processor/tailsamplingprocessor/processor.go b/processor/tailsamplingprocessor/processor.go index 572aa3e60160..22ab882b099a 100644 --- a/processor/tailsamplingprocessor/processor.go +++ b/processor/tailsamplingprocessor/processor.go @@ -518,11 +518,14 @@ func (tsp *tailSamplingSpanProcessor) processTraces(resourceSpans ptrace.Resourc tsp.releaseSampledTrace(tsp.ctx, id, traceTd) case sampling.NotSampled: tsp.nonSampledIDCache.Put(id, true) - tsp.telemetry.ProcessorTailSamplingSamplingLateSpanAge.Record(tsp.ctx, int64(time.Since(actualData.DecisionTime)/time.Second)) default: tsp.logger.Warn("Encountered unexpected sampling decision", zap.Int("decision", int(finalDecision))) } + + if !actualData.DecisionTime.IsZero() { + tsp.telemetry.ProcessorTailSamplingSamplingLateSpanAge.Record(tsp.ctx, int64(time.Since(actualData.DecisionTime)/time.Second)) + } } } From af5807fd2a9f3b57a4a3e6f9aef5f122a68f5fa2 Mon Sep 17 00:00:00 2001 From: Sean Porter Date: Mon, 13 Jan 2025 14:51:51 -0800 Subject: [PATCH 2/3] added changelog entry Signed-off-by: Sean Porter --- ...lingprocessor-late-span-age-histogram.yaml | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 .chloggen/tailsamplingprocessor-late-span-age-histogram.yaml diff --git a/.chloggen/tailsamplingprocessor-late-span-age-histogram.yaml b/.chloggen/tailsamplingprocessor-late-span-age-histogram.yaml new file mode 100644 index 000000000000..88ccab2ab587 --- /dev/null +++ b/.chloggen/tailsamplingprocessor-late-span-age-histogram.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: bug_fix + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: tailsamplingprocessor + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Late span age histogram should include sampled traces. + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [37180] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [user] From 3b41658ecaa5f17d0647f1227b5636533556c900 Mon Sep 17 00:00:00 2001 From: Sean Porter Date: Mon, 13 Jan 2025 22:52:31 -0800 Subject: [PATCH 3/3] adjust late span age histogram test to hit sampled traces Signed-off-by: Sean Porter --- .../processor_telemetry_test.go | 41 ++++++++++++------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/processor/tailsamplingprocessor/processor_telemetry_test.go b/processor/tailsamplingprocessor/processor_telemetry_test.go index 0f3b215703ca..9fd3eae233f0 100644 --- a/processor/tailsamplingprocessor/processor_telemetry_test.go +++ b/processor/tailsamplingprocessor/processor_telemetry_test.go @@ -451,10 +451,10 @@ func TestProcessorTailSamplingSamplingLateSpanAge(t *testing.T) { PolicyCfgs: []PolicyCfg{ { sharedPolicyCfg: sharedPolicyCfg{ - Name: "never-sample", + Name: "sample-half", Type: Probabilistic, ProbabilisticCfg: ProbabilisticCfg{ - SamplingPercentage: 0, + SamplingPercentage: 50, }, }, }, @@ -472,22 +472,24 @@ func TestProcessorTailSamplingSamplingLateSpanAge(t *testing.T) { err = proc.Start(context.Background(), componenttest.NewNopHost()) require.NoError(t, err) - traces := simpleTraces() - traceID := traces.ResourceSpans().At(0).ScopeSpans().AppendEmpty().Spans().AppendEmpty().TraceID() - - lateSpan := ptrace.NewTraces() - lateSpan.ResourceSpans().AppendEmpty().ScopeSpans().AppendEmpty().Spans().AppendEmpty().SetTraceID(traceID) - // test - err = proc.ConsumeTraces(context.Background(), traces) - require.NoError(t, err) + traceIDs, batches := generateIDsAndBatches(10) + for _, batch := range batches { + err = proc.ConsumeTraces(context.Background(), batch) + require.NoError(t, err) + } tsp := proc.(*tailSamplingSpanProcessor) tsp.policyTicker.OnTick() // the first tick always gets an empty batch tsp.policyTicker.OnTick() - err = proc.ConsumeTraces(context.Background(), lateSpan) - require.NoError(t, err) + for _, traceID := range traceIDs { + lateSpan := ptrace.NewTraces() + lateSpan.ResourceSpans().AppendEmpty().ScopeSpans().AppendEmpty().Spans().AppendEmpty().SetTraceID(traceID) + + err = proc.ConsumeTraces(context.Background(), lateSpan) + require.NoError(t, err) + } // verify var md metricdata.ResourceMetrics @@ -499,9 +501,20 @@ func TestProcessorTailSamplingSamplingLateSpanAge(t *testing.T) { Unit: "s", Data: metricdata.Histogram[int64]{ Temporality: metricdata.CumulativeTemporality, - DataPoints: []metricdata.HistogramDataPoint[int64]{{}}, + DataPoints: []metricdata.HistogramDataPoint[int64]{ + { + Count: 10, + Bounds: []float64{0, 5, 10, 25, 50, 75, 100, 250, 500, 750, 1000, 2500, 5000, 7500, 10000}, + BucketCounts: []uint64{10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + Min: metricdata.NewExtrema[int64](0), + Max: metricdata.NewExtrema[int64](0), + Sum: 0, + }, + }, }, } + got := s.getMetric(m.Name, md) - metricdatatest.AssertEqual(t, m, got, metricdatatest.IgnoreTimestamp(), metricdatatest.IgnoreValue()) + + metricdatatest.AssertEqual(t, m, got, metricdatatest.IgnoreTimestamp()) }