[processor/tailsampling] Improved "not sampled" decision cache usage

.chloggen/tailsamplingprocessor-improved-not-sampled-decision-cache.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: tailsamplingprocessor
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Improved not sampled decision cache usage and deleting traces from the internal map when they are in a decision cache.
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [37189]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [user]

processor/tailsamplingprocessor/processor.go

@@ -350,8 +350,11 @@ func (tsp *tailSamplingSpanProcessor) samplingPolicyOnTick() {
 		trace.ReceivedBatches = ptrace.NewTraces()
 		trace.Unlock()
 
-		if decision == sampling.Sampled {
+		switch decision {
+		case sampling.Sampled:
 			tsp.releaseSampledTrace(context.Background(), id, allSpans)
+		case sampling.NotSampled:
+			tsp.releaseNotSampledTrace(id)
 		}
 	}
 
@@ -517,7 +520,7 @@ func (tsp *tailSamplingSpanProcessor) processTraces(resourceSpans ptrace.Resourc
 				appendToTraces(traceTd, resourceSpans, spans)
 				tsp.releaseSampledTrace(tsp.ctx, id, traceTd)
 			case sampling.NotSampled:
-				tsp.nonSampledIDCache.Put(id, true)
+				tsp.releaseNotSampledTrace(id)
 				tsp.telemetry.ProcessorTailSamplingSamplingLateSpanAge.Record(tsp.ctx, int64(time.Since(actualData.DecisionTime)/time.Second))
 			default:
 				tsp.logger.Warn("Encountered unexpected sampling decision",
@@ -562,16 +565,30 @@ func (tsp *tailSamplingSpanProcessor) dropTrace(traceID pcommon.TraceID, deletio
 	tsp.telemetry.ProcessorTailSamplingSamplingTraceRemovalAge.Record(tsp.ctx, int64(deletionTime.Sub(trace.ArrivalTime)/time.Second))
 }
 
-// releaseSampledTrace sends the trace data to the next consumer.
-// It additionally adds the trace ID to the cache of sampled trace IDs.
-// It does not (yet) delete the spans from the internal map.
+// releaseSampledTrace sends the trace data to the next consumer. It
+// additionally adds the trace ID to the cache of sampled trace IDs. If the
+// trace ID is cached, it deletes the spans from the internal map.
 func (tsp *tailSamplingSpanProcessor) releaseSampledTrace(ctx context.Context, id pcommon.TraceID, td ptrace.Traces) {
 	tsp.sampledIDCache.Put(id, true)
 	if err := tsp.nextConsumer.ConsumeTraces(ctx, td); err != nil {
 		tsp.logger.Warn(
 			"Error sending spans to destination",
 			zap.Error(err))
 	}
+	_, ok := tsp.sampledIDCache.Get(id)
+	if ok {
+		tsp.dropTrace(id, time.Now())
+	}
+}
+
+// releaseNotSampledTrace adds the trace ID to the cache of not sampled trace
+// IDs. If the trace ID is cached, it deletes the spans from the internal map.
+func (tsp *tailSamplingSpanProcessor) releaseNotSampledTrace(id pcommon.TraceID) {
+	tsp.nonSampledIDCache.Put(id, true)
+	_, ok := tsp.nonSampledIDCache.Get(id)
+	if ok {
+		tsp.dropTrace(id, time.Now())
+	}
 }
 
 func appendToTraces(dest ptrace.Traces, rss ptrace.ResourceSpans, spanAndScopes []spanAndScope) {

processor/tailsamplingprocessor/processor_decisions_test.go

@@ -6,7 +6,6 @@ package tailsamplingprocessor
 import (
 	"context"
 	"testing"
-	"time"
 
 	"github.com/stretchr/testify/require"
 	"go.opentelemetry.io/collector/component/componenttest"
@@ -386,8 +385,7 @@ func TestLateArrivingSpanUsesDecisionCache(t *testing.T) {
 	// The final decision SHOULD be Sampled.
 	require.EqualValues(t, 1, nextConsumer.SpanCount())
 
-	// Drop the trace to force cache to make decision
-	tsp.dropTrace(traceID, time.Now())
+	// The trace should have been dropped after its id was added to the decision cache
 	_, ok := tsp.idToTrace.Load(traceID)
 	require.False(t, ok)
 
@@ -461,8 +459,7 @@ func TestLateSpanUsesNonSampledDecisionCache(t *testing.T) {
 	// The final decision SHOULD be NOT Sampled.
 	require.EqualValues(t, 0, nextConsumer.SpanCount())
 
-	// Drop the trace to force cache to make decision
-	tsp.dropTrace(traceID, time.Now())
+	// The trace should have been dropped after its id was added to the decision cache
 	_, ok := tsp.idToTrace.Load(traceID)
 	require.False(t, ok)