Implement config to ignore the sampled flag in traceparent

docs/configuration.asciidoc

@@ -1045,6 +1045,36 @@ you must instead set the `LogLevel` for the internal APM logger under the `Loggi
 | `Error`                 | String
 |============
 
+[float]
+[[config-trace-context-ignore-sampled-false]]
+==== `TraceContextIgnoreSampledFalse`
+
+The agent uses the https://www.w3.org/TR/trace-context/[W3C Trace Context] specification and standards for distributed tracing. The traceparent header from the W3C Trace Context specification defines a https://www.w3.org/TR/trace-context/#sampled-flag[sampled flag] which is propagated from a caller service to a callee service, and determines whether a trace is sampled in the callee service. The default behavior of the agent honors the sampled flag value and behaves accordingly.
+
+There may be cases where you wish to change the default behavior of the agent with respect to the sampled flag. By setting the `TraceContextIgnoreSampled` configuration value to `true`, the agent ignores the sampled flag of the W3C Trace Context traceparent header when it has a value of `false` **and** and there is no agent specific tracestate header value present. In ignoring the sampled flag, the agent makes a sampling decision based on the <<config-transaction-sample-rate, sample rate>>. This can be useful when a caller service always sets a sampled flag value of `false`, that results in the agent never sampling any transactions.
+
+[IMPORTANT]
+--
+:dotnet5: .NET 5
+
+{dotnet5} applications set the W3C Trace Context for outgoing HTTP requests by default, but with the traceparent header sampled flag set to `false`. If a {dotnet5} application has an active agent, the agent ensures that the sampled flag is propagated with the agent's sampling decision. If a {dotnet5} application does not have an active agent however, and the application calls another service that does have an active agent, the propagation of a sampled flag value of `false` results in no sampled transactions in the callee service.
+
+If your application is called by an {dotnet5} application that does not have an active agent, setting the `TraceContextIgnoreSampledFalse` configuration value to `true` instructs the agent to start a new transaction and make a sampling decision based on the <<config-transaction-sample-rate, sample rate>>, when the traceparent header sampled flag has a value of `false` **and** there is no agent specific tracestate header value present.
+--
+
+[options="header"]
+|============
+| Environment variable name            | IConfiguration or Web.config key
+| `TRACE_CONTEXT_IGNORE_SAMPLED_FALSE` | `ElasticApm:TraceContextIgnoreSampledFalse`
+|============
+
+[options="header"]
+|============
+| Default                 | Type
+| `false`                 | Boolean
+|============
+
+
 [[config-all-options-summary]]
 === All options summary
 
@@ -1080,6 +1110,7 @@ you must instead set the `LogLevel` for the internal APM logger under the `Loggi
 | <<config-service-version,`ServiceVersion`>> | No | Core
 | <<config-span-frames-min-duration,`SpanFramesMinDuration`>> | No | Stacktrace, Performance
 | <<config-stack-trace-limit,`StackTraceLimit`>> | No | Stacktrace, Performance
+| <<config-trace-context-ignore-sampled-false,`TraceContextIgnoreSampledFalse`>> | No | Core
 | <<config-transaction-ignore-urls,`TransactionIgnoreUrls`>>  | Yes | HTTP, Performance
 | <<config-transaction-max-spans,`TransactionMaxSpans`>>  | Yes | Core, Performance
 | <<config-transaction-sample-rate,`TransactionSampleRate`>> | Yes | Core, Performance

src/Elastic.Apm/BackendComm/CentralConfig/CentralConfigFetcher.cs

@@ -306,6 +306,7 @@ internal WrappingConfigSnapshot(IConfigSnapshot wrapped, CentralConfigReader cen
 			public bool UseElasticTraceparentHeader => _wrapped.UseElasticTraceparentHeader;
 
 			public bool VerifyServerCert => _wrapped.VerifyServerCert;
+			public bool TraceContextIgnoreSampledFalse => _wrapped.TraceContextIgnoreSampledFalse;
 		}
 	}
 }

src/Elastic.Apm/Config/AbstractConfigurationReader.cs

@@ -199,6 +199,13 @@ protected bool ParseRecording(ConfigurationKeyValue kv)
 			return true;
 		}
 
+		protected bool ParseTraceContextIgnoreSampledFalse(ConfigurationKeyValue kv)
+		{
+			if (kv == null || string.IsNullOrEmpty(kv.Value)) return DefaultValues.TraceContextIgnoreSampledFalse;
+			// ReSharper disable once SimplifyConditionalTernaryExpression
+			return bool.TryParse(kv.Value, out var value) ? value : DefaultValues.TraceContextIgnoreSampledFalse;
+		}
+
 		protected bool ParseVerifyServerCert(ConfigurationKeyValue kv)
 		{
 			if (kv == null || string.IsNullOrEmpty(kv.Value)) return DefaultValues.VerifyServerCert;

src/Elastic.Apm/Config/AbstractConfigurationWithEnvFallbackReader.cs

@@ -1,4 +1,5 @@
-// Licensed to Elasticsearch B.V under one or more agreements.
+// Licensed to Elasticsearch B.V under
+// one or more agreements.
 // Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information
 
@@ -93,21 +94,6 @@ internal AbstractConfigurationWithEnvFallbackReader(IApmLogger logger, string de
 
 		public virtual string ServerCert => ParseServerCert(Read(KeyNames.ServerCert, EnvVarNames.ServerCert));
 
-		/// <inheritdoc />
-		[Obsolete("Use ServerUrl")]
-		public virtual IReadOnlyList<Uri> ServerUrls
-		{
-			get
-			{
-				// Use ServerUrl if there's no value for ServerUrls so that usage of ServerUrls
-				// outside of the agent will work with ServerUrl
-				var configurationKeyValue = Read(KeyNames.ServerUrls, EnvVarNames.ServerUrls);
-				return ParseServerUrls(!string.IsNullOrEmpty(configurationKeyValue.Value)
-					? configurationKeyValue
-					: Read(KeyNames.ServerUrl, EnvVarNames.ServerUrl));
-			}
-		}
-
 		/// <inheritdoc />
 		public virtual Uri ServerUrl
 		{
@@ -123,6 +109,21 @@ public virtual Uri ServerUrl
 			}
 		}
 
+		/// <inheritdoc />
+		[Obsolete("Use ServerUrl")]
+		public virtual IReadOnlyList<Uri> ServerUrls
+		{
+			get
+			{
+				// Use ServerUrl if there's no value for ServerUrls so that usage of ServerUrls
+				// outside of the agent will work with ServerUrl
+				var configurationKeyValue = Read(KeyNames.ServerUrls, EnvVarNames.ServerUrls);
+				return ParseServerUrls(!string.IsNullOrEmpty(configurationKeyValue.Value)
+					? configurationKeyValue
+					: Read(KeyNames.ServerUrl, EnvVarNames.ServerUrl));
+			}
+		}
+
 		public virtual string ServiceName => ParseServiceName(Read(KeyNames.ServiceName, EnvVarNames.ServiceName));
 
 		public string ServiceNodeName => ParseServiceNodeName(Read(KeyNames.ServiceNodeName, EnvVarNames.ServiceNodeName));
@@ -134,6 +135,9 @@ public virtual Uri ServerUrl
 
 		public virtual int StackTraceLimit => _stackTraceLimit.Value;
 
+		public bool TraceContextIgnoreSampledFalse =>
+			ParseTraceContextIgnoreSampledFalse(Read(KeyNames.TraceContextIgnoreSampledFalse, EnvVarNames.TraceContextIgnoreSampledFalse));
+
 		public IReadOnlyList<WildcardMatcher> TransactionIgnoreUrls =>
 			ParseTransactionIgnoreUrls(Read(KeyNames.TransactionIgnoreUrls, EnvVarNames.TransactionIgnoreUrls));
 

src/Elastic.Apm/Config/ConfigConsts.cs

@@ -34,6 +34,7 @@ public static class DefaultValues
 			public const string SpanFramesMinDuration = "5ms";
 			public const double SpanFramesMinDurationInMilliseconds = 5;
 			public const int StackTraceLimit = 50;
+			public const bool TraceContextIgnoreSampledFalse = false;
 			public const int TransactionMaxSpans = 500;
 			public const double TransactionSampleRate = 1.0;
 			public const string UnknownServiceName = "unknown";
@@ -142,6 +143,7 @@ public static class EnvVarNames
 			public const string ServiceVersion = Prefix + "SERVICE_VERSION";
 			public const string SpanFramesMinDuration = Prefix + "SPAN_FRAMES_MIN_DURATION";
 			public const string StackTraceLimit = Prefix + "STACK_TRACE_LIMIT";
+			public const string TraceContextIgnoreSampledFalse = Prefix + "TRACE_CONTEXT_IGNORE_SAMPLED_FALSE";
 			public const string TransactionIgnoreUrls = Prefix + "TRANSACTION_IGNORE_URLS";
 			public const string TransactionMaxSpans = Prefix + "TRANSACTION_MAX_SPANS";
 			public const string TransactionSampleRate = Prefix + "TRANSACTION_SAMPLE_RATE";
@@ -183,6 +185,7 @@ public static class KeyNames
 			public const string ServiceNodeName = Prefix + nameof(ServiceNodeName);
 			public const string ServiceVersion = Prefix + nameof(ServiceVersion);
 			public const string SpanFramesMinDuration = Prefix + nameof(SpanFramesMinDuration);
+			public const string TraceContextIgnoreSampledFalse = Prefix + nameof(TraceContextIgnoreSampledFalse);
 			public const string StackTraceLimit = Prefix + nameof(StackTraceLimit);
 			public const string TransactionIgnoreUrls = Prefix + nameof(TransactionIgnoreUrls);
 			public const string TransactionMaxSpans = Prefix + nameof(TransactionMaxSpans);

src/Elastic.Apm/Config/ConfigSnapshotFromReader.cs

@@ -1,4 +1,5 @@
-// Licensed to Elasticsearch B.V under one or more agreements.
+// Licensed to Elasticsearch B.V under
+// one or more agreements.
 // Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information
 
@@ -43,15 +44,19 @@ internal ConfigSnapshotFromReader(IConfigurationReader content, string dbgDescri
 		public bool Recording => _content.Recording;
 		public IReadOnlyList<WildcardMatcher> SanitizeFieldNames => _content.SanitizeFieldNames;
 		public string SecretToken => _content.SecretToken;
-		[Obsolete("Use ServerUrl")]
-		public IReadOnlyList<Uri> ServerUrls => _content.ServerUrls;
 		public string ServerCert => _content.ServerCert;
 		public Uri ServerUrl => _content.ServerUrl;
+
+		[Obsolete("Use ServerUrl")]
+		public IReadOnlyList<Uri> ServerUrls => _content.ServerUrls;
+
 		public string ServiceName => _content.ServiceName;
 		public string ServiceNodeName => _content.ServiceNodeName;
 		public string ServiceVersion => _content.ServiceVersion;
 		public double SpanFramesMinDurationInMilliseconds => _content.SpanFramesMinDurationInMilliseconds;
 		public int StackTraceLimit => _content.StackTraceLimit;
+
+		public bool TraceContextIgnoreSampledFalse => _content.TraceContextIgnoreSampledFalse;
 		public IReadOnlyList<WildcardMatcher> TransactionIgnoreUrls => _content.TransactionIgnoreUrls;
 		public int TransactionMaxSpans => _content.TransactionMaxSpans;
 		public double TransactionSampleRate => _content.TransactionSampleRate;

src/Elastic.Apm/Config/EnvironmentConfigurationReader.cs

@@ -123,6 +123,7 @@ public Uri ServerUrl
 
 		public bool VerifyServerCert => ParseVerifyServerCert(Read(ConfigConsts.EnvVarNames.VerifyServerCert));
 
+		public bool TraceContextIgnoreSampledFalse => ParseTraceContextIgnoreSampledFalse(Read(ConfigConsts.EnvVarNames.TraceContextIgnoreSampledFalse));
 		private ConfigurationKeyValue Read(string key) =>
 			new ConfigurationKeyValue(key, ReadEnvVarValue(key), Origin);
 	}

src/Elastic.Apm/Config/IConfigurationReader.cs

@@ -1,4 +1,5 @@
-// Licensed to Elasticsearch B.V under one or more agreements.
+// Licensed to Elasticsearch B.V under
+// one or more agreements.
 // Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information
 
@@ -51,8 +52,10 @@ public interface IConfigurationReader
 		bool CentralConfig { get; }
 
 		/// <summary>
-		/// Specify which cloud provider should be assumed for metadata collection. By default, the agent will attempt to detect the cloud
-		/// provider or, if that fails, will use trial and error to collect the metadata. Valid options are "aws", "gcp", and "azure".
+		/// Specify which cloud provider should be assumed for metadata collection. By default, the agent will attempt to detect
+		/// the cloud
+		/// provider or, if that fails, will use trial and error to collect the metadata. Valid options are "aws", "gcp", and
+		/// "azure".
 		/// If this config value is set to "False", no cloud metadata will be collected.
 		/// </summary>
 		string CloudProvider { get; }
@@ -212,15 +215,15 @@ public interface IConfigurationReader
 		string ServerCert { get; }
 
 		/// <summary>
-		/// The URLs for APM server.
+		/// The URL for APM server.
 		/// </summary>
-		[Obsolete("Use ServerUrl")]
-		IReadOnlyList<Uri> ServerUrls { get; }
+		Uri ServerUrl { get; }
 
 		/// <summary>
-		/// The URL for APM server.
+		/// The URLs for APM server.
 		/// </summary>
-		Uri ServerUrl { get; }
+		[Obsolete("Use ServerUrl")]
+		IReadOnlyList<Uri> ServerUrls { get; }
 
 		/// <summary>
 		/// The name of service instrumented by the APM agent. This is used to group all the errors and transactions
@@ -258,6 +261,17 @@ public interface IConfigurationReader
 		/// </summary>
 		int StackTraceLimit { get; }
 
+		/// <summary>
+		/// By setting this to <code>true</code> the agent will ignore the sampled part of the W3C TraceContext traceparent header
+		/// when it's false and when the upstream transaction is not coming from an Elastic APM agent.
+		/// In practice this means that in case a caller service calls another service where this value is <code>true</code>,
+		/// the agent will ignore the sampling decision of the upstream service, and it will make a new sampling decision.
+		///
+		/// This can be useful when a caller service always sets the sampled flag to false and the agent would have no chance to
+		/// create any sampled transaction.
+		/// </summary>
+		bool TraceContextIgnoreSampledFalse { get; }
+
 		/// <summary>
 		/// A list of patterns to match HTTP requests to ignore. An incoming HTTP request whose request line matches any of the
 		/// patterns will not be reported as a transaction.
@@ -283,14 +297,17 @@ public interface IConfigurationReader
 
 		/// <summary>
 		/// The sample rate for transactions.
-		/// By default, the agent will sample every transaction (e.g. a request to your service). To reduce overhead and storage requirements,
-		/// the sample rate can be set to a value between 0.0 and 1.0. The agent still records the overall time and result for unsampled
+		/// By default, the agent will sample every transaction (e.g. a request to your service). To reduce overhead and storage
+		/// requirements,
+		/// the sample rate can be set to a value between 0.0 and 1.0. The agent still records the overall time and result for
+		/// unsampled
 		/// transactions, but no context information, labels, or spans are recorded.
 		/// </summary>
 		double TransactionSampleRate { get; }
 
 		/// <summary>
-		/// If <c>true</c>, for all outgoing HTTP requests the agent stores the traceparent in a "elastic-apm-traceparent" header name.
+		/// If <c>true</c>, for all outgoing HTTP requests the agent stores the traceparent in a "elastic-apm-traceparent" header
+		/// name.
 		/// Otherwise, it'll use the official w3c "traceparent" header name.
 		/// </summary>
 		bool UseElasticTraceparentHeader { get; }

src/Elastic.Apm/Model/Transaction.cs

@@ -16,8 +16,6 @@
 using Elastic.Apm.Report;
 using Elastic.Apm.ServerInfo;
 using Elastic.Apm.Libraries.Newtonsoft.Json;
-using Elastic.Apm.Libraries.Newtonsoft.Json.Converters;
-using Elastic.Apm.Libraries.Newtonsoft.Json.Serialization;
 
 namespace Elastic.Apm.Model
 {
@@ -34,6 +32,7 @@ internal class Transaction : ITransaction
 
 		// This constructor is meant for serialization
 		[JsonConstructor]
+		// ReSharper disable once UnusedMember.Local
 		private Transaction(Context context, string name, string type, double duration, long timestamp, string id, string traceId, string parentId,
 			bool isSampled, string result, SpanCount spanCount
 		)
@@ -54,8 +53,7 @@ private Transaction(Context context, string name, string type, double duration,
 		// This constructor is used only by tests that don't care about sampling and distributed tracing
 		internal Transaction(ApmAgent agent, string name, string type)
 			: this(agent.Logger, name, type, new Sampler(1.0), null, agent.PayloadSender, agent.ConfigStore.CurrentSnapshot,
-				agent.TracerInternal.CurrentExecutionSegmentsContainer, null)
-		{ }
+				agent.TracerInternal.CurrentExecutionSegmentsContainer, null) { }
 
 		/// <summary>
 		/// Creates a new transaction
@@ -178,9 +176,12 @@ internal Transaction(
 			}
 			else
 			{
+				var idBytes = new byte[8];
+
 				if (_activity != null)
 				{
 					Id = _activity.SpanId.ToHexString();
+					_activity.SpanId.CopyTo(new Span<byte>(idBytes));
 
 					// try to set the parent id and tracestate on the created activity, based on passed distributed tracing data.
 					// This is so that the distributed tracing data will flow to any child activities
@@ -200,17 +201,25 @@ internal Transaction(
 					}
 				}
 				else
-				{
-					var idBytes = new byte[8];
 					Id = RandomGenerator.GenerateRandomBytesAsString(idBytes);
-				}
 
 				TraceId = distributedTracingData.TraceId;
 				ParentId = distributedTracingData.ParentId;
-				IsSampled = distributedTracingData.FlagRecorded;
 				isSamplingFromDistributedTracingData = true;
 				_traceState = distributedTracingData.TraceState;
 
+				// If TraceContextIgnoreSampledFalse is set and the upstream service is not from our agent (aka no sample rate set)
+				// ignore the sampled flag and make a new sampling decision.
+				if (configSnapshot.TraceContextIgnoreSampledFalse && (distributedTracingData.TraceState == null
+					|| !distributedTracingData.TraceState.SampleRate.HasValue && !distributedTracingData.FlagRecorded))
+				{
+					IsSampled = sampler.DecideIfToSample(idBytes);
+					_traceState?.SetSampleRate(sampler.Rate);
+				}
+				else
+					IsSampled = distributedTracingData.FlagRecorded;
+
+
 				// If there is no tracestate or no valid "es" vendor entry with an "s" (sample rate) attribute, then the agent must
 				// omit sample rate from non-root transactions and their spans.
 				// See https://github.com/elastic/apm/blob/master/specs/agents/tracing-sampling.md#propagation
@@ -231,7 +240,8 @@ internal Transaction(
 			{
 				_logger.Trace()
 					?.Log("New Transaction instance created: {Transaction}. " +
-						"IsSampled ({IsSampled}) and SampleRate ({SampleRate}) is based on incoming distributed tracing data ({DistributedTracingData})." +
+						"IsSampled ({IsSampled}) and SampleRate ({SampleRate}) is based on incoming distributed tracing data ({DistributedTracingData})."
+						+
 						" Start time: {Time} (as timestamp: {Timestamp})",
 						this, IsSampled, SampleRate, distributedTracingData, TimeUtils.FormatTimestampForLog(Timestamp), Timestamp);
 			}
@@ -341,9 +351,9 @@ public Outcome Outcome
 		}
 
 		/// <summary>
-		/// In general if there is an error on the span, the outcome will be <see cref="Outcome.Failure"/>, otherwise it'll be <see cref="Outcome.Success"/>.
-		/// There are some exceptions to this (see spec: https://github.com/elastic/apm/blob/master/specs/agents/tracing-spans.md#span-outcome) when it can be <see cref="Outcome.Unknown"/>.
-		/// Use <see cref="_outcomeChangedThroughApi"/> to check if it was specifically set to <see cref="Outcome.Unknown"/>, or if it's just the default value.
+		/// In general if there is an error on the span, the outcome will be <see cref="Elastic.Apm.Api.Outcome.Failure"/>, otherwise it'll be <see cref="Elastic.Apm.Api.Outcome.Success"/>.
+		/// There are some exceptions to this (see spec: https://github.com/elastic/apm/blob/master/specs/agents/tracing-spans.md#span-outcome) when it can be <see cref="Elastic.Apm.Api.Outcome.Unknown"/>.
+		/// Use <see cref="_outcomeChangedThroughApi"/> to check if it was specifically set to <see cref="Elastic.Apm.Api.Outcome.Unknown"/>, or if it's just the default value.
 		/// </summary>
 		internal Outcome _outcome;