Fix issue where querier trace spans are not nested correctly (#6893)

* Fix issue where querier trace spans are not nested correctly.

Resolves #5341

* Add changelog entry.

CHANGELOG.md

@@ -15,6 +15,7 @@
 * [BUGFIX] Fix panic during tsdb Commit #6766
 * [BUGFIX] tsdb/head: wlog exemplars after samples #6766
 * [BUGFIX] Ruler: fix issue where "failed to remotely evaluate query expression, will retry" messages are logged without context such as the trace ID and do not appear in trace events. #6789
+* [BUGFIX] Querier: fix issue where spans in query request traces were not nested correctly. #6893
 
 ### Mixin
 

pkg/querier/worker/scheduler_processor.go

@@ -210,6 +210,10 @@ func (sp *schedulerProcessor) querierLoop(execCtx context.Context, c schedulerpb
 				defer queueSpan.Finish()
 
 				ctx = spanCtx
+
+				if err := sp.updateTracingHeaders(request.HttpRequest, queueSpan); err != nil {
+					level.Warn(sp.log).Log("msg", "could not update trace headers on httpgrpc request, trace may be malformed", "err", err)
+				}
 			}
 			logger := util_log.WithContext(ctx, sp.log)
 
@@ -298,6 +302,30 @@ func (sp *schedulerProcessor) runRequest(ctx context.Context, logger log.Logger,
 	}
 }
 
+func (sp *schedulerProcessor) updateTracingHeaders(request *httpgrpc.HTTPRequest, span opentracing.Span) error {
+	// Reset any trace headers on the HTTP request with the new parent span ID: the child span for the HTTP request created
+	// by the HTTP tracing infrastructure uses the trace information in the HTTP request headers, ignoring the trace
+	// information in the Golang context.
+	return span.Tracer().Inject(span.Context(), opentracing.HTTPHeaders, httpGrpcHeaderWriter{request})
+}
+
+type httpGrpcHeaderWriter struct {
+	request *httpgrpc.HTTPRequest
+}
+
+var _ opentracing.TextMapWriter = httpGrpcHeaderWriter{}
+
+func (w httpGrpcHeaderWriter) Set(key, val string) {
+	for _, h := range w.request.Headers {
+		if h.Key == key {
+			h.Values = []string{val}
+			return
+		}
+	}
+
+	w.request.Headers = append(w.request.Headers, &httpgrpc.Header{Key: key, Values: []string{val}})
+}
+
 func (sp *schedulerProcessor) createFrontendClient(addr string) (client.PoolClient, error) {
 	opts, err := sp.grpcConfig.DialOption([]grpc.UnaryClientInterceptor{
 		otgrpc.OpenTracingClientInterceptor(opentracing.GlobalTracer()),