Add initial instrumentation to Azure

CHANGELOG.md

@@ -3,6 +3,7 @@
 * [FEATURE] Added the ability to hedge requests with all backends [#750](https://github.com/grafana/tempo/pull/750) (@joe-elliott)
 * [ENHANCEMENT] Performance: improve compaction speed with concurrent reads and writes [#754](https://github.com/grafana/tempo/pull/754) (@mdisibio)
 * [ENHANCEMENT] Improve readability of cpu and memory metrics on operational dashboard [#764](https://github.com/grafana/tempo/pull/764) (@bboreham)
+* [ENHANCEMENT] Add `azure_request_duration_seconds` metric. [#767](https://github.com/grafana/tempo/pull/767) (@JosephWoodward)
 
 ## v1.0.1
 

tempodb/backend/azure/azure_helpers.go

@@ -3,6 +3,7 @@ package azure
 import (
 	"context"
 	"fmt"
+	"net/http"
 	"net/url"
 	"strings"
 	"time"
@@ -17,8 +18,8 @@ const (
 	uptoHedgedRequests = 2
 )
 
-func GetContainerURL(ctx context.Context, conf *Config, hedge bool) (blob.ContainerURL, error) {
-	c, err := blob.NewSharedKeyCredential(conf.StorageAccountName.String(), conf.StorageAccountKey.String())
+func GetContainerURL(ctx context.Context, cfg *Config, hedge bool) (blob.ContainerURL, error) {
+	c, err := blob.NewSharedKeyCredential(cfg.StorageAccountName.String(), cfg.StorageAccountKey.String())
 	if err != nil {
 		return blob.ContainerURL{}, err
 	}
@@ -31,32 +32,40 @@ func GetContainerURL(ctx context.Context, conf *Config, hedge bool) (blob.Contai
 		retryOptions.TryTimeout = time.Until(deadline)
 	}
 
-	var httpSender pipeline.Factory
-	if hedge && conf.HedgeRequestsAt != 0 {
-		httpSender = pipeline.FactoryFunc(func(next pipeline.Policy, po *pipeline.PolicyOptions) pipeline.PolicyFunc {
-			return func(ctx context.Context, request pipeline.Request) (pipeline.Response, error) {
-				client := hedgedhttp.NewClient(conf.HedgeRequestsAt, uptoHedgedRequests, nil)
+	customTransport := http.DefaultTransport.(*http.Transport).Clone()
 
-				// Send the request over the network
-				resp, err := client.Do(request.WithContext(ctx))
+	// add instrumentation
+	transport := newInstrumentedTransport(customTransport)
 
-				return pipeline.NewHTTPResponse(resp), err
-			}
-		})
+	// hedge if desired (0 means disabled)
+	if hedge && cfg.HedgeRequestsAt != 0 {
+		transport = hedgedhttp.NewRoundTripper(cfg.HedgeRequestsAt, uptoHedgedRequests, transport)
 	}
 
+	client := http.Client{Transport: transport}
+
+	httpSender := pipeline.FactoryFunc(func(next pipeline.Policy, po *pipeline.PolicyOptions) pipeline.PolicyFunc {
+		return func(ctx context.Context, request pipeline.Request) (pipeline.Response, error) {
+
+			// Send the request over the network
+			resp, err := client.Do(request.WithContext(ctx))
+
+			return pipeline.NewHTTPResponse(resp), err
+		}
+	})
+
 	p := blob.NewPipeline(c, blob.PipelineOptions{
 		Retry:      retryOptions,
 		Telemetry:  blob.TelemetryOptions{Value: "Tempo"},
 		HTTPSender: httpSender,
 	})
 
-	u, err := url.Parse(fmt.Sprintf("https://%s.%s", conf.StorageAccountName, conf.Endpoint))
+	u, err := url.Parse(fmt.Sprintf("https://%s.%s", cfg.StorageAccountName, cfg.Endpoint))
 
 	// If the endpoint doesn't start with blob.core we can assume Azurite is being used
 	// So the endpoint should follow Azurite URL style
-	if !strings.HasPrefix(conf.Endpoint, "blob.core") {
-		u, err = url.Parse(fmt.Sprintf("http://%s/%s", conf.Endpoint, conf.StorageAccountName))
+	if !strings.HasPrefix(cfg.Endpoint, "blob.core") {
+		u, err = url.Parse(fmt.Sprintf("http://%s/%s", cfg.Endpoint, cfg.StorageAccountName))
 	}
 
 	if err != nil {
@@ -65,7 +74,7 @@ func GetContainerURL(ctx context.Context, conf *Config, hedge bool) (blob.Contai
 
 	service := blob.NewServiceURL(*u, p)
 
-	return service.NewContainerURL(conf.ContainerName), nil
+	return service.NewContainerURL(cfg.ContainerName), nil
 }
 
 func GetContainer(ctx context.Context, conf *Config, hedge bool) (blob.ContainerURL, error) {

tempodb/backend/azure/instrumentation.go

@@ -0,0 +1,41 @@
+package azure
+
+import (
+	"net/http"
+	"strconv"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+var (
+	azureRequestDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace: "tempodb",
+		Name:      "azure_request_duration_seconds",
+		Help:      "Time spent doing Azure requests.",
+
+		Buckets: prometheus.ExponentialBuckets(0.005, 4, 6),
+	}, []string{"operation", "status_code"})
+)
+
+type instrumentedTransport struct {
+	observer prometheus.ObserverVec
+	next     http.RoundTripper
+}
+
+func newInstrumentedTransport(next http.RoundTripper) http.RoundTripper {
+	return instrumentedTransport{
+		observer: azureRequestDuration,
+		next:     next,
+	}
+}
+
+func (i instrumentedTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	start := time.Now()
+	resp, err := i.next.RoundTrip(req)
+	if err == nil {
+		i.observer.WithLabelValues(req.Method, strconv.Itoa(resp.StatusCode)).Observe(time.Since(start).Seconds())
+	}
+	return resp, err
+}