basic tracing support draft

.gitignore

@@ -41,3 +41,6 @@ cacheKey.expiration
 
 #log testing
 out.log
+
+vendor
+trickster

cmd/trickster/main.go

@@ -30,6 +30,7 @@ import (
 	"github.com/Comcast/trickster/internal/runtime"
 	"github.com/Comcast/trickster/internal/util/log"
 	"github.com/Comcast/trickster/internal/util/metrics"
+	"github.com/Comcast/trickster/internal/util/tracing"
 
 	"github.com/gorilla/handlers"
 )
@@ -84,6 +85,8 @@ func main() {
 	}
 
 	metrics.Init()
+	flush := tracing.Init(config.Tracing)
+	defer flush()
 	cr.LoadCachesFromConfig()
 	th.RegisterPingHandler()
 	th.RegisterConfigHandler()

go.mod

@@ -5,39 +5,30 @@ replace gotest.tools => github.com/gotestyourself/gotest.tools v2.2.0+incompatib
 require (
 	github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect
 	github.com/BurntSushi/toml v0.3.1
-	github.com/alecthomas/gometalinter v3.0.0+incompatible // indirect
 	github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6 // indirect
 	github.com/alicebob/miniredis v2.5.0+incompatible
-	github.com/boltdb/bolt v1.3.1 // indirect
 	github.com/coreos/bbolt v1.3.3
 	github.com/dgraph-io/badger v1.6.0
 	github.com/dgryski/go-farm v0.0.0-20191112170834-c2139c5d712b // indirect
-	github.com/fatih/color v1.7.0 // indirect
 	github.com/go-kit/kit v0.9.0
 	github.com/go-logfmt/logfmt v0.5.0 // indirect
 	github.com/go-redis/redis v6.15.6+incompatible
 	github.com/go-stack/stack v1.8.0
-	github.com/gojp/goreportcard v0.0.0-20200102082842-9f8184163627 // indirect
 	github.com/golang/snappy v0.0.1
 	github.com/gomodule/redigo v2.0.0+incompatible // indirect
-	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
 	github.com/gorilla/handlers v1.4.2
 	github.com/gorilla/mux v1.7.3
 	github.com/influxdata/influxdb v1.7.9
-	github.com/mattn/go-colorable v0.1.4 // indirect
-	github.com/mattn/go-isatty v0.0.10 // indirect
-	github.com/mattn/goveralls v0.0.4 // indirect
 	github.com/philhofer/fwd v1.0.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/prometheus/client_golang v1.3.0
 	github.com/prometheus/common v0.8.0
-	github.com/rakyll/gotest v0.0.0-20191108192113-45d501058f2a // indirect
 	github.com/tinylib/msgp v1.1.1
 	github.com/yuin/gopher-lua v0.0.0-20190514113301-1cd887cd7036 // indirect
+	go.opentelemetry.io/otel v0.2.0
+	go.opentelemetry.io/otel/exporter/trace/jaeger v0.2.0
 	golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa
 	golang.org/x/sys v0.0.0-20200116001909-b77594299b42
-	golang.org/x/tools v0.0.0-20191206204035-259af5ff87bd // indirect
-	gopkg.in/alecthomas/kingpin.v3-unstable v3.0.0-20191105091915-95d230a53780 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
 )
 

internal/config/config.go

@@ -45,6 +45,9 @@ var Logging *LoggingConfig
 // Metrics is the Metrics subsection of the Running Configuration
 var Metrics *MetricsConfig
 
+// Tracing defines destricbuted trace options for the Running Configuration
+var Tracing *TracingConfig
+
 // NegativeCacheConfigs is the NegativeCacheConfig subsection of the Running Configuration
 var NegativeCacheConfigs map[string]NegativeCacheConfig
 
@@ -71,6 +74,8 @@ type TricksterConfig struct {
 	Logging *LoggingConfig `toml:"logging"`
 	// Metrics provides configurations for collecting Metrics about the application
 	Metrics *MetricsConfig `toml:"metrics"`
+	// Tracing provides the distributed tracing configuration
+	Tracing *TracingConfig `toml:"tracing"`
 	// NegativeCacheConfigs is a map of NegativeCacheConfigs
 	NegativeCacheConfigs map[string]NegativeCacheConfig `toml:"negative_caches"`
 
@@ -339,6 +344,16 @@ type MetricsConfig struct {
 	ListenPort int `toml:"listen_port"`
 }
 
+// Tracing provides the distributed tracing configuration
+type TracingConfig struct {
+	// Implementation is the particular implementation to use TODO generate with Rob Pike's Stringer
+	Implementation string `toml:"tracer_implementation"`
+	// CollectorEndpoint is the URL of the trace collector it MUST be of Implementation implementation
+	CollectorEndpoint string `toml:"tracing_collector"`
+	// SampleRate sets the probability that a span will be recorded. Values between 0 and 1 are accepted.
+	SampleRate float64 `toml:"sample_rate"`
+}
+
 // NegativeCacheConfig is a collection of response codes and their TTLs
 type NegativeCacheConfig map[string]int
 
@@ -368,6 +383,10 @@ func NewConfig() *TricksterConfig {
 		Metrics: &MetricsConfig{
 			ListenPort: defaultMetricsListenPort,
 		},
+		Tracing: &TracingConfig{
+			Implementation:    defaultTracerImplemetation,
+			CollectorEndpoint: "",
+		},
 		Origins: map[string]*OriginConfig{
 			"default": NewOriginConfig(),
 		},
@@ -823,6 +842,9 @@ func (c *TricksterConfig) copy() *TricksterConfig {
 	nc.Metrics.ListenAddress = c.Metrics.ListenAddress
 	nc.Metrics.ListenPort = c.Metrics.ListenPort
 
+	nc.Tracing.Implementation = c.Tracing.Implementation
+	nc.Tracing.CollectorEndpoint = c.Tracing.CollectorEndpoint
+
 	nc.Frontend.ListenAddress = c.Frontend.ListenAddress
 	nc.Frontend.ListenPort = c.Frontend.ListenPort
 	nc.Frontend.TLSListenAddress = c.Frontend.TLSListenAddress

internal/config/defaults.go

@@ -23,8 +23,11 @@ const (
 	defaultMetricsListenPort    = 8082
 	defaultMetricsListenAddress = ""
 
-	defaultCacheType   = "memory"
-	defaultCacheTypeID = CacheTypeMemory
+	defaultTracerImplemetation = "stdout"
+
+	defaultCacheType        = "memory"
+	defaultCacheTypeID      = CacheTypeMemory
+	defaultCacheCompression = true
 
 	defaultTimeseriesTTLSecs  = 21600
 	defaultFastForwardTTLSecs = 15

internal/config/loader.go

@@ -80,6 +80,7 @@ func Load(applicationName string, applicationVersion string, arguments []string)
 	Frontend = c.Frontend
 	Logging = c.Logging
 	Metrics = c.Metrics
+	Tracing = c.Tracing
 	NegativeCacheConfigs = c.NegativeCacheConfigs
 
 	for k, n := range NegativeCacheConfigs {

internal/proxy/engines/cache.go

@@ -14,6 +14,7 @@
 package engines
 
 import (
+	"context"
 	"mime"
 	"net/http"
 	"strings"
@@ -26,10 +27,19 @@ import (
 	"github.com/Comcast/trickster/internal/proxy/headers"
 	"github.com/Comcast/trickster/internal/proxy/ranges/byterange"
 	"github.com/Comcast/trickster/internal/util/log"
+	"github.com/Comcast/trickster/internal/util/tracing"
+
+	kv "go.opentelemetry.io/otel/api/key"
 )
 
 // QueryCache queries the cache for an HTTPDocument and returns it
-func QueryCache(c cache.Cache, key string, ranges byterange.Ranges) (*HTTPDocument, status.LookupStatus, byterange.Ranges, error) {
+func QueryCache(ctx context.Context, c cache.Cache, key string, ranges byterange.Ranges) (*HTTPDocument, status.LookupStatus, byterange.Ranges, error) {
+	ctx, span := tracing.NewChildSpan(ctx, "QueryCache")
+	defer func() {
+
+		span.End()
+
+	}()
 
 	d := &HTTPDocument{}
 	var lookupStatus status.LookupStatus
@@ -106,6 +116,10 @@ func QueryCache(c cache.Cache, key string, ranges byterange.Ranges) (*HTTPDocume
 	d.isFulfillment = (d.Ranges != nil && len(d.Ranges) > 0) && (ranges == nil || len(ranges) == 0)
 
 	if d.isFulfillment {
+		span.AddEvent(
+			ctx,
+			"Cache Fulfillment",
+		)
 		ranges = byterange.Ranges{byterange.Range{Start: 0, End: d.ContentLength - 1}}
 	}
 
@@ -131,7 +145,12 @@ func stripConditionalHeaders(h http.Header) {
 }
 
 // WriteCache writes an HTTPDocument to the cache
-func WriteCache(c cache.Cache, key string, d *HTTPDocument, ttl time.Duration, compressTypes map[string]bool) error {
+func WriteCache(ctx context.Context, c cache.Cache, key string, d *HTTPDocument, ttl time.Duration, compressTypes map[string]bool) error {
+	ctx, span := tracing.NewChildSpan(ctx, "WriteCache")
+	defer func() {
+
+		span.End()
+	}()
 
 	h := http.Header(d.Headers)
 	h.Del(headers.NameDate)
@@ -150,6 +169,7 @@ func WriteCache(c cache.Cache, key string, d *HTTPDocument, ttl time.Duration, c
 				compress = true
 			}
 		}
+
 	}
 
 	// for memory cache, don't serialize the document, since we can retrieve it by reference.
@@ -180,7 +200,22 @@ func WriteCache(c cache.Cache, key string, d *HTTPDocument, ttl time.Duration, c
 	} else {
 		bytes = append([]byte{0}, bytes...)
 	}
-	return c.Store(key, bytes, ttl)
+
+	err := c.Store(key, bytes, ttl)
+	if err != nil {
+		span.AddEvent(
+			ctx,
+			"Cache Write Failure",
+			kv.String("Error", err.Error()),
+		)
+		return err
+	}
+	span.AddEvent(
+		ctx,
+		"Cache Write",
+		kv.Int("bytesWritten", len(bytes)),
+	)
+	return nil
 
 }