overhaul tracing, add tests, docs (#416)

* refactor tracing

* update modules

* add zipkin exporter support

* bump influxdb to v1.8.0

* update docs and example configs

* add spin release instructions

* ensure upstream response body is closed

cmd/trickster/conf/example.conf

@@ -1,5 +1,5 @@
 #
-# Trickster 1.0 Example Configuration File - Exhaustive
+# Trickster 1.1 Example Configuration File - Exhaustive
 #
 # To use this, run: trickster -config /path/to/example.conf
 #
@@ -238,6 +238,11 @@ listen_port = 8480
     # access this origin via http[s]://trickster-fqdn/default/ unless path_routing_disabled is true
     [origins.default]
 
+    # origin_type identifies the origin type.
+    # Valid options are: 'prometheus', 'influxdb', 'clickhouse', 'irondb', 'reverseproxycache' (or just 'rpc')
+    # origin_type is a required configuration value
+    origin_type = 'prometheus'
+
     ## is_default describes whether this origin is the default origin considered when routing http requests
     ## it is false, by default; but if you only have a single origin configured, is_default will be true unless explicitly set to false
     # is_default = true
@@ -247,11 +252,6 @@ listen_port = 8480
     ## default is false
     # path_routing_disabled = false
 
-    # origin_type identifies the origin type.
-    # Valid options are: 'prometheus', 'influxdb', 'clickhouse', 'irondb', 'reverseproxycache' (or just 'rpc')
-    # origin_type is a required configuration value
-    origin_type = 'prometheus'
-
     ## hosts indicates which FQDNs requested by the client should route to this Origin (in addition to path-based routing)
     ## if you are using TLS, all FQDNs should be included in the certfiicate common names to avoid insecure warnings to clients
     ## default setting is empty list. List format is: hosts = [ '1.example.com', '2.example.com' ]
@@ -466,29 +466,64 @@ listen_port = 8480
     ## to origins by the 'tracing_name' value in origin configs, which, by default, use 'default'
     # [tracing.default]
 
-    ## implementation specifies the Tracing API that you use in your platform.
-    ## options are: 'opentelemetry' (default) and 'opentracing'
-    # implementation: 'opentelemetry'
+    ## tracer_type specifies the type backend tracing system where traces are sent
+    ## options are: jaeger, zipkin, stdout or none.  none is the default
+    # tracer_type = 'none'
+
+    ## service_name specifies the service.name tag/attribute value included in traces
+    ## default is 'trickster'
+    # service_name = 'trickster'
 
-    ## exporter specifies the tracing format sent to the collection system
-    ## options are: jaeger, recorder, stdout or noop.  noop is the default
-    # exporter = 'noop'
+    ## collector_url is the URL of the tracing backend
+    ## required for zipkin and jaeger, unused for stdout
+    # collector_url = ''
 
-    ## collector is the URL of the trace collector which MUST be in the exporter format
-    ## the default is empty string, meaning no traces are sent to the collector
-    # collector = ''
+    ## collector_user is the username credential for authenticating with the tracing backend
+    ## optional jaeger; unused for zipkin and stdout
+    # collector_user = ''
 
-	## An integer or floating point value of 0 to 1.0 (inclusive) is permitted.
-	## sampleRate sets the probability that a span will be recorded.
+    ## collector_pass is the username credential for authenticating with the tracing backend
+    ## optional jaeger; unused for zipkin and stdout
+    # collector_pass = ''
+
+	## sample_rate sets the probability that a span will be recorded.
+	## A floating point value of 0.0 to 1.0 (inclusive) is permitted
     ## default is 1.0 (meaning 100% of requests are recorded)
 	# sample_rate = 1.0
 
-    ## another example tracing config named 'example' using jaeger backend and a 50% sample rate
+      ## tags will append these tags/attributes to each trace that is recorded
+      ## only string key/value tags are supported. numeric values, etc are not.
+      ## default tags list is empty
+      # [tracing.default.tags]
+      # key1 = "value1"
+      # key2 = "value2"
+
+      ## configurations for this tracer, specific to jaeger
+      # [tracing.default.jaeger]
+      ## endpoint_type indicates whether the jaeger tracing backend is a 'collector' or 'agent'
+      ## default is 'collector'
+      # endpoint_type = 'collector'
+
+      ## configurations for this tracer, specific to stdout
+      # [tracing.default.stdout]
+      ## pretty_print indicates whether the output to stdout is formatted better human readability
+      ## default is false
+      # pretty_print = false
+
+    ## another example tracing config named 'example' using jaeger agent backend and a 50% sample rate
     # [tracing.example]
-    # implementation = 'opentelemetry'
-    # exporter = 'jaeger'
-    # collector = 'https://jaeger.example.com/'
+    # tracer_type = 'jaeger'
+    # collector_url = 'https://jaeger.example.com/'
 	# sample_rate = .5
+    #   [tracing.example.jaeger]
+    #   endpoint_type = 'agent'
+
+    ## another example tracing config named 'zipkin-example' using zipkin backend and a 10% sample rate
+    # [tracing.zipkin-example]
+    # tracer_type = 'zipkin'
+    # collector_url = 'https://zipkin.example.com/'
+	# sample_rate = .1
+
 
 ## Configuration Options for Metrics Instrumentation
 # [metrics]

cmd/trickster/conf/simple.prometheus.conf

@@ -1,5 +1,5 @@
 #
-# Trickster 1.0 Example Configuration File - Simple Prometheus Reverse Proxy Cache
+# Trickster 1.x Example Configuration File - Simple Prometheus Reverse Proxy Cache
 #
 # To use this, run: trickster -config /path/to/simple.prometheus.conf
 #

cmd/trickster/conf/simple.reverseproxycache.conf

@@ -1,5 +1,5 @@
 #
-# Trickster 1.0 Example Configuration File - Simple HTTP Reverse Proxy Cache
+# Trickster 1.x Example Configuration File - Simple HTTP Reverse Proxy Cache
 #
 # To use this, run: trickster -config /path/to/simple.reverseproxycache.conf
 #

cmd/trickster/config.go

@@ -35,18 +35,19 @@ import (
 	th "github.com/tricksterproxy/trickster/pkg/proxy/handlers"
 	"github.com/tricksterproxy/trickster/pkg/routing"
 	"github.com/tricksterproxy/trickster/pkg/runtime"
+	tr "github.com/tricksterproxy/trickster/pkg/tracing/registration"
 	"github.com/tricksterproxy/trickster/pkg/util/log"
 	tl "github.com/tricksterproxy/trickster/pkg/util/log"
 	"github.com/tricksterproxy/trickster/pkg/util/metrics"
-	tr "github.com/tricksterproxy/trickster/pkg/util/tracing/registration"
 )
 
 var cfgLock = &sync.Mutex{}
 
 func runConfig(oldConf *config.Config, wg *sync.WaitGroup, log *log.Logger,
 	oldCaches map[string]cache.Cache, args []string, errorsFatal bool) {
 
-	metrics.BuildInfo.WithLabelValues(applicationGoVersion, applicationGitCommitID, applicationVersion).Set(1)
+	metrics.BuildInfo.WithLabelValues(applicationGoVersion,
+		applicationGitCommitID, applicationVersion).Set(1)
 
 	cfgLock.Lock()
 	defer cfgLock.Unlock()
@@ -71,7 +72,8 @@ func runConfig(oldConf *config.Config, wg *sync.WaitGroup, log *log.Logger,
 
 	err = validateConfig(conf)
 	if err != nil {
-		handleStartupIssue("ERROR: Could not load configuration: "+err.Error(), nil, nil, errorsFatal)
+		handleStartupIssue("ERROR: Could not load configuration: "+err.Error(),
+			nil, nil, errorsFatal)
 	}
 	if flags.ValidateConfig {
 		fmt.Println("Trickster configuration validation succeeded.")
@@ -99,37 +101,29 @@ func applyConfig(conf, oldConf *config.Config, wg *sync.WaitGroup, log *log.Logg
 		log.Warn(w, tl.Pairs{})
 	}
 
-	// TODO: move to function and  handle differences
 	//Register Tracing Configurations
-	tracerFlushers, err := tr.RegisterAll(conf, log)
+	tracers, err := tr.RegisterAll(conf, log)
 	if err != nil {
 		handleStartupIssue("tracing registration failed", tl.Pairs{"detail": err.Error()},
 			log, errorsFatal)
 		return
 	}
 
-	if len(tracerFlushers) > 0 {
-		for _, f := range tracerFlushers {
-			// TODO: Move elsewhere, these will close the tracer flushers prematurely here
-			defer f()
-		}
-	}
-
 	// every config (re)load is a new router
 	router := mux.NewRouter()
 	router.HandleFunc(conf.Main.PingHandlerPath, th.PingHandleFunc(conf)).Methods(http.MethodGet)
 
 	var caches = applyCachingConfig(conf, oldConf, log, oldCaches)
 	rh := handlers.ReloadHandleFunc(runConfig, conf, wg, log, caches, args)
 
-	_, err = routing.RegisterProxyRoutes(conf, router, caches, log, false)
+	_, err = routing.RegisterProxyRoutes(conf, router, caches, tracers, log, false)
 	if err != nil {
 		handleStartupIssue("route registration failed", tl.Pairs{"detail": err.Error()},
 			log, errorsFatal)
 		return
 	}
 
-	applyListenerConfigs(conf, oldConf, router, http.HandlerFunc(rh), log)
+	applyListenerConfigs(conf, oldConf, router, http.HandlerFunc(rh), log, tracers)
 
 	metrics.LastReloadSuccessfulTimestamp.Set(float64(time.Now().Unix()))
 	metrics.LastReloadSuccessful.Set(1)
@@ -278,16 +272,20 @@ func validateConfig(conf *config.Config) error {
 		fmt.Println(w)
 	}
 
-	// TODO: Tracers w/ Dry Run
-
 	var caches = make(map[string]cache.Cache)
 	for k := range conf.Caches {
 		caches[k] = nil
 	}
 
 	router := mux.NewRouter()
 	log := log.ConsoleLogger(conf.Logging.LogLevel)
-	_, err := routing.RegisterProxyRoutes(conf, router, caches, log, false)
+
+	tracers, err := tr.RegisterAll(conf, log)
+	if err != nil {
+		return err
+	}
+
+	_, err = routing.RegisterProxyRoutes(conf, router, caches, tracers, log, false)
 	if err != nil {
 		return err
 	}

cmd/trickster/listeners.go

@@ -29,6 +29,7 @@ import (
 	ph "github.com/tricksterproxy/trickster/pkg/proxy/handlers"
 	sw "github.com/tricksterproxy/trickster/pkg/proxy/tls"
 	"github.com/tricksterproxy/trickster/pkg/routing"
+	"github.com/tricksterproxy/trickster/pkg/tracing"
 	"github.com/tricksterproxy/trickster/pkg/util/log"
 	tl "github.com/tricksterproxy/trickster/pkg/util/log"
 	"github.com/tricksterproxy/trickster/pkg/util/metrics"
@@ -47,7 +48,7 @@ type listenerGroup struct {
 }
 
 func startListener(listenerName, address string, port int, connectionsLimit int,
-	tlsConfig *tls.Config, router http.Handler, wg *sync.WaitGroup,
+	tlsConfig *tls.Config, router http.Handler, wg *sync.WaitGroup, tracers tracing.Tracers,
 	exitOnError bool, log *tl.Logger) error {
 	if wg != nil {
 		defer wg.Done()
@@ -78,6 +79,15 @@ func startListener(listenerName, address string, port int, connectionsLimit int,
 	lg.listener = l
 	listeners[listenerName] = lg
 
+	// defer the tracer flush here where the listener connection ends
+	if tracers != nil {
+		for _, v := range tracers {
+			if v != nil && v.Flusher != nil {
+				defer v.Flusher()
+			}
+		}
+	}
+
 	err = http.Serve(l, handlers.CompressHandler(lg.routeSwapper))
 	if err != nil {
 		log.Error("http listener stopping", tl.Pairs{"name": listenerName, "detail": err})
@@ -90,15 +100,16 @@ func startListener(listenerName, address string, port int, connectionsLimit int,
 
 func startListenerRouter(listenerName, address string, port int, connectionsLimit int,
 	tlsConfig *tls.Config, path string, handler http.Handler, wg *sync.WaitGroup,
-	exitOnError bool, log *tl.Logger) error {
+	tracers tracing.Tracers, exitOnError bool, log *tl.Logger) error {
 	router := http.NewServeMux()
 	router.Handle(path, handler)
 	return startListener(listenerName, address, port, connectionsLimit,
-		tlsConfig, router, wg, exitOnError, log)
+		tlsConfig, router, wg, tracers, exitOnError, log)
 }
 
 func applyListenerConfigs(conf, oldConf *config.Config,
-	router, reloadHandler http.Handler, log *log.Logger) {
+	router, reloadHandler http.Handler, log *log.Logger,
+	tracers tracing.Tracers) {
 
 	var err error
 	var routerRefreshed bool
@@ -137,6 +148,8 @@ func applyListenerConfigs(conf, oldConf *config.Config,
 
 	drainTime := time.Duration(conf.ReloadConfig.DrainTimeoutSecs) * time.Second
 
+	var tracerFlusherSet bool
+
 	// if TLS port is configured and at least one origin is mapped to a good tls config,
 	// then set up the tls server listener instance
 	if conf.Frontend.ServeTLS && conf.Frontend.TLSListenPort > 0 && (!hasOldFC ||
@@ -152,9 +165,10 @@ func applyListenerConfigs(conf, oldConf *config.Config,
 		} else {
 			wg.Add(1)
 			routerRefreshed = true
+			tracerFlusherSet = true
 			go startListener("tlsListener",
 				conf.Frontend.TLSListenAddress, conf.Frontend.TLSListenPort,
-				conf.Frontend.ConnectionsLimit, tlsConfig, router, wg, true, log)
+				conf.Frontend.ConnectionsLimit, tlsConfig, router, wg, tracers, true, log)
 		}
 
 	} else if !conf.Frontend.ServeTLS && hasOldFC && oldConf.Frontend.ServeTLS {
@@ -180,9 +194,15 @@ func applyListenerConfigs(conf, oldConf *config.Config,
 		spinDownListener("httpListener", drainTime)
 		wg.Add(1)
 		routerRefreshed = true
+
+		var t2 tracing.Tracers
+		if !tracerFlusherSet {
+			t2 = tracers
+		}
+
 		go startListener("httpListener",
 			conf.Frontend.ListenAddress, conf.Frontend.ListenPort,
-			conf.Frontend.ConnectionsLimit, nil, router, wg, true, log)
+			conf.Frontend.ConnectionsLimit, nil, router, wg, t2, true, log)
 
 	}
 
@@ -200,7 +220,7 @@ func applyListenerConfigs(conf, oldConf *config.Config,
 		wg.Add(1)
 		go startListener("metricsListener",
 			conf.Metrics.ListenAddress, conf.Metrics.ListenPort,
-			conf.Frontend.ConnectionsLimit, nil, mr, wg, true, log)
+			conf.Frontend.ConnectionsLimit, nil, mr, wg, nil, true, log)
 	}
 
 	// if the Reload HTTP port is configured, then set up the http listener instance
@@ -218,7 +238,7 @@ func applyListenerConfigs(conf, oldConf *config.Config,
 
 		go startListener("reloadListener",
 			conf.ReloadConfig.ListenAddress, conf.ReloadConfig.ListenPort,
-			conf.Frontend.ConnectionsLimit, nil, mr, wg, true, log)
+			conf.Frontend.ConnectionsLimit, nil, mr, wg, nil, true, log)
 	}
 
 	if routerRefreshed {

cmd/trickster/listeners_test.go

@@ -30,10 +30,10 @@ func TestListeners(t *testing.T) {
 
 	wg := &sync.WaitGroup{}
 	var err error
+	wg.Add(1)
 	go func() {
-		wg.Add(1)
 		err = startListener("httpListener",
-			"", 0, 20, nil, http.NewServeMux(), wg, false, log.ConsoleLogger("info"))
+			"", 0, 20, nil, http.NewServeMux(), wg, nil, false, log.ConsoleLogger("info"))
 	}()
 
 	time.Sleep(time.Millisecond * 300)
@@ -44,10 +44,11 @@ func TestListeners(t *testing.T) {
 		t.Error("expected non-nil err")
 	}
 
+	wg.Add(1)
 	go func() {
-		wg.Add(1)
 		err = startListenerRouter("httpListener2",
-			"", 0, 20, nil, "/", http.HandlerFunc(handlers.HandleLocalResponse), wg, false, log.ConsoleLogger("info"))
+			"", 0, 20, nil, "/", http.HandlerFunc(handlers.HandleLocalResponse), wg,
+			nil, false, log.ConsoleLogger("info"))
 	}()
 	time.Sleep(time.Millisecond * 300)
 	l = listeners["httpListener2"]