From 5db897a4fd967f9d75e99dbdb04b321b29b16b8a Mon Sep 17 00:00:00 2001 From: Joshua MacDonald Date: Thu, 4 Jan 2024 12:32:57 -0800 Subject: [PATCH 1/4] Let the trace exporter trace itself --- .../metric/exporters/otlp/otelcol/client.go | 6 +-- .../trace/exporters/otlp/otelcol/client.go | 52 +++++++++++++++++-- 2 files changed, 52 insertions(+), 6 deletions(-) diff --git a/lightstep/sdk/metric/exporters/otlp/otelcol/client.go b/lightstep/sdk/metric/exporters/otlp/otelcol/client.go index f6d54bee..c3da32fd 100644 --- a/lightstep/sdk/metric/exporters/otlp/otelcol/client.go +++ b/lightstep/sdk/metric/exporters/otlp/otelcol/client.go @@ -75,7 +75,7 @@ func NewDefaultConfig() Config { Timeout: 0, SendBatchSize: 1000, SendBatchMaxSize: 1500, - MaxInFlightSizeMiB: 32 * 1024 * 1024, + MaxInFlightSizeMiB: 32, }, Exporter: otelarrowexporter.Config{ TimeoutSettings: exporterhelper.TimeoutSettings{ @@ -216,11 +216,11 @@ func (c *client) ExportMetrics(ctx context.Context, data data.Metrics) error { ctx, "otelsdk_export_metrics", ) + defer span.End() + converted := c.d2pd(data) points := int64(converted.DataPointCount()) - defer span.End() - err := c.batcher.ConsumeMetrics(ctx, converted) success := err == nil diff --git a/lightstep/sdk/trace/exporters/otlp/otelcol/client.go b/lightstep/sdk/trace/exporters/otlp/otelcol/client.go index b9cf957a..a88bd5da 100644 --- a/lightstep/sdk/trace/exporters/otlp/otelcol/client.go +++ b/lightstep/sdk/trace/exporters/otlp/otelcol/client.go @@ -16,6 +16,7 @@ package otelcol import ( "context" + "errors" "time" "github.com/lightstep/otel-launcher-go/lightstep/sdk/internal" @@ -31,8 +32,11 @@ import ( "go.opentelemetry.io/collector/exporter/exporterhelper" "go.opentelemetry.io/collector/processor" "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + otelcodes "go.opentelemetry.io/otel/codes" noopmetric "go.opentelemetry.io/otel/metric/noop" "go.opentelemetry.io/otel/sdk/trace" + traceapi "go.opentelemetry.io/otel/trace" nooptrace "go.opentelemetry.io/otel/trace/noop" "go.uber.org/multierr" "go.uber.org/zap" @@ -55,10 +59,43 @@ type client struct { exporter exporter.Traces batcher processor.Traces settings exporter.CreateSettings + tracer traceapi.Tracer } func (c *client) ExportSpans(ctx context.Context, spans []trace.ReadOnlySpan) error { - return c.batcher.ConsumeTraces(ctx, c.d2pd(spans)) + ctx, span := c.tracer.Start( + ctx, + "otelsdk_export_traces", + ) + defer span.End() + + converted := c.d2pd(spans) + count := converted.SpanCount() + + err := c.batcher.ConsumeTraces(ctx, converted) + success := err == nil + var state string + if success { + state = "ok" + } else if errors.Is(err, context.Canceled) { + state = "canceled" + } else if errors.Is(err, context.DeadlineExceeded) { + state = "timeout" + } else { + state = "error" + } + + var attrs = []attribute.KeyValue{ + attribute.Bool("success", success), + attribute.String("state", state), + } + span.SetAttributes(append(attrs, attribute.Int("num_spans", count))...) + if err == nil { + span.SetStatus(otelcodes.Ok, state) + } else { + span.SetStatus(otelcodes.Error, state) + } + return err } func (c *client) Shutdown(ctx context.Context) error { @@ -75,12 +112,12 @@ func (c *client) Shutdown(ctx context.Context) error { func NewDefaultConfig() Config { return Config{ SelfMetrics: true, - SelfSpans: false, + SelfSpans: true, Batcher: concurrentbatchprocessor.Config{ Timeout: time.Second, SendBatchSize: 1000, SendBatchMaxSize: 1500, - MaxInFlightSizeMiB: 32 * 1024 * 1024, + MaxInFlightSizeMiB: 32, }, Exporter: otelarrowexporter.Config{ TimeoutSettings: exporterhelper.TimeoutSettings{ @@ -166,12 +203,21 @@ func NewExporter(ctx context.Context, cfg Config) (trace.SpanExporter, error) { if cfg.SelfSpans { c.settings.TelemetrySettings.TracerProvider = otel.GetTracerProvider() + c.tracer = c.settings.TelemetrySettings.TracerProvider.Tracer("lightstep-go/sdk/trace") } else { c.settings.TelemetrySettings.TracerProvider = nooptrace.NewTracerProvider() } if cfg.SelfMetrics { c.settings.TelemetrySettings.MeterProvider = otel.GetMeterProvider() c.settings.TelemetrySettings.MetricsLevel = configtelemetry.LevelNormal + // Note: the metrics SDK creates a counter at this + // point and counts points. The same is not done here + // because there is another layer above the exporter + // that is capable of dropping, and we want the + // otelsdk.* metrics to count all items, not only + // exported ones. There is no such intermediate + // processor between the periodic metric reader and + // the exporter. } else { c.settings.TelemetrySettings.MeterProvider = noopmetric.NewMeterProvider() } From 40cebf595296b9f8afe461d04be33867eb266bd4 Mon Sep 17 00:00:00 2001 From: Joshua MacDonald Date: Thu, 4 Jan 2024 13:09:45 -0800 Subject: [PATCH 2/4] BSP calls shutdown --- pipelines/common.go | 29 +++++++++++++++-------------- pipelines/go.mod | 23 +++++++++++------------ pipelines/go.sum | 4 ---- pipelines/trace.go | 26 ++++++++++++++------------ 4 files changed, 40 insertions(+), 42 deletions(-) diff --git a/pipelines/common.go b/pipelines/common.go index 8da5bc94..4ae453ac 100644 --- a/pipelines/common.go +++ b/pipelines/common.go @@ -15,14 +15,13 @@ package pipelines import ( - "github.com/lightstep/otel-launcher-go/lightstep/sdk/metric/exporters/otlp/otelcol" + otelcolmetric "github.com/lightstep/otel-launcher-go/lightstep/sdk/metric/exporters/otlp/otelcol" + otelcoltrace "github.com/lightstep/otel-launcher-go/lightstep/sdk/trace/exporters/otlp/otelcol" "go.opentelemetry.io/collector/config/configtls" oldotlpmetricgrpc "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc" - "google.golang.org/grpc/credentials" - - "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" "go.opentelemetry.io/otel/sdk/resource" + "google.golang.org/grpc/credentials" ) type PipelineConfig struct { @@ -68,28 +67,30 @@ type PipelineConfig struct { type PipelineSetupFunc func(PipelineConfig) (func() error, error) -func (p PipelineConfig) secureMetricOption() (otelcol.Option, oldotlpmetricgrpc.Option) { +func (p PipelineConfig) secureMetricOption() (otelcolmetric.Option, oldotlpmetricgrpc.Option) { if p.Insecure { - return otelcol.WithInsecure(), oldotlpmetricgrpc.WithInsecure() + return otelcolmetric.WithInsecure(), oldotlpmetricgrpc.WithInsecure() } else if p.Credentials != nil { return nil, oldotlpmetricgrpc.WithTLSCredentials(p.Credentials) } else if p.TLSSetting != nil { - return otelcol.WithTLSSetting(*p.TLSSetting), nil + return otelcolmetric.WithTLSSetting(*p.TLSSetting), nil } - return otelcol.WithTLSSetting(configtls.TLSClientSetting{ + return otelcolmetric.WithTLSSetting(configtls.TLSClientSetting{ Insecure: false, }), oldotlpmetricgrpc.WithTLSCredentials( credentials.NewClientTLSFromCert(nil, ""), ) } -func (p PipelineConfig) secureTraceOption() otlptracegrpc.Option { +func (p PipelineConfig) secureTraceOption() otelcoltrace.Option { if p.Insecure { - return otlptracegrpc.WithInsecure() + return otelcoltrace.WithInsecure() } else if p.Credentials != nil { - return otlptracegrpc.WithTLSCredentials(p.Credentials) + return nil + } else if p.TLSSetting != nil { + return otelcoltrace.WithTLSSetting(*p.TLSSetting) } - return otlptracegrpc.WithTLSCredentials( - credentials.NewClientTLSFromCert(nil, ""), - ) + return otelcoltrace.WithTLSSetting(configtls.TLSClientSetting{ + Insecure: false, + }) } diff --git a/pipelines/go.mod b/pipelines/go.mod index e76f7606..24531f95 100644 --- a/pipelines/go.mod +++ b/pipelines/go.mod @@ -5,7 +5,11 @@ go 1.21 require ( github.com/lightstep/otel-launcher-go/lightstep/instrumentation v1.23.0 github.com/lightstep/otel-launcher-go/lightstep/sdk/metric v1.23.0 + github.com/lightstep/otel-launcher-go/lightstep/sdk/trace v1.23.0 + + // For otelcol-based exporter configuration. go.opentelemetry.io/collector/config/configtls v0.90.0 + // Host and runtime instrumentation go.opentelemetry.io/contrib/instrumentation/host v0.45.0 go.opentelemetry.io/contrib/instrumentation/runtime v0.45.0 @@ -15,10 +19,6 @@ require ( go.opentelemetry.io/contrib/propagators/ot v1.20.0 go.opentelemetry.io/otel v1.21.0 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.44.0 - - // Standard trace SDK and gRPC OTLP exporter - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0 go.opentelemetry.io/otel/metric v1.21.0 go.opentelemetry.io/otel/sdk v1.21.0 go.opentelemetry.io/otel/sdk/metric v1.21.0 @@ -48,11 +48,13 @@ require ( github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect github.com/google/flatbuffers v2.0.8+incompatible // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect + github.com/hashicorp/go-version v1.6.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/asmfmt v1.3.2 // indirect github.com/klauspost/compress v1.17.3 // indirect github.com/klauspost/cpuid/v2 v2.0.9 // indirect github.com/knadh/koanf/maps v0.1.1 // indirect + github.com/knadh/koanf/providers/confmap v0.1.0 // indirect github.com/knadh/koanf/v2 v2.0.1 // indirect github.com/lightstep/go-expohisto v1.0.0 // indirect github.com/lightstep/otel-launcher-go/lightstep/sdk/internal v1.23.0 // indirect @@ -67,6 +69,8 @@ require ( github.com/mostynb/go-grpc-compression v1.2.2 // indirect github.com/open-telemetry/otel-arrow v0.13.0 // indirect github.com/open-telemetry/otel-arrow/collector v0.13.0 // indirect + github.com/open-telemetry/otel-arrow/collector/exporter/otelarrowexporter v0.13.0 // indirect + github.com/open-telemetry/otel-arrow/collector/processor/concurrentbatchprocessor v0.13.0 // indirect github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect @@ -104,6 +108,7 @@ require ( go.uber.org/zap v1.26.0 // indirect golang.org/x/mod v0.11.0 // indirect golang.org/x/net v0.18.0 // indirect + golang.org/x/sync v0.4.0 // indirect golang.org/x/sys v0.14.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/tools v0.10.0 // indirect @@ -114,18 +119,12 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect ) -require ( - github.com/hashicorp/go-version v1.6.0 // indirect - github.com/knadh/koanf/providers/confmap v0.1.0 // indirect - github.com/open-telemetry/otel-arrow/collector/exporter/otelarrowexporter v0.13.0 // indirect - github.com/open-telemetry/otel-arrow/collector/processor/concurrentbatchprocessor v0.13.0 // indirect - golang.org/x/sync v0.4.0 // indirect -) - replace github.com/lightstep/otel-launcher-go/lightstep/sdk/internal => ../lightstep/sdk/internal replace github.com/lightstep/otel-launcher-go/lightstep/sdk/metric => ../lightstep/sdk/metric +replace github.com/lightstep/otel-launcher-go/lightstep/sdk/trace => ../lightstep/sdk/trace + replace github.com/lightstep/otel-launcher-go/lightstep/instrumentation => ../lightstep/instrumentation // ambiguous import: found package cloud.google.com/go/compute/metadata in multiple modules diff --git a/pipelines/go.sum b/pipelines/go.sum index 87c521fc..7d950d00 100644 --- a/pipelines/go.sum +++ b/pipelines/go.sum @@ -958,10 +958,6 @@ go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc= go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.44.0 h1:jd0+5t/YynESZqsSyPz+7PAFdEop0dlN0+PkyHYo8oI= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.44.0/go.mod h1:U707O40ee1FpQGyhvqnzmCJm1Wh6OX6GGBVn0E6Uyyk= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0 h1:cl5P5/GIfFh4t6xyruOgJP5QiA1pw4fYYdv6nc6CBWw= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0/go.mod h1:zgBdWWAu7oEEMC06MMKc5NLbA/1YDXV1sMpSqEeLQLg= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0 h1:tIqheXEFWAZ7O8A7m+J0aPTmpJN3YQ7qetUAdkkkKpk= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0/go.mod h1:nUeKExfxAQVbiVFn32YXpXZZHZ61Cc3s3Rn1pDBGAb0= go.opentelemetry.io/otel/exporters/prometheus v0.44.0 h1:08qeJgaPC0YEBu2PQMbqU3rogTlyzpjhCI2b58Yn00w= go.opentelemetry.io/otel/exporters/prometheus v0.44.0/go.mod h1:ERL2uIeBtg4TxZdojHUwzZfIFlUIjZtxubT5p4h1Gjg= go.opentelemetry.io/otel/metric v1.21.0 h1:tlYWfeo+Bocx5kLEloTjbcDwBuELRrIFxwdQ36PlJu4= diff --git a/pipelines/trace.go b/pipelines/trace.go index 46a7f28d..694aebb0 100644 --- a/pipelines/trace.go +++ b/pipelines/trace.go @@ -18,22 +18,25 @@ import ( "context" "fmt" + "github.com/lightstep/otel-launcher-go/lightstep/sdk/trace/exporters/otlp/otelcol" "go.opentelemetry.io/contrib/propagators/b3" "go.opentelemetry.io/contrib/propagators/ot" "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/exporters/otlp/otlptrace" - "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" "go.opentelemetry.io/otel/propagation" "go.opentelemetry.io/otel/sdk/trace" "google.golang.org/grpc/encoding/gzip" ) func NewTracePipeline(c PipelineConfig) (func() error, error) { - spanExporter, err := c.newTraceExporter() + spanExporter, err := c.newTraceExporter(c.secureTraceOption()) if err != nil { return nil, fmt.Errorf("failed to create span exporter: %v", err) } + // Note: this processor does not metric the spans it drops. + // TODO: either improve the spec, so the otel-go SDK BSP will do + // this, or else add the lightstep-internal BSP w/ export metrics + // to this repo. bsp := trace.NewBatchSpanProcessor(spanExporter) tp := trace.NewTracerProvider( trace.WithSampler(trace.AlwaysSample()), @@ -48,19 +51,18 @@ func NewTracePipeline(c PipelineConfig) (func() error, error) { otel.SetTracerProvider(tp) return func() error { - _ = bsp.Shutdown(context.Background()) - return spanExporter.Shutdown(context.Background()) + return bsp.Shutdown(context.Background()) }, nil } -func (c PipelineConfig) newTraceExporter() (*otlptrace.Exporter, error) { - return otlptrace.New( +func (c PipelineConfig) newTraceExporter(secure otelcol.Option) (trace.SpanExporter, error) { + return otelcol.NewExporter( context.Background(), - otlptracegrpc.NewClient( - c.secureTraceOption(), - otlptracegrpc.WithEndpoint(c.Endpoint), - otlptracegrpc.WithHeaders(c.Headers), - otlptracegrpc.WithCompressor(gzip.Name), + otelcol.NewConfig( + secure, + otelcol.WithEndpoint(c.Endpoint), + otelcol.WithHeaders(c.Headers), + otelcol.WithCompressor(gzip.Name), ), ) } From 6f5e91e0b28da500e325fe46cddd8f84aa98fe90 Mon Sep 17 00:00:00 2001 From: Joshua MacDonald Date: Thu, 4 Jan 2024 13:51:13 -0800 Subject: [PATCH 3/4] tidy --- go.mod | 3 +-- go.sum | 4 ---- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/go.mod b/go.mod index b2c8272b..d5502db3 100644 --- a/go.mod +++ b/go.mod @@ -45,6 +45,7 @@ require ( github.com/lightstep/go-expohisto v1.0.0 // indirect github.com/lightstep/otel-launcher-go/lightstep/instrumentation v1.23.0 // indirect github.com/lightstep/otel-launcher-go/lightstep/sdk/internal v1.23.0 // indirect + github.com/lightstep/otel-launcher-go/lightstep/sdk/trace v1.23.0 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect @@ -94,8 +95,6 @@ require ( go.opentelemetry.io/contrib/propagators/b3 v1.21.1 // indirect go.opentelemetry.io/contrib/propagators/ot v1.20.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.44.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0 // indirect go.opentelemetry.io/otel/sdk/metric v1.21.0 // indirect go.opentelemetry.io/proto/otlp v1.0.0 // indirect go.uber.org/multierr v1.11.0 // indirect diff --git a/go.sum b/go.sum index d474ad8a..60d64725 100644 --- a/go.sum +++ b/go.sum @@ -960,10 +960,6 @@ go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc= go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.44.0 h1:jd0+5t/YynESZqsSyPz+7PAFdEop0dlN0+PkyHYo8oI= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.44.0/go.mod h1:U707O40ee1FpQGyhvqnzmCJm1Wh6OX6GGBVn0E6Uyyk= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0 h1:cl5P5/GIfFh4t6xyruOgJP5QiA1pw4fYYdv6nc6CBWw= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0/go.mod h1:zgBdWWAu7oEEMC06MMKc5NLbA/1YDXV1sMpSqEeLQLg= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0 h1:tIqheXEFWAZ7O8A7m+J0aPTmpJN3YQ7qetUAdkkkKpk= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0/go.mod h1:nUeKExfxAQVbiVFn32YXpXZZHZ61Cc3s3Rn1pDBGAb0= go.opentelemetry.io/otel/exporters/prometheus v0.44.0 h1:08qeJgaPC0YEBu2PQMbqU3rogTlyzpjhCI2b58Yn00w= go.opentelemetry.io/otel/exporters/prometheus v0.44.0/go.mod h1:ERL2uIeBtg4TxZdojHUwzZfIFlUIjZtxubT5p4h1Gjg= go.opentelemetry.io/otel/metric v1.21.0 h1:tlYWfeo+Bocx5kLEloTjbcDwBuELRrIFxwdQ36PlJu4= From 9269600ecd269442ec9799ee1aa305a72d9d5d65 Mon Sep 17 00:00:00 2001 From: Joshua MacDonald Date: Thu, 4 Jan 2024 14:29:15 -0800 Subject: [PATCH 4/4] fix test --- pipelines/trace_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pipelines/trace_test.go b/pipelines/trace_test.go index 52a1d56a..0ef810da 100644 --- a/pipelines/trace_test.go +++ b/pipelines/trace_test.go @@ -21,7 +21,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "google.golang.org/grpc/credentials" "google.golang.org/protobuf/encoding/prototext" "github.com/lightstep/otel-launcher-go/pipelines/test" @@ -82,7 +81,7 @@ func TestSecureTrace(t *testing.T) { attribute.String("test-r1", "test-v1"), ), ReportingPeriod: "24h", - Credentials: credentials.NewTLS(newTLSConfig()), + TLSSetting: newTLSClientSetting(), Propagators: []string{"tracecontext", "baggage"}, }) assert.NoError(t, err)