From c0f2bdeefa61e076a71347c372f894d0ca7eee5d Mon Sep 17 00:00:00 2001 From: Carrie Edwards Date: Tue, 18 Jun 2024 08:05:46 -0700 Subject: [PATCH 1/5] Add initial support for metrics in Datadog receiver --- .chloggen/datadogreceiver_metrics.yaml | 27 ++++ receiver/datadogreceiver/README.md | 66 ++++++-- receiver/datadogreceiver/factory.go | 28 +++- receiver/datadogreceiver/factory_test.go | 14 +- .../generated_component_test.go | 19 +-- receiver/datadogreceiver/go.mod | 14 +- receiver/datadogreceiver/httpmetrics.go | 46 ++++++ .../internal/metadata/generated_status.go | 3 +- receiver/datadogreceiver/metadata.yaml | 4 + .../datadogreceiver/metrics_translator.go | 24 +++ receiver/datadogreceiver/receiver.go | 146 +++++++++++++++--- receiver/datadogreceiver/receiver_test.go | 20 ++- .../{translator.go => traces_translator.go} | 2 +- ...ator_test.go => traces_translator_test.go} | 4 +- testbed/go.mod | 2 + testbed/go.sum | 2 + 16 files changed, 365 insertions(+), 56 deletions(-) create mode 100644 .chloggen/datadogreceiver_metrics.yaml create mode 100644 receiver/datadogreceiver/httpmetrics.go create mode 100644 receiver/datadogreceiver/metrics_translator.go rename receiver/datadogreceiver/{translator.go => traces_translator.go} (99%) rename receiver/datadogreceiver/{translator_test.go => traces_translator_test.go} (98%) diff --git a/.chloggen/datadogreceiver_metrics.yaml b/.chloggen/datadogreceiver_metrics.yaml new file mode 100644 index 000000000000..39ae7be25fd4 --- /dev/null +++ b/.chloggen/datadogreceiver_metrics.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: datadogreceiver + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Add support for metrics in Datadog receiver + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [18278] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [user] \ No newline at end of file diff --git a/receiver/datadogreceiver/README.md b/receiver/datadogreceiver/README.md index bbbb70ba572c..c37c99078597 100644 --- a/receiver/datadogreceiver/README.md +++ b/receiver/datadogreceiver/README.md @@ -1,36 +1,50 @@ -# Datadog APM Receiver +# Datadog Receiver | Status | | | ------------- |-----------| -| Stability | [alpha]: traces | +| Stability | [development]: metrics | +| | [alpha]: traces | | Distributions | [contrib] | | Issues | [![Open issues](https://img.shields.io/github/issues-search/open-telemetry/opentelemetry-collector-contrib?query=is%3Aissue%20is%3Aopen%20label%3Areceiver%2Fdatadog%20&label=open&color=orange&logo=opentelemetry)](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues?q=is%3Aopen+is%3Aissue+label%3Areceiver%2Fdatadog) [![Closed issues](https://img.shields.io/github/issues-search/open-telemetry/opentelemetry-collector-contrib?query=is%3Aissue%20is%3Aclosed%20label%3Areceiver%2Fdatadog%20&label=closed&color=blue&logo=opentelemetry)](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues?q=is%3Aclosed+is%3Aissue+label%3Areceiver%2Fdatadog) | | [Code Owners](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/CONTRIBUTING.md#becoming-a-code-owner) | [@boostchicken](https://www.github.com/boostchicken), [@gouthamve](https://www.github.com/gouthamve), [@jpkrohling](https://www.github.com/jpkrohling), [@MovieStoreGuy](https://www.github.com/MovieStoreGuy) | +[development]: https://github.com/open-telemetry/opentelemetry-collector#development [alpha]: https://github.com/open-telemetry/opentelemetry-collector#alpha [contrib]: https://github.com/open-telemetry/opentelemetry-collector-releases/tree/main/distributions/otelcol-contrib ## Overview -Accepts traces in the Datadog APM format. -### Supported Datadog APIs - -- v0.3 (msgpack and json) -- v0.4 (msgpack and json) -- v0.5 (msgpack custom format) -- v0.6 -- v0.7 + +The Datadog receiver enables translation between Datadog and OpenTelemetry-compatible backends. +It currently has support for Datadog's APM traces and Datadog metrics. + ## Configuration -Example: +Configuration wise is very simple, just need to specify where the Datadog receiver should listen and the read timeout. + +Then, the receiver must be configured in the pipeline where it will be used. + ```yaml receivers: datadog: endpoint: localhost:8126 read_timeout: 60s + +exporters: + debug: + +service: + pipelines: + metrics: + receivers: [datadog] + exporters: [debug] + traces: + receivers: [datadog] + exporters: [debug] ``` + ### read_timeout (Optional) The read timeout of the HTTP Server @@ -45,3 +59,33 @@ https://github.com/open-telemetry/opentelemetry-collector/tree/main/config/confi ### Default Attributes - `dd.span.Resource`: The datadog resource name (as distinct from the span name) + +### Datadog's API support + +**Traces** + +| Datadog API Endpoint | Status | Notes | +|----------------------|---------|------------------------------| +| /v0.3/traces | Alpha | Support for msgpack and json | +| /v0.4/traces | Alpha | Support for msgpack and json | +| /v0.5/traces | Alpha | Msgpack custom format | +| /v0.7/traces | Alpha | | +| /api/v0.2/traces | Alpha | | + +**Metrics** + +| Datadog API Endpoint | Status | Notes | +|-----------------------------|-------------|-------| +| /api/v1/series | Development | | +| /api/v2/series | Development | | +| /api/v1/check_run | Development | | +| /api/v1/sketches | Development | | +| /api/beta/sketches | Development | | +| /api/v1/distribution_points | Development | | +| /intake | Development | | + +### Temporality considerations + +Some backends use a different [timestamp temporality](https://opentelemetry.io/docs/specs/otel/metrics/data-model/#temporality) than Datadog uses. Both delta and cumulative temporalities are allowed in the spec. +In order to store metrics on these backends, the receiver will need to translate the timestamps. +For backends that use cumulative temporality, a [delta to cumulative processor](../../processor/deltatocumulativeprocessor/README.md) can be configured in the pipeline. diff --git a/receiver/datadogreceiver/factory.go b/receiver/datadogreceiver/factory.go index 4e954ade601d..6403be658d3b 100644 --- a/receiver/datadogreceiver/factory.go +++ b/receiver/datadogreceiver/factory.go @@ -21,6 +21,7 @@ func NewFactory() receiver.Factory { return receiver.NewFactory( metadata.Type, createDefaultConfig, + receiver.WithMetrics(createMetricsReceiver, metadata.MetricsStability), receiver.WithTraces(createTracesReceiver, metadata.TracesStability)) } @@ -34,12 +35,33 @@ func createDefaultConfig() component.Config { } } -func createTracesReceiver(_ context.Context, params receiver.Settings, cfg component.Config, consumer consumer.Traces) (r receiver.Traces, err error) { +func createTracesReceiver(_ context.Context, params receiver.Settings, cfg component.Config, consumer consumer.Traces) (receiver.Traces, error) { + var err error rcfg := cfg.(*Config) - r = receivers.GetOrAdd(cfg, func() component.Component { - dd, _ := newDataDogReceiver(rcfg, consumer, params) + r := receivers.GetOrAdd(rcfg, func() (dd component.Component) { + dd, err = newDataDogReceiver(rcfg, params) return dd }) + if err != nil { + return nil, err + } + + r.Unwrap().(*datadogReceiver).nextTracesConsumer = consumer + return r, nil +} + +func createMetricsReceiver(_ context.Context, params receiver.Settings, cfg component.Config, consumer consumer.Metrics) (receiver.Metrics, error) { + var err error + rcfg := cfg.(*Config) + r := receivers.GetOrAdd(cfg, func() (dd component.Component) { + dd, err = newDataDogReceiver(rcfg, params) + return dd + }) + if err != nil { + return nil, err + } + + r.Unwrap().(*datadogReceiver).nextMetricsConsumer = consumer return r, nil } diff --git a/receiver/datadogreceiver/factory_test.go b/receiver/datadogreceiver/factory_test.go index 0f09761962ad..9910d840f781 100644 --- a/receiver/datadogreceiver/factory_test.go +++ b/receiver/datadogreceiver/factory_test.go @@ -12,12 +12,22 @@ import ( "go.opentelemetry.io/collector/receiver/receivertest" ) -func TestCreateReceiver(t *testing.T) { +func TestCreateTracesReceiver(t *testing.T) { factory := NewFactory() cfg := factory.CreateDefaultConfig() cfg.(*Config).Endpoint = "http://localhost:0" tReceiver, err := factory.CreateTracesReceiver(context.Background(), receivertest.NewNopSettings(), cfg, consumertest.NewNop()) assert.NoError(t, err) - assert.NotNil(t, tReceiver, "receiver creation failed") + assert.NotNil(t, tReceiver, "traces receiver creation failed") +} + +func TestCreateMetricsReceiver(t *testing.T) { + factory := NewFactory() + cfg := factory.CreateDefaultConfig() + cfg.(*Config).Endpoint = "http://localhost:0" + + tReceiver, err := factory.CreateMetricsReceiver(context.Background(), receivertest.NewNopSettings(), cfg, consumertest.NewNop()) + assert.NoError(t, err) + assert.NotNil(t, tReceiver, "metrics receiver creation failed") } diff --git a/receiver/datadogreceiver/generated_component_test.go b/receiver/datadogreceiver/generated_component_test.go index dff4f1f72cec..5f853383057b 100644 --- a/receiver/datadogreceiver/generated_component_test.go +++ b/receiver/datadogreceiver/generated_component_test.go @@ -31,6 +31,13 @@ func TestComponentLifecycle(t *testing.T) { createFn func(ctx context.Context, set receiver.Settings, cfg component.Config) (component.Component, error) }{ + { + name: "metrics", + createFn: func(ctx context.Context, set receiver.Settings, cfg component.Config) (component.Component, error) { + return factory.CreateMetricsReceiver(ctx, set, cfg, consumertest.NewNop()) + }, + }, + { name: "traces", createFn: func(ctx context.Context, set receiver.Settings, cfg component.Config) (component.Component, error) { @@ -53,17 +60,5 @@ func TestComponentLifecycle(t *testing.T) { err = c.Shutdown(context.Background()) require.NoError(t, err) }) - t.Run(test.name+"-lifecycle", func(t *testing.T) { - firstRcvr, err := test.createFn(context.Background(), receivertest.NewNopSettings(), cfg) - require.NoError(t, err) - host := componenttest.NewNopHost() - require.NoError(t, err) - require.NoError(t, firstRcvr.Start(context.Background(), host)) - require.NoError(t, firstRcvr.Shutdown(context.Background())) - secondRcvr, err := test.createFn(context.Background(), receivertest.NewNopSettings(), cfg) - require.NoError(t, err) - require.NoError(t, secondRcvr.Start(context.Background(), host)) - require.NoError(t, secondRcvr.Shutdown(context.Background())) - }) } } diff --git a/receiver/datadogreceiver/go.mod b/receiver/datadogreceiver/go.mod index e5c3c01c3379..3c1e4347613b 100644 --- a/receiver/datadogreceiver/go.mod +++ b/receiver/datadogreceiver/go.mod @@ -4,6 +4,7 @@ go 1.21.0 require ( github.com/DataDog/datadog-agent/pkg/proto v0.56.0-devel.0.20240621152414-10454a30138d + github.com/open-telemetry/opentelemetry-collector-contrib/internal/exp/metrics v0.104.0 github.com/open-telemetry/opentelemetry-collector-contrib/internal/sharedcomponent v0.104.0 github.com/stretchr/testify v1.9.0 github.com/vmihailenco/msgpack/v4 v4.3.13 @@ -14,9 +15,11 @@ require ( go.opentelemetry.io/collector/pdata v1.11.0 go.opentelemetry.io/collector/receiver v0.104.0 go.opentelemetry.io/collector/semconv v0.104.0 + go.opentelemetry.io/otel v1.27.0 go.opentelemetry.io/otel/metric v1.27.0 go.opentelemetry.io/otel/trace v1.27.0 go.uber.org/goleak v1.3.0 + go.uber.org/zap v1.27.0 google.golang.org/protobuf v1.34.2 ) @@ -43,6 +46,7 @@ require ( github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.103.0 // indirect github.com/philhofer/fwd v1.1.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_golang v1.19.1 // indirect @@ -63,12 +67,10 @@ require ( go.opentelemetry.io/collector/extension/auth v0.104.0 // indirect go.opentelemetry.io/collector/featuregate v1.11.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0 // indirect - go.opentelemetry.io/otel v1.27.0 // indirect go.opentelemetry.io/otel/exporters/prometheus v0.49.0 // indirect go.opentelemetry.io/otel/sdk v1.27.0 // indirect go.opentelemetry.io/otel/sdk/metric v1.27.0 // indirect go.uber.org/multierr v1.11.0 // indirect - go.uber.org/zap v1.27.0 // indirect golang.org/x/net v0.26.0 // indirect golang.org/x/sys v0.21.0 // indirect golang.org/x/text v0.16.0 // indirect @@ -84,3 +86,11 @@ retract ( v0.76.2 v0.76.1 ) + +replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/exp/metrics => ../../internal/exp/metrics + +replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest => ../../pkg/pdatatest + +replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden => ../../pkg/golden + +replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil => ../../pkg/pdatautil diff --git a/receiver/datadogreceiver/httpmetrics.go b/receiver/datadogreceiver/httpmetrics.go new file mode 100644 index 000000000000..a3a67c0f7fae --- /dev/null +++ b/receiver/datadogreceiver/httpmetrics.go @@ -0,0 +1,46 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package datadogreceiver // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/datadogreceiver" + +import ( + "context" + "net/http" + "time" + + "go.opentelemetry.io/otel/metric" + semconv "go.opentelemetry.io/otel/semconv/v1.25.0" +) + +func httpMetrics(meter metric.Meter, next http.Handler) http.Handler { + hist, err := meter.Int64Histogram("datadog.http.server.duration", + metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(0.005, 0.01, 0.025, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75, 1, 2.5, 5, 7.5, 10), + ) + if err != nil { + panic(err) + } + + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + start := time.Now() + + rw := recrw{ResponseWriter: w, Code: http.StatusOK} + next.ServeHTTP(&rw, r) + + hist.Record(context.Background(), int64(time.Since(start)), metric.WithAttributes( + semconv.HTTPMethod(r.Method), + semconv.HTTPRoute(r.URL.Path), + semconv.HTTPStatusCode(rw.Code), + )) + }) +} + +type recrw struct { + Code int + http.ResponseWriter +} + +func (rw *recrw) WriteHeader(code int) { + rw.Code = code + rw.ResponseWriter.WriteHeader(code) +} diff --git a/receiver/datadogreceiver/internal/metadata/generated_status.go b/receiver/datadogreceiver/internal/metadata/generated_status.go index 861204a31a25..603d91c58fbe 100644 --- a/receiver/datadogreceiver/internal/metadata/generated_status.go +++ b/receiver/datadogreceiver/internal/metadata/generated_status.go @@ -11,5 +11,6 @@ var ( ) const ( - TracesStability = component.StabilityLevelAlpha + MetricsStability = component.StabilityLevelDevelopment + TracesStability = component.StabilityLevelAlpha ) diff --git a/receiver/datadogreceiver/metadata.yaml b/receiver/datadogreceiver/metadata.yaml index bd323d86f4b5..9d310afe79c5 100644 --- a/receiver/datadogreceiver/metadata.yaml +++ b/receiver/datadogreceiver/metadata.yaml @@ -5,6 +5,10 @@ status: class: receiver stability: alpha: [traces] + development: [metrics] distributions: [contrib] codeowners: active: [boostchicken, gouthamve, jpkrohling, MovieStoreGuy] + +tests: + skip_lifecycle: true # Skip lifecycle tests since there are multiple receivers that run on the same port \ No newline at end of file diff --git a/receiver/datadogreceiver/metrics_translator.go b/receiver/datadogreceiver/metrics_translator.go new file mode 100644 index 000000000000..25bcda2a0896 --- /dev/null +++ b/receiver/datadogreceiver/metrics_translator.go @@ -0,0 +1,24 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package datadogreceiver // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/datadogreceiver" +import ( + "sync" + + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/pdata/pcommon" + + "github.com/open-telemetry/opentelemetry-collector-contrib/internal/exp/metrics/identity" +) + +type MetricsTranslator struct { + sync.RWMutex + buildInfo component.BuildInfo + lastTs map[identity.Stream]pcommon.Timestamp +} + +func newMetricsTranslator() *MetricsTranslator { + return &MetricsTranslator{ + lastTs: make(map[identity.Stream]pcommon.Timestamp), + } +} diff --git a/receiver/datadogreceiver/receiver.go b/receiver/datadogreceiver/receiver.go index 4a8b98056cce..7825185b0031 100644 --- a/receiver/datadogreceiver/receiver.go +++ b/receiver/datadogreceiver/receiver.go @@ -14,28 +14,32 @@ import ( "go.opentelemetry.io/collector/consumer" "go.opentelemetry.io/collector/receiver" "go.opentelemetry.io/collector/receiver/receiverhelper" + "go.uber.org/zap" ) type datadogReceiver struct { - address string - config *Config - params receiver.Settings - nextConsumer consumer.Traces - server *http.Server - tReceiver *receiverhelper.ObsReport -} + address string + config *Config + params receiver.Settings + + nextTracesConsumer consumer.Traces + nextMetricsConsumer consumer.Metrics -func newDataDogReceiver(config *Config, nextConsumer consumer.Traces, params receiver.Settings) (receiver.Traces, error) { + metricsTranslator *MetricsTranslator + + server *http.Server + tReceiver *receiverhelper.ObsReport +} +func newDataDogReceiver(config *Config, params receiver.Settings) (component.Component, error) { instance, err := receiverhelper.NewObsReport(receiverhelper.ObsReportSettings{LongLivedCtx: false, ReceiverID: params.ID, Transport: "http", ReceiverCreateSettings: params}) if err != nil { return nil, err } return &datadogReceiver{ - params: params, - config: config, - nextConsumer: nextConsumer, + params: params, + config: config, server: &http.Server{ ReadTimeout: config.ReadTimeout, }, @@ -45,18 +49,38 @@ func newDataDogReceiver(config *Config, nextConsumer consumer.Traces, params rec func (ddr *datadogReceiver) Start(ctx context.Context, host component.Host) error { ddmux := http.NewServeMux() - ddmux.HandleFunc("/v0.3/traces", ddr.handleTraces) - ddmux.HandleFunc("/v0.4/traces", ddr.handleTraces) - ddmux.HandleFunc("/v0.5/traces", ddr.handleTraces) - ddmux.HandleFunc("/v0.7/traces", ddr.handleTraces) - ddmux.HandleFunc("/api/v0.2/traces", ddr.handleTraces) + + ddmux.HandleFunc("/", func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + }) + + if ddr.nextTracesConsumer != nil { + ddmux.HandleFunc("/v0.3/traces", ddr.handleTraces) + ddmux.HandleFunc("/v0.4/traces", ddr.handleTraces) + ddmux.HandleFunc("/v0.5/traces", ddr.handleTraces) + ddmux.HandleFunc("/v0.7/traces", ddr.handleTraces) + ddmux.HandleFunc("/api/v0.2/traces", ddr.handleTraces) + } + + if ddr.nextMetricsConsumer != nil { + ddr.metricsTranslator = newMetricsTranslator() + ddr.metricsTranslator.buildInfo = ddr.params.BuildInfo + + ddmux.HandleFunc("/api/v1/series", ddr.handleV1Series) + ddmux.HandleFunc("/api/v2/series", ddr.handleV2Series) + ddmux.HandleFunc("/api/v1/check_run", ddr.handleCheckRun) + ddmux.HandleFunc("/api/v1/sketches", ddr.handleSketches) + ddmux.HandleFunc("/api/beta/sketches", ddr.handleSketches) + ddmux.HandleFunc("/intake", ddr.handleIntake) + ddmux.HandleFunc("/api/v1/distribution_points", ddr.handleDistributionPoints) + } var err error ddr.server, err = ddr.config.ServerConfig.ToServer( ctx, host, ddr.params.TelemetrySettings, - ddmux, + httpMetrics(ddr.params.MeterProvider.Meter("datadogreceiver"), ddmux), ) if err != nil { return fmt.Errorf("failed to create server definition: %w", err) @@ -89,7 +113,7 @@ func (ddr *datadogReceiver) handleTraces(w http.ResponseWriter, req *http.Reques }(&spanCount) var ddTraces []*pb.TracerPayload - ddTraces, err = handlePayload(req) + ddTraces, err = handleTracesPayload(req) if err != nil { http.Error(w, "Unable to unmarshal reqs", http.StatusBadRequest) ddr.params.Logger.Error("Unable to unmarshal reqs") @@ -98,7 +122,7 @@ func (ddr *datadogReceiver) handleTraces(w http.ResponseWriter, req *http.Reques for _, ddTrace := range ddTraces { otelTraces := toTraces(ddTrace, req) spanCount = otelTraces.SpanCount() - err = ddr.nextConsumer.ConsumeTraces(obsCtx, otelTraces) + err = ddr.nextTracesConsumer.ConsumeTraces(obsCtx, otelTraces) if err != nil { http.Error(w, "Trace consumer errored out", http.StatusInternalServerError) ddr.params.Logger.Error("Trace consumer errored out") @@ -109,3 +133,87 @@ func (ddr *datadogReceiver) handleTraces(w http.ResponseWriter, req *http.Reques _, _ = w.Write([]byte("OK")) } + +// handleV1Series handles the v1 series endpoint https://docs.datadoghq.com/api/latest/metrics/#submit-metrics +func (ddr *datadogReceiver) handleV1Series(w http.ResponseWriter, req *http.Request) { + obsCtx := ddr.tReceiver.StartMetricsOp(req.Context()) + var err error + var metricsCount int + defer func(metricsCount *int) { + ddr.tReceiver.EndMetricsOp(obsCtx, "datadog", *metricsCount, err) + }(&metricsCount) + + err = fmt.Errorf("series v1 endpoint not implemented") + http.Error(w, err.Error(), http.StatusMethodNotAllowed) + ddr.params.Logger.Warn("metrics consumer errored out", zap.Error(err)) +} + +// handleV2Series handles the v2 series endpoint https://docs.datadoghq.com/api/latest/metrics/#submit-metrics +func (ddr *datadogReceiver) handleV2Series(w http.ResponseWriter, req *http.Request) { + obsCtx := ddr.tReceiver.StartMetricsOp(req.Context()) + var err error + var metricsCount int + defer func(metricsCount *int) { + ddr.tReceiver.EndMetricsOp(obsCtx, "datadog", *metricsCount, err) + }(&metricsCount) + + err = fmt.Errorf("series v2 endpoint not implemented") + http.Error(w, err.Error(), http.StatusMethodNotAllowed) + ddr.params.Logger.Warn("metrics consumer errored out", zap.Error(err)) +} + +// handleCheckRun handles the service checks endpoint https://docs.datadoghq.com/api/latest/service-checks/ +func (ddr *datadogReceiver) handleCheckRun(w http.ResponseWriter, req *http.Request) { + obsCtx := ddr.tReceiver.StartMetricsOp(req.Context()) + var err error + var metricsCount int + defer func(metricsCount *int) { + ddr.tReceiver.EndMetricsOp(obsCtx, "datadog", *metricsCount, err) + }(&metricsCount) + + err = fmt.Errorf("service checks endpoint not implemented") + http.Error(w, err.Error(), http.StatusMethodNotAllowed) + ddr.params.Logger.Warn("metrics consumer errored out", zap.Error(err)) +} + +// handleSketches handles sketches, the underlying data structure of distributions https://docs.datadoghq.com/metrics/distributions/ +func (ddr *datadogReceiver) handleSketches(w http.ResponseWriter, req *http.Request) { + obsCtx := ddr.tReceiver.StartMetricsOp(req.Context()) + var err error + var metricsCount int + defer func(metricsCount *int) { + ddr.tReceiver.EndMetricsOp(obsCtx, "datadog", *metricsCount, err) + }(&metricsCount) + + err = fmt.Errorf("sketches endpoint not implemented") + http.Error(w, err.Error(), http.StatusMethodNotAllowed) + ddr.params.Logger.Warn("metrics consumer errored out", zap.Error(err)) +} + +// handleIntake handles operational calls made by the agent to submit host tags and other metadata to the backend. +func (ddr *datadogReceiver) handleIntake(w http.ResponseWriter, req *http.Request) { + obsCtx := ddr.tReceiver.StartMetricsOp(req.Context()) + var err error + var metricsCount int + defer func(metricsCount *int) { + ddr.tReceiver.EndMetricsOp(obsCtx, "datadog", *metricsCount, err) + }(&metricsCount) + + err = fmt.Errorf("intake endpoint not implemented") + http.Error(w, err.Error(), http.StatusMethodNotAllowed) + ddr.params.Logger.Warn("metrics consumer errored out", zap.Error(err)) +} + +// handleDistributionPoints handles the distribution points endpoint https://docs.datadoghq.com/api/latest/metrics/#submit-distribution-points +func (ddr *datadogReceiver) handleDistributionPoints(w http.ResponseWriter, req *http.Request) { + obsCtx := ddr.tReceiver.StartMetricsOp(req.Context()) + var err error + var metricsCount int + defer func(metricsCount *int) { + ddr.tReceiver.EndMetricsOp(obsCtx, "datadog", *metricsCount, err) + }(&metricsCount) + + err = fmt.Errorf("distribution points endpoint not implemented") + http.Error(w, err.Error(), http.StatusMethodNotAllowed) + ddr.params.Logger.Warn("metrics consumer errored out", zap.Error(err)) +} diff --git a/receiver/datadogreceiver/receiver_test.go b/receiver/datadogreceiver/receiver_test.go index 5f6babcfd69c..74518c0759f5 100644 --- a/receiver/datadogreceiver/receiver_test.go +++ b/receiver/datadogreceiver/receiver_test.go @@ -19,13 +19,27 @@ import ( "go.opentelemetry.io/collector/receiver/receivertest" ) -func TestDatadogReceiver_Lifecycle(t *testing.T) { +func TestDatadogTracesReceiver_Lifecycle(t *testing.T) { factory := NewFactory() cfg := factory.CreateDefaultConfig() cfg.(*Config).Endpoint = "localhost:0" ddr, err := factory.CreateTracesReceiver(context.Background(), receivertest.NewNopSettings(), cfg, consumertest.NewNop()) - assert.NoError(t, err, "Receiver should be created") + assert.NoError(t, err, "Traces receiver should be created") + + err = ddr.Start(context.Background(), componenttest.NewNopHost()) + assert.NoError(t, err, "Server should start") + + err = ddr.Shutdown(context.Background()) + assert.NoError(t, err, "Server should stop") +} + +func TestDatadogMetricsReceiver_Lifecycle(t *testing.T) { + factory := NewFactory() + cfg := factory.CreateDefaultConfig() + cfg.(*Config).Endpoint = "localhost:0" + ddr, err := factory.CreateMetricsReceiver(context.Background(), receivertest.NewNopCreateSettings(), cfg, consumertest.NewNop()) + assert.NoError(t, err, "Metrics receiver should be created") err = ddr.Start(context.Background(), componenttest.NewNopHost()) assert.NoError(t, err, "Server should start") @@ -39,9 +53,9 @@ func TestDatadogServer(t *testing.T) { cfg.Endpoint = "localhost:0" // Using a randomly assigned address dd, err := newDataDogReceiver( cfg, - consumertest.NewNop(), receivertest.NewNopSettings(), ) + dd.(*datadogReceiver).nextTracesConsumer = consumertest.NewNop() require.NoError(t, err, "Must not error when creating receiver") ctx, cancel := context.WithCancel(context.Background()) diff --git a/receiver/datadogreceiver/translator.go b/receiver/datadogreceiver/traces_translator.go similarity index 99% rename from receiver/datadogreceiver/translator.go rename to receiver/datadogreceiver/traces_translator.go index 3c7ac3fae026..0e167a92a578 100644 --- a/receiver/datadogreceiver/translator.go +++ b/receiver/datadogreceiver/traces_translator.go @@ -197,7 +197,7 @@ func putBuffer(buffer *bytes.Buffer) { bufferPool.Put(buffer) } -func handlePayload(req *http.Request) (tp []*pb.TracerPayload, err error) { +func handleTracesPayload(req *http.Request) (tp []*pb.TracerPayload, err error) { var tracerPayloads []*pb.TracerPayload defer func() { diff --git a/receiver/datadogreceiver/translator_test.go b/receiver/datadogreceiver/traces_translator_test.go similarity index 98% rename from receiver/datadogreceiver/translator_test.go rename to receiver/datadogreceiver/traces_translator_test.go index c65f876d30d7..eb8c0dc90165 100644 --- a/receiver/datadogreceiver/translator_test.go +++ b/receiver/datadogreceiver/traces_translator_test.go @@ -115,7 +115,7 @@ func TestTracePayloadV07Unmarshalling(t *testing.T) { bytez, _ := apiPayload.MarshalMsg(reqBytes) req, _ := http.NewRequest(http.MethodPost, "/v0.7/traces", io.NopCloser(bytes.NewReader(bytez))) - translatedPayloads, _ := handlePayload(req) + translatedPayloads, _ := handleTracesPayload(req) assert.Equal(t, len(translatedPayloads), 1, "Expected one translated payload") translated := translatedPayloads[0] span := translated.GetChunks()[0].GetSpans()[0] @@ -151,7 +151,7 @@ func TestTracePayloadApiV02Unmarshalling(t *testing.T) { bytez, _ := proto.Marshal(&agentPayload) req, _ := http.NewRequest(http.MethodPost, "/api/v0.2/traces", io.NopCloser(bytes.NewReader(bytez))) - translatedPayloads, _ := handlePayload(req) + translatedPayloads, _ := handleTracesPayload(req) assert.Equal(t, len(translatedPayloads), 2, "Expected two translated payload") for _, translated := range translatedPayloads { assert.NotNil(t, translated) diff --git a/testbed/go.mod b/testbed/go.mod index 0d51eaf2586b..5bc6e1e0df5a 100644 --- a/testbed/go.mod +++ b/testbed/go.mod @@ -374,3 +374,5 @@ replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden => replace github.com/open-telemetry/opentelemetry-collector-contrib/extension/ackextension => ../extension/ackextension replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl => ../pkg/ottl + +replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/exp/metrics => ../internal/exp/metrics diff --git a/testbed/go.sum b/testbed/go.sum index b29faffc9d66..4caa938b16a1 100644 --- a/testbed/go.sum +++ b/testbed/go.sum @@ -502,6 +502,8 @@ github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4 github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o= github.com/onsi/gomega v1.29.0 h1:KIA/t2t5UBzoirT4H9tsML45GEbo3ouUnBHsCfD2tVg= github.com/onsi/gomega v1.29.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= +github.com/open-telemetry/opentelemetry-collector-contrib/internal/exp/metrics v0.103.0 h1:Pt7k4F2PBhxLsXEOmtPa0ruGKWxkz+knOpQhGjFCCj0= +github.com/open-telemetry/opentelemetry-collector-contrib/internal/exp/metrics v0.103.0/go.mod h1:94YwfWflNwjx+d2nW/O7PsYcGC5HiRxcsSIQza5cJp0= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= From 6151c833edd05d16cbf97553caab7f108e5f2183 Mon Sep 17 00:00:00 2001 From: Carrie Edwards Date: Fri, 21 Jun 2024 08:38:21 -0700 Subject: [PATCH 2/5] Fix lint failures --- exporter/elasticsearchexporter/integrationtest/go.mod | 2 ++ receiver/datadogreceiver/go.mod | 2 +- receiver/datadogreceiver/receiver_test.go | 2 +- testbed/go.sum | 2 -- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/exporter/elasticsearchexporter/integrationtest/go.mod b/exporter/elasticsearchexporter/integrationtest/go.mod index fb50376df106..86dd683ecf34 100644 --- a/exporter/elasticsearchexporter/integrationtest/go.mod +++ b/exporter/elasticsearchexporter/integrationtest/go.mod @@ -270,3 +270,5 @@ replace github.com/open-telemetry/opentelemetry-collector-contrib/connector/span replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl => ../../../pkg/ottl replace github.com/open-telemetry/opentelemetry-collector-contrib/connector/routingconnector => ../../../connector/routingconnector + +replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/exp/metrics => ../../../internal/exp/metrics diff --git a/receiver/datadogreceiver/go.mod b/receiver/datadogreceiver/go.mod index 3c1e4347613b..e278a9d3c26f 100644 --- a/receiver/datadogreceiver/go.mod +++ b/receiver/datadogreceiver/go.mod @@ -17,6 +17,7 @@ require ( go.opentelemetry.io/collector/semconv v0.104.0 go.opentelemetry.io/otel v1.27.0 go.opentelemetry.io/otel/metric v1.27.0 + go.opentelemetry.io/otel/sdk/metric v1.27.0 go.opentelemetry.io/otel/trace v1.27.0 go.uber.org/goleak v1.3.0 go.uber.org/zap v1.27.0 @@ -69,7 +70,6 @@ require ( go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0 // indirect go.opentelemetry.io/otel/exporters/prometheus v0.49.0 // indirect go.opentelemetry.io/otel/sdk v1.27.0 // indirect - go.opentelemetry.io/otel/sdk/metric v1.27.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/net v0.26.0 // indirect golang.org/x/sys v0.21.0 // indirect diff --git a/receiver/datadogreceiver/receiver_test.go b/receiver/datadogreceiver/receiver_test.go index 74518c0759f5..445ecad2ef66 100644 --- a/receiver/datadogreceiver/receiver_test.go +++ b/receiver/datadogreceiver/receiver_test.go @@ -38,7 +38,7 @@ func TestDatadogMetricsReceiver_Lifecycle(t *testing.T) { factory := NewFactory() cfg := factory.CreateDefaultConfig() cfg.(*Config).Endpoint = "localhost:0" - ddr, err := factory.CreateMetricsReceiver(context.Background(), receivertest.NewNopCreateSettings(), cfg, consumertest.NewNop()) + ddr, err := factory.CreateMetricsReceiver(context.Background(), receivertest.NewNopSettings(), cfg, consumertest.NewNop()) assert.NoError(t, err, "Metrics receiver should be created") err = ddr.Start(context.Background(), componenttest.NewNopHost()) diff --git a/testbed/go.sum b/testbed/go.sum index 4caa938b16a1..b29faffc9d66 100644 --- a/testbed/go.sum +++ b/testbed/go.sum @@ -502,8 +502,6 @@ github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4 github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o= github.com/onsi/gomega v1.29.0 h1:KIA/t2t5UBzoirT4H9tsML45GEbo3ouUnBHsCfD2tVg= github.com/onsi/gomega v1.29.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= -github.com/open-telemetry/opentelemetry-collector-contrib/internal/exp/metrics v0.103.0 h1:Pt7k4F2PBhxLsXEOmtPa0ruGKWxkz+knOpQhGjFCCj0= -github.com/open-telemetry/opentelemetry-collector-contrib/internal/exp/metrics v0.103.0/go.mod h1:94YwfWflNwjx+d2nW/O7PsYcGC5HiRxcsSIQza5cJp0= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= From e0be7f26470e5f49fa2512b65211f8d85800a7a3 Mon Sep 17 00:00:00 2001 From: Carrie Edwards Date: Fri, 21 Jun 2024 08:40:45 -0700 Subject: [PATCH 3/5] Remove unnecessary telemetry --- receiver/datadogreceiver/httpmetrics.go | 46 ------------------------- receiver/datadogreceiver/receiver.go | 2 +- 2 files changed, 1 insertion(+), 47 deletions(-) delete mode 100644 receiver/datadogreceiver/httpmetrics.go diff --git a/receiver/datadogreceiver/httpmetrics.go b/receiver/datadogreceiver/httpmetrics.go deleted file mode 100644 index a3a67c0f7fae..000000000000 --- a/receiver/datadogreceiver/httpmetrics.go +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright The OpenTelemetry Authors -// SPDX-License-Identifier: Apache-2.0 - -package datadogreceiver // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/datadogreceiver" - -import ( - "context" - "net/http" - "time" - - "go.opentelemetry.io/otel/metric" - semconv "go.opentelemetry.io/otel/semconv/v1.25.0" -) - -func httpMetrics(meter metric.Meter, next http.Handler) http.Handler { - hist, err := meter.Int64Histogram("datadog.http.server.duration", - metric.WithUnit("s"), - metric.WithExplicitBucketBoundaries(0.005, 0.01, 0.025, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75, 1, 2.5, 5, 7.5, 10), - ) - if err != nil { - panic(err) - } - - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - start := time.Now() - - rw := recrw{ResponseWriter: w, Code: http.StatusOK} - next.ServeHTTP(&rw, r) - - hist.Record(context.Background(), int64(time.Since(start)), metric.WithAttributes( - semconv.HTTPMethod(r.Method), - semconv.HTTPRoute(r.URL.Path), - semconv.HTTPStatusCode(rw.Code), - )) - }) -} - -type recrw struct { - Code int - http.ResponseWriter -} - -func (rw *recrw) WriteHeader(code int) { - rw.Code = code - rw.ResponseWriter.WriteHeader(code) -} diff --git a/receiver/datadogreceiver/receiver.go b/receiver/datadogreceiver/receiver.go index 7825185b0031..780dc1a68fbf 100644 --- a/receiver/datadogreceiver/receiver.go +++ b/receiver/datadogreceiver/receiver.go @@ -80,7 +80,7 @@ func (ddr *datadogReceiver) Start(ctx context.Context, host component.Host) erro ctx, host, ddr.params.TelemetrySettings, - httpMetrics(ddr.params.MeterProvider.Meter("datadogreceiver"), ddmux), + ddmux, ) if err != nil { return fmt.Errorf("failed to create server definition: %w", err) From 3e97bd1ba8cacf7f6a8ed58aee419a03f210b3ca Mon Sep 17 00:00:00 2001 From: Carrie Edwards Date: Mon, 24 Jun 2024 07:49:15 -0700 Subject: [PATCH 4/5] Update go.mod --- receiver/datadogreceiver/go.mod | 1 + 1 file changed, 1 insertion(+) diff --git a/receiver/datadogreceiver/go.mod b/receiver/datadogreceiver/go.mod index e278a9d3c26f..acd837d0a05b 100644 --- a/receiver/datadogreceiver/go.mod +++ b/receiver/datadogreceiver/go.mod @@ -68,6 +68,7 @@ require ( go.opentelemetry.io/collector/extension/auth v0.104.0 // indirect go.opentelemetry.io/collector/featuregate v1.11.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0 // indirect + go.opentelemetry.io/otel v1.27.0 // indirect go.opentelemetry.io/otel/exporters/prometheus v0.49.0 // indirect go.opentelemetry.io/otel/sdk v1.27.0 // indirect go.uber.org/multierr v1.11.0 // indirect From fbf713a0522e8742d040b9273f33c589fe74ba44 Mon Sep 17 00:00:00 2001 From: Carrie Edwards Date: Tue, 2 Jul 2024 11:23:47 -0700 Subject: [PATCH 5/5] Update vendoring --- receiver/datadogreceiver/go.mod | 5 ++--- testbed/go.mod | 1 + 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/receiver/datadogreceiver/go.mod b/receiver/datadogreceiver/go.mod index acd837d0a05b..a446778a883f 100644 --- a/receiver/datadogreceiver/go.mod +++ b/receiver/datadogreceiver/go.mod @@ -15,9 +15,7 @@ require ( go.opentelemetry.io/collector/pdata v1.11.0 go.opentelemetry.io/collector/receiver v0.104.0 go.opentelemetry.io/collector/semconv v0.104.0 - go.opentelemetry.io/otel v1.27.0 go.opentelemetry.io/otel/metric v1.27.0 - go.opentelemetry.io/otel/sdk/metric v1.27.0 go.opentelemetry.io/otel/trace v1.27.0 go.uber.org/goleak v1.3.0 go.uber.org/zap v1.27.0 @@ -47,7 +45,7 @@ require ( github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.103.0 // indirect + github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.104.0 // indirect github.com/philhofer/fwd v1.1.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_golang v1.19.1 // indirect @@ -71,6 +69,7 @@ require ( go.opentelemetry.io/otel v1.27.0 // indirect go.opentelemetry.io/otel/exporters/prometheus v0.49.0 // indirect go.opentelemetry.io/otel/sdk v1.27.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.27.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/net v0.26.0 // indirect golang.org/x/sys v0.21.0 // indirect diff --git a/testbed/go.mod b/testbed/go.mod index 5bc6e1e0df5a..20e00ffc546d 100644 --- a/testbed/go.mod +++ b/testbed/go.mod @@ -186,6 +186,7 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect github.com/open-telemetry/opentelemetry-collector-contrib/extension/ackextension v0.104.0 // indirect + github.com/open-telemetry/opentelemetry-collector-contrib/internal/exp/metrics v0.104.0 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/internal/sharedcomponent v0.104.0 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/pkg/batchperresourceattr v0.104.0 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/pkg/experimentalmetricmetadata v0.104.0 // indirect