Skip to content

Commit

Permalink
Embedded Otel Collector into Agent (#3800)
Browse files Browse the repository at this point in the history 
* basic otel with testdata

* updated UT

* updated UT

* do not pass modifiers

* lint: use Fprint instead of println"

* path fixed

* test cases

* forgotten files

* Added file ingestion test

* Added file ingestion test

* e2e with apm

* imports

* add group

* add --config option to align with collector usage

* Update testing/integration/otel_test.go

Co-authored-by: Paolo Chilà <[email protected]>

* Update testing/integration/otel_test.go

Co-authored-by: Paolo Chilà <[email protected]>

* Update pkg/testing/tools/estools/elasticsearch.go

Co-authored-by: Shaunak Kashyap <[email protected]>

* review

* typos

* fixed unit tests

* try guarding against lower version

* try guarding against lower version

* try with windows

* comment for skip

* remove windows

* quick review resolutions

* check suffix for otel configs only

* require yml or yaml config

* e2e test with logger watching

* some test cleanup

* more cleanup

* typo

* name based resolution

* e2e fixed

* unit test fixed

* missing files

---------

Co-authored-by: Paolo Chilà <[email protected]>
Co-authored-by: Shaunak Kashyap <[email protected]>
  • Loading branch information
@michalpristas @pchila @ycombinator
3 people authored Dec 19, 2023
1 parent ad7e1b5 commit 1410d2a
Show file tree
Hide file tree
Showing 20 changed files with 24,192 additions and 4,102 deletions.
26,821 changes: 22,804 additions & 4,017 deletions NOTICE.txt
View file

Large diffs are not rendered by default.

16 changes: 16 additions & 0 deletions dev-tools/packaging/packages.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,10 @@ shared:
source: 'elastic-agent.yml'
mode: 0600
config: true
/etc/{{.BeatName}}/otel.yml:
source: 'otel.yml'
mode: 0600
config: true
/etc/{{.BeatName}}/.elastic-agent.active.commit:
content: >
{{ commit }}
Expand Down Expand Up @@ -109,6 +113,10 @@ shared:
source: 'elastic-agent.yml'
mode: 0600
config: true
/etc/{{.BeatName}}/otel.yml:
source: 'otel.yml'
mode: 0600
config: true
/etc/{{.BeatName}}/.elastic-agent.active.commit:
content: >
{{ commit }}
Expand Down Expand Up @@ -143,6 +151,10 @@ shared:
source: 'elastic-agent.yml'
mode: 0600
config: true
'otel.yml':
source: 'otel.yml'
mode: 0600
config: true
'.elastic-agent.active.commit':
content: >
{{ commit }}
Expand Down Expand Up @@ -231,6 +243,10 @@ shared:
source: 'elastic-agent.docker.yml'
mode: 0600
config: true
'otel.yml':
source: 'otel.yml'
mode: 0600
config: true
'.elastic-agent.active.commit':
content: >
{{ commit }}
Expand Down
153 changes: 123 additions & 30 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,12 @@ module github.com/elastic/elastic-agent
go 1.20

require (
github.com/Microsoft/go-winio v0.6.0
github.com/Microsoft/go-winio v0.6.1
github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230321174746-8dcc6526cfb1
github.com/billgraziano/dpapi v0.4.0
github.com/blakesmith/ar v0.0.0-20150311145944-8bd4349a67f2
github.com/cavaliercoder/go-rpm v0.0.0-20190131055624-7a9c54e3d83e
github.com/cenkalti/backoff/v4 v4.1.3
github.com/cenkalti/backoff/v4 v4.2.1
github.com/docker/go-units v0.5.0
github.com/dolmen-go/contextio v0.0.0-20200217195037-68fc5150bcd5
github.com/elastic/e2e-testing v1.1.0
Expand All @@ -22,12 +22,12 @@ require (
github.com/elastic/go-sysinfo v1.11.2
github.com/elastic/go-ucfg v0.8.6
github.com/fatih/color v1.15.0
github.com/fsnotify/fsnotify v1.6.0
github.com/fsnotify/fsnotify v1.7.0
github.com/gofrs/flock v0.8.1
github.com/gofrs/uuid v4.4.0+incompatible
github.com/google/go-cmp v0.5.9
github.com/google/go-cmp v0.6.0
github.com/google/pprof v0.0.0-20230426061923-93006964c1fc
github.com/google/uuid v1.3.0
github.com/google/uuid v1.4.0
github.com/gorilla/mux v1.8.0
github.com/hashicorp/go-multierror v1.1.1
github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95
Expand All @@ -39,31 +39,48 @@ require (
github.com/magefile/mage v1.15.0
github.com/mitchellh/gox v1.0.1
github.com/mitchellh/hashstructure v1.1.0
github.com/mitchellh/mapstructure v1.5.0
github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4
github.com/oklog/ulid v1.3.1
github.com/open-telemetry/opentelemetry-collector-contrib/exporter/fileexporter v0.89.0
github.com/open-telemetry/opentelemetry-collector-contrib/processor/attributesprocessor v0.89.0
github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourceprocessor v0.89.0
github.com/open-telemetry/opentelemetry-collector-contrib/processor/transformprocessor v0.89.0
github.com/open-telemetry/opentelemetry-collector-contrib/receiver/filelogreceiver v0.89.0
github.com/otiai10/copy v1.14.0
github.com/pierrre/gotestcover v0.0.0-20160517101806-924dca7d15f0
github.com/pkg/errors v0.9.1
github.com/rs/zerolog v1.27.0
github.com/schollz/progressbar/v3 v3.13.1
github.com/shirou/gopsutil/v3 v3.23.2
github.com/sirupsen/logrus v1.9.0
github.com/spf13/cobra v1.7.0
github.com/shirou/gopsutil/v3 v3.23.10
github.com/sirupsen/logrus v1.9.3
github.com/spf13/cobra v1.8.0
github.com/stretchr/testify v1.8.4
github.com/tsg/go-daemon v0.0.0-20200207173439-e704b93fd89b
go.elastic.co/apm/module/apmgorilla v1.15.0
go.elastic.co/ecszap v1.0.1
go.elastic.co/go-licence-detector v0.5.0
go.uber.org/zap v1.25.0
golang.org/x/crypto v0.14.0
go.opentelemetry.io/collector/component v0.89.0
go.opentelemetry.io/collector/confmap v0.89.0
go.opentelemetry.io/collector/exporter v0.89.0
go.opentelemetry.io/collector/exporter/debugexporter v0.89.0
go.opentelemetry.io/collector/exporter/otlpexporter v0.89.0
go.opentelemetry.io/collector/otelcol v0.89.0
go.opentelemetry.io/collector/processor v0.89.0
go.opentelemetry.io/collector/processor/batchprocessor v0.89.0
go.opentelemetry.io/collector/processor/memorylimiterprocessor v0.89.0
go.opentelemetry.io/collector/receiver v0.89.0
go.opentelemetry.io/collector/receiver/otlpreceiver v0.89.0
go.uber.org/zap v1.26.0
golang.org/x/crypto v0.15.0
golang.org/x/lint v0.0.0-20210508222113-6edffad5e616
golang.org/x/sync v0.3.0
golang.org/x/sys v0.13.0
golang.org/x/term v0.13.0
golang.org/x/text v0.13.0
golang.org/x/sync v0.5.0
golang.org/x/sys v0.14.0
golang.org/x/term v0.14.0
golang.org/x/text v0.14.0
golang.org/x/time v0.3.0
golang.org/x/tools v0.7.0
google.golang.org/grpc v1.58.3
golang.org/x/tools v0.15.0
golang.org/x/tools/go/vcs v0.1.0-deprecated
google.golang.org/grpc v1.59.0
google.golang.org/protobuf v1.31.0
gopkg.in/ini.v1 v1.67.0
gopkg.in/yaml.v2 v2.4.0
Expand All @@ -77,11 +94,17 @@ require (
)

require (
contrib.go.opencensus.io/exporter/prometheus v0.4.2 // indirect
github.com/Jeffail/gabs/v2 v2.6.0 // indirect
github.com/StackExchange/wmi v1.2.1 // indirect
github.com/akavel/rsrc v0.8.0 // indirect
github.com/alecthomas/participle/v2 v2.1.0 // indirect
github.com/antonmedv/expr v1.15.3 // indirect
github.com/armon/go-radix v1.0.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bmatcuk/doublestar/v4 v4.6.1 // indirect
github.com/cavaliercoder/badio v0.0.0-20160213150051-ce5280129e9e // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/cyphar/filepath-securejoin v0.2.4 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dnephin/pflag v1.0.7 // indirect
Expand All @@ -93,68 +116,139 @@ require (
github.com/elastic/gosigar v0.14.2 // indirect
github.com/emicklei/go-restful/v3 v3.10.1 // indirect
github.com/evanphx/json-patch v5.6.0+incompatible // indirect
github.com/felixge/httpsnoop v1.0.3 // indirect
github.com/ghodss/yaml v1.0.0 // indirect
github.com/go-logr/logr v1.2.3 // indirect
github.com/go-kit/log v0.2.1 // indirect
github.com/go-logfmt/logfmt v0.5.1 // indirect
github.com/go-logr/logr v1.3.0 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-ole/go-ole v1.2.6 // indirect
github.com/go-openapi/jsonpointer v0.19.5 // indirect
github.com/go-openapi/jsonreference v0.20.0 // indirect
github.com/go-openapi/swag v0.22.3 // indirect
github.com/gobuffalo/here v0.6.0 // indirect
github.com/gobwas/glob v0.2.3 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/gnostic v0.6.9 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/licenseclassifier v0.0.0-20200402202327-879cb1424de0 // indirect
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/iancoleman/strcase v0.3.0 // indirect
github.com/imdario/mergo v0.3.13 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/influxdata/go-syslog/v3 v3.0.1-0.20210608084020-ac565dc76ba6 // indirect
github.com/jaypipes/pcidb v1.0.0 // indirect
github.com/jcchavezs/porto v0.1.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/karrick/godirwalk v1.16.1 // indirect
github.com/klauspost/compress v1.17.2 // indirect
github.com/knadh/koanf/maps v0.1.1 // indirect
github.com/knadh/koanf/providers/confmap v0.1.0 // indirect
github.com/knadh/koanf/v2 v2.0.1 // indirect
github.com/leodido/ragel-machinery v0.0.0-20181214104525-299bdde78165 // indirect
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/markbates/pkger v0.17.0 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.17 // indirect
github.com/mattn/go-runewidth v0.0.15 // indirect
github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
github.com/mitchellh/copystructure v1.2.0 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/mitchellh/iochan v1.0.0 // indirect
github.com/mitchellh/reflectwalk v1.0.2 // indirect
github.com/moby/spdystream v0.2.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/mostynb/go-grpc-compression v1.2.2 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/onsi/ginkgo/v2 v2.9.0 // indirect
github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.89.0 // indirect
github.com/open-telemetry/opentelemetry-collector-contrib/internal/filter v0.89.0 // indirect
github.com/open-telemetry/opentelemetry-collector-contrib/internal/sharedcomponent v0.89.0 // indirect
github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl v0.89.0 // indirect
github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.89.0 // indirect
github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza v0.89.0 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.1.0-rc2 // indirect
github.com/opencontainers/image-spec v1.1.0-rc5 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
github.com/prometheus/procfs v0.9.0 // indirect
github.com/prometheus/client_golang v1.17.0 // indirect
github.com/prometheus/client_model v0.5.0 // indirect
github.com/prometheus/common v0.45.0 // indirect
github.com/prometheus/procfs v0.11.1 // indirect
github.com/prometheus/statsd_exporter v0.22.7 // indirect
github.com/rivo/uniseg v0.4.4 // indirect
github.com/rs/cors v1.10.1 // indirect
github.com/santhosh-tekuri/jsonschema v1.2.4 // indirect
github.com/sergi/go-diff v1.2.0 // indirect
github.com/shirou/gopsutil v3.21.11+incompatible // indirect
github.com/shoenig/go-m1cpu v0.1.6 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/stretchr/objx v0.5.0 // indirect
github.com/tklauser/go-sysconf v0.3.11 // indirect
github.com/tklauser/numcpus v0.6.0 // indirect
github.com/tklauser/go-sysconf v0.3.12 // indirect
github.com/tklauser/numcpus v0.6.1 // indirect
github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a // indirect
github.com/yusufpapurcu/wmi v1.2.2 // indirect
github.com/yusufpapurcu/wmi v1.2.3 // indirect
go.elastic.co/apm/module/apmhttp v1.15.0 // indirect
go.elastic.co/apm/module/apmhttp/v2 v2.0.0 // indirect
go.elastic.co/apm/v2 v2.0.0 // indirect
go.elastic.co/fastjson v1.1.0 // indirect
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/collector v0.89.0 // indirect
go.opentelemetry.io/collector/config/configauth v0.89.0 // indirect
go.opentelemetry.io/collector/config/configcompression v0.89.0 // indirect
go.opentelemetry.io/collector/config/configgrpc v0.89.0 // indirect
go.opentelemetry.io/collector/config/confighttp v0.89.0 // indirect
go.opentelemetry.io/collector/config/confignet v0.89.0 // indirect
go.opentelemetry.io/collector/config/configopaque v0.89.0 // indirect
go.opentelemetry.io/collector/config/configtelemetry v0.89.0 // indirect
go.opentelemetry.io/collector/config/configtls v0.89.0 // indirect
go.opentelemetry.io/collector/config/internal v0.89.0 // indirect
go.opentelemetry.io/collector/connector v0.89.0 // indirect
go.opentelemetry.io/collector/consumer v0.89.0 // indirect
go.opentelemetry.io/collector/extension v0.89.0 // indirect
go.opentelemetry.io/collector/extension/auth v0.89.0 // indirect
go.opentelemetry.io/collector/featuregate v1.0.0-rcv0018 // indirect
go.opentelemetry.io/collector/pdata v1.0.0-rcv0018 // indirect
go.opentelemetry.io/collector/semconv v0.89.0 // indirect
go.opentelemetry.io/collector/service v0.89.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 // indirect
go.opentelemetry.io/contrib/propagators/b3 v1.20.0 // indirect
go.opentelemetry.io/otel v1.20.0 // indirect
go.opentelemetry.io/otel/bridge/opencensus v0.43.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.43.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.43.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.20.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.20.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.20.0 // indirect
go.opentelemetry.io/otel/exporters/prometheus v0.43.0 // indirect
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v0.43.0 // indirect
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.20.0 // indirect
go.opentelemetry.io/otel/metric v1.20.0 // indirect
go.opentelemetry.io/otel/sdk v1.20.0 // indirect
go.opentelemetry.io/otel/sdk/metric v1.20.0 // indirect
go.opentelemetry.io/otel/trace v1.20.0 // indirect
go.opentelemetry.io/proto/otlp v1.0.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect
golang.org/x/mod v0.9.0 // indirect
golang.org/x/net v0.17.0 // indirect
golang.org/x/oauth2 v0.10.0 // indirect
golang.org/x/exp v0.0.0-20230711023510-fffb14384f22 // indirect
golang.org/x/mod v0.14.0 // indirect
golang.org/x/net v0.18.0 // indirect
golang.org/x/oauth2 v0.12.0 // indirect
gonum.org/v1/gonum v0.14.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20231012201019-e917dd12ba7a // indirect
google.golang.org/grpc/examples v0.0.0-20220304170021-431ea809a767 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
howett.net/plist v1.0.0 // indirect
k8s.io/kube-openapi v0.0.0-20221207184640-f3cff1453715 // indirect
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
Expand All @@ -163,7 +257,7 @@ require (
)

require (
github.com/hashicorp/go-version v1.2.1 // indirect
github.com/hashicorp/go-version v1.6.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/onsi/gomega v1.27.3 // indirect
go.elastic.co/apm v1.15.0
Expand All @@ -176,7 +270,6 @@ replace (
github.com/Shopify/sarama => github.com/elastic/sarama v1.19.1-0.20220310193331-ebc2b0d8eef3
github.com/dop251/goja => github.com/andrewkroh/goja v0.0.0-20190128172624-dd2ac4456e20
github.com/dop251/goja_nodejs => github.com/dop251/goja_nodejs v0.0.0-20171011081505-adff31b136e6
github.com/fsnotify/fsnotify => github.com/adriansr/fsnotify v1.4.8-0.20211018144411-a81f2b630e7c
github.com/tonistiigi/fifo => github.com/containerd/fifo v0.0.0-20190816180239-bda0ff6ed73c
)

Expand Down
Loading

0 comments on commit 1410d2a

Please sign in to comment.