diff --git a/CHANGELOG.md b/CHANGELOG.md index 2038d37a91..5aa44dbc61 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +### 💡 Enhancements 💡 + +- (Splunk) Add bundled collectd/nginx Smart Agent receiver discovery rules ([#3321](https://github.com/signalfx/splunk-otel-collector/pull/3321)) + ## v0.80.0 This Splunk OpenTelemetry Collector release includes changes from the [opentelemetry-collector v0.80.0](https://github.com/open-telemetry/opentelemetry-collector/releases/tag/v0.80.0) and the [opentelemetry-collector-contrib v0.80.0](https://github.com/open-telemetry/opentelemetry-collector-contrib/releases/tag/v0.80.0) releases where appropriate. diff --git a/cmd/otelcol/config/collector/config.d.linux/receivers/smartagent-collectd-nginx.discovery.yaml b/cmd/otelcol/config/collector/config.d.linux/receivers/smartagent-collectd-nginx.discovery.yaml new file mode 100644 index 0000000000..378c7e47de --- /dev/null +++ b/cmd/otelcol/config/collector/config.d.linux/receivers/smartagent-collectd-nginx.discovery.yaml @@ -0,0 +1,37 @@ +##################################################################################### +# This file is generated by the Splunk Distribution of the OpenTelemetry Collector. # +##################################################################################### +# smartagent/collectd/nginx: +# enabled: true +# rule: +# docker_observer: type == "container" and any([name, image, command], {# matches "(?i)nginx"}) and not (command matches "splunk.discovery") +# host_observer: type == "hostport" and command matches "(?i)nginx" and not (command matches "splunk.discovery") +# k8s_observer: type == "port" and pod.name matches "(?i:nginx)" +# config: +# default: +# type: collectd/nginx +# url: '`(port in [443, 8443] ? "https" : "http") + "://{{.Host}}:{{.Port}}/nginx_status"`' +# timeout: 5000 +# isolatedCollectd: true +# status: +# metrics: +# successful: +# - strict: connections.accepted +# first_only: true +# log_record: +# severity_text: info +# body: smartagent/collectd/nginx receiver is working! +# statements: +# failed: +# - regexp: "nginx plugin: curl_easy_perform failed: Operation timed out after" +# first_only: true +# log_record: +# severity_text: info +# append_pattern: true +# body: The container is not serving http connections. +# - regexp: "read-function of plugin .* failed" +# first_only: true +# log_record: +# severity_text: info +# append_pattern: true +# body: The integration is unable to read metrics from this endpoint. \ No newline at end of file diff --git a/internal/confmapprovider/discovery/bundle/bundle.d/receivers/smartagent-collectd-nginx.discovery.yaml b/internal/confmapprovider/discovery/bundle/bundle.d/receivers/smartagent-collectd-nginx.discovery.yaml new file mode 100644 index 0000000000..e7900b6f87 --- /dev/null +++ b/internal/confmapprovider/discovery/bundle/bundle.d/receivers/smartagent-collectd-nginx.discovery.yaml @@ -0,0 +1,37 @@ +##################################################################################### +# This file is generated by the Splunk Distribution of the OpenTelemetry Collector. # +##################################################################################### +smartagent/collectd/nginx: + enabled: true + rule: + docker_observer: type == "container" and any([name, image, command], {# matches "(?i)nginx"}) and not (command matches "splunk.discovery") + host_observer: type == "hostport" and command matches "(?i)nginx" and not (command matches "splunk.discovery") + k8s_observer: type == "port" and pod.name matches "(?i:nginx)" + config: + default: + type: collectd/nginx + url: '`(port in [443, 8443] ? "https" : "http") + "://{{.Host}}:{{.Port}}/nginx_status"`' + timeout: 5000 + isolatedCollectd: true + status: + metrics: + successful: + - strict: connections.accepted + first_only: true + log_record: + severity_text: info + body: smartagent/collectd/nginx receiver is working! + statements: + failed: + - regexp: "nginx plugin: curl_easy_perform failed: Operation timed out after" + first_only: true + log_record: + severity_text: info + append_pattern: true + body: The container is not serving http connections. + - regexp: "read-function of plugin .* failed" + first_only: true + log_record: + severity_text: info + append_pattern: true + body: The integration is unable to read metrics from this endpoint. \ No newline at end of file diff --git a/internal/confmapprovider/discovery/bundle/bundle.d/receivers/smartagent-collectd-nginx.discovery.yaml.tmpl b/internal/confmapprovider/discovery/bundle/bundle.d/receivers/smartagent-collectd-nginx.discovery.yaml.tmpl new file mode 100644 index 0000000000..82b0e13756 --- /dev/null +++ b/internal/confmapprovider/discovery/bundle/bundle.d/receivers/smartagent-collectd-nginx.discovery.yaml.tmpl @@ -0,0 +1,34 @@ +{{ receiver "smartagent/collectd/nginx" }}: + enabled: true + rule: + docker_observer: type == "container" and any([name, image, command], {# matches "(?i)nginx"}) and not (command matches "splunk.discovery") + host_observer: type == "hostport" and command matches "(?i)nginx" and not (command matches "splunk.discovery") + k8s_observer: type == "port" and pod.name matches "(?i:nginx)" + config: + default: + type: collectd/nginx + url: '`(port in [443, 8443] ? "https" : "http") + "{{`://{{.Host}}:{{.Port}}/nginx_status`}}"`' + timeout: 5000 + isolatedCollectd: true + status: + metrics: + successful: + - strict: connections.accepted + first_only: true + log_record: + severity_text: info + body: smartagent/collectd/nginx receiver is working! + statements: + failed: + - regexp: "nginx plugin: curl_easy_perform failed: Operation timed out after" + first_only: true + log_record: + severity_text: info + append_pattern: true + body: The container is not serving http connections. + - regexp: "read-function of plugin .* failed" + first_only: true + log_record: + severity_text: info + append_pattern: true + body: The integration is unable to read metrics from this endpoint. \ No newline at end of file diff --git a/internal/confmapprovider/discovery/bundle/bundle_gen.go b/internal/confmapprovider/discovery/bundle/bundle_gen.go index 25adef36d9..5748baa7e6 100644 --- a/internal/confmapprovider/discovery/bundle/bundle_gen.go +++ b/internal/confmapprovider/discovery/bundle/bundle_gen.go @@ -23,6 +23,8 @@ //go:generate discoverybundler -r -c -d ../../../../cmd/otelcol/config/collector/config.d.linux/receivers -t bundle.d/receivers/smartagent-postgresql.discovery.yaml.tmpl //go:generate discoverybundler -r -t bundle.d/receivers/smartagent-collectd-mysql.discovery.yaml.tmpl //go:generate discoverybundler -r -c -d ../../../../cmd/otelcol/config/collector/config.d.linux/receivers -t bundle.d/receivers/smartagent-collectd-mysql.discovery.yaml.tmpl +//go:generate discoverybundler -r -t bundle.d/receivers/smartagent-collectd-nginx.discovery.yaml.tmpl +//go:generate discoverybundler -r -c -d ../../../../cmd/otelcol/config/collector/config.d.linux/receivers -t bundle.d/receivers/smartagent-collectd-nginx.discovery.yaml.tmpl //go:generate discoverybundler -r -t bundle.d/extensions/docker-observer.discovery.yaml.tmpl //go:generate discoverybundler -r -c -d ../../../../cmd/otelcol/config/collector/config.d.linux/extensions -t bundle.d/extensions/docker-observer.discovery.yaml.tmpl diff --git a/internal/confmapprovider/discovery/bundle/bundle_other_test.go b/internal/confmapprovider/discovery/bundle/bundle_other_test.go index e8f39a6985..6ea3b2385f 100644 --- a/internal/confmapprovider/discovery/bundle/bundle_other_test.go +++ b/internal/confmapprovider/discovery/bundle/bundle_other_test.go @@ -28,6 +28,7 @@ func TestBundleDir(t *testing.T) { require.NoError(t, err) require.Equal(t, []string{ "bundle.d/receivers/smartagent-collectd-mysql.discovery.yaml", + "bundle.d/receivers/smartagent-collectd-nginx.discovery.yaml", "bundle.d/receivers/smartagent-postgresql.discovery.yaml", }, receivers) diff --git a/tests/general/discoverymode/docker_observer_discovery_test.go b/tests/general/discoverymode/docker_observer_discovery_test.go index d81ec57c8c..0c19b853f8 100644 --- a/tests/general/discoverymode/docker_observer_discovery_test.go +++ b/tests/general/discoverymode/docker_observer_discovery_test.go @@ -21,7 +21,6 @@ import ( "os" "path/filepath" "runtime" - "syscall" "testing" "time" @@ -43,13 +42,6 @@ func TestDockerObserver(t *testing.T) { defer tc.PrintLogsOnFailure() defer tc.ShutdownOTLPReceiverSink() - finfo, err := os.Stat("/var/run/docker.sock") - require.NoError(t, err) - fsys := finfo.Sys() - stat, ok := fsys.(*syscall.Stat_t) - require.True(t, ok) - dockerGID := stat.Gid - cc, shutdown := tc.SplunkOtelCollectorContainer( "docker-otlp-exporter-no-internal-prometheus.yaml", func(c testutils.Collector) testutils.Collector { @@ -61,7 +53,7 @@ func TestDockerObserver(t *testing.T) { cc.Container = cc.Container.WillWaitForLogs("Discovering for next") // uid check is for basic collector functionality not using the splunk-otel-collector user // but the docker gid is required to reach the daemon - cc.Container = cc.Container.WithUser(fmt.Sprintf("%d:%d", os.Getuid(), dockerGID)) + cc.Container = cc.Container.WithUser(fmt.Sprintf("%d:%d", os.Getuid(), testutils.GetDockerGID(t))) return cc }, func(c testutils.Collector) testutils.Collector { diff --git a/tests/receivers/smartagent/collectd-nginx/bundled_test.go b/tests/receivers/smartagent/collectd-nginx/bundled_test.go new file mode 100644 index 0000000000..149dfc3af5 --- /dev/null +++ b/tests/receivers/smartagent/collectd-nginx/bundled_test.go @@ -0,0 +1,77 @@ +// Copyright Splunk, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build integration + +package tests + +import ( + "fmt" + "path" + "runtime" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/signalfx/splunk-otel-collector/tests/testutils" +) + +func TestDockerObserver(t *testing.T) { + testutils.SkipIfNotContainerTest(t) + if runtime.GOOS == "darwin" { + t.Skip("unable to share sockets between mac and d4m vm: https://github.com/docker/for-mac/issues/483#issuecomment-758836836") + } + tc := testutils.NewTestcase(t) + defer tc.PrintLogsOnFailure() + defer tc.ShutdownOTLPReceiverSink() + + _, stop := tc.Containers( + testutils.NewContainer().WithContext( + path.Join(".", "testdata", "server"), + ).WithExposedPorts( + fmt.Sprintf("%d:80", testutils.GetAvailablePort(t)), + ).WithName("nginx").WillWaitForPorts("80"), + ) + defer stop() + + _, shutdown := tc.SplunkOtelCollectorContainer( + "otlp_exporter.yaml", + func(c testutils.Collector) testutils.Collector { + cc := c.(*testutils.CollectorContainer) + cc.Container = cc.Container.WithBinds("/var/run/docker.sock:/var/run/docker.sock:ro") + cc.Container = cc.Container.WillWaitForLogs("Discovering for next") + cc.Container = cc.Container.WithUser(fmt.Sprintf("999:%d", testutils.GetDockerGID(t))) + return cc + }, + func(c testutils.Collector) testutils.Collector { + return c.WithEnv(map[string]string{ + // runner seems to be slow + "SPLUNK_DISCOVERY_DURATION": "20s", + // confirm that debug logging doesn't affect runtime + "SPLUNK_DISCOVERY_LOG_LEVEL": "debug", + }).WithArgs( + "--discovery", + "--set", "splunk.discovery.receivers.smartagent/collectd/nginx.config.username=some_user", + "--set", "splunk.discovery.receivers.smartagent/collectd/nginx.config.password=some_password", + "--set", `splunk.discovery.extensions.k8s_observer.enabled=false`, + "--set", `splunk.discovery.extensions.host_observer.enabled=false`, + ) + }, + ) + defer shutdown() + + expectedResourceMetrics := tc.ResourceMetrics("default.yaml") + require.NoError(t, tc.OTLPReceiverSink.AssertAllMetricsReceived(t, *expectedResourceMetrics, 30*time.Second)) +} diff --git a/tests/receivers/smartagent/collectd-nginx/testdata/all_metrics_config.yaml b/tests/receivers/smartagent/collectd-nginx/testdata/all_metrics_config.yaml index 5509d59cd9..481ebb12c0 100644 --- a/tests/receivers/smartagent/collectd-nginx/testdata/all_metrics_config.yaml +++ b/tests/receivers/smartagent/collectd-nginx/testdata/all_metrics_config.yaml @@ -3,6 +3,8 @@ receivers: type: collectd/nginx host: localhost port: 8123 + username: some_user + password: some_password extraMetrics: ["*"] intervalSeconds: 1 diff --git a/tests/receivers/smartagent/collectd-nginx/testdata/otlp_exporter.yaml b/tests/receivers/smartagent/collectd-nginx/testdata/otlp_exporter.yaml new file mode 100644 index 0000000000..fa00f3c646 --- /dev/null +++ b/tests/receivers/smartagent/collectd-nginx/testdata/otlp_exporter.yaml @@ -0,0 +1,13 @@ +exporters: + otlp: + endpoint: "${OTLP_ENDPOINT}" + tls: + insecure: true + +service: + telemetry: + logs: + level: debug + pipelines: + metrics: + exporters: [otlp] diff --git a/tests/receivers/smartagent/collectd-nginx/testdata/resource_metrics/default.yaml b/tests/receivers/smartagent/collectd-nginx/testdata/resource_metrics/default.yaml new file mode 100644 index 0000000000..8075ae01b0 --- /dev/null +++ b/tests/receivers/smartagent/collectd-nginx/testdata/resource_metrics/default.yaml @@ -0,0 +1,22 @@ +resource_metrics: + - scope_metrics: + - metrics: + - name: connections.accepted + type: IntMonotonicCumulativeSum + attributes: + dsname: value + host: + plugin: nginx + system.type: nginx + - name: connections.handled + type: IntMonotonicCumulativeSum + - name: nginx_connections.active + type: IntGauge + - name: nginx_connections.reading + type: IntGauge + - name: nginx_connections.waiting + type: IntGauge + - name: nginx_connections.writing + type: IntGauge + - name: nginx_requests + type: IntMonotonicCumulativeSum diff --git a/tests/receivers/smartagent/collectd-nginx/testdata/server/Dockerfile b/tests/receivers/smartagent/collectd-nginx/testdata/server/Dockerfile index ebff16c579..053c8c0619 100644 --- a/tests/receivers/smartagent/collectd-nginx/testdata/server/Dockerfile +++ b/tests/receivers/smartagent/collectd-nginx/testdata/server/Dockerfile @@ -2,3 +2,4 @@ FROM nginx:1.13.8 RUN rm /etc/nginx/conf.d/default.conf COPY status.conf /etc/nginx/conf.d/status.conf +COPY htpasswd /etc/nginx/.htpasswd diff --git a/tests/receivers/smartagent/collectd-nginx/testdata/server/htpasswd b/tests/receivers/smartagent/collectd-nginx/testdata/server/htpasswd new file mode 100644 index 0000000000..02a2c435d0 --- /dev/null +++ b/tests/receivers/smartagent/collectd-nginx/testdata/server/htpasswd @@ -0,0 +1,2 @@ +some_user:$apr1$mn7FX9UF$2wjlEnZe50o./Vd4mpSLc0 + diff --git a/tests/receivers/smartagent/collectd-nginx/testdata/server/status.conf b/tests/receivers/smartagent/collectd-nginx/testdata/server/status.conf index dfaee0497e..c65cc00776 100644 --- a/tests/receivers/smartagent/collectd-nginx/testdata/server/status.conf +++ b/tests/receivers/smartagent/collectd-nginx/testdata/server/status.conf @@ -1,6 +1,9 @@ server { listen 80; + auth_basic "restricted"; + auth_basic_user_file /etc/nginx/.htpasswd; + location /nginx_status { stub_status; } diff --git a/tests/testutils/container.go b/tests/testutils/container.go index d652aa343d..ca91325ca9 100644 --- a/tests/testutils/container.go +++ b/tests/testutils/container.go @@ -19,6 +19,8 @@ import ( "context" "fmt" "io" + "os" + "syscall" "testing" "time" @@ -484,3 +486,12 @@ func (container *Container) createNetworksIfNecessary(req testcontainers.Generic } return nil } + +func GetDockerGID(t testing.TB) uint32 { + finfo, err := os.Stat("/var/run/docker.sock") + require.NoError(t, err) + fsys := finfo.Sys() + stat, ok := fsys.(*syscall.Stat_t) + require.True(t, ok) + return stat.Gid +}