Merge sending logs through otel-collector (#147)

Send fluentd logs through otel (#109) This sends logs collected by fluentd to otelcol via fluentforward. There are some limitations noted by TODOs that I will file issues to track but they should not affect the common cases. It's mostly around configuring of hec_exporter TLS settings. It still attaches k8s metadata on the fluentd side as it uses various annotations to construct source/sourcetype in some cases. May not be worth trying to fix with move to filelog receiver. * Fluentd cleanup (#123) * Remove hec token * remove ingestHost, ingestPort, ingestProtocol * Disable receivers and pipelines if telemetry type disabled (#127) - Fix changelog ordering - Don't include fluentd configmap when agent not enabled - Enable http-forwarder for all telemetry types since signalfx exporter sends metadata updates * remove fluentforward from service (no longer configured in gateway) * use <ingest>/v1/log for hec endpoint
signalfx · May 21, 2021 · 358cd3f · 358cd3f
1 parent 3612827
commit 358cd3f
Show file tree

Hide file tree

Showing 31 changed files with 407 additions and 441 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -6,11 +6,18 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ### Changed
 
+- fluentd logs are now sent through the collector instead of being sent directly to the backend (#109)
+- Logs are sent through the OpenTelemetry Agent on the local node by default. `otelAgent.enabled` value must be set to `true` when using logs (#127)
 - `otelAgent.ports` and `otelCollector.ports` are selectively enabled depending on what telemetry types are enabled with `metricsEnabled`, `tracesEnabled`, and `logsEnabled`
 - Removed setting `host.name` through the `resource` processor as it is already set by the `resourcedetection/system` processor
 
 ### Removed
 
+- Removed `ingestHost`, `ingestPort`, `ingestProtocol`, use `ingestUrl` instead (#123)
+- Removed `logsBackend`, configure `splunk_hec` exporter directly (#123)
+- Removed `splunk.com/index` annotation for logs (#123)
+- Removed `fluentd.config.indexFields` as all fields sent are indexed (#123)
+- Removed `fluentforward` receiver from gateway (#127)
 - Removed `service.ports`, sourced from `otelCollector.ports` instead (#140)
 
 ## [0.25.0] - 2021-05-07

diff --git a/Makefile b/Makefile
@@ -33,7 +33,7 @@ render:
 		--namespace default \
 		--values rendered/values.yaml \
 		--output-dir "$$dir" \
-		--set otelAgent.enabled=false,otelCollector.enabled=true,otelK8sClusterReceiver.enabled=false,logsEnabled=false \
+		--set otelAgent.enabled=false,otelCollector.enabled=true,otelK8sClusterReceiver.enabled=false \
 		default helm-charts/splunk-otel-collector; \
 	mv "$$dir"/splunk-otel-collector/templates/* "$$dir"; \
 	rm -rf "$$dir"/splunk-otel-collector
diff --git a/helm-charts/splunk-otel-collector/templates/_helpers.tpl b/helm-charts/splunk-otel-collector/templates/_helpers.tpl
@@ -50,25 +50,12 @@ Create the name of the service account to use
     {{ default (include "splunk-otel-collector.fullname" .) .Values.serviceAccount.name }}
 {{- end -}}
 
-{{/*
-Get Splunk ingest host
-*/}}
-{{- define "splunk-otel-collector.ingestHost" -}}
-{{- $_ := required "splunkRealm or ingestHost must be provided" (or .Values.ingestHost .Values.splunkRealm) }}
-{{- .Values.ingestHost | default (printf "ingest.%s.signalfx.com" .Values.splunkRealm) }}
-{{- end -}}
-
 {{/*
 Get Splunk ingest URL
 */}}
 {{- define "splunk-otel-collector.ingestUrl" -}}
-{{- $host := include "splunk-otel-collector.ingestHost" . }}
-{{- $endpoint := printf "%s://%s" .Values.ingestProtocol $host }}
-{{- if or (and (eq .Values.ingestProtocol "http") (ne (toString .Values.ingestPort) "80")) (and (eq .Values.ingestProtocol "https") (ne (toString .Values.ingestPort) "443")) }}
-{{- printf "%s:%s" $endpoint (toString .Values.ingestPort) }}
-{{- else }}
-{{- $endpoint }}
-{{- end }}
+{{- $_ := required "splunkRealm or ingestUrl must be provided" (or .Values.ingestUrl .Values.splunkRealm) }}
+{{- .Values.ingestUrl | default (printf "https://ingest.%s.signalfx.com" .Values.splunkRealm) }}
 {{- end -}}
 
 {{/*

diff --git a/helm-charts/splunk-otel-collector/templates/config/_common.tpl b/helm-charts/splunk-otel-collector/templates/config/_common.tpl
@@ -15,10 +15,12 @@ memory_limiter:
 Common config for the otel-collector sapm exporter
 */}}
 {{- define "splunk-otel-collector.otelSapmExporter" -}}
+{{- if .Values.tracesEnabled }}
 sapm:
   endpoint: {{ include "splunk-otel-collector.ingestUrl" . }}/v2/trace
   access_token: ${SPLUNK_ACCESS_TOKEN}
 {{- end }}
+{{- end }}
 
 {{/*
 Common config for the otel-collector traces receivers
@@ -30,6 +32,8 @@ otlp:
       endpoint: 0.0.0.0:4317
     http:
       endpoint: 0.0.0.0:55681
+
+{{- if .Values.tracesEnabled }}
 sapm:
   endpoint: 0.0.0.0:7276
 jaeger:
@@ -41,3 +45,4 @@ jaeger:
 zipkin:
   endpoint: 0.0.0.0:9411
 {{- end }}
+{{- end }}
diff --git a/helm-charts/splunk-otel-collector/templates/config/_otel-agent.tpl b/helm-charts/splunk-otel-collector/templates/config/_otel-agent.tpl
@@ -14,6 +14,11 @@ extensions:
 
 receivers:
   {{- include "splunk-otel-collector.otelTraceReceivers" . | nindent 2 }}
+  {{- if .Values.logsEnabled }}
+  fluentforward:
+    endpoint: 0.0.0.0:8006
+  {{- end }}
+
   # Prometheus receiver scraping metrics from the pod itself
   prometheus/agent:
     config:
@@ -26,6 +31,7 @@ receivers:
           # Fluend metrics collection disabled by default
           # - "${K8S_POD_IP}:24231"
 
+  {{- if .Values.metricsEnabled }}
   hostmetrics:
     collection_interval: 10s
     scrapers:
@@ -73,10 +79,13 @@ receivers:
 
   signalfx:
     endpoint: 0.0.0.0:9943
+  {{- end }}
 
+  {{- if .Values.tracesEnabled }}
   smartagent/signalfx-forwarder:
     type: signalfx-forwarder
     listenAddress: 0.0.0.0:9080
+  {{- end }}
 
 # By default k8s_tagger and batch processors enabled.
 processors:
@@ -175,14 +184,20 @@ processors:
 exporters:
 
   {{- if .Values.otelCollector.enabled }}
-  # If collector is enabled, metrics and traces will be sent to collector
+  # If collector is enabled, metrics, logs and traces will be sent to collector
   otlp:
     endpoint: {{ include "splunk-otel-collector.fullname" . }}:4317
     insecure: true
   {{- else }}
-  # If collector is disabled, metrics and traces will be set to to SignalFx backend
+  # If collector is disabled, metrics, logs and traces will be sent to to SignalFx backend
   {{- include "splunk-otel-collector.otelSapmExporter" . | nindent 2 }}
+  {{- if .Values.logsEnabled }}
+  splunk_hec:
+    endpoint: {{ include "splunk-otel-collector.ingestUrl" . }}/v1/log
+    token: "${SPLUNK_ACCESS_TOKEN}"
+  {{- end }}
   {{- end }}
+
   signalfx:
     correlation:
     {{- if .Values.otelCollector.enabled }}
@@ -203,8 +218,27 @@ service:
   # The default pipelines should to be changed. You can add any custom pipeline instead.
   # In order to disable a default pipeline just set it to `null` in otelAgent.config overrides.
   pipelines:
+    {{- if .Values.logsEnabled }}
+    logs:
+      receivers: [fluentforward]
+      processors:
+        - memory_limiter
+        - batch
+        - resource
+        - resourcedetection
+        {{- if .Values.environment }}
+        - resource/add_environment
+        {{- end }}
+      exporters:
+        {{- if .Values.otelCollector.enabled }}
+        - otlp
+        {{- else }}
+        - splunk_hec
+        {{- end }}
+    {{- end }}
 
-    # default traces pipeline
+    {{- if .Values.tracesEnabled }}
+    # Default traces pipeline.
     traces:
       receivers: [otlp, jaeger, smartagent/signalfx-forwarder, zipkin]
       processors:
@@ -222,9 +256,14 @@ service:
         {{- else }}
         - sapm
         {{- end }}
+        {{- if .Values.metricsEnabled }}
+        # For trace/metric correlation.
         - signalfx
+        {{- end }}
+    {{- end }}
 
-    # default metrics pipeline
+    {{- if .Values.metricsEnabled }}
+    # Default metrics pipeline.
     metrics:
       receivers: [hostmetrics, kubeletstats, receiver_creator, signalfx]
       processors:
@@ -238,6 +277,7 @@ service:
         {{- else }}
         - signalfx
         {{- end }}
+    {{- end }}
 
     # Pipeline for metrics collected about the agent pod itself.
     metrics/agent:

diff --git a/helm-charts/splunk-otel-collector/templates/config/_otel-collector.tpl b/helm-charts/splunk-otel-collector/templates/config/_otel-collector.tpl
@@ -22,10 +22,6 @@ receivers:
         scrape_interval: 10s
         static_configs:
         - targets: ["${K8S_POD_IP}:8888"]
-
-  fluentforward:
-    endpoint: 0.0.0.0:8006
-
   signalfx:
     endpoint: 0.0.0.0:9943
     access_token_passthrough: true
@@ -71,6 +67,11 @@ processors:
 
 exporters:
   {{- include "splunk-otel-collector.otelSapmExporter" . | nindent 2 }}
+  {{- if .Values.logsEnabled }}
+  splunk_hec:
+    endpoint: {{ include "splunk-otel-collector.ingestUrl" . }}/v1/log
+    token: "${SPLUNK_ACCESS_TOKEN}"
+  {{- end }}
   signalfx:
     ingest_url: {{ include "splunk-otel-collector.ingestUrl" . }}
     api_url: {{ include "splunk-otel-collector.apiUrl" . }}
@@ -82,7 +83,7 @@ service:
   # The default pipelines should not need to be changed. You can add any custom pipeline instead.
   # In order to disable a default pipeline just set it to `null` in otelCollector.config overrides.
   pipelines:
-
+    {{- if .Values.tracesEnabled }}
     # default traces pipeline
     traces:
       receivers: [otlp, jaeger, zipkin, sapm]
@@ -95,16 +96,25 @@ service:
         - resource/add_environment
         {{- end }}
       exporters: [sapm]
+    {{- end }}
 
     # default metrics pipeline
     metrics:
       receivers: [otlp, prometheus, signalfx]
       processors: [memory_limiter, batch, resource/add_cluster_name]
       exporters: [signalfx]
 
-    # default logs pipeline
-    logs:
+    # logs pipeline for receiving and exporting SignalFx events
+    logs/signalfx-events:
       receivers: [signalfx]
       processors: [memory_limiter, batch]
       exporters: [signalfx]
+
+    {{- if .Values.logsEnabled }}
+    # default logs pipeline
+    logs:
+      receivers: [otlp]
+      processors: [memory_limiter, batch]
+      exporters: [splunk_hec]
+    {{- end }}
 {{- end }}