From e7f0204cc3753443e17216a4c048516733e77b4f Mon Sep 17 00:00:00 2001 From: "James Hughes (Splunk)" Date: Wed, 18 Jan 2023 16:20:11 -0800 Subject: [PATCH] Switch to gcp from gce/gke (#2488) * Switch to gcp from gce as specified in https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/10348 Tracking issue: https://github.com/signalfx/splunk-otel-collector/issues/2474 * Update cmd/otelcol/config/collector/full_config_linux.yaml Co-authored-by: Antoine Toulme Co-authored-by: Antoine Toulme --- .../config/collector/agent_config.yaml | 2 +- .../config/collector/full_config_linux.yaml | 6 +- .../config/collector/gateway_config.yaml | 2 +- .../config/collector/otlp_config_linux.yaml | 2 +- .../collector/upstream_agent_config.yaml | 2 +- cmd/translatesfx/README.md | 2 +- cmd/translatesfx/translatesfx/otel.go | 2 +- cmd/translatesfx/translatesfx/otel_test.go | 2 +- .../testdata/otel-e2e-expected.yaml | 2 +- docs/apm-infra-correlation.md | 4 +- internal/configconverter/README.md | 0 internal/configconverter/normalize_gcp.go | 74 ++++++++++++ .../configconverter/normalize_gcp_test.go | 113 ++++++++++++++++++ .../normalize_gcp/upstream_agent_config.yaml | 9 ++ .../upstream_agent_config_many.yaml | 4 + .../upstream_agent_config_no_op.yaml | 4 + .../upstream_agent_config_post_migration.yaml | 4 + .../upstream_agent_config_subresources.yaml | 9 ++ ...nt_config_subresources_post_migration.yaml | 9 ++ internal/settings/settings.go | 1 + internal/settings/settings_test.go | 1 + 21 files changed, 241 insertions(+), 13 deletions(-) create mode 100644 internal/configconverter/README.md create mode 100644 internal/configconverter/normalize_gcp.go create mode 100644 internal/configconverter/normalize_gcp_test.go create mode 100644 internal/configconverter/testdata/normalize_gcp/upstream_agent_config.yaml create mode 100644 internal/configconverter/testdata/normalize_gcp/upstream_agent_config_many.yaml create mode 100644 internal/configconverter/testdata/normalize_gcp/upstream_agent_config_no_op.yaml create mode 100644 internal/configconverter/testdata/normalize_gcp/upstream_agent_config_post_migration.yaml create mode 100644 internal/configconverter/testdata/normalize_gcp/upstream_agent_config_subresources.yaml create mode 100644 internal/configconverter/testdata/normalize_gcp/upstream_agent_config_subresources_post_migration.yaml diff --git a/cmd/otelcol/config/collector/agent_config.yaml b/cmd/otelcol/config/collector/agent_config.yaml index d8915c4a05..08df72aa99 100644 --- a/cmd/otelcol/config/collector/agent_config.yaml +++ b/cmd/otelcol/config/collector/agent_config.yaml @@ -110,7 +110,7 @@ processors: # libraries can send wrong values from container environments. # https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/resourcedetectionprocessor#ordering resourcedetection: - detectors: [gce, ecs, ec2, azure, system] + detectors: [gcp, ecs, ec2, azure, system] override: true # Optional: The following processor can be used to add a default "deployment.environment" attribute to the logs and diff --git a/cmd/otelcol/config/collector/full_config_linux.yaml b/cmd/otelcol/config/collector/full_config_linux.yaml index a0f8d12537..5c8b2794f0 100644 --- a/cmd/otelcol/config/collector/full_config_linux.yaml +++ b/cmd/otelcol/config/collector/full_config_linux.yaml @@ -204,8 +204,8 @@ processors: # Enables the resource detection processor with default settings # Full configuration here: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/resourcedetectionprocessor # NOTE: This processor settings need to be change when using this processor - #resouredection: - # # a list of resource detectors to run, valid options are: "env", "system", "gce", "ec2", "ecs", "elastic_beanstalk" + #resourcedetection: + # # a list of resource detectors to run, valid options are: "env", "system", "gcp", "ec2", "ecs", "elastic_beanstalk" #detectors: [ ] # # determines if existing resource attributes should be overridden or preserved, defaults to true #override: @@ -214,7 +214,7 @@ processors: # Detector order is important: the `system` detector goes last so it can't preclude cloud detectors from setting host/os info. # https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/resourcedetectionprocessor#ordering resourcedetection/internal: - detectors: [gce, ecs, ec2, azure, system] + detectors: [gcp, ecs, ec2, azure, system] override: true # Enables the memory limiter processor with default settings diff --git a/cmd/otelcol/config/collector/gateway_config.yaml b/cmd/otelcol/config/collector/gateway_config.yaml index 58cca22740..34a15ae7a1 100644 --- a/cmd/otelcol/config/collector/gateway_config.yaml +++ b/cmd/otelcol/config/collector/gateway_config.yaml @@ -83,7 +83,7 @@ processors: # Detector order is important: the `system` detector goes last so it can't preclude cloud detectors from setting host/os info. # https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/resourcedetectionprocessor#ordering resourcedetection/internal: - detectors: [gce, ecs, ec2, azure, system] + detectors: [gcp, ecs, ec2, azure, system] override: true exporters: diff --git a/cmd/otelcol/config/collector/otlp_config_linux.yaml b/cmd/otelcol/config/collector/otlp_config_linux.yaml index ff6b95c6c6..2f6b005dcc 100644 --- a/cmd/otelcol/config/collector/otlp_config_linux.yaml +++ b/cmd/otelcol/config/collector/otlp_config_linux.yaml @@ -70,7 +70,7 @@ processors: # Detector order is important: the `system` detector goes last so it can't preclude cloud detectors from setting host/os info. # https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/resourcedetectionprocessor#ordering resourcedetection/internal: - detectors: [gce, ecs, ec2, azure, system] + detectors: [gcp, ecs, ec2, azure, system] override: true exporters: diff --git a/cmd/otelcol/config/collector/upstream_agent_config.yaml b/cmd/otelcol/config/collector/upstream_agent_config.yaml index 0837c3c601..20a7eb88c2 100644 --- a/cmd/otelcol/config/collector/upstream_agent_config.yaml +++ b/cmd/otelcol/config/collector/upstream_agent_config.yaml @@ -100,7 +100,7 @@ processors: # libraries can send wrong values from container environments. # https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/resourcedetectionprocessor#ordering resourcedetection: - detectors: [gce, ecs, ec2, azure, system] + detectors: [gcp, ecs, ec2, azure, system] override: true # Optional: The following processor can be used to add a default "deployment.environment" attribute to the logs and diff --git a/cmd/translatesfx/README.md b/cmd/translatesfx/README.md index f01fc596a2..b32ef376ff 100644 --- a/cmd/translatesfx/README.md +++ b/cmd/translatesfx/README.md @@ -396,7 +396,7 @@ by the `signalfx` exporter to generate cloud resource IDs. resourcedetection: detectors: - env - - gce + - gcp - ecs - ec2 - azure diff --git a/cmd/translatesfx/translatesfx/otel.go b/cmd/translatesfx/translatesfx/otel.go index f04c5f6b70..a928672ab1 100644 --- a/cmd/translatesfx/translatesfx/otel.go +++ b/cmd/translatesfx/translatesfx/otel.go @@ -63,7 +63,7 @@ func newOtelCfg() *otelCfg { Receivers: map[string]map[string]any{}, Processors: map[string]map[string]any{ resourceDetection: { - "detectors": []string{"system", "env", "gce", "ecs", "ec2", "azure"}, + "detectors": []string{"system", "env", "gcp", "ecs", "ec2", "azure"}, }, }, Extensions: map[string]map[string]any{}, diff --git a/cmd/translatesfx/translatesfx/otel_test.go b/cmd/translatesfx/translatesfx/otel_test.go index fa3b61ca82..918d85baae 100644 --- a/cmd/translatesfx/translatesfx/otel_test.go +++ b/cmd/translatesfx/translatesfx/otel_test.go @@ -159,7 +159,7 @@ func TestInfoToOtelConfig_ResourceDetectionProcessor(t *testing.T) { assert.NotNil(t, oc.Processors) rdProc := oc.Processors["resourcedetection"] assert.Equal(t, map[string]any{ - "detectors": []string{"system", "env", "gce", "ecs", "ec2", "azure"}, + "detectors": []string{"system", "env", "gcp", "ecs", "ec2", "azure"}, }, rdProc) assert.Equal(t, []string{"resourcedetection"}, oc.Service.Pipelines["metrics"].Processors) } diff --git a/cmd/translatesfx/translatesfx/testdata/otel-e2e-expected.yaml b/cmd/translatesfx/translatesfx/testdata/otel-e2e-expected.yaml index cee58e583b..5608882494 100644 --- a/cmd/translatesfx/translatesfx/testdata/otel-e2e-expected.yaml +++ b/cmd/translatesfx/translatesfx/testdata/otel-e2e-expected.yaml @@ -56,7 +56,7 @@ processors: detectors: - system - env - - gce + - gcp - ecs - ec2 - azure diff --git a/docs/apm-infra-correlation.md b/docs/apm-infra-correlation.md index 39cdaa8452..e2b7f291b9 100644 --- a/docs/apm-infra-correlation.md +++ b/docs/apm-infra-correlation.md @@ -67,7 +67,7 @@ receivers: ... processors: resourcedetection: - detectors: [system,env,gce,ec2] + detectors: [system,env,gcp,ec2] override: true resource/add_environment: attributes: @@ -164,7 +164,7 @@ receivers: ... processors: resourcedetection: - detectors: [system,env,gce,ec2] + detectors: [system,env,gcp,ec2] override: true resource/add_environment: attributes: diff --git a/internal/configconverter/README.md b/internal/configconverter/README.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/internal/configconverter/normalize_gcp.go b/internal/configconverter/normalize_gcp.go new file mode 100644 index 0000000000..662f79e652 --- /dev/null +++ b/internal/configconverter/normalize_gcp.go @@ -0,0 +1,74 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package configconverter + +import ( + "context" + "log" + "regexp" + + "go.opentelemetry.io/collector/confmap" +) + +type NormalizeGcp struct{} + +func (NormalizeGcp) Convert(_ context.Context, in *confmap.Conf) error { + if in == nil { + return nil + } + + const resourceDetector = "processors::resourcedetection(?P/(?:[^:]|:[^:])+)?::detectors" + resourceDetectorRE := regexp.MustCompile(resourceDetector) + out := map[string]any{} + nonNormalizedGcpDetectorFound := false + + for _, k := range in.AllKeys() { + v := in.Get(k) + match := resourceDetectorRE.FindStringSubmatch(k) + if match != nil { + if vArr, ok := v.([]interface{}); ok { + normalizedV := make([]interface{}, 0, len(vArr)) + found := false + for _, item := range vArr { + switch item.(type) { + case string: + if item == "gce" || item == "gke" { + if !found { + normalizedV = append(normalizedV, "gcp") + } + found = true + nonNormalizedGcpDetectorFound = true + } else if item != nil { + normalizedV = append(normalizedV, item) + } + default: + if item != nil { + normalizedV = append(normalizedV, item) + } + } + } + out[k] = normalizedV + } + } + } + if nonNormalizedGcpDetectorFound { + log.Println("[WARNING] `processors` -> `resourcedetection` -> `detectors` parameter " + + "contains a deprecated configuration. Please update the config according to the guideline: " + + "https://github.com/signalfx/splunk-otel-collector#from-0680-to-0690.") + } + + in.Merge(confmap.NewFromStringMap(out)) + return nil +} diff --git a/internal/configconverter/normalize_gcp_test.go b/internal/configconverter/normalize_gcp_test.go new file mode 100644 index 0000000000..9e3ed635ee --- /dev/null +++ b/internal/configconverter/normalize_gcp_test.go @@ -0,0 +1,113 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package configconverter + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/confmap/confmaptest" +) + +func TestNormalizeGcp(t *testing.T) { + expectedCfgMap, err := confmaptest.LoadConf("testdata/normalize_gcp/upstream_agent_config_post_migration.yaml") + require.NotNil(t, expectedCfgMap) + require.NoError(t, err) + + cfgMap, err := confmaptest.LoadConf("testdata/normalize_gcp/upstream_agent_config.yaml") + require.NoError(t, err) + require.NotNil(t, cfgMap) + + err = NormalizeGcp{}.Convert(context.Background(), cfgMap) + require.NoError(t, err) + + assert.Equal(t, expectedCfgMap, cfgMap) +} + +func TestNormalizeGcpMany(t *testing.T) { + expectedCfgMap, err := confmaptest.LoadConf("testdata/normalize_gcp/upstream_agent_config_post_migration.yaml") + require.NotNil(t, expectedCfgMap) + require.NoError(t, err) + + cfgMap, err := confmaptest.LoadConf("testdata/normalize_gcp/upstream_agent_config_many.yaml") + require.NoError(t, err) + require.NotNil(t, cfgMap) + + err = NormalizeGcp{}.Convert(context.Background(), cfgMap) + require.NoError(t, err) + + assert.Equal(t, expectedCfgMap, cfgMap) +} + +func TestNormalizeGcpSame(t *testing.T) { + expectedCfgMap, err := confmaptest.LoadConf("testdata/normalize_gcp/upstream_agent_config_post_migration.yaml") + require.NotNil(t, expectedCfgMap) + require.NoError(t, err) + + cfgMap, err := confmaptest.LoadConf("testdata/normalize_gcp/upstream_agent_config_post_migration.yaml") + require.NoError(t, err) + require.NotNil(t, cfgMap) + + err = NormalizeGcp{}.Convert(context.Background(), cfgMap) + require.NoError(t, err) + + assert.Equal(t, expectedCfgMap, cfgMap) +} + +func TestNormalizeGcpNoop(t *testing.T) { + expectedCfgMap, err := confmaptest.LoadConf("testdata/normalize_gcp/upstream_agent_config_no_op.yaml") + require.NotNil(t, expectedCfgMap) + require.NoError(t, err) + + cfgMap, err := confmaptest.LoadConf("testdata/normalize_gcp/upstream_agent_config_no_op.yaml") + require.NoError(t, err) + require.NotNil(t, cfgMap) + + err = NormalizeGcp{}.Convert(context.Background(), cfgMap) + require.NoError(t, err) + + assert.Equal(t, expectedCfgMap, cfgMap) +} + +func TestNormalizeGcpSubresources(t *testing.T) { + expectedCfgMap, err := confmaptest.LoadConf("testdata/normalize_gcp/upstream_agent_config_subresources_post_migration.yaml") + require.NotNil(t, expectedCfgMap) + require.NoError(t, err) + + cfgMap, err := confmaptest.LoadConf("testdata/normalize_gcp/upstream_agent_config_subresources.yaml") + require.NoError(t, err) + require.NotNil(t, cfgMap) + + err = NormalizeGcp{}.Convert(context.Background(), cfgMap) + require.NoError(t, err) + + assert.Equal(t, expectedCfgMap, cfgMap) +} diff --git a/internal/configconverter/testdata/normalize_gcp/upstream_agent_config.yaml b/internal/configconverter/testdata/normalize_gcp/upstream_agent_config.yaml new file mode 100644 index 0000000000..e6976c625e --- /dev/null +++ b/internal/configconverter/testdata/normalize_gcp/upstream_agent_config.yaml @@ -0,0 +1,9 @@ +processors: + # Detect if the collector is running on a cloud system, which is important for creating unique cloud provider dimensions. + # Detector order is important: the `system` detector goes last so it can't preclude cloud detectors from setting host/os info. + # Resource detection processor is configured to override all host and cloud attributes because instrumentation + # libraries can send wrong values from container environments. + # https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/resourcedetectionprocessor#ordering + resourcedetection: + detectors: [gce, ecs, ec2, azure, system] + override: true \ No newline at end of file diff --git a/internal/configconverter/testdata/normalize_gcp/upstream_agent_config_many.yaml b/internal/configconverter/testdata/normalize_gcp/upstream_agent_config_many.yaml new file mode 100644 index 0000000000..0fcfd85f6a --- /dev/null +++ b/internal/configconverter/testdata/normalize_gcp/upstream_agent_config_many.yaml @@ -0,0 +1,4 @@ +processors: + resourcedetection: + detectors: [gce, ecs, gke, ec2, azure, system] + override: true \ No newline at end of file diff --git a/internal/configconverter/testdata/normalize_gcp/upstream_agent_config_no_op.yaml b/internal/configconverter/testdata/normalize_gcp/upstream_agent_config_no_op.yaml new file mode 100644 index 0000000000..745a8a4325 --- /dev/null +++ b/internal/configconverter/testdata/normalize_gcp/upstream_agent_config_no_op.yaml @@ -0,0 +1,4 @@ +processors: + resourcedetection: + detectors: [ecs, ec2, azure, system] + override: true \ No newline at end of file diff --git a/internal/configconverter/testdata/normalize_gcp/upstream_agent_config_post_migration.yaml b/internal/configconverter/testdata/normalize_gcp/upstream_agent_config_post_migration.yaml new file mode 100644 index 0000000000..23d2be6717 --- /dev/null +++ b/internal/configconverter/testdata/normalize_gcp/upstream_agent_config_post_migration.yaml @@ -0,0 +1,4 @@ +processors: + resourcedetection: + detectors: [gcp, ecs, ec2, azure, system] + override: true \ No newline at end of file diff --git a/internal/configconverter/testdata/normalize_gcp/upstream_agent_config_subresources.yaml b/internal/configconverter/testdata/normalize_gcp/upstream_agent_config_subresources.yaml new file mode 100644 index 0000000000..dc6c4d6526 --- /dev/null +++ b/internal/configconverter/testdata/normalize_gcp/upstream_agent_config_subresources.yaml @@ -0,0 +1,9 @@ +processors: + resourcedetection: + detectors: [gce, ecs, gke, ec2, azure, system] + resourcedetection/1: + detectors: [gce, ecs, gke, ec2, azure, system] + resourcedetection/abc: + detectors: [gce, ecs, gke, ec2, azure, system] + resourcedetection/abc123_+!@#%^&*(): + detectors: [gce, ecs, gke, ec2, azure, system] diff --git a/internal/configconverter/testdata/normalize_gcp/upstream_agent_config_subresources_post_migration.yaml b/internal/configconverter/testdata/normalize_gcp/upstream_agent_config_subresources_post_migration.yaml new file mode 100644 index 0000000000..6fb1c58c06 --- /dev/null +++ b/internal/configconverter/testdata/normalize_gcp/upstream_agent_config_subresources_post_migration.yaml @@ -0,0 +1,9 @@ +processors: + resourcedetection: + detectors: [gcp, ecs, ec2, azure, system] + resourcedetection/1: + detectors: [gcp, ecs, ec2, azure, system] + resourcedetection/abc: + detectors: [gcp, ecs, ec2, azure, system] + resourcedetection/abc123_+!@#%^&*(): + detectors: [gcp, ecs, ec2, azure, system] diff --git a/internal/settings/settings.go b/internal/settings/settings.go index ccbf1edcc9..ed7a2de71b 100644 --- a/internal/settings/settings.go +++ b/internal/settings/settings.go @@ -147,6 +147,7 @@ func (s *Settings) ConfMapConverters() []confmap.Converter { configconverter.MoveOTLPInsecureKey{}, configconverter.MoveHecTLS{}, configconverter.RenameK8sTagger{}, + configconverter.NormalizeGcp{}, ) } return confMapConverters diff --git a/internal/settings/settings_test.go b/internal/settings/settings_test.go index a5d139cf1e..df2bb3933b 100644 --- a/internal/settings/settings_test.go +++ b/internal/settings/settings_test.go @@ -133,6 +133,7 @@ func TestNewSettingsConvertConfig(t *testing.T) { configconverter.MoveOTLPInsecureKey{}, configconverter.MoveHecTLS{}, configconverter.RenameK8sTagger{}, + configconverter.NormalizeGcp{}, }, settings.ConfMapConverters()) require.Equal(t, []string{"--feature-gates", "foo", "--feature-gates", "-bar"}, settings.ColCoreArgs()) }