confgenerator: allow relabelling project_id (#40)

* confgenerator: allow relabelling project_id This change does the following: - Removes validation that prevents the project_id being relabelled - Moves the project_id to the `gcp.project.id` resource attribute so the exporter can set it correctly in the monitored resource label - Adds a confgenerator test - Fix broken prometheus tests This is being tested by running the sidecar in one project and sending the metrics to another using a relabel rule. Change-Id: I930e8c985c48491f6debb8181432b3b5c3646c05 * integration_test: simplify getAvailablePort Change-Id: Ide2fa4d33f5557be3eeadce3d78945a3bc28f9e7
GoogleCloudPlatform · Aug 9, 2024 · f330dd7 · f330dd7
1 parent b10a57f
commit f330dd7
Show file tree

Hide file tree

Showing 18 changed files with 338 additions and 75 deletions.
diff --git a/collector/exporter/googlemanagedprometheusexporter/generated_component_test.go b/collector/exporter/googlemanagedprometheusexporter/generated_component_test.go
diff --git a/collector/exporter/googlemanagedprometheusexporter/generated_package_test.go b/collector/exporter/googlemanagedprometheusexporter/generated_package_test.go
diff --git a/collector/exporter/googlemanagedprometheusexporter/metadata.yaml b/collector/exporter/googlemanagedprometheusexporter/metadata.yaml
@@ -6,3 +6,11 @@ status:
   stability:
     beta: [metrics]
   distributions: [contrib, observiq]
+
+tests:
+  skip_lifecycle: true
+  goleak:
+    ignore:
+      top:
+        # See https://github.com/census-instrumentation/opencensus-go/issues/1191 for more information.
+        - "go.opencensus.io/stats/view.(*worker).start"
diff --git a/collector/receiver/prometheusreceiver/config_test.go b/collector/receiver/prometheusreceiver/config_test.go
@@ -26,6 +26,8 @@ import (
 	"github.com/stretchr/testify/require"
 	"go.opentelemetry.io/collector/component"
 	"go.opentelemetry.io/collector/confmap/confmaptest"
+
+	"github.com/GoogleCloudPlatform/run-gmp-sidecar/collector/receiver/prometheusreceiver/internal/metadata"
 )
 
 func TestLoadConfig(t *testing.T) {
@@ -34,14 +36,14 @@ func TestLoadConfig(t *testing.T) {
 	factory := NewFactory()
 	cfg := factory.CreateDefaultConfig()
 
-	sub, err := cm.Sub(component.NewIDWithName(typeStr, "").String())
+	sub, err := cm.Sub(component.NewIDWithName(metadata.Type, "").String())
 	require.NoError(t, err)
 	require.NoError(t, component.UnmarshalConfig(sub, cfg))
 
 	r0 := cfg.(*Config)
 	assert.Equal(t, r0, factory.CreateDefaultConfig())
 
-	sub, err = cm.Sub(component.NewIDWithName(typeStr, "customname").String())
+	sub, err = cm.Sub(component.NewIDWithName(metadata.Type, "customname").String())
 	require.NoError(t, err)
 	require.NoError(t, component.UnmarshalConfig(sub, cfg))
 
@@ -66,7 +68,7 @@ func TestLoadTargetAllocatorConfig(t *testing.T) {
 	factory := NewFactory()
 	cfg := factory.CreateDefaultConfig()
 
-	sub, err := cm.Sub(component.NewIDWithName(typeStr, "").String())
+	sub, err := cm.Sub(component.NewIDWithName(metadata.Type, "").String())
 	require.NoError(t, err)
 	require.NoError(t, component.UnmarshalConfig(sub, cfg))
 
@@ -76,7 +78,7 @@ func TestLoadTargetAllocatorConfig(t *testing.T) {
 	assert.Equal(t, 30*time.Second, r0.TargetAllocator.Interval)
 	assert.Equal(t, "collector-1", r0.TargetAllocator.CollectorID)
 
-	sub, err = cm.Sub(component.NewIDWithName(typeStr, "withScrape").String())
+	sub, err = cm.Sub(component.NewIDWithName(metadata.Type, "withScrape").String())
 	require.NoError(t, err)
 	cfg = factory.CreateDefaultConfig()
 	require.NoError(t, component.UnmarshalConfig(sub, cfg))
@@ -91,7 +93,7 @@ func TestLoadTargetAllocatorConfig(t *testing.T) {
 	assert.Equal(t, "demo", r1.PrometheusConfig.ScrapeConfigs[0].JobName)
 	assert.Equal(t, promModel.Duration(5*time.Second), r1.PrometheusConfig.ScrapeConfigs[0].ScrapeInterval)
 
-	sub, err = cm.Sub(component.NewIDWithName(typeStr, "withOnlyScrape").String())
+	sub, err = cm.Sub(component.NewIDWithName(metadata.Type, "withOnlyScrape").String())
 	require.NoError(t, err)
 	cfg = factory.CreateDefaultConfig()
 	require.NoError(t, component.UnmarshalConfig(sub, cfg))
@@ -108,7 +110,7 @@ func TestLoadConfigFailsOnUnknownSection(t *testing.T) {
 	factory := NewFactory()
 	cfg := factory.CreateDefaultConfig()
 
-	sub, err := cm.Sub(component.NewIDWithName(typeStr, "").String())
+	sub, err := cm.Sub(component.NewIDWithName(metadata.Type, "").String())
 	require.NoError(t, err)
 	require.Error(t, component.UnmarshalConfig(sub, cfg))
 }
@@ -122,7 +124,7 @@ func TestLoadConfigFailsOnUnknownPrometheusSection(t *testing.T) {
 	factory := NewFactory()
 	cfg := factory.CreateDefaultConfig()
 
-	sub, err := cm.Sub(component.NewIDWithName(typeStr, "").String())
+	sub, err := cm.Sub(component.NewIDWithName(metadata.Type, "").String())
 	require.NoError(t, err)
 	require.Error(t, component.UnmarshalConfig(sub, cfg))
 }
@@ -134,7 +136,7 @@ func TestLoadConfigFailsOnRenameDisallowed(t *testing.T) {
 	factory := NewFactory()
 	cfg := factory.CreateDefaultConfig()
 
-	sub, err := cm.Sub(component.NewIDWithName(typeStr, "").String())
+	sub, err := cm.Sub(component.NewIDWithName(metadata.Type, "").String())
 	require.NoError(t, err)
 	require.NoError(t, component.UnmarshalConfig(sub, cfg))
 	assert.Error(t, component.ValidateConfig(cfg))
@@ -147,7 +149,7 @@ func TestRejectUnsupportedPrometheusFeatures(t *testing.T) {
 	factory := NewFactory()
 	cfg := factory.CreateDefaultConfig()
 
-	sub, err := cm.Sub(component.NewIDWithName(typeStr, "").String())
+	sub, err := cm.Sub(component.NewIDWithName(metadata.Type, "").String())
 	require.NoError(t, err)
 	require.NoError(t, component.UnmarshalConfig(sub, cfg))
 
@@ -172,7 +174,7 @@ func TestNonExistentAuthCredentialsFile(t *testing.T) {
 	factory := NewFactory()
 	cfg := factory.CreateDefaultConfig()
 
-	sub, err := cm.Sub(component.NewIDWithName(typeStr, "").String())
+	sub, err := cm.Sub(component.NewIDWithName(metadata.Type, "").String())
 	require.NoError(t, err)
 	require.NoError(t, component.UnmarshalConfig(sub, cfg))
 
@@ -191,7 +193,7 @@ func TestTLSConfigNonExistentCertFile(t *testing.T) {
 	factory := NewFactory()
 	cfg := factory.CreateDefaultConfig()
 
-	sub, err := cm.Sub(component.NewIDWithName(typeStr, "").String())
+	sub, err := cm.Sub(component.NewIDWithName(metadata.Type, "").String())
 	require.NoError(t, err)
 	require.NoError(t, component.UnmarshalConfig(sub, cfg))
 
@@ -210,7 +212,7 @@ func TestTLSConfigNonExistentKeyFile(t *testing.T) {
 	factory := NewFactory()
 	cfg := factory.CreateDefaultConfig()
 
-	sub, err := cm.Sub(component.NewIDWithName(typeStr, "").String())
+	sub, err := cm.Sub(component.NewIDWithName(metadata.Type, "").String())
 	require.NoError(t, err)
 	require.NoError(t, component.UnmarshalConfig(sub, cfg))
 
@@ -229,7 +231,7 @@ func TestTLSConfigCertFileWithoutKeyFile(t *testing.T) {
 	factory := NewFactory()
 	cfg := factory.CreateDefaultConfig()
 
-	sub, err := cm.Sub(component.NewIDWithName(typeStr, "").String())
+	sub, err := cm.Sub(component.NewIDWithName(metadata.Type, "").String())
 	require.NoError(t, err)
 	err = component.UnmarshalConfig(sub, cfg)
 	if assert.Error(t, err) {
@@ -243,7 +245,7 @@ func TestTLSConfigKeyFileWithoutCertFile(t *testing.T) {
 	factory := NewFactory()
 	cfg := factory.CreateDefaultConfig()
 
-	sub, err := cm.Sub(component.NewIDWithName(typeStr, "").String())
+	sub, err := cm.Sub(component.NewIDWithName(metadata.Type, "").String())
 	require.NoError(t, err)
 	err = component.UnmarshalConfig(sub, cfg)
 	if assert.Error(t, err) {
@@ -257,7 +259,7 @@ func TestKubernetesSDConfigWithoutKeyFile(t *testing.T) {
 	factory := NewFactory()
 	cfg := factory.CreateDefaultConfig()
 
-	sub, err := cm.Sub(component.NewIDWithName(typeStr, "").String())
+	sub, err := cm.Sub(component.NewIDWithName(metadata.Type, "").String())
 	require.NoError(t, err)
 	err = component.UnmarshalConfig(sub, cfg)
 	if assert.Error(t, err) {
@@ -271,7 +273,7 @@ func TestFileSDConfigJsonNilTargetGroup(t *testing.T) {
 	factory := NewFactory()
 	cfg := factory.CreateDefaultConfig()
 
-	sub, err := cm.Sub(component.NewIDWithName(typeStr, "").String())
+	sub, err := cm.Sub(component.NewIDWithName(metadata.Type, "").String())
 	require.NoError(t, err)
 	require.NoError(t, component.UnmarshalConfig(sub, cfg))
 
@@ -290,7 +292,7 @@ func TestFileSDConfigYamlNilTargetGroup(t *testing.T) {
 	factory := NewFactory()
 	cfg := factory.CreateDefaultConfig()
 
-	sub, err := cm.Sub(component.NewIDWithName(typeStr, "").String())
+	sub, err := cm.Sub(component.NewIDWithName(metadata.Type, "").String())
 	require.NoError(t, err)
 	require.NoError(t, component.UnmarshalConfig(sub, cfg))
 

diff --git a/collector/receiver/prometheusreceiver/generated_package_test.go b/collector/receiver/prometheusreceiver/generated_package_test.go
diff --git a/collector/receiver/prometheusreceiver/internal/staleness_end_to_end_test.go b/collector/receiver/prometheusreceiver/internal/staleness_end_to_end_test.go
@@ -158,19 +158,14 @@ service:
 		Processors: processors,
 	}
 
-	fmp := fileprovider.New()
-	configProvider, err := otelcol.NewConfigProvider(
-		otelcol.ConfigProviderSettings{
+	appSettings := otelcol.CollectorSettings{
+		Factories: func() (otelcol.Factories, error) { return factories, nil },
+		ConfigProviderSettings: otelcol.ConfigProviderSettings{
 			ResolverSettings: confmap.ResolverSettings{
-				URIs:      []string{confFile.Name()},
-				Providers: map[string]confmap.Provider{fmp.Scheme(): fmp},
+				URIs:              []string{confFile.Name()},
+				ProviderFactories: []confmap.ProviderFactory{fileprovider.NewFactory()},
 			},
-		})
-	require.NoError(t, err)
-
-	appSettings := otelcol.CollectorSettings{
-		Factories:      func() (otelcol.Factories, error) { return factories, nil },
-		ConfigProvider: configProvider,
+		},
 		BuildInfo: component.BuildInfo{
 			Command:     "otelcol",
 			Description: "OpenTelemetry Collector",

diff --git a/collector/receiver/prometheusreceiver/internal/transaction_test.go b/collector/receiver/prometheusreceiver/internal/transaction_test.go
@@ -264,7 +264,7 @@ func TestAppendExemplarWithEmptyLabelArray(t *testing.T) {
 
 func nopObsRecv(t *testing.T) *receiverhelper.ObsReport {
 	obsrecv, err := receiverhelper.NewObsReport(receiverhelper.ObsReportSettings{
-		ReceiverID:             component.NewID("prometheus"),
+		ReceiverID:             component.MustNewID("prometheus"),
 		Transport:              transport,
 		ReceiverCreateSettings: receivertest.NewNopCreateSettings(),
 	})

diff --git a/collector/receiver/prometheusreceiver/metadata.yaml b/collector/receiver/prometheusreceiver/metadata.yaml
@@ -5,4 +5,15 @@ status:
   class: receiver
   stability:
     beta: [metrics]
-  distributions: [core, contrib]
+  distributions: [core, contrib]
+
+tests:
+  config:
+    config:
+      scrape_configs:
+        - job_name: 'test'
+  goleak:
+    ignore:
+      top:
+        # See https://github.com/census-instrumentation/opencensus-go/issues/1191 for more information.
+        - "go.opencensus.io/stats/view.(*worker).start"
diff --git a/confgenerator/config.go b/confgenerator/config.go
@@ -217,6 +217,10 @@ func (rc *RunMonitoringConfig) OTelReceiverPipeline() (*otel.ReceiverPipeline, e
 	// Group by the GMP attributes.
 	processors = append(processors, otel.GroupByGMPAttrs())
 
+	// If the user updates the `project_id` label, we need to update the gcp.project.id resource attribute
+	// so the exporter can pick it up.
+	processors = append(processors, otel.TransformationMetrics(otel.GroupByAttribute("gcp.project.id", "project_id"), otel.DeleteMetricAttribute("project_id")))
+
 	return &otel.ReceiverPipeline{
 		Receiver: otel.Component{
 			Type: "prometheus",

diff --git a/confgenerator/otel/processors.go b/confgenerator/otel/processors.go
@@ -179,6 +179,16 @@ func FlattenResourceAttribute(resourceAttribute, metricAttribute string) Transfo
 	return TransformQuery(fmt.Sprintf(`set(attributes["%s"], resource.attributes["%s"])`, metricAttribute, resourceAttribute))
 }
 
+// GroupByAttribute returns an expression that makes a metric attribute into a resource attribute.
+func GroupByAttribute(resourceAttribute, metricAttribute string) TransformQuery {
+	return TransformQuery(fmt.Sprintf(`set(resource.attributes["%s"], attributes["%s"]) where attributes["%s"] != nil`, resourceAttribute, metricAttribute, metricAttribute))
+}
+
+// DeleteMetricAttribute returns an expression that removes the metric attribute specified.
+func DeleteMetricAttribute(metricAttribute string) TransformQuery {
+	return TransformQuery(fmt.Sprintf(`delete_key(attributes, "%s")`, metricAttribute))
+}
+
 // PrefixResourceAttribute prefixes the resource attribute with another resource
 // attribute.
 //

diff --git a/confgenerator/testdata/add-metadata-labels/golden/otel.yaml b/confgenerator/testdata/add-metadata-labels/golden/otel.yaml
@@ -78,6 +78,13 @@ processors:
       statements:
       - replace_pattern(resource.attributes["service.instance.id"], "^(\\d+)$$", Concat([resource.attributes["faas.id"],
         "$$1"], ":"))
+  transform/application-metrics_3:
+    metric_statements:
+      context: datapoint
+      statements:
+      - set(resource.attributes["gcp.project.id"], attributes["project_id"]) where
+        attributes["project_id"] != nil
+      - delete_key(attributes, "project_id")
   transform/run-gmp-self-metrics_3:
     metric_statements:
       context: datapoint
@@ -99,6 +106,7 @@ receivers:
         sample_limit: 1000
         follow_redirects: false
         enable_http2: false
+        http_headers: null
         relabel_configs:
         - source_labels: [__address__]
           target_label: service_name
@@ -148,6 +156,7 @@ service:
       - resourcedetection/application-metrics_0
       - transform/application-metrics_1
       - groupbyattrs/application-metrics_2
+      - transform/application-metrics_3
       receivers:
       - prometheus/application-metrics
     metrics/run-gmp-self-metrics:

diff --git a/confgenerator/testdata/builtin/golden/otel.yaml b/confgenerator/testdata/builtin/golden/otel.yaml
@@ -83,6 +83,13 @@ processors:
       context: datapoint
       statements:
       - set(attributes["instanceId"], resource.attributes["faas.id"])
+  transform/application-metrics_4:
+    metric_statements:
+      context: datapoint
+      statements:
+      - set(resource.attributes["gcp.project.id"], attributes["project_id"]) where
+        attributes["project_id"] != nil
+      - delete_key(attributes, "project_id")
   transform/run-gmp-self-metrics_3:
     metric_statements:
       context: datapoint
@@ -103,6 +110,7 @@ receivers:
         metrics_path: /metrics
         follow_redirects: false
         enable_http2: false
+        http_headers: null
         relabel_configs:
         - source_labels: [__address__]
           target_label: service_name
@@ -161,6 +169,7 @@ service:
       - transform/application-metrics_1
       - transform/application-metrics_2
       - groupbyattrs/application-metrics_3
+      - transform/application-metrics_4
       receivers:
       - prometheus/application-metrics
     metrics/run-gmp-self-metrics: