From 20ba66449b06198459f4a20e424838839af4be7a Mon Sep 17 00:00:00 2001 From: Itiel Olenick Date: Thu, 11 Jul 2024 10:56:55 +0300 Subject: [PATCH 1/5] Configuring httpSD http client from TAs clienthttp, added changelog and tests --- ...tpSD-client-config-from-TA-confighttp.yaml | 28 +++++++ .../prometheusreceiver/metrics_receiver.go | 84 ++++++++++++++++++- .../metrics_receiver_target_allocator_test.go | 53 ++++++++++++ 3 files changed, 164 insertions(+), 1 deletion(-) create mode 100644 .chloggen/httpSD-client-config-from-TA-confighttp.yaml diff --git a/.chloggen/httpSD-client-config-from-TA-confighttp.yaml b/.chloggen/httpSD-client-config-from-TA-confighttp.yaml new file mode 100644 index 000000000000..dd247004c8c0 --- /dev/null +++ b/.chloggen/httpSD-client-config-from-TA-confighttp.yaml @@ -0,0 +1,28 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: prometheusreceiver + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Use confighttp for httpSD client config + + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [33370] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [] diff --git a/receiver/prometheusreceiver/metrics_receiver.go b/receiver/prometheusreceiver/metrics_receiver.go index 65dbd580ea11..a8205bef5519 100644 --- a/receiver/prometheusreceiver/metrics_receiver.go +++ b/receiver/prometheusreceiver/metrics_receiver.go @@ -6,6 +6,7 @@ package prometheusreceiver // import "github.com/open-telemetry/opentelemetry-co import ( "bytes" "context" + "encoding/base64" "errors" "fmt" "io" @@ -14,6 +15,7 @@ import ( "os" "reflect" "regexp" + "strings" "sync" "time" "unsafe" @@ -145,6 +147,80 @@ func (r *pReceiver) startTargetAllocator(allocConf *TargetAllocator, baseCfg *Pr return nil } +// ConvertTLSVersion converts a string TLS version to the corresponding config.TLSVersion value in prometheus common. +func convertTLSVersion(version string) (commonconfig.TLSVersion, error) { + normalizedVersion := "TLS" + strings.ReplaceAll(version, ".", "") + + if tlsVersion, exists := commonconfig.TLSVersions[normalizedVersion]; exists { + return tlsVersion, nil + } + return 0, fmt.Errorf("unsupported TLS version: %s", version) +} + +// configureSDHTTPClientConfig configures the http client for the service discovery manager +// based on the provided TargetAllocator configuration. +func configureSDHTTPClientConfigFromTA(httpSD *promHTTP.SDConfig, allocConf *TargetAllocator) error { + httpSD.HTTPClientConfig.FollowRedirects = false + + httpSD.HTTPClientConfig.TLSConfig = commonconfig.TLSConfig{ + InsecureSkipVerify: allocConf.TLSSetting.InsecureSkipVerify, + ServerName: allocConf.TLSSetting.ServerName, + CAFile: allocConf.TLSSetting.CAFile, + CertFile: allocConf.TLSSetting.CertFile, + KeyFile: allocConf.TLSSetting.KeyFile, + } + + if allocConf.TLSSetting.CAPem != "" { + decodedCA, err := base64.StdEncoding.DecodeString(string(allocConf.TLSSetting.CAPem)) + if err != nil { + return fmt.Errorf("failed to decode CA: %w", err) + } + httpSD.HTTPClientConfig.TLSConfig.CA = string(decodedCA) + } + + if allocConf.TLSSetting.CertPem != "" { + decodedCert, err := base64.StdEncoding.DecodeString(string(allocConf.TLSSetting.CertPem)) + if err != nil { + return fmt.Errorf("failed to decode Cert: %w", err) + } + httpSD.HTTPClientConfig.TLSConfig.Cert = string(decodedCert) + } + + if allocConf.TLSSetting.KeyPem != "" { + decodedKey, err := base64.StdEncoding.DecodeString(string(allocConf.TLSSetting.KeyPem)) + if err != nil { + return fmt.Errorf("failed to decode Key: %w", err) + } + httpSD.HTTPClientConfig.TLSConfig.Key = commonconfig.Secret(decodedKey) + } + + if allocConf.TLSSetting.MinVersion != "" { + minVersion, err := convertTLSVersion(allocConf.TLSSetting.MinVersion) + if err != nil { + return err + } + httpSD.HTTPClientConfig.TLSConfig.MinVersion = minVersion + } + + if allocConf.TLSSetting.MaxVersion != "" { + maxVersion, err := convertTLSVersion(allocConf.TLSSetting.MaxVersion) + if err != nil { + return err + } + httpSD.HTTPClientConfig.TLSConfig.MaxVersion = maxVersion + } + + if allocConf.ProxyURL != "" { + proxyURL, err := url.Parse(allocConf.ProxyURL) + if err != nil { + return err + } + httpSD.HTTPClientConfig.ProxyURL = commonconfig.URL{URL: proxyURL} + } + + return nil +} + // syncTargetAllocator request jobs from targetAllocator and update underlying receiver, if the response does not match the provided compareHash. // baseDiscoveryCfg can be used to provide additional ScrapeConfigs which will be added to the retrieved jobs. func (r *pReceiver) syncTargetAllocator(compareHash uint64, allocConf *TargetAllocator, baseCfg *PromConfig) (uint64, error) { @@ -179,7 +255,13 @@ func (r *pReceiver) syncTargetAllocator(compareHash uint64, allocConf *TargetAll } escapedJob := url.QueryEscape(jobName) httpSD.URL = fmt.Sprintf("%s/jobs/%s/targets?collector_id=%s", allocConf.Endpoint, escapedJob, allocConf.CollectorID) - httpSD.HTTPClientConfig.FollowRedirects = false + + err = configureSDHTTPClientConfigFromTA(&httpSD, allocConf) + if err != nil { + r.settings.Logger.Error("Failed to configure http client config", zap.Error(err)) + return 0, err + } + scrapeConfig.ServiceDiscoveryConfigs = discovery.Configs{ &httpSD, } diff --git a/receiver/prometheusreceiver/metrics_receiver_target_allocator_test.go b/receiver/prometheusreceiver/metrics_receiver_target_allocator_test.go index 69e3c1a7b061..63320d2f6c83 100644 --- a/receiver/prometheusreceiver/metrics_receiver_target_allocator_test.go +++ b/receiver/prometheusreceiver/metrics_receiver_target_allocator_test.go @@ -7,9 +7,11 @@ package prometheusreceiver import ( "context" + "encoding/base64" "encoding/json" "net/http" "net/http/httptest" + "net/url" "strings" "sync" "sync/atomic" @@ -20,8 +22,11 @@ import ( "github.com/prometheus/common/model" promconfig "github.com/prometheus/prometheus/config" promHTTP "github.com/prometheus/prometheus/discovery/http" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/component/componenttest" + "go.opentelemetry.io/collector/config/configopaque" + "go.opentelemetry.io/collector/config/configtls" "go.opentelemetry.io/collector/consumer/consumertest" "go.opentelemetry.io/collector/receiver/receivertest" ) @@ -540,3 +545,51 @@ func TestTargetAllocatorJobRetrieval(t *testing.T) { }) } } + +func TestConfigureSDHTTPClientConfigFromTA(t *testing.T) { + ta := &TargetAllocator{} + ta.TLSSetting = configtls.ClientConfig{ + InsecureSkipVerify: true, + ServerName: "test.server", + Config: configtls.Config{ + CAFile: "/path/to/ca", + CertFile: "/path/to/cert", + KeyFile: "/path/to/key", + CAPem: configopaque.String(base64.StdEncoding.EncodeToString([]byte("test-ca"))), + CertPem: configopaque.String(base64.StdEncoding.EncodeToString([]byte("test-cert"))), + KeyPem: configopaque.String(base64.StdEncoding.EncodeToString([]byte("test-key"))), + MinVersion: "1.2", + MaxVersion: "1.3", + }, + } + ta.ProxyURL = "http://proxy.test" + + httpSD := &promHTTP.SDConfig{RefreshInterval: model.Duration(30 * time.Second)} + + err := configureSDHTTPClientConfigFromTA(httpSD, ta) + + assert.NoError(t, err) + + assert.Equal(t, false, httpSD.HTTPClientConfig.FollowRedirects) + assert.Equal(t, true, httpSD.HTTPClientConfig.TLSConfig.InsecureSkipVerify) + assert.Equal(t, "test.server", httpSD.HTTPClientConfig.TLSConfig.ServerName) + assert.Equal(t, "/path/to/ca", httpSD.HTTPClientConfig.TLSConfig.CAFile) + assert.Equal(t, "/path/to/cert", httpSD.HTTPClientConfig.TLSConfig.CertFile) + assert.Equal(t, "/path/to/key", httpSD.HTTPClientConfig.TLSConfig.KeyFile) + assert.Equal(t, "test-ca", httpSD.HTTPClientConfig.TLSConfig.CA) + assert.Equal(t, "test-cert", httpSD.HTTPClientConfig.TLSConfig.Cert) + assert.Equal(t, commonconfig.Secret("test-key"), httpSD.HTTPClientConfig.TLSConfig.Key) + assert.Equal(t, commonconfig.TLSVersions["TLS12"], httpSD.HTTPClientConfig.TLSConfig.MinVersion) + assert.Equal(t, commonconfig.TLSVersions["TLS13"], httpSD.HTTPClientConfig.TLSConfig.MaxVersion) + + parsedProxyURL, _ := url.Parse("http://proxy.test") + assert.Equal(t, commonconfig.URL{URL: parsedProxyURL}, httpSD.HTTPClientConfig.ProxyURL) + + // Test case with empty TargetAllocator + emptyTA := &TargetAllocator{} + emptyHTTPSD := &promHTTP.SDConfig{RefreshInterval: model.Duration(30 * time.Second)} + + err = configureSDHTTPClientConfigFromTA(emptyHTTPSD, emptyTA) + + assert.NoError(t, err) +} From dd323f22f2af78d9120d82bf7d5ef6f9678df4ba Mon Sep 17 00:00:00 2001 From: Itiel Olenick Date: Thu, 8 Aug 2024 10:04:24 +0300 Subject: [PATCH 2/5] Ran make modtidy --- receiver/prometheusreceiver/go.mod | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/receiver/prometheusreceiver/go.mod b/receiver/prometheusreceiver/go.mod index 785210788935..1947b566629b 100644 --- a/receiver/prometheusreceiver/go.mod +++ b/receiver/prometheusreceiver/go.mod @@ -16,6 +16,8 @@ require ( github.com/stretchr/testify v1.9.0 go.opentelemetry.io/collector/component v0.106.1 go.opentelemetry.io/collector/config/confighttp v0.106.1 + go.opentelemetry.io/collector/config/configopaque v1.12.0 + go.opentelemetry.io/collector/config/configtls v1.12.0 go.opentelemetry.io/collector/confmap v0.106.1 go.opentelemetry.io/collector/confmap/provider/fileprovider v0.106.1 go.opentelemetry.io/collector/consumer v0.106.1 @@ -166,10 +168,8 @@ require ( go.opentelemetry.io/collector/client v0.106.1 // indirect go.opentelemetry.io/collector/config/configauth v0.106.1 // indirect go.opentelemetry.io/collector/config/configcompression v1.12.0 // indirect - go.opentelemetry.io/collector/config/configopaque v1.12.0 // indirect go.opentelemetry.io/collector/config/configretry v1.12.0 // indirect go.opentelemetry.io/collector/config/configtelemetry v0.106.1 // indirect - go.opentelemetry.io/collector/config/configtls v1.12.0 // indirect go.opentelemetry.io/collector/config/internal v0.106.1 // indirect go.opentelemetry.io/collector/connector v0.106.1 // indirect go.opentelemetry.io/collector/consumer/consumerprofiles v0.106.1 // indirect From dc0fba12f4552351e2ea469c88739fabd4af8284 Mon Sep 17 00:00:00 2001 From: Itiel Olenick Date: Sun, 11 Aug 2024 08:44:53 +0300 Subject: [PATCH 3/5] Ran make modtidy --- receiver/prometheusreceiver/go.mod | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/receiver/prometheusreceiver/go.mod b/receiver/prometheusreceiver/go.mod index eb51915ebaac..ff3425968b37 100644 --- a/receiver/prometheusreceiver/go.mod +++ b/receiver/prometheusreceiver/go.mod @@ -14,24 +14,10 @@ require ( github.com/prometheus/common v0.55.0 github.com/prometheus/prometheus v0.53.1 github.com/stretchr/testify v1.9.0 - go.opentelemetry.io/collector/component v0.106.1 - go.opentelemetry.io/collector/config/confighttp v0.106.1 - go.opentelemetry.io/collector/config/configopaque v1.12.0 - go.opentelemetry.io/collector/config/configtls v1.12.0 - go.opentelemetry.io/collector/confmap v0.106.1 - go.opentelemetry.io/collector/confmap/provider/fileprovider v0.106.1 - go.opentelemetry.io/collector/consumer v0.106.1 - go.opentelemetry.io/collector/consumer/consumertest v0.106.1 - go.opentelemetry.io/collector/exporter v0.106.1 - go.opentelemetry.io/collector/featuregate v1.12.0 - go.opentelemetry.io/collector/otelcol v0.106.1 - go.opentelemetry.io/collector/pdata v1.12.0 - go.opentelemetry.io/collector/processor v0.106.1 - go.opentelemetry.io/collector/processor/batchprocessor v0.106.1 - go.opentelemetry.io/collector/receiver v0.106.1 - go.opentelemetry.io/collector/semconv v0.106.1 go.opentelemetry.io/collector/component v0.106.2-0.20240809191011-ef07ea073562 go.opentelemetry.io/collector/config/confighttp v0.106.2-0.20240809191011-ef07ea073562 + go.opentelemetry.io/collector/config/configopaque v1.12.1-0.20240809191011-ef07ea073562 + go.opentelemetry.io/collector/config/configtls v1.12.1-0.20240809191011-ef07ea073562 go.opentelemetry.io/collector/confmap v0.106.2-0.20240809191011-ef07ea073562 go.opentelemetry.io/collector/confmap/provider/fileprovider v0.106.2-0.20240809191011-ef07ea073562 go.opentelemetry.io/collector/consumer v0.106.2-0.20240809191011-ef07ea073562 @@ -183,10 +169,8 @@ require ( go.opentelemetry.io/collector/component/componentprofiles v0.106.2-0.20240809191011-ef07ea073562 // indirect go.opentelemetry.io/collector/config/configauth v0.106.2-0.20240809191011-ef07ea073562 // indirect go.opentelemetry.io/collector/config/configcompression v1.12.1-0.20240809191011-ef07ea073562 // indirect - go.opentelemetry.io/collector/config/configopaque v1.12.1-0.20240809191011-ef07ea073562 // indirect go.opentelemetry.io/collector/config/configretry v1.12.1-0.20240809191011-ef07ea073562 // indirect go.opentelemetry.io/collector/config/configtelemetry v0.106.2-0.20240809191011-ef07ea073562 // indirect - go.opentelemetry.io/collector/config/configtls v1.12.1-0.20240809191011-ef07ea073562 // indirect go.opentelemetry.io/collector/config/internal v0.106.2-0.20240809191011-ef07ea073562 // indirect go.opentelemetry.io/collector/connector v0.106.2-0.20240809191011-ef07ea073562 // indirect go.opentelemetry.io/collector/consumer/consumerprofiles v0.106.2-0.20240809191011-ef07ea073562 // indirect From d1509120fe0e47e4bbbe3ade44963e453c0ace4a Mon Sep 17 00:00:00 2001 From: ItielOlenick <67790309+ItielOlenick@users.noreply.github.com> Date: Sun, 18 Aug 2024 17:04:15 +0300 Subject: [PATCH 4/5] Update .chloggen/httpSD-client-config-from-TA-confighttp.yaml --- .chloggen/httpSD-client-config-from-TA-confighttp.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.chloggen/httpSD-client-config-from-TA-confighttp.yaml b/.chloggen/httpSD-client-config-from-TA-confighttp.yaml index dd247004c8c0..3c7c3b0b526e 100644 --- a/.chloggen/httpSD-client-config-from-TA-confighttp.yaml +++ b/.chloggen/httpSD-client-config-from-TA-confighttp.yaml @@ -7,7 +7,7 @@ change_type: enhancement component: prometheusreceiver # A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). -note: Use confighttp for httpSD client config +note: Ensure Target Allocator's confighttp is used in the receiver's service discovery # Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. From 2727a211df05fe5e149e30f106c5009caa7fb090 Mon Sep 17 00:00:00 2001 From: Itiel Date: Sun, 18 Aug 2024 18:09:36 +0300 Subject: [PATCH 5/5] Ran make gotidy --- receiver/prometheusreceiver/go.mod | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/receiver/prometheusreceiver/go.mod b/receiver/prometheusreceiver/go.mod index 31031a40d090..d8b3e8d3d93c 100644 --- a/receiver/prometheusreceiver/go.mod +++ b/receiver/prometheusreceiver/go.mod @@ -17,8 +17,8 @@ require ( go.opentelemetry.io/collector/component v0.107.1-0.20240816132030-9fd84668bb02 go.opentelemetry.io/collector/component/componentstatus v0.107.1-0.20240816132030-9fd84668bb02 go.opentelemetry.io/collector/config/confighttp v0.107.1-0.20240816132030-9fd84668bb02 - go.opentelemetry.io/collector/config/configopaque v1.13.0 - go.opentelemetry.io/collector/config/configtls v1.13.0 + go.opentelemetry.io/collector/config/configopaque v1.13.1-0.20240816132030-9fd84668bb02 + go.opentelemetry.io/collector/config/configtls v1.13.1-0.20240816132030-9fd84668bb02 go.opentelemetry.io/collector/confmap v0.107.1-0.20240816132030-9fd84668bb02 go.opentelemetry.io/collector/confmap/provider/fileprovider v0.107.1-0.20240816132030-9fd84668bb02 go.opentelemetry.io/collector/consumer v0.107.1-0.20240816132030-9fd84668bb02 @@ -170,10 +170,8 @@ require ( go.opentelemetry.io/collector/component/componentprofiles v0.107.1-0.20240816132030-9fd84668bb02 // indirect go.opentelemetry.io/collector/config/configauth v0.107.1-0.20240816132030-9fd84668bb02 // indirect go.opentelemetry.io/collector/config/configcompression v1.13.1-0.20240816132030-9fd84668bb02 // indirect - go.opentelemetry.io/collector/config/configopaque v1.13.1-0.20240816132030-9fd84668bb02 // indirect go.opentelemetry.io/collector/config/configretry v1.13.1-0.20240816132030-9fd84668bb02 // indirect go.opentelemetry.io/collector/config/configtelemetry v0.107.1-0.20240816132030-9fd84668bb02 // indirect - go.opentelemetry.io/collector/config/configtls v1.13.1-0.20240816132030-9fd84668bb02 // indirect go.opentelemetry.io/collector/config/internal v0.107.1-0.20240816132030-9fd84668bb02 // indirect go.opentelemetry.io/collector/connector v0.107.1-0.20240816132030-9fd84668bb02 // indirect go.opentelemetry.io/collector/consumer/consumerprofiles v0.107.1-0.20240816132030-9fd84668bb02 // indirect