[prometheusreciever]: query TargetAllocator for HTTP_SD jobs

CHANGELOG.md

@@ -44,6 +44,7 @@
 - `attributesprocessor`: Support filter by severity (#9132)
 - `processor/transform`: Add transformation of logs (#9368)
 - `datadogexporter`: Add `metrics::summaries::mode` to specify export mode for summaries (#8846)
+- `prometheusreceiver`: Add ability to request HTTP_SD jobs from OpenTelemetryOperator TargetAllocator (#8055)
 
 ### 🧰 Bug fixes 🧰
 

cmd/configschema/go.mod

@@ -209,6 +209,7 @@ require (
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/mitchellh/hashstructure v1.1.0 // indirect
+	github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/moby/sys/mountinfo v0.5.0 // indirect

exporter/prometheusexporter/go.mod

@@ -86,6 +86,7 @@ require (
 	github.com/miekg/dns v1.1.46 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect

go.mod

@@ -329,6 +329,7 @@ require (
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/mitchellh/hashstructure v1.1.0 // indirect
+	github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/moby/sys/mountinfo v0.5.0 // indirect

receiver/prometheusexecreceiver/go.mod

@@ -82,6 +82,7 @@ require (
 	github.com/miekg/dns v1.1.46 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect

receiver/prometheusreceiver/config.go

@@ -19,6 +19,7 @@ import (
 	"errors"
 	"fmt"
 	"io/ioutil"
+	"net/url"
 	"os"
 	"path/filepath"
 	"sort"
@@ -28,6 +29,7 @@ import (
 	commonconfig "github.com/prometheus/common/config"
 	promconfig "github.com/prometheus/prometheus/config"
 	"github.com/prometheus/prometheus/discovery/file"
+	promHTTP "github.com/prometheus/prometheus/discovery/http"
 	"github.com/prometheus/prometheus/discovery/kubernetes"
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 	"go.opentelemetry.io/collector/config"
@@ -37,6 +39,10 @@ import (
 const (
 	// The key for Prometheus scraping configs.
 	prometheusConfigKey = "config"
+
+	// keys to access the http_sd_config from config root
+	targetAllocatorConfigKey       = "target_allocator"
+	targetAllocatorHTTPSDConfigKey = "http_sd_config"
 )
 
 // Config defines configuration for Prometheus receiver.
@@ -54,12 +60,25 @@ type Config struct {
 	UseStartTimeMetric   bool   `mapstructure:"use_start_time_metric"`
 	StartTimeMetricRegex string `mapstructure:"start_time_metric_regex"`
 
+	TargetAllocator *TargetAllocator `mapstructure:"target_allocator"`
+
 	// ConfigPlaceholder is just an entry to make the configuration pass a check
 	// that requires that all keys present in the config actually exist on the
 	// structure, ie.: it will error if an unknown key is present.
 	ConfigPlaceholder interface{} `mapstructure:"config"`
 }
 
+type TargetAllocator struct {
+	Endpoint    string
+	Interval    time.Duration
+	CollectorID string `mapstructure:"collector_id"`
+	// ConfigPlaceholder is just an entry to make the configuration pass a check
+	// that requires that all keys present in the config actually exist on the
+	// structure, ie.: it will error if an unknown key is present.
+	ConfigPlaceholder interface{}        `mapstructure:"http_sd_config"`
+	HTTPSDConfig      *promHTTP.SDConfig `mapstructure:"-"`
+}
+
 var _ config.Receiver = (*Config)(nil)
 var _ config.Unmarshallable = (*Config)(nil)
 
@@ -205,6 +224,21 @@ func (cfg *Config) Validate() error {
 			}
 		}
 	}
+
+	// validate TargetAllocator
+	targetAllocatorConfig := cfg.TargetAllocator
+	if targetAllocatorConfig == nil {
+		return nil
+	}
+	// ensure valid endpoint
+	if _, err := url.ParseRequestURI(targetAllocatorConfig.Endpoint); err != nil {
+		return fmt.Errorf("TargetAllocator endpoint is not valid: \"%s\"", targetAllocatorConfig.Endpoint)
+	}
+	// ensure valid collectorID without variables
+	if targetAllocatorConfig.CollectorID == "" || strings.Contains(targetAllocatorConfig.CollectorID, "${") {
+		return fmt.Errorf("CollectorID is not a valid ID")
+	}
+
 	return nil
 }
 
@@ -236,5 +270,24 @@ func (cfg *Config) Unmarshal(componentParser *config.Map) error {
 		return fmt.Errorf("prometheus receiver failed to unmarshal yaml to prometheus config: %s", err)
 	}
 
+	// Unmarshal TargetAllocator configs
+	targetAllocatorCfg, err := componentParser.Sub(targetAllocatorConfigKey)
+	if err != nil || len(targetAllocatorCfg.ToStringMap()) == 0 {
+		return err
+	}
+	targetAllocatorHTTPSDCfg, err := targetAllocatorCfg.Sub(targetAllocatorHTTPSDConfigKey)
+	if err != nil || len(targetAllocatorHTTPSDCfg.ToStringMap()) == 0 {
+		return err
+	}
+	targetAllocatorHTTPSDCfg.Set("url", "http://placeholder") // we have to set it as else the marshal will fail
+	httpSDConf, err := yaml.Marshal(targetAllocatorHTTPSDCfg.ToStringMap())
+	if err != nil {
+		return fmt.Errorf("prometheus receiver failed to marshal config to yaml: %s", err)
+	}
+	err = yaml.UnmarshalStrict(httpSDConf, &cfg.TargetAllocator.HTTPSDConfig)
+	if err != nil {
+		return fmt.Errorf("prometheus receiver failed to unmarshal yaml to prometheus config: %s", err)
+	}
+
 	return nil
 }

receiver/prometheusreceiver/config_test.go

@@ -20,6 +20,8 @@ import (
 	"testing"
 	"time"
 
+	promConfig "github.com/prometheus/common/config"
+	promModel "github.com/prometheus/common/model"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"go.opentelemetry.io/collector/component/componenttest"
@@ -48,6 +50,13 @@ func TestLoadConfig(t *testing.T) {
 	assert.Equal(t, time.Duration(r1.PrometheusConfig.ScrapeConfigs[0].ScrapeInterval), 5*time.Second)
 	assert.Equal(t, r1.UseStartTimeMetric, true)
 	assert.Equal(t, r1.StartTimeMetricRegex, "^(.+_)*process_start_time_seconds$")
+
+	assert.Equal(t, "http://my-targetallocator-service", r1.TargetAllocator.Endpoint)
+	assert.Equal(t, 30*time.Second, r1.TargetAllocator.Interval)
+	assert.Equal(t, "collector-1", r1.TargetAllocator.CollectorID)
+	assert.Equal(t, promModel.Duration(60*time.Second), r1.TargetAllocator.HTTPSDConfig.RefreshInterval)
+	assert.Equal(t, "prometheus", r1.TargetAllocator.HTTPSDConfig.HTTPClientConfig.BasicAuth.Username)
+	assert.Equal(t, promConfig.Secret("changeme"), r1.TargetAllocator.HTTPSDConfig.HTTPClientConfig.BasicAuth.Password)
 }
 
 func TestLoadConfigFailsOnUnknownSection(t *testing.T) {

receiver/prometheusreceiver/go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/go-kit/log v0.2.0
 	github.com/gogo/protobuf v1.3.2
 	github.com/golang/snappy v0.0.4
+	github.com/mitchellh/hashstructure/v2 v2.0.2
 	github.com/open-telemetry/opentelemetry-collector-contrib/exporter/prometheusremotewriteexporter v0.50.0
 	github.com/prometheus/common v0.34.0
 	github.com/prometheus/prometheus v1.8.2-0.20220324155304-4d8bbfd4164c
@@ -104,6 +105,7 @@ require (
 	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheusremotewrite v0.50.0 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.0.2 // indirect
+	github.com/pelletier/go-toml v1.9.4 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect