Support passing in confmap.Providers in the builder

.chloggen/builder-confmap-providers.yaml

@@ -0,0 +1,31 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. otlpreceiver)
+component: cmd/builder
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Allow configuring `confmap.Provider`s in the builder.
+
+# One or more tracking issues or pull requests related to the change
+issues: [4759]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: |
+  If no providers or converters are specified, the defaults are used.
+  The default providers are: env, file, http, https, and yaml.
+
+  To configure providers, use the `providers` key in your OCB build
+  manifest with a list of Go modules for your providers.
+  The modules will work the same as other Collector components.
+
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [user]

cmd/builder/README.md

@@ -12,14 +12,20 @@ dist:
   description: Local OpenTelemetry Collector binary
   output_path: /tmp/dist
 exporters:
-  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/alibabacloudlogserviceexporter v0.86.0
-  - gomod: go.opentelemetry.io/collector/exporter/debugexporter v0.86.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/alibabacloudlogserviceexporter v0.96.0
+  - gomod: go.opentelemetry.io/collector/exporter/debugexporter v0.96.0
 
 receivers:
-  - gomod: go.opentelemetry.io/collector/receiver/otlpreceiver v0.86.0
+  - gomod: go.opentelemetry.io/collector/receiver/otlpreceiver v0.96.0
 
 processors:
-  - gomod: go.opentelemetry.io/collector/processor/batchprocessor v0.86.0
+  - gomod: go.opentelemetry.io/collector/processor/batchprocessor v0.96.0
+
+providers:
+  - gomod: go.opentelemetry.io/collector/confmap/provider/fileprovider v0.96.0
+
+converters:
+  - gomod: go.opentelemetry.io/collector/confmap/converter/expandconverter v0.96.0
 EOF
 $ builder --config=otelcol-builder.yaml
 $ cat > /tmp/otelcol.yaml <<EOF

cmd/builder/internal/builder/config.go

@@ -36,6 +36,7 @@ type Config struct {
 	Receivers    []Module     `mapstructure:"receivers"`
 	Processors   []Module     `mapstructure:"processors"`
 	Connectors   []Module     `mapstructure:"connectors"`
+	Providers    *[]Module    `mapstructure:"providers"`
 	Replaces     []string     `mapstructure:"replaces"`
 	Excludes     []string     `mapstructure:"excludes"`
 }
@@ -86,12 +87,18 @@ func NewDefaultConfig() Config {
 
 // Validate checks whether the current configuration is valid
 func (c *Config) Validate() error {
+	var providersError, convertersError error
+	if c.Providers != nil {
+		providersError = validateModules(*c.Providers)
+	}
 	return multierr.Combine(
 		validateModules(c.Extensions),
 		validateModules(c.Receivers),
 		validateModules(c.Exporters),
 		validateModules(c.Processors),
 		validateModules(c.Connectors),
+		providersError,
+		convertersError,
 	)
 }
 
@@ -155,6 +162,36 @@ func (c *Config) ParseModules() error {
 		return err
 	}
 
+	if c.Providers != nil {
+		providers, err := parseModules(*c.Providers)
+		if err != nil {
+			return err
+		}
+		c.Providers = &providers
+	} else {
+		providers, err := parseModules([]Module{
+			{
+				GoMod: "go.opentelemetry.io/collector/confmap/provider/envprovider v" + c.Distribution.OtelColVersion,
+			},
+			{
+				GoMod: "go.opentelemetry.io/collector/confmap/provider/fileprovider v" + c.Distribution.OtelColVersion,
+			},
+			{
+				GoMod: "go.opentelemetry.io/collector/confmap/provider/httpprovider v" + c.Distribution.OtelColVersion,
+			},
+			{
+				GoMod: "go.opentelemetry.io/collector/confmap/provider/httpsprovider v" + c.Distribution.OtelColVersion,
+			},
+			{
+				GoMod: "go.opentelemetry.io/collector/confmap/provider/yamlprovider v" + c.Distribution.OtelColVersion,
+			},
+		})
+		if err != nil {
+			return err
+		}
+		c.Providers = &providers
+	}
+
 	return nil
 }
 

cmd/builder/internal/builder/config_test.go

@@ -225,3 +225,16 @@ func TestRequireOtelColModule(t *testing.T) {
 		})
 	}
 }
+
+func TestAddsDefaultProviders(t *testing.T) {
+	cfg := NewDefaultConfig()
+	cfg.Providers = nil
+	assert.NoError(t, cfg.ParseModules())
+	assert.Len(t, *cfg.Providers, 5)
+}
+
+func TestSkipsNilFieldValidation(t *testing.T) {
+	cfg := NewDefaultConfig()
+	cfg.Providers = nil
+	assert.NoError(t, cfg.Validate())
+}

cmd/builder/internal/builder/main_test.go

@@ -15,27 +15,37 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+func newInitializedConfig(t *testing.T) Config {
+	cfg := NewDefaultConfig()
+	// Validate and ParseModules will be called before the config is
+	// given to Generate.
+	assert.NoError(t, cfg.Validate())
+	assert.NoError(t, cfg.ParseModules())
+
+	return cfg
+}
+
 func TestGenerateDefault(t *testing.T) {
-	require.NoError(t, Generate(NewDefaultConfig()))
+	require.NoError(t, Generate(newInitializedConfig(t)))
 }
 
 func TestGenerateInvalidCollectorVersion(t *testing.T) {
-	cfg := NewDefaultConfig()
+	cfg := newInitializedConfig(t)
 	cfg.Distribution.OtelColVersion = "invalid"
 	err := Generate(cfg)
 	require.NoError(t, err)
 }
 
 func TestGenerateInvalidOutputPath(t *testing.T) {
-	cfg := NewDefaultConfig()
+	cfg := newInitializedConfig(t)
 	cfg.Distribution.OutputPath = "/:invalid"
 	err := Generate(cfg)
 	require.Error(t, err)
 	require.Contains(t, err.Error(), "failed to create output path")
 }
 
 func TestSkipGenerate(t *testing.T) {
-	cfg := NewDefaultConfig()
+	cfg := newInitializedConfig(t)
 	cfg.Distribution.OutputPath = t.TempDir()
 	cfg.SkipGenerate = true
 	err := Generate(cfg)
@@ -120,13 +130,24 @@ func TestGenerateAndCompile(t *testing.T) {
 				return cfg
 			},
 		},
+		{
+			testCase: "No providers",
+			cfgBuilder: func(t *testing.T) Config {
+				cfg := NewDefaultConfig()
+				cfg.Distribution.OutputPath = t.TempDir()
+				cfg.Replaces = append(cfg.Replaces, replaces...)
+				cfg.Providers = &[]Module{}
+				return cfg
+			},
+		},
 	}
 
 	for _, tt := range testCases {
 		t.Run(tt.testCase, func(t *testing.T) {
 			cfg := tt.cfgBuilder(t)
 			assert.NoError(t, cfg.Validate())
 			assert.NoError(t, cfg.SetGoPath())
+			assert.NoError(t, cfg.ParseModules())
 			require.NoError(t, GenerateAndCompile(cfg))
 		})
 	}

cmd/builder/internal/builder/templates/go.mod.tmpl

@@ -5,6 +5,10 @@ module {{.Distribution.Module}}
 go 1.21
 
 require (
+	go.opentelemetry.io/collector/confmap/converter/expandconverter v{{.Distribution.OtelColVersion}}
+	{{- range .Providers}}
+	{{if .GoMod}}{{.GoMod}}{{end}}
+	{{- end}}
 	{{- range .Connectors}}
 	{{if .GoMod}}{{.GoMod}}{{end}}
 	{{- end}}

cmd/builder/internal/builder/templates/main.go.tmpl

@@ -7,6 +7,11 @@ import (
 	"log"
 
 	"go.opentelemetry.io/collector/component"
+	"go.opentelemetry.io/collector/confmap"
+	"go.opentelemetry.io/collector/confmap/converter/expandconverter"
+	{{- range .Providers}}
+	{{.Name}} "{{.Import}}"
+	{{- end}}
 	"go.opentelemetry.io/collector/otelcol"
 )
 
@@ -16,8 +21,25 @@ func main() {
 		Description: "{{ .Distribution.Description }}",
 		Version:     "{{ .Distribution.Version }}",
 	}
+	providers := []confmap.Provider{
+		{{- range .Providers}}
+		{{.Name}}.NewWithSettings(confmap.ProviderSettings{}),
+		{{- end}}
+	}
+	set := otelcol.CollectorSettings{
+		BuildInfo: info,
+		Factories: components,
+		ConfigProviderSettings: otelcol.ConfigProviderSettings{
+			ResolverSettings: confmap.ResolverSettings{
+				Providers:  makeMapProvidersMap(providers...),
+				Converters: []confmap.Converter{
+					expandconverter.New(confmap.ConverterSettings{}),
+				},
+			},
+		},
+	}
 
-	if err := run(otelcol.CollectorSettings{BuildInfo: info, Factories: components}); err != nil {
+	if err := run(set); err != nil {
 		log.Fatal(err)
 	}
 }
@@ -30,3 +52,11 @@ func runInteractive(params otelcol.CollectorSettings) error {
 
 	return nil
 }
+
+func makeMapProvidersMap(providers ...confmap.Provider) map[string]confmap.Provider {
+	ret := make(map[string]confmap.Provider, len(providers))
+	for _, provider := range providers {
+		ret[provider.Scheme()] = provider
+	}
+	return ret
+}

cmd/builder/internal/command.go

@@ -168,6 +168,7 @@ func applyCfgFromFile(flags *flag.FlagSet, cfgFromFile builder.Config) {
 	cfg.Receivers = cfgFromFile.Receivers
 	cfg.Processors = cfgFromFile.Processors
 	cfg.Connectors = cfgFromFile.Connectors
+	cfg.Providers = cfgFromFile.Providers
 	cfg.Replaces = cfgFromFile.Replaces
 	cfg.Excludes = cfgFromFile.Excludes
 

cmd/builder/internal/config/default.yaml

@@ -22,3 +22,10 @@ processors:
 connectors:
   - gomod: go.opentelemetry.io/collector/connector/forwardconnector v0.96.0
 
+providers:
+  - gomod: go.opentelemetry.io/collector/confmap/provider/envprovider v0.96.0
+  - gomod: go.opentelemetry.io/collector/confmap/provider/fileprovider v0.96.0
+  - gomod: go.opentelemetry.io/collector/confmap/provider/httpprovider v0.96.0
+  - gomod: go.opentelemetry.io/collector/confmap/provider/httpsprovider v0.96.0
+  - gomod: go.opentelemetry.io/collector/confmap/provider/yamlprovider v0.96.0
+

cmd/otelcorecol/builder-config.yaml

@@ -22,6 +22,13 @@ processors:
 connectors:
   - gomod: go.opentelemetry.io/collector/connector/forwardconnector v0.96.0
 
+providers:
+  - gomod: go.opentelemetry.io/collector/confmap/provider/envprovider v0.95.0
+  - gomod: go.opentelemetry.io/collector/confmap/provider/fileprovider v0.95.0
+  - gomod: go.opentelemetry.io/collector/confmap/provider/httpprovider v0.95.0
+  - gomod: go.opentelemetry.io/collector/confmap/provider/httpsprovider v0.95.0
+  - gomod: go.opentelemetry.io/collector/confmap/provider/yamlprovider v0.95.0
+
 replaces:
   - go.opentelemetry.io/collector => ../../
   - go.opentelemetry.io/collector/otelcol => ../../otelcol

cmd/otelcorecol/go.mod

@@ -7,6 +7,13 @@ go 1.21
 require (
 	github.com/stretchr/testify v1.9.0
 	go.opentelemetry.io/collector/component v0.96.0
+	go.opentelemetry.io/collector/confmap v0.96.0
+	go.opentelemetry.io/collector/confmap/converter/expandconverter v0.96.0
+	go.opentelemetry.io/collector/confmap/provider/envprovider v0.96.0
+	go.opentelemetry.io/collector/confmap/provider/fileprovider v0.96.0
+	go.opentelemetry.io/collector/confmap/provider/httpprovider v0.96.0
+	go.opentelemetry.io/collector/confmap/provider/httpsprovider v0.96.0
+	go.opentelemetry.io/collector/confmap/provider/yamlprovider v0.96.0
 	go.opentelemetry.io/collector/connector v0.96.0
 	go.opentelemetry.io/collector/connector/forwardconnector v0.96.0
 	go.opentelemetry.io/collector/exporter v0.96.0
@@ -85,13 +92,6 @@ require (
 	go.opentelemetry.io/collector/config/configtelemetry v0.96.0 // indirect
 	go.opentelemetry.io/collector/config/configtls v0.96.0 // indirect
 	go.opentelemetry.io/collector/config/internal v0.96.0 // indirect
-	go.opentelemetry.io/collector/confmap v0.96.0 // indirect
-	go.opentelemetry.io/collector/confmap/converter/expandconverter v0.96.0 // indirect
-	go.opentelemetry.io/collector/confmap/provider/envprovider v0.96.0 // indirect
-	go.opentelemetry.io/collector/confmap/provider/fileprovider v0.96.0 // indirect
-	go.opentelemetry.io/collector/confmap/provider/httpprovider v0.96.0 // indirect
-	go.opentelemetry.io/collector/confmap/provider/httpsprovider v0.96.0 // indirect
-	go.opentelemetry.io/collector/confmap/provider/yamlprovider v0.96.0 // indirect
 	go.opentelemetry.io/collector/consumer v0.96.0 // indirect
 	go.opentelemetry.io/collector/extension/auth v0.96.0 // indirect
 	go.opentelemetry.io/collector/featuregate v1.3.0 // indirect