support uris as --config args (#3182)

CHANGELOG.md

@@ -6,6 +6,7 @@
 
 - Evaluate `--set` properties as yaml values ([#3175](https://github.com/signalfx/splunk-otel-collector/pull/3175))
 - Evaluate config converter updates to `--dry-run` content ([#3176](https://github.com/signalfx/splunk-otel-collector/pull/3176))
+- Support config provider uris in `--config` option values ([#3182](https://github.com/signalfx/splunk-otel-collector/pull/3182))
 
 ## v0.77.0
 

internal/settings/settings.go

@@ -18,6 +18,8 @@ import (
 	"fmt"
 	"log"
 	"os"
+	"regexp"
+	"sort"
 	"strconv"
 	"strings"
 
@@ -70,6 +72,7 @@ type Settings struct {
 	discoveryProperties []string
 	setProperties       []string
 	colCoreArgs         []string
+	supportedURISchemes []string
 	versionFlag         bool
 	noConvertConfig     bool
 	configD             bool
@@ -153,6 +156,29 @@ func (s *Settings) ConfMapProviders() map[string]confmap.Provider {
 	return s.confMapProviders
 }
 
+func loadConfMapProviders(s *Settings) error {
+	var err error
+	if s.discovery, err = discovery.New(); err != nil {
+		return fmt.Errorf("failed to create discovery provider: %w", err)
+	}
+
+	s.confMapProviders = map[string]confmap.Provider{
+		envProvider.Scheme():  envProvider,
+		fileProvider.Scheme(): fileProvider,
+	}
+
+	for p := range s.confMapProviders {
+		s.supportedURISchemes = append(s.supportedURISchemes, p)
+	}
+	sort.Strings(s.supportedURISchemes)
+
+	// though supported, these schemes shouldn't be advertised for use w/ --config
+	s.confMapProviders[s.discovery.PropertyScheme()] = s.discovery.PropertyProvider()
+	s.confMapProviders[s.discovery.ConfigDScheme()] = s.discovery.ConfigDProvider()
+	s.confMapProviders[s.discovery.DiscoveryModeScheme()] = s.discovery.DiscoveryModeProvider()
+	return nil
+}
+
 // ConfMapConverters returns confmap.Converters for the collector core service.
 func (s *Settings) ConfMapConverters() []confmap.Converter {
 	confMapConverters := []confmap.Converter{
@@ -192,17 +218,8 @@ func parseArgs(args []string) (*Settings, error) {
 		configDir:          new(stringPointerFlagValue),
 	}
 
-	var err error
-	if settings.discovery, err = discovery.New(); err != nil {
-		return nil, fmt.Errorf("failed to create discovery provider: %w", err)
-	}
-
-	settings.confMapProviders = map[string]confmap.Provider{
-		envProvider.Scheme():                     envProvider,
-		fileProvider.Scheme():                    fileProvider,
-		settings.discovery.PropertyScheme():      settings.discovery.PropertyProvider(),
-		settings.discovery.ConfigDScheme():       settings.discovery.ConfigDProvider(),
-		settings.discovery.DiscoveryModeScheme(): settings.discovery.DiscoveryModeProvider(),
+	if err := loadConfMapProviders(settings); err != nil {
+		return nil, fmt.Errorf("failed loading confmap.Providers: %w", err)
 	}
 
 	flagSet.Var(settings.configPaths, "config", "Locations to the config file(s), "+
@@ -430,21 +447,47 @@ func checkInputConfigs(settings *Settings) error {
 	configPathVar := os.Getenv(ConfigEnvVar)
 	configYaml := os.Getenv(ConfigYamlEnvVar)
 
+	var configFilePaths []string
 	for _, filePath := range settings.configPaths.value {
+		scheme, location, isURI := parseURI(filePath)
+		if isURI {
+			if _, ok := settings.confMapProviders[scheme]; !ok {
+				log.Printf("%q is an unsupported config provider scheme for this Collector distribution (not in %v).", scheme, settings.supportedURISchemes)
+				continue
+			}
+			if scheme != fileProvider.Scheme() {
+				continue
+			}
+			filePath = location
+		}
 		if _, err := os.Stat(filePath); err != nil {
 			return fmt.Errorf("unable to find the configuration file %s, ensure flag '--config' is set properly: %w", filePath, err)
 		}
+		configFilePaths = append(configFilePaths, filePath)
 	}
 
-	if configPathVar != "" && !settings.configPaths.contains(configPathVar) {
-		log.Printf("Both environment variable %v and flag '--config' were specified. Using the flag values and ignoring the environment variable value %s in this session", ConfigEnvVar, configPathVar)
+	if len(configFilePaths) == 0 {
+		return nil
+	}
+
+	if configPathVar != "" {
+		differingVals := true
+		for _, p := range configFilePaths {
+			if p == configPathVar {
+				differingVals = false
+				break
+			}
+		}
+		if differingVals {
+			log.Printf("Both environment variable %v and flag '--config' were specified. Using the flag values and ignoring the environment variable value %s in this session", ConfigEnvVar, configPathVar)
+		}
 	}
 
 	if configYaml != "" {
 		log.Printf("Both environment variable %s and flag '--config' were specified. Using the flag values and ignoring the environment variable in this session", ConfigYamlEnvVar)
 	}
 
-	return confirmRequiredEnvVarsForDefaultConfigs(settings.configPaths.value)
+	return confirmRequiredEnvVarsForDefaultConfigs(configFilePaths)
 }
 
 func checkConfigPathEnvVar(settings *Settings) error {
@@ -537,3 +580,16 @@ func (s *stringPointerFlagValue) String() string {
 	}
 	return *s.value
 }
+
+// From https://github.com/open-telemetry/opentelemetry-collector/blob/18a11ec09b3f4883d0360a41054ce8f4a8736ea8/confmap/expand.go
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+var uriRegexp = regexp.MustCompile(`(?s:^(?P<Scheme>[A-Za-z][A-Za-z0-9+.-]+):(?P<OpaqueValue>.*)$)`)
+
+func parseURI(uri string) (scheme string, location string, isURI bool) {
+	submatches := uriRegexp.FindStringSubmatch(uri)
+	if len(submatches) != 3 {
+		return "", "", false
+	}
+	return submatches[1], submatches[2], true
+}

internal/settings/settings_test.go

@@ -479,6 +479,45 @@ func TestConfigDirFromEnvVar(t *testing.T) {
 	require.Equal(t, "/from/env/var", getConfigDir(settings))
 }
 
+func TestConfigArgFileURIForm(t *testing.T) {
+	t.Cleanup(clearEnv(t))
+	uriPath := fmt.Sprintf("file:%s", configPath)
+	settings, err := New([]string{"--config", uriPath})
+	require.NoError(t, err)
+	require.Equal(t, []string{uriPath}, settings.configPaths.value)
+	require.Equal(t, settings.configPaths.value, settings.ResolverURIs())
+}
+
+func TestConfigArgEnvURIForm(t *testing.T) {
+	t.Cleanup(clearEnv(t))
+	settings, err := New([]string{"--config", "env:SOME_ENV_VAR"})
+	require.NoError(t, err)
+	require.Equal(t, []string{"env:SOME_ENV_VAR"}, settings.configPaths.value)
+	require.Equal(t, settings.configPaths.value, settings.ResolverURIs())
+
+}
+
+func TestConfigArgUnsupportedURI(t *testing.T) {
+	t.Cleanup(clearEnv(t))
+
+	oldWriter := log.Default().Writer()
+	defer func() {
+		log.Default().SetOutput(oldWriter)
+	}()
+
+	logs := new(bytes.Buffer)
+	log.Default().SetOutput(logs)
+
+	settings, err := New([]string{"--config", "invalid:invalid"})
+	// though invalid, we defer failing to collector service
+	require.NoError(t, err)
+	require.NotNil(t, settings)
+	require.Equal(t, []string{"invalid:invalid"}, settings.configPaths.value)
+	require.Equal(t, settings.configPaths.value, settings.ResolverURIs())
+
+	require.Contains(t, logs.String(), `"invalid" is an unsupported config provider scheme for this Collector distribution (not in [env file]).`)
+}
+
 // to satisfy Settings generation
 func setRequiredEnvVars(t *testing.T) func() {
 	cleanup := clearEnv(t)

tests/general/configproviders/env_test.go

@@ -0,0 +1,54 @@
+// Copyright  Splunk, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build integration
+
+package tests
+
+import (
+	"testing"
+
+	"github.com/signalfx/splunk-otel-collector/tests/testutils"
+)
+
+func TestEnvProvider(t *testing.T) {
+	config := `receivers:
+  hostmetrics:
+    collection_interval: 1s
+    scrapers:
+      memory:
+
+exporters:
+  otlp:
+    endpoint: ${env:OTLP_ENDPOINT}
+    tls:
+      insecure: true
+
+service:
+  pipelines:
+    metrics:
+      receivers: [hostmetrics]
+      exporters: [otlp]
+`
+	testutils.AssertAllMetricsReceived(
+		t, "memory.yaml", "", nil,
+		[]testutils.CollectorBuilder{
+			func(collector testutils.Collector) testutils.Collector {
+				return collector.WithEnv(
+					map[string]string{"SOME_ENV_VAR": config},
+				).WithArgs("--config", "env:SOME_ENV_VAR")
+			},
+		},
+	)
+}

tests/general/configproviders/file_test.go

@@ -30,13 +30,13 @@ func TestFileProvider(t *testing.T) {
 	testdataPath, err := filepath.Abs(path.Join(".", "testdata"))
 	require.NoError(t, err)
 	testutils.AssertAllMetricsReceived(
-		t, "memory.yaml", "file_config.yaml", nil,
+		t, "memory.yaml", "", nil,
 		[]testutils.CollectorBuilder{
 			func(collector testutils.Collector) testutils.Collector {
 				if cc, ok := collector.(*testutils.CollectorContainer); ok {
-					return cc.WithMount(testdataPath, "/testdata")
+					collector = cc.WithMount(testdataPath, "/testdata")
 				}
-				return collector
+				return collector.WithArgs("--config", "file:./testdata/file_config.yaml")
 			},
 		},
 	)