Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Use loadfile that allows reading Config from local file or uri #558

Merged
merged 8 commits into from
Jul 17, 2023
Merged

feat: Use loadfile that allows reading Config from local file or uri #558

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
078d553
feat: Use loadfile that allows reading from local file or uri
brian-intel Jun 30, 2023
1af5f7d
fix: Add default timeout to file.go
brian-intel Jul 7, 2023
a80f8bb
fix: Add request timeout to variables and flags
brian-intel Jul 11, 2023
812304d
fix: remove flag and update function
brian-intel Jul 13, 2023
615ae35
fix: Rebase from origin
brian-intel Jul 17, 2023
fd1fba1
fix: Test update from PR suggestions
brian-intel Jul 17, 2023
db811fe
fix: Get secret provider from dic
brian-intel Jul 17, 2023
1f7c48b
fix: Add absolute and relative path unit tests
brian-intel Jul 17, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 27 additions & 12 deletions bootstrap/config/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,14 +20,15 @@ import (
"errors"
"fmt"
"math"
"os"
"net/url"
"path/filepath"
"reflect"
"strconv"
"strings"
"sync"
"time"

"github.com/edgexfoundry/go-mod-bootstrap/v3/bootstrap/file"
"github.com/edgexfoundry/go-mod-bootstrap/v3/bootstrap/utils"
"github.com/edgexfoundry/go-mod-core-contracts/v3/common"
"github.com/mitchellh/copystructure"
Expand Down Expand Up @@ -62,6 +63,8 @@ const (
// UpdatedStream defines the stream type that is notified by ListenForChanges when a configuration update is received.
type UpdatedStream chan struct{}

var defaultTimeout = 15 * time.Second

type Processor struct {
lc logger.LoggingClient
flags flags.Common
Expand Down Expand Up @@ -197,7 +200,7 @@ func (cp *Processor) Process(
// NOTE: Some security services don't use any common configuration and don't use the configuration provider.
commonConfigLocation := environment.GetCommonConfigFileName(cp.lc, cp.flags.CommonConfig())
if commonConfigLocation != "" {
err := cp.loadCommonConfigFromFile(commonConfigLocation, serviceConfig, serviceType)
err := cp.loadCommonConfigFromFile(commonConfigLocation, serviceConfig, serviceType, secretProvider)
if err != nil {
return err
}
Expand All @@ -213,7 +216,7 @@ func (cp *Processor) Process(
// Now load the private config from a local file if any of these conditions are true
if !useProvider || !cp.providerHasConfig || cp.overwriteConfig {
filePath := GetConfigFileLocation(cp.lc, cp.flags)
configMap, err := cp.loadConfigYamlFromFile(filePath)
configMap, err := cp.loadConfigYamlFromFile(filePath, defaultTimeout, secretProvider)
if err != nil {
return err
}
Expand Down Expand Up @@ -393,11 +396,12 @@ func (cp *Processor) loadCommonConfig(
func (cp *Processor) loadCommonConfigFromFile(
configFile string,
serviceConfig interfaces.Configuration,
serviceType string) error {
serviceType string,
secretProvider interfaces.SecretProviderExt) error {

var err error

commonConfig, err := cp.loadConfigYamlFromFile(configFile)
commonConfig, err := cp.loadConfigYamlFromFile(configFile, defaultTimeout, secretProvider)
if err != nil {
return err
}
Expand Down Expand Up @@ -465,7 +469,7 @@ func (cp *Processor) getAccessTokenCallback(serviceKey string, secretProvider in
// LoadCustomConfigSection loads the specified custom configuration section from file or Configuration provider.
// Section will be seed if Configuration provider does yet have it. This is used for structures custom configuration
// in App and Device services
func (cp *Processor) LoadCustomConfigSection(updatableConfig interfaces.UpdatableConfig, sectionName string) error {
func (cp *Processor) LoadCustomConfigSection(updatableConfig interfaces.UpdatableConfig, sectionName string, secretProvider interfaces.SecretProviderExt) error {
if cp.envVars == nil {
cp.envVars = environment.NewVariables(cp.lc)
}
Expand All @@ -474,7 +478,7 @@ func (cp *Processor) LoadCustomConfigSection(updatableConfig interfaces.Updatabl
if configClient == nil {
cp.lc.Info("Skipping use of Configuration Provider for custom configuration: Provider not available")
filePath := GetConfigFileLocation(cp.lc, cp.flags)
configMap, err := cp.loadConfigYamlFromFile(filePath)
configMap, err := cp.loadConfigYamlFromFile(filePath, defaultTimeout, secretProvider)
if err != nil {
return err
}
Expand Down Expand Up @@ -508,7 +512,7 @@ func (cp *Processor) LoadCustomConfigSection(updatableConfig interfaces.Updatabl
cp.lc.Info("Loaded custom configuration from Configuration Provider, no overrides applied")
} else {
filePath := GetConfigFileLocation(cp.lc, cp.flags)
configMap, err := cp.loadConfigYamlFromFile(filePath)
configMap, err := cp.loadConfigYamlFromFile(filePath, defaultTimeout, secretProvider)
if err != nil {
return err
}
Expand Down Expand Up @@ -636,9 +640,9 @@ func CreateProviderClient(
}

// loadConfigYamlFromFile attempts to read the specified configuration yaml file
func (cp *Processor) loadConfigYamlFromFile(yamlFile string) (map[string]any, error) {
func (cp *Processor) loadConfigYamlFromFile(yamlFile string, timeout time.Duration, provider interfaces.SecretProvider) (map[string]any, error) {
cp.lc.Infof("Loading configuration file from %s", yamlFile)
contents, err := os.ReadFile(yamlFile)
contents, err := file.Load(yamlFile, timeout, provider)
if err != nil {
return nil, fmt.Errorf("failed to read configuration file %s: %s", yamlFile, err.Error())
}
Expand All @@ -654,10 +658,21 @@ func (cp *Processor) loadConfigYamlFromFile(yamlFile string) (map[string]any, er

// GetConfigFileLocation uses the environment variables and flags to determine the location of the configuration
func GetConfigFileLocation(lc logger.LoggingClient, flags flags.Common) string {
configDir := environment.GetConfigDir(lc, flags.ConfigDirectory())
profileDir := environment.GetProfileDir(lc, flags.Profile())
configFileName := environment.GetConfigFileName(lc, flags.ConfigFileName())

// Check for uri path
parsedUrl, err := url.Parse(configFileName)
if err != nil {
lc.Errorf("Could not parse file path: %v", err)
return ""
}
lc.Infof("name %s, parsedurl: %v", configFileName, parsedUrl)
if parsedUrl.Scheme == "http" || parsedUrl.Scheme == "https" {
return configFileName
}

configDir := environment.GetConfigDir(lc, flags.ConfigDirectory())
profileDir := environment.GetProfileDir(lc, flags.Profile())
return filepath.Join(configDir, profileDir, configFileName)
}

Expand Down
56 changes: 43 additions & 13 deletions bootstrap/config/config_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -299,7 +299,7 @@ func TestLoadCommonConfigFromFile(t *testing.T) {
proc := NewProcessor(f, env, timer, ctx, &wg, nil, dic)

// call load common config
err := proc.loadCommonConfigFromFile(tc.config, tc.serviceConfig, tc.serviceType)
err := proc.loadCommonConfigFromFile(tc.config, tc.serviceConfig, tc.serviceType, nil)
// make assertions
require.NotNil(t, cancel)
if tc.expectedErr == "" {
Expand Down Expand Up @@ -378,21 +378,51 @@ func TestFindChangedKey(t *testing.T) {
}

func TestGetConfigFileLocation(t *testing.T) {
dir := "myRes"
profile := "myProfile"
file := "myFile.yaml"
expected := filepath.Join(dir, profile, file)
defer os.Clearenv()
tests := []struct {
name string
dir string
profile string
path string
secretName string
expected string
}{
{
name: "valid - file",
dir: "myRes",
profile: "myProfile",
path: "myFile.yaml",
expected: filepath.Join("myRes", "myProfile", "myFile.yaml"),
},
{
name: "valid - url",
dir: "",
profile: "",
path: "https://raw.githubusercontent.com/edgexfoundry/go-mod-bootstrap/main/bootstrap/config/testdata/configuration.yaml",
expected: "https://raw.githubusercontent.com/edgexfoundry/go-mod-bootstrap/main/bootstrap/config/testdata/configuration.yaml",
},
{
name: "invalid - url",
dir: "",
profile: "",
path: "{test:\"test\"}",
expected: "",
},
}

lc := logger.NewMockClient()
flags := flags.New()
for _, test := range tests {
t.Run(test.secretName, func(t *testing.T) {
lc := logger.NewMockClient()
flags := flags.New()

os.Setenv("EDGEX_CONFIG_DIR", dir)
os.Setenv("EDGEX_PROFILE", profile)
os.Setenv("EDGEX_CONFIG_FILE", file)
defer os.Clearenv()
os.Setenv("EDGEX_CONFIG_DIR", test.dir)
os.Setenv("EDGEX_PROFILE", test.profile)
os.Setenv("EDGEX_CONFIG_FILE", test.path)

actual := GetConfigFileLocation(lc, flags)
assert.Equal(t, expected, actual)
actual := GetConfigFileLocation(lc, flags)
assert.Equal(t, test.expected, actual)
})
}
}

func TestGetInsecureSecretNameFullPath(t *testing.T) {
Expand Down
4 changes: 3 additions & 1 deletion bootstrap/file/file_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,13 @@ func TestLoadFile(t *testing.T) {
{"Valid - load from JSON file", path.Join(".", "testdata", "configuration.json"), 142, "", nil},
{"Valid - load from HTTP", "http://raw.githubusercontent.com/edgexfoundry/go-mod-bootstrap/main/bootstrap/config/testdata/configuration.yaml", 4533, "", nil},
{"Valid - load from HTTPS", "https://raw.githubusercontent.com/edgexfoundry/go-mod-bootstrap/main/bootstrap/config/testdata/configuration.yaml", 4533, "", nil},
{"Valid - load from HTTPS with secret", "https://raw.githubusercontent.com/edgexfoundry/go-mod-bootstrap/main/bootstrap/config/testdata/configuration.yaml?edgexSecretName=mySecretName", 4533, "", map[string]string{"type": "httpheader", "headername": "Authorization", "headercontents": "Basic 1234567890"}},
{"Invalid - File not found", "bogus", 0, "Could not read file", nil},
{"Invalid - parse uri fail", "{test:\"test\"}", 0, "Could not parse file path", nil},
{"Invalid - load from invalid HTTP", "http://raw.githubusercontent.com/edgexfoundry/go-mod-bootstrap/main/bootstrap/config/configuration.yaml", 1, "Invalid status code", nil},
{"Invalid - load from invalid HTTPS", "https://raw.githubusercontent.com/edgexfoundry/go-mod-bootstrap/main/bootstrap/config/configuration.yaml", 1, "Invalid status code", nil},
{"Valid - load from HTTPS with secret", "https://raw.githubusercontent.com/edgexfoundry/go-mod-bootstrap/main/bootstrap/config/testdata/configuration.yaml?edgexSecretName=mySecretName", 4533, "", map[string]string{"type": "httpheader", "headername": "Authorization", "headercontents": "Basic 1234567890"}},
{"Invalid - load from HTTPS with invalid secret", "https://raw.githubusercontent.com/edgexfoundry/go-mod-bootstrap/main/bootstrap/config/testdata/configuration.yaml?edgexSecretName=mySecretName", 4533, "Secret type is not httpheader", map[string]string{"type": "invalidheader", "headername": "Authorization", "headercontents": "Basic 1234567890"}},
{"Invalid - load from HTTPS with empty secret", "https://raw.githubusercontent.com/edgexfoundry/go-mod-bootstrap/main/bootstrap/config/testdata/configuration.yaml?edgexSecretName=mySecretName", 0, "Secret headername and headercontents can not be empty", map[string]string{"type": "httpheader", "headername": "", "headercontents": ""}},
}

for _, tc := range tests {
Expand Down