WIP Secret Source Implementation

cmd/state/state.go

@@ -14,7 +14,9 @@ import (
 
 	"go.k6.io/k6/internal/event"
 	"go.k6.io/k6/internal/ui/console"
+	"go.k6.io/k6/internal/usage"
 	"go.k6.io/k6/lib/fsext"
+	"go.k6.io/k6/secretsource"
 )
 
 const defaultConfigFileName = "config.json"
@@ -54,6 +56,9 @@ type GlobalState struct {
 
 	Logger         *logrus.Logger //nolint:forbidigo //TODO:change to FieldLogger
 	FallbackLogger logrus.FieldLogger
+
+	SecretsManager *secretsource.SecretsManager
+	Usage          *usage.Usage
 }
 
 // NewGlobalState returns a new GlobalState with the given ctx.
@@ -129,6 +134,7 @@ func NewGlobalState(ctx context.Context) *GlobalState {
 			Hooks:     make(logrus.LevelHooks),
 			Level:     logrus.InfoLevel,
 		},
+		Usage: usage.New(),
 	}
 }
 
@@ -140,6 +146,7 @@ type GlobalFlags struct {
 	Address          string
 	ProfilingEnabled bool
 	LogOutput        string
+	SecretSource     []string
 	LogFormat        string
 	Verbose          bool
 }

ext/ext.go

@@ -25,6 +25,7 @@ type ExtensionType uint8
 const (
 	JSExtension ExtensionType = iota + 1
 	OutputExtension
+	SecretSourceExtension
 )
 
 func (e ExtensionType) String() string {
@@ -34,6 +35,8 @@ func (e ExtensionType) String() string {
 		s = "js"
 	case OutputExtension:
 		s = "output"
+	case SecretSourceExtension:
+		s = "secret-source"
 	}
 	return s
 }
@@ -157,4 +160,5 @@ func extractModuleInfo(mod interface{}) (path, version string) {
 func init() {
 	extensions[JSExtension] = make(map[string]*Extension)
 	extensions[OutputExtension] = make(map[string]*Extension)
+	extensions[SecretSourceExtension] = make(map[string]*Extension)
 }

internal/cmd/root.go

@@ -19,7 +19,11 @@
 	"go.k6.io/k6/cmd/state"
 	"go.k6.io/k6/errext"
 	"go.k6.io/k6/errext/exitcodes"
+	"go.k6.io/k6/ext"
 	"go.k6.io/k6/internal/log"
+	"go.k6.io/k6/secretsource"
+
+	_ "go.k6.io/k6/internal/secretsource" //nolint:revive
 )
 
 const waitLoggerCloseTimeout = time.Second * 5
@@ -162,6 +166,10 @@
 	// `gs.DefaultFlags.<value>`, so that the `k6 --help` message is
 	// not messed up...
 
+	// TODO(@mstoykov): likely needs work - no env variables and such. No config.json.
+	flags.StringSliceVar(&gs.Flags.SecretSource, "secret-source", gs.Flags.SecretSource,
+		"setting secret sources for k6 file[=./path.fileformat],")
+
 	flags.StringVar(&gs.Flags.LogOutput, "log-output", gs.Flags.LogOutput,
 		"change the output for k6 logs, possible values are stderr,stdout,none,loki[=host:port],file[=./path.fileformat]")
 	flags.Lookup("log-output").DefValue = gs.DefaultFlags.LogOutput
@@ -257,6 +265,22 @@
 		c.globalState.Logger.Debug("Logger format: TEXT")
 	}
 
+	secretsources, err := createSecretSources(c.globalState)
+	if err != nil {
+		return err
+	}
+	// it is important that we add this hook first as hooks are executed in order of addition
+	// and this means no other hook will get secrets
+	SecretsHook := log.NewSecretsRedactionHook()
+	if len(secretsources) != 0 {
+		// don't actually filter anything if there will be no secrets
+		c.globalState.Logger.AddHook(SecretsHook)
+	}
+	c.globalState.SecretsManager, err = secretsource.NewSecretsManager(SecretsHook, secretsources)
+	if err != nil {
+		return err
+	}
+
 	cancel := func() {} // noop as default
 	if hook != nil {
 		ctx := context.Background()
@@ -289,3 +313,49 @@
 	c.globalState.Logger.AddHook(h)
 	c.globalState.Logger.SetOutput(io.Discard) // don't output to anywhere else
 }
+
+func createSecretSources(gs *state.GlobalState) (map[string]secretsource.SecretSource, error) {
+	baseParams := secretsource.Params{
+		Logger:      gs.Logger,
+		Environment: gs.Env,
+		FS:          gs.FS,
+		Usage:       gs.Usage,
+	}
+
+	result := make(map[string]secretsource.SecretSource)
+	for _, line := range gs.Flags.SecretSource {
+		t, config, ok := strings.Cut(line, "=")
+		if !ok {
+			return nil, fmt.Errorf("couldn't parse secret source configuration %q", line)
+		}
+		secretSources := ext.Get(ext.SecretSourceExtension)
+		found, ok := secretSources[t]
+		if !ok {
+			return nil, fmt.Errorf("no secret source for type %q for configuration %q", t, line)
+		}
+		c := found.Module.(secretsource.Constructor) //nolint:forcetypeassert
+		params := baseParams
+		params.ConfigArgument = config
+		// TODO(@mstoykov): make it not configurable just from cmd line
+		// params.JSONConfig = test.derivedConfig.Collectors[outputType]
+
+		secretSource, err := c(params)
+		if err != nil {
+			return nil, err
+		}
+		name := secretSource.Name()
+		_, alreadRegistered := result[name]
+		if alreadRegistered {
+			return nil, fmt.Errorf("secret source for name %q already registered before configuration %q", t, line)
+		}
+		result[name] = secretSource
+	}
+
+	if len(result) == 1 {
+		for _, l := range result {
+			result["default"] = l
+		}
+	}
+
+	return result, nil
+}

internal/cmd/test_load.go

@@ -18,7 +18,6 @@ import (
 	"go.k6.io/k6/errext/exitcodes"
 	"go.k6.io/k6/internal/js"
 	"go.k6.io/k6/internal/loader"
-	"go.k6.io/k6/internal/usage"
 	"go.k6.io/k6/js/modules"
 	"go.k6.io/k6/lib"
 	"go.k6.io/k6/lib/fsext"
@@ -84,7 +83,8 @@ func loadLocalTest(gs *state.GlobalState, cmd *cobra.Command, args []string) (*l
 			val, ok := gs.Env[key]
 			return val, ok
 		},
-		Usage: usage.New(),
+		Usage:          gs.Usage,
+		SecretsManager: gs.SecretsManager,
 	}
 
 	test := &loadedTest{

internal/cmd/tests/test_state.go

@@ -18,6 +18,7 @@ import (
 	"go.k6.io/k6/internal/event"
 	"go.k6.io/k6/internal/lib/testutils"
 	"go.k6.io/k6/internal/ui/console"
+	"go.k6.io/k6/internal/usage"
 	"go.k6.io/k6/lib/fsext"
 )
 
@@ -111,6 +112,7 @@ func NewGlobalTestState(tb testing.TB) *GlobalTestState {
 		SignalStop:     signal.Stop,
 		Logger:         logger,
 		FallbackLogger: testutils.NewLogger(tb).WithField("fallback", true),
+		Usage:          usage.New(),
 	}
 
 	return ts

internal/log/secrets.go

@@ -0,0 +1,72 @@
+package log
+
+import (
+	"fmt"
+	"strings"
+	"sync"
+
+	"github.com/sirupsen/logrus"
+)
+
+// SecretsHook is a Logrus hook for hiding secrets from entries before they get logged
+type SecretsHook struct {
+	secrets  []string
+	mx       sync.RWMutex
+	replacer *strings.Replacer
+}
+
+// NewSecretsRedactionHook makes a new secrets hook that is used to redact secrets
+// from the logs before other parts can log them
+func NewSecretsRedactionHook() *SecretsHook {
+	return &SecretsHook{}
+}
+
+// Levels is part of the [logrus.Hook]
+func (s *SecretsHook) Levels() []logrus.Level { return logrus.AllLevels }
+
+// Add is used to add a new secret to be redacted.
+// Adding the same secret multiple times will not error, but is not recommended.
+// It is users job to not keep adding the same secret over time but only once.
+func (s *SecretsHook) Add(secret string) {
+	s.mx.Lock()
+	defer s.mx.Unlock()
+	s.secrets = append(s.secrets, secret, "***SECRET_REDACTED***")
+	s.replacer = strings.NewReplacer(s.secrets...)
+}
+
+// Fire is part of the [logrus.Hook]
+func (s *SecretsHook) Fire(entry *logrus.Entry) error {
+	s.mx.Lock()
+	// there is no way for us to get a secret after we got a log for it so we can use that to cache the replacer
+	replacer := s.replacer
+	s.mx.Unlock()
+	if replacer == nil { // no secrets no work
+		return nil
+	}
+	entry.Message = replacer.Replace(entry.Message)
+
+	// replace both keys and values with
+	for k, v := range entry.Data {
+		newk := replacer.Replace(k)
+		if newk != k {
+			entry.Data[newk] = v
+			delete(entry.Data, k)
+			k = newk
+		}
+		entry.Data[k] = recursiveReplace(v, replacer)
+	}
+
+	return nil
+}
+
+func recursiveReplace(v any, replacer *strings.Replacer) any {
+	switch s := v.(type) {
+	case string:
+		return replacer.Replace(s)
+	case int, uint, int64, int32, int16, int8, uint64, uint32, uint16, uint8:
+		// if the secret is encodable in 64 bits ... it is probably not a great secret
+		return v
+	}
+	// replace this with a log after more testing
+	panic(fmt.Sprintf("Had a logrus.fields value with type %T, please report that this is unsupported", v))
+}

internal/secretsource/file/file.go

@@ -0,0 +1,55 @@
+package file
+
+import (
+	"bufio"
+	"errors"
+	"fmt"
+	"strings"
+
+	"go.k6.io/k6/secretsource"
+)
+
+func init() {
+	secretsource.RegisterExtension("file", func(params secretsource.Params) (secretsource.SecretSource, error) {
+		f, err := params.FS.Open(params.ConfigArgument)
+		if err != nil {
+			return nil, err
+		}
+		scanner := bufio.NewScanner(f)
+
+		r := make(map[string]string)
+		for scanner.Scan() {
+			line := scanner.Text()
+			k, v, ok := strings.Cut(line, "=")
+			if !ok {
+				return nil, fmt.Errorf("parsing %q, needs =", line)
+			}
+
+			r[k] = v
+		}
+		return &fileSecretSource{
+			internal: r,
+		}, nil
+	})
+}
+
+type fileSecretSource struct {
+	internal map[string]string
+	filename string
+}
+
+func (mss *fileSecretSource) Name() string {
+	return "file" // TODO(@mstoykov): make this configurable
+}
+
+func (mss *fileSecretSource) Description() string {
+	return fmt.Sprintf("file source from %s", mss.filename)
+}
+
+func (mss *fileSecretSource) Get(key string) (string, error) {
+	v, ok := mss.internal[key]
+	if !ok {
+		return "", errors.New("no value")
+	}
+	return v, nil
+}

internal/secretsource/init.go

@@ -0,0 +1,7 @@
+package secretsource
+
+// TODO(@mstoykov): do we want this? or do we want to have a function like createOutputs?
+import (
+	_ "go.k6.io/k6/internal/secretsource/file" //nolint:revive
+	_ "go.k6.io/k6/internal/secretsource/mock" //nolint:revive
+)

internal/secretsource/mock/mock.go

@@ -0,0 +1,48 @@
+package mock
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+
+	"go.k6.io/k6/secretsource"
+)
+
+func init() {
+	secretsource.RegisterExtension("mock", func(params secretsource.Params) (secretsource.SecretSource, error) {
+		list := strings.Split(params.ConfigArgument, ":")
+		r := make(map[string]string, len(list))
+		for _, kv := range list {
+			k, v, ok := strings.Cut(kv, "=")
+			if !ok {
+				return nil, fmt.Errorf("parsing %q, needs =", kv)
+			}
+
+			r[k] = v
+		}
+		return &mockSecretSource{
+			internal: r,
+		}, nil
+	})
+}
+
+// TODO remove - this was only for quicker testing
+type mockSecretSource struct {
+	internal map[string]string
+}
+
+func (mss *mockSecretSource) Name() string {
+	return "some"
+}
+
+func (mss *mockSecretSource) Description() string {
+	return "something cool for description"
+}
+
+func (mss *mockSecretSource) Get(key string) (string, error) {
+	v, ok := mss.internal[key]
+	if !ok {
+		return "", errors.New("no value")
+	}
+	return v, nil
+}