From 98be923335e1c3eaf16113e2ad6ef9ef8615fffa Mon Sep 17 00:00:00 2001 From: Tom Wieczorek Date: Thu, 9 Jan 2025 08:33:52 +0100 Subject: [PATCH] Let API subcommand accept the runtime config via stdin The API subcommand is not intended to run as a standalone process. It's always run under the supervision of a k0s controller process. Therefore, the usual configuration loading process is inappropriate. Instead, accept the runtime configuration via stdin. This way, there's no way to fallback to a generated default configuration, or to load the configuration from a possibly existing default configuration file that has nothing to do with the one used by the supervising process. Signed-off-by: Tom Wieczorek --- cmd/api/api.go | 47 ++++++++++++++++------- cmd/controller/controller.go | 7 +--- pkg/component/controller/k0scontrolapi.go | 23 +++++++---- pkg/config/runtime.go | 46 +++++++++++++--------- pkg/config/runtime_test.go | 7 ++-- pkg/supervisor/supervisor.go | 5 +++ 6 files changed, 88 insertions(+), 47 deletions(-) diff --git a/cmd/api/api.go b/cmd/api/api.go index f34e393fd6e4..c784c4e4e6b4 100644 --- a/cmd/api/api.go +++ b/cmd/api/api.go @@ -22,6 +22,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "net/http" "os" "path" @@ -42,37 +43,61 @@ import ( "github.com/sirupsen/logrus" "github.com/spf13/cobra" + "github.com/spf13/pflag" ) func NewAPICmd() *cobra.Command { cmd := &cobra.Command{ Use: "api", Short: "Run the controller API", - Args: cobra.NoArgs, + Long: `Run the controller API. +Reads the runtime configuration from standard input.`, + Args: cobra.NoArgs, PersistentPreRunE: func(cmd *cobra.Command, args []string) error { logrus.SetOutput(cmd.OutOrStdout()) internallog.SetInfoLevel() return config.CallParentPersistentPreRun(cmd, args) }, RunE: func(cmd *cobra.Command, _ []string) error { - opts, err := config.GetCmdOpts(cmd) - if err != nil { - return err - } + var run func() error - run, err := buildServer(opts.K0sVars) - if err != nil { + if runtimeConfig, err := loadRuntimeConfig(cmd.InOrStdin()); err != nil { + return err + } else if run, err = buildServer(runtimeConfig.Spec.K0sVars, runtimeConfig.Spec.NodeConfig); err != nil { return err } return run() }, } - cmd.PersistentFlags().AddFlagSet(config.GetPersistentFlagSet()) + + flags := cmd.Flags() + config.GetPersistentFlagSet().VisitAll(func(f *pflag.Flag) { + switch f.Name { + case "debug", "debugListenOn", "verbose": + flags.AddFlag(f) + } + }) + return cmd } -func buildServer(k0sVars *config.CfgVars) (func() error, error) { +func loadRuntimeConfig(stdin io.Reader) (*config.RuntimeConfig, error) { + logrus.Info("Reading runtime configuration from standard input ...") + bytes, err := io.ReadAll(stdin) + if err != nil { + return nil, fmt.Errorf("failed to read from standard input: %w", err) + } + + runtimeConfig, err := config.ParseRuntimeConfig(bytes) + if err != nil { + return nil, fmt.Errorf("failed to load runtime configuration: %w", err) + } + + return runtimeConfig, nil +} + +func buildServer(k0sVars *config.CfgVars, nodeConfig *v1beta1.ClusterConfig) (func() error, error) { // Single kube client for whole lifetime of the API client, err := kubeutil.NewClientFromFile(k0sVars.AdminKubeConfigPath) if err != nil { @@ -82,10 +107,6 @@ func buildServer(k0sVars *config.CfgVars) (func() error, error) { prefix := "/v1beta1" mux := http.NewServeMux() - nodeConfig, err := k0sVars.NodeConfig() - if err != nil { - return nil, err - } storage := nodeConfig.Spec.Storage if storage.Type == v1beta1.EtcdStorageType && !storage.Etcd.IsExternalClusterUsed() { diff --git a/cmd/controller/controller.go b/cmd/controller/controller.go index 40bcd9ace826..e3f5cb0b9fc3 100644 --- a/cmd/controller/controller.go +++ b/cmd/controller/controller.go @@ -162,7 +162,7 @@ func (c *command) start(ctx context.Context) error { return fmt.Errorf("failed to initialize runtime config: %w", err) } defer func() { - if err := rtc.Cleanup(); err != nil { + if err := rtc.Spec.Cleanup(); err != nil { logrus.WithError(err).Warn("Failed to cleanup runtime config") } }() @@ -326,10 +326,7 @@ func (c *command) start(ctx context.Context) error { } if !c.SingleNode && !slices.Contains(c.DisableComponents, constant.ControlAPIComponentName) { - nodeComponents.Add(ctx, &controller.K0SControlAPI{ - ConfigPath: c.CfgFile, - K0sVars: c.K0sVars, - }) + nodeComponents.Add(ctx, &controller.K0SControlAPI{RuntimeConfig: rtc}) } if !slices.Contains(c.DisableComponents, constant.CsrApproverComponentName) { diff --git a/pkg/component/controller/k0scontrolapi.go b/pkg/component/controller/k0scontrolapi.go index 64da68b51e76..b15109e999c1 100644 --- a/pkg/component/controller/k0scontrolapi.go +++ b/pkg/component/controller/k0scontrolapi.go @@ -17,18 +17,21 @@ limitations under the License. package controller import ( + "bytes" "context" + "io" "os" "github.com/k0sproject/k0s/pkg/component/manager" "github.com/k0sproject/k0s/pkg/config" "github.com/k0sproject/k0s/pkg/supervisor" + "sigs.k8s.io/yaml" ) // K0SControlAPI implements the k0s control API component type K0SControlAPI struct { - ConfigPath string - K0sVars *config.CfgVars + RuntimeConfig *config.RuntimeConfig + supervisor supervisor.Supervisor } @@ -48,15 +51,19 @@ func (m *K0SControlAPI) Start(_ context.Context) error { if err != nil { return err } + + runtimeConfig, err := yaml.Marshal(m.RuntimeConfig) + if err != nil { + return err + } + m.supervisor = supervisor.Supervisor{ Name: "k0s-control-api", BinPath: selfExe, - RunDir: m.K0sVars.RunDir, - DataDir: m.K0sVars.DataDir, - Args: []string{ - "api", - "--data-dir=" + m.K0sVars.DataDir, - }, + RunDir: m.RuntimeConfig.Spec.K0sVars.RunDir, + DataDir: m.RuntimeConfig.Spec.K0sVars.DataDir, + Args: []string{"api"}, + Stdin: func() io.Reader { return bytes.NewReader(runtimeConfig) }, } return m.supervisor.Supervise() diff --git a/pkg/config/runtime.go b/pkg/config/runtime.go index 7bcfded69e9c..f800888835a7 100644 --- a/pkg/config/runtime.go +++ b/pkg/config/runtime.go @@ -42,7 +42,7 @@ const ( var ( ErrK0sNotRunning = errors.New("k0s is not running") ErrK0sAlreadyRunning = errors.New("an instance of k0s is already running") - ErrInvalidRuntimeConfig = errors.New("invalid runtime config") + ErrInvalidRuntimeConfig = errors.New("invalid runtime configuration") ) // Runtime config is a static copy of the start up config and CfgVars that is used by @@ -74,23 +74,11 @@ func LoadRuntimeConfig(k0sVars *CfgVars) (*RuntimeConfigSpec, error) { return migrateLegacyRuntimeConfig(k0sVars, content) } - config := &RuntimeConfig{} - if err := yaml.Unmarshal(content, config); err != nil { - return nil, err - } - - if config.APIVersion != v1beta1.ClusterConfigAPIVersion { - return nil, fmt.Errorf("%w: invalid api version: %s", ErrInvalidRuntimeConfig, config.APIVersion) - } - - if config.Kind != RuntimeConfigKind { - return nil, fmt.Errorf("%w: invalid kind: %s", ErrInvalidRuntimeConfig, config.Kind) + config, err := ParseRuntimeConfig(content) + if err != nil { + return nil, fmt.Errorf("failed to parse runtime configuration: %w", err) } - spec := config.Spec - if spec == nil { - return nil, fmt.Errorf("%w: spec is nil", ErrInvalidRuntimeConfig) - } // If a pid is defined but there's no process found, the instance of k0s is // expected to have died, in which case the existing config is removed and @@ -106,6 +94,28 @@ func LoadRuntimeConfig(k0sVars *CfgVars) (*RuntimeConfigSpec, error) { return spec, nil } +func ParseRuntimeConfig(content []byte) (*RuntimeConfig, error) { + var config RuntimeConfig + + if err := yaml.Unmarshal(content, &config); err != nil { + return nil, err + } + + if config.APIVersion != v1beta1.ClusterConfigAPIVersion { + return nil, fmt.Errorf("%w: invalid api version: %q", ErrInvalidRuntimeConfig, config.APIVersion) + } + + if config.Kind != RuntimeConfigKind { + return nil, fmt.Errorf("%w: invalid kind: %q", ErrInvalidRuntimeConfig, config.Kind) + } + + if config.Spec == nil { + return nil, fmt.Errorf("%w: spec is nil", ErrInvalidRuntimeConfig) + } + + return &config, nil +} + func migrateLegacyRuntimeConfig(k0sVars *CfgVars, content []byte) (*RuntimeConfigSpec, error) { cfg := &v1beta1.ClusterConfig{} @@ -136,7 +146,7 @@ func isLegacy(data []byte) bool { return false } -func NewRuntimeConfig(k0sVars *CfgVars) (*RuntimeConfigSpec, error) { +func NewRuntimeConfig(k0sVars *CfgVars) (*RuntimeConfig, error) { if _, err := LoadRuntimeConfig(k0sVars); err == nil { return nil, ErrK0sAlreadyRunning } @@ -179,7 +189,7 @@ func NewRuntimeConfig(k0sVars *CfgVars) (*RuntimeConfigSpec, error) { return nil, fmt.Errorf("failed to write runtime config: %w", err) } - return cfg.Spec, nil + return cfg, nil } func (r *RuntimeConfigSpec) Cleanup() error { diff --git a/pkg/config/runtime_test.go b/pkg/config/runtime_test.go index 0e57882e5870..f980bdcd5901 100644 --- a/pkg/config/runtime_test.go +++ b/pkg/config/runtime_test.go @@ -110,15 +110,16 @@ func TestNewRuntimeConfig(t *testing.T) { } // create a new runtime config and check if it's valid - spec, err := NewRuntimeConfig(k0sVars) + cfg, err := NewRuntimeConfig(k0sVars) + spec := cfg.Spec assert.NoError(t, err) assert.NotNil(t, spec) assert.Equal(t, tempDir, spec.K0sVars.DataDir) assert.Equal(t, os.Getpid(), spec.Pid) assert.NotNil(t, spec.NodeConfig) - cfg, err := spec.K0sVars.NodeConfig() + nodeConfig, err := spec.K0sVars.NodeConfig() assert.NoError(t, err) - assert.Equal(t, "10.0.0.1", cfg.Spec.API.Address) + assert.Equal(t, "10.0.0.1", nodeConfig.Spec.API.Address) // try to create a new runtime config when one is already active and check if it returns an error _, err = NewRuntimeConfig(k0sVars) diff --git a/pkg/supervisor/supervisor.go b/pkg/supervisor/supervisor.go index 1594ba9551e4..e878fbfdaa89 100644 --- a/pkg/supervisor/supervisor.go +++ b/pkg/supervisor/supervisor.go @@ -20,6 +20,7 @@ import ( "context" "errors" "fmt" + "io" "os" "os/exec" "path" @@ -44,6 +45,7 @@ type Supervisor struct { BinPath string RunDir string DataDir string + Stdin func() io.Reader Args []string PidFile string UID int @@ -174,6 +176,9 @@ func (s *Supervisor) Supervise() error { s.cmd = exec.Command(s.BinPath, s.Args...) s.cmd.Dir = s.DataDir s.cmd.Env = getEnv(s.DataDir, s.Name, s.KeepEnvPrefix) + if s.Stdin != nil { + s.cmd.Stdin = s.Stdin() + } // detach from the process group so children don't // get signals sent directly to parent.