Replace kubelet-config with worker-config component

There will be the need to stuff more things into the worker profiles in the future. Prepare for that by refactoring the kubelet-config component into the new worker-config component. Unify / move the worker porfile loading bits found in worker.Kubelet and worker.KubeletConfigClient into the new workerconfig.Loader and use this at the beginning of the k0s worker bootstrapping. Have a new workerconfig.Profile struct that holds the KubeletConfiguration (and more stuff in the future). That struct is somewhat a counterpart to v1beta1.ClusterSpec for workers. Signed-off-by: Tom Wieczorek <[email protected]>
wizpresso-steve-cy-fan · Dec 15, 2022 · 9d0a317 · 9d0a317
1 parent e1917e7
commit 9d0a317
Show file tree

Hide file tree

Showing 25 changed files with 1,957 additions and 173 deletions.
diff --git a/cmd/api/api.go b/cmd/api/api.go
@@ -79,7 +79,7 @@ func NewAPICmd() *cobra.Command {
 
 func (c *command) start() (err error) {
 	// Single kube client for whole lifetime of the API
-	c.client, err = kubeutil.NewClient(c.K0sVars.AdminKubeConfigPath)
+	c.client, err = kubeutil.NewClientFromFile(c.K0sVars.AdminKubeConfigPath)
 	if err != nil {
 		return err
 	}

diff --git a/cmd/controller/controller.go b/cmd/controller/controller.go
@@ -38,6 +38,7 @@ import (
 	"github.com/k0sproject/k0s/pkg/component/controller"
 	"github.com/k0sproject/k0s/pkg/component/controller/clusterconfig"
 	"github.com/k0sproject/k0s/pkg/component/controller/leaderelector"
+	"github.com/k0sproject/k0s/pkg/component/controller/workerconfig"
 	"github.com/k0sproject/k0s/pkg/component/manager"
 	"github.com/k0sproject/k0s/pkg/component/prober"
 	"github.com/k0sproject/k0s/pkg/component/status"
@@ -429,7 +430,12 @@ func (c *command) start(ctx context.Context) error {
 		c.ClusterComponents.Add(ctx, metrics)
 	}
 
-	if !slices.Contains(c.DisableComponents, constant.KubeletConfigComponentName) {
+	if !slices.Contains(c.DisableComponents, constant.WorkerConfigComponentName) {
+		reconciler, err := workerconfig.NewReconciler(c.K0sVars, c.NodeConfig.Spec, adminClientFactory, leaderElector)
+		if err != nil {
+			return err
+		}
+		c.ClusterComponents.Add(ctx, reconciler)
 		c.ClusterComponents.Add(ctx, controller.NewKubeletConfig(c.K0sVars, adminClientFactory))
 	}
 

diff --git a/cmd/worker/worker.go b/cmd/worker/worker.go
@@ -31,7 +31,9 @@ import (
 	"github.com/k0sproject/k0s/pkg/component/prober"
 	"github.com/k0sproject/k0s/pkg/component/status"
 	"github.com/k0sproject/k0s/pkg/component/worker"
+	workerconfig "github.com/k0sproject/k0s/pkg/component/worker/config"
 	"github.com/k0sproject/k0s/pkg/config"
+	"github.com/k0sproject/k0s/pkg/kubernetes"
 
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
@@ -106,7 +108,12 @@ func (c *Command) Start(ctx context.Context) error {
 		return err
 	}
 
-	kubeletConfigClient, err := worker.LoadKubeletConfigClient(c.K0sVars)
+	kubeletKubeconfigPath := c.K0sVars.KubeletAuthConfigPath
+	workerConfig, err := workerconfig.LoadProfile(
+		ctx,
+		kubernetes.KubeconfigFromFile(kubeletKubeconfigPath),
+		c.WorkerProfile,
+	)
 	if err != nil {
 		return err
 	}
@@ -132,9 +139,9 @@ func (c *Command) Start(ctx context.Context) error {
 		CRISocket:           c.CriSocket,
 		EnableCloudProvider: c.CloudProvider,
 		K0sVars:             c.K0sVars,
-		KubeletConfigClient: kubeletConfigClient,
+		Kubeconfig:          kubeletKubeconfigPath,
+		Configuration:       *workerConfig.KubeletConfiguration.DeepCopy(),
 		LogLevel:            c.Logging["kubelet"],
-		Profile:             c.WorkerProfile,
 		Labels:              c.Labels,
 		Taints:              c.Taints,
 		ExtraArgs:           c.KubeletExtraArgs,
@@ -158,7 +165,7 @@ func (c *Command) Start(ctx context.Context) error {
 		})
 	}
 
-	certManager := worker.NewCertificateManager(ctx, c.K0sVars.KubeletAuthConfigPath)
+	certManager := worker.NewCertificateManager(ctx, kubeletKubeconfigPath)
 	if !c.SingleNode && !c.EnableWorker {
 		clusterConfig, err := config.LoadClusterConfig(c.K0sVars)
 		if err != nil {
@@ -202,7 +209,7 @@ func (c *Command) Start(ctx context.Context) error {
 
 	// Stop components
 	if err := componentManager.Stop(); err != nil {
-		logrus.WithError(err).Error("error while stoping component manager")
+		logrus.WithError(err).Error("error while stopping component manager")
 	}
 	return nil
 }
diff --git a/docs/configuration.md b/docs/configuration.md
@@ -243,18 +243,23 @@ CALICO_IPV6POOL_CIDR: "{{ spec.network.dualStack.IPv6podCIDR }}"
 
 ### `spec.workerProfiles`
 
-Worker profiles are used to set kubelet parameters can for a worker. Each worker profile is then used to generate a config map containing a custom `kubelet.config.k8s.io` object.
+Worker profiles are used to manage worker-specific configuration in a
+centralized manner. A ConfigMap is generated for each worker profile. Based on
+the `--profile` argument given to the `k0s worker`, the configuration in the
+corresponding ConfigMap is is picked up during startup.
 
-For a list of possible kubelet configuration keys: [go here](https://kubernetes.io/docs/reference/config-api/kubelet-config.v1beta1/).
+The worker profiles are defined as an array. Each element has following
+properties:
 
-The worker profiles are defined as an array of `spec.workerProfiles.workerProfile`. Each element has following properties:
+| Property | Description                                                                      |
+| -------- | -------------------------------------------------------------------------------- |
+| `name`   | String; name to use as profile selector for the worker process                   |
+| `values` | Object; [Kubelet configuration][kubelet-config] overrides, see below for details |
 
-| Property | Description                                                    |
-| -------- | -------------------------------------------------------------- |
-| `name`   | String; name to use as profile selector for the worker process |
-| `values` | Mapping object                                                 |
+#### `spec.workerProfiles[].values` (Kubelet configuration overrides)
 
-For each profile, the control plane creates a separate ConfigMap with `kubelet-config yaml`. Based on the `--profile` argument given to the `k0s worker`, the corresponding ConfigMap is used to extract the `kubelet-config.yaml` file. `values` are recursively merged with default `kubelet-config.yaml`
+The Kubelet configuration overrides of a profile override the defaults defined
+by k0s.
 
 Note that there are several fields that cannot be overridden:
 
@@ -264,6 +269,8 @@ Note that there are several fields that cannot be overridden:
 - `kind`
 - `staticPodURL`
 
+[kubelet-config]: https://kubernetes.io/docs/reference/config-api/kubelet-config.v1beta1/
+
 #### Examples
 
 ##### Feature Gates
@@ -403,21 +410,30 @@ spec:
 
 ## Disabling controller components
 
-k0s allows completely disabling some of the system components. This allows the user to build a minimal Kubernetes control plane and use what ever components they need to fullfill their need for the controlplane. Disabling the system components happens through a commandline flag for the controller process:
+k0s allows to completely disable some of the system components. This allows
+users to build a minimal Kubernetes control plane and use what ever components
+they need to fulfill their need for the control plane. Disabling the system
+components happens through a command line flag for the controller process:
 
 ```sh
---disable-components strings                     disable components (valid items: api-config,autopilot,control-api,coredns,csr-approver,endpoint-reconciler,helm,konnectivity-server,kube-controller-manager,kube-proxy,kube-scheduler,kubelet-config,metrics-server,network-provider,node-role,system-rbac)
-
+--disable-components strings                     disable components (valid items: api-config,autopilot,control-api,coredns,csr-approver,endpoint-reconciler,helm,konnectivity-server,kube-controller-manager,kube-proxy,kube-scheduler,metrics-server,network-provider,node-role,system-rbac,worker-config)
 ```
 
-If you use k0sctl just add the flag when installing the cluster for the first controller at `spec.hosts.installFlags` in the config file like e.g.:
+**Note:** As of k0s 1.26, the kubelet-config component has been replaced by the
+worker-config component. k0s will issue a warning when the old component name is
+being used. It is scheduled for removal in k0s 1.27. Please update to the new
+component name.
+
+If you use k0sctl, just add the flag when installing the cluster for the first
+controller at `spec.hosts.installFlags` in the config file like e.g.:
 
 ```yaml
 spec:
   hosts:
   - role: controller
     installFlags:
-    - --disable-components metrics-server
+    - --disable-components=metrics-server
 ```
 
-As seen from the component list, the only always-on component is the Kubernetes api-server, without that k0s serves no purpose.
+As seen from the component list, the only always-on component is the Kubernetes
+api-server, without that k0s serves no purpose.
diff --git a/docs/worker-node-config.md b/docs/worker-node-config.md
@@ -46,17 +46,25 @@ controller0   [map[effect:NoSchedule key:node-role.kubernetes.io/master]]
 worker0       <none>
 ```
 
-## Kubelet args
+## Kubelet configuration
 
-The `k0s worker` command accepts a generic flag to pass in any set of arguments for kubelet process.
+The `k0s worker` command accepts a generic flag to pass in any set of arguments
+for kubelet process.
 
-For example, running `k0s worker --token-file=k0s.token --kubelet-extra-args="--node-ip=1.2.3.4 --address=0.0.0.0"` passes in the given flags to kubelet as-is. As such, you must confirm that any flags you are passing in are properly formatted and valued as k0s will not validate those flags.
+For example, running `k0s worker --token-file=k0s.token
+--kubelet-extra-args="--node-ip=1.2.3.4 --address=0.0.0.0"` passes in the given
+flags to Kubelet as-is. As such, you must confirm that any flags you are passing
+in are properly formatted and valued as k0s will not validate those flags.
 
-## Worker Profiles
+### Worker Profiles
 
-kubelet parameters can also be set via a worker profile. Worker profiles are defined in the main k0s.yaml and are used to generate a config map containing a custom `kubelet.config.k8s.io` object.
-To see examples of k0s.yaml containing worker profiles: [go here](./configuration.md#specworkerprofiles).
-For a list of possible kubelet configuration keys: [go here](https://kubernetes.io/docs/reference/config-api/kubelet-config.v1beta1/).
+Kubelet configuration fields can also be set via a worker profiles. Worker
+profiles are defined in the main k0s.yaml and are used to generate ConfigMaps
+containing a custom `kubelet.config.k8s.io/v1beta1/KubeletConfiguration` object.
+To see examples of k0s.yaml containing worker profiles: [go
+here](./configuration.md#specworkerprofiles). For a list of possible Kubelet
+configuration fields: [go
+here](https://kubernetes.io/docs/reference/config-api/kubelet-config.v1beta1/).
 
 ## IPTables Mode
 

diff --git a/inttest/disabledcomponents/disabled_components_test.go b/inttest/disabledcomponents/disabled_components_test.go
@@ -32,7 +32,7 @@ type DisabledComponentsSuite struct {
 
 func (s *DisabledComponentsSuite) TestK0sGetsUp() {
 
-	s.NoError(s.InitController(0, "--disable-components konnectivity-server,kube-scheduler,kube-controller-manager,control-api,csr-approver,kube-proxy,coredns,network-provider,helm,metrics-server,kubelet-config,system-rbac"))
+	s.NoError(s.InitController(0, "--disable-components control-api,coredns,csr-approver,helm,konnectivity-server,kube-controller-manager,kube-proxy,kube-scheduler,metrics-server,network-provider,system-rbac,worker-config"))
 
 	kc, err := s.KubeClient(s.ControllerNode(0))
 	s.Require().NoError(err)

diff --git a/pkg/component/controller/clusterConfig.go b/pkg/component/controller/clusterConfig.go
@@ -22,7 +22,7 @@ import (
 	"os"
 	"time"
 
-	cfgClient "github.com/k0sproject/k0s/pkg/apis/k0s.k0sproject.io/clientset/typed/k0s.k0sproject.io/v1beta1"
+	k0sclient "github.com/k0sproject/k0s/pkg/apis/k0s.k0sproject.io/clientset/typed/k0s.k0sproject.io/v1beta1"
 	"github.com/k0sproject/k0s/pkg/apis/k0s.k0sproject.io/v1beta1"
 	"github.com/k0sproject/k0s/pkg/component/controller/clusterconfig"
 	"github.com/k0sproject/k0s/pkg/component/controller/leaderelector"
@@ -47,8 +47,7 @@ type ClusterConfigReconciler struct {
 	ComponentManager  *manager.Manager
 	KubeClientFactory kubeutil.ClientFactoryInterface
 
-	configClient  cfgClient.ClusterConfigInterface
-	kubeConfig    string
+	configClient  k0sclient.ClusterConfigInterface
 	leaderElector leaderelector.Interface
 	log           *logrus.Entry
 	saver         manifestsSaver
@@ -72,7 +71,6 @@ func NewClusterConfigReconciler(leaderElector leaderelector.Interface, k0sVars c
 		ComponentManager:  mgr,
 		YamlConfig:        cfg,
 		KubeClientFactory: kubeClientFactory,
-		kubeConfig:        k0sVars.AdminKubeConfigPath,
 		leaderElector:     leaderElector,
 		log:               logrus.WithFields(logrus.Fields{"component": "clusterConfig-reconciler"}),
 		saver:             s,

diff --git a/pkg/component/controller/kubeletconfig.go b/pkg/component/controller/kubeletconfig.go
@@ -180,6 +180,15 @@ func (k *KubeletConfig) save(data []byte) error {
 	if err := file.WriteContentAtomically(filePath, data, constant.CertMode); err != nil {
 		return fmt.Errorf("can't write kubelet configuration config map: %v", err)
 	}
+
+	deprecationNotice := []byte(`The kubelet-config component has been replaced by the worker-config component in k0s 1.26.
+It is scheduled for removal in k0s 1.27.
+`)
+
+	if err := file.WriteContentAtomically(filepath.Join(kubeletDir, "deprecated.txt"), deprecationNotice, constant.CertMode); err != nil {
+		k.log.WithError(err).Warn("Failed to write deprecation notice")
+	}
+
 	return nil
 }
 
@@ -257,6 +266,12 @@ kind: ConfigMap
 metadata:
   name: {{.Name}}
   namespace: kube-system
+  labels:
+    k0s.k0sproject.io/deprecated-since: "1.26"
+  annotations:
+    k0s.k0sproject.io/deprecated: |
+      The kubelet-config component has been replaced by the worker-config component in k0s 1.26.
+      It is scheduled for removal in k0s 1.27.
 data:
   kubelet: |
 {{ .KubeletConfigYAML | nindent 4 }}
@@ -268,6 +283,12 @@ kind: Role
 metadata:
   name: system:bootstrappers:kubelet-configmaps
   namespace: kube-system
+  labels:
+    k0s.k0sproject.io/deprecated-since: "1.26"
+  annotations:
+    k0s.k0sproject.io/deprecated: |
+      The kubelet-config component has been replaced by the worker-config component in k0s 1.26.
+      It is scheduled for removal in k0s 1.27.
 rules:
 - apiGroups: [""]
   resources: ["configmaps"]
@@ -282,6 +303,12 @@ kind: RoleBinding
 metadata:
   name: system:bootstrappers:kubelet-configmaps
   namespace: kube-system
+  labels:
+    k0s.k0sproject.io/deprecated-since: "1.26"
+  annotations:
+    k0s.k0sproject.io/deprecated: |
+      The kubelet-config component has been replaced by the worker-config component in k0s 1.26.
+      It is scheduled for removal in k0s 1.27.
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: Role

diff --git a/pkg/component/controller/kubeletconfig_test.go b/pkg/component/controller/kubeletconfig_test.go
@@ -30,9 +30,8 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-var k0sVars = constant.GetConfig("")
-
 func Test_KubeletConfig(t *testing.T) {
+	k0sVars := constant.GetConfig(t.TempDir())
 	dnsAddr, _ := cfg.Spec.Network.DNSAddress()
 	t.Run("default_profile_only", func(t *testing.T) {
 		k := NewKubeletConfig(k0sVars, testutil.NewFakeClientFactory())
@@ -118,6 +117,7 @@ func Test_KubeletConfig(t *testing.T) {
 }
 
 func defaultConfigWithUserProvidedProfiles(t *testing.T) *KubeletConfig {
+	k0sVars := constant.GetConfig(t.TempDir())
 	k := NewKubeletConfig(k0sVars, testutil.NewFakeClientFactory())
 
 	cfgProfileX := map[string]interface{}{