Patch kube-controller-manager static manifest

Signed-off-by: Artiom Diomin <[email protected]>

pkg/certificate/cabundle/ca-bundle.go

@@ -26,10 +26,11 @@ import (
 )
 
 const (
-	CertsDir        = "/etc/kubeone/certs"
-	FileName        = "ca-certificates.crt"
-	SSLCertFilePath = CertsDir + "/" + FileName
-	ConfigMapName   = "ca-bundle"
+	OriginalCertsDir = "/etc/ssl/certs"
+	CustomCertsDir   = "/etc/kubeone/certs"
+	FileName         = "ca-certificates.crt"
+	SSLCertFilePath  = CustomCertsDir + "/" + FileName
+	ConfigMapName    = "ca-bundle"
 
 	SSLCertFileENV = "SSL_CERT_FILE"
 )
@@ -74,7 +75,7 @@ func VolumeMount() corev1.VolumeMount {
 	return corev1.VolumeMount{
 		Name:      ConfigMapName,
 		ReadOnly:  true,
-		MountPath: CertsDir,
+		MountPath: CustomCertsDir,
 	}
 }
 
@@ -94,6 +95,6 @@ func Volume() corev1.Volume {
 func EnvVar() corev1.EnvVar {
 	return corev1.EnvVar{
 		Name:  SSLCertFileENV,
-		Value: filepath.Join(CertsDir, FileName),
+		Value: filepath.Join(CustomCertsDir, FileName),
 	}
 }

pkg/cmd/config.go

@@ -666,13 +666,14 @@ features:
       # authorities in the oidc-ca-file, otherwise the host's root CA set will
       # be used.
       caFile: ""
+
   # Enable Kubernetes Encryption Providers
   # For more information: https://kubernetes.io/docs/tasks/administer-cluster/encrypt-data/
   encryptionProviders:
     # disabled by default
     enable: {{ .EnableEncryptionProviders }}
-    customEncryptionConfiguration: # inline string
-      caFile: ""
+    # inline string
+    customEncryptionConfiguration: ""
 
 ## Bundle of Root CA Certificates extracted from Mozilla
 ## can be found here: https://curl.se/ca/cacert.pem

pkg/scripts/configs.go

@@ -100,7 +100,7 @@ func DeleteEncryptionProvidersConfig(fileName string) string {
 func SaveCABundle(workdir string) (string, error) {
 	return Render(caBundleTemplate, Data{
 		"CA_BUNDLE_FILENAME": cabundle.FileName,
-		"CA_CERTS_DIR":       cabundle.CertsDir,
+		"CA_CERTS_DIR":       cabundle.CustomCertsDir,
 		"WORK_DIR":           workdir,
 	})
 }

pkg/tasks/certs.go

@@ -142,7 +142,7 @@ func earliestCertExpiry(conn ssh.Connection) (time.Time, error) {
 	return earliestCertExpirationTime, nil
 }
 
-func ensureConfigMap(s *state.State) error {
+func ensureCABundleConfigMap(s *state.State) error {
 	s.Logger.Infoln("Creating ca-bundle configMap...")
 
 	cm := cabundle.ConfigMap(s.Cluster.CABundle)

pkg/tasks/encryption_providers.go

@@ -39,7 +39,7 @@ import (
 )
 
 // download the configuration from leader
-func FetchEncryptionProvidersFile(s *state.State) error {
+func fetchEncryptionProvidersFile(s *state.State) error {
 	s.Logger.Infof("Downloading EncryptionProviders configuration file...")
 	host, err := s.Cluster.Leader()
 	if err != nil {
@@ -62,7 +62,7 @@ func FetchEncryptionProvidersFile(s *state.State) error {
 	return err
 }
 
-func UploadIdentityFirstEncryptionConfiguration(s *state.State) error {
+func uploadIdentityFirstEncryptionConfiguration(s *state.State) error {
 	s.Logger.Infof("Uploading EncryptionProviders configuration file...")
 
 	if s.LiveCluster.EncryptionConfiguration == nil ||
@@ -88,7 +88,7 @@ func UploadIdentityFirstEncryptionConfiguration(s *state.State) error {
 	return s.RunTaskOnControlPlane(pushEncryptionConfigurationOnNode, state.RunParallel)
 }
 
-func UploadEncryptionConfigurationWithNewKey(s *state.State) error {
+func uploadEncryptionConfigurationWithNewKey(s *state.State) error {
 	s.Logger.Infof("Uploading EncryptionProviders configuration file...")
 
 	if s.LiveCluster.EncryptionConfiguration == nil ||
@@ -109,7 +109,7 @@ func UploadEncryptionConfigurationWithNewKey(s *state.State) error {
 	return s.RunTaskOnControlPlane(pushEncryptionConfigurationOnNode, state.RunParallel)
 }
 
-func UploadEncryptionConfigurationWithoutOldKey(s *state.State) error {
+func uploadEncryptionConfigurationWithoutOldKey(s *state.State) error {
 	s.Logger.Infof("Uploading EncryptionProviders configuration file...")
 
 	if s.LiveCluster.EncryptionConfiguration == nil ||
@@ -141,7 +141,7 @@ func pushEncryptionConfigurationOnNode(s *state.State, node *kubeoneapi.HostConf
 	return err
 }
 
-func RewriteClusterSecrets(s *state.State) error {
+func rewriteClusterSecrets(s *state.State) error {
 	s.Logger.Infof("Rewriting cluster secrets...")
 	secrets := corev1.SecretList{}
 	err := s.DynamicClient.List(context.Background(), &secrets, &dynclient.ListOptions{})
@@ -163,7 +163,7 @@ func RewriteClusterSecrets(s *state.State) error {
 	return nil
 }
 
-func RemoveEncryptionProviderFile(s *state.State) error {
+func removeEncryptionProviderFile(s *state.State) error {
 	s.Logger.Infof("Removing EncryptionProviders configuration file...")
 	return s.RunTaskOnControlPlane(func(s *state.State, _ *kubeoneapi.HostConfig, _ ssh.Connection) error {
 		cmd := scripts.DeleteEncryptionProvidersConfig(s.GetEncryptionProviderConfigName())

pkg/tasks/nodes.go

@@ -17,9 +17,14 @@ limitations under the License.
 package tasks
 
 import (
+	"bytes"
+	"io"
+	"path/filepath"
+
 	"github.com/pkg/errors"
 
 	kubeoneapi "k8c.io/kubeone/pkg/apis/kubeone"
+	"k8c.io/kubeone/pkg/certificate/cabundle"
 	"k8c.io/kubeone/pkg/scripts"
 	"k8c.io/kubeone/pkg/ssh"
 	"k8c.io/kubeone/pkg/state"
@@ -28,6 +33,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/client-go/util/retry"
+	"sigs.k8s.io/yaml"
 )
 
 func drainNode(s *state.State, node kubeoneapi.HostConfig) error {
@@ -180,3 +186,72 @@ func labelNodeOSes(s *state.State) error {
 
 	return nil
 }
+
+const (
+	getControllerManagerScript = `sudo dd if=/etc/kubernetes/manifests/kube-controller-manager.yaml`
+	putControllerManagerScript = `cat | sudo dd of=/etc/kubernetes/manifests/kube-controller-manager.yaml`
+)
+
+func patchStaticPods(s *state.State) error {
+	return s.RunTaskOnControlPlane(func(s *state.State, node *kubeoneapi.HostConfig, conn ssh.Connection) error {
+		s.Logger.Infoln("Patching static pods...")
+
+		popenConn, ok := conn.(interface {
+			POpen(cmd string, stdin io.Reader, stdout io.Writer, stderr io.Writer) (int, error)
+		})
+		if !ok {
+			return errors.New("ssh.Connection does not implement POpen()")
+		}
+
+		var stdout, stderr bytes.Buffer
+		_, err := popenConn.POpen(getControllerManagerScript, nil, &stdout, &stderr)
+		if err != nil {
+			return errors.Wrapf(err, "failed to get kube-controller-manager.yaml, stderr: %s", stderr.String())
+		}
+
+		pod := corev1.Pod{}
+		if err = yaml.Unmarshal(stdout.Bytes(), &pod); err != nil {
+			return errors.Wrap(err, "failed to unmarshal kube-controller-manager.yaml")
+		}
+
+		cacertDir := cabundle.OriginalCertsDir
+		if s.Cluster.CABundle != "" {
+			cacertDir = cabundle.CustomCertsDir
+		}
+
+		for idx := range pod.Spec.Volumes {
+			volume := pod.Spec.Volumes[idx]
+			if volume.Name == "ca-certs" {
+				volume.HostPath.Path = cacertDir
+			}
+		}
+
+		foundEnvVar := false
+		envVar := corev1.EnvVar{
+			Name:  cabundle.SSLCertFileENV,
+			Value: filepath.Join("/etc/ssl/certs", cabundle.FileName),
+		}
+
+		for idx := range pod.Spec.Containers[0].Env {
+			env := pod.Spec.Containers[0].Env[idx]
+			if env.Name == envVar.Name {
+				env.Value = envVar.Value
+				foundEnvVar = true
+			}
+		}
+		if !foundEnvVar {
+			pod.Spec.Containers[0].Env = append(pod.Spec.Containers[0].Env, envVar)
+		}
+
+		buf, err := yaml.Marshal(&pod)
+		if err != nil {
+			return errors.Wrap(err, "failed to marshal kube-controller-manager.yaml")
+		}
+
+		stdin := bytes.NewBuffer(buf)
+		stderr.Reset()
+		stdout.Reset()
+		_, err = popenConn.POpen(putControllerManagerScript, stdin, &stdout, &stderr)
+		return errors.Wrapf(err, "failed to write kube-controller-manager.yaml\nstdout: %s\nstderr: %s", stdout.String(), stderr.String())
+	}, state.RunParallel)
+}

pkg/tasks/prerequisites.go

@@ -215,17 +215,6 @@ func uploadConfigurationFilesToNode(s *state.State, node *kubeoneapi.HostConfig,
 		return err
 	}
 
-	if s.Cluster.CABundle != "" {
-		cmd, err = scripts.SaveCABundle(s.WorkDir)
-		if err != nil {
-			return err
-		}
-		_, _, err = s.Runner.RunRaw(cmd)
-		if err != nil {
-			return err
-		}
-	}
-
 	return nil
 }
 

pkg/tasks/tasks.go

@@ -20,11 +20,14 @@ import (
 	"github.com/pkg/errors"
 
 	"k8c.io/kubeone/pkg/addons"
+	"k8c.io/kubeone/pkg/apis/kubeone"
 	"k8c.io/kubeone/pkg/certificate"
 	"k8c.io/kubeone/pkg/clusterstatus"
 	"k8c.io/kubeone/pkg/credentials"
 	"k8c.io/kubeone/pkg/features"
 	"k8c.io/kubeone/pkg/kubeconfig"
+	"k8c.io/kubeone/pkg/scripts"
+	"k8c.io/kubeone/pkg/ssh"
 	"k8c.io/kubeone/pkg/state"
 	"k8c.io/kubeone/pkg/templates/externalccm"
 	"k8c.io/kubeone/pkg/templates/machinecontroller"
@@ -126,6 +129,25 @@ func WithFullInstall(t Tasks) Tasks {
 func WithResources(t Tasks) Tasks {
 	return t.append(
 		Tasks{
+			{
+				Fn: func(s *state.State) error {
+					return s.RunTaskOnControlPlane(func(ctx *state.State, node *kubeone.HostConfig, conn ssh.Connection) error {
+						cmd, err := scripts.SaveCABundle(ctx.WorkDir)
+						if err != nil {
+							return err
+						}
+						_, _, err = ctx.Runner.RunRaw(cmd)
+						return err
+					}, state.RunParallel)
+				},
+				Predicate: func(s *state.State) bool {
+					return s.Cluster.CABundle != ""
+				},
+			},
+			{
+				Fn:     patchStaticPods,
+				ErrMsg: "failed to patch static pods",
+			},
 			{
 				Fn:          renewControlPlaneCerts,
 				ErrMsg:      "failed to renew certificates",
@@ -158,7 +180,7 @@ func WithResources(t Tasks) Tasks {
 				Predicate:   func(s *state.State) bool { return s.Cluster.ClusterNetwork.CNI.External == nil },
 			},
 			{
-				Fn:          ensureConfigMap,
+				Fn:          ensureCABundleConfigMap,
 				ErrMsg:      "failed to ensure caBundle configMap",
 				Description: "ensure caBundle configMap",
 				Predicate:   func(s *state.State) bool { return s.Cluster.CABundle != "" },
@@ -268,7 +290,7 @@ func WithDisableEncryptionProviders(t Tasks, customConfig bool) Tasks {
 	if customConfig {
 		return t.append(Tasks{
 			{
-				Fn:          RemoveEncryptionProviderFile,
+				Fn:          removeEncryptionProviderFile,
 				ErrMsg:      "failed to remove encryption providers configuration",
 				Description: "remove old Encryption Providers configuration file",
 			},
@@ -279,19 +301,19 @@ func WithDisableEncryptionProviders(t Tasks, customConfig bool) Tasks {
 			},
 
 			{
-				Fn:          RewriteClusterSecrets,
+				Fn:          rewriteClusterSecrets,
 				ErrMsg:      "failed to rewrite cluster secrets",
 				Description: "rewrite all cluster secrets",
 			},
 		}...)
 	}
 	return t.append(Tasks{
 		{
-			Fn:          FetchEncryptionProvidersFile,
+			Fn:          fetchEncryptionProvidersFile,
 			ErrMsg:      "failed to fetch EncryptionProviders config",
 			Description: "fetch current Encryption Providers configuration file "},
 		{
-			Fn:          UploadIdentityFirstEncryptionConfiguration,
+			Fn:          uploadIdentityFirstEncryptionConfiguration,
 			ErrMsg:      "failed to upload encryption providers configuration",
 			Description: "upload updated Encryption Providers configuration file"},
 		{
@@ -300,12 +322,12 @@ func WithDisableEncryptionProviders(t Tasks, customConfig bool) Tasks {
 			Description: "restart KubeAPI containers",
 		},
 		{
-			Fn:          RewriteClusterSecrets,
+			Fn:          rewriteClusterSecrets,
 			ErrMsg:      "failed to rewrite cluster secrets",
 			Description: "rewrite all cluster secrets",
 		},
 		{
-			Fn:          RemoveEncryptionProviderFile,
+			Fn:          removeEncryptionProviderFile,
 			ErrMsg:      "failed to remove encryption providers configuration",
 			Description: "remove old Encryption Providers configuration file",
 		},
@@ -315,7 +337,7 @@ func WithDisableEncryptionProviders(t Tasks, customConfig bool) Tasks {
 func WithRewriteSecrets(t Tasks) Tasks {
 	return t.append(
 		Task{
-			Fn:          RewriteClusterSecrets,
+			Fn:          rewriteClusterSecrets,
 			ErrMsg:      "failed to rewrite cluster secrets",
 			Description: "rewrite all cluster secrets",
 		})
@@ -325,12 +347,12 @@ func WithRotateKey(t Tasks) Tasks {
 	return WithHostnameOSAndProbes(t).
 		append(Tasks{
 			{
-				Fn:          FetchEncryptionProvidersFile,
+				Fn:          fetchEncryptionProvidersFile,
 				ErrMsg:      "failed to fetch EncryptionProviders config",
 				Description: "fetch current Encryption Providers configuration file ",
 			},
 			{
-				Fn:          UploadEncryptionConfigurationWithNewKey,
+				Fn:          uploadEncryptionConfigurationWithNewKey,
 				ErrMsg:      "failed to upload encryption providers configuration",
 				Description: "upload updated Encryption Providers configuration file",
 			},
@@ -340,12 +362,12 @@ func WithRotateKey(t Tasks) Tasks {
 				Description: "restart KubeAPI containers",
 			},
 			{
-				Fn:          RewriteClusterSecrets,
+				Fn:          rewriteClusterSecrets,
 				ErrMsg:      "failed to rewrite cluster secrets",
 				Description: "rewrite all cluster secrets",
 			},
 			{
-				Fn:          UploadEncryptionConfigurationWithoutOldKey,
+				Fn:          uploadEncryptionConfigurationWithoutOldKey,
 				ErrMsg:      "failed to upload encryption providers configuration",
 				Description: "upload updated Encryption Providers configuration file",
 			},

pkg/templates/kubeadm/v1beta2/kubeadm.go

@@ -203,20 +203,6 @@ func NewConfig(s *state.State, host kubeoneapi.HostConfig) ([]runtime.Object, er
 		}
 	}
 
-	// TODO: Fix double mounted /etc/ssl/certs
-	// if cluster.CABundle != "" {
-	// 	clusterConfig.ControllerManager.ExtraVolumes = append(
-	// 		clusterConfig.ControllerManager.ExtraVolumes, kubeadmv1beta2.HostPathMount{
-	// 			Name:     "ca-bundle",
-	// 			HostPath: cabundle.CertsDir,
-	// 			// this is path that k8s.gcr.io/kube-* images use in ENV
-	// 			// "SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt"
-	// 			MountPath: "/etc/ssl/certs",
-	// 			ReadOnly:  true,
-	// 		},
-	// 	)
-	// }
-
 	if cluster.CloudProvider.External {
 		delete(clusterConfig.APIServer.ExtraArgs, "cloud-provider")
 		delete(clusterConfig.ControllerManager.ExtraArgs, "cloud-provider")