Patch CNI (Flannel) when external CCM is working

* add FLANNELD_IFACE = status.hostIP env to flannel
* when external CCM is deployed, wait for Nodes to be initialized
* introduced new package clientutil
* removed all simpleCreateOrUpdate and placed with clientutil.CreateOrUpdate
* FIX UPDATES of the objects
* removed TypeMeta
* group kubernetes objects to be deployed into packs

Signed-off-by: Artiom Diomin <[email protected]>

pkg/certificate/ca.go

@@ -27,6 +27,8 @@ import (
 
 // DownloadCA grabs CA certs/keys from leader host
 func DownloadCA(s *state.State) error {
+	s.Logger.Info("Downloading PKI…")
+
 	return s.RunTaskOnLeader(func(s *state.State, _ *kubeoneapi.HostConfig, conn ssh.Connection) error {
 		_, _, err := s.Runner.Run(`
 mkdir -p ./{{ .WORK_DIR }}/pki/etcd

pkg/clientutil/create_or_update.go

@@ -0,0 +1,57 @@
+/*
+Copyright 2019 The KubeOne Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package clientutil
+
+import (
+	"context"
+
+	"github.com/pkg/errors"
+
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+// CreateOrUpdate makes it easy to "apply" objects to kubernetes API server
+func CreateOrUpdate(ctx context.Context, c client.Client, obj runtime.Object) error {
+	existing := obj.DeepCopyObject()
+	existingMetaObj, ok := existing.(metav1.Object)
+	if !ok {
+		return errors.Errorf("%T does not implement metav1.Object interface", obj)
+	}
+
+	key := client.ObjectKey{
+		Name:      existingMetaObj.GetName(),
+		Namespace: existingMetaObj.GetNamespace(),
+	}
+
+	err := c.Get(ctx, key, existing)
+
+	switch {
+	case k8serrors.IsNotFound(err):
+		return errors.Wrapf(c.Create(ctx, obj), "failed to create %T", obj)
+	case err != nil:
+		return errors.Wrapf(err, "failed to get %T object", obj)
+	}
+
+	updateMetaObj := obj.(metav1.Object)
+	updateMetaObj.SetResourceVersion(existingMetaObj.GetResourceVersion())
+	obj = updateMetaObj.(runtime.Object)
+
+	return errors.Wrapf(c.Update(ctx, obj), "failed to update %T object", obj)
+}

pkg/credentials/secret.go

@@ -21,13 +21,11 @@ import (
 
 	"github.com/pkg/errors"
 
+	"github.com/kubermatic/kubeone/pkg/clientutil"
 	"github.com/kubermatic/kubeone/pkg/state"
 
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/runtime"
-	dynclient "sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 )
 
 const (
@@ -37,12 +35,6 @@ const (
 	SecretNamespace = "kube-system"
 )
 
-func simpleCreateOrUpdate(ctx context.Context, client dynclient.Client, obj runtime.Object) error {
-	okFunc := func(runtime.Object) error { return nil }
-	_, err := controllerutil.CreateOrUpdate(ctx, client, obj, okFunc)
-	return err
-}
-
 // Ensure creates/updates the credentials secret
 func Ensure(s *state.State) error {
 	if !s.Cluster.MachineController.Deploy && !s.Cluster.CloudProvider.External {
@@ -57,9 +49,8 @@ func Ensure(s *state.State) error {
 		return errors.Wrap(err, "unable to fetch cloud provider credentials")
 	}
 
-	bgCtx := context.Background()
 	secret := credentialsSecret(creds)
-	if err := simpleCreateOrUpdate(bgCtx, s.DynamicClient, secret); err != nil {
+	if err := clientutil.CreateOrUpdate(context.Background(), s.DynamicClient, secret); err != nil {
 		return errors.Wrap(err, "failed to ensure credentials secret")
 	}
 
@@ -68,10 +59,6 @@ func Ensure(s *state.State) error {
 
 func credentialsSecret(credentials map[string]string) *corev1.Secret {
 	return &corev1.Secret{
-		TypeMeta: metav1.TypeMeta{
-			APIVersion: "v1",
-			Kind:       "Secret",
-		},
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      SecretName,
 			Namespace: SecretNamespace,

pkg/features/activate.go

@@ -27,6 +27,8 @@ import (
 // Activate configured features.
 // Installing CRDs, creating policies and so on
 func Activate(s *state.State) error {
+	s.Logger.Info("Activating additional features…")
+
 	if err := installKubeSystemPSP(s.Cluster.Features.PodSecurityPolicy, s); err != nil {
 		return errors.Wrap(err, "failed to install PodSecurityPolicy")
 	}

pkg/features/psp.go

@@ -22,6 +22,7 @@ import (
 	"github.com/pkg/errors"
 
 	kubeoneapi "github.com/kubermatic/kubeone/pkg/apis/kubeone"
+	"github.com/kubermatic/kubeone/pkg/clientutil"
 	"github.com/kubermatic/kubeone/pkg/state"
 	"github.com/kubermatic/kubeone/pkg/templates/kubeadm/kubeadmargs"
 
@@ -30,7 +31,6 @@ import (
 	rbacv1 "k8s.io/api/rbac/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
-	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 )
 
 const (
@@ -52,33 +52,26 @@ func installKubeSystemPSP(psp *kubeoneapi.PodSecurityPolicy, s *state.State) err
 		return nil
 	}
 
-	bgContext := context.Background()
-	okFunc := func(runtime.Object) error { return nil }
-
-	_, err := controllerutil.CreateOrUpdate(bgContext, s.DynamicClient, privilegedPSP(), okFunc)
-	if err != nil {
-		return errors.Wrap(err, "failed to ensure PodSecurityPolicy")
-	}
-
-	_, err = controllerutil.CreateOrUpdate(bgContext, s.DynamicClient, privilegedPSPClusterRole(), okFunc)
-	if err != nil {
-		return errors.Wrap(err, "failed to ensure PodSecurityPolicy cluster role")
+	ctx := context.Background()
+	k8sobjects := []runtime.Object{
+		privilegedPSP(),
+		privilegedPSPClusterRole(),
+		privilegedPSPRoleBinding(),
 	}
 
-	_, err = controllerutil.CreateOrUpdate(bgContext, s.DynamicClient, privilegedPSPRoleBinding(), okFunc)
-	if err != nil {
-		return errors.Wrap(err, "failed to ensure PodSecurityPolicy role binding")
+	for _, obj := range k8sobjects {
+		if err := clientutil.CreateOrUpdate(ctx, s.DynamicClient, obj); err != nil {
+			return errors.Wrap(err, "failed to ensure PodSecurityPolicy role binding")
+		}
 	}
 
 	return nil
 }
 
 func privilegedPSP() *policybeta1.PodSecurityPolicy {
+	t := true
+
 	return &policybeta1.PodSecurityPolicy{
-		TypeMeta: metav1.TypeMeta{
-			APIVersion: "policy/v1beta1",
-			Kind:       "PodSecurityPolicy",
-		},
 		ObjectMeta: metav1.ObjectMeta{
 			Name: "privileged",
 		},
@@ -87,7 +80,7 @@ func privilegedPSP() *policybeta1.PodSecurityPolicy {
 			HostNetwork:              true,
 			HostIPC:                  true,
 			HostPID:                  true,
-			AllowPrivilegeEscalation: boolPtr(true),
+			AllowPrivilegeEscalation: &t,
 			AllowedCapabilities:      []corev1.Capability{"*"},
 			Volumes:                  []policybeta1.FSType{policybeta1.All},
 			HostPorts: []policybeta1.HostPortRange{
@@ -111,10 +104,6 @@ func privilegedPSP() *policybeta1.PodSecurityPolicy {
 
 func privilegedPSPClusterRole() *rbacv1.ClusterRole {
 	return &rbacv1.ClusterRole{
-		TypeMeta: metav1.TypeMeta{
-			APIVersion: "rbac.authorization.k8s.io/v1",
-			Kind:       "ClusterRole",
-		},
 		ObjectMeta: metav1.ObjectMeta{
 			Name: "privileged-psp",
 		},
@@ -131,10 +120,6 @@ func privilegedPSPClusterRole() *rbacv1.ClusterRole {
 
 func privilegedPSPRoleBinding() *rbacv1.RoleBinding {
 	return &rbacv1.RoleBinding{
-		TypeMeta: metav1.TypeMeta{
-			APIVersion: "rbac.authorization.k8s.io/v1",
-			Kind:       "RoleBinding",
-		},
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "privileged-psp",
 			Namespace: pspRoleNamespace,
@@ -158,7 +143,3 @@ func privilegedPSPRoleBinding() *rbacv1.RoleBinding {
 		},
 	}
 }
-
-func boolPtr(b bool) *bool {
-	return &b
-}

pkg/installer/installation/cni.go

@@ -45,3 +45,12 @@ func ensureCNICanal(s *state.State) error {
 	s.Logger.Infoln("Applying canal CNI plugin…")
 	return canal.Deploy(s)
 }
+
+func patchCNI(s *state.State) error {
+	if !s.PatchCNI {
+		return nil
+	}
+
+	s.Logger.Info("Patching CNI…")
+	return ensureCNI(s)
+}

pkg/installer/installation/coredns.go

@@ -40,14 +40,14 @@ func patchCoreDNS(s *state.State) error {
 		return errors.New("kubernetes client not initialized")
 	}
 
-	bgCtx := context.Background()
+	ctx := context.Background()
 	dep := &appsv1.Deployment{}
 	key := client.ObjectKey{
 		Name:      "coredns",
 		Namespace: metav1.NamespaceSystem,
 	}
 
-	err := s.DynamicClient.Get(bgCtx, key, dep)
+	err := s.DynamicClient.Get(ctx, key, dep)
 	if err != nil {
 		return errors.Wrap(err, "failed to get coredns deployment")
 	}
@@ -60,5 +60,5 @@ func patchCoreDNS(s *state.State) error {
 		},
 	)
 
-	return errors.Wrap(s.DynamicClient.Update(bgCtx, dep), "failed to update coredns deployment")
+	return errors.Wrap(s.DynamicClient.Update(ctx, dep), "failed to update coredns deployment")
 }

pkg/installer/installation/install.go

@@ -45,10 +45,11 @@ func Install(s *state.State) error {
 		{Fn: saveKubeconfig, ErrMsg: "unable to save kubeconfig to the local machine", Retries: 3},
 		{Fn: kubeconfig.BuildKubernetesClientset, ErrMsg: "unable to build kubernetes clientset", Retries: 3},
 		{Fn: features.Activate, ErrMsg: "unable to activate features"},
-		{Fn: credentials.Ensure, ErrMsg: "unable to ensure credentials secret"},
-		{Fn: externalccm.Ensure, ErrMsg: "failed to install external CCM"},
-		{Fn: patchCoreDNS, ErrMsg: "failed to patch CoreDNS", Retries: 3},
 		{Fn: ensureCNI, ErrMsg: "failed to install cni plugin", Retries: 3},
+		{Fn: patchCoreDNS, ErrMsg: "failed to patch CoreDNS", Retries: 3},
+		{Fn: credentials.Ensure, ErrMsg: "unable to ensure credentials secret", Retries: 3},
+		{Fn: externalccm.Ensure, ErrMsg: "failed to install external CCM", Retries: 3},
+		{Fn: patchCNI, ErrMsg: "failed to patch CNI", Retries: 3},
 		{Fn: machinecontroller.Ensure, ErrMsg: "failed to install machine-controller", Retries: 3},
 		{Fn: machinecontroller.WaitReady, ErrMsg: "failed to wait for machine-controller", Retries: 3},
 		{Fn: createWorkerMachines, ErrMsg: "failed to create worker machines", Retries: 3},

pkg/installer/installation/kubeconfig.go

@@ -47,6 +47,8 @@ sudo chown $(id -u):$(id -g) $HOME/.kube/config
 }
 
 func saveKubeconfig(s *state.State) error {
+	s.Logger.Info("Downloading kubeconfig…")
+
 	kc, err := kubeconfig.Download(s.Cluster)
 	if err != nil {
 		return err

pkg/state/context.go

@@ -47,9 +47,10 @@ type State struct {
 	RemoveBinaries            bool
 	ForceUpgrade              bool
 	UpgradeMachineDeployments bool
+	PatchCNI                  bool
 }
 
-// Clone returns a shallow copy of the context.
+// Clone returns a shallow copy of the State.
 func (s *State) Clone() *State {
 	newState := *s
 	return &newState