Implement core scripts and logic for running upgrades

pkg/upgrader/upgrade/kubeadm_package.go

@@ -0,0 +1,84 @@
+package upgrade
+
+import (
+	"github.com/pkg/errors"
+
+	"github.com/kubermatic/kubeone/pkg/config"
+	"github.com/kubermatic/kubeone/pkg/installer/util"
+)
+
+const (
+	upgradeKubeadmDebianCommand = `
+source /etc/os-release
+source /etc/kubeone/proxy-env
+
+sudo apt-get update
+
+kube_ver=$(apt-cache madison kubelet | grep "{{ .KUBERNETES_VERSION }}" | head -1 | awk '{print $3}')
+
+sudo apt-mark unhold kubeadm
+sudo apt-get install kubeadm=${kube_ver}
+sudo apt-mark hold kubeadm
+`
+	upgradeKubeadmCentOSCommand = `
+source /etc/kubeone/proxy-env
+
+sudo yum install -y --disableexcludes=kubernetes \
+			kubeadm-{{ .KUBERNETES_VERSION }}-0
+`
+	upgradeKubeadmCoreOSCommand = `
+source /etc/kubeone/proxy-env
+
+RELEASE="v{{ .KUBERNETES_VERSION }}"
+
+sudo mkdir -p /opt/bin
+cd /opt/bin
+sudo curl -L --remote-name-all \
+     https://storage.googleapis.com/kubernetes-release/release/${RELEASE}/bin/linux/amd64/kubeadm
+sudo chmod +x kubeadm
+`
+)
+
+func upgradeKubeadm(ctx *util.Context, node *config.HostConfig) error {
+	var err error
+
+	switch node.OperatingSystem {
+	case "ubuntu", "debian":
+		err = upgradeKubeadmDebian(ctx)
+
+	case "coreos":
+		err = upgradeKubeadmCoreOS(ctx)
+
+	case "centos":
+		err = upgradeKubeadmCentOS(ctx)
+
+	default:
+		err = errors.Errorf("'%s' is not a supported operating system", node.OperatingSystem)
+	}
+
+	return err
+}
+
+func upgradeKubeadmDebian(ctx *util.Context) error {
+	_, _, err := ctx.Runner.Run(upgradeKubeadmDebianCommand, util.TemplateVariables{
+		"KUBERNETES_VERSION": ctx.Cluster.Versions.Kubernetes,
+	})
+
+	return errors.WithStack(err)
+}
+
+func upgradeKubeadmCentOS(ctx *util.Context) error {
+	_, _, err := ctx.Runner.Run(upgradeKubeadmCentOSCommand, util.TemplateVariables{
+		"KUBERNETES_VERSION": ctx.Cluster.Versions.Kubernetes,
+	})
+
+	return errors.WithStack(err)
+}
+
+func upgradeKubeadmCoreOS(ctx *util.Context) error {
+	_, _, err := ctx.Runner.Run(upgradeKubeadmCoreOSCommand, util.TemplateVariables{
+		"KUBERNETES_VERSION": ctx.Cluster.Versions.Kubernetes,
+	})
+
+	return errors.WithStack(err)
+}

pkg/upgrader/upgrade/kubeadm_upgrade.go

@@ -0,0 +1,28 @@
+package upgrade
+
+import (
+	"github.com/kubermatic/kubeone/pkg/installer/util"
+)
+
+const (
+	kubeadmUpgradeLeaderCommand = `
+if [[ -f /etc/kubernetes/kubelet.conf ]]; then exit 0; fi
+sudo kubeadm upgrade {{ .VERSION }}
+`
+	kubeadmUpgradeFollowerCommand = `
+if [[ -f /etc/kubernetes/kubelet.conf ]]; then exit 0; fi
+sudo kubeadm upgrade node experimental-control-plane
+`
+)
+
+func upgradeLeaderControlPlane(ctx *util.Context) error {
+	_, _, err := ctx.Runner.Run(kubeadmUpgradeLeaderCommand, util.TemplateVariables{
+		"VERSION": ctx.Cluster.Versions.Kubernetes,
+	})
+	return err
+}
+
+func upgradeFollowerControlPlane(ctx *util.Context) error {
+	_, _, err := ctx.Runner.Run(kubeadmUpgradeFollowerCommand, util.TemplateVariables{})
+	return err
+}

pkg/upgrader/upgrade/kubelet_package.go

@@ -0,0 +1,84 @@
+package upgrade
+
+import (
+	"github.com/pkg/errors"
+
+	"github.com/kubermatic/kubeone/pkg/config"
+	"github.com/kubermatic/kubeone/pkg/installer/util"
+)
+
+const (
+	upgradeKubeletDebianCommand = `
+source /etc/os-release
+source /etc/kubeone/proxy-env
+
+sudo apt-get update
+
+kube_ver=$(apt-cache madison kubelet | grep "{{ .KUBERNETES_VERSION }}" | head -1 | awk '{print $3}')
+
+sudo apt-mark unhold kubelet
+sudo apt-get install kubelet=${kube_ver}
+sudo apt-mark hold kubelet
+`
+	upgradeKubeletCentOSCommand = `
+source /etc/kubeone/proxy-env
+
+sudo yum install -y --disableexcludes=kubernetes \
+			kubelet-{{ .KUBERNETES_VERSION }}-0
+`
+	upgradeKubeletCoreOSCommand = `
+source /etc/kubeone/proxy-env
+
+RELEASE="v{{ .KUBERNETES_VERSION }}"
+
+sudo mkdir -p /opt/bin
+cd /opt/bin
+sudo curl -L --remote-name-all \
+     https://storage.googleapis.com/kubernetes-release/release/${RELEASE}/bin/linux/amd64/kubelet
+sudo chmod +x kubelet
+`
+)
+
+func upgradeKubelet(ctx *util.Context, node *config.HostConfig) error {
+	var err error
+
+	switch node.OperatingSystem {
+	case "ubuntu", "debian":
+		err = upgradeKubeletDebian(ctx)
+
+	case "coreos":
+		err = upgradeKubeletCoreOS(ctx)
+
+	case "centos":
+		err = upgradeKubeletCentOS(ctx)
+
+	default:
+		err = errors.Errorf("'%s' is not a supported operating system", node.OperatingSystem)
+	}
+
+	return err
+}
+
+func upgradeKubeletDebian(ctx *util.Context) error {
+	_, _, err := ctx.Runner.Run(upgradeKubeletDebianCommand, util.TemplateVariables{
+		"KUBERNETES_VERSION": ctx.Cluster.Versions.Kubernetes,
+	})
+
+	return errors.WithStack(err)
+}
+
+func upgradeKubeletCentOS(ctx *util.Context) error {
+	_, _, err := ctx.Runner.Run(upgradeKubeletCentOSCommand, util.TemplateVariables{
+		"KUBERNETES_VERSION": ctx.Cluster.Versions.Kubernetes,
+	})
+
+	return errors.WithStack(err)
+}
+
+func upgradeKubeletCoreOS(ctx *util.Context) error {
+	_, _, err := ctx.Runner.Run(upgradeKubeletCoreOSCommand, util.TemplateVariables{
+		"KUBERNETES_VERSION": ctx.Cluster.Versions.Kubernetes,
+	})
+
+	return errors.WithStack(err)
+}

pkg/upgrader/upgrade/upgrade.go

@@ -7,18 +7,30 @@ import (
 )
 
 const (
-	labelUpgradeLock      = "kubeone.io/upgrading-in-process"
+	labelUpgradeLock      = "kubeone.io/upgrade-in-progress"
 	labelControlPlaneNode = "node-role.kubernetes.io/master"
 )
 
 // Upgrade performs all the steps required to upgrade Kubernetes on
 // cluster provisioned using KubeOne
 func Upgrade(ctx *util.Context) error {
-	if err := util.BuildKubernetesClientset(ctx); err != nil {
-		return errors.Wrap(err, "unable to build kubernetes clientset")
+	// commonSteps are same for all worker nodes and they are safe to be run in parallel
+	commonSteps := []struct {
+		fn     func(ctx *util.Context) error
+		errMsg string
+	}{
+		{fn: util.BuildKubernetesClientset, errMsg: "unable to build kubernetes clientset"},
+		{fn: determineHostname, errMsg: "unable to determine hostname"},
+		{fn: determineOS, errMsg: "unable to determine operating system"},
+		{fn: runPreflightChecks, errMsg: "preflight checks failed"},
+		{fn: upgradeLeader, errMsg: "unable to upgrade leader control plane"},
+		{fn: upgradeFollower, errMsg: "unable to upgrade follower control plane"},
 	}
-	if err := runPreflightChecks(ctx); err != nil {
-		return errors.Wrap(err, "preflight checks failed")
+
+	for _, step := range commonSteps {
+		if err := step.fn(ctx); err != nil {
+			return errors.Wrap(err, step.errMsg)
+		}
 	}
 
 	return nil

pkg/upgrader/upgrade/upgrade_follower.go

@@ -0,0 +1,47 @@
+package upgrade
+
+import (
+	"github.com/pkg/errors"
+
+	"github.com/kubermatic/kubeone/pkg/config"
+	"github.com/kubermatic/kubeone/pkg/installer/util"
+	"github.com/kubermatic/kubeone/pkg/ssh"
+)
+
+func upgradeFollower(ctx *util.Context) error {
+	return ctx.RunTaskOnFollowers(upgradeFollowerExecutor, false)
+}
+
+func upgradeFollowerExecutor(ctx *util.Context, node *config.HostConfig, conn ssh.Connection) error {
+	ctx.Logger.Infoln("Labeling follower control plane…")
+	err := labelNode(ctx.Clientset.CoreV1().Nodes(), node)
+	if err != nil {
+		return errors.Wrap(err, "failed to label leader control plane node")
+	}
+
+	ctx.Logger.Infoln("Upgrading kubeadm on follower control plane…")
+	err = upgradeKubeadm(ctx, node)
+	if err != nil {
+		return errors.Wrap(err, "failed to upgrade kubeadm on follower control plane")
+	}
+
+	ctx.Logger.Infoln("Running 'kubeadm upgrade' on the follower control plane node…")
+	err = upgradeFollowerControlPlane(ctx)
+	if err != nil {
+		return errors.Wrap(err, "failed to upgrade follower control plane")
+	}
+
+	ctx.Logger.Infoln("Upgrading kubelet…")
+	err = upgradeKubelet(ctx, node)
+	if err != nil {
+		return errors.Wrap(err, "failed to upgrade kubelet")
+	}
+
+	ctx.Logger.Infoln("Unlabeling follower control plane…")
+	err = unlabelNode(ctx.Clientset.CoreV1().Nodes(), node)
+	if err != nil {
+		return errors.Wrap(err, "failed to unlabel follower control plane node")
+	}
+
+	return nil
+}

pkg/upgrader/upgrade/upgrade_leader.go

@@ -0,0 +1,49 @@
+package upgrade
+
+import (
+	"github.com/pkg/errors"
+
+	"github.com/kubermatic/kubeone/pkg/config"
+	"github.com/kubermatic/kubeone/pkg/installer/util"
+	"github.com/kubermatic/kubeone/pkg/ssh"
+)
+
+func upgradeLeader(ctx *util.Context) error {
+	return ctx.RunTaskOnLeader(upgradeLeaderExecutor)
+}
+
+func upgradeLeaderExecutor(ctx *util.Context, node *config.HostConfig, conn ssh.Connection) error {
+	logger := ctx.Logger.WithField("node", node.PublicAddress)
+
+	logger.Infoln("Labeling leader control plane…")
+	err := labelNode(ctx.Clientset.CoreV1().Nodes(), node)
+	if err != nil {
+		return errors.Wrap(err, "failed to label leader control plane node")
+	}
+
+	logger.Infoln("Upgrading kubeadm on leader control plane…")
+	err = upgradeKubeadm(ctx, node)
+	if err != nil {
+		return errors.Wrap(err, "failed to upgrade kubeadm on leader control plane")
+	}
+
+	logger.Infoln("Running 'kubeadm upgrade' on leader control plane node…")
+	err = upgradeLeaderControlPlane(ctx)
+	if err != nil {
+		return errors.Wrap(err, "failed to run 'kubeadm upgrade' on leader control plane")
+	}
+
+	logger.Infoln("Upgrading kubelet on leader control plane…")
+	err = upgradeKubelet(ctx, node)
+	if err != nil {
+		return errors.Wrap(err, "failed to upgrade kubelet on leader control plane")
+	}
+
+	logger.Infoln("Unlabeling leader control plane…")
+	err = unlabelNode(ctx.Clientset.CoreV1().Nodes(), node)
+	if err != nil {
+		return errors.Wrap(err, "failed to unlabel leader control plane node")
+	}
+
+	return nil
+}