diff --git a/pkg/scripts/node.go b/pkg/scripts/node.go new file mode 100644 index 000000000..3eb7ecd25 --- /dev/null +++ b/pkg/scripts/node.go @@ -0,0 +1,39 @@ +/* +Copyright 2019 The KubeOne Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package scripts + +const ( + drainNodeScriptTemplate = ` +kubectl drain {{ .NODE_NAME }} --ignore-daemonsets --delete-local-data +` + + uncordonNodeScriptTemplate = ` +kubectl uncordon {{ .NODE_NAME }} +` +) + +func DrainNode(nodeName string) (string, error) { + return Render(drainNodeScriptTemplate, Data{ + "NODE_NAME": nodeName, + }) +} + +func UncordonNode(nodeName string) (string, error) { + return Render(uncordonNodeScriptTemplate, Data{ + "NODE_NAME": nodeName, + }) +} diff --git a/pkg/scripts/os.go b/pkg/scripts/os.go index e939473aa..2f1693244 100644 --- a/pkg/scripts/os.go +++ b/pkg/scripts/os.go @@ -207,34 +207,30 @@ sudo rm -rf /opt/cni /opt/bin/kubeadm /opt/bin/kubectl /opt/bin/kubelet sudo rm /etc/systemd/system/kubelet.service /etc/systemd/system/kubelet.service.d/10-kubeadm.conf ` - upgradeKubeBinariesDebianScriptTemplate = ` + upgradeKubeadmAndCNIDebianScriptTemplate = ` source /etc/os-release source /etc/kubeone/proxy-env sudo apt-get update -kube_ver=$(apt-cache madison kubelet | grep "{{ .KUBERNETES_VERSION }}" | head -1 | awk '{print $3}') +kube_ver=$(apt-cache madison kubeadm | grep "{{ .KUBERNETES_VERSION }}" | head -1 | awk '{print $3}') cni_ver=$(apt-cache madison kubernetes-cni | grep "{{ .CNI_VERSION }}" | head -1 | awk '{print $3}') -sudo apt-mark unhold kubeadm kubelet kubectl kubernetes-cni +sudo apt-mark unhold kubeadm kubernetes-cni sudo DEBIAN_FRONTEND=noninteractive apt-get install --option "Dpkg::Options::=--force-confold" -y --no-install-recommends \ kubeadm=${kube_ver} \ - kubectl=${kube_ver} \ - kubelet=${kube_ver} \ kubernetes-cni=${cni_ver} -sudo apt-mark hold kubeadm kubelet kubectl kubernetes-cni +sudo apt-mark hold kubeadm kubernetes-cni ` - upgradeKubeBinariesCentOSScriptTemplate = ` + upgradeKubeadmAndCNICentOSScriptTemplate = ` source /etc/kubeone/proxy-env sudo yum install -y --disableexcludes=kubernetes \ - kubelet-{{ .KUBERNETES_VERSION }}-0 \ kubeadm-{{ .KUBERNETES_VERSION }}-0 \ - kubectl-{{ .KUBERNETES_VERSION }}-0 \ kubernetes-cni-{{ .CNI_VERSION }}-0 ` - upgradeKubeBinariesCoreOSScriptTemplate = ` + upgradeKubeadmAndCNICoreOSScriptTemplate = ` source /etc/kubeone/proxy-env sudo mkdir -p /opt/cni/bin @@ -246,13 +242,53 @@ RELEASE="v{{ .KUBERNETES_VERSION }}" sudo mkdir -p /var/tmp/kube-binaries cd /var/tmp/kube-binaries sudo curl -L --remote-name-all \ - https://storage.googleapis.com/kubernetes-release/release/${RELEASE}/bin/linux/amd64/{kubeadm,kubelet,kubectl} + https://storage.googleapis.com/kubernetes-release/release/${RELEASE}/bin/linux/amd64/kubeadm + +sudo mkdir -p /opt/bin +cd /opt/bin +sudo systemctl stop kubelet +sudo mv /var/tmp/kube-binaries/kubeadm . +sudo chmod +x kubeadm +` + + upgradeKubeletAndKubectlDebianScriptTemplate = ` +source /etc/os-release +source /etc/kubeone/proxy-env + +sudo apt-get update + +kube_ver=$(apt-cache madison kubelet | grep "{{ .KUBERNETES_VERSION }}" | head -1 | awk '{print $3}') + +sudo apt-mark unhold kubelet kubectl +sudo DEBIAN_FRONTEND=noninteractive apt-get install --option "Dpkg::Options::=--force-confold" -y --no-install-recommends \ + kubelet=${kube_ver} \ + kubectl=${kube_ver} +sudo apt-mark hold kubelet kubectl +` + + upgradeKubeletAndKubectlCentOSScriptTemplate = ` +source /etc/kubeone/proxy-env + +sudo yum install -y --disableexcludes=kubernetes \ + kubelet-{{ .KUBERNETES_VERSION }}-0 \ + kubectl-{{ .KUBERNETES_VERSION }}-0 +` + + upgradeKubeletAndKubectlCoreOSScriptTemplate = ` +source /etc/kubeone/proxy-env + +RELEASE="v{{ .KUBERNETES_VERSION }}" + +sudo mkdir -p /var/tmp/kube-binaries +cd /var/tmp/kube-binaries +sudo curl -L --remote-name-all \ + https://storage.googleapis.com/kubernetes-release/release/${RELEASE}/bin/linux/amd64/{kubelet,kubectl} sudo mkdir -p /opt/bin cd /opt/bin sudo systemctl stop kubelet -sudo mv /var/tmp/kube-binaries/{kubeadm,kubelet,kubectl} . -sudo chmod +x {kubeadm,kubelet,kubectl} +sudo mv /var/tmp/kube-binaries/{kubelet,kubectl} . +sudo chmod +x {kubelet,kubectl} curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/kubelet.service" | sed "s:/usr/bin:/opt/bin:g" | @@ -262,6 +298,7 @@ sudo mkdir -p /etc/systemd/system/kubelet.service.d curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/10-kubeadm.conf" | sed "s:/usr/bin:/opt/bin:g" | sudo tee /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + sudo systemctl daemon-reload sudo systemctl start kubelet @@ -313,23 +350,41 @@ func RemoveBinariesCoreOS() (string, error) { return Render(removeBinariesCoreOSScriptTemplate, nil) } -func UpgradeKubeBinariesDebian(k8sVersion, cniVersion string) (string, error) { - return Render(upgradeKubeBinariesDebianScriptTemplate, Data{ +func UpgradeKubeadmAndCNIDebian(k8sVersion, cniVersion string) (string, error) { + return Render(upgradeKubeadmAndCNIDebianScriptTemplate, Data{ "KUBERNETES_VERSION": k8sVersion, "CNI_VERSION": cniVersion, }) } -func UpgradeKubeBinariesCentOS(k8sVersion, cniVersion string) (string, error) { - return Render(upgradeKubeBinariesCentOSScriptTemplate, Data{ +func UpgradeKubeadmAndCNICentOS(k8sVersion, cniVersion string) (string, error) { + return Render(upgradeKubeadmAndCNICentOSScriptTemplate, Data{ "KUBERNETES_VERSION": k8sVersion, "CNI_VERSION": cniVersion, }) } -func UpgradeKubeBinariesCoreOS(k8sVersion, cniVersion string) (string, error) { - return Render(upgradeKubeBinariesCoreOSScriptTemplate, Data{ +func UpgradeKubeadmAndCNICoreOS(k8sVersion, cniVersion string) (string, error) { + return Render(upgradeKubeadmAndCNICoreOSScriptTemplate, Data{ "KUBERNETES_VERSION": k8sVersion, "CNI_VERSION": cniVersion, }) } + +func UpgradeKubeletAndKubectlDebian(k8sVersion string) (string, error) { + return Render(upgradeKubeletAndKubectlDebianScriptTemplate, Data{ + "KUBERNETES_VERSION": k8sVersion, + }) +} + +func UpgradeKubeletAndKubectlCentOS(k8sVersion string) (string, error) { + return Render(upgradeKubeletAndKubectlCentOSScriptTemplate, Data{ + "KUBERNETES_VERSION": k8sVersion, + }) +} + +func UpgradeKubeletAndKubectlCoreOS(k8sVersion string) (string, error) { + return Render(upgradeKubeletAndKubectlCoreOSScriptTemplate, Data{ + "KUBERNETES_VERSION": k8sVersion, + }) +} diff --git a/pkg/upgrader/upgrade/drain_nodes.go b/pkg/upgrader/upgrade/drain_nodes.go new file mode 100644 index 000000000..c5ce62e8a --- /dev/null +++ b/pkg/upgrader/upgrade/drain_nodes.go @@ -0,0 +1,45 @@ +/* +Copyright 2019 The KubeOne Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package upgrade + +import ( + kubeoneapi "github.com/kubermatic/kubeone/pkg/apis/kubeone" + "github.com/kubermatic/kubeone/pkg/scripts" + "github.com/kubermatic/kubeone/pkg/state" +) + +func drainNode(s *state.State, node kubeoneapi.HostConfig) error { + cmd, err := scripts.DrainNode(node.Hostname) + if err != nil { + return err + } + + _, _, err = s.Runner.RunRaw(cmd) + + return err +} + +func uncordonNode(s *state.State, node kubeoneapi.HostConfig) error { + cmd, err := scripts.UncordonNode(node.Hostname) + if err != nil { + return err + } + + _, _, err = s.Runner.RunRaw(cmd) + + return err +} diff --git a/pkg/upgrader/upgrade/kubernetes_binaries.go b/pkg/upgrader/upgrade/kubernetes_binaries.go index c57b3bea6..9f24d6640 100644 --- a/pkg/upgrader/upgrade/kubernetes_binaries.go +++ b/pkg/upgrader/upgrade/kubernetes_binaries.go @@ -42,7 +42,7 @@ func upgradeKubernetesBinaries(s *state.State, node kubeoneapi.HostConfig) error } func upgradeKubernetesBinariesDebian(s *state.State) error { - cmd, err := scripts.UpgradeKubeBinariesDebian(s.Cluster.Versions.Kubernetes, s.Cluster.Versions.KubernetesCNIVersion()) + cmd, err := scripts.UpgradeKubeadmAndCNIDebian(s.Cluster.Versions.Kubernetes, s.Cluster.Versions.KubernetesCNIVersion()) if err != nil { return err } @@ -53,7 +53,7 @@ func upgradeKubernetesBinariesDebian(s *state.State) error { } func upgradeKubernetesBinariesCentOS(s *state.State) error { - cmd, err := scripts.UpgradeKubeBinariesCentOS(s.Cluster.Versions.Kubernetes, s.Cluster.Versions.KubernetesCNIVersion()) + cmd, err := scripts.UpgradeKubeadmAndCNICentOS(s.Cluster.Versions.Kubernetes, s.Cluster.Versions.KubernetesCNIVersion()) if err != nil { return err } @@ -64,7 +64,7 @@ func upgradeKubernetesBinariesCentOS(s *state.State) error { } func upgradeKubernetesBinariesCoreOS(s *state.State) error { - cmd, err := scripts.UpgradeKubeBinariesCoreOS(s.Cluster.Versions.Kubernetes, s.Cluster.Versions.KubernetesCNIVersion()) + cmd, err := scripts.UpgradeKubeadmAndCNICoreOS(s.Cluster.Versions.Kubernetes, s.Cluster.Versions.KubernetesCNIVersion()) if err != nil { return err } diff --git a/pkg/upgrader/upgrade/kubernetes_node_binaries.go b/pkg/upgrader/upgrade/kubernetes_node_binaries.go new file mode 100644 index 000000000..5d72423fa --- /dev/null +++ b/pkg/upgrader/upgrade/kubernetes_node_binaries.go @@ -0,0 +1,96 @@ +/* +Copyright 2019 The KubeOne Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package upgrade + +import ( + "time" + + "github.com/pkg/errors" + + kubeoneapi "github.com/kubermatic/kubeone/pkg/apis/kubeone" + "github.com/kubermatic/kubeone/pkg/scripts" + "github.com/kubermatic/kubeone/pkg/ssh" + "github.com/kubermatic/kubeone/pkg/state" +) + +func upgradeKubernetesNodeBinaries(s *state.State) error { + return s.RunTaskOnAllNodes(upgradeKubernetesNodeBinariesExecutor, false) +} + +func upgradeKubernetesNodeBinariesExecutor(s *state.State, node *kubeoneapi.HostConfig, conn ssh.Connection) error { + logger := s.Logger.WithField("node", node.PublicAddress) + + logger.Infoln("Upgrading Kubernetes node binaries on control planes…") + if err := upgradeKubernetesNodeBinariesScript(s, *node); err != nil { + return errors.Wrap(err, "failed to upgrade kubernetes binaries on leader control plane") + } + + logger.Infof("Waiting %v to ensure kubelet is up…", timeoutKubeletUpgrade) + time.Sleep(timeoutKubeletUpgrade) + + return nil +} + +func upgradeKubernetesNodeBinariesScript(s *state.State, node kubeoneapi.HostConfig) error { + var err error + + switch node.OperatingSystem { + case "ubuntu", "debian": + err = upgradeKubernetesNodeBinariesDebian(s) + case "coreos": + err = upgradeKubernetesNodeBinariesCoreOS(s) + case "centos": + err = upgradeKubernetesNodeBinariesCentOS(s) + default: + err = errors.Errorf("'%s' is not a supported operating system", node.OperatingSystem) + } + + return err +} + +func upgradeKubernetesNodeBinariesDebian(s *state.State) error { + cmd, err := scripts.UpgradeKubeletAndKubectlDebian(s.Cluster.Versions.Kubernetes) + if err != nil { + return err + } + + _, _, err = s.Runner.RunRaw(cmd) + + return errors.WithStack(err) +} + +func upgradeKubernetesNodeBinariesCentOS(s *state.State) error { + cmd, err := scripts.UpgradeKubeletAndKubectlCentOS(s.Cluster.Versions.Kubernetes) + if err != nil { + return err + } + + _, _, err = s.Runner.RunRaw(cmd) + + return errors.WithStack(err) +} + +func upgradeKubernetesNodeBinariesCoreOS(s *state.State) error { + cmd, err := scripts.UpgradeKubeletAndKubectlCoreOS(s.Cluster.Versions.Kubernetes) + if err != nil { + return err + } + + _, _, err = s.Runner.RunRaw(cmd) + + return errors.WithStack(err) +} diff --git a/pkg/upgrader/upgrade/upgrade.go b/pkg/upgrader/upgrade/upgrade.go index c67ee45d9..723bab79b 100644 --- a/pkg/upgrader/upgrade/upgrade.go +++ b/pkg/upgrader/upgrade/upgrade.go @@ -40,7 +40,7 @@ const ( timeoutKubeletUpgrade = 1 * time.Minute // timeoutNodeUpgrade is time for how long kubeone will wait after finishing the upgrade // process on the node - timeoutNodeUpgrade = 15 * time.Second + timeoutNodeUpgrade = 30 * time.Second ) // Upgrade performs all the steps required to upgrade Kubernetes on @@ -54,6 +54,7 @@ func Upgrade(s *state.State) error { {Fn: runPreflightChecks, ErrMsg: "preflight checks failed"}, {Fn: upgradeLeader, ErrMsg: "unable to upgrade leader control plane", Retries: 3}, {Fn: upgradeFollower, ErrMsg: "unable to upgrade follower control plane", Retries: 3}, + {Fn: upgradeKubernetesNodeBinaries, ErrMsg: "unable to upgrade kubernetes node binaries", Retries: 3}, {Fn: nodelocaldns.Deploy, ErrMsg: "unable to deploy nodelocaldns", Retries: 3}, {Fn: features.Activate, ErrMsg: "unable to activate features"}, {Fn: certificate.DownloadCA, ErrMsg: "unable to download ca from leader", Retries: 3}, diff --git a/pkg/upgrader/upgrade/upgrade_follower.go b/pkg/upgrader/upgrade/upgrade_follower.go index 78ca12588..3471dfada 100644 --- a/pkg/upgrader/upgrade/upgrade_follower.go +++ b/pkg/upgrader/upgrade/upgrade_follower.go @@ -39,22 +39,31 @@ func upgradeFollowerExecutor(s *state.State, node *kubeoneapi.HostConfig, conn s return errors.Wrap(err, "failed to label leader control plane node") } + logger.Infoln("Draining follower control plane…") + err = drainNode(s, *node) + if err != nil { + return errors.Wrap(err, "failed to drain leader control plane node") + } + logger.Infoln("Upgrading Kubernetes binaries on follower control plane…") err = upgradeKubernetesBinaries(s, *node) if err != nil { return errors.Wrap(err, "failed to upgrade kubernetes binaries on follower control plane") } - logger.Infof("Waiting %v seconds to ensure kubelet is up…", timeoutKubeletUpgrade.String()) - time.Sleep(timeoutKubeletUpgrade) - logger.Infoln("Running 'kubeadm upgrade' on the follower control plane node…") err = upgradeFollowerControlPlane(s) if err != nil { return errors.Wrap(err, "failed to upgrade follower control plane") } - logger.Infof("Waiting %v seconds to ensure all components are up…", timeoutNodeUpgrade.String()) + logger.Infoln("Uncordoning follower control plane…") + err = uncordonNode(s, *node) + if err != nil { + return errors.Wrap(err, "failed to uncordon follower control plane node") + } + + logger.Infof("Waiting %v to ensure all components are up…", timeoutNodeUpgrade) time.Sleep(timeoutNodeUpgrade) logger.Infoln("Unlabeling follower control plane…") diff --git a/pkg/upgrader/upgrade/upgrade_leader.go b/pkg/upgrader/upgrade/upgrade_leader.go index 9194473ca..38eb6d602 100644 --- a/pkg/upgrader/upgrade/upgrade_leader.go +++ b/pkg/upgrader/upgrade/upgrade_leader.go @@ -38,14 +38,16 @@ func upgradeLeaderExecutor(s *state.State, node *kubeoneapi.HostConfig, conn ssh return errors.Wrap(err, "failed to label leader control plane node") } + logger.Infoln("Draining leader control plane…") + if err := drainNode(s, *node); err != nil { + return errors.Wrap(err, "failed to drain leader control plane node") + } + logger.Infoln("Upgrading Kubernetes binaries on leader control plane…") if err := upgradeKubernetesBinaries(s, *node); err != nil { return errors.Wrap(err, "failed to upgrade kubernetes binaries on leader control plane") } - logger.Infof("Waiting %v seconds to ensure kubelet is up…", timeoutKubeletUpgrade.String()) - time.Sleep(timeoutKubeletUpgrade) - logger.Infoln("Generating kubeadm config …") if err := generateKubeadmConfig(s, *node); err != nil { return errors.Wrap(err, "failed to generate kubeadm config") @@ -61,7 +63,12 @@ func upgradeLeaderExecutor(s *state.State, node *kubeoneapi.HostConfig, conn ssh return errors.Wrap(err, "failed to run 'kubeadm upgrade' on leader control plane") } - logger.Infof("Waiting %v seconds to ensure all components are up…", timeoutNodeUpgrade.String()) + logger.Infoln("Uncordoning leader control plane…") + if err := uncordonNode(s, *node); err != nil { + return errors.Wrap(err, "failed to uncordon leader control plane node") + } + + logger.Infof("Waiting %v to ensure all components are up…", timeoutNodeUpgrade) time.Sleep(timeoutNodeUpgrade) logger.Infoln("Unlabeling leader control plane…")