-
Notifications
You must be signed in to change notification settings - Fork 242
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Ensure correct order of operations for cluster upgrades (#764)
* Ensure correct order of operations for cluster upgrades * Fix lint errors * Replace cordon with uncordon * Fix CoreOS Kubeadm upgrade script * Drain nodes with --delete-local-data flag * Fix CoreOS typo * Increase wait after upgrading components * Fix format for printing times * Fix format for printing times
- Loading branch information
1 parent
0f90d86
commit 63afc8f
Showing
8 changed files
with
283 additions
and
31 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,39 @@ | ||
| /* | ||
| Copyright 2019 The KubeOne Authors. | ||
| Licensed under the Apache License, Version 2.0 (the "License"); | ||
| you may not use this file except in compliance with the License. | ||
| You may obtain a copy of the License at | ||
| http://www.apache.org/licenses/LICENSE-2.0 | ||
| Unless required by applicable law or agreed to in writing, software | ||
| distributed under the License is distributed on an "AS IS" BASIS, | ||
| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| See the License for the specific language governing permissions and | ||
| limitations under the License. | ||
| */ | ||
|
|
||
| package scripts | ||
|
|
||
| const ( | ||
| drainNodeScriptTemplate = ` | ||
| kubectl drain {{ .NODE_NAME }} --ignore-daemonsets --delete-local-data | ||
| ` | ||
|
|
||
| uncordonNodeScriptTemplate = ` | ||
| kubectl uncordon {{ .NODE_NAME }} | ||
| ` | ||
| ) | ||
|
|
||
| func DrainNode(nodeName string) (string, error) { | ||
| return Render(drainNodeScriptTemplate, Data{ | ||
| "NODE_NAME": nodeName, | ||
| }) | ||
| } | ||
|
|
||
| func UncordonNode(nodeName string) (string, error) { | ||
| return Render(uncordonNodeScriptTemplate, Data{ | ||
| "NODE_NAME": nodeName, | ||
| }) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,45 @@ | ||
| /* | ||
| Copyright 2019 The KubeOne Authors. | ||
| Licensed under the Apache License, Version 2.0 (the "License"); | ||
| you may not use this file except in compliance with the License. | ||
| You may obtain a copy of the License at | ||
| http://www.apache.org/licenses/LICENSE-2.0 | ||
| Unless required by applicable law or agreed to in writing, software | ||
| distributed under the License is distributed on an "AS IS" BASIS, | ||
| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| See the License for the specific language governing permissions and | ||
| limitations under the License. | ||
| */ | ||
|
|
||
| package upgrade | ||
|
|
||
| import ( | ||
| kubeoneapi "github.com/kubermatic/kubeone/pkg/apis/kubeone" | ||
| "github.com/kubermatic/kubeone/pkg/scripts" | ||
| "github.com/kubermatic/kubeone/pkg/state" | ||
| ) | ||
|
|
||
| func drainNode(s *state.State, node kubeoneapi.HostConfig) error { | ||
| cmd, err := scripts.DrainNode(node.Hostname) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| _, _, err = s.Runner.RunRaw(cmd) | ||
|
|
||
| return err | ||
| } | ||
|
|
||
| func uncordonNode(s *state.State, node kubeoneapi.HostConfig) error { | ||
| cmd, err := scripts.UncordonNode(node.Hostname) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| _, _, err = s.Runner.RunRaw(cmd) | ||
|
|
||
| return err | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,96 @@ | ||
| /* | ||
| Copyright 2019 The KubeOne Authors. | ||
| Licensed under the Apache License, Version 2.0 (the "License"); | ||
| you may not use this file except in compliance with the License. | ||
| You may obtain a copy of the License at | ||
| http://www.apache.org/licenses/LICENSE-2.0 | ||
| Unless required by applicable law or agreed to in writing, software | ||
| distributed under the License is distributed on an "AS IS" BASIS, | ||
| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| See the License for the specific language governing permissions and | ||
| limitations under the License. | ||
| */ | ||
|
|
||
| package upgrade | ||
|
|
||
| import ( | ||
| "time" | ||
|
|
||
| "github.com/pkg/errors" | ||
|
|
||
| kubeoneapi "github.com/kubermatic/kubeone/pkg/apis/kubeone" | ||
| "github.com/kubermatic/kubeone/pkg/scripts" | ||
| "github.com/kubermatic/kubeone/pkg/ssh" | ||
| "github.com/kubermatic/kubeone/pkg/state" | ||
| ) | ||
|
|
||
| func upgradeKubernetesNodeBinaries(s *state.State) error { | ||
| return s.RunTaskOnAllNodes(upgradeKubernetesNodeBinariesExecutor, false) | ||
| } | ||
|
|
||
| func upgradeKubernetesNodeBinariesExecutor(s *state.State, node *kubeoneapi.HostConfig, conn ssh.Connection) error { | ||
| logger := s.Logger.WithField("node", node.PublicAddress) | ||
|
|
||
| logger.Infoln("Upgrading Kubernetes node binaries on control planes…") | ||
| if err := upgradeKubernetesNodeBinariesScript(s, *node); err != nil { | ||
| return errors.Wrap(err, "failed to upgrade kubernetes binaries on leader control plane") | ||
| } | ||
|
|
||
| logger.Infof("Waiting %v to ensure kubelet is up…", timeoutKubeletUpgrade) | ||
| time.Sleep(timeoutKubeletUpgrade) | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| func upgradeKubernetesNodeBinariesScript(s *state.State, node kubeoneapi.HostConfig) error { | ||
| var err error | ||
|
|
||
| switch node.OperatingSystem { | ||
| case "ubuntu", "debian": | ||
| err = upgradeKubernetesNodeBinariesDebian(s) | ||
| case "coreos": | ||
| err = upgradeKubernetesNodeBinariesCoreOS(s) | ||
| case "centos": | ||
| err = upgradeKubernetesNodeBinariesCentOS(s) | ||
| default: | ||
| err = errors.Errorf("'%s' is not a supported operating system", node.OperatingSystem) | ||
| } | ||
|
|
||
| return err | ||
| } | ||
|
|
||
| func upgradeKubernetesNodeBinariesDebian(s *state.State) error { | ||
| cmd, err := scripts.UpgradeKubeletAndKubectlDebian(s.Cluster.Versions.Kubernetes) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| _, _, err = s.Runner.RunRaw(cmd) | ||
|
|
||
| return errors.WithStack(err) | ||
| } | ||
|
|
||
| func upgradeKubernetesNodeBinariesCentOS(s *state.State) error { | ||
| cmd, err := scripts.UpgradeKubeletAndKubectlCentOS(s.Cluster.Versions.Kubernetes) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| _, _, err = s.Runner.RunRaw(cmd) | ||
|
|
||
| return errors.WithStack(err) | ||
| } | ||
|
|
||
| func upgradeKubernetesNodeBinariesCoreOS(s *state.State) error { | ||
| cmd, err := scripts.UpgradeKubeletAndKubectlCoreOS(s.Cluster.Versions.Kubernetes) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| _, _, err = s.Runner.RunRaw(cmd) | ||
|
|
||
| return errors.WithStack(err) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.