[CPDEV-93171] - clean kubernetes tmp dir before upgrade

documentation/Maintenance.md

@@ -255,6 +255,12 @@ UPGRADING KUBERNETES v1.18.8 ⭢ v1.19.3
 
 The script upgrades Kubernetes versions one-by-one. After each upgrade, the `cluster.yaml` is regenerated to reflect the actual cluster state. Use the latest updated `cluster.yaml` configuration to further work with the cluster.
 
+Additionally, before the upgrade kubemarine cleans up `/etc/kubernetes/tmp` directory, where kubeadm stores [backup files](https://kubernetes.io/docs/tasks/administer-cluster/kubeadm/kubeadm-upgrade/#recovering-from-a-failure-state)
+during upgrade. For this reason only backups for the latest upgrade via kubemarine will be placed here after the upgrade procedure.
+
+**Note**: we do not recommend to use those backup files for rolling back after upgrade, because it can follow inconsistent state 
+for `cluster.yaml`. Please, use kubemarine [backup](#backup-procedure) and [restore](#restore-procedure) procedures instead of manual restoration.
+
 #### Upgrading Specific Nodes
 
 **Note**: Kubemarine automatically determines already upgraded nodes and excludes them from the Kubernetes upgrade procedure. Use manual nodes specifying for updating in exceptional cases when the problem cannot be solved automatically. Also, if any of the nodes are not available, first remove the node from the cluster, instead of changing the list of nodes for the upgrade.

kubemarine/procedures/upgrade.py

@@ -12,8 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-
+import copy
 from collections import OrderedDict
 from itertools import chain
 from typing import List
@@ -27,6 +26,13 @@
 from kubemarine.procedures import install
 
 
+def cleanup_tmp_dir(cluster: KubernetesCluster) -> None:
+    # Clean up kubernetes tmp dir, where backup files from previous upgrades are located
+    nodes = cluster.make_group_from_roles(roles=["control-plane", "worker"])
+    nodes.sudo("rm -rf /etc/kubernetes/tmp/*")
+    cluster.log.debug("Backup files for previous upgrades were cleaned")
+
+
 def system_prepare_thirdparties(cluster: KubernetesCluster) -> None:
     if not cluster.inventory['services'].get('thirdparties', {}):
         cluster.log.debug("Skipped - no thirdparties defined in config file")
@@ -108,6 +114,7 @@ def upgrade_plugins(cluster: KubernetesCluster) -> None:
 
 
 tasks = OrderedDict({
+    "cleanup_tmp_dir": cleanup_tmp_dir,
     "verify_upgrade_versions": kubernetes.verify_upgrade_versions,
     "thirdparties": system_prepare_thirdparties,
     "prepull_images": prepull_images,
@@ -116,7 +123,6 @@ def upgrade_plugins(cluster: KubernetesCluster) -> None:
     "packages": upgrade_packages,
     "plugins": upgrade_plugins,
     "overview": install.overview
-
 })
 
 
@@ -142,17 +148,21 @@ def _run(self, resources: DynamicResources) -> None:
             raise Exception("Usage of '--tasks' and '--exclude' is not allowed when upgrading to more than one version")
 
         # todo inventory is preserved few times, probably need to preserve it once instead.
-        actions = [UpgradeAction(version) for version in upgrade_plan]
+        actions = [UpgradeAction(version, i == 0) for i, version in enumerate(upgrade_plan)]
         flow.run_actions(resources, actions)
 
 
 class UpgradeAction(Action):
-    def __init__(self, upgrade_version: str) -> None:
+    def __init__(self, upgrade_version: str, first_upgrade: bool) -> None:
         super().__init__('upgrade to ' + upgrade_version, recreate_inventory=True)
         self.upgrade_version = upgrade_version
+        self.first_upgrade = first_upgrade
 
     def run(self, res: DynamicResources) -> None:
-        flow.run_tasks(res, tasks)
+        action_tasks = copy.deepcopy(tasks)
+        if not self.first_upgrade:
+            del action_tasks['cleanup_tmp_dir']
+        flow.run_tasks(res, action_tasks)
         res.make_final_inventory()
 
     def prepare_context(self, context: dict) -> None:

test/unit/test_upgrade.py

@@ -576,9 +576,9 @@ def prepare_inventory(self, upgrade_plan: List[str]):
         self.upgrade = demo.generate_procedure_inventory('upgrade')
         self.upgrade['upgrade_plan'] = upgrade_plan
         self.actions = []
-        for ver in upgrade_plan:
+        for i, ver in enumerate(upgrade_plan):
             self.upgrade[ver] = {}
-            self.actions.append(upgrade.UpgradeAction(ver))
+            self.actions.append(upgrade.UpgradeAction(ver, i == 0))
 
         self.resources: Optional[demo.FakeResources] = None