diff --git a/roles/openshift_health_checker/openshift_checks/docker_storage_check.py b/roles/openshift_health_checker/openshift_checks/docker_storage_check.py index 6fab4bb0243..818ba7e2ba1 100644 --- a/roles/openshift_health_checker/openshift_checks/docker_storage_check.py +++ b/roles/openshift_health_checker/openshift_checks/docker_storage_check.py @@ -1,41 +1,137 @@ # pylint: disable=missing-docstring from openshift_checks import OpenShiftCheck, get_var +import json + class DockerStorageCheck(OpenShiftCheck): """Check Docker storage sanity. - This check ensures that Docker is using the devicemapper - storage driver, that thinpool usage is not close to 100%, - and that Loopback is not being used. + This check ensures that Docker is using a supported storage driver, + that thinpool usage is not close to 100%, and that Loopback is not being used. """ name = "docker_storage_check" tags = ["preflight"] + storage_drivers = ["devicemapper", "overlay", "overlay2"] + + max_thinpool_data_usage_percent = 90.0 + max_thinpool_metadata_usage_percent = 90.0 + def run(self, tmp, task_vars): + self.max_thinpool_data_usage_percent = get_var(task_vars, "max_thinpool_data_usage_percent") + self.max_thinpool_metadata_usage_percent = get_var(task_vars, "max_thinpool_metadata_usage_percent") + info = self.module_executor("docker_info", {}, task_vars).get("info", {}) - if not self.is_devicemapper_used(info): - msg = "Unsupported Docker storage driver detected. Only \"devicemapper\" is currently supported." - return {"failed": True, "msg": msg} + if not self.is_supported_storage_driver(info): + msg = "Unsupported Docker storage driver detected. Supported storage drivers: {drivers}" + return {"failed": True, "msg": msg.format(drivers=self.storage_drivers)} if self.is_using_loopback_device(info): msg = "Use of loopback devices is discouraged. Try running Docker with `--storage-opt dm.thinpooldev`" return {"failed": True, "msg": msg} - return {"failed": True, "msg": "%s" % info.get("Data Space Available")} + failed, msg = self.check_thinpool_usage(task_vars) + if failed: + return {"failed": True, "msg": msg} return {"changed": False} - @staticmethod - def is_devicemapper_used(docker_info): - return docker_info.get("Driver", None) == "devicemapper" + def is_supported_storage_driver(self, docker_info): + return docker_info.get("Driver", "") in self.storage_drivers @staticmethod - def is_using_loopback_device(info): - for status in info.get("DriverStatus"): + def is_using_loopback_device(docker_info): + # Loopback device usage is only an issue if using devicemapper. + # Skip this check if using any other storage driver. + if docker_info.get("Driver", "") != "devicemapper": + return False + + for status in docker_info.get("DriverStatus", []): if status[0] == "Data loop file": return bool(status[1]) return False + + def check_thinpool_usage(self, task_vars): + no_data_msg = "no thinpool usage data returned by the host." + + data, failed, msg = self.get_lvs_data(task_vars) + if failed: + if not msg: + msg = no_data_msg + return failed, msg + + failed, data_percent = self.get_thinpool_data_usage(data) + if failed: + return failed, no_data_msg + + failed, metadata_percent = self.get_thinpool_metadata_usage(data) + if failed: + return failed, no_data_msg + + if data_percent > self.max_thinpool_data_usage_percent: + msg = "thinpool data usage above maximum threshold of {threshold}%%" + return True, msg.format(threshold=data_percent) + + if metadata_percent > self.max_thinpool_metadata_usage_percent: + msg = "thinpool metadata usage above maximum threshold of {threshold}%%" + return True, msg.format(threshold=metadata_percent) + + return False, "" + + def get_lvs_data(self, task_vars): + lvs_cmd = "/sbin/lvs --select vg_name=docker --select lv_name=docker-pool --report-format json" + result = self.exec_cmd(lvs_cmd, task_vars) + data_json = {} + failed = False + + if result.get("failed", False): + return {}, True, result.get("msg", "") + + try: + data_json = json.loads(result.get("stdout")) + except ValueError: + failed = True + + return data_json.get("report", {}), failed, "" + + def get_thinpool_data_usage(self, thinpool_data): + lv = self.extract_thinpool_obj(thinpool_data) + if not lv: + return True, 0 + + data = lv.get("data_percent") + if not data: + return True, 0 + + return False, float(data) + + def get_thinpool_metadata_usage(self, thinpool_data): + lv = self.extract_thinpool_obj(thinpool_data) + if not lv: + return True, 0 + + data = lv.get("metadata_percent") + if not data: + return True, 0 + + return False, float(data) + + @staticmethod + def extract_thinpool_obj(thinpool_data): + if not thinpool_data or not thinpool_data[0]: + return None + + lv = thinpool_data[0].get("lv") + if not lv or not lv[0]: + return None + + return lv[0] + + def exec_cmd(self, cmd_str, task_vars): + return self.module_executor("command", { + "_raw_params": cmd_str, + }, task_vars) \ No newline at end of file