add thinpool usage check

roles/openshift_health_checker/openshift_checks/docker_storage_check.py

@@ -1,41 +1,137 @@
 # pylint: disable=missing-docstring
 from openshift_checks import OpenShiftCheck, get_var
 
+import json
+
 
 class DockerStorageCheck(OpenShiftCheck):
     """Check Docker storage sanity.
 
-    This check ensures that Docker is using the devicemapper
-    storage driver, that thinpool usage is not close to 100%,
-    and that Loopback is not being used.
+    This check ensures that Docker is using a supported storage driver,
+    that thinpool usage is not close to 100%, and that Loopback is not being used.
     """
 
     name = "docker_storage_check"
     tags = ["preflight"]
 
+    storage_drivers = ["devicemapper", "overlay", "overlay2"]
+
+    max_thinpool_data_usage_percent = 90.0
+    max_thinpool_metadata_usage_percent = 90.0
+
     def run(self, tmp, task_vars):
+        self.max_thinpool_data_usage_percent = get_var(task_vars, "max_thinpool_data_usage_percent")
+        self.max_thinpool_metadata_usage_percent = get_var(task_vars, "max_thinpool_metadata_usage_percent")
+
         info = self.module_executor("docker_info", {}, task_vars).get("info", {})
 
-        if not self.is_devicemapper_used(info):
-            msg = "Unsupported Docker storage driver detected. Only \"devicemapper\" is currently supported."
-            return {"failed": True, "msg": msg}
+        if not self.is_supported_storage_driver(info):
+            msg = "Unsupported Docker storage driver detected. Supported storage drivers: {drivers}"
+            return {"failed": True, "msg": msg.format(drivers=self.storage_drivers)}
 
         if self.is_using_loopback_device(info):
             msg = "Use of loopback devices is discouraged. Try running Docker with `--storage-opt dm.thinpooldev`"
             return {"failed": True, "msg": msg}
 
-        return {"failed": True, "msg": "%s" % info.get("Data Space Available")}
+        failed, msg = self.check_thinpool_usage(task_vars)
+        if failed:
+            return {"failed": True, "msg": msg}
 
         return {"changed": False}
 
-    @staticmethod
-    def is_devicemapper_used(docker_info):
-        return docker_info.get("Driver", None) == "devicemapper"
+    def is_supported_storage_driver(self, docker_info):
+        return docker_info.get("Driver", "") in self.storage_drivers
 
     @staticmethod
-    def is_using_loopback_device(info):
-        for status in info.get("DriverStatus"):
+    def is_using_loopback_device(docker_info):
+        # Loopback device usage is only an issue if using devicemapper.
+        # Skip this check if using any other storage driver.
+        if docker_info.get("Driver", "") != "devicemapper":
+            return False
+
+        for status in docker_info.get("DriverStatus", []):
             if status[0] == "Data loop file":
                 return bool(status[1])
 
         return False
+
+    def check_thinpool_usage(self, task_vars):
+        no_data_msg = "no thinpool usage data returned by the host."
+
+        data, failed, msg = self.get_lvs_data(task_vars)
+        if failed:
+            if not msg:
+                msg = no_data_msg
+            return failed, msg
+
+        failed, data_percent = self.get_thinpool_data_usage(data)
+        if failed:
+            return failed, no_data_msg
+
+        failed, metadata_percent = self.get_thinpool_metadata_usage(data)
+        if failed:
+            return failed, no_data_msg
+
+        if data_percent > self.max_thinpool_data_usage_percent:
+            msg = "thinpool data usage above maximum threshold of {threshold}%%"
+            return True, msg.format(threshold=data_percent)
+
+        if metadata_percent > self.max_thinpool_metadata_usage_percent:
+            msg = "thinpool metadata usage above maximum threshold of {threshold}%%"
+            return True, msg.format(threshold=metadata_percent)
+
+        return False, ""
+
+    def get_lvs_data(self, task_vars):
+        lvs_cmd = "/sbin/lvs --select vg_name=docker --select lv_name=docker-pool --report-format json"
+        result = self.exec_cmd(lvs_cmd, task_vars)
+        data_json = {}
+        failed = False
+
+        if result.get("failed", False):
+            return {}, True, result.get("msg", "")
+
+        try:
+            data_json = json.loads(result.get("stdout"))
+        except ValueError:
+            failed = True
+
+        return data_json.get("report", {}), failed, ""
+
+    def get_thinpool_data_usage(self, thinpool_data):
+        lv = self.extract_thinpool_obj(thinpool_data)
+        if not lv:
+            return True, 0
+
+        data = lv.get("data_percent")
+        if not data:
+            return True, 0
+
+        return False, float(data)
+
+    def get_thinpool_metadata_usage(self, thinpool_data):
+        lv = self.extract_thinpool_obj(thinpool_data)
+        if not lv:
+            return True, 0
+
+        data = lv.get("metadata_percent")
+        if not data:
+            return True, 0
+
+        return False, float(data)
+
+    @staticmethod
+    def extract_thinpool_obj(thinpool_data):
+        if not thinpool_data or not thinpool_data[0]:
+            return None
+
+        lv = thinpool_data[0].get("lv")
+        if not lv or not lv[0]:
+            return None
+
+        return lv[0]
+
+    def exec_cmd(self, cmd_str, task_vars):
+        return self.module_executor("command", {
+            "_raw_params": cmd_str,
+        }, task_vars)