WIP: Implement grommunio-admin shell batch

grommunio_exporter.yaml

@@ -9,6 +9,4 @@ http_server:
 grommunio:
   # Optional overrides
   cli_binary: /usr/sbin/grommunio-admin
-  # concurrent api calls, defaults to 4
-  api_concurrency:
   alternative_hostname:

grommunio_exporter/grommunio_api.py

@@ -11,6 +11,7 @@
 __license__ = "GPL-3.0-only"
 __build__ = "2024110501"
 
+from typing import List
 from ofunctions.misc import fn_name
 import logging
 from pathlib import Path
@@ -165,7 +166,67 @@ def get_mailboxes(self):
             logger.debug("Trace", exc_info=True)
             self.api_status = False
 
-    def _get_mailbox_properties(self, username: str):
+    def get_usernames_from_mailboxes(self, mailboxes: list, filter_no_domain: bool = True) -> List[str]:
+        """
+        Extract a list of usernames from mailboxes
+        """
+        usernames = []
+        for mailbox in mailboxes:
+            if filter_no_domain:
+                domain = self._get_domain_from_username(mailbox["username"])
+                if domain == "no_domain":
+                    continue
+            usernames.append(mailbox["username"])
+        return usernames
+
+    def _get_mailbox_properties(self, usernames: List[str]):
+        """
+        Get various properties of mailboxes
+        """
+        mailbox_properties = {}
+        grommunio_shell_cmd = ""
+        for username in usernames:
+            grommunio_shell_cmd += f"exmdb {username} store get\n"
+        cmd = f"{self.cli_binary} shell -x << EOF\n{grommunio_shell_cmd}\nEOF"
+        exit_code, result = command_runner(cmd, shell=True)
+        if exit_code == 0:
+            print(result)
+        else:
+            logger.error(
+                f"Could not execute {cmd}: Failed with error code {exit_code}: {result}"
+            )
+            self.api_status = False
+            # Since we used awk, we should definitly reset the output
+            mailbox_properties = {}
+
+        labels = (self.hostname, domain, username)
+        for key, value in mailbox_properties.items():
+            if key == "messagesizeextended":
+                self.gauge_grommunio_mailbox_messagesize.labels(*labels).set(value)
+            elif key == "storagequotalimit":
+                # Value given in KB iec, we need to convert it to bytes
+                value = BytesConverter(f"{value} KiB")
+                self.gauge_grommunio_mailbox_storage_quota_limit.labels(*labels).set(
+                    value
+                )
+            elif key == "prohibitreceivequota":
+                value = BytesConverter(f"{value} KiB")
+                self.gauge_grommunio_mailbox_prohibit_reveive_quota.labels(*labels).set(
+                    value
+                )
+            elif key == "prohibitsendquota":
+                value = BytesConverter(f"{value} KiB")
+                self.gauge_grommunio_mailbox_prohibit_send_quota.labels(*labels).set(
+                    value
+                )
+            elif key == "creationtime":
+                # Creationtime is an 18-digit LDAP/FILETIME timestamp we need to convert first to epoch
+                value = convert_from_file_time(value).timestamp()
+                self.gauge_grommunio_mailbox_creation_time.labels(*labels).set(value)
+        return mailbox_properties
+
+
+    def _get_mailbox_properties_old(self, username: str):
         """
         Get size of mailbox
 
@@ -255,7 +316,7 @@ def _get_mailbox_properties(self, username: str):
                 self.gauge_grommunio_mailbox_creation_time.labels(*labels).set(value)
         return mailbox_properties
 
-    def get_mailbox_properties(self, username: str):
+    def get_mailbox_properties_old(self, username: str):
         """
         Just a wrapper to get exceptions from threads
         """
@@ -266,6 +327,14 @@ def get_mailbox_properties(self, username: str):
             logger.debug("Trace", exc_info=True)
             self.api_status = False
 
+    def get_mailbox_properties(self, usernames: List[str]):
+        try:
+            return self._get_mailbox_properties(usernames)
+        except Exception as exc:
+            logger.error(f"Could not get mailboxes properties: {exc}")
+            logger.debug("Trace", exc_info=True)
+            self.api_status = False     
+
     def api_status_reset(self):
         self.api_status = True
 

grommunio_exporter/metrics.py

@@ -23,8 +23,6 @@
 from fastapi_offline import FastAPIOffline
 import prometheus_client
 import socket
-from time import sleep
-import concurrent.futures
 from grommunio_exporter.__version__ import __version__
 from grommunio_exporter.configuration import load_config, get_default_config
 from grommunio_exporter.grommunio_api import GrommunioExporter
@@ -67,13 +65,6 @@
 hostname = config_dict.g("grommunio.hostname")
 if not hostname:
     hostname = socket.getfqdn()
-api_concurrency = config_dict.g("grommunio.api_concurrency")
-if not api_concurrency:
-    api_concurrency = 4
-if api_concurrency > 1:
-    USE_THREADS = True
-else:
-    USE_THREADS = False
 
 
 app = FastAPIOffline()
@@ -91,36 +82,10 @@ def run_metrics():
     # Let's reset api status first
     api.api_status_reset()
 
-    if USE_THREADS:
-        thread_pool = concurrent.futures.ThreadPoolExecutor(max_workers=api_concurrency)
-        thread_list = []
-    for mailbox in api.get_mailboxes():
-        if mailbox["status"] != 4:
-            if USE_THREADS:
-                thread_list.append(
-                    thread_pool.submit(api.get_mailbox_properties, mailbox["username"])
-                )
-            else:
-                api.get_mailbox_properties(mailbox["username"])
-
-    if USE_THREADS:
-        # Let's have a proper wait function for threads to finish
-        threads_still_run = True
-        elapsed_loops = 0
-        while elapsed_loops < 120 and threads_still_run:
-            threads_still_run = False
-            for thread in thread_list:
-                if thread.running():
-                    threads_still_run = True
-            sleep(1)
-            elapsed_loops += 1
-
-        logger.debug(f"Thread list: {thread_list}")
-
-        for thread in thread_list:
-            thread.cancel()
-        thread_pool.shutdown()
-    # Create a gauge for the API status
+
+    mailboxes = api.get_mailboxes()
+    usernames = api.get_usernames_from_mailboxes(mailboxes, filter_no_domain=True)
+    api.get_mailbox_properties(usernames)
     api.api_status_result()