feat(RHINENG-5449): add new prometheus metrics

app/queue/event_producer.py

@@ -4,6 +4,8 @@
 from app.instrumentation import message_not_produced
 from app.instrumentation import message_produced
 from app.logging import get_logger
+from app.queue.metrics import produce_large_message_failure
+from app.queue.metrics import produced_message_size
 
 logger = get_logger(__name__)
 
@@ -22,9 +24,15 @@ def __init__(self, topic: str, event: str, headers: list(tuple()), key: str):
         self.topic = topic
 
     def on_delivered(self, error, message):
+        message_to_send = None
         if error:
-            message_not_produced(logger, error, self.topic, self.event, self.key, self.headers)
+            if "value too long" in error:
+                message_to_send = message
+                produce_large_message_failure.inc()
+
+            message_not_produced(logger, error, self.topic, self.event, self.key, self.headers, message_to_send)
         else:
+            produced_message_size.observe(len(message.encode("utf8")))
             message_produced(logger, message, self.headers)
 
 

app/queue/metrics.py

@@ -32,6 +32,11 @@
 ingress_message_handler_time = Summary(
     "inventory_ingress_message_handler_seconds", "Total time spent handling messages from the ingress queue"
 )
+consumed_message_size = Summary("inventory_consumed_message_size", "Size of incoming messages in bytes")
+produced_message_size = Summary("invenotry_produced_message_size", "Size of outgoing messages in bytes")
+produce_large_message_failure = Counter(
+    "inventory_produce_large_message_failures", "Total amount of failures producing messages due to their size"
+)
 version = Info("inventory_mq_service_version", "Build version for the inventory message queue service")
 version.info({"version": get_build_version()})
 event_producer_success = Counter(

app/queue/queue.py

@@ -344,6 +344,7 @@ def event_loop(consumer, flask_app, event_producer, notification_event_producer,
 
                     try:
                         handler(msg.value(), event_producer, notification_event_producer=notification_event_producer)
+                        metrics.consumed_message_size.observe(len(msg.encode("utf8")))
                         metrics.ingress_message_handler_success.inc()
                     except OperationalError as oe:
                         """sqlalchemy.exc.OperationalError: This error occurs when an