vdk-structlog: LTSV formatting (#2887)

This PR incorporates the LTSV formatting which was present in the vdk-logging-format plugin to vdk-structlog, rendering vdk-logging- format unneccessary. Addresses #2812 Testing done: unit tests cover LTSV formatting --------- Signed-off-by: Gabriel Georgiev <[email protected]> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
vmware · Nov 14, 2023 · 4e5862d · 4e5862d
1 parent 745b14d
commit 4e5862d
Show file tree

Hide file tree

Showing 2 changed files with 44 additions and 2 deletions.
diff --git a/projects/vdk-plugins/vdk-structlog/src/vdk/plugin/structlog/formatters.py b/projects/vdk-plugins/vdk-structlog/src/vdk/plugin/structlog/formatters.py
@@ -39,6 +39,24 @@ def add_fields(
             del log_record["created"]
 
 
+class LtsvFormatter(Formatter):
+    """
+    Formats logs in the Labelled Tab Separated Values (LTSV) format.
+    You can read more about this format here: http://ltsv.org/
+
+    """
+
+    def format(self, record: LogRecord) -> LogRecord:
+        record.message = record.getMessage()
+        record.timestamp = self.formatTime(record, self.datefmt)
+        record.line_number = record.lineno
+        if ("level" not in record.__dict__) or (not record.__dict__["level"]):
+            record.level = record.levelname
+
+        s = self.formatMessage(record)
+        return s
+
+
 class ConsoleFormatter(Formatter):
     """
     Console formatter. Basically the same as logging.Formatter.
@@ -102,6 +120,17 @@ def build_console_format(self) -> str:
         out.append("- %(message)s")
         return " ".join(out)
 
+    def build_ltsv_format(self) -> str:
+        out = []
+        for key in self._metadata_keys:
+            if key in STRUCTLOG_LOGGING_METADATA_ALL:
+                out.append(key + ":" + STRUCTLOG_LOGGING_METADATA_ALL[key])
+            else:
+                out.append(key + ":" + f"%({key})s")
+        out.append("message:%(message)s")
+        ltsv_format = "\t".join(out)
+        return ltsv_format
+
 
 def create_formatter(logging_format: str, metadata_keys: str) -> [Formatter, Filter]:
     """
@@ -118,6 +147,11 @@ def create_formatter(logging_format: str, metadata_keys: str) -> [Formatter, Fil
             StructlogMetadataBuilder(metadata_keys).build_json_format()
         )
         custom_key_filter = JsonMetadataFilter(key_set)
+    elif logging_format == "ltsv":
+        formatter = LtsvFormatter(
+            fmt=StructlogMetadataBuilder(metadata_keys).build_ltsv_format()
+        )
+        custom_key_filter = ConsoleMetadataFilter(key_set)
     else:
         formatter = ConsoleFormatter(
             fmt=StructlogMetadataBuilder(metadata_keys).build_console_format()

diff --git a/projects/vdk-plugins/vdk-structlog/tests/test_structlog.py b/projects/vdk-plugins/vdk-structlog/tests/test_structlog.py
@@ -37,6 +37,14 @@
         "function_name": "run",
         "vdk_job_name": JOB_NAME,
     },
+    "ltsv": {
+        "timestamp": r"timestamp:\d+\.\d+",
+        "level": r"level:INFO",
+        "file_name": r"file_name:10_dummy\.py",
+        "line_number": "line_number:[0-9]+",
+        "function_name": "function_name:run",
+        "vdk_job_name": f"vdk_job_name:{JOB_NAME}",
+    },
     "json": {
         "timestamp": r'"timestamp": \d+\.\d+',
         "level": '"level": "INFO"',
@@ -48,7 +56,7 @@
 }
 
 
-@pytest.mark.parametrize("log_format", ["console", "json"])
+@pytest.mark.parametrize("log_format", ["console", "ltsv", "json"])
 def test_structlog(log_format):
     with mock.patch.dict(
         os.environ,
@@ -76,7 +84,7 @@ def test_structlog(log_format):
         )
 
 
-@pytest.mark.parametrize("log_format", ["console", "json"])
+@pytest.mark.parametrize("log_format", ["console", "ltsv", "json"])
 def test_stock_fields_removal(log_format):
     stock_field_reps = STOCK_FIELD_REPRESENTATIONS[log_format]