vdk-structlog: default formatter

projects/vdk-plugins/vdk-structlog/src/vdk/plugin/structlog/constants.py

@@ -4,6 +4,7 @@
 
 STRUCTLOG_LOGGING_METADATA_KEY = "logging_metadata"
 STRUCTLOG_LOGGING_FORMAT_KEY = "logging_format"
+STRUCTLOG_CONSOLE_LOG_PATTERN = "logging_custom_format"
 
 STRUCTLOG_LOGGING_FORMAT_POSSIBLE_VALUES = ["console", "json"]
 STRUCTLOG_LOGGING_FORMAT_DEFAULT = "console"

projects/vdk-plugins/vdk-structlog/src/vdk/plugin/structlog/formatters.py

@@ -132,7 +132,9 @@ def build_ltsv_format(self) -> str:
         return ltsv_format
 
 
-def create_formatter(logging_format: str, metadata_keys: str) -> [Formatter, Filter]:
+def create_formatter(
+    logging_format: str, metadata_keys: str, custom_console_format: str = None
+) -> [Formatter, Filter]:
     """
     Creates a formatter and a filter based on the logging format configuration
     and metadata_keys configuration that are passed. The formatter takes care
@@ -153,8 +155,12 @@ def create_formatter(logging_format: str, metadata_keys: str) -> [Formatter, Fil
         )
         custom_key_filter = ConsoleMetadataFilter(key_set)
     else:
-        formatter = ConsoleFormatter(
-            fmt=StructlogMetadataBuilder(metadata_keys).build_console_format()
-        )
+        if custom_console_format:
+            format_string = custom_console_format
+        else:
+            format_string = StructlogMetadataBuilder(
+                metadata_keys
+            ).build_console_format()
+        formatter = ConsoleFormatter(fmt=format_string)
         custom_key_filter = ConsoleMetadataFilter(key_set)
     return formatter, custom_key_filter

projects/vdk-plugins/vdk-structlog/src/vdk/plugin/structlog/structlog_plugin.py

@@ -16,6 +16,7 @@
 from vdk.internal.core.config import ConfigurationBuilder
 from vdk.internal.core.context import CoreContext
 from vdk.plugin.structlog.constants import JSON_STRUCTLOG_LOGGING_METADATA_DEFAULT
+from vdk.plugin.structlog.constants import STRUCTLOG_CONSOLE_LOG_PATTERN
 from vdk.plugin.structlog.constants import STRUCTLOG_LOGGING_FORMAT_DEFAULT
 from vdk.plugin.structlog.constants import STRUCTLOG_LOGGING_FORMAT_KEY
 from vdk.plugin.structlog.constants import STRUCTLOG_LOGGING_FORMAT_POSSIBLE_VALUES
@@ -79,6 +80,12 @@ def vdk_configure(self, config_builder: ConfigurationBuilder):
             ),
         )
 
+        config_builder.add(
+            key=STRUCTLOG_CONSOLE_LOG_PATTERN,
+            default_value="",
+            description="Custom format string for console logging. Leave empty for default format.",
+        )
+
         config_builder.add(
             key=STRUCTLOG_LOGGING_FORMAT_KEY,
             default_value=STRUCTLOG_LOGGING_FORMAT_DEFAULT,
@@ -93,8 +100,13 @@ def vdk_initialize(self, context: CoreContext):
         logging_formatter = context.configuration.get_value(
             STRUCTLOG_LOGGING_FORMAT_KEY
         )
+        custom_format_string = context.configuration.get_value(
+            STRUCTLOG_CONSOLE_LOG_PATTERN
+        )
 
-        formatter, metadata_filter = create_formatter(logging_formatter, metadata_keys)
+        formatter, metadata_filter = create_formatter(
+            logging_formatter, metadata_keys, custom_format_string
+        )
 
         root_logger = logging.getLogger()
         root_logger.removeHandler(root_logger.handlers[0])
@@ -107,14 +119,19 @@ def vdk_initialize(self, context: CoreContext):
 
     @hookimpl(hookwrapper=True)
     def initialize_job(self, context: JobContext) -> None:
+        metadata_keys = context.core_context.configuration.get_value(
+            STRUCTLOG_LOGGING_METADATA_KEY
+        )
         logging_formatter = context.core_context.configuration.get_value(
             STRUCTLOG_LOGGING_FORMAT_KEY
         )
-        metadata_keys = context.core_context.configuration.get_value(
-            STRUCTLOG_LOGGING_METADATA_KEY
+        custom_format_string = context.core_context.configuration.get_value(
+            STRUCTLOG_CONSOLE_LOG_PATTERN
         )
 
-        formatter, metadata_filter = create_formatter(logging_formatter, metadata_keys)
+        formatter, metadata_filter = create_formatter(
+            logging_formatter, metadata_keys, custom_format_string
+        )
         job_name_adder = AttributeAdder("vdk_job_name", context.name)
 
         root_logger = logging.getLogger()
@@ -134,14 +151,19 @@ def initialize_job(self, context: JobContext) -> None:
 
     @hookimpl(hookwrapper=True)
     def run_job(self, context: JobContext) -> Optional[ExecutionResult]:
+        metadata_keys = context.core_context.configuration.get_value(
+            STRUCTLOG_LOGGING_METADATA_KEY
+        )
         logging_formatter = context.core_context.configuration.get_value(
             STRUCTLOG_LOGGING_FORMAT_KEY
         )
-        metadata_keys = context.core_context.configuration.get_value(
-            STRUCTLOG_LOGGING_METADATA_KEY
+        custom_format_string = context.core_context.configuration.get_value(
+            STRUCTLOG_CONSOLE_LOG_PATTERN
         )
 
-        formatter, metadata_filter = create_formatter(logging_formatter, metadata_keys)
+        formatter, metadata_filter = create_formatter(
+            logging_formatter, metadata_keys, custom_format_string
+        )
         job_name_adder = AttributeAdder("vdk_job_name", context.name)
 
         root_logger = logging.getLogger()

projects/vdk-plugins/vdk-structlog/tests/test_structlog.py

@@ -113,6 +113,34 @@ def test_stock_fields_removal(log_format):
                 assert re.search(stock_field_reps[shown_field], test_log) is not None
 
 
+@pytest.mark.parametrize("log_format", ["console"])
+def test_custom_console_format(log_format):
+    custom_format_string = "%(asctime)s %(name)-12s %(levelname)-8s %(message)s"
+
+    with mock.patch.dict(
+        os.environ,
+        {
+            "VDK_LOGGING_METADATA": "timestamp,level,file_name,line_number,vdk_job_name",
+            "VDK_LOGGING_FORMAT": log_format,
+            "STRUCTLOG_CONSOLE_LOG_PATTERN": custom_format_string,
+        },
+    ):
+        logs = _run_job_and_get_logs()
+        for log in logs:
+            if "Log statement with no bound context" in log:
+                assert _matches_custom_format(log)
+                break
+        else:
+            pytest.fail("Log statement with no bound context not found in logs")
+
+
+def _matches_custom_format(log):
+    pattern = re.compile(
+        r"\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}\s+\S{1,12}\s+\S{1,8}"
+    )
+    return bool(pattern.search(log))
+
+
 def _run_job_and_get_logs():
     """
     Runs the necessary job and returns the output logs.