From b9b9459e9221ba17a7456cf7f5ad25eff12ab3b6 Mon Sep 17 00:00:00 2001 From: Duygu Hasan Date: Thu, 23 Nov 2023 11:51:51 +0200 Subject: [PATCH 01/19] add custom format string --- .../src/vdk/plugin/structlog/constants.py | 1 + .../src/vdk/plugin/structlog/formatters.py | 10 ++-- .../vdk/plugin/structlog/structlog_plugin.py | 51 +++++++++++-------- 3 files changed, 37 insertions(+), 25 deletions(-) diff --git a/projects/vdk-plugins/vdk-structlog/src/vdk/plugin/structlog/constants.py b/projects/vdk-plugins/vdk-structlog/src/vdk/plugin/structlog/constants.py index 7e0987e281..6938d3ac4b 100644 --- a/projects/vdk-plugins/vdk-structlog/src/vdk/plugin/structlog/constants.py +++ b/projects/vdk-plugins/vdk-structlog/src/vdk/plugin/structlog/constants.py @@ -4,6 +4,7 @@ STRUCTLOG_LOGGING_METADATA_KEY = "logging_metadata" STRUCTLOG_LOGGING_FORMAT_KEY = "logging_format" +STRUCTLOG_LOGGING_CUSTOM_FORMAT_KEY = "logging_custom_format" STRUCTLOG_LOGGING_FORMAT_POSSIBLE_VALUES = ["console", "json"] STRUCTLOG_LOGGING_FORMAT_DEFAULT = "console" diff --git a/projects/vdk-plugins/vdk-structlog/src/vdk/plugin/structlog/formatters.py b/projects/vdk-plugins/vdk-structlog/src/vdk/plugin/structlog/formatters.py index 5b58259f8e..9ac8c4a42b 100644 --- a/projects/vdk-plugins/vdk-structlog/src/vdk/plugin/structlog/formatters.py +++ b/projects/vdk-plugins/vdk-structlog/src/vdk/plugin/structlog/formatters.py @@ -132,7 +132,7 @@ def build_ltsv_format(self) -> str: return ltsv_format -def create_formatter(logging_format: str, metadata_keys: str) -> [Formatter, Filter]: +def create_formatter(logging_format: str, metadata_keys: str, custom_console_format: str = None) -> [Formatter, Filter]: """ Creates a formatter and a filter based on the logging format configuration and metadata_keys configuration that are passed. The formatter takes care @@ -153,8 +153,10 @@ def create_formatter(logging_format: str, metadata_keys: str) -> [Formatter, Fil ) custom_key_filter = ConsoleMetadataFilter(key_set) else: - formatter = ConsoleFormatter( - fmt=StructlogMetadataBuilder(metadata_keys).build_console_format() - ) + if custom_console_format: + format_string = custom_console_format + else: + format_string = StructlogMetadataBuilder(metadata_keys).build_console_format() + formatter = ConsoleFormatter(fmt=format_string) custom_key_filter = ConsoleMetadataFilter(key_set) return formatter, custom_key_filter diff --git a/projects/vdk-plugins/vdk-structlog/src/vdk/plugin/structlog/structlog_plugin.py b/projects/vdk-plugins/vdk-structlog/src/vdk/plugin/structlog/structlog_plugin.py index cea68a26d8..f66319ee8a 100644 --- a/projects/vdk-plugins/vdk-structlog/src/vdk/plugin/structlog/structlog_plugin.py +++ b/projects/vdk-plugins/vdk-structlog/src/vdk/plugin/structlog/structlog_plugin.py @@ -16,6 +16,7 @@ from vdk.internal.core.config import ConfigurationBuilder from vdk.internal.core.context import CoreContext from vdk.plugin.structlog.constants import JSON_STRUCTLOG_LOGGING_METADATA_DEFAULT +from vdk.plugin.structlog.constants import STRUCTLOG_LOGGING_CUSTOM_FORMAT_KEY from vdk.plugin.structlog.constants import STRUCTLOG_LOGGING_FORMAT_DEFAULT from vdk.plugin.structlog.constants import STRUCTLOG_LOGGING_FORMAT_KEY from vdk.plugin.structlog.constants import STRUCTLOG_LOGGING_FORMAT_POSSIBLE_VALUES @@ -79,6 +80,12 @@ def vdk_configure(self, config_builder: ConfigurationBuilder): ), ) + config_builder.add( + key=STRUCTLOG_LOGGING_CUSTOM_FORMAT_KEY, + default_value="", + description="Custom format string for console logging. Leave empty for default format.", + ) + config_builder.add( key=STRUCTLOG_LOGGING_FORMAT_KEY, default_value=STRUCTLOG_LOGGING_FORMAT_DEFAULT, @@ -89,12 +96,14 @@ def vdk_configure(self, config_builder: ConfigurationBuilder): @hookimpl def vdk_initialize(self, context: CoreContext): - metadata_keys = context.configuration.get_value(STRUCTLOG_LOGGING_METADATA_KEY) - logging_formatter = context.configuration.get_value( - STRUCTLOG_LOGGING_FORMAT_KEY - ) + metadata_keys = context.configuration. \ + get_value(STRUCTLOG_LOGGING_METADATA_KEY) + logging_formatter = context.configuration. \ + get_value(STRUCTLOG_LOGGING_FORMAT_KEY) + custom_format_string = context.configuration. \ + get_value(STRUCTLOG_LOGGING_CUSTOM_FORMAT_KEY) - formatter, metadata_filter = create_formatter(logging_formatter, metadata_keys) + formatter, metadata_filter = create_formatter(logging_formatter, metadata_keys, custom_format_string) root_logger = logging.getLogger() root_logger.removeHandler(root_logger.handlers[0]) @@ -107,14 +116,14 @@ def vdk_initialize(self, context: CoreContext): @hookimpl(hookwrapper=True) def initialize_job(self, context: JobContext) -> None: - logging_formatter = context.core_context.configuration.get_value( - STRUCTLOG_LOGGING_FORMAT_KEY - ) - metadata_keys = context.core_context.configuration.get_value( - STRUCTLOG_LOGGING_METADATA_KEY - ) - - formatter, metadata_filter = create_formatter(logging_formatter, metadata_keys) + metadata_keys = context.core_context.configuration.\ + get_value(STRUCTLOG_LOGGING_METADATA_KEY) + logging_formatter = context.core_context.configuration.\ + get_value(STRUCTLOG_LOGGING_FORMAT_KEY) + custom_format_string = context.core_context.configuration.\ + get_value(STRUCTLOG_LOGGING_CUSTOM_FORMAT_KEY) + + formatter, metadata_filter = create_formatter(logging_formatter, metadata_keys, custom_format_string) job_name_adder = AttributeAdder("vdk_job_name", context.name) root_logger = logging.getLogger() @@ -134,14 +143,14 @@ def initialize_job(self, context: JobContext) -> None: @hookimpl(hookwrapper=True) def run_job(self, context: JobContext) -> Optional[ExecutionResult]: - logging_formatter = context.core_context.configuration.get_value( - STRUCTLOG_LOGGING_FORMAT_KEY - ) - metadata_keys = context.core_context.configuration.get_value( - STRUCTLOG_LOGGING_METADATA_KEY - ) - - formatter, metadata_filter = create_formatter(logging_formatter, metadata_keys) + metadata_keys = context.core_context.configuration. \ + get_value(STRUCTLOG_LOGGING_METADATA_KEY) + logging_formatter = context.core_context.configuration. \ + get_value(STRUCTLOG_LOGGING_FORMAT_KEY) + custom_format_string = context.core_context.configuration. \ + get_value(STRUCTLOG_LOGGING_CUSTOM_FORMAT_KEY) + + formatter, metadata_filter = create_formatter(logging_formatter, metadata_keys, custom_format_string) job_name_adder = AttributeAdder("vdk_job_name", context.name) root_logger = logging.getLogger() From 2b73fb89ff68d49959d1eedbed68a264b40ef2f1 Mon Sep 17 00:00:00 2001 From: Duygu Hasan Date: Thu, 23 Nov 2023 11:59:31 +0200 Subject: [PATCH 02/19] add test --- .../vdk-structlog/tests/test_structlog.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/projects/vdk-plugins/vdk-structlog/tests/test_structlog.py b/projects/vdk-plugins/vdk-structlog/tests/test_structlog.py index 7bbda006a6..e5fe9b0424 100644 --- a/projects/vdk-plugins/vdk-structlog/tests/test_structlog.py +++ b/projects/vdk-plugins/vdk-structlog/tests/test_structlog.py @@ -187,3 +187,30 @@ def _get_job_arguments(): extra_fields = f'{{"{EXTRA_TEST_KEY}": "{EXTRA_TEST_VALUE}"}}' return f'{{"bound_fields": {bound_fields}, "extra_fields": {extra_fields}}}' + + +@pytest.mark.parametrize("log_format", ["console"]) +def test_custom_console_format(log_format): + custom_format_string = "(asctime)s %(name)-12s %(levelname)-8s %(message)s" + + with mock.patch.dict( + os.environ, + { + "VDK_LOGGING_METADATA": "timestamp,level,file_name,line_number,vdk_job_name", + "VDK_LOGGING_FORMAT": log_format, + "VDK_LOGGING_CUSTOM_FORMAT": custom_format_string, # Custom format string environment variable + }, + ): + logs = _run_job_and_get_logs() + log_with_custom_format = _get_log_containing_s(logs, "Log statement with no bound context") + + _assert_custom_format(log_with_custom_format, custom_format_string) + + +def _assert_custom_format(log, custom_format_string): + """ + Assert that the log line matches the custom format string. + """ + pattern = re.compile(r"\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3} \S+ \[INFO ]") + assert pattern.search(log), "Log line does not match the custom format string" + From c1c5bf532bfd441bbfdbe9973a90e2d1e7ad86cb Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 23 Nov 2023 10:00:54 +0000 Subject: [PATCH 03/19] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../src/vdk/plugin/structlog/formatters.py | 8 ++- .../vdk/plugin/structlog/structlog_plugin.py | 59 +++++++++++-------- .../vdk-structlog/tests/test_structlog.py | 17 +++--- 3 files changed, 51 insertions(+), 33 deletions(-) diff --git a/projects/vdk-plugins/vdk-structlog/src/vdk/plugin/structlog/formatters.py b/projects/vdk-plugins/vdk-structlog/src/vdk/plugin/structlog/formatters.py index 9ac8c4a42b..eecc0ebc4e 100644 --- a/projects/vdk-plugins/vdk-structlog/src/vdk/plugin/structlog/formatters.py +++ b/projects/vdk-plugins/vdk-structlog/src/vdk/plugin/structlog/formatters.py @@ -132,7 +132,9 @@ def build_ltsv_format(self) -> str: return ltsv_format -def create_formatter(logging_format: str, metadata_keys: str, custom_console_format: str = None) -> [Formatter, Filter]: +def create_formatter( + logging_format: str, metadata_keys: str, custom_console_format: str = None +) -> [Formatter, Filter]: """ Creates a formatter and a filter based on the logging format configuration and metadata_keys configuration that are passed. The formatter takes care @@ -156,7 +158,9 @@ def create_formatter(logging_format: str, metadata_keys: str, custom_console_for if custom_console_format: format_string = custom_console_format else: - format_string = StructlogMetadataBuilder(metadata_keys).build_console_format() + format_string = StructlogMetadataBuilder( + metadata_keys + ).build_console_format() formatter = ConsoleFormatter(fmt=format_string) custom_key_filter = ConsoleMetadataFilter(key_set) return formatter, custom_key_filter diff --git a/projects/vdk-plugins/vdk-structlog/src/vdk/plugin/structlog/structlog_plugin.py b/projects/vdk-plugins/vdk-structlog/src/vdk/plugin/structlog/structlog_plugin.py index f66319ee8a..17ec82af6c 100644 --- a/projects/vdk-plugins/vdk-structlog/src/vdk/plugin/structlog/structlog_plugin.py +++ b/projects/vdk-plugins/vdk-structlog/src/vdk/plugin/structlog/structlog_plugin.py @@ -96,14 +96,17 @@ def vdk_configure(self, config_builder: ConfigurationBuilder): @hookimpl def vdk_initialize(self, context: CoreContext): - metadata_keys = context.configuration. \ - get_value(STRUCTLOG_LOGGING_METADATA_KEY) - logging_formatter = context.configuration. \ - get_value(STRUCTLOG_LOGGING_FORMAT_KEY) - custom_format_string = context.configuration. \ - get_value(STRUCTLOG_LOGGING_CUSTOM_FORMAT_KEY) + metadata_keys = context.configuration.get_value(STRUCTLOG_LOGGING_METADATA_KEY) + logging_formatter = context.configuration.get_value( + STRUCTLOG_LOGGING_FORMAT_KEY + ) + custom_format_string = context.configuration.get_value( + STRUCTLOG_LOGGING_CUSTOM_FORMAT_KEY + ) - formatter, metadata_filter = create_formatter(logging_formatter, metadata_keys, custom_format_string) + formatter, metadata_filter = create_formatter( + logging_formatter, metadata_keys, custom_format_string + ) root_logger = logging.getLogger() root_logger.removeHandler(root_logger.handlers[0]) @@ -116,14 +119,19 @@ def vdk_initialize(self, context: CoreContext): @hookimpl(hookwrapper=True) def initialize_job(self, context: JobContext) -> None: - metadata_keys = context.core_context.configuration.\ - get_value(STRUCTLOG_LOGGING_METADATA_KEY) - logging_formatter = context.core_context.configuration.\ - get_value(STRUCTLOG_LOGGING_FORMAT_KEY) - custom_format_string = context.core_context.configuration.\ - get_value(STRUCTLOG_LOGGING_CUSTOM_FORMAT_KEY) - - formatter, metadata_filter = create_formatter(logging_formatter, metadata_keys, custom_format_string) + metadata_keys = context.core_context.configuration.get_value( + STRUCTLOG_LOGGING_METADATA_KEY + ) + logging_formatter = context.core_context.configuration.get_value( + STRUCTLOG_LOGGING_FORMAT_KEY + ) + custom_format_string = context.core_context.configuration.get_value( + STRUCTLOG_LOGGING_CUSTOM_FORMAT_KEY + ) + + formatter, metadata_filter = create_formatter( + logging_formatter, metadata_keys, custom_format_string + ) job_name_adder = AttributeAdder("vdk_job_name", context.name) root_logger = logging.getLogger() @@ -143,14 +151,19 @@ def initialize_job(self, context: JobContext) -> None: @hookimpl(hookwrapper=True) def run_job(self, context: JobContext) -> Optional[ExecutionResult]: - metadata_keys = context.core_context.configuration. \ - get_value(STRUCTLOG_LOGGING_METADATA_KEY) - logging_formatter = context.core_context.configuration. \ - get_value(STRUCTLOG_LOGGING_FORMAT_KEY) - custom_format_string = context.core_context.configuration. \ - get_value(STRUCTLOG_LOGGING_CUSTOM_FORMAT_KEY) - - formatter, metadata_filter = create_formatter(logging_formatter, metadata_keys, custom_format_string) + metadata_keys = context.core_context.configuration.get_value( + STRUCTLOG_LOGGING_METADATA_KEY + ) + logging_formatter = context.core_context.configuration.get_value( + STRUCTLOG_LOGGING_FORMAT_KEY + ) + custom_format_string = context.core_context.configuration.get_value( + STRUCTLOG_LOGGING_CUSTOM_FORMAT_KEY + ) + + formatter, metadata_filter = create_formatter( + logging_formatter, metadata_keys, custom_format_string + ) job_name_adder = AttributeAdder("vdk_job_name", context.name) root_logger = logging.getLogger() diff --git a/projects/vdk-plugins/vdk-structlog/tests/test_structlog.py b/projects/vdk-plugins/vdk-structlog/tests/test_structlog.py index e5fe9b0424..d51835226b 100644 --- a/projects/vdk-plugins/vdk-structlog/tests/test_structlog.py +++ b/projects/vdk-plugins/vdk-structlog/tests/test_structlog.py @@ -194,15 +194,17 @@ def test_custom_console_format(log_format): custom_format_string = "(asctime)s %(name)-12s %(levelname)-8s %(message)s" with mock.patch.dict( - os.environ, - { - "VDK_LOGGING_METADATA": "timestamp,level,file_name,line_number,vdk_job_name", - "VDK_LOGGING_FORMAT": log_format, - "VDK_LOGGING_CUSTOM_FORMAT": custom_format_string, # Custom format string environment variable - }, + os.environ, + { + "VDK_LOGGING_METADATA": "timestamp,level,file_name,line_number,vdk_job_name", + "VDK_LOGGING_FORMAT": log_format, + "VDK_LOGGING_CUSTOM_FORMAT": custom_format_string, # Custom format string environment variable + }, ): logs = _run_job_and_get_logs() - log_with_custom_format = _get_log_containing_s(logs, "Log statement with no bound context") + log_with_custom_format = _get_log_containing_s( + logs, "Log statement with no bound context" + ) _assert_custom_format(log_with_custom_format, custom_format_string) @@ -213,4 +215,3 @@ def _assert_custom_format(log, custom_format_string): """ pattern = re.compile(r"\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3} \S+ \[INFO ]") assert pattern.search(log), "Log line does not match the custom format string" - From 407dd95e9f016a139107de081bbcc109a48a9b2c Mon Sep 17 00:00:00 2001 From: Duygu Hasan Date: Thu, 23 Nov 2023 12:42:37 +0200 Subject: [PATCH 04/19] remove test --- .../vdk-structlog/tests/test_structlog.py | 25 ------------------- 1 file changed, 25 deletions(-) diff --git a/projects/vdk-plugins/vdk-structlog/tests/test_structlog.py b/projects/vdk-plugins/vdk-structlog/tests/test_structlog.py index e5fe9b0424..771321aa6b 100644 --- a/projects/vdk-plugins/vdk-structlog/tests/test_structlog.py +++ b/projects/vdk-plugins/vdk-structlog/tests/test_structlog.py @@ -189,28 +189,3 @@ def _get_job_arguments(): return f'{{"bound_fields": {bound_fields}, "extra_fields": {extra_fields}}}' -@pytest.mark.parametrize("log_format", ["console"]) -def test_custom_console_format(log_format): - custom_format_string = "(asctime)s %(name)-12s %(levelname)-8s %(message)s" - - with mock.patch.dict( - os.environ, - { - "VDK_LOGGING_METADATA": "timestamp,level,file_name,line_number,vdk_job_name", - "VDK_LOGGING_FORMAT": log_format, - "VDK_LOGGING_CUSTOM_FORMAT": custom_format_string, # Custom format string environment variable - }, - ): - logs = _run_job_and_get_logs() - log_with_custom_format = _get_log_containing_s(logs, "Log statement with no bound context") - - _assert_custom_format(log_with_custom_format, custom_format_string) - - -def _assert_custom_format(log, custom_format_string): - """ - Assert that the log line matches the custom format string. - """ - pattern = re.compile(r"\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3} \S+ \[INFO ]") - assert pattern.search(log), "Log line does not match the custom format string" - From 4f95038d70c4528fb771c12b659edf4bc9824f9e Mon Sep 17 00:00:00 2001 From: Duygu Hasan Date: Thu, 23 Nov 2023 14:27:56 +0200 Subject: [PATCH 05/19] try test --- .../vdk-structlog/tests/test_structlog.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/projects/vdk-plugins/vdk-structlog/tests/test_structlog.py b/projects/vdk-plugins/vdk-structlog/tests/test_structlog.py index 771321aa6b..e78481cd27 100644 --- a/projects/vdk-plugins/vdk-structlog/tests/test_structlog.py +++ b/projects/vdk-plugins/vdk-structlog/tests/test_structlog.py @@ -113,6 +113,34 @@ def test_stock_fields_removal(log_format): assert re.search(stock_field_reps[shown_field], test_log) is not None +@pytest.mark.parametrize("log_format", ["console"]) +def test_custom_console_format(log_format): + custom_format_string = "%(asctime)s %(name)-12s %(levelname)-8s %(message)s" + + with mock.patch.dict( + os.environ, + { + "VDK_LOGGING_METADATA": "timestamp,level,file_name,line_number,vdk_job_name", + "VDK_LOGGING_FORMAT": log_format, + "VDK_LOGGING_CUSTOM_FORMAT": custom_format_string, + }, + ): + logs = _run_job_and_get_logs() + for log in logs: + if "Log statement with no bound context" in log: + assert _matches_custom_format(log) + break + else: + pytest.fail("Log statement with no bound context not found in logs") + + +def _matches_custom_format(log): + # This example pattern is assuming a structure + # based on the format string "%(asctime)s %(name)-12s %(levelname)-8s %(message)s" + pattern = re.compile(r"\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} \S+ \[INFO ]") + return bool(pattern.search(log)) + + def _run_job_and_get_logs(): """ Runs the necessary job and returns the output logs. From cca5524c03c6787e1876a3c3d8a06de6dc0e0ad9 Mon Sep 17 00:00:00 2001 From: Duygu Hasan Date: Thu, 23 Nov 2023 15:18:01 +0200 Subject: [PATCH 06/19] fix regex --- projects/vdk-plugins/vdk-structlog/tests/test_structlog.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/projects/vdk-plugins/vdk-structlog/tests/test_structlog.py b/projects/vdk-plugins/vdk-structlog/tests/test_structlog.py index e78481cd27..0fdc1c2b88 100644 --- a/projects/vdk-plugins/vdk-structlog/tests/test_structlog.py +++ b/projects/vdk-plugins/vdk-structlog/tests/test_structlog.py @@ -135,9 +135,7 @@ def test_custom_console_format(log_format): def _matches_custom_format(log): - # This example pattern is assuming a structure - # based on the format string "%(asctime)s %(name)-12s %(levelname)-8s %(message)s" - pattern = re.compile(r"\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} \S+ \[INFO ]") + pattern = re.compile(r"\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}\s+\S{1,12}\s+\S{1,8}") return bool(pattern.search(log)) From 3c1129fad56dabda560907dd7b71abfdec4d4596 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 23 Nov 2023 13:19:27 +0000 Subject: [PATCH 07/19] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- projects/vdk-plugins/vdk-structlog/tests/test_structlog.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/projects/vdk-plugins/vdk-structlog/tests/test_structlog.py b/projects/vdk-plugins/vdk-structlog/tests/test_structlog.py index 0fdc1c2b88..e520021a15 100644 --- a/projects/vdk-plugins/vdk-structlog/tests/test_structlog.py +++ b/projects/vdk-plugins/vdk-structlog/tests/test_structlog.py @@ -135,7 +135,9 @@ def test_custom_console_format(log_format): def _matches_custom_format(log): - pattern = re.compile(r"\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}\s+\S{1,12}\s+\S{1,8}") + pattern = re.compile( + r"\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}\s+\S{1,12}\s+\S{1,8}" + ) return bool(pattern.search(log)) @@ -213,5 +215,3 @@ def _get_job_arguments(): extra_fields = f'{{"{EXTRA_TEST_KEY}": "{EXTRA_TEST_VALUE}"}}' return f'{{"bound_fields": {bound_fields}, "extra_fields": {extra_fields}}}' - - From 54f57e65b8e27d15c03d279c3d8060bb23ca4bc8 Mon Sep 17 00:00:00 2001 From: Duygu Hasan Date: Fri, 24 Nov 2023 11:38:35 +0200 Subject: [PATCH 08/19] rename constant --- .../src/vdk/plugin/structlog/constants.py | 2 +- .../src/vdk/plugin/structlog/structlog_plugin.py | 10 +++++----- .../vdk-plugins/vdk-structlog/tests/test_structlog.py | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/projects/vdk-plugins/vdk-structlog/src/vdk/plugin/structlog/constants.py b/projects/vdk-plugins/vdk-structlog/src/vdk/plugin/structlog/constants.py index 6938d3ac4b..98df328102 100644 --- a/projects/vdk-plugins/vdk-structlog/src/vdk/plugin/structlog/constants.py +++ b/projects/vdk-plugins/vdk-structlog/src/vdk/plugin/structlog/constants.py @@ -4,7 +4,7 @@ STRUCTLOG_LOGGING_METADATA_KEY = "logging_metadata" STRUCTLOG_LOGGING_FORMAT_KEY = "logging_format" -STRUCTLOG_LOGGING_CUSTOM_FORMAT_KEY = "logging_custom_format" +STRUCTLOG_CONSOLE_LOG_PATTERN = "logging_custom_format" STRUCTLOG_LOGGING_FORMAT_POSSIBLE_VALUES = ["console", "json"] STRUCTLOG_LOGGING_FORMAT_DEFAULT = "console" diff --git a/projects/vdk-plugins/vdk-structlog/src/vdk/plugin/structlog/structlog_plugin.py b/projects/vdk-plugins/vdk-structlog/src/vdk/plugin/structlog/structlog_plugin.py index 17ec82af6c..8047aa2c79 100644 --- a/projects/vdk-plugins/vdk-structlog/src/vdk/plugin/structlog/structlog_plugin.py +++ b/projects/vdk-plugins/vdk-structlog/src/vdk/plugin/structlog/structlog_plugin.py @@ -16,7 +16,7 @@ from vdk.internal.core.config import ConfigurationBuilder from vdk.internal.core.context import CoreContext from vdk.plugin.structlog.constants import JSON_STRUCTLOG_LOGGING_METADATA_DEFAULT -from vdk.plugin.structlog.constants import STRUCTLOG_LOGGING_CUSTOM_FORMAT_KEY +from vdk.plugin.structlog.constants import STRUCTLOG_CONSOLE_LOG_PATTERN from vdk.plugin.structlog.constants import STRUCTLOG_LOGGING_FORMAT_DEFAULT from vdk.plugin.structlog.constants import STRUCTLOG_LOGGING_FORMAT_KEY from vdk.plugin.structlog.constants import STRUCTLOG_LOGGING_FORMAT_POSSIBLE_VALUES @@ -81,7 +81,7 @@ def vdk_configure(self, config_builder: ConfigurationBuilder): ) config_builder.add( - key=STRUCTLOG_LOGGING_CUSTOM_FORMAT_KEY, + key=STRUCTLOG_CONSOLE_LOG_PATTERN, default_value="", description="Custom format string for console logging. Leave empty for default format.", ) @@ -101,7 +101,7 @@ def vdk_initialize(self, context: CoreContext): STRUCTLOG_LOGGING_FORMAT_KEY ) custom_format_string = context.configuration.get_value( - STRUCTLOG_LOGGING_CUSTOM_FORMAT_KEY + STRUCTLOG_CONSOLE_LOG_PATTERN ) formatter, metadata_filter = create_formatter( @@ -126,7 +126,7 @@ def initialize_job(self, context: JobContext) -> None: STRUCTLOG_LOGGING_FORMAT_KEY ) custom_format_string = context.core_context.configuration.get_value( - STRUCTLOG_LOGGING_CUSTOM_FORMAT_KEY + STRUCTLOG_CONSOLE_LOG_PATTERN ) formatter, metadata_filter = create_formatter( @@ -158,7 +158,7 @@ def run_job(self, context: JobContext) -> Optional[ExecutionResult]: STRUCTLOG_LOGGING_FORMAT_KEY ) custom_format_string = context.core_context.configuration.get_value( - STRUCTLOG_LOGGING_CUSTOM_FORMAT_KEY + STRUCTLOG_CONSOLE_LOG_PATTERN ) formatter, metadata_filter = create_formatter( diff --git a/projects/vdk-plugins/vdk-structlog/tests/test_structlog.py b/projects/vdk-plugins/vdk-structlog/tests/test_structlog.py index e520021a15..b2a79869c3 100644 --- a/projects/vdk-plugins/vdk-structlog/tests/test_structlog.py +++ b/projects/vdk-plugins/vdk-structlog/tests/test_structlog.py @@ -122,7 +122,7 @@ def test_custom_console_format(log_format): { "VDK_LOGGING_METADATA": "timestamp,level,file_name,line_number,vdk_job_name", "VDK_LOGGING_FORMAT": log_format, - "VDK_LOGGING_CUSTOM_FORMAT": custom_format_string, + "STRUCTLOG_CONSOLE_LOG_PATTERN": custom_format_string, }, ): logs = _run_job_and_get_logs() From b9de33e98ae1ecede73c2d03cbc8703a779a0abc Mon Sep 17 00:00:00 2001 From: Duygu Hasan Date: Fri, 24 Nov 2023 11:43:06 +0200 Subject: [PATCH 09/19] refactor structlog class to reuse code --- .../vdk/plugin/structlog/structlog_plugin.py | 78 +++++-------------- 1 file changed, 21 insertions(+), 57 deletions(-) diff --git a/projects/vdk-plugins/vdk-structlog/src/vdk/plugin/structlog/structlog_plugin.py b/projects/vdk-plugins/vdk-structlog/src/vdk/plugin/structlog/structlog_plugin.py index 8047aa2c79..77ddc33b66 100644 --- a/projects/vdk-plugins/vdk-structlog/src/vdk/plugin/structlog/structlog_plugin.py +++ b/projects/vdk-plugins/vdk-structlog/src/vdk/plugin/structlog/structlog_plugin.py @@ -94,87 +94,51 @@ def vdk_configure(self, config_builder: ConfigurationBuilder): ), ) - @hookimpl - def vdk_initialize(self, context: CoreContext): + def configure_logger(self, context, additional_filters=[]): metadata_keys = context.configuration.get_value(STRUCTLOG_LOGGING_METADATA_KEY) - logging_formatter = context.configuration.get_value( - STRUCTLOG_LOGGING_FORMAT_KEY - ) - custom_format_string = context.configuration.get_value( - STRUCTLOG_CONSOLE_LOG_PATTERN - ) + logging_formatter = context.configuration.get_value(STRUCTLOG_LOGGING_FORMAT_KEY) + custom_format_string = context.configuration.get_value(STRUCTLOG_CONSOLE_LOG_PATTERN) formatter, metadata_filter = create_formatter( logging_formatter, metadata_keys, custom_format_string ) root_logger = logging.getLogger() - root_logger.removeHandler(root_logger.handlers[0]) + if root_logger.handlers: + root_logger.removeHandler(root_logger.handlers[0]) handler = logging.StreamHandler(sys.stderr) - handler.addFilter(metadata_filter) handler.setFormatter(formatter) + handler.addFilter(metadata_filter) - root_logger.addHandler(handler) + for filter in additional_filters: + handler.addFilter(filter) - @hookimpl(hookwrapper=True) - def initialize_job(self, context: JobContext) -> None: - metadata_keys = context.core_context.configuration.get_value( - STRUCTLOG_LOGGING_METADATA_KEY - ) - logging_formatter = context.core_context.configuration.get_value( - STRUCTLOG_LOGGING_FORMAT_KEY - ) - custom_format_string = context.core_context.configuration.get_value( - STRUCTLOG_CONSOLE_LOG_PATTERN - ) - - formatter, metadata_filter = create_formatter( - logging_formatter, metadata_keys, custom_format_string - ) - job_name_adder = AttributeAdder("vdk_job_name", context.name) + root_logger.addHandler(handler) + def remove_handler(self): root_logger = logging.getLogger() - root_logger.removeHandler(root_logger.handlers[0]) + if root_logger.handlers: + root_logger.removeHandler(root_logger.handlers[-1]) - handler = logging.StreamHandler(sys.stderr) - handler.setFormatter(formatter) - handler.addFilter(job_name_adder) - handler.addFilter(metadata_filter) + @hookimpl + def vdk_initialize(self, context: CoreContext): + self.configure_logger(context) - root_logger.addHandler(handler) + @hookimpl(hookwrapper=True) + def initialize_job(self, context: JobContext) -> None: + job_name_adder = AttributeAdder("vdk_job_name", context.name) + self.configure_logger(context.core_context, [job_name_adder]) out: HookCallResult out = yield - root_logger.removeHandler(handler) + self.remove_handler() @hookimpl(hookwrapper=True) def run_job(self, context: JobContext) -> Optional[ExecutionResult]: - metadata_keys = context.core_context.configuration.get_value( - STRUCTLOG_LOGGING_METADATA_KEY - ) - logging_formatter = context.core_context.configuration.get_value( - STRUCTLOG_LOGGING_FORMAT_KEY - ) - custom_format_string = context.core_context.configuration.get_value( - STRUCTLOG_CONSOLE_LOG_PATTERN - ) - - formatter, metadata_filter = create_formatter( - logging_formatter, metadata_keys, custom_format_string - ) job_name_adder = AttributeAdder("vdk_job_name", context.name) - - root_logger = logging.getLogger() - root_logger.removeHandler(root_logger.handlers[0]) - - handler = logging.StreamHandler(sys.stderr) - handler.setFormatter(formatter) - handler.addFilter(job_name_adder) - handler.addFilter(metadata_filter) - - root_logger.addHandler(handler) + self.configure_logger(context.core_context, [job_name_adder]) out: HookCallResult out = yield From 0dedc3c15807dbf15ddc37cf7a7a5e36b0960b0b Mon Sep 17 00:00:00 2001 From: Duygu Hasan Date: Fri, 24 Nov 2023 12:06:28 +0200 Subject: [PATCH 10/19] revert --- .../vdk/plugin/structlog/structlog_plugin.py | 80 ++++++++++++++----- 1 file changed, 58 insertions(+), 22 deletions(-) diff --git a/projects/vdk-plugins/vdk-structlog/src/vdk/plugin/structlog/structlog_plugin.py b/projects/vdk-plugins/vdk-structlog/src/vdk/plugin/structlog/structlog_plugin.py index 77ddc33b66..5179dd583e 100644 --- a/projects/vdk-plugins/vdk-structlog/src/vdk/plugin/structlog/structlog_plugin.py +++ b/projects/vdk-plugins/vdk-structlog/src/vdk/plugin/structlog/structlog_plugin.py @@ -94,51 +94,87 @@ def vdk_configure(self, config_builder: ConfigurationBuilder): ), ) - def configure_logger(self, context, additional_filters=[]): + @hookimpl + def vdk_initialize(self, context: CoreContext): metadata_keys = context.configuration.get_value(STRUCTLOG_LOGGING_METADATA_KEY) - logging_formatter = context.configuration.get_value(STRUCTLOG_LOGGING_FORMAT_KEY) - custom_format_string = context.configuration.get_value(STRUCTLOG_CONSOLE_LOG_PATTERN) + logging_formatter = context.configuration.get_value( + STRUCTLOG_LOGGING_FORMAT_KEY + ) + custom_format_string = context.configuration.get_value( + STRUCTLOG_CONSOLE_LOG_PATTERN + ) formatter, metadata_filter = create_formatter( logging_formatter, metadata_keys, custom_format_string ) root_logger = logging.getLogger() - if root_logger.handlers: - root_logger.removeHandler(root_logger.handlers[0]) + root_logger.removeHandler(root_logger.handlers[0]) handler = logging.StreamHandler(sys.stderr) - handler.setFormatter(formatter) handler.addFilter(metadata_filter) - - for filter in additional_filters: - handler.addFilter(filter) + handler.setFormatter(formatter) root_logger.addHandler(handler) - def remove_handler(self): - root_logger = logging.getLogger() - if root_logger.handlers: - root_logger.removeHandler(root_logger.handlers[-1]) - - @hookimpl - def vdk_initialize(self, context: CoreContext): - self.configure_logger(context) - @hookimpl(hookwrapper=True) def initialize_job(self, context: JobContext) -> None: + metadata_keys = context.core_context.configuration.get_value( + STRUCTLOG_LOGGING_METADATA_KEY + ) + logging_formatter = context.core_context.configuration.get_value( + STRUCTLOG_LOGGING_FORMAT_KEY + ) + custom_format_string = context.core_context.configuration.get_value( + STRUCTLOG_CONSOLE_LOG_PATTERN + ) + + formatter, metadata_filter = create_formatter( + logging_formatter, metadata_keys, custom_format_string + ) job_name_adder = AttributeAdder("vdk_job_name", context.name) - self.configure_logger(context.core_context, [job_name_adder]) + + root_logger = logging.getLogger() + root_logger.removeHandler(root_logger.handlers[0]) + + handler = logging.StreamHandler(sys.stderr) + handler.setFormatter(formatter) + handler.addFilter(job_name_adder) + handler.addFilter(metadata_filter) + + root_logger.addHandler(handler) out: HookCallResult out = yield - self.remove_handler() + root_logger.removeHandler(handler) @hookimpl(hookwrapper=True) def run_job(self, context: JobContext) -> Optional[ExecutionResult]: + metadata_keys = context.core_context.configuration.get_value( + STRUCTLOG_LOGGING_METADATA_KEY + ) + logging_formatter = context.core_context.configuration.get_value( + STRUCTLOG_LOGGING_FORMAT_KEY + ) + custom_format_string = context.core_context.configuration.get_value( + STRUCTLOG_CONSOLE_LOG_PATTERN + ) + + formatter, metadata_filter = create_formatter( + logging_formatter, metadata_keys, custom_format_string + ) job_name_adder = AttributeAdder("vdk_job_name", context.name) - self.configure_logger(context.core_context, [job_name_adder]) + + root_logger = logging.getLogger() + root_logger.removeHandler(root_logger.handlers[0]) + + handler = logging.StreamHandler(sys.stderr) + handler.setFormatter(formatter) + handler.addFilter(job_name_adder) + handler.addFilter(metadata_filter) + + root_logger.addHandler(handler) out: HookCallResult out = yield @@ -162,4 +198,4 @@ def run_step(self, context: JobContext, step: Step) -> Optional[StepResult]: @hookimpl def vdk_start(plugin_registry: IPluginRegistry, command_line_args: List): - plugin_registry.load_plugin_with_hooks_impl(StructlogPlugin(), "StructlogPlugin") + plugin_registry.load_plugin_with_hooks_impl(StructlogPlugin(), "StructlogPlugin") \ No newline at end of file From 7836a0e7cf7bd468ec85e37095ded2c4b2d32a1f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 24 Nov 2023 10:07:56 +0000 Subject: [PATCH 11/19] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../vdk-structlog/src/vdk/plugin/structlog/structlog_plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/vdk-plugins/vdk-structlog/src/vdk/plugin/structlog/structlog_plugin.py b/projects/vdk-plugins/vdk-structlog/src/vdk/plugin/structlog/structlog_plugin.py index 5179dd583e..8047aa2c79 100644 --- a/projects/vdk-plugins/vdk-structlog/src/vdk/plugin/structlog/structlog_plugin.py +++ b/projects/vdk-plugins/vdk-structlog/src/vdk/plugin/structlog/structlog_plugin.py @@ -198,4 +198,4 @@ def run_step(self, context: JobContext, step: Step) -> Optional[StepResult]: @hookimpl def vdk_start(plugin_registry: IPluginRegistry, command_line_args: List): - plugin_registry.load_plugin_with_hooks_impl(StructlogPlugin(), "StructlogPlugin") \ No newline at end of file + plugin_registry.load_plugin_with_hooks_impl(StructlogPlugin(), "StructlogPlugin") From 8d3a894418e238105c7e91b683530e9266f10709 Mon Sep 17 00:00:00 2001 From: Duygu Hasan Date: Tue, 28 Nov 2023 15:51:10 +0200 Subject: [PATCH 12/19] fix naming --- projects/vdk-plugins/vdk-structlog/tests/test_structlog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/vdk-plugins/vdk-structlog/tests/test_structlog.py b/projects/vdk-plugins/vdk-structlog/tests/test_structlog.py index b2a79869c3..e520021a15 100644 --- a/projects/vdk-plugins/vdk-structlog/tests/test_structlog.py +++ b/projects/vdk-plugins/vdk-structlog/tests/test_structlog.py @@ -122,7 +122,7 @@ def test_custom_console_format(log_format): { "VDK_LOGGING_METADATA": "timestamp,level,file_name,line_number,vdk_job_name", "VDK_LOGGING_FORMAT": log_format, - "STRUCTLOG_CONSOLE_LOG_PATTERN": custom_format_string, + "VDK_LOGGING_CUSTOM_FORMAT": custom_format_string, }, ): logs = _run_job_and_get_logs() From 7ee1e9659bf662f1447080c01dde449452c4804f Mon Sep 17 00:00:00 2001 From: Duygu Hasan Date: Tue, 28 Nov 2023 16:22:53 +0200 Subject: [PATCH 13/19] add changes to README --- projects/vdk-plugins/vdk-structlog/README.md | 35 +++++++++++++++++--- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/projects/vdk-plugins/vdk-structlog/README.md b/projects/vdk-plugins/vdk-structlog/README.md index 91b9dcc7a4..4c212ef416 100644 --- a/projects/vdk-plugins/vdk-structlog/README.md +++ b/projects/vdk-plugins/vdk-structlog/README.md @@ -15,10 +15,37 @@ pip install vdk-structlog (`vdk config-help` is useful command to browse all config options of your installation of vdk) -| Name | Description | Example Value | Possible Values | -| ---------------- | --------------------------------------------------------------------- |---------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------| -| logging_metadata | Configure the metadata that will be output along with the log message | "timestamp, level, logger_name, file_name, vdk_job_name | "timestamp, level, logger_name, file_name, line_number, function_name, vdk_job_name, vdk_step_name, vdk_step_type" | -| logging_format | Configure the logging output format. Available formats: json, console | "console" | "console", "json" | + +| Name | Description | Example Value | Possible Values | +|----------------------------|-----------------------------------------------------------------------------------------------------|---------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------| +| logging_metadata | Configure the metadata that will be output along with the log message | "timestamp, level, logger_name, file_name, vdk_job_name | "timestamp, level, logger_name, file_name, line_number, function_name, vdk_job_name, vdk_step_name, vdk_step_type" | +| logging_format | Configure the logging output format. Available formats: json, console | "console" | "console", "json" | +| custom_console_log_pattern | Custom format string for console logging. Overrides `logging_metadata` and `logging_format` if set. | "%(asctime)s %(name)-12s %(levelname)-8s %(message)s" | Any valid Python logging format string | + +### Example: Configure Custom Console Format + +If you wish to have a specific format for your console logs, you can define a custom format using the `custom_console_log_pattern` configuration. This format string will override the default format specified in `logging_format` and the metadata fields in `logging_metadata`. + +For example, add the following to your data job configuration: + +``` +[vdk] +custom_console_log_pattern=%(asctime)s %(name)-12s %(levelname)-8s %(message)s +``` +When you run your data job, regardless of other logging settings, your logs will strictly follow this custom format, displaying the timestamp, logger's name, log level, and the log message as per the format string specified. + +``` +2023-10-17 11:20:59,202 managed_cursor INFO ingest-from-db-example-job - Executing query SUCCEEDED. Query duration 00h:00m:00s +2023-10-17 11:20:59,202 managed_connectio INFO ingest-from-db-example-job - Fetching query result... +2023-10-17 11:20:59,202 managed_cursor INFO ingest-from-db-example-job - Fetching all results from query ... +2023-10-17 11:20:59,202 managed_cursor INFO ingest-from-db-example-job - Fetching all results from query SUCCEEDED. +2023-10-17 11:20:59,202 managed_cursor INFO ingest-from-db-example-job - Closing DB cursor ... +2023-10-17 11:20:59,202 managed_cursor INFO ingest-from-db-example-job - Closing DB cursor SUCCEEDED. +2023-10-17 11:20:59,203 file_based_step INFO ingest-from-db-example-job - Entering 30_ingest_to_table.py#run(...) ... +2023-10-17 11:20:59,203 ingester_router INFO ingest-from-db-example-job - Sending tabular data for ingestion with method: sqlite and target: None +2023-10-17 11:20:59,204 file_based_step INFO ingest-from-db-example-job - Exiting 30_ingest_to_table.py#run(...) SUCCESS +``` + ### Example: Configure metadata From 3b910bf17ab7e8fc44e2f617398d4b75df03ca0e Mon Sep 17 00:00:00 2001 From: Duygu Hasan Date: Tue, 28 Nov 2023 16:30:38 +0200 Subject: [PATCH 14/19] change tests --- .../vdk-structlog/tests/test_structlog.py | 33 +++++++++++++++---- 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/projects/vdk-plugins/vdk-structlog/tests/test_structlog.py b/projects/vdk-plugins/vdk-structlog/tests/test_structlog.py index e520021a15..19e8146f26 100644 --- a/projects/vdk-plugins/vdk-structlog/tests/test_structlog.py +++ b/projects/vdk-plugins/vdk-structlog/tests/test_structlog.py @@ -114,18 +114,18 @@ def test_stock_fields_removal(log_format): @pytest.mark.parametrize("log_format", ["console"]) -def test_custom_console_format(log_format): +def test_custom_format_applied(log_format): custom_format_string = "%(asctime)s %(name)-12s %(levelname)-8s %(message)s" with mock.patch.dict( os.environ, { - "VDK_LOGGING_METADATA": "timestamp,level,file_name,line_number,vdk_job_name", "VDK_LOGGING_FORMAT": log_format, - "VDK_LOGGING_CUSTOM_FORMAT": custom_format_string, + "VDK_CUSTOM_CONSOLE_LOG_PATTERN": custom_format_string, }, ): logs = _run_job_and_get_logs() + for log in logs: if "Log statement with no bound context" in log: assert _matches_custom_format(log) @@ -134,10 +134,31 @@ def test_custom_console_format(log_format): pytest.fail("Log statement with no bound context not found in logs") +@pytest.mark.parametrize("log_format", ["json", "ltsv"]) +def test_custom_format_overrides_default(log_format): + custom_format_string = "%(asctime)s %(name)-12s %(levelname)-8s %(message)s" + + with mock.patch.dict( + os.environ, + { + "VDK_LOGGING_METADATA": "timestamp,level,file_name,vdk_job_name", + "VDK_LOGGING_FORMAT": log_format, + "VDK_CUSTOM_CONSOLE_LOG_PATTERN": custom_format_string, + }, + ): + logs = _run_job_and_get_logs() + + for log in logs: + if "Log statement with no bound context" in log: + assert _matches_custom_format(log), \ + f"Log format did not match the custom format. Log: {log}" + break + else: + pytest.fail("Log statement with no bound context not found in logs") + + def _matches_custom_format(log): - pattern = re.compile( - r"\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}\s+\S{1,12}\s+\S{1,8}" - ) + pattern = re.compile(r"\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}\s+\S{1,12}\s+\S{1,8}") return bool(pattern.search(log)) From f35d80fd8ea84d8dec046b5708b659dcf9b8777b Mon Sep 17 00:00:00 2001 From: Duygu Hasan Date: Tue, 28 Nov 2023 16:48:25 +0200 Subject: [PATCH 15/19] fix regex --- projects/vdk-plugins/vdk-structlog/tests/test_structlog.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/projects/vdk-plugins/vdk-structlog/tests/test_structlog.py b/projects/vdk-plugins/vdk-structlog/tests/test_structlog.py index 19e8146f26..070b4c320c 100644 --- a/projects/vdk-plugins/vdk-structlog/tests/test_structlog.py +++ b/projects/vdk-plugins/vdk-structlog/tests/test_structlog.py @@ -158,7 +158,9 @@ def test_custom_format_overrides_default(log_format): def _matches_custom_format(log): - pattern = re.compile(r"\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}\s+\S{1,12}\s+\S{1,8}") + pattern = re.compile( + r"\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3} \S{1,12} \S{1,8} .+" + ) return bool(pattern.search(log)) From 3dacb6e9331ee9eacf615345e7be5eef66bf2398 Mon Sep 17 00:00:00 2001 From: Duygu Hasan Date: Wed, 29 Nov 2023 09:18:58 +0200 Subject: [PATCH 16/19] fix create formatter --- .../src/vdk/plugin/structlog/formatters.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/projects/vdk-plugins/vdk-structlog/src/vdk/plugin/structlog/formatters.py b/projects/vdk-plugins/vdk-structlog/src/vdk/plugin/structlog/formatters.py index eecc0ebc4e..190a495948 100644 --- a/projects/vdk-plugins/vdk-structlog/src/vdk/plugin/structlog/formatters.py +++ b/projects/vdk-plugins/vdk-structlog/src/vdk/plugin/structlog/formatters.py @@ -144,7 +144,10 @@ def create_formatter( key_set = set(metadata_keys.split(",")) formatter = None custom_key_filter = None - if logging_format == "json": + if custom_console_format: + formatter = ConsoleFormatter(fmt=custom_console_format) + custom_key_filter = ConsoleMetadataFilter(key_set) + elif logging_format == "json": formatter = JsonFormatter( StructlogMetadataBuilder(metadata_keys).build_json_format() ) @@ -155,12 +158,6 @@ def create_formatter( ) custom_key_filter = ConsoleMetadataFilter(key_set) else: - if custom_console_format: - format_string = custom_console_format - else: - format_string = StructlogMetadataBuilder( - metadata_keys - ).build_console_format() - formatter = ConsoleFormatter(fmt=format_string) + formatter = ConsoleFormatter(fmt=StructlogMetadataBuilder(metadata_keys).build_console_format()) custom_key_filter = ConsoleMetadataFilter(key_set) return formatter, custom_key_filter From e67d945308f128212d95d5972a49659ab4d3597e Mon Sep 17 00:00:00 2001 From: Duygu Hasan Date: Thu, 30 Nov 2023 09:13:42 +0200 Subject: [PATCH 17/19] fix create formatter --- .../src/vdk/plugin/structlog/formatters.py | 12 ++++++------ .../vdk-structlog/tests/test_structlog.py | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/projects/vdk-plugins/vdk-structlog/src/vdk/plugin/structlog/formatters.py b/projects/vdk-plugins/vdk-structlog/src/vdk/plugin/structlog/formatters.py index 190a495948..2dc5e15415 100644 --- a/projects/vdk-plugins/vdk-structlog/src/vdk/plugin/structlog/formatters.py +++ b/projects/vdk-plugins/vdk-structlog/src/vdk/plugin/structlog/formatters.py @@ -144,10 +144,7 @@ def create_formatter( key_set = set(metadata_keys.split(",")) formatter = None custom_key_filter = None - if custom_console_format: - formatter = ConsoleFormatter(fmt=custom_console_format) - custom_key_filter = ConsoleMetadataFilter(key_set) - elif logging_format == "json": + if logging_format == "json": formatter = JsonFormatter( StructlogMetadataBuilder(metadata_keys).build_json_format() ) @@ -157,7 +154,10 @@ def create_formatter( fmt=StructlogMetadataBuilder(metadata_keys).build_ltsv_format() ) custom_key_filter = ConsoleMetadataFilter(key_set) - else: - formatter = ConsoleFormatter(fmt=StructlogMetadataBuilder(metadata_keys).build_console_format()) + elif logging_format == "console": + format_string = custom_console_format if custom_console_format else StructlogMetadataBuilder( + metadata_keys).build_console_format() + formatter = ConsoleFormatter(fmt=format_string) custom_key_filter = ConsoleMetadataFilter(key_set) + return formatter, custom_key_filter diff --git a/projects/vdk-plugins/vdk-structlog/tests/test_structlog.py b/projects/vdk-plugins/vdk-structlog/tests/test_structlog.py index 070b4c320c..9dae436392 100644 --- a/projects/vdk-plugins/vdk-structlog/tests/test_structlog.py +++ b/projects/vdk-plugins/vdk-structlog/tests/test_structlog.py @@ -135,7 +135,7 @@ def test_custom_format_applied(log_format): @pytest.mark.parametrize("log_format", ["json", "ltsv"]) -def test_custom_format_overrides_default(log_format): +def test_custom_format_not_applied_for_non_console_formats(log_format): custom_format_string = "%(asctime)s %(name)-12s %(levelname)-8s %(message)s" with mock.patch.dict( @@ -150,8 +150,8 @@ def test_custom_format_overrides_default(log_format): for log in logs: if "Log statement with no bound context" in log: - assert _matches_custom_format(log), \ - f"Log format did not match the custom format. Log: {log}" + assert not _matches_custom_format(log), \ + f"Custom format was incorrectly applied for {log_format} format. Log: {log}" break else: pytest.fail("Log statement with no bound context not found in logs") From 7434292d2db65461e3b8580fbc0d387d5833c94e Mon Sep 17 00:00:00 2001 From: Duygu Hasan Date: Thu, 30 Nov 2023 09:23:57 +0200 Subject: [PATCH 18/19] refactor readme --- projects/vdk-plugins/vdk-structlog/README.md | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/projects/vdk-plugins/vdk-structlog/README.md b/projects/vdk-plugins/vdk-structlog/README.md index 4c212ef416..57fae3cbb9 100644 --- a/projects/vdk-plugins/vdk-structlog/README.md +++ b/projects/vdk-plugins/vdk-structlog/README.md @@ -15,16 +15,15 @@ pip install vdk-structlog (`vdk config-help` is useful command to browse all config options of your installation of vdk) - -| Name | Description | Example Value | Possible Values | -|----------------------------|-----------------------------------------------------------------------------------------------------|---------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------| -| logging_metadata | Configure the metadata that will be output along with the log message | "timestamp, level, logger_name, file_name, vdk_job_name | "timestamp, level, logger_name, file_name, line_number, function_name, vdk_job_name, vdk_step_name, vdk_step_type" | -| logging_format | Configure the logging output format. Available formats: json, console | "console" | "console", "json" | -| custom_console_log_pattern | Custom format string for console logging. Overrides `logging_metadata` and `logging_format` if set. | "%(asctime)s %(name)-12s %(levelname)-8s %(message)s" | Any valid Python logging format string | +| Name | Description | Example Value | Possible Values | +|----------------------------|---------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------| +| logging_metadata | Configure the metadata that will be output along with the log message | "timestamp, level, logger_name, file_name, vdk_job_name | "timestamp, level, logger_name, file_name, line_number, function_name, vdk_job_name, vdk_step_name, vdk_step_type" | +| logging_format | Configure the logging output format. Available formats: json, console | "console" | "console", "json" | +| custom_console_log_pattern | Custom format string for console logging, applied only when`logging_format` is 'console' and overrides `logging_metadata` | "%(asctime)s %(name)-12s %(levelname)-8s %(message)s" | Any valid Python logging format string | ### Example: Configure Custom Console Format -If you wish to have a specific format for your console logs, you can define a custom format using the `custom_console_log_pattern` configuration. This format string will override the default format specified in `logging_format` and the metadata fields in `logging_metadata`. +If you wish to apply a specific format to your console logs, you can define a custom format using the `custom_console_log_pattern` configuration. This custom format string will be used only when the `logging_format` is set to 'console'. For example, add the following to your data job configuration: From 22115e9a22891ba29dbe201c09920651e00135ae Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 30 Nov 2023 07:24:51 +0000 Subject: [PATCH 19/19] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../vdk-structlog/src/vdk/plugin/structlog/formatters.py | 7 +++++-- projects/vdk-plugins/vdk-structlog/tests/test_structlog.py | 5 +++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/projects/vdk-plugins/vdk-structlog/src/vdk/plugin/structlog/formatters.py b/projects/vdk-plugins/vdk-structlog/src/vdk/plugin/structlog/formatters.py index 2dc5e15415..7c17b835dd 100644 --- a/projects/vdk-plugins/vdk-structlog/src/vdk/plugin/structlog/formatters.py +++ b/projects/vdk-plugins/vdk-structlog/src/vdk/plugin/structlog/formatters.py @@ -155,8 +155,11 @@ def create_formatter( ) custom_key_filter = ConsoleMetadataFilter(key_set) elif logging_format == "console": - format_string = custom_console_format if custom_console_format else StructlogMetadataBuilder( - metadata_keys).build_console_format() + format_string = ( + custom_console_format + if custom_console_format + else StructlogMetadataBuilder(metadata_keys).build_console_format() + ) formatter = ConsoleFormatter(fmt=format_string) custom_key_filter = ConsoleMetadataFilter(key_set) diff --git a/projects/vdk-plugins/vdk-structlog/tests/test_structlog.py b/projects/vdk-plugins/vdk-structlog/tests/test_structlog.py index 9dae436392..7f791b1a7c 100644 --- a/projects/vdk-plugins/vdk-structlog/tests/test_structlog.py +++ b/projects/vdk-plugins/vdk-structlog/tests/test_structlog.py @@ -150,8 +150,9 @@ def test_custom_format_not_applied_for_non_console_formats(log_format): for log in logs: if "Log statement with no bound context" in log: - assert not _matches_custom_format(log), \ - f"Custom format was incorrectly applied for {log_format} format. Log: {log}" + assert not _matches_custom_format( + log + ), f"Custom format was incorrectly applied for {log_format} format. Log: {log}" break else: pytest.fail("Log statement with no bound context not found in logs")