vdk-core: introduce sections to configurations (#3319)

Solving: #3305

The main change is that now we have sections list in Configuration and
ConfigurationBuilder instead of having only key-value lists

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Dilyan Marinov <[email protected]>

projects/vdk-core/src/vdk/internal/builtin_plugins/config/config_help.py

@@ -112,7 +112,7 @@ def config_help(ctx: click.Context) -> None:
 
     vars_to_descriptions = {}
     providers_descriptions = {}
-    for k in configuration.list_config_keys():
+    for k in configuration.list_config_keys_from_main_sections():
         description = configuration.get_description(k)
         if description:
             if k.startswith("__config_provider__"):

projects/vdk-core/src/vdk/internal/builtin_plugins/config/vdk_config.py

@@ -187,7 +187,9 @@ def vdk_configure(self, config_builder: ConfigurationBuilder) -> None:
             description=description,
         )
 
-        config_keys = config_builder.list_config_keys()
+        config_keys = (
+            config_builder.list_config_keys_from_main_sections()
+        )  # TODO: make it work with all sections
         log.debug(
             f"Founds config keys: {config_keys}. Will check if environment variable is set for any"
         )
@@ -289,7 +291,5 @@ def vdk_configure(self, config_builder: ConfigurationBuilder) -> None:
                 for key, value in job_config.get_vdk_subsection_values(
                     subsection
                 ).items():
-                    # TODO: after the configuration builder is changed, we need to pass the subsection as
-                    #  section not as key
-                    config_builder.add(key=subsection + "_" + key, default_value="")
-                    config_builder.set_value(key=subsection + "_" + key, value=value)
+                    config_builder.add(key=key, default_value="", section=subsection)
+                    config_builder.set_value(key=key, value=value, section=subsection)

projects/vdk-core/tests/functional/run/test_run_notifications.py

@@ -2,6 +2,7 @@
 # SPDX-License-Identifier: Apache-2.0
 import os
 from email.message import Message
+from random import randint
 from unittest import mock
 
 from click.testing import Result
@@ -18,7 +19,6 @@
 
 def __get_smtp_env(smtpd: SMTPDFix):
     # https://github.com/bebleo/smtpdfix#using
-    smtpd.config.host = "127.0.0.1"
     env = {
         "VDK_NOTIFICATION_ENABLED": "true",
         "VDK_ENABLE_ATTEMPT_NOTIFICATIONS": "true",
@@ -33,88 +33,101 @@ def __get_smtp_env(smtpd: SMTPDFix):
     return env
 
 
-def test_run_successfull(smtpd: SMTPDFix):
+def test_run_successfull():
     errors.resolvable_context().clear()
-    with mock.patch.dict(
-        os.environ,
-        {
-            **__get_smtp_env(smtpd),
-            "VDK_NOTIFIED_ON_JOB_SUCCESS": "[email protected]",
-        },
-    ):
-        runner = CliEntryBasedTestRunner()
 
-        result: Result = runner.invoke(["run", util.job_path("simple-job")])
+    port = randint(10000, 20000)
+    host = "127.0.0.1"
+    with SMTPDFix(host, port) as server:
+        with mock.patch.dict(
+            os.environ,
+            {
+                **__get_smtp_env(server),
+                "VDK_NOTIFIED_ON_JOB_SUCCESS": "[email protected]",
+            },
+        ):
+            runner = CliEntryBasedTestRunner()
 
-        cli_assert_equal(0, result)
+            result: Result = runner.invoke(["run", util.job_path("simple-job")])
 
-        assert len(smtpd.messages) == 1
-        message: Message = smtpd.messages[0]
-        assert "[email protected]" == message.get("To")
-        assert "simple-job" in message.get("Subject")
+            cli_assert_equal(0, result)
 
+            assert len(server.messages) == 1
+            message: Message = server.messages[0]
+            assert "[email protected]" == message.get("To")
+            assert "simple-job" in message.get("Subject")
 
-def test_run_successfull_notify_multiple_users(smtpd: SMTPDFix):
+
+def test_run_successfull_notify_multiple_users():
     errors.resolvable_context().clear()
-    with mock.patch.dict(
-        os.environ,
-        {
-            **__get_smtp_env(smtpd),
-            "VDK_NOTIFIED_ON_JOB_SUCCESS": "[email protected];[email protected];[email protected]",
-        },
-    ):
-        runner = CliEntryBasedTestRunner()
+    port = randint(10000, 20000)
+    host = "127.0.0.1"
+    with SMTPDFix(host, port) as server:
+        with mock.patch.dict(
+            os.environ,
+            {
+                **__get_smtp_env(server),
+                "VDK_NOTIFIED_ON_JOB_SUCCESS": "[email protected];[email protected];[email protected]",
+            },
+        ):
+            runner = CliEntryBasedTestRunner()
 
-        result: Result = runner.invoke(["run", util.job_path("simple-job")])
+            result: Result = runner.invoke(["run", util.job_path("simple-job")])
 
-        cli_assert_equal(0, result)
+            cli_assert_equal(0, result)
 
-        assert len(smtpd.messages) == 1
-        message: Message = smtpd.messages[0]
-        assert "[email protected]" in message.get("To")
-        assert "[email protected]" in message.get("To")
-        assert "[email protected]" in message.get("To")
+            assert len(server.messages) == 1
+            message: Message = server.messages[0]
+            assert "[email protected]" in message.get("To")
+            assert "[email protected]" in message.get("To")
+            assert "[email protected]" in message.get("To")
 
 
 @mock.patch.dict(os.environ, {"VDK_DB_DEFAULT_TYPE": DB_TYPE_SQLITE_MEMORY})
-def test_run_query_failed_user_error_notification_sent(smtpd: SMTPDFix):
+def test_run_query_failed_user_error_notification_sent():
     errors.resolvable_context().clear()
     db_plugin = DecoratedSqLite3MemoryDbPlugin()
     runner = CliEntryBasedTestRunner(db_plugin)
 
-    with mock.patch.dict(
-        os.environ,
-        {
-            **__get_smtp_env(smtpd),
-            "VDK_NOTIFIED_ON_JOB_FAILURE_USER_ERROR": "[email protected]",
-        },
-    ):
-        result: Result = runner.invoke(
-            ["run", util.job_path("simple-create-insert-failed")]
-        )
+    port = randint(10000, 20000)
+    host = "127.0.0.1"
+    with SMTPDFix(host, port) as server:
+        with mock.patch.dict(
+            os.environ,
+            {
+                **__get_smtp_env(server),
+                "VDK_NOTIFIED_ON_JOB_FAILURE_USER_ERROR": "[email protected]",
+            },
+        ):
+            result: Result = runner.invoke(
+                ["run", util.job_path("simple-create-insert-failed")]
+            )
 
-        cli_assert_equal(1, result)
-        assert len(smtpd.messages) == 1
+            cli_assert_equal(1, result)
+            assert len(server.messages) == 1
 
 
 @mock.patch.dict(os.environ, {"VDK_DB_DEFAULT_TYPE": DB_TYPE_SQLITE_MEMORY})
-def test_run_query_failed_user_error_no_notification_configured(smtpd: SMTPDFix):
+def test_run_query_failed_user_error_no_notification_configured():
     errors.resolvable_context().clear()
     db_plugin = DecoratedSqLite3MemoryDbPlugin()
     runner = CliEntryBasedTestRunner(db_plugin)
 
-    with mock.patch.dict(
-        os.environ,
-        {
-            **__get_smtp_env(smtpd),
-            # our job will fail with user error but we have configure platform error
-            # so we should NOT get mail
-            "NOTIFIED_ON_JOB_FAILURE_PLATFORM_ERROR": "[email protected]",
-        },
-    ):
-        result: Result = runner.invoke(
-            ["run", util.job_path("simple-create-insert-failed")]
-        )
-
-        cli_assert_equal(1, result)
-        assert len(smtpd.messages) == 0
+    port = randint(10000, 20000)
+    host = "127.0.0.1"
+    with SMTPDFix(host, port) as server:
+        with mock.patch.dict(
+            os.environ,
+            {
+                **__get_smtp_env(server),
+                # our job will fail with user error but we have configure platform error
+                # so we should NOT get mail
+                "NOTIFIED_ON_JOB_FAILURE_PLATFORM_ERROR": "[email protected]",
+            },
+        ):
+            result: Result = runner.invoke(
+                ["run", util.job_path("simple-create-insert-failed")]
+            )
+
+            cli_assert_equal(1, result)
+            assert len(server.messages) == 0

projects/vdk-core/tests/vdk/internal/builtin_plugins/config/test_config_help.py

@@ -43,7 +43,7 @@ def test_cli_config_help():
     assert "to-be-shown-default-value" in result.output
     assert "key_bool" in result.output
     assert "key_int" in result.output
-    assert "key_no_description" not in result.output
+    assert "key_no_description" in result.output
     assert "key_misconfigured_description" in result.output
     assert "key_sensitive" in result.output
     assert "This option is marked as sensitive." in result.output

projects/vdk-core/tests/vdk/internal/builtin_plugins/ingestion/test_ingester_base.py

@@ -16,6 +16,7 @@
 )
 from vdk.internal.core import errors
 from vdk.internal.core.config import Configuration
+from vdk.internal.core.config import ConfigurationBuilder
 
 shared_test_values = {
     "test_payload1": {"key1": "val1", "key2": "val2", "key3": "val3"},
@@ -42,7 +43,11 @@ def create_ingester_base(kwargs=None, config_dict=None, ingester=None) -> Ingest
     }
     if config_dict is not None:
         config_key_value_pairs.update(config_dict)
-    test_config = Configuration(None, config_key_value_pairs, {})
+    test_config_builder = ConfigurationBuilder()
+    for k, v in config_key_value_pairs.items():
+        test_config_builder.add(key=k, default_value=v)
+
+    test_config = test_config_builder.build()
 
     return IngesterBase(
         data_job_name="test_job",

projects/vdk-core/tests/vdk/internal/builtin_plugins/ingestion/test_ingester_router.py

@@ -7,6 +7,7 @@
 from vdk.api.plugin.plugin_input import IIngesterPlugin
 from vdk.internal.builtin_plugins.ingestion.ingester_base import IngesterBase
 from vdk.internal.builtin_plugins.ingestion.ingester_router import IngesterRouter
+from vdk.internal.core.config import ConfigEntry
 from vdk.internal.core.config import Configuration
 from vdk.internal.core.errors import UserCodeError
 from vdk.internal.core.errors import VdkConfigurationError
@@ -15,15 +16,17 @@
 
 def create_ingester_router(configs) -> IngesterRouter:
     config_key_value_pairs = {
-        "ingester_number_of_worker_threads": 1,
-        "ingester_payload_size_bytes_threshold": 100,
-        "ingester_objects_queue_size": 1,
-        "ingester_payloads_queue_size": 1,
-        "ingester_log_upload_errors": False,
-        "ingestion_payload_aggregator_timeout_seconds": 2,
+        "ingester_number_of_worker_threads": ConfigEntry(value=1),
+        "ingester_payload_size_bytes_threshold": ConfigEntry(value=100),
+        "ingester_objects_queue_size": ConfigEntry(value=1),
+        "ingester_payloads_queue_size": ConfigEntry(value=1),
+        "ingester_log_upload_errors": ConfigEntry(value=False),
+        "ingestion_payload_aggregator_timeout_seconds": ConfigEntry(value=2),
     }
-    config_key_value_pairs.update(configs)
-    test_config = Configuration({}, config_key_value_pairs, {})
+    for i in configs.keys():
+        config_key_value_pairs[i] = ConfigEntry(configs[i])
+    section = {"vdk": config_key_value_pairs}
+    test_config = Configuration(section)
     state_store = MagicMock(spec=StateStore)
     return IngesterRouter(test_config, state_store)