From 2f459c05c03ad2490f48219bbb200d94aef7433f Mon Sep 17 00:00:00 2001 From: Dilyan Marinov Date: Wed, 25 Oct 2023 10:56:34 +0300 Subject: [PATCH] vdk-core: remove redundant logs Why As part of the run logs initiative, we should audit all log statements and error messages in vdk-core and make sure there aren't any redundancies, e.g. overly verbose messages or messages that don't add value. What Remove MSG_CONSEQUENCE_DELEGATING_TO_CALLER__LIKELY_EXECUTION_FAILURE MSG_COUNTERMEASURE_FIX_PARENT_EXCEPTION MSG_CONSEQUENCE_TERMINATING_APP where possible Re-word log statements and exception messages that are too verbose How was this tested? CI What kind of change is this? Feature/non-breaking Signed-off-by: Dilyan Marinov --- .../builtin_plugins/config/log_config.py | 2 -- .../builtin_plugins/connection/impl/router.py | 2 -- .../connection/managed_connection_base.py | 2 -- .../connection/managed_cursor.py | 6 ----- .../ingestion/ingester_router.py | 6 ----- .../job_properties/properties_router.py | 1 - .../job_secrets/secrets_router.py | 1 - .../internal/builtin_plugins/run/cli_run.py | 18 +++++++-------- .../internal/builtin_plugins/run/data_job.py | 1 - .../builtin_plugins/run/file_based_step.py | 6 ++--- .../templates/template_impl.py | 1 - .../vdk-core/src/vdk/internal/core/errors.py | 2 ++ .../src/vdk/internal/plugin/plugin.py | 22 ++++++++----------- 13 files changed, 21 insertions(+), 49 deletions(-) diff --git a/projects/vdk-core/src/vdk/internal/builtin_plugins/config/log_config.py b/projects/vdk-core/src/vdk/internal/builtin_plugins/config/log_config.py index 01a35773f0..fa93e6396a 100644 --- a/projects/vdk-core/src/vdk/internal/builtin_plugins/config/log_config.py +++ b/projects/vdk-core/src/vdk/internal/builtin_plugins/config/log_config.py @@ -61,7 +61,6 @@ def _parse_log_level_module(log_level_module): errors.VdkConfigurationError( "Invalid logging configuration passed to LOG_LEVEL_MODULE.", f"Error is: {e}. log_level_module was set to {log_level_module}.", - "Logging will not be initialized and exception is raised", "Set correctly configuration to log_level_debug configuration in format 'module=level;module2=level2'", ) ) @@ -148,7 +147,6 @@ def configure_loggers( errors.VdkConfigurationError( f"Provided configuration variable for {SYSLOG_SOCK_TYPE} has invalid value.", f"VDK was run with {SYSLOG_SOCK_TYPE}={syslog_sock_type}, however {syslog_sock_type} is invalid value for this variable.", - errors.MSG_CONSEQUENCE_DELEGATING_TO_CALLER__LIKELY_EXECUTION_FAILURE, f"Provide a valid value for {SYSLOG_SOCK_TYPE}." f"Currently possible values are {list(SYSLOG_SOCK_TYPE_VALUES_DICT.keys())}", ) diff --git a/projects/vdk-core/src/vdk/internal/builtin_plugins/connection/impl/router.py b/projects/vdk-core/src/vdk/internal/builtin_plugins/connection/impl/router.py index 0f0276f252..0e620b042c 100644 --- a/projects/vdk-core/src/vdk/internal/builtin_plugins/connection/impl/router.py +++ b/projects/vdk-core/src/vdk/internal/builtin_plugins/connection/impl/router.py @@ -90,7 +90,6 @@ def open_connection(self, dbtype: str) -> ManagedConnectionBase: errors.VdkConfigurationError( f"Provided configuration variable for {DB_DEFAULT_TYPE} has invalid value.", f"VDK was run with {DB_DEFAULT_TYPE}={dbtype}, however {dbtype} is invalid value for this variable.", - errors.MSG_CONSEQUENCE_DELEGATING_TO_CALLER__LIKELY_EXECUTION_FAILURE, f"Provide either valid value for {DB_DEFAULT_TYPE} or install database plugin that supports this type. " f"Currently possible values are {list(self._connection_builders.keys())}", ) @@ -125,7 +124,6 @@ def __create_connection(self, dbtype: str): errors.VdkConfigurationError( f"Could not create new connection of db type {dbtype}.", f"VDK was run with {DB_DEFAULT_TYPE}={dbtype}, however no valid connection was created.", - errors.MSG_CONSEQUENCE_DELEGATING_TO_CALLER__LIKELY_EXECUTION_FAILURE, f"Seems to be a bug in the plugin for dbtype {dbtype}. Make sure it's correctly installed. " f"If upgraded recently consider reverting to previous version. Or use another db type. " f"Currently possible values are {list(self._connection_builders.keys())}", diff --git a/projects/vdk-core/src/vdk/internal/builtin_plugins/connection/managed_connection_base.py b/projects/vdk-core/src/vdk/internal/builtin_plugins/connection/managed_connection_base.py index 92979fb376..3a34d3326a 100644 --- a/projects/vdk-core/src/vdk/internal/builtin_plugins/connection/managed_connection_base.py +++ b/projects/vdk-core/src/vdk/internal/builtin_plugins/connection/managed_connection_base.py @@ -152,8 +152,6 @@ def execute_query(self, query: str) -> List[List[Any]]: [ "Fetching all results from query FAILED.", errors.MSG_WHY_FROM_EXCEPTION(e), - errors.MSG_CONSEQUENCE_DELEGATING_TO_CALLER__LIKELY_EXECUTION_FAILURE, - errors.MSG_COUNTERMEASURE_FIX_PARENT_EXCEPTION, ] ) ) diff --git a/projects/vdk-core/src/vdk/internal/builtin_plugins/connection/managed_cursor.py b/projects/vdk-core/src/vdk/internal/builtin_plugins/connection/managed_cursor.py index 5782ee13d3..a5f169d412 100644 --- a/projects/vdk-core/src/vdk/internal/builtin_plugins/connection/managed_cursor.py +++ b/projects/vdk-core/src/vdk/internal/builtin_plugins/connection/managed_cursor.py @@ -116,8 +116,6 @@ def execute( [ "Executing query FAILED.", errors.MSG_WHY_FROM_EXCEPTION(e), - errors.MSG_CONSEQUENCE_DELEGATING_TO_CALLER__LIKELY_EXECUTION_FAILURE, - errors.MSG_COUNTERMEASURE_FIX_PARENT_EXCEPTION, ] ) ) @@ -142,8 +140,6 @@ def _decorate_operation(self, managed_operation: ManagedOperation, operation: st [ "Decorating query FAILED.", errors.MSG_WHY_FROM_EXCEPTION(e), - errors.MSG_CONSEQUENCE_DELEGATING_TO_CALLER__LIKELY_EXECUTION_FAILURE, - errors.MSG_COUNTERMEASURE_FIX_PARENT_EXCEPTION, ] ) ) @@ -162,8 +158,6 @@ def _validate_operation(self, operation: str, parameters: Optional[Container]): [ "Validating query FAILED.", errors.MSG_WHY_FROM_EXCEPTION(e), - errors.MSG_CONSEQUENCE_DELEGATING_TO_CALLER__LIKELY_EXECUTION_FAILURE, - errors.MSG_COUNTERMEASURE_FIX_PARENT_EXCEPTION, ] ) ) diff --git a/projects/vdk-core/src/vdk/internal/builtin_plugins/ingestion/ingester_router.py b/projects/vdk-core/src/vdk/internal/builtin_plugins/ingestion/ingester_router.py index 735ccf1629..11eddf37df 100644 --- a/projects/vdk-core/src/vdk/internal/builtin_plugins/ingestion/ingester_router.py +++ b/projects/vdk-core/src/vdk/internal/builtin_plugins/ingestion/ingester_router.py @@ -87,7 +87,6 @@ def __get_ingester(self, method: str) -> IngesterBase: VdkConfigurationError( "Provided method, {method}, has invalid value.", "VDK was run with method={method}, however {method} is not part of the available ingestion mechanisms.", - errors.MSG_CONSEQUENCE_DELEGATING_TO_CALLER__LIKELY_EXECUTION_FAILURE, f"Provide either valid value for method, or install ingestion plugin that supports this type. " f"Currently possible values are {list(self._ingester_builders.keys())}", ) @@ -177,7 +176,6 @@ def __initialize_ingester(self, method) -> IngesterBase: VdkConfigurationError( f"Could not create new ingester plugin of type {method}.", f"VDK was run with method={method}, however no valid ingester plugin was created.", - errors.MSG_CONSEQUENCE_DELEGATING_TO_CALLER__LIKELY_EXECUTION_FAILURE, f"Seems to be a bug in the plugin for method {method}. Make sure it's correctly installed. " f"If upgraded recently consider reverting to previous version. Or use another method type.", ) @@ -223,7 +221,6 @@ def __initialize_processor(self, method) -> Optional[IIngesterPlugin]: f"VDK was run with method={method}, however " "no valid ingestion processor plugin was " "created.", - errors.MSG_CONSEQUENCE_DELEGATING_TO_CALLER__LIKELY_EXECUTION_FAILURE, "Seems to be a bug in the plugin for method" f" {method}. Make sure it's correctly " f"installed. If upgraded recently, consider" @@ -257,9 +254,6 @@ def close(self): f"On close some following ingest queues types reported errors: {list(errors_list.keys())}.", f"There were errors while closing ingestion. Exceptions were: {errors_list}.", "Some data was partially ingested or not ingested at all.", - "Follow the instructions in the error messages and log warnings. " - "Make sure to inspect any errors or warning logs generated" - "Re-try the job if necessary", ] if any( diff --git a/projects/vdk-core/src/vdk/internal/builtin_plugins/job_properties/properties_router.py b/projects/vdk-core/src/vdk/internal/builtin_plugins/job_properties/properties_router.py index e159a6d009..a526d65504 100644 --- a/projects/vdk-core/src/vdk/internal/builtin_plugins/job_properties/properties_router.py +++ b/projects/vdk-core/src/vdk/internal/builtin_plugins/job_properties/properties_router.py @@ -92,7 +92,6 @@ def __choose_from_default_type(self, properties_type: str): errors.VdkConfigurationError( f"properties default type was configured to be {properties_type} " f"no such properties api implementation has been registered", - f"", f"Check if the job has not been mis-configured - for example misspelling error. " f"See config-help for help on configuration. Existing properties types are: {list(self.__properties_builders.keys())} " f"Alternatively make sure the correct plugin has been installed " diff --git a/projects/vdk-core/src/vdk/internal/builtin_plugins/job_secrets/secrets_router.py b/projects/vdk-core/src/vdk/internal/builtin_plugins/job_secrets/secrets_router.py index 933d6a664e..d432b90b75 100644 --- a/projects/vdk-core/src/vdk/internal/builtin_plugins/job_secrets/secrets_router.py +++ b/projects/vdk-core/src/vdk/internal/builtin_plugins/job_secrets/secrets_router.py @@ -90,7 +90,6 @@ def __choose_from_default_type(self, secrets_type: str): errors.VdkConfigurationError( f"secrets default type was configured to be {secrets_type} " "no such secrets api implementation has been registered", - "", "Check if the job has not been mis-configured - for example misspelling error. " f"See config-help for help on configuration. Existing secrets types are: {list(self.__secrets_builders.keys())} " "Alternatively make sure the correct plugin has been installed " diff --git a/projects/vdk-core/src/vdk/internal/builtin_plugins/run/cli_run.py b/projects/vdk-core/src/vdk/internal/builtin_plugins/run/cli_run.py index 3026b58961..d06991e11d 100644 --- a/projects/vdk-core/src/vdk/internal/builtin_plugins/run/cli_run.py +++ b/projects/vdk-core/src/vdk/internal/builtin_plugins/run/cli_run.py @@ -45,8 +45,6 @@ def __validate_and_parse_args(arguments: str) -> Optional[Dict]: [ "Failed to validate job arguments.", errors.MSG_WHY_FROM_EXCEPTION(e), - errors.MSG_CONSEQUENCE_TERMINATING_APP, - errors.MSG_COUNTERMEASURE_FIX_PARENT_EXCEPTION, ] ) ) @@ -104,14 +102,16 @@ def __warn_on_python_version_disparity( log.warning( f""" {os.linesep + (' ' * 20) + ('*' * 80)} - What: Python version mismatch between local python and configure python. - Why: The Python version specified in the job's config.ini file is ({configured_python_version}) + Python version mismatch between local python and configure python. + The Python version specified in the job's config.ini file is ({configured_python_version}) while the local python version used to execute the data job is ({local_py_version}). - Consequences: Developing a data job using one Python version and deploying + + Developing a data job using one Python version and deploying it with a different version can result in unexpected and difficult-to-troubleshoot errors like module incompatibilities, or unexpected behavior during execution. - Countermeasures: To resolve this issue, ensure that the Python version + + To resolve this issue, ensure that the Python version specified in the python_version property of the config.ini file matches the Python version of your execution environment by either editing the python_version property in config.ini, or switching local environment @@ -165,10 +165,8 @@ def create_and_run_data_job( [ "Failed executing job.", errors.MSG_WHY_FROM_EXCEPTION(e), - errors.MSG_CONSEQUENCE_TERMINATING_APP, - errors.MSG_COUNTERMEASURE_FIX_PARENT_EXCEPTION - + " Most likely a prerequisite or plugin of one of the key VDK components failed, see" - + " logs for details and ensure the prerequisite for the failed component (details in stacktrace).", + " Most likely a prerequisite or plugin of one of the key VDK components failed, see" + + " logs for details and ensure the prerequisite for the failed component.", ] ) ) diff --git a/projects/vdk-core/src/vdk/internal/builtin_plugins/run/data_job.py b/projects/vdk-core/src/vdk/internal/builtin_plugins/run/data_job.py index 86d4fa00fb..59c1c36415 100644 --- a/projects/vdk-core/src/vdk/internal/builtin_plugins/run/data_job.py +++ b/projects/vdk-core/src/vdk/internal/builtin_plugins/run/data_job.py @@ -121,7 +121,6 @@ def run_job(context: JobContext) -> ExecutionResult: errors.UserCodeError( "Data Job execution has failed.", "Data Job has no steps.", - "Data job execution will not continue.", "Please include at least 1 valid step in your Data Job. Also make sure you are passing the correct data job directory.", ) ) diff --git a/projects/vdk-core/src/vdk/internal/builtin_plugins/run/file_based_step.py b/projects/vdk-core/src/vdk/internal/builtin_plugins/run/file_based_step.py index 88bfa5f4a3..9692994865 100644 --- a/projects/vdk-core/src/vdk/internal/builtin_plugins/run/file_based_step.py +++ b/projects/vdk-core/src/vdk/internal/builtin_plugins/run/file_based_step.py @@ -88,10 +88,8 @@ def run(job_input) [ "Failed loading job sources of %s" % filename, errors.MSG_WHY_FROM_EXCEPTION(e), - errors.MSG_CONSEQUENCE_TERMINATING_APP, - errors.MSG_COUNTERMEASURE_FIX_PARENT_EXCEPTION - + " Most likely importing a dependency or data job step failed, see" - + " logs for details and fix the failed step (details in stacktrace).", + " Most likely importing a dependency or data job step failed, see" + + " logs for details and fix the failed step.", ] ) ) diff --git a/projects/vdk-core/src/vdk/internal/builtin_plugins/templates/template_impl.py b/projects/vdk-core/src/vdk/internal/builtin_plugins/templates/template_impl.py index 14395c15a7..c3e18989c2 100644 --- a/projects/vdk-core/src/vdk/internal/builtin_plugins/templates/template_impl.py +++ b/projects/vdk-core/src/vdk/internal/builtin_plugins/templates/template_impl.py @@ -77,7 +77,6 @@ def get_template_directory(self, name: str) -> pathlib.Path: errors.UserCodeError( f"No registered template with name: {name}.", "Template with that name has not been registered", - errors.MSG_CONSEQUENCE_DELEGATING_TO_CALLER__LIKELY_EXECUTION_FAILURE, "Make sure you have not misspelled the name of the template " "or the plugin(s) providing the template is installed. " f"Current list of templated is: {list(self._registered_templates.keys())}", diff --git a/projects/vdk-core/src/vdk/internal/core/errors.py b/projects/vdk-core/src/vdk/internal/core/errors.py index 0f3e2c22ea..ad730dc412 100644 --- a/projects/vdk-core/src/vdk/internal/core/errors.py +++ b/projects/vdk-core/src/vdk/internal/core/errors.py @@ -10,6 +10,8 @@ """ from __future__ import annotations +from logging import Logger + from vdk.internal.core.error_classifiers import * # Due to the expansion of this file's responsibilities over time, diff --git a/projects/vdk-core/src/vdk/internal/plugin/plugin.py b/projects/vdk-core/src/vdk/internal/plugin/plugin.py index 136fe0e305..1bf3b11a67 100644 --- a/projects/vdk-core/src/vdk/internal/plugin/plugin.py +++ b/projects/vdk-core/src/vdk/internal/plugin/plugin.py @@ -97,19 +97,15 @@ def load_plugin_with_hooks_impl( f"Failed to register plugin {name}. Most likely the plugin name has been forbidden" ) except Exception as e: - message = ErrorMessage( - summary=f"Failed to load plugin ", - what=f"Failed to load plugin with name '{name}' and module/class '{module_or_class_with_hook_impls}'", - why="Most likely a bug in the plugin", - consequences="The CLI tool will likely abort.", - countermeasures="Re-try again. " - "Revert to previous stable version of the plugin or CLI " - "plugin (pip install vdk-plugin-name==version) " - "Or see what plugins are installed (use `pip list` command) and if " - "there are not issues. " - "Or try to reinstall the app in a new clean environment ", - ) - raise PluginException(message) from e + raise PluginException( + f"""Failed to load plugin + Failed to load plugin with name '{name}' and module/class '{module_or_class_with_hook_impls}' + Troubleshooting options: + 1. Check what plugins are installed (use `pip list` command) and see if there are any issues. + 2. Revert to previous stable version of the plugin or CLI plugin (pip install vdk-plugin-name==version) + 3. Reinstall the app in a new clean environment + """ + ) from e def add_hook_specs(self, module_or_class_with_hookspecs: object): self.__plugin_manager.add_hookspecs(module_or_class_with_hookspecs)