Check all options are valid and present in validate_cfg

CHANGELOG.md

@@ -37,6 +37,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 #### Changed
 
+- Updated validation of config file to check that all options are present and valid [#373](https://github.com/askap-vast/vast-pipeline/pull/373).
 - Rewritten relation functions to improve speed [#307](https://github.com/askap-vast/vast-pipeline/pull/307).
 - Minor changes to association to increase speed [#307](https://github.com/askap-vast/vast-pipeline/pull/307).
 - Changes to decrease memory usage during the calculation of the ideal coverage dataframe [#307](https://github.com/askap-vast/vast-pipeline/pull/307).
@@ -66,6 +67,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 #### List of PRs
 
+- [#373](https://github.com/askap-vast/vast-pipeline/pull/373) fix: Check all options are valid and present in validate_cfg.
 - [#307](https://github.com/askap-vast/vast-pipeline/pull/307) feat: Improve relation functions and general association speed ups.
 - [#277](https://github.com/askap-vast/vast-pipeline/pull/277) feat,model: Parallel and epoch based association.
 - [#380](https://github.com/askap-vast/vast-pipeline/pull/380) feat, dep: Enable Excel export button.

vast_pipeline/pipeline/main.py

@@ -99,6 +99,23 @@ def validate_cfg(self):
         validate a pipeline run configuration against default parameters and
         for different settings (e.g. force extraction)
         """
+        # validate every config from the config template
+        config_keys = [k for k in dir(self.config) if k.isupper()]
+        valid_keys = settings.PIPE_RUN_CONFIG_DEFAULTS.keys()
+
+        # Check that there are no 'unknown' options given by the user
+        for key in config_keys:
+            if key.lower() not in valid_keys:
+                raise PipelineConfigError(
+                    f'Configuration not valid, unknown option: {key}!'
+                )
+        # Check that all options are provided by the user
+        for key in settings.PIPE_RUN_CONFIG_DEFAULTS.keys():
+            if key.upper() not in config_keys:
+                raise PipelineConfigError(
+                    f'Configuration not valid, missing option: {key.upper()}!'
+                )
+
         # do sanity checks
         if (getattr(self.config, 'IMAGE_FILES') and
             getattr(self.config, 'SELAVY_FILES') and
@@ -148,43 +165,13 @@ def validate_cfg(self):
                 f' Must be a value contained in: {association_methods}.'
             ))
 
-        # validate config keys for each association method
-        ass_method = getattr(self.config, 'ASSOCIATION_METHOD')
-        if ass_method == 'basic' or ass_method == 'advanced':
-            if not getattr(self.config, 'ASSOCIATION_RADIUS'):
-                raise PipelineConfigError('ASSOCIATION_RADIUS missing!')
-        else:
-            # deruiter association
-            if (
-                not getattr(self.config, 'ASSOCIATION_DE_RUITER_RADIUS') or not
-                getattr(self.config, 'ASSOCIATION_BEAMWIDTH_LIMIT')
-                ):
-                raise PipelineConfigError((
-                    'ASSOCIATION_DE_RUITER_RADIUS or '
-                    'ASSOCIATION_BEAMWIDTH_LIMIT missing!'
-                ))
-
-        # validate min_new_source_sigma value
-        if 'NEW_SOURCE_MIN_SIGMA' not in dir(self.config):
-            raise PipelineConfigError('NEW_SOURCE_MIN_SIGMA must be defined!')
-
         # validate Forced extraction settings
         if getattr(self.config, 'MONITOR'):
             if not getattr(self.config, 'BACKGROUND_FILES'):
                 raise PipelineConfigError(
                     'Expecting list of background MAP files!'
                 )
 
-            monitor_settings = [
-                'MONITOR_MIN_SIGMA',
-                'MONITOR_EDGE_BUFFER_SCALE',
-                'MONITOR_CLUSTER_THRESHOLD',
-                'MONITOR_ALLOW_NAN',
-            ]
-            for mon_set in monitor_settings:
-                if mon_set not in dir(self.config):
-                    raise PipelineConfigError(mon_set + ' must be defined!')
-
         # if defined, check background files regardless of monitor
         if getattr(self.config, 'BACKGROUND_FILES'):
             # check for duplicated values
@@ -209,14 +196,6 @@ def validate_cfg(self):
                         raise PipelineConfigError(
                             f'file:\n{file}\ndoes not exists!'
                         )
-
-        # validate every config from the config template
-        for key in [k for k in dir(self.config) if k.isupper()]:
-            if key.lower() not in settings.PIPE_RUN_CONFIG_DEFAULTS.keys():
-                raise PipelineConfigError(
-                    f'configuration not valid, missing key: {key}!'
-                )
-
         pass
 
     def match_images_to_data(self):