Check all options are valid and present in validate_cfg

CHANGELOG.md

@@ -30,6 +30,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 #### Changed
 
+- Updated validation of config file to check that all options are present and valid [#373](https://github.com/askap-vast/vast-pipeline/pull/373).
 - Bumped datatables to 1.10.22 [#363](https://github.com/askap-vast/vast-pipeline/pull/363).
 - Changed `dom` layout on datatables [#363](https://github.com/askap-vast/vast-pipeline/pull/363).
 - Changed external results table pagination buttons on source detail webinterface page pagination to include less numbers to avoid overlap [#363](https://github.com/askap-vast/vast-pipeline/pull/363).
@@ -53,6 +54,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 #### List of PRs
 
+- [#373](https://github.com/askap-vast/vast-pipeline/pull/373) fix: Check all options are valid and present in validate_cfg.
 - [#379](https://github.com/askap-vast/vast-pipeline/pull/372) feat: Add links to source detail template.
 - [#377](https://github.com/askap-vast/vast-pipeline/pull/377) fix: Update image bkg path when not originally provided.
 - [#363](https://github.com/askap-vast/vast-pipeline/pull/363) feat, dep: Add export and column visibility buttons to tables.

vast_pipeline/pipeline/main.py

@@ -84,6 +84,23 @@ def validate_cfg(self):
         validate a pipeline run configuration against default parameters and
         for different settings (e.g. force extraction)
         """
+        # validate every config from the config template
+        config_keys = [k for k in dir(self.config) if k.isupper()]
+        valid_keys = settings.PIPE_RUN_CONFIG_DEFAULTS.keys()
+
+        # Check that there are no 'unknown' options given by the user
+        for key in config_keys:
+            if key.lower() not in valid_keys:
+                raise PipelineConfigError(
+                    f'Configuration not valid, unknown option: {key}!'
+                )
+        # Check that all options are provided by the user
+        for key in settings.PIPE_RUN_CONFIG_DEFAULTS.keys():
+            if key.upper() not in config_keys:
+                raise PipelineConfigError(
+                    f'Configuration not valid, missing option: {key.upper()}!'
+                )
+
         # do sanity checks
         if (getattr(self.config, 'IMAGE_FILES') and
             getattr(self.config, 'SELAVY_FILES') and
@@ -124,26 +141,6 @@ def validate_cfg(self):
                 f' Must be a value contained in: {association_methods}.'
             ))
 
-        # validate config keys for each association method
-        ass_method = getattr(self.config, 'ASSOCIATION_METHOD')
-        if ass_method == 'basic' or ass_method == 'advanced':
-            if not getattr(self.config, 'ASSOCIATION_RADIUS'):
-                raise PipelineConfigError('ASSOCIATION_RADIUS missing!')
-        else:
-            # deruiter association
-            if (
-                not getattr(self.config, 'ASSOCIATION_DE_RUITER_RADIUS') or not
-                getattr(self.config, 'ASSOCIATION_BEAMWIDTH_LIMIT')
-                ):
-                raise PipelineConfigError((
-                    'ASSOCIATION_DE_RUITER_RADIUS or '
-                    'ASSOCIATION_BEAMWIDTH_LIMIT missing!'
-                ))
-
-        # validate min_new_source_sigma value
-        if 'NEW_SOURCE_MIN_SIGMA' not in dir(self.config):
-            raise PipelineConfigError('NEW_SOURCE_MIN_SIGMA must be defined!')
-
         # validate Forced extraction settings
         if getattr(self.config, 'MONITOR'):
             if not getattr(self.config, 'BACKGROUND_FILES'):
@@ -169,23 +166,6 @@ def validate_cfg(self):
                         f'file:\n{file}\ndoes not exists!'
                     )
 
-            monitor_settings = [
-                'MONITOR_MIN_SIGMA',
-                'MONITOR_EDGE_BUFFER_SCALE',
-                'MONITOR_CLUSTER_THRESHOLD',
-                'MONITOR_ALLOW_NAN',
-            ]
-            for mon_set in monitor_settings:
-                if mon_set not in dir(self.config):
-                    raise PipelineConfigError(mon_set + ' must be defined!')
-
-        # validate every config from the config template
-        for key in [k for k in dir(self.config) if k.isupper()]:
-            if key.lower() not in settings.PIPE_RUN_CONFIG_DEFAULTS.keys():
-                raise PipelineConfigError(
-                    f'configuration not valid, missing key: {key}!'
-                )
-
         pass
 
     def process_pipeline(self, p_run):