Add config validation checks

DEVELOPING.md

@@ -163,3 +163,13 @@ Run the tests with the following:
 ```bash
 (pipeline_env)$ ./manage.py test pipeline
 ```
+
+Two example tests, currently working are run with the following:
+
+```bash
+(pipeline_env)$ ./manage.py test pipeline.tests.test_runpipeline.CheckRunConfigValidationTest
+```
+
+```bash
+(pipeline_env)$ ./manage.py test pipeline.tests.test_webserver
+```

pipeline/pipeline/main.py

@@ -88,8 +88,19 @@ def validate_cfg(self):
         if (getattr(self.config, 'IMAGE_FILES') and
             getattr(self.config, 'SELAVY_FILES') and
             getattr(self.config, 'NOISE_FILES')):
+            img_f_list = getattr(self.config, 'IMAGE_FILES')
             for lst in ['IMAGE_FILES', 'SELAVY_FILES', 'NOISE_FILES']:
-                for file in getattr(self.config, lst):
+                # checks for duplicates in each list
+                cfg_list = getattr(self.config, lst)
+                if len(set(cfg_list)) != len(cfg_list):
+                    raise PipelineConfigError(f'Duplicated files in: \n{lst}')
+                # check if nr of files match nr of images
+                if len(cfg_list) != len(img_f_list):
+                    raise PipelineConfigError(
+                        f'Number of {lst} files not matching number of images'
+                    )
+                # check if file exists
+                for file in cfg_list:
                     if not os.path.exists(file):
                         raise PipelineConfigError(
                             f'file:\n{file}\ndoes not exists!'
@@ -139,7 +150,20 @@ def validate_cfg(self):
                 raise PipelineConfigError(
                     'Expecting list of background MAP files!'
                 )
-            for file in getattr(self.config, 'BACKGROUND_FILES'):
+            # check for duplicated values
+            backgrd_f_list = getattr(self.config, 'BACKGROUND_FILES')
+            if len(set(backgrd_f_list)) != len(backgrd_f_list):
+                raise PipelineConfigError(
+                    'Duplicated files in: BACKGROUND_FILES list'
+                )
+            # check if provided more background files than images
+            if len(backgrd_f_list) != len(getattr(self.config, 'IMAGE_FILES')):
+                raise PipelineConfigError((
+                    'Number of BACKGROUND_FILES different from number of'
+                    ' IMAGE_FILES files'
+                ))
+            # check if all background files exist
+            for file in backgrd_f_list:
                 if not os.path.exists(file):
                     raise PipelineConfigError(
                         f'file:\n{file}\ndoes not exists!'

pipeline/tests/test_runpipeline.py

@@ -1,11 +1,17 @@
 import os
+from copy import copy
 
 from django.conf import settings as s
-from django.test import TestCase, override_settings
+from django.test import SimpleTestCase, TestCase, override_settings
 from django.core.management import call_command
 
+from pipeline.pipeline.main import Pipeline
+from pipeline.pipeline.errors import PipelineConfigError
+
+
 TEST_ROOT = os.path.join(s.BASE_DIR, 'pipeline', 'tests')
 
+
 @override_settings(
     PIPELINE_WORKING_DIR=os.path.join(TEST_ROOT, 'pipeline-runs'),
     RAW_IMAGE_DIR=os.path.join(TEST_ROOT, 'data'),
@@ -21,3 +27,42 @@ def setUp(self):
     def test_check_run(self):
         run_log_path = os.path.join(self.basic_assoc_run, 'log.txt')
         self.assertTrue(os.path.exists(run_log_path))
+
+
+@override_settings(
+    PIPELINE_WORKING_DIR=os.path.join(TEST_ROOT, 'pipeline-runs')
+)
+class CheckRunConfigValidationTest(SimpleTestCase):
+
+    def setUp(self):
+        config_path = os.path.join(
+            s.PIPELINE_WORKING_DIR,
+            'basic-association',
+            'config.py'
+        )
+
+        self.pipe_run = Pipeline(
+            name='dj_test_basic-association',
+            config_path=config_path
+        )
+        self.config_path = config_path
+
+    def test_duplicated_files(self):
+        for key in ['IMAGE_FILES', 'SELAVY_FILES', 'NOISE_FILES']:
+            f_list = getattr(self.pipe_run.config, key)
+            f_list.append(f_list[0])
+            setattr(self.pipe_run.config, key, f_list)
+            with self.assertRaises(PipelineConfigError):
+                self.pipe_run.validate_cfg()
+            # restore old config
+            self.pipe_run.config = self.pipe_run.load_cfg(self.config_path)
+
+    def test_nr_files_differs(self):
+        for key in ['IMAGE_FILES', 'SELAVY_FILES', 'NOISE_FILES']:
+            f_list = getattr(self.pipe_run.config, key)
+            f_list.append(f_list[0].replace('04', '05'))
+            setattr(self.pipe_run.config, key, f_list)
+            with self.assertRaises(PipelineConfigError):
+                self.pipe_run.validate_cfg()
+            # restore old config
+            self.pipe_run.config = self.pipe_run.load_cfg(self.config_path)