Implement three levels of CMOR checks strictness

esmvalcore/_main.py

@@ -98,6 +98,15 @@ def get_args():
         '--diagnostics',
         nargs='*',
         help="Only run the named diagnostics from the recipe.")
+    parser.add_argument(
+        '--cmor-checks',
+        type=str,
+        action='store_true',
+        help="Metadata CMOR checks strictness; \
+             Optional: true; possible values: \
+             none: no checks performed \
+             relaxed: checks are performed but no errors raised; \
+             default: full checks and errors raised.")
     args = parser.parse_args()
     return args
 
@@ -143,6 +152,7 @@ def main(args):
         pattern if TASKSEP in pattern else pattern + TASKSEP + '*'
         for pattern in args.diagnostics or ()
     }
+    cfg['cmor_checks_type'] = args.cmor_checks
     cfg['synda_download'] = args.synda_download
     for limit in ('max_datasets', 'max_years'):
         value = getattr(args, limit)

esmvalcore/_recipe.py

@@ -332,11 +332,19 @@ def _get_default_settings(variable, config_user, derive=False):
 
     # Configure CMOR metadata check
     if variable.get('cmor_table'):
+        raise_exception = True
+        report_only_warning = False
+        if config_user['cmor_checks_type'] in ['none', 'relaxed']:
+            raise_exception = False
+            if config_user['cmor_checks_type'] == 'relaxed':
+                report_only_warning = True
         settings['cmor_check_metadata'] = {
             'cmor_table': variable['cmor_table'],
             'mip': variable['mip'],
             'short_name': variable['short_name'],
             'frequency': variable['frequency'],
+            'raise_exception': raise_exception,
+            'report_only_warning': report_only_warning,
         }
     # Configure final CMOR data check
     if variable.get('cmor_table'):

esmvalcore/cmor/check.py

@@ -57,11 +57,16 @@ def __init__(self,
                  var_info,
                  frequency=None,
                  fail_on_error=False,
+                 raise_exception=True,
+                 report_only_warning=False,
                  automatic_fixes=False):
 
         self._cube = cube
         self._failerr = fail_on_error
+        self._raise_exception = raise_exception
+        self._report_only_warning = report_only_warning
         self._errors = list()
+        self._strict_errors = list()
         self._warnings = list()
         self._debug_messages = list()
         self._cmor_var = var_info
@@ -104,13 +109,16 @@ def check_metadata(self, logger=None):
 
         self.report_debug_messages(logger)
         self.report_warnings(logger)
-        self.report_errors()
+        self.report_errors(logger)
 
         if self.frequency != 'fx':
             self._add_auxiliar_time_coordinates()
-        return self._cube
+        if self.has_errors():
+            return None
+        else:
+            return self._cube
 
-    def report_errors(self):
+    def report_errors(self, logger):
         """Report detected errors.
 
         Raises
@@ -120,10 +128,22 @@ def report_errors(self):
 
         """
         if self.has_errors():
-            msg = 'There were errors in variable {}:\n{}\nin cube:\n{}'
+            msg = 'There are REGULAR CMOR errors in var {}:\n{}\nin cube:\n{}'
             msg = msg.format(self._cube.var_name, '\n '.join(self._errors),
                              self._cube)
-            raise CMORCheckError(msg)
+            if self._raise_exception:
+                logger.error(msg)
+                raise CMORCheckError(msg)
+            if self._report_only_warning:
+                logger.warning(msg)
+        if self.has_strict_errors():
+            msg = 'There are STRICT CMOR errors in var {}:\n{}\nin cube:\n{}'
+            msg = msg.format(
+                self._cube.var_name, '\n '.join(self._strict_errors),
+                self._cube)
+            if self._raise_exception:
+                raise CMORCheckError(msg)
+            logger.error(msg)
 
     def report_warnings(self, logger):
         """Report detected warnings to the given logger.
@@ -179,7 +199,7 @@ def check_data(self, logger=None):
         self._check_coords_data()
 
         self.report_warnings(logger)
-        self.report_errors()
+        self.report_errors(logger)
         return self._cube
 
     def _check_fill_value(self):
@@ -576,6 +596,17 @@ def has_errors(self):
         """
         return len(self._errors) > 0
 
+    def has_strict_errors(self):
+        """Check if there are reported errors of strict nature.
+
+        Returns
+        -------
+        bool:
+            True if there are pending errors, False otherwise.
+
+        """
+        return len(self._strict_errors) > 0
+
     def has_warnings(self):
         """Check if there are reported warnings.
 
@@ -617,6 +648,25 @@ def report_error(self, message, *args):
             raise CMORCheckError(msg + '\nin cube:\n{}'.format(self._cube))
         self._errors.append(msg)
 
+    def report_strict_error(self, message, *args):
+        """Report a STRICT error.
+
+        If fail_on_error is set to True, raises automatically.
+        If fail_on_error is set to False, stores it for later reports.
+
+        Parameters
+        ----------
+        message: str: unicode
+            Message for the error.
+        *args:
+            arguments to format the message string.
+
+        """
+        msg = message.format(*args)
+        if self._failerr:
+            raise CMORCheckError(msg + '\nin cube:\n{}'.format(self._cube))
+        self._strict_errors.append(msg)
+
     def report_warning(self, message, *args):
         """Report a warning.
 
@@ -668,6 +718,7 @@ def _get_cmor_checker(table,
                       short_name,
                       frequency,
                       fail_on_error=True,
+                      raise_exception=True,
                       automatic_fixes=False):
     """Get a CMOR checker/fixer."""
     if table not in CMOR_TABLES:
@@ -687,12 +738,16 @@ def _checker(cube):
             var_info,
             frequency=frequency,
             fail_on_error=fail_on_error,
+            raise_exception=raise_exception,
+            report_only_warning=report_only_warning,
             automatic_fixes=automatic_fixes)
 
     return _checker
 
 
-def cmor_check_metadata(cube, cmor_table, mip, short_name, frequency):
+def cmor_check_metadata(cube, cmor_table, mip,
+                        short_name, frequency,
+                        raise_exception, report_only_warning):
     """Check if metadata conforms to variable's CMOR definiton.
 
     None of the checks at this step will force the cube to load the data.
@@ -709,9 +764,15 @@ def cmor_check_metadata(cube, cmor_table, mip, short_name, frequency):
         Variable's short name.
     frequency: basestring
         Data frequency.
+    raise_exception: bool
+        Boolean operator that raises or not
+        the exception resulted from checker.
 
     """
-    checker = _get_cmor_checker(cmor_table, mip, short_name, frequency)
+    checker = _get_cmor_checker(cmor_table, mip,
+                                short_name, frequency,
+                                raise_exception=raise_exception,
+                                report_only_warning=report_only_warning)
     checker(cube).check_metadata()
     return cube