Implement three levels of CMOR checks strictness

doc/esmvalcore/fixing_data.rst

@@ -266,3 +266,42 @@ missing coordinate you can create a fix for this model:
         data_cube = cubes.extract_strict('VAR_NAME')
         data_cube.add_aux_coord(coord, DIMENSIONS_INDEX_TUPLE)
         return [data_cube]
+
+
+Customizing checker strictness
+==============================
+
+The data checker classifies its issues using four different levels of
+severity. From highest to lowest:
+
+ - ``CRITICAL``: issues that most of the time will have severe consequences.
+
+ - ``ERROR``: issues that usually lead to unexpected errors, but can be safely
+    ignored sometimes.
+
+ - ``WARNING``: something is not up to the standard but is unlikely to have
+    consequences later.
+
+ - ``DEBUG``: any info that the checker wants to communicate. Regardless of
+    checker strictness, those will always be reported as debug messages.
+
+Users can have control about which levels of issues are interpreted as errors,
+and therefore make the checker fail or warnings or debug messages.
+For this purpose there is an optional command line option `--check-level`
+that can take a number of values, listed below from the lowest level of
+strictness to the highest:
+
+- ``ignore``: all issues, regardless of severity, will be reported as
+    warnings. Checker will never fail. Use this at your own risk.
+
+- ``relaxed``: only CRITICAL issues are treated as errors. We recommend not to
+    rely on this mode, although it can be useful if there are errors preventing
+    the run that you are sure you can manage on the diagnostics or that will
+    not affect you.
+
+- ``default``: fail if there are any CRITICAL or ERROR issues (DEFAULT); Provides
+    a good measure of safety.
+
+- ``strict``: fail if there are any warnings, this is the highest level of
+    strictness. Mostly useful for checking datasets that you have produced, to
+    be sure that future users will not be distracted by inoffensive warnings.

esmvalcore/_main.py

@@ -41,6 +41,7 @@
 from ._config import configure_logging, read_config_user_file, DIAGNOSTICS_PATH
 from ._recipe import TASKSEP, read_recipe_file
 from ._task import resource_usage_logger
+from .cmor.check import CheckLevels
 
 # set up logging
 logger = logging.getLogger(__name__)
@@ -97,6 +98,24 @@ def get_args():
         '--diagnostics',
         nargs='*',
         help="Only run the named diagnostics from the recipe.")
+    parser.add_argument(
+        '--check-level',
+        type=str,
+        choices=[
+            val.name.lower() for val in CheckLevels if val != CheckLevels.DEBUG
+            ],
+        default='default',
+        help="""
+            Configure the severity of the errors that will make the CMOR check
+            fail.
+            Optional: true;
+            Possible values:
+            ignore: all errors will be reported as warnings
+            relaxed: only fail if there are critical errors
+            default: fail if there are any errors
+            strict: fail if there are any warnings
+        """
+    )
     args = parser.parse_args()
     return args
 
@@ -142,6 +161,7 @@ def main(args):
         pattern if TASKSEP in pattern else pattern + TASKSEP + '*'
         for pattern in args.diagnostics or ()
     }
+    cfg['check_level'] = CheckLevels[args.check_level.upper()]
     cfg['synda_download'] = args.synda_download
     for limit in ('max_datasets', 'max_years'):
         value = getattr(args, limit)

esmvalcore/_recipe.py

@@ -299,6 +299,7 @@ def _get_default_settings(variable, config_user, derive=False):
     settings['fix_file']['output_dir'] = fix_dir
     # Cube fixes
     fix['frequency'] = variable['frequency']
+    fix['check_level'] = config_user['check_level']
     settings['fix_metadata'] = dict(fix)
     settings['fix_data'] = dict(fix)
 
@@ -328,13 +329,15 @@ def _get_default_settings(variable, config_user, derive=False):
         'mip': variable['mip'],
         'short_name': variable['short_name'],
         'frequency': variable['frequency'],
+        'check_level': config_user['check_level']
     }
     # Configure final CMOR data check
     settings['cmor_check_data'] = {
         'cmor_table': variable['project'],
         'mip': variable['mip'],
         'short_name': variable['short_name'],
         'frequency': variable['frequency'],
+        'check_level': config_user['check_level']
     }
 
     # Clean up fixed files