Merge pull request #3977 from jasonb5/add_chksum_argument

dds --chksum argument to case.submit and create_test allowing
users to verify checksums without explicitly calling
check_input_data with --chksum.

Test suite: scripts_regression_tests.py
Test baseline: n/a
Test namelist changes: n/a
Test status: bit for bit

Fixes #3467

User interface changes?: n/a
Update gh-pages html (Y/N)?: N

scripts/Tools/case.submit

@@ -74,6 +74,9 @@ def parse_command_line(args, description):
     parser.add_argument("-a", "--batch-args",
                         help="Used to pass additional arguments to batch system.")
 
+    parser.add_argument("--chksum", action="store_true",
+                        help="Verifies input data checksums.")
+
     args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser)
 
     CIME.utils.resolve_mail_type_args(args)
@@ -90,13 +93,13 @@ def parse_command_line(args, description):
 
     return (args.caseroot, job, args.no_batch, args.prereq, args.prereq_allow_failure,
             args.resubmit, args.resubmit_immediate, args.skip_preview_namelist, args.mail_user,
-            args.mail_type, args.batch_args, workflow)
+            args.mail_type, args.batch_args, workflow, args.chksum)
 
 ###############################################################################
 def _main_func(description, test_args=False):
 ###############################################################################
     caseroot, job, no_batch, prereq, allow_fail, resubmit, resubmit_immediate, skip_pnl, \
-        mail_user, mail_type, batch_args, workflow = parse_command_line(sys.argv, description)
+        mail_user, mail_type, batch_args, workflow, chksum = parse_command_line(sys.argv, description)
 
     # save these options to a hidden file for use during resubmit
     config_file = os.path.join(caseroot,".submit_options")
@@ -120,7 +123,8 @@ def _main_func(description, test_args=False):
         with Case(caseroot, read_only=False, record=True) as case:
             case.submit(job=job, no_batch=no_batch, prereq=prereq, allow_fail=allow_fail,
                         resubmit=resubmit, resubmit_immediate=resubmit_immediate, skip_pnl=skip_pnl,
-                        mail_user=mail_user, mail_type=mail_type, batch_args=batch_args, workflow=workflow)
+                        mail_user=mail_user, mail_type=mail_type,
+                        batch_args=batch_args, workflow=workflow, chksum=chksum)
 
 if __name__ == "__main__":
     _main_func(__doc__)

scripts/create_test

@@ -305,6 +305,9 @@ def parse_command_line(args, description):
     parser.add_argument("--workflow",default=workflow_default,
                         help="A workflow from config_workflow.xml to apply to this case. ")
 
+    parser.add_argument("--chksum", action="store_true",
+                        help="Verifies input data checksums.")
+
     CIME.utils.add_mail_type_args(parser)
 
     args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser)
@@ -463,7 +466,12 @@ def parse_command_line(args, description):
         args.namelists_only, args.project, \
         args.test_id, args.parallel_jobs, args.walltime, \
         args.single_submit, args.proc_pool, args.use_existing, args.save_timing, args.queue, \
-        args.allow_baseline_overwrite, args.output_root, args.wait, args.force_procs, args.force_threads, args.mpilib, args.input_dir, args.pesfile, args.retry, args.mail_user, args.mail_type, args.check_throughput, args.check_memory, args.ignore_namelists, args.ignore_memleak, args.allow_pnl, args.non_local, args.single_exe, args.workflow
+        args.allow_baseline_overwrite, args.output_root, args.wait, \
+        args.force_procs, args.force_threads, args.mpilib, args.input_dir, args.pesfile, \
+        args.retry, args.mail_user, args.mail_type, args.check_throughput, \
+        args.check_memory, args.ignore_namelists, args.ignore_memleak, args.allow_pnl, \
+        args.non_local, args.single_exe, args.workflow, args.chksum
+
 
 ###############################################################################
 def get_default_setting(config, varname, default_if_not_found, check_main=False):
@@ -572,7 +580,7 @@ def create_test(test_names, test_data, compiler, machine_name, no_run, no_build,
                 walltime, single_submit, proc_pool, use_existing, save_timing, queue, allow_baseline_overwrite, output_root, wait,
                 force_procs, force_threads, mpilib, input_dir, pesfile, mail_user, mail_type,
                 check_throughput, check_memory, ignore_namelists, ignore_memleak,
-                allow_pnl, non_local, single_exe, workflow):
+                allow_pnl, non_local, single_exe, workflow, chksum):
 ###############################################################################
     impl = TestScheduler(test_names, test_data=test_data,
                          no_run=no_run, no_build=no_build, no_setup=no_setup, no_batch=no_batch,
@@ -586,7 +594,8 @@ def create_test(test_names, test_data, compiler, machine_name, no_run, no_build,
                          queue=queue, allow_baseline_overwrite=allow_baseline_overwrite,
                          output_root=output_root, force_procs=force_procs, force_threads=force_threads,
                          mpilib=mpilib, input_dir=input_dir, pesfile=pesfile, mail_user=mail_user, mail_type=mail_type, allow_pnl=allow_pnl,
-                         non_local=non_local, single_exe=single_exe, workflow=workflow)
+                         non_local=non_local, single_exe=single_exe,
+                         workflow=workflow, chksum=chksum)
 
     success = impl.run_tests(wait=wait,
                              check_throughput=check_throughput,
@@ -626,7 +635,7 @@ def _main_func(description):
     project, test_id, parallel_jobs, walltime, single_submit, proc_pool, use_existing, \
     save_timing, queue, allow_baseline_overwrite, output_root, wait, force_procs, force_threads, mpilib, input_dir, pesfile, \
     retry, mail_user, mail_type, check_throughput, check_memory, ignore_namelists, ignore_memleak, allow_pnl, \
-    non_local, single_exe, workflow = \
+    non_local, single_exe, workflow, chksum = \
         parse_command_line(sys.argv, description)
 
     success = False
@@ -639,7 +648,7 @@ def _main_func(description):
                               project, test_id, parallel_jobs, walltime, single_submit, proc_pool, use_existing, save_timing,
                               queue, allow_baseline_overwrite, output_root, wait, force_procs, force_threads, mpilib, input_dir, pesfile,
                               mail_user, mail_type, check_throughput, check_memory, ignore_namelists, ignore_memleak,
-                              allow_pnl, non_local, single_exe, workflow)
+                              allow_pnl, non_local, single_exe, workflow, chksum)
         run_count += 1
 
         # For testing only

scripts/lib/CIME/case/case_submit.py

@@ -25,7 +25,7 @@ def _build_prereq_str(case, prev_job_ids):
 
 def _submit(case, job=None, no_batch=False, prereq=None, allow_fail=False, resubmit=False,
             resubmit_immediate=False, skip_pnl=False, mail_user=None, mail_type=None,
-            batch_args=None, workflow=True):
+            batch_args=None, workflow=True, chksum=False):
     if job is None:
         job = case.get_first_job()
 
@@ -130,7 +130,7 @@ def _submit(case, job=None, no_batch=False, prereq=None, allow_fail=False, resub
         unlock_file(os.path.basename(env_batch.filename))
         lock_file(os.path.basename(env_batch.filename))
 
-        case.check_case(skip_pnl=skip_pnl)
+        case.check_case(skip_pnl=skip_pnl, chksum=chksum)
         if job == case.get_primary_job():
             case.check_DA_settings()
             if case.get_value("MACH") == "mira":
@@ -162,7 +162,7 @@ def _submit(case, job=None, no_batch=False, prereq=None, allow_fail=False, resub
 
 def submit(self, job=None, no_batch=False, prereq=None, allow_fail=False, resubmit=False,
            resubmit_immediate=False, skip_pnl=False, mail_user=None, mail_type=None,
-           batch_args=None, workflow=True):
+           batch_args=None, workflow=True, chksum=False):
     if resubmit_immediate and self.get_value("MACH") in ['mira', 'cetus']:
         logger.warning("resubmit_immediate does not work on Mira/Cetus, submitting normally")
         resubmit_immediate = False
@@ -202,7 +202,8 @@ def submit(self, job=None, no_batch=False, prereq=None, allow_fail=False, resubm
                                   allow_fail=allow_fail, resubmit=resubmit,
                                   resubmit_immediate=resubmit_immediate, skip_pnl=skip_pnl,
                                   mail_user=mail_user, mail_type=mail_type,
-                                  batch_args=batch_args, workflow=workflow)
+                                  batch_args=batch_args, workflow=workflow,
+                                  chksum=chksum)
         run_and_log_case_status(functor, "case.submit", caseroot=caseroot,
                                 custom_success_msg_functor=lambda x: x.split(":")[-1],
                                 is_batch=is_batch)
@@ -215,12 +216,12 @@ def submit(self, job=None, no_batch=False, prereq=None, allow_fail=False, resubm
 
         raise
 
-def check_case(self, skip_pnl=False):
+def check_case(self, skip_pnl=False, chksum=False):
     self.check_lockedfiles()
     if not skip_pnl:
         self.create_namelists() # Must be called before check_all_input_data
     logger.info("Checking that inputdata is available as part of case submission")
-    self.check_all_input_data()
+    self.check_all_input_data(chksum=chksum)
 
     if self.get_value('COMP_WAV') == 'ww':
         # the ww3 buildnml has dependencies on inputdata so we must run it again

scripts/lib/CIME/test_scheduler.py

@@ -123,7 +123,7 @@ def __init__(self, test_names, test_data=None,
                  allow_baseline_overwrite=False, output_root=None,
                  force_procs=None, force_threads=None, mpilib=None,
                  input_dir=None, pesfile=None, mail_user=None, mail_type=None, allow_pnl=False,
-                 non_local=False, single_exe=False, workflow=None):
+                 non_local=False, single_exe=False, workflow=None, chksum=False):
     ###########################################################################
         self._cime_root       = get_cime_root()
         self._cime_model      = get_model()
@@ -317,6 +317,7 @@ def __init__(self, test_names, test_data=None,
             for test_name in build_group:
                 logger.debug("{}{}".format("  " if test_name == build_group[0] else "    ", test_name))
 
+        self._chksum = chksum
         # By the end of this constructor, this program should never hard abort,
         # instead, errors will be placed in the TestStatus files for the various
         # tests cases
@@ -798,6 +799,8 @@ def _run_phase(self, test):
                 cmd += " --mail-user={}".format(self._mail_user)
             if self._mail_type:
                 cmd += " -M={}".format(",".join(self._mail_type))
+            if self._chksum:
+                cmd += " --chksum"
 
             return self._shell_cmd_for_phase(test, cmd, RUN_PHASE, from_dir=test_dir)
 

scripts/lib/CIME/tests/test_case.py

@@ -1,11 +1,63 @@
 import os
 import unittest
-import tempfile
 from unittest import mock
+import tempfile
 
+from CIME.case import case_submit
 from CIME.case import Case
 from CIME import utils as cime_utils
 
+class TestCaseSubmit(unittest.TestCase):
+
+    def test_check_case(self):
+        case = mock.MagicMock()
+
+        case_submit.check_case(case, chksum=True)
+
+        case.check_all_input_data.assert_called_with(chksum=True)
+
+    @mock.patch("CIME.case.case_submit.lock_file")
+    def test__submit(self, lock_file): # pylint: disable=unused-argument
+        case = mock.MagicMock()
+
+        case_submit._submit(case, chksum=True) # pylint: disable=protected-access
+
+        case.check_case.assert_called_with(skip_pnl=False, chksum=True)
+
+    @mock.patch("CIME.case.case_submit._submit")
+    @mock.patch("CIME.case.case.Case.initialize_derived_attributes")
+    @mock.patch("CIME.case.case.Case.get_value")
+    @mock.patch("CIME.case.case.Case.read_xml")
+    def test_submit(self, read_xml, get_value, init, _submit): # pylint: disable=unused-argument
+        with tempfile.TemporaryDirectory() as tempdir:
+            get_value.side_effect = [
+                tempdir,
+                tempdir,
+                True,
+                "baseid",
+                None,
+                True,
+            ]
+
+            with Case(tempdir) as case:
+                case.submit(chksum=True)
+
+            _submit.assert_called_with(
+                case,
+                job=None,
+                no_batch=False,
+                prereq=None,
+                allow_fail=False,
+                resubmit=False,
+                resubmit_immediate=False,
+                skip_pnl=False,
+                mail_user=None,
+                mail_type=None,
+                batch_args=None,
+                workflow=True,
+                chksum=True
+            )
+
 class TestCase(unittest.TestCase):
 
     def setUp(self):

scripts/lib/CIME/tests/test_test_scheduler.py

@@ -0,0 +1,27 @@
+import os
+import unittest
+from unittest import mock
+
+from CIME.test_scheduler import TestScheduler
+
+class TestTestScheduler(unittest.TestCase):
+
+    @mock.patch.dict(os.environ, {"CIME_MODEL": "cesm"})
+    @mock.patch("time.strftime", return_value="00:00:00")
+    def test_chksum(self, strftime): # pylint: disable=unused-argument
+        ts = TestScheduler(
+            ["SEQ_Ln9.f19_g16_rx1.A.ubuntu-latest_gnu"],
+            machine_name="ubuntu-latest",
+            chksum=True,
+            test_root="/tests",
+        )
+
+        with mock.patch.object(ts, "_shell_cmd_for_phase") as _shell_cmd_for_phase:
+            ts._run_phase("SEQ_Ln9.f19_g16_rx1.A.ubuntu-latest_gnu") # pylint: disable=protected-access
+
+            _shell_cmd_for_phase.assert_called_with(
+                "SEQ_Ln9.f19_g16_rx1.A.ubuntu-latest_gnu",
+                "./case.submit --skip-preview-namelist --no-batch --chksum",
+                "RUN",
+                from_dir="/tests/SEQ_Ln9.f19_g16_rx1.A.ubuntu-latest_gnu.00:00:00",
+            )