diff --git a/scripts/Tools/case.submit b/scripts/Tools/case.submit index 4ad64109559..49979d86237 100755 --- a/scripts/Tools/case.submit +++ b/scripts/Tools/case.submit @@ -74,6 +74,9 @@ def parse_command_line(args, description): parser.add_argument("-a", "--batch-args", help="Used to pass additional arguments to batch system.") + parser.add_argument("--chksum", action="store_true", + help="Verifies input data checksums.") + args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) CIME.utils.resolve_mail_type_args(args) @@ -90,13 +93,13 @@ def parse_command_line(args, description): return (args.caseroot, job, args.no_batch, args.prereq, args.prereq_allow_failure, args.resubmit, args.resubmit_immediate, args.skip_preview_namelist, args.mail_user, - args.mail_type, args.batch_args, workflow) + args.mail_type, args.batch_args, workflow, args.chksum) ############################################################################### def _main_func(description, test_args=False): ############################################################################### caseroot, job, no_batch, prereq, allow_fail, resubmit, resubmit_immediate, skip_pnl, \ - mail_user, mail_type, batch_args, workflow = parse_command_line(sys.argv, description) + mail_user, mail_type, batch_args, workflow, chksum = parse_command_line(sys.argv, description) # save these options to a hidden file for use during resubmit config_file = os.path.join(caseroot,".submit_options") @@ -120,7 +123,8 @@ def _main_func(description, test_args=False): with Case(caseroot, read_only=False, record=True) as case: case.submit(job=job, no_batch=no_batch, prereq=prereq, allow_fail=allow_fail, resubmit=resubmit, resubmit_immediate=resubmit_immediate, skip_pnl=skip_pnl, - mail_user=mail_user, mail_type=mail_type, batch_args=batch_args, workflow=workflow) + mail_user=mail_user, mail_type=mail_type, + batch_args=batch_args, workflow=workflow, chksum=chksum) if __name__ == "__main__": _main_func(__doc__) diff --git a/scripts/create_test b/scripts/create_test index db3cf968af9..dff85a1cc45 100755 --- a/scripts/create_test +++ b/scripts/create_test @@ -305,6 +305,9 @@ def parse_command_line(args, description): parser.add_argument("--workflow",default=workflow_default, help="A workflow from config_workflow.xml to apply to this case. ") + parser.add_argument("--chksum", action="store_true", + help="Verifies input data checksums.") + CIME.utils.add_mail_type_args(parser) args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) @@ -463,7 +466,12 @@ def parse_command_line(args, description): args.namelists_only, args.project, \ args.test_id, args.parallel_jobs, args.walltime, \ args.single_submit, args.proc_pool, args.use_existing, args.save_timing, args.queue, \ - args.allow_baseline_overwrite, args.output_root, args.wait, args.force_procs, args.force_threads, args.mpilib, args.input_dir, args.pesfile, args.retry, args.mail_user, args.mail_type, args.check_throughput, args.check_memory, args.ignore_namelists, args.ignore_memleak, args.allow_pnl, args.non_local, args.single_exe, args.workflow + args.allow_baseline_overwrite, args.output_root, args.wait, \ + args.force_procs, args.force_threads, args.mpilib, args.input_dir, args.pesfile, \ + args.retry, args.mail_user, args.mail_type, args.check_throughput, \ + args.check_memory, args.ignore_namelists, args.ignore_memleak, args.allow_pnl, \ + args.non_local, args.single_exe, args.workflow, args.chksum + ############################################################################### def get_default_setting(config, varname, default_if_not_found, check_main=False): @@ -572,7 +580,7 @@ def create_test(test_names, test_data, compiler, machine_name, no_run, no_build, walltime, single_submit, proc_pool, use_existing, save_timing, queue, allow_baseline_overwrite, output_root, wait, force_procs, force_threads, mpilib, input_dir, pesfile, mail_user, mail_type, check_throughput, check_memory, ignore_namelists, ignore_memleak, - allow_pnl, non_local, single_exe, workflow): + allow_pnl, non_local, single_exe, workflow, chksum): ############################################################################### impl = TestScheduler(test_names, test_data=test_data, no_run=no_run, no_build=no_build, no_setup=no_setup, no_batch=no_batch, @@ -586,7 +594,8 @@ def create_test(test_names, test_data, compiler, machine_name, no_run, no_build, queue=queue, allow_baseline_overwrite=allow_baseline_overwrite, output_root=output_root, force_procs=force_procs, force_threads=force_threads, mpilib=mpilib, input_dir=input_dir, pesfile=pesfile, mail_user=mail_user, mail_type=mail_type, allow_pnl=allow_pnl, - non_local=non_local, single_exe=single_exe, workflow=workflow) + non_local=non_local, single_exe=single_exe, + workflow=workflow, chksum=chksum) success = impl.run_tests(wait=wait, check_throughput=check_throughput, @@ -626,7 +635,7 @@ def _main_func(description): project, test_id, parallel_jobs, walltime, single_submit, proc_pool, use_existing, \ save_timing, queue, allow_baseline_overwrite, output_root, wait, force_procs, force_threads, mpilib, input_dir, pesfile, \ retry, mail_user, mail_type, check_throughput, check_memory, ignore_namelists, ignore_memleak, allow_pnl, \ - non_local, single_exe, workflow = \ + non_local, single_exe, workflow, chksum = \ parse_command_line(sys.argv, description) success = False @@ -639,7 +648,7 @@ def _main_func(description): project, test_id, parallel_jobs, walltime, single_submit, proc_pool, use_existing, save_timing, queue, allow_baseline_overwrite, output_root, wait, force_procs, force_threads, mpilib, input_dir, pesfile, mail_user, mail_type, check_throughput, check_memory, ignore_namelists, ignore_memleak, - allow_pnl, non_local, single_exe, workflow) + allow_pnl, non_local, single_exe, workflow, chksum) run_count += 1 # For testing only diff --git a/scripts/lib/CIME/case/case_submit.py b/scripts/lib/CIME/case/case_submit.py index ebf182b2f88..54fd4054b2d 100644 --- a/scripts/lib/CIME/case/case_submit.py +++ b/scripts/lib/CIME/case/case_submit.py @@ -25,7 +25,7 @@ def _build_prereq_str(case, prev_job_ids): def _submit(case, job=None, no_batch=False, prereq=None, allow_fail=False, resubmit=False, resubmit_immediate=False, skip_pnl=False, mail_user=None, mail_type=None, - batch_args=None, workflow=True): + batch_args=None, workflow=True, chksum=False): if job is None: job = case.get_first_job() @@ -130,7 +130,7 @@ def _submit(case, job=None, no_batch=False, prereq=None, allow_fail=False, resub unlock_file(os.path.basename(env_batch.filename)) lock_file(os.path.basename(env_batch.filename)) - case.check_case(skip_pnl=skip_pnl) + case.check_case(skip_pnl=skip_pnl, chksum=chksum) if job == case.get_primary_job(): case.check_DA_settings() if case.get_value("MACH") == "mira": @@ -162,7 +162,7 @@ def _submit(case, job=None, no_batch=False, prereq=None, allow_fail=False, resub def submit(self, job=None, no_batch=False, prereq=None, allow_fail=False, resubmit=False, resubmit_immediate=False, skip_pnl=False, mail_user=None, mail_type=None, - batch_args=None, workflow=True): + batch_args=None, workflow=True, chksum=False): if resubmit_immediate and self.get_value("MACH") in ['mira', 'cetus']: logger.warning("resubmit_immediate does not work on Mira/Cetus, submitting normally") resubmit_immediate = False @@ -202,7 +202,8 @@ def submit(self, job=None, no_batch=False, prereq=None, allow_fail=False, resubm allow_fail=allow_fail, resubmit=resubmit, resubmit_immediate=resubmit_immediate, skip_pnl=skip_pnl, mail_user=mail_user, mail_type=mail_type, - batch_args=batch_args, workflow=workflow) + batch_args=batch_args, workflow=workflow, + chksum=chksum) run_and_log_case_status(functor, "case.submit", caseroot=caseroot, custom_success_msg_functor=lambda x: x.split(":")[-1], is_batch=is_batch) @@ -215,12 +216,12 @@ def submit(self, job=None, no_batch=False, prereq=None, allow_fail=False, resubm raise -def check_case(self, skip_pnl=False): +def check_case(self, skip_pnl=False, chksum=False): self.check_lockedfiles() if not skip_pnl: self.create_namelists() # Must be called before check_all_input_data logger.info("Checking that inputdata is available as part of case submission") - self.check_all_input_data() + self.check_all_input_data(chksum=chksum) if self.get_value('COMP_WAV') == 'ww': # the ww3 buildnml has dependencies on inputdata so we must run it again diff --git a/scripts/lib/CIME/test_scheduler.py b/scripts/lib/CIME/test_scheduler.py index 14b046d6626..90d8e450c2e 100644 --- a/scripts/lib/CIME/test_scheduler.py +++ b/scripts/lib/CIME/test_scheduler.py @@ -123,7 +123,7 @@ def __init__(self, test_names, test_data=None, allow_baseline_overwrite=False, output_root=None, force_procs=None, force_threads=None, mpilib=None, input_dir=None, pesfile=None, mail_user=None, mail_type=None, allow_pnl=False, - non_local=False, single_exe=False, workflow=None): + non_local=False, single_exe=False, workflow=None, chksum=False): ########################################################################### self._cime_root = get_cime_root() self._cime_model = get_model() @@ -317,6 +317,7 @@ def __init__(self, test_names, test_data=None, for test_name in build_group: logger.debug("{}{}".format(" " if test_name == build_group[0] else " ", test_name)) + self._chksum = chksum # By the end of this constructor, this program should never hard abort, # instead, errors will be placed in the TestStatus files for the various # tests cases @@ -798,6 +799,8 @@ def _run_phase(self, test): cmd += " --mail-user={}".format(self._mail_user) if self._mail_type: cmd += " -M={}".format(",".join(self._mail_type)) + if self._chksum: + cmd += " --chksum" return self._shell_cmd_for_phase(test, cmd, RUN_PHASE, from_dir=test_dir) diff --git a/scripts/lib/CIME/tests/test_case.py b/scripts/lib/CIME/tests/test_case.py index 6ea84fc9b83..88e7821443d 100644 --- a/scripts/lib/CIME/tests/test_case.py +++ b/scripts/lib/CIME/tests/test_case.py @@ -1,11 +1,63 @@ import os import unittest -import tempfile from unittest import mock +import tempfile +from CIME.case import case_submit from CIME.case import Case from CIME import utils as cime_utils +class TestCaseSubmit(unittest.TestCase): + + def test_check_case(self): + case = mock.MagicMock() + + case_submit.check_case(case, chksum=True) + + case.check_all_input_data.assert_called_with(chksum=True) + + @mock.patch("CIME.case.case_submit.lock_file") + def test__submit(self, lock_file): # pylint: disable=unused-argument + case = mock.MagicMock() + + case_submit._submit(case, chksum=True) # pylint: disable=protected-access + + case.check_case.assert_called_with(skip_pnl=False, chksum=True) + + @mock.patch("CIME.case.case_submit._submit") + @mock.patch("CIME.case.case.Case.initialize_derived_attributes") + @mock.patch("CIME.case.case.Case.get_value") + @mock.patch("CIME.case.case.Case.read_xml") + def test_submit(self, read_xml, get_value, init, _submit): # pylint: disable=unused-argument + with tempfile.TemporaryDirectory() as tempdir: + get_value.side_effect = [ + tempdir, + tempdir, + True, + "baseid", + None, + True, + ] + + with Case(tempdir) as case: + case.submit(chksum=True) + + _submit.assert_called_with( + case, + job=None, + no_batch=False, + prereq=None, + allow_fail=False, + resubmit=False, + resubmit_immediate=False, + skip_pnl=False, + mail_user=None, + mail_type=None, + batch_args=None, + workflow=True, + chksum=True + ) + class TestCase(unittest.TestCase): def setUp(self): diff --git a/scripts/lib/CIME/tests/test_test_scheduler.py b/scripts/lib/CIME/tests/test_test_scheduler.py new file mode 100644 index 00000000000..b907151e6b9 --- /dev/null +++ b/scripts/lib/CIME/tests/test_test_scheduler.py @@ -0,0 +1,27 @@ +import os +import unittest +from unittest import mock + +from CIME.test_scheduler import TestScheduler + +class TestTestScheduler(unittest.TestCase): + + @mock.patch.dict(os.environ, {"CIME_MODEL": "cesm"}) + @mock.patch("time.strftime", return_value="00:00:00") + def test_chksum(self, strftime): # pylint: disable=unused-argument + ts = TestScheduler( + ["SEQ_Ln9.f19_g16_rx1.A.ubuntu-latest_gnu"], + machine_name="ubuntu-latest", + chksum=True, + test_root="/tests", + ) + + with mock.patch.object(ts, "_shell_cmd_for_phase") as _shell_cmd_for_phase: + ts._run_phase("SEQ_Ln9.f19_g16_rx1.A.ubuntu-latest_gnu") # pylint: disable=protected-access + + _shell_cmd_for_phase.assert_called_with( + "SEQ_Ln9.f19_g16_rx1.A.ubuntu-latest_gnu", + "./case.submit --skip-preview-namelist --no-batch --chksum", + "RUN", + from_dir="/tests/SEQ_Ln9.f19_g16_rx1.A.ubuntu-latest_gnu.00:00:00", + )