From 7760115c4f59db99ebca632dd47ea551c6adbae4 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Fri, 21 May 2021 22:33:03 +0000 Subject: [PATCH 1/5] Adds --chksum to case.submit and create_test --- scripts/Tools/case.submit | 10 +++- scripts/create_test | 19 ++++-- scripts/lib/CIME/case/case_submit.py | 15 +++-- scripts/lib/CIME/test_scheduler.py | 5 +- scripts/lib/CIME/tests/test_case.py | 60 ++++++++++++++++++- scripts/lib/CIME/tests/test_test_scheduler.py | 27 +++++++++ 6 files changed, 118 insertions(+), 18 deletions(-) create mode 100644 scripts/lib/CIME/tests/test_test_scheduler.py diff --git a/scripts/Tools/case.submit b/scripts/Tools/case.submit index 4ad64109559..49979d86237 100755 --- a/scripts/Tools/case.submit +++ b/scripts/Tools/case.submit @@ -74,6 +74,9 @@ def parse_command_line(args, description): parser.add_argument("-a", "--batch-args", help="Used to pass additional arguments to batch system.") + parser.add_argument("--chksum", action="store_true", + help="Verifies input data checksums.") + args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) CIME.utils.resolve_mail_type_args(args) @@ -90,13 +93,13 @@ def parse_command_line(args, description): return (args.caseroot, job, args.no_batch, args.prereq, args.prereq_allow_failure, args.resubmit, args.resubmit_immediate, args.skip_preview_namelist, args.mail_user, - args.mail_type, args.batch_args, workflow) + args.mail_type, args.batch_args, workflow, args.chksum) ############################################################################### def _main_func(description, test_args=False): ############################################################################### caseroot, job, no_batch, prereq, allow_fail, resubmit, resubmit_immediate, skip_pnl, \ - mail_user, mail_type, batch_args, workflow = parse_command_line(sys.argv, description) + mail_user, mail_type, batch_args, workflow, chksum = parse_command_line(sys.argv, description) # save these options to a hidden file for use during resubmit config_file = os.path.join(caseroot,".submit_options") @@ -120,7 +123,8 @@ def _main_func(description, test_args=False): with Case(caseroot, read_only=False, record=True) as case: case.submit(job=job, no_batch=no_batch, prereq=prereq, allow_fail=allow_fail, resubmit=resubmit, resubmit_immediate=resubmit_immediate, skip_pnl=skip_pnl, - mail_user=mail_user, mail_type=mail_type, batch_args=batch_args, workflow=workflow) + mail_user=mail_user, mail_type=mail_type, + batch_args=batch_args, workflow=workflow, chksum=chksum) if __name__ == "__main__": _main_func(__doc__) diff --git a/scripts/create_test b/scripts/create_test index db3cf968af9..dff85a1cc45 100755 --- a/scripts/create_test +++ b/scripts/create_test @@ -305,6 +305,9 @@ def parse_command_line(args, description): parser.add_argument("--workflow",default=workflow_default, help="A workflow from config_workflow.xml to apply to this case. ") + parser.add_argument("--chksum", action="store_true", + help="Verifies input data checksums.") + CIME.utils.add_mail_type_args(parser) args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) @@ -463,7 +466,12 @@ def parse_command_line(args, description): args.namelists_only, args.project, \ args.test_id, args.parallel_jobs, args.walltime, \ args.single_submit, args.proc_pool, args.use_existing, args.save_timing, args.queue, \ - args.allow_baseline_overwrite, args.output_root, args.wait, args.force_procs, args.force_threads, args.mpilib, args.input_dir, args.pesfile, args.retry, args.mail_user, args.mail_type, args.check_throughput, args.check_memory, args.ignore_namelists, args.ignore_memleak, args.allow_pnl, args.non_local, args.single_exe, args.workflow + args.allow_baseline_overwrite, args.output_root, args.wait, \ + args.force_procs, args.force_threads, args.mpilib, args.input_dir, args.pesfile, \ + args.retry, args.mail_user, args.mail_type, args.check_throughput, \ + args.check_memory, args.ignore_namelists, args.ignore_memleak, args.allow_pnl, \ + args.non_local, args.single_exe, args.workflow, args.chksum + ############################################################################### def get_default_setting(config, varname, default_if_not_found, check_main=False): @@ -572,7 +580,7 @@ def create_test(test_names, test_data, compiler, machine_name, no_run, no_build, walltime, single_submit, proc_pool, use_existing, save_timing, queue, allow_baseline_overwrite, output_root, wait, force_procs, force_threads, mpilib, input_dir, pesfile, mail_user, mail_type, check_throughput, check_memory, ignore_namelists, ignore_memleak, - allow_pnl, non_local, single_exe, workflow): + allow_pnl, non_local, single_exe, workflow, chksum): ############################################################################### impl = TestScheduler(test_names, test_data=test_data, no_run=no_run, no_build=no_build, no_setup=no_setup, no_batch=no_batch, @@ -586,7 +594,8 @@ def create_test(test_names, test_data, compiler, machine_name, no_run, no_build, queue=queue, allow_baseline_overwrite=allow_baseline_overwrite, output_root=output_root, force_procs=force_procs, force_threads=force_threads, mpilib=mpilib, input_dir=input_dir, pesfile=pesfile, mail_user=mail_user, mail_type=mail_type, allow_pnl=allow_pnl, - non_local=non_local, single_exe=single_exe, workflow=workflow) + non_local=non_local, single_exe=single_exe, + workflow=workflow, chksum=chksum) success = impl.run_tests(wait=wait, check_throughput=check_throughput, @@ -626,7 +635,7 @@ def _main_func(description): project, test_id, parallel_jobs, walltime, single_submit, proc_pool, use_existing, \ save_timing, queue, allow_baseline_overwrite, output_root, wait, force_procs, force_threads, mpilib, input_dir, pesfile, \ retry, mail_user, mail_type, check_throughput, check_memory, ignore_namelists, ignore_memleak, allow_pnl, \ - non_local, single_exe, workflow = \ + non_local, single_exe, workflow, chksum = \ parse_command_line(sys.argv, description) success = False @@ -639,7 +648,7 @@ def _main_func(description): project, test_id, parallel_jobs, walltime, single_submit, proc_pool, use_existing, save_timing, queue, allow_baseline_overwrite, output_root, wait, force_procs, force_threads, mpilib, input_dir, pesfile, mail_user, mail_type, check_throughput, check_memory, ignore_namelists, ignore_memleak, - allow_pnl, non_local, single_exe, workflow) + allow_pnl, non_local, single_exe, workflow, chksum) run_count += 1 # For testing only diff --git a/scripts/lib/CIME/case/case_submit.py b/scripts/lib/CIME/case/case_submit.py index ebf182b2f88..362654e790d 100644 --- a/scripts/lib/CIME/case/case_submit.py +++ b/scripts/lib/CIME/case/case_submit.py @@ -25,7 +25,7 @@ def _build_prereq_str(case, prev_job_ids): def _submit(case, job=None, no_batch=False, prereq=None, allow_fail=False, resubmit=False, resubmit_immediate=False, skip_pnl=False, mail_user=None, mail_type=None, - batch_args=None, workflow=True): + batch_args=None, workflow=True, chksum=False): if job is None: job = case.get_first_job() @@ -130,7 +130,7 @@ def _submit(case, job=None, no_batch=False, prereq=None, allow_fail=False, resub unlock_file(os.path.basename(env_batch.filename)) lock_file(os.path.basename(env_batch.filename)) - case.check_case(skip_pnl=skip_pnl) + case.check_case(skip_pnl=skip_pnl, chksum=chksum) if job == case.get_primary_job(): case.check_DA_settings() if case.get_value("MACH") == "mira": @@ -148,6 +148,8 @@ def _submit(case, job=None, no_batch=False, prereq=None, allow_fail=False, resub allow_fail=allow_fail, mail_user=mail_user, mail_type=mail_type, batch_args=batch_args, workflow=workflow) + print(job_ids) + xml_jobids = [] for jobname, jobid in job_ids.items(): logger.info("Submitted job {} with id {}".format(jobname, jobid)) @@ -162,7 +164,7 @@ def _submit(case, job=None, no_batch=False, prereq=None, allow_fail=False, resub def submit(self, job=None, no_batch=False, prereq=None, allow_fail=False, resubmit=False, resubmit_immediate=False, skip_pnl=False, mail_user=None, mail_type=None, - batch_args=None, workflow=True): + batch_args=None, workflow=True, chksum=False): if resubmit_immediate and self.get_value("MACH") in ['mira', 'cetus']: logger.warning("resubmit_immediate does not work on Mira/Cetus, submitting normally") resubmit_immediate = False @@ -202,7 +204,8 @@ def submit(self, job=None, no_batch=False, prereq=None, allow_fail=False, resubm allow_fail=allow_fail, resubmit=resubmit, resubmit_immediate=resubmit_immediate, skip_pnl=skip_pnl, mail_user=mail_user, mail_type=mail_type, - batch_args=batch_args, workflow=workflow) + batch_args=batch_args, workflow=workflow, + chksum=chksum) run_and_log_case_status(functor, "case.submit", caseroot=caseroot, custom_success_msg_functor=lambda x: x.split(":")[-1], is_batch=is_batch) @@ -215,12 +218,12 @@ def submit(self, job=None, no_batch=False, prereq=None, allow_fail=False, resubm raise -def check_case(self, skip_pnl=False): +def check_case(self, skip_pnl=False, chksum=False): self.check_lockedfiles() if not skip_pnl: self.create_namelists() # Must be called before check_all_input_data logger.info("Checking that inputdata is available as part of case submission") - self.check_all_input_data() + self.check_all_input_data(chksum=chksum) if self.get_value('COMP_WAV') == 'ww': # the ww3 buildnml has dependencies on inputdata so we must run it again diff --git a/scripts/lib/CIME/test_scheduler.py b/scripts/lib/CIME/test_scheduler.py index 14b046d6626..90d8e450c2e 100644 --- a/scripts/lib/CIME/test_scheduler.py +++ b/scripts/lib/CIME/test_scheduler.py @@ -123,7 +123,7 @@ def __init__(self, test_names, test_data=None, allow_baseline_overwrite=False, output_root=None, force_procs=None, force_threads=None, mpilib=None, input_dir=None, pesfile=None, mail_user=None, mail_type=None, allow_pnl=False, - non_local=False, single_exe=False, workflow=None): + non_local=False, single_exe=False, workflow=None, chksum=False): ########################################################################### self._cime_root = get_cime_root() self._cime_model = get_model() @@ -317,6 +317,7 @@ def __init__(self, test_names, test_data=None, for test_name in build_group: logger.debug("{}{}".format(" " if test_name == build_group[0] else " ", test_name)) + self._chksum = chksum # By the end of this constructor, this program should never hard abort, # instead, errors will be placed in the TestStatus files for the various # tests cases @@ -798,6 +799,8 @@ def _run_phase(self, test): cmd += " --mail-user={}".format(self._mail_user) if self._mail_type: cmd += " -M={}".format(",".join(self._mail_type)) + if self._chksum: + cmd += " --chksum" return self._shell_cmd_for_phase(test, cmd, RUN_PHASE, from_dir=test_dir) diff --git a/scripts/lib/CIME/tests/test_case.py b/scripts/lib/CIME/tests/test_case.py index 8823bb669b2..4661ee7d24c 100644 --- a/scripts/lib/CIME/tests/test_case.py +++ b/scripts/lib/CIME/tests/test_case.py @@ -1,12 +1,66 @@ import os import sys import unittest +from unittest import mock +import tempfile +from CIME.case import case_submit from CIME.case import Case from CIME import utils as cime_utils from . import utils +class TestCaseSubmit(unittest.TestCase): + + def test_check_case(self): + case = mock.MagicMock() + + case_submit.check_case(case, chksum=True) + + case.check_all_input_data.assert_called_with(chksum=True) + + @mock.patch("CIME.case.case_submit.lock_file") + def test__submit(self, lock_file): # pylint: disable=unused-argument + case = mock.MagicMock() + + case_submit._submit(case, chksum=True) # pylint: disable=protected-access + + case.check_case.assert_called_with(skip_pnl=False, chksum=True) + + @mock.patch("CIME.case.case_submit._submit") + @mock.patch("CIME.case.case.Case.initialize_derived_attributes") + @mock.patch("CIME.case.case.Case.get_value") + @mock.patch("CIME.case.case.Case.read_xml") + def test_submit(self, read_xml, get_value, init, _submit): # pylint: disable=unused-argument + with tempfile.TemporaryDirectory() as tempdir: + get_value.side_effect = [ + tempdir, + tempdir, + True, + "baseid", + None, + True, + ] + + with Case(tempdir) as case: + case.submit(chksum=True) + + _submit.assert_called_with( + case, + job=None, + no_batch=False, + prereq=None, + allow_fail=False, + resubmit=False, + resubmit_immediate=False, + skip_pnl=False, + mail_user=None, + mail_type=None, + batch_args=None, + workflow=True, + chksum=True + ) + class TestCase(unittest.TestCase): def setUp(self): @@ -156,7 +210,7 @@ def assert_calls_match(self, calls, expected): def test_init(self): with self.tempdir as tempdir: - mock = utils.Mocker() + mock = utils.Mocker() # pylint: disable=redefined-outer-name open_mock = mock.patch( "builtins.open" if sys.version_info.major > 2 else "__builtin__.open", @@ -189,7 +243,7 @@ def test_init(self): def test_sub_relative(self): with self.tempdir as tempdir: - mock = utils.Mocker() + mock = utils.Mocker() # pylint: disable=redefined-outer-name open_mock = mock.patch( "builtins.open" if sys.version_info.major > 2 else "__builtin__.open", @@ -220,7 +274,7 @@ def test_sub_relative(self): def test_cmd_arg(self): with self.tempdir as tempdir: - mock = utils.Mocker() + mock = utils.Mocker() # pylint: disable=redefined-outer-name open_mock = mock.patch( "builtins.open" if sys.version_info.major > 2 else "__builtin__.open", diff --git a/scripts/lib/CIME/tests/test_test_scheduler.py b/scripts/lib/CIME/tests/test_test_scheduler.py new file mode 100644 index 00000000000..31e488cc111 --- /dev/null +++ b/scripts/lib/CIME/tests/test_test_scheduler.py @@ -0,0 +1,27 @@ +import os +import unittest +from unittest import mock + +from CIME.test_scheduler import TestScheduler + +class TestTestScheduler(unittest.TestCase): + + @mock.patch.dict(os.environ, {"CIME_MODEL": "cesm"}) + @mock.patch("time.strftime", return_value="00:00:00") + def test_chksum(self, strftime): + ts = TestScheduler( + ["SEQ_Ln9.f19_g16_rx1.A.ubuntu-latest_gnu"], + machine_name="ubuntu-latest", + chksum=True, + test_root="/tests", + ) + + with mock.patch.object(ts, "_shell_cmd_for_phase") as _shell_cmd_for_phase: + ts._run_phase("SEQ_Ln9.f19_g16_rx1.A.ubuntu-latest_gnu") + + _shell_cmd_for_phase.assert_called_with( + "SEQ_Ln9.f19_g16_rx1.A.ubuntu-latest_gnu", + "./case.submit --skip-preview-namelist --no-batch --chksum", + "RUN", + from_dir="/tests/SEQ_Ln9.f19_g16_rx1.A.ubuntu-latest_gnu.00:00:00", + ) From 4b0a1e234edc2f7ba18107099452cf504e67bf38 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Fri, 21 May 2021 22:39:56 +0000 Subject: [PATCH 2/5] Remove print --- scripts/lib/CIME/case/case_submit.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/lib/CIME/case/case_submit.py b/scripts/lib/CIME/case/case_submit.py index 362654e790d..54fd4054b2d 100644 --- a/scripts/lib/CIME/case/case_submit.py +++ b/scripts/lib/CIME/case/case_submit.py @@ -148,8 +148,6 @@ def _submit(case, job=None, no_batch=False, prereq=None, allow_fail=False, resub allow_fail=allow_fail, mail_user=mail_user, mail_type=mail_type, batch_args=batch_args, workflow=workflow) - print(job_ids) - xml_jobids = [] for jobname, jobid in job_ids.items(): logger.info("Submitted job {} with id {}".format(jobname, jobid)) From ca6367fffaef08fbfc95e23f3b2cd205f912369f Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Fri, 21 May 2021 23:10:58 +0000 Subject: [PATCH 3/5] Fixes linting --- scripts/lib/CIME/tests/test_test_scheduler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/lib/CIME/tests/test_test_scheduler.py b/scripts/lib/CIME/tests/test_test_scheduler.py index 31e488cc111..b907151e6b9 100644 --- a/scripts/lib/CIME/tests/test_test_scheduler.py +++ b/scripts/lib/CIME/tests/test_test_scheduler.py @@ -8,7 +8,7 @@ class TestTestScheduler(unittest.TestCase): @mock.patch.dict(os.environ, {"CIME_MODEL": "cesm"}) @mock.patch("time.strftime", return_value="00:00:00") - def test_chksum(self, strftime): + def test_chksum(self, strftime): # pylint: disable=unused-argument ts = TestScheduler( ["SEQ_Ln9.f19_g16_rx1.A.ubuntu-latest_gnu"], machine_name="ubuntu-latest", @@ -17,7 +17,7 @@ def test_chksum(self, strftime): ) with mock.patch.object(ts, "_shell_cmd_for_phase") as _shell_cmd_for_phase: - ts._run_phase("SEQ_Ln9.f19_g16_rx1.A.ubuntu-latest_gnu") + ts._run_phase("SEQ_Ln9.f19_g16_rx1.A.ubuntu-latest_gnu") # pylint: disable=protected-access _shell_cmd_for_phase.assert_called_with( "SEQ_Ln9.f19_g16_rx1.A.ubuntu-latest_gnu", From d08f23c5fb4da7d0b7e850988fc2ba6887ffe392 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Wed, 26 May 2021 17:23:33 +0000 Subject: [PATCH 4/5] Fixes linting --- scripts/lib/CIME/tests/test_case.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/lib/CIME/tests/test_case.py b/scripts/lib/CIME/tests/test_case.py index 5248c234b90..df4ce93bd82 100644 --- a/scripts/lib/CIME/tests/test_case.py +++ b/scripts/lib/CIME/tests/test_case.py @@ -7,8 +7,6 @@ from CIME.case import Case from CIME import utils as cime_utils -from . import utils - class TestCaseSubmit(unittest.TestCase): def test_check_case(self): From 9af1d6eb1a41c0e0c06f4ab19996a0efbff65c7e Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Wed, 2 Jun 2021 16:30:18 +0000 Subject: [PATCH 5/5] Removes whitespace --- scripts/lib/CIME/tests/test_case.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/lib/CIME/tests/test_case.py b/scripts/lib/CIME/tests/test_case.py index df4ce93bd82..88e7821443d 100644 --- a/scripts/lib/CIME/tests/test_case.py +++ b/scripts/lib/CIME/tests/test_case.py @@ -255,7 +255,7 @@ def test_init(self, strftime, get_value, flush, init): # pylint: disable=unused- @mock.patch("sys.argv", ["/src/scripts/create_newcase"]) def test_sub_relative(self, strftime, get_value, flush, init): # pylint: disable=unused-argument Case._force_read_only = False # pylint: disable=protected-access - + mocked_open = mock.mock_open() with self.tempdir as tempdir, mock.patch("CIME.case.case.open",