From e9980c16d422f89a959aaa165dfd20f0773a56e5 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Mon, 29 Aug 2022 16:38:39 -0600 Subject: [PATCH 01/56] add auto-rt to my forked repository --- tests/auto/jobs/bl.py | 41 ++++++++--------- tests/auto/jobs/rt.py | 55 ++++++++++------------ tests/auto/rt_auto.py | 92 ++++++++++++++++++++++++++++++++++--- tests/auto/start_rt_auto.sh | 3 ++ 4 files changed, 132 insertions(+), 59 deletions(-) diff --git a/tests/auto/jobs/bl.py b/tests/auto/jobs/bl.py index 1b758211a4..20c627ecd6 100644 --- a/tests/auto/jobs/bl.py +++ b/tests/auto/jobs/bl.py @@ -13,7 +13,7 @@ def run(job_obj): bldir = f'{blstore}/develop-{bldate}/{job_obj.compiler.upper()}' bldirbool = check_for_bl_dir(bldir, job_obj) run_regression_test(job_obj, pr_repo_loc) - post_process(job_obj, pr_repo_loc, repo_dir_str, rtbldir, bldir) + post_process(job_obj, pr_repo_loc, repo_dir_str, rtbldir, bldir, bldate, blstore) def set_directories(job_obj): @@ -39,9 +39,9 @@ def set_directories(job_obj): rtbldir = '/work/noaa/stmp/bcurtis/stmp/bcurtis/FV3_RT/'\ f'REGRESSION_TEST_{job_obj.compiler.upper()}' elif job_obj.machine == 'cheyenne': - workdir = '/glade/scratch/dtcufsrt/autort/tests/auto/pr' + workdir = '/glade/scratch/dswales/UFS/autort/tests/auto/pr' blstore = '/glade/p/ral/jntp/GMTB/ufs-weather-model/RT/NEMSfv3gfs' - rtbldir = '/glade/scratch/dtcufsrt/FV3_RT/'\ + rtbldir = '/glade/scratch/dswales/UFS/dswales/FV3_RT/'\ f'REGRESSION_TEST_{job_obj.compiler.upper()}' else: logger.critical(f'Machine {job_obj.machine} is not supported for this job') @@ -60,8 +60,8 @@ def check_for_bl_dir(bldir, job_obj): logger.info('Checking if baseline directory exists') if os.path.exists(bldir): logger.critical(f'Baseline dir: {bldir} exists. It should not, yet.') - job_obj.comment_text_append(f'{bldir}\n Exists already. ' - 'It should not yet. Please delete.') + job_obj.comment_text_append(f'[BL] ERROR: Baseline location exists before ' + f'creation:\n{bldir}') raise FileExistsError return False @@ -124,23 +124,25 @@ def clone_pr_repo(job_obj, workdir): logger = logging.getLogger('BL/CLONE_PR_REPO') repo_name = job_obj.preq_dict['preq'].head.repo.name branch = job_obj.preq_dict['preq'].head.ref - git_url = job_obj.preq_dict['preq'].head.repo.html_url.split('//') - git_url = f'{git_url[0]}//${{ghapitoken}}@{git_url[1]}' - logger.debug(f'GIT URL: {git_url}') + #git_url = job_obj.preq_dict['preq'].head.repo.html_url.split('//') + #git_url = f'{git_url[0]}//${{ghapitoken}}@{git_url[1]}' + #logger.debug(f'GIT URL: {git_url}') + git_ssh_url = job_obj.preq_dict['preq'].head.repo.ssh_url + logger.debug(f'GIT SSH_URL: {git_ssh_url}') logger.info('Starting repo clone') repo_dir_str = f'{workdir}/'\ f'{str(job_obj.preq_dict["preq"].id)}/'\ f'{datetime.datetime.now().strftime("%Y%m%d%H%M%S")}' pr_repo_loc = f'{repo_dir_str}/{repo_name}' - job_obj.comment_text_append(f'Repo location: {pr_repo_loc}') + job_obj.comment_text_append(f'[BL] Repo location: {pr_repo_loc}') create_repo_commands = [ [f'mkdir -p "{repo_dir_str}"', os.getcwd()], - [f'git clone -b {branch} {git_url}', repo_dir_str], + [f'git clone -b {branch} {git_ssh_url}', repo_dir_str], ['git submodule update --init --recursive', f'{repo_dir_str}/{repo_name}'], - ['git config user.email "brian.curtis@noaa.gov"', + ['git config user.email "dustin.swales@noaa.gov"', f'{repo_dir_str}/{repo_name}'], - ['git config user.name "Brian Curtis"', + ['git config user.name "Dustin Swales"', f'{repo_dir_str}/{repo_name}'] ] @@ -150,7 +152,7 @@ def clone_pr_repo(job_obj, workdir): return pr_repo_loc, repo_dir_str -def post_process(job_obj, pr_repo_loc, repo_dir_str, rtbldir, bldir): +def post_process(job_obj, pr_repo_loc, repo_dir_str, rtbldir, bldir, bldate, blstore): logger = logging.getLogger('BL/MOVE_RT_LOGS') rt_log = f'tests/RegressionTests_{job_obj.machine}'\ f'.{job_obj.compiler}.log' @@ -159,14 +161,11 @@ def post_process(job_obj, pr_repo_loc, repo_dir_str, rtbldir, bldir): if logfile_pass: create_bl_dir(bldir, job_obj) move_bl_command = [[f'mv {rtbldir}/* {bldir}/', pr_repo_loc]] - if job_obj.machine == 'orion': - move_bl_command.append([f'/bin/bash --login adjust_permissions.sh orion develop-{bldate}', blstore]) job_obj.run_commands(logger, move_bl_command) - job_obj.comment_text_append('Baseline creation and move successful') + job_obj.comment_text_append('[BL] Baseline creation and move successful') logger.info('Starting RT Job') rt.run(job_obj) logger.info('Finished with RT Job') - remove_pr_data(job_obj, pr_repo_loc, repo_dir_str, rt_dir) def get_bl_date(job_obj, pr_repo_loc): @@ -190,9 +189,7 @@ def get_bl_date(job_obj, pr_repo_loc): logger.info(f'Date {bldate} is not formatted YYYYMMDD') raise ValueError if not BLDATEFOUND: - job_obj.comment_text_append('BL_DATE not found in rt.sh.' - 'Please manually edit rt.sh ' - 'with BL_DATE={bldate}') + job_obj.comment_text_append('[BL] ERROR: Variable "BL_DATE" not found in rt.sh.') job_obj.job_failed(logger, 'get_bl_date()') logger.info('Finished get_bl_date') @@ -208,13 +205,11 @@ def process_logfile(job_obj, logfile): for line in f: if all(x in line for x in fail_string_list): # if 'FAIL' in line and 'Test' in line: - job_obj.comment_text_append(f'{line.rstrip(chr(10))}') + job_obj.comment_text_append(f'[BL] Error: {line.rstrip(chr(10))}') elif 'working dir' in line and not rt_dir: logger.info(f'Found "working dir" in line: {line}') rt_dir = os.path.split(line.split()[-1])[0] logger.info(f'It is: {rt_dir}') - job_obj.comment_text_append(f'Please manually delete: ' - f'{rt_dir}') elif 'SUCCESSFUL' in line: logger.info('RT Successful') return rt_dir, True diff --git a/tests/auto/jobs/rt.py b/tests/auto/jobs/rt.py index c69bcda669..ef57eb6355 100644 --- a/tests/auto/jobs/rt.py +++ b/tests/auto/jobs/rt.py @@ -23,7 +23,7 @@ def set_directories(job_obj): elif job_obj.machine == 'orion': workdir = '/work/noaa/nems/emc.nemspara/autort/pr' elif job_obj.machine == 'cheyenne': - workdir = '/glade/scratch/dtcufsrt/autort/tests/auto/pr' + workdir = '/glade/scratch/dswales/UFS/autort/tests/auto/pr' else: print(f'Machine {job_obj.machine} is not supported for this job') raise KeyError @@ -60,23 +60,23 @@ def clone_pr_repo(job_obj, workdir): logger = logging.getLogger('RT/CLONE_PR_REPO') repo_name = job_obj.preq_dict['preq'].head.repo.name branch = job_obj.preq_dict['preq'].head.ref - git_url = job_obj.preq_dict['preq'].head.repo.html_url.split('//') - git_url = f'{git_url[0]}//${{ghapitoken}}@{git_url[1]}' - logger.debug(f'GIT URL: {git_url}') + #git_url = job_obj.preq_dict['preq'].head.repo.html_url.split('//') + git_ssh_url = job_obj.preq_dict['preq'].head.repo.ssh_url + logger.debug(f'GIT SSH_URL: {git_ssh_url}') logger.info('Starting repo clone') repo_dir_str = f'{workdir}/'\ f'{str(job_obj.preq_dict["preq"].id)}/'\ f'{datetime.datetime.now().strftime("%Y%m%d%H%M%S")}' pr_repo_loc = f'{repo_dir_str}/{repo_name}' - job_obj.comment_text_append(f'Repo location: {pr_repo_loc}') + job_obj.comment_text_append(f'[RT] Repo location: {pr_repo_loc}') create_repo_commands = [ [f'mkdir -p "{repo_dir_str}"', os.getcwd()], - [f'git clone -b {branch} {git_url}', repo_dir_str], + [f'git clone -b {branch} {git_ssh_url}', repo_dir_str], ['git submodule update --init --recursive', f'{repo_dir_str}/{repo_name}'], - ['git config user.email "brian.curtis@noaa.gov"', + ['git config user.email "dustin.swales@noaa.gov"', f'{repo_dir_str}/{repo_name}'], - ['git config user.name "Brian Curtis"', + ['git config user.name "Dustin Swales"', f'{repo_dir_str}/{repo_name}'] ] @@ -94,25 +94,22 @@ def post_process(job_obj, pr_repo_loc, repo_dir_str, branch): filepath = f'{pr_repo_loc}/{rt_log}' rt_dir, logfile_pass = process_logfile(job_obj, filepath) if logfile_pass: - if job_obj.preq_dict['preq'].maintainer_can_modify: - move_rt_commands = [ - [f'git pull --ff-only origin {branch}', pr_repo_loc], - [f'git add {rt_log}', pr_repo_loc], - [f'git commit -m "RT JOBS PASSED: {job_obj.machine}' - f'.{job_obj.compiler}. Log file uploaded.\n\n' - 'on-behalf-of @ufs-community"', - pr_repo_loc], - ['sleep 10', pr_repo_loc], - [f'git push origin {branch}', pr_repo_loc] - ] - job_obj.run_commands(logger, move_rt_commands) - remove_pr_data(job_obj, pr_repo_loc, repo_dir_str, rt_dir) - else: - job_obj.comment_text_append(f'Cannot upload {job_obj.machine}.'\ - f'{job_obj.compiler} RT Log'\ - 'It is blocked by PR owner') - job_obj.comment_text_append(f'Please obtain logs from {pr_repo_loc}') - job_obj.preq_dict['preq'].create_issue_comment(job_obj.comment_text) + #if job_obj.preq_dict['preq'].maintainer_can_modify: + move_rt_commands = [ + [f'git pull --ff-only origin {branch}', pr_repo_loc], + [f'git add {rt_log}', pr_repo_loc], + [f'git commit -m "[AutoRT] {job_obj.machine}' + f'.{job_obj.compiler} Job Completed.\n\n\n' + 'on-behalf-of @ufs-community "', + pr_repo_loc], + ['sleep 10', pr_repo_loc], + [f'git push origin {branch}', pr_repo_loc] + ] + job_obj.run_commands(logger, move_rt_commands) + else: + job_obj.comment_text_append(f'[RT] Log file shows failures.') + job_obj.comment_text_append(f'[RT] Please obtain logs from {pr_repo_loc}') + job_obj.preq_dict['preq'].create_issue_comment(job_obj.comment_text) def process_logfile(job_obj, logfile): @@ -124,11 +121,9 @@ def process_logfile(job_obj, logfile): for line in f: if all(x in line for x in fail_string_list): # if 'FAIL' in line and 'Test' in line: - job_obj.comment_text_append(f'{line.rstrip(chr(10))}') + job_obj.comment_text_append(f'[RT] Error: {line.rstrip(chr(10))}') elif 'working dir' in line and not rt_dir: rt_dir = os.path.split(line.split()[-1])[0] - job_obj.comment_text_append(f'Please manually delete: ' - f'{rt_dir}') elif 'SUCCESSFUL' in line: return rt_dir, True job_obj.job_failed(logger, f'{job_obj.preq_dict["action"]}') diff --git a/tests/auto/rt_auto.py b/tests/auto/rt_auto.py index bed530b3d0..e4c9229d05 100644 --- a/tests/auto/rt_auto.py +++ b/tests/auto/rt_auto.py @@ -11,9 +11,10 @@ import subprocess import re import os +from glob import glob import logging import importlib - +from shutil import rmtree class GHInterface: ''' @@ -74,6 +75,79 @@ def set_action_from_label(machine, actions, label): logging.info(f'Compiler: {label_compiler}, Action: {action_match}') return label_compiler, action_match +def delete_pr_dirs(each_pr, machine): + if machine == 'hera': + workdir = '/scratch1/NCEPDEV/nems/emc.nemspara/autort/pr' + elif machine == 'jet': + workdir = '/lfs4/HFIP/h-nems/emc.nemspara/autort/pr' + elif machine == 'gaea': + workdir = '/lustre/f2/pdata/ncep/emc.nemspara/autort/pr' + elif machine == 'orion': + workdir = '/work/noaa/nems/emc.nemspara/autort/pr' + elif machine == 'cheyenne': + workdir = '/glade/scratch/dswales/UFS/autort/tests/auto/pr' + else: + logging.error(f'Machine {machine} is not supported for this job') + raise KeyError + ids = [str(pr.id) for pr in each_pr] + logging.debug(f'ids are: {ids}') + dirs = [x.split('/')[-2] for x in glob(f'{workdir}/*/')] + logging.debug(f'dirs: {dirs}') + for dir in dirs: + if dir != 'pr': + logging.debug(f'Checking dir {dir}') + if not dir in ids: + logging.debug(f'ID NOT A MATCH, DELETING {dir}') + delete_rt_dirs(dir, machine, workdir) + if os.path.isdir(f'{workdir}/{dir}'): + logging.debug(f'Executing rmtree in "{workdir}/{dir}"') + rmtree(f'{workdir}/{dir}') + else: + logging.debug(f'{workdir}/{dir} does not exist, not attempting to remove') + else: + logging.debug(f'ID A MATCH, NOT DELETING {dir}') + # job_obj.preq_dict["preq"].id + + +def delete_rt_dirs(in_dir, machine, workdir): + if machine == 'hera': + rt_dir ='/scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT' + elif machine == 'jet': + rt_dir ='/lfs4/HFIP/h-nems/emc.nemspara/RT_BASELINE/'\ + f'emc.nemspara/FV3_RT' + elif machine == 'gaea': + rt_dir = '/lustre/f2/scratch/emc.nemspara/FV3_RT' + elif machine == 'orion': + rt_dir = '/work/noaa/stmp/bcurtis/stmp/bcurtis/FV3_RT' + elif machine == 'cheyenne': + rt_dir = '/glade/scratch/dswales/UFS/dswales/FV3_RT/' + else: + logging.error(f'Machine {machine} is not supported for this job') + raise KeyError + globdir = f'{workdir}/{in_dir}/**/compile_*.log' + logging.debug(f'globdir: {globdir}') + logfiles = glob(globdir, recursive=True) + if not logfiles: + return + logging.debug(f'logfiles: {logfiles}') + matches = [] + for logfile in logfiles: + with open(logfile, "r") as fp: + lines = [line.split('/') for line in fp if 'rt_' in line] + lines = list(set([item for sublist in lines for item in sublist])) + lines = [s for s in lines if 'rt_' in s and '\n' not in s] + if lines: + matches.append(lines) + logging.debug(f'lines: {lines}') + matches = list(set([item for sublist in matches for item in sublist])) + logging.debug(f'matches: {matches}') + for match in matches: + if os.path.isdir(f'{rt_dir}/{match}'): + logging.debug(f'Executing rmtree in "{rt_dir}/{match}"') + rmtree(f'{rt_dir}/{match}') + else: + logging.debug(f'{rt_dir}/{match} does not exist, not attempting to remove') + def get_preqs_with_actions(repos, machine, ghinterface_obj, actions): ''' Create list of dictionaries of a pull request @@ -84,6 +158,7 @@ def get_preqs_with_actions(repos, machine, ghinterface_obj, actions): .get_pulls(state='open', sort='created', base=repo['base']) for repo in repos] each_pr = [preq for gh_preq in gh_preqs for preq in gh_preq] + delete_pr_dirs(each_pr, machine) preq_labels = [{'preq': pr, 'label': label} for pr in each_pr for label in pr.get_labels()] @@ -125,7 +200,7 @@ def __init__(self, preq_dict, ghinterface_obj, machine, compiler): self.ghinterface_obj = ghinterface_obj self.machine = machine self.compiler = compiler - self.comment_text = '' + self.comment_text = '***Automated RT Failure Notification***\n' self.failed_tests = [] def comment_text_append(self, newtext): @@ -193,8 +268,8 @@ def send_comment_text(self): logger = logging.getLogger('JOB/SEND_COMMENT_TEXT') logger.info(f'Comment Text: {self.comment_text}') self.comment_text_append('Please make changes and add ' - 'the following label back:') - self.comment_text_append(f'{self.machine}' + 'the following label back: ' + f'{self.machine}' f'-{self.compiler}' f'-{self.preq_dict["action"]}') @@ -217,6 +292,9 @@ def setup_env(): elif bool(re.match(re.compile('fe.+'), hostname)): machine = 'jet' os.environ['ACCNR'] = 'h-nems' + elif bool(re.match(re.compile('tfe.+'), hostname)): + machine = 'jet' + os.environ['ACCNR'] = 'h-nems' elif bool(re.match(re.compile('gaea.+'), hostname)): machine = 'gaea' os.environ['ACCNR'] = 'nggps_emc' @@ -232,7 +310,7 @@ def setup_env(): # Dictionary of GitHub repositories to check repo_dict = [{ 'name': 'ufs-weather-model', - 'address': 'ufs-community/ufs-weather-model', + 'address': 'dustinswales/ufs-weather-model', 'base': 'develop' }] @@ -247,8 +325,10 @@ def main(): # handle logging log_filename = f'rt_auto_'\ f'{datetime.datetime.now().strftime("%Y%m%d%H%M%S")}.log' + #logging.basicConfig(filename=log_filename, filemode='w', + # level=logging.INFO) logging.basicConfig(filename=log_filename, filemode='w', - level=logging.INFO) + level=logging.DEBUG) logger = logging.getLogger('MAIN') logger.info('Starting Script') diff --git a/tests/auto/start_rt_auto.sh b/tests/auto/start_rt_auto.sh index af85cdccbb..24c17e7389 100644 --- a/tests/auto/start_rt_auto.sh +++ b/tests/auto/start_rt_auto.sh @@ -13,6 +13,9 @@ elif [[ $HOSTNAME == Orion-login-* ]]; then elif [[ $HOSTNAME == fe* ]]; then export PATH=/lfs4/HFIP/hfv3gfs/software/miniconda3/4.8.3/envs/ufs-weather-model/bin:/lfs4/HFIP/hfv3gfs/software/miniconda3/4.8.3/bin:$PATH export PYTHONPATH=/lfs4/HFIP/hfv3gfs/software/miniconda3/4.8.3/envs/ufs-weather-model/lib/python3.8/site-packages:/lfs4/HFIP/hfv3gfs/software/miniconda3/4.8.3/lib/python3.8/site-packages +elif [[ $HOSTNAME == tfe* ]]; then + export PATH=/lfs4/HFIP/hfv3gfs/software/miniconda3/4.8.3/envs/ufs-weather-model/bin:/lfs4/HFIP/hfv3gfs/software/miniconda3/4.8.3/bin:$PATH + export PYTHONPATH=/lfs4/HFIP/hfv3gfs/software/miniconda3/4.8.3/envs/ufs-weather-model/lib/python3.8/site-packages:/lfs4/HFIP/hfv3gfs/software/miniconda3/4.8.3/lib/python3.8/site-packages elif [[ $HOSTNAME == gaea* ]]; then export PATH=/lustre/f2/pdata/esrl/gsd/contrib/miniconda3/4.8.3/envs/ufs-weather-model/bin:$PATH export PYTHONPATH=/lustre/f2/pdata/esrl/gsd/contrib/miniconda3/4.8.3/lib/python3.8/site-packages From be6f5a6ef3e10464bf82f5cff1155ba66e681d3b Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 23 Sep 2022 01:46:40 +0000 Subject: [PATCH 02/56] Initial commit --- tests/auto/jobs/bl.py | 6 +++--- tests/auto/jobs/rt.py | 8 ++++---- tests/auto/rt_auto.py | 8 ++++---- tests/auto/start_rt_auto.sh | 0 4 files changed, 11 insertions(+), 11 deletions(-) mode change 100644 => 100755 tests/auto/start_rt_auto.sh diff --git a/tests/auto/jobs/bl.py b/tests/auto/jobs/bl.py index 20c627ecd6..df92839eeb 100644 --- a/tests/auto/jobs/bl.py +++ b/tests/auto/jobs/bl.py @@ -19,7 +19,7 @@ def run(job_obj): def set_directories(job_obj): logger = logging.getLogger('BL/SET_DIRECTORIES') if job_obj.machine == 'hera': - workdir = '/scratch1/NCEPDEV/nems/emc.nemspara/autort/pr' + workdir = '/scratch1/BMC/gmtb/Dustin.Swales/UFS/auto-RT/Pull_Requests' blstore = '/scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs' rtbldir = '/scratch1/NCEPDEV/stmp4/emc.nemspara/FV3_RT/'\ f'REGRESSION_TEST_{job_obj.compiler.upper()}' @@ -140,9 +140,9 @@ def clone_pr_repo(job_obj, workdir): [f'git clone -b {branch} {git_ssh_url}', repo_dir_str], ['git submodule update --init --recursive', f'{repo_dir_str}/{repo_name}'], - ['git config user.email "dustin.swales@noaa.gov"', + ['git config user.email "dswales@ucar.edu"', f'{repo_dir_str}/{repo_name}'], - ['git config user.name "Dustin Swales"', + ['git config user.name "dustinswales"', f'{repo_dir_str}/{repo_name}'] ] diff --git a/tests/auto/jobs/rt.py b/tests/auto/jobs/rt.py index ef57eb6355..0e9ad28b32 100644 --- a/tests/auto/jobs/rt.py +++ b/tests/auto/jobs/rt.py @@ -15,7 +15,7 @@ def run(job_obj): def set_directories(job_obj): logger = logging.getLogger('RT/SET_DIRECTORIES') if job_obj.machine == 'hera': - workdir = '/scratch1/NCEPDEV/nems/emc.nemspara/autort/pr' + workdir = '/scratch1/BMC/gmtb/Dustin.Swales/UFS/auto-RT/Pull_Requests' elif job_obj.machine == 'jet': workdir = '/lfs4/HFIP/h-nems/emc.nemspara/autort/pr' elif job_obj.machine == 'gaea': @@ -74,9 +74,9 @@ def clone_pr_repo(job_obj, workdir): [f'git clone -b {branch} {git_ssh_url}', repo_dir_str], ['git submodule update --init --recursive', f'{repo_dir_str}/{repo_name}'], - ['git config user.email "dustin.swales@noaa.gov"', + ['git config user.email "dswales@ucar.edu"', f'{repo_dir_str}/{repo_name}'], - ['git config user.name "Dustin Swales"', + ['git config user.name "dustinswales"', f'{repo_dir_str}/{repo_name}'] ] @@ -100,7 +100,7 @@ def post_process(job_obj, pr_repo_loc, repo_dir_str, branch): [f'git add {rt_log}', pr_repo_loc], [f'git commit -m "[AutoRT] {job_obj.machine}' f'.{job_obj.compiler} Job Completed.\n\n\n' - 'on-behalf-of @ufs-community "', + 'on-behalf-of @ufs-community "', pr_repo_loc], ['sleep 10', pr_repo_loc], [f'git push origin {branch}', pr_repo_loc] diff --git a/tests/auto/rt_auto.py b/tests/auto/rt_auto.py index e4c9229d05..6c2471f4a1 100644 --- a/tests/auto/rt_auto.py +++ b/tests/auto/rt_auto.py @@ -76,8 +76,8 @@ def set_action_from_label(machine, actions, label): return label_compiler, action_match def delete_pr_dirs(each_pr, machine): - if machine == 'hera': - workdir = '/scratch1/NCEPDEV/nems/emc.nemspara/autort/pr' + if machine == 'hera': + workdir = '/scratch1/BMC/gmtb/Dustin.Swales/UFS/auto-RT/Pull_Requests' elif machine == 'jet': workdir = '/lfs4/HFIP/h-nems/emc.nemspara/autort/pr' elif machine == 'gaea': @@ -110,8 +110,8 @@ def delete_pr_dirs(each_pr, machine): def delete_rt_dirs(in_dir, machine, workdir): - if machine == 'hera': - rt_dir ='/scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT' + if machine == 'hera': + rt_dir = '/scratch1/BMC/gmtb/Dustin.Swales/UFS/stmp2/Dustin.Swales/FV3_RT' elif machine == 'jet': rt_dir ='/lfs4/HFIP/h-nems/emc.nemspara/RT_BASELINE/'\ f'emc.nemspara/FV3_RT' diff --git a/tests/auto/start_rt_auto.sh b/tests/auto/start_rt_auto.sh old mode 100644 new mode 100755 From 4e55b1c177680661e2bcbc9096e7b50e2aeb222e Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Tue, 4 Oct 2022 05:54:21 +0000 Subject: [PATCH 03/56] Auto-RTs working --- tests/auto/jobs/bl.py | 39 ++++++++++++--------------------------- tests/auto/jobs/rt.py | 16 +++++----------- tests/auto/rt_auto.py | 4 ++-- 3 files changed, 19 insertions(+), 40 deletions(-) diff --git a/tests/auto/jobs/bl.py b/tests/auto/jobs/bl.py index df92839eeb..9d196beab8 100644 --- a/tests/auto/jobs/bl.py +++ b/tests/auto/jobs/bl.py @@ -10,7 +10,7 @@ def run(job_obj): workdir, rtbldir, blstore = set_directories(job_obj) pr_repo_loc, repo_dir_str = clone_pr_repo(job_obj, workdir) bldate = get_bl_date(job_obj, pr_repo_loc) - bldir = f'{blstore}/develop-{bldate}/{job_obj.compiler.upper()}' + bldir = f'{blstore}/main-{bldate}/{job_obj.compiler.upper()}' bldirbool = check_for_bl_dir(bldir, job_obj) run_regression_test(job_obj, pr_repo_loc) post_process(job_obj, pr_repo_loc, repo_dir_str, rtbldir, bldir, bldate, blstore) @@ -19,29 +19,14 @@ def run(job_obj): def set_directories(job_obj): logger = logging.getLogger('BL/SET_DIRECTORIES') if job_obj.machine == 'hera': - workdir = '/scratch1/BMC/gmtb/Dustin.Swales/UFS/auto-RT/Pull_Requests' - blstore = '/scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs' - rtbldir = '/scratch1/NCEPDEV/stmp4/emc.nemspara/FV3_RT/'\ - f'REGRESSION_TEST_{job_obj.compiler.upper()}' - elif job_obj.machine == 'jet': - workdir = '/lfs4/HFIP/h-nems/emc.nemspara/autort/pr' - blstore = '/lfs4/HFIP/h-nems/emc.nemspara/RT/NEMSfv3gfs/' - rtbldir = '/lfs4/HFIP/h-nems/emc.nemspara/RT_BASELINE/'\ - f'emc.nemspara/FV3_RT/REGRESSION_TEST_{job_obj.compiler.upper()}' - elif job_obj.machine == 'gaea': - workdir = '/lustre/f2/pdata/ncep/emc.nemspara/autort/pr' - blstore = '/lustre/f2/pdata/ncep_shared/emc.nemspara/RT/NEMSfv3gfs' - rtbldir = '/lustre/f2/scratch/emc.nemspara/FV3_RT/'\ - f'REGRESSION_TEST_{job_obj.compiler.upper()}' - elif job_obj.machine == 'orion': - workdir = '/work/noaa/nems/emc.nemspara/autort/pr' - blstore = '/work/noaa/nems/emc.nemspara/RT/NEMSfv3gfs' - rtbldir = '/work/noaa/stmp/bcurtis/stmp/bcurtis/FV3_RT/'\ + workdir = '/scratch1/BMC/gmtb/RT/auto_RT/Pull_Requests' + blstore = '/scratch1/BMC/gmtb/RT/NCAR' + rtbldir = '/scratch1/BMC/gmtb/RT/FV3_RT/'\ f'REGRESSION_TEST_{job_obj.compiler.upper()}' elif job_obj.machine == 'cheyenne': - workdir = '/glade/scratch/dswales/UFS/autort/tests/auto/pr' - blstore = '/glade/p/ral/jntp/GMTB/ufs-weather-model/RT/NEMSfv3gfs' - rtbldir = '/glade/scratch/dswales/UFS/dswales/FV3_RT/'\ + workdir = '/glade/scratch/epicufsrt/GMTB/ufs-weather-model/RT/auto_RT/Pull_Requests' + blstore = '/glade/scratch/epicufsrt/GMTB/ufs-weather-model/RT/NCAR' + rtbldir = '/glade/scratch/epicufsrt/GMTB/ufs-weather-model/RT/'\ f'REGRESSION_TEST_{job_obj.compiler.upper()}' else: logger.critical(f'Machine {job_obj.machine} is not supported for this job') @@ -102,11 +87,11 @@ def run_regression_test(job_obj, pr_repo_loc): logger = logging.getLogger('BL/RUN_REGRESSION_TEST') if job_obj.compiler == 'gnu': rt_command = [[f'export RT_COMPILER="{job_obj.compiler}" && cd tests ' - '&& /bin/bash --login ./rt.sh -e -c -l rt_gnu.conf', + '&& /bin/bash --login ./rt.ncar.sh -e -c -l rt_gnu.conf', pr_repo_loc]] elif job_obj.compiler == 'intel': rt_command = [[f'export RT_COMPILER="{job_obj.compiler}" && cd tests ' - '&& /bin/bash --login ./rt.sh -e -c', pr_repo_loc]] + '&& /bin/bash --login ./rt.ncar.sh -e -c', pr_repo_loc]] job_obj.run_commands(logger, rt_command) @@ -169,9 +154,9 @@ def post_process(job_obj, pr_repo_loc, repo_dir_str, rtbldir, bldir, bldate, bls def get_bl_date(job_obj, pr_repo_loc): - logger = logging.getLogger('BL/UPDATE_RT_SH') + logger = logging.getLogger('BL/UPDATE_RT_NCAR_SH') BLDATEFOUND = False - with open(f'{pr_repo_loc}/tests/rt.sh', 'r') as f: + with open(f'{pr_repo_loc}/tests/rt.ncar.sh', 'r') as f: for line in f: if 'BL_DATE=' in line: logger.info('Found BL_DATE in line') @@ -189,7 +174,7 @@ def get_bl_date(job_obj, pr_repo_loc): logger.info(f'Date {bldate} is not formatted YYYYMMDD') raise ValueError if not BLDATEFOUND: - job_obj.comment_text_append('[BL] ERROR: Variable "BL_DATE" not found in rt.sh.') + job_obj.comment_text_append('[BL] ERROR: Variable "BL_DATE" not found in rt.ncar.sh.') job_obj.job_failed(logger, 'get_bl_date()') logger.info('Finished get_bl_date') diff --git a/tests/auto/jobs/rt.py b/tests/auto/jobs/rt.py index 0e9ad28b32..f5ec6b09f6 100644 --- a/tests/auto/jobs/rt.py +++ b/tests/auto/jobs/rt.py @@ -15,15 +15,9 @@ def run(job_obj): def set_directories(job_obj): logger = logging.getLogger('RT/SET_DIRECTORIES') if job_obj.machine == 'hera': - workdir = '/scratch1/BMC/gmtb/Dustin.Swales/UFS/auto-RT/Pull_Requests' - elif job_obj.machine == 'jet': - workdir = '/lfs4/HFIP/h-nems/emc.nemspara/autort/pr' - elif job_obj.machine == 'gaea': - workdir = '/lustre/f2/pdata/ncep/emc.nemspara/autort/pr' - elif job_obj.machine == 'orion': - workdir = '/work/noaa/nems/emc.nemspara/autort/pr' + workdir = '/scratch1/BMC/gmtb/RT/auto_RT/Pull_Requests' elif job_obj.machine == 'cheyenne': - workdir = '/glade/scratch/dswales/UFS/autort/tests/auto/pr' + workdir = '/glade/scratch/epicufsrt/GMTB/ufs-weather-model/RT/auto_RT/Pull_Requests' else: print(f'Machine {job_obj.machine} is not supported for this job') raise KeyError @@ -38,11 +32,11 @@ def run_regression_test(job_obj, pr_repo_loc): logger = logging.getLogger('RT/RUN_REGRESSION_TEST') if job_obj.compiler == 'gnu': rt_command = [[f'export RT_COMPILER="{job_obj.compiler}" && cd tests ' - '&& /bin/bash --login ./rt.sh -e -l rt_gnu.conf', + '&& /bin/bash --login ./rt.ncar.sh -e -l rt_gnu.conf', pr_repo_loc]] elif job_obj.compiler == 'intel': rt_command = [[f'export RT_COMPILER="{job_obj.compiler}" && cd tests ' - '&& /bin/bash --login ./rt.sh -e', pr_repo_loc]] + '&& /bin/bash --login ./rt.ncar.sh -e', pr_repo_loc]] job_obj.run_commands(logger, rt_command) @@ -100,7 +94,7 @@ def post_process(job_obj, pr_repo_loc, repo_dir_str, branch): [f'git add {rt_log}', pr_repo_loc], [f'git commit -m "[AutoRT] {job_obj.machine}' f'.{job_obj.compiler} Job Completed.\n\n\n' - 'on-behalf-of @ufs-community "', + 'on-behalf-of @NCAR "', pr_repo_loc], ['sleep 10', pr_repo_loc], [f'git push origin {branch}', pr_repo_loc] diff --git a/tests/auto/rt_auto.py b/tests/auto/rt_auto.py index 6c2471f4a1..46330283c1 100644 --- a/tests/auto/rt_auto.py +++ b/tests/auto/rt_auto.py @@ -310,8 +310,8 @@ def setup_env(): # Dictionary of GitHub repositories to check repo_dict = [{ 'name': 'ufs-weather-model', - 'address': 'dustinswales/ufs-weather-model', - 'base': 'develop' + 'address': 'NCAR/ufs-weather-model', + 'base': 'main' }] # Approved Actions From c8e86d5fd43a0227470157d3234d9d71a8ab1243 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Wed, 5 Jul 2023 21:37:58 +0000 Subject: [PATCH 04/56] Starting removal of hard-coding of test parameters (machine, workdir). --- tests/auto/jobs/bl.py | 4 +- tests/auto/jobs/rt.py | 29 +++--------- tests/auto/rt_auto.py | 91 +++++++++++-------------------------- tests/auto/start_rt_auto.sh | 34 ++++++++------ 4 files changed, 56 insertions(+), 102 deletions(-) diff --git a/tests/auto/jobs/bl.py b/tests/auto/jobs/bl.py index 9d196beab8..e5adc4e2a1 100644 --- a/tests/auto/jobs/bl.py +++ b/tests/auto/jobs/bl.py @@ -125,9 +125,9 @@ def clone_pr_repo(job_obj, workdir): [f'git clone -b {branch} {git_ssh_url}', repo_dir_str], ['git submodule update --init --recursive', f'{repo_dir_str}/{repo_name}'], - ['git config user.email "dswales@ucar.edu"', + ['git config user.email "kavulich@ucar.edu"', f'{repo_dir_str}/{repo_name}'], - ['git config user.name "dustinswales"', + ['git config user.name "mkavulich"', f'{repo_dir_str}/{repo_name}'] ] diff --git a/tests/auto/jobs/rt.py b/tests/auto/jobs/rt.py index f5ec6b09f6..142d6d61a7 100644 --- a/tests/auto/jobs/rt.py +++ b/tests/auto/jobs/rt.py @@ -6,28 +6,11 @@ def run(job_obj): logger = logging.getLogger('RT/RUN') - workdir = set_directories(job_obj) - branch, pr_repo_loc, repo_dir_str = clone_pr_repo(job_obj, workdir) + branch, pr_repo_loc, repo_dir_str = clone_pr_repo(job_obj) run_regression_test(job_obj, pr_repo_loc) post_process(job_obj, pr_repo_loc, repo_dir_str, branch) -def set_directories(job_obj): - logger = logging.getLogger('RT/SET_DIRECTORIES') - if job_obj.machine == 'hera': - workdir = '/scratch1/BMC/gmtb/RT/auto_RT/Pull_Requests' - elif job_obj.machine == 'cheyenne': - workdir = '/glade/scratch/epicufsrt/GMTB/ufs-weather-model/RT/auto_RT/Pull_Requests' - else: - print(f'Machine {job_obj.machine} is not supported for this job') - raise KeyError - - logger.info(f'machine: {job_obj.machine}') - logger.info(f'workdir: {workdir}') - - return workdir - - def run_regression_test(job_obj, pr_repo_loc): logger = logging.getLogger('RT/RUN_REGRESSION_TEST') if job_obj.compiler == 'gnu': @@ -49,7 +32,7 @@ def remove_pr_data(job_obj, pr_repo_loc, repo_dir_str, rt_dir): job_obj.run_commands(logger, rm_command) -def clone_pr_repo(job_obj, workdir): +def clone_pr_repo(job_obj): ''' clone the GitHub pull request repo, via command line ''' logger = logging.getLogger('RT/CLONE_PR_REPO') repo_name = job_obj.preq_dict['preq'].head.repo.name @@ -58,7 +41,7 @@ def clone_pr_repo(job_obj, workdir): git_ssh_url = job_obj.preq_dict['preq'].head.repo.ssh_url logger.debug(f'GIT SSH_URL: {git_ssh_url}') logger.info('Starting repo clone') - repo_dir_str = f'{workdir}/'\ + repo_dir_str = f'{job_obj.workdir}/'\ f'{str(job_obj.preq_dict["preq"].id)}/'\ f'{datetime.datetime.now().strftime("%Y%m%d%H%M%S")}' pr_repo_loc = f'{repo_dir_str}/{repo_name}' @@ -68,9 +51,9 @@ def clone_pr_repo(job_obj, workdir): [f'git clone -b {branch} {git_ssh_url}', repo_dir_str], ['git submodule update --init --recursive', f'{repo_dir_str}/{repo_name}'], - ['git config user.email "dswales@ucar.edu"', + ['git config user.email "kavulich@ucar.edu"', f'{repo_dir_str}/{repo_name}'], - ['git config user.name "dustinswales"', + ['git config user.name "mkavulich"', f'{repo_dir_str}/{repo_name}'] ] @@ -94,7 +77,7 @@ def post_process(job_obj, pr_repo_loc, repo_dir_str, branch): [f'git add {rt_log}', pr_repo_loc], [f'git commit -m "[AutoRT] {job_obj.machine}' f'.{job_obj.compiler} Job Completed.\n\n\n' - 'on-behalf-of @NCAR "', + 'on-behalf-of @NCAR "', pr_repo_loc], ['sleep 10', pr_repo_loc], [f'git push origin {branch}', pr_repo_loc] diff --git a/tests/auto/rt_auto.py b/tests/auto/rt_auto.py index 46330283c1..5e9acffcb7 100644 --- a/tests/auto/rt_auto.py +++ b/tests/auto/rt_auto.py @@ -7,6 +7,7 @@ prior to start. """ from github import Github as gh +import argparse import datetime import subprocess import re @@ -75,20 +76,7 @@ def set_action_from_label(machine, actions, label): logging.info(f'Compiler: {label_compiler}, Action: {action_match}') return label_compiler, action_match -def delete_pr_dirs(each_pr, machine): - if machine == 'hera': - workdir = '/scratch1/BMC/gmtb/Dustin.Swales/UFS/auto-RT/Pull_Requests' - elif machine == 'jet': - workdir = '/lfs4/HFIP/h-nems/emc.nemspara/autort/pr' - elif machine == 'gaea': - workdir = '/lustre/f2/pdata/ncep/emc.nemspara/autort/pr' - elif machine == 'orion': - workdir = '/work/noaa/nems/emc.nemspara/autort/pr' - elif machine == 'cheyenne': - workdir = '/glade/scratch/dswales/UFS/autort/tests/auto/pr' - else: - logging.error(f'Machine {machine} is not supported for this job') - raise KeyError +def delete_pr_dirs(each_pr, machine, workdir): ids = [str(pr.id) for pr in each_pr] logging.debug(f'ids are: {ids}') dirs = [x.split('/')[-2] for x in glob(f'{workdir}/*/')] @@ -110,20 +98,6 @@ def delete_pr_dirs(each_pr, machine): def delete_rt_dirs(in_dir, machine, workdir): - if machine == 'hera': - rt_dir = '/scratch1/BMC/gmtb/Dustin.Swales/UFS/stmp2/Dustin.Swales/FV3_RT' - elif machine == 'jet': - rt_dir ='/lfs4/HFIP/h-nems/emc.nemspara/RT_BASELINE/'\ - f'emc.nemspara/FV3_RT' - elif machine == 'gaea': - rt_dir = '/lustre/f2/scratch/emc.nemspara/FV3_RT' - elif machine == 'orion': - rt_dir = '/work/noaa/stmp/bcurtis/stmp/bcurtis/FV3_RT' - elif machine == 'cheyenne': - rt_dir = '/glade/scratch/dswales/UFS/dswales/FV3_RT/' - else: - logging.error(f'Machine {machine} is not supported for this job') - raise KeyError globdir = f'{workdir}/{in_dir}/**/compile_*.log' logging.debug(f'globdir: {globdir}') logfiles = glob(globdir, recursive=True) @@ -149,7 +123,7 @@ def delete_rt_dirs(in_dir, machine, workdir): logging.debug(f'{rt_dir}/{match} does not exist, not attempting to remove') -def get_preqs_with_actions(repos, machine, ghinterface_obj, actions): +def get_preqs_with_actions(repos, machine, workdir, ghinterface_obj, actions): ''' Create list of dictionaries of a pull request and its machine label and action ''' logger = logging.getLogger('GET_PREQS_WITH_ACTIONS') @@ -158,7 +132,7 @@ def get_preqs_with_actions(repos, machine, ghinterface_obj, actions): .get_pulls(state='open', sort='created', base=repo['base']) for repo in repos] each_pr = [preq for gh_preq in gh_preqs for preq in gh_preq] - delete_pr_dirs(each_pr, machine) + delete_pr_dirs(each_pr, machine, workdir) preq_labels = [{'preq': pr, 'label': label} for pr in each_pr for label in pr.get_labels()] @@ -170,7 +144,7 @@ def get_preqs_with_actions(repos, machine, ghinterface_obj, actions): if match: pr_label['action'] = match # return_preq.append(pr_label.copy()) - jobs.append(Job(pr_label.copy(), ghinterface_obj, machine, compiler)) + jobs.append(Job(pr_label.copy(), ghinterface_obj, machine, compiler, workdir)) return jobs @@ -192,7 +166,7 @@ class Job: provided by the bash script ''' - def __init__(self, preq_dict, ghinterface_obj, machine, compiler): + def __init__(self, preq_dict, ghinterface_obj, machine, compiler, workdir): self.logger = logging.getLogger('JOB') self.preq_dict = preq_dict self.job_mod = importlib.import_module( @@ -200,6 +174,7 @@ def __init__(self, preq_dict, ghinterface_obj, machine, compiler): self.ghinterface_obj = ghinterface_obj self.machine = machine self.compiler = compiler + self.workdir = workdir self.comment_text = '***Automated RT Failure Notification***\n' self.failed_tests = [] @@ -209,7 +184,7 @@ def comment_text_append(self, newtext): def remove_pr_label(self): ''' Removes the PR label that initiated the job run from PR ''' self.logger.info(f'Removing Label: {self.preq_dict["label"]}') - self.preq_dict['preq'].remove_from_labels(self.preq_dict['label']) +# self.preq_dict['preq'].remove_from_labels(self.preq_dict['label']) def check_label_before_job_start(self): # LETS Check the label still exists before the start of the job in the @@ -260,7 +235,7 @@ def run(self): except Exception: self.job_failed(logger, 'run()') logger.info('Sending comment text') - self.send_comment_text() +# self.send_comment_text() else: logger.info(f'Cannot find label {self.preq_dict["label"]}') @@ -283,41 +258,18 @@ def job_failed(self, logger, job_name, exception=Exception, STDOUT=False, logger.critical(f'STDOUT: {[item for item in out if not None]}') logger.critical(f'STDERR: {[eitem for eitem in err if not None]}') -def setup_env(): - hostname = os.getenv('HOSTNAME') - if bool(re.match(re.compile('hfe.+'), hostname)): - machine = 'hera' - elif bool(re.match(re.compile('hecflow.+'), hostname)): - machine = 'hera' - elif bool(re.match(re.compile('fe.+'), hostname)): - machine = 'jet' - os.environ['ACCNR'] = 'h-nems' - elif bool(re.match(re.compile('tfe.+'), hostname)): - machine = 'jet' - os.environ['ACCNR'] = 'h-nems' - elif bool(re.match(re.compile('gaea.+'), hostname)): - machine = 'gaea' - os.environ['ACCNR'] = 'nggps_emc' - elif bool(re.match(re.compile('Orion-login.+'), hostname)): - machine = 'orion' - elif bool(re.match(re.compile('chadmin.+'), hostname)): - machine = 'cheyenne' - os.environ['ACCNR'] = 'P48503002' - else: - raise KeyError(f'Hostname: {hostname} does not match '\ - 'for a supported system. Exiting.') - +def setup_env(name='ufs-weather-model', address='NCAR/ufs-weather-model', base='main'): # Dictionary of GitHub repositories to check repo_dict = [{ - 'name': 'ufs-weather-model', - 'address': 'NCAR/ufs-weather-model', - 'base': 'main' + 'name': name, + 'address': address, + 'base': base }] # Approved Actions action_list = ['RT', 'BL'] - return machine, repo_dict, action_list + return repo_dict, action_list def main(): @@ -332,9 +284,20 @@ def main(): logger = logging.getLogger('MAIN') logger.info('Starting Script') + parser = argparse.ArgumentParser() + parser.add_argument('-m','--machine', help='current machine name', required=True) + parser.add_argument('-w','--workdir', help='directory where tests will be staged and run', required=True) + args = parser.parse_args() + + machine = args.machine + workdir = args.workdir + + logger.info(f'Machine: {machine}') + logger.info(f'Working directory: {workdir}') + # setup environment logger.info('Getting the environment setup') - machine, repos, actions = setup_env() + repos, actions = setup_env() # setup interface with GitHub logger.info('Setting up GitHub interface.') @@ -344,7 +307,7 @@ def main(): # and turn them into Job objects logger.info('Getting all pull requests, ' 'labels and actions applicable to this machine.') - jobs = get_preqs_with_actions(repos, machine, + jobs = get_preqs_with_actions(repos, machine, workdir, ghinterface_obj, actions) [job.run() for job in jobs] diff --git a/tests/auto/start_rt_auto.sh b/tests/auto/start_rt_auto.sh index 24c17e7389..f724fb76de 100755 --- a/tests/auto/start_rt_auto.sh +++ b/tests/auto/start_rt_auto.sh @@ -1,33 +1,41 @@ #!/bin/bash --login set -eux -if [[ $HOSTNAME == hfe* ]]; then - export PATH=/scratch1/NCEPDEV/nems/emc.nemspara/soft/miniconda3/bin:$PATH - export PYTHONPATH=/scratch1/NCEPDEV/nems/emc.nemspara/soft/miniconda3/lib/python3.8/site-packages -elif [[ $HOSTNAME == hecflow* ]]; then +function usage { + echo "usage: $0 machine workdir" + echo " machine: Name of current machine [hera/cheyenne]" + echo " workdir: Working directory for checking out and running tests" + exit 1 +} +if [ "$#" -lt 2 ]; then + echo "Need to provide machine and working directory" + usage +fi + +machine=$1 +workdir=$2 + +if [[ $machine == hera ]]; then export PATH=/scratch1/NCEPDEV/nems/emc.nemspara/soft/miniconda3/bin:$PATH export PYTHONPATH=/scratch1/NCEPDEV/nems/emc.nemspara/soft/miniconda3/lib/python3.8/site-packages -elif [[ $HOSTNAME == Orion-login-* ]]; then +elif [[ $machine == orion ]]; then export PATH=/work/noaa/nems/emc.nemspara/soft/miniconda3/bin:$PATH export PYTHONPATH=/work/noaa/nems/emc.nemspara/soft/miniconda3/lib/python3.8/site-packages -elif [[ $HOSTNAME == fe* ]]; then - export PATH=/lfs4/HFIP/hfv3gfs/software/miniconda3/4.8.3/envs/ufs-weather-model/bin:/lfs4/HFIP/hfv3gfs/software/miniconda3/4.8.3/bin:$PATH - export PYTHONPATH=/lfs4/HFIP/hfv3gfs/software/miniconda3/4.8.3/envs/ufs-weather-model/lib/python3.8/site-packages:/lfs4/HFIP/hfv3gfs/software/miniconda3/4.8.3/lib/python3.8/site-packages -elif [[ $HOSTNAME == tfe* ]]; then +elif [[ $machine == jet ]]; then export PATH=/lfs4/HFIP/hfv3gfs/software/miniconda3/4.8.3/envs/ufs-weather-model/bin:/lfs4/HFIP/hfv3gfs/software/miniconda3/4.8.3/bin:$PATH export PYTHONPATH=/lfs4/HFIP/hfv3gfs/software/miniconda3/4.8.3/envs/ufs-weather-model/lib/python3.8/site-packages:/lfs4/HFIP/hfv3gfs/software/miniconda3/4.8.3/lib/python3.8/site-packages -elif [[ $HOSTNAME == gaea* ]]; then +elif [[ $machine == gaea ]]; then export PATH=/lustre/f2/pdata/esrl/gsd/contrib/miniconda3/4.8.3/envs/ufs-weather-model/bin:$PATH export PYTHONPATH=/lustre/f2/pdata/esrl/gsd/contrib/miniconda3/4.8.3/lib/python3.8/site-packages -elif [[ $HOSTNAME == *chadmin* ]]; then +elif [[ $machine == cheyenne ]]; then export MACHINE_ID=cheyenne export PATH=/glade/p/ral/jntp/tools/miniconda3/4.8.3/envs/ufs-weather-model/bin:/glade/p/ral/jntp/tools/miniconda3/4.8.3/bin:$PATH export PYTHONPATH=/glade/p/ral/jntp/tools/miniconda3/4.8.3/envs/ufs-weather-model/lib/python3.8/site-packages:/glade/p/ral/jntp/tools/miniconda3/4.8.3/lib/python3.8/site-packages else - echo "No Python Path for this machine." + echo "No Python Path has been set up for this machine ($machine)." exit 1 fi -python rt_auto.py +python rt_auto.py -m=$machine -w=$workdir exit 0 From dea25d96c2be94785ef077c917052e4a3c9aef33 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Wed, 27 Sep 2023 15:24:16 +0000 Subject: [PATCH 05/56] I think this will work! Adding workdir as a command line argument all the way through --- tests/auto/jobs/bl.py | 26 ++++++----- tests/auto/jobs/rt.py | 4 +- tests/auto/rt_auto.py | 21 +++++---- tests/auto/start_rt_auto.sh | 6 +-- tests/rt.ncar.sh | 92 ++++++++++++++++++++++++++++++++++--- 5 files changed, 117 insertions(+), 32 deletions(-) diff --git a/tests/auto/jobs/bl.py b/tests/auto/jobs/bl.py index e5adc4e2a1..e25520e66a 100644 --- a/tests/auto/jobs/bl.py +++ b/tests/auto/jobs/bl.py @@ -7,37 +7,39 @@ def run(job_obj): logger = logging.getLogger('BL/RUN') - workdir, rtbldir, blstore = set_directories(job_obj) + workdir, newbldir, blstore = set_directories(job_obj) pr_repo_loc, repo_dir_str = clone_pr_repo(job_obj, workdir) bldate = get_bl_date(job_obj, pr_repo_loc) bldir = f'{blstore}/main-{bldate}/{job_obj.compiler.upper()}' bldirbool = check_for_bl_dir(bldir, job_obj) run_regression_test(job_obj, pr_repo_loc) - post_process(job_obj, pr_repo_loc, repo_dir_str, rtbldir, bldir, bldate, blstore) + post_process(job_obj, pr_repo_loc, repo_dir_str, newbldir, bldir, bldate, blstore) def set_directories(job_obj): logger = logging.getLogger('BL/SET_DIRECTORIES') if job_obj.machine == 'hera': - workdir = '/scratch1/BMC/gmtb/RT/auto_RT/Pull_Requests' - blstore = '/scratch1/BMC/gmtb/RT/NCAR' - rtbldir = '/scratch1/BMC/gmtb/RT/FV3_RT/'\ + rt_dir = '/scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/' + blstore = f'{rt_dir}/baselines' + newbldir = f'{rt_dir}/FV3_RT/'\ f'REGRESSION_TEST_{job_obj.compiler.upper()}' elif job_obj.machine == 'cheyenne': workdir = '/glade/scratch/epicufsrt/GMTB/ufs-weather-model/RT/auto_RT/Pull_Requests' blstore = '/glade/scratch/epicufsrt/GMTB/ufs-weather-model/RT/NCAR' - rtbldir = '/glade/scratch/epicufsrt/GMTB/ufs-weather-model/RT/'\ + newbldir = '/glade/scratch/epicufsrt/GMTB/ufs-weather-model/RT/'\ f'REGRESSION_TEST_{job_obj.compiler.upper()}' else: logger.critical(f'Machine {job_obj.machine} is not supported for this job') raise KeyError + workdir = job_obj.workdir + logger.info(f'machine: {job_obj.machine}') logger.info(f'workdir: {workdir}') logger.info(f'blstore: {blstore}') - logger.info(f'rtbldir: {rtbldir}') + logger.info(f'newbldir: {newbldir}') - return workdir, rtbldir, blstore + return workdir, newbldir, blstore def check_for_bl_dir(bldir, job_obj): @@ -87,11 +89,11 @@ def run_regression_test(job_obj, pr_repo_loc): logger = logging.getLogger('BL/RUN_REGRESSION_TEST') if job_obj.compiler == 'gnu': rt_command = [[f'export RT_COMPILER="{job_obj.compiler}" && cd tests ' - '&& /bin/bash --login ./rt.ncar.sh -e -c -l rt_gnu.conf', + '&& /bin/bash --login ./rt.ncar.sh -e -c -l rt_gnu.conf -p {job_obj.machine}', pr_repo_loc]] elif job_obj.compiler == 'intel': rt_command = [[f'export RT_COMPILER="{job_obj.compiler}" && cd tests ' - '&& /bin/bash --login ./rt.ncar.sh -e -c', pr_repo_loc]] + '&& /bin/bash --login ./rt.ncar.sh -e -c -p {job_obj.machine}', pr_repo_loc]] job_obj.run_commands(logger, rt_command) @@ -137,7 +139,7 @@ def clone_pr_repo(job_obj, workdir): return pr_repo_loc, repo_dir_str -def post_process(job_obj, pr_repo_loc, repo_dir_str, rtbldir, bldir, bldate, blstore): +def post_process(job_obj, pr_repo_loc, repo_dir_str, newbldir, bldir, bldate, blstore): logger = logging.getLogger('BL/MOVE_RT_LOGS') rt_log = f'tests/RegressionTests_{job_obj.machine}'\ f'.{job_obj.compiler}.log' @@ -145,7 +147,7 @@ def post_process(job_obj, pr_repo_loc, repo_dir_str, rtbldir, bldir, bldate, bls rt_dir, logfile_pass = process_logfile(job_obj, filepath) if logfile_pass: create_bl_dir(bldir, job_obj) - move_bl_command = [[f'mv {rtbldir}/* {bldir}/', pr_repo_loc]] + move_bl_command = [[f'mv {newbldir}/* {bldir}/', pr_repo_loc]] job_obj.run_commands(logger, move_bl_command) job_obj.comment_text_append('[BL] Baseline creation and move successful') logger.info('Starting RT Job') diff --git a/tests/auto/jobs/rt.py b/tests/auto/jobs/rt.py index 142d6d61a7..664bd18b16 100644 --- a/tests/auto/jobs/rt.py +++ b/tests/auto/jobs/rt.py @@ -15,11 +15,11 @@ def run_regression_test(job_obj, pr_repo_loc): logger = logging.getLogger('RT/RUN_REGRESSION_TEST') if job_obj.compiler == 'gnu': rt_command = [[f'export RT_COMPILER="{job_obj.compiler}" && cd tests ' - '&& /bin/bash --login ./rt.ncar.sh -e -l rt_gnu.conf', + '&& /bin/bash --login ./rt.ncar.sh -e -l rt_gnu.conf -p {job_obj.machine}', pr_repo_loc]] elif job_obj.compiler == 'intel': rt_command = [[f'export RT_COMPILER="{job_obj.compiler}" && cd tests ' - '&& /bin/bash --login ./rt.ncar.sh -e', pr_repo_loc]] + '&& /bin/bash --login ./rt.ncar.sh -e -p {job_obj.machine}', pr_repo_loc]] job_obj.run_commands(logger, rt_command) diff --git a/tests/auto/rt_auto.py b/tests/auto/rt_auto.py index 5e9acffcb7..2aceab9506 100644 --- a/tests/auto/rt_auto.py +++ b/tests/auto/rt_auto.py @@ -184,7 +184,7 @@ def comment_text_append(self, newtext): def remove_pr_label(self): ''' Removes the PR label that initiated the job run from PR ''' self.logger.info(f'Removing Label: {self.preq_dict["label"]}') -# self.preq_dict['preq'].remove_from_labels(self.preq_dict['label']) + self.preq_dict['preq'].remove_from_labels(self.preq_dict['label']) def check_label_before_job_start(self): # LETS Check the label still exists before the start of the job in the @@ -228,14 +228,14 @@ def run(self): self.comment_text_append(f'Job: {self.preq_dict["action"]}') if self.check_label_before_job_start(): try: - logger.info('Calling remove_pr_label') - self.remove_pr_label() +# logger.info('Calling remove_pr_label') +# self.remove_pr_label() logger.info('Calling Job to Run') self.job_mod.run(self) except Exception: self.job_failed(logger, 'run()') logger.info('Sending comment text') -# self.send_comment_text() + self.send_comment_text() else: logger.info(f'Cannot find label {self.preq_dict["label"]}') @@ -277,21 +277,26 @@ def main(): # handle logging log_filename = f'rt_auto_'\ f'{datetime.datetime.now().strftime("%Y%m%d%H%M%S")}.log' - #logging.basicConfig(filename=log_filename, filemode='w', - # level=logging.INFO) logging.basicConfig(filename=log_filename, filemode='w', - level=logging.DEBUG) + level=logging.INFO) logger = logging.getLogger('MAIN') logger.info('Starting Script') parser = argparse.ArgumentParser() parser.add_argument('-m','--machine', help='current machine name', required=True) - parser.add_argument('-w','--workdir', help='directory where tests will be staged and run', required=True) + parser.add_argument('-w','--workdir', help='directory where tests will be staged and run', required=False) + parser.add_argument('-d','--debug', help='Set logging to more verbose output', action='store_true') + args = parser.parse_args() machine = args.machine workdir = args.workdir + if args.debug: + logger.info('Setting logging level to debug') + for handler in logger.handlers: + handler.setLevel(logging.DEBUG) + logger.info(f'Machine: {machine}') logger.info(f'Working directory: {workdir}') diff --git a/tests/auto/start_rt_auto.sh b/tests/auto/start_rt_auto.sh index f724fb76de..f3f0482480 100755 --- a/tests/auto/start_rt_auto.sh +++ b/tests/auto/start_rt_auto.sh @@ -2,13 +2,14 @@ set -eux function usage { + set +x echo "usage: $0 machine workdir" echo " machine: Name of current machine [hera/cheyenne]" echo " workdir: Working directory for checking out and running tests" exit 1 } if [ "$#" -lt 2 ]; then - echo "Need to provide machine and working directory" + echo "Need to provide machine and workdir" usage fi @@ -28,7 +29,6 @@ elif [[ $machine == gaea ]]; then export PATH=/lustre/f2/pdata/esrl/gsd/contrib/miniconda3/4.8.3/envs/ufs-weather-model/bin:$PATH export PYTHONPATH=/lustre/f2/pdata/esrl/gsd/contrib/miniconda3/4.8.3/lib/python3.8/site-packages elif [[ $machine == cheyenne ]]; then - export MACHINE_ID=cheyenne export PATH=/glade/p/ral/jntp/tools/miniconda3/4.8.3/envs/ufs-weather-model/bin:/glade/p/ral/jntp/tools/miniconda3/4.8.3/bin:$PATH export PYTHONPATH=/glade/p/ral/jntp/tools/miniconda3/4.8.3/envs/ufs-weather-model/lib/python3.8/site-packages:/glade/p/ral/jntp/tools/miniconda3/4.8.3/lib/python3.8/site-packages else @@ -36,6 +36,6 @@ else exit 1 fi -python rt_auto.py -m=$machine -w=$workdir +python rt_auto.py -m=$machine -w=$workdir -d exit 0 diff --git a/tests/rt.ncar.sh b/tests/rt.ncar.sh index 71c71f0a1c..616a9dfc87 100755 --- a/tests/rt.ncar.sh +++ b/tests/rt.ncar.sh @@ -21,7 +21,8 @@ usage() { echo " -n run single test " echo " -r use Rocoto workflow manager" echo " -w for weekly_test, skip comparing baseline results" - echo " -d delete run direcotries that are not used by other tests" + echo " -d delete run directories that are not used by other tests" + echo " -p platform (machine name)" echo set -x exit 1 @@ -127,7 +128,73 @@ fi readonly RT_SINGLE_CONF='rt_single.conf' -source detect_machine.sh # Note: this does not set ACCNR. The "if" block below does. +CREATE_BASELINE=false +ROCOTO=false +ECFLOW=false +KEEP_RUNDIR=false +SINGLE_NAME='' +TEST_35D=false +export skip_check_results=false +export delete_rundir=false + +TESTS_FILE='rt.conf' + +while getopts ":cl:mn:dwkreh" opt; do + case $opt in + c) + CREATE_BASELINE=true + ;; + l) + TESTS_FILE=$OPTARG + ;; + m) + # redefine RTPWD to point to newly created baseline outputs + RTPWD=${NEW_BASELINE} + ;; + n) + SINGLE_NAME=$OPTARG + ;; + d) + export delete_rundir=true + awk -F "|" '{print $5}' rt.conf | grep "\S" > keep_tests.tmp + ;; + s) + export skip_check_results=true + ;; + k) + KEEP_RUNDIR=true + ;; + r) + ROCOTO=true + ECFLOW=false + ;; + e) + ECFLOW=true + ROCOTO=false + ;; + p) + MACHINE_ID=$OPTARG + ;; + w) + dprefix=$OPTARG + ;; + h) + usage + ;; + \?) + usage + die "Invalid option: -$OPTARG" + ;; + :) + usage + die "Option -$OPTARG requires an argument." + ;; + esac +done + +# Default compiler "intel" +export RT_COMPILER=${RT_COMPILER:-intel} + source rt_utils.sh source module-setup.sh @@ -304,7 +371,7 @@ elif [[ $MACHINE_ID = hera ]]; then ACCNR=gmtb PARTITION= - dprefix=/scratch1/BMC/gmtb + dprefix=${dprefix:-/scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model} DISKNM=$dprefix/RT STMP=$dprefix/RT/stmp4 PTMP=$dprefix/RT/stmp2 @@ -313,8 +380,6 @@ elif [[ $MACHINE_ID = hera ]]; then elif [[ $MACHINE_ID = orion ]]; then - module load git/2.28.0 - module load gcc/8.3.0 module load contrib rocoto/1.3.1 @@ -428,7 +493,7 @@ elif [[ $MACHINE_ID = expanse ]]; then PTMP=$dprefix SCHEDULER=slurm - elif [[ $MACHINE_ID = noaacloud.* ]]; then +elif [[ $MACHINE_ID = noaacloud.* ]]; then module use /apps/modules/modulefiles module load rocoto/1.3.3 @@ -447,11 +512,17 @@ elif [[ $MACHINE_ID = expanse ]]; then PTMP=$dprefix/stmp2 SCHEDULER=slurm - else die "Unknown machine ID, please edit detect_machine.sh file" fi +# If account is unspecified, assume the machine has a "nems" +# accounting code. +export ACCNR="${ACCNR:-nems}" + +# Display the machine and account +echo "Machine: " $MACHINE_ID " Account: " $ACCNR + mkdir -p ${STMP}/${USER} NEW_BASELINE=${STMP}/${USER}/FV3_RT/REGRESSION_TEST @@ -627,6 +698,13 @@ EOF die "ecFlow is not supported on this machine $MACHINE_ID" fi +else + + if [[ $MACHINE_ID = hera ]] && [[ $HOSTNAME = hecflow ]]; then + echo "ERROR: To run without using ECFlow on Hera, please do not use ecflow node." + exit 1 + fi + fi ## From 2bf3cdc37d2ab4f917ea4720246302dab73f0a8f Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Thu, 20 Jul 2023 23:34:08 +0000 Subject: [PATCH 06/56] maybe this will work --- tests/rt.ncar.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/rt.ncar.sh b/tests/rt.ncar.sh index 616a9dfc87..87de56fef2 100755 --- a/tests/rt.ncar.sh +++ b/tests/rt.ncar.sh @@ -139,7 +139,7 @@ export delete_rundir=false TESTS_FILE='rt.conf' -while getopts ":cl:mn:dwkreh" opt; do +while getopts ":cl:mn:dw:krepsh" opt; do case $opt in c) CREATE_BASELINE=true From 9869ed61b7ef00f9bc1a819dc27e1f110a4119de Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Thu, 20 Jul 2023 23:47:54 +0000 Subject: [PATCH 07/56] I somehow did something weird...hopefully now it succeeds. --- tests/rt.ncar.sh | 37 ++++++++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/tests/rt.ncar.sh b/tests/rt.ncar.sh index 87de56fef2..c0a1fd2467 100755 --- a/tests/rt.ncar.sh +++ b/tests/rt.ncar.sh @@ -139,20 +139,44 @@ export delete_rundir=false TESTS_FILE='rt.conf' -while getopts ":cl:mn:dw:krepsh" opt; do +SKIP_ORDER=false + +while getopts "a:cl:mn:dw:krepsh" opt; do case $opt in + a) + ACCNR=$OPTARG + ;; c) CREATE_BASELINE=true ;; l) TESTS_FILE=$OPTARG + SKIP_ORDER=true ;; m) # redefine RTPWD to point to newly created baseline outputs RTPWD=${NEW_BASELINE} ;; n) - SINGLE_NAME=$OPTARG + SINGLE_OPTS=("$OPTARG") + until [[ $(eval "echo \${$OPTIND}") =~ ^-.* ]] || [ -z $(eval "echo \${$OPTIND}") ]; do + SINGLE_OPTS+=($(eval "echo \${$OPTIND}")) + OPTIND=$((OPTIND + 1)) + done + + if [[ ${#SINGLE_OPTS[@]} != 2 ]]; then + echo "The -n option needs AND , i.e. -n control_p8 intel" + exit 1 + fi + SINGLE_NAME=${SINGLE_OPTS[0],,} + export RT_COMPILER=${SINGLE_OPTS[1],,} + + if [[ "$RT_COMPILER" == "intel" ]] || [[ "$RT_COMPILER" == "gnu" ]]; then + echo "COMPILER set to ${RT_COMPILER}" + else + echo "Compiler must be either 'intel' or 'gnu'." + exit 1 + fi ;; d) export delete_rundir=true @@ -192,9 +216,6 @@ while getopts ":cl:mn:dw:krepsh" opt; do esac done -# Default compiler "intel" -export RT_COMPILER=${RT_COMPILER:-intel} - source rt_utils.sh source module-setup.sh @@ -369,7 +390,6 @@ elif [[ $MACHINE_ID = hera ]]; then QUEUE=batch COMPILE_QUEUE=batch - ACCNR=gmtb PARTITION= dprefix=${dprefix:-/scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model} DISKNM=$dprefix/RT @@ -455,7 +475,6 @@ elif [[ $MACHINE_ID = cheyenne ]]; then ECFLOW_START=/glade/p/ral/jntp/tools/miniconda3/4.8.3/envs/ufs-weather-model/bin/ecflow_start.sh ECF_PORT=$(( $(id -u) + 1500 )) - ACCNR=P48503002 QUEUE=regular COMPILE_QUEUE=regular PARTITION= @@ -516,10 +535,6 @@ else die "Unknown machine ID, please edit detect_machine.sh file" fi -# If account is unspecified, assume the machine has a "nems" -# accounting code. -export ACCNR="${ACCNR:-nems}" - # Display the machine and account echo "Machine: " $MACHINE_ID " Account: " $ACCNR From 0cd3f0d93868adb79ebf40408a228b88528c6b91 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Thu, 20 Jul 2023 23:54:14 +0000 Subject: [PATCH 08/56] Okay, got to the ECflow step, let's see what happens next --- tests/auto/jobs/rt.py | 4 ++-- tests/rt.ncar.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/auto/jobs/rt.py b/tests/auto/jobs/rt.py index 664bd18b16..b2f35b6f10 100644 --- a/tests/auto/jobs/rt.py +++ b/tests/auto/jobs/rt.py @@ -15,11 +15,11 @@ def run_regression_test(job_obj, pr_repo_loc): logger = logging.getLogger('RT/RUN_REGRESSION_TEST') if job_obj.compiler == 'gnu': rt_command = [[f'export RT_COMPILER="{job_obj.compiler}" && cd tests ' - '&& /bin/bash --login ./rt.ncar.sh -e -l rt_gnu.conf -p {job_obj.machine}', + '&& /bin/bash --login ./rt.ncar.sh -e -l rt_gnu.conf -p {job_obj.machine} -w {job_obj.workdir}', pr_repo_loc]] elif job_obj.compiler == 'intel': rt_command = [[f'export RT_COMPILER="{job_obj.compiler}" && cd tests ' - '&& /bin/bash --login ./rt.ncar.sh -e -p {job_obj.machine}', pr_repo_loc]] + '&& /bin/bash --login ./rt.ncar.sh -e -p {job_obj.machine} -w {job_obj.workdir}', pr_repo_loc]] job_obj.run_commands(logger, rt_command) diff --git a/tests/rt.ncar.sh b/tests/rt.ncar.sh index c0a1fd2467..d6c22c4be4 100755 --- a/tests/rt.ncar.sh +++ b/tests/rt.ncar.sh @@ -141,7 +141,7 @@ TESTS_FILE='rt.conf' SKIP_ORDER=false -while getopts "a:cl:mn:dw:krepsh" opt; do +while getopts "a:cl:mn:dw:krep:sh" opt; do case $opt in a) ACCNR=$OPTARG From 43bb1d44fcb563521f207019fe8523fab6325372 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Thu, 20 Jul 2023 23:58:25 +0000 Subject: [PATCH 09/56] argh! another dumb error --- tests/auto/jobs/rt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/auto/jobs/rt.py b/tests/auto/jobs/rt.py index b2f35b6f10..f0d6e1517b 100644 --- a/tests/auto/jobs/rt.py +++ b/tests/auto/jobs/rt.py @@ -15,11 +15,11 @@ def run_regression_test(job_obj, pr_repo_loc): logger = logging.getLogger('RT/RUN_REGRESSION_TEST') if job_obj.compiler == 'gnu': rt_command = [[f'export RT_COMPILER="{job_obj.compiler}" && cd tests ' - '&& /bin/bash --login ./rt.ncar.sh -e -l rt_gnu.conf -p {job_obj.machine} -w {job_obj.workdir}', + f'&& /bin/bash --login ./rt.ncar.sh -e -l rt_gnu.conf -p {job_obj.machine} -w {job_obj.workdir}', pr_repo_loc]] elif job_obj.compiler == 'intel': rt_command = [[f'export RT_COMPILER="{job_obj.compiler}" && cd tests ' - '&& /bin/bash --login ./rt.ncar.sh -e -p {job_obj.machine} -w {job_obj.workdir}', pr_repo_loc]] + f'&& /bin/bash --login ./rt.ncar.sh -e -p {job_obj.machine} -w {job_obj.workdir}', pr_repo_loc]] job_obj.run_commands(logger, rt_command) From 41c555d2b34b7f41a937115dc5ff434772729da1 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Fri, 21 Jul 2023 02:40:28 +0000 Subject: [PATCH 10/56] Finally figured out how to manually tests properly...this is close to working...for real this time! --- tests/default_vars.sh | 18 +++---- tests/rt.ncar.sh | 110 ++++-------------------------------------- 2 files changed, 19 insertions(+), 109 deletions(-) diff --git a/tests/default_vars.sh b/tests/default_vars.sh index 2b2b7beb1b..5ab6568125 100755 --- a/tests/default_vars.sh +++ b/tests/default_vars.sh @@ -94,7 +94,7 @@ ocn_omp_num_threads=1 wav_omp_num_threads=1 -if [[ $MACHINE_ID = wcoss2 || $MACHINE_ID = acorn ]]; then +if [[ $MACHINE_ID = wcoss2.* || $MACHINE_ID = acorn.* ]]; then TPN=128 @@ -107,7 +107,7 @@ if [[ $MACHINE_ID = wcoss2 || $MACHINE_ID = acorn ]]; then INPES_cpl_atmw_gdas=6; JNPES_cpl_atmw_gdas=8; WPG_cpl_atmw_gdas=24 WAV_tasks_atmw_gdas=248 -elif [[ $MACHINE_ID = orion ]]; then +elif [[ $MACHINE_ID = orion.* ]]; then TPN=40 @@ -120,7 +120,7 @@ elif [[ $MACHINE_ID = orion ]]; then INPES_cpl_atmw_gdas=6; JNPES_cpl_atmw_gdas=8; WPG_cpl_atmw_gdas=24 WAV_tasks_atmw_gdas=248 -elif [[ $MACHINE_ID = hera ]]; then +elif [[ $MACHINE_ID = hera.* ]]; then TPN=40 @@ -133,7 +133,7 @@ elif [[ $MACHINE_ID = hera ]]; then INPES_cpl_atmw_gdas=6; JNPES_cpl_atmw_gdas=8; WPG_cpl_atmw_gdas=24 WAV_tasks_atmw_gdas=248 -elif [[ $MACHINE_ID = linux ]]; then +elif [[ $MACHINE_ID = linux.* ]]; then TPN=40 @@ -152,7 +152,7 @@ elif [[ $MACHINE_ID = linux ]]; then ICE_tasks_cpl_thrd=10 WAV_tasks_cpl_thrd=12 -elif [[ $MACHINE_ID = jet ]]; then +elif [[ $MACHINE_ID = jet.* ]]; then TPN=24 @@ -181,7 +181,7 @@ elif [[ $MACHINE_ID = jet ]]; then WAV_tasks_cpl_c192=80 WLCLK_cpl_c192=120 -elif [[ $MACHINE_ID = s4 ]]; then +elif [[ $MACHINE_ID = s4.* ]]; then TPN=32 @@ -200,7 +200,7 @@ elif [[ $MACHINE_ID = s4 ]]; then ICE_tasks_cpl_bmrk=48 WAV_tasks_cpl_bmrk=80 -elif [[ $MACHINE_ID = gaea ]]; then +elif [[ $MACHINE_ID = gaea.* ]]; then TPN=24 @@ -213,7 +213,7 @@ elif [[ $MACHINE_ID = gaea ]]; then INPES_cpl_atmw_gdas=6; JNPES_cpl_atmw_gdas=8; WPG_cpl_atmw_gdas=24 WAV_tasks_atmw_gdas=264 -elif [[ $MACHINE_ID = cheyenne ]]; then +elif [[ $MACHINE_ID = cheyenne.* ]]; then TPN=36 @@ -256,7 +256,7 @@ elif [[ $MACHINE_ID = cheyenne ]]; then ICE_tasks_cpl_bmrk=48 WAV_tasks_cpl_bmrk=100 -elif [[ $MACHINE_ID = stampede ]]; then +elif [[ $MACHINE_ID = stampede.* ]]; then echo "Unknown MACHINE_ID ${MACHINE_ID}. Please update tasks configurations in default_vars.sh" exit 1 diff --git a/tests/rt.ncar.sh b/tests/rt.ncar.sh index d6c22c4be4..6a588f8788 100755 --- a/tests/rt.ncar.sh +++ b/tests/rt.ncar.sh @@ -219,97 +219,7 @@ done source rt_utils.sh source module-setup.sh -CREATE_BASELINE=false -ROCOTO=false -ECFLOW=false -KEEP_RUNDIR=false -SINGLE_NAME='' -TEST_35D=false -export skip_check_results=false -export delete_rundir=false -SKIP_ORDER=false -RTPWD_NEW_BASELINE=false -TESTS_FILE='rt.conf' -ACCNR=${ACCNR:-""} - -while getopts ":a:cl:mn:dwkreh" opt; do - case $opt in - a) - ACCNR=$OPTARG - ;; - c) - CREATE_BASELINE=true - ;; - l) - TESTS_FILE=$OPTARG - SKIP_ORDER=true - ;; - m) - # redefine RTPWD to point to newly created baseline outputs - RTPWD_NEW_BASELINE=true - ;; - n) - SINGLE_OPTS=("$OPTARG") - until [[ $(eval "echo \${$OPTIND}") =~ ^-.* ]] || [ -z $(eval "echo \${$OPTIND}") ]; do - SINGLE_OPTS+=($(eval "echo \${$OPTIND}")) - OPTIND=$((OPTIND + 1)) - done - - if [[ ${#SINGLE_OPTS[@]} != 2 ]]; then - echo "The -n option needs AND , i.e. -n control_p8 intel" - exit 1 - fi - SINGLE_NAME=${SINGLE_OPTS[0],,} - export RT_COMPILER=${SINGLE_OPTS[1],,} - - if [[ "$RT_COMPILER" == "intel" ]] || [[ "$RT_COMPILER" == "gnu" ]]; then - echo "COMPILER set to ${RT_COMPILER}" - else - echo "Compiler must be either 'intel' or 'gnu'." - exit 1 - fi - ;; - d) - export delete_rundir=true - awk -F "|" '{print $5}' rt.conf | grep "\S" > keep_tests.tmp - ;; - w) - export skip_check_results=true - ;; - k) - KEEP_RUNDIR=true - ;; - r) - ROCOTO=true - ECFLOW=false - ;; - e) - ECFLOW=true - ROCOTO=false - ;; - h) - usage - ;; - \?) - usage - die "Invalid option: -$OPTARG" - ;; - :) - usage - die "Option -$OPTARG requires an argument." - ;; - esac -done - -if [[ -z "$ACCNR" ]]; then - echo "Please use -a to set group account to use on HPC" - exit 1 -fi - -# Display the machine and account using the format detect_machine.sh used: -echo "Machine: " $MACHINE_ID " Account: " $ACCNR - -if [[ $MACHINE_ID = wcoss2 ]]; then +if [[ $MACHINE_ID = wcoss2.* ]]; then #module use /usrx/local/dev/emc_rocoto/modulefiles #module load ruby/2.5.1 rocoto/1.3.0rc2 @@ -336,7 +246,7 @@ if [[ $MACHINE_ID = wcoss2 ]]; then PTMP=/lfs/h2/emc/ptmp SCHEDULER=pbs -elif [[ $MACHINE_ID = acorn ]]; then +elif [[ $MACHINE_ID = acorn.* ]]; then module load ecflow/5.6.0.13 module load gcc/10.3.0 python/3.8.6 @@ -356,7 +266,7 @@ elif [[ $MACHINE_ID = acorn ]]; then PTMP=/lfs/h2/emc/ptmp SCHEDULER=pbs -elif [[ $MACHINE_ID = gaea ]]; then +elif [[ $MACHINE_ID = gaea.* ]]; then export PATH=/lustre/f2/pdata/esrl/gsd/contrib/miniconda3/4.8.3/envs/ufs-weather-model/bin:/lustre/f2/pdata/esrl/gsd/contrib/miniconda3/4.8.3/bin:$PATH export PYTHONPATH=/lustre/f2/pdata/esrl/gsd/contrib/miniconda3/4.8.3/envs/ufs-weather-model/lib/python3.8/site-packages:/lustre/f2/pdata/esrl/gsd/contrib/miniconda3/4.8.3/lib/python3.8/site-packages @@ -372,7 +282,7 @@ elif [[ $MACHINE_ID = gaea ]]; then SCHEDULER=slurm -elif [[ $MACHINE_ID = hera ]]; then +elif [[ $MACHINE_ID = hera.* ]]; then module load rocoto ROCOTORUN=$(which rocotorun) @@ -398,7 +308,7 @@ elif [[ $MACHINE_ID = hera ]]; then SCHEDULER=slurm -elif [[ $MACHINE_ID = orion ]]; then +elif [[ $MACHINE_ID = orion.* ]]; then module load gcc/8.3.0 @@ -421,7 +331,7 @@ elif [[ $MACHINE_ID = orion ]]; then SCHEDULER=slurm -elif [[ $MACHINE_ID = jet ]]; then +elif [[ $MACHINE_ID = jet.* ]]; then module load rocoto/1.3.2 ROCOTORUN=$(which rocotorun) @@ -444,7 +354,7 @@ elif [[ $MACHINE_ID = jet ]]; then SCHEDULER=slurm -elif [[ $MACHINE_ID = s4 ]]; then +elif [[ $MACHINE_ID = s4.* ]]; then module load rocoto/1.3.2 module load ecflow/5.6.0 @@ -468,7 +378,7 @@ elif [[ $MACHINE_ID = s4 ]]; then SCHEDULER=slurm -elif [[ $MACHINE_ID = cheyenne ]]; then +elif [[ $MACHINE_ID = cheyenne.* ]]; then export PATH=/glade/p/ral/jntp/tools/miniconda3/4.8.3/envs/ufs-weather-model/bin:/glade/p/ral/jntp/tools/miniconda3/4.8.3/bin:$PATH export PYTHONPATH=/glade/p/ral/jntp/tools/miniconda3/4.8.3/envs/ufs-weather-model/lib/python3.8/site-packages:/glade/p/ral/jntp/tools/miniconda3/4.8.3/lib/python3.8/site-packages @@ -484,7 +394,7 @@ elif [[ $MACHINE_ID = cheyenne ]]; then PTMP=$dprefix SCHEDULER=pbs -elif [[ $MACHINE_ID = stampede ]]; then +elif [[ $MACHINE_ID = stampede.* ]]; then export PYTHONPATH= ECFLOW_START= @@ -499,7 +409,7 @@ elif [[ $MACHINE_ID = stampede ]]; then MPIEXEC=ibrun MPIEXECOPTS= -elif [[ $MACHINE_ID = expanse ]]; then +elif [[ $MACHINE_ID = expanse.* ]]; then export PYTHONPATH= ECFLOW_START= From b59b926d0d852ed0217bd775accaac517c422e4d Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Fri, 21 Jul 2023 02:53:10 +0000 Subject: [PATCH 11/56] One more fix --- tests/rt.ncar.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/rt.ncar.sh b/tests/rt.ncar.sh index 6a588f8788..73e6209494 100755 --- a/tests/rt.ncar.sh +++ b/tests/rt.ncar.sh @@ -605,19 +605,19 @@ suite ${ECFLOW_SUITE} limit max_jobs ${MAX_JOBS} EOF - if [[ $MACHINE_ID = wcoss2 || $MACHINE_ID = acorn ]]; then + if [[ $MACHINE_ID = wcoss2.* || $MACHINE_ID = acorn.* ]]; then QUEUE=dev - elif [[ $MACHINE_ID = hera ]]; then + elif [[ $MACHINE_ID = hera.* ]]; then QUEUE=batch - elif [[ $MACHINE_ID = orion ]]; then + elif [[ $MACHINE_ID = orion.* ]]; then QUEUE=batch - elif [[ $MACHINE_ID = jet ]]; then + elif [[ $MACHINE_ID = jet.* ]]; then QUEUE=batch - elif [[ $MACHINE_ID = s4 ]]; then + elif [[ $MACHINE_ID = s4.* ]]; then QUEUE=s4 - elif [[ $MACHINE_ID = gaea ]]; then + elif [[ $MACHINE_ID = gaea.* ]]; then QUEUE=normal - elif [[ $MACHINE_ID = cheyenne ]]; then + elif [[ $MACHINE_ID = cheyenne.* ]]; then QUEUE=regular else die "ecFlow is not supported on this machine $MACHINE_ID" @@ -625,7 +625,7 @@ EOF else - if [[ $MACHINE_ID = hera ]] && [[ $HOSTNAME = hecflow ]]; then + if [[ $MACHINE_ID = hera.* ]] && [[ $HOSTNAME = hecflow ]]; then echo "ERROR: To run without using ECFlow on Hera, please do not use ecflow node." exit 1 fi From 40e26b5a3f1233fd8434890e2d3150fd8c4ca548 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Fri, 21 Jul 2023 04:48:28 +0000 Subject: [PATCH 12/56] Updating paths, almost there! --- tests/rt.ncar.sh | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/tests/rt.ncar.sh b/tests/rt.ncar.sh index 73e6209494..12c1f19461 100755 --- a/tests/rt.ncar.sh +++ b/tests/rt.ncar.sh @@ -302,7 +302,7 @@ elif [[ $MACHINE_ID = hera.* ]]; then PARTITION= dprefix=${dprefix:-/scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model} - DISKNM=$dprefix/RT + DISKNM=$dprefix STMP=$dprefix/RT/stmp4 PTMP=$dprefix/RT/stmp2 @@ -467,13 +467,9 @@ fi source bl_date.ncar.conf -if [[ "$RTPWD_NEW_BASELINE" == true ]] ; then - RTPWD=${NEW_BASELINE} -else - RTPWD=${RTPWD:-$DISKNM/NCAR/main-${BL_DATE}} -fi +RTPWD=${RTPWD:-$DISKNM/baselines/main-${BL_DATE}/${RT_COMPILER^^}} -INPUTDATA_ROOT=${INPUTDATA_ROOT:-$DISKNM/NCAR/input-data-20221101} +INPUTDATA_ROOT=${INPUTDATA_ROOT:-$DISKNM/input_data/} INPUTDATA_ROOT_WW3=${INPUTDATA_ROOT}/WW3_input_data_20220624 INPUTDATA_ROOT_BMIC=${INPUTDATA_ROOT_BMIC:-$DISKNM/NCAR/BM_IC-20220207} From 6f2af79028fb46b140bb5df88bb181c1f1915a60 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Fri, 21 Jul 2023 05:41:30 +0000 Subject: [PATCH 13/56] things are compiling and starting to run. Just need to get them to finish running... --- tests/rt.ncar.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/rt.ncar.sh b/tests/rt.ncar.sh index 12c1f19461..b3f3d8d33b 100755 --- a/tests/rt.ncar.sh +++ b/tests/rt.ncar.sh @@ -302,7 +302,7 @@ elif [[ $MACHINE_ID = hera.* ]]; then PARTITION= dprefix=${dprefix:-/scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model} - DISKNM=$dprefix + DISKNM=$dprefix/RT STMP=$dprefix/RT/stmp4 PTMP=$dprefix/RT/stmp2 @@ -469,7 +469,7 @@ source bl_date.ncar.conf RTPWD=${RTPWD:-$DISKNM/baselines/main-${BL_DATE}/${RT_COMPILER^^}} -INPUTDATA_ROOT=${INPUTDATA_ROOT:-$DISKNM/input_data/} +INPUTDATA_ROOT=${INPUTDATA_ROOT:-$DISKNM/../input_data/} INPUTDATA_ROOT_WW3=${INPUTDATA_ROOT}/WW3_input_data_20220624 INPUTDATA_ROOT_BMIC=${INPUTDATA_ROOT_BMIC:-$DISKNM/NCAR/BM_IC-20220207} From f9622de29c42beaf18ff86aa628cc3027744dcfb Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Fri, 21 Jul 2023 06:16:57 +0000 Subject: [PATCH 14/56] This hack should get it working. A better solution will be to pass the input data directory as an argument --- tests/rt.ncar.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/rt.ncar.sh b/tests/rt.ncar.sh index b3f3d8d33b..c0aaa7bb98 100755 --- a/tests/rt.ncar.sh +++ b/tests/rt.ncar.sh @@ -469,7 +469,7 @@ source bl_date.ncar.conf RTPWD=${RTPWD:-$DISKNM/baselines/main-${BL_DATE}/${RT_COMPILER^^}} -INPUTDATA_ROOT=${INPUTDATA_ROOT:-$DISKNM/../input_data/} +INPUTDATA_ROOT=${INPUTDATA_ROOT:-$DISKNM/../../input_data/${BL_DATE}} INPUTDATA_ROOT_WW3=${INPUTDATA_ROOT}/WW3_input_data_20220624 INPUTDATA_ROOT_BMIC=${INPUTDATA_ROOT_BMIC:-$DISKNM/NCAR/BM_IC-20220207} From 2fabe8884e3d5bc9835213531c3121dc84497b27 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Fri, 21 Jul 2023 06:50:54 +0000 Subject: [PATCH 15/56] Incorrect input data and baseline directories --- tests/rt.ncar.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/rt.ncar.sh b/tests/rt.ncar.sh index c0aaa7bb98..9f81c19b14 100755 --- a/tests/rt.ncar.sh +++ b/tests/rt.ncar.sh @@ -466,8 +466,9 @@ if [[ $TESTS_FILE =~ '35d' ]] || [[ $TESTS_FILE =~ 'weekly' ]]; then fi source bl_date.ncar.conf +INPUT_DATE=20230314 -RTPWD=${RTPWD:-$DISKNM/baselines/main-${BL_DATE}/${RT_COMPILER^^}} +RTPWD=${RTPWD:-$DISKNM/../../baselines/main-${BL_DATE}/${RT_COMPILER^^}} INPUTDATA_ROOT=${INPUTDATA_ROOT:-$DISKNM/../../input_data/${BL_DATE}} INPUTDATA_ROOT_WW3=${INPUTDATA_ROOT}/WW3_input_data_20220624 From 4c9070414172236655ca16ef73c70dda77b4ee97 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Fri, 21 Jul 2023 07:29:27 +0000 Subject: [PATCH 16/56] Last bad directory (HAH) --- tests/rt.ncar.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/rt.ncar.sh b/tests/rt.ncar.sh index 9f81c19b14..6466e89c74 100755 --- a/tests/rt.ncar.sh +++ b/tests/rt.ncar.sh @@ -470,7 +470,7 @@ INPUT_DATE=20230314 RTPWD=${RTPWD:-$DISKNM/../../baselines/main-${BL_DATE}/${RT_COMPILER^^}} -INPUTDATA_ROOT=${INPUTDATA_ROOT:-$DISKNM/../../input_data/${BL_DATE}} +INPUTDATA_ROOT=${INPUTDATA_ROOT:-$DISKNM/../../input_data/${INPUT_DATE}} INPUTDATA_ROOT_WW3=${INPUTDATA_ROOT}/WW3_input_data_20220624 INPUTDATA_ROOT_BMIC=${INPUTDATA_ROOT_BMIC:-$DISKNM/NCAR/BM_IC-20220207} From 67f9976b7649b55e33cf46f05819253687e21b1e Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Fri, 21 Jul 2023 14:41:54 +0000 Subject: [PATCH 17/56] Add back label-remover, testing appears to work on Hera! --- tests/auto/rt_auto.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/auto/rt_auto.py b/tests/auto/rt_auto.py index 2aceab9506..6fa7c7c87a 100644 --- a/tests/auto/rt_auto.py +++ b/tests/auto/rt_auto.py @@ -228,8 +228,8 @@ def run(self): self.comment_text_append(f'Job: {self.preq_dict["action"]}') if self.check_label_before_job_start(): try: -# logger.info('Calling remove_pr_label') -# self.remove_pr_label() + logger.info('Calling remove_pr_label') + self.remove_pr_label() logger.info('Calling Job to Run') self.job_mod.run(self) except Exception: From e5e079c11e0ea746d60daa035b47e823cc8e339a Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Fri, 21 Jul 2023 14:29:51 -0600 Subject: [PATCH 18/56] Update paths for Cheyenne --- tests/rt.ncar.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/rt.ncar.sh b/tests/rt.ncar.sh index 6466e89c74..ff705193a2 100755 --- a/tests/rt.ncar.sh +++ b/tests/rt.ncar.sh @@ -388,8 +388,8 @@ elif [[ $MACHINE_ID = cheyenne.* ]]; then QUEUE=regular COMPILE_QUEUE=regular PARTITION= - dprefix=/glade/scratch - DISKNM=/glade/scratch/epicufsrt/GMTB/ufs-weather-model/RT + dprefix=${dprefix:-/scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model} + DISKNM=$dprefix STMP=$dprefix PTMP=$dprefix SCHEDULER=pbs From fcc3a5b755c754368eea3b50f467ba03abc3910a Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Mon, 24 Jul 2023 18:22:01 -0600 Subject: [PATCH 19/56] Need to purge module environment, my default environment has conda which makes the python load fail --- tests/rt.ncar.sh | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/rt.ncar.sh b/tests/rt.ncar.sh index ff705193a2..14db56548f 100755 --- a/tests/rt.ncar.sh +++ b/tests/rt.ncar.sh @@ -15,7 +15,7 @@ usage() { echo " -c create new baseline results" echo " -e use ecFlow workflow manager" echo " -h display this help" - echo " -k keep run directory after rt.sh is completed" + echo " -k keep run directory after $0 is completed" echo " -l runs test specified in " echo " -m compare against new baseline results" echo " -n run single test " @@ -104,11 +104,11 @@ cleanup() { exit } -trap '{ echo "rt.sh interrupted"; rt_trap ; }' INT -trap '{ echo "rt.sh quit"; rt_trap ; }' QUIT -trap '{ echo "rt.sh terminated"; rt_trap ; }' TERM -trap '{ echo "rt.sh error on line $LINENO"; cleanup ; }' ERR -trap '{ echo "rt.sh finished"; cleanup ; }' EXIT +trap '{ echo "$0 interrupted"; rt_trap ; }' INT +trap '{ echo "$0 quit"; rt_trap ; }' QUIT +trap '{ echo "$0 terminated"; rt_trap ; }' TERM +trap '{ echo "$0 error on line $LINENO"; cleanup ; }' ERR +trap '{ echo "$0 finished"; cleanup ; }' EXIT # PATHRT - Path to regression tests directory readonly PATHRT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd -P )" @@ -122,7 +122,7 @@ readonly LOCKDIR="${PATHRT}"/lock if mkdir "${LOCKDIR}" ; then echo $(hostname) $$ > "${LOCKDIR}/PID" else - echo "Only one instance of rt.sh can be running at a time" + echo "Only one instance of $0 can be running at a time" exit 1 fi @@ -380,6 +380,7 @@ elif [[ $MACHINE_ID = s4.* ]]; then elif [[ $MACHINE_ID = cheyenne.* ]]; then + module purge export PATH=/glade/p/ral/jntp/tools/miniconda3/4.8.3/envs/ufs-weather-model/bin:/glade/p/ral/jntp/tools/miniconda3/4.8.3/bin:$PATH export PYTHONPATH=/glade/p/ral/jntp/tools/miniconda3/4.8.3/envs/ufs-weather-model/lib/python3.8/site-packages:/glade/p/ral/jntp/tools/miniconda3/4.8.3/lib/python3.8/site-packages ECFLOW_START=/glade/p/ral/jntp/tools/miniconda3/4.8.3/envs/ufs-weather-model/bin/ecflow_start.sh From 2ea384019519566235e2e326c08c9139c24202b6 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Wed, 23 Aug 2023 10:54:52 -0600 Subject: [PATCH 20/56] Put module purge in wrong place; need it prior to building. --- tests/compile.sh | 1 + tests/rt.ncar.sh | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/compile.sh b/tests/compile.sh index 8702efe135..17a5199568 100755 --- a/tests/compile.sh +++ b/tests/compile.sh @@ -62,6 +62,7 @@ else source /lustre/f2/dev/role.epic/contrib/Lmod_init.sh fi # Load fv3 module + module purge module use $PATHTR/modulefiles modulefile="ufs_${MACHINE_ID}.${RT_COMPILER}" module load $modulefile diff --git a/tests/rt.ncar.sh b/tests/rt.ncar.sh index 14db56548f..680be25ec1 100755 --- a/tests/rt.ncar.sh +++ b/tests/rt.ncar.sh @@ -380,7 +380,6 @@ elif [[ $MACHINE_ID = s4.* ]]; then elif [[ $MACHINE_ID = cheyenne.* ]]; then - module purge export PATH=/glade/p/ral/jntp/tools/miniconda3/4.8.3/envs/ufs-weather-model/bin:/glade/p/ral/jntp/tools/miniconda3/4.8.3/bin:$PATH export PYTHONPATH=/glade/p/ral/jntp/tools/miniconda3/4.8.3/envs/ufs-weather-model/lib/python3.8/site-packages:/glade/p/ral/jntp/tools/miniconda3/4.8.3/lib/python3.8/site-packages ECFLOW_START=/glade/p/ral/jntp/tools/miniconda3/4.8.3/envs/ufs-weather-model/bin/ecflow_start.sh From 2c59a022ff2e2a382075ee1e558a83976c5445b1 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Wed, 23 Aug 2023 16:45:19 -0600 Subject: [PATCH 21/56] Final fixes to Cheyenne build; now working! - Correct paths for Cheyenne data - Move "input date" variable to platform section --- tests/rt.ncar.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/rt.ncar.sh b/tests/rt.ncar.sh index 680be25ec1..60bc6a9eed 100755 --- a/tests/rt.ncar.sh +++ b/tests/rt.ncar.sh @@ -301,6 +301,7 @@ elif [[ $MACHINE_ID = hera.* ]]; then COMPILE_QUEUE=batch PARTITION= + INPUT_DATE=20230314 dprefix=${dprefix:-/scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model} DISKNM=$dprefix/RT STMP=$dprefix/RT/stmp4 @@ -388,10 +389,11 @@ elif [[ $MACHINE_ID = cheyenne.* ]]; then QUEUE=regular COMPILE_QUEUE=regular PARTITION= - dprefix=${dprefix:-/scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model} - DISKNM=$dprefix - STMP=$dprefix - PTMP=$dprefix + INPUT_DATE=20221101 + dprefix=${dprefix:-/glade/p/ral/jntp/CCPP_regression_testing/NCAR_ufs-weather-model/} + DISKNM=$dprefix/RT + STMP=$dprefix/RT + PTMP=$dprefix/RT SCHEDULER=pbs elif [[ $MACHINE_ID = stampede.* ]]; then From 55a36496494dfbf8fc5a91062aa7367aee662ec7 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Thu, 24 Aug 2023 16:08:09 -0600 Subject: [PATCH 22/56] Fix stuff that got messed up in rebase --- tests/default_vars.sh | 24 +++++++++--------------- tests/rt.ncar.sh | 22 +++++++++++----------- 2 files changed, 20 insertions(+), 26 deletions(-) diff --git a/tests/default_vars.sh b/tests/default_vars.sh index 5ab6568125..f57dcc00ea 100755 --- a/tests/default_vars.sh +++ b/tests/default_vars.sh @@ -94,7 +94,7 @@ ocn_omp_num_threads=1 wav_omp_num_threads=1 -if [[ $MACHINE_ID = wcoss2.* || $MACHINE_ID = acorn.* ]]; then +if [[ $MACHINE_ID = wcoss2 || $MACHINE_ID = acorn ]]; then TPN=128 @@ -107,7 +107,7 @@ if [[ $MACHINE_ID = wcoss2.* || $MACHINE_ID = acorn.* ]]; then INPES_cpl_atmw_gdas=6; JNPES_cpl_atmw_gdas=8; WPG_cpl_atmw_gdas=24 WAV_tasks_atmw_gdas=248 -elif [[ $MACHINE_ID = orion.* ]]; then +elif [[ $MACHINE_ID = orion ]]; then TPN=40 @@ -120,7 +120,7 @@ elif [[ $MACHINE_ID = orion.* ]]; then INPES_cpl_atmw_gdas=6; JNPES_cpl_atmw_gdas=8; WPG_cpl_atmw_gdas=24 WAV_tasks_atmw_gdas=248 -elif [[ $MACHINE_ID = hera.* ]]; then +elif [[ $MACHINE_ID = hera ]]; then TPN=40 @@ -133,7 +133,7 @@ elif [[ $MACHINE_ID = hera.* ]]; then INPES_cpl_atmw_gdas=6; JNPES_cpl_atmw_gdas=8; WPG_cpl_atmw_gdas=24 WAV_tasks_atmw_gdas=248 -elif [[ $MACHINE_ID = linux.* ]]; then +elif [[ $MACHINE_ID = linux ]]; then TPN=40 @@ -152,7 +152,7 @@ elif [[ $MACHINE_ID = linux.* ]]; then ICE_tasks_cpl_thrd=10 WAV_tasks_cpl_thrd=12 -elif [[ $MACHINE_ID = jet.* ]]; then +elif [[ $MACHINE_ID = jet ]]; then TPN=24 @@ -181,7 +181,7 @@ elif [[ $MACHINE_ID = jet.* ]]; then WAV_tasks_cpl_c192=80 WLCLK_cpl_c192=120 -elif [[ $MACHINE_ID = s4.* ]]; then +elif [[ $MACHINE_ID = s4 ]]; then TPN=32 @@ -200,7 +200,7 @@ elif [[ $MACHINE_ID = s4.* ]]; then ICE_tasks_cpl_bmrk=48 WAV_tasks_cpl_bmrk=80 -elif [[ $MACHINE_ID = gaea.* ]]; then +elif [[ $MACHINE_ID = gaea ]]; then TPN=24 @@ -213,7 +213,7 @@ elif [[ $MACHINE_ID = gaea.* ]]; then INPES_cpl_atmw_gdas=6; JNPES_cpl_atmw_gdas=8; WPG_cpl_atmw_gdas=24 WAV_tasks_atmw_gdas=264 -elif [[ $MACHINE_ID = cheyenne.* ]]; then +elif [[ $MACHINE_ID = cheyenne ]]; then TPN=36 @@ -256,10 +256,7 @@ elif [[ $MACHINE_ID = cheyenne.* ]]; then ICE_tasks_cpl_bmrk=48 WAV_tasks_cpl_bmrk=100 -elif [[ $MACHINE_ID = stampede.* ]]; then - - echo "Unknown MACHINE_ID ${MACHINE_ID}. Please update tasks configurations in default_vars.sh" - exit 1 +elif [[ $MACHINE_ID = stampede ]]; then TPN_dflt=48 ; INPES_dflt=3 ; JNPES_dflt=8 TPN_thrd=24 ; INPES_thrd=3 ; JNPES_thrd=4 @@ -300,9 +297,6 @@ elif [[ ${MACHINE_ID} = noaacloud ]] ; then elif [[ $MACHINE_ID = expanse ]]; then - echo "Unknown MACHINE_ID ${MACHINE_ID}. Please update tasks configurations in default_vars.sh" - exit 1 - TPN_dflt=64 ; INPES_dflt=3 ; JNPES_dflt=8 TPN_thrd=64 ; INPES_thrd=3 ; JNPES_thrd=4 TPN_stretch=12 ; INPES_stretch=2 ; JNPES_stretch=4 diff --git a/tests/rt.ncar.sh b/tests/rt.ncar.sh index 60bc6a9eed..9c1fe9726b 100755 --- a/tests/rt.ncar.sh +++ b/tests/rt.ncar.sh @@ -219,7 +219,7 @@ done source rt_utils.sh source module-setup.sh -if [[ $MACHINE_ID = wcoss2.* ]]; then +if [[ $MACHINE_ID = wcoss2 ]]; then #module use /usrx/local/dev/emc_rocoto/modulefiles #module load ruby/2.5.1 rocoto/1.3.0rc2 @@ -246,7 +246,7 @@ if [[ $MACHINE_ID = wcoss2.* ]]; then PTMP=/lfs/h2/emc/ptmp SCHEDULER=pbs -elif [[ $MACHINE_ID = acorn.* ]]; then +elif [[ $MACHINE_ID = acorn ]]; then module load ecflow/5.6.0.13 module load gcc/10.3.0 python/3.8.6 @@ -266,7 +266,7 @@ elif [[ $MACHINE_ID = acorn.* ]]; then PTMP=/lfs/h2/emc/ptmp SCHEDULER=pbs -elif [[ $MACHINE_ID = gaea.* ]]; then +elif [[ $MACHINE_ID = gaea ]]; then export PATH=/lustre/f2/pdata/esrl/gsd/contrib/miniconda3/4.8.3/envs/ufs-weather-model/bin:/lustre/f2/pdata/esrl/gsd/contrib/miniconda3/4.8.3/bin:$PATH export PYTHONPATH=/lustre/f2/pdata/esrl/gsd/contrib/miniconda3/4.8.3/envs/ufs-weather-model/lib/python3.8/site-packages:/lustre/f2/pdata/esrl/gsd/contrib/miniconda3/4.8.3/lib/python3.8/site-packages @@ -282,7 +282,7 @@ elif [[ $MACHINE_ID = gaea.* ]]; then SCHEDULER=slurm -elif [[ $MACHINE_ID = hera.* ]]; then +elif [[ $MACHINE_ID = hera ]]; then module load rocoto ROCOTORUN=$(which rocotorun) @@ -309,7 +309,7 @@ elif [[ $MACHINE_ID = hera.* ]]; then SCHEDULER=slurm -elif [[ $MACHINE_ID = orion.* ]]; then +elif [[ $MACHINE_ID = orion ]]; then module load gcc/8.3.0 @@ -332,7 +332,7 @@ elif [[ $MACHINE_ID = orion.* ]]; then SCHEDULER=slurm -elif [[ $MACHINE_ID = jet.* ]]; then +elif [[ $MACHINE_ID = jet ]]; then module load rocoto/1.3.2 ROCOTORUN=$(which rocotorun) @@ -355,7 +355,7 @@ elif [[ $MACHINE_ID = jet.* ]]; then SCHEDULER=slurm -elif [[ $MACHINE_ID = s4.* ]]; then +elif [[ $MACHINE_ID = s4 ]]; then module load rocoto/1.3.2 module load ecflow/5.6.0 @@ -379,7 +379,7 @@ elif [[ $MACHINE_ID = s4.* ]]; then SCHEDULER=slurm -elif [[ $MACHINE_ID = cheyenne.* ]]; then +elif [[ $MACHINE_ID = cheyenne ]]; then export PATH=/glade/p/ral/jntp/tools/miniconda3/4.8.3/envs/ufs-weather-model/bin:/glade/p/ral/jntp/tools/miniconda3/4.8.3/bin:$PATH export PYTHONPATH=/glade/p/ral/jntp/tools/miniconda3/4.8.3/envs/ufs-weather-model/lib/python3.8/site-packages:/glade/p/ral/jntp/tools/miniconda3/4.8.3/lib/python3.8/site-packages @@ -396,7 +396,7 @@ elif [[ $MACHINE_ID = cheyenne.* ]]; then PTMP=$dprefix/RT SCHEDULER=pbs -elif [[ $MACHINE_ID = stampede.* ]]; then +elif [[ $MACHINE_ID = stampede ]]; then export PYTHONPATH= ECFLOW_START= @@ -411,7 +411,7 @@ elif [[ $MACHINE_ID = stampede.* ]]; then MPIEXEC=ibrun MPIEXECOPTS= -elif [[ $MACHINE_ID = expanse.* ]]; then +elif [[ $MACHINE_ID = expanse ]]; then export PYTHONPATH= ECFLOW_START= @@ -424,7 +424,7 @@ elif [[ $MACHINE_ID = expanse.* ]]; then PTMP=$dprefix SCHEDULER=slurm -elif [[ $MACHINE_ID = noaacloud.* ]]; then +elif [[ $MACHINE_ID = noaacloud ]]; then module use /apps/modules/modulefiles module load rocoto/1.3.3 From b4dc8dd8def4af0dc1278e1911d7379941135a38 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Thu, 24 Aug 2023 19:48:37 -0600 Subject: [PATCH 23/56] Need to specify account, and running rocoto vs ecflow depends on platform --- tests/auto/jobs/bl.py | 4 ++-- tests/auto/jobs/rt.py | 7 +++++-- tests/auto/rt_auto.py | 13 ++++++++----- tests/auto/start_rt_auto.sh | 10 +++++----- 4 files changed, 20 insertions(+), 14 deletions(-) diff --git a/tests/auto/jobs/bl.py b/tests/auto/jobs/bl.py index e25520e66a..077c6418cc 100644 --- a/tests/auto/jobs/bl.py +++ b/tests/auto/jobs/bl.py @@ -89,11 +89,11 @@ def run_regression_test(job_obj, pr_repo_loc): logger = logging.getLogger('BL/RUN_REGRESSION_TEST') if job_obj.compiler == 'gnu': rt_command = [[f'export RT_COMPILER="{job_obj.compiler}" && cd tests ' - '&& /bin/bash --login ./rt.ncar.sh -e -c -l rt_gnu.conf -p {job_obj.machine}', + '&& /bin/bash --login ./rt.ncar.sh -e -c -l rt_gnu.conf -p {job_obj.machine} -k', pr_repo_loc]] elif job_obj.compiler == 'intel': rt_command = [[f'export RT_COMPILER="{job_obj.compiler}" && cd tests ' - '&& /bin/bash --login ./rt.ncar.sh -e -c -p {job_obj.machine}', pr_repo_loc]] + '&& /bin/bash --login ./rt.ncar.sh -e -c -p {job_obj.machine} -k', pr_repo_loc]] job_obj.run_commands(logger, rt_command) diff --git a/tests/auto/jobs/rt.py b/tests/auto/jobs/rt.py index f0d6e1517b..8f76005f76 100644 --- a/tests/auto/jobs/rt.py +++ b/tests/auto/jobs/rt.py @@ -13,13 +13,16 @@ def run(job_obj): def run_regression_test(job_obj, pr_repo_loc): logger = logging.getLogger('RT/RUN_REGRESSION_TEST') + workflow_flag='-e' + if job_obj.machine == 'cheyenne': + workflow_flag='-r' if job_obj.compiler == 'gnu': rt_command = [[f'export RT_COMPILER="{job_obj.compiler}" && cd tests ' - f'&& /bin/bash --login ./rt.ncar.sh -e -l rt_gnu.conf -p {job_obj.machine} -w {job_obj.workdir}', + f'&& /bin/bash --login ./rt.ncar.sh {workflow_flag} -l rt_gnu.conf -a {job_obj.account} -p {job_obj.machine} -w {job_obj.workdir}', pr_repo_loc]] elif job_obj.compiler == 'intel': rt_command = [[f'export RT_COMPILER="{job_obj.compiler}" && cd tests ' - f'&& /bin/bash --login ./rt.ncar.sh -e -p {job_obj.machine} -w {job_obj.workdir}', pr_repo_loc]] + f'&& /bin/bash --login ./rt.ncar.sh {workflow_flag} -a {job_obj.account} -p {job_obj.machine} -w {job_obj.workdir} -n control_p8 intel -k', pr_repo_loc]] job_obj.run_commands(logger, rt_command) diff --git a/tests/auto/rt_auto.py b/tests/auto/rt_auto.py index 6fa7c7c87a..757739722e 100644 --- a/tests/auto/rt_auto.py +++ b/tests/auto/rt_auto.py @@ -123,7 +123,7 @@ def delete_rt_dirs(in_dir, machine, workdir): logging.debug(f'{rt_dir}/{match} does not exist, not attempting to remove') -def get_preqs_with_actions(repos, machine, workdir, ghinterface_obj, actions): +def get_preqs_with_actions(repos, machine, account, workdir, ghinterface_obj, actions): ''' Create list of dictionaries of a pull request and its machine label and action ''' logger = logging.getLogger('GET_PREQS_WITH_ACTIONS') @@ -144,7 +144,7 @@ def get_preqs_with_actions(repos, machine, workdir, ghinterface_obj, actions): if match: pr_label['action'] = match # return_preq.append(pr_label.copy()) - jobs.append(Job(pr_label.copy(), ghinterface_obj, machine, compiler, workdir)) + jobs.append(Job(pr_label.copy(), ghinterface_obj, machine, account, compiler, workdir)) return jobs @@ -166,13 +166,14 @@ class Job: provided by the bash script ''' - def __init__(self, preq_dict, ghinterface_obj, machine, compiler, workdir): + def __init__(self, preq_dict, ghinterface_obj, machine, account, compiler, workdir): self.logger = logging.getLogger('JOB') self.preq_dict = preq_dict self.job_mod = importlib.import_module( f'jobs.{self.preq_dict["action"].lower()}') self.ghinterface_obj = ghinterface_obj self.machine = machine + self.account = account self.compiler = compiler self.workdir = workdir self.comment_text = '***Automated RT Failure Notification***\n' @@ -184,7 +185,7 @@ def comment_text_append(self, newtext): def remove_pr_label(self): ''' Removes the PR label that initiated the job run from PR ''' self.logger.info(f'Removing Label: {self.preq_dict["label"]}') - self.preq_dict['preq'].remove_from_labels(self.preq_dict['label']) +# self.preq_dict['preq'].remove_from_labels(self.preq_dict['label']) def check_label_before_job_start(self): # LETS Check the label still exists before the start of the job in the @@ -284,12 +285,14 @@ def main(): parser = argparse.ArgumentParser() parser.add_argument('-m','--machine', help='current machine name', required=True) + parser.add_argument('-a','--account', help='account to charge', required=True) parser.add_argument('-w','--workdir', help='directory where tests will be staged and run', required=False) parser.add_argument('-d','--debug', help='Set logging to more verbose output', action='store_true') args = parser.parse_args() machine = args.machine + account = args.account workdir = args.workdir if args.debug: @@ -312,7 +315,7 @@ def main(): # and turn them into Job objects logger.info('Getting all pull requests, ' 'labels and actions applicable to this machine.') - jobs = get_preqs_with_actions(repos, machine, workdir, + jobs = get_preqs_with_actions(repos, machine, account, workdir, ghinterface_obj, actions) [job.run() for job in jobs] diff --git a/tests/auto/start_rt_auto.sh b/tests/auto/start_rt_auto.sh index f3f0482480..cdb7aa9158 100755 --- a/tests/auto/start_rt_auto.sh +++ b/tests/auto/start_rt_auto.sh @@ -8,13 +8,14 @@ function usage { echo " workdir: Working directory for checking out and running tests" exit 1 } -if [ "$#" -lt 2 ]; then - echo "Need to provide machine and workdir" +if [ "$#" -lt 3 ]; then + echo "Need to provide machine, account, and workdir" usage fi machine=$1 -workdir=$2 +account=$2 +workdir=$3 if [[ $machine == hera ]]; then export PATH=/scratch1/NCEPDEV/nems/emc.nemspara/soft/miniconda3/bin:$PATH @@ -36,6 +37,5 @@ else exit 1 fi -python rt_auto.py -m=$machine -w=$workdir -d +python rt_auto.py -m=$machine -w=$workdir -a=$account -d -exit 0 From 494c5d50cf3d45c685f56bb81622069482c6c952 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Thu, 24 Aug 2023 20:09:31 -0600 Subject: [PATCH 24/56] Apparently Cheyenne *does* support ECFlow, I just missed a different error --- tests/auto/jobs/rt.py | 6 ++---- tests/rt.ncar.sh | 14 +++++++------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/tests/auto/jobs/rt.py b/tests/auto/jobs/rt.py index 8f76005f76..d6c8b51dfd 100644 --- a/tests/auto/jobs/rt.py +++ b/tests/auto/jobs/rt.py @@ -14,15 +14,13 @@ def run(job_obj): def run_regression_test(job_obj, pr_repo_loc): logger = logging.getLogger('RT/RUN_REGRESSION_TEST') workflow_flag='-e' - if job_obj.machine == 'cheyenne': - workflow_flag='-r' if job_obj.compiler == 'gnu': rt_command = [[f'export RT_COMPILER="{job_obj.compiler}" && cd tests ' - f'&& /bin/bash --login ./rt.ncar.sh {workflow_flag} -l rt_gnu.conf -a {job_obj.account} -p {job_obj.machine} -w {job_obj.workdir}', + f'&& /bin/bash --login ./rt.ncar.sh -e -l rt_gnu.conf -a {job_obj.account} -p {job_obj.machine} -w {job_obj.workdir}', pr_repo_loc]] elif job_obj.compiler == 'intel': rt_command = [[f'export RT_COMPILER="{job_obj.compiler}" && cd tests ' - f'&& /bin/bash --login ./rt.ncar.sh {workflow_flag} -a {job_obj.account} -p {job_obj.machine} -w {job_obj.workdir} -n control_p8 intel -k', pr_repo_loc]] + f'&& /bin/bash --login ./rt.ncar.sh -e -a {job_obj.account} -p {job_obj.machine} -w {job_obj.workdir} -n control_p8 intel -k', pr_repo_loc]] job_obj.run_commands(logger, rt_command) diff --git a/tests/rt.ncar.sh b/tests/rt.ncar.sh index 9c1fe9726b..1e54f30e91 100755 --- a/tests/rt.ncar.sh +++ b/tests/rt.ncar.sh @@ -604,19 +604,19 @@ suite ${ECFLOW_SUITE} limit max_jobs ${MAX_JOBS} EOF - if [[ $MACHINE_ID = wcoss2.* || $MACHINE_ID = acorn.* ]]; then + if [[ $MACHINE_ID = wcoss2 || $MACHINE_ID = acorn ]]; then QUEUE=dev - elif [[ $MACHINE_ID = hera.* ]]; then + elif [[ $MACHINE_ID = hera ]]; then QUEUE=batch - elif [[ $MACHINE_ID = orion.* ]]; then + elif [[ $MACHINE_ID = orion ]]; then QUEUE=batch - elif [[ $MACHINE_ID = jet.* ]]; then + elif [[ $MACHINE_ID = jet ]]; then QUEUE=batch - elif [[ $MACHINE_ID = s4.* ]]; then + elif [[ $MACHINE_ID = s4 ]]; then QUEUE=s4 - elif [[ $MACHINE_ID = gaea.* ]]; then + elif [[ $MACHINE_ID = gaea ]]; then QUEUE=normal - elif [[ $MACHINE_ID = cheyenne.* ]]; then + elif [[ $MACHINE_ID = cheyenne ]]; then QUEUE=regular else die "ecFlow is not supported on this machine $MACHINE_ID" From c36a056dc13982758bf2087042cb4f3f8153919a Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Fri, 25 Aug 2023 15:41:13 -0600 Subject: [PATCH 25/56] Provide account as command line argument --- tests/auto/jobs/bl.py | 4 ++-- tests/auto/jobs/rt.py | 2 +- tests/auto/rt_auto.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/auto/jobs/bl.py b/tests/auto/jobs/bl.py index 077c6418cc..8b74af264c 100644 --- a/tests/auto/jobs/bl.py +++ b/tests/auto/jobs/bl.py @@ -89,11 +89,11 @@ def run_regression_test(job_obj, pr_repo_loc): logger = logging.getLogger('BL/RUN_REGRESSION_TEST') if job_obj.compiler == 'gnu': rt_command = [[f'export RT_COMPILER="{job_obj.compiler}" && cd tests ' - '&& /bin/bash --login ./rt.ncar.sh -e -c -l rt_gnu.conf -p {job_obj.machine} -k', + '&& /bin/bash --login ./rt.ncar.sh -e -a {job_obj.account} -c -l rt_gnu.conf -p {job_obj.machine} -w {job_obj.workdir} -k', pr_repo_loc]] elif job_obj.compiler == 'intel': rt_command = [[f'export RT_COMPILER="{job_obj.compiler}" && cd tests ' - '&& /bin/bash --login ./rt.ncar.sh -e -c -p {job_obj.machine} -k', pr_repo_loc]] + '&& /bin/bash --login ./rt.ncar.sh -e -a {job_obj.account} -c -p {job_obj.machine} -w {job_obj.workdir} -k', pr_repo_loc]] job_obj.run_commands(logger, rt_command) diff --git a/tests/auto/jobs/rt.py b/tests/auto/jobs/rt.py index d6c8b51dfd..d7faa20694 100644 --- a/tests/auto/jobs/rt.py +++ b/tests/auto/jobs/rt.py @@ -20,7 +20,7 @@ def run_regression_test(job_obj, pr_repo_loc): pr_repo_loc]] elif job_obj.compiler == 'intel': rt_command = [[f'export RT_COMPILER="{job_obj.compiler}" && cd tests ' - f'&& /bin/bash --login ./rt.ncar.sh -e -a {job_obj.account} -p {job_obj.machine} -w {job_obj.workdir} -n control_p8 intel -k', pr_repo_loc]] + f'&& /bin/bash --login ./rt.ncar.sh -e -a {job_obj.account} -p {job_obj.machine} -w {job_obj.workdir}', pr_repo_loc]] job_obj.run_commands(logger, rt_command) diff --git a/tests/auto/rt_auto.py b/tests/auto/rt_auto.py index 757739722e..b427155ba9 100644 --- a/tests/auto/rt_auto.py +++ b/tests/auto/rt_auto.py @@ -185,7 +185,7 @@ def comment_text_append(self, newtext): def remove_pr_label(self): ''' Removes the PR label that initiated the job run from PR ''' self.logger.info(f'Removing Label: {self.preq_dict["label"]}') -# self.preq_dict['preq'].remove_from_labels(self.preq_dict['label']) + self.preq_dict['preq'].remove_from_labels(self.preq_dict['label']) def check_label_before_job_start(self): # LETS Check the label still exists before the start of the job in the From 231434d0c15b1d5c9e97c4ad85e1ddd669bff682 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Wed, 13 Sep 2023 10:59:28 -0600 Subject: [PATCH 26/56] Going rogue here: removing ncar-specific script and going straight for rt.sh --- tests/auto/jobs/bl.py | 8 +- tests/auto/jobs/rt.py | 4 +- tests/rt.ncar.sh | 878 ------------------------------------------ tests/rt.sh | 119 +++++- 4 files changed, 111 insertions(+), 898 deletions(-) delete mode 100755 tests/rt.ncar.sh diff --git a/tests/auto/jobs/bl.py b/tests/auto/jobs/bl.py index 8b74af264c..8afff5758b 100644 --- a/tests/auto/jobs/bl.py +++ b/tests/auto/jobs/bl.py @@ -89,11 +89,11 @@ def run_regression_test(job_obj, pr_repo_loc): logger = logging.getLogger('BL/RUN_REGRESSION_TEST') if job_obj.compiler == 'gnu': rt_command = [[f'export RT_COMPILER="{job_obj.compiler}" && cd tests ' - '&& /bin/bash --login ./rt.ncar.sh -e -a {job_obj.account} -c -l rt_gnu.conf -p {job_obj.machine} -w {job_obj.workdir} -k', + '&& /bin/bash --login ./rt.sh -e -a {job_obj.account} -c -l rt_gnu.conf -p {job_obj.machine} -w {job_obj.workdir} -k', pr_repo_loc]] elif job_obj.compiler == 'intel': rt_command = [[f'export RT_COMPILER="{job_obj.compiler}" && cd tests ' - '&& /bin/bash --login ./rt.ncar.sh -e -a {job_obj.account} -c -p {job_obj.machine} -w {job_obj.workdir} -k', pr_repo_loc]] + '&& /bin/bash --login ./rt.sh -e -a {job_obj.account} -c -p {job_obj.machine} -w {job_obj.workdir} -k', pr_repo_loc]] job_obj.run_commands(logger, rt_command) @@ -158,7 +158,7 @@ def post_process(job_obj, pr_repo_loc, repo_dir_str, newbldir, bldir, bldate, bl def get_bl_date(job_obj, pr_repo_loc): logger = logging.getLogger('BL/UPDATE_RT_NCAR_SH') BLDATEFOUND = False - with open(f'{pr_repo_loc}/tests/rt.ncar.sh', 'r') as f: + with open(f'{pr_repo_loc}/tests/rt.sh', 'r') as f: for line in f: if 'BL_DATE=' in line: logger.info('Found BL_DATE in line') @@ -176,7 +176,7 @@ def get_bl_date(job_obj, pr_repo_loc): logger.info(f'Date {bldate} is not formatted YYYYMMDD') raise ValueError if not BLDATEFOUND: - job_obj.comment_text_append('[BL] ERROR: Variable "BL_DATE" not found in rt.ncar.sh.') + job_obj.comment_text_append('[BL] ERROR: Variable "BL_DATE" not found in rt.sh.') job_obj.job_failed(logger, 'get_bl_date()') logger.info('Finished get_bl_date') diff --git a/tests/auto/jobs/rt.py b/tests/auto/jobs/rt.py index d7faa20694..252ca29705 100644 --- a/tests/auto/jobs/rt.py +++ b/tests/auto/jobs/rt.py @@ -16,11 +16,11 @@ def run_regression_test(job_obj, pr_repo_loc): workflow_flag='-e' if job_obj.compiler == 'gnu': rt_command = [[f'export RT_COMPILER="{job_obj.compiler}" && cd tests ' - f'&& /bin/bash --login ./rt.ncar.sh -e -l rt_gnu.conf -a {job_obj.account} -p {job_obj.machine} -w {job_obj.workdir}', + f'&& /bin/bash --login ./rt.sh -e -l rt_gnu.conf -a {job_obj.account} -p {job_obj.machine} -w {job_obj.workdir}', pr_repo_loc]] elif job_obj.compiler == 'intel': rt_command = [[f'export RT_COMPILER="{job_obj.compiler}" && cd tests ' - f'&& /bin/bash --login ./rt.ncar.sh -e -a {job_obj.account} -p {job_obj.machine} -w {job_obj.workdir}', pr_repo_loc]] + f'&& /bin/bash --login ./rt.sh -e -a {job_obj.account} -p {job_obj.machine} -w {job_obj.workdir}', pr_repo_loc]] job_obj.run_commands(logger, rt_command) diff --git a/tests/rt.ncar.sh b/tests/rt.ncar.sh deleted file mode 100755 index 1e54f30e91..0000000000 --- a/tests/rt.ncar.sh +++ /dev/null @@ -1,878 +0,0 @@ -#!/bin/bash -set -eux - -SECONDS=0 - -hostname - -die() { echo "$@" >&2; exit 1; } -usage() { - set +x - echo - echo "Usage: $0 -a | -c | -e | -h | -k | -w | -d | -l | -m | -n | -r " - echo - echo " -a to use on for HPC queue" - echo " -c create new baseline results" - echo " -e use ecFlow workflow manager" - echo " -h display this help" - echo " -k keep run directory after $0 is completed" - echo " -l runs test specified in " - echo " -m compare against new baseline results" - echo " -n run single test " - echo " -r use Rocoto workflow manager" - echo " -w for weekly_test, skip comparing baseline results" - echo " -d delete run directories that are not used by other tests" - echo " -p platform (machine name)" - echo - set -x - exit 1 -} - -[[ $# -eq 0 ]] && usage - -rt_single() { - rm -f $RT_SINGLE_CONF - local compile_line='' - local run_line='' - while read -r line || [ "$line" ]; do - line="${line#"${line%%[![:space:]]*}"}" - [[ ${#line} == 0 ]] && continue - [[ $line == \#* ]] && continue - - if [[ $line == COMPILE* ]] ; then - MACHINES=$(echo $line | cut -d'|' -f5 | sed -e 's/^ *//' -e 's/ *$//') - RT_COMPILER_IN=$(echo $line | cut -d'|' -f3 | sed -e 's/^ *//' -e 's/ *$//') - if [[ ! $RT_COMPILER_IN == $RT_COMPILER ]]; then - continue - fi - if [[ ${MACHINES} == '' ]]; then - compile_line=$line - elif [[ ${MACHINES} == -* ]]; then - [[ ${MACHINES} =~ ${MACHINE_ID} ]] || compile_line=$line - elif [[ ${MACHINES} == +* ]]; then - [[ ${MACHINES} =~ ${MACHINE_ID} ]] && compile_line=$line - fi - fi - - if [[ $line =~ RUN ]]; then - tmp_test=$(echo $line | cut -d'|' -f2 | sed -e 's/^ *//' -e 's/ *$//') - if [[ $SINGLE_NAME == $tmp_test && $compile_line != '' ]]; then - echo $compile_line > $RT_SINGLE_CONF - dep_test=$(echo $line | grep -w $tmp_test | cut -d'|' -f5 | sed -e 's/^ *//' -e 's/ *$//') - if [[ $dep_test != '' ]]; then - dep_line=$(cat rt.conf | grep -w "$dep_test" | grep -v "$tmp_test") - dep_line="${dep_line#"${dep_line%%[![:space:]]*}"}" - echo $dep_line >> $RT_SINGLE_CONF - fi - echo $line >> $RT_SINGLE_CONF - break - fi - fi - done < $TESTS_FILE - - if [[ ! -f $RT_SINGLE_CONF ]]; then - echo "$SINGLE_NAME does not exist or cannot be run on $MACHINE_ID" - exit 1 - fi -} - -rt_35d() { -if [[ $TEST_NAME =~ '35d' ]] ; then - local sy=$(echo ${DATE_35D} | cut -c 1-4) - local sm=$(echo ${DATE_35D} | cut -c 5-6) - local new_test_name="tests/${TEST_NAME}_${DATE_35D}" - rm -f $new_test_name - cp tests/$TEST_NAME $new_test_name - - sed -i -e "s/\(export SYEAR\)/\1=\"$sy\"/" $new_test_name - sed -i -e "s/\(export SMONTH\)/\1=\"$sm\"/" $new_test_name - - TEST_NAME=${new_test_name#tests/} -fi -} - -rt_trap() { - [[ ${ROCOTO:-false} == true ]] && rocoto_kill - [[ ${ECFLOW:-false} == true ]] && ecflow_kill - cleanup -} - -cleanup() { - [[ $(awk '{print $2}' < "${LOCKDIR}/PID") == $$ ]] && rm -rf ${LOCKDIR} - [[ ${ECFLOW:-false} == true ]] && ecflow_stop - trap 0 - exit -} - -trap '{ echo "$0 interrupted"; rt_trap ; }' INT -trap '{ echo "$0 quit"; rt_trap ; }' QUIT -trap '{ echo "$0 terminated"; rt_trap ; }' TERM -trap '{ echo "$0 error on line $LINENO"; cleanup ; }' ERR -trap '{ echo "$0 finished"; cleanup ; }' EXIT - -# PATHRT - Path to regression tests directory -readonly PATHRT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd -P )" -cd ${PATHRT} - -# PATHTR - Path to nmmb trunk directory -readonly PATHTR=$( cd ${PATHRT}/.. && pwd ) - -# make sure only one instance of rt.sh is running -readonly LOCKDIR="${PATHRT}"/lock -if mkdir "${LOCKDIR}" ; then - echo $(hostname) $$ > "${LOCKDIR}/PID" -else - echo "Only one instance of $0 can be running at a time" - exit 1 -fi - -readonly RT_SINGLE_CONF='rt_single.conf' - -CREATE_BASELINE=false -ROCOTO=false -ECFLOW=false -KEEP_RUNDIR=false -SINGLE_NAME='' -TEST_35D=false -export skip_check_results=false -export delete_rundir=false - -TESTS_FILE='rt.conf' - -SKIP_ORDER=false - -while getopts "a:cl:mn:dw:krep:sh" opt; do - case $opt in - a) - ACCNR=$OPTARG - ;; - c) - CREATE_BASELINE=true - ;; - l) - TESTS_FILE=$OPTARG - SKIP_ORDER=true - ;; - m) - # redefine RTPWD to point to newly created baseline outputs - RTPWD=${NEW_BASELINE} - ;; - n) - SINGLE_OPTS=("$OPTARG") - until [[ $(eval "echo \${$OPTIND}") =~ ^-.* ]] || [ -z $(eval "echo \${$OPTIND}") ]; do - SINGLE_OPTS+=($(eval "echo \${$OPTIND}")) - OPTIND=$((OPTIND + 1)) - done - - if [[ ${#SINGLE_OPTS[@]} != 2 ]]; then - echo "The -n option needs AND , i.e. -n control_p8 intel" - exit 1 - fi - SINGLE_NAME=${SINGLE_OPTS[0],,} - export RT_COMPILER=${SINGLE_OPTS[1],,} - - if [[ "$RT_COMPILER" == "intel" ]] || [[ "$RT_COMPILER" == "gnu" ]]; then - echo "COMPILER set to ${RT_COMPILER}" - else - echo "Compiler must be either 'intel' or 'gnu'." - exit 1 - fi - ;; - d) - export delete_rundir=true - awk -F "|" '{print $5}' rt.conf | grep "\S" > keep_tests.tmp - ;; - s) - export skip_check_results=true - ;; - k) - KEEP_RUNDIR=true - ;; - r) - ROCOTO=true - ECFLOW=false - ;; - e) - ECFLOW=true - ROCOTO=false - ;; - p) - MACHINE_ID=$OPTARG - ;; - w) - dprefix=$OPTARG - ;; - h) - usage - ;; - \?) - usage - die "Invalid option: -$OPTARG" - ;; - :) - usage - die "Option -$OPTARG requires an argument." - ;; - esac -done - -source rt_utils.sh -source module-setup.sh - -if [[ $MACHINE_ID = wcoss2 ]]; then - - #module use /usrx/local/dev/emc_rocoto/modulefiles - #module load ruby/2.5.1 rocoto/1.3.0rc2 - #ROCOTORUN=$(which rocotorun) - #ROCOTOSTAT=$(which rocotostat) - #ROCOTOCOMPLETE=$(which rocotocomplete) - #ROCOTO_SCHEDULER=lsf - - module load ecflow/5.6.0.13 - module load gcc/10.3.0 python/3.8.6 - ECFLOW_START=${ECF_ROOT}/scripts/server_check.sh - export ECF_OUTPUTDIR=${PATHRT}/ecf_outputdir - export ECF_COMDIR=${PATHRT}/ecf_comdir - rm -rf ${ECF_OUTPUTDIR} ${ECF_COMDIR} - mkdir -p ${ECF_OUTPUTDIR} - mkdir -p ${ECF_COMDIR} - export colonifnco=":output" # hack - - DISKNM=/lfs/h2/emc/nems/noscrub/emc.nems/RT - QUEUE=dev - COMPILE_QUEUE=dev - PARTITION= - STMP=/lfs/h2/emc/ptmp - PTMP=/lfs/h2/emc/ptmp - SCHEDULER=pbs - -elif [[ $MACHINE_ID = acorn ]]; then - - module load ecflow/5.6.0.13 - module load gcc/10.3.0 python/3.8.6 - ECFLOW_START=${ECF_ROOT}/scripts/server_check.sh - export ECF_OUTPUTDIR=${PATHRT}/ecf_outputdir - export ECF_COMDIR=${PATHRT}/ecf_comdir - rm -rf ${ECF_OUTPUTDIR} ${ECF_COMDIR} - mkdir -p ${ECF_OUTPUTDIR} - mkdir -p ${ECF_COMDIR} - export colonifnco=":output" # hack - - DISKNM=/lfs/h1/emc/nems/noscrub/emc.nems/RT - QUEUE=dev - COMPILE_QUEUE=dev - PARTITION= - STMP=/lfs/h2/emc/ptmp - PTMP=/lfs/h2/emc/ptmp - SCHEDULER=pbs - -elif [[ $MACHINE_ID = gaea ]]; then - - export PATH=/lustre/f2/pdata/esrl/gsd/contrib/miniconda3/4.8.3/envs/ufs-weather-model/bin:/lustre/f2/pdata/esrl/gsd/contrib/miniconda3/4.8.3/bin:$PATH - export PYTHONPATH=/lustre/f2/pdata/esrl/gsd/contrib/miniconda3/4.8.3/envs/ufs-weather-model/lib/python3.8/site-packages:/lustre/f2/pdata/esrl/gsd/contrib/miniconda3/4.8.3/lib/python3.8/site-packages - ECFLOW_START=/lustre/f2/pdata/esrl/gsd/contrib/miniconda3/4.8.3/envs/ufs-weather-model/bin/ecflow_start.sh - ECF_PORT=$(( $(id -u) + 1500 )) - - DISKNM=/lustre/f2/pdata/ncep_shared/emc.nemspara/RT - QUEUE=normal - COMPILE_QUEUE=normal - PARTITION=c4 - STMP=/lustre/f2/scratch - PTMP=/lustre/f2/scratch - - SCHEDULER=slurm - -elif [[ $MACHINE_ID = hera ]]; then - - module load rocoto - ROCOTORUN=$(which rocotorun) - ROCOTOSTAT=$(which rocotostat) - ROCOTOCOMPLETE=$(which rocotocomplete) - ROCOTO_SCHEDULER=slurm - - PYTHONHOME=/scratch1/NCEPDEV/nems/emc.nemspara/soft/miniconda3_new_20210629 - export PATH=$PYTHONHOME/bin:$PATH - export PYTHONPATH=$PYTHONHOME/lib/python3.7/site-packages - - module load ecflow - ECFLOW_START=ecflow_start.sh - - QUEUE=batch - COMPILE_QUEUE=batch - - PARTITION= - INPUT_DATE=20230314 - dprefix=${dprefix:-/scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model} - DISKNM=$dprefix/RT - STMP=$dprefix/RT/stmp4 - PTMP=$dprefix/RT/stmp2 - - SCHEDULER=slurm - -elif [[ $MACHINE_ID = orion ]]; then - - module load gcc/8.3.0 - - module load contrib rocoto/1.3.1 - ROCOTORUN=$(which rocotorun) - ROCOTOSTAT=$(which rocotostat) - ROCOTOCOMPLETE=$(which rocotocomplete) - export PATH=/work/noaa/nems/emc.nemspara/soft/miniconda3/bin:$PATH - export PYTHONPATH=/work/noaa/nems/emc.nemspara/soft/miniconda3/lib/python3.8/site-packages - ECFLOW_START=/work/noaa/nems/emc.nemspara/soft/miniconda3/bin/ecflow_start.sh - ECF_PORT=$(( $(id -u) + 1500 )) - - QUEUE=batch - COMPILE_QUEUE=batch - PARTITION=orion - dprefix=/work/noaa/stmp/${USER} - DISKNM=/work/noaa/nems/emc.nemspara/RT - STMP=$dprefix/stmp - PTMP=$dprefix/stmp - - SCHEDULER=slurm - -elif [[ $MACHINE_ID = jet ]]; then - - module load rocoto/1.3.2 - ROCOTORUN=$(which rocotorun) - ROCOTOSTAT=$(which rocotostat) - ROCOTOCOMPLETE=$(which rocotocomplete) - ROCOTO_SCHEDULER=slurm - - export PATH=/lfs4/HFIP/hfv3gfs/software/miniconda3/4.8.3/envs/ufs-weather-model/bin:/lfs4/HFIP/hfv3gfs/software/miniconda3/4.8.3/bin:$PATH - export PYTHONPATH=/lfs4/HFIP/hfv3gfs/software/miniconda3/4.8.3/envs/ufs-weather-model/lib/python3.8/site-packages:/lfs4/HFIP/hfv3gfs/software/miniconda3/4.8.3/lib/python3.8/site-packages - module load ecflow - ECFLOW_START=/apps/ecflow/5.5.3/bin/ecflow_start.sh - - QUEUE=batch - COMPILE_QUEUE=batch - PARTITION=xjet - DISKNM=/mnt/lfs4/HFIP/hfv3gfs/role.epic/RT - dprefix=${dprefix:-/lfs4/HFIP/$ACCNR/$USER} - STMP=${STMP:-$dprefix/RT_BASELINE} - PTMP=${PTMP:-$dprefix/RT_RUNDIRS} - - SCHEDULER=slurm - -elif [[ $MACHINE_ID = s4 ]]; then - - module load rocoto/1.3.2 - module load ecflow/5.6.0 - module load miniconda/3.8-s4 - ROCOTORUN=$(which rocotorun) - ROCOTOSTAT=$(which rocotostat) - ROCOTOCOMPLETE=$(which rocotocomplete) - ROCOTO_SCHEDULER=slurm - - ECFLOW_START=/opt/ecflow/5.6.0/bin/ecflow_start.sh - ECF_PORT=$(( $(id -u) + 1500 )) - - QUEUE=s4 - COMPILE_QUEUE=s4 - - PARTITION=s4 - dprefix=/data/prod - DISKNM=$dprefix/emc.nemspara/RT - STMP=/scratch/short/users - PTMP=/scratch/users - - SCHEDULER=slurm - -elif [[ $MACHINE_ID = cheyenne ]]; then - - export PATH=/glade/p/ral/jntp/tools/miniconda3/4.8.3/envs/ufs-weather-model/bin:/glade/p/ral/jntp/tools/miniconda3/4.8.3/bin:$PATH - export PYTHONPATH=/glade/p/ral/jntp/tools/miniconda3/4.8.3/envs/ufs-weather-model/lib/python3.8/site-packages:/glade/p/ral/jntp/tools/miniconda3/4.8.3/lib/python3.8/site-packages - ECFLOW_START=/glade/p/ral/jntp/tools/miniconda3/4.8.3/envs/ufs-weather-model/bin/ecflow_start.sh - ECF_PORT=$(( $(id -u) + 1500 )) - - QUEUE=regular - COMPILE_QUEUE=regular - PARTITION= - INPUT_DATE=20221101 - dprefix=${dprefix:-/glade/p/ral/jntp/CCPP_regression_testing/NCAR_ufs-weather-model/} - DISKNM=$dprefix/RT - STMP=$dprefix/RT - PTMP=$dprefix/RT - SCHEDULER=pbs - -elif [[ $MACHINE_ID = stampede ]]; then - - export PYTHONPATH= - ECFLOW_START= - QUEUE=skx-normal - COMPILE_QUEUE=skx-dev - PARTITION= - dprefix=$SCRATCH/ufs-weather-model/run - DISKNM=/work2/07736/minsukji/stampede2/ufs-weather-model/RT - STMP=$dprefix - PTMP=$dprefix - SCHEDULER=slurm - MPIEXEC=ibrun - MPIEXECOPTS= - -elif [[ $MACHINE_ID = expanse ]]; then - - export PYTHONPATH= - ECFLOW_START= - QUEUE=compute - COMPILE_QUEUE=shared - PARTITION= - dprefix=/expanse/lustre/scratch/$USER/temp_project/run - DISKNM=/expanse/lustre/scratch/domh/temp_project/RT - STMP=$dprefix - PTMP=$dprefix - SCHEDULER=slurm - -elif [[ $MACHINE_ID = noaacloud ]]; then - - module use /apps/modules/modulefiles - module load rocoto/1.3.3 - - ROCOTORUN=$(which rocotorun) - ROCOTOSTAT=$(which rocotostat) - ROCOTOCOMPLETE=$(which rocotocomplete) - ROCOTO_SCHEDULER=slurm - - QUEUE=batch - COMPILE_QUEUE=batch - PARTITION= - dprefix=/lustre/ - DISKNM=/contrib/ufs-weather-model/RT - STMP=$dprefix/stmp4 - PTMP=$dprefix/stmp2 - SCHEDULER=slurm - -else - die "Unknown machine ID, please edit detect_machine.sh file" -fi - -# Display the machine and account -echo "Machine: " $MACHINE_ID " Account: " $ACCNR - -mkdir -p ${STMP}/${USER} - -NEW_BASELINE=${STMP}/${USER}/FV3_RT/REGRESSION_TEST - -# Overwrite default RUNDIR_ROOT if environment variable RUNDIR_ROOT is set -RUNDIR_ROOT=${RUNDIR_ROOT:-${PTMP}/${USER}/FV3_RT}/rt_$$ -mkdir -p ${RUNDIR_ROOT} - -if [[ $SINGLE_NAME != '' ]]; then - rt_single - TESTS_FILE=$RT_SINGLE_CONF -fi - -if [[ $TESTS_FILE =~ '35d' ]] || [[ $TESTS_FILE =~ 'weekly' ]]; then - TEST_35D=true -fi - -source bl_date.ncar.conf -INPUT_DATE=20230314 - -RTPWD=${RTPWD:-$DISKNM/../../baselines/main-${BL_DATE}/${RT_COMPILER^^}} - -INPUTDATA_ROOT=${INPUTDATA_ROOT:-$DISKNM/../../input_data/${INPUT_DATE}} -INPUTDATA_ROOT_WW3=${INPUTDATA_ROOT}/WW3_input_data_20220624 -INPUTDATA_ROOT_BMIC=${INPUTDATA_ROOT_BMIC:-$DISKNM/NCAR/BM_IC-20220207} - -shift $((OPTIND-1)) -[[ $# -gt 1 ]] && usage - -if [[ $CREATE_BASELINE == true ]]; then - # - # prepare new regression test directory - # - rm -rf "${NEW_BASELINE}" - mkdir -p "${NEW_BASELINE}" -fi - -if [[ $skip_check_results == true ]]; then - REGRESSIONTEST_LOG=${PATHRT}/logs/RegressionTests_weekly_$MACHINE_ID.log -else - REGRESSIONTEST_LOG=${PATHRT}/logs/RegressionTests_$MACHINE_ID.log -fi - -date > ${REGRESSIONTEST_LOG} -echo "Start Regression test" >> ${REGRESSIONTEST_LOG} -echo >> ${REGRESSIONTEST_LOG} -echo "Testing UFSWM Hash:" `git rev-parse HEAD` >> ${REGRESSIONTEST_LOG} -echo "Testing With Submodule Hashes:" >> ${REGRESSIONTEST_LOG} -git submodule status >> ${REGRESSIONTEST_LOG} - -source default_vars.sh - -JOB_NR=0 -TEST_NR=0 -COMPILE_COUNTER=0 -rm -f fail_test* fail_compile* - -export LOG_DIR=${PATHRT}/logs/log_$MACHINE_ID -rm -rf ${LOG_DIR} -mkdir -p ${LOG_DIR} - -if [[ $ROCOTO == true ]]; then - - ROCOTO_XML=${PATHRT}/rocoto_workflow.xml - ROCOTO_STATE=${PATHRT}/rocoto_workflow.state - ROCOTO_DB=${PATHRT}/rocoto_workflow.db - - rm -f $ROCOTO_XML $ROCOTO_DB $ROCOTO_STATE *_lock.db - - if [[ $MACHINE_ID = wcoss2 || $MACHINE_ID = acorn ]]; then - QUEUE=dev - COMPILE_QUEUE=dev - ROCOTO_SCHEDULER=pbs - elif [[ $MACHINE_ID = hera ]]; then - QUEUE=batch - COMPILE_QUEUE=batch - ROCOTO_SCHEDULER=slurm - elif [[ $MACHINE_ID = orion ]]; then - QUEUE=batch - COMPILE_QUEUE=batch - ROCOTO_SCHEDULER=slurm - elif [[ $MACHINE_ID = s4 ]]; then - QUEUE=s4 - COMPILE_QUEUE=s4 - ROCOTO_SCHEDULER=slurm - elif [[ $MACHINE_ID = noaacloud ]]; then - QUEUE=batch - COMPILE_QUEUE=batch - ROCOTO_SCHEDULER=slurm - elif [[ $MACHINE_ID = jet ]]; then - QUEUE=batch - COMPILE_QUEUE=batch - ROCOTO_SCHEDULER=slurm - else - die "Rocoto is not supported on this machine $MACHINE_ID" - fi - - cat << EOF > $ROCOTO_XML - - - - - - - - - - -]> - - 197001010000 197001010000 01:00:00 - &LOG;/workflow.log -EOF - -fi - -if [[ $ECFLOW == true ]]; then - - # Default maximum number of compile and run jobs - MAX_BUILDS=10 - MAX_JOBS=30 - - # Default number of tries to run jobs - on wcoss, no error tolerance - ECF_TRIES=2 - if [[ $MACHINE_ID = wcoss ]]; then - ECF_TRIES=1 - fi - - # Reduce maximum number of compile jobs on jet.intel and s4.intel because of licensing issues - if [[ $MACHINE_ID = jet ]]; then - MAX_BUILDS=5 - elif [[ $MACHINE_ID = s4 ]]; then - MAX_BUILDS=1 - fi - - ECFLOW_RUN=${PATHRT}/ecflow_run - ECFLOW_SUITE=regtest_$$ - rm -rf ${ECFLOW_RUN} - mkdir -p ${ECFLOW_RUN}/${ECFLOW_SUITE} - cp head.h tail.h ${ECFLOW_RUN} - cat << EOF > ${ECFLOW_RUN}/${ECFLOW_SUITE}.def -suite ${ECFLOW_SUITE} - edit ECF_HOME '${ECFLOW_RUN}' - edit ECF_INCLUDE '${ECFLOW_RUN}' - edit ECF_KILL_CMD kill -15 %ECF_RID% > %ECF_JOB%.kill 2>&1 - edit ECF_TRIES ${ECF_TRIES} - label src_dir '${PATHTR}' - label run_dir '${RUNDIR_ROOT}' - limit max_builds ${MAX_BUILDS} - limit max_jobs ${MAX_JOBS} -EOF - - if [[ $MACHINE_ID = wcoss2 || $MACHINE_ID = acorn ]]; then - QUEUE=dev - elif [[ $MACHINE_ID = hera ]]; then - QUEUE=batch - elif [[ $MACHINE_ID = orion ]]; then - QUEUE=batch - elif [[ $MACHINE_ID = jet ]]; then - QUEUE=batch - elif [[ $MACHINE_ID = s4 ]]; then - QUEUE=s4 - elif [[ $MACHINE_ID = gaea ]]; then - QUEUE=normal - elif [[ $MACHINE_ID = cheyenne ]]; then - QUEUE=regular - else - die "ecFlow is not supported on this machine $MACHINE_ID" - fi - -else - - if [[ $MACHINE_ID = hera.* ]] && [[ $HOSTNAME = hecflow ]]; then - echo "ERROR: To run without using ECFlow on Hera, please do not use ecflow node." - exit 1 - fi - -fi - -## -## read rt.conf and then either execute the test script directly or create -## workflow description file -## - -new_compile=false -in_metatask=false - -[[ -f $TESTS_FILE ]] || die "$TESTS_FILE does not exist" - -LAST_COMPILER_NR=-9999 - -declare -A compiles - -while read -r line || [ "$line" ]; do - - line="${line#"${line%%[![:space:]]*}"}" - [[ ${#line} == 0 ]] && continue - [[ $line == \#* ]] && continue - - JOB_NR=$( printf '%03d' $(( 10#$JOB_NR + 1 )) ) - - if [[ $line == COMPILE* ]]; then - - COMPILE_NAME=$( echo $line | cut -d'|' -f2 | sed -e 's/^ *//' -e 's/ *$//') - RT_COMPILER=$(echo $line | cut -d'|' -f3 | sed -e 's/^ *//' -e 's/ *$//') - MAKE_OPT=$( echo $line | cut -d'|' -f4 | sed -e 's/^ *//' -e 's/ *$//') - MACHINES=$( echo $line | cut -d'|' -f5 | sed -e 's/^ *//' -e 's/ *$//') - CB=$( echo $line | cut -d'|' -f6) - COMPILE_NR=${COMPILE_NAME}_${RT_COMPILER} - - set +u - if [[ ! -z ${compiles[$COMPILE_NR]} ]] ; then - echo "Error! Duplicated compilation $COMPILE_NAME for compiler $RT_COMPILER!" - exit 1 - fi - set -u - compiles[$COMPILE_NR]=$COMPILE_NR - echo "COMPILING ${compiles[${COMPILE_NR}]}" - - [[ $CREATE_BASELINE == true && $CB != *fv3* ]] && continue - - if [[ ${MACHINES} != '' ]]; then - if [[ ${MACHINES} == -* ]]; then - [[ ${MACHINES} =~ ${MACHINE_ID} ]] && continue - elif [[ ${MACHINES} == +* ]]; then - [[ ${MACHINES} =~ ${MACHINE_ID} ]] || continue - else - echo "MACHINES=|${MACHINES}|" - die "MACHINES spec must be either an empty string or start with either '+' or '-'" - fi - fi - - cat << EOF > ${RUNDIR_ROOT}/compile_${COMPILE_NR}.env - export JOB_NR=${JOB_NR} - export COMPILE_NR=${COMPILE_NR} - export MACHINE_ID=${MACHINE_ID} - export RT_COMPILER=${RT_COMPILER} - export PATHRT=${PATHRT} - export PATHTR=${PATHTR} - export SCHEDULER=${SCHEDULER} - export ACCNR=${ACCNR} - export QUEUE=${COMPILE_QUEUE} - export PARTITION=${PARTITION} - export ROCOTO=${ROCOTO} - export ECFLOW=${ECFLOW} - export REGRESSIONTEST_LOG=${REGRESSIONTEST_LOG} - export LOG_DIR=${LOG_DIR} -EOF - - if [[ $ROCOTO == true ]]; then - rocoto_create_compile_task - elif [[ $ECFLOW == true ]]; then - ecflow_create_compile_task - else - ./run_compile.sh ${PATHRT} ${RUNDIR_ROOT} "${MAKE_OPT}" ${COMPILE_NR} > ${LOG_DIR}/compile_${COMPILE_NR}.log 2>&1 - fi - - RT_SUFFIX="" - BL_SUFFIX="" - - continue - - elif [[ $line == RUN* ]] ; then - - TEST_NAME=$(echo $line | cut -d'|' -f2 | sed -e 's/^ *//' -e 's/ *$//') - MACHINES=$( echo $line | cut -d'|' -f3 | sed -e 's/^ *//' -e 's/ *$//') - CB=$( echo $line | cut -d'|' -f4) - DEP_RUN=$( echo $line | cut -d'|' -f5 | sed -e 's/^ *//' -e 's/ *$//') - DATE_35D=$( echo $line | cut -d'|' -f6 | sed -e 's/^ *//' -e 's/ *$//') - - if [[ $DEP_RUN != '' ]]; then - DEP_RUN=${DEP_RUN}_${RT_COMPILER} - fi - - [[ -e "tests/$TEST_NAME" ]] || die "run test file tests/$TEST_NAME does not exist" - [[ $CREATE_BASELINE == true && $CB != *baseline* ]] && continue - - if [[ ${MACHINES} != '' ]]; then - if [[ ${MACHINES} == -* ]]; then - [[ ${MACHINES} =~ ${MACHINE_ID} ]] && continue - elif [[ ${MACHINES} == +* ]]; then - [[ ${MACHINES} =~ ${MACHINE_ID} ]] || continue - else - echo "MACHINES=|${MACHINES}|" - die "MACHINES spec must be either an empty string or start with either '+' or '-'" - fi - fi - - # 35 day tests - [[ $TEST_35D == true ]] && rt_35d - - # Avoid uninitialized RT_SUFFIX/BL_SUFFIX (see definition above) - RT_SUFFIX=${RT_SUFFIX:-""} - BL_SUFFIX=${BL_SUFFIX:-""} - - if [[ $ROCOTO == true && $new_compile == true ]]; then - new_compile=false - in_metatask=true - cat << EOF >> $ROCOTO_XML - 0 -EOF - fi - - TEST_NR=$( printf '%03d' $(( 10#$TEST_NR + 1 )) ) - - ( - source ${PATHRT}/tests/$TEST_NAME - - compute_petbounds_and_tasks - - TPN=$(( TPN / THRD )) - NODES=$(( TASKS / TPN )) - if (( NODES * TPN < TASKS )); then - NODES=$(( NODES + 1 )) - fi - - cat << EOF > ${RUNDIR_ROOT}/run_test_${TEST_NR}.env - export JOB_NR=${JOB_NR} - export MACHINE_ID=${MACHINE_ID} - export RT_COMPILER=${RT_COMPILER} - export RTPWD=${RTPWD} - export INPUTDATA_ROOT=${INPUTDATA_ROOT} - export INPUTDATA_ROOT_WW3=${INPUTDATA_ROOT_WW3} - export INPUTDATA_ROOT_BMIC=${INPUTDATA_ROOT_BMIC} - export PATHRT=${PATHRT} - export PATHTR=${PATHTR} - export NEW_BASELINE=${NEW_BASELINE} - export CREATE_BASELINE=${CREATE_BASELINE} - export RT_SUFFIX=${RT_SUFFIX} - export BL_SUFFIX=${BL_SUFFIX} - export SCHEDULER=${SCHEDULER} - export ACCNR=${ACCNR} - export QUEUE=${QUEUE} - export PARTITION=${PARTITION} - export ROCOTO=${ROCOTO} - export ECFLOW=${ECFLOW} - export REGRESSIONTEST_LOG=${REGRESSIONTEST_LOG} - export LOG_DIR=${LOG_DIR} - export DEP_RUN=${DEP_RUN} - export skip_check_results=${skip_check_results} - export delete_rundir=${delete_rundir} - export WLCLK=${WLCLK} -EOF - if [[ $MACHINE_ID = jet ]]; then - cat << EOF >> ${RUNDIR_ROOT}/run_test_${TEST_NR}.env - export PATH=/lfs4/HFIP/hfv3gfs/software/miniconda3/4.8.3/envs/ufs-weather-model/bin:/lfs4/HFIP/hfv3gfs/software/miniconda3/4.8.3/bin:$PATH - export PYTHONPATH=/lfs4/HFIP/hfv3gfs/software/miniconda3/4.8.3/envs/ufs-weather-model/lib/python3.8/site-packages:/lfs4/HFIP/hfv3gfs/software/miniconda3/4.8.3/lib/python3.8/site-packages -EOF - fi - - if [[ $ROCOTO == true ]]; then - rocoto_create_run_task - elif [[ $ECFLOW == true ]]; then - ecflow_create_run_task - else - ./run_test.sh ${PATHRT} ${RUNDIR_ROOT} ${TEST_NAME} ${TEST_NR} ${COMPILE_NR} > ${LOG_DIR}/run_${TEST_NAME}_${RT_COMPILER}${RT_SUFFIX}.log 2>&1 - fi - ) - - continue - else - die "Unknown command $line" - fi -done < $TESTS_FILE - -## -## run regression test workflow (currently Rocoto or ecFlow are supported) -## - -if [[ $ROCOTO == true ]]; then - if [[ $in_metatask == true ]]; then - echo " " >> $ROCOTO_XML - fi - echo "" >> $ROCOTO_XML - # run rocoto workflow until done - rocoto_run -fi - -if [[ $ECFLOW == true ]]; then - echo "endsuite" >> ${ECFLOW_RUN}/${ECFLOW_SUITE}.def - # run ecflow workflow until done - ecflow_run -fi - -## -## regression test is either failed or successful -## -set +e -cat ${LOG_DIR}/compile_*_time.log >> ${REGRESSIONTEST_LOG} -cat ${LOG_DIR}/rt_*.log >> ${REGRESSIONTEST_LOG} - -FILES="fail_test_* fail_compile_*" -for f in $FILES; do - if [[ -f "$f" ]]; then - cat "$f" >> fail_test - fi -done - -if [[ -e fail_test ]]; then - echo "FAILED TESTS: " - echo "FAILED TESTS: " >> ${REGRESSIONTEST_LOG} - while read -r failed_test_name - do - echo "${failed_test_name}" - echo "${failed_test_name}" >> ${REGRESSIONTEST_LOG} - done < fail_test - echo ; echo REGRESSION TEST FAILED - (echo ; echo REGRESSION TEST FAILED) >> ${REGRESSIONTEST_LOG} -else - echo ; echo REGRESSION TEST WAS SUCCESSFUL - (echo ; echo REGRESSION TEST WAS SUCCESSFUL) >> ${REGRESSIONTEST_LOG} - - rm -f fv3_*.x fv3_*.exe modules.fv3_* modulefiles/modules.fv3_* keep_tests.tmp - [[ ${KEEP_RUNDIR} == false ]] && rm -rf ${RUNDIR_ROOT} - [[ ${ROCOTO} == true ]] && rm -f ${ROCOTO_XML} ${ROCOTO_DB} ${ROCOTO_STATE} *_lock.db - [[ ${TEST_35D} == true ]] && rm -f tests/cpld_bmark*_20* - [[ ${SINGLE_NAME} != '' ]] && rm -f $RT_SINGLE_CONF -fi - -date >> ${REGRESSIONTEST_LOG} - -elapsed_time=$( printf '%02dh:%02dm:%02ds\n' $((SECONDS%86400/3600)) $((SECONDS%3600/60)) $((SECONDS%60)) ) -echo "Elapsed time: ${elapsed_time}. Have a nice day!" >> ${REGRESSIONTEST_LOG} -echo "Elapsed time: ${elapsed_time}. Have a nice day!" diff --git a/tests/rt.sh b/tests/rt.sh index 456a33a883..e4af05583e 100755 --- a/tests/rt.sh +++ b/tests/rt.sh @@ -21,7 +21,8 @@ usage() { echo " -n run single test " echo " -r use Rocoto workflow manager" echo " -w for weekly_test, skip comparing baseline results" - echo " -d delete run direcotries that are not used by other tests" + echo " -d delete run directories that are not used by other tests" + echo " -p platform (machine name)" echo set -x exit 1 @@ -127,7 +128,94 @@ fi readonly RT_SINGLE_CONF='rt_single.conf' -source detect_machine.sh # Note: this does not set ACCNR. The "if" block below does. +CREATE_BASELINE=false +ROCOTO=false +ECFLOW=false +KEEP_RUNDIR=false +SINGLE_NAME='' +TEST_35D=false +export skip_check_results=false +export delete_rundir=false + +TESTS_FILE='rt.conf' + +SKIP_ORDER=false + +while getopts "a:cl:mn:dw:krep:sh" opt; do + case $opt in + a) + ACCNR=$OPTARG + ;; + c) + CREATE_BASELINE=true + ;; + l) + TESTS_FILE=$OPTARG + SKIP_ORDER=true + ;; + m) + # redefine RTPWD to point to newly created baseline outputs + RTPWD=${NEW_BASELINE} + ;; + n) + SINGLE_OPTS=("$OPTARG") + until [[ $(eval "echo \${$OPTIND}") =~ ^-.* ]] || [ -z $(eval "echo \${$OPTIND}") ]; do + SINGLE_OPTS+=($(eval "echo \${$OPTIND}")) + OPTIND=$((OPTIND + 1)) + done + + if [[ ${#SINGLE_OPTS[@]} != 2 ]]; then + echo "The -n option needs AND , i.e. -n control_p8 intel" + exit 1 + fi + SINGLE_NAME=${SINGLE_OPTS[0],,} + export RT_COMPILER=${SINGLE_OPTS[1],,} + + if [[ "$RT_COMPILER" == "intel" ]] || [[ "$RT_COMPILER" == "gnu" ]]; then + echo "COMPILER set to ${RT_COMPILER}" + else + echo "Compiler must be either 'intel' or 'gnu'." + exit 1 + fi + ;; + d) + export delete_rundir=true + awk -F "|" '{print $5}' rt.conf | grep "\S" > keep_tests.tmp + ;; + s) + export skip_check_results=true + ;; + k) + KEEP_RUNDIR=true + ;; + r) + ROCOTO=true + ECFLOW=false + ;; + e) + ECFLOW=true + ROCOTO=false + ;; + p) + MACHINE_ID=$OPTARG + ;; + w) + dprefix=$OPTARG + ;; + h) + usage + ;; + \?) + usage + die "Invalid option: -$OPTARG" + ;; + :) + usage + die "Option -$OPTARG requires an argument." + ;; + esac +done + source rt_utils.sh source module-setup.sh @@ -303,10 +391,11 @@ elif [[ $MACHINE_ID = hera ]]; then COMPILE_QUEUE=batch PARTITION= - dprefix=/scratch1/NCEPDEV - DISKNM=$dprefix/nems/emc.nemspara/RT - STMP=$dprefix/stmp4 - PTMP=$dprefix/stmp2 + INPUT_DATE=20230314 + dprefix=${dprefix:-/scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model} + DISKNM=$dprefix/RT + STMP=$dprefix/RT/stmp4 + PTMP=$dprefix/RT/stmp2 SCHEDULER=slurm @@ -392,10 +481,11 @@ elif [[ $MACHINE_ID = cheyenne ]]; then QUEUE=regular COMPILE_QUEUE=regular PARTITION= - dprefix=/glade/scratch - DISKNM=/glade/scratch/epicufsrt/GMTB/ufs-weather-model/RT - STMP=$dprefix - PTMP=$dprefix + INPUT_DATE=20221101 + dprefix=${dprefix:-/glade/p/ral/jntp/CCPP_regression_testing/NCAR_ufs-weather-model/} + DISKNM=$dprefix/RT + STMP=$dprefix/RT + PTMP=$dprefix/RT SCHEDULER=pbs elif [[ $MACHINE_ID = stampede ]]; then @@ -426,7 +516,7 @@ elif [[ $MACHINE_ID = expanse ]]; then PTMP=$dprefix SCHEDULER=slurm - elif [[ $MACHINE_ID = noaacloud.* ]]; then +elif [[ $MACHINE_ID = noaacloud.* ]]; then module use /apps/modules/modulefiles module load rocoto/1.3.3 @@ -467,7 +557,8 @@ if [[ $TESTS_FILE =~ '35d' ]] || [[ $TESTS_FILE =~ 'weekly' ]]; then TEST_35D=true fi -source bl_date.conf +source bl_date.ncar.conf +INPUT_DATE=20230314 if [[ "$RTPWD_NEW_BASELINE" == true ]] ; then RTPWD=${NEW_BASELINE} @@ -476,9 +567,9 @@ else fi -INPUTDATA_ROOT=${INPUTDATA_ROOT:-$DISKNM/NEMSfv3gfs/input-data-20221101} +INPUTDATA_ROOT=${INPUTDATA_ROOT:-$DISKNM/../../input_data/${INPUT_DATE}} INPUTDATA_ROOT_WW3=${INPUTDATA_ROOT}/WW3_input_data_20220624 -INPUTDATA_ROOT_BMIC=${INPUTDATA_ROOT_BMIC:-$DISKNM/NEMSfv3gfs/BM_IC-20220207} +INPUTDATA_ROOT_BMIC=${INPUTDATA_ROOT_BMIC:-$DISKNM/NCAR/BM_IC-20220207} shift $((OPTIND-1)) [[ $# -gt 1 ]] && usage From 916efdd4bb9a8e7c000c4cfeee6529796d2ad4ff Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Wed, 30 Aug 2023 15:28:39 +0000 Subject: [PATCH 27/56] Fix usage statement --- tests/auto/start_rt_auto.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/auto/start_rt_auto.sh b/tests/auto/start_rt_auto.sh index cdb7aa9158..0b0f363978 100755 --- a/tests/auto/start_rt_auto.sh +++ b/tests/auto/start_rt_auto.sh @@ -3,8 +3,9 @@ set -eux function usage { set +x - echo "usage: $0 machine workdir" + echo "usage: $0 machine account workdir" echo " machine: Name of current machine [hera/cheyenne]" + echo " account: HPC account to charge" echo " workdir: Working directory for checking out and running tests" exit 1 } From 45a35d3807a1f81bace3a3a4ec14c89d9e36d5b4 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Wed, 30 Aug 2023 15:55:31 +0000 Subject: [PATCH 28/56] Restore old flag letters for rt.sh, want to keep things as back-compatible as possible --- tests/auto/jobs/bl.py | 4 ++-- tests/auto/jobs/rt.py | 4 ++-- tests/rt.sh | 7 ++++--- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/auto/jobs/bl.py b/tests/auto/jobs/bl.py index 8afff5758b..17ace97ef7 100644 --- a/tests/auto/jobs/bl.py +++ b/tests/auto/jobs/bl.py @@ -89,11 +89,11 @@ def run_regression_test(job_obj, pr_repo_loc): logger = logging.getLogger('BL/RUN_REGRESSION_TEST') if job_obj.compiler == 'gnu': rt_command = [[f'export RT_COMPILER="{job_obj.compiler}" && cd tests ' - '&& /bin/bash --login ./rt.sh -e -a {job_obj.account} -c -l rt_gnu.conf -p {job_obj.machine} -w {job_obj.workdir} -k', + '&& /bin/bash --login ./rt.sh -e -a {job_obj.account} -c -l rt_gnu.conf -p {job_obj.machine} -z {job_obj.workdir} -k', pr_repo_loc]] elif job_obj.compiler == 'intel': rt_command = [[f'export RT_COMPILER="{job_obj.compiler}" && cd tests ' - '&& /bin/bash --login ./rt.sh -e -a {job_obj.account} -c -p {job_obj.machine} -w {job_obj.workdir} -k', pr_repo_loc]] + '&& /bin/bash --login ./rt.sh -e -a {job_obj.account} -c -p {job_obj.machine} -z {job_obj.workdir} -k', pr_repo_loc]] job_obj.run_commands(logger, rt_command) diff --git a/tests/auto/jobs/rt.py b/tests/auto/jobs/rt.py index 252ca29705..068b7f9e76 100644 --- a/tests/auto/jobs/rt.py +++ b/tests/auto/jobs/rt.py @@ -16,11 +16,11 @@ def run_regression_test(job_obj, pr_repo_loc): workflow_flag='-e' if job_obj.compiler == 'gnu': rt_command = [[f'export RT_COMPILER="{job_obj.compiler}" && cd tests ' - f'&& /bin/bash --login ./rt.sh -e -l rt_gnu.conf -a {job_obj.account} -p {job_obj.machine} -w {job_obj.workdir}', + f'&& /bin/bash --login ./rt.sh -e -l rt_gnu.conf -a {job_obj.account} -p {job_obj.machine} -z {job_obj.workdir}', pr_repo_loc]] elif job_obj.compiler == 'intel': rt_command = [[f'export RT_COMPILER="{job_obj.compiler}" && cd tests ' - f'&& /bin/bash --login ./rt.sh -e -a {job_obj.account} -p {job_obj.machine} -w {job_obj.workdir}', pr_repo_loc]] + f'&& /bin/bash --login ./rt.sh -e -a {job_obj.account} -p {job_obj.machine} -z {job_obj.workdir}', pr_repo_loc]] job_obj.run_commands(logger, rt_command) diff --git a/tests/rt.sh b/tests/rt.sh index e4af05583e..f7d1b3cdd1 100755 --- a/tests/rt.sh +++ b/tests/rt.sh @@ -23,6 +23,7 @@ usage() { echo " -w for weekly_test, skip comparing baseline results" echo " -d delete run directories that are not used by other tests" echo " -p platform (machine name)" + echo " -z specify working directory for regression test run" echo set -x exit 1 @@ -141,7 +142,7 @@ TESTS_FILE='rt.conf' SKIP_ORDER=false -while getopts "a:cl:mn:dw:krep:sh" opt; do +while getopts "a:cl:mn:dz:krep:wh" opt; do case $opt in a) ACCNR=$OPTARG @@ -182,7 +183,7 @@ while getopts "a:cl:mn:dw:krep:sh" opt; do export delete_rundir=true awk -F "|" '{print $5}' rt.conf | grep "\S" > keep_tests.tmp ;; - s) + w) export skip_check_results=true ;; k) @@ -199,7 +200,7 @@ while getopts "a:cl:mn:dw:krep:sh" opt; do p) MACHINE_ID=$OPTARG ;; - w) + z) dprefix=$OPTARG ;; h) From 155c0d12e88b1c40c77a6f6dbd3cbeae59b51f1e Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Wed, 30 Aug 2023 16:16:34 +0000 Subject: [PATCH 29/56] Pull out all machine-specific settings into their own source files under machine/ subdirectory, overwritable by command-line option "-s" --- tests/machine/acorn | 20 +++ tests/machine/cheyenne | 17 ++ tests/machine/cheyenne.ncar | 18 ++ tests/machine/expanse | 12 ++ tests/machine/gaea | 16 ++ tests/machine/hera | 24 +++ tests/machine/hera.ncar | 27 +++ tests/machine/jet | 23 +++ tests/machine/noaacloud | 19 +++ tests/machine/orion | 25 +++ tests/machine/s4 | 24 +++ tests/machine/stampede | 14 ++ tests/machine/wcoss | 27 +++ tests/rt.sh | 325 +----------------------------------- 14 files changed, 274 insertions(+), 317 deletions(-) create mode 100644 tests/machine/acorn create mode 100644 tests/machine/cheyenne create mode 100644 tests/machine/cheyenne.ncar create mode 100644 tests/machine/expanse create mode 100644 tests/machine/gaea create mode 100644 tests/machine/hera create mode 100644 tests/machine/hera.ncar create mode 100644 tests/machine/jet create mode 100644 tests/machine/noaacloud create mode 100644 tests/machine/orion create mode 100644 tests/machine/s4 create mode 100644 tests/machine/stampede create mode 100644 tests/machine/wcoss diff --git a/tests/machine/acorn b/tests/machine/acorn new file mode 100644 index 0000000000..0982088601 --- /dev/null +++ b/tests/machine/acorn @@ -0,0 +1,20 @@ + module load ecflow/5.6.0.13 + module load gcc/10.3.0 python/3.8.6 + ECFLOW_START=${ECF_ROOT}/scripts/server_check.sh + export ECF_OUTPUTDIR=${PATHRT}/ecf_outputdir + export ECF_COMDIR=${PATHRT}/ecf_comdir + rm -rf ${ECF_OUTPUTDIR} ${ECF_COMDIR} + mkdir -p ${ECF_OUTPUTDIR} + mkdir -p ${ECF_COMDIR} + export colonifnco=":output" # hack + + DISKNM=/lfs/h1/emc/nems/noscrub/emc.nems/RT + QUEUE=dev + COMPILE_QUEUE=dev + PARTITION= + STMP=/lfs/h2/emc/ptmp + PTMP=/lfs/h2/emc/ptmp + SCHEDULER=pbs + cp fv3_conf/fv3_qsub.IN_wcoss2 fv3_conf/fv3_qsub.IN + cp fv3_conf/compile_qsub.IN_wcoss2 fv3_conf/compile_qsub.IN + diff --git a/tests/machine/cheyenne b/tests/machine/cheyenne new file mode 100644 index 0000000000..54919f143d --- /dev/null +++ b/tests/machine/cheyenne @@ -0,0 +1,17 @@ + export PATH=/glade/work/jedipara/cheyenne/spack-stack/miniconda-3.9.12/bin:$PATH + export PYTHONPATH=/glade/work/jedipara/cheyenne/spack-stack/miniconda-3.9.12/lib/python3.9/site-packages + + module use /glade/work/jedipara/cheyenne/spack-stack/modulefiles/misc + module load ecflow/5.8.4 + ECFLOW_START=/glade/work/jedipara/cheyenne/spack-stack/ecflow-5.8.4/bin/ecflow_start.sh + ECF_PORT=$(( $(id -u) + 1500 )) + + QUEUE=regular + COMPILE_QUEUE=regular + PARTITION= + dprefix=/glade/scratch + DISKNM=/glade/scratch/epicufsrt/GMTB/ufs-weather-model/RT + STMP=$dprefix + PTMP=$dprefix + SCHEDULER=pbs + diff --git a/tests/machine/cheyenne.ncar b/tests/machine/cheyenne.ncar new file mode 100644 index 0000000000..af20f0102d --- /dev/null +++ b/tests/machine/cheyenne.ncar @@ -0,0 +1,18 @@ + export PATH=/glade/p/ral/jntp/tools/miniconda3/4.8.3/envs/ufs-weather-model/bin:/glade/p/ral/jntp/tools/miniconda3/4.8.3/bin:$PATH + export PYTHONPATH=/glade/p/ral/jntp/tools/miniconda3/4.8.3/envs/ufs-weather-model/lib/python3.8/site-packages:/glade/p/ral/jntp/tools/miniconda3/4.8.3/lib/python3.8/site-packages + ECFLOW_START=/glade/p/ral/jntp/tools/miniconda3/4.8.3/envs/ufs-weather-model/bin/ecflow_start.sh + ECF_PORT=$(( $(id -u) + 1500 )) + + QUEUE=regular + COMPILE_QUEUE=regular + PARTITION= + INPUT_DATE=20221101 + dprefix=${dprefix:-/glade/p/ral/jntp/CCPP_regression_testing/NCAR_ufs-weather-model/} + DISKNM=$dprefix/RT + STMP=$dprefix/RT + PTMP=$dprefix/RT + SCHEDULER=pbs + cp fv3_conf/fv3_qsub.IN_cheyenne fv3_conf/fv3_qsub.IN + cp fv3_conf/compile_qsub.IN_cheyenne fv3_conf/compile_qsub.IN + + diff --git a/tests/machine/expanse b/tests/machine/expanse new file mode 100644 index 0000000000..055aa07519 --- /dev/null +++ b/tests/machine/expanse @@ -0,0 +1,12 @@ + export PYTHONPATH= + ECFLOW_START= + QUEUE=compute + COMPILE_QUEUE=shared + PARTITION= + dprefix=/expanse/lustre/scratch/$USER/temp_project/run + DISKNM=/expanse/lustre/scratch/domh/temp_project/RT + STMP=$dprefix + PTMP=$dprefix + SCHEDULER=slurm + cp fv3_conf/fv3_slurm.IN_expanse fv3_conf/fv3_slurm.IN + diff --git a/tests/machine/gaea b/tests/machine/gaea new file mode 100644 index 0000000000..16bef941cc --- /dev/null +++ b/tests/machine/gaea @@ -0,0 +1,16 @@ + export PATH=/lustre/f2/pdata/esrl/gsd/contrib/miniconda3/4.8.3/envs/ufs-weather-model/bin:/lustre/f2/pdata/esrl/gsd/contrib/miniconda3/4.8.3/bin:$PATH + export PYTHONPATH=/lustre/f2/pdata/esrl/gsd/contrib/miniconda3/4.8.3/envs/ufs-weather-model/lib/python3.8/site-packages:/lustre/f2/pdata/esrl/gsd/contrib/miniconda3/4.8.3/lib/python3.8/site-packages + ECFLOW_START=/lustre/f2/pdata/esrl/gsd/contrib/miniconda3/4.8.3/envs/ufs-weather-model/bin/ecflow_start.sh + ECF_PORT=$(( $(id -u) + 1500 )) + + DISKNM=/lustre/f2/pdata/ncep_shared/emc.nemspara/RT + QUEUE=normal + COMPILE_QUEUE=normal + PARTITION=c4 + STMP=/lustre/f2/scratch + PTMP=/lustre/f2/scratch + + SCHEDULER=slurm + cp fv3_conf/fv3_slurm.IN_gaea fv3_conf/fv3_slurm.IN + cp fv3_conf/compile_slurm.IN_gaea fv3_conf/compile_slurm.IN + diff --git a/tests/machine/hera b/tests/machine/hera new file mode 100644 index 0000000000..f3ac945d42 --- /dev/null +++ b/tests/machine/hera @@ -0,0 +1,24 @@ + module load rocoto + ROCOTORUN=$(which rocotorun) + ROCOTOSTAT=$(which rocotostat) + ROCOTOCOMPLETE=$(which rocotocomplete) + ROCOTO_SCHEDULER=slurm + + PYTHONHOME=/scratch1/NCEPDEV/jcsda/jedipara/spack-stack/miniconda-3.9.12 + export PATH=$PYTHONHOME/bin:$PATH + export PYTHONPATH=$PYTHONHOME/lib/python3.9/site-packages + + module load ecflow/5.5.3 + ECFLOW_START=ecflow_start.sh + + QUEUE=batch + COMPILE_QUEUE=batch + + PARTITION= + dprefix=/scratch1/NCEPDEV + DISKNM=/scratch2/NAGAPE/epic/UFS-WM_RT + STMP=$dprefix/stmp4 + PTMP=$dprefix/stmp2 + + SCHEDULER=slurm + diff --git a/tests/machine/hera.ncar b/tests/machine/hera.ncar new file mode 100644 index 0000000000..92d533740f --- /dev/null +++ b/tests/machine/hera.ncar @@ -0,0 +1,27 @@ + module load rocoto + ROCOTORUN=$(which rocotorun) + ROCOTOSTAT=$(which rocotostat) + ROCOTOCOMPLETE=$(which rocotocomplete) + ROCOTO_SCHEDULER=slurm + + PYTHONHOME=/scratch1/NCEPDEV/nems/emc.nemspara/soft/miniconda3_new_20210629 + export PATH=$PYTHONHOME/bin:$PATH + export PYTHONPATH=$PYTHONHOME/lib/python3.7/site-packages + + module load ecflow + ECFLOW_START=ecflow_start.sh + + QUEUE=batch + COMPILE_QUEUE=batch + + PARTITION= + INPUT_DATE=20230314 + dprefix=${dprefix:-/scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model} + DISKNM=$dprefix/RT + STMP=$dprefix/RT/stmp4 + PTMP=$dprefix/RT/stmp2 + + SCHEDULER=slurm + cp fv3_conf/fv3_slurm.IN_hera fv3_conf/fv3_slurm.IN + cp fv3_conf/compile_slurm.IN_hera fv3_conf/compile_slurm.IN + diff --git a/tests/machine/jet b/tests/machine/jet new file mode 100644 index 0000000000..a82b8eccd5 --- /dev/null +++ b/tests/machine/jet @@ -0,0 +1,23 @@ + module load rocoto/1.3.2 + ROCOTORUN=$(which rocotorun) + ROCOTOSTAT=$(which rocotostat) + ROCOTOCOMPLETE=$(which rocotocomplete) + ROCOTO_SCHEDULER=slurm + + export PATH=/lfs4/HFIP/hfv3gfs/software/miniconda3/4.8.3/envs/ufs-weather-model/bin:/lfs4/HFIP/hfv3gfs/software/miniconda3/4.8.3/bin:$PATH + export PYTHONPATH=/lfs4/HFIP/hfv3gfs/software/miniconda3/4.8.3/envs/ufs-weather-model/lib/python3.8/site-packages:/lfs4/HFIP/hfv3gfs/software/miniconda3/4.8.3/lib/python3.8/site-packages + module load ecflow + ECFLOW_START=/apps/ecflow/5.5.3/bin/ecflow_start.sh + + QUEUE=batch + COMPILE_QUEUE=batch + PARTITION=xjet + DISKNM=/mnt/lfs4/HFIP/hfv3gfs/role.epic/RT + dprefix=${dprefix:-/lfs4/HFIP/$ACCNR/$USER} + STMP=${STMP:-$dprefix/RT_BASELINE} + PTMP=${PTMP:-$dprefix/RT_RUNDIRS} + + SCHEDULER=slurm + cp fv3_conf/fv3_slurm.IN_jet fv3_conf/fv3_slurm.IN + cp fv3_conf/compile_slurm.IN_jet fv3_conf/compile_slurm.IN + diff --git a/tests/machine/noaacloud b/tests/machine/noaacloud new file mode 100644 index 0000000000..53333ef6a7 --- /dev/null +++ b/tests/machine/noaacloud @@ -0,0 +1,19 @@ + module use /apps/modules/modulefiles + module load rocoto/1.3.3 + + ROCOTORUN=$(which rocotorun) + ROCOTOSTAT=$(which rocotostat) + ROCOTOCOMPLETE=$(which rocotocomplete) + ROCOTO_SCHEDULER=slurm + + QUEUE=batch + COMPILE_QUEUE=batch + PARTITION= + dprefix=/lustre/ + DISKNM=/contrib/ufs-weather-model/RT + STMP=$dprefix/stmp4 + PTMP=$dprefix/stmp2 + SCHEDULER=slurm + cp fv3_conf/fv3_slurm.IN_noaacloud fv3_conf/fv3_slurm.IN + cp fv3_conf/compile_slurm.IN_noaacloud fv3_conf/compile_slurm.IN + diff --git a/tests/machine/orion b/tests/machine/orion new file mode 100644 index 0000000000..679210d1a7 --- /dev/null +++ b/tests/machine/orion @@ -0,0 +1,25 @@ + module load git/2.28.0 + + module load gcc/8.3.0 + + module load contrib rocoto/1.3.1 + ROCOTORUN=$(which rocotorun) + ROCOTOSTAT=$(which rocotostat) + ROCOTOCOMPLETE=$(which rocotocomplete) + export PATH=/work/noaa/nems/emc.nemspara/soft/miniconda3/bin:$PATH + export PYTHONPATH=/work/noaa/nems/emc.nemspara/soft/miniconda3/lib/python3.8/site-packages + ECFLOW_START=/work/noaa/nems/emc.nemspara/soft/miniconda3/bin/ecflow_start.sh + ECF_PORT=$(( $(id -u) + 1500 )) + + QUEUE=batch + COMPILE_QUEUE=batch + PARTITION=orion + dprefix=/work/noaa/stmp/${USER} + DISKNM=/work/noaa/nems/emc.nemspara/RT + STMP=$dprefix/stmp + PTMP=$dprefix/stmp + + SCHEDULER=slurm + cp fv3_conf/fv3_slurm.IN_orion fv3_conf/fv3_slurm.IN + cp fv3_conf/compile_slurm.IN_orion fv3_conf/compile_slurm.IN + diff --git a/tests/machine/s4 b/tests/machine/s4 new file mode 100644 index 0000000000..79069c7087 --- /dev/null +++ b/tests/machine/s4 @@ -0,0 +1,24 @@ + module load rocoto/1.3.2 + module load ecflow/5.6.0 + module load miniconda/3.8-s4 + ROCOTORUN=$(which rocotorun) + ROCOTOSTAT=$(which rocotostat) + ROCOTOCOMPLETE=$(which rocotocomplete) + ROCOTO_SCHEDULER=slurm + + ECFLOW_START=/opt/ecflow/5.6.0/bin/ecflow_start.sh + ECF_PORT=$(( $(id -u) + 1500 )) + + QUEUE=s4 + COMPILE_QUEUE=s4 + + PARTITION=s4 + dprefix=/data/prod + DISKNM=$dprefix/emc.nemspara/RT + STMP=/scratch/short/users + PTMP=/scratch/users + + SCHEDULER=slurm + cp fv3_conf/fv3_slurm.IN_s4 fv3_conf/fv3_slurm.IN + cp fv3_conf/compile_slurm.IN_s4 fv3_conf/compile_slurm.IN + diff --git a/tests/machine/stampede b/tests/machine/stampede new file mode 100644 index 0000000000..6579ffe234 --- /dev/null +++ b/tests/machine/stampede @@ -0,0 +1,14 @@ + export PYTHONPATH= + ECFLOW_START= + QUEUE=skx-normal + COMPILE_QUEUE=skx-dev + PARTITION= + dprefix=$SCRATCH/ufs-weather-model/run + DISKNM=/work2/07736/minsukji/stampede2/ufs-weather-model/RT + STMP=$dprefix + PTMP=$dprefix + SCHEDULER=slurm + MPIEXEC=ibrun + MPIEXECOPTS= + cp fv3_conf/fv3_slurm.IN_stampede fv3_conf/fv3_slurm.IN + diff --git a/tests/machine/wcoss b/tests/machine/wcoss new file mode 100644 index 0000000000..1d77b3afa0 --- /dev/null +++ b/tests/machine/wcoss @@ -0,0 +1,27 @@ + #module use /usrx/local/dev/emc_rocoto/modulefiles + #module load ruby/2.5.1 rocoto/1.3.0rc2 + #ROCOTORUN=$(which rocotorun) + #ROCOTOSTAT=$(which rocotostat) + #ROCOTOCOMPLETE=$(which rocotocomplete) + #ROCOTO_SCHEDULER=lsf + + module load ecflow/5.6.0.13 + module load gcc/10.3.0 python/3.8.6 + ECFLOW_START=${ECF_ROOT}/scripts/server_check.sh + export ECF_OUTPUTDIR=${PATHRT}/ecf_outputdir + export ECF_COMDIR=${PATHRT}/ecf_comdir + rm -rf ${ECF_OUTPUTDIR} ${ECF_COMDIR} + mkdir -p ${ECF_OUTPUTDIR} + mkdir -p ${ECF_COMDIR} + export colonifnco=":output" # hack + + DISKNM=/lfs/h2/emc/nems/noscrub/emc.nems/RT + QUEUE=dev + COMPILE_QUEUE=dev + PARTITION= + STMP=/lfs/h2/emc/ptmp + PTMP=/lfs/h2/emc/ptmp + SCHEDULER=pbs + cp fv3_conf/fv3_qsub.IN_wcoss2 fv3_conf/fv3_qsub.IN + cp fv3_conf/compile_qsub.IN_wcoss2 fv3_conf/compile_qsub.IN + diff --git a/tests/rt.sh b/tests/rt.sh index f7d1b3cdd1..acf45e89c3 100755 --- a/tests/rt.sh +++ b/tests/rt.sh @@ -24,6 +24,7 @@ usage() { echo " -d delete run directories that are not used by other tests" echo " -p platform (machine name)" echo " -z specify working directory for regression test run" + echo " -s specify sourced environment file in machine/ subdirectory (default is {platform})" echo set -x exit 1 @@ -203,6 +204,9 @@ while getopts "a:cl:mn:dz:krep:wh" opt; do z) dprefix=$OPTARG ;; + s) + sourcefile=$OPTARG + ;; h) usage ;; @@ -220,326 +224,13 @@ done source rt_utils.sh source module-setup.sh -CREATE_BASELINE=false -ROCOTO=false -ECFLOW=false -KEEP_RUNDIR=false -SINGLE_NAME='' -TEST_35D=false -export skip_check_results=false -export delete_rundir=false -SKIP_ORDER=false -RTPWD_NEW_BASELINE=false -TESTS_FILE='rt.conf' -ACCNR=${ACCNR:-""} - -while getopts ":a:cl:mn:dwkreh" opt; do - case $opt in - a) - ACCNR=$OPTARG - ;; - c) - CREATE_BASELINE=true - ;; - l) - TESTS_FILE=$OPTARG - SKIP_ORDER=true - ;; - m) - # redefine RTPWD to point to newly created baseline outputs - RTPWD_NEW_BASELINE=true - ;; - n) - SINGLE_OPTS=("$OPTARG") - until [[ $(eval "echo \${$OPTIND}") =~ ^-.* ]] || [ -z $(eval "echo \${$OPTIND}") ]; do - SINGLE_OPTS+=($(eval "echo \${$OPTIND}")) - OPTIND=$((OPTIND + 1)) - done - - if [[ ${#SINGLE_OPTS[@]} != 2 ]]; then - echo "The -n option needs AND , i.e. -n control_p8 intel" - exit 1 - fi - SINGLE_NAME=${SINGLE_OPTS[0],,} - export RT_COMPILER=${SINGLE_OPTS[1],,} - - if [[ "$RT_COMPILER" == "intel" ]] || [[ "$RT_COMPILER" == "gnu" ]]; then - echo "COMPILER set to ${RT_COMPILER}" - else - echo "Compiler must be either 'intel' or 'gnu'." - exit 1 - fi - ;; - d) - export delete_rundir=true - awk -F "|" '{print $5}' rt.conf | grep "\S" > keep_tests.tmp - ;; - w) - export skip_check_results=true - ;; - k) - KEEP_RUNDIR=true - ;; - r) - ROCOTO=true - ECFLOW=false - ;; - e) - ECFLOW=true - ROCOTO=false - ;; - h) - usage - ;; - \?) - usage - die "Invalid option: -$OPTARG" - ;; - :) - usage - die "Option -$OPTARG requires an argument." - ;; - esac -done +sourcefile=${sourcefile:-"machine/$MACHINE_ID"} -if [[ -z "$ACCNR" ]]; then - echo "Please use -a to set group account to use on HPC" - exit 1 +if ! [ -f "$sourcefile" ]; then + die "ERROR: Invalid source file, check that valid platform has been specified\n and that source file '$sourcefile' exists" fi -# Display the machine and account using the format detect_machine.sh used: -echo "Machine: " $MACHINE_ID " Account: " $ACCNR - -if [[ $MACHINE_ID = wcoss2 ]]; then - - #module use /usrx/local/dev/emc_rocoto/modulefiles - #module load ruby/2.5.1 rocoto/1.3.0rc2 - #ROCOTORUN=$(which rocotorun) - #ROCOTOSTAT=$(which rocotostat) - #ROCOTOCOMPLETE=$(which rocotocomplete) - #ROCOTO_SCHEDULER=lsf - - module load ecflow/5.6.0.13 - module load gcc/10.3.0 python/3.8.6 - ECFLOW_START=${ECF_ROOT}/scripts/server_check.sh - export ECF_OUTPUTDIR=${PATHRT}/ecf_outputdir - export ECF_COMDIR=${PATHRT}/ecf_comdir - rm -rf ${ECF_OUTPUTDIR} ${ECF_COMDIR} - mkdir -p ${ECF_OUTPUTDIR} - mkdir -p ${ECF_COMDIR} - export colonifnco=":output" # hack - - DISKNM=/lfs/h2/emc/nems/noscrub/emc.nems/RT - QUEUE=dev - COMPILE_QUEUE=dev - PARTITION= - STMP=/lfs/h2/emc/ptmp - PTMP=/lfs/h2/emc/ptmp - SCHEDULER=pbs - -elif [[ $MACHINE_ID = acorn ]]; then - - module load ecflow/5.6.0.13 - module load gcc/10.3.0 python/3.8.6 - ECFLOW_START=${ECF_ROOT}/scripts/server_check.sh - export ECF_OUTPUTDIR=${PATHRT}/ecf_outputdir - export ECF_COMDIR=${PATHRT}/ecf_comdir - rm -rf ${ECF_OUTPUTDIR} ${ECF_COMDIR} - mkdir -p ${ECF_OUTPUTDIR} - mkdir -p ${ECF_COMDIR} - export colonifnco=":output" # hack - - DISKNM=/lfs/h1/emc/nems/noscrub/emc.nems/RT - QUEUE=dev - COMPILE_QUEUE=dev - PARTITION= - STMP=/lfs/h2/emc/ptmp - PTMP=/lfs/h2/emc/ptmp - SCHEDULER=pbs - -elif [[ $MACHINE_ID = gaea ]]; then - - export PATH=/lustre/f2/pdata/esrl/gsd/contrib/miniconda3/4.8.3/envs/ufs-weather-model/bin:/lustre/f2/pdata/esrl/gsd/contrib/miniconda3/4.8.3/bin:$PATH - export PYTHONPATH=/lustre/f2/pdata/esrl/gsd/contrib/miniconda3/4.8.3/envs/ufs-weather-model/lib/python3.8/site-packages:/lustre/f2/pdata/esrl/gsd/contrib/miniconda3/4.8.3/lib/python3.8/site-packages - ECFLOW_START=/lustre/f2/pdata/esrl/gsd/contrib/miniconda3/4.8.3/envs/ufs-weather-model/bin/ecflow_start.sh - ECF_PORT=$(( $(id -u) + 1500 )) - - DISKNM=/lustre/f2/pdata/ncep_shared/emc.nemspara/RT - QUEUE=normal - COMPILE_QUEUE=normal - PARTITION=c4 - STMP=/lustre/f2/scratch - PTMP=/lustre/f2/scratch - - SCHEDULER=slurm - -elif [[ $MACHINE_ID = hera ]]; then - - module load rocoto - ROCOTORUN=$(which rocotorun) - ROCOTOSTAT=$(which rocotostat) - ROCOTOCOMPLETE=$(which rocotocomplete) - ROCOTO_SCHEDULER=slurm - - PYTHONHOME=/scratch1/NCEPDEV/nems/emc.nemspara/soft/miniconda3_new_20210629 - export PATH=$PYTHONHOME/bin:$PATH - export PYTHONPATH=$PYTHONHOME/lib/python3.7/site-packages - - module load ecflow - ECFLOW_START=ecflow_start.sh - - QUEUE=batch - COMPILE_QUEUE=batch - - PARTITION= - INPUT_DATE=20230314 - dprefix=${dprefix:-/scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model} - DISKNM=$dprefix/RT - STMP=$dprefix/RT/stmp4 - PTMP=$dprefix/RT/stmp2 - - SCHEDULER=slurm - -elif [[ $MACHINE_ID = orion ]]; then - - module load git/2.28.0 - - module load gcc/8.3.0 - - module load contrib rocoto/1.3.1 - ROCOTORUN=$(which rocotorun) - ROCOTOSTAT=$(which rocotostat) - ROCOTOCOMPLETE=$(which rocotocomplete) - export PATH=/work/noaa/nems/emc.nemspara/soft/miniconda3/bin:$PATH - export PYTHONPATH=/work/noaa/nems/emc.nemspara/soft/miniconda3/lib/python3.8/site-packages - ECFLOW_START=/work/noaa/nems/emc.nemspara/soft/miniconda3/bin/ecflow_start.sh - ECF_PORT=$(( $(id -u) + 1500 )) - - QUEUE=batch - COMPILE_QUEUE=batch - PARTITION=orion - dprefix=/work/noaa/stmp/${USER} - DISKNM=/work/noaa/nems/emc.nemspara/RT - STMP=$dprefix/stmp - PTMP=$dprefix/stmp - - SCHEDULER=slurm - -elif [[ $MACHINE_ID = jet ]]; then - - module load rocoto/1.3.2 - ROCOTORUN=$(which rocotorun) - ROCOTOSTAT=$(which rocotostat) - ROCOTOCOMPLETE=$(which rocotocomplete) - ROCOTO_SCHEDULER=slurm - - export PATH=/lfs4/HFIP/hfv3gfs/software/miniconda3/4.8.3/envs/ufs-weather-model/bin:/lfs4/HFIP/hfv3gfs/software/miniconda3/4.8.3/bin:$PATH - export PYTHONPATH=/lfs4/HFIP/hfv3gfs/software/miniconda3/4.8.3/envs/ufs-weather-model/lib/python3.8/site-packages:/lfs4/HFIP/hfv3gfs/software/miniconda3/4.8.3/lib/python3.8/site-packages - module load ecflow - ECFLOW_START=/apps/ecflow/5.5.3/bin/ecflow_start.sh - - QUEUE=batch - COMPILE_QUEUE=batch - PARTITION=xjet - DISKNM=/mnt/lfs4/HFIP/hfv3gfs/role.epic/RT - dprefix=${dprefix:-/lfs4/HFIP/$ACCNR/$USER} - STMP=${STMP:-$dprefix/RT_BASELINE} - PTMP=${PTMP:-$dprefix/RT_RUNDIRS} - - SCHEDULER=slurm - -elif [[ $MACHINE_ID = s4 ]]; then - - module load rocoto/1.3.2 - module load ecflow/5.6.0 - module load miniconda/3.8-s4 - ROCOTORUN=$(which rocotorun) - ROCOTOSTAT=$(which rocotostat) - ROCOTOCOMPLETE=$(which rocotocomplete) - ROCOTO_SCHEDULER=slurm - - ECFLOW_START=/opt/ecflow/5.6.0/bin/ecflow_start.sh - ECF_PORT=$(( $(id -u) + 1500 )) - - QUEUE=s4 - COMPILE_QUEUE=s4 - - PARTITION=s4 - dprefix=/data/prod - DISKNM=$dprefix/emc.nemspara/RT - STMP=/scratch/short/users - PTMP=/scratch/users - - SCHEDULER=slurm - -elif [[ $MACHINE_ID = cheyenne ]]; then - - export PATH=/glade/p/ral/jntp/tools/miniconda3/4.8.3/envs/ufs-weather-model/bin:/glade/p/ral/jntp/tools/miniconda3/4.8.3/bin:$PATH - export PYTHONPATH=/glade/p/ral/jntp/tools/miniconda3/4.8.3/envs/ufs-weather-model/lib/python3.8/site-packages:/glade/p/ral/jntp/tools/miniconda3/4.8.3/lib/python3.8/site-packages - ECFLOW_START=/glade/p/ral/jntp/tools/miniconda3/4.8.3/envs/ufs-weather-model/bin/ecflow_start.sh - ECF_PORT=$(( $(id -u) + 1500 )) - - QUEUE=regular - COMPILE_QUEUE=regular - PARTITION= - INPUT_DATE=20221101 - dprefix=${dprefix:-/glade/p/ral/jntp/CCPP_regression_testing/NCAR_ufs-weather-model/} - DISKNM=$dprefix/RT - STMP=$dprefix/RT - PTMP=$dprefix/RT - SCHEDULER=pbs - -elif [[ $MACHINE_ID = stampede ]]; then - - export PYTHONPATH= - ECFLOW_START= - QUEUE=skx-normal - COMPILE_QUEUE=skx-dev - PARTITION= - dprefix=$SCRATCH/ufs-weather-model/run - DISKNM=/work2/07736/minsukji/stampede2/ufs-weather-model/RT - STMP=$dprefix - PTMP=$dprefix - SCHEDULER=slurm - MPIEXEC=ibrun - MPIEXECOPTS= - -elif [[ $MACHINE_ID = expanse ]]; then - - export PYTHONPATH= - ECFLOW_START= - QUEUE=compute - COMPILE_QUEUE=shared - PARTITION= - dprefix=/expanse/lustre/scratch/$USER/temp_project/run - DISKNM=/expanse/lustre/scratch/domh/temp_project/RT - STMP=$dprefix - PTMP=$dprefix - SCHEDULER=slurm - -elif [[ $MACHINE_ID = noaacloud.* ]]; then - - module use /apps/modules/modulefiles - module load rocoto/1.3.3 - - ROCOTORUN=$(which rocotorun) - ROCOTOSTAT=$(which rocotostat) - ROCOTOCOMPLETE=$(which rocotocomplete) - ROCOTO_SCHEDULER=slurm - - QUEUE=batch - COMPILE_QUEUE=batch - PARTITION= - dprefix=/lustre/ - DISKNM=/contrib/ufs-weather-model/RT - STMP=$dprefix/stmp4 - PTMP=$dprefix/stmp2 - SCHEDULER=slurm - - -else - die "Unknown machine ID, please edit detect_machine.sh file" -fi +source ${sourcefile:-"machine/$MACHINE_ID"} mkdir -p ${STMP}/${USER} From c67ab50f2fef63b06ae27aace3774913e168f4da Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Wed, 30 Aug 2023 16:27:35 +0000 Subject: [PATCH 30/56] Turns out you could already overwrite the working directory, but only via environment variable --- tests/auto/jobs/rt.py | 8 ++++---- tests/rt.sh | 4 ---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/tests/auto/jobs/rt.py b/tests/auto/jobs/rt.py index 068b7f9e76..31fbdc599a 100644 --- a/tests/auto/jobs/rt.py +++ b/tests/auto/jobs/rt.py @@ -15,12 +15,12 @@ def run_regression_test(job_obj, pr_repo_loc): logger = logging.getLogger('RT/RUN_REGRESSION_TEST') workflow_flag='-e' if job_obj.compiler == 'gnu': - rt_command = [[f'export RT_COMPILER="{job_obj.compiler}" && cd tests ' - f'&& /bin/bash --login ./rt.sh -e -l rt_gnu.conf -a {job_obj.account} -p {job_obj.machine} -z {job_obj.workdir}', + rt_command = [[f'export RT_COMPILER="{job_obj.compiler}" && export RUNDIR_ROOT={job_obj.workdir} && cd tests ' + f'&& /bin/bash --login ./rt.sh -e -l rt_gnu.conf -a {job_obj.account} -p {job_obj.machine}', pr_repo_loc]] elif job_obj.compiler == 'intel': - rt_command = [[f'export RT_COMPILER="{job_obj.compiler}" && cd tests ' - f'&& /bin/bash --login ./rt.sh -e -a {job_obj.account} -p {job_obj.machine} -z {job_obj.workdir}', pr_repo_loc]] + rt_command = [[f'export RT_COMPILER="{job_obj.compiler}" && export RUNDIR_ROOT={job_obj.workdir} && cd tests ' + f'&& /bin/bash --login ./rt.sh -e -a {job_obj.account} -p {job_obj.machine}', pr_repo_loc]] job_obj.run_commands(logger, rt_command) diff --git a/tests/rt.sh b/tests/rt.sh index acf45e89c3..072d5df4c1 100755 --- a/tests/rt.sh +++ b/tests/rt.sh @@ -23,7 +23,6 @@ usage() { echo " -w for weekly_test, skip comparing baseline results" echo " -d delete run directories that are not used by other tests" echo " -p platform (machine name)" - echo " -z specify working directory for regression test run" echo " -s specify sourced environment file in machine/ subdirectory (default is {platform})" echo set -x @@ -201,9 +200,6 @@ while getopts "a:cl:mn:dz:krep:wh" opt; do p) MACHINE_ID=$OPTARG ;; - z) - dprefix=$OPTARG - ;; s) sourcefile=$OPTARG ;; From 3fd441e8742aac9f94df00df5b652cf744e8a7f7 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Wed, 30 Aug 2023 18:04:26 +0000 Subject: [PATCH 31/56] Forgot to add new option to getopts list --- tests/rt.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/rt.sh b/tests/rt.sh index 072d5df4c1..e9bb04193e 100755 --- a/tests/rt.sh +++ b/tests/rt.sh @@ -23,7 +23,7 @@ usage() { echo " -w for weekly_test, skip comparing baseline results" echo " -d delete run directories that are not used by other tests" echo " -p platform (machine name)" - echo " -s specify sourced environment file in machine/ subdirectory (default is {platform})" + echo " -s specify sourced environment file (default is machine/\$platform)" echo set -x exit 1 @@ -142,7 +142,7 @@ TESTS_FILE='rt.conf' SKIP_ORDER=false -while getopts "a:cl:mn:dz:krep:wh" opt; do +while getopts "a:cl:mn:dz:krep:s:wh" opt; do case $opt in a) ACCNR=$OPTARG From be2f0be59672d8ecae9031261c92b9245e72442c Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Wed, 30 Aug 2023 18:37:39 +0000 Subject: [PATCH 32/56] No reason to try to create STMP directory if we aren't creating a new baseline --- tests/machine/cheyenne.ncar | 2 +- tests/machine/hera.ncar | 2 +- tests/rt.sh | 5 +---- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/tests/machine/cheyenne.ncar b/tests/machine/cheyenne.ncar index af20f0102d..a74089a3a5 100644 --- a/tests/machine/cheyenne.ncar +++ b/tests/machine/cheyenne.ncar @@ -7,7 +7,7 @@ COMPILE_QUEUE=regular PARTITION= INPUT_DATE=20221101 - dprefix=${dprefix:-/glade/p/ral/jntp/CCPP_regression_testing/NCAR_ufs-weather-model/} + dprefix=${RUNDIR_ROOT:-/glade/p/ral/jntp/CCPP_regression_testing/NCAR_ufs-weather-model/} DISKNM=$dprefix/RT STMP=$dprefix/RT PTMP=$dprefix/RT diff --git a/tests/machine/hera.ncar b/tests/machine/hera.ncar index 92d533740f..a607be5eff 100644 --- a/tests/machine/hera.ncar +++ b/tests/machine/hera.ncar @@ -16,7 +16,7 @@ PARTITION= INPUT_DATE=20230314 - dprefix=${dprefix:-/scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model} + dprefix=${RUNDIR_ROOT:-/scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model} DISKNM=$dprefix/RT STMP=$dprefix/RT/stmp4 PTMP=$dprefix/RT/stmp2 diff --git a/tests/rt.sh b/tests/rt.sh index e9bb04193e..4eed6c0fee 100755 --- a/tests/rt.sh +++ b/tests/rt.sh @@ -228,10 +228,6 @@ fi source ${sourcefile:-"machine/$MACHINE_ID"} -mkdir -p ${STMP}/${USER} - -NEW_BASELINE=${STMP}/${USER}/FV3_RT/REGRESSION_TEST - # Overwrite default RUNDIR_ROOT if environment variable RUNDIR_ROOT is set RUNDIR_ROOT=${RUNDIR_ROOT:-${PTMP}/${USER}/FV3_RT}/rt_$$ mkdir -p ${RUNDIR_ROOT} @@ -266,6 +262,7 @@ if [[ $CREATE_BASELINE == true ]]; then # # prepare new regression test directory # + NEW_BASELINE=${STMP}/${USER}/FV3_RT/REGRESSION_TEST rm -rf "${NEW_BASELINE}" mkdir -p "${NEW_BASELINE}" fi From 19289b0d09dd4b6142128049b69bdcc2783a5a79 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Wed, 30 Aug 2023 19:15:42 +0000 Subject: [PATCH 33/56] Need to define NEW_BASELINE even if it isn't needed *sigh* --- tests/rt.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/rt.sh b/tests/rt.sh index 4eed6c0fee..4089ee8812 100755 --- a/tests/rt.sh +++ b/tests/rt.sh @@ -258,11 +258,12 @@ INPUTDATA_ROOT_BMIC=${INPUTDATA_ROOT_BMIC:-$DISKNM/NCAR/BM_IC-20220207} shift $((OPTIND-1)) [[ $# -gt 1 ]] && usage +NEW_BASELINE=${STMP}/${USER}/FV3_RT/REGRESSION_TEST + if [[ $CREATE_BASELINE == true ]]; then # # prepare new regression test directory # - NEW_BASELINE=${STMP}/${USER}/FV3_RT/REGRESSION_TEST rm -rf "${NEW_BASELINE}" mkdir -p "${NEW_BASELINE}" fi From 5a103d737d3ed6912b98aa971ffd4115a296b9c8 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Wed, 13 Sep 2023 11:00:02 -0600 Subject: [PATCH 34/56] Fix specification of baseline and input locations for hera --- tests/machine/cheyenne.ncar | 2 +- tests/machine/hera.ncar | 12 ++++++++---- tests/rt.sh | 4 ---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/machine/cheyenne.ncar b/tests/machine/cheyenne.ncar index a74089a3a5..a25735de3d 100644 --- a/tests/machine/cheyenne.ncar +++ b/tests/machine/cheyenne.ncar @@ -1,3 +1,4 @@ + source bl_date.ncar.conf export PATH=/glade/p/ral/jntp/tools/miniconda3/4.8.3/envs/ufs-weather-model/bin:/glade/p/ral/jntp/tools/miniconda3/4.8.3/bin:$PATH export PYTHONPATH=/glade/p/ral/jntp/tools/miniconda3/4.8.3/envs/ufs-weather-model/lib/python3.8/site-packages:/glade/p/ral/jntp/tools/miniconda3/4.8.3/lib/python3.8/site-packages ECFLOW_START=/glade/p/ral/jntp/tools/miniconda3/4.8.3/envs/ufs-weather-model/bin/ecflow_start.sh @@ -15,4 +16,3 @@ cp fv3_conf/fv3_qsub.IN_cheyenne fv3_conf/fv3_qsub.IN cp fv3_conf/compile_qsub.IN_cheyenne fv3_conf/compile_qsub.IN - diff --git a/tests/machine/hera.ncar b/tests/machine/hera.ncar index a607be5eff..e8f3b373c9 100644 --- a/tests/machine/hera.ncar +++ b/tests/machine/hera.ncar @@ -1,3 +1,6 @@ + source bl_date.ncar.conf + INPUT_DATE=20230314 + module load rocoto ROCOTORUN=$(which rocotorun) ROCOTOSTAT=$(which rocotostat) @@ -15,11 +18,12 @@ COMPILE_QUEUE=batch PARTITION= - INPUT_DATE=20230314 dprefix=${RUNDIR_ROOT:-/scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model} - DISKNM=$dprefix/RT - STMP=$dprefix/RT/stmp4 - PTMP=$dprefix/RT/stmp2 + DISKNM=$dprefix/ + STMP=$dprefix/stmp4 + PTMP=$dprefix/stmp2 + RTPWD=${RTPWD:-/scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-${BL_DATE}/} + INPUTDATA_ROOT=${INPUTDATA_ROOT:-/scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/input_data/${INPUT_DATE}/} SCHEDULER=slurm cp fv3_conf/fv3_slurm.IN_hera fv3_conf/fv3_slurm.IN diff --git a/tests/rt.sh b/tests/rt.sh index 4089ee8812..edf1072ae5 100755 --- a/tests/rt.sh +++ b/tests/rt.sh @@ -241,16 +241,12 @@ if [[ $TESTS_FILE =~ '35d' ]] || [[ $TESTS_FILE =~ 'weekly' ]]; then TEST_35D=true fi -source bl_date.ncar.conf -INPUT_DATE=20230314 - if [[ "$RTPWD_NEW_BASELINE" == true ]] ; then RTPWD=${NEW_BASELINE} else RTPWD=${RTPWD:-$DISKNM/NEMSfv3gfs/develop-${BL_DATE}} fi - INPUTDATA_ROOT=${INPUTDATA_ROOT:-$DISKNM/../../input_data/${INPUT_DATE}} INPUTDATA_ROOT_WW3=${INPUTDATA_ROOT}/WW3_input_data_20220624 INPUTDATA_ROOT_BMIC=${INPUTDATA_ROOT_BMIC:-$DISKNM/NCAR/BM_IC-20220207} From fa4c938b740355fea205bbd1303ed2d5accaae6c Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Wed, 30 Aug 2023 21:32:51 +0000 Subject: [PATCH 35/56] Fix baseline location --- tests/machine/hera.ncar | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/machine/hera.ncar b/tests/machine/hera.ncar index e8f3b373c9..6dd574386a 100644 --- a/tests/machine/hera.ncar +++ b/tests/machine/hera.ncar @@ -22,7 +22,7 @@ DISKNM=$dprefix/ STMP=$dprefix/stmp4 PTMP=$dprefix/stmp2 - RTPWD=${RTPWD:-/scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-${BL_DATE}/} + RTPWD=${RTPWD:-/scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-${BL_DATE}/${RT_COMPILER^^}} INPUTDATA_ROOT=${INPUTDATA_ROOT:-/scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/input_data/${INPUT_DATE}/} SCHEDULER=slurm From f97d96dd6f4a6bf9477e0b43a4fcbae0d16a898a Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Wed, 30 Aug 2023 22:56:22 +0000 Subject: [PATCH 36/56] Update for latest hera baseline --- tests/machine/hera.ncar | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/machine/hera.ncar b/tests/machine/hera.ncar index 6dd574386a..e8f3b373c9 100644 --- a/tests/machine/hera.ncar +++ b/tests/machine/hera.ncar @@ -22,7 +22,7 @@ DISKNM=$dprefix/ STMP=$dprefix/stmp4 PTMP=$dprefix/stmp2 - RTPWD=${RTPWD:-/scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-${BL_DATE}/${RT_COMPILER^^}} + RTPWD=${RTPWD:-/scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-${BL_DATE}/} INPUTDATA_ROOT=${INPUTDATA_ROOT:-/scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/input_data/${INPUT_DATE}/} SCHEDULER=slurm From 92f426c3c28776cab9ec56dbc0738d60238a576c Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Wed, 13 Sep 2023 17:57:32 +0000 Subject: [PATCH 37/56] Working on Hera, needs refinement of python hard-coding --- tests/auto/jobs/bl.py | 3 +-- tests/auto/jobs/rt.py | 10 ++++------ tests/auto/rt_auto.py | 2 +- tests/machine/hera.ncar | 1 + 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/tests/auto/jobs/bl.py b/tests/auto/jobs/bl.py index 17ace97ef7..0074a43fdf 100644 --- a/tests/auto/jobs/bl.py +++ b/tests/auto/jobs/bl.py @@ -141,8 +141,7 @@ def clone_pr_repo(job_obj, workdir): def post_process(job_obj, pr_repo_loc, repo_dir_str, newbldir, bldir, bldate, blstore): logger = logging.getLogger('BL/MOVE_RT_LOGS') - rt_log = f'tests/RegressionTests_{job_obj.machine}'\ - f'.{job_obj.compiler}.log' + rt_log = f'tests/RegressionTests_{job_obj.machine}.log' filepath = f'{pr_repo_loc}/{rt_log}' rt_dir, logfile_pass = process_logfile(job_obj, filepath) if logfile_pass: diff --git a/tests/auto/jobs/rt.py b/tests/auto/jobs/rt.py index 31fbdc599a..66aa1ab58c 100644 --- a/tests/auto/jobs/rt.py +++ b/tests/auto/jobs/rt.py @@ -20,7 +20,7 @@ def run_regression_test(job_obj, pr_repo_loc): pr_repo_loc]] elif job_obj.compiler == 'intel': rt_command = [[f'export RT_COMPILER="{job_obj.compiler}" && export RUNDIR_ROOT={job_obj.workdir} && cd tests ' - f'&& /bin/bash --login ./rt.sh -e -a {job_obj.account} -p {job_obj.machine}', pr_repo_loc]] + f'&& /bin/bash --login ./rt.sh -e -a {job_obj.account} -p {job_obj.machine} -s machine/{job_obj.machine}.ncar', pr_repo_loc]] job_obj.run_commands(logger, rt_command) @@ -38,7 +38,6 @@ def clone_pr_repo(job_obj): logger = logging.getLogger('RT/CLONE_PR_REPO') repo_name = job_obj.preq_dict['preq'].head.repo.name branch = job_obj.preq_dict['preq'].head.ref - #git_url = job_obj.preq_dict['preq'].head.repo.html_url.split('//') git_ssh_url = job_obj.preq_dict['preq'].head.repo.ssh_url logger.debug(f'GIT SSH_URL: {git_ssh_url}') logger.info('Starting repo clone') @@ -67,8 +66,7 @@ def clone_pr_repo(job_obj): def post_process(job_obj, pr_repo_loc, repo_dir_str, branch): ''' This is the callback function associated with the "RT" command ''' logger = logging.getLogger('RT/MOVE_RT_LOGS') - rt_log = f'tests/RegressionTests_{job_obj.machine}'\ - f'.{job_obj.compiler}.log' + rt_log = f'tests/logs/RegressionTests_{job_obj.machine}.log' filepath = f'{pr_repo_loc}/{rt_log}' rt_dir, logfile_pass = process_logfile(job_obj, filepath) if logfile_pass: @@ -108,7 +106,7 @@ def process_logfile(job_obj, logfile): else: logger.critical(f'Could not find {job_obj.machine}' f'.{job_obj.compiler} ' - f'{job_obj.preq_dict["action"]} log') + f'{job_obj.preq_dict["action"]} log:\n{logfile}') print(f'Could not find {job_obj.machine}.{job_obj.compiler} ' - f'{job_obj.preq_dict["action"]} log') + f'{job_obj.preq_dict["action"]} log:\n{logfile}') raise FileNotFoundError diff --git a/tests/auto/rt_auto.py b/tests/auto/rt_auto.py index b427155ba9..757739722e 100644 --- a/tests/auto/rt_auto.py +++ b/tests/auto/rt_auto.py @@ -185,7 +185,7 @@ def comment_text_append(self, newtext): def remove_pr_label(self): ''' Removes the PR label that initiated the job run from PR ''' self.logger.info(f'Removing Label: {self.preq_dict["label"]}') - self.preq_dict['preq'].remove_from_labels(self.preq_dict['label']) +# self.preq_dict['preq'].remove_from_labels(self.preq_dict['label']) def check_label_before_job_start(self): # LETS Check the label still exists before the start of the job in the diff --git a/tests/machine/hera.ncar b/tests/machine/hera.ncar index e8f3b373c9..da14f83b44 100644 --- a/tests/machine/hera.ncar +++ b/tests/machine/hera.ncar @@ -24,6 +24,7 @@ PTMP=$dprefix/stmp2 RTPWD=${RTPWD:-/scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-${BL_DATE}/} INPUTDATA_ROOT=${INPUTDATA_ROOT:-/scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/input_data/${INPUT_DATE}/} + INPUTDATA_ROOT_BMIC=/scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/BM_IC-20220207 SCHEDULER=slurm cp fv3_conf/fv3_slurm.IN_hera fv3_conf/fv3_slurm.IN From f5d65f13bd85881b2a0a4f8172124ad08b698184 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Wed, 13 Sep 2023 12:09:21 -0600 Subject: [PATCH 38/56] Paths for Cheyenne --- tests/machine/cheyenne.ncar | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/machine/cheyenne.ncar b/tests/machine/cheyenne.ncar index a25735de3d..fa1b78beb5 100644 --- a/tests/machine/cheyenne.ncar +++ b/tests/machine/cheyenne.ncar @@ -12,6 +12,11 @@ DISKNM=$dprefix/RT STMP=$dprefix/RT PTMP=$dprefix/RT + RTPWD=${RTPWD:-/glade/p/ral/jntp/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-${BL_DATE}/} + INPUTDATA_ROOT=${INPUTDATA_ROOT:-/scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/input_data/${INPUT_DATE}/} + INPUTDATA_ROOT_BMIC=/glade/p/ral/jntp/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/BM_IC-20220207 + + SCHEDULER=pbs cp fv3_conf/fv3_qsub.IN_cheyenne fv3_conf/fv3_qsub.IN cp fv3_conf/compile_qsub.IN_cheyenne fv3_conf/compile_qsub.IN From 46ec486d6269ff718309803a52e85cf04fbbdd50 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Wed, 13 Sep 2023 22:13:32 -0600 Subject: [PATCH 39/56] Fix Cheyenne data location --- tests/machine/cheyenne.ncar | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/machine/cheyenne.ncar b/tests/machine/cheyenne.ncar index fa1b78beb5..0e71913ea7 100644 --- a/tests/machine/cheyenne.ncar +++ b/tests/machine/cheyenne.ncar @@ -13,7 +13,7 @@ STMP=$dprefix/RT PTMP=$dprefix/RT RTPWD=${RTPWD:-/glade/p/ral/jntp/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-${BL_DATE}/} - INPUTDATA_ROOT=${INPUTDATA_ROOT:-/scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/input_data/${INPUT_DATE}/} + INPUTDATA_ROOT=${INPUTDATA_ROOT:-/glade/p/ral/jntp/CCPP_regression_testing/NCAR_ufs-weather-model/input_data/${INPUT_DATE}/} INPUTDATA_ROOT_BMIC=/glade/p/ral/jntp/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/BM_IC-20220207 From 0537c1bdcb7a90bc75ad73423fbf7a93977c7de4 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Thu, 14 Sep 2023 17:45:08 -0600 Subject: [PATCH 40/56] Pass through baseline and new baseline locations as command-line arguments --- tests/auto/jobs/bl.py | 47 ++++++++++++++++++++++++++++++++++--------- tests/auto/jobs/rt.py | 18 ++++++++--------- tests/auto/rt_auto.py | 33 ++++++++++++------------------ 3 files changed, 59 insertions(+), 39 deletions(-) diff --git a/tests/auto/jobs/bl.py b/tests/auto/jobs/bl.py index 0074a43fdf..ef614a5c52 100644 --- a/tests/auto/jobs/bl.py +++ b/tests/auto/jobs/bl.py @@ -18,27 +18,54 @@ def run(job_obj): def set_directories(job_obj): logger = logging.getLogger('BL/SET_DIRECTORIES') - if job_obj.machine == 'hera': + workdir = '' + blstore = '' + newbldir = '' + machine = job_obj.clargs.machine + if machine == 'hera': rt_dir = '/scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/' blstore = f'{rt_dir}/baselines' newbldir = f'{rt_dir}/FV3_RT/'\ f'REGRESSION_TEST_{job_obj.compiler.upper()}' - elif job_obj.machine == 'cheyenne': + elif machine == 'jet': + workdir = '/lfs4/HFIP/h-nems/emc.nemspara/autort/pr' + blstore = '/lfs4/HFIP/h-nems/emc.nemspara/RT/NEMSfv3gfs/' + rtbldir = '/lfs4/HFIP/h-nems/emc.nemspara/RT_BASELINE/'\ + f'emc.nemspara/FV3_RT/REGRESSION_TEST_{job_obj.compiler.upper()}' + elif machine == 'gaea': + workdir = '/lustre/f2/pdata/ncep/emc.nemspara/autort/pr' + blstore = '/lustre/f2/pdata/ncep_shared/emc.nemspara/RT/NEMSfv3gfs' + rtbldir = '/lustre/f2/scratch/emc.nemspara/FV3_RT/'\ + f'REGRESSION_TEST_{job_obj.compiler.upper()}' + elif machine == 'orion': + workdir = '/work/noaa/nems/emc.nemspara/autort/pr' + blstore = '/work/noaa/nems/emc.nemspara/RT/NEMSfv3gfs' + rtbldir = '/work/noaa/stmp/bcurtis/stmp/bcurtis/FV3_RT/'\ + f'REGRESSION_TEST_{job_obj.compiler.upper()}' + elif machine == 'cheyenne': workdir = '/glade/scratch/epicufsrt/GMTB/ufs-weather-model/RT/auto_RT/Pull_Requests' blstore = '/glade/scratch/epicufsrt/GMTB/ufs-weather-model/RT/NCAR' newbldir = '/glade/scratch/epicufsrt/GMTB/ufs-weather-model/RT/'\ f'REGRESSION_TEST_{job_obj.compiler.upper()}' - else: - logger.critical(f'Machine {job_obj.machine} is not supported for this job') - raise KeyError - workdir = job_obj.workdir + if job_obj.clargs.workdir is not None: + workdir = job_obj.clargs.workdir + if job_obj.clargs.blstore is not None: + blstore = job_obj.clargs.blstore + if job_obj.clargs.newbldir is not None: + newbldir = job_obj.clargs.newbldir - logger.info(f'machine: {job_obj.machine}') + logger.info(f'machine: {machine}') logger.info(f'workdir: {workdir}') logger.info(f'blstore: {blstore}') logger.info(f'newbldir: {newbldir}') + if not workdir or not blstore or not newbldir: + logger.critical(f'One of workdir, blstore, or newbldir has not been specified') + logger.critical(f'Provide these on the command line or specify a supported machine') + raise KeyError + + return workdir, newbldir, blstore @@ -89,11 +116,11 @@ def run_regression_test(job_obj, pr_repo_loc): logger = logging.getLogger('BL/RUN_REGRESSION_TEST') if job_obj.compiler == 'gnu': rt_command = [[f'export RT_COMPILER="{job_obj.compiler}" && cd tests ' - '&& /bin/bash --login ./rt.sh -e -a {job_obj.account} -c -l rt_gnu.conf -p {job_obj.machine} -z {job_obj.workdir} -k', + '&& /bin/bash --login ./rt.sh -e -a {job_obj.clargs.account} -c -l rt_gnu.conf -p {job_obj.clargs.machine} -z {job_obj.clargs.workdir} -k', pr_repo_loc]] elif job_obj.compiler == 'intel': rt_command = [[f'export RT_COMPILER="{job_obj.compiler}" && cd tests ' - '&& /bin/bash --login ./rt.sh -e -a {job_obj.account} -c -p {job_obj.machine} -z {job_obj.workdir} -k', pr_repo_loc]] + '&& /bin/bash --login ./rt.sh -e -a {job_obj.clargs.account} -c -p {job_obj.clargs.machine} -z {job_obj.clargs.workdir} -k', pr_repo_loc]] job_obj.run_commands(logger, rt_command) @@ -141,7 +168,7 @@ def clone_pr_repo(job_obj, workdir): def post_process(job_obj, pr_repo_loc, repo_dir_str, newbldir, bldir, bldate, blstore): logger = logging.getLogger('BL/MOVE_RT_LOGS') - rt_log = f'tests/RegressionTests_{job_obj.machine}.log' + rt_log = f'tests/RegressionTests_{job_obj.clargs.machine}.log' filepath = f'{pr_repo_loc}/{rt_log}' rt_dir, logfile_pass = process_logfile(job_obj, filepath) if logfile_pass: diff --git a/tests/auto/jobs/rt.py b/tests/auto/jobs/rt.py index 66aa1ab58c..33027ef369 100644 --- a/tests/auto/jobs/rt.py +++ b/tests/auto/jobs/rt.py @@ -15,12 +15,12 @@ def run_regression_test(job_obj, pr_repo_loc): logger = logging.getLogger('RT/RUN_REGRESSION_TEST') workflow_flag='-e' if job_obj.compiler == 'gnu': - rt_command = [[f'export RT_COMPILER="{job_obj.compiler}" && export RUNDIR_ROOT={job_obj.workdir} && cd tests ' - f'&& /bin/bash --login ./rt.sh -e -l rt_gnu.conf -a {job_obj.account} -p {job_obj.machine}', + rt_command = [[f'export RT_COMPILER="{job_obj.compiler}" && export RUNDIR_ROOT={job_obj.clargs.workdir} && cd tests ' + f'&& /bin/bash --login ./rt.sh -e -l rt_gnu.conf -a {job_obj.clargs.account} -p {job_obj.clargs.machine}', pr_repo_loc]] elif job_obj.compiler == 'intel': - rt_command = [[f'export RT_COMPILER="{job_obj.compiler}" && export RUNDIR_ROOT={job_obj.workdir} && cd tests ' - f'&& /bin/bash --login ./rt.sh -e -a {job_obj.account} -p {job_obj.machine} -s machine/{job_obj.machine}.ncar', pr_repo_loc]] + rt_command = [[f'export RT_COMPILER="{job_obj.compiler}" && export RUNDIR_ROOT={job_obj.clargs.workdir} && cd tests ' + f'&& /bin/bash --login ./rt.sh -e -a {job_obj.clargs.account} -p {job_obj.clargs.machine} -s machine/{job_obj.clargs.machine}.ncar -n control_p8 intel', pr_repo_loc]] job_obj.run_commands(logger, rt_command) @@ -41,7 +41,7 @@ def clone_pr_repo(job_obj): git_ssh_url = job_obj.preq_dict['preq'].head.repo.ssh_url logger.debug(f'GIT SSH_URL: {git_ssh_url}') logger.info('Starting repo clone') - repo_dir_str = f'{job_obj.workdir}/'\ + repo_dir_str = f'{job_obj.clargs.workdir}/'\ f'{str(job_obj.preq_dict["preq"].id)}/'\ f'{datetime.datetime.now().strftime("%Y%m%d%H%M%S")}' pr_repo_loc = f'{repo_dir_str}/{repo_name}' @@ -66,7 +66,7 @@ def clone_pr_repo(job_obj): def post_process(job_obj, pr_repo_loc, repo_dir_str, branch): ''' This is the callback function associated with the "RT" command ''' logger = logging.getLogger('RT/MOVE_RT_LOGS') - rt_log = f'tests/logs/RegressionTests_{job_obj.machine}.log' + rt_log = f'tests/logs/RegressionTests_{job_obj.clargs.machine}.log' filepath = f'{pr_repo_loc}/{rt_log}' rt_dir, logfile_pass = process_logfile(job_obj, filepath) if logfile_pass: @@ -74,7 +74,7 @@ def post_process(job_obj, pr_repo_loc, repo_dir_str, branch): move_rt_commands = [ [f'git pull --ff-only origin {branch}', pr_repo_loc], [f'git add {rt_log}', pr_repo_loc], - [f'git commit -m "[AutoRT] {job_obj.machine}' + [f'git commit -m "[AutoRT] {job_obj.clargs.machine}' f'.{job_obj.compiler} Job Completed.\n\n\n' 'on-behalf-of @NCAR "', pr_repo_loc], @@ -104,9 +104,9 @@ def process_logfile(job_obj, logfile): return rt_dir, True job_obj.job_failed(logger, f'{job_obj.preq_dict["action"]}') else: - logger.critical(f'Could not find {job_obj.machine}' + logger.critical(f'Could not find {job_obj.clargs.machine}' f'.{job_obj.compiler} ' f'{job_obj.preq_dict["action"]} log:\n{logfile}') - print(f'Could not find {job_obj.machine}.{job_obj.compiler} ' + print(f'Could not find {job_obj.clargs.machine}.{job_obj.compiler} ' f'{job_obj.preq_dict["action"]} log:\n{logfile}') raise FileNotFoundError diff --git a/tests/auto/rt_auto.py b/tests/auto/rt_auto.py index 757739722e..405a187926 100644 --- a/tests/auto/rt_auto.py +++ b/tests/auto/rt_auto.py @@ -123,28 +123,29 @@ def delete_rt_dirs(in_dir, machine, workdir): logging.debug(f'{rt_dir}/{match} does not exist, not attempting to remove') -def get_preqs_with_actions(repos, machine, account, workdir, ghinterface_obj, actions): +def get_preqs_with_actions(repos, args, ghinterface_obj, actions): ''' Create list of dictionaries of a pull request and its machine label and action ''' logger = logging.getLogger('GET_PREQS_WITH_ACTIONS') + logger.info('Getting Pull Requests with Actions') gh_preqs = [ghinterface_obj.client.get_repo(repo['address']) .get_pulls(state='open', sort='created', base=repo['base']) for repo in repos] each_pr = [preq for gh_preq in gh_preqs for preq in gh_preq] - delete_pr_dirs(each_pr, machine, workdir) +# delete_pr_dirs(each_pr, args.machine, args.workdir) preq_labels = [{'preq': pr, 'label': label} for pr in each_pr for label in pr.get_labels()] jobs = [] # return_preq = [] for pr_label in preq_labels: - compiler, match = set_action_from_label(machine, actions, + compiler, match = set_action_from_label(args.machine, actions, pr_label['label']) if match: pr_label['action'] = match # return_preq.append(pr_label.copy()) - jobs.append(Job(pr_label.copy(), ghinterface_obj, machine, account, compiler, workdir)) + jobs.append(Job(pr_label.copy(), ghinterface_obj, args, compiler)) return jobs @@ -166,16 +167,14 @@ class Job: provided by the bash script ''' - def __init__(self, preq_dict, ghinterface_obj, machine, account, compiler, workdir): + def __init__(self, preq_dict, ghinterface_obj, args, compiler): self.logger = logging.getLogger('JOB') self.preq_dict = preq_dict self.job_mod = importlib.import_module( f'jobs.{self.preq_dict["action"].lower()}') self.ghinterface_obj = ghinterface_obj - self.machine = machine - self.account = account + self.clargs = args self.compiler = compiler - self.workdir = workdir self.comment_text = '***Automated RT Failure Notification***\n' self.failed_tests = [] @@ -190,7 +189,7 @@ def remove_pr_label(self): def check_label_before_job_start(self): # LETS Check the label still exists before the start of the job in the # case of multiple jobs - label_to_check = f'{self.machine}'\ + label_to_check = f'{self.clargs.machine}'\ f'-{self.compiler}'\ f'-{self.preq_dict["action"]}' labels = self.preq_dict['preq'].get_labels() @@ -224,7 +223,7 @@ def run_commands(self, logger, commands_with_cwd): def run(self): logger = logging.getLogger('JOB/RUN') logger.info(f'Starting Job: {self.preq_dict["label"]}') - self.comment_text_append(newtext=f'Machine: {self.machine}') + self.comment_text_append(newtext=f'Machine: {self.clargs.machine}') self.comment_text_append(f'Compiler: {self.compiler}') self.comment_text_append(f'Job: {self.preq_dict["action"]}') if self.check_label_before_job_start(): @@ -245,7 +244,7 @@ def send_comment_text(self): logger.info(f'Comment Text: {self.comment_text}') self.comment_text_append('Please make changes and add ' 'the following label back: ' - f'{self.machine}' + f'{self.clargs.machine}' f'-{self.compiler}' f'-{self.preq_dict["action"]}') @@ -287,22 +286,17 @@ def main(): parser.add_argument('-m','--machine', help='current machine name', required=True) parser.add_argument('-a','--account', help='account to charge', required=True) parser.add_argument('-w','--workdir', help='directory where tests will be staged and run', required=False) + parser.add_argument('-b','--baseline', help='directory where baseline data is stored', required=False) + parser.add_argument('--new_baseline', help='if creating a new baseline, directory where new baseline data is stored', required=False) parser.add_argument('-d','--debug', help='Set logging to more verbose output', action='store_true') args = parser.parse_args() - machine = args.machine - account = args.account - workdir = args.workdir - if args.debug: logger.info('Setting logging level to debug') for handler in logger.handlers: handler.setLevel(logging.DEBUG) - logger.info(f'Machine: {machine}') - logger.info(f'Working directory: {workdir}') - # setup environment logger.info('Getting the environment setup') repos, actions = setup_env() @@ -315,8 +309,7 @@ def main(): # and turn them into Job objects logger.info('Getting all pull requests, ' 'labels and actions applicable to this machine.') - jobs = get_preqs_with_actions(repos, machine, account, workdir, - ghinterface_obj, actions) + jobs = get_preqs_with_actions(repos, args, ghinterface_obj, actions) [job.run() for job in jobs] logger.info('Script Finished') From 4c6f62a048645b58f87a1dfe1b4742c0df1376a4 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Fri, 15 Sep 2023 15:24:03 -0600 Subject: [PATCH 41/56] More improvements and configurability to whole system - Fix bl.py for creating new baseline, make changes back-compatible - Modify rt.sh to allow custom NEW_BASELINE path - Implement command-line args for specifying baseline and creating new one - Rename some variables to be consistent with new command-line args - Remove outdated, commented code - Fix code for finding baseline date - Fix "job_failed" function to print exception details by raising if applicable --- tests/auto/jobs/bl.py | 94 ++++++++++++++++--------------------------- tests/auto/rt_auto.py | 14 ++++--- tests/rt.sh | 2 +- 3 files changed, 44 insertions(+), 66 deletions(-) diff --git a/tests/auto/jobs/bl.py b/tests/auto/jobs/bl.py index ef614a5c52..b9155e96bd 100644 --- a/tests/auto/jobs/bl.py +++ b/tests/auto/jobs/bl.py @@ -7,66 +7,68 @@ def run(job_obj): logger = logging.getLogger('BL/RUN') - workdir, newbldir, blstore = set_directories(job_obj) + workdir, new_baseline, blstore = set_directories(job_obj) pr_repo_loc, repo_dir_str = clone_pr_repo(job_obj, workdir) bldate = get_bl_date(job_obj, pr_repo_loc) bldir = f'{blstore}/main-{bldate}/{job_obj.compiler.upper()}' bldirbool = check_for_bl_dir(bldir, job_obj) run_regression_test(job_obj, pr_repo_loc) - post_process(job_obj, pr_repo_loc, repo_dir_str, newbldir, bldir, bldate, blstore) + post_process(job_obj, pr_repo_loc, repo_dir_str, new_baseline, bldir, bldate, blstore) def set_directories(job_obj): logger = logging.getLogger('BL/SET_DIRECTORIES') workdir = '' blstore = '' - newbldir = '' + new_baseline = '' machine = job_obj.clargs.machine if machine == 'hera': - rt_dir = '/scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/' - blstore = f'{rt_dir}/baselines' - newbldir = f'{rt_dir}/FV3_RT/'\ + rt_dir = '/scratch1/NCEPDEV/nems/emc.nemspara/' + workdir = f'{rt_dir}/autort/pr' + blstore = f'{rt_dir}/RT/NEMSfv3gfs' + new_baseline = f'{rt_dir}/FV3_RT/'\ f'REGRESSION_TEST_{job_obj.compiler.upper()}' elif machine == 'jet': - workdir = '/lfs4/HFIP/h-nems/emc.nemspara/autort/pr' - blstore = '/lfs4/HFIP/h-nems/emc.nemspara/RT/NEMSfv3gfs/' - rtbldir = '/lfs4/HFIP/h-nems/emc.nemspara/RT_BASELINE/'\ + rt_dir = '/lfs4/HFIP/h-nems/emc.nemspara/' + workdir = f'{rt_dir}/autort/pr' + blstore = f'{rt_dir}/RT/NEMSfv3gfs' + new_baseline = '{rt_dir}/RT_BASELINE/'\ f'emc.nemspara/FV3_RT/REGRESSION_TEST_{job_obj.compiler.upper()}' elif machine == 'gaea': workdir = '/lustre/f2/pdata/ncep/emc.nemspara/autort/pr' blstore = '/lustre/f2/pdata/ncep_shared/emc.nemspara/RT/NEMSfv3gfs' - rtbldir = '/lustre/f2/scratch/emc.nemspara/FV3_RT/'\ + new_baseline = '/lustre/f2/scratch/emc.nemspara/FV3_RT/'\ f'REGRESSION_TEST_{job_obj.compiler.upper()}' elif machine == 'orion': workdir = '/work/noaa/nems/emc.nemspara/autort/pr' blstore = '/work/noaa/nems/emc.nemspara/RT/NEMSfv3gfs' - rtbldir = '/work/noaa/stmp/bcurtis/stmp/bcurtis/FV3_RT/'\ + new_baseline = '/work/noaa/stmp/bcurtis/stmp/bcurtis/FV3_RT/'\ f'REGRESSION_TEST_{job_obj.compiler.upper()}' elif machine == 'cheyenne': - workdir = '/glade/scratch/epicufsrt/GMTB/ufs-weather-model/RT/auto_RT/Pull_Requests' - blstore = '/glade/scratch/epicufsrt/GMTB/ufs-weather-model/RT/NCAR' - newbldir = '/glade/scratch/epicufsrt/GMTB/ufs-weather-model/RT/'\ + workdir = '/glade/scratch/dtcufsrt/autort/tests/auto/pr' + blstore = '/glade/p/ral/jntp/GMTB/ufs-weather-model/RT/NEMSfv3gfs' + new_baseline = '/glade/scratch/dtcufsrt/FV3_RT/'\ f'REGRESSION_TEST_{job_obj.compiler.upper()}' - if job_obj.clargs.workdir is not None: + if job_obj.clargs.workdir: workdir = job_obj.clargs.workdir - if job_obj.clargs.blstore is not None: - blstore = job_obj.clargs.blstore - if job_obj.clargs.newbldir is not None: - newbldir = job_obj.clargs.newbldir + if job_obj.clargs.baseline: + blstore = job_obj.clargs.baseline + if job_obj.clargs.new_baseline: + new_baseline = job_obj.clargs.new_baseline logger.info(f'machine: {machine}') logger.info(f'workdir: {workdir}') logger.info(f'blstore: {blstore}') - logger.info(f'newbldir: {newbldir}') + logger.info(f'new_baseline: {new_baseline}') - if not workdir or not blstore or not newbldir: - logger.critical(f'One of workdir, blstore, or newbldir has not been specified') + if not workdir or not blstore or not new_baseline: + logger.critical(f'One of workdir, blstore, or new_baseline has not been specified') logger.critical(f'Provide these on the command line or specify a supported machine') raise KeyError - return workdir, newbldir, blstore + return workdir, new_baseline, blstore def check_for_bl_dir(bldir, job_obj): @@ -89,38 +91,15 @@ def create_bl_dir(bldir, job_obj): raise FileNotFoundError -#def get_bl_date(job_obj): -# logger = logging.getLogger('BL/GET_BL_DATE') -# for line in job_obj.preq_dict['preq'].body.splitlines(): -# if 'BL_DATE:' in line: -# bldate = line -# bldate = bldate.replace('BL_DATE:', '') -# bldate = bldate.replace(' ', '') -# if len(bldate) != 8: -# print(f'Date: {bldate} is not formatted YYYYMMDD') -# raise ValueError -# logger.info(f'BL_DATE: {bldate}') -# bl_format = '%Y%m%d' -# try: -# datetime.datetime.strptime(bldate, bl_format) -# except ValueError: -# logger.info(f'Date {bldate} is not formatted YYYYMMDD') -# raise ValueError -# return bldate -# logger.critical('"BL_DATE:YYYYMMDD" needs to be in the PR body.'\ -# 'On its own line. Stopping') -# raise ValueError - - def run_regression_test(job_obj, pr_repo_loc): logger = logging.getLogger('BL/RUN_REGRESSION_TEST') if job_obj.compiler == 'gnu': rt_command = [[f'export RT_COMPILER="{job_obj.compiler}" && cd tests ' - '&& /bin/bash --login ./rt.sh -e -a {job_obj.clargs.account} -c -l rt_gnu.conf -p {job_obj.clargs.machine} -z {job_obj.clargs.workdir} -k', + f'&& /bin/bash --login ./rt.sh -e -a {job_obj.clargs.account} -c -l rt_gnu.conf -p {job_obj.clargs.machine} -z {job_obj.clargs.workdir} -k', pr_repo_loc]] elif job_obj.compiler == 'intel': - rt_command = [[f'export RT_COMPILER="{job_obj.compiler}" && cd tests ' - '&& /bin/bash --login ./rt.sh -e -a {job_obj.clargs.account} -c -p {job_obj.clargs.machine} -z {job_obj.clargs.workdir} -k', pr_repo_loc]] + rt_command = [[f'export RT_COMPILER="{job_obj.compiler}" && export RUNDIR_ROOT={job_obj.clargs.workdir} && export NEW_BASELINE={job_obj.clargs.new_baseline} && cd tests ' + f'&& /bin/bash --login ./rt.sh -e -a {job_obj.clargs.account} -c -p {job_obj.clargs.machine} -s machine/{job_obj.clargs.machine}.ncar -n control_p8 intel -k', pr_repo_loc]] job_obj.run_commands(logger, rt_command) @@ -166,14 +145,14 @@ def clone_pr_repo(job_obj, workdir): return pr_repo_loc, repo_dir_str -def post_process(job_obj, pr_repo_loc, repo_dir_str, newbldir, bldir, bldate, blstore): +def post_process(job_obj, pr_repo_loc, repo_dir_str, new_baseline, bldir, bldate, blstore): logger = logging.getLogger('BL/MOVE_RT_LOGS') - rt_log = f'tests/RegressionTests_{job_obj.clargs.machine}.log' + rt_log = f'tests/logs/RegressionTests_{job_obj.clargs.machine}.log' filepath = f'{pr_repo_loc}/{rt_log}' rt_dir, logfile_pass = process_logfile(job_obj, filepath) if logfile_pass: create_bl_dir(bldir, job_obj) - move_bl_command = [[f'mv {newbldir}/* {bldir}/', pr_repo_loc]] + move_bl_command = [[f'mv {new_baseline}/* {bldir}/', pr_repo_loc]] job_obj.run_commands(logger, move_bl_command) job_obj.comment_text_append('[BL] Baseline creation and move successful') logger.info('Starting RT Job') @@ -184,15 +163,13 @@ def post_process(job_obj, pr_repo_loc, repo_dir_str, newbldir, bldir, bldate, bl def get_bl_date(job_obj, pr_repo_loc): logger = logging.getLogger('BL/UPDATE_RT_NCAR_SH') BLDATEFOUND = False - with open(f'{pr_repo_loc}/tests/rt.sh', 'r') as f: + with open(f'{pr_repo_loc}/tests/bl_date.ncar.conf', 'r') as f: for line in f: if 'BL_DATE=' in line: logger.info('Found BL_DATE in line') BLDATEFOUND = True - bldate = line + bldate = line.split('=')[1].strip() bldate = bldate.rstrip('\n') - bldate = bldate.replace('BL_DATE=', '') - bldate = bldate.strip(' ') logger.info(f'bldate is "{bldate}"') logger.info(f'Type bldate: {type(bldate)}') bl_format = '%Y%m%d' @@ -229,7 +206,6 @@ def process_logfile(job_obj, logfile): logger.critical(f'Log file exists but is not complete') job_obj.job_failed(logger, f'{job_obj.preq_dict["action"]}') else: - logger.critical(f'Could not find {job_obj.machine}' - f'.{job_obj.compiler} ' - f'{job_obj.preq_dict["action"]} log') + logger.critical(f'Could not find {job_obj.clargs.machine}.{job_obj.compiler} ' + f'{job_obj.preq_dict["action"]} log: {logfile}') raise FileNotFoundError diff --git a/tests/auto/rt_auto.py b/tests/auto/rt_auto.py index 405a187926..894aa54451 100644 --- a/tests/auto/rt_auto.py +++ b/tests/auto/rt_auto.py @@ -235,7 +235,7 @@ def run(self): except Exception: self.job_failed(logger, 'run()') logger.info('Sending comment text') - self.send_comment_text() +# self.send_comment_text() else: logger.info(f'Cannot find label {self.preq_dict["label"]}') @@ -250,13 +250,15 @@ def send_comment_text(self): self.preq_dict['preq'].create_issue_comment(self.comment_text) - def job_failed(self, logger, job_name, exception=Exception, STDOUT=False, + def job_failed(self, logger, job_name, exception=None, STDOUT=False, out=None, err=None): - logger.critical(f'{job_name} FAILED. Exception:{exception}') + logger.critical(f'{job_name} FAILED.') if STDOUT: logger.critical(f'STDOUT: {[item for item in out if not None]}') logger.critical(f'STDERR: {[eitem for eitem in err if not None]}') + if exception is not None: + raise def setup_env(name='ufs-weather-model', address='NCAR/ufs-weather-model', base='main'): # Dictionary of GitHub repositories to check @@ -285,9 +287,9 @@ def main(): parser = argparse.ArgumentParser() parser.add_argument('-m','--machine', help='current machine name', required=True) parser.add_argument('-a','--account', help='account to charge', required=True) - parser.add_argument('-w','--workdir', help='directory where tests will be staged and run', required=False) - parser.add_argument('-b','--baseline', help='directory where baseline data is stored', required=False) - parser.add_argument('--new_baseline', help='if creating a new baseline, directory where new baseline data is stored', required=False) + parser.add_argument('-w','--workdir', help='directory where tests will be staged and run', required=False, default='') + parser.add_argument('-b','--baseline', help='directory where baseline data is stored', required=False, default='') + parser.add_argument('--new_baseline', help='if creating a new baseline, directory where new baseline data is stored', required=False, default='') parser.add_argument('-d','--debug', help='Set logging to more verbose output', action='store_true') args = parser.parse_args() diff --git a/tests/rt.sh b/tests/rt.sh index edf1072ae5..7f594967e0 100755 --- a/tests/rt.sh +++ b/tests/rt.sh @@ -254,7 +254,7 @@ INPUTDATA_ROOT_BMIC=${INPUTDATA_ROOT_BMIC:-$DISKNM/NCAR/BM_IC-20220207} shift $((OPTIND-1)) [[ $# -gt 1 ]] && usage -NEW_BASELINE=${STMP}/${USER}/FV3_RT/REGRESSION_TEST +NEW_BASELINE=${NEW_BASELINE:-${STMP}/${USER}/FV3_RT/REGRESSION_TEST} if [[ $CREATE_BASELINE == true ]]; then # From 560c7d7aa7ab0390b090dae20c826dd27cf2c0eb Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Fri, 15 Sep 2023 23:40:49 -0600 Subject: [PATCH 42/56] Apparently nested `source` calls doesnt work, need to re-think this... --- tests/machine/cheyenne.ncar | 1 - tests/machine/hera.ncar | 1 - tests/rt.sh | 1 + 3 files changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/machine/cheyenne.ncar b/tests/machine/cheyenne.ncar index 0e71913ea7..fd6ae59abd 100644 --- a/tests/machine/cheyenne.ncar +++ b/tests/machine/cheyenne.ncar @@ -1,4 +1,3 @@ - source bl_date.ncar.conf export PATH=/glade/p/ral/jntp/tools/miniconda3/4.8.3/envs/ufs-weather-model/bin:/glade/p/ral/jntp/tools/miniconda3/4.8.3/bin:$PATH export PYTHONPATH=/glade/p/ral/jntp/tools/miniconda3/4.8.3/envs/ufs-weather-model/lib/python3.8/site-packages:/glade/p/ral/jntp/tools/miniconda3/4.8.3/lib/python3.8/site-packages ECFLOW_START=/glade/p/ral/jntp/tools/miniconda3/4.8.3/envs/ufs-weather-model/bin/ecflow_start.sh diff --git a/tests/machine/hera.ncar b/tests/machine/hera.ncar index da14f83b44..68aa56f34e 100644 --- a/tests/machine/hera.ncar +++ b/tests/machine/hera.ncar @@ -1,4 +1,3 @@ - source bl_date.ncar.conf INPUT_DATE=20230314 module load rocoto diff --git a/tests/rt.sh b/tests/rt.sh index 7f594967e0..f1335af404 100755 --- a/tests/rt.sh +++ b/tests/rt.sh @@ -226,6 +226,7 @@ if ! [ -f "$sourcefile" ]; then die "ERROR: Invalid source file, check that valid platform has been specified\n and that source file '$sourcefile' exists" fi +source bl_date.ncar.conf source ${sourcefile:-"machine/$MACHINE_ID"} # Overwrite default RUNDIR_ROOT if environment variable RUNDIR_ROOT is set From 05ba82687a37483a9a5be1254785443c30f240d7 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Sat, 16 Sep 2023 00:16:54 -0600 Subject: [PATCH 43/56] Restore some default variables in rt.sh --- tests/rt.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/rt.sh b/tests/rt.sh index f1335af404..e57fdb9ad7 100755 --- a/tests/rt.sh +++ b/tests/rt.sh @@ -248,9 +248,9 @@ else RTPWD=${RTPWD:-$DISKNM/NEMSfv3gfs/develop-${BL_DATE}} fi -INPUTDATA_ROOT=${INPUTDATA_ROOT:-$DISKNM/../../input_data/${INPUT_DATE}} +INPUTDATA_ROOT=${INPUTDATA_ROOT:-$DISKNM/NEMSfv3gfs/input-data-20221101} INPUTDATA_ROOT_WW3=${INPUTDATA_ROOT}/WW3_input_data_20220624 -INPUTDATA_ROOT_BMIC=${INPUTDATA_ROOT_BMIC:-$DISKNM/NCAR/BM_IC-20220207} +INPUTDATA_ROOT_BMIC=${INPUTDATA_ROOT_BMIC:-$DISKNM/NEMSfv3gfs/BM_IC-20220207} shift $((OPTIND-1)) [[ $# -gt 1 ]] && usage From 527b72bbd14a31cbb5cf1f170cbaba315399c659 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Sat, 16 Sep 2023 12:21:33 -0600 Subject: [PATCH 44/56] I was wrong, nested sourcing DOES work, I was just passing the wrong files --- tests/machine/cheyenne.ncar | 1 + tests/machine/hera.ncar | 1 + tests/rt.sh | 1 - 3 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/machine/cheyenne.ncar b/tests/machine/cheyenne.ncar index fd6ae59abd..b443c8abab 100644 --- a/tests/machine/cheyenne.ncar +++ b/tests/machine/cheyenne.ncar @@ -1,3 +1,4 @@ + source ../bl_date.ncar.conf export PATH=/glade/p/ral/jntp/tools/miniconda3/4.8.3/envs/ufs-weather-model/bin:/glade/p/ral/jntp/tools/miniconda3/4.8.3/bin:$PATH export PYTHONPATH=/glade/p/ral/jntp/tools/miniconda3/4.8.3/envs/ufs-weather-model/lib/python3.8/site-packages:/glade/p/ral/jntp/tools/miniconda3/4.8.3/lib/python3.8/site-packages ECFLOW_START=/glade/p/ral/jntp/tools/miniconda3/4.8.3/envs/ufs-weather-model/bin/ecflow_start.sh diff --git a/tests/machine/hera.ncar b/tests/machine/hera.ncar index 68aa56f34e..df19d3eaf3 100644 --- a/tests/machine/hera.ncar +++ b/tests/machine/hera.ncar @@ -1,3 +1,4 @@ + source ../bl_date.ncar.conf INPUT_DATE=20230314 module load rocoto diff --git a/tests/rt.sh b/tests/rt.sh index e57fdb9ad7..7667089cd5 100755 --- a/tests/rt.sh +++ b/tests/rt.sh @@ -226,7 +226,6 @@ if ! [ -f "$sourcefile" ]; then die "ERROR: Invalid source file, check that valid platform has been specified\n and that source file '$sourcefile' exists" fi -source bl_date.ncar.conf source ${sourcefile:-"machine/$MACHINE_ID"} # Overwrite default RUNDIR_ROOT if environment variable RUNDIR_ROOT is set From 87f72847190f110e271970b15930e7a7c166bdf9 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Sat, 16 Sep 2023 12:26:51 -0600 Subject: [PATCH 45/56] Relative paths will be the death of me --- tests/machine/cheyenne.ncar | 2 +- tests/machine/hera.ncar | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/machine/cheyenne.ncar b/tests/machine/cheyenne.ncar index b443c8abab..0e71913ea7 100644 --- a/tests/machine/cheyenne.ncar +++ b/tests/machine/cheyenne.ncar @@ -1,4 +1,4 @@ - source ../bl_date.ncar.conf + source bl_date.ncar.conf export PATH=/glade/p/ral/jntp/tools/miniconda3/4.8.3/envs/ufs-weather-model/bin:/glade/p/ral/jntp/tools/miniconda3/4.8.3/bin:$PATH export PYTHONPATH=/glade/p/ral/jntp/tools/miniconda3/4.8.3/envs/ufs-weather-model/lib/python3.8/site-packages:/glade/p/ral/jntp/tools/miniconda3/4.8.3/lib/python3.8/site-packages ECFLOW_START=/glade/p/ral/jntp/tools/miniconda3/4.8.3/envs/ufs-weather-model/bin/ecflow_start.sh diff --git a/tests/machine/hera.ncar b/tests/machine/hera.ncar index df19d3eaf3..da14f83b44 100644 --- a/tests/machine/hera.ncar +++ b/tests/machine/hera.ncar @@ -1,4 +1,4 @@ - source ../bl_date.ncar.conf + source bl_date.ncar.conf INPUT_DATE=20230314 module load rocoto From ae013abf445605ba2e124d318afb9d97ee938e68 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Wed, 20 Sep 2023 10:25:07 -0600 Subject: [PATCH 46/56] More python-side improvements - Consolidate logic for rt.sh run command in rt.py and bl.py - Add envfile as a command-line argument rather than hard-coded - Add an '--additional_args' flag for passing arbitrary extra commands to rt.sh --- tests/auto/jobs/bl.py | 19 ++++++++++++------- tests/auto/jobs/rt.py | 23 +++++++++++++++-------- tests/auto/rt_auto.py | 3 +++ 3 files changed, 30 insertions(+), 15 deletions(-) diff --git a/tests/auto/jobs/bl.py b/tests/auto/jobs/bl.py index b9155e96bd..a888d89d07 100644 --- a/tests/auto/jobs/bl.py +++ b/tests/auto/jobs/bl.py @@ -93,14 +93,19 @@ def create_bl_dir(bldir, job_obj): def run_regression_test(job_obj, pr_repo_loc): logger = logging.getLogger('BL/RUN_REGRESSION_TEST') + + rt_command = 'cd tests' + rt_command += f' && export RT_COMPILER="{job_obj.compiler}"' + if job_obj.clargs.workdir: + rt_command += f' && export RUNDIR_ROOT={job_obj.clargs.workdir}' + rt_command += f' && /bin/bash --login ./rt.sh -e -a {job_obj.clargs.account} -c -p {job_obj.clargs.machine} -n control_p8 intel' if job_obj.compiler == 'gnu': - rt_command = [[f'export RT_COMPILER="{job_obj.compiler}" && cd tests ' - f'&& /bin/bash --login ./rt.sh -e -a {job_obj.clargs.account} -c -l rt_gnu.conf -p {job_obj.clargs.machine} -z {job_obj.clargs.workdir} -k', - pr_repo_loc]] - elif job_obj.compiler == 'intel': - rt_command = [[f'export RT_COMPILER="{job_obj.compiler}" && export RUNDIR_ROOT={job_obj.clargs.workdir} && export NEW_BASELINE={job_obj.clargs.new_baseline} && cd tests ' - f'&& /bin/bash --login ./rt.sh -e -a {job_obj.clargs.account} -c -p {job_obj.clargs.machine} -s machine/{job_obj.clargs.machine}.ncar -n control_p8 intel -k', pr_repo_loc]] - job_obj.run_commands(logger, rt_command) + rt_command += f' -l rt_gnu.conf' + if job_obj.clargs.envfile: + rt_command += f' -s {job_obj.clargs.envfile}' + rt_command += job_obj.clargs.additional_args + + job_obj.run_commands(logger, [[rt_command, pr_repo_loc]]) def remove_pr_data(job_obj, pr_repo_loc, repo_dir_str, rt_dir): diff --git a/tests/auto/jobs/rt.py b/tests/auto/jobs/rt.py index 33027ef369..6db49bb138 100644 --- a/tests/auto/jobs/rt.py +++ b/tests/auto/jobs/rt.py @@ -13,15 +13,22 @@ def run(job_obj): def run_regression_test(job_obj, pr_repo_loc): logger = logging.getLogger('RT/RUN_REGRESSION_TEST') - workflow_flag='-e' + + rt_command = 'cd tests' + rt_command += f' && export RT_COMPILER="{job_obj.compiler}"' + if job_obj.clargs.workdir: + rt_command += f' && export RUNDIR_ROOT={job_obj.clargs.workdir}' + rt_command += f' && /bin/bash --login ./rt.sh -e -a {job_obj.clargs.account} -p {job_obj.clargs.machine} -n control_p8 intel' if job_obj.compiler == 'gnu': - rt_command = [[f'export RT_COMPILER="{job_obj.compiler}" && export RUNDIR_ROOT={job_obj.clargs.workdir} && cd tests ' - f'&& /bin/bash --login ./rt.sh -e -l rt_gnu.conf -a {job_obj.clargs.account} -p {job_obj.clargs.machine}', - pr_repo_loc]] - elif job_obj.compiler == 'intel': - rt_command = [[f'export RT_COMPILER="{job_obj.compiler}" && export RUNDIR_ROOT={job_obj.clargs.workdir} && cd tests ' - f'&& /bin/bash --login ./rt.sh -e -a {job_obj.clargs.account} -p {job_obj.clargs.machine} -s machine/{job_obj.clargs.machine}.ncar -n control_p8 intel', pr_repo_loc]] - job_obj.run_commands(logger, rt_command) + rt_command += f' -l rt_gnu.conf' + if job_obj.clargs.envfile: + rt_command += f' -s {job_obj.clargs.envfile}' + rt_command += job_obj.clargs.additional_args + + print(rt_command) + print(pr_repo_loc) + + job_obj.run_commands(logger, [[rt_command, pr_repo_loc]]) def remove_pr_data(job_obj, pr_repo_loc, repo_dir_str, rt_dir): diff --git a/tests/auto/rt_auto.py b/tests/auto/rt_auto.py index 894aa54451..2b4c03f003 100644 --- a/tests/auto/rt_auto.py +++ b/tests/auto/rt_auto.py @@ -236,6 +236,7 @@ def run(self): self.job_failed(logger, 'run()') logger.info('Sending comment text') # self.send_comment_text() + raise else: logger.info(f'Cannot find label {self.preq_dict["label"]}') @@ -289,8 +290,10 @@ def main(): parser.add_argument('-a','--account', help='account to charge', required=True) parser.add_argument('-w','--workdir', help='directory where tests will be staged and run', required=False, default='') parser.add_argument('-b','--baseline', help='directory where baseline data is stored', required=False, default='') + parser.add_argument('-e','--envfile', help='environment file sourced by rt.sh', required=False, default='') parser.add_argument('--new_baseline', help='if creating a new baseline, directory where new baseline data is stored', required=False, default='') parser.add_argument('-d','--debug', help='Set logging to more verbose output', action='store_true') + parser.add_argument('--additional_args', help='Additional arguments to pass to rt.sh', required=False, default='') args = parser.parse_args() From 7946885a13b0b24d63c9b4ff53a040023a9013e8 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Wed, 20 Sep 2023 12:07:52 -0600 Subject: [PATCH 47/56] Getting to the final stretch here! Implement yaml config file to supplement command-line arguments --- tests/auto/jobs/bl.py | 2 +- tests/auto/jobs/rt.py | 7 ++----- tests/auto/rt_auto.py | 39 ++++++++++++++++++++++++++++++------- tests/auto/rt_auto.yaml | 6 ++++++ tests/auto/start_rt_auto.sh | 8 ++++---- 5 files changed, 45 insertions(+), 17 deletions(-) create mode 100644 tests/auto/rt_auto.yaml diff --git a/tests/auto/jobs/bl.py b/tests/auto/jobs/bl.py index a888d89d07..2075cbac96 100644 --- a/tests/auto/jobs/bl.py +++ b/tests/auto/jobs/bl.py @@ -103,7 +103,7 @@ def run_regression_test(job_obj, pr_repo_loc): rt_command += f' -l rt_gnu.conf' if job_obj.clargs.envfile: rt_command += f' -s {job_obj.clargs.envfile}' - rt_command += job_obj.clargs.additional_args + rt_command += f' {job_obj.clargs.additional_args}' job_obj.run_commands(logger, [[rt_command, pr_repo_loc]]) diff --git a/tests/auto/jobs/rt.py b/tests/auto/jobs/rt.py index 6db49bb138..4aa87229ec 100644 --- a/tests/auto/jobs/rt.py +++ b/tests/auto/jobs/rt.py @@ -18,15 +18,12 @@ def run_regression_test(job_obj, pr_repo_loc): rt_command += f' && export RT_COMPILER="{job_obj.compiler}"' if job_obj.clargs.workdir: rt_command += f' && export RUNDIR_ROOT={job_obj.clargs.workdir}' - rt_command += f' && /bin/bash --login ./rt.sh -e -a {job_obj.clargs.account} -p {job_obj.clargs.machine} -n control_p8 intel' + rt_command += f' && /bin/bash --login ./rt.sh -e -a {job_obj.clargs.account} -p {job_obj.clargs.machine}' if job_obj.compiler == 'gnu': rt_command += f' -l rt_gnu.conf' if job_obj.clargs.envfile: rt_command += f' -s {job_obj.clargs.envfile}' - rt_command += job_obj.clargs.additional_args - - print(rt_command) - print(pr_repo_loc) + rt_command += f' {job_obj.clargs.additional_args}' job_obj.run_commands(logger, [[rt_command, pr_repo_loc]]) diff --git a/tests/auto/rt_auto.py b/tests/auto/rt_auto.py index 2b4c03f003..06fd004609 100644 --- a/tests/auto/rt_auto.py +++ b/tests/auto/rt_auto.py @@ -16,6 +16,7 @@ import logging import importlib from shutil import rmtree +import yaml class GHInterface: ''' @@ -286,22 +287,46 @@ def main(): logger.info('Starting Script') parser = argparse.ArgumentParser() - parser.add_argument('-m','--machine', help='current machine name', required=True) - parser.add_argument('-a','--account', help='account to charge', required=True) - parser.add_argument('-w','--workdir', help='directory where tests will be staged and run', required=False, default='') - parser.add_argument('-b','--baseline', help='directory where baseline data is stored', required=False, default='') - parser.add_argument('-e','--envfile', help='environment file sourced by rt.sh', required=False, default='') - parser.add_argument('--new_baseline', help='if creating a new baseline, directory where new baseline data is stored', required=False, default='') + parser.add_argument('-m','--machine', help='current machine name', default='') + parser.add_argument('-a','--account', help='account to charge', default='') + parser.add_argument('-w','--workdir', help='directory where tests will be staged and run', default='') + parser.add_argument('-b','--baseline', help='directory where baseline data is stored', default='') + parser.add_argument('-e','--envfile', help='environment file sourced by rt.sh', default='') + parser.add_argument('--new_baseline', help='if creating a new baseline, directory where new baseline data is stored', default='') + parser.add_argument('-y','--yamlfile', help='yaml file to load additional arguments from', default='rt_auto.yaml') parser.add_argument('-d','--debug', help='Set logging to more verbose output', action='store_true') - parser.add_argument('--additional_args', help='Additional arguments to pass to rt.sh', required=False, default='') + parser.add_argument('--additional_args', help='Additional arguments to pass to rt.sh', default='') args = parser.parse_args() + # Get command-line arguments as a dictionary + dargs = vars(args) + + # Load yamlfile for additional arguments + with open(args.yamlfile) as f: + cfg = yaml.safe_load(f) + + # For each mandatory command-line argument, if not provided, check yaml file, and fail if not there either + mandatory = ['machine','account'] + for md in mandatory: + if not dargs[md]: + if not cfg['args'].get(md): + raise argparse.ArgumentTypeError(f'"{md}" is a required argument; you must provide it via command line or yaml config file "{args.yamlfile}"') + + # For each optional command-line argument, if it was not provided, attempt to get it from the yaml file + for arg in dargs: + if arg in mandatory: + pass + if not dargs[arg]: + if cfg['args'].get(arg): + logger.info(f"Reading argument from yaml file:\n{arg} = {cfg['args'][arg]}") + dargs[arg] = cfg['args'][arg] if args.debug: logger.info('Setting logging level to debug') for handler in logger.handlers: handler.setLevel(logging.DEBUG) + # setup environment logger.info('Getting the environment setup') repos, actions = setup_env() diff --git a/tests/auto/rt_auto.yaml b/tests/auto/rt_auto.yaml new file mode 100644 index 0000000000..6934a25bf3 --- /dev/null +++ b/tests/auto/rt_auto.yaml @@ -0,0 +1,6 @@ +args: + machine: cheyenne + account: P48503002 + workdir: /glade/p/ral/jntp/CCPP_regression_testing/NCAR_ufs-weather-model/run/ + envfile: machine/cheyenne.ncar + additional_args: -n control_p8 intel diff --git a/tests/auto/start_rt_auto.sh b/tests/auto/start_rt_auto.sh index 0b0f363978..0d2b0e5080 100755 --- a/tests/auto/start_rt_auto.sh +++ b/tests/auto/start_rt_auto.sh @@ -9,14 +9,14 @@ function usage { echo " workdir: Working directory for checking out and running tests" exit 1 } -if [ "$#" -lt 3 ]; then - echo "Need to provide machine, account, and workdir" +if [ "$#" -lt 1 ]; then + echo "Need to provide machine" usage fi machine=$1 -account=$2 -workdir=$3 +account=${2:-''} +workdir=${3:-''} if [[ $machine == hera ]]; then export PATH=/scratch1/NCEPDEV/nems/emc.nemspara/soft/miniconda3/bin:$PATH From b523dc51bc29d60f93d560b28e21a5dd6d396579 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Wed, 20 Sep 2023 16:42:11 -0600 Subject: [PATCH 48/56] I think I have all the major features that I want! Now to test on Hera... - Add git section to yaml read that gets git user information, and optionally specifies github info - Convert rt_auto.yaml to a README so users don't get other peoples' settings --- tests/auto/README.rt_auto.yaml | 19 +++++++++++++++++++ tests/auto/jobs/bl.py | 7 ++----- tests/auto/jobs/rt.py | 6 +++--- tests/auto/rt_auto.py | 29 +++++++++++++++++++---------- tests/auto/rt_auto.yaml | 6 ------ 5 files changed, 43 insertions(+), 24 deletions(-) create mode 100644 tests/auto/README.rt_auto.yaml delete mode 100644 tests/auto/rt_auto.yaml diff --git a/tests/auto/README.rt_auto.yaml b/tests/auto/README.rt_auto.yaml new file mode 100644 index 0000000000..8e9ada0ef8 --- /dev/null +++ b/tests/auto/README.rt_auto.yaml @@ -0,0 +1,19 @@ +# This README describes the format of rt_auto.yaml, which is read by rt_auto.py in order to get git- and GitHub-related settings, and optionally to provide arguments (to avoid having to type long command-line arguments). +# The required section is named 'git', and has two sub-sections, one required, and one optional: +# config (required): Specifies user.email and user.name settings tied to the user's GitHub account +# github (optional): Specifies the repository and base branch (i.e. the destination of the PR) to check +# An example configuration can be found below: +args: + machine: cheyenne + account: P48503002 + workdir: /glade/p/ral/jntp/CCPP_regression_testing/NCAR_ufs-weather-model/run/ + envfile: machine/cheyenne.ncar # The default machine file is machine/{machine_name}, but you can specify your own + additional_args: -n control_p8 intel # Specify any additional options you'd like to pass to rt.sh; here we specify to only run the "control_p8" test for intel compilers +git: + config: + user.email: user@test.edu + user.name: test_user + github: + org: NCAR + repo: ufs-weather-model + base: main diff --git a/tests/auto/jobs/bl.py b/tests/auto/jobs/bl.py index 2075cbac96..ce6714c7a0 100644 --- a/tests/auto/jobs/bl.py +++ b/tests/auto/jobs/bl.py @@ -122,9 +122,6 @@ def clone_pr_repo(job_obj, workdir): logger = logging.getLogger('BL/CLONE_PR_REPO') repo_name = job_obj.preq_dict['preq'].head.repo.name branch = job_obj.preq_dict['preq'].head.ref - #git_url = job_obj.preq_dict['preq'].head.repo.html_url.split('//') - #git_url = f'{git_url[0]}//${{ghapitoken}}@{git_url[1]}' - #logger.debug(f'GIT URL: {git_url}') git_ssh_url = job_obj.preq_dict['preq'].head.repo.ssh_url logger.debug(f'GIT SSH_URL: {git_ssh_url}') logger.info('Starting repo clone') @@ -138,9 +135,9 @@ def clone_pr_repo(job_obj, workdir): [f'git clone -b {branch} {git_ssh_url}', repo_dir_str], ['git submodule update --init --recursive', f'{repo_dir_str}/{repo_name}'], - ['git config user.email "kavulich@ucar.edu"', + [f'git config user.email {job_obj.gitargs["config"]["user.email"]}', f'{repo_dir_str}/{repo_name}'], - ['git config user.name "mkavulich"', + [f'git config user.name {job_obj.gitargs["config"]["user.name"]}', f'{repo_dir_str}/{repo_name}'] ] diff --git a/tests/auto/jobs/rt.py b/tests/auto/jobs/rt.py index 4aa87229ec..c8c1823d2c 100644 --- a/tests/auto/jobs/rt.py +++ b/tests/auto/jobs/rt.py @@ -55,9 +55,9 @@ def clone_pr_repo(job_obj): [f'git clone -b {branch} {git_ssh_url}', repo_dir_str], ['git submodule update --init --recursive', f'{repo_dir_str}/{repo_name}'], - ['git config user.email "kavulich@ucar.edu"', + [f'git config user.email {job_obj.gitargs["config"]["user.email"]}', f'{repo_dir_str}/{repo_name}'], - ['git config user.name "mkavulich"', + [f'git config user.name job_obj.gitargs["config"]["user.name"]', f'{repo_dir_str}/{repo_name}'] ] @@ -80,7 +80,7 @@ def post_process(job_obj, pr_repo_loc, repo_dir_str, branch): [f'git add {rt_log}', pr_repo_loc], [f'git commit -m "[AutoRT] {job_obj.clargs.machine}' f'.{job_obj.compiler} Job Completed.\n\n\n' - 'on-behalf-of @NCAR "', + f'on-behalf-of {job_obj.gitargs["github"]["org"]} @{job_obj.gitargs["config"]["user.name"]}"', pr_repo_loc], ['sleep 10', pr_repo_loc], [f'git push origin {branch}', pr_repo_loc] diff --git a/tests/auto/rt_auto.py b/tests/auto/rt_auto.py index 06fd004609..77a8934b44 100644 --- a/tests/auto/rt_auto.py +++ b/tests/auto/rt_auto.py @@ -124,7 +124,7 @@ def delete_rt_dirs(in_dir, machine, workdir): logging.debug(f'{rt_dir}/{match} does not exist, not attempting to remove') -def get_preqs_with_actions(repos, args, ghinterface_obj, actions): +def get_preqs_with_actions(repos, args, ghinterface_obj, actions, git_cfg): ''' Create list of dictionaries of a pull request and its machine label and action ''' logger = logging.getLogger('GET_PREQS_WITH_ACTIONS') @@ -134,7 +134,7 @@ def get_preqs_with_actions(repos, args, ghinterface_obj, actions): .get_pulls(state='open', sort='created', base=repo['base']) for repo in repos] each_pr = [preq for gh_preq in gh_preqs for preq in gh_preq] -# delete_pr_dirs(each_pr, args.machine, args.workdir) + delete_pr_dirs(each_pr, args.machine, args.workdir) preq_labels = [{'preq': pr, 'label': label} for pr in each_pr for label in pr.get_labels()] @@ -146,7 +146,7 @@ def get_preqs_with_actions(repos, args, ghinterface_obj, actions): if match: pr_label['action'] = match # return_preq.append(pr_label.copy()) - jobs.append(Job(pr_label.copy(), ghinterface_obj, args, compiler)) + jobs.append(Job(pr_label.copy(), ghinterface_obj, args, compiler, git_cfg)) return jobs @@ -168,7 +168,7 @@ class Job: provided by the bash script ''' - def __init__(self, preq_dict, ghinterface_obj, args, compiler): + def __init__(self, preq_dict, ghinterface_obj, args, compiler, gitargs): self.logger = logging.getLogger('JOB') self.preq_dict = preq_dict self.job_mod = importlib.import_module( @@ -176,6 +176,7 @@ def __init__(self, preq_dict, ghinterface_obj, args, compiler): self.ghinterface_obj = ghinterface_obj self.clargs = args self.compiler = compiler + self.gitargs = gitargs self.comment_text = '***Automated RT Failure Notification***\n' self.failed_tests = [] @@ -262,12 +263,20 @@ def job_failed(self, logger, job_name, exception=None, STDOUT=False, if exception is not None: raise -def setup_env(name='ufs-weather-model', address='NCAR/ufs-weather-model', base='main'): +def setup_env(git_cfg): # Dictionary of GitHub repositories to check + + if not git_cfg.get('repo'): + git_cfg['repo'] = 'ufs-weather-model' + if not git_cfg.get('org'): + git_cfg['org'] = 'ufs-community' + if not git_cfg.get('base'): + git_cfg['base'] = 'main' + repo_dict = [{ - 'name': name, - 'address': address, - 'base': base + 'name': git_cfg['repo'], + 'address': f"{git_cfg['org']}/{git_cfg['repo']}", + 'base': git_cfg['base'] }] # Approved Actions @@ -329,7 +338,7 @@ def main(): # setup environment logger.info('Getting the environment setup') - repos, actions = setup_env() + repos, actions = setup_env(cfg['git']['github']) # setup interface with GitHub logger.info('Setting up GitHub interface.') @@ -339,7 +348,7 @@ def main(): # and turn them into Job objects logger.info('Getting all pull requests, ' 'labels and actions applicable to this machine.') - jobs = get_preqs_with_actions(repos, args, ghinterface_obj, actions) + jobs = get_preqs_with_actions(repos, args, ghinterface_obj, actions, cfg['git']) [job.run() for job in jobs] logger.info('Script Finished') diff --git a/tests/auto/rt_auto.yaml b/tests/auto/rt_auto.yaml deleted file mode 100644 index 6934a25bf3..0000000000 --- a/tests/auto/rt_auto.yaml +++ /dev/null @@ -1,6 +0,0 @@ -args: - machine: cheyenne - account: P48503002 - workdir: /glade/p/ral/jntp/CCPP_regression_testing/NCAR_ufs-weather-model/run/ - envfile: machine/cheyenne.ncar - additional_args: -n control_p8 intel From d2620a2e10d1bacc4dc7a86debcea9cc42c7f30e Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Thu, 21 Sep 2023 04:31:45 +0000 Subject: [PATCH 49/56] Update input data for Hera --- tests/machine/hera.ncar | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/machine/hera.ncar b/tests/machine/hera.ncar index da14f83b44..56b0e3a33d 100644 --- a/tests/machine/hera.ncar +++ b/tests/machine/hera.ncar @@ -1,5 +1,5 @@ source bl_date.ncar.conf - INPUT_DATE=20230314 + INPUT_DATE=20230920 module load rocoto ROCOTORUN=$(which rocotorun) From ff76d7c255f169828495ef242de08a57174157bc Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Wed, 27 Sep 2023 14:38:39 +0000 Subject: [PATCH 50/56] Handle "workdir" as a job_obj element rather than referencing the command-line argument directly; this allows for the logic in bl.py to assign a workdir even if none is provided. --- tests/auto/jobs/bl.py | 22 +++++++++++----------- tests/auto/jobs/rt.py | 9 ++++++--- tests/auto/rt_auto.py | 9 +++++---- 3 files changed, 22 insertions(+), 18 deletions(-) diff --git a/tests/auto/jobs/bl.py b/tests/auto/jobs/bl.py index ce6714c7a0..47ce530f19 100644 --- a/tests/auto/jobs/bl.py +++ b/tests/auto/jobs/bl.py @@ -7,8 +7,8 @@ def run(job_obj): logger = logging.getLogger('BL/RUN') - workdir, new_baseline, blstore = set_directories(job_obj) - pr_repo_loc, repo_dir_str = clone_pr_repo(job_obj, workdir) + new_baseline, blstore = set_directories(job_obj) + pr_repo_loc, repo_dir_str = clone_pr_repo(job_obj) bldate = get_bl_date(job_obj, pr_repo_loc) bldir = f'{blstore}/main-{bldate}/{job_obj.compiler.upper()}' bldirbool = check_for_bl_dir(bldir, job_obj) @@ -50,25 +50,25 @@ def set_directories(job_obj): new_baseline = '/glade/scratch/dtcufsrt/FV3_RT/'\ f'REGRESSION_TEST_{job_obj.compiler.upper()}' - if job_obj.clargs.workdir: - workdir = job_obj.clargs.workdir + if not job_obj.clargs.workdir: + job_obj.workdir = workdir if job_obj.clargs.baseline: blstore = job_obj.clargs.baseline if job_obj.clargs.new_baseline: new_baseline = job_obj.clargs.new_baseline logger.info(f'machine: {machine}') - logger.info(f'workdir: {workdir}') + logger.info(f'workdir: {job_obj.workdir}') logger.info(f'blstore: {blstore}') logger.info(f'new_baseline: {new_baseline}') - if not workdir or not blstore or not new_baseline: + if not job_obj.workdir or not blstore or not new_baseline: logger.critical(f'One of workdir, blstore, or new_baseline has not been specified') logger.critical(f'Provide these on the command line or specify a supported machine') raise KeyError - return workdir, new_baseline, blstore + return new_baseline, blstore def check_for_bl_dir(bldir, job_obj): @@ -96,8 +96,8 @@ def run_regression_test(job_obj, pr_repo_loc): rt_command = 'cd tests' rt_command += f' && export RT_COMPILER="{job_obj.compiler}"' - if job_obj.clargs.workdir: - rt_command += f' && export RUNDIR_ROOT={job_obj.clargs.workdir}' + if job_obj.workdir: + rt_command += f' && export RUNDIR_ROOT={job_obj.workdir}' rt_command += f' && /bin/bash --login ./rt.sh -e -a {job_obj.clargs.account} -c -p {job_obj.clargs.machine} -n control_p8 intel' if job_obj.compiler == 'gnu': rt_command += f' -l rt_gnu.conf' @@ -117,7 +117,7 @@ def remove_pr_data(job_obj, pr_repo_loc, repo_dir_str, rt_dir): job_obj.run_commands(logger, rm_command) -def clone_pr_repo(job_obj, workdir): +def clone_pr_repo(job_obj): ''' clone the GitHub pull request repo, via command line ''' logger = logging.getLogger('BL/CLONE_PR_REPO') repo_name = job_obj.preq_dict['preq'].head.repo.name @@ -125,7 +125,7 @@ def clone_pr_repo(job_obj, workdir): git_ssh_url = job_obj.preq_dict['preq'].head.repo.ssh_url logger.debug(f'GIT SSH_URL: {git_ssh_url}') logger.info('Starting repo clone') - repo_dir_str = f'{workdir}/'\ + repo_dir_str = f'{job_obj.workdir}/'\ f'{str(job_obj.preq_dict["preq"].id)}/'\ f'{datetime.datetime.now().strftime("%Y%m%d%H%M%S")}' pr_repo_loc = f'{repo_dir_str}/{repo_name}' diff --git a/tests/auto/jobs/rt.py b/tests/auto/jobs/rt.py index c8c1823d2c..de04dcd4f3 100644 --- a/tests/auto/jobs/rt.py +++ b/tests/auto/jobs/rt.py @@ -16,8 +16,8 @@ def run_regression_test(job_obj, pr_repo_loc): rt_command = 'cd tests' rt_command += f' && export RT_COMPILER="{job_obj.compiler}"' - if job_obj.clargs.workdir: - rt_command += f' && export RUNDIR_ROOT={job_obj.clargs.workdir}' + if job_obj.workdir: + rt_command += f' && export RUNDIR_ROOT={job_obj.workdir}' rt_command += f' && /bin/bash --login ./rt.sh -e -a {job_obj.clargs.account} -p {job_obj.clargs.machine}' if job_obj.compiler == 'gnu': rt_command += f' -l rt_gnu.conf' @@ -45,7 +45,7 @@ def clone_pr_repo(job_obj): git_ssh_url = job_obj.preq_dict['preq'].head.repo.ssh_url logger.debug(f'GIT SSH_URL: {git_ssh_url}') logger.info('Starting repo clone') - repo_dir_str = f'{job_obj.clargs.workdir}/'\ + repo_dir_str = f'{job_obj.workdir}/'\ f'{str(job_obj.preq_dict["preq"].id)}/'\ f'{datetime.datetime.now().strftime("%Y%m%d%H%M%S")}' pr_repo_loc = f'{repo_dir_str}/{repo_name}' @@ -72,6 +72,7 @@ def post_process(job_obj, pr_repo_loc, repo_dir_str, branch): logger = logging.getLogger('RT/MOVE_RT_LOGS') rt_log = f'tests/logs/RegressionTests_{job_obj.clargs.machine}.log' filepath = f'{pr_repo_loc}/{rt_log}' + print(filepath) rt_dir, logfile_pass = process_logfile(job_obj, filepath) if logfile_pass: #if job_obj.preq_dict['preq'].maintainer_can_modify: @@ -101,10 +102,12 @@ def process_logfile(job_obj, logfile): for line in f: if all(x in line for x in fail_string_list): # if 'FAIL' in line and 'Test' in line: + print(f'[RT] Error: {line.rstrip(chr(10))}') job_obj.comment_text_append(f'[RT] Error: {line.rstrip(chr(10))}') elif 'working dir' in line and not rt_dir: rt_dir = os.path.split(line.split()[-1])[0] elif 'SUCCESSFUL' in line: + print(rt_dir) return rt_dir, True job_obj.job_failed(logger, f'{job_obj.preq_dict["action"]}') else: diff --git a/tests/auto/rt_auto.py b/tests/auto/rt_auto.py index 77a8934b44..f1539b2d4d 100644 --- a/tests/auto/rt_auto.py +++ b/tests/auto/rt_auto.py @@ -179,6 +179,7 @@ def __init__(self, preq_dict, ghinterface_obj, args, compiler, gitargs): self.gitargs = gitargs self.comment_text = '***Automated RT Failure Notification***\n' self.failed_tests = [] + self.workdir = args.workdir def comment_text_append(self, newtext): self.comment_text += f'{newtext}\n' @@ -186,7 +187,7 @@ def comment_text_append(self, newtext): def remove_pr_label(self): ''' Removes the PR label that initiated the job run from PR ''' self.logger.info(f'Removing Label: {self.preq_dict["label"]}') -# self.preq_dict['preq'].remove_from_labels(self.preq_dict['label']) + self.preq_dict['preq'].remove_from_labels(self.preq_dict['label']) def check_label_before_job_start(self): # LETS Check the label still exists before the start of the job in the @@ -237,7 +238,7 @@ def run(self): except Exception: self.job_failed(logger, 'run()') logger.info('Sending comment text') -# self.send_comment_text() + self.send_comment_text() raise else: logger.info(f'Cannot find label {self.preq_dict["label"]}') @@ -260,8 +261,8 @@ def job_failed(self, logger, job_name, exception=None, STDOUT=False, if STDOUT: logger.critical(f'STDOUT: {[item for item in out if not None]}') logger.critical(f'STDERR: {[eitem for eitem in err if not None]}') - if exception is not None: - raise +# if exception is not None: +# raise def setup_env(git_cfg): # Dictionary of GitHub repositories to check From 9d5542d67b61cf84410c261a28b964cc250c2492 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Wed, 27 Sep 2023 16:07:18 +0000 Subject: [PATCH 51/56] Post-rebase fixes: - Add verbose error message if user doesn't have a yaml config file - Revert some accidental changes to rt.sh since last merge - Move check for bad arguments in rt.sh to right after case statement --- tests/auto/rt_auto.py | 9 +++++++-- tests/rt.sh | 16 ++++++++-------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/tests/auto/rt_auto.py b/tests/auto/rt_auto.py index f1539b2d4d..b76768036e 100644 --- a/tests/auto/rt_auto.py +++ b/tests/auto/rt_auto.py @@ -312,8 +312,13 @@ def main(): dargs = vars(args) # Load yamlfile for additional arguments - with open(args.yamlfile) as f: - cfg = yaml.safe_load(f) + try: + with open(args.yamlfile) as f: + cfg = yaml.safe_load(f) + except FileNotFoundError: + logger.error(f'Could not find yaml config file {args.yamlfile}') + logger.error('See README.rt_auto.yaml for info on creating this file') + raise # For each mandatory command-line argument, if not provided, check yaml file, and fail if not there either mandatory = ['machine','account'] diff --git a/tests/rt.sh b/tests/rt.sh index 7667089cd5..b0ec6665fd 100755 --- a/tests/rt.sh +++ b/tests/rt.sh @@ -142,7 +142,7 @@ TESTS_FILE='rt.conf' SKIP_ORDER=false -while getopts "a:cl:mn:dz:krep:s:wh" opt; do +while getopts ":a:cl:mn:dkrep:s:wh" opt; do case $opt in a) ACCNR=$OPTARG @@ -155,8 +155,7 @@ while getopts "a:cl:mn:dz:krep:s:wh" opt; do SKIP_ORDER=true ;; m) - # redefine RTPWD to point to newly created baseline outputs - RTPWD=${NEW_BASELINE} + RTPWD_NEW_BASELINE=true ;; n) SINGLE_OPTS=("$OPTARG") @@ -217,6 +216,10 @@ while getopts "a:cl:mn:dz:krep:s:wh" opt; do esac done +# If there are extra arguments, quit and print usage function +shift $((OPTIND-1)) +[[ $# -gt 1 ]] && usage + source rt_utils.sh source module-setup.sh @@ -241,6 +244,8 @@ if [[ $TESTS_FILE =~ '35d' ]] || [[ $TESTS_FILE =~ 'weekly' ]]; then TEST_35D=true fi +NEW_BASELINE=${NEW_BASELINE:-${STMP}/${USER}/FV3_RT/REGRESSION_TEST} + if [[ "$RTPWD_NEW_BASELINE" == true ]] ; then RTPWD=${NEW_BASELINE} else @@ -251,11 +256,6 @@ INPUTDATA_ROOT=${INPUTDATA_ROOT:-$DISKNM/NEMSfv3gfs/input-data-20221101} INPUTDATA_ROOT_WW3=${INPUTDATA_ROOT}/WW3_input_data_20220624 INPUTDATA_ROOT_BMIC=${INPUTDATA_ROOT_BMIC:-$DISKNM/NEMSfv3gfs/BM_IC-20220207} -shift $((OPTIND-1)) -[[ $# -gt 1 ]] && usage - -NEW_BASELINE=${NEW_BASELINE:-${STMP}/${USER}/FV3_RT/REGRESSION_TEST} - if [[ $CREATE_BASELINE == true ]]; then # # prepare new regression test directory From 5598dd0b240c961be9b1e81726586f964c22e945 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Wed, 27 Sep 2023 19:31:24 +0000 Subject: [PATCH 52/56] Forgot to initialize new variable RTPWD_NEW_BASELINE --- tests/rt.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/rt.sh b/tests/rt.sh index b0ec6665fd..36c64e9824 100755 --- a/tests/rt.sh +++ b/tests/rt.sh @@ -135,13 +135,13 @@ ECFLOW=false KEEP_RUNDIR=false SINGLE_NAME='' TEST_35D=false +RTPWD_NEW_BASELINE=false +SKIP_ORDER=false export skip_check_results=false export delete_rundir=false TESTS_FILE='rt.conf' -SKIP_ORDER=false - while getopts ":a:cl:mn:dkrep:s:wh" opt; do case $opt in a) From 6a97b95331fc4621b953d26178016134e698e2a0 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Mon, 9 Oct 2023 15:49:28 +0000 Subject: [PATCH 53/56] pass "new baseline" command-line arg --- tests/auto/jobs/bl.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/auto/jobs/bl.py b/tests/auto/jobs/bl.py index 47ce530f19..ef6f3bb365 100644 --- a/tests/auto/jobs/bl.py +++ b/tests/auto/jobs/bl.py @@ -98,6 +98,8 @@ def run_regression_test(job_obj, pr_repo_loc): rt_command += f' && export RT_COMPILER="{job_obj.compiler}"' if job_obj.workdir: rt_command += f' && export RUNDIR_ROOT={job_obj.workdir}' + if job_obj.clargs.new_baseline: + rt_command += f' && export NEW_BASELINE={job_obj.clargs.new_baseline}' rt_command += f' && /bin/bash --login ./rt.sh -e -a {job_obj.clargs.account} -c -p {job_obj.clargs.machine} -n control_p8 intel' if job_obj.compiler == 'gnu': rt_command += f' -l rt_gnu.conf' From a85bc659cd7edf7cfaabe8a2dd1dcedad91100b3 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Tue, 10 Oct 2023 14:44:27 -0600 Subject: [PATCH 54/56] Fix logfile parsing problem with test failures --- tests/auto/jobs/rt.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/auto/jobs/rt.py b/tests/auto/jobs/rt.py index de04dcd4f3..b9856ce55d 100644 --- a/tests/auto/jobs/rt.py +++ b/tests/auto/jobs/rt.py @@ -96,20 +96,19 @@ def post_process(job_obj, pr_repo_loc, repo_dir_str, branch): def process_logfile(job_obj, logfile): logger = logging.getLogger('RT/PROCESS_LOGFILE') rt_dir = [] - fail_string_list = ['Test', 'failed'] + fail_string_list = ['Test', 'FAIL'] if os.path.exists(logfile): + logger.debug(f'processing log file {logfile}') with open(logfile) as f: for line in f: if all(x in line for x in fail_string_list): - # if 'FAIL' in line and 'Test' in line: - print(f'[RT] Error: {line.rstrip(chr(10))}') job_obj.comment_text_append(f'[RT] Error: {line.rstrip(chr(10))}') elif 'working dir' in line and not rt_dir: rt_dir = os.path.split(line.split()[-1])[0] elif 'SUCCESSFUL' in line: - print(rt_dir) return rt_dir, True job_obj.job_failed(logger, f'{job_obj.preq_dict["action"]}') + return rt_dir, False else: logger.critical(f'Could not find {job_obj.clargs.machine}' f'.{job_obj.compiler} ' From fc745c4cd69ba1c82b865d7d19e4c30e1c62c60d Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Wed, 11 Oct 2023 15:57:09 +0000 Subject: [PATCH 55/56] Remove module purge per PR comments --- tests/compile.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/compile.sh b/tests/compile.sh index 17a5199568..8702efe135 100755 --- a/tests/compile.sh +++ b/tests/compile.sh @@ -62,7 +62,6 @@ else source /lustre/f2/dev/role.epic/contrib/Lmod_init.sh fi # Load fv3 module - module purge module use $PATHTR/modulefiles modulefile="ufs_${MACHINE_ID}.${RT_COMPILER}" module load $modulefile From 055ae969762b0e59c681d8379253e3b91a396b75 Mon Sep 17 00:00:00 2001 From: "job_obj.gitargs[config][user.name]" Date: Wed, 11 Oct 2023 23:41:24 +0000 Subject: [PATCH 56/56] [AutoRT] hera.intel Job Completed. on-behalf-of NCAR @mkavulich --- tests/logs/RegressionTests_hera.log | 2102 +++++++++++++-------------- 1 file changed, 1051 insertions(+), 1051 deletions(-) diff --git a/tests/logs/RegressionTests_hera.log b/tests/logs/RegressionTests_hera.log index df1ad1dd33..d2afb73600 100644 --- a/tests/logs/RegressionTests_hera.log +++ b/tests/logs/RegressionTests_hera.log @@ -1,74 +1,74 @@ -Wed Oct 4 21:18:06 UTC 2023 +Wed Oct 11 21:28:26 UTC 2023 Start Regression test -Testing UFSWM Hash: 94d15265837e9df085f4a4b98426d4476f63e072 +Testing UFSWM Hash: fc745c4cd69ba1c82b865d7d19e4c30e1c62c60d Testing With Submodule Hashes: 37cbb7d6840ae7515a9a8f0dfd4d89461b3396d1 ../AQM (v0.2.0-37-g37cbb7d) 2aa6bfbb62ebeecd7da964b8074f6c3c41c7d1eb ../CDEPS-interface/CDEPS (cdeps0.4.17-38-g2aa6bfb) 5840cd1931e2e32b9dfded0c19049d0f1ec3d04c ../CICE-interface/CICE (CICE6.0.0-440-g5840cd1) 9923d6d17700daf502d9a016138bf8eb8aad7f09 ../CMEPS-interface/CMEPS (cmeps_v0.4.1-1402-g9923d6d) cabd7753ae17f7bfcc6dad56daf10868aa51c3f4 ../CMakeModules (v1.0.0-28-gcabd775) - 376e17f718bfa77e7ce57fddd4d1841235052209 ../FV3 (EP4-61-g376e17f) + 8db61ecc752cda63046092649a338e34c61520f9 ../FV3 (EP4-63-g8db61ec) b94145fca46169bbc53ec6b8d4ed849715dc5130 ../GOCART (rt-v5_29_1_BPL91_1-exRT4-514-gb94145f) 24437531dcf8580aadaf6ebeb9de544ccfc674f9 ../HYCOM-interface/HYCOM (2.3.00-120-g2443753) fdbfa2523650b81a0771f3fb1791ea3e3dce66db ../MOM6-interface/MOM6 (dev/master/repository_split_2014.10.10-9713-gfdbfa2523) 569e354ababbde7a7cd68647533769a5c966468d ../NOAHMP-interface/noahmp (v3.7.1-303-g569e354) c4b116886b5ef9af5fb4942d7161074df3402732 ../WW3 (6.07.1-325-gc4b11688) 3bfa4468d85e5b63980c28434f494967f38b10a3 ../stochastic_physics (ufs-v2.0.0-171-g3bfa446) -Compile atmaero_intel elapsed time 547 seconds. -DAPP=ATMAERO -DCCPP_SUITES=FV3_GFS_v17_p8 -D32BIT=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Release -Compile atmaq_debug_intel elapsed time 220 seconds. -DAPP=ATMAQ -DCCPP_SUITES=FV3_GFS_v15p2 -DDEBUG=ON -D32BIT=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Debug -Compile atmaq_faster_intel elapsed time 552 seconds. -DAPP=ATMAQ -DCCPP_SUITES=FV3_GFS_v15p2 -DFASTER=ON -D32BIT=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Release -Compile atmaq_intel elapsed time 520 seconds. -DAPP=ATMAQ -DCCPP_SUITES=FV3_GFS_v15p2 -D32BIT=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Release -Compile atm_debug_dyn32_intel elapsed time 225 seconds. -DAPP=ATM -DDEBUG=ON -D32BIT=ON -DCCPP_SUITES=FV3_HRRR,FV3_GFS_v16,FV3_GFS_v16_csawmg,FV3_GFS_v16_ras,FV3_GFS_v17_p8,FV3_GFS_v15_thompson_mynn_lam3km,FV3_RAP,FV3_RAP_unified_ugwp,FV3_RAP_cires_ugwp,FV3_RAP_flake,FV3_RAP_clm_lake,FV3_RAP_noah,FV3_RAP_sfcdiff,FV3_RAP_noah_sfcdiff_cires_ugwp,FV3_RRFS_v1beta -DMPI=ON -DCMAKE_BUILD_TYPE=Debug -Compile atm_dyn32_debug_gnu elapsed time 177 seconds. -DAPP=ATM -D32BIT=ON -DDEBUG=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Debug -Compile atm_dyn32_intel elapsed time 596 seconds. -DAPP=ATM -DCCPP_SUITES=FV3_GFS_v16,FV3_GFS_v16_flake,FV3_GFS_v17_p8,FV3_GFS_v17_p8_rrtmgp,FV3_GFS_v15_thompson_mynn_lam3km,FV3_WoFS_v0,FV3_GFS_v17_p8_mynn -D32BIT=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Release -Compile atm_dyn32_phy32_debug_gnu elapsed time 176 seconds. -DAPP=ATM -D32BIT=ON -DCCPP_32BIT=ON -DDEBUG=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Debug +Compile atmaero_intel elapsed time 532 seconds. -DAPP=ATMAERO -DCCPP_SUITES=FV3_GFS_v17_p8 -D32BIT=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Release +Compile atmaq_debug_intel elapsed time 178 seconds. -DAPP=ATMAQ -DCCPP_SUITES=FV3_GFS_v15p2 -DDEBUG=ON -D32BIT=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Debug +Compile atmaq_faster_intel elapsed time 546 seconds. -DAPP=ATMAQ -DCCPP_SUITES=FV3_GFS_v15p2 -DFASTER=ON -D32BIT=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Release +Compile atmaq_intel elapsed time 514 seconds. -DAPP=ATMAQ -DCCPP_SUITES=FV3_GFS_v15p2 -D32BIT=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Release +Compile atm_debug_dyn32_intel elapsed time 234 seconds. -DAPP=ATM -DDEBUG=ON -D32BIT=ON -DCCPP_SUITES=FV3_HRRR,FV3_GFS_v16,FV3_GFS_v16_csawmg,FV3_GFS_v16_ras,FV3_GFS_v17_p8,FV3_GFS_v15_thompson_mynn_lam3km,FV3_RAP,FV3_RAP_unified_ugwp,FV3_RAP_cires_ugwp,FV3_RAP_flake,FV3_RAP_clm_lake,FV3_RAP_noah,FV3_RAP_sfcdiff,FV3_RAP_noah_sfcdiff_cires_ugwp,FV3_RRFS_v1beta -DMPI=ON -DCMAKE_BUILD_TYPE=Debug +Compile atm_dyn32_debug_gnu elapsed time 183 seconds. -DAPP=ATM -D32BIT=ON -DDEBUG=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Debug +Compile atm_dyn32_intel elapsed time 593 seconds. -DAPP=ATM -DCCPP_SUITES=FV3_GFS_v16,FV3_GFS_v16_flake,FV3_GFS_v17_p8,FV3_GFS_v17_p8_rrtmgp,FV3_GFS_v15_thompson_mynn_lam3km,FV3_WoFS_v0,FV3_GFS_v17_p8_mynn -D32BIT=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Release +Compile atm_dyn32_phy32_debug_gnu elapsed time 181 seconds. -DAPP=ATM -D32BIT=ON -DCCPP_32BIT=ON -DDEBUG=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Debug Compile atm_dyn64_phy32_debug_gnu elapsed time 176 seconds. -DAPP=ATM -DCCPP_32BIT=ON -DDEBUG=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Debug -Compile atm_dyn64_phy32_gnu elapsed time 281 seconds. -DAPP=ATM -DCCPP_32BIT=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Release -Compile atm_faster_dyn32_intel elapsed time 536 seconds. -DAPP=ATM -DCCPP_SUITES=FV3_GFS_v17_p8,FV3_GFS_v15_thompson_mynn_lam3km -D32BIT=ON -DFASTER=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Release -Compile atm_gnu elapsed time 196 seconds. -DAPP=ATM -DCCPP_SUITES=FV3_GFS_v16,FV3_GFS_v16_flake,FV3_GFS_v16_ras,FV3_GFS_v17_p8 -DMPI=ON -DCMAKE_BUILD_TYPE=Release -Compile atml_intel elapsed time 562 seconds. -DAPP=ATML -DCCPP_SUITES=FV3_GFS_v16,FV3_GFS_v17_p8,FV3_GFS_v15_thompson_mynn_lam3km -DMPI=ON -DCMAKE_BUILD_TYPE=Release -Compile atmw_intel elapsed time 561 seconds. -DAPP=ATMW -DCCPP_SUITES=FV3_GFS_v17_p8 -D32BIT=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Release -Compile atmwm_intel elapsed time 588 seconds. -DAPP=ATMWM -DCCPP_SUITES=FV3_GFS_v16 -D32BIT=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Release +Compile atm_dyn64_phy32_gnu elapsed time 274 seconds. -DAPP=ATM -DCCPP_32BIT=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Release +Compile atm_faster_dyn32_intel elapsed time 550 seconds. -DAPP=ATM -DCCPP_SUITES=FV3_GFS_v17_p8,FV3_GFS_v15_thompson_mynn_lam3km -D32BIT=ON -DFASTER=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Release +Compile atm_gnu elapsed time 200 seconds. -DAPP=ATM -DCCPP_SUITES=FV3_GFS_v16,FV3_GFS_v16_flake,FV3_GFS_v16_ras,FV3_GFS_v17_p8 -DMPI=ON -DCMAKE_BUILD_TYPE=Release +Compile atml_intel elapsed time 532 seconds. -DAPP=ATML -DCCPP_SUITES=FV3_GFS_v16,FV3_GFS_v17_p8,FV3_GFS_v15_thompson_mynn_lam3km -DMPI=ON -DCMAKE_BUILD_TYPE=Release +Compile atmw_intel elapsed time 562 seconds. -DAPP=ATMW -DCCPP_SUITES=FV3_GFS_v17_p8 -D32BIT=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Release +Compile atmwm_intel elapsed time 572 seconds. -DAPP=ATMWM -DCCPP_SUITES=FV3_GFS_v16 -D32BIT=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Release Compile csawmg_intel elapsed time 528 seconds. -DAPP=ATM -DCCPP_SUITES=FV3_GFS_v16_csawmg,FV3_GFS_v16_ras -DMPI=ON -DCMAKE_BUILD_TYPE=Release -Compile datm_cdeps_debug_intel elapsed time 105 seconds. -DAPP=NG-GODAS -DDEBUG=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Debug -DMOM6SOLO=ON -Compile datm_cdeps_faster_intel elapsed time 180 seconds. -DAPP=NG-GODAS -DFASTER=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Release -DMOM6SOLO=ON -Compile datm_cdeps_gnu elapsed time 119 seconds. -DAPP=NG-GODAS -DMPI=ON -DCMAKE_BUILD_TYPE=Release -DMOM6SOLO=ON -Compile datm_cdeps_intel elapsed time 180 seconds. -DAPP=NG-GODAS -DMPI=ON -DCMAKE_BUILD_TYPE=Release -DMOM6SOLO=ON -Compile datm_cdeps_land_intel elapsed time 89 seconds. -DAPP=LND -DMPI=ON -DCMAKE_BUILD_TYPE=Release -Compile hafs_all_intel elapsed time 608 seconds. -DAPP=HAFS-ALL -DCCPP_SUITES=FV3_HAFS_v1_gfdlmp_tedmf,FV3_HAFS_v1_gfdlmp_tedmf_nonsst -D32BIT=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Release -Compile hafsw_debug_intel elapsed time 197 seconds. -DAPP=HAFSW -DMOVING_NEST=ON -DCCPP_SUITES=FV3_HAFS_v1_gfdlmp_tedmf_nonsst -D32BIT=ON -DDEBUG=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Debug -Compile hafsw_faster_intel elapsed time 603 seconds. -DAPP=HAFSW -DMOVING_NEST=ON -DCCPP_SUITES=FV3_HAFS_v1_gfdlmp_tedmf_nonsst -D32BIT=ON -DFASTER=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Release -Compile hafsw_intel elapsed time 596 seconds. -DAPP=HAFSW -DMOVING_NEST=ON -DCCPP_SUITES=FV3_HAFS_v1_gfdlmp_tedmf,FV3_HAFS_v1_gfdlmp_tedmf_nonsst,FV3_HAFS_v1_thompson_tedmf_gfdlsf -D32BIT=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Release -Compile rrfs_dyn32_phy32_debug_intel elapsed time 177 seconds. -DAPP=ATM -DCCPP_SUITES=FV3_RAP,FV3_HRRR -D32BIT=ON -DCCPP_32BIT=ON -DDEBUG=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Debug -Compile rrfs_dyn32_phy32_faster_intel elapsed time 654 seconds. -DAPP=ATM -DCCPP_SUITES=FV3_RAP,FV3_HRRR -D32BIT=ON -DCCPP_32BIT=ON -DFASTER=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Release -Compile rrfs_dyn32_phy32_gnu elapsed time 197 seconds. -DAPP=ATM -DCCPP_SUITES=FV3_RAP,FV3_HRRR -D32BIT=ON -DCCPP_32BIT=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Release -Compile rrfs_dyn32_phy32_intel elapsed time 511 seconds. -DAPP=ATM -DCCPP_SUITES=FV3_RAP,FV3_HRRR -D32BIT=ON -DCCPP_32BIT=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Release -Compile rrfs_dyn64_phy32_debug_intel elapsed time 174 seconds. -DAPP=ATM -DCCPP_SUITES=FV3_RAP,FV3_HRRR -DCCPP_32BIT=ON -DDEBUG=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Debug -Compile rrfs_dyn64_phy32_intel elapsed time 510 seconds. -DAPP=ATM -DCCPP_SUITES=FV3_RAP,FV3_HRRR -DCCPP_32BIT=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Release -Compile rrfs_gnu elapsed time 200 seconds. -DAPP=ATM -DCCPP_SUITES=FV3_RAP,FV3_RAP_sfcdiff,FV3_HRRR,FV3_RRFS_v1beta -D32BIT=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Release +Compile datm_cdeps_debug_intel elapsed time 117 seconds. -DAPP=NG-GODAS -DDEBUG=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Debug -DMOM6SOLO=ON +Compile datm_cdeps_faster_intel elapsed time 211 seconds. -DAPP=NG-GODAS -DFASTER=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Release -DMOM6SOLO=ON +Compile datm_cdeps_gnu elapsed time 113 seconds. -DAPP=NG-GODAS -DMPI=ON -DCMAKE_BUILD_TYPE=Release -DMOM6SOLO=ON +Compile datm_cdeps_intel elapsed time 182 seconds. -DAPP=NG-GODAS -DMPI=ON -DCMAKE_BUILD_TYPE=Release -DMOM6SOLO=ON +Compile datm_cdeps_land_intel elapsed time 48 seconds. -DAPP=LND -DMPI=ON -DCMAKE_BUILD_TYPE=Release +Compile hafs_all_intel elapsed time 584 seconds. -DAPP=HAFS-ALL -DCCPP_SUITES=FV3_HAFS_v1_gfdlmp_tedmf,FV3_HAFS_v1_gfdlmp_tedmf_nonsst -D32BIT=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Release +Compile hafsw_debug_intel elapsed time 229 seconds. -DAPP=HAFSW -DMOVING_NEST=ON -DCCPP_SUITES=FV3_HAFS_v1_gfdlmp_tedmf_nonsst -D32BIT=ON -DDEBUG=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Debug +Compile hafsw_faster_intel elapsed time 601 seconds. -DAPP=HAFSW -DMOVING_NEST=ON -DCCPP_SUITES=FV3_HAFS_v1_gfdlmp_tedmf_nonsst -D32BIT=ON -DFASTER=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Release +Compile hafsw_intel elapsed time 599 seconds. -DAPP=HAFSW -DMOVING_NEST=ON -DCCPP_SUITES=FV3_HAFS_v1_gfdlmp_tedmf,FV3_HAFS_v1_gfdlmp_tedmf_nonsst,FV3_HAFS_v1_thompson_tedmf_gfdlsf -D32BIT=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Release +Compile rrfs_dyn32_phy32_debug_intel elapsed time 190 seconds. -DAPP=ATM -DCCPP_SUITES=FV3_RAP,FV3_HRRR -D32BIT=ON -DCCPP_32BIT=ON -DDEBUG=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Debug +Compile rrfs_dyn32_phy32_faster_intel elapsed time 661 seconds. -DAPP=ATM -DCCPP_SUITES=FV3_RAP,FV3_HRRR -D32BIT=ON -DCCPP_32BIT=ON -DFASTER=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Release +Compile rrfs_dyn32_phy32_gnu elapsed time 202 seconds. -DAPP=ATM -DCCPP_SUITES=FV3_RAP,FV3_HRRR -D32BIT=ON -DCCPP_32BIT=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Release +Compile rrfs_dyn32_phy32_intel elapsed time 504 seconds. -DAPP=ATM -DCCPP_SUITES=FV3_RAP,FV3_HRRR -D32BIT=ON -DCCPP_32BIT=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Release +Compile rrfs_dyn64_phy32_debug_intel elapsed time 205 seconds. -DAPP=ATM -DCCPP_SUITES=FV3_RAP,FV3_HRRR -DCCPP_32BIT=ON -DDEBUG=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Debug +Compile rrfs_dyn64_phy32_intel elapsed time 529 seconds. -DAPP=ATM -DCCPP_SUITES=FV3_RAP,FV3_HRRR -DCCPP_32BIT=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Release +Compile rrfs_gnu elapsed time 197 seconds. -DAPP=ATM -DCCPP_SUITES=FV3_RAP,FV3_RAP_sfcdiff,FV3_HRRR,FV3_RRFS_v1beta -D32BIT=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Release Compile rrfs_intel elapsed time 547 seconds. -DAPP=ATM -DCCPP_SUITES=FV3_RAP,FV3_RAP_sfcdiff,FV3_HRRR,FV3_RRFS_v1beta,FV3_RRFS_v1nssl -D32BIT=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Release -Compile s2s_aoflux_intel elapsed time 581 seconds. -DAPP=S2S -DCCPP_SUITES=FV3_GFS_v17_coupled_p8_sfcocn -DCMEPS_AOFLUX=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Release -DMOM6SOLO=ON -Compile s2s_gnu elapsed time 253 seconds. -DAPP=S2S -DCCPP_SUITES=FV3_GFS_v17_coupled_p8 -DMPI=ON -DCMAKE_BUILD_TYPE=Release -DMOM6SOLO=ON -Compile s2s_intel elapsed time 580 seconds. -DAPP=S2S -DCCPP_SUITES=FV3_GFS_v17_coupled_p8 -DMPI=ON -DCMAKE_BUILD_TYPE=Release -DMOM6SOLO=ON -Compile s2swa_32bit_intel elapsed time 645 seconds. -DAPP=S2SWA -D32BIT=ON -DCCPP_SUITES=FV3_GFS_v17_coupled_p8 -DMPI=ON -DCMAKE_BUILD_TYPE=Release -DMOM6SOLO=ON -Compile s2swa_debug_gnu elapsed time 140 seconds. -DAPP=S2SWA -DDEBUG=ON -DCCPP_SUITES=FV3_GFS_v17_coupled_p8 -DMPI=ON -DCMAKE_BUILD_TYPE=Debug -DMOM6SOLO=ON -Compile s2swa_debug_intel elapsed time 226 seconds. -DAPP=S2SWA -DDEBUG=ON -DCCPP_SUITES=FV3_GFS_v17_coupled_p8 -DMPI=ON -DCMAKE_BUILD_TYPE=Debug -DMOM6SOLO=ON -Compile s2swa_faster_intel elapsed time 693 seconds. -DAPP=S2SWA -DCCPP_SUITES=FV3_GFS_v17_coupled_p8 -DFASTER=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Release -DMOM6SOLO=ON -Compile s2swa_gnu elapsed time 281 seconds. -DAPP=S2SWA -DCCPP_SUITES=FV3_GFS_v17_coupled_p8 -DMPI=ON -DCMAKE_BUILD_TYPE=Release -DMOM6SOLO=ON -Compile s2swa_intel elapsed time 653 seconds. -DAPP=S2SWA -DCCPP_SUITES=FV3_GFS_v17_coupled_p8 -DMPI=ON -DCMAKE_BUILD_TYPE=Release -DMOM6SOLO=ON -Compile s2sw_debug_intel elapsed time 220 seconds. -DAPP=S2SW -DDEBUG=ON -DCCPP_SUITES=FV3_GFS_v17_coupled_p8 -DMPI=ON -DCMAKE_BUILD_TYPE=Debug -DMOM6SOLO=ON -Compile s2sw_intel elapsed time 629 seconds. -DAPP=S2SW -DCCPP_SUITES=FV3_GFS_v17_coupled_p8 -DMPI=ON -DCMAKE_BUILD_TYPE=Release -DMOM6SOLO=ON -Compile s2sw_pdlib_debug_gnu elapsed time 162 seconds. -DAPP=S2SW -DCCPP_SUITES=FV3_GFS_v17_coupled_p8 -DPDLIB=ON -DDEBUG=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Debug -DMOM6SOLO=ON -Compile s2sw_pdlib_debug_intel elapsed time 222 seconds. -DAPP=S2SW -DCCPP_SUITES=FV3_GFS_v17_coupled_p8 -DPDLIB=ON -DDEBUG=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Debug -DMOM6SOLO=ON +Compile s2s_aoflux_intel elapsed time 576 seconds. -DAPP=S2S -DCCPP_SUITES=FV3_GFS_v17_coupled_p8_sfcocn -DCMEPS_AOFLUX=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Release -DMOM6SOLO=ON +Compile s2s_gnu elapsed time 235 seconds. -DAPP=S2S -DCCPP_SUITES=FV3_GFS_v17_coupled_p8 -DMPI=ON -DCMAKE_BUILD_TYPE=Release -DMOM6SOLO=ON +Compile s2s_intel elapsed time 576 seconds. -DAPP=S2S -DCCPP_SUITES=FV3_GFS_v17_coupled_p8 -DMPI=ON -DCMAKE_BUILD_TYPE=Release -DMOM6SOLO=ON +Compile s2swa_32bit_intel elapsed time 632 seconds. -DAPP=S2SWA -D32BIT=ON -DCCPP_SUITES=FV3_GFS_v17_coupled_p8 -DMPI=ON -DCMAKE_BUILD_TYPE=Release -DMOM6SOLO=ON +Compile s2swa_debug_gnu elapsed time 141 seconds. -DAPP=S2SWA -DDEBUG=ON -DCCPP_SUITES=FV3_GFS_v17_coupled_p8 -DMPI=ON -DCMAKE_BUILD_TYPE=Debug -DMOM6SOLO=ON +Compile s2swa_debug_intel elapsed time 235 seconds. -DAPP=S2SWA -DDEBUG=ON -DCCPP_SUITES=FV3_GFS_v17_coupled_p8 -DMPI=ON -DCMAKE_BUILD_TYPE=Debug -DMOM6SOLO=ON +Compile s2swa_faster_intel elapsed time 698 seconds. -DAPP=S2SWA -DCCPP_SUITES=FV3_GFS_v17_coupled_p8 -DFASTER=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Release -DMOM6SOLO=ON +Compile s2swa_gnu elapsed time 267 seconds. -DAPP=S2SWA -DCCPP_SUITES=FV3_GFS_v17_coupled_p8 -DMPI=ON -DCMAKE_BUILD_TYPE=Release -DMOM6SOLO=ON +Compile s2swa_intel elapsed time 645 seconds. -DAPP=S2SWA -DCCPP_SUITES=FV3_GFS_v17_coupled_p8 -DMPI=ON -DCMAKE_BUILD_TYPE=Release -DMOM6SOLO=ON +Compile s2sw_debug_intel elapsed time 204 seconds. -DAPP=S2SW -DDEBUG=ON -DCCPP_SUITES=FV3_GFS_v17_coupled_p8 -DMPI=ON -DCMAKE_BUILD_TYPE=Debug -DMOM6SOLO=ON +Compile s2sw_intel elapsed time 621 seconds. -DAPP=S2SW -DCCPP_SUITES=FV3_GFS_v17_coupled_p8 -DMPI=ON -DCMAKE_BUILD_TYPE=Release -DMOM6SOLO=ON +Compile s2sw_pdlib_debug_gnu elapsed time 136 seconds. -DAPP=S2SW -DCCPP_SUITES=FV3_GFS_v17_coupled_p8 -DPDLIB=ON -DDEBUG=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Debug -DMOM6SOLO=ON +Compile s2sw_pdlib_debug_intel elapsed time 207 seconds. -DAPP=S2SW -DCCPP_SUITES=FV3_GFS_v17_coupled_p8 -DPDLIB=ON -DDEBUG=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Debug -DMOM6SOLO=ON Compile s2sw_pdlib_gnu elapsed time 247 seconds. -DAPP=S2SW -DCCPP_SUITES=FV3_GFS_v17_coupled_p8 -DPDLIB=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Release -DMOM6SOLO=ON -Compile s2sw_pdlib_intel elapsed time 887 seconds. -DAPP=S2SW -DCCPP_SUITES=FV3_GFS_v17_coupled_p8 -DPDLIB=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Release -DMOM6SOLO=ON -Compile wam_debug_gnu elapsed time 112 seconds. -DAPP=ATM -DCCPP_SUITES=FV3_GFS_v16_fv3wam -D32BIT=ON -DMULTI_GASES=ON -DDEBUG=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Debug -Compile wam_debug_intel elapsed time 175 seconds. -DAPP=ATM -DCCPP_SUITES=FV3_GFS_v16_fv3wam -D32BIT=ON -DMULTI_GASES=ON -DDEBUG=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Debug -Compile wam_intel elapsed time 481 seconds. -DAPP=ATM -DCCPP_SUITES=FV3_GFS_v16_fv3wam -D32BIT=ON -DMULTI_GASES=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Release +Compile s2sw_pdlib_intel elapsed time 902 seconds. -DAPP=S2SW -DCCPP_SUITES=FV3_GFS_v17_coupled_p8 -DPDLIB=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Release -DMOM6SOLO=ON +Compile wam_debug_gnu elapsed time 106 seconds. -DAPP=ATM -DCCPP_SUITES=FV3_GFS_v16_fv3wam -D32BIT=ON -DMULTI_GASES=ON -DDEBUG=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Debug +Compile wam_debug_intel elapsed time 183 seconds. -DAPP=ATM -DCCPP_SUITES=FV3_GFS_v16_fv3wam -D32BIT=ON -DMULTI_GASES=ON -DDEBUG=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Debug +Compile wam_intel elapsed time 487 seconds. -DAPP=ATM -DCCPP_SUITES=FV3_GFS_v16_fv3wam -D32BIT=ON -DMULTI_GASES=ON -DMPI=ON -DCMAKE_BUILD_TYPE=Release -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/cpld_control_p8_mixedmode_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/cpld_control_p8_mixedmode_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//cpld_control_p8_mixedmode_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/cpld_control_p8_mixedmode_intel Checking test 001 cpld_control_p8_mixedmode_intel results .... Comparing sfcf021.tile1.nc .........OK Comparing sfcf021.tile2.nc .........OK @@ -133,14 +133,14 @@ Checking test 001 cpld_control_p8_mixedmode_intel results .... Comparing 20210323.060000.out_pnt.ww3 .........OK Comparing 20210323.060000.out_grd.ww3 .........OK - 0: The total amount of wall time = 312.775373 - 0: The maximum resident set size (KB) = 3085636 + 0: The total amount of wall time = 312.629905 + 0: The maximum resident set size (KB) = 3116272 Test 001 cpld_control_p8_mixedmode_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/cpld_control_gfsv17_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/cpld_control_gfsv17_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//cpld_control_gfsv17_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/cpld_control_gfsv17_intel Checking test 002 cpld_control_gfsv17_intel results .... Comparing sfcf021.tile1.nc .........OK Comparing sfcf021.tile2.nc .........OK @@ -204,14 +204,14 @@ Checking test 002 cpld_control_gfsv17_intel results .... Comparing 20210323.060000.out_pnt.ww3 .........OK Comparing 20210323.060000.out_grd.ww3 .........OK - 0: The total amount of wall time = 220.803662 - 0: The maximum resident set size (KB) = 1673080 + 0: The total amount of wall time = 219.802474 + 0: The maximum resident set size (KB) = 1674764 Test 002 cpld_control_gfsv17_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/cpld_control_p8_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/cpld_control_p8_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//cpld_control_p8_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/cpld_control_p8_intel Checking test 003 cpld_control_p8_intel results .... Comparing sfcf021.tile1.nc .........OK Comparing sfcf021.tile2.nc .........OK @@ -276,14 +276,14 @@ Checking test 003 cpld_control_p8_intel results .... Comparing 20210323.060000.out_pnt.ww3 .........OK Comparing 20210323.060000.out_grd.ww3 .........OK - 0: The total amount of wall time = 343.985731 - 0: The maximum resident set size (KB) = 3164808 + 0: The total amount of wall time = 348.927739 + 0: The maximum resident set size (KB) = 3146932 Test 003 cpld_control_p8_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/cpld_control_p8_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/cpld_restart_p8_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//cpld_control_p8_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/cpld_restart_p8_intel Checking test 004 cpld_restart_p8_intel results .... Comparing sfcf024.tile1.nc .........OK Comparing sfcf024.tile2.nc .........OK @@ -336,14 +336,14 @@ Checking test 004 cpld_restart_p8_intel results .... Comparing 20210323.060000.out_pnt.ww3 .........OK Comparing 20210323.060000.out_grd.ww3 .........OK - 0: The total amount of wall time = 202.091562 - 0: The maximum resident set size (KB) = 3047488 + 0: The total amount of wall time = 205.243128 + 0: The maximum resident set size (KB) = 2985932 Test 004 cpld_restart_p8_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/cpld_control_p8_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/cpld_control_qr_p8_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//cpld_control_p8_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/cpld_control_qr_p8_intel Checking test 005 cpld_control_qr_p8_intel results .... Comparing sfcf021.tile1.nc .........OK Comparing sfcf021.tile2.nc .........OK @@ -408,14 +408,14 @@ Checking test 005 cpld_control_qr_p8_intel results .... Comparing 20210323.060000.out_pnt.ww3 .........OK Comparing 20210323.060000.out_grd.ww3 .........OK - 0: The total amount of wall time = 343.449837 - 0: The maximum resident set size (KB) = 3178172 + 0: The total amount of wall time = 345.292408 + 0: The maximum resident set size (KB) = 3176032 Test 005 cpld_control_qr_p8_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/cpld_control_p8_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/cpld_restart_qr_p8_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//cpld_control_p8_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/cpld_restart_qr_p8_intel Checking test 006 cpld_restart_qr_p8_intel results .... Comparing sfcf024.tile1.nc .........OK Comparing sfcf024.tile2.nc .........OK @@ -468,14 +468,14 @@ Checking test 006 cpld_restart_qr_p8_intel results .... Comparing 20210323.060000.out_pnt.ww3 .........OK Comparing 20210323.060000.out_grd.ww3 .........OK - 0: The total amount of wall time = 209.903041 - 0: The maximum resident set size (KB) = 3060800 + 0: The total amount of wall time = 198.463307 + 0: The maximum resident set size (KB) = 3015640 Test 006 cpld_restart_qr_p8_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/cpld_control_p8_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/cpld_2threads_p8_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//cpld_control_p8_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/cpld_2threads_p8_intel Checking test 007 cpld_2threads_p8_intel results .... Comparing sfcf024.tile1.nc .........OK Comparing sfcf024.tile2.nc .........OK @@ -528,14 +528,14 @@ Checking test 007 cpld_2threads_p8_intel results .... Comparing 20210323.060000.out_pnt.ww3 .........OK Comparing 20210323.060000.out_grd.ww3 .........OK - 0: The total amount of wall time = 350.009632 - 0: The maximum resident set size (KB) = 3506856 + 0: The total amount of wall time = 354.480882 + 0: The maximum resident set size (KB) = 3454324 Test 007 cpld_2threads_p8_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/cpld_control_p8_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/cpld_decomp_p8_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//cpld_control_p8_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/cpld_decomp_p8_intel Checking test 008 cpld_decomp_p8_intel results .... Comparing sfcf024.tile1.nc .........OK Comparing sfcf024.tile2.nc .........OK @@ -588,14 +588,14 @@ Checking test 008 cpld_decomp_p8_intel results .... Comparing 20210323.060000.out_pnt.ww3 .........OK Comparing 20210323.060000.out_grd.ww3 .........OK - 0: The total amount of wall time = 341.077389 - 0: The maximum resident set size (KB) = 3151908 + 0: The total amount of wall time = 344.637559 + 0: The maximum resident set size (KB) = 3113968 Test 008 cpld_decomp_p8_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/cpld_control_p8_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/cpld_mpi_p8_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//cpld_control_p8_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/cpld_mpi_p8_intel Checking test 009 cpld_mpi_p8_intel results .... Comparing sfcf024.tile1.nc .........OK Comparing sfcf024.tile2.nc .........OK @@ -648,14 +648,14 @@ Checking test 009 cpld_mpi_p8_intel results .... Comparing 20210323.060000.out_pnt.ww3 .........OK Comparing 20210323.060000.out_grd.ww3 .........OK - 0: The total amount of wall time = 283.294793 - 0: The maximum resident set size (KB) = 3011420 + 0: The total amount of wall time = 285.675869 + 0: The maximum resident set size (KB) = 3011936 Test 009 cpld_mpi_p8_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/cpld_control_ciceC_p8_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/cpld_control_ciceC_p8_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//cpld_control_ciceC_p8_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/cpld_control_ciceC_p8_intel Checking test 010 cpld_control_ciceC_p8_intel results .... Comparing sfcf021.tile1.nc .........OK Comparing sfcf021.tile2.nc .........OK @@ -720,14 +720,14 @@ Checking test 010 cpld_control_ciceC_p8_intel results .... Comparing 20210323.060000.out_pnt.ww3 .........OK Comparing 20210323.060000.out_grd.ww3 .........OK - 0: The total amount of wall time = 344.858882 - 0: The maximum resident set size (KB) = 3166320 + 0: The total amount of wall time = 344.903300 + 0: The maximum resident set size (KB) = 3156964 Test 010 cpld_control_ciceC_p8_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/cpld_control_c192_p8_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/cpld_control_c192_p8_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//cpld_control_c192_p8_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/cpld_control_c192_p8_intel Checking test 011 cpld_control_c192_p8_intel results .... Comparing sfcf030.tile1.nc .........OK Comparing sfcf030.tile2.nc .........OK @@ -780,14 +780,14 @@ Checking test 011 cpld_control_c192_p8_intel results .... Comparing 20210323.120000.out_grd.ww3 .........OK Comparing 20210323.120000.out_pnt.ww3 .........OK - 0: The total amount of wall time = 598.544432 - 0: The maximum resident set size (KB) = 3186572 + 0: The total amount of wall time = 599.085556 + 0: The maximum resident set size (KB) = 3200352 Test 011 cpld_control_c192_p8_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/cpld_control_c192_p8_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/cpld_restart_c192_p8_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//cpld_control_c192_p8_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/cpld_restart_c192_p8_intel Checking test 012 cpld_restart_c192_p8_intel results .... Comparing sfcf030.tile1.nc .........OK Comparing sfcf030.tile2.nc .........OK @@ -840,14 +840,14 @@ Checking test 012 cpld_restart_c192_p8_intel results .... Comparing 20210323.120000.out_grd.ww3 .........OK Comparing 20210323.120000.out_pnt.ww3 .........OK - 0: The total amount of wall time = 402.877950 - 0: The maximum resident set size (KB) = 3132660 + 0: The total amount of wall time = 388.832251 + 0: The maximum resident set size (KB) = 3154948 Test 012 cpld_restart_c192_p8_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/cpld_bmark_p8_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/cpld_bmark_p8_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//cpld_bmark_p8_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/cpld_bmark_p8_intel Checking test 013 cpld_bmark_p8_intel results .... Comparing sfcf006.nc .........OK Comparing atmf006.nc .........OK @@ -895,14 +895,14 @@ Checking test 013 cpld_bmark_p8_intel results .... Comparing 20130401.060000.out_pnt.ww3 .........OK Comparing 20130401.060000.out_grd.ww3 .........OK - 0: The total amount of wall time = 745.775586 - 0: The maximum resident set size (KB) = 4029092 + 0: The total amount of wall time = 733.161661 + 0: The maximum resident set size (KB) = 4014276 Test 013 cpld_bmark_p8_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/cpld_bmark_p8_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/cpld_restart_bmark_p8_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//cpld_bmark_p8_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/cpld_restart_bmark_p8_intel Checking test 014 cpld_restart_bmark_p8_intel results .... Comparing sfcf006.nc .........OK Comparing atmf006.nc .........OK @@ -950,14 +950,14 @@ Checking test 014 cpld_restart_bmark_p8_intel results .... Comparing 20130401.060000.out_pnt.ww3 .........OK Comparing 20130401.060000.out_grd.ww3 .........OK - 0: The total amount of wall time = 481.090107 - 0: The maximum resident set size (KB) = 3945424 + 0: The total amount of wall time = 503.883192 + 0: The maximum resident set size (KB) = 3939404 Test 014 cpld_restart_bmark_p8_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/cpld_control_noaero_p8_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/cpld_control_noaero_p8_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//cpld_control_noaero_p8_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/cpld_control_noaero_p8_intel Checking test 015 cpld_control_noaero_p8_intel results .... Comparing sfcf021.tile1.nc .........OK Comparing sfcf021.tile2.nc .........OK @@ -1021,14 +1021,14 @@ Checking test 015 cpld_control_noaero_p8_intel results .... Comparing 20210323.060000.out_pnt.ww3 .........OK Comparing 20210323.060000.out_grd.ww3 .........OK - 0: The total amount of wall time = 266.490311 - 0: The maximum resident set size (KB) = 1704544 + 0: The total amount of wall time = 273.074580 + 0: The maximum resident set size (KB) = 1705056 Test 015 cpld_control_noaero_p8_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/cpld_control_c96_noaero_p8_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/cpld_control_nowave_noaero_p8_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//cpld_control_c96_noaero_p8_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/cpld_control_nowave_noaero_p8_intel Checking test 016 cpld_control_nowave_noaero_p8_intel results .... Comparing sfcf021.tile1.nc .........OK Comparing sfcf021.tile2.nc .........OK @@ -1090,14 +1090,14 @@ Checking test 016 cpld_control_nowave_noaero_p8_intel results .... Comparing RESTART/iced.2021-03-23-21600.nc .........OK Comparing RESTART/ufs.cpld.cpl.r.2021-03-23-21600.nc .........OK - 0: The total amount of wall time = 269.134765 - 0: The maximum resident set size (KB) = 1722720 + 0: The total amount of wall time = 272.138991 + 0: The maximum resident set size (KB) = 1709288 Test 016 cpld_control_nowave_noaero_p8_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/cpld_debug_p8_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/cpld_debug_p8_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//cpld_debug_p8_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/cpld_debug_p8_intel Checking test 017 cpld_debug_p8_intel results .... Comparing sfcf003.tile1.nc .........OK Comparing sfcf003.tile2.nc .........OK @@ -1150,14 +1150,14 @@ Checking test 017 cpld_debug_p8_intel results .... Comparing 20210322.090000.out_pnt.ww3 .........OK Comparing 20210322.090000.out_grd.ww3 .........OK - 0: The total amount of wall time = 527.141944 - 0: The maximum resident set size (KB) = 3237556 + 0: The total amount of wall time = 523.005721 + 0: The maximum resident set size (KB) = 3192876 Test 017 cpld_debug_p8_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/cpld_debug_noaero_p8_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/cpld_debug_noaero_p8_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//cpld_debug_noaero_p8_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/cpld_debug_noaero_p8_intel Checking test 018 cpld_debug_noaero_p8_intel results .... Comparing sfcf003.tile1.nc .........OK Comparing sfcf003.tile2.nc .........OK @@ -1209,14 +1209,14 @@ Checking test 018 cpld_debug_noaero_p8_intel results .... Comparing 20210322.090000.out_pnt.ww3 .........OK Comparing 20210322.090000.out_grd.ww3 .........OK - 0: The total amount of wall time = 355.215840 - 0: The maximum resident set size (KB) = 1736268 + 0: The total amount of wall time = 361.377430 + 0: The maximum resident set size (KB) = 1720708 Test 018 cpld_debug_noaero_p8_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/cpld_control_noaero_p8_agrid_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/cpld_control_noaero_p8_agrid_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//cpld_control_noaero_p8_agrid_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/cpld_control_noaero_p8_agrid_intel Checking test 019 cpld_control_noaero_p8_agrid_intel results .... Comparing sfcf021.tile1.nc .........OK Comparing sfcf021.tile2.nc .........OK @@ -1278,14 +1278,14 @@ Checking test 019 cpld_control_noaero_p8_agrid_intel results .... Comparing RESTART/iced.2021-03-23-21600.nc .........OK Comparing RESTART/ufs.cpld.cpl.r.2021-03-23-21600.nc .........OK - 0: The total amount of wall time = 280.008734 - 0: The maximum resident set size (KB) = 1752520 + 0: The total amount of wall time = 280.502584 + 0: The maximum resident set size (KB) = 1766332 Test 019 cpld_control_noaero_p8_agrid_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/cpld_control_c48_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/cpld_control_c48_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//cpld_control_c48_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/cpld_control_c48_intel Checking test 020 cpld_control_c48_intel results .... Comparing sfcf024.tile1.nc .........OK Comparing sfcf024.tile2.nc .........OK @@ -1335,14 +1335,14 @@ Checking test 020 cpld_control_c48_intel results .... Comparing RESTART/iced.2021-03-23-21600.nc .........OK Comparing RESTART/ufs.cpld.cpl.r.2021-03-23-21600.nc .........OK - 0: The total amount of wall time = 579.787649 - 0: The maximum resident set size (KB) = 2801324 + 0: The total amount of wall time = 588.071622 + 0: The maximum resident set size (KB) = 2806624 Test 020 cpld_control_c48_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/cpld_control_p8_faster_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/cpld_control_p8_faster_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//cpld_control_p8_faster_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/cpld_control_p8_faster_intel Checking test 021 cpld_control_p8_faster_intel results .... Comparing sfcf021.tile1.nc .........OK Comparing sfcf021.tile2.nc .........OK @@ -1407,14 +1407,14 @@ Checking test 021 cpld_control_p8_faster_intel results .... Comparing 20210323.060000.out_pnt.ww3 .........OK Comparing 20210323.060000.out_grd.ww3 .........OK - 0: The total amount of wall time = 324.034891 - 0: The maximum resident set size (KB) = 3169232 + 0: The total amount of wall time = 326.689359 + 0: The maximum resident set size (KB) = 3177344 Test 021 cpld_control_p8_faster_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/cpld_control_pdlib_p8_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/cpld_control_pdlib_p8_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//cpld_control_pdlib_p8_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/cpld_control_pdlib_p8_intel Checking test 022 cpld_control_pdlib_p8_intel results .... Comparing sfcf021.tile1.nc .........OK Comparing sfcf021.tile2.nc .........OK @@ -1478,14 +1478,14 @@ Checking test 022 cpld_control_pdlib_p8_intel results .... Comparing 20210323.060000.out_pnt.ww3 .........OK Comparing 20210323.060000.out_grd.ww3 .........OK - 0: The total amount of wall time = 1209.043536 - 0: The maximum resident set size (KB) = 1742580 + 0: The total amount of wall time = 1190.954813 + 0: The maximum resident set size (KB) = 1730444 Test 022 cpld_control_pdlib_p8_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/cpld_control_pdlib_p8_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/cpld_restart_pdlib_p8_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//cpld_control_pdlib_p8_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/cpld_restart_pdlib_p8_intel Checking test 023 cpld_restart_pdlib_p8_intel results .... Comparing sfcf024.tile1.nc .........OK Comparing sfcf024.tile2.nc .........OK @@ -1537,14 +1537,14 @@ Checking test 023 cpld_restart_pdlib_p8_intel results .... Comparing 20210323.060000.out_pnt.ww3 .........OK Comparing 20210323.060000.out_grd.ww3 .........OK - 0: The total amount of wall time = 565.553788 - 0: The maximum resident set size (KB) = 1026632 + 0: The total amount of wall time = 566.459536 + 0: The maximum resident set size (KB) = 983508 Test 023 cpld_restart_pdlib_p8_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/cpld_control_pdlib_p8_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/cpld_mpi_pdlib_p8_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//cpld_control_pdlib_p8_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/cpld_mpi_pdlib_p8_intel Checking test 024 cpld_mpi_pdlib_p8_intel results .... Comparing sfcf021.tile1.nc .........OK Comparing sfcf021.tile2.nc .........OK @@ -1608,14 +1608,14 @@ Checking test 024 cpld_mpi_pdlib_p8_intel results .... Comparing 20210323.060000.out_pnt.ww3 .........OK Comparing 20210323.060000.out_grd.ww3 .........OK - 0: The total amount of wall time = 1084.449432 - 0: The maximum resident set size (KB) = 1628888 + 0: The total amount of wall time = 1083.914863 + 0: The maximum resident set size (KB) = 1620052 Test 024 cpld_mpi_pdlib_p8_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/cpld_debug_pdlib_p8_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/cpld_debug_pdlib_p8_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//cpld_debug_pdlib_p8_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/cpld_debug_pdlib_p8_intel Checking test 025 cpld_debug_pdlib_p8_intel results .... Comparing sfcf003.tile1.nc .........OK Comparing sfcf003.tile2.nc .........OK @@ -1667,14 +1667,14 @@ Checking test 025 cpld_debug_pdlib_p8_intel results .... Comparing 20210322.090000.out_pnt.ww3 .........OK Comparing 20210322.090000.out_grd.ww3 .........OK - 0: The total amount of wall time = 1385.771389 - 0: The maximum resident set size (KB) = 1693092 + 0: The total amount of wall time = 1424.049690 + 0: The maximum resident set size (KB) = 1693964 Test 025 cpld_debug_pdlib_p8_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/control_flake_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/control_flake_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//control_flake_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/control_flake_intel Checking test 026 control_flake_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf024.nc .........OK @@ -1685,14 +1685,14 @@ Checking test 026 control_flake_intel results .... Comparing GFSPRS.GrbF00 .........OK Comparing GFSPRS.GrbF24 .........OK - 0: The total amount of wall time = 190.405383 - 0: The maximum resident set size (KB) = 679452 + 0: The total amount of wall time = 184.906262 + 0: The maximum resident set size (KB) = 677200 Test 026 control_flake_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/control_CubedSphereGrid_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/control_CubedSphereGrid_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//control_CubedSphereGrid_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/control_CubedSphereGrid_intel Checking test 027 control_CubedSphereGrid_intel results .... Comparing sfcf000.tile1.nc .........OK Comparing sfcf000.tile2.nc .........OK @@ -1719,28 +1719,28 @@ Checking test 027 control_CubedSphereGrid_intel results .... Comparing atmf024.tile5.nc .........OK Comparing atmf024.tile6.nc .........OK - 0: The total amount of wall time = 132.288638 - 0: The maximum resident set size (KB) = 615016 + 0: The total amount of wall time = 130.171998 + 0: The maximum resident set size (KB) = 622200 Test 027 control_CubedSphereGrid_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/control_CubedSphereGrid_parallel_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/control_CubedSphereGrid_parallel_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//control_CubedSphereGrid_parallel_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/control_CubedSphereGrid_parallel_intel Checking test 028 control_CubedSphereGrid_parallel_intel results .... Comparing sfcf000.nc ............ALT CHECK......OK - Comparing sfcf024.nc .........OK - Comparing atmf000.nc ............ALT CHECK......OK + Comparing sfcf024.nc ............ALT CHECK......OK + Comparing atmf000.nc .........OK Comparing atmf024.nc ............ALT CHECK......OK - 0: The total amount of wall time = 128.682177 - 0: The maximum resident set size (KB) = 620516 + 0: The total amount of wall time = 128.355421 + 0: The maximum resident set size (KB) = 596724 Test 028 control_CubedSphereGrid_parallel_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/control_latlon_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/control_latlon_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//control_latlon_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/control_latlon_intel Checking test 029 control_latlon_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf024.nc .........OK @@ -1751,16 +1751,16 @@ Checking test 029 control_latlon_intel results .... Comparing GFSPRS.GrbF00 .........OK Comparing GFSPRS.GrbF24 .........OK - 0: The total amount of wall time = 133.371912 - 0: The maximum resident set size (KB) = 622612 + 0: The total amount of wall time = 132.930112 + 0: The maximum resident set size (KB) = 604884 Test 029 control_latlon_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/control_wrtGauss_netcdf_parallel_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/control_wrtGauss_netcdf_parallel_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//control_wrtGauss_netcdf_parallel_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/control_wrtGauss_netcdf_parallel_intel Checking test 030 control_wrtGauss_netcdf_parallel_intel results .... - Comparing sfcf000.nc .........OK + Comparing sfcf000.nc ............ALT CHECK......OK Comparing sfcf024.nc .........OK Comparing atmf000.nc .........OK Comparing atmf024.nc .........OK @@ -1769,14 +1769,14 @@ Checking test 030 control_wrtGauss_netcdf_parallel_intel results .... Comparing GFSPRS.GrbF00 .........OK Comparing GFSPRS.GrbF24 .........OK - 0: The total amount of wall time = 137.574548 - 0: The maximum resident set size (KB) = 622376 + 0: The total amount of wall time = 138.211312 + 0: The maximum resident set size (KB) = 624520 Test 030 control_wrtGauss_netcdf_parallel_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/control_c48_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/control_c48_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//control_c48_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/control_c48_intel Checking test 031 control_c48_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf024.nc .........OK @@ -1815,14 +1815,14 @@ Checking test 031 control_c48_intel results .... Comparing RESTART/20210323.060000.sfc_data.tile5.nc .........OK Comparing RESTART/20210323.060000.sfc_data.tile6.nc .........OK -0: The total amount of wall time = 370.421487 -0: The maximum resident set size (KB) = 816684 +0: The total amount of wall time = 368.564003 +0: The maximum resident set size (KB) = 814840 Test 031 control_c48_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/control_c192_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/control_c192_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//control_c192_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/control_c192_intel Checking test 032 control_c192_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf024.nc .........OK @@ -1833,14 +1833,14 @@ Checking test 032 control_c192_intel results .... Comparing GFSPRS.GrbF00 .........OK Comparing GFSPRS.GrbF24 .........OK - 0: The total amount of wall time = 529.222671 - 0: The maximum resident set size (KB) = 760428 + 0: The total amount of wall time = 526.748273 + 0: The maximum resident set size (KB) = 764200 Test 032 control_c192_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/control_c384_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/control_c384_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//control_c384_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/control_c384_intel Checking test 033 control_c384_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf012.nc .........OK @@ -1851,14 +1851,14 @@ Checking test 033 control_c384_intel results .... Comparing GFSPRS.GrbF00 .........OK Comparing GFSPRS.GrbF12 .........OK - 0: The total amount of wall time = 526.273862 - 0: The maximum resident set size (KB) = 1270720 + 0: The total amount of wall time = 523.386873 + 0: The maximum resident set size (KB) = 1266136 Test 033 control_c384_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/control_c384gdas_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/control_c384gdas_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//control_c384gdas_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/control_c384gdas_intel Checking test 034 control_c384gdas_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf006.nc .........OK @@ -1901,14 +1901,14 @@ Checking test 034 control_c384gdas_intel results .... Comparing RESTART/20210322.060000.sfc_data.tile5.nc .........OK Comparing RESTART/20210322.060000.sfc_data.tile6.nc .........OK - 0: The total amount of wall time = 471.477143 - 0: The maximum resident set size (KB) = 1372188 + 0: The total amount of wall time = 469.465498 + 0: The maximum resident set size (KB) = 1366584 Test 034 control_c384gdas_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/control_stochy_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/control_stochy_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//control_stochy_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/control_stochy_intel Checking test 035 control_stochy_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf012.nc .........OK @@ -1919,28 +1919,28 @@ Checking test 035 control_stochy_intel results .... Comparing GFSPRS.GrbF00 .........OK Comparing GFSPRS.GrbF12 .........OK - 0: The total amount of wall time = 90.249403 - 0: The maximum resident set size (KB) = 632464 + 0: The total amount of wall time = 88.030536 + 0: The maximum resident set size (KB) = 630688 Test 035 control_stochy_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/control_stochy_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/control_stochy_restart_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//control_stochy_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/control_stochy_restart_intel Checking test 036 control_stochy_restart_intel results .... Comparing sfcf012.nc .........OK Comparing atmf012.nc .........OK Comparing GFSFLX.GrbF12 .........OK Comparing GFSPRS.GrbF12 .........OK - 0: The total amount of wall time = 48.810637 - 0: The maximum resident set size (KB) = 472192 + 0: The total amount of wall time = 48.998345 + 0: The maximum resident set size (KB) = 473748 Test 036 control_stochy_restart_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/control_lndp_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/control_lndp_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//control_lndp_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/control_lndp_intel Checking test 037 control_lndp_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf012.nc .........OK @@ -1951,14 +1951,14 @@ Checking test 037 control_lndp_intel results .... Comparing GFSPRS.GrbF00 .........OK Comparing GFSPRS.GrbF12 .........OK - 0: The total amount of wall time = 85.290116 - 0: The maximum resident set size (KB) = 625876 + 0: The total amount of wall time = 81.848034 + 0: The maximum resident set size (KB) = 624472 Test 037 control_lndp_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/control_iovr4_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/control_iovr4_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//control_iovr4_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/control_iovr4_intel Checking test 038 control_iovr4_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf021.nc .........OK @@ -1973,14 +1973,14 @@ Checking test 038 control_iovr4_intel results .... Comparing GFSPRS.GrbF21 .........OK Comparing GFSPRS.GrbF24 .........OK - 0: The total amount of wall time = 139.537757 - 0: The maximum resident set size (KB) = 628084 + 0: The total amount of wall time = 138.157455 + 0: The maximum resident set size (KB) = 626004 Test 038 control_iovr4_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/control_iovr5_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/control_iovr5_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//control_iovr5_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/control_iovr5_intel Checking test 039 control_iovr5_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf021.nc .........OK @@ -1995,14 +1995,14 @@ Checking test 039 control_iovr5_intel results .... Comparing GFSPRS.GrbF21 .........OK Comparing GFSPRS.GrbF24 .........OK - 0: The total amount of wall time = 139.364985 - 0: The maximum resident set size (KB) = 626448 + 0: The total amount of wall time = 137.637632 + 0: The maximum resident set size (KB) = 629624 Test 039 control_iovr5_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/control_p8_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/control_p8_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//control_p8_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/control_p8_intel Checking test 040 control_p8_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf021.nc .........OK @@ -2049,14 +2049,14 @@ Checking test 040 control_p8_intel results .... Comparing RESTART/20210323.060000.sfc_data.tile5.nc .........OK Comparing RESTART/20210323.060000.sfc_data.tile6.nc .........OK - 0: The total amount of wall time = 173.853463 - 0: The maximum resident set size (KB) = 1601016 + 0: The total amount of wall time = 171.583302 + 0: The maximum resident set size (KB) = 1567676 Test 040 control_p8_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/control_p8_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/control_restart_p8_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//control_p8_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/control_restart_p8_intel Checking test 041 control_restart_p8_intel results .... Comparing sfcf024.nc .........OK Comparing atmf024.nc .........OK @@ -2095,14 +2095,14 @@ Checking test 041 control_restart_p8_intel results .... Comparing RESTART/20210323.060000.sfc_data.tile5.nc .........OK Comparing RESTART/20210323.060000.sfc_data.tile6.nc .........OK - 0: The total amount of wall time = 88.055005 - 0: The maximum resident set size (KB) = 874048 + 0: The total amount of wall time = 89.107190 + 0: The maximum resident set size (KB) = 866656 Test 041 control_restart_p8_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/control_p8_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/control_qr_p8_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//control_p8_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/control_qr_p8_intel Checking test 042 control_qr_p8_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf021.nc .........OK @@ -2149,14 +2149,14 @@ Checking test 042 control_qr_p8_intel results .... Comparing RESTART/20210323.060000.sfc_data.tile5.nc ............ALT CHECK......OK Comparing RESTART/20210323.060000.sfc_data.tile6.nc ............ALT CHECK......OK - 0: The total amount of wall time = 172.562389 - 0: The maximum resident set size (KB) = 1594172 + 0: The total amount of wall time = 169.149110 + 0: The maximum resident set size (KB) = 1582572 Test 042 control_qr_p8_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/control_p8_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/control_restart_qr_p8_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//control_p8_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/control_restart_qr_p8_intel Checking test 043 control_restart_qr_p8_intel results .... Comparing sfcf024.nc .........OK Comparing atmf024.nc .........OK @@ -2195,14 +2195,14 @@ Checking test 043 control_restart_qr_p8_intel results .... Comparing RESTART/20210323.060000.sfc_data.tile5.nc ............ALT CHECK......OK Comparing RESTART/20210323.060000.sfc_data.tile6.nc ............ALT CHECK......OK - 0: The total amount of wall time = 90.634243 - 0: The maximum resident set size (KB) = 856600 + 0: The total amount of wall time = 90.599775 + 0: The maximum resident set size (KB) = 863704 Test 043 control_restart_qr_p8_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/control_p8_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/control_decomp_p8_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//control_p8_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/control_decomp_p8_intel Checking test 044 control_decomp_p8_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf024.nc .........OK @@ -2245,14 +2245,14 @@ Checking test 044 control_decomp_p8_intel results .... Comparing RESTART/20210323.060000.sfc_data.tile5.nc .........OK Comparing RESTART/20210323.060000.sfc_data.tile6.nc .........OK - 0: The total amount of wall time = 180.226427 - 0: The maximum resident set size (KB) = 1578612 + 0: The total amount of wall time = 172.147367 + 0: The maximum resident set size (KB) = 1592584 Test 044 control_decomp_p8_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/control_p8_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/control_2threads_p8_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//control_p8_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/control_2threads_p8_intel Checking test 045 control_2threads_p8_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf024.nc .........OK @@ -2295,14 +2295,14 @@ Checking test 045 control_2threads_p8_intel results .... Comparing RESTART/20210323.060000.sfc_data.tile5.nc .........OK Comparing RESTART/20210323.060000.sfc_data.tile6.nc .........OK - 0: The total amount of wall time = 158.910470 - 0: The maximum resident set size (KB) = 1679064 + 0: The total amount of wall time = 157.506704 + 0: The maximum resident set size (KB) = 1676464 Test 045 control_2threads_p8_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/control_p8_lndp_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/control_p8_lndp_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//control_p8_lndp_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/control_p8_lndp_intel Checking test 046 control_p8_lndp_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf021.nc .........OK @@ -2321,14 +2321,14 @@ Checking test 046 control_p8_lndp_intel results .... Comparing GFSPRS.GrbF24 .........OK Comparing GFSPRS.GrbF48 .........OK - 0: The total amount of wall time = 313.444549 - 0: The maximum resident set size (KB) = 1600620 + 0: The total amount of wall time = 314.151427 + 0: The maximum resident set size (KB) = 1594328 Test 046 control_p8_lndp_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/control_p8_rrtmgp_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/control_p8_rrtmgp_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//control_p8_rrtmgp_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/control_p8_rrtmgp_intel Checking test 047 control_p8_rrtmgp_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf021.nc .........OK @@ -2375,14 +2375,14 @@ Checking test 047 control_p8_rrtmgp_intel results .... Comparing RESTART/20210323.060000.sfc_data.tile5.nc .........OK Comparing RESTART/20210323.060000.sfc_data.tile6.nc .........OK - 0: The total amount of wall time = 225.528307 - 0: The maximum resident set size (KB) = 1645628 + 0: The total amount of wall time = 226.001540 + 0: The maximum resident set size (KB) = 1669088 Test 047 control_p8_rrtmgp_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/control_p8_mynn_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/control_p8_mynn_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//control_p8_mynn_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/control_p8_mynn_intel Checking test 048 control_p8_mynn_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf021.nc .........OK @@ -2429,14 +2429,14 @@ Checking test 048 control_p8_mynn_intel results .... Comparing RESTART/20210323.060000.sfc_data.tile5.nc .........OK Comparing RESTART/20210323.060000.sfc_data.tile6.nc .........OK - 0: The total amount of wall time = 172.611219 - 0: The maximum resident set size (KB) = 1591644 + 0: The total amount of wall time = 168.840111 + 0: The maximum resident set size (KB) = 1602604 Test 048 control_p8_mynn_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/merra2_thompson_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/merra2_thompson_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//merra2_thompson_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/merra2_thompson_intel Checking test 049 merra2_thompson_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf021.nc .........OK @@ -2483,14 +2483,14 @@ Checking test 049 merra2_thompson_intel results .... Comparing RESTART/20210323.060000.sfc_data.tile5.nc .........OK Comparing RESTART/20210323.060000.sfc_data.tile6.nc .........OK - 0: The total amount of wall time = 195.439306 - 0: The maximum resident set size (KB) = 1597796 + 0: The total amount of wall time = 193.445143 + 0: The maximum resident set size (KB) = 1595856 Test 049 merra2_thompson_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/regional_control_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/regional_control_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//regional_control_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/regional_control_intel Checking test 050 regional_control_intel results .... Comparing dynf000.nc .........OK Comparing dynf006.nc .........OK @@ -2501,28 +2501,28 @@ Checking test 050 regional_control_intel results .... Comparing NATLEV.GrbF00 .........OK Comparing NATLEV.GrbF06 .........OK - 0: The total amount of wall time = 296.259801 - 0: The maximum resident set size (KB) = 846884 + 0: The total amount of wall time = 294.106336 + 0: The maximum resident set size (KB) = 867092 Test 050 regional_control_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/regional_control_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/regional_restart_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//regional_control_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/regional_restart_intel Checking test 051 regional_restart_intel results .... Comparing dynf006.nc .........OK Comparing phyf006.nc .........OK Comparing PRSLEV.GrbF06 .........OK Comparing NATLEV.GrbF06 .........OK - 0: The total amount of wall time = 149.945934 - 0: The maximum resident set size (KB) = 861564 + 0: The total amount of wall time = 150.227514 + 0: The maximum resident set size (KB) = 843816 Test 051 regional_restart_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/regional_control_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/regional_control_qr_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//regional_control_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/regional_control_qr_intel Checking test 052 regional_control_qr_intel results .... Comparing dynf000.nc .........OK Comparing dynf006.nc .........OK @@ -2533,28 +2533,28 @@ Checking test 052 regional_control_qr_intel results .... Comparing NATLEV.GrbF00 .........OK Comparing NATLEV.GrbF06 .........OK - 0: The total amount of wall time = 292.431419 - 0: The maximum resident set size (KB) = 862844 + 0: The total amount of wall time = 292.961398 + 0: The maximum resident set size (KB) = 865408 Test 052 regional_control_qr_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/regional_control_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/regional_restart_qr_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//regional_control_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/regional_restart_qr_intel Checking test 053 regional_restart_qr_intel results .... Comparing dynf006.nc .........OK Comparing phyf006.nc .........OK Comparing PRSLEV.GrbF06 .........OK Comparing NATLEV.GrbF06 .........OK - 0: The total amount of wall time = 149.239507 - 0: The maximum resident set size (KB) = 863444 + 0: The total amount of wall time = 150.250183 + 0: The maximum resident set size (KB) = 851544 Test 053 regional_restart_qr_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/regional_control_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/regional_decomp_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//regional_control_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/regional_decomp_intel Checking test 054 regional_decomp_intel results .... Comparing dynf000.nc .........OK Comparing dynf006.nc .........OK @@ -2565,14 +2565,14 @@ Checking test 054 regional_decomp_intel results .... Comparing NATLEV.GrbF00 .........OK Comparing NATLEV.GrbF06 .........OK - 0: The total amount of wall time = 310.329110 - 0: The maximum resident set size (KB) = 854396 + 0: The total amount of wall time = 306.316680 + 0: The maximum resident set size (KB) = 854564 Test 054 regional_decomp_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/regional_control_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/regional_2threads_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//regional_control_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/regional_2threads_intel Checking test 055 regional_2threads_intel results .... Comparing dynf000.nc .........OK Comparing dynf006.nc .........OK @@ -2583,14 +2583,14 @@ Checking test 055 regional_2threads_intel results .... Comparing NATLEV.GrbF00 .........OK Comparing NATLEV.GrbF06 .........OK - 0: The total amount of wall time = 177.287887 - 0: The maximum resident set size (KB) = 844380 + 0: The total amount of wall time = 177.963517 + 0: The maximum resident set size (KB) = 833516 Test 055 regional_2threads_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/regional_noquilt_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/regional_noquilt_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//regional_noquilt_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/regional_noquilt_intel Checking test 056 regional_noquilt_intel results .... Comparing atmos_4xdaily.nc .........OK Comparing fv3_history2d.nc .........OK @@ -2598,28 +2598,28 @@ Checking test 056 regional_noquilt_intel results .... Comparing RESTART/fv_core.res.tile1_new.nc .........OK Comparing RESTART/fv_tracer.res.tile1_new.nc .........OK - 0: The total amount of wall time = 295.889809 - 0: The maximum resident set size (KB) = 853152 + 0: The total amount of wall time = 293.256774 + 0: The maximum resident set size (KB) = 838772 Test 056 regional_noquilt_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/regional_netcdf_parallel_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/regional_netcdf_parallel_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//regional_netcdf_parallel_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/regional_netcdf_parallel_intel Checking test 057 regional_netcdf_parallel_intel results .... Comparing dynf000.nc .........OK Comparing dynf006.nc .........OK Comparing phyf000.nc .........OK Comparing phyf006.nc .........OK - 0: The total amount of wall time = 289.754002 - 0: The maximum resident set size (KB) = 860640 + 0: The total amount of wall time = 289.698293 + 0: The maximum resident set size (KB) = 841004 Test 057 regional_netcdf_parallel_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/regional_control_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/regional_2dwrtdecomp_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//regional_control_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/regional_2dwrtdecomp_intel Checking test 058 regional_2dwrtdecomp_intel results .... Comparing dynf000.nc .........OK Comparing dynf006.nc .........OK @@ -2630,14 +2630,14 @@ Checking test 058 regional_2dwrtdecomp_intel results .... Comparing NATLEV.GrbF00 .........OK Comparing NATLEV.GrbF06 .........OK - 0: The total amount of wall time = 294.015253 - 0: The maximum resident set size (KB) = 864700 + 0: The total amount of wall time = 292.834125 + 0: The maximum resident set size (KB) = 869900 Test 058 regional_2dwrtdecomp_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/fv3_regional_wofs_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/regional_wofs_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//fv3_regional_wofs_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/regional_wofs_intel Checking test 059 regional_wofs_intel results .... Comparing dynf000.nc .........OK Comparing dynf006.nc .........OK @@ -2648,14 +2648,14 @@ Checking test 059 regional_wofs_intel results .... Comparing NATLEV.GrbF00 .........OK Comparing NATLEV.GrbF06 .........OK - 0: The total amount of wall time = 370.676350 - 0: The maximum resident set size (KB) = 623928 + 0: The total amount of wall time = 369.824297 + 0: The maximum resident set size (KB) = 629348 Test 059 regional_wofs_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rap_control_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rap_control_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rap_control_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rap_control_intel Checking test 060 rap_control_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf021.nc .........OK @@ -2702,14 +2702,14 @@ Checking test 060 rap_control_intel results .... Comparing RESTART/20210323.060000.sfc_data.tile5.nc .........OK Comparing RESTART/20210323.060000.sfc_data.tile6.nc .........OK - 0: The total amount of wall time = 446.739542 - 0: The maximum resident set size (KB) = 1051592 + 0: The total amount of wall time = 447.212478 + 0: The maximum resident set size (KB) = 1045268 Test 060 rap_control_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/regional_spp_sppt_shum_skeb_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/regional_spp_sppt_shum_skeb_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//regional_spp_sppt_shum_skeb_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/regional_spp_sppt_shum_skeb_intel Checking test 061 regional_spp_sppt_shum_skeb_intel results .... Comparing dynf000.nc .........OK Comparing dynf001.nc .........OK @@ -2720,14 +2720,14 @@ Checking test 061 regional_spp_sppt_shum_skeb_intel results .... Comparing NATLEV.GrbF00 .........OK Comparing NATLEV.GrbF01 .........OK - 0: The total amount of wall time = 231.674275 - 0: The maximum resident set size (KB) = 1182880 + 0: The total amount of wall time = 227.842921 + 0: The maximum resident set size (KB) = 1162992 Test 061 regional_spp_sppt_shum_skeb_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rap_control_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rap_decomp_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rap_control_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rap_decomp_intel Checking test 062 rap_decomp_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf021.nc .........OK @@ -2774,14 +2774,14 @@ Checking test 062 rap_decomp_intel results .... Comparing RESTART/20210323.060000.sfc_data.tile5.nc .........OK Comparing RESTART/20210323.060000.sfc_data.tile6.nc .........OK - 0: The total amount of wall time = 466.306077 - 0: The maximum resident set size (KB) = 1001956 + 0: The total amount of wall time = 466.727909 + 0: The maximum resident set size (KB) = 992456 Test 062 rap_decomp_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rap_control_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rap_2threads_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rap_control_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rap_2threads_intel Checking test 063 rap_2threads_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf021.nc .........OK @@ -2828,14 +2828,14 @@ Checking test 063 rap_2threads_intel results .... Comparing RESTART/20210323.060000.sfc_data.tile5.nc .........OK Comparing RESTART/20210323.060000.sfc_data.tile6.nc .........OK - 0: The total amount of wall time = 422.113639 - 0: The maximum resident set size (KB) = 1144720 + 0: The total amount of wall time = 417.360828 + 0: The maximum resident set size (KB) = 1129064 Test 063 rap_2threads_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rap_control_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rap_restart_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rap_control_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rap_restart_intel Checking test 064 rap_restart_intel results .... Comparing sfcf024.nc .........OK Comparing atmf024.nc .........OK @@ -2874,14 +2874,14 @@ Checking test 064 rap_restart_intel results .... Comparing RESTART/20210323.060000.sfc_data.tile5.nc .........OK Comparing RESTART/20210323.060000.sfc_data.tile6.nc .........OK - 0: The total amount of wall time = 227.925240 - 0: The maximum resident set size (KB) = 961192 + 0: The total amount of wall time = 226.968828 + 0: The maximum resident set size (KB) = 962108 Test 064 rap_restart_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rap_sfcdiff_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rap_sfcdiff_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rap_sfcdiff_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rap_sfcdiff_intel Checking test 065 rap_sfcdiff_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf009.nc .........OK @@ -2928,14 +2928,14 @@ Checking test 065 rap_sfcdiff_intel results .... Comparing RESTART/20210323.060000.sfc_data.tile5.nc .........OK Comparing RESTART/20210323.060000.sfc_data.tile6.nc .........OK - 0: The total amount of wall time = 448.730173 - 0: The maximum resident set size (KB) = 1048400 + 0: The total amount of wall time = 445.895873 + 0: The maximum resident set size (KB) = 1044040 Test 065 rap_sfcdiff_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rap_sfcdiff_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rap_sfcdiff_decomp_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rap_sfcdiff_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rap_sfcdiff_decomp_intel Checking test 066 rap_sfcdiff_decomp_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf009.nc .........OK @@ -2982,14 +2982,14 @@ Checking test 066 rap_sfcdiff_decomp_intel results .... Comparing RESTART/20210323.060000.sfc_data.tile5.nc .........OK Comparing RESTART/20210323.060000.sfc_data.tile6.nc .........OK - 0: The total amount of wall time = 470.169370 - 0: The maximum resident set size (KB) = 990388 + 0: The total amount of wall time = 469.422085 + 0: The maximum resident set size (KB) = 987984 Test 066 rap_sfcdiff_decomp_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rap_sfcdiff_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rap_sfcdiff_restart_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rap_sfcdiff_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rap_sfcdiff_restart_intel Checking test 067 rap_sfcdiff_restart_intel results .... Comparing sfcf012.nc .........OK Comparing atmf012.nc .........OK @@ -3028,14 +3028,14 @@ Checking test 067 rap_sfcdiff_restart_intel results .... Comparing RESTART/20210323.060000.sfc_data.tile5.nc .........OK Comparing RESTART/20210323.060000.sfc_data.tile6.nc .........OK - 0: The total amount of wall time = 334.316648 - 0: The maximum resident set size (KB) = 958436 + 0: The total amount of wall time = 333.150823 + 0: The maximum resident set size (KB) = 974576 Test 067 rap_sfcdiff_restart_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/hrrr_control_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/hrrr_control_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//hrrr_control_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/hrrr_control_intel Checking test 068 hrrr_control_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf009.nc .........OK @@ -3082,14 +3082,14 @@ Checking test 068 hrrr_control_intel results .... Comparing RESTART/20210322.120000.sfc_data.tile5.nc .........OK Comparing RESTART/20210322.120000.sfc_data.tile6.nc .........OK - 0: The total amount of wall time = 430.877446 - 0: The maximum resident set size (KB) = 1019296 + 0: The total amount of wall time = 428.937833 + 0: The maximum resident set size (KB) = 1046208 Test 068 hrrr_control_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/hrrr_control_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/hrrr_control_qr_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//hrrr_control_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/hrrr_control_qr_intel Checking test 069 hrrr_control_qr_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf009.nc .........OK @@ -3136,14 +3136,14 @@ Checking test 069 hrrr_control_qr_intel results .... Comparing RESTART/20210322.120000.sfc_data.tile5.nc ............ALT CHECK......OK Comparing RESTART/20210322.120000.sfc_data.tile6.nc ............ALT CHECK......OK - 0: The total amount of wall time = 426.090771 - 0: The maximum resident set size (KB) = 1060516 + 0: The total amount of wall time = 426.310890 + 0: The maximum resident set size (KB) = 1052280 Test 069 hrrr_control_qr_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/hrrr_control_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/hrrr_control_decomp_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//hrrr_control_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/hrrr_control_decomp_intel Checking test 070 hrrr_control_decomp_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf009.nc .........OK @@ -3190,14 +3190,14 @@ Checking test 070 hrrr_control_decomp_intel results .... Comparing RESTART/20210322.120000.sfc_data.tile5.nc .........OK Comparing RESTART/20210322.120000.sfc_data.tile6.nc .........OK - 0: The total amount of wall time = 452.170402 - 0: The maximum resident set size (KB) = 998744 + 0: The total amount of wall time = 447.623106 + 0: The maximum resident set size (KB) = 988192 Test 070 hrrr_control_decomp_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/hrrr_control_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/hrrr_control_2threads_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//hrrr_control_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/hrrr_control_2threads_intel Checking test 071 hrrr_control_2threads_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf009.nc .........OK @@ -3244,42 +3244,42 @@ Checking test 071 hrrr_control_2threads_intel results .... Comparing RESTART/20210322.120000.sfc_data.tile5.nc .........OK Comparing RESTART/20210322.120000.sfc_data.tile6.nc .........OK - 0: The total amount of wall time = 395.577096 - 0: The maximum resident set size (KB) = 1064936 + 0: The total amount of wall time = 395.768129 + 0: The maximum resident set size (KB) = 1070084 Test 071 hrrr_control_2threads_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/hrrr_control_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/hrrr_control_restart_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//hrrr_control_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/hrrr_control_restart_intel Checking test 072 hrrr_control_restart_intel results .... Comparing sfcf012.nc .........OK Comparing atmf012.nc .........OK Comparing GFSFLX.GrbF12 .........OK Comparing GFSPRS.GrbF12 .........OK - 0: The total amount of wall time = 322.795944 - 0: The maximum resident set size (KB) = 966264 + 0: The total amount of wall time = 320.223689 + 0: The maximum resident set size (KB) = 942288 Test 072 hrrr_control_restart_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/hrrr_control_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/hrrr_control_restart_qr_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//hrrr_control_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/hrrr_control_restart_qr_intel Checking test 073 hrrr_control_restart_qr_intel results .... Comparing sfcf012.nc .........OK Comparing atmf012.nc .........OK Comparing GFSFLX.GrbF12 .........OK Comparing GFSPRS.GrbF12 .........OK - 0: The total amount of wall time = 328.785549 - 0: The maximum resident set size (KB) = 986492 + 0: The total amount of wall time = 322.303052 + 0: The maximum resident set size (KB) = 987136 Test 073 hrrr_control_restart_qr_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rrfs_v1beta_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rrfs_v1beta_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rrfs_v1beta_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rrfs_v1beta_intel Checking test 074 rrfs_v1beta_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf009.nc .........OK @@ -3326,14 +3326,14 @@ Checking test 074 rrfs_v1beta_intel results .... Comparing RESTART/20210323.060000.sfc_data.tile5.nc .........OK Comparing RESTART/20210323.060000.sfc_data.tile6.nc .........OK - 0: The total amount of wall time = 447.441481 - 0: The maximum resident set size (KB) = 1037628 + 0: The total amount of wall time = 441.032624 + 0: The maximum resident set size (KB) = 1048904 Test 074 rrfs_v1beta_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rrfs_v1nssl_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rrfs_v1nssl_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rrfs_v1nssl_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rrfs_v1nssl_intel Checking test 075 rrfs_v1nssl_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf009.nc .........OK @@ -3348,14 +3348,14 @@ Checking test 075 rrfs_v1nssl_intel results .... Comparing GFSPRS.GrbF09 .........OK Comparing GFSPRS.GrbF12 .........OK - 0: The total amount of wall time = 523.019165 - 0: The maximum resident set size (KB) = 687696 + 0: The total amount of wall time = 521.949863 + 0: The maximum resident set size (KB) = 654992 Test 075 rrfs_v1nssl_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rrfs_v1nssl_nohailnoccn_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rrfs_v1nssl_nohailnoccn_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rrfs_v1nssl_nohailnoccn_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rrfs_v1nssl_nohailnoccn_intel Checking test 076 rrfs_v1nssl_nohailnoccn_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf009.nc .........OK @@ -3370,14 +3370,14 @@ Checking test 076 rrfs_v1nssl_nohailnoccn_intel results .... Comparing GFSPRS.GrbF09 .........OK Comparing GFSPRS.GrbF12 .........OK - 0: The total amount of wall time = 509.728582 - 0: The maximum resident set size (KB) = 753536 + 0: The total amount of wall time = 508.549728 + 0: The maximum resident set size (KB) = 733620 Test 076 rrfs_v1nssl_nohailnoccn_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rrfs_smoke_conus13km_hrrr_warm_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rrfs_smoke_conus13km_hrrr_warm_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rrfs_smoke_conus13km_hrrr_warm_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rrfs_smoke_conus13km_hrrr_warm_intel Checking test 077 rrfs_smoke_conus13km_hrrr_warm_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf001.nc .........OK @@ -3393,14 +3393,14 @@ Checking test 077 rrfs_smoke_conus13km_hrrr_warm_intel results .... Comparing RESTART/20210512.170000.phy_data.nc .........OK Comparing RESTART/20210512.170000.sfc_data.nc .........OK - 0: The total amount of wall time = 152.815085 - 0: The maximum resident set size (KB) = 993552 + 0: The total amount of wall time = 147.660570 + 0: The maximum resident set size (KB) = 1013920 Test 077 rrfs_smoke_conus13km_hrrr_warm_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rrfs_smoke_conus13km_hrrr_warm_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rrfs_smoke_conus13km_hrrr_warm_qr_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rrfs_smoke_conus13km_hrrr_warm_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rrfs_smoke_conus13km_hrrr_warm_qr_intel Checking test 078 rrfs_smoke_conus13km_hrrr_warm_qr_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf001.nc .........OK @@ -3416,14 +3416,14 @@ Checking test 078 rrfs_smoke_conus13km_hrrr_warm_qr_intel results .... Comparing RESTART/20210512.170000.phy_data.nc ............ALT CHECK......OK Comparing RESTART/20210512.170000.sfc_data.nc ............ALT CHECK......OK - 0: The total amount of wall time = 145.713531 - 0: The maximum resident set size (KB) = 937984 + 0: The total amount of wall time = 143.999077 + 0: The maximum resident set size (KB) = 971408 Test 078 rrfs_smoke_conus13km_hrrr_warm_qr_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rrfs_smoke_conus13km_hrrr_warm_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rrfs_smoke_conus13km_hrrr_warm_2threads_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rrfs_smoke_conus13km_hrrr_warm_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rrfs_smoke_conus13km_hrrr_warm_2threads_intel Checking test 079 rrfs_smoke_conus13km_hrrr_warm_2threads_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf001.nc .........OK @@ -3432,14 +3432,14 @@ Checking test 079 rrfs_smoke_conus13km_hrrr_warm_2threads_intel results .... Comparing atmf001.nc .........OK Comparing atmf002.nc .........OK - 0: The total amount of wall time = 93.788388 - 0: The maximum resident set size (KB) = 941860 + 0: The total amount of wall time = 91.639153 + 0: The maximum resident set size (KB) = 922812 Test 079 rrfs_smoke_conus13km_hrrr_warm_2threads_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rrfs_conus13km_hrrr_warm_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rrfs_conus13km_hrrr_warm_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rrfs_conus13km_hrrr_warm_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rrfs_conus13km_hrrr_warm_intel Checking test 080 rrfs_conus13km_hrrr_warm_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf001.nc .........OK @@ -3455,14 +3455,14 @@ Checking test 080 rrfs_conus13km_hrrr_warm_intel results .... Comparing RESTART/20210512.170000.phy_data.nc .........OK Comparing RESTART/20210512.170000.sfc_data.nc .........OK - 0: The total amount of wall time = 132.645649 - 0: The maximum resident set size (KB) = 979884 + 0: The total amount of wall time = 131.372733 + 0: The maximum resident set size (KB) = 967728 Test 080 rrfs_conus13km_hrrr_warm_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rrfs_smoke_conus13km_hrrr_warm_radar_tten_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rrfs_smoke_conus13km_radar_tten_warm_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rrfs_smoke_conus13km_hrrr_warm_radar_tten_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rrfs_smoke_conus13km_radar_tten_warm_intel Checking test 081 rrfs_smoke_conus13km_radar_tten_warm_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf001.nc .........OK @@ -3471,38 +3471,38 @@ Checking test 081 rrfs_smoke_conus13km_radar_tten_warm_intel results .... Comparing atmf001.nc .........OK Comparing atmf002.nc .........OK - 0: The total amount of wall time = 151.107222 - 0: The maximum resident set size (KB) = 1024792 + 0: The total amount of wall time = 150.106795 + 0: The maximum resident set size (KB) = 1003632 Test 081 rrfs_smoke_conus13km_radar_tten_warm_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rrfs_smoke_conus13km_hrrr_warm_restart_mismatch_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rrfs_smoke_conus13km_hrrr_warm_restart_mismatch_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rrfs_smoke_conus13km_hrrr_warm_restart_mismatch_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rrfs_smoke_conus13km_hrrr_warm_restart_mismatch_intel Checking test 082 rrfs_smoke_conus13km_hrrr_warm_restart_mismatch_intel results .... Comparing sfcf002.nc .........OK Comparing atmf002.nc .........OK - 0: The total amount of wall time = 90.575245 - 0: The maximum resident set size (KB) = 1014872 + 0: The total amount of wall time = 83.781506 + 0: The maximum resident set size (KB) = 1010768 Test 082 rrfs_smoke_conus13km_hrrr_warm_restart_mismatch_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rrfs_smoke_conus13km_hrrr_warm_restart_mismatch_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rrfs_smoke_conus13km_hrrr_warm_restart_qr_mismatch_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rrfs_smoke_conus13km_hrrr_warm_restart_mismatch_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rrfs_smoke_conus13km_hrrr_warm_restart_qr_mismatch_intel Checking test 083 rrfs_smoke_conus13km_hrrr_warm_restart_qr_mismatch_intel results .... Comparing sfcf002.nc .........OK Comparing atmf002.nc .........OK - 0: The total amount of wall time = 103.338684 - 0: The maximum resident set size (KB) = 1011696 + 0: The total amount of wall time = 88.761641 + 0: The maximum resident set size (KB) = 1000704 Test 083 rrfs_smoke_conus13km_hrrr_warm_restart_qr_mismatch_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/control_csawmg_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/control_csawmg_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//control_csawmg_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/control_csawmg_intel Checking test 084 control_csawmg_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf024.nc .........OK @@ -3513,14 +3513,14 @@ Checking test 084 control_csawmg_intel results .... Comparing GFSPRS.GrbF00 .........OK Comparing GFSPRS.GrbF24 .........OK - 0: The total amount of wall time = 341.552636 - 0: The maximum resident set size (KB) = 729076 + 0: The total amount of wall time = 333.482240 + 0: The maximum resident set size (KB) = 722664 Test 084 control_csawmg_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/control_csawmgt_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/control_csawmgt_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//control_csawmgt_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/control_csawmgt_intel Checking test 085 control_csawmgt_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf024.nc .........OK @@ -3531,14 +3531,14 @@ Checking test 085 control_csawmgt_intel results .... Comparing GFSPRS.GrbF00 .........OK Comparing GFSPRS.GrbF24 .........OK - 0: The total amount of wall time = 333.753097 - 0: The maximum resident set size (KB) = 722216 + 0: The total amount of wall time = 331.767306 + 0: The maximum resident set size (KB) = 723744 Test 085 control_csawmgt_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/control_ras_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/control_ras_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//control_ras_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/control_ras_intel Checking test 086 control_ras_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf024.nc .........OK @@ -3549,26 +3549,26 @@ Checking test 086 control_ras_intel results .... Comparing GFSPRS.GrbF00 .........OK Comparing GFSPRS.GrbF24 .........OK - 0: The total amount of wall time = 185.478281 - 0: The maximum resident set size (KB) = 692152 + 0: The total amount of wall time = 182.916688 + 0: The maximum resident set size (KB) = 721336 Test 086 control_ras_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/control_wam_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/control_wam_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//control_wam_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/control_wam_intel Checking test 087 control_wam_intel results .... Comparing sfcf024.nc .........OK Comparing atmf024.nc .........OK - 0: The total amount of wall time = 113.049960 - 0: The maximum resident set size (KB) = 636144 + 0: The total amount of wall time = 114.084401 + 0: The maximum resident set size (KB) = 635060 Test 087 control_wam_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/control_p8_faster_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/control_p8_faster_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//control_p8_faster_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/control_p8_faster_intel Checking test 088 control_p8_faster_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf021.nc .........OK @@ -3615,14 +3615,14 @@ Checking test 088 control_p8_faster_intel results .... Comparing RESTART/20210323.060000.sfc_data.tile5.nc .........OK Comparing RESTART/20210323.060000.sfc_data.tile6.nc .........OK - 0: The total amount of wall time = 156.773330 - 0: The maximum resident set size (KB) = 1595552 + 0: The total amount of wall time = 154.517155 + 0: The maximum resident set size (KB) = 1598412 Test 088 control_p8_faster_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/regional_control_faster_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/regional_control_faster_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//regional_control_faster_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/regional_control_faster_intel Checking test 089 regional_control_faster_intel results .... Comparing dynf000.nc .........OK Comparing dynf006.nc .........OK @@ -3633,56 +3633,56 @@ Checking test 089 regional_control_faster_intel results .... Comparing NATLEV.GrbF00 .........OK Comparing NATLEV.GrbF06 .........OK - 0: The total amount of wall time = 269.901503 - 0: The maximum resident set size (KB) = 865892 + 0: The total amount of wall time = 265.602258 + 0: The maximum resident set size (KB) = 867896 Test 089 regional_control_faster_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rrfs_smoke_conus13km_hrrr_warm_debug_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rrfs_smoke_conus13km_hrrr_warm_debug_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rrfs_smoke_conus13km_hrrr_warm_debug_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rrfs_smoke_conus13km_hrrr_warm_debug_intel Checking test 090 rrfs_smoke_conus13km_hrrr_warm_debug_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf001.nc .........OK Comparing atmf000.nc .........OK Comparing atmf001.nc .........OK - 0: The total amount of wall time = 826.733265 - 0: The maximum resident set size (KB) = 1056952 + 0: The total amount of wall time = 840.415869 + 0: The maximum resident set size (KB) = 1044632 Test 090 rrfs_smoke_conus13km_hrrr_warm_debug_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rrfs_smoke_conus13km_hrrr_warm_debug_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rrfs_smoke_conus13km_hrrr_warm_debug_2threads_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rrfs_smoke_conus13km_hrrr_warm_debug_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rrfs_smoke_conus13km_hrrr_warm_debug_2threads_intel Checking test 091 rrfs_smoke_conus13km_hrrr_warm_debug_2threads_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf001.nc .........OK Comparing atmf000.nc .........OK Comparing atmf001.nc .........OK - 0: The total amount of wall time = 477.230110 - 0: The maximum resident set size (KB) = 972896 + 0: The total amount of wall time = 470.600591 + 0: The maximum resident set size (KB) = 965680 Test 091 rrfs_smoke_conus13km_hrrr_warm_debug_2threads_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rrfs_conus13km_hrrr_warm_debugs_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rrfs_conus13km_hrrr_warm_debug_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rrfs_conus13km_hrrr_warm_debugs_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rrfs_conus13km_hrrr_warm_debug_intel Checking test 092 rrfs_conus13km_hrrr_warm_debug_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf001.nc .........OK Comparing atmf000.nc .........OK Comparing atmf001.nc .........OK - 0: The total amount of wall time = 748.974215 - 0: The maximum resident set size (KB) = 1020988 + 0: The total amount of wall time = 731.759084 + 0: The maximum resident set size (KB) = 1010640 Test 092 rrfs_conus13km_hrrr_warm_debug_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/control_CubedSphereGrid_debug_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/control_CubedSphereGrid_debug_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//control_CubedSphereGrid_debug_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/control_CubedSphereGrid_debug_intel Checking test 093 control_CubedSphereGrid_debug_intel results .... Comparing sfcf000.tile1.nc .........OK Comparing sfcf000.tile2.nc .........OK @@ -3709,348 +3709,348 @@ Checking test 093 control_CubedSphereGrid_debug_intel results .... Comparing atmf001.tile5.nc .........OK Comparing atmf001.tile6.nc .........OK - 0: The total amount of wall time = 145.573810 - 0: The maximum resident set size (KB) = 785508 + 0: The total amount of wall time = 146.611106 + 0: The maximum resident set size (KB) = 791908 Test 093 control_CubedSphereGrid_debug_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/control_wrtGauss_netcdf_parallel_debug_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/control_wrtGauss_netcdf_parallel_debug_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//control_wrtGauss_netcdf_parallel_debug_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/control_wrtGauss_netcdf_parallel_debug_intel Checking test 094 control_wrtGauss_netcdf_parallel_debug_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf001.nc .........OK Comparing atmf000.nc .........OK Comparing atmf001.nc .........OK - 0: The total amount of wall time = 148.584572 - 0: The maximum resident set size (KB) = 785992 + 0: The total amount of wall time = 146.266752 + 0: The maximum resident set size (KB) = 783932 Test 094 control_wrtGauss_netcdf_parallel_debug_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/control_stochy_debug_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/control_stochy_debug_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//control_stochy_debug_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/control_stochy_debug_intel Checking test 095 control_stochy_debug_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf001.nc .........OK Comparing atmf000.nc .........OK Comparing atmf001.nc .........OK - 0: The total amount of wall time = 169.209649 - 0: The maximum resident set size (KB) = 791756 + 0: The total amount of wall time = 167.970117 + 0: The maximum resident set size (KB) = 766052 Test 095 control_stochy_debug_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/control_lndp_debug_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/control_lndp_debug_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//control_lndp_debug_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/control_lndp_debug_intel Checking test 096 control_lndp_debug_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf001.nc .........OK Comparing atmf000.nc .........OK Comparing atmf001.nc .........OK - 0: The total amount of wall time = 150.284063 - 0: The maximum resident set size (KB) = 793384 + 0: The total amount of wall time = 149.180773 + 0: The maximum resident set size (KB) = 789556 Test 096 control_lndp_debug_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/control_csawmg_debug_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/control_csawmg_debug_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//control_csawmg_debug_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/control_csawmg_debug_intel Checking test 097 control_csawmg_debug_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf001.nc .........OK Comparing atmf000.nc .........OK Comparing atmf001.nc .........OK - 0: The total amount of wall time = 230.316912 - 0: The maximum resident set size (KB) = 842360 + 0: The total amount of wall time = 231.317705 + 0: The maximum resident set size (KB) = 836772 Test 097 control_csawmg_debug_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/control_csawmgt_debug_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/control_csawmgt_debug_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//control_csawmgt_debug_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/control_csawmgt_debug_intel Checking test 098 control_csawmgt_debug_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf001.nc .........OK Comparing atmf000.nc .........OK Comparing atmf001.nc .........OK - 0: The total amount of wall time = 223.410077 - 0: The maximum resident set size (KB) = 843556 + 0: The total amount of wall time = 224.670374 + 0: The maximum resident set size (KB) = 837976 Test 098 control_csawmgt_debug_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/control_ras_debug_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/control_ras_debug_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//control_ras_debug_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/control_ras_debug_intel Checking test 099 control_ras_debug_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf001.nc .........OK Comparing atmf000.nc .........OK Comparing atmf001.nc .........OK - 0: The total amount of wall time = 150.817268 - 0: The maximum resident set size (KB) = 802400 + 0: The total amount of wall time = 146.960082 + 0: The maximum resident set size (KB) = 799920 Test 099 control_ras_debug_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/control_diag_debug_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/control_diag_debug_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//control_diag_debug_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/control_diag_debug_intel Checking test 100 control_diag_debug_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf001.nc .........OK Comparing atmf000.nc .........OK Comparing atmf001.nc .........OK - 0: The total amount of wall time = 154.706356 - 0: The maximum resident set size (KB) = 838480 + 0: The total amount of wall time = 152.860385 + 0: The maximum resident set size (KB) = 849004 Test 100 control_diag_debug_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/control_debug_p8_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/control_debug_p8_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//control_debug_p8_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/control_debug_p8_intel Checking test 101 control_debug_p8_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf001.nc .........OK Comparing atmf000.nc .........OK Comparing atmf001.nc .........OK - 0: The total amount of wall time = 166.262034 - 0: The maximum resident set size (KB) = 1618988 + 0: The total amount of wall time = 165.694826 + 0: The maximum resident set size (KB) = 1621356 Test 101 control_debug_p8_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/regional_debug_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/regional_debug_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//regional_debug_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/regional_debug_intel Checking test 102 regional_debug_intel results .... Comparing dynf000.nc .........OK Comparing dynf001.nc .........OK Comparing phyf000.nc .........OK Comparing phyf001.nc .........OK - 0: The total amount of wall time = 987.223895 - 0: The maximum resident set size (KB) = 896808 + 0: The total amount of wall time = 968.390005 + 0: The maximum resident set size (KB) = 883560 Test 102 regional_debug_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rap_control_debug_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rap_control_debug_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rap_control_debug_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rap_control_debug_intel Checking test 103 rap_control_debug_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf001.nc .........OK Comparing atmf000.nc .........OK Comparing atmf001.nc .........OK - 0: The total amount of wall time = 272.147995 - 0: The maximum resident set size (KB) = 1169640 + 0: The total amount of wall time = 269.846871 + 0: The maximum resident set size (KB) = 1165300 Test 103 rap_control_debug_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/hrrr_control_debug_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/hrrr_control_debug_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//hrrr_control_debug_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/hrrr_control_debug_intel Checking test 104 hrrr_control_debug_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf001.nc .........OK Comparing atmf000.nc .........OK Comparing atmf001.nc .........OK - 0: The total amount of wall time = 263.131439 - 0: The maximum resident set size (KB) = 1136536 + 0: The total amount of wall time = 263.117833 + 0: The maximum resident set size (KB) = 1158176 Test 104 hrrr_control_debug_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rap_control_debug_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rap_unified_drag_suite_debug_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rap_control_debug_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rap_unified_drag_suite_debug_intel Checking test 105 rap_unified_drag_suite_debug_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf001.nc .........OK Comparing atmf000.nc .........OK Comparing atmf001.nc .........OK - 0: The total amount of wall time = 275.097533 - 0: The maximum resident set size (KB) = 1165812 + 0: The total amount of wall time = 271.581307 + 0: The maximum resident set size (KB) = 1162688 Test 105 rap_unified_drag_suite_debug_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rap_diag_debug_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rap_diag_debug_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rap_diag_debug_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rap_diag_debug_intel Checking test 106 rap_diag_debug_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf001.nc .........OK Comparing atmf000.nc .........OK Comparing atmf001.nc .........OK - 0: The total amount of wall time = 286.790277 - 0: The maximum resident set size (KB) = 1253540 + 0: The total amount of wall time = 290.987711 + 0: The maximum resident set size (KB) = 1245648 Test 106 rap_diag_debug_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rap_cires_ugwp_debug_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rap_cires_ugwp_debug_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rap_cires_ugwp_debug_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rap_cires_ugwp_debug_intel Checking test 107 rap_cires_ugwp_debug_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf001.nc .........OK Comparing atmf000.nc .........OK Comparing atmf001.nc .........OK - 0: The total amount of wall time = 279.659504 - 0: The maximum resident set size (KB) = 1146644 + 0: The total amount of wall time = 279.213034 + 0: The maximum resident set size (KB) = 1167456 Test 107 rap_cires_ugwp_debug_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rap_cires_ugwp_debug_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rap_unified_ugwp_debug_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rap_cires_ugwp_debug_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rap_unified_ugwp_debug_intel Checking test 108 rap_unified_ugwp_debug_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf001.nc .........OK Comparing atmf000.nc .........OK Comparing atmf001.nc .........OK - 0: The total amount of wall time = 283.634479 - 0: The maximum resident set size (KB) = 1140620 + 0: The total amount of wall time = 278.970865 + 0: The maximum resident set size (KB) = 1164500 Test 108 rap_unified_ugwp_debug_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rap_lndp_debug_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rap_lndp_debug_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rap_lndp_debug_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rap_lndp_debug_intel Checking test 109 rap_lndp_debug_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf001.nc .........OK Comparing atmf000.nc .........OK Comparing atmf001.nc .........OK - 0: The total amount of wall time = 276.319560 - 0: The maximum resident set size (KB) = 1164452 + 0: The total amount of wall time = 268.762777 + 0: The maximum resident set size (KB) = 1148820 Test 109 rap_lndp_debug_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rap_progcld_thompson_debug_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rap_progcld_thompson_debug_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rap_progcld_thompson_debug_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rap_progcld_thompson_debug_intel Checking test 110 rap_progcld_thompson_debug_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf001.nc .........OK Comparing atmf000.nc .........OK Comparing atmf001.nc .........OK - 0: The total amount of wall time = 269.910330 - 0: The maximum resident set size (KB) = 1167152 + 0: The total amount of wall time = 272.474943 + 0: The maximum resident set size (KB) = 1165880 Test 110 rap_progcld_thompson_debug_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rap_noah_debug_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rap_noah_debug_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rap_noah_debug_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rap_noah_debug_intel Checking test 111 rap_noah_debug_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf001.nc .........OK Comparing atmf000.nc .........OK Comparing atmf001.nc .........OK - 0: The total amount of wall time = 274.255152 - 0: The maximum resident set size (KB) = 1139852 + 0: The total amount of wall time = 264.938492 + 0: The maximum resident set size (KB) = 1165472 Test 111 rap_noah_debug_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rap_sfcdiff_debug_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rap_sfcdiff_debug_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rap_sfcdiff_debug_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rap_sfcdiff_debug_intel Checking test 112 rap_sfcdiff_debug_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf001.nc .........OK Comparing atmf000.nc .........OK Comparing atmf001.nc .........OK - 0: The total amount of wall time = 277.480043 - 0: The maximum resident set size (KB) = 1171196 + 0: The total amount of wall time = 274.281897 + 0: The maximum resident set size (KB) = 1145184 Test 112 rap_sfcdiff_debug_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rap_noah_sfcdiff_cires_ugwp_debug_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rap_noah_sfcdiff_cires_ugwp_debug_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rap_noah_sfcdiff_cires_ugwp_debug_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rap_noah_sfcdiff_cires_ugwp_debug_intel Checking test 113 rap_noah_sfcdiff_cires_ugwp_debug_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf001.nc .........OK Comparing atmf000.nc .........OK Comparing atmf001.nc .........OK - 0: The total amount of wall time = 443.613506 - 0: The maximum resident set size (KB) = 1161460 + 0: The total amount of wall time = 444.346094 + 0: The maximum resident set size (KB) = 1140876 Test 113 rap_noah_sfcdiff_cires_ugwp_debug_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rrfs_v1beta_debug_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rrfs_v1beta_debug_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rrfs_v1beta_debug_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rrfs_v1beta_debug_intel Checking test 114 rrfs_v1beta_debug_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf001.nc .........OK Comparing atmf000.nc .........OK Comparing atmf001.nc .........OK - 0: The total amount of wall time = 269.193685 - 0: The maximum resident set size (KB) = 1165084 + 0: The total amount of wall time = 269.769267 + 0: The maximum resident set size (KB) = 1141456 Test 114 rrfs_v1beta_debug_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rap_clm_lake_debug_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rap_clm_lake_debug_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rap_clm_lake_debug_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rap_clm_lake_debug_intel Checking test 115 rap_clm_lake_debug_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf001.nc .........OK Comparing atmf000.nc .........OK Comparing atmf001.nc .........OK - 0: The total amount of wall time = 359.366318 - 0: The maximum resident set size (KB) = 1163920 + 0: The total amount of wall time = 348.384173 + 0: The maximum resident set size (KB) = 1160780 Test 115 rap_clm_lake_debug_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rap_flake_debug_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rap_flake_debug_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rap_flake_debug_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rap_flake_debug_intel Checking test 116 rap_flake_debug_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf001.nc .........OK Comparing atmf000.nc .........OK Comparing atmf001.nc .........OK - 0: The total amount of wall time = 272.351756 - 0: The maximum resident set size (KB) = 1152672 + 0: The total amount of wall time = 265.252464 + 0: The maximum resident set size (KB) = 1168744 Test 116 rap_flake_debug_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/control_wam_debug_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/control_wam_debug_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//control_wam_debug_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/control_wam_debug_intel Checking test 117 control_wam_debug_intel results .... Comparing sfcf019.nc .........OK Comparing atmf019.nc .........OK - 0: The total amount of wall time = 272.877196 - 0: The maximum resident set size (KB) = 511644 + 0: The total amount of wall time = 273.692241 + 0: The maximum resident set size (KB) = 518804 Test 117 control_wam_debug_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/regional_spp_sppt_shum_skeb_dyn32_phy32_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/regional_spp_sppt_shum_skeb_dyn32_phy32_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//regional_spp_sppt_shum_skeb_dyn32_phy32_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/regional_spp_sppt_shum_skeb_dyn32_phy32_intel Checking test 118 regional_spp_sppt_shum_skeb_dyn32_phy32_intel results .... Comparing dynf000.nc .........OK Comparing dynf001.nc .........OK @@ -4061,14 +4061,14 @@ Checking test 118 regional_spp_sppt_shum_skeb_dyn32_phy32_intel results .... Comparing NATLEV.GrbF00 .........OK Comparing NATLEV.GrbF01 .........OK - 0: The total amount of wall time = 218.172374 - 0: The maximum resident set size (KB) = 1072012 + 0: The total amount of wall time = 213.174668 + 0: The maximum resident set size (KB) = 1072132 Test 118 regional_spp_sppt_shum_skeb_dyn32_phy32_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rap_control_dyn32_phy32_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rap_control_dyn32_phy32_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rap_control_dyn32_phy32_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rap_control_dyn32_phy32_intel Checking test 119 rap_control_dyn32_phy32_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf009.nc .........OK @@ -4115,14 +4115,14 @@ Checking test 119 rap_control_dyn32_phy32_intel results .... Comparing RESTART/20210323.060000.sfc_data.tile5.nc .........OK Comparing RESTART/20210323.060000.sfc_data.tile6.nc .........OK - 0: The total amount of wall time = 381.635324 - 0: The maximum resident set size (KB) = 999204 + 0: The total amount of wall time = 368.184558 + 0: The maximum resident set size (KB) = 998940 Test 119 rap_control_dyn32_phy32_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/hrrr_control_dyn32_phy32_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/hrrr_control_dyn32_phy32_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//hrrr_control_dyn32_phy32_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/hrrr_control_dyn32_phy32_intel Checking test 120 hrrr_control_dyn32_phy32_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf009.nc .........OK @@ -4169,14 +4169,14 @@ Checking test 120 hrrr_control_dyn32_phy32_intel results .... Comparing RESTART/20210322.120000.sfc_data.tile5.nc .........OK Comparing RESTART/20210322.120000.sfc_data.tile6.nc .........OK - 0: The total amount of wall time = 194.689344 - 0: The maximum resident set size (KB) = 946604 + 0: The total amount of wall time = 188.922928 + 0: The maximum resident set size (KB) = 952684 Test 120 hrrr_control_dyn32_phy32_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/hrrr_control_qr_dyn32_phy32_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/hrrr_control_qr_dyn32_phy32_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//hrrr_control_qr_dyn32_phy32_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/hrrr_control_qr_dyn32_phy32_intel Checking test 121 hrrr_control_qr_dyn32_phy32_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf009.nc .........OK @@ -4223,14 +4223,14 @@ Checking test 121 hrrr_control_qr_dyn32_phy32_intel results .... Comparing RESTART/20210322.120000.sfc_data.tile5.nc .........OK Comparing RESTART/20210322.120000.sfc_data.tile6.nc .........OK - 0: The total amount of wall time = 195.752656 - 0: The maximum resident set size (KB) = 964084 + 0: The total amount of wall time = 190.619993 + 0: The maximum resident set size (KB) = 926436 Test 121 hrrr_control_qr_dyn32_phy32_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rap_control_dyn32_phy32_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rap_2threads_dyn32_phy32_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rap_control_dyn32_phy32_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rap_2threads_dyn32_phy32_intel Checking test 122 rap_2threads_dyn32_phy32_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf009.nc .........OK @@ -4277,14 +4277,14 @@ Checking test 122 rap_2threads_dyn32_phy32_intel results .... Comparing RESTART/20210323.060000.sfc_data.tile5.nc .........OK Comparing RESTART/20210323.060000.sfc_data.tile6.nc .........OK - 0: The total amount of wall time = 350.875599 - 0: The maximum resident set size (KB) = 1016228 + 0: The total amount of wall time = 348.227194 + 0: The maximum resident set size (KB) = 1018148 Test 122 rap_2threads_dyn32_phy32_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/hrrr_control_dyn32_phy32_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/hrrr_control_2threads_dyn32_phy32_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//hrrr_control_dyn32_phy32_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/hrrr_control_2threads_dyn32_phy32_intel Checking test 123 hrrr_control_2threads_dyn32_phy32_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf009.nc .........OK @@ -4331,14 +4331,14 @@ Checking test 123 hrrr_control_2threads_dyn32_phy32_intel results .... Comparing RESTART/20210322.120000.sfc_data.tile5.nc .........OK Comparing RESTART/20210322.120000.sfc_data.tile6.nc .........OK - 0: The total amount of wall time = 175.797174 - 0: The maximum resident set size (KB) = 935820 + 0: The total amount of wall time = 175.098220 + 0: The maximum resident set size (KB) = 933968 Test 123 hrrr_control_2threads_dyn32_phy32_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/hrrr_control_dyn32_phy32_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/hrrr_control_decomp_dyn32_phy32_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//hrrr_control_dyn32_phy32_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/hrrr_control_decomp_dyn32_phy32_intel Checking test 124 hrrr_control_decomp_dyn32_phy32_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf009.nc .........OK @@ -4385,14 +4385,14 @@ Checking test 124 hrrr_control_decomp_dyn32_phy32_intel results .... Comparing RESTART/20210322.120000.sfc_data.tile5.nc .........OK Comparing RESTART/20210322.120000.sfc_data.tile6.nc .........OK - 0: The total amount of wall time = 206.797253 - 0: The maximum resident set size (KB) = 874100 + 0: The total amount of wall time = 201.933773 + 0: The maximum resident set size (KB) = 889572 Test 124 hrrr_control_decomp_dyn32_phy32_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rap_control_dyn32_phy32_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rap_restart_dyn32_phy32_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rap_control_dyn32_phy32_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rap_restart_dyn32_phy32_intel Checking test 125 rap_restart_dyn32_phy32_intel results .... Comparing sfcf012.nc .........OK Comparing atmf012.nc .........OK @@ -4431,42 +4431,42 @@ Checking test 125 rap_restart_dyn32_phy32_intel results .... Comparing RESTART/20210323.060000.sfc_data.tile5.nc .........OK Comparing RESTART/20210323.060000.sfc_data.tile6.nc .........OK - 0: The total amount of wall time = 275.590248 - 0: The maximum resident set size (KB) = 905804 + 0: The total amount of wall time = 273.406448 + 0: The maximum resident set size (KB) = 941740 Test 125 rap_restart_dyn32_phy32_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/hrrr_control_dyn32_phy32_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/hrrr_control_restart_dyn32_phy32_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//hrrr_control_dyn32_phy32_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/hrrr_control_restart_dyn32_phy32_intel Checking test 126 hrrr_control_restart_dyn32_phy32_intel results .... Comparing sfcf012.nc .........OK Comparing atmf012.nc .........OK Comparing GFSFLX.GrbF12 .........OK Comparing GFSPRS.GrbF12 .........OK - 0: The total amount of wall time = 100.546020 - 0: The maximum resident set size (KB) = 858624 + 0: The total amount of wall time = 97.643561 + 0: The maximum resident set size (KB) = 858148 Test 126 hrrr_control_restart_dyn32_phy32_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/hrrr_control_qr_dyn32_phy32_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/hrrr_control_restart_qr_dyn32_phy32_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//hrrr_control_qr_dyn32_phy32_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/hrrr_control_restart_qr_dyn32_phy32_intel Checking test 127 hrrr_control_restart_qr_dyn32_phy32_intel results .... Comparing sfcf012.nc .........OK Comparing atmf012.nc .........OK Comparing GFSFLX.GrbF12 .........OK Comparing GFSPRS.GrbF12 .........OK - 0: The total amount of wall time = 102.773207 - 0: The maximum resident set size (KB) = 879052 + 0: The total amount of wall time = 100.846124 + 0: The maximum resident set size (KB) = 869688 Test 127 hrrr_control_restart_qr_dyn32_phy32_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rrfs_smoke_conus13km_fast_phy32_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rrfs_smoke_conus13km_fast_phy32_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rrfs_smoke_conus13km_fast_phy32_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rrfs_smoke_conus13km_fast_phy32_intel Checking test 128 rrfs_smoke_conus13km_fast_phy32_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf001.nc .........OK @@ -4482,14 +4482,14 @@ Checking test 128 rrfs_smoke_conus13km_fast_phy32_intel results .... Comparing RESTART/20210512.170000.phy_data.nc .........OK Comparing RESTART/20210512.170000.sfc_data.nc .........OK - 0: The total amount of wall time = 118.951505 - 0: The maximum resident set size (KB) = 899872 + 0: The total amount of wall time = 114.136248 + 0: The maximum resident set size (KB) = 900972 Test 128 rrfs_smoke_conus13km_fast_phy32_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rrfs_smoke_conus13km_fast_phy32_qr_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rrfs_smoke_conus13km_fast_phy32_qr_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rrfs_smoke_conus13km_fast_phy32_qr_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rrfs_smoke_conus13km_fast_phy32_qr_intel Checking test 129 rrfs_smoke_conus13km_fast_phy32_qr_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf001.nc .........OK @@ -4505,38 +4505,38 @@ Checking test 129 rrfs_smoke_conus13km_fast_phy32_qr_intel results .... Comparing RESTART/20210512.170000.phy_data.nc .........OK Comparing RESTART/20210512.170000.sfc_data.nc .........OK - 0: The total amount of wall time = 127.627747 - 0: The maximum resident set size (KB) = 927708 + 0: The total amount of wall time = 111.023086 + 0: The maximum resident set size (KB) = 891964 Test 129 rrfs_smoke_conus13km_fast_phy32_qr_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rrfs_smoke_conus13km_fast_phy32_restart_mismatch_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rrfs_smoke_conus13km_fast_phy32_restart_mismatch_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rrfs_smoke_conus13km_fast_phy32_restart_mismatch_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rrfs_smoke_conus13km_fast_phy32_restart_mismatch_intel Checking test 130 rrfs_smoke_conus13km_fast_phy32_restart_mismatch_intel results .... Comparing sfcf002.nc .........OK Comparing atmf002.nc .........OK - 0: The total amount of wall time = 65.714152 - 0: The maximum resident set size (KB) = 903820 + 0: The total amount of wall time = 64.079056 + 0: The maximum resident set size (KB) = 898844 Test 130 rrfs_smoke_conus13km_fast_phy32_restart_mismatch_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rrfs_smoke_conus13km_fast_phy32_restart_mismatch_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rrfs_smoke_conus13km_fast_phy32_restart_qr_mismatch_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rrfs_smoke_conus13km_fast_phy32_restart_mismatch_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rrfs_smoke_conus13km_fast_phy32_restart_qr_mismatch_intel Checking test 131 rrfs_smoke_conus13km_fast_phy32_restart_qr_mismatch_intel results .... Comparing sfcf002.nc .........OK Comparing atmf002.nc .........OK - 0: The total amount of wall time = 73.186216 - 0: The maximum resident set size (KB) = 913648 + 0: The total amount of wall time = 69.333129 + 0: The maximum resident set size (KB) = 910472 Test 131 rrfs_smoke_conus13km_fast_phy32_restart_qr_mismatch_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rap_control_dyn64_phy32_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rap_control_dyn64_phy32_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rap_control_dyn64_phy32_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rap_control_dyn64_phy32_intel Checking test 132 rap_control_dyn64_phy32_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf009.nc .........OK @@ -4583,81 +4583,81 @@ Checking test 132 rap_control_dyn64_phy32_intel results .... Comparing RESTART/20210322.180000.sfc_data.tile5.nc .........OK Comparing RESTART/20210322.180000.sfc_data.tile6.nc .........OK - 0: The total amount of wall time = 239.737796 - 0: The maximum resident set size (KB) = 937164 + 0: The total amount of wall time = 238.468165 + 0: The maximum resident set size (KB) = 949580 Test 132 rap_control_dyn64_phy32_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rap_control_debug_dyn32_phy32_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rap_control_debug_dyn32_phy32_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rap_control_debug_dyn32_phy32_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rap_control_debug_dyn32_phy32_intel Checking test 133 rap_control_debug_dyn32_phy32_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf001.nc .........OK Comparing atmf000.nc .........OK Comparing atmf001.nc .........OK - 0: The total amount of wall time = 266.638823 - 0: The maximum resident set size (KB) = 1054300 + 0: The total amount of wall time = 273.305983 + 0: The maximum resident set size (KB) = 1048748 Test 133 rap_control_debug_dyn32_phy32_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/hrrr_control_debug_dyn32_phy32_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/hrrr_control_debug_dyn32_phy32_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//hrrr_control_debug_dyn32_phy32_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/hrrr_control_debug_dyn32_phy32_intel Checking test 134 hrrr_control_debug_dyn32_phy32_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf001.nc .........OK Comparing atmf000.nc .........OK Comparing atmf001.nc .........OK - 0: The total amount of wall time = 259.251442 - 0: The maximum resident set size (KB) = 1049888 + 0: The total amount of wall time = 262.388901 + 0: The maximum resident set size (KB) = 1047860 Test 134 hrrr_control_debug_dyn32_phy32_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rap_control_debug_dyn64_phy32_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rap_control_dyn64_phy32_debug_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rap_control_debug_dyn64_phy32_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rap_control_dyn64_phy32_debug_intel Checking test 135 rap_control_dyn64_phy32_debug_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf001.nc .........OK Comparing atmf000.nc .........OK Comparing atmf001.nc .........OK - 0: The total amount of wall time = 284.456348 - 0: The maximum resident set size (KB) = 1092112 + 0: The total amount of wall time = 273.000263 + 0: The maximum resident set size (KB) = 1089976 Test 135 rap_control_dyn64_phy32_debug_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/hafs_regional_atm_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/hafs_regional_atm_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//hafs_regional_atm_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/hafs_regional_atm_intel Checking test 136 hafs_regional_atm_intel results .... Comparing atmf006.nc .........OK Comparing sfcf006.nc .........OK Comparing HURPRS.GrbF06 .........OK - 0: The total amount of wall time = 227.798438 - 0: The maximum resident set size (KB) = 1046044 + 0: The total amount of wall time = 225.600688 + 0: The maximum resident set size (KB) = 1016384 Test 136 hafs_regional_atm_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/hafs_regional_atm_thompson_gfdlsf_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/hafs_regional_atm_thompson_gfdlsf_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//hafs_regional_atm_thompson_gfdlsf_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/hafs_regional_atm_thompson_gfdlsf_intel Checking test 137 hafs_regional_atm_thompson_gfdlsf_intel results .... Comparing atmf006.nc .........OK Comparing sfcf006.nc .........OK - 0: The total amount of wall time = 337.361815 - 0: The maximum resident set size (KB) = 1419680 + 0: The total amount of wall time = 326.407833 + 0: The maximum resident set size (KB) = 1373612 Test 137 hafs_regional_atm_thompson_gfdlsf_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/hafs_regional_atm_ocn_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/hafs_regional_atm_ocn_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//hafs_regional_atm_ocn_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/hafs_regional_atm_ocn_intel Checking test 138 hafs_regional_atm_ocn_intel results .... Comparing atmf006.nc .........OK Comparing sfcf006.nc .........OK @@ -4666,14 +4666,14 @@ Checking test 138 hafs_regional_atm_ocn_intel results .... Comparing ufs.hafs.cpl.hi.2019-08-29-21600.nc .........OK Comparing ufs.hafs.cpl.r.2019-08-29-21600.nc .........OK - 0: The total amount of wall time = 386.642363 - 0: The maximum resident set size (KB) = 1216804 + 0: The total amount of wall time = 384.900409 + 0: The maximum resident set size (KB) = 1214568 Test 138 hafs_regional_atm_ocn_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/hafs_regional_atm_wav_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/hafs_regional_atm_wav_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//hafs_regional_atm_wav_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/hafs_regional_atm_wav_intel Checking test 139 hafs_regional_atm_wav_intel results .... Comparing atmf006.nc .........OK Comparing sfcf006.nc .........OK @@ -4682,14 +4682,14 @@ Checking test 139 hafs_regional_atm_wav_intel results .... Comparing ufs.hafs.ww3.r.2019-08-29-21600 .........OK Comparing ufs.hafs.cpl.r.2019-08-29-21600.nc .........OK - 0: The total amount of wall time = 740.447046 - 0: The maximum resident set size (KB) = 1234816 + 0: The total amount of wall time = 743.219393 + 0: The maximum resident set size (KB) = 1209208 Test 139 hafs_regional_atm_wav_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/hafs_regional_atm_ocn_wav_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/hafs_regional_atm_ocn_wav_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//hafs_regional_atm_ocn_wav_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/hafs_regional_atm_ocn_wav_intel Checking test 140 hafs_regional_atm_ocn_wav_intel results .... Comparing atmf006.nc .........OK Comparing sfcf006.nc .........OK @@ -4700,14 +4700,14 @@ Checking test 140 hafs_regional_atm_ocn_wav_intel results .... Comparing ufs.hafs.ww3.r.2019-08-29-21600 .........OK Comparing ufs.hafs.cpl.r.2019-08-29-21600.nc .........OK - 0: The total amount of wall time = 844.309706 - 0: The maximum resident set size (KB) = 1266088 + 0: The total amount of wall time = 838.594453 + 0: The maximum resident set size (KB) = 1261860 Test 140 hafs_regional_atm_ocn_wav_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/hafs_regional_1nest_atm_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/hafs_regional_1nest_atm_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//hafs_regional_1nest_atm_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/hafs_regional_1nest_atm_intel Checking test 141 hafs_regional_1nest_atm_intel results .... Comparing atmf006.nc .........OK Comparing sfcf006.nc .........OK @@ -4729,14 +4729,14 @@ Checking test 141 hafs_regional_1nest_atm_intel results .... Comparing RESTART/fv_BC_ne.res.nest02.nc .........OK Comparing RESTART/fv_BC_sw.res.nest02.nc .........OK - 0: The total amount of wall time = 296.029387 - 0: The maximum resident set size (KB) = 502632 + 0: The total amount of wall time = 296.265454 + 0: The maximum resident set size (KB) = 496712 Test 141 hafs_regional_1nest_atm_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/hafs_regional_1nest_atm_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/hafs_regional_1nest_atm_qr_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//hafs_regional_1nest_atm_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/hafs_regional_1nest_atm_qr_intel Checking test 142 hafs_regional_1nest_atm_qr_intel results .... Comparing atmf006.nc .........OK Comparing sfcf006.nc .........OK @@ -4758,14 +4758,14 @@ Checking test 142 hafs_regional_1nest_atm_qr_intel results .... Comparing RESTART/fv_BC_ne.res.nest02.nc .........OK Comparing RESTART/fv_BC_sw.res.nest02.nc .........OK - 0: The total amount of wall time = 313.005283 - 0: The maximum resident set size (KB) = 469840 + 0: The total amount of wall time = 305.635623 + 0: The maximum resident set size (KB) = 474720 Test 142 hafs_regional_1nest_atm_qr_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/hafs_regional_telescopic_2nests_atm_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/hafs_regional_telescopic_2nests_atm_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//hafs_regional_telescopic_2nests_atm_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/hafs_regional_telescopic_2nests_atm_intel Checking test 143 hafs_regional_telescopic_2nests_atm_intel results .... Comparing atmf006.nc .........OK Comparing sfcf006.nc .........OK @@ -4774,14 +4774,14 @@ Checking test 143 hafs_regional_telescopic_2nests_atm_intel results .... Comparing atm.nest03.f006.nc .........OK Comparing sfc.nest03.f006.nc .........OK - 0: The total amount of wall time = 339.282115 - 0: The maximum resident set size (KB) = 508356 + 0: The total amount of wall time = 338.774708 + 0: The maximum resident set size (KB) = 509264 Test 143 hafs_regional_telescopic_2nests_atm_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/hafs_global_1nest_atm_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/hafs_global_1nest_atm_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//hafs_global_1nest_atm_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/hafs_global_1nest_atm_intel Checking test 144 hafs_global_1nest_atm_intel results .... Comparing atmf006.nc .........OK Comparing sfcf006.nc .........OK @@ -4828,14 +4828,14 @@ Checking test 144 hafs_global_1nest_atm_intel results .... Comparing RESTART/fv_BC_ne.res.nest02.nc .........OK Comparing RESTART/fv_BC_sw.res.nest02.nc .........OK - 0: The total amount of wall time = 136.731444 - 0: The maximum resident set size (KB) = 349284 + 0: The total amount of wall time = 137.309747 + 0: The maximum resident set size (KB) = 348260 Test 144 hafs_global_1nest_atm_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/hafs_global_1nest_atm_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/hafs_global_1nest_atm_qr_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//hafs_global_1nest_atm_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/hafs_global_1nest_atm_qr_intel Checking test 145 hafs_global_1nest_atm_qr_intel results .... Comparing atmf006.nc .........OK Comparing sfcf006.nc .........OK @@ -4882,14 +4882,14 @@ Checking test 145 hafs_global_1nest_atm_qr_intel results .... Comparing RESTART/fv_BC_ne.res.nest02.nc .........OK Comparing RESTART/fv_BC_sw.res.nest02.nc .........OK - 0: The total amount of wall time = 149.200136 - 0: The maximum resident set size (KB) = 352228 + 0: The total amount of wall time = 142.170040 + 0: The maximum resident set size (KB) = 318936 Test 145 hafs_global_1nest_atm_qr_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/hafs_global_multiple_4nests_atm_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/hafs_global_multiple_4nests_atm_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//hafs_global_multiple_4nests_atm_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/hafs_global_multiple_4nests_atm_intel Checking test 146 hafs_global_multiple_4nests_atm_intel results .... Comparing atmf006.nc .........OK Comparing sfcf006.nc .........OK @@ -4971,14 +4971,14 @@ Checking test 146 hafs_global_multiple_4nests_atm_intel results .... Comparing RESTART/fv_BC_sw.res.nest04.nc .........OK Comparing RESTART/fv_BC_sw.res.nest05.nc .........OK - 0: The total amount of wall time = 392.780159 - 0: The maximum resident set size (KB) = 448504 + 0: The total amount of wall time = 389.857180 + 0: The maximum resident set size (KB) = 416104 Test 146 hafs_global_multiple_4nests_atm_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/hafs_global_multiple_4nests_atm_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/hafs_global_multiple_4nests_atm_qr_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//hafs_global_multiple_4nests_atm_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/hafs_global_multiple_4nests_atm_qr_intel Checking test 147 hafs_global_multiple_4nests_atm_qr_intel results .... Comparing atmf006.nc .........OK Comparing sfcf006.nc .........OK @@ -5060,14 +5060,14 @@ Checking test 147 hafs_global_multiple_4nests_atm_qr_intel results .... Comparing RESTART/fv_BC_sw.res.nest04.nc .........OK Comparing RESTART/fv_BC_sw.res.nest05.nc .........OK - 0: The total amount of wall time = 425.525552 - 0: The maximum resident set size (KB) = 466580 + 0: The total amount of wall time = 408.050421 + 0: The maximum resident set size (KB) = 467948 Test 147 hafs_global_multiple_4nests_atm_qr_intel PASS Tries: 2 -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/hafs_regional_specified_moving_1nest_atm_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/hafs_regional_specified_moving_1nest_atm_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//hafs_regional_specified_moving_1nest_atm_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/hafs_regional_specified_moving_1nest_atm_intel Checking test 148 hafs_regional_specified_moving_1nest_atm_intel results .... Comparing atmf006.nc .........OK Comparing sfcf006.nc .........OK @@ -5076,14 +5076,14 @@ Checking test 148 hafs_regional_specified_moving_1nest_atm_intel results .... Comparing HURPRS.GrbF06 .........OK Comparing HURPRS.GrbF06.nest02 .........OK - 0: The total amount of wall time = 194.352563 - 0: The maximum resident set size (KB) = 519028 + 0: The total amount of wall time = 192.010365 + 0: The maximum resident set size (KB) = 520012 Test 148 hafs_regional_specified_moving_1nest_atm_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/hafs_regional_storm_following_1nest_atm_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/hafs_regional_storm_following_1nest_atm_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//hafs_regional_storm_following_1nest_atm_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/hafs_regional_storm_following_1nest_atm_intel Checking test 149 hafs_regional_storm_following_1nest_atm_intel results .... Comparing atmf006.nc .........OK Comparing sfcf006.nc .........OK @@ -5105,14 +5105,14 @@ Checking test 149 hafs_regional_storm_following_1nest_atm_intel results .... Comparing RESTART/fv_BC_ne.res.nest02.nc .........OK Comparing RESTART/fv_BC_sw.res.nest02.nc .........OK - 0: The total amount of wall time = 182.163000 - 0: The maximum resident set size (KB) = 517244 + 0: The total amount of wall time = 181.067505 + 0: The maximum resident set size (KB) = 511660 Test 149 hafs_regional_storm_following_1nest_atm_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/hafs_regional_storm_following_1nest_atm_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/hafs_regional_storm_following_1nest_atm_qr_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//hafs_regional_storm_following_1nest_atm_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/hafs_regional_storm_following_1nest_atm_qr_intel Checking test 150 hafs_regional_storm_following_1nest_atm_qr_intel results .... Comparing atmf006.nc .........OK Comparing sfcf006.nc .........OK @@ -5134,14 +5134,14 @@ Checking test 150 hafs_regional_storm_following_1nest_atm_qr_intel results .... Comparing RESTART/fv_BC_ne.res.nest02.nc .........OK Comparing RESTART/fv_BC_sw.res.nest02.nc .........OK - 0: The total amount of wall time = 196.638985 - 0: The maximum resident set size (KB) = 501176 + 0: The total amount of wall time = 193.426499 + 0: The maximum resident set size (KB) = 493832 Test 150 hafs_regional_storm_following_1nest_atm_qr_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/hafs_regional_storm_following_1nest_atm_ocn_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/hafs_regional_storm_following_1nest_atm_ocn_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//hafs_regional_storm_following_1nest_atm_ocn_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/hafs_regional_storm_following_1nest_atm_ocn_intel Checking test 151 hafs_regional_storm_following_1nest_atm_ocn_intel results .... Comparing atmf006.nc .........OK Comparing sfcf006.nc .........OK @@ -5150,42 +5150,42 @@ Checking test 151 hafs_regional_storm_following_1nest_atm_ocn_intel results .... Comparing archv.2020_238_18.a .........OK Comparing archs.2020_238_18.a .........OK - 0: The total amount of wall time = 219.952170 - 0: The maximum resident set size (KB) = 561320 + 0: The total amount of wall time = 221.927778 + 0: The maximum resident set size (KB) = 564764 Test 151 hafs_regional_storm_following_1nest_atm_ocn_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/hafs_global_storm_following_1nest_atm_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/hafs_global_storm_following_1nest_atm_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//hafs_global_storm_following_1nest_atm_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/hafs_global_storm_following_1nest_atm_intel Checking test 152 hafs_global_storm_following_1nest_atm_intel results .... Comparing atmf006.nc .........OK Comparing sfcf006.nc .........OK Comparing atm.nest02.f006.nc .........OK Comparing sfc.nest02.f006.nc .........OK - 0: The total amount of wall time = 56.031971 - 0: The maximum resident set size (KB) = 363508 + 0: The total amount of wall time = 56.905576 + 0: The maximum resident set size (KB) = 365136 Test 152 hafs_global_storm_following_1nest_atm_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/hafs_regional_storm_following_1nest_atm_ocn_debug_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/hafs_regional_storm_following_1nest_atm_ocn_debug_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//hafs_regional_storm_following_1nest_atm_ocn_debug_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/hafs_regional_storm_following_1nest_atm_ocn_debug_intel Checking test 153 hafs_regional_storm_following_1nest_atm_ocn_debug_intel results .... Comparing atmf001.nc .........OK Comparing sfcf001.nc .........OK Comparing atm.nest02.f001.nc .........OK Comparing sfc.nest02.f001.nc .........OK - 0: The total amount of wall time = 706.010175 - 0: The maximum resident set size (KB) = 584800 + 0: The total amount of wall time = 721.197339 + 0: The maximum resident set size (KB) = 575092 Test 153 hafs_regional_storm_following_1nest_atm_ocn_debug_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/hafs_regional_storm_following_1nest_atm_ocn_wav_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/hafs_regional_storm_following_1nest_atm_ocn_wav_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//hafs_regional_storm_following_1nest_atm_ocn_wav_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/hafs_regional_storm_following_1nest_atm_ocn_wav_intel Checking test 154 hafs_regional_storm_following_1nest_atm_ocn_wav_intel results .... Comparing atmf006.nc .........OK Comparing sfcf006.nc .........OK @@ -5196,14 +5196,14 @@ Checking test 154 hafs_regional_storm_following_1nest_atm_ocn_wav_intel results Comparing 20200825.180000.out_grd.ww3 .........OK Comparing 20200825.180000.out_pnt.ww3 .........OK - 0: The total amount of wall time = 491.158699 - 0: The maximum resident set size (KB) = 616064 + 0: The total amount of wall time = 488.740252 + 0: The maximum resident set size (KB) = 588864 Test 154 hafs_regional_storm_following_1nest_atm_ocn_wav_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/hafs_regional_docn_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/hafs_regional_docn_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//hafs_regional_docn_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/hafs_regional_docn_intel Checking test 155 hafs_regional_docn_intel results .... Comparing atmf006.nc .........OK Comparing sfcf006.nc .........OK @@ -5211,14 +5211,14 @@ Checking test 155 hafs_regional_docn_intel results .... Comparing ufs.hafs.cpl.r.2019-08-29-21600.nc .........OK Comparing ufs.hafs.docn.r.2019-08-29-21600.nc .........OK - 0: The total amount of wall time = 347.865979 - 0: The maximum resident set size (KB) = 1227280 + 0: The total amount of wall time = 340.317617 + 0: The maximum resident set size (KB) = 1224712 Test 155 hafs_regional_docn_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/hafs_regional_docn_oisst_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/hafs_regional_docn_oisst_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//hafs_regional_docn_oisst_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/hafs_regional_docn_oisst_intel Checking test 156 hafs_regional_docn_oisst_intel results .... Comparing atmf006.nc .........OK Comparing sfcf006.nc .........OK @@ -5226,131 +5226,131 @@ Checking test 156 hafs_regional_docn_oisst_intel results .... Comparing ufs.hafs.cpl.r.2019-08-29-21600.nc .........OK Comparing ufs.hafs.docn.r.2019-08-29-21600.nc .........OK - 0: The total amount of wall time = 345.899870 - 0: The maximum resident set size (KB) = 1218876 + 0: The total amount of wall time = 340.272225 + 0: The maximum resident set size (KB) = 1204940 Test 156 hafs_regional_docn_oisst_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/hafs_regional_datm_cdeps_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/hafs_regional_datm_cdeps_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//hafs_regional_datm_cdeps_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/hafs_regional_datm_cdeps_intel Checking test 157 hafs_regional_datm_cdeps_intel results .... Comparing ufs.hafs.cpl.hi.2019-08-30-00000.nc .........OK Comparing ufs.hafs.cpl.r.2019-08-30-00000.nc .........OK Comparing ufs.hafs.datm.r.2019-08-30-00000.nc .........OK - 0: The total amount of wall time = 964.283768 - 0: The maximum resident set size (KB) = 1038380 + 0: The total amount of wall time = 961.673312 + 0: The maximum resident set size (KB) = 1036080 Test 157 hafs_regional_datm_cdeps_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/datm_cdeps_control_cfsr_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/datm_cdeps_control_cfsr_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//datm_cdeps_control_cfsr_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/datm_cdeps_control_cfsr_intel Checking test 158 datm_cdeps_control_cfsr_intel results .... Comparing RESTART/20111002.000000.MOM.res.nc .........OK Comparing RESTART/iced.2011-10-02-00000.nc .........OK Comparing RESTART/DATM_CFSR.cpl.r.2011-10-02-00000.nc .........OK - 0: The total amount of wall time = 160.476033 - 0: The maximum resident set size (KB) = 1056428 + 0: The total amount of wall time = 158.788510 + 0: The maximum resident set size (KB) = 1071160 Test 158 datm_cdeps_control_cfsr_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/datm_cdeps_control_cfsr_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/datm_cdeps_restart_cfsr_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//datm_cdeps_control_cfsr_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/datm_cdeps_restart_cfsr_intel Checking test 159 datm_cdeps_restart_cfsr_intel results .... Comparing RESTART/20111002.000000.MOM.res.nc .........OK Comparing RESTART/iced.2011-10-02-00000.nc .........OK Comparing RESTART/DATM_CFSR.cpl.r.2011-10-02-00000.nc .........OK - 0: The total amount of wall time = 90.274600 - 0: The maximum resident set size (KB) = 1017864 + 0: The total amount of wall time = 89.644601 + 0: The maximum resident set size (KB) = 999452 Test 159 datm_cdeps_restart_cfsr_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/datm_cdeps_control_gefs_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/datm_cdeps_control_gefs_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//datm_cdeps_control_gefs_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/datm_cdeps_control_gefs_intel Checking test 160 datm_cdeps_control_gefs_intel results .... Comparing RESTART/20111002.000000.MOM.res.nc .........OK Comparing RESTART/iced.2011-10-02-00000.nc .........OK Comparing RESTART/DATM_GEFS_NEW.cpl.r.2011-10-02-00000.nc .........OK - 0: The total amount of wall time = 149.887590 - 0: The maximum resident set size (KB) = 951200 + 0: The total amount of wall time = 146.222700 + 0: The maximum resident set size (KB) = 960328 Test 160 datm_cdeps_control_gefs_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/datm_cdeps_iau_gefs_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/datm_cdeps_iau_gefs_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//datm_cdeps_iau_gefs_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/datm_cdeps_iau_gefs_intel Checking test 161 datm_cdeps_iau_gefs_intel results .... Comparing RESTART/20111002.000000.MOM.res.nc .........OK Comparing RESTART/iced.2011-10-02-00000.nc .........OK Comparing RESTART/DATM_GEFS_NEW.cpl.r.2011-10-02-00000.nc .........OK - 0: The total amount of wall time = 148.361401 - 0: The maximum resident set size (KB) = 959304 + 0: The total amount of wall time = 152.879097 + 0: The maximum resident set size (KB) = 953920 Test 161 datm_cdeps_iau_gefs_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/datm_cdeps_stochy_gefs_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/datm_cdeps_stochy_gefs_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//datm_cdeps_stochy_gefs_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/datm_cdeps_stochy_gefs_intel Checking test 162 datm_cdeps_stochy_gefs_intel results .... Comparing RESTART/20111002.000000.MOM.res.nc .........OK Comparing RESTART/iced.2011-10-02-00000.nc .........OK Comparing RESTART/DATM_GEFS_NEW.cpl.r.2011-10-02-00000.nc .........OK - 0: The total amount of wall time = 151.339504 - 0: The maximum resident set size (KB) = 955316 + 0: The total amount of wall time = 147.870529 + 0: The maximum resident set size (KB) = 958748 Test 162 datm_cdeps_stochy_gefs_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/datm_cdeps_ciceC_cfsr_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/datm_cdeps_ciceC_cfsr_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//datm_cdeps_ciceC_cfsr_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/datm_cdeps_ciceC_cfsr_intel Checking test 163 datm_cdeps_ciceC_cfsr_intel results .... Comparing RESTART/20111002.000000.MOM.res.nc .........OK Comparing RESTART/iced.2011-10-02-00000.nc .........OK Comparing RESTART/DATM_CFSR.cpl.r.2011-10-02-00000.nc .........OK - 0: The total amount of wall time = 154.546208 - 0: The maximum resident set size (KB) = 1040900 + 0: The total amount of wall time = 154.522755 + 0: The maximum resident set size (KB) = 1065064 Test 163 datm_cdeps_ciceC_cfsr_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/datm_cdeps_bulk_cfsr_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/datm_cdeps_bulk_cfsr_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//datm_cdeps_bulk_cfsr_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/datm_cdeps_bulk_cfsr_intel Checking test 164 datm_cdeps_bulk_cfsr_intel results .... Comparing RESTART/20111002.000000.MOM.res.nc .........OK Comparing RESTART/iced.2011-10-02-00000.nc .........OK Comparing RESTART/DATM_CFSR.cpl.r.2011-10-02-00000.nc .........OK - 0: The total amount of wall time = 151.735389 - 0: The maximum resident set size (KB) = 1055872 + 0: The total amount of wall time = 152.274936 + 0: The maximum resident set size (KB) = 1060156 Test 164 datm_cdeps_bulk_cfsr_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/datm_cdeps_bulk_gefs_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/datm_cdeps_bulk_gefs_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//datm_cdeps_bulk_gefs_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/datm_cdeps_bulk_gefs_intel Checking test 165 datm_cdeps_bulk_gefs_intel results .... Comparing RESTART/20111002.000000.MOM.res.nc .........OK Comparing RESTART/iced.2011-10-02-00000.nc .........OK Comparing RESTART/DATM_GEFS_NEW.cpl.r.2011-10-02-00000.nc .........OK - 0: The total amount of wall time = 142.437584 - 0: The maximum resident set size (KB) = 958776 + 0: The total amount of wall time = 149.159191 + 0: The maximum resident set size (KB) = 954324 Test 165 datm_cdeps_bulk_gefs_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/datm_cdeps_mx025_cfsr_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/datm_cdeps_mx025_cfsr_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//datm_cdeps_mx025_cfsr_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/datm_cdeps_mx025_cfsr_intel Checking test 166 datm_cdeps_mx025_cfsr_intel results .... Comparing RESTART/20111001.120000.MOM.res.nc .........OK Comparing RESTART/20111001.120000.MOM.res_1.nc .........OK @@ -5359,14 +5359,14 @@ Checking test 166 datm_cdeps_mx025_cfsr_intel results .... Comparing RESTART/iced.2011-10-01-43200.nc .........OK Comparing RESTART/DATM_CFSR.cpl.r.2011-10-01-43200.nc .........OK - 0: The total amount of wall time = 425.896000 - 0: The maximum resident set size (KB) = 872068 + 0: The total amount of wall time = 468.089401 + 0: The maximum resident set size (KB) = 879924 Test 166 datm_cdeps_mx025_cfsr_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/datm_cdeps_mx025_gefs_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/datm_cdeps_mx025_gefs_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//datm_cdeps_mx025_gefs_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/datm_cdeps_mx025_gefs_intel Checking test 167 datm_cdeps_mx025_gefs_intel results .... Comparing RESTART/20111001.120000.MOM.res.nc .........OK Comparing RESTART/20111001.120000.MOM.res_1.nc .........OK @@ -5375,77 +5375,77 @@ Checking test 167 datm_cdeps_mx025_gefs_intel results .... Comparing RESTART/iced.2011-10-01-43200.nc .........OK Comparing RESTART/DATM_GEFS_NEW.cpl.r.2011-10-01-43200.nc .........OK - 0: The total amount of wall time = 425.123876 - 0: The maximum resident set size (KB) = 933816 + 0: The total amount of wall time = 445.498808 + 0: The maximum resident set size (KB) = 938488 Test 167 datm_cdeps_mx025_gefs_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/datm_cdeps_control_cfsr_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/datm_cdeps_multiple_files_cfsr_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//datm_cdeps_control_cfsr_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/datm_cdeps_multiple_files_cfsr_intel Checking test 168 datm_cdeps_multiple_files_cfsr_intel results .... Comparing RESTART/DATM_CFSR.cpl.r.2011-10-02-00000.nc .........OK - 0: The total amount of wall time = 157.119890 - 0: The maximum resident set size (KB) = 1052444 + 0: The total amount of wall time = 151.945452 + 0: The maximum resident set size (KB) = 1060192 Test 168 datm_cdeps_multiple_files_cfsr_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/datm_cdeps_3072x1536_cfsr_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/datm_cdeps_3072x1536_cfsr_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//datm_cdeps_3072x1536_cfsr_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/datm_cdeps_3072x1536_cfsr_intel Checking test 169 datm_cdeps_3072x1536_cfsr_intel results .... Comparing RESTART/20111002.000000.MOM.res.nc .........OK Comparing RESTART/iced.2011-10-02-00000.nc .........OK Comparing RESTART/DATM_CFSR3072x1536.cpl.r.2011-10-02-00000.nc .........OK - 0: The total amount of wall time = 217.520657 - 0: The maximum resident set size (KB) = 2343376 + 0: The total amount of wall time = 222.585274 + 0: The maximum resident set size (KB) = 2348108 Test 169 datm_cdeps_3072x1536_cfsr_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/datm_cdeps_gfs_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/datm_cdeps_gfs_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//datm_cdeps_gfs_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/datm_cdeps_gfs_intel Checking test 170 datm_cdeps_gfs_intel results .... Comparing RESTART/20210323.060000.MOM.res.nc .........OK Comparing RESTART/iced.2021-03-23-21600.nc .........OK Comparing RESTART/DATM_GFS.cpl.r.2021-03-23-21600.nc .........OK - 0: The total amount of wall time = 228.107984 - 0: The maximum resident set size (KB) = 2360024 + 0: The total amount of wall time = 213.700263 + 0: The maximum resident set size (KB) = 2356656 Test 170 datm_cdeps_gfs_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/datm_cdeps_debug_cfsr_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/datm_cdeps_debug_cfsr_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//datm_cdeps_debug_cfsr_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/datm_cdeps_debug_cfsr_intel Checking test 171 datm_cdeps_debug_cfsr_intel results .... Comparing RESTART/20111001.060000.MOM.res.nc .........OK Comparing RESTART/iced.2011-10-01-21600.nc .........OK Comparing RESTART/DATM_CFSR.cpl.r.2011-10-01-21600.nc .........OK - 0: The total amount of wall time = 348.619314 - 0: The maximum resident set size (KB) = 980788 + 0: The total amount of wall time = 357.438564 + 0: The maximum resident set size (KB) = 985436 Test 171 datm_cdeps_debug_cfsr_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/datm_cdeps_control_cfsr_faster_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/datm_cdeps_control_cfsr_faster_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//datm_cdeps_control_cfsr_faster_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/datm_cdeps_control_cfsr_faster_intel Checking test 172 datm_cdeps_control_cfsr_faster_intel results .... Comparing RESTART/20111002.000000.MOM.res.nc .........OK Comparing RESTART/iced.2011-10-02-00000.nc .........OK Comparing RESTART/DATM_CFSR.cpl.r.2011-10-02-00000.nc .........OK - 0: The total amount of wall time = 152.047541 - 0: The maximum resident set size (KB) = 1055488 + 0: The total amount of wall time = 151.619903 + 0: The maximum resident set size (KB) = 1045324 Test 172 datm_cdeps_control_cfsr_faster_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/datm_cdeps_lnd_gswp3_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/datm_cdeps_lnd_gswp3_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//datm_cdeps_lnd_gswp3_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/datm_cdeps_lnd_gswp3_intel Checking test 173 datm_cdeps_lnd_gswp3_intel results .... Comparing ufs.cpld.lnd.out.2000-01-02-00000.tile1.nc .........OK Comparing ufs.cpld.lnd.out.2000-01-02-00000.tile2.nc .........OK @@ -5454,14 +5454,14 @@ Checking test 173 datm_cdeps_lnd_gswp3_intel results .... Comparing ufs.cpld.lnd.out.2000-01-02-00000.tile5.nc .........OK Comparing ufs.cpld.lnd.out.2000-01-02-00000.tile6.nc .........OK - 0: The total amount of wall time = 7.045600 - 0: The maximum resident set size (KB) = 262416 + 0: The total amount of wall time = 7.277277 + 0: The maximum resident set size (KB) = 256772 Test 173 datm_cdeps_lnd_gswp3_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/datm_cdeps_lnd_gswp3_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/datm_cdeps_lnd_gswp3_rst_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//datm_cdeps_lnd_gswp3_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/datm_cdeps_lnd_gswp3_rst_intel Checking test 174 datm_cdeps_lnd_gswp3_rst_intel results .... Comparing ufs.cpld.lnd.out.2000-01-02-00000.tile1.nc .........OK Comparing ufs.cpld.lnd.out.2000-01-02-00000.tile2.nc .........OK @@ -5470,14 +5470,14 @@ Checking test 174 datm_cdeps_lnd_gswp3_rst_intel results .... Comparing ufs.cpld.lnd.out.2000-01-02-00000.tile5.nc .........OK Comparing ufs.cpld.lnd.out.2000-01-02-00000.tile6.nc .........OK - 0: The total amount of wall time = 12.681403 - 0: The maximum resident set size (KB) = 238488 + 0: The total amount of wall time = 12.127870 + 0: The maximum resident set size (KB) = 262384 Test 174 datm_cdeps_lnd_gswp3_rst_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/control_p8_atmlnd_sbs_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/control_p8_atmlnd_sbs_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//control_p8_atmlnd_sbs_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/control_p8_atmlnd_sbs_intel Checking test 175 control_p8_atmlnd_sbs_intel results .... Comparing sfcf000.tile1.nc .........OK Comparing sfcf000.tile2.nc .........OK @@ -5562,14 +5562,14 @@ Checking test 175 control_p8_atmlnd_sbs_intel results .... Comparing ufs.cpld.lnd.out.2021-03-23-21600.tile5.nc .........OK Comparing ufs.cpld.lnd.out.2021-03-23-21600.tile6.nc .........OK - 0: The total amount of wall time = 207.150934 - 0: The maximum resident set size (KB) = 1586492 + 0: The total amount of wall time = 204.331308 + 0: The maximum resident set size (KB) = 1604140 Test 175 control_p8_atmlnd_sbs_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/atmwav_control_noaero_p8_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/atmwav_control_noaero_p8_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//atmwav_control_noaero_p8_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/atmwav_control_noaero_p8_intel Checking test 176 atmwav_control_noaero_p8_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf012.nc .........OK @@ -5612,14 +5612,14 @@ Checking test 176 atmwav_control_noaero_p8_intel results .... Comparing 20210322.180000.out_grd.ww3 .........OK Comparing ufs.atmw.ww3.r.2021-03-22-64800 .........OK - 0: The total amount of wall time = 92.419003 - 0: The maximum resident set size (KB) = 1630960 + 0: The total amount of wall time = 93.983300 + 0: The maximum resident set size (KB) = 1635200 Test 176 atmwav_control_noaero_p8_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/control_atmwav_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/control_atmwav_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//control_atmwav_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/control_atmwav_intel Checking test 177 control_atmwav_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf012.nc .........OK @@ -5663,14 +5663,14 @@ Checking test 177 control_atmwav_intel results .... Comparing RESTART/20210322.180000.sfc_data.tile6.nc .........OK Comparing 20210322.180000.restart.glo_1deg .........OK - 0: The total amount of wall time = 87.609841 - 0: The maximum resident set size (KB) = 657128 + 0: The total amount of wall time = 88.854224 + 0: The maximum resident set size (KB) = 662936 Test 177 control_atmwav_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/atmaero_control_p8_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/atmaero_control_p8_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//atmaero_control_p8_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/atmaero_control_p8_intel Checking test 178 atmaero_control_p8_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf024.nc .........OK @@ -5714,14 +5714,14 @@ Checking test 178 atmaero_control_p8_intel results .... Comparing RESTART/20210323.060000.sfc_data.tile5.nc .........OK Comparing RESTART/20210323.060000.sfc_data.tile6.nc .........OK - 0: The total amount of wall time = 230.034592 - 0: The maximum resident set size (KB) = 2923432 + 0: The total amount of wall time = 228.813591 + 0: The maximum resident set size (KB) = 2972624 Test 178 atmaero_control_p8_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/atmaero_control_p8_rad_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/atmaero_control_p8_rad_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//atmaero_control_p8_rad_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/atmaero_control_p8_rad_intel Checking test 179 atmaero_control_p8_rad_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf024.nc .........OK @@ -5765,14 +5765,14 @@ Checking test 179 atmaero_control_p8_rad_intel results .... Comparing RESTART/20210323.060000.sfc_data.tile5.nc .........OK Comparing RESTART/20210323.060000.sfc_data.tile6.nc .........OK - 0: The total amount of wall time = 286.665156 - 0: The maximum resident set size (KB) = 3013864 + 0: The total amount of wall time = 283.816316 + 0: The maximum resident set size (KB) = 3006652 Test 179 atmaero_control_p8_rad_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/atmaero_control_p8_rad_micro_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/atmaero_control_p8_rad_micro_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//atmaero_control_p8_rad_micro_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/atmaero_control_p8_rad_micro_intel Checking test 180 atmaero_control_p8_rad_micro_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf024.nc .........OK @@ -5816,14 +5816,14 @@ Checking test 180 atmaero_control_p8_rad_micro_intel results .... Comparing RESTART/20210323.060000.sfc_data.tile5.nc .........OK Comparing RESTART/20210323.060000.sfc_data.tile6.nc .........OK - 0: The total amount of wall time = 284.864780 - 0: The maximum resident set size (KB) = 3006060 + 0: The total amount of wall time = 288.731158 + 0: The maximum resident set size (KB) = 3042592 Test 180 atmaero_control_p8_rad_micro_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/regional_atmaq_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/regional_atmaq_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//regional_atmaq_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/regional_atmaq_intel Checking test 181 regional_atmaq_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf003.nc .........OK @@ -5839,14 +5839,14 @@ Checking test 181 regional_atmaq_intel results .... Comparing RESTART/20190801.180000.phy_data.nc .........OK Comparing RESTART/20190801.180000.sfc_data.nc .........OK - 0: The total amount of wall time = 671.832866 - 0: The maximum resident set size (KB) = 1456820 + 0: The total amount of wall time = 632.374902 + 0: The maximum resident set size (KB) = 1462944 Test 181 regional_atmaq_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/regional_atmaq_debug_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/regional_atmaq_debug_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//regional_atmaq_debug_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/regional_atmaq_debug_intel Checking test 182 regional_atmaq_debug_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf001.nc .........OK @@ -5860,14 +5860,14 @@ Checking test 182 regional_atmaq_debug_intel results .... Comparing RESTART/20190801.130000.phy_data.nc .........OK Comparing RESTART/20190801.130000.sfc_data.nc .........OK - 0: The total amount of wall time = 1207.614128 - 0: The maximum resident set size (KB) = 1313672 + 0: The total amount of wall time = 1186.161533 + 0: The maximum resident set size (KB) = 1269728 Test 182 regional_atmaq_debug_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/regional_atmaq_faster_intel -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/regional_atmaq_faster_intel +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//regional_atmaq_faster_intel +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/regional_atmaq_faster_intel Checking test 183 regional_atmaq_faster_intel results .... Comparing sfcf000.nc .........OK Comparing sfcf003.nc .........OK @@ -5883,14 +5883,14 @@ Checking test 183 regional_atmaq_faster_intel results .... Comparing RESTART/20190801.180000.phy_data.nc .........OK Comparing RESTART/20190801.180000.sfc_data.nc .........OK - 0: The total amount of wall time = 593.227728 - 0: The maximum resident set size (KB) = 1457892 + 0: The total amount of wall time = 562.251920 + 0: The maximum resident set size (KB) = 1287268 Test 183 regional_atmaq_faster_intel PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/control_c48_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/control_c48_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//control_c48_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/control_c48_gnu Checking test 184 control_c48_gnu results .... Comparing sfcf000.nc .........OK Comparing sfcf024.nc .........OK @@ -5929,14 +5929,14 @@ Checking test 184 control_c48_gnu results .... Comparing RESTART/20210323.060000.sfc_data.tile5.nc .........OK Comparing RESTART/20210323.060000.sfc_data.tile6.nc .........OK -0: The total amount of wall time = 685.425924 -0: The maximum resident set size (KB) = 702812 +0: The total amount of wall time = 682.465801 +0: The maximum resident set size (KB) = 704828 Test 184 control_c48_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/control_stochy_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/control_stochy_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//control_stochy_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/control_stochy_gnu Checking test 185 control_stochy_gnu results .... Comparing sfcf000.nc .........OK Comparing sfcf012.nc .........OK @@ -5947,14 +5947,14 @@ Checking test 185 control_stochy_gnu results .... Comparing GFSPRS.GrbF00 .........OK Comparing GFSPRS.GrbF12 .........OK - 0: The total amount of wall time = 658.739038 - 0: The maximum resident set size (KB) = 478832 + 0: The total amount of wall time = 639.221462 + 0: The maximum resident set size (KB) = 479772 Test 185 control_stochy_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/control_ras_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/control_ras_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//control_ras_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/control_ras_gnu Checking test 186 control_ras_gnu results .... Comparing sfcf000.nc .........OK Comparing sfcf024.nc .........OK @@ -5965,14 +5965,14 @@ Checking test 186 control_ras_gnu results .... Comparing GFSPRS.GrbF00 .........OK Comparing GFSPRS.GrbF24 .........OK - 0: The total amount of wall time = 848.168845 - 0: The maximum resident set size (KB) = 488504 + 0: The total amount of wall time = 815.364544 + 0: The maximum resident set size (KB) = 486356 Test 186 control_ras_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/control_p8_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/control_p8_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//control_p8_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/control_p8_gnu Checking test 187 control_p8_gnu results .... Comparing sfcf000.nc .........OK Comparing sfcf021.nc .........OK @@ -6019,14 +6019,14 @@ Checking test 187 control_p8_gnu results .... Comparing RESTART/20210323.060000.sfc_data.tile5.nc .........OK Comparing RESTART/20210323.060000.sfc_data.tile6.nc .........OK - 0: The total amount of wall time = 898.482834 - 0: The maximum resident set size (KB) = 1233708 + 0: The total amount of wall time = 912.605062 + 0: The maximum resident set size (KB) = 1235696 Test 187 control_p8_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/control_flake_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/control_flake_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//control_flake_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/control_flake_gnu Checking test 188 control_flake_gnu results .... Comparing sfcf000.nc .........OK Comparing sfcf024.nc .........OK @@ -6037,14 +6037,14 @@ Checking test 188 control_flake_gnu results .... Comparing GFSPRS.GrbF00 .........OK Comparing GFSPRS.GrbF24 .........OK - 0: The total amount of wall time = 1559.586260 - 0: The maximum resident set size (KB) = 522804 + 0: The total amount of wall time = 1561.172048 + 0: The maximum resident set size (KB) = 525168 Test 188 control_flake_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rap_control_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rap_control_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rap_control_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rap_control_gnu Checking test 189 rap_control_gnu results .... Comparing sfcf000.nc .........OK Comparing sfcf021.nc .........OK @@ -6091,14 +6091,14 @@ Checking test 189 rap_control_gnu results .... Comparing RESTART/20210323.060000.sfc_data.tile5.nc .........OK Comparing RESTART/20210323.060000.sfc_data.tile6.nc .........OK - 0: The total amount of wall time = 1449.540279 - 0: The maximum resident set size (KB) = 828968 + 0: The total amount of wall time = 1457.061463 + 0: The maximum resident set size (KB) = 830400 Test 189 rap_control_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rap_control_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rap_decomp_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rap_control_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rap_decomp_gnu Checking test 190 rap_decomp_gnu results .... Comparing sfcf000.nc .........OK Comparing sfcf021.nc .........OK @@ -6145,14 +6145,14 @@ Checking test 190 rap_decomp_gnu results .... Comparing RESTART/20210323.060000.sfc_data.tile5.nc .........OK Comparing RESTART/20210323.060000.sfc_data.tile6.nc .........OK - 0: The total amount of wall time = 1457.742332 - 0: The maximum resident set size (KB) = 831752 + 0: The total amount of wall time = 1413.153235 + 0: The maximum resident set size (KB) = 826688 Test 190 rap_decomp_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rap_control_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rap_2threads_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rap_control_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rap_2threads_gnu Checking test 191 rap_2threads_gnu results .... Comparing sfcf000.nc .........OK Comparing sfcf021.nc .........OK @@ -6199,14 +6199,14 @@ Checking test 191 rap_2threads_gnu results .... Comparing RESTART/20210323.060000.sfc_data.tile5.nc .........OK Comparing RESTART/20210323.060000.sfc_data.tile6.nc .........OK - 0: The total amount of wall time = 1344.019606 - 0: The maximum resident set size (KB) = 897196 + 0: The total amount of wall time = 1338.850277 + 0: The maximum resident set size (KB) = 893848 Test 191 rap_2threads_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rap_control_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rap_restart_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rap_control_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rap_restart_gnu Checking test 192 rap_restart_gnu results .... Comparing sfcf024.nc .........OK Comparing atmf024.nc .........OK @@ -6245,14 +6245,14 @@ Checking test 192 rap_restart_gnu results .... Comparing RESTART/20210323.060000.sfc_data.tile5.nc .........OK Comparing RESTART/20210323.060000.sfc_data.tile6.nc .........OK - 0: The total amount of wall time = 736.339088 - 0: The maximum resident set size (KB) = 546956 + 0: The total amount of wall time = 729.833554 + 0: The maximum resident set size (KB) = 547756 Test 192 rap_restart_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rap_sfcdiff_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rap_sfcdiff_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rap_sfcdiff_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rap_sfcdiff_gnu Checking test 193 rap_sfcdiff_gnu results .... Comparing sfcf000.nc .........OK Comparing sfcf009.nc .........OK @@ -6299,14 +6299,14 @@ Checking test 193 rap_sfcdiff_gnu results .... Comparing RESTART/20210323.060000.sfc_data.tile5.nc .........OK Comparing RESTART/20210323.060000.sfc_data.tile6.nc .........OK - 0: The total amount of wall time = 1478.461888 - 0: The maximum resident set size (KB) = 824704 + 0: The total amount of wall time = 1466.920722 + 0: The maximum resident set size (KB) = 825780 Test 193 rap_sfcdiff_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rap_sfcdiff_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rap_sfcdiff_decomp_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rap_sfcdiff_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rap_sfcdiff_decomp_gnu Checking test 194 rap_sfcdiff_decomp_gnu results .... Comparing sfcf000.nc .........OK Comparing sfcf009.nc .........OK @@ -6353,14 +6353,14 @@ Checking test 194 rap_sfcdiff_decomp_gnu results .... Comparing RESTART/20210323.060000.sfc_data.tile5.nc .........OK Comparing RESTART/20210323.060000.sfc_data.tile6.nc .........OK - 0: The total amount of wall time = 1449.755783 - 0: The maximum resident set size (KB) = 826952 + 0: The total amount of wall time = 1450.701008 + 0: The maximum resident set size (KB) = 829204 Test 194 rap_sfcdiff_decomp_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rap_sfcdiff_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rap_sfcdiff_restart_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rap_sfcdiff_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rap_sfcdiff_restart_gnu Checking test 195 rap_sfcdiff_restart_gnu results .... Comparing sfcf012.nc .........OK Comparing atmf012.nc .........OK @@ -6399,14 +6399,14 @@ Checking test 195 rap_sfcdiff_restart_gnu results .... Comparing RESTART/20210323.060000.sfc_data.tile5.nc .........OK Comparing RESTART/20210323.060000.sfc_data.tile6.nc .........OK - 0: The total amount of wall time = 1081.008163 - 0: The maximum resident set size (KB) = 549672 + 0: The total amount of wall time = 1064.518160 + 0: The maximum resident set size (KB) = 548820 Test 195 rap_sfcdiff_restart_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/hrrr_control_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/hrrr_control_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//hrrr_control_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/hrrr_control_gnu Checking test 196 hrrr_control_gnu results .... Comparing sfcf000.nc .........OK Comparing sfcf009.nc .........OK @@ -6453,14 +6453,14 @@ Checking test 196 hrrr_control_gnu results .... Comparing RESTART/20210322.120000.sfc_data.tile5.nc .........OK Comparing RESTART/20210322.120000.sfc_data.tile6.nc .........OK - 0: The total amount of wall time = 1437.771836 - 0: The maximum resident set size (KB) = 823280 + 0: The total amount of wall time = 1474.445232 + 0: The maximum resident set size (KB) = 826272 Test 196 hrrr_control_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/hrrr_control_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/hrrr_control_qr_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//hrrr_control_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/hrrr_control_qr_gnu Checking test 197 hrrr_control_qr_gnu results .... Comparing sfcf000.nc .........OK Comparing sfcf009.nc .........OK @@ -6507,14 +6507,14 @@ Checking test 197 hrrr_control_qr_gnu results .... Comparing RESTART/20210322.120000.sfc_data.tile5.nc ............ALT CHECK......OK Comparing RESTART/20210322.120000.sfc_data.tile6.nc ............ALT CHECK......OK - 0: The total amount of wall time = 1465.397596 - 0: The maximum resident set size (KB) = 836840 + 0: The total amount of wall time = 1463.076746 + 0: The maximum resident set size (KB) = 832552 Test 197 hrrr_control_qr_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/hrrr_control_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/hrrr_control_2threads_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//hrrr_control_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/hrrr_control_2threads_gnu Checking test 198 hrrr_control_2threads_gnu results .... Comparing sfcf000.nc .........OK Comparing sfcf009.nc .........OK @@ -6561,14 +6561,14 @@ Checking test 198 hrrr_control_2threads_gnu results .... Comparing RESTART/20210322.120000.sfc_data.tile5.nc .........OK Comparing RESTART/20210322.120000.sfc_data.tile6.nc .........OK - 0: The total amount of wall time = 1494.112148 - 0: The maximum resident set size (KB) = 892148 + 0: The total amount of wall time = 1558.181677 + 0: The maximum resident set size (KB) = 890200 Test 198 hrrr_control_2threads_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/hrrr_control_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/hrrr_control_decomp_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//hrrr_control_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/hrrr_control_decomp_gnu Checking test 199 hrrr_control_decomp_gnu results .... Comparing sfcf000.nc .........OK Comparing sfcf009.nc .........OK @@ -6615,42 +6615,42 @@ Checking test 199 hrrr_control_decomp_gnu results .... Comparing RESTART/20210322.120000.sfc_data.tile5.nc .........OK Comparing RESTART/20210322.120000.sfc_data.tile6.nc .........OK - 0: The total amount of wall time = 1450.143002 - 0: The maximum resident set size (KB) = 822172 + 0: The total amount of wall time = 1458.758520 + 0: The maximum resident set size (KB) = 821176 Test 199 hrrr_control_decomp_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/hrrr_control_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/hrrr_control_restart_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//hrrr_control_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/hrrr_control_restart_gnu Checking test 200 hrrr_control_restart_gnu results .... Comparing sfcf012.nc .........OK Comparing atmf012.nc .........OK Comparing GFSFLX.GrbF12 .........OK Comparing GFSPRS.GrbF12 .........OK - 0: The total amount of wall time = 1063.309605 - 0: The maximum resident set size (KB) = 545412 + 0: The total amount of wall time = 1040.936438 + 0: The maximum resident set size (KB) = 545076 Test 200 hrrr_control_restart_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/hrrr_control_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/hrrr_control_restart_qr_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//hrrr_control_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/hrrr_control_restart_qr_gnu Checking test 201 hrrr_control_restart_qr_gnu results .... Comparing sfcf012.nc .........OK Comparing atmf012.nc .........OK Comparing GFSFLX.GrbF12 .........OK Comparing GFSPRS.GrbF12 .........OK - 0: The total amount of wall time = 1089.803313 - 0: The maximum resident set size (KB) = 562724 + 0: The total amount of wall time = 1049.534945 + 0: The maximum resident set size (KB) = 560064 Test 201 hrrr_control_restart_qr_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rrfs_v1beta_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rrfs_v1beta_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rrfs_v1beta_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rrfs_v1beta_gnu Checking test 202 rrfs_v1beta_gnu results .... Comparing sfcf000.nc .........OK Comparing sfcf009.nc .........OK @@ -6697,14 +6697,14 @@ Checking test 202 rrfs_v1beta_gnu results .... Comparing RESTART/20210323.060000.sfc_data.tile5.nc .........OK Comparing RESTART/20210323.060000.sfc_data.tile6.nc .........OK - 0: The total amount of wall time = 1466.936692 - 0: The maximum resident set size (KB) = 822672 + 0: The total amount of wall time = 1439.184734 + 0: The maximum resident set size (KB) = 825648 Test 202 rrfs_v1beta_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rrfs_smoke_conus13km_hrrr_warm_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rrfs_smoke_conus13km_hrrr_warm_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rrfs_smoke_conus13km_hrrr_warm_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rrfs_smoke_conus13km_hrrr_warm_gnu Checking test 203 rrfs_smoke_conus13km_hrrr_warm_gnu results .... Comparing sfcf000.nc .........OK Comparing sfcf001.nc .........OK @@ -6720,14 +6720,14 @@ Checking test 203 rrfs_smoke_conus13km_hrrr_warm_gnu results .... Comparing RESTART/20210512.170000.phy_data.nc .........OK Comparing RESTART/20210512.170000.sfc_data.nc .........OK - 0: The total amount of wall time = 791.777019 - 0: The maximum resident set size (KB) = 664524 + 0: The total amount of wall time = 806.553809 + 0: The maximum resident set size (KB) = 664384 Test 203 rrfs_smoke_conus13km_hrrr_warm_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rrfs_smoke_conus13km_hrrr_warm_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rrfs_smoke_conus13km_hrrr_warm_qr_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rrfs_smoke_conus13km_hrrr_warm_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rrfs_smoke_conus13km_hrrr_warm_qr_gnu Checking test 204 rrfs_smoke_conus13km_hrrr_warm_qr_gnu results .... Comparing sfcf000.nc .........OK Comparing sfcf001.nc .........OK @@ -6743,14 +6743,14 @@ Checking test 204 rrfs_smoke_conus13km_hrrr_warm_qr_gnu results .... Comparing RESTART/20210512.170000.phy_data.nc ............ALT CHECK......OK Comparing RESTART/20210512.170000.sfc_data.nc ............ALT CHECK......OK - 0: The total amount of wall time = 867.671090 - 0: The maximum resident set size (KB) = 617544 + 0: The total amount of wall time = 857.405122 + 0: The maximum resident set size (KB) = 622176 Test 204 rrfs_smoke_conus13km_hrrr_warm_qr_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rrfs_smoke_conus13km_hrrr_warm_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rrfs_smoke_conus13km_hrrr_warm_2threads_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rrfs_smoke_conus13km_hrrr_warm_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rrfs_smoke_conus13km_hrrr_warm_2threads_gnu Checking test 205 rrfs_smoke_conus13km_hrrr_warm_2threads_gnu results .... Comparing sfcf000.nc .........OK Comparing sfcf001.nc .........OK @@ -6759,14 +6759,14 @@ Checking test 205 rrfs_smoke_conus13km_hrrr_warm_2threads_gnu results .... Comparing atmf001.nc .........OK Comparing atmf002.nc .........OK - 0: The total amount of wall time = 1044.696611 - 0: The maximum resident set size (KB) = 660988 + 0: The total amount of wall time = 1078.678562 + 0: The maximum resident set size (KB) = 659016 Test 205 rrfs_smoke_conus13km_hrrr_warm_2threads_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rrfs_smoke_conus13km_hrrr_warm_radar_tten_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rrfs_smoke_conus13km_radar_tten_warm_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rrfs_smoke_conus13km_hrrr_warm_radar_tten_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rrfs_smoke_conus13km_radar_tten_warm_gnu Checking test 206 rrfs_smoke_conus13km_radar_tten_warm_gnu results .... Comparing sfcf000.nc .........OK Comparing sfcf001.nc .........OK @@ -6775,14 +6775,14 @@ Checking test 206 rrfs_smoke_conus13km_radar_tten_warm_gnu results .... Comparing atmf001.nc .........OK Comparing atmf002.nc .........OK - 0: The total amount of wall time = 820.613248 - 0: The maximum resident set size (KB) = 665820 + 0: The total amount of wall time = 852.051749 + 0: The maximum resident set size (KB) = 663144 Test 206 rrfs_smoke_conus13km_radar_tten_warm_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rrfs_conus13km_hrrr_warm_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rrfs_conus13km_hrrr_warm_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rrfs_conus13km_hrrr_warm_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rrfs_conus13km_hrrr_warm_gnu Checking test 207 rrfs_conus13km_hrrr_warm_gnu results .... Comparing sfcf000.nc .........OK Comparing sfcf001.nc .........OK @@ -6798,274 +6798,274 @@ Checking test 207 rrfs_conus13km_hrrr_warm_gnu results .... Comparing RESTART/20210512.170000.phy_data.nc .........OK Comparing RESTART/20210512.170000.sfc_data.nc .........OK - 0: The total amount of wall time = 783.615012 - 0: The maximum resident set size (KB) = 642892 + 0: The total amount of wall time = 737.730211 + 0: The maximum resident set size (KB) = 642808 Test 207 rrfs_conus13km_hrrr_warm_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rrfs_smoke_conus13km_hrrr_warm_restart_mismatch_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rrfs_smoke_conus13km_hrrr_warm_restart_mismatch_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rrfs_smoke_conus13km_hrrr_warm_restart_mismatch_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rrfs_smoke_conus13km_hrrr_warm_restart_mismatch_gnu Checking test 208 rrfs_smoke_conus13km_hrrr_warm_restart_mismatch_gnu results .... Comparing sfcf002.nc .........OK Comparing atmf002.nc .........OK - 0: The total amount of wall time = 426.409910 - 0: The maximum resident set size (KB) = 654880 + 0: The total amount of wall time = 425.979169 + 0: The maximum resident set size (KB) = 650984 Test 208 rrfs_smoke_conus13km_hrrr_warm_restart_mismatch_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rrfs_smoke_conus13km_hrrr_warm_restart_mismatch_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rrfs_smoke_conus13km_hrrr_warm_restart_qr_mismatch_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rrfs_smoke_conus13km_hrrr_warm_restart_mismatch_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rrfs_smoke_conus13km_hrrr_warm_restart_qr_mismatch_gnu Checking test 209 rrfs_smoke_conus13km_hrrr_warm_restart_qr_mismatch_gnu results .... Comparing sfcf002.nc .........OK Comparing atmf002.nc .........OK - 0: The total amount of wall time = 494.185946 - 0: The maximum resident set size (KB) = 677040 + 0: The total amount of wall time = 457.419556 + 0: The maximum resident set size (KB) = 672996 Test 209 rrfs_smoke_conus13km_hrrr_warm_restart_qr_mismatch_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/control_diag_debug_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/control_diag_debug_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//control_diag_debug_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/control_diag_debug_gnu Checking test 210 control_diag_debug_gnu results .... Comparing sfcf000.nc .........OK Comparing sfcf001.nc .........OK Comparing atmf000.nc .........OK Comparing atmf001.nc .........OK - 0: The total amount of wall time = 123.614593 - 0: The maximum resident set size (KB) = 530600 + 0: The total amount of wall time = 121.807788 + 0: The maximum resident set size (KB) = 527332 Test 210 control_diag_debug_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/regional_debug_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/regional_debug_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//regional_debug_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/regional_debug_gnu Checking test 211 regional_debug_gnu results .... Comparing dynf000.nc .........OK Comparing dynf001.nc .........OK Comparing phyf000.nc .........OK Comparing phyf001.nc .........OK - 0: The total amount of wall time = 660.270454 - 0: The maximum resident set size (KB) = 593852 + 0: The total amount of wall time = 622.992239 + 0: The maximum resident set size (KB) = 594556 Test 211 regional_debug_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rap_control_debug_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rap_control_debug_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rap_control_debug_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rap_control_debug_gnu Checking test 212 rap_control_debug_gnu results .... Comparing sfcf000.nc .........OK Comparing sfcf001.nc .........OK Comparing atmf000.nc .........OK Comparing atmf001.nc .........OK - 0: The total amount of wall time = 176.347211 - 0: The maximum resident set size (KB) = 838600 + 0: The total amount of wall time = 171.184227 + 0: The maximum resident set size (KB) = 837096 Test 212 rap_control_debug_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/hrrr_control_debug_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/hrrr_control_debug_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//hrrr_control_debug_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/hrrr_control_debug_gnu Checking test 213 hrrr_control_debug_gnu results .... Comparing sfcf000.nc .........OK Comparing sfcf001.nc .........OK Comparing atmf000.nc .........OK Comparing atmf001.nc .........OK - 0: The total amount of wall time = 170.229418 - 0: The maximum resident set size (KB) = 835336 + 0: The total amount of wall time = 163.537092 + 0: The maximum resident set size (KB) = 835464 Test 213 hrrr_control_debug_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rap_diag_debug_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rap_diag_debug_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rap_diag_debug_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rap_diag_debug_gnu Checking test 214 rap_diag_debug_gnu results .... Comparing sfcf000.nc .........OK Comparing sfcf001.nc .........OK Comparing atmf000.nc .........OK Comparing atmf001.nc .........OK - 0: The total amount of wall time = 211.025738 - 0: The maximum resident set size (KB) = 923240 + 0: The total amount of wall time = 211.609335 + 0: The maximum resident set size (KB) = 921656 Test 214 rap_diag_debug_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rap_noah_sfcdiff_cires_ugwp_debug_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rap_noah_sfcdiff_cires_ugwp_debug_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rap_noah_sfcdiff_cires_ugwp_debug_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rap_noah_sfcdiff_cires_ugwp_debug_gnu Checking test 215 rap_noah_sfcdiff_cires_ugwp_debug_gnu results .... Comparing sfcf000.nc .........OK Comparing sfcf001.nc .........OK Comparing atmf000.nc .........OK Comparing atmf001.nc .........OK - 0: The total amount of wall time = 272.116833 - 0: The maximum resident set size (KB) = 834996 + 0: The total amount of wall time = 270.876031 + 0: The maximum resident set size (KB) = 834540 Test 215 rap_noah_sfcdiff_cires_ugwp_debug_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rap_progcld_thompson_debug_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rap_progcld_thompson_debug_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rap_progcld_thompson_debug_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rap_progcld_thompson_debug_gnu Checking test 216 rap_progcld_thompson_debug_gnu results .... Comparing sfcf000.nc .........OK Comparing sfcf001.nc .........OK Comparing atmf000.nc .........OK Comparing atmf001.nc .........OK - 0: The total amount of wall time = 173.620279 - 0: The maximum resident set size (KB) = 836136 + 0: The total amount of wall time = 176.956774 + 0: The maximum resident set size (KB) = 842068 Test 216 rap_progcld_thompson_debug_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rrfs_v1beta_debug_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rrfs_v1beta_debug_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rrfs_v1beta_debug_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rrfs_v1beta_debug_gnu Checking test 217 rrfs_v1beta_debug_gnu results .... Comparing sfcf000.nc .........OK Comparing sfcf001.nc .........OK Comparing atmf000.nc .........OK Comparing atmf001.nc .........OK - 0: The total amount of wall time = 170.297293 - 0: The maximum resident set size (KB) = 834184 + 0: The total amount of wall time = 170.163937 + 0: The maximum resident set size (KB) = 838088 Test 217 rrfs_v1beta_debug_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/control_ras_debug_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/control_ras_debug_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//control_ras_debug_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/control_ras_debug_gnu Checking test 218 control_ras_debug_gnu results .... Comparing sfcf000.nc .........OK Comparing sfcf001.nc .........OK Comparing atmf000.nc .........OK Comparing atmf001.nc .........OK - 0: The total amount of wall time = 103.021665 - 0: The maximum resident set size (KB) = 484024 + 0: The total amount of wall time = 98.760274 + 0: The maximum resident set size (KB) = 481140 Test 218 control_ras_debug_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/control_stochy_debug_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/control_stochy_debug_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//control_stochy_debug_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/control_stochy_debug_gnu Checking test 219 control_stochy_debug_gnu results .... Comparing sfcf000.nc .........OK Comparing sfcf001.nc .........OK Comparing atmf000.nc .........OK Comparing atmf001.nc .........OK - 0: The total amount of wall time = 116.910805 - 0: The maximum resident set size (KB) = 475844 + 0: The total amount of wall time = 116.208070 + 0: The maximum resident set size (KB) = 473576 Test 219 control_stochy_debug_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/control_debug_p8_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/control_debug_p8_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//control_debug_p8_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/control_debug_p8_gnu Checking test 220 control_debug_p8_gnu results .... Comparing sfcf000.nc .........OK Comparing sfcf001.nc .........OK Comparing atmf000.nc .........OK Comparing atmf001.nc .........OK - 0: The total amount of wall time = 117.222676 - 0: The maximum resident set size (KB) = 1225364 + 0: The total amount of wall time = 115.428684 + 0: The maximum resident set size (KB) = 1226352 Test 220 control_debug_p8_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rrfs_smoke_conus13km_hrrr_warm_debug_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rrfs_smoke_conus13km_hrrr_warm_debug_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rrfs_smoke_conus13km_hrrr_warm_debug_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rrfs_smoke_conus13km_hrrr_warm_debug_gnu Checking test 221 rrfs_smoke_conus13km_hrrr_warm_debug_gnu results .... Comparing sfcf000.nc .........OK Comparing sfcf001.nc .........OK Comparing atmf000.nc .........OK Comparing atmf001.nc .........OK - 0: The total amount of wall time = 649.434207 - 0: The maximum resident set size (KB) = 679732 + 0: The total amount of wall time = 601.648553 + 0: The maximum resident set size (KB) = 681112 Test 221 rrfs_smoke_conus13km_hrrr_warm_debug_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rrfs_smoke_conus13km_hrrr_warm_debug_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rrfs_smoke_conus13km_hrrr_warm_debug_2threads_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rrfs_smoke_conus13km_hrrr_warm_debug_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rrfs_smoke_conus13km_hrrr_warm_debug_2threads_gnu Checking test 222 rrfs_smoke_conus13km_hrrr_warm_debug_2threads_gnu results .... Comparing sfcf000.nc .........OK Comparing sfcf001.nc .........OK Comparing atmf000.nc .........OK Comparing atmf001.nc .........OK - 0: The total amount of wall time = 775.518645 - 0: The maximum resident set size (KB) = 678176 + 0: The total amount of wall time = 788.010143 + 0: The maximum resident set size (KB) = 675980 Test 222 rrfs_smoke_conus13km_hrrr_warm_debug_2threads_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rrfs_conus13km_hrrr_warm_debugs_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rrfs_conus13km_hrrr_warm_debug_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rrfs_conus13km_hrrr_warm_debugs_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rrfs_conus13km_hrrr_warm_debug_gnu Checking test 223 rrfs_conus13km_hrrr_warm_debug_gnu results .... Comparing sfcf000.nc .........OK Comparing sfcf001.nc .........OK Comparing atmf000.nc .........OK Comparing atmf001.nc .........OK - 0: The total amount of wall time = 584.301495 - 0: The maximum resident set size (KB) = 657460 + 0: The total amount of wall time = 565.580641 + 0: The maximum resident set size (KB) = 658176 Test 223 rrfs_conus13km_hrrr_warm_debug_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rap_flake_debug_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rap_flake_debug_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rap_flake_debug_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rap_flake_debug_gnu Checking test 224 rap_flake_debug_gnu results .... Comparing sfcf000.nc .........OK Comparing sfcf001.nc .........OK Comparing atmf000.nc .........OK Comparing atmf001.nc .........OK - 0: The total amount of wall time = 175.462925 - 0: The maximum resident set size (KB) = 836872 + 0: The total amount of wall time = 174.111532 + 0: The maximum resident set size (KB) = 839584 Test 224 rap_flake_debug_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rap_clm_lake_debug_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rap_clm_lake_debug_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rap_clm_lake_debug_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rap_clm_lake_debug_gnu Checking test 225 rap_clm_lake_debug_gnu results .... Comparing sfcf000.nc .........OK Comparing sfcf001.nc .........OK Comparing atmf000.nc .........OK Comparing atmf001.nc .........OK - 0: The total amount of wall time = 193.375624 - 0: The maximum resident set size (KB) = 838884 + 0: The total amount of wall time = 199.791491 + 0: The maximum resident set size (KB) = 840600 Test 225 rap_clm_lake_debug_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/control_wam_debug_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/control_wam_debug_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//control_wam_debug_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/control_wam_debug_gnu Checking test 226 control_wam_debug_gnu results .... Comparing sfcf019.nc .........OK Comparing atmf019.nc .........OK - 0: The total amount of wall time = 186.550126 - 0: The maximum resident set size (KB) = 190920 + 0: The total amount of wall time = 180.791758 + 0: The maximum resident set size (KB) = 194344 Test 226 control_wam_debug_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rap_control_dyn32_phy32_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rap_control_dyn32_phy32_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rap_control_dyn32_phy32_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rap_control_dyn32_phy32_gnu Checking test 227 rap_control_dyn32_phy32_gnu results .... Comparing sfcf000.nc .........OK Comparing sfcf009.nc .........OK @@ -7112,14 +7112,14 @@ Checking test 227 rap_control_dyn32_phy32_gnu results .... Comparing RESTART/20210323.060000.sfc_data.tile5.nc .........OK Comparing RESTART/20210323.060000.sfc_data.tile6.nc .........OK - 0: The total amount of wall time = 1496.063076 - 0: The maximum resident set size (KB) = 685804 + 0: The total amount of wall time = 1486.482859 + 0: The maximum resident set size (KB) = 680388 Test 227 rap_control_dyn32_phy32_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/hrrr_control_dyn32_phy32_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/hrrr_control_dyn32_phy32_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//hrrr_control_dyn32_phy32_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/hrrr_control_dyn32_phy32_gnu Checking test 228 hrrr_control_dyn32_phy32_gnu results .... Comparing sfcf000.nc .........OK Comparing sfcf009.nc .........OK @@ -7166,14 +7166,14 @@ Checking test 228 hrrr_control_dyn32_phy32_gnu results .... Comparing RESTART/20210322.120000.sfc_data.tile5.nc .........OK Comparing RESTART/20210322.120000.sfc_data.tile6.nc .........OK - 0: The total amount of wall time = 736.503118 - 0: The maximum resident set size (KB) = 680892 + 0: The total amount of wall time = 759.086976 + 0: The maximum resident set size (KB) = 682704 Test 228 hrrr_control_dyn32_phy32_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/hrrr_control_qr_dyn32_phy32_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/hrrr_control_qr_dyn32_phy32_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//hrrr_control_qr_dyn32_phy32_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/hrrr_control_qr_dyn32_phy32_gnu Checking test 229 hrrr_control_qr_dyn32_phy32_gnu results .... Comparing sfcf000.nc .........OK Comparing sfcf009.nc .........OK @@ -7220,14 +7220,14 @@ Checking test 229 hrrr_control_qr_dyn32_phy32_gnu results .... Comparing RESTART/20210322.120000.sfc_data.tile5.nc .........OK Comparing RESTART/20210322.120000.sfc_data.tile6.nc .........OK - 0: The total amount of wall time = 742.379529 - 0: The maximum resident set size (KB) = 696816 + 0: The total amount of wall time = 759.456708 + 0: The maximum resident set size (KB) = 687008 Test 229 hrrr_control_qr_dyn32_phy32_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rap_control_dyn32_phy32_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rap_2threads_dyn32_phy32_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rap_control_dyn32_phy32_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rap_2threads_dyn32_phy32_gnu Checking test 230 rap_2threads_dyn32_phy32_gnu results .... Comparing sfcf000.nc .........OK Comparing sfcf009.nc .........OK @@ -7274,14 +7274,14 @@ Checking test 230 rap_2threads_dyn32_phy32_gnu results .... Comparing RESTART/20210323.060000.sfc_data.tile5.nc .........OK Comparing RESTART/20210323.060000.sfc_data.tile6.nc .........OK - 0: The total amount of wall time = 1363.337234 - 0: The maximum resident set size (KB) = 733652 + 0: The total amount of wall time = 1366.680896 + 0: The maximum resident set size (KB) = 727056 Test 230 rap_2threads_dyn32_phy32_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/hrrr_control_dyn32_phy32_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/hrrr_control_2threads_dyn32_phy32_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//hrrr_control_dyn32_phy32_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/hrrr_control_2threads_dyn32_phy32_gnu Checking test 231 hrrr_control_2threads_dyn32_phy32_gnu results .... Comparing sfcf000.nc .........OK Comparing sfcf009.nc .........OK @@ -7328,14 +7328,14 @@ Checking test 231 hrrr_control_2threads_dyn32_phy32_gnu results .... Comparing RESTART/20210322.120000.sfc_data.tile5.nc .........OK Comparing RESTART/20210322.120000.sfc_data.tile6.nc .........OK - 0: The total amount of wall time = 738.783463 - 0: The maximum resident set size (KB) = 729516 + 0: The total amount of wall time = 766.428943 + 0: The maximum resident set size (KB) = 722928 Test 231 hrrr_control_2threads_dyn32_phy32_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/hrrr_control_dyn32_phy32_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/hrrr_control_decomp_dyn32_phy32_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//hrrr_control_dyn32_phy32_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/hrrr_control_decomp_dyn32_phy32_gnu Checking test 232 hrrr_control_decomp_dyn32_phy32_gnu results .... Comparing sfcf000.nc .........OK Comparing sfcf009.nc .........OK @@ -7382,14 +7382,14 @@ Checking test 232 hrrr_control_decomp_dyn32_phy32_gnu results .... Comparing RESTART/20210322.120000.sfc_data.tile5.nc .........OK Comparing RESTART/20210322.120000.sfc_data.tile6.nc .........OK - 0: The total amount of wall time = 738.311974 - 0: The maximum resident set size (KB) = 681772 + 0: The total amount of wall time = 733.661927 + 0: The maximum resident set size (KB) = 678668 Test 232 hrrr_control_decomp_dyn32_phy32_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rap_control_dyn32_phy32_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rap_restart_dyn32_phy32_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rap_control_dyn32_phy32_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rap_restart_dyn32_phy32_gnu Checking test 233 rap_restart_dyn32_phy32_gnu results .... Comparing sfcf012.nc .........OK Comparing atmf012.nc .........OK @@ -7428,42 +7428,42 @@ Checking test 233 rap_restart_dyn32_phy32_gnu results .... Comparing RESTART/20210323.060000.sfc_data.tile5.nc .........OK Comparing RESTART/20210323.060000.sfc_data.tile6.nc .........OK - 0: The total amount of wall time = 1065.270220 - 0: The maximum resident set size (KB) = 512304 + 0: The total amount of wall time = 1070.659496 + 0: The maximum resident set size (KB) = 513852 Test 233 rap_restart_dyn32_phy32_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/hrrr_control_dyn32_phy32_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/hrrr_control_restart_dyn32_phy32_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//hrrr_control_dyn32_phy32_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/hrrr_control_restart_dyn32_phy32_gnu Checking test 234 hrrr_control_restart_dyn32_phy32_gnu results .... Comparing sfcf012.nc .........OK Comparing atmf012.nc .........OK Comparing GFSFLX.GrbF12 .........OK Comparing GFSPRS.GrbF12 .........OK - 0: The total amount of wall time = 367.066223 - 0: The maximum resident set size (KB) = 505580 + 0: The total amount of wall time = 366.099845 + 0: The maximum resident set size (KB) = 506268 Test 234 hrrr_control_restart_dyn32_phy32_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/hrrr_control_qr_dyn32_phy32_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/hrrr_control_restart_qr_dyn32_phy32_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//hrrr_control_qr_dyn32_phy32_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/hrrr_control_restart_qr_dyn32_phy32_gnu Checking test 235 hrrr_control_restart_qr_dyn32_phy32_gnu results .... Comparing sfcf012.nc .........OK Comparing atmf012.nc .........OK Comparing GFSFLX.GrbF12 .........OK Comparing GFSPRS.GrbF12 .........OK - 0: The total amount of wall time = 387.056569 - 0: The maximum resident set size (KB) = 524764 + 0: The total amount of wall time = 374.800054 + 0: The maximum resident set size (KB) = 522256 Test 235 hrrr_control_restart_qr_dyn32_phy32_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rrfs_smoke_conus13km_phy32_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rrfs_smoke_conus13km_phy32_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rrfs_smoke_conus13km_phy32_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rrfs_smoke_conus13km_phy32_gnu Checking test 236 rrfs_smoke_conus13km_phy32_gnu results .... Comparing sfcf000.nc .........OK Comparing sfcf001.nc .........OK @@ -7479,14 +7479,14 @@ Checking test 236 rrfs_smoke_conus13km_phy32_gnu results .... Comparing RESTART/20210512.170000.phy_data.nc .........OK Comparing RESTART/20210512.170000.sfc_data.nc .........OK - 0: The total amount of wall time = 777.192740 - 0: The maximum resident set size (KB) = 584464 + 0: The total amount of wall time = 773.114968 + 0: The maximum resident set size (KB) = 580692 Test 236 rrfs_smoke_conus13km_phy32_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rrfs_smoke_conus13km_phy32_qr_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rrfs_smoke_conus13km_phy32_qr_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rrfs_smoke_conus13km_phy32_qr_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rrfs_smoke_conus13km_phy32_qr_gnu Checking test 237 rrfs_smoke_conus13km_phy32_qr_gnu results .... Comparing sfcf000.nc .........OK Comparing sfcf001.nc .........OK @@ -7502,38 +7502,38 @@ Checking test 237 rrfs_smoke_conus13km_phy32_qr_gnu results .... Comparing RESTART/20210512.170000.phy_data.nc .........OK Comparing RESTART/20210512.170000.sfc_data.nc .........OK - 0: The total amount of wall time = 798.525337 - 0: The maximum resident set size (KB) = 577864 + 0: The total amount of wall time = 812.211076 + 0: The maximum resident set size (KB) = 567376 Test 237 rrfs_smoke_conus13km_phy32_qr_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rrfs_smoke_conus13km_phy32_restart_mismatch_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rrfs_smoke_conus13km_phy32_restart_mismatch_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rrfs_smoke_conus13km_phy32_restart_mismatch_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rrfs_smoke_conus13km_phy32_restart_mismatch_gnu Checking test 238 rrfs_smoke_conus13km_phy32_restart_mismatch_gnu results .... Comparing sfcf002.nc .........OK Comparing atmf002.nc .........OK - 0: The total amount of wall time = 404.688091 - 0: The maximum resident set size (KB) = 573364 + 0: The total amount of wall time = 392.964131 + 0: The maximum resident set size (KB) = 573204 Test 238 rrfs_smoke_conus13km_phy32_restart_mismatch_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rrfs_smoke_conus13km_phy32_restart_mismatch_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rrfs_smoke_conus13km_phy32_restart_qr_mismatch_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rrfs_smoke_conus13km_phy32_restart_mismatch_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rrfs_smoke_conus13km_phy32_restart_qr_mismatch_gnu Checking test 239 rrfs_smoke_conus13km_phy32_restart_qr_mismatch_gnu results .... Comparing sfcf002.nc .........OK Comparing atmf002.nc .........OK - 0: The total amount of wall time = 401.504896 - 0: The maximum resident set size (KB) = 592540 + 0: The total amount of wall time = 407.783929 + 0: The maximum resident set size (KB) = 593312 Test 239 rrfs_smoke_conus13km_phy32_restart_qr_mismatch_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rap_control_dyn64_phy32_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rap_control_dyn64_phy32_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rap_control_dyn64_phy32_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rap_control_dyn64_phy32_gnu Checking test 240 rap_control_dyn64_phy32_gnu results .... Comparing sfcf000.nc .........OK Comparing sfcf009.nc .........OK @@ -7580,56 +7580,56 @@ Checking test 240 rap_control_dyn64_phy32_gnu results .... Comparing RESTART/20210322.180000.sfc_data.tile5.nc .........OK Comparing RESTART/20210322.180000.sfc_data.tile6.nc .........OK - 0: The total amount of wall time = 1078.067923 - 0: The maximum resident set size (KB) = 704652 + 0: The total amount of wall time = 1066.356488 + 0: The maximum resident set size (KB) = 712756 Test 240 rap_control_dyn64_phy32_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rap_control_debug_dyn32_phy32_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rap_control_debug_dyn32_phy32_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rap_control_debug_dyn32_phy32_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rap_control_debug_dyn32_phy32_gnu Checking test 241 rap_control_debug_dyn32_phy32_gnu results .... Comparing sfcf000.nc .........OK Comparing sfcf001.nc .........OK Comparing atmf000.nc .........OK Comparing atmf001.nc .........OK - 0: The total amount of wall time = 174.124804 - 0: The maximum resident set size (KB) = 700588 + 0: The total amount of wall time = 171.730339 + 0: The maximum resident set size (KB) = 696896 Test 241 rap_control_debug_dyn32_phy32_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/hrrr_control_debug_dyn32_phy32_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/hrrr_control_debug_dyn32_phy32_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//hrrr_control_debug_dyn32_phy32_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/hrrr_control_debug_dyn32_phy32_gnu Checking test 242 hrrr_control_debug_dyn32_phy32_gnu results .... Comparing sfcf000.nc .........OK Comparing sfcf001.nc .........OK Comparing atmf000.nc .........OK Comparing atmf001.nc .........OK - 0: The total amount of wall time = 170.162083 - 0: The maximum resident set size (KB) = 691860 + 0: The total amount of wall time = 167.980914 + 0: The maximum resident set size (KB) = 690208 Test 242 hrrr_control_debug_dyn32_phy32_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/rap_control_debug_dyn64_phy32_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/rap_control_dyn64_phy32_debug_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//rap_control_debug_dyn64_phy32_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/rap_control_dyn64_phy32_debug_gnu Checking test 243 rap_control_dyn64_phy32_debug_gnu results .... Comparing sfcf000.nc .........OK Comparing sfcf001.nc .........OK Comparing atmf000.nc .........OK Comparing atmf001.nc .........OK - 0: The total amount of wall time = 205.113660 - 0: The maximum resident set size (KB) = 714292 + 0: The total amount of wall time = 199.135493 + 0: The maximum resident set size (KB) = 711620 Test 243 rap_control_dyn64_phy32_debug_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/cpld_control_p8_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/cpld_control_p8_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//cpld_control_p8_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/cpld_control_p8_gnu Checking test 244 cpld_control_p8_gnu results .... Comparing sfcf021.tile1.nc .........OK Comparing sfcf021.tile2.nc .........OK @@ -7694,14 +7694,14 @@ Checking test 244 cpld_control_p8_gnu results .... Comparing 20210323.060000.out_pnt.ww3 .........OK Comparing 20210323.060000.out_grd.ww3 .........OK - 0: The total amount of wall time = 1884.008600 - 0: The maximum resident set size (KB) = 1423752 + 0: The total amount of wall time = 1864.994421 + 0: The maximum resident set size (KB) = 1430784 Test 244 cpld_control_p8_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/cpld_control_c96_noaero_p8_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/cpld_control_nowave_noaero_p8_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//cpld_control_c96_noaero_p8_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/cpld_control_nowave_noaero_p8_gnu Checking test 245 cpld_control_nowave_noaero_p8_gnu results .... Comparing sfcf021.tile1.nc .........OK Comparing sfcf021.tile2.nc .........OK @@ -7763,14 +7763,14 @@ Checking test 245 cpld_control_nowave_noaero_p8_gnu results .... Comparing RESTART/iced.2021-03-23-21600.nc .........OK Comparing RESTART/ufs.cpld.cpl.r.2021-03-23-21600.nc .........OK - 0: The total amount of wall time = 1239.247182 - 0: The maximum resident set size (KB) = 1330352 + 0: The total amount of wall time = 1237.587726 + 0: The maximum resident set size (KB) = 1325588 Test 245 cpld_control_nowave_noaero_p8_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/cpld_debug_p8_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/cpld_debug_p8_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//cpld_debug_p8_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/cpld_debug_p8_gnu Checking test 246 cpld_debug_p8_gnu results .... Comparing sfcf003.tile1.nc .........OK Comparing sfcf003.tile2.nc .........OK @@ -7823,14 +7823,14 @@ Checking test 246 cpld_debug_p8_gnu results .... Comparing 20210322.090000.out_pnt.ww3 .........OK Comparing 20210322.090000.out_grd.ww3 .........OK - 0: The total amount of wall time = 949.289910 - 0: The maximum resident set size (KB) = 1431492 + 0: The total amount of wall time = 1441.568197 + 0: The maximum resident set size (KB) = 1430276 Test 246 cpld_debug_p8_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/cpld_control_pdlib_p8_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/cpld_control_pdlib_p8_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//cpld_control_pdlib_p8_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/cpld_control_pdlib_p8_gnu Checking test 247 cpld_control_pdlib_p8_gnu results .... Comparing sfcf021.tile1.nc .........OK Comparing sfcf021.tile2.nc .........OK @@ -7894,14 +7894,14 @@ Checking test 247 cpld_control_pdlib_p8_gnu results .... Comparing 20210323.060000.out_pnt.ww3 .........OK Comparing 20210323.060000.out_grd.ww3 .........OK - 0: The total amount of wall time = 1715.077477 - 0: The maximum resident set size (KB) = 1289752 + 0: The total amount of wall time = 1689.018184 + 0: The maximum resident set size (KB) = 1292876 Test 247 cpld_control_pdlib_p8_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/cpld_debug_pdlib_p8_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/cpld_debug_pdlib_p8_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//cpld_debug_pdlib_p8_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/cpld_debug_pdlib_p8_gnu Checking test 248 cpld_debug_pdlib_p8_gnu results .... Comparing sfcf003.tile1.nc .........OK Comparing sfcf003.tile2.nc .........OK @@ -7953,25 +7953,25 @@ Checking test 248 cpld_debug_pdlib_p8_gnu results .... Comparing 20210322.090000.out_pnt.ww3 .........OK Comparing 20210322.090000.out_grd.ww3 .........OK - 0: The total amount of wall time = 899.861398 - 0: The maximum resident set size (KB) = 1302124 + 0: The total amount of wall time = 913.145853 + 0: The maximum resident set size (KB) = 1297368 Test 248 cpld_debug_pdlib_p8_gnu PASS -baseline dir = /scratch1/BMC/gmtb/RT/NCAR/main-20231004/datm_cdeps_control_cfsr_gnu -working dir = /scratch1/BMC/gmtb/Grant.Firl/stmp2/Grant.Firl/FV3_RT/rt_16483/datm_cdeps_control_cfsr_gnu +baseline dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model/baselines/main-20231004//datm_cdeps_control_cfsr_gnu +working dir = /scratch1/BMC/gmtb/CCPP_regression_testing/NCAR_ufs-weather-model//run//rt_16158/datm_cdeps_control_cfsr_gnu Checking test 249 datm_cdeps_control_cfsr_gnu results .... Comparing RESTART/20111002.000000.MOM.res.nc .........OK Comparing RESTART/iced.2011-10-02-00000.nc .........OK Comparing RESTART/DATM_CFSR.cpl.r.2011-10-02-00000.nc .........OK - 0: The total amount of wall time = 167.409719 - 0: The maximum resident set size (KB) = 665676 + 0: The total amount of wall time = 167.558810 + 0: The maximum resident set size (KB) = 667636 Test 249 datm_cdeps_control_cfsr_gnu PASS REGRESSION TEST WAS SUCCESSFUL -Thu Oct 5 00:19:47 UTC 2023 -Elapsed time: 03h:01m:42s. Have a nice day! +Wed Oct 11 23:41:20 UTC 2023 +Elapsed time: 02h:12m:57s. Have a nice day!