From d465ea06e8b2a8f3a5eb1120647c1e2ce5197d66 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Thu, 15 Feb 2024 19:25:02 +0000 Subject: [PATCH 1/2] Set HOMEgfs for module_setup in CI driver (#2321) Hotfixes to CI Bash system from updates with sourcing `detect_machine.sh` in `ush/module-setup.sh` using **HOMEgfs**. --- ci/scripts/check_ci.sh | 23 ++++++++++++----------- ci/scripts/driver.sh | 3 +++ ci/scripts/run_ci.sh | 13 +++++++------ 3 files changed, 22 insertions(+), 17 deletions(-) diff --git a/ci/scripts/check_ci.sh b/ci/scripts/check_ci.sh index cda2d4e9f2..4ff7eefd26 100755 --- a/ci/scripts/check_ci.sh +++ b/ci/scripts/check_ci.sh @@ -8,7 +8,7 @@ set -eux # to run from within a cron job in the CI Managers account ##################################################################################### -ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd )" +HOMEgfs="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd )" scriptname=$(basename "${BASH_SOURCE[0]}") echo "Begin ${scriptname} at $(date -u)" || true export PS4='+ $(basename ${BASH_SOURCE})[${LINENO}]' @@ -20,11 +20,11 @@ REPO_URL="https://github.com/NOAA-EMC/global-workflow.git" # Set up runtime environment varibles for accounts on supproted machines ######################################################################### -source "${ROOT_DIR}/ush/detect_machine.sh" +source "${HOMEgfs}/ush/detect_machine.sh" case ${MACHINE_ID} in hera | orion | hercules) echo "Running Automated Testing on ${MACHINE_ID}" - source "${ROOT_DIR}/ci/platforms/config.${MACHINE_ID}" + source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}" ;; *) echo "Unsupported platform. Exiting with error." @@ -32,9 +32,10 @@ case ${MACHINE_ID} in ;; esac set +x -source "${ROOT_DIR}/ush/module-setup.sh" -source "${ROOT_DIR}/ci/scripts/utils/ci_utils.sh" -module use "${ROOT_DIR}/modulefiles" +export HOMEgfs +source "${HOMEgfs}/ush/module-setup.sh" +source "${HOMEgfs}/ci/scripts/utils/ci_utils.sh" +module use "${HOMEgfs}/modulefiles" module load "module_gwsetup.${MACHINE_ID}" module list set -x @@ -57,7 +58,7 @@ pr_list_dbfile="${GFS_CI_ROOT}/open_pr_list.db" pr_list="" if [[ -f "${pr_list_dbfile}" ]]; then - pr_list=$("${ROOT_DIR}/ci/scripts/pr_list_database.py" --dbfile "${pr_list_dbfile}" --display | grep -v Failed | grep Running | awk '{print $1}') || true + pr_list=$("${HOMEgfs}/ci/scripts/pr_list_database.py" --dbfile "${pr_list_dbfile}" --display | grep -v Failed | grep Running | awk '{print $1}') || true fi if [[ -z "${pr_list+x}" ]]; then echo "no PRs open and ready to run cases on .. exiting" @@ -89,13 +90,13 @@ for pr in ${pr_list}; do sed -i "1 i\`\`\`" "${output_ci}" sed -i "1 i\All CI Test Cases Passed on ${MACHINE_ID^}:" "${output_ci}" "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci}" - "${ROOT_DIR}/ci/scripts/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}" + "${HOMEgfs}/ci/scripts/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}" # Check to see if this PR that was opened by the weekly tests and if so close it if it passed on all platforms weekly_labels=$(${GH} pr view "${pr}" --repo "${REPO_URL}" --json headRefName,labels,author --jq 'select(.author.login | contains("emcbot")) | select(.headRefName | contains("weekly_ci")) | .labels[].name ') || true if [[ -n "${weekly_labels}" ]]; then - num_platforms=$(find "${ROOT_DIR}/ci/platforms" -type f -name "config.*" | wc -l) + num_platforms=$(find "${HOMEgfs}/ci/platforms" -type f -name "config.*" | wc -l) passed=0 - for platforms in "${ROOT_DIR}"/ci/platforms/config.*; do + for platforms in "${HOMEgfs}"/ci/platforms/config.*; do machine=$(basename "${platforms}" | cut -d. -f2) if [[ "${weekly_labels}" == *"CI-${machine^}-Passed"* ]]; then ((passed=passed+1)) @@ -139,7 +140,7 @@ for pr in ${pr_list}; do } >> "${output_ci}" sed -i "1 i\`\`\`" "${output_ci}" "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci}" - "${ROOT_DIR}/ci/scripts/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}" + "${HOMEgfs}/ci/scripts/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}" for kill_cases in "${pr_dir}/RUNTESTS/"*; do pslot=$(basename "${kill_cases}") cancel_slurm_jobs "${pslot}" diff --git a/ci/scripts/driver.sh b/ci/scripts/driver.sh index 5fc13ea524..f37b5e3f2e 100755 --- a/ci/scripts/driver.sh +++ b/ci/scripts/driver.sh @@ -47,12 +47,15 @@ esac ###################################################### # setup runtime env for correct python install and git ###################################################### +HOMEgfs=${ROOT_DIR} +export HOMEgfs set +x source "${ROOT_DIR}/ci/scripts/utils/ci_utils.sh" source "${ROOT_DIR}/ush/module-setup.sh" module use "${ROOT_DIR}/modulefiles" module load "module_gwsetup.${MACHINE_ID}" set -x +unset HOMEgfs ############################################################ # query repo and get list of open PRs with tags {machine}-CI diff --git a/ci/scripts/run_ci.sh b/ci/scripts/run_ci.sh index 4a390a23f2..f50a4465d0 100755 --- a/ci/scripts/run_ci.sh +++ b/ci/scripts/run_ci.sh @@ -9,7 +9,7 @@ set -eux # Abstract TODO ##################################################################################### -ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd )" +HOMEgfs="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd )" scriptname=$(basename "${BASH_SOURCE[0]}") echo "Begin ${scriptname} at $(date -u)" || true export PS4='+ $(basename ${BASH_SOURCE})[${LINENO}]' @@ -18,11 +18,11 @@ export PS4='+ $(basename ${BASH_SOURCE})[${LINENO}]' # Set up runtime environment varibles for accounts on supproted machines ######################################################################### -source "${ROOT_DIR}/ush/detect_machine.sh" +source "${HOMEgfs}/ush/detect_machine.sh" case ${MACHINE_ID} in hera | orion | hercules) echo "Running Automated Testing on ${MACHINE_ID}" - source "${ROOT_DIR}/ci/platforms/config.${MACHINE_ID}" + source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}" ;; *) echo "Unsupported platform. Exiting with error." @@ -30,8 +30,9 @@ case ${MACHINE_ID} in ;; esac set +x -source "${ROOT_DIR}/ush/module-setup.sh" -module use "${ROOT_DIR}/modulefiles" +export HOMEgfs +source "${HOMEgfs}/ush/module-setup.sh" +module use "${HOMEgfs}/modulefiles" module load "module_gwsetup.${MACHINE_ID}" module list set -eux @@ -47,7 +48,7 @@ pr_list_dbfile="${GFS_CI_ROOT}/open_pr_list.db" pr_list="" if [[ -f "${pr_list_dbfile}" ]]; then - pr_list=$("${ROOT_DIR}/ci/scripts/pr_list_database.py" --display --dbfile "${pr_list_dbfile}" | grep -v Failed | grep Open | grep Running | awk '{print $1}' | head -"${max_concurrent_pr}") || true + pr_list=$("${HOMEgfs}/ci/scripts/pr_list_database.py" --display --dbfile "${pr_list_dbfile}" | grep -v Failed | grep Open | grep Running | awk '{print $1}' | head -"${max_concurrent_pr}") || true fi if [[ -z "${pr_list}" ]]; then echo "no open and built PRs that are ready for the cases to advance with rocotorun .. exiting" From 094e3b86da44f1d3fc1d99f68f6fdfcd36deb09f Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Thu, 15 Feb 2024 14:43:55 -0500 Subject: [PATCH 2/2] Move IMS remapping files from COM_OBS to FIXgdas (#2322) * Add in IMS obs fix directory and update submodule for gdas --- parm/config/gfs/config.base.emc.dyn | 1 + sorc/gdas.cd | 2 +- sorc/link_workflow.sh | 2 +- ush/python/pygfs/task/land_analysis.py | 2 +- versions/fix.ver | 1 + 5 files changed, 5 insertions(+), 3 deletions(-) diff --git a/parm/config/gfs/config.base.emc.dyn b/parm/config/gfs/config.base.emc.dyn index 1f6568c3ee..48b66ab1b1 100644 --- a/parm/config/gfs/config.base.emc.dyn +++ b/parm/config/gfs/config.base.emc.dyn @@ -39,6 +39,7 @@ export FIXcice="${FIXgfs}/cice" export FIXmom="${FIXgfs}/mom6" export FIXreg2grb2="${FIXgfs}/reg2grb2" export FIXugwd="${FIXgfs}/ugwd" +export FIXgdas="${FIXgfs}/gdas" ######################################################################## diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 831b08a3f9..b37680be78 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 831b08a3f947e8d743e2afbd6d38ecc4b0dec3b1 +Subproject commit b37680be78560343b940352b9b2c6727190a642c diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index 21bc30faa4..3e0ff4db64 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -190,7 +190,7 @@ if [[ -d "${HOMEgfs}/sorc/gdas.cd" ]]; then cd "${HOMEgfs}/fix" || exit 1 [[ ! -d gdas ]] && mkdir -p gdas cd gdas || exit 1 - for gdas_sub in fv3jedi gsibec; do + for gdas_sub in fv3jedi gsibec obs; do if [[ -d "${gdas_sub}" ]]; then rm -rf "${gdas_sub}" fi diff --git a/ush/python/pygfs/task/land_analysis.py b/ush/python/pygfs/task/land_analysis.py index 821caf2305..90ab997de1 100644 --- a/ush/python/pygfs/task/land_analysis.py +++ b/ush/python/pygfs/task/land_analysis.py @@ -153,7 +153,7 @@ def prepare_IMS(self) -> None: # create a temporary dict of all keys needed in this method localconf = AttrDict() keys = ['DATA', 'current_cycle', 'COM_OBS', 'COM_ATMOS_RESTART_PREV', - 'OPREFIX', 'CASE', 'OCNRES', 'ntiles'] + 'OPREFIX', 'CASE', 'OCNRES', 'ntiles', 'FIXgdas'] for key in keys: localconf[key] = self.task_config[key] diff --git a/versions/fix.ver b/versions/fix.ver index 13d9b56dd2..f230188dbb 100644 --- a/versions/fix.ver +++ b/versions/fix.ver @@ -10,6 +10,7 @@ export datm_ver=20220805 export gdas_crtm_ver=20220805 export gdas_fv3jedi_ver=20220805 export gdas_gsibec_ver=20221031 +export gdas_obs_ver=20240213 export glwu_ver=20220805 export gsi_ver=20230911 export lut_ver=20220805