diff --git a/ci/cases/pr/C48_S2SWA_gefs.yaml b/ci/cases/pr/C48_S2SWA_gefs.yaml index 310d0ea615..a924b416c3 100644 --- a/ci/cases/pr/C48_S2SWA_gefs.yaml +++ b/ci/cases/pr/C48_S2SWA_gefs.yaml @@ -17,3 +17,5 @@ arguments: edate: 2021032312 yaml: {{ HOMEgfs }}/ci/cases/yamls/gefs_ci_defaults.yaml +skip_ci_on_hosts: + - wcoss2 diff --git a/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml b/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml index d9156e38f3..3356f2030b 100644 --- a/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml +++ b/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml @@ -18,6 +18,7 @@ arguments: yaml: {{ HOMEgfs }}/ci/cases/yamls/soca_gfs_defaults_ci.yaml skip_ci_on_hosts: + - wcoss2 - orion - hera - hercules diff --git a/ci/cases/pr/C96C48_ufs_hybatmDA.yaml b/ci/cases/pr/C96C48_ufs_hybatmDA.yaml index da68b0f86c..1b57be63eb 100644 --- a/ci/cases/pr/C96C48_ufs_hybatmDA.yaml +++ b/ci/cases/pr/C96C48_ufs_hybatmDA.yaml @@ -21,3 +21,5 @@ skip_ci_on_hosts: - hera - orion - hercules + - wcoss2 + diff --git a/ci/cases/pr/C96_atmaerosnowDA.yaml b/ci/cases/pr/C96_atmaerosnowDA.yaml index 7e22955a37..edde37cbf7 100644 --- a/ci/cases/pr/C96_atmaerosnowDA.yaml +++ b/ci/cases/pr/C96_atmaerosnowDA.yaml @@ -19,3 +19,4 @@ arguments: skip_ci_on_hosts: - orion - hercules + - wcoss2 diff --git a/ci/platforms/config.wcoss2 b/ci/platforms/config.wcoss2 new file mode 100644 index 0000000000..994f92e0d2 --- /dev/null +++ b/ci/platforms/config.wcoss2 @@ -0,0 +1,8 @@ +#!/usr/bin/bash + +export GFS_CI_ROOT=/lfs/h2/emc/global/noscrub/globalworkflow.ci/GFS_CI_ROOT +export ICSDIR_ROOT=/lfs/h2/emc/global/noscrub/emc.global/data/ICSDIR +export STMP="/lfs/h2/emc/stmp/${USER}" +export SLURM_ACCOUNT=GFS-DEV +export max_concurrent_cases=5 +export max_concurrent_pr=4 diff --git a/ci/scripts/check_ci.sh b/ci/scripts/check_ci.sh index 4ff7eefd26..8cf5110d9f 100755 --- a/ci/scripts/check_ci.sh +++ b/ci/scripts/check_ci.sh @@ -13,8 +13,7 @@ scriptname=$(basename "${BASH_SOURCE[0]}") echo "Begin ${scriptname} at $(date -u)" || true export PS4='+ $(basename ${BASH_SOURCE})[${LINENO}]' -GH=${HOME}/bin/gh -REPO_URL="https://github.com/NOAA-EMC/global-workflow.git" +REPO_URL=${REPO_URL:-"git@github.com:NOAA-EMC/global-workflow.git"} ######################################################################### # Set up runtime environment varibles for accounts on supproted machines @@ -22,7 +21,7 @@ REPO_URL="https://github.com/NOAA-EMC/global-workflow.git" source "${HOMEgfs}/ush/detect_machine.sh" case ${MACHINE_ID} in - hera | orion | hercules) + hera | orion | hercules | wcoss2) echo "Running Automated Testing on ${MACHINE_ID}" source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}" ;; @@ -38,7 +37,18 @@ source "${HOMEgfs}/ci/scripts/utils/ci_utils.sh" module use "${HOMEgfs}/modulefiles" module load "module_gwsetup.${MACHINE_ID}" module list +# Load machine specific modules for ci (only wcoss2 is current) +if [[ "${MACHINE_ID}" == "wcoss2" ]]; then + module load "module_gwci.${MACHINE_ID}" +fi set -x +if ! command -v gh > /dev/null; then + GH="${HOME}/bin/gh" +else + GH=$(command -v gh) +fi +export GH + rocotostat=$(command -v rocotostat) if [[ -z ${rocotostat+x} ]]; then echo "rocotostat not found on system" @@ -141,10 +151,7 @@ for pr in ${pr_list}; do sed -i "1 i\`\`\`" "${output_ci}" "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci}" "${HOMEgfs}/ci/scripts/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}" - for kill_cases in "${pr_dir}/RUNTESTS/"*; do - pslot=$(basename "${kill_cases}") - cancel_slurm_jobs "${pslot}" - done + cancel_all_batch_jobs "${pr_dir}/RUNTESTS/" break fi if [[ "${num_done}" -eq "${num_cycles}" ]]; then diff --git a/ci/scripts/clone-build_ci.sh b/ci/scripts/clone-build_ci.sh index 798c98bf50..940f9e7393 100755 --- a/ci/scripts/clone-build_ci.sh +++ b/ci/scripts/clone-build_ci.sh @@ -74,7 +74,7 @@ set +e source "${HOMEgfs}/ush/module-setup.sh" export BUILD_JOBS=8 rm -rf log.build -./build_all.sh -gu >> log.build 2>&1 +./build_all.sh -g >> log.build 2>&1 build_status=$? DATE=$(date +'%D %r') diff --git a/ci/scripts/driver.sh b/ci/scripts/driver.sh index f37b5e3f2e..76eaf4c448 100755 --- a/ci/scripts/driver.sh +++ b/ci/scripts/driver.sh @@ -14,11 +14,7 @@ set -eux # It then is ready to run a suite of regression tests with various configurations ####################################################################################### -################################################################# -# TODO using static build for GitHub CLI until fixed in HPC-Stack -################################################################# -export GH=${HOME}/bin/gh -export REPO_URL=${REPO_URL:-"https://github.com/NOAA-EMC/global-workflow.git"} +export REPO_URL=${REPO_URL:-"git@github.com:NOAA-EMC/global-workflow.git"} ################################################################ # Setup the reletive paths to scripts and PS4 for better logging @@ -34,7 +30,7 @@ export PS4='+ $(basename ${BASH_SOURCE})[${LINENO}]' source "${ROOT_DIR}/ush/detect_machine.sh" case ${MACHINE_ID} in - hera | orion | hercules) + hera | orion | hercules | wcoss2) echo "Running Automated Testing on ${MACHINE_ID}" source "${ROOT_DIR}/ci/platforms/config.${MACHINE_ID}" ;; @@ -54,8 +50,18 @@ source "${ROOT_DIR}/ci/scripts/utils/ci_utils.sh" source "${ROOT_DIR}/ush/module-setup.sh" module use "${ROOT_DIR}/modulefiles" module load "module_gwsetup.${MACHINE_ID}" +# Load machine specific modules for ci (only wcoss2 is current) +if [[ "${MACHINE_ID}" == "wcoss2" ]]; then + module load "module_gwci.${MACHINE_ID}" +fi set -x unset HOMEgfs +if ! command -v gh > /dev/null; then + GH="${HOME}/bin/gh" +else + GH=$(command -v gh) +fi +export GH ############################################################ # query repo and get list of open PRs with tags {machine}-CI @@ -113,7 +119,7 @@ for pr in ${pr_list}; do else for case in ${experiments}; do case_name=$(basename "${case}") - cancel_slurm_jobs "${case_name}" + cancel_batch_jobs "${case_name}" { echo "Canceled all jobs for experiment ${case_name} in PR:${pr} on ${MACHINE_ID^}" } >> "${output_ci_single}" diff --git a/ci/scripts/driver_weekly.sh b/ci/scripts/driver_weekly.sh index 9460e0b0a4..6cd2493769 100755 --- a/ci/scripts/driver_weekly.sh +++ b/ci/scripts/driver_weekly.sh @@ -38,7 +38,7 @@ export PS4='+ $(basename ${BASH_SOURCE[0]})[${LINENO}]' source "${ROOT_DIR}/ush/detect_machine.sh" case ${MACHINE_ID} in - hera | orion | hercules) + hera | orion | hercules | wcoss2) echo "Running Automated Testing on ${MACHINE_ID}" source "${ROOT_DIR}/ci/platforms/config.${MACHINE_ID}" ;; diff --git a/ci/scripts/run_ci.sh b/ci/scripts/run_ci.sh index f50a4465d0..37a9da162b 100755 --- a/ci/scripts/run_ci.sh +++ b/ci/scripts/run_ci.sh @@ -20,7 +20,7 @@ export PS4='+ $(basename ${BASH_SOURCE})[${LINENO}]' source "${HOMEgfs}/ush/detect_machine.sh" case ${MACHINE_ID} in - hera | orion | hercules) + hera | orion | hercules | wcoss2) echo "Running Automated Testing on ${MACHINE_ID}" source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}" ;; diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh index ce2e039307..9c0e8ac997 100755 --- a/ci/scripts/utils/ci_utils.sh +++ b/ci/scripts/utils/ci_utils.sh @@ -23,7 +23,7 @@ function cancel_batch_jobs() { scheduler=$(determine_scheduler) - if [[ "${schduler}" == "torque" ]]; then + if [[ "${scheduler}" == "torque" ]]; then job_ids=$(qstat -u "${USER}" | awk '{print $1}') || true for job_id in ${job_ids}; do diff --git a/modulefiles/module_gwci.wcoss2.lua b/modulefiles/module_gwci.wcoss2.lua new file mode 100644 index 0000000000..50eaeefee0 --- /dev/null +++ b/modulefiles/module_gwci.wcoss2.lua @@ -0,0 +1,8 @@ +help([[ +Load environment to run GFS workflow ci scripts on WCOSS2 +]]) + +prepend_path("MODULEPATH", "/apps/ops/para/nco/modulefiles/core") +load(pathJoin("gh","2.28.0")) + +whatis("Description: GFS run setup ci environment")