Upadte and tested CI Bash for WCOSS2 (#2481)

Updates to the driver scripts to the CI framework to work on WCOSS with
Cron Bash.

Resolves #2268

ci/cases/pr/C48_S2SWA_gefs.yaml

@@ -17,3 +17,5 @@ arguments:
   edate: 2021032312
   yaml: {{ HOMEgfs }}/ci/cases/yamls/gefs_ci_defaults.yaml
 
+skip_ci_on_hosts:
+  - wcoss2

ci/cases/pr/C48mx500_3DVarAOWCDA.yaml

@@ -18,6 +18,7 @@ arguments:
   yaml: {{ HOMEgfs }}/ci/cases/yamls/soca_gfs_defaults_ci.yaml
 
 skip_ci_on_hosts:
+  - wcoss2
   - orion
   - hera
   - hercules

ci/cases/pr/C96C48_ufs_hybatmDA.yaml

@@ -21,3 +21,5 @@ skip_ci_on_hosts:
   - hera
   - orion
   - hercules
+  - wcoss2
+

ci/cases/pr/C96_atmaerosnowDA.yaml

@@ -19,3 +19,4 @@ arguments:
 skip_ci_on_hosts:
   - orion
   - hercules
+  - wcoss2

ci/platforms/config.wcoss2

@@ -0,0 +1,8 @@
+#!/usr/bin/bash
+
+export GFS_CI_ROOT=/lfs/h2/emc/global/noscrub/globalworkflow.ci/GFS_CI_ROOT
+export ICSDIR_ROOT=/lfs/h2/emc/global/noscrub/emc.global/data/ICSDIR
+export STMP="/lfs/h2/emc/stmp/${USER}"
+export SLURM_ACCOUNT=GFS-DEV
+export max_concurrent_cases=5
+export max_concurrent_pr=4

ci/scripts/check_ci.sh

@@ -13,16 +13,15 @@ scriptname=$(basename "${BASH_SOURCE[0]}")
 echo "Begin ${scriptname} at $(date -u)" || true
 export PS4='+ $(basename ${BASH_SOURCE})[${LINENO}]'
 
-GH=${HOME}/bin/gh
-REPO_URL="https://github.com/NOAA-EMC/global-workflow.git"
+REPO_URL=${REPO_URL:-"[email protected]:NOAA-EMC/global-workflow.git"}
 
 #########################################################################
 #  Set up runtime environment varibles for accounts on supproted machines
 #########################################################################
 
 source "${HOMEgfs}/ush/detect_machine.sh"
 case ${MACHINE_ID} in
-  hera | orion | hercules)
+  hera | orion | hercules | wcoss2)
    echo "Running Automated Testing on ${MACHINE_ID}"
    source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}"
    ;;
@@ -38,7 +37,18 @@ source "${HOMEgfs}/ci/scripts/utils/ci_utils.sh"
 module use "${HOMEgfs}/modulefiles"
 module load "module_gwsetup.${MACHINE_ID}"
 module list
+# Load machine specific modules for ci (only wcoss2 is current)
+if [[ "${MACHINE_ID}" == "wcoss2" ]]; then
+  module load "module_gwci.${MACHINE_ID}"
+fi
 set -x
+if ! command -v gh > /dev/null; then
+   GH="${HOME}/bin/gh"
+else
+   GH=$(command -v gh)
+fi
+export GH
+
 rocotostat=$(command -v rocotostat)
 if [[ -z ${rocotostat+x} ]]; then
   echo "rocotostat not found on system"
@@ -141,10 +151,7 @@ for pr in ${pr_list}; do
       sed -i "1 i\`\`\`" "${output_ci}"
       "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci}"
       "${HOMEgfs}/ci/scripts/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}"
-      for kill_cases in "${pr_dir}/RUNTESTS/"*; do
-         pslot=$(basename "${kill_cases}")
-         cancel_slurm_jobs "${pslot}"
-      done
+      cancel_all_batch_jobs "${pr_dir}/RUNTESTS/"
       break
     fi
     if [[ "${num_done}" -eq  "${num_cycles}" ]]; then

ci/scripts/clone-build_ci.sh

@@ -74,7 +74,7 @@ set +e
 source "${HOMEgfs}/ush/module-setup.sh"
 export BUILD_JOBS=8
 rm -rf log.build
-./build_all.sh -gu  >> log.build 2>&1
+./build_all.sh -g  >> log.build 2>&1
 build_status=$?
 
 DATE=$(date +'%D %r')

ci/scripts/driver.sh

@@ -14,11 +14,7 @@ set -eux
 # It then is ready to run a suite of regression tests with various configurations
 #######################################################################################
 
-#################################################################
-# TODO using static build for GitHub CLI until fixed in HPC-Stack
-#################################################################
-export GH=${HOME}/bin/gh
-export REPO_URL=${REPO_URL:-"https://github.com/NOAA-EMC/global-workflow.git"}
+export REPO_URL=${REPO_URL:-"[email protected]:NOAA-EMC/global-workflow.git"}
 
 ################################################################
 # Setup the reletive paths to scripts and PS4 for better logging
@@ -34,7 +30,7 @@ export PS4='+ $(basename ${BASH_SOURCE})[${LINENO}]'
 
 source "${ROOT_DIR}/ush/detect_machine.sh"
 case ${MACHINE_ID} in
-  hera | orion | hercules)
+  hera | orion | hercules | wcoss2)
     echo "Running Automated Testing on ${MACHINE_ID}"
     source "${ROOT_DIR}/ci/platforms/config.${MACHINE_ID}"
     ;;
@@ -54,8 +50,18 @@ source "${ROOT_DIR}/ci/scripts/utils/ci_utils.sh"
 source "${ROOT_DIR}/ush/module-setup.sh"
 module use "${ROOT_DIR}/modulefiles"
 module load "module_gwsetup.${MACHINE_ID}"
+# Load machine specific modules for ci (only wcoss2 is current)
+if [[ "${MACHINE_ID}" == "wcoss2" ]]; then
+  module load "module_gwci.${MACHINE_ID}"
+fi
 set -x
 unset HOMEgfs
+if ! command -v gh > /dev/null; then
+   GH="${HOME}/bin/gh"
+else
+   GH=$(command -v gh)
+fi
+export GH
 
 ############################################################
 # query repo and get list of open PRs with tags {machine}-CI
@@ -113,7 +119,7 @@ for pr in ${pr_list}; do
     else
       for case in ${experiments}; do
         case_name=$(basename "${case}")
-        cancel_slurm_jobs "${case_name}"
+        cancel_batch_jobs "${case_name}"
         {
           echo "Canceled all jobs for experiment ${case_name} in PR:${pr} on ${MACHINE_ID^}"
         } >> "${output_ci_single}"

ci/scripts/driver_weekly.sh

@@ -38,7 +38,7 @@ export PS4='+ $(basename ${BASH_SOURCE[0]})[${LINENO}]'
 
 source "${ROOT_DIR}/ush/detect_machine.sh"
 case ${MACHINE_ID} in
-  hera | orion | hercules)
+  hera | orion | hercules | wcoss2)
     echo "Running Automated Testing on ${MACHINE_ID}"
     source "${ROOT_DIR}/ci/platforms/config.${MACHINE_ID}"
     ;;

ci/scripts/run_ci.sh

@@ -20,7 +20,7 @@ export PS4='+ $(basename ${BASH_SOURCE})[${LINENO}]'
 
 source "${HOMEgfs}/ush/detect_machine.sh"
 case ${MACHINE_ID} in
-  hera | orion | hercules)
+  hera | orion | hercules | wcoss2)
    echo "Running Automated Testing on ${MACHINE_ID}"
    source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}"
    ;;

ci/scripts/utils/ci_utils.sh

@@ -23,7 +23,7 @@ function cancel_batch_jobs() {
 
   scheduler=$(determine_scheduler)
 
-  if [[ "${schduler}" == "torque" ]]; then
+  if [[ "${scheduler}" == "torque" ]]; then
     job_ids=$(qstat -u "${USER}" | awk '{print $1}') || true
 
     for job_id in ${job_ids}; do

modulefiles/module_gwci.wcoss2.lua

@@ -0,0 +1,8 @@
+help([[
+Load environment to run GFS workflow ci scripts on WCOSS2
+]])
+
+prepend_path("MODULEPATH", "/apps/ops/para/nco/modulefiles/core")
+load(pathJoin("gh","2.28.0"))
+
+whatis("Description: GFS run setup ci environment")