From 423e400dd4ccedeec02fc59d16fae5a5b4e70510 Mon Sep 17 00:00:00 2001
From: terry mcguinness
 <terry.mcguinness@dlogin07.dogwood.wcoss2.ncep.noaa.gov>
Date: Wed, 10 Apr 2024 17:32:34 +0000
Subject: [PATCH 01/22] added wcoss2 bash ci config file for platform

---
 ci/platforms/config.wcoss2 | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 ci/platforms/config.wcoss2

diff --git a/ci/platforms/config.wcoss2 b/ci/platforms/config.wcoss2
new file mode 100644
index 0000000000..5f73ccadbd
--- /dev/null
+++ b/ci/platforms/config.wcoss2
@@ -0,0 +1,8 @@
+#!/usr/bin/bash
+
+export GFS_CI_ROOT=/lfs/h2/emc/global/noscrub/globalworkflow.ci/GFS_CI_ROOT
+export ICSDIR_ROOT=/lfs/h2/emc/global/noscrub/emc.global/data/ICSDIR
+export STMP=/lfs/h2/emc/stmp/$USER
+#export SLURM_ACCOUNT=nems
+export max_concurrent_cases=5
+export max_concurrent_pr=4

From d1a543f86242b19f0c13e955ed8ee5246cb6a148 Mon Sep 17 00:00:00 2001
From: Terry McGuinness <terry.mcguinness@noaa.gov>
Date: Wed, 10 Apr 2024 18:14:04 +0000
Subject: [PATCH 02/22] updated already defaulted ACCOUNT still miss-named
 SLURM though

---
 ci/platforms/config.wcoss2 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/platforms/config.wcoss2 b/ci/platforms/config.wcoss2
index 5f73ccadbd..072680ee81 100644
--- a/ci/platforms/config.wcoss2
+++ b/ci/platforms/config.wcoss2
@@ -3,6 +3,6 @@
 export GFS_CI_ROOT=/lfs/h2/emc/global/noscrub/globalworkflow.ci/GFS_CI_ROOT
 export ICSDIR_ROOT=/lfs/h2/emc/global/noscrub/emc.global/data/ICSDIR
 export STMP=/lfs/h2/emc/stmp/$USER
-#export SLURM_ACCOUNT=nems
+export SLURM_ACCOUNT=GFS-DEV
 export max_concurrent_cases=5
 export max_concurrent_pr=4

From 659d74837b45cfcf1a21c3bbc10b6a44494c5313 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 10 Apr 2024 18:22:36 +0000
Subject: [PATCH 03/22] added wcoss2 for checking if system is supported in the
 driver ci scripts

---
 ci/scripts/check_ci.sh      | 2 +-
 ci/scripts/driver.sh        | 2 +-
 ci/scripts/driver_weekly.sh | 2 +-
 ci/scripts/run_ci.sh        | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/ci/scripts/check_ci.sh b/ci/scripts/check_ci.sh
index 4ff7eefd26..98b8455f90 100755
--- a/ci/scripts/check_ci.sh
+++ b/ci/scripts/check_ci.sh
@@ -22,7 +22,7 @@ REPO_URL="https://github.com/NOAA-EMC/global-workflow.git"
 
 source "${HOMEgfs}/ush/detect_machine.sh"
 case ${MACHINE_ID} in
-  hera | orion | hercules)
+  hera | orion | hercules | wcoss2)
    echo "Running Automated Testing on ${MACHINE_ID}"
    source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}"
    ;;
diff --git a/ci/scripts/driver.sh b/ci/scripts/driver.sh
index f37b5e3f2e..38f247f340 100755
--- a/ci/scripts/driver.sh
+++ b/ci/scripts/driver.sh
@@ -34,7 +34,7 @@ export PS4='+ $(basename ${BASH_SOURCE})[${LINENO}]'
 
 source "${ROOT_DIR}/ush/detect_machine.sh"
 case ${MACHINE_ID} in
-  hera | orion | hercules)
+  hera | orion | hercules | wcoss2)
     echo "Running Automated Testing on ${MACHINE_ID}"
     source "${ROOT_DIR}/ci/platforms/config.${MACHINE_ID}"
     ;;
diff --git a/ci/scripts/driver_weekly.sh b/ci/scripts/driver_weekly.sh
index 9460e0b0a4..6cd2493769 100755
--- a/ci/scripts/driver_weekly.sh
+++ b/ci/scripts/driver_weekly.sh
@@ -38,7 +38,7 @@ export PS4='+ $(basename ${BASH_SOURCE[0]})[${LINENO}]'
 
 source "${ROOT_DIR}/ush/detect_machine.sh"
 case ${MACHINE_ID} in
-  hera | orion | hercules)
+  hera | orion | hercules | wcoss2)
     echo "Running Automated Testing on ${MACHINE_ID}"
     source "${ROOT_DIR}/ci/platforms/config.${MACHINE_ID}"
     ;;
diff --git a/ci/scripts/run_ci.sh b/ci/scripts/run_ci.sh
index f50a4465d0..37a9da162b 100755
--- a/ci/scripts/run_ci.sh
+++ b/ci/scripts/run_ci.sh
@@ -20,7 +20,7 @@ export PS4='+ $(basename ${BASH_SOURCE})[${LINENO}]'
 
 source "${HOMEgfs}/ush/detect_machine.sh"
 case ${MACHINE_ID} in
-  hera | orion | hercules)
+  hera | orion | hercules | wcoss2)
    echo "Running Automated Testing on ${MACHINE_ID}"
    source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}"
    ;;

From f989ea897e8cf0b352dfc75aec68c7f19ddd7635 Mon Sep 17 00:00:00 2001
From: Terry McGuinness <terry.mcguinness@noaa.gov>
Date: Wed, 10 Apr 2024 19:19:36 +0000
Subject: [PATCH 04/22] update gh assignment in check too because WCOSS2 uses a
 module load

---
 ci/scripts/check_ci.sh | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/ci/scripts/check_ci.sh b/ci/scripts/check_ci.sh
index 98b8455f90..9152c0648a 100755
--- a/ci/scripts/check_ci.sh
+++ b/ci/scripts/check_ci.sh
@@ -13,7 +13,11 @@ scriptname=$(basename "${BASH_SOURCE[0]}")
 echo "Begin ${scriptname} at $(date -u)" || true
 export PS4='+ $(basename ${BASH_SOURCE})[${LINENO}]'
 
-GH=${HOME}/bin/gh
+export GH=$(which gh)
+if [[ $? != "0" ]]; then
+   export GH="${HOME}/bin/gh"
+fi
+
 REPO_URL="https://github.com/NOAA-EMC/global-workflow.git"
 
 #########################################################################

From c78f7b12b829c87740cdc523713babd78278b3d7 Mon Sep 17 00:00:00 2001
From: Terry McGuinness <terry.mcguinness@noaa.gov>
Date: Wed, 10 Apr 2024 20:37:05 +0000
Subject: [PATCH 05/22] upated GH assignment and REPO_URL from env if needed
 over ride

---
 ci/scripts/check_ci.sh | 2 +-
 ci/scripts/driver.sh   | 8 ++++++--
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/ci/scripts/check_ci.sh b/ci/scripts/check_ci.sh
index 9152c0648a..2754a262c2 100755
--- a/ci/scripts/check_ci.sh
+++ b/ci/scripts/check_ci.sh
@@ -18,7 +18,7 @@ if [[ $? != "0" ]]; then
    export GH="${HOME}/bin/gh"
 fi
 
-REPO_URL="https://github.com/NOAA-EMC/global-workflow.git"
+REPO_URL=${REPO_URL:-"git@github.com:NOAA-EMC/global-workflow.git"}
 
 #########################################################################
 #  Set up runtime environment varibles for accounts on supproted machines
diff --git a/ci/scripts/driver.sh b/ci/scripts/driver.sh
index 38f247f340..334ebb5b3f 100755
--- a/ci/scripts/driver.sh
+++ b/ci/scripts/driver.sh
@@ -17,8 +17,12 @@ set -eux
 #################################################################
 # TODO using static build for GitHub CLI until fixed in HPC-Stack
 #################################################################
-export GH=${HOME}/bin/gh
-export REPO_URL=${REPO_URL:-"https://github.com/NOAA-EMC/global-workflow.git"}
+
+export GH=$(which gh)
+if [[ $? != "0" ]]; then
+   export GH="${HOME}/bin/gh"
+fi
+export REPO_URL=${REPO_URL:-"git@github.com:NOAA-EMC/global-workflow.git"}
 
 ################################################################
 # Setup the reletive paths to scripts and PS4 for better logging

From 196026816563035ac82eff169071eb99554be962 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 11 Apr 2024 19:35:49 +0000
Subject: [PATCH 06/22] updated slurm util calls to bash in the drivers for
 bash ci

---
 ci/scripts/check_ci.sh | 4 +---
 ci/scripts/driver.sh   | 2 +-
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/ci/scripts/check_ci.sh b/ci/scripts/check_ci.sh
index 98b8455f90..8f7eb5d42f 100755
--- a/ci/scripts/check_ci.sh
+++ b/ci/scripts/check_ci.sh
@@ -141,9 +141,7 @@ for pr in ${pr_list}; do
       sed -i "1 i\`\`\`" "${output_ci}"
       "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci}"
       "${HOMEgfs}/ci/scripts/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}"
-      for kill_cases in "${pr_dir}/RUNTESTS/"*; do
-         pslot=$(basename "${kill_cases}")
-         cancel_slurm_jobs "${pslot}"
+      cancel_all_batch_jobs "${pr_dir}/RUNTESTS/"
       done
       break
     fi
diff --git a/ci/scripts/driver.sh b/ci/scripts/driver.sh
index 38f247f340..1998e78cba 100755
--- a/ci/scripts/driver.sh
+++ b/ci/scripts/driver.sh
@@ -113,7 +113,7 @@ for pr in ${pr_list}; do
     else
       for case in ${experiments}; do
         case_name=$(basename "${case}")
-        cancel_slurm_jobs "${case_name}"
+        cancel_batch_jobs "${case_name}"
         {
           echo "Canceled all jobs for experiment ${case_name} in PR:${pr} on ${MACHINE_ID^}"
         } >> "${output_ci_single}"

From 960a16179d37c44902aa9165dcb9b997f4b9737b Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 15 Apr 2024 16:28:39 +0000
Subject: [PATCH 07/22] added ci runtime module file for wcoss2

---
 modulefiles/module_gwci.wcoss2.lua | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 modulefiles/module_gwci.wcoss2.lua

diff --git a/modulefiles/module_gwci.wcoss2.lua b/modulefiles/module_gwci.wcoss2.lua
new file mode 100644
index 0000000000..929ef9a06d
--- /dev/null
+++ b/modulefiles/module_gwci.wcoss2.lua
@@ -0,0 +1,8 @@
+help([[
+Load environment to run GFS workflow ci scripts on WCOSS2
+]])
+
+prepend_path("MODULEPATH", "/apps/ops/para/nco/moudulefiles/core")
+load(pathJoin("gh","2.28.0"))
+
+whatis("Description: GFS run setup ci environment")

From 07294e46d22d3f99980dde2182068165e97bf747 Mon Sep 17 00:00:00 2001
From: Terry McGuinness <terry.mcguinness@noaa.gov>
Date: Mon, 15 Apr 2024 16:37:07 +0000
Subject: [PATCH 08/22] fixed path in ci modulefile for wcoss2

---
 modulefiles/module_gwci.wcoss2.lua | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modulefiles/module_gwci.wcoss2.lua b/modulefiles/module_gwci.wcoss2.lua
index 929ef9a06d..50eaeefee0 100644
--- a/modulefiles/module_gwci.wcoss2.lua
+++ b/modulefiles/module_gwci.wcoss2.lua
@@ -2,7 +2,7 @@ help([[
 Load environment to run GFS workflow ci scripts on WCOSS2
 ]])
 
-prepend_path("MODULEPATH", "/apps/ops/para/nco/moudulefiles/core")
+prepend_path("MODULEPATH", "/apps/ops/para/nco/modulefiles/core")
 load(pathJoin("gh","2.28.0"))
 
 whatis("Description: GFS run setup ci environment")

From 049149b64b7d1cf23679723099aed66596c82100 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 15 Apr 2024 16:42:24 +0000
Subject: [PATCH 09/22] update GH assignment for WCOSS2 updates

---
 ci/scripts/driver.sh | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/ci/scripts/driver.sh b/ci/scripts/driver.sh
index 7c5cfe782d..4fec2cdf6d 100755
--- a/ci/scripts/driver.sh
+++ b/ci/scripts/driver.sh
@@ -14,14 +14,6 @@ set -eux
 # It then is ready to run a suite of regression tests with various configurations
 #######################################################################################
 
-#################################################################
-# TODO using static build for GitHub CLI until fixed in HPC-Stack
-#################################################################
-
-export GH=$(which gh)
-if [[ $? != "0" ]]; then
-   export GH="${HOME}/bin/gh"
-fi
 export REPO_URL=${REPO_URL:-"git@github.com:NOAA-EMC/global-workflow.git"}
 
 ################################################################
@@ -58,8 +50,16 @@ source "${ROOT_DIR}/ci/scripts/utils/ci_utils.sh"
 source "${ROOT_DIR}/ush/module-setup.sh"
 module use "${ROOT_DIR}/modulefiles"
 module load "module_gwsetup.${MACHINE_ID}"
+# Load machine specific modules for ci (only wcoss2 is current)
+if [[ "${MACHINE_ID}" == "wcoss2" ]]; then
+  module load "module_gwci.${MACHINE_ID}"
+fi
 set -x
 unset HOMEgfs
+export GH=$(which gh)
+if [[ $? != "0" ]]; then
+   export GH="${HOME}/bin/gh"
+fi
 
 ############################################################
 # query repo and get list of open PRs with tags {machine}-CI

From 0e1abe7562b7d477757f18861fea455dca914039 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 15 Apr 2024 16:51:58 +0000
Subject: [PATCH 10/22] fixed shellnorms including GH export

---
 ci/platforms/config.wcoss2 | 2 +-
 ci/scripts/driver.sh       | 4 +++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/ci/platforms/config.wcoss2 b/ci/platforms/config.wcoss2
index 072680ee81..994f92e0d2 100644
--- a/ci/platforms/config.wcoss2
+++ b/ci/platforms/config.wcoss2
@@ -2,7 +2,7 @@
 
 export GFS_CI_ROOT=/lfs/h2/emc/global/noscrub/globalworkflow.ci/GFS_CI_ROOT
 export ICSDIR_ROOT=/lfs/h2/emc/global/noscrub/emc.global/data/ICSDIR
-export STMP=/lfs/h2/emc/stmp/$USER
+export STMP="/lfs/h2/emc/stmp/${USER}"
 export SLURM_ACCOUNT=GFS-DEV
 export max_concurrent_cases=5
 export max_concurrent_pr=4
diff --git a/ci/scripts/driver.sh b/ci/scripts/driver.sh
index 4fec2cdf6d..6eaa3ea717 100755
--- a/ci/scripts/driver.sh
+++ b/ci/scripts/driver.sh
@@ -56,9 +56,11 @@ if [[ "${MACHINE_ID}" == "wcoss2" ]]; then
 fi
 set -x
 unset HOMEgfs
-export GH=$(which gh)
+GH=$(which gh)
 if [[ $? != "0" ]]; then
    export GH="${HOME}/bin/gh"
+else
+   export GH
 fi
 
 ############################################################

From bfc37c2a765f87e1e1d4edb117b451688d43c791 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 15 Apr 2024 17:01:56 +0000
Subject: [PATCH 11/22] replace with with command -v for gh

---
 ci/scripts/driver.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/scripts/driver.sh b/ci/scripts/driver.sh
index 6eaa3ea717..24b6bc9e25 100755
--- a/ci/scripts/driver.sh
+++ b/ci/scripts/driver.sh
@@ -56,7 +56,7 @@ if [[ "${MACHINE_ID}" == "wcoss2" ]]; then
 fi
 set -x
 unset HOMEgfs
-GH=$(which gh)
+GH=$(command -v gh)
 if [[ $? != "0" ]]; then
    export GH="${HOME}/bin/gh"
 else

From 3566d666960bbd0ff01fa4abfeac8b6165f8092c Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 15 Apr 2024 17:09:02 +0000
Subject: [PATCH 12/22] check_ci.sh had an extra done in the wrong place

---
 ci/scripts/check_ci.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/ci/scripts/check_ci.sh b/ci/scripts/check_ci.sh
index 6af4eea39f..df53804f10 100755
--- a/ci/scripts/check_ci.sh
+++ b/ci/scripts/check_ci.sh
@@ -146,7 +146,6 @@ for pr in ${pr_list}; do
       "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci}"
       "${HOMEgfs}/ci/scripts/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}"
       cancel_all_batch_jobs "${pr_dir}/RUNTESTS/"
-      done
       break
     fi
     if [[ "${num_done}" -eq  "${num_cycles}" ]]; then

From 6dfc13336afd100d8d1890002b4fce543710d1a4 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 15 Apr 2024 17:14:02 +0000
Subject: [PATCH 13/22] updated gh assignment for shellnorms again

---
 ci/scripts/check_ci.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/ci/scripts/check_ci.sh b/ci/scripts/check_ci.sh
index df53804f10..cd6a73ab53 100755
--- a/ci/scripts/check_ci.sh
+++ b/ci/scripts/check_ci.sh
@@ -13,9 +13,11 @@ scriptname=$(basename "${BASH_SOURCE[0]}")
 echo "Begin ${scriptname} at $(date -u)" || true
 export PS4='+ $(basename ${BASH_SOURCE})[${LINENO}]'
 
-export GH=$(which gh)
+GH=$(command -v gh)
 if [[ $? != "0" ]]; then
    export GH="${HOME}/bin/gh"
+else
+   export GH
 fi
 
 REPO_URL=${REPO_URL:-"git@github.com:NOAA-EMC/global-workflow.git"}

From 95b4399245e72367ac8e81e049434e336b8f5b71 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 15 Apr 2024 17:18:51 +0000
Subject: [PATCH 14/22] update shellnorms recomondation for testig agains ?$

---
 ci/scripts/check_ci.sh | 3 +--
 ci/scripts/driver.sh   | 3 +--
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/ci/scripts/check_ci.sh b/ci/scripts/check_ci.sh
index cd6a73ab53..92b79dc5e3 100755
--- a/ci/scripts/check_ci.sh
+++ b/ci/scripts/check_ci.sh
@@ -13,8 +13,7 @@ scriptname=$(basename "${BASH_SOURCE[0]}")
 echo "Begin ${scriptname} at $(date -u)" || true
 export PS4='+ $(basename ${BASH_SOURCE})[${LINENO}]'
 
-GH=$(command -v gh)
-if [[ $? != "0" ]]; then
+if ! GH=$(command -v gh); then
    export GH="${HOME}/bin/gh"
 else
    export GH
diff --git a/ci/scripts/driver.sh b/ci/scripts/driver.sh
index 24b6bc9e25..5e59d8b77b 100755
--- a/ci/scripts/driver.sh
+++ b/ci/scripts/driver.sh
@@ -56,8 +56,7 @@ if [[ "${MACHINE_ID}" == "wcoss2" ]]; then
 fi
 set -x
 unset HOMEgfs
-GH=$(command -v gh)
-if [[ $? != "0" ]]; then
+if not GH=$(command -v gh); then
    export GH="${HOME}/bin/gh"
 else
    export GH

From a01c1e80be1ec0d557f2f6c7a9e0fba9658a4092 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 15 Apr 2024 17:30:54 +0000
Subject: [PATCH 15/22] copilot consulted and was succent this was best way to
 comply to shellnorms and use the command command twice for assign gh

---
 ci/scripts/check_ci.sh | 7 ++++---
 ci/scripts/driver.sh   | 7 ++++---
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/ci/scripts/check_ci.sh b/ci/scripts/check_ci.sh
index 92b79dc5e3..6aaf244a3d 100755
--- a/ci/scripts/check_ci.sh
+++ b/ci/scripts/check_ci.sh
@@ -13,11 +13,12 @@ scriptname=$(basename "${BASH_SOURCE[0]}")
 echo "Begin ${scriptname} at $(date -u)" || true
 export PS4='+ $(basename ${BASH_SOURCE})[${LINENO}]'
 
-if ! GH=$(command -v gh); then
-   export GH="${HOME}/bin/gh"
+if ! command -v gh > /dev/null; then
+   GH="${HOME}/bin/gh"
 else
-   export GH
+   GH=$(command -v gh)
 fi
+export GH
 
 REPO_URL=${REPO_URL:-"git@github.com:NOAA-EMC/global-workflow.git"}
 
diff --git a/ci/scripts/driver.sh b/ci/scripts/driver.sh
index 5e59d8b77b..76eaf4c448 100755
--- a/ci/scripts/driver.sh
+++ b/ci/scripts/driver.sh
@@ -56,11 +56,12 @@ if [[ "${MACHINE_ID}" == "wcoss2" ]]; then
 fi
 set -x
 unset HOMEgfs
-if not GH=$(command -v gh); then
-   export GH="${HOME}/bin/gh"
+if ! command -v gh > /dev/null; then
+   GH="${HOME}/bin/gh"
 else
-   export GH
+   GH=$(command -v gh)
 fi
+export GH
 
 ############################################################
 # query repo and get list of open PRs with tags {machine}-CI

From aa7d8f97ca74097eaac964e6e49f130f5d2ec603 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 15 Apr 2024 17:42:23 +0000
Subject: [PATCH 16/22] need to load the ci runtime module file for wcoss2 to
 git gh in check script in bash

---
 ci/scripts/check_ci.sh | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/ci/scripts/check_ci.sh b/ci/scripts/check_ci.sh
index 6aaf244a3d..c4deaeea51 100755
--- a/ci/scripts/check_ci.sh
+++ b/ci/scripts/check_ci.sh
@@ -13,13 +13,6 @@ scriptname=$(basename "${BASH_SOURCE[0]}")
 echo "Begin ${scriptname} at $(date -u)" || true
 export PS4='+ $(basename ${BASH_SOURCE})[${LINENO}]'
 
-if ! command -v gh > /dev/null; then
-   GH="${HOME}/bin/gh"
-else
-   GH=$(command -v gh)
-fi
-export GH
-
 REPO_URL=${REPO_URL:-"git@github.com:NOAA-EMC/global-workflow.git"}
 
 #########################################################################
@@ -44,7 +37,19 @@ source "${HOMEgfs}/ci/scripts/utils/ci_utils.sh"
 module use "${HOMEgfs}/modulefiles"
 module load "module_gwsetup.${MACHINE_ID}"
 module list
+# Load machine specific modules for ci (only wcoss2 is current)
+if [[ "${MACHINE_ID}" == "wcoss2" ]]; then
+  module load "module_gwci.${MACHINE_ID}"
+fi
 set -x
+unset HOMEgfs
+if ! command -v gh > /dev/null; then
+   GH="${HOME}/bin/gh"
+else
+   GH=$(command -v gh)
+fi
+export GH
+
 rocotostat=$(command -v rocotostat)
 if [[ -z ${rocotostat+x} ]]; then
   echo "rocotostat not found on system"

From f5322e1396584b200c09f4d4d74bde868fa36c8f Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 15 Apr 2024 17:51:24 +0000
Subject: [PATCH 17/22] was not intended to unset HOMEgfs in the check script

---
 ci/scripts/check_ci.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/ci/scripts/check_ci.sh b/ci/scripts/check_ci.sh
index c4deaeea51..8cf5110d9f 100755
--- a/ci/scripts/check_ci.sh
+++ b/ci/scripts/check_ci.sh
@@ -42,7 +42,6 @@ if [[ "${MACHINE_ID}" == "wcoss2" ]]; then
   module load "module_gwci.${MACHINE_ID}"
 fi
 set -x
-unset HOMEgfs
 if ! command -v gh > /dev/null; then
    GH="${HOME}/bin/gh"
 else

From afa62059e1dbe9cf0b6984a6572014ec02cdbe9d Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 15 Apr 2024 18:01:12 +0000
Subject: [PATCH 18/22] misspelled scheduler varible when checkout for torque
 in bash utils

---
 ci/scripts/utils/ci_utils.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/scripts/utils/ci_utils.sh b/ci/scripts/utils/ci_utils.sh
index ce2e039307..9c0e8ac997 100755
--- a/ci/scripts/utils/ci_utils.sh
+++ b/ci/scripts/utils/ci_utils.sh
@@ -23,7 +23,7 @@ function cancel_batch_jobs() {
 
   scheduler=$(determine_scheduler)
 
-  if [[ "${schduler}" == "torque" ]]; then
+  if [[ "${scheduler}" == "torque" ]]; then
     job_ids=$(qstat -u "${USER}" | awk '{print $1}') || true
 
     for job_id in ${job_ids}; do

From 82db78812e898d6a03783c61f6d7cf672a8b03a2 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 15 Apr 2024 18:04:33 +0000
Subject: [PATCH 19/22] removed gefs case for this PR as it is for bash

---
 ci/cases/pr/C48_S2SWA_gefs.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/ci/cases/pr/C48_S2SWA_gefs.yaml b/ci/cases/pr/C48_S2SWA_gefs.yaml
index 310d0ea615..5f4ea3c36b 100644
--- a/ci/cases/pr/C48_S2SWA_gefs.yaml
+++ b/ci/cases/pr/C48_S2SWA_gefs.yaml
@@ -17,3 +17,5 @@ arguments:
   edate: 2021032312
   yaml: {{ HOMEgfs }}/ci/cases/yamls/gefs_ci_defaults.yaml
 
+  skip_ci_on_hosts:
+  - wcoss2

From ea3d4954aa166764f7bbaa5f1a032bc81d32984e Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 15 Apr 2024 18:37:53 +0000
Subject: [PATCH 20/22] miss indeented skipp on wcoss2

---
 ci/cases/pr/C48_S2SWA_gefs.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/cases/pr/C48_S2SWA_gefs.yaml b/ci/cases/pr/C48_S2SWA_gefs.yaml
index 5f4ea3c36b..a924b416c3 100644
--- a/ci/cases/pr/C48_S2SWA_gefs.yaml
+++ b/ci/cases/pr/C48_S2SWA_gefs.yaml
@@ -17,5 +17,5 @@ arguments:
   edate: 2021032312
   yaml: {{ HOMEgfs }}/ci/cases/yamls/gefs_ci_defaults.yaml
 
-  skip_ci_on_hosts:
+skip_ci_on_hosts:
   - wcoss2

From 8d0c593ad88660744645f5445a1337aa650e5bc7 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 16 Apr 2024 17:19:27 +0000
Subject: [PATCH 21/22] removed casses that needed gdasapp and removed the -u
 switch from the build as for now only wcoss2 will be using the bash ci

---
 ci/cases/pr/C48mx500_3DVarAOWCDA.yaml | 1 +
 ci/cases/pr/C96C48_ufs_hybatmDA.yaml  | 1 +
 ci/cases/pr/C96_atmaerosnowDA.yaml    | 1 +
 ci/scripts/clone-build_ci.sh          | 2 +-
 4 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml b/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml
index d9156e38f3..3356f2030b 100644
--- a/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml
+++ b/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml
@@ -18,6 +18,7 @@ arguments:
   yaml: {{ HOMEgfs }}/ci/cases/yamls/soca_gfs_defaults_ci.yaml
 
 skip_ci_on_hosts:
+  - wcoss2
   - orion
   - hera
   - hercules
diff --git a/ci/cases/pr/C96C48_ufs_hybatmDA.yaml b/ci/cases/pr/C96C48_ufs_hybatmDA.yaml
index da68b0f86c..e8007aab2a 100644
--- a/ci/cases/pr/C96C48_ufs_hybatmDA.yaml
+++ b/ci/cases/pr/C96C48_ufs_hybatmDA.yaml
@@ -21,3 +21,4 @@ skip_ci_on_hosts:
   - hera
   - orion
   - hercules
+  - wcoss2
\ No newline at end of file
diff --git a/ci/cases/pr/C96_atmaerosnowDA.yaml b/ci/cases/pr/C96_atmaerosnowDA.yaml
index 7e22955a37..edde37cbf7 100644
--- a/ci/cases/pr/C96_atmaerosnowDA.yaml
+++ b/ci/cases/pr/C96_atmaerosnowDA.yaml
@@ -19,3 +19,4 @@ arguments:
 skip_ci_on_hosts:
   - orion
   - hercules
+  - wcoss2
diff --git a/ci/scripts/clone-build_ci.sh b/ci/scripts/clone-build_ci.sh
index 798c98bf50..940f9e7393 100755
--- a/ci/scripts/clone-build_ci.sh
+++ b/ci/scripts/clone-build_ci.sh
@@ -74,7 +74,7 @@ set +e
 source "${HOMEgfs}/ush/module-setup.sh"
 export BUILD_JOBS=8
 rm -rf log.build
-./build_all.sh -gu  >> log.build 2>&1
+./build_all.sh -g  >> log.build 2>&1
 build_status=$?
 
 DATE=$(date +'%D %r')

From b98c66a192f61495648f034e6fd15f4499a9de8c Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terrence.mcguinness@cox.net>
Date: Tue, 16 Apr 2024 13:26:17 -0400
Subject: [PATCH 22/22] Update C96C48_ufs_hybatmDA.yaml

trailing space at end of file
---
 ci/cases/pr/C96C48_ufs_hybatmDA.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ci/cases/pr/C96C48_ufs_hybatmDA.yaml b/ci/cases/pr/C96C48_ufs_hybatmDA.yaml
index e8007aab2a..1b57be63eb 100644
--- a/ci/cases/pr/C96C48_ufs_hybatmDA.yaml
+++ b/ci/cases/pr/C96C48_ufs_hybatmDA.yaml
@@ -21,4 +21,5 @@ skip_ci_on_hosts:
   - hera
   - orion
   - hercules
-  - wcoss2
\ No newline at end of file
+  - wcoss2
+