From 2eb3f12103a72a672ddd77286256e6e68e6e9210 Mon Sep 17 00:00:00 2001 From: RatkoVasic-NOAA <37597874+RatkoVasic-NOAA@users.noreply.github.com> Date: Tue, 20 Feb 2024 09:17:36 -0500 Subject: [PATCH 01/11] Update cmake version --- modulefiles/build_hera_intel.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modulefiles/build_hera_intel.lua b/modulefiles/build_hera_intel.lua index 500d410dc5..3edc20b75a 100644 --- a/modulefiles/build_hera_intel.lua +++ b/modulefiles/build_hera_intel.lua @@ -20,7 +20,7 @@ load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) stack_python_ver=os.getenv("stack_python_ver") or "3.10.8" load(pathJoin("stack-python", stack_python_ver)) -cmake_ver=os.getenv("cmake_ver") or "3.20.1" +cmake_ver=os.getenv("cmake_ver") or "3.23.1" load(pathJoin("cmake", cmake_ver)) load("srw_common") From 527538cd8826d0667aae4b7c01ebd694b368b7b5 Mon Sep 17 00:00:00 2001 From: RatkoVasic-NOAA <37597874+RatkoVasic-NOAA@users.noreply.github.com> Date: Tue, 20 Feb 2024 14:23:29 -0500 Subject: [PATCH 02/11] Increase memory for HPSS Using new slurm and rocoto, some HPSS jobs were failing due to lack of memory. Increased from 2GB to 4GB. --- parm/wflow/coldstart.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/parm/wflow/coldstart.yaml b/parm/wflow/coldstart.yaml index e707b51d92..002d7f7b96 100644 --- a/parm/wflow/coldstart.yaml +++ b/parm/wflow/coldstart.yaml @@ -28,7 +28,7 @@ task_get_extrn_ics: <<: *default_vars ICS_OR_LBCS: ICS join: !cycstr '&LOGDIR;/{{ jobname }}_@Y@m@d@H&LOGEXT;' - memory: 2G + memory: 4G nnodes: 1 native: '{% if platform.get("SCHED_NATIVE_CMD_HPSS") %}{{ platform.SCHED_NATIVE_CMD_HPSS }}{% else %}{{ platform.SCHED_NATIVE_CMD}}{% endif %}' partition: '{% if platform.get("PARTITION_HPSS") %}&PARTITION_HPSS;{% else %}None{% endif %}' @@ -59,7 +59,7 @@ task_get_extrn_lbcs: <<: *default_vars ICS_OR_LBCS: LBCS join: !cycstr '&LOGDIR;/{{ jobname }}_@Y@m@d@H&LOGEXT;' - memory: 2G + memory: 4G nnodes: 1 native: '{% if platform.get("SCHED_NATIVE_CMD_HPSS") %}{{ platform.SCHED_NATIVE_CMD_HPSS }}{% else %}{{ platform.SCHED_NATIVE_CMD}}{% endif %}' partition: '{% if platform.get("PARTITION_HPSS") %}&PARTITION_HPSS;{% else %}None{% endif %}' From edf5e384aaf1bba3b4309e35e8ede99c76d6b0e7 Mon Sep 17 00:00:00 2001 From: RatkoVasic-NOAA <37597874+RatkoVasic-NOAA@users.noreply.github.com> Date: Wed, 21 Feb 2024 12:04:54 -0500 Subject: [PATCH 03/11] Fix in exregional_make_lbcs.sh --- scripts/exregional_make_lbcs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/exregional_make_lbcs.sh b/scripts/exregional_make_lbcs.sh index 72f9369ff6..695af1b409 100755 --- a/scripts/exregional_make_lbcs.sh +++ b/scripts/exregional_make_lbcs.sh @@ -410,7 +410,7 @@ for (( ii=0; ii<${num_fhrs}; ii=ii+bcgrpnum10 )); do fi ;; "GDAS") - fn_atm="${EXTRN_MDL_FNS[0][$i]}" + fn_atm="${EXTRN_MDL_FNS[$i]}" ;; "GEFS") fn_grib2="${EXTRN_MDL_FNS[$i]}" From 8799cb5359a2d701bb3f1303ceedc2039c267a0f Mon Sep 17 00:00:00 2001 From: RatkoVasic-NOAA <37597874+RatkoVasic-NOAA@users.noreply.github.com> Date: Wed, 21 Feb 2024 15:05:12 -0500 Subject: [PATCH 04/11] Update path to spack-stack on Rocky OS --- modulefiles/build_hera_intel.lua | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modulefiles/build_hera_intel.lua b/modulefiles/build_hera_intel.lua index 3edc20b75a..339711f40d 100644 --- a/modulefiles/build_hera_intel.lua +++ b/modulefiles/build_hera_intel.lua @@ -8,6 +8,9 @@ whatis([===[Loads libraries needed for building the UFS SRW App on Hera ]===]) prepend_path("MODULEPATH","/contrib/sutils/modulefiles") load("sutils") +-- When Hera switces from CentOS to Rocky, replace line withh correct path to spack-stack +-- If you want to use Rocky OS now, use line below +--prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env-rocky8/install/modulefiles/Core") prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env-noavx512/install/modulefiles/Core") prepend_path("MODULEPATH", "/scratch1/NCEPDEV/jcsda/jedipara/spack-stack/modulefiles") From a86977ab3ace1eba60c41495e4b792c650d296ec Mon Sep 17 00:00:00 2001 From: RatkoVasic-NOAA <37597874+RatkoVasic-NOAA@users.noreply.github.com> Date: Wed, 21 Feb 2024 15:08:22 -0500 Subject: [PATCH 05/11] Update spack-stack path on Rocky OS --- modulefiles/build_hera_gnu.lua | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modulefiles/build_hera_gnu.lua b/modulefiles/build_hera_gnu.lua index c1f57e2115..722910dc9a 100644 --- a/modulefiles/build_hera_gnu.lua +++ b/modulefiles/build_hera_gnu.lua @@ -5,6 +5,9 @@ the NOAA RDHPC machine Hera using GNU 9.2.0 whatis([===[Loads libraries needed for building the UFS SRW App on Hera using GNU 9.2.0 ]===]) +-- When Hera switces from CentOS to Rocky, replace line withh correct path to spack-stack +-- If you want to use Rocky OS now, use line below +--prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env-rocky8/install/modulefiles/Core") prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env-noavx512/install/modulefiles/Core") prepend_path("MODULEPATH", "/scratch1/NCEPDEV/jcsda/jedipara/spack-stack/modulefiles") From cd5bb4d0574f91a023421598bb769ab5a54852b7 Mon Sep 17 00:00:00 2001 From: RatkoVasic-NOAA <37597874+RatkoVasic-NOAA@users.noreply.github.com> Date: Thu, 22 Feb 2024 13:05:15 -0500 Subject: [PATCH 06/11] Correct misspelling --- modulefiles/build_hera_intel.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modulefiles/build_hera_intel.lua b/modulefiles/build_hera_intel.lua index 339711f40d..314fd89183 100644 --- a/modulefiles/build_hera_intel.lua +++ b/modulefiles/build_hera_intel.lua @@ -8,7 +8,7 @@ whatis([===[Loads libraries needed for building the UFS SRW App on Hera ]===]) prepend_path("MODULEPATH","/contrib/sutils/modulefiles") load("sutils") --- When Hera switces from CentOS to Rocky, replace line withh correct path to spack-stack +-- When Hera switches from CentOS to Rocky, replace line withh correct path to spack-stack -- If you want to use Rocky OS now, use line below --prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env-rocky8/install/modulefiles/Core") prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env-noavx512/install/modulefiles/Core") From c038dba5f27455d247712501acebec81fe713e06 Mon Sep 17 00:00:00 2001 From: RatkoVasic-NOAA <37597874+RatkoVasic-NOAA@users.noreply.github.com> Date: Thu, 22 Feb 2024 13:05:48 -0500 Subject: [PATCH 07/11] Corrected misspelling --- modulefiles/build_hera_gnu.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modulefiles/build_hera_gnu.lua b/modulefiles/build_hera_gnu.lua index 722910dc9a..90bd671b5a 100644 --- a/modulefiles/build_hera_gnu.lua +++ b/modulefiles/build_hera_gnu.lua @@ -5,7 +5,7 @@ the NOAA RDHPC machine Hera using GNU 9.2.0 whatis([===[Loads libraries needed for building the UFS SRW App on Hera using GNU 9.2.0 ]===]) --- When Hera switces from CentOS to Rocky, replace line withh correct path to spack-stack +-- When Hera switches from CentOS to Rocky, replace line withh correct path to spack-stack -- If you want to use Rocky OS now, use line below --prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env-rocky8/install/modulefiles/Core") prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env-noavx512/install/modulefiles/Core") From b7fd251ef5c91737ce935b4448a64f85eaf54b08 Mon Sep 17 00:00:00 2001 From: RatkoVasic-NOAA <37597874+RatkoVasic-NOAA@users.noreply.github.com> Date: Thu, 22 Feb 2024 13:29:57 -0500 Subject: [PATCH 08/11] Replace xJet with vJet --- ush/machine/jet.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/machine/jet.yaml b/ush/machine/jet.yaml index 93d375ee02..bef698f874 100644 --- a/ush/machine/jet.yaml +++ b/ush/machine/jet.yaml @@ -9,7 +9,7 @@ platform: DOMAIN_PREGEN_BASEDIR: /mnt/lfs4/HFIP/hfv3gfs/role.epic/UFS_SRW_data/develop/FV3LAM_pregen PARTITION_DEFAULT: sjet,vjet,kjet,xjet QUEUE_DEFAULT: batch - PARTITION_FCST: xjet + PARTITION_FCST: vjet QUEUE_FCST: batch PARTITION_HPSS: service QUEUE_HPSS: batch From 2ccb09a4c56dc5a88422e1752500d581016f242c Mon Sep 17 00:00:00 2001 From: Ratko Vasic Date: Fri, 23 Feb 2024 17:38:01 +0000 Subject: [PATCH 09/11] Modulefile changes for OS switch on Jet (CentOS -> Rocky) --- modulefiles/build_jet_intel.lua | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modulefiles/build_jet_intel.lua b/modulefiles/build_jet_intel.lua index e687531ac8..9c52cac497 100644 --- a/modulefiles/build_jet_intel.lua +++ b/modulefiles/build_jet_intel.lua @@ -5,6 +5,9 @@ the NOAA RDHPC machine Jet using Intel-2021.5.0 whatis([===[Loads libraries needed for building the UFS SRW App on Jet ]===]) +-- When Jet switches from CentOS to Rocky, replace line withh correct path to spack-stack +-- If you want to use Rocky OS now, use line below +--prepend_path("MODULEPATH","/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env-rocky8/install/modulefiles/Core") prepend_path("MODULEPATH","/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core") prepend_path("MODULEPATH", "/lfs4/HFIP/hfv3gfs/spack-stack/modulefiles") @@ -15,7 +18,6 @@ load("cmake/3.23.1") load("srw_common") -load("prod-util/1.2.2") load("nccmp/1.9.0.1") load("nco/5.0.6") From c33c8f34d91db6cc077bf2e3af2880e850015d56 Mon Sep 17 00:00:00 2001 From: RatkoVasic-NOAA <37597874+RatkoVasic-NOAA@users.noreply.github.com> Date: Fri, 1 Mar 2024 14:59:48 -0500 Subject: [PATCH 10/11] Add description of usage Rocky OS on Jet --- modulefiles/build_jet_intel.lua | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/modulefiles/build_jet_intel.lua b/modulefiles/build_jet_intel.lua index 9c52cac497..5e46f0dc93 100644 --- a/modulefiles/build_jet_intel.lua +++ b/modulefiles/build_jet_intel.lua @@ -6,7 +6,10 @@ the NOAA RDHPC machine Jet using Intel-2021.5.0 whatis([===[Loads libraries needed for building the UFS SRW App on Jet ]===]) -- When Jet switches from CentOS to Rocky, replace line withh correct path to spack-stack --- If you want to use Rocky OS now, use line below +-- If you want to use Rocky OS now on Jet: +-- 1. login to one of Rocky Jet nodes: fe5 - fe8 +-- 2. uncomment line to Rocky spack-stack, and comment current line with path to spack-stack +-- 3. in file ush/machine/jet.yaml set PARTITION_FCST=xjet --prepend_path("MODULEPATH","/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env-rocky8/install/modulefiles/Core") prepend_path("MODULEPATH","/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core") prepend_path("MODULEPATH", "/lfs4/HFIP/hfv3gfs/spack-stack/modulefiles") From 64f00d6b60d3e2ff589d74a0cd40a87e852b2cab Mon Sep 17 00:00:00 2001 From: "michael.lueken" Date: Tue, 12 Mar 2024 14:39:04 +0000 Subject: [PATCH 11/11] [Jet-Rocky8] Update path to use rocky8 spack-stack in build_jet_intel.lua and set PARTITION_FCST=xjet in ush/machine/jet.yaml --- modulefiles/build_jet_intel.lua | 8 +------- ush/machine/jet.yaml | 2 +- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/modulefiles/build_jet_intel.lua b/modulefiles/build_jet_intel.lua index 5e46f0dc93..eb2893d3cc 100644 --- a/modulefiles/build_jet_intel.lua +++ b/modulefiles/build_jet_intel.lua @@ -5,13 +5,7 @@ the NOAA RDHPC machine Jet using Intel-2021.5.0 whatis([===[Loads libraries needed for building the UFS SRW App on Jet ]===]) --- When Jet switches from CentOS to Rocky, replace line withh correct path to spack-stack --- If you want to use Rocky OS now on Jet: --- 1. login to one of Rocky Jet nodes: fe5 - fe8 --- 2. uncomment line to Rocky spack-stack, and comment current line with path to spack-stack --- 3. in file ush/machine/jet.yaml set PARTITION_FCST=xjet ---prepend_path("MODULEPATH","/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env-rocky8/install/modulefiles/Core") -prepend_path("MODULEPATH","/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core") +prepend_path("MODULEPATH","/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env-rocky8/install/modulefiles/Core") prepend_path("MODULEPATH", "/lfs4/HFIP/hfv3gfs/spack-stack/modulefiles") load("stack-intel/2021.5.0") diff --git a/ush/machine/jet.yaml b/ush/machine/jet.yaml index bef698f874..93d375ee02 100644 --- a/ush/machine/jet.yaml +++ b/ush/machine/jet.yaml @@ -9,7 +9,7 @@ platform: DOMAIN_PREGEN_BASEDIR: /mnt/lfs4/HFIP/hfv3gfs/role.epic/UFS_SRW_data/develop/FV3LAM_pregen PARTITION_DEFAULT: sjet,vjet,kjet,xjet QUEUE_DEFAULT: batch - PARTITION_FCST: vjet + PARTITION_FCST: xjet QUEUE_FCST: batch PARTITION_HPSS: service QUEUE_HPSS: batch