[develop] Update modulefiles to use hdf/1.14.0 and netcdf/4.9.2-based…

… software stacks on Tier 1 systems (#889) * Update SRW modulefiles to use hpc-stacks with higher versions of software modules on Tier 1 platforms, similar to currently used for UFS-WM. * Update UFS_SRW_data location on Orion to use new dedicated role-epic space --------- Co-authored-by: Natalie Perlin <[email protected]> Co-authored-by: Natalie Perlin <[email protected]>
ufs-community · Sep 15, 2023 · c00ca73 · c00ca73
1 parent 8d5c355
commit c00ca73
Show file tree

Hide file tree

Showing 14 changed files with 41 additions and 42 deletions.
diff --git a/modulefiles/build_cheyenne_gnu.lua b/modulefiles/build_cheyenne_gnu.lua
@@ -14,19 +14,13 @@ setenv("MKLROOT", "/glade/u/apps/opt/intel/2022.1/mkl/latest")
 load(pathJoin("ncarcompilers", os.getenv("ncarcompilers_ver") or "0.5.0"))
 unload("netcdf")
 
-prepend_path("MODULEPATH","/glade/work/epicufsrt/contrib/hpc-stack/gnu11.2.0/modulefiles/stack")
+prepend_path("MODULEPATH","/glade/work/epicufsrt/contrib/hpc-stack/gnu11.2.0_ncdf492/modulefiles/stack")
 load(pathJoin("hpc", os.getenv("hpc_ver") or "1.2.0"))
 load(pathJoin("hpc-gnu", os.getenv("hpc_gnu_ver") or "11.2.0"))
 load(pathJoin("hpc-mpt", os.getenv("hpc_mpt_ver") or "2.25"))
 
 load("srw_common")
 
-load(pathJoin("g2", os.getenv("g2_ver") or "3.4.5"))
-load(pathJoin("esmf", os.getenv("esmf_ver") or "8.3.0b09"))
-load(pathJoin("netcdf", os.getenv("netcdf_ver") or "4.7.4"))
-load(pathJoin("libpng", os.getenv("libpng_ver") or "1.6.37"))
-load(pathJoin("pio", os.getenv("pio_ver") or "2.5.7"))
-load(pathJoin("fms", os.getenv("fms_ver") or "2022.04"))
 load(pathJoin("openblas", os.getenv("openblas_ver") or "0.3.23"))
 
 unsetenv("MKLROOT")

diff --git a/modulefiles/build_cheyenne_intel.lua b/modulefiles/build_cheyenne_intel.lua
@@ -14,20 +14,13 @@ load(pathJoin("python", os.getenv("python_ver") or "3.7.9"))
 load(pathJoin("ncarcompilers", os.getenv("ncarcompilers_ver") or "0.5.0"))
 unload("netcdf")
 
-prepend_path("MODULEPATH","/glade/work/epicufsrt/contrib/hpc-stack/intel2022.1/modulefiles/stack")
+prepend_path("MODULEPATH","/glade/work/epicufsrt/contrib/hpc-stack/intel2022.1_ncdf492/modulefiles/stack")
 load(pathJoin("hpc", os.getenv("hpc_ver") or "1.2.0"))
 load(pathJoin("hpc-intel", os.getenv("hpc_intel_ver") or "2022.1"))
 load(pathJoin("hpc-mpt", os.getenv("hpc_mpt_ver") or "2.25"))
 
 load("srw_common")
 
-load(pathJoin("g2", os.getenv("g2_ver") or "3.4.5"))
-load(pathJoin("esmf", os.getenv("esmf_ver") or "8.3.0b09"))
-load(pathJoin("netcdf", os.getenv("netcdf_ver") or "4.7.4"))
-load(pathJoin("libpng", os.getenv("libpng_ver") or "1.6.37"))
-load(pathJoin("pio", os.getenv("pio_ver") or "2.5.7"))
-load(pathJoin("fms", os.getenv("fms_ver") or "2022.04"))
-
 setenv("CMAKE_C_COMPILER","mpicc")
 setenv("CMAKE_CXX_COMPILER","mpicpc")
 setenv("CMAKE_Fortran_COMPILER","mpif90")

diff --git a/modulefiles/build_gaea_intel.lua b/modulefiles/build_gaea_intel.lua
@@ -7,9 +7,9 @@ whatis([===[Loads libraries needed for building the UFS SRW App on Gaea ]===])
 
 load(pathJoin("cmake", os.getenv("cmake_ver") or "3.20.1"))
 
-prepend_path("MODULEPATH","/lustre/f2/dev/role.epic/contrib/hpc-stack/intel-classic-2022.0.2/modulefiles/stack")
+prepend_path("MODULEPATH","/lustre/f2/dev/role.epic/contrib/hpc-stack/intel-classic-2023.1.0/modulefiles/stack")
 load(pathJoin("hpc", os.getenv("hpc_ver") or "1.2.0"))
-load(pathJoin("hpc-intel-classic", os.getenv("hpc_intel_classic_ver") or "2022.0.2"))
+load(pathJoin("hpc-intel-classic", os.getenv("hpc_intel_classic_ver") or "2023.1.0"))
 load(pathJoin("hpc-cray-mpich", os.getenv("hpc_cray_mpich_ver") or "7.7.20"))
 
 load("srw_common")
@@ -24,3 +24,7 @@ setenv("CMAKE_CXX_COMPILER","CC")
 setenv("CMAKE_Fortran_COMPILER","ftn")
 setenv("CMAKE_Platform","gaea.intel")
 
+setenv("CFLAGS","-diag-disable=10441")
+setenv("FFLAGS","-diag-disable=10441 -fp-model source")
+
+
diff --git a/modulefiles/build_hera_gnu.lua b/modulefiles/build_hera_gnu.lua
@@ -13,16 +13,16 @@ load(pathJoin("cmake", os.getenv("cmake_ver") or "3.20.1"))
 gnu_ver=os.getenv("gnu_ver") or "9.2.0"
 load(pathJoin("gnu", gnu_ver))
 
-prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/hpc-stack/libs/gnu-9.2/modulefiles/stack")
+prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/hpc-stack/libs/gnu-9.2_ncdf492/modulefiles/stack")
 
 load(pathJoin("hpc", os.getenv("hpc_ver") or "1.2.0"))
 load(pathJoin("hpc-gnu", os.getenv("hpc-gnu_ver") or "9.2"))
 load(pathJoin("hpc-mpich", os.getenv("hpc-mpich_ver") or "3.3.2"))
 
 load("srw_common")
 
-load(pathJoin("nccmp", os.getenv("nccmp_ver") or "1.8.9"))
-load(pathJoin("nco", os.getenv("nco_ver") or "4.9.3"))
+load(pathJoin("nccmp", os.getenv("nccmp_ver") or "1.9.1.0"))
+load(pathJoin("nco", os.getenv("nco_ver") or "5.0.6"))
 load(pathJoin("openblas", os.getenv("openblas_ver") or "0.3.23"))
 
 unsetenv("MKLROOT")

diff --git a/modulefiles/build_hera_intel.lua b/modulefiles/build_hera_intel.lua
@@ -16,7 +16,7 @@ load(pathJoin("intel", intel_ver))
 impi_ver=os.getenv("impi_ver") or "2022.1.2"
 load(pathJoin("impi", impi_ver))
 
-prepend_path("MODULEPATH","/scratch1/NCEPDEV/nems/role.epic/hpc-stack/libs/intel-2022.1.2/modulefiles/stack")
+prepend_path("MODULEPATH","/scratch1/NCEPDEV/nems/role.epic/hpc-stack/libs/intel-2022.1.2_ncdf492/modulefiles/stack")
 
 load(pathJoin("hpc", os.getenv("hpc_ver") or "1.2.0"))
 load(pathJoin("hpc-intel", os.getenv("hpc_intel_ver") or "2022.1.2"))
@@ -25,7 +25,7 @@ load(pathJoin("hpc-impi", os.getenv("hpc_impi_ver") or "2022.1.2"))
 load("srw_common")
 
 load(pathJoin("nccmp", os.getenv("nccmp_ver") or "1.8.9.0"))
-load(pathJoin("nco", os.getenv("nco_ver") or "4.9.3"))
+load(pathJoin("nco", os.getenv("nco_ver") or "5.0.6"))
 
 setenv("CMAKE_C_COMPILER","mpiicc")
 setenv("CMAKE_CXX_COMPILER","mpiicpc")

diff --git a/modulefiles/build_jet_intel.lua b/modulefiles/build_jet_intel.lua
@@ -10,7 +10,7 @@ load("sutils")
 
 load(pathJoin("cmake", os.getenv("cmake_ver") or "3.20.1"))
 
-prepend_path("MODULEPATH","/mnt/lfs4/HFIP/hfv3gfs/role.epic/hpc-stack/libs/intel-2022.1.2/modulefiles/stack")
+prepend_path("MODULEPATH","/mnt/lfs4/HFIP/hfv3gfs/role.epic/hpc-stack/libs/intel-2022.1.2_ncdf492/modulefiles/stack")
 load(pathJoin("hpc", os.getenv("hpc_ver") or "1.2.0"))
 load(pathJoin("hpc-intel", os.getenv("hpc_intel_ver") or "2022.1.2"))
 load(pathJoin("hpc-impi", os.getenv("hpc_impi_ver") or "2022.1.2"))

diff --git a/modulefiles/build_orion_intel.lua b/modulefiles/build_orion_intel.lua
@@ -11,15 +11,15 @@ load("noaatools")
 load(pathJoin("cmake", os.getenv("cmake_ver") or "3.22.1"))
 load(pathJoin("python", os.getenv("python_ver") or "3.9.2"))
 
-prepend_path("MODULEPATH","/work/noaa/epic/role-epic/contrib/orion/hpc-stack/intel-2022.1.2/modulefiles/stack")
+prepend_path("MODULEPATH","/work/noaa/epic/role-epic/contrib/orion/hpc-stack/intel-2022.1.2_ncdf492/modulefiles/stack")
 load(pathJoin("hpc", os.getenv("hpc_ver") or "1.2.0"))
 load(pathJoin("hpc-intel", os.getenv("hpc_intel_ver") or "2022.1.2"))
 load(pathJoin("hpc-impi", os.getenv("hpc_impi_ver") or "2022.1.2"))
 
 load("srw_common")
 
 load(pathJoin("nccmp", os.getenv("nccmp_ver") or "1.8.9.0"))
-load(pathJoin("nco", os.getenv("nco_ver") or "4.9.3"))
+load(pathJoin("nco", os.getenv("nco_ver") or "5.0.6"))
 
 setenv("CMAKE_C_COMPILER","mpiicc")
 setenv("CMAKE_CXX_COMPILER","mpiicpc")

diff --git a/modulefiles/srw_common.lua b/modulefiles/srw_common.lua
@@ -1,12 +1,12 @@
-load("jasper/2.0.25")
-load("zlib/1.2.11")
-load_any("png/1.6.35", "libpng/1.6.37")
+load_any("jasper/2.0.25","jasper/2.0.32")
+load_any("zlib/1.2.11","zlib/1.2.13")
+load("libpng/1.6.37")
 
-load_any("netcdf/4.7.4", "netcdf-c/4.7.4")
-load_any("netcdf/4.7.4", "netcdf-fortran/4.5.4")
-load_any("pio/2.5.7", "parallelio/2.5.2")
-load_any("esmf/8.3.0b09", "esmf/8.2.0")
-load("fms/2022.04")
+load_any("netcdf/4.9.2", "netcdf-c/4.9.2")
+load_any("netcdf/4.9.2", "netcdf-fortran/4.6.0")
+load_any("pio/2.5.10","parallelio/2.5.10")
+load("esmf/8.4.2")
+load("fms/2023.01")
 
 load("bacio/2.4.1")
 load("crtm/2.4.0")
@@ -18,12 +18,12 @@ load("w3emc/2.9.2")
 
 load_any("gftl-shared/v1.5.0", "gftl-shared/1.5.0")
 load_any("yafyaml/v0.5.1", "yafyaml/0.5.1")
-load_any("mapl/2.22.0-esmf-8.3.0b09", "mapl/2.11.0-esmf-8.2.0")
+load("mapl/2.35.2-esmf-8.4.2")
 
 load("nemsio/2.5.4")
 load("sfcio/1.4.1")
 load("sigio/2.3.2")
 load("w3nco/2.4.1")
-load("wrf_io/1.2.0")
+load_any("wrf_io/1.2.0","wrf-io/1.2.0")
 
 load("wgrib2/2.0.8")
diff --git a/modulefiles/tasks/orion/miniconda_regional_workflow_cmaq.lua b/modulefiles/tasks/orion/miniconda_regional_workflow_cmaq.lua
@@ -1,4 +1,4 @@
-prepend_path("MODULEPATH","/work/noaa/epic-ps/role-epic-ps/miniconda3/modulefiles")
+prepend_path("MODULEPATH","/work/noaa/epic/role-epic/contrib/orion/miniconda3/modulefiles")
 load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0"))
 
 setenv("SRW_ENV", "regional_workflow_cmaq")
diff --git a/modulefiles/tasks/orion/plot_allvars.local.lua b/modulefiles/tasks/orion/plot_allvars.local.lua
@@ -1,5 +1,5 @@
 unload("python")
-append_path("MODULEPATH","/work/noaa/epic-ps/role-epic-ps/miniconda3/modulefiles")
+append_path("MODULEPATH","/work/noaa/epic/role-epic/contrib/orion/miniconda3/modulefiles")
 load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0"))
 
 setenv("SRW_ENV", "regional_workflow")
diff --git a/modulefiles/wflow_gaea.lua b/modulefiles/wflow_gaea.lua
@@ -13,7 +13,9 @@ prepend_path("MODULEPATH","/lustre/f2/dev/role.epic/contrib/rocoto/modulefiles")
 load("rocoto")
 load("alps")
 
-pushenv("MKLROOT", "/opt/intel/oneapi/mkl/2022.0.2/")
+local MKLROOT="/opt/intel/oneapi/mkl/2023.1.0/"
+prepend_path("LD_LIBRARY_PATH",pathJoin(MKLROOT,"lib/intel64"))
+pushenv("MKLROOT", MKLROOT)
 pushenv("GSI_BINARY_SOURCE_DIR", "/lustre/f2/dev/role.epic/contrib/GSI_data/fix/20230601")
 setenv("PMI_NO_PREINITIALIZE","1")
 

diff --git a/parm/wflow/coldstart.yaml b/parm/wflow/coldstart.yaml
@@ -22,6 +22,9 @@ default_task: &default_task
 task_get_extrn_ics:
   <<: *default_task
   command: '&LOAD_MODULES_RUN_TASK_FP; "get_extrn_ics" "&JOBSdir;/JREGIONAL_GET_EXTRN_MDL_FILES"'
+  attrs:
+    cycledefs: forecast
+    maxtries: '2'
   envars:
     <<: *default_vars
     ICS_OR_LBCS: ICS
@@ -32,7 +35,7 @@ task_get_extrn_ics:
   partition: '{% if platform.get("PARTITION_HPSS") %}&PARTITION_HPSS;{% else %}None{% endif %}'
   ppn: 1
   queue: '&QUEUE_HPSS;'
-  walltime: 00:45:00
+  walltime: 01:30:00
   dependency:
     or:
       and:
@@ -50,6 +53,9 @@ task_get_extrn_ics:
 task_get_extrn_lbcs:
   <<: *default_task
   command: '&LOAD_MODULES_RUN_TASK_FP; "get_extrn_lbcs" "&JOBSdir;/JREGIONAL_GET_EXTRN_MDL_FILES"'
+  attrs:
+    cycledefs: forecast
+    maxtries: '2'
   envars:
     <<: *default_vars
     ICS_OR_LBCS: LBCS
@@ -60,7 +66,7 @@ task_get_extrn_lbcs:
   partition: '{% if platform.get("PARTITION_HPSS") %}&PARTITION_HPSS;{% else %}None{% endif %}'
   ppn: 1
   queue: '&QUEUE_HPSS;'
-  walltime: 00:45:00
+  walltime: 01:30:00
   dependency:
     or:
       and:

diff --git a/ush/machine/jet.yaml b/ush/machine/jet.yaml
@@ -8,7 +8,7 @@ platform:
   DOMAIN_PREGEN_BASEDIR: /mnt/lfs4/HFIP/hfv3gfs/role.epic/UFS_SRW_data/develop/FV3LAM_pregen
   PARTITION_DEFAULT: sjet,vjet,kjet,xjet
   QUEUE_DEFAULT: batch
-  PARTITION_FCST: sjet,vjet,kjet,xjet
+  PARTITION_FCST: xjet
   QUEUE_FCST: batch
   PARTITION_HPSS: service
   QUEUE_HPSS: batch

diff --git a/ush/machine/orion.yaml b/ush/machine/orion.yaml
@@ -24,7 +24,7 @@ platform:
   TEST_EXTRN_MDL_SOURCE_BASEDIR: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/input_model_data
   TEST_AQM_INPUT_BASEDIR: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/aqm_data
   TEST_PREGEN_BASEDIR: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/FV3LAM_pregen
-  TEST_VX_FCST_INPUT_BASEDIR: '{{ "/work/noaa/epic-ps/role-epic-ps/UFS_SRW_data/develop/output_data/fcst_" }}{{ "ens" if (global.NUM_ENS_MEMBERS > 0) else "det" }}{{ "/{{workflow.PREDEF_GRID_NAME}}" }}{% raw %}{% endraw %}'
+  TEST_VX_FCST_INPUT_BASEDIR: '{{ "/work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/output_data/fcst_" }}{{ "ens" if (global.NUM_ENS_MEMBERS > 0) else "det" }}{{ "/{{workflow.PREDEF_GRID_NAME}}" }}{% raw %}{% endraw %}'
   FIXaer: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/fix/fix_aer
   FIXgsm: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/fix/fix_am
   FIXlut: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/fix/fix_lut