diff --git a/ci/rt.sh b/ci/rt.sh index cf839e660..57cea7942 100755 --- a/ci/rt.sh +++ b/ci/rt.sh @@ -65,7 +65,7 @@ if [ $mac2 = hf ]; then # for HERA export machine=HERA export homedir=${homedir:-"/scratch2/NAGAPE/epic/UPP/test_suite"} export rundir=${rundir:-"/scratch1/NCEPDEV/stmp2/${USER}"} - module use /scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core + module use /scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/unified-env-rocky8/install/modulefiles/Core module load stack-intel/2021.5.0 module load stack-intel-oneapi-mpi/2021.5.1 module load prod_util/2.1.1 @@ -85,6 +85,7 @@ elif [ $mac3 = herc ] ; then module load stack-intel/2021.9.0 module load stack-intel-oneapi-mpi/2021.9.0 module load prod_util/2.1.1 + module load stack-python/3.10.13 fi #set working directory diff --git a/modulefiles/gaea-c5.lua b/modulefiles/gaea-c5.lua deleted file mode 100644 index 09c7e6593..000000000 --- a/modulefiles/gaea-c5.lua +++ /dev/null @@ -1,58 +0,0 @@ -help([[ - This module loads libraries required for building and running UPP - on the NOAA RDHPC machine Gaea C5 using Intel-2023.1.0. -]]) - -whatis([===[Loads libraries needed for building the UPP on Gaea ]===]) - -load("PrgEnv-intel/8.3.3") -load("intel-classic/2023.1.0") -load("cray-mpich/8.1.25") -load("python/3.9.12") - -prepend_path("MODULEPATH", "/lustre/f2/dev/wpo/role.epic/contrib/spack-stack/c5/spack-stack-dev-20230717/envs/unified-env/install/modulefiles/Core") -prepend_path("MODULEPATH", "/lustre/f2/dev/wpo/role.epic/contrib/spack-stack/c5/modulefiles") - -stack_intel_ver=os.getenv("stack_intel_ver") or "2023.1.0" -load(pathJoin("stack-intel", stack_intel_ver)) - -stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "8.1.25" -load(pathJoin("stack-cray-mpich", stack_cray_mpich_ver)) - -stack_python_ver=os.getenv("stack_python_ver") or "3.9.12" -load(pathJoin("stack-python", stack_python_ver)) - -local ufs_modules = { - {["jasper"] = "2.0.32"}, - {["zlib"] = "1.2.13"}, - {["libpng"] = "1.6.37"}, - {["hdf5"] = "1.14.0"}, - {["netcdf-c"] = "4.9.2"}, - {["netcdf-fortran"] = "4.6.0"}, - {["bacio"] = "2.4.1"}, - {["crtm"] = "2.4.0"}, - {["g2"] = "3.4.5"}, - {["g2tmpl"] = "1.10.2"}, - {["ip"] = "3.3.3"}, - {["nemsio"] = "2.5.4"}, - {["sfcio"] = "1.4.1"}, - {["sigio"] = "2.3.2"}, - {["sp"] = "2.3.3"}, - {["w3emc"] = "2.9.2"}, - {["wrf-io"] = "1.2.0"}, -} - -for i = 1, #ufs_modules do - for name, default_version in pairs(ufs_modules[i]) do - local env_version_name = string.gsub(name, "-", "_") .. "_ver" - load(pathJoin(name, os.getenv(env_version_name) or default_version)) - end -end - -unload("darshan-runtime") -unload("cray-libsci") - -setenv("CC","cc") -setenv("CXX","CC") -setenv("FC","ftn") -setenv("CMAKE_Platform","gaea_c5.intel") diff --git a/modulefiles/gaea.lua b/modulefiles/gaea.lua index 7a87da323..0e610cfd7 100644 --- a/modulefiles/gaea.lua +++ b/modulefiles/gaea.lua @@ -1,56 +1,34 @@ help([[ This module loads libraries required for building and running UPP - on the NOAA RDHPC machine Gaea using Intel-2022.0.2 + on the NOAA RDHPC machine Gaea C5 using Intel-2023.1.0. ]]) whatis([===[Loads libraries needed for building the UPP on Gaea ]===]) -load_any(pathJoin("cmake", os.getenv("cmake_ver") or "3.20.1"),"cmake") - -prepend_path("MODULEPATH","/lustre/f2/dev/role.epic/contrib/hpc-stack/intel-classic-2022.0.2/modulefiles/stack") -load(pathJoin("hpc", os.getenv("hpc_ver") or "1.2.0")) - -load(pathJoin("intel-classic", os.getenv("intel_classic_ver") or "2022.0.2")) -load(pathJoin("cray-mpich", os.getenv("cray_mpich_ver") or "7.7.20")) -load(pathJoin("hpc-intel-classic", os.getenv("hpc_intel_classic_ver") or "2022.0.2")) -load(pathJoin("hpc-cray-mpich", os.getenv("hpc_cray_mpich_ver") or "7.7.20")) -load(pathJoin("libpng", os.getenv("libpng_ver") or "1.6.37")) - --- Needed at runtime: -load("alps") - -local ufs_modules = { - {["jasper"] = "2.0.25"}, - {["zlib"] = "1.2.11"}, - {["libpng"] = "1.6.37"}, - {["hdf5"] = "1.10.6"}, - {["netcdf"] = "4.7.4"}, - {["pio"] = "2.5.7"}, - {["esmf"] = "8.3.0b09"}, - {["fms"] = "2022.04"}, - {["bacio"] = "2.4.1"}, - {["crtm"] = "2.4.0"}, - {["g2"] = "3.4.5"}, - {["g2tmpl"] = "1.10.2"}, - {["ip"] = "3.3.3"}, - {["sp"] = "2.3.3"}, - {["w3emc"] = "2.9.2"}, - {["gftl-shared"] = "v1.5.0"}, - {["mapl"] = "2.22.0-esmf-8.3.0b09"}, - {["nemsio"] = "2.5.4"}, - {["sigio"] = "2.3.2"}, - {["sfcio"] = "1.4.1"}, - {["wrf_io"] = "1.2.0"}, -} - -for i = 1, #ufs_modules do - for name, default_version in pairs(ufs_modules[i]) do - local env_version_name = string.gsub(name, "-", "_") .. "_ver" - load(pathJoin(name, os.getenv(env_version_name) or default_version)) - end -end +load("PrgEnv-intel/8.3.3") +load("intel-classic/2023.1.0") +load("cray-mpich/8.1.25") +load("python/3.9.12") + +prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core") +prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/modulefiles") + +stack_intel_ver=os.getenv("stack_intel_ver") or "2023.1.0" +load(pathJoin("stack-intel", stack_intel_ver)) + +stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "8.1.25" +load(pathJoin("stack-cray-mpich", stack_cray_mpich_ver)) + +stack_python_ver=os.getenv("stack_python_ver") or "3.10.13" +load(pathJoin("stack-python", stack_python_ver)) + +load("upp_common") + +unload("darshan-runtime") +unload("cray-libsci") setenv("CC","cc") -setenv("FC","ftn") setenv("CXX","CC") +setenv("FC","ftn") + setenv("CMAKE_Platform","gaea.intel") diff --git a/modulefiles/hera_c.lua b/modulefiles/hera_c.lua deleted file mode 100644 index 5bd691da6..000000000 --- a/modulefiles/hera_c.lua +++ /dev/null @@ -1,23 +0,0 @@ -help([[ -Load environment to build UPP on hera -]]) - - -prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core") - -stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0" -load(pathJoin("stack-intel", stack_intel_ver)) - -stack_impi_ver=os.getenv("stack_impi_ver") or "2021.5.1" -load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) - -cmake_ver=os.getenv("cmake_ver") or "3.23.1" -load(pathJoin("cmake", cmake_ver)) - -load("upp_common") - -setenv("CC","mpiicc") -setenv("CXX","mpiicpc") -setenv("FC","mpiifort") - -whatis("Description: UPP build environment") diff --git a/modulefiles/hercules.lua b/modulefiles/hercules.lua index b057df81e..da1722e68 100644 --- a/modulefiles/hercules.lua +++ b/modulefiles/hercules.lua @@ -14,6 +14,9 @@ load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) cmake_ver=os.getenv("cmake_ver") or "3.23.1" load(pathJoin("cmake", cmake_ver)) +stack_python_ver=os.getenv("stack_python_ver") or "3.10.13" +load(pathJoin("stack-python", stack_python_ver)) + load("upp_common") setenv("CC","mpiicc") diff --git a/tests/compile_upp.sh b/tests/compile_upp.sh index 2ecfe9f5e..c1cb507a1 100755 --- a/tests/compile_upp.sh +++ b/tests/compile_upp.sh @@ -83,24 +83,12 @@ source ${PATHTR}/tests/detect_machine.sh if [[ $MACHINE_ID != "unknown" ]]; then if [ $MACHINE_ID == "wcoss2" -o $MACHINE_ID == "wcoss2_a" ]; then module reset - elif [[ "$MACHINE_ID" =~ gaea.* ]] ; then - if [[ $( hostname ) =~ gaea5[0-9] ]] ; then - module purge - # Unset the read-only variables $PELOCAL_PRGENV and $RCLOCAL_PRGENV - gdb -ex 'call (int) unbind_variable("PELOCAL_PRGENV")' \ - -ex 'call (int) unbind_variable("RCLOCAL_PRGENV")' \ - --pid=$$ --batch - # Reload system default modules: - set +eu - source /etc/bash.bashrc.local - source /lustre/f2/dev/role.epic/contrib/Lmod_init.sh - set -eu - else - # There is no safe "module purge" on GAEA compute nodes. - set +eu - source /lustre/f2/dev/role.epic/contrib/Lmod_init.sh - set -eu - fi + elif [[ "$MACHINE_ID" =~ gaea* ]] ; then + module reset + # Unset the read-only variables $PELOCAL_PRGENV and $RCLOCAL_PRGENV + gdb -ex 'call (int) unbind_variable("PELOCAL_PRGENV")' \ + -ex 'call (int) unbind_variable("RCLOCAL_PRGENV")' \ + --pid=$$ --batch else module purge fi diff --git a/tests/detect_machine.sh b/tests/detect_machine.sh index cbc6a96bc..ba157a0eb 100755 --- a/tests/detect_machine.sh +++ b/tests/detect_machine.sh @@ -1,112 +1,95 @@ #!/bin/bash +# The authoritative copy of this script lives in the ufs-weather-model at: +# https://github.com/ufs-community/ufs-weather-model/blob/develop/tests/detect_machine.sh +# If any local modifications are made or new platform support added, +# please consider opening an issue and a PR to the ufs-weather-model +# so that this copy remains in sync with its authoritative source +# +# Thank you for your contribution + +# If the MACHINE_ID variable is set, skip this script. +[[ -n ${MACHINE_ID:-} ]] && return + +# First detect w/ hostname case $(hostname -f) in - alogin01.acorn.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2_a ;; ### acorn - alogin02.acorn.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2_a ;; ### acorn - adecflow01.acorn.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### acorn - adecflow02.acorn.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### acorn - dlogin01.dogwood.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### dodwood - dlogin02.dogwood.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### dodwood - dlogin03.dogwood.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### dodwood - dlogin04.dogwood.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### dodwood - dlogin05.dogwood.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### dodwood - dlogin06.dogwood.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### dodwood - dlogin07.dogwood.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### dodwood - dlogin08.dogwood.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### dodwood - dlogin09.dogwood.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### dodwood - clogin01.cactus.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### cactus - clogin02.cactus.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### cactus - clogin03.cactus.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### cactus - clogin04.cactus.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### cactus - clogin05.cactus.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### cactus - clogin06.cactus.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### cactus - clogin07.cactus.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### cactus - clogin08.cactus.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### cactus - clogin09.cactus.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### cactus - - gaea9) MACHINE_ID=gaea ;; ### gaea9 - gaea10) MACHINE_ID=gaea ;; ### gaea10 - gaea11) MACHINE_ID=gaea ;; ### gaea11 - gaea12) MACHINE_ID=gaea ;; ### gaea12 - gaea13) MACHINE_ID=gaea ;; ### gaea13 - gaea14) MACHINE_ID=gaea ;; ### gaea14 - gaea15) MACHINE_ID=gaea ;; ### gaea15 - gaea16) MACHINE_ID=gaea ;; ### gaea16 - gaea9.ncrc.gov) MACHINE_ID=gaea ;; ### gaea9 - gaea10.ncrc.gov) MACHINE_ID=gaea ;; ### gaea10 - gaea11.ncrc.gov) MACHINE_ID=gaea ;; ### gaea11 - gaea12.ncrc.gov) MACHINE_ID=gaea ;; ### gaea12 - gaea13.ncrc.gov) MACHINE_ID=gaea ;; ### gaea13 - gaea14.ncrc.gov) MACHINE_ID=gaea ;; ### gaea14 - gaea15.ncrc.gov) MACHINE_ID=gaea ;; ### gaea15 - gaea16.ncrc.gov) MACHINE_ID=gaea ;; ### gaea16 - - gaea5[0-9]*) MACHINE_ID=gaea-c5 ;; ### GAEA C5 login node - - hfe01) MACHINE_ID=hera_c ;; ### hera01 - hfe02) MACHINE_ID=hera_c ;; ### hera02 - hfe03) MACHINE_ID=hera_c ;; ### hera03 - hfe04) MACHINE_ID=hera_c ;; ### hera04 - hfe05) MACHINE_ID=hera_c ;; ### hera05 - hfe06) MACHINE_ID=hera_c ;; ### hera06 - hfe07) MACHINE_ID=hera_c ;; ### hera07 - hfe08) MACHINE_ID=hera_c ;; ### hera08 - hfe09) MACHINE_ID=hera ;; ### hera09 - hfe10) MACHINE_ID=hera ;; ### hera10 - hfe11) MACHINE_ID=hera ;; ### hera11 - hfe12) MACHINE_ID=hera ;; ### hera12 - - fe1) MACHINE_ID=jet_c ;; ### jet01 - fe2) MACHINE_ID=jet_c ;; ### jet02 - fe3) MACHINE_ID=jet_c ;; ### jet03 - fe4) MACHINE_ID=jet_c ;; ### jet04 - fe5) MACHINE_ID=jet ;; ### jet05 - fe6) MACHINE_ID=jet ;; ### jet06 - fe7) MACHINE_ID=jet ;; ### jet07 - fe8) MACHINE_ID=jet ;; ### jet08 - tfe1) MACHINE_ID=jet ;; ### jet09 - tfe2) MACHINE_ID=jet ;; ### jet10 - - Orion-login-1.HPC.MsState.Edu) MACHINE_ID=orion ;; ### orion1 - Orion-login-2.HPC.MsState.Edu) MACHINE_ID=orion ;; ### orion2 - Orion-login-3.HPC.MsState.Edu) MACHINE_ID=orion ;; ### orion3 - Orion-login-4.HPC.MsState.Edu) MACHINE_ID=orion ;; ### orion4 - - Hercules-login-1.HPC.MsState.Edu) MACHINE_ID=hercules ;; ### hercules1 - Hercules-login-2.HPC.MsState.Edu) MACHINE_ID=hercules ;; ### hercules2 - Hercules-login-3.HPC.MsState.Edu) MACHINE_ID=hercules ;; ### hercules3 - Hercules-login-4.HPC.MsState.Edu) MACHINE_ID=hercules ;; ### hercules4 - - cheyenne1.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne1 - cheyenne2.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne2 - cheyenne3.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne3 - cheyenne4.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne4 - cheyenne5.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne5 - cheyenne6.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne6 - cheyenne1.ib0.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne1 - cheyenne2.ib0.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne2 - cheyenne3.ib0.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne3 - cheyenne4.ib0.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne4 - cheyenne5.ib0.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne5 - cheyenne6.ib0.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne6 - - login1.stampede2.tacc.utexas.edu) MACHINE_ID=stampede ;; ### stampede1 - login2.stampede2.tacc.utexas.edu) MACHINE_ID=stampede ;; ### stampede2 - login3.stampede2.tacc.utexas.edu) MACHINE_ID=stampede ;; ### stampede3 - login4.stampede2.tacc.utexas.edu) MACHINE_ID=stampede ;; ### stampede4 - - s4-submit.ssec.wisc.edu) MACHINE_ID=s4 ;; ### S4 - - *) MACHINE_ID=unknown + adecflow0[12].acorn.wcoss2.ncep.noaa.gov) MACHINE_ID=acorn ;; ### acorn + alogin0[123].acorn.wcoss2.ncep.noaa.gov) MACHINE_ID=acorn ;; ### acorn + clogin0[1-9].cactus.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### cactus01-9 + clogin10.cactus.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### cactus10 + dlogin0[1-9].dogwood.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### dogwood01-9 + dlogin10.dogwood.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### dogwood10 + + gaea5[1-8]) MACHINE_ID=gaea ;; ### gaea51-58 + gaea5[1-8].ncrc.gov) MACHINE_ID=gaea ;; ### gaea51-58 + + hfe0[1-9]) MACHINE_ID=hera ;; ### hera01-09 + hfe1[0-2]) MACHINE_ID=hera ;; ### hera10-12 + hecflow01) MACHINE_ID=hera ;; ### heraecflow01 + + s4-submit.ssec.wisc.edu) MACHINE_ID=s4 ;; ### s4 + + fe[1-4]) MACHINE_ID=jet_c ;; ### jet1-4 + fe[5-8]) MACHINE_ID=jet ;; ### jet1-8 + tfe[12]) MACHINE_ID=jet ;; ### tjet1-2 + + Orion-login-[1-4].HPC.MsState.Edu) MACHINE_ID=orion ;; ### orion1-4 + + [Hh]ercules-login-[1-4].[Hh][Pp][Cc].[Mm]s[Ss]tate.[Ee]du) MACHINE_ID=hercules ;; ### hercules1-4 + + derecho[1-8].hsn.de.hpc.ucar.edu) MACHINE_ID=derecho ;; ### derecho1-8 + + login[1-4].stampede2.tacc.utexas.edu) MACHINE_ID=stampede ;; ### stampede1-4 + + login0[1-2].expanse.sdsc.edu) MACHINE_ID=expanse ;; ### expanse1-2 + + discover3[1-5].prv.cube) MACHINE_ID=discover ;; ### discover31-35 + *) MACHINE_ID=UNKNOWN ;; # Unknown platform esac -# Overwrite auto-detect with RT_MACHINE if set -MACHINE_ID=${RT_MACHINE:-${MACHINE_ID}} +if [[ ${MACHINE_ID} == "UNKNOWN" ]]; then + case ${PW_CSP:-} in + "aws" | "google" | "azure") MACHINE_ID=noaacloud ;; + *) PW_CSP="UNKNOWN" + esac +fi + +# Overwrite auto-detect with MACHINE if set +MACHINE_ID=${MACHINE:-${MACHINE_ID}} -# Append compiler -#if [ $MACHINE_ID = orion ] || [ $MACHINE_ID = hera ] || [ $MACHINE_ID = cheyenne ] || [ $MACHINE_ID = jet ] || [ $MACHINE_ID = gaea ] || [ $MACHINE_ID = stampede ] || [ $MACHINE_ID = s4 ]; then -# MACHINE_ID=${MACHINE_ID}.${RT_COMPILER} -#fi +# If MACHINE_ID is no longer UNKNNOWN, return it +if [[ "${MACHINE_ID}" != "UNKNOWN" ]]; then + return +fi -echo "Machine: " $MACHINE_ID +# Try searching based on paths since hostname may not match on compute nodes +if [[ -d /lfs/h3 ]]; then + # We are on NOAA Cactus or Dogwood + MACHINE_ID=wcoss2 +elif [[ -d /lfs/h1 && ! -d /lfs/h3 ]]; then + # We are on NOAA TDS Acorn + MACHINE_ID=acorn +elif [[ -d /mnt/lfs1 ]]; then + # We are on NOAA Jet + MACHINE_ID=jet +elif [[ -d /scratch1 ]]; then + # We are on NOAA Hera + MACHINE_ID=hera +elif [[ -d /work ]]; then + # We are on MSU Orion or Hercules + if [[ -d /apps/other ]]; then + # We are on Hercules + MACHINE_ID=hercules + else + MACHINE_ID=orion + fi +elif [[ -d /gpfs && -d /ncrc ]]; then + # We are on GAEA. + MACHINE_ID=gaea +elif [[ -d /data/prod ]]; then + # We are on SSEC's S4 + MACHINE_ID=s4 +else + echo WARNING: UNKNOWN PLATFORM 1>&2 +fi