From b1ccc146bfd683be9ac58ebebeb00f5b53826a75 Mon Sep 17 00:00:00 2001 From: Yunheng Wang <47898913+ywangwof@users.noreply.github.com> Date: Fri, 26 Jun 2020 15:27:48 -0500 Subject: [PATCH] Added support on Odin and Stampede (#242) * Added support on Odin and Stampede * Removed unnecessary module files --- jobs/JREGIONAL_MAKE_ICS | 4 ++++ jobs/JREGIONAL_MAKE_LBCS | 5 +++++ modulefiles/codes/odin/global_equiv_resol | 13 +++++++++++++ modulefiles/codes/odin/mosaic_file | 13 +++++++++++++ modulefiles/codes/odin/regional_grid | 11 +++++++++++ modulefiles/codes/stampede/global_equiv_resol | 10 ++++++++++ modulefiles/codes/stampede/mosaic_file | 12 ++++++++++++ modulefiles/codes/stampede/regional_grid | 10 ++++++++++ modulefiles/tasks/odin/get_extrn_ics | 8 ++++++++ modulefiles/tasks/odin/get_extrn_lbcs | 8 ++++++++ modulefiles/tasks/odin/make_grid | 14 ++++++++++++++ modulefiles/tasks/stampede/get_extrn_ics | 8 ++++++++ modulefiles/tasks/stampede/get_extrn_lbcs | 8 ++++++++ modulefiles/tasks/stampede/make_grid | 11 +++++++++++ scripts/exregional_make_grid.sh | 7 +++++++ scripts/exregional_make_orog.sh | 7 ++++++- scripts/exregional_make_sfc_climo.sh | 10 ++++++++++ scripts/exregional_run_fcst.sh | 9 +++++++++ scripts/exregional_run_post.sh | 5 +++++ ush/get_extrn_mdl_file_dir_info.sh | 5 ++++- ush/set_extrn_mdl_params.sh | 10 ++++++++-- ush/setup.sh | 16 +++++++++++++++- ush/valid_param_vals.sh | 2 +- 23 files changed, 200 insertions(+), 6 deletions(-) create mode 100644 modulefiles/codes/odin/global_equiv_resol create mode 100644 modulefiles/codes/odin/mosaic_file create mode 100644 modulefiles/codes/odin/regional_grid create mode 100644 modulefiles/codes/stampede/global_equiv_resol create mode 100644 modulefiles/codes/stampede/mosaic_file create mode 100644 modulefiles/codes/stampede/regional_grid create mode 100644 modulefiles/tasks/odin/get_extrn_ics create mode 100644 modulefiles/tasks/odin/get_extrn_lbcs create mode 100644 modulefiles/tasks/odin/make_grid create mode 100644 modulefiles/tasks/stampede/get_extrn_ics create mode 100644 modulefiles/tasks/stampede/get_extrn_lbcs create mode 100644 modulefiles/tasks/stampede/make_grid diff --git a/jobs/JREGIONAL_MAKE_ICS b/jobs/JREGIONAL_MAKE_ICS index abb36922c8..b8e8728371 100755 --- a/jobs/JREGIONAL_MAKE_ICS +++ b/jobs/JREGIONAL_MAKE_ICS @@ -85,12 +85,16 @@ case "$MACHINE" in ;; "ODIN") + APRUN="srun" ;; "CHEYENNE") nprocs=$(( NNODES_MAKE_ICS*PPN_MAKE_ICS )) APRUN="mpirun -np $nprocs" ;; + "STAMPEDE") + APRUN="ibrun" + ;; esac # diff --git a/jobs/JREGIONAL_MAKE_LBCS b/jobs/JREGIONAL_MAKE_LBCS index af64a106a5..7b53dab319 100755 --- a/jobs/JREGIONAL_MAKE_LBCS +++ b/jobs/JREGIONAL_MAKE_LBCS @@ -85,6 +85,7 @@ case "$MACHINE" in ;; "ODIN") + APRUN="srun" ;; "CHEYENNE") @@ -92,6 +93,10 @@ case "$MACHINE" in APRUN="mpirun -np $nprocs" ;; + "STAMPEDE") + APRUN="ibrun" + ;; + esac # #----------------------------------------------------------------------- diff --git a/modulefiles/codes/odin/global_equiv_resol b/modulefiles/codes/odin/global_equiv_resol new file mode 100644 index 0000000000..123d202eaa --- /dev/null +++ b/modulefiles/codes/odin/global_equiv_resol @@ -0,0 +1,13 @@ +#%Module##################################################### +## Module file for regional_grid +############################################################# +module load PrgEnv-intel +module swap intel/19.0.5.281 +module load cray-mpich/7.7.10 +module load cray-libsci +module load cray-netcdf-hdf5parallel +module load cray-parallel-netcdf +module load cray-hdf5-parallel + +export NETCDF=/opt/cray/pe/netcdf-hdf5parallel/4.6.3.2/INTEL/19.0 +export HDF5=/opt/cray/pe/hdf5-parallel/1.10.5.2/INTEL/19.0 diff --git a/modulefiles/codes/odin/mosaic_file b/modulefiles/codes/odin/mosaic_file new file mode 100644 index 0000000000..123d202eaa --- /dev/null +++ b/modulefiles/codes/odin/mosaic_file @@ -0,0 +1,13 @@ +#%Module##################################################### +## Module file for regional_grid +############################################################# +module load PrgEnv-intel +module swap intel/19.0.5.281 +module load cray-mpich/7.7.10 +module load cray-libsci +module load cray-netcdf-hdf5parallel +module load cray-parallel-netcdf +module load cray-hdf5-parallel + +export NETCDF=/opt/cray/pe/netcdf-hdf5parallel/4.6.3.2/INTEL/19.0 +export HDF5=/opt/cray/pe/hdf5-parallel/1.10.5.2/INTEL/19.0 diff --git a/modulefiles/codes/odin/regional_grid b/modulefiles/codes/odin/regional_grid new file mode 100644 index 0000000000..2d4b9a9a31 --- /dev/null +++ b/modulefiles/codes/odin/regional_grid @@ -0,0 +1,11 @@ +#%Module##################################################### +## Module file for regional_grid +############################################################# +module load PrgEnv-intel +module swap intel/19.0.5.281 +module load cray-mpich/7.7.10 +module load cray-libsci +module load cray-netcdf-hdf5parallel +module load cray-parallel-netcdf +module load cray-hdf5-parallel + diff --git a/modulefiles/codes/stampede/global_equiv_resol b/modulefiles/codes/stampede/global_equiv_resol new file mode 100644 index 0000000000..0f78231fa0 --- /dev/null +++ b/modulefiles/codes/stampede/global_equiv_resol @@ -0,0 +1,10 @@ +#%Module##################################################### +## Module file for regional_grid +############################################################# +module load intel/18.0.2 +module load impi/18.0.2 +module load hdf5/1.10.4 +module load netcdf/4.6.2 + +setenv NETCDF /opt/apps/intel18/netcdf/4.6.2/x86_64 +setenv HDF5 /opt/apps/intel18/hdf5/1.10.4/x86_64 diff --git a/modulefiles/codes/stampede/mosaic_file b/modulefiles/codes/stampede/mosaic_file new file mode 100644 index 0000000000..bab633db74 --- /dev/null +++ b/modulefiles/codes/stampede/mosaic_file @@ -0,0 +1,12 @@ +#%Module##################################################### +## Module file for regional_grid +############################################################# + + +module load intel/18.0.2 +module load impi/18.0.2 +module load hdf5/1.10.4 +module load netcdf/4.6.2 + +setenv NETCDF /opt/apps/intel18/netcdf/4.6.2/x86_64 +setenv HDF5 /opt/apps/intel18/hdf5/1.10.4/x86_64 diff --git a/modulefiles/codes/stampede/regional_grid b/modulefiles/codes/stampede/regional_grid new file mode 100644 index 0000000000..0f78231fa0 --- /dev/null +++ b/modulefiles/codes/stampede/regional_grid @@ -0,0 +1,10 @@ +#%Module##################################################### +## Module file for regional_grid +############################################################# +module load intel/18.0.2 +module load impi/18.0.2 +module load hdf5/1.10.4 +module load netcdf/4.6.2 + +setenv NETCDF /opt/apps/intel18/netcdf/4.6.2/x86_64 +setenv HDF5 /opt/apps/intel18/hdf5/1.10.4/x86_64 diff --git a/modulefiles/tasks/odin/get_extrn_ics b/modulefiles/tasks/odin/get_extrn_ics new file mode 100644 index 0000000000..77e8616036 --- /dev/null +++ b/modulefiles/tasks/odin/get_extrn_ics @@ -0,0 +1,8 @@ +#%Module##################################################### +## Module file for get_extrn_ics task. +############################################################# + +#module purge + +#module load hpss + diff --git a/modulefiles/tasks/odin/get_extrn_lbcs b/modulefiles/tasks/odin/get_extrn_lbcs new file mode 100644 index 0000000000..e0b1eb8abd --- /dev/null +++ b/modulefiles/tasks/odin/get_extrn_lbcs @@ -0,0 +1,8 @@ +#%Module##################################################### +## Module file for get_extrn_lbcs task. +############################################################# + +#module purge + +#module load hpss + diff --git a/modulefiles/tasks/odin/make_grid b/modulefiles/tasks/odin/make_grid new file mode 100644 index 0000000000..d37cb0e9d7 --- /dev/null +++ b/modulefiles/tasks/odin/make_grid @@ -0,0 +1,14 @@ +#%Module##################################################### +## Module file for make_grid task. +############################################################# + +#module purge + +module load PrgEnv-intel +module swap intel/19.0.5.281 +module load cray-libsci +module load cray-netcdf-hdf5parallel +module load cray-parallel-netcdf +module load cray-hdf5-parallel +module load gcc/6.1.0 +module load slurm diff --git a/modulefiles/tasks/stampede/get_extrn_ics b/modulefiles/tasks/stampede/get_extrn_ics new file mode 100644 index 0000000000..77e8616036 --- /dev/null +++ b/modulefiles/tasks/stampede/get_extrn_ics @@ -0,0 +1,8 @@ +#%Module##################################################### +## Module file for get_extrn_ics task. +############################################################# + +#module purge + +#module load hpss + diff --git a/modulefiles/tasks/stampede/get_extrn_lbcs b/modulefiles/tasks/stampede/get_extrn_lbcs new file mode 100644 index 0000000000..e0b1eb8abd --- /dev/null +++ b/modulefiles/tasks/stampede/get_extrn_lbcs @@ -0,0 +1,8 @@ +#%Module##################################################### +## Module file for get_extrn_lbcs task. +############################################################# + +#module purge + +#module load hpss + diff --git a/modulefiles/tasks/stampede/make_grid b/modulefiles/tasks/stampede/make_grid new file mode 100644 index 0000000000..aa867cfbfc --- /dev/null +++ b/modulefiles/tasks/stampede/make_grid @@ -0,0 +1,11 @@ +#%Module##################################################### +## Module file for make_grid task. +############################################################# + +module purge + +module load intel/18.0.2 +module load impi/18.0.2 +module load hdf5/1.10.4 +module load netcdf/4.6.2 +module load python3/3.7.0 diff --git a/scripts/exregional_make_grid.sh b/scripts/exregional_make_grid.sh index 37b7bcbf1e..48cfb49a25 100755 --- a/scripts/exregional_make_grid.sh +++ b/scripts/exregional_make_grid.sh @@ -148,6 +148,13 @@ case $MACHINE in APRUN="time" ;; +"STAMPEDE") +# + export APRUN="time" + + ulimit -s unlimited + ulimit -a + ;; esac # #----------------------------------------------------------------------- diff --git a/scripts/exregional_make_orog.sh b/scripts/exregional_make_orog.sh index 5f51796668..a8a39dcc0c 100755 --- a/scripts/exregional_make_orog.sh +++ b/scripts/exregional_make_orog.sh @@ -145,6 +145,11 @@ case $MACHINE in APRUN="time" ;; +"STAMPEDE") + export APRUN="time" + export topo_dir="/work/00315/tg455890/stampede2/regional_fv3/fix_orog" + ;; + esac # #----------------------------------------------------------------------- @@ -317,7 +322,7 @@ ${tmp_dir}" \ ;; -"CHEYENNE" | "HERA" | "JET" | "ODIN") +"CHEYENNE" | "HERA" | "JET" | "ODIN" | "STAMPEDE") $APRUN "${exec_fp}" < "${input_redirect_fn}" || \ print_err_msg_exit "\ Call to executable (exec_fp) that generates the raw orography file returned diff --git a/scripts/exregional_make_sfc_climo.sh b/scripts/exregional_make_sfc_climo.sh index 48b383f48f..938ede5e9d 100755 --- a/scripts/exregional_make_sfc_climo.sh +++ b/scripts/exregional_make_sfc_climo.sh @@ -151,6 +151,16 @@ case $MACHINE in APRUN="mpirun -np $nprocs" ;; +"ODIN") + nprocs=$(( NNODES_MAKE_SFC_CLIMO*PPN_MAKE_SFC_CLIMO )) + APRUN="srun -n $nprocs" + ;; + +"STAMPEDE") + nprocs=$(( NNODES_MAKE_SFC_CLIMO*PPN_MAKE_SFC_CLIMO )) + APRUN="ibrun -np ${nprocs}" + ;; + *) print_err_msg_exit "\ Run command has not been specified for this machine: diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index 6f74dd8b02..10b7fed073 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -120,6 +120,15 @@ case $MACHINE in LD_LIBRARY_PATH="${UFS_WTHR_MDL_DIR}/FV3/ccpp/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" ;; # + +"STAMPEDE") +# + module list + + APRUN="ibrun -np ${PE_MEMBER01}" + #LD_LIBRARY_PATH="${UFS_WTHR_MDL_DIR}/FV3/ccpp/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" + ;; + esac # #----------------------------------------------------------------------- diff --git a/scripts/exregional_run_post.sh b/scripts/exregional_run_post.sh index 1832daaada..60758381e7 100755 --- a/scripts/exregional_run_post.sh +++ b/scripts/exregional_run_post.sh @@ -128,6 +128,11 @@ case $MACHINE in APRUN="mpirun -np $nprocs" ;; +"STAMPEDE") + nprocs=$(( NNODES_RUN_POST*PPN_RUN_POST )) + APRUN="ibrun -n $nprocs" + ;; + esac # #----------------------------------------------------------------------- diff --git a/ush/get_extrn_mdl_file_dir_info.sh b/ush/get_extrn_mdl_file_dir_info.sh index 1e52e6ae27..fde6b8e2da 100755 --- a/ush/get_extrn_mdl_file_dir_info.sh +++ b/ush/get_extrn_mdl_file_dir_info.sh @@ -545,11 +545,14 @@ bination of external model (extrn_mdl_name) and analysis or forecast sysdir="" ;; "ODIN") - sysdir="" + sysdir="$sysbasedir" ;; "CHEYENNE") sysdir="" ;; + "STAMPEDE") + sysdir="$sysbasedir" + ;; *) print_err_msg_exit "\ The system directory in which to look for external model output files diff --git a/ush/set_extrn_mdl_params.sh b/ush/set_extrn_mdl_params.sh index 5943fa1ee4..c434f8d60e 100644 --- a/ush/set_extrn_mdl_params.sh +++ b/ush/set_extrn_mdl_params.sh @@ -68,11 +68,14 @@ else EXTRN_MDL_FILES_SYSBASEDIR_ICS="" ;; "ODIN") - EXTRN_MDL_FILES_SYSBASEDIR_ICS="" + EXTRN_MDL_FILES_SYSBASEDIR_ICS="/scratch/ywang/EPIC/GDAS/2019053000_mem001" ;; "CHEYENNE") EXTRN_MDL_FILES_SYSBASEDIR_ICS="/glade/p/ral/jntp/UFS_CAM/COMGFS" ;; + "STAMPEDE") + EXTRN_MDL_FILES_SYSBASEDIR_ICS="/scratch/00315/tg455890/GDAS/20190530/2019053000_mem001" + ;; *) print_err_msg_exit "\ The system directory in which to look for the files generated by the ex- @@ -223,11 +226,14 @@ else EXTRN_MDL_FILES_SYSBASEDIR_LBCS="" ;; "ODIN") - EXTRN_MDL_FILES_SYSBASEDIR_LBCS="" + EXTRN_MDL_FILES_SYSBASEDIR_LBCS="/scratch/ywang/EPIC/GDAS/2019053000_mem001" ;; "CHEYENNE") EXTRN_MDL_FILES_SYSBASEDIR_LBCS="/glade/p/ral/jntp/UFS_CAM/COMGFS" ;; + "STAMPEDE") + EXTRN_MDL_FILES_SYSBASEDIR_LBCS="/scratch/00315/tg455890/GDAS/20190530/2019053000_mem001" + ;; *) print_err_msg_exit "\ The system directory in which to look for the files generated by the ex- diff --git a/ush/setup.sh b/ush/setup.sh index 5aff206f47..8fcabfbc9c 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -312,7 +312,16 @@ case $MACHINE in QUEUE_HPSS=${QUEUE_HPSS:-"regular"} QUEUE_HPSS_TAG="queue" # pbspro does not support "partition" tag QUEUE_FCST=${QUEUE_FCST:-"regular"} + ;; +# +"STAMPEDE") # + NCORES_PER_NODE=68 + SCHED="slurm" + QUEUE_DEFAULT=${QUEUE_DEFAULT:-"normal"} + QUEUE_HPSS=${QUEUE_HPSS:-"development"} + QUEUE_FCST=${QUEUE_FCST:-"normal"} + ;; esac # #----------------------------------------------------------------------- @@ -670,13 +679,18 @@ case $MACHINE in "ODIN") FIXgsm="/scratch/ywang/fix/theia_fix/fix_am" - SFC_CLIMO_INPUT_DIR="/scratch1/NCEPDEV/da/George.Gayno/ufs_utils.git/climo_fields_netcdf" + SFC_CLIMO_INPUT_DIR="/scratch/ywang/fix/climo_fields_netcdf" ;; "CHEYENNE") FIXgsm="/glade/p/ral/jntp/UFS_CAM/fix/fix_am" SFC_CLIMO_INPUT_DIR="/glade/p/ral/jntp/UFS_CAM/fix/climo_fields_netcdf" ;; +"STAMPEDE") + FIXgsm="/work/00315/tg455890/stampede2/regional_fv3/fix_am" + SFC_CLIMO_INPUT_DIR="/work/00315/tg455890/stampede2/regional_fv3/climo_fields_netcdf" + ;; + *) print_err_msg_exit "\ Directories have not been specified for this machine: diff --git a/ush/valid_param_vals.sh b/ush/valid_param_vals.sh index 59caeec391..793c568c8b 100644 --- a/ush/valid_param_vals.sh +++ b/ush/valid_param_vals.sh @@ -1,6 +1,6 @@ valid_vals_RUN_ENVIR=("nco" "community") valid_vals_VERBOSE=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") -valid_vals_MACHINE=("WCOSS_CRAY" "WCOSS_DELL_P3" "THEIA" "HERA" "JET" "ODIN" "CHEYENNE") +valid_vals_MACHINE=("WCOSS_CRAY" "WCOSS_DELL_P3" "THEIA" "HERA" "JET" "ODIN" "CHEYENNE" "STAMPEDE") valid_vals_SCHED=("slurm" "pbspro" "lsf" "lsfcray" "none") valid_vals_PREDEF_GRID_NAME=( \ "EMC_CONUS_3km" \