diff --git a/.gitmodules b/.gitmodules index 4843d9ff5e..08036fb8e3 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,8 +4,10 @@ branch = dtc/develop [submodule "NEMS"] path = NEMS - url = https://github.com/NOAA-EMC/NEMS - branch = develop + #url = https://github.com/NOAA-EMC/NEMS + #branch = develop + url = https://github.com/climbfuji/NEMS + branch = final_pr_before_merging_to_develop_or_master_20200313 [submodule "FMS"] path = FMS url = https://github.com/NOAA-GFDL/FMS diff --git a/FV3 b/FV3 index 722339d1cf..8672795f5a 160000 --- a/FV3 +++ b/FV3 @@ -1 +1 @@ -Subproject commit 722339d1cfa75cc63dedde486c733f3907a716ac +Subproject commit 8672795f5a6def43cc9faeecc8c8b67b77ddadc8 diff --git a/NEMS b/NEMS index b64c32ded6..cd9d88a3fd 160000 --- a/NEMS +++ b/NEMS @@ -1 +1 @@ -Subproject commit b64c32ded665b185622328b7805dae2c151ed8b6 +Subproject commit cd9d88a3fdc7d3e9a4c58b2145b305d899fdfed5 diff --git a/cmake/configure_hera.gnu.cmake b/cmake/configure_hera.gnu.cmake new file mode 100644 index 0000000000..7efcb1633e --- /dev/null +++ b/cmake/configure_hera.gnu.cmake @@ -0,0 +1,35 @@ +message("") +message("Setting configuration for $ENV{CMAKE_Platform}") +message("") + +get_filename_component (C_COMPILER_NAME ${CMAKE_C_COMPILER} NAME) +get_filename_component (CXX_COMPILER_NAME ${CMAKE_CXX_COMPILER} NAME) +get_filename_component (Fortran_COMPILER_NAME ${CMAKE_Fortran_COMPILER} NAME) +message("C compiler: ${CMAKE_C_COMPILER_ID} ${CMAKE_C_COMPILER_VERSION} (${C_COMPILER_NAME})") +message("CXX compiler: ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION} (${CXX_COMPILER_NAME})") +message("Fortran compiler: ${CMAKE_Fortran_COMPILER_ID} ${CMAKE_Fortran_COMPILER_VERSION} (${Fortran_COMPILER_NAME})") +message("") + +option(DEBUG "Enable DEBUG mode" OFF) +option(REPRO "Enable REPRO mode" OFF) +option(VERBOSE "Enable VERBOSE mode" OFF) +option(32BIT "Enable 32BIT (single precision arithmetic in dycore)" OFF) +option(OPENMP "Enable OpenMP threading" ON) +option(AVX2 "Enable AVX2 instruction set" OFF) + +option(INLINE_POST "Enable inline post" OFF) + +include( cmake/${CMAKE_Fortran_COMPILER_ID}.cmake ) + +set(NEMSIO_INC $ENV{NEMSIO_INC}) +set(POST_INC $ENV{POST_INC}) +set(NCEP_LIBS $ENV{POST_LIB} $ENV{NEMSIO_LIB} $ENV{G2_LIB4} $ENV{G2TMPL_LIB} $ENV{BACIO_LIB4} $ENV{SP_LIBd} $ENV{W3EMC_LIBd} $ENV{W3NCO_LIBd} $ENV{CRTM_LIB} $ENV{PNG_LIB} $ENV{JASPER_LIB} $ENV{Z_LIB}) + +set(ESMF_MOD ${ESMF_F90COMPILEPATHS}) +set(ESMF_LIBS "${ESMF_F90ESMFLINKRPATHS} ${ESMF_F90ESMFLINKPATHS} ${ESMF_F90ESMFLINKLIBS}") + +set(NETCDF_INC_DIR $ENV{NETCDF}/include) +set(NETCDF_LIBDIR $ENV{NETCDF}/lib) +set(NETCDF_LIBS -L$ENV{NETCDF}/lib -lnetcdff -lnetcdf) + +message("") diff --git a/compsets/hera.input b/compsets/hera.input index f8ae2baa0f..213ec9fbf7 100644 --- a/compsets/hera.input +++ b/compsets/hera.input @@ -31,8 +31,8 @@ platform hera.intel { # INPUTS is the input directory, which should contain fix and parm # files, plus any restarts or other inputs. - BASELINE="/scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200313" - BASELINE_TEMPLATE="/scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200313" + BASELINE="/scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200317/INTEL" + BASELINE_TEMPLATE="/scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200317/INTEL" INPUTS="@[BASELINE]" default_resources={ diff --git a/compsets/wcoss.input b/compsets/wcoss.input index de07434e05..d8f3c5cb51 100644 --- a/compsets/wcoss.input +++ b/compsets/wcoss.input @@ -28,8 +28,8 @@ platform wcoss.phase2 { LONG_TEST_QUEUE='&LONGQ;' BUILD_QUEUE='&BUILDQ;' MACHINE_ID='wcoss.phase2' - BASELINE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200313" - BASELINE_TEMPLATE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200313" + BASELINE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200317" + BASELINE_TEMPLATE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200317" default_resources={ TASKS=156 @@ -126,8 +126,8 @@ platform wcoss.phase1 { C768_THRD=4 MACHINE_ID='wcoss.phase1' - BASELINE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200313" - BASELINE_TEMPLATE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200313" + BASELINE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200317" + BASELINE_TEMPLATE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200317" # WCOSS Phase 2 has been slow of late. DEFAULT_TEST_WALLTIME=2700 @@ -213,8 +213,8 @@ platform wcoss_dell_p3 { LONG_TEST_QUEUE='&LONGQ;' BUILD_QUEUE='&BUILDQ;' - BASELINE="/gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200313/" - BASELINE_TEMPLATE="/gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200313/" + BASELINE="/gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200317/" + BASELINE_TEMPLATE="/gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200317/" default_resources={ TASKS=156 @@ -314,8 +314,8 @@ platform wcoss.cray { BUILD_WALLTIME="3600" DEFAULT_TEST_WALLTIME=1800 - BASELINE="/gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200313" - BASELINE_TEMPLATE="/gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200313" + BASELINE="/gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200317" + BASELINE_TEMPLATE="/gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200317" execution_time_modules=[[[ module load alps diff --git a/conf/configure.fv3.hera.gnu b/conf/configure.fv3.hera.gnu new file mode 100644 index 0000000000..e55daa83b7 --- /dev/null +++ b/conf/configure.fv3.hera.gnu @@ -0,0 +1,175 @@ +## NEMS configuration file +## +## Platform: Hera +## Compiler: GNU with OpenMPI + +SHELL=/bin/sh + +################################################################################ +## Include the common configuration parts + +ifdef InNemsMakefile +include $(TOP)/conf/configure.nems.NUOPC +endif + +############ +# commands # +############ +FC = mpif90 +CC = mpicc +CXX = mpicxx +LD = $(FC) + +######### +# flags # +######### +# default is 64-bit OpenMP non-hydrostatic build using AVX2 +DEBUG = +REPRO = +VERBOSE = +OPENMP = Y +AVX2 = Y +HYDRO = N +CCPP = N +STATIC = N + +include $(ESMFMKFILE) +ESMF_INC = $(ESMF_F90COMPILEPATHS) + +NEMSIOINC = -I$(NEMSIO_INC) +NCEPLIBS = $(NEMSIO_LIB) $(BACIO_LIB4) $(SP_LIBd) $(W3EMC_LIBd) $(W3NCO_LIBd) + +############################################## +# Need to use at least GNU Make version 3.81 # +############################################## +need := 3.81 +ok := $(filter $(need),$(firstword $(sort $(MAKE_VERSION) $(need)))) +ifneq ($(need),$(ok)) +$(error Need at least make version $(need). Load module gmake/3.81) +endif + +NETCDF_ROOT = $(NETCDF) +INCLUDE = -I$(NETCDF_ROOT)/include +NETCDF_INC = -I$(NETCDF_ROOT)/include +ifneq ($(findstring netcdf/4,$(LOADEDMODULES)),) + NETCDF_LIB += -L$(NETCDF)/lib -lnetcdff -lnetcdf +else + NETCDF_LIB += -L$(NETCDF)/lib -lnetcdff -lnetcdf +endif + +FPPFLAGS := -cpp -Wp,-w $(INCLUDE) +CFLAGS := $(INCLUDE) + +FFLAGS := $(INCLUDE) -fcray-pointer -ffree-line-length-none -fno-range-check -fbacktrace + +CPPDEFS += -Duse_libMPI -Duse_netCDF -DSPMD -DUSE_LOG_DIAG_FIELD_INFO -DUSE_GFSL63 -DGFS_PHYS -Duse_WRTCOMP +CPPDEFS += -DNEW_TAUCTMAX -DINTERNAL_FILE_NML -DNO_INLINE_POST + +ifeq ($(HYDRO),Y) +CPPDEFS += +else +CPPDEFS += -DMOIST_CAPPA -DUSE_COND +endif + +ifeq ($(NAM_phys),Y) +CPPDEFS += -DNAM_phys +endif + +ifeq ($(32BIT),Y) +CPPDEFS += -DOVERLOAD_R4 -DOVERLOAD_R8 +FFLAGS += +else +FFLAGS += -fdefault-real-8 -fdefault-double-8 +endif + +ifeq ($(AVX2),Y) +FFLAGS += +CFLAGS += +else +FFLAGS += +CFLAGS += +endif + +ifeq ($(MULTI_GASES),Y) +CPPDEFS += -DMULTI_GASES +endif + +FFLAGS_OPT = -O2 -fno-range-check +FFLAGS_REPRO = -O2 -g -fbacktrace -fno-range-check +FFLAGS_DEBUG = -g -O0 -fno-unsafe-math-optimizations -frounding-math -fsignaling-nans -ffpe-trap=invalid,zero,overflow -fbounds-check -fbacktrace -fno-range-check -Wall + +TRANSCENDENTALS := +FFLAGS_OPENMP = -fopenmp +FFLAGS_VERBOSE = -v -V + +CFLAGS += -D__IFC + +CFLAGS_OPT = -O2 +CFLAGS_REPRO = -O2 +CFLAGS_OPENMP = -fopenmp +CFLAGS_DEBUG = -O0 -g + +# Optional Testing compile flags. Mutually exclusive from DEBUG, REPRO, and OPT +# *_TEST will match the production if no new option(s) is(are) to be tested. +FFLAGS_TEST = -O3 +CFLAGS_TEST = -O2 + +LDFLAGS := +LDFLAGS_OPENMP := -fopenmp +LDFLAGS_VERBOSE := -Wl,-V,--verbose,-cref,-M + +# start with blank LIBS +LIBS := + +ifeq ($(REPRO),Y) +CPPDEFS += -DREPRO +CFLAGS += $(CFLAGS_REPRO) +FFLAGS += $(FFLAGS_REPRO) +FAST := +else ifeq ($(DEBUG),Y) +CPPDEFS += -DDEBUG +CFLAGS += $(CFLAGS_DEBUG) +FFLAGS += $(FFLAGS_DEBUG) +FAST := +else ifeq ($(TEST),Y) +CFLAGS += $(CFLAGS_TEST) +FFLAGS += $(FFLAGS_TEST) +FAST := +else +CFLAGS += $(CFLAGS_OPT) +FFLAGS += $(FFLAGS_OPT) +FAST := $(TRANSCENDENTALS) +endif + +ifeq ($(OPENMP),Y) +CPPDEFS += -DOPENMP +CFLAGS += $(CFLAGS_OPENMP) +FFLAGS += $(FFLAGS_OPENMP) +LDFLAGS += $(LDFLAGS_OPENMP) +endif + +ifeq ($(VERBOSE),Y) +CFLAGS += $(CFLAGS_VERBOSE) +FFLAGS += $(FFLAGS_VERBOSE) +LDFLAGS += $(LDFLAGS_VERBOSE) +endif + +ifeq ($(CCPP),Y) +CPPDEFS += -DCCPP +CFLAGS += -I$(PATH_CCPP)/include +FFLAGS += -I$(PATH_CCPP)/include +ifeq ($(STATIC),Y) +CPPDEFS += -DSTATIC +LDFLAGS += -L$(PATH_CCPP)/lib -lccppphys -lccpp $(NCEPLIBS) -lxml2 +else +LDFLAGS += -L$(PATH_CCPP)/lib -lccpp +endif +endif + +LDFLAGS += $(LIBS) + +ifdef InNemsMakefile +FFLAGS += $(ESMF_INC) +CPPFLAGS += -cpp -traditional +EXTLIBS = $(NCEPLIBS) $(ESMF_LIB) $(LDFLAGS) $(NETCDF_LIB) +endif diff --git a/modulefiles/hera.gnu/fv3 b/modulefiles/hera.gnu/fv3 new file mode 100644 index 0000000000..ec7a2d5aae --- /dev/null +++ b/modulefiles/hera.gnu/fv3 @@ -0,0 +1,42 @@ +#%Module###################################################################### +## +## NEMS FV3 Prerequisites: Hera/GNU + +proc ModulesHelp {} { + puts stderr "\tcit - loads modules required for building and running FV3 under NEMS on Hera/GNU" +} + +module-whatis "loads NEMS FV3 prerequisites for Hera/GNU" + +# NOTE: the "module purge" and loading of the module command are +# handled by the module-setup.sh (or .csh) script. +setenv NCEPLIBS /scratch2/NCEPDEV/nwprod/NCEPLIBS +## +## load contrib environment +## load slurm utils (arbitrary.pl layout.pl) +## +module load contrib sutils + +## +## load programming environment +## this typically includes compiler, MPI and job scheduler +## +module load gnu/9.2.0 +module load openmpi/3.1.4 +module load netcdf/4.7.2 + +## +## use pre-compiled EMSF library and NCEP libraries for above compiler / MPI combination +## +module use -a /scratch1/BMC/gmtb/software/modulefiles/gnu-9.2.0/openmpi-3.1.4 +module load NCEPlibs/1.0.0 + +## +## load cmake +## +module use -a /scratch1/BMC/gmtb/software/modulefiles/generic +module load cmake/3.16.3 +setenv CMAKE_C_COMPILER mpicc +setenv CMAKE_CXX_COMPILER mpicxx +setenv CMAKE_Fortran_COMPILER mpif90 +setenv CMAKE_Platform hera.gnu diff --git a/tests/rt.sh b/tests/rt.sh index 5fe6360a0e..39f9047f73 100755 --- a/tests/rt.sh +++ b/tests/rt.sh @@ -315,7 +315,7 @@ mkdir -p ${STMP}/${USER} # Different own baseline directories for different compilers on Theia/Cheyenne NEW_BASELINE=${STMP}/${USER}/FV3_RT/REGRESSION_TEST -if [[ $MACHINE_ID = theia.* ]] || [[ $MACHINE_ID = cheyenne.* ]] || [[ $MACHINE_ID = jet.* ]] || [[ $MACHINE_ID = gaea.* ]]; then +if [[ $MACHINE_ID = hera.* ]] || [[ $MACHINE_ID = cheyenne.* ]]; then NEW_BASELINE=${NEW_BASELINE}_${COMPILER^^} fi @@ -381,10 +381,13 @@ while getopts ":cfsl:mkreh" opt; do esac done -if [[ $MACHINE_ID = cheyenne.* ]]; then - RTPWD=${RTPWD:-$DISKNM/develop-20200313/${COMPILER^^}} +# Fix me - make those definitions and DISKNM consistent +if [[ $MACHINE_ID = hera.* ]]; then + RTPWD=${RTPWD:-$DISKNM/NEMSfv3gfs/develop-20200317/${COMPILER^^}} +elif [[ $MACHINE_ID = cheyenne.* ]]; then + RTPWD=${RTPWD:-$DISKNM/develop-20200317/${COMPILER^^}} else - RTPWD=${RTPWD:-$DISKNM/NEMSfv3gfs/develop-20200313} + RTPWD=${RTPWD:-$DISKNM/NEMSfv3gfs/develop-20200317} fi shift $((OPTIND-1)) @@ -552,11 +555,11 @@ while read -r line; do APP='' NEMS_VER=$(echo $line | cut -d'|' -f2 | sed -e 's/^ *//' -e 's/ *$//') SET=$( echo $line | cut -d'|' -f3) - MACHINES=$(echo $line | cut -d'|' -f4 | sed -e 's/^ *//' -e 's/ *$//') + MACHINES=$(echo $line | cut -d'|' -f4) CB=$( echo $line | cut -d'|' -f5) [[ $SET_ID != ' ' && $SET != *${SET_ID}* ]] && continue - [[ $MACHINES != ' ' && $MACHINES != "${MACHINE_ID}" ]] && continue + [[ $MACHINES != ' ' && $MACHINES != *${MACHINE_ID}* ]] && continue [[ $CREATE_BASELINE == true && $CB != *fv3* ]] && continue (( COMPILE_NR += 1 )) @@ -591,11 +594,11 @@ while read -r line; do APP=$( echo $line | cut -d'|' -f2 | sed -e 's/^ *//' -e 's/ *$//') SET=$( echo $line | cut -d'|' -f3) - MACHINES=$(echo $line | cut -d'|' -f4 | sed -e 's/^ *//' -e 's/ *$//') + MACHINES=$(echo $line | cut -d'|' -f4) CB=$( echo $line | cut -d'|' -f5) [[ $SET_ID != ' ' && $SET != *${SET_ID}* ]] && continue - [[ $MACHINES != ' ' && $MACHINES != "${MACHINE_ID}" ]] && continue + [[ $MACHINES != ' ' && $MACHINES != *${MACHINE_ID}* ]] && continue [[ $CREATE_BASELINE == true && $CB != *fv3* ]] && continue [[ ${ROCOTO} == true || ${ECFLOW} == true ]] && continue diff --git a/tests/rt_gnu.conf b/tests/rt_gnu.conf index 30b5f2e164..81951f6337 100644 --- a/tests/rt_gnu.conf +++ b/tests/rt_gnu.conf @@ -2,37 +2,37 @@ # IPD PROD tests # ####################################################################################################################################################################################### -COMPILE | | standard | cheyenne.gnu | fv3 | +COMPILE | | standard | | fv3 | RUN | fv3_gfdlmp | standard | cheyenne.gnu | fv3 | -COMPILE | 32BIT=Y DEBUG=Y | standard | cheyenne.gnu | fv3 | -RUN | fv3_control_debug | standard | cheyenne.gnu | fv3 | +COMPILE | 32BIT=Y DEBUG=Y | standard | | fv3 | +RUN | fv3_control_debug | standard | | fv3 | ####################################################################################################################################################################################### # IPD REPRO tests # ####################################################################################################################################################################################### -COMPILE | REPRO=Y | standard | cheyenne.gnu | fv3 | +COMPILE | REPRO=Y | standard | | fv3 | RUN | fv3_gfdlmp | standard | cheyenne.gnu | fv3 | ####################################################################################################################################################################################### # CCPP REPRO tests # ####################################################################################################################################################################################### -COMPILE | REPRO=Y CCPP=Y STATIC=Y SUITES=FV3_GFS_2017_gfdlmp | standard | cheyenne.gnu | | +COMPILE | REPRO=Y CCPP=Y STATIC=Y SUITES=FV3_GFS_2017_gfdlmp | standard | | | RUN | fv3_ccpp_gfdlmp | standard | cheyenne.gnu | | ####################################################################################################################################################################################### # CCPP PROD tests # ####################################################################################################################################################################################### -COMPILE | CCPP=Y STATIC=Y SUITES=FV3_GFS_2017_gfdlmp | standard | cheyenne.gnu | fv3 | +COMPILE | CCPP=Y STATIC=Y SUITES=FV3_GFS_2017_gfdlmp | standard | | fv3 | RUN | fv3_ccpp_gfdlmp | standard | cheyenne.gnu | fv3 | -COMPILE | CCPP=Y STATIC=Y SUITES=FV3_GFS_2017 32BIT=Y DEBUG=Y | standard | cheyenne.gnu | fv3 | -RUN | fv3_ccpp_control_debug | standard | cheyenne.gnu | fv3 | +COMPILE | CCPP=Y STATIC=Y SUITES=FV3_GFS_2017 32BIT=Y DEBUG=Y | standard | | fv3 | +RUN | fv3_ccpp_control_debug | standard | | fv3 | -COMPILE | CCPP=Y STATIC=Y SUITES=FV3_GFS_v15p2,FV3_GFS_v16beta | standard | cheyenne.gnu | fv3 | +COMPILE | CCPP=Y STATIC=Y SUITES=FV3_GFS_v15p2,FV3_GFS_v16beta | standard | | fv3 | RUN | fv3_ccpp_gfs_v15p2 | standard | cheyenne.gnu | fv3 | RUN | fv3_ccpp_gfs_v16beta | standard | cheyenne.gnu | fv3 | @@ -41,7 +41,7 @@ RUN | fv3_ccpp_gfs_v16beta # CCPP DEBUG tests # ####################################################################################################################################################################################### -COMPILE | CCPP=Y STATIC=Y SUITES=FV3_GFS_v15p2,FV3_GFS_v16beta DEBUG=Y | standard | cheyenne.gnu | fv3 | +COMPILE | CCPP=Y STATIC=Y SUITES=FV3_GFS_v15p2,FV3_GFS_v16beta DEBUG=Y | standard | | fv3 | RUN | fv3_ccpp_gfs_v15p2_debug | standard | cheyenne.gnu | fv3 | RUN | fv3_ccpp_gfs_v16beta_debug | standard | cheyenne.gnu | fv3 |