Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Port to Orion #105

Merged
merged 10 commits into from
Apr 22, 2020
Merged

Port to Orion #105

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
b171ab6
Porting to Orion
DusanJovic-NOAA Apr 15, 2020
e11d158
Merge branch 'develop' into orion
DusanJovic-NOAA Apr 17, 2020
2b9591b
Add WW3 orion port.
DusanJovic-NOAA Apr 19, 2020
4f4ce63
Update NEMS submodule
DusanJovic-NOAA Apr 20, 2020
4653732
Revert changes that removed support for WCOSS phase1/2
DusanJovic-NOAA Apr 20, 2020
14ca8c3
Update FV3 submodule
DusanJovic-NOAA Apr 20, 2020
70f4141
Regression test log files on Hera
DusanJovic-NOAA Apr 21, 2020
c51cdcd
Regression test log files on WCOSS Cray
DusanJovic-NOAA Apr 21, 2020
b25199c
Regression test log files on WCOSS Dell
DusanJovic-NOAA Apr 22, 2020
3787d6e
Update FV3 and WW3 submodules
DusanJovic-NOAA Apr 22, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions .gitmodules
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,19 +1,19 @@
[submodule "FV3"]
path = FV3
url = https://github.com/NOAA-EMC/fv3atm
branch = develop
url = https://github.com/DusanJovic-NOAA/fv3atm
branch = orion
[submodule "NEMS"]
path = NEMS
url = https://github.com/NOAA-EMC/NEMS
branch = develop
url = https://github.com/DusanJovic-NOAA/NEMS
branch = orion
[submodule "FMS"]
path = FMS
url = https://github.com/NOAA-GFDL/FMS
branch = master
[submodule "WW3"]
path = WW3
url = https://github.com/NOAA-EMC/WW3
branch = develop
url = https://github.com/aliabdolali/WW3
branch = OrionPort
[submodule "stochastic_physics"]
path = stochastic_physics
url = https://github.com/noaa-psd/stochastic_physics
Expand Down
3 changes: 3 additions & 0 deletions CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,9 @@ if(WW3)
if(${CMAKE_Platform} STREQUAL "hera.intel")
set(WW3_COMP "hera")
endif()
if(${CMAKE_Platform} STREQUAL "orion.intel")
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should really try to get rid of these extra machine dependencies when moving to cmake.

set(WW3_COMP "orion")
endif()
message("Build WW3:")
message(" run: ${CMAKE_BUILD_TOOL} WW3_PARCOMPN=4 WW3_COMP=${WW3_COMP} ww3_nems")
message(" in: ${PROJECT_SOURCE_DIR}/WW3/model/esmf")
Expand Down
2 changes: 1 addition & 1 deletion FV3
Open in desktop
Submodule FV3 updated 5 files
+1 −1 ccpp/build_ccpp.sh
+7 −0 ccpp/set_compilers.sh
+13 −13 gfsphysics/GFS_layer/GFS_driver.F90
+13 −13 gfsphysics/GFS_layer/GFS_physics_driver.F90
+13 −13 gfsphysics/GFS_layer/GFS_radiation_driver.F90
2 changes: 1 addition & 1 deletion NEMS
Open in desktop
Submodule NEMS updated from 8e1f15 to 5e94fb
2 changes: 1 addition & 1 deletion WW3
Open in desktop
Submodule WW3 updated 4 files
+2 −2 model/bin/cmplr.env
+2 −2 model/bin/w3_setup
+1 −1 model/esmf/Makefile
+9 −0 regtests/bin/matrix_ncep
35 changes: 35 additions & 0 deletions cmake/configure_orion.intel.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
message("")
message("Setting configuration for $ENV{CMAKE_Platform}")
message("")

get_filename_component (C_COMPILER_NAME ${CMAKE_C_COMPILER} NAME)
get_filename_component (CXX_COMPILER_NAME ${CMAKE_CXX_COMPILER} NAME)
get_filename_component (Fortran_COMPILER_NAME ${CMAKE_Fortran_COMPILER} NAME)
message("C compiler: ${CMAKE_C_COMPILER_ID} ${CMAKE_C_COMPILER_VERSION} (${C_COMPILER_NAME})")
message("CXX compiler: ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION} (${CXX_COMPILER_NAME})")
message("Fortran compiler: ${CMAKE_Fortran_COMPILER_ID} ${CMAKE_Fortran_COMPILER_VERSION} (${Fortran_COMPILER_NAME})")
message("")

option(DEBUG "Enable DEBUG mode" OFF)
option(REPRO "Enable REPRO mode" OFF)
option(VERBOSE "Enable VERBOSE mode" OFF)
option(32BIT "Enable 32BIT (single precision arithmetic in dycore)" OFF)
option(OPENMP "Enable OpenMP threading" ON)
option(AVX2 "Enable AVX2 instruction set" ON)

option(INLINE_POST "Enable inline post" ON)

include( cmake/${CMAKE_Fortran_COMPILER_ID}.cmake )

set(NEMSIO_INC $ENV{NEMSIO_INC})
set(POST_INC $ENV{POST_INC})
set(NCEP_LIBS $ENV{POST_LIB} $ENV{NEMSIO_LIB} $ENV{G2_LIB4} $ENV{G2TMPL_LIB} $ENV{BACIO_LIB4} $ENV{SP_LIBd} $ENV{W3EMC_LIBd} $ENV{W3NCO_LIBd} $ENV{CRTM_LIB} $ENV{PNG_LIB} $ENV{JASPER_LIB} $ENV{Z_LIB})

set(ESMF_MOD ${ESMF_F90COMPILEPATHS})
set(ESMF_LIBS "${ESMF_F90ESMFLINKRPATHS} ${ESMF_F90ESMFLINKPATHS} ${ESMF_F90ESMFLINKLIBS}")

set(NETCDF_INC_DIR $ENV{NETCDF}/include)
set(NETCDF_LIBDIR $ENV{NETCDF}/lib)
set(NETCDF_LIBS -L$ENV{NETCDF}/lib -lnetcdff -lnetcdf)

message("")
187 changes: 187 additions & 0 deletions conf/configure.fv3.orion.intel
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,187 @@
## NEMS configuration file
##
## Platform: Hera
## Compiler: Intel with IntelMPI

SHELL=/bin/sh

################################################################################
## Include the common configuration parts

ifdef InNemsMakefile
include $(TOP)/conf/configure.nems.NUOPC
endif

############
# commands #
############
FC = mpiifort
CC = mpiicc
CXX = mpiicpc
LD = mpiifort -mkl=sequential

#########
# flags #
#########
# default is 64-bit OpenMP non-hydrostatic build using AVX2
DEBUG =
REPRO =
VERBOSE =
OPENMP = Y
AVX2 = Y
HYDRO = N
CCPP = N
STATIC = N

include $(ESMFMKFILE)
ESMF_INC = $(ESMF_F90COMPILEPATHS)

NEMSIOINC = -I$(NEMSIO_INC)
NCEPLIBS = $(POST_LIB) $(NEMSIO_LIB) $(G2_LIB4) $(G2TMPL_LIB) $(BACIO_LIB4) $(SP_LIBd) $(W3EMC_LIBd) $(W3NCO_LIBd) $(CRTM_LIB) $(PNG_LIB) $(JASPER_LIB) $(Z_LIB)

##############################################
# Need to use at least GNU Make version 3.81 #
##############################################
need := 3.81
ok := $(filter $(need),$(firstword $(sort $(MAKE_VERSION) $(need))))
ifneq ($(need),$(ok))
$(error Need at least make version $(need). Load module gmake/3.81)
endif

NETCDF_ROOT = $(NETCDF)
INCLUDE = -I$(NETCDF_ROOT)/include
NETCDF_INC = -I$(NETCDF_ROOT)/include
ifneq ($(findstring netcdf/4,$(LOADEDMODULES)),)
NETCDF_LIB += -L$(NETCDF)/lib -lnetcdff -lnetcdf
else
NETCDF_LIB = -L$(NETCDF)/lib -lnetcdff -lnetcdf
endif

FPPFLAGS := -fpp -Wp,-w $(INCLUDE)
CFLAGS := $(INCLUDE)

FFLAGS := $(INCLUDE) -fno-alias -auto -safe-cray-ptr -save-temps -ftz -assume byterecl -nowarn -sox -align array64byte

CPPDEFS += -Duse_libMPI -Duse_netCDF -DSPMD -DUSE_LOG_DIAG_FIELD_INFO -DUSE_GFSL63 -DGFS_PHYS -Duse_WRTCOMP
CPPDEFS += -DNEW_TAUCTMAX -DINTERNAL_FILE_NML

ifeq ($(HYDRO),Y)
CPPDEFS +=
else
CPPDEFS += -DMOIST_CAPPA -DUSE_COND
endif

ifeq ($(NAM_phys),Y)
CPPDEFS += -DNAM_phys
endif

ifeq ($(32BIT),Y)
CPPDEFS += -DOVERLOAD_R4 -DOVERLOAD_R8
FFLAGS += -i4 -real-size 32
else
ifeq ($(REPRO),Y)
FFLAGS += -i4 -real-size 64
else
FFLAGS += -i4 -real-size 64 -no-prec-div -no-prec-sqrt
endif
endif

ifeq ($(REPRO),Y)
FFLAGS += -qno-opt-dynamic-align
CFLAGS += -qno-opt-dynamic-align
else
ifeq ($(AVX2),Y)
# Don't use the AVX512 flags yet on hera
#FFLAGS += -axSSE4.2,AVX,CORE-AVX2,CORE-AVX512 -qno-opt-dynamic-align
#CFLAGS += -axSSE4.2,AVX,CORE-AVX2,CORE-AVX512 -qno-opt-dynamic-align
FFLAGS += -xCORE-AVX2 -qno-opt-dynamic-align
CFLAGS += -xCORE-AVX2 -qno-opt-dynamic-align
else
FFLAGS += -qno-opt-dynamic-align
CFLAGS += -qno-opt-dynamic-align
endif
endif

ifeq ($(MULTI_GASES),Y)
CPPDEFS += -DMULTI_GASES
endif

FFLAGS_OPT = -O2 -debug minimal -fp-model source -qoverride-limits -qopt-prefetch=3
FFLAGS_REPRO = -O2 -debug minimal -fp-model consistent -qoverride-limits -g -traceback
FFLAGS_DEBUG = -g -O0 -check all -check noarg_temp_created -check nopointer -warn -warn noerrors -fp-stack-check -fstack-protector-all -fpe0 -debug -traceback -ftrapuv

TRANSCENDENTALS := -fast-transcendentals
FFLAGS_OPENMP = -qopenmp
FFLAGS_VERBOSE = -v -V -what

CFLAGS += -D__IFC -sox -fp-model source

CFLAGS_OPT = -O2 -debug minimal
CFLAGS_REPRO = -O2 -debug minimal
CFLAGS_OPENMP = -qopenmp
CFLAGS_DEBUG = -O0 -g -ftrapuv -traceback

# Optional Testing compile flags. Mutually exclusive from DEBUG, REPRO, and OPT
# *_TEST will match the production if no new option(s) is(are) to be tested.
FFLAGS_TEST = -O3 -debug minimal -fp-model source -qoverride-limits
CFLAGS_TEST = -O2

LDFLAGS :=
LDFLAGS_OPENMP := -qopenmp
LDFLAGS_VERBOSE := -Wl,-V,--verbose,-cref,-M

# start with blank LIBS
LIBS :=

ifeq ($(REPRO),Y)
CPPDEFS += -DREPRO
CFLAGS += $(CFLAGS_REPRO)
FFLAGS += $(FFLAGS_REPRO)
FAST :=
else ifeq ($(DEBUG),Y)
CPPDEFS += -DDEBUG
CFLAGS += $(CFLAGS_DEBUG)
FFLAGS += $(FFLAGS_DEBUG)
FAST :=
else ifeq ($(TEST),Y)
CFLAGS += $(CFLAGS_TEST)
FFLAGS += $(FFLAGS_TEST)
FAST :=
else
CFLAGS += $(CFLAGS_OPT)
FFLAGS += $(FFLAGS_OPT)
FAST := $(TRANSCENDENTALS)
endif

ifeq ($(OPENMP),Y)
CPPDEFS += -DOPENMP
CFLAGS += $(CFLAGS_OPENMP)
FFLAGS += $(FFLAGS_OPENMP)
LDFLAGS += $(LDFLAGS_OPENMP)
endif

ifeq ($(VERBOSE),Y)
CFLAGS += $(CFLAGS_VERBOSE)
FFLAGS += $(FFLAGS_VERBOSE)
LDFLAGS += $(LDFLAGS_VERBOSE)
endif

ifeq ($(CCPP),Y)
CPPDEFS += -DCCPP
CFLAGS += -I$(PATH_CCPP)/include
FFLAGS += -I$(PATH_CCPP)/include
ifeq ($(STATIC),Y)
CPPDEFS += -DSTATIC
LDFLAGS += -L$(PATH_CCPP)/lib -lccppphys -lccpp $(NCEPLIBS) -lxml2
else
LDFLAGS += -L$(PATH_CCPP)/lib -lccpp
endif
endif

LDFLAGS += $(LIBS)

ifdef InNemsMakefile
FFLAGS += $(ESMF_INC)
CPPFLAGS += -traditional
EXTLIBS = $(NCEPLIBS) $(ESMF_LIB) $(LDFLAGS) $(NETCDF_LIB)
endif
57 changes: 57 additions & 0 deletions modulefiles/orion.intel/fv3
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
#%Module######################################################################
##
## NEMS FV3 Prerequisites: Orion/Intel

proc ModulesHelp {} {
puts stderr "\tcit - loads modules required for building and running FV3 under NEMS on Orion/Intel"
}

module-whatis "loads NEMS FV3 prerequisites for Orion/Intel"

##
## load contrib environment
## load noaatools (slurm utils (arbitrary.pl layout.pl))
##
module load contrib noaatools

##
## load programming environment
## this typically includes compiler, MPI and job scheduler
##
module load intel/2018
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Going back to 2018? For the UFS public release, we used intel/2020. I guess you need this to provide developers a stable alternative platform, as close as possible to hera, when the latter will be occupied with the retrospectives.

module load impi/2018

##
## NCEP libraries (temporary version to match the CCPP requirements)
##
module use /apps/contrib/NCEPLIBS/orion/modulefiles
module load bacio/2.0.3
module load ip/3.0.2
module load nemsio/2.2.4
module load sp/2.0.3
module load w3emc/2.4.0
module load w3nco/2.0.7
module load g2/3.1.1
module load g2tmpl/1.6.0
module load crtm/2.3.0
module load jasper/1.900.2
module load png/1.2.44
module load z/1.2.6

##
## load ESMF library for above compiler / MPI combination
## use pre-compiled EMSF library for above compiler / MPI combination
##
module use /apps/contrib/NCEPLIBS/lib/modulefiles
module load netcdfp/4.7.4
module load esmflocal/8.0.0.para
module load post-intel-sandybridge/8.0.5

##
## load cmake
##
module load cmake/3.15.4
setenv CMAKE_C_COMPILER mpiicc
setenv CMAKE_CXX_COMPILER mpiicpc
setenv CMAKE_Fortran_COMPILER mpiifort
setenv CMAKE_Platform orion.intel
2 changes: 1 addition & 1 deletion tests/default_vars.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ elif [ $MACHINE_ID = wcoss_dell_p3 ]; then
TASKS_stretch=48 ; TPN_stretch=28 ; INPES_stretch=2 ; JNPES_stretch=4
TASKS_strnest=96 ; TPN_strnest=28 ; INPES_strnest=2 ; JNPES_strnest=4

elif [[ $MACHINE_ID = theia.* ]]; then
elif [[ $MACHINE_ID = orion.* ]]; then

TASKS_dflt=150 ; TPN_dflt=24 ; INPES_dflt=3 ; JNPES_dflt=8
TASKS_thrd=84 ; TPN_thrd=12 ; INPES_thrd=3 ; JNPES_thrd=4
Expand Down
Loading