Merge NCAR dtc/develop into NOAA-EMC develop, merge selected updates …

…from NOAA-EMC ufs_public_release, updates for jet and gaea

.gitmodules

@@ -1,15 +1,15 @@
 [submodule "FV3"]
 	path = FV3
-	url = https://github.com/NOAA-EMC/fv3atm
-	branch = develop
+	url = https://github.com/climbfuji/fv3atm
+	branch = emc_devemc_update_from_dtc_develop_20191217
 [submodule "NEMS"]
 	path = NEMS
 	url = https://github.com/NOAA-EMC/NEMS
 	branch = develop
 [submodule "FMS"]
 	path = FMS
-	url = https://github.com/NOAA-EMC/FMS
-	branch = GFS-FMS
+	url = https://github.com/NOAA-GFDL/FMS
+	branch = master
 [submodule "WW3"]
 	path = WW3
 	url = https://github.com/NOAA-EMC/WW3

CMakeLists.txt

@@ -125,7 +125,7 @@ add_library(
     FMS/mosaic/interp.c
     FMS/mosaic/mosaic_util.c
     FMS/mosaic/read_mosaic.c
-    FMS/mpp/affinity.c
+    FMS/affinity/affinity.c
     FMS/mpp/nsclock.c
     FMS/mpp/threadloc.c
 )

cmake/configure_cheyenne.gnu.cmake

@@ -0,0 +1,35 @@
+message("")
+message("Setting configuration for $ENV{CMAKE_Platform}")
+message("")
+
+get_filename_component (C_COMPILER_NAME ${CMAKE_C_COMPILER} NAME)
+get_filename_component (CXX_COMPILER_NAME ${CMAKE_CXX_COMPILER} NAME)
+get_filename_component (Fortran_COMPILER_NAME ${CMAKE_Fortran_COMPILER} NAME)
+message("C       compiler: ${CMAKE_C_COMPILER_ID} ${CMAKE_C_COMPILER_VERSION} (${C_COMPILER_NAME})")
+message("CXX     compiler: ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION} (${CXX_COMPILER_NAME})")
+message("Fortran compiler: ${CMAKE_Fortran_COMPILER_ID} ${CMAKE_Fortran_COMPILER_VERSION} (${Fortran_COMPILER_NAME})")
+message("")
+
+option(DEBUG   "Enable DEBUG mode" OFF)
+option(REPRO   "Enable REPRO mode" OFF)
+option(VERBOSE "Enable VERBOSE mode" OFF)
+option(32BIT   "Enable 32BIT (single precision arithmetic in dycore)" OFF)
+option(OPENMP  "Enable OpenMP threading" ON)
+option(AVX2    "Enable AVX2 instruction set" OFF)
+
+option(INLINE_POST "Enable inline post" OFF)
+
+include( cmake/${CMAKE_Fortran_COMPILER_ID}.cmake )
+
+set(NEMSIO_INC $ENV{NEMSIO_INC})
+set(POST_INC $ENV{POST_INC})
+set(NCEP_LIBS $ENV{POST_LIB} $ENV{NEMSIO_LIB} $ENV{G2_LIB4} $ENV{G2TMPL_LIB} $ENV{BACIO_LIB4} $ENV{SP_LIBd} $ENV{W3EMC_LIBd} $ENV{W3NCO_LIBd} $ENV{CRTM_LIB} $ENV{PNG_LIB} $ENV{JASPER_LIB} $ENV{Z_LIB})
+
+set(ESMF_MOD ${ESMF_F90COMPILEPATHS})
+set(ESMF_LIBS "${ESMF_F90ESMFLINKRPATHS} ${ESMF_F90ESMFLINKPATHS} ${ESMF_F90ESMFLINKLIBS}")
+
+set(NETCDF_INC_DIR $ENV{NETCDF}/include)
+set(NETCDF_LIBDIR $ENV{NETCDF}/lib)
+set(NETCDF_LIBS -L$ENV{NETCDF}/lib -lnetcdff -lnetcdf)
+
+message("")

cmake/configure_cheyenne.intel-impi.cmake

@@ -0,0 +1,35 @@
+message("")
+message("Setting configuration for $ENV{CMAKE_Platform}")
+message("")
+
+get_filename_component (C_COMPILER_NAME ${CMAKE_C_COMPILER} NAME)
+get_filename_component (CXX_COMPILER_NAME ${CMAKE_CXX_COMPILER} NAME)
+get_filename_component (Fortran_COMPILER_NAME ${CMAKE_Fortran_COMPILER} NAME)
+message("C       compiler: ${CMAKE_C_COMPILER_ID} ${CMAKE_C_COMPILER_VERSION} (${C_COMPILER_NAME})")
+message("CXX     compiler: ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION} (${CXX_COMPILER_NAME})")
+message("Fortran compiler: ${CMAKE_Fortran_COMPILER_ID} ${CMAKE_Fortran_COMPILER_VERSION} (${Fortran_COMPILER_NAME})")
+message("")
+
+option(DEBUG   "Enable DEBUG mode" OFF)
+option(REPRO   "Enable REPRO mode" OFF)
+option(VERBOSE "Enable VERBOSE mode" OFF)
+option(32BIT   "Enable 32BIT (single precision arithmetic in dycore)" OFF)
+option(OPENMP  "Enable OpenMP threading" ON)
+option(AVX2    "Enable AVX2 instruction set" ON)
+
+option(INLINE_POST "Enable inline post" OFF)
+
+include( cmake/${CMAKE_Fortran_COMPILER_ID}.cmake )
+
+set(NEMSIO_INC $ENV{NEMSIO_INC})
+set(POST_INC $ENV{POST_INC})
+set(NCEP_LIBS $ENV{POST_LIB} $ENV{NEMSIO_LIB} $ENV{G2_LIB4} $ENV{G2TMPL_LIB} $ENV{BACIO_LIB4} $ENV{SP_LIBd} $ENV{W3EMC_LIBd} $ENV{W3NCO_LIBd} $ENV{CRTM_LIB} $ENV{PNG_LIB} $ENV{JASPER_LIB} $ENV{Z_LIB})
+
+set(ESMF_MOD ${ESMF_F90COMPILEPATHS})
+set(ESMF_LIBS "${ESMF_F90ESMFLINKRPATHS} ${ESMF_F90ESMFLINKPATHS} ${ESMF_F90ESMFLINKLIBS}")
+
+set(NETCDF_INC_DIR $ENV{NETCDF}/include)
+set(NETCDF_LIBDIR $ENV{NETCDF}/lib)
+set(NETCDF_LIBS -L$ENV{NETCDF}/lib -lnetcdff -lnetcdf)
+
+message("")

cmake/configure_cheyenne.intel.cmake

@@ -0,0 +1,35 @@
+message("")
+message("Setting configuration for $ENV{CMAKE_Platform}")
+message("")
+
+get_filename_component (C_COMPILER_NAME ${CMAKE_C_COMPILER} NAME)
+get_filename_component (CXX_COMPILER_NAME ${CMAKE_CXX_COMPILER} NAME)
+get_filename_component (Fortran_COMPILER_NAME ${CMAKE_Fortran_COMPILER} NAME)
+message("C       compiler: ${CMAKE_C_COMPILER_ID} ${CMAKE_C_COMPILER_VERSION} (${C_COMPILER_NAME})")
+message("CXX     compiler: ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION} (${CXX_COMPILER_NAME})")
+message("Fortran compiler: ${CMAKE_Fortran_COMPILER_ID} ${CMAKE_Fortran_COMPILER_VERSION} (${Fortran_COMPILER_NAME})")
+message("")
+
+option(DEBUG   "Enable DEBUG mode" OFF)
+option(REPRO   "Enable REPRO mode" OFF)
+option(VERBOSE "Enable VERBOSE mode" OFF)
+option(32BIT   "Enable 32BIT (single precision arithmetic in dycore)" OFF)
+option(OPENMP  "Enable OpenMP threading" ON)
+option(AVX2    "Enable AVX2 instruction set" ON)
+
+option(INLINE_POST "Enable inline post" OFF)
+
+include( cmake/${CMAKE_Fortran_COMPILER_ID}.cmake )
+
+set(NEMSIO_INC $ENV{NEMSIO_INC})
+set(POST_INC $ENV{POST_INC})
+set(NCEP_LIBS $ENV{POST_LIB} $ENV{NEMSIO_LIB} $ENV{G2_LIB4} $ENV{G2TMPL_LIB} $ENV{BACIO_LIB4} $ENV{SP_LIBd} $ENV{W3EMC_LIBd} $ENV{W3NCO_LIBd} $ENV{CRTM_LIB} $ENV{PNG_LIB} $ENV{JASPER_LIB} $ENV{Z_LIB})
+
+set(ESMF_MOD ${ESMF_F90COMPILEPATHS})
+set(ESMF_LIBS "${ESMF_F90ESMFLINKRPATHS} ${ESMF_F90ESMFLINKPATHS} ${ESMF_F90ESMFLINKLIBS}")
+
+set(NETCDF_INC_DIR $ENV{NETCDF}/include)
+set(NETCDF_LIBDIR $ENV{NETCDF}/lib)
+set(NETCDF_LIBS -L$ENV{NETCDF}/lib -lnetcdff -lnetcdf)
+
+message("")

compsets/fv3.input

@@ -201,7 +201,10 @@ fv3_defaults = {
     MAKE_NH='.T.'
     MOUNTAIN='.F.'
     SATMEDMF='.F.'
+    ISATMEDMF='0'
     HYBEDMF='.T.'
+    IMFSHALCNV='2'
+    IMFDEEPCNV='2'
     LHEATSTRG='.F.'
     LGFDLMPRAD='.F.'
     EFFR_IN='.F.'

conf/configure.fv3.macosx.gnu

@@ -122,8 +122,8 @@ CPPDEFS += -DMULTI_GASES
 endif
 
 FFLAGS_OPT = -O2 -g -fno-range-check
-FFLAGS_REPRO = -O2 -g -fbacktrace -fno-range-check
-FFLAGS_DEBUG = -g -O0 -ggdb -fno-unsafe-math-optimizations -frounding-math -fsignaling-nans -Wuninitialized -ffpe-trap=invalid,zero,overflow -fbounds-check -fbacktrace -fno-range-check -Wall
+FFLAGS_REPRO = -O0 -g -fbacktrace -fno-range-check
+FFLAGS_DEBUG = -g -O0 -ggdb -fno-unsafe-math-optimizations -frounding-math -fsignaling-nans -ffpe-trap=invalid,zero,overflow -fbounds-check -fbacktrace -fno-range-check -Wall
 
 TRANSCENDENTALS :=
 FFLAGS_OPENMP = -fopenmp

modulefiles/cheyenne.gnu/fv3

@@ -37,3 +37,11 @@ module load NCEPlibs/9.9.9
 ## SIONlib library
 ##
 module load SIONlib/1.7.4
+
+##
+## set cmake compilers
+##
+setenv CMAKE_C_COMPILER mpicc
+setenv CMAKE_CXX_COMPILER mpicxx
+setenv CMAKE_Fortran_COMPILER mpif90
+setenv CMAKE_Platform cheyenne.gnu

modulefiles/cheyenne.intel-impi/fv3

@@ -37,3 +37,11 @@ module load NCEPlibs/9.9.9
 ## SIONlib library
 ##
 module load SIONlib/1.7.4
+
+##
+## set cmake compilers
+##
+setenv CMAKE_C_COMPILER mpicc
+setenv CMAKE_CXX_COMPILER mpicxx
+setenv CMAKE_Fortran_COMPILER mpif90
+setenv CMAKE_Platform cheyenne.intel-impi

modulefiles/cheyenne.intel/fv3

@@ -37,3 +37,11 @@ module load NCEPlibs/9.9.9
 ## SIONlib library
 ##
 module load SIONlib/1.7.4
+
+##
+## set cmake compilers
+##
+setenv CMAKE_C_COMPILER mpicc
+setenv CMAKE_CXX_COMPILER mpicxx
+setenv CMAKE_Fortran_COMPILER mpif90
+setenv CMAKE_Platform cheyenne.intel

modulefiles/gaea.intel/fv3

@@ -20,7 +20,7 @@ module rm intel
 module load intel/18.0.3.222
 module load cray-netcdf
 
-module use -a /lustre/f2/pdata/esrl/gsd/gmtb/NEMSfv3gfs/modules/modulefiles/intel-18.0.3.222
+module use -a /lustre/f2/pdata/esrl/gsd/ufs/modules/modulefiles/intel-18.0.3.222
 
 ##
 ## Libxml2 - compiled manually because of issues with linking the code
@@ -46,5 +46,5 @@ module load alps
 setenv CMAKE_C_COMPILER cc
 setenv CMAKE_CXX_COMPILER CC
 setenv CMAKE_Fortran_COMPILER ftn
-setenv CMAKE_Platform gaea
+setenv CMAKE_Platform gaea.intel
 setenv NETCDF ${NETCDF_DIR}

modulefiles/gaea.intel/fv3.intel-16.0.3.210

@@ -22,7 +22,7 @@ module load cray-netcdf
 setenv NETCDF /opt/cray/pe/netcdf/4.4.0/INTEL/15.0
 setenv HDF5   /opt/cray/pe/hdf5/1.8.16/INTEL/15.0
 
-module use -a /lustre/f2/pdata/esrl/gsd/gmtb/NEMSfv3gfs/modules/modulefiles/intel-16.0.3.210
+module use -a /lustre/f2/pdata/esrl/gsd/ufs/modules/modulefiles/intel-16.0.3.210
 
 ##
 ## Libxml2 - compiled manually because of issues with linking the code
@@ -37,7 +37,7 @@ module load NCEPlibs/9.9.9
 ##
 ## use pre-compiled EMSF library for above compiler / MPI combination
 ##
-module load esmf/8.0.0_bs50
+module load esmf/8.0.0
 
 # Needed at runtime:
 module load alps
@@ -51,4 +51,5 @@ setenv F90 mpif90
 setenv CMAKE_C_COMPILER mpicc
 setenv CMAKE_CXX_COMPILER mpic++
 setenv CMAKE_Fortran_COMPILER mpif90
-setenv CMAKE_Platform theia.gnu
+setenv CMAKE_Platform gaea.intel
+setenv NETCDF ${NETCDF_DIR}

parm/ccpp_control.nml.IN

@@ -159,7 +159,7 @@
        cnvcld         = .true.
        imfshalcnv     = 2
        imfdeepcnv     = 2
-       isatmedmf      = 0
+       isatmedmf      = @[ISATMEDMF]
        cdmbgwd        = 3.5,0.25
        prslrd0        = 0.
        ivegsrc        = 1

parm/ccpp_gfdlmp.nml.IN

@@ -178,6 +178,8 @@
        oz_phys_2015   = @[OZ_PHYS_NEW]
        h2o_phys       = @[H2O_PHYS]
        nstf_name      = @[NSTF_NAME]
+       cplflx         = @[CPLFLX]
+       cplwav         = @[CPLWAV]
        xkzminv        = 0.3
        xkzm_m         = 1.0
        xkzm_h         = 1.0

parm/ccpp_gsd_sar-model_configure.IN

@@ -0,0 +1,48 @@
+total_member:            1
+PE_MEMBER01:             840
+start_year:              2019
+start_month:             07
+start_day:               12
+start_hour:              12
+start_minute:            0
+start_second:            0
+nhours_fcst:             3
+RUN_CONTINUE:            .false.
+ENS_SPS:                 .false.
+dt_atmos:                50
+cpl:                     .false.
+calendar:                'julian'
+memuse_verbose:          .false.
+atmos_nthreads:          2
+use_hyper_thread:        .false.
+ncores_per_node:         24
+debug_affinity:          .true.
+restart_interval:        0
+output_1st_tstep_rst:    .false.
+quilting:                .true.
+print_esmf:              .false.
+
+write_groups:            1
+write_tasks_per_group:   24
+num_files:               2
+filename_base:           'dyn''phy'
+output_file:             'netcdf'
+write_nemsioflip:        .false.
+write_fsyncflag:         .false.
+
+output_grid:             'lambert_conformal'
+cen_lon:                 -97.5
+cen_lat:                 38.5
+stdlat1:                 38.5
+stdlat2:                 38.5
+nx:                      1738
+ny:                      974
+lon1:                    -122.21414225
+lat1:                    22.41403305
+dx:                      3000.0
+dy:                      3000.0
+
+nfhout:                  1
+nfhmax_hf:               60
+nfhout_hf:               1
+nsout:                   -1