NOAA-GSL · DomHeinzeller · Jan 30, 2020 · Dec 4, 2019 · Dec 10, 2019 · Dec 26, 2019
diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,4 @@
 *.o
 *.mod
 *.a
+*.pyc
diff --git a/.gitmodules b/.gitmodules
@@ -1,12 +1,12 @@
 [submodule "atmos_cubed_sphere"]
 	path = atmos_cubed_sphere
-	url = https://github.com/noaa-gsd/GFDL_atmos_cubed_sphere
-	branch = gsd/develop
+	url = https://github.com/climbfuji/GFDL_atmos_cubed_sphere
+	branch = update_dtc_develop_from_dev-emc
 [submodule "ccpp/framework"]
 	path = ccpp/framework
-	url = https://github.com/noaa-gsd/ccpp-framework
-	branch = gsd/develop
+	url = https://github.com/climbfuji/ccpp-framework
+	branch = update_dtc_develop_from_master
 [submodule "ccpp/physics"]
 	path = ccpp/physics
-	url = https://github.com/noaa-gsd/ccpp-physics
-	branch = gsd/develop
+	url = https://github.com/climbfuji/ccpp-physics
+	branch = update_dtc_develop_from_master
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -0,0 +1,130 @@
+if( NOT DEFINED PHYS)
+  set(PHYS gfs)
+endif()
+
+if (PHYS MATCHES "nam")
+  add_definitions(-DNAM_phys)
+endif()
+
+if(CCPP)
+  add_definitions(-DCCPP)
+  if(STATIC)
+    add_definitions(-DSTATIC)
+  endif()
+endif()
+
+add_subdirectory(cpl)
+add_subdirectory(${PHYS}physics)
+add_subdirectory(ipd)
+add_subdirectory(io)
+
+# fv3core library
+
+if(NOT CCPP)
+set(FAST_PHYSICS_SRCS_F90 atmos_cubed_sphere/model/fv_cmp.F90)
+endif()
+
+add_library(
+    fv3core
+
+    atmos_cubed_sphere/model/a2b_edge.F90
+    atmos_cubed_sphere/model/multi_gases.F90
+    atmos_cubed_sphere/model/boundary.F90
+    atmos_cubed_sphere/model/dyn_core.F90
+    atmos_cubed_sphere/model/fv_arrays.F90
+    ${FAST_PHYSICS_SRCS_F90}
+    atmos_cubed_sphere/model/fv_control.F90
+    atmos_cubed_sphere/model/fv_dynamics.F90
+    atmos_cubed_sphere/model/fv_fill.F90
+    atmos_cubed_sphere/model/fv_grid_utils.F90
+    atmos_cubed_sphere/model/fv_mapz.F90
+    atmos_cubed_sphere/model/fv_nesting.F90
+    atmos_cubed_sphere/model/fv_regional_bc.F90
+    atmos_cubed_sphere/model/fv_sg.F90
+    atmos_cubed_sphere/model/fv_tracer2d.F90
+    atmos_cubed_sphere/model/fv_update_phys.F90
+    atmos_cubed_sphere/model/sw_core.F90
+    atmos_cubed_sphere/model/tp_core.F90
+    atmos_cubed_sphere/model/nh_core.F90
+    atmos_cubed_sphere/model/nh_utils.F90
+    atmos_cubed_sphere/tools/external_ic.F90
+    atmos_cubed_sphere/tools/external_sst.F90
+    atmos_cubed_sphere/tools/fv_diagnostics.F90
+    atmos_cubed_sphere/tools/fv_eta.F90
+    atmos_cubed_sphere/tools/fv_grid_tools.F90
+    atmos_cubed_sphere/tools/fv_io.F90
+    atmos_cubed_sphere/tools/fv_mp_mod.F90
+    atmos_cubed_sphere/tools/fv_nudge.F90
+    atmos_cubed_sphere/tools/fv_treat_da_inc.F90
+    atmos_cubed_sphere/tools/fv_iau_mod.F90
+    atmos_cubed_sphere/tools/fv_restart.F90
+    atmos_cubed_sphere/tools/fv_surf_map.F90
+    atmos_cubed_sphere/tools/fv_timing.F90
+    atmos_cubed_sphere//tools/init_hydro.F90
+    atmos_cubed_sphere/tools/sim_nc_mod.F90
+    atmos_cubed_sphere/tools/sorted_index.F90
+    atmos_cubed_sphere/tools/test_cases.F90
+    atmos_cubed_sphere/driver/fvGFS/DYCORE_typedefs.F90
+    atmos_cubed_sphere/driver/fvGFS/fv_nggps_diag.F90
+    atmos_cubed_sphere/driver/fvGFS/atmosphere.F90
+)
+
+set_property(SOURCE atmos_cubed_sphere/model/nh_utils.F90 APPEND_STRING PROPERTY COMPILE_FLAGS "${FAST}")
+set_property(SOURCE atmos_cubed_sphere/model/fv_mapz.F90  APPEND_STRING PROPERTY COMPILE_FLAGS "${FAST}")
+
+target_link_libraries(fv3core PRIVATE fms)
+target_link_libraries(fv3core PRIVATE ${PHYS}physics)
+target_link_libraries(fv3core PRIVATE ipd)
+
+target_include_directories(fv3core PRIVATE ${ESMF_MOD})
+
+# end of fv3core
+
+###############################################################################
+### ccpp
+###############################################################################
+if(CCPP)
+  add_subdirectory(ccpp)
+  add_subdirectory(ccpp/driver)
+  add_dependencies(${PHYS}physics ccpp ccppphys)
+  add_dependencies(ccppdriver ccpp ccppphys)
+  add_dependencies(ccppphys ccpp)
+
+  target_include_directories(fv3core PRIVATE ${CMAKE_BINARY_DIR}/FV3/ccpp/framework/src)
+  target_include_directories(fv3core PRIVATE ${CMAKE_BINARY_DIR}/FV3/ccpp/driver)
+endif()
+
+set_target_properties(fv3core PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mod)
+target_include_directories(fv3core PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/mod)
+
+# When specifying an out-of-tree source a binary directory must be explicitly specified.
+#add_subdirectory(../stochastic_physics stochastic_physics)
+add_subdirectory(stochastic_physics)
+
+add_library(
+    fv3cap
+
+    atmos_model.F90
+    fv3_cap.F90
+    module_fv3_config.F90
+    module_fcst_grid_comp.F90
+    time_utils.F90
+)
+
+if(CCPP)
+target_include_directories(fv3cap PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/ccpp/driver/mod)
+endif()
+target_include_directories(fv3cap PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/stochastic_physics)
+target_compile_definitions(fv3cap PRIVATE -DESMF_VERSION_MAJOR=${ESMF_VERSION_MAJOR})
+target_include_directories(fv3cap PRIVATE ${ESMF_MOD})
+
+target_link_libraries(fv3cap PRIVATE fms)
+target_link_libraries(fv3cap PRIVATE fv3cpl)
+target_link_libraries(fv3cap PRIVATE ${PHYS}physics)
+target_link_libraries(fv3cap PRIVATE ipd)
+target_link_libraries(fv3cap PRIVATE io)
+target_link_libraries(fv3cap PRIVATE fv3core)
+target_link_libraries(fv3cap PRIVATE stochastic_physics)
+
+set_target_properties(fv3cap PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mod)
+target_include_directories(fv3cap PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/mod)
diff --git a/atmos_cubed_sphere b/atmos_cubed_sphere
diff --git a/atmos_model.F90 b/atmos_model.F90
@@ -113,7 +113,8 @@ module atmos_model_mod
                               FV3GFS_diag_register, FV3GFS_diag_output,  &
                               DIAG_SIZE
 use fv_iau_mod,         only: iau_external_data_type,getiauforcing,iau_initialize
-use module_fv3_config,  only: output_1st_tstep_rst, first_kdt, nsout
+use module_fv3_config,  only: output_1st_tstep_rst, first_kdt, nsout,    &
+                              frestart, restart_endfcst
 
 !-----------------------------------------------------------------------
 
@@ -221,7 +222,8 @@ module atmos_model_mod
   logical,parameter :: flip_vc = .true.
 #endif
 
-  real(kind=IPD_kind_phys), parameter :: zero=0.0, one=1.0
+  real(kind=IPD_kind_phys), parameter :: zero = 0.0_IPD_kind_phys, &
+                                         one  = 1.0_IPD_kind_phys
 
 contains
 
@@ -945,17 +947,19 @@ end subroutine update_atmos_model_state
 subroutine atmos_model_end (Atmos)
   type (atmos_data_type), intent(inout) :: Atmos
 !---local variables
-  integer :: idx
+  integer :: idx, seconds
 #ifdef CCPP
   integer :: ierr
 #endif
 
 !-----------------------------------------------------------------------
 !---- termination routine for atmospheric model ----
 
-    call atmosphere_end (Atmos % Time, Atmos%grid)
-    call FV3GFS_restart_write (IPD_Data, IPD_Restart, Atm_block, &
-                               IPD_Control, Atmos%domain)
+    call atmosphere_end (Atmos % Time, Atmos%grid, restart_endfcst)
+    if(restart_endfcst) then
+      call FV3GFS_restart_write (IPD_Data, IPD_Restart, Atm_block, &
+                                 IPD_Control, Atmos%domain)
+    endif
 
 #ifdef CCPP
 !   Fast physics (from dynamics) are finalized in atmosphere_end above;
@@ -1458,6 +1462,24 @@ subroutine update_atmos_chemistry(state, rc)
         enddo
       enddo
 
+      ! -- zero out accumulated fields
+!$OMP parallel do default (none) &
+!$OMP             shared  (nj, ni, Atm_block, IPD_Control, IPD_Data) &
+!$OMP             private (j, jb, i, ib, nb, ix)
+      do j = 1, nj
+        jb = j + Atm_block%jsc - 1
+        do i = 1, ni
+          ib = i + Atm_block%isc - 1
+          nb = Atm_block%blkno(ib,jb)
+          ix = Atm_block%ixp(ib,jb)
+          IPD_Data(nb)%coupling%rainc_cpl(ix)  = zero
+          if (.not.IPD_Control%cplflx) then
+            IPD_Data(nb)%coupling%rain_cpl(ix) = zero
+            IPD_Data(nb)%coupling%snow_cpl(ix) = zero
+          end if
+        enddo
+      enddo
+
       if (IPD_Control%debug) then
         ! -- diagnostics
         write(6,'("update_atmos: prsi   - min/max/avg",3g16.6)') minval(prsi),   maxval(prsi),   sum(prsi)/size(prsi)
@@ -1699,7 +1721,7 @@ subroutine assign_importdata(rc)
                   IPD_Data(nb)%Coupling%ficein_cpl(ix)   = zero
                   if (IPD_Data(nb)%Sfcprop%oceanfrac(ix) > zero) then
                     if (datar8(i,j) >= IPD_control%min_seaice*IPD_Data(nb)%Sfcprop%oceanfrac(ix)) then
-                      IPD_Data(nb)%Coupling%ficein_cpl(ix) = datar8(i,j)
+                      IPD_Data(nb)%Coupling%ficein_cpl(ix) = max(zero, min(datar8(i,j),one))
 !                     if (IPD_Data(nb)%Sfcprop%oceanfrac(ix) == one) IPD_Data(nb)%Sfcprop%slmsk(ix) = 2. !slmsk=2 crashes in gcycle on partial land points
                       IPD_Data(nb)%Sfcprop%slmsk(ix)         = 2.                                        !slmsk=2 crashes in gcycle on partial land points
                       IPD_Data(nb)%Coupling%slimskin_cpl(ix) = 4.

diff --git a/ccpp/CMakeLists.txt b/ccpp/CMakeLists.txt
@@ -259,12 +259,20 @@ endif(MKL_DIR)
 
 #------------------------------------------------------------------------------
 # Set netCDF flags for preprocessor, compiler and linker (if defined)
+# Legacy settings for old make build
 if(NETCDF_DIR)
   set (NETCDF_INC "-I${NETCDF_DIR}/include")
   set (NETCDF_LIB "-L${NETCDF_DIR}/lib -lnetcdff -lnetcdf")
   set (CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} ${NETCDF_INC} ${NETCDF_LIB}")
   ADD_DEFINITIONS(-DNETCDF)
   message (STATUS "Enable netCDF support")
+# Settings for new cmake build
+elseif (NETCDF_INC_DIR AND NETCDF_LIBDIR)
+  set (NETCDF_INC "-I${NETCDF_INC_DIR}")
+  set (NETCDF_LIB "-L${NETCDF_LIBDIR} -lnetcdff -lnetcdf")
+  set (CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} ${NETCDF_INC} ${NETCDF_LIB}")
+  ADD_DEFINITIONS(-DNETCDF)
+  message (STATUS "Enable netCDF support")
 else(NETCDF_DIR)
   message (STATUS "Disable netCDF support")
 endif(NETCDF_DIR)

diff --git a/ccpp/config/ccpp_prebuild_config.py b/ccpp/config/ccpp_prebuild_config.py
@@ -231,7 +231,6 @@
     'FV3/ccpp/physics/physics/sfc_cice.f'                        : [ 'slow_physics' ],
     'FV3/ccpp/physics/physics/sfc_diff.f'                        : [ 'slow_physics' ],
     'FV3/ccpp/physics/physics/sfc_drv.f'                         : [ 'slow_physics' ],
-    'FV3/ccpp/physics/physics/sfc_noahmp_pre.F90'                : [ 'slow_physics' ],
     'FV3/ccpp/physics/physics/sfc_noahmp_drv.f'                  : [ 'slow_physics' ],
     'FV3/ccpp/physics/physics/sfc_nst.f'                         : [ 'slow_physics' ],
     'FV3/ccpp/physics/physics/sfc_ocean.F'                       : [ 'slow_physics' ],

diff --git a/ccpp/driver/CMakeLists.txt b/ccpp/driver/CMakeLists.txt
@@ -0,0 +1,27 @@
+
+if(NOT DYN32)
+remove_definitions(-DOVERLOAD_R8)
+remove_definitions(-DOVERLOAD_R4)
+endif()
+
+message ("Force 64 bits in CCPP_layer")
+if(CMAKE_Fortran_COMPILER_ID MATCHES "Intel")
+    string (REPLACE "-i4 -real-size 32" "-i4 -real-size 64 -no-prec-div -no-prec-sqrt" CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}")
+elseif(CMAKE_Fortran_COMPILER_ID MATCHES "GNU")
+    set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fdefault-real-8")
+endif()
+
+add_library(
+    ccppdriver
+
+    CCPP_driver.F90
+)
+
+target_link_libraries(ccppdriver ${PHYS}physics)
+target_link_libraries(ccppdriver ccpp)
+target_link_libraries(ccppdriver ccppphys)
+
+target_include_directories(ccppdriver PRIVATE ${CMAKE_BINARY_DIR}/FV3/ccpp/framework/src)
+
+set_target_properties(ccppdriver PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mod)
+target_include_directories(ccppdriver PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/mod)
diff --git a/ccpp/framework b/ccpp/framework
diff --git a/ccpp/physics b/ccpp/physics
diff --git a/ccpp/suites/suite_FV3_GFS_2017_coupled.xml b/ccpp/suites/suite_FV3_GFS_2017_coupled.xml
@@ -0,0 +1,88 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<suite name="FV3_GFS_2017_coupled" lib="ccppphys" ver="3">
+  <!-- <init></init> -->
+  <group name="time_vary">
+    <subcycle loop="1">
+      <scheme>GFS_time_vary_pre</scheme>
+      <scheme>GFS_rrtmg_setup</scheme>
+      <scheme>GFS_rad_time_vary</scheme>
+      <scheme>GFS_phys_time_vary</scheme>
+    </subcycle>
+  </group>
+  <group name="radiation">
+    <subcycle loop="1">
+      <scheme>GFS_suite_interstitial_rad_reset</scheme>
+      <scheme>GFS_rrtmg_pre</scheme>
+      <scheme>rrtmg_sw_pre</scheme>
+      <scheme>rrtmg_sw</scheme>
+      <scheme>rrtmg_sw_post</scheme>
+      <scheme>rrtmg_lw_pre</scheme>
+      <scheme>rrtmg_lw</scheme>
+      <scheme>rrtmg_lw_post</scheme>
+      <scheme>GFS_rrtmg_post</scheme>
+    </subcycle>
+  </group>
+  <group name="physics">
+    <subcycle loop="1">
+      <scheme>GFS_suite_interstitial_phys_reset</scheme>
+      <scheme>GFS_suite_stateout_reset</scheme>
+      <scheme>get_prs_fv3</scheme>
+      <scheme>GFS_suite_interstitial_1</scheme>
+      <scheme>GFS_surface_generic_pre</scheme>
+      <scheme>GFS_surface_composites_pre</scheme>
+      <scheme>dcyc2t3</scheme>
+      <scheme>GFS_surface_composites_inter</scheme>
+      <scheme>GFS_suite_interstitial_2</scheme>
+    </subcycle>
+    <!-- Surface iteration loop -->
+    <subcycle loop="2">
+      <scheme>sfc_diff</scheme>
+      <scheme>GFS_surface_loop_control_part1</scheme>
+      <scheme>sfc_ocean</scheme>
+      <scheme>lsm_noah</scheme>
+      <scheme>sfc_cice</scheme>
+      <scheme>sfc_sice</scheme>
+      <scheme>GFS_surface_loop_control_part2</scheme>
+    </subcycle>
+    <!-- End of surface iteration loop -->
+    <subcycle loop="1">
+      <scheme>GFS_surface_composites_post</scheme>
+      <scheme>dcyc2t3_post</scheme>
+      <scheme>sfc_diag</scheme>
+      <scheme>sfc_diag_post</scheme>
+      <scheme>GFS_surface_generic_post</scheme>
+      <scheme>GFS_PBL_generic_pre</scheme>
+      <scheme>hedmf</scheme>
+      <scheme>GFS_PBL_generic_post</scheme>
+      <scheme>GFS_GWD_generic_pre</scheme>
+      <scheme>cires_ugwp</scheme>
+      <scheme>cires_ugwp_post</scheme>
+       <scheme>GFS_GWD_generic_post</scheme>
+      <scheme>rayleigh_damp</scheme>
+      <scheme>GFS_suite_stateout_update</scheme>
+      <scheme>ozphys</scheme>
+      <scheme>GFS_DCNV_generic_pre</scheme>
+      <scheme>get_phi_fv3</scheme>
+      <scheme>GFS_suite_interstitial_3</scheme>
+      <scheme>samfdeepcnv</scheme>
+      <scheme>GFS_DCNV_generic_post</scheme>
+      <scheme>GFS_SCNV_generic_pre</scheme>
+      <scheme>samfshalcnv</scheme>
+      <scheme>GFS_SCNV_generic_post</scheme>
+      <scheme>GFS_suite_interstitial_4</scheme>
+      <scheme>cnvc90</scheme>
+      <scheme>GFS_MP_generic_pre</scheme>
+      <scheme>zhaocarr_gscond</scheme>
+      <scheme>zhaocarr_precpd</scheme>
+      <scheme>GFS_MP_generic_post</scheme>
+      <scheme>maximum_hourly_diagnostics</scheme>
+    </subcycle>
+  </group>
+  <group name="stochastics">
+    <subcycle loop="1">
+      <scheme>GFS_stochastics</scheme>
+    </subcycle>
+  </group>
+  <!-- <finalize></finalize> -->
+</suite>
diff --git a/ccpp/suites/suite_FV3_GFS_2017_gfdlmp_noahmp.xml b/ccpp/suites/suite_FV3_GFS_2017_gfdlmp_noahmp.xml
@@ -47,7 +47,6 @@
       <scheme>sfc_nst_pre</scheme>
       <scheme>sfc_nst</scheme>
       <scheme>sfc_nst_post</scheme>
-      <scheme>sfc_noahmp_pre</scheme>
       <scheme>noahmpdrv</scheme>
       <scheme>sfc_sice</scheme>
       <scheme>GFS_surface_loop_control_part2</scheme>
-Original file line number
+Diff line change
@@ -1,3 +1,4 @@
     *.o
     *.mod
     *.a
+    *.pyc
+3 −2		driver/fvGFS/atmosphere.F90
+3 −2		model/fv_control.F90
+9 −5		tools/fv_restart.F90
+7 −0		CMakeLists.txt
+3 −6		README.md
+5 −5		schemes/check/ccpp_prebuild_config.py
+5 −3		src/CMakeLists.txt
+12 −5		physics/GFS_DCNV_generic.F90
+17 −0		physics/GFS_DCNV_generic.meta
+22 −8		physics/GFS_MP_generic.F90
+53 −0		physics/GFS_MP_generic.meta
+32 −10		physics/GFS_rrtmg_post.F90
+8 −0		physics/GFS_rrtmg_post.meta
+5 −4		physics/GFS_surface_generic.F90
+7 −1		physics/cires_ugwp.F90
+4 −0		physics/cires_ugwp_triggers.F90
+10 −0		physics/docs/ccpp_doxyfile
+348 −30		physics/docs/library.bib
+41 −0		physics/docs/pdftxt/NoahMP.txt
+21 −0		physics/docs/pdftxt/UGWPv0.txt
+2 −0		physics/docs/pdftxt/all_shemes_list.txt
+14 −12		physics/module_MP_FER_HIRES.F90
+1 −1		physics/module_gfdl_cloud_microphys.F90
+32 −1		physics/module_sf_noahmp_glacier.f90
+61 −2		physics/module_sf_noahmplsm.f90
+1 −1		physics/mp_fer_hires.F90
+9 −0		physics/noahmp_tables.f90
+14 −14		physics/samfdeepcnv.f
+9 −6		physics/satmedmfvdifq.F
+37 −7		physics/sfc_noahmp_drv.f
+0 −65		physics/sfc_noahmp_pre.F90
+0 −167		physics/sfc_noahmp_pre.meta
+1 −0		physics/sflx.f
+10 −2		physics/ugwp_driver_v0.F