[production/RRFS.v1] Parallel IO enhancements from GDIT (#818)

* fv3atm changes for parallel io speedup items courtesy of GDIT
(Dan Kokron contributions).

CMakeLists.txt

@@ -58,6 +58,14 @@ if(NOT PARALLEL_NETCDF)
   list(APPEND _fv3atm_defs_private NO_PARALLEL_NETCDF)
 endif()
 
+if(ENABLE_PARALLELRESTART)
+  list(APPEND _fv3atm_defs_private ENABLE_PARALLELRESTART)
+endif()
+
+if(ENABLE_RRFS_WAR)
+  list(APPEND _fv3atm_defs_private ENABLE_RRFS_WAR)
+endif()
+
 if(MOVING_NEST)
   list(APPEND _fv3atm_defs_private MOVING_NEST MOIST_CAPPA USE_COND)
   if(DEBUG)

io/fv3atm_restart_io.F90

@@ -14,8 +14,15 @@ module fv3atm_restart_io_mod
                                 register_axis, register_restart_field, &
                                 register_variable_attribute, register_field, &
                                 read_restart, write_restart, write_data,     &
-                                get_global_io_domain_indices, get_dimension_size
-  use mpp_domains_mod,    only: domain2d
+                                get_global_io_domain_indices, get_dimension_size, &
+                                global_att_exists, get_global_attribute
+#ifdef ENABLE_PARALLELRESTART
+  use mpp_domains_mod,    only: domain2d, mpp_get_domain_tile_commid, mpp_copy_domain, &
+                                  mpp_define_io_domain, mpp_get_layout
+#else
+  use mpp_domains_mod,    only: domain2d, mpp_copy_domain, &
+                                  mpp_define_io_domain, mpp_get_layout
+#endif
   use fv3atm_common_io,   only: create_2d_field_and_add_to_bundle, &
        create_3d_field_and_add_to_bundle, copy_from_gfs_data, axis_type
   use fv3atm_sfc_io
@@ -504,6 +511,7 @@ end subroutine fv3atm_checksum
   !!  Also calculates sncovr if it is not present in the restart file.
   subroutine sfc_prop_restart_read (Sfcprop, Atm_block, Model, fv_domain, warm_start, ignore_rst_cksum)
     use fv3atm_rrfs_sd_io
+    use atmosphere_mod,     only: Atm,mygrid
     implicit none
     !--- interface variable definitions
     type(GFS_sfcprop_type),    intent(inout) :: Sfcprop(:)
@@ -529,12 +537,25 @@ subroutine sfc_prop_restart_read (Sfcprop, Atm_block, Model, fv_domain, warm_sta
 
     type(FmsNetcdfDomainFile_t) :: Oro_restart, Sfc_restart, dust12m_restart, emi_restart, rrfssd_restart
     type(FmsNetcdfDomainFile_t) :: Oro_ls_restart, Oro_ss_restart
+    type(domain2D) :: domain_for_read
+    integer :: read_layout(2)
 
     !--- OROGRAPHY FILE
 
     !--- open file
+#ifdef ENABLE_PARALLELRESTART
+    Oro_restart%use_collective = .true.
+    call mpp_get_layout(Atm(mygrid)%domain, read_layout)
+    call mpp_copy_domain(Atm(mygrid)%domain, domain_for_read)
+    call mpp_define_io_domain(domain_for_read, read_layout)
+    Oro_restart%tile_comm = mpp_get_domain_tile_commid(Atm(mygrid)%domain)
+
+    infile=trim(indir)//'/'//trim(fn_oro)
+    amiopen=open_file(Oro_restart, trim(infile), 'read', domain=domain_for_read, is_restart=.true., dont_add_res_to_filename=.true.)
+#else
     infile=trim(indir)//'/'//trim(fn_oro)
     amiopen=open_file(Oro_restart, trim(infile), 'read', domain=fv_domain, is_restart=.true., dont_add_res_to_filename=.true.)
+#endif
     if (.not.amiopen) call mpp_error( FATAL, 'Error with opening file '//trim(infile) )
 
     call oro%register(Model,Oro_restart,Atm_block)
@@ -640,8 +661,16 @@ subroutine sfc_prop_restart_read (Sfcprop, Atm_block, Model, fv_domain, warm_sta
     !--- SURFACE FILE
 
     !--- open file
+#ifdef ENABLE_PARALLELRESTART
+    Sfc_restart%use_collective = .true.
+    Sfc_restart%tile_comm = mpp_get_domain_tile_commid(Atm(mygrid)%domain)
+
+    infile=trim(indir)//'/'//trim(fn_srf)
+    amiopen=open_file(Sfc_restart, trim(infile), "read", domain=domain_for_read, is_restart=.true., dont_add_res_to_filename=.true.)
+#else
     infile=trim(indir)//'/'//trim(fn_srf)
     amiopen=open_file(Sfc_restart, trim(infile), "read", domain=fv_domain, is_restart=.true., dont_add_res_to_filename=.true.)
+#endif
     if( .not.amiopen ) call mpp_error(FATAL, 'Error opening file'//trim(infile))
 
     if(sfc%allocate_arrays(Model, Atm_block, .true., warm_start)) then
@@ -800,6 +829,7 @@ end subroutine sfc_prop_restart_write
   !! restart variables with the GFDL FMS restart subsystem.
   !! Calls a GFDL FMS routine to restore the data from a restart file.
   subroutine phys_restart_read (GFS_Restart, Atm_block, Model, fv_domain, ignore_rst_cksum)
+    use atmosphere_mod,     only: Atm,mygrid
     implicit none
     !--- interface variable definitions
     type(GFS_restart_type),      intent(in) :: GFS_Restart
@@ -819,6 +849,8 @@ subroutine phys_restart_read (GFS_Restart, Atm_block, Model, fv_domain, ignore_r
 
     type(phy_data_type) :: phy
     type(FmsNetcdfDomainFile_t) :: Phy_restart
+    type(domain2D) :: domain_for_read
+    integer :: read_layout(2)
 
     isc = Atm_block%isc
     iec = Atm_block%iec
@@ -831,7 +863,17 @@ subroutine phys_restart_read (GFS_Restart, Atm_block, Model, fv_domain, ignore_r
 
     !--- open restart file and register axes
     fname = trim(indir)//'/'//trim(fn_phy)
+#ifdef ENABLE_PARALLELRESTART
+    Phy_restart%use_collective = .true.
+    call mpp_get_layout(Atm(mygrid)%domain, read_layout)
+    call mpp_copy_domain(Atm(mygrid)%domain, domain_for_read)
+    call mpp_define_io_domain(domain_for_read, read_layout)
+    Phy_restart%tile_comm = mpp_get_domain_tile_commid(Atm(mygrid)%domain)
+
+    amiopen=open_file(Phy_restart, trim(fname), 'read', domain=domain_for_read, is_restart=.true., dont_add_res_to_filename=.true.)
+#else
     amiopen=open_file(Phy_restart, trim(fname), 'read', domain=fv_domain, is_restart=.true., dont_add_res_to_filename=.true.)
+#endif
     if( amiopen ) then
       call register_axis(Phy_restart, 'xaxis_1', 'X')
       call register_axis(Phy_restart, 'yaxis_1', 'Y')