From ec0ae9eb9b1cfc5954ef300a5b0c582e94c14519 Mon Sep 17 00:00:00 2001 From: MatthewPyle-NOAA <48285220+MatthewPyle-NOAA@users.noreply.github.com> Date: Fri, 3 May 2024 12:22:15 -0400 Subject: [PATCH] [production/RRFS.v1] Parallel IO enhancements from GDIT (#818) * fv3atm changes for parallel io speedup items courtesy of GDIT (Dan Kokron contributions). --- CMakeLists.txt | 8 +++++++ atmos_cubed_sphere | 2 +- io/fv3atm_restart_io.F90 | 46 ++++++++++++++++++++++++++++++++++++++-- 3 files changed, 53 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5d0dcf6f6..89426b531 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -58,6 +58,14 @@ if(NOT PARALLEL_NETCDF) list(APPEND _fv3atm_defs_private NO_PARALLEL_NETCDF) endif() +if(ENABLE_PARALLELRESTART) + list(APPEND _fv3atm_defs_private ENABLE_PARALLELRESTART) +endif() + +if(ENABLE_RRFS_WAR) + list(APPEND _fv3atm_defs_private ENABLE_RRFS_WAR) +endif() + if(MOVING_NEST) list(APPEND _fv3atm_defs_private MOVING_NEST MOIST_CAPPA USE_COND) if(DEBUG) diff --git a/atmos_cubed_sphere b/atmos_cubed_sphere index fab198c58..84c30284b 160000 --- a/atmos_cubed_sphere +++ b/atmos_cubed_sphere @@ -1 +1 @@ -Subproject commit fab198c5815d707a8e8421e553d199eff945ffa9 +Subproject commit 84c30284b2a1e7efb4c6ef9c05566c1a27d1a8fd diff --git a/io/fv3atm_restart_io.F90 b/io/fv3atm_restart_io.F90 index d32be0586..7dc8460ba 100644 --- a/io/fv3atm_restart_io.F90 +++ b/io/fv3atm_restart_io.F90 @@ -14,8 +14,15 @@ module fv3atm_restart_io_mod register_axis, register_restart_field, & register_variable_attribute, register_field, & read_restart, write_restart, write_data, & - get_global_io_domain_indices, get_dimension_size - use mpp_domains_mod, only: domain2d + get_global_io_domain_indices, get_dimension_size, & + global_att_exists, get_global_attribute +#ifdef ENABLE_PARALLELRESTART + use mpp_domains_mod, only: domain2d, mpp_get_domain_tile_commid, mpp_copy_domain, & + mpp_define_io_domain, mpp_get_layout +#else + use mpp_domains_mod, only: domain2d, mpp_copy_domain, & + mpp_define_io_domain, mpp_get_layout +#endif use fv3atm_common_io, only: create_2d_field_and_add_to_bundle, & create_3d_field_and_add_to_bundle, copy_from_gfs_data, axis_type use fv3atm_sfc_io @@ -504,6 +511,7 @@ end subroutine fv3atm_checksum !! Also calculates sncovr if it is not present in the restart file. subroutine sfc_prop_restart_read (Sfcprop, Atm_block, Model, fv_domain, warm_start, ignore_rst_cksum) use fv3atm_rrfs_sd_io + use atmosphere_mod, only: Atm,mygrid implicit none !--- interface variable definitions type(GFS_sfcprop_type), intent(inout) :: Sfcprop(:) @@ -529,12 +537,25 @@ subroutine sfc_prop_restart_read (Sfcprop, Atm_block, Model, fv_domain, warm_sta type(FmsNetcdfDomainFile_t) :: Oro_restart, Sfc_restart, dust12m_restart, emi_restart, rrfssd_restart type(FmsNetcdfDomainFile_t) :: Oro_ls_restart, Oro_ss_restart + type(domain2D) :: domain_for_read + integer :: read_layout(2) !--- OROGRAPHY FILE !--- open file +#ifdef ENABLE_PARALLELRESTART + Oro_restart%use_collective = .true. + call mpp_get_layout(Atm(mygrid)%domain, read_layout) + call mpp_copy_domain(Atm(mygrid)%domain, domain_for_read) + call mpp_define_io_domain(domain_for_read, read_layout) + Oro_restart%tile_comm = mpp_get_domain_tile_commid(Atm(mygrid)%domain) + + infile=trim(indir)//'/'//trim(fn_oro) + amiopen=open_file(Oro_restart, trim(infile), 'read', domain=domain_for_read, is_restart=.true., dont_add_res_to_filename=.true.) +#else infile=trim(indir)//'/'//trim(fn_oro) amiopen=open_file(Oro_restart, trim(infile), 'read', domain=fv_domain, is_restart=.true., dont_add_res_to_filename=.true.) +#endif if (.not.amiopen) call mpp_error( FATAL, 'Error with opening file '//trim(infile) ) call oro%register(Model,Oro_restart,Atm_block) @@ -640,8 +661,16 @@ subroutine sfc_prop_restart_read (Sfcprop, Atm_block, Model, fv_domain, warm_sta !--- SURFACE FILE !--- open file +#ifdef ENABLE_PARALLELRESTART + Sfc_restart%use_collective = .true. + Sfc_restart%tile_comm = mpp_get_domain_tile_commid(Atm(mygrid)%domain) + + infile=trim(indir)//'/'//trim(fn_srf) + amiopen=open_file(Sfc_restart, trim(infile), "read", domain=domain_for_read, is_restart=.true., dont_add_res_to_filename=.true.) +#else infile=trim(indir)//'/'//trim(fn_srf) amiopen=open_file(Sfc_restart, trim(infile), "read", domain=fv_domain, is_restart=.true., dont_add_res_to_filename=.true.) +#endif if( .not.amiopen ) call mpp_error(FATAL, 'Error opening file'//trim(infile)) if(sfc%allocate_arrays(Model, Atm_block, .true., warm_start)) then @@ -800,6 +829,7 @@ end subroutine sfc_prop_restart_write !! restart variables with the GFDL FMS restart subsystem. !! Calls a GFDL FMS routine to restore the data from a restart file. subroutine phys_restart_read (GFS_Restart, Atm_block, Model, fv_domain, ignore_rst_cksum) + use atmosphere_mod, only: Atm,mygrid implicit none !--- interface variable definitions type(GFS_restart_type), intent(in) :: GFS_Restart @@ -819,6 +849,8 @@ subroutine phys_restart_read (GFS_Restart, Atm_block, Model, fv_domain, ignore_r type(phy_data_type) :: phy type(FmsNetcdfDomainFile_t) :: Phy_restart + type(domain2D) :: domain_for_read + integer :: read_layout(2) isc = Atm_block%isc iec = Atm_block%iec @@ -831,7 +863,17 @@ subroutine phys_restart_read (GFS_Restart, Atm_block, Model, fv_domain, ignore_r !--- open restart file and register axes fname = trim(indir)//'/'//trim(fn_phy) +#ifdef ENABLE_PARALLELRESTART + Phy_restart%use_collective = .true. + call mpp_get_layout(Atm(mygrid)%domain, read_layout) + call mpp_copy_domain(Atm(mygrid)%domain, domain_for_read) + call mpp_define_io_domain(domain_for_read, read_layout) + Phy_restart%tile_comm = mpp_get_domain_tile_commid(Atm(mygrid)%domain) + + amiopen=open_file(Phy_restart, trim(fname), 'read', domain=domain_for_read, is_restart=.true., dont_add_res_to_filename=.true.) +#else amiopen=open_file(Phy_restart, trim(fname), 'read', domain=fv_domain, is_restart=.true., dont_add_res_to_filename=.true.) +#endif if( amiopen ) then call register_axis(Phy_restart, 'xaxis_1', 'X') call register_axis(Phy_restart, 'yaxis_1', 'Y')