Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

parallel netcdf IO #41

Merged
merged 33 commits into from
Feb 4, 2020
Merged

parallel netcdf IO #41

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
33 commits
Select commit Hold shift + click to select a range
45d0ce2
netcdf parallel writing and lon/lat in netcdf file
junwang-noaa Jan 8, 2020
6b9b865
some changes to get it to (almost) run on hera
jswhit2 Jan 8, 2020
ccdbf27
lat should be in degrees - remove conversion to radians.
jswhit2 Jan 9, 2020
b3e52a0
import updates from jswhit/fv3atm
jswhit2 Jan 9, 2020
a201d6c
more bug fixes - now works on hera
jswhit2 Jan 9, 2020
656c546
specify collective access if compression turned on.
jswhit2 Jan 9, 2020
ade86ed
use classic model
jswhit2 Jan 9, 2020
49ffb5e
bug fixes for parallel IO with compression
jswhit2 Jan 10, 2020
134ebc0
fix calculation of max compression error
jswhit2 Jan 10, 2020
5fce869
turn off shuffle filter
jswhit2 Jan 10, 2020
30516fc
code simplification
jswhit2 Jan 10, 2020
930d622
remove debug print
jswhit2 Jan 17, 2020
6cb687d
don't use parallel IO for 2d file (since it seems to increase run time)
jswhit2 Jan 22, 2020
c883717
allow multiple values of output_file, as long as they all start with …
jswhit2 Jan 23, 2020
ffb33e0
use default chunksize for 2d vars
jswhit2 Jan 24, 2020
89184bd
delete commented out macro ESMF_ERR_ABORT
jswhit2 Jan 24, 2020
5af9ccf
delete rad2dg
jswhit2 Jan 24, 2020
b49ef58
add module_write_netcdf_parallel.F90
jswhit2 Jan 25, 2020
2e63f49
add option to build without parallel netcdf (-DNO_PARALLEL_NETCDF)
jswhit2 Jan 25, 2020
4a45855
fix typo
jswhit2 Jan 25, 2020
14dfcab
stub file for building without parallel netcdf lib
jswhit2 Jan 26, 2020
5beba44
Merge remote-tracking branch 'upstream/develop' into netcdf_parallel
jswhit2 Jan 26, 2020
dab1dd8
allow chunksizes for 2d arrays to be set in model_configure (ichunk2d…
jswhit2 Jan 26, 2020
a6c36fb
add ichunk3d,jchunk3d,kchunk3d to specify 3d chunksizes.
jswhit2 Jan 27, 2020
1cf5a6c
fix typo
jswhit2 Jan 27, 2020
a3f0306
put ifdefs in module_write_netcdf_parallel.F90 so no stub file needed
jswhit2 Jan 27, 2020
1525cc0
don't need this file anymore
jswhit2 Jan 27, 2020
51d3dc1
remove module_write_netcdf_parallel_stub.o target
jswhit2 Jan 27, 2020
8ddde89
use specified chunksizes for serial IO. If chunksize parameter negati…
jswhit2 Jan 27, 2020
98948bd
update comments
jswhit2 Jan 27, 2020
45d35d5
get output_file without esmf call error
junwang-noaa Jan 30, 2020
a0f9ffa
syntax fix
junwang-noaa Jan 30, 2020
715f1ae
fix stub interface in module_write_netcdf_parallel.F90 for cmake build
junwang-noaa Feb 1, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion fv3_cap.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ module fv3gfs_cap_mod
wrttasks_per_group, n_group, &
lead_wrttask, last_wrttask, &
output_grid, output_file, &
imo, jmo, write_nemsioflip, &
imo,jmo,ichunk2d,jchunk2d,write_nemsioflip,&
write_fsyncflag, nsout_io, &
cen_lon, cen_lat, ideflate, &
lon1, lat1, lon2, lat2, dlon, dlat, &
Expand Down Expand Up @@ -307,6 +307,11 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc)
call ESMF_ConfigGetAttribute(config=CF,value=iau_offset,default=0,label ='iau_offset:',rc=rc)
if (iau_offset < 0) iau_offset=0

! chunksizes for netcdf_parallel
call ESMF_ConfigGetAttribute(config=CF,value=ichunk2d,default=0,label ='ichunk2d:',rc=rc)
call ESMF_ConfigGetAttribute(config=CF,value=jchunk2d,default=0,label ='jchunk2d:',rc=rc)

! zlib compression flag
call ESMF_ConfigGetAttribute(config=CF,value=ideflate,default=0,label ='ideflate:',rc=rc)
if (ideflate < 0) ideflate=0

Expand Down
1 change: 1 addition & 0 deletions io/module_fv3_io_def.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ module module_fv3_io_def
character(255) :: app_domain
character(255) :: output_grid
integer :: imo,jmo
integer :: ichunk2d,jchunk2d
integer :: nbdlphys
integer :: nsout_io, iau_offset, ideflate, nbits
real :: cen_lon, cen_lat, lon1, lat1, lon2, lat2, dlon, dlat
Expand Down
9 changes: 5 additions & 4 deletions io/module_write_netcdf_parallel.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,14 +19,14 @@ module module_write_netcdf_parallel
contains

!----------------------------------------------------------------------------------------
subroutine write_netcdf_parallel(fieldbundle, wrtfb, filename, mpi_comm, mype, im, jm, rc)
subroutine write_netcdf_parallel(fieldbundle, wrtfb, filename, mpi_comm, mype, im, jm, ichunk2d, jchunk2d, rc)
!
type(ESMF_FieldBundle), intent(in) :: fieldbundle
type(ESMF_FieldBundle), intent(in) :: wrtfb
character(*), intent(in) :: filename
integer, intent(in) :: mpi_comm
integer, intent(in) :: mype
integer, intent(in) :: im, jm
integer, intent(in) :: im, jm, ichunk2d, jchunk2d
integer, optional,intent(out) :: rc
!
!** local vars
Expand Down Expand Up @@ -149,7 +149,8 @@ subroutine write_netcdf_parallel(fieldbundle, wrtfb, filename, mpi_comm, mype, i
if (ideflate > 0) then
ncerr = nf90_def_var(ncid, trim(fldName), NF90_FLOAT, &
(/im_dimid,jm_dimid,time_dimid/), varids(i), &
shuffle=.true.,deflate_level=ideflate); NC_ERR_STOP(ncerr)
shuffle=.false.,deflate_level=ideflate,&
chunksizes=(/ichunk2d,jchunk2d,1/)); NC_ERR_STOP(ncerr)
! compression filters require collective access.
ncerr = nf90_var_par_access(ncid, varids(i), NF90_COLLECTIVE)
else
Expand All @@ -171,7 +172,7 @@ subroutine write_netcdf_parallel(fieldbundle, wrtfb, filename, mpi_comm, mype, i
ncerr = nf90_def_var(ncid, trim(fldName), NF90_FLOAT, &
(/im_dimid,jm_dimid,pfull_dimid,time_dimid/), varids(i), &
shuffle=.false.,deflate_level=ideflate,&
chunksizes=(/im,jm,1,1/),cache_size=40*im*jm); NC_ERR_STOP(ncerr)
chunksizes=(/im,jm,1,1/)); NC_ERR_STOP(ncerr)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Must two of the chunksizes be 1?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No, but this seems to be the most common access pattern (reading a 2d horizontal slice).

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Now changed so that chunksize can be specified at run time. Default is to use MPI decompisition size..

! compression filters require collective access.
ncerr = nf90_var_par_access(ncid, varids(i), NF90_COLLECTIVE)
else
Expand Down
17 changes: 14 additions & 3 deletions io/module_wrt_grid_comp.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ module module_wrt_grid_comp
use module_fv3_io_def, only : num_pes_fcst,lead_wrttask, last_wrttask, &
n_group, num_files, app_domain, &
filename_base, output_grid, output_file, &
imo, jmo, write_nemsioflip, &
imo,jmo,ichunk2d,jchunk2d,write_nemsioflip,&
nsout => nsout_io, &
cen_lon, cen_lat, &
lon1, lat1, lon2, lat2, dlon, dlat, &
Expand Down Expand Up @@ -1444,10 +1444,21 @@ subroutine wrt_run(wrt_comp, imp_state_write, exp_state_write,clock,rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, &
line=__LINE__, file=__FILE__)) return
#endif

! set default chunksizes for 2d arrays
if (ichunk2d <= 0) then
ichunk2d = wrt_int_state%lon_end-wrt_int_state%lon_start+1
call mpi_bcast(ichunk2d,1,mpi_integer,0,wrt_mpi_comm,rc)
endif
if (jchunk2d <= 0) then
jchunk2d = wrt_int_state%lat_end-wrt_int_state%lat_start+1
call mpi_bcast(jchunk2d,1,mpi_integer,0,wrt_mpi_comm,rc)
endif
if (wrt_int_state%mype == 0) then
print *,'ichunk2d,jchunk2d',ichunk2d,jchunk2d
endif
wbeg = MPI_Wtime()
call write_netcdf_parallel(file_bundle,wrt_int_state%wrtFB(nbdl), &
trim(filename), wrt_mpi_comm,wrt_int_state%mype,imo,jmo,rc)
trim(filename), wrt_mpi_comm,wrt_int_state%mype,imo,jmo,ichunk2d,jchunk2d,rc)
wend = MPI_Wtime()
if (lprnt) then
write(*,'(A,F10.5,A,I4.2,A,I2.2)')' parallel netcdf Write Time is ',wend-wbeg &
Expand Down