Parallel netcdf-4 IO option (#1552)

TYPE: new feature

KEYWORDS: parallel netcdf4

SOURCE: Ted Mansell (NOAA/National Severe Storms Lab)

DESCRIPTION OF CHANGES:
This PR adds an I/O option to use the new netcdf capability in 4.7.4 (and later versions) to write both in parallel 
and with variable compression for the same file. Versions of pnetcdf permitted parallel I/O, but did not permit 
standard HDF5 compression. A new I/O library directory, `external/io_netcdfpar`, has been added that is 
modeled on the similar `external/io_netcdf` directory. Users should see the file doc/README.netcdf4par for 
details on configuring and running.

In a nutshell, steps to use this feature:
1. This capability to use HDF5 compression and to have parallel I/O (similar to pnetcdf) requires NetCDF v4.7.4 
or later.
2. The NetCDF library must be built with MPI.
3. Once the NetCDF library / module is chosen and the NETCDF env variable is correctly pointing to the right 
location, then prior to the `configure` step, the user sets a new environment variable. For example, in csh or 
bash `setenv NETCDFPAR $NETCDF` or `export NETCDFPAR=$NETCDF`, respectively.
4. Set the io_form to 13 to use the parallel compressed netcdf option (usually just for the model output or restarts).
5. This I/O option requires activating the NOCOLONS switch (which is why only model output or restarts are 
recommended, since metgrid files have colons imbedded in the file names).

LIST OF MODIFIED FILES: 
M       Makefile
M       Registry/Registry.EM_COMMON
M       arch/Config.pl
M       arch/md_calls.inc
M       arch/postamble
M       arch/preamble
M       configure
A       doc/README.netcdf4par
M       external/Makefile
A       external/io_netcdfpar/Makefile
A       external/io_netcdfpar/diffwrf.F90
A       external/io_netcdfpar/ext_ncdpar_get_dom_ti.code
A       external/io_netcdfpar/ext_ncdpar_get_var_td.code
A       external/io_netcdfpar/ext_ncdpar_get_var_ti.code
A       external/io_netcdfpar/ext_ncdpar_put_dom_ti.code
A       external/io_netcdfpar/ext_ncdpar_put_var_td.code
A       external/io_netcdfpar/ext_ncdpar_put_var_ti.code
A       external/io_netcdfpar/field_routines.F90
A       external/io_netcdfpar/module_wrfsi_static.F90
A       external/io_netcdfpar/transpose.code
A       external/io_netcdfpar/wrf_io.F90
M       frame/md_calls.m4
M       frame/module_io.F
M       share/mediation_integrate.F
M       share/module_io_domain.F
M       share/output_wrf.F
M       share/wrf_ext_write_field.F

TESTS CONDUCTED: 
1. Tested on lustre file system (cray) with domain sizes up to about 700x700. Will leave the existing chunking as
as the initial implementation.
2. Parallel I/O with compression successfully runs on the NCAR cheyenne system, a GPFS file system. The 
following were set up for modules during the build and within the job script:
```
module purge
module load intel
module load ncarcompilers
module load mpt
module load netcdf-mpi
module load ncarenv
```
3. The tests on cheyenne worked with both GNU/10.1.0 and Intel/19.1.1
4. Jenkins testing is OK.
5. Additional information is now output at the end of the `configure` step regarding I/O options:
```
NetCDF version: 4.8.1
Enabled NetCDF-4/HDF-5: yes
NetCDF built with PnetCDF: no
Enabled NetCDF parallel: yes
Using parallel NetCDF via NETCDFPAR option
```

RELEASE NOTE: Added the ability to write compressed NetCDF4 files in parallel via NetCDF 4.7.4 (and later). The performance is slower than pnetcdf, but can be notably faster than regular NetCDF4 on parallel file systems. As expected, the compression provides files significantly smaller than pnetcdf generates.

Makefile

@@ -908,6 +908,18 @@ framework :
                CPP="$(CPP)" LDFLAGS="$(LDFLAGS)" TRADFLAG="$(TRADFLAG)" ESMF_IO_LIB_EXT="$(ESMF_IO_LIB_EXT)" \
 	       LIB_LOCAL="$(LIB_LOCAL)" \
                ESMF_MOD_DEPENDENCE="$(ESMF_MOD_DEPENDENCE)" AR="INTERNAL_BUILD_ERROR_SHOULD_NOT_NEED_AR"; \
+          cd ../io_netcdfpar ; \
+          $(MAKE) NETCDFPARPATH="$(NETCDFPATH)" \
+               FC="$(FC) $(FCBASEOPTS) $(PROMOTION) $(FCDEBUG) $(OMP)" RANLIB="$(RANLIB)" \
+               CPP="$(CPP)" LDFLAGS="$(LDFLAGS)" TRADFLAG="$(TRADFLAG)" ESMF_IO_LIB_EXT="$(ESMF_IO_LIB_EXT)" \
+               LIB_LOCAL="$(LIB_LOCAL)" \
+               ESMF_MOD_DEPENDENCE="$(ESMF_MOD_DEPENDENCE)" AR="INTERNAL_BUILD_ERROR_SHOULD_NOT_NEED_AR" diffwrf; \
+          cd ../io_netcdfpar ; \
+          $(MAKE) NETCDFPARPATH="$(NETCDFPATH)" \
+               FC="$(SFC) $(FCBASEOPTS) $(PROMOTION) $(FCDEBUG) $(OMP)" RANLIB="$(RANLIB)" \
+               CPP="$(CPP)" LDFLAGS="$(LDFLAGS)" TRADFLAG="$(TRADFLAG)" ESMF_IO_LIB_EXT="$(ESMF_IO_LIB_EXT)" \
+	       LIB_LOCAL="$(LIB_LOCAL)" \
+               ESMF_MOD_DEPENDENCE="$(ESMF_MOD_DEPENDENCE)" AR="INTERNAL_BUILD_ERROR_SHOULD_NOT_NEED_AR"; \
           cd ../io_pio ; \
           echo SKIPPING PIO BUILD $(MAKE) NETCDFPATH="$(PNETCDFPATH)" \
                FC="$(SFC) $(FCBASEOPTS) $(PROMOTION) $(FCDEBUG) $(OMP)" RANLIB="$(RANLIB)" \
@@ -1027,7 +1039,11 @@ fseek_test :
 
 # rule used by configure to test if this will compile with netcdf4
 nc4_test:
-	@cd tools ; /bin/rm -f nc4_test.{exe,nc,o} ; $(SCC) -o nc4_test.exe nc4_test.c -I$(NETCDF)/include -L$(NETCDF)/lib $(USENETCDF) ; cd ..
+	if [ $(USENETCDFPAR) -eq 0 ] ; then \
+	 ( cd tools ; /bin/rm -f nc4_test.{exe,nc,o} ; $(SCC) -o nc4_test.exe nc4_test.c -I$(NETCDF)/include -L$(NETCDF)/lib -lnetcdf $(NETCDF4_DEP_LIB) ; cd .. ) ; \
+	else \
+	 ( cd tools ; /bin/rm -f nc4_test.{exe,nc,o} ; $(DM_CC) -o nc4_test.exe nc4_test.c -I$(NETCDF)/include -L$(NETCDF)/lib -lnetcdf $(NETCDF4_DEP_LIB) ; cd ..  ) ; \
+	fi
 
 # rule used by configure to test if Fortran 2003 IEEE signaling is available
 fortran_2003_ieee_test:

Registry/Registry.EM_COMMON

@@ -3290,6 +3290,7 @@ package   io_zzz      io_form_restart==9                     -             -
 package   io_grib2    io_form_restart==10                    -             -
 package   io_pnetcdf  io_form_restart==11                    -             -
 package   io_pio      io_form_restart==12                    -             -
+package   io_netcdfpar io_form_restart==13                   -             -
 
 #WRF Hydro
 package   no_wrfhydro    wrf_hydro==0                -            -

arch/Config.pl

@@ -9,6 +9,7 @@
 $sw_perl_path = perl ;
 $sw_netcdf_path = "" ;
 $sw_pnetcdf_path = "" ;
+$sw_netcdfpar_path = "" ;
 $sw_hdf5_path=""; 
 $sw_phdf5_path=""; 
 $sw_jasperlib_path=""; 
@@ -95,6 +96,10 @@
   {
     $sw_pnetcdf_path = substr( $ARGV[0], 9 ) ;
   }
+  if ( substr( $ARGV[0], 1, 10 ) eq "netcdfpar=" )
+  {
+    $sw_netcdfpar_path = substr( $ARGV[0], 11 ) ;
+  }
   if ( substr( $ARGV[0], 1, 5 ) eq "hdf5=" )
   {
     $sw_hdf5_path = substr( $ARGV[0], 6 ) ;
@@ -459,6 +464,7 @@
     $_ =~ s/CONFIGURE_PERL_PATH/$sw_perl_path/g ;
     $_ =~ s/CONFIGURE_NETCDF_PATH/$sw_netcdf_path/g ;
     $_ =~ s/CONFIGURE_PNETCDF_PATH/$sw_pnetcdf_path/g ;
+    $_ =~ s/CONFIGURE_NETCDFPAR_PATH/$sw_netcdfpar_path/g ;
     $_ =~ s/CONFIGURE_HDF5_PATH/$sw_hdf5_path/g ;
     $_ =~ s/CONFIGURE_PHDF5_PATH/$sw_phdf5_path/g ;
     $_ =~ s/CONFIGURE_LDFLAGS/$sw_ldflags/g ;
@@ -496,6 +502,25 @@
        $_ =~ s/#// ;
        $_ =~ s/#// ;
     }
+
+# put netcdfpar ahead of netcdf so that part of the name does not get clobbered
+    if ( $sw_netcdfpar_path )
+      { $_ =~ s/CONFIGURE_WRFIO_NFPAR/wrfio_nfpar/g ;
+        $_ =~ s:CONFIGURE_NETCDFPAR_FLAG:-DNETCDFPAR: ;
+        if ( $ENV{NETCDFPAR_LDFLAGS} ) {
+          $_ =~ s:CONFIGURE_NETCDFPAR_LIB_PATH:\$\(WRF_SRC_ROOT_DIR\)/external/io_netcdfpar/libwrfio_nfpar.a $ENV{NETCDFPAR_LDFLAGS} : ;
+        } elsif ( $sw_os eq "Interix" ) {
+          $_ =~ s:CONFIGURE_NETCDFPAR_LIB_PATH:\$\(WRF_SRC_ROOT_DIR\)/external/io_netcdfpar/libwrfio_nfpar.a -L$sw_netcdfpar_path/lib $sw_usenetcdff $sw_usenetcdf : ;
+        } else {
+          $_ =~ s:CONFIGURE_NETCDFPAR_LIB_PATH:-L\$\(WRF_SRC_ROOT_DIR\)/external/io_netcdfpar -lwrfio_nfpar -L$sw_netcdfpar_path/lib $sw_usenetcdff $sw_usenetcdf : ;
+        }
+         }
+    else
+      { $_ =~ s/CONFIGURE_WRFIO_NFPAR//g ;
+        $_ =~ s:CONFIGURE_NETCDFPAR_FLAG::g ;
+        $_ =~ s:CONFIGURE_NETCDFPAR_LIB_PATH::g ;
+         }
+
     if ( $sw_netcdf_path ) 
       { $_ =~ s/CONFIGURE_WRFIO_NF/wrfio_nf/g ;
 	$_ =~ s:CONFIGURE_NETCDF_FLAG:-DNETCDF: ;
@@ -837,6 +862,25 @@
     if ( $sw_os ne "CYGWIN_NT" ) {
       $_ =~ s/#NOWIN// ;
     }
+
+    if ( $sw_netcdfpar_path )
+      { #print("set sw_netcdfpar_path stuff\n");
+        $_ =~ s/CONFIGURE_WRFIO_NFPAR/wrfio_nfpar/g ;
+        $_ =~ s:CONFIGURE_NETCDFPAR_FLAG:-DNETCDFPAR: ;
+        if ( $ENV{NETCDFPAR_LDFLAGS} ) {
+          $_ =~ s:CONFIGURE_NETCDFPAR_LIB_PATH:\$\(WRF_SRC_ROOT_DIR\)/external/io_netcdfpar/libwrfio_nfpar.a $ENV{NETCDFPAR_LDFLAGS} : ;
+        } elsif ( $sw_os eq "Interix" ) {
+          $_ =~ s:CONFIGURE_NETCDFPAR_LIB_PATH:\$\(WRF_SRC_ROOT_DIR\)/external/io_netcdfpar/libwrfio_nfpar.a -L$sw_netcdfpar_path/lib $sw_usenetcdff $sw_usenetcdf : ;
+        } else {
+          $_ =~ s:CONFIGURE_NETCDFPAR_LIB_PATH:-L\$\(WRF_SRC_ROOT_DIR\)/external/io_netcdfpar -lwrfio_nfpar -L$sw_netcdfpar_path/lib $sw_usenetcdff $sw_usenetcdf : ;
+        }
+         }
+    else
+      { $_ =~ s/CONFIGURE_WRFIO_NFPAR//g ;
+        $_ =~ s:CONFIGURE_NETCDFPAR_FLAG::g ;
+        $_ =~ s:CONFIGURE_NETCDFPAR_LIB_PATH::g ;
+         }
+
     if ( $sw_netcdf_path )
       { $_ =~ s/CONFIGURE_WRFIO_NF/wrfio_nf/g ;
 	$_ =~ s:CONFIGURE_NETCDF_FLAG:-DNETCDF: ;