From 90777d6465ebef36f3f611265311c65e587b2ef7 Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Wed, 8 Jul 2020 12:17:37 -0600 Subject: [PATCH 1/9] added new test tst_h_par_compress.c --- h5_test/Makefile.am | 2 +- h5_test/tst_h_par_compress.c | 245 +++++++++++++++++++++++++++++++++++ 2 files changed, 246 insertions(+), 1 deletion(-) create mode 100644 h5_test/tst_h_par_compress.c diff --git a/h5_test/Makefile.am b/h5_test/Makefile.am index dfea153bb4..115709c0b7 100644 --- a/h5_test/Makefile.am +++ b/h5_test/Makefile.am @@ -34,7 +34,7 @@ TESTS = $(H5TESTS) # The parallel program is run from a script. if TEST_PARALLEL4 -check_PROGRAMS += tst_h_par +check_PROGRAMS += tst_h_par tst_h_par_compress TESTS += run_par_tests.sh endif diff --git a/h5_test/tst_h_par_compress.c b/h5_test/tst_h_par_compress.c new file mode 100644 index 0000000000..3cd89b8f61 --- /dev/null +++ b/h5_test/tst_h_par_compress.c @@ -0,0 +1,245 @@ +/* This is part of the netCDF package. Copyright 2020 University + Corporation for Atmospheric Research/Unidata See COPYRIGHT file for + conditions of use. + + Test HDF5 file code. These are not intended to be exhaustive tests, + but they use HDF5 the same way that netCDF-4 does, so if these + tests don't work, than netCDF-4 won't work either. + + This files tests parallel I/O using compression filters. This + functionality is only available in HDF5-1.10.3 and later versions. + + Ed Hartnett +*/ +#include +#include "err_macros.h" +#include + +/* Defining USE_MPE causes the MPE trace library to be used (and you + * must also relink with -llmpe -lmpe). This causes clog2 output to be + * written, which can be converted to slog2 (by the program + * clog2TOslog2) and then used in the analysis program jumpshot. */ +/*#define USE_MPE 1*/ + +#ifdef USE_MPE +#include +#endif /* USE_MPE */ + +#define FILE_NAME "tst_h_par_compress.h5" +#define VAR_NAME "HALs_memory" +#define NDIMS 1 +#define MILLION 1000000 +#define DIM2_LEN 16000000 +#define SC1 100000 /* slice count. */ + +int +main(int argc, char **argv) +{ + int p, my_rank; + +#ifdef USE_MPE + int s_init, e_init, s_define, e_define, s_write, e_write, s_close, e_close; +#endif /* USE_MPE */ + + MPI_Init(&argc, &argv); + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + MPI_Comm_size(MPI_COMM_WORLD, &p); + +#ifdef USE_MPE + MPE_Init_log(); + s_init = MPE_Log_get_event_number(); + e_init = MPE_Log_get_event_number(); + s_define = MPE_Log_get_event_number(); + e_define = MPE_Log_get_event_number(); + s_write = MPE_Log_get_event_number(); + e_write = MPE_Log_get_event_number(); + s_close = MPE_Log_get_event_number(); + e_close = MPE_Log_get_event_number(); + MPE_Describe_state(s_init, e_init, "Init", "red"); + MPE_Describe_state(s_define, e_define, "Define", "yellow"); + MPE_Describe_state(s_write, e_write, "Write", "green"); + MPE_Describe_state(s_close, e_close, "Close", "purple"); + MPE_Start_log(); + MPE_Log_event(s_init, 0, "start init"); +#endif /* USE_MPE */ + + /* For builds with HDF5 prior to 1.10.3, just return success. */ +#ifdef HDF5_SUPPORTS_PAR_FILTERS + if (!my_rank) + printf("*** Creating file for parallel I/O read, and rereading it..."); + { + hid_t fapl_id, fileid, whole_spaceid, dsid, slice_spaceid, whole_spaceid1, xferid; + hsize_t start[NDIMS], count[NDIMS]; + hsize_t dims[1]; + int data[SC1], data_in[SC1]; + int num_steps; + double ftime; + int write_us, read_us; + int max_write_us, max_read_us; + float write_rate, read_rate; + int i, s; + + /* We will write the same slice of random data over and over to + * fill the file. */ + for (i = 0; i < SC1; i++) + data[i] = rand(); + +#ifdef USE_MPE + MPE_Log_event(e_init, 0, "end init"); + MPE_Log_event(s_define, 0, "start define file"); +#endif /* USE_MPE */ + + /* Create file. */ + if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) ERR; + if (H5Pset_fapl_mpio(fapl_id, MPI_COMM_WORLD, MPI_INFO_NULL) < 0) ERR; + if ((fileid = H5Fcreate(FILE_NAME, H5F_ACC_TRUNC, H5P_DEFAULT, + fapl_id)) < 0) ERR; + + /* Create a space to deal with one slice in memory. */ + dims[0] = SC1; + if ((slice_spaceid = H5Screate_simple(NDIMS, dims, NULL)) < 0) ERR; + + /* Create a space to write all slices. */ + dims[0] = DIM2_LEN; + if ((whole_spaceid = H5Screate_simple(NDIMS, dims, NULL)) < 0) ERR; + + /* Create dataset. */ + if ((dsid = H5Dcreate1(fileid, VAR_NAME, H5T_NATIVE_INT, + whole_spaceid, H5P_DEFAULT)) < 0) ERR; + + /* Use collective write operations. */ + if ((xferid = H5Pcreate(H5P_DATASET_XFER)) < 0) ERR; + if (H5Pset_dxpl_mpio(xferid, H5FD_MPIO_COLLECTIVE) < 0) ERR; + +#ifdef USE_MPE + MPE_Log_event(e_define, 0, "end define file"); + if (my_rank) + sleep(my_rank); +#endif /* USE_MPE */ + + /* Write the data in num_step steps. */ + ftime = MPI_Wtime(); + num_steps = (DIM2_LEN/SC1) / p; + for (s = 0; s < num_steps; s++) + { +#ifdef USE_MPE + MPE_Log_event(s_write, 0, "start write slab"); +#endif /* USE_MPE */ + + /* Select hyperslab for write of one slice. */ + start[0] = s * SC1 * p + my_rank * SC1; + count[0] = SC1; + if (H5Sselect_hyperslab(whole_spaceid, H5S_SELECT_SET, + start, NULL, count, NULL) < 0) ERR; + + if (H5Dwrite(dsid, H5T_NATIVE_INT, slice_spaceid, whole_spaceid, + xferid, data) < 0) ERR; + +#ifdef USE_MPE + MPE_Log_event(e_write, 0, "end write file"); +#endif /* USE_MPE */ + } + write_us = (MPI_Wtime() - ftime) * MILLION; + MPI_Reduce(&write_us, &max_write_us, 1, MPI_INT, MPI_MAX, 0, MPI_COMM_WORLD); + if (!my_rank) + { + write_rate = (float)(DIM2_LEN * sizeof(int))/(float)max_write_us; + printf("\np=%d, write_rate=%g", p, write_rate); + } + +#ifdef USE_MPE + MPE_Log_event(s_close, 0, "start close file"); +#endif /* USE_MPE */ + + /* Close. These collective operations will allow every process + * to catch up. */ + if (H5Dclose(dsid) < 0 || + H5Sclose(whole_spaceid) < 0 || + H5Sclose(slice_spaceid) < 0 || + H5Pclose(fapl_id) < 0 || + H5Fclose(fileid) < 0) + ERR; + +#ifdef USE_MPE + MPE_Log_event(e_close, 0, "end close file"); +#endif /* USE_MPE */ + + /* Open the file. */ + if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) ERR; + if (H5Pset_fapl_mpio(fapl_id, MPI_COMM_WORLD, MPI_INFO_NULL) < 0) ERR; + + + if (H5Pset_libver_bounds(fapl_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) ERR; + if ((fileid = H5Fopen(FILE_NAME, H5F_ACC_RDONLY, fapl_id)) < 0) ERR; + + /* Create a space to deal with one slice in memory. */ + dims[0] = SC1; + if ((slice_spaceid = H5Screate_simple(NDIMS, dims, NULL)) < 0) ERR; + + /* Open the dataset. */ + if ((dsid = H5Dopen(fileid, VAR_NAME)) < 0) ERR; + if ((whole_spaceid1 = H5Dget_space(dsid)) < 0) ERR; + + ftime = MPI_Wtime(); + + /* Read the data, a slice at a time. */ + for (s = 0; s < num_steps; s++) + { + /* Select hyperslab for read of one slice. */ + start[0] = s * SC1 * p + my_rank * SC1; + count[0] = SC1; + if (H5Sselect_hyperslab(whole_spaceid1, H5S_SELECT_SET, + start, NULL, count, NULL) < 0) + { + ERR; + return 2; + } + + if (H5Dread(dsid, H5T_NATIVE_INT, slice_spaceid, whole_spaceid1, + H5P_DEFAULT, data_in) < 0) + { + ERR; + return 2; + } + + /* Check the slice of data. */ + for (i = 0; i < SC1; i++) + if (data[i] != data_in[i]) + { + ERR; + return 2; + } + } + read_us = (MPI_Wtime() - ftime) * MILLION; + MPI_Reduce(&read_us, &max_read_us, 1, MPI_INT, MPI_MAX, 0, MPI_COMM_WORLD); + if (!my_rank) + { + read_rate = (float)(DIM2_LEN * sizeof(int))/(float)max_read_us; + printf(", read_rate=%g\n", read_rate); + } + + /* Close down. */ + if (H5Dclose(dsid) < 0 || + H5Sclose(slice_spaceid) < 0 || + H5Sclose(whole_spaceid1) < 0 || + H5Pclose(fapl_id) < 0 || + H5Fclose(fileid) < 0) + ERR; + } +#else + { + if (!my_rank) + printf("*** HDF5 1.10.3 or greater required for this test..."); + } + +#endif /* HDF5_SUPPORTS_PAR_FILTERS */ + + if (!my_rank) + SUMMARIZE_ERR; + + MPI_Finalize(); + + if (!my_rank) + FINAL_RESULTS; + return 0; +} From 1d79a47c6d5ac0c5d5de82152d3a1794044dc978 Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Wed, 8 Jul 2020 12:24:48 -0600 Subject: [PATCH 2/9] fixed warning in tst_parallel5.c --- h5_test/run_par_tests.sh.in | 6 ++++++ nc_test4/tst_parallel5.c | 3 +-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/h5_test/run_par_tests.sh.in b/h5_test/run_par_tests.sh.in index 70716aec80..d8f6297a86 100644 --- a/h5_test/run_par_tests.sh.in +++ b/h5_test/run_par_tests.sh.in @@ -17,4 +17,10 @@ echo "Testing parallel I/O with HDF5..." @MPIEXEC@ -n 4 ./tst_h_par echo "SUCCESS!!!" +echo "Testing parallel I/O with HDF5 and compression..." +@MPIEXEC@ -n 1 ./tst_h_par_compress +@MPIEXEC@ -n 2 ./tst_h_par_compress +@MPIEXEC@ -n 4 ./tst_h_par_compress +echo "SUCCESS!!!" + exit 0 diff --git a/nc_test4/tst_parallel5.c b/nc_test4/tst_parallel5.c index ba9cf5242b..9c2e490548 100644 --- a/nc_test4/tst_parallel5.c +++ b/nc_test4/tst_parallel5.c @@ -322,8 +322,7 @@ main(int argc, char **argv) { /* This test is related to * https://github.com/Unidata/netcdf-c/issues/1715. */ - int ncid, varid; - int test_data_in, test_data = 42; + int ncid; size_t size, nelems; float preemption; From c0b6792515dc6e915883ea432a1e49cf33ae0a9c Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Wed, 8 Jul 2020 12:36:02 -0600 Subject: [PATCH 3/9] changed tst_h_par_compress to use H5Dcreate2() just like code in nc4hdf5.c does --- h5_test/tst_h_par_compress.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/h5_test/tst_h_par_compress.c b/h5_test/tst_h_par_compress.c index 3cd89b8f61..c4240353ea 100644 --- a/h5_test/tst_h_par_compress.c +++ b/h5_test/tst_h_par_compress.c @@ -66,7 +66,7 @@ main(int argc, char **argv) /* For builds with HDF5 prior to 1.10.3, just return success. */ #ifdef HDF5_SUPPORTS_PAR_FILTERS if (!my_rank) - printf("*** Creating file for parallel I/O read, and rereading it..."); + printf("*** Creating file for parallel I/O read with compression, and rereading it..."); { hid_t fapl_id, fileid, whole_spaceid, dsid, slice_spaceid, whole_spaceid1, xferid; hsize_t start[NDIMS], count[NDIMS]; @@ -104,8 +104,8 @@ main(int argc, char **argv) if ((whole_spaceid = H5Screate_simple(NDIMS, dims, NULL)) < 0) ERR; /* Create dataset. */ - if ((dsid = H5Dcreate1(fileid, VAR_NAME, H5T_NATIVE_INT, - whole_spaceid, H5P_DEFAULT)) < 0) ERR; + if ((dsid = H5Dcreate2(fileid, VAR_NAME, H5T_NATIVE_INT, + whole_spaceid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) ERR; /* Use collective write operations. */ if ((xferid = H5Pcreate(H5P_DATASET_XFER)) < 0) ERR; From d1a13b28cc6d2551a223861a2a2e3c257c628958 Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Wed, 8 Jul 2020 12:39:35 -0600 Subject: [PATCH 4/9] adding property list for dataset creation to tst_h_par_compress.c --- h5_test/tst_h_par_compress.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/h5_test/tst_h_par_compress.c b/h5_test/tst_h_par_compress.c index c4240353ea..192c063f1c 100644 --- a/h5_test/tst_h_par_compress.c +++ b/h5_test/tst_h_par_compress.c @@ -69,6 +69,7 @@ main(int argc, char **argv) printf("*** Creating file for parallel I/O read with compression, and rereading it..."); { hid_t fapl_id, fileid, whole_spaceid, dsid, slice_spaceid, whole_spaceid1, xferid; + hid_t plistid; hsize_t start[NDIMS], count[NDIMS]; hsize_t dims[1]; int data[SC1], data_in[SC1]; @@ -104,8 +105,10 @@ main(int argc, char **argv) if ((whole_spaceid = H5Screate_simple(NDIMS, dims, NULL)) < 0) ERR; /* Create dataset. */ + if ((plistid = H5Pcreate(H5P_DATASET_CREATE)) < 0) ERR; + if ((dsid = H5Dcreate2(fileid, VAR_NAME, H5T_NATIVE_INT, - whole_spaceid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) ERR; + whole_spaceid, H5P_DEFAULT, plistid, H5P_DEFAULT)) < 0) ERR; /* Use collective write operations. */ if ((xferid = H5Pcreate(H5P_DATASET_XFER)) < 0) ERR; @@ -157,6 +160,7 @@ main(int argc, char **argv) H5Sclose(whole_spaceid) < 0 || H5Sclose(slice_spaceid) < 0 || H5Pclose(fapl_id) < 0 || + H5Pclose(plistid) < 0 || H5Fclose(fileid) < 0) ERR; From 4936edd840acf18c7d2293d9931664f4df8846f5 Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Wed, 8 Jul 2020 12:56:34 -0600 Subject: [PATCH 5/9] adding other settings of dataset property list to match what is in nc4hdf5.c --- h5_test/tst_h_par_compress.c | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/h5_test/tst_h_par_compress.c b/h5_test/tst_h_par_compress.c index 192c063f1c..a493c7d0d0 100644 --- a/h5_test/tst_h_par_compress.c +++ b/h5_test/tst_h_par_compress.c @@ -71,13 +71,14 @@ main(int argc, char **argv) hid_t fapl_id, fileid, whole_spaceid, dsid, slice_spaceid, whole_spaceid1, xferid; hid_t plistid; hsize_t start[NDIMS], count[NDIMS]; - hsize_t dims[1]; + hsize_t dims[1], chunksize = SC1; int data[SC1], data_in[SC1]; int num_steps; double ftime; int write_us, read_us; int max_write_us, max_read_us; float write_rate, read_rate; + int deflate_level = 4; int i, s; /* We will write the same slice of random data over and over to @@ -104,9 +105,26 @@ main(int argc, char **argv) dims[0] = DIM2_LEN; if ((whole_spaceid = H5Screate_simple(NDIMS, dims, NULL)) < 0) ERR; - /* Create dataset. */ + /* Create property list for dataset. */ if ((plistid = H5Pcreate(H5P_DATASET_CREATE)) < 0) ERR; + /* Turn off object tracking times in HDF5 (as is done in nc4hdf.c). */ + if (H5Pset_obj_track_times(plistid, 0) < 0) ERR; + + /* Required to truly turn HDF5 fill values off */ + if (H5Pset_fill_time(plistid, H5D_FILL_TIME_NEVER) < 0) ERR; + + /* if (H5Pset_shuffle(plistid) < 0) */ + if (H5Pset_deflate(plistid, deflate_level) < 0) ERR; + + /* Set chunking. */ + if (H5Pset_chunk(plistid, NDIMS, &chunksize) < 0) ERR; + + /* Turn on creation order tracking. */ + if (H5Pset_attr_creation_order(plistid, H5P_CRT_ORDER_TRACKED| + H5P_CRT_ORDER_INDEXED) < 0) ERR; + + /* Create dataset. */ if ((dsid = H5Dcreate2(fileid, VAR_NAME, H5T_NATIVE_INT, whole_spaceid, H5P_DEFAULT, plistid, H5P_DEFAULT)) < 0) ERR; From c19aa14eaf6053efaa8406de51980a8b32f47e17 Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Wed, 8 Jul 2020 13:14:58 -0600 Subject: [PATCH 6/9] turned off szip testing while HDF5 issue is resolved --- h5_test/tst_h_par_compress.c | 392 ++++++++++++++++------------------- 1 file changed, 178 insertions(+), 214 deletions(-) diff --git a/h5_test/tst_h_par_compress.c b/h5_test/tst_h_par_compress.c index a493c7d0d0..fce580a9c3 100644 --- a/h5_test/tst_h_par_compress.c +++ b/h5_test/tst_h_par_compress.c @@ -15,16 +15,6 @@ #include "err_macros.h" #include -/* Defining USE_MPE causes the MPE trace library to be used (and you - * must also relink with -llmpe -lmpe). This causes clog2 output to be - * written, which can be converted to slog2 (by the program - * clog2TOslog2) and then used in the analysis program jumpshot. */ -/*#define USE_MPE 1*/ - -#ifdef USE_MPE -#include -#endif /* USE_MPE */ - #define FILE_NAME "tst_h_par_compress.h5" #define VAR_NAME "HALs_memory" #define NDIMS 1 @@ -32,233 +22,207 @@ #define DIM2_LEN 16000000 #define SC1 100000 /* slice count. */ +/* The following code, when uncommented, adds szip testing for + * parallel I/O. However, this currently fails. I have a support + * request in to HDF5 about this. Ed 7/8/20 */ +/* #ifdef USE_SZIP */ +/* #define NUM_COMPRESS_FILTERS 2 */ +/* #else */ +/* #define NUM_COMPRESS_FILTERS 1 */ +/* #endif /\* USE_SZIP *\/ */ +#define NUM_COMPRESS_FILTERS 1 + int main(int argc, char **argv) { int p, my_rank; -#ifdef USE_MPE - int s_init, e_init, s_define, e_define, s_write, e_write, s_close, e_close; -#endif /* USE_MPE */ - MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); MPI_Comm_size(MPI_COMM_WORLD, &p); -#ifdef USE_MPE - MPE_Init_log(); - s_init = MPE_Log_get_event_number(); - e_init = MPE_Log_get_event_number(); - s_define = MPE_Log_get_event_number(); - e_define = MPE_Log_get_event_number(); - s_write = MPE_Log_get_event_number(); - e_write = MPE_Log_get_event_number(); - s_close = MPE_Log_get_event_number(); - e_close = MPE_Log_get_event_number(); - MPE_Describe_state(s_init, e_init, "Init", "red"); - MPE_Describe_state(s_define, e_define, "Define", "yellow"); - MPE_Describe_state(s_write, e_write, "Write", "green"); - MPE_Describe_state(s_close, e_close, "Close", "purple"); - MPE_Start_log(); - MPE_Log_event(s_init, 0, "start init"); -#endif /* USE_MPE */ - /* For builds with HDF5 prior to 1.10.3, just return success. */ #ifdef HDF5_SUPPORTS_PAR_FILTERS - if (!my_rank) - printf("*** Creating file for parallel I/O read with compression, and rereading it..."); + for (int cf = 0; cf < NUM_COMPRESS_FILTERS; cf++) { - hid_t fapl_id, fileid, whole_spaceid, dsid, slice_spaceid, whole_spaceid1, xferid; - hid_t plistid; - hsize_t start[NDIMS], count[NDIMS]; - hsize_t dims[1], chunksize = SC1; - int data[SC1], data_in[SC1]; - int num_steps; - double ftime; - int write_us, read_us; - int max_write_us, max_read_us; - float write_rate, read_rate; - int deflate_level = 4; - int i, s; - - /* We will write the same slice of random data over and over to - * fill the file. */ - for (i = 0; i < SC1; i++) - data[i] = rand(); - -#ifdef USE_MPE - MPE_Log_event(e_init, 0, "end init"); - MPE_Log_event(s_define, 0, "start define file"); -#endif /* USE_MPE */ - - /* Create file. */ - if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) ERR; - if (H5Pset_fapl_mpio(fapl_id, MPI_COMM_WORLD, MPI_INFO_NULL) < 0) ERR; - if ((fileid = H5Fcreate(FILE_NAME, H5F_ACC_TRUNC, H5P_DEFAULT, - fapl_id)) < 0) ERR; - - /* Create a space to deal with one slice in memory. */ - dims[0] = SC1; - if ((slice_spaceid = H5Screate_simple(NDIMS, dims, NULL)) < 0) ERR; - - /* Create a space to write all slices. */ - dims[0] = DIM2_LEN; - if ((whole_spaceid = H5Screate_simple(NDIMS, dims, NULL)) < 0) ERR; - - /* Create property list for dataset. */ - if ((plistid = H5Pcreate(H5P_DATASET_CREATE)) < 0) ERR; + if (!my_rank) + printf("*** Testing parallel I/O with %s compression...", cf ? "szip" : "zlib"); + { + hid_t fapl_id, fileid, whole_spaceid, dsid, slice_spaceid, whole_spaceid1, xferid; + hid_t plistid; + hsize_t start[NDIMS], count[NDIMS]; + hsize_t dims[1], chunksize = SC1; + int data[SC1], data_in[SC1]; + int num_steps; + double ftime; + int write_us, read_us; + int max_write_us, max_read_us; + float write_rate, read_rate; + int deflate_level = 4; + int i, s; + + /* We will write the same slice of random data over and over to + * fill the file. */ + for (i = 0; i < SC1; i++) + data[i] = rand(); + + /* Create file. */ + if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) ERR; + if (H5Pset_fapl_mpio(fapl_id, MPI_COMM_WORLD, MPI_INFO_NULL) < 0) ERR; + if ((fileid = H5Fcreate(FILE_NAME, H5F_ACC_TRUNC, H5P_DEFAULT, + fapl_id)) < 0) ERR; + + /* Create a space to deal with one slice in memory. */ + dims[0] = SC1; + if ((slice_spaceid = H5Screate_simple(NDIMS, dims, NULL)) < 0) ERR; + + /* Create a space to write all slices. */ + dims[0] = DIM2_LEN; + if ((whole_spaceid = H5Screate_simple(NDIMS, dims, NULL)) < 0) ERR; + + /* Create property list for dataset. */ + if ((plistid = H5Pcreate(H5P_DATASET_CREATE)) < 0) ERR; - /* Turn off object tracking times in HDF5 (as is done in nc4hdf.c). */ - if (H5Pset_obj_track_times(plistid, 0) < 0) ERR; + /* Turn off object tracking times in HDF5 (as is done in nc4hdf.c). */ + if (H5Pset_obj_track_times(plistid, 0) < 0) ERR; - /* Required to truly turn HDF5 fill values off */ - if (H5Pset_fill_time(plistid, H5D_FILL_TIME_NEVER) < 0) ERR; + /* Required to truly turn HDF5 fill values off */ + if (H5Pset_fill_time(plistid, H5D_FILL_TIME_NEVER) < 0) ERR; - /* if (H5Pset_shuffle(plistid) < 0) */ - if (H5Pset_deflate(plistid, deflate_level) < 0) ERR; - - /* Set chunking. */ - if (H5Pset_chunk(plistid, NDIMS, &chunksize) < 0) ERR; - - /* Turn on creation order tracking. */ - if (H5Pset_attr_creation_order(plistid, H5P_CRT_ORDER_TRACKED| - H5P_CRT_ORDER_INDEXED) < 0) ERR; + /* Set compression, either deflate or szip. */ + if (cf == 0) + { + if (H5Pset_deflate(plistid, deflate_level) < 0) ERR; + } + else + { + int options_mask = 32; + int bits_per_pixel = 32; + if (H5Pset_szip(plistid, options_mask, bits_per_pixel)) ERR; + } + + /* Set chunking. */ + if (H5Pset_chunk(plistid, NDIMS, &chunksize) < 0) ERR; + + /* Turn on creation order tracking. */ + if (H5Pset_attr_creation_order(plistid, H5P_CRT_ORDER_TRACKED| + H5P_CRT_ORDER_INDEXED) < 0) ERR; - /* Create dataset. */ - if ((dsid = H5Dcreate2(fileid, VAR_NAME, H5T_NATIVE_INT, - whole_spaceid, H5P_DEFAULT, plistid, H5P_DEFAULT)) < 0) ERR; - - /* Use collective write operations. */ - if ((xferid = H5Pcreate(H5P_DATASET_XFER)) < 0) ERR; - if (H5Pset_dxpl_mpio(xferid, H5FD_MPIO_COLLECTIVE) < 0) ERR; - -#ifdef USE_MPE - MPE_Log_event(e_define, 0, "end define file"); - if (my_rank) - sleep(my_rank); -#endif /* USE_MPE */ - - /* Write the data in num_step steps. */ - ftime = MPI_Wtime(); - num_steps = (DIM2_LEN/SC1) / p; - for (s = 0; s < num_steps; s++) - { -#ifdef USE_MPE - MPE_Log_event(s_write, 0, "start write slab"); -#endif /* USE_MPE */ - - /* Select hyperslab for write of one slice. */ - start[0] = s * SC1 * p + my_rank * SC1; - count[0] = SC1; - if (H5Sselect_hyperslab(whole_spaceid, H5S_SELECT_SET, - start, NULL, count, NULL) < 0) ERR; - - if (H5Dwrite(dsid, H5T_NATIVE_INT, slice_spaceid, whole_spaceid, - xferid, data) < 0) ERR; - -#ifdef USE_MPE - MPE_Log_event(e_write, 0, "end write file"); -#endif /* USE_MPE */ - } - write_us = (MPI_Wtime() - ftime) * MILLION; - MPI_Reduce(&write_us, &max_write_us, 1, MPI_INT, MPI_MAX, 0, MPI_COMM_WORLD); - if (!my_rank) - { - write_rate = (float)(DIM2_LEN * sizeof(int))/(float)max_write_us; - printf("\np=%d, write_rate=%g", p, write_rate); - } - -#ifdef USE_MPE - MPE_Log_event(s_close, 0, "start close file"); -#endif /* USE_MPE */ - - /* Close. These collective operations will allow every process - * to catch up. */ - if (H5Dclose(dsid) < 0 || - H5Sclose(whole_spaceid) < 0 || - H5Sclose(slice_spaceid) < 0 || - H5Pclose(fapl_id) < 0 || - H5Pclose(plistid) < 0 || - H5Fclose(fileid) < 0) - ERR; - -#ifdef USE_MPE - MPE_Log_event(e_close, 0, "end close file"); -#endif /* USE_MPE */ - - /* Open the file. */ - if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) ERR; - if (H5Pset_fapl_mpio(fapl_id, MPI_COMM_WORLD, MPI_INFO_NULL) < 0) ERR; - - - if (H5Pset_libver_bounds(fapl_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) ERR; - if ((fileid = H5Fopen(FILE_NAME, H5F_ACC_RDONLY, fapl_id)) < 0) ERR; - - /* Create a space to deal with one slice in memory. */ - dims[0] = SC1; - if ((slice_spaceid = H5Screate_simple(NDIMS, dims, NULL)) < 0) ERR; - - /* Open the dataset. */ - if ((dsid = H5Dopen(fileid, VAR_NAME)) < 0) ERR; - if ((whole_spaceid1 = H5Dget_space(dsid)) < 0) ERR; - - ftime = MPI_Wtime(); - - /* Read the data, a slice at a time. */ - for (s = 0; s < num_steps; s++) - { - /* Select hyperslab for read of one slice. */ - start[0] = s * SC1 * p + my_rank * SC1; - count[0] = SC1; - if (H5Sselect_hyperslab(whole_spaceid1, H5S_SELECT_SET, - start, NULL, count, NULL) < 0) - { - ERR; - return 2; - } - - if (H5Dread(dsid, H5T_NATIVE_INT, slice_spaceid, whole_spaceid1, - H5P_DEFAULT, data_in) < 0) - { - ERR; - return 2; - } - - /* Check the slice of data. */ - for (i = 0; i < SC1; i++) - if (data[i] != data_in[i]) - { - ERR; - return 2; - } - } - read_us = (MPI_Wtime() - ftime) * MILLION; - MPI_Reduce(&read_us, &max_read_us, 1, MPI_INT, MPI_MAX, 0, MPI_COMM_WORLD); - if (!my_rank) - { - read_rate = (float)(DIM2_LEN * sizeof(int))/(float)max_read_us; - printf(", read_rate=%g\n", read_rate); - } - - /* Close down. */ - if (H5Dclose(dsid) < 0 || - H5Sclose(slice_spaceid) < 0 || - H5Sclose(whole_spaceid1) < 0 || - H5Pclose(fapl_id) < 0 || - H5Fclose(fileid) < 0) - ERR; - } + /* Create dataset. */ + if ((dsid = H5Dcreate2(fileid, VAR_NAME, H5T_NATIVE_INT, + whole_spaceid, H5P_DEFAULT, plistid, H5P_DEFAULT)) < 0) ERR; + + /* Use collective write operations. */ + if ((xferid = H5Pcreate(H5P_DATASET_XFER)) < 0) ERR; + if (H5Pset_dxpl_mpio(xferid, H5FD_MPIO_COLLECTIVE) < 0) ERR; + + /* Write the data in num_step steps. */ + ftime = MPI_Wtime(); + num_steps = (DIM2_LEN/SC1) / p; + for (s = 0; s < num_steps; s++) + { + /* Select hyperslab for write of one slice. */ + start[0] = s * SC1 * p + my_rank * SC1; + count[0] = SC1; + if (H5Sselect_hyperslab(whole_spaceid, H5S_SELECT_SET, + start, NULL, count, NULL) < 0) ERR; + + if (H5Dwrite(dsid, H5T_NATIVE_INT, slice_spaceid, whole_spaceid, + xferid, data) < 0) ERR; + + } + write_us = (MPI_Wtime() - ftime) * MILLION; + MPI_Reduce(&write_us, &max_write_us, 1, MPI_INT, MPI_MAX, 0, MPI_COMM_WORLD); + if (!my_rank) + { + write_rate = (float)(DIM2_LEN * sizeof(int))/(float)max_write_us; + printf("\np=%d, write_rate=%g", p, write_rate); + } + + /* Close. These collective operations will allow every process + * to catch up. */ + if (H5Dclose(dsid) < 0 || + H5Sclose(whole_spaceid) < 0 || + H5Sclose(slice_spaceid) < 0 || + H5Pclose(fapl_id) < 0 || + H5Pclose(plistid) < 0 || + H5Fclose(fileid) < 0) + ERR; + + /* Open the file. */ + if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) ERR; + if (H5Pset_fapl_mpio(fapl_id, MPI_COMM_WORLD, MPI_INFO_NULL) < 0) ERR; + + + if (H5Pset_libver_bounds(fapl_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) ERR; + if ((fileid = H5Fopen(FILE_NAME, H5F_ACC_RDONLY, fapl_id)) < 0) ERR; + + /* Create a space to deal with one slice in memory. */ + dims[0] = SC1; + if ((slice_spaceid = H5Screate_simple(NDIMS, dims, NULL)) < 0) ERR; + + /* Open the dataset. */ + if ((dsid = H5Dopen(fileid, VAR_NAME)) < 0) ERR; + if ((whole_spaceid1 = H5Dget_space(dsid)) < 0) ERR; + + ftime = MPI_Wtime(); + + /* Read the data, a slice at a time. */ + for (s = 0; s < num_steps; s++) + { + /* Select hyperslab for read of one slice. */ + start[0] = s * SC1 * p + my_rank * SC1; + count[0] = SC1; + if (H5Sselect_hyperslab(whole_spaceid1, H5S_SELECT_SET, + start, NULL, count, NULL) < 0) + { + ERR; + return 2; + } + + if (H5Dread(dsid, H5T_NATIVE_INT, slice_spaceid, whole_spaceid1, + H5P_DEFAULT, data_in) < 0) + { + ERR; + return 2; + } + + /* Check the slice of data. */ + for (i = 0; i < SC1; i++) + if (data[i] != data_in[i]) + { + ERR; + return 2; + } + } + read_us = (MPI_Wtime() - ftime) * MILLION; + MPI_Reduce(&read_us, &max_read_us, 1, MPI_INT, MPI_MAX, 0, MPI_COMM_WORLD); + if (!my_rank) + { + read_rate = (float)(DIM2_LEN * sizeof(int))/(float)max_read_us; + printf(", read_rate=%g\n", read_rate); + } + + /* Close down. */ + if (H5Dclose(dsid) < 0 || + H5Sclose(slice_spaceid) < 0 || + H5Sclose(whole_spaceid1) < 0 || + H5Pclose(fapl_id) < 0 || + H5Fclose(fileid) < 0) + ERR; + } + if (!my_rank) + SUMMARIZE_ERR; + + } /* next cf */ #else { if (!my_rank) - printf("*** HDF5 1.10.3 or greater required for this test..."); + printf("*** HDF5 1.10.3 or greater required for this test.\n"); } #endif /* HDF5_SUPPORTS_PAR_FILTERS */ - if (!my_rank) - SUMMARIZE_ERR; - MPI_Finalize(); if (!my_rank) From 73e21b4fd66cf165e2bc3d5ed6512fb6fad9bd10 Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Wed, 8 Jul 2020 13:21:33 -0600 Subject: [PATCH 7/9] took out timing code --- h5_test/tst_h_par_compress.c | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/h5_test/tst_h_par_compress.c b/h5_test/tst_h_par_compress.c index fce580a9c3..e24313ab7a 100644 --- a/h5_test/tst_h_par_compress.c +++ b/h5_test/tst_h_par_compress.c @@ -54,10 +54,6 @@ main(int argc, char **argv) hsize_t dims[1], chunksize = SC1; int data[SC1], data_in[SC1]; int num_steps; - double ftime; - int write_us, read_us; - int max_write_us, max_read_us; - float write_rate, read_rate; int deflate_level = 4; int i, s; @@ -117,7 +113,6 @@ main(int argc, char **argv) if (H5Pset_dxpl_mpio(xferid, H5FD_MPIO_COLLECTIVE) < 0) ERR; /* Write the data in num_step steps. */ - ftime = MPI_Wtime(); num_steps = (DIM2_LEN/SC1) / p; for (s = 0; s < num_steps; s++) { @@ -131,13 +126,6 @@ main(int argc, char **argv) xferid, data) < 0) ERR; } - write_us = (MPI_Wtime() - ftime) * MILLION; - MPI_Reduce(&write_us, &max_write_us, 1, MPI_INT, MPI_MAX, 0, MPI_COMM_WORLD); - if (!my_rank) - { - write_rate = (float)(DIM2_LEN * sizeof(int))/(float)max_write_us; - printf("\np=%d, write_rate=%g", p, write_rate); - } /* Close. These collective operations will allow every process * to catch up. */ @@ -165,8 +153,6 @@ main(int argc, char **argv) if ((dsid = H5Dopen(fileid, VAR_NAME)) < 0) ERR; if ((whole_spaceid1 = H5Dget_space(dsid)) < 0) ERR; - ftime = MPI_Wtime(); - /* Read the data, a slice at a time. */ for (s = 0; s < num_steps; s++) { @@ -195,13 +181,6 @@ main(int argc, char **argv) return 2; } } - read_us = (MPI_Wtime() - ftime) * MILLION; - MPI_Reduce(&read_us, &max_read_us, 1, MPI_INT, MPI_MAX, 0, MPI_COMM_WORLD); - if (!my_rank) - { - read_rate = (float)(DIM2_LEN * sizeof(int))/(float)max_read_us; - printf(", read_rate=%g\n", read_rate); - } /* Close down. */ if (H5Dclose(dsid) < 0 || From b1c0fa07af78cb5d65b3e3fc496038ce6f20f766 Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Wed, 8 Jul 2020 13:26:50 -0600 Subject: [PATCH 8/9] added tst_h_par_compress to CMake build --- h5_test/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/h5_test/CMakeLists.txt b/h5_test/CMakeLists.txt index 40702e9ddb..3f040c9cbc 100644 --- a/h5_test/CMakeLists.txt +++ b/h5_test/CMakeLists.txt @@ -22,5 +22,6 @@ ENDFOREACH() IF(TEST_PARALLEL4) build_bin_test(tst_h_par) + build_bin_test(tst_h_par_compress) add_sh_test(h5_test run_par_tests) ENDIF() From 3f7197cc07ff3c88cc1cbac50cec0ff2bb71c36e Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Wed, 8 Jul 2020 13:28:15 -0600 Subject: [PATCH 9/9] updated RELEASE_NOTES --- RELEASE_NOTES.md | 1 + 1 file changed, 1 insertion(+) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 48a8b2fcb4..0ab86f7443 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -7,6 +7,7 @@ This file contains a high-level description of this package's evolution. Release ## 4.8.0 - TBD +* [Enhancement] Added new test for using compression with parallel I/O: nc_test4/tst_h_par_compress.c. See [https://github.com/Unidata/netcdf-c/pull/1784]. * [Bug Fix] Now allow szip to be used on variables with unlimited dimension [https://github.com/Unidata/netcdf-c/issues/1774]. * [Enhancement] Add support for cloud storage using a variant of the Zarr storage format. Warning: this feature is highly experimental and is subject to rapid evolution [https://www.unidata.ucar.edu/blogs/developer/en/entry/overview-of-zarr-support-in]. * [Bug Fix] Fix nccopy to properly set default chunking parameters when not otherwise specified. This can significantly improve performance in selected cases. Note that if seeing slow performance with nccopy, then, as a work-around, specifically set the chunking parameters. [https://github.com/Unidata/netcdf-c/issues/1763].