Merge pull request ESMCI#1343 from NCAR/ejh_benchmark

Fix memory leak, starting to move to heap instead of stack for maps, adding some tests

src/clib/pio_darray_int.c

@@ -404,13 +404,15 @@ int write_darray_multi_par(file_desc_t *file, int nvars, int fndims, const int *
     /* If this is an IO task write the data. */
     if (ios->ioproc)
     {
-        int rrcnt = 0; /* Number of subarray requests (pnetcdf only). */
         void *bufptr;
         size_t start[fndims];
         size_t count[fndims];
         int ndims = iodesc->ndims;
+#ifdef _PNETCDF
+        int rrcnt = 0; /* Number of subarray requests (pnetcdf only). */
         PIO_Offset *startlist[num_regions]; /* Array of start arrays for ncmpi_iput_varn(). */
         PIO_Offset *countlist[num_regions]; /* Array of count  arrays for ncmpi_iput_varn(). */
+#endif /* _PNETCDF */
 
         /* Process each region of data to be written. */
         for (int regioncnt = 0; regioncnt < num_regions; regioncnt++)
@@ -962,8 +964,12 @@ int recv_and_write_data(file_desc_t *file, const int *varids, const int *frame,
                         }
                     }
 
-                    /* Call the netCDF functions to write the data. */
+#ifdef LOGGING
+                    for (int i = 1; i < fndims; i++)
+                        LOG((3, "start[%d] %d count[%d] %d", i, start[i], i, count[i]));
+#endif /* LOGGING */
 
+                    /* Call the netCDF functions to write the data. */
                     if ((ierr = nc_put_vara(file->fh, varids[nv], start, count, bufptr)))
                         return check_netcdf2(ios, NULL, ierr, __FILE__, __LINE__);
 
@@ -1017,7 +1023,7 @@ int write_darray_multi_serial(file_desc_t *file, int nvars, int fndims, const in
               varids[0] <= PIO_MAX_VARS && iodesc, "invalid input", __FILE__, __LINE__);
 
     LOG((1, "write_darray_multi_serial nvars = %d fndims = %d iodesc->ndims = %d "
-         "iodesc->mpitype = %d", nvars, iodesc->ndims, fndims, iodesc->mpitype));
+         "iodesc->mpitype = %d", nvars, fndims, iodesc->ndims, iodesc->mpitype));
 
     /* Get the iosystem info. */
     ios = file->iosystem;
@@ -1120,6 +1126,7 @@ int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, int vid, void *iobu
 
     /* Get the IO system info. */
     ios = file->iosystem;
+    LOG((3, "pio_read_darray_nc ios->ioproc %d", ios->ioproc));
 
     /* Get the variable info. */
     if ((ierr = get_var_desc(vid, &file->varlist, &vdesc)))
@@ -1135,6 +1142,7 @@ int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, int vid, void *iobu
     if(!ios->async || !ios->ioproc)
 	ierr = get_gdim0(file, iodesc, vid, fndims, &gdim0);
 #endif
+    /* LOG((4, "fndims %d ndims %d", fndims, ndims)); */
 
     /* IO procs will read the data. */
     if (ios->ioproc)

src/clib/pio_rearrange.c

@@ -549,7 +549,10 @@ int compute_counts(iosystem_desc_t *ios, io_desc_t *iodesc,
     int recv_displs[ios->num_uniontasks];
 
     /* The list of indeces on each compute task */
-    PIO_Offset s2rindex[iodesc->ndof];
+    PIO_Offset *s2rindex;
+
+    if (!(s2rindex = malloc(sizeof(PIO_Offset) * iodesc->ndof)))
+        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
 
     /* Allocate memory for the array of counts and init to zero. */
     if (!(iodesc->scount = calloc(ios->num_iotasks, sizeof(int))))
@@ -767,6 +770,8 @@ int compute_counts(iosystem_desc_t *ios, io_desc_t *iodesc,
                           &iodesc->rearr_opts.comp2io)))
         return pio_err(ios, NULL, ierr, __FILE__, __LINE__);
 
+    free(s2rindex);
+
     return PIO_NOERR;
 }
 
@@ -1200,15 +1205,21 @@ int box_rearrange_create(iosystem_desc_t *ios, int maplen, const PIO_Offset *com
          "ios->num_iotasks = %d", maplen, ndims, ios->num_comptasks, ios->num_iotasks));
 
     /* Allocate arrays needed for this function. */
-    int dest_ioproc[maplen]; /* Destination IO task for each data element on compute task. */
-    PIO_Offset dest_ioindex[maplen];    /* Offset into IO task array for each data element. */
+    int *dest_ioproc; /* Destination IO task for each data element on compute task. */
+    PIO_Offset *dest_ioindex;    /* Offset into IO task array for each data element. */
     int sendcounts[ios->num_uniontasks]; /* Send counts for swapm call. */
     int sdispls[ios->num_uniontasks];    /* Send displacements for swapm. */
     int recvcounts[ios->num_uniontasks]; /* Receive counts for swapm. */
     int rdispls[ios->num_uniontasks];    /* Receive displacements for swapm. */
     MPI_Datatype dtypes[ios->num_uniontasks]; /* Array of MPI_OFFSET types for swapm. */
     PIO_Offset iomaplen[ios->num_iotasks];   /* Gets the llen of all IO tasks. */
 
+    /* Allocate storage. */
+    if (!(dest_ioproc = malloc(sizeof(int) * maplen)))
+        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+    if (!(dest_ioindex = malloc(sizeof(PIO_Offset) * maplen)))
+        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
     /* This is the box rearranger. */
     iodesc->rearranger = PIO_REARR_BOX;
 
@@ -1393,6 +1404,10 @@ int box_rearrange_create(iosystem_desc_t *ios, int maplen, const PIO_Offset *com
     if ((ret = compute_counts(ios, iodesc, dest_ioproc, dest_ioindex)))
         return pio_err(ios, NULL, ret, __FILE__, __LINE__);
 
+    /* Free resources. */
+    free(dest_ioproc);
+    free(dest_ioindex);
+
     /* Compute the max io buffer size needed for an iodesc. */
     if (ios->ioproc)
     {

src/clib/pioc.c

@@ -548,7 +548,8 @@ int PIOc_InitDecomp(int iosysid, int pio_type, int ndims, const int *gdimlen, in
 	    iodesc->map[m] = compmap[iodesc->remap[m]];
 	for (int m=1; m < maplen; m++)
 	    if (iodesc->map[m] < iodesc->map[m-1])
-		printf("%d: compmap[%d] %ld map[%d] %ld remap[%d] %d\n",ios->comp_rank, m, compmap[m], m, iodesc->map[m], m, iodesc->remap[m]);
+		printf("%d: compmap[%d] %lld map[%d] %lld remap[%d] %d\n",ios->comp_rank, m,
+                       compmap[m], m, iodesc->map[m], m, iodesc->remap[m]);
     }
     else
     {

src/clib/pioc_sc.c

@@ -131,7 +131,7 @@ void compute_one_dim(int gdim, int ioprocs, int rank, PIO_Offset *start,
  * @param arrlen
  * @param arr_in
  * @returns the size of the block
- * @author Jim Edwards
+ * @author Jim Edwards, Ed Hartnett
  */
 PIO_Offset GCDblocksize(int arrlen, const PIO_Offset *arr_in)
 {
@@ -144,12 +144,21 @@ PIO_Offset GCDblocksize(int arrlen, const PIO_Offset *arr_in)
     PIO_Offset bsize;     /* Size of the block. */
     PIO_Offset bsizeg;    /* Size of gap block. */
     PIO_Offset blklensum; /* Sum of all block lengths. */
-    PIO_Offset del_arr[arrlen - 1]; /* Array of deltas between adjacent elements in arr_in. */
-    PIO_Offset loc_arr[arrlen - 1];
+    PIO_Offset *del_arr; /* Array of deltas between adjacent elements in arr_in. */
+    PIO_Offset *loc_arr;
 
     /* Check inputs. */
     pioassert(arrlen > 0 && arr_in, "invalid input", __FILE__, __LINE__);
 
+    /* Allocate arrays. */
+    if (!(loc_arr = malloc(sizeof(PIO_Offset) * (arrlen - 1))))
+        return PIO_ENOMEM;
+    if (!(del_arr = malloc(sizeof(PIO_Offset) * (arrlen - 1))))
+    {
+        free(loc_arr);
+        return PIO_ENOMEM;
+    }
+
     /* Count the number of contiguous blocks in arr_in. If any if
        these blocks is of size 1, we are done and can return.
        Otherwise numtimes is the number of blocks. */
@@ -160,7 +169,11 @@ PIO_Offset GCDblocksize(int arrlen, const PIO_Offset *arr_in)
         {
             numtimes++;
             if ( i > 0 && del_arr[i - 1] > 1)
-                return(1);
+            {
+                free(loc_arr);
+                free(del_arr);
+                return 1;
+            }
         }
     }
 
@@ -226,6 +239,9 @@ PIO_Offset GCDblocksize(int arrlen, const PIO_Offset *arr_in)
             bsize = lgcd(bsize, arr_in[0]);
     }
 
+    free(loc_arr);
+    free(del_arr);
+
     return bsize;
 }
 

src/clib/pioc_support.c

@@ -994,6 +994,8 @@ int PIOc_write_nc_decomp(int iosysid, const char *filename, int cmode, int ioid,
     iosystem_desc_t *ios; /* IO system info. */
     io_desc_t *iodesc;    /* Decomposition info. */
     int max_maplen;       /* The maximum maplen used for any task. */
+    int *full_map;        /* 2D array holds all map info for all tasks. */
+    int *my_map;          /* 1D array holds all map info for this task. */
     int mpierr;
     int ret;
 
@@ -1035,35 +1037,39 @@ int PIOc_write_nc_decomp(int iosysid, const char *filename, int cmode, int ioid,
         return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
     LOG((3, "max_maplen = %d", max_maplen));
 
-    /* 2D array that will hold all the map information for all
-     * tasks. */
-    int full_map[ios->num_comptasks][max_maplen];
+    if (!(full_map = malloc(sizeof(int) * ios->num_comptasks * max_maplen)))
+        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+    if (!(my_map = malloc(sizeof(int) * max_maplen)))
+        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
 
     /* Fill local array with my map. Use the fill value for unused */
     /* elements at the end if max_maplen is longer than maplen. Also
      * subtract 1 because the iodesc->map is 1-based. */
-    int my_map[max_maplen];
     for (int e = 0; e < max_maplen; e++)
     {
         my_map[e] = e < iodesc->maplen ? iodesc->map[e] - 1 : NC_FILL_INT;
         LOG((3, "my_map[%d] = %d", e, my_map[e]));
     }
 
     /* Gather my_map from all computation tasks and fill the full_map array. */
-    if ((mpierr = MPI_Allgather(&my_map, max_maplen, MPI_INT, full_map, max_maplen,
+    if ((mpierr = MPI_Allgather(my_map, max_maplen, MPI_INT, full_map, max_maplen,
                                 MPI_INT, ios->comp_comm)))
         return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
 
+    free(my_map);
+
     for (int p = 0; p < ios->num_comptasks; p++)
         for (int e = 0; e < max_maplen; e++)
-            LOG((3, "full_map[%d][%d] = %d", p, e, full_map[p][e]));
+            LOG((3, "full_map[%d][%d] = %d", p, e, full_map[p * max_maplen + e]));
 
     /* Write the netCDF decomp file. */
     if ((ret = pioc_write_nc_decomp_int(ios, filename, cmode, iodesc->ndims, iodesc->dimlen,
-                                        ios->num_comptasks, task_maplen, (int *)full_map, title,
+                                        ios->num_comptasks, task_maplen, full_map, title,
                                         history, fortran_order)))
         return ret;
 
+    free(full_map);
     return PIO_NOERR;
 }
 
@@ -1530,19 +1536,24 @@ int pioc_read_nc_decomp_int(int iosysid, const char *filename, int *ndims, int *
 
     /* Read the map. */
     int map_varid;
-    int map_in[num_tasks_in][max_maplen_in];
+    int *map_in;
+
+    if (!(map_in = malloc(sizeof(int) * num_tasks_in * max_maplen_in)))
+        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
     if ((ret = PIOc_inq_varid(ncid, DECOMP_MAP_VAR_NAME, &map_varid)))
         return pio_err(ios, NULL, ret, __FILE__, __LINE__);
-    if ((ret = PIOc_get_var_int(ncid, map_varid, (int *)map_in)))
+    if ((ret = PIOc_get_var_int(ncid, map_varid, map_in)))
         return pio_err(ios, NULL, ret, __FILE__, __LINE__);
     if (map)
     {
         if (!(*map = malloc(num_tasks_in * max_maplen_in * sizeof(int))))
             return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
         for (int t = 0; t < num_tasks_in; t++)
             for (int l = 0; l < max_maplen_in; l++)
-                (*map)[t * max_maplen_in + l] = map_in[t][l];
+                (*map)[t * max_maplen_in + l] = map_in[t * max_maplen_in + l];
     }
+    free(map_in);
 
     /* Close the netCDF decomp file. */
     LOG((2, "pioc_read_nc_decomp_int about to close file ncid = %d", ncid));

tests/cunit/CMakeLists.txt

@@ -93,6 +93,8 @@ if (NOT PIO_USE_MPISERIAL)
   add_executable (test_decomp_frame EXCLUDE_FROM_ALL test_decomp_frame.c test_common.c)
   target_link_libraries (test_decomp_frame pioc)
   if (PIO_USE_MALLOC)
+    add_executable (test_perf2 EXCLUDE_FROM_ALL test_perf2.c test_common.c)
+    target_link_libraries (test_perf2)
     add_executable (test_darray_async_simple EXCLUDE_FROM_ALL test_darray_async_simple.c test_common.c)
     target_link_libraries (test_darray_async_simple pioc)
     add_executable (test_darray_async EXCLUDE_FROM_ALL test_darray_async.c test_common.c)
@@ -130,6 +132,7 @@ add_dependencies (tests test_decomps)
 add_dependencies (tests test_darray_fill)
 add_dependencies (tests test_decomp_frame)
 if(PIO_USE_MALLOC)
+#  add_dependencies (tests test_perf2)
   add_dependencies (tests test_darray_async_simple)
   add_dependencies (tests test_darray_async)
   add_dependencies (tests test_darray_async_many)
@@ -271,6 +274,10 @@ else ()
       EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_darray_async_simple
       NUMPROCS ${AT_LEAST_FOUR_TASKS}
       TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+    # add_mpi_test(test_perf2
+    #   EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_perf2
+    #   NUMPROCS ${AT_LEAST_FOUR_TASKS}
+    #   TIMEOUT ${DEFAULT_TEST_TIMEOUT})
     add_mpi_test(test_darray_async
       EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_darray_async
       NUMPROCS ${AT_LEAST_FOUR_TASKS}

tests/cunit/Makefile.am

@@ -19,7 +19,7 @@ test_darray_multivar3 test_darray_1d test_darray_3d			\
 test_decomp_uneven test_decomps test_rearr test_darray_async_simple	\
 test_darray_async test_darray_async_many test_darray_2sync		\
 test_async_multicomp test_async_multi2 test_async_manyproc		\
-test_darray_fill test_decomp_frame
+test_darray_fill test_decomp_frame test_perf1 test_perf2
 
 # Tests will run from a bash script.
 TESTS = run_tests.sh
@@ -59,6 +59,8 @@ test_async_multi2_SOURCES = test_async_multi2.c test_common.c pio_tests.h
 test_async_manyproc_SOURCES = test_async_manyproc.c test_common.c pio_tests.h
 test_darray_fill_SOURCES = test_darray_fill.c test_common.c pio_tests.h
 test_decomp_frame_SOURCES = test_decomp_frame.c test_common.c pio_tests.h
+test_perf1_SOURCES = test_perf1.c test_common.c pio_tests.h
+test_perf2_SOURCES = test_perf2.c test_common.c pio_tests.h
 
 # Distribute the test script.
 EXTRA_DIST = run_tests.sh

tests/cunit/test_darray_fill.c

@@ -360,12 +360,11 @@ int main(int argc, char **argv)
 
                 } /* next type */
             } /* next fill value test case */
-        } /* next rearranger */
-
-        /* Finalize PIO system. */
-        if ((ret = PIOc_finalize(iosysid)))
-            return ret;
 
+            /* Finalize PIO iosysid. */
+            if ((ret = PIOc_finalize(iosysid)))
+                return ret;
+        } /* next rearranger */
     } /* endif my_rank < TARGET_NTASKS */
 
     /* Finalize the MPI library. */