diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index fcf5eaa28a..bb3de63f33 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -7,6 +7,7 @@ This file contains a high-level description of this package's evolution. Release ## 4.7.0 - TBD +* [Enhancement] Expanded the capabilities of `NC_INMEMORY` to support writing and accessing the final modified memory. See [GitHub #879](https://github.com/Unidata/netcdf-c/pull/879) for more information. * [Enhancement] Made CDF5 support enabled by default. See [Github #931](https://github.com/Unidata/netcdf-c/issues/931) for more information. diff --git a/cf b/cf index f42254c2a5..e60b3dcabf 100644 --- a/cf +++ b/cf @@ -106,20 +106,18 @@ FLAGS="$FLAGS --enable-extreme-numbers" #FLAGS="$FLAGS --disable-testsets" #FLAGS="$FLAGS --disable-dap-remote-tests" #FLAGS="$FLAGS --enable-dap-auth-tests" -- requires a new remotetest server -FLAGS="$FLAGS --enable-doxygen --enable-internal-docs" +#FLAGS="$FLAGS --enable-doxygen --enable-internal-docs" FLAGS="$FLAGS --enable-logging" #FLAGS="$FLAGS --disable-diskless" #FLAGS="$FLAGS --enable-mmap" #FLAGS="$FLAGS --with-udunits" #FLAGS="$FLAGS --with-libcf" -#valgrind => not shared -#FLAGS="$FLAGS --enable-valgrind-tests" #FLAGS="$FLAGS --enable-jna" #FLAGS="$FLAGS --disable-properties-attribute" #FLAGS="$FLAGS --disable-silent-rules" #FLAGS="$FLAGS --with-testservers=remotestserver.localhost:8083" #FLAGS="$FLAGS --disable-filter-testing" -#FLAGS="$FLAGS --enable-metadata-perf" +FLAGS="$FLAGS --enable-metadata-perf" if test "x$PAR4" != x1 ; then FLAGS="$FLAGS --disable-parallel4" diff --git a/cf.cmake b/cf.cmake index 8aa94ce620..e3c31afade 100644 --- a/cf.cmake +++ b/cf.cmake @@ -36,19 +36,32 @@ FLAGS="-DCMAKE_PREFIX_PATH=c:/tools/nccmake" fi FLAGS="$FLAGS -DCMAKE_INSTALL_PREFIX=/tmp/netcdf" -if test "x$DAP" = x ; then FLAGS="$FLAGS -DENABLE_DAP=false"; fi -if test "x$NC4" = x ; then FLAGS="$FLAGS -DENABLE_NETCDF_4=false"; fi -if test "x$CDF5" != x ; then FLAGS="$FLAGS -DENABLE_CDF5=true"; fi -if test "x$HDF4" != x ; then FLAGS="$FLAGS -DENABLE_HDF4=true"; fi -FLAGS="$FLAGS -DENABLE_CONVERSION_WARNINGS=false" +if test "x$DAP" = x ; then +FLAGS="$FLAGS -DENABLE_DAP=false" +fi +if test "x$NC4" = x ; then +FLAGS="$FLAGS -DENABLE_NETCDF_4=false" +fi +if test "x$CDF5" != x ; then +FLAGS="$FLAGS -DENABLE_CDF5=true" +fi +if test "x$HDF4" != x ; then +FLAGS="$FLAGS -DENABLE_HDF4=true" +fi + +# Enables FLAGS="$FLAGS -DENABLE_DAP_REMOTE_TESTS=true" -FLAGS="$FLAGS -DENABLE_TESTS=true" -FLAGS="$FLAGS -DENABLE_EXAMPLES=false" -FLAGS="$FLAGS -DENABLE_DYNAMIC_LOADING=false" -FLAGS="$FLAGS -DENABLE_WINSOCK2=false" +FLAGS="$FLAGS -DENABLE_LOGGING=true" +#FLAGS="$FLAGS -DENABLE_DOXYGEN=true -DENABLE_INTERNAL_DOCS=true" #FLAGS="$FLAGS -DENABLE_LARGE_FILE_TESTS=true" FLAGS="$FLAGS -DENABLE_FILTER_TESTING=true" +# Disables +FLAGS="$FLAGS -DENABLE_EXAMPLES=false" +FLAGS="$FLAGS -DENABLE_CONVERSION_WARNINGS=false" +#FLAGS="$FLAGS -DENABLE_TESTS=false" +#FLAGS="$FLAGS -DENABLE_DISKLESS=false" + rm -fr build mkdir build cd build @@ -56,6 +69,7 @@ cd build NCLIB=`pwd` if test "x$VS" != x ; then + # Visual Studio CFG="Release" NCLIB="${NCLIB}/liblib" diff --git a/configure.ac b/configure.ac index 24be602902..9f7e9d509d 100644 --- a/configure.ac +++ b/configure.ac @@ -786,7 +786,7 @@ fi # Setup the diskless and mmap conditionals if test "x$enable_diskless" = xyes ; then - AC_DEFINE([USE_DISKLESS], [1], [if true, include NC_DISKLESS code]) + AC_DEFINE([USE_DISKLESS], [1], [if true, include NC_DISKLESS and NC_INMEMORY code]) if test "x$enable_mmap" = xyes; then AC_DEFINE([USE_MMAP], [1], [if true, use mmap for in-memory files]) fi diff --git a/docs/CMakeLists.txt b/docs/CMakeLists.txt index e31f3257a3..44ce86b8ac 100644 --- a/docs/CMakeLists.txt +++ b/docs/CMakeLists.txt @@ -74,6 +74,16 @@ IF(ENABLE_DOXYGEN) ENDIF(ENABLE_DOXYGEN) -SET(CUR_EXTRA_DIST ${CUR_EXTRA_DIST} CMakeLists.txt Makefile.am netcdf.m4 DoxygenLayout.xml Doxyfile.in Doxyfile.guide.in footer.html mainpage.dox tutorial.dox guide.dox types.dox notes.md cdl.dox architecture.dox internal.dox install-fortran.dox Doxyfile.in.cmake windows-binaries.md building-with-cmake.md install.md) +# Should match list in Makefile.am +SET(CUR_EXTRA_DIST ${CUR_EXTRA_DIST} +netcdf.m4 DoxygenLayout.xml Doxyfile.in footer.html +mainpage.dox tutorial.dox guide.dox types.dox cdl.dox +architecture.dox internal.dox windows-binaries.md +building-with-cmake.md CMakeLists.txt groups.dox install.md notes.md +install-fortran.md all-error-codes.md credits.md auth.md +obsolete/fan_utils.html bestpractices.md filters.md indexing.md +inmemory.md DAP4.dox OPeNDAP.dox attribute_conventions.md FAQ.md +file_format_specifications.md known_problems.md +COPYRIGHT.dox) ADD_EXTRA_DIST("${CUR_EXTRA_DIST}") diff --git a/docs/Doxyfile.in b/docs/Doxyfile.in index be2f8a1236..be95288c40 100644 --- a/docs/Doxyfile.in +++ b/docs/Doxyfile.in @@ -757,7 +757,9 @@ INPUT = \ @abs_top_srcdir@/docs/DAP4.dox \ @abs_top_srcdir@/docs/attribute_conventions.md \ @abs_top_srcdir@/docs/file_format_specifications.md \ - @abs_top_srcdir@/docs/tutorial.dox \ + @abs_top_srcdir@/docs/auth.md \ + @abs_top_srcdir@/docs/filters.md \ + @abs_top_srcdir@/docs/inmemory.md \ @abs_top_srcdir@/docs/notes.md \ @abs_top_srcdir@/docs/auth.md \ @abs_top_srcdir@/docs/filters.md \ @@ -767,6 +769,7 @@ INPUT = \ @abs_top_srcdir@/docs/COPYRIGHT.dox \ @abs_top_srcdir@/docs/credits.md \ @abs_top_srcdir@/docs/bestpractices.md \ + @abs_top_srcdir@/docs/tutorial.dox \ @abs_top_srcdir@/include/netcdf.h \ @abs_top_srcdir@/include/netcdf_mem.h \ @abs_top_srcdir@/include/netcdf_par.h \ diff --git a/docs/Makefile.am b/docs/Makefile.am index 6d76b661dd..4f7ad0d950 100644 --- a/docs/Makefile.am +++ b/docs/Makefile.am @@ -9,7 +9,10 @@ mainpage.dox tutorial.dox guide.dox types.dox cdl.dox \ architecture.dox internal.dox windows-binaries.md \ building-with-cmake.md CMakeLists.txt groups.dox install.md notes.md \ install-fortran.md all-error-codes.md credits.md auth.md \ -obsolete/fan_utils.html bestpractices.md filters.md indexing.dox +obsolete/fan_utils.html bestpractices.md filters.md indexing.dox \ +inmemory.md DAP4.dox OPeNDAP.dox attribute_conventions.md FAQ.md \ +file_format_specifications.md known_problems.md \ +COPYRIGHT.dox # Turn off parallel builds in this directory. .NOTPARALLEL: diff --git a/docs/auth.md b/docs/auth.md index 84038cb2d1..e9bb39058d 100644 --- a/docs/auth.md +++ b/docs/auth.md @@ -512,9 +512,10 @@ what you changed to the author so this document can be updated. done exit -## Provenance +## Point of Contact __Author__: Dennis Heimbigner
+__Email__: dmh at ucar dot edu __Initial Version__: 11/21/2014
__Last Revised__: 08/24/2017 diff --git a/docs/filters.md b/docs/filters.md index 7ca8106748..76cb45e3cc 100644 --- a/docs/filters.md +++ b/docs/filters.md @@ -1,8 +1,8 @@ -Filter Support in netCDF-4 (Enhanced) +NetCDF-4 Filter Support ============================ -Filter Support in netCDF-4 (Enhanced) {#compress} +NetCDF-4 Filter Support {#compress} ================================= [TOC] @@ -434,8 +434,15 @@ Test for Machine Endianness static const unsigned char b[4] = {0x0,0x0,0x0,0x1}; /* value 1 in big-endian*/ int endianness = (1 == *(unsigned int*)b); /* 1=>big 0=>little endian ```` +References {#References} +======================== -Provenance +1. https://support.hdfgroup.org/HDF5/doc/Advanced/DynamicallyLoadedFilters/HDF5DynamicallyLoadedFilters.pdf +2. https://support.hdfgroup.org/HDF5/doc/TechNotes/TechNote-HDF5-CompressionTroubleshooting.pdf +3. https://portal.hdfgroup.org/display/support/Contributions#Contributions-filters +4. https://support.hdfgroup.org/services/contributions.html#filters + +Point of Contact ================ __Author__: Dennis Heimbigner
@@ -443,10 +450,3 @@ __Email__: dmh at ucar dot edu __Initial Version__: 1/10/2018
__Last Revised__: 2/5/2018 -References {#References} -========== - -1. https://support.hdfgroup.org/HDF5/doc/Advanced/DynamicallyLoadedFilters/HDF5DynamicallyLoadedFilters.pdf -2. https://support.hdfgroup.org/HDF5/doc/TechNotes/TechNote-HDF5-CompressionTroubleshooting.pdf -3. https://portal.hdfgroup.org/display/support/Contributions#Contributions-filters -4. https://support.hdfgroup.org/services/contributions.html#filters diff --git a/docs/inmemory.md b/docs/inmemory.md new file mode 100644 index 0000000000..b3bc88ba0a --- /dev/null +++ b/docs/inmemory.md @@ -0,0 +1,215 @@ +NetCDF In-Memory Support +==================================== + + + +NetCDF In-Memory Support {#inmemory} +==================================== + +[TOC] + +Introduction {#inmemory_intro} +-------------- + +It can be convenient to operate on a netcdf file whose +content is held in memory instead of in a disk file. +The netcdf API has been modified in a number of ways +to support this capability. + +Actually, three distinct but related capabilities are provided. + +1. DISKLESS -- Read a file into memory, operate on it, and optionally +write it back out to disk when nc_close() is called. +2. INMEMORY -- Tell the netcdf-c library to treat a provided block +of memory as if it were a netcdf file. At close, it is possible to ask +for the final contents of the memory chunk. Be warned that there is +some complexity to this as described below. +4. MMAP -- Tell the netcdf-c library to use the *mmap()* operating +system functionality to access a file. + +The first two capabilities are intertwined in the sense that the *diskless* +capability makes use internally of the *inmemory* capability. But, the +*inmemory* capability can be used independently of the *diskless* capability. + +The *mmap()* capability provides a capability similar to *diskless* but +using special capabilities of the underlying operating system. + +Note also that *diskless* and *inmemory* can be used for both +*netcdf-3* (classic) and *netcdf-4* (enhanced) data. The *mmap* +capability can only be used with *netcdf-3*. + +Enabling Diskless File Access {#Enable_Diskless} +-------------- +The *diskless* capability can be used relatively transparently +using the *NC_DISKLESS* mode flag. + +Note that since the file is stored in memory, size limitations apply. +If you are on using a 32-bit pointer then the file size must be less than 2^32 +bytes in length. On a 64-bit machine, the size must be less than 2^64 bytes. + +### Diskless File Open +Calling *nc_open()* using the mode flag *NC_DISKLESS* will cause +the file being opened to be read into memory. When calling *nc_close()*, +the file will optionally be re-written (aka "persisted") to disk. This +persist capability will be invoked if and only if *NC_WRITE* is specified +in the mode flags at the call to *nc_open()*. + +### Diskless File Create +Calling *nc_create()* using the mode flag *NC_DISKLESS* will cause +the file to initially be created and kept in memory. +When calling *nc_close()*, the file will be written +to disk. +Note that if it is desired to create the file in memory, +but not write to a disk file, then one can either set +the NC_NOCLOBBER mode flag or one can call *nc_abort()* +instead of *nc_close()*. + +Enabling Inmemory File Access {#Enable_Inmemory} +-------------- + +The netcdf API has been extended to support the inmemory capability. +The relevant API is defined in the file `netcdf_mem.h`. + +The important data structure to use is `NC_memio`. +```` +typedef struct NC_memio { + size_t size; + void* memory; + int flags; +} NC_memio; + +```` +An instance of this data structure is used when providing or +retrieving a block of data. It specifies the memory and its size +and also some relevant flags that define how to manage the memory. + +Current only one flag is defined -- *NC_MEMIO_LOCKED*. +This tells the netcdf library that it should never try to +*realloc()* the memory nor to *free()* the memory. Note +that this does not mean that the memory cannot be modified, but +only that the modifications will be within the confines of the provided +memory. If doing such modifications is impossible without +reallocating the memory, then the modification will fail. + +### In-Memory API + +The new API consists of the following functions. +```` +int nc_open_mem(const char* path, int mode, size_t size, void* memory, int* ncidp); + +int nc_create_mem(const char* path, int mode, size_t initialsize, int* ncidp); + +int nc_open_memio(const char* path, int mode, NC_memio* info, int* ncidp); + +int nc_close_memio(int ncid, NC_memio* info); + +```` +### The **nc_open_mem** Function + +The *nc_open_mem()* function is actually a convenience +function that internally invokes *nc_open_memio()*. +It essentially provides simple read-only access to a chunk of memory +of some specified size. + +### The **nc_open_memio** Function + +This function provides a more general read/write capability with respect +to a chunk of memory. It has a number of constraints and its +semantics are somewhat complex. This is primarily due to limitations +imposed by the underlying HDF5 library. + +The constraints are as follows. + +1. If the *NC_MEMIO_LOCKED* flag is set, then the netcdf library will +make no attempt to reallocate or free the provided memory. +If the caller invokes the *nc_close_memio()* function to retrieve the +final memory block, it should be the same +memory block as was provided when *nc_open_memio* was called. +Note that it is still possible to modify the in-memory file if the NC_WRITE +mode flag was set. However, failures can occur if an operation +cannot complete because the memory needs to be expanded. +2. If the *NC_MEMIO_LOCKED* flag is not set, then +the netcdf library will take control of the incoming memory +and will feel free to reallocate the provided +memory block to obtain a larger block when an attempt to modify +the in-memory file requires more space. Note that implicit in this +is that the old block -- the one originally provided -- may be +free'd as a side effect of re-allocating the memory using the +*realloc()* function. +If the caller invokes the *nc_close_memio()* function to retrieve the +final memory block, the returned block must always be freed +by the caller and that the original block should not be freed. + +### The **nc_create_mem** Function + +This function allows a user to create an in-memory file, write to it, +and then retrieve the final memory using *nc_close_memio()*. +The *initialsize* argument to *nc_create_mem()* tells the library +how much initial memory to allocate. Technically, this is advisory only +because it may be ignored by the underlying HDF5 library. +It is used, however, for netcdf-3 files. + +### The **nc_close_memio** Function + +The ordinary *nc_close()* function can be called to close an in-memory file. +However, it is often desirable to obtain the final size and memory block +for the in-memory file when that file has been modified. +The *nc_close_memio()* function provides a means to do this. +Its second argument is a pointer to an *NC_memio* object +into which the final memory and size are stored. WARNING, +the returned memory is owned by the caller and so the caller +is responsible for calling *free()* on that returned memory. + +### Support for Writing with *NC_MEMIO_LOCKED* + +When the NC_MEMIO_LOCKED flag is set in the *NC_memio* object +passed to *nc_open_memio()*, it is still possible to modify +the opened in-memory file (using the NC_WRITE mode flag). + +The big problem is that any changes must fit into the memory provided +by the caller via the *NC_memio* object. This problem can be +mitigated, however, by using the "trick" of overallocating +the caller supplied memory. That is, if the original file is, say, 300 bytes, +then it is possible to allocate, say, 65000 bytes and copy the original file +into the first 300 bytes of the larger memory block. This will allow +the netcdf-c library to add to the file up to that 65000 byte limit. +In this way, it is possible to avoid memory reallocation while still +allowing modifications to the file. You will still need to call +*nc_close_memio()* to obtain the size of the final, modified, file. + +Enabling MMAP File Access {#Enable_MMAP} +-------------- + +Some operating systems provide a capability called MMAP. +This allows disk files to automatically be mapped to chunks of memory. +It operates in a fashion somewhat similar to operating system virtual +memory, except with respect to a file. + +By setting mode flag NC_MMAP, it is possible to do the equivalent +of NC_DISKLESS but using the operating system's mmap capabilities. + +Currently, MMAP support is only available when using netcdf-3 or cdf5 +files. + +Known Bugs {#Inmemory_Bugs} +-------------- + +1. If you are modifying a locked memory chunk (using + NC_MEMIO_LOCKED) and are accessing it as a netcdf-4 file, and + you overrun the available space, then the HDF5 library will + fail with a segmentation fault. + +References {#Inmemory_References} +-------------- + +1. https://support.hdfgroup.org/HDF5/doc1.8/Advanced/FileImageOperations/HDF5FileImageOperations.pdf + +Point of Contact +-------------- + +__Author__: Dennis Heimbigner
+__Email__: dmh at ucar dot edu +__Initial Version__: 2/3/2018
+__Last Revised__: 2/5/2018 + + diff --git a/docs/static-pages/software.html b/docs/static-pages/software.html index 6959af09df..6da1f11b78 100644 --- a/docs/static-pages/software.html +++ b/docs/static-pages/software.html @@ -95,6 +95,9 @@

Freely Available Software

  • Gfdnavi (Geophysical fluid data navigator)
    • +
  • + Gliderscope +
  • GMT (Generic Mapping Tools)
    • @@ -200,6 +203,9 @@

    Freely Available Software

  • ncvtk
    • +
  • + NetCDF Ninja +
  • netcdf tools
    • @@ -927,6 +933,36 @@

    Gfdnavi (Geophysical fluid data navigator)

    +

    Gliderccope

    +

    + Dr L. Mun Woo + at ANFOG (Australian National Facility for Ocean Gliders) + has developed + Gliderscope. + Gliderscope is an IMOS (Integrated Marine Observing System) + oceanographic software package allow users quick easy visualisation + of ocean glider data, via a convenient graphical user interface. + Originally developed for use with ANFOG NetCDF data, it has now + been expanded to handle NetCDF data files from IOOS (U.S. + Integrated Ocean Observing System) and EGO (Everyone's Gliding + Observatories) also. +

    + Being interactive, Gliderscope speaks to users via an onscreen dialogue + box, helping the user decide what to do. With a few simple clicks of + the mouse, users can choose and extract segments of data, filter out the + bad data, perform calculations (e.g. for water density, sound velocity + in water, light attenuation, 1% photic depth) and apply a variety of + high level graphical data visualisation techniques to produce elegant + three/four-dimensional plots of water properties, interpolated contour + charts, vertical profile plots, water properties comparison charts + etc. Additionally, users can also export their data to text or NetCDF + files for easy access in other applications. Gliderscope is available + on Windows and Macintosh platforms, as standalone executable software + as well as an App for use within Matlab. +

    + +

    +

    GMT

    GMT (Generic Mapping Tools) is @@ -1965,6 +2001,21 @@

    ncvtk

    +

    netCDF Ninja

    +

    + Dr L. Mun Woo of University of Western Australia + has developed + NetCDF Ninja, + a graphical user interface that allows users to browse all + the metadata contained in NetCDF files, scrutinise the data using an + interactive graphical plot and even make small alterations or export + the data in text format without having any knowledge of coding. NetCDF + Ninja is available on Windows and Macintosh platforms, as standalone + executable software as well as an App for use within Matlab. +

    + +

    +

    Ivan Shmakov's netcdf tools

    diff --git a/include/nc3dispatch.h b/include/nc3dispatch.h index b3a93f8346..5b88aa7d82 100644 --- a/include/nc3dispatch.h +++ b/include/nc3dispatch.h @@ -80,7 +80,7 @@ extern int NC3_abort(int ncid); extern int -NC3_close(int ncid); +NC3_close(int ncid,void*); extern int NC3_set_fill(int ncid, int fillmode, int *old_modep); diff --git a/include/nc4dispatch.h b/include/nc4dispatch.h index 53e0b62434..33b34921ec 100644 --- a/include/nc4dispatch.h +++ b/include/nc4dispatch.h @@ -46,7 +46,7 @@ extern int NC4_abort(int ncid); extern int -NC4_close(int ncid); +NC4_close(int ncid,void*); extern int NC4_set_fill(int ncid, int fillmode, int *old_modep); diff --git a/include/nc4internal.h b/include/nc4internal.h index 0f2c69bf7e..a180d5e280 100644 --- a/include/nc4internal.h +++ b/include/nc4internal.h @@ -18,6 +18,7 @@ #include "ncdimscale.h" #include "nc_logging.h" +#include "netcdf_mem.h" #include "ncindex.h" #ifdef USE_PARALLEL @@ -326,6 +327,17 @@ typedef struct NC_HDF5_FILE_INFO void *format_file_info; #endif /* USE_HDF4 */ struct NCFILEINFO* fileinfo; + struct NC4_Memio { + NC_memio memio; + int locked; /* do not copy and do not release */ + int persist; /* Should file be persisted out on close? */ + int inmemory; + int diskless; + unsigned int flags; /* for H5LTopen_file_image */ + int fapl; + size_t initialsize; + int created; /* 1 => create, 0 => open */ + } mem; } NC_HDF5_FILE_INFO_T; diff --git a/include/ncdispatch.h b/include/ncdispatch.h index 439cdc2252..a235afeda5 100644 --- a/include/ncdispatch.h +++ b/include/ncdispatch.h @@ -105,11 +105,6 @@ typedef struct NC_MPI_INFO { MPI_Info info; } NC_MPI_INFO; -typedef struct NC_MEM_INFO { - size_t size; - void* memory; -} NC_MEM_INFO; - /* Define known dispatch tables and initializers */ /*Forward*/ @@ -211,7 +206,7 @@ int (*redef)(int); int (*_enddef)(int,size_t,size_t,size_t,size_t); int (*sync)(int); int (*abort)(int); -int (*close)(int); +int (*close)(int,void*); int (*set_fill)(int,int,int*); int (*inq_base_pe)(int,int*); int (*set_base_pe)(int,int); diff --git a/include/netcdf.h b/include/netcdf.h index 3b6a309871..3102c69fb9 100644 --- a/include/netcdf.h +++ b/include/netcdf.h @@ -154,7 +154,7 @@ Use this in mode flags for both nc_create() and nc_open(). */ Use this in mode flags for both nc_create() and nc_open(). */ #define NC_MPIPOSIX 0x4000 /**< \deprecated As of libhdf5 1.8.13. */ -#define NC_INMEMORY 0x8000 /**< Read from memory. Mode flag for nc_open() or nc_create(). */ +#define NC_INMEMORY 0x8000 /**< Read from memory. Mode flag for nc_open() or nc_create() => NC_DISKLESS */ #define NC_PNETCDF (NC_MPIIO) /**< Use parallel-netcdf library; alias for NC_MPIIO. */ @@ -459,7 +459,8 @@ by the desired type. */ #define NC_EFILTER (-132) /**< Filter operation failed. */ #define NC_ERCFILE (-133) /**< RC file failure */ #define NC_ENULLPAD (-134) /**< Header Bytes not Null-Byte padded */ -#define NC4_LAST_ERROR (-135) /**< @internal All netCDF errors > this. */ +#define NC_EINMEMORY (-135) /**< In-memory file error */ +#define NC4_LAST_ERROR (-136) /**< @internal All netCDF errors > this. */ /** @internal This is used in netCDF-4 files for dimensions without * coordinate vars. */ diff --git a/include/netcdf_mem.h b/include/netcdf_mem.h index a388d286d2..c8079896ca 100644 --- a/include/netcdf_mem.h +++ b/include/netcdf_mem.h @@ -14,12 +14,30 @@ #include +typedef struct NC_memio { + size_t size; + void* memory; +#define NC_MEMIO_LOCKED 1 /* Do not try to realloc or free provided memory */ + int flags; +} NC_memio; + #if defined(__cplusplus) extern "C" { #endif +/* Treate a memory block as a file; read-only */ EXTERNL int nc_open_mem(const char* path, int mode, size_t size, void* memory, int* ncidp); +EXTERNL int nc_create_mem(const char* path, int mode, size_t initialsize, int* ncidp); + +/* Alternative to nc_open_mem with extended capabilites + See docs/inmemory.md + */ +EXTERNL int nc_open_memio(const char* path, int mode, NC_memio* info, int* ncidp); + +/* Close memory file and return the final memory state */ +EXTERNL int nc_close_memio(int ncid, NC_memio* info); + #if defined(__cplusplus) } #endif diff --git a/libdap2/ncd2dispatch.c b/libdap2/ncd2dispatch.c index 5a95659b3d..cd9db60de3 100644 --- a/libdap2/ncd2dispatch.c +++ b/libdap2/ncd2dispatch.c @@ -225,7 +225,7 @@ NCD2_sync(int ncid) static int NCD2_abort(int ncid) { - return NCD2_close(ncid); + return NCD2_close(ncid,NULL); } static int @@ -596,13 +596,13 @@ fprintf(stderr,"ncdap3: final constraint: %s\n",dapcomm->oc.url->query); return ncstat; done: - if(drno != NULL) NCD2_close(drno->ext_ncid); + if(drno != NULL) NCD2_close(drno->ext_ncid,NULL); if(ocstat != OC_NOERR) ncstat = ocerrtoncerr(ocstat); return THROW(ncstat); } int -NCD2_close(int ncid) +NCD2_close(int ncid, void* ignore) { NC* drno; NCDAPCOMMON* dapcomm; diff --git a/libdap2/ncd2dispatch.h b/libdap2/ncd2dispatch.h index 697c18a873..4b97b4bedd 100644 --- a/libdap2/ncd2dispatch.h +++ b/libdap2/ncd2dispatch.h @@ -50,7 +50,7 @@ NCD2_open(const char *path, int mode, struct NC_Dispatch* dispatch, NC* ncp); extern int -NCD2_close(int ncid); +NCD2_close(int ncid,void*); extern int NCD2_inq_format_extended(int ncid, int* formatp, int* modep); diff --git a/libdap4/d4file.c b/libdap4/d4file.c index 6574efe0eb..ef21347683 100644 --- a/libdap4/d4file.c +++ b/libdap4/d4file.c @@ -245,7 +245,7 @@ NCD4_open(const char * path, int mode, } int -NCD4_close(int ncid) +NCD4_close(int ncid, void* ignore) { int ret = NC_NOERR; NC* nc; @@ -278,7 +278,7 @@ NCD4_close(int ncid) int NCD4_abort(int ncid) { - return NCD4_close(ncid); + return NCD4_close(ncid,NULL); } /**************************************************/ diff --git a/libdap4/ncd4dispatch.h b/libdap4/ncd4dispatch.h index d14071c7f8..9a740d427f 100644 --- a/libdap4/ncd4dispatch.h +++ b/libdap4/ncd4dispatch.h @@ -21,7 +21,7 @@ NCD4_open(const char *path, int mode, struct NC_Dispatch* dispatch, NC* ncp); extern int -NCD4_close(int ncid); +NCD4_close(int ncid,void*); extern int NCD4_abort(int ncid); diff --git a/libdispatch/derror.c b/libdispatch/derror.c index c3c1bfae99..4ee2b93b75 100644 --- a/libdispatch/derror.c +++ b/libdispatch/derror.c @@ -262,9 +262,11 @@ const char *nc_strerror(int ncerr1) case NC_EMPI: return "NetCDF: MPI operation failed."; case NC_ERCFILE: return "NetCDF: RC File Failure."; - case NC_ENULLPAD: - return "NetCDF: File fails strict Null-Byte Header check."; - default: + case NC_ENULLPAD: + return "NetCDF: File fails strict Null-Byte Header check."; + case NC_EINMEMORY: + return "NetCDF: In-memory File operation failed."; + default: #ifdef USE_PNETCDF /* The behavior of ncmpi_strerror here is to return NULL, not a string. This causes problems in (at least) diff --git a/libdispatch/dfile.c b/libdispatch/dfile.c index 2d81f661ed..014e6c5739 100644 --- a/libdispatch/dfile.c +++ b/libdispatch/dfile.c @@ -181,10 +181,10 @@ NC_check_file_type(const char *path, int flags, void *parameters, int status = NC_NOERR; int diskless = ((flags & NC_DISKLESS) == NC_DISKLESS); + int inmemory = (!diskless && ((flags & NC_INMEMORY) == NC_INMEMORY)); #ifdef USE_PARALLEL int use_parallel = ((flags & NC_MPIIO) == NC_MPIIO); #endif /* USE_PARALLEL */ - int inmemory = (diskless && ((flags & NC_INMEMORY) == NC_INMEMORY)); struct MagicFile file; *model = 0; @@ -194,7 +194,7 @@ NC_check_file_type(const char *path, int flags, void *parameters, file.path = path; /* do not free */ file.parameters = parameters; if(inmemory && parameters == NULL) - {status = NC_EDISKLESS; goto done;} + {status = NC_EINMEMORY; goto done;} if(inmemory) { file.inmemory = inmemory; goto next; @@ -263,8 +263,8 @@ and attributes. NC_64BIT_DATA (Alias NC_CDF5) (create CDF-5 file), NC_NETCDF4 (create netCDF-4/HDF5 file), NC_CLASSIC_MODEL (enforce netCDF classic mode on netCDF-4/HDF5 files), - NC_DISKLESS (store data only in memory), - NC_MMAP (use MMAP for NC_DISKLESS), + NC_DISKLESS (store data in memory), + NC_MMAP (use MMAP for NC_DISKLESS instead of NC_INMEMORY), and NC_WRITE. See discussion below. @@ -278,7 +278,10 @@ aspects of how it may be used. Setting NC_NOCLOBBER means you do not want to clobber (overwrite) an existing dataset; an error (NC_EEXIST) is returned if the specified -dataset already exists. +dataset already exists. As a slight variation on this, if you +specify NC_DISKLESS and NC_NOCLOBBER, the file will be created +in-memory, but no attempt will be made to persiste the in-memory +data to a disk file. The NC_SHARE flag is appropriate when one process may be writing the dataset and one or more other processes reading the dataset @@ -528,6 +531,58 @@ nc__create(const char *path, int cmode, size_t initialsz, chunksizehintp, 0, NULL, ncidp); } + +/** \ingroup datasets +Create a netCDF file with the contents stored in memory. + +\param path Must be non-null, but otherwise only used to set the dataset name. + +\param mode the mode flags; Note that this procedure uses a limited set of flags because it forcibly sets NC_INMEMORY. + +\param initialsize (advisory) size to allocate for the created file + +\param ncidp Pointer to location where returned netCDF ID is to be +stored. + +\returns ::NC_NOERR No error. + +\returns ::NC_ENOMEM Out of memory. + +\returns ::NC_EDISKLESS diskless io is not enabled for fails. + +\returns ::NC_EINVAL, etc. other errors also returned by nc_open. + +

    Examples

    + +In this example we use nc_create_mem() to create a classic netCDF dataset +named foo.nc. The initial size is set to 4096. + +@code + #include + ... + int status = NC_NOERR; + int ncid; + int mode = 0; + size_t initialsize = 4096; + ... + status = nc_create_mem("foo.nc", mode, initialsize, &ncid); + if (status != NC_NOERR) handle_error(status); +@endcode +*/ + +int +nc_create_mem(const char* path, int mode, size_t initialsize, int* ncidp) +{ +#ifdef USE_DISKLESS + if(mode & (NC_MPIIO|NC_MPIPOSIX|NC_MMAP)) + return NC_EINVAL; + mode |= (NC_INMEMORY|NC_NOCLOBBER); /* Specifically, do not set NC_DISKLESS */ + return NC_create(path, mode, initialsize, 0, NULL, 0, NULL, ncidp); +#else + return NC_EDISKLESS; +#endif +} + /** * @internal Create a file with special (deprecated) Cray settings. * @@ -740,7 +795,7 @@ Open a netCDF file with the contents taken from a block of memory. \param path Must be non-null, but otherwise only used to set the dataset name. -\param mode the mode flags; Note that this procedure uses a limited set of flags because it forcibly sets NC_NOWRITE|NC_DISKLESS|NC_INMEMORY. +\param mode the mode flags; Note that this procedure uses a limited set of flags because it forcibly sets NC_INMEMORY. \param size The length of the block of memory being passed. @@ -784,22 +839,89 @@ int nc_open_mem(const char* path, int mode, size_t size, void* memory, int* ncidp) { #ifdef USE_DISKLESS - NC_MEM_INFO meminfo; + NC_memio meminfo; /* Sanity checks */ if(memory == NULL || size < MAGIC_NUMBER_LEN || path == NULL) return NC_EINVAL; if(mode & (NC_WRITE|NC_MPIIO|NC_MPIPOSIX|NC_MMAP)) return NC_EINVAL; - mode |= (NC_INMEMORY|NC_DISKLESS); + mode |= (NC_INMEMORY); /* DO not set NC_DISKLESS */ meminfo.size = size; meminfo.memory = memory; + meminfo.flags = NC_MEMIO_LOCKED; return NC_open(path, mode, 0, NULL, 0, &meminfo, ncidp); #else return NC_EDISKLESS; #endif } +/** \ingroup datasets +Open a netCDF file with the contents taken from a block of memory. +Similar to nc_open_mem, but with parameters. Warning: if you do +specify that the provided memory is locked, then never +pass in non-heap allocated memory. Additionally, if not locked, +then do not assume that the memory returned by nc_close_mem +is the same as passed to nc_open_memio. You must check +before attempting to free the original memory. + +\param path Must be non-null, but otherwise only used to set the dataset name. + +\param mode the mode flags; Note that this procedure uses a limited set of flags because it forcibly sets NC_INMEMORY. + +\param params controlling parameters + +\param ncidp Pointer to location where returned netCDF ID is to be +stored. + +\returns ::NC_NOERR No error. + +\returns ::NC_ENOMEM Out of memory. + +\returns ::NC_EDISKLESS diskless io is not enabled for fails. + +\returns ::NC_EINVAL, etc. other errors also returned by nc_open. + +

    Examples

    + +Here is an example using nc_open_memio() to open an existing netCDF dataset +named foo.nc for read-only, non-shared access. It differs from the nc_open_mem() +example in that it uses a parameter block. + +@code +#include +#include + ... +int status = NC_NOERR; +int ncid; +NC_memio params; + ... +params.size = ; +params.memory = malloc(size); +params.flags = + ... +status = nc_open_memio("foo.nc", 0, ¶ms, &ncid); +if (status != NC_NOERR) handle_error(status); +@endcode +*/ +int +nc_open_memio(const char* path, int mode, NC_memio* params, int* ncidp) +{ +#ifdef USE_DISKLESS + /* Sanity checks */ + if(path == NULL || params == NULL) + return NC_EINVAL; + if(params->memory == NULL || params->size < MAGIC_NUMBER_LEN) + return NC_EINVAL; + if(mode & (NC_MPIIO|NC_MPIPOSIX|NC_MMAP)) + return NC_EINVAL; + mode |= (NC_INMEMORY); + return NC_open(path, mode, 0, NULL, 0, params, ncidp); +#else + return NC_EINMEMORY; +#endif +} + /** * @internal Open a netCDF file with extra parameters for Cray. * @@ -1263,8 +1385,73 @@ nc_close(int ncid) if(ncp->refcount <= 0) #endif { + stat = ncp->dispatch->close(ncid,NULL); + /* Remove from the nc list */ + if (!stat) + { + del_from_NCList(ncp); + free_NC(ncp); + } + } + return stat; +} + +/** \ingroup datasets +Do a normal close (see nc_close()) on an in-memory dataset, +then return a copy of the final memory contents of the dataset. + +\param ncid NetCDF ID, from a previous call to nc_open() or nc_create(). + +\param memio a pointer to an NC_memio object into which the final valid memory +size and memory will be returned. + +\returns ::NC_NOERR No error. + +\returns ::NC_EBADID Invalid id passed. + +\returns ::NC_ENOMEM Out of memory. - stat = ncp->dispatch->close(ncid); +\returns ::NC_EDISKLESS if the file was not created as an inmemory file. + +\returns ::NC_EBADGRPID ncid did not contain the root group id of this +file. (NetCDF-4 only). + +

    Example

    + +Here is an example using nc_close_mem to finish the definitions of a new +netCDF dataset named foo.nc, return the final memory, +and release its netCDF ID: + +\code + #include + ... + int status = NC_NOERR; + int ncid; + NC_memio finalmem; + size_t initialsize = 65000; + ... + status = nc_create_mem("foo.nc", NC_NOCLOBBER, initialsize, &ncid); + if (status != NC_NOERR) handle_error(status); + ... create dimensions, variables, attributes + status = nc_close_memio(ncid,&finalmem); + if (status != NC_NOERR) handle_error(status); +\endcode + + */ +int +nc_close_memio(int ncid, NC_memio* memio) +{ +#ifdef USE_DISKLESS + NC* ncp; + int stat = NC_check_id(ncid, &ncp); + if(stat != NC_NOERR) return stat; + +#ifdef USE_REFCOUNT + ncp->refcount--; + if(ncp->refcount <= 0) +#endif + { + stat = ncp->dispatch->close(ncid,memio); /* Remove from the nc list */ if (!stat) { @@ -1273,6 +1460,9 @@ nc_close(int ncid) } } return stat; +#else + return NC_EINMEMORY; +#endif } /** \ingroup datasets @@ -1774,7 +1964,7 @@ NC_create(const char *path0, int cmode, size_t initialsz, } #ifndef USE_DISKLESS - cmode &= (~ NC_DISKLESS); /* Force off */ + cmode &= (~ (NC_DISKLESS|NC_INMEMORY)); /* Force off */ #endif #ifdef WINPATH @@ -1964,13 +2154,12 @@ NC_open(const char *path0, int cmode, int basepe, size_t *chunksizehintp, #ifndef USE_DISKLESS /* Clean up cmode */ - cmode &= (~ NC_DISKLESS); + cmode &= (~ (NC_DISKLESS|NC_INMEMORY)); #endif inmemory = ((cmode & NC_INMEMORY) == NC_INMEMORY); diskless = ((cmode & NC_DISKLESS) == NC_DISKLESS); - #ifdef WINPATH path = NCpathcvt(path0); #else @@ -2195,7 +2384,7 @@ openmagic(struct MagicFile* file) int status = NC_NOERR; if(file->inmemory) { /* Get its length */ - NC_MEM_INFO* meminfo = (NC_MEM_INFO*)file->parameters; + NC_memio* meminfo = (NC_memio*)file->parameters; file->filelen = (long long)meminfo->size; goto done; } @@ -2260,7 +2449,7 @@ readmagic(struct MagicFile* file, long pos, char* magic) memset(magic,0,MAGIC_NUMBER_LEN); if(file->inmemory) { char* mempos; - NC_MEM_INFO* meminfo = (NC_MEM_INFO*)file->parameters; + NC_memio* meminfo = (NC_memio*)file->parameters; if((pos + MAGIC_NUMBER_LEN) > meminfo->size) {status = NC_EDISKLESS; goto done;} mempos = ((char*)meminfo->memory) + pos; diff --git a/libdispatch/dutil.c b/libdispatch/dutil.c index 7ec8625a6e..d8d76e1ade 100644 --- a/libdispatch/dutil.c +++ b/libdispatch/dutil.c @@ -169,9 +169,9 @@ NC_readfile(const char* filename, NCbytes* content) char part[1024]; #ifdef _MSC_VER - stream = NCfopen(filename,"r"); -#else stream = NCfopen(filename,"rb"); +#else + stream = NCfopen(filename,"r"); #endif if(stream == NULL) {ret=errno; goto done;} for(;;) { diff --git a/libsrc/memio.c b/libsrc/memio.c index 15c0cae6e9..c8a5f15918 100644 --- a/libsrc/memio.c +++ b/libsrc/memio.c @@ -2,36 +2,39 @@ * Copyright 1996, University Corporation for Atmospheric Research * See netcdf/COPYRIGHT file for copying and redistribution conditions. */ -#if defined (_WIN32) || defined (_WIN64) -#include -#include -#include -#define lseek64 lseek -#endif #if HAVE_CONFIG_H #include #endif - #include #include +#include #include #include -#ifdef _MSC_VER /* Microsoft Compilers */ -#include -#endif #ifdef HAVE_UNISTD_H #include #endif #ifdef HAVE_FCNTL_H #include #endif +#ifdef _MSC_VER /* Microsoft Compilers */ +#include +#include +#include +#define access(path,mode) _access(path,mode) +#endif #include "ncdispatch.h" #include "nc3internal.h" +#include "netcdf_mem.h" +#include "ncwinpath.h" #undef DEBUG +#ifndef HAVE_SSIZE_T +typedef int ssize_t; +#endif + #ifdef DEBUG #include #endif @@ -73,15 +76,33 @@ #undef X_ALIGN #endif +#undef REALLOCBUG +#ifdef REALLOCBUG +/* There is some kind of realloc bug that I cannot solve yet */ +#define reallocx(m,new,old) realloc(m,new) +#else +static void* +reallocx(void* mem, size_t newsize, size_t oldsize) +{ + void* m = malloc(newsize); + memcpy(m,mem,oldsize); + return m; +} +#endif + /* Private data for memio */ typedef struct NCMEMIO { - int locked; /* => we cannot realloc */ + int locked; /* => we cannot realloc or free*/ + int modified; /* => we realloc'd memory at least once */ int persist; /* => save to a file; triggered by NC_WRITE */ char* memory; - off_t alloc; - off_t size; - off_t pos; + size_t alloc; + size_t size; + size_t pos; + /* Convenience flags */ + int diskless; + int inmemory; /* assert(inmemory iff !diskless */ } NCMEMIO; /* Forward */ @@ -92,25 +113,28 @@ static int memio_sync(ncio *const nciop); static int memio_filesize(ncio* nciop, off_t* filesizep); static int memio_pad_length(ncio* nciop, off_t length); static int memio_close(ncio* nciop, int); +static int readfile(const char* path, NC_memio*); +static int writefile(const char* path, NCMEMIO*); +static int fileiswriteable(const char* path); /* Mnemonic */ #define DOOPEN 1 -static long pagesize = 0; +static size_t pagesize = 0; /*! Create a new ncio struct to hold info about the file. */ -static int memio_new(const char* path, int ioflags, off_t initialsize, void* memory, ncio** nciopp, NCMEMIO** memiop) +static int +memio_new(const char* path, int ioflags, off_t initialsize, ncio** nciopp, NCMEMIO** memiop) { int status = NC_NOERR; ncio* nciop = NULL; NCMEMIO* memio = NULL; - off_t minsize = initialsize; - int inmemory = (fIsSet(ioflags,NC_INMEMORY)); + size_t minsize = (size_t)initialsize; /* use asserts because this is an internal function */ + assert(fIsSet(ioflags,NC_INMEMORY)); assert(memiop != NULL && nciopp != NULL); - assert(path != NULL || (memory != NULL && initialsize > 0)); - assert(!inmemory || (memory != NULL && initialsize > 0)); + assert(path != NULL); if(pagesize == 0) { #if defined (_WIN32) || defined(_WIN64) @@ -118,22 +142,20 @@ static int memio_new(const char* path, int ioflags, off_t initialsize, void* mem GetSystemInfo (&info); pagesize = info.dwPageSize; #elif defined HAVE_SYSCONF - pagesize = sysconf(_SC_PAGE_SIZE); + long pgval = -1; + pgval = sysconf(_SC_PAGE_SIZE); + if(pgval < 0) { + status = NC_EIO; + goto fail; + } + pagesize = (size_t)pgval; #elif defined HAVE_GETPAGESIZE - pagesize = getpagesize(); + pagesize = (size_t)getpagesize(); #else - pagesize = 4096; /* good guess */ + pagesize = 4096; /* good guess */ #endif } - /* We need to catch errors. - sysconf, at least, can return a negative value - when there is an error. */ - if(pagesize < 0) { - status = NC_EIO; - goto fail; - } - errno = 0; /* Always force the allocated size to be a multiple of pagesize */ @@ -161,24 +183,24 @@ static int memio_new(const char* path, int ioflags, off_t initialsize, void* mem *((char**)&nciop->path) = strdup(path); if(nciop->path == NULL) {status = NC_ENOMEM; goto fail;} - memio->alloc = initialsize; - memio->pos = 0; - memio->size = minsize; - memio->memory = NULL; - memio->persist = fIsSet(ioflags,NC_WRITE); + if(memiop && memio) *memiop = memio; else free(memio); if(nciopp && nciop) *nciopp = nciop; else { if(nciop->path != NULL) free((char*)nciop->path); free(nciop); } - if(inmemory) { - memio->memory = memory; - } else { - /* malloc memory */ - memio->memory = (char*)malloc((size_t)memio->alloc); - if(memio->memory == NULL) {status = NC_ENOMEM; goto fail;} - } + memio->alloc = (size_t)initialsize; + memio->pos = 0; + memio->size = minsize; + memio->memory = NULL; /* filled in by caller */ + + if(fIsSet(ioflags,NC_DISKLESS)) + memio->diskless = 1; + if(fIsSet(ioflags,NC_INMEMORY) && !memio->diskless) + memio->inmemory = 1; + if(fIsSet(ioflags,NC_WRITE) && !fIsSet(ioflags,NC_NOCLOBBER) && memio->diskless) + memio->persist = 1; done: return status; @@ -210,41 +232,31 @@ int memio_create(const char* path, int ioflags, size_t initialsz, off_t igeto, size_t igetsz, size_t* sizehintp, - void* parameters, + void* parameters /*ignored*/, ncio* *nciopp, void** const mempp) { ncio* nciop; int fd; int status; NCMEMIO* memio = NULL; - int persist = (ioflags & NC_WRITE?1:0); - int oflags; if(path == NULL ||* path == 0) return NC_EINVAL; - - status = memio_new(path, ioflags, (off_t)initialsz, NULL, &nciop, &memio); + + status = memio_new(path, ioflags, initialsz, &nciop, &memio); if(status != NC_NOERR) return status; - if(persist) { - /* Open the file just tomake sure we can write it if needed */ - oflags = (persist ? O_RDWR : O_RDONLY); -#ifdef O_BINARY - fSet(oflags, O_BINARY); -#endif - oflags |= (O_CREAT|O_TRUNC); - if(fIsSet(ioflags,NC_NOCLOBBER)) - oflags |= O_EXCL; -#ifdef vms - fd = open(path, oflags, 0, "ctx=stm"); -#else - fd = open(path, oflags, OPENMODE); -#endif - if(fd < 0) {status = errno; goto unwind_open;} + if(memio->persist) { + /* Verify the file is writeable */ + if(!fileiswriteable(path)) + {status = EPERM; goto unwind_open;} + } - (void)close(fd); /* will reopen at nc_close */ - } /*!persist*/ + /* Allocate the memory for this file */ + memio->memory = (char*)malloc((size_t)memio->alloc); + if(memio->memory == NULL) {status = NC_ENOMEM; goto unwind_open;} + memio->locked = 0; #ifdef DEBUG fprintf(stderr,"memio_create: initial memory: %lu/%lu\n",(unsigned long)memio->memory,(unsigned long)memio->alloc); @@ -276,8 +288,7 @@ fprintf(stderr,"memio_create: initial memory: %lu/%lu\n",(unsigned long)memio->m return status; } -/* This function opens the data file. - +/* This function opens the data file or inmemory data path - path of data file. ioflags - flags passed into nc_open. igeto - looks like this function can do an initial page get, and @@ -301,85 +312,75 @@ memio_open(const char* path, ncio* nciop = NULL; int fd = -1; int status = NC_NOERR; - int persist = (fIsSet(ioflags,NC_WRITE)?1:0); - int inmemory = (fIsSet(ioflags,NC_INMEMORY)); - int oflags = 0; - NCMEMIO* memio = NULL; size_t sizehint = 0; - off_t filesize = 0; - off_t red = 0; - char* pos = NULL; - NC_MEM_INFO* meminfo = (NC_MEM_INFO*)parameters; + NC_memio meminfo; /* use struct to avoid worrying about free'ing it */ + NCMEMIO* memio = NULL; + size_t initialsize; + /* Should be the case that diskless => inmemory but not converse */ + int diskless = (fIsSet(ioflags,NC_DISKLESS)); + int inmemory = (fIsSet(ioflags,NC_INMEMORY) && !diskless); + int locked = 0; if(path == NULL || strlen(path) == 0) return NC_EINVAL; assert(sizehintp != NULL); + sizehint = *sizehintp; - if(inmemory) { - filesize = (off_t)meminfo->size; - } else { - /* Open the file,and make sure we can write it if needed */ - oflags = (persist ? O_RDWR : O_RDONLY); -#ifdef O_BINARY - fSet(oflags, O_BINARY); -#endif - oflags |= O_EXCL; -#ifdef vms - fd = open(path, oflags, 0, "ctx=stm"); -#else - fd = open(path, oflags, OPENMODE); -#endif -#ifdef DEBUG - if(fd < 0) { - fprintf(stderr,"open failed: file=%s err=",path); - perror(""); - } -#endif - if(fd < 0) {status = errno; goto unwind_open;} - - /* get current filesize = max(|file|,initialize)*/ - filesize = lseek(fd,0,SEEK_END); - if(filesize < 0) {status = errno; goto unwind_open;} - /* move pointer back to beginning of file */ - (void)lseek(fd,0,SEEK_SET); - if(filesize < (off_t)sizehint) - filesize = (off_t)sizehint; + if(inmemory) { /* parameters provide the memory chunk */ + NC_memio* memparams = (NC_memio*)parameters; + meminfo = *memparams; + locked = fIsSet(meminfo.flags,NC_MEMIO_LOCKED); + /* As a safeguard, if !locked and NC_WRITE is set, + then we must take control of the incoming memory */ + if(!locked && fIsSet(ioflags,NC_WRITE)) { + memparams->memory = NULL; + } + } else { /* read the file into a chunk of memory*/ + assert(diskless); + status = readfile(path,&meminfo); + if(status != NC_NOERR) + {goto unwind_open;} } - if(inmemory) - status = memio_new(path, ioflags, filesize, meminfo->memory, &nciop, &memio); - else - status = memio_new(path, ioflags, filesize, NULL, &nciop, &memio); - if(status != NC_NOERR) { - if(fd >= 0) - close(fd); - return status; + /* Fix up initial size */ + initialsize = meminfo.size; + + /* create the NCMEMIO structure */ + status = memio_new(path, ioflags, initialsize, &nciop, &memio); + if(status != NC_NOERR) + {goto unwind_open;} + memio->locked = locked; + + /* Initialize the memio memory */ + memio->memory = meminfo.memory; + + /* memio_new may have modified the allocated size, in which case, + reallocate the memory unless the memory is locked. */ + if(memio->alloc > meminfo.size) { + if(memio->locked) + memio->alloc = meminfo.size; /* force it back to what it was */ + else { + void* oldmem = memio->memory; + memio->memory = reallocx(oldmem,memio->alloc,meminfo.size); + if(memio->memory == NULL) + {status = NC_ENOMEM; goto unwind_open;} + } } #ifdef DEBUG fprintf(stderr,"memio_open: initial memory: %lu/%lu\n",(unsigned long)memio->memory,(unsigned long)memio->alloc); #endif - if(!inmemory) { - /* Read the file into the memio memory */ - /* We need to do multiple reads because there is no - guarantee that the amount read will be the full amount */ - red = memio->size; - pos = memio->memory; - while(red > 0) { - ssize_t count = (ssize_t)read(fd, pos, (size_t)red); - if(count < 0) {status = errno; goto unwind_open;} - if(count == 0) {status = NC_ENOTNC; goto unwind_open;} - red -= count; - pos += count; - } - (void)close(fd); + if(memio->persist) { + /* Verify the file is writeable */ + if(!fileiswriteable(path)) + {status = EPERM; goto unwind_open;} } /* Use half the filesize as the blocksize ; why? */ - sizehint = (size_t)filesize/2; + sizehint = (size_t)(memio->alloc/2); /* sizehint must be multiple of 8 */ sizehint = (sizehint / 8) * 8; @@ -433,26 +434,34 @@ static int memio_pad_length(ncio* nciop, off_t length) { NCMEMIO* memio; + size_t len = (size_t)length; if(nciop == NULL || nciop->pvt == NULL) return NC_EINVAL; memio = (NCMEMIO*)nciop->pvt; - if(!fIsSet(nciop->ioflags, NC_WRITE)) + if(!memio->persist) return EPERM; /* attempt to write readonly file*/ - - if(memio->locked > 0) + if(memio->locked) return NC_EDISKLESS; - if(length > memio->alloc) { + if(len > memio->alloc) { /* Realloc the allocated memory to a multiple of the pagesize*/ - off_t newsize = length; + size_t newsize = (size_t)len; void* newmem = NULL; /* Round to a multiple of pagesize */ if((newsize % pagesize) != 0) newsize += (pagesize - (newsize % pagesize)); - newmem = (char*)realloc(memio->memory,(size_t)newsize); + newmem = (char*)reallocx(memio->memory,newsize,memio->alloc); if(newmem == NULL) return NC_ENOMEM; - + /* If not copy is set, then fail if the newmem address is different + from old address */ + if(newmem != memio->memory) { + memio->modified++; + if(memio->locked) { + free(newmem); + return NC_EINMEMORY; + } + } /* zero out the extra memory */ memset((void*)((char*)newmem+memio->alloc),0,(size_t)(newsize - memio->alloc)); @@ -461,10 +470,13 @@ fprintf(stderr,"realloc: %lu/%lu -> %lu/%lu\n", (unsigned long)memio->memory,(unsigned long)memio->alloc, (unsigned long)newmem,(unsigned long)newsize); #endif + if(memio->memory != NULL && (!memio->locked || memio->modified)) + free(memio->memory); memio->memory = newmem; memio->alloc = newsize; + memio->modified = 1; } - memio->size = length; + memio->size = len; return NC_NOERR; } @@ -482,46 +494,23 @@ memio_close(ncio* nciop, int doUnlink) { int status = NC_NOERR; NCMEMIO* memio ; - int fd = -1; - int inmemory = 0; if(nciop == NULL || nciop->pvt == NULL) return NC_NOERR; - inmemory = (fIsSet(nciop->ioflags,NC_INMEMORY)); memio = (NCMEMIO*)nciop->pvt; assert(memio != NULL); /* See if the user wants the contents persisted to a file */ - if(!inmemory && memio->persist) { - /* Try to open the file for writing */ - int oflags = O_WRONLY|O_CREAT|O_TRUNC; -#ifdef O_BINARY - fSet(oflags, O_BINARY); -#endif - fd = open(nciop->path, oflags, OPENMODE); - if(fd >= 0) { - /* We need to do multiple writes because there is no - guarantee that the amount written will be the full amount */ - off_t written = memio->size; - char* pos = memio->memory; - while(written > 0) { - ssize_t count = (ssize_t)write(fd, pos, (size_t)written); - if(count < 0) - {status = errno; goto done;} - if(count == 0) - {status = NC_ENOTNC; goto done;} - written -= count; - pos += count; - } - } else - status = errno; - } + if(memio->persist && memio->memory != NULL) { + status = writefile(nciop->path,memio); + } -done: - if(!inmemory && memio->memory != NULL) + /* We only free the memio memory if file is not locked or has been modified */ + if(memio->memory != NULL && (!memio->locked || memio->modified)) { free(memio->memory); + memio->memory = NULL; + } /* do cleanup */ - if(fd >= 0) (void)close(fd); if(memio != NULL) free(memio); if(nciop->path != NULL) free((char*)nciop->path); free(nciop); @@ -529,9 +518,10 @@ memio_close(ncio* nciop, int doUnlink) } static int -guarantee(ncio* nciop, off_t endpoint) +guarantee(ncio* nciop, off_t endpoint0) { NCMEMIO* memio = (NCMEMIO*)nciop->pvt; + size_t endpoint = (size_t)endpoint0; if(endpoint > memio->alloc) { /* extend the allocated memory and size */ int status = memio_pad_length(nciop,endpoint); @@ -633,3 +623,117 @@ memio_sync(ncio* const nciop) { return NC_NOERR; /* do nothing */ } + +/* "Hidden" Internal function to extract a copy of + the size and/or contents of the memory +*/ +int +memio_extract(ncio* const nciop, size_t* sizep, void** memoryp) +{ + int status = NC_NOERR; + NCMEMIO* memio = NULL; + + if(nciop == NULL || nciop->pvt == NULL) return NC_NOERR; + memio = (NCMEMIO*)nciop->pvt; + assert(memio != NULL); + if(sizep) *sizep = memio->size; + + if(memoryp && memio->memory != NULL) { + *memoryp = memio->memory; + memio->memory = NULL; /* make sure it does not get free'd */ + } + return status; +} + +static int +fileiswriteable(const char* path) +{ + int ok; + ok = access(path,O_RDWR); + if(ok < 0) + return 0; + return 1; +} + +/* Read contents of a disk file into a memory chunk */ +static int +readfile(const char* path, NC_memio* memio) +{ + int status = NC_NOERR; + FILE* f = NULL; + size_t filesize = 0; + size_t count = 0; + char* memory = NULL; + char* p = NULL; + + /* Open the file for reading */ +#ifdef _MSC_VER + f = NCfopen(path,"rb"); +#else + f = NCfopen(path,"r"); +#endif + if(f == NULL) + {status = errno; goto done;} + /* get current filesize */ + if(fseek(f,0,SEEK_END) < 0) + {status = errno; goto done;} + filesize = (size_t)ftell(f); + /* allocate memory */ + memory = malloc((size_t)filesize); + if(memory == NULL) + {status = NC_ENOMEM; goto done;} + /* move pointer back to beginning of file */ + rewind(f); + count = filesize; + p = memory; + while(count > 0) { + size_t actual; + actual = fread(p,1,count,f); + if(actual == 0 || ferror(f)) + {status = NC_EIO; goto done;} + count -= actual; + p += actual; + } + if(memio) { + memio->size = (size_t)filesize; + memio->memory = memory; + } +done: + if(status != NC_NOERR && memory != NULL) + free(memory); + if(f != NULL) fclose(f); + return status; +} + +/* write contents of a memory chunk back into a disk file */ +static int +writefile(const char* path, NCMEMIO* memio) +{ + int status = NC_NOERR; + FILE* f = NULL; + size_t count = 0; + char* p = NULL; + + /* Open the file for writing*/ +#ifdef _MSC_VER + f = NCfopen(path,"rwb"); +#else + f = NCfopen(path,"rw"); +#endif + if(f == NULL) + {status = errno; goto done;} + rewind(f); + count = memio->size; + p = memio->memory; + while(count > 0) { + size_t actual; + actual = fwrite(p,1,count,f); + if(actual == 0 || ferror(f)) + {status = NC_EIO; goto done;} + count -= actual; + p += actual; + } +done: + if(f != NULL) fclose(f); + return status; +} diff --git a/libsrc/nc3internal.c b/libsrc/nc3internal.c index 88eb83ae54..8b8de63e12 100644 --- a/libsrc/nc3internal.c +++ b/libsrc/nc3internal.c @@ -19,6 +19,7 @@ #endif #include "nc3internal.h" +#include "netcdf_mem.h" #include "rnd.h" #include "ncx.h" @@ -35,6 +36,10 @@ /* For cdf5 */ #define NC_NUMRECS_EXTENT5 8 +/* Internal function; breaks ncio abstraction */ +extern int memio_extract(ncio* const nciop, size_t* sizep, void** memoryp); + + static void free_NC3INFO(NC3_INFO *nc3) { @@ -1317,7 +1322,7 @@ NC3_abort(int ncid) } int -NC3_close(int ncid) +NC3_close(int ncid, void* params) { int status = NC_NOERR; NC *nc; @@ -1365,6 +1370,12 @@ NC3_close(int ncid) } } + if(params != NULL && (nc->mode & NC_INMEMORY) != 0) { + NC_memio* memio = (NC_memio*)params; + /* Extract the final memory size &/or contents */ + status = memio_extract(nc3->nciop,&memio->size,&memio->memory); + } + (void) ncio_close(nc3->nciop, 0); nc3->nciop = NULL; diff --git a/libsrc/ncio.c b/libsrc/ncio.c index fb7756914f..f2e5137aee 100644 --- a/libsrc/ncio.c +++ b/libsrc/ncio.c @@ -46,9 +46,9 @@ ncio_create(const char *path, int ioflags, size_t initialsz, ncio** iopp, void** const mempp) { #ifdef USE_DISKLESS - if(fIsSet(ioflags,NC_DISKLESS)) { + if(fIsSet(ioflags,NC_INMEMORY)) { # ifdef USE_MMAP - if(fIsSet(ioflags,NC_MMAP)) + if(fIsSet(ioflags,NC_MMAP) && fIsSet(ioflags, NC_DISKLESS)) return mmapio_create(path,ioflags,initialsz,igeto,igetsz,sizehintp,parameters,iopp,mempp); else # endif /*USE_MMAP*/ @@ -72,12 +72,12 @@ ncio_open(const char *path, int ioflags, ncio** iopp, void** const mempp) { /* Diskless open has the following constraints: - 1. file must be classic version 1 or 2 + 1. file must be classic version 1 or 2 or 5 */ #ifdef USE_DISKLESS - if(fIsSet(ioflags,NC_DISKLESS)) { + if(fIsSet(ioflags,NC_INMEMORY)) { # ifdef USE_MMAP - if(fIsSet(ioflags,NC_MMAP)) + if(fIsSet(ioflags,NC_MMAP) && fIsSet(ioflags, NC_DISKLESS)) return mmapio_open(path,ioflags,igeto,igetsz,sizehintp,parameters,iopp,mempp); else # endif /*USE_MMAP*/ diff --git a/libsrc4/CMakeLists.txt b/libsrc4/CMakeLists.txt index 75ca5208d8..0aa4b69d28 100644 --- a/libsrc4/CMakeLists.txt +++ b/libsrc4/CMakeLists.txt @@ -1,6 +1,6 @@ # Process these files with m4. -SET(libsrc4_SOURCES nc4dispatch.c nc4attr.c nc4dim.c nc4file.c nc4grp.c nc4type.c nc4var.c ncfunc.c nc4internal.c nc4hdf.c nc4info.c ncindex.c) +SET(libsrc4_SOURCES nc4dispatch.c nc4attr.c nc4dim.c nc4file.c nc4grp.c nc4type.c nc4var.c ncfunc.c nc4internal.c nc4hdf.c nc4info.c ncindex.c nc4mem.c nc4memcb.c ) IF(LOGGING) SET(libsrc4_SOURCES ${libsrc4_SOURCES} error4.c) @@ -38,6 +38,3 @@ ENDIF(BUILD_DAP) IF (BUILD_PARALLEL) SET(ARGS_MANPAGE ${ARGS_MANPAGE} -DPARALLEL_IO=TRUE) ENDIF(BUILD_PARALLEL) - - - diff --git a/libsrc4/Makefile.am b/libsrc4/Makefile.am index e6d74e2ee5..8e3218f815 100644 --- a/libsrc4/Makefile.am +++ b/libsrc4/Makefile.am @@ -12,7 +12,7 @@ libnetcdf4_la_CPPFLAGS = ${AM_CPPFLAGS} noinst_LTLIBRARIES = libnetcdf4.la libnetcdf4_la_SOURCES = nc4dispatch.c nc4attr.c nc4dim.c nc4file.c \ nc4grp.c nc4hdf.c nc4internal.c nc4type.c nc4var.c ncfunc.c error4.c \ -nc4info.c nc4printer.c ncindex.c +nc4info.c nc4printer.c ncindex.c nc4mem.c nc4memcb.c EXTRA_DIST = CMakeLists.txt diff --git a/libsrc4/nc4file.c b/libsrc4/nc4file.c index e689f4521e..b37ca9d610 100644 --- a/libsrc4/nc4file.c +++ b/libsrc4/nc4file.c @@ -16,10 +16,18 @@ #include "nc.h" #include "nc4internal.h" #include "nc4dispatch.h" -#include /* must be after nc4internal.h */ -#include +#include "netcdf_mem.h" +#ifdef USE_HDF4 +#include +#endif #include +extern int nc4_vararray_add(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var); + +/* From nc4mem.c */ +extern int NC4_open_image_file(NC_HDF5_FILE_INFO_T* h5); +extern int NC4_create_image_file(NC_HDF5_FILE_INFO_T* h5, size_t); +extern int NC4_extract_file_image(NC_HDF5_FILE_INFO_T* h5); /** @internal When we have open objects at file close, should we log them or print to stdout. Default is to log. */ @@ -490,7 +498,7 @@ att_read_var_callbk(hid_t loc_id, const char *att_name, const H5A_info_t *ainfo, static const int ILLEGAL_OPEN_FLAGS = (NC_MMAP|NC_64BIT_OFFSET); /** @internal These flags may not be set for create. */ -static const int ILLEGAL_CREATE_FLAGS = (NC_NOWRITE|NC_MMAP|NC_INMEMORY|NC_64BIT_OFFSET|NC_CDF5); +static const int ILLEGAL_CREATE_FLAGS = (NC_NOWRITE|NC_MMAP|NC_64BIT_OFFSET|NC_CDF5); extern void reportopenobjects(int log, hid_t); @@ -585,12 +593,13 @@ sync_netcdf4_file(NC_HDF5_FILE_INFO_T *h5) * * @param h5 Pointer to HDF5 file info struct. * @param abort True if this is an abort. + * @param extractmem True if we need to extract and save final inmemory * * @return ::NC_NOERR No error. * @author Ed Hartnett */ static int -close_netcdf4_file(NC_HDF5_FILE_INFO_T *h5, int abort) +close_netcdf4_file(NC_HDF5_FILE_INFO_T *h5, int abort, int extractmem) { int retval = NC_NOERR; @@ -631,6 +640,16 @@ close_netcdf4_file(NC_HDF5_FILE_INFO_T *h5, int abort) if(h5->fileinfo) free(h5->fileinfo); + /* Check to see if this is an in-memory file and we want to get its + final content + */ + if(extractmem) { + /* File must be read/write */ + if(!h5->no_write) { + retval = NC4_extract_file_image(h5); + } + } + if (H5Fclose(h5->hdfid) < 0) { dumpopenobjects(h5); @@ -877,8 +896,8 @@ nc4typelen(nc_type type) * * @param path The file name of the new file. * @param cmode The creation mode flag. - * @param comm MPI communicator (parallel IO only). - * @param info MPI info (parallel IO only). + * @param initialsz The proposed initial file size (advisory) + * @param parameters extra parameter info (like MPI communicator) * @param nc Pointer to an instance of NC. * * @return ::NC_NOERR No error. @@ -889,25 +908,45 @@ nc4typelen(nc_type type) * @author Ed Hartnett, Dennis Heimbigner */ static int -nc4_create_file(const char *path, int cmode, MPI_Comm comm, MPI_Info info, - NC *nc) +nc4_create_file(const char *path, int cmode, size_t initialsz, void* parameters, NC *nc) { hid_t fcpl_id, fapl_id = -1; unsigned flags; FILE *fp; int retval = NC_NOERR; NC_HDF5_FILE_INFO_T* nc4_info = NULL; + #ifdef USE_PARALLEL4 + NC_MPI_INFO* mpiinfo = NULL; int comm_duped = 0; /* Whether the MPI Communicator was duplicated */ int info_duped = 0; /* Whether the MPI Info object was duplicated */ -#else /* !USE_PARALLEL4 */ - int persist = 0; /* Should diskless try to persist its data into file?*/ -#endif +#endif /* !USE_PARALLEL4 */ assert(nc && path); LOG((3, "%s: path %s mode 0x%x", __func__, path, cmode)); - if(cmode & NC_DISKLESS) + /* Add necessary structs to hold netcdf-4 file data. */ + if ((retval = nc4_nc4f_list_add(nc, path, (NC_WRITE | cmode)))) + BAIL(retval); + + nc4_info = NC4_DATA(nc); + assert(nc4_info && nc4_info->root_grp); + + nc4_info->mem.inmemory = (cmode & NC_INMEMORY) == NC_INMEMORY; + nc4_info->mem.diskless = (cmode & NC_DISKLESS) == NC_DISKLESS; + nc4_info->mem.created = 1; + nc4_info->mem.initialsize = initialsz; + + if(nc4_info->mem.inmemory && parameters) + nc4_info->mem.memio = *(NC_memio*)parameters; +#ifdef USE_PARALLEL4 + else if(parameters) { + mpinfo = (NC_MPI_INFO *)parameters; + comm = mpiinfo->comm; + info = mpiinfo->info; + } +#endif + if(nc4_info->mem.diskless) flags = H5F_ACC_TRUNC; else if(cmode & NC_NOCLOBBER) flags = H5F_ACC_EXCL; @@ -915,25 +954,19 @@ nc4_create_file(const char *path, int cmode, MPI_Comm comm, MPI_Info info, flags = H5F_ACC_TRUNC; /* If this file already exists, and NC_NOCLOBBER is specified, - return an error. */ - if (cmode & NC_DISKLESS) { -#ifndef USE_PARALLEL4 - if(cmode & NC_WRITE) - persist = 1; -#endif + return an error (unless diskless|inmemory) */ + if (nc4_info->mem.diskless) { + if((cmode & NC_WRITE) && (cmode & NC_NOCLOBBER) == 0) + nc4_info->mem.persist = 1; + } else if (nc4_info->mem.inmemory) { + /* ok */ } else if ((cmode & NC_NOCLOBBER) && (fp = fopen(path, "r"))) { fclose(fp); return NC_EEXIST; } - /* Add necessary structs to hold netcdf-4 file data. */ - if ((retval = nc4_nc4f_list_add(nc, path, (NC_WRITE | cmode)))) - BAIL(retval); - nc4_info = NC4_DATA(nc); - assert(nc4_info && nc4_info->root_grp); - /* Need this access plist to control how HDF5 handles open objects - * on file close. (Setting H5F_CLOSE_SEMI will cause H5Fclose to + * on file close. Setting H5F_CLOSE_SEMI will cause H5Fclose to * fail if there are any open objects in the file. */ if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) BAIL(NC_EHDFERR); @@ -943,8 +976,7 @@ nc4_create_file(const char *path, int cmode, MPI_Comm comm, MPI_Info info, #ifdef USE_PARALLEL4 /* If this is a parallel file create, set up the file creation property list. */ - if ((cmode & NC_MPIIO) || (cmode & NC_MPIPOSIX)) - { + if ((cmode & NC_MPIIO) || (cmode & NC_MPIPOSIX)) { nc4_info->parallel = NC_TRUE; if (cmode & NC_MPIIO) /* MPI/IO */ { @@ -986,7 +1018,7 @@ nc4_create_file(const char *path, int cmode, MPI_Comm comm, MPI_Info info, } #else /* only set cache for non-parallel... */ if(cmode & NC_DISKLESS) { - if (H5Pset_fapl_core(fapl_id, 4096, persist)) + if (H5Pset_fapl_core(fapl_id, 4096, nc4_info->mem.persist)) BAIL(NC_EDISKLESS); } if (H5Pset_cache(fapl_id, 0, nc4_chunk_cache_nelems, nc4_chunk_cache_size, @@ -1029,7 +1061,21 @@ nc4_create_file(const char *path, int cmode, MPI_Comm comm, MPI_Info info, H5Pset_coll_metadata_write(fapl_id, 1); #endif - if ((nc4_info->hdfid = H5Fcreate(path, flags, fcpl_id, fapl_id)) < 0) + if(nc4_info->mem.inmemory) { +#if 0 + if(nc4_info->mem.memio.size == 0) + nc4_info->memio.size = DEFAULT_CREATE_MEMSIZE; /* last ditch fix */ + if(nc4_info->memio.memory == NULL) { /* last ditch fix */ + nc4_info->memio.memory = malloc(nc4_info->memio.size); + if(nc4_info->memio.memory == NULL) + BAIL(NC_ENOMEM); + } + assert(nc4_info->memio.size > 0 && nc4_info->memio.memory != NULL); +#endif + retval = NC4_create_image_file(nc4_info,initialsz); + if(retval) + BAIL(retval); + } else if ((nc4_info->hdfid = H5Fcreate(path, flags, fcpl_id, fapl_id)) < 0) /*Change the return error from NC_EFILEMETADATA to System error EACCES because that is the more likely problem */ BAIL(EACCES); @@ -1058,7 +1104,7 @@ nc4_create_file(const char *path, int cmode, MPI_Comm comm, MPI_Info info, #endif if (fapl_id != H5P_DEFAULT) H5Pclose(fapl_id); if(!nc4_info) return retval; - close_netcdf4_file(nc4_info,1); /* treat like abort */ + close_netcdf4_file(nc4_info,1,0); /* treat like abort */ return retval; } @@ -1086,22 +1132,12 @@ NC4_create(const char* path, int cmode, size_t initialsz, int basepe, size_t *chunksizehintp, int use_parallel, void *parameters, NC_Dispatch *dispatch, NC* nc_file) { - MPI_Comm comm = MPI_COMM_WORLD; - MPI_Info info = MPI_INFO_NULL; int res; assert(nc_file && path); - LOG((1, "%s: path %s cmode 0x%x comm %d info %d", - __func__, path, cmode, comm, info)); - -#ifdef USE_PARALLEL4 - if (parameters) - { - comm = ((NC_MPI_INFO *)parameters)->comm; - info = ((NC_MPI_INFO *)parameters)->info; - } -#endif /* USE_PARALLEL4 */ + LOG((1, "%s: path %s cmode 0x%x parameters %p", + __func__, path, cmode, parameters)); /* If this is our first file, turn off HDF5 error messages. */ if (!nc4_hdf5_initialized) @@ -1144,7 +1180,7 @@ NC4_create(const char* path, int cmode, size_t initialsz, int basepe, nc_file->int_ncid = nc_file->ext_ncid; - res = nc4_create_file(path, cmode, comm, info, nc_file); + res = nc4_create_file(path, cmode, initialsz, parameters, nc_file); done: return res; @@ -2480,27 +2516,47 @@ static int nc4_open_file(const char *path, int mode, void* parameters, NC *nc) { hid_t fapl_id = H5P_DEFAULT; - unsigned flags = (mode & NC_WRITE) ? - H5F_ACC_RDWR : H5F_ACC_RDONLY; int retval; + unsigned flags; NC_HDF5_FILE_INFO_T* nc4_info = NULL; - int inmemory = ((mode & NC_INMEMORY) == NC_INMEMORY); - NC_MEM_INFO* meminfo = (NC_MEM_INFO*)parameters; + #ifdef USE_PARALLEL4 - NC_MPI_INFO* mpiinfo = (NC_MPI_INFO*)parameters; - int comm_duped = 0; /* Whether the MPI Communicator was duplicated */ - int info_duped = 0; /* Whether the MPI Info object was duplicated */ -#endif /* !USE_PARALLEL4 */ + NC_MPI_INFO* mpiinfo = NULL; + int comm_duped = 0; /* Whether the MPI Communicator was duplicated */ + int info_duped = 0; /* Whether the MPI Info object was duplicated */ +#endif LOG((3, "%s: path %s mode %d", __func__, path, mode)); assert(path && nc); + flags = (mode & NC_WRITE) ? H5F_ACC_RDWR : H5F_ACC_RDONLY; + /* Add necessary structs to hold netcdf-4 file data. */ if ((retval = nc4_nc4f_list_add(nc, path, mode))) BAIL(retval); nc4_info = NC4_DATA(nc); assert(nc4_info && nc4_info->root_grp); + nc4_info->mem.inmemory = ((mode & NC_INMEMORY) == NC_INMEMORY); + nc4_info->mem.diskless = ((mode & NC_DISKLESS) == NC_DISKLESS); + if(nc4_info->mem.inmemory) { + NC_memio* memparams = NULL; + if(parameters == NULL) + BAIL(NC_EINMEMORY); + memparams = (NC_memio*)parameters; + nc4_info->mem.memio = *memparams; /* keep local copy */ + /* As a safeguard, if !locked and NC_WRITE is set, + then we must take control of the incoming memory */ + nc4_info->mem.locked = (nc4_info->mem.memio.flags & NC_MEMIO_LOCKED) == NC_MEMIO_LOCKED; + if(!nc4_info->mem.locked && ((mode & NC_WRITE) == NC_WRITE)) { + memparams->memory = NULL; + } +#ifdef USE_PARALLEL4 + } else { + mpiinfo = (NC_MPI_INFO*)parameters; +#endif /* !USE_PARALLEL4 */ + } + /* Need this access plist to control how HDF5 handles open objects * on file close. (Setting H5F_CLOSE_SEMI will cause H5Fclose to * fail if there are any open objects in the file. */ @@ -2568,12 +2624,13 @@ nc4_open_file(const char *path, int mode, void* parameters, NC *nc) #ifdef HDF5_HAS_COLL_METADATA_OPS H5Pset_all_coll_metadata_ops(fapl_id, 1 ); #endif - if(inmemory) { - if((nc4_info->hdfid = H5LTopen_file_image(meminfo->memory,meminfo->size, - H5LT_FILE_IMAGE_DONT_COPY|H5LT_FILE_IMAGE_DONT_RELEASE - )) < 0) + if(nc4_info->mem.inmemory) { + /* validate */ + if(nc4_info->mem.memio.size == 0 || nc4_info->mem.memio.memory == NULL) + BAIL(NC_INMEMORY); + retval = NC4_open_image_file(nc4_info); + if(retval) BAIL(NC_EHDFERR); - nc4_info->no_write = NC_TRUE; } else if ((nc4_info->hdfid = H5Fopen(path, flags, fapl_id)) < 0) BAIL(NC_EHDFERR); @@ -2614,7 +2671,7 @@ nc4_open_file(const char *path, int mode, void* parameters, NC *nc) #endif if (fapl_id != H5P_DEFAULT) H5Pclose(fapl_id); if (!nc4_info) return retval; - close_netcdf4_file(nc4_info,1); /* treat like abort*/ + close_netcdf4_file(nc4_info,1,0); /* treat like abort*/ return retval; } @@ -2905,7 +2962,7 @@ NC4_abort(int ncid) /* Free any resources the netcdf-4 library has for this file's * metadata. */ - if ((retval = close_netcdf4_file(nc4_info, 1))) + if ((retval = close_netcdf4_file(nc4_info, 1, 0))) return retval; /* Delete the file, if we should. */ @@ -2920,17 +2977,19 @@ NC4_abort(int ncid) * @internal Close the netcdf file, writing any changes first. * * @param ncid File and group ID. + * @param params any extra parameters in/out of close * * @return ::NC_NOERR No error. * @author Ed Hartnett */ int -NC4_close(int ncid) +NC4_close(int ncid, void* params) { NC_GRP_INFO_T *grp; NC *nc; NC_HDF5_FILE_INFO_T *h5; int retval; + int inmemory; LOG((1, "%s: ncid 0x%x", __func__, ncid)); @@ -2944,9 +3003,57 @@ NC4_close(int ncid) if (grp->parent) return NC_EBADGRPID; + inmemory = ((h5->cmode & NC_INMEMORY) == NC_INMEMORY); + /* Call the nc4 close. */ - if ((retval = close_netcdf4_file(grp->nc4_info, 0))) + if ((retval = close_netcdf4_file(grp->nc4_info, 0, (inmemory?1:0)))) return retval; + if(inmemory && params != NULL) { + NC_memio* memio = (NC_memio*)params; + *memio = h5->mem.memio; + } + + return NC_NOERR; +} + +/** + * @internal Close an in-memory netcdf file, writing any changes first. + * + * @param ncid File and group ID. + * @param sizep ptr into which the final size is stored + * @param memp ptr into which the final memory is stored + * + * @return ::NC_NOERR No error. + * @author Ed Hartnett +*/ +int +NC4_close_mem(int ncid, size_t* sizep, void** memp) +{ + NC_GRP_INFO_T *grp; + NC *nc; + NC_HDF5_FILE_INFO_T *h5; + int retval; + + LOG((1, "%s: ncid 0x%x", __func__, ncid)); + + /* Find our metadata for this file. */ + if ((retval = nc4_find_nc_grp_h5(ncid, &nc, &grp, &h5))) + return retval; + + assert(nc && h5 && grp); + + /* This must be the root group. */ + if (grp->parent) + return NC_EBADGRPID; + + /* If the file is not an in-memory file, then treat like normal close */ + if((h5->cmode & NC_INMEMORY) == 0) + return NC4_close(ncid,NULL); + + /* Call the nc4 close and extract memory */ + if ((retval = close_netcdf4_file(grp->nc4_info, 0, 1))) + return retval; + return NC_NOERR; } @@ -3047,4 +3154,3 @@ nc4_enddef_netcdf4_file(NC_HDF5_FILE_INFO_T *h5) return sync_netcdf4_file(h5); } - diff --git a/libsrc4/nc4mem.c b/libsrc4/nc4mem.c new file mode 100644 index 0000000000..f59b21ca14 --- /dev/null +++ b/libsrc4/nc4mem.c @@ -0,0 +1,113 @@ +/********************************************************************* +* Copyright 2018, UCAR/Unidata +* See netcdf/COPYRIGHT file for copying and redistribution conditions. +* ********************************************************************/ + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * +* Copyright by The HDF Group. * +* Copyright by the Board of Trustees of the University of Illinois. * +* All rights reserved. * +* * +* This file is part of HDF5. The full HDF5 copyright notice, including * +* terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * +* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include "config.h" +#include +#include +#include +#include +#include /* must be after nc4internal.h */ +#include +#include + +#include "netcdf.h" +#include "nc4internal.h" + +#undef DEBUG + +#ifndef HDrealloc + #define HDrealloc(M,Z) realloc(M,Z) +#endif /* HDrealloc */ + +extern int NC4_image_init(NC_HDF5_FILE_INFO_T* h5); + +int +NC4_open_image_file(NC_HDF5_FILE_INFO_T* h5) +{ + int stat = NC_NOERR; + hid_t hdfid; + + /* check arguments */ + if(h5->mem.memio.memory == NULL || h5->mem.memio.size == 0) + {stat = NC_EINVAL; goto done;} + + /* Figure out the flags */ + h5->mem.flags = 0; + if(h5->mem.locked) { + h5->mem.flags |= (H5LT_FILE_IMAGE_DONT_COPY | H5LT_FILE_IMAGE_DONT_RELEASE); + } + if(!h5->no_write) + h5->mem.flags |= H5LT_FILE_IMAGE_OPEN_RW; + + /* Create the file but using our version of H5LTopen_file_image */ + hdfid = NC4_image_init(h5); + if(hdfid < 0) + {stat = NC_EHDFERR; goto done;} + /* Return file identifier */ + h5->hdfid = hdfid; + +done: + return stat; +} + +int +NC4_create_image_file(NC_HDF5_FILE_INFO_T* h5, size_t initialsz) +{ + int stat = NC_NOERR; + int hdfid; + + /* Create the file but using our version of H5LTopen_file_image */ + h5->mem.created = 1; + h5->mem.initialsize = initialsz; + hdfid = NC4_image_init(h5); + if(hdfid < 0) + {stat = NC_EHDFERR; goto done;} + /* Return file identifier */ + h5->hdfid = hdfid; +done: + return stat; +} + +int +NC4_extract_file_image(NC_HDF5_FILE_INFO_T* h5) +{ + int stat = NC_NOERR; + hid_t fapl; + herr_t herr; + NC_memio mem; + + assert(h5 && h5->hdfid && !h5->no_write); + + /* Get the file access property list */ + fapl = h5->mem.fapl; + if(fapl < 0) + {stat = NC_EHDFERR; goto done;} + + memset(&mem,0,sizeof(mem)); + herr = H5Pget_file_image(fapl, &mem.memory, &mem.size); + if(herr < 0) + {stat = NC_EHDFERR; goto done;} + h5->mem.memio = mem; + + /* Close FAPL */ + if (H5Pclose(fapl) < 0) + {stat = NC_EHDFERR; goto done;} + h5->mem.fapl = 0; +done: + return stat; +} diff --git a/libsrc4/nc4memcb.c b/libsrc4/nc4memcb.c new file mode 100644 index 0000000000..5ec166154f --- /dev/null +++ b/libsrc4/nc4memcb.c @@ -0,0 +1,857 @@ +/********************************************************************* +* Copyright 2018, UCAR/Unidata +* See netcdf/COPYRIGHT file for copying and redistribution conditions. +* ********************************************************************/ + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * +* Copyright by The HDF Group. * +* Copyright by the Board of Trustees of the University of Illinois. * +* All rights reserved. * +* * +* This file is part of HDF5. The full HDF5 copyright notice, including * +* terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * +* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * @internal Inmemory support + * + * This code is derived from H5Lt.c#H5LTopen_file_image. + * In order to make the netcdf inmemory code work, it is necessary + * to modify some of the callback functions; specifically + * image_malloc, image_realloc, and image_memcpy. + *

    + * The changes are directed at allowing the caller to + * specify two things. + *

      + *
    1. To specify (indirectly) the H5LT_FILE_IMAGE_DONT_COPY flag. + * This means that no attempt to realloc the caller provided memory + * should be made. This also means that the memory block pointer + * provided by the caller will be thesame returned by nc_close_memio(). + *
    2. The caller overallocates the memory so that there is space + * to allow the file to be modified in place. + *
    + *

    + * The existing implementation of H5LTopen_file_image has two flaws + * with respect to these properties. + *

      + *
    1. The image_realloc callback fails if + * H5LT_FILE_IMAGE_DONT_COPY flag is set even if there is room + * to allow the memory block to pretend to expand (because + * of overallocation). + *
    2. When the caller attempts to get the final memory block, + * the HDF5 library makes a copy, unless theH5LT_FILE_IMAGE_DONT_COPY + * flag is set. This is unnecessary. Note that in this situation, + * the HDF5 library will use + * image_malloc() + * followed by + * image_memcpy() + *
    + *

    + * So, the callback changes to support this properly are as follows. + *

    + *
    image_realloc
    + *
    If there is sufficient space (because of overallocation), + * the pretend to realloc and return the incoming memory block + * instead of taking the chance of doing a real realloc.
    + *
    image_malloc
    + *
    If the operation being performed is to obtain + * the space to copy the final memory, then just return + * the original memory block. Note that this case is detectable + * because the callback is given the value + * H5FD_FILE_IMAGE_OP_PROPERTY_LIST_GET.
    + *
    image_memcpy
    + *
    Similar to the image_malloc change. + * Namely, if the operation being performed is to copy + * out the final memory contents, and the final memory block + * is the same as that provided by the caller originally, then + * just do nothing. Again, this case can be detected + * by the occurrence of H5FD_FILE_IMAGE_OP_PROPERTY_LIST_GET.
    + *
    + * @author Dennis Heimbigner +*/ + +#include +#include +#include +#include + +#include +#include + +#include "nc4internal.h" + +#ifndef HDrealloc +#define HDrealloc(x,y) realloc(x,y) +#endif +#ifndef SUCCEED +#define SUCCEED 0 +#define FAIL -1 +#endif +#ifndef FALSE +#define FALSE 0 +#endif + +#undef TRACE + +#ifdef TRACE +#include +static void trace(const char* fcn, void* _udata, ...); +static void traceend(const char* fcn, void* _udata); +static void tracefail(const char* fcn); +/* In case we do not have variadic macros */ +#define TRACE1(fcn,udata,x1) trace(fcn,udata,x1) +#define TRACE2(fcn,udata,x1,x2) trace(fcn,udata,x1,x2) +#define TRACE3(fcn,udata,x1,x2,x3) trace(fcn,udata,x1,x2,x3) +#define TRACEEND(fcn,udata) traceend(fcn,udata); +#define TRACEFAIL(fcn) tracefail(fcn) +#else /*!TRACE*/ +#define TRACE1(fcn,udata,x1) +#define TRACE2(fcn,udata,x1,x2) +#define TRACE3(fcn,udata,x1,x2,x3) +#define TRACEEND(fcn,udata) +#define TRACEFAIL(fcn) +#endif + +#define DEFAULT_CREATE_MEMSIZE ((size_t)1<<16) + +#ifndef H5LT_FILE_IMAGE_DONT_COPY + +/* Flag definitions for H5LTopen_file_image() */ +#define H5LT_FILE_IMAGE_OPEN_RW 0x0001 /* Open image for read-write */ +#define H5LT_FILE_IMAGE_DONT_COPY 0x0002 /* The HDF5 lib won't copy */ +/* user supplied image buffer. The same image is open with the core driver. */ +#define H5LT_FILE_IMAGE_DONT_RELEASE 0x0004 /* The HDF5 lib won't */ +/* deallocate user supplied image buffer. The user application is reponsible */ +/* for doing so. */ +#define H5LT_FILE_IMAGE_ALL 0x0007 + +#endif /*H5LT_FILE_IMAGE_DONT_COPY*/ + + +#if 0 +/* For Lex and Yacc */ +#define COL 3 +#define LIMIT 512 +#define INCREMENT 1024 +#define TMP_LEN 256 +#define MAX(a,b) (((a)>(b)) ? (a) : (b)) +size_t input_len; +char *myinput; +size_t indent = 0; +#endif /*0*/ + +/* File Image operations + + A file image is a representation of an HDF5 file in a memory + buffer. In order to perform operations on an image in a similar way + to a file, the application buffer is copied to a FAPL buffer, which + in turn is copied to a VFD buffer. Buffer copying can decrease + performance, especially when using large file images. A solution to + this issue is to simulate the copying of the application buffer, + when actually the same buffer is used for the FAPL and the VFD. + This is implemented by using callbacks that simulate the standard + functions for memory management (additional callbacks are used for + the management of associated data structures). From the application + standpoint, a file handle can be obtained from a file image by using + the API routine H5LTopen_file_image(). This function takes a flag + argument that indicates the HDF5 library how to handle the given image; + several flag values can be combined by using the bitwise OR operator. + Valid flag values include: + + H5LT_FILE_IMAGE_OPEN_RW indicates the HDF5 library to open the file + image in read/write mode. Default is read-only mode. + + H5LT_FILE_IMAGE_DONT_COPY indicates the HDF5 library to not copy the + supplied user buffer; the same buffer will be handled by the FAPL and + the VFD driver. Default operation copies the user buffer to the FAPL and + VFD driver. + + H5LT_FILE_IMAGE_DONT_RELEASE indicates the HDF5 library to not release + the buffer handled by the FAPL and the VFD upon closing. This flag value + is only applicable when the flag value H5LT_FILE_IMAGE_DONT_COPY is set as + well. The application is responsible to release the image buffer. +*/ + +/* Data structure to pass application data to callbacks. */ +/* Modified to add NC_FILE_INFO_T ptr */ +typedef struct { + void *app_image_ptr; /* Pointer to application buffer */ + size_t app_image_size; /* Size of application buffer */ + void *fapl_image_ptr; /* Pointer to FAPL buffer */ + size_t fapl_image_size; /* Size of FAPL buffer */ + int fapl_ref_count; /* Reference counter for FAPL buffer */ + void *vfd_image_ptr; /* Pointer to VFD buffer */ + size_t vfd_image_size; /* Size of VFD buffer */ + int vfd_ref_count; /* Reference counter for VFD buffer */ + unsigned flags; /* Flags indicate how the file image will */ + /* be open */ + int ref_count; /* Reference counter on udata struct */ + NC_HDF5_FILE_INFO_T* h5; +} H5LT_file_image_ud_t; + +/* callbacks prototypes for file image ops */ +static void *local_image_malloc(size_t size, H5FD_file_image_op_t file_image_op, void *udata); +static void *local_image_memcpy(void *dest, const void *src, size_t size, H5FD_file_image_op_t file_image_op, void *udata); +static herr_t local_image_free(void *ptr, H5FD_file_image_op_t file_image_op, void *udata); +static void *local_udata_copy(void *udata); +static herr_t local_udata_free(void *udata); + +/* Definition of callbacks for file image operations. */ + + +/*------------------------------------------------------------------------- +* Function: image_malloc +* +* Purpose: Simulates malloc() function to avoid copying file images. +* The application buffer is set to the buffer on only one FAPL. +* Then the FAPL buffer can be copied to other FAPL buffers or +* to only one VFD buffer. +* +* Return: Address of "allocated" buffer, if successful. Otherwise, it returns +* NULL. +* +* Programmer: Christian Chilan +* +* Date: October 3, 2011 +* +*------------------------------------------------------------------------- +*/ +static void * +local_image_malloc(size_t size, H5FD_file_image_op_t file_image_op, void *_udata) +{ + H5LT_file_image_ud_t *udata = (H5LT_file_image_ud_t *)_udata; + void * return_value = NULL; + + TRACE1("malloc",_udata, size); + + /* callback is only used if the application buffer is not actually copied */ + if (!(udata->flags & H5LT_FILE_IMAGE_DONT_COPY)) + goto out; + + switch ( file_image_op ) { + /* the app buffer is "copied" to only one FAPL. Afterwards, FAPLs can be "copied" */ + case H5FD_FILE_IMAGE_OP_PROPERTY_LIST_SET: + if (udata->app_image_ptr == NULL) + goto out; + if (udata->app_image_size != size) + goto out; + if (udata->fapl_image_ptr != NULL) + goto out; + if (udata->fapl_image_size != 0) + goto out; + if (udata->fapl_ref_count != 0) + goto out; + + udata->fapl_image_ptr = udata->app_image_ptr; + udata->fapl_image_size = udata->app_image_size; + return_value = udata->fapl_image_ptr; + udata->fapl_ref_count++; + break; + + case H5FD_FILE_IMAGE_OP_PROPERTY_LIST_COPY: + if (udata->fapl_image_ptr == NULL) + goto out; + if (udata->fapl_image_size != size) + goto out; + if (udata->fapl_ref_count == 0) + goto out; + + return_value = udata->fapl_image_ptr; + udata->fapl_ref_count++; + break; + + case H5FD_FILE_IMAGE_OP_PROPERTY_LIST_GET: + if(!(udata->flags & H5LT_FILE_IMAGE_DONT_COPY)) + goto out; + /* fake the malloc by returning the original memory */ + return_value = udata->fapl_image_ptr; + break; + + case H5FD_FILE_IMAGE_OP_FILE_OPEN: + /* FAPL buffer is "copied" to only one VFD buffer */ + if (udata->vfd_image_ptr != NULL) + goto out; + if (udata->vfd_image_size != 0) + goto out; + if (udata->vfd_ref_count != 0) + goto out; + if (udata->fapl_image_ptr == NULL) + goto out; + if (udata->fapl_image_size != size) + goto out; + if (udata->fapl_ref_count == 0) + goto out; + + udata->vfd_image_ptr = udata->fapl_image_ptr; + udata->vfd_image_size = size; + udata->vfd_ref_count++; + return_value = udata->vfd_image_ptr; + break; + + /* added unused labels to shut the compiler up */ + case H5FD_FILE_IMAGE_OP_NO_OP: + case H5FD_FILE_IMAGE_OP_PROPERTY_LIST_CLOSE: + case H5FD_FILE_IMAGE_OP_FILE_RESIZE: + case H5FD_FILE_IMAGE_OP_FILE_CLOSE: + default: + goto out; + } /* end switch */ + + TRACEEND("malloc",_udata); + + return(return_value); + +out: + TRACEFAIL("malloc"); + return NULL; +} /* end image_malloc() */ + + +/*------------------------------------------------------------------------- +* Function: image_memcpy +* +* Purpose: Simulates memcpy() function to avoid copying file images. +* The image buffer can be set to only one FAPL buffer, and +* "copied" to only one VFD buffer. The FAPL buffer can be +* "copied" to other FAPLs buffers. +* +* Return: The address of the destination buffer, if successful. Otherwise, it +* returns NULL. +* +* Programmer: Christian Chilan +* +* Date: October 3, 2011 +* +*------------------------------------------------------------------------- +*/ +static void * +local_image_memcpy(void *dest, const void *src, size_t size, H5FD_file_image_op_t file_image_op, + void *_udata) +{ + H5LT_file_image_ud_t *udata = (H5LT_file_image_ud_t *)_udata; + + TRACE3("memcpy",_udata,dest,src,size); + + /* callback is only used if the application buffer is not actually copied */ + if (!(udata->flags & H5LT_FILE_IMAGE_DONT_COPY)) + goto out; + + switch(file_image_op) { + case H5FD_FILE_IMAGE_OP_PROPERTY_LIST_SET: + if (dest != udata->fapl_image_ptr) + goto out; + if (src != udata->app_image_ptr) + goto out; + if (size != udata->fapl_image_size) + goto out; + if (size != udata->app_image_size) + goto out; + if (udata->fapl_ref_count == 0) + goto out; + break; + + case H5FD_FILE_IMAGE_OP_PROPERTY_LIST_COPY: + if (dest != udata->fapl_image_ptr) + goto out; + if (src != udata->fapl_image_ptr) + goto out; + if (size != udata->fapl_image_size) + goto out; + if (udata->fapl_ref_count < 2) + goto out; + break; + + case H5FD_FILE_IMAGE_OP_PROPERTY_LIST_GET: + if (!(udata->flags & H5LT_FILE_IMAGE_DONT_COPY)) + goto out; + /* test: src == dest == original */ + if(src != dest || src != udata->fapl_image_ptr) + goto out; + break; + + case H5FD_FILE_IMAGE_OP_FILE_OPEN: + if (dest != udata->vfd_image_ptr) + goto out; + if (src != udata->fapl_image_ptr) + goto out; + if (size != udata->vfd_image_size) + goto out; + if (size != udata->fapl_image_size) + goto out; + if (udata->fapl_ref_count == 0) + goto out; + if (udata->vfd_ref_count != 1) + goto out; + break; + + /* added unused labels to shut the compiler up */ + case H5FD_FILE_IMAGE_OP_NO_OP: + case H5FD_FILE_IMAGE_OP_PROPERTY_LIST_CLOSE: + case H5FD_FILE_IMAGE_OP_FILE_RESIZE: + case H5FD_FILE_IMAGE_OP_FILE_CLOSE: + default: + goto out; + } /* end switch */ + +#ifdef TRACE + fprintf(stderr,"trace: memcpy: dest=%p\n",dest); +#endif + TRACEEND("memcpy",_udata); + return(dest); + +out: + TRACEFAIL("memcpy"); + return NULL; +} /* end image_memcpy() */ + + +/*------------------------------------------------------------------------- +* Function: image_realloc +* +* Purpose: Reallocates the shared application image buffer and updates data +* structures that manage buffer "copying". +* +* Return: Address of reallocated buffer, if successful. Otherwise, it returns +* NULL. +* +* Programmer: Christian Chilan +* +* Date: October 3, 2011 +* +*------------------------------------------------------------------------- +*/ +static void * +local_image_realloc(void *ptr, size_t size, H5FD_file_image_op_t file_image_op, void *_udata) +{ + H5LT_file_image_ud_t *udata = (H5LT_file_image_ud_t *)_udata; + void * return_value = NULL; + + TRACE2("realloc",_udata, ptr, size); + + /* callback is only used if the application buffer is not actually copied */ + if (!(udata->flags & H5LT_FILE_IMAGE_DONT_COPY)) + goto out; + + /* realloc() is not allowed if the image is open in read-only mode */ + if (!(udata->flags & H5LT_FILE_IMAGE_OPEN_RW)) + goto out; + + if (file_image_op == H5FD_FILE_IMAGE_OP_FILE_RESIZE) { + if (udata->vfd_image_ptr != ptr) + goto out; + + if (udata->vfd_ref_count != 1) + goto out; + + /* Modified: + 1. If the realloc new size is <= existing size, + then pretend we did a realloc and return success. + This avoids unneccessary heap operations. + 2. If the H5LT_FILE_IMAGE_DONT_COPY or + H5LT_FILE_IMAGE_DONT_RELEASE flag is set and the + realloc new size is > existing size, then fail + because the realloc() call may change the address + of the buffer. The new address cannot be + communicated to the application to release it. + 3. Otherwise, use realloc(). Note that this may have the + side effect of freeing the previous memory chunk. + */ + if(size <= udata->vfd_image_size) { + /* Ok, pretend we did a realloc */ + } else if((udata->flags & H5LT_FILE_IMAGE_DONT_RELEASE) + || (udata->flags & H5LT_FILE_IMAGE_DONT_COPY)) { + goto out; /* realloc MAY violate these flags */ + } else { + if (NULL == (udata->vfd_image_ptr = HDrealloc(ptr, size))) + goto out; + udata->vfd_image_size = size; + } + return_value = udata->vfd_image_ptr; + } /* end if */ + else + goto out; + +#ifdef TRACE + fprintf(stderr,"trace: realloc: return=%p\n",return_value); +#endif + TRACEEND("realloc",_udata); + return(return_value); + +out: + TRACEFAIL("realloc"); + return NULL; +} /* end local_image_realloc() */ + +/*------------------------------------------------------------------------- +* Function: image_free +* +* Purpose: Simulates deallocation of FAPL and VFD buffers by decreasing +* reference counters. Shared application buffer is actually +* deallocated if there are no outstanding references. +* +* Return: SUCCEED or FAIL +* +* Programmer: Christian Chilan +* +* Date: October 3, 2011 +* +*------------------------------------------------------------------------- +*/ +static herr_t +local_image_free(void *ptr, H5FD_file_image_op_t file_image_op, void *_udata) +{ + H5LT_file_image_ud_t *udata = (H5LT_file_image_ud_t *)_udata; + + TRACE1("free",_udata,ptr); + + /* callback is only used if the application buffer is not actually copied */ + if (!(udata->flags & H5LT_FILE_IMAGE_DONT_COPY)) + goto out; + + switch(file_image_op) { + case H5FD_FILE_IMAGE_OP_PROPERTY_LIST_CLOSE: + if (udata->fapl_image_ptr != ptr) + goto out; + if (udata->fapl_ref_count == 0) + goto out; + + udata->fapl_ref_count--; + + /* release the shared buffer only if indicated by the respective flag and there are no outstanding references */ + if (udata->fapl_ref_count == 0 && udata->vfd_ref_count == 0 && + !(udata->flags & H5LT_FILE_IMAGE_DONT_RELEASE)) { + free(udata->fapl_image_ptr); + udata->app_image_ptr = NULL; + udata->fapl_image_ptr = NULL; + udata->vfd_image_ptr = NULL; + } /* end if */ + break; + + case H5FD_FILE_IMAGE_OP_FILE_CLOSE: + if (udata->vfd_image_ptr != ptr) + goto out; + if (udata->vfd_ref_count != 1) + goto out; + + udata->vfd_ref_count--; + + /* release the shared buffer only if indicated by the respective flag and there are no outstanding references */ + if (udata->fapl_ref_count == 0 && udata->vfd_ref_count == 0 && + !(udata->flags & H5LT_FILE_IMAGE_DONT_RELEASE)) { + free(udata->vfd_image_ptr); + udata->app_image_ptr = NULL; + udata->fapl_image_ptr = NULL; + udata->vfd_image_ptr = NULL; + } /* end if */ + break; + + /* added unused labels to keep the compiler quite */ + case H5FD_FILE_IMAGE_OP_NO_OP: + case H5FD_FILE_IMAGE_OP_PROPERTY_LIST_SET: + case H5FD_FILE_IMAGE_OP_PROPERTY_LIST_COPY: + case H5FD_FILE_IMAGE_OP_PROPERTY_LIST_GET: + case H5FD_FILE_IMAGE_OP_FILE_OPEN: + case H5FD_FILE_IMAGE_OP_FILE_RESIZE: + default: + goto out; + } /* end switch */ + + TRACEEND("free",_udata); + return(SUCCEED); + +out: + TRACEFAIL("free"); + return(FAIL); +} /* end image_free() */ + + +/*------------------------------------------------------------------------- +* Function: udata_copy +* +* Purpose: Simulates the copying of the user data structure utilized in the +* management of the "copying" of file images. +* +* Return: Address of "newly allocated" structure, if successful. Otherwise, it +* returns NULL. +* +* Programmer: Christian Chilan +* +* Date: October 3, 2011 +* +*------------------------------------------------------------------------- +*/ +static void * +local_udata_copy(void *_udata) +{ + H5LT_file_image_ud_t *udata = (H5LT_file_image_ud_t *)_udata; + + /* callback is only used if the application buffer is not actually copied */ + if (!(udata->flags & H5LT_FILE_IMAGE_DONT_COPY)) + goto out; + + if (udata->ref_count == 0) + goto out; + + udata->ref_count++; + + return(udata); + +out: + return NULL; +} /* end udata_copy */ + + +/*------------------------------------------------------------------------- +* Function: udata_free +* +* Purpose: Simulates deallocation of the user data structure utilized in the +* management of the "copying" of file images. The data structure is +* actually deallocated when there are no outstanding references. +* +* Return: SUCCEED or FAIL +* +* Programmer: Christian Chilan +* +* Date: October 3, 2011 +* +*------------------------------------------------------------------------- +*/ +static herr_t +local_udata_free(void *_udata) +{ + H5LT_file_image_ud_t *udata = (H5LT_file_image_ud_t *)_udata; + + /* callback is only used if the application buffer is not actually copied */ + if (!(udata->flags & H5LT_FILE_IMAGE_DONT_COPY)) + goto out; + + if (udata->ref_count == 0) + goto out; + + udata->ref_count--; + + /* checks that there are no references outstanding before deallocating udata */ + if (udata->ref_count == 0 && udata->fapl_ref_count == 0 && + udata->vfd_ref_count == 0) + free(udata); + + return(SUCCEED); + +out: + return(FAIL); +} /* end udata_free */ + +/* End of callbacks definitions for file image operations */ + +/*------------------------------------------------------------------------- +* +* Public functions +* +*------------------------------------------------------------------------- +*/ + +/*------------------------------------------------------------------------- +* Function: H5LTopen_file_image +* +* Purpose: Open a user supplied file image using the core file driver. +* +* Return: File identifier, Failure: -1 +* +* Programmer: Christian Chilan +* +* Date: October 3, 2011 +* +*------------------------------------------------------------------------- +*/ +int +NC4_image_init(NC_HDF5_FILE_INFO_T* h5) +{ + hid_t fapl = -1, file_id = -1; /* HDF5 identifiers */ + unsigned file_open_flags;/* Flags for image open */ + char file_name[64]; /* Filename buffer */ + size_t alloc_incr; /* Buffer allocation increment */ + size_t min_incr = 65536; /* Minimum buffer increment */ + double buf_prcnt = 0.1f; /* Percentage of buffer size to set + as increment */ + size_t buf_size = h5->mem.memio.size; + void* buf_ptr = h5->mem.memio.memory; + unsigned flags = h5->mem.flags; + + static long file_name_counter; + H5FD_file_image_callbacks_t callbacks = {&local_image_malloc, &local_image_memcpy, + &local_image_realloc, &local_image_free, + &local_udata_copy, &local_udata_free, + (void *)NULL}; + /* check arguments */ + if (buf_ptr == NULL) { + if(h5->mem.created) { + if(h5->mem.memio.size == 0) h5->mem.memio.size = DEFAULT_CREATE_MEMSIZE; + h5->mem.memio.memory = malloc(h5->mem.memio.size); + } else + goto out; /* open requires an input buffer */ + } + // reset + buf_size = h5->mem.memio.size; + buf_ptr = h5->mem.memio.memory; + /* validate */ + if (buf_ptr == NULL || buf_size == 0) + goto out; + if (flags & (unsigned)~(H5LT_FILE_IMAGE_ALL)) + goto out; + + /* Create FAPL to transmit file image */ + if ((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) + goto out; + + /* set allocation increment to a percentage of the supplied buffer size, or + * a pre-defined minimum increment value, whichever is larger + */ + if ((buf_prcnt * buf_size) > min_incr) + alloc_incr = (size_t)(buf_prcnt * buf_size); + else + alloc_incr = min_incr; + + /* Configure FAPL to use the core file driver */ + if (H5Pset_fapl_core(fapl, alloc_incr, FALSE) < 0) + goto out; + + /* Set callbacks for file image ops ONLY if the file image is NOT copied */ + if (flags & H5LT_FILE_IMAGE_DONT_COPY) + { + H5LT_file_image_ud_t *udata; /* Pointer to udata structure */ + + /* Allocate buffer to communicate user data to callbacks */ + if (NULL == (udata = (H5LT_file_image_ud_t *)malloc(sizeof(H5LT_file_image_ud_t)))) + goto out; + + /* Initialize udata with info about app buffer containing file image and flags */ + udata->app_image_ptr = buf_ptr; + udata->app_image_size = buf_size; + udata->fapl_image_ptr = NULL; + udata->fapl_image_size = 0; + udata->fapl_ref_count = 0; + udata->vfd_image_ptr = NULL; + udata->vfd_image_size = 0; + udata->vfd_ref_count = 0; + udata->flags = flags; + udata->ref_count = 1; /* corresponding to the first FAPL */ + udata->h5 = h5; + + /* copy address of udata into callbacks */ + callbacks.udata = (void *)udata; + + /* Set file image callbacks */ + if (H5Pset_file_image_callbacks(fapl, &callbacks) < 0) { + free(udata); + goto out; + } /* end if */ + } + + /* Assign file image in user buffer to FAPL */ + if (H5Pset_file_image(fapl, buf_ptr, buf_size) < 0) + goto out; + + /* set file open flags */ + if (flags & H5LT_FILE_IMAGE_OPEN_RW) + file_open_flags = H5F_ACC_RDWR; + else + file_open_flags = H5F_ACC_RDONLY; + + /* define a unique file name */ + snprintf(file_name, (sizeof(file_name) - 1), "file_image_%ld", file_name_counter++); + + /* Assign file image in FAPL to the core file driver */ + if(h5->mem.created) { + file_open_flags |= H5F_ACC_TRUNC; + if ((file_id = H5Fcreate(file_name, file_open_flags, H5P_DEFAULT, fapl)) < 0) + goto out; + } else { + if ((file_id = H5Fopen(file_name, file_open_flags, fapl)) < 0) + goto out; + } + + h5->mem.fapl = fapl; + + /* Return file identifier */ + return file_id; + +out: + H5E_BEGIN_TRY { + if(fapl >= 0) + H5Pclose(fapl); + } H5E_END_TRY; + return -1; +} /* end H5LTopen_file_image() */ + +#ifdef TRACE + +static void +printudata(H5LT_file_image_ud_t* udata) +{ + if(udata == NULL) return; + fprintf(stderr," flags=0x%0x",udata->flags); + fprintf(stderr," flags=0x%0x",udata->flags); + fprintf(stderr," ref_count=%d",udata->ref_count); + fprintf(stderr," app=(%p,%lld)",udata->app_image_ptr,(long long)udata->app_image_size); + fprintf(stderr," fapl=(%p,%lld)[%d]",udata->fapl_image_ptr,(long long)udata->fapl_image_size,udata->fapl_ref_count); + fprintf(stderr," vfd=(%p,%lld)[%d]",udata->vfd_image_ptr,(long long)udata->vfd_image_size,udata->vfd_ref_count); +} + +static void trace(const char* fcn, void* _udata, ...) +{ + H5LT_file_image_ud_t *udata = NULL; + va_list ap; + + va_start(ap, _udata); /* Requires the last fixed parameter (to get the address) */ + udata = (H5LT_file_image_ud_t *)_udata; + if(strcmp("malloc",fcn)==0) { + size_t size = va_arg(ap,size_t); + fprintf(stderr,"trace: %s: size=%lld udata=",fcn,(long long)size); + printudata(udata); + fprintf(stderr,"\n"); + } else if(strcmp("realloc",fcn)==0) { + void* ptr = va_arg(ap,void*); + size_t size = va_arg(ap,size_t); + fprintf(stderr,"trace: %s: ptr=%p, size=%lld udata=",fcn,ptr,(long long)size); + printudata(udata); + fprintf(stderr,"\n"); + } else if(strcmp("free",fcn)==0) { + void* ptr = va_arg(ap,void*); + fprintf(stderr,"trace: %s: ptr=%p udata=",fcn,ptr); + printudata(udata); + fprintf(stderr,"\n"); + } else if(strcmp("memcpy",fcn)==0) { + void* dest = va_arg(ap,void*); + void* src = va_arg(ap,void*); + size_t size = va_arg(ap,size_t); + fprintf(stderr,"trace: %s: dest=%p src=%p size=%lld udata=",fcn,dest,src,(long long)size); + printudata(udata); + fprintf(stderr,"\n"); + } else { + fprintf(stderr,"unknown fcn: %s\n",fcn); + } + va_end(ap); + fflush(stderr); +} + +static void traceend(const char* fcn, void* _udata) +{ + H5LT_file_image_ud_t *udata = (H5LT_file_image_ud_t *)_udata; + fprintf(stderr,"traceend: %s: udata=",fcn); + printudata(udata); + fprintf(stderr,"\n"); + fflush(stderr); +} + +static void +tracefail(const char* fcn) +{ + fprintf(stderr,"fail: %s",fcn); + fflush(stderr); +} + +#endif /*TRACE*/ diff --git a/libsrcp/ncpdispatch.c b/libsrcp/ncpdispatch.c index 8bd2a5f7c9..d651abc4f4 100644 --- a/libsrcp/ncpdispatch.c +++ b/libsrcp/ncpdispatch.c @@ -236,7 +236,7 @@ NCP_abort(int ncid) static int -NCP_close(int ncid) +NCP_close(int ncid, void* ignored) { NC* nc; NCP_INFO* nc5; diff --git a/nc_test/CMakeLists.txt b/nc_test/CMakeLists.txt index f6e5854069..0d0140e087 100644 --- a/nc_test/CMakeLists.txt +++ b/nc_test/CMakeLists.txt @@ -53,7 +53,7 @@ IF(LARGE_FILE_TESTS) ENDIF() IF(BUILD_DISKLESS) - SET(TESTFILES ${TESTFILES} tst_diskless tst_diskless3 tst_diskless4 tst_diskless5) + SET(TESTFILES ${TESTFILES} tst_diskless tst_diskless3 tst_diskless4 tst_diskless5 tst_inmemory) IF(USE_NETCDF4) SET(TESTFILES ${TESTFILES} tst_diskless2) ENDIF() @@ -83,6 +83,7 @@ IF(BUILD_UTILITIES) add_sh_test(nc_test run_diskless2) ENDIF(LARGE_FILE_TESTS) add_sh_test(nc_test run_diskless5) + add_sh_test(nc_test run_inmemory) ENDIF(BUILD_DISKLESS) ENDIF(BUILD_UTILITIES) diff --git a/nc_test/Make0 b/nc_test/Make0 index 65eb29dc5e..59bdb93411 100644 --- a/nc_test/Make0 +++ b/nc_test/Make0 @@ -1,9 +1,7 @@ # Test c output -T=tst_diskless2 +T=tst_inmemory -#SRC=error.c test_get.c test_put.c test_read.c test_write.c util.c - -#ARGS=netcdf4 +#ARGS=t #CMD=valgrind --leak-check=full #CMD=export NETCDF_LOG_LEVEL=5 ;gdb --args diff --git a/nc_test/Makefile.am b/nc_test/Makefile.am index 5ed4f80901..9952af2000 100644 --- a/nc_test/Makefile.am +++ b/nc_test/Makefile.am @@ -54,7 +54,7 @@ check_PROGRAMS = $(TESTPROGRAMS) # Build Diskless test helpers if BUILD_DISKLESS -check_PROGRAMS += tst_diskless tst_diskless3 tst_diskless4 tst_diskless5 +check_PROGRAMS += tst_diskless tst_diskless3 tst_diskless4 tst_diskless5 tst_inmemory if USE_NETCDF4 check_PROGRAMS += tst_diskless2 endif @@ -64,7 +64,7 @@ TESTS = $(TESTPROGRAMS) if BUILD_UTILITIES if BUILD_DISKLESS -TESTS += run_diskless.sh run_diskless5.sh +TESTS += run_diskless.sh run_diskless5.sh run_inmemory.sh if BUILD_MMAP TESTS += run_mmap.sh run_mmap.log: run_diskless.log @@ -81,9 +81,13 @@ endif # Distribute the .c files so that m4 isn't required on the users # machine. -EXTRA_DIST = test_get.m4 test_put.m4 run_diskless.sh run_diskless2.sh \ -run_diskless5.sh run_mmap.sh run_pnetcdf_test.sh test_read.m4 \ -test_write.m4 ref_tst_diskless2.cdl tst_diskless5.cdl CMakeLists.txt + +# Distribute the .c files so that m4 isn't required on the users +# machine. +EXTRA_DIST = test_get.m4 test_put.m4 run_diskless.sh run_diskless2.sh \ +run_diskless5.sh run_mmap.sh run_pnetcdf_test.sh test_read.m4 \ +test_write.m4 ref_tst_diskless2.cdl tst_diskless5.cdl run_inmemory.sh \ +CMakeLists.txt # These files are created by the tests. CLEANFILES = nc_test_classic.nc nc_test_64bit.nc nc_test_netcdf4.nc \ diff --git a/nc_test/run_inmemory.sh b/nc_test/run_inmemory.sh new file mode 100755 index 0000000000..8a61277a2c --- /dev/null +++ b/nc_test/run_inmemory.sh @@ -0,0 +1,49 @@ +#!/bin/sh + + +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ../test_common.sh + +set -e + +# Get the target OS and CPU +CPU=`uname -p` +OS=`uname` + +#Constants +FILE3=tst_inmemory3 +CREATE3=tst_inmemory3_create +FILE4=tst_inmemory4 +CREATE4=tst_inmemory4_create + +echo "" +echo "*** Testing in-memory operations" + +HASNC4=`${top_builddir}/nc-config --has-nc4` + +# Execute the core of the inmemory tests +if ! ${execdir}/tst_inmemory; then + echo "FAIL: tst.inmemory" +fi + +echo "**** Test ncdump of the resulting inmemory data" +${NCDUMP} -n "${FILE3}" ${FILE3}.nc > ${FILE3}.cdl +${NCDUMP} -n "${FILE3}" ${CREATE3}.nc > ${CREATE3}.cdl +if ! diff -wb ${FILE3}.cdl ${CREATE3}.cdl ; then + echo "FAIL: ${FILE3}.cdl vs ${CREATE3}.cdl +fi + +if test "x$HASNC4" = "xyes" ; then +${NCDUMP} ${FILE4}.nc > ${FILE4}.cdl +${NCDUMP} ${CREATE4}.nc > ${CREATE4}.cdl +if diff -wb ${FILE4}.cdl ${CREATE4}.cdl ; then + echo "FAIL: ${FILE4}.cdl vs ${CREATE4}.cdl +fi + +# cleanup +rm -f ${FILE3}.nc ${FILE4}.nc ${CREATE3}.nc ${CREATE4}.nc +rm -f ${FILE3}.cdl ${FILE4}.cdl ${CREATE3}.cdl ${CREATE4}.cdl + +echo "PASS: all inmemory tests" + +exit 0 diff --git a/nc_test/tst_def_var_fill.c b/nc_test/tst_def_var_fill.c index 30d256b11e..96e7f91450 100644 --- a/nc_test/tst_def_var_fill.c +++ b/nc_test/tst_def_var_fill.c @@ -10,11 +10,6 @@ #include #include #include -#ifdef HAVE_LIBGEN_H -#include /* basename() */ -#else -#define basename(X) X -#endif #include @@ -52,7 +47,7 @@ int main(int argc, char** argv) { else strcpy(filename, "tst_def_var_fill.nc"); char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for def_var_fill ", basename(argv[0])); + sprintf(cmd_str, "*** TESTING C %s for def_var_fill ", argv[0]); printf("%-66s ------ ", cmd_str); fflush(stdout); free(cmd_str); diff --git a/nc_test/tst_diskless5.c b/nc_test/tst_diskless5.c index 1dcf1b3bed..8113d89cb0 100644 --- a/nc_test/tst_diskless5.c +++ b/nc_test/tst_diskless5.c @@ -17,7 +17,7 @@ #include "netcdf_mem.h" #undef MEM -#define DISKLESS +#undef DISKLESS #define USEINT #undef DUMP diff --git a/nc_test/tst_inmemory.c b/nc_test/tst_inmemory.c new file mode 100644 index 0000000000..b0f8e4bec6 --- /dev/null +++ b/nc_test/tst_inmemory.c @@ -0,0 +1,645 @@ +/** \file \internal +Basic NC_INMEMORY API tests both for netcdf-3 and netcdf-4 + +Copyright 2011, UCAR/Unidata. See COPYRIGHT file for copying and +redistribution conditions. +*/ + +#undef DDBG + +#include +#include +#include +#ifdef HAVE_UNISTD_H +#include +#endif + +#include "netcdf.h" +#include "netcdf_mem.h" +#include "ncbytes.h" +#include "nc_tests.h" +#include "err_macros.h" + +#include + +#ifdef USE_NETCDF4 +extern int H5Eprint1(FILE * stream); +#endif + +#define FLAGS4 (NC_INMEMORY|NC_NETCDF4|NC_CLOBBER) +#define FLAGS3 (NC_INMEMORY|NCCLOBBER) + +#define NC_NETCDF3 0 +#define MODIFIED 1 + +#define LARGE_SPACE (1<<18) + +#define FILE3 "tst_inmemory3.nc" +#define FILE4 "tst_inmemory4.nc" +#define CREATE3 "tst_inmemory3_create.nc" +#define CREATE4 "tst_inmemory4_create.nc" +#define XFAIL "tst_xfail.nc" +#define MISC "tst_misc.nc" + +#define CREATEFILE3 "tst_memcreate3.nc" +#define CREATEFILE4 "tst_memcreate4.nc" + +/* Make no dimension larger than this */ +#define MAXDIMLEN 100 +#define NDIMS 2 +#define UNLIM_LEN 2 +#define DIM0_NAME "fun" +#define DIM1_NAME "money" +#define DIM1_LEN 8 +#define ATT0_NAME "home" +#define ATT0_TEXT "earthship" +#define NVARS 3 +#define VAR0_NAME "nightlife" +#define VAR1_NAME "time" +#define VAR2_NAME "taxi_distance" +#define VAR3_NAME "miles" + +#define FLOATVAL ((float)42.22) + +/* +CDL for created file: +netcdf tst_inmemory3 { +dimensions: + fun = UNLIMITED ; // (2 currently) + money = 8 ; +variables: + int nightlife(fun, money) ; + float time ; + short taxi_distance(money) ; + +// global attributes: + :home = "earthship" ; +data: + + nightlife = + 0, 100, 200, 300, 400, 500, 600, 700, + 800, 900, 1000, 1100, 1200, 1300, 1400, 1500 ; + + time = 42.22 ; + + taxi_distance = 0, 1, 2, 3, 4, 5, 6, 7 ; +} +*/ + +#ifdef DDBG +#undef ERR +static void +fail(int line) { + fflush(stdout); + fprintf(stderr,"\nline=%d\n",line); + fflush(stderr); + exit(1); +} +#define ERR fail(__LINE__) +#endif + +/* Store any error message */ +static char errmsg[4096]; + +static int +check(int stat, const char* file, int line, int xfail) +{ + int pass = ((!xfail && stat == NC_NOERR) || (xfail && stat != NC_NOERR)); + fflush(stdout); + if(!xfail) { + if(stat != NC_NOERR) { + fprintf(stderr,"***Fail: line: %d; status=%d %s\n", + line,stat,nc_strerror(stat)); + err++; + } + } else { /*xfail*/ + if(stat == NC_NOERR) { + fprintf(stderr,"***XFail Fail: line: %d; passed instead of failed\n", + line); + err++; + } else { + fprintf(stderr,"\t***XFail: status=%d %s\n", + stat,nc_strerror(stat)); + } + stat = NC_NOERR; /* because xfail */ + } + fflush(stderr); + return stat; +} + +#define CHECK(expr) {stat = check((expr),__FILE__,__LINE__,0); if(stat) return stat;} +#define XCHECK(expr) {stat = check((expr),__FILE__,__LINE__,1); if(stat) return stat;} + +#define REPORT(xfail,expr) {if((xfail)) {XCHECK((expr));} else {CHECK((expr));}} + +/**************************************************/ + +static void +removefile(const char* path) +{ + unlink(path); +} + +static int +readfile(const char* path, NC_memio* memio) +{ + int status = NC_NOERR; + FILE* f = NULL; + size_t filesize = 0; + size_t count = 0; + char* memory = NULL; + char* p = NULL; + + /* Open the file for reading */ +#ifdef _MSC_VER + f = fopen(path,"rb"); +#else + f = fopen(path,"r"); +#endif + if(f == NULL) + {status = errno; goto done;} + /* get current filesize */ + if(fseek(f,0,SEEK_END) < 0) + {status = errno; goto done;} + filesize = (size_t)ftell(f); + /* allocate memory */ + memory = malloc((size_t)filesize); + if(memory == NULL) + {status = NC_ENOMEM; goto done;} + /* move pointer back to beginning of file */ + rewind(f); + count = filesize; + p = memory; + while(count > 0) { + size_t actual; + actual = fread(p,1,count,f); + if(actual == 0 || ferror(f)) + {status = NC_EIO; goto done;} + count -= actual; + p += actual; + } + if(memio) { + memio->size = (size_t)filesize; + memio->memory = memory; + } +done: + if(status != NC_NOERR && memory != NULL) + free(memory); + if(f != NULL) fclose(f); + return status; +} + + +static int +writefile(const char* path, NC_memio* memio) +{ + int status = NC_NOERR; + FILE* f = NULL; + size_t count = 0; + char* memory = NULL; + char* p = NULL; + + /* Open the file for writing */ +#ifdef _MSC_VER + f = fopen(path,"wb"); +#else + f = fopen(path,"w"); +#endif + if(f == NULL) + {status = errno; goto done;} + count = memio->size; + p = memio->memory; + while(count > 0) { + size_t actual; + actual = fwrite(p,1,count,f); + if(actual == 0 || ferror(f)) + {status = NC_EIO; goto done;} + count -= actual; + p += actual; + } +done: + if(f != NULL) fclose(f); + return status; +} + + +/* Duplicate an NC_memio instance; needed to avoid + attempting to use memory that might have been realloc'd + Allow the new memory to be larger than the src memory +*/ +int +duplicatememory(NC_memio* src, NC_memio* target, size_t alloc) +{ + if(src == NULL || target == NULL || src->size == 0 || src->memory == NULL) + return NC_EINVAL; + *target = *src; + if(alloc == 0) alloc = src->size; + target->memory = malloc(alloc); + if(target->memory == NULL) + return NC_ENOMEM; + memcpy(target->memory,src->memory,src->size); + target->size = alloc; + return NC_NOERR; +} + +/* +Given an ncid of a created file, fill in the meta-data +and data as described by the above CDL. Do not close +the file. +*/ + +static int +define_metadata(int ncid) +{ + int stat = NC_NOERR; + int dimid[NDIMS], varid0, varid1, varid2; + short short_data[DIM1_LEN]; + size_t start[1] = {0}; + size_t count[1] = {DIM1_LEN}; + int dimprod = (UNLIM_LEN*DIM1_LEN); + int i; + float float_data; + int nightdata[UNLIM_LEN*DIM1_LEN] ; + + /* Create data to write */ + float_data = FLOATVAL; + + for (i = 0; i < DIM1_LEN; i++) + short_data[i] = i; + + for (i = 0; i < dimprod; i++) + nightdata[i] = (100*i); + + CHECK(nc_put_att_text(ncid, NC_GLOBAL, ATT0_NAME, + sizeof(ATT0_TEXT), ATT0_TEXT)); + + CHECK(nc_def_dim(ncid, DIM0_NAME, NC_UNLIMITED, &dimid[0])); + CHECK(nc_def_dim(ncid, DIM1_NAME, DIM1_LEN, &dimid[1])); + + CHECK(nc_def_var(ncid, VAR0_NAME, NC_INT, NDIMS, dimid, &varid0)); + CHECK(nc_def_var(ncid, VAR1_NAME, NC_FLOAT, 0, NULL, &varid1)); + CHECK(nc_def_var(ncid, VAR2_NAME, NC_SHORT, 1, &dimid[1], &varid2)); + + CHECK(nc_enddef(ncid)); + + CHECK(nc_put_vara_float(ncid, varid1, NULL, NULL, &float_data)); + CHECK(nc_put_vara_short(ncid, varid2, start, count, short_data)); + + { + size_t start[2] = {0,0}; + size_t count[2] = {2,DIM1_LEN}; + CHECK(nc_put_vara_int(ncid, varid0, start, count, nightdata)); + } + + return stat; +} + + +/* +Create our reference file as a real on-disk file +and read it back into memory +*/ + +static int +create_reference_file(const char* filename, int mode, NC_memio* filedata) +{ + int stat = NC_NOERR; + int ncid; + + CHECK(nc_create(filename, mode|NC_CLOBBER, &ncid)); /* overwrite */ + CHECK(define_metadata(ncid)); + CHECK(nc_close(ncid)); + + /* Read back the contents of the file into memory */ + if(filedata != NULL) { + memset(filedata,0,sizeof(NC_memio)); + CHECK(readfile(filename,filedata)); + } + return stat; +} + +static int +modify_file(int ncid) +{ + int stat = NC_NOERR; + size_t i; + int varid3; + int dimid[1]; + size_t len; + int data[MAXDIMLEN]; + + /* Get id of the unlimited dimension */ + if((stat=nc_inq_dimid(ncid, DIM0_NAME, dimid))) goto done; + /* get current dim length */ + if((stat=nc_inq_dimlen(ncid, dimid[0], &len))) goto done; + /* open file for new meta-data */ + if((stat=nc_redef(ncid))) goto done; + /* Define a new variable */ + if((stat=nc_def_var(ncid, VAR3_NAME, NC_INT, 1, dimid, &varid3))) goto done; + /* close metadata */ + if((stat=nc_enddef(ncid))) goto done; + /* Write data to new variable */ + for(i=0;imemory != NULL) + free(memio->memory); + } +} + +static int +test_open(const char* path, NC_memio* filedata, int mode) +{ + int stat = NC_NOERR; + NC_memio duplicate; + NC_memio* finaldata = NULL; + int ncid; + int xmode = mode; /* modified mode */ + + finaldata = calloc(1,sizeof(NC_memio)); + + fprintf(stderr,"\n\t***Test open 1: nc_open_mem(): read-only\n"); + CHECK(duplicatememory(filedata,&duplicate,0)); + CHECK(nc_open_mem(path, xmode, duplicate.size, duplicate.memory, &ncid)); + CHECK(verify_file(ncid,!MODIFIED)); + CHECK(nc_close(ncid)); + free(duplicate.memory); + + fprintf(stderr,"\n\t***Test open 2: nc_open_memio(): read-only\n"); + CHECK(duplicatememory(filedata,&duplicate,0)); + duplicate.flags = NC_MEMIO_LOCKED; + CHECK(nc_open_memio(path, xmode, &duplicate, &ncid)) + CHECK(verify_file(ncid,!MODIFIED)); + CHECK(nc_close_memio(ncid,finaldata)); + /* Published returned finaldata */ + fprintf(stderr,"\tfinaldata: size=%lld memory=%p\n",(unsigned long long)finaldata->size,finaldata->memory); + /* Verify that finaldata is same */ + if(finaldata->size != duplicate.size) CHECK(NC_EINVAL); + if(finaldata->memory != duplicate.memory) CHECK(NC_EINVAL); + free(finaldata->memory); + + fprintf(stderr,"\n\t***Test open 3: nc_open_memio(): read-write, copy\n"); + xmode |= NC_WRITE; /* allow file to be modified */ + CHECK(duplicatememory(filedata,&duplicate,0)); + CHECK(nc_open_memio(path, xmode, &duplicate, &ncid)) + /* modify file */ + CHECK(modify_file(ncid)); + CHECK(verify_file(ncid,MODIFIED)); + CHECK(nc_close_memio(ncid,finaldata)); + /* Published returned finaldata */ + fprintf(stderr,"\tfinaldata: size=%lld memory=%p\n",(unsigned long long)finaldata->size,finaldata->memory); + /* Verify that finaldata is same */ + if(finaldata->size < filedata->size) CHECK(NC_EINVAL); + /* As a safeguard, the memory in duplicate should have been set to NULL*/ + free(finaldata->memory); + + fprintf(stderr,"\n\t***Test open 4: nc_open_memio(): read-write, locked, extra space\n"); + /* Store the filedata in a memory chunk that leaves room for modification */ + CHECK(duplicatememory(filedata,&duplicate,LARGE_SPACE)); + /* Lock the duplicate memory */ + duplicate.flags |= NC_MEMIO_LOCKED; + xmode |= NC_WRITE; /* allow file to be modified */ + CHECK(nc_open_memio(path, xmode, &duplicate, &ncid)) + /* modify file */ + CHECK(modify_file(ncid)); + CHECK(verify_file(ncid,MODIFIED)); + CHECK(nc_close_memio(ncid,finaldata)); + /* Published returned finaldata */ + fprintf(stderr,"\tfinaldata: size=%lld memory=%p\n",(unsigned long long)finaldata->size,finaldata->memory); + /* Check returned finaldata */ + if(finaldata->size != duplicate.size) CHECK(NC_EINVAL); + if(finaldata->memory != duplicate.memory) CHECK(NC_EINVAL); + free(finaldata->memory); + + return stat; +} + +static int +test_create(const char* path, int mode) +{ + int stat = NC_NOERR; + NC_memio* finaldata = NULL; + int ncid; + int xmode = mode; + + fprintf(stderr,"\n\t***Test create 1: nc_create_memio(): no initialsize\n"); + CHECK(nc_create_mem(path, xmode, 0, &ncid)) + /* create file metadata */ + CHECK(define_metadata(ncid)); + CHECK(verify_file(ncid,!MODIFIED)); + finaldata = calloc(1,sizeof(NC_memio)); + CHECK(nc_close_memio(ncid,finaldata)); + /* Published returned finaldata */ + fprintf(stderr,"\tfinaldata: size=%lld memory=%p\n",(unsigned long long)finaldata->size,finaldata->memory); + free(finaldata->memory); + + fprintf(stderr,"\n\t***Test create 2: nc_create_memio(): initialsize; save file\n"); + CHECK(nc_create_mem(path, xmode, LARGE_SPACE, &ncid)) + /* create file metadata */ + CHECK(define_metadata(ncid)); + CHECK(verify_file(ncid,!MODIFIED)); + finaldata = calloc(1,sizeof(NC_memio)); + CHECK(nc_close_memio(ncid,finaldata)); + /* Published returned finaldata */ + fprintf(stderr,"\tfinaldata: size=%lld memory=%p\n",(unsigned long long)finaldata->size,finaldata->memory); + /* Write out the final data as a .nc file */ + CHECK(writefile(path,finaldata)); + free(finaldata->memory); + return stat; +} + +static int +test_misc(const char* path, int mode, NC_memio* filedata) +{ + int stat = NC_NOERR; + int ncid; + int xmode = mode; + NC_memio duplicate; + + fprintf(stderr,"\n\t***Test misc 1: use nc_close on created inmemory file\n"); + CHECK(nc_create_mem(MISC, xmode, 0, &ncid)) + CHECK(nc_close(ncid)); + removefile(MISC); + + fprintf(stderr,"\n\t***Test misc 2: use nc_close on opened inmemory file\n"); + CHECK(duplicatememory(filedata,&duplicate,0)); + CHECK(nc_open_memio(path, xmode, &duplicate, &ncid)) + CHECK(verify_file(ncid,!MODIFIED)); + CHECK(nc_close(ncid)); + removefile(MISC); + + return stat; +} + +/* Test various edge conditions to ensure they fail correctly */ +static int +test_xfail(const char* path, int mode, NC_memio* filedata) +{ + int stat = NC_NOERR; + NC_memio duplicate; + int ncid; + int xmode = mode; /* modified mode */ + + fprintf(stderr,"\n\t***Test xfail 1: nc_open_mem(): write to read-only\n"); + CHECK(duplicatememory(filedata,&duplicate,0)); + CHECK(nc_open_mem(XFAIL, xmode, duplicate.size, duplicate.memory, &ncid)); + XCHECK(nc_redef(ncid)); + CHECK(nc_abort(ncid)); + free(duplicate.memory); + + fprintf(stderr,"\n\t***Test xfail 2: nc_open_memio(): modify without overallocating\n"); + if((mode & NC_NETCDF4)) { + fprintf(stderr,"\tSuppressed because of HDF5 library bug\n"); + } else { + /* With HDF5 1.8.20, and possibly other versions, + this tests causes a seg fault in the HDF5 Library. + So until it is fixed, just leave well enough alone */ + NC_memio* finaldata = NULL; + finaldata = calloc(1,sizeof(NC_memio)); + CHECK(duplicatememory(filedata,&duplicate,0)); + duplicate.flags = NC_MEMIO_LOCKED; + xmode |= NC_WRITE; + CHECK(nc_open_memio(XFAIL, xmode, &duplicate, &ncid)) + XCHECK(modify_file(ncid)); + CHECK(nc_abort(ncid)); + free(finaldata->memory); + } + + return stat; +} + +int +main(int argc, char **argv) +{ + int stat = NC_NOERR; + int i; + void* filedata = NULL; + NC_memio filedata3; + NC_memio filedata4; + + nc_set_log_level(0); + +#ifdef USE_NETCDF4 + H5Eprint1(stderr); +#endif + + fprintf(stderr,"\n*** Testing the inmemory API: netcdf-3.\n"); + CHECK(create_reference_file(FILE3,NC_NETCDF3,&filedata3)); /* netcdf-3 */ + CHECK(test_open(FILE3,&filedata3,NC_NETCDF3)); + CHECK(test_create(CREATE3,NC_NETCDF3)); + CHECK(test_misc(FILE3, NC_NETCDF3, &filedata3)); + CHECK(test_xfail(FILE3, NC_NETCDF3, &filedata3)); + memiofree(&filedata3); + +#ifdef USE_NETCDF4 + fprintf(stderr,"\n*** Testing the inmemory API: netcdf-4.\n"); + CHECK(create_reference_file(FILE4,NC_NETCDF4,&filedata4)); + CHECK(test_open(FILE4,&filedata4,NC_NETCDF4)); + CHECK(test_create(CREATE4,NC_NETCDF4)); + CHECK(test_misc(FILE4,NC_NETCDF4, &filedata4)); + CHECK(test_xfail(FILE4, NC_NETCDF4, &filedata4)); + memiofree(&filedata4); +#endif + + SUMMARIZE_ERR; + + FINAL_RESULTS; + + return 0; +} diff --git a/nc_test4/perftest.sh b/nc_test4/perftest.sh index 46f0fd28ae..04caa4900a 100644 --- a/nc_test4/perftest.sh +++ b/nc_test4/perftest.sh @@ -7,6 +7,7 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh +set -x set -e echo "Testing performance of nc_create and nc_open on file with large metadata"