diff --git a/.github/workflows/run_tests_ubuntu.yml b/.github/workflows/run_tests_ubuntu.yml index 927ea68e4b..a21ac5b001 100644 --- a/.github/workflows/run_tests_ubuntu.yml +++ b/.github/workflows/run_tests_ubuntu.yml @@ -4,7 +4,7 @@ name: Run Ubuntu/Linux netCDF Tests -on: [ pull_request ] +on: [pull_request] jobs: diff --git a/CMakeLists.txt b/CMakeLists.txt index e89e351c5e..846ceb6416 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -623,26 +623,6 @@ ENDIF(ENABLE_STRICT_NULL_BYTE_HEADER_PADDING) # 3. is nczarr enabled? # We need separate flags for cases 1 and 2 -# We need to determine if libsz is available both for HDF5 and NCZarr -# If user has specified the `SZIP_LIBRARY`, use it; otherwise try to find... -IF(NOT SZIP_LIBRARY) - FIND_LIBRARY(SZIP PATH NAMES szip sz sz2) - IF(SZIP) - SET(SZIP_LIBRARY ${SZIP}) - ELSE() - UNSET(SZIP_LIBRARY) - UNSET(SZIP) - ENDIF() -ENDIF() - -IF(SZIP_LIBRARY) - SET(SZIP_FOUND yes) - SET(HAVE_SZ yes) -ELSE() - SET(SZIP_FOUND no) - SET(HAVE_SZ no) -ENDIF() - ## # Option to Enable HDF5 # @@ -905,15 +885,6 @@ IF(USE_HDF5) int x = 1;}" USE_HDF5_SZIP) IF(USE_HDF5_SZIP) SET(HAVE_H5Z_SZIP yes) - # If user has specified the `SZIP_LIBRARY`, use it; otherwise try to find... - IF(SZIP_FOUND) - SET(CMAKE_REQUIRED_LIBRARIES ${SZIP_LIBRARY} ${CMAKE_REQUIRED_LIBRARIES}) - MESSAGE(STATUS "HDF5 has szip.") - ELSE() - MESSAGE(FATAL_ERROR "HDF5 Requires SZIP, but cannot find libszip or libsz.") - ENDIF() - ELSE() - SET(HAVE_H5Z_SZIP no) ENDIF() #### @@ -1114,31 +1085,53 @@ string(TOLOWER "${filter}" downfilter) IF(${filter}_FOUND) INCLUDE_DIRECTORIES(${filter}_INCLUDE_DIRS}) SET(ENABLE_${upfilter} TRUE) - SET(STD_FILTERS "${STD_FILTERS},${downfilter}") + SET(HAVE_${upfilter} ON) + SET(STD_FILTERS "${STD_FILTERS} ${downfilter}") + MESSAGE(">>> Standard Filter: ${downfilter}") ELSE() SET(ENABLE_${upfilter} FALSE) + SET(HAVE_${upfilter} OFF) ENDIF() endmacro(set_std_filter) # Locate some compressors +FIND_PACKAGE(Szip) FIND_PACKAGE(Bz2) FIND_PACKAGE(Blosc) FIND_PACKAGE(Zstd) -IF(Zstd_FOUND) - SET(HAVE_ZSTD yes) -ELSE() - SET(HAVE_ZSTD no) -ENDIF() - # Accumulate standard filters -set(STD_FILTERS "deflate") # Always have deflate */ -set_std_filter(SZIP) +set(STD_FILTERS "deflate") # Always have deflate*/ +set_std_filter(Szip) + SET(HAVE_SZ ${Szip_FOUND}) set_std_filter(Blosc) set_std_filter(Zstd) -set_std_filter(Bz2) -IF(NOT Bz2_FOUND) -set(STD_FILTERS "${STD_FILTERS},bzip2") # Always have bzip2 */ +IF(Bz2_FOUND) + set_std_filter(Bz2) +ELSE() + # The reason we use a local version is to support a more comples test case + MESSAGE(WARNING "libbz2 not found using built-in version") + SET(HAVE_LOCAL_BZ2 ON) + SET(HAVE_BZ2 ON) + set(STD_FILTERS "${STD_FILTERS} bz2") +ENDIF() + + +# If user wants, then install selected plugins +SET(PLUGIN_INSTALL_DIR "" CACHE STRING "Whether and where we should install plugins") +SET(ENABLE_PLUGIN_INSTALL OFF) +if(DEFINED PLUGIN_INSTALL_DIR OR DEFINED CACHE{PLUGIN_INSTALL_DIR}) + IF(PLUGIN_INSTALL_DIR STREQUAL "") + MESSAGE(WARNING "No plugin directory value specified; option ignored.") + UNSET(PLUGIN_INSTALL_DIR) + UNSET(PLUGIN_INSTALL_DIR CACHE) + SET(PLUGIN_INSTALL_DIR_SETTING "N.A.") + ELSE() + SET(PLUGIN_INSTALL_DIR_SETTING "${PLUGIN_INSTALL_DIR}") + SET(ENABLE_PLUGIN_INSTALL ON) + ENDIF() +ELSE() + SET(PLUGIN_INSTALL_DIR_SETTING "N.A.") ENDIF() # See if we have libzip @@ -2528,6 +2521,10 @@ IF(ENABLE_NCZARR) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nc_test4/findplugin.in ${CMAKE_CURRENT_BINARY_DIR}/nczarr_test/findplugin.sh @ONLY NEWLINE_STYLE LF) ENDIF() +IF(ENABLE_PLUGINS) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nc_test4/findplugin.in ${CMAKE_CURRENT_BINARY_DIR}/plugins/findplugin.sh @ONLY NEWLINE_STYLE LF) +ENDIF() + IF(ENABLE_EXAMPLES) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nc_test4/findplugin.in ${CMAKE_CURRENT_BINARY_DIR}/examples/C/findplugin.sh @ONLY NEWLINE_STYLE LF) ENDIF() diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 9d0d2398d5..d0ce10764d 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -7,6 +7,7 @@ This file contains a high-level description of this package's evolution. Release ## 4.8.2 - TBD +* [Enhancement] Support installation of pre-built standard filters into user-specified location. See [Github #2318](https://github.com/Unidata/netcdf-c/pull/2318). * [Enhancement] Improve filter support. More specifically (1) add nc_inq_filter_avail to check if a filter is available, (2) add the notion of standard filters, (3) cleanup szip support to fix interaction with NCZarr. See [Github #2245](https://github.com/Unidata/netcdf-c/pull/2245). * [Enhancement] Switch to tinyxml2 as the default xml parser implementation. See [Github #2170](https://github.com/Unidata/netcdf-c/pull/2170). * [Bug Fix] Require that the type of the variable in nc_def_var_filter is not variable length. See [Github #/2231](https://github.com/Unidata/netcdf-c/pull/2231). diff --git a/cmake/modules/FindBzip2.cmake b/cmake/modules/FindBzip2.cmake new file mode 100644 index 0000000000..9b6c9471d0 --- /dev/null +++ b/cmake/modules/FindBzip2.cmake @@ -0,0 +1,64 @@ +# Searches for an installation of the bzip2 library. On success, it sets the following variables: +# +# Bzip2_FOUND Set to true to indicate the bzip2 library was found +# Bzip2_INCLUDE_DIRS The directory containing the header file bzip2/bzip2.h +# Bzip2_LIBRARIES The libraries needed to use the bzip2 library +# +# To specify an additional directory to search, set Bzip2_ROOT. +# +# Author: Siddhartha Chaudhuri, 2009 +# + +# Look for the header, first in the user-specified location and then in the system locations +SET(Bzip2_INCLUDE_DOC "The directory containing the header file bzip2.h") +FIND_PATH(Bzip2_INCLUDE_DIRS NAMES bzip2.h bzip2/bzip2.h PATHS ${Bzip2_ROOT} ${Bzip2_ROOT}/include DOC ${Bzip2_INCLUDE_DOC} NO_DEFAULT_PATH) +IF(NOT Bzip2_INCLUDE_DIRS) # now look in system locations + FIND_PATH(Bzip2_INCLUDE_DIRS NAMES bzlib.h DOC ${Bzip2_INCLUDE_DOC}) +ENDIF(NOT Bzip2_INCLUDE_DIRS) + +SET(Bzip2_FOUND FALSE) + +IF(Bzip2_INCLUDE_DIRS) + SET(Bzip2_LIBRARY_DIRS ${Bzip2_INCLUDE_DIRS}) + + IF("${Bzip2_LIBRARY_DIRS}" MATCHES "/include$") + # Strip off the trailing "/include" in the path. + GET_FILENAME_COMPONENT(Bzip2_LIBRARY_DIRS ${Bzip2_LIBRARY_DIRS} PATH) + ENDIF("${Bzip2_LIBRARY_DIRS}" MATCHES "/include$") + + IF(EXISTS "${Bzip2_LIBRARY_DIRS}/lib") + SET(Bzip2_LIBRARY_DIRS ${Bzip2_LIBRARY_DIRS}/lib) + ENDIF(EXISTS "${Bzip2_LIBRARY_DIRS}/lib") + + # Find Bzip2 libraries + FIND_LIBRARY(Bzip2_DEBUG_LIBRARY NAMES bzip2d bzip2_d libbzip2d libbzip2_d libbzip2 + PATH_SUFFIXES Debug ${CMAKE_LIBRARY_ARCHITECTURE} ${CMAKE_LIBRARY_ARCHITECTURE}/Debug + PATHS ${Bzip2_LIBRARY_DIRS} NO_DEFAULT_PATH) + FIND_LIBRARY(Bzip2_RELEASE_LIBRARY NAMES bzip2 libbzip2 + PATH_SUFFIXES Release ${CMAKE_LIBRARY_ARCHITECTURE} ${CMAKE_LIBRARY_ARCHITECTURE}/Release + PATHS ${Bzip2_LIBRARY_DIRS} NO_DEFAULT_PATH) + + SET(Bzip2_LIBRARIES ) + IF(Bzip2_DEBUG_LIBRARY AND Bzip2_RELEASE_LIBRARY) + SET(Bzip2_LIBRARIES debug ${Bzip2_DEBUG_LIBRARY} optimized ${Bzip2_RELEASE_LIBRARY}) + ELSEIF(Bzip2_DEBUG_LIBRARY) + SET(Bzip2_LIBRARIES ${Bzip2_DEBUG_LIBRARY}) + ELSEIF(Bzip2_RELEASE_LIBRARY) + SET(Bzip2_LIBRARIES ${Bzip2_RELEASE_LIBRARY}) + ENDIF(Bzip2_DEBUG_LIBRARY AND Bzip2_RELEASE_LIBRARY) + + IF(Bzip2_LIBRARIES) + SET(Bzip2_FOUND TRUE) + ENDIF(Bzip2_LIBRARIES) +ENDIF(Bzip2_INCLUDE_DIRS) + +IF(Bzip2_FOUND) +# IF(NOT Bzip2_FIND_QUIETLY) + MESSAGE(STATUS "Found Bzip2: headers at ${Bzip2_INCLUDE_DIRS}, libraries at ${Bzip2_LIBRARY_DIRS}") + MESSAGE(STATUS " library is ${Bzip2_LIBRARIES}") +# ENDIF(NOT Bzip2_FIND_QUIETLY) +ELSE(Bzip2_FOUND) + IF(Bzip2_FIND_REQUIRED) + MESSAGE(FATAL_ERROR "Bzip2 library not found") + ENDIF(Bzip2_FIND_REQUIRED) +ENDIF(Bzip2_FOUND) diff --git a/cmake/modules/FindSZIP.cmake b/cmake/modules/FindSZIP.cmake deleted file mode 100644 index dc972e5bd0..0000000000 --- a/cmake/modules/FindSZIP.cmake +++ /dev/null @@ -1,179 +0,0 @@ - -# - Find SZIP library -# - Derived from the FindTiff.cmake that is included with cmake -# Find the native SZIP includes and library -# This module defines -# SZIP_INCLUDE_DIRS, where to find tiff.h, etc. -# SZIP_LIBRARIES, libraries to link against to use SZIP. -# SZIP_FOUND, If false, do not try to use SZIP. -# also defined, but not for general use are -# SZIP_LIBRARY, where to find the SZIP library. -# SZIP_LIBRARY_DEBUG - Debug version of SZIP library -# SZIP_LIBRARY_RELEASE - Release Version of SZIP library - -# MESSAGE (STATUS "Finding SZIP library and headers..." ) - -############################################ -# -# Check the existence of the libraries. -# -############################################ -# This macro was taken directly from the FindQt4.cmake file that is included -# with the CMake distribution. This is NOT my work. All work was done by the -# original authors of the FindQt4.cmake file. Only minor modifications were -# made to remove references to Qt and make this file more generally applicable -######################################################################### - -MACRO (SZIP_ADJUST_LIB_VARS basename) - IF (${basename}_INCLUDE_DIR) - - # if only the release version was found, set the debug variable also to the release version - IF (${basename}_LIBRARY_RELEASE AND NOT ${basename}_LIBRARY_DEBUG) - SET (${basename}_LIBRARY_DEBUG ${${basename}_LIBRARY_RELEASE}) - SET (${basename}_LIBRARY ${${basename}_LIBRARY_RELEASE}) - SET (${basename}_LIBRARIES ${${basename}_LIBRARY_RELEASE}) - ENDIF (${basename}_LIBRARY_RELEASE AND NOT ${basename}_LIBRARY_DEBUG) - - # if only the debug version was found, set the release variable also to the debug version - IF (${basename}_LIBRARY_DEBUG AND NOT ${basename}_LIBRARY_RELEASE) - SET (${basename}_LIBRARY_RELEASE ${${basename}_LIBRARY_DEBUG}) - SET (${basename}_LIBRARY ${${basename}_LIBRARY_DEBUG}) - SET (${basename}_LIBRARIES ${${basename}_LIBRARY_DEBUG}) - ENDIF (${basename}_LIBRARY_DEBUG AND NOT ${basename}_LIBRARY_RELEASE) - IF (${basename}_LIBRARY_DEBUG AND ${basename}_LIBRARY_RELEASE) - # if the generator supports configuration types then set - # optimized and debug libraries, or if the CMAKE_BUILD_TYPE has a value - IF (CMAKE_CONFIGURATION_TYPES OR CMAKE_BUILD_TYPE) - SET (${basename}_LIBRARY optimized ${${basename}_LIBRARY_RELEASE} debug ${${basename}_LIBRARY_DEBUG}) - ELSE(CMAKE_CONFIGURATION_TYPES OR CMAKE_BUILD_TYPE) - # if there are no configuration types and CMAKE_BUILD_TYPE has no value - # then just use the release libraries - SET (${basename}_LIBRARY ${${basename}_LIBRARY_RELEASE} ) - ENDIF (CMAKE_CONFIGURATION_TYPES OR CMAKE_BUILD_TYPE) - SET (${basename}_LIBRARIES optimized ${${basename}_LIBRARY_RELEASE} debug ${${basename}_LIBRARY_DEBUG}) - ENDIF (${basename}_LIBRARY_DEBUG AND ${basename}_LIBRARY_RELEASE) - - SET (${basename}_LIBRARY ${${basename}_LIBRARY} CACHE FILEPATH "The ${basename} library") - - IF (${basename}_LIBRARY) - SET (${basename}_FOUND 1) - ENDIF (${basename}_LIBRARY) - - ENDIF (${basename}_INCLUDE_DIR ) - - # Make variables changeble to the advanced user - MARK_AS_ADVANCED (${basename}_LIBRARY ${basename}_LIBRARY_RELEASE ${basename}_LIBRARY_DEBUG ${basename}_INCLUDE_DIR ) -ENDMACRO (SZIP_ADJUST_LIB_VARS) - - -# Look for the header file. -SET (SZIP_INCLUDE_SEARCH_DIRS - $ENV{SZIP_INSTALL}/include - $ENV{SZIP_INSTALL}/include/szip - /usr/include - /usr/include/szip -) - -SET (SZIP_LIB_SEARCH_DIRS - $ENV{SZIP_INSTALL}/lib - /usr/lib -) - -SET (SZIP_BIN_SEARCH_DIRS - $ENV{SZIP_INSTALL}/bin - /usr/bin -) - -FIND_PATH (SZIP_INCLUDE_DIR - NAMES szlib.h - PATHS ${SZIP_INCLUDE_SEARCH_DIRS} - NO_DEFAULT_PATH -) - -IF (WIN32 AND NOT MINGW) - SET (SZIP_SEARCH_DEBUG_NAMES "sz_d;libsz_d") - SET (SZIP_SEARCH_RELEASE_NAMES "sz;libsz;szip") -ELSE (WIN32 AND NOT MINGW) - SET (SZIP_SEARCH_DEBUG_NAMES "sz_d") - SET (SZIP_SEARCH_RELEASE_NAMES "sz") -ENDIF (WIN32 AND NOT MINGW) - -# Look for the library. -FIND_LIBRARY (SZIP_LIBRARY_DEBUG - NAMES ${SZIP_SEARCH_DEBUG_NAMES} - PATHS ${SZIP_LIB_SEARCH_DIRS} - NO_DEFAULT_PATH -) - -FIND_LIBRARY (SZIP_LIBRARY_RELEASE - NAMES ${SZIP_SEARCH_RELEASE_NAMES} - PATHS ${SZIP_LIB_SEARCH_DIRS} - NO_DEFAULT_PATH -) - -SZIP_ADJUST_LIB_VARS (SZIP) - -IF (SZIP_INCLUDE_DIR AND SZIP_LIBRARY) - SET (SZIP_FOUND 1) - SET (SZIP_LIBRARIES ${SZIP_LIBRARY}) - SET (SZIP_INCLUDE_DIRS ${SZIP_INCLUDE_DIR}) - IF (SZIP_LIBRARY_DEBUG) - GET_FILENAME_COMPONENT (SZIP_LIBRARY_PATH ${SZIP_LIBRARY_DEBUG} PATH) - SET (SZIP_LIB_DIR ${SZIP_LIBRARY_PATH}) - ELSEIF (SZIP_LIBRARY_RELEASE) - GET_FILENAME_COMPONENT (SZIP_LIBRARY_PATH ${SZIP_LIBRARY_RELEASE} PATH) - SET (SZIP_LIB_DIR ${SZIP_LIBRARY_PATH}) - ENDIF (SZIP_LIBRARY_DEBUG) - -ELSE (SZIP_INCLUDE_DIR AND SZIP_LIBRARY) - SET (SZIP_FOUND 0) - SET (SZIP_LIBRARIES) - SET (SZIP_INCLUDE_DIRS) -ENDIF (SZIP_INCLUDE_DIR AND SZIP_LIBRARY) - -# Report the results. -IF (NOT SZIP_FOUND) - SET (SZIP_DIR_MESSAGE - "SZip was not found. Make sure SZIP_LIBRARY and SZIP_INCLUDE_DIR are set or set the SZIP_INSTALL environment variable." - ) - IF (NOT SZIP_FIND_QUIETLY) - MESSAGE (STATUS "${SZIP_DIR_MESSAGE}") - ELSE (NOT SZIP_FIND_QUIETLY) - IF (SZIP_FIND_REQUIRED) - MESSAGE (FATAL_ERROR "SZip was NOT found and is Required by this project") - ENDIF (SZIP_FIND_REQUIRED) - ENDIF (NOT SZIP_FIND_QUIETLY) -ENDIF (NOT SZIP_FOUND) - -IF (SZIP_FOUND) - INCLUDE (CheckSymbolExists) - ############################################# - # Find out if SZIP was build using dll's - ############################################# - # Save required variable - SET (CMAKE_REQUIRED_INCLUDES_SAVE ${CMAKE_REQUIRED_INCLUDES}) - SET (CMAKE_REQUIRED_FLAGS_SAVE ${CMAKE_REQUIRED_FLAGS}) - # Add SZIP_INCLUDE_DIR to CMAKE_REQUIRED_INCLUDES - SET (CMAKE_REQUIRED_INCLUDES "${CMAKE_REQUIRED_INCLUDES};${SZIP_INCLUDE_DIRS}") - - CHECK_SYMBOL_EXISTS (SZIP_BUILT_AS_DYNAMIC_LIB "SZconfig.h" HAVE_SZIP_DLL) - - IF (HAVE_SZIP_DLL STREQUAL "TRUE") - SET (HAVE_SZIP_DLL "1") - ENDIF (HAVE_SZIP_DLL STREQUAL "TRUE") - - # Restore CMAKE_REQUIRED_INCLUDES and CMAKE_REQUIRED_FLAGS variables - SET (CMAKE_REQUIRED_INCLUDES ${CMAKE_REQUIRED_INCLUDES_SAVE}) - SET (CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS_SAVE}) - # - ############################################# -ENDIF (SZIP_FOUND) - -IF (FIND_SZIP_DEBUG) - MESSAGE (STATUS "SZIP_INCLUDE_DIR: ${SZIP_INCLUDE_DIR}") - MESSAGE (STATUS "SZIP_INCLUDE_DIRS: ${SZIP_INCLUDE_DIRS}") - MESSAGE (STATUS "SZIP_LIBRARY_DEBUG: ${SZIP_LIBRARY_DEBUG}") - MESSAGE (STATUS "SZIP_LIBRARY_RELEASE: ${SZIP_LIBRARY_RELEASE}") - MESSAGE (STATUS "HAVE_SZIP_DLL: ${HAVE_SZIP_DLL}") - MESSAGE (STATUS "CMAKE_BUILD_TYPE: ${CMAKE_BUILD_TYPE}") -ENDIF (FIND_SZIP_DEBUG) diff --git a/cmake/modules/FindSzip.cmake b/cmake/modules/FindSzip.cmake new file mode 100644 index 0000000000..4deca3347a --- /dev/null +++ b/cmake/modules/FindSzip.cmake @@ -0,0 +1,64 @@ +# Searches for an installation of the szip library. On success, it sets the following variables: +# +# Szip_FOUND Set to true to indicate the szip library was found +# Szip_INCLUDE_DIRS The directory containing the header file szip/szip.h +# Szip_LIBRARIES The libraries needed to use the szip library +# +# To specify an additional directory to search, set Szip_ROOT. +# +# Author: Siddhartha Chaudhuri, 2009 +# + +# Look for the header, first in the user-specified location and then in the system locations +SET(Szip_INCLUDE_DOC "The directory containing the header file szip.h") +FIND_PATH(Szip_INCLUDE_DIRS NAMES szlib.h szip.h szip/szip.h PATHS ${Szip_ROOT} ${Szip_ROOT}/include DOC ${Szip_INCLUDE_DOC} NO_DEFAULT_PATH) +IF(NOT Szip_INCLUDE_DIRS) # now look in system locations + FIND_PATH(Szip_INCLUDE_DIRS NAMES szlib.h szip.h szip/szip.h DOC ${Szip_INCLUDE_DOC}) +ENDIF(NOT Szip_INCLUDE_DIRS) + +SET(Szip_FOUND FALSE) + +IF(Szip_INCLUDE_DIRS) + SET(Szip_LIBRARY_DIRS ${Szip_INCLUDE_DIRS}) + + IF("${Szip_LIBRARY_DIRS}" MATCHES "/include$") + # Strip off the trailing "/include" in the path. + GET_FILENAME_COMPONENT(Szip_LIBRARY_DIRS ${Szip_LIBRARY_DIRS} PATH) + ENDIF("${Szip_LIBRARY_DIRS}" MATCHES "/include$") + + IF(EXISTS "${Szip_LIBRARY_DIRS}/lib") + SET(Szip_LIBRARY_DIRS ${Szip_LIBRARY_DIRS}/lib) + ENDIF(EXISTS "${Szip_LIBRARY_DIRS}/lib") + + # Find Szip libraries + FIND_LIBRARY(Szip_DEBUG_LIBRARY NAMES szipd szip_d libszipd libszip_d szip libszip sz2 libsz2 + PATH_SUFFIXES Debug ${CMAKE_LIBRARY_ARCHITECTURE} ${CMAKE_LIBRARY_ARCHITECTURE}/Debug + PATHS ${Szip_LIBRARY_DIRS} NO_DEFAULT_PATH) + FIND_LIBRARY(Szip_RELEASE_LIBRARY NAMES szip libszip sz libsz sz2 libsz2 + PATH_SUFFIXES Release ${CMAKE_LIBRARY_ARCHITECTURE} ${CMAKE_LIBRARY_ARCHITECTURE}/Release + PATHS ${Szip_LIBRARY_DIRS} NO_DEFAULT_PATH) + + SET(Szip_LIBRARIES ) + IF(Szip_DEBUG_LIBRARY AND Szip_RELEASE_LIBRARY) + SET(Szip_LIBRARIES debug ${Szip_DEBUG_LIBRARY} optimized ${Szip_RELEASE_LIBRARY}) + ELSEIF(Szip_DEBUG_LIBRARY) + SET(Szip_LIBRARIES ${Szip_DEBUG_LIBRARY}) + ELSEIF(Szip_RELEASE_LIBRARY) + SET(Szip_LIBRARIES ${Szip_RELEASE_LIBRARY}) + ENDIF(Szip_DEBUG_LIBRARY AND Szip_RELEASE_LIBRARY) + + IF(Szip_LIBRARIES) + SET(Szip_FOUND TRUE) + ENDIF(Szip_LIBRARIES) +ENDIF(Szip_INCLUDE_DIRS) + +IF(Szip_FOUND) +# IF(NOT Szip_FIND_QUIETLY) + MESSAGE(STATUS "Found Szip: headers at ${Szip_INCLUDE_DIRS}, libraries at ${Szip_LIBRARY_DIRS}") + MESSAGE(STATUS " library is ${Szip_LIBRARIES}") +# ENDIF(NOT Szip_FIND_QUIETLY) +ELSE(Szip_FOUND) + IF(Szip_FIND_REQUIRED) + MESSAGE(FATAL_ERROR "Szip library not found") + ENDIF(Szip_FIND_REQUIRED) +ENDIF(Szip_FOUND) diff --git a/config.h.cmake.in b/config.h.cmake.in index b77c1bab24..97004f62c1 100644 --- a/config.h.cmake.in +++ b/config.h.cmake.in @@ -173,7 +173,7 @@ are set when opening a binary file on Windows. */ #cmakedefine HAVE_ATEXIT 1 /* Define to 1 if bzip2 library available. */ -#cmakedefine HAVE_BZIP2 1 +#cmakedefine HAVE_BZ2 1 /* Define to 1 if zstd library available. */ #cmakedefine HAVE_ZSTD 1 diff --git a/configure.ac b/configure.ac index eb1abf6601..74ae167344 100644 --- a/configure.ac +++ b/configure.ac @@ -667,24 +667,21 @@ fi AC_MSG_CHECKING([whether libzstd library is available]) AC_MSG_RESULT([${have_zstd}]) -# See if we have libbzip2 or libbz2 -AC_CHECK_LIB([bz2],[BZ2_bzCompress],[have_bzip2=yes],[have_bzip2=no]) -if test "x$have_bzip2" = "xyes" ; then +# See if we have libbz2 +AC_CHECK_LIB([bz2],[BZ2_bzCompress],[have_bz2=yes],[have_bz2=no]) +if test "x$have_bz2" = "xyes" ; then AC_SEARCH_LIBS([BZ2_bzCompress],[bz2 bz2.dll cygbz2.dll], [], []) - AC_DEFINE([HAVE_BZIP2], [1], [if true, bz2 library is installed]) + AC_DEFINE([HAVE_BZ2], [1], [if true, bz2 library is installed]) fi AC_MSG_CHECKING([whether libbz2 library is available]) -AC_MSG_RESULT([${have_bzip2}]) +AC_MSG_RESULT([${have_bz2}]) -if test "x$have_bzip2" = "xno" ; then -AC_CHECK_LIB([bzip2],[BZ2_bzCompress],[have_bzip2=yes],[have_bzip2=no]) -if test "x$have_bzip2" = "xyes" ; then - AC_SEARCH_LIBS([BZ2_bzCompress],[bzip2 bzip2.dll cygbzip2.dll], [], []) - AC_DEFINE([HAVE_BZIP2], [1], [if true, bzip2 library is installed]) -fi -AC_MSG_CHECKING([whether libbzip2 library is available]) -AC_MSG_RESULT([${have_bzip2}]) +if test "x$have_bz2" = "xno" ; then + have_local_bz2=yes +else + have_local_bz2=no fi +AM_CONDITIONAL(HAVE_LOCAL_BZ2, [test "x$have_local_bz2" = xyes]) # Note that szip management is tricky. # This is because we have three things to consider: @@ -1753,7 +1750,7 @@ AM_CONDITIONAL(HAS_MULTIFILTERS, [test "x$has_multifilters" = xyes]) AM_CONDITIONAL(HAVE_SZ, [test "x$have_sz" = xyes]) AM_CONDITIONAL(HAVE_H5Z_SZIP, [test "x$enable_hdf5_szip" = xyes]) AM_CONDITIONAL(HAVE_BLOSC, [test "x$have_blosc" = xyes]) -AM_CONDITIONAL(HAVE_BZIP2, [test "x$have_bzip2" = xyes]) +AM_CONDITIONAL(HAVE_BZ2, [test "x$have_bz2" = xyes]) AM_CONDITIONAL(HAVE_ZSTD, [test "x$have_zstd" = xyes]) # If the machine doesn't have a long long, and we want netCDF-4, then @@ -1864,26 +1861,45 @@ AC_SUBST(HAS_SZLIB_WRITE, [$have_sz]) AC_SUBST(HAS_ZSTD,[$have_zstd]) # Always available -std_filters="deflate,bzip2" +std_filters="deflate bz2" if test "x$enable_szlib" = xyes ; then -std_filters="${std_filters},szip" +std_filters="${std_filters} szip" fi # We need to include szip iff HDF5 && HDF5_HAS_SZIP || !HDF5 && NCZARR && libsz if test "x$enable_hdf5" = xyes && test "x$enable_hdf5_szip" = xyes ; then - std_filters="${std_filters},szip" + std_filters="${std_filters} szip" fi if test "x$enable_hdf5" = xno && test "x$have_sz" = xyes ; then -std_filters="${std_filters},szip" +std_filters="${std_filters} szip" fi if test "x$have_blosc" = xyes ; then -std_filters="${std_filters},blosc" +std_filters="${std_filters} blosc" fi if test "x$have_zstd" = xyes ; then -std_filters="${std_filters},zst" +std_filters="${std_filters} zstd" fi AC_SUBST(STD_FILTERS,[$std_filters]) +# If user wants, then install selected standard filters +AC_MSG_CHECKING([whether and where we should install plugins]) +AC_ARG_WITH([plugin-dir], [AS_HELP_STRING([--with-plugin-dir=], + [Install selected standard filters in specified directory])]) +AC_MSG_RESULT([$with_plugin_dir]) +if test "x$with_plugin_dir" = x ; then + AC_MSG_WARN([No plugin directory value specified; option ignored.]) + with_plugin_dir= + with_plugin_dir_setting="N.A." + enable_plugin_dir=no +else + with_plugin_dir_setting="$with_plugin_dir" + enable_plugin_dir=yes +fi +AM_CONDITIONAL([ENABLE_PLUGIN_DIR], [test "x$enable_plugin_dir" = xyes]) +AC_SUBST([PLUGIN_INSTALL_DIR], [$with_plugin_dir]) +# Better value for libnetcdf.settings +AC_SUBST([PLUGIN_INSTALL_DIR_SETTING], [$with_plugin_dir_setting]) + # Access netcdf specific version of config.h AH_BOTTOM([#include "ncconfigure.h"]) @@ -1973,6 +1989,7 @@ AC_MSG_NOTICE([generating header files and makefiles]) AC_CONFIG_FILES(test_common.sh:test_common.in) AC_CONFIG_FILES(nc_test4/findplugin.sh:nc_test4/findplugin.in) AC_CONFIG_FILES(nczarr_test/findplugin.sh:nc_test4/findplugin.in) +AC_CONFIG_FILES(plugins/findplugin.sh:nc_test4/findplugin.in) AC_CONFIG_FILES(examples/C/findplugin.sh:nc_test4/findplugin.in) AC_CONFIG_FILES(ncdap_test/findtestserver.c:ncdap_test/findtestserver.c.in) AC_CONFIG_FILES(dap4_test/findtestserver4.c:ncdap_test/findtestserver.c.in) diff --git a/docs/filters.md b/docs/filters.md index 303a2644d4..00dbe59304 100644 --- a/docs/filters.md +++ b/docs/filters.md @@ -932,28 +932,40 @@ This is for internal use only. # Appendix F. Pre-built Filters -As part of the overall build process, a number of filters are built as shared libraries in the "plugins" directory. -They may be in that directory or the "plugins/.libs" subdirectory. -It may be possible for users to utilize some of those libraries to provide filter support for general use. -One simple way to reuse these libraries is to make the environment variable "HDF5_PLUGIN_PATH" refer to the plugin directory or the subdirectory, whichever contains the shared libraries. -This may not be possible if the user already has other filter shared libraries in use. +As part of the overall build process, a number of filters are built as shared libraries in the "plugins" directory +— in that directory or the "plugins/.libs" subdirectory. -The second way is to copy the necessary shared libraries from the plugins directory to the user's HDF5_PLUGIN_PATH directory. -If the user is using HDF5, then the following filters are probably usable: +An option exists to allow some of those filters to be installed into a user-specified directory. The relevant options are as follows: +```` +./configure: --with-plugin-dir= +cmake: -DPLUGIN_INSTALL_DIR= +```` +If the value of the environment variable "HDF5_PLUGIN_PATH" is a single directory, then +a good value for the install directory is "$HDF5_PLUGIN_PATH", so for example: +```` +./configure ... --with-plugin-dir="$HDF5_PLUGIN_DIR" +```` -* ''libh5bzip2.so'' -- an HDF5 filter for bzip2 compression -* ''libh5blosc.so'' -- an HDF5 filter for blosc compression +If this option is specified, then as part of the "install" build action, +a specified set of filter shared libraries will be copied into the specified directory. +Any existing library of the same name will be overwritten. If the specified directory +itself does not exist, then it will be created. -If the user is using NCZarr filters, then you can install the following additional shared libraries: +Currently, if the following filters are available, they will be installed; +* ''libh5bzip2.so'' -- an HDF5 filter wrapper for bzip2 compression +* ''libh5blosc.so'' -- an HDF5 filter wrapper for blosc compression +* ''libh5zstd.so'' -- an HDF5 filter wrapper for zstandardcompression +If the user is using NCZarr filters, then the following additional filters will be installed. * libh5shuffle.so -- shuffle filter * libh5fletcher32.so -- fletcher32 checksum * libh5deflate.so -- deflate compression +* libh5szip.so -- szip compression * libnczdefaults.so -- provide NCZarr support for shuffle, fletcher32, and deflate. +* libnczszip.so -- provide NCZarr support for szip. -The shuffle, fletcher32, and deflate filters in this case will be ignored by HDF5 and only used by the NCZarr code. -But in order to use them, it needs additional Codec capabilities provided by the libnczdefauts.so shared library. -Note also that if you disable HDF5 support, but leave NCZarr support enabled, then all of the above filters +The shuffle, fletcher32, deflate, and szip filters in this case will be ignored by HDF5 and only used by the NCZarr code. +Note that if you disable HDF5 support, but leave NCZarr support enabled, then all of the above filters should continue to work. # Point of Contact {#filters_poc} diff --git a/examples/C/Makefile.am b/examples/C/Makefile.am index e9f087adac..34dbfbec41 100644 --- a/examples/C/Makefile.am +++ b/examples/C/Makefile.am @@ -7,7 +7,9 @@ # Un comment to use a more verbose test driver #SH_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose +#sh_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose #LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose +#TESTS_ENVIRONMENT = export SETX=1; # Put together AM_CPPFLAGS and AM_LDFLAGS. include $(top_srcdir)/lib_flags.am diff --git a/examples/C/run_filter.sh b/examples/C/run_filter.sh index 37dc8f8068..90e0935fe9 100755 --- a/examples/C/run_filter.sh +++ b/examples/C/run_filter.sh @@ -10,26 +10,25 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../../test_common.sh echo "*** Running filter example for netCDF-4." + set -e -if test -f ${builddir}/findplugin.sh ; then echo "*** running test_filter example..." . ${builddir}/findplugin.sh # Locate the plugin path and the library names; argument order is critical # Find bzip2 and capture findplugin h5bzip2 -BZIP2PATH="${HDF5_PLUGIN_PATH}/${HDF5_PLUGIN_LIB}" +BZIP2PATH="${HDF5_PLUGIN_DIR}/${HDF5_PLUGIN_LIB}" + # Verify if ! test -f ${BZIP2PATH} ; then echo "Unable to locate ${BZIP2PATH}"; exit 1; fi -export HDF5_PLUGIN_PATH -echo "*** running filter_example..." rm -f ./bzip2.nc +export HDF5_PLUGIN_PATH="${HDF5_PLUGIN_DIR}" +echo "*** running filter_example..." ${execdir}/filter_example -#rm -f ./bzip2.nc - -fi # Filter enabled +rm -f ./bzip2.nc echo "*** Filter example successful!" exit 0 diff --git a/include/netcdf_filter_build.h b/include/netcdf_filter_build.h index 6c6f8e9481..11c80f8095 100644 --- a/include/netcdf_filter_build.h +++ b/include/netcdf_filter_build.h @@ -34,17 +34,46 @@ /* Defined flags for filter invocation (not stored); powers of two */ #define NCZ_FILTER_DECODE 0x00000001 -/* External Discovery Function */ +/* External Discovery Functions */ /* Obtain a pointer to an instance of NCZ_codec_class_t. +NCZ_get_codec_info(void) -- returns pointer to instance of NCZ_codec_class_t. + Instance an be recast based on version+sort to the plugin type specific info. +So the void* return value is typically actually of type NCZ_codec_class_t*. + +Signature: typedef const void* (*NCZ_get_codec_info_proto)(void); + +The current object returned by NCZ_get_codec_info is a + pointer to an instance of NCZ_codec_t. + +The key to this struct are the several function pointers that do +initialize/finalize and conversion between codec JSON and HDF5 +parameters. The function pointers defined in NCZ_codec_t +manipulate HDF5 parameters and NumCodec JSON. + +Obtain a pointer to an instance of NCZ_codec_class_t. + NCZ_get_codec_info(void) -- returns pointer to instance of NCZ_codec_class_t. Instance an be recast based on version+sort to the plugin type specific info. So the void* return value is typically actually of type NCZ_codec_class_t*. */ typedef const void* (*NCZ_get_codec_info_proto)(void); +/* +Obtain a pointer to a NULL terminated vector of NCZ_codec_class_t*. + +NCZ_codec_info_defaults(void) -- returns pointer to a vector of pointers to instances of NCZ_codec_class_t. The vector is NULL terminated. +So the void* return value is typically actually of type NCZ_codec_class_t**. + +Signature: typedef const void* (*NCZ_codec_info_defaults_proto)(void); + +This entry point is used to return the codec information for +multiple filters that otherwise do not have codec information defined. +*/ +typedef const void* (*NCZ_codec_info_defaults_proto)(void); + /* The current object returned by NCZ_get_plugin_info is a pointer to an instance of NCZ_codec_t. @@ -86,7 +115,7 @@ int (*NCZ_hdf5_to_codec)(size_t nparams, const unsigned* params, char** codecp); * Convert a set of visible parameters to a set of working parameters using extra environmental information. Also allows for changes to the visible parameters. Invoked before filter is actually used. -int (*NCZ_build_parameters)(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp); +int (*NCZ_modify_parameters)(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp); @param ncid -- (in) ncid of the variable's group @param varid -- (in) varid of the variable @@ -96,8 +125,15 @@ int (*NCZ_build_parameters)(int ncid, int varid, size_t* vnparamsp, unsigned** v @params wparamsp -- (out) vector of working parameters @return -- a netcdf-c error code. -* Convert a set of working parameters to a set of visible parameters using extra environmental information, if needed. -Invoked before filter metadata is written. +* Convert an HDF5 vector of visible parameters to a JSON representation. + +int (*NCZ_hdf5_to_codec)(size_t nparams, const unsigned* params, char** codecp); + +@param nparams -- (in) the length of the HDF5 unsigned vector +@param params -- (in) pointer to the HDF5 unsigned vector. +@param codecp -- (out) store the string representation of the codec; caller must free. +@return -- a netcdf-c error code. + */ /* diff --git a/libdispatch/dutil.c b/libdispatch/dutil.c index b8f48e17cf..d4b37a3448 100644 --- a/libdispatch/dutil.c +++ b/libdispatch/dutil.c @@ -211,11 +211,11 @@ NC_mktmp(const char* base) int fd = -1; char* tmp = NULL; size_t len; +#ifndef HAVE_MKSTEMP + int tries; #define MAXTRIES 4 -#ifdef HAVE_MKSTEMP - mode_t mask; #else - int tries; + mode_t mask; #endif len = strlen(base)+6+1; diff --git a/libnczarr/zfilter.c b/libnczarr/zfilter.c index 58bb0984d4..963a977183 100644 --- a/libnczarr/zfilter.c +++ b/libnczarr/zfilter.c @@ -52,9 +52,11 @@ #include "netcdf_filter_build.h" #include "netcdf_aux.h" -#undef DEBUG -#undef DEBUGF -#undef DEBUGL +#if 0 +#define DEBUG +#define DEBUGF +#define DEBUGL +#endif /* If set, then triage potential shared libraries based on extension */ #define NAMEOPT @@ -177,7 +179,7 @@ NCJtrace(const NCjson* j) const char* printplugin(const NCZ_Plugin* plugin) { - char* plbuf = malloc(4096); + static char plbuf[4096]; char plbuf2[2000]; char plbuf1[2000]; @@ -194,8 +196,7 @@ printplugin(const NCZ_Plugin* plugin) static char* printparams(size_t nparams, const unsigned* params) { - char* ppbuf = malloc(4096); - + static char ppbuf[4096]; if(nparams == 0) snprintf(ppbuf,4096,"{0,%p}",params); else @@ -1378,20 +1379,27 @@ NCZ_load_plugin(const char* path, struct NCZ_Plugin** plugp) /* See what we have */ { - H5PL_get_plugin_type_proto gpt = (H5PL_get_plugin_type_proto)ncpgetsymbol(lib,"H5PLget_plugin_type"); - H5PL_get_plugin_info_proto gpi = (H5PL_get_plugin_info_proto)ncpgetsymbol(lib,"H5PLget_plugin_info"); - NCZ_get_codec_info_proto npi = (NCZ_get_codec_info_proto)ncpgetsymbol(lib,"NCZ_get_codec_info"); - NCZ_codec_info_defaults_proto cpd = (NCZ_codec_info_defaults_proto)ncpgetsymbol(lib,"NCZ_codec_info_defaults"); + const H5PL_get_plugin_type_proto gpt = (H5PL_get_plugin_type_proto)ncpgetsymbol(lib,"H5PLget_plugin_type"); + const H5PL_get_plugin_info_proto gpi = (H5PL_get_plugin_info_proto)ncpgetsymbol(lib,"H5PLget_plugin_info"); + const NCZ_get_codec_info_proto npi = (NCZ_get_codec_info_proto)ncpgetsymbol(lib,"NCZ_get_codec_info"); + const NCZ_codec_info_defaults_proto cpd = (NCZ_codec_info_defaults_proto)ncpgetsymbol(lib,"NCZ_codec_info_defaults"); if(gpt == NULL && gpi == NULL && npi == NULL && cpd == NULL) {stat = NC_ENOFILTER; goto done;} if(cpd != NULL) { /* Deal with defaults first */ - NCZ_codec_t** cp = NULL; + const NCZ_codec_t** cp = NULL; nclistpush(default_libs,lib); - for(cp=cpd();*cp;cp++) { + cp = (const NCZ_codec_t**)cpd(); +#ifdef DEBUGL + fprintf(stderr,"@@@ %s: default codec library found: %p\n",path,cp); +#endif + for(;*cp;cp++) { struct CodecAPI* c0; +#ifdef DEBUGL + fprintf(stderr,"@@@ %s: %s = %u\n",path,(*cp)->codecid,(*cp)->hdf5id); +#endif c0 = (struct CodecAPI*)calloc(1,sizeof(struct CodecAPI)); if(c0 == NULL) {stat = NC_ENOMEM; goto done1;} c0->codec = *cp; diff --git a/libnczarr/zfilter.h b/libnczarr/zfilter.h index 4aa7de80aa..38d8b0c3d8 100644 --- a/libnczarr/zfilter.h +++ b/libnczarr/zfilter.h @@ -25,16 +25,6 @@ #define plugin_dir_win "%s/hdf5/lib/plugin" #define win32_root_env "ALLUSERSPROFILE" -/* -Return a NULL terminated vector of pointers to instances of ''NCZ_codec_t''. -The value returned is actually of type ''NCZ_codec_t**'', -but is of type ''void*'' to allow for extensions. -The list of returned items are used to try to provide defaults -for any HDF5 filters that have no corresponding Codec. -This is for internal use only. -*/ -typedef void* (*NCZ_codec_info_defaults_proto)(void); - /* Opaque */ struct NCZ_Filter; diff --git a/libnetcdf.settings.in b/libnetcdf.settings.in index d9cc1af96f..6dea12d5c7 100644 --- a/libnetcdf.settings.in +++ b/libnetcdf.settings.in @@ -9,6 +9,7 @@ Configured On: @CONFIG_DATE@ Host System: @host_cpu@-@host_vendor@-@host_os@ Build Directory: @abs_top_builddir@ Install Prefix: @prefix@ +Plugin Install Prefix: @PLUGIN_INSTALL_DIR_SETTING@ # Compiling Options ----------------- @@ -49,4 +50,4 @@ Logging: @HAS_LOGGING@ SZIP Write Support: @HAS_SZLIB_WRITE@ Standard Filters: @STD_FILTERS@ ZSTD Support: @HAS_ZSTD@ -Benchmarks: @HAS_BENCHMARKS@ \ No newline at end of file +Benchmarks: @HAS_BENCHMARKS@ diff --git a/nc_test4/Makefile.am b/nc_test4/Makefile.am index 8ce7e97ded..9524339f3e 100644 --- a/nc_test4/Makefile.am +++ b/nc_test4/Makefile.am @@ -109,9 +109,15 @@ ref_filteredvv.cdl ref_multi.cdl \ ref_ncgenF.cdl ref_nccopyF.cdl \ ref_filter_repeat.txt ref_fillonly.cdl test_fillonly.sh \ ref_filter_order_create.txt ref_filter_order_read.txt \ -ref_any.cdl tst_specific_filters.sh \ +ref_any.cdl tst_specific_filters.sh \ tst_virtual_datasets.c +# The tst_filterinstall test can only be run after an install +# occurred with --with-plugin-dir enabled. So there is no point +#in running it via make check. It is kept here so it can be +# manually invoked if desired +EXTRA_DIST += tst_filterinstall.sh + CLEANFILES = tst_mpi_parallel.bin cdm_sea_soundings.nc bm_chunking.nc \ tst_floats_1D.cdl floats_1D_3.nc floats_1D.cdl tst_*.nc tmp_*.txt \ tst_floats2_*.cdl tst_ints2_*.cdl tst_shorts2_*.cdl tst_elena_*.cdl \ diff --git a/nc_test4/findplugin.in b/nc_test4/findplugin.in index fb30686307..a8f2a4e3b8 100644 --- a/nc_test4/findplugin.in +++ b/nc_test4/findplugin.in @@ -9,12 +9,10 @@ # $2 is 1 if we are running under cmake # $3 is 1 if we are running using Visual Studio, blank otherwise # $4 is the build type; only used if $3 is 1 -# Optional Input: -# HDF5_PLUGIN_PATH environment variable # Outputs: # return code is 0 is success, 1 if failed # Variable HDF5_PLUGIN_LIB is set to the library file name -# Variable HDF5_PLUGIN_PATH is setthe absolute path to the +# Variable HDF5_PLUGIN_DIR is set to the absolute path to the # directory containing the plugin library file # Local variables are prefixed with FP_ # @@ -27,7 +25,7 @@ findplugin() { FP_NAME="$1" FP_PLUGIN_LIB= -FP_PLUGIN_PATH= +FP_PLUGIN_DIR= # Figure out the plugin file name # Test for visual studio before cygwin since both might be true @@ -43,14 +41,6 @@ else # Presumably some form on *nix" FP_PLUGIN_LIB="lib${FP_NAME}.so" fi -# If HDF5_PLUGIN_PATH is defined, then it overrides everything else. -if test "x$HDF5_PLUGIN_PATH" != x ; then - HDF5_PLUGIN_LIB="$FP_PLUGIN_LIB" - # HDF5_PLUGIN_PATH already set - HDF5_PLUGIN_PATH=`${NCPATHCVT} $HDF5_PLUGIN_PATH` - return 0; -fi - # Figure out the path to where the lib is stored # This can probably be simplified @@ -59,40 +49,38 @@ cd ${TOPBUILDDIR}/plugins FP_PLUGINS=`pwd` cd ${CURWD} -# Case 1: Cmake with Visual Studio -# Do not know where to look for a dylib # Case 1: Cmake with Visual Studio if test "x$FP_ISCMAKE" != x -a "x${FP_ISMSVC}" != x ; then # Case 1a: ignore the build type directory if test -f "${FP_PLUGINS}/${FP_PLUGIN_LIB}" ; then - FP_PLUGIN_PATH="${FP_PLUGINS}" + FP_PLUGIN_DIR="${FP_PLUGINS}" fi else # Case 2: automake # Case 2a: look in .libs if test -f "${FP_PLUGINS}/.libs/${FP_PLUGIN_LIB}" ; then - FP_PLUGIN_PATH="${FP_PLUGINS}/.libs" + FP_PLUGIN_DIR="${FP_PLUGINS}/.libs" else # Case 2: look in FP_PLUGINS directly if test -f "${FP_PLUGINS}/${FP_PLUGIN_LIB}" ; then - FP_PLUGIN_PATH="${FP_PLUGINS}" + FP_PLUGIN_DIR="${FP_PLUGINS}" fi fi fi # Verify -if test "x$FP_PLUGIN_PATH" = x ; then - echo "***Fail: Could not locate a usable HDF5_PLUGIN_PATH" +if test "x$FP_PLUGIN_DIR" = x ; then + echo "***Fail: Could not locate a usable HDF5_PLUGIN_DIR" return 1 fi -if ! test -f "$FP_PLUGIN_PATH/$FP_PLUGIN_LIB" ; then +if ! test -f "$FP_PLUGIN_DIR/$FP_PLUGIN_LIB" ; then echo "***Fail: Could not locate a usable HDF5_PLUGIN_LIB" return 1 fi -FP_PLUGIN_PATH=`${NCPATHCVT} $FP_PLUGIN_PATH` +FP_PLUGIN_DIR=`${NCPATHCVT} $FP_PLUGIN_DIR` # Set the final output variables HDF5_PLUGIN_LIB="$FP_PLUGIN_LIB" -HDF5_PLUGIN_PATH="$FP_PLUGIN_PATH" +HDF5_PLUGIN_DIR="$FP_PLUGIN_DIR" return 0 } diff --git a/nc_test4/tst_filter.sh b/nc_test4/tst_filter.sh index 44bcf211ad..b478079092 100755 --- a/nc_test4/tst_filter.sh +++ b/nc_test4/tst_filter.sh @@ -43,36 +43,37 @@ sed -e 's/[ ]*\([^ ].*\)/\1/' <$1 >$2 # Hide/unhide the noop filter hidenoop() { - rm -fr ${HDF5_PLUGIN_PATH}/save - mkdir ${HDF5_PLUGIN_PATH}/save - mv ${NOOPPATH} ${HDF5_PLUGIN_PATH}/save + rm -fr ${HDF5_PLUGIN_DIR}/save + mkdir ${HDF5_PLUGIN_DIR}/save + mv ${NOOPDIR} ${HDF5_PLUGIN_DIR}/save } unhidenoop() { - mv ${HDF5_PLUGIN_PATH}/save/${NOOPLIB} ${HDF5_PLUGIN_PATH} - rm -fr ${HDF5_PLUGIN_PATH}/save + mv ${HDF5_PLUGIN_DIR}/save/${NOOPLIB} ${HDF5_PLUGIN_DIR} + rm -fr ${HDF5_PLUGIN_DIR}/save } -# Locate the plugin path and the library names; argument order is critical +# Locate the plugin dir and the library names; argument order is critical # Find bzip2 and capture findplugin h5bzip2 BZIP2LIB="${HDF5_PLUGIN_LIB}" -BZIP2PATH="${HDF5_PLUGIN_PATH}/${BZIP2LIB}" +BZIP2DIR="${HDF5_PLUGIN_DIR}/${BZIP2LIB}" # Find misc and capture findplugin h5misc -MISCPATH="${HDF5_PLUGIN_PATH}/${HDF5_PLUGIN_LIB}" +MISCDIR="${HDF5_PLUGIN_DIR}/${HDF5_PLUGIN_LIB}" # Find noop and capture findplugin h5noop NOOPLIB="${HDF5_PLUGIN_LIB}" -NOOPPATH="${HDF5_PLUGIN_PATH}/${HDF5_PLUGIN_LIB}" +NOOPDIR="${HDF5_PLUGIN_DIR}/${HDF5_PLUGIN_LIB}" -echo "final HDF5_PLUGIN_PATH=${HDF5_PLUGIN_PATH}" -export HDF5_PLUGIN_PATH +echo "final HDF5_PLUGIN_DIR=${HDF5_PLUGIN_DIR}" +export HDF5_PLUGIN_DIR +export HDF5_PLUGIN_PATH="$HDF5_PLUGIN_DIR" # Verify -if ! test -f ${BZIP2PATH} ; then echo "Unable to locate ${BZIP2PATH}"; exit 1; fi -if ! test -f ${MISCPATH} ; then echo "Unable to locate ${MISCPATH}"; exit 1; fi -if ! test -f ${NOOPPATH} ; then echo "Unable to locate ${NOOPPATH}"; exit 1; fi +if ! test -f ${BZIP2DIR} ; then echo "Unable to locate ${BZIP2DIR}"; exit 1; fi +if ! test -f ${MISCDIR} ; then echo "Unable to locate ${MISCDIR}"; exit 1; fi +if ! test -f ${NOOPDIR} ; then echo "Unable to locate ${NOOPDIR}"; exit 1; fi # See if we have szip if avail szip; then HAVE_SZIP=1; else HAVE_SZIP=0; fi diff --git a/nc_test4/tst_filterinstall.sh b/nc_test4/tst_filterinstall.sh new file mode 100755 index 0000000000..465a42141d --- /dev/null +++ b/nc_test4/tst_filterinstall.sh @@ -0,0 +1,111 @@ +#!/bin/bash + +# Test the filter install + +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ../test_common.sh + +if test "x$TESTNCZARR" = x1 ; then +. ./test_nczarr.sh +fi + +set -e + +# Use this plugin path +export HDF5_PLUGIN_PATH="${FEATURE_PLUGIN_INSTALL_DIR}" + +# Function to remove selected -s attributes from file; +# These attributes might be platform dependent +sclean() { + cat $1 \ + | sed -e '/:_IsNetcdf4/d' \ + | sed -e '/:_Endianness/d' \ + | sed -e '/_NCProperties/d' \ + | sed -e '/_SuperblockVersion/d' \ + | sed -e '/_Format/d' \ + | sed -e '/global attributes:/d' \ + | cat > $2 +} + +# Function to extract _Filter attribute from a file +# These attributes might be platform dependent +getfilterattr() { +V="$1" +sed -e '/${V}.*:_Filter/p' -ed <$2 >$3 +} + +# Function to extract _Codecs attribute from a file +# These attributes might be platform dependent +getcodecsattr() { +V="$1" +sed -e '/${V}.*:_Codecs/p' -ed <$2 >$3 +} + +trimleft() { +sed -e 's/[ ]*\([^ ].*\)/\1/' <$1 >$2 +} + +setfilter() { + FF="$1" + FSRC="$2" + FDST="$3" + FIH5="$4" + FICX="$5" + FFH5="$6" + FFCX="$7" + if test "x$FFH5" = x ; then FFH5="$FIH5" ; fi + if test "x$FFCX" = x ; then FFCX="$FICX" ; fi + rm -f $FDST + cat ${srcdir}/$FSRC \ + | sed -e "s/ref_any/${FF}/" \ + | sed -e "s/IH5/${FIH5}/" -e "s/FH5/${FFH5}/" \ + | sed -e "s/ICX/${FICX}/" -e "s/FCX/${FFCX}/" \ + | sed -e 's/"/\\"/g' -e 's/@/"/g' \ + | cat > $FDST +} + +# Execute the specified tests + +runfilter() { +zext=$1 +zfilt="$2" +zparams="$3" +zcodec="$4" +echo "*** Testing processing of filter $zfilt for map $zext" +if test "x$TESTNCZARR" = x1 ; then +fileargs "tmp_filt_${zfilt}" +deletemap $zext $file +else +file="tmp_filt_${zfilt}.nc" +rm -f $file +fi +setfilter $zfilt ref_any.cdl "tmp_filt_${zfilt}.cdl" "$zparams" "$zcodec" +if test "x$TESTNCZARR" = x1 ; then +${NCGEN} -4 -lb -o $fileurl "tmp_filt_${zfilt}.cdl" +${NCDUMP} -n $zfilt -sF $fileurl > "tmp_filt_${zfilt}.tmp" +else +${NCGEN} -4 -lb -o $file "tmp_filt_${zfilt}.cdl" +${NCDUMP} -n $zfilt -sF $file > "tmp_filt_${zfilt}.tmp" +fi +sclean "tmp_filt_${zfilt}.tmp" "tmp_filt_${zfilt}.dump" +} + +testbzip2() { + zext=$1 + if ! avail bzip2; then return 0; fi + runfilter $zext bzip2 '307,9' '[{\"id\": \"bz2\",\"level\": \"9\"}]' + diff -b -w "tmp_filt_bzip2.cdl" "tmp_filt_bzip2.dump" +} + +testset() { +# Which test cases to exercise + testbzip2 $1 +} + +if test "x$TESTNCZARR" = x1 ; then + testset file + if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testset zip ; fi + if test "x$FEATURE_S3TESTS" = xyes ; then testset s3 ; fi +else + testset nc +fi diff --git a/nc_test4/tst_specific_filters.sh b/nc_test4/tst_specific_filters.sh index 874aabcd0c..43235d2869 100755 --- a/nc_test4/tst_specific_filters.sh +++ b/nc_test4/tst_specific_filters.sh @@ -28,9 +28,6 @@ echo "findplugin.sh loaded" # Assume all test filters are in same plugin dir findplugin h5bzip2 -echo "final HDF5_PLUGIN_PATH=${HDF5_PLUGIN_PATH}" -export HDF5_PLUGIN_PATH - # Function to remove selected -s attributes from file; # These attributes might be platform dependent sclean() { @@ -167,14 +164,14 @@ testzstd() { testset() { # Which test cases to exercise if test "x$TESTNCZARR" = x1 ; then -# testfletcher32 $1 + testfletcher32 $1 testshuffle $1 fi -# testdeflate $1 -# testszip $1 -# testbzip2 $1 -# testblosc $1 -# testzstd $1 + testdeflate $1 + testszip $1 + testbzip2 $1 + testblosc $1 + testzstd $1 } if test "x$TESTNCZARR" = x1 ; then diff --git a/ncdap_test/Makefile.am b/ncdap_test/Makefile.am index b8de229ca4..25b19cde26 100644 --- a/ncdap_test/Makefile.am +++ b/ncdap_test/Makefile.am @@ -10,7 +10,7 @@ include $(top_srcdir)/lib_flags.am #SH_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose #LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose #TEST_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose -#TESTS_ENVIRONMENT += export SETX=1; +#TESTS_ENVIRONMENT = export SETX=1; # Note which tests depend on other tests. Necessary for make -j check. TEST_EXTENSIONS = .sh diff --git a/ncdump/Makefile.am b/ncdump/Makefile.am index d3513794d4..c381f30d99 100644 --- a/ncdump/Makefile.am +++ b/ncdump/Makefile.am @@ -4,10 +4,10 @@ # Ed Hartnett, Dennis Heimbigner, Ward Fisher -SH_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose -sh_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose -LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose -TESTS_ENVIRONMENT = export SETX=1; +#SH_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose +#sh_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose +#LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose +#TESTS_ENVIRONMENT = export SETX=1; # Put together AM_CPPFLAGS and AM_LDFLAGS. include $(top_srcdir)/lib_flags.am diff --git a/ncdump/tst_nccopy3.sh b/ncdump/tst_nccopy3.sh index 449e896865..7dcfc910c5 100755 --- a/ncdump/tst_nccopy3.sh +++ b/ncdump/tst_nccopy3.sh @@ -8,7 +8,9 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh +set -x set -e + echo "" # get some config.h parameters diff --git a/ncdump/tst_output.sh b/ncdump/tst_output.sh index 40c7e7cd47..18da9c3e6f 100755 --- a/ncdump/tst_output.sh +++ b/ncdump/tst_output.sh @@ -4,6 +4,7 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh # This shell script tests the output from several previous tests. +set -x set -e echo "" diff --git a/nczarr_test/Makefile.am b/nczarr_test/Makefile.am index c7c15565d8..6563344861 100644 --- a/nczarr_test/Makefile.am +++ b/nczarr_test/Makefile.am @@ -154,7 +154,7 @@ EXTRA_DIST += ref_power_901_constants.zip ref_power_901_constants.cdl ref_quotes CLEANFILES = ut_*.txt ut*.cdl tmp*.nc tmp*.cdl tmp*.txt tmp*.dmp tmp*.zip tmp*.nc tmp*.dump tmp*.tmp tmp_ngc.c ref_zarr_test_data.cdl -BUILT_SOURCES = test_quantize.c test_filter_avail.c run_specific_filters.sh +BUILT_SOURCES = test_quantize.c test_filter_avail.c run_specific_filters.sh run_filterinstall.sh test_quantize.c: $(top_srcdir)/nc_test4/tst_quantize.c rm -f $@ echo "#define TESTNCZARR" > $@ @@ -174,6 +174,15 @@ run_specific_filters.sh: $(top_srcdir)/nc_test4/tst_specific_filters.sh chmod a+x $@ rm -f run_specific_filters.tmp +run_filterinstall.sh: $(top_srcdir)/nc_test4/tst_filterinstall.sh + rm -f $@ run_filterinstall.tmp + echo "#!/bin/bash" > run_filterinstall.tmp + echo "TESTNCZARR=1" >> run_filterinstall.tmp + cat $(top_srcdir)/nc_test4/tst_filterinstall.sh >> run_filterinstall.tmp + tr -d '\r' < run_filterinstall.tmp > $@ + chmod a+x $@ + rm -f run_filterinstall.tmp + # Remove directories clean-local: rm -fr tmp*.file results.file results.s3 results.zip diff --git a/nczarr_test/run_filter.sh b/nczarr_test/run_filter.sh index fce92ba606..da653a985c 100755 --- a/nczarr_test/run_filter.sh +++ b/nczarr_test/run_filter.sh @@ -51,21 +51,21 @@ sed -e 's/[ ]*\([^ ].*\)/\1/' <$1 >$2 # Find misc and capture findplugin h5misc -MISCPATH="${HDF5_PLUGIN_PATH}/${HDF5_PLUGIN_LIB}" +MISCDIR="${HDF5_PLUGIN_DIR}/${HDF5_PLUGIN_LIB}" # Find noop and capture findplugin h5noop NOOPLIB="${HDF5_PLUGIN_LIB}" -NOOPPATH="${HDF5_PLUGIN_PATH}/${NOOPLIB}" +NOOPDIR="${HDF5_PLUGIN_DIR}/${NOOPLIB}" # Find bzip2 and capture findplugin h5bzip2 BZIP2LIB="${HDF5_PLUGIN_LIB}" -BZIP2PATH="${HDF5_PLUGIN_PATH}/${BZIP2LIB}" +BZIP2DIR="${HDF5_PLUGIN_DIR}/${BZIP2LIB}" # Verify -if ! test -f ${BZIP2PATH} ; then echo "Unable to locate ${BZIP2PATH}"; exit 1; fi -if ! test -f ${MISCPATH} ; then echo "Unable to locate ${MISCPATH}"; exit 1; fi +if ! test -f ${BZIP2DIR} ; then echo "Unable to locate ${BZIP2DIR}"; exit 1; fi +if ! test -f ${MISCDIR} ; then echo "Unable to locate ${MISCDIR}"; exit 1; fi # Execute the specified tests @@ -144,11 +144,11 @@ ${NCDUMP} -hs $fileurl > ./tmp_known_$zext.txt # Remove irrelevant -s output sclean ./tmp_known_$zext.txt tmp_known_$zext.dump # Now hide the filter code -mv ${NOOPPATH} ./${NOOPLIB}.save +mv ${NOOPDIR} ./${NOOPLIB}.save # dump and clean noop.nc header when filter is not avail ${NCDUMP} -hs $fileurl > ./tmp_unk_$zext.txt # Restore the filter code -mv ./${NOOPLIB}.save ${NOOPPATH} +mv ./${NOOPLIB}.save ${NOOPDIR} # Verify that the filter is no longer defined UNK=`sed -e '/var:_Filter/p' -e d ./tmp_unk_$zext.txt` test "x$UNK" = x diff --git a/nczarr_test/test_nczarr.sh b/nczarr_test/test_nczarr.sh index 489bc0512f..e63b82a42d 100755 --- a/nczarr_test/test_nczarr.sh +++ b/nczarr_test/test_nczarr.sh @@ -133,12 +133,12 @@ if test "x$FP_USEPLUGINS" = xyes; then echo "findplugin.sh loaded" # Locate the plugin path and the library names; argument order is critical -# Find bzip2 and capture +# Find misc in order to determine HDF5_PLUGIN+PATH. # Assume all test filters are in same plugin dir findplugin h5misc -echo "final HDF5_PLUGIN_PATH=${HDF5_PLUGIN_PATH}" -export HDF5_PLUGIN_PATH +echo "final HDF5_PLUGIN_DIR=${HDF5_PLUGIN_DIR}" +export HDF5_PLUGIN_PATH="${HDF5_PLUGIN_DIR}" fi # USEPLUGINS resetrc() { diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt index 71a6af7c40..e606b392a3 100644 --- a/plugins/CMakeLists.txt +++ b/plugins/CMakeLists.txt @@ -10,12 +10,6 @@ FILE(READ H5Znoop.c NOOP_SOURCE) FILE(WRITE ${CMAKE_CURRENT_BINARY_DIR}/H5Znoop1.c "#define NOOP_INSTANCE 1\n") FILE(APPEND ${CMAKE_CURRENT_BINARY_DIR}/H5Znoop1.c "${NOOP_SOURCE}") -SET(h5bzip2_SOURCES H5Zbzip2.c) - -IF(NOT HAVE_BZIP2) -SET(h5bzip2_SOURCES ${h5bzip2_SOURCES} blocksort.c huffman.c crctable.c randtable.c compress.c decompress.c bzlib.c) -ENDIF() - SET(h5misc_SOURCES H5Zmisc.c H5Zutil.c h5misc.h) SET(h5noop_SOURCES H5Znoop.c H5Zutil.c h5noop.h) @@ -29,11 +23,10 @@ SET(h5fletcher32_SOURCES H5Zfletcher32.c H5checksum.c) SET(h5deflate_SOURCES H5Zdeflate.c) SET(nczmisc_SOURCES NCZmisc.c) -SET(nczdefaults_SOURCES NCZdefaults.c) - -IF(ENABLE_FILTER_TESTING) -IF(BUILD_UTILITIES) +SET(nczhdf5filters_SOURCES NCZhdf5filters.c) +SET(nczstdfilters_SOURCES NCZstdfilters.c) +IF(ENABLE_PLUGINS) # LDFLAGS = -module -avoid-version -shared -export-dynamic -no-undefined SET(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}") @@ -64,7 +57,8 @@ buildplugin(h5fletcher32 "h5fletcher32") buildplugin(h5deflate "h5deflate") buildplugin(nczmisc "nczmisc") -buildplugin(nczdefaults "nczdefaults" netcdf) +buildplugin(nczhdf5filters "nczhdf5filters" netcdf) +buildplugin(nczstdfilters "nczstdfilters" netcdf) IF(ENABLE_BLOSC) SET(h5blosc_SOURCES H5Zblosc.c) @@ -80,17 +74,54 @@ ENDIF() IF(HAVE_SZ) SET(h5szip_SOURCES H5Zszip.c H5Zszip.h) buildplugin(h5szip "h5szip" ${Szip_LIBRARIES}) - SET(nczszip_SOURCES NCZszip.c) - buildplugin(nczszip "nczszip" netcdf) ENDIF() +IF(HAVE_LOCAL_BZ2) +SET(h5bzip2_SOURCES H5Zbzip2.c blocksort.c huffman.c crctable.c randtable.c compress.c decompress.c bzlib.c bzlib.h bzlib_private.h) +buildplugin(h5bzip2 "h5bzip2") +ELSE() +SET(h5bzip2_SOURCES H5Zbzip2.c) buildplugin(h5bzip2 "h5bzip2" ${Bzip2_LIBRARIES}) -# Note we use name h5bzip2 instead of bzip2 to avoid logical -# target name clash with examples/C/hdf5plugins -SET_TARGET_PROPERTIES(h5bzip2 PROPERTIES OUTPUT_NAME "bzip2") +ENDIF() + + +# Installation +IF(ENABLE_PLUGIN_INSTALL) + +MACRO(installplugin PLUG) +IF(MSVC) + SET(BUILD_PLUGIN_LIB "${PLUG}.dll") +ELSE() + SET(BUILD_PLUGIN_LIB "lib${PLUG}.so") +ENDIF() +MESSAGE(STATUS "Installing: ${BUILD_PLUGIN_LIB} into ${PLUGIN_INSTALL_DIR}") +install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/${BUILD_PLUGIN_LIB} DESTINATION ${PLUGIN_INSTALL_DIR}) +ENDMACRO() + +install(DIRECTORY DESTINATION ${PLUGIN_INSTALL_DIR}) +IF(Bzip2_FOUND) +installplugin(h5bzip2) +ENDIF() +IF(Zstd_FOUND) +installplugin(h5zstd) +ENDIF() +IF(Blosc_FOUND) +installplugin(h5blosc) +ENDIF() +IF(ENABLE_NCZARR) + installplugin(h5fletcher32) + installplugin(h5shuffle) + installplugin(h5deflate) + installplugin(nczhdf5filters) + installplugin(nczstdfilters) +IF(Szip_FOUND) + installplugin(h5szip) +ENDIF() +ENDIF() + +ENDIF(ENABLE_PLUGIN_INSTALL) -ENDIF(BUILD_UTILITIES) -ENDIF(ENABLE_FILTER_TESTING) +ENDIF(ENABLE_PLUGINS) # Copy some test files from current source dir to out-of-tree build dir. FILE(COPY ${COPY_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/) diff --git a/plugins/H5Zblosc.c b/plugins/H5Zblosc.c index 06afc4c719..3b36d568ce 100755 --- a/plugins/H5Zblosc.c +++ b/plugins/H5Zblosc.c @@ -125,8 +125,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define PUSH_ERR(f,m,s,...) fprintf(stderr,"%s\n",s) #endif /*USE_HDF5*/ -static int h5z_blosc_initialized = 0; - static size_t blosc_filter(unsigned flags, size_t cd_nelmts, const unsigned cd_values[], size_t nbytes, size_t* buf_size, void** buf); @@ -162,7 +160,7 @@ herr_t blosc_set_local(hid_t dcpl, hid_t type, hid_t space) hid_t super_type; H5T_class_t classt; - r = GET_FILTER(dcpl, FILTER_BLOSC, &flags, &nelements, values, 0, NULL); + r = GET_FILTER(dcpl, H5Z_FILTER_BLOSC, &flags, &nelements, values, 0, NULL); if (r < 0) return -1; if (nelements < 4) nelements = 4; /* First 4 slots reserved. */ @@ -209,7 +207,7 @@ herr_t blosc_set_local(hid_t dcpl, hid_t type, hid_t space) fprintf(stderr, "Blosc: Computed buffer size %d\n", bufsize); #endif - r = H5Pmodify_filter(dcpl, FILTER_BLOSC, flags, nelements, values); + r = H5Pmodify_filter(dcpl, H5Z_FILTER_BLOSC, flags, nelements, values); if (r < 0) return -1; return 1; @@ -365,7 +363,7 @@ size_t blosc_filter(unsigned flags, size_t cd_nelmts, const H5Z_class2_t blosc_H5Filter[1] = { { H5Z_CLASS_T_VERS, - (H5Z_filter_t)(FILTER_BLOSC), + (H5Z_filter_t)(H5Z_FILTER_BLOSC), 1, /* encoder_present flag (set to true) */ 1, /* decoder_present flag (set to true) */ "blosc", @@ -379,235 +377,3 @@ const H5Z_class2_t blosc_H5Filter[1] = { H5PL_type_t H5PLget_plugin_type(void) { return H5PL_TYPE_FILTER; } const void* H5PLget_plugin_info(void) { return blosc_H5Filter; } - - -/* Provide the codec support for the HDF5 blosc library */ - -/* NCZarr Filter Objects */ - -#define DEFAULT_LEVEL 9 -#define DEFAULT_BLOCKSIZE 1 -#define DEFAULT_TYPESIZE 1 -#define DEFAULT_COMPCODE BLOSC_LZ4 - -/* Forward */ -static void NCZ_blosc_codec_finalize(void); -static int NCZ_blosc_codec_to_hdf5(const char* codec, size_t* nparamsp, unsigned** paramsp); -static int NCZ_blosc_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp); -static int NCZ_blosc_modify_parameters(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp); - -/* Structure for NCZ_PLUGIN_CODEC */ -static NCZ_codec_t NCZ_blosc_codec = {/* NCZ_codec_t codec fields */ - NCZ_CODEC_CLASS_VER, /* Struct version number */ - NCZ_CODEC_HDF5, /* Struct sort */ - "blosc", /* Standard name/id of the codec */ - FILTER_BLOSC, /* HDF5 alias for blosc */ - NULL, /*NCZ_blosc_codec_initialize*/ - NCZ_blosc_codec_finalize, - NCZ_blosc_codec_to_hdf5, - NCZ_blosc_hdf5_to_codec, - NCZ_blosc_modify_parameters, -}; - -/* External Export API */ -const void* -NCZ_get_codec_info(void) -{ - if(!h5z_blosc_initialized) { - h5z_blosc_initialized = 1; - blosc_init(); - } - return (void*)&NCZ_blosc_codec; -} - -/* NCZarr Interface Functions */ - -/* Create the true parameter set: -Visible parameters: -param[0] -- reserved -param[1] -- reserved -param[2] -- reserved -param[3] -- variable chunksize in bytes | 0 (=>default) -param[4] -- compression level -param[5] -- BLOSC_SHUFFLE|BLOSC_BITSHUFFLE -param[6] -- compressor index - -Working parameters: -param[0] -- filter revision -param[1] -- blosc version -param[2] -- variable type size in bytes -param[3] -- variable chunksize in bytes -param[4] -- compression level -param[5] -- BLOSC_SHUFFLE|BLOSC_BITSHUFFLE -param[6] -- compressor index -*/ - -static void -NCZ_blosc_codec_finalize(void) -{ - if(h5z_blosc_initialized) { - blosc_destroy(); - h5z_blosc_initialized = 0; - } -} - -static int -NCZ_blosc_modify_parameters(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp) -{ - int i,stat = NC_NOERR; - nc_type vtype; - int storage, ndims; - size_t* chunklens = NULL; - size_t typesize, chunksize; - char vname[NC_MAX_NAME+1]; - unsigned* params = NULL; - size_t nparams; - size_t vnparams = *vnparamsp; - unsigned* vparams = *vparamsp; - - if(vnparams < 7) - {stat = NC_EFILTER; goto done;} - nparams = 7; - - if(vparams == NULL) - {stat = NC_EFILTER; goto done;} - - if(wnparamsp == NULL || wparamsp == NULL) - {stat = NC_EFILTER; goto done;} - - vnparams = *vnparamsp; - vparams = *vparamsp; - - /* Get variable info */ - if((stat = nc_inq_var(ncid,varid,vname,&vtype,&ndims,NULL,NULL))) goto done; - - if(ndims == 0) {stat = NC_EFILTER; goto done;} - - /* Get the typesize */ - if((stat = nc_inq_type(ncid,vtype,NULL,&typesize))) goto done; - - /* Compute chunksize */ - if((chunklens = (size_t*)calloc(ndims,sizeof(size_t)))==NULL) goto done; - if((stat = nc_inq_var_chunking(ncid,varid,&storage,chunklens))) goto done; - if(storage != NC_CHUNKED) {stat = NC_EFILTER; goto done;} - chunksize = typesize; - for(i=0;i NC_MAX_UINT) {stat = NC_EFILTER; goto done;} - params[4] = (unsigned)jc.ival; - - /* Get blocksize */ - if(NCJdictget(jcodec,"blocksize",&jtmp)) {stat = NC_EFILTER; goto done;} - if(jtmp) { - if(NCJcvt(jtmp,NCJ_INT,&jc)) {stat = NC_EFILTER; goto done;} - } else - jc.ival = DEFAULT_BLOCKSIZE; - if(jc.ival < 0 || jc.ival > NC_MAX_UINT) {stat = NC_EFILTER; goto done;} - params[3] = (unsigned)jc.ival; - - /* Get shuffle */ - if(NCJdictget(jcodec,"shuffle",&jtmp)) {stat = NC_EFILTER; goto done;} - if(jtmp) { - if(NCJcvt(jtmp,NCJ_INT,&jc)) {stat = NC_EFILTER; goto done;} - } else - jc.ival = BLOSC_NOSHUFFLE; - params[5] = (unsigned)jc.ival; - - /* Get compname */ - if(NCJdictget(jcodec,"cname",&jtmp)) {stat = NC_EFILTER; goto done;} - if(jtmp) { - if(NCJcvt(jtmp,NCJ_STRING,&jc)) {stat = NC_EFILTER; goto done;} - if(jc.sval == NULL || strlen(jc.sval) == 0) {stat = NC_EFILTER; goto done;} - if((compcode = blosc_compname_to_compcode(jc.sval)) < 0) {stat = NC_EFILTER; goto done;} - } else - compcode = DEFAULT_COMPCODE; - params[6] = (unsigned)compcode; - - if(nparamsp) *nparamsp = 7; - if(paramsp) {*paramsp = params; params = NULL;} - -done: - if(jc.sval) { - free(jc.sval); - } - if(params) { - free(params); - } - NCJreclaim(jcodec); - return stat; -} - -static int -NCZ_blosc_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp) -{ - int stat = NC_NOERR; - char json[1024]; - const char* compname = NULL; - - if(nparams == 0 || params == NULL) - {stat = NC_EINVAL; goto done;} - - /* Get the sub-compressor name */ - if(blosc_compcode_to_compname((int)params[6],&compname) < 0) {stat = NC_EFILTER; goto done;} - - snprintf(json,sizeof(json), - "{\"id\": \"blosc\",\"clevel\": %u,\"blocksize\": %u,\"cname\": \"%s\",\"shuffle\": %d}", - params[4],params[3],compname,params[5]); - - if(codecp) { - if((*codecp = strdup(json))==NULL) {stat = NC_ENOMEM; goto done;} - } - -done: - return stat; -} diff --git a/plugins/H5Zblosc.h b/plugins/H5Zblosc.h index 6e834a2931..911f24c2b3 100755 --- a/plugins/H5Zblosc.h +++ b/plugins/H5Zblosc.h @@ -29,13 +29,6 @@ extern "C" { #include "netcdf_filter_build.h" #include "blosc.h" -/* Filter revision number, starting at 1 */ -/* #define FILTER_BLOSC_VERSION 1 */ -#define FILTER_BLOSC_VERSION 2 /* multiple compressors since Blosc 1.3 */ - -/* Filter ID registered with the HDF Group */ -#define FILTER_BLOSC 32001 - #ifdef _MSC_VER #ifdef DLL_EXPORT /* define when building the library */ #define DECLSPEC __declspec(dllexport) @@ -46,6 +39,15 @@ extern "C" { #define DECLSPEC extern #endif +/* Filter revision number, starting at 1 */ +/* #define FILTER_BLOSC_VERSION 1 */ +#define FILTER_BLOSC_VERSION 2 /* multiple compressors since Blosc 1.3 */ + +#define DEFAULT_LEVEL 9 +#define DEFAULT_BLOCKSIZE 1 +#define DEFAULT_TYPESIZE 1 +#define DEFAULT_COMPCODE BLOSC_LZ4 + /* HDF5 Plugin API */ DECLSPEC H5PL_type_t H5PLget_plugin_type(void); DECLSPEC const void* H5PLget_plugin_info(void); diff --git a/plugins/H5Zbzip2.c b/plugins/H5Zbzip2.c index fa9ed8a6c0..6f1128c48e 100644 --- a/plugins/H5Zbzip2.c +++ b/plugins/H5Zbzip2.c @@ -214,92 +214,3 @@ H5Z_filter_bzip2(unsigned int flags, size_t cd_nelmts, return 0; } -/**************************************************/ -/* NCZarr Filter Objects */ - -/* Provide the codec support for the HDF5 bzip library */ - -static int NCZ_bzip2_codec_to_hdf5(const char* codec, size_t* nparamsp, unsigned** paramsp); -static int NCZ_bzip2_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp); - -static NCZ_codec_t NCZ_bzip2_codec = {/* NCZ_codec_t codec fields */ - NCZ_CODEC_CLASS_VER, /* Struct version number */ - NCZ_CODEC_HDF5, /* Struct sort */ - "bz2", /* Standard name/id of the codec */ - H5Z_FILTER_BZIP2, /* HDF5 alias for bzip2 */ - NULL, /*NCZ_bzip2_codec_initialize*/ - NULL, /*NCZ_bzip2_codec_finalize*/ - NCZ_bzip2_codec_to_hdf5, - NCZ_bzip2_hdf5_to_codec, - NULL, /*NCZ_bzip2_modify_parameters*/ -}; - -/* External Export API */ -DLLEXPORT -const void* -NCZ_get_codec_info(void) -{ - return (void*)&NCZ_bzip2_codec; -} - -static int -NCZ_bzip2_codec_to_hdf5(const char* codec_json, size_t* nparamsp, unsigned** paramsp) -{ - int stat = NC_NOERR; - NCjson* jcodec = NULL; - NCjson* jtmp = NULL; - unsigned* params = NULL; - struct NCJconst jc; - - if(nparamsp == NULL || paramsp == NULL) - {stat = NC_EINTERNAL; goto done;} - - if((params = (unsigned*)calloc(1,sizeof(unsigned)))== NULL) - {stat = NC_ENOMEM; goto done;} - - /* parse the JSON */ - if(NCJparse(codec_json,0,&jcodec)) - {stat = NC_EFILTER; goto done;} - if(NCJsort(jcodec) != NCJ_DICT) {stat = NC_EPLUGIN; goto done;} - /* Verify the codec ID */ - if(NCJdictget(jcodec,"id",&jtmp)) - {stat = NC_EFILTER; goto done;} - if(jtmp == NULL || !NCJisatomic(jtmp)) {stat = NC_EFILTER; goto done;} - if(strcmp(NCJstring(jtmp),NCZ_bzip2_codec.codecid)!=0) {stat = NC_EINVAL; goto done;} - - /* Get Level */ - if(NCJdictget(jcodec,"level",&jtmp)) - {stat = NC_EFILTER; goto done;} - if(NCJcvt(jtmp,NCJ_INT,&jc)) - {stat = NC_EFILTER; goto done;} - if(jc.ival < 0 || jc.ival > NC_MAX_UINT) {stat = NC_EINVAL; goto done;} - params[0] = (unsigned)jc.ival; - *nparamsp = 1; - *paramsp = params; params = NULL; - -done: - if(params) free(params); - NCJreclaim(jcodec); - return stat; -} - -static int -NCZ_bzip2_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp) -{ - int stat = NC_NOERR; - unsigned level = 0; - char json[1024]; - - if(nparams == 0 || params == NULL) - {stat = NC_EFILTER; goto done;} - - level = params[0]; - snprintf(json,sizeof(json),"{\"id\": \"%s\", \"level\": \"%u\"}",NCZ_bzip2_codec.codecid,level); - if(codecp) { - if((*codecp = strdup(json))==NULL) {stat = NC_ENOMEM; goto done;} - } - -done: - return stat; -} - diff --git a/plugins/H5Zzstd.c b/plugins/H5Zzstd.c index 9d56425c71..58759ce5ae 100644 --- a/plugins/H5Zzstd.c +++ b/plugins/H5Zzstd.c @@ -166,92 +166,3 @@ H5Z_filter_zstd(unsigned int flags, size_t cd_nelmts, return 0; } -/**************************************************/ -/* NCZarr Filter Objects */ - -/* Provide the codec support for the HDF5 zstandard library */ - -static int NCZ_zstd_codec_to_hdf5(const char* codec, size_t* nparamsp, unsigned** paramsp); -static int NCZ_zstd_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp); - -static NCZ_codec_t NCZ_zstd_codec = {/* NCZ_codec_t codec fields */ - NCZ_CODEC_CLASS_VER, /* Struct version number */ - NCZ_CODEC_HDF5, /* Struct sort */ - "zstd", /* Standard name/id of the codec */ - H5Z_FILTER_ZSTD, /* HDF5 alias for zstd */ - NULL, /*NCZ_zstd_codec_initialize*/ - NULL, /*NCZ_zstd_codec_finalize*/ - NCZ_zstd_codec_to_hdf5, - NCZ_zstd_hdf5_to_codec, - NULL, /*NCZ_zstd_modify_parameters*/ -}; - -/* External Export API */ -DLLEXPORT -const void* -NCZ_get_codec_info(void) -{ - return (void*)&NCZ_zstd_codec; -} - -static int -NCZ_zstd_codec_to_hdf5(const char* codec_json, size_t* nparamsp, unsigned** paramsp) -{ - int stat = NC_NOERR; - NCjson* jcodec = NULL; - NCjson* jtmp = NULL; - unsigned* params = NULL; - struct NCJconst jc; - - if(nparamsp == NULL || paramsp == NULL) - {stat = NC_EINTERNAL; goto done;} - - if((params = (unsigned*)calloc(1,sizeof(unsigned)))== NULL) - {stat = NC_ENOMEM; goto done;} - - /* parse the JSON */ - if(NCJparse(codec_json,0,&jcodec)) - {stat = NC_EFILTER; goto done;} - if(NCJsort(jcodec) != NCJ_DICT) {stat = NC_EPLUGIN; goto done;} - /* Verify the codec ID */ - if(NCJdictget(jcodec,"id",&jtmp)) - {stat = NC_EFILTER; goto done;} - if(jtmp == NULL || !NCJisatomic(jtmp)) {stat = NC_EFILTER; goto done;} - if(strcmp(NCJstring(jtmp),NCZ_zstd_codec.codecid)!=0) {stat = NC_EINVAL; goto done;} - - /* Get Level */ - if(NCJdictget(jcodec,"level",&jtmp)) - {stat = NC_EFILTER; goto done;} - if(NCJcvt(jtmp,NCJ_INT,&jc)) - {stat = NC_EFILTER; goto done;} - if(jc.ival < 0 || jc.ival > NC_MAX_UINT) {stat = NC_EINVAL; goto done;} - params[0] = (unsigned)jc.ival; - *nparamsp = 1; - *paramsp = params; params = NULL; - -done: - if(params) free(params); - NCJreclaim(jcodec); - return stat; -} - -static int -NCZ_zstd_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp) -{ - int stat = NC_NOERR; - unsigned level = 0; - char json[1024]; - - if(nparams == 0 || params == NULL) - {stat = NC_EFILTER; goto done;} - - level = params[0]; - snprintf(json,sizeof(json),"{\"id\": \"%s\", \"level\": \"%u\"}",NCZ_zstd_codec.codecid,level); - if(codecp) { - if((*codecp = strdup(json))==NULL) {stat = NC_ENOMEM; goto done;} - } - -done: - return stat; -} - diff --git a/plugins/Makefile.am b/plugins/Makefile.am index b367c98d04..ded5b30760 100644 --- a/plugins/Makefile.am +++ b/plugins/Makefile.am @@ -14,34 +14,22 @@ if ISMINGW LDADD = ${top_builddir}/liblib/libnetcdf.la endif -# BZIP2 version 1.0.8 (https://sourceware.org/bzip2/) -BZIP2SRC = blocksort.c huffman.c crctable.c randtable.c compress.c decompress.c bzlib.c bzlib.h bzlib_private.h - EXTRA_DIST = CMakeLists.txt # The HDF5 filter wrappers -EXTRA_DIST += \ +EXTRA_DIST += stdinstall.sh \ H5Ztemplate.c H5Zmisc.c H5Zutil.c H5Znoop.c h5noop.h NCZmisc.c \ H5Zshuffle.c H5Zdeflate.c H5Zszip.c H5Zszip.h \ H5Zbzip2.c h5bzip2.h H5Zblosc.c H5Zblosc.h H5Zzstd.c H5Zzstd.h + # The Codec filter wrappers -EXTRA_DIST += NCZdefaults.c NCZszip.c +EXTRA_DIST += NCZhdf5filters.c NCZstdfilters.c # The Filter implementations EXTRA_DIST += H5checksum.c - -EXTRA_DIST += ${BZIP2SRC} BZIP2_LICENSE - if ENABLE_FILTER_TESTING -lib_LTLIBRARIES += libh5bzip2.la -libh5bzip2_la_SOURCES = H5Zbzip2.c h5bzip2.h -if ! HAVE_BZIP2 -libh5bzip2_la_SOURCES += ${BZIP2SRC} -endif - noinst_LTLIBRARIES += libh5misc.la libh5noop.la libh5noop1.la libnczmisc.la -noinst_LTLIBRARIES += libnczdefaults.la if ENABLE_NCZARR_FILTERS noinst_LTLIBRARIES += libh5fletcher32.la libh5shuffle.la libh5deflate.la @@ -51,27 +39,39 @@ libh5deflate_la_SOURCES = H5Zdeflate.c # Need our version of szip if libsz available and we are not using HDF5 if HAVE_SZ -noinst_LTLIBRARIES += libh5szip.la libnczszip.la +noinst_LTLIBRARIES += libh5szip.la libh5szip_la_SOURCES = H5Zszip.c H5Zszip.h -libnczszip_la_SOURCES = NCZszip.c endif -libnczdefaults_la_SOURCES = NCZdefaults.c - endif # ENABLE_NCZARR_FILTERS +if ENABLE_PLUGINS + +libnczstdfilters_la_SOURCES = NCZstdfilters.c + +libnczhdf5filters_la_SOURCES = NCZhdf5filters.c + if HAVE_BLOSC -noinst_LTLIBRARIES += libh5blosc.la libh5blosc_la_SOURCES = H5Zblosc.c H5Zblosc.h endif if HAVE_ZSTD -noinst_LTLIBRARIES += libh5zstd.la libh5zstd_la_SOURCES = H5Zzstd.c H5Zzstd.h endif -libh5misc_la_SOURCES = H5Zmisc.c H5Zutil.c h5misc.h +noinst_LTLIBRARIES += libnczhdf5filters.la +noinst_LTLIBRARIES += libnczstdfilters.la +if HAVE_BLOSC +noinst_LTLIBRARIES += libh5blosc.la +endif +if HAVE_ZSTD +noinst_LTLIBRARIES += libh5zstd.la +endif + +endif #ENABLE_PLUGINS + +libh5misc_la_SOURCES = H5Zmisc.c H5Zutil.c h5misc.h libnczmisc_la_SOURCES = NCZmisc.c # The noop filter is to allow testing of multifilters and filter order @@ -79,6 +79,14 @@ libnczmisc_la_SOURCES = NCZmisc.c libh5noop_la_SOURCES = H5Znoop.c H5Zutil.c h5noop.h libh5noop1_la_SOURCES = H5Znoop1.c H5Zutil.c h5noop.h +# Bzip2 is used to test more complex filters +libh5bzip2_la_SOURCES = H5Zbzip2.c h5bzip2.h +BZIP2SRC = blocksort.c huffman.c crctable.c randtable.c compress.c decompress.c bzlib.c bzlib.h bzlib_private.h +EXTRA_DIST += ${BZIP2SRC} BZIP2_LICENSE +if HAVE_LOCAL_BZ2 +libh5bzip2_la_SOURCES += ${BZIP2SRC} +endif + endif #ENABLE_FILTER_TESTING BUILT_SOURCES = H5Znoop1.c @@ -87,6 +95,9 @@ H5Znoop1.c: Makefile H5Znoop.c echo '#define NOOP_INSTANCE 1' > $@ cat ${srcdir}/H5Znoop.c >> $@ +noinst_LTLIBRARIES += libh5bzip2.la + +# Record where bzip2 came from; may be out of date BZIP2VER = 1.0.8 BZIP2DIR = bzip2-${BZIP2VER} BZIP2URL = https://sourceware.org/pub/bzip2/${BZIP2DIR}.tar.gz @@ -96,3 +107,7 @@ bzip2:: tar -zxf ${BZIP2DIR}.tar.gz cd ${BZIP2DIR}; cp ${BZIP2SRC} ..; cp LICENSE ../BZIP2_LICENSE ; cd .. rm -fr ./${BZIP2DIR} + +# Custom install +install-exec-hook: + sh ./stdinstall.sh diff --git a/plugins/NCZdefaults.c b/plugins/NCZhdf5filters.c similarity index 53% rename from plugins/NCZdefaults.c rename to plugins/NCZhdf5filters.c index 48dc3439b4..a8336ad730 100644 --- a/plugins/NCZdefaults.c +++ b/plugins/NCZhdf5filters.c @@ -21,6 +21,11 @@ Author: Dennis Heimbigner #include "netcdf_filter_build.h" #include "netcdf_json.h" +#ifdef HAVE_SZ +#include +#include "H5Zszip.h" +#endif + #define H5Z_FILTER_DEFLATE 1 /*deflation like gzip */ #define H5Z_FILTER_SHUFFLE 2 /*shuffle the data */ #define H5Z_FILTER_FLETCHER32 3 /*fletcher32 checksum of EDC */ @@ -40,6 +45,12 @@ static int NCZ_fletcher32_modify_parameters(int ncid, int varid, size_t* vnparam static int NCZ_deflate_codec_to_hdf5(const char* codec, size_t* nparamsp, unsigned** paramsp); static int NCZ_deflate_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp); +#ifdef HAVE_SZ +static int NCZ_szip_codec_to_hdf5(const char* codec, size_t* nparamsp, unsigned** paramsp); +static int NCZ_szip_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp); +static int NCZ_szip_modify_parameters(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp); +#endif + /**************************************************/ static NCZ_codec_t NCZ_shuffle_codec = { @@ -302,10 +313,224 @@ NCZ_deflate_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp) /**************************************************/ +#ifdef HAVE_SZ + +static NCZ_codec_t NCZ_szip_codec = { + NCZ_CODEC_CLASS_VER, /* Struct version number */ + NCZ_CODEC_HDF5, /* Struct sort */ + "szip", /* Standard name/id of the codec */ + H5Z_FILTER_SZIP, /* HDF5 alias for szip */ + NULL, /*NCZ_szip_codec_initialize*/ + NULL, /*NCZ_szip_codec_finalize*/ + NCZ_szip_codec_to_hdf5, + NCZ_szip_hdf5_to_codec, + NCZ_szip_modify_parameters, +}; + +static int +NCZ_szip_codec_to_hdf5(const char* codec_json, size_t* nparamsp, unsigned** paramsp) +{ + int stat = NC_NOERR; + unsigned* params = NULL; + size_t nparams = 2; /* No. of visible parameters */ + NCjson* json = NULL; + NCjson* jtmp = NULL; + struct NCJconst jc = {0,0,0,NULL}; + + if(nparamsp == NULL || paramsp == NULL) + {stat = NC_EINTERNAL; goto done;} + + if((params = (unsigned*)calloc(nparams,sizeof(unsigned)))== NULL) + {stat = NC_ENOMEM; goto done;} + + if(NCJparse(codec_json,0,&json)) + {stat = NC_EFILTER; goto done;} + + if(NCJdictget(json,"mask",&jtmp) || jtmp == NULL) + {stat = NC_EFILTER; goto done;} + if(NCJcvt(jtmp,NCJ_INT,&jc)) + {stat = NC_EFILTER; goto done;} + params[H5Z_SZIP_PARM_MASK] = (unsigned)jc.ival; + + jtmp = NULL; + if(NCJdictget(json,"pixels-per-block",&jtmp) || jtmp == NULL) + {stat = NC_EFILTER; goto done;} + if(NCJcvt(jtmp,NCJ_INT,&jc)) + {stat = NC_EFILTER; goto done;} + params[H5Z_SZIP_PARM_PPB] = (unsigned)jc.ival; + + *nparamsp = nparams; + *paramsp = params; params = NULL; + +done: + NCJreclaim(json); + nullfree(params); + return stat; +} + +static int +NCZ_szip_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp) +{ + int stat = NC_NOERR; + char json[2048]; + + snprintf(json,sizeof(json),"{\"id\": \"%s\", \"mask\": %u, \"pixels-per-block\": %u}", + NCZ_szip_codec.codecid, + params[H5Z_SZIP_PARM_MASK], + params[H5Z_SZIP_PARM_PPB]); + if(codecp) { + if((*codecp = strdup(json))==NULL) {stat = NC_ENOMEM; goto done;} + } + +done: + return stat; +} + +static int +NCZ_szip_modify_parameters(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp) +{ + int i,ret_value = NC_NOERR; + nc_type vtype; + size_t typesize, scanline, dtype_precision, npoints; + int ndims, storage, dtype_order; + int dimids[NC_MAX_VAR_DIMS]; + char vname[NC_MAX_NAME+1]; + size_t chunklens[NC_MAX_VAR_DIMS]; + unsigned* params = NULL; + unsigned* vparams = NULL; + size_t wnparams = 4; + + if(wnparamsp == NULL || wparamsp == NULL) + {ret_value = NC_EFILTER; goto done;} + if(vnparamsp == NULL || vparamsp == NULL) + {ret_value = NC_EFILTER; goto done;} + if(*vnparamsp > 0 && *vparamsp == NULL) + {ret_value = NC_EFILTER; goto done;} + + vparams = *vparamsp; + + /* Get variable info */ + if((ret_value = nc_inq_var(ncid,varid,vname,&vtype,&ndims,dimids,NULL))) goto done; + + /* Get the typesize */ + if((ret_value = nc_inq_type(ncid,vtype,NULL,&typesize))) goto done; + + /* Get datatype's precision, in case is less than full bits */ + dtype_precision = typesize*8; + + if(dtype_precision > 24) { + if(dtype_precision <= 32) + dtype_precision = 32; + else if(dtype_precision <= 64) + dtype_precision = 64; + } /* end if */ + + if(ndims == 0) {ret_value = NC_EFILTER; goto done;} + + /* Set "local" parameter for this dataset's "pixels-per-scanline" */ + if((ret_value = nc_inq_dimlen(ncid,dimids[ndims-1],&scanline))) goto done; + + /* Get number of elements for the dataspace; use + total number of elements in the chunk to define the new 'scanline' size */ + /* Compute chunksize */ + if((ret_value = nc_inq_var_chunking(ncid,varid,&storage,chunklens))) goto done; + if(storage != NC_CHUNKED) {ret_value = NC_EFILTER; goto done;} + npoints = 1; + for(i=0;i +#include +#include +#include + +#include "netcdf.h" +#include "netcdf_filter.h" +#include "netcdf_filter_build.h" +#include "netcdf_json.h" + +/* Provide Codec information for the standard filters */ + +#ifndef H5Z_FILTER_BZIP2 +#define H5Z_FILTER_BZIP2 1 +#define H5Z_FILTER_ZSTD 2 +#define H5Z_FILTER_BLOSC 3 +#endif + +#ifdef HAVE_BLOSC +#include "H5Zblosc.h" +#endif + +/**************************************************/ +/* NCZarr Filter Objects */ + +/* Forward */ + +#ifdef HAVE_BZ2 +static int NCZ_bzip2_codec_to_hdf5(const char* codec, size_t* nparamsp, unsigned** paramsp); +static int NCZ_bzip2_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp); +#endif + +#ifdef HAVE_ZSTD +static int NCZ_zstd_codec_to_hdf5(const char* codec, size_t* nparamsp, unsigned** paramsp); +static int NCZ_zstd_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp); +#endif + +#ifdef HAVE_BLOSC +static int NCZ_blosc_codec_to_hdf5(const char* codec, size_t* nparamsp, unsigned** paramsp); +static int NCZ_blosc_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp); +static int NCZ_blosc_modify_parameters(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp); +static void NCZ_blosc_codec_finalize(void); +#endif + +/**************************************************/ +/* Provide the codec support for bzip2 filter */ + +#ifdef HAVE_BZ2 +static NCZ_codec_t NCZ_bzip2_codec = {/* NCZ_codec_t codec fields */ + NCZ_CODEC_CLASS_VER, /* Struct version number */ + NCZ_CODEC_HDF5, /* Struct sort */ + "bz2", /* Standard name/id of the codec */ + H5Z_FILTER_BZIP2, /* HDF5 alias for bzip2 */ + NULL, /*NCZ_bzip2_codec_initialize*/ + NULL, /*NCZ_bzip2_codec_finalize*/ + NCZ_bzip2_codec_to_hdf5, + NCZ_bzip2_hdf5_to_codec, + NULL, /*NCZ_bzip2_modify_parameters*/ +}; + +/* External Export API */ +DLLEXPORT +const void* +NCZ_get_codec_info(void) +{ + return (void*)&NCZ_bzip2_codec; +} + +static int +NCZ_bzip2_codec_to_hdf5(const char* codec_json, size_t* nparamsp, unsigned** paramsp) +{ + int stat = NC_NOERR; + NCjson* jcodec = NULL; + NCjson* jtmp = NULL; + unsigned* params = NULL; + struct NCJconst jc; + + if(nparamsp == NULL || paramsp == NULL) + {stat = NC_EINTERNAL; goto done;} + + if((params = (unsigned*)calloc(1,sizeof(unsigned)))== NULL) + {stat = NC_ENOMEM; goto done;} + + /* parse the JSON */ + if(NCJparse(codec_json,0,&jcodec)) + {stat = NC_EFILTER; goto done;} + if(NCJsort(jcodec) != NCJ_DICT) {stat = NC_EPLUGIN; goto done;} + /* Verify the codec ID */ + if(NCJdictget(jcodec,"id",&jtmp)) + {stat = NC_EFILTER; goto done;} + if(jtmp == NULL || !NCJisatomic(jtmp)) {stat = NC_EFILTER; goto done;} + if(strcmp(NCJstring(jtmp),NCZ_bzip2_codec.codecid)!=0) {stat = NC_EINVAL; goto done;} + + /* Get Level */ + if(NCJdictget(jcodec,"level",&jtmp)) + {stat = NC_EFILTER; goto done;} + if(NCJcvt(jtmp,NCJ_INT,&jc)) + {stat = NC_EFILTER; goto done;} + if(jc.ival < 0 || jc.ival > NC_MAX_UINT) {stat = NC_EINVAL; goto done;} + params[0] = (unsigned)jc.ival; + *nparamsp = 1; + *paramsp = params; params = NULL; + +done: + if(params) free(params); + NCJreclaim(jcodec); + return stat; +} + +static int +NCZ_bzip2_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp) +{ + int stat = NC_NOERR; + unsigned level = 0; + char json[1024]; + + if(nparams == 0 || params == NULL) + {stat = NC_EFILTER; goto done;} + + level = params[0]; + snprintf(json,sizeof(json),"{\"id\": \"%s\", \"level\": \"%u\"}",NCZ_bzip2_codec.codecid,level); + if(codecp) { + if((*codecp = strdup(json))==NULL) {stat = NC_ENOMEM; goto done;} + } + +done: + return stat; +} +#endif + +/**************************************************/ +/* Provide the codec support for zstandard filter */ + +#ifdef HAVE_ZSTD + +static NCZ_codec_t NCZ_zstd_codec = {/* NCZ_codec_t codec fields */ + NCZ_CODEC_CLASS_VER, /* Struct version number */ + NCZ_CODEC_HDF5, /* Struct sort */ + "zstd", /* Standard name/id of the codec */ + H5Z_FILTER_ZSTD, /* HDF5 alias for zstd */ + NULL, /*NCZ_zstd_codec_initialize*/ + NULL, /*NCZ_zstd_codec_finalize*/ + NCZ_zstd_codec_to_hdf5, + NCZ_zstd_hdf5_to_codec, + NULL, /*NCZ_zstd_modify_parameters*/ +}; + +static int +NCZ_zstd_codec_to_hdf5(const char* codec_json, size_t* nparamsp, unsigned** paramsp) +{ + int stat = NC_NOERR; + NCjson* jcodec = NULL; + NCjson* jtmp = NULL; + unsigned* params = NULL; + struct NCJconst jc; + + if(nparamsp == NULL || paramsp == NULL) + {stat = NC_EINTERNAL; goto done;} + + if((params = (unsigned*)calloc(1,sizeof(unsigned)))== NULL) + {stat = NC_ENOMEM; goto done;} + + /* parse the JSON */ + if(NCJparse(codec_json,0,&jcodec)) + {stat = NC_EFILTER; goto done;} + if(NCJsort(jcodec) != NCJ_DICT) {stat = NC_EPLUGIN; goto done;} + /* Verify the codec ID */ + if(NCJdictget(jcodec,"id",&jtmp)) + {stat = NC_EFILTER; goto done;} + if(jtmp == NULL || !NCJisatomic(jtmp)) {stat = NC_EFILTER; goto done;} + if(strcmp(NCJstring(jtmp),NCZ_zstd_codec.codecid)!=0) {stat = NC_EINVAL; goto done;} + + /* Get Level */ + if(NCJdictget(jcodec,"level",&jtmp)) + {stat = NC_EFILTER; goto done;} + if(NCJcvt(jtmp,NCJ_INT,&jc)) + {stat = NC_EFILTER; goto done;} + if(jc.ival < 0 || jc.ival > NC_MAX_UINT) {stat = NC_EINVAL; goto done;} + params[0] = (unsigned)jc.ival; + *nparamsp = 1; + *paramsp = params; params = NULL; + +done: + if(params) free(params); + NCJreclaim(jcodec); + return stat; +} + +static int +NCZ_zstd_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp) +{ + int stat = NC_NOERR; + unsigned level = 0; + char json[1024]; + + if(nparams == 0 || params == NULL) + {stat = NC_EFILTER; goto done;} + + level = params[0]; + snprintf(json,sizeof(json),"{\"id\": \"%s\", \"level\": \"%u\"}",NCZ_zstd_codec.codecid,level); + if(codecp) { + if((*codecp = strdup(json))==NULL) {stat = NC_ENOMEM; goto done;} + } + +done: + return stat; +} +#endif + +/**************************************************/ +/* Provide the codec support for blosc filter */ + +#ifdef HAVE_BLOSC + +/* Structure for NCZ_PLUGIN_CODEC */ +static NCZ_codec_t NCZ_blosc_codec = {/* NCZ_codec_t codec fields */ + NCZ_CODEC_CLASS_VER, /* Struct version number */ + NCZ_CODEC_HDF5, /* Struct sort */ + "blosc", /* Standard name/id of the codec */ + H5Z_FILTER_BLOSC, /* HDF5 alias for blosc */ + NULL, /*NCZ_blosc_codec_initialize*/ + NCZ_blosc_codec_finalize, + NCZ_blosc_codec_to_hdf5, + NCZ_blosc_hdf5_to_codec, + NCZ_blosc_modify_parameters, +}; + +/* NCZarr Interface Functions */ + +/* Create the true parameter set: +Visible parameters: +param[0] -- reserved +param[1] -- reserved +param[2] -- reserved +param[3] -- variable chunksize in bytes | 0 (=>default) +param[4] -- compression level +param[5] -- BLOSC_SHUFFLE|BLOSC_BITSHUFFLE +param[6] -- compressor index + +Working parameters: +param[0] -- filter revision +param[1] -- blosc version +param[2] -- variable type size in bytes +param[3] -- variable chunksize in bytes +param[4] -- compression level +param[5] -- BLOSC_SHUFFLE|BLOSC_BITSHUFFLE +param[6] -- compressor index +*/ + +void blosc_destroy(void); + +static int ncz_blosc_initialized = 0; + +static void +NCZ_blosc_codec_finalize(void) +{ + if(ncz_blosc_initialized) { + blosc_destroy(); + ncz_blosc_initialized = 0; + } +} + +static int +NCZ_blosc_modify_parameters(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp) +{ + int i,stat = NC_NOERR; + nc_type vtype; + int storage, ndims; + size_t* chunklens = NULL; + size_t typesize, chunksize; + char vname[NC_MAX_NAME+1]; + unsigned* params = NULL; + size_t nparams; + size_t vnparams = *vnparamsp; + unsigned* vparams = *vparamsp; + + if(vnparams < 7) + {stat = NC_EFILTER; goto done;} + nparams = 7; + + if(vparams == NULL) + {stat = NC_EFILTER; goto done;} + + if(wnparamsp == NULL || wparamsp == NULL) + {stat = NC_EFILTER; goto done;} + + vnparams = *vnparamsp; + vparams = *vparamsp; + + /* Get variable info */ + if((stat = nc_inq_var(ncid,varid,vname,&vtype,&ndims,NULL,NULL))) goto done; + + if(ndims == 0) {stat = NC_EFILTER; goto done;} + + /* Get the typesize */ + if((stat = nc_inq_type(ncid,vtype,NULL,&typesize))) goto done; + + /* Compute chunksize */ + if((chunklens = (size_t*)calloc(ndims,sizeof(size_t)))==NULL) goto done; + if((stat = nc_inq_var_chunking(ncid,varid,&storage,chunklens))) goto done; + if(storage != NC_CHUNKED) {stat = NC_EFILTER; goto done;} + chunksize = typesize; + for(i=0;i NC_MAX_UINT) {stat = NC_EFILTER; goto done;} + params[4] = (unsigned)jc.ival; + + /* Get blocksize */ + if(NCJdictget(jcodec,"blocksize",&jtmp)) {stat = NC_EFILTER; goto done;} + if(jtmp) { + if(NCJcvt(jtmp,NCJ_INT,&jc)) {stat = NC_EFILTER; goto done;} + } else + jc.ival = DEFAULT_BLOCKSIZE; + if(jc.ival < 0 || jc.ival > NC_MAX_UINT) {stat = NC_EFILTER; goto done;} + params[3] = (unsigned)jc.ival; + + /* Get shuffle */ + if(NCJdictget(jcodec,"shuffle",&jtmp)) {stat = NC_EFILTER; goto done;} + if(jtmp) { + if(NCJcvt(jtmp,NCJ_INT,&jc)) {stat = NC_EFILTER; goto done;} + } else + jc.ival = BLOSC_NOSHUFFLE; + params[5] = (unsigned)jc.ival; + + /* Get compname */ + if(NCJdictget(jcodec,"cname",&jtmp)) {stat = NC_EFILTER; goto done;} + if(jtmp) { + if(NCJcvt(jtmp,NCJ_STRING,&jc)) {stat = NC_EFILTER; goto done;} + if(jc.sval == NULL || strlen(jc.sval) == 0) {stat = NC_EFILTER; goto done;} + if((compcode = blosc_compname_to_compcode(jc.sval)) < 0) {stat = NC_EFILTER; goto done;} + } else + compcode = DEFAULT_COMPCODE; + params[6] = (unsigned)compcode; + + if(nparamsp) *nparamsp = 7; + if(paramsp) {*paramsp = params; params = NULL;} + +done: + if(jc.sval) { + free(jc.sval); + } + if(params) { + free(params); + } + NCJreclaim(jcodec); + return stat; +} + +static int +NCZ_blosc_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp) +{ + int stat = NC_NOERR; + char json[1024]; + const char* compname = NULL; + + if(nparams == 0 || params == NULL) + {stat = NC_EINVAL; goto done;} + + /* Get the sub-compressor name */ + if(blosc_compcode_to_compname((int)params[6],&compname) < 0) {stat = NC_EFILTER; goto done;} + + snprintf(json,sizeof(json), + "{\"id\": \"blosc\",\"clevel\": %u,\"blocksize\": %u,\"cname\": \"%s\",\"shuffle\": %d}", + params[4],params[3],compname,params[5]); + + if(codecp) { + if((*codecp = strdup(json))==NULL) {stat = NC_ENOMEM; goto done;} + } + +done: + return stat; +} +#endif + +/**************************************************/ + +NCZ_codec_t* NCZ_stdfilters_codecs[] = { +#ifdef HAVE_BZ2 +&NCZ_bzip2_codec, +#endif +#ifdef HAVE_ZSTD +&NCZ_zstd_codec, +#endif +#ifdef HAVE_BLOSC +&NCZ_blosc_codec, +#endif +NULL +}; + +/* External Export API */ +DLLEXPORT +const void* +NCZ_codec_info_defaults(void) +{ + return (void*)&NCZ_stdfilters_codecs; +} + diff --git a/plugins/stdinstall.in b/plugins/stdinstall.in new file mode 100755 index 0000000000..74a73765a8 --- /dev/null +++ b/plugins/stdinstall.in @@ -0,0 +1,44 @@ +#!/bin/bash + +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ../test_common.sh + +set -x + +INSTALLDIR=@PLUGIN_INSTALL_DIR@ + +# Load the findplugins function +. ${builddir}/findplugin.sh +echo "findplugin.sh loaded" + +installplugin() { +PLUG="$1" +# Locate the plugin path and the library name; argument order is critical +findplugin ${PLUG} +if ! test -f "$HDF5_PLUGIN_DIR/$HDF5_PLUGIN_LIB" ; then + echo "Not exists: ${HDF5_PLUGIN_DIR}/$HDF5_PLUGIN_LIB ; ignored" + return +fi +if ! test -d "${INSTALLDIR}" ; then + echo "Not exists: ${INSTALLDIR} ; creating" + mkdir ${INSTALLDIR} +fi +echo "Installing: $HDF5_PLUGIN_DIR/$HDF5_PLUGIN_LIB into $INSTALLDIR" +cp -f "$HDF5_PLUGIN_DIR/$HDF5_PLUGIN_LIB" $INSTALLDIR +} + +if test "x$USEPLUGINS" != x ; then + if test "x$INSTALLDIR" != x ; then + installplugin h5bzip2 + installplugin h5zstd + installplugin h5blosc + if test "x$FEATURE_NCZARR" ; then + installplugin h5fletcher32 + installplugin h5shuffle + installplugin h5deflate + installplugin h5szip + installplugin nczdefaults + installplugin nczszip + fi + fi +fi diff --git a/test_common.in b/test_common.in index e010870b45..eff6baf8e5 100644 --- a/test_common.in +++ b/test_common.in @@ -28,9 +28,11 @@ FEATURE_PARALLEL=@HAS_PARALLEL@ # Define selected features of the build FEATURE_HDF5=@HAS_HDF5@ +FEATURE_NCZARR=@HAS_NCZARR@ FEATURE_S3TESTS=@DO_NCZARR_S3_TESTS@ FEATURE_NCZARR_ZIP=@DO_NCZARR_ZIP_TESTS@ FEATURE_FILTERTESTS=@DO_FILTER_TESTS@ +FEATURE_PLUGIN_INSTALL_DIR=@PLUGIN_INSTALL_DIR@ set -e diff --git a/unit_test/test_pathcvt.c b/unit_test/test_pathcvt.c index 6c12b107f4..b0230f53b3 100644 --- a/unit_test/test_pathcvt.c +++ b/unit_test/test_pathcvt.c @@ -108,12 +108,12 @@ static Test PATHTESTS[] = { char* macros[128]; /*Forward */ -#ifdef DEBUG -static const char* kind2string(int kind); -#endif static char* expand(const char* s); static void setmacros(void); static void reclaimmacros(void); +#ifdef DEBUG +static const char* kind2string(int kind); +#endif int main(int argc, char** argv)