Add zstd test and fix plugin build for CMake/Cygwin

CMakeLists.txt

@@ -423,15 +423,15 @@ if(NC_EXTRA_DEPS)
   foreach(_DEP ${DEPS_LIST})
     string(REGEX REPLACE "^-l" "" _LIB ${_DEP})
     FIND_LIBRARY("${_LIB}_DEP" NAMES "${_LIB}" "lib${_LIB}")
-    message(${${_LIB}_DEP})
+    message(STATUS ${${_LIB}_DEP})
     if("${${_LIB}_DEP}" STREQUAL "${_LIB}_DEP-NOTFOUND")
       message(FATAL_ERROR "Error finding ${_LIB}.")
     else()
       message(STATUS "Found ${_LIB}: ${${_LIB}_DEP}")
     endif()
     set(EXTRA_DEPS ${EXTRA_DEPS} "${${_LIB}_DEP}")
   endforeach()
-  message("Extra deps: ${EXTRA_DEPS}")
+  message(STATUS "Extra deps: ${EXTRA_DEPS}")
   list(REMOVE_DUPLICATES EXTRA_DEPS)
   set(CMAKE_REQUIRED_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES} ${EXTRA_DEPS})
 endif()
@@ -624,7 +624,7 @@ option(NETCDF_ENABLE_FILTER_BLOSC "Enable use of blosc compression library if it
 option(NETCDF_ENABLE_FILTER_ZSTD  "Enable use of Zstd compression library if it is available." ON)
 
 # If user wants, then install selected plugins (default on)
-set(PLUGIN_INSTALL_DIR "NO" CACHE STRING "Whether and where we should install plugins; defaults to yes")
+set(PLUGIN_INSTALL_DIR "YES" CACHE STRING "Whether and where we should install plugins; defaults to yes")
 
 # This is ugly, but seems necessary because of CMake's boolean structure
 set(boolval FALSE)
@@ -663,7 +663,7 @@ if(ENABLE_PLUGIN_INSTALL)
 	  set(PLUGIN_INSTALL_DIR "/usr/local/hdf5/lib/plugin")
 	endif()
       endif()
-        message("Defaulting to -DPLUGIN_INSTALL_DIR=${PLUGIN_INSTALL_DIR}")
+        message(STATUS "Defaulting to -DPLUGIN_INSTALL_DIR=${PLUGIN_INSTALL_DIR}")
     endif()
 endif()
 
@@ -678,13 +678,15 @@ if(ENABLE_PLUGIN_INSTALL)
   # Get last element
   list(GET PATH_LIST -1 PLUGIN_INSTALL_DIR)
   set(PLUGIN_INSTALL_DIR_SETTING "${PLUGIN_INSTALL_DIR}")
-  message("Final value of-DPLUGIN_INSTALL_DIR=${PLUGIN_INSTALL_DIR}")
+  message(STATUS "Final value of-DPLUGIN_INSTALL_DIR=${PLUGIN_INSTALL_DIR}")
 else() # No option specified
     unset(PLUGIN_INSTALL_DIR)
     unset(PLUGIN_INSTALL_DIR CACHE)
     set(PLUGIN_INSTALL_DIR_SETTING "N.A.")
 endif()
 
+message(STATUS "ENABLE_PLUGIN_INSTALL=${ENABLE_PLUGIN_INSTALL} PLUGIN_INSTALL_DIR=${PLUGIN_INSTALL_DIR}")
+
 # Try to enable NCZarr zip support
 option(NETCDF_ENABLE_NCZARR_ZIP "Enable NCZarr ZIP support." ${NETCDF_ENABLE_NCZARR})
 

configure.ac

@@ -2102,7 +2102,7 @@ AC_ARG_WITH([plugin-dir], [AS_HELP_STRING([--with-plugin-dir=<absolute directory
               [Install selected standard filters in specified or default directory])],
 	      [],[with_plugin_dir=no])
 if test "x$have_zstd" = xyes; then
-   if test "x$with_plugin_dir" = xno; then
+   if test "x$enable_plugins" = xyes && test "x$with_plugin_dir" = xno; then
       with_plugin_dir=yes
    fi
 fi

nc_test4/CMakeLists.txt

@@ -102,6 +102,12 @@ IF(HAVE_H5Z_SZIP)
   ENDIF()
 ENDIF()
 
+IF(USE_HDF5 AND NETCDF_ENABLE_FILTER_TESTING)
+  IF(HAVE_ZSTD)
+    BUILD_BIN_TEST(tst_zstd)
+    add_sh_test(nc_test4 run_zstd_test)
+  ENDIF()
+ENDIF()
 
 # Copy some test files from current source dir to out-of-tree build dir.
 FILE(GLOB COPY_FILES ${CMAKE_CURRENT_SOURCE_DIR}/*.nc ${CMAKE_CURRENT_SOURCE_DIR}/ref_bzip2.c ${CMAKE_CURRENT_SOURCE_DIR}/*.sh ${CMAKE_CURRENT_SOURCE_DIR}/*.h5 ${CMAKE_CURRENT_SOURCE_DIR}/*.cdl)

nc_test4/Makefile.am

@@ -11,10 +11,10 @@
 include $(top_srcdir)/lib_flags.am
 
 # Un comment to use a more verbose test driver
-#SH_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
-#sh_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
-#LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
-#TESTS_ENVIRONMENT = export SETX=1;
+# SH_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
+# sh_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
+# LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
+# TESTS_ENVIRONMENT = export SETX=1;
 
 TEST_EXTENSIONS = .sh
 
@@ -113,6 +113,14 @@ endif # NETCDF_ENABLE_PLUGINS
 endif # USE_HDF5
 endif # NETCDF_BUILD_UTILITIES
 
+# Zstandard tests.
+if NETCDF_ENABLE_PLUGINS
+if HAVE_ZSTD
+check_PROGRAMS += tst_zstd
+TESTS += run_zstd_test.sh
+endif # HAVE_ZSTD
+endif # NETCDF_ENABLE_PLUGINS
+
 # This are extra tests that will only be run if netcdf-4 is configured
 # with --enable-parallel-tests.
 if TEST_PARALLEL4
@@ -129,16 +137,15 @@ ref_tst_interops4.nc CMakeLists.txt run_grp_rename.sh			\
 run_empty_vlen_test.sh ref_hdf5_compat1.nc ref_hdf5_compat2.nc		\
 ref_hdf5_compat3.nc tst_misc.sh tdset.h5 tst_szip.sh ref_szip.h5	\
 ref_szip.cdl tst_filter.sh bzip2.cdl noop.cdl ref_filtered.cdl		\
-ref_unfiltered.cdl ref_bzip2.c ref_unfilteredvv.cdl	\
-ref_filteredvv.cdl ref_multi.cdl                                        \
-ref_ncgenF.cdl ref_nccopyF.cdl                                          \
-ref_filter_repeat.txt ref_fillonly.cdl test_fillonly.sh                 \
-ref_filter_order_create.txt ref_filter_order_read.txt			\
-ref_any.cdl tst_specific_filters.sh tst_unknown.sh                      \
-tst_virtual_datasets.c noop1.cdl unknown.cdl  \
-tst_broken_files.c ref_bloscx.cdl tst_bloscfail.sh			\
-tst_fixedstring.sh ref_fixedstring.h5 ref_fixedstring.cdl               \
-tst_filterinstall.sh tst_filter_vlen.sh tst_filter_misc.sh
+ref_unfiltered.cdl ref_bzip2.c ref_unfilteredvv.cdl			\
+ref_filteredvv.cdl ref_multi.cdl ref_ncgenF.cdl ref_nccopyF.cdl		\
+ref_filter_repeat.txt ref_fillonly.cdl test_fillonly.sh			\
+ref_filter_order_create.txt ref_filter_order_read.txt ref_any.cdl	\
+tst_specific_filters.sh tst_unknown.sh tst_virtual_datasets.c		\
+noop1.cdl unknown.cdl tst_broken_files.c ref_bloscx.cdl			\
+tst_bloscfail.sh tst_fixedstring.sh ref_fixedstring.h5			\
+ref_fixedstring.cdl tst_filterinstall.sh tst_filter_vlen.sh		\
+tst_filter_misc.sh run_zstd_test.sh
 
 CLEANFILES = tst_mpi_parallel.bin cdm_sea_soundings.nc bm_chunking.nc	\
 tst_floats_1D.cdl floats_1D_3.nc floats_1D.cdl tst_*.nc tmp_*.txt       \

nc_test4/run_zstd_test.sh

@@ -0,0 +1,33 @@
+#!/bin/sh
+
+# This .in file is processed at build time into a shell that runs some
+# zstd for netCDF.
+
+# Ed Hartnett, 8/10/24
+
+set -e
+
+if test "x$srcdir" = x ; then srcdir=`pwd`; fi
+. ../test_common.sh
+
+# Load the findplugins function
+. ${builddir}/findplugin.sh
+echo "findplugin.sh loaded"
+echo "${HDF5_PLUGIN_DIR}"
+
+findplugin h5zstd
+echo
+echo "Testing Zstandard compression..."
+
+export HDF5_PLUGIN_PATH="${HDF5_PLUGIN_DIR}"
+echo "HDF5_PLUGIN_PATH=$HDF5_PLUGIN_PATH"
+ls -l "${HDF5_PLUGIN_PATH}"
+
+echo "set HDF5_PLUGIN_PATH: ${HDF5_PLUGIN_PATH}"
+
+./tst_zstd
+
+echo "SUCCESS!"
+
+
+

nc_test4/tst_zstd.c

@@ -0,0 +1,78 @@
+/* This is part of the netCDF package.
+   Copyright 2018 University Corporation for Atmospheric Research/Unidata
+   See COPYRIGHT file for conditions of use.
+
+   Test netcdf-4 variables.
+   Ed Hartnett, 8/7/24
+*/
+
+#include "nc_tests.h"
+#include "err_macros.h"
+#include "netcdf.h"
+
+#define FILE_NAME "tst_zstd.nc"
+#define VAR_FLOAT_NAME "GALE_data"
+#define VAR_DOUBLE_NAME "ERICA_data"
+#define NDIM2 2
+#define DIM1_NAME "x"
+#define DIM1_LEN 2
+#define DIM2_NAME "y"
+#define DIM2_LEN 3
+
+int
+main(int argc, char **argv)
+{
+   int ncid, dimids[NDIM2];
+   int float_varid, double_varid;
+   float float_in[DIM1_LEN][DIM2_LEN], float_out[DIM1_LEN][DIM2_LEN] = {{-.1f, 9999.99f, 100.001f},{-.1f, 9999.99f, 100.001f}};
+   double double_in[DIM1_LEN][DIM2_LEN], double_out[DIM1_LEN][DIM2_LEN] = {{0.02, .1128, 1090.1},{0.02, .1128, 1090.1}};
+   int i, j, ret;
+
+
+   printf("\n*** Testing netcdf-4 zstd compression.\n");
+
+   printf("*** testing netcdf-4 zstd with float...");
+   {
+      int nvars_in, varids_in[2];
+
+      /* Create a netcdf file with one dim and two vars. */
+      if (nc_create(FILE_NAME, NC_NETCDF4, &ncid)) ERR;
+      if (nc_def_dim(ncid, DIM1_NAME, DIM1_LEN, &dimids[0])) ERR;
+      if (nc_def_dim(ncid, DIM2_NAME, DIM2_LEN, &dimids[1])) ERR;
+      if (nc_def_var(ncid, VAR_FLOAT_NAME, NC_FLOAT, 2, dimids, &float_varid)) ERR;
+      if (nc_def_var(ncid, VAR_DOUBLE_NAME, NC_DOUBLE, 2, dimids, &double_varid)) ERR;
+      if ((ret = nc_def_var_zstandard(ncid, float_varid, 4)))
+      {
+	  printf("error: %s\n", nc_strerror(ret));
+	  ERR;
+      }
+      if (nc_def_var_zstandard(ncid, double_varid, 4)) ERR;
+      if (nc_put_var_float(ncid, float_varid, (float *)float_out)) ERR;
+      if (nc_put_var_double(ncid, double_varid, (double *)double_out)) ERR;
+      if (nc_close(ncid)) ERR;
+
+      /* Open the file and make sure nc_inq_varids yields correct
+       * result. */
+      if (nc_open(FILE_NAME, NC_NOWRITE, &ncid)) ERR;
+      if (nc_inq_varids(ncid, &nvars_in, varids_in)) ERR;
+      if (nvars_in != 2 || varids_in[0] != 0 || varids_in[1] != 1) ERR;
+      if (nc_get_var_float(ncid, float_varid, (float *)float_in)) ERR;
+      if (nc_get_var_double(ncid, double_varid, (double *)double_in)) ERR;
+      for (i = 0; i < DIM1_LEN; i++)
+      {
+          for (j = 0; j < DIM2_LEN; j++)
+          {
+              if (float_in[i][j] != float_out[i][j]) ERR;
+              if (double_in[i][j] != double_out[i][j]) ERR;
+          }
+      }
+
+
+      if (nc_close(ncid)) ERR;
+   }
+
+   SUMMARIZE_ERR;
+   SUMMARIZE_ERR;
+
+   FINAL_RESULTS;
+}

ncdump/CMakeLists.txt

@@ -250,7 +250,8 @@ endif()
     endif()
 
     set_tests_properties(ncdump_tst_nccopy3 PROPERTIES DEPENDS
-        "ncdump_tst_calendars;ncdump_run_utf8_tests;ncdump_tst_output;ncdump_tst_64bit;ncdump_run_tests;ncdump_tst_lengths")
+      "ncdump_tst_calendars;ncdump_run_utf8_tests;ncdump_tst_output;ncdump_tst_64bit;ncdump_run_tests;ncdump_tst_lengths")
+    set_tests_properties(ncdump_tst_nccopy3 PROPERTIES RUN_SERIAL TRUE)
 
   endif()
 
@@ -271,9 +272,12 @@ endif()
     add_sh_test(ncdump tst_ncgen4)
     add_sh_test(ncdump tst_netcdf4_4)
     add_sh_test(ncdump tst_nccopy4)
+    set_tests_properties(ncdump_tst_nccopy4 PROPERTIES DEPENDS
+      "ncdump_ncgen_tests;ncdump_tst_ncgen4;ncdump_tst_fillbug;ncdump_tst_netcdf4_4;ncdump_tst_h_scalar;ncdump_tst_netcdf4;ncdump_run_ncgen_tests;ncdump_tst_output;ncdump_tst_ncgen4;ncdump_sh_tst_fillbug;tst_comp;tst_comp2;tst_nans;tst_opaque_data;tst_create_files;tst_special_atts;ncdump_tst_nccopy3")
+    set_tests_properties(ncdump_tst_nccopy4 PROPERTIES RUN_SERIAL TRUE)
     add_sh_test(ncdump tst_calendars_nc4)
 
-    set_tests_properties(ncdump_tst_nccopy4 PROPERTIES DEPENDS "ncdump_run_ncgen_tests;ncdump_tst_output;ncdump_tst_ncgen4;ncdump_sh_tst_fillbug;ncdump_tst_netcdf4_4;ncdump_tst_h_scalar;tst_comp;tst_comp2;tst_nans;tst_opaque_data;tst_create_files;tst_special_atts")
+    set_tests_properties(ncdump_tst_nccopy4 PROPERTIES DEPENDS "")
     set_tests_properties(ncdump_tst_nccopy5 PROPERTIES DEPENDS "ncdump_tst_nccopy4")
 
   endif(USE_HDF5)

plugins/CMakeLists.txt

@@ -11,7 +11,7 @@ if(WIN32)
   set(PLUGINPRE "__nc")
 elseif(CYGWIN)
   set(PLUGINEXT "dll")
-  set(PLUGINPRE "lib__nc")
+  set(PLUGINPRE "cyg__nc")
 else()
    set(PLUGINPRE "lib__nc")
    if(APPLE)