diff --git a/.github/workflows/run_tests_osx.yml b/.github/workflows/run_tests_osx.yml index 8e9d65a6fb..aa35ae8795 100644 --- a/.github/workflows/run_tests_osx.yml +++ b/.github/workflows/run_tests_osx.yml @@ -7,7 +7,7 @@ name: Run macOS-based netCDF Tests -on: [pull_request] +on: [pull_request,push] jobs: diff --git a/.github/workflows/run_tests_ubuntu.yml b/.github/workflows/run_tests_ubuntu.yml index eb410325a7..1ede4ad90c 100644 --- a/.github/workflows/run_tests_ubuntu.yml +++ b/.github/workflows/run_tests_ubuntu.yml @@ -4,7 +4,7 @@ name: Run Ubuntu/Linux netCDF Tests -on: [pull_request] +on: [pull_request,push] jobs: diff --git a/.github/workflows/run_tests_win_mingw.yml b/.github/workflows/run_tests_win_mingw.yml index 919c50f235..aaf999e6a4 100644 --- a/.github/workflows/run_tests_win_mingw.yml +++ b/.github/workflows/run_tests_win_mingw.yml @@ -7,7 +7,7 @@ name: Run MSYS2, MinGW64-based Tests -on: [ pull_request ] +on: [pull_request,push] jobs: diff --git a/CMakeLists.txt b/CMakeLists.txt index b90a7ceb61..fe76cfb5ca 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -55,6 +55,21 @@ IF(UNAME) set(TMP_BUILDNAME "${osname}-${osrel}-${cpu}") ENDIF() +# Define some Platforms +if(osname MATCHES "CYGWIN.*") +SET(ISCYGWIN yes) +endif() +if(osname MATCHES "Darwin.*") +SET(ISOSX yes) +endif() +if(MSVC) +SET(ISMSVC yes) +endif() +if(osname MATCHES "MINGW.*") +SET(ISMINGW yes) +SET(MINGW yes) +endif() + ### # Allow for some customization of the buildname. # This will make it easier to identify different builds, @@ -86,6 +101,7 @@ INCLUDE(GNUInstallDirs) IF(MSVC) SET(GLOBAL PROPERTY USE_FOLDERS ON) + ADD_COMPILE_OPTIONS("/utf-8") ENDIF() #Add custom CMake Module @@ -412,7 +428,6 @@ IF(NOT MSVC) ENDIF(BUILD_FORTRAN) ENDIF() - ### # Allow the user to specify libraries # to link against, similar to automakes 'LIBS' variable. @@ -1794,6 +1809,26 @@ IF(ENABLE_MMAP) ENDIF(ENABLE_MMAP) #CHECK_FUNCTION_EXISTS(alloca HAVE_ALLOCA) + +# Used in the `configure_file` calls below +SET(ISCMAKE "1") +IF(MSVC) +SET(ISMSVC ON CACHE BOOL "" FORCE) +SET(REGEDIT ON CACHE BOOL "" FORCE) +# Get windows major version and build number +EXECUTE_PROCESS(COMMAND "systeminfo" OUTPUT_VARIABLE WININFO) +IF(WININFO STREQUAL "") + SET(WVM 0) + SET(WVB 0) +ELSE() + STRING(REGEX MATCH "\nOS Version:[ \t]+[0-9.]+" WINVERLINE "${WININFO}") + STRING(REGEX REPLACE "[^0-9]*([0-9]+)[.]([0-9])+[.]([0-9]+)" "\\1" WVM "${WINVERLINE}") + STRING(REGEX REPLACE "[^0-9]*([0-9]+)[.]([0-9])+[.]([0-9]+)" "\\3" WVB "${WINVERLINE}") +ENDIF() +SET(WINVERMAJOR ${WVM} CACHE STRING "" FORCE) +SET(WINVERBUILD ${WVB} CACHE STRING "" FORCE) +ENDIF() + ##### # End system inspection checks. ##### @@ -2418,13 +2453,6 @@ configure_file( ${netCDF_SOURCE_DIR}/include/netcdf_dispatch.h.in ${netCDF_BINARY_DIR}/include/netcdf_dispatch.h @ONLY NEWLINE_STYLE LF) -# Used in the `configure_file` calls below -SET(ISCMAKE "1") -IF(MSVC) -SET(ISMSVC "1") -SET(REGEDIT 1) -ENDIF() - #### # Build test_common.sh ##### diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index e64d4f4517..6a19f7029f 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -7,8 +7,9 @@ This file contains a high-level description of this package's evolution. Release ## 4.8.2 - TBD +* [Bug Fix] Improve UTF8 support on windows so that it can use utf8 natively. See [Github #2222](https://github.com/Unidata/netcdf-c/pull/2222). * [Enhancement] Add complete bitgroom support to NCZarr. See [Github #2197](https://github.com/Unidata/netcdf-c/pull/2197). -* [Bug Fix] Clean up the handling of deeply nested VLEN types. Marks nc_free_vlen() and nc_free_string as deprecated in favor of ncaux_reclaim_data(). See [Github #2179(https://github.com/Unidata/netcdf-c/pull/2179). +* [Bug Fix] Clean up the handling of deeply nested VLEN types. Marks nc_free_vlen() and nc_free_string as deprecated in favor of ncaux_reclaim_data(). See [Github #2179](https://github.com/Unidata/netcdf-c/pull/2179). * [Bug Fix] Make sure that netcdf.h accurately defines the flags in the open/create mode flags. See [Github #2183](https://github.com/Unidata/netcdf-c/pull/2183). * [Enhancement] Improve support for msys2+mingw platform. See [Github #2171](https://github.com/Unidata/netcdf-c/pull/2171). * [Bug Fix] Clean up the various inter-test dependencies in ncdump for CMake. See [Github #2168](https://github.com/Unidata/netcdf-c/pull/2168). diff --git a/config.h.cmake.in b/config.h.cmake.in index 87f473e1f3..693eab6128 100644 --- a/config.h.cmake.in +++ b/config.h.cmake.in @@ -157,12 +157,6 @@ are set when opening a binary file on Windows. */ /* if true, Allow dynamically loaded plugins */ #cmakedefine ENABLE_PLUGINS 1 -/* Do we have access to the Windows Registry */ -#cmakedefine REGEDIT 1 - -/* define the possible sources for remote test servers */ -#cmakedefine REMOTETESTSERVERS "${REMOTETESTSERVERS}" - /* if true, run extra tests which may not work yet */ #cmakedefine EXTRA_TESTS 1 @@ -505,6 +499,12 @@ with zip */ /* Define to the version of this package. */ #cmakedefine PACKAGE_VERSION "${netCDF_VERSION}" +/* Do we have access to the Windows Registry */ +#cmakedefine REGEDIT 1 + +/* define the possible sources for remote test servers */ +#cmakedefine REMOTETESTSERVERS "${REMOTETESTSERVERS}" + /* The size of `ulonglong` as computed by sizeof. */ #cmakedefine SIZEOF_ULONGLONG @SIZEOF_ULONGLONG@ @@ -626,6 +626,10 @@ with zip */ /* Version number of package */ #cmakedefine VERSION "${netCDF_VERSION}" +/* Capture Windows version and build */ +#cmakedefine WINVERMAJOR ${WINVERMAJOR} +#cmakedefine WINVERBUILD ${WINVERBUILD} + /* Define WORDS_BIGENDIAN to 1 if your processor stores words with the most significant byte first (like Motorola and SPARC, unlike Intel). */ #if defined AC_APPLE_UNIVERSAL_BUILD diff --git a/configure.ac b/configure.ac index 929d12f055..844bfff096 100644 --- a/configure.ac +++ b/configure.ac @@ -106,6 +106,15 @@ ISMINGW=yes ISMSYS=yes fi +# Get windows version info +WINVER=`systeminfo | sed -e '/^OS Version:/p' -ed | sed -e 's|[^0-9]*\([0-9.]*\).*|\1|'` +WINVERMAJOR=`echo $WVER | sed -e 's|\([^.]*\)[.]\([^.]*\)[.]\(.*\)|\1|'` +WINVERBUILD=`echo $WVER | sed -e 's|\([^.]*\)[.]\([^.]*\)[.]\(.*\)|\3|'` +if test "x$WINVERMAJOR" = x ; then WINVERMAJOR=0; fi +if test "x$WINVERBUILD" = x ; then WINVERBUILD=0; fi +AC_DEFINE_UNQUOTED([WINVERMAJOR], [$WINVERMAJOR], [windows version major]) +AC_DEFINE_UNQUOTED([WINVERBUILD], [$WINVERBUILD], [windows version build]) + AC_MSG_NOTICE([checking supported formats]) # An explicit disable of netcdf-4 | netcdf4 is treated as if it was disable-hdf5 @@ -1218,6 +1227,8 @@ AM_CONDITIONAL(ISMINGW, [test "x$ISMINGW" = xyes]) AM_CONDITIONAL(ISMSYS, [test "x$ISMSYS" = xyes]) AC_SUBST([ISMSVC], [${ISMSVC}]) + AC_SUBST([WINVERMAJOR], [${WINVERMAJOR}]) + AC_SUBST([WINVERBUILD], [${WINVERBUILD}]) AC_SUBST([ISCYGWIN], [${ISCYGWIN}]) AC_SUBST([ISOSX], [${ISOSX}]) AC_SUBST([ISMINGW], [${ISMINGW}]) diff --git a/dap4_test/test_common.h b/dap4_test/test_common.h index b2f4805ad0..70b1c22319 100644 --- a/dap4_test/test_common.h +++ b/dap4_test/test_common.h @@ -26,26 +26,6 @@ static char* outfile = NULL; static int ncid = 0; static int translatenc4 = 0; -static int -readfile(const char* filename, NCbytes* content) -{ - FILE* stream; - char part[1024]; - - stream = fopen(filename,"r"); - if(stream == NULL) return errno; - for(;;) { - size_t count = fread(part, 1, sizeof(part), stream); - if(count <= 0) break; - ncbytesappendn(content,part,count); - if(ferror(stream)) {fclose(stream); return NC_EIO;} - if(feof(stream)) break; - } - ncbytesnull(content); - fclose(stream); - return NC_NOERR; -} - static void fail(int code) { @@ -86,7 +66,7 @@ setup(int tdmr, int argc, char** argv) outfile = NULL; input = ncbytesnew(); output = ncbytesnew(); - if((ret = readfile(infile,input))) fail(ret); + if((ret = NC_readfile(infile,input))) fail(ret); { const char* trans = getenv("translatenc4"); if(trans != NULL) diff --git a/dap4_test/test_data.c b/dap4_test/test_data.c index 946c37b2ab..ab9f8f3a8d 100644 --- a/dap4_test/test_data.c +++ b/dap4_test/test_data.c @@ -12,7 +12,7 @@ Test the netcdf-4 data building process. #include #include "netcdf.h" -#define DEBUG +#undef DEBUG static void fail(int code) diff --git a/include/nc_tests.h b/include/nc_tests.h index 35e2ae3504..79cc9094c0 100644 --- a/include/nc_tests.h +++ b/include/nc_tests.h @@ -20,6 +20,8 @@ #include "netcdf.h" #include "netcdf_filter.h" #include "nc_logging.h" +#include "ncpathmgr.h" +#include "ncrc.h" #ifdef USE_PARALLEL #include "netcdf_par.h" #endif diff --git a/include/ncrc.h b/include/ncrc.h index 1b3228674b..342b13d024 100644 --- a/include/ncrc.h +++ b/include/ncrc.h @@ -103,6 +103,8 @@ EXTERNL int NC__testurl(const char* path, char** basenamep); EXTERNL int NC_isLittleEndian(void); EXTERNL char* NC_entityescape(const char* s); EXTERNL int NC_readfile(const char* filename, NCbytes* content); +EXTERNL int NC_readfilen(const char* filename, NCbytes* content, long long len); +EXTERNL int NC_readfileF(FILE* fp, NCbytes* content, long long len); EXTERNL int NC_writefile(const char* filename, size_t size, void* content); EXTERNL char* NC_mktmp(const char* base); EXTERNL int NC_getmodelist(const char* modestr, NClist** modelistp); diff --git a/libdap2/dceconstraints.c b/libdap2/dceconstraints.c index 800e5eb089..b4895768f8 100644 --- a/libdap2/dceconstraints.c +++ b/libdap2/dceconstraints.c @@ -13,7 +13,7 @@ #include "dapincludes.h" #include "dceparselex.h" -#define DEBUG +#undef DEBUG #define LBRACE "{" #define RBRACE "}" diff --git a/libdap4/d4file.c b/libdap4/d4file.c index 1c94040104..d0e41db059 100644 --- a/libdap4/d4file.c +++ b/libdap4/d4file.c @@ -395,17 +395,17 @@ set_curl_properties(NCD4INFO* d4info) FILE* f = NULL; char* fname = d4info->auth->curlflags.cookiejar; /* See if the file exists already */ - f = fopen(fname,"r"); + f = NCfopen(fname,"r"); if(f == NULL) { /* Ok, create it */ - f = fopen(fname,"w+"); + f = NCfopen(fname,"w+"); if(f == NULL) { fprintf(stderr,"Cookie file cannot be read and written: %s\n",fname); {ret= NC_EPERM; goto fail;} } } else { /* test if file can be written */ fclose(f); - f = fopen(fname,"r+"); + f = NCfopen(fname,"r+"); if(f == NULL) { fprintf(stderr,"Cookie file is cannot be written: %s\n",fname); {ret = NC_EPERM; goto fail;} diff --git a/libdap4/d4includes.h b/libdap4/d4includes.h index cc8b191c69..04151a683a 100644 --- a/libdap4/d4includes.h +++ b/libdap4/d4includes.h @@ -42,6 +42,7 @@ #include "ncuri.h" #include "nclog.h" #include "ncdap.h" +#include "ncpathmgr.h" #include "d4util.h" diff --git a/libdap4/d4util.c b/libdap4/d4util.c index a68041ecd2..b84d67877d 100644 --- a/libdap4/d4util.c +++ b/libdap4/d4util.c @@ -346,73 +346,6 @@ NCD4_elidenuls(char* s, size_t slen) return j; } -int -NCD4_readfile(const char* filename, NCbytes* content) -{ - int ret = NC_NOERR; - FILE* stream = NULL; - char part[1024]; - - stream = fopen(filename,"r"); - if(stream == NULL) {ret=errno; goto done;} - for(;;) { - size_t count = fread(part, 1, sizeof(part), stream); - if(count <= 0) break; - ncbytesappendn(content,part,count); - if(ferror(stream)) {ret = NC_EIO; goto done;} - if(feof(stream)) break; - } - ncbytesnull(content); -done: - if(stream) fclose(stream); - return ret; -} - -/** -Wrap mktmp and return the generated name -*/ - -int -NCD4_mktmp(const char* base, char** tmpnamep) -{ - int fd; - char tmp[NC_MAX_PATH]; -#ifdef HAVE_MKSTEMP - mode_t mask; -#endif - - strncpy(tmp,base,sizeof(tmp)); -#ifdef HAVE_MKSTEMP - strlcat(tmp,"XXXXXX", sizeof(tmp)); - /* Note Potential problem: old versions of this function - leave the file in mode 0666 instead of 0600 */ - mask=umask(0077); - fd = mkstemp(tmp); - (void)umask(mask); -#else /* !HAVE_MKSTEMP */ - /* Need to simulate by using some kind of pseudo-random number */ - { - int rno = rand(); - char spid[7]; - if(rno < 0) rno = -rno; - snprintf(spid,sizeof(spid),"%06d",rno); - strlcat(tmp,spid,sizeof(tmp)); -#if defined(_WIN32) || defined(_WIN64) - fd=open(tmp,O_RDWR|O_BINARY|O_CREAT, _S_IREAD|_S_IWRITE); -# else - fd=open(tmp,O_RDWR|O_CREAT|O_EXCL, S_IRWXU); -# endif - } -#endif /* !HAVE_MKSTEMP */ - if(fd < 0) { - nclog(NCLOGERR, "Could not create temp file: %s",tmp); - return THROW(NC_EPERM); - } else - close(fd); - if(tmpnamep) *tmpnamep = strdup(tmp); - return THROW(NC_NOERR); -} - void NCD4_hostport(NCURI* uri, char* space, size_t len) { diff --git a/libdap4/ncd4.h b/libdap4/ncd4.h index fe13d7f520..db9f74e3a5 100644 --- a/libdap4/ncd4.h +++ b/libdap4/ncd4.h @@ -64,114 +64,114 @@ defined here, including function-like #defines. /* DSP API wrappers */ #ifdef FIX -extern int dsp_getDMR(ND4dsp* dsp, DCR** dcrp); -extern int dsp_getDAP(ND4dsp* dsp, DCR** dcrp); -extern int dsp_close(ND4dsp* dsp); +EXTERNL int dsp_getDMR(ND4dsp* dsp, DCR** dcrp); +EXTERNL int dsp_getDAP(ND4dsp* dsp, DCR** dcrp); +EXTERNL int dsp_close(ND4dsp* dsp); /* DSP API */ -extern int dsp_open(const char* path, ND4dsp** dspp); +EXTERNL int dsp_open(const char* path, ND4dsp** dspp); #endif /**************************************************/ /* From d4http.c */ -extern long NCD4_fetchhttpcode(CURL* curl); -extern int NCD4_fetchurl_file(CURL* curl, const char* url, FILE* stream, d4size_t* sizep, long* filetime); -extern int NCD4_fetchurl(CURL* curl, const char* url, NCbytes* buf, long* filetime, int* httpcode); -extern int NCD4_curlopen(CURL** curlp); -extern void NCD4_curlclose(CURL* curl); -extern int NCD4_fetchlastmodified(CURL* curl, char* url, long* filetime); -extern int NCD4_ping(const char* url); +EXTERNL long NCD4_fetchhttpcode(CURL* curl); +EXTERNL int NCD4_fetchurl_file(CURL* curl, const char* url, FILE* stream, d4size_t* sizep, long* filetime); +EXTERNL int NCD4_fetchurl(CURL* curl, const char* url, NCbytes* buf, long* filetime, int* httpcode); +EXTERNL int NCD4_curlopen(CURL** curlp); +EXTERNL void NCD4_curlclose(CURL* curl); +EXTERNL int NCD4_fetchlastmodified(CURL* curl, char* url, long* filetime); +EXTERNL int NCD4_ping(const char* url); /* From d4read.c */ -extern int NCD4_readDMR(NCD4INFO* state, int flags); -extern int NCD4_readDAP(NCD4INFO* state, int flags); -extern int NCD4_seterrormessage(NCD4meta* metadata, size_t len, char* msg); +EXTERNL int NCD4_readDMR(NCD4INFO* state, int flags); +EXTERNL int NCD4_readDAP(NCD4INFO* state, int flags); +EXTERNL int NCD4_seterrormessage(NCD4meta* metadata, size_t len, char* msg); /* From d4parser.c */ -extern int NCD4_parse(NCD4meta*); -extern NCD4node* NCD4_findAttr(NCD4node* container, const char* attrname); -extern NCD4node* NCD4_groupFor(NCD4node* node); -extern int NCD4_defineattr(NCD4meta* meta, NCD4node* parent, const char* aname, const char* typename, NCD4node** attrp); +EXTERNL int NCD4_parse(NCD4meta*); +EXTERNL NCD4node* NCD4_findAttr(NCD4node* container, const char* attrname); +EXTERNL NCD4node* NCD4_groupFor(NCD4node* node); +EXTERNL int NCD4_defineattr(NCD4meta* meta, NCD4node* parent, const char* aname, const char* typename, NCD4node** attrp); /* From d4printer.c */ -extern int NCD4_print(NCD4meta*, NCbytes* output); +EXTERNL int NCD4_print(NCD4meta*, NCbytes* output); /* From d4meta.c */ -extern NCD4meta* NCD4_newmeta(NCD4INFO*); -extern void NCD4_attachraw(NCD4meta*, size_t size, void* rawdata); -extern void NCD4_reclaimMeta(NCD4meta*); -extern void NCD4_resetMeta(NCD4meta*); -extern void reclaimNode(NCD4node* node); -extern void NCD4_setdebuglevel(NCD4meta*,int); -extern int NCD4_metabuild(NCD4meta*, int ncid); -extern size_t NCD4_computeTypeSize(NCD4meta*, NCD4node* type); -extern int NCD4_findvar(NC* ncp, int ncid, int varid, NCD4node** varp, NCD4node** grpp); +EXTERNL NCD4meta* NCD4_newmeta(NCD4INFO*); +EXTERNL void NCD4_attachraw(NCD4meta*, size_t size, void* rawdata); +EXTERNL void NCD4_reclaimMeta(NCD4meta*); +EXTERNL void NCD4_resetMeta(NCD4meta*); +EXTERNL void reclaimNode(NCD4node* node); +EXTERNL void NCD4_setdebuglevel(NCD4meta*,int); +EXTERNL int NCD4_metabuild(NCD4meta*, int ncid); +EXTERNL size_t NCD4_computeTypeSize(NCD4meta*, NCD4node* type); +EXTERNL int NCD4_findvar(NC* ncp, int ncid, int varid, NCD4node** varp, NCD4node** grpp); /* From d4chunk.c */ -extern int NCD4_dechunk(NCD4meta*); -extern int NCD4_infermode(NCD4meta* meta); +EXTERNL int NCD4_dechunk(NCD4meta*); +EXTERNL int NCD4_infermode(NCD4meta* meta); struct NCD4serial; -extern void NCD4_resetSerial(struct NCD4serial* serial, size_t rawsize, void* rawdata); +EXTERNL void NCD4_resetSerial(struct NCD4serial* serial, size_t rawsize, void* rawdata); /* From d4swap.c */ -extern int NCD4_swapdata(NCD4meta*, NClist* topvars); +EXTERNL int NCD4_swapdata(NCD4meta*, NClist* topvars); /* From d4fix.c */ -extern int NCD4_delimit(NCD4meta*, NCD4node* var, void** offsetp); -extern int NCD4_moveto(NCD4meta*, NCD4node* var, d4size_t count, void** offsetp); -extern int NCD4_toposort(NCD4meta*); +EXTERNL int NCD4_delimit(NCD4meta*, NCD4node* var, void** offsetp); +EXTERNL int NCD4_moveto(NCD4meta*, NCD4node* var, d4size_t count, void** offsetp); +EXTERNL int NCD4_toposort(NCD4meta*); /* From d4data.c */ -extern int NCD4_processdata(NCD4meta*); -extern int NCD4_fillinstance(NCD4meta*, NCD4node* type, void** offsetp, void** dstp, NClist* blobs); -extern int NCD4_getToplevelVars(NCD4meta* meta, NCD4node* group, NClist* toplevel); +EXTERNL int NCD4_processdata(NCD4meta*); +EXTERNL int NCD4_fillinstance(NCD4meta*, NCD4node* type, void** offsetp, void** dstp, NClist* blobs); +EXTERNL int NCD4_getToplevelVars(NCD4meta* meta, NCD4node* group, NClist* toplevel); /* From d4util.c */ -extern d4size_t NCD4_dimproduct(NCD4node* node); -extern size_t NCD4_typesize(nc_type tid); -extern int NCD4_isLittleEndian(void);/* Return 1 if this machine is little endian */ -extern int NCD4_errorNC(int code, const int line, const char* file); -extern int NCD4_error(int code, const int line, const char* file, const char* fmt, ...); -extern char* NCD4_makeFQN(NCD4node* node); -extern char* NCD4_makeName(NCD4node*,const char* sep); -extern int NCD4_parseFQN(const char* fqn0, NClist* pieces); -extern char* NCD4_deescape(const char* esc); -extern char* NCD4_entityescape(const char* s); -extern size_t NCD4_elidenuls(char* s, size_t slen); -extern void* NCD4_getheader(void* p, NCD4HDR* hdr, int hostlittleendian); -extern void NCD4_reporterror(NCD4INFO* state); +EXTERNL d4size_t NCD4_dimproduct(NCD4node* node); +EXTERNL size_t NCD4_typesize(nc_type tid); +EXTERNL int NCD4_isLittleEndian(void);/* Return 1 if this machine is little endian */ +EXTERNL int NCD4_errorNC(int code, const int line, const char* file); +EXTERNL int NCD4_error(int code, const int line, const char* file, const char* fmt, ...); +EXTERNL char* NCD4_makeFQN(NCD4node* node); +EXTERNL char* NCD4_makeName(NCD4node*,const char* sep); +EXTERNL int NCD4_parseFQN(const char* fqn0, NClist* pieces); +EXTERNL char* NCD4_deescape(const char* esc); +EXTERNL char* NCD4_entityescape(const char* s); +EXTERNL size_t NCD4_elidenuls(char* s, size_t slen); +EXTERNL void* NCD4_getheader(void* p, NCD4HDR* hdr, int hostlittleendian); +EXTERNL void NCD4_reporterror(NCD4INFO* state); /* From d4dump.c */ -extern void NCD4_dumpbytes(size_t size, const void* data0, int swap); -extern void NCD4_tagdump(size_t size, const void* data0, int swap, const char* tag); -extern void NCD4_dumpvars(NCD4node* group); -extern union ATOMICS* NCD4_dumpatomic(NCD4node* var, void* data); +EXTERNL void NCD4_dumpbytes(size_t size, const void* data0, int swap); +EXTERNL void NCD4_tagdump(size_t size, const void* data0, int swap, const char* tag); +EXTERNL void NCD4_dumpvars(NCD4node* group); +EXTERNL union ATOMICS* NCD4_dumpatomic(NCD4node* var, void* data); /* From d4rc.c */ -extern int NCD4_rcload(void); -extern int NCD4_rcprocess(NCD4INFO* info); -extern void NCD4_rcfree(NClist* rc); -extern char* NCD4_rclookup(char* key, char* hostport); -extern int NCD4_parseproxy(NCD4INFO* info, const char* surl); -extern int NCD4_rcdefault(NCD4INFO*); +EXTERNL int NCD4_rcload(void); +EXTERNL int NCD4_rcprocess(NCD4INFO* info); +EXTERNL void NCD4_rcfree(NClist* rc); +EXTERNL char* NCD4_rclookup(char* key, char* hostport); +EXTERNL int NCD4_parseproxy(NCD4INFO* info, const char* surl); +EXTERNL int NCD4_rcdefault(NCD4INFO*); /* From d4cvt.c */ -extern int NCD4_convert(nc_type srctype, nc_type dsttype, char* memory0, char* value0, size_t count); +EXTERNL int NCD4_convert(nc_type srctype, nc_type dsttype, char* memory0, char* value0, size_t count); /* d4file.c */ -extern void NCD4_applyclientparamcontrols(NCD4INFO*); -extern int NCD4_readDMRorDAP(NCD4INFO* d4info, NCD4mode mode); +EXTERNL void NCD4_applyclientparamcontrols(NCD4INFO*); +EXTERNL int NCD4_readDMRorDAP(NCD4INFO* d4info, NCD4mode mode); /* ncd4dispatch.c */ struct NC_reservedatt; /*forward*/ -extern const struct NC_reservedatt* NCD4_lookupreserved(const char* name); +EXTERNL const struct NC_reservedatt* NCD4_lookupreserved(const char* name); /* Add an extra function whose sole purpose is to allow configure(.ac) to test for the presence of this code. */ -extern int nc__dap4(void); +EXTERNL int nc__dap4(void); /**************************************************/ /* Macro defined functions */ diff --git a/libdispatch/dcrc32.c b/libdispatch/dcrc32.c index 3878841e88..d804774e3d 100644 --- a/libdispatch/dcrc32.c +++ b/libdispatch/dcrc32.c @@ -199,7 +199,7 @@ local void make_crc_table() { FILE *out; - out = fopen("dcrc32.h", "w"); + out = NCfopen("dcrc32.h", "w"); if (out == NULL) return; fprintf(out, "/* dcrc32.h -- tables for rapid CRC calculation\n"); fprintf(out, " * Generated automatically by dcrc32.c\n */\n\n"); diff --git a/libdispatch/dinfermodel.c b/libdispatch/dinfermodel.c index fb58d18c23..d406908a88 100644 --- a/libdispatch/dinfermodel.c +++ b/libdispatch/dinfermodel.c @@ -1263,9 +1263,8 @@ openmagic(struct MagicFile* file) } else #endif /* USE_PARALLEL */ { - if(file->path == NULL || strlen(file->path)==0) - {status = NC_EINVAL; goto done;} - + if (file->path == NULL || strlen(file->path) == 0) + {status = NC_EINVAL; goto done;} file->fp = NCfopen(file->path, "r"); if(file->fp == NULL) {status = errno; goto done;} @@ -1296,6 +1295,8 @@ static int readmagic(struct MagicFile* file, long pos, char* magic) { int status = NC_NOERR; + NCbytes* buf = ncbytesnew(); + memset(magic,0,MAGIC_NUMBER_LEN); if(fIsSet(file->omode,NC_INMEMORY)) { char* mempos; @@ -1315,19 +1316,18 @@ readmagic(struct MagicFile* file, long pos, char* magic) if(file->iss3) { if((status = NC_s3sdkread(file->s3client,file->s3.bucket,file->s3.rootkey,start,count,(void*)magic,&file->errmsg))) {goto done;} - } else + } + else #endif - { - NCbytes* buf = ncbytesnew(); - status = nc_http_read(file->state,file->curlurl,start,count,buf); - if(status == NC_NOERR) { - if(ncbyteslength(buf) != count) - status = NC_EINVAL; - else - memcpy(magic,ncbytescontents(buf),count); - } - ncbytesfree(buf); - } + { + status = nc_http_read(file->state, file->curlurl, start, count, buf); + if (status == NC_NOERR) { + if (ncbyteslength(buf) != count) + status = NC_EINVAL; + else + memcpy(magic, ncbytescontents(buf), count); + } + } #endif } else { #ifdef USE_PARALLEL @@ -1337,23 +1337,21 @@ readmagic(struct MagicFile* file, long pos, char* magic) if((retval = MPI_File_read_at_all(file->fh, pos, magic, MAGIC_NUMBER_LEN, MPI_CHAR, &mstatus)) != MPI_SUCCESS) {status = NC_EPARINIT; goto done;} - } else + } + else #endif /* USE_PARALLEL */ - { - int count; - int i = fseek(file->fp,pos,SEEK_SET); - if(i < 0) - {status = errno; goto done;} - for(i=0;ifp); - if(count == 0 || ferror(file->fp)) - {status = errno; goto done;} - i += count; - } - } + { /* Ordinary read */ + long i; + i = fseek(file->fp, pos, SEEK_SET); + if (i < 0) { status = errno; goto done; } + ncbytessetlength(buf, 0); + if ((status = NC_readfileF(file->fp, buf, MAGIC_NUMBER_LEN))) goto done; + memcpy(magic, ncbytescontents(buf), MAGIC_NUMBER_LEN); + } } done: + ncbytesfree(buf); if(file && file->fp) clearerr(file->fp); return check(status); } diff --git a/libdispatch/dinstance.c b/libdispatch/dinstance.c index 5a95d3cbc9..64756929bd 100644 --- a/libdispatch/dinstance.c +++ b/libdispatch/dinstance.c @@ -153,14 +153,15 @@ reclaim_datar(int ncid, nc_type xtype, Position* offset) int klass, isf; if((stat = NC4_inq_type_fixed_size(ncid,xtype,&isf))) goto done; + + /* Get relevant type info */ + if((stat = NC_inq_any_type(ncid,xtype,NULL,&xsize,&basetype,&nfields,&klass))) goto done; + if(isf) { /* no need to reclaim anything */ offset->offset += xsize; goto done; } - /* Get relevant type info */ - if((stat = NC_inq_any_type(ncid,xtype,NULL,&xsize,&basetype,&nfields,&klass))) goto done; - switch (xtype) { case NC_STRING: { char** sp = (char**)(offset->memory + offset->offset); diff --git a/libdispatch/dpathmgr.c b/libdispatch/dpathmgr.c index c95273892d..2ced44ae2e 100644 --- a/libdispatch/dpathmgr.c +++ b/libdispatch/dpathmgr.c @@ -25,13 +25,9 @@ #include #include #include -#include #include #endif -#ifdef __hpux #include -#endif - #include "netcdf.h" #include "ncpathmgr.h" #include "nclog.h" @@ -42,7 +38,7 @@ #undef DEBUGPATH static int pathdebug = -1; -#define DEBUG +#undef DEBUG #ifdef DEBUG #define REPORT(e,msg) report((e),(msg),__LINE__) @@ -55,6 +51,14 @@ static int pathdebug = -1; #define mkdir _mkdir #define rmdir _rmdir #define getcwd _getcwd + +#if WINVERMAJOR > 10 || (WINVERMAJOR == 10 && WINVERBUILD >= 17134) +/* Should be possible to use UTF8 directly */ +#define WINUTF8 +#else +#undef WINUTF8 +#endif + #endif /* @@ -87,6 +91,11 @@ static const struct Path { /* Keep the working directory kind and drive */ static char wdprefix[8192]; +/* The current codepage */ +#ifdef _WIN32 +static int acp = -1; +#endif + /* Keep CYGWIN/MSYS2 mount point */ static struct MountPoint { int defined; @@ -265,6 +274,7 @@ static void pathinit(void) { if(pathinitialized) return; + pathinitialized = 1; /* avoid recursion */ /* Check for path debug env vars */ if(pathdebug < 0) { @@ -272,6 +282,12 @@ pathinit(void) pathdebug = (s == NULL ? 0 : 1); } (void)getwdpath(); + +#ifdef _WIN32 + /* Get the current code page */ + acp = GetACP(); +#endif + memset(&mountpoint,0,sizeof(mountpoint)); #ifdef REGEDIT { /* See if we can get the MSYS2 prefix from the registry */ @@ -367,31 +383,44 @@ NCfopen(const char* path, const char* flags) { int stat = NC_NOERR; FILE* f = NULL; - char* bflags = NULL; + char bflags[64]; + char* path8 = NULL; /* ACP -> UTF=8 */ + char* bflags8 = NULL; char* cvtpath = NULL; wchar_t* wpath = NULL; wchar_t* wflags = NULL; size_t flaglen = strlen(flags)+1+1; - bflags = (char*)malloc(flaglen); bflags[0] = '\0'; - strlcat(bflags,flags,flaglen); + strlcat(bflags, flags, sizeof(bflags)); #ifdef _WIN32 - strlcat(bflags,"b",flaglen); + strlcat(bflags,"b",sizeof(bflags)); +#endif + + /* First, convert from current Code Page to utf8 */ + if((stat = ansi2utf8(path,&path8))) goto done; + if((stat = ansi2utf8(bflags,&bflags8))) goto done; + + /* Localize */ + if((cvtpath = NCpathcvt(path8))==NULL) goto done; + +#ifdef WINUTF8 + if(acp == CP_UTF8) { + /* This should take utf8 directly */ + f = fopen(cvtpath,bflags8); + } else #endif - cvtpath = NCpathcvt(path); - if(cvtpath == NULL) return NULL; - /* Convert from local to wide */ - if((stat = utf82wide(cvtpath,&wpath))) - {REPORT(stat,"utf282wide"); goto done;} - if((stat = ansi2wide(bflags,&wflags))) - {REPORT(stat,"ansi2wide"); goto done;} - f = _wfopen(wpath,wflags); + { + /* Convert from utf8 to wide */ + if((stat = utf82wide(cvtpath,&wpath))) goto done; + if((stat = utf82wide(bflags8,&wflags))) goto done; + f = _wfopen(wpath,wflags); + } done: nullfree(cvtpath); + nullfree(path8); nullfree(wpath); nullfree(wflags); - nullfree(bflags); return f; } @@ -402,17 +431,32 @@ NCopen3(const char* path, int flags, int mode) int stat = NC_NOERR; int fd = -1; char* cvtpath = NULL; + char* path8 = NULL; wchar_t* wpath = NULL; - cvtpath = NCpathcvt(path); - if(cvtpath == NULL) goto done; - /* Convert from utf8 to wide */ - if((stat = utf82wide(cvtpath,&wpath))) goto done; + + /* First, convert from current Code Page to utf8 */ + if((stat = ansi2utf8(path,&path8))) goto done; + + if((cvtpath = NCpathcvt(path8))==NULL) goto done; + #ifdef _WIN32 flags |= O_BINARY; #endif - fd = _wopen(wpath,flags,mode); +#ifdef WINUTF8 + if(acp == CP_UTF8) { + /* This should take utf8 directly */ + fd = _open(cvtpath,flags,mode); + } else +#endif + { + /* Convert from utf8 to wide */ + if((stat = utf82wide(cvtpath,&wpath))) goto done; + fd = _wopen(wpath,flags,mode); + } + done: nullfree(cvtpath); + nullfree(path8); nullfree(wpath); return fd; } @@ -433,7 +477,7 @@ NCopendir(const char* path) char* cvtname = NCpathcvt(path); if(cvtname == NULL) return NULL; ent = opendir(cvtname); - free(cvtname); + nullfree(cvtname); return ent; } @@ -456,17 +500,32 @@ EXTERNL int NCaccess(const char* path, int mode) { - int status = 0; + int stat = 0; char* cvtpath = NULL; + char* path8 = NULL; wchar_t* wpath = NULL; - if((cvtpath = NCpathcvt(path)) == NULL) - {status = EINVAL; goto done;} - if((status = utf82wide(cvtpath,&wpath))) {status = ENOENT; goto done;} - if(_waccess(wpath,mode) < 0) {status = errno; goto done;} + + /* First, convert from current Code Page to utf8 */ + if((stat = ansi2utf8(path,&path8))) goto done; + + if((cvtpath = NCpathcvt(path8)) == NULL) {stat = EINVAL; goto done;} +#ifdef WINUTF8 + if(acp == CP_UTF8) { + /* This should take utf8 directly */ + if(_access(cvtpath,mode) < 0) {stat = errno; goto done;} + } else +#endif + { + /* Convert from utf8 to wide */ + if((stat = utf82wide(cvtpath,&wpath))) goto done; + if(_waccess(wpath,mode) < 0) {stat = errno; goto done;} + } + done: - free(cvtpath); - free(wpath); - errno = status; + nullfree(cvtpath); + nullfree(path8); + nullfree(wpath); + errno = stat; return (errno?-1:0); } @@ -475,14 +534,29 @@ int NCremove(const char* path) { int status = 0; + char* path8 = NULL; char* cvtpath = NULL; wchar_t* wpath = NULL; - if((cvtpath = NCpathcvt(path)) == NULL) {status=ENOMEM; goto done;} - if((status = utf82wide(cvtpath,&wpath))) {status = ENOENT; goto done;} - if(_wremove(wpath) < 0) {status = errno; goto done;} + + /* First, convert from current Code Page to utf8 */ + if((status = ansi2utf8(path,&path8))) goto done; + + if((cvtpath = NCpathcvt(path8)) == NULL) {status=ENOMEM; goto done;} +#ifdef WINUTF8 + if(acp == CP_UTF8) { + /* This should take utf8 directly */ + if(remove(cvtpath) < 0) {status = errno; goto done;} + } else +#endif + { + if((status = utf82wide(cvtpath,&wpath))) {status = ENOENT; goto done;} + if(_wremove(wpath) < 0) {status = errno; goto done;} + } + done: - free(cvtpath); - free(wpath); + nullfree(cvtpath); + nullfree(path8); + nullfree(wpath); errno = status; return (errno?-1:0); } @@ -493,13 +567,28 @@ NCmkdir(const char* path, int mode) { int status = 0; char* cvtpath = NULL; + char* path8 = NULL; wchar_t* wpath = NULL; - if((cvtpath = NCpathcvt(path)) == NULL) {status=ENOMEM; goto done;} - if((status = utf82wide(cvtpath,&wpath))) {status = ENOENT; goto done;} - if(_wmkdir(wpath) < 0) {status = errno; goto done;} + + /* First, convert from current Code Page to utf8 */ + if((status = ansi2utf8(path,&path8))) goto done; + + if((cvtpath = NCpathcvt(path8)) == NULL) {status=ENOMEM; goto done;} +#ifdef WINUTF8 + if(acp == CP_UTF8) { + /* This should take utf8 directly */ + if(_mkdir(cvtpath) < 0) {status = errno; goto done;} + } else +#endif + { + if((status = utf82wide(cvtpath,&wpath))) {status = ENOENT; goto done;} + if(_wmkdir(wpath) < 0) {status = errno; goto done;} + } + done: - free(cvtpath); - free(wpath); + nullfree(cvtpath); + nullfree(path8); + nullfree(wpath); errno = status; return (errno?-1:0); } @@ -509,10 +598,18 @@ int NCrmdir(const char* path) { int status = 0; - char* cvtname = NCpathcvt(path); - if(cvtname == NULL) {errno = ENOENT; return -1;} + char* cvtname = NULL; + char* path8 = NULL; + + /* First, convert from current Code Page to utf8 */ + if((status = ansi2utf8(path,&path8))) goto done; + + cvtname = NCpathcvt(path8); + if(cvtname == NULL) {errno = ENOENT; status = -1;} status = rmdir(cvtname); - free(cvtname); +done: + nullfree(cvtname); + nullfree(path8); return status; } @@ -589,13 +686,26 @@ NCstat(const char* path, struct stat* buf) { int status = 0; char* cvtpath = NULL; + char* path8 = NULL; wchar_t* wpath = NULL; - if((cvtpath = NCpathcvt(path)) == NULL) {status=ENOMEM; goto done;} - if((status = utf82wide(cvtpath,&wpath))) {status = ENOENT; goto done;} - if(_wstat64(wpath,buf) < 0) {status = errno; goto done;} + + if((status = ansi2utf8(path,&path8))) goto done; + + if((cvtpath = NCpathcvt(path8)) == NULL) {status=ENOMEM; goto done;} +#ifdef WINUTF8 + if(acp == CP_UTF8) { + if(_stat64(cvtpath,buf) < 0) {status = errno; goto done;} + } else +#endif + { + if((status = utf82wide(cvtpath,&wpath))) {status = ENOENT; goto done;} + if(_wstat64(wpath,buf) < 0) {status = errno; goto done;} + } + done: - free(cvtpath); - free(wpath); + nullfree(cvtpath); + nullfree(path8); + nullfree(wpath); errno = status; return (errno?-1:0); } @@ -672,6 +782,7 @@ parsepath(const char* inpath, struct Path* path) memset(path,0,sizeof(struct Path)); if(inpath == NULL) goto done; /* defensive driving */ + if(!pathinitialized) pathinit(); #if 0 /* Convert to UTF8 */ @@ -999,7 +1110,7 @@ getwdpath(void) wpath = _wgetcwd(wcwd, 8192); path = NULL; stat = wide2utf8(wpath, &path); - free(wcwd); + nullfree(wcwd); if (stat) return stat; strlcat(wdprefix,path,sizeof(wdprefix)); } @@ -1067,9 +1178,9 @@ NCgetkindname(int kind) #ifdef WINPATH /** - * Converts the filename from Locale character set (presumably some + * Converts the file path from current character set (presumably some * ANSI character set like ISO-Latin-1 or UTF-8 to UTF-8 - * @param local Pointer to a nul-terminated string in locale char set. + * @param local Pointer to a nul-terminated string in current char set. * @param u8p Pointer for returning the output utf8 string * * @return NC_NOERR return converted filename @@ -1078,21 +1189,32 @@ NCgetkindname(int kind) * */ static int -ansi2utf8(const char* local, char** u8p) +ansi2utf8(const char* path, char** u8p) { int stat=NC_NOERR; char* u8 = NULL; int n; wchar_t* u16 = NULL; + if(path == NULL) goto done; + + if(!pathinitialized) pathinit(); + +#ifdef WINUTF8 + if(acp == CP_UTF8) { /* Current code page is UTF8 and Windows supports */ + u8 = strdup(path); + if(u8 == NULL) stat = NC_ENOMEM; + } else +#endif + /* Use wide character conversion plus current code page*/ { /* Get length of the converted string */ - n = MultiByteToWideChar(CP_ACP, 0, local, -1, NULL, 0); + n = MultiByteToWideChar(acp, 0, path, -1, NULL, 0); if (!n) {stat = NC_EINVAL; goto done;} - if((u16 = malloc(sizeof(wchar_t) * n))==NULL) + if((u16 = malloc(sizeof(wchar_t) * (n)))==NULL) {stat = NCTHROW(NC_ENOMEM); goto done;} /* do the conversion */ - if (!MultiByteToWideChar(CP_ACP, 0, local, -1, u16, n)) + if (!MultiByteToWideChar(CP_ACP, 0, path, -1, u16, n)) {stat = NC_EINVAL; goto done;} /* Now reverse the process to produce utf8 */ n = WideCharToMultiByte(CP_UTF8, 0, u16, -1, NULL, 0, NULL, NULL); @@ -1102,8 +1224,8 @@ ansi2utf8(const char* local, char** u8p) if (!WideCharToMultiByte(CP_UTF8, 0, u16, -1, u8, n, NULL, NULL)) {stat = NC_EINVAL; goto done;} } - if(u8p) {*u8p = u8; u8 = NULL;} done: + if(u8p) {*u8p = u8; u8 = NULL;} nullfree(u8); nullfree(u16); return stat; @@ -1116,6 +1238,8 @@ ansi2wide(const char* local, wchar_t** u16p) wchar_t* u16 = NULL; int n; + if(!pathinitialized) pathinit(); + /* Get length of the converted string */ n = MultiByteToWideChar(CP_ACP, 0, local, -1, NULL, 0); if (!n) {stat = NC_EINVAL; goto done;} @@ -1137,6 +1261,8 @@ utf82wide(const char* utf8, wchar_t** u16p) wchar_t* u16 = NULL; int n; + if(!pathinitialized) pathinit(); + /* Get length of the converted string */ n = MultiByteToWideChar(CP_UTF8, 0, utf8, -1, NULL, 0); if (!n) {stat = NC_EINVAL; goto done;} @@ -1158,6 +1284,8 @@ wide2utf8(const wchar_t* u16, char** u8p) char* u8 = NULL; int n; + if(!pathinitialized) pathinit(); + /* Get length of the converted string */ n = WideCharToMultiByte(CP_UTF8, 0, u16, -1, NULL, 0, NULL, NULL); if (!n) {stat = NC_EINVAL; goto done;} @@ -1172,17 +1300,6 @@ wide2utf8(const wchar_t* u16, char** u8p) return stat; } -/* Set locale */ -void -nc_setlocale_utf8(void) -{ -#ifdef __hpux - setlocale(LC_CTYPE,""); -#else - setlocale(LC_ALL,"C.UTF8"); -#endif -} - #endif /*WINPATH*/ static char* diff --git a/libdispatch/dutil.c b/libdispatch/dutil.c index fc8a018d1a..06f68fe87e 100644 --- a/libdispatch/dutil.c +++ b/libdispatch/dutil.c @@ -195,7 +195,7 @@ NC_shellUnescape(const char* esc) } /** -Wrap mktmp and return the generated path, +Wrap mkstemp and return the generated path, or null if failed. Base is the base file path. XXXXX is appended to allow mktmp add its unique id. @@ -208,16 +208,41 @@ NC_mktmp(const char* base) int fd = -1; char* tmp = NULL; size_t len; + int tries; +#define MAXTRIES 4 +#ifdef HAVE_MKSTEMP + mode_t mask; +#endif len = strlen(base)+6+1; - if((tmp = (char*)malloc(len))==NULL) + if((tmp = (char*)calloc(1,len))==NULL) goto done; - strncpy(tmp,base,len); +#ifdef HAVE_MKSTEMP + strlcat(tmp,base,len); strlcat(tmp, "XXXXXX", len); + mask=umask(0077); fd = NCmkstemp(tmp); + (void)umask(mask); +#else /* !HAVE_MKSTEMP */ + /* Need to simulate by using some kind of pseudo-random number */ + for(tries=0;tries= 0) break; /* sucess */ + fd = -1; /* try again */ + } +#endif /* !HAVE_MKSTEMP */ if(fd < 0) { - nclog(NCLOGERR, "Could not create temp file: %s",tmp); - goto done; + nclog(NCLOGERR, "Could not create temp file: %s",tmp); + nullfree(tmp); + tmp = NULL; + goto done; } done: if(fd >= 0) close(fd); @@ -226,23 +251,47 @@ NC_mktmp(const char* base) int NC_readfile(const char* filename, NCbytes* content) +{ + int stat; + stat = NC_readfilen(filename, content, -1); + return stat; +} + +int +NC_readfilen(const char* filename, NCbytes* content, long long amount) { int ret = NC_NOERR; FILE* stream = NULL; - char part[1024]; stream = NCfopen(filename,"r"); if(stream == NULL) {ret=errno; goto done;} - for(;;) { + ret = NC_readfileF(stream,content,amount); + if (stream) fclose(stream); +done: + return ret; +} + +int +NC_readfileF(FILE* stream, NCbytes* content, long long amount) +{ + int ret = NC_NOERR; + long long red = 0; + char part[1024]; + + while(amount < 0 || red < amount) { size_t count = fread(part, 1, sizeof(part), stream); - if(count <= 0) break; - ncbytesappendn(content,part,count); if(ferror(stream)) {ret = NC_EIO; goto done;} - if(feof(stream)) break; + if(count > 0) ncbytesappendn(content,part,(unsigned long)count); + red += count; + if (feof(stream)) break; + } + /* Keep only amount */ + if(amount >= 0) { + if(red > amount) ncbytessetlength(content,amount); /* read too much */ + if(red < amount) ret = NC_ETRUNC; /* |file| < amount */ } ncbytesnull(content); done: - if(stream) fclose(stream); return ret; } @@ -263,8 +312,8 @@ NC_writefile(const char* filename, size_t size, void* content) while(remain > 0) { size_t written = fwrite(p, 1, remain, stream); if(ferror(stream)) {ret = NC_EIO; goto done;} - if(feof(stream)) break; remain -= written; + if (feof(stream)) break; } done: if(stream) fclose(stream); diff --git a/libdispatch/ncs3sdk.cpp b/libdispatch/ncs3sdk.cpp index 2c51b3e762..1b0feaec02 100644 --- a/libdispatch/ncs3sdk.cpp +++ b/libdispatch/ncs3sdk.cpp @@ -73,9 +73,8 @@ NC_s3sdkinitialize(void) ncs3_finalized = 0; NCTRACE(11,NULL); Aws::InitAPI(ncs3options); - NCUNTRACE(NC_NOERR); } - return 1; + return NCUNTRACE(NC_NOERR); } EXTERNL int @@ -86,9 +85,8 @@ NC_s3sdkfinalize(void) ncs3_finalized = 1; NCTRACE(11,NULL); Aws::ShutdownAPI(ncs3options); - NCUNTRACE(NC_NOERR); } - return 1; + return NCUNTRACE(NC_NOERR); } static char* diff --git a/libsrc/XGetopt.c b/libsrc/XGetopt.c index 5a4894ca75..08a28a1d98 100644 --- a/libsrc/XGetopt.c +++ b/libsrc/XGetopt.c @@ -1,220 +1,284 @@ -// XGetopt.cpp Version 1.2 -// -// Author: Hans Dietrich -// hdietrich2@hotmail.com -// -// Description: -// XGetopt.cpp implements getopt(), a function to parse command lines. -// -// History -// Version 1.2 - 2003 May 17 -// - Added Unicode support -// -// Version 1.1 - 2002 March 10 -// - Added example to XGetopt.cpp module header -// -// This software is released into the public domain. -// You are free to use it in any way you like. -// -// This software is provided "as is" with no expressed -// or implied warranty. I accept no liability for any -// damage or loss of business that this software may cause. -// -/////////////////////////////////////////////////////////////////////////////// - - -/////////////////////////////////////////////////////////////////////////////// -// if you are not using precompiled headers then include these lines: - -/////////////////////////////////////////////////////////////////////////////// - - +// XGetopt.cpp Version 1.2 +// +// Author: Hans Dietrich +// hdietrich2@hotmail.com +// +// Description: +// XGetopt.cpp implements getopt(), a function to parse command lines. +// +// History +// Version 1.2 - 2003 May 17 +// - Added Unicode support +// +// Version 1.1 - 2002 March 10 +// - Added example to XGetopt.cpp module header +// +// This software is released into the public domain. +// You are free to use it in any way you like. +// +// This software is provided "as is" with no expressed +// or implied warranty. I accept no liability for any +// damage or loss of business that this software may cause. +// +/////////////////////////////////////////////////////////////////////////////// + + +/////////////////////////////////////////////////////////////////////////////// +// if you are not using precompiled headers then include these lines: + +/////////////////////////////////////////////////////////////////////////////// + + #ifndef DLL_EXPORT #define DLL_EXPORT #endif -#include "XGetopt.h" - - -/////////////////////////////////////////////////////////////////////////////// -// -// X G e t o p t . c p p -// -// -// NAME -// getopt -- parse command line options -// -// SYNOPSIS -// int getopt(int argc, TCHAR *argv[], TCHAR *optstring) -// -// extern TCHAR *optarg; -// extern int optind; -// -// DESCRIPTION -// The getopt() function parses the command line arguments. Its -// arguments argc and argv are the argument count and array as -// passed into the application on program invocation. In the case -// of Visual C++ programs, argc and argv are available via the -// variables __argc and __argv (double underscores), respectively. -// getopt returns the next option letter in argv that matches a -// letter in optstring. (Note: Unicode programs should use -// __targv instead of __argv. Also, all character and string -// literals should be enclosed in _T( ) ). -// -// optstring is a string of recognized option letters; if a letter -// is followed by a colon, the option is expected to have an argument -// that may or may not be separated from it by white space. optarg -// is set to point to the start of the option argument on return from -// getopt. -// -// Option letters may be combined, e.g., "-ab" is equivalent to -// "-a -b". Option letters are case sensitive. -// -// getopt places in the external variable optind the argv index -// of the next argument to be processed. optind is initialized -// to 0 before the first call to getopt. -// -// When all options have been processed (i.e., up to the first -// non-option argument), getopt returns EOF, optarg will point -// to the argument, and optind will be set to the argv index of -// the argument. If there are no non-option arguments, optarg -// will be set to NULL. -// -// The special option "--" may be used to delimit the end of the -// options; EOF will be returned, and "--" (and everything after it) -// will be skipped. -// -// RETURN VALUE -// For option letters contained in the string optstring, getopt -// will return the option letter. getopt returns a question mark (?) -// when it encounters an option letter not included in optstring. -// EOF is returned when processing is finished. -// -// BUGS -// 1) Long options are not supported. -// 2) The GNU double-colon extension is not supported. -// 3) The environment variable POSIXLY_CORRECT is not supported. -// 4) The + syntax is not supported. -// 5) The automatic permutation of arguments is not supported. -// 6) This implementation of getopt() returns EOF if an error is -// encountered, instead of -1 as the latest standard requires. -// -// EXAMPLE -// BOOL CMyApp::ProcessCommandLine(int argc, TCHAR *argv[]) -// { -// int c; -// -// while ((c = getopt(argc, argv, _T("aBn:"))) != EOF) -// { -// switch (c) -// { -// case _T('a'): -// TRACE(_T("option a\n")); -// // -// // set some flag here -// // -// break; -// -// case _T('B'): -// TRACE( _T("option B\n")); -// // -// // set some other flag here -// // -// break; -// -// case _T('n'): -// TRACE(_T("option n: value=%d\n"), atoi(optarg)); -// // -// // do something with value here -// // -// break; -// -// case _T('?'): -// TRACE(_T("ERROR: illegal option %s\n"), argv[optind-1]); -// return FALSE; -// break; -// -// default: -// TRACE(_T("WARNING: no handler for option %c\n"), c); -// return FALSE; -// break; -// } -// } -// // -// // check for non-option args here -// // -// return TRUE; -// } -// -/////////////////////////////////////////////////////////////////////////////// - -GTOPT_EXTRA TCHAR *optarg; // global argument pointer -GTOPT_EXTRA int optind = 0; // global argv index +#include "XGetopt.h" +#include "nclist.h" +#include "ncbytes.h" + +#ifdef _MSC_VER +static void XCommandLineToArgvA(int* argcp, char*** argvp); +#endif + +/////////////////////////////////////////////////////////////////////////////// +// +// X G e t o p t . c p p +// +// +// NAME +// getopt -- parse command line options +// +// SYNOPSIS +// int getopt(int argc, TCHAR *argv[], TCHAR *optstring) +// +// extern TCHAR *optarg; +// extern int optind; +// +// DESCRIPTION +// The getopt() function parses the command line arguments. Its +// arguments argc and argv are the argument count and array as +// passed into the application on program invocation. In the case +// of Visual C++ programs, argc and argv are available via the +// variables __argc and __argv (double underscores), respectively. +// getopt returns the next option letter in argv that matches a +// letter in optstring. (Note: Unicode programs should use +// __targv instead of __argv. Also, all character and string +// literals should be enclosed in _T( ) ). +// +// optstring is a string of recognized option letters; if a letter +// is followed by a colon, the option is expected to have an argument +// that may or may not be separated from it by white space. optarg +// is set to point to the start of the option argument on return from +// getopt. +// +// Option letters may be combined, e.g., "-ab" is equivalent to +// "-a -b". Option letters are case sensitive. +// +// getopt places in the external variable optind the argv index +// of the next argument to be processed. optind is initialized +// to 0 before the first call to getopt. +// +// When all options have been processed (i.e., up to the first +// non-option argument), getopt returns EOF, optarg will point +// to the argument, and optind will be set to the argv index of +// the argument. If there are no non-option arguments, optarg +// will be set to NULL. +// +// The special option "--" may be used to delimit the end of the +// options; EOF will be returned, and "--" (and everything after it) +// will be skipped. +// +// RETURN VALUE +// For option letters contained in the string optstring, getopt +// will return the option letter. getopt returns a question mark (?) +// when it encounters an option letter not included in optstring. +// EOF is returned when processing is finished. +// +// BUGS +// 1) Long options are not supported. +// 2) The GNU double-colon extension is not supported. +// 3) The environment variable POSIXLY_CORRECT is not supported. +// 4) The + syntax is not supported. +// 5) The automatic permutation of arguments is not supported. +// 6) This implementation of getopt() returns EOF if an error is +// encountered, instead of -1 as the latest standard requires. +// +// EXAMPLE +// BOOL CMyApp::ProcessCommandLine(int argc, TCHAR *argv[]) +// { +// int c; +// +// while ((c = getopt(argc, argv, _T("aBn:"))) != EOF) +// { +// switch (c) +// { +// case _T('a'): +// TRACE(_T("option a\n")); +// // +// // set some flag here +// // +// break; +// +// case _T('B'): +// TRACE( _T("option B\n")); +// // +// // set some other flag here +// // +// break; +// +// case _T('n'): +// TRACE(_T("option n: value=%d\n"), atoi(optarg)); +// // +// // do something with value here +// // +// break; +// +// case _T('?'): +// TRACE(_T("ERROR: illegal option %s\n"), argv[optind-1]); +// return FALSE; +// break; +// +// default: +// TRACE(_T("WARNING: no handler for option %c\n"), c); +// return FALSE; +// break; +// } +// } +// // +// // check for non-option args here +// // +// return TRUE; +// } +// +/////////////////////////////////////////////////////////////////////////////// + +GTOPT_EXTRA TCHAR *optarg; // global argument pointer +GTOPT_EXTRA int optind = 0; // global argv index GTOPT_EXTRA int opterr; // print error - + GTOPT_EXTRA -int getopt(int argc, TCHAR *argv[], TCHAR *optstring) -{ - static TCHAR *next = NULL; - TCHAR c; - TCHAR *cp = malloc(sizeof(TCHAR)*1024); - - if (optind == 0) - next = NULL; - - optarg = NULL; - - if (next == NULL || *next == _T('\0')) - { - if (optind == 0) - optind++; - - if (optind >= argc || argv[optind][0] != _T('-') || argv[optind][1] == _T('\0')) - { - optarg = NULL; - if (optind < argc) - optarg = argv[optind]; - return EOF; - } - - if (_tcscmp(argv[optind], _T("--")) == 0) - { - optind++; - optarg = NULL; - if (optind < argc) - optarg = argv[optind]; - return EOF; - } - - next = argv[optind]; - next++; // skip past - - optind++; - } - - c = *next++; - cp = strchr(optstring, c); - - if (cp == NULL || c == _T(':')) - return _T('?'); - - cp++; - if (*cp == _T(':')) - { - if (*next != _T('\0')) - { - optarg = next; - next = NULL; - } - else if (optind < argc) - { - optarg = argv[optind]; - optind++; - } - else - { - return _T('?'); - } - } - - return c; -} +int getopt(int argc, TCHAR *argv[], TCHAR *optstring) +{ + static TCHAR *next = NULL; + TCHAR c; + TCHAR *cp = malloc(sizeof(TCHAR)*1024); + +#ifdef _MSC_VER + { + int xargc = -1; + char** xargv = NULL; + XCommandLineToArgvA(&xargc, &xargv); + } +#endif + + if (optind == 0) + next = NULL; + + optarg = NULL; + + if (next == NULL || *next == _T('\0')) + { + if (optind == 0) + optind++; + + if (optind >= argc || argv[optind][0] != _T('-') || argv[optind][1] == _T('\0')) + { + optarg = NULL; + if (optind < argc) + optarg = argv[optind]; + return EOF; + } + + if (_tcscmp(argv[optind], _T("--")) == 0) + { + optind++; + optarg = NULL; + if (optind < argc) + optarg = argv[optind]; + return EOF; + } + + next = argv[optind]; + next++; // skip past - + optind++; + } + + c = *next++; + cp = strchr(optstring, c); + + if (cp == NULL || c == _T(':')) + return _T('?'); + + cp++; + if (*cp == _T(':')) + { + if (*next != _T('\0')) + { + optarg = next; + next = NULL; + } + else if (optind < argc) + { + optarg = argv[optind]; + optind++; + } + else + { + return _T('?'); + } + } + + return c; +} + +/**************************************************/ +#define ESCAPE '\\' +#define SQUOTE '\'' +#define DQUOTE '"' + +/* Convert a UTF8 command line into a series of words for use by XGetOpt */ +/* Note only checks for ASCII '\' '\'' '"' */ +static void +XCommandLineToArgvA(int* argcp, char*** argvp) +{ + const char* line = NULL; + size_t len; + int whitespace; + int quote = 0; + enum State state; + NClist* argv = NULL; + NCbytes* word = NULL; + char** p; + + line = GetCommandLineA(); + len = strlen(line); + argv = nclistnew(); + word = ncbytesnew(); + whitespace = 1; /* start in whitespace mode */ + for(p=line;*p;p++) { + int c = *p; + if(whitespace && c <= ' ' || c == 127) continue; /* more whitespace */ + if(!whitespace && c <= ' ' || c == 127) { + whitespace = 1; /* end of word */ + ncbytesnull(word); + nclistpush(argv,ncbytesextract(word)); /* capture the word */ + continue; + } + whitespace = 0; /* end whitespace */ + if(c == ESCAPE) { + c = *(++p); /* move to next char */ + } else if(c == SQUOTE || c == DQUOTE) { + if(!quote) {quote = c; continue;} /* Start quoted text */ + if(quote == c) {quote = 0; continue;} /* end quoted text */ + } + /* Just collect the character as part of the current word */ + ncbytesappend(word,c); + } + /* Return parsed words */ + if(argcp) *argcp = nclistlength(argv); + nclistpush(argv,NULL); /* Just to be sure */ + if(argvp) *argvp = (char**)nclistextract(argv); + nclistfree(argv); + ncbytesfree(word); +} diff --git a/libsrc/ncFile.c b/libsrc/ncFile.c index e448d32e94..6d152ecaad 100644 --- a/libsrc/ncFile.c +++ b/libsrc/ncFile.c @@ -63,7 +63,7 @@ ncFile_create(const char *path, int ioflags, ncstdio** filepp) File* f; struct ncFileState* state; - f = fopen(path,"w+"); + f = NCfopen(path,"w+"); if(f == NULL) return errno; filep = (ncstdio*)calloc(sizeof(ncstdio),1); diff --git a/libsrc/ncio.c b/libsrc/ncio.c index 24bb92da10..9ee5d99bf5 100644 --- a/libsrc/ncio.c +++ b/libsrc/ncio.c @@ -83,7 +83,9 @@ ncio_open(const char *path, int ioflags, void* parameters, ncio** iopp, void** const mempp) { +#ifdef ENABLE_BYTERANGE int modetest = urlmodetest(path); +#endif /* Diskless open has the following constraints: 1. file must be classic version 1 or 2 or 5 diff --git a/nc_perf/tst_compress.c b/nc_perf/tst_compress.c index 15f2cba41c..354ac422cb 100644 --- a/nc_perf/tst_compress.c +++ b/nc_perf/tst_compress.c @@ -83,7 +83,7 @@ get_file_size(char *filename, size_t *file_size) FILE *fp; assert(filename && file_size); - fp = fopen(filename, "r"); + fp = NCfopen(filename, "r"); if (fp) { fseek(fp, 0 , SEEK_END); diff --git a/nc_perf/tst_compress_par.c b/nc_perf/tst_compress_par.c index 72084d1195..39b769c960 100644 --- a/nc_perf/tst_compress_par.c +++ b/nc_perf/tst_compress_par.c @@ -82,7 +82,7 @@ get_file_size(char *filename, size_t *file_size) FILE *fp; assert(filename && file_size); - fp = fopen(filename, "r"); + fp = NCfopen(filename, "r"); if (fp) { fseek(fp, 0 , SEEK_END); diff --git a/nc_perf/tst_files2.c b/nc_perf/tst_files2.c index 1e6145ac4a..ca9c8211ac 100644 --- a/nc_perf/tst_files2.c +++ b/nc_perf/tst_files2.c @@ -45,7 +45,7 @@ get_mem_used1(int *mem_used) system(cmd); /* Read the results and delete temp file. */ - if (!(fp = fopen(TMP_FILE_NAME, "r"))) ERR; + if (!(fp = NCfopen(TMP_FILE_NAME, "r"))) ERR; fread(blob, MAX_LEN, 1, fp); sscanf(blob, "%d", mem_used); fclose(fp); @@ -60,7 +60,7 @@ get_mem_used2(int *mem_used) FILE *pf; snprintf(buf, 30, "/proc/%u/statm", (unsigned)getpid()); - pf = fopen(buf, "r"); + pf = NCfopen(buf, "r"); if (pf) { unsigned size; /* total program size */ unsigned resident;/* resident set size */ diff --git a/nc_perf/tst_files3.c b/nc_perf/tst_files3.c index 7e72090de8..ab04f7ec06 100644 --- a/nc_perf/tst_files3.c +++ b/nc_perf/tst_files3.c @@ -167,7 +167,7 @@ get_mem_used2(int *mem_used) FILE *pf; snprintf(buf, 30, "/proc/%u/statm", (unsigned)getpid()); - pf = fopen(buf, "r"); + pf = NCfopen(buf, "r"); if (pf) { unsigned size; /* total program size */ unsigned resident;/* resident set size */ diff --git a/nc_perf/tst_mem.c b/nc_perf/tst_mem.c index a22a2f0960..ae4860e595 100644 --- a/nc_perf/tst_mem.c +++ b/nc_perf/tst_mem.c @@ -30,7 +30,7 @@ get_mem_used2(int *mem_used) FILE *pf; snprintf(buf, 30, "/proc/%u/statm", (unsigned)getpid()); - pf = fopen(buf, "r"); + pf = NCfopen(buf, "r"); if (pf) { unsigned size; /* total program size */ unsigned resident;/* resident set size */ diff --git a/nc_test/tst_diskless5.c b/nc_test/tst_diskless5.c index ab06f36df3..77b9f181a2 100644 --- a/nc_test/tst_diskless5.c +++ b/nc_test/tst_diskless5.c @@ -91,7 +91,7 @@ main(int argc, const char* argv[]) #endif #ifdef MEM - fd = open(PATH,O_RDONLY); + fd = NCopen2(PATH,O_RDONLY); if(fd < 0) { fprintf(stderr,"could not open foo.nc\n"); assert(0); diff --git a/nc_test/tst_diskless6.c b/nc_test/tst_diskless6.c index d968d71b14..69aa805dac 100644 --- a/nc_test/tst_diskless6.c +++ b/nc_test/tst_diskless6.c @@ -31,6 +31,7 @@ See \ref copyright file for more info. #endif #include "netcdf.h" +#include "ncpathmgr.h" #define CLEANUP @@ -131,7 +132,8 @@ fail(int ret) void exists(const char* file) { - FILE* f = fopen(file,"r"); + int stat = 0; + FILE* f = NCfopen(file, "r"); if(f == NULL) fail(NC_EPERM); fclose(f); } @@ -139,7 +141,7 @@ exists(const char* file) void notexists(const char* file) { - FILE* f = fopen(file,"r"); + FILE* f = NCfopen(file,"r"); if(f != NULL) {fclose(f); fail(NC_EEXIST);} } diff --git a/nc_test/tst_inmemory.c b/nc_test/tst_inmemory.c index 5119af820f..2075dd2957 100644 --- a/nc_test/tst_inmemory.c +++ b/nc_test/tst_inmemory.c @@ -162,80 +162,27 @@ static int readfile(const char* path, NC_memio* memio) { int status = NC_NOERR; - FILE* f = NULL; - size_t filesize = 0; - size_t count = 0; - char* memory = NULL; - char* p = NULL; - - /* Open the file for reading */ - f = NCfopen(path,"r"); - if(f == NULL) - {status = errno; goto done;} - /* get current filesize */ - if(fseek(f,0,SEEK_END) < 0) - {status = errno; goto done;} - filesize = (size_t)ftell(f); - /* allocate memory */ - memory = malloc((size_t)filesize); - if(memory == NULL) - {status = NC_ENOMEM; goto done;} - /* move pointer back to beginning of file */ - rewind(f); - count = filesize; - p = memory; - while(count > 0) { - size_t actual; - actual = fread(p,1,count,f); - if(actual == 0 || ferror(f)) - {status = NC_EIO; goto done;} - count -= actual; - p += actual; - } + NCbytes* buf = ncbytesnew(); + if((status = NC_readfile(path,buf))) goto done; if(memio) { - memio->size = (size_t)filesize; - memio->memory = memory; + memio->size = (size_t)ncbyteslength(buf); + memio->memory = ncbytesextract(buf); } done: - if(status != NC_NOERR && memory != NULL) - free(memory); - if(f != NULL) fclose(f); + ncbytesfree(buf); return status; } - static int writefile(const char* path, NC_memio* memio) { int status = NC_NOERR; - FILE* f = NULL; - size_t count = 0; - char* p = NULL; - /* Open the file for writing */ -#ifdef _WIN32 - f = fopen(path,"wb"); -#else - f = fopen(path,"w"); -#endif - if(f == NULL) - {status = errno; goto done;} - count = memio->size; - p = memio->memory; - while(count > 0) { - size_t actual; - actual = fwrite(p,1,count,f); - if(actual == 0 || ferror(f)) - {status = NC_EIO; goto done;} - count -= actual; - p += actual; - } + if((status = NC_writefile(path,memio->size,memio->memory))) goto done; done: - if(f != NULL) fclose(f); return status; } - /* Duplicate an NC_memio instance; needed to avoid attempting to use memory that might have been realloc'd Allow the new memory to be larger than the src memory diff --git a/nc_test/tst_misc.c b/nc_test/tst_misc.c index 7c55a1ddf3..93a96a3412 100644 --- a/nc_test/tst_misc.c +++ b/nc_test/tst_misc.c @@ -43,7 +43,7 @@ main(int argc, char **argv) for (i = DATA_LEN; i >= 0; i--) { /* Create a small file which is not a netCDF file. */ - if (!(file = fopen(FILE_NAME, "w+"))) ERR; + if (!(file = NCfopen(FILE_NAME, "w+"))) ERR; if (fwrite(dummy_data, 1, i, file) != i) ERR; if (fclose(file)) ERR; diff --git a/nc_test4/tst_camrun.c b/nc_test4/tst_camrun.c index bd3b4c8266..fb6ed42e3c 100644 --- a/nc_test4/tst_camrun.c +++ b/nc_test4/tst_camrun.c @@ -680,7 +680,7 @@ get_mem_used2(int *mem_used) assert(mem_used); snprintf(buf, 30, "/proc/%u/statm", (unsigned)getpid()); - if ((pf = fopen(buf, "r"))) + if ((pf = NCfopen(buf, "r"))) { (void)fscanf(pf, "%u %u %u %u %u %u", &size, &resident, &share, &text, &lib, &data); diff --git a/nc_test4/tst_udf.c b/nc_test4/tst_udf.c index 43d44a016c..12024bb2ea 100644 --- a/nc_test4/tst_udf.c +++ b/nc_test4/tst_udf.c @@ -311,7 +311,7 @@ main(int argc, char **argv) int i; /* Create a file with magic number at start. */ - if (!(FP = fopen(FILE_NAME, "w"))) ERR; + if (!(FP = NCfopen(FILE_NAME, "w"))) ERR; if (fwrite(magic_number, sizeof(char), strlen(magic_number), FP) != strlen(magic_number)) ERR; if (fwrite(dummy_data, sizeof(char), strlen(dummy_data), FP) diff --git a/ncdap_test/t_auth.c b/ncdap_test/t_auth.c index e70a068e87..32d3a4c24c 100644 --- a/ncdap_test/t_auth.c +++ b/ncdap_test/t_auth.c @@ -16,7 +16,7 @@ See \ref copyright file for more info. #include #endif -#define DEBUG +#undef DEBUG #include "netcdf.h" #include "nctestserver.h" @@ -150,7 +150,7 @@ testrc(const char* prefix, const char* url) FILE* rc; snprintf(rcpath,sizeof(rcpath),"%s/%s",prefix,RC); - rc = fopen(rcpath,"w"); + rc = NCfopen(rcpath,"w"); if(rc == NULL) { fprintf(stderr,"Cannot create ./%s\n",RC); exit(1); @@ -178,7 +178,7 @@ fillrc(const char* path) FILE* rc; killrc(); - rc = fopen(path,"w"); + rc = NCfopen(path,"w"); if(rc == NULL) { fprintf(stderr,"cannot create rc file: %s\n",path); exit(1); diff --git a/ncdump/CMakeLists.txt b/ncdump/CMakeLists.txt index c0aee2a7f3..7b0cca34d6 100644 --- a/ncdump/CMakeLists.txt +++ b/ncdump/CMakeLists.txt @@ -117,6 +117,7 @@ IF(MSVC) ${CMAKE_CURRENT_BINARY_DIR}) ENDIF(ENABLE_DAP) + ENDIF() IF(ENABLE_TESTS) @@ -304,14 +305,11 @@ endif() ENDIF() ENDIF(EXTRA_TESTS) - # The unicode tests are complicated - IF(USE_HDF5) - IF(NOT MSVC AND NOT MINGW) - # These tests do not work under windows - add_sh_test(ncdump test_unicode_directory) - add_sh_test(ncdump test_unicode_path) - ENDIF() - ENDIF(USE_HDF5) + # The unicode tests + if(NOT ISMINGW) + add_sh_test(ncdump test_unicode_directory) + add_sh_test(ncdump test_unicode_path) + ENDIF() IF(USE_CDF5) add_sh_test(ncdump test_keywords) diff --git a/ncdump/Makefile.am b/ncdump/Makefile.am index d5db8cf3a8..b629c495c7 100644 --- a/ncdump/Makefile.am +++ b/ncdump/Makefile.am @@ -144,7 +144,6 @@ if !ISMINGW if !ISCYGWIN TESTS += tst_output.sh TESTS += tst_nccopy3.sh -TESTS += test_unicode_directory.sh test_unicode_path.sh if USE_HDF5 TESTS += run_back_comp_tests.sh tst_netcdf4_4.sh TESTS += tst_nccopy4.sh tst_nccopy5.sh @@ -152,6 +151,11 @@ endif endif endif +# The unicode tests +if !ISMINGW +TESTS += test_unicode_directory.sh test_unicode_path.sh +endif + endif BUILD_TESTSETS # These files all have to be included with the distribution. diff --git a/ncdump/acpget.c b/ncdump/acpget.c new file mode 100644 index 0000000000..32753f768c --- /dev/null +++ b/ncdump/acpget.c @@ -0,0 +1,30 @@ +#include +#include +#include +#include +#include +#include + +int +main(int argc, char** argv) +{ + int acp = -1; + const char* acpid = NULL; + const char* lcall = NULL; + const char* lcctype = NULL; + char digits[16]; + + switch (acp = GetACP()) { + case 1252: acpid = "CP_1252"; break; + case 65001: acpid = "CP_UTF8"; break; + default: + snprintf(digits,sizeof(digits),"%d",acp); + acpid = digits; + break; + } + + lcall = setlocale(LC_ALL,NULL); + lcctype = setlocale(LC_CTYPE,NULL); + printf("ACP=%s locale: LC_ALL=%s LC_CTYPE=%s\n",acpid,lcall,lcctype); + return 0; +} diff --git a/ncdump/nctrunc.c b/ncdump/nctrunc.c index f0f9f2007e..67ce45c969 100644 --- a/ncdump/nctrunc.c +++ b/ncdump/nctrunc.c @@ -6,6 +6,7 @@ #include #include #include +#include "nc_tests.h" #define BUFLEN 100000 @@ -19,7 +20,7 @@ main(int argc, char** argv) FILE* input = stdin; if(argc > 1) { - input = fopen(argv[1],"r"); + input = NCfopen(argv[1],"r"); } /* Read the whole file */ diff --git a/ncdump/ocprint.c b/ncdump/ocprint.c index 1d16e436ef..72a45fb3ce 100755 --- a/ncdump/ocprint.c +++ b/ncdump/ocprint.c @@ -243,7 +243,7 @@ main(int argc, char **argv) if(ocopt.output != NULL) fclose(ocopt.output); if(optarg == NULL) usage("-o does not specify a file name"); - ocopt.output = fopen(optarg,"w"); + ocopt.output = NCfopen(optarg,"w"); if(ocopt.output == NULL) usage("-o file not writeable"); break; diff --git a/ncdump/run_cygutf8.sh b/ncdump/run_cygutf8.sh new file mode 100755 index 0000000000..ddf4985d64 --- /dev/null +++ b/ncdump/run_cygutf8.sh @@ -0,0 +1,10 @@ +#!/bin/bash -x + +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ../test_common.sh + +echo "Test unicode paths" +${execdir}/acpget +${execdir}/tst_cygutf8 +ls xutf8* +rm xutf8* diff --git a/ncdump/test_unicode_path.sh b/ncdump/test_unicode_path.sh index b9e08b3048..eeb4ed69d2 100755 --- a/ncdump/test_unicode_path.sh +++ b/ncdump/test_unicode_path.sh @@ -10,9 +10,6 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi set -e -LC_ALL="C.UTF-8" -export LC_ALL - # Passing a utf8 name using either \x or actual characters # to Visual Studio does not work well. if test "x$FP_ISMSVC" = x ; then diff --git a/ncdump/tst_chunking.c b/ncdump/tst_chunking.c index 95cd92010b..46c9ddc5ba 100644 --- a/ncdump/tst_chunking.c +++ b/ncdump/tst_chunking.c @@ -9,7 +9,7 @@ #include #include "err_macros.h" -#define DEBUG +#undef DEBUG static int ret = NC_NOERR; diff --git a/ncdump/tst_cygutf8.c b/ncdump/tst_cygutf8.c new file mode 100644 index 0000000000..201c3511ed --- /dev/null +++ b/ncdump/tst_cygutf8.c @@ -0,0 +1,55 @@ +/* +https://www.cygwin.com/cygwin-ug-net/using-specialnames.html +https://docs.microsoft.com/en-us/archive/msdn-magazine/2016/september/c-unicode-encoding-conversions-with-stl-strings-and-win32-apis +*/ + +#include +#include +#include +#ifdef _WIN32 +#include +#endif +#include "netcdf.h" +#include "ncpathmgr.h" + +static const unsigned char name1[] = { +'x','u','t','f','8','_', +'\xe6', '\xb5', '\xb7', +'\0' +}; + +static unsigned char name2[] = { +'x','u','t','f','8','_', + 0xCE, 0x9A, /* GREEK CAPITAL LETTER KAPPA : 2-bytes utf8 */ + 0xCE, 0xB1, /* GREEK SMALL LETTER LAMBDA : 2-bytes utf8 */ + 0xCE, 0xBB, /* GREEK SMALL LETTER ALPHA : 2-bytes utf8 */ + 0xCE, 0xB7, /* GREEK SMALL LETTER ETA : 2-bytes utf8 */ + 0xCE, 0xBC, /* GREEK SMALL LETTER MU : 2-bytes utf8 */ + 0xCE, 0xAD, /* GREEK SMALL LETTER EPSILON WITH TONOS + : 2-bytes utf8 */ + 0xCF, 0x81, /* GREEK SMALL LETTER RHO : 2-bytes utf8 */ + 0xCE, 0xB1, /* GREEK SMALL LETTER ALPHA : 2-bytes utf8 */ + 0x00 + }; + +static char* name3 = "xutf8_사람/접는사람"; + +/* This is CP_1252 */ +//static char* name4 = "xutf8_Å"; +static char name4[8] = {'x','u','t','f','8','_',0XC5,0x00} ; + +int +main() +{ + FILE* f; + f = NCfopen((char*)name1,"w"); + if (f) fclose(f); + f = NCfopen((char*)name2,"w"); + if (f) fclose(f); + f = NCfopen((char*)name3,"w"); + if (f) fclose(f); + printf("|name4|=%u\n",(unsigned)strlen(name4)); + f = NCfopen((char*)name4,"w"); + if (f) fclose(f); + return 0; +} diff --git a/ncdump/tst_unicode.c b/ncdump/tst_unicode.c index e742260db5..a3369559b6 100644 --- a/ncdump/tst_unicode.c +++ b/ncdump/tst_unicode.c @@ -23,7 +23,7 @@ #include #endif -#define DEBUG +#undef DEBUG /* The data file we will create. */ static const unsigned char prefix[] = { diff --git a/ncgen/dump.c b/ncgen/dump.c index c4e493dc4d..bd4d775b86 100644 --- a/ncgen/dump.c +++ b/ncgen/dump.c @@ -8,7 +8,7 @@ #include "includes.h" #include "dump.h" -#define DEBUGSRC +#undef DEBUGSRC #define MAXELEM 8 #define MAXDEPTH 4 diff --git a/ncgen3/run_nc4_tests.sh b/ncgen3/run_nc4_tests.sh index 52b3360d17..97caa5a53d 100755 --- a/ncgen3/run_nc4_tests.sh +++ b/ncgen3/run_nc4_tests.sh @@ -5,9 +5,10 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh +set -e echo "*** Testing ncgen3 for netCDF-4." -set -e + echo "*** creating netCDF-4 file c0_4.nc from c0.cdl..." ${NCGEN3} -k3 -b -o c0_4.nc ${ncgen3c0} echo "*** creating netCDF-4 classic model file c0_4c.nc from c0.cdl..." diff --git a/nczarr_test/ref_zarr_test_data.cdl.gz b/nczarr_test/ref_zarr_test_data.cdl.gz new file mode 100644 index 0000000000..9d688eb0e2 Binary files /dev/null and b/nczarr_test/ref_zarr_test_data.cdl.gz differ diff --git a/nczarr_test/run_interop.sh b/nczarr_test/run_interop.sh index 17d059e02c..a2ee6f3fc0 100755 --- a/nczarr_test/run_interop.sh +++ b/nczarr_test/run_interop.sh @@ -73,11 +73,11 @@ case "$zext" in # Move into position rm -f ${execdir}/ref_zarr_test_data.cdl # Use gunzip because it always appears to be available - if gunzip ${srcdir}/ref_zarr_test_data.cdl.gz ; then ignore=1; fi - if test -f ${srcdir}/ref_zarr_test_data.cdl ; then - testcases3 zarr_test_data.zarr ref_zarr_test_data xarray - fi - ;; + if ! test -f ${srcdir}/ref_zarr_test_data.cdl ; then + gunzip -c ${srcdir}/ref_zarr_test_data.cdl.gz > ${srcdir}/ref_zarr_test_data.cdl + fi + testcases3 zarr_test_data.zarr ref_zarr_test_data xarray + ;; *) echo "unimplemented kind: $1" ; exit 1;; esac } diff --git a/nczarr_test/s3util.c b/nczarr_test/s3util.c index 03ec172a23..07e03ef1c9 100644 --- a/nczarr_test/s3util.c +++ b/nczarr_test/s3util.c @@ -27,7 +27,7 @@ #undef NODELETE -#define DEBUG +#undef DEBUG #define DATANAME "data" diff --git a/nczarr_test/tst_zchunks3.c b/nczarr_test/tst_zchunks3.c index b3d238d88a..fa0c575194 100644 --- a/nczarr_test/tst_zchunks3.c +++ b/nczarr_test/tst_zchunks3.c @@ -8,7 +8,7 @@ #include "ut_includes.h" #include "test_nczarr_utils.h" -#define DEBUG +#undef DEBUG static int ret = NC_NOERR; #define FILE_NAME "tmp_chunks3.nc" diff --git a/nczarr_test/ut_json.c b/nczarr_test/ut_json.c index 22600fcb56..37ab65d231 100644 --- a/nczarr_test/ut_json.c +++ b/nczarr_test/ut_json.c @@ -5,7 +5,7 @@ #include "ut_includes.h" -#define DEBUG +#undef DEBUG typedef enum Cmds { cmd_none = 0, diff --git a/nczarr_test/zhex.c b/nczarr_test/zhex.c index 3a49ff6495..972407cb46 100644 --- a/nczarr_test/zhex.c +++ b/nczarr_test/zhex.c @@ -12,7 +12,7 @@ #include #endif -#define DEBUG +#undef DEBUG static char hex[16] = "0123456789abcdef"; diff --git a/nczarr_test/zmapio.c b/nczarr_test/zmapio.c index b1e06295ef..c9c982c881 100644 --- a/nczarr_test/zmapio.c +++ b/nczarr_test/zmapio.c @@ -23,7 +23,7 @@ #include "nclog.h" #include "ncuri.h" -#define DEBUG +#undef DEBUG #define DATANAME "data" diff --git a/nczarr_test/zs3parse.c b/nczarr_test/zs3parse.c index e1dba30cad..7286dd75a2 100644 --- a/nczarr_test/zs3parse.c +++ b/nczarr_test/zs3parse.c @@ -21,7 +21,7 @@ #include "zincludes.h" #include "ncpathmgr.h" -#define DEBUG +#undef DEBUG #define AWSHOST ".amazonaws.com" diff --git a/test_common.in b/test_common.in index 85d22323e7..fb02fe0d9d 100644 --- a/test_common.in +++ b/test_common.in @@ -10,11 +10,13 @@ TOPSRCDIR='@abs_top_srcdir@' TOPBUILDDIR='@abs_top_builddir@' FP_ISCMAKE=@ISCMAKE@ FP_ISMSVC=@ISMSVC@ + FP_WINVERMAJOR=@WINVERMAJOR@ + FP_WINVERBUILD=@WINVERBUILD@ FP_ISCYGWIN=@ISCYGWIN@ FP_ISMINGW=@ISMINGW@ FP_ISMSYS=@ISMSYS@ -FP_ISREGEDIT=@ISREGEDIT@ FP_USEPLUGINS=@USEPLUGINS@ +FP_ISREGEDIT=@ISREGEDIT@ # Feature flags FEATURE_HDF5=@HAS_HDF5@ @@ -150,5 +152,7 @@ ncgen3c0="${top_srcdir}/ncgen3/c0.cdl" ncgenc0="${top_srcdir}/ncgen/c0.cdl" ncgenc04="${top_srcdir}/ncgen/c0_4.cdl" +if test "x$FP_ISMSVC" = xyes || test "x$FP_ISCYGWIN" = xyes; then export LC_ALL="en_US.utf8"; fi + # Make sure we are in builddir (not execdir) cd $builddir diff --git a/unit_test/test_aws.c b/unit_test/test_aws.c index 826d3f82c9..8dd9e92764 100644 --- a/unit_test/test_aws.c +++ b/unit_test/test_aws.c @@ -15,7 +15,7 @@ Test the handling of aws profiles and regions. #include "ncrc.h" #include "ncpathmgr.h" -#define DEBUG +#undef DEBUG typedef struct ProfileTest { const char* profile; diff --git a/unit_test/test_pathcvt.c b/unit_test/test_pathcvt.c index 7f1d67b6be..7426e54953 100644 --- a/unit_test/test_pathcvt.c +++ b/unit_test/test_pathcvt.c @@ -13,7 +13,7 @@ Test the NCpathcvt #include "netcdf.h" #include "ncpathmgr.h" -#define DEBUG +#undef DEBUG #define NKINDS 4 static const int kinds[NKINDS] = {NCPD_NIX,NCPD_MSYS,NCPD_CYGWIN,NCPD_WIN};