From 36102e3c32b9d292dd139373d008f83e6b33fa94 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Tue, 8 Feb 2022 20:53:30 -0700 Subject: [PATCH 1/3] Improve UTF8 Support On Windows re: Issue https://github.com/Unidata/netcdf-c/issues/2190 The primary purpose of this PR is to improve the utf8 support for windows. This is persuant to a change in Windows that supports utf8 natively (almost). The almost means that it is still utf16 internally and the set of characters representable by utf8 is larger than those representable by utf16. This leaves open the question in the Issue about handling the Windows 1252 character set. This required the following changes: 1. Test the Windows build and major version in order to see if native utf8 is supported. 2. If native utf8 is supported, Modify dpathmgr.c to call the 8-bit version of the windows fopen() and open() functions. 3. In support of this, programs that use XGetOpt (Windows versions) need to get the command line as utf8 and then parse to arc+argv as utf8. This requires using a homegrown command line parser named XCommandLineToArgvA. 4. Add a utility program called "acpget" that prints out the current Windows code page and locale. Additionally, some technical debt was cleaned up as follows: 1. Unify all the places which attempt to read all or a part of a file into the dutil.c#NC_readfile code. 2. Similary unify all the code that creates temp files into dutil.c#NC_mktmp code. 3. Convert almost all remaining calls to fopen() and open() to NCfopen() and NCopen3(). This is to ensure that path management is used consistently. This touches a number of files. 4. extern->EXTERNL as needed to get it to work under Windows. --- .github/workflows/run_tests_osx.yml | 2 +- .github/workflows/run_tests_ubuntu.yml | 2 +- .github/workflows/run_tests_win_mingw.yml | 2 +- CMakeLists.txt | 44 +- RELEASE_NOTES.md | 3 +- config.h.cmake.in | 16 +- configure.ac | 11 + dap4_test/test_common.h | 22 +- dap4_test/test_data.c | 2 +- include/nc_tests.h | 2 + include/ncrc.h | 2 + libdap2/dceconstraints.c | 2 +- libdap4/d4file.c | 6 +- libdap4/d4includes.h | 1 + libdap4/d4util.c | 67 --- libdap4/ncd4.h | 132 +++--- libdispatch/dcrc32.c | 2 +- libdispatch/dinfermodel.c | 54 ++- libdispatch/dinstance.c | 7 +- libdispatch/dpathmgr.c | 255 ++++++++--- libdispatch/dutil.c | 73 +++- libdispatch/ncs3sdk.cpp | 6 +- libsrc/XGetopt.c | 492 ++++++++++++---------- libsrc/ncFile.c | 2 +- libsrc/ncio.c | 2 + nc_perf/tst_compress.c | 2 +- nc_perf/tst_compress_par.c | 2 +- nc_perf/tst_files2.c | 4 +- nc_perf/tst_files3.c | 2 +- nc_perf/tst_mem.c | 2 +- nc_test/tst_diskless5.c | 2 +- nc_test/tst_diskless6.c | 6 +- nc_test/tst_inmemory.c | 65 +-- nc_test/tst_misc.c | 2 +- nc_test4/tst_camrun.c | 2 +- nc_test4/tst_udf.c | 2 +- ncdap_test/t_auth.c | 6 +- ncdump/CMakeLists.txt | 14 +- ncdump/Makefile.am | 6 +- ncdump/acpget.c | 30 ++ ncdump/nctrunc.c | 3 +- ncdump/ocprint.c | 2 +- ncdump/run_cygutf8.sh | 10 + ncdump/test_unicode_path.sh | 3 - ncdump/tst_chunking.c | 2 +- ncdump/tst_cygutf8.c | 55 +++ ncdump/tst_unicode.c | 2 +- ncgen/dump.c | 2 +- ncgen3/run_nc4_tests.sh | 3 +- nczarr_test/ref_zarr_test_data.cdl.gz | Bin 0 -> 2702 bytes nczarr_test/run_interop.sh | 10 +- nczarr_test/s3util.c | 2 +- nczarr_test/tst_zchunks3.c | 2 +- nczarr_test/ut_json.c | 2 +- nczarr_test/zhex.c | 2 +- nczarr_test/zmapio.c | 2 +- nczarr_test/zs3parse.c | 2 +- test_common.in | 6 +- unit_test/test_aws.c | 2 +- unit_test/test_pathcvt.c | 2 +- 60 files changed, 855 insertions(+), 615 deletions(-) create mode 100644 ncdump/acpget.c create mode 100755 ncdump/run_cygutf8.sh create mode 100644 ncdump/tst_cygutf8.c create mode 100644 nczarr_test/ref_zarr_test_data.cdl.gz diff --git a/.github/workflows/run_tests_osx.yml b/.github/workflows/run_tests_osx.yml index 8e9d65a6fb..aa35ae8795 100644 --- a/.github/workflows/run_tests_osx.yml +++ b/.github/workflows/run_tests_osx.yml @@ -7,7 +7,7 @@ name: Run macOS-based netCDF Tests -on: [pull_request] +on: [pull_request,push] jobs: diff --git a/.github/workflows/run_tests_ubuntu.yml b/.github/workflows/run_tests_ubuntu.yml index eb410325a7..1ede4ad90c 100644 --- a/.github/workflows/run_tests_ubuntu.yml +++ b/.github/workflows/run_tests_ubuntu.yml @@ -4,7 +4,7 @@ name: Run Ubuntu/Linux netCDF Tests -on: [pull_request] +on: [pull_request,push] jobs: diff --git a/.github/workflows/run_tests_win_mingw.yml b/.github/workflows/run_tests_win_mingw.yml index 919c50f235..aaf999e6a4 100644 --- a/.github/workflows/run_tests_win_mingw.yml +++ b/.github/workflows/run_tests_win_mingw.yml @@ -7,7 +7,7 @@ name: Run MSYS2, MinGW64-based Tests -on: [ pull_request ] +on: [pull_request,push] jobs: diff --git a/CMakeLists.txt b/CMakeLists.txt index b90a7ceb61..fe76cfb5ca 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -55,6 +55,21 @@ IF(UNAME) set(TMP_BUILDNAME "${osname}-${osrel}-${cpu}") ENDIF() +# Define some Platforms +if(osname MATCHES "CYGWIN.*") +SET(ISCYGWIN yes) +endif() +if(osname MATCHES "Darwin.*") +SET(ISOSX yes) +endif() +if(MSVC) +SET(ISMSVC yes) +endif() +if(osname MATCHES "MINGW.*") +SET(ISMINGW yes) +SET(MINGW yes) +endif() + ### # Allow for some customization of the buildname. # This will make it easier to identify different builds, @@ -86,6 +101,7 @@ INCLUDE(GNUInstallDirs) IF(MSVC) SET(GLOBAL PROPERTY USE_FOLDERS ON) + ADD_COMPILE_OPTIONS("/utf-8") ENDIF() #Add custom CMake Module @@ -412,7 +428,6 @@ IF(NOT MSVC) ENDIF(BUILD_FORTRAN) ENDIF() - ### # Allow the user to specify libraries # to link against, similar to automakes 'LIBS' variable. @@ -1794,6 +1809,26 @@ IF(ENABLE_MMAP) ENDIF(ENABLE_MMAP) #CHECK_FUNCTION_EXISTS(alloca HAVE_ALLOCA) + +# Used in the `configure_file` calls below +SET(ISCMAKE "1") +IF(MSVC) +SET(ISMSVC ON CACHE BOOL "" FORCE) +SET(REGEDIT ON CACHE BOOL "" FORCE) +# Get windows major version and build number +EXECUTE_PROCESS(COMMAND "systeminfo" OUTPUT_VARIABLE WININFO) +IF(WININFO STREQUAL "") + SET(WVM 0) + SET(WVB 0) +ELSE() + STRING(REGEX MATCH "\nOS Version:[ \t]+[0-9.]+" WINVERLINE "${WININFO}") + STRING(REGEX REPLACE "[^0-9]*([0-9]+)[.]([0-9])+[.]([0-9]+)" "\\1" WVM "${WINVERLINE}") + STRING(REGEX REPLACE "[^0-9]*([0-9]+)[.]([0-9])+[.]([0-9]+)" "\\3" WVB "${WINVERLINE}") +ENDIF() +SET(WINVERMAJOR ${WVM} CACHE STRING "" FORCE) +SET(WINVERBUILD ${WVB} CACHE STRING "" FORCE) +ENDIF() + ##### # End system inspection checks. ##### @@ -2418,13 +2453,6 @@ configure_file( ${netCDF_SOURCE_DIR}/include/netcdf_dispatch.h.in ${netCDF_BINARY_DIR}/include/netcdf_dispatch.h @ONLY NEWLINE_STYLE LF) -# Used in the `configure_file` calls below -SET(ISCMAKE "1") -IF(MSVC) -SET(ISMSVC "1") -SET(REGEDIT 1) -ENDIF() - #### # Build test_common.sh ##### diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index e64d4f4517..b4fda7981a 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -7,8 +7,9 @@ This file contains a high-level description of this package's evolution. Release ## 4.8.2 - TBD +* [Bug Fix] Improve UTF8 support on windows to use utf8 natively. See [Github #????](https://github.com/Unidata/netcdf-c/pull/????). * [Enhancement] Add complete bitgroom support to NCZarr. See [Github #2197](https://github.com/Unidata/netcdf-c/pull/2197). -* [Bug Fix] Clean up the handling of deeply nested VLEN types. Marks nc_free_vlen() and nc_free_string as deprecated in favor of ncaux_reclaim_data(). See [Github #2179(https://github.com/Unidata/netcdf-c/pull/2179). +* [Bug Fix] Clean up the handling of deeply nested VLEN types. Marks nc_free_vlen() and nc_free_string as deprecated in favor of ncaux_reclaim_data(). See [Github #2179](https://github.com/Unidata/netcdf-c/pull/2179). * [Bug Fix] Make sure that netcdf.h accurately defines the flags in the open/create mode flags. See [Github #2183](https://github.com/Unidata/netcdf-c/pull/2183). * [Enhancement] Improve support for msys2+mingw platform. See [Github #2171](https://github.com/Unidata/netcdf-c/pull/2171). * [Bug Fix] Clean up the various inter-test dependencies in ncdump for CMake. See [Github #2168](https://github.com/Unidata/netcdf-c/pull/2168). diff --git a/config.h.cmake.in b/config.h.cmake.in index 87f473e1f3..693eab6128 100644 --- a/config.h.cmake.in +++ b/config.h.cmake.in @@ -157,12 +157,6 @@ are set when opening a binary file on Windows. */ /* if true, Allow dynamically loaded plugins */ #cmakedefine ENABLE_PLUGINS 1 -/* Do we have access to the Windows Registry */ -#cmakedefine REGEDIT 1 - -/* define the possible sources for remote test servers */ -#cmakedefine REMOTETESTSERVERS "${REMOTETESTSERVERS}" - /* if true, run extra tests which may not work yet */ #cmakedefine EXTRA_TESTS 1 @@ -505,6 +499,12 @@ with zip */ /* Define to the version of this package. */ #cmakedefine PACKAGE_VERSION "${netCDF_VERSION}" +/* Do we have access to the Windows Registry */ +#cmakedefine REGEDIT 1 + +/* define the possible sources for remote test servers */ +#cmakedefine REMOTETESTSERVERS "${REMOTETESTSERVERS}" + /* The size of `ulonglong` as computed by sizeof. */ #cmakedefine SIZEOF_ULONGLONG @SIZEOF_ULONGLONG@ @@ -626,6 +626,10 @@ with zip */ /* Version number of package */ #cmakedefine VERSION "${netCDF_VERSION}" +/* Capture Windows version and build */ +#cmakedefine WINVERMAJOR ${WINVERMAJOR} +#cmakedefine WINVERBUILD ${WINVERBUILD} + /* Define WORDS_BIGENDIAN to 1 if your processor stores words with the most significant byte first (like Motorola and SPARC, unlike Intel). */ #if defined AC_APPLE_UNIVERSAL_BUILD diff --git a/configure.ac b/configure.ac index 929d12f055..844bfff096 100644 --- a/configure.ac +++ b/configure.ac @@ -106,6 +106,15 @@ ISMINGW=yes ISMSYS=yes fi +# Get windows version info +WINVER=`systeminfo | sed -e '/^OS Version:/p' -ed | sed -e 's|[^0-9]*\([0-9.]*\).*|\1|'` +WINVERMAJOR=`echo $WVER | sed -e 's|\([^.]*\)[.]\([^.]*\)[.]\(.*\)|\1|'` +WINVERBUILD=`echo $WVER | sed -e 's|\([^.]*\)[.]\([^.]*\)[.]\(.*\)|\3|'` +if test "x$WINVERMAJOR" = x ; then WINVERMAJOR=0; fi +if test "x$WINVERBUILD" = x ; then WINVERBUILD=0; fi +AC_DEFINE_UNQUOTED([WINVERMAJOR], [$WINVERMAJOR], [windows version major]) +AC_DEFINE_UNQUOTED([WINVERBUILD], [$WINVERBUILD], [windows version build]) + AC_MSG_NOTICE([checking supported formats]) # An explicit disable of netcdf-4 | netcdf4 is treated as if it was disable-hdf5 @@ -1218,6 +1227,8 @@ AM_CONDITIONAL(ISMINGW, [test "x$ISMINGW" = xyes]) AM_CONDITIONAL(ISMSYS, [test "x$ISMSYS" = xyes]) AC_SUBST([ISMSVC], [${ISMSVC}]) + AC_SUBST([WINVERMAJOR], [${WINVERMAJOR}]) + AC_SUBST([WINVERBUILD], [${WINVERBUILD}]) AC_SUBST([ISCYGWIN], [${ISCYGWIN}]) AC_SUBST([ISOSX], [${ISOSX}]) AC_SUBST([ISMINGW], [${ISMINGW}]) diff --git a/dap4_test/test_common.h b/dap4_test/test_common.h index b2f4805ad0..70b1c22319 100644 --- a/dap4_test/test_common.h +++ b/dap4_test/test_common.h @@ -26,26 +26,6 @@ static char* outfile = NULL; static int ncid = 0; static int translatenc4 = 0; -static int -readfile(const char* filename, NCbytes* content) -{ - FILE* stream; - char part[1024]; - - stream = fopen(filename,"r"); - if(stream == NULL) return errno; - for(;;) { - size_t count = fread(part, 1, sizeof(part), stream); - if(count <= 0) break; - ncbytesappendn(content,part,count); - if(ferror(stream)) {fclose(stream); return NC_EIO;} - if(feof(stream)) break; - } - ncbytesnull(content); - fclose(stream); - return NC_NOERR; -} - static void fail(int code) { @@ -86,7 +66,7 @@ setup(int tdmr, int argc, char** argv) outfile = NULL; input = ncbytesnew(); output = ncbytesnew(); - if((ret = readfile(infile,input))) fail(ret); + if((ret = NC_readfile(infile,input))) fail(ret); { const char* trans = getenv("translatenc4"); if(trans != NULL) diff --git a/dap4_test/test_data.c b/dap4_test/test_data.c index 946c37b2ab..ab9f8f3a8d 100644 --- a/dap4_test/test_data.c +++ b/dap4_test/test_data.c @@ -12,7 +12,7 @@ Test the netcdf-4 data building process. #include #include "netcdf.h" -#define DEBUG +#undef DEBUG static void fail(int code) diff --git a/include/nc_tests.h b/include/nc_tests.h index 35e2ae3504..79cc9094c0 100644 --- a/include/nc_tests.h +++ b/include/nc_tests.h @@ -20,6 +20,8 @@ #include "netcdf.h" #include "netcdf_filter.h" #include "nc_logging.h" +#include "ncpathmgr.h" +#include "ncrc.h" #ifdef USE_PARALLEL #include "netcdf_par.h" #endif diff --git a/include/ncrc.h b/include/ncrc.h index 1b3228674b..342b13d024 100644 --- a/include/ncrc.h +++ b/include/ncrc.h @@ -103,6 +103,8 @@ EXTERNL int NC__testurl(const char* path, char** basenamep); EXTERNL int NC_isLittleEndian(void); EXTERNL char* NC_entityescape(const char* s); EXTERNL int NC_readfile(const char* filename, NCbytes* content); +EXTERNL int NC_readfilen(const char* filename, NCbytes* content, long long len); +EXTERNL int NC_readfileF(FILE* fp, NCbytes* content, long long len); EXTERNL int NC_writefile(const char* filename, size_t size, void* content); EXTERNL char* NC_mktmp(const char* base); EXTERNL int NC_getmodelist(const char* modestr, NClist** modelistp); diff --git a/libdap2/dceconstraints.c b/libdap2/dceconstraints.c index 800e5eb089..b4895768f8 100644 --- a/libdap2/dceconstraints.c +++ b/libdap2/dceconstraints.c @@ -13,7 +13,7 @@ #include "dapincludes.h" #include "dceparselex.h" -#define DEBUG +#undef DEBUG #define LBRACE "{" #define RBRACE "}" diff --git a/libdap4/d4file.c b/libdap4/d4file.c index 1c94040104..d0e41db059 100644 --- a/libdap4/d4file.c +++ b/libdap4/d4file.c @@ -395,17 +395,17 @@ set_curl_properties(NCD4INFO* d4info) FILE* f = NULL; char* fname = d4info->auth->curlflags.cookiejar; /* See if the file exists already */ - f = fopen(fname,"r"); + f = NCfopen(fname,"r"); if(f == NULL) { /* Ok, create it */ - f = fopen(fname,"w+"); + f = NCfopen(fname,"w+"); if(f == NULL) { fprintf(stderr,"Cookie file cannot be read and written: %s\n",fname); {ret= NC_EPERM; goto fail;} } } else { /* test if file can be written */ fclose(f); - f = fopen(fname,"r+"); + f = NCfopen(fname,"r+"); if(f == NULL) { fprintf(stderr,"Cookie file is cannot be written: %s\n",fname); {ret = NC_EPERM; goto fail;} diff --git a/libdap4/d4includes.h b/libdap4/d4includes.h index cc8b191c69..04151a683a 100644 --- a/libdap4/d4includes.h +++ b/libdap4/d4includes.h @@ -42,6 +42,7 @@ #include "ncuri.h" #include "nclog.h" #include "ncdap.h" +#include "ncpathmgr.h" #include "d4util.h" diff --git a/libdap4/d4util.c b/libdap4/d4util.c index a68041ecd2..b84d67877d 100644 --- a/libdap4/d4util.c +++ b/libdap4/d4util.c @@ -346,73 +346,6 @@ NCD4_elidenuls(char* s, size_t slen) return j; } -int -NCD4_readfile(const char* filename, NCbytes* content) -{ - int ret = NC_NOERR; - FILE* stream = NULL; - char part[1024]; - - stream = fopen(filename,"r"); - if(stream == NULL) {ret=errno; goto done;} - for(;;) { - size_t count = fread(part, 1, sizeof(part), stream); - if(count <= 0) break; - ncbytesappendn(content,part,count); - if(ferror(stream)) {ret = NC_EIO; goto done;} - if(feof(stream)) break; - } - ncbytesnull(content); -done: - if(stream) fclose(stream); - return ret; -} - -/** -Wrap mktmp and return the generated name -*/ - -int -NCD4_mktmp(const char* base, char** tmpnamep) -{ - int fd; - char tmp[NC_MAX_PATH]; -#ifdef HAVE_MKSTEMP - mode_t mask; -#endif - - strncpy(tmp,base,sizeof(tmp)); -#ifdef HAVE_MKSTEMP - strlcat(tmp,"XXXXXX", sizeof(tmp)); - /* Note Potential problem: old versions of this function - leave the file in mode 0666 instead of 0600 */ - mask=umask(0077); - fd = mkstemp(tmp); - (void)umask(mask); -#else /* !HAVE_MKSTEMP */ - /* Need to simulate by using some kind of pseudo-random number */ - { - int rno = rand(); - char spid[7]; - if(rno < 0) rno = -rno; - snprintf(spid,sizeof(spid),"%06d",rno); - strlcat(tmp,spid,sizeof(tmp)); -#if defined(_WIN32) || defined(_WIN64) - fd=open(tmp,O_RDWR|O_BINARY|O_CREAT, _S_IREAD|_S_IWRITE); -# else - fd=open(tmp,O_RDWR|O_CREAT|O_EXCL, S_IRWXU); -# endif - } -#endif /* !HAVE_MKSTEMP */ - if(fd < 0) { - nclog(NCLOGERR, "Could not create temp file: %s",tmp); - return THROW(NC_EPERM); - } else - close(fd); - if(tmpnamep) *tmpnamep = strdup(tmp); - return THROW(NC_NOERR); -} - void NCD4_hostport(NCURI* uri, char* space, size_t len) { diff --git a/libdap4/ncd4.h b/libdap4/ncd4.h index fe13d7f520..db9f74e3a5 100644 --- a/libdap4/ncd4.h +++ b/libdap4/ncd4.h @@ -64,114 +64,114 @@ defined here, including function-like #defines. /* DSP API wrappers */ #ifdef FIX -extern int dsp_getDMR(ND4dsp* dsp, DCR** dcrp); -extern int dsp_getDAP(ND4dsp* dsp, DCR** dcrp); -extern int dsp_close(ND4dsp* dsp); +EXTERNL int dsp_getDMR(ND4dsp* dsp, DCR** dcrp); +EXTERNL int dsp_getDAP(ND4dsp* dsp, DCR** dcrp); +EXTERNL int dsp_close(ND4dsp* dsp); /* DSP API */ -extern int dsp_open(const char* path, ND4dsp** dspp); +EXTERNL int dsp_open(const char* path, ND4dsp** dspp); #endif /**************************************************/ /* From d4http.c */ -extern long NCD4_fetchhttpcode(CURL* curl); -extern int NCD4_fetchurl_file(CURL* curl, const char* url, FILE* stream, d4size_t* sizep, long* filetime); -extern int NCD4_fetchurl(CURL* curl, const char* url, NCbytes* buf, long* filetime, int* httpcode); -extern int NCD4_curlopen(CURL** curlp); -extern void NCD4_curlclose(CURL* curl); -extern int NCD4_fetchlastmodified(CURL* curl, char* url, long* filetime); -extern int NCD4_ping(const char* url); +EXTERNL long NCD4_fetchhttpcode(CURL* curl); +EXTERNL int NCD4_fetchurl_file(CURL* curl, const char* url, FILE* stream, d4size_t* sizep, long* filetime); +EXTERNL int NCD4_fetchurl(CURL* curl, const char* url, NCbytes* buf, long* filetime, int* httpcode); +EXTERNL int NCD4_curlopen(CURL** curlp); +EXTERNL void NCD4_curlclose(CURL* curl); +EXTERNL int NCD4_fetchlastmodified(CURL* curl, char* url, long* filetime); +EXTERNL int NCD4_ping(const char* url); /* From d4read.c */ -extern int NCD4_readDMR(NCD4INFO* state, int flags); -extern int NCD4_readDAP(NCD4INFO* state, int flags); -extern int NCD4_seterrormessage(NCD4meta* metadata, size_t len, char* msg); +EXTERNL int NCD4_readDMR(NCD4INFO* state, int flags); +EXTERNL int NCD4_readDAP(NCD4INFO* state, int flags); +EXTERNL int NCD4_seterrormessage(NCD4meta* metadata, size_t len, char* msg); /* From d4parser.c */ -extern int NCD4_parse(NCD4meta*); -extern NCD4node* NCD4_findAttr(NCD4node* container, const char* attrname); -extern NCD4node* NCD4_groupFor(NCD4node* node); -extern int NCD4_defineattr(NCD4meta* meta, NCD4node* parent, const char* aname, const char* typename, NCD4node** attrp); +EXTERNL int NCD4_parse(NCD4meta*); +EXTERNL NCD4node* NCD4_findAttr(NCD4node* container, const char* attrname); +EXTERNL NCD4node* NCD4_groupFor(NCD4node* node); +EXTERNL int NCD4_defineattr(NCD4meta* meta, NCD4node* parent, const char* aname, const char* typename, NCD4node** attrp); /* From d4printer.c */ -extern int NCD4_print(NCD4meta*, NCbytes* output); +EXTERNL int NCD4_print(NCD4meta*, NCbytes* output); /* From d4meta.c */ -extern NCD4meta* NCD4_newmeta(NCD4INFO*); -extern void NCD4_attachraw(NCD4meta*, size_t size, void* rawdata); -extern void NCD4_reclaimMeta(NCD4meta*); -extern void NCD4_resetMeta(NCD4meta*); -extern void reclaimNode(NCD4node* node); -extern void NCD4_setdebuglevel(NCD4meta*,int); -extern int NCD4_metabuild(NCD4meta*, int ncid); -extern size_t NCD4_computeTypeSize(NCD4meta*, NCD4node* type); -extern int NCD4_findvar(NC* ncp, int ncid, int varid, NCD4node** varp, NCD4node** grpp); +EXTERNL NCD4meta* NCD4_newmeta(NCD4INFO*); +EXTERNL void NCD4_attachraw(NCD4meta*, size_t size, void* rawdata); +EXTERNL void NCD4_reclaimMeta(NCD4meta*); +EXTERNL void NCD4_resetMeta(NCD4meta*); +EXTERNL void reclaimNode(NCD4node* node); +EXTERNL void NCD4_setdebuglevel(NCD4meta*,int); +EXTERNL int NCD4_metabuild(NCD4meta*, int ncid); +EXTERNL size_t NCD4_computeTypeSize(NCD4meta*, NCD4node* type); +EXTERNL int NCD4_findvar(NC* ncp, int ncid, int varid, NCD4node** varp, NCD4node** grpp); /* From d4chunk.c */ -extern int NCD4_dechunk(NCD4meta*); -extern int NCD4_infermode(NCD4meta* meta); +EXTERNL int NCD4_dechunk(NCD4meta*); +EXTERNL int NCD4_infermode(NCD4meta* meta); struct NCD4serial; -extern void NCD4_resetSerial(struct NCD4serial* serial, size_t rawsize, void* rawdata); +EXTERNL void NCD4_resetSerial(struct NCD4serial* serial, size_t rawsize, void* rawdata); /* From d4swap.c */ -extern int NCD4_swapdata(NCD4meta*, NClist* topvars); +EXTERNL int NCD4_swapdata(NCD4meta*, NClist* topvars); /* From d4fix.c */ -extern int NCD4_delimit(NCD4meta*, NCD4node* var, void** offsetp); -extern int NCD4_moveto(NCD4meta*, NCD4node* var, d4size_t count, void** offsetp); -extern int NCD4_toposort(NCD4meta*); +EXTERNL int NCD4_delimit(NCD4meta*, NCD4node* var, void** offsetp); +EXTERNL int NCD4_moveto(NCD4meta*, NCD4node* var, d4size_t count, void** offsetp); +EXTERNL int NCD4_toposort(NCD4meta*); /* From d4data.c */ -extern int NCD4_processdata(NCD4meta*); -extern int NCD4_fillinstance(NCD4meta*, NCD4node* type, void** offsetp, void** dstp, NClist* blobs); -extern int NCD4_getToplevelVars(NCD4meta* meta, NCD4node* group, NClist* toplevel); +EXTERNL int NCD4_processdata(NCD4meta*); +EXTERNL int NCD4_fillinstance(NCD4meta*, NCD4node* type, void** offsetp, void** dstp, NClist* blobs); +EXTERNL int NCD4_getToplevelVars(NCD4meta* meta, NCD4node* group, NClist* toplevel); /* From d4util.c */ -extern d4size_t NCD4_dimproduct(NCD4node* node); -extern size_t NCD4_typesize(nc_type tid); -extern int NCD4_isLittleEndian(void);/* Return 1 if this machine is little endian */ -extern int NCD4_errorNC(int code, const int line, const char* file); -extern int NCD4_error(int code, const int line, const char* file, const char* fmt, ...); -extern char* NCD4_makeFQN(NCD4node* node); -extern char* NCD4_makeName(NCD4node*,const char* sep); -extern int NCD4_parseFQN(const char* fqn0, NClist* pieces); -extern char* NCD4_deescape(const char* esc); -extern char* NCD4_entityescape(const char* s); -extern size_t NCD4_elidenuls(char* s, size_t slen); -extern void* NCD4_getheader(void* p, NCD4HDR* hdr, int hostlittleendian); -extern void NCD4_reporterror(NCD4INFO* state); +EXTERNL d4size_t NCD4_dimproduct(NCD4node* node); +EXTERNL size_t NCD4_typesize(nc_type tid); +EXTERNL int NCD4_isLittleEndian(void);/* Return 1 if this machine is little endian */ +EXTERNL int NCD4_errorNC(int code, const int line, const char* file); +EXTERNL int NCD4_error(int code, const int line, const char* file, const char* fmt, ...); +EXTERNL char* NCD4_makeFQN(NCD4node* node); +EXTERNL char* NCD4_makeName(NCD4node*,const char* sep); +EXTERNL int NCD4_parseFQN(const char* fqn0, NClist* pieces); +EXTERNL char* NCD4_deescape(const char* esc); +EXTERNL char* NCD4_entityescape(const char* s); +EXTERNL size_t NCD4_elidenuls(char* s, size_t slen); +EXTERNL void* NCD4_getheader(void* p, NCD4HDR* hdr, int hostlittleendian); +EXTERNL void NCD4_reporterror(NCD4INFO* state); /* From d4dump.c */ -extern void NCD4_dumpbytes(size_t size, const void* data0, int swap); -extern void NCD4_tagdump(size_t size, const void* data0, int swap, const char* tag); -extern void NCD4_dumpvars(NCD4node* group); -extern union ATOMICS* NCD4_dumpatomic(NCD4node* var, void* data); +EXTERNL void NCD4_dumpbytes(size_t size, const void* data0, int swap); +EXTERNL void NCD4_tagdump(size_t size, const void* data0, int swap, const char* tag); +EXTERNL void NCD4_dumpvars(NCD4node* group); +EXTERNL union ATOMICS* NCD4_dumpatomic(NCD4node* var, void* data); /* From d4rc.c */ -extern int NCD4_rcload(void); -extern int NCD4_rcprocess(NCD4INFO* info); -extern void NCD4_rcfree(NClist* rc); -extern char* NCD4_rclookup(char* key, char* hostport); -extern int NCD4_parseproxy(NCD4INFO* info, const char* surl); -extern int NCD4_rcdefault(NCD4INFO*); +EXTERNL int NCD4_rcload(void); +EXTERNL int NCD4_rcprocess(NCD4INFO* info); +EXTERNL void NCD4_rcfree(NClist* rc); +EXTERNL char* NCD4_rclookup(char* key, char* hostport); +EXTERNL int NCD4_parseproxy(NCD4INFO* info, const char* surl); +EXTERNL int NCD4_rcdefault(NCD4INFO*); /* From d4cvt.c */ -extern int NCD4_convert(nc_type srctype, nc_type dsttype, char* memory0, char* value0, size_t count); +EXTERNL int NCD4_convert(nc_type srctype, nc_type dsttype, char* memory0, char* value0, size_t count); /* d4file.c */ -extern void NCD4_applyclientparamcontrols(NCD4INFO*); -extern int NCD4_readDMRorDAP(NCD4INFO* d4info, NCD4mode mode); +EXTERNL void NCD4_applyclientparamcontrols(NCD4INFO*); +EXTERNL int NCD4_readDMRorDAP(NCD4INFO* d4info, NCD4mode mode); /* ncd4dispatch.c */ struct NC_reservedatt; /*forward*/ -extern const struct NC_reservedatt* NCD4_lookupreserved(const char* name); +EXTERNL const struct NC_reservedatt* NCD4_lookupreserved(const char* name); /* Add an extra function whose sole purpose is to allow configure(.ac) to test for the presence of this code. */ -extern int nc__dap4(void); +EXTERNL int nc__dap4(void); /**************************************************/ /* Macro defined functions */ diff --git a/libdispatch/dcrc32.c b/libdispatch/dcrc32.c index 3878841e88..d804774e3d 100644 --- a/libdispatch/dcrc32.c +++ b/libdispatch/dcrc32.c @@ -199,7 +199,7 @@ local void make_crc_table() { FILE *out; - out = fopen("dcrc32.h", "w"); + out = NCfopen("dcrc32.h", "w"); if (out == NULL) return; fprintf(out, "/* dcrc32.h -- tables for rapid CRC calculation\n"); fprintf(out, " * Generated automatically by dcrc32.c\n */\n\n"); diff --git a/libdispatch/dinfermodel.c b/libdispatch/dinfermodel.c index fb58d18c23..d406908a88 100644 --- a/libdispatch/dinfermodel.c +++ b/libdispatch/dinfermodel.c @@ -1263,9 +1263,8 @@ openmagic(struct MagicFile* file) } else #endif /* USE_PARALLEL */ { - if(file->path == NULL || strlen(file->path)==0) - {status = NC_EINVAL; goto done;} - + if (file->path == NULL || strlen(file->path) == 0) + {status = NC_EINVAL; goto done;} file->fp = NCfopen(file->path, "r"); if(file->fp == NULL) {status = errno; goto done;} @@ -1296,6 +1295,8 @@ static int readmagic(struct MagicFile* file, long pos, char* magic) { int status = NC_NOERR; + NCbytes* buf = ncbytesnew(); + memset(magic,0,MAGIC_NUMBER_LEN); if(fIsSet(file->omode,NC_INMEMORY)) { char* mempos; @@ -1315,19 +1316,18 @@ readmagic(struct MagicFile* file, long pos, char* magic) if(file->iss3) { if((status = NC_s3sdkread(file->s3client,file->s3.bucket,file->s3.rootkey,start,count,(void*)magic,&file->errmsg))) {goto done;} - } else + } + else #endif - { - NCbytes* buf = ncbytesnew(); - status = nc_http_read(file->state,file->curlurl,start,count,buf); - if(status == NC_NOERR) { - if(ncbyteslength(buf) != count) - status = NC_EINVAL; - else - memcpy(magic,ncbytescontents(buf),count); - } - ncbytesfree(buf); - } + { + status = nc_http_read(file->state, file->curlurl, start, count, buf); + if (status == NC_NOERR) { + if (ncbyteslength(buf) != count) + status = NC_EINVAL; + else + memcpy(magic, ncbytescontents(buf), count); + } + } #endif } else { #ifdef USE_PARALLEL @@ -1337,23 +1337,21 @@ readmagic(struct MagicFile* file, long pos, char* magic) if((retval = MPI_File_read_at_all(file->fh, pos, magic, MAGIC_NUMBER_LEN, MPI_CHAR, &mstatus)) != MPI_SUCCESS) {status = NC_EPARINIT; goto done;} - } else + } + else #endif /* USE_PARALLEL */ - { - int count; - int i = fseek(file->fp,pos,SEEK_SET); - if(i < 0) - {status = errno; goto done;} - for(i=0;ifp); - if(count == 0 || ferror(file->fp)) - {status = errno; goto done;} - i += count; - } - } + { /* Ordinary read */ + long i; + i = fseek(file->fp, pos, SEEK_SET); + if (i < 0) { status = errno; goto done; } + ncbytessetlength(buf, 0); + if ((status = NC_readfileF(file->fp, buf, MAGIC_NUMBER_LEN))) goto done; + memcpy(magic, ncbytescontents(buf), MAGIC_NUMBER_LEN); + } } done: + ncbytesfree(buf); if(file && file->fp) clearerr(file->fp); return check(status); } diff --git a/libdispatch/dinstance.c b/libdispatch/dinstance.c index 5a95d3cbc9..64756929bd 100644 --- a/libdispatch/dinstance.c +++ b/libdispatch/dinstance.c @@ -153,14 +153,15 @@ reclaim_datar(int ncid, nc_type xtype, Position* offset) int klass, isf; if((stat = NC4_inq_type_fixed_size(ncid,xtype,&isf))) goto done; + + /* Get relevant type info */ + if((stat = NC_inq_any_type(ncid,xtype,NULL,&xsize,&basetype,&nfields,&klass))) goto done; + if(isf) { /* no need to reclaim anything */ offset->offset += xsize; goto done; } - /* Get relevant type info */ - if((stat = NC_inq_any_type(ncid,xtype,NULL,&xsize,&basetype,&nfields,&klass))) goto done; - switch (xtype) { case NC_STRING: { char** sp = (char**)(offset->memory + offset->offset); diff --git a/libdispatch/dpathmgr.c b/libdispatch/dpathmgr.c index c95273892d..2ced44ae2e 100644 --- a/libdispatch/dpathmgr.c +++ b/libdispatch/dpathmgr.c @@ -25,13 +25,9 @@ #include #include #include -#include #include #endif -#ifdef __hpux #include -#endif - #include "netcdf.h" #include "ncpathmgr.h" #include "nclog.h" @@ -42,7 +38,7 @@ #undef DEBUGPATH static int pathdebug = -1; -#define DEBUG +#undef DEBUG #ifdef DEBUG #define REPORT(e,msg) report((e),(msg),__LINE__) @@ -55,6 +51,14 @@ static int pathdebug = -1; #define mkdir _mkdir #define rmdir _rmdir #define getcwd _getcwd + +#if WINVERMAJOR > 10 || (WINVERMAJOR == 10 && WINVERBUILD >= 17134) +/* Should be possible to use UTF8 directly */ +#define WINUTF8 +#else +#undef WINUTF8 +#endif + #endif /* @@ -87,6 +91,11 @@ static const struct Path { /* Keep the working directory kind and drive */ static char wdprefix[8192]; +/* The current codepage */ +#ifdef _WIN32 +static int acp = -1; +#endif + /* Keep CYGWIN/MSYS2 mount point */ static struct MountPoint { int defined; @@ -265,6 +274,7 @@ static void pathinit(void) { if(pathinitialized) return; + pathinitialized = 1; /* avoid recursion */ /* Check for path debug env vars */ if(pathdebug < 0) { @@ -272,6 +282,12 @@ pathinit(void) pathdebug = (s == NULL ? 0 : 1); } (void)getwdpath(); + +#ifdef _WIN32 + /* Get the current code page */ + acp = GetACP(); +#endif + memset(&mountpoint,0,sizeof(mountpoint)); #ifdef REGEDIT { /* See if we can get the MSYS2 prefix from the registry */ @@ -367,31 +383,44 @@ NCfopen(const char* path, const char* flags) { int stat = NC_NOERR; FILE* f = NULL; - char* bflags = NULL; + char bflags[64]; + char* path8 = NULL; /* ACP -> UTF=8 */ + char* bflags8 = NULL; char* cvtpath = NULL; wchar_t* wpath = NULL; wchar_t* wflags = NULL; size_t flaglen = strlen(flags)+1+1; - bflags = (char*)malloc(flaglen); bflags[0] = '\0'; - strlcat(bflags,flags,flaglen); + strlcat(bflags, flags, sizeof(bflags)); #ifdef _WIN32 - strlcat(bflags,"b",flaglen); + strlcat(bflags,"b",sizeof(bflags)); +#endif + + /* First, convert from current Code Page to utf8 */ + if((stat = ansi2utf8(path,&path8))) goto done; + if((stat = ansi2utf8(bflags,&bflags8))) goto done; + + /* Localize */ + if((cvtpath = NCpathcvt(path8))==NULL) goto done; + +#ifdef WINUTF8 + if(acp == CP_UTF8) { + /* This should take utf8 directly */ + f = fopen(cvtpath,bflags8); + } else #endif - cvtpath = NCpathcvt(path); - if(cvtpath == NULL) return NULL; - /* Convert from local to wide */ - if((stat = utf82wide(cvtpath,&wpath))) - {REPORT(stat,"utf282wide"); goto done;} - if((stat = ansi2wide(bflags,&wflags))) - {REPORT(stat,"ansi2wide"); goto done;} - f = _wfopen(wpath,wflags); + { + /* Convert from utf8 to wide */ + if((stat = utf82wide(cvtpath,&wpath))) goto done; + if((stat = utf82wide(bflags8,&wflags))) goto done; + f = _wfopen(wpath,wflags); + } done: nullfree(cvtpath); + nullfree(path8); nullfree(wpath); nullfree(wflags); - nullfree(bflags); return f; } @@ -402,17 +431,32 @@ NCopen3(const char* path, int flags, int mode) int stat = NC_NOERR; int fd = -1; char* cvtpath = NULL; + char* path8 = NULL; wchar_t* wpath = NULL; - cvtpath = NCpathcvt(path); - if(cvtpath == NULL) goto done; - /* Convert from utf8 to wide */ - if((stat = utf82wide(cvtpath,&wpath))) goto done; + + /* First, convert from current Code Page to utf8 */ + if((stat = ansi2utf8(path,&path8))) goto done; + + if((cvtpath = NCpathcvt(path8))==NULL) goto done; + #ifdef _WIN32 flags |= O_BINARY; #endif - fd = _wopen(wpath,flags,mode); +#ifdef WINUTF8 + if(acp == CP_UTF8) { + /* This should take utf8 directly */ + fd = _open(cvtpath,flags,mode); + } else +#endif + { + /* Convert from utf8 to wide */ + if((stat = utf82wide(cvtpath,&wpath))) goto done; + fd = _wopen(wpath,flags,mode); + } + done: nullfree(cvtpath); + nullfree(path8); nullfree(wpath); return fd; } @@ -433,7 +477,7 @@ NCopendir(const char* path) char* cvtname = NCpathcvt(path); if(cvtname == NULL) return NULL; ent = opendir(cvtname); - free(cvtname); + nullfree(cvtname); return ent; } @@ -456,17 +500,32 @@ EXTERNL int NCaccess(const char* path, int mode) { - int status = 0; + int stat = 0; char* cvtpath = NULL; + char* path8 = NULL; wchar_t* wpath = NULL; - if((cvtpath = NCpathcvt(path)) == NULL) - {status = EINVAL; goto done;} - if((status = utf82wide(cvtpath,&wpath))) {status = ENOENT; goto done;} - if(_waccess(wpath,mode) < 0) {status = errno; goto done;} + + /* First, convert from current Code Page to utf8 */ + if((stat = ansi2utf8(path,&path8))) goto done; + + if((cvtpath = NCpathcvt(path8)) == NULL) {stat = EINVAL; goto done;} +#ifdef WINUTF8 + if(acp == CP_UTF8) { + /* This should take utf8 directly */ + if(_access(cvtpath,mode) < 0) {stat = errno; goto done;} + } else +#endif + { + /* Convert from utf8 to wide */ + if((stat = utf82wide(cvtpath,&wpath))) goto done; + if(_waccess(wpath,mode) < 0) {stat = errno; goto done;} + } + done: - free(cvtpath); - free(wpath); - errno = status; + nullfree(cvtpath); + nullfree(path8); + nullfree(wpath); + errno = stat; return (errno?-1:0); } @@ -475,14 +534,29 @@ int NCremove(const char* path) { int status = 0; + char* path8 = NULL; char* cvtpath = NULL; wchar_t* wpath = NULL; - if((cvtpath = NCpathcvt(path)) == NULL) {status=ENOMEM; goto done;} - if((status = utf82wide(cvtpath,&wpath))) {status = ENOENT; goto done;} - if(_wremove(wpath) < 0) {status = errno; goto done;} + + /* First, convert from current Code Page to utf8 */ + if((status = ansi2utf8(path,&path8))) goto done; + + if((cvtpath = NCpathcvt(path8)) == NULL) {status=ENOMEM; goto done;} +#ifdef WINUTF8 + if(acp == CP_UTF8) { + /* This should take utf8 directly */ + if(remove(cvtpath) < 0) {status = errno; goto done;} + } else +#endif + { + if((status = utf82wide(cvtpath,&wpath))) {status = ENOENT; goto done;} + if(_wremove(wpath) < 0) {status = errno; goto done;} + } + done: - free(cvtpath); - free(wpath); + nullfree(cvtpath); + nullfree(path8); + nullfree(wpath); errno = status; return (errno?-1:0); } @@ -493,13 +567,28 @@ NCmkdir(const char* path, int mode) { int status = 0; char* cvtpath = NULL; + char* path8 = NULL; wchar_t* wpath = NULL; - if((cvtpath = NCpathcvt(path)) == NULL) {status=ENOMEM; goto done;} - if((status = utf82wide(cvtpath,&wpath))) {status = ENOENT; goto done;} - if(_wmkdir(wpath) < 0) {status = errno; goto done;} + + /* First, convert from current Code Page to utf8 */ + if((status = ansi2utf8(path,&path8))) goto done; + + if((cvtpath = NCpathcvt(path8)) == NULL) {status=ENOMEM; goto done;} +#ifdef WINUTF8 + if(acp == CP_UTF8) { + /* This should take utf8 directly */ + if(_mkdir(cvtpath) < 0) {status = errno; goto done;} + } else +#endif + { + if((status = utf82wide(cvtpath,&wpath))) {status = ENOENT; goto done;} + if(_wmkdir(wpath) < 0) {status = errno; goto done;} + } + done: - free(cvtpath); - free(wpath); + nullfree(cvtpath); + nullfree(path8); + nullfree(wpath); errno = status; return (errno?-1:0); } @@ -509,10 +598,18 @@ int NCrmdir(const char* path) { int status = 0; - char* cvtname = NCpathcvt(path); - if(cvtname == NULL) {errno = ENOENT; return -1;} + char* cvtname = NULL; + char* path8 = NULL; + + /* First, convert from current Code Page to utf8 */ + if((status = ansi2utf8(path,&path8))) goto done; + + cvtname = NCpathcvt(path8); + if(cvtname == NULL) {errno = ENOENT; status = -1;} status = rmdir(cvtname); - free(cvtname); +done: + nullfree(cvtname); + nullfree(path8); return status; } @@ -589,13 +686,26 @@ NCstat(const char* path, struct stat* buf) { int status = 0; char* cvtpath = NULL; + char* path8 = NULL; wchar_t* wpath = NULL; - if((cvtpath = NCpathcvt(path)) == NULL) {status=ENOMEM; goto done;} - if((status = utf82wide(cvtpath,&wpath))) {status = ENOENT; goto done;} - if(_wstat64(wpath,buf) < 0) {status = errno; goto done;} + + if((status = ansi2utf8(path,&path8))) goto done; + + if((cvtpath = NCpathcvt(path8)) == NULL) {status=ENOMEM; goto done;} +#ifdef WINUTF8 + if(acp == CP_UTF8) { + if(_stat64(cvtpath,buf) < 0) {status = errno; goto done;} + } else +#endif + { + if((status = utf82wide(cvtpath,&wpath))) {status = ENOENT; goto done;} + if(_wstat64(wpath,buf) < 0) {status = errno; goto done;} + } + done: - free(cvtpath); - free(wpath); + nullfree(cvtpath); + nullfree(path8); + nullfree(wpath); errno = status; return (errno?-1:0); } @@ -672,6 +782,7 @@ parsepath(const char* inpath, struct Path* path) memset(path,0,sizeof(struct Path)); if(inpath == NULL) goto done; /* defensive driving */ + if(!pathinitialized) pathinit(); #if 0 /* Convert to UTF8 */ @@ -999,7 +1110,7 @@ getwdpath(void) wpath = _wgetcwd(wcwd, 8192); path = NULL; stat = wide2utf8(wpath, &path); - free(wcwd); + nullfree(wcwd); if (stat) return stat; strlcat(wdprefix,path,sizeof(wdprefix)); } @@ -1067,9 +1178,9 @@ NCgetkindname(int kind) #ifdef WINPATH /** - * Converts the filename from Locale character set (presumably some + * Converts the file path from current character set (presumably some * ANSI character set like ISO-Latin-1 or UTF-8 to UTF-8 - * @param local Pointer to a nul-terminated string in locale char set. + * @param local Pointer to a nul-terminated string in current char set. * @param u8p Pointer for returning the output utf8 string * * @return NC_NOERR return converted filename @@ -1078,21 +1189,32 @@ NCgetkindname(int kind) * */ static int -ansi2utf8(const char* local, char** u8p) +ansi2utf8(const char* path, char** u8p) { int stat=NC_NOERR; char* u8 = NULL; int n; wchar_t* u16 = NULL; + if(path == NULL) goto done; + + if(!pathinitialized) pathinit(); + +#ifdef WINUTF8 + if(acp == CP_UTF8) { /* Current code page is UTF8 and Windows supports */ + u8 = strdup(path); + if(u8 == NULL) stat = NC_ENOMEM; + } else +#endif + /* Use wide character conversion plus current code page*/ { /* Get length of the converted string */ - n = MultiByteToWideChar(CP_ACP, 0, local, -1, NULL, 0); + n = MultiByteToWideChar(acp, 0, path, -1, NULL, 0); if (!n) {stat = NC_EINVAL; goto done;} - if((u16 = malloc(sizeof(wchar_t) * n))==NULL) + if((u16 = malloc(sizeof(wchar_t) * (n)))==NULL) {stat = NCTHROW(NC_ENOMEM); goto done;} /* do the conversion */ - if (!MultiByteToWideChar(CP_ACP, 0, local, -1, u16, n)) + if (!MultiByteToWideChar(CP_ACP, 0, path, -1, u16, n)) {stat = NC_EINVAL; goto done;} /* Now reverse the process to produce utf8 */ n = WideCharToMultiByte(CP_UTF8, 0, u16, -1, NULL, 0, NULL, NULL); @@ -1102,8 +1224,8 @@ ansi2utf8(const char* local, char** u8p) if (!WideCharToMultiByte(CP_UTF8, 0, u16, -1, u8, n, NULL, NULL)) {stat = NC_EINVAL; goto done;} } - if(u8p) {*u8p = u8; u8 = NULL;} done: + if(u8p) {*u8p = u8; u8 = NULL;} nullfree(u8); nullfree(u16); return stat; @@ -1116,6 +1238,8 @@ ansi2wide(const char* local, wchar_t** u16p) wchar_t* u16 = NULL; int n; + if(!pathinitialized) pathinit(); + /* Get length of the converted string */ n = MultiByteToWideChar(CP_ACP, 0, local, -1, NULL, 0); if (!n) {stat = NC_EINVAL; goto done;} @@ -1137,6 +1261,8 @@ utf82wide(const char* utf8, wchar_t** u16p) wchar_t* u16 = NULL; int n; + if(!pathinitialized) pathinit(); + /* Get length of the converted string */ n = MultiByteToWideChar(CP_UTF8, 0, utf8, -1, NULL, 0); if (!n) {stat = NC_EINVAL; goto done;} @@ -1158,6 +1284,8 @@ wide2utf8(const wchar_t* u16, char** u8p) char* u8 = NULL; int n; + if(!pathinitialized) pathinit(); + /* Get length of the converted string */ n = WideCharToMultiByte(CP_UTF8, 0, u16, -1, NULL, 0, NULL, NULL); if (!n) {stat = NC_EINVAL; goto done;} @@ -1172,17 +1300,6 @@ wide2utf8(const wchar_t* u16, char** u8p) return stat; } -/* Set locale */ -void -nc_setlocale_utf8(void) -{ -#ifdef __hpux - setlocale(LC_CTYPE,""); -#else - setlocale(LC_ALL,"C.UTF8"); -#endif -} - #endif /*WINPATH*/ static char* diff --git a/libdispatch/dutil.c b/libdispatch/dutil.c index fc8a018d1a..06f68fe87e 100644 --- a/libdispatch/dutil.c +++ b/libdispatch/dutil.c @@ -195,7 +195,7 @@ NC_shellUnescape(const char* esc) } /** -Wrap mktmp and return the generated path, +Wrap mkstemp and return the generated path, or null if failed. Base is the base file path. XXXXX is appended to allow mktmp add its unique id. @@ -208,16 +208,41 @@ NC_mktmp(const char* base) int fd = -1; char* tmp = NULL; size_t len; + int tries; +#define MAXTRIES 4 +#ifdef HAVE_MKSTEMP + mode_t mask; +#endif len = strlen(base)+6+1; - if((tmp = (char*)malloc(len))==NULL) + if((tmp = (char*)calloc(1,len))==NULL) goto done; - strncpy(tmp,base,len); +#ifdef HAVE_MKSTEMP + strlcat(tmp,base,len); strlcat(tmp, "XXXXXX", len); + mask=umask(0077); fd = NCmkstemp(tmp); + (void)umask(mask); +#else /* !HAVE_MKSTEMP */ + /* Need to simulate by using some kind of pseudo-random number */ + for(tries=0;tries= 0) break; /* sucess */ + fd = -1; /* try again */ + } +#endif /* !HAVE_MKSTEMP */ if(fd < 0) { - nclog(NCLOGERR, "Could not create temp file: %s",tmp); - goto done; + nclog(NCLOGERR, "Could not create temp file: %s",tmp); + nullfree(tmp); + tmp = NULL; + goto done; } done: if(fd >= 0) close(fd); @@ -226,23 +251,47 @@ NC_mktmp(const char* base) int NC_readfile(const char* filename, NCbytes* content) +{ + int stat; + stat = NC_readfilen(filename, content, -1); + return stat; +} + +int +NC_readfilen(const char* filename, NCbytes* content, long long amount) { int ret = NC_NOERR; FILE* stream = NULL; - char part[1024]; stream = NCfopen(filename,"r"); if(stream == NULL) {ret=errno; goto done;} - for(;;) { + ret = NC_readfileF(stream,content,amount); + if (stream) fclose(stream); +done: + return ret; +} + +int +NC_readfileF(FILE* stream, NCbytes* content, long long amount) +{ + int ret = NC_NOERR; + long long red = 0; + char part[1024]; + + while(amount < 0 || red < amount) { size_t count = fread(part, 1, sizeof(part), stream); - if(count <= 0) break; - ncbytesappendn(content,part,count); if(ferror(stream)) {ret = NC_EIO; goto done;} - if(feof(stream)) break; + if(count > 0) ncbytesappendn(content,part,(unsigned long)count); + red += count; + if (feof(stream)) break; + } + /* Keep only amount */ + if(amount >= 0) { + if(red > amount) ncbytessetlength(content,amount); /* read too much */ + if(red < amount) ret = NC_ETRUNC; /* |file| < amount */ } ncbytesnull(content); done: - if(stream) fclose(stream); return ret; } @@ -263,8 +312,8 @@ NC_writefile(const char* filename, size_t size, void* content) while(remain > 0) { size_t written = fwrite(p, 1, remain, stream); if(ferror(stream)) {ret = NC_EIO; goto done;} - if(feof(stream)) break; remain -= written; + if (feof(stream)) break; } done: if(stream) fclose(stream); diff --git a/libdispatch/ncs3sdk.cpp b/libdispatch/ncs3sdk.cpp index 2c51b3e762..1b0feaec02 100644 --- a/libdispatch/ncs3sdk.cpp +++ b/libdispatch/ncs3sdk.cpp @@ -73,9 +73,8 @@ NC_s3sdkinitialize(void) ncs3_finalized = 0; NCTRACE(11,NULL); Aws::InitAPI(ncs3options); - NCUNTRACE(NC_NOERR); } - return 1; + return NCUNTRACE(NC_NOERR); } EXTERNL int @@ -86,9 +85,8 @@ NC_s3sdkfinalize(void) ncs3_finalized = 1; NCTRACE(11,NULL); Aws::ShutdownAPI(ncs3options); - NCUNTRACE(NC_NOERR); } - return 1; + return NCUNTRACE(NC_NOERR); } static char* diff --git a/libsrc/XGetopt.c b/libsrc/XGetopt.c index 5a4894ca75..08a28a1d98 100644 --- a/libsrc/XGetopt.c +++ b/libsrc/XGetopt.c @@ -1,220 +1,284 @@ -// XGetopt.cpp Version 1.2 -// -// Author: Hans Dietrich -// hdietrich2@hotmail.com -// -// Description: -// XGetopt.cpp implements getopt(), a function to parse command lines. -// -// History -// Version 1.2 - 2003 May 17 -// - Added Unicode support -// -// Version 1.1 - 2002 March 10 -// - Added example to XGetopt.cpp module header -// -// This software is released into the public domain. -// You are free to use it in any way you like. -// -// This software is provided "as is" with no expressed -// or implied warranty. I accept no liability for any -// damage or loss of business that this software may cause. -// -/////////////////////////////////////////////////////////////////////////////// - - -/////////////////////////////////////////////////////////////////////////////// -// if you are not using precompiled headers then include these lines: - -/////////////////////////////////////////////////////////////////////////////// - - +// XGetopt.cpp Version 1.2 +// +// Author: Hans Dietrich +// hdietrich2@hotmail.com +// +// Description: +// XGetopt.cpp implements getopt(), a function to parse command lines. +// +// History +// Version 1.2 - 2003 May 17 +// - Added Unicode support +// +// Version 1.1 - 2002 March 10 +// - Added example to XGetopt.cpp module header +// +// This software is released into the public domain. +// You are free to use it in any way you like. +// +// This software is provided "as is" with no expressed +// or implied warranty. I accept no liability for any +// damage or loss of business that this software may cause. +// +/////////////////////////////////////////////////////////////////////////////// + + +/////////////////////////////////////////////////////////////////////////////// +// if you are not using precompiled headers then include these lines: + +/////////////////////////////////////////////////////////////////////////////// + + #ifndef DLL_EXPORT #define DLL_EXPORT #endif -#include "XGetopt.h" - - -/////////////////////////////////////////////////////////////////////////////// -// -// X G e t o p t . c p p -// -// -// NAME -// getopt -- parse command line options -// -// SYNOPSIS -// int getopt(int argc, TCHAR *argv[], TCHAR *optstring) -// -// extern TCHAR *optarg; -// extern int optind; -// -// DESCRIPTION -// The getopt() function parses the command line arguments. Its -// arguments argc and argv are the argument count and array as -// passed into the application on program invocation. In the case -// of Visual C++ programs, argc and argv are available via the -// variables __argc and __argv (double underscores), respectively. -// getopt returns the next option letter in argv that matches a -// letter in optstring. (Note: Unicode programs should use -// __targv instead of __argv. Also, all character and string -// literals should be enclosed in _T( ) ). -// -// optstring is a string of recognized option letters; if a letter -// is followed by a colon, the option is expected to have an argument -// that may or may not be separated from it by white space. optarg -// is set to point to the start of the option argument on return from -// getopt. -// -// Option letters may be combined, e.g., "-ab" is equivalent to -// "-a -b". Option letters are case sensitive. -// -// getopt places in the external variable optind the argv index -// of the next argument to be processed. optind is initialized -// to 0 before the first call to getopt. -// -// When all options have been processed (i.e., up to the first -// non-option argument), getopt returns EOF, optarg will point -// to the argument, and optind will be set to the argv index of -// the argument. If there are no non-option arguments, optarg -// will be set to NULL. -// -// The special option "--" may be used to delimit the end of the -// options; EOF will be returned, and "--" (and everything after it) -// will be skipped. -// -// RETURN VALUE -// For option letters contained in the string optstring, getopt -// will return the option letter. getopt returns a question mark (?) -// when it encounters an option letter not included in optstring. -// EOF is returned when processing is finished. -// -// BUGS -// 1) Long options are not supported. -// 2) The GNU double-colon extension is not supported. -// 3) The environment variable POSIXLY_CORRECT is not supported. -// 4) The + syntax is not supported. -// 5) The automatic permutation of arguments is not supported. -// 6) This implementation of getopt() returns EOF if an error is -// encountered, instead of -1 as the latest standard requires. -// -// EXAMPLE -// BOOL CMyApp::ProcessCommandLine(int argc, TCHAR *argv[]) -// { -// int c; -// -// while ((c = getopt(argc, argv, _T("aBn:"))) != EOF) -// { -// switch (c) -// { -// case _T('a'): -// TRACE(_T("option a\n")); -// // -// // set some flag here -// // -// break; -// -// case _T('B'): -// TRACE( _T("option B\n")); -// // -// // set some other flag here -// // -// break; -// -// case _T('n'): -// TRACE(_T("option n: value=%d\n"), atoi(optarg)); -// // -// // do something with value here -// // -// break; -// -// case _T('?'): -// TRACE(_T("ERROR: illegal option %s\n"), argv[optind-1]); -// return FALSE; -// break; -// -// default: -// TRACE(_T("WARNING: no handler for option %c\n"), c); -// return FALSE; -// break; -// } -// } -// // -// // check for non-option args here -// // -// return TRUE; -// } -// -/////////////////////////////////////////////////////////////////////////////// - -GTOPT_EXTRA TCHAR *optarg; // global argument pointer -GTOPT_EXTRA int optind = 0; // global argv index +#include "XGetopt.h" +#include "nclist.h" +#include "ncbytes.h" + +#ifdef _MSC_VER +static void XCommandLineToArgvA(int* argcp, char*** argvp); +#endif + +/////////////////////////////////////////////////////////////////////////////// +// +// X G e t o p t . c p p +// +// +// NAME +// getopt -- parse command line options +// +// SYNOPSIS +// int getopt(int argc, TCHAR *argv[], TCHAR *optstring) +// +// extern TCHAR *optarg; +// extern int optind; +// +// DESCRIPTION +// The getopt() function parses the command line arguments. Its +// arguments argc and argv are the argument count and array as +// passed into the application on program invocation. In the case +// of Visual C++ programs, argc and argv are available via the +// variables __argc and __argv (double underscores), respectively. +// getopt returns the next option letter in argv that matches a +// letter in optstring. (Note: Unicode programs should use +// __targv instead of __argv. Also, all character and string +// literals should be enclosed in _T( ) ). +// +// optstring is a string of recognized option letters; if a letter +// is followed by a colon, the option is expected to have an argument +// that may or may not be separated from it by white space. optarg +// is set to point to the start of the option argument on return from +// getopt. +// +// Option letters may be combined, e.g., "-ab" is equivalent to +// "-a -b". Option letters are case sensitive. +// +// getopt places in the external variable optind the argv index +// of the next argument to be processed. optind is initialized +// to 0 before the first call to getopt. +// +// When all options have been processed (i.e., up to the first +// non-option argument), getopt returns EOF, optarg will point +// to the argument, and optind will be set to the argv index of +// the argument. If there are no non-option arguments, optarg +// will be set to NULL. +// +// The special option "--" may be used to delimit the end of the +// options; EOF will be returned, and "--" (and everything after it) +// will be skipped. +// +// RETURN VALUE +// For option letters contained in the string optstring, getopt +// will return the option letter. getopt returns a question mark (?) +// when it encounters an option letter not included in optstring. +// EOF is returned when processing is finished. +// +// BUGS +// 1) Long options are not supported. +// 2) The GNU double-colon extension is not supported. +// 3) The environment variable POSIXLY_CORRECT is not supported. +// 4) The + syntax is not supported. +// 5) The automatic permutation of arguments is not supported. +// 6) This implementation of getopt() returns EOF if an error is +// encountered, instead of -1 as the latest standard requires. +// +// EXAMPLE +// BOOL CMyApp::ProcessCommandLine(int argc, TCHAR *argv[]) +// { +// int c; +// +// while ((c = getopt(argc, argv, _T("aBn:"))) != EOF) +// { +// switch (c) +// { +// case _T('a'): +// TRACE(_T("option a\n")); +// // +// // set some flag here +// // +// break; +// +// case _T('B'): +// TRACE( _T("option B\n")); +// // +// // set some other flag here +// // +// break; +// +// case _T('n'): +// TRACE(_T("option n: value=%d\n"), atoi(optarg)); +// // +// // do something with value here +// // +// break; +// +// case _T('?'): +// TRACE(_T("ERROR: illegal option %s\n"), argv[optind-1]); +// return FALSE; +// break; +// +// default: +// TRACE(_T("WARNING: no handler for option %c\n"), c); +// return FALSE; +// break; +// } +// } +// // +// // check for non-option args here +// // +// return TRUE; +// } +// +/////////////////////////////////////////////////////////////////////////////// + +GTOPT_EXTRA TCHAR *optarg; // global argument pointer +GTOPT_EXTRA int optind = 0; // global argv index GTOPT_EXTRA int opterr; // print error - + GTOPT_EXTRA -int getopt(int argc, TCHAR *argv[], TCHAR *optstring) -{ - static TCHAR *next = NULL; - TCHAR c; - TCHAR *cp = malloc(sizeof(TCHAR)*1024); - - if (optind == 0) - next = NULL; - - optarg = NULL; - - if (next == NULL || *next == _T('\0')) - { - if (optind == 0) - optind++; - - if (optind >= argc || argv[optind][0] != _T('-') || argv[optind][1] == _T('\0')) - { - optarg = NULL; - if (optind < argc) - optarg = argv[optind]; - return EOF; - } - - if (_tcscmp(argv[optind], _T("--")) == 0) - { - optind++; - optarg = NULL; - if (optind < argc) - optarg = argv[optind]; - return EOF; - } - - next = argv[optind]; - next++; // skip past - - optind++; - } - - c = *next++; - cp = strchr(optstring, c); - - if (cp == NULL || c == _T(':')) - return _T('?'); - - cp++; - if (*cp == _T(':')) - { - if (*next != _T('\0')) - { - optarg = next; - next = NULL; - } - else if (optind < argc) - { - optarg = argv[optind]; - optind++; - } - else - { - return _T('?'); - } - } - - return c; -} +int getopt(int argc, TCHAR *argv[], TCHAR *optstring) +{ + static TCHAR *next = NULL; + TCHAR c; + TCHAR *cp = malloc(sizeof(TCHAR)*1024); + +#ifdef _MSC_VER + { + int xargc = -1; + char** xargv = NULL; + XCommandLineToArgvA(&xargc, &xargv); + } +#endif + + if (optind == 0) + next = NULL; + + optarg = NULL; + + if (next == NULL || *next == _T('\0')) + { + if (optind == 0) + optind++; + + if (optind >= argc || argv[optind][0] != _T('-') || argv[optind][1] == _T('\0')) + { + optarg = NULL; + if (optind < argc) + optarg = argv[optind]; + return EOF; + } + + if (_tcscmp(argv[optind], _T("--")) == 0) + { + optind++; + optarg = NULL; + if (optind < argc) + optarg = argv[optind]; + return EOF; + } + + next = argv[optind]; + next++; // skip past - + optind++; + } + + c = *next++; + cp = strchr(optstring, c); + + if (cp == NULL || c == _T(':')) + return _T('?'); + + cp++; + if (*cp == _T(':')) + { + if (*next != _T('\0')) + { + optarg = next; + next = NULL; + } + else if (optind < argc) + { + optarg = argv[optind]; + optind++; + } + else + { + return _T('?'); + } + } + + return c; +} + +/**************************************************/ +#define ESCAPE '\\' +#define SQUOTE '\'' +#define DQUOTE '"' + +/* Convert a UTF8 command line into a series of words for use by XGetOpt */ +/* Note only checks for ASCII '\' '\'' '"' */ +static void +XCommandLineToArgvA(int* argcp, char*** argvp) +{ + const char* line = NULL; + size_t len; + int whitespace; + int quote = 0; + enum State state; + NClist* argv = NULL; + NCbytes* word = NULL; + char** p; + + line = GetCommandLineA(); + len = strlen(line); + argv = nclistnew(); + word = ncbytesnew(); + whitespace = 1; /* start in whitespace mode */ + for(p=line;*p;p++) { + int c = *p; + if(whitespace && c <= ' ' || c == 127) continue; /* more whitespace */ + if(!whitespace && c <= ' ' || c == 127) { + whitespace = 1; /* end of word */ + ncbytesnull(word); + nclistpush(argv,ncbytesextract(word)); /* capture the word */ + continue; + } + whitespace = 0; /* end whitespace */ + if(c == ESCAPE) { + c = *(++p); /* move to next char */ + } else if(c == SQUOTE || c == DQUOTE) { + if(!quote) {quote = c; continue;} /* Start quoted text */ + if(quote == c) {quote = 0; continue;} /* end quoted text */ + } + /* Just collect the character as part of the current word */ + ncbytesappend(word,c); + } + /* Return parsed words */ + if(argcp) *argcp = nclistlength(argv); + nclistpush(argv,NULL); /* Just to be sure */ + if(argvp) *argvp = (char**)nclistextract(argv); + nclistfree(argv); + ncbytesfree(word); +} diff --git a/libsrc/ncFile.c b/libsrc/ncFile.c index e448d32e94..6d152ecaad 100644 --- a/libsrc/ncFile.c +++ b/libsrc/ncFile.c @@ -63,7 +63,7 @@ ncFile_create(const char *path, int ioflags, ncstdio** filepp) File* f; struct ncFileState* state; - f = fopen(path,"w+"); + f = NCfopen(path,"w+"); if(f == NULL) return errno; filep = (ncstdio*)calloc(sizeof(ncstdio),1); diff --git a/libsrc/ncio.c b/libsrc/ncio.c index 24bb92da10..9ee5d99bf5 100644 --- a/libsrc/ncio.c +++ b/libsrc/ncio.c @@ -83,7 +83,9 @@ ncio_open(const char *path, int ioflags, void* parameters, ncio** iopp, void** const mempp) { +#ifdef ENABLE_BYTERANGE int modetest = urlmodetest(path); +#endif /* Diskless open has the following constraints: 1. file must be classic version 1 or 2 or 5 diff --git a/nc_perf/tst_compress.c b/nc_perf/tst_compress.c index 15f2cba41c..354ac422cb 100644 --- a/nc_perf/tst_compress.c +++ b/nc_perf/tst_compress.c @@ -83,7 +83,7 @@ get_file_size(char *filename, size_t *file_size) FILE *fp; assert(filename && file_size); - fp = fopen(filename, "r"); + fp = NCfopen(filename, "r"); if (fp) { fseek(fp, 0 , SEEK_END); diff --git a/nc_perf/tst_compress_par.c b/nc_perf/tst_compress_par.c index 72084d1195..39b769c960 100644 --- a/nc_perf/tst_compress_par.c +++ b/nc_perf/tst_compress_par.c @@ -82,7 +82,7 @@ get_file_size(char *filename, size_t *file_size) FILE *fp; assert(filename && file_size); - fp = fopen(filename, "r"); + fp = NCfopen(filename, "r"); if (fp) { fseek(fp, 0 , SEEK_END); diff --git a/nc_perf/tst_files2.c b/nc_perf/tst_files2.c index 1e6145ac4a..ca9c8211ac 100644 --- a/nc_perf/tst_files2.c +++ b/nc_perf/tst_files2.c @@ -45,7 +45,7 @@ get_mem_used1(int *mem_used) system(cmd); /* Read the results and delete temp file. */ - if (!(fp = fopen(TMP_FILE_NAME, "r"))) ERR; + if (!(fp = NCfopen(TMP_FILE_NAME, "r"))) ERR; fread(blob, MAX_LEN, 1, fp); sscanf(blob, "%d", mem_used); fclose(fp); @@ -60,7 +60,7 @@ get_mem_used2(int *mem_used) FILE *pf; snprintf(buf, 30, "/proc/%u/statm", (unsigned)getpid()); - pf = fopen(buf, "r"); + pf = NCfopen(buf, "r"); if (pf) { unsigned size; /* total program size */ unsigned resident;/* resident set size */ diff --git a/nc_perf/tst_files3.c b/nc_perf/tst_files3.c index 7e72090de8..ab04f7ec06 100644 --- a/nc_perf/tst_files3.c +++ b/nc_perf/tst_files3.c @@ -167,7 +167,7 @@ get_mem_used2(int *mem_used) FILE *pf; snprintf(buf, 30, "/proc/%u/statm", (unsigned)getpid()); - pf = fopen(buf, "r"); + pf = NCfopen(buf, "r"); if (pf) { unsigned size; /* total program size */ unsigned resident;/* resident set size */ diff --git a/nc_perf/tst_mem.c b/nc_perf/tst_mem.c index a22a2f0960..ae4860e595 100644 --- a/nc_perf/tst_mem.c +++ b/nc_perf/tst_mem.c @@ -30,7 +30,7 @@ get_mem_used2(int *mem_used) FILE *pf; snprintf(buf, 30, "/proc/%u/statm", (unsigned)getpid()); - pf = fopen(buf, "r"); + pf = NCfopen(buf, "r"); if (pf) { unsigned size; /* total program size */ unsigned resident;/* resident set size */ diff --git a/nc_test/tst_diskless5.c b/nc_test/tst_diskless5.c index ab06f36df3..77b9f181a2 100644 --- a/nc_test/tst_diskless5.c +++ b/nc_test/tst_diskless5.c @@ -91,7 +91,7 @@ main(int argc, const char* argv[]) #endif #ifdef MEM - fd = open(PATH,O_RDONLY); + fd = NCopen2(PATH,O_RDONLY); if(fd < 0) { fprintf(stderr,"could not open foo.nc\n"); assert(0); diff --git a/nc_test/tst_diskless6.c b/nc_test/tst_diskless6.c index d968d71b14..69aa805dac 100644 --- a/nc_test/tst_diskless6.c +++ b/nc_test/tst_diskless6.c @@ -31,6 +31,7 @@ See \ref copyright file for more info. #endif #include "netcdf.h" +#include "ncpathmgr.h" #define CLEANUP @@ -131,7 +132,8 @@ fail(int ret) void exists(const char* file) { - FILE* f = fopen(file,"r"); + int stat = 0; + FILE* f = NCfopen(file, "r"); if(f == NULL) fail(NC_EPERM); fclose(f); } @@ -139,7 +141,7 @@ exists(const char* file) void notexists(const char* file) { - FILE* f = fopen(file,"r"); + FILE* f = NCfopen(file,"r"); if(f != NULL) {fclose(f); fail(NC_EEXIST);} } diff --git a/nc_test/tst_inmemory.c b/nc_test/tst_inmemory.c index 5119af820f..2075dd2957 100644 --- a/nc_test/tst_inmemory.c +++ b/nc_test/tst_inmemory.c @@ -162,80 +162,27 @@ static int readfile(const char* path, NC_memio* memio) { int status = NC_NOERR; - FILE* f = NULL; - size_t filesize = 0; - size_t count = 0; - char* memory = NULL; - char* p = NULL; - - /* Open the file for reading */ - f = NCfopen(path,"r"); - if(f == NULL) - {status = errno; goto done;} - /* get current filesize */ - if(fseek(f,0,SEEK_END) < 0) - {status = errno; goto done;} - filesize = (size_t)ftell(f); - /* allocate memory */ - memory = malloc((size_t)filesize); - if(memory == NULL) - {status = NC_ENOMEM; goto done;} - /* move pointer back to beginning of file */ - rewind(f); - count = filesize; - p = memory; - while(count > 0) { - size_t actual; - actual = fread(p,1,count,f); - if(actual == 0 || ferror(f)) - {status = NC_EIO; goto done;} - count -= actual; - p += actual; - } + NCbytes* buf = ncbytesnew(); + if((status = NC_readfile(path,buf))) goto done; if(memio) { - memio->size = (size_t)filesize; - memio->memory = memory; + memio->size = (size_t)ncbyteslength(buf); + memio->memory = ncbytesextract(buf); } done: - if(status != NC_NOERR && memory != NULL) - free(memory); - if(f != NULL) fclose(f); + ncbytesfree(buf); return status; } - static int writefile(const char* path, NC_memio* memio) { int status = NC_NOERR; - FILE* f = NULL; - size_t count = 0; - char* p = NULL; - /* Open the file for writing */ -#ifdef _WIN32 - f = fopen(path,"wb"); -#else - f = fopen(path,"w"); -#endif - if(f == NULL) - {status = errno; goto done;} - count = memio->size; - p = memio->memory; - while(count > 0) { - size_t actual; - actual = fwrite(p,1,count,f); - if(actual == 0 || ferror(f)) - {status = NC_EIO; goto done;} - count -= actual; - p += actual; - } + if((status = NC_writefile(path,memio->size,memio->memory))) goto done; done: - if(f != NULL) fclose(f); return status; } - /* Duplicate an NC_memio instance; needed to avoid attempting to use memory that might have been realloc'd Allow the new memory to be larger than the src memory diff --git a/nc_test/tst_misc.c b/nc_test/tst_misc.c index 7c55a1ddf3..93a96a3412 100644 --- a/nc_test/tst_misc.c +++ b/nc_test/tst_misc.c @@ -43,7 +43,7 @@ main(int argc, char **argv) for (i = DATA_LEN; i >= 0; i--) { /* Create a small file which is not a netCDF file. */ - if (!(file = fopen(FILE_NAME, "w+"))) ERR; + if (!(file = NCfopen(FILE_NAME, "w+"))) ERR; if (fwrite(dummy_data, 1, i, file) != i) ERR; if (fclose(file)) ERR; diff --git a/nc_test4/tst_camrun.c b/nc_test4/tst_camrun.c index bd3b4c8266..fb6ed42e3c 100644 --- a/nc_test4/tst_camrun.c +++ b/nc_test4/tst_camrun.c @@ -680,7 +680,7 @@ get_mem_used2(int *mem_used) assert(mem_used); snprintf(buf, 30, "/proc/%u/statm", (unsigned)getpid()); - if ((pf = fopen(buf, "r"))) + if ((pf = NCfopen(buf, "r"))) { (void)fscanf(pf, "%u %u %u %u %u %u", &size, &resident, &share, &text, &lib, &data); diff --git a/nc_test4/tst_udf.c b/nc_test4/tst_udf.c index 43d44a016c..12024bb2ea 100644 --- a/nc_test4/tst_udf.c +++ b/nc_test4/tst_udf.c @@ -311,7 +311,7 @@ main(int argc, char **argv) int i; /* Create a file with magic number at start. */ - if (!(FP = fopen(FILE_NAME, "w"))) ERR; + if (!(FP = NCfopen(FILE_NAME, "w"))) ERR; if (fwrite(magic_number, sizeof(char), strlen(magic_number), FP) != strlen(magic_number)) ERR; if (fwrite(dummy_data, sizeof(char), strlen(dummy_data), FP) diff --git a/ncdap_test/t_auth.c b/ncdap_test/t_auth.c index e70a068e87..32d3a4c24c 100644 --- a/ncdap_test/t_auth.c +++ b/ncdap_test/t_auth.c @@ -16,7 +16,7 @@ See \ref copyright file for more info. #include #endif -#define DEBUG +#undef DEBUG #include "netcdf.h" #include "nctestserver.h" @@ -150,7 +150,7 @@ testrc(const char* prefix, const char* url) FILE* rc; snprintf(rcpath,sizeof(rcpath),"%s/%s",prefix,RC); - rc = fopen(rcpath,"w"); + rc = NCfopen(rcpath,"w"); if(rc == NULL) { fprintf(stderr,"Cannot create ./%s\n",RC); exit(1); @@ -178,7 +178,7 @@ fillrc(const char* path) FILE* rc; killrc(); - rc = fopen(path,"w"); + rc = NCfopen(path,"w"); if(rc == NULL) { fprintf(stderr,"cannot create rc file: %s\n",path); exit(1); diff --git a/ncdump/CMakeLists.txt b/ncdump/CMakeLists.txt index c0aee2a7f3..7b0cca34d6 100644 --- a/ncdump/CMakeLists.txt +++ b/ncdump/CMakeLists.txt @@ -117,6 +117,7 @@ IF(MSVC) ${CMAKE_CURRENT_BINARY_DIR}) ENDIF(ENABLE_DAP) + ENDIF() IF(ENABLE_TESTS) @@ -304,14 +305,11 @@ endif() ENDIF() ENDIF(EXTRA_TESTS) - # The unicode tests are complicated - IF(USE_HDF5) - IF(NOT MSVC AND NOT MINGW) - # These tests do not work under windows - add_sh_test(ncdump test_unicode_directory) - add_sh_test(ncdump test_unicode_path) - ENDIF() - ENDIF(USE_HDF5) + # The unicode tests + if(NOT ISMINGW) + add_sh_test(ncdump test_unicode_directory) + add_sh_test(ncdump test_unicode_path) + ENDIF() IF(USE_CDF5) add_sh_test(ncdump test_keywords) diff --git a/ncdump/Makefile.am b/ncdump/Makefile.am index d5db8cf3a8..b629c495c7 100644 --- a/ncdump/Makefile.am +++ b/ncdump/Makefile.am @@ -144,7 +144,6 @@ if !ISMINGW if !ISCYGWIN TESTS += tst_output.sh TESTS += tst_nccopy3.sh -TESTS += test_unicode_directory.sh test_unicode_path.sh if USE_HDF5 TESTS += run_back_comp_tests.sh tst_netcdf4_4.sh TESTS += tst_nccopy4.sh tst_nccopy5.sh @@ -152,6 +151,11 @@ endif endif endif +# The unicode tests +if !ISMINGW +TESTS += test_unicode_directory.sh test_unicode_path.sh +endif + endif BUILD_TESTSETS # These files all have to be included with the distribution. diff --git a/ncdump/acpget.c b/ncdump/acpget.c new file mode 100644 index 0000000000..32753f768c --- /dev/null +++ b/ncdump/acpget.c @@ -0,0 +1,30 @@ +#include +#include +#include +#include +#include +#include + +int +main(int argc, char** argv) +{ + int acp = -1; + const char* acpid = NULL; + const char* lcall = NULL; + const char* lcctype = NULL; + char digits[16]; + + switch (acp = GetACP()) { + case 1252: acpid = "CP_1252"; break; + case 65001: acpid = "CP_UTF8"; break; + default: + snprintf(digits,sizeof(digits),"%d",acp); + acpid = digits; + break; + } + + lcall = setlocale(LC_ALL,NULL); + lcctype = setlocale(LC_CTYPE,NULL); + printf("ACP=%s locale: LC_ALL=%s LC_CTYPE=%s\n",acpid,lcall,lcctype); + return 0; +} diff --git a/ncdump/nctrunc.c b/ncdump/nctrunc.c index f0f9f2007e..67ce45c969 100644 --- a/ncdump/nctrunc.c +++ b/ncdump/nctrunc.c @@ -6,6 +6,7 @@ #include #include #include +#include "nc_tests.h" #define BUFLEN 100000 @@ -19,7 +20,7 @@ main(int argc, char** argv) FILE* input = stdin; if(argc > 1) { - input = fopen(argv[1],"r"); + input = NCfopen(argv[1],"r"); } /* Read the whole file */ diff --git a/ncdump/ocprint.c b/ncdump/ocprint.c index 1d16e436ef..72a45fb3ce 100755 --- a/ncdump/ocprint.c +++ b/ncdump/ocprint.c @@ -243,7 +243,7 @@ main(int argc, char **argv) if(ocopt.output != NULL) fclose(ocopt.output); if(optarg == NULL) usage("-o does not specify a file name"); - ocopt.output = fopen(optarg,"w"); + ocopt.output = NCfopen(optarg,"w"); if(ocopt.output == NULL) usage("-o file not writeable"); break; diff --git a/ncdump/run_cygutf8.sh b/ncdump/run_cygutf8.sh new file mode 100755 index 0000000000..ddf4985d64 --- /dev/null +++ b/ncdump/run_cygutf8.sh @@ -0,0 +1,10 @@ +#!/bin/bash -x + +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ../test_common.sh + +echo "Test unicode paths" +${execdir}/acpget +${execdir}/tst_cygutf8 +ls xutf8* +rm xutf8* diff --git a/ncdump/test_unicode_path.sh b/ncdump/test_unicode_path.sh index b9e08b3048..eeb4ed69d2 100755 --- a/ncdump/test_unicode_path.sh +++ b/ncdump/test_unicode_path.sh @@ -10,9 +10,6 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi set -e -LC_ALL="C.UTF-8" -export LC_ALL - # Passing a utf8 name using either \x or actual characters # to Visual Studio does not work well. if test "x$FP_ISMSVC" = x ; then diff --git a/ncdump/tst_chunking.c b/ncdump/tst_chunking.c index 95cd92010b..46c9ddc5ba 100644 --- a/ncdump/tst_chunking.c +++ b/ncdump/tst_chunking.c @@ -9,7 +9,7 @@ #include #include "err_macros.h" -#define DEBUG +#undef DEBUG static int ret = NC_NOERR; diff --git a/ncdump/tst_cygutf8.c b/ncdump/tst_cygutf8.c new file mode 100644 index 0000000000..201c3511ed --- /dev/null +++ b/ncdump/tst_cygutf8.c @@ -0,0 +1,55 @@ +/* +https://www.cygwin.com/cygwin-ug-net/using-specialnames.html +https://docs.microsoft.com/en-us/archive/msdn-magazine/2016/september/c-unicode-encoding-conversions-with-stl-strings-and-win32-apis +*/ + +#include +#include +#include +#ifdef _WIN32 +#include +#endif +#include "netcdf.h" +#include "ncpathmgr.h" + +static const unsigned char name1[] = { +'x','u','t','f','8','_', +'\xe6', '\xb5', '\xb7', +'\0' +}; + +static unsigned char name2[] = { +'x','u','t','f','8','_', + 0xCE, 0x9A, /* GREEK CAPITAL LETTER KAPPA : 2-bytes utf8 */ + 0xCE, 0xB1, /* GREEK SMALL LETTER LAMBDA : 2-bytes utf8 */ + 0xCE, 0xBB, /* GREEK SMALL LETTER ALPHA : 2-bytes utf8 */ + 0xCE, 0xB7, /* GREEK SMALL LETTER ETA : 2-bytes utf8 */ + 0xCE, 0xBC, /* GREEK SMALL LETTER MU : 2-bytes utf8 */ + 0xCE, 0xAD, /* GREEK SMALL LETTER EPSILON WITH TONOS + : 2-bytes utf8 */ + 0xCF, 0x81, /* GREEK SMALL LETTER RHO : 2-bytes utf8 */ + 0xCE, 0xB1, /* GREEK SMALL LETTER ALPHA : 2-bytes utf8 */ + 0x00 + }; + +static char* name3 = "xutf8_사람/접는사람"; + +/* This is CP_1252 */ +//static char* name4 = "xutf8_Å"; +static char name4[8] = {'x','u','t','f','8','_',0XC5,0x00} ; + +int +main() +{ + FILE* f; + f = NCfopen((char*)name1,"w"); + if (f) fclose(f); + f = NCfopen((char*)name2,"w"); + if (f) fclose(f); + f = NCfopen((char*)name3,"w"); + if (f) fclose(f); + printf("|name4|=%u\n",(unsigned)strlen(name4)); + f = NCfopen((char*)name4,"w"); + if (f) fclose(f); + return 0; +} diff --git a/ncdump/tst_unicode.c b/ncdump/tst_unicode.c index e742260db5..a3369559b6 100644 --- a/ncdump/tst_unicode.c +++ b/ncdump/tst_unicode.c @@ -23,7 +23,7 @@ #include #endif -#define DEBUG +#undef DEBUG /* The data file we will create. */ static const unsigned char prefix[] = { diff --git a/ncgen/dump.c b/ncgen/dump.c index c4e493dc4d..bd4d775b86 100644 --- a/ncgen/dump.c +++ b/ncgen/dump.c @@ -8,7 +8,7 @@ #include "includes.h" #include "dump.h" -#define DEBUGSRC +#undef DEBUGSRC #define MAXELEM 8 #define MAXDEPTH 4 diff --git a/ncgen3/run_nc4_tests.sh b/ncgen3/run_nc4_tests.sh index 52b3360d17..97caa5a53d 100755 --- a/ncgen3/run_nc4_tests.sh +++ b/ncgen3/run_nc4_tests.sh @@ -5,9 +5,10 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh +set -e echo "*** Testing ncgen3 for netCDF-4." -set -e + echo "*** creating netCDF-4 file c0_4.nc from c0.cdl..." ${NCGEN3} -k3 -b -o c0_4.nc ${ncgen3c0} echo "*** creating netCDF-4 classic model file c0_4c.nc from c0.cdl..." diff --git a/nczarr_test/ref_zarr_test_data.cdl.gz b/nczarr_test/ref_zarr_test_data.cdl.gz new file mode 100644 index 0000000000000000000000000000000000000000..9d688eb0e2045937c8b60d4f578bbc0b22446fd1 GIT binary patch literal 2702 zcmb2|=HPhm@h6dixhOR)zACY(D83}MxFkL$u_RG1IVFeT?cLbyTLBUb4_^P$=G%SC z;YH6kXZQEV{<(ErUdOmv=yGrAm;LWm)WYWG>E6DI5 z<$LycZr-F5W~krNA2~7o@l6x8wt`9LPk+#h6`7nl@tD-j=hJns@xJw6uO#X0U-roD z@tTgWaVe)$x5UYC?(A3GTWGGb@q4m-4?kO+zL;Ub1QYjg^X^4FpDT7=KKb$U&Uw0< zjC3|HIGGucl+B# zd8Ki;Coh|y|FwIUd-~ja`M0h9Zpxi~=U>?Vj9*WexvR@bLhBy?Hh^@A=kWoaVb97W`P^n|gfxSv~`a zKeazrt~}q&pDPmVv+@7x8;$H$LG~B^pFbVyV|Y{~_|1HsvNvCMYTo!)&;4(2!RJdG zj9H&8bp%O)>-px_r(2nG z=ezBe;cGwqbLqLqXLr3%JNvM}0>q~kb#DIyn>vYy2igUTXlf4)y)bGtEfdIS=#GZ& zXvsj^LU=Tfjpi|edF ${srcdir}/ref_zarr_test_data.cdl + fi + testcases3 zarr_test_data.zarr ref_zarr_test_data xarray + ;; *) echo "unimplemented kind: $1" ; exit 1;; esac } diff --git a/nczarr_test/s3util.c b/nczarr_test/s3util.c index 03ec172a23..07e03ef1c9 100644 --- a/nczarr_test/s3util.c +++ b/nczarr_test/s3util.c @@ -27,7 +27,7 @@ #undef NODELETE -#define DEBUG +#undef DEBUG #define DATANAME "data" diff --git a/nczarr_test/tst_zchunks3.c b/nczarr_test/tst_zchunks3.c index b3d238d88a..fa0c575194 100644 --- a/nczarr_test/tst_zchunks3.c +++ b/nczarr_test/tst_zchunks3.c @@ -8,7 +8,7 @@ #include "ut_includes.h" #include "test_nczarr_utils.h" -#define DEBUG +#undef DEBUG static int ret = NC_NOERR; #define FILE_NAME "tmp_chunks3.nc" diff --git a/nczarr_test/ut_json.c b/nczarr_test/ut_json.c index 22600fcb56..37ab65d231 100644 --- a/nczarr_test/ut_json.c +++ b/nczarr_test/ut_json.c @@ -5,7 +5,7 @@ #include "ut_includes.h" -#define DEBUG +#undef DEBUG typedef enum Cmds { cmd_none = 0, diff --git a/nczarr_test/zhex.c b/nczarr_test/zhex.c index 3a49ff6495..972407cb46 100644 --- a/nczarr_test/zhex.c +++ b/nczarr_test/zhex.c @@ -12,7 +12,7 @@ #include #endif -#define DEBUG +#undef DEBUG static char hex[16] = "0123456789abcdef"; diff --git a/nczarr_test/zmapio.c b/nczarr_test/zmapio.c index b1e06295ef..c9c982c881 100644 --- a/nczarr_test/zmapio.c +++ b/nczarr_test/zmapio.c @@ -23,7 +23,7 @@ #include "nclog.h" #include "ncuri.h" -#define DEBUG +#undef DEBUG #define DATANAME "data" diff --git a/nczarr_test/zs3parse.c b/nczarr_test/zs3parse.c index e1dba30cad..7286dd75a2 100644 --- a/nczarr_test/zs3parse.c +++ b/nczarr_test/zs3parse.c @@ -21,7 +21,7 @@ #include "zincludes.h" #include "ncpathmgr.h" -#define DEBUG +#undef DEBUG #define AWSHOST ".amazonaws.com" diff --git a/test_common.in b/test_common.in index 85d22323e7..fb02fe0d9d 100644 --- a/test_common.in +++ b/test_common.in @@ -10,11 +10,13 @@ TOPSRCDIR='@abs_top_srcdir@' TOPBUILDDIR='@abs_top_builddir@' FP_ISCMAKE=@ISCMAKE@ FP_ISMSVC=@ISMSVC@ + FP_WINVERMAJOR=@WINVERMAJOR@ + FP_WINVERBUILD=@WINVERBUILD@ FP_ISCYGWIN=@ISCYGWIN@ FP_ISMINGW=@ISMINGW@ FP_ISMSYS=@ISMSYS@ -FP_ISREGEDIT=@ISREGEDIT@ FP_USEPLUGINS=@USEPLUGINS@ +FP_ISREGEDIT=@ISREGEDIT@ # Feature flags FEATURE_HDF5=@HAS_HDF5@ @@ -150,5 +152,7 @@ ncgen3c0="${top_srcdir}/ncgen3/c0.cdl" ncgenc0="${top_srcdir}/ncgen/c0.cdl" ncgenc04="${top_srcdir}/ncgen/c0_4.cdl" +if test "x$FP_ISMSVC" = xyes || test "x$FP_ISCYGWIN" = xyes; then export LC_ALL="en_US.utf8"; fi + # Make sure we are in builddir (not execdir) cd $builddir diff --git a/unit_test/test_aws.c b/unit_test/test_aws.c index 826d3f82c9..8dd9e92764 100644 --- a/unit_test/test_aws.c +++ b/unit_test/test_aws.c @@ -15,7 +15,7 @@ Test the handling of aws profiles and regions. #include "ncrc.h" #include "ncpathmgr.h" -#define DEBUG +#undef DEBUG typedef struct ProfileTest { const char* profile; diff --git a/unit_test/test_pathcvt.c b/unit_test/test_pathcvt.c index 7f1d67b6be..7426e54953 100644 --- a/unit_test/test_pathcvt.c +++ b/unit_test/test_pathcvt.c @@ -13,7 +13,7 @@ Test the NCpathcvt #include "netcdf.h" #include "ncpathmgr.h" -#define DEBUG +#undef DEBUG #define NKINDS 4 static const int kinds[NKINDS] = {NCPD_NIX,NCPD_MSYS,NCPD_CYGWIN,NCPD_WIN}; From c0f086fc1410d5a3c8171da50c79271b6fafe231 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Tue, 8 Feb 2022 20:55:23 -0700 Subject: [PATCH 2/3] Update Release Notes --- RELEASE_NOTES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index b4fda7981a..e4ae005014 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -7,7 +7,7 @@ This file contains a high-level description of this package's evolution. Release ## 4.8.2 - TBD -* [Bug Fix] Improve UTF8 support on windows to use utf8 natively. See [Github #????](https://github.com/Unidata/netcdf-c/pull/????). +* [Bug Fix] Improve UTF8 support on windows to use utf8 natively. See [Github #2222](https://github.com/Unidata/netcdf-c/pull/2222). * [Enhancement] Add complete bitgroom support to NCZarr. See [Github #2197](https://github.com/Unidata/netcdf-c/pull/2197). * [Bug Fix] Clean up the handling of deeply nested VLEN types. Marks nc_free_vlen() and nc_free_string as deprecated in favor of ncaux_reclaim_data(). See [Github #2179](https://github.com/Unidata/netcdf-c/pull/2179). * [Bug Fix] Make sure that netcdf.h accurately defines the flags in the open/create mode flags. See [Github #2183](https://github.com/Unidata/netcdf-c/pull/2183). From 2a9edc8969e6339312a52ae740d0d3918c571335 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Tue, 8 Feb 2022 22:25:51 -0700 Subject: [PATCH 3/3] Update Release Notes (2) --- RELEASE_NOTES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index e4ae005014..6a19f7029f 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -7,7 +7,7 @@ This file contains a high-level description of this package's evolution. Release ## 4.8.2 - TBD -* [Bug Fix] Improve UTF8 support on windows to use utf8 natively. See [Github #2222](https://github.com/Unidata/netcdf-c/pull/2222). +* [Bug Fix] Improve UTF8 support on windows so that it can use utf8 natively. See [Github #2222](https://github.com/Unidata/netcdf-c/pull/2222). * [Enhancement] Add complete bitgroom support to NCZarr. See [Github #2197](https://github.com/Unidata/netcdf-c/pull/2197). * [Bug Fix] Clean up the handling of deeply nested VLEN types. Marks nc_free_vlen() and nc_free_string as deprecated in favor of ncaux_reclaim_data(). See [Github #2179](https://github.com/Unidata/netcdf-c/pull/2179). * [Bug Fix] Make sure that netcdf.h accurately defines the flags in the open/create mode flags. See [Github #2183](https://github.com/Unidata/netcdf-c/pull/2183).