diff --git a/.github/workflows/run_tests_osx.yml b/.github/workflows/run_tests_osx.yml index 05f2a568ec..8e9d65a6fb 100644 --- a/.github/workflows/run_tests_osx.yml +++ b/.github/workflows/run_tests_osx.yml @@ -7,7 +7,7 @@ name: Run macOS-based netCDF Tests -on: [ pull_request ] +on: [pull_request] jobs: diff --git a/.github/workflows/run_tests_ubuntu.yml b/.github/workflows/run_tests_ubuntu.yml index b57cf765fa..22377985ef 100644 --- a/.github/workflows/run_tests_ubuntu.yml +++ b/.github/workflows/run_tests_ubuntu.yml @@ -4,8 +4,7 @@ name: Run Ubuntu/Linux netCDF Tests -on: [ pull_request ] - +on: [pull_request] jobs: diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 92eef327af..243dd590a2 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -7,6 +7,7 @@ This file contains a high-level description of this package's evolution. Release ## 4.8.2 - TBD +* [Enhancement] Add complete bitgroom support to NCZarr. See [Github #????](https://github.com/Unidata/netcdf-c/pull/????). * [Bug Fix] Clean up the handling of deeply nested VLEN types. Marks nc_free_vlen() and nc_free_string as deprecated in favor of ncaux_reclaim_data(). See [Github #2179(https://github.com/Unidata/netcdf-c/pull/2179). * [Bug Fix] Make sure that netcdf.h accurately defines the flags in the open/create mode flags. See [Github #2183](https://github.com/Unidata/netcdf-c/pull/2183). * [Enhancement] Improve support for msys2+mingw platform. See [Github #2171](https://github.com/Unidata/netcdf-c/pull/2171). diff --git a/acinclude.m4 b/acinclude.m4 index 315eeec1f2..a3fdc712a7 100644 --- a/acinclude.m4 +++ b/acinclude.m4 @@ -384,3 +384,4 @@ MOSTLYCLEANFILES += $(valgrind_log_files) AC_SUBST([VALGRIND_CHECK_RULES]) m4_ifdef([_AM_SUBST_NOTMAKE], [_AM_SUBST_NOTMAKE([VALGRIND_CHECK_RULES])]) ]) +AC_DEFUN([_AC_FINALIZE],[]) diff --git a/include/ncuri.h b/include/ncuri.h index 33f7068f19..c4fabd4049 100644 --- a/include/ncuri.h +++ b/include/ncuri.h @@ -79,6 +79,9 @@ EXTERNL int ncurisetquery(NCURI*,const char* query); /* Replace the fragment list */ EXTERNL int ncurisetfragments(NCURI*, const char* fragments); +/* Rebuild the uri */ +EXTERNL int ncurirebuild(NCURI*); + /* Replace a specific &key=...& in uri fragment */ EXTERNL int ncurisetfragmentkey(NCURI* duri,const char* key, const char* value); diff --git a/libdap4/ncd4dispatch.c b/libdap4/ncd4dispatch.c index 7c64e7280d..7f85759c8e 100644 --- a/libdap4/ncd4dispatch.c +++ b/libdap4/ncd4dispatch.c @@ -763,6 +763,42 @@ NCD4_get_var_chunk_cache(int ncid, int p2, size_t* p3, size_t* p4, float* p5) return (ret); } +static int +NCD4_inq_var_quantize(int ncid, int varid, int *quantize_modep, int *nsdp) +{ + NC* ncp; + int ret; + int substrateid; + if((ret = NC_check_id(ncid, (NC**)&ncp)) != NC_NOERR) return (ret); + substrateid = makenc4id(ncp,ncid); + ret = nc_inq_var_quantize(substrateid, varid, quantize_modep, nsdp); + return (ret); +} + +static int +NCD4_inq_var_filter_ids(int ncid, int varid, size_t* nfilters, unsigned int* filterids) +{ + NC* ncp; + int ret; + int substrateid; + if((ret = NC_check_id(ncid, (NC**)&ncp)) != NC_NOERR) return (ret); + substrateid = makenc4id(ncp,ncid); + ret = nc_inq_var_filter_ids(substrateid, varid, nfilters, filterids); + return (ret); +} + +static int +NCD4_inq_var_filter_info(int ncid, int varid, unsigned int id, size_t* nparams, unsigned int* params) +{ + NC* ncp; + int ret; + int substrateid; + if((ret = NC_check_id(ncid, (NC**)&ncp)) != NC_NOERR) return (ret); + substrateid = makenc4id(ncp,ncid); + ret = nc_inq_var_filter_info(substrateid, varid, id, nparams, params); + return (ret); +} + /**************************************************/ /* Following functions are overridden to handle @@ -970,9 +1006,9 @@ NCD4_def_var_filter, NCD4_set_var_chunk_cache, NCD4_get_var_chunk_cache, -NC_NOTNC4_inq_var_filter_ids, -NC_NOTNC4_inq_var_filter_info, +NCD4_inq_var_filter_ids, +NCD4_inq_var_filter_info, NC_NOTNC4_def_var_quantize, -NC_NOTNC4_inq_var_quantize, +NCD4_inq_var_quantize, }; diff --git a/libdispatch/dinfermodel.c b/libdispatch/dinfermodel.c index 39387d7a08..97eeb18a5f 100644 --- a/libdispatch/dinfermodel.c +++ b/libdispatch/dinfermodel.c @@ -122,33 +122,43 @@ static struct FORMATMODES { {NULL,0}, }; -/* For some reason, compiler complains */ +/* Replace top-level name with defkey=defvalue */ static const struct MACRODEF { char* name; char* defkey; - char* defvalue; + char* defvalues[4]; } macrodefs[] = { -{"zarr","mode","nczarr,zarr"}, -{"dap2","mode","dap2"}, -{"dap4","mode","dap4"}, -{"s3","mode","nczarr,s3"}, -{"bytes","mode","bytes"}, -{"xarray","mode","nczarr,zarr,xarray"}, -{"noxarray","mode","nczarr,zarr,noxarray"}, -{NULL,NULL,NULL} +{"zarr","mode",{"nczarr","zarr",NULL}}, +{"dap2","mode",{"dap2",NULL}}, +{"dap4","mode",{"dap4",NULL}}, +{"s3","mode",{"s3","nczarr",NULL}}, +{"bytes","mode",{"bytes",NULL}}, +{"xarray","mode",{"nczarr","zarr","xarray",NULL}}, +{"noxarray","mode",{"nczarr","zarr","noxarray",NULL}}, +{"zarr","mode",{"nczarr","zarr","xarray",NULL}}, +{NULL,NULL,{NULL}} }; -/* Mode inferences */ +/* Mode inferences: if mode contains key, then add the inference and infer again */ static const struct MODEINFER { char* key; char* inference; } modeinferences[] = { {"zarr","nczarr"}, +{"zarr","xarray"}, {"xarray","zarr"}, {"noxarray","zarr"}, {NULL,NULL} }; +/* Mode negations: if mode contains key, then remove all occurrences of the inference and repeat */ +static const struct MODEINFER modenegations[] = { +{"bytes","nczarr"}, /* bytes negates (nc)zarr */ +{"bytes","zarr"}, +{"noxarray","xarray"}, +{NULL,NULL} +}; + /* Map FORMATX to readability to get magic number */ static struct Readable { int impl; @@ -175,9 +185,9 @@ static struct NCPROTOCOLLIST { {"http",NULL,NULL}, {"https",NULL,NULL}, {"file",NULL,NULL}, - {"dods","http","dap2"}, - {"dap4","http","dap4"}, - {"s3","s3","s3"}, + {"dods","http","mode=dap2"}, + {"dap4","http","mode=dap4"}, + {"s3","s3","mode=s3"}, {NULL,NULL,NULL} /* Terminate search */ }; @@ -201,6 +211,11 @@ static void printlist(NClist* list, const char* tag); static int isreadable(NCURI*,NCmodel*); static char* list2string(NClist*); static int parsepair(const char* pair, char** keyp, char** valuep); +static NClist* parsemode(const char* modeval); +static const char* getmodekey(const NClist* envv); +static int replacemode(NClist* envv, const char* newval); +static int inferone(const char* mode, NClist* newmodes); +static int negateone(const char* mode, NClist* modes); /* If the path looks like a URL, then parse it, reformat it. @@ -456,8 +471,10 @@ processmacros(NClist** fraglenvp) if(strlen(value) == 0) { /* must be a singleton */ for(macros=macrodefs;macros->name;macros++) { if(strcmp(macros->name,key)==0) { + char* const * p; nclistpush(expanded,strdup(macros->defkey)); - nclistpush(expanded,strdup(macros->defvalue)); + for(p=macros->defvalues;*p;p++) + nclistpush(expanded,strdup(*p)); found = 1; break; } @@ -483,68 +500,85 @@ static int processinferences(NClist* fraglenv) { int stat = NC_NOERR; - const struct MODEINFER* inferences = NULL; + const char* modeval = NULL; NClist* modes = NULL; - int inferred,i,pos = -1; - char* modeval = NULL; + NClist* newmodes = nclistnew(); + int i,inferred = 0; char* newmodeval = NULL; if(fraglenv == NULL || nclistlength(fraglenv) == 0) goto done; /* Get "mode" entry */ - for(i=0;ikey;inferences++) { - if(strcasecmp(inferences->key,mode)==0) { - int j; - int exists = 0; - for(j=0;jinference)==0) - {exists = 1; break;} - } - if(!exists) { - /* append the inferred mode if not already present */ - nclistpush(modes,strdup(inferences->inference)); - inferred = 1; - } - } - } + inferred = inferone(mode,newmodes); + nclistpush(newmodes,strdup(mode)); /* keep key */ + if(!inferred) nclistpush(newmodes,strdup(mode)); } } while(inferred); + /* Remove negative inferences */ + for(i=0;ikey;tests++) { + int i; + if(strcasecmp(tests->key,mode)==0) { + /* Find and remove all instances of the inference value */ + for(i=nclistlength(newmodes)-1;i>=0;i--) { + char* candidate = nclistget(newmodes,i); + if(strcasecmp(candidate,tests->inference)==0) { + nclistremove(newmodes,i); + nullfree(candidate); + changed = 1; + } + } + } + } + return changed; +} + +static int +inferone(const char* mode, NClist* newmodes) +{ + const struct MODEINFER* tests = modeinferences; + int changed = 0; + for(;tests->key;tests++) { + if(strcasecmp(tests->key,mode)==0) { + /* Append the inferred mode; dups removed later */ + nclistpush(newmodes,strdup(tests->inference)); + changed = 1; + } + } + return changed; +} static int mergekey(NClist** valuesp) @@ -694,7 +728,7 @@ NC_omodeinfer(int useparallel, int cmode, NCmodel* model) /* Process the cmode; may override some already set flags. The * user-defined formats must be checked first. They may choose to - * use some of the other flags, like NC_NETCDF4, so we must fist + * use some of the other flags, like NC_NETCDF4, so we must first * check NC_UDF0 and NC_UDF1 before checking for any other * flag. */ if(fIsSet(cmode,(NC_UDF0|NC_UDF1))) { @@ -789,6 +823,7 @@ NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void NClist* modeargs = nclistnew(); char* sfrag = NULL; const char* modeval = NULL; + char* abspath = NULL; /* Phase 1: 1. convert special protocols to http|https @@ -807,6 +842,9 @@ NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void printlist(fraglenv,"processmacros"); #endif + /* Cleanup the fragment list */ + if((stat = cleanfragments(&fraglenv))) goto done; + /* Phase 2a: Expand mode inferences and add to fraglenv */ if((stat = processinferences(fraglenv))) goto done; #ifdef DEBUG @@ -828,13 +866,29 @@ NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void ncurisetfragments(uri,sfrag); nullfree(sfrag); sfrag = NULL; + /* If s3, then rebuild the url */ + if(NC_iss3(uri)) { + NCURI* newuri = NULL; + if((stat = NC_s3urlrebuild(uri,&newuri,NULL,NULL))) goto done; + ncurifree(uri); + uri = newuri; + } else if(strcmp(uri->protocol,"file")==0) { + /* convert path to absolute */ + char* canon = NULL; + abspath = NCpathabsolute(uri->path); + if((stat = NCpathcanonical(abspath,&canon))) goto done; + nullfree(abspath); + abspath = canon; canon = NULL; + if((stat = ncurisetpath(uri,abspath))) goto done; + } + /* rebuild the path */ - if(newpathp) + if(newpathp) { *newpathp = ncuribuild(uri,NULL,NULL,NCURIALL); #ifdef DEBUG - fprintf(stderr,"newpath=|%s|\n",*newpathp); fflush(stderr); + fprintf(stderr,"newpath=|%s|\n",*newpathp); fflush(stderr); #endif - + } /* Phase 5: Process the mode key to see if we can tell the formatx */ modeval = ncurifragmentlookup(uri,"mode"); if(modeval != NULL) { @@ -921,6 +975,7 @@ NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void done: nullfree(sfrag); + nullfree(abspath); ncurifree(uri); nclistfreeall(modeargs); nclistfreeall(fraglenv); @@ -931,15 +986,16 @@ NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void static int isreadable(NCURI* uri, NCmodel* model) { + int canread = 0; struct Readable* r; /* Step 1: Look up the implementation */ for(r=readable;r->impl;r++) { - if(model->impl == r->impl) return r->readable; + if(model->impl == r->impl) {canread = r->readable; break;} } /* Step 2: check for bytes mode */ - if(NC_testmode(uri,"bytes")) return 1; - - return 0; + if(!canread && NC_testmode(uri,"bytes") && (model->impl == NC_FORMATX_NC4 || model->impl == NC_FORMATX_NC_HDF5)) + canread = 1; + return canread; } #if 0 @@ -996,7 +1052,50 @@ nc__testurl(const char* path0, char** basenamep) return ok; } +/**************************************************/ +/* Envv list utilities */ + +static const char* +getmodekey(const NClist* envv) +{ + int i; + /* Get "mode" entry */ + for(i=0;iurl */ + ncurirebuild(newurl); /* return various items */ #ifdef AWSDEBUG - { - char* s = ncuribuild(newurl,NULL,NULL,NCURIALL); - fprintf(stderr,">>> NC_s3urlrebuild: final=%s bucket=%s region=%s\n",s,bucket,region); - nullfree(s); - } + fprintf(stderr,">>> NC_s3urlrebuild: final=%s bucket=%s region=%s\n",uri->uri,bucket,region); #endif if(newurlp) {*newurlp = newurl; newurl = NULL;} if(bucketp) {*bucketp = bucket; bucket = NULL;} diff --git a/libdispatch/ncuri.c b/libdispatch/ncuri.c index e311d2b7ea..824e2e601e 100644 --- a/libdispatch/ncuri.c +++ b/libdispatch/ncuri.c @@ -541,6 +541,16 @@ ncurisetfragments(NCURI* duri,const char* fragments) return ret; } +/* Replace the path */ +int +ncurirebuild(NCURI* duri) +{ + char* surl = ncuribuild(duri,NULL,NULL,NCURIALL); + nullfree(duri->uri); + duri->uri = surl; + return (NC_NOERR); +} + /* Replace a specific fragment key*/ int ncurisetfragmentkey(NCURI* duri,const char* key, const char* value) diff --git a/libnczarr/zarr.h b/libnczarr/zarr.h index 8e531297aa..9f78e19297 100644 --- a/libnczarr/zarr.h +++ b/libnczarr/zarr.h @@ -68,10 +68,11 @@ EXTERNL int NCZ_comma_parse(const char* s, NClist* list); EXTERNL int NCZ_swapatomicdata(size_t datalen, void* data, int typesize); EXTERNL char** NCZ_clonestringvec(size_t len, const char** vec); EXTERNL void NCZ_freestringvec(size_t len, char** vec); -EXTERNL int NCZ_create_fill_chunk(size64_t chunksize, size_t typesize, const void* fill, void** fillchunkp); EXTERNL int NCZ_s3clear(NCS3INFO* s3map); EXTERNL int NCZ_ischunkname(const char* name,char dimsep); EXTERNL char* NCZ_chunkpath(struct ChunkKey key); +EXTERNL int NCZ_reclaim_fill_value(NC_VAR_INFO_T* var); +EXTERNL int NCZ_copy_fill_value(NC_VAR_INFO_T* var, void** dstp); /* zwalk.c */ EXTERNL int NCZ_read_chunk(int ncid, int varid, size64_t* zindices, void* chunkdata); diff --git a/libnczarr/zattr.c b/libnczarr/zattr.c index ec24c225f4..a69b5b264b 100644 --- a/libnczarr/zattr.c +++ b/libnczarr/zattr.c @@ -577,7 +577,7 @@ ncz_put_att(NC_GRP_INFO_T* grp, int varid, const char *name, nc_type file_type, /* If this is the _FillValue attribute, then we will also have to * copy the value to the fill_vlue pointer of the NC_VAR_INFO_T * struct for this var. (But ignore a global _FillValue - * attribute). */ + * attribute). Also kill the cache fillchunk as no longer valid */ if (!strcmp(att->hdr.name, _FillValue) && varid != NC_GLOBAL) { /* Fill value must have exactly one value */ @@ -636,6 +636,8 @@ ncz_put_att(NC_GRP_INFO_T* grp, int varid, const char *name, nc_type file_type, * been created in the file, so the dataset gets deleted and re-created. */ if (var->created) var->fill_val_changed = NC_TRUE; + /* Reclaim any existing fill_chunk */ + if((retval = NCZ_reclaim_fill_chunk(((NCZ_VAR_INFO_T*)var->format_var_info)->cache))) BAIL(retval); } /* Copy the attribute data, if there is any. */ @@ -701,7 +703,7 @@ ncz_put_att(NC_GRP_INFO_T* grp, int varid, const char *name, nc_type file_type, if(fillsave.data != NULL) { assert(fillsave.len > 0); if(att->data) - (void)nc_reclaim_data_all(h5->controller->ext_ncid,fillsave.type,var->fill_value,1); + (void)nc_reclaim_data_all(h5->controller->ext_ncid,fillsave.type,var->fill_value,1); var->fill_value = fillsave.data; } } @@ -850,22 +852,23 @@ int NCZ_inq_attname(int ncid, int varid, int attnum, char *name) { NC_ATT_INFO_T *att; - int retval; + int retval = NC_NOERR; + ZTRACE(1,"ncid=%d varid=%d attnum=%d name=%s",ncid,varid,attnum); LOG((2, "%s: ncid 0x%x varid %d", __func__, ncid, varid)); /* Find the file, group, and var info, and do lazy att read if * needed. */ if ((retval = ncz_find_grp_var_att(ncid, varid, NULL, attnum, 0, NULL, NULL, NULL, NULL, &att))) - return retval; + goto done; assert(att); /* Get the name. */ if (name) strcpy(name, att->hdr.name); - - return NC_NOERR; +done: + return ZUNTRACEX(retval,"name=%s",(retval?"":name)); } /** diff --git a/libnczarr/zcache.h b/libnczarr/zcache.h index c5065d1667..f25f30185d 100644 --- a/libnczarr/zcache.h +++ b/libnczarr/zcache.h @@ -37,9 +37,11 @@ typedef struct NCZCacheEntry { } NCZCacheEntry; typedef struct NCZChunkCache { + int valid; /* 0 => following fields need to be re-calculated */ NC_VAR_INFO_T* var; /* backlink */ size64_t ndims; /* true ndims == var->ndims + scalar */ size64_t chunksize; /* for real data */ + size64_t chunkcount; /* cross product of chunksizes */ void* fillchunk; /* enough fillvalues to fill a real chunk */ size_t maxentries; /* Max number of entries allowed; maxsize can override */ size_t maxsize; /* Maximum space used by cache; 0 => nolimit */ @@ -63,5 +65,7 @@ extern size64_t NCZ_cache_entrysize(NCZChunkCache* cache); extern NCZCacheEntry* NCZ_cache_entry(NCZChunkCache* cache, const size64_t* indices); extern size64_t NCZ_cache_size(NCZChunkCache* cache); extern int NCZ_buildchunkpath(NCZChunkCache* cache, const size64_t* chunkindices, struct ChunkKey* key); +extern int NCZ_ensure_fill_chunk(NCZChunkCache* cache); +extern int NCZ_reclaim_fill_chunk(NCZChunkCache* cache); #endif /*ZCACHE_H*/ diff --git a/libnczarr/zdebug.h b/libnczarr/zdebug.h index 4ed97975dc..97b255db41 100644 --- a/libnczarr/zdebug.h +++ b/libnczarr/zdebug.h @@ -8,8 +8,8 @@ #undef ZDEBUG /* general debug */ #undef ZDEBUG1 /* detailed debug */ -#define ZCATCH /* Warning: significant performance impact */ -#undef ZTRACING /* Warning: significant performance impact */ +#undef ZCATCH /* Warning: significant performance impact */ +#define ZTRACING /* Warning: significant performance impact */ #include "ncexternl.h" #include "nclog.h" diff --git a/libnczarr/zdispatch.c b/libnczarr/zdispatch.c index e90f0291a6..b2fff3d9ef 100644 --- a/libnczarr/zdispatch.c +++ b/libnczarr/zdispatch.c @@ -29,28 +29,28 @@ static const NC_Dispatch NCZ_dispatcher = { NCZ_abort, NCZ_close, NCZ_set_fill, - NC4_inq_format, + NCZ_inq_format, NCZ_inq_format_extended, NCZ_inq, - NC4_inq_type, + NCZ_inq_type, NCZ_def_dim, - NC4_inq_dimid, + NCZ_inq_dimid, NCZ_inq_dim, - NC4_inq_unlimdim, + NCZ_inq_unlimdim, NCZ_rename_dim, NCZ_inq_att, NCZ_inq_attid, - NC4_inq_attname, + NCZ_inq_attname, NCZ_rename_att, NCZ_del_att, NCZ_get_att, NCZ_put_att, NCZ_def_var, - NC4_inq_varid, + NCZ_inq_varid, NCZ_rename_var, NCZ_get_vara, NCZ_put_vara, @@ -67,20 +67,20 @@ static const NC_Dispatch NCZ_dispatcher = { NCZ_show_metadata, NCZ_inq_unlimdims, - NC4_inq_ncid, - NC4_inq_grps, - NC4_inq_grpname, - NC4_inq_grpname_full, - NC4_inq_grp_parent, - NC4_inq_grp_full_ncid, - NC4_inq_varids, - NC4_inq_dimids, - NC4_inq_typeids, + NCZ_inq_ncid, + NCZ_inq_grps, + NCZ_inq_grpname, + NCZ_inq_grpname_full, + NCZ_inq_grp_parent, + NCZ_inq_grp_full_ncid, + NCZ_inq_varids, + NCZ_inq_dimids, + NCZ_inq_typeids, NCZ_inq_type_equal, NCZ_def_grp, NCZ_rename_grp, - NC4_inq_user_type, - NC4_inq_typeid, + NCZ_inq_user_type, + NCZ_inq_typeid, NC_NOTNC4_def_compound, NC_NOTNC4_insert_compound, @@ -104,16 +104,12 @@ static const NC_Dispatch NCZ_dispatcher = { NC4_get_var_chunk_cache, NCZ_inq_var_filter_ids, NCZ_inq_var_filter_info, - NC_NOTNC4_def_var_quantize, - NC_NOTNC4_inq_var_quantize, + NCZ_def_var_quantize, + NCZ_inq_var_quantize, }; const NC_Dispatch* NCZ_dispatch_table = NULL; /* moved here from ddispatch.c */ -#ifdef ZTRACING -#include "ztracedispatch.h" -#endif - /** * @internal Initialize the ZARR dispatch layer. * @@ -127,9 +123,6 @@ NCZ_initialize(void) { int stat; NCZ_dispatch_table = &NCZ_dispatcher; -#ifdef ZTRACING - NCZ_dispatch_table = &NCZ_dispatcher_trace; -#endif if (!ncz_initialized) NCZ_initialize_internal(); stat = NCZ_provenance_init(); @@ -196,3 +189,123 @@ NCZ_inq_var_filter_info(int ncid, int varid, unsigned int id, size_t* nparams, u return REPORT(NC_ENOFILTER,"inq_var_filter_info"); } #endif /*ENABLE_NCZARR_FILTERS*/ + +/**************************************************/ +/* Following functions call into libsrc4 */ + +int +NCZ_inq_type(int ncid, nc_type xtype, char *name, size_t *size) +{ + int stat = NC_NOERR; + ZTRACE(0,"NC4_inq_type(ncid,xtype,name,size)"); + stat = NC4_inq_type(ncid,xtype,name,size); + return ZUNTRACE(stat); +} + +int +NCZ_inq_dimid(int ncid, const char *name, int *idp) +{ + int stat = NC_NOERR; + ZTRACE(0,"NC4_inq_dimid(ncid,name,idp)"); + stat = NC4_inq_dimid(ncid,name,idp); + return ZUNTRACE(stat); +} + +int +NCZ_inq_unlimdim(int ncid, int *unlimdimidp) +{ + int stat = NC_NOERR; + ZTRACE(0,"NC4_inq_unlimdim(ncid,unlimdimidp)"); + stat = NC4_inq_unlimdim(ncid,unlimdimidp); + return ZUNTRACE(stat); +} + +int +NCZ_inq_varid(int ncid, const char* name, int *varidp) +{ + int stat = NC_NOERR; + ZTRACE(0,"NC4_inq_varid(ncid,name,varidp)"); + stat = NC4_inq_varid(ncid,name,varidp); + return ZUNTRACE(stat); +} + +int +NCZ_inq_ncid(int ncid, const char* name, int* grpidp) +{ + int stat = NC_NOERR; + ZTRACE(0,"NC4_inq_ncid(ncid,name,grpidp)"); + stat = NC4_inq_ncid(ncid,name,grpidp); + return ZUNTRACE(stat); +} + +int +NCZ_inq_grps(int ncid, int* n, int* ncids) +{ + int stat = NC_NOERR; + ZTRACE(0,"NC4_inq_grps(ncid,n,ncids)"); + stat = NC4_inq_grps(ncid,n,ncids); + return ZUNTRACE(stat); +} + +int +NCZ_inq_grpname(int ncid, char* name) +{ + int stat = NC_NOERR; + ZTRACE(0,"NC4_inq_grpname(ncid,name)"); + stat = NC4_inq_grpname(ncid,name); + return ZUNTRACE(stat); +} + +int +NCZ_inq_grpname_full(int ncid, size_t* lenp, char* fullname) +{ + int stat = NC_NOERR; + ZTRACE(0,"NC4_inq_grpname_full(ncid,lenp,fullname)"); + stat = NC4_inq_grpname_full(ncid,lenp,fullname); + return ZUNTRACE(stat); +} + +int +NCZ_inq_grp_parent(int ncid, int* parentidp) +{ + int stat = NC_NOERR; + ZTRACE(0,"NC4_inq_grp_parent(ncid,parentidp)"); + stat = NC4_inq_grp_parent(ncid,parentidp); + return ZUNTRACE(stat); +} + +int +NCZ_inq_grp_full_ncid(int ncid, const char* fullname, int* grpidp) +{ + int stat = NC_NOERR; + ZTRACE(0,"NC4_inq_grp_full_ncid(ncid,fullname,grpidp)"); + stat = NC4_inq_grp_full_ncid(ncid,fullname,grpidp); + return ZUNTRACE(stat); +} + +int +NCZ_inq_varids(int ncid, int* nvars, int* varids) +{ + int stat = NC_NOERR; + ZTRACE(0,"NC4_inq_varids(ncid,nvars,varids)"); + stat = NC4_inq_varids(ncid,nvars,varids); + return ZUNTRACE(stat); +} + +int +NCZ_inq_dimids(int ncid, int* ndims, int* dimids, int inclparents) +{ + int stat = NC_NOERR; + ZTRACE(0,"NC4_inq_dimids(ncid,ndims,dimids,inclparents)"); + stat = NC4_inq_dimids(ncid,ndims,dimids,inclparents); + return ZUNTRACE(stat); +} + +int +NCZ_inq_user_type(int ncid, nc_type xtype, char* name, size_t* size, nc_type* basetid, size_t* nfields, int* classp) +{ + int stat = NC_NOERR; + ZTRACE(0,"NC4_inq_user_type(ncid,xtype,name,size,basetid,nfields,classp)"); + stat = NC4_inq_user_type(ncid,xtype,name,size,basetid,nfields,classp); + return ZUNTRACE(stat); +} diff --git a/libnczarr/zdispatch.h b/libnczarr/zdispatch.h index fdbd03828a..09391d03f8 100644 --- a/libnczarr/zdispatch.h +++ b/libnczarr/zdispatch.h @@ -173,9 +173,28 @@ EXTERNL int NCZ_def_var_filter(int ncid, int varid, unsigned int filterid, size_ EXTERNL int NCZ_inq_var_filter_ids(int ncid, int varid, size_t* nfiltersp, unsigned int *filterids); EXTERNL int NCZ_inq_var_filter_info(int ncid, int varid, unsigned int filterid, size_t* nparamsp, unsigned int *params); -EXTERNL int NCZ_def_var_filterx(int ncid, int varid, const char* text); -EXTERNL int NCZ_inq_var_filterx_ids(int ncid, int varid, char** textp); -EXTERNL int NCZ_inq_var_filterx_info(int ncid, int varid, const char* id, char** textp); +EXTERNL int NCZ_def_var_quantize(int ncid, int varid, int quantize_mode, int nsd); +EXTERNL int NCZ_inq_var_quantize(int ncid, int varid, int *quantize_modep, int *nsdp); + +/**************************************************/ +/* Following functions wrap libsrc4 */ +EXTERNL int NCZ_inq_type(int ncid, nc_type xtype, char *name, size_t *size); +EXTERNL int NCZ_inq_dimid(int ncid, const char *name, int *idp); +EXTERNL int NCZ_inq_unlimdim(int ncid, int *unlimdimidp); +EXTERNL int NCZ_inq_attname(int ncid, int varid, int attnum, char *name); +EXTERNL int NCZ_inq_varid(int ncid, const char* name, int *varidp); +EXTERNL int NCZ_inq_ncid(int ncid, const char* name, int* grpidp); +EXTERNL int NCZ_inq_grps(int ncid, int* n, int* ncids); +EXTERNL int NCZ_inq_grpname(int ncid, char* name); +EXTERNL int NCZ_inq_grpname_full(int ncid, size_t* lenp, char* fullname); +EXTERNL int NCZ_inq_grp_parent(int ncid, int* parentidp); +EXTERNL int NCZ_inq_grp_full_ncid(int ncid, const char* fullname, int* grpidp); +EXTERNL int NCZ_inq_varids(int ncid, int* nvars, int* varids); +EXTERNL int NCZ_inq_dimids(int ncid, int* ndims, int* dimids, int inclparents); +EXTERNL int NCZ_inq_typeids(int ncid, int* ntypes, int* typeids); +EXTERNL int NCZ_inq_user_type(int ncid, nc_type xtype, char* name, size_t* size, nc_type* basetid, size_t* nfields, int* classp); + +/**************************************************/ #if defined(__cplusplus) } diff --git a/libnczarr/zfile.c b/libnczarr/zfile.c index 96070567c8..69e13cb3f6 100644 --- a/libnczarr/zfile.c +++ b/libnczarr/zfile.c @@ -16,7 +16,7 @@ #include "zfilter.h" /* Forward */ -static int NCZ_enddef(int ncid); +static int NCZ_enddef(NC_FILE_INFO_T* h5); static int ncz_sync_netcdf4_file(NC_FILE_INFO_T* file, int isclose); /** @@ -34,20 +34,20 @@ NCZ_redef(int ncid) NC_FILE_INFO_T* zinfo = NULL; int stat = NC_NOERR; - LOG((1, "%s: ncid 0x%x", __func__, ncid)); + ZTRACE(0,"NCZ_redef(ncid)"); /* Find this file's metadata. */ if ((stat = nc4_find_grp_h5(ncid, NULL, &zinfo))) - return stat; + goto done; assert(zinfo); /* If we're already in define mode, return an error. */ if (zinfo->flags & NC_INDEF) - return NC_EINDEFINE; + {stat = NC_EINDEFINE; goto done;} /* If the file is read-only, return an error. */ if (zinfo->no_write) - return NC_EPERM; + {stat = NC_EPERM; goto done;} /* Set define mode. */ zinfo->flags |= NC_INDEF; @@ -56,7 +56,8 @@ NCZ_redef(int ncid) redef. */ zinfo->redef = NC_TRUE; - return NC_NOERR; +done: + return ZUNTRACE(stat); } /** @@ -77,8 +78,13 @@ NCZ__enddef(int ncid, size_t h_minfree, size_t v_align, size_t v_minfree, size_t r_align) { int stat = NC_NOERR; + NC_FILE_INFO_T* h5 = NULL; + NC_GRP_INFO_T* grp = NULL; ZTRACE(0,"ncid=%d",ncid); - stat = NCZ_enddef(ncid); + if ((stat = nc4_find_grp_h5(ncid, &grp, &h5))) + goto done; + stat = NCZ_enddef(h5); +done: return ZUNTRACE(stat); } @@ -86,7 +92,7 @@ NCZ__enddef(int ncid, size_t h_minfree, size_t v_align, * @internal Take the file out of define mode. This is called * automatically for netcdf-4 files, if the user forgets. * - * @param ncid File and group ID. + * @param h5 File object * * @return ::NC_NOERR No error. * @return ::NC_EBADID Bad ncid. @@ -94,41 +100,36 @@ NCZ__enddef(int ncid, size_t h_minfree, size_t v_align, * @author Dennis Heimbigner, Ed Hartnett */ static int -NCZ_enddef(int ncid) +NCZ_enddef(NC_FILE_INFO_T* h5) { - NC_FILE_INFO_T* h5 = NULL; - NC_GRP_INFO_T *grp; NC_VAR_INFO_T *var; int i,j; int stat = NC_NOERR; - ZTRACE(1,"ncid=%d",ncid); - - /* Find pointer to group and zinfo. */ - if ((stat = nc4_find_grp_h5(ncid, &grp, &h5))) - goto done; + ZTRACE(1,"h5=%s",h5->hdr.name); - /* Why is this here? Especially since it is not recursive so it - only applies to the this grp */ /* When exiting define mode, process all variables */ for (i = 0; i < nclistlength(h5->allgroups); i++) { NC_GRP_INFO_T* g = nclistget(h5->allgroups,i); for (j = 0; j < ncindexsize(g->vars); j++) { var = (NC_VAR_INFO_T *)ncindexith(g->vars, j); assert(var); - /* set the fill value and _FillValue attribute */ - if((stat = ncz_get_fill_value(h5,var,NULL))) goto done; /* ensure var->fill_value is set */ - assert(var->fill_value != NULL); var->written_to = NC_TRUE; /* mark it written */ + var->created = 1; +#if 0 + /* set the fill value and _FillValue attribute */ + if((stat = NCZ_ensure_fill_value(var))) goto done; /* ensure var->fill_value is set */ + assert(var->no_fill || var->fill_value != NULL); /* rebuild the fill chunk */ if((stat = NCZ_adjust_var_cache(var))) goto done; #ifdef ENABLE_NCZARR_FILTERS /* Build the filter working parameters for any filters */ if((stat = NCZ_filter_setup(var))) goto done; #endif +#endif /*0|1*/ } } - stat = ncz_enddef_netcdf4_file(h5); + if((stat = ncz_enddef_netcdf4_file(h5))) goto done; done: return ZUNTRACE(stat); } @@ -163,7 +164,7 @@ NCZ_sync(int ncid) { if (file->cmode & NC_CLASSIC_MODEL) return NC_EINDEFINE; - if ((stat = NCZ_enddef(ncid))) + if ((stat = NCZ_enddef(file))) return stat; } @@ -252,7 +253,7 @@ ncz_closeorabort(NC_FILE_INFO_T* h5, void* params, int abort) /* If we're in define mode, but not redefing the file, delete it. */ if(!abort) { - /* Invoke enddef if needed, which mean sync first */ + /* Invoke enddef if needed, which includes sync first */ if(h5->flags & NC_INDEF) h5->flags ^= NC_INDEF; /* Sync the file unless this is a read-only file. */ if(!h5->no_write) { @@ -412,7 +413,7 @@ ncz_sync_netcdf4_file(NC_FILE_INFO_T* file, int isclose) } /** - * @internal This function will do the enddef stuff for a netcdf-4 file. + * @internal This function will do the enddef stuff for an nczarr file. * * @param file Pointer to ZARR file info struct. * @@ -456,20 +457,20 @@ NCZ_set_fill(int ncid, int fillmode, int *old_modep) NC_FILE_INFO_T* h5 = NULL; int stat = NC_NOERR; - LOG((2, "%s: ncid 0x%x fillmode %d", __func__, ncid, fillmode)); + ZTRACE(0,"NCZ_set_fill(ncid,fillmode,old)"); /* Get pointer to file info. */ if ((stat = nc4_find_grp_h5(ncid, NULL, &h5))) - return stat; + goto done; assert(h5); /* Trying to set fill on a read-only file? You sicken me! */ if (h5->no_write) - return NC_EPERM; + {stat = NC_EPERM; goto done;} /* Did you pass me some weird fillmode? */ if (fillmode != NC_FILL && fillmode != NC_NOFILL) - return NC_EINVAL; + {stat = NC_EINVAL; goto done;} /* If the user wants to know, tell him what the old mode was. */ if (old_modep) @@ -477,5 +478,6 @@ NCZ_set_fill(int ncid, int fillmode, int *old_modep) h5->fill_mode = fillmode; - return NC_NOERR; +done: + return ZUNTRACE(stat); } diff --git a/libnczarr/zfilter.c b/libnczarr/zfilter.c index 3d64b66d67..d2d24b9048 100644 --- a/libnczarr/zfilter.c +++ b/libnczarr/zfilter.c @@ -53,8 +53,8 @@ #include "netcdf_aux.h" #undef DEBUG -#define DEBUGF -#define DEBUGL +#undef DEBUGF +#undef DEBUGL #define NULLIFY(x) ((x)?(x):"NULL") @@ -486,6 +486,7 @@ nc_var_filter_remove(int ncid, int varid, unsigned int filterid) } #endif +#ifdef ENABLE_NCZARR_FILTERS int NCZ_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams, const unsigned int* params) @@ -678,7 +679,7 @@ NCZ_inq_var_filter_info(int ncid, int varid, unsigned int id, size_t* nparamsp, done: return ZUNTRACEX(stat,"nparams=%u",(unsigned)(nparamsp?*nparamsp:0)); } - +#endif /*ENABLE_NCZARR_FILTERS*/ /**************************************************/ /* Filter application functions */ @@ -778,7 +779,6 @@ NCZ_applyfilterchain(const NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, NClist* cha void* next_buf = NULL; size_t next_used = 0; - #ifdef DEBUG fprintf(stderr,"current: alloc=%u used=%u buf=%p\n",(unsigned)current_alloc,(unsigned)current_used,current_buf); #endif @@ -796,6 +796,7 @@ fprintf(stderr,"current: alloc=%u used=%u buf=%p\n",(unsigned)current_alloc,(uns fprintf(stderr,"next: alloc=%u used=%u buf=%p\n",(unsigned)next_alloc,(unsigned)next_used,next_buf); #endif if(next_used == 0) {stat = NC_EFILTER; lastbuffer = next_buf; goto done; } + /* If the filter did not need to create a new buffer, then next == current else current was reclaimed */ current_buf = next_buf; current_alloc = next_alloc; current_used = next_used; @@ -814,6 +815,7 @@ fprintf(stderr,"next: alloc=%u used=%u buf=%p\n",(unsigned)next_alloc,(unsigned) fprintf(stderr,"next: alloc=%u used=%u buf=%p\n",(unsigned)next_alloc,(unsigned)next_used,next_buf); #endif if(next_used == 0) {stat = NC_EFILTER; lastbuffer = next_buf; goto done;} + /* If the filter did not need to create a new buffer, then next == current else current was reclaimed */ current_buf = next_buf; current_alloc = next_alloc; current_used = next_used; @@ -1451,28 +1453,34 @@ ensure_working(const NC_VAR_INFO_T* var, NCZ_Filter* filter) { int stat = NC_NOERR; if(!(filter->flags & FLAG_WORKING)) { - size_t vnparams = filter->hdf5.visible.nparams; - unsigned* vparams = filter->hdf5.visible.params; + const size_t oldnparams = filter->hdf5.visible.nparams; + const unsigned* oldparams = filter->hdf5.visible.params; assert(filter->flags & FLAG_VISIBLE); - /* Convert the visible parameters to working parameters (and back) */ - if(filter->plugin->codec.codec->NCZ_modify_parameters) { - stat = filter->plugin->codec.codec->NCZ_modify_parameters(ncidfor(var),var->hdr.id, + /* Convert the visible parameters to working parameters; may also modify the visible params */ + if(filter->plugin && filter->plugin->codec.codec->NCZ_modify_parameters) { + stat = filter->plugin->codec.codec->NCZ_modify_parameters(ncidfor(var),var->hdr.id, &filter->hdf5.visible.nparams, &filter->hdf5.visible.params, &filter->hdf5.working.nparams, &filter->hdf5.working.params); - if(stat) goto done; +#ifdef DEBUGF + fprintf(stderr,"DEBUGF: NCZ_modify_parameters: ncid=%d varid=%d filter=%s\n", (int)ncidfor(var),(int)var->hdr.id, + printfilter(filter)); +#endif + if(stat) goto done; + /* See if the visible parameters were changed */ + if(oldnparams != filter->hdf5.visible.nparams || oldparams != filter->hdf5.visible.params) + filter->flags |= FLAG_NEWVISIBLE; } else { - /* Just use the visible parameters */ + /* assume visible are unchanged */ + assert(oldnparams == filter->hdf5.visible.nparams && oldparams == filter->hdf5.visible.params); /* unchanged */ + /* Just copy the visible parameters */ nullfree(filter->hdf5.working.params); if((stat = paramnczclone(&filter->hdf5.working,&filter->hdf5.visible))) goto done; } filter->flags |= FLAG_WORKING; - /* See if the visible parameters were changed */ - if(vnparams != filter->hdf5.visible.nparams || vparams != filter->hdf5.visible.params) - filter->flags |= FLAG_NEWVISIBLE; } #ifdef DEBUGF - fprintf(stderr,"DEBUGF: ensure_working_parameters: ncid=%lu varid=%u filter=%s\n", ncidfor(var), (unsigned)var->hdr.id,printfilter(filter)); + fprintf(stderr,"DEBUGF: ensure_working_parameters: ncid=%lu varid=%u filter=%s\n", ncidfor(var), (unsigned)var->hdr.id,printfilter(filter)); #endif done: return THROW(stat); @@ -1540,40 +1548,16 @@ NCZ_filter_setup(NC_VAR_INFO_T* var) filters = (NClist*)var->filters; for(i=0;iplugin != NULL); + assert(filter != NULL && filter->plugin != NULL); assert((filter->flags & FLAG_VISIBLE)); /* Assume visible params are defined */ /* verify */ assert(filter->hdf5.id > 0 && (filter->hdf5.visible.nparams == 0 || filter->hdf5.visible.params != NULL)); - assert((filter->flags & FLAG_WORKING)==0); /* Assume working params are not defined */ - assert(filter->hdf5.working.nparams == 0 && filter->hdf5.working.params == NULL); - vnparams = filter->hdf5.visible.nparams; - vparams = filter->hdf5.visible.params; /* Initialize the working parameters */ - if(filter->plugin && filter->plugin->codec.codec->NCZ_modify_parameters) { - stat = filter->plugin->codec.codec->NCZ_modify_parameters(ncidfor(var),var->hdr.id, - &filter->hdf5.visible.nparams,&filter->hdf5.visible.params, - &filter->hdf5.working.nparams,&filter->hdf5.working.params); -#ifdef DEBUGF - fprintf(stderr,"DEBUGF: NCZ_modify_parameters: ncid=%d varid=%d filter=%s\n", (int)ncidfor(var),(int)var->hdr.id, - printfilter(filter)); -#endif - if(stat) goto done; - } else {/* Just copy over the visible params */ - filter->hdf5.working.nparams = vnparams; - if(vnparams > 0) { - if((stat=paramclone(vnparams,&filter->hdf5.working.params,vparams))) goto done; - } - } + if((stat = ensure_working(var,filter))) goto done; #ifdef DEBUGF fprintf(stderr,"DEBUGF: NCZ_filter_setup: ncid=%d varid=%d filter=%s\n", (int)ncidfor(var),(int)var->hdr.id, printfilter(filter)); #endif - filter->flags |= FLAG_WORKING; - /* See if the visible parameters were changed */ - if(vnparams != filter->hdf5.visible.nparams || vparams != filter->hdf5.visible.params) - filter->flags |= FLAG_NEWVISIBLE; } done: diff --git a/libnczarr/zinternal.c b/libnczarr/zinternal.c index 724e7d7d3d..6980c17851 100644 --- a/libnczarr/zinternal.c +++ b/libnczarr/zinternal.c @@ -635,22 +635,25 @@ ncz_find_grp_var_att(int ncid, int varid, const char *name, int attnum, } /** - * @internal What fill value should be used for a variable? + * @internal Ensure that either var->no_fill || var->fill_value != NULL. * Side effects: set as default if necessary and build _FillValue attribute. * * @param h5 Pointer to file info struct. * @param var Pointer to variable info struct. - * @param fillp Pointer that gets pointer to fill value. * * @returns NC_NOERR No error. * @returns NC_ENOMEM Out of memory. * @author Ed Hartnett, Dennis Heimbigner */ int -ncz_get_fill_value(NC_FILE_INFO_T *h5, NC_VAR_INFO_T *var, void **fillp) +NCZ_ensure_fill_value(NC_VAR_INFO_T *var) { size_t size; int retval = NC_NOERR; + NC_FILE_INFO_T *h5 = var->container->nc4_info; + + if(var->no_fill) + return NC_NOERR; #if 0 /*LOOK*/ /* Find out how much space we need for this type's fill value. */ @@ -660,9 +663,8 @@ ncz_get_fill_value(NC_FILE_INFO_T *h5, NC_VAR_INFO_T *var, void **fillp) size = sizeof(char *); else #endif - { - if ((retval = nc4_get_typelen_mem(h5, var->type_info->hdr.id, &size))) goto done; - } + + if ((retval = nc4_get_typelen_mem(h5, var->type_info->hdr.id, &size))) goto done; assert(size); /* If the user has set a fill_value for this var, use, otherwise find the default fill value. */ @@ -674,8 +676,7 @@ ncz_get_fill_value(NC_FILE_INFO_T *h5, NC_VAR_INFO_T *var, void **fillp) {retval = NC_ENOMEM; goto done;} if((retval = nc4_get_default_fill_value(var->type_info, var->fill_value))) { /* Note: release memory, but don't return error on failure */ - free(var->fill_value); - var->fill_value = NULL; + (void)NCZ_reclaim_fill_value(var); retval = NC_NOERR; goto done; } @@ -714,18 +715,6 @@ ncz_get_fill_value(NC_FILE_INFO_T *h5, NC_VAR_INFO_T *var, void **fillp) } } #endif /*0*/ - /* Create _FillValue Attribute */ - if((retval = ncz_create_fillvalue(var))) goto done; - if(fillp) { - void* fill = NULL; - /* Allocate the return space. */ - if((fill = calloc(1, size))==NULL) - {retval = NC_ENOMEM; goto done;} - memcpy(fill, var->fill_value, size); - *fillp = fill; - fill = NULL; - } - done: return retval; } @@ -769,6 +758,27 @@ NCZ_set_log_level() } #endif /* LOGGING */ +/** + * @internal Get the format (i.e. NC_FORMAT_NETCDF4 pr + * NC_FORMAT_NETCDF4_CLASSIC) of an open netCDF-4 file. + * + * @param ncid File ID (ignored). + * @param formatp Pointer that gets the constant indicating format. + + * @return ::NC_NOERR No error. + * @return ::NC_EBADID Bad ncid. + * @author Ed Hartnett + */ +int +NCZ_inq_format(int ncid, int *formatp) +{ + int stat = NC_NOERR; + + ZTRACE(0,"ncid=%d formatp=%p",ncid,formatp); + stat = NC4_inq_format(ncid,formatp); + return ZUNTRACEX(stat,"formatp=%d",(formatp?-1:*formatp)); +} + /** * @internal Return the extended format (i.e. the dispatch model), * plus the mode associated with an open file. diff --git a/libnczarr/zinternal.h b/libnczarr/zinternal.h index 471bf2f1b0..95f6060e52 100644 --- a/libnczarr/zinternal.h +++ b/libnczarr/zinternal.h @@ -225,7 +225,7 @@ int NCZ_initialize(void); int NCZ_finalize(void); int NCZ_initialize_internal(void); int NCZ_finalize_internal(void); -int ncz_get_fill_value(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, void **fillp); +int NCZ_ensure_fill_value(NC_VAR_INFO_T* var); int ncz_find_grp_var_att(int ncid, int varid, const char *name, int attnum, int use_name, char *norm_name, NC_FILE_INFO_T** file, NC_GRP_INFO_T** grp, NC_VAR_INFO_T** var, @@ -251,6 +251,7 @@ int ncz_makeattr(NC_OBJ*, NCindex* attlist, const char* name, nc_type typid, siz /* zvar.c */ int ncz_gettype(NC_FILE_INFO_T*, NC_GRP_INFO_T*, int xtype, NC_TYPE_INFO_T** typep); int ncz_find_default_chunksizes2(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var); +int NCZ_ensure_quantizer(int ncid, NC_VAR_INFO_T* var); /* Undefined */ /* Find var, doing lazy var metadata read if needed. */ diff --git a/libnczarr/zsync.c b/libnczarr/zsync.c index 9241badea8..5a6e395385 100644 --- a/libnczarr/zsync.c +++ b/libnczarr/zsync.c @@ -270,6 +270,21 @@ ncz_sync_var_meta(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, int isclose) zinfo = file->format_file_info; map = zinfo->map; +#if 1 + /* Make sure that everything is established */ + /* ensure the fill value */ + if((stat = NCZ_ensure_fill_value(var))) goto done; /* ensure var->fill_value is set */ + assert(var->no_fill || var->fill_value != NULL); + /* ensure the chunk cache */ + if((stat = NCZ_adjust_var_cache(var))) goto done; + /* rebuild the fill chunk */ + if((stat = NCZ_ensure_fill_chunk(zvar->cache))) goto done; +#ifdef ENABLE_NCZARR_FILTERS + /* Build the filter working parameters for any filters */ + if((stat = NCZ_filter_setup(var))) goto done; +#endif +#endif /*0|1*/ + /* Construct var path */ if((stat = NCZ_varkey(var,&fullpath))) goto done; @@ -344,7 +359,9 @@ ncz_sync_var_meta(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, int isclose) if((stat=NCJnew(NCJ_NULL,&jfill))) goto done; } else {/*!var->no_fill*/ int atomictype = var->type_info->hdr.id; - assert(var->fill_value != NULL); + if(var->fill_value == NULL) { + if((stat = NCZ_ensure_fill_value(var))) goto done; + } /* Convert var->fill_value to a string */ if((stat = NCZ_stringconvert(atomictype,1,var->fill_value,&jfill))) goto done; assert(jfill->sort != NCJ_ARRAY); @@ -473,6 +490,8 @@ ncz_sync_var_meta(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, int isclose) goto done; nullfree(key); key = NULL; + var->created = 1; + /* Build .zattrs object */ assert(var->att); if((stat = ncz_sync_atts(file,(NC_OBJ*)var, var->att, isclose))) @@ -506,7 +525,7 @@ ncz_sync_var(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, int isclose) int stat = NC_NOERR; NCZ_VAR_INFO_T* zvar = var->format_var_info; - if(!isclose) { + if(isclose) { if((stat = ncz_sync_var_meta(file,var,isclose))) goto done; } @@ -609,6 +628,7 @@ ncz_sync_atts(NC_FILE_INFO_T* file, NC_OBJ* container, NCindex* attlist, int isc NCjson* jtype = NULL; NCjson* jdimrefs = NULL; NCjson* jdict = NULL; + NCjson* jint = NULL; NCZMAP* map = NULL; char* fullpath = NULL; char* key = NULL; @@ -616,8 +636,15 @@ ncz_sync_atts(NC_FILE_INFO_T* file, NC_OBJ* container, NCindex* attlist, int isc char* dimpath = NULL; int isxarray = 0; int isrootgroup = 0; - + NC_VAR_INFO_T* var = NULL; + NC_GRP_INFO_T* grp = NULL; + LOG((3, "%s", __func__)); + + if(container->sort == NCVAR) + var = (NC_VAR_INFO_T*)container; + else if(container->sort == NCGRP) + grp = (NC_GRP_INFO_T*)container; zinfo = file->format_file_info; map = zinfo->map; @@ -625,14 +652,10 @@ ncz_sync_atts(NC_FILE_INFO_T* file, NC_OBJ* container, NCindex* attlist, int isc if(zinfo->controls.flags & FLAG_XARRAYDIMS) isxarray = 1; if(container->sort == NCVAR) { - NC_VAR_INFO_T* var = (NC_VAR_INFO_T*)container; if(var->container && var->container->parent == NULL) isrootgroup = 1; } - if(!isxarray && ncindexsize(attlist) == 0) - goto done; /* do nothing */ - if(ncindexsize(attlist) > 0) { /* Create the jncattr.types object */ if((stat = NCJnew(NCJ_DICT,&jtypes))) @@ -657,9 +680,9 @@ ncz_sync_atts(NC_FILE_INFO_T* file, NC_OBJ* container, NCindex* attlist, int isc /* Construct container path */ if(container->sort == NCGRP) - stat = NCZ_grpkey((NC_GRP_INFO_T*)container,&fullpath); + stat = NCZ_grpkey(grp,&fullpath); else - stat = NCZ_varkey((NC_VAR_INFO_T*)container,&fullpath); + stat = NCZ_varkey(var,&fullpath); if(stat) goto done; @@ -667,9 +690,8 @@ ncz_sync_atts(NC_FILE_INFO_T* file, NC_OBJ* container, NCindex* attlist, int isc if((stat = ncz_jsonize_atts(attlist,&jatts))) goto done; - if(container->sort == NCVAR) { + if(container->sort == NCVAR) { if(isrootgroup && isxarray) { - NC_VAR_INFO_T* var = (NC_VAR_INFO_T*)container; /* Insert the XARRAY _ARRAY_ATTRIBUTE attribute */ if((stat = NCJnew(NCJ_ARRAY,&jdimrefs))) goto done; @@ -686,23 +708,44 @@ ncz_sync_atts(NC_FILE_INFO_T* file, NC_OBJ* container, NCindex* attlist, int isc jdimrefs = NULL; } } - if(!(zinfo->controls.flags & FLAG_PUREZARR)) { - /* Insert the _NCZARR_ATTR attribute */ - if((stat = NCJnew(NCJ_DICT,&jdict))) + /* Add Quantize Attribute */ + if(container->sort == NCVAR && var && var->quantize_mode > 0) { + char mode[64]; + snprintf(mode,sizeof(mode),"%d",var->nsd); + if((stat = NCJnewstring(NCJ_INT,mode,&jint))) goto done; - if((stat = NCJinsert(jdict,"types",jtypes))) goto done; - jtypes = NULL; - if((stat = NCJinsert(jatts,NCZ_V2_ATTR,jdict))) goto done; - jdict = NULL; + /* Insert the quantize attribute */ + switch (var->quantize_mode) { + case NC_QUANTIZE_BITGROOM: + if((stat = NCJinsert(jatts,NC_QUANTIZE_BITGROOM_ATT_NAME,jint))) goto done; + jint = NULL; + break; + case NC_QUANTIZE_GRANULARBG: + if((stat = NCJinsert(jatts,NC_QUANTIZE_GRANULARBG_ATT_NAME,jint))) goto done; + jint = NULL; + break; + default: break; + } } - /* write .zattrs path */ - if((stat = nczm_concat(fullpath,ZATTRS,&key))) - goto done; - /* Write to map */ - if((stat=NCZ_uploadjson(map,key,jatts))) - goto done; - nullfree(key); key = NULL; + if(NCJlength(jatts) > 0) { + if(!(zinfo->controls.flags & FLAG_PUREZARR)) { + /* Insert the _NCZARR_ATTR attribute */ + if((stat = NCJnew(NCJ_DICT,&jdict))) + goto done; + if((stat = NCJinsert(jdict,"types",jtypes))) goto done; + jtypes = NULL; + if((stat = NCJinsert(jatts,NCZ_V2_ATTR,jdict))) goto done; + jdict = NULL; + } + /* write .zattrs path */ + if((stat = nczm_concat(fullpath,ZATTRS,&key))) + goto done; + /* Write to map */ + if((stat=NCZ_uploadjson(map,key,jatts))) + goto done; + nullfree(key); key = NULL; + } done: nullfree(fullpath); @@ -714,6 +757,7 @@ ncz_sync_atts(NC_FILE_INFO_T* file, NC_OBJ* container, NCindex* attlist, int isc NCJreclaim(jtype); NCJreclaim(jdimrefs); NCJreclaim(jdict); + NCJreclaim(jint); return THROW(stat); } @@ -1419,6 +1463,12 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames) var->format_var_info = zvar; zvar->common.file = file; + /* pretend it was created */ + var->created = 1; + + /* Indicate we do not have quantizer yet */ + var->quantize_mode = -1; + /* Set filter list */ assert(var->filters == NULL); var->filters = (void*)nclistnew(); @@ -2123,6 +2173,10 @@ ncz_get_var_meta(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var) if ((retval = nc4_adjust_var_cache(var))) BAIL(retval); + /* Is there an attribute which means quantization was used? */ + if ((retval = get_quantize_info(var))) + BAIL(retval); + if (var->coords_read && !var->dimscale) if ((retval = get_attached_info(var, hdf5_var, var->ndims, hdf5_var->hdf_datasetid))) return retval; diff --git a/libnczarr/ztracedispatch.h b/libnczarr/ztracedispatch.h deleted file mode 100644 index 3db5ffb98c..0000000000 --- a/libnczarr/ztracedispatch.h +++ /dev/null @@ -1,572 +0,0 @@ -/* Copyright 2005-2018 University Corporation for Atmospheric - Research/Unidata. */ - -/** - * @file - * @internal This header file contains prototypes and initialization - * for the ZARR dispatch layer. - * - * @author Dennis Heimbigner, Ed Hartnett - */ - -#include "zincludes.h" - -static int NCZTR_create(const char *path, int cmode, size_t initialsz, int basepe, size_t *chunksizehintp, void *parameters, const struct NC_Dispatch *table, int ncid) -{ - int stat = NC_NOERR; - stat = NCZ_create(path,cmode,initialsz,basepe,chunksizehintp,parameters,table,ncid); - return stat; -} - -static int NCZTR_open(const char *path, int mode, int basepe, size_t *chunksizehintp,void *parameters, const struct NC_Dispatch *table, int ncid) -{ - int stat = NC_NOERR; - stat = NCZ_open(path,mode,basepe,chunksizehintp,parameters,table,ncid); - return stat; -} - -static int NCZTR_redef(int ncid) -{ - int stat = NC_NOERR; - ZTRACE(0,"NCZ_redef(ncid)"); - stat = NCZ_redef(ncid); - return ZUNTRACE(stat); -} - -static int NCZTR__enddef(int ncid,size_t h_minfree,size_t v_align,size_t v_minfree,size_t r_align) -{ - int stat = NC_NOERR; - stat = NCZ__enddef(ncid,h_minfree,v_align,v_minfree,r_align); - return stat; -} - -static int NCZTR_sync(int ncid) -{ - int stat = NC_NOERR; - stat = NCZ_sync(ncid); - return stat; -} - -static int NCZTR_abort(int ncid) -{ - int stat = NC_NOERR; - stat = NCZ_abort(ncid); - return stat; -} - -static int NCZTR_close(int ncid, void* params) -{ - int stat = NC_NOERR; - stat = NCZ_close(ncid,params); - return stat; -} - -static int NCZTR_set_fill(int ncid, int fillmode, int *old) -{ - int stat = NC_NOERR; - ZTRACE(0,"NCZ_set_fill(ncid,fillmode,old)"); - stat = NCZ_set_fill(ncid,fillmode,old); - return ZUNTRACE(stat); -} - -static int NCZTR_inq_format(int ncid, int* formatp) -{ - int stat = NC_NOERR; - ZTRACE(0,"NC4_inq_format(ncid,formatp)"); - stat = NC4_inq_format(ncid,formatp); - return ZUNTRACE(stat); -} - -static int NCZTR_inq_format_extended(int ncid, int *formatp, int *modep) -{ - int stat = NC_NOERR; - ZTRACE(0,"NCZ_inq_format_extended(ncid,formatp,modep)"); - stat = NCZ_inq_format_extended(ncid,formatp,modep); - return ZUNTRACE(stat); -} - -static int NCZTR_inq(int ncid, int *ndimsp, int *nvarsp, int *nattsp, int *udimp) -{ - int stat = NC_NOERR; - ZTRACE(0,"NCZ_inq(ncid,ndimsp,nvarsp,nattsp,udimp)"); - stat = NCZ_inq(ncid,ndimsp,nvarsp,nattsp,udimp); - return ZUNTRACE(stat); -} - -static int NCZTR_inq_type(int ncid, nc_type xtype, char *name, size_t *size) -{ - int stat = NC_NOERR; - ZTRACE(0,"NC4_inq_type(ncid,xtype,name,size)"); - stat = NC4_inq_type(ncid,xtype,name,size); - return ZUNTRACE(stat); -} - -static int NCZTR_def_dim(int ncid, const char *name, size_t len, int *idp) -{ - int stat = NC_NOERR; - ZTRACE(0,"NCZ_def_dim(ncid,name,len,idp)"); - stat = NCZ_def_dim(ncid,name,len,idp); - return ZUNTRACE(stat); -} - -static int NCZTR_inq_dimid(int ncid, const char *name, int *idp) -{ - int stat = NC_NOERR; - ZTRACE(0,"NC4_inq_dimid(ncid,name,idp)"); - stat = NC4_inq_dimid(ncid,name,idp); - return ZUNTRACE(stat); -} - -static int NCZTR_inq_dim(int ncid, int dimid, char *name, size_t *lenp) -{ - int stat = NC_NOERR; - ZTRACE(0,"NCZ_inq_dim(ncid,dimid,name,lenp)"); - stat = NCZ_inq_dim(ncid,dimid,name,lenp); - return ZUNTRACE(stat); -} - -static int NCZTR_inq_unlimdim(int ncid, int *unlimdimidp) -{ - int stat = NC_NOERR; - ZTRACE(0,"NC4_inq_unlimdim(ncid,unlimdimidp)"); - stat = NC4_inq_unlimdim(ncid,unlimdimidp); - return ZUNTRACE(stat); -} - -static int NCZTR_rename_dim(int ncid, int dimid, const char *name) -{ - int stat = NC_NOERR; - ZTRACE(0,"NCZ_rename_dim(ncid,dimid,name)"); - stat = NCZ_rename_dim(ncid,dimid,name); - return ZUNTRACE(stat); -} - -static int NCZTR_inq_att(int ncid, int varid, const char *name, nc_type *xtypep, size_t *lenp) -{ - int stat = NC_NOERR; - ZTRACE(0,"NCZ_inq_att(ncid,varid,name,xtypep,lenp)"); - stat = NCZ_inq_att(ncid,varid,name,xtypep,lenp); - return ZUNTRACE(stat); -} - -static int NCZTR_inq_attid(int ncid, int varid, const char* name, int *idp) -{ - int stat = NC_NOERR; - ZTRACE(0,"NCZ_inq_attid(ncid,varid,name,idp)"); - stat = NCZ_inq_attid(ncid,varid,name,idp); - return ZUNTRACE(stat); -} - -static int NCZTR_inq_attname(int ncid, int varid, int attnum, char *name) -{ - int stat = NC_NOERR; - ZTRACE(0,"NCZ_inq_attname(ncid,varid,attnum,name)"); - stat = NCZ_inq_attname(ncid,varid,attnum,name); - return ZUNTRACE(stat); -} - -static int NCZTR_rename_att(int ncid, int varid, const char* name, const char *newname) -{ - int stat = NC_NOERR; - ZTRACE(0,"NCZ_rename_att(ncid,varid,name,newname)"); - stat = NCZ_rename_att(ncid,varid,name,newname); - return ZUNTRACE(stat); -} - -static int NCZTR_del_att(int ncid, int varid, const char *name) -{ - int stat = NC_NOERR; - ZTRACE(0,"NCZ_del_att(ncid,varid,name)"); - stat = NCZ_del_att(ncid,varid,name); - return ZUNTRACE(stat); -} - -static int NCZTR_get_att(int ncid, int varid, const char* name, void *data, nc_type memtype) -{ - int stat = NC_NOERR; - ZTRACE(0,"NCZ_get_att(ncid,varid,name,data,memtype)"); - stat = NCZ_get_att(ncid,varid,name,data,memtype); - return ZUNTRACE(stat); -} - -static int NCZTR_put_att(int ncid, int varid, const char* name, nc_type filetype, size_t len, const void *data, nc_type memtype) -{ - int stat = NC_NOERR; - ZTRACE(0,"NCZ_put_att(ncid,varid,name,filetype,len,data,memtype)"); - stat = NCZ_put_att(ncid,varid,name,filetype,len,data,memtype); - return ZUNTRACE(stat); -} - -static int NCZTR_def_var(int ncid, const char* name, nc_type xtype, int ndims, const int *dimidsp, int *varidp) -{ - int stat = NC_NOERR; - stat = NCZ_def_var(ncid,name,xtype,ndims,dimidsp,varidp); - return stat; -} - -static int NCZTR_inq_varid(int ncid, const char* name, int *varidp) -{ - int stat = NC_NOERR; - ZTRACE(0,"NC4_inq_varid(ncid,name,varidp)"); - stat = NC4_inq_varid(ncid,name,varidp); - return ZUNTRACE(stat); -} - -static int NCZTR_rename_var(int ncid, int varid, const char *name) -{ - int stat = NC_NOERR; - stat = NCZ_rename_var(ncid,varid,name); - return stat; -} - -static int NCZTR_get_vara(int ncid, int varid, const size_t *startp, const size_t *countp, void *ip, nc_type memtype) -{ - int stat = NC_NOERR; - ZTRACE(0,"NCZ_get_vara(ncid,varid,startp,countp,ip,memtype)"); - stat = NCZ_get_vara(ncid,varid,startp,countp,ip,memtype); - return ZUNTRACE(stat); -} - -static int NCZTR_put_vara(int ncid, int varid, const size_t *startp, const size_t *countp, const void *ip, nc_type memtype) -{ - int stat = NC_NOERR; - ZTRACE(0,"NCZ_put_vara(ncid,varid,startp,countp,ip,memtype)"); - stat = NCZ_put_vara(ncid,varid,startp,countp,ip,memtype); - return ZUNTRACE(stat); -} - -static int NCZTR_inq_var_all(int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp, int *dimidsp, int *nattsp, int *shufflep, int *deflatep, int *deflate_levelp, int *fletcher32p, int *contiguousp, size_t *chunksizesp, int *no_fill, void *fill_valuep, int *endiannessp, unsigned int *idp, size_t *nparamsp, unsigned int *params) -{ - int stat = NC_NOERR; - stat = NCZ_inq_var_all(ncid,varid,name,xtypep,ndimsp,dimidsp,nattsp,shufflep,deflatep,deflate_levelp,fletcher32p,contiguousp,chunksizesp,no_fill,fill_valuep,endiannessp,idp,nparamsp,params); - return stat; -} - -static int NCZTR_var_par_access(int ncid, int varid, int par_access) -{ - int stat = NC_NOERR; - ZTRACE(0,"NCZ_var_par_access(ncid,varid,par_access)"); - stat = NCZ_var_par_access(ncid,varid,par_access); - return ZUNTRACE(stat); -} - -static int NCZTR_def_var_fill(int ncid, int varid, int no_fill, const void *fill_value) -{ - int stat = NC_NOERR; - ZTRACE(0,"NCZ_def_var_fill(ncid,varid,no_fill,fill_value)"); - stat = NCZ_def_var_fill(ncid,varid,no_fill,fill_value); - return ZUNTRACE(stat); -} - -static int NCZTR_show_metadata(int ncid) -{ - int stat = NC_NOERR; - ZTRACE(0,"NCZ_show_metadata(ncid)"); - stat = NCZ_show_metadata(ncid); - return ZUNTRACE(stat); -} - -static int NCZTR_inq_unlimdims(int ncid, int* n, int* uidsp) -{ - int stat = NC_NOERR; - ZTRACE(0,"NCZ_inq_unlimdims(ncid,n,uidsp)"); - stat = NCZ_inq_unlimdims(ncid,n,uidsp); - return ZUNTRACE(stat); -} - -static int NCZTR_inq_ncid(int ncid, const char* name, int* grpidp) -{ - int stat = NC_NOERR; - ZTRACE(0,"NC4_inq_ncid(ncid,name,grpidp)"); - stat = NC4_inq_ncid(ncid,name,grpidp); - return ZUNTRACE(stat); -} - -static int NCZTR_inq_grps(int ncid, int* n, int* ncids) -{ - int stat = NC_NOERR; - ZTRACE(0,"NC4_inq_grps(ncid,n,ncids)"); - stat = NC4_inq_grps(ncid,n,ncids); - return ZUNTRACE(stat); -} - -static int NCZTR_inq_grpname(int ncid, char* name) -{ - int stat = NC_NOERR; - ZTRACE(0,"NC4_inq_grpname(ncid,name)"); - stat = NC4_inq_grpname(ncid,name); - return ZUNTRACE(stat); -} - -static int NCZTR_inq_grpname_full(int ncid, size_t* lenp, char* fullname) -{ - int stat = NC_NOERR; - ZTRACE(0,"NC4_inq_grpname_full(ncid,lenp,fullname)"); - stat = NC4_inq_grpname_full(ncid,lenp,fullname); - return ZUNTRACE(stat); -} - -static int NCZTR_inq_grp_parent(int ncid, int* parentidp) -{ - int stat = NC_NOERR; - ZTRACE(0,"NC4_inq_grp_parent(ncid,parentidp)"); - stat = NC4_inq_grp_parent(ncid,parentidp); - return ZUNTRACE(stat); -} - -static int NCZTR_inq_grp_full_ncid(int ncid, const char* fullname, int* grpidp) -{ - int stat = NC_NOERR; - ZTRACE(0,"NC4_inq_grp_full_ncid(ncid,fullname,grpidp)"); - stat = NC4_inq_grp_full_ncid(ncid,fullname,grpidp); - return ZUNTRACE(stat); -} - -static int NCZTR_inq_varids(int ncid, int* nvars, int* varids) -{ - int stat = NC_NOERR; - ZTRACE(0,"NC4_inq_varids(ncid,nvars,varids)"); - stat = NC4_inq_varids(ncid,nvars,varids); - return ZUNTRACE(stat); -} - -static int NCZTR_inq_dimids(int ncid, int* ndims, int* dimids, int inclparents) -{ - int stat = NC_NOERR; - ZTRACE(0,"NC4_inq_dimids(ncid,ndims,dimids,inclparents)"); - stat = NC4_inq_dimids(ncid,ndims,dimids,inclparents); - return ZUNTRACE(stat); -} - -static int NCZTR_inq_typeids(int ncid, int* ntypes, int* typeids) -{ - int stat = NC_NOERR; - ZTRACE(0,"NCZ_inq_typeids(ncid,ntypes,typeids)"); - stat = NCZ_inq_typeids(ncid,ntypes,typeids); - return ZUNTRACE(stat); -} - -static int NCZTR_inq_type_equal(int ncid1, nc_type tid1, int ncid2, nc_type tid2, int* eq) -{ - int stat = NC_NOERR; - ZTRACE(0,"NCZ_inq_type_equal(ncid1,tid1,ncid2,tid2,eq)"); - stat = NCZ_inq_type_equal(ncid1,tid1,ncid2,tid2,eq); - return ZUNTRACE(stat); -} - -static int NCZTR_def_grp(int parent, const char* name, int* grpid) -{ - int stat = NC_NOERR; - ZTRACE(0,"NCZ_def_grp(parent,name,grpid)"); - stat = NCZ_def_grp(parent,name,grpid); - return ZUNTRACE(stat); -} - -static int NCZTR_rename_grp(int ncid, const char* name) -{ - int stat = NC_NOERR; - ZTRACE(0,"NCZ_rename_grp(ncid,name)"); - stat = NCZ_rename_grp(ncid,name); - return ZUNTRACE(stat); -} - -static int NCZTR_inq_user_type(int ncid, nc_type xtype, char* name, size_t* size, nc_type* basetid, size_t* nfields, int* classp) -{ - int stat = NC_NOERR; - ZTRACE(0,"NC4_inq_user_type(ncid,xtype,name,size,basetid,nfields,classp)"); - stat = NC4_inq_user_type(ncid,xtype,name,size,basetid,nfields,classp); - return ZUNTRACE(stat); -} - -static int NCZTR_inq_typeid(int ncid, const char* name, nc_type* tidp) -{ - int stat = NC_NOERR; - ZTRACE(0,"NCZ_inq_typeid(ncid,name,tidp)"); - stat = NCZ_inq_typeid(ncid,name,tidp); - return ZUNTRACE(stat); -} - -static int NCZTR_def_var_chunking(int ncid, int varid, int storage, const size_t *chunksizes) -{ - int stat = NC_NOERR; - ZTRACE(0,"NCZ_def_var_chunking(ncid,varid,storage,chunksizes)"); - stat = NCZ_def_var_chunking(ncid,varid,storage,chunksizes); - return ZUNTRACE(stat); -} - -static int NCZTR_def_var_endian(int ncid, int varid, int endian) -{ - int stat = NC_NOERR; - ZTRACE(0,"NCZ_def_var_endian(ncid,varid,endian)"); - stat = NCZ_def_var_endian(ncid,varid,endian); - return ZUNTRACE(stat); -} - -static int NCZTR_set_var_chunk_cache(int ncid, int varid, size_t size, size_t nelems, float preemption) -{ - int stat = NC_NOERR; - ZTRACE(0,"NCZ_set_var_chunk_cache(ncid,varid,size,nelems,preemption)"); - stat = NCZ_set_var_chunk_cache(ncid,varid,size,nelems,preemption); - return ZUNTRACE(stat); -} - -static int NCZTR_get_var_chunk_cache(int ncid, int varid, size_t *sizep, size_t *nelemsp, float *preemptionp) -{ - int stat = NC_NOERR; - ZTRACE(0,"NC4_get_var_chunk_cache(ncid,varid,sizep,nelemsp,preemptionp)"); - stat = NC4_get_var_chunk_cache(ncid,varid,sizep,nelemsp,preemptionp); - return ZUNTRACE(stat); -} - -static int NCZTR_inq_var_filter_ids(int ncid, int varid, size_t* nfilters, unsigned int* filterids) -{ - int stat = NC_NOERR; - ZTRACE(0,"NCZ_inq_var_filter_ids(ncid,varid,nfilters,filterids)"); - stat = NCZ_inq_var_filter_ids(ncid,varid,nfilters,filterids); - return ZUNTRACE(stat); -} - -static int NCZTR_inq_var_filter_info(int ncid, int varid, unsigned int id, size_t* nparams, unsigned int* params) -{ - int stat = NC_NOERR; - ZTRACE(0,"NCZ_inq_var_filter_info(ncid,varid,id,nparams,params)"); - stat = NCZ_inq_var_filter_info(ncid,varid,id,nparams,params); - return ZUNTRACE(stat); -} - - -static int NCZTR_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp, const ptrdiff_t *stridep, void *ip, nc_type memtype) -{ - int stat = NC_NOERR; - ZTRACE(0,"NCZ_get_vars(ncid,varid,startp,countp,stridep,ip,memtype)"); - stat = NCZ_get_vars(ncid,varid,startp,countp,stridep,ip,memtype); - return ZUNTRACE(stat); -} - -static int NCZTR_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp, const ptrdiff_t *stridep, const void *ip, nc_type memtype) -{ - int stat = NC_NOERR; - ZTRACE(0,"NCZ_put_vars(ncid,varid,startp,countp,stridep,ip,memtype)"); - stat = NCZ_put_vars(ncid,varid,startp,countp,stridep,ip,memtype); - return ZUNTRACE(stat); -} - -#if 0 -static int NCZTR_def_var_deflate(int ncid, int varid, int shuffle, int deflate, int level) -{ - int stat = NC_NOERR; - ZTRACE(0,"NCZ_def_var_deflate(ncid,varid,shuffle,deflate,level)"); - stat = NCZ_def_var_deflate(ncid,varid,shuffle,deflate,level); - return ZUNTRACE(stat); -} - -static int NCZTR_def_var_fletcher32(int ncid, int varid, int fletcher32) -{ - int stat = NC_NOERR; - ZTRACE(0,"NCZ_def_var_fletcher32(ncid,varid,fletcher32)"); - stat = NCZ_def_var_fletcher32(ncid,varid,fletcher32); - return ZUNTRACE(stat); -} - -static int NCZTR_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams, const unsigned int *params) -{ - int stat = NC_NOERR; - stat = NCZ_def_var_filter(ncid,varid,id,nparams,params); - return (stat); -} - -#endif - -static const NC_Dispatch NCZ_dispatcher_trace = { - - NC_FORMATX_NCZARR, - NC_DISPATCH_VERSION, - - NCZTR_create, - NCZTR_open, - - NCZTR_redef, - NCZTR__enddef, - NCZTR_sync, - NCZTR_abort, - NCZTR_close, - NCZTR_set_fill, - NCZTR_inq_format, - NCZTR_inq_format_extended, - - NCZTR_inq, - NCZTR_inq_type, - - NCZTR_def_dim, - NCZTR_inq_dimid, - NCZTR_inq_dim, - NCZTR_inq_unlimdim, - NCZTR_rename_dim, - - NCZTR_inq_att, - NCZTR_inq_attid, - NCZTR_inq_attname, - NCZTR_rename_att, - NCZTR_del_att, - NCZTR_get_att, - NCZTR_put_att, - - NCZTR_def_var, - NCZTR_inq_varid, - NCZTR_rename_var, - NCZTR_get_vara, - NCZTR_put_vara, - NCZTR_get_vars, - NCZTR_put_vars, - NCDEFAULT_get_varm, - NCDEFAULT_put_varm, - - NCZTR_inq_var_all, - - NCZTR_var_par_access, - NCZTR_def_var_fill, - - NCZTR_show_metadata, - NCZTR_inq_unlimdims, - - NCZTR_inq_ncid, - NCZTR_inq_grps, - NCZTR_inq_grpname, - NCZTR_inq_grpname_full, - NCZTR_inq_grp_parent, - NCZTR_inq_grp_full_ncid, - NCZTR_inq_varids, - NCZTR_inq_dimids, - NCZTR_inq_typeids, - NCZTR_inq_type_equal, - NCZTR_def_grp, - NCZTR_rename_grp, - NCZTR_inq_user_type, - NCZTR_inq_typeid, - - NC_NOTNC4_def_compound, - NC_NOTNC4_insert_compound, - NC_NOTNC4_insert_array_compound, - NC_NOTNC4_inq_compound_field, - NC_NOTNC4_inq_compound_fieldindex, - NC_NOTNC4_def_vlen, - NC_NOTNC4_put_vlen_element, - NC_NOTNC4_get_vlen_element, - NC_NOTNC4_def_enum, - NC_NOTNC4_insert_enum, - NC_NOTNC4_inq_enum_member, - NC_NOTNC4_inq_enum_ident, - NC_NOTNC4_def_opaque, - NC_NOTNC4_def_var_deflate, - NC_NOTNC4_def_var_fletcher32, - NCZTR_def_var_chunking, - NCZTR_def_var_endian, - NCZ_def_var_filter, - NCZTR_set_var_chunk_cache, - NCZTR_get_var_chunk_cache, - NCZTR_inq_var_filter_ids, - NCZTR_inq_var_filter_info, - NC_NOTNC4_def_var_quantize, - NC_NOTNC4_inq_var_quantize, -}; - diff --git a/libnczarr/ztype.c b/libnczarr/ztype.c index b8463aaaa8..12035dd0dd 100644 --- a/libnczarr/ztype.c +++ b/libnczarr/ztype.c @@ -133,8 +133,9 @@ NCZ_inq_typeid(int ncid, const char *name, nc_type *typeidp) int NCZ_inq_typeids(int ncid, int *ntypes, int *typeids) { + ZTRACE(0,"ncid=%d",ncid); if(ntypes) *ntypes = 0; - return NC_NOERR; + return ZUNTRACEX(NC_NOERR,"ntypes=%d typeids=%p",(ntypes?-1:*ntypes),typeids); } #ifdef LOOK diff --git a/libnczarr/zutil.c b/libnczarr/zutil.c index acf4dcb417..fb04dd38a3 100644 --- a/libnczarr/zutil.c +++ b/libnczarr/zutil.c @@ -729,51 +729,6 @@ NCZ_freestringvec(size_t len, char** vec) nullfree(vec); } -/* create a fill chunk */ -int -NCZ_create_fill_chunk(size64_t chunksize, size_t typesize, const void* fill, void** fillchunkp) -{ - int i; - void* fillchunk = NULL; - if((fillchunk = malloc(chunksize))==NULL) - return NC_ENOMEM; - if(fill == NULL) { - /* use zeros */ - memset(fillchunk,0,chunksize); - goto done; - } - switch (typesize) { - case 1: { - unsigned char c = *((unsigned char*)fill); - memset(fillchunk,c,chunksize); - } break; - case 2: { - unsigned short fv = *((unsigned short*)fill); - unsigned short* p2 = (unsigned short*)fillchunk; - for(i=0;ifill_value) { + int ncid = var->container->nc4_info->controller->ext_ncid; + int tid = var->type_info->hdr.id; + stat = nc_reclaim_data_all(ncid,tid,var->fill_value,1); + var->fill_value = NULL; + } + /* Reclaim any existing fill_chunk */ + if(!stat) stat = NCZ_reclaim_fill_chunk(((NCZ_VAR_INFO_T*)var->format_var_info)->cache); + return stat; +} + +int +NCZ_copy_fill_value(NC_VAR_INFO_T* var, void** dstp) +{ + int stat = NC_NOERR; + int ncid = var->container->nc4_info->controller->ext_ncid; + int tid = var->type_info->hdr.id; + void* dst = NULL; + + if(var->fill_value) { + if((stat = nc_copy_data_all(ncid,tid,var->fill_value,1,&dst))) goto done; + } + if(dstp) {*dstp = dst; dst = NULL;} +done: + if(dst) (void)nc_reclaim_data_all(ncid,tid,dst,1); + return stat; +} diff --git a/libnczarr/zvar.c b/libnczarr/zvar.c index b0812d5bec..c32288e69b 100644 --- a/libnczarr/zvar.c +++ b/libnczarr/zvar.c @@ -436,16 +436,18 @@ var->type_info->rc++; * variables which may be contiguous.) */ LOG((4, "allocating array of %d size_t to hold chunksizes for var %s", var->ndims, var->hdr.name)); - if (var->ndims) { - if (!(var->chunksizes = calloc(var->ndims, sizeof(size_t)))) - BAIL(NC_ENOMEM); - if ((retval = ncz_find_default_chunksizes2(grp, var))) - BAIL(retval); - } else { - /* Pretend that scalars are like var[1] */ - if (!(var->chunksizes = calloc(1, sizeof(size_t)))) - BAIL(NC_ENOMEM); - var->chunksizes[0] = 1; + if(!var->chunksizes) { + if(var->ndims) { + if (!(var->chunksizes = calloc(var->ndims+zvar->scalar, sizeof(size_t)))) + BAIL(NC_ENOMEM); + if ((retval = ncz_find_default_chunksizes2(grp, var))) + BAIL(retval); + } else { + /* Pretend that scalars are like var[1] */ + if (!(var->chunksizes = calloc(1, sizeof(size_t)))) + BAIL(NC_ENOMEM); + var->chunksizes[0] = 1; + } } /* Compute the chunksize cross product */ @@ -510,7 +512,8 @@ static int ncz_def_var_extra(int ncid, int varid, int *shuffle, int *unused1, int *unused2, int *fletcher32, int *storagep, const size_t *chunksizes, int *no_fill, - const void *fill_value, int *endianness) + const void *fill_value, int *endianness, + int *quantize_mode, int *nsd) { NC_GRP_INFO_T *grp; NC_FILE_INFO_T *h5; @@ -522,13 +525,16 @@ ncz_def_var_extra(int ncid, int varid, int *shuffle, int *unused1, LOG((2, "%s: ncid 0x%x varid %d", __func__, ncid, varid)); - ZTRACE(2,"ncid=%d varid=%d shuffle=%d fletcher32=%d no_fill=%d, fill_value=%p endianness=%d", + ZTRACE(2,"ncid=%d varid=%d shuffle=%d fletcher32=%d no_fill=%d, fill_value=%p endianness=%d quantize_mode=%d nsd=%d", ncid,varid, (shuffle?*shuffle:-1), (fletcher32?*fletcher32:-1), (no_fill?*no_fill:-1), fill_value, - (endianness?*endianness:-1)); + (endianness?*endianness:-1), + (quantize_mode?*quantize_mode:-1), + (nsd?*nsd:-1) + ); /* Find info for this file and group, and set pointer to each. */ if ((retval = nc4_find_nc_grp_h5(ncid, NULL, &grp, &h5))) @@ -544,6 +550,8 @@ ncz_def_var_extra(int ncid, int varid, int *shuffle, int *unused1, {retval = NC_ENOTVAR; goto done;} assert(var && var->hdr.id == varid); + zvar = var->format_var_info; + ZTRACEMORE(1,"\tstoragep=%d chunksizes=%s",(storagep?*storagep:-1),(chunksizes?nczprint_sizevector(var->ndims,chunksizes):"null")); /* Can't turn on parallel and deflate/fletcher32/szip/shuffle @@ -633,7 +641,7 @@ ncz_def_var_extra(int ncid, int varid, int *shuffle, int *unused1, {retval = NC_EINVAL; goto done;} } else if (storage == NC_CHUNKED && var->ndims > 0) { var->storage = NC_CHUNKED; - + /* If the user provided chunksizes, check that they are valid * and that their total size of chunk is less than 4 GB. */ if (chunksizes) @@ -661,6 +669,7 @@ ncz_def_var_extra(int ncid, int varid, int *shuffle, int *unused1, int anyzero = 0; /* check for any zero length chunksizes */ zvar = var->format_var_info; assert(zvar->cache != NULL); + zvar->cache->valid = 0; if(chunksizes) { for (d = 0; d < var->ndims; d++) { var->chunksizes[d] = chunksizes[d]; @@ -679,6 +688,8 @@ ncz_def_var_extra(int ncid, int varid, int *shuffle, int *unused1, zvar->chunkproduct *= var->chunksizes[d]; zvar->chunksize = zvar->chunkproduct * var->type_info->size; } + /* Adjust cache */ + if((retval = NCZ_adjust_var_cache(var))) goto done; #ifdef LOGGING { @@ -722,12 +733,13 @@ ncz_def_var_extra(int ncid, int varid, int *shuffle, int *unused1, if ((retval = nc_put_att(ncid, varid, _FillValue, var->type_info->hdr.id, 1, fill_value))) goto done; + /* Reclaim any existing fill_chunk */ + if((retval = NCZ_reclaim_fill_chunk(zvar->cache))) goto done; } else if (var->fill_value && no_fill && (*no_fill)) { /* Turning off fill value? */ /* If there's a _FillValue attribute, delete it. */ retval = NCZ_del_att(ncid, varid, _FillValue); if (retval && retval != NC_ENOTATT) return retval; - if((retval = nc_reclaim_data_all(ncid,var->type_info->hdr.id,var->fill_value,1))) return retval; - var->fill_value = NULL; + if((retval = NCZ_reclaim_fill_value(var))) return retval; } /* Is the user setting the endianness? */ @@ -756,6 +768,46 @@ ncz_def_var_extra(int ncid, int varid, int *shuffle, int *unused1, var->endianness = *endianness; } + /* Remember quantization settings. They will be used when data are + * written. */ + if (quantize_mode) + { + /* Only two valid mode settings. */ + if (*quantize_mode != NC_NOQUANTIZE && + *quantize_mode != NC_QUANTIZE_BITGROOM) + return NC_EINVAL; + + if (*quantize_mode == NC_QUANTIZE_BITGROOM) + { + /* Only float and double types can have quantization. */ + if (var->type_info->hdr.id != NC_FLOAT && + var->type_info->hdr.id != NC_DOUBLE) + return NC_EINVAL; + + /* For bitgroom, number of significant digits is required. */ + if (!nsd) + return NC_EINVAL; + + /* NSD must be in range. */ + if (*nsd <= 0) + return NC_EINVAL; + if (var->type_info->hdr.id == NC_FLOAT && + *nsd > NC_QUANTIZE_MAX_FLOAT_NSD) + return NC_EINVAL; + if (var->type_info->hdr.id == NC_DOUBLE && + *nsd > NC_QUANTIZE_MAX_DOUBLE_NSD) + return NC_EINVAL; + + var->nsd = *nsd; + } + + var->quantize_mode = *quantize_mode; + + /* If quantization is turned off, then set nsd to 0. */ + if (*quantize_mode == NC_NOQUANTIZE) + var->nsd = 0; + } + done: return ZUNTRACE(retval); } @@ -788,7 +840,7 @@ NCZ_def_var_deflate(int ncid, int varid, int shuffle, int deflate, int stat = NC_NOERR; unsigned int level = (unsigned int)deflate_level; /* Set shuffle first */ - if((stat = ncz_def_var_extra(ncid, varid, &shuffle, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL))) goto done; + if((stat = ncz_def_var_extra(ncid, varid, &shuffle, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL))) goto done; if(deflate) stat = nc_def_var_filter(ncid, varid, H5Z_FILTER_DEFLATE,1,&level); if(stat) goto done; @@ -818,7 +870,7 @@ int NCZ_def_var_fletcher32(int ncid, int varid, int fletcher32) { return ncz_def_var_extra(ncid, varid, NULL, NULL, NULL, &fletcher32, - NULL, NULL, NULL, NULL, NULL); + NULL, NULL, NULL, NULL, NULL, NULL, NULL); } /** @@ -847,7 +899,7 @@ int NCZ_def_var_chunking(int ncid, int varid, int contiguous, const size_t *chunksizesp) { return ncz_def_var_extra(ncid, varid, NULL, NULL, NULL, NULL, - &contiguous, chunksizesp, NULL, NULL, NULL); + &contiguous, chunksizesp, NULL, NULL, NULL, NULL, NULL); } /** @@ -892,7 +944,7 @@ ncz_def_var_chunking_ints(int ncid, int varid, int contiguous, int *chunksizesp) cs[i] = chunksizesp[i]; retval = ncz_def_var_extra(ncid, varid, NULL, NULL, NULL, NULL, - &contiguous, cs, NULL, NULL, NULL); + &contiguous, cs, NULL, NULL, NULL, NULL, NULL); if (var->ndims) free(cs); @@ -926,7 +978,7 @@ int NCZ_def_var_fill(int ncid, int varid, int no_fill, const void *fill_value) { return ncz_def_var_extra(ncid, varid, NULL, NULL, NULL, NULL, NULL, - NULL, &no_fill, fill_value, NULL); + NULL, &no_fill, fill_value, NULL, NULL, NULL); } /** @@ -955,9 +1007,143 @@ int NCZ_def_var_endian(int ncid, int varid, int endianness) { return ncz_def_var_extra(ncid, varid, NULL, NULL, NULL, NULL, NULL, - NULL, NULL, NULL, &endianness); + NULL, NULL, NULL, &endianness, NULL, NULL); +} + +/** + * @internal Set quantization settings on a variable. This is + * called by nc_def_var_quantize(). + * + * Quantization allows the user to specify a number of significant + * digits for variables of type ::NC_FLOAT or ::NC_DOUBLE. (Attempting + * to set quantize for other types will result in an ::NC_EINVAL + * error.) + * + * When quantize is turned on, and the number of significant digits + * has been specified, then the netCDF library will apply all zeros or + * all ones (alternating) to bits which are not needed to specify the + * value to the number of significant digits. This will change the + * value of the data, but will make it more compressable. + * + * Quantizing the data does not reduce the size of the data on disk, + * but combining quantize with compression will allow for better + * compression. Since the data values are changed, the use of quantize + * and compression such as deflate constitute lossy compression. + * + * Producers of large datasets may find that using quantize with + * compression will result in significant improvent in the final data + * size. + * + * Variables which use quantize will have added an attribute with either the name + * ::NC_QUANTIZE_BITGROOM_ATT_NAME or ::NC_QUANTIZE_GRANULARBG, but in either case + * will contain the number of significant digits. + * Users should not delete or change this + * attribute. This is the only record that quantize has been applied + * to the data. + * + * As with the deflate settings, quantize settings may only be + * modified before the first call to nc_enddef(). Once nc_enddef() is + * called for the file, quantize settings for any variable in the file + * may not be changed. + * + * Use of quantization is fully backwards compatible with existing + * versions and packages that can read compressed netCDF data. A + * variable which has been quantized is readable to older versions of + * the netCDF libraries, and to netCDF-Java. + * + * @param ncid File ID. + * @param varid Variable ID. NC_GLOBAL may not be used. + * @param quantize_mode Quantization mode. May be ::NC_NOQUANTIZE or + * ::NC_QUANTIZE_BITGROOM or ::NC_QUANTIZE_GRANULARBG. + * @param nsd Number of significant digits. May be any integer from 1 + * to ::NC_QUANTIZE_MAX_FLOAT_NSD (for variables of type ::NC_FLOAT) or + * ::NC_QUANTIZE_MAX_DOUBLE_NSD (for variables of type ::NC_DOUBLE). + * + * @returns ::NC_NOERR No error. + * @returns ::NC_EBADID Bad ncid. + * @returns ::NC_ENOTVAR Invalid variable ID. + * @returns ::NC_ENOTNC4 Attempting netcdf-4 operation on file that is + * not netCDF-4/HDF5. + * @returns ::NC_ELATEDEF Too late to change settings for this variable. + * @returns ::NC_ENOTINDEFINE Not in define mode. + * @returns ::NC_EINVAL Invalid input + * @author Ed Hartnett, Dennis Heimbigner + */ +int +NCZ_def_var_quantize(int ncid, int varid, int quantize_mode, int nsd) +{ + return ncz_def_var_extra(ncid, varid, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, + &quantize_mode, &nsd); } +/** +Ensure that the quantize information for a variable is defined. +Keep a flag in the NCZ_VAR_INFO_T struct to indicate if quantize +info is defined, and if not, read the attribute. +*/ +int +NCZ_ensure_quantizer(int ncid, NC_VAR_INFO_T* var) +{ + int nsd = 0; + + /* Read the attribute */ + if(NCZ_get_att(ncid,var->hdr.id,NC_QUANTIZE_BITGROOM_ATT_NAME,&nsd,NC_INT)==NC_NOERR) { + var->quantize_mode = NC_QUANTIZE_BITGROOM; + var->nsd = nsd; + } else if(NCZ_get_att(ncid,var->hdr.id,NC_QUANTIZE_GRANULARBG_ATT_NAME,&nsd,NC_INT)==NC_NOERR) { + var->quantize_mode = NC_QUANTIZE_GRANULARBG; + var->nsd = nsd; + } else { + var->quantize_mode = NC_NOQUANTIZE; + var->nsd = 0; + } + if(var->quantize_mode < 0) var->quantize_mode = 0; + return NC_NOERR; +} + +/** + * @internal Get quantize information about a variable. Pass NULL for + * whatever you don't care about. Note that this can require reading + * all the attributes for the variable. + * + * @param ncid File ID. + * @param varid Variable ID. + * @param quantize_modep Gets quantize mode. + * @param nsdp Gets Number of Significant Digits if quantize is in use. + * + * @returns ::NC_NOERR No error. + * @returns ::NC_EBADID Bad ncid. + * @returns ::NC_ENOTVAR Bad varid. + * @returns ::NC_EINVAL Invalid input. + * @author Ed Hartnett + */ +int +NCZ_inq_var_quantize(int ncid, int varid, int *quantize_modep, + int *nsdp) +{ + NC_VAR_INFO_T *var; + int retval; + + LOG((2, "%s: ncid 0x%x varid %d", __func__, ncid, varid)); + + /* Find info for this file and group, and set pointer to each. */ + /* Get pointer to the var. */ + if ((retval = nc4_find_grp_h5_var(ncid, varid, NULL, NULL, &var))) + return retval; + if (!var) + return NC_ENOTVAR; + assert(var->hdr.id == varid); + if(var->quantize_mode == -1) + {if((retval = NCZ_ensure_quantizer(ncid, var))) return retval;} + /* Copy the data to the user's data buffers. */ + if (quantize_modep) + *quantize_modep = var->quantize_mode; + if (nsdp) + *nsdp = var->nsd; + return NC_NOERR; + } + /** * @internal Rename a var to "bubba," for example. This is called by * nc_rename_var() for netCDF-4 files. This results in complexities @@ -1319,6 +1505,7 @@ NCZ_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp, size64_t stride[NC_MAX_VAR_DIMS]; int retval, range_error = 0, i, d2; void *bufr = NULL; + int bufrd = 0; /* 1 => we allocated bufr */ int need_to_convert = 0; int zero_count = 0; /* true if a count is zero */ size_t len = 1; @@ -1432,18 +1619,23 @@ NCZ_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp, } #endif - /* Are we going to convert any data? (No converting of user-defined types - except enum). */ - if (mem_nc_type != var->type_info->hdr.id && + /* Are we going to convert any data? (No converting of compound or + * opaque or vlen types.) We also need to call this code if we are doing + * quantization. */ + if ((mem_nc_type != var->type_info->hdr.id && mem_nc_type != NC_COMPOUND && mem_nc_type != NC_OPAQUE && mem_nc_type != NC_VLEN) + || var->quantize_mode > 0) { size_t file_type_size; /* We must convert - allocate a buffer. */ need_to_convert++; - for (d2=0; d2<(var->ndims+zvar->scalar); d2++) + if(zvar->scalar) + len = 1; + else for (d2=0; d2ndims; d2++) len *= countp[d2]; + LOG((4, "converting data for var %s type=%d len=%d", var->hdr.name, var->type_info->hdr.id, len)); @@ -1455,9 +1647,12 @@ NCZ_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp, /* If we're reading, we need bufr to have enough memory to store * the data in the file. If we're writing, we need bufr to be * big enough to hold all the data in the file's type. */ - if (len > 0) + if (len > 0) { + assert(bufr == NULL); if (!(bufr = malloc(len * file_type_size))) BAIL(NC_ENOMEM); + bufrd = 1; + } } else bufr = (void *)data; @@ -1531,6 +1726,8 @@ NCZ_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp, /* Do we need to convert the data? */ if (need_to_convert) { + if(var->quantize_mode < 0) {if((retval = NCZ_ensure_quantizer(ncid,var))) BAIL(retval);} + assert(bufr != NULL); if ((retval = nc4_convert_type(data, bufr, mem_nc_type, var->type_info->hdr.id, len, &range_error, var->fill_value, (h5->cmode & NC_CLASSIC_MODEL), @@ -1573,7 +1770,7 @@ NCZ_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp, if (xfer_plistid && (H5Pclose(xfer_plistid) < 0)) BAIL2(NC_EPARINIT); #endif - if (need_to_convert && bufr) free(bufr); + if (bufrd && bufr) free(bufr); /* If there was an error return it, otherwise return any potential range error value. If none, return NC_NOERR as usual.*/ @@ -1627,7 +1824,6 @@ NCZ_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp, size64_t fmaxdims[NC_MAX_VAR_DIMS]; size64_t start[NC_MAX_VAR_DIMS]; size64_t stride[NC_MAX_VAR_DIMS]; - void *fillvalue = NULL; int no_read = 0, provide_fill = 0; int fill_value_size[NC_MAX_VAR_DIMS]; int retval, range_error = 0, i, d2; @@ -1867,8 +2063,8 @@ NCZ_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp, /* Get the fill value from the ZARR variable. Memory will be * allocated. */ - if (ncz_get_fill_value(h5, var, &fillvalue) < 0) - BAIL(NC_EHDFERR); + if (NCZ_ensure_fill_value(var)) + BAIL(NC_EINVAL); /* How many fill values do we need? */ for (fill_len = 1, d2 = 0; d2 < var->ndims; d2++) @@ -1878,9 +2074,9 @@ NCZ_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp, filldata = (char *)data + real_data_size; for (i = 0; i < fill_len; i++) { - /* Copy one instance of the fill_value */ - if((retval = nc_copy_data(ncid,var->type_info->hdr.id,fillvalue,1,filldata))) - BAIL(retval); + /* Copy one instance of the fill_value */ + if((retval = nc_copy_data(ncid,var->type_info->hdr.id,var->fill_value,1,filldata))) + BAIL(retval); filldata = (char *)filldata + file_type_size; } } @@ -1888,6 +2084,7 @@ NCZ_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp, /* Convert data type if needed. */ if (need_to_convert) { + if(var->quantize_mode < 0) {if((retval = NCZ_ensure_quantizer(ncid,var))) BAIL(retval);} if ((retval = nc4_convert_type(bufr, data, var->type_info->hdr.id, mem_nc_type, len, &range_error, var->fill_value, (h5->cmode & NC_CLASSIC_MODEL), var->quantize_mode, @@ -1917,15 +2114,6 @@ NCZ_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp, #endif if (need_to_convert && bufr) free(bufr); - if (fillvalue) - { - if (var->type_info->nc_type_class == NC_VLEN) - nc_free_vlen((nc_vlen_t *)fillvalue); - else if (var->type_info->nc_type_class == NC_STRING && *(char **)fillvalue) - free(*(char **)fillvalue); - free(fillvalue); - } - /* If there was an error return it, otherwise return any potential range error value. If none, return NC_NOERR as usual.*/ if (retval) diff --git a/libnczarr/zwalk.c b/libnczarr/zwalk.c index 635925ab36..87f490c592 100644 --- a/libnczarr/zwalk.c +++ b/libnczarr/zwalk.c @@ -19,7 +19,7 @@ static unsigned int optimize = 0; extern int NCZ_buildchunkkey(size_t R, const size64_t* chunkindices, char** keyp); /* 0 => no debug */ -static unsigned int wdebug = 0; +static unsigned int wdebug = 1; /* Forward */ static int NCZ_walk(NCZProjection** projv, NCZOdometer* chunkodom, NCZOdometer* slpodom, NCZOdometer* memodom, const struct Common* common, void* chunkdata); @@ -123,7 +123,7 @@ NCZ_transferslice(NC_VAR_INFO_T* var, int reading, common.cache = zvar->cache; /* We need to talk scalar into account */ - common.rank = var->ndims + zvar->scalar; + common.rank = var->ndims; common.scalar = zvar->scalar; common.swap = (zfile->native_endianness == var->endianness ? 0 : 1); @@ -157,9 +157,6 @@ NCZ_transferslice(NC_VAR_INFO_T* var, int reading, common.reader.source = ((NCZ_VAR_INFO_T*)(var->format_var_info))->cache; common.reader.read = readfromcache; - /* verify */ - assert(var->no_fill || var->fill_value != NULL); - if(common.scalar) { if((stat = NCZ_transferscalar(&common))) goto done; } diff --git a/libnczarr/zxcache.c b/libnczarr/zxcache.c index f5cbd6c24f..52a8c7d351 100644 --- a/libnczarr/zxcache.c +++ b/libnczarr/zxcache.c @@ -105,13 +105,20 @@ NCZ_adjust_var_cache(NC_VAR_INFO_T *var) { int stat = NC_NOERR; NCZ_VAR_INFO_T* zvar = (NCZ_VAR_INFO_T*)var->format_var_info; - /* completely empty the cache */ - flushcache(zvar->cache); + NCZChunkCache* zcache = NULL; + + zcache = zvar->cache; + if(zcache->valid) goto done; #ifdef DEBUG fprintf(stderr,"xxx: adjusting cache for: %s\n",var->hdr.name); #endif + /* completely empty the cache */ + flushcache(zcache); + + /* Reclaim any existing fill_chunk */ + if((stat = NCZ_reclaim_fill_chunk(zcache))) goto done; /* Reset the parameters */ zvar->cache->maxsize = var->chunk_cache_size; zvar->cache->maxentries = var->chunk_cache_nelems; @@ -119,22 +126,21 @@ fprintf(stderr,"xxx: adjusting cache for: %s\n",var->hdr.name); fprintf(stderr,"%s.cache.adjust: size=%ld nelems=%ld\n", var->hdr.name,(unsigned long)zvar->cache->maxsize,(unsigned long)zvar->cache->maxentries); #endif - /* One more thing, adjust the chunksize */ - zvar->cache->chunksize = zvar->chunksize; - /* and also rebuild the fillchunk */ - nullfree(zvar->cache->fillchunk); - zvar->cache->fillchunk = NULL; - if(var->no_fill) - stat = NCZ_create_fill_chunk(zvar->cache->chunksize,var->type_info->size,NULL,&zvar->cache->fillchunk); - else { - assert(var->fill_value != NULL); - stat = NCZ_create_fill_chunk(zvar->cache->chunksize,var->type_info->size,var->fill_value,&zvar->cache->fillchunk); + /* One more thing, adjust the chunksize and count*/ + zcache->chunksize = zvar->chunksize; + zcache->chunkcount = 1; + if(var->ndims > 0) { + int i; + for(i=0;indims;i++) { + zcache->chunkcount *= var->chunksizes[i]; + } } + zcache->valid = 1; +done: return stat; } /**************************************************/ - /** * Create a chunk cache object * @@ -162,12 +168,19 @@ NCZ_create_chunk_cache(NC_VAR_INFO_T* var, size64_t chunksize, char dimsep, NCZC {stat = NC_ENOMEM; goto done;} cache->var = var; cache->ndims = var->ndims + zvar->scalar; - assert(cache->fillchunk == NULL); cache->fillchunk = NULL; cache->chunksize = chunksize; cache->dimension_separator = dimsep; zvar->cache = cache; + cache->chunkcount = 1; + if(var->ndims > 0) { + int i; + for(i=0;indims;i++) { + cache->chunkcount *= var->chunksizes[i]; + } + } + #ifdef FLUSH cache->maxentries = 1; #endif @@ -180,6 +193,7 @@ NCZ_create_chunk_cache(NC_VAR_INFO_T* var, size64_t chunksize, char dimsep, NCZC if((cache->mru = nclistnew()) == NULL) {stat = NC_ENOMEM; goto done;} nclistsetalloc(cache->mru,cache->maxentries); + if(cachep) {*cachep = cache; cache = NULL;} done: nullfree(fill); @@ -219,7 +233,7 @@ fprintf(stderr,"|cache.free|=%ld\n",nclistlength(cache->mru)); ncxcachefree(cache->xcache); nclistfree(cache->mru); cache->mru = NULL; - nullfree(cache->fillchunk); + (void)NCZ_reclaim_fill_chunk(cache); nullfree(cache); (void)ZUNTRACE(NC_NOERR); } @@ -271,13 +285,14 @@ NCZ_read_cache_chunk(NCZChunkCache* cache, const size64_t* indices, void** datap /* Create the key for this cache */ if((stat = NCZ_buildchunkpath(cache,indices,&entry->key))) goto done; entry->hashkey = hkey; - /* Try to read the object from "disk" */ + assert(entry->data == NULL && entry->size == 0); + /* Try to read the object from "disk"; might change size; will create if non-existent */ if((stat=get_chunk(cache,entry))) goto done; + assert(entry->data != NULL); + /* Ensure cache constraints not violated; but do it before entry is added */ + if((stat=makeroom(cache))) goto done; nclistpush(cache->mru,entry); - cache->used += entry->size; if((stat = ncxcacheinsert(cache->xcache,entry->hashkey,entry))) goto done; - /* Ensure cache constraints not violated */ - if((stat=makeroom(cache))) goto done; } #ifdef DEBUG @@ -360,6 +375,7 @@ flushcache(NCZChunkCache* cache) /* Remove entries to ensure cache is not violating any of its constraints. On entry, constraints might be violated. + Make sure that the entryinuse (NULL => no constraint) is not reclaimed. */ static int @@ -367,25 +383,28 @@ constraincache(NCZChunkCache* cache) { int stat = NC_NOERR; + /* If the cache is empty then do nothing */ + if(cache->used == 0) goto done; + /* Flush from LRU end if we are at capacity */ while(nclistlength(cache->mru) > cache->maxentries || cache->used > cache->maxsize) { int i; void* ptr; NCZCacheEntry* e = ncxcachelast(cache->xcache); /* last entry is the least recently used */ if((stat = ncxcacheremove(cache->xcache,e->hashkey,&ptr))) goto done; - assert(e == ptr); + assert(e == ptr); for(i=0;imru);i++) { e = nclistget(cache->mru,i); if(ptr == e) break; } - assert(e != NULL); + assert(e != NULL); assert(i >= 0 && i < nclistlength(cache->mru)); nclistremove(cache->mru,i); + assert(cache->used >= e->size); + /* Note that |old chunk data| may not be same as |new chunk data| because of filters */ + cache->used -= e->size; /* old size */ if(e->modified) /* flush to file */ stat=put_chunk(cache,e); - /* Decrement space used */ - assert(cache->used >= e->size); - cache->used -= e->size; /* reclaim */ nullfree(e->data); nullfree(e->key.varkey); nullfree(e->key.chunkkey); nullfree(e); } @@ -410,9 +429,12 @@ NCZ_flush_chunk_cache(NCZChunkCache* cache) for(i=0;imru);i++) { NCZCacheEntry* entry = nclistget(cache->mru,i); if(entry->modified) { + /* Make cache used be consistent across filter application */ + cache->used -= entry->size; /* Write out this chunk in toto*/ if((stat=put_chunk(cache,entry))) goto done; + cache->used += entry->size; } entry->modified = 0; } @@ -421,6 +443,69 @@ NCZ_flush_chunk_cache(NCZChunkCache* cache) return ZUNTRACE(stat); } +/* Ensure existence of some kind of fill chunk */ +int +NCZ_ensure_fill_chunk(NCZChunkCache* cache) +{ + int i, stat = NC_NOERR; + NC_VAR_INFO_T* var = cache->var; + size_t typesize = var->type_info->size; + + if(cache->fillchunk) goto done; + + if((cache->fillchunk = malloc(cache->chunksize))==NULL) + {stat = NC_ENOMEM; goto done;} + if(var->no_fill) { + /* use zeros */ + memset(cache->fillchunk,0,cache->chunksize); + goto done; + } + if((stat = NCZ_ensure_fill_value(var))) goto done; + switch (typesize) { + case 1: { + unsigned char c = *((unsigned char*)var->fill_value); + memset(cache->fillchunk,c,cache->chunksize); + } break; + case 2: { + unsigned short fv = *((unsigned short*)var->fill_value); + unsigned short* p2 = (unsigned short*)cache->fillchunk; + for(i=0;ichunksize;i+=typesize) *p2++ = fv; + } break; + case 4: { + unsigned int fv = *((unsigned int*)var->fill_value); + unsigned int* p4 = (unsigned int*)cache->fillchunk; + for(i=0;ichunksize;i+=typesize) *p4++ = fv; + } break; + case 8: { + unsigned long long fv = *((unsigned long long*)var->fill_value); + unsigned long long* p8 = (unsigned long long*)cache->fillchunk; + for(i=0;ichunksize;i+=typesize) *p8++ = fv; + } break; + default: { + unsigned char* p; + for(p=cache->fillchunk,i=0;ichunksize;i+=typesize,p+=typesize) + memcpy(p,var->fill_value,typesize); + } break; + } +done: + return NC_NOERR; +} + +int +NCZ_reclaim_fill_chunk(NCZChunkCache* zcache) +{ + int stat = NC_NOERR; + if(zcache && zcache->fillchunk) { + NC_VAR_INFO_T* var = zcache->var; + int ncid = var->container->nc4_info->controller->ext_ncid; + int tid = var->type_info->hdr.id; + size_t chunkcount = zcache->chunkcount; + stat = nc_reclaim_data_all(ncid,tid,zcache->fillchunk,chunkcount); + zcache->fillchunk = NULL; + } + return stat; +} + #if 0 int NCZ_chunk_cache_modified(NCZChunkCache* cache, const size64_t* indices) @@ -529,7 +614,7 @@ put_chunk(NCZChunkCache* cache, NCZCacheEntry* entry) /* Get the filter chain to apply */ NClist* filterchain = (NClist*)var->filters; if(nclistlength(filterchain) > 0) { - /* Apply the filter chain to get the filtered data */ + /* Apply the filter chain to get the filtered data; will reclaim entry->data */ if((stat = NCZ_applyfilterchain(file,var,filterchain,entry->size,entry->data,&flen,&filtered,ENCODING))) goto done; /* Fix up the cache entry */ /* Note that if filtered is different from entry->data, then entry->data will have been freed */ @@ -618,7 +703,8 @@ get_chunk(NCZChunkCache* cache, NCZCacheEntry* entry) if((entry->data = (void*)malloc(entry->size)) == NULL) {stat = NC_ENOMEM; goto done;} /* apply fill value */ - assert(cache->fillchunk); + if(cache->fillchunk == NULL) + {if((stat = NCZ_ensure_fill_chunk(cache))) goto done;} memcpy(entry->data,cache->fillchunk,entry->size); entry->isfiltered = 0; stat = NC_NOERR; @@ -669,3 +755,76 @@ NCZ_buildchunkpath(NCZChunkCache* cache, const size64_t* chunkindices, struct Ch nullfree(varkey); return THROW(stat); } + +void +NCZ_dumpxcacheentry(NCZChunkCache* cache, NCZCacheEntry* e, NCbytes* buf) +{ + char s[8192]; + char idx[64]; + int i; + + ncbytescat(buf,"{"); + snprintf(s,sizeof(s),"modified=%u isfiltered=%u indices=", + (unsigned)e->modified, + (unsigned)e->isfiltered + ); + ncbytescat(buf,s); + for(i=0;indims;i++) { + snprintf(idx,sizeof(idx),"%s%llu",(i==0?"":"."),e->indices[i]); + ncbytescat(buf,idx); + } + snprintf(s,sizeof(s),"size=%llu data=%p", + e->size, + e->data + ); + ncbytescat(buf,s); + ncbytescat(buf,"}"); +} + +void +NCZ_printxcache(NCZChunkCache* cache) +{ + static char xs[20000]; + NCbytes* buf = ncbytesnew(); + char s[8192]; + int i; + + ncbytescat(buf,"NCZChunkCache:\n"); + snprintf(s,sizeof(s),"\tvar=%s\n\tndims=%u\n\tchunksize=%u\n\tchunkcount=%u\n\tfillchunk=%p\n", + cache->var->hdr.name, + (unsigned)cache->ndims, + (unsigned)cache->chunksize, + (unsigned)cache->chunkcount, + cache->fillchunk + ); + ncbytescat(buf,s); + + snprintf(s,sizeof(s),"\tmaxentries=%u\n\tmaxsize=%u\n\tused=%u\n\tdimsep='%c'\n", + (unsigned)cache->maxentries, + (unsigned)cache->maxsize, + (unsigned)cache->used, + cache->dimension_separator + ); + ncbytescat(buf,s); + + snprintf(s,sizeof(s),"\tmru: (%u)\n",(unsigned)nclistlength(cache->mru)); + ncbytescat(buf,s); + if(nclistlength(cache->mru)==0) + ncbytescat(buf,"\t\t\n"); + for(i=0;imru);i++) { + NCZCacheEntry* e = (NCZCacheEntry*)nclistget(cache->mru,i); + snprintf(s,sizeof(s),"\t\t[%d] ",i); + ncbytescat(buf,s); + if(e == NULL) + ncbytescat(buf,""); + else + NCZ_dumpxcacheentry(cache, e, buf); + ncbytescat(buf,"\n"); + } + + xs[0] = '\0'; + strlcat(xs,ncbytescontents(buf),sizeof(xs)); + ncbytesfree(buf); + fprintf(stderr,"%s\n",xs); +// return xs; +} diff --git a/libsrc/ncio.c b/libsrc/ncio.c index 133eddad40..24bb92da10 100644 --- a/libsrc/ncio.c +++ b/libsrc/ncio.c @@ -185,11 +185,10 @@ urlmodetest(const char* path) ncuriparse(path,&uri); if(uri == NULL) return 0; /* Not URL */ - if(NC_testmode(uri, "bytes")) - kind = NC_HTTP; - else if(NC_testmode(uri, "s3")) - kind = NC_S3SDK; - else + if(NC_testmode(uri, "bytes")) { + /* NC_S3SDK takes priority over NC_HTTP */ + if(NC_testmode(uri, "s3")) kind = NC_S3SDK; else kind = NC_HTTP; + } else kind = 0; ncurifree(uri); return kind; diff --git a/libsrc4/nc4internal.c b/libsrc4/nc4internal.c index 935f1500e2..c4377c3aa9 100644 --- a/libsrc4/nc4internal.c +++ b/libsrc4/nc4internal.c @@ -1358,8 +1358,12 @@ var_free(NC_VAR_INFO_T *var) free(var->dim); /* Delete any fill value allocation. */ - if (var->fill_value) - {free(var->fill_value); var->fill_value = NULL;} + if (var->fill_value) { + int ncid = var->container->nc4_info->controller->ext_ncid; + int tid = var->type_info->hdr.id; + if((retval = nc_reclaim_data_all(ncid, tid, var->fill_value, 1))) return retval; + var->fill_value = NULL; + } /* Release type information */ if (var->type_info) diff --git a/nc_test/test_byterange.sh b/nc_test/test_byterange.sh index 83d121536f..28b5e6374d 100755 --- a/nc_test/test_byterange.sh +++ b/nc_test/test_byterange.sh @@ -7,13 +7,13 @@ set -e # Test Urls URL3="https://thredds-test.unidata.ucar.edu/thredds/fileServer/pointData/cf_dsg/example/point.nc#mode=bytes&aws.profile=none" -#URL3="https://remotetest.unidata.ucar.edu/thredds/fileServer/testdata/2004050300_eta_211.nc#bytes&aws.profile=none" +#URL3a="https://remotetest.unidata.ucar.edu/thredds/fileServer/testdata/2004050300_eta_211.nc#bytes&aws.profile=none" URL4a="https://s3.us-east-1.amazonaws.com/noaa-goes16/ABI-L1b-RadC/2017/059/03/OR_ABI-L1b-RadC-M3C13_G16_s20170590337505_e20170590340289_c20170590340316.nc#mode=bytes&aws.profile=none" URL4b="https://thredds-test.unidata.ucar.edu/thredds/fileServer/irma/metar/files/METAR_20170910_0000.nc#bytes&aws.profile=none" URL4c="s3://noaa-goes16/ABI-L1b-RadC/2017/059/03/OR_ABI-L1b-RadC-M3C13_G16_s20170590337505_e20170590340289_c20170590340316.nc#mode=bytes&aws.profile=none" # Requires auth -URL4d="s3://unidata-zarr-test-data/byterangefiles/upload3.nc#bytes&aws.profile=unidata" -URL4e="s3://unidata-zarr-test-data/byterangefiles/upload4.nc#bytes&aws.profile=unidata" +URL3b="s3://unidata-zarr-test-data/byterangefiles/upload3.nc#bytes&aws.profile=unidata" +URL4d="s3://unidata-zarr-test-data/byterangefiles/upload4.nc#bytes&aws.profile=unidata" # Do not use unless we know it has some permanence (note the segment 'testing' in the URL); URL4x="https://s3.us-west-2.amazonaws.com/coawst-public/testing/HadCRUT.4.6.0.0.median.nc#mode=bytes,&aws.profile=none" @@ -94,11 +94,11 @@ if test "x$FEATURE_S3TESTS" = xyes && test "x$FEATURE_HDF5" = xyes ; then echo "***Test remote netcdf-4 file: s3" testbytes nc4c netCDF-4 "$URL4c" echo "***Test remote netcdf-4 file: s3 auth" - tests3auth nc4e netCDF-4 "$URL4e" + tests3auth nc4d netCDF-4 "$URL4d" fi if test "x$FEATURE_S3TESTS" = xyes ; then echo "***Test remote netcdf-3 file: s3 auth" - tests3auth nc4d classic "$URL4d" + tests3auth nc3b classic "$URL3b" fi # Cleanup diff --git a/nc_test4/findplugin.in b/nc_test4/findplugin.in index f1623e73aa..fb30686307 100644 --- a/nc_test4/findplugin.in +++ b/nc_test4/findplugin.in @@ -23,7 +23,6 @@ # variables: see hdf5plugins/CMakeLists.txt findplugin() { -set -x FP_NAME="$1" diff --git a/nc_test4/tst_quantize.c b/nc_test4/tst_quantize.c index df31c17966..e8e8761fc8 100644 --- a/nc_test4/tst_quantize.c +++ b/nc_test4/tst_quantize.c @@ -7,6 +7,7 @@ precision. Ed Hartnett, 8/19/21 + Dennis Heimbigner, 1/16/22 */ #include @@ -72,12 +73,28 @@ pd(double myd) int main(int argc, char **argv) { +#ifdef TESTNCZARR + const char* template = NULL; + char file_url[4096]; + + if(argc == 1) + {fprintf(stderr,"usage: test_quantize \n"); exit(1);} + + template = argv[1]; + + snprintf(file_url,sizeof(file_url),template,FILE_NAME); + +#undef FILE_NAME +#define FILE_NAME file_url +#endif + printf("\n*** Testing netcdf-4 variable quantization functions.\n"); printf("**** testing quantization setting and error conditions..."); { int ncid, dimid, varid1, varid2; int quantize_mode_in, nsd_in; +#ifndef TESTNCZARR /* Create a netcdf classic file with one var. Attempt * quantization. It will not work. */ if (nc_create(FILE_NAME, NC_CLOBBER, &ncid)) ERR; @@ -86,6 +103,7 @@ main(int argc, char **argv) if (nc_def_var_quantize(ncid, varid1, NC_QUANTIZE_BITGROOM, NSD_3) != NC_ENOTNC4) ERR; if (nc_inq_var_quantize(ncid, varid1, &quantize_mode_in, &nsd_in) != NC_ENOTNC4) ERR; if (nc_close(ncid)) ERR; +#endif /* Create a netcdf-4 file with two vars. Attempt * quantization. It will work, eventually... */ @@ -147,10 +165,13 @@ main(int argc, char **argv) /* Open the file and check. */ if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR; - if (nc_inq_var_quantize(ncid, 0, &quantize_mode_in, &nsd_in)) ERR; + /* Don't assume the varid !!! */ + if (nc_inq_varid(ncid, VAR_NAME_1, &varid1)) ERR; + if (nc_inq_varid(ncid, VAR_NAME_2, &varid2)) ERR; + if (nc_inq_var_quantize(ncid, varid1, &quantize_mode_in, &nsd_in)) ERR; if (quantize_mode_in != NC_QUANTIZE_BITGROOM) ERR; if (nsd_in != NSD_3) ERR; - if (nc_inq_var_quantize(ncid, 1, &quantize_mode_in, &nsd_in)) ERR; + if (nc_inq_var_quantize(ncid, varid2, &quantize_mode_in, &nsd_in)) ERR; if (quantize_mode_in != NC_QUANTIZE_BITGROOM) ERR; if (nsd_in != NSD_9) ERR; if (nc_close(ncid)) ERR; @@ -172,7 +193,11 @@ main(int argc, char **argv) int varid; int nsd_in, quantize_mode; int nsd_out = 3; +#ifdef TESTNCZARR + char file_name[4096]; +#else char file_name[NC_MAX_NAME + 1]; +#endif int xtype[NTYPES] = {NC_CHAR, NC_SHORT, NC_INT, NC_BYTE, NC_UBYTE, NC_USHORT, NC_UINT, NC_INT64, NC_UINT64}; int t; @@ -180,7 +205,14 @@ main(int argc, char **argv) for (t = 0; t < NTYPES; t++) { sprintf(file_name, "%s_bitgroom_type_%d.nc", TEST, xtype[t]); - +#ifdef TESTNCZARR + { + char url[4096]; + snprintf(url,sizeof(url),template,file_name); + strcpy(file_name,url); +fprintf(stderr,"\n>>> type url = |%s|\n",file_name); + } +#endif /* Create file. */ if (nc_create(file_name, NC_NETCDF4, &ncid)) ERR; if (nc_def_dim(ncid, X_NAME, NX_BIG, &dimid[0])) ERR; @@ -195,6 +227,7 @@ main(int argc, char **argv) /* Check file. */ { if (nc_open(file_name, NC_NETCDF4, &ncid)) ERR; + if (nc_inq_varid(ncid,VAR_NAME,&varid)) ERR; if (nc_inq_var_quantize(ncid, varid, &quantize_mode, &nsd_in)) ERR; if (quantize_mode) ERR; if (nc_close(ncid)) ERR; @@ -236,9 +269,11 @@ main(int argc, char **argv) /* Open the file and check metadata. */ if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR; - if (nc_inq_var_quantize(ncid, 0, &quantize_mode_in, &nsd_in)) ERR; + if (nc_inq_varid(ncid, VAR_NAME_1, &varid1)) ERR; + if (nc_inq_varid(ncid, VAR_NAME_2, &varid2)) ERR; + if (nc_inq_var_quantize(ncid, varid1, &quantize_mode_in, &nsd_in)) ERR; if (quantize_mode_in != NC_QUANTIZE_BITGROOM || nsd_in != NSD_3) ERR; - if (nc_inq_var_quantize(ncid, 1, &quantize_mode_in, &nsd_in)) ERR; + if (nc_inq_var_quantize(ncid, varid2, &quantize_mode_in, &nsd_in)) ERR; if (quantize_mode_in != NC_QUANTIZE_BITGROOM || nsd_in != NSD_3) ERR; /* Each var now has an attribute describing the quantize settings. */ @@ -300,9 +335,11 @@ main(int argc, char **argv) /* Open the file and check metadata. */ if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR; - if (nc_inq_var_quantize(ncid, 0, &quantize_mode_in, &nsd_in)) ERR; + if (nc_inq_varid(ncid,VAR_NAME_1,&varid1)) ERR; + if (nc_inq_varid(ncid,VAR_NAME_2,&varid2)) ERR; + if (nc_inq_var_quantize(ncid, varid1, &quantize_mode_in, &nsd_in)) ERR; if (quantize_mode_in != NC_QUANTIZE_BITGROOM || nsd_in != NSD_3) ERR; - if (nc_inq_var_quantize(ncid, 1, &quantize_mode_in, &nsd_in)) ERR; + if (nc_inq_var_quantize(ncid, varid2, &quantize_mode_in, &nsd_in)) ERR; if (quantize_mode_in != NC_QUANTIZE_BITGROOM || nsd_in != NSD_3) ERR; /* Check the data. */ @@ -376,9 +413,11 @@ main(int argc, char **argv) /* Open the file and check metadata. */ if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR; - if (nc_inq_var_quantize(ncid, 0, &quantize_mode_in, &nsd_in)) ERR; + if (nc_inq_varid(ncid,VAR_NAME_1,&varid1)) ERR; + if (nc_inq_varid(ncid,VAR_NAME_2,&varid2)) ERR; + if (nc_inq_var_quantize(ncid, varid1, &quantize_mode_in, &nsd_in)) ERR; if (quantize_mode_in != NC_QUANTIZE_BITGROOM || nsd_in != NSD_3) ERR; - if (nc_inq_var_quantize(ncid, 1, &quantize_mode_in, &nsd_in)) ERR; + if (nc_inq_var_quantize(ncid, varid2, &quantize_mode_in, &nsd_in)) ERR; if (quantize_mode_in != NC_QUANTIZE_BITGROOM || nsd_in != NSD_3) ERR; /* Check the data. */ @@ -441,9 +480,11 @@ main(int argc, char **argv) /* Open the file and check metadata. */ if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR; - if (nc_inq_var_quantize(ncid, 0, &quantize_mode_in, &nsd_in)) ERR; + if (nc_inq_varid(ncid,VAR_NAME_1,&varid1)) ERR; + if (nc_inq_varid(ncid,VAR_NAME_2,&varid2)) ERR; + if (nc_inq_var_quantize(ncid, varid1, &quantize_mode_in, &nsd_in)) ERR; if (quantize_mode_in != NC_QUANTIZE_BITGROOM || nsd_in != NSD_3) ERR; - if (nc_inq_var_quantize(ncid, 1, &quantize_mode_in, &nsd_in)) ERR; + if (nc_inq_var_quantize(ncid, varid2, &quantize_mode_in, &nsd_in)) ERR; if (quantize_mode_in != NC_QUANTIZE_BITGROOM || nsd_in != NSD_3) ERR; /* Each var now has an attribute describing the quantize settings. */ @@ -523,9 +564,11 @@ main(int argc, char **argv) /* Open the file and check metadata. */ if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR; - if (nc_inq_var_quantize(ncid, 0, &quantize_mode_in, &nsd_in)) ERR; + if (nc_inq_varid(ncid,VAR_NAME_1,&varid1)) ERR; + if (nc_inq_varid(ncid,VAR_NAME_2,&varid2)) ERR; + if (nc_inq_var_quantize(ncid, varid1, &quantize_mode_in, &nsd_in)) ERR; if (quantize_mode_in != NC_QUANTIZE_BITGROOM || nsd_in != NSD_3) ERR; - if (nc_inq_var_quantize(ncid, 1, &quantize_mode_in, &nsd_in)) ERR; + if (nc_inq_var_quantize(ncid, varid2, &quantize_mode_in, &nsd_in)) ERR; if (quantize_mode_in != NC_QUANTIZE_BITGROOM || nsd_in != NSD_3) ERR; /* Check the data. */ @@ -603,9 +646,11 @@ main(int argc, char **argv) /* Open the file and check metadata. */ if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR; - if (nc_inq_var_quantize(ncid, 0, &quantize_mode_in, &nsd_in)) ERR; + if (nc_inq_varid(ncid,VAR_NAME_1,&varid1)) ERR; + if (nc_inq_varid(ncid,VAR_NAME_2,&varid2)) ERR; + if (nc_inq_var_quantize(ncid, varid1, &quantize_mode_in, &nsd_in)) ERR; if (quantize_mode_in != NC_QUANTIZE_BITGROOM || nsd_in != NSD_3) ERR; - if (nc_inq_var_quantize(ncid, 1, &quantize_mode_in, &nsd_in)) ERR; + if (nc_inq_var_quantize(ncid, varid2, &quantize_mode_in, &nsd_in)) ERR; if (quantize_mode_in != NC_QUANTIZE_BITGROOM || nsd_in != NSD_3) ERR; /* Check the data. */ @@ -689,9 +734,11 @@ main(int argc, char **argv) /* Open the file and check metadata. */ if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR; - if (nc_inq_var_quantize(ncid, 0, &quantize_mode_in, &nsd_in)) ERR; + if (nc_inq_varid(ncid,VAR_NAME_1,&varid1)) ERR; + if (nc_inq_varid(ncid,VAR_NAME_2,&varid2)) ERR; + if (nc_inq_var_quantize(ncid, varid1, &quantize_mode_in, &nsd_in)) ERR; if (quantize_mode_in != NC_QUANTIZE_BITGROOM || nsd_in != NSD_3) ERR; - if (nc_inq_var_quantize(ncid, 1, &quantize_mode_in, &nsd_in)) ERR; + if (nc_inq_var_quantize(ncid, varid2, &quantize_mode_in, &nsd_in)) ERR; if (quantize_mode_in != NC_QUANTIZE_BITGROOM || nsd_in != NSD_3) ERR; /* Check the data. */ @@ -782,6 +829,8 @@ main(int argc, char **argv) /* Now reopen the file and check. */ if (nc_open(FILE_NAME, NC_NETCDF4, &ncid)) ERR; + if (nc_inq_varid(ncid,VAR_NAME,&varid1)) ERR; + if (nc_inq_varid(ncid,VAR_NAME_2,&varid2)) ERR; /* Read the data. */ if (nc_get_var_float(ncid, varid1, float_data_in)) ERR; @@ -875,13 +924,16 @@ main(int argc, char **argv) if (nc_def_var_quantize(ncid, varid1, NC_QUANTIZE_BITGROOM, NSD_3)) ERR; if (nc_def_var_quantize(ncid, varid2, NC_QUANTIZE_BITGROOM, NSD_3)) ERR; +#ifdef TESTNCZARR +#ifdef ENABLE_NCZARR_FILTERS /* Set up zlib compression. This will work better because the * data are quantized, yielding a smaller output file. We will * set compression level to 1, which is usually the best * choice. */ if (nc_def_var_deflate(ncid, varid1, 0, 1, 1)) ERR; if (nc_def_var_deflate(ncid, varid2, 0, 1, 1)) ERR; - +#endif +#endif /* Write the data. */ if (nc_put_var_float(ncid, varid1, float_data)) ERR; if (nc_put_var_double(ncid, varid2, double_data)) ERR; @@ -896,6 +948,8 @@ main(int argc, char **argv) /* Now reopen the file and check. */ if (nc_open(FILE_NAME, NC_NETCDF4, &ncid)) ERR; + if (nc_inq_varid(ncid,"var1",&varid1)) ERR; + if (nc_inq_varid(ncid,"var2",&varid2)) ERR; /* Read the data. */ if (nc_get_var_float(ncid, varid1, float_data_in)) ERR; diff --git a/ncdump/tst_dimsizes.sh b/ncdump/tst_dimsizes.sh index e08faba862..175e60ff48 100755 --- a/ncdump/tst_dimsizes.sh +++ b/ncdump/tst_dimsizes.sh @@ -1,6 +1,6 @@ #!/bin/sh -#set -e +set -e if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh diff --git a/ncgen/genbin.c b/ncgen/genbin.c index 9c60e1dcee..5dd5d18717 100644 --- a/ncgen/genbin.c +++ b/ncgen/genbin.c @@ -257,11 +257,15 @@ genbin_definespecialattributes(Symbol* var) } CHECK_ERR(stat); } + if(special->flags & _QUANTIZE_FLAG) { + stat = nc_def_var_quantize(var->container->nc_id, + var->nc_id, special->_Quantizer, special->_NSD); + CHECK_ERR(stat); + } return stat; } #endif /*USE_NETCDF4*/ - void genbin_close(void) { diff --git a/ncgen/genc.c b/ncgen/genc.c index b3b0c414b8..9e5dcad464 100644 --- a/ncgen/genc.c +++ b/ncgen/genc.c @@ -539,6 +539,22 @@ genc_definespecialattributes(Symbol* vsym) codelined(1,"CHECK_ERR(stat);"); } } + if(special->flags & _QUANTIZE_FLAG) { + const char* alg = NULL; + switch(special->_Quantizer) { + case NC_QUANTIZE_BITGROOM: alg = "NC_QUANTIZE_BITGROOM"; + case NC_QUANTIZE_GRANULARBG: alg = "NC_QUANTIZE_GRANULARBG"; + default: alg = "NC_NOQUANTIZE"; + } + bbprintf0(stmt, + " stat = nc_def_var_quantize(%s, %s, %s, %d);\n", + groupncid(vsym->container), + varncid(vsym), + alg, special->_NSD + ); + codedump(stmt); + codelined(1,"CHECK_ERR(stat);"); + } } #endif /*USE_NETCDF4*/ diff --git a/ncgen/ncgen.h b/ncgen/ncgen.h index 523b2d1e19..c50a516b82 100644 --- a/ncgen/ncgen.h +++ b/ncgen/ncgen.h @@ -76,6 +76,7 @@ various C global variables #define _FORMAT_FLAG 0x800 #define _FILTER_FLAG 0x1000 #define _CODECS_FLAG 0x2000 +#define _QUANTIZE_FLAG 0x4000 extern struct Specialtoken { char* name; @@ -118,6 +119,8 @@ typedef struct Specialdata { int _Shuffle; /* 0 => false, 1 => true*/ int _Endianness; /* 1 =>little, 2 => big*/ int _Fill ; /* 0 => false, 1 => true WATCHOUT: this is inverse of NOFILL*/ + int _Quantizer; /* algorithm */ + int _NSD; /* No. of significant digits */ NC_H5_Filterspec** _Filters; size_t nfilters; /* |filters| */ char* _Codecs; /* in JSON form */ diff --git a/ncgen/ncgen.l b/ncgen/ncgen.l index d382d8154d..a030d17891 100644 --- a/ncgen/ncgen.l +++ b/ncgen/ncgen.l @@ -138,6 +138,7 @@ struct Specialtoken specials[] = { {"_SuperblockVersion",_SUPERBLOCK,_SUPERBLOCK_FLAG}, {"_Filter",_FILTER,_FILTER_FLAG}, {"_Codecs",_CODECS,_CODECS_FLAG}, +{"_QuantizeBitgroomNumberOfSignificantDigits",_QUANTIZE,_QUANTIZE_FLAG}, {NULL,0} /* null terminate */ }; @@ -216,7 +217,7 @@ NUMBER [+-]?[0-9][0-9]*[Uu]?([BbSs]|[Ll]|[Ll][Ll])? DBLNUMBER [+-]?[0-9]*\.[0-9]*{exp}?[LlDd]?|[+-]?[0-9]*{exp}[LlDd]? FLTNUMBER [+-]?[0-9]*\.[0-9]*{exp}?[Ff]|[+-]?[0-9]*{exp}[Ff] -SPECIAL "_FillValue"|"_Format"|"_Storage"|"_ChunkSizes"|"_Fletcher32"|"_DeflateLevel"|"_Shuffle"|"_Endianness"|"_NoFill"|"_NCProperties"|"_IsNetcdf4"|"_SuperblockVersion"|"_Filter"|"_Codecs" +SPECIAL "_FillValue"|"_Format"|"_Storage"|"_ChunkSizes"|"_Fletcher32"|"_DeflateLevel"|"_Shuffle"|"_Endianness"|"_NoFill"|"_NCProperties"|"_IsNetcdf4"|"_SuperblockVersion"|"_Filter"|"_Codecs"|"_QuantizeBitgroomNumberOfSignificantDIgits"|"_QuantizeGranularBitGroomNumberOfSignificantDigits" USASCII [\x01-\x7F] diff --git a/ncgen/ncgen.y b/ncgen/ncgen.y index b0951fbe45..70bc6437bf 100644 --- a/ncgen/ncgen.y +++ b/ncgen/ncgen.y @@ -216,6 +216,7 @@ NCConstant* constant; _SUPERBLOCK _FILTER _CODECS + _QUANTIZE DATASETID %type ident typename primtype dimd varspec @@ -772,6 +773,8 @@ attrdecl: {$$ = makespecial(_FILTER_FLAG,$1,NULL,(void*)$5,ISCONST);} | ambiguous_ref ':' _CODECS '=' conststring {$$ = makespecial(_CODECS_FLAG,$1,NULL,(void*)$5,ISCONST);} + | ambiguous_ref ':' _QUANTIZE '=' constint + {$$ = makespecial(_QUANTIZE_FLAG,$1,NULL,(void*)$5,ISCONST);} | ambiguous_ref ':' _NOFILL '=' constbool {$$ = makespecial(_NOFILL_FLAG,$1,NULL,(void*)$5,ISCONST);} | ':' _FORMAT '=' conststring @@ -1240,6 +1243,7 @@ makespecial(int tag, Symbol* vsym, Symbol* tsym, void* data, int isconst) break; case _SUPERBLOCK_FLAG: case _DEFLATE_FLAG: + case _QUANTIZE_FLAG: tmp = nullconst(); tmp->nctype = NC_INT; convert1(con,tmp); @@ -1336,6 +1340,11 @@ makespecial(int tag, Symbol* vsym, Symbol* tsym, void* data, int isconst) special->_DeflateLevel = idata; special->flags |= _DEFLATE_FLAG; break; + case _QUANTIZE_FLAG: + special->_Quantizer = NC_QUANTIZE_BITGROOM; + special->_NSD = idata; + special->flags |= _QUANTIZE_FLAG; + break; case _SHUFFLE_FLAG: special->_Shuffle = tf; special->flags |= _SHUFFLE_FLAG; diff --git a/nczarr_test/CMakeLists.txt b/nczarr_test/CMakeLists.txt index 3d0a8f93bb..b1f23d5aca 100644 --- a/nczarr_test/CMakeLists.txt +++ b/nczarr_test/CMakeLists.txt @@ -9,6 +9,10 @@ SET(abs_top_srcdir ${CMAKE_CURRENT_SOURCE_DIR}) remove_definitions(-DDLL_EXPORT) +FILE(READ ${CMAKE_CURRENT_SOURCE_DIR}/../nc_test4/tst_quantize.c QSOURCE) +FILE(WRITE ${CMAKE_CURRENT_BINARY_DIR}/test_quantize.c "#define TESTNCZARR\n") +FILE(APPEND ${CMAKE_CURRENT_BINARY_DIR}/test_quantize.c "${QSOURCE}") + FILE(GLOB COPY_FILES ${CMAKE_CURRENT_SOURCE_DIR}/*.sh ${CMAKE_CURRENT_SOURCE_DIR}/ref*.cdl ${CMAKE_CURRENT_SOURCE_DIR}/ref*.txt) @@ -92,6 +96,9 @@ IF(ENABLE_TESTS) add_sh_test(nczarr_test run_misc) add_sh_test(nczarr_test run_nczarr_fill) + BUILD_BIN_TEST(test_quantize ${TSTCOMMONSRC}) + add_sh_test(nczarr_test run_quantize) + if(ENABLE_NCZARR_S3) add_sh_test(nczarr_test run_s3_cleanup) ENDIF() diff --git a/nczarr_test/Makefile.am b/nczarr_test/Makefile.am index d2278ae17c..b466a68510 100644 --- a/nczarr_test/Makefile.am +++ b/nczarr_test/Makefile.am @@ -35,7 +35,7 @@ ut_projections_SOURCES = ut_projections.c ${commonsrc} ut_chunking_SOURCES = ut_chunking.c ${commonsrc} tst_fillonlyz_SOURCES = tst_fillonlyz.c ${tstcommonsrc} -check_PROGRAMS += tst_zchunks tst_zchunks2 tst_zchunks3 tst_fillonlyz +check_PROGRAMS += tst_zchunks tst_zchunks2 tst_zchunks3 tst_fillonlyz test_quantize TESTS += run_ut_map.sh TESTS += run_ut_mapapi.sh @@ -55,6 +55,8 @@ check_PROGRAMS += tst_chunkcases tst_chunkcases_SOURCES = tst_chunkcases.c ${tstcommonsrc} TESTS += run_chunkcases.sh +TESTS += run_quantize.sh + TESTS += run_purezarr.sh TESTS += run_interop.sh TESTS += run_misc.sh @@ -126,7 +128,7 @@ run_ut_map.sh run_ut_mapapi.sh run_ut_misc.sh run_ut_chunk.sh run_ncgen4.sh \ run_nccopyz.sh run_fillonlyz.sh run_chunkcases.sh test_nczarr.sh run_perf_chunks1.sh run_s3_cleanup.sh \ run_purezarr.sh run_interop.sh run_misc.sh \ run_filter.sh run_specific_filters.sh \ -run_newformat.sh run_nczarr_fill.sh +run_newformat.sh run_nczarr_fill.sh run_quantize.sh EXTRA_DIST += \ ref_ut_map_create.cdl ref_ut_map_writedata.cdl ref_ut_map_writemeta2.cdl ref_ut_map_writemeta.cdl \ @@ -153,12 +155,19 @@ EXTRA_DIST += ref_power_901_constants.zip ref_power_901_constants.cdl ref_quotes CLEANFILES = ut_*.txt ut*.cdl tmp*.nc tmp*.cdl tmp*.txt tmp*.dmp tmp*.zip tmp*.nc tmp*.dump tmp*.tmp tmp_ngc.c ref_zarr_test_data.cdl +BUILT_SOURCES = test_quantize.c +test_quantize.c: ../nc_test4/tst_quantize.c + rm -f test_quantize.c + echo "#define TESTNCZARR" > test_quantize.c + cat ../nc_test4/tst_quantize.c >> test_quantize.c + # Remove directories clean-local: rm -fr tmp*.file results.file results.s3 results.zip rm -fr rcmiscdir -DISTCLEANFILES = findplugin.sh +DISTCLEANFILES = findplugin.sh test_quantize.c # If valgrind is present, add valgrind targets. @VALGRIND_CHECK_RULES@ + diff --git a/nczarr_test/ref_filtered.cdl b/nczarr_test/ref_filtered.cdl index 6afa77b2e2..cf9e6febb4 100644 --- a/nczarr_test/ref_filtered.cdl +++ b/nczarr_test/ref_filtered.cdl @@ -11,7 +11,6 @@ dimensions: group: g { variables: float var(dim0, dim1, dim2, dim3) ; - var:_FillValue = 9.96921e+36f ; var:_Storage = "chunked" ; var:_ChunkSizes = 4, 4, 4, 4 ; var:_Filter = "307,9" ; diff --git a/nczarr_test/ref_purezarr.cdl b/nczarr_test/ref_purezarr.cdl index f9b9720608..edc00790f7 100644 --- a/nczarr_test/ref_purezarr.cdl +++ b/nczarr_test/ref_purezarr.cdl @@ -4,7 +4,6 @@ dimensions: _zdim_5 = 5 ; variables: int i(_zdim_2, _zdim_5) ; - i:_FillValue = -2147483647 ; data: i = diff --git a/nczarr_test/ref_xarray.cdl b/nczarr_test/ref_xarray.cdl index cd3e3c3874..69ef9f8e8b 100644 --- a/nczarr_test/ref_xarray.cdl +++ b/nczarr_test/ref_xarray.cdl @@ -4,7 +4,6 @@ dimensions: y = 5 ; variables: int i(x, y) ; - i:_FillValue = -2147483647 ; data: i = diff --git a/nczarr_test/ref_zarr_test_data.cdl.gz b/nczarr_test/ref_zarr_test_data.cdl.gz deleted file mode 100644 index 27cc5e8541..0000000000 Binary files a/nczarr_test/ref_zarr_test_data.cdl.gz and /dev/null differ diff --git a/nczarr_test/run_interop.sh b/nczarr_test/run_interop.sh index 8867e9f549..7006509cc1 100755 --- a/nczarr_test/run_interop.sh +++ b/nczarr_test/run_interop.sh @@ -72,7 +72,9 @@ case "$zext" in # Read a test case created by netcdf-java zarr. # Move into position rm -f ${execdir}/ref_zarr_test_data.cdl - if gunzip -c < ${srcdir}/ref_zarr_test_data.cdl.gz > ${execdir}/ref_zarr_test_data.cdl ; then + # Use gunzip because it always appears to be available + if gunzip ${srcdir}/ref_zarr_test_data.cdl.gz ; then ignore=1; fi + if test -f ${srcdir}/ref_zarr_test_data.cdl ; then testcases3 zarr_test_data.zarr ref_zarr_test_data xarray fi ;; diff --git a/nczarr_test/run_quantize.sh b/nczarr_test/run_quantize.sh new file mode 100755 index 0000000000..5df10f2358 --- /dev/null +++ b/nczarr_test/run_quantize.sh @@ -0,0 +1,26 @@ +#!/bin/sh + +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ../test_common.sh + +. "$srcdir/test_nczarr.sh" + +# This shell script runs test_quantize + +set -e + +testcase() { + zext=$1 + fileargs tmp_quantize "mode=$zarr,$zext" + case "$zext" in + file) template="file://${execdir}/%s.zarr#mode=zarr,$zext" ;; + zip) template="file://${execdir}/%s.zip#mode=zarr,$zext" ;; + s3) template="s3://${NCZARR_S3_TEST_BUCKET}/netcdf-c/%s.zarr#mode=zarr,$zext" ;; + *) echo "unknown file type"; exit 1 ;; + esac + ${execdir}/test_quantize "$template" +} + +testcase file +#if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testcase zip; fi +#if test "x$FEATURE_S3TESTS" = xyes ; then testcase s3; fi