From f376c2332952b9951e1feacd88ae866ef2162c2b Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Sat, 29 Feb 2020 12:06:21 -0700 Subject: [PATCH 1/7] Make utilities support NC_COMPACT re: https://github.com/Unidata/netcdf-c/issues/1642 Modify ncdump, nccopy, and ncgen to support the NC_COMPACT storage option. Added test cases and added description to the man pages for the utilities. 1. ncdump: For compact storage variable, print special attribute __Storage_ as ```` : _Storage = "compact"; ```` 2. ncgen: parse and implement ```` : _Storage = "compact"; ```` in a .cdl file 3. nccopy: Extend the chunk specification (-c flag) to support compact using the forms ```` nccopy ... -c :compact and nccopy ... -c :contiguous ```` Misc. other changes 1. cleanup the copy_chunking function in ncdump/nccopy.c --- libhdf5/hdf5var.c | 7 +- ncdump/cdl/ref_tst_special_atts3.cdl | 4 + ncdump/chunkspec.c | 39 +++++ ncdump/chunkspec.h | 7 + ncdump/nccopy.1 | 8 +- ncdump/nccopy.c | 225 +++++++++++++-------------- ncdump/ncdump.1 | 2 +- ncdump/ncdump.c | 17 +- ncdump/ref_ctest_special_atts_4.c | 20 +++ ncdump/ref_tst_special_atts.cdl | 8 +- ncdump/tst_nccopy5.sh | 13 +- ncdump/tst_special_atts.c | 17 +- ncgen/genbin.c | 5 +- ncgen/genc.c | 9 +- ncgen/ncgen.1 | 2 +- ncgen/ncgen.y | 2 + ncgen/ncgeny.c | 2 + 17 files changed, 240 insertions(+), 147 deletions(-) diff --git a/libhdf5/hdf5var.c b/libhdf5/hdf5var.c index 79ebd44033..c83b1eaa38 100644 --- a/libhdf5/hdf5var.c +++ b/libhdf5/hdf5var.c @@ -736,7 +736,7 @@ nc_def_var_extra(int ncid, int varid, int *shuffle, int *unused1, /* Does the user want a contiguous or compact dataset? Not so * fast! Make sure that there are no unlimited dimensions, and * no filters in use for this data. */ - if (*storage) + if (*storage != NC_CHUNKED) { if (nclistlength(var->filters) > 0 || var->fletcher32 || var->shuffle) return NC_EINVAL; @@ -747,9 +747,12 @@ nc_def_var_extra(int ncid, int varid, int *shuffle, int *unused1, } /* Handle chunked storage settings. */ - if (*storage == NC_CHUNKED) + if (*storage == NC_CHUNKED && var->ndims == 0) { + var->contiguous = NC_TRUE; + } else if (*storage == NC_CHUNKED) { var->contiguous = NC_FALSE; + var->compact = NC_FALSE; /* If the user provided chunksizes, check that they are not too * big, and that their total size of chunk is less than 4 GB. */ diff --git a/ncdump/cdl/ref_tst_special_atts3.cdl b/ncdump/cdl/ref_tst_special_atts3.cdl index 600d6df359..235a4a8fb9 100644 --- a/ncdump/cdl/ref_tst_special_atts3.cdl +++ b/ncdump/cdl/ref_tst_special_atts3.cdl @@ -27,6 +27,10 @@ variables: var4:_Shuffle = "true" ; var4:_Endianness = "little" ; var4:_NoFill = "true" ; + int var5(dim1, dim2, dim3) ; + var5:_Storage = "compact" ; + int var6; + var6:_Storage = "compact" ; float slp(time, lat, lon) ; slp:_FillValue = 1.e+15f ; slp:_DeflateLevel = 1 ; diff --git a/ncdump/chunkspec.c b/ncdump/chunkspec.c index e48df5f7af..8029530bc0 100644 --- a/ncdump/chunkspec.c +++ b/ncdump/chunkspec.c @@ -24,6 +24,7 @@ struct VarChunkSpec { size_t rank; /* number of dimensions in chunkspec string */ size_t chunksizes[NC_MAX_VAR_DIMS]; /* corresponding chunk sizes */ bool_t omit; /* true if chunking to be turned off */ + int kind; int igrpid; /* container of the (input) variable */ int ivarid; /* (input) Variable whose chunks are specified */ }; @@ -208,6 +209,20 @@ dimchunkspec_omit(void) { } +/* Return whether chunking should be omitted, due to explicit + * command-line specification. */ +bool_t +dimchunkspec_exists(int indimid) { + int idim; + for(idim = 0; idim < dimchunkspecs.ndims; idim++) { + if(indimid == dimchunkspecs.idimids[idim]) { + return 1; + } + } + return 0; +} + + /* * Parse per-variable chunkspec string and convert into varchunkspec structure. * ncid: location ID of open netCDF file or group in an open file @@ -260,6 +275,16 @@ varchunkspec_parse(int igrp, const char *spec0) goto done; } + /* See if the remainder matches 'compact' or 'contiguous' */ + if(strcasecmp(p,"compact")==0) { + chunkspec->kind = NC_COMPACT; + goto notchunked; + } if(strcasecmp(p,"contiguous")==0) { + chunkspec->kind = NC_CONTIGUOUS; + goto notchunked; + } else + chunkspec->kind = NC_CHUNKED; + /* Iterate over dimension sizes */ while(*p) { unsigned long dimsize; @@ -293,6 +318,7 @@ varchunkspec_parse(int igrp, const char *spec0) if(chunkspec->chunksizes[i] > len) {ret = NC_EBADCHUNK; goto done;} } +notchunked: /* add the chunkspec to our list */ listpush(varchunkspecs,chunkspec); chunkspec = NULL; @@ -307,6 +333,19 @@ varchunkspec_parse(int igrp, const char *spec0) /* Accessors */ +/* Return NC_CHUNKED || NC_CONTIGUOUS || NC_COMPACT */ +int +varchunkspec_kind(int grpid, int varid) +{ + int i; + for(i=0;iigrpid == grpid && spec->ivarid == varid) + return spec->kind; + } + return NC_CONTIGUOUS; /* default */ +} + bool_t varchunkspec_exists(int igrpid, int ivarid) { diff --git a/ncdump/chunkspec.h b/ncdump/chunkspec.h index 9b965ae40d..8e2f9f06c8 100644 --- a/ncdump/chunkspec.h +++ b/ncdump/chunkspec.h @@ -17,6 +17,11 @@ chunkspec_parse(int ncid, const char *spec); extern size_t dimchunkspec_size(int dimid); +/* Return 1 if a dimension spec is defined + * corresponding to dimid, 0 if not found */ +extern bool_t +dimchunkspec_exists(int indimid); + /* Return number of dimensions for which chunking was specified in * chunkspec string on command line, 0 if no chunkspec string was * specified. */ @@ -36,6 +41,8 @@ extern size_t varchunkspec_ndims(int grpid, int varid); extern bool_t varchunkspec_exists(int grpid, int varid); +extern int varchunkspec_kind(int grpid, int varid); + extern void chunkspecinit(void); diff --git a/ncdump/nccopy.1 b/ncdump/nccopy.1 index 5c1fa0a037..1a8bf1d59e 100644 --- a/ncdump/nccopy.1 +++ b/ncdump/nccopy.1 @@ -142,7 +142,7 @@ other filters such as checksums. Changing the chunking in a netCDF file can also greatly speedup access, by choosing chunk shapes that are appropriate for the most common access patterns. .IP -The \fIchunkspec\fP argument has two forms. The first form is the +The \fIchunkspec\fP argument has several forms. The first form is the original, deprecated form and is a string of comma-separated associations, each specifying a dimension name, a '/' character, and optionally the corresponding chunk length for that dimension. No blanks should @@ -186,6 +186,12 @@ If the same variable is specified more than once, the second and later specifications are ignored. Also, this second form, per-variable chunking, takes precedence over any per-dimension chunking except the bare "/" case. +.IP +The third form of the \fIchunkspec\fP has the +syntax: \fI var:compact\fP or \fI var:contiguous\fP. +This explicitly attempts to set the variable storage type as +compact or contiguous, respectively. These may be overridden +if other flags require the variable to be chunked. .IP "\fB \-v \fP \fI var1,... \fP" The output will include data values for the specified variables, in addition to the declarations of all dimensions, variables, and diff --git a/ncdump/nccopy.c b/ncdump/nccopy.c index b433e43920..7accdeae9e 100644 --- a/ncdump/nccopy.c +++ b/ncdump/nccopy.c @@ -400,14 +400,14 @@ filterspecsforvar(const char* ofqn) } /* Return size of chunk in bytes for a variable varid in a group igrp, or 0 if - * layout is contiguous */ + * layout is contiguous|compact */ static int inq_var_chunksize(int igrp, int varid, size_t* chunksizep) { int stat = NC_NOERR; int ndims; size_t *chunksizes; int dim; - int contig = 1; + int contig = NC_CONTIGUOUS; nc_type vartype; size_t value_size; size_t prod; @@ -418,10 +418,9 @@ inq_var_chunksize(int igrp, int varid, size_t* chunksizep) { prod = value_size; NC_CHECK(nc_inq_varndims(igrp, varid, &ndims)); chunksizes = (size_t *) emalloc((ndims + 1) * sizeof(size_t)); - if(ndims > 0) { - NC_CHECK(nc_inq_var_chunking(igrp, varid, &contig, NULL)); - } - if(contig == 1) { + contig = NC_CHUNKED; + NC_CHECK(nc_inq_var_chunking(igrp, varid, &contig, NULL)); + if(contig != NC_CHUNKED) { *chunksizep = 0; } else { NC_CHECK(nc_inq_var_chunking(igrp, varid, &contig, chunksizes)); @@ -448,7 +447,7 @@ inq_var_chunking_params(int igrp, int ivarid, int ogrp, int ovarid, int ndims; size_t *ichunksizes, *ochunksizes; int dim; - int icontig = 1, ocontig = 1; + int icontig = NC_CONTIGUOUS, ocontig = NC_CONTIGUOUS; nc_type vartype; size_t value_size; size_t prod, iprod, oprod; @@ -458,11 +457,10 @@ inq_var_chunking_params(int igrp, int ivarid, int ogrp, int ovarid, *chunkcache_preemptionp = COPY_CHUNKCACHE_PREEMPTION; NC_CHECK(nc_inq_varndims(igrp, ivarid, &ndims)); - if(ndims > 0) { - NC_CHECK(nc_inq_var_chunking(igrp, ivarid, &icontig, NULL)); - NC_CHECK(nc_inq_var_chunking(ogrp, ovarid, &ocontig, NULL)); - } - if(icontig == 1 && ocontig == 1) { /* no chunking in input or output */ + icontig = (ocontig = NC_CHUNKED); + NC_CHECK(nc_inq_var_chunking(igrp, ivarid, &icontig, NULL)); + NC_CHECK(nc_inq_var_chunking(ogrp, ovarid, &ocontig, NULL)); + if(icontig != NC_CHUNKED && ocontig != NC_CHUNKED) { /* no chunking in input or output */ *chunkcache_nelemsp = 0; *chunkcache_sizep = 0; *chunkcache_preemptionp = 0; @@ -473,7 +471,7 @@ inq_var_chunking_params(int igrp, int ivarid, int ogrp, int ovarid, NC_CHECK(nc_inq_type(igrp, vartype, NULL, &value_size)); iprod = value_size; - if(icontig == 0 && ocontig == 1) { /* chunking only in input */ + if(icontig == NC_CHUNKED && ocontig != NC_CHUNKED) { /* chunking only in input */ *chunkcache_nelemsp = 1; /* read one input chunk at a time */ *chunkcache_sizep = iprod; *chunkcache_preemptionp = 1.0f; @@ -481,7 +479,7 @@ inq_var_chunking_params(int igrp, int ivarid, int ogrp, int ovarid, } ichunksizes = (size_t *) emalloc((ndims + 1) * sizeof(size_t)); - if(icontig == 1) { /* if input contiguous, treat as if chunked on + if(icontig != NC_CHUNKED) { /* if input contiguous|compact, treat as if chunked on * first dimension */ ichunksizes[0] = 1; for(dim = 1; dim < ndims; dim++) { @@ -491,7 +489,7 @@ inq_var_chunking_params(int igrp, int ivarid, int ogrp, int ovarid, NC_CHECK(nc_inq_var_chunking(igrp, ivarid, &icontig, ichunksizes)); } - /* now can assume chunking in both input and output */ + /* now can pretend chunking in both input and output */ ochunksizes = (size_t *) emalloc((ndims + 1) * sizeof(size_t)); NC_CHECK(nc_inq_var_chunking(ogrp, ovarid, &ocontig, ochunksizes)); @@ -903,148 +901,136 @@ copy_chunking(int igrp, int i_varid, int ogrp, int o_varid, int ndims, int inkin int outnc4 = (outkind == NC_FORMAT_NETCDF4 || outkind == NC_FORMAT_NETCDF4_CLASSIC); VarID ovid; char* ofqn = NULL; - int nochunkspec = 1 ; /* 1 => no -c option applies to this variable */ + int icontig = NC_CONTIGUOUS; + int ocontig = NC_CONTIGUOUS; + size_t ichunkp[NC_MAX_VAR_DIMS]; + size_t ochunkp[NC_MAX_VAR_DIMS]; + size_t dimlens[NC_MAX_VAR_DIMS]; + int is_unlimited = 0; /* First, check the file kinds */ if(!outnc4) return stat; /* no chunking */ - /* See if a scalar */ - if(ndims == 0) - return stat; /* scalars cannot be chunked */ + memset(ichunkp,0,sizeof(ichunkp)); + memset(ochunkp,0,sizeof(ochunkp)); + memset(dimlens,0,sizeof(dimlens)); + + /* Get the chunking, if any, on the current input variable */ + if(innc4) { + NC_CHECK(nc_inq_var_chunking(igrp, i_varid, &icontig, ichunkp)); + /* pretend that this is same as a -c option */ + } else { /* !innc4 */ + icontig = NC_CONTIGUOUS; + ichunkp[0] = 0; + } /* If var specific chunking was specified for this output variable then it overrides all else. */ - /* Note, using goto done instead of nested if-then-else */ - if(varchunkspec_exists(igrp,i_varid)) { - if(varchunkspec_omit(igrp,i_varid)) { - NC_CHECK(nc_def_var_chunking(ogrp, o_varid, NC_CONTIGUOUS, NULL)); - } else { - size_t* ochunkp = varchunkspec_chunksizes(igrp,i_varid); - NC_CHECK(nc_def_var_chunking(ogrp, o_varid, NC_CHUNKED, ochunkp)); - } - goto done; - } + /* Note, using goto done instead of nested if-then-else */ - /* See about dim-specific chunking */ + /* First check on output contiguous'ness */ + /* Note: the chunkspecs are defined in terms of input variable+grp ids. + The grp may differ if !innc4 && outnc4 */ + if(varchunkspec_omit(igrp,i_varid)) + ocontig = NC_CONTIGUOUS; + else if(varchunkspec_exists(igrp,i_varid)) + ocontig = varchunkspec_kind(igrp,i_varid); + else + ocontig = icontig; + + /* Figure out the chunking even if we do not decide to do so*/ + if(varchunkspec_exists(igrp,i_varid) + && varchunkspec_kind(igrp,i_varid) == NC_CHUNKED) + memcpy(ochunkp,varchunkspec_chunksizes(igrp,i_varid),ndims*sizeof(size_t)); + + /* If any kind of output filter was specified, then not contiguous */ + ovid.grpid = ogrp; + ovid.varid = o_varid; + if((stat=computeFQN(ovid,&ofqn))) goto done; + if(option_deflate_level >= 0 || filterspecforvar(ofqn) != NULL) + ocontig = NC_CHUNKED; + + /* See about dim-specific chunking; does not override -c spec*/ { int idim; /* size of a chunk: product of dimension chunksizes and size of value */ size_t csprod; - int is_unlimited = 0; size_t typesize; - size_t ichunkp[NC_MAX_VAR_DIMS]; - size_t ochunkp[NC_MAX_VAR_DIMS]; int dimids[NC_MAX_VAR_DIMS]; - int icontig = 1; - int ocontig = 1; /* until proven otherwise */ /* See if dim-specific chunking was suppressed */ - if(dimchunkspec_omit()) - goto next2; + if(dimchunkspec_omit()) /* use input chunksizes */ + goto next2; - /* Setup for output chunking */ - typesize = val_size(ogrp, o_varid); + /* Setup for possible output chunking */ + typesize = val_size(igrp, i_varid); csprod = typesize; memset(&dimids,0,sizeof(dimids)); - memset(&ichunkp,0,sizeof(ichunkp)); - memset(&ochunkp,0,sizeof(ochunkp)); - - /* Get the chunking, if any, on the current input variable */ - if(innc4) { - NC_CHECK(nc_inq_var_chunking(igrp, i_varid, &icontig, ichunkp)); - /* pretend that this is same as a -c option */ - nochunkspec = 0; - } else { - icontig = 1; - ichunkp[0] = 0; - } - if(!icontig) - ocontig = 0; /* If input is chunked, then so is output */ /* Prepare to iterate over the dimids of this input variable */ NC_CHECK(nc_inq_vardimid(igrp, i_varid, dimids)); - /* Assign chunk sizes for all dimensions of variable; + /* Capture dimension lengts for all dimensions of variable; even if we decide to not chunk */ for(idim = 0; idim < ndims; idim++) { int idimid = dimids[idim]; int odimid = dimmap_odimid(idimid); - size_t chunksize; - size_t dimlen; /* Get input dimension length */ - NC_CHECK(nc_inq_dimlen(igrp, idimid, &dimlen)); + NC_CHECK(nc_inq_dimlen(igrp, idimid, &dimlens[idim])); /* Check for unlimited */ if(dimmap_ounlim(odimid)) { is_unlimited = 1; - ocontig = 0; /* force chunking */ - } - - /* If the -c set a chunk size for this dimension, use it */ - chunksize = dimchunkspec_size(idimid); - if(chunksize > 0) { /* found in chunkspec */ - ochunkp[idim] = chunksize; - ocontig = 0; /* cannot use contiguous */ - nochunkspec = 0; /* form of explicit chunking */ - goto next; - } - - /* Not specified in -c; Apply defaulting rules as defined in nccopy.1 */ - - /* If input is chunked, then use that chunk size */ - if(!icontig) { - ochunkp[idim] = ichunkp[idim]; - ocontig = 0; - goto next; + ocontig = NC_CHUNKED; /* force chunking */ } - /* If input is not netcdf-4 then use the input size as the chunk size; - but do not force chunking. - */ - if(!innc4) { - ochunkp[idim] = dimlen; - goto next; - } + if(dimchunkspec_exists(idimid)) { + /* If the -c set a chunk size for this dimension, use it */ + dimlens[idim] = dimchunkspec_size(idimid); /* Save it */ + ocontig = NC_CHUNKED; /* force chunking */ + } /* Default for unlimited is max(4 megabytes, current dim size) */ if(is_unlimited) { size_t mb4dimsize = DFALTUNLIMSIZE / typesize; - ochunkp[idim] = (dimlen > mb4dimsize ? mb4dimsize : dimlen); - } else { - /* final default is the current dimension size */ - ochunkp[idim] = dimlen; + if(dimlens[idim] > mb4dimsize) + dimlens[idim] = mb4dimsize; } - -next: + } + /* compute the final ochunksizes: precedence is output, input, dimeln */ + for(idim = 0; idim < ndims; idim++) { + if(ochunkp[idim] == 0) { + if(ichunkp[idim] != 0) + ochunkp[idim] = ichunkp[idim]; + } + if(ochunkp[idim] == 0) { + if(dimlens[idim] != 0) + ochunkp[idim] = dimlens[idim]; + } + if(ochunkp[idim] == 0) {stat = NC_EINTERNAL; goto done;} /* compute on-going dimension product */ csprod *= ochunkp[idim]; } - - /* Finally, if total chunksize is too small (and dim is not unlimited) => do not chunk */ - if(csprod < option_min_chunk_bytes && !is_unlimited) { - ocontig = 1; /* Force contiguous */ - } + /* if total chunksize is too small (and dim is not unlimited) => do not chunk */ + if(csprod < option_min_chunk_bytes && !is_unlimited) + ocontig = NC_CONTIGUOUS; /* Force contiguous */ + } next2: - /* If any kind of output filter was specified, then not contiguous */ - ovid.grpid = ogrp; - ovid.varid = o_varid; - if((stat=computeFQN(ovid,&ofqn))) goto done; - if(option_deflate_level >= 0 || hasfilterspecsforvar(ofqn)) - ocontig = 0; - - /* Apply the chunking, if any */ - if(!nochunkspec) {/* explicitly set chunking */ - if(ocontig) { /* We can use contiguous output */ - NC_CHECK(nc_def_var_chunking(ogrp, o_varid, NC_CONTIGUOUS, NULL)); - } else { - NC_CHECK(nc_def_var_chunking(ogrp, o_varid, NC_CHUNKED, ochunkp)); - } - } /* else no chunk spec at all, let defaults set at nc_def_var() be used */ + /* Apply the chunking, if any */ + switch (ocontig) { + case NC_CHUNKED: + NC_CHECK(nc_def_var_chunking(ogrp, o_varid, NC_CHUNKED, ochunkp)); + break; + case NC_CONTIGUOUS: + case NC_COMPACT: + NC_CHECK(nc_def_var_chunking(ogrp, o_varid, ocontig, NULL)); + break; + default: stat = NC_EINVAL; goto done; } #ifdef USE_NETCDF4 @@ -1052,18 +1038,15 @@ copy_chunking(int igrp, int i_varid, int ogrp, int o_varid, int ndims, int inkin { int d; size_t chunksizes[NC_MAX_VAR_DIMS]; char name[NC_MAX_NAME]; - int contig = 0; - unsigned long long totalsize = 1; - NC_CHECK(nc_inq_var(ogrp,o_varid,name,NULL,NULL,NULL,NULL)); - NC_CHECK(nc_inq_var_chunking(ogrp, o_varid, &contig, chunksizes)); - fprintf(stderr,"xxx: chunk sizes: %s[",name); - if(contig) { + if(ocontig == NC_CONTIGUOUS) { fprintf(stderr,"contig]\n"); + } else if(ocontig == NC_COMPACT) { + fprintf(stderr,"compact]\n"); } else { for(d=0;d 0) fprintf(stderr,","); - fprintf(stderr,"%lu",(unsigned long)chunksizes[d]); + fprintf(stderr,"%lu",(unsigned long)ochunkp[d]); } fprintf(stderr,"]=%llu\n",totalsize); } @@ -1267,9 +1250,9 @@ free_var_chunk_cache(int grp, int varid) int kind; NC_CHECK(nc_inq_format(grp, &kind)); if(kind == NC_FORMAT_NETCDF4 || kind == NC_FORMAT_NETCDF4_CLASSIC) { - int contig = 1; + int contig = NC_CONTIGUOUS NC_CHECK(nc_inq_var_chunking(grp, varid, &contig, NULL)); - if(contig == 0) { /* chunked */ + if(contig == NC_CHUNKED) { /* chunked */ NC_CHECK(nc_set_var_chunk_cache(grp, varid, chunk_cache_size, cache_nelems, cache_preemp)); } } @@ -1576,9 +1559,9 @@ copy_var_data(int igrp, int varid, int ogrp) { NC_CHECK(nc_inq_format(ogrp, &okind)); if(okind == NC_FORMAT_NETCDF4 || okind == NC_FORMAT_NETCDF4_CLASSIC) { /* if this variable chunked, set variable chunk cache size */ - int contig = 1; + int contig = NC_CONTIGUOUS; NC_CHECK(nc_inq_var_chunking(ogrp, ovarid, &contig, NULL)); - if(contig == 0) { /* chunked */ + if(contig == NC_CHUNKED) { /* chunked */ if(option_compute_chunkcaches) { /* Try to estimate variable-specific chunk cache, * depending on specific size and shape of this diff --git a/ncdump/ncdump.1 b/ncdump/ncdump.1 index 9e241bb887..bc835e0965 100644 --- a/ncdump/ncdump.1 +++ b/ncdump/ncdump.1 @@ -195,7 +195,7 @@ classic model'. `_NoFill' is `true' if the persistent NoFill property was set for the variable when it was defined. `_Shuffle' is `true' if use of the shuffle filter was specified for the variable. -`_Storage' is `contiguous' or `chunked', depending on how the +`_Storage' is `contiguous' or `compact' or `chunked', depending on how the variable's data is stored. .IP "\fB-t\fP" Controls display of time data, if stored in a variable that uses diff --git a/ncdump/ncdump.c b/ncdump/ncdump.c index bbf13d68a0..f534f56152 100644 --- a/ncdump/ncdump.c +++ b/ncdump/ncdump.c @@ -983,17 +983,19 @@ pr_att_specials( const ncvar_t *varp ) { + int contig = NC_CHUNKED; /* No special variable attributes for classic or 64-bit offset data */ if(kind == 1 || kind == 2) return; - /* _Chunking */ - if (varp->ndims > 0) { /* no chunking for scalar variables */ - int contig = 0; - NC_CHECK( nc_inq_var_chunking(ncid, varid, &contig, NULL ) ); - if(contig == 1) { - pr_att_name(ncid, varp->name, NC_ATT_STORAGE); + /* _Chunking tests */ + NC_CHECK( nc_inq_var_chunking(ncid, varid, &contig, NULL ) ); + if(contig == NC_CONTIGUOUS) { + pr_att_name(ncid, varp->name, NC_ATT_STORAGE); printf(" = \"contiguous\" ;\n"); - } else { + } else if(contig == NC_COMPACT) { + pr_att_name(ncid, varp->name, NC_ATT_STORAGE); + printf(" = \"compact\" ;\n"); + } else { size_t *chunkp; int i; pr_att_name(ncid, varp->name, NC_ATT_STORAGE); @@ -1007,7 +1009,6 @@ pr_att_specials( printf("%lu%s", (unsigned long)chunkp[i], i+1 < varp->ndims ? ", " : " ;\n"); } free(chunkp); - } } /* _Filter (including deflate and shuffle) */ diff --git a/ncdump/ref_ctest_special_atts_4.c b/ncdump/ref_ctest_special_atts_4.c index dfea7761dd..ae4297189d 100644 --- a/ncdump/ref_ctest_special_atts_4.c +++ b/ncdump/ref_ctest_special_atts_4.c @@ -57,6 +57,8 @@ main() {/* create ref_tst_special_atts.nc */ int var3_id; int var4_id; int var5_id; + int var6_id; + int var7_id; /* rank (number of dimensions) for each variable */ # define RANK_var1 1 @@ -64,6 +66,8 @@ main() {/* create ref_tst_special_atts.nc */ # define RANK_var3 3 # define RANK_var4 3 # define RANK_var5 1 +# define RANK_var6 1 +# define RANK_var7 0 /* variable shapes */ int var1_dims[RANK_var1]; @@ -71,6 +75,7 @@ main() {/* create ref_tst_special_atts.nc */ int var3_dims[RANK_var3]; int var4_dims[RANK_var4]; int var5_dims[RANK_var5]; + int var6_dims[RANK_var6]; /* enter define mode */ stat = nc_create("ref_tst_special_atts.nc", NC_CLOBBER|NC_NETCDF4, &ncid); @@ -158,6 +163,21 @@ main() {/* create ref_tst_special_atts.nc */ stat = nc_def_var_fill(tst_special_atts_grp, var5_id, NC_NOFILL, NULL); check_err(stat,__LINE__,__FILE__); + var6_dims[0] = dim1_dim; + stat = nc_def_var(tst_special_atts_grp, "var6", NC_INT, RANK_var6, var6_dims, &var6_id); + check_err(stat,__LINE__,__FILE__); + stat = nc_def_var_chunking(tst_special_atts_grp, var6_id, NC_COMPACT, NULL); + check_err(stat,__LINE__,__FILE__); + stat = nc_def_var_endian(tst_special_atts_grp, var6_id, NC_ENDIAN_LITTLE); + check_err(stat,__LINE__,__FILE__); + + stat = nc_def_var(tst_special_atts_grp, "var7", NC_INT, RANK_var7, 0, &var7_id); + check_err(stat,__LINE__,__FILE__); + stat = nc_def_var_chunking(tst_special_atts_grp, var7_id, NC_COMPACT, NULL); + check_err(stat,__LINE__,__FILE__); + stat = nc_def_var_endian(tst_special_atts_grp, var7_id, NC_ENDIAN_LITTLE); + check_err(stat,__LINE__,__FILE__); + /* leave define mode */ stat = nc_enddef (tst_special_atts_grp); check_err(stat,__LINE__,__FILE__); diff --git a/ncdump/ref_tst_special_atts.cdl b/ncdump/ref_tst_special_atts.cdl index b846dde9d4..41ca757cfc 100644 --- a/ncdump/ref_tst_special_atts.cdl +++ b/ncdump/ref_tst_special_atts.cdl @@ -43,9 +43,15 @@ variables: var5:_Shuffle = "true" ; var5:_Fletcher32 = "true" ; var5:_NoFill = "true" ; + int var6(dim1) ; + var6:_Storage = "compact" ; + var6:_Endianness = "little" ; + int var7 ; + var7:_Storage = "compact" ; + var7:_Endianness = "little" ; // global attributes: - :_NCProperties = "version=2,netcdf=4.6.2-development,hdf5=1.10.1" ; + :_NCProperties = "version=2,netcdf=4.7.4-development,hdf5=1.10.4," ; :_SuperblockVersion = 0 ; :_IsNetcdf4 = 1 ; :_Format = "netCDF-4" ; diff --git a/ncdump/tst_nccopy5.sh b/ncdump/tst_nccopy5.sh index 6d369c3749..8374cfc4e4 100755 --- a/ncdump/tst_nccopy5.sh +++ b/ncdump/tst_nccopy5.sh @@ -86,7 +86,8 @@ ${NCDUMP} tst_nc5.nc > tst_nc5.cdl echo "*** Test nccopy -c with per-variable chunking; classic->enhanced" # This should produce same as -c dim0/,dim1/1,dim2/,dim3/1,dim4/,dim5/1,dim6/ -${NCCOPY} -c ivar:7,1,2,1,5,1,9 tst_nc5.nc tmp_nc5.nc +# But note that the chunk product is less than default, so we need to reduce it (-M) +${NCCOPY} -M1000 -c ivar:7,1,2,1,5,1,9 tst_nc5.nc tmp_nc5.nc ${NCDUMP} -n tst_nc5 tmp_nc5.nc > tmp_nc5.cdl # Verify that the core cdl is the same diff tst_nc5.cdl tmp_nc5.cdl @@ -109,7 +110,8 @@ echo "*** Test nccopy -c with per-variable chunking; enhanced->enhanced" reset ./tst_chunking tst_nc5.nc deflate ${NCDUMP} -n tst_nc5 tst_nc5.nc > tst_nc5.cdl -${NCCOPY} -c ivar:4,1,2,1,5,2,3 tst_nc5.nc tmp_nc5.nc +# Use -M to ensure that chunking takes effect +${NCCOPY} -M500 -c ivar:4,1,2,1,5,2,3 tst_nc5.nc tmp_nc5.nc ${NCDUMP} -n tst_nc5 tmp_nc5.nc > tmp_nc5.cdl diff tst_nc5.cdl tmp_nc5.cdl @@ -121,7 +123,6 @@ TESTLINE=`sed -e '/ivar:_ChunkSizes/p' -e d enhanced" reset ./tst_chunking tst_nc5.nc group ${NCDUMP} -n tst_nc5 tst_nc5.nc > tst_nc5.cdl -${NCCOPY} -c /g/ivar:4,1,2,1,5,2,3 tst_nc5.nc tmp_nc5.nc +${NCCOPY} -M500 -c /g/ivar:4,1,2,1,5,2,3 tst_nc5.nc tmp_nc5.nc ${NCDUMP} -n tst_nc5 tmp_nc5.nc > tmp_nc5.cdl diff tst_nc5.cdl tmp_nc5.cdl @@ -153,7 +154,7 @@ echo "*** Test nccopy -c with unlimited dimension; classic ->enhanced" reset ./tst_chunking tst_nc5.nc unlimited # should produce modified tmp_nc5.nc with ivar of rank 2 ${NCDUMP} -n tst_nc5 tst_nc5.nc > tst_nc5.cdl -${NCCOPY} -c ivar:5,3 tst_nc5.nc tmp_nc5.nc +${NCCOPY} -M500 -c ivar:5,3 tst_nc5.nc tmp_nc5.nc ${NCDUMP} -n tst_nc5 tmp_nc5.nc > tmp_nc5.cdl diff tst_nc5.cdl tmp_nc5.cdl @@ -176,7 +177,7 @@ echo "*** Test nccopy -c fvar: to suppress chunking; classic ->enhanced" reset ./tst_chunking tst_nc5_omit.nc ${NCDUMP} -n tst_nc5_omit tst_nc5_omit.nc > tst_nc5_omit.cdl -${NCCOPY} -c ivar:7,1,2,1,5,1,9 -c fvar: tst_nc5_omit.nc tmp_nc5_omit.nc +${NCCOPY} -M500 -c ivar:7,1,2,1,5,1,9 -c fvar: tst_nc5_omit.nc tmp_nc5_omit.nc ${NCDUMP} -n tst_nc5_omit tmp_nc5_omit.nc > tmp_nc5_omit.cdl diff tst_nc5_omit.cdl tmp_nc5_omit.cdl diff --git a/ncdump/tst_special_atts.c b/ncdump/tst_special_atts.c index 7343bb6674..9655b86bb9 100644 --- a/ncdump/tst_special_atts.c +++ b/ncdump/tst_special_atts.c @@ -28,6 +28,11 @@ #define VAR3_RANK 3 #define VAR4_NAME "var4" #define VAR4_RANK 3 +#define VAR5_NAME "var5" +#define VAR5_RANK VAR1_RANK +#define VAR6_NAME "var6" +#define VAR6_RANK 1 +#define VAR7_NAME "var7" #define CHUNK1 (DIM1_LEN/2 + 1) #define CHUNK2 (DIM2_LEN/3 + 1) #define CHUNK3 (DIM3_LEN/4 + 1) @@ -42,7 +47,7 @@ main(int argc, char **argv) int i, j, k, m; int dimids[VAR3_RANK]; - int var1id, var2id, var3id, var4id, var5id; + int var1id, var2id, var3id, var4id, var5id, var6id, var7id; size_t chunksizes[] = {CHUNK1, CHUNK2, CHUNK3}; int data1[DIM1_LEN]; int data1_in[DIM1_LEN]; @@ -105,9 +110,11 @@ main(int argc, char **argv) NC_COMPOUND_OFFSET(struct obs_t, attention_span), NC_INT64)) ERR; /* create a variable of that compound type */ - if (nc_def_var(ncid, "var5", typeid, VAR1_RANK, dimids, &var5id)) + if (nc_def_var(ncid, VAR5_NAME, typeid, VAR5_RANK, dimids, &var5id)) ERR; } + if (nc_def_var(ncid, VAR6_NAME, NC_INT, VAR6_RANK, dimids, &var6id)) ERR; + if (nc_def_var(ncid, VAR7_NAME, NC_INT, 0, NULL, &var7id)) ERR; /* Specify contiguous storage and endianness explicitly for var1. */ if (nc_def_var_chunking(ncid, var1id, NC_CONTIGUOUS, NULL)) ERR; @@ -135,6 +142,10 @@ main(int argc, char **argv) if (nc_def_var_fletcher32(ncid, var5id, NC_FLETCHER32)) ERR; if (nc_def_var_deflate(ncid, var5id, NC_SHUFFLE, COMPRESS, DEFLATE_LEVEL)) ERR; + /* Set _Storage as compact */ + if (nc_def_var_chunking(ncid, var6id, NC_COMPACT, NULL)) ERR; + if (nc_def_var_chunking(ncid, var7id, NC_COMPACT, NULL)) ERR; + if (nc_enddef(ncid)) ERR; /* Some artificial data */ @@ -154,6 +165,8 @@ main(int argc, char **argv) if(nc_put_var(ncid, var2id, &data2[0][0])) ERR; if(nc_put_var(ncid, var3id, &data3[0][0][0])) ERR; if(nc_put_var(ncid, var4id, &data3[0][0][0])) ERR; + if(nc_put_var(ncid, var6id, &data3[0][0][0])) ERR; + if(nc_put_var(ncid, var7id, &data3[0][0][0])) ERR; if (nc_close(ncid)) ERR; diff --git a/ncgen/genbin.c b/ncgen/genbin.c index 84cc3670a1..70ae4ea487 100644 --- a/ncgen/genbin.c +++ b/ncgen/genbin.c @@ -199,8 +199,9 @@ genbin_definespecialattributes(Symbol* var) int stat = NC_NOERR; Specialdata* special = var->var.special; if(special->flags & _STORAGE_FLAG) { - if(special->_Storage == NC_CONTIGUOUS) { - stat = nc_def_var_chunking(var->container->nc_id, var->nc_id, NC_CONTIGUOUS, NULL); + if(special->_Storage == NC_CONTIGUOUS + || special->_Storage == NC_COMPACT) { + stat = nc_def_var_chunking(var->container->nc_id, var->nc_id, special->_Storage, NULL); } else { /* chunked */ if(special->nchunks == 0 || special->_ChunkSizes == NULL) derror("NC_CHUNKED requested, but no chunksizes specified"); diff --git a/ncgen/genc.c b/ncgen/genc.c index 7655567ad9..2ded919cb9 100644 --- a/ncgen/genc.c +++ b/ncgen/genc.c @@ -453,13 +453,18 @@ genc_definespecialattributes(Symbol* vsym) Specialdata* special = vsym->var.special; if(usingclassic) return; if(special->flags & _STORAGE_FLAG) { - int storage = special->_Storage; + const char* storage = NULL; size_t* chunks = special->_ChunkSizes; + switch (special->_Storage) { + case NC_CONTIGUOUS: storage = "NC_CONTIGUOUS"; break; + case NC_COMPACT: storage = "NC_COMPACT"; break; + case NC_CHUNKED: storage = "NC_CHUNKED"; break; + } bbprintf0(stmt, " stat = nc_def_var_chunking(%s, %s, %s, ", groupncid(vsym->container), varncid(vsym), - (storage == NC_CONTIGUOUS?"NC_CONTIGUOUS":"NC_CHUNKED")); + storage); codedump(stmt); if(special->nchunks == 0 || chunks == NULL) codepartial("NULL"); diff --git a/ncgen/ncgen.1 b/ncgen/ncgen.1 index 84dd428831..9b6646fd86 100644 --- a/ncgen/ncgen.1 +++ b/ncgen/ncgen.1 @@ -808,7 +808,7 @@ for the variable. .IP 4. 3 `_Shuffle' specifies if the the shuffle filter should be used. .IP 5. 3 -`_Storage' is `contiguous' or `chunked'. +`_Storage' is `contiguous' or `compact` or `chunked'. .IP 6. 3 `_ChunkSizes' is a list of chunk sizes for each dimension of the variable diff --git a/ncgen/ncgen.y b/ncgen/ncgen.y index d6fe826538..da88855a4d 100644 --- a/ncgen/ncgen.y +++ b/ncgen/ncgen.y @@ -1305,6 +1305,8 @@ makespecial(int tag, Symbol* vsym, Symbol* tsym, void* data, int isconst) derror("_Storage: illegal NULL value"); else if(strcmp(sdata,"contiguous") == 0) special->_Storage = NC_CONTIGUOUS; + else if(strcmp(sdata,"compact") == 0) + special->_Storage = NC_COMPACT; else if(strcmp(sdata,"chunked") == 0) special->_Storage = NC_CHUNKED; else diff --git a/ncgen/ncgeny.c b/ncgen/ncgeny.c index e427db7f92..c7058ec287 100644 --- a/ncgen/ncgeny.c +++ b/ncgen/ncgeny.c @@ -3329,6 +3329,8 @@ makespecial(int tag, Symbol* vsym, Symbol* tsym, void* data, int isconst) derror("_Storage: illegal NULL value"); else if(strcmp(sdata,"contiguous") == 0) special->_Storage = NC_CONTIGUOUS; + else if(strcmp(sdata,"compact") == 0) + special->_Storage = NC_COMPACT; else if(strcmp(sdata,"chunked") == 0) special->_Storage = NC_CHUNKED; else From 7ff42a284830c5eea6ed5ae6dd9a8ea5f577a04c Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Sat, 29 Feb 2020 13:02:49 -0700 Subject: [PATCH 2/7] Fix typo --- ncdump/nccopy.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ncdump/nccopy.c b/ncdump/nccopy.c index 7accdeae9e..4d858603eb 100644 --- a/ncdump/nccopy.c +++ b/ncdump/nccopy.c @@ -368,7 +368,7 @@ parsefilterspec(const char* optarg0, List* speclist) } static int -hasfilterspecsforvar(const char* ofqn) +hasfilterspecforvar(const char* ofqn) { int i; /* See which output filter specs are defined for this output variable */ @@ -950,7 +950,7 @@ copy_chunking(int igrp, int i_varid, int ogrp, int o_varid, int ndims, int inkin ovid.grpid = ogrp; ovid.varid = o_varid; if((stat=computeFQN(ovid,&ofqn))) goto done; - if(option_deflate_level >= 0 || filterspecforvar(ofqn) != NULL) + if(option_deflate_level >= 0 || hasfilterspecforvar(ofqn) != NULL) ocontig = NC_CHUNKED; /* See about dim-specific chunking; does not override -c spec*/ From e66c727c28e819b79b1a3d40599f1f60f89d0dd3 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Sat, 29 Feb 2020 15:33:27 -0700 Subject: [PATCH 3/7] Fix Filters x compact --- libhdf5/hdf5var.c | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/libhdf5/hdf5var.c b/libhdf5/hdf5var.c index c83b1eaa38..1e3790ce34 100644 --- a/libhdf5/hdf5var.c +++ b/libhdf5/hdf5var.c @@ -173,6 +173,11 @@ nc4_find_default_chunksizes2(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var) total_chunk_size = (double) type_size; #endif + if(var->chunksizes == NULL) { + if((var->chunksizes = calloc(1,sizeof(size_t)*var->ndims)) == NULL) + return NC_ENOMEM; + } + /* How many values in the variable (or one record, if there are * unlimited dimensions). */ for (d = 0; d < var->ndims; d++) @@ -697,6 +702,7 @@ nc_def_var_extra(int ncid, int varid, int *shuffle, int *unused1, /* Set the deflate settings. */ var->contiguous = NC_FALSE; + var->compact = NC_FALSE; var->deflate = *deflate; if (*deflate) var->deflate_level = *deflate_level; @@ -704,18 +710,27 @@ nc_def_var_extra(int ncid, int varid, int *shuffle, int *unused1, } #endif + /* Cannot set filters of any sort on scalars */ + if(var->ndims == 0) { + if(shuffle && *shuffle) + return NC_NOERR; /* ignore */ + if(fletcher32 && *fletcher32) + return NC_NOERR; /* ignore */ + } + /* Shuffle filter? */ if (shuffle) { var->shuffle = *shuffle; var->contiguous = NC_FALSE; - } + var->compact = NC_FALSE; } /* Fletcher32 checksum error protection? */ if (fletcher32) { var->fletcher32 = *fletcher32; var->contiguous = NC_FALSE; + var->compact = NC_FALSE; } #ifdef USE_PARALLEL @@ -749,6 +764,7 @@ nc_def_var_extra(int ncid, int varid, int *shuffle, int *unused1, /* Handle chunked storage settings. */ if (*storage == NC_CHUNKED && var->ndims == 0) { var->contiguous = NC_TRUE; + var->compact = NC_FALSE; } else if (*storage == NC_CHUNKED) { var->contiguous = NC_FALSE; @@ -776,6 +792,7 @@ nc_def_var_extra(int ncid, int varid, int *shuffle, int *unused1, else if (*storage == NC_CONTIGUOUS) { var->contiguous = NC_TRUE; + var->compact = NC_FALSE; } else if (*storage == NC_COMPACT) { @@ -801,7 +818,7 @@ nc_def_var_extra(int ncid, int varid, int *shuffle, int *unused1, { /* Determine default chunksizes for this variable (do nothing * for scalar vars). */ - if (var->chunksizes && !var->chunksizes[0]) + if (var->chunksizes == NULL || var->chunksizes[0] == 0) if ((retval = nc4_find_default_chunksizes2(grp, var))) return retval; From 7d1ca9ac85f7ad539bb9be3a169feef87f4ee7a8 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Mon, 2 Mar 2020 11:12:30 -0700 Subject: [PATCH 4/7] fix references to var->deflate' --- libhdf5/hdf5var.c | 33 --------------------------------- libsrc4/nc4var.c | 2 +- 2 files changed, 1 insertion(+), 34 deletions(-) diff --git a/libhdf5/hdf5var.c b/libhdf5/hdf5var.c index 1e3790ce34..e2bda5ee83 100644 --- a/libhdf5/hdf5var.c +++ b/libhdf5/hdf5var.c @@ -677,39 +677,6 @@ nc_def_var_extra(int ncid, int varid, int *shuffle, int *unused1, if (var->created) return NC_ELATEDEF; -#if 0 - /* Check compression options. */ - if (deflate && !deflate_level) - return NC_EINVAL; - - /* Valid deflate level? */ - if (deflate) - { - if (*deflate) - if (*deflate_level < NC_MIN_DEFLATE_LEVEL || - *deflate_level > NC_MAX_DEFLATE_LEVEL) - return NC_EINVAL; - - /* For scalars, just ignore attempt to deflate. */ - if (!var->ndims) - return NC_NOERR; - - /* If szip is in use, return an error. */ - if ((retval = nc_inq_var_szip(ncid, varid, &option_mask, NULL))) - return retval; - if (option_mask) - return NC_EINVAL; - - /* Set the deflate settings. */ - var->contiguous = NC_FALSE; - var->compact = NC_FALSE; - var->deflate = *deflate; - if (*deflate) - var->deflate_level = *deflate_level; - LOG((3, "%s: *deflate_level %d", __func__, *deflate_level)); - } -#endif - /* Cannot set filters of any sort on scalars */ if(var->ndims == 0) { if(shuffle && *shuffle) diff --git a/libsrc4/nc4var.c b/libsrc4/nc4var.c index 2cba6b6307..79d6c5d062 100644 --- a/libsrc4/nc4var.c +++ b/libsrc4/nc4var.c @@ -446,7 +446,7 @@ NC4_var_par_access(int ncid, int varid, int par_access) /* If zlib, shuffle, or fletcher32 filters are in use, then access * must be collective. Fail an attempt to set such a variable to * independent access. */ - if ((var->deflate || var->shuffle || var->fletcher32) && + if ((nclistlength(var->filters) > 0 || var->shuffle || var->fletcher32) && par_access == NC_INDEPENDENT) return NC_EINVAL; From 420fdf462555333df40803d3d7fe3ac097401be0 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Mon, 2 Mar 2020 11:45:41 -0700 Subject: [PATCH 5/7] fix memory allocation failure in hdf5var.c --- libhdf5/hdf5var.c | 17 ++++++++--------- ncdump/nccopy.c | 2 +- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/libhdf5/hdf5var.c b/libhdf5/hdf5var.c index e2bda5ee83..b5453f2c0c 100644 --- a/libhdf5/hdf5var.c +++ b/libhdf5/hdf5var.c @@ -562,17 +562,16 @@ NC4_def_var(int ncid, const char *name, nc_type xtype, int ndims, * variables which may be contiguous.) */ LOG((4, "allocating array of %d size_t to hold chunksizes for var %s", var->ndims, var->hdr.name)); - if (var->ndims) + if (var->ndims) { if (!(var->chunksizes = calloc(var->ndims, sizeof(size_t)))) BAIL(NC_ENOMEM); - - if ((retval = nc4_find_default_chunksizes2(grp, var))) - BAIL(retval); - - /* Is this a variable with a chunksize greater than the current - * cache size? */ - if ((retval = nc4_adjust_var_cache(grp, var))) - BAIL(retval); + if ((retval = nc4_find_default_chunksizes2(grp, var))) + BAIL(retval); + /* Is this a variable with a chunksize greater than the current + * cache size? */ + if ((retval = nc4_adjust_var_cache(grp, var))) + BAIL(retval); + } /* If the user names this variable the same as a dimension, but * doesn't use that dimension first in its list of dimension ids, diff --git a/ncdump/nccopy.c b/ncdump/nccopy.c index 4d858603eb..60ade0d3d1 100644 --- a/ncdump/nccopy.c +++ b/ncdump/nccopy.c @@ -950,7 +950,7 @@ copy_chunking(int igrp, int i_varid, int ogrp, int o_varid, int ndims, int inkin ovid.grpid = ogrp; ovid.varid = o_varid; if((stat=computeFQN(ovid,&ofqn))) goto done; - if(option_deflate_level >= 0 || hasfilterspecforvar(ofqn) != NULL) + if(option_deflate_level >= 0 || hasfilterspecforvar(ofqn)) ocontig = NC_CHUNKED; /* See about dim-specific chunking; does not override -c spec*/ From e82d33a9bb9c024ee1d043021025dc667fad9de6 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Mon, 2 Mar 2020 11:58:22 -0700 Subject: [PATCH 6/7] fix error in tst_h_vars.c --- h5_test/tst_h_vars.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/h5_test/tst_h_vars.c b/h5_test/tst_h_vars.c index c290809cad..1cfb4e26d3 100644 --- a/h5_test/tst_h_vars.c +++ b/h5_test/tst_h_vars.c @@ -414,7 +414,7 @@ main() #define SZIP_VAR_NAME "szip_var" #define SZIP_DIM1_LEN 32 { - int data[DIM1_LEN]; + int data[SZIP_DIM1_LEN]; hid_t plistid; hsize_t chunksize[NDIM1] = {SZIP_DIM1_LEN}; int options_mask = 32, pixels_per_block = 4; From 73537603e228a8f0f1d874c777a30754626ce73f Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Mon, 2 Mar 2020 15:10:54 -0700 Subject: [PATCH 7/7] Make scalar X filter return an error instead of ignoring it --- libhdf5/hdf5filter.c | 4 ++-- libhdf5/hdf5var.c | 4 ++-- nc_test4/tst_vars2.c | 4 ++-- ncdump/nccopy.c | 7 ++++--- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/libhdf5/hdf5filter.c b/libhdf5/hdf5filter.c index 40f0348652..74bc58d122 100644 --- a/libhdf5/hdf5filter.c +++ b/libhdf5/hdf5filter.c @@ -223,7 +223,7 @@ NC4_filter_actions(int ncid, int varid, int op, NC_Filterobject* args) /* If the HDF5 dataset has already been created, then it is too * late to set all the extra stuff. */ if (!(h5->flags & NC_INDEF)) return THROW(NC_EINDEFINE); - if (!var->ndims) return NC_NOERR; /* For scalars, ignore */ + if (!var->ndims) return NC_EINVAL; /* For scalars, complain */ if (var->created) return THROW(NC_ELATEDEF); /* Can't turn on parallel and szip before HDF5 1.10.2. */ @@ -315,7 +315,7 @@ NC4_filter_actions(int ncid, int varid, int op, NC_Filterobject* args) #endif /* USE_PARALLEL */ } break; case NCFILTER_INQ: { - if (!var->ndims) return THROW(NC_ENOFILTER); /* For scalars, fail */ + if (!var->ndims) return THROW(NC_EINVAL); /* For scalars, fail */ if(obj->sort != NC_FILTER_SORT_SPEC) return THROW(NC_EFILTER); idp = &obj->u.spec.filterid; nparamsp = &obj->u.spec.nparams; diff --git a/libhdf5/hdf5var.c b/libhdf5/hdf5var.c index b5453f2c0c..79ae6e6ca9 100644 --- a/libhdf5/hdf5var.c +++ b/libhdf5/hdf5var.c @@ -679,9 +679,9 @@ nc_def_var_extra(int ncid, int varid, int *shuffle, int *unused1, /* Cannot set filters of any sort on scalars */ if(var->ndims == 0) { if(shuffle && *shuffle) - return NC_NOERR; /* ignore */ + return NC_EINVAL; if(fletcher32 && *fletcher32) - return NC_NOERR; /* ignore */ + return NC_EINVAL; } /* Shuffle filter? */ diff --git a/nc_test4/tst_vars2.c b/nc_test4/tst_vars2.c index de429ac64b..5da5d793a5 100644 --- a/nc_test4/tst_vars2.c +++ b/nc_test4/tst_vars2.c @@ -1478,8 +1478,8 @@ main(int argc, char **argv) if (shuffle_in || deflate_in) ERR; if (nc_inq_var_deflate(ncid, varid, NULL, NULL, NULL)) ERR; - /* Deflate is ignored for scalar. */ - if (nc_def_var_deflate(ncid, varid_scalar, 0, 1, 4)) ERR; + /* Deflate fails for scalar. */ + if (nc_def_var_deflate(ncid, varid_scalar, 0, 1, 4) != NC_EINVAL) ERR; if (nc_inq_var_deflate(ncid, varid, &shuffle_in, &deflate_in, &deflate_level_in)) ERR; if (shuffle_in || deflate_in) ERR; diff --git a/ncdump/nccopy.c b/ncdump/nccopy.c index 60ade0d3d1..48dd3648ab 100644 --- a/ncdump/nccopy.c +++ b/ncdump/nccopy.c @@ -1069,18 +1069,19 @@ copy_var_specials(int igrp, int varid, int ogrp, int o_varid, int inkind, int ou int innc4 = (inkind == NC_FORMAT_NETCDF4 || inkind == NC_FORMAT_NETCDF4_CLASSIC); int outnc4 = (outkind == NC_FORMAT_NETCDF4 || outkind == NC_FORMAT_NETCDF4_CLASSIC); int deflated = 0; /* true iff deflation is applied */ + int ndims; if(!outnc4) return stat; /* Ignore non-netcdf4 files */ { /* handle chunking parameters */ - int ndims; NC_CHECK(nc_inq_varndims(igrp, varid, &ndims)); if (ndims > 0) { /* no chunking for scalar variables */ NC_CHECK(copy_chunking(igrp, varid, ogrp, o_varid, ndims, inkind, outkind)); } } + if(ndims > 0) { /* handle compression parameters, copying from input, overriding * with command-line options */ int shuffle_in=0, deflate_in=0, deflate_level_in=0; @@ -1112,7 +1113,7 @@ copy_var_specials(int igrp, int varid, int ogrp, int o_varid, int inkind, int ou } } - if(innc4 && outnc4) + if(innc4 && outnc4 && ndims > 0) { /* handle checksum parameters */ int fletcher32 = 0; NC_CHECK(nc_inq_var_fletcher32(igrp, varid, &fletcher32)); @@ -1130,7 +1131,7 @@ copy_var_specials(int igrp, int varid, int ogrp, int o_varid, int inkind, int ou } } - if(!deflated) { + if(!deflated && ndims > 0) { /* handle other general filters */ NC_CHECK(copy_var_filter(igrp, varid, ogrp, o_varid, inkind, outkind)); }