Merge branch 'ncgenkw.dmh' of https://github.com/DennisHeimbigner/net…

…cdf-c into gh1753.wif

ncdump/CMakeLists.txt

@@ -307,11 +307,18 @@ ENDIF(MSVC)
 
   add_sh_test(ncdump tst_ctests)
 
+<<<<<<< HEAD
   IF(ENABLE_UNFIXED_MEMORY_LEAKS)
   SET_TESTS_PROPERTIES(ncdump_run_ncgen_nc4_tests.sh ncdump_tst_nccopy4.sh ncdump_tst_ncgen_shared.sh ncdump_tst_netcdf4.sh
                        PROPERTIES ENVIRONMENT NC_VLEN_NOTEST=1)
   ENDIF()
 
+=======
+  IF(USE_CDF5)
+  add_sh_test(ncdump test_keywords)
+  ENDIF()
+
+>>>>>>> 90b912b7e8675f857b618a440610ada3327f4190
 ENDIF()
 
 ENDIF()

ncdump/Makefile.am

@@ -4,7 +4,6 @@
 
 # Ed Hartnett, Dennis Heimbigner, Ward Fisher
 
-
 #SH_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
 #sh_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
 #LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
@@ -130,6 +129,11 @@ if USE_HDF5
 TESTS += tst_ctests.sh
 endif
 
+if ENABLE_CDF5
+# Test for keywords as identifiers
+TESTS += test_keywords.sh
+endif
+
 endif BUILD_TESTSETS
 
 # These files all have to be included with the distribution.
@@ -169,7 +173,9 @@ ref_null_byte_padding_test.nc ref_tst_irish_rover.nc			\
 ref_provenance_v1.nc ref_tst_radix.cdl tst_radix.cdl test_radix.sh	\
 ref_nccopy_w.cdl tst_nccopy_w3.sh tst_nccopy_w4.sh			\
 ref_no_ncproperty.nc test_unicode_directory.sh				\
-ref_roman_szip_simple.cdl ref_roman_szip_unlim.cdl ref_tst_perdimspecs.cdl
+ref_roman_szip_simple.cdl ref_roman_szip_unlim.cdl ref_tst_perdimspecs.cdl \
+test_keywords.sh ref_keyword1.cdl ref_keyword2.cdl
+
 
 # The L512.bin file is file containing exactly 512 bytes each of value 0.
 # It is used for creating hdf5 files with varying offsets for testing.
@@ -205,4 +211,5 @@ tst_ncf199.cdl tst_tst_gattenum.cdl tst_tst_usuffix.cdl ctest.c		\
 ctest64.c nccopy3_subset_out.nc camrun.c tst_ncf213.cdl tst_ncf213.nc	\
 tst_radix.nc tmp_radix.cdl ctest_small_3.c ctest_small_4.c		\
 ctest_special_atts_4.c tst_roman_szip_simple.cdl			\
-tst_roman_szip_unlim.cdl tst_perdimpspecs.nc tmppds.*
+tst_roman_szip_unlim.cdl tst_perdimpspecs.nc tmppds.* \
+keyword1.nc keyword2.nc tmp_keyword1.cdl tmp_keyword2.cdl

ncdump/nccopy.c

@@ -34,6 +34,7 @@
 #include "ncpathmgr.h"
 
 #undef DEBUGFILTER
+#undef DEBUGCHUNK
 
 /* default bytes of memory we are willing to allocate for variable
  * values during copy */
@@ -307,7 +308,7 @@ parsefilterspec(const char* optarg0, List* speclist)
     }
     /* Parse the variable list */
     if((vlist = listnew()) == NULL) {stat = NC_ENOMEM; goto done;}
-    if((stat=parsevarlist(optarg,vlist))) goto done;        
+    if((stat=parsevarlist(optarg,vlist))) goto done;
 
     if(strcasecmp(remainder,"none") != 0) {
         /* Collect the id+parameters */
@@ -324,7 +325,7 @@ parsefilterspec(const char* optarg0, List* speclist)
 	    filters[0] = nilspec; nilspec = NULL;
 	}
     }
-    
+
     /* Construct a spec entry for each element in vlist */
     for(i=0;i<listlength(vlist);i++) {
 	int k;
@@ -358,7 +359,7 @@ parsefilterspec(const char* optarg0, List* speclist)
 	    filtopt = NULL;
 	}
     }
-    
+
 done:
     freefilterlist(nfilters,filters);
     if(vlist) listfreeall(vlist);
@@ -369,7 +370,7 @@ parsefilterspec(const char* optarg0, List* speclist)
 static int
 hasfilteroptforvar(const char* ofqn)
 {
-  int i;	 
+  int i;
   /* See which output filter options are defined for this output variable */
   for(i=0;i<listlength(filteroptions);i++) {
       struct FilterOption* opt = listget(filteroptions,i);
@@ -382,7 +383,7 @@ hasfilteroptforvar(const char* ofqn)
 static List*
 filteroptsforvar(const char* ofqn)
 {
-  int i;	 
+  int i;
   List* list = listnew();
   /* See which output filter options are defined for this output variable */
   for(i=0;i<listlength(filteroptions);i++) {
@@ -798,15 +799,15 @@ copy_var_filter(int igrp, int varid, int ogrp, int o_varid, int inkind, int outk
     /* Only bother to look if input is netcdf-4 variant */
     if(innc4) {
       size_t nfilters;
-      unsigned int* ids = NULL;      
+      unsigned int* ids = NULL;
       int k;
       if((stat = nc_inq_var_filter_ids(vid.grpid,vid.varid,&nfilters,NULL)))
 	goto done;
       if(nfilters > 0) ids = (unsigned int*)calloc(nfilters,sizeof(unsigned int));
       if((stat = nc_inq_var_filter_ids(vid.grpid,vid.varid,&nfilters,ids)))
 	goto done;
       memset(&inspec,0,sizeof(inspec));
-      
+
       for(k=0;k<nfilters;k++) {
 	  inspec.pfs.filterid = ids[k];
           stat=nc_inq_var_filter_info(vid.grpid,vid.varid,inspec.pfs.filterid,&inspec.pfs.nparams,NULL);
@@ -926,7 +927,7 @@ copy_chunking(int igrp, int i_varid, int ogrp, int o_varid, int ndims, int inkin
 	    /* pretend that this is same as a -c option */
     } else { /* !innc4 */
 	icontig = NC_CONTIGUOUS;
-	ichunkp[0] = 0;	    	
+	ichunkp[0] = 0;
     }
 
     /* If var specific chunking was specified for this output variable
@@ -1009,7 +1010,7 @@ copy_chunking(int igrp, int i_varid, int ogrp, int o_varid, int ndims, int inkin
 	}
 
         /* Get the current default chunking on the output variable */
-        /* Unfortunately, there is no way to get this info except by 
+        /* Unfortunately, there is no way to get this info except by
            forcing chunking */
         if(ocontig == NC_CHUNKED) {
 	    /* this may fail if chunking is not possible, in which case ignore */
@@ -1082,7 +1083,7 @@ copy_chunking(int igrp, int i_varid, int ogrp, int o_varid, int ndims, int inkin
     }
 #endif /*DEBUGFILTER*/
 #endif /*USE_NETCDF4*/
- 
+
 done:
     if(ofqn) free(ofqn);
     return stat;
@@ -1487,23 +1488,23 @@ copy_vars(int igrp, int ogrp)
     return stat;
 }
 
-#if 0
+#if DEBUGCHUNK
 static void
 report(int rank, size_t* start, size_t* count, void* buf)
 {
     int i;
-    size_t prod = 1;    
+    size_t prod = 1;
     for(i=0;i<rank;i++) prod *= count[i];
     fprintf(stderr,"start=");
-    for(i=0;i<rank;i++)	
+    for(i=0;i<rank;i++)
 	fprintf(stderr,"%s%ld",(i==0?"(":" "),(long)start[i]);
     fprintf(stderr,")");
     fprintf(stderr," count=");
-    for(i=0;i<rank;i++)	
+    for(i=0;i<rank;i++)
 	fprintf(stderr,"%s%ld",(i==0?"(":" "),(long)count[i]);
     fprintf(stderr,")");
     fprintf(stderr," data=");
-    for(i=0;i<prod;i++)	
+    for(i=0;i<prod;i++)
 	fprintf(stderr,"%s%d",(i==0?"(":" "),((int*)buf)[i]);
     fprintf(stderr,"\n");
     fflush(stderr);
@@ -1663,6 +1664,9 @@ copy_var_data(int igrp, int varid, int ogrp)
      * subsequent calls. */
     while((ntoget = nc_next_iter(iterp, start, count)) > 0) {
 	NC_CHECK(nc_get_vara(igrp, varid, start, count, buf));
+#ifdef DEBUGCHUNK
+	report(iterp->rank,start,count,buf);
+#endif
 	NC_CHECK(nc_put_vara(ogrp, ovarid, start, count, buf));
 #ifdef USE_NETCDF4
 	/* we have to explicitly free values for strings and vlens */

ncdump/ref_keyword1.cdl

@@ -0,0 +1,8 @@
+netcdf keyword1 {
+dimensions:
+    string = 128;
+    int64 = 64;
+variables:
+    int string(string);
+    int int64(int64);
+}

ncdump/ref_keyword2.cdl

@@ -0,0 +1,6 @@
+netcdf keyword2 {
+dimensions:
+string = 128;
+variables:
+int string(string);
+}

ncdump/test_keywords.sh

@@ -0,0 +1,23 @@
+#!/bin/sh
+
+if test "x$srcdir" = x ; then srcdir=`pwd`; fi 
+. ../test_common.sh
+
+set -e
+
+echo "*** Test use of keywords for formats where the keyword is not defined"
+echo "*** classic: creating keyword1.nc from ref_keyword1.cdl..."
+${NCGEN} -3 -lb -o keyword1.nc $srcdir/ref_keyword1.cdl
+echo "*** creating tmp_keyword1.cdl from keyword1.nc..."
+${NCDUMP} -h keyword1.nc > tmp_keyword1.cdl
+echo "*** comparing tmp_keyword1.cdl to ref_keyword1.cdl..."
+diff -b -w tmp_keyword1.cdl $srcdir/ref_keyword1.cdl
+
+echo "*** cdf5: creating keyword2.nc from ref_keyword2.cdl..."
+${NCGEN} -5 -lb -o keyword2.nc $srcdir/ref_keyword2.cdl
+echo "*** creating tmp_keyword2.cdl from keyword2.nc..."
+${NCDUMP} -h keyword2.nc > tmp_keyword2.cdl
+echo "*** comparing tmp_keyword2.cdl to ref_keyword2.cdl..."
+diff -b -w tmp_keyword2.cdl $srcdir/ref_keyword2.cdl
+
+exit 0

ncgen/ncgen.l

@@ -140,6 +140,35 @@ struct Specialtoken specials[] = {
 {NULL,0} /* null terminate */
 };
 
+/* Track keywords that may be identifiers depending on
+   format being produced */
+/* Define the possible format classes */
+#define KWALL (1<<NC_FORMAT_CLASSIC|1<<NC_FORMAT_64BIT_OFFSET|1<<NC_FORMAT_NETCDF4|1<<NC_FORMAT_NETCDF4_CLASSIC|1<<NC_FORMAT_64BIT_DATA) /* Used in all formats */
+#define KWCDF5 (1<<NC_FORMAT_64BIT_DATA) /* Used in cdf5 */
+#define KWNC4 (1<<NC_FORMAT_NETCDF4) /* Used in netcdf-4 */
+
+#define NKWIDENT 12
+struct KWIDENT {
+    int token;
+    const char* keyword;
+    int formats; /* Which formats use this keyword */
+} kwident[NKWIDENT] = {
+/* Order by token for binary search */
+{CHAR_K, "char", KWALL},
+{BYTE_K, "byte", KWALL},
+{SHORT_K, "short", KWALL},
+{INT_K, "int", KWALL},
+{FLOAT_K, "float", KWALL},
+{DOUBLE_K, "double", KWALL},
+{UBYTE_K, "ubyte", KWCDF5|KWNC4},
+{USHORT_K, "ushort", KWCDF5|KWNC4},
+{UINT_K, "uint", KWCDF5|KWNC4},
+{INT64_K, "int64", KWCDF5|KWNC4},
+{UINT64_K, "uint64", KWCDF5|KWNC4},
+{STRING_K, "string", KWNC4}
+};
+static int identorkw(int token);
+
 %}
 %x ST_C_COMMENT
 %x TEXT
@@ -261,26 +290,27 @@ yytext[MAXTRST-1] = '\0';
 		return lexdebug(OPAQUESTRING);
 		}
 
-compound|struct|structure {return lexdebug(COMPOUND);}
-enum				{return lexdebug(ENUM);}
+compound|struct|structure {return lexdebug(identorkw(COMPOUND));}
+enum				{return lexdebug(identorkw(ENUM));}
 opaque				{return lexdebug(OPAQUE_);}
 
-float|real		{return lexdebug(FLOAT_K);}
-char			{return lexdebug(CHAR_K);}
-byte			{return lexdebug(BYTE_K);}
-ubyte			{return lexdebug(UBYTE_K);}
-short			{return lexdebug(SHORT_K);}
-ushort			{return lexdebug(USHORT_K);}
-long|int|integer	{return lexdebug(INT_K);}
-ulong|uint|uinteger	{return lexdebug(UINT_K);}
-int64			{return lexdebug(INT64_K);}
-uint64			{return lexdebug(UINT64_K);}
-double			{return lexdebug(DOUBLE_K);}
-string                  {return lexdebug(STRING_K);}
+float|real		{return lexdebug(identorkw(FLOAT_K));}
+char			{return lexdebug(identorkw(CHAR_K));}
+byte			{return lexdebug(identorkw(BYTE_K));}
+ubyte			{return lexdebug(identorkw(UBYTE_K));}
+short			{return lexdebug(identorkw(SHORT_K));}
+ushort			{return lexdebug(identorkw(USHORT_K));}
+long|int|integer	{return lexdebug(identorkw(INT_K));}
+ulong|uint|uinteger	{return lexdebug(identorkw(UINT_K));}
+int64			{return lexdebug(identorkw(INT64_K));}
+uint64			{return lexdebug(identorkw(UINT64_K));}
+double			{return lexdebug(identorkw(DOUBLE_K));}
+string                  {return lexdebug(identorkw(STRING_K));}
 
 unlimited|UNLIMITED	{int32_val = -1;
-			 return lexdebug(NC_UNLIMITED_K);}
+			 return lexdebug(identorkw(NC_UNLIMITED_K));}
 
+			/* These are currently only keywords */
 types:			{return lexdebug(TYPES);}
 dimensions:		{return lexdebug(DIMENSIONS);}
 variables:		{return lexdebug(VARIABLES);}
@@ -876,3 +906,39 @@ collecttag(char* text, char** stagp)
     }
     return tag;
 }
+
+/* Depending on the format, a name may be a keword or an ident */
+static int
+identorkw(int token)
+{
+    /* Binary search for yytext */
+    int n = NKWIDENT;
+    int L = 0;
+    int R = (n - 1);
+    int m, cmp;
+    struct KWIDENT* p;
+    int found = 0;
+    size_t len;
+    char* id = NULL;
+
+    for(;;) {
+	if(L > R) break;
+        m = (L + R) / 2;
+	p = &kwident[m];
+	cmp = (p->token - token);
+	if(cmp == 0) {found = 1; break;}
+	if(cmp < 0)
+	    L = (m + 1);
+	else /*cmp > 0*/
+	    R = (m - 1);
+    }
+    if(!found) return token; /* Not a keyword of interest */
+    /* See if the format applies */
+    if(p->formats & ((int)1<<k_flag)) return token;
+    /* Need to convert a non-ident token to an ident symbol */
+    len = strlen(yytext);
+    len = unescape(yytext,len,ISIDENT,&id);
+    yylval.sym = install(id);
+    efree(id);
+    return IDENT; /* treat as identifier */
+}