TDSv71 and other fixes

.gitignore

@@ -99,10 +99,6 @@ missing
 Output.osm
 /output.osm
 /output.pgm
-translations/*_schema.js
-translations/etds*_rules.js
-translations/emgcp*_rules.js
-translations/eggdm*_rules.js
 .qmake.stash
 translations-local
 **/*.pro.user*

Makefile.am

@@ -65,6 +65,7 @@ help:
 	@echo "docs                    - Build all project documentation."
 	@echo "dist                    - Make a distributable tarball."
 	@echo "eclipse                 - If configured '--with-services', then build Eclipse IDE project files."
+	@echo "js-schema               - Build the Javascript schema files."
 	@echo "osmapidb                - If configured '--with-services', then build a clean copy of an OSM API database."
 	@echo "translations-test       - Run the mocha translations tests. Should take less than five minutes."
 	@echo "services-build          - If configured '--with-services', then build the web services."
@@ -123,6 +124,9 @@ ui-clean-coverage:
 ui2x-build:
 	scripts/HootEnv.sh $(MAKE) -f Makefile.hoot ui2x-build
 
+js-schema:
+	scripts/HootEnv.sh $(MAKE) -f Makefile.hoot js-make
+
 translations-test:
 	scripts/HootEnv.sh $(MAKE) -f Makefile.hoot translations-test
 

Makefile.hoot

@@ -11,9 +11,9 @@ SHELL=/bin/bash
 
 .PHONY: clean-ccache
 
-# If you're adding another schema file be sure and update the archive target too.
-# NOTE: This is the list that gets deleted in "make clean"
-SCHEMA_FILES=translations/tds61_schema.js translations/tds61_full_schema.js translations/etds61_rules.js translations/etds61_osm_rules.js translations/tds40_schema.js translations/tds40_full_schema.js translations/etds40_rules.js translations/etds40_osm_rules.js translations/mgcp_schema.js translations/emgcp_rules.js translations/emgcp_osm_rules.js translations/ggdm30_schema.js translations/ggdm30_full_schema.js translations/eggdm30_rules.js translations/eggdm30_osm_rules.js translations/tds70_schema.js translations/tds70_full_schema.js translations/etds70_rules.js translations/etds70_osm_rules.js
+# This list is for manually rebuilding the schema files.  The files have been commited to the main GitHub repo
+SCHEMA_FILES= translations/tds71_schema.js translations/tds70_schema.js translations/tds61_schema.js translations/tds40_schema.js translations/mgcp_schema.js translations/ggdm30_schema.js translations/tds71_thematic_schema.js translations/tds71_full_schema.js translations/tds61_full_schema.js translations/tds70_full_schema.js translations/tds40_full_schema.js translations/ggdm30_full_schema.js translations/etds71_rules.js translations/etds70_rules.js translations/etds61_rules.js translations/etds40_rules.js translations/emgcp_rules.js translations/eggdm30_rules.js translations/etds71_osm_rules.js translations/etds61_osm_rules.js translations/etds40_osm_rules.js translations/emgcp_osm_rules.js translations/eggdm30_osm_rules.js translations/etds70_osm_rules.js
+
 
 -include Makefile.inc
 
@@ -54,7 +54,6 @@ clean: Makefile.qmake ui-clean services-clean clean-coverage
 	rm -f conf/core/ConfigOptions.json
 	rm -f conf/services/conflationTypeGroups.json
 	rm -f translations/*.pyc
-	rm -f $(SCHEMA_FILES)
 	rm -f hoot-core/src/main/cpp/hoot/core/HootConfig.h
 	# The following bits are due to moving files in git. Since the Output.osm
 	# file is written to the test-files/cases/**/ directory git won't remove
@@ -177,7 +176,34 @@ translations/emgcp_rules.js: scripts/schema/ConvertMGCPSchema_XML.py conf/transl
 translations/emgcp_osm_rules.js: scripts/schema/ConvertMGCPSchema_XML.py conf/translations/MGCP_FeatureCatalogue_TRD4_v4.6_20210129.xml.gz
 	mkdir -p $(@D)
 	$< --fromenglish $(word 2,$^) > $@ || (rm -f $@ ; exit -1)
-#	$< --fromenglish $(word 2,$^) $(word 3,$^) > $@ || (rm -f $@ ; exit -1)
+
+#### TDSv71
+
+# Build the TDS71 "From English" rules
+translations/etds71_osm_rules.js: scripts/schema/ConvertTDSv71Schema_XML.py conf/translations/TDSv71.xml.gz
+	mkdir -p $(@D)
+	$< --fromenglish $(word 2,$^) > $@ || (rm -f $@ ; exit -1)
+
+# Build the TDS71 "To English" rules
+translations/etds71_rules.js: scripts/schema/ConvertTDSv71Schema_XML.py conf/translations/TDSv71.xml.gz
+	mkdir -p $(@D)
+	$< --toenglish $(word 2,$^) > $@ || (rm -f $@ ; exit -1)
+
+# TDSv71 full schema is built from an XML file made from a sample FGDB
+# This one has all of the Text Enumerations
+translations/tds71_full_schema.js: scripts/schema/ConvertTDSv71Schema_XML.py conf/translations/TDSv71.xml.gz
+	mkdir -p $(@D)
+	$< --fullschema $(word 2,$^) > $@ || (rm -f $@ ; exit -1)
+
+# TDSv71 thematic schema is built from an XML file made from a sample FGDB
+translations/tds71_thematic_schema.js: scripts/schema/ConvertTDSv71Schema_XML.py conf/translations/TDSv71.xml.gz
+	mkdir -p $(@D)
+	$< --thematic  $(word 2,$^) > $@ || (rm -f $@ ; exit -1)
+
+# TDSv71 schema is built from an XML file made from a sample FGDB
+translations/tds71_schema.js: scripts/schema/ConvertTDSv71Schema_XML.py conf/translations/TDSv71.xml.gz
+	mkdir -p $(@D)
+	$^ > $@ || (rm -f $@ ; exit -1)
 
 #### TDSv40
 translations/tds40_schema.js: scripts/schema/ConvertTDSv40Schema.py conf/translations/TDSv40.csv.gz
@@ -198,7 +224,6 @@ translations/etds40_rules.js: scripts/schema/ConvertTDSv40Schema.py conf/transla
 translations/etds40_osm_rules.js: scripts/schema/ConvertTDSv40Schema.py conf/translations/TDSv40.csv.gz
 	mkdir -p $(@D)
 	$< --fromenglish $(word 2,$^) > $@ || (rm -f $@ ; exit -1)
-#	$< --fromenglish $(word 2,$^) $(word 3,$^) > $@ || (rm -f $@ ; exit -1)
 
 #### TDSv61
 # TDSv61 is built from the TDSv60 spec with NGA's extensions
@@ -433,32 +458,7 @@ archive: js-make services-docs docs
 	# Copy the protocol buffer generated files. Eases a dependency when building
 	# on CentOS
 	mkdir -p $(HOOT_TARNAME)/hoot-core/src/main/cpp/hoot/core/proto/
-	# Copy the auto-generated schema definitions. This eases another dependency on CentOS
-	mkdir -p $(HOOT_TARNAME)/translations/
-	# TDSv40
-	cp translations/tds40_schema.js $(HOOT_TARNAME)/translations/tds40_schema.js
-	cp translations/tds40_full_schema.js $(HOOT_TARNAME)/translations/tds40_full_schema.js
-	cp translations/etds40_rules.js $(HOOT_TARNAME)/translations/etds40_osm_rules.js
-	cp translations/etds40_osm_rules.js $(HOOT_TARNAME)/translations/etds40_osm_rules.js
-	# TDSv61
-	cp translations/tds61_schema.js $(HOOT_TARNAME)/translations/tds61_schema.js
-	cp translations/tds61_full_schema.js $(HOOT_TARNAME)/translations/tds61_full_schema.js
-	cp translations/etds61_rules.js $(HOOT_TARNAME)/translations/etds61_rules.js
-	cp translations/etds61_osm_rules.js $(HOOT_TARNAME)/translations/etds61_osm_rules.js
-	# TDSv70
-	cp translations/tds70_schema.js $(HOOT_TARNAME)/translations/tds70_schema.js
-	cp translations/tds70_full_schema.js $(HOOT_TARNAME)/translations/tds70_full_schema.js
-	cp translations/etds70_rules.js $(HOOT_TARNAME)/translations/etds70_rules.js
-	cp translations/etds70_osm_rules.js $(HOOT_TARNAME)/translations/etds70_osm_rules.js
-	# MGCP
-	cp translations/mgcp_schema.js $(HOOT_TARNAME)/translations/mgcp_schema.js
-	cp translations/emgcp_rules.js $(HOOT_TARNAME)/translations/emgcp_rules.js
-	cp translations/emgcp_osm_rules.js $(HOOT_TARNAME)/translations/emgcp_osm_rules.js
-	# GGDMv30
-	cp translations/ggdm30_schema.js $(HOOT_TARNAME)/translations/ggdm30_schema.js
-	cp translations/ggdm30_full_schema.js $(HOOT_TARNAME)/translations/ggdm30_full_schema.js
-	cp translations/eggdm30_rules.js $(HOOT_TARNAME)/translations/eggdm30_osm_rules.js
-	cp translations/eggdm30_osm_rules.js $(HOOT_TARNAME)/translations/eggdm30_osm_rules.js
+	# NOTE: The autogenerated schema files are now in the main repo
 	# Create the version file
 	echo $(HOOT_VERSION) > $(HOOT_TARNAME)/version
 	# tar the file and zip it

conf/services/DefaultTranslations.json

@@ -1,4 +1,10 @@
 [
+    {
+        "name" : "TDSv71",
+        "description" : "NSG Topographic Data Store (TDS) v7.1",
+        "canExport" : true,
+        "path" : "translations/TDSv71.js"
+    },
     {
         "name" : "TDSv61",
         "description" : "NSG Topographic Data Store (TDS) v6.1",

conf/translations/TDS_NGA_README.txt

@@ -1,6 +1,5 @@
 This file lists changes to the published TDS specs.
 
-
 TDSv6.1
 NGA added to the TDSv60 specification to make TDSv61.
 The NGA additions include attributes and changes that are not included in the ESRI TDSv61 template FGDB.
@@ -19,3 +18,6 @@ TDSv70
 21 Aug 20: Added attributes to each feature to store image metadata.
 Dec 20: Added attributes to "not extracted" fcodes.
 
+TDSv71
+22 Feb 22: Added 'OTH' to carto features and also image metadata attributes
+

node-export-server/config.json

@@ -20,6 +20,7 @@
     "TDSv40": "translations/TDSv40.js",
     "TDSv61": "translations/TDSv61.js",
     "TDSv70": "translations/TDSv70.js",
+    "TDSv71": "translations/TDSv71.js",
     "MGCP": "translations/MGCP_TRD4.js",
     "DNC": "translations/DNC.js",
     "ENC": "translations/ENCv311_to_OGR.js",

scripts/ci/archive.sh

@@ -30,6 +30,10 @@ automake --add-missing --copy
 
 # Make the archive.
 make -j"$(nproc)" clean
+
+# Make the generated schema BEFORE trying to build everything.
+# Some of the python scripts get killed due to lack of resources.
+make -j$(( `nproc` > 4 ? 4 : `nproc` )) js-schema
 make -j"$(nproc)" archive
 
 # All done.

scripts/schema/ConvertGGDMv30Schema.py

@@ -48,98 +48,91 @@
 # Note: This uses double quotes ( " ) around data elements in the output.  The csv files have values with
 # single quotes ( ' ) in them.  These quotes are also in the DFDD and NFDD specs.
 def printJavascript(schema,withDefs):
-    print '    var schema = [' # And so it begins...
+    print 'var schema = [' # And so it begins...
 
     num_feat = len(schema.keys()) # How many features in the schema?
     for f in sorted(schema.keys()):
         # Skip all of the 'Table' features
         if schema[f]['geom'] == 'Table':
             continue
 
-        print '        { name:"%s",' % (f); # name = geom + FCODE
-        print '          fcode:"%s",' % (schema[f]['fcode'])
-        print '          desc:"%s",' % (schema[f]['desc'])
+        pString = ' {name:"%s",fcode:"%s",desc:"%s",geom:"%s",' % (f,schema[f]['fcode'],schema[f]['desc'],schema[f]['geom']);
         if withDefs and 'definition' in schema[f]:
-            print '          definition:"%s",' % (schema[f]['definition'])
-        print '          geom:"%s",' % (schema[f]['geom'])
-        print '          columns:['
+            pString += 'definition:"%s",' % (schema[f]['definition'])
+
+        print pString
+        print '  columns:['
 
         num_attrib = len(schema[f]['columns'].keys()) # How many attributes does the feature have?
         for k in sorted(schema[f]['columns'].keys()):
-            print '                     { name:"%s",' % (k)
-            print '                       desc:"%s",' % (schema[f]['columns'][k]['desc'])
-            print '                       optional:"%s",' % (schema[f]['columns'][k]['optional'])
-
-            if withDefs and 'definition' in schema[f]['columns'][k]:
-                print '                       definition:"%s",' % (schema[f]['columns'][k]['definition'])
-
+            aString = '   {name:"%s",desc:"%s",optional:"%s",' % (k,schema[f]['columns'][k]['desc'],schema[f]['columns'][k]['optional'])
 
             if 'length' in schema[f]['columns'][k]:
-                print '                       length:"%s",' % (schema[f]['columns'][k]['length'])
+                aString += 'length:"%s",' % (schema[f]['columns'][k]['length'])
 
             if 'units' in schema[f]['columns'][k]:
-                print '                       units:"%s",' % (schema[f]['columns'][k]['units'])
+                aString += 'units:"%s",' % (schema[f]['columns'][k]['units'])
+
+            if withDefs and 'definition' in schema[f]['columns'][k]:
+                aString += 'definition:"%s",' % (schema[f]['columns'][k]['definition'])
 
             if 'func' in schema[f]['columns'][k]:
-                print '                       type:"enumeration",'
-                print '                       defValue:"%s",' % (schema[f]['columns'][k]['defValue'])
-                print '                       enumerations: %s' % (schema[f]['columns'][k]['func'])
+                aString += 'type:"enumeration",defValue:"%s",enumerations: %s}' % (schema[f]['columns'][k]['defValue'],schema[f]['columns'][k]['func'])
+                if num_attrib > 1:  # Are we at the last attribute? yes = no trailing comma
+                    aString += ','
+                    num_attrib -= 1
+                print aString
 
             elif schema[f]['columns'][k]['type'] == 'enumeration':
-                #print '                       type:"%s",' % (schema[f]['columns'][k]['type'])
-                print '                       type:"enumeration",'
-                print '                       defValue:"%s",' % (schema[f]['columns'][k]['defValue'])
-                print '                       enumerations:['
+                aString += 'type:"enumeration",defValue:"%s",' % (schema[f]['columns'][k]['defValue'])
+                print aString
+                print '    enumerations:['
+                num_enum = len(schema[f]['columns'][k]['enum']) # How many attributes does the feature have?
                 for l in schema[f]['columns'][k]['enum']:
-                    #print '                           { name:"%s", value:"%s" }, ' % (l['name'],l['value'])
-                    print '                           { value:"%s", name:"%s" }, ' % (l['value'],l['name'])
-                # print '                        ] // End of Enumerations '
-                print '                        ]'
-
-            else:
-                print '                       type:"%s",' % (schema[f]['columns'][k]['type'])
-                print '                       defValue:"%s" ' % (schema[f]['columns'][k]['defValue'])
-
-            if num_attrib == 1:  # Are we at the last attribute? yes = no trailing comma
-                # print '                     } // End of %s' % (k)
-                print '                     }'
+                    if num_enum == 1:
+                        print '     {name:"%s",value:"%s"}' % (l['name'],l['value'])
+                    else:
+                        print '     {name:"%s",value:"%s"},' % (l['name'],l['value'])
+                        num_enum -= 1
+                print '    ]'
+                if num_attrib == 1:  # Are we at the last attribute? yes = no trailing comma
+                    print '   }'
+                else:
+                    print '   },'
+                    num_attrib -= 1
             else:
-                # print '                     }, // End of %s' % (k)
-                print '                     },'
-                num_attrib -= 1
+                aString += 'type:"%s",defValue:"%s"}' % (schema[f]['columns'][k]['type'],schema[f]['columns'][k]['defValue'])
+                if num_attrib > 1:  # Are we at the last attribute? yes = no trailing comma
+                    aString += ','
+                    num_attrib -= 1
+                print aString
 
-        # print '                    ] // End of Columns'
-        print '                    ]'
+        print '  ]' # End of the attributes
 
         if num_feat == 1: # Are we at the last feature? yes = no trailing comma
-            # print '          } // End of feature %s\n' % (schema[f]['fcode'])
-            print '          }'
+            print ' }'
         else:
-            # print '          }, // End of feature %s\n' % (schema[f]['fcode'])
-            print '          },'
+            print ' },'
             num_feat -= 1
 
-    # print '    ]; // End of schema\n'
-    print '    ];'
+    print '];' # End of schema
 # End printJavascript
 
 
 # Print out a codelist as a JS variable
 def printFunctions(eList):
     for i in eList:
-        print '    var %s = [' % (i)
+        print 'var %s = [' % (i)
 
         num_vals = len(eList[i]['values'].keys()) # How many values does the thing have?
         for j in sorted(eList[i]['values'].keys(), key=asint):
             if num_vals == 1: # Are we at the last feature? yes = no trailing comma
-                #print '              { name:"%s", value:"%s" } ' % (eList[i]['values'][j],j)
-                print '              { value:"%s", name:"%s" } ' % (j,eList[i]['values'][j])
+                print ' {value:"%s",name:"%s"}' % (j,eList[i]['values'][j])
             else:
-                #print '              { name:"%s", value:"%s" }, ' % (eList[i]['values'][j],j)
-                print '              { value:"%s", name:"%s" }, ' % (j,eList[i]['values'][j])
+                print ' {value:"%s",name:"%s"},' % (j,eList[i]['values'][j])
                 num_vals -= 1
 
-        print '             ];'
+        print '];'
         print
 # End printFunctions