Develop develop-ref after #2217 (#2219)

Co-authored-by: John Halley Gotway <[email protected]>
Co-authored-by: Julie Prestopnik <[email protected]>
Co-authored-by: jprestop <[email protected]>
Co-authored-by: Howard Soh <[email protected]>
Co-authored-by: j-opatz <[email protected]>
Co-authored-by: Randy Bullock <[email protected]>
Co-authored-by: davidfillmore <[email protected]>
Co-authored-by: rgbullock <[email protected]>
Co-authored-by: John Halley Gotway <[email protected]>
Co-authored-by: johnhg <[email protected]>
Co-authored-by: Seth Linden <[email protected]>
Co-authored-by: George McCabe <[email protected]>
Co-authored-by: Seth Linden <[email protected]>
Co-authored-by: Howard Soh <[email protected]>
Co-authored-by: Seth Linden <[email protected]>
Co-authored-by: hsoh-u <[email protected]>
Co-authored-by: John Halley Gotway <[email protected]>
Co-authored-by: MET Tools Test Account <[email protected]>
Co-authored-by: mo-mglover <[email protected]>
Co-authored-by: davidalbo <[email protected]>
Co-authored-by: lisagoodrich <[email protected]>
Co-authored-by: Dan Adriaansen <[email protected]>
Co-authored-by: Dave Albo <[email protected]>
Co-authored-by: Molly Smith <[email protected]>
Co-authored-by: Jonathan Vigh <[email protected]>

data/config/IODA2NCConfig_default

@@ -88,8 +88,8 @@ obs_name_map = [];
 // Default mapping for Metadata.
 //
 metadata_map = [
-   { key = "message_type"; val = "msg_type"; },
-   { key = "station_id";   val = "report_identifier"; },
+   { key = "message_type"; val = "msg_type,station_ob"; },
+   { key = "station_id";   val = "station_id,report_identifier"; },
    { key = "pressure";     val = "air_pressure,pressure"; },
    { key = "height";       val = "height,height_above_mean_sea_level"; },
    { key = "elevation";    val = ""; }

docs/Users_Guide/reformat_point.rst

@@ -951,8 +951,8 @@ _____________________
 .. code-block:: none
 		
 		metadata_map = [
-		{ key = "message_type"; val = "msg_type"; },
-		{ key = "station_id";   val = "report_identifier"; },
+		{ key = "message_type"; val = "msg_type,station_ob"; },
+		{ key = "station_id";   val = "station_id,report_identifier"; },
 		{ key = "pressure";     val = "air_pressure,pressure"; },
 		{ key = "height";       val = "height,height_above_mean_sea_level"; },
 		{ key = "elevation";    val = ""; }

scripts/python/met_point_obs.py

@@ -5,7 +5,7 @@
 
 - This is the base class and the customized script should extend the met_point_obs.
 - The customized script (for example "custom_reader") must implement "def read_data(self, args)"
-  which fills the array (list) variables at __init__().
+  which fills the array (python list or numpy array) variables at __init__().
 - The args can be 1) single string argument, 2) the list of arguments, and 3) the dictionary of arguments.
 - The variable "met_point_data" must be set for MET tools
 - The customized script is expected to include following codes:
@@ -17,6 +17,7 @@
     met_point_data = point_obs_data.get_point_data()
 '''
 
+import os
 from abc import ABC, abstractmethod
 import numpy as np
 
@@ -28,52 +29,128 @@ class met_point_obs(ABC):
     '''
     classdocs
     '''
+    ERROR_P = " ==ERROR_PYTHON=="
+    INFO_P = " ==INFO_PYTHON=="
 
     python_prefix = 'PYTHON_POINT_RAW'
 
     def __init__(self, use_var_id=True):
         '''
         Constructor
         '''
+        self.input_name = None
         self.use_var_id = use_var_id    # True if variable index, False if GRIB code
 
         # Header
         self.nhdr = 0
         self.npbhdr = 0
         self.nhdr_typ = 0   # type table
-        self.nhdr_sid = 0   #station_uid table
+        self.nhdr_sid = 0   # station_id table
         self.nhdr_vld = 0   # valid time strings
-        self.hdr_typ = []   # (nhdr)
-        self.hdr_sid = []   # (nhdr)
-        self.hdr_vld = []   # (nhdr)
-        self.hdr_lat = []   # (nhdr)
-        self.hdr_lon = []   # (nhdr)
-        self.hdr_elv = []   # (nhdr)
-        self.hdr_typ_table = [] # (nhdr_typ, mxstr2)
-        self.hdr_sid_table = [] # (nhdr_sid, mxstr2)
-        self.hdr_vld_table = [] # (nhdr_vld, mxstr)
-        self.hdr_prpt_typ  = [] # optional
-        self.hdr_irpt_typ  = [] # optional
-        self.hdr_inst_typ  = [] # optional
+        self.hdr_typ = []   # (nhdr) integer
+        self.hdr_sid = []   # (nhdr) integer
+        self.hdr_vld = []   # (nhdr) integer
+        self.hdr_lat = []   # (nhdr) float
+        self.hdr_lon = []   # (nhdr) float
+        self.hdr_elv = []   # (nhdr) float
+        self.hdr_typ_table = [] # (nhdr_typ, mxstr2) string
+        self.hdr_sid_table = [] # (nhdr_sid, mxstr2) string
+        self.hdr_vld_table = [] # (nhdr_vld, mxstr) string
 
         #Observation data
         self.nobs = 0
         self.nobs_qty = 0
         self.nobs_var = 0
-        self.obs_qty = []  # (nobs_qty )
-        self.obs_hid = []  # (nobs)
-        self.obs_vid = []  # (nobs) veriable index or GRIB code
-        self.obs_lvl = []  # (nobs)
-        self.obs_hgt = []  # (nobs)
-        self.obs_val = []  # (nobs)
-        self.obs_qty_table = []  # (nobs_qty, mxstr)
-        self.obs_var_table = []  # (nobs_var, mxstr2) optional if GRIB code at self.obs_vid
+        self.obs_qty = []  # (nobs_qty) integer, index of self.obs_qty_table
+        self.obs_hid = []  # (nobs) integer
+        self.obs_vid = []  # (nobs) integer, veriable index from self.obs_var_table or GRIB code
+        self.obs_lvl = []  # (nobs) float
+        self.obs_hgt = []  # (nobs) float
+        self.obs_val = []  # (nobs) float
+        self.obs_qty_table = []  # (nobs_qty, mxstr) string
+        self.obs_var_table = []  # (nobs_var, mxstr2) string, required if self.use_var_id is True
+
+        # Optional variables for PREPBUFR, not supported yet
+        self.hdr_prpt_typ  = [] # optional
+        self.hdr_irpt_typ  = [] # optional
+        self.hdr_inst_typ  = [] # optional
 
     @abstractmethod
     def read_data(self, args):
-        # args should be list or dictionary
+        # args can be input_file_name, list, or dictionary
+        # - The variables at __init__ should be filled as python list or numpy array
+        # - set self.input_name
+        #
+        # Here is a template
+        '''
+        if isinstance(args, dict):
+            in_filename = args.get('in_name',None)
+        elif isinstance(args, list):
+            in_filename = args[0]
+        else:
+            in_filename = args
+        self.input_name = in_filename
+        '''
         pass
 
+    def check_data_member_float(self, local_var, var_name):
+        if 0 == len(local_var):
+            self.log_error("{v} is empty".format(v=var_name))
+        elif isinstance(local_var, list):
+            if 0 <= str(type(local_var[0])).find('numpy'):
+                self.log_info("Recommend using numpy instead of python list for {v} ({t}) to avoid side effect".format(
+                        v=var_name, t=type(local_var[0])))
+            elif not isinstance(local_var[0], (int, float)):
+                self.log_error("Not supported data type ({t}) for {v}[0] (int or float only)".format(
+                        v=var_name, t=local_var[0].type))
+        elif not isinstance(local_var, np.ndarray):
+            self.log_error("Not supported data type ({t}) for {v} (list and numpy.ndarray)".format(
+                    v=var_name, t=type(local_var)))
+
+    def check_data_member_int(self, local_var, var_name):
+        if 0 == len(local_var):
+            self.log_error("{v} is empty".format(v=var_name))
+        elif isinstance(local_var, list):
+            if 0 <= str(type(local_var[0])).find('numpy'):
+                self.log_info("Recommend using numpy instead of python list for {v} ({t}) to avoid side effect".format(
+                        v=var_name, t=type(local_var[0])))
+            elif not isinstance(local_var[0], int):
+                self.log_error("Not supported data type ({t}) for {v}[0] (int only)".format(
+                        v=var_name, t=type(local_var[0])))
+        elif not isinstance(local_var, np.ndarray):
+            self.log_error("Not supported data type ({t}) for {v} (list and numpy.ndarray)".format(
+                    v=var_name, t=type(local_var)))
+
+    def check_data_member_string(self, local_var, var_name):
+        if 0 == len(local_var):
+            self.log_error("{v} is empty".format(v=var_name))
+        elif not isinstance(local_var, (list)):
+            self.log_error("Not supported data type ({t}) for {v} (list)".format(
+                    v=var_name, t=type(local_var)))
+
+    def check_point_data(self):
+        if self.input_name is not None and not os.path.exists(self.input_name):
+            self.log_error('The netcdf input {f} does not exist'.format(f=self.input_name))
+        else:
+            self.check_data_member_int(self.hdr_typ,'hdr_typ')
+            self.check_data_member_int(self.hdr_sid,'hdr_sid')
+            self.check_data_member_int(self.hdr_vld,'hdr_vld')
+            self.check_data_member_float(self.hdr_lat,'hdr_lat')
+            self.check_data_member_float(self.hdr_lon,'hdr_lon')
+            self.check_data_member_float(self.hdr_elv,'hdr_elv')
+            self.check_data_member_string(self.hdr_typ_table,'hdr_typ_table')
+            self.check_data_member_string(self.hdr_sid_table,'hdr_sid_table')
+            self.check_data_member_string(self.hdr_vld_table,'hdr_vld_table')
+
+            self.check_data_member_int(self.obs_qty,'obs_qty')
+            self.check_data_member_int(self.obs_hid,'obs_hid')
+            self.check_data_member_int(self.obs_vid,'obs_vid')
+            self.check_data_member_float(self.obs_lvl,'obs_lvl')
+            self.check_data_member_float(self.obs_hgt,'obs_hgt')
+            self.check_data_member_float(self.obs_val,'obs_val')
+            self.check_data_member_string(self.obs_qty_table,'obs_qty_table')
+            self.check_data_member_string(self.obs_var_table,'obs_var_table')
+
     def get_point_data(self):
         if self.nhdr <= 0:
             self.nhdr = len(self.hdr_lat)
@@ -91,8 +168,18 @@ def get_point_data(self):
             self.nobs_qty = len(self.obs_qty_table)
         if self.nobs_var <= 0:
             self.nobs_var = len(self.obs_var_table)
+        self.check_point_data()
         return self.__dict__
 
+    def log_error(self, err_msgs):
+        print(self.ERROR_P)
+        for err_line in err_msgs.split('\n'):
+            print('{p} {m}'.format(p=self.ERROR_P, m=err_line))
+        print(self.ERROR_P)
+
+    def log_info(self, info_msg):
+        print('{p} {m}'.format(p=self.INFO_P, m=info_msg))
+
     @staticmethod
     def is_python_script(arg_value):
         return arg_value.startswith(met_point_obs.python_prefix)
@@ -120,31 +207,50 @@ def print_data(key, data_array, show_count=COUNT_SHOW):
     def print_point_data(met_point_data, print_subset=True):
         print(' === MET point data by python embedding ===')
         if print_subset:
-            met_point_obs.print_data('nhdr', met_point_data['nhdr'])
-            met_point_obs.print_data('nobs' , met_point_data['nobs'])
+            met_point_obs.print_data('nhdr',met_point_data['nhdr'])
+            met_point_obs.print_data('nobs',met_point_data['nobs'])
             met_point_obs.print_data('use_var_id',met_point_data['use_var_id'])
             met_point_obs.print_data('hdr_typ',met_point_data['hdr_typ'])
+            met_point_obs.print_data('hdr_typ_table',met_point_data['hdr_typ_table'])
+            met_point_obs.print_data('hdr_sid',met_point_data['hdr_sid'])
+            met_point_obs.print_data('hdr_sid_table',met_point_data['hdr_sid_table'])
+            met_point_obs.print_data('hdr_vld',met_point_data['hdr_vld'])
+            met_point_obs.print_data('hdr_vld_table',met_point_data['hdr_vld_table'])
+            met_point_obs.print_data('hdr_lat',met_point_data['hdr_lat'])
+            met_point_obs.print_data('hdr_lon',met_point_data['hdr_lon'])
+            met_point_obs.print_data('hdr_elv',met_point_data['hdr_elv'])
             met_point_obs.print_data('obs_hid',met_point_data['obs_hid'])
             met_point_obs.print_data('obs_vid',met_point_data['obs_vid'])
-            met_point_obs.print_data('obs_val',met_point_data['obs_val'])
             met_point_obs.print_data('obs_var_table',met_point_data['obs_var_table'])
+            met_point_obs.print_data('obs_qty',met_point_data['obs_qty'])
             met_point_obs.print_data('obs_qty_table',met_point_data['obs_qty_table'])
-            met_point_obs.print_data('hdr_typ_table',met_point_data['hdr_typ_table'])
-            met_point_obs.print_data('hdr_sid_table',met_point_data['hdr_sid_table'])
-            met_point_obs.print_data('hdr_vld_table',met_point_data['hdr_vld_table'])
+            met_point_obs.print_data('obs_lvl',met_point_data['obs_lvl'])
+            met_point_obs.print_data('obs_hgt',met_point_data['obs_hgt'])
+            met_point_obs.print_data('obs_val',met_point_data['obs_val'])
+
         else:
             print('All',met_point_data)
             print("      nhdr: ",met_point_data['nhdr'])
             print("      nobs: ",met_point_data['nobs'])
             print('use_var_id: ',met_point_data['use_var_id'])
             print('hdr_typ: ',met_point_data['hdr_typ'])
-            print('obs_vid: ',met_point_data['obs_vid'])
-            print('obs_val: ',met_point_data['obs_val'])
-            print('obs_var_table: ',met_point_data['obs_var_table'])
-            print('obs_qty_table: ',met_point_data['obs_qty_table'])
             print('hdr_typ_table: ',met_point_data['hdr_typ_table'])
+            print('hdr_sid: ',met_point_data['hdr_sid'])
             print('hdr_sid_table: ',met_point_data['hdr_sid_table'])
+            print('hdr_vld: ',met_point_data['hdr_vld'])
             print('hdr_vld_table: ',met_point_data['hdr_vld_table'])
+            print('hdr_lat: ',met_point_data['hdr_lat'])
+            print('hdr_lon: ',met_point_data['hdr_lon'])
+            print('hdr_elv: ',met_point_data['hdr_elv'])
+            print('obs_hid: ',met_point_data['obs_hid'])
+            print('obs_vid: ',met_point_data['obs_vid'])
+            print('obs_var_table: ',met_point_data['obs_var_table'])
+            print('obs_qty: ',met_point_data['obs_qty'])
+            print('obs_qty_table: ',met_point_data['obs_qty_table'])
+            print('obs_lvl: ',met_point_data['obs_lvl'])
+            print('obs_hgt: ',met_point_data['obs_hgt'])
+            print('obs_val: ',met_point_data['obs_val'])
+
         print(' === MET point data by python embedding ===')
 
 
@@ -160,9 +266,9 @@ def read_data(self, arg_map={}):
         self.hdr_lon = np.array([ -89., -89., -89., -89., -89., -89., -89., -89., -89., -92., -92., -92., -92., -89., -89., -89., -89., -89., -89., -89., -89., -89., -92., -92., -92., -92. ])
         self.hdr_elv = np.array([ 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220. ])
 
-        self.obs_qid = np.array([ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ])
         self.obs_hid = np.array([ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 25 ])
         self.obs_vid = np.array([ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ])
+        self.obs_qty = np.array([ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ])
         self.obs_lvl = np.array([ 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000. ])
         self.obs_hgt = np.array([ 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2. ])
         self.obs_val = np.array([ 292., 292.5, 293., 293.5, 294., 294.5, 295., 295.5, 296., 292., 293.4, 293., 296., 294., 92., 92.5, 93., 93.5, 94., 94.5, 95., 95.5, 96., 92., 93.4, 93., 96., 94. ])

src/basic/vx_config/dictionary.cc

@@ -1949,7 +1949,8 @@ if ( Entry )  {
 
       mlog << Warning
            << "\nDictionary::lookup_thresh() -> "
-           << "threshold lookup failed for name \"" << name << "\"\n\n";
+           << "threshold lookup failed for name \""
+           << name << "\"\n\n";
 
    }
 }
@@ -1962,7 +1963,9 @@ if ( !Entry || !is_correct_type )  {
 
       mlog << Error
            << "\nDictionary::lookup_thresh() -> "
-           << "lookup failed for name \"" << name << "\"\n\n";
+           << "threshold \"" << name
+           << "\" must start with a valid comparison operator "
+           << "(<, <=, ==, !=, >, >=, lt, le, eq, ne, gt, ge).\n\n";
 
       exit ( 1 );
 
@@ -2008,7 +2011,6 @@ if ( Entry )  {
    }
 }
 
-
 LastLookupStatus = is_correct_type;
 
 if ( !Entry || !is_correct_type )  {
@@ -2017,7 +2019,9 @@ if ( !Entry || !is_correct_type )  {
 
       mlog << Error
            << "\nDictionary::lookup_thresh_array() -> "
-           << "lookup failed for name \"" << name << "\"\n\n";
+           << "each threshold in array \"" << name
+           << "\" must start with a valid comparison operator "
+           << "(<, <=, ==, !=, >, >=, lt, le, eq, ne, gt, ge).\n\n";
 
       exit ( 1 );
 
@@ -2056,8 +2060,9 @@ if ( Dict->n_entries() > 0 )  {
 
       mlog << Error
            << "\nDictionary::lookup_thresh_array() -> "
-           << "array \"" << name
-           << "\" does not contain ThreshType.\n\n";
+           << "each threshold in array \"" << name
+           << "\" must start with a valid comparison operator "
+           << "(<, <=, ==, !=, >, >=, lt, le, eq, ne, gt, ge).\n\n";
 
       exit ( 1 );
 

src/basic/vx_log/Makefile.am

@@ -15,6 +15,7 @@ libvx_log_a_SOURCES = concat_string.cc concat_string.h \
                file_fxns.cc file_fxns.h \
                indent.cc indent.h \
                logger.cc logger.h \
+               main.cc main.h \
                string_array.cc string_array.h \
                str_wrappers.cc str_wrappers.h \
                vx_log.h

src/basic/vx_log/Makefile.in

@@ -108,8 +108,8 @@ am__v_AR_1 =
 libvx_log_a_AR = $(AR) $(ARFLAGS)
 libvx_log_a_LIBADD =
 am_libvx_log_a_OBJECTS = concat_string.$(OBJEXT) file_fxns.$(OBJEXT) \
-	indent.$(OBJEXT) logger.$(OBJEXT) string_array.$(OBJEXT) \
-	str_wrappers.$(OBJEXT)
+	indent.$(OBJEXT) logger.$(OBJEXT) main.$(OBJEXT) \
+	string_array.$(OBJEXT) str_wrappers.$(OBJEXT)
 libvx_log_a_OBJECTS = $(am_libvx_log_a_OBJECTS)
 AM_V_P = $(am__v_P_@AM_V@)
 am__v_P_ = $(am__v_P_@AM_DEFAULT_V@)
@@ -128,8 +128,8 @@ depcomp = $(SHELL) $(top_srcdir)/depcomp
 am__maybe_remake_depfiles = depfiles
 am__depfiles_remade = ./$(DEPDIR)/concat_string.Po \
 	./$(DEPDIR)/file_fxns.Po ./$(DEPDIR)/indent.Po \
-	./$(DEPDIR)/logger.Po ./$(DEPDIR)/str_wrappers.Po \
-	./$(DEPDIR)/string_array.Po
+	./$(DEPDIR)/logger.Po ./$(DEPDIR)/main.Po \
+	./$(DEPDIR)/str_wrappers.Po ./$(DEPDIR)/string_array.Po
 am__mv = mv -f
 CXXCOMPILE = $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \
 	$(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS)
@@ -336,6 +336,7 @@ libvx_log_a_SOURCES = concat_string.cc concat_string.h \
                file_fxns.cc file_fxns.h \
                indent.cc indent.h \
                logger.cc logger.h \
+               main.cc main.h \
                string_array.cc string_array.h \
                str_wrappers.cc str_wrappers.h \
                vx_log.h
@@ -392,6 +393,7 @@ distclean-compile:
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/file_fxns.Po@am__quote@ # am--include-marker
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/indent.Po@am__quote@ # am--include-marker
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/logger.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/main.Po@am__quote@ # am--include-marker
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/str_wrappers.Po@am__quote@ # am--include-marker
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/string_array.Po@am__quote@ # am--include-marker
 
@@ -547,6 +549,7 @@ distclean: distclean-am
 	-rm -f ./$(DEPDIR)/file_fxns.Po
 	-rm -f ./$(DEPDIR)/indent.Po
 	-rm -f ./$(DEPDIR)/logger.Po
+	-rm -f ./$(DEPDIR)/main.Po
 	-rm -f ./$(DEPDIR)/str_wrappers.Po
 	-rm -f ./$(DEPDIR)/string_array.Po
 	-rm -f Makefile
@@ -598,6 +601,7 @@ maintainer-clean: maintainer-clean-am
 	-rm -f ./$(DEPDIR)/file_fxns.Po
 	-rm -f ./$(DEPDIR)/indent.Po
 	-rm -f ./$(DEPDIR)/logger.Po
+	-rm -f ./$(DEPDIR)/main.Po
 	-rm -f ./$(DEPDIR)/str_wrappers.Po
 	-rm -f ./$(DEPDIR)/string_array.Po
 	-rm -f Makefile