Merge pull request #2557 from dtcenter/feature_2525_fill_value_at_dat…

…aplane Feature #2525 fill value at dataplane
dtcenter · Jun 13, 2023 · 76770fa · 76770fa
2 parents cf9bdc2 + 84f59e7
commit 76770fa
Show file tree

Hide file tree

Showing 10 changed files with 382 additions and 116 deletions.
diff --git a/docs/Users_Guide/appendixF.rst b/docs/Users_Guide/appendixF.rst
@@ -110,55 +110,88 @@ Python Script Requirements for 2D Gridded Dataplanes
 
 3. The data inside the **met_data** variable must be **double precision floating point** type
 
-4. A Python dictionary named **attrs** must be defined in the user's script and contain the :ref:`required attributes<pyembed-2d-attrs>`
+4. A Python dictionary named **attrs** must be defined in the user's script and contain the :ref:`required attributes<pyembed-2d-attrs>` and
+   any :ref:`optional attributes<pyembed-2d-attrs>`
 
 .. _pyembed-2d-attrs:
 
-Required Attributes for 2D Gridded Dataplanes
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-The **attrs** dictionary must contain the following information:
+Attributes for 2D Gridded Dataplanes
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 .. list-table:: 2D Dataplane Attributes
-   :widths: 5 5 10
+   :widths: 5 5 10 5
    :header-rows: 1
 
    * - key
      - description
      - data type/format
+     - required/optional
    * - valid
      - valid time
      - string (YYYYMMDD_HHMMSS)
+     - required
    * - init
      - initialization time
      - string (YYYYMMDD_HHMMSS)
+     - required
    * - lead
      - forecast lead
      - string (HHMMSS)
+     - required
    * - accum
      - accumulation interval
      - string (HHMMSS)
+     - required
    * - name
      - variable name
      - string
+     - required
    * - long_name
      - variable long name
      - string
+     - required
    * - level
      - variable level
      - string
+     - required
    * - units
      - variable units
      - string
+     - required
    * - grid
-     - grid information
+     - :ref:`grid information<pyembed-grid-attrs>`
      - string or dict
+     - required
+   * - fill_value
+     - :ref:`missing data value<pyembed-fillvalue-attrs>`
+     - int or float
+     - optional
 
 .. note::
 
    Often times Xarray DataArray objects come with their own set of attributes available as a property. To avoid conflict with the required attributes
    for MET, it is advised to strip these attributes and rely on the **attrs** dictionary defined in your script.
 
+.. _pyembed-fillvalue-attrs:
+
+Python embedding for 2D gridded dataplanes provides support for a user-defined missing data (or fill value). By default, the MET tools will respect (and ignore) the following special values in a user's **met_data** variable:
+
+1. NaN
+2. Inf
+3. -9999
+4. -9999.
+
+If a user has a 2D dataplane with another value that should be considered a fill value by MET, then the user must use the **fill_value** attribute in the **attrs** dictionary. An example would be if a user had a 2D dataplane with missing data indicated with -99. A user can use the **fill_value** attribute in their **attrs** dictionary which will tell MET to ignore those values:
+
+.. code-block:: none
+   :caption: User Fill Value for 2D Dataplane
+   
+   'fill_value': -99
+
+Alternatively, the user can choose to replace their special values with one of the four supported values instead of setting the **fill_value** attribute. Note that only a single user-defined fill value is supported at this time.
+
+.. _pyembed-grid-attrs:
+
 The grid entry in the **attrs** dictionary must contain the grid size and projection information in the same format that is used in the netCDF files written out by the MET tools. The value of this item in the dictionary can either be a string, or another dictionary. Examples of the **grid** entry defined as a string are:
 
 • Using a named grid supported by MET:

diff --git a/scripts/python/examples/read_ascii_numpy.py b/scripts/python/examples/read_ascii_numpy.py
@@ -4,82 +4,96 @@
 
 ###########################################
 
-print("Python Script:\t" + repr(sys.argv[0]))
+def log(msg):
+   dataplane.log_msg(msg)
+
+def set_dataplane_attrs():
+   # attrs is a dictionary which contains attributes describing the dataplane.
+   # attrs should have 9 items, each of data type string:
+   #      'name': data name
+   # 'long_name': descriptive name
+   #     'valid': valid time (format = 'yyyymmdd_hhmmss')
+   #      'init': init time (format = 'yyyymmdd_hhmmss')
+   #      'lead': lead time (format = 'hhmmss')
+   #     'accum': accumulation time (format = 'hhmmss')
+   #     'level': vertilcal level
+   #     'units': units of the data
+   #      'grid': contains the grid information
+   #              - a grid name (G212)
+   #              - a gridded data file name
+   #              - MET specific grid string, "lambert 185 129 12.19 -133.459 -95 40.635 6371.2 25 25 N"
+   #              - a dictionary for the grid information
+
+   valid_time = '20050807_120000'
+   init_time = '20050807_000000'
+   lead_time = '120000'
+   accum_time = '120000'
+   v_level = 'Surface'
+   units = 'None'
+
+   grid_lambert_conformal = {
+      'type': 'Lambert Conformal',
+      'hemisphere': 'N',
+
+      'name': 'FooGrid',
+
+      'scale_lat_1': 25.0,
+      'scale_lat_2': 25.0,
+
+      'lat_pin': 12.19,
+      'lon_pin': -135.459,
+
+      'x_pin': 0.0,
+      'y_pin': 0.0,
+
+      'lon_orient': -95.0,
+
+      'd_km': 40.635,
+      'r_km': 6371.2,
+
+      'nx': 185,
+      'ny': 129,
+   }
+
+   long_name = data_name + "_word"
+   return dataplane.set_dataplane_attrs(data_name, valid_time, init_time,
+                                      lead_time, accum_time, v_level, units,
+                                      grid_lambert_conformal, long_name)
+
+log("Python Script:\t" + repr(sys.argv[0]))
 
-   ##
-   ##  input file specified on the command line
-   ##  load the data into the numpy array
-   ##
+##
+##  input file specified on the command line
+##  load the data into the numpy array
+##
 
 if len(sys.argv) != 3:
-    print("ERROR: read_ascii_numpy.py -> Must specify exactly one input file and a name for the data.")
-    sys.exit(1)
+   dataplane.quit("read_ascii_numpy.py -> Must specify exactly one input file and a name for the data.")
 
 # Read the input file as the first argument
 input_file = os.path.expandvars(sys.argv[1])
 data_name  = sys.argv[2]
 
 try:
-   print("Input File:\t" + repr(input_file))
-   print("Data Name:\t" + repr(data_name))
-   # read_2d_text_input() reads n by m text data and returns 2D numpy array
-   met_data = dataplane.read_2d_text_input(input_file)
-   print("Data Shape:\t" + repr(met_data.shape))
-   print("Data Type:\t" + repr(met_data.dtype))
-except NameError:
-   met_data = None
-   print("Can't find the input file")
-
-# attrs is a dictionary which contains attributes describing the dataplane.
-# attrs should have 9 items, each of data type string:
-#      'name': data name
-# 'long_name': descriptive name
-#     'valid': valid time (format = 'yyyymmdd_hhmmss')
-#      'init': init time (format = 'yyyymmdd_hhmmss')
-#      'lead': lead time (format = 'hhmmss')
-#     'accum': accumulation time (format = 'hhmmss')
-#     'level': vertilcal level
-#     'units': units of the data
-#      'grid': contains the grid information
-#              - a grid name (G212)
-#              - a gridded data file name
-#              - MET specific grid string, "lambert 185 129 12.19 -133.459 -95 40.635 6371.2 25 25 N"
-#              - a dictionary for the grid information
-
-valid_time = '20050807_120000'
-init_time = '20050807_000000'
-lead_time = '120000'
-accum_time = '120000'
-v_level = 'Surface'
-units = 'None'
-
-grid_lambert_conformal = {
-   'type': 'Lambert Conformal',
-   'hemisphere': 'N',
-
-   'name': 'FooGrid',
-
-   'scale_lat_1': 25.0,
-   'scale_lat_2': 25.0,
-
-   'lat_pin': 12.19,
-   'lon_pin': -135.459,
-
-   'x_pin': 0.0,
-   'y_pin': 0.0,
-
-   'lon_orient': -95.0,
-
-   'd_km': 40.635,
-   'r_km': 6371.2,
-
-   'nx': 185,
-   'ny': 129,
-}
-
-long_name = data_name + "_word"
-attrs = dataplane.set_dataplane_attrs(data_name, valid_time, init_time,
-                                      lead_time, accum_time, v_level, units,
-                                      grid_lambert_conformal, long_name)
-
-print("Attributes:\t" + repr(attrs))
+   log("Input File:\t" + repr(input_file))
+   log("Data Name:\t" + repr(data_name))
+   if os.path.exists(input_file):
+      # read_2d_text_input() reads n by m text data and returns 2D numpy array
+      met_data = dataplane.read_2d_text_input(input_file)
+      if met_data is None:
+         dataplane.quit(f" Fail to build met_data from {input_file}")
+      else:
+         log("Data Shape:\t" + repr(met_data.shape))
+         log("Data Type:\t" + repr(met_data.dtype))
+   else:
+      dataplane.quit(f"input {input_file} does exist!!!")
+except:
+   import traceback
+   traceback.print_exc()
+   dataplane.quit(f"Unknown error with {sys.argv[0]}: ")
+
+attrs = set_dataplane_attrs()
+log("Attributes:\t" + repr(attrs))
+
+# Sets fill_value if it exists
+#attrs['fill_value'] = 255  # for letter.txt
diff --git a/scripts/python/examples/read_ascii_xarray.py b/scripts/python/examples/read_ascii_xarray.py
@@ -5,31 +5,41 @@
 
 ###########################################
 
-print("Python Script:\t" + repr(sys.argv[0]))
+def log(msg):
+   dataplane.log_msg(msg)
+
+log("Python Script:\t" + repr(sys.argv[0]))
 
    ##
    ##  input file specified on the command line
    ##  load the data into the numpy array
    ##
 
 if len(sys.argv) != 3:
-    print("ERROR: read_ascii_xarray.py -> Must specify exactly one input file and a name for the data.")
-    sys.exit(1)
+   dataplane.quit("read_ascii_xarray.py -> Must specify exactly one input file and a name for the data.")
 
 # Read the input file as the first argument
 input_file = os.path.expandvars(sys.argv[1])
 data_name  = sys.argv[2]
 
 try:
-   print("Input File:\t" + repr(input_file))
-   print("Data Name:\t" + repr(data_name))
-   # read_2d_text_input() reads n by m text data and returns 2D numpy array
-   met_data = dataplane.read_2d_text_input(input_file)
-   print("Data Shape:\t" + repr(met_data.shape))
-   print("Data Type:\t" + repr(met_data.dtype))
-except NameError:
+   log("Input File:\t" + repr(input_file))
+   log("Data Name:\t" + repr(data_name))
+   if os.path.exists(input_file):
+      # read_2d_text_input() reads n by m text data and returns 2D numpy array
+      met_data = dataplane.read_2d_text_input(input_file)
+      if met_data is None:
+         dataplane.quit(f" Fail to build met_data from {input_file}")
+      else:
+         log("Data Shape:\t" + repr(met_data.shape))
+         log("Data Type:\t" + repr(met_data.dtype))
+   else:
+      dataplane.quit(f"input {input_file} does exist!!!")
+except:
    met_data = None
-   print("Can't read the input file")
+   import traceback
+   traceback.print_exc()
+   dataplane.quit(f"Unknown error with {sys.argv[0]}: ")
 
 ###########################################
 
@@ -105,5 +115,8 @@
 # Delete the met_data variable, and reset it to be the Xarray object
 del met_data
 
+# Sets fill_value/min_value/max_value if it exists
+#ds.attrs['fill_value'] = 255
+
 # Create met_data and specify attrs because XR doesn't persist them.
 met_data = xr.DataArray(ds.fcst, attrs=ds.attrs)
diff --git a/scripts/python/met/Makefile.am b/scripts/python/met/Makefile.am
@@ -25,6 +25,7 @@
 pythonmetscriptsdir = $(pkgdatadir)/python/met
 
 pythonmetscripts_DATA = \
+	logger.py \
 	dataplane.py \
 	mprbase.py \
 	point.py

diff --git a/scripts/python/met/Makefile.in b/scripts/python/met/Makefile.in
@@ -298,6 +298,7 @@ top_builddir = @top_builddir@
 top_srcdir = @top_srcdir@
 pythonmetscriptsdir = $(pkgdatadir)/python/met
 pythonmetscripts_DATA = \
+	logger.py \
 	dataplane.py \
 	mprbase.py \
 	point.py