Feature #2772 python embedding json

docs/Users_Guide/config_options.rst

@@ -494,6 +494,49 @@ Where code is running in a production context, it is worth being familiar with
 the binding / affinitization method on the particular system and building it
 into any relevant scripting.
 
+.. _met_keep_temp_file:
+
+MET_KEEP_TEMP_FILE
+------------------
+
+The MET_KEEP_TEMP_FILE environment variable can be set to control the runtime
+behavior of the MET tools. The MET tools write temporary files in several places
+in the application and library code. By default, those temporary files are deleted
+when they are no longer needed. However it can be useful for development, testing,
+and debugging to keep them for further inspection. Setting this environment variable
+to a value of :code:`yes` or :code:`true` instructs the MET tools to retain temporary
+files instead of deleting them.
+
+Note that doing so may fill up the temporary directory. It is the responsiblity of
+the user to monitor the temporary directory usage and remove temporary files that
+are no longer needed.
+
+When running with this option, users are advised to refer to section
+:numref:`config_tmp_dir` and write temporary files to a personal location rather than
+the default shared :code:`/tmp` directory.
+
+.. _met_python_debug:
+
+MET_PYTHON_DEBUG
+----------------
+
+The MET_PYTHON_DEBUG environment variable can be set to enable debugging log messages
+related to Python embedding. These log messages are disabled by default. The environment
+variable can be set to a value of :code:`all` for all log messages, :code:`dataplane`
+for log messages when reading gridded data, or :code:`point` for log messages when
+reading point data.
+
+.. _met_python_tmp_format:
+
+MET_PYTHON_TMP_FORMAT
+---------------------
+
+The MET_PYTHON_TMP_FORMAT environment variable defines whether temporary files for
+Python embedding should be written as NetCDF files or using JSON/NumPy serialization.
+By default, they are written using JSON for attributes and NumPy serialization for data
+to avoid NetCDF library conflicts between MET and Python. Setting this environment
+variable to :code:`netcdf` enables the use of temporary NetCDF files instead.
+
 Settings Common to Multiple Tools
 =================================
 

internal/test_unit/xml/unit_python.xml

@@ -206,6 +206,7 @@
   <test name="python_numpy_point_stat">
     <exec>&MET_BIN;/point_stat</exec>
     <env>
+      <pair><name>MET_PYTHON_EXE</name> <value>&MET_PYTHON_EXE;</value></pair>
       <pair><name>FCST_COMMAND</name> <value>&MET_BASE;/python/examples/read_ascii_numpy.py &DATA_DIR_PYTHON;/fcst.txt FCST</value></pair>
       <pair><name>OBS_COMMAND</name>  <value>&MET_BASE;/python/examples/read_ascii_numpy.py &DATA_DIR_PYTHON;/obs.txt OBS</value></pair>
     </env>
@@ -480,21 +481,6 @@
     </output>
   </test>
 
-  <test name="python_point2grid_pb2nc_TMP">
-    <exec>&MET_BIN;/point2grid</exec>
-    <param> \
-      'PYTHON_NUMPY=&MET_BASE;/python/examples/read_met_point_obs.py &OUTPUT_DIR;/pb2nc/ndas.20120409.t12z.prepbufr.tm00.nc' \
-      G212 \
-      &OUTPUT_DIR;/python/pb2nc_TMP.nc \
-      -field 'name="TMP"; level="*"; valid_time="20120409_120000"; censor_thresh=[ &lt;0 ]; censor_val=[0];' \
-      -name TEMP \
-      -v 1
-    </param>
-    <output>
-      <grid_nc>&OUTPUT_DIR;/python/pb2nc_TMP.nc</grid_nc>
-    </output>
-  </test>
-
   <!-- Invokes user-python logic to read a point obs -->
   <test name="python_point2grid_pb2nc_TMP_user_python">
     <exec>&MET_BIN;/point2grid</exec>
@@ -535,6 +521,7 @@
   <test name="python_plot_point_obs_CONFIG">
     <exec>&MET_BIN;/plot_point_obs</exec>
     <env>
+      <pair><name>MET_PYTHON_EXE</name> <value>&MET_PYTHON_EXE;</value></pair>
       <pair><name>TO_GRID</name> <value>NONE</value></pair>
     </env>
     <param> \
@@ -561,6 +548,7 @@
                 > &OUTPUT_DIR;/python/ensemble_stat/input_file_list; \
           &MET_BIN;/ensemble_stat</exec>
     <env>
+      <pair><name>MET_PYTHON_EXE</name> <value>&MET_PYTHON_EXE;</value></pair>
       <pair><name>DESC</name>           <value>NA</value></pair>
       <pair><name>OBS_ERROR_FLAG</name> <value>FALSE</value></pair>
       <pair><name>SKIP_CONST</name>     <value>FALSE</value></pair>
@@ -587,6 +575,7 @@
   <test name="python_point_stat_GRIB1_NAM_GDAS_WINDS">
     <exec>&MET_BIN;/point_stat</exec>
     <env>
+      <pair><name>MET_PYTHON_EXE</name> <value>&MET_PYTHON_EXE;</value></pair>
       <pair><name>BEG_DS</name>         <value>-1800</value></pair>
       <pair><name>END_DS</name>         <value>1800</value></pair>
       <pair><name>OUTPUT_PREFIX</name>  <value>GRIB1_NAM_GDAS_WINDS</value></pair>
@@ -605,6 +594,9 @@
   </test>
 
   <test name="python_plot_data_plane_SEMILATLON_ZONAL_MEAN">
+    <env>
+      <pair><name>MET_PYTHON_EXE</name> <value>&MET_PYTHON_EXE;</value></pair>
+    </env>
     <exec>&MET_BIN;/plot_data_plane</exec>
     <param> \
     PYTHON_NUMPY \
@@ -619,6 +611,9 @@
   </test>
 
   <test name="python_pcp_combine_SEMILATLON_MERIDIONAL_MEAN">
+    <env>
+      <pair><name>MET_PYTHON_EXE</name> <value>&MET_PYTHON_EXE;</value></pair>
+    </env>
     <exec>&MET_BIN;/pcp_combine</exec>
     <param> \
     -add PYTHON_NUMPY \

scripts/python/examples/read_ascii_numpy.py

@@ -5,7 +5,7 @@
 ###########################################
 
 def log(msg):
-   dataplane.log_msg(msg)
+   dataplane.log_message(msg)
 
 def set_dataplane_attrs():
    # attrs is a dictionary which contains attributes describing the dataplane.
@@ -95,5 +95,5 @@ def set_dataplane_attrs():
 attrs = set_dataplane_attrs()
 log("Attributes:\t" + repr(attrs))
 
-# Sets fill_value if it exists
+# Sets fill_value if it exists at the dataplane
 #attrs['fill_value'] = 255  # for letter.txt

scripts/python/examples/read_ascii_numpy_grid.py

@@ -27,8 +27,11 @@
    met_data = dataplane.read_2d_text_input(input_file)
    print("Data Shape:\t" + repr(met_data.shape))
    print("Data Type:\t" + repr(met_data.dtype))
-except NameError:
-   print("Can't find the input file")
+except NameError as ex:
+   print(" === ERROR from read_ascii_numpy_grid.py")
+   print(f"   Exception: {type(ex)}  {ex}")
+   print(f"   sys.argv: {sys.argv}")
+   print("   Can't find the input file")
 
 # attrs is a dictionary which contains attributes describing the dataplane.
 # attrs should have 9 items, each of data type string:

scripts/python/examples/read_ascii_xarray.py

@@ -6,7 +6,7 @@
 ###########################################
 
 def log(msg):
-   dataplane.log_msg(msg)
+   dataplane.log_message(msg)
 
 log("Python Script:\t" + repr(sys.argv[0]))
 

scripts/python/examples/read_met_point_obs.py

@@ -21,6 +21,7 @@
 from datetime import datetime
 
 from met.point import met_point_tools
+from met.point_nc import met_point_nc_tools
 from pyembed.python_embedding import pyembed_tools
 
 ARG_PRINT_DATA = 'show_data'
@@ -44,14 +45,15 @@
    netcdf_filename = os.path.expandvars(input_name)
    args = [ netcdf_filename ]
    #args = { 'nc_name': netcdf_filename }
-   point_obs_data = met_point_tools.get_nc_point_obs()
+   point_obs_data = met_point_nc_tools.get_nc_point_obs()
    point_obs_data.read_data(point_obs_data.get_nc_filename(args))
 
 if point_obs_data is not None:
    met_point_data = point_obs_data.get_point_data() 
    met_point_data['met_point_data'] = point_obs_data
-   print("met_point_data: ", met_point_data)
-   print(met_point_data)
+   if os.getenv("MET_PYTHON_DEBUG", "") != "":
+      print("met_point_data: ", met_point_data)
+      print(met_point_data)
 
    if DO_PRINT_DATA:
       point_obs_data.dump()

scripts/python/met/Makefile.am

@@ -28,6 +28,7 @@ pythonmetscripts_DATA = \
 	logger.py \
 	dataplane.py \
 	mprbase.py \
+	point_nc.py \
 	point.py
 
 EXTRA_DIST = ${pythonmetscripts_DATA}

scripts/python/met/Makefile.in

@@ -314,6 +314,7 @@ pythonmetscripts_DATA = \
 	logger.py \
 	dataplane.py \
 	mprbase.py \
+	point_nc.py \
 	point.py
 
 EXTRA_DIST = ${pythonmetscripts_DATA}