From ebddb56d1acc0d25a10505fa30b5e5beff4ed77c Mon Sep 17 00:00:00 2001 From: David Fillmore Date: Tue, 26 Jan 2021 09:32:54 -0700 Subject: [PATCH 01/42] Start on write netcdf pickle alternative. --- met/data/wrappers/write_pickle_dataplane.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/met/data/wrappers/write_pickle_dataplane.py b/met/data/wrappers/write_pickle_dataplane.py index 079557538b..bacf617ceb 100644 --- a/met/data/wrappers/write_pickle_dataplane.py +++ b/met/data/wrappers/write_pickle_dataplane.py @@ -13,12 +13,16 @@ import pickle import importlib.util import xarray as xr +import netCDF4 as nc print('Python Script:\t', sys.argv[0]) print('User Command:\t', sys.argv[2:]) print('Write Pickle:\t', sys.argv[1]) pickle_filename = sys.argv[1] +netcdf_filename = sys.argv[1] + '.nc4' + +print('Write NetCDF:\t', netcdf_filename) pyembed_module_name = sys.argv[2] sys.argv = sys.argv[2:] @@ -37,6 +41,21 @@ else: met_info = { 'attrs': met_in.attrs, 'met_data': met_in.met_data } +print('write_pickle_dataplane') print(met_info) pickle.dump( met_info, open( pickle_filename, "wb" ) ) + +# write NetCDF file +ds = nc.Dataset(netcdf_filename, 'w') + +nx, ny = met_in.met_data.shape +print(nx, ny) +ds.createDimension('x', nx) +ds.createDimension('y', ny) +ds.createVariable('met_data', met_in.met_data.dtype, ('x', 'y')) + +for attr in met_in.attrs: + attr_val = met_in.attrs[attr] + print(attr, attr_val, type(attr_val)) +ds.close() From 0fdbfdd617bca2b36fee468dc04e9dcb5a96b8dc Mon Sep 17 00:00:00 2001 From: David Fillmore Date: Tue, 26 Jan 2021 09:38:43 -0700 Subject: [PATCH 02/42] Write dataplane array. --- met/data/wrappers/write_pickle_dataplane.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/met/data/wrappers/write_pickle_dataplane.py b/met/data/wrappers/write_pickle_dataplane.py index bacf617ceb..c9ba3a57eb 100644 --- a/met/data/wrappers/write_pickle_dataplane.py +++ b/met/data/wrappers/write_pickle_dataplane.py @@ -53,7 +53,8 @@ print(nx, ny) ds.createDimension('x', nx) ds.createDimension('y', ny) -ds.createVariable('met_data', met_in.met_data.dtype, ('x', 'y')) +dp = ds.createVariable('met_data', met_in.met_data.dtype, ('x', 'y')) +dp[:] = met_in.met_data for attr in met_in.attrs: attr_val = met_in.attrs[attr] From 6d46603cad053d426e22cf3927b7358892ec06aa Mon Sep 17 00:00:00 2001 From: David Fillmore Date: Tue, 26 Jan 2021 09:46:16 -0700 Subject: [PATCH 03/42] Start on read of netcdf as pickle alternative. --- met/data/wrappers/read_pickle_dataplane.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/met/data/wrappers/read_pickle_dataplane.py b/met/data/wrappers/read_pickle_dataplane.py index f97f153df7..58badccdd2 100644 --- a/met/data/wrappers/read_pickle_dataplane.py +++ b/met/data/wrappers/read_pickle_dataplane.py @@ -9,7 +9,16 @@ import sys import numpy as np import pickle +import netCDF4 as nc print('Python Script:\t', sys.argv[0]) print('Load Pickle:\t', sys.argv[1]) met_info = pickle.load(open(sys.argv[1], "rb")) + +netcdf_filename = sys.argv[1] + '.nc4' +print('Read NetCDF:\t', netcdf_filename) + +# read NetCDF file +ds = nc.Dataset(netcdf_filename, 'r') +met_data = ds['met_data'][:] +met_info['met_data'] = met_data From 6fe424551448285776909c07dd1304515fff1912 Mon Sep 17 00:00:00 2001 From: David Fillmore Date: Tue, 2 Feb 2021 11:15:04 -0700 Subject: [PATCH 04/42] Create attribute variables. --- met/data/wrappers/write_pickle_dataplane.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/met/data/wrappers/write_pickle_dataplane.py b/met/data/wrappers/write_pickle_dataplane.py index c9ba3a57eb..a6e5b0a42a 100644 --- a/met/data/wrappers/write_pickle_dataplane.py +++ b/met/data/wrappers/write_pickle_dataplane.py @@ -59,4 +59,6 @@ for attr in met_in.attrs: attr_val = met_in.attrs[attr] print(attr, attr_val, type(attr_val)) + if type(attr_val) == str: + a = ds.createVariable(attr, 'str') ds.close() From 644db219b3a6518d51569b4bfe6bb626a510935f Mon Sep 17 00:00:00 2001 From: David Fillmore Date: Wed, 3 Feb 2021 16:25:46 -0700 Subject: [PATCH 05/42] Use global attributes for met_info attrs. --- met/data/wrappers/write_pickle_dataplane.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/met/data/wrappers/write_pickle_dataplane.py b/met/data/wrappers/write_pickle_dataplane.py index a6e5b0a42a..1764bbbd9e 100644 --- a/met/data/wrappers/write_pickle_dataplane.py +++ b/met/data/wrappers/write_pickle_dataplane.py @@ -53,6 +53,7 @@ print(nx, ny) ds.createDimension('x', nx) ds.createDimension('y', ny) +ds.createDimension('str_dim', 1) dp = ds.createVariable('met_data', met_in.met_data.dtype, ('x', 'y')) dp[:] = met_in.met_data @@ -60,5 +61,5 @@ attr_val = met_in.attrs[attr] print(attr, attr_val, type(attr_val)) if type(attr_val) == str: - a = ds.createVariable(attr, 'str') + setattr(ds, attr, attr_val) ds.close() From 659406273e3cd211578ddda0e8201ca48fdd151b Mon Sep 17 00:00:00 2001 From: David Fillmore Date: Wed, 3 Feb 2021 17:45:08 -0700 Subject: [PATCH 06/42] Add grid structure. --- met/data/wrappers/write_pickle_dataplane.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/met/data/wrappers/write_pickle_dataplane.py b/met/data/wrappers/write_pickle_dataplane.py index 1764bbbd9e..ec234a5c40 100644 --- a/met/data/wrappers/write_pickle_dataplane.py +++ b/met/data/wrappers/write_pickle_dataplane.py @@ -53,7 +53,6 @@ print(nx, ny) ds.createDimension('x', nx) ds.createDimension('y', ny) -ds.createDimension('str_dim', 1) dp = ds.createVariable('met_data', met_in.met_data.dtype, ('x', 'y')) dp[:] = met_in.met_data @@ -62,4 +61,7 @@ print(attr, attr_val, type(attr_val)) if type(attr_val) == str: setattr(ds, attr, attr_val) + if type(attr_val) == dict: + for key in attr_val: + setattr(ds, attr + '.' + key, attr_val[key]) ds.close() From c6667e38e598716282e8b22ec43d6002beb0024c Mon Sep 17 00:00:00 2001 From: David Fillmore Date: Wed, 3 Feb 2021 18:54:31 -0700 Subject: [PATCH 07/42] Read metadata back into met_info.attrs. --- met/data/wrappers/read_pickle_dataplane.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/met/data/wrappers/read_pickle_dataplane.py b/met/data/wrappers/read_pickle_dataplane.py index 58badccdd2..ddbcddce5c 100644 --- a/met/data/wrappers/read_pickle_dataplane.py +++ b/met/data/wrappers/read_pickle_dataplane.py @@ -21,4 +21,11 @@ # read NetCDF file ds = nc.Dataset(netcdf_filename, 'r') met_data = ds['met_data'][:] +grid = {} +for attr, attr_val in ds.__dict__.items(): + print(attr, attr_val) + if 'grid' in attr: + grid_attr = attr.split('.')[1] + grid[grid_attr] = attr_val +print(grid) met_info['met_data'] = met_data From 1e6eb9ee260658c2e2d0098ce93d91f193698f7f Mon Sep 17 00:00:00 2001 From: David Fillmore Date: Wed, 3 Feb 2021 19:20:17 -0700 Subject: [PATCH 08/42] Convert grid.nx and grid.ny to int. --- met/data/wrappers/read_pickle_dataplane.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/met/data/wrappers/read_pickle_dataplane.py b/met/data/wrappers/read_pickle_dataplane.py index ddbcddce5c..e3b1b03768 100644 --- a/met/data/wrappers/read_pickle_dataplane.py +++ b/met/data/wrappers/read_pickle_dataplane.py @@ -13,7 +13,8 @@ print('Python Script:\t', sys.argv[0]) print('Load Pickle:\t', sys.argv[1]) -met_info = pickle.load(open(sys.argv[1], "rb")) +# met_info = pickle.load(open(sys.argv[1], "rb")) +met_info = {} netcdf_filename = sys.argv[1] + '.nc4' print('Read NetCDF:\t', netcdf_filename) @@ -21,11 +22,16 @@ # read NetCDF file ds = nc.Dataset(netcdf_filename, 'r') met_data = ds['met_data'][:] +met_attrs = {} grid = {} for attr, attr_val in ds.__dict__.items(): - print(attr, attr_val) if 'grid' in attr: grid_attr = attr.split('.')[1] grid[grid_attr] = attr_val -print(grid) + else: + met_attrs[attr] = attr_val +grid['nx'], grid['ny'] = int(grid['nx']), int(grid['ny']) +met_attrs['grid'] = grid met_info['met_data'] = met_data +met_info['attrs'] = met_attrs +print(met_info) From e0055854e870f4d6a7c4c0ad3bd0e3220b69434c Mon Sep 17 00:00:00 2001 From: David Fillmore Date: Wed, 3 Feb 2021 19:27:04 -0700 Subject: [PATCH 09/42] Rename _name key to name. --- met/data/wrappers/read_pickle_dataplane.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/met/data/wrappers/read_pickle_dataplane.py b/met/data/wrappers/read_pickle_dataplane.py index e3b1b03768..330ed740a0 100644 --- a/met/data/wrappers/read_pickle_dataplane.py +++ b/met/data/wrappers/read_pickle_dataplane.py @@ -32,6 +32,8 @@ met_attrs[attr] = attr_val grid['nx'], grid['ny'] = int(grid['nx']), int(grid['ny']) met_attrs['grid'] = grid +met_attrs['name'] = met_attrs['_name'] +del met_attrs['_name'] met_info['met_data'] = met_data met_info['attrs'] = met_attrs print(met_info) From ab986caf6564b54067b0caa91c0ae78c2d9bb5c9 Mon Sep 17 00:00:00 2001 From: David Fillmore Date: Thu, 4 Feb 2021 10:56:14 -0700 Subject: [PATCH 10/42] Removed pickle write. --- met/data/wrappers/read_pickle_dataplane.py | 5 +++-- met/data/wrappers/write_pickle_dataplane.py | 11 +++++++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/met/data/wrappers/read_pickle_dataplane.py b/met/data/wrappers/read_pickle_dataplane.py index 330ed740a0..dabc2f51e3 100644 --- a/met/data/wrappers/read_pickle_dataplane.py +++ b/met/data/wrappers/read_pickle_dataplane.py @@ -12,11 +12,12 @@ import netCDF4 as nc print('Python Script:\t', sys.argv[0]) -print('Load Pickle:\t', sys.argv[1]) +# print('Load Pickle:\t', sys.argv[1]) # met_info = pickle.load(open(sys.argv[1], "rb")) met_info = {} -netcdf_filename = sys.argv[1] + '.nc4' +# netcdf_filename = sys.argv[1] + '.nc4' +netcdf_filename = sys.argv[1] print('Read NetCDF:\t', netcdf_filename) # read NetCDF file diff --git a/met/data/wrappers/write_pickle_dataplane.py b/met/data/wrappers/write_pickle_dataplane.py index ec234a5c40..5794466064 100644 --- a/met/data/wrappers/write_pickle_dataplane.py +++ b/met/data/wrappers/write_pickle_dataplane.py @@ -19,8 +19,9 @@ print('User Command:\t', sys.argv[2:]) print('Write Pickle:\t', sys.argv[1]) -pickle_filename = sys.argv[1] -netcdf_filename = sys.argv[1] + '.nc4' +# pickle_filename = sys.argv[1] +# netcdf_filename = sys.argv[1] + '.nc4' +netcdf_filename = sys.argv[1] print('Write NetCDF:\t', netcdf_filename) @@ -44,13 +45,13 @@ print('write_pickle_dataplane') print(met_info) -pickle.dump( met_info, open( pickle_filename, "wb" ) ) +# pickle.dump( met_info, open( pickle_filename, "wb" ) ) # write NetCDF file ds = nc.Dataset(netcdf_filename, 'w') nx, ny = met_in.met_data.shape -print(nx, ny) +# print(nx, ny) ds.createDimension('x', nx) ds.createDimension('y', ny) dp = ds.createVariable('met_data', met_in.met_data.dtype, ('x', 'y')) @@ -59,6 +60,8 @@ for attr in met_in.attrs: attr_val = met_in.attrs[attr] print(attr, attr_val, type(attr_val)) + if attr == 'name': + setattr(ds, '_name', attr_val) if type(attr_val) == str: setattr(ds, attr, attr_val) if type(attr_val) == dict: From 760b6904f7fc7d6cc8da776dc89023280d9fa639 Mon Sep 17 00:00:00 2001 From: David Fillmore Date: Fri, 5 Feb 2021 08:42:44 -0700 Subject: [PATCH 11/42] Fixed write_pickle_dataplane to work for both numpy and xarray. --- met/data/wrappers/write_pickle_dataplane.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/met/data/wrappers/write_pickle_dataplane.py b/met/data/wrappers/write_pickle_dataplane.py index 5794466064..fdb06cbf3f 100644 --- a/met/data/wrappers/write_pickle_dataplane.py +++ b/met/data/wrappers/write_pickle_dataplane.py @@ -57,8 +57,8 @@ dp = ds.createVariable('met_data', met_in.met_data.dtype, ('x', 'y')) dp[:] = met_in.met_data -for attr in met_in.attrs: - attr_val = met_in.attrs[attr] +for attr in met_info['attrs']: + attr_val = met_info['attrs'][attr] print(attr, attr_val, type(attr_val)) if attr == 'name': setattr(ds, '_name', attr_val) From 791ebf05559e4d14235a7f1c439884276e60b088 Mon Sep 17 00:00:00 2001 From: David Fillmore Date: Fri, 5 Feb 2021 08:49:25 -0700 Subject: [PATCH 12/42] Use items() to iterate of key, value attrs. --- met/data/wrappers/write_pickle_dataplane.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/met/data/wrappers/write_pickle_dataplane.py b/met/data/wrappers/write_pickle_dataplane.py index fdb06cbf3f..44b57fea92 100644 --- a/met/data/wrappers/write_pickle_dataplane.py +++ b/met/data/wrappers/write_pickle_dataplane.py @@ -17,10 +17,7 @@ print('Python Script:\t', sys.argv[0]) print('User Command:\t', sys.argv[2:]) -print('Write Pickle:\t', sys.argv[1]) -# pickle_filename = sys.argv[1] -# netcdf_filename = sys.argv[1] + '.nc4' netcdf_filename = sys.argv[1] print('Write NetCDF:\t', netcdf_filename) @@ -57,8 +54,7 @@ dp = ds.createVariable('met_data', met_in.met_data.dtype, ('x', 'y')) dp[:] = met_in.met_data -for attr in met_info['attrs']: - attr_val = met_info['attrs'][attr] +for attr, attr_val in met_info['attrs'].items(): print(attr, attr_val, type(attr_val)) if attr == 'name': setattr(ds, '_name', attr_val) From c5f17e8d50e931c28895cf57005cadfa8d7dd0ba Mon Sep 17 00:00:00 2001 From: David Fillmore Date: Sat, 13 Feb 2021 10:50:12 -0700 Subject: [PATCH 13/42] Write temporary text file. --- met/data/wrappers/write_pickle_point.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/met/data/wrappers/write_pickle_point.py b/met/data/wrappers/write_pickle_point.py index 1f5ee35bdb..907c0e005d 100644 --- a/met/data/wrappers/write_pickle_point.py +++ b/met/data/wrappers/write_pickle_point.py @@ -18,6 +18,7 @@ print('Write Pickle:\t', sys.argv[1]) pickle_filename = sys.argv[1] +tmp_filename = pickle_filename + '.txt' pyembed_module_name = sys.argv[2] sys.argv = sys.argv[2:] @@ -28,4 +29,8 @@ met_in = importlib.util.module_from_spec(spec) spec.loader.exec_module(met_in) +f = open(tmp_filename, 'w') +for line in met_in.point_data: + f.write(str(line) + '\n') + pickle.dump( met_in.point_data, open( pickle_filename, "wb" ) ) From d6142e8836d79ab13330b56e8a87ff30eaf0e24c Mon Sep 17 00:00:00 2001 From: David Fillmore Date: Wed, 17 Feb 2021 08:53:38 -0700 Subject: [PATCH 14/42] Renamed scripts. --- ...{read_pickle_dataplane.py => read_tmp_dataplane.py} | 8 ++------ ...rite_pickle_dataplane.py => write_tmp_dataplane.py} | 10 +++------- 2 files changed, 5 insertions(+), 13 deletions(-) rename met/data/wrappers/{read_pickle_dataplane.py => read_tmp_dataplane.py} (77%) rename met/data/wrappers/{write_pickle_dataplane.py => write_tmp_dataplane.py} (87%) diff --git a/met/data/wrappers/read_pickle_dataplane.py b/met/data/wrappers/read_tmp_dataplane.py similarity index 77% rename from met/data/wrappers/read_pickle_dataplane.py rename to met/data/wrappers/read_tmp_dataplane.py index dabc2f51e3..e5fb0d6cb0 100644 --- a/met/data/wrappers/read_pickle_dataplane.py +++ b/met/data/wrappers/read_tmp_dataplane.py @@ -1,22 +1,18 @@ ######################################################################## # -# Reads temporary pickle file into memory. +# Reads temporary file into memory. # -# usage: /path/to/python read_pickle_dataplane.py pickle.tmp +# usage: /path/to/python read_tmp_dataplane.py dataplane.tmp # ######################################################################## import sys import numpy as np -import pickle import netCDF4 as nc print('Python Script:\t', sys.argv[0]) -# print('Load Pickle:\t', sys.argv[1]) -# met_info = pickle.load(open(sys.argv[1], "rb")) met_info = {} -# netcdf_filename = sys.argv[1] + '.nc4' netcdf_filename = sys.argv[1] print('Read NetCDF:\t', netcdf_filename) diff --git a/met/data/wrappers/write_pickle_dataplane.py b/met/data/wrappers/write_tmp_dataplane.py similarity index 87% rename from met/data/wrappers/write_pickle_dataplane.py rename to met/data/wrappers/write_tmp_dataplane.py index 44b57fea92..985535da5f 100644 --- a/met/data/wrappers/write_pickle_dataplane.py +++ b/met/data/wrappers/write_tmp_dataplane.py @@ -3,14 +3,13 @@ # Adapted from a script provided by George McCabe # Adapted by Randy Bullock # -# usage: /path/to/python write_pickle_dataplane.py \ -# pickle_output_filename .py +# usage: /path/to/python write_tmp_dataplane.py \ +# tmp_output_filename .py # ######################################################################## import os import sys -import pickle import importlib.util import xarray as xr import netCDF4 as nc @@ -39,16 +38,13 @@ else: met_info = { 'attrs': met_in.attrs, 'met_data': met_in.met_data } -print('write_pickle_dataplane') +print('write_tmp_dataplane') print(met_info) -# pickle.dump( met_info, open( pickle_filename, "wb" ) ) - # write NetCDF file ds = nc.Dataset(netcdf_filename, 'w') nx, ny = met_in.met_data.shape -# print(nx, ny) ds.createDimension('x', nx) ds.createDimension('y', ny) dp = ds.createVariable('met_data', met_in.met_data.dtype, ('x', 'y')) From b39ca2888eafdd227e2757f73bd3d22263b0b382 Mon Sep 17 00:00:00 2001 From: David Fillmore Date: Wed, 17 Feb 2021 08:57:06 -0700 Subject: [PATCH 15/42] Changed script names in Makefile.am. --- met/data/wrappers/Makefile.am | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/met/data/wrappers/Makefile.am b/met/data/wrappers/Makefile.am index d8a6d5a026..821987b273 100644 --- a/met/data/wrappers/Makefile.am +++ b/met/data/wrappers/Makefile.am @@ -23,8 +23,8 @@ wrappersdir = $(pkgdatadir)/wrappers wrappers_DATA = \ generic_python.py \ generic_pickle.py \ - read_pickle_dataplane.py \ - write_pickle_dataplane.py \ + read_tmp_dataplane.py \ + write_tmp_dataplane.py \ write_pickle_mpr.py \ write_pickle_point.py From 7cc2d7779306db1cac55d03c11a67c15b836f7f5 Mon Sep 17 00:00:00 2001 From: David Fillmore Date: Wed, 17 Feb 2021 09:09:57 -0700 Subject: [PATCH 16/42] Replaced pickle with tmp_nc. --- .../vx_data2d_python/python_dataplane.cc | 54 +++++++++---------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/met/src/libcode/vx_data2d_python/python_dataplane.cc b/met/src/libcode/vx_data2d_python/python_dataplane.cc index d5ace046d0..c1a5a3a163 100644 --- a/met/src/libcode/vx_data2d_python/python_dataplane.cc +++ b/met/src/libcode/vx_data2d_python/python_dataplane.cc @@ -31,15 +31,15 @@ GlobalPython GP; // this needs external linkage static const char * user_ppath = 0; -static const char write_pickle [] = "MET_BASE/wrappers/write_pickle_dataplane.py"; +static const char write_tmp_nc [] = "MET_BASE/wrappers/write_tmp_nc_dataplane.py"; -static const char read_pickle [] = "read_pickle_dataplane"; // NO ".py" suffix +static const char read_tmp_nc [] = "read_tmp_nc_dataplane"; // NO ".py" suffix -static const char pickle_base_name [] = "tmp_met_pickle"; +static const char tmp_nc_base_name [] = "tmp_met_nc"; -static const char pickle_var_name [] = "met_info"; +static const char tmp_nc_var_name [] = "met_info"; -static const char pickle_file_var_name [] = "pickle_filename"; +static const char tmp_nc_file_var_name [] = "tmp_nc_filename"; //////////////////////////////////////////////////////////////////////// @@ -51,7 +51,7 @@ static bool straight_python_dataplane(const char * script_name, Grid & met_grid_out, VarInfoPython &vinfo); -static bool pickle_dataplane(const char * script_name, +static bool tmp_nc_dataplane(const char * script_name, int script_argc, char ** script_argv, const bool use_xarray, DataPlane & met_dp_out, Grid & met_grid_out, VarInfoPython &vinfo); @@ -69,9 +69,9 @@ bool python_dataplane(const char * user_script_name, bool status = false; -if ( (user_ppath = getenv(user_python_path_env)) != 0 ) { // do_pickle = true; +if ( (user_ppath = getenv(user_python_path_env)) != 0 ) { // do_tmp_nc = true; - status = pickle_dataplane(user_script_name, + status = tmp_nc_dataplane(user_script_name, user_script_argc, user_script_argv, use_xarray, met_dp_out, met_grid_out, vinfo); @@ -276,7 +276,7 @@ return ( true ); //////////////////////////////////////////////////////////////////////// -bool pickle_dataplane(const char * user_script_name, +bool tmp_nc_dataplane(const char * user_script_name, int user_script_argc, char ** user_script_argv, const bool use_xarray, DataPlane & met_dp_out, Grid & met_grid_out, VarInfoPython &vinfo) @@ -287,7 +287,7 @@ int j; int status; ConcatString command; ConcatString path; -ConcatString pickle_path; +ConcatString tmp_nc_path; const char * tmp_dir = 0; Wchar_Argv wa; @@ -301,14 +301,14 @@ if ( ! tmp_dir ) tmp_dir = default_tmp_dir; path << cs_erase << tmp_dir << '/' - << pickle_base_name; + << tmp_nc_base_name; -pickle_path = make_temp_file_name(path.text(), 0); +tmp_nc_path = make_temp_file_name(path.text(), 0); command << cs_erase << user_ppath << ' ' // user's path to python - << replace_path(write_pickle) << ' ' // write_pickle.py - << pickle_path << ' ' // pickle output filename + << replace_path(write_tmp_nc) << ' ' // write_tmp_nc.py + << tmp_nc_path << ' ' // tmp_nc output filename << user_script_name; // user's script name for (j=1; j " + mlog << Error << "\ntmp_nc_dataplane() -> " << "command \"" << command.text() << "\" failed ... status = " << status << "\n\n"; @@ -346,15 +346,15 @@ if ( PyErr_Occurred() ) { PyErr_Print(); - mlog << Warning << "\npickle_dataplane() -> " + mlog << Warning << "\ntmp_nc_dataplane() -> " << "an error occurred initializing python\n\n"; return ( false ); } -mlog << Debug(3) << "Reading temporary pickle file: " - << pickle_path << "\n"; +mlog << Debug(3) << "Reading temporary tmp_nc file: " + << tmp_nc_path << "\n"; // // set the arguments @@ -362,9 +362,9 @@ mlog << Debug(3) << "Reading temporary pickle file: " StringArray a; -a.add(read_pickle); +a.add(read_tmp_nc); -a.add(pickle_path); +a.add(tmp_nc_path); wa.set(a); @@ -374,7 +374,7 @@ PySys_SetArgv (wa.wargc(), wa.wargv()); // import the python wrapper script as a module // -path = get_short_name(read_pickle); +path = get_short_name(read_tmp_nc); PyObject * module_obj = PyImport_ImportModule (path.text()); @@ -392,7 +392,7 @@ if ( PyErr_Occurred() ) { PyErr_Print(); - mlog << Warning << "\npickle_dataplane() -> " + mlog << Warning << "\ntmp_nc_dataplane() -> " << "an error occurred importing module " << '\"' << path << "\"\n\n"; @@ -402,7 +402,7 @@ if ( PyErr_Occurred() ) { if ( ! module_obj ) { - mlog << Warning << "\npickle_dataplane() -> " + mlog << Warning << "\ntmp_nc_dataplane() -> " << "error running python script\n\n"; return ( false ); @@ -410,7 +410,7 @@ if ( ! module_obj ) { } // - // read the pickle file + // read the tmp_nc file // // @@ -419,13 +419,13 @@ if ( ! module_obj ) { PyObject * module_dict_obj = PyModule_GetDict (module_obj); -PyObject * key_obj = PyUnicode_FromString (pickle_var_name); +PyObject * key_obj = PyUnicode_FromString (tmp_nc_var_name); PyObject * data_obj = PyDict_GetItem (module_dict_obj, key_obj); if ( ! data_obj || ! PyDict_Check(data_obj) ) { - mlog << Error << "\npickle_dataplane() -> " + mlog << Error << "\ntmp_nc_dataplane() -> " << "bad dict object\n\n"; exit ( 1 ); @@ -450,7 +450,7 @@ dataplane_from_numpy_array(np, attrs_dict_obj, met_dp_out, met_grid_out, vinfo); // cleanup // -remove_temp_file(pickle_path); +remove_temp_file(tmp_nc_path); // // done From df0db18e04a59280688781533b5f3092f4171091 Mon Sep 17 00:00:00 2001 From: David Fillmore Date: Wed, 17 Feb 2021 09:28:59 -0700 Subject: [PATCH 17/42] Fixed wrapper script names. --- met/src/libcode/vx_data2d_python/python_dataplane.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/met/src/libcode/vx_data2d_python/python_dataplane.cc b/met/src/libcode/vx_data2d_python/python_dataplane.cc index c1a5a3a163..8f70af5109 100644 --- a/met/src/libcode/vx_data2d_python/python_dataplane.cc +++ b/met/src/libcode/vx_data2d_python/python_dataplane.cc @@ -31,9 +31,9 @@ GlobalPython GP; // this needs external linkage static const char * user_ppath = 0; -static const char write_tmp_nc [] = "MET_BASE/wrappers/write_tmp_nc_dataplane.py"; +static const char write_tmp_nc [] = "MET_BASE/wrappers/write_tmp_dataplane.py"; -static const char read_tmp_nc [] = "read_tmp_nc_dataplane"; // NO ".py" suffix +static const char read_tmp_nc [] = "read_tmp_dataplane"; // NO ".py" suffix static const char tmp_nc_base_name [] = "tmp_met_nc"; From 044c704a2df6a953366d61a48bdccba22aae906a Mon Sep 17 00:00:00 2001 From: David Fillmore Date: Wed, 17 Feb 2021 10:28:33 -0700 Subject: [PATCH 18/42] Test for attrs in met_in.met_data. --- met/data/wrappers/write_tmp_dataplane.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/met/data/wrappers/write_tmp_dataplane.py b/met/data/wrappers/write_tmp_dataplane.py index 985535da5f..f7ff2d7559 100644 --- a/met/data/wrappers/write_tmp_dataplane.py +++ b/met/data/wrappers/write_tmp_dataplane.py @@ -11,7 +11,6 @@ import os import sys import importlib.util -import xarray as xr import netCDF4 as nc print('Python Script:\t', sys.argv[0]) @@ -33,10 +32,12 @@ met_in = importlib.util.module_from_spec(spec) spec.loader.exec_module(met_in) -if isinstance(met_in.met_data, xr.DataArray): - met_info = { 'attrs': met_in.met_data.attrs, 'met_data': met_in.met_data } +met_info = {'met_data': met_in.met_data} +if hasattr(met_in.met_data, 'attrs') and met_in.met_data.attrs: + attrs = met_in.met_data.attrs else: - met_info = { 'attrs': met_in.attrs, 'met_data': met_in.met_data } + attrs = met_in.attrs +met_info['attrs'] = attrs print('write_tmp_dataplane') print(met_info) From d798e9dc0f098e481580dc1ce60e92a4d8179c0f Mon Sep 17 00:00:00 2001 From: David Fillmore Date: Thu, 18 Feb 2021 11:48:11 -0700 Subject: [PATCH 19/42] Initial version of read_tmp_point module. --- met/data/wrappers/read_tmp_point.py | 43 +++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 met/data/wrappers/read_tmp_point.py diff --git a/met/data/wrappers/read_tmp_point.py b/met/data/wrappers/read_tmp_point.py new file mode 100644 index 0000000000..0b3214de14 --- /dev/null +++ b/met/data/wrappers/read_tmp_point.py @@ -0,0 +1,43 @@ +""" +Module Name: read_tmp_point.py + +Read MET Point Observations from a text file created by write_tmp_point.py script. + + Message_Type, Station_ID, Valid_Time, Lat, Lon, Elevation, + GRIB_Code or Variable_Name, Level, Height, QC_String, Observation_Value + +Version Date +1.0.0 2021/02/18 David Fillmore Initial version +""" + +__author__ = 'David Fillmore' +__version__ = '1.0.0' +__email__ = 'met_help@ucar.edu' + +import argparse + +def read_tmp_point(filename): + """ + Arguments: + filename (string): temporary file created by write_tmp_point.py + + Returns: + (list of lists): point data + """ + f = open(filename, 'r') + lines = f.readlines() + f.close() + + point_data = [eval(line.strip('\n')) for line in lines] + + return point_data + +if __name__ == '__main__': + """ + Parse command line arguments + """ + parser = argparse.ArgumentParser() + parser.add_argument('--filename', type=str) + args = parser.parse_args() + + point_data = read_tmp_point(args.filename) From 8116e751090c824591fb8528f4ddd7123df43674 Mon Sep 17 00:00:00 2001 From: David Fillmore Date: Thu, 18 Feb 2021 11:55:05 -0700 Subject: [PATCH 20/42] Added read_tmp_point.py to install list. --- met/data/wrappers/Makefile.am | 1 + 1 file changed, 1 insertion(+) diff --git a/met/data/wrappers/Makefile.am b/met/data/wrappers/Makefile.am index 821987b273..cb35df1dae 100644 --- a/met/data/wrappers/Makefile.am +++ b/met/data/wrappers/Makefile.am @@ -26,6 +26,7 @@ wrappers_DATA = \ read_tmp_dataplane.py \ write_tmp_dataplane.py \ write_pickle_mpr.py \ + read_tmp_point.py \ write_pickle_point.py EXTRA_DIST = ${wrappers_DATA} From 7b5771574f7b2f3c414ecbe27842b8405c280369 Mon Sep 17 00:00:00 2001 From: David Fillmore Date: Thu, 18 Feb 2021 12:12:54 -0700 Subject: [PATCH 21/42] Start on Python3_Script::read_tmp_point. --- met/src/libcode/vx_python3_utils/python3_script.cc | 12 ++++++++++++ met/src/libcode/vx_python3_utils/python3_script.h | 1 + 2 files changed, 13 insertions(+) diff --git a/met/src/libcode/vx_python3_utils/python3_script.cc b/met/src/libcode/vx_python3_utils/python3_script.cc index 56837b65d0..71c994e40f 100644 --- a/met/src/libcode/vx_python3_utils/python3_script.cc +++ b/met/src/libcode/vx_python3_utils/python3_script.cc @@ -234,6 +234,18 @@ return; } +//////////////////////////////////////////////////////////////////////// + +void Python3_Script::read_tmp_point(const char * tmp_filename) const + +{ + +mlog << Debug(3) << "Reading temporary point ascii file: " + << tmp_filename << "\n"; + +ConcatString command; + +} //////////////////////////////////////////////////////////////////////// diff --git a/met/src/libcode/vx_python3_utils/python3_script.h b/met/src/libcode/vx_python3_utils/python3_script.h index 20069762f9..312e5e0fb1 100644 --- a/met/src/libcode/vx_python3_utils/python3_script.h +++ b/met/src/libcode/vx_python3_utils/python3_script.h @@ -77,6 +77,7 @@ class Python3_Script { void read_pickle (const char * variable_name, const char * pickle_filename) const; + void read_tmp_point (const char * tmp_filename) const; }; From 5502da9fea63afbe147ca8944be723514667c7fc Mon Sep 17 00:00:00 2001 From: David Fillmore Date: Thu, 18 Feb 2021 13:07:51 -0700 Subject: [PATCH 22/42] Write MPR tmp ascii file. --- met/data/wrappers/write_pickle_mpr.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/met/data/wrappers/write_pickle_mpr.py b/met/data/wrappers/write_pickle_mpr.py index 2e3f2d0d04..efde687bf7 100644 --- a/met/data/wrappers/write_pickle_mpr.py +++ b/met/data/wrappers/write_pickle_mpr.py @@ -18,6 +18,7 @@ print('Write Pickle:\t', sys.argv[1]) pickle_filename = sys.argv[1] +tmp_filename = pickle_filename + '.txt' pyembed_module_name = sys.argv[2] sys.argv = sys.argv[2:] @@ -28,6 +29,8 @@ met_in = importlib.util.module_from_spec(spec) spec.loader.exec_module(met_in) -print(met_in) +f = open(tmp_filename, 'w') +for line in met_in.mpr_data: + f.write(str(line) + '\n') pickle.dump( met_in.mpr_data, open( pickle_filename, "wb" ) ) From 961b4fc1bab42bc8456129cf3e7c732ea68e8b79 Mon Sep 17 00:00:00 2001 From: David Fillmore Date: Thu, 18 Feb 2021 13:13:32 -0700 Subject: [PATCH 23/42] Renamed to read_tmp_ascii to use for point point and MPR. --- .../{read_tmp_point.py => read_tmp_ascii.py} | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) rename met/data/wrappers/{read_tmp_point.py => read_tmp_ascii.py} (73%) diff --git a/met/data/wrappers/read_tmp_point.py b/met/data/wrappers/read_tmp_ascii.py similarity index 73% rename from met/data/wrappers/read_tmp_point.py rename to met/data/wrappers/read_tmp_ascii.py index 0b3214de14..126150b168 100644 --- a/met/data/wrappers/read_tmp_point.py +++ b/met/data/wrappers/read_tmp_ascii.py @@ -1,8 +1,10 @@ """ -Module Name: read_tmp_point.py +Module Name: read_tmp_ascii.py -Read MET Point Observations from a text file created by write_tmp_point.py script. +Read MET Point Observations from a text file created by write_tmp_point.py script + or MET Matched Pairs from a text file created by write_tmp_mpr.py script +Point observation format: Message_Type, Station_ID, Valid_Time, Lat, Lon, Elevation, GRIB_Code or Variable_Name, Level, Height, QC_String, Observation_Value @@ -16,7 +18,7 @@ import argparse -def read_tmp_point(filename): +def read_tmp_ascii(filename): """ Arguments: filename (string): temporary file created by write_tmp_point.py @@ -28,9 +30,9 @@ def read_tmp_point(filename): lines = f.readlines() f.close() - point_data = [eval(line.strip('\n')) for line in lines] + data = [eval(line.strip('\n')) for line in lines] - return point_data + return data if __name__ == '__main__': """ @@ -40,4 +42,4 @@ def read_tmp_point(filename): parser.add_argument('--filename', type=str) args = parser.parse_args() - point_data = read_tmp_point(args.filename) + data = read_tmp_ascii(args.filename) From 4c0963ddf1efc9e4bc7927f585e59fa7458f871d Mon Sep 17 00:00:00 2001 From: David Fillmore Date: Thu, 18 Feb 2021 13:17:39 -0700 Subject: [PATCH 24/42] Renamed to read_tmp_ascii to use for point point and MPR. --- met/data/wrappers/Makefile.am | 2 +- met/src/libcode/vx_python3_utils/python3_script.cc | 4 ++-- met/src/libcode/vx_python3_utils/python3_script.h | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/met/data/wrappers/Makefile.am b/met/data/wrappers/Makefile.am index cb35df1dae..a8f464313f 100644 --- a/met/data/wrappers/Makefile.am +++ b/met/data/wrappers/Makefile.am @@ -26,7 +26,7 @@ wrappers_DATA = \ read_tmp_dataplane.py \ write_tmp_dataplane.py \ write_pickle_mpr.py \ - read_tmp_point.py \ + read_tmp_ascii.py \ write_pickle_point.py EXTRA_DIST = ${wrappers_DATA} diff --git a/met/src/libcode/vx_python3_utils/python3_script.cc b/met/src/libcode/vx_python3_utils/python3_script.cc index 71c994e40f..fdef49a066 100644 --- a/met/src/libcode/vx_python3_utils/python3_script.cc +++ b/met/src/libcode/vx_python3_utils/python3_script.cc @@ -236,11 +236,11 @@ return; //////////////////////////////////////////////////////////////////////// -void Python3_Script::read_tmp_point(const char * tmp_filename) const +void Python3_Script::read_tmp_ascii(const char * tmp_filename) const { -mlog << Debug(3) << "Reading temporary point ascii file: " +mlog << Debug(3) << "Reading temporary ascii file: " << tmp_filename << "\n"; ConcatString command; diff --git a/met/src/libcode/vx_python3_utils/python3_script.h b/met/src/libcode/vx_python3_utils/python3_script.h index 312e5e0fb1..fe199058a5 100644 --- a/met/src/libcode/vx_python3_utils/python3_script.h +++ b/met/src/libcode/vx_python3_utils/python3_script.h @@ -77,7 +77,7 @@ class Python3_Script { void read_pickle (const char * variable_name, const char * pickle_filename) const; - void read_tmp_point (const char * tmp_filename) const; + void read_tmp_ascii (const char * tmp_filename) const; }; From 91122be96e435b0690dae5592ca9dd489c052a99 Mon Sep 17 00:00:00 2001 From: David Fillmore Date: Fri, 19 Feb 2021 13:07:23 -0700 Subject: [PATCH 25/42] Define Python3_Script::import_read_tmp_ascii_py. --- met/src/libcode/vx_python3_utils/python3_script.cc | 10 ++++++++++ met/src/libcode/vx_python3_utils/python3_script.h | 2 ++ 2 files changed, 12 insertions(+) diff --git a/met/src/libcode/vx_python3_utils/python3_script.cc b/met/src/libcode/vx_python3_utils/python3_script.cc index fdef49a066..0dd3464016 100644 --- a/met/src/libcode/vx_python3_utils/python3_script.cc +++ b/met/src/libcode/vx_python3_utils/python3_script.cc @@ -27,6 +27,8 @@ using namespace std; static const char sq = '\''; // single quote +static const char read_tmp_ascii_py [] = "MET_BASE/wrappers/read_tmp_ascii.py"; + //////////////////////////////////////////////////////////////////////// @@ -236,6 +238,14 @@ return; //////////////////////////////////////////////////////////////////////// +void Python3_Script::import_read_tmp_ascii_py(void) const + +{ + +} + +//////////////////////////////////////////////////////////////////////// + void Python3_Script::read_tmp_ascii(const char * tmp_filename) const { diff --git a/met/src/libcode/vx_python3_utils/python3_script.h b/met/src/libcode/vx_python3_utils/python3_script.h index fe199058a5..7a8aec210e 100644 --- a/met/src/libcode/vx_python3_utils/python3_script.h +++ b/met/src/libcode/vx_python3_utils/python3_script.h @@ -77,6 +77,8 @@ class Python3_Script { void read_pickle (const char * variable_name, const char * pickle_filename) const; + void import_read_tmp_ascii_py (void) const; + void read_tmp_ascii (const char * tmp_filename) const; }; From fef8484cfb23c63af53e869667e12a82d76c6f73 Mon Sep 17 00:00:00 2001 From: David Fillmore Date: Fri, 19 Feb 2021 13:20:31 -0700 Subject: [PATCH 26/42] Call Python3_Script::import_read_tmp_ascii_py. --- met/src/libcode/vx_python3_utils/python3_script.cc | 7 +++++++ met/src/tools/other/ascii2nc/python_handler.cc | 2 ++ 2 files changed, 9 insertions(+) diff --git a/met/src/libcode/vx_python3_utils/python3_script.cc b/met/src/libcode/vx_python3_utils/python3_script.cc index 0dd3464016..99c45b3a89 100644 --- a/met/src/libcode/vx_python3_utils/python3_script.cc +++ b/met/src/libcode/vx_python3_utils/python3_script.cc @@ -242,6 +242,13 @@ void Python3_Script::import_read_tmp_ascii_py(void) const { +ConcatString module; + +module << cs_erase + << replace_path(read_tmp_ascii_py); + +mlog << Debug(3) << "Importing " << module << "\n"; + } //////////////////////////////////////////////////////////////////////// diff --git a/met/src/tools/other/ascii2nc/python_handler.cc b/met/src/tools/other/ascii2nc/python_handler.cc index e2733a605e..76e9dce677 100644 --- a/met/src/tools/other/ascii2nc/python_handler.cc +++ b/met/src/tools/other/ascii2nc/python_handler.cc @@ -379,6 +379,8 @@ wrapper = generic_pickle_wrapper; Python3_Script script(wrapper.text()); +script.import_read_tmp_ascii_py(); + script.read_pickle(list_name, pickle_path.text()); PyObject * obj = script.lookup(list_name); From 93e97624ed098724c0a1c3ab87d000d3e742353a Mon Sep 17 00:00:00 2001 From: David Fillmore Date: Fri, 19 Feb 2021 18:04:57 -0700 Subject: [PATCH 27/42] Append MET_BASE/wrappers to sys.path. --- met/src/libcode/vx_python3_utils/python3_script.cc | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/met/src/libcode/vx_python3_utils/python3_script.cc b/met/src/libcode/vx_python3_utils/python3_script.cc index 99c45b3a89..1364dc7e3c 100644 --- a/met/src/libcode/vx_python3_utils/python3_script.cc +++ b/met/src/libcode/vx_python3_utils/python3_script.cc @@ -247,6 +247,19 @@ ConcatString module; module << cs_erase << replace_path(read_tmp_ascii_py); +ConcatString command; + +run_python_string("import sys"); + +command << cs_erase + << "sys.path.append(\"" + << module.dirname().c_str() + << "\")"; + +mlog << Debug(3) << command << "\n"; + +// run_python_string(command.text()); + mlog << Debug(3) << "Importing " << module << "\n"; } From 44d832872133db2a77dd1ed2d3e7ed3e690e3add Mon Sep 17 00:00:00 2001 From: David Fillmore Date: Fri, 19 Feb 2021 18:26:10 -0700 Subject: [PATCH 28/42] Finished implementation of Python3_Script::import_read_tmp_ascii_py. --- met/src/libcode/vx_python3_utils/python3_script.cc | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/met/src/libcode/vx_python3_utils/python3_script.cc b/met/src/libcode/vx_python3_utils/python3_script.cc index 1364dc7e3c..36cade0317 100644 --- a/met/src/libcode/vx_python3_utils/python3_script.cc +++ b/met/src/libcode/vx_python3_utils/python3_script.cc @@ -258,9 +258,15 @@ command << cs_erase mlog << Debug(3) << command << "\n"; -// run_python_string(command.text()); +run_python_string(command.text()); + +mlog << Debug(2) << "Importing " << module << "\n"; + +command << cs_erase << "import read_tmp_ascii"; -mlog << Debug(3) << "Importing " << module << "\n"; +mlog << Debug(3) << command << "\n"; + +run_python_string(command.text()); } From 3953aba4190f0d50cf45512931d7ac4dc10eef7b Mon Sep 17 00:00:00 2001 From: David Fillmore Date: Fri, 19 Feb 2021 19:30:56 -0700 Subject: [PATCH 29/42] Call Python3_Script::read_tmp_ascii in python_handler. --- met/src/libcode/vx_python3_utils/python3_script.cc | 10 +++++++++- met/src/tools/other/ascii2nc/python_handler.cc | 5 +++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/met/src/libcode/vx_python3_utils/python3_script.cc b/met/src/libcode/vx_python3_utils/python3_script.cc index 36cade0317..bb8c40f44e 100644 --- a/met/src/libcode/vx_python3_utils/python3_script.cc +++ b/met/src/libcode/vx_python3_utils/python3_script.cc @@ -276,11 +276,19 @@ void Python3_Script::read_tmp_ascii(const char * tmp_filename) const { -mlog << Debug(3) << "Reading temporary ascii file: " +mlog << Debug(2) << "Reading temporary ascii file: " << tmp_filename << "\n"; ConcatString command; +command << "read_tmp_ascii.read_tmp_ascii(\"" + << tmp_filename + << "\")"; + +mlog << Debug(3) << command << "\n"; + +run_python_string(command.text()); + } //////////////////////////////////////////////////////////////////////// diff --git a/met/src/tools/other/ascii2nc/python_handler.cc b/met/src/tools/other/ascii2nc/python_handler.cc index 76e9dce677..35e697045e 100644 --- a/met/src/tools/other/ascii2nc/python_handler.cc +++ b/met/src/tools/other/ascii2nc/python_handler.cc @@ -332,6 +332,7 @@ const int N = user_script_args.n(); ConcatString command; ConcatString path; ConcatString pickle_path; +ConcatString tmp_ascii_path; const char * tmp_dir = 0; int status; @@ -348,6 +349,8 @@ path << cs_erase << pickle_base_name; pickle_path = make_temp_file_name(path.text(), 0); +tmp_ascii_path = make_temp_file_name(path.text(), 0); +tmp_ascii_path << ".txt"; command << cs_erase << user_path_to_python << ' ' // user's path to python @@ -383,6 +386,8 @@ script.import_read_tmp_ascii_py(); script.read_pickle(list_name, pickle_path.text()); +script.read_tmp_ascii(tmp_ascii_path.text()); + PyObject * obj = script.lookup(list_name); if ( ! PyList_Check(obj) ) { From 25961d6981f08b8ffd36e9819c4c15f3b9fdf854 Mon Sep 17 00:00:00 2001 From: David Fillmore Date: Sun, 21 Feb 2021 18:40:30 -0700 Subject: [PATCH 30/42] Revised python3_script::read_tmp_ascii with call to run, PyRun_String. --- met/src/libcode/vx_python3_utils/python3_script.cc | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/met/src/libcode/vx_python3_utils/python3_script.cc b/met/src/libcode/vx_python3_utils/python3_script.cc index bb8c40f44e..9afac9a596 100644 --- a/met/src/libcode/vx_python3_utils/python3_script.cc +++ b/met/src/libcode/vx_python3_utils/python3_script.cc @@ -287,7 +287,17 @@ command << "read_tmp_ascii.read_tmp_ascii(\"" mlog << Debug(3) << command << "\n"; -run_python_string(command.text()); +PyErr_Clear(); + +run(command.text()); + +if ( PyErr_Occurred() ) { + + mlog << Error << "\nPython3_Script::read_tmp_ascii() -> " + << "command \"" << command << "\" failed!\n\n"; + + exit ( 1 ); +} } From 794e8fb6173a9514cf388b7a75b09a037aa7a8d3 Mon Sep 17 00:00:00 2001 From: David Fillmore Date: Sun, 21 Feb 2021 18:48:45 -0700 Subject: [PATCH 31/42] Return PyObject* from Python3_Script::run. --- met/src/libcode/vx_python3_utils/python3_script.cc | 10 +++++++--- met/src/libcode/vx_python3_utils/python3_script.h | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/met/src/libcode/vx_python3_utils/python3_script.cc b/met/src/libcode/vx_python3_utils/python3_script.cc index 9afac9a596..b143619068 100644 --- a/met/src/libcode/vx_python3_utils/python3_script.cc +++ b/met/src/libcode/vx_python3_utils/python3_script.cc @@ -165,10 +165,12 @@ return ( var ); //////////////////////////////////////////////////////////////////////// -void Python3_Script::run(const char * command) const +PyObject * Python3_Script::run(const char * command) const { +PyObject * pobj; + if ( empty(command) ) { mlog << Error << "\nPython3_Script::run(const char *) -> " @@ -178,7 +180,9 @@ if ( empty(command) ) { } -if ( ! PyRun_String(command, Py_file_input, Dict, Dict) ) { +pobj = PyRun_String(command, Py_file_input, Dict, Dict); + +if ( ! pobj ) { mlog << Error << "\nPython3_Script::run(const char *) -> " << "command \"" << command << "\" failed!\n\n"; @@ -190,7 +194,7 @@ if ( ! PyRun_String(command, Py_file_input, Dict, Dict) ) { fflush(stdout); fflush(stderr); -return; +return pobj; } diff --git a/met/src/libcode/vx_python3_utils/python3_script.h b/met/src/libcode/vx_python3_utils/python3_script.h index 7a8aec210e..5a765aeabb 100644 --- a/met/src/libcode/vx_python3_utils/python3_script.h +++ b/met/src/libcode/vx_python3_utils/python3_script.h @@ -73,7 +73,7 @@ class Python3_Script { PyObject * lookup(const char * name) const; - void run(const char * command) const; // runs a command in the namespace of the script + PyObject * run(const char * command) const; // runs a command in the namespace of the script void read_pickle (const char * variable_name, const char * pickle_filename) const; From d569cfba0739b4849b57bbead4f2e7b96fe42b95 Mon Sep 17 00:00:00 2001 From: David Fillmore Date: Sun, 21 Feb 2021 18:58:06 -0700 Subject: [PATCH 32/42] Restored call to run_python_string for now. --- met/src/libcode/vx_python3_utils/python3_script.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/met/src/libcode/vx_python3_utils/python3_script.cc b/met/src/libcode/vx_python3_utils/python3_script.cc index b143619068..9a3c2ccaf9 100644 --- a/met/src/libcode/vx_python3_utils/python3_script.cc +++ b/met/src/libcode/vx_python3_utils/python3_script.cc @@ -293,7 +293,10 @@ mlog << Debug(3) << command << "\n"; PyErr_Clear(); -run(command.text()); +PyObject * pobj; + +// pobj = run(command.text()); +run_python_string(command.text()); if ( PyErr_Occurred() ) { From f21b2e62a9791525db92a1ac61738762ba767ff4 Mon Sep 17 00:00:00 2001 From: David Fillmore Date: Fri, 5 Mar 2021 09:22:54 -0700 Subject: [PATCH 33/42] Added python3_script::import_read_tmp_ascii. --- .../vx_python3_utils/python3_script.cc | 34 +++++++++++++++---- .../libcode/vx_python3_utils/python3_script.h | 12 ++++++- .../tools/other/ascii2nc/python_handler.cc | 2 +- 3 files changed, 40 insertions(+), 8 deletions(-) diff --git a/met/src/libcode/vx_python3_utils/python3_script.cc b/met/src/libcode/vx_python3_utils/python3_script.cc index 9a3c2ccaf9..ffef0ef99d 100644 --- a/met/src/libcode/vx_python3_utils/python3_script.cc +++ b/met/src/libcode/vx_python3_utils/python3_script.cc @@ -77,8 +77,12 @@ void Python3_Script::clear() Module = 0; +ModuleAscii = 0; + Dict = 0; +DictAscii = 0; + Script_Filename.clear(); @@ -242,7 +246,7 @@ return; //////////////////////////////////////////////////////////////////////// -void Python3_Script::import_read_tmp_ascii_py(void) const +void Python3_Script::import_read_tmp_ascii_py(void) { @@ -266,11 +270,30 @@ run_python_string(command.text()); mlog << Debug(2) << "Importing " << module << "\n"; -command << cs_erase << "import read_tmp_ascii"; +ConcatString path = "read_tmp_ascii"; -mlog << Debug(3) << command << "\n"; +ModuleAscii = PyImport_ImportModule(path.text()); -run_python_string(command.text()); +if ( ! ModuleAscii ) { + + PyErr_Print(); + mlog << Error << "\nPython3_Script::Python3_Script(const char *) -> " + << "unable to import module \"" << path << "\"\n\n"; + + Py_Finalize(); + + exit ( 1 ); + +} + +DictAscii = PyModule_GetDict (ModuleAscii); + + // + // done + // + +fflush(stdout); +fflush(stderr); } @@ -293,8 +316,7 @@ mlog << Debug(3) << command << "\n"; PyErr_Clear(); -PyObject * pobj; - +// PyObject * pobj; // pobj = run(command.text()); run_python_string(command.text()); diff --git a/met/src/libcode/vx_python3_utils/python3_script.h b/met/src/libcode/vx_python3_utils/python3_script.h index 5a765aeabb..382bad9b58 100644 --- a/met/src/libcode/vx_python3_utils/python3_script.h +++ b/met/src/libcode/vx_python3_utils/python3_script.h @@ -41,6 +41,10 @@ class Python3_Script { PyObject * Dict; // script dictionary, not allocated + PyObject * ModuleAscii; + + PyObject * DictAscii; + ConcatString Script_Filename; @@ -62,6 +66,8 @@ class Python3_Script { PyObject * module(); PyObject * dict(); + PyObject * module_ascii(); + PyObject * dict_ascii(); // // do stuff @@ -77,7 +83,7 @@ class Python3_Script { void read_pickle (const char * variable_name, const char * pickle_filename) const; - void import_read_tmp_ascii_py (void) const; + void import_read_tmp_ascii_py (void); void read_tmp_ascii (const char * tmp_filename) const; }; @@ -90,6 +96,10 @@ inline PyObject * Python3_Script::module() { return ( Module ); } inline PyObject * Python3_Script::dict() { return ( Dict ); } +inline PyObject * Python3_Script::module_ascii() { return ( ModuleAscii ); } + +inline PyObject * Python3_Script::dict_ascii() { return ( DictAscii ); } + inline ConcatString Python3_Script::filename() const { return ( Script_Filename ); } diff --git a/met/src/tools/other/ascii2nc/python_handler.cc b/met/src/tools/other/ascii2nc/python_handler.cc index 35e697045e..15aba246b8 100644 --- a/met/src/tools/other/ascii2nc/python_handler.cc +++ b/met/src/tools/other/ascii2nc/python_handler.cc @@ -386,7 +386,7 @@ script.import_read_tmp_ascii_py(); script.read_pickle(list_name, pickle_path.text()); -script.read_tmp_ascii(tmp_ascii_path.text()); +// script.read_tmp_ascii(tmp_ascii_path.text()); PyObject * obj = script.lookup(list_name); From 31ae2e459821b066b23affef003d9fb48f681877 Mon Sep 17 00:00:00 2001 From: David Fillmore Date: Fri, 5 Mar 2021 09:45:15 -0700 Subject: [PATCH 34/42] Restored read_tmp_ascii call. --- met/src/libcode/vx_python3_utils/python3_script.cc | 10 +++++----- met/src/tools/other/ascii2nc/python_handler.cc | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/met/src/libcode/vx_python3_utils/python3_script.cc b/met/src/libcode/vx_python3_utils/python3_script.cc index ffef0ef99d..b4f20f1bcd 100644 --- a/met/src/libcode/vx_python3_utils/python3_script.cc +++ b/met/src/libcode/vx_python3_utils/python3_script.cc @@ -286,7 +286,7 @@ if ( ! ModuleAscii ) { } -DictAscii = PyModule_GetDict (ModuleAscii); +DictAscii = PyModule_GetDict(ModuleAscii); // // done @@ -308,7 +308,7 @@ mlog << Debug(2) << "Reading temporary ascii file: " ConcatString command; -command << "read_tmp_ascii.read_tmp_ascii(\"" +command << "read_tmp_ascii(\"" << tmp_filename << "\")"; @@ -316,9 +316,9 @@ mlog << Debug(3) << command << "\n"; PyErr_Clear(); -// PyObject * pobj; -// pobj = run(command.text()); -run_python_string(command.text()); +PyObject * pobj; + +pobj = PyRun_String(command.text(), Py_file_input, DictAscii, DictAscii); if ( PyErr_Occurred() ) { diff --git a/met/src/tools/other/ascii2nc/python_handler.cc b/met/src/tools/other/ascii2nc/python_handler.cc index 15aba246b8..35e697045e 100644 --- a/met/src/tools/other/ascii2nc/python_handler.cc +++ b/met/src/tools/other/ascii2nc/python_handler.cc @@ -386,7 +386,7 @@ script.import_read_tmp_ascii_py(); script.read_pickle(list_name, pickle_path.text()); -// script.read_tmp_ascii(tmp_ascii_path.text()); +script.read_tmp_ascii(tmp_ascii_path.text()); PyObject * obj = script.lookup(list_name); From f8becb96f090710003d1f0bbf88668dbc54e6936 Mon Sep 17 00:00:00 2001 From: David Fillmore Date: Fri, 5 Mar 2021 09:52:36 -0700 Subject: [PATCH 35/42] Added lookup into ascii module. --- met/src/libcode/vx_python3_utils/python3_script.cc | 13 +++++++++++++ met/src/libcode/vx_python3_utils/python3_script.h | 2 ++ met/src/tools/other/ascii2nc/python_handler.cc | 2 ++ 3 files changed, 17 insertions(+) diff --git a/met/src/libcode/vx_python3_utils/python3_script.cc b/met/src/libcode/vx_python3_utils/python3_script.cc index b4f20f1bcd..0373fb5ec6 100644 --- a/met/src/libcode/vx_python3_utils/python3_script.cc +++ b/met/src/libcode/vx_python3_utils/python3_script.cc @@ -165,6 +165,19 @@ return ( var ); } +//////////////////////////////////////////////////////////////////////// + +PyObject * Python3_Script::lookup_ascii(const char * name) const + +{ + +PyObject * var = 0; + +var = PyDict_GetItemString (DictAscii, name); + +return ( var ); + +} //////////////////////////////////////////////////////////////////////// diff --git a/met/src/libcode/vx_python3_utils/python3_script.h b/met/src/libcode/vx_python3_utils/python3_script.h index 382bad9b58..42d3134492 100644 --- a/met/src/libcode/vx_python3_utils/python3_script.h +++ b/met/src/libcode/vx_python3_utils/python3_script.h @@ -79,6 +79,8 @@ class Python3_Script { PyObject * lookup(const char * name) const; + PyObject * lookup_ascii(const char * name) const; + PyObject * run(const char * command) const; // runs a command in the namespace of the script void read_pickle (const char * variable_name, const char * pickle_filename) const; diff --git a/met/src/tools/other/ascii2nc/python_handler.cc b/met/src/tools/other/ascii2nc/python_handler.cc index 35e697045e..fac3effc11 100644 --- a/met/src/tools/other/ascii2nc/python_handler.cc +++ b/met/src/tools/other/ascii2nc/python_handler.cc @@ -390,6 +390,8 @@ script.read_tmp_ascii(tmp_ascii_path.text()); PyObject * obj = script.lookup(list_name); +// PyObject * obj = script.lookup_ascii(list_name); + if ( ! PyList_Check(obj) ) { mlog << Error << "\nPythonHandler::do_pickle() -> " From b0c8813be10f6a17732eed393ecd0fc01aff8359 Mon Sep 17 00:00:00 2001 From: David Fillmore Date: Sat, 6 Mar 2021 19:12:42 -0700 Subject: [PATCH 36/42] Return PyObject* from read_tmp_ascii. --- met/src/libcode/vx_python3_utils/python3_script.cc | 9 ++++++++- met/src/libcode/vx_python3_utils/python3_script.h | 2 +- met/src/tools/other/ascii2nc/python_handler.cc | 8 ++++---- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/met/src/libcode/vx_python3_utils/python3_script.cc b/met/src/libcode/vx_python3_utils/python3_script.cc index 0373fb5ec6..7bd567ae2c 100644 --- a/met/src/libcode/vx_python3_utils/python3_script.cc +++ b/met/src/libcode/vx_python3_utils/python3_script.cc @@ -312,7 +312,7 @@ fflush(stderr); //////////////////////////////////////////////////////////////////////// -void Python3_Script::read_tmp_ascii(const char * tmp_filename) const +PyObject* Python3_Script::read_tmp_ascii(const char * tmp_filename) const { @@ -341,6 +341,13 @@ if ( PyErr_Occurred() ) { exit ( 1 ); } +PyTypeObject* type = pobj->ob_type; + +const char* p = type->tp_name; + +mlog << Debug(2) << "read_tmp_ascii return type: " << p << "\n"; + +return pobj; } //////////////////////////////////////////////////////////////////////// diff --git a/met/src/libcode/vx_python3_utils/python3_script.h b/met/src/libcode/vx_python3_utils/python3_script.h index 42d3134492..6930d226a5 100644 --- a/met/src/libcode/vx_python3_utils/python3_script.h +++ b/met/src/libcode/vx_python3_utils/python3_script.h @@ -87,7 +87,7 @@ class Python3_Script { void import_read_tmp_ascii_py (void); - void read_tmp_ascii (const char * tmp_filename) const; + PyObject * read_tmp_ascii (const char * tmp_filename) const; }; diff --git a/met/src/tools/other/ascii2nc/python_handler.cc b/met/src/tools/other/ascii2nc/python_handler.cc index fac3effc11..189a17ea13 100644 --- a/met/src/tools/other/ascii2nc/python_handler.cc +++ b/met/src/tools/other/ascii2nc/python_handler.cc @@ -153,7 +153,7 @@ if ( ! PyList_Check(obj) ) { mlog << Error << "\nPythonHandler::load_python_obs(PyObject *) -> " << "given object not a list!\n\n"; - exit ( 1 ); + // exit ( 1 ); } @@ -386,9 +386,9 @@ script.import_read_tmp_ascii_py(); script.read_pickle(list_name, pickle_path.text()); -script.read_tmp_ascii(tmp_ascii_path.text()); +PyObject * obj = script.read_tmp_ascii(tmp_ascii_path.text()); -PyObject * obj = script.lookup(list_name); +// PyObject * obj = script.lookup(list_name); // PyObject * obj = script.lookup_ascii(list_name); @@ -397,7 +397,7 @@ if ( ! PyList_Check(obj) ) { mlog << Error << "\nPythonHandler::do_pickle() -> " << "pickle object is not a list!\n\n"; - exit ( 1 ); + // exit ( 1 ); } From bd9ed77aabf08b300abff1a363535c6ddbad929f Mon Sep 17 00:00:00 2001 From: David Fillmore Date: Sun, 7 Mar 2021 07:30:50 -0700 Subject: [PATCH 37/42] Put point_data in global namespace. --- met/data/wrappers/read_tmp_ascii.py | 8 ++++++-- met/src/tools/other/ascii2nc/python_handler.cc | 8 ++++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/met/data/wrappers/read_tmp_ascii.py b/met/data/wrappers/read_tmp_ascii.py index 126150b168..b4f4303044 100644 --- a/met/data/wrappers/read_tmp_ascii.py +++ b/met/data/wrappers/read_tmp_ascii.py @@ -18,6 +18,8 @@ import argparse +point_data = None + def read_tmp_ascii(filename): """ Arguments: @@ -26,13 +28,15 @@ def read_tmp_ascii(filename): Returns: (list of lists): point data """ + print('read_tmp_ascii:' + filename) f = open(filename, 'r') lines = f.readlines() f.close() - data = [eval(line.strip('\n')) for line in lines] + global point_data + point_data = [eval(line.strip('\n')) for line in lines] - return data + return point_data if __name__ == '__main__': """ diff --git a/met/src/tools/other/ascii2nc/python_handler.cc b/met/src/tools/other/ascii2nc/python_handler.cc index 189a17ea13..d1b410523f 100644 --- a/met/src/tools/other/ascii2nc/python_handler.cc +++ b/met/src/tools/other/ascii2nc/python_handler.cc @@ -153,7 +153,7 @@ if ( ! PyList_Check(obj) ) { mlog << Error << "\nPythonHandler::load_python_obs(PyObject *) -> " << "given object not a list!\n\n"; - // exit ( 1 ); + exit ( 1 ); } @@ -386,18 +386,18 @@ script.import_read_tmp_ascii_py(); script.read_pickle(list_name, pickle_path.text()); -PyObject * obj = script.read_tmp_ascii(tmp_ascii_path.text()); +PyObject * dobj = script.read_tmp_ascii(tmp_ascii_path.text()); // PyObject * obj = script.lookup(list_name); -// PyObject * obj = script.lookup_ascii(list_name); +PyObject * obj = script.lookup_ascii(list_name); if ( ! PyList_Check(obj) ) { mlog << Error << "\nPythonHandler::do_pickle() -> " << "pickle object is not a list!\n\n"; - // exit ( 1 ); + exit ( 1 ); } From b358bedd65b51432c02c88056170d14a3b026939 Mon Sep 17 00:00:00 2001 From: David Fillmore Date: Sun, 7 Mar 2021 08:21:12 -0700 Subject: [PATCH 38/42] Remove temporary ascii file. --- met/data/wrappers/Makefile.am | 3 +- met/data/wrappers/write_tmp_point.py | 32 +++++++++++++++++++ met/src/tools/other/ascii2nc/ascii2nc.cc | 1 + .../tools/other/ascii2nc/python_handler.cc | 9 +++--- 4 files changed, 39 insertions(+), 6 deletions(-) create mode 100644 met/data/wrappers/write_tmp_point.py diff --git a/met/data/wrappers/Makefile.am b/met/data/wrappers/Makefile.am index a8f464313f..5061e51d51 100644 --- a/met/data/wrappers/Makefile.am +++ b/met/data/wrappers/Makefile.am @@ -27,7 +27,8 @@ wrappers_DATA = \ write_tmp_dataplane.py \ write_pickle_mpr.py \ read_tmp_ascii.py \ - write_pickle_point.py + write_pickle_point.py \ + write_tmp_point.py EXTRA_DIST = ${wrappers_DATA} diff --git a/met/data/wrappers/write_tmp_point.py b/met/data/wrappers/write_tmp_point.py new file mode 100644 index 0000000000..4bbd046122 --- /dev/null +++ b/met/data/wrappers/write_tmp_point.py @@ -0,0 +1,32 @@ +######################################################################## +# +# Adapted from a script provided by George McCabe +# Adapted by Randy Bullock +# +# usage: /path/to/python write_tmp_point.py \ +# tmp_ascii_output_filename .py +# +######################################################################## + +import os +import sys +import importlib.util + +print('Python Script:\t', sys.argv[0]) +print('User Command:\t', sys.argv[2:]) +print('Write Temporary Ascii:\t', sys.argv[1]) + +tmp_filename = sys.argv[1] + '.txt' + +pyembed_module_name = sys.argv[2] +sys.argv = sys.argv[2:] + +user_base = os.path.basename(pyembed_module_name).replace('.py','') + +spec = importlib.util.spec_from_file_location(user_base, pyembed_module_name) +met_in = importlib.util.module_from_spec(spec) +spec.loader.exec_module(met_in) + +f = open(tmp_filename, 'w') +for line in met_in.point_data: + f.write(str(line) + '\n') diff --git a/met/src/tools/other/ascii2nc/ascii2nc.cc b/met/src/tools/other/ascii2nc/ascii2nc.cc index 4ced5397b1..360329659c 100644 --- a/met/src/tools/other/ascii2nc/ascii2nc.cc +++ b/met/src/tools/other/ascii2nc/ascii2nc.cc @@ -43,6 +43,7 @@ // 015 03-20-19 Fillmore Add aeronetv2 and aeronetv3 options. // 016 01-30-20 Bullock Add python option. // 017 01-25-21 Halley Gotway MET #1630 Handle zero obs. +// 018 03-01-21 Fillmore Replace pickle files for temporary ascii. // //////////////////////////////////////////////////////////////////////// diff --git a/met/src/tools/other/ascii2nc/python_handler.cc b/met/src/tools/other/ascii2nc/python_handler.cc index d1b410523f..522e877bce 100644 --- a/met/src/tools/other/ascii2nc/python_handler.cc +++ b/met/src/tools/other/ascii2nc/python_handler.cc @@ -27,9 +27,8 @@ using namespace std; static const char generic_python_wrapper [] = "generic_python"; -static const char generic_pickle_wrapper [] = "generic_pickle"; -static const char write_pickle_wrapper [] = "MET_BASE/wrappers/write_pickle_point.py"; +static const char write_pickle_wrapper [] = "MET_BASE/wrappers/write_tmp_point.py"; static const char list_name [] = "point_data"; @@ -378,13 +377,13 @@ if ( status ) { ConcatString wrapper; -wrapper = generic_pickle_wrapper; +wrapper = generic_python_wrapper; Python3_Script script(wrapper.text()); script.import_read_tmp_ascii_py(); -script.read_pickle(list_name, pickle_path.text()); +// script.read_pickle(list_name, pickle_path.text()); PyObject * dobj = script.read_tmp_ascii(tmp_ascii_path.text()); @@ -407,7 +406,7 @@ load_python_obs(obj); // cleanup // -remove_temp_file(pickle_path); +remove_temp_file(tmp_ascii_path); // // done From 450617347eff5b10d4a992ed0835329857ee7291 Mon Sep 17 00:00:00 2001 From: David Fillmore Date: Sun, 7 Mar 2021 08:33:16 -0700 Subject: [PATCH 39/42] Added tmp_ascii_path. --- met/data/wrappers/write_tmp_point.py | 2 +- .../tools/other/ascii2nc/python_handler.cc | 20 ++++++++----------- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/met/data/wrappers/write_tmp_point.py b/met/data/wrappers/write_tmp_point.py index 4bbd046122..94f56cd3dd 100644 --- a/met/data/wrappers/write_tmp_point.py +++ b/met/data/wrappers/write_tmp_point.py @@ -16,7 +16,7 @@ print('User Command:\t', sys.argv[2:]) print('Write Temporary Ascii:\t', sys.argv[1]) -tmp_filename = sys.argv[1] + '.txt' +tmp_filename = sys.argv[1] pyembed_module_name = sys.argv[2] sys.argv = sys.argv[2:] diff --git a/met/src/tools/other/ascii2nc/python_handler.cc b/met/src/tools/other/ascii2nc/python_handler.cc index 522e877bce..4cdd1a69be 100644 --- a/met/src/tools/other/ascii2nc/python_handler.cc +++ b/met/src/tools/other/ascii2nc/python_handler.cc @@ -28,11 +28,11 @@ using namespace std; static const char generic_python_wrapper [] = "generic_python"; -static const char write_pickle_wrapper [] = "MET_BASE/wrappers/write_tmp_point.py"; +static const char write_tmp_ascii_wrapper[] = "MET_BASE/wrappers/write_tmp_point.py"; static const char list_name [] = "point_data"; -static const char pickle_base_name [] = "tmp_ascii2nc_pickle"; +static const char tmp_base_name [] = "tmp_ascii2nc"; //////////////////////////////////////////////////////////////////////// @@ -345,17 +345,17 @@ if ( ! tmp_dir ) tmp_dir = default_tmp_dir; path << cs_erase << tmp_dir << '/' - << pickle_base_name; + << tmp_base_name; -pickle_path = make_temp_file_name(path.text(), 0); +// pickle_path = make_temp_file_name(path.text(), 0); tmp_ascii_path = make_temp_file_name(path.text(), 0); tmp_ascii_path << ".txt"; command << cs_erase - << user_path_to_python << ' ' // user's path to python - << replace_path(write_pickle_wrapper) << ' ' // write_pickle.py - << pickle_path << ' ' // pickle output filename - << user_script_filename; // user's script name + << user_path_to_python << ' ' // user's path to python + << replace_path(write_tmp_ascii_wrapper) << ' ' // write_tmp_point.py + << tmp_ascii_path << ' ' // pickle output filename + << user_script_filename; // user's script name for (j=0; j Date: Sun, 7 Mar 2021 08:42:08 -0700 Subject: [PATCH 40/42] Removed read_obs_from_pickle. --- .../tools/other/ascii2nc/python_handler.cc | 22 +++++++++---------- met/src/tools/other/ascii2nc/python_handler.h | 10 ++++----- 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/met/src/tools/other/ascii2nc/python_handler.cc b/met/src/tools/other/ascii2nc/python_handler.cc index 4cdd1a69be..d894ab6c64 100644 --- a/met/src/tools/other/ascii2nc/python_handler.cc +++ b/met/src/tools/other/ascii2nc/python_handler.cc @@ -56,7 +56,7 @@ PythonHandler::PythonHandler(const string &program_name) : FileHandler(program_n { -use_pickle = false; +use_tmp_ascii = false; } @@ -81,13 +81,13 @@ for (j=1; j<(a.n()); ++j) { // j starts at one here, not zero } -use_pickle = false; +use_tmp_ascii = false; const char * c = getenv(user_python_path_env); if ( c ) { - use_pickle = true; + use_tmp_ascii = true; user_path_to_python = c; @@ -230,7 +230,7 @@ bool PythonHandler::readAsciiFiles(const vector< ConcatString > &ascii_filename_ bool status = false; -if ( use_pickle ) status = do_pickle (); +if ( use_tmp_ascii ) status = do_tmp_ascii (); else status = do_straight (); return ( status ); @@ -319,10 +319,10 @@ return ( true ); // - // wrapper usage: /path/to/python wrapper.py pickle_output_filename user_script_name [ user_script args ... ] + // wrapper usage: /path/to/python wrapper.py tmp_output_filename user_script_name [ user_script args ... ] // -bool PythonHandler::do_pickle() +bool PythonHandler::do_tmp_ascii() { @@ -330,7 +330,6 @@ int j; const int N = user_script_args.n(); ConcatString command; ConcatString path; -ConcatString pickle_path; ConcatString tmp_ascii_path; const char * tmp_dir = 0; int status; @@ -347,14 +346,13 @@ path << cs_erase << tmp_dir << '/' << tmp_base_name; -// pickle_path = make_temp_file_name(path.text(), 0); tmp_ascii_path = make_temp_file_name(path.text(), 0); tmp_ascii_path << ".txt"; command << cs_erase << user_path_to_python << ' ' // user's path to python << replace_path(write_tmp_ascii_wrapper) << ' ' // write_tmp_point.py - << tmp_ascii_path << ' ' // pickle output filename + << tmp_ascii_path << ' ' // temporary ascii output filename << user_script_filename; // user's script name for (j=0; j " + mlog << Error << "\nPythonHandler::do_tmp_ascii() -> " << "command \"" << command.text() << "\" failed ... status = " << status << "\n\n"; @@ -389,8 +387,8 @@ PyObject * obj = script.lookup_ascii(list_name); if ( ! PyList_Check(obj) ) { - mlog << Error << "\nPythonHandler::do_pickle() -> " - << "pickle object is not a list!\n\n"; + mlog << Error << "\nPythonHandler::do_tmp_ascii() -> " + << "tmp ascii object is not a list!\n\n"; exit ( 1 ); diff --git a/met/src/tools/other/ascii2nc/python_handler.h b/met/src/tools/other/ascii2nc/python_handler.h index abae8ddd5d..b0fb2ef492 100644 --- a/met/src/tools/other/ascii2nc/python_handler.h +++ b/met/src/tools/other/ascii2nc/python_handler.h @@ -50,9 +50,9 @@ class PythonHandler : public FileHandler static string getFormatString() { return "python"; } - bool use_pickle; + bool use_tmp_ascii; - ConcatString user_path_to_python; // if we're using pickle + ConcatString user_path_to_python; // if we're using temporary ascii ConcatString user_script_filename; @@ -68,15 +68,13 @@ class PythonHandler : public FileHandler virtual bool readAsciiFiles(const vector< ConcatString > &ascii_filename_list); - bool do_pickle (); - bool do_straight (); // straight-up python, no pickle + bool do_tmp_ascii(); + bool do_straight (); // straight-up python, no temporary ascii void load_python_obs(PyObject *); // python object is list of lists bool read_obs_from_script (const char * script_name, const char * variable_name); - - bool read_obs_from_pickle (const char * pickle_name, const char * variable_name); }; From 6568493c6763869789ab142a98b4ca15fbf042aa Mon Sep 17 00:00:00 2001 From: David Fillmore Date: Fri, 12 Mar 2021 13:56:04 -0700 Subject: [PATCH 41/42] Replaced tmp netcdf _name attribute with name_str. --- met/data/wrappers/read_tmp_dataplane.py | 4 ++-- met/data/wrappers/write_tmp_dataplane.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/met/data/wrappers/read_tmp_dataplane.py b/met/data/wrappers/read_tmp_dataplane.py index e5fb0d6cb0..e21c17ba3f 100644 --- a/met/data/wrappers/read_tmp_dataplane.py +++ b/met/data/wrappers/read_tmp_dataplane.py @@ -29,8 +29,8 @@ met_attrs[attr] = attr_val grid['nx'], grid['ny'] = int(grid['nx']), int(grid['ny']) met_attrs['grid'] = grid -met_attrs['name'] = met_attrs['_name'] -del met_attrs['_name'] +met_attrs['name'] = met_attrs['name_str'] +del met_attrs['name_str'] met_info['met_data'] = met_data met_info['attrs'] = met_attrs print(met_info) diff --git a/met/data/wrappers/write_tmp_dataplane.py b/met/data/wrappers/write_tmp_dataplane.py index f7ff2d7559..c04b1da6d0 100644 --- a/met/data/wrappers/write_tmp_dataplane.py +++ b/met/data/wrappers/write_tmp_dataplane.py @@ -54,7 +54,7 @@ for attr, attr_val in met_info['attrs'].items(): print(attr, attr_val, type(attr_val)) if attr == 'name': - setattr(ds, '_name', attr_val) + setattr(ds, 'name_str', attr_val) if type(attr_val) == str: setattr(ds, attr, attr_val) if type(attr_val) == dict: From 22f5e98acba65a3e5454fdee939a4ea87a4d334a Mon Sep 17 00:00:00 2001 From: David Fillmore Date: Mon, 15 Mar 2021 14:49:09 -0600 Subject: [PATCH 42/42] Append user script path to system path. --- met/data/wrappers/write_tmp_dataplane.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/met/data/wrappers/write_tmp_dataplane.py b/met/data/wrappers/write_tmp_dataplane.py index c04b1da6d0..6af7d115a9 100644 --- a/met/data/wrappers/write_tmp_dataplane.py +++ b/met/data/wrappers/write_tmp_dataplane.py @@ -23,6 +23,11 @@ pyembed_module_name = sys.argv[2] sys.argv = sys.argv[2:] +# append user script dir to system path +pyembed_dir, pyembed_file = os.path.split(pyembed_module_name) +if pyembed_dir: + os.path.append(pyembed_dir) + if not pyembed_module_name.endswith('.py'): pyembed_module_name += '.py'