From ccfcdfcba00ac4188523565ba5fb44793e4e7f42 Mon Sep 17 00:00:00 2001 From: Dave Albo Date: Thu, 15 Jun 2023 12:58:58 -0600 Subject: [PATCH] Fixed units/level problem in output files, prepended superobject name to variable names in output files, display full raw field in netcdf and postscript output files, changed unit test to include matching --- .../config/MODEConfig_multivar_fake_data | 4 +- ...Config_multivar_fake_data_with_intensities | 4 +- src/basic/vx_util/data_plane.cc | 17 ++++++ src/basic/vx_util/data_plane.h | 2 + src/libcode/vx_shapedata/engine.cc | 15 +++++- src/libcode/vx_shapedata/engine.h | 4 +- src/tools/core/mode/mode_exec.cc | 52 +++++++++++++++++-- src/tools/core/mode/mode_exec.h | 12 ++++- src/tools/core/mode/mode_frontend.cc | 14 ++++- src/tools/core/mode/multivar_data.cc | 11 +++- src/tools/core/mode/multivar_data.h | 10 ++-- 11 files changed, 127 insertions(+), 18 deletions(-) diff --git a/internal/test_unit/config/MODEConfig_multivar_fake_data b/internal/test_unit/config/MODEConfig_multivar_fake_data index c687d39c4e..c8941f8b92 100644 --- a/internal/test_unit/config/MODEConfig_multivar_fake_data +++ b/internal/test_unit/config/MODEConfig_multivar_fake_data @@ -107,12 +107,12 @@ mask_missing_flag = NONE; // // Match objects between the forecast and observation fields // -match_flag = NONE; +match_flag = MERGE_BOTH; // // Maximum centroid distance for objects to be compared // -max_centroid_dist = 800.0/grid_res; +max_centroid_dist = 800.0; //////////////////////////////////////////////////////////////////////////////// diff --git a/internal/test_unit/config/MODEConfig_multivar_fake_data_with_intensities b/internal/test_unit/config/MODEConfig_multivar_fake_data_with_intensities index 8f646002e0..af6d1720a5 100644 --- a/internal/test_unit/config/MODEConfig_multivar_fake_data_with_intensities +++ b/internal/test_unit/config/MODEConfig_multivar_fake_data_with_intensities @@ -107,12 +107,12 @@ mask_missing_flag = NONE; // // Match objects between the forecast and observation fields // -match_flag = NONE; +match_flag = MERGE_BOTH; // // Maximum centroid distance for objects to be compared // -max_centroid_dist = 800.0/grid_res; +max_centroid_dist = 800.0; //////////////////////////////////////////////////////////////////////////////// diff --git a/src/basic/vx_util/data_plane.cc b/src/basic/vx_util/data_plane.cc index 8d34e9b7e2..cbbdec7840 100644 --- a/src/basic/vx_util/data_plane.cc +++ b/src/basic/vx_util/data_plane.cc @@ -399,6 +399,23 @@ void DataPlane::set_accum(int s) { return; } +/////////////////////////////////////////////////////////////////////////////// +void DataPlane::set_all(float *data, int nx, int ny) { + if (nx != Nx || ny != Ny) { + mlog << Error << "\nDataPlane::set_all() -> " + << "the data dimensions do not match: (" + << Nx << ", " << Ny << ") != (" + << nx << ", " << ny << ")!\n\n"; + exit(1); + } + for (int x=0; xvar_info->name_attr())); } else { s = check_hdr_str(conf_key_fcst_var, eng.conf_info.Fcst->var_info->name_attr()); @@ -3368,6 +3376,11 @@ void write_header_columns(ModeFuzzyEngine & eng, const Grid & grid, AsciiTable & // Observation Variable Name if (eng.is_multivar_super) { s = eng.conf_info.obs_multivar_name; + } else if (eng.is_multivar_intensity) { + s = eng.conf_info.obs_multivar_name; + s.add("_"); + s.add(check_hdr_str(conf_key_fcst_var, + eng.conf_info.Obs->var_info->name_attr())); } else { s = check_hdr_str(conf_key_obs_var, eng.conf_info.Obs->var_info->name_attr()); diff --git a/src/libcode/vx_shapedata/engine.h b/src/libcode/vx_shapedata/engine.h index a1662938c1..c975a06691 100644 --- a/src/libcode/vx_shapedata/engine.h +++ b/src/libcode/vx_shapedata/engine.h @@ -315,6 +315,7 @@ class ModeFuzzyEngine { bool need_obs_clus_split; bool is_multivar_super; // used when producing output files + bool is_multivar_intensity; // used when producing output files const Grid * grid; // not allocated @@ -376,7 +377,8 @@ class ModeFuzzyEngine { extern double total_interest (ModeConfInfo &, const PairFeature &, int, int, bool is_single); extern double interest_percentile(ModeFuzzyEngine &, const double, const int); -extern void write_engine_stats (ModeFuzzyEngine &, const Grid &, AsciiTable &, bool isMultiVarSuper=false); +extern void write_engine_stats (ModeFuzzyEngine &, const Grid &, AsciiTable &, bool isMultiVarSuper=false, + bool isMultivarIntensity=false); extern void write_header_row (ModeFuzzyEngine &, AsciiTable &, const int row); // row usually zero extern void write_header_columns (ModeFuzzyEngine &, const Grid &, AsciiTable &, const int row); extern void write_fcst_single (ModeFuzzyEngine &, const int, const Grid &, AsciiTable &, const int); diff --git a/src/tools/core/mode/mode_exec.cc b/src/tools/core/mode/mode_exec.cc index 91b4a394a5..104c6c31cf 100644 --- a/src/tools/core/mode/mode_exec.cc +++ b/src/tools/core/mode/mode_exec.cc @@ -179,7 +179,6 @@ void ModeExecutive::init(int n_files) exit(1); } - // Read forecast file if(!(fcst_mtddf = mtddf_factory.new_met_2d_data_file(fcst_file.c_str(), ftype))) { mlog << Error << "\nTrouble reading forecast file \"" @@ -220,7 +219,6 @@ void ModeExecutive::init(int n_files) /////////////////////////////////////////////////////////////////////// - void ModeExecutive::init_multivar(GrdFileType ftype, GrdFileType otype) { @@ -408,6 +406,14 @@ void ModeExecutive::setup_fcst_obs_data() engine.conf_info.set_perc_thresh(Fcst_sd.data, Obs_sd.data); + // store the input data units + funits = engine.conf_info.Fcst->var_info->units(); + ounits = engine.conf_info.Obs->var_info->units(); + + // store the input data level + flevel = engine.conf_info.Fcst->var_info->level_name(); + olevel = engine.conf_info.Obs->var_info->level_name(); + // // done // @@ -496,6 +502,14 @@ void ModeExecutive::setup_fcst_obs_data(const MultiVarData &mvd) // do this in the second pass using masked Fcst_sd and Obs_sd engine.conf_info.set_perc_thresh(Fcst_sd.data, Obs_sd.data); + // store the input data units + funits = engine.conf_info.Fcst->var_info->units(); + ounits = engine.conf_info.Obs->var_info->units(); + + // store the input data level + flevel = engine.conf_info.Fcst->var_info->level_name(); + olevel = engine.conf_info.Obs->var_info->level_name(); + // // done // @@ -851,7 +865,21 @@ void ModeExecutive::process_masks(ShapeData & fcst_sd, ShapeData & obs_sd) /////////////////////////////////////////////////////////////////////// -void ModeExecutive::process_output(bool isMultivar, bool isMultivarSuper) +void ModeExecutive::set_raw_to_full(float *fcst_raw_data, + float *obs_raw_data, + int nx, int ny, + double idata_min, double idata_max) +{ + engine.fcst_raw->data.set_all(fcst_raw_data, nx, ny); + engine.obs_raw->data.set_all(obs_raw_data, nx, ny); + data_min = idata_min; + data_max = idata_max; +} + +/////////////////////////////////////////////////////////////////////// + +void ModeExecutive::process_output(bool isMultivar, bool isMultivarSuper, + const MultiVarData *mvd) { // store to class member so don't have to pass it around @@ -898,6 +926,18 @@ void ModeExecutive::process_output(bool isMultivar, bool isMultivarSuper) if ( engine.conf_info.ct_stats_flag ) write_ct_stats(); + if (isMultivar) { + if (mvd) { + set_raw_to_full(mvd->_simple->_fcst_raw_data, + mvd->_simple->_obs_raw_data, + mvd->_nx, mvd->_ny, + mvd->_data_min, mvd->_data_max); + } else { + mlog << Error << "\nprocess_output() -> " + << "no multivar data when data is expected\n\n"; + exit(1); + } + } write_obj_netcdf(engine.conf_info.nc_info); if ( engine.conf_info.ps_plot_flag ) plot_engine(); @@ -1121,7 +1161,8 @@ void ModeExecutive::write_obj_stats() // // Write the output statistics to an AsciiTable object // - write_engine_stats(engine, grid, obj_at, isMultivarSuperOutput); + write_engine_stats(engine, grid, obj_at, isMultivarSuperOutput, + isMultivarOutput); // // Write the AsciiTable object to the output file @@ -1213,7 +1254,8 @@ MultiVarData *ModeExecutive::get_multivar_data() replace(fcst_magic_string.begin(), fcst_magic_string.end(), '/', '_'); replace(obs_magic_string.begin(), obs_magic_string.end(), '/', '_'); - mvd->init(fcst_magic_string, obs_magic_string, grid, ftype, otype); + mvd->init(fcst_magic_string, obs_magic_string, grid, ftype, otype, + funits, ounits, flevel, olevel, data_min, data_max); mvd->set_fcst_obj(engine.fcst_split, simple); mvd->set_fcst_raw(engine.fcst_raw, simple); mvd->set_obs_obj(engine.obs_split, simple); diff --git a/src/tools/core/mode/mode_exec.h b/src/tools/core/mode/mode_exec.h index e774fccb68..89d0102d2a 100644 --- a/src/tools/core/mode/mode_exec.h +++ b/src/tools/core/mode/mode_exec.h @@ -116,6 +116,8 @@ class ModeExecutive { ShapeData Fcst_sd, Obs_sd; GrdFileType ftype, otype; + string funits, ounits; + string flevel, olevel; bool isMultivarOutput; bool isMultivarSuperOutput; @@ -135,8 +137,14 @@ class ModeExecutive { void do_match_merge(ShapeData &f_merge, ShapeData &o_merge, bool isMultivarSuper=false); void process_masks(ShapeData &, ShapeData &); - void process_output(bool isMultivar=false, bool isMultivarSuper=false); - + void process_output(bool isMultivar=false, bool isMultivarSuper=false, + const MultiVarData *mvd=NULL); + + void set_raw_to_full(float *fcst_raw_data, + float *obs_raw_data, + int nx, int ny, + double data_min, double data_max); + // owned by caller MultiVarData *get_multivar_data(); diff --git a/src/tools/core/mode/mode_frontend.cc b/src/tools/core/mode/mode_frontend.cc index 0cdde2d502..329bcaa0e7 100644 --- a/src/tools/core/mode/mode_frontend.cc +++ b/src/tools/core/mode/mode_frontend.cc @@ -140,6 +140,15 @@ int ModeFrontEnd::run_multivar_pass2(const StringArray & Argv, const MultiVarDat if (compress_level >= 0) conf.nc_info.set_compress_level(compress_level); //if ( field_index >= 0 ) conf.set_field_index(field_index); + + // for multivar pass2, explicity set the level and units using stored values + // from pass1 + + conf.Fcst->var_info->set_level_name(mvd._flevel.c_str()); + conf.Obs->var_info->set_level_name(mvd._olevel.c_str()); + conf.Fcst->var_info->set_units(mvd._funits.c_str()); + conf.Obs->var_info->set_units(mvd._ounits.c_str()); + if (has_union && (conf.Fcst->merge_flag == MergeType_Thresh || conf.Obs->merge_flag == MergeType_Thresh)) { mlog << Warning << "\nModeFrontEnd::run() -> " @@ -291,7 +300,10 @@ void ModeFrontEnd::do_straight(Processing_t ptype, const MultiVarData &mvd, mode_exec->do_conv_thresh(index, index, false, true); mode_exec->do_match_merge(f_merge, o_merge, ptype == MULTIVAR_SUPER); - mode_exec->process_output(true, false); + + // here replace raw data and min/max for plotting + + mode_exec->process_output(true, false, &mvd); } mode_exec->clear_internal_r_index(); diff --git a/src/tools/core/mode/multivar_data.cc b/src/tools/core/mode/multivar_data.cc index 4d5daac67b..703ad54e8a 100644 --- a/src/tools/core/mode/multivar_data.cc +++ b/src/tools/core/mode/multivar_data.cc @@ -209,7 +209,10 @@ void MultiVarData::checkFileTypeConsistency(const MultiVarData &mvdi, int j) void MultiVarData::init(const string &fname, const string &oname, const Grid &grid, GrdFileType ftype, - GrdFileType otype) + GrdFileType otype, + const string &funits, const string &ounits, + const string &flevel, const string &olevel, + double data_min, double data_max) { _clear(); _f_name = fname; @@ -219,6 +222,12 @@ void MultiVarData::init(const string &fname, const string &oname, _grid = new Grid(grid); _ftype = ftype; _otype = otype; + _funits = funits; + _ounits = ounits; + _flevel = flevel; + _olevel = olevel; + _data_min = data_min; + _data_max = data_max; _simple = new MultiVarData1(_nx, _ny, "Simple"); _merge = new MultiVarData1(_nx, _ny, "Merge"); } diff --git a/src/tools/core/mode/multivar_data.h b/src/tools/core/mode/multivar_data.h index a7cf038298..99ead96419 100644 --- a/src/tools/core/mode/multivar_data.h +++ b/src/tools/core/mode/multivar_data.h @@ -23,7 +23,6 @@ #include "data_file_type.h" #include "two_d_array.h" - class MultiVarData1 { private: @@ -94,7 +93,10 @@ class MultiVarData { void checkFileTypeConsistency(const MultiVarData &mvdi, int j); void init(const string &fname, const string &oname, - const Grid &grid, GrdFileType ftype, GrdFileType otype); + const Grid &grid, GrdFileType ftype, GrdFileType otype, + const string &funits, const string &ounits, + const string &flevel, const string &olevel, + double data_min, double data_max); void set_fcst_obj(ShapeData *sd, bool simple); void set_fcst_raw(ShapeData *sd, bool simple); void set_obs_obj(ShapeData *sd, bool simple); @@ -116,7 +118,9 @@ class MultiVarData { Grid *_grid; GrdFileType _ftype; GrdFileType _otype; - + string _funits, _ounits; + string _flevel, _olevel; + double _data_min, _data_max; };