Fixed units/level problem in output files, prepended superobject name…

… to variable names in output files, display full raw field in netcdf and postscript output files, changed unit test to include matching

internal/test_unit/config/MODEConfig_multivar_fake_data

@@ -107,12 +107,12 @@ mask_missing_flag = NONE;
 //
 // Match objects between the forecast and observation fields
 //
-match_flag = NONE;
+match_flag = MERGE_BOTH;
 
 //
 // Maximum centroid distance for objects to be compared
 //
-max_centroid_dist = 800.0/grid_res;
+max_centroid_dist = 800.0;
 
 ////////////////////////////////////////////////////////////////////////////////
 

internal/test_unit/config/MODEConfig_multivar_fake_data_with_intensities

@@ -107,12 +107,12 @@ mask_missing_flag = NONE;
 //
 // Match objects between the forecast and observation fields
 //
-match_flag = NONE;
+match_flag = MERGE_BOTH;
 
 //
 // Maximum centroid distance for objects to be compared
 //
-max_centroid_dist = 800.0/grid_res;
+max_centroid_dist = 800.0;
 
 ////////////////////////////////////////////////////////////////////////////////
 

src/basic/vx_util/data_plane.cc

@@ -399,6 +399,23 @@ void DataPlane::set_accum(int s) {
    return;
 }
 
+///////////////////////////////////////////////////////////////////////////////
+void DataPlane::set_all(float *data, int nx, int ny) {
+   if (nx != Nx || ny != Ny) {
+      mlog << Error << "\nDataPlane::set_all() -> "
+           << "the data dimensions do not match: ("
+           << Nx << ", " << Ny << ") != ("
+           << nx << ", " << ny << ")!\n\n";
+      exit(1);
+   }
+   for (int x=0; x<nx; ++x) {
+      for (int y=0; y<ny; ++y) {
+         int index = two_to_one(x, y);
+         Data[index] = data[index];
+      }
+   }
+}
+
 ///////////////////////////////////////////////////////////////////////////////
 
 bool DataPlane::is_all_bad_data() const {

src/basic/vx_util/data_plane.h

@@ -85,6 +85,8 @@ class DataPlane {
       void set_lead(int);
       void set_accum(int);
 
+      void set_all(float *data, int nx, int ny);
+
          //
          // Get functions
          //

src/libcode/vx_shapedata/engine.cc

@@ -158,6 +158,7 @@ void ModeFuzzyEngine::init_from_scratch() {
    need_obs_clus_split  = true;
 
    is_multivar_super    = false;
+   is_multivar_intensity= false;
 
    need_match           = true;
 
@@ -3111,11 +3112,13 @@ double interest_percentile(ModeFuzzyEngine &eng, const double p, const int flag)
 
 ///////////////////////////////////////////////////////////////////////
 
-void write_engine_stats(ModeFuzzyEngine & eng, const Grid & grid, AsciiTable & at, bool isMultivarSuper)
+void write_engine_stats(ModeFuzzyEngine & eng, const Grid & grid, AsciiTable & at,
+                        bool isMultivarSuper, bool isMultivarIntensity)
 
 {
 
    eng.is_multivar_super = isMultivarSuper;
+   eng.is_multivar_intensity = isMultivarIntensity;
 
    int i, j, row;
 
@@ -3341,6 +3344,11 @@ void write_header_columns(ModeFuzzyEngine & eng, const Grid & grid, AsciiTable &
    // Forecast Variable Name
    if (eng.is_multivar_super) {
       s = eng.conf_info.fcst_multivar_name;
+   } else if (eng.is_multivar_intensity) {
+      s = eng.conf_info.fcst_multivar_name;
+      s.add("_");
+      s.add(check_hdr_str(conf_key_fcst_var,
+                          eng.conf_info.Fcst->var_info->name_attr()));
    } else {
       s = check_hdr_str(conf_key_fcst_var,
                         eng.conf_info.Fcst->var_info->name_attr());
@@ -3368,6 +3376,11 @@ void write_header_columns(ModeFuzzyEngine & eng, const Grid & grid, AsciiTable &
    // Observation Variable Name
    if (eng.is_multivar_super) {
       s = eng.conf_info.obs_multivar_name;
+   } else if (eng.is_multivar_intensity) {
+      s = eng.conf_info.obs_multivar_name;
+      s.add("_");
+      s.add(check_hdr_str(conf_key_fcst_var,
+                          eng.conf_info.Obs->var_info->name_attr()));
    } else {
       s = check_hdr_str(conf_key_obs_var,
                      eng.conf_info.Obs->var_info->name_attr());

src/libcode/vx_shapedata/engine.h

@@ -315,6 +315,7 @@ class ModeFuzzyEngine {
       bool need_obs_clus_split;
 
       bool is_multivar_super;        // used when producing output files
+      bool is_multivar_intensity;    // used when producing output files
 
       const Grid * grid;             //  not allocated
 
@@ -376,7 +377,8 @@ class ModeFuzzyEngine {
 extern double total_interest     (ModeConfInfo &, const PairFeature &, int, int, bool is_single);
 extern double interest_percentile(ModeFuzzyEngine &, const double, const int);
 
-extern void write_engine_stats   (ModeFuzzyEngine &, const Grid &, AsciiTable &, bool isMultiVarSuper=false);
+extern void write_engine_stats   (ModeFuzzyEngine &, const Grid &, AsciiTable &, bool isMultiVarSuper=false,
+                                  bool isMultivarIntensity=false);
 extern void write_header_row     (ModeFuzzyEngine &, AsciiTable &, const int row);   //  row usually zero
 extern void write_header_columns (ModeFuzzyEngine &, const Grid &, AsciiTable &, const int row);
 extern void write_fcst_single    (ModeFuzzyEngine &, const int, const Grid &, AsciiTable &, const int);

src/tools/core/mode/mode_exec.cc

@@ -179,7 +179,6 @@ void ModeExecutive::init(int n_files)
       exit(1);
    }
 
-
    // Read forecast file
    if(!(fcst_mtddf = mtddf_factory.new_met_2d_data_file(fcst_file.c_str(), ftype))) {
       mlog << Error << "\nTrouble reading forecast file \""
@@ -220,7 +219,6 @@ void ModeExecutive::init(int n_files)
 
 ///////////////////////////////////////////////////////////////////////
 
-
 void ModeExecutive::init_multivar(GrdFileType ftype, GrdFileType otype)
 
 {
@@ -408,6 +406,14 @@ void ModeExecutive::setup_fcst_obs_data()
 
    engine.conf_info.set_perc_thresh(Fcst_sd.data, Obs_sd.data);
 
+   // store the input data units
+   funits = engine.conf_info.Fcst->var_info->units();
+   ounits = engine.conf_info.Obs->var_info->units();
+
+   // store the input data level
+   flevel = engine.conf_info.Fcst->var_info->level_name();
+   olevel = engine.conf_info.Obs->var_info->level_name();
+
    //
    //  done
    //
@@ -496,6 +502,14 @@ void ModeExecutive::setup_fcst_obs_data(const MultiVarData &mvd)
    // do this in the second pass using masked Fcst_sd and Obs_sd
    engine.conf_info.set_perc_thresh(Fcst_sd.data, Obs_sd.data);
 
+   // store the input data units
+   funits = engine.conf_info.Fcst->var_info->units();
+   ounits = engine.conf_info.Obs->var_info->units();
+
+   // store the input data level
+   flevel = engine.conf_info.Fcst->var_info->level_name();
+   olevel = engine.conf_info.Obs->var_info->level_name();
+
    //
    //  done
    //
@@ -851,7 +865,21 @@ void ModeExecutive::process_masks(ShapeData & fcst_sd, ShapeData & obs_sd)
 
 ///////////////////////////////////////////////////////////////////////
 
-void ModeExecutive::process_output(bool isMultivar, bool isMultivarSuper)
+void ModeExecutive::set_raw_to_full(float *fcst_raw_data,
+                                    float *obs_raw_data,
+                                    int nx, int ny,
+                                    double idata_min, double idata_max)
+{
+   engine.fcst_raw->data.set_all(fcst_raw_data, nx, ny);
+   engine.obs_raw->data.set_all(obs_raw_data, nx, ny);
+   data_min = idata_min;
+   data_max = idata_max;
+}
+
+///////////////////////////////////////////////////////////////////////
+
+void ModeExecutive::process_output(bool isMultivar, bool isMultivarSuper,
+                                   const MultiVarData *mvd)
 
 {
    // store to class member so don't have to pass it around
@@ -898,6 +926,18 @@ void ModeExecutive::process_output(bool isMultivar, bool isMultivarSuper)
 
    if ( engine.conf_info.ct_stats_flag )  write_ct_stats();
 
+   if (isMultivar) {
+      if (mvd) {
+         set_raw_to_full(mvd->_simple->_fcst_raw_data,
+                         mvd->_simple->_obs_raw_data,
+                         mvd->_nx, mvd->_ny,
+                         mvd->_data_min, mvd->_data_max);
+      } else {
+         mlog << Error << "\nprocess_output() -> "
+              << "no multivar data when data is expected\n\n";
+         exit(1);
+      }
+   }
    write_obj_netcdf(engine.conf_info.nc_info);
 
    if ( engine.conf_info.ps_plot_flag )   plot_engine();
@@ -1121,7 +1161,8 @@ void ModeExecutive::write_obj_stats()
    //
    // Write the output statistics to an AsciiTable object
    //
-   write_engine_stats(engine, grid, obj_at, isMultivarSuperOutput); 
+   write_engine_stats(engine, grid, obj_at, isMultivarSuperOutput,
+                      isMultivarOutput); 
 
    //
    // Write the AsciiTable object to the output file
@@ -1213,7 +1254,8 @@ MultiVarData *ModeExecutive::get_multivar_data()
    replace(fcst_magic_string.begin(), fcst_magic_string.end(), '/', '_');   
    replace(obs_magic_string.begin(), obs_magic_string.end(), '/', '_');   
 
-   mvd->init(fcst_magic_string, obs_magic_string, grid, ftype, otype);
+   mvd->init(fcst_magic_string, obs_magic_string, grid, ftype, otype,
+             funits, ounits, flevel, olevel, data_min, data_max);
    mvd->set_fcst_obj(engine.fcst_split, simple);
    mvd->set_fcst_raw(engine.fcst_raw, simple);
    mvd->set_obs_obj(engine.obs_split, simple);

src/tools/core/mode/mode_exec.h

@@ -116,6 +116,8 @@ class ModeExecutive {
    ShapeData Fcst_sd, Obs_sd;
 
    GrdFileType ftype, otype;
+   string funits, ounits;
+   string flevel, olevel;
 
    bool isMultivarOutput;
    bool isMultivarSuperOutput;
@@ -135,8 +137,14 @@ class ModeExecutive {
    void do_match_merge(ShapeData &f_merge, ShapeData &o_merge, bool isMultivarSuper=false);
 
    void process_masks(ShapeData &, ShapeData &);
-   void process_output(bool isMultivar=false, bool isMultivarSuper=false);
-
+   void process_output(bool isMultivar=false, bool isMultivarSuper=false,
+                       const MultiVarData *mvd=NULL);
+
+   void set_raw_to_full(float *fcst_raw_data,
+                        float *obs_raw_data,
+                        int nx, int ny,
+                        double data_min, double data_max);
+
 
    // owned by caller
    MultiVarData *get_multivar_data();

src/tools/core/mode/mode_frontend.cc

@@ -140,6 +140,15 @@ int ModeFrontEnd::run_multivar_pass2(const StringArray & Argv, const MultiVarDat
    if (compress_level >= 0) conf.nc_info.set_compress_level(compress_level);
    //if ( field_index >= 0 )
    conf.set_field_index(field_index);
+
+   // for multivar pass2, explicity set the level and units using stored values
+   // from pass1
+
+   conf.Fcst->var_info->set_level_name(mvd._flevel.c_str());
+   conf.Obs->var_info->set_level_name(mvd._olevel.c_str());
+   conf.Fcst->var_info->set_units(mvd._funits.c_str());
+   conf.Obs->var_info->set_units(mvd._ounits.c_str());
+
    if (has_union && (conf.Fcst->merge_flag == MergeType_Thresh ||
                      conf.Obs->merge_flag == MergeType_Thresh)) {
       mlog << Warning << "\nModeFrontEnd::run() -> "
@@ -291,7 +300,10 @@ void ModeFrontEnd::do_straight(Processing_t ptype, const MultiVarData &mvd,
 
       mode_exec->do_conv_thresh(index, index, false, true);
       mode_exec->do_match_merge(f_merge, o_merge, ptype == MULTIVAR_SUPER);
-      mode_exec->process_output(true, false);
+
+      // here replace raw data and min/max for plotting
+
+      mode_exec->process_output(true, false, &mvd);
    }
 
    mode_exec->clear_internal_r_index();

src/tools/core/mode/multivar_data.cc

@@ -209,7 +209,10 @@ void MultiVarData::checkFileTypeConsistency(const MultiVarData &mvdi, int j)
 
 void MultiVarData::init(const string &fname, const string &oname,
                         const Grid &grid, GrdFileType ftype,
-                        GrdFileType otype)
+                        GrdFileType otype, 
+                        const string &funits, const string &ounits,
+                        const string &flevel, const string &olevel,
+                        double data_min, double data_max)
 {
    _clear();
    _f_name = fname;
@@ -219,6 +222,12 @@ void MultiVarData::init(const string &fname, const string &oname,
    _grid = new Grid(grid);
    _ftype = ftype;
    _otype = otype;
+   _funits = funits;
+   _ounits = ounits;
+   _flevel = flevel;
+   _olevel = olevel;
+   _data_min = data_min;
+   _data_max = data_max;
    _simple = new MultiVarData1(_nx, _ny, "Simple");
    _merge = new MultiVarData1(_nx, _ny, "Merge");
 }

src/tools/core/mode/multivar_data.h

@@ -23,7 +23,6 @@
 #include "data_file_type.h"
 #include "two_d_array.h"
 
-
 class MultiVarData1 {
 
  private:
@@ -94,7 +93,10 @@ class MultiVarData {
    void checkFileTypeConsistency(const MultiVarData &mvdi, int j);
 
    void init(const string &fname, const string &oname,
-             const Grid &grid, GrdFileType ftype, GrdFileType otype);
+             const Grid &grid, GrdFileType ftype, GrdFileType otype,
+             const string &funits, const string &ounits,
+             const string &flevel, const string &olevel,
+             double data_min, double data_max);
    void set_fcst_obj(ShapeData *sd, bool simple);
    void set_fcst_raw(ShapeData *sd, bool simple);
    void set_obs_obj(ShapeData *sd, bool simple);
@@ -116,7 +118,9 @@ class MultiVarData {
    Grid *_grid;
    GrdFileType _ftype;
    GrdFileType _otype;
-
+   string _funits, _ounits;
+   string _flevel, _olevel;
+   double _data_min, _data_max;
 };