Update develop-ref after #2075 (#2076)

Co-authored-by: Julie Prestopnik <[email protected]> Co-authored-by: johnhg <[email protected]> Co-authored-by: Seth Linden <[email protected]> Co-authored-by: John Halley Gotway <[email protected]> Co-authored-by: j-opatz <[email protected]> Co-authored-by: Howard Soh <[email protected]> Co-authored-by: John Halley Gotway <[email protected]> Co-authored-by: jprestop <[email protected]> Co-authored-by: Howard Soh <[email protected]> Co-authored-by: Seth Linden <[email protected]> Co-authored-by: hsoh-u <[email protected]> Co-authored-by: George McCabe <[email protected]> Co-authored-by: John Halley Gotway <[email protected]> Co-authored-by: MET Tools Test Account <[email protected]> Co-authored-by: mo-mglover <[email protected]> Co-authored-by: davidalbo <[email protected]> Co-authored-by: lisagoodrich <[email protected]>
dtcenter · Mar 2, 2022 · 49a6050 · 49a6050
1 parent 3fee8f0
commit 49a6050
Show file tree

Hide file tree

Showing 16 changed files with 304 additions and 41 deletions.
diff --git a/met/docs/Users_Guide/gen-ens-prod.rst b/met/docs/Users_Guide/gen-ens-prod.rst
@@ -173,6 +173,18 @@ Each value in the array will replace the text **MET_ENS_MEMBER_ID**.
     ];
   }
 
+This replacement behavior can also be applied to climatology file name entry, in the 
+climo_mean and climo_stdev dictionaries.
+
+.. code-block:: none
+
+  climo_mean = {
+     file_name = ["/path/to/file/memberMET_ENS_MEMBER_ID-mean.nc"];
+     }
+     
+This substitution method can only be used if **ens_member_ids** has at least one entry 
+and the **normalize** option is set to **CLIMO_ANOM** or **CLIMO_STD_ANOM**.
+
 **control_id** is a string that is substituted in the same way as the **ens_member_ids** values
 to read a control member. This value is only used when the **-ctrl** command line argument is
 used. The value should not be found in the **ens_member_ids** array.

diff --git a/met/src/basic/vx_config/threshold.cc b/met/src/basic/vx_config/threshold.cc
@@ -139,7 +139,7 @@ if ( tf_left )  return ( true );
 
 const bool tf_right = right_child->check(x, cmn, csd);
 
-return ( tf_left || tf_right );
+return ( tf_right );
 
 }
 

diff --git a/met/src/basic/vx_util/ascii_table.cc b/met/src/basic/vx_util/ascii_table.cc
@@ -1002,9 +1002,10 @@ if ( DoCommaString )  {
 
    p = strchr(junk, '.');
 
-   if ( p )  *p = (char) 0;
-
-   ++p;
+   if ( p ) {
+      *p = (char) 0;
+      ++p;
+   }
 
    X = atol(junk);
 
@@ -1014,7 +1015,7 @@ if ( DoCommaString )  {
 
    s << j2;
 
-   if ( Precision > 0 )  s << '.' << p;
+   if ( Precision > 0 && p )  s << '.' << p;
 
    set_entry(r, c, s.string());
 

diff --git a/met/src/libcode/vx_data2d/var_info.cc b/met/src/libcode/vx_data2d/var_info.cc
@@ -892,6 +892,12 @@ int EnsVarInfo::get_file_index(int index) {
 
 ////////////////////////////////////////////////////////////////////////
 
+ConcatString EnsVarInfo::get_ens_member_id(int index) {
+   return inputs[index].ens_member_id;
+}
+
+////////////////////////////////////////////////////////////////////////
+
 ConcatString raw_magic_str(Dictionary i_edict, GrdFileType file_type) {
    ConcatString magic_str;
 

diff --git a/met/src/libcode/vx_data2d/var_info.h b/met/src/libcode/vx_data2d/var_info.h
@@ -258,9 +258,10 @@ inline int          VarInfo::accum_attr()     const { return(SetAttrAccum);    }
 //
 
 struct InputInfo {
-   VarInfo * var_info;      // Variable information to read
-   int file_index;          // Index in file_list of file to read
-   StringArray * file_list; // Array of files (unallocated)
+   VarInfo * var_info;         // Variable information to read
+   int file_index;             // Index in file_list of file to read
+   StringArray * file_list;    // Array of files (unallocated)
+   ConcatString ens_member_id; // MET_ENS_MEMBER_ID string
 };
 
 ////////////////////////////////////////////////////////////////////////
@@ -292,6 +293,7 @@ class EnsVarInfo {
       VarInfo * get_var_info(int index=0);
       ConcatString get_file(int index=0);
       int get_file_index(int index=0);
+      ConcatString get_ens_member_id(int index=0);
 
       ConcatString nc_var_str;      // Ensemble variable name strings
       ThreshArray cat_ta;           // Ensemble categorical thresholds

diff --git a/met/src/libcode/vx_nc_obs/met_point_data.cc b/met/src/libcode/vx_nc_obs/met_point_data.cc
@@ -403,6 +403,7 @@ void MetPointHeader::reset_counters() {
    strl_len = 0;
    strll_len = 0;
    hdr_count = 0;
+   hdr_type_count = 0;
 
    min_vld_time = -1;
    max_vld_time = -1;

diff --git a/met/src/libcode/vx_nc_obs/nc_obs_util.cc b/met/src/libcode/vx_nc_obs/nc_obs_util.cc
@@ -63,6 +63,7 @@ void NcDataBuffer::reset_counters() {
    hdr_data_offset = 0;
    pb_hdr_count = 0;
    pb_hdr_data_offset = 0;
+   prev_hdr_vld = 0;
 }
 
 ///////////////////////////////////////////////////////////////////////////////
@@ -988,6 +989,7 @@ void NetcdfObsVars::reset(bool _use_var_id) {
    deflate_level = 0;
    hdr_cnt     = 0;     // header array length (fixed dimension if hdr_cnt > 0)
    obs_cnt     = 0;     // obs. array length (fixed dimension if obs_cnt > 0)
+   raw_hdr_cnt = 0;
    //hdr_str_len = 0;    // string length for header (message) type header
 }
 

diff --git a/met/src/libcode/vx_shapedata/interest.cc b/met/src/libcode/vx_shapedata/interest.cc
@@ -601,23 +601,33 @@ void get_percentiles(DistributionPercentiles &ptile,
                      const int perc, const bool precip_flag)
 
 {
-   int i, x, y, count, n_values;
+   int i, x, y, n_values;
    int nx, ny;
    double *v = (double *) 0;
+   double *v_tmp = (double *) 0;
+   const char *method_name = "get_percentiles() -> ";
 
    nx = raw.data.nx();
    ny = raw.data.ny();
 
+   v_tmp = new double[nx*ny];
+   if(!v_tmp) {
+      mlog << Error << "\n" << method_name << "memory allocation error, v_tmp\n\n";
+      exit(1);
+   }
+
    //
    // Count values.
-   // Only check precipitation for values greater than zero.
+   // Only collect precipitation values greater than zero.
    //
    n_values = 0;
    for(x=0; x<nx; ++x) {
       for(y=0; y<ny; ++y) {
          if((mask.s_is_on(x, y)) &&
             (raw.is_valid_xy(x, y)) &&
-            (!precip_flag || (precip_flag && raw.data(x, y) > 0))) ++n_values;
+            (!precip_flag || (precip_flag && raw.data(x, y) > 0))) {
+            v_tmp[n_values++] = (double)(raw.data(x, y));
+         }
       }
    }
 
@@ -627,25 +637,15 @@ void get_percentiles(DistributionPercentiles &ptile,
    v = new double [n_values];
 
    if(!v) {
-      mlog << Error << "\nget_percentiles() -> "
-           << "memory allocation error\n\n";
+      mlog << Error << "\n" << method_name << "memory allocation error\n\n";
       exit(1);
    }
 
    //
    // Fill values
    //
-   count = 0;
-   for(x=0; x<nx; ++x) {
-      for(y=0; y<ny; ++y) {
-         if((mask.s_is_on(x, y)) &&
-            (raw.is_valid_xy(x, y)) &&
-            (!precip_flag || (precip_flag && raw.data(x, y) > 0))) {
-
-            v[count++] = (double) (raw.data(x, y));
-         }
-      }
-   }
+   for(x=0; x<n_values; ++x)
+      v[x] = v_tmp[x];
 
    //
    // Sort
@@ -685,6 +685,7 @@ void get_percentiles(DistributionPercentiles &ptile,
    //
    // Free memory
    //
+   if(v_tmp) { delete [] v_tmp;  v_tmp = (double *) 0; }
    if(v) { delete [] v;  v = (double *) 0; }
 
    //

diff --git a/met/src/libcode/vx_statistics/read_climo.cc b/met/src/libcode/vx_statistics/read_climo.cc
@@ -243,6 +243,12 @@ void read_climo_file(const char *climo_file, GrdFileType ctype,
          }
       }
 
+      // Print log message for matching record
+      mlog << Debug(4)
+           << "Found matching " << cur_ut_cs << " \""
+           << info->magic_str() << "\" climatology field in file \""
+           << climo_file << "\".\n"; 
+
       // Regrid, if needed
       if(!(mtddf->grid() == vx_grid)) {
          mlog << Debug(2)

diff --git a/met/src/libcode/vx_tc_util/genesis_info.h b/met/src/libcode/vx_tc_util/genesis_info.h
@@ -53,8 +53,6 @@ class GenesisInfo : public TrackInfo {
 
       void assign(const GenesisInfo &);
 
-      bool IsSet;
-
       // Genesis Information
       int      GenesisIndex;
       unixtime GenesisTime;

diff --git a/met/src/tools/other/gen_ens_prod/gen_ens_prod.cc b/met/src/tools/other/gen_ens_prod/gen_ens_prod.cc
@@ -55,6 +55,8 @@ static void process_command_line(int, char **);
 static void process_grid(const Grid &);
 static void process_ensemble();
 
+static void get_climo_mean_stdev(GenEnsProdVarInfo *, int,
+                                 bool, int, DataPlane &, DataPlane &);
 static void get_ens_mean_stdev(GenEnsProdVarInfo *, DataPlane &, DataPlane &);
 static bool get_data_plane(const char *, GrdFileType, VarInfo *, DataPlane &);
 
@@ -272,7 +274,7 @@ void process_grid(const Grid &fcst_grid) {
 
 void process_ensemble() {
    int i_var, i_ens, n_ens_vld, n_ens_inputs;
-   bool need_reset;
+   bool need_reset, set_climo_ens_mem_id;
    DataPlane ens_dp, ctrl_dp;
    DataPlane cmn_dp, csd_dp;
    DataPlane emn_dp, esd_dp;
@@ -287,6 +289,13 @@ void process_ensemble() {
       // Need to reinitialize counts and sums for each ensemble field
       need_reset = true;
 
+      // When normalizing relative to climatology with MET_ENS_MEMBER_ID set,
+      // read climatology separately for each member
+      set_climo_ens_mem_id =
+         (conf_info.ens_member_ids.n() > 1) &&
+         ((*var_it)->normalize == NormalizeType_ClimoAnom ||
+          (*var_it)->normalize == NormalizeType_ClimoStdAnom);
+
       // Print out the normalization flag
       cs << cs_erase;
       if((*var_it)->normalize != NormalizeType_None) {
@@ -310,9 +319,9 @@ void process_ensemble() {
          // Skip bad data files
          if(!ens_file_vld[(*var_it)->get_file_index(i_ens)]) continue;
 
-         mlog << Debug(3) << "\n"
-              << "Reading field: "
-              << var_info->magic_str() << "\n";
+         mlog << Debug(3)
+              << "\nReading ensemble field \""
+              << var_info->magic_str() << "\".\n";
 
          // Read data and track the valid data count
          if(!get_data_plane(ens_file.c_str(), etype,
@@ -335,17 +344,13 @@ void process_ensemble() {
             clear_counts();
 
             // Read climatology data for this field
-            cmn_dp = read_climo_data_plane(
-                        conf_info.conf.lookup_array(conf_key_climo_mean_field, false),
-                        i_var, ens_valid_ut, grid);
-
-            csd_dp = read_climo_data_plane(
-                        conf_info.conf.lookup_array(conf_key_climo_stdev_field, false),
-                        i_var, ens_valid_ut, grid);
+            get_climo_mean_stdev((*var_it), i_var,
+                                 set_climo_ens_mem_id,
+                                 i_ens, cmn_dp, csd_dp);
 
             // Compute the ensemble summary data, if needed
             if((*var_it)->normalize == NormalizeType_FcstAnom ||
-               (*var_it)->normalize == NormalizeType_FcstStdAnom ) {
+               (*var_it)->normalize == NormalizeType_FcstStdAnom) {
                get_ens_mean_stdev((*var_it), emn_dp, esd_dp);
             }
             else {
@@ -371,6 +376,13 @@ void process_ensemble() {
                   exit(1);
                }
 
+               // Read climo data with MET_ENS_MEMBER_ID set
+               if(set_climo_ens_mem_id) {
+                  get_climo_mean_stdev((*var_it), i_var,
+                                       set_climo_ens_mem_id, i_ens,
+                                       cmn_dp, csd_dp);
+               }
+
                // Normalize, if requested
                if((*var_it)->normalize != NormalizeType_None) {
                   normalize_data(ctrl_dp, (*var_it)->normalize,
@@ -391,6 +403,13 @@ void process_ensemble() {
 
          } // end if need_reset
 
+         // Read climo data with MET_ENS_MEMBER_ID set
+         if(set_climo_ens_mem_id) {
+             get_climo_mean_stdev((*var_it), i_var,
+                                  set_climo_ens_mem_id, i_ens,
+                                  cmn_dp, csd_dp);
+         }
+
          // Normalize, if requested
          if((*var_it)->normalize != NormalizeType_None) {
             normalize_data(ens_dp, (*var_it)->normalize,
@@ -411,7 +430,7 @@ void process_ensemble() {
       if(((double) n_ens_vld/n_ens_inputs) < conf_info.vld_ens_thresh) {
          mlog << Error << "\nprocess_ensemble() -> "
               << n_ens_vld << " of " << n_ens_inputs
-	      << " (" << (double)n_ens_vld/n_ens_inputs << ")"
+              << " (" << (double)n_ens_vld/n_ens_inputs << ")"
               << " fields found for \"" << (*var_it)->get_var_info()->magic_str()
               << "\" does not meet the threshold specified by \""
               << conf_key_ens_ens_thresh << "\" (" << conf_info.vld_ens_thresh
@@ -430,6 +449,41 @@ void process_ensemble() {
 
 ////////////////////////////////////////////////////////////////////////
 
+void get_climo_mean_stdev(GenEnsProdVarInfo *ens_info, int i_var,
+                          bool set_ens_mem_id, int i_ens,
+                          DataPlane &cmn_dp, DataPlane &csd_dp) {
+
+   // Set the MET_ENS_MEMBER_ID environment variable
+   if(set_ens_mem_id) {
+      setenv(met_ens_member_id, ens_info->get_ens_member_id(i_ens).c_str(), 1);
+   }
+
+   mlog << Debug(4)
+        << "Reading climatology mean data for ensemble field \""
+        << ens_info->get_var_info(i_ens)->magic_str() << "\".\n";
+
+   cmn_dp = read_climo_data_plane(
+               conf_info.conf.lookup_array(conf_key_climo_mean_field, false),
+               i_var, ens_valid_ut, grid);
+
+   mlog << Debug(4)
+        << "Reading climatology standard deviation data for ensemble field \""
+        << ens_info->get_var_info(i_ens)->magic_str() << "\".\n";
+
+   csd_dp = read_climo_data_plane(
+               conf_info.conf.lookup_array(conf_key_climo_stdev_field, false),
+               i_var, ens_valid_ut, grid);
+
+   // Unset the MET_ENS_MEMBER_ID environment variable
+   if(set_ens_mem_id) {
+      unsetenv(met_ens_member_id);
+   }
+
+   return;
+}
+
+////////////////////////////////////////////////////////////////////////
+
 void get_ens_mean_stdev(GenEnsProdVarInfo *ens_info,
                         DataPlane &emn_dp, DataPlane &esd_dp) {
    int i_ens, nxy, j;
@@ -451,6 +505,7 @@ void get_ens_mean_stdev(GenEnsProdVarInfo *ens_info,
         << "Computing the ensemble mean and standard deviation for "
         << ens_info->raw_magic_str << ".\n";
 
+   nxy = 0;
    // Loop over the ensemble inputs
    for(i_ens=0; i_ens < ens_info->inputs_n(); i_ens++) {
 

diff --git a/met/src/tools/other/gen_ens_prod/gen_ens_prod.h b/met/src/tools/other/gen_ens_prod/gen_ens_prod.h
@@ -14,7 +14,7 @@
 //
 //   Mod#   Date      Name            Description
 //   ----   ----      ----            -----------
-//   000    09/10/21  Halley Gotway   Initial version (MET #1904).
+//   000    09/10/21  Halley Gotway   MET #1904 Initial version.
 //
 ////////////////////////////////////////////////////////////////////////
 

diff --git a/met/src/tools/other/gen_ens_prod/gen_ens_prod_conf_info.cc b/met/src/tools/other/gen_ens_prod/gen_ens_prod_conf_info.cc
@@ -213,6 +213,7 @@ void GenEnsProdConfInfo::process_config(GrdFileType etype, StringArray * ens_fil
          input_info.var_info = next_var;
          input_info.file_index = 0;
          input_info.file_list = ens_files;
+         input_info.ens_member_id = ens_member_ids[j];
          ens_info->add_input(input_info);
 
          // Add InputInfo to ens info list for each ensemble file provided
@@ -221,6 +222,7 @@ void GenEnsProdConfInfo::process_config(GrdFileType etype, StringArray * ens_fil
             input_info.var_info = NULL;
             input_info.file_index = k;
             input_info.file_list = ens_files;
+            input_info.ens_member_id = ens_member_ids[j];
             ens_info->add_input(input_info);
          } // end for k
-Original file line number
+Diff line change
@@ Expand Up / @@ -139,7 +139,7 @@ if ( tf_left ) return ( true ); @@
     const bool tf_right = right_child->check(x, cmn, csd);
-    return ( tf_left || tf_right );
+    return ( tf_right );
     }
@@ Expand Down @@