From 39a5233d324a735410036a30aa7edc7360e2a2af Mon Sep 17 00:00:00 2001
From: Howard Soh <hsoh@kiowa.rap.ucar.edu>
Date: Mon, 29 Nov 2021 13:47:45 -0700
Subject: [PATCH 1/2] #1936 Excludes precip3hr, precip6hr, precip12hr, and
 precip10min from required variables for MESONET

---
 met/src/tools/other/madis2nc/madis2nc.cc | 26 +++++++++---------------
 1 file changed, 10 insertions(+), 16 deletions(-)

diff --git a/met/src/tools/other/madis2nc/madis2nc.cc b/met/src/tools/other/madis2nc/madis2nc.cc
index b1192f566f..6f48a30ccc 100644
--- a/met/src/tools/other/madis2nc/madis2nc.cc
+++ b/met/src/tools/other/madis2nc/madis2nc.cc
@@ -84,7 +84,7 @@ static void clean_up();
 static void setup_netcdf_out(int nhdr);
 
 static bool get_filtered_nc_data(NcVar var, float *data, const long dim,
-                                 const long cur, const char *var_name);
+                                 const long cur, const char *var_name, bool required=true);
 static bool get_filtered_nc_data_2d(NcVar var, int *data, const long *dim,
                                     const long *cur, const char *var_name, bool count_bad=false);
 static bool get_filtered_nc_data_2d(NcVar var, float *data, const long *dim,
@@ -415,7 +415,7 @@ void setup_netcdf_out(int nhdr) {
 
 static bool get_filtered_nc_data(NcVar var, float *data,
                                  const long dim, const long cur,
-                                 const char *var_name) {
+                                 const char *var_name, bool required) {
 
    bool status = false;
    float in_fill_value;
@@ -439,15 +439,13 @@ static bool get_filtered_nc_data(NcVar var, float *data,
               << "Fail to read data [" << var_name << "].\n\n";
       }
    }
-   else {
+   else if (required) {
       mlog << Error << "\n" << method_name
            << "Can not read a NetCDF data because the variable [" << var_name << "] is missing.\n\n";
    }
-   if (!status) {
-      for (int idx=0; idx<dim; idx++) {
-         data[idx] = bad_data_float;
-         rej_fill++;
-      }
+   if(!status) {
+      for (int idx=0; idx<dim; idx++) data[idx] = bad_data_float;
+      rej_fill += dim;
    }
    return status;
 
@@ -2715,10 +2713,6 @@ void process_madis_mesonet(NcFile *&f_in) {
    if (IS_INVALID_NC(in_soilTemperature_var)) missing_vars.add("soilTemperature");
    if (IS_INVALID_NC(in_minTemp24Hour_var)) missing_vars.add("minTemp24Hour");
    if (IS_INVALID_NC(in_maxTemp24Hour_var)) missing_vars.add("maxTemp24Hour");
-   if (IS_INVALID_NC(in_precip3hr_var)) missing_vars.add("precip3hr");
-   if (IS_INVALID_NC(in_precip6hr_var)) missing_vars.add("precip6hr");
-   if (IS_INVALID_NC(in_precip12hr_var)) missing_vars.add("precip12hr");
-   if (IS_INVALID_NC(in_precip10min_var)) missing_vars.add("precip10min");
    if (IS_INVALID_NC(in_precip1min_var)) missing_vars.add("precip1min");
    if (IS_INVALID_NC(in_windDir10_var)) missing_vars.add("windDir10");
    if (IS_INVALID_NC(in_windSpeed10_var)) missing_vars.add("windSpeed10");
@@ -2922,10 +2916,10 @@ void process_madis_mesonet(NcFile *&f_in) {
       get_filtered_nc_data(in_soilTemperature_var,  soilTemperature_arr,  buf_size, i_hdr_s, "soilTemperature" );
       get_filtered_nc_data(in_minTemp24Hour_var,    minTemp24Hour_arr,    buf_size, i_hdr_s, "minTemp24Hour"   );
       get_filtered_nc_data(in_maxTemp24Hour_var,    maxTemp24Hour_arr,    buf_size, i_hdr_s, "maxTemp24Hour"   );
-      get_filtered_nc_data(in_precip3hr_var,        precip3hr_arr,        buf_size, i_hdr_s, "precip3hr"       );
-      get_filtered_nc_data(in_precip6hr_var,        precip6hr_arr,        buf_size, i_hdr_s, "precip6hr"       );
-      get_filtered_nc_data(in_precip12hr_var,       precip12hr_arr,       buf_size, i_hdr_s, "precip12hr"      );
-      get_filtered_nc_data(in_precip10min_var,      precip10min_arr,      buf_size, i_hdr_s, "precip10min"     );
+      get_filtered_nc_data(in_precip3hr_var,        precip3hr_arr,        buf_size, i_hdr_s, "precip3hr"       , false);
+      get_filtered_nc_data(in_precip6hr_var,        precip6hr_arr,        buf_size, i_hdr_s, "precip6hr"       , false);
+      get_filtered_nc_data(in_precip12hr_var,       precip12hr_arr,       buf_size, i_hdr_s, "precip12hr"      , false);
+      get_filtered_nc_data(in_precip10min_var,      precip10min_arr,      buf_size, i_hdr_s, "precip10min"     , false);
       get_filtered_nc_data(in_precip1min_var,       precip1min_arr,       buf_size, i_hdr_s, "precip1min"      );
       get_filtered_nc_data(in_windDir10_var,        windDir10_arr,        buf_size, i_hdr_s, "windDir10"       );
       get_filtered_nc_data(in_windSpeed10_var,      windSpeed10_arr,      buf_size, i_hdr_s, "windSpeed10"     );

From d35190e8c1ad5b3d60b7e5a2863ece30bc0d15c0 Mon Sep 17 00:00:00 2001
From: Howard Soh <hsoh@kiowa.rap.ucar.edu>
Date: Wed, 1 Dec 2021 13:50:10 -0700
Subject: [PATCH 2/2] #1936 Added an unit test for newer MESONET file

---
 test/xml/unit_madis2nc.xml | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/test/xml/unit_madis2nc.xml b/test/xml/unit_madis2nc.xml
index 0b0020e8ba..802eebc3c7 100644
--- a/test/xml/unit_madis2nc.xml
+++ b/test/xml/unit_madis2nc.xml
@@ -130,6 +130,18 @@
     </output>
   </test>
 
+  <test name="madis2nc_MESONET_optional_vars">
+    <exec>&MET_BIN;/madis2nc</exec>
+    <param> \
+      &DATA_DIR_OBS;/madis/mesonet/mesonet_20170101_0000.nc \
+      &OUTPUT_DIR;/madis2nc/mesonet_20170101_0000_F000.nc \
+      -type mesonet -mask_grid G207 -v 2
+    </param>
+    <output>
+      <point_nc>&OUTPUT_DIR;/madis2nc/mesonet_20170101_0000_F000.nc</point_nc>
+    </output>
+  </test>
+
   <test name="madis2nc_ACARS_PROFILES">
     <exec>&MET_BIN;/madis2nc</exec>
     <param> \