Feature #2745 mvmode enhancements

docs/Users_Guide/mode.rst

@@ -118,6 +118,23 @@ The **multivar_intensity_compare_fcst** and **multivar_intensity_compare_obs** c
 
 When regridding to the FCST or OBS field (e.g. to_grid = FCST), the first field of the field array is used from the forecast and observation field dictionaries, respectively. All regridding is then done to that grid. Other regrid options described in :ref:`regrid` can also be used as normal.
 
+"file_type" can be set independently for each input in multivariate mode. If not set for an input, MET uses file names and file content to determine the type.
+
+When setting a threshold to a percentile, some choices require both an observation input and a forecast input.  When this is the case, it's assumed the indices match, so for example if forecast input 1 has such a percentile setting, then observation input 1 will be used to compute the percentile.  Percentiles in which this will happen are:
+
+* SFP in an observation input
+  * The matching forecast input will be used to determine the threshold.  e.g. ">SFP33.3" in the 2nd observation input means greater than 33.3-rd percentile of the 2nd forecast input will be used as the threshold for that observation input.
+
+* SOP in a forecast input
+  * The matching observation input will be used to determine the threshold. e.g. ">SOP33.3" in the 2nd forecast input means greater than 33.3-rd percentile of the 2nd observation input will be used as the threshold for that forecast input.
+
+* "==FBIAS" in an observation input.
+  * e.g. "==FBIAS1" in an observation input to automatically de-bias the data, using a simple threshold in the matching forecast input. For example, when observation input 3 has "==FBIAS1", and forecast input 3 has ">5.0", MET applies the >5.0 threshold to the forecast and then chooses an observation threshold which results in a frequency bias of 1. The frequency bias can be any float value > 0.0.
+
+* "==FBIAS" in a forecast input.
+  * e.g. "==FBIAS1" in a forecast input to automatically de-bias the data, using a simple threshold in the matching observation input. For example, when forecast input 2 has "==FBIAS1", and observation input 2 has ">5.0", MET applies the >5.0 threshold to the observation and then chooses a forecast threshold which results in a frequency bias of 1.  The frequency bias can be any float value > 0.0.
+
+
 Practical Information
 =====================
 

internal/scripts/environment/development.seneca

@@ -1,34 +1,35 @@
 # Define the development environment for NCAR project machine seneca
-# Based on settings in /usr/local/src/met/README.snat
+# Based on settings in /nrit/ral/src/met/README.ral-it
 
 # Top-level MET project directory
 MET_PROJ_DIR=/d1/projects/MET
 
 # Variables required to build MET
 export MET_DEVELOPMENT=true
-export MET_DST=/usr/local
+export MET_DST=/nrit/ral
 export MET_PROJ=${MET_DST}/proj-9.2.1
-export MET_NETCDF=${MET_DST}/netcdf-4.7.0/gcc-8.3.0
-export MET_HDF5=${MET_DST}/hdf5-1.8.21
-export MET_HDFINC=${MET_DST}/hdf4-4.2.15/include/hdf
-export MET_HDFLIB=${MET_DST}/hdf4-4.2.15/lib
-export MET_HDFEOS=${MET_DST}/hdf-eos2-20v1
-export MET_BUFR=${MET_DST}
-export MET_GRIB2C=${MET_DST}
-export MET_GSL=${MET_PROJ_DIR}/MET_releases/external_libs/gnu_8.3.0
+export MET_NETCDF=${MET_DST}/netcdf-4.9.2/gnu-12.2.0
+export MET_HDF5=${MET_DST}/hdf5-1.14.2
+export MET_HDFINC=${MET_DST}/hdf4-4.2.16-2/include/hdf
+export MET_HDFLIB=${MET_DST}/hdf4-4.2.16-2/lib
+export MET_HDFEOS=${MET_DST}/hdf-eos2-3.0
+export MET_BUFR=${MET_DST}/nceplibs-1.4.0/bufr-12.0.1
+export MET_GRIB2C=${MET_DST}/nceplibs-1.4.0/g2c-1.6.2
+export MET_GSLINC=/usr/include
+export MET_GSLLIB=/usr/lib/x86_64-linux-gnu
 export MET_CAIROINC=/usr/include/cairo
 export MET_CAIROLIB=/usr/lib/x86_64-linux-gnu
 export MET_FREETYPEINC=/usr/include/freetype2
 export MET_FREETYPELIB=/usr/lib/x86_64-linux-gnu
-export MET_JASPER=${MET_DST}/jasper-1.900.1
-export MET_ATLAS=${MET_PROJ_DIR}/MET_releases/external_libs/gnu_8.3.0/atlas
-export MET_ECKIT=${MET_PROJ_DIR}/MET_releases/external_libs/gnu_8.3.0/eckit
+export MET_ATLAS=${MET_DST}/atlas-0.30.0
+export MET_ECKIT=${MET_DST}/eckit-1.20.2
+export JASPER=${MET_DST}/jasper-1.900.1
 
 # For Python 3 in met-9.0
-export MET_PYTHON=/usr/local/met-python3
-export MET_PYTHON_BIN_EXE=${MET_PYTHON}/bin/python3.8
-export MET_PYTHON_CC="-I${MET_PYTHON}/include/python3.8"
-export MET_PYTHON_LD="-L${MET_PYTHON}/lib -lpython3.8 -lcrypt -lpthread -ldl -lutil -lm"
+export MET_PYTHON=${MET_DST}/met-python3
+export MET_PYTHON_BIN_EXE=${MET_PYTHON}/bin/python3.10
+export MET_PYTHON_CC="-I${MET_PYTHON}/include/python3.10"
+export MET_PYTHON_LD="-L${MET_PYTHON}/lib -lpython3.10 -lcrypt -lpthread -ldl -lutil -lm"
 
 # -D__64BIT__ is required because we've compiled libgrib2c.a with that flag
 export CFLAGS="-DUNDERSCORE -fPIC -D__64BIT__ -g"
@@ -37,21 +38,24 @@ export CXXFLAGS=${CFLAGS}
 # Set LDFLAGS to include -rpath settings when compiling MET
 export LDFLAGS="-Wl,--disable-new-dtags"
 export LDFLAGS="${LDFLAGS} -Wl,-rpath,${MET_DST}/lib:${MET_HDFEOS}/lib:${MET_PROJ}/lib:${MET_NETCDF}/lib:${MET_DST}/zlib-1.2.11/lib:${MET_DST}/szip-2.1.1/lib"
-export LDFLAGS="${LDFLAGS} -Wl,-rpath,${MET_HDFLIB}:${MET_HDF5}/lib:${MET_GSL}/lib:${MET_PYTHON}/lib:${MET_JASPER}/lib"
-export LDFLAGS="${LDFLAGS} -L${MET_JASPER}/lib"
+export LDFLAGS="${LDFLAGS} -Wl,-rpath,${MET_HDFLIB}:${MET_HDF5}/lib:${MET_GSL}/lib:${MET_PYTHON}/lib:${JASPER}/lib"
+export LDFLAGS="${LDFLAGS} -L${JASPER}/lib"
 
 # Variables required to run MET
 export MET_TEST_INPUT=${MET_PROJ_DIR}/MET_test_data/unit_test
 export MET_FONT_DIR=${MET_TEST_INPUT}/fonts
 
 # Define Rscript to use a version with the ncdf4 package 1.17 or later 
-export MET_TEST_RSCRIPT=/usr/local/R-4.1.2/bin/Rscript
+export MET_TEST_RSCRIPT=/nrit/ral/R-4.3.1/bin/Rscript
+
+# Define runtime Python version
+export MET_TEST_MET_PYTHON_EXE=${MET_PYTHON_BIN_EXE}
 
 # This is a cron script -- create the shell environment for this job
 # - NCO is for ncdiff
 # - NetCDF is for ncdump.
-export PATH="/usr/local/nco/bin:/usr/local/netcdf/bin:\
-             /usr/local/sbin:/usr/local/bin:/usr/sbin:\
+export PATH="/nrit/ral/nco/bin:/nrit/ral/netcdf4/bin:\
+             /nrit/ral/sbin:/nrit/ral/bin:/usr/sbin:\
              /usr/bin:/sbin:/bin:/usr/bin/X11:/opt/bin:$PATH"
 
 # SonarQube

internal/test_unit/config/MODEConfig_multivar

@@ -0,0 +1,274 @@
+////////////////////////////////////////////////////////////////////////////////
+//
+// MODE configuration file.
+//
+// For additional information, see the MET_BASE/config/README file.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+//
+// Output model name to be written
+//
+// model =
+model = "HRRR";
+
+//
+// Output description to be written
+//
+// desc =
+
+//
+// Output observation type to be written
+//
+// obtype =
+obtype = "ANALYSIS";
+
+////////////////////////////////////////////////////////////////////////////////
+
+//
+// Verification grid
+//
+regrid = {to_grid = FCST;method = NEAREST;width = 1;vld_thresh = 0.5;}
+
+////////////////////////////////////////////////////////////////////////////////
+
+//
+`// Approximate grid resolution (km)
+//
+// grid_res =
+// 
+
+////////////////////////////////////////////////////////////////////////////////
+
+//
+// Run all permutations of radius and threshold
+//
+// quilt =
+// ${METPLUS_QUILT}
+
+//
+// MODE Multivar boolean combination logic
+//
+//multivar_logic =
+//${METPLUS_MULTIVAR_LOGIC}
+multivar_logic = "#1 && #2 && #3";
+
+// keep this around to compare to older versions, the new version doesn't read this
+multivar_intensity_flag = [FALSE, TRUE, TRUE];  
+
+multivar_intensity_compare_fcst = [ 2, 3 ];
+multivar_intensity_compare_obs = [ 2, 3 ];
+
+//
+// Forecast and observation fields to be verified
+//
+fcst = {
+   //${METPLUS_FCST_FIELD}
+   field = [{ name="CSNOW"; level="L0"; conv_radius = 0; conv_thresh = ==1; file_type = GRIB2; merge_flag = NONE; },
+            { name="VIS"; level="L0"; conv_radius = 5; conv_thresh = <=804.672; merge_thresh = <=1207.008; merge_flag = THRESH; },
+            { name="WIND"; level="Z10"; conv_radius = 5; conv_thresh = >=8.9408; merge_thresh = >=6.7056; merge_flag = THRESH; }];
+
+   //${METPLUS_FCST_CENSOR_THRESH}
+   //${METPLUS_FCST_CENSOR_VAL}
+   //${METPLUS_FCST_CONV_RADIUS}
+   //${METPLUS_FCST_CONV_THRESH}
+   //${METPLUS_FCST_VLD_THRESH}
+   //${METPLUS_FCST_FILTER_ATTR_NAME}
+   filter_attr_name = ["AREA"];
+
+   //${METPLUS_FCST_FILTER_ATTR_THRESH}
+   filter_attr_thresh = [>=25];
+
+   //${METPLUS_FCST_MERGE_THRESH}
+   //${METPLUS_FCST_MERGE_FLAG}
+   //${METPLUS_FCST_FILE_TYPE}
+   //file_type = GRIB2;
+   multivar_name = "Snow";
+   multivar_level = "LO";
+}
+
+obs = {
+   //${METPLUS_OBS_FIELD}
+   field = [{ name="PrecipFlag"; level="L0"; conv_radius = 0; conv_thresh = ==3; file_type = GRIB2; merge_flag = NONE; },
+            { name="VIS"; level="L0"; conv_radius = 5; conv_thresh = <=804.672; merge_thresh = <=1207.008; merge_flag = THRESH; },
+	    { name="WIND"; level="Z10"; conv_radius = 5; conv_thresh = >=8.9408; merge_thresh = >=6.7056; merge_flag = THRESH; } ];
+
+   //${METPLUS_OBS_CENSOR_THRESH}
+   //${METPLUS_OBS_CENSOR_VAL}
+   //${METPLUS_OBS_CONV_RADIUS}
+   //${METPLUS_OBS_CONV_THRESH}
+   //${METPLUS_OBS_VLD_THRESH}
+
+   //${METPLUS_OBS_FILTER_ATTR_NAME}
+   filter_attr_name = ["AREA"];
+
+   //${METPLUS_OBS_FILTER_ATTR_THRESH}
+   filter_attr_thresh = [>=25];
+
+   // ${METPLUS_OBS_MERGE_THRESH}
+   // ${METPLUS_OBS_MERGE_FLAG}
+   // ${METPLUS_OBS_FILE_TYPE}
+   multivar_name = "Precip";
+   multivar_level = "LO";
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+//
+// Handle missing data
+//
+// mask_missing_flag =
+// ${METPLUS_MASK_MISSING_FLAG}
+
+//
+// Match objects between the forecast and observation fields
+//
+//match_flag =
+//${METPLUS_MATCH_FLAG}
+match_flag = MERGE_BOTH;
+//
+// Maximum centroid distance for objects to be compared
+//
+//max_centroid_dist =
+//${METPLUS_MAX_CENTROID_DIST}
+
+////////////////////////////////////////////////////////////////////////////////
+
+//
+// Verification masking regions
+//
+//mask = {
+//${METPLUS_MASK_DICT}
+
+////////////////////////////////////////////////////////////////////////////////
+
+//
+// Fuzzy engine weights
+//
+//weight = {
+//${METPLUS_WEIGHT_DICT}
+
+////////////////////////////////////////////////////////////////////////////////
+
+//
+// Fuzzy engine interest functions
+//
+interest_function = {
+
+   //${METPLUS_INTEREST_FUNCTION_CENTROID_DIST}
+
+   //${METPLUS_INTEREST_FUNCTION_BOUNDARY_DIST}
+
+   //${METPLUS_INTEREST_FUNCTION_CONVEX_HULL_DIST}
+
+   angle_diff = (
+      (  0.0, 1.0 )
+      ( 30.0, 1.0 )
+      ( 90.0, 0.0 )
+   );
+
+   aspect_diff = (
+      (  0.00, 1.0 )
+      (  0.10, 1.0 )
+      (  0.75, 0.0 )
+   );
+
+   corner   = 0.8;
+   ratio_if = (
+      (    0.0, 0.0 )
+      ( corner, 1.0 )
+      (    1.0, 1.0 )
+   );
+
+   area_ratio = ratio_if;
+
+   int_area_ratio = (
+      ( 0.00, 0.00 )
+      ( 0.10, 0.50 )
+      ( 0.25, 1.00 )
+      ( 1.00, 1.00 )
+   );
+
+   curvature_ratio = ratio_if;
+
+   complexity_ratio = ratio_if;
+
+   inten_perc_ratio = ratio_if;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+//
+// Total interest threshold for determining matches
+//
+//total_interest_thresh =
+//${METPLUS_TOTAL_INTEREST_THRESH}
+
+//
+// Interest threshold for printing output pair information
+//
+print_interest_thresh = 0.0;
+
+////////////////////////////////////////////////////////////////////////////////
+
+//
+// Plotting information
+//
+met_data_dir = "MET_BASE";
+
+fcst_raw_plot = {
+   color_table      = "MET_BASE/colortables/met_default.ctable";
+   plot_min         = 0.0;
+   plot_max         = 0.0;
+}
+
+obs_raw_plot = {
+   color_table      = "MET_BASE/colortables/met_default.ctable";
+   plot_min         = 0.0;
+   plot_max         = 0.0;
+}
+
+object_plot = {
+   color_table      = "MET_BASE/colortables/mode_obj.ctable";
+}
+
+//
+// Boolean for plotting on the region of valid data within the domain
+//
+plot_valid_flag = FALSE;
+
+//
+// Plot polyline edges using great circle arcs instead of straight lines
+//
+plot_gcarc_flag = FALSE;
+
+////////////////////////////////////////////////////////////////////////////////
+
+//
+// NetCDF matched pairs, PostScript, and contingency table output files
+//
+//ps_plot_flag =
+//${METPLUS_PS_PLOT_FLAG}
+
+//nc_pairs_flag = {
+//${METPLUS_NC_PAIRS_FLAG_DICT}
+
+//ct_stats_flag =
+//${METPLUS_CT_STATS_FLAG}
+
+
+////////////////////////////////////////////////////////////////////////////////
+
+shift_right = 0;   //  grid squares
+
+////////////////////////////////////////////////////////////////////////////////
+
+//${METPLUS_OUTPUT_PREFIX}
+//version        = "V10.0";
+
+//tmp_dir = "${MET_TMP_DIR}";
+tmp_dir = "/tmp";
+
+////////////////////////////////////////////////////////////////////////////////
+
+//${METPLUS_MET_CONFIG_OVERRIDES}