From 5e8baca6ce1e6816f07ef07ab7c0e1ddc9320e3f Mon Sep 17 00:00:00 2001 From: johnhg Date: Tue, 17 Nov 2020 15:27:20 -0700 Subject: [PATCH] Update main_v9.1-ref for the nightly build. (#1566) * Add debug level 4 message to list out the number of GRIB2 records inventoried. This helps debugging issues with MET potentially not reading all input GRIB2 records on WCOSS. * Bugfix 1554 main_v9.1 ncdump (#1555) * Bugfix 1562 main_v9.1 grid_diag (#1563) * Per #1562, check for bad data values before adding data to the PDF's for grid_diag. * Per #1562, removing the poly = CONUS.poly mask from GridDiagConfig_TMP. That settting masked a problem in the handling of missing data. Exercising the mask.poly option is tested in another unit test. This will change the output and break the nightly build, but that's good since we'll do more thorough testing. * Per #1508, change the verbosity in unit_tc_gen.xml from -v 2 to -v 5 to print out some additional log messages that may help in debugging the intermittent file list failure. --- met/src/libcode/vx_data2d_grib2/data2d_grib2.cc | 4 ++++ met/src/libcode/vx_series_data/series_pdf.cc | 9 ++++++--- scripts/docker/Dockerfile | 8 +++++--- test/config/GridDiagConfig_TMP | 2 +- test/xml/unit_tc_gen.xml | 2 +- 5 files changed, 17 insertions(+), 8 deletions(-) diff --git a/met/src/libcode/vx_data2d_grib2/data2d_grib2.cc b/met/src/libcode/vx_data2d_grib2/data2d_grib2.cc index d2c9abc7a0..2629ce50b5 100644 --- a/met/src/libcode/vx_data2d_grib2/data2d_grib2.cc +++ b/met/src/libcode/vx_data2d_grib2/data2d_grib2.cc @@ -930,6 +930,10 @@ void MetGrib2DataFile::read_grib2_record_list() { } // END: while( read_grib2_record() ) + mlog << Debug(4) << "MetGrib2DataFile::read_grib2_record_list() - " + << "processed " << rec_num - 1 << " records from GRIB2 file: " + << Filename << "\n"; + return; } diff --git a/met/src/libcode/vx_series_data/series_pdf.cc b/met/src/libcode/vx_series_data/series_pdf.cc index 8f7eeb0bf6..15f1c8cf15 100644 --- a/met/src/libcode/vx_series_data/series_pdf.cc +++ b/met/src/libcode/vx_series_data/series_pdf.cc @@ -66,8 +66,9 @@ void update_pdf( for(int i = 0; i < dp.nx(); i++) { for(int j = 0; j < dp.ny(); j++) { - if(!mp.s_is_on(i, j)) continue; double value = dp.get(i, j); + if(!mp.s_is_on(i, j) || + is_bad_data(value)) continue; int k = floor((value - min) / delta); if(k < 0) k = 0; if(k >= pdf.size()) k = pdf.size() - 1; @@ -92,12 +93,14 @@ void update_joint_pdf( for(int i = 0; i < dp_A.nx(); i++) { for(int j = 0; j < dp_A.ny(); j++) { - if(!mp.s_is_on(i, j)) continue; double value_A = dp_A.get(i, j); + double value_B = dp_B.get(i, j); + if(!mp.s_is_on(i, j) || + is_bad_data(value_A) || + is_bad_data(value_B)) continue; int k_A = floor((value_A - min_A) / delta_A); if(k_A < 0) k_A = 0; if(k_A >= n_A) k_A = n_A - 1; - double value_B = dp_B.get(i, j); int k_B = floor((value_B - min_B) / delta_B); if(k_B < 0) k_B = 0; if(k_B >= n_B) k_B = n_B - 1; diff --git a/scripts/docker/Dockerfile b/scripts/docker/Dockerfile index 5256542507..9a80592f6b 100644 --- a/scripts/docker/Dockerfile +++ b/scripts/docker/Dockerfile @@ -131,7 +131,7 @@ RUN echo "Downloading HDF4.2r3 from ${HDF4_URL}" \ && cd /met/external_libs/HDF4.2r3 \ && LOG_FILE=/met/logs/HDF4.2r3_configure.log \ && echo "Configuring HDF4.2r3 and writing log file ${LOG_FILE}" \ - && ./configure --prefix=/usr/local --disable-netcdf > ${LOG_FILE} \ + && ./configure --prefix=/usr/local/hdf --disable-netcdf > ${LOG_FILE} \ && cat mfhdf/hdiff/Makefile | sed 's/LIBS = -ljpeg -lz/LIBS = -ljpeg -lz -lm/g' > Makefile_NEW \ && mv -f Makefile_NEW mfhdf/hdiff/Makefile \ && LOG_FILE=/met/logs/HDF4.2r3_make_install.log \ @@ -142,11 +142,12 @@ RUN echo "Downloading HDF4.2r3 from ${HDF4_URL}" \ && cd /met/external_libs/hdfeos \ && LOG_FILE=/met/logs/hdfeos_configure.log \ && echo "Configuring hdfeos and writing log file ${LOG_FILE}" \ - && ./configure --prefix=/usr/local --with-hdf4=/usr/local CC=/usr/local/bin/h4cc > ${LOG_FILE} \ + && ./configure --prefix=/usr/local/hdfeos --with-hdf4=/usr/local/hdf CC=/usr/local/hdf/bin/h4cc > ${LOG_FILE} \ && LOG_FILE=/met/logs/hdfeos_make_install.log \ && echo "Compiling hdfeos and writing log file ${LOG_FILE}" \ && make install > ${LOG_FILE} \ - && cp include/*.h /usr/local/include/. \ + && mkdir /usr/local/hdfeos/include \ + && cp include/*.h /usr/local/hdfeos/include/. \ && cd /met/external_libs \ && rm -rf HDF4.2r3 hdfeos @@ -163,6 +164,7 @@ RUN echo "Checking out MET ${MET_GIT_NAME} from ${MET_GIT_URL}" \ && ./bootstrap \ && echo "Configuring MET ${MET_GIT_NAME} and writing log file ${LOG_FILE}" \ && ./configure --enable-grib2 --enable-mode_graphics --enable-modis --enable-lidar2nc --enable-python \ + MET_HDF=/usr/local/hdf MET_HDFEOS=/usr/local/hdfeos \ MET_FREETYPEINC=/usr/include/freetype2 MET_FREETYPELIB=/usr/lib \ MET_CAIROINC=/usr/include/cairo MET_CAIROLIB=/usr/lib \ MET_PYTHON_CC='-I/usr/include/python3.6m' MET_PYTHON_LD='-lpython3.6m' > ${LOG_FILE} \ diff --git a/test/config/GridDiagConfig_TMP b/test/config/GridDiagConfig_TMP index ef1a4c3dcf..c8d41e2715 100644 --- a/test/config/GridDiagConfig_TMP +++ b/test/config/GridDiagConfig_TMP @@ -62,7 +62,7 @@ data = { mask = { grid = ""; - poly = "MET_BASE/poly/CONUS.poly"; + poly = ""; } //////////////////////////////////////////////////////////////////////////////// diff --git a/test/xml/unit_tc_gen.xml b/test/xml/unit_tc_gen.xml index 4f4d67ecf3..9f50eeca5a 100644 --- a/test/xml/unit_tc_gen.xml +++ b/test/xml/unit_tc_gen.xml @@ -22,7 +22,7 @@ -config &CONFIG_DIR;/TCGenConfig_2016 \ -out &OUTPUT_DIR;/tc_gen/tc_gen_2016 \ -log &OUTPUT_DIR;/tc_gen/tc_gen_2016.log \ - -v 2 + -v 5 &OUTPUT_DIR;/tc_gen/tc_gen_2016.stat