From ef855c5ab26fcb7de8ec97300aae897a0093b231 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Tue, 4 Oct 2022 12:35:36 -0600 Subject: [PATCH 01/58] Per #392, add hooks for storing diagnostic info alongside the track info. --- src/libcode/vx_tc_util/track_info.cc | 28 ++++++++++++++++++++++----- src/libcode/vx_tc_util/track_info.h | 8 ++++++++ src/libcode/vx_tc_util/track_point.cc | 6 ++++++ src/libcode/vx_tc_util/track_point.h | 4 ++++ 4 files changed, 41 insertions(+), 5 deletions(-) diff --git a/src/libcode/vx_tc_util/track_info.cc b/src/libcode/vx_tc_util/track_info.cc index 4a4c34900c..bb85eaa598 100644 --- a/src/libcode/vx_tc_util/track_info.cc +++ b/src/libcode/vx_tc_util/track_info.cc @@ -92,6 +92,7 @@ void TrackInfo::clear() { MaxValidTime = (unixtime) 0; MinWarmCore = (unixtime) 0; MaxWarmCore = (unixtime) 0; + DiagName.clear(); TrackLines.clear(); clear_points(); @@ -133,7 +134,8 @@ void TrackInfo::dump(ostream &out, int indent_depth) const { out << prefix << "MaxWarmCore = \"" << (MaxWarmCore > 0 ? unix_to_yyyymmdd_hhmmss(MaxWarmCore).text() : na_str) << "\n"; out << prefix << "NPoints = " << NPoints << "\n"; out << prefix << "NAlloc = " << NAlloc << "\n"; - out << prefix << "NTrackLines = " << TrackLines.n_elements() << "\n"; + out << prefix << "NDiag = " << DiagName.n() << "\n"; + out << prefix << "NTrackLines = " << TrackLines.n() << "\n"; for(i=0; i " + << "range check error for index value " << i << "\n\n"; + exit(1); + } + + return(DiagName[i].c_str()); +} + +//////////////////////////////////////////////////////////////////////// + void TrackInfo::add(const TrackPoint &p) { extend(NPoints + 1, false); @@ -920,7 +938,7 @@ TrackInfo consensus(const TrackInfoArray &tracks, } // Loop through the lead times and construct a TrackPoint for each - for(i=0, skip=false; i 180.0 && plon[j] < 0.0 ? 360.0 : 0.0); lon_avg += (plon[j] + lon_shift); } @@ -1030,7 +1048,7 @@ bool has_storm_id(const StringArray &storm_id, bool match = false; // Loop over the storm id entries - for(i=0; i Date: Tue, 4 Oct 2022 12:36:18 -0600 Subject: [PATCH 02/58] Per #392, add hooks for writing a new TCDIAG output line type. --- src/libcode/vx_tc_util/tc_columns.cc | 87 ++++++++++++++++++++++++++ src/libcode/vx_tc_util/tc_columns.h | 13 +++- src/libcode/vx_tc_util/tc_stat_line.cc | 2 + src/libcode/vx_tc_util/tc_stat_line.h | 2 + 4 files changed, 103 insertions(+), 1 deletion(-) diff --git a/src/libcode/vx_tc_util/tc_columns.cc b/src/libcode/vx_tc_util/tc_columns.cc index 1c71d754ff..ab3d939096 100644 --- a/src/libcode/vx_tc_util/tc_columns.cc +++ b/src/libcode/vx_tc_util/tc_columns.cc @@ -139,6 +139,37 @@ void write_prob_rirw_header_row(int hdr_flag, int n_thresh, AsciiTable &at, //////////////////////////////////////////////////////////////////////// +void write_tc_diag_header_row(int hdr_flag, int n_diag, AsciiTable &at, + int r, int c) { + int i; + ConcatString s; + char tmp_str[max_str_len]; + + // Write the header column names if requested + if(hdr_flag) { + for(i=0; i i) { + hdr.set_desc((string)p.line(i)->get_item("DESC", false)); + } + + // Write the header columns + write_tc_header_cols(hdr, at, i_row); + + // Write the data columns + write_tc_diag_cols(p, i, at, i_row, n_tc_header_cols); + + // Increment the row counter + i_row++; + } + + return; +} + + //////////////////////////////////////////////////////////////////////// void write_tc_header_cols(const TcHdrColumns &hdr, @@ -364,6 +435,22 @@ void write_prob_rirw_cols(const ProbRIRWPairInfo &p, int i, //////////////////////////////////////////////////////////////////////// +void write_tc_diag_cols(const TrackPairInfo &p, int i, + AsciiTable &at, int r, int c) { + int j; + + // Write all the diagnostics + at.set_entry(r, c++, p.adeck().n_diag()); + for(j=0; j Date: Tue, 4 Oct 2022 17:03:47 -0600 Subject: [PATCH 03/58] Per #392, continue working on hooks to store/write TCDIAG output lines interleaved with the TCMPR data. --- data/config/TCPairsConfig_default | 5 + src/basic/vx_config/config_constants.h | 1 + src/libcode/vx_tc_util/tc_columns.cc | 149 ++++++++---------- src/libcode/vx_tc_util/tc_columns.h | 28 ++-- src/libcode/vx_tc_util/tc_stat_line.cc | 4 +- src/libcode/vx_tc_util/tc_stat_line.h | 4 +- src/libcode/vx_tc_util/track_info.cc | 10 ++ src/libcode/vx_tc_util/track_info.h | 1 + src/tools/tc_utils/tc_pairs/tc_pairs.cc | 31 +++- .../tc_utils/tc_pairs/tc_pairs_conf_info.cc | 4 + .../tc_utils/tc_pairs/tc_pairs_conf_info.h | 3 + src/tools/tc_utils/tc_stat/tc_stat_job.cc | 2 +- 12 files changed, 134 insertions(+), 108 deletions(-) diff --git a/data/config/TCPairsConfig_default b/data/config/TCPairsConfig_default index ed3c659d5f..263dcb0445 100644 --- a/data/config/TCPairsConfig_default +++ b/data/config/TCPairsConfig_default @@ -137,6 +137,11 @@ watch_warn = { time_offset = -14400; } +// +// Diagnostics to be extracted +// +diag_name = []; + // // Modify basin names to make them consistent across ATCF input files. // diff --git a/src/basic/vx_config/config_constants.h b/src/basic/vx_config/config_constants.h index 5cb03579d4..621a634325 100644 --- a/src/basic/vx_config/config_constants.h +++ b/src/basic/vx_config/config_constants.h @@ -1066,6 +1066,7 @@ static const char conf_key_dland_file[] = "dland_file"; static const char conf_key_basin_file[] = "basin_file"; static const char conf_key_track_watch_warn[] = "track_watch_warn"; static const char conf_key_watch_warn[] = "watch_warn"; +static const char conf_key_diag_name[] = "diag_name"; static const char conf_key_basin_map[] = "basin_map"; static const char conf_key_time_offset[] = "time_offset"; static const char conf_key_amodel[] = "amodel"; diff --git a/src/libcode/vx_tc_util/tc_columns.cc b/src/libcode/vx_tc_util/tc_columns.cc index ab3d939096..7953bac78d 100644 --- a/src/libcode/vx_tc_util/tc_columns.cc +++ b/src/libcode/vx_tc_util/tc_columns.cc @@ -108,7 +108,7 @@ void write_tc_mpr_header_row(int hdr_flag, AsciiTable &at, //////////////////////////////////////////////////////////////////////// -void write_prob_rirw_header_row(int hdr_flag, int n_thresh, AsciiTable &at, +void write_tc_diag_header_row(int hdr_flag, int n_diag, AsciiTable &at, int r, int c) { int i; ConcatString s; @@ -120,18 +120,18 @@ void write_prob_rirw_header_row(int hdr_flag, int n_thresh, AsciiTable &at, at.set_entry(r, c++, tc_header_cols[i]); } - // Write the static PROBRIRW header columns - for(i=0; i 0) { + + // TCDIAG line type + hdr.set_line_type((string) TCStatLineType_TCDIAG_Str); + + // Write the header columns + write_tc_header_cols(hdr, at, i_row); + + // Write the data columns + write_tc_diag_cols(p, i, at, i_row, n_tc_header_cols); + + // Increment the row counter + i_row++; + } } return; @@ -209,11 +223,11 @@ void write_tc_mpr_row(TcHdrColumns &hdr, const TrackPairInfo &p, //////////////////////////////////////////////////////////////////////// -void write_prob_rirw_row(TcHdrColumns &hdr, const ProbRIRWPairInfo &p, - AsciiTable &at, int &i_row) { +void write_prob_rirw_pair_info(TcHdrColumns &hdr, const ProbRIRWPairInfo &p, + AsciiTable &at, int &i_row) { // PROBRIRW line type - hdr.set_line_type((string)"PROBRIRW"); + hdr.set_line_type((string) TCStatLineType_ProbRIRW_Str); // Timing information hdr.set_init (p.prob_rirw().init()); @@ -233,46 +247,6 @@ void write_prob_rirw_row(TcHdrColumns &hdr, const ProbRIRWPairInfo &p, return; } -//////////////////////////////////////////////////////////////////////// - -void write_tc_diag_row(TcHdrColumns &hdr, const TrackPairInfo &p, - AsciiTable &at, int &i_row) { - int i; - - // TCDIAG line type - hdr.set_line_type((string)"TCDIAG"); - - // Loop through the TrackPairInfo points - for(i=0; i i) { - hdr.set_desc((string)p.line(i)->get_item("DESC", false)); - } - - // Write the header columns - write_tc_header_cols(hdr, at, i_row); - - // Write the data columns - write_tc_diag_cols(p, i, at, i_row, n_tc_header_cols); - - // Increment the row counter - i_row++; - } - - return; -} - - //////////////////////////////////////////////////////////////////////// void write_tc_header_cols(const TcHdrColumns &hdr, @@ -314,7 +288,7 @@ void write_tc_mpr_cols(const TrackPairInfo &p, int i, AsciiTable &at, int r, int c) { int j; - // Write tc_mpr columns + // Write TCMPR columns at.set_entry(r, c++, p.n_points()); at.set_entry(r, c++, i+1); at.set_entry(r, c++, cyclonelevel_to_string(p.bdeck()[i].level())); @@ -374,6 +348,25 @@ void write_tc_mpr_cols(const TrackPairInfo &p, int i, //////////////////////////////////////////////////////////////////////// +void write_tc_diag_cols(const TrackPairInfo &p, int i, + AsciiTable &at, int r, int c) { + int j; + + // Write TCDIAG columns + at.set_entry(r, c++, p.n_points()); + at.set_entry(r, c++, i+1); + at.set_entry(r, c++, p.adeck().n_diag()); + + for(j=0; j 0) { + n_row += p.n_points(); + n_col = max(n_col, get_n_tc_diag_cols(conf_info.DiagName.n())); + } + // Initialize the output AsciiTable - out_at.set_size(p.n_points() + 1, n_tc_header_cols + n_tc_mpr_cols); + out_at.set_size(n_row, n_tc_header_cols + n_col); setup_table(out_at); // Write the header row @@ -1999,7 +2022,7 @@ void write_tracks(const TrackPairInfoArray &p) { tchc.set_storm_name(p[i].bdeck().storm_name()); // Write the current TrackPairInfo object - write_tc_mpr_row(tchc, p[i], out_at, i_row); + write_track_pair_info(tchc, p[i], out_at, i_row); } // Write the AsciiTable contents and clean up @@ -2077,7 +2100,7 @@ void write_prob_rirw(const ProbRIRWPairInfoArray &p) { tchc.set_storm_name(p[i].bdeck()->storm_name()); // Write the current ProbRIRWPairInfo object - write_prob_rirw_row(tchc, p[i], out_at, i_row); + write_prob_rirw_pair_info(tchc, p[i], out_at, i_row); } // Write the AsciiTable contents and clean up diff --git a/src/tools/tc_utils/tc_pairs/tc_pairs_conf_info.cc b/src/tools/tc_utils/tc_pairs/tc_pairs_conf_info.cc index a378bbbd1d..72fdccd819 100644 --- a/src/tools/tc_utils/tc_pairs/tc_pairs_conf_info.cc +++ b/src/tools/tc_utils/tc_pairs/tc_pairs_conf_info.cc @@ -96,6 +96,7 @@ void TCPairsConfInfo::clear() { DLandFile.clear(); WatchWarnFile.clear(); WatchWarnOffset = bad_data_int; + DiagName.clear(); BasinMap.clear(); Version.clear(); @@ -305,6 +306,9 @@ void TCPairsConfInfo::process_config() { // Conf: WatchWarnOffset WatchWarnOffset = dict->lookup_int(conf_key_time_offset); + // Conf: DiagName + DiagName = dict->lookup_string_array(conf_key_diag_name); + // Conf: BasinMap BasinMap = parse_conf_key_value_map(dict, conf_key_basin_map); diff --git a/src/tools/tc_utils/tc_pairs/tc_pairs_conf_info.h b/src/tools/tc_utils/tc_pairs/tc_pairs_conf_info.h index 3fc82f2671..dff8b1acbc 100644 --- a/src/tools/tc_utils/tc_pairs/tc_pairs_conf_info.h +++ b/src/tools/tc_utils/tc_pairs/tc_pairs_conf_info.h @@ -111,6 +111,9 @@ class TCPairsConfInfo { // Watch/warnings time offset int WatchWarnOffset; + // Diagnostics to be extracted + StringArray DiagName; + // Basin Map std::map BasinMap; diff --git a/src/tools/tc_utils/tc_stat/tc_stat_job.cc b/src/tools/tc_utils/tc_stat/tc_stat_job.cc index 79e217ffa4..f2d47ede1d 100644 --- a/src/tools/tc_utils/tc_stat/tc_stat_job.cc +++ b/src/tools/tc_utils/tc_stat/tc_stat_job.cc @@ -1138,7 +1138,7 @@ void TCStatJob::dump_pair(const TrackPairInfo &pair, ofstream *out) { // Write the TrackPairInfo object i_row = hdr_row; - write_tc_mpr_row(tchc, pair, out_at, i_row); + write_track_pair_info(tchc, pair, out_at, i_row); // Write the AsciiTable to the file *out << out_at; From 6048ae6427a2eb639a8e6bc004a4629c17ddee1f Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Thu, 6 Oct 2022 13:00:49 -0600 Subject: [PATCH 04/58] Per #392, add -tcdiag and -lsdiag command line options for TC-Pairs and add vx_tc_util/diag_file.h/.cc. This compiles but the code doesn't actually do much yet. --- src/libcode/vx_tc_util/Makefile.am | 1 + src/libcode/vx_tc_util/diag_file.cc | 101 +++++++++++++++++++++ src/libcode/vx_tc_util/diag_file.h | 103 +++++++++++++++++++++ src/tools/tc_utils/tc_pairs/tc_pairs.cc | 114 ++++++++++++++++++++---- src/tools/tc_utils/tc_pairs/tc_pairs.h | 3 + 5 files changed, 303 insertions(+), 19 deletions(-) create mode 100644 src/libcode/vx_tc_util/diag_file.cc create mode 100644 src/libcode/vx_tc_util/diag_file.h diff --git a/src/libcode/vx_tc_util/Makefile.am b/src/libcode/vx_tc_util/Makefile.am index 390eb1e5ae..1000ecde48 100644 --- a/src/libcode/vx_tc_util/Makefile.am +++ b/src/libcode/vx_tc_util/Makefile.am @@ -28,6 +28,7 @@ libvx_tc_util_a_SOURCES = \ pair_data_genesis.cc pair_data_genesis.h \ tc_hdr_columns.cc tc_hdr_columns.h \ tc_columns.cc tc_columns.h \ + tc_diag.cc tc_diag.h \ tc_stat_line.cc tc_stat_line.h \ vx_tc_nc_util.h vx_tc_nc_util.cc \ vx_tc_util.h diff --git a/src/libcode/vx_tc_util/diag_file.cc b/src/libcode/vx_tc_util/diag_file.cc new file mode 100644 index 0000000000..a7b5a3209a --- /dev/null +++ b/src/libcode/vx_tc_util/diag_file.cc @@ -0,0 +1,101 @@ +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* +// ** Copyright UCAR (c) 1992 - 2022 +// ** University Corporation for Atmospheric Research (UCAR) +// ** National Center for Atmospheric Research (NCAR) +// ** Research Applications Lab (RAL) +// ** P.O.Box 3000, Boulder, Colorado, 80307-3000, USA +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* + +//////////////////////////////////////////////////////////////////////// + +using namespace std; + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "vx_util.h" +#include "vx_log.h" + +#include "diag_file.h" + +//////////////////////////////////////////////////////////////////////// +// +// Code for class DiagFile +// +//////////////////////////////////////////////////////////////////////// + +DiagFile::DiagFile() { + init_from_scratch(); +} + +//////////////////////////////////////////////////////////////////////// + +DiagFile::~DiagFile() { + close(); +} + +//////////////////////////////////////////////////////////////////////// + +DiagFile::DiagFile(const DiagFile &) { + mlog << Error << "\nDiagFile::DiagFile(const DiagFile &) -> " + << "should never be called!\n\n"; + exit(1); +} + +//////////////////////////////////////////////////////////////////////// + +DiagFile & DiagFile::operator=(const DiagFile &) { + mlog << Error + << "\nDiagFile::operator=(const DiagFile &) -> " + << "should never be called!\n\n"; + exit(1); + + return(*this); +} + +//////////////////////////////////////////////////////////////////////// + +void DiagFile::init_from_scratch() { + + // Initialize values + StormID.clear(); + Basin.clear(); + Cyclone.clear(); + Technique.clear(); + InitTime = (unixtime) 0; + + close(); + + return; +} + +//////////////////////////////////////////////////////////////////////// + +bool DiagFile::open_tcdiag(const char *path, const char *model_name) { + + close(); + + // JHG work here + + return ( false ); +} + +//////////////////////////////////////////////////////////////////////// + +bool DiagFile::open_lsdiag(const char *path, const char *model_name) { + + close(); + + // JHG work here + + return ( false ); +} + +//////////////////////////////////////////////////////////////////////// diff --git a/src/libcode/vx_tc_util/diag_file.h b/src/libcode/vx_tc_util/diag_file.h new file mode 100644 index 0000000000..9c7a023276 --- /dev/null +++ b/src/libcode/vx_tc_util/diag_file.h @@ -0,0 +1,103 @@ +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* +// ** Copyright UCAR (c) 1992 - 2022 +// ** University Corporation for Atmospheric Research (UCAR) +// ** National Center for Atmospheric Research (NCAR) +// ** Research Applications Lab (RAL) +// ** P.O.Box 3000, Boulder, Colorado, 80307-3000, USA +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* + +//////////////////////////////////////////////////////////////////////// + +#ifndef __DIAG_FILE_H__ +#define __DIAG_FILE_H__ + +//////////////////////////////////////////////////////////////////////// + +#include +#include + +#include "data_line.h" + +//////////////////////////////////////////////////////////////////////// +// +// LSDIAG files: +// - https://ftp.nhc.noaa.gov/atcf/lsdiag +// - Header: +// BBCC YYMMDD HH WS LAT LON 9999 BBCCYYYY +// BB is 2-letter basin name +// CC is 2-digit cyclone number +// YYMMDD HH is the initialization time +// WS is the wind speed +// +// TCDIAG files: +// - Add link to sample data +// - Header: +// * ATCF_ID YYYYMMDDHH * +// * BBNN BBNN * +// ATCF_ID is the technique (model) name +// YYYYMMDDHH is the initialization time +// BB is the 2-letter basin name +// CC is the 2-digit cyclone number +// +//////////////////////////////////////////////////////////////////////// + +class DiagFile : public LineDataFile { + + private: + + DiagFile(const DiagFile &); + DiagFile & operator=(const DiagFile &); + + protected: + + void init_from_scratch(); + + // Storm and model identification + ConcatString StormId; + ConcatString Basin; + ConcatString Cyclone; + ConcatString Technique; + unixtime InitTime; + + public: + + DiagFile(); + ~DiagFile(); + + // + // set stuff + // + + // + // get stuff + // + + const ConcatString & storm_id() const; + const ConcatString & basin() const; + const ConcatString & cyclone() const; + const ConcatString & technique() const; + const ConcatString & initials() const; + unixtime init() const; + + // + // do stuff + // + + bool open_tcdiag(const char *path, const char *model_name); + bool open_lsdiag(const char *path, const char *model_name); + +}; + +//////////////////////////////////////////////////////////////////////// + +inline const ConcatString & DiagFile::storm_id() const { return(StormId); } +inline const ConcatString & DiagFile::basin() const { return(Basin); } +inline const ConcatString & DiagFile::cyclone() const { return(Cyclone); } +inline const ConcatString & DiagFile::technique() const { return(Technique); } +inline unixtime DiagFile::init() const { return(InitTime); } + +//////////////////////////////////////////////////////////////////////// + +#endif /* __DIAG_FILE_H__ */ + +//////////////////////////////////////////////////////////////////////// diff --git a/src/tools/tc_utils/tc_pairs/tc_pairs.cc b/src/tools/tc_utils/tc_pairs/tc_pairs.cc index c8159dca15..6b9e91e1d2 100644 --- a/src/tools/tc_utils/tc_pairs/tc_pairs.cc +++ b/src/tools/tc_utils/tc_pairs/tc_pairs.cc @@ -35,6 +35,7 @@ // the gen_vx_mask tool. // 012 07/06/22 Howard Soh METplus-Internal #19 Rename main to met_main // 013 09/28/22 Prestopnik MET #2227 Remove namespace std from header files +// 014 10/06/22 Halley Gotway MET #392 Incorporate diagnostics // //////////////////////////////////////////////////////////////////////// @@ -123,7 +124,7 @@ static void process_prob_files (const StringArray &, static bool is_keeper (const ATCFLineBase *); static void filter_tracks (TrackInfoArray &); static void filter_probs (ProbInfoArray &); -static void process_diag_files (TrackInfoArray &); +static void process_diags (TrackInfoArray &); static bool check_masks (const MaskPoly &, const Grid &, const MaskPlane &, @@ -151,6 +152,10 @@ static void set_edeck (const StringArray &); static void set_bdeck (const StringArray &); static void set_atcf_source (const StringArray &, StringArray &, StringArray &); +static void set_tcdiag (const StringArray &); +static void set_lsdiag (const StringArray &); +static void set_diag_source (const StringArray &, + StringArray &, StringArray &); static void set_config (const StringArray &); static void set_out (const StringArray &); @@ -196,20 +201,20 @@ void process_command_line(int argc, char **argv) { cline.set_usage(usage); // Add function calls for the arguments - cline.add(set_adeck, "-adeck", -1); - cline.add(set_edeck, "-edeck", -1); - cline.add(set_bdeck, "-bdeck", -1); - cline.add(set_config, "-config", 1); - cline.add(set_out, "-out", 1); + cline.add(set_adeck, "-adeck", -1); + cline.add(set_edeck, "-edeck", -1); + cline.add(set_bdeck, "-bdeck", -1); + cline.add(set_bdeck, "-tcdiag", -1); + cline.add(set_bdeck, "-lsdiag", -1); + cline.add(set_config, "-config", 1); + cline.add(set_out, "-out", 1); // Parse the command line cline.parse(); // Check for the minimum number of arguments - if((adeck_source.n() == 0 && - edeck_source.n() == 0) || - bdeck_source.n() == 0 || - config_file.length() == 0) { + if((adeck_source.n() == 0 && edeck_source.n() == 0) || + bdeck_source.n() == 0 || config_file.length() == 0) { mlog << Error << "\nprocess_command_line(int argc, char **argv) -> " << "You must specify at least one source of ADECK or EDECK " @@ -243,6 +248,22 @@ void process_command_line(int argc, char **argv) { << bdeck_model_suffix[i] << "\n"; } + // List the input TCDIAG files + for(i=0; i " + << "the model name must be specified as " + << "\"model=string\".\n\n"; + usage(); + } + else { + name = sa[1]; + } + } + } + + // Parse the remaining sources + for(i=0; i Date: Thu, 6 Oct 2022 13:03:11 -0600 Subject: [PATCH 05/58] Per #392, run bootstrap on seneca to add diag_file.h/.cc to vx_tc_util/Makefile.in. --- src/libcode/vx_tc_util/Makefile.in | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/libcode/vx_tc_util/Makefile.in b/src/libcode/vx_tc_util/Makefile.in index b059aaf546..0a93f231bc 100644 --- a/src/libcode/vx_tc_util/Makefile.in +++ b/src/libcode/vx_tc_util/Makefile.in @@ -123,6 +123,7 @@ am_libvx_tc_util_a_OBJECTS = libvx_tc_util_a-atcf_line_base.$(OBJEXT) \ libvx_tc_util_a-pair_data_genesis.$(OBJEXT) \ libvx_tc_util_a-tc_hdr_columns.$(OBJEXT) \ libvx_tc_util_a-tc_columns.$(OBJEXT) \ + libvx_tc_util_a-tc_diag.$(OBJEXT) \ libvx_tc_util_a-tc_stat_line.$(OBJEXT) \ libvx_tc_util_a-vx_tc_nc_util.$(OBJEXT) libvx_tc_util_a_OBJECTS = $(am_libvx_tc_util_a_OBJECTS) @@ -153,6 +154,7 @@ am__depfiles_remade = ./$(DEPDIR)/libvx_tc_util_a-atcf_line_base.Po \ ./$(DEPDIR)/libvx_tc_util_a-prob_rirw_info.Po \ ./$(DEPDIR)/libvx_tc_util_a-prob_rirw_pair_info.Po \ ./$(DEPDIR)/libvx_tc_util_a-tc_columns.Po \ + ./$(DEPDIR)/libvx_tc_util_a-tc_diag.Po \ ./$(DEPDIR)/libvx_tc_util_a-tc_hdr_columns.Po \ ./$(DEPDIR)/libvx_tc_util_a-tc_stat_line.Po \ ./$(DEPDIR)/libvx_tc_util_a-track_info.Po \ @@ -382,6 +384,7 @@ libvx_tc_util_a_SOURCES = \ pair_data_genesis.cc pair_data_genesis.h \ tc_hdr_columns.cc tc_hdr_columns.h \ tc_columns.cc tc_columns.h \ + tc_diag.cc tc_diag.h \ tc_stat_line.cc tc_stat_line.h \ vx_tc_nc_util.h vx_tc_nc_util.cc \ vx_tc_util.h @@ -447,6 +450,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libvx_tc_util_a-prob_rirw_info.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libvx_tc_util_a-prob_rirw_pair_info.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libvx_tc_util_a-tc_columns.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libvx_tc_util_a-tc_diag.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libvx_tc_util_a-tc_hdr_columns.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libvx_tc_util_a-tc_stat_line.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libvx_tc_util_a-track_info.Po@am__quote@ # am--include-marker @@ -698,6 +702,20 @@ libvx_tc_util_a-tc_columns.obj: tc_columns.cc @AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ @am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libvx_tc_util_a_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o libvx_tc_util_a-tc_columns.obj `if test -f 'tc_columns.cc'; then $(CYGPATH_W) 'tc_columns.cc'; else $(CYGPATH_W) '$(srcdir)/tc_columns.cc'; fi` +libvx_tc_util_a-tc_diag.o: tc_diag.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libvx_tc_util_a_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT libvx_tc_util_a-tc_diag.o -MD -MP -MF $(DEPDIR)/libvx_tc_util_a-tc_diag.Tpo -c -o libvx_tc_util_a-tc_diag.o `test -f 'tc_diag.cc' || echo '$(srcdir)/'`tc_diag.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libvx_tc_util_a-tc_diag.Tpo $(DEPDIR)/libvx_tc_util_a-tc_diag.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='tc_diag.cc' object='libvx_tc_util_a-tc_diag.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libvx_tc_util_a_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o libvx_tc_util_a-tc_diag.o `test -f 'tc_diag.cc' || echo '$(srcdir)/'`tc_diag.cc + +libvx_tc_util_a-tc_diag.obj: tc_diag.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libvx_tc_util_a_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT libvx_tc_util_a-tc_diag.obj -MD -MP -MF $(DEPDIR)/libvx_tc_util_a-tc_diag.Tpo -c -o libvx_tc_util_a-tc_diag.obj `if test -f 'tc_diag.cc'; then $(CYGPATH_W) 'tc_diag.cc'; else $(CYGPATH_W) '$(srcdir)/tc_diag.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libvx_tc_util_a-tc_diag.Tpo $(DEPDIR)/libvx_tc_util_a-tc_diag.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='tc_diag.cc' object='libvx_tc_util_a-tc_diag.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libvx_tc_util_a_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o libvx_tc_util_a-tc_diag.obj `if test -f 'tc_diag.cc'; then $(CYGPATH_W) 'tc_diag.cc'; else $(CYGPATH_W) '$(srcdir)/tc_diag.cc'; fi` + libvx_tc_util_a-tc_stat_line.o: tc_stat_line.cc @am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libvx_tc_util_a_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT libvx_tc_util_a-tc_stat_line.o -MD -MP -MF $(DEPDIR)/libvx_tc_util_a-tc_stat_line.Tpo -c -o libvx_tc_util_a-tc_stat_line.o `test -f 'tc_stat_line.cc' || echo '$(srcdir)/'`tc_stat_line.cc @am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libvx_tc_util_a-tc_stat_line.Tpo $(DEPDIR)/libvx_tc_util_a-tc_stat_line.Po @@ -863,6 +881,7 @@ distclean: distclean-am -rm -f ./$(DEPDIR)/libvx_tc_util_a-prob_rirw_info.Po -rm -f ./$(DEPDIR)/libvx_tc_util_a-prob_rirw_pair_info.Po -rm -f ./$(DEPDIR)/libvx_tc_util_a-tc_columns.Po + -rm -f ./$(DEPDIR)/libvx_tc_util_a-tc_diag.Po -rm -f ./$(DEPDIR)/libvx_tc_util_a-tc_hdr_columns.Po -rm -f ./$(DEPDIR)/libvx_tc_util_a-tc_stat_line.Po -rm -f ./$(DEPDIR)/libvx_tc_util_a-track_info.Po @@ -926,6 +945,7 @@ maintainer-clean: maintainer-clean-am -rm -f ./$(DEPDIR)/libvx_tc_util_a-prob_rirw_info.Po -rm -f ./$(DEPDIR)/libvx_tc_util_a-prob_rirw_pair_info.Po -rm -f ./$(DEPDIR)/libvx_tc_util_a-tc_columns.Po + -rm -f ./$(DEPDIR)/libvx_tc_util_a-tc_diag.Po -rm -f ./$(DEPDIR)/libvx_tc_util_a-tc_hdr_columns.Po -rm -f ./$(DEPDIR)/libvx_tc_util_a-tc_stat_line.Po -rm -f ./$(DEPDIR)/libvx_tc_util_a-track_info.Po From d93606fa477924fe313d8379d9c2049c4de4a743 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Thu, 6 Oct 2022 13:05:55 -0600 Subject: [PATCH 06/58] Per #392, just including the new diag_file.h in vx_tc_util.h --- src/libcode/vx_tc_util/vx_tc_util.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libcode/vx_tc_util/vx_tc_util.h b/src/libcode/vx_tc_util/vx_tc_util.h index f0c1825cf2..902aa3f74c 100644 --- a/src/libcode/vx_tc_util/vx_tc_util.h +++ b/src/libcode/vx_tc_util/vx_tc_util.h @@ -14,6 +14,7 @@ //////////////////////////////////////////////////////////////////////// #include "atcf_track_line.h" +#include "diag_file.h" #include "track_point.h" #include "track_info.h" #include "track_pair_info.h" From 0be4b69f9406dfd91c5db815aa986e4a0ab4934b Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Thu, 6 Oct 2022 13:13:47 -0600 Subject: [PATCH 07/58] Per #392, rename tc_diag.h/.cc to diag_file.h/.cc --- src/libcode/vx_tc_util/Makefile.am | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libcode/vx_tc_util/Makefile.am b/src/libcode/vx_tc_util/Makefile.am index 1000ecde48..463cb5495a 100644 --- a/src/libcode/vx_tc_util/Makefile.am +++ b/src/libcode/vx_tc_util/Makefile.am @@ -15,6 +15,7 @@ libvx_tc_util_a_SOURCES = \ atcf_line_base.cc atcf_line_base.h atcf_offsets.h \ atcf_track_line.cc atcf_track_line.h \ atcf_prob_line.cc atcf_prob_line.h \ + diag_file.cc diag_file.h \ track_point.cc track_point.h \ track_info.cc track_info.h \ track_pair_info.cc track_pair_info.h \ @@ -28,7 +29,6 @@ libvx_tc_util_a_SOURCES = \ pair_data_genesis.cc pair_data_genesis.h \ tc_hdr_columns.cc tc_hdr_columns.h \ tc_columns.cc tc_columns.h \ - tc_diag.cc tc_diag.h \ tc_stat_line.cc tc_stat_line.h \ vx_tc_nc_util.h vx_tc_nc_util.cc \ vx_tc_util.h From 7c29c058d554798baed09280d937ce05fe316f97 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Thu, 6 Oct 2022 13:14:44 -0600 Subject: [PATCH 08/58] Per #392, updating Makefile.in accordingly. --- src/libcode/vx_tc_util/Makefile.in | 40 +++++++++++++++--------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/src/libcode/vx_tc_util/Makefile.in b/src/libcode/vx_tc_util/Makefile.in index 0a93f231bc..236c0abd6f 100644 --- a/src/libcode/vx_tc_util/Makefile.in +++ b/src/libcode/vx_tc_util/Makefile.in @@ -110,6 +110,7 @@ libvx_tc_util_a_LIBADD = am_libvx_tc_util_a_OBJECTS = libvx_tc_util_a-atcf_line_base.$(OBJEXT) \ libvx_tc_util_a-atcf_track_line.$(OBJEXT) \ libvx_tc_util_a-atcf_prob_line.$(OBJEXT) \ + libvx_tc_util_a-diag_file.$(OBJEXT) \ libvx_tc_util_a-track_point.$(OBJEXT) \ libvx_tc_util_a-track_info.$(OBJEXT) \ libvx_tc_util_a-track_pair_info.$(OBJEXT) \ @@ -123,7 +124,6 @@ am_libvx_tc_util_a_OBJECTS = libvx_tc_util_a-atcf_line_base.$(OBJEXT) \ libvx_tc_util_a-pair_data_genesis.$(OBJEXT) \ libvx_tc_util_a-tc_hdr_columns.$(OBJEXT) \ libvx_tc_util_a-tc_columns.$(OBJEXT) \ - libvx_tc_util_a-tc_diag.$(OBJEXT) \ libvx_tc_util_a-tc_stat_line.$(OBJEXT) \ libvx_tc_util_a-vx_tc_nc_util.$(OBJEXT) libvx_tc_util_a_OBJECTS = $(am_libvx_tc_util_a_OBJECTS) @@ -145,6 +145,7 @@ am__maybe_remake_depfiles = depfiles am__depfiles_remade = ./$(DEPDIR)/libvx_tc_util_a-atcf_line_base.Po \ ./$(DEPDIR)/libvx_tc_util_a-atcf_prob_line.Po \ ./$(DEPDIR)/libvx_tc_util_a-atcf_track_line.Po \ + ./$(DEPDIR)/libvx_tc_util_a-diag_file.Po \ ./$(DEPDIR)/libvx_tc_util_a-gen_shape_info.Po \ ./$(DEPDIR)/libvx_tc_util_a-genesis_info.Po \ ./$(DEPDIR)/libvx_tc_util_a-pair_data_genesis.Po \ @@ -154,7 +155,6 @@ am__depfiles_remade = ./$(DEPDIR)/libvx_tc_util_a-atcf_line_base.Po \ ./$(DEPDIR)/libvx_tc_util_a-prob_rirw_info.Po \ ./$(DEPDIR)/libvx_tc_util_a-prob_rirw_pair_info.Po \ ./$(DEPDIR)/libvx_tc_util_a-tc_columns.Po \ - ./$(DEPDIR)/libvx_tc_util_a-tc_diag.Po \ ./$(DEPDIR)/libvx_tc_util_a-tc_hdr_columns.Po \ ./$(DEPDIR)/libvx_tc_util_a-tc_stat_line.Po \ ./$(DEPDIR)/libvx_tc_util_a-track_info.Po \ @@ -371,6 +371,7 @@ libvx_tc_util_a_SOURCES = \ atcf_line_base.cc atcf_line_base.h atcf_offsets.h \ atcf_track_line.cc atcf_track_line.h \ atcf_prob_line.cc atcf_prob_line.h \ + diag_file.cc diag_file.h \ track_point.cc track_point.h \ track_info.cc track_info.h \ track_pair_info.cc track_pair_info.h \ @@ -384,7 +385,6 @@ libvx_tc_util_a_SOURCES = \ pair_data_genesis.cc pair_data_genesis.h \ tc_hdr_columns.cc tc_hdr_columns.h \ tc_columns.cc tc_columns.h \ - tc_diag.cc tc_diag.h \ tc_stat_line.cc tc_stat_line.h \ vx_tc_nc_util.h vx_tc_nc_util.cc \ vx_tc_util.h @@ -441,6 +441,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libvx_tc_util_a-atcf_line_base.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libvx_tc_util_a-atcf_prob_line.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libvx_tc_util_a-atcf_track_line.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libvx_tc_util_a-diag_file.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libvx_tc_util_a-gen_shape_info.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libvx_tc_util_a-genesis_info.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libvx_tc_util_a-pair_data_genesis.Po@am__quote@ # am--include-marker @@ -450,7 +451,6 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libvx_tc_util_a-prob_rirw_info.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libvx_tc_util_a-prob_rirw_pair_info.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libvx_tc_util_a-tc_columns.Po@am__quote@ # am--include-marker -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libvx_tc_util_a-tc_diag.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libvx_tc_util_a-tc_hdr_columns.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libvx_tc_util_a-tc_stat_line.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libvx_tc_util_a-track_info.Po@am__quote@ # am--include-marker @@ -520,6 +520,20 @@ libvx_tc_util_a-atcf_prob_line.obj: atcf_prob_line.cc @AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ @am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libvx_tc_util_a_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o libvx_tc_util_a-atcf_prob_line.obj `if test -f 'atcf_prob_line.cc'; then $(CYGPATH_W) 'atcf_prob_line.cc'; else $(CYGPATH_W) '$(srcdir)/atcf_prob_line.cc'; fi` +libvx_tc_util_a-diag_file.o: diag_file.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libvx_tc_util_a_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT libvx_tc_util_a-diag_file.o -MD -MP -MF $(DEPDIR)/libvx_tc_util_a-diag_file.Tpo -c -o libvx_tc_util_a-diag_file.o `test -f 'diag_file.cc' || echo '$(srcdir)/'`diag_file.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libvx_tc_util_a-diag_file.Tpo $(DEPDIR)/libvx_tc_util_a-diag_file.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='diag_file.cc' object='libvx_tc_util_a-diag_file.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libvx_tc_util_a_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o libvx_tc_util_a-diag_file.o `test -f 'diag_file.cc' || echo '$(srcdir)/'`diag_file.cc + +libvx_tc_util_a-diag_file.obj: diag_file.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libvx_tc_util_a_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT libvx_tc_util_a-diag_file.obj -MD -MP -MF $(DEPDIR)/libvx_tc_util_a-diag_file.Tpo -c -o libvx_tc_util_a-diag_file.obj `if test -f 'diag_file.cc'; then $(CYGPATH_W) 'diag_file.cc'; else $(CYGPATH_W) '$(srcdir)/diag_file.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libvx_tc_util_a-diag_file.Tpo $(DEPDIR)/libvx_tc_util_a-diag_file.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='diag_file.cc' object='libvx_tc_util_a-diag_file.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libvx_tc_util_a_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o libvx_tc_util_a-diag_file.obj `if test -f 'diag_file.cc'; then $(CYGPATH_W) 'diag_file.cc'; else $(CYGPATH_W) '$(srcdir)/diag_file.cc'; fi` + libvx_tc_util_a-track_point.o: track_point.cc @am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libvx_tc_util_a_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT libvx_tc_util_a-track_point.o -MD -MP -MF $(DEPDIR)/libvx_tc_util_a-track_point.Tpo -c -o libvx_tc_util_a-track_point.o `test -f 'track_point.cc' || echo '$(srcdir)/'`track_point.cc @am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libvx_tc_util_a-track_point.Tpo $(DEPDIR)/libvx_tc_util_a-track_point.Po @@ -702,20 +716,6 @@ libvx_tc_util_a-tc_columns.obj: tc_columns.cc @AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ @am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libvx_tc_util_a_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o libvx_tc_util_a-tc_columns.obj `if test -f 'tc_columns.cc'; then $(CYGPATH_W) 'tc_columns.cc'; else $(CYGPATH_W) '$(srcdir)/tc_columns.cc'; fi` -libvx_tc_util_a-tc_diag.o: tc_diag.cc -@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libvx_tc_util_a_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT libvx_tc_util_a-tc_diag.o -MD -MP -MF $(DEPDIR)/libvx_tc_util_a-tc_diag.Tpo -c -o libvx_tc_util_a-tc_diag.o `test -f 'tc_diag.cc' || echo '$(srcdir)/'`tc_diag.cc -@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libvx_tc_util_a-tc_diag.Tpo $(DEPDIR)/libvx_tc_util_a-tc_diag.Po -@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='tc_diag.cc' object='libvx_tc_util_a-tc_diag.o' libtool=no @AMDEPBACKSLASH@ -@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ -@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libvx_tc_util_a_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o libvx_tc_util_a-tc_diag.o `test -f 'tc_diag.cc' || echo '$(srcdir)/'`tc_diag.cc - -libvx_tc_util_a-tc_diag.obj: tc_diag.cc -@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libvx_tc_util_a_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT libvx_tc_util_a-tc_diag.obj -MD -MP -MF $(DEPDIR)/libvx_tc_util_a-tc_diag.Tpo -c -o libvx_tc_util_a-tc_diag.obj `if test -f 'tc_diag.cc'; then $(CYGPATH_W) 'tc_diag.cc'; else $(CYGPATH_W) '$(srcdir)/tc_diag.cc'; fi` -@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libvx_tc_util_a-tc_diag.Tpo $(DEPDIR)/libvx_tc_util_a-tc_diag.Po -@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='tc_diag.cc' object='libvx_tc_util_a-tc_diag.obj' libtool=no @AMDEPBACKSLASH@ -@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ -@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libvx_tc_util_a_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o libvx_tc_util_a-tc_diag.obj `if test -f 'tc_diag.cc'; then $(CYGPATH_W) 'tc_diag.cc'; else $(CYGPATH_W) '$(srcdir)/tc_diag.cc'; fi` - libvx_tc_util_a-tc_stat_line.o: tc_stat_line.cc @am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libvx_tc_util_a_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT libvx_tc_util_a-tc_stat_line.o -MD -MP -MF $(DEPDIR)/libvx_tc_util_a-tc_stat_line.Tpo -c -o libvx_tc_util_a-tc_stat_line.o `test -f 'tc_stat_line.cc' || echo '$(srcdir)/'`tc_stat_line.cc @am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libvx_tc_util_a-tc_stat_line.Tpo $(DEPDIR)/libvx_tc_util_a-tc_stat_line.Po @@ -872,6 +872,7 @@ distclean: distclean-am -rm -f ./$(DEPDIR)/libvx_tc_util_a-atcf_line_base.Po -rm -f ./$(DEPDIR)/libvx_tc_util_a-atcf_prob_line.Po -rm -f ./$(DEPDIR)/libvx_tc_util_a-atcf_track_line.Po + -rm -f ./$(DEPDIR)/libvx_tc_util_a-diag_file.Po -rm -f ./$(DEPDIR)/libvx_tc_util_a-gen_shape_info.Po -rm -f ./$(DEPDIR)/libvx_tc_util_a-genesis_info.Po -rm -f ./$(DEPDIR)/libvx_tc_util_a-pair_data_genesis.Po @@ -881,7 +882,6 @@ distclean: distclean-am -rm -f ./$(DEPDIR)/libvx_tc_util_a-prob_rirw_info.Po -rm -f ./$(DEPDIR)/libvx_tc_util_a-prob_rirw_pair_info.Po -rm -f ./$(DEPDIR)/libvx_tc_util_a-tc_columns.Po - -rm -f ./$(DEPDIR)/libvx_tc_util_a-tc_diag.Po -rm -f ./$(DEPDIR)/libvx_tc_util_a-tc_hdr_columns.Po -rm -f ./$(DEPDIR)/libvx_tc_util_a-tc_stat_line.Po -rm -f ./$(DEPDIR)/libvx_tc_util_a-track_info.Po @@ -936,6 +936,7 @@ maintainer-clean: maintainer-clean-am -rm -f ./$(DEPDIR)/libvx_tc_util_a-atcf_line_base.Po -rm -f ./$(DEPDIR)/libvx_tc_util_a-atcf_prob_line.Po -rm -f ./$(DEPDIR)/libvx_tc_util_a-atcf_track_line.Po + -rm -f ./$(DEPDIR)/libvx_tc_util_a-diag_file.Po -rm -f ./$(DEPDIR)/libvx_tc_util_a-gen_shape_info.Po -rm -f ./$(DEPDIR)/libvx_tc_util_a-genesis_info.Po -rm -f ./$(DEPDIR)/libvx_tc_util_a-pair_data_genesis.Po @@ -945,7 +946,6 @@ maintainer-clean: maintainer-clean-am -rm -f ./$(DEPDIR)/libvx_tc_util_a-prob_rirw_info.Po -rm -f ./$(DEPDIR)/libvx_tc_util_a-prob_rirw_pair_info.Po -rm -f ./$(DEPDIR)/libvx_tc_util_a-tc_columns.Po - -rm -f ./$(DEPDIR)/libvx_tc_util_a-tc_diag.Po -rm -f ./$(DEPDIR)/libvx_tc_util_a-tc_hdr_columns.Po -rm -f ./$(DEPDIR)/libvx_tc_util_a-tc_stat_line.Po -rm -f ./$(DEPDIR)/libvx_tc_util_a-track_info.Po From b5e6c7889ad60f7412f0de092d36e2ac360ca5b3 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Thu, 6 Oct 2022 13:57:38 -0600 Subject: [PATCH 09/58] Per #392, fix capitalization bug. --- src/libcode/vx_tc_util/diag_file.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libcode/vx_tc_util/diag_file.cc b/src/libcode/vx_tc_util/diag_file.cc index a7b5a3209a..45a54c757b 100644 --- a/src/libcode/vx_tc_util/diag_file.cc +++ b/src/libcode/vx_tc_util/diag_file.cc @@ -65,7 +65,7 @@ DiagFile & DiagFile::operator=(const DiagFile &) { void DiagFile::init_from_scratch() { // Initialize values - StormID.clear(); + StormId.clear(); Basin.clear(); Cyclone.clear(); Technique.clear(); From 27007e164424d297b8813afe3a58f938d1ae8665 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Fri, 7 Oct 2022 14:57:13 -0600 Subject: [PATCH 10/58] Per #392, work in progress. Parsing the tcdiag and lsdiag files well. Next I need to read the diag data into the tracks. --- src/libcode/vx_tc_util/diag_file.cc | 228 ++++++++++++++++++++++-- src/libcode/vx_tc_util/diag_file.h | 28 ++- src/tools/tc_utils/tc_pairs/tc_pairs.cc | 59 +++++- 3 files changed, 295 insertions(+), 20 deletions(-) diff --git a/src/libcode/vx_tc_util/diag_file.cc b/src/libcode/vx_tc_util/diag_file.cc index 45a54c757b..c23a6df7f8 100644 --- a/src/libcode/vx_tc_util/diag_file.cc +++ b/src/libcode/vx_tc_util/diag_file.cc @@ -25,6 +25,17 @@ using namespace std; #include "diag_file.h" +//////////////////////////////////////////////////////////////////////// + +static const char default_lsdiag_technique[] = "AVNO"; + +static const int lsdiag_wdth[] = { + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5 +}; +static int n_lsdiag_wdth = sizeof(lsdiag_wdth)/sizeof(*lsdiag_wdth); + //////////////////////////////////////////////////////////////////////// // // Code for class DiagFile @@ -43,6 +54,63 @@ DiagFile::~DiagFile() { //////////////////////////////////////////////////////////////////////// +int DiagFile::lead(int i) const { + + // Check range + if(i < 0 || i >= LeadTime.n()) { + mlog << Error << "\nTrackInfo::lead(int) -> " + << "range check error for index value " << i << "\n\n"; + exit(1); + } + + return(LeadTime[i]); +} + +//////////////////////////////////////////////////////////////////////// + +unixtime DiagFile::valid(int i) const { + + // Check range + if(i < 0 || i >= LeadTime.n()) { + mlog << Error << "\nTrackInfo::valid(int) -> " + << "range check error for index value " << i << "\n\n"; + exit(1); + } + + return(InitTime == 0 || is_bad_data(LeadTime[i]) ? + 0 : InitTime + LeadTime[i]); +} + +//////////////////////////////////////////////////////////////////////// + +double DiagFile::lat(int i) const { + + // Check range + if(i < 0 || i >= Lat.n()) { + mlog << Error << "\nTrackInfo::lat(int) -> " + << "range check error for index value " << i << "\n\n"; + exit(1); + } + + return(Lat[i]); +} + +//////////////////////////////////////////////////////////////////////// + +double DiagFile::lon(int i) const { + + // Check range + if(i < 0 || i >= Lon.n()) { + mlog << Error << "\nTrackInfo::lon(int) -> " + << "range check error for index value " << i << "\n\n"; + exit(1); + } + + return(Lon[i]); +} + +//////////////////////////////////////////////////////////////////////// + DiagFile::DiagFile(const DiagFile &) { mlog << Error << "\nDiagFile::DiagFile(const DiagFile &) -> " << "should never be called!\n\n"; @@ -52,8 +120,7 @@ DiagFile::DiagFile(const DiagFile &) { //////////////////////////////////////////////////////////////////////// DiagFile & DiagFile::operator=(const DiagFile &) { - mlog << Error - << "\nDiagFile::operator=(const DiagFile &) -> " + mlog << Error << "\nDiagFile::operator=(const DiagFile &) -> " << "should never be called!\n\n"; exit(1); @@ -65,12 +132,18 @@ DiagFile & DiagFile::operator=(const DiagFile &) { void DiagFile::init_from_scratch() { // Initialize values + FileType = DiagFileType_None; StormId.clear(); Basin.clear(); Cyclone.clear(); Technique.clear(); InitTime = (unixtime) 0; + NTime = 0; + LeadTime.clear(); + Lat.clear(); + Lon.clear(); + close(); return; @@ -78,24 +151,159 @@ void DiagFile::init_from_scratch() { //////////////////////////////////////////////////////////////////////// -bool DiagFile::open_tcdiag(const char *path, const char *model_name) { +bool DiagFile::open_tcdiag(const std::string &path, const std::string &model_name) { + int i; - close(); + FileType = TCDiagFileType; + + // Initialize the technique name + Technique = model_name; + + // Open the file + open(path.c_str()); + + // Parse the header information + DataLine dl; + ConcatString cs; + while(dl.read_line(this)) { + + // Skip empty lines + if(dl.n_items() == 0) continue; + + cs = dl[0]; + + // Parse header lines + if(cs == "*") { + + // First header line: Technique InitTime (ATCFID YYYMMDDHH) + if(InitTime == 0) { + if(Technique.empty()) Technique = dl[1]; + InitTime = timestring_to_unix(dl[2]); + } + // Second header line: Basin Cyclone Number (BBCC) + else if(StormId.empty()) { + string bbcc = dl[1]; + Basin = bbcc.substr(0, 2); + Cyclone = bbcc.substr(2, 2); + int mon, day, yr, hr, min, sec; + unix_to_mdyhms(InitTime, mon, day, yr, hr, min, sec); + StormId << Basin << Cyclone << yr; + } + } + // Parse time and location info + else if(cs == "NTIME") { + NTime = atoi(dl[1]); + } + else if(cs == "TIME") { + for(int i=2; i " + << "the NTIME value (" << NTime + << ") does not match the actual number of times (" + << LeadTime.n() << "), latitudes (" << Lat.n() + << "), or longitudes (" << Lon.n() << ")!\n\n"; + exit(1); + } - return ( false ); + mlog << Debug(4) << "Parsing " << StormId << " " + << Technique << " " << unix_to_yyyymmddhh(InitTime) + << " TC diagnostics file: " << path << "\n"; + + return(true); } //////////////////////////////////////////////////////////////////////// -bool DiagFile::open_lsdiag(const char *path, const char *model_name) { +bool DiagFile::open_lsdiag(const std::string &path, const std::string &model_name) { + int i; - close(); + FileType = LSDiagFileType; + + // Store the default lsdiag technique, unless otherwise specified + if(model_name.size() > 0) Technique = model_name; + else Technique = default_lsdiag_technique; + + // Open the file + open(path.c_str()); + + // Parse the header information from the first line + DataLine dl; + ConcatString cs; + dl.read_line(this); + + // Check for the expected number of items + if(dl.n_items() != 9 || strncasecmp(dl[8], "HEAD", strlen("HEAD") != 0)) { + mlog << Error << "\nDiagFile::open_lsdiag() -> " + << "unexpected header line in file: " << path << "\n\n"; + exit(1); + } + + // Parse storm information + StormId = dl[7]; + Basin = StormId.string().substr(0, 2); + Cyclone = StormId.string().substr(2, 2); + + // Parse timing info + cs = dl[1]; + int yr = atoi(StormId.string().substr(4, 4).c_str()); + int mon = atoi(cs.string().substr(2, 2).c_str()); + int day = atoi(cs.string().substr(4, 2).c_str()); + int hr = atoi(dl[2]); + InitTime = mdyhms_to_unix(mon, day, yr, hr, 0, 0); + + // Parse time and location info + while(read_fwf_line(dl, lsdiag_wdth, n_lsdiag_wdth)) { + + // Fixed width column 24 has the data name + cs = dl[23]; + + if(cs == "TIME") { + for(i=2; i<23; i++) { + LeadTime.add(atoi(dl[i])*sec_per_hour); + } + NTime = LeadTime.n(); + } + else if(cs == "LAT") { + // Tenths of degree north + for(i=2; i<23; i++) { + Lat.add(atof(dl[i])/10.0); + } + } + else if(cs == "LON") { + for(i=2; i<23; i++) { + // Tenths of degrees west (convert to east) + Lon.add(rescale_lon(-1.0*atof(dl[i])/10.0)); + } + + // Finished parsing header + break; + } + } // end while - // JHG work here + mlog << Debug(4) << "Parsing " << StormId << " " + << Technique << " " << unix_to_yyyymmddhh(InitTime) + << " LS diagnostics file: " << path << "\n"; - return ( false ); + return(true); } //////////////////////////////////////////////////////////////////////// diff --git a/src/libcode/vx_tc_util/diag_file.h b/src/libcode/vx_tc_util/diag_file.h index 9c7a023276..d9f12b3716 100644 --- a/src/libcode/vx_tc_util/diag_file.h +++ b/src/libcode/vx_tc_util/diag_file.h @@ -16,8 +16,17 @@ #include #include +#include "vx_cal.h" #include "data_line.h" +//////////////////////////////////////////////////////////////////////// + +enum DiagFileType { + DiagFileType_None, // Default + LSDiagFileType, // Large Scale Diagnostics + TCDiagFileType // Tropical Cyclone Diagnostics +}; + //////////////////////////////////////////////////////////////////////// // // LSDIAG files: @@ -52,6 +61,9 @@ class DiagFile : public LineDataFile { void init_from_scratch(); + // Diagnostics file type + DiagFileType FileType; + // Storm and model identification ConcatString StormId; ConcatString Basin; @@ -59,6 +71,11 @@ class DiagFile : public LineDataFile { ConcatString Technique; unixtime InitTime; + int NTime; + IntArray LeadTime; + NumArray Lat; + NumArray Lon; + public: DiagFile(); @@ -79,12 +96,18 @@ class DiagFile : public LineDataFile { const ConcatString & initials() const; unixtime init() const; + int n_time() const; + int lead(int) const; + unixtime valid(int) const; + double lat(int) const; + double lon(int) const; + // // do stuff // - bool open_tcdiag(const char *path, const char *model_name); - bool open_lsdiag(const char *path, const char *model_name); + bool open_tcdiag(const std::string &, const std::string &); + bool open_lsdiag(const std::string &, const std::string &); }; @@ -95,6 +118,7 @@ inline const ConcatString & DiagFile::basin() const { return(Basin); } inline const ConcatString & DiagFile::cyclone() const { return(Cyclone); } inline const ConcatString & DiagFile::technique() const { return(Technique); } inline unixtime DiagFile::init() const { return(InitTime); } +inline int DiagFile::n_time() const { return(NTime); } //////////////////////////////////////////////////////////////////////// diff --git a/src/tools/tc_utils/tc_pairs/tc_pairs.cc b/src/tools/tc_utils/tc_pairs/tc_pairs.cc index 6b9e91e1d2..65f167d834 100644 --- a/src/tools/tc_utils/tc_pairs/tc_pairs.cc +++ b/src/tools/tc_utils/tc_pairs/tc_pairs.cc @@ -125,6 +125,9 @@ static bool is_keeper (const ATCFLineBase *); static void filter_tracks (TrackInfoArray &); static void filter_probs (ProbInfoArray &); static void process_diags (TrackInfoArray &); +static void get_diag_files (const StringArray &, + const StringArray &, + StringArray &, StringArray &); static bool check_masks (const MaskPoly &, const Grid &, const MaskPlane &, @@ -204,8 +207,8 @@ void process_command_line(int argc, char **argv) { cline.add(set_adeck, "-adeck", -1); cline.add(set_edeck, "-edeck", -1); cline.add(set_bdeck, "-bdeck", -1); - cline.add(set_bdeck, "-tcdiag", -1); - cline.add(set_bdeck, "-lsdiag", -1); + cline.add(set_tcdiag, "-tcdiag", -1); + cline.add(set_lsdiag, "-lsdiag", -1); cline.add(set_config, "-config", 1); cline.add(set_out, "-out", 1); @@ -1021,10 +1024,50 @@ void filter_probs(ProbInfoArray &probs) { //////////////////////////////////////////////////////////////////////// -// JHG: Add logic to actually process this data -// For now just add the names of the requested diagnostics void process_diags(TrackInfoArray &tracks) { - tracks.set_diag_name(conf_info.DiagName); + StringArray tcdiag_files, tcdiag_files_model_name; + StringArray lsdiag_files, lsdiag_files_model_name; + StringArray files, files_model_name; + DiagFile dfile; + + // Process TCDIAG inputs + if(tcdiag_source.n() > 0) { + get_atcf_files(tcdiag_source, tcdiag_model_name, + tcdiag_files, tcdiag_files_model_name); + + mlog << Debug(2) + << "Processing " << tcdiag_files.n() + << " TCDIAG diagnostic file(s).\n"; + } + + // Process LSDIAG inputs + if(lsdiag_source.n() > 0) { + get_atcf_files(lsdiag_source, lsdiag_model_name, + lsdiag_files, lsdiag_files_model_name); + + mlog << Debug(2) + << "Processing " << lsdiag_files.n() + << " LSDIAG diagnostic file(s).\n"; + } + + // Append them to loop once + files = tcdiag_files; + files.add(lsdiag_files); + files_model_name = tcdiag_files_model_name; + files_model_name.add(lsdiag_model_name); + + // Loop over the input files + for(int i=0; i 0) { if(!mask_poly.latlon_is_inside_dege(lat, lon)) { @@ -1043,7 +1086,7 @@ bool check_masks(const MaskPoly &mask_poly, const Grid &mask_grid, } // - // Check grid masking. + // Check grid masking // if(mask_grid.nx() > 0 || mask_grid.ny() > 0) { mask_grid.latlon_to_xy(lat, -1.0*lon, grid_x, grid_y); @@ -1053,7 +1096,7 @@ bool check_masks(const MaskPoly &mask_poly, const Grid &mask_grid, } // - // Check area mask. + // Check area mask // if(mask_area.nx() > 0 || mask_area.ny() > 0) { if(!mask_area.s_is_on(nint(grid_x), nint(grid_y))) { From 8ea797a9f6311818170eff597594c1aa908352f1 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Fri, 7 Oct 2022 17:12:00 -0600 Subject: [PATCH 11/58] Per #392, add logic to read lines of diagnostics data. --- src/libcode/vx_tc_util/diag_file.cc | 132 ++++++++++++++++++++++++ src/libcode/vx_tc_util/diag_file.h | 3 + src/tools/tc_utils/tc_pairs/tc_pairs.cc | 54 +++++----- 3 files changed, 159 insertions(+), 30 deletions(-) diff --git a/src/libcode/vx_tc_util/diag_file.cc b/src/libcode/vx_tc_util/diag_file.cc index c23a6df7f8..6edb4e8fa9 100644 --- a/src/libcode/vx_tc_util/diag_file.cc +++ b/src/libcode/vx_tc_util/diag_file.cc @@ -36,6 +36,9 @@ static const int lsdiag_wdth[] = { }; static int n_lsdiag_wdth = sizeof(lsdiag_wdth)/sizeof(*lsdiag_wdth); +static const int tcdiag_fill_value = 9999; +static const int lsdiag_fill_value = 9999; + //////////////////////////////////////////////////////////////////////// // // Code for class DiagFile @@ -307,3 +310,132 @@ bool DiagFile::open_lsdiag(const std::string &path, const std::string &model_nam } //////////////////////////////////////////////////////////////////////// + +bool DiagFile::read_diag_data(ConcatString &name, NumArray &data) { + bool status = false; + + if(FileType == TCDiagFileType) { + status = read_tcdiag_data(name, data); + } + else if(FileType == LSDiagFileType) { + status = read_lsdiag_data(name, data); + } + else { + mlog << Error << "\nDiagFile::read_diag_data() -> " + << "unexpected file type!\n\n"; + exit(1); + } + + return(status); +} + +//////////////////////////////////////////////////////////////////////// + +bool DiagFile::read_tcdiag_data(ConcatString &name, NumArray &data) { + DataLine dl; + ConcatString cs; + int i; + + // Initialize + name.clear(); + data.erase(); + bool status = false; + + // Read until a finding a line of data to parse + while(dl.read_line(this)) { + + // Skip empty lines + if(dl.n_items() == 0) continue; + + // Quit reading at the COMMENTS section + if((cs = dl[1]) == "COMMENTS") { + status = false; + break; + } + + // Set name from the first column + name = dl[0]; + + // Skip certain lines + if(name.startswith("----") || name.startswith("TIME") || + name.startswith("NLEV") || name.startswith("NVAR")) continue; + + // Parse the data values + for(i=2; i " + << "the number of \"" << name << "\" diagnostic values (" + << data.n() << ") does not match the expected number (" + << NTime << ")!\n\n"; + exit(1); + } + else { + + // Break out after reading a good line of data + status = true; + break; + } + + } // end while + + return(status); +} + +//////////////////////////////////////////////////////////////////////// + +bool DiagFile::read_lsdiag_data(ConcatString &name, NumArray &data) { + DataLine dl; + int i; + + // Initialize + name.clear(); + data.erase(); + bool status = false; + + // Read until a finding a line of data to parse + while(read_fwf_line(dl, lsdiag_wdth, n_lsdiag_wdth)) { + + // Skip empty lines + if(dl.n_items() == 0) continue; + + // Set name from column 24 + name = dl[24]; + + // Quit reading at the LAST line + if(name == "LAST") { + status = false; + break; + } + + // Parse the data values + for(i=2; i<23; i++) { + data.add(atoi(dl[i]) == lsdiag_fill_value ? + bad_data_double : atof(dl[i])); + } + + // Check for the expected number of items + if(NTime != data.n()) { + mlog << Error << "\nDiagFile::read_lsdiag_data() -> " + << "the number of \"" << name << "\" diagnostic values (" + << data.n() << ") does not match the expected number (" + << NTime << ")!\n\n"; + exit(1); + } + else { + + // Break out after reading a good line of data + status = true; + break; + } + + } // end while + + return(status); +} + +//////////////////////////////////////////////////////////////////////// diff --git a/src/libcode/vx_tc_util/diag_file.h b/src/libcode/vx_tc_util/diag_file.h index d9f12b3716..4f9b71797b 100644 --- a/src/libcode/vx_tc_util/diag_file.h +++ b/src/libcode/vx_tc_util/diag_file.h @@ -109,6 +109,9 @@ class DiagFile : public LineDataFile { bool open_tcdiag(const std::string &, const std::string &); bool open_lsdiag(const std::string &, const std::string &); + bool read_diag_data (ConcatString &, NumArray &); + bool read_tcdiag_data(ConcatString &, NumArray &); + bool read_lsdiag_data(ConcatString &, NumArray &); }; //////////////////////////////////////////////////////////////////////// diff --git a/src/tools/tc_utils/tc_pairs/tc_pairs.cc b/src/tools/tc_utils/tc_pairs/tc_pairs.cc index 65f167d834..e3e0d945bf 100644 --- a/src/tools/tc_utils/tc_pairs/tc_pairs.cc +++ b/src/tools/tc_utils/tc_pairs/tc_pairs.cc @@ -1025,47 +1025,41 @@ void filter_probs(ProbInfoArray &probs) { //////////////////////////////////////////////////////////////////////// void process_diags(TrackInfoArray &tracks) { - StringArray tcdiag_files, tcdiag_files_model_name; - StringArray lsdiag_files, lsdiag_files_model_name; StringArray files, files_model_name; - DiagFile dfile; + DiagFile diag_file; + int i; // Process TCDIAG inputs if(tcdiag_source.n() > 0) { - get_atcf_files(tcdiag_source, tcdiag_model_name, - tcdiag_files, tcdiag_files_model_name); + get_atcf_files(tcdiag_source, tcdiag_model_name, + files, files_model_name); + + mlog << Debug(2) + << "Processing " << files.n() + << " TCDIAG diagnostic file(s).\n"; - mlog << Debug(2) - << "Processing " << tcdiag_files.n() - << " TCDIAG diagnostic file(s).\n"; + // Loop over the input files + for(i=0; i 0) { - get_atcf_files(lsdiag_source, lsdiag_model_name, - lsdiag_files, lsdiag_files_model_name); - - mlog << Debug(2) - << "Processing " << lsdiag_files.n() - << " LSDIAG diagnostic file(s).\n"; - } - - // Append them to loop once - files = tcdiag_files; - files.add(lsdiag_files); - files_model_name = tcdiag_files_model_name; - files_model_name.add(lsdiag_model_name); + get_atcf_files(lsdiag_source, lsdiag_model_name, + files, files_model_name); - // Loop over the input files - for(int i=0; i Date: Fri, 7 Oct 2022 17:13:43 -0600 Subject: [PATCH 12/58] Per #392, update TrackInfo class to add the diagnostic data to the tracks. Still need to do more work there. --- src/libcode/vx_tc_util/track_info.cc | 34 +++++++++++++++++++++++++--- src/libcode/vx_tc_util/track_info.h | 6 ++--- 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/src/libcode/vx_tc_util/track_info.cc b/src/libcode/vx_tc_util/track_info.cc index 38efb50d39..526dc72ffc 100644 --- a/src/libcode/vx_tc_util/track_info.cc +++ b/src/libcode/vx_tc_util/track_info.cc @@ -535,6 +535,28 @@ void TrackInfo::add_watch_warn(const ConcatString &ww_sid, //////////////////////////////////////////////////////////////////////// +bool TrackInfo::add_diag_data(DiagFile &diag_file, const StringArray &diag_name) { + + // Check for a match + if(StormId != diag_file.storm_id() || + Technique != diag_file.technique() || + InitTime != diag_file.init()) return(false); + + // Store the requested diagnostic names + if(DiagName.n() == 0) DiagName = diag_name; + + // Read the diagnostics from the file + ConcatString cur_name; + NumArray cur_data; + while(diag_file.read_diag_data(cur_name, cur_data)) { + cout << "JHG for " << cur_name << ", read " << cur_data.n() << " values\n"; + } // end while + + return(true); +} + +//////////////////////////////////////////////////////////////////////// + bool TrackInfo::has(const ATCFTrackLine &l) const { return(TrackLines.has(l.get_line())); } @@ -864,12 +886,18 @@ bool TrackInfoArray::erase_storm_id(const ConcatString &s) { //////////////////////////////////////////////////////////////////////// -void TrackInfoArray::set_diag_name(const StringArray &sa) { +bool TrackInfoArray::add_diag_data(DiagFile &diag_file, const StringArray &diag_name) { + bool match = false; // Set the names for each track - for(int i=0; i Date: Fri, 7 Oct 2022 23:29:54 -0600 Subject: [PATCH 13/58] Per #392, switch ANV to GFS in the diagnostic file code. --- src/libcode/vx_tc_util/diag_file.cc | 22 ++++++++++++++++++---- src/libcode/vx_tc_util/diag_file.h | 2 ++ 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/libcode/vx_tc_util/diag_file.cc b/src/libcode/vx_tc_util/diag_file.cc index 6edb4e8fa9..6994235fb6 100644 --- a/src/libcode/vx_tc_util/diag_file.cc +++ b/src/libcode/vx_tc_util/diag_file.cc @@ -154,13 +154,27 @@ void DiagFile::init_from_scratch() { //////////////////////////////////////////////////////////////////////// +void DiagFile::set_technique(const string &str) { + + Technique = str; + + // Replace instances of AVN with GFS + if(strstr(Technique.c_str(), "AVN") != NULL) { + Technique.replace("AVN", "GFS"); + } + + return; +} + +//////////////////////////////////////////////////////////////////////// + bool DiagFile::open_tcdiag(const std::string &path, const std::string &model_name) { int i; FileType = TCDiagFileType; // Initialize the technique name - Technique = model_name; + set_technique(model_name); // Open the file open(path.c_str()); @@ -180,7 +194,7 @@ bool DiagFile::open_tcdiag(const std::string &path, const std::string &model_nam // First header line: Technique InitTime (ATCFID YYYMMDDHH) if(InitTime == 0) { - if(Technique.empty()) Technique = dl[1]; + if(Technique.empty()) set_technique(dl[1]); InitTime = timestring_to_unix(dl[2]); } // Second header line: Basin Cyclone Number (BBCC) @@ -242,8 +256,8 @@ bool DiagFile::open_lsdiag(const std::string &path, const std::string &model_nam FileType = LSDiagFileType; // Store the default lsdiag technique, unless otherwise specified - if(model_name.size() > 0) Technique = model_name; - else Technique = default_lsdiag_technique; + if(model_name.size() > 0) set_technique(model_name); + else set_technique(default_lsdiag_technique); // Open the file open(path.c_str()); diff --git a/src/libcode/vx_tc_util/diag_file.h b/src/libcode/vx_tc_util/diag_file.h index 4f9b71797b..1406a62062 100644 --- a/src/libcode/vx_tc_util/diag_file.h +++ b/src/libcode/vx_tc_util/diag_file.h @@ -85,6 +85,8 @@ class DiagFile : public LineDataFile { // set stuff // + void set_technique(const std::string &); + // // get stuff // From 44f889229208c3d2df6f67fd3a4cedc252df2b33 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Fri, 7 Oct 2022 23:31:19 -0600 Subject: [PATCH 14/58] Per #392, update TrackPoint class to store the actual diagnostic name and value. That makes the logic easier. --- src/libcode/vx_tc_util/track_point.cc | 45 +++++++++++++++++++++++++-- src/libcode/vx_tc_util/track_point.h | 6 +++- 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/src/libcode/vx_tc_util/track_point.cc b/src/libcode/vx_tc_util/track_point.cc index 1b78aa41df..fc99d7a5a6 100644 --- a/src/libcode/vx_tc_util/track_point.cc +++ b/src/libcode/vx_tc_util/track_point.cc @@ -422,6 +422,8 @@ void TrackPoint::clear() { Depth = NoSystemsDepth; WarmCore = false; WatchWarn = NoWatchWarnType; + DiagName.clear(); + DiagVal.clear(); // Call clear for each Wind object and then set intensity value for(i=0; i " + << "the diagnostic location (" << diag_lat << ", " << diag_lon + << ") does not match the track location (" << Lat << ", " << Lon + << ")\n"; + } + + // Store the diagnostic + DiagName.add(name); + DiagVal.add(val); + + return; +} + +//////////////////////////////////////////////////////////////////////// diff --git a/src/libcode/vx_tc_util/track_point.h b/src/libcode/vx_tc_util/track_point.h index 8cf61d3ccd..cfd69e8ead 100644 --- a/src/libcode/vx_tc_util/track_point.h +++ b/src/libcode/vx_tc_util/track_point.h @@ -167,7 +167,8 @@ class TrackPoint { // Wind Radii QuadInfo Wind[NWinds]; - // Diagnostic Values + // Diagnostic names and values + StringArray DiagName; NumArray DiagVal; public: @@ -232,6 +233,8 @@ class TrackPoint { SystemsDepth depth() const; bool warm_core() const; WatchWarnType watch_warn() const; + int n_diag() const; + const char * diag_name(int) const; double diag_val(int) const; // @@ -241,6 +244,7 @@ class TrackPoint { void set_wind(int, const QuadInfo &); bool set(const ATCFTrackLine &); bool is_match(const ATCFTrackLine &) const; + void add_diag_data(double, double, const ConcatString &, double); }; From e4806e775085a0baa32d7f8137139bdb194352d6 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Fri, 7 Oct 2022 23:31:58 -0600 Subject: [PATCH 15/58] Per #392, remove the DiagName setting from TrackInfo class since we're storing it in TrackPoint instead. --- src/libcode/vx_tc_util/track_info.cc | 41 ++++++++++++---------------- src/libcode/vx_tc_util/track_info.h | 6 ---- 2 files changed, 18 insertions(+), 29 deletions(-) diff --git a/src/libcode/vx_tc_util/track_info.cc b/src/libcode/vx_tc_util/track_info.cc index 526dc72ffc..9a42628a9b 100644 --- a/src/libcode/vx_tc_util/track_info.cc +++ b/src/libcode/vx_tc_util/track_info.cc @@ -92,7 +92,6 @@ void TrackInfo::clear() { MaxValidTime = (unixtime) 0; MinWarmCore = (unixtime) 0; MaxWarmCore = (unixtime) 0; - DiagName.clear(); TrackLines.clear(); clear_points(); @@ -134,7 +133,6 @@ void TrackInfo::dump(ostream &out, int indent_depth) const { out << prefix << "MaxWarmCore = \"" << (MaxWarmCore > 0 ? unix_to_yyyymmdd_hhmmss(MaxWarmCore).text() : na_str) << "\n"; out << prefix << "NPoints = " << NPoints << "\n"; out << prefix << "NAlloc = " << NAlloc << "\n"; - out << prefix << "NDiag = " << DiagName.n() << "\n"; out << prefix << "NTrackLines = " << TrackLines.n() << "\n"; for(i=0; i " - << "range check error for index value " << i << "\n\n"; - exit(1); - } - - return(DiagName[i].c_str()); -} - -//////////////////////////////////////////////////////////////////////// - void TrackInfo::add(const TrackPoint &p) { extend(NPoints + 1, false); @@ -536,20 +518,33 @@ void TrackInfo::add_watch_warn(const ConcatString &ww_sid, //////////////////////////////////////////////////////////////////////// bool TrackInfo::add_diag_data(DiagFile &diag_file, const StringArray &diag_name) { + int i, i_pnt; // Check for a match if(StormId != diag_file.storm_id() || Technique != diag_file.technique() || InitTime != diag_file.init()) return(false); - // Store the requested diagnostic names - if(DiagName.n() == 0) DiagName = diag_name; - // Read the diagnostics from the file ConcatString cur_name; NumArray cur_data; while(diag_file.read_diag_data(cur_name, cur_data)) { - cout << "JHG for " << cur_name << ", read " << cur_data.n() << " values\n"; + + // Only add requested diagnostics + if(diag_name.has(cur_name)) { + + // Add diagnostic values to the TrackPoints + for(i=0; i Date: Fri, 7 Oct 2022 23:33:03 -0600 Subject: [PATCH 16/58] Per #392, update TC-Pairs to add the diag data to the parsed tracks. --- src/libcode/vx_tc_util/tc_columns.cc | 11 +++++------ src/tools/tc_utils/tc_pairs/tc_pairs.cc | 18 +++++++++++++----- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/src/libcode/vx_tc_util/tc_columns.cc b/src/libcode/vx_tc_util/tc_columns.cc index 7953bac78d..94fb10b289 100644 --- a/src/libcode/vx_tc_util/tc_columns.cc +++ b/src/libcode/vx_tc_util/tc_columns.cc @@ -172,10 +172,9 @@ void write_prob_rirw_header_row(int hdr_flag, int n_thresh, AsciiTable &at, void write_track_pair_info(TcHdrColumns &hdr, const TrackPairInfo &p, AsciiTable &at, int &i_row) { - int i; // Loop through the TrackPairInfo points - for(i=0; i 0) { + if(p.adeck()[i].n_diag() > 0) { // TCDIAG line type hdr.set_line_type((string) TCStatLineType_TCDIAG_Str); @@ -355,10 +354,10 @@ void write_tc_diag_cols(const TrackPairInfo &p, int i, // Write TCDIAG columns at.set_entry(r, c++, p.n_points()); at.set_entry(r, c++, i+1); - at.set_entry(r, c++, p.adeck().n_diag()); + at.set_entry(r, c++, p.adeck()[i].n_diag()); - for(j=0; j 0) { @@ -1039,10 +1039,14 @@ void process_diags(TrackInfoArray &tracks) { << " TCDIAG diagnostic file(s).\n"; // Loop over the input files - for(i=0; i Date: Tue, 11 Oct 2022 10:45:42 -0600 Subject: [PATCH 17/58] Per #392, restructure the logic for storing the TC diagnostics data by parsing it into an STL map first. If diag_name is an empty list, write all available diagnostics to the output. --- src/libcode/vx_tc_util/diag_file.cc | 192 +++++++++------------- src/libcode/vx_tc_util/diag_file.h | 29 ++-- src/libcode/vx_tc_util/tc_columns.cc | 14 +- src/libcode/vx_tc_util/track_info.cc | 46 ++++-- src/libcode/vx_tc_util/track_info.h | 6 + src/libcode/vx_tc_util/track_pair_info.cc | 12 ++ src/libcode/vx_tc_util/track_pair_info.h | 5 +- src/libcode/vx_tc_util/track_point.cc | 46 ++---- src/libcode/vx_tc_util/track_point.h | 43 +++-- src/tools/tc_utils/tc_pairs/tc_pairs.cc | 10 +- 10 files changed, 198 insertions(+), 205 deletions(-) diff --git a/src/libcode/vx_tc_util/diag_file.cc b/src/libcode/vx_tc_util/diag_file.cc index 6994235fb6..9daa79df86 100644 --- a/src/libcode/vx_tc_util/diag_file.cc +++ b/src/libcode/vx_tc_util/diag_file.cc @@ -114,6 +114,27 @@ double DiagFile::lon(int i) const { //////////////////////////////////////////////////////////////////////// +bool DiagFile::has_diag(const string &str) const { + return(DiagMap.count(str) > 0); +} + +//////////////////////////////////////////////////////////////////////// + +const NumArray & DiagFile::get_diag(const string &str) const { + return(DiagMap.at(str)); +} + +//////////////////////////////////////////////////////////////////////// + +void DiagFile::get_diag_name(StringArray &diag_name) const { + diag_name.clear(); + for(map::const_iterator it = DiagMap.begin(); + it != DiagMap.end(); it++) diag_name.add(it->first); + return; +} + +//////////////////////////////////////////////////////////////////////// + DiagFile::DiagFile(const DiagFile &) { mlog << Error << "\nDiagFile::DiagFile(const DiagFile &) -> " << "should never be called!\n\n"; @@ -146,6 +167,7 @@ void DiagFile::init_from_scratch() { LeadTime.clear(); Lat.clear(); Lon.clear(); + DiagMap.clear(); close(); @@ -168,8 +190,9 @@ void DiagFile::set_technique(const string &str) { //////////////////////////////////////////////////////////////////////// -bool DiagFile::open_tcdiag(const std::string &path, const std::string &model_name) { +void DiagFile::read_tcdiag(const std::string &path, const std::string &model_name) { int i; + NumArray data; FileType = TCDiagFileType; @@ -187,6 +210,7 @@ bool DiagFile::open_tcdiag(const std::string &path, const std::string &model_nam // Skip empty lines if(dl.n_items() == 0) continue; + // First column cs = dl[0]; // Parse header lines @@ -233,7 +257,7 @@ bool DiagFile::open_tcdiag(const std::string &path, const std::string &model_nam // Check for the expected number of items if(NTime != LeadTime.n() || NTime != Lat.n() || NTime != Lon.n()) { - mlog << Error << "\nDiagFile::open_tcdiag() -> " + mlog << Error << "\nDiagFile::read_tcdiag() -> " << "the NTIME value (" << NTime << ") does not match the actual number of times (" << LeadTime.n() << "), latitudes (" << Lat.n() @@ -245,13 +269,54 @@ bool DiagFile::open_tcdiag(const std::string &path, const std::string &model_nam << Technique << " " << unix_to_yyyymmddhh(InitTime) << " TC diagnostics file: " << path << "\n"; - return(true); + // Store the diagnostics data + while(dl.read_line(this)) { + + // Skip empty lines + if(dl.n_items() == 0) continue; + + // Quit reading at the COMMENTS section + if((cs = dl[1]) == "COMMENTS") break; + + // first column contains the name + cs = dl[0]; + + // Skip certain lines + if(cs.startswith("----") || cs.startswith("TIME") || + cs.startswith("NLEV") || cs.startswith("NVAR")) continue; + + // Parse the data values + data.erase(); + for(i=2; i " + << "the number of \"" << cs << "\" diagnostic values (" + << data.n() << ") does not match the expected number (" + << NTime << ")!\n\n"; + exit(1); + } + // Store the data in the map + else { + DiagMap[cs] = data; + } + } // end while + + // Close the input file + close(); + + return; } //////////////////////////////////////////////////////////////////////// -bool DiagFile::open_lsdiag(const std::string &path, const std::string &model_name) { +void DiagFile::read_lsdiag(const std::string &path, const std::string &model_name) { int i; + NumArray data; FileType = LSDiagFileType; @@ -320,113 +385,20 @@ bool DiagFile::open_lsdiag(const std::string &path, const std::string &model_nam << Technique << " " << unix_to_yyyymmddhh(InitTime) << " LS diagnostics file: " << path << "\n"; - return(true); -} - -//////////////////////////////////////////////////////////////////////// - -bool DiagFile::read_diag_data(ConcatString &name, NumArray &data) { - bool status = false; - - if(FileType == TCDiagFileType) { - status = read_tcdiag_data(name, data); - } - else if(FileType == LSDiagFileType) { - status = read_lsdiag_data(name, data); - } - else { - mlog << Error << "\nDiagFile::read_diag_data() -> " - << "unexpected file type!\n\n"; - exit(1); - } - - return(status); -} - -//////////////////////////////////////////////////////////////////////// - -bool DiagFile::read_tcdiag_data(ConcatString &name, NumArray &data) { - DataLine dl; - ConcatString cs; - int i; - - // Initialize - name.clear(); - data.erase(); - bool status = false; - - // Read until a finding a line of data to parse - while(dl.read_line(this)) { - - // Skip empty lines - if(dl.n_items() == 0) continue; - - // Quit reading at the COMMENTS section - if((cs = dl[1]) == "COMMENTS") { - status = false; - break; - } - - // Set name from the first column - name = dl[0]; - - // Skip certain lines - if(name.startswith("----") || name.startswith("TIME") || - name.startswith("NLEV") || name.startswith("NVAR")) continue; - - // Parse the data values - for(i=2; i " - << "the number of \"" << name << "\" diagnostic values (" - << data.n() << ") does not match the expected number (" - << NTime << ")!\n\n"; - exit(1); - } - else { - - // Break out after reading a good line of data - status = true; - break; - } - - } // end while - - return(status); -} - -//////////////////////////////////////////////////////////////////////// - -bool DiagFile::read_lsdiag_data(ConcatString &name, NumArray &data) { - DataLine dl; - int i; - - // Initialize - name.clear(); - data.erase(); - bool status = false; - - // Read until a finding a line of data to parse + // Store the diagnostics data while(read_fwf_line(dl, lsdiag_wdth, n_lsdiag_wdth)) { // Skip empty lines if(dl.n_items() == 0) continue; - // Set name from column 24 - name = dl[24]; + // Check the 25th column + cs = dl[24]; // Quit reading at the LAST line - if(name == "LAST") { - status = false; - break; - } + if(cs == "LAST") break; // Parse the data values + data.erase(); for(i=2; i<23; i++) { data.add(atoi(dl[i]) == lsdiag_fill_value ? bad_data_double : atof(dl[i])); @@ -434,22 +406,22 @@ bool DiagFile::read_lsdiag_data(ConcatString &name, NumArray &data) { // Check for the expected number of items if(NTime != data.n()) { - mlog << Error << "\nDiagFile::read_lsdiag_data() -> " - << "the number of \"" << name << "\" diagnostic values (" + mlog << Error << "\nDiagFile::read_lsdiag() -> " + << "the number of \"" << cs << "\" diagnostic values (" << data.n() << ") does not match the expected number (" << NTime << ")!\n\n"; exit(1); } + // Store the data in the map else { - - // Break out after reading a good line of data - status = true; - break; + DiagMap[cs] = data; } - } // end while - return(status); + // Close the input file + close(); + + return; } //////////////////////////////////////////////////////////////////////// diff --git a/src/libcode/vx_tc_util/diag_file.h b/src/libcode/vx_tc_util/diag_file.h index 1406a62062..d6245d4322 100644 --- a/src/libcode/vx_tc_util/diag_file.h +++ b/src/libcode/vx_tc_util/diag_file.h @@ -15,6 +15,7 @@ #include #include +#include #include "vx_cal.h" #include "data_line.h" @@ -76,6 +77,9 @@ class DiagFile : public LineDataFile { NumArray Lat; NumArray Lon; + // Diagnostic values + std::map DiagMap; + public: DiagFile(); @@ -104,26 +108,29 @@ class DiagFile : public LineDataFile { double lat(int) const; double lon(int) const; + int n_diag() const; + bool has_diag(const std::string &) const; + const NumArray & get_diag(const std::string &) const; + void get_diag_name(StringArray &) const; + // // do stuff // - bool open_tcdiag(const std::string &, const std::string &); - bool open_lsdiag(const std::string &, const std::string &); + void read_tcdiag(const std::string &, const std::string &); + void read_lsdiag(const std::string &, const std::string &); - bool read_diag_data (ConcatString &, NumArray &); - bool read_tcdiag_data(ConcatString &, NumArray &); - bool read_lsdiag_data(ConcatString &, NumArray &); }; //////////////////////////////////////////////////////////////////////// -inline const ConcatString & DiagFile::storm_id() const { return(StormId); } -inline const ConcatString & DiagFile::basin() const { return(Basin); } -inline const ConcatString & DiagFile::cyclone() const { return(Cyclone); } -inline const ConcatString & DiagFile::technique() const { return(Technique); } -inline unixtime DiagFile::init() const { return(InitTime); } -inline int DiagFile::n_time() const { return(NTime); } +inline const ConcatString & DiagFile::storm_id() const { return(StormId); } +inline const ConcatString & DiagFile::basin() const { return(Basin); } +inline const ConcatString & DiagFile::cyclone() const { return(Cyclone); } +inline const ConcatString & DiagFile::technique() const { return(Technique); } +inline unixtime DiagFile::init() const { return(InitTime); } +inline int DiagFile::n_time() const { return(NTime); } +inline int DiagFile::n_diag() const { return(DiagMap.size()); } //////////////////////////////////////////////////////////////////////// diff --git a/src/libcode/vx_tc_util/tc_columns.cc b/src/libcode/vx_tc_util/tc_columns.cc index 94fb10b289..0a54e7a99d 100644 --- a/src/libcode/vx_tc_util/tc_columns.cc +++ b/src/libcode/vx_tc_util/tc_columns.cc @@ -36,8 +36,7 @@ void open_tc_txt_file(ofstream *&out, const char *file_name) { out->open(file_name); if(!(*out)) { - mlog << Error - << "\nopen_tc_txt_file()-> " + mlog << Error << "\nopen_tc_txt_file()-> " << "can't open the output file \"" << file_name << "\" for writing!\n\n"; exit(1); @@ -356,8 +355,17 @@ void write_tc_diag_cols(const TrackPairInfo &p, int i, at.set_entry(r, c++, i+1); at.set_entry(r, c++, p.adeck()[i].n_diag()); + // Check the number of names and values match + if(p.adeck().n_diag() != p.adeck()[i].n_diag()) { + mlog << Error << "\nwrite_tc_diag_cols()-> " + << "the number of diagnostic names (" << p.adeck().n_diag() + << ") and values (" << p.adeck()[i].n_diag() + << ") do not match!\n\n"; + exit(1); + } + for(j=0; j 0 ? unix_to_yyyymmdd_hhmmss(MaxValidTime).text() : na_str) << "\n"; out << prefix << "MinWarmCore = \"" << (MinWarmCore > 0 ? unix_to_yyyymmdd_hhmmss(MinWarmCore).text() : na_str) << "\n"; out << prefix << "MaxWarmCore = \"" << (MaxWarmCore > 0 ? unix_to_yyyymmdd_hhmmss(MaxWarmCore).text() : na_str) << "\n"; + out << prefix << "NDiag = " << DiagName.n() << "\n"; out << prefix << "NPoints = " << NPoints << "\n"; out << prefix << "NAlloc = " << NAlloc << "\n"; out << prefix << "NTrackLines = " << TrackLines.n() << "\n"; @@ -168,6 +170,7 @@ ConcatString TrackInfo::serialize() const { << ", MaxValidTime = " << (MaxValidTime > 0 ? unix_to_yyyymmdd_hhmmss(MaxValidTime).text() : na_str) << ", MinWarmCore = " << (MinWarmCore > 0 ? unix_to_yyyymmdd_hhmmss(MinWarmCore).text() : na_str) << ", MaxWarmCore = " << (MaxWarmCore > 0 ? unix_to_yyyymmdd_hhmmss(MaxWarmCore).text() : na_str) + << ", NDiag = " << DiagName.n() << ", NPoints = " << NPoints << ", NAlloc = " << NAlloc << ", NTrackLines = " << TrackLines.n(); @@ -217,6 +220,7 @@ void TrackInfo::assign(const TrackInfo &t) { MaxValidTime = t.MaxValidTime; MinWarmCore = t.MinWarmCore; MaxWarmCore = t.MaxWarmCore; + DiagName = t.DiagName; TrackLines = t.TrackLines; if(t.NPoints == 0) return; @@ -392,6 +396,12 @@ int TrackInfo::warm_core_dur() const { //////////////////////////////////////////////////////////////////////// +const char * TrackInfo::diag_name(int i) const { + return(i>=0 && i 0) DiagName = diag_name; + else diag_file.get_diag_name(DiagName); - // Only add requested diagnostics - if(diag_name.has(cur_name)) { + int i_name, i_time, i_pnt; - // Add diagnostic values to the TrackPoints - for(i=0; i n) n = Pair[i].adeck().n_diag(); + } + + return(n); +} + +//////////////////////////////////////////////////////////////////////// + void TrackPairInfoArray::add(const TrackPairInfo &p) { extend(NPairs + 1, false); diff --git a/src/libcode/vx_tc_util/track_pair_info.h b/src/libcode/vx_tc_util/track_pair_info.h index b4e92739ab..d6c0ea3b75 100644 --- a/src/libcode/vx_tc_util/track_pair_info.h +++ b/src/libcode/vx_tc_util/track_pair_info.h @@ -206,8 +206,9 @@ class TrackPairInfoArray { // const TrackPairInfo & operator[](int) const; - int n_pairs() const; - int n_points() const; + int n_pairs() const; + int n_points() const; + int max_n_diag() const; // // do stuff diff --git a/src/libcode/vx_tc_util/track_point.cc b/src/libcode/vx_tc_util/track_point.cc index fc99d7a5a6..d1c2208d63 100644 --- a/src/libcode/vx_tc_util/track_point.cc +++ b/src/libcode/vx_tc_util/track_point.cc @@ -66,8 +66,7 @@ QuadInfo & QuadInfo::operator+=(const QuadInfo &t) { // Check intensity if(is_bad_data(Intensity)) Intensity = t.intensity(); else if(Intensity != t.intensity()) { - mlog << Error - << "\nQuadInfo::operator+=(const QuadInfo &t) -> " + mlog << Error << "\nQuadInfo::operator+=(const QuadInfo &t) -> " << "cannot call += for two different intensity values (" << Intensity << " != " << t.intensity() << ").\n\n"; exit(1); @@ -265,8 +264,7 @@ void QuadInfo::set_quad_vals(QuadrantType ref_quad, break; default: - mlog << Error - << "\nQuadInfo::set_quad_vals() -> " + mlog << Error << "\nQuadInfo::set_quad_vals() -> " << "unexpected quadrant type encountered \"" << quadranttype_to_string(ref_quad) << "\".\n\n"; exit(1); @@ -343,8 +341,7 @@ TrackPoint & TrackPoint::operator+=(const TrackPoint &p) { // Check valid time if(ValidTime == (unixtime) 0) ValidTime = p.valid(); else if(ValidTime != p.valid()) { - mlog << Error - << "\nTrackPoint::operator+=(const TrackPoint &p) -> " + mlog << Error << "\nTrackPoint::operator+=(const TrackPoint &p) -> " << "cannot call += for two different valid times (" << unix_to_yyyymmdd_hhmmss(ValidTime) << " != " << unix_to_yyyymmdd_hhmmss(p.valid()) << ").\n\n"; @@ -354,8 +351,7 @@ TrackPoint & TrackPoint::operator+=(const TrackPoint &p) { // Check lead time if(is_bad_data(LeadTime)) LeadTime = p.lead(); else if(LeadTime != p.lead()) { - mlog << Error - << "\nTrackPoint::operator+=(const TrackPoint &p) -> " + mlog << Error << "\nTrackPoint::operator+=(const TrackPoint &p) -> " << "cannot call += for two different lead times (" << sec_to_hhmmss(LeadTime) << " != " << sec_to_hhmmss(p.lead()) << ").\n\n"; @@ -422,7 +418,6 @@ void TrackPoint::clear() { Depth = NoSystemsDepth; WarmCore = false; WatchWarn = NoWatchWarnType; - DiagName.clear(); DiagVal.clear(); // Call clear for each Wind object and then set intensity value @@ -456,10 +451,8 @@ void TrackPoint::dump(ostream &out, int indent_depth) const { out << prefix << "Speed = " << Speed << "\n"; out << prefix << "Depth = " << systemsdepth_to_string(Depth) << "\n"; out << prefix << "WarmCore = " << bool_to_string(WarmCore) << "\n"; - - for(i=0; i " + mlog << Error << "\nTrackPoint::operator[](int) -> " << "range check error for index value " << n << "\n\n"; exit(1); } @@ -601,18 +592,6 @@ void TrackPoint::set_watch_warn(WatchWarnType ww_type, unixtime ww_ut) { //////////////////////////////////////////////////////////////////////// -int TrackPoint::n_diag() const { - return(DiagName.n()); -} - -//////////////////////////////////////////////////////////////////////// - -const char * TrackPoint::diag_name(int i) const { - return(i>=0 && i=0 && i= NWinds)) { - mlog << Error - << "\nQuadInfo::set_wind(int, const QuadInfo) -> " + mlog << Error << "\nQuadInfo::set_wind(int, const QuadInfo) -> " << "range check error (" << n << ").\n\n"; exit(1); } @@ -669,8 +647,7 @@ bool TrackPoint::is_match(const ATCFTrackLine &l) const { //////////////////////////////////////////////////////////////////////// -void TrackPoint::add_diag_data(double diag_lat, double diag_lon, - const ConcatString &name, double val) { +void TrackPoint::add_diag_data(double diag_lat, double diag_lon, double val) { // Check for consistent location if(!is_eq(diag_lat, Lat) || !is_eq(diag_lon, Lon)) { @@ -680,8 +657,7 @@ void TrackPoint::add_diag_data(double diag_lat, double diag_lon, << ")\n"; } - // Store the diagnostic - DiagName.add(name); + // Store the diagnostic value DiagVal.add(val); return; diff --git a/src/libcode/vx_tc_util/track_point.h b/src/libcode/vx_tc_util/track_point.h index cfd69e8ead..3250bf4dc6 100644 --- a/src/libcode/vx_tc_util/track_point.h +++ b/src/libcode/vx_tc_util/track_point.h @@ -167,9 +167,8 @@ class TrackPoint { // Wind Radii QuadInfo Wind[NWinds]; - // Diagnostic names and values - StringArray DiagName; - NumArray DiagVal; + // Diagnostic values + NumArray DiagVal; public: @@ -234,7 +233,6 @@ class TrackPoint { bool warm_core() const; WatchWarnType watch_warn() const; int n_diag() const; - const char * diag_name(int) const; double diag_val(int) const; // @@ -244,7 +242,7 @@ class TrackPoint { void set_wind(int, const QuadInfo &); bool set(const ATCFTrackLine &); bool is_match(const ATCFTrackLine &) const; - void add_diag_data(double, double, const ConcatString &, double); + void add_diag_data(double, double, double); }; @@ -268,24 +266,25 @@ inline void TrackPoint::set_depth(SystemsDepth t) { Depth = t; } inline void TrackPoint::set_warm_core(bool v) { WarmCore = v; } inline void TrackPoint::set_watch_warn(WatchWarnType t) { WatchWarn = t; } -inline unixtime TrackPoint::valid() const { return(ValidTime); } +inline unixtime TrackPoint::valid() const { return(ValidTime); } inline int TrackPoint::valid_hour() const { return(unix_to_sec_of_day(ValidTime)); } -inline int TrackPoint::lead() const { return(LeadTime); } -inline double TrackPoint::lat() const { return(Lat); } -inline double TrackPoint::lon() const { return(Lon); } -inline double TrackPoint::v_max() const { return(Vmax); } -inline double TrackPoint::mslp() const { return(MSLP); } -inline CycloneLevel TrackPoint::level() const { return(Level); } -inline double TrackPoint::radp() const { return(RadP); } -inline double TrackPoint::rrp() const { return(RRP); } -inline double TrackPoint::mrd() const { return(MRD); } -inline double TrackPoint::gusts() const { return(Gusts); } -inline double TrackPoint::eye() const { return(Eye); } -inline double TrackPoint::direction() const { return(Direction); } -inline double TrackPoint::speed() const { return(Speed); } -inline SystemsDepth TrackPoint::depth() const { return(Depth); } -inline bool TrackPoint::warm_core() const { return(WarmCore); } -inline WatchWarnType TrackPoint::watch_warn() const { return(WatchWarn); } +inline int TrackPoint::lead() const { return(LeadTime); } +inline double TrackPoint::lat() const { return(Lat); } +inline double TrackPoint::lon() const { return(Lon); } +inline double TrackPoint::v_max() const { return(Vmax); } +inline double TrackPoint::mslp() const { return(MSLP); } +inline CycloneLevel TrackPoint::level() const { return(Level); } +inline double TrackPoint::radp() const { return(RadP); } +inline double TrackPoint::rrp() const { return(RRP); } +inline double TrackPoint::mrd() const { return(MRD); } +inline double TrackPoint::gusts() const { return(Gusts); } +inline double TrackPoint::eye() const { return(Eye); } +inline double TrackPoint::direction() const { return(Direction); } +inline double TrackPoint::speed() const { return(Speed); } +inline SystemsDepth TrackPoint::depth() const { return(Depth); } +inline bool TrackPoint::warm_core() const { return(WarmCore); } +inline WatchWarnType TrackPoint::watch_warn() const { return(WatchWarn); } +inline int TrackPoint::n_diag() const { return(DiagVal.n()); } //////////////////////////////////////////////////////////////////////// diff --git a/src/tools/tc_utils/tc_pairs/tc_pairs.cc b/src/tools/tc_utils/tc_pairs/tc_pairs.cc index 927b9828e8..24e46be8c0 100644 --- a/src/tools/tc_utils/tc_pairs/tc_pairs.cc +++ b/src/tools/tc_utils/tc_pairs/tc_pairs.cc @@ -1040,7 +1040,7 @@ void process_diags(TrackInfoArray &tracks) { // Loop over the input files for(i=0,n=0; i 0) { + if((n_diag = p.max_n_diag()) > 0) { n_row += p.n_points(); - n_col = max(n_col, get_n_tc_diag_cols(conf_info.DiagName.n())); + n_col = max(n_col, get_n_tc_diag_cols(n_diag)); } // Initialize the output AsciiTable From f8822f2d32cb6fd400b6be7066a2dce9c561455f Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Tue, 11 Oct 2022 10:54:13 -0600 Subject: [PATCH 18/58] Per #392, add TCDIAG header columns for tc_stat and the unit tests to use. --- data/table_files/met_header_columns_V11.0.txt | 1 + internal/test_unit/hdr/met_11_0.hdr | 1 + 2 files changed, 2 insertions(+) diff --git a/data/table_files/met_header_columns_V11.0.txt b/data/table_files/met_header_columns_V11.0.txt index 7a448d8d17..31ec795403 100644 --- a/data/table_files/met_header_columns_V11.0.txt +++ b/data/table_files/met_header_columns_V11.0.txt @@ -37,4 +37,5 @@ V11.0 : MODE : OBJ : VERSION MODEL N_VALID GRID_RES DESC FCST_LEAD FCST_VAL V11.0 : MODE : CTS : VERSION MODEL N_VALID GRID_RES DESC FCST_LEAD FCST_VALID FCST_ACCUM OBS_LEAD OBS_VALID OBS_ACCUM FCST_RAD FCST_THR OBS_RAD OBS_THR FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE FIELD TOTAL FY_OY FY_ON FN_OY FN_ON BASER FMEAN ACC FBIAS PODY PODN POFD FAR CSI GSS HK HSS ODDS V11.0 : TCST : TCMPR : VERSION AMODEL BMODEL DESC STORM_ID BASIN CYCLONE STORM_NAME INIT LEAD VALID INIT_MASK VALID_MASK LINE_TYPE TOTAL INDEX LEVEL WATCH_WARN INITIALS ALAT ALON BLAT BLON TK_ERR X_ERR Y_ERR ALTK_ERR CRTK_ERR ADLAND BDLAND AMSLP BMSLP AMAX_WIND BMAX_WIND AAL_WIND_34 BAL_WIND_34 ANE_WIND_34 BNE_WIND_34 ASE_WIND_34 BSE_WIND_34 ASW_WIND_34 BSW_WIND_34 ANW_WIND_34 BNW_WIND_34 AAL_WIND_50 BAL_WIND_50 ANE_WIND_50 BNE_WIND_50 ASE_WIND_50 BSE_WIND_50 ASW_WIND_50 BSW_WIND_50 ANW_WIND_50 BNW_WIND_50 AAL_WIND_64 BAL_WIND_64 ANE_WIND_64 BNE_WIND_64 ASE_WIND_64 BSE_WIND_64 ASW_WIND_64 BSW_WIND_64 ANW_WIND_64 BNW_WIND_64 ARADP BRADP ARRP BRRP AMRD BMRD AGUSTS BGUSTS AEYE BEYE ADIR BDIR ASPEED BSPEED ADEPTH BDEPTH +V11.0 : TCST : TCDIAG : VERSION AMODEL BMODEL DESC STORM_ID BASIN CYCLONE STORM_NAME INIT LEAD VALID INIT_MASK VALID_MASK LINE_TYPE TOTAL INDEX (N_DIAG) DIAG_[0-9]* VALUE_[0-9]* V11.0 : TCST : PROBRIRW : VERSION AMODEL BMODEL DESC STORM_ID BASIN CYCLONE STORM_NAME INIT LEAD VALID INIT_MASK VALID_MASK LINE_TYPE ALAT ALON BLAT BLON INITIALS TK_ERR X_ERR Y_ERR ADLAND BDLAND RIRW_BEG RIRW_END RIRW_WINDOW AWIND_END BWIND_BEG BWIND_END BDELTA BDELTA_MAX BLEVEL_BEG BLEVEL_END (N_THRESH) THRESH_[0-9]* PROB_[0-9]* diff --git a/internal/test_unit/hdr/met_11_0.hdr b/internal/test_unit/hdr/met_11_0.hdr index 99e888ee32..2cff17ec5d 100644 --- a/internal/test_unit/hdr/met_11_0.hdr +++ b/internal/test_unit/hdr/met_11_0.hdr @@ -34,4 +34,5 @@ MODE_SOA : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_L MODE_POA : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE N_VALID GRID_RES OBJECT_ID OBJECT_CAT CENTROID_DIST BOUNDARY_DIST CONVEX_HULL_DIST ANGLE_DIFF ASPECT_DIFF AREA_RATIO INTERSECTION_AREA UNION_AREA SYMMETRIC_DIFF INTERSECTION_OVER_AREA CURVATURE_RATIO COMPLEXITY_RATIO PERCENTILE_INTENSITY_RATIO INTEREST MODE_CTS : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE N_VALID GRID_RES FIELD TOTAL FY_OY FY_ON FN_OY FN_ON BASER FMEAN ACC FBIAS PODY PODN POFD FAR CSI GSS HK HSS ODDS TCST_TCMPR : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE AMODEL BMODEL STORM_ID BASIN CYCLONE STORM_NAME INIT_MASK VALID_MASK TOTAL INDEX LEVEL WATCH_WARN INITIALS ALAT ALON BLAT BLON TK_ERR X_ERR Y_ERR ALTK_ERR CRTK_ERR ADLAND BDLAND AMSLP BMSLP AMAX_WIND BMAX_WIND AAL_WIND_34 BAL_WIND_34 ANE_WIND_34 BNE_WIND_34 ASE_WIND_34 BSE_WIND_34 ASW_WIND_34 BSW_WIND_34 ANW_WIND_34 BNW_WIND_34 AAL_WIND_50 BAL_WIND_50 ANE_WIND_50 BNE_WIND_50 ASE_WIND_50 BSE_WIND_50 ASW_WIND_50 BSW_WIND_50 ANW_WIND_50 BNW_WIND_50 AAL_WIND_64 BAL_WIND_64 ANE_WIND_64 BNE_WIND_64 ASE_WIND_64 BSE_WIND_64 ASW_WIND_64 BSW_WIND_64 ANW_WIND_64 BNW_WIND_64 ARADP BRADP ARRP BRRP AMRD BMRD AGUSTS BGUSTS AEYE BEYE ADIR BDIR ASPEED BSPEED ADEPTH BDEPTH +TCST_TCDIAG : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE AMODEL BMODEL STORM_ID BASIN CYCLONE STORM_NAME INIT_MASK VALID_MASK TOTAL INDEX N_DIAG _VAR_ TCST_PROBRIRW : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE ALAT ALON BLAT BLON INITIALS TK_ERR X_ERR Y_ERR ADLAND BDLAND RI_BEG RI_END RI_WINDOW AWIND_END BWIND_BEG BWIND_END BDELTA BDELTA_MAX BLEVEL_BEG BLEVEL_END N_THRESH _VAR_ From 4759ea1b7d4c52bb6c20f5d386d5bc42db607017 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Thu, 13 Oct 2022 10:33:12 -0600 Subject: [PATCH 19/58] Per #392, still more work to do. But for TC-Stat, I'm now tracking the input TCMPR and TCDIAG lines separately. --- src/libcode/vx_tc_util/tc_columns.cc | 2 +- src/libcode/vx_tc_util/track_info.cc | 24 +++- src/libcode/vx_tc_util/track_info.h | 31 +++-- src/libcode/vx_tc_util/track_pair_info.cc | 155 ++++++++++++---------- src/libcode/vx_tc_util/track_pair_info.h | 18 ++- src/libcode/vx_tc_util/track_point.cc | 4 +- src/libcode/vx_tc_util/track_point.h | 2 +- src/tools/tc_utils/tc_pairs/tc_pairs.cc | 2 +- 8 files changed, 143 insertions(+), 95 deletions(-) diff --git a/src/libcode/vx_tc_util/tc_columns.cc b/src/libcode/vx_tc_util/tc_columns.cc index 0a54e7a99d..987b88c350 100644 --- a/src/libcode/vx_tc_util/tc_columns.cc +++ b/src/libcode/vx_tc_util/tc_columns.cc @@ -187,7 +187,7 @@ void write_track_pair_info(TcHdrColumns &hdr, const TrackPairInfo &p, // Set the description if(p.n_lines() > i) { - hdr.set_desc((string)p.line(i)->get_item("DESC", false)); + hdr.set_desc((string)p.tcmpr_line(i)->get_item("DESC", false)); } // Write the header columns diff --git a/src/libcode/vx_tc_util/track_info.cc b/src/libcode/vx_tc_util/track_info.cc index 7e295e758a..7ae2a95c79 100644 --- a/src/libcode/vx_tc_util/track_info.cc +++ b/src/libcode/vx_tc_util/track_info.cc @@ -552,9 +552,9 @@ bool TrackInfo::add_diag_data(DiagFile &diag_file, const StringArray &diag_name) if((i_pnt = lead_index(nint(diag_file.lead(i_time)))) < 0) continue; // Store this diagnostic value in the TrackPoint - Point[i_pnt].add_diag_data(diag_file.lat(i_time), - diag_file.lon(i_time), - diag_val[i_time]); + Point[i_pnt].add_diag_value(diag_file.lat(i_time), + diag_file.lon(i_time), + diag_val[i_time]); } // end for i_time } // end for i_name @@ -564,6 +564,24 @@ bool TrackInfo::add_diag_data(DiagFile &diag_file, const StringArray &diag_name) //////////////////////////////////////////////////////////////////////// +void TrackInfo::add_diag_value(int i_pnt, double val) { + + // Range check + if(i_pnt < 0 || i_pnt >= NPoints) { + mlog << Error << "\nTrackInfo::add_diag_value() -> " + << "range check error for point " << i_pnt << "\n\n"; + exit(1); + } + + Point[i_pnt].add_diag_value(Point[i_pnt].lat(), + Point[i_pnt].lon(), + val); + + return; +} + +//////////////////////////////////////////////////////////////////////// + bool TrackInfo::has(const ATCFTrackLine &l) const { return(TrackLines.has(l.get_line())); } diff --git a/src/libcode/vx_tc_util/track_info.h b/src/libcode/vx_tc_util/track_info.h index 7746d8ae5d..82c49e2cc9 100644 --- a/src/libcode/vx_tc_util/track_info.h +++ b/src/libcode/vx_tc_util/track_info.h @@ -112,6 +112,7 @@ class TrackInfo { void set_valid_min(const unixtime); void set_valid_max(const unixtime); void set_point(int, const TrackPoint &); + void set_diag_name(const StringArray &); // // get stuff @@ -143,6 +144,7 @@ class TrackInfo { int valid_inc() const; int n_points() const; int n_diag() const; + const StringArray & diag_name() const; const char * diag_name(int) const; StringArray track_lines() const; @@ -155,6 +157,7 @@ class TrackInfo { bool add(const ATCFTrackLine &, bool check_dup = false, bool check_anly = false); void add_watch_warn(const ConcatString &, WatchWarnType, unixtime); bool add_diag_data(DiagFile &, const StringArray &); + void add_diag_value(int, double); bool has(const ATCFTrackLine &) const; @@ -167,19 +170,20 @@ class TrackInfo { //////////////////////////////////////////////////////////////////////// -inline bool TrackInfo::is_best_track() const { return(IsBestTrack); } -inline bool TrackInfo::is_oper_track() const { return(IsOperTrack); } -inline bool TrackInfo::is_anly_track() const { return(IsAnlyTrack); } -inline void TrackInfo::set_storm_id(const char *s) { StormId = s; } -inline void TrackInfo::set_basin(const char *s) { Basin = s; } -inline void TrackInfo::set_cyclone(const char *s) { Cyclone = s; } -inline void TrackInfo::set_storm_name(const char *s) { StormName = s; } -inline void TrackInfo::set_technique_number(int i) { TechniqueNumber = i; } -inline void TrackInfo::set_technique(const char *s) { Technique = s; } -inline void TrackInfo::set_initials(const char *s) { Initials = s; } -inline void TrackInfo::set_init(const unixtime u) { InitTime = u; } -inline void TrackInfo::set_valid_min(const unixtime u) { MinValidTime = u; } -inline void TrackInfo::set_valid_max(const unixtime u) { MaxValidTime = u; } +inline bool TrackInfo::is_best_track() const { return(IsBestTrack); } +inline bool TrackInfo::is_oper_track() const { return(IsOperTrack); } +inline bool TrackInfo::is_anly_track() const { return(IsAnlyTrack); } +inline void TrackInfo::set_storm_id(const char *s) { StormId = s; } +inline void TrackInfo::set_basin(const char *s) { Basin = s; } +inline void TrackInfo::set_cyclone(const char *s) { Cyclone = s; } +inline void TrackInfo::set_storm_name(const char *s) { StormName = s; } +inline void TrackInfo::set_technique_number(int i) { TechniqueNumber = i; } +inline void TrackInfo::set_technique(const char *s) { Technique = s; } +inline void TrackInfo::set_initials(const char *s) { Initials = s; } +inline void TrackInfo::set_init(const unixtime u) { InitTime = u; } +inline void TrackInfo::set_valid_min(const unixtime u) { MinValidTime = u; } +inline void TrackInfo::set_valid_max(const unixtime u) { MaxValidTime = u; } +inline void TrackInfo::set_diag_name(const StringArray &s) { DiagName = s; } inline const ConcatString & TrackInfo::storm_id() const { return(StormId); } inline const ConcatString & TrackInfo::basin() const { return(Basin); } @@ -196,6 +200,7 @@ inline unixtime TrackInfo::warm_core_min() const { return(MinWarm inline unixtime TrackInfo::warm_core_max() const { return(MaxWarmCore); } inline int TrackInfo::n_points() const { return(NPoints); } inline int TrackInfo::n_diag() const { return(DiagName.n()); } +inline const StringArray & TrackInfo::diag_name() const { return(DiagName); } inline StringArray TrackInfo::track_lines() const { return(TrackLines); } //////////////////////////////////////////////////////////////////////// diff --git a/src/libcode/vx_tc_util/track_pair_info.cc b/src/libcode/vx_tc_util/track_pair_info.cc index f517c6ce92..f163bf22de 100644 --- a/src/libcode/vx_tc_util/track_pair_info.cc +++ b/src/libcode/vx_tc_util/track_pair_info.cc @@ -63,9 +63,6 @@ TrackPairInfo & TrackPairInfo::operator=(const TrackPairInfo & t) { void TrackPairInfo::init_from_scratch() { - // Initialize pointers - Line = (TCStatLine *) 0; - clear(); return; @@ -76,6 +73,7 @@ void TrackPairInfo::init_from_scratch() { void TrackPairInfo::clear() { NPoints = 0; + NLines = 0; ADeck.clear(); BDeck.clear(); ADeckDLand.clear(); @@ -90,9 +88,8 @@ void TrackPairInfo::clear() { ADeckPrvInt.clear(); BDeckPrvInt.clear(); Keep.clear(); - - if(Line) { delete [] Line; Line = (TCStatLine *) 0; } - NLines = NAlloc = 0; + TCMPRLine.clear(); + TCDIAGLine.clear(); return; } @@ -103,6 +100,7 @@ void TrackPairInfo::dump(ostream &out, int indent_depth) const { Indent prefix(indent_depth); out << prefix << "NPoints = " << NPoints << "\n"; + out << prefix << "NLines = " << NLines << "\n"; out << prefix << "ADeck:\n"; ADeck.dump(out, indent_depth+1); out << prefix << "BDeck:\n"; @@ -131,7 +129,6 @@ void TrackPairInfo::dump(ostream &out, int indent_depth) const { BDeckPrvInt.dump(out, indent_depth+1); out << prefix << "Keep:\n"; Keep.dump(out, indent_depth+1); - out << prefix << "NLines = " << NLines << "\n"; out << flush; return; @@ -199,6 +196,7 @@ void TrackPairInfo::assign(const TrackPairInfo &t) { clear(); NPoints = t.NPoints; + NLines = t.NLines; ADeck = t.ADeck; BDeck = t.BDeck; ADeckDLand = t.ADeckDLand; @@ -213,54 +211,8 @@ void TrackPairInfo::assign(const TrackPairInfo &t) { ADeckPrvInt = t.ADeckPrvInt; BDeckPrvInt = t.BDeckPrvInt; Keep = t.Keep; - - extend(t.NLines); - - for(i=0; i= n) return; - - // Compute the allocation size - if(!exact) { - k = n/TrackPairLineAllocInc; - if(n%TrackPairLineAllocInc) k++; - n = k*TrackPairLineAllocInc; - } - - // Allocate a new TCStatLine array of the required length - new_line = new TCStatLine [n]; - - if(!new_line) { - mlog << Error - << "\nvoid TrackPairInfo::extend(int, bool) -> " - << "memory allocation error\n\n"; - exit(1); - } - - // Copy the array contents and delete the old one - if(Line) { - for(j=0; j= NPoints) { - mlog << Error - << "\nTrackPairInfo::set_keep(int, int) -> " + mlog << Error << "\nTrackPairInfo::set_keep(int, int) -> " << "range check error for index value " << i << "\n\n"; exit(1); } @@ -348,12 +299,25 @@ void TrackPairInfo::add(const TrackPoint &a, const TrackPoint &b, BDeckPrvInt.add(bad_data_double); Keep.add(1); + // JHG what about diags here? + return; } //////////////////////////////////////////////////////////////////////// void TrackPairInfo::add(const TCStatLine &l) { + + // Check the line type + if(l.type() == TCStatLineType_TCMPR) add_tcmpr_line(l); + else if(l.type() == TCStatLineType_TCDIAG) add_tcdiag_line(l); + + return; +} + +//////////////////////////////////////////////////////////////////////// + +void TrackPairInfo::add_tcmpr_line(const TCStatLine &l) { TrackPoint apoint, bpoint; TrackPoint *tp = (TrackPoint *) 0; QuadInfo wind; @@ -363,8 +327,14 @@ void TrackPairInfo::add(const TCStatLine &l) { // Check the line type if(l.type() != TCStatLineType_TCMPR) return; - // Increment the count + // Store the input TCMPR line and TCDIAG placeholder + TCMPRLine.push_back(l); + TCStatLine empty_line; + TCDIAGLine.push_back(empty_line); + + // Increment the point and line count NPoints++; + NLines++; // Initialize the ADECK/BDECK tracks if(NPoints == 1) initialize(l); @@ -404,7 +374,7 @@ void TrackPairInfo::add(const TCStatLine &l) { cs << cs_erase << deck[i] << "SPEED"; tp->set_speed(atof(l.get_item(cs.c_str()))); cs << cs_erase << deck[i] << "DEPTH"; - tp->set_eye(string_to_systemsdepth(l.get_item(cs.c_str()))); + tp->set_depth(string_to_systemsdepth(l.get_item(cs.c_str()))); tp->set_watch_warn(string_to_watchwarntype(l.get_item("WATCH_WARN"))); // Loop over the winds @@ -450,9 +420,61 @@ void TrackPairInfo::add(const TCStatLine &l) { BDeckPrvInt.add(bad_data_double); Keep.add(1); - // Store the input line - extend(NLines + 1, false); - Line[NLines++] = l; + return; +} + +//////////////////////////////////////////////////////////////////////// + +void TrackPairInfo::add_tcdiag_line(const TCStatLine &l) { + int n_diag, i; + ConcatString cs; + + // Check the line type + if(l.type() != TCStatLineType_TCDIAG) return; + + // Update the TCDIAGLine for this TrackPoint + TCDIAGLine[NPoints-1] = l; + + // Increment the line count + NLines++; + + // Check for a match + if(ADeck.storm_id() != l.storm_id() || + ADeck.technique() != l.amodel() || + ADeck.init() != l.init() || + ADeck[NPoints-1].lead() != l.lead()) { + mlog << Error << "\nadd_tcdiag_line() -> " + << "the TCDIAG data does not match the track data!\n\n"; + exit(1); + } + + // Name of diagnostics read + StringArray diag_name; + + // Number of diagnostics + n_diag = atoi(l.get_item("N_DIAG")); + + // Parse all the diagnostics entries + for(i=0; i " + << "the number of TCDIAG diagnostics have changed (" + << ADeck.diag_name().n() << " != " << diag_name.n() + << ")!\n\n"; + exit(1); + } return; } @@ -795,7 +817,8 @@ TrackPairInfo TrackPairInfo::keep_subset() const { // Add from TC-Stat line data if(NLines == NPoints) { - tpi.add(Line[i]); + tpi.add_tcmpr_line(TCMPRLine[i]); + tpi.add_tcdiag_line(TCDIAGLine[i]); } // Otherwise, add from TC-Pairs track pair else { @@ -956,8 +979,7 @@ void TrackPairInfoArray::extend(int n, bool exact) { new_info = new TrackPairInfo [n]; if(!new_info) { - mlog << Error - << "\nvoid TrackPairInfoArray::extend(int, bool) -> " + mlog << Error << "\nvoid TrackPairInfoArray::extend(int, bool) -> " << "memory allocation error\n\n"; exit(1); } @@ -984,8 +1006,7 @@ const TrackPairInfo & TrackPairInfoArray::operator[](int n) const { // Check range if((n < 0) || (n >= NPairs)) { - mlog << Error - << "\nTrackPairInfoArray::operator[](int) -> " + mlog << Error << "\nTrackPairInfoArray::operator[](int) -> " << "range check error for index value " << n << "\n\n"; exit(1); } diff --git a/src/libcode/vx_tc_util/track_pair_info.h b/src/libcode/vx_tc_util/track_pair_info.h index d6c0ea3b75..69db58d9e8 100644 --- a/src/libcode/vx_tc_util/track_pair_info.h +++ b/src/libcode/vx_tc_util/track_pair_info.h @@ -14,6 +14,7 @@ //////////////////////////////////////////////////////////////////////// #include +#include #include "track_point.h" #include "track_info.h" @@ -42,10 +43,10 @@ class TrackPairInfo { void init_from_scratch(); void assign(const TrackPairInfo &); - void extend(int, bool exact = true); - // Number of track points + // Number of track points and input lines int NPoints; + int NLines; // ADECK and BDECK tracks TrackInfo ADeck; @@ -63,9 +64,8 @@ class TrackPairInfo { NumArray CrossTrackErr; // TCStatLines used to construct this track - int NLines; - int NAlloc; - TCStatLine * Line; + std::vector TCMPRLine; // sized by NPoint + std::vector TCDIAGLine; // sized by NPoint // Status for whether RI/RW occurred NumArray ADeckRIRW; @@ -121,7 +121,8 @@ class TrackPairInfo { int bdeck_prv_int(int) const; int n_lines() const; - const TCStatLine * line(int i) const; + const TCStatLine * tcmpr_line(int i) const; + const TCStatLine * tcdiag_line(int i) const; int i_init() const; bool keep(int) const; @@ -134,6 +135,8 @@ class TrackPairInfo { void add(const TrackPoint &, const TrackPoint &, double, double, double, double, double, double, double); void add(const TCStatLine &); + void add_tcmpr_line(const TCStatLine &); + void add_tcdiag_line(const TCStatLine &); void add_watch_warn(const ConcatString &, WatchWarnType, unixtime); int check_water_only(); @@ -163,7 +166,8 @@ inline int TrackPairInfo::bdeck_rirw(int i) const { return(n inline int TrackPairInfo::adeck_prv_int(int i) const { return(nint(ADeckPrvInt[i])); } inline int TrackPairInfo::bdeck_prv_int(int i) const { return(nint(BDeckPrvInt[i])); } inline int TrackPairInfo::n_lines() const { return(NLines); } -inline const TCStatLine * TrackPairInfo::line(int i) const { return(&Line[i]); } +inline const TCStatLine * TrackPairInfo::tcmpr_line(int i) const { return(&TCMPRLine[i]); } +inline const TCStatLine * TrackPairInfo::tcdiag_line(int i) const { return(&TCDIAGLine[i]); } inline bool TrackPairInfo::keep(int i) const { return(Keep[i] != 0); } //////////////////////////////////////////////////////////////////////// diff --git a/src/libcode/vx_tc_util/track_point.cc b/src/libcode/vx_tc_util/track_point.cc index d1c2208d63..dabe5ae869 100644 --- a/src/libcode/vx_tc_util/track_point.cc +++ b/src/libcode/vx_tc_util/track_point.cc @@ -647,11 +647,11 @@ bool TrackPoint::is_match(const ATCFTrackLine &l) const { //////////////////////////////////////////////////////////////////////// -void TrackPoint::add_diag_data(double diag_lat, double diag_lon, double val) { +void TrackPoint::add_diag_value(double diag_lat, double diag_lon, double val) { // Check for consistent location if(!is_eq(diag_lat, Lat) || !is_eq(diag_lon, Lon)) { - mlog << Warning << "\nTrackPoint::add_diag_data() -> " + mlog << Warning << "\nTrackPoint::add_diag_value() -> " << "the diagnostic location (" << diag_lat << ", " << diag_lon << ") does not match the track location (" << Lat << ", " << Lon << ")\n"; diff --git a/src/libcode/vx_tc_util/track_point.h b/src/libcode/vx_tc_util/track_point.h index 3250bf4dc6..4b92121d68 100644 --- a/src/libcode/vx_tc_util/track_point.h +++ b/src/libcode/vx_tc_util/track_point.h @@ -242,7 +242,7 @@ class TrackPoint { void set_wind(int, const QuadInfo &); bool set(const ATCFTrackLine &); bool is_match(const ATCFTrackLine &) const; - void add_diag_data(double, double, double); + void add_diag_value(double, double, double); }; diff --git a/src/tools/tc_utils/tc_pairs/tc_pairs.cc b/src/tools/tc_utils/tc_pairs/tc_pairs.cc index 24e46be8c0..a2e63b05cf 100644 --- a/src/tools/tc_utils/tc_pairs/tc_pairs.cc +++ b/src/tools/tc_utils/tc_pairs/tc_pairs.cc @@ -184,7 +184,7 @@ int met_main(int argc, char *argv[]) { //////////////////////////////////////////////////////////////////////// const string get_tool_name() { - return "tc_gen"; + return "tc_pairs"; } //////////////////////////////////////////////////////////////////////// From 19b2df827499cf95c98450ad68fecfdbf2c1ae1a Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Thu, 13 Oct 2022 10:34:00 -0600 Subject: [PATCH 20/58] Per #392, couple for changes for the previous commit. --- src/tools/tc_utils/tc_stat/tc_stat_files.cc | 12 ++- src/tools/tc_utils/tc_stat/tc_stat_job.cc | 91 +++++++++++++-------- 2 files changed, 63 insertions(+), 40 deletions(-) diff --git a/src/tools/tc_utils/tc_stat/tc_stat_files.cc b/src/tools/tc_utils/tc_stat/tc_stat_files.cc index bd77dad0f4..917be384fa 100644 --- a/src/tools/tc_utils/tc_stat/tc_stat_files.cc +++ b/src/tools/tc_utils/tc_stat/tc_stat_files.cc @@ -161,12 +161,16 @@ bool TCStatFiles::operator>>(TrackPairInfo &pair) { // Read lines to the end of the track or file while(CurLDF >> line) { - // Skip header and non-TCMPR lines - if(line.is_header() || line.type() != TCStatLineType_TCMPR) continue; + // Skip header and non-TCMPR/TCDIAG lines + if(line.is_header() || + (line.type() != TCStatLineType_TCMPR && + line.type() != TCStatLineType_TCDIAG)) continue; // Add the current point pair.add(line); +// JHG need to work on this logic! + // Break out of the loop at the end of the track if(atoi(line.get_item("TOTAL")) == atoi(line.get_item("INDEX"))) break; @@ -224,7 +228,7 @@ bool TCStatFiles::operator>>(ProbRIRWPairInfo &pair) { break; - } //end while + } // end while return(status); } @@ -270,7 +274,7 @@ bool TCStatFiles::operator>>(TCStatLine &line) { break; - } //end while + } // end while return(status); } diff --git a/src/tools/tc_utils/tc_stat/tc_stat_job.cc b/src/tools/tc_utils/tc_stat/tc_stat_job.cc index f2d47ede1d..19c6ccf4e6 100644 --- a/src/tools/tc_utils/tc_stat/tc_stat_job.cc +++ b/src/tools/tc_utils/tc_stat/tc_stat_job.cc @@ -575,7 +575,7 @@ bool TCStatJob::is_keeper_track(const TrackPairInfo &pair, for(thr_it=InitThreshMap.begin(); thr_it!= InitThreshMap.end(); thr_it++) { // Get the numeric init column value - v_dbl = get_column_double(*pair.line(i_init), thr_it->first); + v_dbl = get_column_double(*pair.tcmpr_line(i_init), thr_it->first); // Check the column threshold if(!thr_it->second.check_dbl(v_dbl)) { @@ -592,7 +592,7 @@ bool TCStatJob::is_keeper_track(const TrackPairInfo &pair, for(str_it=InitStrIncMap.begin(); str_it!= InitStrIncMap.end(); str_it++) { // Retrieve the column value - v_str = pair.line(i_init)->get_item(str_it->first.c_str()); + v_str = pair.tcmpr_line(i_init)->get_item(str_it->first.c_str()); // Check the string value if(!str_it->second.has(v_str)) { @@ -609,7 +609,7 @@ bool TCStatJob::is_keeper_track(const TrackPairInfo &pair, for(str_it=InitStrExcMap.begin(); str_it!= InitStrExcMap.end(); str_it++) { // Retrieve the column value - v_str = pair.line(i_init)->get_item(str_it->first.c_str()); + v_str = pair.tcmpr_line(i_init)->get_item(str_it->first.c_str()); // Check the string value if(str_it->second.has(v_str)) { @@ -1102,15 +1102,25 @@ void TCStatJob::dump_pair(const TrackPairInfo &pair, ofstream *out) { TcHdrColumns tchc; AsciiTable out_at; - int i_row, hdr_row; + int i_row, hdr_row, n_row, n_col, n_diag; + + // Number of output rows and columns + n_row = pair.n_points(); + n_col = n_tc_mpr_cols; // Determine if we need to write a header row if((int) out->tellp() == 0) hdr_row = 1; else hdr_row = 0; + // Update rows and columns for diagnosics output + if((n_diag = pair.adeck().n_diag()) > 0) { + n_row += pair.n_points(); + n_col = max(n_col, get_n_tc_diag_cols(n_diag)); + } + // Initialize the output AsciiTable - out_at.set_size(pair.n_points() + hdr_row, - n_tc_header_cols + n_tc_mpr_cols); + out_at.set_size(n_row + hdr_row, + n_tc_header_cols + n_col); // Write the TCMPR header row write_tc_mpr_header_row(1, out_at, 0, 0); @@ -1366,7 +1376,7 @@ void TCStatJob::event_equalize_tracks() { // Store the current map key and value key = pair.adeck().technique(); // ADECK model name - val = pair.line(i)->header(); // Track line header + val = pair.tcmpr_line(i)->header(); // Track line header // Add a new map entry, if necessary if(case_map.count(key) == 0) { @@ -1491,8 +1501,8 @@ void TCStatJob::subset_track_pair(TrackPairInfo &pair, TCLineCounts &n) { if(pair.n_points() == 0) return; // Increment the read and keep counts - n.NRead += pair.n_points(); - n.NKeep += pair.n_points(); + n.NRead += pair.n_lines(); + n.NKeep += pair.n_lines(); // Determine if the whole track can be discarded if(!is_keeper_track(pair, n)) { @@ -1537,13 +1547,14 @@ void TCStatJob::subset_track_pair(TrackPairInfo &pair, TCLineCounts &n) { } // Loop over the track points to check line by line - for(i=0; i " << "the track-based " << TCStatLineType_TCMPR_Str - << " line type cannot be mixed with the " + << " and " << TCStatLineType_TCDIAG_Str + << " line types cannot be mixed with the " << LineType[i] << " line type.\n\n"; exit(1); } @@ -1721,7 +1737,7 @@ void TCStatJobFilter::filter_tracks(TCLineCounts &n) { void TCStatJobFilter::filter_lines(TCLineCounts &n) { TCStatLine line; - +cout << "JHGa\n"; // Apply the event equalization logic to build a list of common cases if(EventEqual == true) event_equalize_lines(); @@ -1742,7 +1758,7 @@ void TCStatJobFilter::filter_lines(TCLineCounts &n) { // Increment the read and keep counts n.NRead++; n.NKeep++; - +cout << "JHG read/keep = " << n.NRead << "/" << n.NKeep << "\n"; // Check if this line should be kept if(!is_keeper_line(line, n)) continue; @@ -1988,12 +2004,14 @@ void TCStatJobSummary::do_job(const StringArray &file_list, // Process track-based filter job if(LineType.has(TCStatLineType_TCMPR_Str)) { - // TCMPR and non-TCMPR LineTypes cannot be mixed + // TCMPR and non-TCMPR/TCDIAG LineTypes cannot be mixed for(i=0; i " << "the track-based " << TCStatLineType_TCMPR_Str - << " line type cannot be mixed with the " + << " and " << TCStatLineType_TCDIAG_Str + << " line types cannot be mixed with the " << LineType[i] << " line type.\n\n"; exit(1); } @@ -2121,19 +2139,19 @@ void TCStatJobSummary::process_pair(TrackPairInfo &pair) { // For -column_union, put all columns in the same key if(ColumnUnion) prefix = write_css(ReqColumn); else prefix = Column[j]; - key = build_map_key(prefix.c_str(), *pair.line(i), ByColumn); - val = get_column_double(*pair.line(i), Column[j]); + key = build_map_key(prefix.c_str(), *pair.tcmpr_line(i), ByColumn); + val = get_column_double(*pair.tcmpr_line(i), Column[j]); // Add map entry for this key, if necessary if(cur_map.count(key) == 0) cur_map[key] = data; // Add values for this key cur_map[key].Val.add(val); - cur_map[key].Hdr.add(pair.line(i)->header()); - cur_map[key].AModel.add(pair.line(i)->amodel()); - cur_map[key].Init.add(pair.line(i)->init()); - cur_map[key].Lead.add(pair.line(i)->lead()); - cur_map[key].Valid.add(pair.line(i)->valid()); + cur_map[key].Hdr.add(pair.tcmpr_line(i)->header()); + cur_map[key].AModel.add(pair.tcmpr_line(i)->amodel()); + cur_map[key].Init.add(pair.tcmpr_line(i)->init()); + cur_map[key].Lead.add(pair.tcmpr_line(i)->lead()); + cur_map[key].Valid.add(pair.tcmpr_line(i)->valid()); } // end for j } // end for i @@ -3039,7 +3057,7 @@ void TCStatJobRIRW::process_pair(TrackPairInfo &pair) { for(i=0; iamodel() << sep - << pair.line(i)->bmodel() << sep - << pair.line(i)->storm_id() << sep - << unix_to_yyyymmdd_hhmmss(pair.line(i)->init()) << sep + << pair.tcmpr_line(i)->amodel() << sep + << pair.tcmpr_line(i)->bmodel() << sep + << pair.tcmpr_line(i)->storm_id() << sep + << unix_to_yyyymmdd_hhmmss(pair.tcmpr_line(i)->init()) << sep << (is_bad_data(lead) ? na_str : sec_to_hhmmss(lead).text()) << sep - << unix_to_yyyymmdd_hhmmss(pair.line(i)->valid()) << sep + << unix_to_yyyymmdd_hhmmss(pair.tcmpr_line(i)->valid()) << sep << aprv << sep << acur << sep << adlt << sep << (is_bad_data(a) ? na_str : bool_to_string(a)) << sep << bprv << sep << bcur << sep << bdlt << sep @@ -3968,6 +3986,7 @@ void TCStatJobProbRIRW::do_output(ostream &out) { //////////////////////////////////////////////////////////////////////// TCLineCounts::TCLineCounts() { + // Read and keep counts NRead = 0; NKeep = 0; From 9c143fc808c91fdf36e2fe299920ddf69aebddd7 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Thu, 13 Oct 2022 15:28:04 -0600 Subject: [PATCH 21/58] Per #392, add support -diag_thresh and -init_diag_thresh job filtering options. Switch from reporting the number of lines read/kept to the number of track points to make the logic easier. This works for the filter job type, but still need to test it for other job types. --- src/tools/tc_utils/tc_stat/tc_stat.cc | 6 +- src/tools/tc_utils/tc_stat/tc_stat_job.cc | 130 ++++++++++++++++++++-- src/tools/tc_utils/tc_stat/tc_stat_job.h | 8 ++ 3 files changed, 131 insertions(+), 13 deletions(-) diff --git a/src/tools/tc_utils/tc_stat/tc_stat.cc b/src/tools/tc_utils/tc_stat/tc_stat.cc index c536227149..665214775e 100644 --- a/src/tools/tc_utils/tc_stat/tc_stat.cc +++ b/src/tools/tc_utils/tc_stat/tc_stat.cc @@ -222,11 +222,12 @@ void process_jobs() { else mlog << Debug(1) << method_name << "job is missing\n"; mlog << Debug(3) - << "Total lines read = " << n.NRead << "\n" - << "Total lines kept = " << n.NKeep << "\n" + << "Total track points read = " << n.NRead << "\n" + << "Total track points kept = " << n.NKeep << "\n" << "Rejected for track watch/warn = " << n.RejTrackWatchWarn << "\n" << "Rejected for init threshold = " << n.RejInitThresh << "\n" << "Rejected for init string = " << n.RejInitStr << "\n" + << "Rejected for init diag threshold = " << n.RejInitDiagThresh << "\n" << "Rejected for out init mask = " << n.RejOutInitMask << "\n" << "Rejected for water only = " << n.RejWaterOnly << "\n" << "Rejected for rapid inten = " << n.RejRIRW << "\n" @@ -249,6 +250,7 @@ void process_jobs() { << "Rejected for line type = " << n.RejLineType << "\n" << "Rejected for numeric threshold = " << n.RejColumnThresh << "\n" << "Rejected for string matching = " << n.RejColumnStr << "\n" + << "Rejected for diag threshold = " << n.RejDiagThresh << "\n" << "Rejected for match points = " << n.RejMatchPoints << "\n" << "Rejected for event equal = " << n.RejEventEqual << "\n" << "Rejected for out init mask = " << n.RejOutInitMask << "\n" diff --git a/src/tools/tc_utils/tc_stat/tc_stat_job.cc b/src/tools/tc_utils/tc_stat/tc_stat_job.cc index 19c6ccf4e6..0eeae343cd 100644 --- a/src/tools/tc_utils/tc_stat/tc_stat_job.cc +++ b/src/tools/tc_utils/tc_stat/tc_stat_job.cc @@ -224,6 +224,8 @@ void TCStatJob::clear() { InitThreshMap.clear(); InitStrIncMap.clear(); InitStrExcMap.clear(); + DiagThreshMap.clear(); + InitDiagThreshMap.clear(); EventEqualLead.clear(); EventEqualCases.clear(); @@ -307,6 +309,8 @@ void TCStatJob::assign(const TCStatJob & j) { InitThreshMap = j.InitThreshMap; InitStrIncMap = j.InitStrIncMap; InitStrExcMap = j.InitStrExcMap; + DiagThreshMap = j.DiagThreshMap; + InitDiagThreshMap = j.InitDiagThreshMap; DumpFile = j.DumpFile; open_dump_file(); @@ -456,6 +460,18 @@ void TCStatJob::dump(ostream & out, int depth) const { str_it->second.dump(out, depth + 1); } + out << prefix << "DiagThreshMap ...\n"; + for(thr_it=DiagThreshMap.begin(); thr_it!= DiagThreshMap.end(); thr_it++) { + out << prefix << thr_it->first << ": \n"; + thr_it->second.dump(out, depth + 1); + } + + out << prefix << "InitDiagThreshMap ...\n"; + for(thr_it=InitDiagThreshMap.begin(); thr_it!= InitDiagThreshMap.end(); thr_it++) { + out << prefix << thr_it->first << ": \n"; + thr_it->second.dump(out, depth + 1); + } + out << prefix << "WaterOnly = " << bool_to_string(WaterOnly) << "\n"; out << prefix << "RIRWTrack = " << tracktype_to_string(RIRWTrack) << "\n"; @@ -546,7 +562,7 @@ bool TCStatJob::is_keeper_track(const TrackPairInfo &pair, if(!keep) n.RejTrackWatchWarn += pair.n_points(); // Get the index of the track initialization point - i_init=pair.i_init(); + i_init = pair.i_init(); // Check for bad track initialization point if(is_bad_data(i_init)) { @@ -562,6 +578,10 @@ bool TCStatJob::is_keeper_track(const TrackPairInfo &pair, keep = false; n.RejInitStr += pair.n_points(); } + else if(InitDiagThreshMap.size() > 0) { + keep = false; + n.RejInitDiagThresh += pair.n_points(); + } else if(OutInitMaskName.nonempty()) { keep = false; n.RejOutInitMask += pair.n_points(); @@ -620,6 +640,31 @@ bool TCStatJob::is_keeper_track(const TrackPairInfo &pair, } } + // Check InitDiagThreshMap + if(keep == true) { + + int i_diag; + + // Loop through the numeric diagnostic thresholds + for(thr_it=InitDiagThreshMap.begin(); thr_it!= InitDiagThreshMap.end(); thr_it++) { + + // Check whether the diagnostic name exists + if(!pair.adeck().diag_name().has(thr_it->first, i_diag)) { + keep = false; + break; + } + + // Check that diagnostic value + if(!thr_it->second.check_dbl(pair.adeck()[i_init].diag_val(i_diag))) { + keep = false; + break; + } + } + + // Update counts + if(!keep) n.RejInitDiagThresh += pair.n_points(); + } + // Check OutInitMask if(keep == true) { @@ -666,6 +711,10 @@ bool TCStatJob::is_keeper_track(const TrackPairInfo &pair, bool TCStatJob::is_keeper_line(const TCStatLine &line, TCLineCounts &n) const { + + // Does not apply to TCDIAG lines + if(line.type() == TCStatLineType_TCDIAG) return(true); + bool keep = true; double v_dbl, alat, alon, blat, blon; ConcatString v_str; @@ -881,6 +930,42 @@ double TCStatJob::get_column_double(const TCStatLine &line, //////////////////////////////////////////////////////////////////////// +bool TCStatJob::is_keeper_tcdiag(const StringArray &diag_name, + const TrackPoint &point, + TCLineCounts &n) const { + bool keep = true; + int i_diag; + map::const_iterator thr_it; + + // Check ColumnThreshMap + + // Loop through the numeric diagnostic thresholds + for(thr_it=DiagThreshMap.begin(); thr_it!= DiagThreshMap.end(); thr_it++) { + + // Check whether the diagnostic name exists + if(!diag_name.has(thr_it->first, i_diag)) { + keep = false; + break; + } + + // Check that diagnostic value + if(!thr_it->second.check_dbl(point.diag_val(i_diag))) { + keep = false; + break; + } + } + + // Update counts + if(!keep) { + n.RejDiagThresh++; + n.NKeep -= 1; + } + + return(keep); +} + +//////////////////////////////////////////////////////////////////////// + StringArray TCStatJob::parse_job_command(const char *jobstring) { StringArray a, b; string c; @@ -940,6 +1025,10 @@ StringArray TCStatJob::parse_job_command(const char *jobstring) { a.shift_down(i, 2); } else if(c.compare("-init_str_exc" ) == 0) { parse_string_option(a[i+1].c_str(), a[i+2].c_str(), InitStrExcMap); a.shift_down(i, 2); } + else if(c.compare("-diag_thresh" ) == 0) { parse_thresh_option(a[i+1].c_str(), a[i+2].c_str(), DiagThreshMap); + a.shift_down(i, 2); } + else if(c.compare("-init_diag_thresh" ) == 0) { parse_thresh_option(a[i+1].c_str(), a[i+2].c_str(), InitDiagThreshMap); + a.shift_down(i, 2); } else if(c.compare("-water_only" ) == 0) { WaterOnly = string_to_bool(a[i+1].c_str()); a.shift_down(i, 1); } else if(c.compare("-rirw_track" ) == 0) { RIRWTrack = string_to_tracktype(a[i+1].c_str()); a.shift_down(i, 1); } else if(c.compare("-rirw_time" ) == 0) { RIRWTimeADeck = timestring_to_sec(a[i+1].c_str()); @@ -1263,6 +1352,18 @@ ConcatString TCStatJob::serialize() const { << str_it->second[i] << " "; } } + for(thr_it=DiagThreshMap.begin(); thr_it!= DiagThreshMap.end(); thr_it++) { + for(i=0; isecond.n(); i++) { + s << "-diag_thresh " << thr_it->first << " " + << thr_it->second[i].get_str() << " "; + } + } + for(thr_it=InitDiagThreshMap.begin(); thr_it!= InitDiagThreshMap.end(); thr_it++) { + for(i=0; isecond.n(); i++) { + s << "-init_diag_thresh " << thr_it->first << " " + << thr_it->second[i].get_str() << " "; + } + } if(WaterOnly != default_water_only) s << "-water_only " << bool_to_string(WaterOnly) << " "; if(RIRWTrack != default_rirw_track) { @@ -1501,8 +1602,8 @@ void TCStatJob::subset_track_pair(TrackPairInfo &pair, TCLineCounts &n) { if(pair.n_points() == 0) return; // Increment the read and keep counts - n.NRead += pair.n_lines(); - n.NKeep += pair.n_lines(); + n.NRead += pair.n_points(); + n.NKeep += pair.n_points(); // Determine if the whole track can be discarded if(!is_keeper_track(pair, n)) { @@ -1549,12 +1650,17 @@ void TCStatJob::subset_track_pair(TrackPairInfo &pair, TCLineCounts &n) { // Loop over the track points to check line by line for(i=0; i InitStrIncMap; std::map InitStrExcMap; + // Numeric diagnostic thresholds + std::map DiagThreshMap; + std::map InitDiagThreshMap; + // Variables to the store the analysis job specification ConcatString DumpFile; // Dump TrackPairInfo used to a file std::ofstream *DumpOut; // Dump output file stream From d087f8edd355a1ae30dd140f82f81eb3dcbe20a8 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Thu, 13 Oct 2022 16:29:42 -0600 Subject: [PATCH 22/58] Per #392, update the tc_stat summary job to process TCDIAG values. --- src/libcode/vx_tc_util/track_pair_info.cc | 8 ++- src/tools/tc_utils/tc_stat/out.tcst | 0 src/tools/tc_utils/tc_stat/tc_stat_files.cc | 4 +- src/tools/tc_utils/tc_stat/tc_stat_job.cc | 70 ++++++++++++++------- src/tools/tc_utils/tc_stat/tc_stat_job.h | 2 + 5 files changed, 56 insertions(+), 28 deletions(-) create mode 100644 src/tools/tc_utils/tc_stat/out.tcst diff --git a/src/libcode/vx_tc_util/track_pair_info.cc b/src/libcode/vx_tc_util/track_pair_info.cc index f163bf22de..481198afad 100644 --- a/src/libcode/vx_tc_util/track_pair_info.cc +++ b/src/libcode/vx_tc_util/track_pair_info.cc @@ -273,6 +273,10 @@ void TrackPairInfo::set_keep(int i, int val) { return; } +//////////////////////////////////////////////////////////////////////// +// +// Add derived TrackPoint pairs such as for consensus tracks +// //////////////////////////////////////////////////////////////////////// void TrackPairInfo::add(const TrackPoint &a, const TrackPoint &b, @@ -299,8 +303,6 @@ void TrackPairInfo::add(const TrackPoint &a, const TrackPoint &b, BDeckPrvInt.add(bad_data_double); Keep.add(1); - // JHG what about diags here? - return; } @@ -816,7 +818,7 @@ TrackPairInfo TrackPairInfo::keep_subset() const { if(Keep[i] == 0) continue; // Add from TC-Stat line data - if(NLines == NPoints) { + if(TCMPRLine.size() > 0) { tpi.add_tcmpr_line(TCMPRLine[i]); tpi.add_tcdiag_line(TCDIAGLine[i]); } diff --git a/src/tools/tc_utils/tc_stat/out.tcst b/src/tools/tc_utils/tc_stat/out.tcst new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/tools/tc_utils/tc_stat/tc_stat_files.cc b/src/tools/tc_utils/tc_stat/tc_stat_files.cc index 917be384fa..08c63ac4ae 100644 --- a/src/tools/tc_utils/tc_stat/tc_stat_files.cc +++ b/src/tools/tc_utils/tc_stat/tc_stat_files.cc @@ -169,12 +169,12 @@ bool TCStatFiles::operator>>(TrackPairInfo &pair) { // Add the current point pair.add(line); -// JHG need to work on this logic! - // Break out of the loop at the end of the track if(atoi(line.get_item("TOTAL")) == atoi(line.get_item("INDEX"))) break; +// JHG need to work on this logic! We'll exit after reading the last TCMPR line but before reading the last TCDIAG line! + } // end while return(true); diff --git a/src/tools/tc_utils/tc_stat/tc_stat_job.cc b/src/tools/tc_utils/tc_stat/tc_stat_job.cc index 0eeae343cd..dc74ee86da 100644 --- a/src/tools/tc_utils/tc_stat/tc_stat_job.cc +++ b/src/tools/tc_utils/tc_stat/tc_stat_job.cc @@ -934,38 +934,49 @@ bool TCStatJob::is_keeper_tcdiag(const StringArray &diag_name, const TrackPoint &point, TCLineCounts &n) const { bool keep = true; - int i_diag; + double v_dbl; map::const_iterator thr_it; - // Check ColumnThreshMap + // Check DiagThreshMap // Loop through the numeric diagnostic thresholds for(thr_it=DiagThreshMap.begin(); thr_it!= DiagThreshMap.end(); thr_it++) { - // Check whether the diagnostic name exists - if(!diag_name.has(thr_it->first, i_diag)) { - keep = false; - break; - } + // Get the numeric diagnostic value + v_dbl = get_diag_double(diag_name, point, thr_it->first); - // Check that diagnostic value - if(!thr_it->second.check_dbl(point.diag_val(i_diag))) { + // Check the diagnostic threshold + if(!thr_it->second.check_dbl(v_dbl)) { keep = false; + n.RejDiagThresh++; break; } } // Update counts - if(!keep) { - n.RejDiagThresh++; - n.NKeep -= 1; - } + if(!keep) n.NKeep -= 1; return(keep); } //////////////////////////////////////////////////////////////////////// +double TCStatJob::get_diag_double(const StringArray &diag_name, + const TrackPoint &point, + const ConcatString &diag_cs) const { + double v = bad_data_double; + int i_diag; + + // Check whether the diagnostic name exists + if(diag_name.has(diag_cs, i_diag)) { + v = point.diag_val(i_diag); + } + + return(v); +} + +//////////////////////////////////////////////////////////////////////// + StringArray TCStatJob::parse_job_command(const char *jobstring) { StringArray a, b; string c; @@ -1473,11 +1484,11 @@ void TCStatJob::event_equalize_tracks() { if(skip) continue; // Add event equalization information for each track point - for(i=0; iheader(); // Track line header + key = pair.adeck().technique(); // ADECK model name + val = pair.tcmpr_line(i)->header(); // Track line header // Add a new map entry, if necessary if(case_map.count(key) == 0) { @@ -2101,8 +2112,11 @@ void TCStatJobSummary::do_job(const StringArray &file_list, // Apply logic based on the LineType // - // If not specified, assume TCMPR by adding it to the LineType - if(LineType.n() == 0) LineType.add(TCStatLineType_TCMPR_Str); + // If not specified, assume TCMPR/TCDIAG by adding them to the LineType + if(LineType.n() == 0) { + LineType.add(TCStatLineType_TCMPR_Str); + LineType.add(TCStatLineType_TCDIAG_Str); + } // Add the input file list TCSTFiles.add_files(file_list); @@ -2235,8 +2249,8 @@ void TCStatJobSummary::process_pair(TrackPairInfo &pair) { // Initialize the map cur_map.clear(); - // Loop over TCStatLines and construct a summary map - for(i=0; i Date: Thu, 13 Oct 2022 16:37:07 -0600 Subject: [PATCH 23/58] Per #392, rename TCLineCounts struct to TCPointCounts to clarify that we're now counting input track points, not lines. --- src/tools/tc_utils/tc_stat/tc_stat.cc | 6 ++--- src/tools/tc_utils/tc_stat/tc_stat_job.cc | 32 +++++++++++------------ src/tools/tc_utils/tc_stat/tc_stat_job.h | 32 +++++++++++------------ 3 files changed, 35 insertions(+), 35 deletions(-) diff --git a/src/tools/tc_utils/tc_stat/tc_stat.cc b/src/tools/tc_utils/tc_stat/tc_stat.cc index 665214775e..52e0c5adaf 100644 --- a/src/tools/tc_utils/tc_stat/tc_stat.cc +++ b/src/tools/tc_utils/tc_stat/tc_stat.cc @@ -167,7 +167,7 @@ void process_jobs() { TCStatJob *cur_job = (TCStatJob *) 0; ConcatString jobstring; int i, n_jobs; - TCLineCounts n; + TCPointCounts n; const char *method_name = "process_jobs() -> "; // Open the output file @@ -210,14 +210,14 @@ void process_jobs() { << cur_job->serialize() << "\n"; // Initialize counts - memset(&n, 0, sizeof(TCLineCounts)); + memset(&n, 0, sizeof(TCPointCounts)); // Do the job cur_job->do_job(tcst_files, n); mlog << Debug(2) << method_name << "Job " << i+1 << " used " << n.NKeep << " out of " - << n.NRead << " lines read.\n"; + << n.NRead << " track points read.\n"; } else mlog << Debug(1) << method_name << "job is missing\n"; diff --git a/src/tools/tc_utils/tc_stat/tc_stat_job.cc b/src/tools/tc_utils/tc_stat/tc_stat_job.cc index dc74ee86da..0bd1b2f054 100644 --- a/src/tools/tc_utils/tc_stat/tc_stat_job.cc +++ b/src/tools/tc_utils/tc_stat/tc_stat_job.cc @@ -523,7 +523,7 @@ void TCStatJob::dump(ostream & out, int depth) const { //////////////////////////////////////////////////////////////////////// bool TCStatJob::is_keeper_track(const TrackPairInfo &pair, - TCLineCounts &n) const { + TCPointCounts &n) const { bool keep = true; int i, i_init; double v_dbl; @@ -710,7 +710,7 @@ bool TCStatJob::is_keeper_track(const TrackPairInfo &pair, //////////////////////////////////////////////////////////////////////// bool TCStatJob::is_keeper_line(const TCStatLine &line, - TCLineCounts &n) const { + TCPointCounts &n) const { // Does not apply to TCDIAG lines if(line.type() == TCStatLineType_TCDIAG) return(true); @@ -932,7 +932,7 @@ double TCStatJob::get_column_double(const TCStatLine &line, bool TCStatJob::is_keeper_tcdiag(const StringArray &diag_name, const TrackPoint &point, - TCLineCounts &n) const { + TCPointCounts &n) const { bool keep = true; double v_dbl; map::const_iterator thr_it; @@ -1431,7 +1431,7 @@ ConcatString TCStatJob::serialize() const { //////////////////////////////////////////////////////////////////////// -void TCStatJob::do_job(const StringArray &file_list, TCLineCounts &n) { +void TCStatJob::do_job(const StringArray &file_list, TCPointCounts &n) { mlog << Error << "\nTCStatJob::do_job() -> " << "the do_job() base class function should never be called!\n\n"; @@ -1448,7 +1448,7 @@ void TCStatJob::do_job(const StringArray &file_list, TCLineCounts &n) { void TCStatJob::event_equalize_tracks() { TrackPairInfo pair; - TCLineCounts n; + TCPointCounts n; ConcatString key, val; StringArray case_list; ConcatString models; @@ -1536,7 +1536,7 @@ void TCStatJob::event_equalize_tracks() { //////////////////////////////////////////////////////////////////////// void TCStatJob::event_equalize_lines() { - TCLineCounts n; + TCPointCounts n; TCStatLine line; ConcatString key, val; StringArray case_list; @@ -1606,7 +1606,7 @@ void TCStatJob::event_equalize_lines() { //////////////////////////////////////////////////////////////////////// -void TCStatJob::subset_track_pair(TrackPairInfo &pair, TCLineCounts &n) { +void TCStatJob::subset_track_pair(TrackPairInfo &pair, TCPointCounts &n) { int i, n_rej; // Check for no points @@ -1752,7 +1752,7 @@ void TCStatJobFilter::assign(const TCStatJobFilter & j) { //////////////////////////////////////////////////////////////////////// void TCStatJobFilter::do_job(const StringArray &file_list, - TCLineCounts &n) { + TCPointCounts &n) { int i; // Check that the -dump_row option has been supplied @@ -1813,7 +1813,7 @@ void TCStatJobFilter::do_job(const StringArray &file_list, //////////////////////////////////////////////////////////////////////// -void TCStatJobFilter::filter_tracks(TCLineCounts &n) { +void TCStatJobFilter::filter_tracks(TCPointCounts &n) { TrackPairInfo pair; // Apply the event equalization logic to build a list of common cases @@ -1852,7 +1852,7 @@ void TCStatJobFilter::filter_tracks(TCLineCounts &n) { //////////////////////////////////////////////////////////////////////// -void TCStatJobFilter::filter_lines(TCLineCounts &n) { +void TCStatJobFilter::filter_lines(TCPointCounts &n) { TCStatLine line; // Apply the event equalization logic to build a list of common cases @@ -2095,7 +2095,7 @@ ConcatString TCStatJobSummary::serialize() const { //////////////////////////////////////////////////////////////////////// void TCStatJobSummary::do_job(const StringArray &file_list, - TCLineCounts &n) { + TCPointCounts &n) { TrackPairInfo pair; int i; @@ -2157,7 +2157,7 @@ void TCStatJobSummary::do_job(const StringArray &file_list, //////////////////////////////////////////////////////////////////////// -void TCStatJobSummary::summarize_tracks(TCLineCounts &n) { +void TCStatJobSummary::summarize_tracks(TCPointCounts &n) { TrackPairInfo pair; // Apply the event equalization logic to build a list of common cases @@ -2199,7 +2199,7 @@ void TCStatJobSummary::summarize_tracks(TCLineCounts &n) { //////////////////////////////////////////////////////////////////////// -void TCStatJobSummary::summarize_lines(TCLineCounts &n) { +void TCStatJobSummary::summarize_lines(TCPointCounts &n) { TCStatLine line; // Apply the event equalization logic to build a list of common cases @@ -3115,7 +3115,7 @@ ConcatString TCStatJobRIRW::serialize() const { //////////////////////////////////////////////////////////////////////// void TCStatJobRIRW::do_job(const StringArray &file_list, - TCLineCounts &n) { + TCPointCounts &n) { TrackPairInfo pair; // Add the input file list @@ -3864,7 +3864,7 @@ ConcatString TCStatJobProbRIRW::serialize() const { //////////////////////////////////////////////////////////////////////// void TCStatJobProbRIRW::do_job(const StringArray &file_list, - TCLineCounts &n) { + TCPointCounts &n) { ProbRIRWPairInfo pair; // Check that -probrirw_thresh has been supplied @@ -4115,7 +4115,7 @@ void TCStatJobProbRIRW::do_output(ostream &out) { //////////////////////////////////////////////////////////////////////// -TCLineCounts::TCLineCounts() { +TCPointCounts::TCPointCounts() { // Read and keep counts NRead = 0; diff --git a/src/tools/tc_utils/tc_stat/tc_stat_job.h b/src/tools/tc_utils/tc_stat/tc_stat_job.h index 837b3e9ca8..475080343f 100644 --- a/src/tools/tc_utils/tc_stat/tc_stat_job.h +++ b/src/tools/tc_utils/tc_stat/tc_stat_job.h @@ -118,8 +118,8 @@ extern ConcatString tcstatjobtype_to_string(const TCStatJobType); //////////////////////////////////////////////////////////////////////// -// Struct for counts of lines read and rejected -struct TCLineCounts { +// Struct for counts of track points read and rejected +struct TCPointCounts { // Read and keep counts int NRead; @@ -161,7 +161,7 @@ struct TCLineCounts { int RejOutValidMask; int RejLeadReq; - TCLineCounts(); + TCPointCounts(); }; //////////////////////////////////////////////////////////////////////// @@ -207,13 +207,13 @@ class TCStatJob { ////////////////////////////////////////////////////////////////// - bool is_keeper_track(const TrackPairInfo &, TCLineCounts &) const; + bool is_keeper_track(const TrackPairInfo &, TCPointCounts &) const; - bool is_keeper_line(const TCStatLine &, TCLineCounts &) const; + bool is_keeper_line(const TCStatLine &, TCPointCounts &) const; double get_column_double(const TCStatLine &, const ConcatString &) const; - bool is_keeper_tcdiag(const StringArray &, const TrackPoint &, TCLineCounts &) const; + bool is_keeper_tcdiag(const StringArray &, const TrackPoint &, TCPointCounts &) const; double get_diag_double(const StringArray &, const TrackPoint &, const ConcatString &) const; @@ -237,13 +237,13 @@ class TCStatJob { ////////////////////////////////////////////////////////////////// - virtual void do_job(const StringArray &, TCLineCounts &); + virtual void do_job(const StringArray &, TCPointCounts &); void event_equalize_tracks(); void event_equalize_lines(); - void subset_track_pair(TrackPairInfo &, TCLineCounts &); + void subset_track_pair(TrackPairInfo &, TCPointCounts &); ////////////////////////////////////////////////////////////////// @@ -381,10 +381,10 @@ class TCStatJobFilter : public TCStatJob { void clear(); - void do_job(const StringArray &, TCLineCounts &); // virtual from base class + void do_job(const StringArray &, TCPointCounts &); // virtual from base class - void filter_tracks(TCLineCounts &); - void filter_lines (TCLineCounts &); + void filter_tracks(TCPointCounts &); + void filter_lines (TCPointCounts &); void do_output(std::ostream &); @@ -415,10 +415,10 @@ class TCStatJobSummary : public TCStatJob { ConcatString serialize() const; - void do_job(const StringArray &, TCLineCounts &); // virtual from base class + void do_job(const StringArray &, TCPointCounts &); // virtual from base class - void summarize_tracks(TCLineCounts &); - void summarize_lines (TCLineCounts &); + void summarize_tracks(TCPointCounts &); + void summarize_lines (TCPointCounts &); void process_pair(TrackPairInfo &); void process_line(TCStatLine &); @@ -479,7 +479,7 @@ class TCStatJobRIRW : public TCStatJob { ConcatString serialize() const; - void do_job(const StringArray &, TCLineCounts &); // virtual from base class + void do_job(const StringArray &, TCPointCounts &); // virtual from base class void process_pair(TrackPairInfo &); @@ -529,7 +529,7 @@ class TCStatJobProbRIRW : public TCStatJob { ConcatString serialize() const; - void do_job(const StringArray &, TCLineCounts &); // virtual from base class + void do_job(const StringArray &, TCPointCounts &); // virtual from base class void process_pair(ProbRIRWPairInfo &); From 342dbc4f9e7673c7748fb37f9a16f75811ed5d92 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Wed, 19 Oct 2022 09:48:16 -0600 Subject: [PATCH 24/58] Per #392, add more error checking. Each TCDIAG line must be preceded by a TCMPR. If they don't match, error out with a useful error message. --- src/libcode/vx_tc_util/track_pair_info.cc | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/src/libcode/vx_tc_util/track_pair_info.cc b/src/libcode/vx_tc_util/track_pair_info.cc index e6402fe24f..5fbaa281eb 100644 --- a/src/libcode/vx_tc_util/track_pair_info.cc +++ b/src/libcode/vx_tc_util/track_pair_info.cc @@ -441,11 +441,14 @@ void TrackPairInfo::add_tcdiag_line(const TCStatLine &l) { // Check the line type if(l.type() != TCStatLineType_TCDIAG) return; - // Update the TCDIAGLine for this TrackPoint - TCDIAGLine[NPoints-1] = l; - - // Increment the line count - NLines++; + // Should have already parsed TCMPR + if(NPoints == 0) { + mlog << Error << "\nadd_tcdiag_line() -> " + << "each TCDIAG line must be preceded by the TCMPR line " + << "to which it corresponds:\n" + << " " << l << "\n\n"; + exit(1); + } // Check for a match if(ADeck.storm_id() != l.storm_id() || @@ -453,10 +456,18 @@ void TrackPairInfo::add_tcdiag_line(const TCStatLine &l) { ADeck.init() != l.init() || ADeck[NPoints-1].lead() != l.lead()) { mlog << Error << "\nadd_tcdiag_line() -> " - << "the TCDIAG data does not match the track data!\n\n"; + << "the TCDIAG data does not match the TCMPR data:\n" + << " TCMPR Line: " << TCMPRLine[NPoints-1] << "\n" + << " TCDIAG Line: " << l << "\n\n"; exit(1); } + // Update the TCDIAGLine for this TrackPoint + TCDIAGLine[NPoints-1] = l; + + // Increment the line count + NLines++; + // Name of diagnostics read StringArray diag_name; From a3804c79d50e4bf21ca429c4bec469d16f7c0997 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Wed, 19 Oct 2022 10:58:28 -0600 Subject: [PATCH 25/58] Per #392, add LineDataFile::peek_line() function and then update tc_stat_files.cc to call it when parsing .tcst files. Need to check for and parse trailing TCDIAG lines. ci-run-unit --- src/basic/vx_util/data_line.cc | 28 +++++++++++++++++++++ src/basic/vx_util/data_line.h | 2 ++ src/tools/tc_utils/tc_stat/tc_stat_files.cc | 26 ++++++++++++------- 3 files changed, 47 insertions(+), 9 deletions(-) diff --git a/src/basic/vx_util/data_line.cc b/src/basic/vx_util/data_line.cc index f7692fdeb0..dade54254c 100644 --- a/src/basic/vx_util/data_line.cc +++ b/src/basic/vx_util/data_line.cc @@ -815,6 +815,34 @@ return ( 1 ); } +//////////////////////////////////////////////////////////////////////// +// +// Read the next line but do not move the read pointer +// +//////////////////////////////////////////////////////////////////////// + + +int LineDataFile::peek_line(DataLine & a) + +{ + +int status = 0; + +if ( ok() ) { + + int cur_pos = in->tellg(); + + status = a.read_line(this); + + in->seekg(cur_pos); + +} + +return ( 1 ); + +} + + //////////////////////////////////////////////////////////////////////// diff --git a/src/basic/vx_util/data_line.h b/src/basic/vx_util/data_line.h index 4608e8a2bd..fd509d3906 100644 --- a/src/basic/vx_util/data_line.h +++ b/src/basic/vx_util/data_line.h @@ -190,6 +190,8 @@ class LineDataFile { virtual int operator>>(DataLine &); + int peek_line(DataLine &); + int read_fwf_line(DataLine &, const int *wdth, int n_wdth); const char * filename() const; diff --git a/src/tools/tc_utils/tc_stat/tc_stat_files.cc b/src/tools/tc_utils/tc_stat/tc_stat_files.cc index 08c63ac4ae..5f63e0fd57 100644 --- a/src/tools/tc_utils/tc_stat/tc_stat_files.cc +++ b/src/tools/tc_utils/tc_stat/tc_stat_files.cc @@ -137,7 +137,7 @@ bool TCStatFiles::operator>>(TrackPairInfo &pair) { CurFile++; // Check for the last file - if(CurFile == FileList.n_elements()) return(false); + if(CurFile == FileList.n()) return(false); else { // Open the next file for reading @@ -152,7 +152,7 @@ bool TCStatFiles::operator>>(TrackPairInfo &pair) { // List file being read mlog << Debug(3) << "Reading file " << CurFile+1 << " of " - << FileList.n_elements() << ": " << FileList[CurFile] + << FileList.n() << ": " << FileList[CurFile] << "\n"; } // end else @@ -171,10 +171,18 @@ bool TCStatFiles::operator>>(TrackPairInfo &pair) { // Break out of the loop at the end of the track if(atoi(line.get_item("TOTAL")) == - atoi(line.get_item("INDEX"))) break; - -// JHG need to work on this logic! We'll exit after reading the last TCMPR line but before reading the last TCDIAG line! + atoi(line.get_item("INDEX"))) { + + // Check for a trailing TCDIAG line + if(CurLDF.peek_line(line)) { + if(line.type() == TCStatLineType_TCDIAG) { + pair.add(line); + CurLDF >> line; + } + } + break; + } } // end while return(true); @@ -196,7 +204,7 @@ bool TCStatFiles::operator>>(ProbRIRWPairInfo &pair) { CurFile++; // Check for the last file - if(CurFile == FileList.n_elements()) return(false); + if(CurFile == FileList.n()) return(false); else { // Open the next file for reading @@ -211,7 +219,7 @@ bool TCStatFiles::operator>>(ProbRIRWPairInfo &pair) { // List file being read mlog << Debug(3) << "Reading file " << CurFile+1 << " of " - << FileList.n_elements() << ": " << FileList[CurFile] + << FileList.n() << ": " << FileList[CurFile] << "\n"; } // end else @@ -245,7 +253,7 @@ bool TCStatFiles::operator>>(TCStatLine &line) { CurFile++; // Check for the last file - if(CurFile == FileList.n_elements()) return(false); + if(CurFile == FileList.n()) return(false); else { // Open the next file for reading @@ -260,7 +268,7 @@ bool TCStatFiles::operator>>(TCStatLine &line) { // List file being read mlog << Debug(3) << "Reading file " << CurFile+1 << " of " - << FileList.n_elements() << ": " << FileList[CurFile] + << FileList.n() << ": " << FileList[CurFile] << "\n"; } // end else From ad36eb1f9dd0a21d1de7d0fad5b286e1bd4c4b16 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Wed, 19 Oct 2022 12:23:15 -0600 Subject: [PATCH 26/58] Per #392, working on error messages. --- src/libcode/vx_tc_util/track_pair_info.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/libcode/vx_tc_util/track_pair_info.cc b/src/libcode/vx_tc_util/track_pair_info.cc index 5fbaa281eb..72e8aa1212 100644 --- a/src/libcode/vx_tc_util/track_pair_info.cc +++ b/src/libcode/vx_tc_util/track_pair_info.cc @@ -443,7 +443,7 @@ void TrackPairInfo::add_tcdiag_line(const TCStatLine &l) { // Should have already parsed TCMPR if(NPoints == 0) { - mlog << Error << "\nadd_tcdiag_line() -> " + mlog << Error << "\nTrackPairInfo::add_tcdiag_line() -> " << "each TCDIAG line must be preceded by the TCMPR line " << "to which it corresponds:\n" << " " << l << "\n\n"; @@ -455,7 +455,7 @@ void TrackPairInfo::add_tcdiag_line(const TCStatLine &l) { ADeck.technique() != l.amodel() || ADeck.init() != l.init() || ADeck[NPoints-1].lead() != l.lead()) { - mlog << Error << "\nadd_tcdiag_line() -> " + mlog << Error << "\nTrackPairInfo::add_tcdiag_line() -> " << "the TCDIAG data does not match the TCMPR data:\n" << " TCMPR Line: " << TCMPRLine[NPoints-1] << "\n" << " TCDIAG Line: " << l << "\n\n"; @@ -489,7 +489,7 @@ void TrackPairInfo::add_tcdiag_line(const TCStatLine &l) { } // Make sure they do not change else if(ADeck.diag_name().n() != diag_name.n()) { - mlog << Error << "\nadd_tcdiag_line() -> " + mlog << Error << "\nTrackPairInfo::add_tcdiag_line() -> " << "the number of TCDIAG diagnostics have changed (" << ADeck.diag_name().n() << " != " << diag_name.n() << ")!\n\n"; @@ -610,7 +610,7 @@ int TrackPairInfo::check_rirw(const TrackType track_type, if(!exact_adeck && st_adeck.get_type() != thresh_lt && st_adeck.get_type() != thresh_le && st_adeck.get_type() != thresh_gt && st_adeck.get_type() != thresh_ge) { - mlog << Error << "\ncheck_rirw() -> " + mlog << Error << "\nTrackPairInfo::check_rirw() -> " << "for non-exact intensity differences the RI/RW threshold (" << st_adeck.get_str() << ") must be of type <, <=, >, or >=.\n\n"; exit(1); @@ -620,7 +620,7 @@ int TrackPairInfo::check_rirw(const TrackType track_type, if(!exact_bdeck && st_bdeck.get_type() != thresh_lt && st_bdeck.get_type() != thresh_le && st_bdeck.get_type() != thresh_gt && st_bdeck.get_type() != thresh_ge) { - mlog << Error << "\ncheck_rirw() -> " + mlog << Error << "\nTrackPairInfo::check_rirw() -> " << "for non-exact intensity differences the RI/RW threshold (" << st_bdeck.get_str() << ") must be of type <, <=, >, or >=.\n\n"; exit(1); @@ -999,7 +999,7 @@ void TrackPairInfoArray::extend(int n, bool exact) { new_info = new TrackPairInfo [n]; if(!new_info) { - mlog << Error << "\nvoid TrackPairInfoArray::extend(int, bool) -> " + mlog << Error << "\nTrackPairInfoArray::extend(int, bool) -> " << "memory allocation error\n\n"; exit(1); } From ec250c4f4a8ee8fc57b3d996850ae0fd9b7ca8c1 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Wed, 19 Oct 2022 13:21:57 -0600 Subject: [PATCH 27/58] Per #392, for TC-Stat add diag_thresh_name/val and init_diag_thresh_name/val options to the config file. For TC-Pairs, add -tcdiag and -lsdiag to the usage statement. For both, update the User's Guide. --- data/config/TCStatConfig_default | 16 ++++- docs/Users_Guide/tc-pairs.rst | 69 +++++++++++++++++-- docs/Users_Guide/tc-stat.rst | 28 ++++++-- src/basic/vx_config/config_constants.h | 4 ++ src/tools/tc_utils/tc_pairs/tc_pairs.cc | 10 +++ src/tools/tc_utils/tc_stat/tc_stat.cc | 1 + .../tc_utils/tc_stat/tc_stat_conf_info.cc | 10 +++ 7 files changed, 127 insertions(+), 11 deletions(-) diff --git a/data/config/TCStatConfig_default b/data/config/TCStatConfig_default index 5063ad8feb..9925185a7c 100644 --- a/data/config/TCStatConfig_default +++ b/data/config/TCStatConfig_default @@ -93,7 +93,7 @@ line_type = []; // // Stratify by checking the watch/warning status for each track point -// common to both the ADECK and BDECK tracks. If the watch/warning status +// common to both the ADECK and BDECK tracks. If the watch/warning status // of any of the track points appears in the list, retain the entire track. // track_watch_warn = []; @@ -134,6 +134,20 @@ init_str_val = []; init_str_exc_name = []; init_str_exc_val = []; +// +// Stratify track points by applying thresholds to numeric +// storm diagnostic values in the TCDIAG lines. +// +diag_thresh_name = []; +diag_thresh_val = []; + +// +// Stratify entire tracks by applying thresholds to numeric +// storm diagnostic values in the TCDIAG lines for lead time 0. +// +init_diag_thresh_name = []; +init_diag_thresh_val = []; + // // Stratify by the ADECK and BDECK distances to land. // diff --git a/docs/Users_Guide/tc-pairs.rst b/docs/Users_Guide/tc-pairs.rst index 77ee2af4c7..f18ddab66f 100644 --- a/docs/Users_Guide/tc-pairs.rst +++ b/docs/Users_Guide/tc-pairs.rst @@ -9,6 +9,16 @@ Introduction The TC-Pairs tool provides verification for tropical cyclone forecasts in ATCF file format. It matches an ATCF format tropical cyclone (TC) forecast with a second ATCF format reference TC dataset (most commonly the Best Track analysis). The TC-Pairs tool processes both track and intensity adeck data and probabilistic edeck data. The adeck matched pairs contain position errors, as well as wind, sea level pressure, and distance to land values for each TC dataset. The edeck matched pairs contain probabilistic forecast values and the verifying observation values. The pair generation can be subset based on user-defined filtering criteria. Practical aspects of the TC-Pairs tool are described in :numref:`TC-Pairs_Practical-information`. +Scientific and statistical aspects +================================== + +.. _TC-Pairs_Diagnostics: + +Storm Diagnostics +----------------- + +TODO: Add a paragraph about storm diagnostics, describing what they are, why they are important, and how they can be generated. + .. _TC-Pairs_Practical-information: Practical information @@ -27,6 +37,8 @@ The usage statement for tc_pairs is shown below: -adeck source and/or -edeck source -bdeck source -config file + [-tcdiag source] + [-lsdiag source] [-out base] [-log file] [-v level] @@ -47,13 +59,17 @@ Required arguments for tc_pairs Optional arguments for tc_pairs ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -5. The -**out base** argument indicates the path of the output file base. This argument overrides the default output file base (**./out_tcmpr**). +5. The **-tcdiag source** argument indicates the TC-Pairs acceptable format data source containing the tropical cyclone diagnostics dataset corresponding to the adeck tracks. This argument specifies the name of a TC-Pairs acceptable format file or top-level directory containing TC-Pairs acceptable format files ending in ".dat" to be processed. + +6. The **-lsdiag source** argument indicates the TC-Pairs acceptable format data source containing the large scale diagnostics dataset corresponding to the adeck tracks. This argument specifies the name of a TC-Pairs acceptable format file or top-level directory containing TC-Pairs acceptable format files ending in ".dat" to be processed. -6. The **-log file** option directs output and errors to the specified log file. All messages will be written to that file as well as standard out and error. Thus, users can save the messages without having to redirect the output on the command line. The default behavior is no log file. +7. The -**out base** argument indicates the path of the output file base. This argument overrides the default output file base (**./out_tcmpr**). -7. The **-v level** option indicates the desired level of verbosity. The contents of "level" will override the default setting of 2. Setting the verbosity to 0 will make the tool run with no log messages, while increasing the verbosity above 1 will increase the amount of logging. +8. The **-log file** option directs output and errors to the specified log file. All messages will be written to that file as well as standard out and error. Thus, users can save the messages without having to redirect the output on the command line. The default behavior is no log file. -This tool currently only supports the rapid intensification (**RI**) edeck probability type but support for additional edeck probability types will be added in future releases. At least one **-adeck** or **-edeck** option must be specified. The **-adeck, -edeck**, and **-bdeck** options may optionally be followed with **suffix=string** to append that string to all model names found within that data source. This option may be useful when processing track data from two different sources which reuse the same model names. +9. The **-v level** option indicates the desired level of verbosity. The contents of "level" will override the default setting of 2. Setting the verbosity to 0 will make the tool run with no log messages, while increasing the verbosity above 1 will increase the amount of logging. + +This tool currently only supports the rapid intensification (**RI**) edeck probability type but support for additional edeck probability types will be added in future releases. At least one **-adeck** or **-edeck** option must be specified. The **-adeck, -edeck**, and **-bdeck** options may optionally be followed with **suffix=string** to append that string to all model names found within that data source. This option may be useful when processing track data from two different sources which reuse the same model names. The **-tcdiag** and **-lsdiag** options may optionally be followed with **model=string** to override the model name of the tracks to which those diagnostics correspond. An example of the tc_pairs calling sequence is shown below: @@ -235,6 +251,18 @@ ____________________ The **watch_warn** field specifies the file name and time applied offset to the **watch_warn** flag. The **file_name** string specifies the path of the watch/warning file to be used to determine when a watch or warning is in effect during the forecast initialization and verification times. The default file is named **wwpts_us.txt**, which is found in the installed *share/met/tc_data/* directory within the MET build. The **time_offset** string is the time window (in seconds) assigned to the watch/warning. Due to the non-uniform time watches and warnings are issued, a time window is assigned for which watch/warnings are included in the verification for each valid time. The default watch/warn file is static, and therefore may not include warned storms beyond the current MET code release date; therefore users may wish to create a post in the `METplus GitHub Discussions Forum `_ in order to obtain the most recent watch/warning file if the static file does not contain storms of interest. +____________________ + +.. code-block:: none + + diag_name = []; + +The **diag_name** entry specifies a comma-separated list of strings for the tropical cyclone diagnostics of interest. This applies when the **-tcdiag** and/or **-lsdiag** command line options have been used to provide storm diagnostics data. If a non-zero list of diagnostic names is specified, only those diagnostics appearing in the list are written to the TCDIAG output line type. If defined as an empty list (default), all diagnostics found in the input are written to the TCDIAG output lines. + +A TCMPR line is written to the output for each track point. If diagnostics data is also defined for that track point, a TCDIAG line is written immediately after the corresponding TCMPR line. The contents of that TCDIAG line is deteremined by diagnostic names requrested in the **diag_name** entry. + +____________________ + .. code-block:: none basin_map = [ @@ -487,7 +515,38 @@ TC-Pairs produces output in TCST format. The default output file name can be ove * - 84 - MAX_WIND_SPREAD - consensus variable: the standard deviation of the member's maximum wind speed values - + +.. _TCDIAG Line Type: + +.. list-table:: Format information for TCDIAG (Tropical Cyclone Diagnostics) output line type. + :widths: auto + :header-rows: 2 + + * - + - + - TCDIAG OUTPUT FORMAT + * - Column Number + - Header Column Name + - Description + * - 13 + - TCDIAG + - Tropical Cyclone Diagnostics line type + * - 14 + - TOTAL + - Total number of pairs in track + * - 15 + - INDEX + - Index of the current track pair + * - 16 + - N_DIAG + - Number of storm diagnostic name and value columns to follow + * - 17 + - DIAG_i + - Name of the of the ith storm diagnostic (repeated) + * - 18 + - VALUE_i + - Value of the ith storm diagnostic (repeated) + .. _PROBRIRW Line Type: .. list-table:: Format information for PROBRIRW (Probability of Rapid Intensification/Weakening) output line type. diff --git a/docs/Users_Guide/tc-stat.rst b/docs/Users_Guide/tc-stat.rst index 6db00f3160..03013bdf94 100644 --- a/docs/Users_Guide/tc-stat.rst +++ b/docs/Users_Guide/tc-stat.rst @@ -245,7 +245,7 @@ _________________________ column_str_name = []; column_str_val = []; -The **column_str_name** and **column_str_val** fields stratify by performing string matching on non-numeric data columns. Specify a comma-separated list of columns names and values to be **included** in the analysis. The length of the **column_str_val** should match that of the **column_str_name**. Using the **-column_str name val** option within the job command lines may further refine these selections. +The **column_str_name** and **column_str_val** fields stratify by performing string matching on non-numeric data columns. Specify a comma-separated list of columns names and values to be **included** in the analysis. The length of the **column_str_val** should match that of the **column_str_name**. Using the **-column_str name value** option within the job command lines may further refine these selections. _________________________ @@ -254,7 +254,7 @@ _________________________ column_str_exc_name = []; column_str_exc_val = []; -The **column_str_exc_name** and **column_str_exc_val** fields stratify by performing string matching on non-numeric data columns. Specify a comma-separated list of columns names and values to be **excluded** from the analysis. The length of the **column_str_exc_val** should match that of the **column_str_exc_name**. Using the **-column_str_exc name val** option within the job command lines may further refine these selections. +The **column_str_exc_name** and **column_str_exc_val** fields stratify by performing string matching on non-numeric data columns. Specify a comma-separated list of columns names and values to be **excluded** from the analysis. The length of the **column_str_exc_val** should match that of the **column_str_exc_name**. Using the **-column_str_exc name value** option within the job command lines may further refine these selections. _________________________ @@ -263,7 +263,7 @@ _________________________ init_thresh_name = []; init_thresh_val = []; -The **init_thresh_name** and **init_thresh_val** fields stratify by applying thresholds to numeric data columns only when lead = 0. If lead = 0, but the value does not meet the threshold, discard the entire track. The length of the **init_thresh_val** should match that of the **init_thresh_name**. Using the **-init_thresh name val** option within the job command lines may further refine these selections. +The **init_thresh_name** and **init_thresh_val** fields stratify by applying thresholds to numeric data columns only when lead = 0. If lead = 0, but the value does not meet the threshold, discard the entire track. The length of the **init_thresh_val** should match that of the **init_thresh_name**. Using the **-init_thresh name thresh** option within the job command lines may further refine these selections. _________________________ @@ -272,7 +272,7 @@ _________________________ init_str_name = []; init_str_val = []; -The **init_str_name** and **init_str_val** fields stratify by performing string matching on non-numeric data columns only when lead = 0. If lead = 0, but the string **does not** match, discard the entire track. The length of the **init_str_val** should match that of the **init_str_name**. Using the **-init_str name val** option within the job command lines may further refine these selections. +The **init_str_name** and **init_str_val** fields stratify by performing string matching on non-numeric data columns only when lead = 0. If lead = 0, but the string **does not** match, discard the entire track. The length of the **init_str_val** should match that of the **init_str_name**. Using the **-init_str name value** option within the job command lines may further refine these selections. _________________________ @@ -281,7 +281,25 @@ _________________________ init_str_exc_name = []; init_str_exc_val = []; -The **init_str_exc_name** and **init_str_exc_val** fields stratify by performing string matching on non-numeric data columns only when lead = 0. If lead = 0, and the string **does** match, discard the entire track. The length of the **init_str_exc_val** should match that of the **init_str_exc_name**. Using the **-init_str_exc name val** option within the job command lines may further refine these selections. +The **init_str_exc_name** and **init_str_exc_val** fields stratify by performing string matching on non-numeric data columns only when lead = 0. If lead = 0, and the string **does** match, discard the entire track. The length of the **init_str_exc_val** should match that of the **init_str_exc_name**. Using the **-init_str_exc name value** option within the job command lines may further refine these selections. + +_________________________ + +.. code-block:: none + + diag_thresh_name = []; + diag_thresh_val = []; + +The **diag_thresh_name** and **diag_thresh_val** fields stratify individual track points by applying thresholds to numeric data columns from the TCDIAG lines. Specify a comma-separated list of diagnostics names and thresholds to be applied. The length of **diag_thresh_val** should match that of **diag_thresh_name**. If the storm diagnostic does not meet the threshold, discard both the TCMPR and TCDIAG lines for that track point. Using the **-diag_thresh name thresh** option within the job command lines may further refine these selections. + +_________________________ + +.. code-block:: none + + init_diag_thresh_name = []; + init_diag_thresh_val = []; + +The **init_diag_thresh_name** and **init_diag_thresh_val** fields stratify entire tracks by applying thresholds to numeric data columns from the TCDIAG lines, but only when lead = 0. If lead = 0, but the storm diagnostic does not meet the threshold, discard the entire track. The length of the **init_diag_thresh_val** should match that of the **init_diag_thresh_name**. Using the **-init_diag_thresh name thresh** option within the job command lines may further refine these selections. _________________________ diff --git a/src/basic/vx_config/config_constants.h b/src/basic/vx_config/config_constants.h index 621a634325..5fad36d25e 100644 --- a/src/basic/vx_config/config_constants.h +++ b/src/basic/vx_config/config_constants.h @@ -1083,6 +1083,10 @@ static const char conf_key_init_str_name[] = "init_str_name"; static const char conf_key_init_str_val[] = "init_str_val"; static const char conf_key_init_str_exc_name[] = "init_str_exc_name"; static const char conf_key_init_str_exc_val[] = "init_str_exc_val"; +static const char conf_key_diag_thresh_name[] = "diag_thresh_name"; +static const char conf_key_diag_thresh_val[] = "diag_thresh_val"; +static const char conf_key_init_diag_thresh_name[] = "init_diag_thresh_name"; +static const char conf_key_init_diag_thresh_val[] = "init_diag_thresh_val"; static const char conf_key_water_only[] = "water_only"; static const char conf_key_rirw_track[] = "rirw.track"; static const char conf_key_rirw_time_adeck[] = "rirw.adeck.time"; diff --git a/src/tools/tc_utils/tc_pairs/tc_pairs.cc b/src/tools/tc_utils/tc_pairs/tc_pairs.cc index a2e63b05cf..837af0af80 100644 --- a/src/tools/tc_utils/tc_pairs/tc_pairs.cc +++ b/src/tools/tc_utils/tc_pairs/tc_pairs.cc @@ -2238,6 +2238,16 @@ void usage() { << "TCPairsConfig file containing the desired configuration " << "settings (required).\n" + << "\t\t\"-tcdiag source\" is used one or more times to " + << "specify a file or top-level directory containing tropical " + << "cyclone diagnostics \"" << atcf_suffix + << "\" data to process (optional).\n" + + << "\t\t\"-lsdiag source\" is used one or more times to " + << "specify a file or top-level directory containing large " + << "scale diagnostics \"" << atcf_suffix + << "\" data to process (optional).\n" + << "\t\t\"-out base\" overrides the default output file base " << "(" << out_base << ") (optional).\n" diff --git a/src/tools/tc_utils/tc_stat/tc_stat.cc b/src/tools/tc_utils/tc_stat/tc_stat.cc index 52e0c5adaf..10c97286e5 100644 --- a/src/tools/tc_utils/tc_stat/tc_stat.cc +++ b/src/tools/tc_utils/tc_stat/tc_stat.cc @@ -21,6 +21,7 @@ // the gen_vx_mask tool. // 004 07/06/22 Howard Soh METplus-Internal #19 Rename main to met_main // 005 09/28/22 Prestopnik MET #2227 Remove namespace std from header files +// 006 10/06/22 Halley Gotway MET #392 Incorporate diagnostics // //////////////////////////////////////////////////////////////////////// diff --git a/src/tools/tc_utils/tc_stat/tc_stat_conf_info.cc b/src/tools/tc_utils/tc_stat/tc_stat_conf_info.cc index d2edb908df..d211057ef6 100644 --- a/src/tools/tc_utils/tc_stat/tc_stat_conf_info.cc +++ b/src/tools/tc_utils/tc_stat/tc_stat_conf_info.cc @@ -212,6 +212,16 @@ void TCStatConfInfo::process_config() { conf_key_init_str_exc_name, conf_key_init_str_exc_val, Filter.InitStrExcMap); + // Conf: TCStatJob::DiagThreshName, TCStatJob::DiagThreshVal + parse_conf_thresh_map(Conf, + conf_key_diag_thresh_name, conf_key_diag_thresh_val, + Filter.DiagThreshMap); + + // Conf: TCStatJob::InitDiagThreshName, TCStatJob::InitDiagThreshVal + parse_conf_thresh_map(Conf, + conf_key_init_diag_thresh_name, conf_key_init_diag_thresh_val, + Filter.InitDiagThreshMap); + // Conf: TCStatJob::WaterOnly Filter.WaterOnly = Conf.lookup_bool(conf_key_water_only); From 4a1ce99003d856e1a73f39f732592453744774a1 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Wed, 19 Oct 2022 15:22:13 -0600 Subject: [PATCH 28/58] Per #392, migrate new TC-Pairs and TC-Stat config options to other config files. --- internal/test_unit/config/TCPairsConfig_ALAL2010 | 13 +++++++++---- internal/test_unit/config/TCPairsConfig_BASIN_MAP | 5 +++++ internal/test_unit/config/TCPairsConfig_CONSENSUS | 5 +++++ internal/test_unit/config/TCPairsConfig_INTERP12 | 6 +++++- internal/test_unit/config/TCPairsConfig_PROBRIRW | 5 +++++ internal/test_unit/config/TCStatConfig_ALAL2010 | 14 ++++++++++++++ internal/test_unit/config/TCStatConfig_PROBRIRW | 14 ++++++++++++++ 7 files changed, 57 insertions(+), 5 deletions(-) diff --git a/internal/test_unit/config/TCPairsConfig_ALAL2010 b/internal/test_unit/config/TCPairsConfig_ALAL2010 index 0dfc50ee28..cad7e58eae 100644 --- a/internal/test_unit/config/TCPairsConfig_ALAL2010 +++ b/internal/test_unit/config/TCPairsConfig_ALAL2010 @@ -14,7 +14,7 @@ // // Models // -model = []; +model = [ "GFSO" ]; // // Description @@ -24,17 +24,17 @@ desc = "NA"; // // Storm identifiers // -storm_id = [ "ALAL2010" ]; +storm_id = []; // // Basins // -basin = []; +basin = [ "AL" ]; // // Cyclone numbers // -cyclone = []; +cyclone = [ "09" ]; // // Storm names @@ -128,6 +128,11 @@ watch_warn = { time_offset = -14400; } +// +// Diagnostics to be extracted +// +diag_name = []; + // // Modify basin names to make them consistent across ATCF input files. // diff --git a/internal/test_unit/config/TCPairsConfig_BASIN_MAP b/internal/test_unit/config/TCPairsConfig_BASIN_MAP index fb0075ee17..150e4ed4fa 100644 --- a/internal/test_unit/config/TCPairsConfig_BASIN_MAP +++ b/internal/test_unit/config/TCPairsConfig_BASIN_MAP @@ -128,6 +128,11 @@ watch_warn = { time_offset = -14400; } +// +// Diagnostics to be extracted +// +diag_name = []; + // // Modify basin names to make them consistent across ATCF input files. // diff --git a/internal/test_unit/config/TCPairsConfig_CONSENSUS b/internal/test_unit/config/TCPairsConfig_CONSENSUS index 16b2106a2a..b157c71264 100644 --- a/internal/test_unit/config/TCPairsConfig_CONSENSUS +++ b/internal/test_unit/config/TCPairsConfig_CONSENSUS @@ -180,6 +180,11 @@ watch_warn = { time_offset = -14400; } +// +// Diagnostics to be extracted +// +diag_name = []; + // // Modify basin names to make them consistent across ATCF input files. // diff --git a/internal/test_unit/config/TCPairsConfig_INTERP12 b/internal/test_unit/config/TCPairsConfig_INTERP12 index 92e00a10b5..2985f596ee 100644 --- a/internal/test_unit/config/TCPairsConfig_INTERP12 +++ b/internal/test_unit/config/TCPairsConfig_INTERP12 @@ -72,7 +72,6 @@ init_hour = []; // lead_req = []; - // // lat/lon polylines defining masking regions // @@ -129,6 +128,11 @@ watch_warn = { time_offset = -14400; } +// +// Diagnostics to be extracted +// +diag_name = []; + // // Modify basin names to make them consistent across ATCF input files. // diff --git a/internal/test_unit/config/TCPairsConfig_PROBRIRW b/internal/test_unit/config/TCPairsConfig_PROBRIRW index 1632603972..c958ec5590 100644 --- a/internal/test_unit/config/TCPairsConfig_PROBRIRW +++ b/internal/test_unit/config/TCPairsConfig_PROBRIRW @@ -128,6 +128,11 @@ watch_warn = { time_offset = -14400; } +// +// Diagnostics to be extracted +// +diag_name = []; + // // Modify basin names to make them consistent across ATCF input files. // diff --git a/internal/test_unit/config/TCStatConfig_ALAL2010 b/internal/test_unit/config/TCStatConfig_ALAL2010 index 85c3141ffe..27d64c0478 100644 --- a/internal/test_unit/config/TCStatConfig_ALAL2010 +++ b/internal/test_unit/config/TCStatConfig_ALAL2010 @@ -136,6 +136,20 @@ init_str_val = []; init_str_exc_name = []; init_str_exc_val = []; +// +// Stratify track points by applying thresholds to numeric +// storm diagnostic values in the TCDIAG lines. +// +diag_thresh_name = []; +diag_thresh_val = []; + +// +// Stratify entire tracks by applying thresholds to numeric +// storm diagnostic values in the TCDIAG lines for lead time 0. +// +init_diag_thresh_name = []; +init_diag_thresh_val = []; + // // Stratify by the ADECK and BDECK distances to land. // diff --git a/internal/test_unit/config/TCStatConfig_PROBRIRW b/internal/test_unit/config/TCStatConfig_PROBRIRW index a48bcc6dba..ba9a79b1d7 100644 --- a/internal/test_unit/config/TCStatConfig_PROBRIRW +++ b/internal/test_unit/config/TCStatConfig_PROBRIRW @@ -136,6 +136,20 @@ init_str_val = []; init_str_exc_name = []; init_str_exc_val = []; +// +// Stratify track points by applying thresholds to numeric +// storm diagnostic values in the TCDIAG lines. +// +diag_thresh_name = []; +diag_thresh_val = []; + +// +// Stratify entire tracks by applying thresholds to numeric +// storm diagnostic values in the TCDIAG lines for lead time 0. +// +init_diag_thresh_name = []; +init_diag_thresh_val = []; + // // Stratify by the ADECK and BDECK distances to land. // From 3f30594594ca71ba7411244a11f3e9ce0223b857 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Thu, 20 Oct 2022 13:41:42 -0600 Subject: [PATCH 29/58] Per #392, fix parsing of the lsdiag input files. --- src/libcode/vx_tc_util/diag_file.cc | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/src/libcode/vx_tc_util/diag_file.cc b/src/libcode/vx_tc_util/diag_file.cc index 9daa79df86..1e4a859de5 100644 --- a/src/libcode/vx_tc_util/diag_file.cc +++ b/src/libcode/vx_tc_util/diag_file.cc @@ -265,10 +265,6 @@ void DiagFile::read_tcdiag(const std::string &path, const std::string &model_nam exit(1); } - mlog << Debug(4) << "Parsing " << StormId << " " - << Technique << " " << unix_to_yyyymmddhh(InitTime) - << " TC diagnostics file: " << path << "\n"; - // Store the diagnostics data while(dl.read_line(this)) { @@ -306,6 +302,10 @@ void DiagFile::read_tcdiag(const std::string &path, const std::string &model_nam } } // end while + mlog << Debug(4) << "Parsed " << DiagMap.size() << " diagnostic values for " + << StormId << " " << Technique << " " << unix_to_yyyymmddhh(InitTime) + << " TC diagnostics file: " << path << "\n"; + // Close the input file close(); @@ -358,6 +358,9 @@ void DiagFile::read_lsdiag(const std::string &path, const std::string &model_nam // Fixed width column 24 has the data name cs = dl[23]; + // Strip any whitespace from the fixed-width column + cs.ws_strip(); + if(cs == "TIME") { for(i=2; i<23; i++) { LeadTime.add(atoi(dl[i])*sec_per_hour); @@ -381,18 +384,14 @@ void DiagFile::read_lsdiag(const std::string &path, const std::string &model_nam } } // end while - mlog << Debug(4) << "Parsing " << StormId << " " - << Technique << " " << unix_to_yyyymmddhh(InitTime) - << " LS diagnostics file: " << path << "\n"; - // Store the diagnostics data while(read_fwf_line(dl, lsdiag_wdth, n_lsdiag_wdth)) { // Skip empty lines if(dl.n_items() == 0) continue; - // Check the 25th column - cs = dl[24]; + // Check the 24th column + cs = dl[23]; // Quit reading at the LAST line if(cs == "LAST") break; @@ -418,6 +417,10 @@ void DiagFile::read_lsdiag(const std::string &path, const std::string &model_nam } } // end while + mlog << Debug(4) << "Parsed " << DiagMap.size() << " diagnostic values for " + << StormId << " " << Technique << " " << unix_to_yyyymmddhh(InitTime) + << " LS diagnostics file: " << path << "\n"; + // Close the input file close(); From 4a28484609084acb539450a87aa689d3e88966e6 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Thu, 20 Oct 2022 13:57:43 -0600 Subject: [PATCH 30/58] Per #392, switch from using DiagMap to DiagName and DiagVal to maintain the order by which the values are read and stored. --- src/libcode/vx_tc_util/diag_file.cc | 45 +++++++++++++++------------- src/libcode/vx_tc_util/diag_file.h | 24 ++++++++------- src/libcode/vx_tc_util/track_info.cc | 4 +-- 3 files changed, 39 insertions(+), 34 deletions(-) diff --git a/src/libcode/vx_tc_util/diag_file.cc b/src/libcode/vx_tc_util/diag_file.cc index 1e4a859de5..0a796ab84e 100644 --- a/src/libcode/vx_tc_util/diag_file.cc +++ b/src/libcode/vx_tc_util/diag_file.cc @@ -61,7 +61,7 @@ int DiagFile::lead(int i) const { // Check range if(i < 0 || i >= LeadTime.n()) { - mlog << Error << "\nTrackInfo::lead(int) -> " + mlog << Error << "\nDiagFile::lead(int) -> " << "range check error for index value " << i << "\n\n"; exit(1); } @@ -75,7 +75,7 @@ unixtime DiagFile::valid(int i) const { // Check range if(i < 0 || i >= LeadTime.n()) { - mlog << Error << "\nTrackInfo::valid(int) -> " + mlog << Error << "\nDiagFile::valid(int) -> " << "range check error for index value " << i << "\n\n"; exit(1); } @@ -90,7 +90,7 @@ double DiagFile::lat(int i) const { // Check range if(i < 0 || i >= Lat.n()) { - mlog << Error << "\nTrackInfo::lat(int) -> " + mlog << Error << "\nDiagFile::lat(int) -> " << "range check error for index value " << i << "\n\n"; exit(1); } @@ -104,7 +104,7 @@ double DiagFile::lon(int i) const { // Check range if(i < 0 || i >= Lon.n()) { - mlog << Error << "\nTrackInfo::lon(int) -> " + mlog << Error << "\nDiagFile::lon(int) -> " << "range check error for index value " << i << "\n\n"; exit(1); } @@ -115,22 +115,22 @@ double DiagFile::lon(int i) const { //////////////////////////////////////////////////////////////////////// bool DiagFile::has_diag(const string &str) const { - return(DiagMap.count(str) > 0); + return(DiagName.has(str)); } //////////////////////////////////////////////////////////////////////// -const NumArray & DiagFile::get_diag(const string &str) const { - return(DiagMap.at(str)); -} +const NumArray & DiagFile::diag_val(const string &str) const { + int i; -//////////////////////////////////////////////////////////////////////// + // Find the index of the name + if(!DiagName.has(str, i)) { + mlog << Error << "\nDiagFile::diag_val() -> " + << "requested diagnostic name \"" << str << "\" not found!\n\n"; + exit(1); + } -void DiagFile::get_diag_name(StringArray &diag_name) const { - diag_name.clear(); - for(map::const_iterator it = DiagMap.begin(); - it != DiagMap.end(); it++) diag_name.add(it->first); - return; + return(DiagVal[i]); } //////////////////////////////////////////////////////////////////////// @@ -167,7 +167,8 @@ void DiagFile::init_from_scratch() { LeadTime.clear(); Lat.clear(); Lon.clear(); - DiagMap.clear(); + DiagName.clear(); + DiagVal.clear(); close(); @@ -296,13 +297,14 @@ void DiagFile::read_tcdiag(const std::string &path, const std::string &model_nam << NTime << ")!\n\n"; exit(1); } - // Store the data in the map + // Store the name and values else { - DiagMap[cs] = data; + DiagName.add(cs); + DiagVal.push_back(data); } } // end while - mlog << Debug(4) << "Parsed " << DiagMap.size() << " diagnostic values for " + mlog << Debug(4) << "Parsed " << DiagName.n() << " diagnostic values for " << StormId << " " << Technique << " " << unix_to_yyyymmddhh(InitTime) << " TC diagnostics file: " << path << "\n"; @@ -411,13 +413,14 @@ void DiagFile::read_lsdiag(const std::string &path, const std::string &model_nam << NTime << ")!\n\n"; exit(1); } - // Store the data in the map + // Store the name and values else { - DiagMap[cs] = data; + DiagName.add(cs); + DiagVal.push_back(data); } } // end while - mlog << Debug(4) << "Parsed " << DiagMap.size() << " diagnostic values for " + mlog << Debug(4) << "Parsed " << DiagName.n() << " diagnostic values for " << StormId << " " << Technique << " " << unix_to_yyyymmddhh(InitTime) << " LS diagnostics file: " << path << "\n"; diff --git a/src/libcode/vx_tc_util/diag_file.h b/src/libcode/vx_tc_util/diag_file.h index d6245d4322..593a40bd79 100644 --- a/src/libcode/vx_tc_util/diag_file.h +++ b/src/libcode/vx_tc_util/diag_file.h @@ -77,8 +77,9 @@ class DiagFile : public LineDataFile { NumArray Lat; NumArray Lon; - // Diagnostic values - std::map DiagMap; + // Diagnostic names and values + StringArray DiagName; + std::vector DiagVal; public: @@ -109,9 +110,9 @@ class DiagFile : public LineDataFile { double lon(int) const; int n_diag() const; + const StringArray & diag_name() const; bool has_diag(const std::string &) const; - const NumArray & get_diag(const std::string &) const; - void get_diag_name(StringArray &) const; + const NumArray & diag_val(const std::string &) const; // // do stuff @@ -124,13 +125,14 @@ class DiagFile : public LineDataFile { //////////////////////////////////////////////////////////////////////// -inline const ConcatString & DiagFile::storm_id() const { return(StormId); } -inline const ConcatString & DiagFile::basin() const { return(Basin); } -inline const ConcatString & DiagFile::cyclone() const { return(Cyclone); } -inline const ConcatString & DiagFile::technique() const { return(Technique); } -inline unixtime DiagFile::init() const { return(InitTime); } -inline int DiagFile::n_time() const { return(NTime); } -inline int DiagFile::n_diag() const { return(DiagMap.size()); } +inline const ConcatString & DiagFile::storm_id() const { return(StormId); } +inline const ConcatString & DiagFile::basin() const { return(Basin); } +inline const ConcatString & DiagFile::cyclone() const { return(Cyclone); } +inline const ConcatString & DiagFile::technique() const { return(Technique); } +inline unixtime DiagFile::init() const { return(InitTime); } +inline int DiagFile::n_time() const { return(NTime); } +inline int DiagFile::n_diag() const { return(DiagName.n()); } +inline const StringArray & DiagFile::diag_name() const { return(DiagName); } //////////////////////////////////////////////////////////////////////// diff --git a/src/libcode/vx_tc_util/track_info.cc b/src/libcode/vx_tc_util/track_info.cc index 65b80a5286..ef750127f4 100644 --- a/src/libcode/vx_tc_util/track_info.cc +++ b/src/libcode/vx_tc_util/track_info.cc @@ -535,14 +535,14 @@ bool TrackInfo::add_diag_data(DiagFile &diag_file, const StringArray &diag_name) // If empty, store all diagnostics if(diag_name.n() > 0) DiagName = diag_name; - else diag_file.get_diag_name(DiagName); + else DiagName = diag_file.diag_name(); int i_name, i_time, i_pnt; // Retrieve data for each diagnostic for(i_name=0; i_name Date: Thu, 20 Oct 2022 13:58:14 -0600 Subject: [PATCH 31/58] Per #392, add tc_pairs.xml tests for diagnostics. Still need to refine them and add tc_stat tests. --- .../config/TCPairsConfig_DIAGNOSTICS | 151 ++++++++++++++++++ internal/test_unit/xml/unit_tc_pairs.xml | 42 +++++ 2 files changed, 193 insertions(+) create mode 100644 internal/test_unit/config/TCPairsConfig_DIAGNOSTICS diff --git a/internal/test_unit/config/TCPairsConfig_DIAGNOSTICS b/internal/test_unit/config/TCPairsConfig_DIAGNOSTICS new file mode 100644 index 0000000000..e18685069b --- /dev/null +++ b/internal/test_unit/config/TCPairsConfig_DIAGNOSTICS @@ -0,0 +1,151 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// TC-Pairs configuration file. +// +// For additional information, please see the MET User's Guide. +// +//////////////////////////////////////////////////////////////////////////////// + +// +// ATCF file format reference: +// http://www.nrlmry.navy.mil/atcf_web/docs/database/new/abrdeck.html +// + +// +// Models +// +model = []; + +// +// Description +// +desc = "NA"; + +// +// Storm identifiers +// +storm_id = [ ${STORM_ID} ]; + +// +// Basins +// +basin = []; + +// +// Cyclone numbers +// +cyclone = []; + +// +// Storm names +// +storm_name = []; + +// +// Model initialization time windows to include or exclude +// +init_beg = ""; +init_end = ""; +init_inc = [ ${INIT_INC} ]; +init_exc = []; + +// +// Valid model time windows to include or exclude +// +valid_beg = ""; +valid_end = ""; +valid_inc = []; +valid_exc = []; + +// +// Valid times for which output should be written +// +write_valid = []; + +// +// Model initialization hours +// +init_hour = []; + +// +// Required lead time in hours +// +lead_req = []; + +// +// lat/lon polylines defining masking regions +// +init_mask = ""; +valid_mask = ""; + +// +// Specify if the code should check for duplicate ATCF lines +// +check_dup = TRUE; + +// +// Specify special processing to be performed for interpolated models. +// Set to NONE, FILL, or REPLACE. +// +interp12 = NONE; + +// +// Specify how consensus forecasts should be defined +// +consensus = []; + +// +// Forecast lag times +// +lag_time = []; + +// +// CLIPER/SHIFOR baseline forecasts to be derived from the BEST +// and operational (CARQ) tracks. +// +best_baseline = []; +oper_baseline = []; + +// +// Specify if only those track points common to both the ADECK and BDECK +// tracks be written out. +// +match_points = FALSE; + +// +// Specify the NetCDF output of the gen_dland tool containing a gridded +// representation of the minimum distance to land. +// +dland_file = "${MET_TEST_OUTPUT}/tc_dland/tc_dland_half_deg.nc"; + +// +// Specify watch/warning information: +// - Input watch/warning filename +// - Watch/warning time offset in seconds +// +watch_warn = { + file_name = "MET_BASE/tc_data/wwpts_us.txt"; + time_offset = -14400; +} + +// +// Diagnostics to be extracted +// +diag_name = [ ${DIAG_NAME} ]; + +// +// Modify basin names to make them consistent across ATCF input files. +// +basin_map = [ + { key = "SI"; val = "SH"; }, + { key = "SP"; val = "SH"; }, + { key = "AU"; val = "SH"; }, + { key = "AB"; val = "IO"; }, + { key = "BB"; val = "IO"; } +]; + +// +// Indicate a version number for the contents of this configuration file. +// The value should generally not be modified. +// +version = "V11.0.0"; diff --git a/internal/test_unit/xml/unit_tc_pairs.xml b/internal/test_unit/xml/unit_tc_pairs.xml index 44f986fd10..bcf8cacbf3 100644 --- a/internal/test_unit/xml/unit_tc_pairs.xml +++ b/internal/test_unit/xml/unit_tc_pairs.xml @@ -182,4 +182,46 @@ + + &MET_BIN;/tc_pairs + + STORM_ID "AL072022" + INIT_INC "20220916_18" + DIAG_NAME + + \ + -adeck &DATA_DIR;/adeck/aal072022_OFCL_AVNO.dat \ + -bdeck &DATA_DIR;/bdeck/bal072022.dat \ + -lsdiag &DATA_DIR;/lsdiag/22091618AL0722_lsdiag.dat \ + -config &CONFIG_DIR;/TCPairsConfig_DIAGNOSTICS \ + -out &OUTPUT_DIR;/tc_pairs/al072022_LSDIAG \ + -log &OUTPUT_DIR;/tc_pairs/tc_pairs_LSDIAG.log \ + -v 4 + + + &OUTPUT_DIR;/tc_pairs/al072022_LSDIAG.tcst + + + + + &MET_BIN;/tc_pairs + + STORM_ID "AL092021" + INIT_INC "20210827_12" + DIAG_NAME + + \ + -adeck &DATA_DIR;/adeck/aal092021_OFCL_AVNO.dat \ + -bdeck &DATA_DIR;/bdeck/bal092021.dat \ + -tcdiag &DATA_DIR;/tcdiag/sal092021_avno_doper_2021082712_diag.dat \ + -config &CONFIG_DIR;/TCPairsConfig_DIAGNOSTICS \ + -out &OUTPUT_DIR;/tc_pairs/al092021_TCDIAG \ + -log &OUTPUT_DIR;/tc_pairs/tc_pairs_TCDIAG.log \ + -v 4 + + + &OUTPUT_DIR;/tc_pairs/al092021_TCDIAG.tcst + + + From 1529ab0da9ef9d8fb07ca633c7e807a1e9b9c15d Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Thu, 20 Oct 2022 14:07:26 -0600 Subject: [PATCH 32/58] Per #392, tweak log message. --- src/libcode/vx_tc_util/diag_file.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libcode/vx_tc_util/diag_file.cc b/src/libcode/vx_tc_util/diag_file.cc index 0a796ab84e..bb7eaa7416 100644 --- a/src/libcode/vx_tc_util/diag_file.cc +++ b/src/libcode/vx_tc_util/diag_file.cc @@ -304,7 +304,7 @@ void DiagFile::read_tcdiag(const std::string &path, const std::string &model_nam } } // end while - mlog << Debug(4) << "Parsed " << DiagName.n() << " diagnostic values for " + mlog << Debug(4) << "Parsed " << DiagName.n() << " diagnostic values from " << StormId << " " << Technique << " " << unix_to_yyyymmddhh(InitTime) << " TC diagnostics file: " << path << "\n"; @@ -420,7 +420,7 @@ void DiagFile::read_lsdiag(const std::string &path, const std::string &model_nam } } // end while - mlog << Debug(4) << "Parsed " << DiagName.n() << " diagnostic values for " + mlog << Debug(4) << "Parsed " << DiagName.n() << " diagnostic values from " << StormId << " " << Technique << " " << unix_to_yyyymmddhh(InitTime) << " LS diagnostics file: " << path << "\n"; From b856723cabeb64dd0083a9a74d5ff04585edd2c8 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Thu, 20 Oct 2022 14:42:03 -0600 Subject: [PATCH 33/58] Per #392, the lsdiag corresponds to ADECK SHIP tracks, as noted in https://ftp.nhc.noaa.gov/atcf/README --- src/libcode/vx_tc_util/diag_file.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libcode/vx_tc_util/diag_file.cc b/src/libcode/vx_tc_util/diag_file.cc index bb7eaa7416..d6b81ff944 100644 --- a/src/libcode/vx_tc_util/diag_file.cc +++ b/src/libcode/vx_tc_util/diag_file.cc @@ -27,7 +27,7 @@ using namespace std; //////////////////////////////////////////////////////////////////////// -static const char default_lsdiag_technique[] = "AVNO"; +static const char default_lsdiag_technique[] = "SHIP"; static const int lsdiag_wdth[] = { 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, From 981baeca58eba99dc07c54e1f75d29f20fb6e9a8 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Fri, 21 Oct 2022 14:33:05 -0600 Subject: [PATCH 34/58] Per #392, add tcdiag_convert_map and lsdiag_convert_map entries to the TC-Pairs configuration file. Still need to figure out how to handle lsdiag values that require special handling. --- data/config/TCPairsConfig_default | 53 +++++++++++++ docs/Users_Guide/tc-pairs.rst | 11 +++ .../config/TCPairsConfig_DIAGNOSTICS | 8 ++ internal/test_unit/xml/unit_tc_pairs.xml | 4 +- src/basic/vx_config/config_constants.h | 2 + src/basic/vx_config/config_util.cc | 77 ++++++++++++++++--- src/basic/vx_config/config_util.h | 2 + src/libcode/vx_tc_util/diag_file.cc | 58 +++++++++++--- src/libcode/vx_tc_util/diag_file.h | 6 +- src/tools/tc_utils/tc_pairs/tc_pairs.cc | 6 +- .../tc_utils/tc_pairs/tc_pairs_conf_info.cc | 8 ++ .../tc_utils/tc_pairs/tc_pairs_conf_info.h | 4 + 12 files changed, 213 insertions(+), 26 deletions(-) diff --git a/data/config/TCPairsConfig_default b/data/config/TCPairsConfig_default index 263dcb0445..c76f036329 100644 --- a/data/config/TCPairsConfig_default +++ b/data/config/TCPairsConfig_default @@ -142,6 +142,59 @@ watch_warn = { // diag_name = []; +// +// Unit conversions to be applied to diagnostic values +// +tcdiag_convert_map = []; +lsdiag_convert_map = [ + { key = [ "CSST", "RSST", "DSST", "DSTA", "XDST", "XNST", "NSST", "NSTA", + "NTMX", "NTFR", "U200", "U20C", "V20C", "E000", "EPOS", "ENEG", "EPSS", "ENSS", + "T000", "TLAT", "TLON", "TWAC", "TWXC", "G150", "G200", "G250", "V000", "V850", + "V500", "V300", "PENC", "SHDC", "SHGC", "T150", "T200", "T250", "SHRD", "SHRS", + "SHRG", "PENV", "HE07", "HE05"]; convert(x) = x / 10; }, + { key = [ "VVAV", "VMFX", "VVAC" ]; convert(x) = x / 100; } +]; + +// +// LSDiagnostics +// https://rammb.cira.colostate.edu/research/tropical_cyclones/ships/data/ships_predictor_file_2022.pdf +// Likley typo: NDMX should acutally be NDTX in there +// +// All diagnostics from sample file: +// "TIME", "DELV", "LAT", "LON", "CSST", "DTL", "RSST", "DSST", "DSTA", "CD20", +// "CD26", "COHC", "XDST", "XNST", "XOHC", "NSST", "NSTA", "NTMX", "NDTX", "NDML", +// "ND30", "ND28", "ND26", "ND24", "ND22", "ND20", "ND18", "ND16", "NDFR", "NTFR", +// "NOHC", "NO20", "IR00", "IRM1", "IRM3", "PC00", "PCM1", "PCM3", "U200", "U20C", +// "V20C", "E000", "EPOS", "ENEG", "EPSS", "ENSS", "RHLO", "RHMD", "RHHI", "PSLV", +// "Z850", "D200", "REFC", "PEFC", "T000", "R000", "Z000", "TLAT", "TLON", "TWAC", +// "TWXC", "G150", "G200", "G250", "V000", "V850", "V500", "V300", "TGRD", "TADV", +// "PENC", "SHDC", "SDDC", "SHGC", "DIVC", "T150", "T200", "T250", "SHRD", "SHTD", +// "SHRS", "SHTS", "SHRG", "PENV", "VMPI", "VVAV", "VMFX", "VVAC", "HE07", "HE05", +// "O500", "O700", "CFLX", "MTPW", "PW01", "PW02", "PW03", "PW04", "PW05", "PW06", +// "PW07", "PW08", "PW09", "PW10", "PW11", "PW12", "PW13", "PW14", "PW15", "PW16", +// "PW17", "PW18", "PW19", "PW20", "PW21", "IRXX" +// +// No conversion required: +// "TIME", "DELV", "DTL", "CD20", "CD26", "COHC", "XOHC", "NDTX", "NDML", "ND30", +// "ND28", "ND26", "ND24", "ND22", "ND20", "ND18", "ND16", "NDFR", "NOHC", "NO20", +// "RHLO", "RHMD", "RHHI", "Z850", "D200", "REFC", "PEFC", "R000", "Z000", "TGRD", +// "TADV", "SDDC", "DIVC", "SHTD", "SHTS", "VMPI", "O500", "O700", "CFLX" +// +// Divide by 10: +// "LAT", "LON", "CSST", "RSST", "DSST", "DSTA", "XDST", "XNST", "NSST", "NSTA", +// "NTMX", "NTFR", "U200", "U20C", "V20C", "E000", "EPOS", "ENEG", "EPSS", "ENSS", +// "T000", "TLAT", "TLON", "TWAC", "TWXC", "G150", "G200", "G250", "V000", "V850", +// "V500", "V300", "PENC", "SHDC", "SHGC", "T150", "T200", "T250", "SHRD", "SHRS", +// "SHRG", "PENV", "HE07", "HE05" +// +// Divide by 100: +// "VVAV", "VMFX", "VVAC" +// +// Special logic required: +// "IR00", "IRM1", "IRM3", "PC00", "PCM1", "PCM3", "PSLV", "MTPW", "PW01", "PW02", +// "PW03", "PW04", "PW05", "PW06", "PW07", "PW08", "PW09", "PW10", "PW11", "PW12", +// "PW13", "PW14", "PW15", "PW16", "PW17", "PW18", "PW19", "PW20", "PW21", "IRXX" + // // Modify basin names to make them consistent across ATCF input files. // diff --git a/docs/Users_Guide/tc-pairs.rst b/docs/Users_Guide/tc-pairs.rst index f18ddab66f..89083769a5 100644 --- a/docs/Users_Guide/tc-pairs.rst +++ b/docs/Users_Guide/tc-pairs.rst @@ -263,6 +263,17 @@ A TCMPR line is written to the output for each track point. If diagnostics data ____________________ +.. code-block:: none + + tcdiag_convert_map = []; + lsdiag_convert_map = []; + +The **tcdiag_convert_map** and **lsdiag_convert_map** entries define conversion functions to be applied to diagnostics data read with the **-tcdiag** and **-lsdiag** command line options, respectively. Each array element is a dictionary consisting of a **key** and **convert(x)** entry. The **key** is an array of strings listing diagnostic names. **convert(x)** is a function which defines how the diagnostic data should be converted after being read. The defined function is applied to any diagnostic value who name appears in the **key**. + +Note that unit conversions are generally handled automatically for **-tcdiag** inputs based on the units defined in the input file. The conversion function **convert(x) = x / 10;** is automatically applied to any diagnostic with units of 10C, 10M/S, or 10KT. However, conversion functions defined in **tcdiag_convert_map** take precedence over that default behavior. + +____________________ + .. code-block:: none basin_map = [ diff --git a/internal/test_unit/config/TCPairsConfig_DIAGNOSTICS b/internal/test_unit/config/TCPairsConfig_DIAGNOSTICS index e18685069b..2e382dc7cd 100644 --- a/internal/test_unit/config/TCPairsConfig_DIAGNOSTICS +++ b/internal/test_unit/config/TCPairsConfig_DIAGNOSTICS @@ -133,6 +133,14 @@ watch_warn = { // diag_name = [ ${DIAG_NAME} ]; +// +// Unit conversions to be applied to diagnostic values +// + +// Commented out to use settings from the default config file +// tcdiag_convert_map = []; +// lsdiag_convert_map = []; + // // Modify basin names to make them consistent across ATCF input files. // diff --git a/internal/test_unit/xml/unit_tc_pairs.xml b/internal/test_unit/xml/unit_tc_pairs.xml index bcf8cacbf3..c7eeadd4f7 100644 --- a/internal/test_unit/xml/unit_tc_pairs.xml +++ b/internal/test_unit/xml/unit_tc_pairs.xml @@ -190,7 +190,7 @@ DIAG_NAME \ - -adeck &DATA_DIR;/adeck/aal072022_OFCL_AVNO.dat \ + -adeck &DATA_DIR;/adeck/aal072022_OFCL_SHIP_AVNO.dat \ -bdeck &DATA_DIR;/bdeck/bal072022.dat \ -lsdiag &DATA_DIR;/lsdiag/22091618AL0722_lsdiag.dat \ -config &CONFIG_DIR;/TCPairsConfig_DIAGNOSTICS \ @@ -211,7 +211,7 @@ DIAG_NAME \ - -adeck &DATA_DIR;/adeck/aal092021_OFCL_AVNO.dat \ + -adeck &DATA_DIR;/adeck/aal092021_OFCL_SHIP_AVNO.dat \ -bdeck &DATA_DIR;/bdeck/bal092021.dat \ -tcdiag &DATA_DIR;/tcdiag/sal092021_avno_doper_2021082712_diag.dat \ -config &CONFIG_DIR;/TCPairsConfig_DIAGNOSTICS \ diff --git a/src/basic/vx_config/config_constants.h b/src/basic/vx_config/config_constants.h index 5fad36d25e..de6943cd58 100644 --- a/src/basic/vx_config/config_constants.h +++ b/src/basic/vx_config/config_constants.h @@ -1067,6 +1067,8 @@ static const char conf_key_basin_file[] = "basin_file"; static const char conf_key_track_watch_warn[] = "track_watch_warn"; static const char conf_key_watch_warn[] = "watch_warn"; static const char conf_key_diag_name[] = "diag_name"; +static const char conf_key_tcdiag_convert_map[] = "tcdiag_convert_map"; +static const char conf_key_lsdiag_convert_map[] = "lsdiag_convert_map"; static const char conf_key_basin_map[] = "basin_map"; static const char conf_key_time_offset[] = "time_offset"; static const char conf_key_amodel[] = "amodel"; diff --git a/src/basic/vx_config/config_util.cc b/src/basic/vx_config/config_util.cc index 5c1778bf39..29c8405147 100644 --- a/src/basic/vx_config/config_util.cc +++ b/src/basic/vx_config/config_util.cc @@ -977,7 +977,7 @@ TimeSummaryInfo parse_conf_time_summary(Dictionary *dict) { void parse_add_conf_key_value_map( Dictionary *dict, const char *conf_key_map_name, map *m) { - Dictionary *msg_typ_dict = (Dictionary *) 0; + Dictionary *map_dict = (Dictionary *) 0; ConcatString key, val; int i; @@ -988,14 +988,14 @@ void parse_add_conf_key_value_map( } // Conf: map_name: message_type_map, obs)var_map, etc - msg_typ_dict = dict->lookup_array(conf_key_map_name); + map_dict = dict->lookup_array(conf_key_map_name); // Loop through the array entries - for(i=0; in_entries(); i++) { + for(i=0; in_entries(); i++) { // Lookup the key and value - key = (*msg_typ_dict)[i]->dict_value()->lookup_string(conf_key_key); - val = (*msg_typ_dict)[i]->dict_value()->lookup_string(conf_key_val); + key = (*map_dict)[i]->dict_value()->lookup_string(conf_key_key); + val = (*map_dict)[i]->dict_value()->lookup_string(conf_key_val); if(m->count(key) >= 1) { (*m)[key] = val; @@ -1011,10 +1011,9 @@ void parse_add_conf_key_value_map( /////////////////////////////////////////////////////////////////////////////// - map parse_conf_key_value_map( Dictionary *dict, const char *conf_key_map_name, const char *caller) { - Dictionary *msg_typ_dict = (Dictionary *) 0; + Dictionary *map_dict = (Dictionary *) 0; map m; ConcatString key, val; int i; @@ -1026,14 +1025,14 @@ map parse_conf_key_value_map( } // Conf: map_name: message_type_map, obs_var_map, etc - msg_typ_dict = dict->lookup_array(conf_key_map_name); + map_dict = dict->lookup_array(conf_key_map_name); // Loop through the array entries - for(i=0; in_entries(); i++) { + for(i=0; in_entries(); i++) { // Lookup the key and value - key = (*msg_typ_dict)[i]->dict_value()->lookup_string(conf_key_key); - val = (*msg_typ_dict)[i]->dict_value()->lookup_string(conf_key_val); + key = (*map_dict)[i]->dict_value()->lookup_string(conf_key_key); + val = (*map_dict)[i]->dict_value()->lookup_string(conf_key_val); if(m.count(key) >= 1) { mlog << Warning << "\n" << method_name @@ -1100,6 +1099,62 @@ map parse_conf_obs_name_map(Dictionary *dict) { /////////////////////////////////////////////////////////////////////////////// +map parse_conf_key_convert_map( + Dictionary *dict, const char *conf_key_map_name, const char *caller) { + Dictionary *map_dict = (Dictionary *) 0; + int i, j; + StringArray sa; + ConcatString key; + UserFunc_1Arg fx; + map m; + const char *method_name = (0 != caller) ? caller : "parse_conf_key_convert_map() -> "; + + if(!dict) { + mlog << Error << "\n" << method_name << "empty dictionary!\n\n"; + exit(1); + } + + // Conf: tcdiag_convert_map, lsdiag_convert_map, etc + map_dict = dict->lookup_array(conf_key_map_name); + + // Loop through the array entries + for(i=0; in_entries(); i++) { + + // Lookup the key and convert function + sa = (*map_dict)[i]->dict_value()->lookup_string_array(conf_key_key); + fx.clear(); + fx.set((*map_dict)[i]->dict_value()->lookup(conf_key_convert)); + + // Check the function + if(!fx.is_set()) { + mlog << Error << "\n" << method_name + << "lookup for \"" << conf_key_convert << "\" failed in the \"" + << conf_key_map_name << "\" map!\n\n"; + exit(1); + } + + // Add map entry for each string + for(j=0; j= 1) { + mlog << Warning << "\n" << method_name + << "found multiple entries for key \"" << key << "\" in the \"" + << conf_key_map_name << "\" map!\n\n"; + } + + // Add entry to the map + m.insert(pair(key,fx)); + + } // end for j + } // end for i + + return(m); +} + +/////////////////////////////////////////////////////////////////////////////// + void BootInfo::clear() { interval = BootIntervalType_None; rep_prop = bad_data_double; diff --git a/src/basic/vx_config/config_util.h b/src/basic/vx_config/config_util.h index 45de85dd0f..63e84a685e 100644 --- a/src/basic/vx_config/config_util.h +++ b/src/basic/vx_config/config_util.h @@ -62,6 +62,8 @@ extern std::map extern std::map parse_conf_metadata_map(Dictionary *dict); extern std::map parse_conf_obs_name_map(Dictionary *dict); +extern std::map parse_conf_key_convert_map( + Dictionary *dict, const char *conf_key_map_name, const char *caller=0); extern BootInfo parse_conf_boot(Dictionary *dict); extern RegridInfo parse_conf_regrid(Dictionary *dict, bool error_out = default_dictionary_error_out); extern InterpInfo parse_conf_interp(Dictionary *dict, const char *); diff --git a/src/libcode/vx_tc_util/diag_file.cc b/src/libcode/vx_tc_util/diag_file.cc index d6b81ff944..af35d50100 100644 --- a/src/libcode/vx_tc_util/diag_file.cc +++ b/src/libcode/vx_tc_util/diag_file.cc @@ -39,6 +39,8 @@ static int n_lsdiag_wdth = sizeof(lsdiag_wdth)/sizeof(*lsdiag_wdth); static const int tcdiag_fill_value = 9999; static const int lsdiag_fill_value = 9999; +static const char tcdiag_div10_units_str[] = "(10C),(10KT),(10M/S)"; + //////////////////////////////////////////////////////////////////////// // // Code for class DiagFile @@ -191,10 +193,18 @@ void DiagFile::set_technique(const string &str) { //////////////////////////////////////////////////////////////////////// -void DiagFile::read_tcdiag(const std::string &path, const std::string &model_name) { - int i; +void DiagFile::read_tcdiag(const std::string &path, const std::string &model_name, + const std::map &convert_map) { + int i, v_int; + double v_dbl; NumArray data; + const UserFunc_1Arg *fx_ptr = 0; + // Units whose values should be divided by 10 + StringArray div10_units; + div10_units.parse_css(tcdiag_div10_units_str); + + // Store the file type FileType = TCDiagFileType; // Initialize the technique name @@ -275,18 +285,30 @@ void DiagFile::read_tcdiag(const std::string &path, const std::string &model_nam // Quit reading at the COMMENTS section if((cs = dl[1]) == "COMMENTS") break; - // first column contains the name + // First column contains the name cs = dl[0]; // Skip certain lines if(cs.startswith("----") || cs.startswith("TIME") || cs.startswith("NLEV") || cs.startswith("NVAR")) continue; + // Check for a conversion function + fx_ptr = (convert_map.count(cs) > 0 ? &convert_map.at(cs) : (UserFunc_1Arg *) 0); + // Parse the data values data.erase(); for(i=2; i &convert_map) { + int i, v_int; + double v_dbl; NumArray data; + const UserFunc_1Arg *fx_ptr = 0; + // Store the file type FileType = LSDiagFileType; // Store the default lsdiag technique, unless otherwise specified @@ -395,14 +421,28 @@ void DiagFile::read_lsdiag(const std::string &path, const std::string &model_nam // Check the 24th column cs = dl[23]; + // Strip any whitespace from the fixed-width column + cs.ws_strip(); + // Quit reading at the LAST line if(cs == "LAST") break; + // Check for a conversion function + fx_ptr = (convert_map.count(cs) > 0 ? &convert_map.at(cs) : (UserFunc_1Arg *) 0); + // Parse the data values data.erase(); for(i=2; i<23; i++) { - data.add(atoi(dl[i]) == lsdiag_fill_value ? - bad_data_double : atof(dl[i])); + + v_int = atoi(dl[i]); + + // Check for bad data and apply conversions + if(v_int == lsdiag_fill_value) v_dbl = bad_data_double; + else if(fx_ptr) v_dbl = (*fx_ptr)(v_int); + else v_dbl = (double) v_int; + + // Store the value + data.add(v_dbl); } // Check for the expected number of items diff --git a/src/libcode/vx_tc_util/diag_file.h b/src/libcode/vx_tc_util/diag_file.h index 593a40bd79..1996612988 100644 --- a/src/libcode/vx_tc_util/diag_file.h +++ b/src/libcode/vx_tc_util/diag_file.h @@ -118,8 +118,10 @@ class DiagFile : public LineDataFile { // do stuff // - void read_tcdiag(const std::string &, const std::string &); - void read_lsdiag(const std::string &, const std::string &); + void read_tcdiag(const std::string &, const std::string &, + const std::map &); + void read_lsdiag(const std::string &, const std::string &, + const std::map &); }; diff --git a/src/tools/tc_utils/tc_pairs/tc_pairs.cc b/src/tools/tc_utils/tc_pairs/tc_pairs.cc index 837af0af80..d114fc3332 100644 --- a/src/tools/tc_utils/tc_pairs/tc_pairs.cc +++ b/src/tools/tc_utils/tc_pairs/tc_pairs.cc @@ -1040,7 +1040,8 @@ void process_diags(TrackInfoArray &tracks) { // Loop over the input files for(i=0,n=0; ilookup_string_array(conf_key_diag_name); + // Conf: TCDiagConvertFxMap + TCDiagConvertFxMap = parse_conf_key_convert_map(dict, conf_key_tcdiag_convert_map); + + // Conf: LSDiagConvertFxMap + LSDiagConvertFxMap = parse_conf_key_convert_map(dict, conf_key_lsdiag_convert_map); + // Conf: BasinMap BasinMap = parse_conf_key_value_map(dict, conf_key_basin_map); diff --git a/src/tools/tc_utils/tc_pairs/tc_pairs_conf_info.h b/src/tools/tc_utils/tc_pairs/tc_pairs_conf_info.h index dff8b1acbc..080d0b9647 100644 --- a/src/tools/tc_utils/tc_pairs/tc_pairs_conf_info.h +++ b/src/tools/tc_utils/tc_pairs/tc_pairs_conf_info.h @@ -114,6 +114,10 @@ class TCPairsConfInfo { // Diagnostics to be extracted StringArray DiagName; + // Diagnostic conversion maps + std::map TCDiagConvertFxMap; + std::map LSDiagConvertFxMap; + // Basin Map std::map BasinMap; From d07334c8da123e377bd686c8eff738da649f58cb Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Fri, 21 Oct 2022 15:07:36 -0600 Subject: [PATCH 35/58] Per #392, move the diagnostic location checking from TrackPoint up into TrackInfo and only check it once. This reduces the warning messages from 1500+ down to 7 in a single run. Only check locations for the first diagnostic processed. --- src/libcode/vx_tc_util/track_info.cc | 19 +++++++++++++------ src/libcode/vx_tc_util/track_point.cc | 10 +--------- src/libcode/vx_tc_util/track_point.h | 2 +- 3 files changed, 15 insertions(+), 16 deletions(-) diff --git a/src/libcode/vx_tc_util/track_info.cc b/src/libcode/vx_tc_util/track_info.cc index ef750127f4..12df7996b1 100644 --- a/src/libcode/vx_tc_util/track_info.cc +++ b/src/libcode/vx_tc_util/track_info.cc @@ -550,10 +550,19 @@ bool TrackInfo::add_diag_data(DiagFile &diag_file, const StringArray &diag_name) // Get the index of the TrackPoint for this lead time if((i_pnt = lead_index(nint(diag_file.lead(i_time)))) < 0) continue; + // Check for consistent location, but only once + if(i_name == 0) { + if(!is_eq(diag_file.lat(i_time), Point[i_pnt].lat()) || + !is_eq(diag_file.lon(i_time), Point[i_pnt].lon())) { + mlog << Warning << "\nTrackInfo::add_diag_data() -> " + << "the diagnostic location (" << diag_file.lat(i_time) << ", " + << diag_file.lon(i_time) << ") does not match the track location (" + << Point[i_pnt].lat() << ", " << Point[i_pnt].lon() << ")\n"; + } + } + // Store this diagnostic value in the TrackPoint - Point[i_pnt].add_diag_value(diag_file.lat(i_time), - diag_file.lon(i_time), - diag_val[i_time]); + Point[i_pnt].add_diag_value(diag_val[i_time]); } // end for i_time } // end for i_name @@ -572,9 +581,7 @@ void TrackInfo::add_diag_value(int i_pnt, double val) { exit(1); } - Point[i_pnt].add_diag_value(Point[i_pnt].lat(), - Point[i_pnt].lon(), - val); + Point[i_pnt].add_diag_value(val); return; } diff --git a/src/libcode/vx_tc_util/track_point.cc b/src/libcode/vx_tc_util/track_point.cc index 36b2833e1c..0badcb827c 100644 --- a/src/libcode/vx_tc_util/track_point.cc +++ b/src/libcode/vx_tc_util/track_point.cc @@ -685,15 +685,7 @@ bool TrackPoint::is_match(const ATCFTrackLine &l) const { //////////////////////////////////////////////////////////////////////// -void TrackPoint::add_diag_value(double diag_lat, double diag_lon, double val) { - - // Check for consistent location - if(!is_eq(diag_lat, Lat) || !is_eq(diag_lon, Lon)) { - mlog << Warning << "\nTrackPoint::add_diag_value() -> " - << "the diagnostic location (" << diag_lat << ", " << diag_lon - << ") does not match the track location (" << Lat << ", " << Lon - << ")\n"; - } +void TrackPoint::add_diag_value(double val) { // Store the diagnostic value DiagVal.add(val); diff --git a/src/libcode/vx_tc_util/track_point.h b/src/libcode/vx_tc_util/track_point.h index ca87637604..27ae223dae 100644 --- a/src/libcode/vx_tc_util/track_point.h +++ b/src/libcode/vx_tc_util/track_point.h @@ -262,7 +262,7 @@ class TrackPoint { void set_wind(int, const QuadInfo &); bool set(const ATCFTrackLine &); bool is_match(const ATCFTrackLine &) const; - void add_diag_value(double, double, double); + void add_diag_value(double); }; From b449814d19cfaedfdd5057caaf930b70c676371e Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Fri, 21 Oct 2022 16:10:57 -0600 Subject: [PATCH 36/58] Per #392, for -tcdiag and -lsdiag, let model=STRING be set to a comma-separated list of values. --- docs/Users_Guide/tc-pairs.rst | 2 +- src/libcode/vx_tc_util/diag_file.cc | 36 ++++++++++--------- src/libcode/vx_tc_util/diag_file.h | 12 +++---- src/libcode/vx_tc_util/track_info.cc | 46 ++++++++++--------------- src/libcode/vx_tc_util/track_info.h | 2 +- src/tools/tc_utils/tc_pairs/tc_pairs.cc | 16 +++++---- 6 files changed, 56 insertions(+), 58 deletions(-) diff --git a/docs/Users_Guide/tc-pairs.rst b/docs/Users_Guide/tc-pairs.rst index 89083769a5..1394b4b0ed 100644 --- a/docs/Users_Guide/tc-pairs.rst +++ b/docs/Users_Guide/tc-pairs.rst @@ -69,7 +69,7 @@ Optional arguments for tc_pairs 9. The **-v level** option indicates the desired level of verbosity. The contents of "level" will override the default setting of 2. Setting the verbosity to 0 will make the tool run with no log messages, while increasing the verbosity above 1 will increase the amount of logging. -This tool currently only supports the rapid intensification (**RI**) edeck probability type but support for additional edeck probability types will be added in future releases. At least one **-adeck** or **-edeck** option must be specified. The **-adeck, -edeck**, and **-bdeck** options may optionally be followed with **suffix=string** to append that string to all model names found within that data source. This option may be useful when processing track data from two different sources which reuse the same model names. The **-tcdiag** and **-lsdiag** options may optionally be followed with **model=string** to override the model name of the tracks to which those diagnostics correspond. +This tool currently only supports the rapid intensification (**RI**) edeck probability type but support for additional edeck probability types will be added in future releases. At least one **-adeck** or **-edeck** option must be specified. The **-adeck, -edeck**, and **-bdeck** options may optionally be followed with **suffix=string** to append that string to all model names found within that data source. This option may be useful when processing track data from two different sources which reuse the same model names. The **-tcdiag** and **-lsdiag** options may optionally be followed with **model=string** to override the model name of the tracks to which those diagnostics correspond. The **string** specifies a comma-separated list of one or more ATCF ID's to which these diagnostics should be paired (e.g. **model=OFCL,SHIP**). An example of the tc_pairs calling sequence is shown below: diff --git a/src/libcode/vx_tc_util/diag_file.cc b/src/libcode/vx_tc_util/diag_file.cc index af35d50100..62a97c8de7 100644 --- a/src/libcode/vx_tc_util/diag_file.cc +++ b/src/libcode/vx_tc_util/diag_file.cc @@ -179,13 +179,17 @@ void DiagFile::init_from_scratch() { //////////////////////////////////////////////////////////////////////// -void DiagFile::set_technique(const string &str) { +void DiagFile::set_technique(const StringArray &sa) { - Technique = str; + Technique = sa; // Replace instances of AVN with GFS - if(strstr(Technique.c_str(), "AVN") != NULL) { - Technique.replace("AVN", "GFS"); + for(int i=0; i &convert_map) { +void DiagFile::read_tcdiag(const ConcatString &path, const StringArray &model_names, + const map &convert_map) { int i, v_int; double v_dbl; NumArray data; @@ -207,8 +211,8 @@ void DiagFile::read_tcdiag(const std::string &path, const std::string &model_nam // Store the file type FileType = TCDiagFileType; - // Initialize the technique name - set_technique(model_name); + // Store user-specified model names or read it from the file below + if(model_names.n() > 0) set_technique(model_names); // Open the file open(path.c_str()); @@ -229,7 +233,7 @@ void DiagFile::read_tcdiag(const std::string &path, const std::string &model_nam // First header line: Technique InitTime (ATCFID YYYMMDDHH) if(InitTime == 0) { - if(Technique.empty()) set_technique(dl[1]); + if(Technique.n() == 0) Technique.add(dl[1]); InitTime = timestring_to_unix(dl[2]); } // Second header line: Basin Cyclone Number (BBCC) @@ -327,7 +331,7 @@ void DiagFile::read_tcdiag(const std::string &path, const std::string &model_nam } // end while mlog << Debug(4) << "Parsed " << DiagName.n() << " diagnostic values from " - << StormId << " " << Technique << " " << unix_to_yyyymmddhh(InitTime) + << StormId << " " << write_css(Technique) << " " << unix_to_yyyymmddhh(InitTime) << " TC diagnostics file: " << path << "\n"; // Close the input file @@ -338,8 +342,8 @@ void DiagFile::read_tcdiag(const std::string &path, const std::string &model_nam //////////////////////////////////////////////////////////////////////// -void DiagFile::read_lsdiag(const std::string &path, const std::string &model_name, - const std::map &convert_map) { +void DiagFile::read_lsdiag(const ConcatString &path, const StringArray &model_names, + const map &convert_map) { int i, v_int; double v_dbl; NumArray data; @@ -348,9 +352,9 @@ void DiagFile::read_lsdiag(const std::string &path, const std::string &model_nam // Store the file type FileType = LSDiagFileType; - // Store the default lsdiag technique, unless otherwise specified - if(model_name.size() > 0) set_technique(model_name); - else set_technique(default_lsdiag_technique); + // Store user-specified model names or the default one + if(model_names.n() > 0) set_technique(model_names); + else Technique.add(default_lsdiag_technique); // Open the file open(path.c_str()); @@ -461,7 +465,7 @@ void DiagFile::read_lsdiag(const std::string &path, const std::string &model_nam } // end while mlog << Debug(4) << "Parsed " << DiagName.n() << " diagnostic values from " - << StormId << " " << Technique << " " << unix_to_yyyymmddhh(InitTime) + << StormId << " " << write_css(Technique) << " " << unix_to_yyyymmddhh(InitTime) << " LS diagnostics file: " << path << "\n"; // Close the input file diff --git a/src/libcode/vx_tc_util/diag_file.h b/src/libcode/vx_tc_util/diag_file.h index 1996612988..d3785f9d69 100644 --- a/src/libcode/vx_tc_util/diag_file.h +++ b/src/libcode/vx_tc_util/diag_file.h @@ -69,7 +69,7 @@ class DiagFile : public LineDataFile { ConcatString StormId; ConcatString Basin; ConcatString Cyclone; - ConcatString Technique; + StringArray Technique; unixtime InitTime; int NTime; @@ -90,7 +90,7 @@ class DiagFile : public LineDataFile { // set stuff // - void set_technique(const std::string &); + void set_technique(const StringArray &); // // get stuff @@ -99,7 +99,7 @@ class DiagFile : public LineDataFile { const ConcatString & storm_id() const; const ConcatString & basin() const; const ConcatString & cyclone() const; - const ConcatString & technique() const; + const StringArray & technique() const; const ConcatString & initials() const; unixtime init() const; @@ -118,9 +118,9 @@ class DiagFile : public LineDataFile { // do stuff // - void read_tcdiag(const std::string &, const std::string &, + void read_tcdiag(const ConcatString &, const StringArray &, const std::map &); - void read_lsdiag(const std::string &, const std::string &, + void read_lsdiag(const ConcatString &, const StringArray &, const std::map &); }; @@ -130,7 +130,7 @@ class DiagFile : public LineDataFile { inline const ConcatString & DiagFile::storm_id() const { return(StormId); } inline const ConcatString & DiagFile::basin() const { return(Basin); } inline const ConcatString & DiagFile::cyclone() const { return(Cyclone); } -inline const ConcatString & DiagFile::technique() const { return(Technique); } +inline const StringArray & DiagFile::technique() const { return(Technique); } inline unixtime DiagFile::init() const { return(InitTime); } inline int DiagFile::n_time() const { return(NTime); } inline int DiagFile::n_diag() const { return(DiagName.n()); } diff --git a/src/libcode/vx_tc_util/track_info.cc b/src/libcode/vx_tc_util/track_info.cc index 12df7996b1..65c6d49289 100644 --- a/src/libcode/vx_tc_util/track_info.cc +++ b/src/libcode/vx_tc_util/track_info.cc @@ -253,8 +253,7 @@ void TrackInfo::extend(int n, bool exact) { new_line = new TrackPoint [n]; if(!new_line) { - mlog << Error - << "\nvoid TrackInfo::extend(int, bool) -> " + mlog << Error << "\nvoid TrackInfo::extend(int, bool) -> " << "memory allocation error\n\n"; exit(1); } @@ -313,8 +312,7 @@ void TrackInfo::set_point(int n, const TrackPoint &p) { // Check range if((n < 0) || (n >= NPoints)) { - mlog << Error - << "\nTrackInfo::set_point(int, const TrackPoint &) -> " + mlog << Error << "\nTrackInfo::set_point(int, const TrackPoint &) -> " << "range check error for index value " << n << "\n\n"; exit(1); } @@ -360,8 +358,7 @@ const TrackPoint & TrackInfo::operator[](int n) const { // Check range if((n < 0) || (n >= NPoints)) { - mlog << Error - << "\nTrackInfo::operator[](int) -> " + mlog << Error << "\nTrackInfo::operator[](int) -> " << "range check error for index value " << n << "\n\n"; exit(1); } @@ -529,9 +526,9 @@ void TrackInfo::add_watch_warn(const ConcatString &ww_sid, bool TrackInfo::add_diag_data(DiagFile &diag_file, const StringArray &diag_name) { // Check for a match - if(StormId != diag_file.storm_id() || - Technique != diag_file.technique() || - InitTime != diag_file.init()) return(false); + if(StormId != diag_file.storm_id() || + InitTime != diag_file.init() || + !diag_file.technique().has(Technique)) return(false); // If empty, store all diagnostics if(diag_name.n() > 0) DiagName = diag_name; @@ -555,9 +552,11 @@ bool TrackInfo::add_diag_data(DiagFile &diag_file, const StringArray &diag_name) if(!is_eq(diag_file.lat(i_time), Point[i_pnt].lat()) || !is_eq(diag_file.lon(i_time), Point[i_pnt].lon())) { mlog << Warning << "\nTrackInfo::add_diag_data() -> " - << "the diagnostic location (" << diag_file.lat(i_time) << ", " - << diag_file.lon(i_time) << ") does not match the track location (" - << Point[i_pnt].lat() << ", " << Point[i_pnt].lon() << ")\n"; + << "the " << StormId << " " << Technique << " " << unix_to_yyyymmddhh(InitTime) + << " lead time " << sec_to_timestring(diag_file.lead(i_time)) + << " track location (" << Point[i_pnt].lat() << ", " + << Point[i_pnt].lon() << ") does not match the diagnostic location (" + << diag_file.lat(i_time) << ", " << diag_file.lon(i_time) << ")\n"; } } @@ -806,8 +805,7 @@ const TrackInfo & TrackInfoArray::operator[](int n) const { // Check range if((n < 0) || (n >= Track.size())) { - mlog << Error - << "\nTrackInfoArray::operator[](int) -> " + mlog << Error << "\nTrackInfoArray::operator[](int) -> " << "range check error for index value " << n << "\n\n"; exit(1); } @@ -830,8 +828,7 @@ void TrackInfoArray::set(int n, const TrackInfo &t) { // Check range if((n < 0) || (n >= Track.size())) { - mlog << Error - << "\nTrackInfoArray::set(int, const TrackInfo &) -> " + mlog << Error << "\nTrackInfoArray::set(int, const TrackInfo &) -> " << "range check error for index value " << n << "\n\n"; exit(1); } @@ -917,18 +914,15 @@ bool TrackInfoArray::erase_storm_id(const ConcatString &s) { //////////////////////////////////////////////////////////////////////// -bool TrackInfoArray::add_diag_data(DiagFile &diag_file, const StringArray &diag_name) { - bool match = false; +int TrackInfoArray::add_diag_data(DiagFile &diag_file, const StringArray &diag_name) { + int n_match = 0; // Set the names for each track for(int i=0; i " + mlog << Error << "\nTrackInfoArray::consensus() -> " << "cannot compute a consensus for zero tracks!\n\n"; exit(1); } @@ -977,8 +970,7 @@ TrackInfo consensus(const TrackInfoArray &tracks, if(tavg.basin() != tracks[i].basin() || tavg.cyclone() != tracks[i].cyclone() || tavg.init() != tracks[i].init()) { - mlog << Error - << "\nTrackInfoArray::consensus() -> " + mlog << Error << "\nTrackInfoArray::consensus() -> " << "the basin, cyclone number, and init time must " << "remain constant.\n\n"; exit(1); diff --git a/src/libcode/vx_tc_util/track_info.h b/src/libcode/vx_tc_util/track_info.h index aa028ef059..c01d9a2fab 100644 --- a/src/libcode/vx_tc_util/track_info.h +++ b/src/libcode/vx_tc_util/track_info.h @@ -244,7 +244,7 @@ class TrackInfoArray { bool add(const ATCFTrackLine &, bool check_dup = false, bool check_anly = false); bool has(const ATCFTrackLine &) const; bool erase_storm_id(const ConcatString &); - bool add_diag_data(DiagFile &, const StringArray &); + int add_diag_data(DiagFile &, const StringArray &); // // get stuff diff --git a/src/tools/tc_utils/tc_pairs/tc_pairs.cc b/src/tools/tc_utils/tc_pairs/tc_pairs.cc index d114fc3332..84583c728b 100644 --- a/src/tools/tc_utils/tc_pairs/tc_pairs.cc +++ b/src/tools/tc_utils/tc_pairs/tc_pairs.cc @@ -255,7 +255,7 @@ void process_command_line(int argc, char **argv) { for(i=0; i Date: Fri, 21 Oct 2022 16:21:32 -0600 Subject: [PATCH 37/58] Per #392, just updating code comments. --- src/tools/tc_utils/tc_stat/tc_stat_job.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tools/tc_utils/tc_stat/tc_stat_job.cc b/src/tools/tc_utils/tc_stat/tc_stat_job.cc index 0bd1b2f054..32d56caa7b 100644 --- a/src/tools/tc_utils/tc_stat/tc_stat_job.cc +++ b/src/tools/tc_utils/tc_stat/tc_stat_job.cc @@ -654,7 +654,7 @@ bool TCStatJob::is_keeper_track(const TrackPairInfo &pair, break; } - // Check that diagnostic value + // Check the diagnostic value threshold if(!thr_it->second.check_dbl(pair.adeck()[i_init].diag_val(i_diag))) { keep = false; break; @@ -945,7 +945,7 @@ bool TCStatJob::is_keeper_tcdiag(const StringArray &diag_name, // Get the numeric diagnostic value v_dbl = get_diag_double(diag_name, point, thr_it->first); - // Check the diagnostic threshold + // Check the diagnostic value threshold if(!thr_it->second.check_dbl(v_dbl)) { keep = false; n.RejDiagThresh++; From 4a23a1c9e96cea358aabd33c4d9024ae71642f35 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Fri, 21 Oct 2022 17:22:04 -0600 Subject: [PATCH 38/58] Per #392, print a single warning from tc_stat for each diagnostic name that is requested but is not actually present in the input data. --- src/tools/tc_utils/tc_stat/tc_stat_job.cc | 31 ++++++++++++++--------- src/tools/tc_utils/tc_stat/tc_stat_job.h | 7 ++--- 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/src/tools/tc_utils/tc_stat/tc_stat_job.cc b/src/tools/tc_utils/tc_stat/tc_stat_job.cc index 32d56caa7b..79f85cb5eb 100644 --- a/src/tools/tc_utils/tc_stat/tc_stat_job.cc +++ b/src/tools/tc_utils/tc_stat/tc_stat_job.cc @@ -226,6 +226,7 @@ void TCStatJob::clear() { InitStrExcMap.clear(); DiagThreshMap.clear(); InitDiagThreshMap.clear(); + PrintDiagWarning.clear(); EventEqualLead.clear(); EventEqualCases.clear(); @@ -311,6 +312,7 @@ void TCStatJob::assign(const TCStatJob & j) { InitStrExcMap = j.InitStrExcMap; DiagThreshMap = j.DiagThreshMap; InitDiagThreshMap = j.InitDiagThreshMap; + PrintDiagWarning = j.PrintDiagWarning; DumpFile = j.DumpFile; open_dump_file(); @@ -523,7 +525,7 @@ void TCStatJob::dump(ostream & out, int depth) const { //////////////////////////////////////////////////////////////////////// bool TCStatJob::is_keeper_track(const TrackPairInfo &pair, - TCPointCounts &n) const { + TCPointCounts &n) { bool keep = true; int i, i_init; double v_dbl; @@ -648,21 +650,17 @@ bool TCStatJob::is_keeper_track(const TrackPairInfo &pair, // Loop through the numeric diagnostic thresholds for(thr_it=InitDiagThreshMap.begin(); thr_it!= InitDiagThreshMap.end(); thr_it++) { - // Check whether the diagnostic name exists - if(!pair.adeck().diag_name().has(thr_it->first, i_diag)) { - keep = false; - break; - } + // Get the numeric diagnostic value + v_dbl = get_diag_double(pair.adeck().diag_name(), + pair.adeck()[i_init], thr_it->first); // Check the diagnostic value threshold - if(!thr_it->second.check_dbl(pair.adeck()[i_init].diag_val(i_diag))) { + if(!thr_it->second.check_dbl(v_dbl)) { keep = false; + n.RejInitDiagThresh += pair.n_points(); break; } } - - // Update counts - if(!keep) n.RejInitDiagThresh += pair.n_points(); } // Check OutInitMask @@ -932,7 +930,7 @@ double TCStatJob::get_column_double(const TCStatLine &line, bool TCStatJob::is_keeper_tcdiag(const StringArray &diag_name, const TrackPoint &point, - TCPointCounts &n) const { + TCPointCounts &n) { bool keep = true; double v_dbl; map::const_iterator thr_it; @@ -963,7 +961,7 @@ bool TCStatJob::is_keeper_tcdiag(const StringArray &diag_name, double TCStatJob::get_diag_double(const StringArray &diag_name, const TrackPoint &point, - const ConcatString &diag_cs) const { + const ConcatString &diag_cs) { double v = bad_data_double; int i_diag; @@ -971,6 +969,15 @@ double TCStatJob::get_diag_double(const StringArray &diag_name, if(diag_name.has(diag_cs, i_diag)) { v = point.diag_val(i_diag); } + // Print a single warning message if diagnostics are present + // but not the requested one + else if(diag_name.n() > 0 && + !PrintDiagWarning.has(diag_cs)) { + mlog << Warning << "\nTCStatJob::get_diag_double() -> " + << "diagnostic \"" << diag_cs + << "\" not defined for TrackPoint.\n\n"; + PrintDiagWarning.add(diag_cs); + } return(v); } diff --git a/src/tools/tc_utils/tc_stat/tc_stat_job.h b/src/tools/tc_utils/tc_stat/tc_stat_job.h index 475080343f..020134b593 100644 --- a/src/tools/tc_utils/tc_stat/tc_stat_job.h +++ b/src/tools/tc_utils/tc_stat/tc_stat_job.h @@ -207,15 +207,15 @@ class TCStatJob { ////////////////////////////////////////////////////////////////// - bool is_keeper_track(const TrackPairInfo &, TCPointCounts &) const; + bool is_keeper_track(const TrackPairInfo &, TCPointCounts &); bool is_keeper_line(const TCStatLine &, TCPointCounts &) const; double get_column_double(const TCStatLine &, const ConcatString &) const; - bool is_keeper_tcdiag(const StringArray &, const TrackPoint &, TCPointCounts &) const; + bool is_keeper_tcdiag(const StringArray &, const TrackPoint &, TCPointCounts &); - double get_diag_double(const StringArray &, const TrackPoint &, const ConcatString &) const; + double get_diag_double(const StringArray &, const TrackPoint &, const ConcatString &); ////////////////////////////////////////////////////////////////// @@ -307,6 +307,7 @@ class TCStatJob { // Numeric diagnostic thresholds std::map DiagThreshMap; std::map InitDiagThreshMap; + StringArray PrintDiagWarning; // Variables to the store the analysis job specification ConcatString DumpFile; // Dump TrackPairInfo used to a file From 1fe0bde1c44f02d3c0b516d84018cdd7dd920d21 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Sat, 22 Oct 2022 10:20:07 -0600 Subject: [PATCH 39/58] Per #392, refine the logic for tcdiag_convert_map config entry. Let it specify the conversion function based on the diagnostic name or units string. If both are provided, the conversion function for the name takes precedence. --- data/config/TCPairsConfig_default | 6 +++++- docs/Users_Guide/tc-pairs.rst | 8 +++++--- src/libcode/vx_tc_util/diag_file.cc | 19 +++++++------------ 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/data/config/TCPairsConfig_default b/data/config/TCPairsConfig_default index c76f036329..a6f2391687 100644 --- a/data/config/TCPairsConfig_default +++ b/data/config/TCPairsConfig_default @@ -145,7 +145,11 @@ diag_name = []; // // Unit conversions to be applied to diagnostic values // -tcdiag_convert_map = []; + +tcdiag_convert_map = [ + { key = [ "(10C)", "(10KT)", "(10M/S)" ]; convert(x) = x / 10; } +] + lsdiag_convert_map = [ { key = [ "CSST", "RSST", "DSST", "DSTA", "XDST", "XNST", "NSST", "NSTA", "NTMX", "NTFR", "U200", "U20C", "V20C", "E000", "EPOS", "ENEG", "EPSS", "ENSS", diff --git a/docs/Users_Guide/tc-pairs.rst b/docs/Users_Guide/tc-pairs.rst index 1394b4b0ed..dd6111e701 100644 --- a/docs/Users_Guide/tc-pairs.rst +++ b/docs/Users_Guide/tc-pairs.rst @@ -265,12 +265,14 @@ ____________________ .. code-block:: none - tcdiag_convert_map = []; + tcdiag_convert_map = [ + { key = [ "(10C)", "(10KT)", "(10M/S)" ]; convert(x) = x / 10; } + ]; lsdiag_convert_map = []; -The **tcdiag_convert_map** and **lsdiag_convert_map** entries define conversion functions to be applied to diagnostics data read with the **-tcdiag** and **-lsdiag** command line options, respectively. Each array element is a dictionary consisting of a **key** and **convert(x)** entry. The **key** is an array of strings listing diagnostic names. **convert(x)** is a function which defines how the diagnostic data should be converted after being read. The defined function is applied to any diagnostic value who name appears in the **key**. +The **tcdiag_convert_map** and **lsdiag_convert_map** entries define conversion functions to be applied to diagnostics data read with the **-tcdiag** and **-lsdiag** command line options, respectively. Each array element is a dictionary consisting of a **key** and **convert(x)** entry. -Note that unit conversions are generally handled automatically for **-tcdiag** inputs based on the units defined in the input file. The conversion function **convert(x) = x / 10;** is automatically applied to any diagnostic with units of 10C, 10M/S, or 10KT. However, conversion functions defined in **tcdiag_convert_map** take precedence over that default behavior. +The **key** is an array of strings. For **-tcdiag** inputs, the strings can specify diagnostic names or units. If both the name and units are specified, the conversion function for the name takes precedence. For **-lsdiag** inputs, the strings specify diagnostic names since no unit strings are provided. **convert(x)** is a function of one variable which defines how the diagnostic data should be converted. The defined function is applied to any diagnostic value who name or units appears in the **key**. ____________________ diff --git a/src/libcode/vx_tc_util/diag_file.cc b/src/libcode/vx_tc_util/diag_file.cc index 62a97c8de7..e8ffda489f 100644 --- a/src/libcode/vx_tc_util/diag_file.cc +++ b/src/libcode/vx_tc_util/diag_file.cc @@ -39,8 +39,6 @@ static int n_lsdiag_wdth = sizeof(lsdiag_wdth)/sizeof(*lsdiag_wdth); static const int tcdiag_fill_value = 9999; static const int lsdiag_fill_value = 9999; -static const char tcdiag_div10_units_str[] = "(10C),(10KT),(10M/S)"; - //////////////////////////////////////////////////////////////////////// // // Code for class DiagFile @@ -204,10 +202,6 @@ void DiagFile::read_tcdiag(const ConcatString &path, const StringArray &model_na NumArray data; const UserFunc_1Arg *fx_ptr = 0; - // Units whose values should be divided by 10 - StringArray div10_units; - div10_units.parse_css(tcdiag_div10_units_str); - // Store the file type FileType = TCDiagFileType; @@ -296,8 +290,10 @@ void DiagFile::read_tcdiag(const ConcatString &path, const StringArray &model_na if(cs.startswith("----") || cs.startswith("TIME") || cs.startswith("NLEV") || cs.startswith("NVAR")) continue; - // Check for a conversion function - fx_ptr = (convert_map.count(cs) > 0 ? &convert_map.at(cs) : (UserFunc_1Arg *) 0); + // Check for a conversion function based on the diagnostic name or units + if(convert_map.count(cs) > 0) fx_ptr = &convert_map.at(cs); + else if(convert_map.count(dl[1]) > 0) fx_ptr = &convert_map.at(dl[1]); + else fx_ptr = (UserFunc_1Arg *) 0; // Parse the data values data.erase(); @@ -306,10 +302,9 @@ void DiagFile::read_tcdiag(const ConcatString &path, const StringArray &model_na v_int = atoi(dl[i]); // Check for bad data and apply conversions - if(v_int == tcdiag_fill_value) v_dbl = bad_data_double; - else if(fx_ptr) v_dbl = (*fx_ptr)(v_int); - else if(div10_units.has(dl[1])) v_dbl = v_int / 10.0; - else v_dbl = (double) v_int; + if(v_int == tcdiag_fill_value) v_dbl = bad_data_double; + else if(fx_ptr) v_dbl = (*fx_ptr)(v_int); + else v_dbl = (double) v_int; // Store the value data.add(v_dbl); From 41347dc8997bc955caaf9e85e47735b8332a97d7 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Sat, 22 Oct 2022 10:42:28 -0600 Subject: [PATCH 40/58] Per #392, add logic to skip over lsdiag diagnostics that require special handling. --- data/config/TCPairsConfig_default | 19 +++++++++++-------- src/libcode/vx_tc_util/diag_file.cc | 15 ++++++++++++--- 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/data/config/TCPairsConfig_default b/data/config/TCPairsConfig_default index a6f2391687..47af93e594 100644 --- a/data/config/TCPairsConfig_default +++ b/data/config/TCPairsConfig_default @@ -155,7 +155,9 @@ lsdiag_convert_map = [ "NTMX", "NTFR", "U200", "U20C", "V20C", "E000", "EPOS", "ENEG", "EPSS", "ENSS", "T000", "TLAT", "TLON", "TWAC", "TWXC", "G150", "G200", "G250", "V000", "V850", "V500", "V300", "PENC", "SHDC", "SHGC", "T150", "T200", "T250", "SHRD", "SHRS", - "SHRG", "PENV", "HE07", "HE05"]; convert(x) = x / 10; }, + "SHRG", "PENV", "HE07", "HE05", "PW01", "PW02", "PW03", "PW04", "PW05", "PW06", + "PW07", "PW08", "PW09", "PW10", "PW11", "PW12", "PW13", "PW14", "PW15", "PW16", + "PW17", "PW18", "PW20", "PW21" ]; convert(x) = x / 10; }, { key = [ "VVAV", "VMFX", "VVAC" ]; convert(x) = x / 100; } ]; @@ -182,22 +184,23 @@ lsdiag_convert_map = [ // "TIME", "DELV", "DTL", "CD20", "CD26", "COHC", "XOHC", "NDTX", "NDML", "ND30", // "ND28", "ND26", "ND24", "ND22", "ND20", "ND18", "ND16", "NDFR", "NOHC", "NO20", // "RHLO", "RHMD", "RHHI", "Z850", "D200", "REFC", "PEFC", "R000", "Z000", "TGRD", -// "TADV", "SDDC", "DIVC", "SHTD", "SHTS", "VMPI", "O500", "O700", "CFLX" +// "TADV", "SDDC", "DIVC", "SHTD", "SHTS", "VMPI", "O500", "O700", "CFLX", "PW19" // // Divide by 10: // "LAT", "LON", "CSST", "RSST", "DSST", "DSTA", "XDST", "XNST", "NSST", "NSTA", // "NTMX", "NTFR", "U200", "U20C", "V20C", "E000", "EPOS", "ENEG", "EPSS", "ENSS", // "T000", "TLAT", "TLON", "TWAC", "TWXC", "G150", "G200", "G250", "V000", "V850", // "V500", "V300", "PENC", "SHDC", "SHGC", "T150", "T200", "T250", "SHRD", "SHRS", -// "SHRG", "PENV", "HE07", "HE05" +// "SHRG", "PENV", "HE07", "HE05", "PW01", "PW02", "PW03", "PW04", "PW05", "PW06", +// "PW07", "PW08", "PW09", "PW10", "PW11", "PW12", "PW13", "PW14", "PW15", "PW16", +// "PW17", "PW18", "PW20", "PW21" // // Divide by 100: // "VVAV", "VMFX", "VVAC" -// -// Special logic required: -// "IR00", "IRM1", "IRM3", "PC00", "PCM1", "PCM3", "PSLV", "MTPW", "PW01", "PW02", -// "PW03", "PW04", "PW05", "PW06", "PW07", "PW08", "PW09", "PW10", "PW11", "PW12", -// "PW13", "PW14", "PW15", "PW16", "PW17", "PW18", "PW19", "PW20", "PW21", "IRXX" +// +// Ignore for now but could add special logic: +// "MTPW" - Because the same data is provided in PW01-PW21 +// "IR00", "IRM1", "IRM3", "PC00", "PCM1", "PCM3", "PSLV", "IRXX" // // Modify basin names to make them consistent across ATCF input files. diff --git a/src/libcode/vx_tc_util/diag_file.cc b/src/libcode/vx_tc_util/diag_file.cc index e8ffda489f..e7d501dd99 100644 --- a/src/libcode/vx_tc_util/diag_file.cc +++ b/src/libcode/vx_tc_util/diag_file.cc @@ -39,6 +39,8 @@ static int n_lsdiag_wdth = sizeof(lsdiag_wdth)/sizeof(*lsdiag_wdth); static const int tcdiag_fill_value = 9999; static const int lsdiag_fill_value = 9999; +static const char lsdiag_skip_str[] = "MTPW,IR00,IRM1,IRM3,PC00,PCM1,PCM3,PSLV,IRXX"; + //////////////////////////////////////////////////////////////////////// // // Code for class DiagFile @@ -354,6 +356,10 @@ void DiagFile::read_lsdiag(const ConcatString &path, const StringArray &model_na // Open the file open(path.c_str()); + // Diagnostic names to ignore + StringArray skip_sa; + skip_sa.parse_css(lsdiag_skip_str); + // Parse the header information from the first line DataLine dl; ConcatString cs; @@ -423,6 +429,9 @@ void DiagFile::read_lsdiag(const ConcatString &path, const StringArray &model_na // Strip any whitespace from the fixed-width column cs.ws_strip(); + // Check for diagnostic names to skip + if(skip_sa.has(cs)) continue; + // Quit reading at the LAST line if(cs == "LAST") break; @@ -436,9 +445,9 @@ void DiagFile::read_lsdiag(const ConcatString &path, const StringArray &model_na v_int = atoi(dl[i]); // Check for bad data and apply conversions - if(v_int == lsdiag_fill_value) v_dbl = bad_data_double; - else if(fx_ptr) v_dbl = (*fx_ptr)(v_int); - else v_dbl = (double) v_int; + if(v_int == lsdiag_fill_value) v_dbl = bad_data_double; + else if(fx_ptr) v_dbl = (*fx_ptr)(v_int); + else v_dbl = (double) v_int; // Store the value data.add(v_dbl); From 2fc945450b23077ec32175b6c413728ebb530e5a Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Sat, 22 Oct 2022 10:49:18 -0600 Subject: [PATCH 41/58] Per #392, update unit_tc_pairs.xml --- data/config/TCPairsConfig_default | 2 +- internal/test_unit/xml/unit_tc_pairs.xml | 40 ++++++++++++------------ 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/data/config/TCPairsConfig_default b/data/config/TCPairsConfig_default index 47af93e594..9fac140fad 100644 --- a/data/config/TCPairsConfig_default +++ b/data/config/TCPairsConfig_default @@ -148,7 +148,7 @@ diag_name = []; tcdiag_convert_map = [ { key = [ "(10C)", "(10KT)", "(10M/S)" ]; convert(x) = x / 10; } -] +]; lsdiag_convert_map = [ { key = [ "CSST", "RSST", "DSST", "DSTA", "XDST", "XNST", "NSST", "NSTA", diff --git a/internal/test_unit/xml/unit_tc_pairs.xml b/internal/test_unit/xml/unit_tc_pairs.xml index c7eeadd4f7..f1d8a8b9c6 100644 --- a/internal/test_unit/xml/unit_tc_pairs.xml +++ b/internal/test_unit/xml/unit_tc_pairs.xml @@ -182,45 +182,45 @@ - - &MET_BIN;/tc_pairs + + &MET_BIN;/tc_pairs - STORM_ID "AL072022" - INIT_INC "20220916_18" + STORM_ID "AL092021" + INIT_INC "20210827_12" DIAG_NAME \ - -adeck &DATA_DIR;/adeck/aal072022_OFCL_SHIP_AVNO.dat \ - -bdeck &DATA_DIR;/bdeck/bal072022.dat \ - -lsdiag &DATA_DIR;/lsdiag/22091618AL0722_lsdiag.dat \ + -adeck &DATA_DIR;/adeck/aal092021_OFCL_SHIP_AVNO.dat \ + -bdeck &DATA_DIR;/bdeck/bal092021.dat \ + -tcdiag &DATA_DIR;/tcdiag/sal092021_avno_doper_2021082712_diag.dat \ -config &CONFIG_DIR;/TCPairsConfig_DIAGNOSTICS \ - -out &OUTPUT_DIR;/tc_pairs/al072022_LSDIAG \ - -log &OUTPUT_DIR;/tc_pairs/tc_pairs_LSDIAG.log \ + -out &OUTPUT_DIR;/tc_pairs/al092021_TCDIAG \ + -log &OUTPUT_DIR;/tc_pairs/tc_pairs_TCDIAG.log \ -v 4 - &OUTPUT_DIR;/tc_pairs/al072022_LSDIAG.tcst + &OUTPUT_DIR;/tc_pairs/al092021_TCDIAG.tcst - - &MET_BIN;/tc_pairs + + &MET_BIN;/tc_pairs - STORM_ID "AL092021" - INIT_INC "20210827_12" + STORM_ID "AL072022" + INIT_INC "20220916_18" DIAG_NAME \ - -adeck &DATA_DIR;/adeck/aal092021_OFCL_SHIP_AVNO.dat \ - -bdeck &DATA_DIR;/bdeck/bal092021.dat \ - -tcdiag &DATA_DIR;/tcdiag/sal092021_avno_doper_2021082712_diag.dat \ + -adeck &DATA_DIR;/adeck/aal072022_OFCL_SHIP_AVNO.dat \ + -bdeck &DATA_DIR;/bdeck/bal072022.dat \ + -lsdiag &DATA_DIR;/lsdiag/22091618AL0722_lsdiag.dat \ -config &CONFIG_DIR;/TCPairsConfig_DIAGNOSTICS \ - -out &OUTPUT_DIR;/tc_pairs/al092021_TCDIAG \ - -log &OUTPUT_DIR;/tc_pairs/tc_pairs_TCDIAG.log \ + -out &OUTPUT_DIR;/tc_pairs/al072022_LSDIAG \ + -log &OUTPUT_DIR;/tc_pairs/tc_pairs_LSDIAG.log \ -v 4 - &OUTPUT_DIR;/tc_pairs/al092021_TCDIAG.tcst + &OUTPUT_DIR;/tc_pairs/al072022_LSDIAG.tcst From 45dffeb3decd972a4ec90353aab73c4a76f785cc Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Sat, 22 Oct 2022 10:54:13 -0600 Subject: [PATCH 42/58] Per #392, revert TCPairsConfig_ALAL2010 back to what it should be. --- internal/test_unit/config/TCPairsConfig_ALAL2010 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/test_unit/config/TCPairsConfig_ALAL2010 b/internal/test_unit/config/TCPairsConfig_ALAL2010 index cad7e58eae..a0c2b582c7 100644 --- a/internal/test_unit/config/TCPairsConfig_ALAL2010 +++ b/internal/test_unit/config/TCPairsConfig_ALAL2010 @@ -14,7 +14,7 @@ // // Models // -model = [ "GFSO" ]; +model = []; // // Description @@ -24,17 +24,17 @@ desc = "NA"; // // Storm identifiers // -storm_id = []; +storm_id = [ "ALAL2010" ]; // // Basins // -basin = [ "AL" ]; +basin = []; // // Cyclone numbers // -cyclone = [ "09" ]; +cyclone = []; // // Storm names From f4f78bbe77095ee9f081229d409b70f023e57cfd Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Sat, 22 Oct 2022 10:58:13 -0600 Subject: [PATCH 43/58] Per #392, update unit_tc_pairs.xml to attach lsdiag to OFCL and SHIP tracks. --- internal/test_unit/xml/unit_tc_pairs.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/test_unit/xml/unit_tc_pairs.xml b/internal/test_unit/xml/unit_tc_pairs.xml index f1d8a8b9c6..c16543d227 100644 --- a/internal/test_unit/xml/unit_tc_pairs.xml +++ b/internal/test_unit/xml/unit_tc_pairs.xml @@ -213,7 +213,7 @@ \ -adeck &DATA_DIR;/adeck/aal072022_OFCL_SHIP_AVNO.dat \ -bdeck &DATA_DIR;/bdeck/bal072022.dat \ - -lsdiag &DATA_DIR;/lsdiag/22091618AL0722_lsdiag.dat \ + -lsdiag &DATA_DIR;/lsdiag/22091618AL0722_lsdiag.dat model=OFCL,SHIP \ -config &CONFIG_DIR;/TCPairsConfig_DIAGNOSTICS \ -out &OUTPUT_DIR;/tc_pairs/al072022_LSDIAG \ -log &OUTPUT_DIR;/tc_pairs/tc_pairs_LSDIAG.log \ From 3ca7d0a6cece134eb8160e112fb5d7633e49e038 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Tue, 25 Oct 2022 11:11:10 -0600 Subject: [PATCH 44/58] Per #392, revise the DiagFile logic. Call clear() at the beginning of each read function. Improved handling of setting the Technique. --- src/libcode/vx_tc_util/diag_file.cc | 45 ++++++++++++++++++++--------- src/libcode/vx_tc_util/diag_file.h | 3 ++ 2 files changed, 35 insertions(+), 13 deletions(-) diff --git a/src/libcode/vx_tc_util/diag_file.cc b/src/libcode/vx_tc_util/diag_file.cc index e7d501dd99..d98d14125a 100644 --- a/src/libcode/vx_tc_util/diag_file.cc +++ b/src/libcode/vx_tc_util/diag_file.cc @@ -157,6 +157,15 @@ DiagFile & DiagFile::operator=(const DiagFile &) { void DiagFile::init_from_scratch() { + clear(); + + return; +} + +//////////////////////////////////////////////////////////////////////// + +void DiagFile::clear() { + // Initialize values FileType = DiagFileType_None; StormId.clear(); @@ -181,16 +190,19 @@ void DiagFile::init_from_scratch() { void DiagFile::set_technique(const StringArray &sa) { - Technique = sa; + for(int i=0; i " << "the number of \"" << cs << "\" diagnostic values (" << data.n() << ") does not match the expected number (" - << NTime << ")!\n\n"; + << NTime << "): " << path << "\n\n"; exit(1); } // Store the name and values @@ -346,12 +362,15 @@ void DiagFile::read_lsdiag(const ConcatString &path, const StringArray &model_na NumArray data; const UserFunc_1Arg *fx_ptr = 0; + // Initialize + clear(); + // Store the file type FileType = LSDiagFileType; // Store user-specified model names or the default one if(model_names.n() > 0) set_technique(model_names); - else Technique.add(default_lsdiag_technique); + else add_technique(default_lsdiag_technique); // Open the file open(path.c_str()); @@ -368,7 +387,7 @@ void DiagFile::read_lsdiag(const ConcatString &path, const StringArray &model_na // Check for the expected number of items if(dl.n_items() != 9 || strncasecmp(dl[8], "HEAD", strlen("HEAD") != 0)) { mlog << Error << "\nDiagFile::open_lsdiag() -> " - << "unexpected header line in file: " << path << "\n\n"; + << "unexpected header line: " << path << "\n\n"; exit(1); } diff --git a/src/libcode/vx_tc_util/diag_file.h b/src/libcode/vx_tc_util/diag_file.h index d3785f9d69..02b47a9bc0 100644 --- a/src/libcode/vx_tc_util/diag_file.h +++ b/src/libcode/vx_tc_util/diag_file.h @@ -86,11 +86,14 @@ class DiagFile : public LineDataFile { DiagFile(); ~DiagFile(); + void clear(); + // // set stuff // void set_technique(const StringArray &); + void add_technique(const std::string &); // // get stuff From d189124e66b1d3e5b561a264803d959f2ac1321a Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Tue, 25 Oct 2022 11:46:21 -0600 Subject: [PATCH 45/58] Per #392, add unit tests. --- .../config/TCPairsConfig_DIAGNOSTICS | 6 +-- internal/test_unit/xml/unit_tc_pairs.xml | 41 +++++-------------- internal/test_unit/xml/unit_tc_stat.xml | 13 ++++++ 3 files changed, 27 insertions(+), 33 deletions(-) diff --git a/internal/test_unit/config/TCPairsConfig_DIAGNOSTICS b/internal/test_unit/config/TCPairsConfig_DIAGNOSTICS index 2e382dc7cd..61976d2d18 100644 --- a/internal/test_unit/config/TCPairsConfig_DIAGNOSTICS +++ b/internal/test_unit/config/TCPairsConfig_DIAGNOSTICS @@ -44,9 +44,9 @@ storm_name = []; // // Model initialization time windows to include or exclude // -init_beg = ""; -init_end = ""; -init_inc = [ ${INIT_INC} ]; +init_beg = ${INIT_BEG}; +init_end = ${INIT_END}; +init_inc = []; init_exc = []; // diff --git a/internal/test_unit/xml/unit_tc_pairs.xml b/internal/test_unit/xml/unit_tc_pairs.xml index c16543d227..c0c6200a15 100644 --- a/internal/test_unit/xml/unit_tc_pairs.xml +++ b/internal/test_unit/xml/unit_tc_pairs.xml @@ -182,45 +182,26 @@ - + &MET_BIN;/tc_pairs - STORM_ID "AL092021" - INIT_INC "20210827_12" + STORM_ID "AL092022" + INIT_BEG "20220926_00" + INIT_END "20220926_18" DIAG_NAME \ - -adeck &DATA_DIR;/adeck/aal092021_OFCL_SHIP_AVNO.dat \ - -bdeck &DATA_DIR;/bdeck/bal092021.dat \ - -tcdiag &DATA_DIR;/tcdiag/sal092021_avno_doper_2021082712_diag.dat \ + -adeck &DATA_DIR;/adeck/aal092022_OFCL_SHIP_AVNO.dat \ + -bdeck &DATA_DIR;/bdeck/bal092022.dat \ + -tcdiag &DATA_DIR;/tcdiag/2022/sal092022_avno_doper_20220926*_diag.dat \ + -lsdiag &DATA_DIR;/lsdiag/2022/220926*AL0922_lsdiag.dat model=OFCL,SHIP \ -config &CONFIG_DIR;/TCPairsConfig_DIAGNOSTICS \ - -out &OUTPUT_DIR;/tc_pairs/al092021_TCDIAG \ - -log &OUTPUT_DIR;/tc_pairs/tc_pairs_TCDIAG.log \ + -out &OUTPUT_DIR;/tc_pairs/al092022_20220926_DIAGNOSTICS \ + -log &OUTPUT_DIR;/tc_pairs/tc_pairs_DIAGNOSTICS.log \ -v 4 - &OUTPUT_DIR;/tc_pairs/al092021_TCDIAG.tcst - - - - - &MET_BIN;/tc_pairs - - STORM_ID "AL072022" - INIT_INC "20220916_18" - DIAG_NAME - - \ - -adeck &DATA_DIR;/adeck/aal072022_OFCL_SHIP_AVNO.dat \ - -bdeck &DATA_DIR;/bdeck/bal072022.dat \ - -lsdiag &DATA_DIR;/lsdiag/22091618AL0722_lsdiag.dat model=OFCL,SHIP \ - -config &CONFIG_DIR;/TCPairsConfig_DIAGNOSTICS \ - -out &OUTPUT_DIR;/tc_pairs/al072022_LSDIAG \ - -log &OUTPUT_DIR;/tc_pairs/tc_pairs_LSDIAG.log \ - -v 4 - - - &OUTPUT_DIR;/tc_pairs/al072022_LSDIAG.tcst + &OUTPUT_DIR;/tc_pairs/al092022_20220926_DIAGNOSTICS.tcst diff --git a/internal/test_unit/xml/unit_tc_stat.xml b/internal/test_unit/xml/unit_tc_stat.xml index 5029a1f9a8..4fd270718d 100644 --- a/internal/test_unit/xml/unit_tc_stat.xml +++ b/internal/test_unit/xml/unit_tc_stat.xml @@ -92,4 +92,17 @@ + + &MET_BIN;/tc_stat + \ + -lookin &OUTPUT_DIR;/tc_pairs/al092022_20220926_DIAGNOSTICS.tcst \ + -config &CONFIG_DIR;/TCStatConfig_DIAGNOSTICS \ + -out &OUTPUT_DIR;/tc_stat/DIAGNOSTICS_stat.out \ + -v 2 + + + &OUTPUT_DIR;/tc_stat/DIAGNOSTICS_stat.out + + + From 19af6dcc231d0f800a9bc24a3e4fc16c5820ad93 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Tue, 25 Oct 2022 12:51:55 -0600 Subject: [PATCH 46/58] Per #392, adding tc_stat config file for it's unit test. --- .../test_unit/config/TCStatConfig_DIAGNOSTICS | 226 ++++++++++++++++++ 1 file changed, 226 insertions(+) create mode 100644 internal/test_unit/config/TCStatConfig_DIAGNOSTICS diff --git a/internal/test_unit/config/TCStatConfig_DIAGNOSTICS b/internal/test_unit/config/TCStatConfig_DIAGNOSTICS new file mode 100644 index 0000000000..1801bf5ad9 --- /dev/null +++ b/internal/test_unit/config/TCStatConfig_DIAGNOSTICS @@ -0,0 +1,226 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// TC-Stat configuration file. +// +// For additional information, please see the MET User's Guide. +// +//////////////////////////////////////////////////////////////////////////////// + +// +// The parameters listed below are used to filter the TC-STAT data down to the +// desired subset of lines over which statistics are to be computed. Only those +// lines which meet ALL of the criteria specified will be retained. +// +// The settings that are common to all jobs may be specified once at the top +// level. If no selection is listed for a parameter, that parameter will not +// be used for filtering. If multiple selections are listed for a parameter, +// the analyses will be performed on their union. +// +// Each configuration filtering option may be overridden by a corresponding +// job command option of the same name, as described in the MET User's Guide. +// + +// +// Stratify by the AMODEL or BMODEL columns. +// +amodel = []; +bmodel = []; + +// +// Stratify by the DESC column. +// +desc = []; + +// +// Stratify by the STORM_ID column. +// +storm_id = []; + +// +// Stratify by the BASIN column. +// +basin = []; + +// +// Stratify by the CYCLONE column. +// +cyclone = []; + +// +// Stratify by the STORM_NAME column. +// +storm_name = []; + +// +// Stratify by the INIT times. +// Model initialization time windows to include or exclude +// +init_beg = ""; +init_end = ""; +init_inc = []; +init_exc = []; + +// +// Stratify by the VALID times. +// +valid_beg = ""; +valid_end = ""; +valid_inc = []; +valid_exc = []; + +// +// Stratify by the initialization and valid hours and lead time. +// +init_hour = []; +valid_hour = []; +lead = []; + +// +// Select tracks which contain all required lead times. +// +lead_req = []; + +// +// Stratify by the INIT_MASK and VALID_MASK columns. +// +init_mask = []; +valid_mask = []; + +// +// Stratify by the LINE_TYPE column. +// +line_type = []; + +// +// Stratify by checking the watch/warning status for each track point +// common to both the ADECK and BDECK tracks. If the watch/warning status +// of any of the track points appears in the list, retain the entire track. +// +track_watch_warn = []; + +// +// Stratify by applying thresholds to numeric data columns. +// +column_thresh_name = []; +column_thresh_val = []; + +// +// Stratify by performing string matching on non-numeric data columns. +// +column_str_name = []; +column_str_val = []; + +// +// Stratify by excluding strings in non-numeric data columns. +// +column_str_exc_name = []; +column_str_exc_val = []; + +// +// Similar to the column_thresh options above +// +init_thresh_name = []; +init_thresh_val = []; + +// +// Similar to the column_str options above +// +init_str_name = []; +init_str_val = []; + +// +// Similar to the column_str_exc options above +// +init_str_exc_name = []; +init_str_exc_val = []; + +// +// Stratify track points by applying thresholds to numeric +// storm diagnostic values in the TCDIAG lines. +// +diag_thresh_name = []; +diag_thresh_val = []; + +// +// Stratify entire tracks by applying thresholds to numeric +// storm diagnostic values in the TCDIAG lines for lead time 0. +// +init_diag_thresh_name = []; +init_diag_thresh_val = []; + +// +// Stratify by the ADECK and BDECK distances to land. +// +water_only = FALSE; + +// +// Specify whether only those track points for which rapid intensification +// or weakening of the maximum wind speed occurred in the previous time +// step should be retained. +// +rirw = { + track = NONE; + adeck = { + time = "24"; + exact = TRUE; + thresh = >=30.0; + } + bdeck = adeck; +} + +// +// Specify whether only those track points occurring near landfall should be +// retained, and define the landfall retention window in HH[MMSS] format +// around the landfall time. +// +landfall = FALSE; +landfall_beg = "-24"; +landfall_end = "00"; + +// +// Specify whether only those track points common to both the ADECK and BDECK +// tracks should be retained. +// +match_points = TRUE; + +// +// Specify whether only those cases common to all models in the dataset should +// be retained. +// +event_equal = FALSE; + +// +// Specify lead times that must be present for a track to be included in the +// event equalization logic. +// +event_equal_lead = []; + +// +// Apply polyline masking logic to the location of the ADECK track at the +// initialization time. +// +out_init_mask = ""; + +// +// Apply polyline masking logic to the location of the ADECK track at the +// valid time. +// +out_valid_mask = ""; + +// +// Array of TCStat analysis jobs to be performed on the filtered data +// +jobs = [ + "-job filter -dump_row ${MET_TEST_OUTPUT}/tc_stat/DIAGNOSTICS_filter_match_points.tcst", + "-job summary -diag_thresh PW01 lt60 -column TK_ERR -by AMODEL", + "-job summary -diag_thresh PW01 ge60 -column TK_ERR -by AMODEL", + "-job summary -diag_thresh TPW lt60 -column TK_ERR -by AMODEL", + "-job summary -diag_thresh TPW ge60 -column TK_ERR -by AMODEL", + "-job summary -init_diag_thresh DTL lt325 -amodel OFCL -column TK_ERR -by AMODEL,LEAD" +]; + +// +// Indicate a version number for the contents of this configuration file. +// The value should generally not be modified. +// +version = "V11.0.0"; From e063200892581ddf7c6d4c9c208a1c4290d3682c Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Wed, 26 Oct 2022 12:58:14 -0600 Subject: [PATCH 47/58] Per #392, add DiagType enumeration along with utility functions to convert between strings and enumerated values. --- src/basic/vx_config/config_constants.h | 17 +++++++++-- src/basic/vx_config/config_util.cc | 39 ++++++++++++++++++++++++++ src/basic/vx_config/config_util.h | 3 ++ 3 files changed, 57 insertions(+), 2 deletions(-) diff --git a/src/basic/vx_config/config_constants.h b/src/basic/vx_config/config_constants.h index de6943cd58..37c0380b6a 100644 --- a/src/basic/vx_config/config_constants.h +++ b/src/basic/vx_config/config_constants.h @@ -85,6 +85,19 @@ enum TrackType { //////////////////////////////////////////////////////////////////////// +// +// Enumeration for tropical cyclone diagnostic types +// + +enum DiagType { + DiagType_None, // Default + TCDiagType, // Tropical Cyclone Diagnostics + LSDiagRTType, // Realtime Large Scale Diagnostics + LSDiagDevType // Development Large Scale Diagnostics +}; + +//////////////////////////////////////////////////////////////////////// + // // Enumeration for 12-hour interpolation logic // @@ -1067,8 +1080,8 @@ static const char conf_key_basin_file[] = "basin_file"; static const char conf_key_track_watch_warn[] = "track_watch_warn"; static const char conf_key_watch_warn[] = "watch_warn"; static const char conf_key_diag_name[] = "diag_name"; -static const char conf_key_tcdiag_convert_map[] = "tcdiag_convert_map"; -static const char conf_key_lsdiag_convert_map[] = "lsdiag_convert_map"; +static const char conf_key_diag_convert_map[] = "diag_convert_map"; +static const char conf_key_source[] = "source"; static const char conf_key_basin_map[] = "basin_map"; static const char conf_key_time_offset[] = "time_offset"; static const char conf_key_amodel[] = "amodel"; diff --git a/src/basic/vx_config/config_util.cc b/src/basic/vx_config/config_util.cc index 29c8405147..e5747c8239 100644 --- a/src/basic/vx_config/config_util.cc +++ b/src/basic/vx_config/config_util.cc @@ -2740,6 +2740,45 @@ ConcatString tracktype_to_string(TrackType type) { /////////////////////////////////////////////////////////////////////////////// +DiagType string_to_diagtype(const char *s) { + DiagType t = DiagType_None; + + // Convert string to enumerated DiagType + if(strcasecmp(s, conf_val_none) == 0) t = DiagType_None; + else if(strcasecmp(s, "TCDIAG") == 0) t = TCDiagType; + else if(strcasecmp(s, "LSDIAG_RT") == 0) t = LSDiagRTType; + else if(strcasecmp(s, "LSDIAG_DEV") == 0) t = LSDiagDevType; + else { + mlog << Error << "\nstring_to_diagtype() -> " + << "Unexpected DiagType string \"" << s << "\".\n\n"; + exit(1); + } + + return(t); +} + +/////////////////////////////////////////////////////////////////////////////// + +ConcatString diagtype_to_string(DiagType type) { + ConcatString s; + + // Convert enumerated DiagType to string + switch(type) { + case(DiagType_None): s = conf_val_none; break; + case(TCDiagType): s = "TCDIAG"; break; + case(LSDiagRTType): s = "LSDIAG_RT"; break; + case(LSDiagDevType): s = "LSDIAG_DEV"; break; + default: + mlog << Error << "\ndiagtype_to_string() -> " + << "Unexpected DiagType value of " << type << ".\n\n"; + exit(1); + } + + return(s); +} + +/////////////////////////////////////////////////////////////////////////////// + Interp12Type int_to_interp12type(int v) { Interp12Type t = Interp12Type_None; diff --git a/src/basic/vx_config/config_util.h b/src/basic/vx_config/config_util.h index 63e84a685e..02c712e715 100644 --- a/src/basic/vx_config/config_util.h +++ b/src/basic/vx_config/config_util.h @@ -111,6 +111,9 @@ extern TrackType int_to_tracktype(int); extern TrackType string_to_tracktype(const char *); extern ConcatString tracktype_to_string(TrackType); +extern DiagType string_to_diagtype(const char *); +extern ConcatString diagtype_to_string(DiagType); + extern Interp12Type int_to_interp12type(int); extern Interp12Type string_to_interp12type(const char *); extern ConcatString interp12type_to_string(Interp12Type); From 027380132d7e2ff7f4149bdb649ee4ecca809681 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Wed, 26 Oct 2022 12:59:10 -0600 Subject: [PATCH 48/58] Per #392, enhance the DiagFile class to make use of the DiagType enumeration. --- src/libcode/vx_tc_util/diag_file.cc | 54 +++++++++++++++++++++++------ src/libcode/vx_tc_util/diag_file.h | 22 +++++------- 2 files changed, 53 insertions(+), 23 deletions(-) diff --git a/src/libcode/vx_tc_util/diag_file.cc b/src/libcode/vx_tc_util/diag_file.cc index d98d14125a..775083d9e5 100644 --- a/src/libcode/vx_tc_util/diag_file.cc +++ b/src/libcode/vx_tc_util/diag_file.cc @@ -167,7 +167,7 @@ void DiagFile::init_from_scratch() { void DiagFile::clear() { // Initialize values - FileType = DiagFileType_None; + SourceType = DiagType_None; StormId.clear(); Basin.clear(); Cyclone.clear(); @@ -209,8 +209,31 @@ void DiagFile::add_technique(const string &str) { //////////////////////////////////////////////////////////////////////// +void DiagFile::read(const DiagType source, + const ConcatString &path, const StringArray &model_names, + const std::map *convert_map) { + + // Read diagnostics baesd on the source type + if(source == TCDiagType) { + read_tcdiag(path, model_names, convert_map); + } + else if(source == LSDiagRTType) { + read_lsdiagrt(path, model_names, convert_map); + } + else { + mlog << Error << "\nDiagFile::read() -> " + << "diagnostics of type " << diagtype_to_string(source) + << " are not currently supported!\n\n"; + exit(1); + } + + return; +} + +//////////////////////////////////////////////////////////////////////// + void DiagFile::read_tcdiag(const ConcatString &path, const StringArray &model_names, - const map &convert_map) { + const map *convert_map) { int i, v_int; double v_dbl; NumArray data; @@ -220,7 +243,7 @@ void DiagFile::read_tcdiag(const ConcatString &path, const StringArray &model_na clear(); // Store the file type - FileType = TCDiagFileType; + SourceType = TCDiagType; // Store user-specified model names or read it from the file below if(model_names.n() > 0) set_technique(model_names); @@ -309,9 +332,14 @@ void DiagFile::read_tcdiag(const ConcatString &path, const StringArray &model_na cs.startswith("NLEV") || cs.startswith("NVAR")) continue; // Check for a conversion function based on the diagnostic name or units - if(convert_map.count(cs) > 0) fx_ptr = &convert_map.at(cs); - else if(convert_map.count(dl[1]) > 0) fx_ptr = &convert_map.at(dl[1]); - else fx_ptr = (UserFunc_1Arg *) 0; + if(convert_map) { + if(convert_map->count(cs) > 0) fx_ptr = &convert_map->at(cs); + else if(convert_map->count(dl[1]) > 0) fx_ptr = &convert_map->at(dl[1]); + else fx_ptr = (UserFunc_1Arg *) 0; + } + else { + fx_ptr = (UserFunc_1Arg *) 0; + } // Parse the data values data.erase(); @@ -355,8 +383,8 @@ void DiagFile::read_tcdiag(const ConcatString &path, const StringArray &model_na //////////////////////////////////////////////////////////////////////// -void DiagFile::read_lsdiag(const ConcatString &path, const StringArray &model_names, - const map &convert_map) { +void DiagFile::read_lsdiagrt(const ConcatString &path, const StringArray &model_names, + const map *convert_map) { int i, v_int; double v_dbl; NumArray data; @@ -366,7 +394,7 @@ void DiagFile::read_lsdiag(const ConcatString &path, const StringArray &model_na clear(); // Store the file type - FileType = LSDiagFileType; + SourceType = LSDiagRTType; // Store user-specified model names or the default one if(model_names.n() > 0) set_technique(model_names); @@ -455,7 +483,13 @@ void DiagFile::read_lsdiag(const ConcatString &path, const StringArray &model_na if(cs == "LAST") break; // Check for a conversion function - fx_ptr = (convert_map.count(cs) > 0 ? &convert_map.at(cs) : (UserFunc_1Arg *) 0); + if(convert_map) { + if(convert_map->count(cs) > 0) fx_ptr = &convert_map->at(cs); + else fx_ptr = (UserFunc_1Arg *) 0; + } + else { + fx_ptr = (UserFunc_1Arg *) 0; + } // Parse the data values data.erase(); diff --git a/src/libcode/vx_tc_util/diag_file.h b/src/libcode/vx_tc_util/diag_file.h index 02b47a9bc0..43e6efb385 100644 --- a/src/libcode/vx_tc_util/diag_file.h +++ b/src/libcode/vx_tc_util/diag_file.h @@ -17,17 +17,10 @@ #include #include +#include "vx_config.h" #include "vx_cal.h" #include "data_line.h" -//////////////////////////////////////////////////////////////////////// - -enum DiagFileType { - DiagFileType_None, // Default - LSDiagFileType, // Large Scale Diagnostics - TCDiagFileType // Tropical Cyclone Diagnostics -}; - //////////////////////////////////////////////////////////////////////// // // LSDIAG files: @@ -63,7 +56,7 @@ class DiagFile : public LineDataFile { void init_from_scratch(); // Diagnostics file type - DiagFileType FileType; + DiagType SourceType; // Storm and model identification ConcatString StormId; @@ -121,10 +114,13 @@ class DiagFile : public LineDataFile { // do stuff // - void read_tcdiag(const ConcatString &, const StringArray &, - const std::map &); - void read_lsdiag(const ConcatString &, const StringArray &, - const std::map &); + void read (const DiagType, + const ConcatString &, const StringArray &, + const std::map *); + void read_tcdiag (const ConcatString &, const StringArray &, + const std::map *); + void read_lsdiagrt (const ConcatString &, const StringArray &, + const std::map *); }; From 05c1e3940080aa56a3d514ddf89be8562835e6a1 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Wed, 26 Oct 2022 13:00:50 -0600 Subject: [PATCH 49/58] Per #392, large changes to TC-Pairs, replacing lsdiag_convert_map and tcdiag_convert_map with a single diag_convert_map. --- data/config/TCPairsConfig_default | 60 +--- .../config/TCPairsConfig_DIAGNOSTICS | 6 +- src/tools/tc_utils/tc_pairs/tc_pairs.cc | 268 ++++++++---------- src/tools/tc_utils/tc_pairs/tc_pairs.h | 10 +- .../tc_utils/tc_pairs/tc_pairs_conf_info.cc | 85 +++++- .../tc_utils/tc_pairs/tc_pairs_conf_info.h | 5 +- 6 files changed, 224 insertions(+), 210 deletions(-) diff --git a/data/config/TCPairsConfig_default b/data/config/TCPairsConfig_default index 9fac140fad..733b6c574a 100644 --- a/data/config/TCPairsConfig_default +++ b/data/config/TCPairsConfig_default @@ -146,61 +146,25 @@ diag_name = []; // Unit conversions to be applied to diagnostic values // -tcdiag_convert_map = [ - { key = [ "(10C)", "(10KT)", "(10M/S)" ]; convert(x) = x / 10; } -]; +diag_convert_map = [ + { source = "TCDIAG"; + key = [ "(10C)", "(10KT)", "(10M/S)" ]; + convert(x) = x / 10; }, -lsdiag_convert_map = [ - { key = [ "CSST", "RSST", "DSST", "DSTA", "XDST", "XNST", "NSST", "NSTA", + { source = "LSDIAG_RT"; + key = [ "CSST", "RSST", "DSST", "DSTA", "XDST", "XNST", "NSST", "NSTA", "NTMX", "NTFR", "U200", "U20C", "V20C", "E000", "EPOS", "ENEG", "EPSS", "ENSS", "T000", "TLAT", "TLON", "TWAC", "TWXC", "G150", "G200", "G250", "V000", "V850", "V500", "V300", "PENC", "SHDC", "SHGC", "T150", "T200", "T250", "SHRD", "SHRS", "SHRG", "PENV", "HE07", "HE05", "PW01", "PW02", "PW03", "PW04", "PW05", "PW06", "PW07", "PW08", "PW09", "PW10", "PW11", "PW12", "PW13", "PW14", "PW15", "PW16", - "PW17", "PW18", "PW20", "PW21" ]; convert(x) = x / 10; }, - { key = [ "VVAV", "VMFX", "VVAC" ]; convert(x) = x / 100; } -]; + "PW17", "PW18", "PW20", "PW21" ]; + convert(x) = x / 10; }, -// -// LSDiagnostics -// https://rammb.cira.colostate.edu/research/tropical_cyclones/ships/data/ships_predictor_file_2022.pdf -// Likley typo: NDMX should acutally be NDTX in there -// -// All diagnostics from sample file: -// "TIME", "DELV", "LAT", "LON", "CSST", "DTL", "RSST", "DSST", "DSTA", "CD20", -// "CD26", "COHC", "XDST", "XNST", "XOHC", "NSST", "NSTA", "NTMX", "NDTX", "NDML", -// "ND30", "ND28", "ND26", "ND24", "ND22", "ND20", "ND18", "ND16", "NDFR", "NTFR", -// "NOHC", "NO20", "IR00", "IRM1", "IRM3", "PC00", "PCM1", "PCM3", "U200", "U20C", -// "V20C", "E000", "EPOS", "ENEG", "EPSS", "ENSS", "RHLO", "RHMD", "RHHI", "PSLV", -// "Z850", "D200", "REFC", "PEFC", "T000", "R000", "Z000", "TLAT", "TLON", "TWAC", -// "TWXC", "G150", "G200", "G250", "V000", "V850", "V500", "V300", "TGRD", "TADV", -// "PENC", "SHDC", "SDDC", "SHGC", "DIVC", "T150", "T200", "T250", "SHRD", "SHTD", -// "SHRS", "SHTS", "SHRG", "PENV", "VMPI", "VVAV", "VMFX", "VVAC", "HE07", "HE05", -// "O500", "O700", "CFLX", "MTPW", "PW01", "PW02", "PW03", "PW04", "PW05", "PW06", -// "PW07", "PW08", "PW09", "PW10", "PW11", "PW12", "PW13", "PW14", "PW15", "PW16", -// "PW17", "PW18", "PW19", "PW20", "PW21", "IRXX" -// -// No conversion required: -// "TIME", "DELV", "DTL", "CD20", "CD26", "COHC", "XOHC", "NDTX", "NDML", "ND30", -// "ND28", "ND26", "ND24", "ND22", "ND20", "ND18", "ND16", "NDFR", "NOHC", "NO20", -// "RHLO", "RHMD", "RHHI", "Z850", "D200", "REFC", "PEFC", "R000", "Z000", "TGRD", -// "TADV", "SDDC", "DIVC", "SHTD", "SHTS", "VMPI", "O500", "O700", "CFLX", "PW19" -// -// Divide by 10: -// "LAT", "LON", "CSST", "RSST", "DSST", "DSTA", "XDST", "XNST", "NSST", "NSTA", -// "NTMX", "NTFR", "U200", "U20C", "V20C", "E000", "EPOS", "ENEG", "EPSS", "ENSS", -// "T000", "TLAT", "TLON", "TWAC", "TWXC", "G150", "G200", "G250", "V000", "V850", -// "V500", "V300", "PENC", "SHDC", "SHGC", "T150", "T200", "T250", "SHRD", "SHRS", -// "SHRG", "PENV", "HE07", "HE05", "PW01", "PW02", "PW03", "PW04", "PW05", "PW06", -// "PW07", "PW08", "PW09", "PW10", "PW11", "PW12", "PW13", "PW14", "PW15", "PW16", -// "PW17", "PW18", "PW20", "PW21" -// -// Divide by 100: -// "VVAV", "VMFX", "VVAC" -// -// Ignore for now but could add special logic: -// "MTPW" - Because the same data is provided in PW01-PW21 -// "IR00", "IRM1", "IRM3", "PC00", "PCM1", "PCM3", "PSLV", "IRXX" + { source = "LSDIAG_RT"; + key = [ "VVAV", "VMFX", "VVAC" ]; + convert(x) = x / 100; } +]; // // Modify basin names to make them consistent across ATCF input files. diff --git a/internal/test_unit/config/TCPairsConfig_DIAGNOSTICS b/internal/test_unit/config/TCPairsConfig_DIAGNOSTICS index 61976d2d18..7ef647a90d 100644 --- a/internal/test_unit/config/TCPairsConfig_DIAGNOSTICS +++ b/internal/test_unit/config/TCPairsConfig_DIAGNOSTICS @@ -135,11 +135,9 @@ diag_name = [ ${DIAG_NAME} ]; // // Unit conversions to be applied to diagnostic values -// - // Commented out to use settings from the default config file -// tcdiag_convert_map = []; -// lsdiag_convert_map = []; +// +// diag_convert_map = []; // // Modify basin names to make them consistent across ATCF input files. diff --git a/src/tools/tc_utils/tc_pairs/tc_pairs.cc b/src/tools/tc_utils/tc_pairs/tc_pairs.cc index 84583c728b..1861e16eb0 100644 --- a/src/tools/tc_utils/tc_pairs/tc_pairs.cc +++ b/src/tools/tc_utils/tc_pairs/tc_pairs.cc @@ -125,9 +125,6 @@ static bool is_keeper (const ATCFLineBase *); static void filter_tracks (TrackInfoArray &); static void filter_probs (ProbInfoArray &); static void process_diags (TrackInfoArray &); -static void get_diag_files (const StringArray &, - const StringArray &, - StringArray &, StringArray &); static bool check_masks (const MaskPoly &, const Grid &, const MaskPlane &, @@ -153,12 +150,9 @@ static void usage (); static void set_adeck (const StringArray &); static void set_edeck (const StringArray &); static void set_bdeck (const StringArray &); -static void set_atcf_source (const StringArray &, - StringArray &, StringArray &); -static void set_tcdiag (const StringArray &); -static void set_lsdiag (const StringArray &); -static void set_diag_source (const StringArray &, +static void set_atcf_path (const StringArray &, StringArray &, StringArray &); +static void set_diag (const StringArray &); static void set_config (const StringArray &); static void set_out (const StringArray &); @@ -207,8 +201,7 @@ void process_command_line(int argc, char **argv) { cline.add(set_adeck, "-adeck", -1); cline.add(set_edeck, "-edeck", -1); cline.add(set_bdeck, "-bdeck", -1); - cline.add(set_tcdiag, "-tcdiag", -1); - cline.add(set_lsdiag, "-lsdiag", -1); + cline.add(set_diag, "-diag", -1); cline.add(set_config, "-config", 1); cline.add(set_out, "-out", 1); @@ -216,11 +209,10 @@ void process_command_line(int argc, char **argv) { cline.parse(); // Check for the minimum number of arguments - if((adeck_source.n() == 0 && edeck_source.n() == 0) || - bdeck_source.n() == 0 || config_file.length() == 0) { - mlog << Error - << "\nprocess_command_line(int argc, char **argv) -> " - << "You must specify at least one source of ADECK or EDECK " + if((adeck_path.n() == 0 && edeck_path.n() == 0) || + bdeck_path.n() == 0 || config_file.length() == 0) { + mlog << Error << "\nprocess_command_line(int argc, char **argv) -> " + << "You must specify at least one path of ADECK or EDECK " << "data, BDECK data, and the config file using the " << "\"-adeck\", \"-edeck\", \"-bdeck\", and \"-config\" " << "command line options.\n\n"; @@ -228,43 +220,36 @@ void process_command_line(int argc, char **argv) { } // List the input ADECK track files - for(i=0; i 0) { + if(adeck_path.n() > 0) { process_adecks(bdeck_tracks); } // Process EDECK files - if(edeck_source.n() > 0) { + if(edeck_path.n() > 0) { process_edecks(bdeck_tracks); } @@ -316,7 +301,7 @@ void process_bdecks(TrackInfoArray &bdeck_tracks) { bdeck_tracks.clear(); // Get the list of track files - get_atcf_files(bdeck_source, bdeck_model_suffix, + get_atcf_files(bdeck_path, bdeck_model_suffix, files, files_model_suffix); mlog << Debug(2) @@ -339,7 +324,7 @@ void process_adecks(const TrackInfoArray &bdeck_tracks) { int i, j, n_match; // Get the list of track files - get_atcf_files(adeck_source, adeck_model_suffix, + get_atcf_files(adeck_path, adeck_model_suffix, files, files_model_suffix); mlog << Debug(2) @@ -459,7 +444,7 @@ void process_edecks(const TrackInfoArray &bdeck_tracks) { int n_match, i, j; // Get the list of ATCF files - get_atcf_files(edeck_source, edeck_model_suffix, + get_atcf_files(edeck_path, edeck_model_suffix, files, files_model_suffix); mlog << Debug(2) @@ -539,17 +524,16 @@ void process_edecks(const TrackInfoArray &bdeck_tracks) { //////////////////////////////////////////////////////////////////////// -void get_atcf_files(const StringArray &source, +void get_atcf_files(const StringArray &path, const StringArray &model_suffix, StringArray &files, StringArray &files_model_suffix) { - StringArray cur_source, cur_files; + StringArray cur_path, cur_files; int i, j; - if(source.n() != model_suffix.n()) { - mlog << Error - << "\nget_atcf_files() -> " - << "the source and suffix arrays must be equal length!\n\n"; + if(path.n() != model_suffix.n()) { + mlog << Error << "\nget_atcf_files() -> " + << "the path and suffix arrays must be equal length!\n\n"; exit(1); } @@ -558,10 +542,10 @@ void get_atcf_files(const StringArray &source, files_model_suffix.clear(); // Build list of files and corresponding model suffix list - for(i=0; i " + mlog << Error << "\nprocess_track_files() -> " << "unable to open file \"" << files[i] << "\"\n\n"; exit(1); } @@ -693,8 +676,7 @@ void process_prob_files(const StringArray &files, // Open the current file if(!f.open(files[i].c_str())) { - mlog << Error - << "\nprocess_prob_files() -> " + mlog << Error << "\nprocess_prob_files() -> " << "unable to open file \"" << files[i] << "\"\n\n"; exit(1); } @@ -1025,52 +1007,62 @@ void filter_probs(ProbInfoArray &probs) { //////////////////////////////////////////////////////////////////////// void process_diags(TrackInfoArray &tracks) { + StringArray cur_path, cur_model_name; StringArray files, files_model_name, model_names; DiagFile diag_file; - int i, n; + int i, j, n; + map n_diag_map; + + // Process the diagnostic inputs + for(i=0; i 0) { - get_atcf_files(tcdiag_source, tcdiag_model_name, + // Get the list of diagnostic files for this source + get_atcf_files(cur_path, cur_model_name, files, files_model_name); mlog << Debug(2) << "Processing " << files.n() - << " TCDIAG diagnostic file(s).\n"; + << " " << diagtype_to_string(diag_source[i]) + << " diagnostic file(s).\n"; // Loop over the input files - for(i=0,n=0; i 0) { - get_atcf_files(lsdiag_source, lsdiag_model_name, - files, files_model_name); + // Pointer to the conversion map for this source + map * convert_ptr = 0; + if(conf_info.DiagConvertMap.count(diag_source[i]) > 0) { + convert_ptr = &conf_info.DiagConvertMap.at(diag_source[i]); + } - mlog << Debug(2) - << "Processing " << files.n() - << " LSDIAG diagnostic file(s).\n"; + // Process the diagnostic files + diag_file.read(diag_source[i], files[j], model_names, convert_ptr); - // Loop over the input files - for(i=0,n=0; i 0) n_diag_map[diag_source[i]] += n; + else n_diag_map[diag_source[i]] = n; + + } // end for i + + // Print the diagnostic track counts + for(map::iterator it = n_diag_map.begin(); + it != n_diag_map.end(); it++) { mlog << Debug(3) - << "Added diagnostics information to " << n - << " tracks.\n"; + << "Added " << diagtype_to_string(it->first) + << " diagnostics to " << it->second << " tracks.\n"; } return; @@ -1534,8 +1526,7 @@ void derive_baseline_model(const ConcatString &model, // Check bounds if(i_start < 0 || i_start >= ti.n_points()) { - mlog << Error - << "\n" << method_name + mlog << Error << "\n" << method_name << "range check error for i_start = " << i_start << "\n\n"; exit(1); } @@ -1651,8 +1642,7 @@ void derive_baseline_model(const ConcatString &model, bl_lat, bl_lon, bl_vmax); } else { - mlog << Error - << "\n" << method_name + mlog << Error << "\n" << method_name << "unsupported baseline model type \"" << model << "\".\n\n"; exit(1); @@ -1894,8 +1884,7 @@ void compute_track_err(const TrackInfo &adeck, const TrackInfo &bdeck, // Check for too many track points if(n_ut > mxp) { - mlog << Error - << "\ncompute_track_err() -> " + mlog << Error << "\ncompute_track_err() -> " << "exceeded the maximum number of allowable track points (" << n_ut << " > " << mxp << ")\n\n"; exit(1); @@ -2214,26 +2203,25 @@ void usage() { << ") ***\n\n" << "Usage: " << program_name << "\n" - << "\t-adeck source and/or -edeck source\n" - << "\t-bdeck source\n" + << "\t-adeck path and/or -edeck path\n" + << "\t-bdeck path\n" << "\t-config file\n" - << "\t[-tcdiag source]\n" - << "\t[-lsdiag source]\n" + << "\t[-diag source path]\n" << "\t[-out base]\n" << "\t[-log file]\n" << "\t[-v level]\n\n" - << "\twhere\t\"-adeck source\" is used one or more times to " + << "\twhere\t\"-adeck path\" is used one or more times to " << "specify a file or top-level directory containing ATCF " << "model output \"" << atcf_suffix << "\" data to process (required if no -edeck).\n" - << "\t\t\"-edeck source\" is used one or more times to " + << "\t\t\"-edeck path\" is used one or more times to " << "specify a file or top-level directory containing ATCF " << "ensemble model output \"" << atcf_suffix << "\" data to process (required if no -adeck).\n" - << "\t\t\"-bdeck source\" is used one or more times to " + << "\t\t\"-bdeck path\" is used one or more times to " << "specify a file or top-level directory containing ATCF " << "best track \"" << atcf_suffix << "\" data to process (required).\n" @@ -2242,15 +2230,11 @@ void usage() { << "TCPairsConfig file containing the desired configuration " << "settings (required).\n" - << "\t\t\"-tcdiag source\" is used one or more times to " + << "\t\t\"-diag source path\" is used one or more times to " << "specify a file or top-level directory containing tropical " << "cyclone diagnostics \"" << atcf_suffix - << "\" data to process (optional).\n" - - << "\t\t\"-lsdiag source\" is used one or more times to " - << "specify a file or top-level directory containing large " - << "scale diagnostics \"" << atcf_suffix - << "\" data to process (optional).\n" + << "\" data to process. The supported formats are TCDIAG, " + << "LSDIAG_RT, LSDIAG_DEV (optional).\n" << "\t\t\"-out base\" overrides the default output file base " << "(" << out_base << ") (optional).\n" @@ -2263,11 +2247,11 @@ void usage() { << "\tNote: The \"-adeck\", \"-edeck\", and \"-bdeck\" options " << "may include \"suffix=string\" to modify the model names " - << "from that source.\n\n" + << "from that path.\n\n" - << "\tNote: The \"-tcdiag\" and \"-lsdiag\" options may include " - << "\"model=string\" to override the model name of the tracks to " - << "which those diagnostics correspond.\n\n"; + << "\tNote: The \"-diag\" option may include \"model=string\" to " + << "override the model name of the tracks to which those diagnostics " + << "correspond.\n\n"; exit(1); } @@ -2275,25 +2259,25 @@ void usage() { //////////////////////////////////////////////////////////////////////// void set_adeck(const StringArray &a) { - set_atcf_source(a, adeck_source, adeck_model_suffix); + set_atcf_path(a, adeck_path, adeck_model_suffix); } //////////////////////////////////////////////////////////////////////// void set_edeck(const StringArray &a) { - set_atcf_source(a, edeck_source, edeck_model_suffix); + set_atcf_path(a, edeck_path, edeck_model_suffix); } //////////////////////////////////////////////////////////////////////// void set_bdeck(const StringArray &a) { - set_atcf_source(a, bdeck_source, bdeck_model_suffix); + set_atcf_path(a, bdeck_path, bdeck_model_suffix); } //////////////////////////////////////////////////////////////////////// -void set_atcf_source(const StringArray &a, - StringArray &source, StringArray &model_suffix) { +void set_atcf_path(const StringArray &a, + StringArray &path, StringArray &model_suffix) { int i; StringArray sa; ConcatString cs, suffix; @@ -2304,8 +2288,7 @@ void set_atcf_source(const StringArray &a, if(cs.startswith("suffix")) { sa = cs.split("="); if(sa.n() != 2) { - mlog << Error - << "\nset_atcf_source() -> " + mlog << Error << "\nset_atcf_path() -> " << "the model suffix must be specified as " << "\"suffix=string\".\n\n"; usage(); @@ -2316,11 +2299,11 @@ void set_atcf_source(const StringArray &a, } } - // Parse the remaining sources + // Parse the remaining paths for(i=0; i " + << "the \"-diag source path\" command line option " + << "must have length >= 2.\n\n"; + exit(1); + } - // Check for optional model sub-argument - for(i=0; i " + mlog << Error << "\nset_diag_path() -> " << "the model name must be specified as " << "\"model=string\".\n\n"; usage(); } else { - name = sa[1]; + model_name = sa[1]; } } } - // Parse the remaining sources - for(i=0; i diag_source; +static StringArray diag_path, diag_model_name; static ConcatString config_file; static TCPairsConfInfo conf_info; diff --git a/src/tools/tc_utils/tc_pairs/tc_pairs_conf_info.cc b/src/tools/tc_utils/tc_pairs/tc_pairs_conf_info.cc index 96fddfad9a..30fab41cc9 100644 --- a/src/tools/tc_utils/tc_pairs/tc_pairs_conf_info.cc +++ b/src/tools/tc_utils/tc_pairs/tc_pairs_conf_info.cc @@ -23,6 +23,11 @@ using namespace std; #include "apply_mask.h" #include "vx_log.h" +//////////////////////////////////////////////////////////////////////// + +void parse_conf_diag_convert_map(Dictionary *, + map> &); + //////////////////////////////////////////////////////////////////////// // // Code for class TCPairsConfInfo @@ -97,8 +102,7 @@ void TCPairsConfInfo::clear() { WatchWarnFile.clear(); WatchWarnOffset = bad_data_int; DiagName.clear(); - TCDiagConvertFxMap.clear(); - LSDiagConvertFxMap.clear(); + DiagConvertMap.clear(); BasinMap.clear(); Version.clear(); @@ -311,11 +315,8 @@ void TCPairsConfInfo::process_config() { // Conf: DiagName DiagName = dict->lookup_string_array(conf_key_diag_name); - // Conf: TCDiagConvertFxMap - TCDiagConvertFxMap = parse_conf_key_convert_map(dict, conf_key_tcdiag_convert_map); - - // Conf: LSDiagConvertFxMap - LSDiagConvertFxMap = parse_conf_key_convert_map(dict, conf_key_lsdiag_convert_map); + // Conf: DiagConvertMap + parse_conf_diag_convert_map(dict, DiagConvertMap); // Conf: BasinMap BasinMap = parse_conf_key_value_map(dict, conf_key_basin_map); @@ -324,3 +325,73 @@ void TCPairsConfInfo::process_config() { } //////////////////////////////////////////////////////////////////////// +// +// Utility functions +// +//////////////////////////////////////////////////////////////////////// + +void parse_conf_diag_convert_map(Dictionary *dict, + map> &source_map) { + int i, j; + Dictionary *map_dict = (Dictionary *) 0; + map cur_map; + DiagType source; + StringArray sa; + ConcatString key; + UserFunc_1Arg fx; + + const char *method_name = "parse_conf_diag_convert_map() -> "; + + if(!dict) { + mlog << Error << "\n" << method_name + << "empty dictionary!\n\n"; + exit(1); + } + + // Conf: diag_convert_map + map_dict = dict->lookup_array(conf_key_diag_convert_map); + + // Loop through the array entries + for(i=0; in_entries(); i++) { + + // Initialize the current map + cur_map.clear(); + + // Lookup the source, key, and convert function + source = string_to_diagtype( + (*map_dict)[i]->dict_value()->lookup_string(conf_key_source).c_str()); + sa = (*map_dict)[i]->dict_value()->lookup_string_array(conf_key_key); + fx.clear(); + fx.set((*map_dict)[i]->dict_value()->lookup(conf_key_convert)); + + // Check the function + if(!fx.is_set()) { + mlog << Error << "\n" << method_name + << "lookup for \"" << conf_key_convert << "\" failed in the \"" + << conf_key_diag_convert_map << "\" map!\n\n"; + exit(1); + } + + // Add entry to the current map for each string + for(j=0; j(sa[j],fx)); + } + + // Append to the existing source entry + if(source_map.count(source) > 0) { + for(map::iterator it = cur_map.begin(); + it != cur_map.end(); it++) { + source_map.at(source).insert(pair(it->first, it->second)); + } + } + // Add a new source entry + else { + source_map.insert(pair>(source, cur_map)); + } + + } // end for i + + return; +} + +//////////////////////////////////////////////////////////////////////// diff --git a/src/tools/tc_utils/tc_pairs/tc_pairs_conf_info.h b/src/tools/tc_utils/tc_pairs/tc_pairs_conf_info.h index 080d0b9647..8c2583b51a 100644 --- a/src/tools/tc_utils/tc_pairs/tc_pairs_conf_info.h +++ b/src/tools/tc_utils/tc_pairs/tc_pairs_conf_info.h @@ -114,9 +114,8 @@ class TCPairsConfInfo { // Diagnostics to be extracted StringArray DiagName; - // Diagnostic conversion maps - std::map TCDiagConvertFxMap; - std::map LSDiagConvertFxMap; + // Diagnostic conversions + std::map> DiagConvertMap; // Basin Map std::map BasinMap; From b618f922b381a6109036ed61086e46050bca7f47 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Wed, 26 Oct 2022 13:01:20 -0600 Subject: [PATCH 50/58] Per #392, update unit_tc_pairs.xml to use the single -diag command line option instead of -lsdiag and -tcdiag. --- internal/test_unit/xml/unit_tc_pairs.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/test_unit/xml/unit_tc_pairs.xml b/internal/test_unit/xml/unit_tc_pairs.xml index c0c6200a15..d12f327f09 100644 --- a/internal/test_unit/xml/unit_tc_pairs.xml +++ b/internal/test_unit/xml/unit_tc_pairs.xml @@ -193,8 +193,8 @@ \ -adeck &DATA_DIR;/adeck/aal092022_OFCL_SHIP_AVNO.dat \ -bdeck &DATA_DIR;/bdeck/bal092022.dat \ - -tcdiag &DATA_DIR;/tcdiag/2022/sal092022_avno_doper_20220926*_diag.dat \ - -lsdiag &DATA_DIR;/lsdiag/2022/220926*AL0922_lsdiag.dat model=OFCL,SHIP \ + -diag TCDIAG &DATA_DIR;/tcdiag/2022/sal092022_avno_doper_20220926*_diag.dat \ + -diag LSDIAG_RT &DATA_DIR;/lsdiag/2022/220926*AL0922_lsdiag.dat model=OFCL,SHIP \ -config &CONFIG_DIR;/TCPairsConfig_DIAGNOSTICS \ -out &OUTPUT_DIR;/tc_pairs/al092022_20220926_DIAGNOSTICS \ -log &OUTPUT_DIR;/tc_pairs/tc_pairs_DIAGNOSTICS.log \ From 10cc706afe7f6ddceff2b7414d4a2a8a78186c9a Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Wed, 26 Oct 2022 13:12:14 -0600 Subject: [PATCH 51/58] Per #392, update TrackInfo to store the DiagType source. --- src/libcode/vx_tc_util/diag_file.h | 2 ++ src/libcode/vx_tc_util/track_info.cc | 7 +++++++ src/libcode/vx_tc_util/track_info.h | 16 ++++++++++++---- 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/src/libcode/vx_tc_util/diag_file.h b/src/libcode/vx_tc_util/diag_file.h index 43e6efb385..5490a9f8fe 100644 --- a/src/libcode/vx_tc_util/diag_file.h +++ b/src/libcode/vx_tc_util/diag_file.h @@ -105,6 +105,7 @@ class DiagFile : public LineDataFile { double lat(int) const; double lon(int) const; + DiagType source() const; int n_diag() const; const StringArray & diag_name() const; bool has_diag(const std::string &) const; @@ -132,6 +133,7 @@ inline const ConcatString & DiagFile::cyclone() const { return(Cyclone); inline const StringArray & DiagFile::technique() const { return(Technique); } inline unixtime DiagFile::init() const { return(InitTime); } inline int DiagFile::n_time() const { return(NTime); } +inline DiagType DiagFile::source() const { return(SourceType); } inline int DiagFile::n_diag() const { return(DiagName.n()); } inline const StringArray & DiagFile::diag_name() const { return(DiagName); } diff --git a/src/libcode/vx_tc_util/track_info.cc b/src/libcode/vx_tc_util/track_info.cc index 65c6d49289..3b53f150c9 100644 --- a/src/libcode/vx_tc_util/track_info.cc +++ b/src/libcode/vx_tc_util/track_info.cc @@ -91,6 +91,7 @@ void TrackInfo::clear() { MaxValidTime = (unixtime) 0; MinWarmCore = (unixtime) 0; MaxWarmCore = (unixtime) 0; + DiagSource = DiagType_None; DiagName.clear(); TrackLines.clear(); @@ -131,6 +132,7 @@ void TrackInfo::dump(ostream &out, int indent_depth) const { out << prefix << "MaxValidTime = \"" << (MaxValidTime > 0 ? unix_to_yyyymmdd_hhmmss(MaxValidTime).text() : na_str) << "\n"; out << prefix << "MinWarmCore = \"" << (MinWarmCore > 0 ? unix_to_yyyymmdd_hhmmss(MinWarmCore).text() : na_str) << "\n"; out << prefix << "MaxWarmCore = \"" << (MaxWarmCore > 0 ? unix_to_yyyymmdd_hhmmss(MaxWarmCore).text() : na_str) << "\n"; + out << prefix << "DiagSource = " << diagtype_to_string(DiagSource) << "\n"; out << prefix << "NDiag = " << DiagName.n() << "\n"; out << prefix << "NPoints = " << NPoints << "\n"; out << prefix << "NAlloc = " << NAlloc << "\n"; @@ -169,6 +171,7 @@ ConcatString TrackInfo::serialize() const { << ", MaxValidTime = " << (MaxValidTime > 0 ? unix_to_yyyymmdd_hhmmss(MaxValidTime).text() : na_str) << ", MinWarmCore = " << (MinWarmCore > 0 ? unix_to_yyyymmdd_hhmmss(MinWarmCore).text() : na_str) << ", MaxWarmCore = " << (MaxWarmCore > 0 ? unix_to_yyyymmdd_hhmmss(MaxWarmCore).text() : na_str) + << ", DiagSource = " << diagtype_to_string(DiagSource) << ", NDiag = " << DiagName.n() << ", NPoints = " << NPoints << ", NAlloc = " << NAlloc @@ -219,6 +222,7 @@ void TrackInfo::assign(const TrackInfo &t) { MaxValidTime = t.MaxValidTime; MinWarmCore = t.MinWarmCore; MaxWarmCore = t.MaxWarmCore; + DiagSource = t.DiagSource; DiagName = t.DiagName; TrackLines = t.TrackLines; @@ -530,6 +534,9 @@ bool TrackInfo::add_diag_data(DiagFile &diag_file, const StringArray &diag_name) InitTime != diag_file.init() || !diag_file.technique().has(Technique)) return(false); + // Store the diagnostic source + DiagSource = diag_file.source(); + // If empty, store all diagnostics if(diag_name.n() > 0) DiagName = diag_name; else DiagName = diag_file.diag_name(); diff --git a/src/libcode/vx_tc_util/track_info.h b/src/libcode/vx_tc_util/track_info.h index c01d9a2fab..e227030f5c 100644 --- a/src/libcode/vx_tc_util/track_info.h +++ b/src/libcode/vx_tc_util/track_info.h @@ -70,7 +70,8 @@ class TrackInfo { unixtime MinWarmCore; unixtime MaxWarmCore; - // Diagnostic names + // Diagnostic source and names + DiagType DiagSource; StringArray DiagName; // TrackPoints @@ -113,6 +114,7 @@ class TrackInfo { void set_valid_min(const unixtime); void set_valid_max(const unixtime); void set_point(int, const TrackPoint &); + void set_diag_source(DiagType); void set_diag_name(const StringArray &); // @@ -144,6 +146,8 @@ class TrackInfo { int warm_core_dur() const; int valid_inc() const; int n_points() const; + + DiagType diag_source() const; int n_diag() const; const StringArray & diag_name() const; const char * diag_name(int) const; @@ -184,6 +188,7 @@ inline void TrackInfo::set_initials(const char *s) { Initials = s; inline void TrackInfo::set_init(const unixtime u) { InitTime = u; } inline void TrackInfo::set_valid_min(const unixtime u) { MinValidTime = u; } inline void TrackInfo::set_valid_max(const unixtime u) { MaxValidTime = u; } +inline void TrackInfo::set_diag_source(DiagType t) { DiagSource = t; } inline void TrackInfo::set_diag_name(const StringArray &s) { DiagName = s; } inline const ConcatString & TrackInfo::storm_id() const { return(StormId); } @@ -200,9 +205,12 @@ inline unixtime TrackInfo::valid_max() const { return(MaxVali inline unixtime TrackInfo::warm_core_min() const { return(MinWarmCore); } inline unixtime TrackInfo::warm_core_max() const { return(MaxWarmCore); } inline int TrackInfo::n_points() const { return(NPoints); } -inline int TrackInfo::n_diag() const { return(DiagName.n()); } -inline const StringArray & TrackInfo::diag_name() const { return(DiagName); } -inline StringArray TrackInfo::track_lines() const { return(TrackLines); } + +inline DiagType TrackInfo::diag_source() const { return(DiagSource); } +inline int TrackInfo::n_diag() const { return(DiagName.n()); } +inline const StringArray & TrackInfo::diag_name() const { return(DiagName); } + +inline StringArray TrackInfo::track_lines() const { return(TrackLines); } //////////////////////////////////////////////////////////////////////// // From 001b0ec57138e0dfb0307c1b6d2fd96226454d44 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Wed, 26 Oct 2022 13:30:17 -0600 Subject: [PATCH 52/58] Per #392, add SOURCE column to the TCDIAG line type from tc_pairs. --- data/table_files/met_header_columns_V11.0.txt | 2 +- internal/test_unit/hdr/met_11_0.hdr | 2 +- src/libcode/vx_tc_util/tc_columns.cc | 1 + src/libcode/vx_tc_util/tc_columns.h | 4 ++-- src/libcode/vx_tc_util/track_pair_info.cc | 16 ++++++++++++++++ 5 files changed, 21 insertions(+), 4 deletions(-) diff --git a/data/table_files/met_header_columns_V11.0.txt b/data/table_files/met_header_columns_V11.0.txt index a3c983a11d..fc371c1347 100644 --- a/data/table_files/met_header_columns_V11.0.txt +++ b/data/table_files/met_header_columns_V11.0.txt @@ -37,5 +37,5 @@ V11.0 : MODE : OBJ : VERSION MODEL N_VALID GRID_RES DESC FCST_LEAD FCST_VAL V11.0 : MODE : CTS : VERSION MODEL N_VALID GRID_RES DESC FCST_LEAD FCST_VALID FCST_ACCUM OBS_LEAD OBS_VALID OBS_ACCUM FCST_RAD FCST_THR OBS_RAD OBS_THR FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE FIELD TOTAL FY_OY FY_ON FN_OY FN_ON BASER FMEAN ACC FBIAS PODY PODN POFD FAR CSI GSS HK HSS ODDS V11.0 : TCST : TCMPR : VERSION AMODEL BMODEL DESC STORM_ID BASIN CYCLONE STORM_NAME INIT LEAD VALID INIT_MASK VALID_MASK LINE_TYPE TOTAL INDEX LEVEL WATCH_WARN INITIALS ALAT ALON BLAT BLON TK_ERR X_ERR Y_ERR ALTK_ERR CRTK_ERR ADLAND BDLAND AMSLP BMSLP AMAX_WIND BMAX_WIND AAL_WIND_34 BAL_WIND_34 ANE_WIND_34 BNE_WIND_34 ASE_WIND_34 BSE_WIND_34 ASW_WIND_34 BSW_WIND_34 ANW_WIND_34 BNW_WIND_34 AAL_WIND_50 BAL_WIND_50 ANE_WIND_50 BNE_WIND_50 ASE_WIND_50 BSE_WIND_50 ASW_WIND_50 BSW_WIND_50 ANW_WIND_50 BNW_WIND_50 AAL_WIND_64 BAL_WIND_64 ANE_WIND_64 BNE_WIND_64 ASE_WIND_64 BSE_WIND_64 ASW_WIND_64 BSW_WIND_64 ANW_WIND_64 BNW_WIND_64 ARADP BRADP ARRP BRRP AMRD BMRD AGUSTS BGUSTS AEYE BEYE ADIR BDIR ASPEED BSPEED ADEPTH BDEPTH NUM_MEMBERS TRACK_SPREAD DIST_MEAN MSLP_SPREAD MAX_WIND_SPREAD -V11.0 : TCST : TCDIAG : VERSION AMODEL BMODEL DESC STORM_ID BASIN CYCLONE STORM_NAME INIT LEAD VALID INIT_MASK VALID_MASK LINE_TYPE TOTAL INDEX (N_DIAG) DIAG_[0-9]* VALUE_[0-9]* +V11.0 : TCST : TCDIAG : VERSION AMODEL BMODEL DESC STORM_ID BASIN CYCLONE STORM_NAME INIT LEAD VALID INIT_MASK VALID_MASK LINE_TYPE TOTAL INDEX SOURCE (N_DIAG) DIAG_[0-9]* VALUE_[0-9]* V11.0 : TCST : PROBRIRW : VERSION AMODEL BMODEL DESC STORM_ID BASIN CYCLONE STORM_NAME INIT LEAD VALID INIT_MASK VALID_MASK LINE_TYPE ALAT ALON BLAT BLON INITIALS TK_ERR X_ERR Y_ERR ADLAND BDLAND RIRW_BEG RIRW_END RIRW_WINDOW AWIND_END BWIND_BEG BWIND_END BDELTA BDELTA_MAX BLEVEL_BEG BLEVEL_END (N_THRESH) THRESH_[0-9]* PROB_[0-9]* diff --git a/internal/test_unit/hdr/met_11_0.hdr b/internal/test_unit/hdr/met_11_0.hdr index c82b051928..171b811992 100644 --- a/internal/test_unit/hdr/met_11_0.hdr +++ b/internal/test_unit/hdr/met_11_0.hdr @@ -34,5 +34,5 @@ MODE_SOA : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_L MODE_POA : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE N_VALID GRID_RES OBJECT_ID OBJECT_CAT CENTROID_DIST BOUNDARY_DIST CONVEX_HULL_DIST ANGLE_DIFF ASPECT_DIFF AREA_RATIO INTERSECTION_AREA UNION_AREA SYMMETRIC_DIFF INTERSECTION_OVER_AREA CURVATURE_RATIO COMPLEXITY_RATIO PERCENTILE_INTENSITY_RATIO INTEREST MODE_CTS : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE N_VALID GRID_RES FIELD TOTAL FY_OY FY_ON FN_OY FN_ON BASER FMEAN ACC FBIAS PODY PODN POFD FAR CSI GSS HK HSS ODDS TCST_TCMPR : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE AMODEL BMODEL STORM_ID BASIN CYCLONE STORM_NAME INIT_MASK VALID_MASK TOTAL INDEX LEVEL WATCH_WARN INITIALS ALAT ALON BLAT BLON TK_ERR X_ERR Y_ERR ALTK_ERR CRTK_ERR ADLAND BDLAND AMSLP BMSLP AMAX_WIND BMAX_WIND AAL_WIND_34 BAL_WIND_34 ANE_WIND_34 BNE_WIND_34 ASE_WIND_34 BSE_WIND_34 ASW_WIND_34 BSW_WIND_34 ANW_WIND_34 BNW_WIND_34 AAL_WIND_50 BAL_WIND_50 ANE_WIND_50 BNE_WIND_50 ASE_WIND_50 BSE_WIND_50 ASW_WIND_50 BSW_WIND_50 ANW_WIND_50 BNW_WIND_50 AAL_WIND_64 BAL_WIND_64 ANE_WIND_64 BNE_WIND_64 ASE_WIND_64 BSE_WIND_64 ASW_WIND_64 BSW_WIND_64 ANW_WIND_64 BNW_WIND_64 ARADP BRADP ARRP BRRP AMRD BMRD AGUSTS BGUSTS AEYE BEYE ADIR BDIR ASPEED BSPEED ADEPTH BDEPTH NUM_MEMBERS TRACK_SPREAD DIST_MEAN MSLP_SPREAD MAX_WIND_SPREAD -TCST_TCDIAG : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE AMODEL BMODEL STORM_ID BASIN CYCLONE STORM_NAME INIT_MASK VALID_MASK TOTAL INDEX N_DIAG _VAR_ +TCST_TCDIAG : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE AMODEL BMODEL STORM_ID BASIN CYCLONE STORM_NAME INIT_MASK VALID_MASK TOTAL INDEX SOURCE N_DIAG _VAR_ TCST_PROBRIRW : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE ALAT ALON BLAT BLON INITIALS TK_ERR X_ERR Y_ERR ADLAND BDLAND RI_BEG RI_END RI_WINDOW AWIND_END BWIND_BEG BWIND_END BDELTA BDELTA_MAX BLEVEL_BEG BLEVEL_END N_THRESH _VAR_ diff --git a/src/libcode/vx_tc_util/tc_columns.cc b/src/libcode/vx_tc_util/tc_columns.cc index 9d56e008ea..4cb90895f4 100644 --- a/src/libcode/vx_tc_util/tc_columns.cc +++ b/src/libcode/vx_tc_util/tc_columns.cc @@ -359,6 +359,7 @@ void write_tc_diag_cols(const TrackPairInfo &p, int i, // Write TCDIAG columns at.set_entry(r, c++, p.n_points()); at.set_entry(r, c++, i+1); + at.set_entry(r, c++, diagtype_to_string(p.adeck().diag_source())); at.set_entry(r, c++, p.adeck()[i].n_diag()); // Check the number of names and values match diff --git a/src/libcode/vx_tc_util/tc_columns.h b/src/libcode/vx_tc_util/tc_columns.h index 3f44157317..7bd6c58814 100644 --- a/src/libcode/vx_tc_util/tc_columns.h +++ b/src/libcode/vx_tc_util/tc_columns.h @@ -122,8 +122,8 @@ static const int n_tc_cols_xy = sizeof(tc_cols_xy)/sizeof(*tc_cols_xy); //////////////////////////////////////////////////////////////////////// static const char * tc_diag_cols [] = { - "TOTAL", "INDEX", "N_DIAG", - "DIAG_", "VALUE_" + "TOTAL", "INDEX", "SOURCE", + "N_DIAG", "DIAG_", "VALUE_" }; static const int n_tc_diag_cols = sizeof(tc_diag_cols)/sizeof(*tc_diag_cols); diff --git a/src/libcode/vx_tc_util/track_pair_info.cc b/src/libcode/vx_tc_util/track_pair_info.cc index 72e8aa1212..4a69d2d0db 100644 --- a/src/libcode/vx_tc_util/track_pair_info.cc +++ b/src/libcode/vx_tc_util/track_pair_info.cc @@ -471,6 +471,22 @@ void TrackPairInfo::add_tcdiag_line(const TCStatLine &l) { // Name of diagnostics read StringArray diag_name; + // Diagnostic source type + DiagType t = string_to_diagtype(l.get_item("SOURCE")); + + // Make sure the source type does not change + if(ADeck.diag_source() != DiagType_None && + ADeck.diag_source() != t) { + mlog << Error << "\nTrackPairInfo::add_tcdiag_line() -> " + << "the diagnostic source type has changed (" + << diagtype_to_string(ADeck.diag_source()) << " != " + << diagtype_to_string(t) << ")!\n\n"; + exit(1); + } + + // Store the source type + ADeck.set_diag_source(t); + // Number of diagnostics n_diag = atoi(l.get_item("N_DIAG")); From 3b401b27125c07f5ce4760ea9a1fe2713dbde493 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Wed, 26 Oct 2022 13:45:26 -0600 Subject: [PATCH 53/58] Per #392, update TC-Pairs documentation based on changes to the TCDIAG line type. --- data/config/TCPairsConfig_default | 3 +- docs/Users_Guide/tc-pairs.rst | 61 +++++++++++++------ .../config/TCPairsConfig_DIAGNOSTICS | 2 +- 3 files changed, 43 insertions(+), 23 deletions(-) diff --git a/data/config/TCPairsConfig_default b/data/config/TCPairsConfig_default index 733b6c574a..b0a20cfd78 100644 --- a/data/config/TCPairsConfig_default +++ b/data/config/TCPairsConfig_default @@ -143,9 +143,8 @@ watch_warn = { diag_name = []; // -// Unit conversions to be applied to diagnostic values +// Unit conversions to be applied based on diagnostic names and units // - diag_convert_map = [ { source = "TCDIAG"; key = [ "(10C)", "(10KT)", "(10M/S)" ]; diff --git a/docs/Users_Guide/tc-pairs.rst b/docs/Users_Guide/tc-pairs.rst index dd6111e701..7809601bfc 100644 --- a/docs/Users_Guide/tc-pairs.rst +++ b/docs/Users_Guide/tc-pairs.rst @@ -34,11 +34,10 @@ The usage statement for tc_pairs is shown below: .. code-block:: none Usage: tc_pairs - -adeck source and/or -edeck source - -bdeck source + -adeck path and/or -edeck path + -bdeck path -config file - [-tcdiag source] - [-lsdiag source] + [-diag source path] [-out base] [-log file] [-v level] @@ -48,28 +47,30 @@ tc_pairs has required arguments and can accept several optional arguments. Required arguments for tc_pairs ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -1. The **-adeck source** argument indicates the adeck TC-Pairs acceptable format data source containing tropical cyclone model forecast (output from tracker) data to be verified. Acceptable data formats are limited to the standard ATCF format and the one column modified ATCF file, generated by running the tracker in genesis mode. It specifies the name of a TC-Pairs acceptable format file or top-level directory containing TC-Pairs acceptable format files ending in ".dat" to be processed. The **-adeck** or **-edeck** option must be used at least once. +1. The **-adeck path** argument indicates the adeck TC-Pairs acceptable format data containing tropical cyclone model forecast (output from tracker) data to be verified. Acceptable data formats are limited to the standard ATCF format and the one column modified ATCF file, generated by running the tracker in genesis mode. It specifies the name of a TC-Pairs acceptable format file or top-level directory containing TC-Pairs acceptable format files ending in ".dat" to be processed. The **-adeck** or **-edeck** option must be used at least once. -2. The **-edeck source** argument indicates the edeck ATCF format data source containing probabilistic track data to be verified. It specifies the name of an ATCF format file or top-level directory containing ATCF format files ending in ".dat" to be processed. The **-adeck** or **-edeck** option must be used at least once. +2. The **-edeck path** argument indicates the edeck ATCF format data containing probabilistic track data to be verified. It specifies the name of an ATCF format file or top-level directory containing ATCF format files ending in ".dat" to be processed. The **-adeck** or **-edeck** option must be used at least once. -3. The **-bdeck source** argument indicates the TC-Pairs acceptable format data source containing the tropical cyclone reference dataset to be used for verifying the adeck source. This source is typically the NHC Best Track Analysis, but could be any TC-Pairs acceptable formatted reference. The acceptable data formats for bdecks are the same as those for adecks. This argument specifies the name of a TC-Pairs acceptable format file or top-level directory containing TC-Pairs acceptable format files ending in ".dat" to be processed. +3. The **-bdeck path** argument indicates the TC-Pairs acceptable format data containing the tropical cyclone reference dataset to be used for verifying the adeck data. This data is typically the NHC Best Track Analysis, but could be any TC-Pairs acceptable formatted reference. The acceptable data formats for bdecks are the same as those for adecks. This argument specifies the name of a TC-Pairs acceptable format file or top-level directory containing TC-Pairs acceptable format files ending in ".dat" to be processed. 4. The **-config file** argument indicates the name of the configuration file to be used. The contents of the configuration file are discussed below. Optional arguments for tc_pairs ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -5. The **-tcdiag source** argument indicates the TC-Pairs acceptable format data source containing the tropical cyclone diagnostics dataset corresponding to the adeck tracks. This argument specifies the name of a TC-Pairs acceptable format file or top-level directory containing TC-Pairs acceptable format files ending in ".dat" to be processed. +5. The **-diag source path** argument indicates the TC-Pairs acceptable format data containing the tropical cyclone diagnostics dataset corresponding to the adeck tracks. The **source** can be set to TCDIAG, LSDIAG_RT, or LSDIAG_DEV to indicate the input diagnostics data source. The **path** argument specifies the name of a TC-Pairs acceptable format file or top-level directory containing TC-Pairs acceptable format files ending in ".dat" to be processed. -6. The **-lsdiag source** argument indicates the TC-Pairs acceptable format data source containing the large scale diagnostics dataset corresponding to the adeck tracks. This argument specifies the name of a TC-Pairs acceptable format file or top-level directory containing TC-Pairs acceptable format files ending in ".dat" to be processed. +6. The -**out base** argument indicates the path of the output file base. This argument overrides the default output file base (**./out_tcmpr**). -7. The -**out base** argument indicates the path of the output file base. This argument overrides the default output file base (**./out_tcmpr**). +7. The **-log file** option directs output and errors to the specified log file. All messages will be written to that file as well as standard out and error. Thus, users can save the messages without having to redirect the output on the command line. The default behavior is no log file. -8. The **-log file** option directs output and errors to the specified log file. All messages will be written to that file as well as standard out and error. Thus, users can save the messages without having to redirect the output on the command line. The default behavior is no log file. +8. The **-v level** option indicates the desired level of verbosity. The contents of "level" will override the default setting of 2. Setting the verbosity to 0 will make the tool run with no log messages, while increasing the verbosity above 1 will increase the amount of logging. -9. The **-v level** option indicates the desired level of verbosity. The contents of "level" will override the default setting of 2. Setting the verbosity to 0 will make the tool run with no log messages, while increasing the verbosity above 1 will increase the amount of logging. +This tool currently only supports the rapid intensification (**RI**) edeck probability type but support for additional edeck probability types will be added in future releases. -This tool currently only supports the rapid intensification (**RI**) edeck probability type but support for additional edeck probability types will be added in future releases. At least one **-adeck** or **-edeck** option must be specified. The **-adeck, -edeck**, and **-bdeck** options may optionally be followed with **suffix=string** to append that string to all model names found within that data source. This option may be useful when processing track data from two different sources which reuse the same model names. The **-tcdiag** and **-lsdiag** options may optionally be followed with **model=string** to override the model name of the tracks to which those diagnostics correspond. The **string** specifies a comma-separated list of one or more ATCF ID's to which these diagnostics should be paired (e.g. **model=OFCL,SHIP**). +At least one **-adeck** or **-edeck** option must be specified. The **-adeck, -edeck**, and **-bdeck** options may optionally be followed with **suffix=string** to append that string to all model names found within that data source. This option may be useful when processing track data from two different sources which reuse the same model names. + +The **-diag** option may optionally be followed with **model=string** to override the model name of the tracks to which those diagnostics correspond. The **string** specifies a comma-separated list of one or more ATCF ID's to which these diagnostics should be paired (e.g. **model=OFCL,SHIP**). An example of the tc_pairs calling sequence is shown below: @@ -83,6 +84,8 @@ The TC-Pairs tool implements the following logic: • Parse the adeck, edeck, and bdeck data files and store them as track objects. +• Parse diagnostics data files and add the requested diagnostics to the existing adeck track objects. + • Apply configuration file settings to filter the adeck, edeck, and bdeck track data down to a subset of interest. • Apply configuration file settings to derive additional adeck track data, such as interpolated tracks, consensus tracks, time-lagged tracks, and statistical track and intensity models. @@ -265,14 +268,29 @@ ____________________ .. code-block:: none - tcdiag_convert_map = [ - { key = [ "(10C)", "(10KT)", "(10M/S)" ]; convert(x) = x / 10; } + diag_convert_map = [ + { source = "TCDIAG"; + key = [ "(10C)", "(10KT)", "(10M/S)" ]; + convert(x) = x / 10; }, + + { source = "LSDIAG_RT"; + key = [ "CSST", "RSST", "DSST", "DSTA", "XDST", "XNST", "NSST", "NSTA", + "NTMX", "NTFR", "U200", "U20C", "V20C", "E000", "EPOS", "ENEG", "EPSS", "ENSS", + "T000", "TLAT", "TLON", "TWAC", "TWXC", "G150", "G200", "G250", "V000", "V850", + "V500", "V300", "PENC", "SHDC", "SHGC", "T150", "T200", "T250", "SHRD", "SHRS", + "SHRG", "PENV", "HE07", "HE05", "PW01", "PW02", "PW03", "PW04", "PW05", "PW06", + "PW07", "PW08", "PW09", "PW10", "PW11", "PW12", "PW13", "PW14", "PW15", "PW16", + "PW17", "PW18", "PW20", "PW21" ]; + convert(x) = x / 10; }, + + { source = "LSDIAG_RT"; + key = [ "VVAV", "VMFX", "VVAC" ]; + convert(x) = x / 100; } ]; - lsdiag_convert_map = []; -The **tcdiag_convert_map** and **lsdiag_convert_map** entries define conversion functions to be applied to diagnostics data read with the **-tcdiag** and **-lsdiag** command line options, respectively. Each array element is a dictionary consisting of a **key** and **convert(x)** entry. +The **diag_convert_map** entries define conversion functions to be applied to diagnostics data read with the **-diag** command line option. Each array element is a dictionary consisting of a **source**, **key**, and **convert(x)** entry. -The **key** is an array of strings. For **-tcdiag** inputs, the strings can specify diagnostic names or units. If both the name and units are specified, the conversion function for the name takes precedence. For **-lsdiag** inputs, the strings specify diagnostic names since no unit strings are provided. **convert(x)** is a function of one variable which defines how the diagnostic data should be converted. The defined function is applied to any diagnostic value who name or units appears in the **key**. +The **source** is one of the supported diagnostics data sources. The **key** is an array of strings. The strings can specify diagnostic names or units, although units are only checked for **TCDIAG** sources. If both the name and units are specified, the conversion function for the name takes precedence. **convert(x)** is a function of one variable which defines how the diagnostic data should be converted. The defined function is applied to any diagnostic value who name or units appears in the **key**. ____________________ @@ -551,12 +569,15 @@ TC-Pairs produces output in TCST format. The default output file name can be ove - INDEX - Index of the current track pair * - 16 + - SOURCE + - Diagnostics data source + * - 17 - N_DIAG - Number of storm diagnostic name and value columns to follow - * - 17 + * - 18 - DIAG_i - Name of the of the ith storm diagnostic (repeated) - * - 18 + * - 19 - VALUE_i - Value of the ith storm diagnostic (repeated) diff --git a/internal/test_unit/config/TCPairsConfig_DIAGNOSTICS b/internal/test_unit/config/TCPairsConfig_DIAGNOSTICS index 7ef647a90d..a2ae20816b 100644 --- a/internal/test_unit/config/TCPairsConfig_DIAGNOSTICS +++ b/internal/test_unit/config/TCPairsConfig_DIAGNOSTICS @@ -134,7 +134,7 @@ watch_warn = { diag_name = [ ${DIAG_NAME} ]; // -// Unit conversions to be applied to diagnostic values +// Unit conversions to be applied based on diagnostic names and units // Commented out to use settings from the default config file // // diag_convert_map = []; From f68d0a41f2f92dba6a0a5b505f205b51f8d50557 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Wed, 26 Oct 2022 14:35:27 -0600 Subject: [PATCH 54/58] Per #392, when using a map of maps, the Docker build step errors out with: tc_pairs_conf_info.h:118:60: error: '>>' should be '> >' within a nested template argument list std::map> DiagConvertMap; Adding the extra space to make the code compile in Docker. --- src/tools/tc_utils/tc_pairs/tc_pairs_conf_info.cc | 6 +++--- src/tools/tc_utils/tc_pairs/tc_pairs_conf_info.h | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/tools/tc_utils/tc_pairs/tc_pairs_conf_info.cc b/src/tools/tc_utils/tc_pairs/tc_pairs_conf_info.cc index 30fab41cc9..fa6c9b318c 100644 --- a/src/tools/tc_utils/tc_pairs/tc_pairs_conf_info.cc +++ b/src/tools/tc_utils/tc_pairs/tc_pairs_conf_info.cc @@ -26,7 +26,7 @@ using namespace std; //////////////////////////////////////////////////////////////////////// void parse_conf_diag_convert_map(Dictionary *, - map> &); + map > &); //////////////////////////////////////////////////////////////////////// // @@ -331,7 +331,7 @@ void TCPairsConfInfo::process_config() { //////////////////////////////////////////////////////////////////////// void parse_conf_diag_convert_map(Dictionary *dict, - map> &source_map) { + map > &source_map) { int i, j; Dictionary *map_dict = (Dictionary *) 0; map cur_map; @@ -386,7 +386,7 @@ void parse_conf_diag_convert_map(Dictionary *dict, } // Add a new source entry else { - source_map.insert(pair>(source, cur_map)); + source_map.insert(pair >(source, cur_map)); } } // end for i diff --git a/src/tools/tc_utils/tc_pairs/tc_pairs_conf_info.h b/src/tools/tc_utils/tc_pairs/tc_pairs_conf_info.h index 8c2583b51a..12dc7eea14 100644 --- a/src/tools/tc_utils/tc_pairs/tc_pairs_conf_info.h +++ b/src/tools/tc_utils/tc_pairs/tc_pairs_conf_info.h @@ -115,7 +115,7 @@ class TCPairsConfInfo { StringArray DiagName; // Diagnostic conversions - std::map> DiagConvertMap; + std::map< DiagType, std::map > DiagConvertMap; // Basin Map std::map BasinMap; From b5ac2f4ea8bf935a09394b719733679fdb3a345f Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Wed, 26 Oct 2022 14:52:42 -0600 Subject: [PATCH 55/58] Per #392, set the default lsdiag ATCFID as OFCL. --- src/libcode/vx_tc_util/diag_file.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libcode/vx_tc_util/diag_file.cc b/src/libcode/vx_tc_util/diag_file.cc index 775083d9e5..4e7c76f551 100644 --- a/src/libcode/vx_tc_util/diag_file.cc +++ b/src/libcode/vx_tc_util/diag_file.cc @@ -27,7 +27,7 @@ using namespace std; //////////////////////////////////////////////////////////////////////// -static const char default_lsdiag_technique[] = "SHIP"; +static const char default_lsdiag_technique[] = "OFCL"; static const int lsdiag_wdth[] = { 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, From bc2b5559fd08e8b41e70938a5c7f0e4540a27a98 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Wed, 26 Oct 2022 16:37:28 -0600 Subject: [PATCH 56/58] Per #392, update diagnostics tests to specify the diag_name strings. Also needed to refine the logic in track_info.cc. --- internal/test_unit/xml/unit_tc_pairs.xml | 2 +- src/libcode/vx_tc_util/track_info.cc | 17 +++++++++++------ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/internal/test_unit/xml/unit_tc_pairs.xml b/internal/test_unit/xml/unit_tc_pairs.xml index d12f327f09..ec40175113 100644 --- a/internal/test_unit/xml/unit_tc_pairs.xml +++ b/internal/test_unit/xml/unit_tc_pairs.xml @@ -188,7 +188,7 @@ STORM_ID "AL092022" INIT_BEG "20220926_00" INIT_END "20220926_18" - DIAG_NAME + DIAG_NAME "DTL", "PW01", "SHRD", "TPW", "LAND", "SHR_MAG", "STM_SPD" \ -adeck &DATA_DIR;/adeck/aal092022_OFCL_SHIP_AVNO.dat \ diff --git a/src/libcode/vx_tc_util/track_info.cc b/src/libcode/vx_tc_util/track_info.cc index 3b53f150c9..cddb2b1536 100644 --- a/src/libcode/vx_tc_util/track_info.cc +++ b/src/libcode/vx_tc_util/track_info.cc @@ -527,7 +527,7 @@ void TrackInfo::add_watch_warn(const ConcatString &ww_sid, //////////////////////////////////////////////////////////////////////// -bool TrackInfo::add_diag_data(DiagFile &diag_file, const StringArray &diag_name) { +bool TrackInfo::add_diag_data(DiagFile &diag_file, const StringArray &req_diag_name) { // Check for a match if(StormId != diag_file.storm_id() || @@ -538,15 +538,20 @@ bool TrackInfo::add_diag_data(DiagFile &diag_file, const StringArray &diag_name) DiagSource = diag_file.source(); // If empty, store all diagnostics - if(diag_name.n() > 0) DiagName = diag_name; - else DiagName = diag_file.diag_name(); + bool store_all_diag = (req_diag_name.n() == 0 ? true : false); int i_name, i_time, i_pnt; - // Retrieve data for each diagnostic - for(i_name=0; i_name Date: Thu, 27 Oct 2022 23:50:16 -0600 Subject: [PATCH 57/58] Per #392, updates based on PR #2315 review from @jvigh. Thanks for the careful review Jonathan. --- data/config/TCPairsConfig_default | 28 ++++++++++++++++++-------- docs/Users_Guide/tc-pairs.rst | 32 ++++++++++++++++++++---------- src/libcode/vx_tc_util/diag_file.h | 2 +- 3 files changed, 43 insertions(+), 19 deletions(-) diff --git a/data/config/TCPairsConfig_default b/data/config/TCPairsConfig_default index b0a20cfd78..07d94992f8 100644 --- a/data/config/TCPairsConfig_default +++ b/data/config/TCPairsConfig_default @@ -151,18 +151,30 @@ diag_convert_map = [ convert(x) = x / 10; }, { source = "LSDIAG_RT"; - key = [ "CSST", "RSST", "DSST", "DSTA", "XDST", "XNST", "NSST", "NSTA", - "NTMX", "NTFR", "U200", "U20C", "V20C", "E000", "EPOS", "ENEG", "EPSS", "ENSS", - "T000", "TLAT", "TLON", "TWAC", "TWXC", "G150", "G200", "G250", "V000", "V850", - "V500", "V300", "PENC", "SHDC", "SHGC", "T150", "T200", "T250", "SHRD", "SHRS", - "SHRG", "PENV", "HE07", "HE05", "PW01", "PW02", "PW03", "PW04", "PW05", "PW06", - "PW07", "PW08", "PW09", "PW10", "PW11", "PW12", "PW13", "PW14", "PW15", "PW16", - "PW17", "PW18", "PW20", "PW21" ]; + key = [ "CSST", "RSST", "DSST", "DSTA", "XDST", "XNST", "NSST", "NSTA", "NTMX", "NTFR", + "U200", "U20C", "V20C", "E000", "EPOS", "ENEG", "EPSS", "ENSS", "T000", "TLAT", + "TLON", "TWAC", "TWXC", "G150", "G200", "G250", "V000", "V850", "V500", "V300", + "SHDC", "SHGC", "T150", "T200", "T250", "SHRD", "SHRS", "SHRG", "HE07", "HE05", + "PW01", "PW02", "PW03", "PW04", "PW05", "PW06", "PW07", "PW08", "PW09", "PW10", + "PW11", "PW12", "PW13", "PW14", "PW15", "PW16", "PW17", "PW18", "PW20", "PW21" ]; convert(x) = x / 10; }, { source = "LSDIAG_RT"; key = [ "VVAV", "VMFX", "VVAC" ]; - convert(x) = x / 100; } + convert(x) = x / 100; }, + + { source = "LSDIAG_RT"; + key = [ "TADV" ]; + convert(x) = x / 1000000; }, + + { source = "LSDIAG_RT"; + key = [ "Z850", "D200", "TGRD", "DIVC" ]; + convert(x) = x / 10000000; }, + + { source = "LSDIAG_RT"; + key = [ "PENC", "PENV" ]; + convert(x) = x / 10 + 1000; } + ]; // diff --git a/docs/Users_Guide/tc-pairs.rst b/docs/Users_Guide/tc-pairs.rst index 7809601bfc..4557aa90fa 100644 --- a/docs/Users_Guide/tc-pairs.rst +++ b/docs/Users_Guide/tc-pairs.rst @@ -262,7 +262,7 @@ ____________________ The **diag_name** entry specifies a comma-separated list of strings for the tropical cyclone diagnostics of interest. This applies when the **-tcdiag** and/or **-lsdiag** command line options have been used to provide storm diagnostics data. If a non-zero list of diagnostic names is specified, only those diagnostics appearing in the list are written to the TCDIAG output line type. If defined as an empty list (default), all diagnostics found in the input are written to the TCDIAG output lines. -A TCMPR line is written to the output for each track point. If diagnostics data is also defined for that track point, a TCDIAG line is written immediately after the corresponding TCMPR line. The contents of that TCDIAG line is deteremined by diagnostic names requrested in the **diag_name** entry. +A TCMPR line is written to the output for each track point. If diagnostics data is also defined for that track point, a TCDIAG line is written immediately after the corresponding TCMPR line. The contents of that TCDIAG line is determined by diagnostic names requested in the **diag_name** entry. ____________________ @@ -274,23 +274,35 @@ ____________________ convert(x) = x / 10; }, { source = "LSDIAG_RT"; - key = [ "CSST", "RSST", "DSST", "DSTA", "XDST", "XNST", "NSST", "NSTA", - "NTMX", "NTFR", "U200", "U20C", "V20C", "E000", "EPOS", "ENEG", "EPSS", "ENSS", - "T000", "TLAT", "TLON", "TWAC", "TWXC", "G150", "G200", "G250", "V000", "V850", - "V500", "V300", "PENC", "SHDC", "SHGC", "T150", "T200", "T250", "SHRD", "SHRS", - "SHRG", "PENV", "HE07", "HE05", "PW01", "PW02", "PW03", "PW04", "PW05", "PW06", - "PW07", "PW08", "PW09", "PW10", "PW11", "PW12", "PW13", "PW14", "PW15", "PW16", - "PW17", "PW18", "PW20", "PW21" ]; + key = [ "CSST", "RSST", "DSST", "DSTA", "XDST", "XNST", "NSST", "NSTA", "NTMX", "NTFR", + "U200", "U20C", "V20C", "E000", "EPOS", "ENEG", "EPSS", "ENSS", "T000", "TLAT", + "TLON", "TWAC", "TWXC", "G150", "G200", "G250", "V000", "V850", "V500", "V300", + "SHDC", "SHGC", "T150", "T200", "T250", "SHRD", "SHRS", "SHRG", "HE07", "HE05", + "PW01", "PW02", "PW03", "PW04", "PW05", "PW06", "PW07", "PW08", "PW09", "PW10", + "PW11", "PW12", "PW13", "PW14", "PW15", "PW16", "PW17", "PW18", "PW20", "PW21" ]; convert(x) = x / 10; }, { source = "LSDIAG_RT"; key = [ "VVAV", "VMFX", "VVAC" ]; - convert(x) = x / 100; } + convert(x) = x / 100; }, + + { source = "LSDIAG_RT"; + key = [ "TADV" ]; + convert(x) = x / 1000000; }, + + { source = "LSDIAG_RT"; + key = [ "Z850", "D200", "TGRD", "DIVC" ]; + convert(x) = x / 10000000; }, + + { source = "LSDIAG_RT"; + key = [ "PENC", "PENV" ]; + convert(x) = x / 10 + 1000; } + ]; The **diag_convert_map** entries define conversion functions to be applied to diagnostics data read with the **-diag** command line option. Each array element is a dictionary consisting of a **source**, **key**, and **convert(x)** entry. -The **source** is one of the supported diagnostics data sources. The **key** is an array of strings. The strings can specify diagnostic names or units, although units are only checked for **TCDIAG** sources. If both the name and units are specified, the conversion function for the name takes precedence. **convert(x)** is a function of one variable which defines how the diagnostic data should be converted. The defined function is applied to any diagnostic value who name or units appears in the **key**. +The **source** is one of the supported diagnostics data sources. The **key** is an array of strings. The strings can specify diagnostic names or units, although units are only checked for **TCDIAG** sources. If both the name and units are specified, the conversion function for the name takes precedence. **convert(x)** is a function of one variable which defines how the diagnostic data should be converted. The defined function is applied to any diagnostic value whose name or units appears in the **key**. ____________________ diff --git a/src/libcode/vx_tc_util/diag_file.h b/src/libcode/vx_tc_util/diag_file.h index 5490a9f8fe..95d6341aed 100644 --- a/src/libcode/vx_tc_util/diag_file.h +++ b/src/libcode/vx_tc_util/diag_file.h @@ -23,7 +23,7 @@ //////////////////////////////////////////////////////////////////////// // -// LSDIAG files: +// Real-time LSDIAG files: // - https://ftp.nhc.noaa.gov/atcf/lsdiag // - Header: // BBCC YYMMDD HH WS LAT LON 9999 BBCCYYYY From 56f97c8aa2d459cec0ed5856fa85c75322e7b2c6 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Fri, 28 Oct 2022 11:05:56 -0600 Subject: [PATCH 58/58] Per #392, hopefully the final commit for this feature branch. These changes enable LAT and LON values to be included on the TCDIAG line as requested in Jonathan's PR review. Note that the longitude values stored for the header are converted from degrees east to west (*-1) and rescaled to a standard range to match the track data. The LON values written to the TCDIAG line are NOT converted in this way. Figure its better to leave them as close to the input as possible. Also renamed the input test data directory from lsdiag to lsdiag_rt, figuing that we'll need add data for lsdiag_dev eventually. --- data/config/TCPairsConfig_default | 13 ++--- docs/Users_Guide/tc-pairs.rst | 13 ++--- internal/test_unit/xml/unit_tc_pairs.xml | 2 +- src/libcode/vx_tc_util/diag_file.cc | 64 +++++++++++++----------- src/libcode/vx_tc_util/diag_file.h | 35 +++++++------ 5 files changed, 70 insertions(+), 57 deletions(-) diff --git a/data/config/TCPairsConfig_default b/data/config/TCPairsConfig_default index 07d94992f8..a768ac23e6 100644 --- a/data/config/TCPairsConfig_default +++ b/data/config/TCPairsConfig_default @@ -151,12 +151,13 @@ diag_convert_map = [ convert(x) = x / 10; }, { source = "LSDIAG_RT"; - key = [ "CSST", "RSST", "DSST", "DSTA", "XDST", "XNST", "NSST", "NSTA", "NTMX", "NTFR", - "U200", "U20C", "V20C", "E000", "EPOS", "ENEG", "EPSS", "ENSS", "T000", "TLAT", - "TLON", "TWAC", "TWXC", "G150", "G200", "G250", "V000", "V850", "V500", "V300", - "SHDC", "SHGC", "T150", "T200", "T250", "SHRD", "SHRS", "SHRG", "HE07", "HE05", - "PW01", "PW02", "PW03", "PW04", "PW05", "PW06", "PW07", "PW08", "PW09", "PW10", - "PW11", "PW12", "PW13", "PW14", "PW15", "PW16", "PW17", "PW18", "PW20", "PW21" ]; + key = [ "LAT", "LON", "CSST", "RSST", "DSST", "DSTA", "XDST", "XNST", "NSST", "NSTA", + "NTMX", "NTFR", "U200", "U20C", "V20C", "E000", "EPOS", "ENEG", "EPSS", "ENSS", + "T000", "TLAT", "TLON", "TWAC", "TWXC", "G150", "G200", "G250", "V000", "V850", + "V500", "V300", "SHDC", "SHGC", "T150", "T200", "T250", "SHRD", "SHRS", "SHRG", + "HE07", "HE05", "PW01", "PW02", "PW03", "PW04", "PW05", "PW06", "PW07", "PW08", + "PW09", "PW10", "PW11", "PW12", "PW13", "PW14", "PW15", "PW16", "PW17", "PW18", + "PW20", "PW21" ]; convert(x) = x / 10; }, { source = "LSDIAG_RT"; diff --git a/docs/Users_Guide/tc-pairs.rst b/docs/Users_Guide/tc-pairs.rst index 4557aa90fa..3e6508446d 100644 --- a/docs/Users_Guide/tc-pairs.rst +++ b/docs/Users_Guide/tc-pairs.rst @@ -274,12 +274,13 @@ ____________________ convert(x) = x / 10; }, { source = "LSDIAG_RT"; - key = [ "CSST", "RSST", "DSST", "DSTA", "XDST", "XNST", "NSST", "NSTA", "NTMX", "NTFR", - "U200", "U20C", "V20C", "E000", "EPOS", "ENEG", "EPSS", "ENSS", "T000", "TLAT", - "TLON", "TWAC", "TWXC", "G150", "G200", "G250", "V000", "V850", "V500", "V300", - "SHDC", "SHGC", "T150", "T200", "T250", "SHRD", "SHRS", "SHRG", "HE07", "HE05", - "PW01", "PW02", "PW03", "PW04", "PW05", "PW06", "PW07", "PW08", "PW09", "PW10", - "PW11", "PW12", "PW13", "PW14", "PW15", "PW16", "PW17", "PW18", "PW20", "PW21" ]; + key = [ "LAT", "LON", "CSST", "RSST", "DSST", "DSTA", "XDST", "XNST", "NSST", "NSTA", + "NTMX", "NTFR", "U200", "U20C", "V20C", "E000", "EPOS", "ENEG", "EPSS", "ENSS", + "T000", "TLAT", "TLON", "TWAC", "TWXC", "G150", "G200", "G250", "V000", "V850", + "V500", "V300", "SHDC", "SHGC", "T150", "T200", "T250", "SHRD", "SHRS", "SHRG", + "HE07", "HE05", "PW01", "PW02", "PW03", "PW04", "PW05", "PW06", "PW07", "PW08", + "PW09", "PW10", "PW11", "PW12", "PW13", "PW14", "PW15", "PW16", "PW17", "PW18", + "PW20", "PW21" ]; convert(x) = x / 10; }, { source = "LSDIAG_RT"; diff --git a/internal/test_unit/xml/unit_tc_pairs.xml b/internal/test_unit/xml/unit_tc_pairs.xml index ec40175113..0080eb53b1 100644 --- a/internal/test_unit/xml/unit_tc_pairs.xml +++ b/internal/test_unit/xml/unit_tc_pairs.xml @@ -194,7 +194,7 @@ -adeck &DATA_DIR;/adeck/aal092022_OFCL_SHIP_AVNO.dat \ -bdeck &DATA_DIR;/bdeck/bal092022.dat \ -diag TCDIAG &DATA_DIR;/tcdiag/2022/sal092022_avno_doper_20220926*_diag.dat \ - -diag LSDIAG_RT &DATA_DIR;/lsdiag/2022/220926*AL0922_lsdiag.dat model=OFCL,SHIP \ + -diag LSDIAG_RT &DATA_DIR;/lsdiag_rt/2022/220926*AL0922_lsdiag.dat model=OFCL,SHIP \ -config &CONFIG_DIR;/TCPairsConfig_DIAGNOSTICS \ -out &OUTPUT_DIR;/tc_pairs/al092022_20220926_DIAGNOSTICS \ -log &OUTPUT_DIR;/tc_pairs/tc_pairs_DIAGNOSTICS.log \ diff --git a/src/libcode/vx_tc_util/diag_file.cc b/src/libcode/vx_tc_util/diag_file.cc index 4e7c76f551..32a37d2f07 100644 --- a/src/libcode/vx_tc_util/diag_file.cc +++ b/src/libcode/vx_tc_util/diag_file.cc @@ -39,7 +39,7 @@ static int n_lsdiag_wdth = sizeof(lsdiag_wdth)/sizeof(*lsdiag_wdth); static const int tcdiag_fill_value = 9999; static const int lsdiag_fill_value = 9999; -static const char lsdiag_skip_str[] = "MTPW,IR00,IRM1,IRM3,PC00,PCM1,PCM3,PSLV,IRXX"; +static const char lsdiag_skip_str[] = "TIME,DELV,MTPW,IR00,IRM1,IRM3,PC00,PCM1,PCM3,PSLV,IRXX"; //////////////////////////////////////////////////////////////////////// // @@ -218,7 +218,7 @@ void DiagFile::read(const DiagType source, read_tcdiag(path, model_names, convert_map); } else if(source == LSDiagRTType) { - read_lsdiagrt(path, model_names, convert_map); + read_lsdiag_rt(path, model_names, convert_map); } else { mlog << Error << "\nDiagFile::read() -> " @@ -234,8 +234,8 @@ void DiagFile::read(const DiagType source, void DiagFile::read_tcdiag(const ConcatString &path, const StringArray &model_names, const map *convert_map) { - int i, v_int; - double v_dbl; + int i; + double v_in, v_out; NumArray data; const UserFunc_1Arg *fx_ptr = 0; @@ -285,7 +285,7 @@ void DiagFile::read_tcdiag(const ConcatString &path, const StringArray &model_na NTime = atoi(dl[1]); } else if(cs == "TIME") { - for(int i=2; i *convert_map) { +void DiagFile::read_lsdiag_rt(const ConcatString &path, const StringArray &model_names, + const map *convert_map) { int i, v_int; double v_dbl; NumArray data; @@ -404,8 +406,8 @@ void DiagFile::read_lsdiagrt(const ConcatString &path, const StringArray &model_ open(path.c_str()); // Diagnostic names to ignore - StringArray skip_sa; - skip_sa.parse_css(lsdiag_skip_str); + StringArray skip_diag_sa; + skip_diag_sa.parse_css(lsdiag_skip_str); // Parse the header information from the first line DataLine dl; @@ -414,7 +416,7 @@ void DiagFile::read_lsdiagrt(const ConcatString &path, const StringArray &model_ // Check for the expected number of items if(dl.n_items() != 9 || strncasecmp(dl[8], "HEAD", strlen("HEAD") != 0)) { - mlog << Error << "\nDiagFile::open_lsdiag() -> " + mlog << Error << "\nDiagFile::read_lsdiag_rt() -> " << "unexpected header line: " << path << "\n\n"; exit(1); } @@ -432,6 +434,9 @@ void DiagFile::read_lsdiagrt(const ConcatString &path, const StringArray &model_ int hr = atoi(dl[2]); InitTime = mdyhms_to_unix(mon, day, yr, hr, 0, 0); + // Store the location of the beginning of the data + int data_start_location = in->tellg(); + // Parse time and location info while(read_fwf_line(dl, lsdiag_wdth, n_lsdiag_wdth)) { @@ -459,25 +464,28 @@ void DiagFile::read_lsdiagrt(const ConcatString &path, const StringArray &model_ Lon.add(rescale_lon(-1.0*atof(dl[i])/10.0)); } - // Finished parsing header + // Finished parsing the metadata break; } } // end while + // Rewind to the beginning of the data + in->seekg(data_start_location); + // Store the diagnostics data while(read_fwf_line(dl, lsdiag_wdth, n_lsdiag_wdth)) { // Skip empty lines if(dl.n_items() == 0) continue; - // Check the 24th column + // The 24th column contains the diagnostic name cs = dl[23]; // Strip any whitespace from the fixed-width column cs.ws_strip(); // Check for diagnostic names to skip - if(skip_sa.has(cs)) continue; + if(skip_diag_sa.has(cs)) continue; // Quit reading at the LAST line if(cs == "LAST") break; @@ -508,7 +516,7 @@ void DiagFile::read_lsdiagrt(const ConcatString &path, const StringArray &model_ // Check for the expected number of items if(NTime != data.n()) { - mlog << Error << "\nDiagFile::read_lsdiag() -> " + mlog << Error << "\nDiagFile::read_lsdiag_rt() -> " << "the number of \"" << cs << "\" diagnostic values (" << data.n() << ") does not match the expected number (" << NTime << ")!\n\n"; diff --git a/src/libcode/vx_tc_util/diag_file.h b/src/libcode/vx_tc_util/diag_file.h index 95d6341aed..6db6848870 100644 --- a/src/libcode/vx_tc_util/diag_file.h +++ b/src/libcode/vx_tc_util/diag_file.h @@ -23,15 +23,6 @@ //////////////////////////////////////////////////////////////////////// // -// Real-time LSDIAG files: -// - https://ftp.nhc.noaa.gov/atcf/lsdiag -// - Header: -// BBCC YYMMDD HH WS LAT LON 9999 BBCCYYYY -// BB is 2-letter basin name -// CC is 2-digit cyclone number -// YYMMDD HH is the initialization time -// WS is the wind speed -// // TCDIAG files: // - Add link to sample data // - Header: @@ -42,6 +33,18 @@ // BB is the 2-letter basin name // CC is the 2-digit cyclone number // +// Real-time LSDIAG files: +// - https://ftp.nhc.noaa.gov/atcf/lsdiag +// - Header: +// BBCC YYMMDD HH WS LAT LON 9999 BBCCYYYY +// BB is 2-letter basin name +// CC is 2-digit cyclone number +// YYMMDD HH is the initialization time +// WS is the wind speed +// +// Developmental LSDIAG files (not currently supported): +// - https://rammb2.cira.colostate.edu/research/tropical-cyclones/ships +// //////////////////////////////////////////////////////////////////////// class DiagFile : public LineDataFile { @@ -115,13 +118,13 @@ class DiagFile : public LineDataFile { // do stuff // - void read (const DiagType, - const ConcatString &, const StringArray &, - const std::map *); - void read_tcdiag (const ConcatString &, const StringArray &, - const std::map *); - void read_lsdiagrt (const ConcatString &, const StringArray &, - const std::map *); + void read (const DiagType, + const ConcatString &, const StringArray &, + const std::map *); + void read_tcdiag (const ConcatString &, const StringArray &, + const std::map *); + void read_lsdiag_rt (const ConcatString &, const StringArray &, + const std::map *); };