From 75f3a41b103442de70dd8c953c305a34c9d3414a Mon Sep 17 00:00:00 2001 From: Howard Soh Date: Sat, 8 Oct 2022 01:47:59 -0600 Subject: [PATCH 01/22] #2068 Added obs_to_qc_map --- data/config/IODA2NCConfig_default | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/data/config/IODA2NCConfig_default b/data/config/IODA2NCConfig_default index 2ac36c27ce..30e170c358 100644 --- a/data/config/IODA2NCConfig_default +++ b/data/config/IODA2NCConfig_default @@ -92,9 +92,18 @@ metadata_map = [ { key = "station_id"; val = "station_id,report_identifier"; }, { key = "pressure"; val = "air_pressure,pressure"; }, { key = "height"; val = "height,height_above_mean_sea_level"; }, + { key = "datetime"; val = "datetime,dateTime"; }, { key = "elevation"; val = ""; } ]; +// +// Default mapping for obs to qc. +// +obs_to_qc_map = [ + { key = "wind_from_direction"; val = "eastward_wind,northward_wind"; }, + { key = "wind_speed"; val = "eastward_wind,northward_wind"; } +]; + missing_thresh = [ <=-1e9, >=1e9, ==-9999 ]; //////////////////////////////////////////////////////////////////////////////// From 95ed64d7bf21b2cea7f98112ba25049b11676e95 Mon Sep 17 00:00:00 2001 From: Howard Soh Date: Sat, 8 Oct 2022 01:49:18 -0600 Subject: [PATCH 02/22] #2068 Support obs_to_qc_map --- src/basic/vx_config/config_constants.h | 1 + src/basic/vx_config/config_util.cc | 7 +++++++ src/basic/vx_config/config_util.h | 1 + src/tools/other/ioda2nc/ioda2nc_conf_info.cc | 1 + src/tools/other/ioda2nc/ioda2nc_conf_info.h | 1 + 5 files changed, 11 insertions(+) diff --git a/src/basic/vx_config/config_constants.h b/src/basic/vx_config/config_constants.h index 5cb03579d4..c719f72298 100644 --- a/src/basic/vx_config/config_constants.h +++ b/src/basic/vx_config/config_constants.h @@ -637,6 +637,7 @@ static const char conf_key_trunc_factor[] = "gaussian_trunc_factor"; static const char conf_key_eclv_points[] = "eclv_points"; static const char conf_key_var_name_map[] = "var_name_map"; static const char conf_key_metadata_map[] = "metadata_map"; +static const char conf_key_obs_to_qc_map[] = "obs_to_qc_map"; static const char conf_key_missing_thresh[] = "missing_thresh"; static const char conf_key_control_id[] = "control_id"; static const char conf_key_ens_member_ids[] = "ens_member_ids"; diff --git a/src/basic/vx_config/config_util.cc b/src/basic/vx_config/config_util.cc index 5c1778bf39..d5e4e592ab 100644 --- a/src/basic/vx_config/config_util.cc +++ b/src/basic/vx_config/config_util.cc @@ -1100,6 +1100,13 @@ map parse_conf_obs_name_map(Dictionary *dict) { /////////////////////////////////////////////////////////////////////////////// +map parse_conf_obs_to_qc_map(Dictionary *dict) { + const char *method_name = "parse_conf_obs_to_qc_map() -> "; + return parse_conf_key_values_map(dict, conf_key_obs_to_qc_map, method_name); +} + +/////////////////////////////////////////////////////////////////////////////// + void BootInfo::clear() { interval = BootIntervalType_None; rep_prop = bad_data_double; diff --git a/src/basic/vx_config/config_util.h b/src/basic/vx_config/config_util.h index 45de85dd0f..61280b8583 100644 --- a/src/basic/vx_config/config_util.h +++ b/src/basic/vx_config/config_util.h @@ -62,6 +62,7 @@ extern std::map extern std::map parse_conf_metadata_map(Dictionary *dict); extern std::map parse_conf_obs_name_map(Dictionary *dict); +extern std::map parse_conf_obs_to_qc_map(Dictionary *dict); extern BootInfo parse_conf_boot(Dictionary *dict); extern RegridInfo parse_conf_regrid(Dictionary *dict, bool error_out = default_dictionary_error_out); extern InterpInfo parse_conf_interp(Dictionary *dict, const char *); diff --git a/src/tools/other/ioda2nc/ioda2nc_conf_info.cc b/src/tools/other/ioda2nc/ioda2nc_conf_info.cc index 29ed7be5f4..4c8f6beb5c 100644 --- a/src/tools/other/ioda2nc/ioda2nc_conf_info.cc +++ b/src/tools/other/ioda2nc/ioda2nc_conf_info.cc @@ -170,6 +170,7 @@ void IODA2NCConfInfo::process_config() { obs_name_map = parse_conf_obs_name_map(&conf); message_type_map = parse_conf_message_type_map(&conf); metadata_map = parse_conf_metadata_map(&conf); + obs_to_qc_map = parse_conf_obs_to_qc_map(&conf); if ( sid_list ) delete [] sid_list; diff --git a/src/tools/other/ioda2nc/ioda2nc_conf_info.h b/src/tools/other/ioda2nc/ioda2nc_conf_info.h index 0f09a18abb..9c64bc03c3 100644 --- a/src/tools/other/ioda2nc/ioda2nc_conf_info.h +++ b/src/tools/other/ioda2nc/ioda2nc_conf_info.h @@ -57,6 +57,7 @@ class IODA2NCConfInfo { map obs_name_map; map message_type_map; map metadata_map; + map obs_to_qc_map; StringArray surface_message_types; TimeSummaryInfo timeSummaryInfo; From e2c72ab1e5ac82941ed9de96e013cfccc4744bf9 Mon Sep 17 00:00:00 2001 From: Howard Soh Date: Sat, 8 Oct 2022 01:50:38 -0600 Subject: [PATCH 03/22] #2068 Support group --- src/libcode/vx_nc_util/nc_utils.cc | 381 ++++++++++++++++++++++++----- src/libcode/vx_nc_util/nc_utils.h | 34 ++- 2 files changed, 344 insertions(+), 71 deletions(-) diff --git a/src/libcode/vx_nc_util/nc_utils.cc b/src/libcode/vx_nc_util/nc_utils.cc index 3441b2744e..447a8ac737 100644 --- a/src/libcode/vx_nc_util/nc_utils.cc +++ b/src/libcode/vx_nc_util/nc_utils.cc @@ -160,19 +160,87 @@ char get_att_value_char(const NcAtt *att) { //////////////////////////////////////////////////////////////////////// -bool get_att_value_chars(const NcAtt *att, ConcatString &value) { +bool get_att_value_chars(const NcAtt *att, ConcatString &value, int nc_id, int var_id) { bool status = false; static const char *method_name = "get_att_value_chars(NcAtt) -> "; if (IS_VALID_NC_P(att)) { nc_type attType = GET_NC_TYPE_ID_P(att); - if (attType == NC_CHAR || attType == NC_STRING) { + if (attType == NC_CHAR) { +cout << method_name << " DEBUG HS " << " attType=NC_CHAR\n"; + try { + char att_value[1024*8]; + att->getValues(att_value); + value = att_value; + } + catch (exceptions::NcChar ex) { + value = ""; + // Handle netCDF::exceptions::NcChar: NetCDF: Attempt to convert between text & numbers + mlog << Warning << "\n" << method_name + << "Exception: " << ex.what() << "\n" + << "Fail to read " << GET_NC_NAME_P(att) << " attribute (" + << GET_NC_TYPE_NAME_P(att) << " type).\n" + << "Please check the encoding of the "<< GET_NC_NAME_P(att) << " attribute.\n\n"; + } + } + else if (attType == NC_STRING) { + size_t att_len = att->getAttLength(); +NcGroup p_group = att->getParentGroup(); +cout << method_name << " DEBUG HS " << " attType=NC_STRING att_name="<< att->getName() << "\n"; +cout << method_name << " DEBUG HS " << " att_len=" << att_len << "\n"; +cout << method_name << " DEBUG HS " << " p_group name=" << p_group.getName() << " id=" << p_group.getId() << "\n"; + try { - string att_value; + //string att_value; + char att_value[1024*8]; att->getValues(att_value); value = att_value; } catch (exceptions::NcChar ex) { value = ""; +try { + int status, stat; + size_t attlen = 0; + + stat = nc_inq_attlen(nc_id, var_id, att->getName().c_str(), &attlen); +cout << method_name << " DEBUG HS ===================== S =============================\n"; +cout << method_name << " DEBUG HS 0 " << " attlen=" << attlen << "\n"; + + attlen = 1024; + char *title; + char **string_attr = (char**)malloc(attlen * sizeof(char*)); + memset(string_attr, 0, attlen * sizeof(char*)); +//cout << " DEBUG HS 1 " << method_name << " attlen=" << attlen << ", string_attr[0]=" << string_attr[0] << "\n"; +//cout << " DEBUG HS 1 " << method_name << " attlen=" << attlen << ", string_attr[0]=" << string_attr[0] << "\n"; +//cout << method_name << " DEBUG HS 2 attlen=" << attlen << "\n"; + +// stat = nc_get_att_string(nc_id, var_id, att->getName().c_str(), string_attr); + +title = (char *) malloc(attlen + 1); +cout << method_name << " DEBUG HS 3\n"; +memset(title, 0, attlen * sizeof(char)); +cout << method_name << " DEBUG HS 4\n"; + +status = nc_get_att_text(nc_id, var_id, att->getName().c_str(), title); +cout << method_name << " DEBUG HS " << " nc_id=" << nc_id << ", var_id=" << var_id << " attr_name=" << att->getName() << "\n"; +cout << method_name << " DEBUG HS " << " attlen=" << attlen << ", title=" << title << "\n"; +//cout << method_name << " DEBUG HS " << " attlen=" << attlen << ", string_attr=" << string_attr << "\n"; +//cout << method_name << " DEBUG HS " << " attlen=" << attlen << ", string_attr[0]=" << string_attr[0] << "\n"; +//cout << method_name << " DEBUG HS " << " attlen=" << attlen << ", string_attr[0][0]=" << string_attr[0][0] << "\n"; +cout << method_name << " DEBUG HS ===================== E ============================\n"; + value = title; + + delete [] string_attr ; + delete [] title ; +} +catch (exceptions::NcException ex) { + mlog << Warning << "\n" << method_name + << " NNNNNNN Exception: " << ex.what() << "\n" + << "Fail to read " << GET_NC_NAME_P(att) << " attribute (" + << GET_NC_TYPE_NAME_P(att) << " type).\n" + << "Please check the encoding of the "<< GET_NC_NAME_P(att) << " attribute.\n\n"; +} + + // Handle netCDF::exceptions::NcChar: NetCDF: Attempt to convert between text & numbers mlog << Warning << "\n" << method_name << "Exception: " << ex.what() << "\n" @@ -445,7 +513,7 @@ NcGroupAtt *get_nc_att(const NcFile * nc, const ConcatString &att_name, bool exi //////////////////////////////////////////////////////////////////////// bool get_nc_att_value(const NcVar *var, const ConcatString &att_name, - ConcatString &att_val, bool exit_on_error) { + ConcatString &att_val, int nc_id, bool exit_on_error) { bool status = false; NcVarAtt *att = (NcVarAtt *) 0; @@ -455,7 +523,7 @@ bool get_nc_att_value(const NcVar *var, const ConcatString &att_name, att = get_nc_att(var, att_name); // Look for a match - status = get_att_value_chars(att, att_val); + status = get_att_value_chars(att, att_val, nc_id, var->getId()); if (att) delete att; return(status); @@ -833,28 +901,49 @@ void add_att(NcVar *var, const string &att_name, const double att_val) { //////////////////////////////////////////////////////////////////////// -int get_var_names(NcFile *nc, StringArray *varNames) { +int get_var_names(NcFile *nc, StringArray *var_names) { - int i, varCount; NcVar var; + int i = 0; + int var_count = nc->getVarCount(); - varCount = nc->getVarCount(); - - i = 0; - multimap::iterator itVar; + multimap::iterator it_var; multimap mapVar = GET_NC_VARS_P(nc); - for (itVar = mapVar.begin(); itVar != mapVar.end(); ++itVar) { - var = (*itVar).second; - varNames->add(var.getName()); + for (it_var = mapVar.begin(); it_var != mapVar.end(); ++it_var) { + var = (*it_var).second; + var_names->add(var.getName()); i++; } - if (i != varCount) { + if (i != var_count) { mlog << Error << "\n\tget_var_names() -> " - << "does not match array, allocated " << varCount << " but assigned " + << "does not match array, allocated " << var_count << " but assigned " << i << ".\n\n"; } - return(varCount); + return(var_count); +} + +//////////////////////////////////////////////////////////////////////// + +int get_var_names(NcFile *nc, StringArray *var_names, StringArray &group_names) { + + NcVar var; + NcGroup nc_group; + int var_count = 0; + multimap var_map; + multimap::iterator it_var; + + for (int idx=0; idxadd(it_var->first); + var_count++; + } + } + } + return(var_count); } //////////////////////////////////////////////////////////////////////// @@ -959,9 +1048,9 @@ bool get_var_standard_name(const NcVar *var, ConcatString &att_val) { //////////////////////////////////////////////////////////////////////// -bool get_var_units(const NcVar *var, ConcatString &att_val) { +bool get_var_units(const NcVar *var, ConcatString &att_val, int nc_id) { - return(get_nc_att_value(var, units_att_name, att_val)); + return(get_nc_att_value(var, units_att_name, att_val, nc_id, var->getId())); } //////////////////////////////////////////////////////////////////////// @@ -2114,6 +2203,18 @@ bool args_ok(const LongArray & a) { } //////////////////////////////////////////////////////////////////////// +// Exit if variable does not exists + +NcGroup get_nc_group(NcFile *nc, const char *group_name) { + NcGroup nc_group; + multimap group_map = nc->getGroups(); + multimap::iterator it = group_map.find(group_name); + if (it != group_map.end()) nc_group = it->second; + return nc_group; +} + +//////////////////////////////////////////////////////////////////////// +// Exit if variable does not exists NcVar get_var(NcFile *nc, const char *var_name) { string new_var_name = var_name; @@ -2123,23 +2224,70 @@ NcVar get_var(NcFile *nc, const char *var_name) { // Retrieve the variable from the NetCDF file. // NcVar var; - multimap varMap = GET_NC_VARS_P(nc); - multimap::iterator it = varMap.find(new_var_name); - if (it != varMap.end()) { - NcVar tmpVar = it->second; - if(IS_INVALID_NC(tmpVar)) { - mlog << Error << "\nget_var() -> " - << "can't read \"" << new_var_name << "\" variable.\n\n"; - exit(1); - } + multimap var_map = GET_NC_VARS_P(nc); + multimap::iterator it = var_map.find(new_var_name); + if (it != var_map.end()) var = it->second; + + if(IS_INVALID_NC(var)) { + mlog << Error << "\nget_var(var_name) -> " + << "can't read \"" << new_var_name << "\" variable.\n\n"; + exit(1); + } + + return(var); +} + +//////////////////////////////////////////////////////////////////////// +// Exit if variable does not exists + +NcVar get_var(NcFile *nc, const ConcatString var_name) { + return get_var(nc, var_name.c_str()); +} + +//////////////////////////////////////////////////////////////////////// +// Exit if variable does not exists + +NcVar get_var(NcFile *nc, const char *var_name, const char *group_name) { + string nc_var_name; + string new_var_name = var_name; + patch_nc_name(&new_var_name); + + // + // Retrieve the variable from the NetCDF file. + // + NcVar var; + multimap var_map; + NcGroup nc_group = get_nc_group(nc, group_name); + if (IS_VALID_NC(nc_group)) { + nc_var_name = new_var_name; + var_map = nc_group.getVars(); + } + else { + // This is for IODA data format 1.0 + nc_var_name = new_var_name + "@" + group_name; + var_map = GET_NC_VARS_P(nc); + } + multimap::iterator it = var_map.find(nc_var_name); + if (it != var_map.end()) var = it->second; - var = tmpVar; + if(IS_INVALID_NC(var)) { + mlog << Error << "\nget_var(var_name, group_name) -> " + << "can't read \"" << new_var_name << "\" variable.\n\n"; + exit(1); } return(var); } //////////////////////////////////////////////////////////////////////// +// Exit if variable does not exists + +NcVar get_var(NcFile *nc, const ConcatString var_name, const char *group_name) { + return get_var(nc, var_name.c_str(), group_name); +} + +//////////////////////////////////////////////////////////////////////// +// Do not exit if variable does not exists NcVar get_nc_var(NcFile *nc, const char *var_name, bool log_as_error) { string new_var_name = var_name; @@ -2151,7 +2299,7 @@ NcVar get_nc_var(NcFile *nc, const char *var_name, bool log_as_error) { NcVar var = nc->getVar(new_var_name); if(IS_INVALID_NC(var)) { ConcatString log_message; - log_message << "\nget_nc_var(NcFile) --> The variable \"" + log_message << "\nget_nc_var(var_name) --> The variable \"" << new_var_name << "\" does not exist!\n\n"; if (log_as_error) mlog << Error << log_message; @@ -2162,6 +2310,61 @@ NcVar get_nc_var(NcFile *nc, const char *var_name, bool log_as_error) { return(var); } +//////////////////////////////////////////////////////////////////////// +// Do not exit if variable does not exists + +NcVar get_nc_var(NcFile *nc, const ConcatString var_name, bool log_as_error) { + return get_nc_var(nc, var_name.c_str(), log_as_error); +} + +/////////////////////////// +// Do not exit if variable does not exists + +NcVar get_nc_var(NcFile *nc, const char *var_name, const char *group_name, + bool log_as_error) { + string nc_var_name; + string new_var_name = var_name; + patch_nc_name(&new_var_name); + + // + // Retrieve the variable from the NetCDF file. + // + NcVar var; + multimap var_map; + NcGroup nc_group = get_nc_group(nc, group_name); + if (IS_VALID_NC(nc_group)) { + nc_var_name = new_var_name; + var_map = nc_group.getVars(); + } + else { + // This is for IODA data format 1.0 + nc_var_name = new_var_name + "@" + group_name; + var_map = GET_NC_VARS_P(nc); + } + multimap::iterator it = var_map.find(nc_var_name); + if (it != var_map.end()) var = it->second; + + if(IS_INVALID_NC(var)) { + ConcatString log_message; + log_message << "\nget_nc_var(var_name, group_name) --> The variable \"" + << nc_var_name << "\" does not exist!\n\n"; + if (log_as_error) + mlog << Error << log_message; + else + mlog << Warning << log_message; + } + + return(var); +} + +/////////////////////////// +// Do not exit if variable does not exists + +NcVar get_nc_var(NcFile *nc, const ConcatString var_name, const char *group_name, + bool log_as_error) { + return get_nc_var(nc, var_name.c_str(), group_name, log_as_error); +} + //////////////////////////////////////////////////////////////////////// void copy_nc_att_byte(NcFile *nc_to, NcGroupAtt *from_att) { @@ -2654,6 +2857,7 @@ void copy_nc_data_short(NcVar *var_from, NcVar *var_to, int data_size) { delete[] data; } +//////////////////////////////////////////////////////////////////////// void copy_nc_var_data(NcVar *var_from, NcVar *var_to) { const string method_name = "copy_nc_var_data()"; @@ -2698,13 +2902,64 @@ void copy_nc_var_dims(NcVar *var_from, NcVar *var_to) { //////////////////////////////////////////////////////////////////////// -bool has_var(NcFile *nc, const char * var_name) { - NcVar v = get_var(nc, var_name); +bool has_nc_group(NcFile *nc, const char *group_name) { + multimap group_map = nc->getGroups(); + multimap::iterator it = group_map.find(group_name); + return (it != group_map.end()); +} + +//////////////////////////////////////////////////////////////////////// + +bool has_var(NcFile *nc, const char *var_name) { + string new_var_name = var_name; + patch_nc_name(&new_var_name); + NcVar v = get_var(nc, new_var_name.c_str()); return IS_VALID_NC(v); } //////////////////////////////////////////////////////////////////////// +bool has_var(NcFile *nc, const ConcatString var_name) { + return has_var(nc, var_name.c_str()); +} + +//////////////////////////////////////////////////////////////////////// + +bool has_var(NcFile *nc, const char *var_name, const char *group_name) { + string nc_var_name; + string new_var_name = var_name; + patch_nc_name(&new_var_name); + + // + // Retrieve the variable from the NetCDF file. + // + multimap var_map; + NcGroup nc_group = get_nc_group(nc, group_name); + if (IS_VALID_NC(nc_group)) { +cout << " DEBUG HS has_var() found group " << group_name << "\n"; + nc_var_name = new_var_name; + var_map = nc_group.getVars(); + } + else { +cout << " DEBUG HS has_var() found no group " << group_name << "\n"; + // This is for IODA data format 1.0 + nc_var_name = new_var_name + "@" + group_name; + var_map = GET_NC_VARS_P(nc); + } +cout << " DEBUG HS has_var() nc_var_name = " << nc_var_name << "\n"; + multimap::iterator it = var_map.find(nc_var_name); + + return (it != var_map.end()); +} + +//////////////////////////////////////////////////////////////////////// + +bool has_var(NcFile *nc, const ConcatString var_name, const char *group_name) { + return has_var(nc, var_name.c_str(), group_name); +} + +//////////////////////////////////////////////////////////////////////// + NcVar add_var(NcFile *nc, const string &var_name, const NcType ncType, const int deflate_level) { vector ncDimVector; string new_var_name = var_name; @@ -3009,26 +3264,24 @@ vector get_dims(const NcVar *var, int *dim_count) { //////////////////////////////////////////////////////////////////////// bool is_nc_name_lat(const ConcatString name) { - bool is_latitude = (name == "lat" || name == "LAT" - || name == "Lat" || name == "Latitude" - || name == "latitude" || name == "LATITUDE"); + ConcatString name_l = to_lower(name); + bool is_latitude = (name_l == "lat" || name_l == "latitude"); return is_latitude; } //////////////////////////////////////////////////////////////////////// bool is_nc_name_lon(const ConcatString name) { - bool is_longitude = (name == "lon" || name == "LON" - || name == "Lon" || name == "Longitude" - || name == "longitude" || name == "LONGITUDE"); + ConcatString name_l = to_lower(name); + bool is_longitude = (name_l == "lon" || name_l == "longitude"); return is_longitude; } //////////////////////////////////////////////////////////////////////// bool is_nc_name_time(const ConcatString name) { - bool is_time = (name == "t" || name == "time" || name == "Time" || name == "TIME" - || name == "datetime" || name == "Datetime" || name == "DATETIME"); + ConcatString name_l = to_lower(name); + bool is_time = (name == "t" || name_l == "time" || name_l == "datetime"); return is_time; } @@ -3061,26 +3314,26 @@ NcVar get_nc_var_lat(const NcFile *nc) { multimap mapVar = GET_NC_VARS_P(nc); static const char *method_name = "get_nc_var_lat() "; - for (multimap::iterator itVar = mapVar.begin(); - itVar != mapVar.end(); ++itVar) { - ConcatString name = (*itVar).first; + for (multimap::iterator it_var = mapVar.begin(); + it_var != mapVar.end(); ++it_var) { + ConcatString name = (*it_var).first; //if (is_nc_name_lat(name)) found = true; - if (get_var_standard_name(&(*itVar).second, name)) { + if (get_var_standard_name(&(*it_var).second, name)) { if (is_nc_name_lat(name)) found = true; } - if (!found && get_var_units(&(*itVar).second, name)) { + if (!found && get_var_units(&(*it_var).second, name)) { if (is_nc_unit_latitude(name.c_str())) { - if (get_nc_att_value(&(*itVar).second, axis_att_name, name)) { + if (get_nc_att_value(&(*it_var).second, axis_att_name, name)) { if (is_nc_attr_lat(name)) found = true; } - else if (get_nc_att_value(&(*itVar).second, + else if (get_nc_att_value(&(*it_var).second, coordinate_axis_type_att_name, name)) { if (is_nc_attr_lat(name)) found = true; } } } if (found) { - var = (*itVar).second; + var = (*it_var).second; break; } } @@ -3103,26 +3356,26 @@ NcVar get_nc_var_lon(const NcFile *nc) { multimap mapVar = GET_NC_VARS_P(nc); static const char *method_name = "get_nc_var_lon() "; - for (multimap::iterator itVar = mapVar.begin(); - itVar != mapVar.end(); ++itVar) { - ConcatString name = (*itVar).first; + for (multimap::iterator it_var = mapVar.begin(); + it_var != mapVar.end(); ++it_var) { + ConcatString name = (*it_var).first; //if (is_nc_name_lon(name)) found = true; - if (get_var_standard_name(&(*itVar).second, name)) { + if (get_var_standard_name(&(*it_var).second, name)) { if (is_nc_name_lon(name)) found = true; } - if (!found && get_var_units(&(*itVar).second, name)) { + if (!found && get_var_units(&(*it_var).second, name)) { if (is_nc_unit_longitude(name.c_str())) { - if (get_nc_att_value(&(*itVar).second, axis_att_name, name)) { + if (get_nc_att_value(&(*it_var).second, axis_att_name, name)) { if (is_nc_attr_lon(name)) found = true; } - else if (get_nc_att_value(&(*itVar).second, + else if (get_nc_att_value(&(*it_var).second, coordinate_axis_type_att_name, name)) { if (is_nc_attr_lon(name)) found = true; } } } if (found) { - var = (*itVar).second; + var = (*it_var).second; break; } } @@ -3145,28 +3398,28 @@ NcVar get_nc_var_time(const NcFile *nc) { multimap mapVar = GET_NC_VARS_P(nc); static const char *method_name = "get_nc_var_time() "; - for (multimap::iterator itVar = mapVar.begin(); - itVar != mapVar.end(); ++itVar) { - ConcatString name = (*itVar).first; + for (multimap::iterator it_var = mapVar.begin(); + it_var != mapVar.end(); ++it_var) { + ConcatString name = (*it_var).first; //if (is_nc_name_time(name)) found = true; - if (get_var_standard_name(&(*itVar).second, name)) { + if (get_var_standard_name(&(*it_var).second, name)) { if (is_nc_name_time(name)) found = true; mlog << Debug(7) << method_name << "checked variable \"" << name << "\" is_time: " << found << "\n"; } - if (!found && get_var_units(&(*itVar).second, name)) { + if (!found && get_var_units(&(*it_var).second, name)) { if (is_nc_unit_time(name.c_str())) { - if (get_nc_att_value(&(*itVar).second, axis_att_name, name)) { + if (get_nc_att_value(&(*it_var).second, axis_att_name, name)) { if (is_nc_attr_time(name)) found = true; } - else if (get_nc_att_value(&(*itVar).second, + else if (get_nc_att_value(&(*it_var).second, coordinate_axis_type_att_name, name)) { if (is_nc_attr_time(name)) found = true; } } } if (found) { - var = (*itVar).second; + var = (*it_var).second; break; } } diff --git a/src/libcode/vx_nc_util/nc_utils.h b/src/libcode/vx_nc_util/nc_utils.h index 199173e263..8755b32cdc 100644 --- a/src/libcode/vx_nc_util/nc_utils.h +++ b/src/libcode/vx_nc_util/nc_utils.h @@ -161,7 +161,7 @@ extern bool get_att_value(const netCDF::NcAtt *, int &value); extern bool get_att_value(const netCDF::NcAtt *, float &value); extern bool get_att_value(const netCDF::NcAtt *, double &value); -extern bool get_att_value_chars (const netCDF::NcAtt *, ConcatString &); +extern bool get_att_value_chars (const netCDF::NcAtt *, ConcatString &, int nc_id=0, int var_id=0); extern int get_att_value_int (const netCDF::NcAtt *); extern long long get_att_value_llong (const netCDF::NcAtt *); extern double get_att_value_double (const netCDF::NcAtt *); @@ -186,7 +186,8 @@ extern bool get_nc_att_value(const netCDF::NcVarAtt *, std::string &); extern bool get_nc_att_value(const netCDF::NcVarAtt *, int &, bool exit_on_error = true); extern bool get_nc_att_value(const netCDF::NcVarAtt *, float &, bool exit_on_error = true); extern bool get_nc_att_value(const netCDF::NcVarAtt *, double &, bool exit_on_error = true); -extern bool get_nc_att_value(const netCDF::NcVar *, const ConcatString &, ConcatString &, bool exit_on_error = false); +extern bool get_nc_att_value(const netCDF::NcVar *, const ConcatString &, ConcatString &, + int nc_id=0, bool exit_on_error = false); extern bool get_nc_att_value(const netCDF::NcVar *, const ConcatString &, int &, bool exit_on_error = false); extern bool get_nc_att_value(const netCDF::NcVar *, const ConcatString &, float &, bool exit_on_error = false); extern bool get_nc_att_value(const netCDF::NcVar *, const ConcatString &, double &, bool exit_on_error = false); @@ -216,7 +217,8 @@ extern void add_att(netCDF::NcVar *, const std::string &, const int ); extern void add_att(netCDF::NcVar *, const std::string &, const float ); extern void add_att(netCDF::NcVar *, const std::string &, const double); -extern int get_var_names(netCDF::NcFile *, StringArray *varNames); +extern int get_var_names(netCDF::NcFile *, StringArray *var_names); +extern int get_var_names(netCDF::NcFile *, StringArray *var_names, StringArray &group_names); extern bool get_var_att_float (const netCDF::NcVar *, const ConcatString &, float &); extern bool get_var_att_double(const netCDF::NcVar *, const ConcatString &, double &); @@ -229,7 +231,7 @@ extern bool get_var_grid_mapping_name(const netCDF::NcVar *var, ConcatString & extern bool get_var_long_name(const netCDF::NcVar *, ConcatString &); extern double get_var_missing_value(const netCDF::NcVar *); extern bool get_var_standard_name(const netCDF::NcVar *, ConcatString &); -extern bool get_var_units(const netCDF::NcVar *, ConcatString &); +extern bool get_var_units(const netCDF::NcVar *, ConcatString &, int nc_id=0); extern bool args_ok(const LongArray &); @@ -315,8 +317,22 @@ extern bool put_nc_data_with_dims(netCDF::NcVar *, const double *data, const int extern bool put_nc_data_with_dims(netCDF::NcVar *, const double *data, const long len0, const long len1=0, const long len2=0); -extern netCDF::NcVar get_var(netCDF::NcFile *, const char * var_name); // exit if not exists -extern netCDF::NcVar get_nc_var(netCDF::NcFile *, const char * var_name, bool log_as_error=false); // continue even though not exists +extern netCDF::NcGroup get_nc_group(netCDF::NcFile *, const char *group_name); // do not exit if not exists + +extern netCDF::NcVar get_var(netCDF::NcFile *, const char *var_name); // exit if not exists +//extern netCDF::NcVar get_var(netCDF::NcFile *, const ConcatString &var_name); // exit if not exists +extern netCDF::NcVar get_var(netCDF::NcFile *, const char *var_name, + const char *group_name); // exit if not exists +//extern netCDF::NcVar get_var(netCDF::NcFile *, const ConcatString &var_name, +// const char *group_name); // exit if not exists +extern netCDF::NcVar get_nc_var(netCDF::NcFile *, const char *var_name, + bool log_as_error=false); // continue even though not exists +//extern netCDF::NcVar get_nc_var(netCDF::NcFile *, const ConcatString &var_name, +// bool log_as_error=false); // continue even though not exists +extern netCDF::NcVar get_nc_var(netCDF::NcFile *, const char *var_name, + const char *group_name, bool log_as_error=false); // continue even though not exists +//extern netCDF::NcVar get_nc_var(netCDF::NcFile *, const ConcatString &var_name, +// const char *group_name, bool log_as_error=false); // continue even though not exists extern netCDF::NcVar *copy_nc_var(netCDF::NcFile *, netCDF::NcVar *, const int deflate_level=DEF_DEFLATE_LEVEL, const bool all_attrs=true); extern void copy_nc_att(netCDF::NcFile *, netCDF::NcVar *, const ConcatString attr_name); @@ -325,7 +341,11 @@ extern void copy_nc_atts(netCDF::NcFile *, netCDF::NcFile *, const bool all_att extern void copy_nc_atts( netCDF::NcVar *, netCDF::NcVar *, const bool all_attrs=true); extern void copy_nc_var_data(netCDF::NcVar *, netCDF::NcVar *); -extern bool has_var(netCDF::NcFile *, const char * var_name); +extern bool has_nc_group(netCDF::NcFile *, const char *group_name); +extern bool has_var(netCDF::NcFile *, const char *var_name); +//extern bool has_var(netCDF::NcFile *, const ConcatString &var_name); +extern bool has_var(netCDF::NcFile *, const char *var_name, const char *group_name); +//extern bool has_var(netCDF::NcFile *, const ConcatString &var_name, const char *group_name); extern netCDF::NcVar add_var(netCDF::NcFile *, const std::string &, const netCDF::NcType, const int deflate_level=DEF_DEFLATE_LEVEL); extern netCDF::NcVar add_var(netCDF::NcFile *, const std::string &, const netCDF::NcType, const netCDF::NcDim, const int deflate_level=DEF_DEFLATE_LEVEL); From 91c111a87ec28ae505796b4ae1ec989caebdfdca Mon Sep 17 00:00:00 2001 From: Howard Soh Date: Sat, 8 Oct 2022 01:50:58 -0600 Subject: [PATCH 04/22] #2068 Support IODA v2 --- src/tools/other/ioda2nc/ioda2nc.cc | 289 +++++++++++++++++++---------- 1 file changed, 196 insertions(+), 93 deletions(-) diff --git a/src/tools/other/ioda2nc/ioda2nc.cc b/src/tools/other/ioda2nc/ioda2nc.cc index 1aaef67ba1..ab39192ca3 100644 --- a/src/tools/other/ioda2nc/ioda2nc.cc +++ b/src/tools/other/ioda2nc/ioda2nc.cc @@ -64,6 +64,14 @@ static const char *program_name = "ioda2nc"; static const int REJECT_DEBUG_LEVEL = 9; +static const char *metadata_group_name = "MetaData"; +static const char *qc_group_name = "QCFlags"; +static const char *qc_postfix = "PreQC"; +static const char *obs_group_name = "ObsValue"; +static const char *derived_obs_group_name = "DerivedObsValue"; + +enum e_ioda_format { ioda_v1, ioda_v2 }; + //////////////////////////////////////////////////////////////////////// // @@ -74,7 +82,8 @@ static const int REJECT_DEBUG_LEVEL = 9; static StringArray ioda_files; static StringArray core_dims; -static StringArray core_vars; +static StringArray core_dims_v1; +static StringArray core_meta_vars; // Output NetCDF file name static ConcatString ncfile; @@ -128,6 +137,7 @@ static void initialize(); static void process_command_line(int, char **); static void open_netcdf(); static void process_ioda_file(int); +static void process_ioda_file(NcFile *nc); static void write_netcdf_hdr_data(); static void clean_up(); @@ -154,13 +164,14 @@ static void set_valid_beg_time(const StringArray &); static void set_valid_end_time(const StringArray &); static void set_verbosity(const StringArray &); -static bool check_core_data(const bool, const bool, StringArray &, StringArray &); +static bool check_core_data(const bool, const bool, StringArray &, StringArray &, e_ioda_format); static bool check_missing_thresh(float value); static ConcatString find_meta_name(StringArray, StringArray); -static bool get_meta_data_float(NcFile *, StringArray &, const char *, float *, const int); -static bool get_meta_data_strings(NcFile *, const ConcatString, char *); -static bool get_obs_data_float(NcFile *, const ConcatString, - NcVar *, float *, int *, const int); +static bool get_meta_data_float(NcFile *, StringArray &, const char *, float *, + const int, e_ioda_format); +static bool get_meta_data_strings(NcFile *, const ConcatString, char *, e_ioda_format); +static bool get_obs_data_float(NcFile *, const ConcatString, NcVar *, + float *, int *, const int, const e_ioda_format); static bool has_postfix(std::string const &, std::string const &); //////////////////////////////////////////////////////////////////////// @@ -214,15 +225,18 @@ void initialize() { nc_point_obs.init_buffer(); core_dims.clear(); - core_dims.add("nvars"); core_dims.add("nlocs"); - //core_dims.add("nstring"); - //core_dims.add("ndatetime"); + + core_dims_v1.clear(); + core_dims_v1.add("nvars"); + core_dims_v1.add("nlocs"); + core_dims_v1.add("nstring"); + core_dims_v1.add("ndatetime"); - core_vars.clear(); - core_vars.add("datetime"); - core_vars.add("latitude"); - core_vars.add("longitude"); + core_meta_vars.clear(); + core_meta_vars.add("datetime"); + core_meta_vars.add("latitude"); + core_meta_vars.add("longitude"); summary_obs = new SummaryObs(); return; @@ -349,6 +363,8 @@ void process_ioda_file(int i_pb) { int rej_elv, rej_nobs; double x, y; + bool status; + bool number_time = false; unixtime file_ut; unixtime adjusted_file_ut; unixtime msg_ut, beg_ut, end_ut; @@ -411,28 +427,62 @@ void process_ioda_file(int i_pb) { file_name << ioda_files[i_pb]; int nrecs = 0; - StringArray var_names, dim_names; + StringArray dim_names; StringArray metadata_vars; StringArray obs_value_vars; - get_var_names(f_in, &var_names); + e_ioda_format ioda_format = ioda_v2; + int nstring, nvars; + bool error_out = true; + int nlocs = get_dim_value(f_in, "nlocs", error_out); // number of locations + + nvars = nstring = bad_data_int ; + if (! has_nc_group(f_in, obs_group_name)) ioda_format = ioda_v1; + get_dim_names(f_in, &dim_names); - for(idx=0; idx= 8) { + for(idx=0; idx= 6) { - for(idx=0; idxgetId()) && (0 < units.length())) { + parse_cf_time_string(units.c_str(), base_ut, sec_per_unit); + } + else { + mlog << Error << "\n" << method_name << "Fail to get time units from " + << GET_NC_NAME(in_hdr_vld_var) << "\n\n"; + sec_per_unit = 1; + base_ut = 300000; + } } - NcVar in_hdr_vld_var = get_var(f_in, "datetime@MetaData"); - NcVar in_hdr_lat_var = get_var(f_in, "latitude@MetaData"); - NcVar in_hdr_lon_var = get_var(f_in, "longitude@MetaData"); - - int ndatetime; - if(dim_names.has("ndatetime")) ndatetime = get_dim_value(f_in, "ndatetime", error_out); else { - NcDim datetime_dim = get_nc_dim(&in_hdr_vld_var, 1); - ndatetime = IS_VALID_NC(datetime_dim) ? get_dim_size(&datetime_dim) : nstring; - mlog << Debug(3) << "\n" << method_name - << "ndatetime dimension does not exist!\n"; + in_hdr_vld_var = get_nc_var(f_in, "datetime", metadata_group_name); + if(dim_names.has("ndatetime")) ndatetime = get_dim_value(f_in, "ndatetime", error_out); + else { + NcDim datetime_dim = get_nc_dim(&in_hdr_vld_var, 1); + ndatetime = IS_VALID_NC(datetime_dim) ? get_dim_size(&datetime_dim) : nstring; + mlog << Debug(3) << "\n" << method_name + << "ndatetime dimension does not exist!\n"; + } + mlog << Debug(5) << method_name << "dimensions: nvars=" << nvars << ", nlocs=" << nlocs + << ", nrecs=" << nrecs << ", nstring=" << nstring << ", ndatetime=" << ndatetime << "\n"; } - mlog << Debug(5) << method_name << "dimensions: nvars=" << nvars << ", nlocs=" << nlocs - << ", nrecs=" << nrecs << ", nstring=" << nstring << ", ndatetime=" << ndatetime << "\n"; npbmsg_total = npbmsg = nlocs; @@ -489,6 +549,7 @@ void process_ioda_file(int i_pb) { ? (npbmsg * nmsg_percent / 100) : nmsg; } +cout << " DEBUG " << method_name << "nlocs=" << nlocs << ", ndatetime=" << ndatetime<< "\n"; long lengths[2] = { nlocs, ndatetime }; long offsets[2] = { 0, 0 }; float *hdr_lat_arr = new float[nlocs]; @@ -496,19 +557,20 @@ void process_ioda_file(int i_pb) { float *hdr_elv_arr = new float[nlocs]; float *obs_pres_arr = new float[nlocs]; float *obs_hght_arr = new float[nlocs]; + float *hdr_time_arr = new float[nlocs]; char *hdr_vld_block = new char[nlocs*ndatetime]; char *hdr_msg_types = 0; char *hdr_station_ids = 0; vector v_qc_data; vector v_obs_data; - get_meta_data_float(f_in, metadata_vars, "pressure", obs_pres_arr, nlocs); - get_meta_data_float(f_in, metadata_vars, "height", obs_hght_arr, nlocs); - get_meta_data_float(f_in, metadata_vars, "elevation", hdr_elv_arr, nlocs); + get_meta_data_float(f_in, metadata_vars, "pressure", obs_pres_arr, nlocs, ioda_format); + get_meta_data_float(f_in, metadata_vars, "height", obs_hght_arr, nlocs, ioda_format); + get_meta_data_float(f_in, metadata_vars, "elevation", hdr_elv_arr, nlocs, ioda_format); if(has_msg_type) { hdr_msg_types = new char[nlocs*nstring]; - get_meta_data_strings(f_in, msg_type_name, hdr_msg_types); + get_meta_data_strings(f_in, msg_type_name, hdr_msg_types, ioda_format); } else { mlog << Debug(1) << method_name @@ -517,7 +579,7 @@ void process_ioda_file(int i_pb) { if(has_station_id) { hdr_station_ids = new char[nlocs*nstring]; - get_meta_data_strings(f_in, station_id_name, hdr_station_ids); + get_meta_data_strings(f_in, station_id_name, hdr_station_ids, ioda_format); } else { mlog << Debug(1) << method_name @@ -534,7 +596,10 @@ void process_ioda_file(int i_pb) { << "trouble getting longitude\n\n"; exit(1); } - if(!get_nc_data(&in_hdr_vld_var, hdr_vld_block, lengths, offsets)) { + + status = number_time ? get_nc_data(&in_hdr_vld_var, hdr_time_arr, lengths, offsets) + : get_nc_data(&in_hdr_vld_var, hdr_vld_block, lengths, offsets); + if(!status) { mlog << Error << "\n" << method_name << "trouble getting datetime\n\n"; exit(1); @@ -550,17 +615,21 @@ void process_ioda_file(int i_pb) { for(idx=0; idx 0) { @@ -620,11 +690,17 @@ void process_ioda_file(int i_pb) { } } - char valid_time[ndatetime+1]; - m_strncpy(valid_time, (const char *)(hdr_vld_block + (i_read * ndatetime)), - ndatetime, method_name_s, "valid_time", true); - valid_time[ndatetime] = 0; - msg_ut = yyyymmddThhmmss_to_unix(valid_time); + if (number_time) { + msg_ut = add_to_unixtime(base_ut, sec_per_unit, + hdr_time_arr[i_read], no_leap_year); + } + else { + char valid_time[ndatetime+1]; + m_strncpy(valid_time, (const char *)(hdr_vld_block + (i_read * ndatetime)), + ndatetime, method_name_s, "valid_time", true); + valid_time[ndatetime] = 0; + msg_ut = yyyymmddThhmmss_to_unix(valid_time); + } // Check to make sure that the message time hasn't changed // from one IODA message to the next @@ -638,8 +714,7 @@ void process_ioda_file(int i_pb) { // Check if valid_beg_ut and valid_end_ut were set on the // command line. If so, use them. If not, use beg_ds and // end_ds. - if(valid_beg_ut != (unixtime) 0 || - valid_end_ut != (unixtime) 0) { + if(valid_beg_ut != (unixtime) 0 || valid_end_ut != (unixtime) 0) { beg_ut = valid_beg_ut; end_ut = valid_end_ut; } @@ -1056,30 +1131,47 @@ bool keep_valid_time(const unixtime ut, //////////////////////////////////////////////////////////////////////// bool check_core_data(const bool has_msg_type, const bool has_station_id, - StringArray &dim_names, StringArray &metadata_vars) { + StringArray &dim_names, StringArray &metadata_vars, + e_ioda_format ioda_format) { bool is_netcdf_ready = true; static const char *method_name = "check_core_data() -> "; - - for(int idx=0; idx " - << "core dimension \"" << core_dims[idx] << "\" is missing.\n\n"; + << "core dimension \"" << t_core_dims[idx] << "\" is missing.\n\n"; is_netcdf_ready = false; } } - if(has_msg_type || has_station_id) { - if(!dim_names.has("nstring")) { - mlog << Error << "\n" << method_name << "-> " - << "core dimension \"nstring\" is missing.\n\n"; - is_netcdf_ready = false; + if (ioda_format == ioda_v1) { + if(has_msg_type || has_station_id) { + if(!dim_names.has("nstring")) { + mlog << Error << "\n" << method_name << "-> " + << "core dimension \"nstring\" is missing.\n\n"; + is_netcdf_ready = false; + } } } - for(int idx=0; idx 0) { + for (int idx2=0; idx2 < alt_names.n(); idx2++) { + if (core_meta_vars[idx] != alt_names[idx2]) { + found = metadata_vars.has(alt_names[idx2]); + if (found) break; + } + } + } + } + if(!found) { mlog << Error << "\n" << method_name << "-> " - << "core variable \"" << core_vars[idx] << "\" is missing.\n\n"; + << "core variable \"" << core_meta_vars[idx] << "\" is missing.\n\n"; is_netcdf_ready = false; } } @@ -1117,16 +1209,15 @@ ConcatString find_meta_name(StringArray metadata_names, StringArray config_names bool get_meta_data_float(NcFile *f_in, StringArray &metadata_vars, const char *metadata_key, float *metadata_buf, - const int nlocs) { + const int nlocs, e_ioda_format ioda_format) { bool status = false; static const char *method_name = "get_meta_data_float() -> "; ConcatString metadata_name = find_meta_name( metadata_vars, conf_info.metadata_map[metadata_key]); + if(metadata_name.length() > 0) { - ConcatString ioda_name = metadata_name; - ioda_name.add("@MetaData"); - NcVar meta_var = get_var(f_in, ioda_name.c_str()); + NcVar meta_var = get_var(f_in, metadata_name.c_str(), metadata_group_name); if(IS_VALID_NC(meta_var)) { status = get_nc_data(&meta_var, metadata_buf, nlocs); if(!status) mlog << Debug(3) << method_name @@ -1149,13 +1240,11 @@ bool get_meta_data_float(NcFile *f_in, StringArray &metadata_vars, //////////////////////////////////////////////////////////////////////// bool get_meta_data_strings(NcFile *f_in, const ConcatString metadata_name, - char *metadata_buf) { + char *metadata_buf, e_ioda_format ioda_format) { bool status = false; static const char *method_name = "get_meta_data_strings() -> "; - ConcatString ioda_name = metadata_name; - ioda_name.add("@MetaData"); - NcVar meta_var = get_var(f_in, ioda_name.c_str()); + NcVar meta_var = get_var(f_in, metadata_name.c_str(), metadata_group_name); if(IS_VALID_NC(meta_var)) { status = get_nc_data(&meta_var, metadata_buf); if(!status) { @@ -1171,7 +1260,7 @@ bool get_meta_data_strings(NcFile *f_in, const ConcatString metadata_name, bool get_obs_data_float(NcFile *f_in, const ConcatString var_name, NcVar *obs_var, float *obs_buf, int *qc_buf, - const int nlocs) { + const int nlocs, const e_ioda_format ioda_format) { bool status = false; static const char *method_name = "get_obs_data_float() -> "; @@ -1185,21 +1274,35 @@ bool get_obs_data_float(NcFile *f_in, const ConcatString var_name, << "trouble getting " << var_name << "\n\n"; } else mlog << Error << "\n" << method_name - << var_name << "@ObsValue does not exist!\n\n"; + << var_name << " or " << var_name << "@ObsValue does not exist!\n\n"; if(!status) exit(1); status = false; if(var_name.length() > 0) { - ConcatString ioda_name = var_name; - ioda_name.add("@PreQC"); - NcVar qc_var = get_var(f_in, ioda_name.c_str()); + ConcatString qc_name = var_name; + ConcatString qc_group = (ioda_format == ioda_v1) ? qc_postfix : qc_group_name; + if (ioda_format == ioda_v2) { + StringArray qc_names = conf_info.obs_to_qc_map[var_name]; + if (0 < qc_names.n()) { + for (int idx=0; idx Date: Mon, 10 Oct 2022 13:45:55 -0600 Subject: [PATCH 05/22] Deleted temp codes --- src/libcode/vx_nc_util/nc_utils.cc | 44 ------------------------------ 1 file changed, 44 deletions(-) diff --git a/src/libcode/vx_nc_util/nc_utils.cc b/src/libcode/vx_nc_util/nc_utils.cc index 447a8ac737..d6f50af00e 100644 --- a/src/libcode/vx_nc_util/nc_utils.cc +++ b/src/libcode/vx_nc_util/nc_utils.cc @@ -197,50 +197,6 @@ cout << method_name << " DEBUG HS " << " p_group name=" << p_group.getName() << } catch (exceptions::NcChar ex) { value = ""; -try { - int status, stat; - size_t attlen = 0; - - stat = nc_inq_attlen(nc_id, var_id, att->getName().c_str(), &attlen); -cout << method_name << " DEBUG HS ===================== S =============================\n"; -cout << method_name << " DEBUG HS 0 " << " attlen=" << attlen << "\n"; - - attlen = 1024; - char *title; - char **string_attr = (char**)malloc(attlen * sizeof(char*)); - memset(string_attr, 0, attlen * sizeof(char*)); -//cout << " DEBUG HS 1 " << method_name << " attlen=" << attlen << ", string_attr[0]=" << string_attr[0] << "\n"; -//cout << " DEBUG HS 1 " << method_name << " attlen=" << attlen << ", string_attr[0]=" << string_attr[0] << "\n"; -//cout << method_name << " DEBUG HS 2 attlen=" << attlen << "\n"; - -// stat = nc_get_att_string(nc_id, var_id, att->getName().c_str(), string_attr); - -title = (char *) malloc(attlen + 1); -cout << method_name << " DEBUG HS 3\n"; -memset(title, 0, attlen * sizeof(char)); -cout << method_name << " DEBUG HS 4\n"; - -status = nc_get_att_text(nc_id, var_id, att->getName().c_str(), title); -cout << method_name << " DEBUG HS " << " nc_id=" << nc_id << ", var_id=" << var_id << " attr_name=" << att->getName() << "\n"; -cout << method_name << " DEBUG HS " << " attlen=" << attlen << ", title=" << title << "\n"; -//cout << method_name << " DEBUG HS " << " attlen=" << attlen << ", string_attr=" << string_attr << "\n"; -//cout << method_name << " DEBUG HS " << " attlen=" << attlen << ", string_attr[0]=" << string_attr[0] << "\n"; -//cout << method_name << " DEBUG HS " << " attlen=" << attlen << ", string_attr[0][0]=" << string_attr[0][0] << "\n"; -cout << method_name << " DEBUG HS ===================== E ============================\n"; - value = title; - - delete [] string_attr ; - delete [] title ; -} -catch (exceptions::NcException ex) { - mlog << Warning << "\n" << method_name - << " NNNNNNN Exception: " << ex.what() << "\n" - << "Fail to read " << GET_NC_NAME_P(att) << " attribute (" - << GET_NC_TYPE_NAME_P(att) << " type).\n" - << "Please check the encoding of the "<< GET_NC_NAME_P(att) << " attribute.\n\n"; -} - - // Handle netCDF::exceptions::NcChar: NetCDF: Attempt to convert between text & numbers mlog << Warning << "\n" << method_name << "Exception: " << ex.what() << "\n" From bf7943b09630398b924cb78830f33921a5611c9b Mon Sep 17 00:00:00 2001 From: Howard Soh Date: Mon, 10 Oct 2022 17:07:24 -0600 Subject: [PATCH 06/22] #20768 Checks use_var_id attribute before getting variables --- src/libcode/vx_nc_obs/nc_obs_util.cc | 35 +++++++++++++++------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/src/libcode/vx_nc_obs/nc_obs_util.cc b/src/libcode/vx_nc_obs/nc_obs_util.cc index 73db28167a..8f5a019280 100644 --- a/src/libcode/vx_nc_obs/nc_obs_util.cc +++ b/src/libcode/vx_nc_obs/nc_obs_util.cc @@ -480,6 +480,9 @@ void NetcdfObsVars::read_dims_vars(NcFile *f_in) { obs_dim = get_nc_dim(f_in, nc_dim_nobs); // Observation array length hdr_dim = get_nc_dim(f_in, nc_dim_nhdr); // Header array length + use_var_id = false; + get_global_att(f_in, nc_att_use_var_id, use_var_id); + // Get netCDF header variables hdr_typ_var = get_var(f_in, nc_var_hdr_typ); // Message type (String or int) hdr_sid_var = get_var(f_in, nc_var_hdr_sid); // Station ID (String or int) @@ -503,10 +506,14 @@ void NetcdfObsVars::read_dims_vars(NcFile *f_in) { obs_arr_var = get_var(f_in, nc_var_obs_arr); } else { obs_hid_var = ncVar; // Obs. header id array - ncVar = get_var(f_in, nc_var_obs_gc); - if (!IS_INVALID_NC(ncVar)) obs_gc_var = ncVar; // Obs. grib code array - ncVar = get_var(f_in, nc_var_obs_vid); - if (!IS_INVALID_NC(ncVar)) obs_vid_var = ncVar; // Obs. variable id array + if (use_var_id) { + ncVar = get_var(f_in, nc_var_obs_vid); + if (!IS_INVALID_NC(ncVar)) obs_vid_var = ncVar; // Obs. variable id array + } + else { + ncVar = get_var(f_in, nc_var_obs_gc); + if (!IS_INVALID_NC(ncVar)) obs_gc_var = ncVar; // Obs. grib code array + } obs_lvl_var = get_var(f_in, nc_var_obs_lvl); // Obs. pressure level array obs_hgt_var = get_var(f_in, nc_var_obs_hgt); // Obs. highth array obs_val_var = get_var(f_in, nc_var_obs_val); // Obs. value array @@ -516,12 +523,14 @@ void NetcdfObsVars::read_dims_vars(NcFile *f_in) { ncVar = get_var(f_in, nc_var_obs_qty_tbl); if (!IS_INVALID_NC(ncVar)) obs_qty_tbl_var = ncVar; - ncVar = get_var(f_in, nc_var_obs_var); - if (!IS_INVALID_NC(ncVar)) obs_var = ncVar; - ncVar = get_var(f_in, nc_var_unit); - if (!IS_INVALID_NC(ncVar)) unit_var = ncVar; - ncVar = get_var(f_in, nc_var_desc); - if (!IS_INVALID_NC(ncVar)) desc_var = ncVar; + if (use_var_id) { + ncVar = get_var(f_in, nc_var_obs_var); + if (!IS_INVALID_NC(ncVar)) obs_var = ncVar; + ncVar = get_var(f_in, nc_var_unit); + if (!IS_INVALID_NC(ncVar)) unit_var = ncVar; + ncVar = get_var(f_in, nc_var_desc); + if (!IS_INVALID_NC(ncVar)) desc_var = ncVar; + } // PrepBufr only headers ncVar = get_var(f_in, nc_var_hdr_prpt_typ); @@ -531,12 +540,6 @@ void NetcdfObsVars::read_dims_vars(NcFile *f_in) { ncVar = get_var(f_in, nc_var_hdr_inst_typ); if (!IS_INVALID_NC(ncVar)) hdr_inst_typ_var = ncVar; - bool _use_var_id = false; - if (!get_global_att(f_in, nc_att_use_var_id, _use_var_id)) { - _use_var_id = IS_VALID_NC(obs_var); - } - - use_var_id = _use_var_id; } //////////////////////////////////////////////////////////////////////// From e842e5fffebb567a69e74d44aec38f7338391d53 Mon Sep 17 00:00:00 2001 From: Howard Soh Date: Mon, 10 Oct 2022 17:10:15 -0600 Subject: [PATCH 07/22] #2068 Updated comments --- src/libcode/vx_nc_util/nc_utils.cc | 79 ++++++++++++------------------ src/libcode/vx_nc_util/nc_utils.h | 22 ++++----- 2 files changed, 41 insertions(+), 60 deletions(-) diff --git a/src/libcode/vx_nc_util/nc_utils.cc b/src/libcode/vx_nc_util/nc_utils.cc index d6f50af00e..bb5fe3d1fc 100644 --- a/src/libcode/vx_nc_util/nc_utils.cc +++ b/src/libcode/vx_nc_util/nc_utils.cc @@ -166,9 +166,8 @@ bool get_att_value_chars(const NcAtt *att, ConcatString &value, int nc_id, int v if (IS_VALID_NC_P(att)) { nc_type attType = GET_NC_TYPE_ID_P(att); if (attType == NC_CHAR) { -cout << method_name << " DEBUG HS " << " attType=NC_CHAR\n"; try { - char att_value[1024*8]; + char att_value[8096]; att->getValues(att_value); value = att_value; } @@ -183,15 +182,13 @@ cout << method_name << " DEBUG HS " << " attType=NC_CHAR\n"; } } else if (attType == NC_STRING) { - size_t att_len = att->getAttLength(); + try { +size_t att_len = att->getAttLength(); NcGroup p_group = att->getParentGroup(); cout << method_name << " DEBUG HS " << " attType=NC_STRING att_name="<< att->getName() << "\n"; cout << method_name << " DEBUG HS " << " att_len=" << att_len << "\n"; cout << method_name << " DEBUG HS " << " p_group name=" << p_group.getName() << " id=" << p_group.getId() << "\n"; - - try { - //string att_value; - char att_value[1024*8]; + string att_value; att->getValues(att_value); value = att_value; } @@ -2159,7 +2156,7 @@ bool args_ok(const LongArray & a) { } //////////////////////////////////////////////////////////////////////// -// Exit if variable does not exists +// Continue even though not exists NcGroup get_nc_group(NcFile *nc, const char *group_name) { NcGroup nc_group; @@ -2170,7 +2167,7 @@ NcGroup get_nc_group(NcFile *nc, const char *group_name) { } //////////////////////////////////////////////////////////////////////// -// Exit if variable does not exists +// Exit if exists but invalid NcVar get_var(NcFile *nc, const char *var_name) { string new_var_name = var_name; @@ -2182,26 +2179,20 @@ NcVar get_var(NcFile *nc, const char *var_name) { NcVar var; multimap var_map = GET_NC_VARS_P(nc); multimap::iterator it = var_map.find(new_var_name); - if (it != var_map.end()) var = it->second; - - if(IS_INVALID_NC(var)) { - mlog << Error << "\nget_var(var_name) -> " - << "can't read \"" << new_var_name << "\" variable.\n\n"; - exit(1); + if (it != var_map.end()) { + var = it->second; + if(IS_INVALID_NC(var)) { + mlog << Error << "\nget_var(var_name) -> " + << "can't read \"" << new_var_name << "\" variable.\n\n"; + exit(1); + } } return(var); } //////////////////////////////////////////////////////////////////////// -// Exit if variable does not exists - -NcVar get_var(NcFile *nc, const ConcatString var_name) { - return get_var(nc, var_name.c_str()); -} - -//////////////////////////////////////////////////////////////////////// -// Exit if variable does not exists +// Exit if exists but invalid NcVar get_var(NcFile *nc, const char *var_name, const char *group_name) { string nc_var_name; @@ -2218,32 +2209,25 @@ NcVar get_var(NcFile *nc, const char *var_name, const char *group_name) { nc_var_name = new_var_name; var_map = nc_group.getVars(); } - else { - // This is for IODA data format 1.0 + else { // This is for IODA data format 1.0 nc_var_name = new_var_name + "@" + group_name; var_map = GET_NC_VARS_P(nc); } multimap::iterator it = var_map.find(nc_var_name); - if (it != var_map.end()) var = it->second; - - if(IS_INVALID_NC(var)) { - mlog << Error << "\nget_var(var_name, group_name) -> " - << "can't read \"" << new_var_name << "\" variable.\n\n"; - exit(1); + if (it != var_map.end()) { + var = it->second; + if(IS_INVALID_NC(var)) { + mlog << Error << "\nget_var(var_name, group_name) -> " + << "can't read \"" << new_var_name << "\" variable.\n\n"; + exit(1); + } } return(var); } //////////////////////////////////////////////////////////////////////// -// Exit if variable does not exists - -NcVar get_var(NcFile *nc, const ConcatString var_name, const char *group_name) { - return get_var(nc, var_name.c_str(), group_name); -} - -//////////////////////////////////////////////////////////////////////// -// Do not exit if variable does not exists +// Continue even though not exists NcVar get_nc_var(NcFile *nc, const char *var_name, bool log_as_error) { string new_var_name = var_name; @@ -2267,14 +2251,14 @@ NcVar get_nc_var(NcFile *nc, const char *var_name, bool log_as_error) { } //////////////////////////////////////////////////////////////////////// -// Do not exit if variable does not exists +// Continue even though not exists -NcVar get_nc_var(NcFile *nc, const ConcatString var_name, bool log_as_error) { +NcVar get_nc_var(NcFile *nc, const ConcatString &var_name, bool log_as_error) { return get_nc_var(nc, var_name.c_str(), log_as_error); } -/////////////////////////// -// Do not exit if variable does not exists +//////////////////////////////////////////////////////////////////////// +// Continue even though not exists NcVar get_nc_var(NcFile *nc, const char *var_name, const char *group_name, bool log_as_error) { @@ -2292,8 +2276,7 @@ NcVar get_nc_var(NcFile *nc, const char *var_name, const char *group_name, nc_var_name = new_var_name; var_map = nc_group.getVars(); } - else { - // This is for IODA data format 1.0 + else { // This is for IODA data format 1.0 nc_var_name = new_var_name + "@" + group_name; var_map = GET_NC_VARS_P(nc); } @@ -2313,10 +2296,10 @@ NcVar get_nc_var(NcFile *nc, const char *var_name, const char *group_name, return(var); } -/////////////////////////// -// Do not exit if variable does not exists +//////////////////////////////////////////////////////////////////////// +// Continue even though not exists -NcVar get_nc_var(NcFile *nc, const ConcatString var_name, const char *group_name, +NcVar get_nc_var(NcFile *nc, const ConcatString &var_name, const char *group_name, bool log_as_error) { return get_nc_var(nc, var_name.c_str(), group_name, log_as_error); } diff --git a/src/libcode/vx_nc_util/nc_utils.h b/src/libcode/vx_nc_util/nc_utils.h index 8755b32cdc..12b997dc17 100644 --- a/src/libcode/vx_nc_util/nc_utils.h +++ b/src/libcode/vx_nc_util/nc_utils.h @@ -317,24 +317,22 @@ extern bool put_nc_data_with_dims(netCDF::NcVar *, const double *data, const int extern bool put_nc_data_with_dims(netCDF::NcVar *, const double *data, const long len0, const long len1=0, const long len2=0); -extern netCDF::NcGroup get_nc_group(netCDF::NcFile *, const char *group_name); // do not exit if not exists +extern netCDF::NcGroup get_nc_group(netCDF::NcFile *, const char *group_name); // continue even though not exists -extern netCDF::NcVar get_var(netCDF::NcFile *, const char *var_name); // exit if not exists -//extern netCDF::NcVar get_var(netCDF::NcFile *, const ConcatString &var_name); // exit if not exists +extern netCDF::NcVar get_var(netCDF::NcFile *, const char *var_name); // exit if exists but invalid extern netCDF::NcVar get_var(netCDF::NcFile *, const char *var_name, - const char *group_name); // exit if not exists -//extern netCDF::NcVar get_var(netCDF::NcFile *, const ConcatString &var_name, -// const char *group_name); // exit if not exists + const char *group_name); // continue even though not exists extern netCDF::NcVar get_nc_var(netCDF::NcFile *, const char *var_name, - bool log_as_error=false); // continue even though not exists -//extern netCDF::NcVar get_nc_var(netCDF::NcFile *, const ConcatString &var_name, -// bool log_as_error=false); // continue even though not exists + bool log_as_error=false); // continue even though not exists +extern netCDF::NcVar get_nc_var(netCDF::NcFile *, const ConcatString &var_name, + bool log_as_error=false); // continue even though not exists extern netCDF::NcVar get_nc_var(netCDF::NcFile *, const char *var_name, const char *group_name, bool log_as_error=false); // continue even though not exists -//extern netCDF::NcVar get_nc_var(netCDF::NcFile *, const ConcatString &var_name, -// const char *group_name, bool log_as_error=false); // continue even though not exists +extern netCDF::NcVar get_nc_var(netCDF::NcFile *, const ConcatString &var_name, + const char *group_name, bool log_as_error=false); // continue even though not exists -extern netCDF::NcVar *copy_nc_var(netCDF::NcFile *, netCDF::NcVar *, const int deflate_level=DEF_DEFLATE_LEVEL, const bool all_attrs=true); +extern netCDF::NcVar *copy_nc_var(netCDF::NcFile *, netCDF::NcVar *, + const int deflate_level=DEF_DEFLATE_LEVEL, const bool all_attrs=true); extern void copy_nc_att(netCDF::NcFile *, netCDF::NcVar *, const ConcatString attr_name); extern void copy_nc_att( netCDF::NcVar *, netCDF::NcVar *, const ConcatString attr_name); extern void copy_nc_atts(netCDF::NcFile *, netCDF::NcFile *, const bool all_attrs=true); From a531011404a979a13e11e4331a9cd77a2071733a Mon Sep 17 00:00:00 2001 From: Howard Soh Date: Wed, 12 Oct 2022 23:41:26 -0600 Subject: [PATCH 08/22] #2068 calls get_cf_conventions --- src/libcode/vx_data2d_factory/is_netcdf_file.cc | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/libcode/vx_data2d_factory/is_netcdf_file.cc b/src/libcode/vx_data2d_factory/is_netcdf_file.cc index 3e72029514..b14fa51fa6 100644 --- a/src/libcode/vx_data2d_factory/is_netcdf_file.cc +++ b/src/libcode/vx_data2d_factory/is_netcdf_file.cc @@ -40,9 +40,6 @@ static const char netcdf_magic [] = "CDF"; static const char hdf_magic [] = "HDF"; static const int netcdf_magic_len = m_strlen(netcdf_magic); -static const string nccf_att_name = "Conventions"; -static const string nccf_att_name_l = "conventions"; -static const string nccf_att_name_U = "CONVENTIONS"; static const string nccf_att_value = "CF-"; static const string nccf_att_value2 = "CF "; static const string nccf_att_value3 = "COARDS"; @@ -96,9 +93,7 @@ bool is_nccf_file(const char * filename) NcFile *nc_file = open_ncfile(filename); if (!IS_INVALID_NC_P(nc_file)) { - bool found = get_global_att(nc_file, nccf_att_name, att_val); - if (!found) found = get_global_att(nc_file, nccf_att_name_l, att_val); - if (!found) found = get_global_att(nc_file, nccf_att_name_U, att_val); + bool found = get_cf_conventions(nc_file, att_val); // "Conventions" attrribute if (found) { status = (att_val.compare(0, nccf_att_value.length(), nccf_att_value) == 0 || From 8e651efe458c06ee1968c6f208bafec1a4bd9261 Mon Sep 17 00:00:00 2001 From: Howard Soh Date: Mon, 17 Oct 2022 15:46:12 -0600 Subject: [PATCH 09/22] Added metadata names for elevation --- data/config/IODA2NCConfig_default | 2 +- docs/Users_Guide/reformat_point.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/data/config/IODA2NCConfig_default b/data/config/IODA2NCConfig_default index 30e170c358..124854005e 100644 --- a/data/config/IODA2NCConfig_default +++ b/data/config/IODA2NCConfig_default @@ -93,7 +93,7 @@ metadata_map = [ { key = "pressure"; val = "air_pressure,pressure"; }, { key = "height"; val = "height,height_above_mean_sea_level"; }, { key = "datetime"; val = "datetime,dateTime"; }, - { key = "elevation"; val = ""; } + { key = "elevation"; val = "elevation,station_elevation"; } ]; // diff --git a/docs/Users_Guide/reformat_point.rst b/docs/Users_Guide/reformat_point.rst index 29ac65e53c..57d3e29a40 100644 --- a/docs/Users_Guide/reformat_point.rst +++ b/docs/Users_Guide/reformat_point.rst @@ -947,7 +947,7 @@ _____________________ { key = "station_id"; val = "station_id,report_identifier"; }, { key = "pressure"; val = "air_pressure,pressure"; }, { key = "height"; val = "height,height_above_mean_sea_level"; }, - { key = "elevation"; val = ""; } + { key = "elevation"; val = "elevation,station_elevation"; } ]; This entry is an array of dictionaries, each containing a **key** string and **val** string which define a mapping of metadata for IODA data files. From a022a80ccf7810b892fa4f18dd42e3c268a06c14 Mon Sep 17 00:00:00 2001 From: Howard Soh Date: Mon, 17 Oct 2022 15:47:12 -0600 Subject: [PATCH 10/22] #2068 Added unit tests for format v2 --- internal/test_unit/xml/unit_ioda2nc.xml | 38 +++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/internal/test_unit/xml/unit_ioda2nc.xml b/internal/test_unit/xml/unit_ioda2nc.xml index 91f2a7cbb4..55165568bd 100644 --- a/internal/test_unit/xml/unit_ioda2nc.xml +++ b/internal/test_unit/xml/unit_ioda2nc.xml @@ -83,4 +83,42 @@ + + &MET_BIN;/ioda2nc + + STATION_ID + MASK_GRID + MASK_POLY + MESSAGE_TYPE + + \ + &DATA_DIR_OBS;/ioda/jopa_satwind_20210701T1200Z_out_0000_reduced.nc4 \ + &OUTPUT_DIR;/ioda2nc/jopa_satwind_20210701T1200Z_int_datetime.nc \ + -config &CONFIG_DIR;/IODA2NCConfig_mask \ + -v 2 + + + &OUTPUT_DIR;/ioda2nc/jopa_satwind_20210701T1200Z_int_datetime.nc + + + + + &MET_BIN;/ioda2nc + + STATION_ID + MASK_GRID + MASK_POLY + MESSAGE_TYPE + + \ + &DATA_DIR_OBS;/ioda/2021081612_sonde_small.nc \ + &OUTPUT_DIR;/ioda2nc/2021081612_sonde_small_sid.nc \ + -config &CONFIG_DIR;/IODA2NCConfig_mask \ + -v 2 + + + &OUTPUT_DIR;/ioda2nc/2021081612_sonde_small_sid.nc + + + From 434fd83d943ae7d8e5c271ca78cedd5b67270850 Mon Sep 17 00:00:00 2001 From: Howard Soh Date: Mon, 17 Oct 2022 16:07:06 -0600 Subject: [PATCH 11/22] #2068 Added get_nc_data for string type variable and get_cf_conventions for string type attribute --- src/libcode/vx_nc_util/nc_utils.cc | 89 +++++++++++++++++++----------- src/libcode/vx_nc_util/nc_utils.h | 12 ++-- 2 files changed, 66 insertions(+), 35 deletions(-) diff --git a/src/libcode/vx_nc_util/nc_utils.cc b/src/libcode/vx_nc_util/nc_utils.cc index bb5fe3d1fc..f69ec6f493 100644 --- a/src/libcode/vx_nc_util/nc_utils.cc +++ b/src/libcode/vx_nc_util/nc_utils.cc @@ -160,7 +160,7 @@ char get_att_value_char(const NcAtt *att) { //////////////////////////////////////////////////////////////////////// -bool get_att_value_chars(const NcAtt *att, ConcatString &value, int nc_id, int var_id) { +bool get_att_value_chars(const NcAtt *att, ConcatString &value) { bool status = false; static const char *method_name = "get_att_value_chars(NcAtt) -> "; if (IS_VALID_NC_P(att)) { @@ -183,23 +183,29 @@ bool get_att_value_chars(const NcAtt *att, ConcatString &value, int nc_id, int v } else if (attType == NC_STRING) { try { -size_t att_len = att->getAttLength(); -NcGroup p_group = att->getParentGroup(); -cout << method_name << " DEBUG HS " << " attType=NC_STRING att_name="<< att->getName() << "\n"; -cout << method_name << " DEBUG HS " << " att_len=" << att_len << "\n"; -cout << method_name << " DEBUG HS " << " p_group name=" << p_group.getName() << " id=" << p_group.getId() << "\n"; string att_value; att->getValues(att_value); value = att_value; } catch (exceptions::NcChar ex) { - value = ""; - // Handle netCDF::exceptions::NcChar: NetCDF: Attempt to convert between text & numbers - mlog << Warning << "\n" << method_name - << "Exception: " << ex.what() << "\n" - << "Fail to read " << GET_NC_NAME_P(att) << " attribute (" - << GET_NC_TYPE_NAME_P(att) << " type).\n" - << "Please check the encoding of the "<< GET_NC_NAME_P(att) << " attribute.\n\n"; + int num_elements_sub = 8096; + int num_elements = att->getAttLength();; + char *att_value[num_elements]; + for (int i = 0; i < num_elements; i++ ) { + att_value[i] = (char*) calloc(num_elements_sub, sizeof(char)); + } + try { + att->getValues(att_value); + value = att_value[0]; + } + catch (exceptions::NcException ex) { + mlog << Warning << "\n" << method_name + << "Exception: " << ex.what() << "\n" + << "Fail to read " << GET_NC_NAME_P(att) << " attribute (" + << GET_NC_TYPE_NAME_P(att) << " type).\n" + << "Please check the encoding of the "<< GET_NC_NAME_P(att) << " attribute.\n\n"; + } + for (int i = 0; i < num_elements; i++ ) delete att_value[i]; } } else { // MET-788: to handle a custom modified NetCDF @@ -357,6 +363,23 @@ bool get_att_no_leap_year(const NcVar *var) { return no_leap_year; } +//////////////////////////////////////////////////////////////////////// + +bool get_cf_conventions(const netCDF::NcFile *nc, ConcatString& conventions_value) { + bool has_attr = false; + multimap::iterator it_att; + multimap map_attrs = nc->getAtts(); + for (it_att = map_attrs.begin(); it_att != map_attrs.end(); it_att++) { + if (to_lower(it_att->first) == to_lower(cf_att_name)) { + NcGroupAtt cf_att = it_att->second; + if (IS_VALID_NC(cf_att)) + has_attr = get_att_value_chars(&cf_att, conventions_value); + } + } + return has_attr; +} + + //////////////////////////////////////////////////////////////////////// ConcatString get_log_msg_for_att(const NcVarAtt *att) { @@ -375,7 +398,7 @@ ConcatString get_log_msg_for_att(const NcVarAtt *att) { //////////////////////////////////////////////////////////////////////// ConcatString get_log_msg_for_att(const NcVarAtt *att, string var_name, - const ConcatString att_name) { + const ConcatString att_name) { ConcatString log_msg; log_msg << "can't read attribute" << " \"" << ((att_name.length() > 0) ? att_name.c_str() : GET_SAFE_NC_NAME_P(att)) @@ -466,7 +489,7 @@ NcGroupAtt *get_nc_att(const NcFile * nc, const ConcatString &att_name, bool exi //////////////////////////////////////////////////////////////////////// bool get_nc_att_value(const NcVar *var, const ConcatString &att_name, - ConcatString &att_val, int nc_id, bool exit_on_error) { + ConcatString &att_val, int grp_id, bool exit_on_error) { bool status = false; NcVarAtt *att = (NcVarAtt *) 0; @@ -476,7 +499,7 @@ bool get_nc_att_value(const NcVar *var, const ConcatString &att_name, att = get_nc_att(var, att_name); // Look for a match - status = get_att_value_chars(att, att_val, nc_id, var->getId()); + status = get_att_value_chars(att, att_val); if (att) delete att; return(status); @@ -860,9 +883,9 @@ int get_var_names(NcFile *nc, StringArray *var_names) { int i = 0; int var_count = nc->getVarCount(); - multimap::iterator it_var; multimap mapVar = GET_NC_VARS_P(nc); - for (it_var = mapVar.begin(); it_var != mapVar.end(); ++it_var) { + for (multimap::iterator it_var = mapVar.begin(); + it_var != mapVar.end(); ++it_var) { var = (*it_var).second; var_names->add(var.getName()); i++; @@ -889,7 +912,7 @@ int get_var_names(NcFile *nc, StringArray *var_names, StringArray &group_names) for (int idx=0; idxadd(it_var->first); var_count++; @@ -1001,9 +1024,9 @@ bool get_var_standard_name(const NcVar *var, ConcatString &att_val) { //////////////////////////////////////////////////////////////////////// -bool get_var_units(const NcVar *var, ConcatString &att_val, int nc_id) { +bool get_var_units(const NcVar *var, ConcatString &att_val) { - return(get_nc_att_value(var, units_att_name, att_val, nc_id, var->getId())); + return(get_nc_att_value(var, units_att_name, att_val)); } //////////////////////////////////////////////////////////////////////// @@ -1699,6 +1722,14 @@ bool get_nc_data(NcVar *var, char *data) { //////////////////////////////////////////////////////////////////////// +bool get_nc_data(NcVar *var, char **data) { + bool return_status = get_nc_data_t(var, data); + + return(return_status); +} + +//////////////////////////////////////////////////////////////////////// + bool get_nc_data(NcVar *var, uchar *data) { bool return_status = false; int data_type = GET_NC_TYPE_ID_P(var); @@ -2207,7 +2238,7 @@ NcVar get_var(NcFile *nc, const char *var_name, const char *group_name) { NcGroup nc_group = get_nc_group(nc, group_name); if (IS_VALID_NC(nc_group)) { nc_var_name = new_var_name; - var_map = nc_group.getVars(); + var_map = GET_NC_VARS(nc_group); } else { // This is for IODA data format 1.0 nc_var_name = new_var_name + "@" + group_name; @@ -2274,7 +2305,7 @@ NcVar get_nc_var(NcFile *nc, const char *var_name, const char *group_name, NcGroup nc_group = get_nc_group(nc, group_name); if (IS_VALID_NC(nc_group)) { nc_var_name = new_var_name; - var_map = nc_group.getVars(); + var_map = GET_NC_VARS(nc_group); } else { // This is for IODA data format 1.0 nc_var_name = new_var_name + "@" + group_name; @@ -2648,8 +2679,8 @@ void copy_nc_atts(NcFile *nc_from, NcFile *nc_to, const bool all_attrs) { for (multimap::iterator itr = ncAttMap.begin(); itr != ncAttMap.end(); ++itr) { if (all_attrs || - ( (itr->first != "Conventions") - && (itr->first != "missing_value") ) ) { + ( (itr->first != cf_att_name) + && (itr->first != missing_value_att_name) ) ) { NcGroupAtt *from_att = &(itr->second); int dataType = GET_NC_TYPE_ID_P(from_att); switch (dataType) { @@ -2875,17 +2906,13 @@ bool has_var(NcFile *nc, const char *var_name, const char *group_name) { multimap var_map; NcGroup nc_group = get_nc_group(nc, group_name); if (IS_VALID_NC(nc_group)) { -cout << " DEBUG HS has_var() found group " << group_name << "\n"; nc_var_name = new_var_name; - var_map = nc_group.getVars(); + var_map = GET_NC_VARS(nc_group); } - else { -cout << " DEBUG HS has_var() found no group " << group_name << "\n"; - // This is for IODA data format 1.0 + else { // This is for IODA data format 1.0 nc_var_name = new_var_name + "@" + group_name; var_map = GET_NC_VARS_P(nc); } -cout << " DEBUG HS has_var() nc_var_name = " << nc_var_name << "\n"; multimap::iterator it = var_map.find(nc_var_name); return (it != var_map.end()); diff --git a/src/libcode/vx_nc_util/nc_utils.h b/src/libcode/vx_nc_util/nc_utils.h index 12b997dc17..2ee19473ff 100644 --- a/src/libcode/vx_nc_util/nc_utils.h +++ b/src/libcode/vx_nc_util/nc_utils.h @@ -133,6 +133,7 @@ static const std::string axis_att_name = "axis"; static const std::string bounds_att_name = "bounds"; static const std::string coordinates_att_name = "coordinates"; static const std::string coordinate_axis_type_att_name = "_CoordinateAxisType"; +static const std::string cf_att_name = "Conventions"; static const std::string description_att_name = "description"; static const std::string fill_value_att_name = "_FillValue"; static const std::string grid_mapping_att_name = "grid_mapping"; @@ -161,7 +162,7 @@ extern bool get_att_value(const netCDF::NcAtt *, int &value); extern bool get_att_value(const netCDF::NcAtt *, float &value); extern bool get_att_value(const netCDF::NcAtt *, double &value); -extern bool get_att_value_chars (const netCDF::NcAtt *, ConcatString &, int nc_id=0, int var_id=0); +extern bool get_att_value_chars (const netCDF::NcAtt *, ConcatString &); extern int get_att_value_int (const netCDF::NcAtt *); extern long long get_att_value_llong (const netCDF::NcAtt *); extern double get_att_value_double (const netCDF::NcAtt *); @@ -179,8 +180,10 @@ extern long long get_att_value_llong (const netCDF::NcFile *, const ConcatString extern double get_att_value_double(const netCDF::NcFile *, const ConcatString& ); extern bool get_att_no_leap_year(const netCDF::NcVar *); -extern netCDF::NcVarAtt *get_nc_att(const netCDF::NcVar *, const ConcatString &, bool exit_on_error = false); -extern netCDF::NcGroupAtt *get_nc_att(const netCDF::NcFile *, const ConcatString &, bool exit_on_error = false); +extern bool get_cf_conventions(const netCDF::NcFile *, ConcatString&); + +extern netCDF::NcVarAtt *get_nc_att(const netCDF::NcVar *, const ConcatString &, bool exit_on_error = false); +extern netCDF::NcGroupAtt *get_nc_att(const netCDF::NcFile *, const ConcatString &, bool exit_on_error = false); extern bool get_nc_att_value(const netCDF::NcVarAtt *, std::string &); extern bool get_nc_att_value(const netCDF::NcVarAtt *, int &, bool exit_on_error = true); @@ -231,7 +234,7 @@ extern bool get_var_grid_mapping_name(const netCDF::NcVar *var, ConcatString & extern bool get_var_long_name(const netCDF::NcVar *, ConcatString &); extern double get_var_missing_value(const netCDF::NcVar *); extern bool get_var_standard_name(const netCDF::NcVar *, ConcatString &); -extern bool get_var_units(const netCDF::NcVar *, ConcatString &, int nc_id=0); +extern bool get_var_units(const netCDF::NcVar *, ConcatString &); extern bool args_ok(const LongArray &); @@ -252,6 +255,7 @@ extern ConcatString* get_string_val(netCDF::NcVar *var, const int index, const i extern bool get_nc_data(netCDF::NcVar *, int *data); extern bool get_nc_data(netCDF::NcVar *, char *data); +extern bool get_nc_data(netCDF::NcVar *, char **data); extern bool get_nc_data(netCDF::NcVar *, uchar *data); extern bool get_nc_data(netCDF::NcVar *, float *data); extern bool get_nc_data(netCDF::NcVar *, double *data); From bc09238291aea0e7122494ad59f2e6c2f656c175 Mon Sep 17 00:00:00 2001 From: Howard Soh Date: Mon, 17 Oct 2022 16:07:54 -0600 Subject: [PATCH 12/22] Added datetime --- src/basic/vx_config/config_constants.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/basic/vx_config/config_constants.h b/src/basic/vx_config/config_constants.h index c719f72298..5bdfdd4fc2 100644 --- a/src/basic/vx_config/config_constants.h +++ b/src/basic/vx_config/config_constants.h @@ -885,6 +885,7 @@ static const char conf_key_do_polylines_flag [] = "do_polylines"; // PB2NC specific parameter key names // +static const char conf_key_datetime[] = "datetime"; static const char conf_key_station_id[] = "station_id"; static const char conf_key_elevation_range[] = "elevation_range"; static const char conf_key_pb_report_type[] = "pb_report_type"; From 05410125d46a3b2faf8ef670785c95d3d268ab15 Mon Sep 17 00:00:00 2001 From: Howard Soh Date: Mon, 17 Oct 2022 16:25:44 -0600 Subject: [PATCH 13/22] #2068 Support string data type valid times --- src/tools/other/ioda2nc/ioda2nc.cc | 194 ++++++++++++++++++++--------- 1 file changed, 132 insertions(+), 62 deletions(-) diff --git a/src/tools/other/ioda2nc/ioda2nc.cc b/src/tools/other/ioda2nc/ioda2nc.cc index ab39192ca3..eb766c42c5 100644 --- a/src/tools/other/ioda2nc/ioda2nc.cc +++ b/src/tools/other/ioda2nc/ioda2nc.cc @@ -63,6 +63,7 @@ static const char * DEF_CONFIG_NAME = "MET_BASE/config/IODA2NCConfig_default"; static const char *program_name = "ioda2nc"; static const int REJECT_DEBUG_LEVEL = 9; +static const int string_data_len = 512; static const char *metadata_group_name = "MetaData"; static const char *qc_group_name = "QCFlags"; @@ -168,8 +169,9 @@ static bool check_core_data(const bool, const bool, StringArray &, StringArray & static bool check_missing_thresh(float value); static ConcatString find_meta_name(StringArray, StringArray); static bool get_meta_data_float(NcFile *, StringArray &, const char *, float *, - const int, e_ioda_format); -static bool get_meta_data_strings(NcFile *, const ConcatString, char *, e_ioda_format); + const int); +static bool get_meta_data_strings(NcVar &, char *); +static bool get_meta_data_strings(NcVar &, char **); static bool get_obs_data_float(NcFile *, const ConcatString, NcVar *, float *, int *, const int, const e_ioda_format); static bool has_postfix(std::string const &, std::string const &); @@ -231,7 +233,7 @@ void initialize() { core_dims_v1.add("nvars"); core_dims_v1.add("nlocs"); core_dims_v1.add("nstring"); - core_dims_v1.add("ndatetime"); + //core_dims_v1.add("ndatetime"); core_meta_vars.clear(); core_meta_vars.add("datetime"); @@ -364,7 +366,8 @@ void process_ioda_file(int i_pb) { double x, y; bool status; - bool number_time = false; + bool is_time_offset = false; + bool is_time_string = false; unixtime file_ut; unixtime adjusted_file_ut; unixtime msg_ut, beg_ut, end_ut; @@ -427,15 +430,16 @@ void process_ioda_file(int i_pb) { file_name << ioda_files[i_pb]; int nrecs = 0; + int nstring, nvars; StringArray dim_names; StringArray metadata_vars; StringArray obs_value_vars; - e_ioda_format ioda_format = ioda_v2; - int nstring, nvars; bool error_out = true; + e_ioda_format ioda_format = ioda_v2; int nlocs = get_dim_value(f_in, "nlocs", error_out); // number of locations - nvars = nstring = bad_data_int ; + nvars = bad_data_int ; + nstring = string_data_len; if (! has_nc_group(f_in, obs_group_name)) ioda_format = ioda_v1; get_dim_names(f_in, &dim_names); @@ -446,7 +450,7 @@ void process_ioda_file(int i_pb) { if(has_postfix(var_names[idx], "@MetaData")) { metadata_vars.add(var_names[idx].substr(0, var_names[idx].find('@'))); } - if(has_postfix(var_names[idx], "@ObsValue")) { + if(has_postfix(var_names[idx], "@ObsValue") || has_postfix(var_names[idx], "@DerivedObsValue")) { obs_value_vars.add(var_names[idx].substr(0, var_names[idx].find('@'))); } } @@ -482,13 +486,15 @@ void process_ioda_file(int i_pb) { for(idx=0; idxgetId()) && (0 < units.length())) { - parse_cf_time_string(units.c_str(), base_ut, sec_per_unit); - } - else { - mlog << Error << "\n" << method_name << "Fail to get time units from " - << GET_NC_NAME(in_hdr_vld_var) << "\n\n"; - sec_per_unit = 1; - base_ut = 300000; - } + if (IS_INVALID_NC(in_hdr_vld_var)) { + mlog << Error << "\n" << method_name << "Fail to get datetime variable\n\n"; + exit(-1); } else { - in_hdr_vld_var = get_nc_var(f_in, "datetime", metadata_group_name); - if(dim_names.has("ndatetime")) ndatetime = get_dim_value(f_in, "ndatetime", error_out); + + if (NC_STRING == GET_NC_TYPE_ID(in_hdr_vld_var)) { + is_time_string = true; + } + else if (NC_CHAR == GET_NC_TYPE_ID(in_hdr_vld_var)) { + if(dim_names.has("ndatetime")) ndatetime = get_dim_value(f_in, "ndatetime", error_out); + else { + NcDim datetime_dim = get_nc_dim(&in_hdr_vld_var, 1); + ndatetime = IS_VALID_NC(datetime_dim) ? get_dim_size(&datetime_dim) : nstring; + mlog << Debug(3) << "\n" << method_name + << "ndatetime dimension does not exist!\n"; + } + mlog << Debug(5) << method_name << "dimensions: nvars=" << nvars << ", nlocs=" << nlocs + << ", nrecs=" << nrecs << ", nstring=" << nstring << ", ndatetime=" << ndatetime << "\n"; + } else { - NcDim datetime_dim = get_nc_dim(&in_hdr_vld_var, 1); - ndatetime = IS_VALID_NC(datetime_dim) ? get_dim_size(&datetime_dim) : nstring; - mlog << Debug(3) << "\n" << method_name - << "ndatetime dimension does not exist!\n"; + ConcatString units; + is_time_offset = true; + no_leap_year = get_att_no_leap_year(&in_hdr_vld_var); + if (get_var_units(&in_hdr_vld_var, units) && (0 < units.length())) { + parse_cf_time_string(units.c_str(), base_ut, sec_per_unit); + } + else { + mlog << Error << "\n" << method_name << "Fail to get time units from " + << GET_NC_NAME(in_hdr_vld_var) << "\n\n"; + exit(-1); + } } - mlog << Debug(5) << method_name << "dimensions: nvars=" << nvars << ", nlocs=" << nlocs - << ", nrecs=" << nrecs << ", nstring=" << nstring << ", ndatetime=" << ndatetime << "\n"; } npbmsg_total = npbmsg = nlocs; @@ -546,12 +561,9 @@ void process_ioda_file(int i_pb) { // are enough present. if(nmsg > 0 && nmsg < npbmsg) { npbmsg = (nmsg_percent > 0 && nmsg_percent <= 100) - ? (npbmsg * nmsg_percent / 100) : nmsg; + ? (npbmsg * nmsg_percent / 100) : nmsg; } -cout << " DEBUG " << method_name << "nlocs=" << nlocs << ", ndatetime=" << ndatetime<< "\n"; - long lengths[2] = { nlocs, ndatetime }; - long offsets[2] = { 0, 0 }; float *hdr_lat_arr = new float[nlocs]; float *hdr_lon_arr = new float[nlocs]; float *hdr_elv_arr = new float[nlocs]; @@ -559,18 +571,34 @@ cout << " DEBUG " << method_name << "nlocs=" << nlocs << ", ndatetime=" << ndate float *obs_hght_arr = new float[nlocs]; float *hdr_time_arr = new float[nlocs]; char *hdr_vld_block = new char[nlocs*ndatetime]; - char *hdr_msg_types = 0; - char *hdr_station_ids = 0; + char *hdr_msg_types = NULL; + char *hdr_station_ids = NULL; + char **hdr_vld_block2 = NULL; + char **hdr_msg_types2 = NULL; + char **hdr_station_ids2 = NULL; vector v_qc_data; vector v_obs_data; + + if (is_time_string) { + hdr_vld_block2 = (char**) calloc(nlocs, sizeof(char*)); + for (int i=0; i "; @@ -1239,17 +1309,15 @@ bool get_meta_data_float(NcFile *f_in, StringArray &metadata_vars, //////////////////////////////////////////////////////////////////////// -bool get_meta_data_strings(NcFile *f_in, const ConcatString metadata_name, - char *metadata_buf, e_ioda_format ioda_format) { +bool get_meta_data_strings(NcVar &var, char *metadata_buf) { bool status = false; static const char *method_name = "get_meta_data_strings() -> "; - NcVar meta_var = get_var(f_in, metadata_name.c_str(), metadata_group_name); - if(IS_VALID_NC(meta_var)) { - status = get_nc_data(&meta_var, metadata_buf); + if(IS_VALID_NC(var)) { + status = get_nc_data(&var, metadata_buf); if(!status) { mlog << Error << "\n" << method_name << " -> " - << "trouble getting " << metadata_name << "\n\n"; + << "trouble getting " << GET_NC_NAME(var) << "\n\n"; exit(1); } } @@ -1274,14 +1342,16 @@ bool get_obs_data_float(NcFile *f_in, const ConcatString var_name, << "trouble getting " << var_name << "\n\n"; } else mlog << Error << "\n" << method_name - << var_name << " or " << var_name << "@ObsValue does not exist!\n\n"; + << var_name << " does not exist!\n\n"; if(!status) exit(1); status = false; if(var_name.length() > 0) { ConcatString qc_name = var_name; - ConcatString qc_group = (ioda_format == ioda_v1) ? qc_postfix : qc_group_name; + ConcatString qc_group = qc_postfix; if (ioda_format == ioda_v2) { + NcGroup nc_grp = get_nc_group(f_in, qc_postfix); + if (IS_INVALID_NC(nc_grp)) qc_group = qc_group_name; StringArray qc_names = conf_info.obs_to_qc_map[var_name]; if (0 < qc_names.n()) { for (int idx=0; idx Date: Tue, 18 Oct 2022 15:04:07 -0600 Subject: [PATCH 14/22] #2068 Cleanup --- src/libcode/vx_nc_util/nc_utils.h | 2 -- src/tools/other/ioda2nc/ioda2nc.cc | 1 - 2 files changed, 3 deletions(-) diff --git a/src/libcode/vx_nc_util/nc_utils.h b/src/libcode/vx_nc_util/nc_utils.h index 2ee19473ff..adb518d402 100644 --- a/src/libcode/vx_nc_util/nc_utils.h +++ b/src/libcode/vx_nc_util/nc_utils.h @@ -345,9 +345,7 @@ extern void copy_nc_var_data(netCDF::NcVar *, netCDF::NcVar *); extern bool has_nc_group(netCDF::NcFile *, const char *group_name); extern bool has_var(netCDF::NcFile *, const char *var_name); -//extern bool has_var(netCDF::NcFile *, const ConcatString &var_name); extern bool has_var(netCDF::NcFile *, const char *var_name, const char *group_name); -//extern bool has_var(netCDF::NcFile *, const ConcatString &var_name, const char *group_name); extern netCDF::NcVar add_var(netCDF::NcFile *, const std::string &, const netCDF::NcType, const int deflate_level=DEF_DEFLATE_LEVEL); extern netCDF::NcVar add_var(netCDF::NcFile *, const std::string &, const netCDF::NcType, const netCDF::NcDim, const int deflate_level=DEF_DEFLATE_LEVEL); diff --git a/src/tools/other/ioda2nc/ioda2nc.cc b/src/tools/other/ioda2nc/ioda2nc.cc index eb766c42c5..ad0c90353f 100644 --- a/src/tools/other/ioda2nc/ioda2nc.cc +++ b/src/tools/other/ioda2nc/ioda2nc.cc @@ -138,7 +138,6 @@ static void initialize(); static void process_command_line(int, char **); static void open_netcdf(); static void process_ioda_file(int); -static void process_ioda_file(NcFile *nc); static void write_netcdf_hdr_data(); static void clean_up(); From 4811522662a5ab0d26380ef197654033adb898ff Mon Sep 17 00:00:00 2001 From: Howard Soh Date: Tue, 18 Oct 2022 15:50:50 -0600 Subject: [PATCH 15/22] #2068 Added obs_to_qc_map --- docs/Users_Guide/reformat_point.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/Users_Guide/reformat_point.rst b/docs/Users_Guide/reformat_point.rst index 1fe03107d3..d1b0cabd6e 100644 --- a/docs/Users_Guide/reformat_point.rst +++ b/docs/Users_Guide/reformat_point.rst @@ -968,6 +968,17 @@ This entry is an array of dictionaries, each containing a **key** string and **v _____________________ +.. code-block:: none + + obs_to_qc_map = [ + { key = "wind_from_direction"; val = "eastward_wind,northward_wind"; }, + { key = "wind_speed"; val = "eastward_wind,northward_wind"; } + ]; + +This entry is an array of dictionaries, each containing a **key** string and **val** string which define a mapping of QC variable name for IODA data files. + +_____________________ + .. code-block:: none missing_thresh = [ <=-1e9, >=1e9, ==-9999 ]; From f43372cf3841ec7be31acd0716de842fc845de8d Mon Sep 17 00:00:00 2001 From: Howard Soh Date: Wed, 19 Oct 2022 13:53:49 -0600 Subject: [PATCH 16/22] #2068 Hanlde float anmd double data type separately --- src/libcode/vx_nc_util/nc_utils.hpp | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/src/libcode/vx_nc_util/nc_utils.hpp b/src/libcode/vx_nc_util/nc_utils.hpp index ceb5bd2807..f7c9c553a6 100644 --- a/src/libcode/vx_nc_util/nc_utils.hpp +++ b/src/libcode/vx_nc_util/nc_utils.hpp @@ -46,12 +46,21 @@ extern void set_def_fill_value(unsigned short *val); template bool get_att_num_value_(const netCDF::NcAtt *att, T &att_val, int matching_type) { - bool status = false; - if (IS_VALID_NC_P(att)) { + bool status = IS_VALID_NC_P(att); + if (status) { int nc_type_id = GET_NC_TYPE_ID_P(att); if (matching_type == nc_type_id) { att->getValues(&att_val); - status = true; + } + else if (NC_FLOAT == nc_type_id) { + float att_value; + att->getValues(&att_value); + att_val = att_value; + } + else if (NC_DOUBLE == nc_type_id) { + double att_value; + att->getValues(&att_value); + att_val = att_value; } else if (NC_CHAR == nc_type_id) { std::string att_value; @@ -62,7 +71,6 @@ bool get_att_num_value_(const netCDF::NcAtt *att, T &att_val, int matching_type) att_val = (double)atof(att_value.c_str()); else // if (matching_type == NC_INT) att_val = atoi(att_value.c_str()); - status = true; } } return(status); @@ -74,8 +82,6 @@ template bool get_nc_att_value_(const netCDF::NcVar *var, const ConcatString &att_name, T &att_val, bool exit_on_error, T bad_data, const char *caller_name) { - bool status = false; - // Initialize att_val = bad_data; @@ -83,7 +89,7 @@ bool get_nc_att_value_(const netCDF::NcVar *var, const ConcatString &att_name, // Retrieve the NetCDF variable attribute. // netCDF::NcVarAtt *att = get_nc_att(var, att_name); - status = get_att_value((netCDF::NcAtt *)att, att_val); + bool status = get_att_value((netCDF::NcAtt *)att, att_val); if (!status) { mlog << Error << "\n" << caller_name << get_log_msg_for_att(att, GET_SAFE_NC_NAME_P(var), att_name); @@ -102,7 +108,6 @@ bool get_nc_att_value_(const netCDF::NcVar *var, const ConcatString &att_name, template bool get_nc_att_value_(const netCDF::NcVarAtt *att, T &att_val, bool exit_on_error, T bad_data, const char *caller_name) { - bool status = true; // Initialize att_val = bad_data; @@ -110,7 +115,7 @@ bool get_nc_att_value_(const netCDF::NcVarAtt *att, T &att_val, bool exit_on_err // // Retrieve the NetCDF variable attribute. // - status = get_att_value((netCDF::NcAtt *)att, att_val); + bool status = get_att_value((netCDF::NcAtt *)att, att_val); if (!status) { mlog << Error << "\n" << caller_name << get_log_msg_for_att(att); @@ -259,12 +264,10 @@ void apply_scale_factor_(T *data, const int cell_count, template bool get_nc_data_t(netCDF::NcVar *var, T *data) { - bool return_status = false; + bool return_status = IS_VALID_NC_P(var); - if (IS_VALID_NC_P(var)) { + if (return_status) { var->getVar(data); - - return_status = true; } return(return_status); } From 411fa68a748d86dbfc5de9ac3f19e51885c2ede1 Mon Sep 17 00:00:00 2001 From: Howard Soh Date: Wed, 19 Oct 2022 15:44:31 -0600 Subject: [PATCH 17/22] #2028 Initialize temp buffer --- src/libcode/vx_nc_util/nc_utils.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libcode/vx_nc_util/nc_utils.cc b/src/libcode/vx_nc_util/nc_utils.cc index e1cf6a620e..b312655396 100644 --- a/src/libcode/vx_nc_util/nc_utils.cc +++ b/src/libcode/vx_nc_util/nc_utils.cc @@ -167,7 +167,8 @@ bool get_att_value_chars(const NcAtt *att, ConcatString &value) { nc_type attType = GET_NC_TYPE_ID_P(att); if (attType == NC_CHAR) { try { - char att_value[8096]; + char att_value[tmp_buf_size]; + memset(att_value, 0, tmp_buf_size); att->getValues(att_value); value = att_value; } From 11250b893e1e3a4c87d7ac4f01f733bf8e23c1ba Mon Sep 17 00:00:00 2001 From: MET Tools Test Account Date: Sat, 29 Oct 2022 09:40:15 -0600 Subject: [PATCH 18/22] Per #2068, update to using METbaseimage version 1.2 which has been updated to support the use of NetCDF groups. --- .github/jobs/set_job_controls.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/jobs/set_job_controls.sh b/.github/jobs/set_job_controls.sh index b26f6d577d..49c4806d98 100755 --- a/.github/jobs/set_job_controls.sh +++ b/.github/jobs/set_job_controls.sh @@ -6,7 +6,7 @@ run_unit_tests=false run_diff=false run_update_truth=false met_base_repo=met-base -met_base_tag=v1.0 +met_base_tag=v1.2 input_data_version=develop truth_data_version=develop From 68cdc0aafcc4f6c5042a0c44a6e5d7461dac024b Mon Sep 17 00:00:00 2001 From: MET Tools Test Account Date: Sat, 29 Oct 2022 12:35:26 -0600 Subject: [PATCH 19/22] Per #2068, update MET_BASE_TAG=v1.2 in three more spots. --- .github/workflows/build_docker_and_trigger_metplus.yml | 2 +- internal/scripts/docker/Dockerfile | 2 +- internal/scripts/docker/Dockerfile.copy | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_docker_and_trigger_metplus.yml b/.github/workflows/build_docker_and_trigger_metplus.yml index e6c8c86a3f..efda02e5f8 100644 --- a/.github/workflows/build_docker_and_trigger_metplus.yml +++ b/.github/workflows/build_docker_and_trigger_metplus.yml @@ -29,7 +29,7 @@ jobs: env: SOURCE_BRANCH: ${{ steps.get_branch_name.outputs.branch_name }}-lite MET_BASE_REPO: met-base - MET_BASE_TAG: v1.0 + MET_BASE_TAG: v1.2 - name: Push Docker Image run: .github/jobs/push_docker_image.sh diff --git a/internal/scripts/docker/Dockerfile b/internal/scripts/docker/Dockerfile index 4dafd977b3..72a02a2229 100644 --- a/internal/scripts/docker/Dockerfile +++ b/internal/scripts/docker/Dockerfile @@ -1,5 +1,5 @@ ARG MET_BASE_REPO=met-base -ARG MET_BASE_TAG=v1.0 +ARG MET_BASE_TAG=v1.2 FROM dtcenter/${MET_BASE_REPO}:${MET_BASE_TAG} MAINTAINER John Halley Gotway diff --git a/internal/scripts/docker/Dockerfile.copy b/internal/scripts/docker/Dockerfile.copy index 5c51408fdf..1e5103aba7 100644 --- a/internal/scripts/docker/Dockerfile.copy +++ b/internal/scripts/docker/Dockerfile.copy @@ -1,5 +1,5 @@ ARG MET_BASE_REPO=met-base-unit-test -ARG MET_BASE_TAG=v1.0 +ARG MET_BASE_TAG=v1.2 FROM dtcenter/${MET_BASE_REPO}:${MET_BASE_TAG} MAINTAINER John Halley Gotway From 4b6fbfe43700c3568134402d83180a564a28ebd5 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Sat, 29 Oct 2022 13:00:47 -0600 Subject: [PATCH 20/22] Per #2068, METbaseimage version 1.2 also uses python version 3.8. Try updating MET to link to that version for the docker environment. --- internal/scripts/environment/development.docker | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/scripts/environment/development.docker b/internal/scripts/environment/development.docker index ec4bd202c3..d525b7dd76 100644 --- a/internal/scripts/environment/development.docker +++ b/internal/scripts/environment/development.docker @@ -24,8 +24,8 @@ export MET_FREETYPELIB=/usr/lib64 export MET_JASPERLIB=/usr/lib64 export MET_PYTHON=/usr/bin/python3 -export MET_PYTHON_CC="-I/usr/include/python3.6m -I/usr/include/python3.6m" -export MET_PYTHON_LD="-L/usr/lib64 -lpython3.6m -lpthread -ldl -lutil -lm" +export MET_PYTHON_CC="-I/usr/include/python3.8m -I/usr/include/python3.8m" +export MET_PYTHON_LD="-L/usr/lib64 -lpython3.8m -lpthread -ldl -lutil -lm" # -D__64BIT__ is required because we've compiled libgrib2c.a with that flag export CFLAGS="-DUNDERSCORE -fPIC -D__64BIT__ -g" From 7203883874d9fb2bc6e55f01528809443592f52c Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Sat, 29 Oct 2022 13:14:04 -0600 Subject: [PATCH 21/22] Per #2068, working on the docker build environment to match METbaseimage v1.2. --- internal/scripts/environment/development.docker | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/scripts/environment/development.docker b/internal/scripts/environment/development.docker index d525b7dd76..15ff8403c2 100644 --- a/internal/scripts/environment/development.docker +++ b/internal/scripts/environment/development.docker @@ -24,8 +24,8 @@ export MET_FREETYPELIB=/usr/lib64 export MET_JASPERLIB=/usr/lib64 export MET_PYTHON=/usr/bin/python3 -export MET_PYTHON_CC="-I/usr/include/python3.8m -I/usr/include/python3.8m" -export MET_PYTHON_LD="-L/usr/lib64 -lpython3.8m -lpthread -ldl -lutil -lm" +export MET_PYTHON_CC="-I/usr/local/include/python3.8" +export MET_PYTHON_LD="-L/usr/local/lib -lpython3.8 -lpthread -ldl -lutil -lm" # -D__64BIT__ is required because we've compiled libgrib2c.a with that flag export CFLAGS="-DUNDERSCORE -fPIC -D__64BIT__ -g" From f860aa62af5c64eb28d2eb628b970b0c2cfc6e04 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Sat, 29 Oct 2022 17:30:26 -0600 Subject: [PATCH 22/22] Per #2068, revert to METbaseimage version 1.1. --- .github/jobs/set_job_controls.sh | 2 +- .github/workflows/build_docker_and_trigger_metplus.yml | 2 +- internal/scripts/docker/Dockerfile | 2 +- internal/scripts/docker/Dockerfile.copy | 2 +- internal/scripts/environment/development.docker | 4 ++-- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/jobs/set_job_controls.sh b/.github/jobs/set_job_controls.sh index 49c4806d98..e897aaa912 100755 --- a/.github/jobs/set_job_controls.sh +++ b/.github/jobs/set_job_controls.sh @@ -6,7 +6,7 @@ run_unit_tests=false run_diff=false run_update_truth=false met_base_repo=met-base -met_base_tag=v1.2 +met_base_tag=v1.1 input_data_version=develop truth_data_version=develop diff --git a/.github/workflows/build_docker_and_trigger_metplus.yml b/.github/workflows/build_docker_and_trigger_metplus.yml index efda02e5f8..15cd652caf 100644 --- a/.github/workflows/build_docker_and_trigger_metplus.yml +++ b/.github/workflows/build_docker_and_trigger_metplus.yml @@ -29,7 +29,7 @@ jobs: env: SOURCE_BRANCH: ${{ steps.get_branch_name.outputs.branch_name }}-lite MET_BASE_REPO: met-base - MET_BASE_TAG: v1.2 + MET_BASE_TAG: v1.1 - name: Push Docker Image run: .github/jobs/push_docker_image.sh diff --git a/internal/scripts/docker/Dockerfile b/internal/scripts/docker/Dockerfile index 72a02a2229..8ec46e33db 100644 --- a/internal/scripts/docker/Dockerfile +++ b/internal/scripts/docker/Dockerfile @@ -1,5 +1,5 @@ ARG MET_BASE_REPO=met-base -ARG MET_BASE_TAG=v1.2 +ARG MET_BASE_TAG=v1.1 FROM dtcenter/${MET_BASE_REPO}:${MET_BASE_TAG} MAINTAINER John Halley Gotway diff --git a/internal/scripts/docker/Dockerfile.copy b/internal/scripts/docker/Dockerfile.copy index 1e5103aba7..7d0fc6ae6f 100644 --- a/internal/scripts/docker/Dockerfile.copy +++ b/internal/scripts/docker/Dockerfile.copy @@ -1,5 +1,5 @@ ARG MET_BASE_REPO=met-base-unit-test -ARG MET_BASE_TAG=v1.2 +ARG MET_BASE_TAG=v1.1 FROM dtcenter/${MET_BASE_REPO}:${MET_BASE_TAG} MAINTAINER John Halley Gotway diff --git a/internal/scripts/environment/development.docker b/internal/scripts/environment/development.docker index 15ff8403c2..c6674b231b 100644 --- a/internal/scripts/environment/development.docker +++ b/internal/scripts/environment/development.docker @@ -24,8 +24,8 @@ export MET_FREETYPELIB=/usr/lib64 export MET_JASPERLIB=/usr/lib64 export MET_PYTHON=/usr/bin/python3 -export MET_PYTHON_CC="-I/usr/local/include/python3.8" -export MET_PYTHON_LD="-L/usr/local/lib -lpython3.8 -lpthread -ldl -lutil -lm" +export MET_PYTHON_CC="-I/usr/include/python3.6m" +export MET_PYTHON_LD="-L/usr/lib64 -lpython3.6m -lpthread -ldl -lutil -lm" # -D__64BIT__ is required because we've compiled libgrib2c.a with that flag export CFLAGS="-DUNDERSCORE -fPIC -D__64BIT__ -g"