From c0f28f735c8d862d1850355f47fa4754ab0a8c75 Mon Sep 17 00:00:00 2001 From: johnhg Date: Mon, 12 Dec 2022 13:16:26 -0700 Subject: [PATCH] Update develop-ref after #2371, #2372, and #2373 (#2376) Co-authored-by: Howard Soh Co-authored-by: Dave Albo Co-authored-by: John Halley Gotway Co-authored-by: Seth Linden Co-authored-by: johnhg Co-authored-by: Lisa Goodrich Co-authored-by: jprestop Co-authored-by: MET Tools Test Account Co-authored-by: j-opatz <59586397+j-opatz@users.noreply.github.com> Co-authored-by: George McCabe <23407799+georgemccabe@users.noreply.github.com> Co-authored-by: Julie Prestopnik Co-authored-by: Jonathan Vigh Co-authored-by: hsoh-u Co-authored-by: bikegeek <3753118+bikegeek@users.noreply.github.com> Co-authored-by: davidalbo Co-authored-by: Seth Linden Co-authored-by: lisagoodrich <33230218+lisagoodrich@users.noreply.github.com> Co-authored-by: Daniel Adriaansen --- .github/workflows/documentation.yml | 2 +- docs/Users_Guide/appendixC.rst | 23 ++ docs/Users_Guide/config_options.rst | 13 +- docs/Users_Guide/grid-stat.rst | 9 + docs/Users_Guide/point-stat.rst | 13 +- docs/Users_Guide/reformat_point.rst | 8 +- docs/Users_Guide/refs.rst | 69 +++- docs/Users_Guide/release-notes.rst | 198 +++++---- docs/conf.py | 4 +- .../test_unit/config/GridStatConfig_python | 10 +- internal/test_unit/xml/unit_aeronet.xml | 12 + src/basic/vx_cal/unix_to_mdyhms.cc | 30 ++ src/basic/vx_cal/vx_cal.h | 1 + src/libcode/vx_data2d/var_info.cc | 11 + src/libcode/vx_data2d/var_info.h | 1 + .../vx_data2d_python/var_info_python.cc | 7 + src/libcode/vx_seeps/seeps.cc | 45 ++- src/libcode/vx_seeps/seeps.h | 4 +- src/libcode/vx_statistics/pair_data_point.cc | 26 +- src/libcode/vx_statistics/pair_data_point.h | 3 + .../core/grid_stat/grid_stat_conf_info.cc | 87 +++- .../core/point_stat/point_stat_conf_info.cc | 93 +++-- src/tools/other/ascii2nc/aeronet_handler.cc | 182 +++++---- src/tools/other/ascii2nc/aeronet_handler.h | 2 +- src/tools/other/ascii2nc/airnow_handler.cc | 377 ++++++++++-------- src/tools/other/ascii2nc/airnow_handler.h | 11 +- src/tools/other/ascii2nc/file_handler.cc | 10 +- src/tools/other/ascii2nc/file_handler.h | 1 + 28 files changed, 791 insertions(+), 461 deletions(-) diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index c43c5a6525..f30b90d0f6 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -21,7 +21,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: - python-version: '3.6' + python-version: '3.8' - name: Install dependencies run: | python -m pip install --upgrade python-dateutil requests sphinx \ diff --git a/docs/Users_Guide/appendixC.rst b/docs/Users_Guide/appendixC.rst index 0ac8b09cbf..6a385f7bfe 100644 --- a/docs/Users_Guide/appendixC.rst +++ b/docs/Users_Guide/appendixC.rst @@ -382,6 +382,29 @@ For cost / loss ratio above the base rate, the ECLV is defined as: .. math:: \text{ECLV } = \frac{(cl \ast (h + f)) + m - b}{b \ast (cl - 1)}. +Stable Equitable Error in Probability Space (SEEPS) +--------------------------------------------------- + +Included in SEEPS output :numref:`table_PS_format_info_SEEPS` and SEEPS_MPR output :numref:`table_PS_format_info_SEEPS_MPR` + +The SEEPS scoring matrix (equation 15 from :ref:`Rodwell et al, 2010 `) is: + +.. math:: \{S^{S}_{vf}\} = \frac{1}{2} + \begin{Bmatrix} + 0 & \frac{1}{1-p_1} & \frac{1}{p_3} + \frac{1}{1-p_1}\\ + \frac{1}{p_1} & 0 & \frac{1}{p_3}\\ + \frac{1}{p_1} + \frac{1}{1-p_3} & \frac{1}{1-p_3} & 0 + \end{Bmatrix} + +In addition, Rodwell et al (2011) note that SEEPS can be written as the mean of two 2-category scores that individually assess the dry/light and light/heavy thresholds (:ref:`Rodwell et al., 2011 `). Each of these scores is like 1 – HK, but written as: + +.. math:: \frac{n_{01}}{\text{Expected n}_{.1}} + \frac{n_{10}}{\text{Expected n}_{.0}} + + +where the word expected refers to the mean value deduced from the climatology, rather than the sample mean. + +SEEPS scores are expected to lie between 0 and 1, with a perfect forecast having a value of 0. Individual values can be much larger than 1. Results can be presented as a skill score by using the value of 1 – SEEPS. + MET verification measures for continuous variables ================================================== diff --git a/docs/Users_Guide/config_options.rst b/docs/Users_Guide/config_options.rst index 195edca40a..5cc2432fd6 100644 --- a/docs/Users_Guide/config_options.rst +++ b/docs/Users_Guide/config_options.rst @@ -240,6 +240,7 @@ Referencing that environment variable inside a MET configuration file: In addition to supporting user-specified environment variables within configuration files, the environment variables listed below have special meaning if set at runtime. +.. _met_airnow_stations: MET_AIRNOW_STATIONS ^^^^^^^^^^^^^^^^^^^ @@ -249,7 +250,17 @@ will override the default file. If set, it should be the full path to the file. The default table can be found in the installed *share/met/table_files/airnow_monitoring_site_locations_v2.dat*. This file contains ascii column data that allows lookups of latitude, longitude, and elevation for all -airnow stations based on stationId and/or AqSid. +AirNow stations based on stationId and/or AqSid. + +Additional information and updated site locations can be found at the +`EPA AirNow website `_. While some monitoring stations are +permanent, others are temporary, and theirs locations can change. When running the +ascii2nc tool with the `-format airnowhourly` option, users should +`download `_ the `Monitoring_Site_Locations_V2.dat` data file +data file corresponding to the date being processed and set the MET_AIRNOW_STATIONS +envrionment variable to define its location. + +.. _met_ndbc_stations: MET_NDBC_STATIONS ^^^^^^^^^^^^^^^^^ diff --git a/docs/Users_Guide/grid-stat.rst b/docs/Users_Guide/grid-stat.rst index 2887cd77a6..55d9daaa4b 100644 --- a/docs/Users_Guide/grid-stat.rst +++ b/docs/Users_Guide/grid-stat.rst @@ -71,6 +71,15 @@ There are several ways to present the results of the neighborhood approaches, su The user must specify several parameters in the grid_stat configuration file to utilize the neighborhood approach, such as the interpolation method, size of the smoothing window, and required fraction of valid data points within the smoothing window. For FSS-specific results, the user must specify the size of the neighborhood window, the required fraction of valid data points within the window, and the fractional coverage threshold from which the contingency tables are defined. These parameters are described further in the practical information section below. +.. _grid-stat_seeps: + +SEEPS scores +------------ + +The Stable Equitable Error in Probability Space (SEEPS) was devised for monitoring global deterministic forecasts of precipitation against the WMO gauge network (:ref:`Rodwell et al., 2010 `; :ref:`Haiden et al., 2012 `) and is a multi-category score which uses a climatology to account for local variations in behavior. Please see Point-Stat documentation :numref:`PS_seeps` for more details. + +The capability to calculate the SEEPS has also been added to Grid-Stat. This follows the method described in :ref:`North et al, 2022 `, which uses the TRMM 3B42 v7 gridded satellite product for the climatological values and interpolates the forecast and observed products onto this grid for evaluation. A 24-hour TRMM climatology (valid at 00 UTC) constructed from data over the time period 1998-2015 is supplied with the release. Expansion of the capability to other fields will occur as well vetted examples and funding allow. + Fourier Decomposition --------------------- diff --git a/docs/Users_Guide/point-stat.rst b/docs/Users_Guide/point-stat.rst index c5bb22b5e9..e4f0e7eeda 100644 --- a/docs/Users_Guide/point-stat.rst +++ b/docs/Users_Guide/point-stat.rst @@ -136,6 +136,17 @@ The HiRA framework provides a unique method for evaluating models in the neighbo Often, the neighborhood size is chosen so that multiple models to be compared have approximately the same horizontal resolution. Then, standard metrics for probabilistic forecasts, such as Brier Score, can be used to compare those forecasts. HiRA was developed using surface observation stations so the neighborhood lies completely within the horizontal plane. With any type of upper air observation, the vertical neighborhood must also be defined. +.. _PS_seeps: + +SEEPS scores +------------ + +The Stable Equitable Error in Probability Space (SEEPS) was devised for monitoring global deterministic forecasts of precipitation against the WMO gauge network (:ref:`Rodwell et al., 2010 `; :ref:`Haiden et al., 2012 `) and is a multi-category score which uses a climatology to account for local variations in behavior. Since the score uses probability space to define categories using the climatology, it can be aggregated over heterogeneous climate regions. Even though it was developed for use with precipitation forecasts, in principle it could be applied to any forecast parameter for which a sufficiently long time period of observations exists to create a suitable climatology. The computation of SEEPS for precipitation is only supported for now. + +For use with precipitation, three categories are used, named ‘dry’, ‘light’ and ‘heavy’. The ‘dry’ category is defined (using the WMO observing guidelines) with any accumulation (rounded to the nearest 0.1 millimeter) that is less than or equal to 0.2 mm. The remaining precipitation is divided into ‘light’ and ‘heavy’ categories whose thresholds are with respect to a climatology and thus location specific. The light precipitation is defined to occur twice as often as heavy precipitation. + +When calculating a single SEEPS value over observing stations for a particular region, the scores should have a density weighting applied which accounts for uneven station distribution in the region of interest (see Section 9.1 in :ref:`Rodwell et al., 2010 `). This density weighting has not yet been implemented in MET. Global precipitation climatologies calculated from the WMO SYNOP records from 1980-2009 are supplied with the release. At the moment, a 24-hour climatology is available (valid at 00 UTC or 12 UTC), but in future a 6-hour climatology will become available. + .. _PS_Statistical-measures: Statistical measures @@ -487,7 +498,7 @@ Note that writing out matched pair data (MPR lines) for a large number of cases If all line types corresponding to a particular verification method are set to NONE, the computation of those statistics will be skipped in the code and thus make the Point-Stat tool run more efficiently. For example, if FHO, CTC, and CTS are all set to NONE, the Point-Stat tool will skip the categorical verification step. -The default SEEPS climo file exists at MET_BASE/share/met/climo/seeps/PPT24_seepsweights.nc. It can be overridden by using the environment variable, MET_SEEPS_POINT_CLIMO_NAME. +The default SEEPS climo file exists at MET_BASE/climo/seeps/PPT24_seepsweights.nc. It can be overridden by using the environment variable, MET_SEEPS_POINT_CLIMO_NAME. .. _point_stat-output: diff --git a/docs/Users_Guide/reformat_point.rst b/docs/Users_Guide/reformat_point.rst index 527d5d3d72..1cd9b4705d 100644 --- a/docs/Users_Guide/reformat_point.rst +++ b/docs/Users_Guide/reformat_point.rst @@ -451,9 +451,9 @@ While initial versions of the ASCII2NC tool only supported a simple 11 column AS • Western Wind and Solar Integration Study (WWSIS) format. WWSIS data are available by request from National Renewable Energy Laboratory (NREL) in Boulder, CO. -• `AirNow DailyData_v2, AirNow HourlyData, and AirNow HourlyAQObs formats `_ +• `AirNow DailyData_v2, AirNow HourlyData, and AirNow HourlyAQObs formats `_. See the :ref:`MET_AIRNOW_STATIONS` environment variable. -• `National Data Buoy (NDBC) Standard Meteorlogical Data format `_ +• `National Data Buoy (NDBC) Standard Meteorlogical Data format `_. See the :ref:`MET_NDBC_STATIONS` environment variable. • `AErosol RObotic NEtwork (AERONET) versions 2 and 3 format `_ @@ -1191,10 +1191,10 @@ The script can be found at: .. code-block:: none - MET_BASE/shared/met/utility/print_pointnc2ascii.py + MET_BASE/utility/print_pointnc2ascii.py For how to use the script, issue the command: .. code-block:: none - python3 MET_BASE/shared/met/utility/print_pointnc2ascii.py -h + python3 MET_BASE/utility/print_pointnc2ascii.py -h diff --git a/docs/Users_Guide/refs.rst b/docs/Users_Guide/refs.rst index 423be1a479..b4045f90b6 100644 --- a/docs/Users_Guide/refs.rst +++ b/docs/Users_Guide/refs.rst @@ -7,14 +7,14 @@ References .. _Aberson-1998: | Aberson, S.D., 1998: Five-day Tropical cyclone track forecasts in the North -| Atlantic Basin. *Weather & Forecasting*, 13, 1005-1015. +| Atlantic Basin. *Weather and Forecasting*, 13, 1005-1015. | .. _Ahijevych-2009: -| Ahijevych, D., E. Gilleland, B.G. Brown, and E.E. Ebert, 2009. Application of +| Ahijevych, D., E. Gilleland, B.G. Brown, and E.E. Ebert, 2009: Application of | spatial verification methods to idealized and NWP-gridded precipitation forecasts. -| Weather Forecast., 24 (6), 1485 - 1497, doi: 10.1175/2009WAF2222298.1. +| *Weather and Forecasting*, 24 (6), 1485 - 1497, doi: 10.1175/2009WAF2222298.1. | .. _Barker-1991: @@ -127,31 +127,30 @@ References .. _Gilleland-2017: -| Gilleland, E., 2017. A new characterization in the spatial verification +| Gilleland, E., 2017: A new characterization in the spatial verification | framework for false alarms, misses, and overall patterns. -| Weather Forecast., 32 (1), 187 - 198, doi: 10.1175/WAF-D-16-0134.1. +| *Weather and Forecasting*, 32 (1), 187 - 198, doi: 10.1175/WAF-D-16-0134.1. | - .. _Gilleland_PartI-2020: -| Gilleland, E., 2020. Bootstrap methods for statistical inference. +| Gilleland, E., 2020: Bootstrap methods for statistical inference. | Part I: Comparative forecast verification for continuous variables. -| Journal of Atmospheric and Oceanic Technology, 37 (11), 2117 - 2134, +| *Journal of Atmospheric and Oceanic Technology*, 37 (11), 2117 - 2134, | doi: 10.1175/JTECH-D-20-0069.1. | .. _Gilleland_PartII-2020: -| Gilleland, E., 2020. Bootstrap methods for statistical inference. -| Part II: Extreme-value analysis. Journal of Atmospheric and Oceanic -| Technology, 37 (11), 2135 - 2144, doi: 10.1175/JTECH-D-20-0070.1. +| Gilleland, E., 2020: Bootstrap methods for statistical inference. +| Part II: Extreme-value analysis. *Journal of Atmospheric and Oceanic* +| *Technology*, 37 (11), 2135 - 2144, doi: 10.1175/JTECH-D-20-0070.1. | .. _Gilleland-2021: -| Gilleland, E., 2021. Novel measures for summarizing high-resolution forecast -| performance. Advances in Statistical Climatology, Meteorology and Oceanography, +| Gilleland, E., 2021: Novel measures for summarizing high-resolution forecast +| performance. *Advances in Statistical Climatology, Meteorology and Oceanography*, | 7 (1), 13 - 34, doi: 10.5194/ascmo-7-13-2021. | @@ -164,6 +163,14 @@ References | http://www.stat.washington.edu/www/research/reports/ | +.. _Haiden-2012: + +| Haiden, T., M.J. Rodwell, D.S. Richardson, A. Okagaki, T. Robinson, T. Hewson, 2012: +| Intercomparison of Global Model Precipitation Forecast Skill in 2010/11 +| Using the SEEPS Score. *Monthly Weather Review*, 140, 2720-2733. +| https://doi.org/10.1175/MWR-D-11-00301.1 +| + .. _Hamill-2001: | Hamill, T. M., 2001: Interpretation of rank histograms for verifying ensemble @@ -186,7 +193,7 @@ References | Knaff, J.A., M. DeMaria, C.R. Sampson, and J.M. Gross, 2003: Statistical, | Five-Day Tropical Cyclone Intensity Forecasts Derived from Climatology -| and Persistence. *Weather & Forecasting,* Vol. 18 Issue 2, p. 80-92. +| and Persistence. *Weather and Forecasting,* Vol. 18 Issue 2, p. 80-92. | .. _Mason-2004: @@ -228,20 +235,44 @@ References | verification. *Monthly Weather Review*, 115, 1330-1338. | + +.. _North-2022: + +| North, R.C., M.P. Mittermaier, S.F. Milton, 2022. *Using SEEPS with a* +| TRMM-derived Climatology to Assess Global NWP Precipitation Forecast Skill. +| *Monthly Weather Review*, 150, 135-155. +| https://doi.org/10.1175/MWR-D-20-0347.1 +| + .. _Ou-2016: | Ou, M. H., Charles, M., & Collins, D. C. 2016: Sensitivity of calibrated week-2 | probabilistic forecast skill to reforecast sampling of the NCEP global -| ensemble forecast system. *Weather and Forecasting,* 31(4), 1093-1107. +| ensemble forecast system. *Weather and Forecasting*, 31(4), 1093-1107. | .. _Roberts-2008: | Roberts, N.M., and H.W. Lean, 2008: Scale-selective verification of rainfall | accumulations from high-resolution forecasts of convective events. -| *Monthly Weather Review,* 136, 78-97. +| *Monthly Weather Review*, 136, 78-97. | +.. _Rodwell-2010: + +| Rodwell, M.J., D.S. Richardson, T.D. Hewson and T. Haiden, 2010: A new equitable +| score suitable for verifying precipitation in numerical weather prediction. +| *Quarterly Journal of the Royal Meteorological Society*, 136: 1344-1463. +| https://doi.org/10.1002/qj.656 +| + +.. _Rodwell-2011: + +| Rodwell, M.J., T. Haiden, D.S. Richardson, 2011: Developments in Precipitation +| Verification. *ECMWF Newsletter* Number 128. +| https://www.ecmwf.int/node/14595 +| + .. _Saetra-2004: | Saetra O., H. Hersbach, J-R Bidlot, D. Richardson, 2004: Effects of @@ -279,7 +310,7 @@ References .. _Todter-2012: | Tödter, J. and B. Ahrens, 2012: Generalization of the Ignorance Score: -| Continuous ranked version and its decomposition. *Mon. Wea. Rev.*, +| Continuous ranked version and its decomposition. *Monthly Weather Review*, | 140 (6), 2005 - 2017, doi: 10.1175/MWR-D-11-00266.1. | @@ -287,14 +318,14 @@ References | Weniger, M., F. Kapp, and P. Friederichs, 2016: Spatial Verification Using | Wavelet Transforms: A Review. *Quarterly Journal of the Royal* -| *Meteorological Society,* 143, 120-136. +| *Meteorological Society*, 143, 120-136. | .. _Wilks-2010: | Wilks, D.S. 2010: Sampling distributions of the Brier score and Brier skill | score under serial dependence. *Quarterly Journal of the Royal* -| *Meteorological Society,*, 136, 2109-2118. doi:10.1002/qj.709 +| *Meteorological Society*, 136, 2109-2118. doi:10.1002/qj.709 | .. _Wilks-2011: diff --git a/docs/Users_Guide/release-notes.rst b/docs/Users_Guide/release-notes.rst index 7c52c7e517..4dfd494c4a 100644 --- a/docs/Users_Guide/release-notes.rst +++ b/docs/Users_Guide/release-notes.rst @@ -9,148 +9,136 @@ When applicable, release notes are followed by the GitHub issue number which des enhancement, or new feature (`MET GitHub issues `_). Important issues are listed **in bold** for emphasis. -.. warning:: **Ensemble post-processing was added to Gen-Ens-Prod in version 10.1.0 and will be removed from Ensemble-Stat in version 11.0.0!** +MET Version 11.0.0 release notes (20221209) +------------------------------------------- -MET Version 11.0.0-beta5 release notes (20221120) -------------------------------------------------- +* Repository, build, and test: -* Repository, build, and testing: - - * MET: Enhance MET to have better signal handling for shutdown events (`dtcenter/METplus-Internal#21 `_). - -* Library Enhancements: - - * Reimplement the pntnc2ascii.R utility Rscript in Python (`#2085 `_). - * **Add a Python helper script/function to transform point_data objects to met_point_data objects for Python Embedding** (`#2302 `_). - -* Application Enhancements: - - * **Enhance Grid-Stat to compute SEEPS for gridded observations and write the SEEPS STAT line type** (`#1943 `_). - * **Enhance Stat-Analysis to aggregate SEEPS_MPR and SEEPS line types** (`#2339 `_). - * **Refine TCDIAG output from TC-Pairs as needed** (`#2321 `_). - * Rename the TCDIAG SOURCE column as DIAG_SOURCE (`#2337 `_). - * Enhance TC-Pairs consensus logic to compute the spread of the location, wind speed, and pressure (`#2036 `_). - * Print ASCII2NC warning message about python embedding support not being compiled (`#2277 `_). - * Eliminate Gen-Ens-Prod warning when parsing the nbhrd_prob dictionary (`#2224 `_). - -MET Version 11.0.0-beta4 release notes (20221102) -------------------------------------------------- - -* Repository, build, and testing: - - * Enhance the MET base image to support NetCDF files using groups in the enhanced data model (`dtcenter/METbaseimage#6 `_). + * **Restructure the contents of the MET repository so that it matches the existing release tarfiles** (`#1920 `_). + * **Add initial files to create the MET compilation environment in the dtcenter/met-base Docker image** (`dtcenter/METbaseimage#1 `_). + * Restructure the MET Dockerfiles to create images based on the new METbaseimage (`#2196 `_). + * Enhance METbaseimage to support NetCDF files using groups in the enhanced data model (`dtcenter/METbaseimage#6 `_). + * Add .zenodo.json file to add metadata about releases (`#2198 `_). * Update the SonarQube version used for routine software scans (`#2270 `_). - * Create test to replicate MET-11.0.0-beta3 linker errors and fix them (`#2281 `_). + * Fix OpenMP compilation error for GCC 9.3.0/9.4.0 (`#2106 `_). + * Fix oom() compile time linker error (`#2238 `_). + * Fix MET-11.0.0-beta3 linker errors (`#2281 `_). * Fix GHA documentation workflow (`#2282 `_). * Fix GHA warnings and update the version of actions (i.e. actions/checkout@v3) (`#2297 `_). -* Code cleanup and documentation: +* Documentation: - * Remove namespace specification from header files (`#2227 `_). + * Create outline for the MET Contributor's Guide (`#1774 `_). + * Document PB2NC's handling of quality markers (`#2278 `_). * Move release notes into its own chapter in the User's Guide (`#2298 `_). * Bugfixes: + * Fix regression test differences in pb2nc and ioda2nc output (`#2102 `_). + * Fix support for reading rotated lat/lon grids from CF-compliant NetCDF files (`#2115 `_). + * Fix support for reading rotated lat/lon grids from GRIB1 files (grid type 10) (`#2118 `_). + * Fix support for int64 NetCDF variable types (`#2123 `_). + * Fix Stat-Analysis to aggregate the ECNT ME and RMSE values correctly (`#2170 `_). + * Fix NetCDF library code to process scale_factor and add_offset attributes independently (`#2187 `_). + * Fix Ensemble-Stat to work with different missing members for two or more variables (`#2208 `_). + * Fix truncated station_id name in the output from IODA2NC (`#2216 `_). * Fix Stat-Analysis aggregation of the neighborhood statistics line types (`#2271 `_). - * Fix the ascii2nc_airnow_hourly test in unit_ascii2nc.xml (`#2306 `_). + * Fix Point-Stat and Ensemble-Stat GRIB table lookup logic for python embedding of point observations (`#2286 `_). + * Fix ascii2nc_airnow_hourly test in unit_ascii2nc.xml (`#2306 `_). * Fix TC-Stat parsing of TCMPR lines (`#2309 `_). - * Fix the Point-Stat and Ensemble-Stat GRIB table lookup logic for python embedding of point observations (`#2286 `_). + * Fix ASCII2NC logic for reading AERONET v3 data (`#2370 `_). -* Library Enhancments: +* Enhancements: - * Add support for point-based climatologies for use in SEEPS (`#1941 `_). - * Enhance MET to handle NC strings when processing CF-Compliant NetCDF files (`#2042 `_). - * Enhance the MET library code to handle CF-compliant time strings with an offset defined in months or years (`#2155 `_). + * NetCDF: -* Application Enhancements: + * **Enhance MET's NetCDF library interface to support level strings that include coordinate variable values instead of just indexes** (`#1815 `_). + * Enhance MET to handle NC strings when processing CF-Compliant NetCDF files (`#2042 `_). + * Enhance MET to handle CF-compliant time strings with an offset defined in months or years (`#2155 `_). + * Refine NetCDF level string handling logic to always interpret @ strings as values (`#2225 `_). - * **Enhance ASCII2NC to read NDBC buoy data** (`#2276 `_). - * **Enhance IODA2NC to support IODA v2.0 format** (`#2068 `_). - * **Add the Mean Absolute Difference (SPREAD_MD) to the ECNT line type** (`#2332 `_). - * **Add MAE to the ECNT line type from Ensemble-Stat and for HiRA** (`#2325 `_). - * **Add new bias ratio statistic to the ECNT line type from Ensemble-Stat and for HiRA** (`#2058 `_). - * **Enhance TC-Pairs consensus logic to compute the spread of the location, wind speed, and pressure** (`#2036 `_). - * **Enhance TC-Pairs to read hurricane model diagnostic files (e.g. SHIPS) and TC-Stat to filter the new data** (`#392 `_). - * Refine Grid-Diag output variable names when specifying two input data sources (`#2232 `_). + * GRIB: -MET Version 11.0.0-beta3 release notes (20220921) -------------------------------------------------- + * Add support for reading National Blend Model GRIB2 data (`#2055 `_). + * Update the GRIB2 MRMS table in MET (`#2081 `_). -* Repository and build: + * Python: - * Add initial files to create the MET compilation environment in the dtcenter/met-base Docker image (`dtcenter/METbaseimage#1 `_). - * Update the METbaseimage to install Python 3.8.6 from source (`dtcenter/METbaseimage#3 `_). - * Restructure the MET Dockerfiles to create images based on the new METbaseimage (`#2196 `_). - * Add .zenodo.json file to add metadata about releases (`#2198 `_). + * Reimplement the pntnc2ascii.R utility Rscript in Python (`#2085 `_). + * Add more error checking for python embedding of point observations (`#2202 `_). + * **Add a Python helper script/function to transform point_data objects to met_point_data objects for Python Embedding** (`#2302 `_). -* Bugfixes: + * METplus-Internal: - * Fix the truncated station_id name in the output from IODA2NC (`#2216 `_). - * Fix oom() compile time linker error (`#2238 `_). - * Store unspecified accumulation interval as 0 rather than bad data (`#2250 `_). + * MET: Replace fixed length character arrays with strings (`dtcenter/METplus-Internal#14 `_). + * MET: Add a timestamp to the log output at the beginning and end of each MET tool run (`dtcenter/METplus-Internal#18 `_). + * MET: Add the user ID and the command line being executed to the log output at beginning and end of each MET tool run (`dtcenter/METplus-Internal#19 `_). + * MET: Enhance MET to have better signal handling for shutdown events (`dtcenter/METplus-Internal#21 `_). -* Enhancements: + * Common Libraries: - * **Remove ensemble post-processing from the Ensemble-Stat tool** (`#1908 `_). - * **Enhance Point-Stat to compute SEEPS for point observations and write new SEEPS and SEEPS_MPR STAT line types** (`#1942 `_). - * **Add the fair CRPS statistic to the ECNT line type in a new CRPS_EMP_FAIR column** (`#2206 `_). - * Update map data with more recent NaturalEarth definitions (`#2207 `_). - * Define new grid class to store semi-structured grid information (e.g. lat or lon vs level or time) (`#1954 `_). - * Add support for EPA AirNow ASCII data in ASCII2NC (`#2142 `_). - * Add tmp_dir configuration option to the Plot-Point-Obs tool (`#2237 `_). - * Refine NetCDF level string handling logic to always interpret @ strings as values (`#2225 `_). - * Add support for reading National Blend Model GRIB2 data (`#2055 `_). + * **Define new grid class to store semi-structured grid information (e.g. lat or lon vs level or time)** (`#1954 `_). + * Refine warning/error messages when parsing thresholds (`#2211 `_). + * Remove namespace specification from header files (`#2227 `_). + * Update MET version number to 11.0.0 (`#2132 `_). + * Store unspecified accumulation interval as 0 rather than bad data (`#2250 `_). + * Add sanity check to error out when both is_u_wind and is_v_wind are set to true (`#2357 `_). -MET Version 11.0.0-beta2 release notes (20220809) -------------------------------------------------- + * Statistics: -* Bugfixes: + * **Add Anomaly Correlation Coefficient to VCNT Line Type** (`#2022 `_). + * **Allow 2x2 HSS calculations to include user-defined EC values** (`#2147 `_). + * **Add the fair CRPS statistic to the ECNT line type in a new CRPS_EMP_FAIR column** (`#2206 `_). + * **Add MAE to the ECNT line type from Ensemble-Stat and for HiRA** (`#2325 `_). + * **Add the Mean Absolute Difference (SPREAD_MD) to the ECNT line type** (`#2332 `_). + * **Add new bias ratio statistic to the ECNT line type from Ensemble-Stat and for HiRA** (`#2058 `_). - * Fix Ensemble-Stat to work with different missing members for two or more variables (`#2208 `_). + * Configuration and masking: -* Enhancements: + * Define the Bukovsky masking regions for use in MET (`#1940 `_). + * **Enhance Gen-Vx-Mask by adding a new poly_xy masking type option** (`#2152 `_). + * Add M_to_KFT and KM_to_KFT functions to ConfigConstants (`#2180 `_). + * Update map data with more recent NaturalEarth definitions (`#2207 `_). - * **Enhance MET's NetCDF library interface to support level strings that include coordinate variable values instead of just indexes** (`#1815 `_). - * **Enhance MTD to process time series with non-uniform time steps, such as monthly data** (`#1971 `_). - * Define the Bukovsky masking regions for use in MET (`#1940 `_). - * Update the GRIB2 MRMS table in MET (`#2081 `_). - * Add more error checking for python embedding of point observations (`#2202 `_). - * Add a sum option to the time summaries computed by the point pre-processing tools (`#2204 `_). - * Refine warning/error messages when parsing thresholds (`#2211 `_). - * Add "station_ob" to metadata_map as a message_type metadata variable for ioda2nc (`#2215 `_). - * MET: Add a timestamp to the log output at the beginning and end of each MET tool run (`dtcenter/METplus-Internal#18 `_). - * MET: Add the user ID and the command line being executed to the log output at beginning and end of each MET tool run (`dtcenter/METplus-Internal#19 `_). - * MET: Enhance MET to have better signal handling for shutdown events (`dtcenter/METplus-Internal#21 `_). + * Point Pre-Processing Tools: -MET Version 11.0.0-beta1 release notes (20220622) -------------------------------------------------- + * **Enhance IODA2NC to support IODA v2.0 format** (`#2068 `_). + * **Add support for EPA AirNow ASCII data in ASCII2NC** (`#2142 `_). + * Add a sum option to the time summaries computed by the point pre-processing tools (`#2204 `_). + * Add "station_ob" to metadata_map as a message_type metadata variable for ioda2nc (`#2215 `_). + * **Enhance ASCII2NC to read NDBC buoy data** (`#2276 `_). + * Print ASCII2NC warning message about python embedding support not being compiled (`#2277 `_). -* Repository and build: + * Point-Stat, Grid-Stat, Stat-Analysis: - * **Restructure the contents of the MET repository so that it matches the existing release tarfiles** (`#1920 `_). - * Fix the OpenMP compilation error for GCC 9.3.0/9.4.0 (`#2106 `_). - * Update the MET version number to 11.0.0 (`#2132 `_). + * Add support for point-based climatologies for use in SEEPS (`#1941 `_). + * **Enhance Point-Stat to compute SEEPS for point observations and write new SEEPS and SEEPS_MPR STAT line types** (`#1942 `_). + * **Enhance Grid-Stat to compute SEEPS for gridded observations and write the SEEPS STAT line type** (`#1943 `_). + * Sort mask.sid station lists to check their contents more efficiently (`#1950 `_). + * **Enhance Stat-Analysis to aggregate SEEPS_MPR and SEEPS line types** (`#2339 `_). + * Relax Point-Stat and Ensemble-Stat logic for the configuration of message_type_group_map (`#2362 `_). + * Fix Point-Stat and Grid-Stat logic for processing U/V winds with python embedding (`#2366 `_). -* Bugfixes: + * Ensemble Tools: - * Fix regression test differences in pb2nc and ioda2nc output (`#2102 `_). - * Fix support for reading rotated lat/lon grids from CF-compliant NetCDF files (`#2115 `_). - * Fix support for reading rotated lat/lon grids from GRIB1 files (grid type 10) (`#2118 `_). - * Fix support for int64 NetCDF variable types (`#2123 `_). - * Fix Stat-Analysis to aggregate the ECNT ME and RMSE values correctly (`#2170 `_). - * Fix NetCDF library code to process scale_factor and add_offset attributes independently (`#2187 `_). + * **Remove ensemble post-processing from the Ensemble-Stat tool** (`#1908 `_). + * Eliminate Gen-Ens-Prod warning when parsing the nbhrd_prob dictionary (`#2224 `_). -* Enhancements: + * Tropical Cyclone Tools: + + * **Enhance TC-Pairs to read hurricane model diagnostic files (e.g. SHIPS) and TC-Stat to filter the new data** (`#392 `_). + * **Enhance TC-Pairs consensus logic to compute the spread of the location, wind speed, and pressure** (`#2036 `_). + * Enhance TC-RMW to compute tangential and radial winds (`#2072 `_). + * Refine TCDIAG output from TC-Pairs as needed (`#2321 `_). + * Rename the TCDIAG SOURCE column as DIAG_SOURCE (`#2337 `_). + + * Miscellaneous: - * Sort mask.sid station lists to check their contents more efficiently (`#1950 `_). - * Add Anomaly Correlation Coefficient to VCNT Line Type (`#2022 `_). - * Enhance TC-RMW to compute tangential and radial winds (`#2072 `_). - * Allow 2x2 HSS calculations to include user-defined EC values (`#2147 `_). - * Enhance Gen-Vx-Mask by adding a new poly_xy masking type option (`#2152 `_). - * Add M_to_KFT and KM_to_KFT functions to ConfigConstants (`#2180 `_). - * MET: Replace fixed length character arrays with strings (`dtcenter/METplus-Internal#14 `_). + * Enhance MTD to process time series with non-uniform time steps, such as monthly data (`#1971 `_). + * Refine Grid-Diag output variable names when specifying two input data sources (`#2232 `_). + * Add tmp_dir configuration option to the Plot-Point-Obs tool (`#2237 `_). MET Upgrade Instructions ======================== -Upgrade instructions will be listed here if they are applicable for this release. +* Ensemble post-processing has been fully removed from Ensemble-Stat in version 11.0.0. It can be performed using the Gen-Ens-Prod tool. diff --git a/docs/conf.py b/docs/conf.py index 739344333f..055e2c9651 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -20,11 +20,11 @@ project = 'MET' author = 'UCAR/NCAR, NOAA, CSU/CIRA, and CU/CIRES' author_list = 'Opatz, J., T. Jensen, J. Prestopnik, H. Soh, L. Goodrich, B. Brown, R. Bullock, J. Halley Gotway, K. Newman' -version = '11.0.0-beta5' +version = '11.0.0' verinfo = version release = f'{version}' release_year = '2022' -release_date = f'{release_year}-11-20' +release_date = f'{release_year}-12-09' copyright = f'{release_year}, {author}' # -- General configuration --------------------------------------------------- diff --git a/internal/test_unit/config/GridStatConfig_python b/internal/test_unit/config/GridStatConfig_python index 37c1c090e9..b6469e7613 100644 --- a/internal/test_unit/config/GridStatConfig_python +++ b/internal/test_unit/config/GridStatConfig_python @@ -60,12 +60,16 @@ rank_corr_flag = FALSE; // fcst = { field = [ { name = "${FCST_COMMAND}"; }, - { name = "${OBS_COMMAND}"; } ]; + { name = "${OBS_COMMAND}"; }, + { name = "${FCST_COMMAND}"; set_attr_name = "FCST_UWIND"; level = "Surface"; is_u_wind = TRUE; }, + { name = "${OBS_COMMAND}"; set_attr_name = "OBS_VWIND"; level = "Surface"; is_v_wind = TRUE; } ]; } obs = { field = [ { name = "${OBS_COMMAND}"; }, - { name = "${FCST_COMMAND}"; } ]; + { name = "${FCST_COMMAND}"; }, + { name = "${OBS_COMMAND}"; set_attr_name = "OBS_UWIND"; level = "Surface"; is_u_wind = TRUE; }, + { name = "${FCST_COMMAND}"; set_attr_name = "FCST_VWIND"; level = "Surface"; is_v_wind = TRUE; } ]; } //////////////////////////////////////////////////////////////////////////////// @@ -173,7 +177,7 @@ output_flag = { cnt = NONE; sl1l2 = STAT; sal1l2 = NONE; - vl1l2 = NONE; + vl1l2 = STAT; val1l2 = NONE; vcnt = NONE; pct = NONE; diff --git a/internal/test_unit/xml/unit_aeronet.xml b/internal/test_unit/xml/unit_aeronet.xml index 0892c623ad..208353a485 100644 --- a/internal/test_unit/xml/unit_aeronet.xml +++ b/internal/test_unit/xml/unit_aeronet.xml @@ -51,6 +51,18 @@ + + &MET_BIN;/ascii2nc + \ + &DATA_DIR_OBS;/aeronet/20221202.v3.lev15.txt \ + &OUTPUT_DIR;/aeronet/20221202.v3.lev15.nc \ + -v 1 -format aeronetv3 + + + &OUTPUT_DIR;/aeronet/20221202.v3.lev15.nc + + + &MET_BIN;/ascii2nc diff --git a/src/basic/vx_cal/unix_to_mdyhms.cc b/src/basic/vx_cal/unix_to_mdyhms.cc index e7d2a11330..72fd5cfcf9 100644 --- a/src/basic/vx_cal/unix_to_mdyhms.cc +++ b/src/basic/vx_cal/unix_to_mdyhms.cc @@ -16,6 +16,7 @@ using namespace std; #include #include +#include #include #include @@ -94,6 +95,35 @@ const char * short_month_name[] = { }; +//////////////////////////////////////////////////////////////////////// +// Converts a month string to month (1 to 12). + +int month_name_to_m (const char *month_str) + +{ + +int month = -1; +int str_len = strlen(month_str); +char t_month_str[str_len + 1]; + +for (int idx=0; idx seeps_climo_grid_map_00; -static const char *def_seeps_filename = "MET_BASE/climo/seeps/PPT24_seepsweights.nc"; -static const char *def_seeps_grid_filename = "MET_BASE/climo/seeps/PPT24_seepsweights_grid.nc"; +static const char *def_seeps_filename = + "MET_BASE/climo/seeps/PPT24_seepsweights.nc"; +static const char *def_seeps_grid_filename = + "MET_BASE/climo/seeps/PPT24_seepsweights_grid.nc"; static const char *var_name_sid = "sid"; static const char *var_name_lat = "lat"; @@ -200,8 +202,10 @@ SeepsClimo::SeepsClimo() { if (seeps_ready) read_seeps_scores(seeps_name); else { mlog << Error << "\nSeepsClimo::SeepsClimo() -> " - << "The SEEPS climo data \"" << seeps_name << "\" is missing!" - << " Turn off SEEPS and SEEPS_MPR to continue\n\n"; + << "The SEEPS point climo data \"" << seeps_name << "\" is missing!\n" + << "Set the " << MET_ENV_SEEPS_POINT_CLIMO_NAME + << " environment variable to define its location " + << "or disable output for SEEPS and SEEPS_MPR.\n\n"; exit(1); } } @@ -312,10 +316,13 @@ SeepsRecord *SeepsClimo::get_record(int sid, int month, int hour) { } else { mlog << Error << "\n" << method_name - << "The SEEPS climo data is missing!" - << " Turn off SEEPS and SEEPS_MPR to continue\n\n"; + << "The SEEPS point climo data is missing!\n" + << "Set the " << MET_ENV_SEEPS_POINT_CLIMO_NAME + << " environment variable to define its location " + << "or disable output for SEEPS and SEEPS_MPR.\n\n"; exit(1); } + return record; } @@ -326,22 +333,22 @@ ConcatString SeepsClimo::get_seeps_climo_filename() { const char *method_name = "SeepsClimo::get_seeps_climo_filename() -> "; // Use the MET_TMP_DIR environment variable, if set. - bool use_env = get_env(MET_ENV_SEEPS_CLIMO_NAME, seeps_filename); + bool use_env = get_env(MET_ENV_SEEPS_POINT_CLIMO_NAME, seeps_filename); if(use_env) seeps_filename = replace_path(seeps_filename); else seeps_filename = replace_path(def_seeps_filename); if (seeps_ready = file_exists(seeps_filename.c_str())) { - mlog << Debug(7) << method_name << "SEEPS climo name=\"" + mlog << Debug(7) << method_name << "SEEPS point climo name=\"" << seeps_filename.c_str() << "\"\n"; } else { - ConcatString message = " "; + ConcatString message = ""; if (use_env) { message.add("from the env. name "); - message.add(MET_ENV_SEEPS_CLIMO_NAME); + message.add(MET_ENV_SEEPS_POINT_CLIMO_NAME); } mlog << Warning << "\n" << method_name - << "The SEEPS climo name \"" << seeps_filename.c_str() + << "The SEEPS point climo name \"" << seeps_filename.c_str() << "\"" << message << " does not exist!\n\n"; } @@ -652,8 +659,10 @@ SeepsClimoGrid::SeepsClimoGrid(int month, int hour) : month{month}, hour{hour} if (seeps_ready) read_seeps_scores(seeps_name); else { mlog << Error << "\nSeepsClimoGrid::SeepsClimoGrid -> " - << "The SEEPS climo data \"" << seeps_name << "\" is missing!" - << " Turn off SEEPS to continue\n\n"; + << "The SEEPS grid climo data \"" << seeps_name << "\" is missing!\n" + << "Set the " << MET_ENV_SEEPS_GRID_CLIMO_NAME + << " environment variable to define its location " + << "or disable output for SEEPS.\n\n"; exit(1); } @@ -769,24 +778,24 @@ ConcatString SeepsClimoGrid::get_seeps_climo_filename() { const char *method_name = "SeepsClimoGrid::get_seeps_climo_filename() -> "; // Use the MET_TMP_DIR environment variable, if set. - bool use_env = get_env(MET_ENV_SEEPS_CLIMO_GRID_NAME, seeps_filename); + bool use_env = get_env(MET_ENV_SEEPS_GRID_CLIMO_NAME, seeps_filename); if(use_env) { seeps_filename = replace_path(seeps_filename); } else seeps_filename = replace_path(def_seeps_grid_filename); if (seeps_ready = file_exists(seeps_filename.c_str())) { - mlog << Debug(7) << method_name << "SEEPS climo name=\"" + mlog << Debug(7) << method_name << "SEEPS grid climo name=\"" << seeps_filename.c_str() << "\"\n"; } else { - ConcatString message = " "; + ConcatString message = ""; if (use_env) { message.add("from the env. name "); - message.add(MET_ENV_SEEPS_CLIMO_GRID_NAME); + message.add(MET_ENV_SEEPS_GRID_CLIMO_NAME); } mlog << Warning << "\n" << method_name - << "The SEEPS climo name \"" << seeps_filename.c_str() + << "The SEEPS grid climo name \"" << seeps_filename.c_str() << "\"" << message << " does not exist!\n\n"; } diff --git a/src/libcode/vx_seeps/seeps.h b/src/libcode/vx_seeps/seeps.h index ae5915298e..47ee2ad3df 100644 --- a/src/libcode/vx_seeps/seeps.h +++ b/src/libcode/vx_seeps/seeps.h @@ -24,8 +24,8 @@ //////////////////////////////////////////////////////////////////////// -static const char *MET_ENV_SEEPS_CLIMO_NAME = "MET_SEEPS_POINT_CLIMO_NAME"; -static const char *MET_ENV_SEEPS_CLIMO_GRID_NAME = "MET_SEEPS_GRID_CLIMO_NAME"; +static const char *MET_ENV_SEEPS_POINT_CLIMO_NAME = "MET_SEEPS_POINT_CLIMO_NAME"; +static const char *MET_ENV_SEEPS_GRID_CLIMO_NAME = "MET_SEEPS_GRID_CLIMO_NAME"; static const char *dim_name_nstn = "nstn"; diff --git a/src/libcode/vx_statistics/pair_data_point.cc b/src/libcode/vx_statistics/pair_data_point.cc index ff7df4262e..ca965a2520 100644 --- a/src/libcode/vx_statistics/pair_data_point.cc +++ b/src/libcode/vx_statistics/pair_data_point.cc @@ -70,8 +70,8 @@ void PairDataPoint::init_from_scratch() { seeps_mpr.clear(); seeps.clear(); + seeps_climo = NULL; clear(); - seeps_climo = get_seeps_climo(); return; } @@ -180,8 +180,16 @@ bool PairDataPoint::add_point_pair(const char *sid, double lat, double lon, //////////////////////////////////////////////////////////////////////// +void PairDataPoint::load_seeps_climo() { + if (NULL == seeps_climo) seeps_climo = get_seeps_climo(); +} + +//////////////////////////////////////////////////////////////////////// + void PairDataPoint::set_seeps_thresh(const SingleThresh &p1_thresh) { - seeps_climo->set_p1_thresh(p1_thresh); + if (NULL != seeps_climo) seeps_climo->set_p1_thresh(p1_thresh); + else mlog << Warning << "\nPairDataPoint::set_seeps_thresh() ignored t1_threshold." + << " Load SEEPS climo first\n\n"; } //////////////////////////////////////////////////////////////////////// @@ -291,7 +299,7 @@ SeepsScore *PairDataPoint::compute_seeps(const char *sid, double f, int month, day, year, hour, minute, second; int sid_no = atoi(sid); - if (sid_no) { + if (sid_no && NULL != seeps_climo) { unix_to_mdyhms(ut, month, day, year, hour, minute, second); seeps = seeps_climo->get_seeps_score(sid_no, f, o, month, hour); if (mlog.verbosity_level() >= seeps_debug_level @@ -1470,6 +1478,18 @@ void VxPairDataPoint::set_obs_perc_value(int percentile) { //////////////////////////////////////////////////////////////////////// +void VxPairDataPoint::load_seeps_climo() { + for(int i=0; i < n_msg_typ; i++){ + for(int j=0; j < n_mask; j++){ + for(int k=0; k < n_interp; k++){ + pd[i][j][k].load_seeps_climo(); + } + } + } +} + +//////////////////////////////////////////////////////////////////////// + void VxPairDataPoint::set_seeps_thresh(const SingleThresh &p1_thresh) { for(int i=0; i < n_msg_typ; i++){ for(int j=0; j < n_mask; j++){ diff --git a/src/libcode/vx_statistics/pair_data_point.h b/src/libcode/vx_statistics/pair_data_point.h index ab44e87d17..f98a1ee376 100644 --- a/src/libcode/vx_statistics/pair_data_point.h +++ b/src/libcode/vx_statistics/pair_data_point.h @@ -34,6 +34,7 @@ class PairDataPoint : public PairBase { void assign(const PairDataPoint &); SeepsClimo *seeps_climo; + public: PairDataPoint(); @@ -58,6 +59,7 @@ class PairDataPoint : public PairBase { bool add_point_pair(const char *, double, double, double, double, unixtime, double, double, double, double, const char *, double, double, double); + void load_seeps_climo(); void set_seeps_thresh(const SingleThresh &p1_thresh); void set_seeps_score(SeepsScore *, int index=-1); @@ -227,6 +229,7 @@ class VxPairDataPoint { void set_mpr_thresh(const StringArray &, const ThreshArray &); + void load_seeps_climo(); void set_seeps_thresh(const SingleThresh &p1_thresh); void set_climo_cdf_info_ptr(const ClimoCDFInfo *); diff --git a/src/tools/core/grid_stat/grid_stat_conf_info.cc b/src/tools/core/grid_stat/grid_stat_conf_info.cc index 2a5f447972..6173eda494 100644 --- a/src/tools/core/grid_stat/grid_stat_conf_info.cc +++ b/src/tools/core/grid_stat/grid_stat_conf_info.cc @@ -173,9 +173,10 @@ void GridStatConfInfo::process_config(GrdFileType ftype, // Summarize output flags across all verification tasks process_flags(); - // If VL1L2 or VAL1L2 is requested, set the uv_index + // If VL1L2, VAL1L2, or VCNT is requested, set the uv_index if(output_flag[i_vl1l2] != STATOutputType_None || - output_flag[i_val1l2] != STATOutputType_None) { + output_flag[i_val1l2] != STATOutputType_None || + output_flag[i_vcnt] != STATOutputType_None) { for(i=0; iis_v_wind() && - vx_opt[j].obs_info->is_v_wind() && - vx_opt[i].fcst_info->req_level_name() == - vx_opt[j].fcst_info->req_level_name() && - vx_opt[i].obs_info->req_level_name() == - vx_opt[j].obs_info->req_level_name() && + if(vx_opt[j].fcst_info->is_v_wind() && + vx_opt[j].obs_info->is_v_wind() && vx_opt[i].is_uv_match(vx_opt[j])) { - vx_opt[i].fcst_info->set_uv_index(j); - vx_opt[i].obs_info->set_uv_index(j); + mlog << Debug(3) << "U-wind field array entry " << i+1 + << " matches V-wind field array entry " << j+1 << ".\n"; + + // Print warning about multiple matches + if(vx_opt[i].fcst_info->uv_index() >= 0 || + vx_opt[i].obs_info->uv_index() >= 0) { + mlog << Warning << "\nGridStatConfInfo::process_config() -> " + << "For U-wind, found multiple matching V-wind field array entries! " + << "Using the first match found. Set the \"level\" strings to " + << "differentiate between them.\n\n"; + } + // Use the first match + else { + vx_opt[i].fcst_info->set_uv_index(j); + vx_opt[i].obs_info->set_uv_index(j); + } } } + + // No match found + if(vx_opt[i].fcst_info->uv_index() < 0 || + vx_opt[i].obs_info->uv_index() < 0) { + mlog << Debug(3) << "U-wind field array entry " << i+1 + << " has no matching V-wind field array entry.\n"; + } + } // Process v-wind else if(vx_opt[i].fcst_info->is_v_wind() && @@ -204,18 +223,36 @@ void GridStatConfInfo::process_config(GrdFileType ftype, // Search for corresponding u-wind for(j=0; jis_u_wind() && - vx_opt[j].obs_info->is_u_wind() && - vx_opt[i].fcst_info->req_level_name() == - vx_opt[j].fcst_info->req_level_name() && - vx_opt[i].obs_info->req_level_name() == - vx_opt[j].obs_info->req_level_name() && + if(vx_opt[j].fcst_info->is_u_wind() && + vx_opt[j].obs_info->is_u_wind() && vx_opt[i].is_uv_match(vx_opt[j])) { - vx_opt[i].fcst_info->set_uv_index(j); - vx_opt[i].obs_info->set_uv_index(j); + mlog << Debug(3) << "V-wind field array entry " << i+1 + << " matches U-wind field array entry " << j+1 << ".\n"; + + // Print warning about multiple matches + if(vx_opt[i].fcst_info->uv_index() >= 0 || + vx_opt[i].obs_info->uv_index() >= 0) { + mlog << Warning << "\nGridStatConfInfo::process_config() -> " + << "For V-wind, found multiple matching U-wind field array entries! " + << "Using the first match found. Set the \"level\" strings to " + << "differentiate between them.\n\n"; + } + // Use the first match + else { + vx_opt[i].fcst_info->set_uv_index(j); + vx_opt[i].obs_info->set_uv_index(j); + } } } + + // No match found + if(vx_opt[i].fcst_info->uv_index() < 0 || + vx_opt[i].obs_info->uv_index() < 0) { + mlog << Debug(3) << "V-wind field array entry " << i+1 + << " has no matching U-wind field array entry.\n"; + } + } } // end for i } // end if @@ -904,6 +941,10 @@ void GridStatVxOpt::parse_nc_info(Dictionary &odict) { bool GridStatVxOpt::is_uv_match(const GridStatVxOpt &v) const { bool match = true; + // + // Check that requested forecast and observation levels match. + // Requested levels are optional for python embedding and may be empty. + // Check that the masking regions and interpolation options match. // // The following do not impact matched pairs: // desc, var_name, var_suffix, @@ -917,12 +958,14 @@ bool GridStatVxOpt::is_uv_match(const GridStatVxOpt &v) const { // hss_ec_value, rank_corr_flag, output_flag, nc_info // - if(!(mask_grid == v.mask_grid ) || + if(!is_req_level_match( fcst_info->req_level_name(), + v.fcst_info->req_level_name()) || + !is_req_level_match( obs_info->req_level_name(), + v.obs_info->req_level_name()) || + !(mask_grid == v.mask_grid ) || !(mask_poly == v.mask_poly ) || !(mask_name == v.mask_name ) || - !(interp_info == v.interp_info)) { - match = false; - } + !(interp_info == v.interp_info)) match = false; return(match); } diff --git a/src/tools/core/point_stat/point_stat_conf_info.cc b/src/tools/core/point_stat/point_stat_conf_info.cc index 7b811ce17a..fadd283b9c 100644 --- a/src/tools/core/point_stat/point_stat_conf_info.cc +++ b/src/tools/core/point_stat/point_stat_conf_info.cc @@ -165,11 +165,12 @@ void PointStatConfInfo::process_config(GrdFileType ftype) { // Summarize output flags across all verification tasks process_flags(); - // If VL1L2 or VAL1L2 is requested, set the uv_index. + // If VL1L2, VAL1L2, or VCNT is requested, set the uv_index. // When processing vectors, need to make sure the message types, // masking regions, and interpolation methods are consistent. if(output_flag[i_vl1l2] != STATOutputType_None || - output_flag[i_val1l2] != STATOutputType_None) { + output_flag[i_val1l2] != STATOutputType_None || + output_flag[i_vcnt] != STATOutputType_None) { for(i=0; iis_v_wind() && - vx_opt[j].vx_pd.obs_info->is_v_wind() && - vx_opt[i].vx_pd.fcst_info->req_level_name() == - vx_opt[j].vx_pd.fcst_info->req_level_name() && - vx_opt[i].vx_pd.obs_info->req_level_name() == - vx_opt[j].vx_pd.obs_info->req_level_name() && + if(vx_opt[j].vx_pd.fcst_info->is_v_wind() && + vx_opt[j].vx_pd.obs_info->is_v_wind() && vx_opt[i].is_uv_match(vx_opt[j])) { - vx_opt[i].vx_pd.fcst_info->set_uv_index(j); - vx_opt[i].vx_pd.obs_info->set_uv_index(j); + mlog << Debug(3) << "U-wind field array entry " << i+1 + << " matches V-wind field array entry " << j+1 << ".\n"; + + // Print warning about multiple matches + if(vx_opt[i].vx_pd.fcst_info->uv_index() >= 0 || + vx_opt[i].vx_pd.obs_info->uv_index() >= 0) { + mlog << Warning << "\nPointStatConfInfo::process_config() -> " + << "For U-wind, found multiple matching V-wind field array entries! " + << "Using the first match found. Set the \"level\" strings to " + << "differentiate between them.\n\n"; + } + // Use the first match + else { + vx_opt[i].vx_pd.fcst_info->set_uv_index(j); + vx_opt[i].vx_pd.obs_info->set_uv_index(j); + } } } + + // No match found + if(vx_opt[i].vx_pd.fcst_info->uv_index() < 0 || + vx_opt[i].vx_pd.obs_info->uv_index() < 0) { + mlog << Debug(3) << "U-wind field array entry " << i+1 + << " has no matching V-wind field array entry.\n"; + } + } // Process v-wind else if(vx_opt[i].vx_pd.fcst_info->is_v_wind() && @@ -198,18 +217,36 @@ void PointStatConfInfo::process_config(GrdFileType ftype) { // Search for corresponding u-wind for(j=0; jis_u_wind() && - vx_opt[j].vx_pd.obs_info->is_u_wind() && - vx_opt[i].vx_pd.fcst_info->req_level_name() == - vx_opt[j].vx_pd.fcst_info->req_level_name() && - vx_opt[i].vx_pd.obs_info->req_level_name() == - vx_opt[j].vx_pd.obs_info->req_level_name() && + if(vx_opt[j].vx_pd.fcst_info->is_u_wind() && + vx_opt[j].vx_pd.obs_info->is_u_wind() && vx_opt[i].is_uv_match(vx_opt[j])) { - vx_opt[i].vx_pd.fcst_info->set_uv_index(j); - vx_opt[i].vx_pd.obs_info->set_uv_index(j); + mlog << Debug(3) << "V-wind field array entry " << i+1 + << " matches U-wind field array entry " << j+1 << ".\n"; + + // Print warning about multiple matches + if(vx_opt[i].vx_pd.fcst_info->uv_index() >= 0 || + vx_opt[i].vx_pd.obs_info->uv_index() >= 0) { + mlog << Warning << "\nPointStatConfInfo::process_config() -> " + << "For U-wind, found multiple matching V-wind field array entries! " + << "Using the first match found. Set the \"level\" strings to " + << "differentiate between them.\n\n"; + } + // Use the first match + else { + vx_opt[i].vx_pd.fcst_info->set_uv_index(j); + vx_opt[i].vx_pd.obs_info->set_uv_index(j); + } } } + + // No match found + if(vx_opt[i].vx_pd.fcst_info->uv_index() < 0 || + vx_opt[i].vx_pd.obs_info->uv_index() < 0) { + mlog << Debug(3) << "V-wind field array entry " << i+1 + << " has no matching U-wind field array entry.\n"; + } + } } // end for i } // end if @@ -704,6 +741,10 @@ void PointStatVxOpt::clear() { bool PointStatVxOpt::is_uv_match(const PointStatVxOpt &v) const { bool match = true; + // + // Check that requested forecast and observation levels match. + // Requested levels are optional for python embedding and may be empty. + // Check that several other config options also match. // // The following do not impact matched pairs: // fcat_ta, ocat_ta, @@ -714,7 +755,11 @@ bool PointStatVxOpt::is_uv_match(const PointStatVxOpt &v) const { // rank_corr_flag, output_flag // - if(!(beg_ds == v.beg_ds ) || + if(!is_req_level_match( vx_pd.fcst_info->req_level_name(), + v.vx_pd.fcst_info->req_level_name()) || + !is_req_level_match( vx_pd.obs_info->req_level_name(), + v.vx_pd.obs_info->req_level_name()) || + !(beg_ds == v.beg_ds ) || !(end_ds == v.end_ds ) || !(land_flag == v.land_flag ) || !(topo_flag == v.topo_flag ) || @@ -727,9 +772,7 @@ bool PointStatVxOpt::is_uv_match(const PointStatVxOpt &v) const { !(msg_typ == v.msg_typ ) || !(duplicate_flag == v.duplicate_flag) || !(obs_summary == v.obs_summary ) || - !(obs_perc == v.obs_perc )) { - match = false; - } + !(obs_perc == v.obs_perc )) match = false; return(match); } @@ -1091,7 +1134,11 @@ void PointStatVxOpt::set_vx_pd(PointStatConfInfo *conf_info) { vx_pd.set_duplicate_flag(duplicate_flag); vx_pd.set_obs_summary(obs_summary); vx_pd.set_obs_perc_value(obs_perc); - vx_pd.set_seeps_thresh(seeps_p1_thresh); + if (output_flag[i_seeps_mpr] != STATOutputType_None + || output_flag[i_seeps] != STATOutputType_None) { + vx_pd.load_seeps_climo(); + vx_pd.set_seeps_thresh(seeps_p1_thresh); + } return; } diff --git a/src/tools/other/ascii2nc/aeronet_handler.cc b/src/tools/other/ascii2nc/aeronet_handler.cc index 7cdc07d7d0..1d6b9dbd2d 100644 --- a/src/tools/other/ascii2nc/aeronet_handler.cc +++ b/src/tools/other/ascii2nc/aeronet_handler.cc @@ -21,7 +21,7 @@ using namespace std; #include #include #include -#include +#include #include "vx_log.h" #include "vx_math.h" @@ -30,13 +30,13 @@ using namespace std; #include "aeronet_handler.h" static const char *AERONET_NA_STR = "N/A"; +static const char *AERONET_V3_STR = "AERONET Version 3"; const int AeronetHandler::NUM_HDR_COLS = 7; const int AeronetHandler::NUM_OBS_COLS = 45; //const int NUM_OBS_COLS_V3 = 113; //const int NUM_OBS_COLS_V3_tot = 57; //const int NUM_OBS_COLS_V3_oneill = 267; - const int version_3_columns[7] = { 113, 81, 53, 33, 41, 259 }; const string site_name_col = "AERONET_Site_Name"; @@ -46,6 +46,8 @@ const string elv_col1 = "Site_Elevation"; // "Site_Elevation(m)"; const string lat_col2 = "Latitude"; // "Latitude(degrees)" const string lon_col2 = "Longitude"; // "Longitude(degrees)" const string elv_col2 = "Elevation"; // "Elevation(meters)" +const string date_col = "Date"; // "Date(dd:mm:yyyy)" +const string month_col = "Month"; // "Month" const string AeronetHandler::HEADER_TYPE = ""; @@ -67,8 +69,10 @@ const string WAVELENGTHS_PW_NAME = "Exact_Wavelengths_of_PW"; // Exact_Wavelen const string WAVELENGTHS_INPUT_AOD_NAME = "Exact_Wavelengths_for_Input_AOD"; // Exact_Wavelengths_for_Input_AOD(um) static int format_version; +const string SITE_MISSING = "site_missing"; -double angstrom_power_interplation(double value_1, double value_2, double level_1, double level_2, double target_level); +double angstrom_power_interplation(double value_1, double value_2, double level_1, + double level_2, double target_level); //////////////////////////////////////////////////////////////////////// @@ -119,7 +123,7 @@ bool AeronetHandler::isFileType(LineDataFile &ascii_file) const string line = dl.get_line(); if (line.length() > 17) { line = line.substr(0, 17); - if (strcmp(line.c_str(), "AERONET Version 3") == 0) { + if (strcmp(line.c_str(), AERONET_V3_STR) == 0) { is_file_type = true; format_version = 3; return is_file_type; @@ -154,7 +158,7 @@ void AeronetHandler::setFormatVersion(int version) { bool AeronetHandler::_readObservations(LineDataFile &ascii_file) { DataLine data_line; - string method_name = "AeronetHandler::_readObservations() "; + string method_name = "AeronetHandler::_readObservations() --> "; // // Read and save the station name, latitude, longitude, and elevation. @@ -169,15 +173,16 @@ bool AeronetHandler::_readObservations(LineDataFile &ascii_file) ascii_file >> data_line; string header_type = "AERONET_AOD"; - + // // Get the field information from the fifth header line // if (format_version == 3) { use_var_id = true; - // Get the field information from the 7-th header line - ascii_file >> data_line; + // Skip lines to get the field information from 7-th header line + // (with site name at line 2) or the 6-th header line (without site name) + if(_stationId != SITE_MISSING) ascii_file >> data_line; ascii_file >> data_line; ascii_file >> data_line; } @@ -195,7 +200,7 @@ bool AeronetHandler::_readObservations(LineDataFile &ascii_file) int flag; int aod_var_id = bad_data_int; - int var_idx, sid_idx, elv_idx, lat_idx, lon_idx; + int var_idx, sid_idx, elv_idx, lat_idx, lon_idx, date_idx, month_idx; double height_from_header; string aot = "AOT"; //string angstrom = "Angstrom"; @@ -204,8 +209,9 @@ bool AeronetHandler::_readObservations(LineDataFile &ascii_file) NumArray header_heights; IntArray header_var_index; StringArray header_var_names; - - sid_idx = elv_idx = lat_idx = lon_idx = -1; + + date_idx = 0; + sid_idx = elv_idx = lat_idx = lon_idx = month_idx = -1; for (int j = 0; j < hdr_tokens.n_elements(); j++) { @@ -239,15 +245,17 @@ bool AeronetHandler::_readObservations(LineDataFile &ascii_file) { if (hdr_field.find("Empty") == string::npos) flag = 1; } - + process_flag.add(flag); - + if (format_version == 3) { if (0 == hdr_field.find(site_name_col)) sid_idx = j; else if (0 == hdr_field.find(lat_col1) || 0 == hdr_field.find(lat_col2)) lat_idx = j; else if (0 == hdr_field.find(lon_col1) || 0 == hdr_field.find(lon_col2)) lon_idx = j; else if (0 == hdr_field.find(elv_col1) || 0 == hdr_field.find(elv_col2)) elv_idx = j; - + else if (0 == strcmp(hdr_field.c_str(), month_col.c_str())) month_idx = j; + else if (0 == hdr_field.find(date_col)) date_idx = j; + // Collect variable names and index var_name = make_var_name_from_header(hdr_field); if (!var_names.has(var_name.c_str(), var_idx)) { @@ -261,17 +269,17 @@ bool AeronetHandler::_readObservations(LineDataFile &ascii_file) header_var_index.add(var_idx); header_var_names.add(var_name.c_str()); header_heights.add(height_from_header); - mlog << Debug(5) << method_name << "header_idx: " << j + mlog << Debug(7) << method_name << "header_idx: " << j << ", var_idx: " << var_idx << ", var: " << var_name << " from " << hdr_field << ", flag: " << flag << ", height: " << height_from_header << "\n"; } } - + int column_cnt = NUM_OBS_COLS; if (format_version == 3) { column_cnt = get_header_count_v3(hdr_tokens); } - + // // Process the observation lines // @@ -288,7 +296,7 @@ bool AeronetHandler::_readObservations(LineDataFile &ascii_file) { bad_line_count++; if (format_version != 3) { - mlog << Error << "\nAeronetHandler" << method_name << "-> " + mlog << Error << "\n" << method_name << "line number " << data_line.line_number() << " does not have the correct number of columns " << data_line.n_items() << " (" << column_cnt << "). Stop processing \"" @@ -296,7 +304,7 @@ bool AeronetHandler::_readObservations(LineDataFile &ascii_file) return false; } else if (data_line.n_items() < column_cnt) { - mlog << Error << "\nAeronetHandler::_readObservations() -> " + mlog << Error << "\n" << method_name << "line number " << data_line.line_number() << " does not have the correct number of columns " << data_line.n_items() << " (" << column_cnt << "). Stop processing \"" @@ -305,7 +313,7 @@ bool AeronetHandler::_readObservations(LineDataFile &ascii_file) } else { if (bad_line_count < 5) { - mlog << Warning << "\nAeronetHandler::_readObservations() -> " + mlog << Warning << "\n" << method_name << "line number " << data_line.line_number() << " has more number of columns " << data_line.n_items() << " (" << column_cnt << ").\n\n"; @@ -317,25 +325,25 @@ bool AeronetHandler::_readObservations(LineDataFile &ascii_file) if (format_version == 3) { // Get the stationId if (elv_idx < 0) { - mlog << Warning << "AeronetHandler::_readObservations() Can not find header column \"" + mlog << Warning << "\n" << method_name << "Can not find header column \"" << elv_col2 << "\". from " << ascii_file.filename() << "\".\n\n"; break; } else if ((lat_idx < 0) || (lon_idx < 0)) { string field_name = (lat_idx < 0) ? lat_col2 : lon_col2; - mlog << Error << "AeronetHandler::_readObservations() Can not find header column \"" + mlog << Error << "\n" << method_name << "Can not find header column \"" << field_name << "\". Skip the input \"" << ascii_file.filename() << "\"\n\n"; break; } else { if (sid_idx < 0) { - mlog << Warning << "AeronetHandler::_readObservations() Can not find header column \"" + mlog << Warning << "\n" << method_name << "Can not find header column \"" << site_name_col << "\" from the input \"" << ascii_file.filename() << "\"\n\n"; } - else if (_stationId != data_line[sid_idx]) { - mlog << Error << "\nAeronetHandler::_readObservations() The header and data columns don't match." + else if (_stationId != data_line[sid_idx] && _stationId != SITE_MISSING) { + mlog << Error << "\n" << method_name << "The header and data columns don't match." << " The station ID from data column (" << data_line[sid_idx] << ") at " << sid_idx << " is different from " << _stationId << ". Skip this input \"" << ascii_file.filename() @@ -343,7 +351,7 @@ bool AeronetHandler::_readObservations(LineDataFile &ascii_file) break; } } - + // Get the stationLat _stationLat = atof(data_line[lat_idx]); // Get the stationLon @@ -351,8 +359,8 @@ bool AeronetHandler::_readObservations(LineDataFile &ascii_file) // Get the stationAlt if (elv_idx >= 0) _stationAlt = atof(data_line[elv_idx]); else _stationAlt = bad_data_float; - - mlog << Debug(5) << "AeronetHandler::_readObservations() stationID: " + + mlog << Debug(7) << "\n" << method_name << "stationID: " << ((sid_idx < 0) ? _stationId : data_line[sid_idx]) << " from index " << sid_idx << " lat: " << _stationLat << " lon: " << _stationLon @@ -364,7 +372,7 @@ bool AeronetHandler::_readObservations(LineDataFile &ascii_file) // Pull the valid time from the data line // - time_t valid_time = _getValidTime(data_line); + time_t valid_time = _getValidTime(data_line, (0 <= month_idx ? month_idx : date_idx)); if (valid_time == 0) return false; @@ -372,10 +380,10 @@ bool AeronetHandler::_readObservations(LineDataFile &ascii_file) bool has_aod_at_550; double aod_at_440, aod_at_675; int var_id = AOT_GRIB_CODE; - + has_aod_at_550 = false; aod_at_440 = aod_at_675 = bad_data_float; - + var_name = AOT_NAME; // // Save the desired observations from the line @@ -386,7 +394,6 @@ bool AeronetHandler::_readObservations(LineDataFile &ascii_file) string hdr_field = hdr_tokens[k]; size_t found_aot = hdr_field.find(aot); - //int found_angstrom = hdr_field.find(angstrom); string height = ""; if (found_aot != string::npos) @@ -394,19 +401,10 @@ bool AeronetHandler::_readObservations(LineDataFile &ascii_file) height = hdr_field.substr((found_aot + 4), hdr_field.size() - 1); } - //if (found_angstrom != string::npos) - //{ - // size_t found_dash = hdr_field.find("-"); - // if (found_dash != string::npos) - // { - // height = hdr_field.substr(0, found_dash); - // } - //} - if(strcmp(data_line[k], AERONET_NA_STR) == 0) continue; var_name = AOT_NAME; - + double dlevel = bad_data_double; double dheight = atoi(height.c_str()); @@ -420,8 +418,9 @@ bool AeronetHandler::_readObservations(LineDataFile &ascii_file) else if (is_eq(dheight, 675)) aod_at_675 = atof(data_line[k]); } } - - _addObservations(Observation(header_type, _stationId, + + _addObservations(Observation(header_type, + (sid_idx<0 ? _stationId : data_line[sid_idx]), valid_time, _stationLat, _stationLon, _stationAlt, @@ -437,28 +436,28 @@ bool AeronetHandler::_readObservations(LineDataFile &ascii_file) var_name = AOD_NAME; double dheight = 550; double aod_at_550 = angstrom_power_interplation(aod_at_675,aod_at_440,675.,440.,dheight); - _addObservations(Observation(header_type, _stationId, valid_time, - _stationLat, _stationLon, _stationAlt, + _addObservations(Observation(header_type, + (sid_idx<0 ? _stationId : data_line[sid_idx]), + valid_time, _stationLat, _stationLon, _stationAlt, na_str, var_id, bad_data_double, dheight, aod_at_550, var_name)); - mlog << Debug(7) << "AeronetHandler::_readObservations() AOD at 550: " + mlog << Debug(7) << method_name << " AOD at 550: " << aod_at_550 << "\t440: " << aod_at_440 << "\t675: " << aod_at_675 << "\n"; } } } // end while if (bad_line_count > 0) { - mlog << Warning << "\nAeronetHandler::_readObservations() -> " - << "Found " << bad_line_count - << " lines with more data columns from " + mlog << Warning << "\n" << method_name + << "Found " << bad_line_count + << " lines with more data columns from " << ascii_file.filename() << "\".\n\n"; } - + if (format_version == 3) { double aod_at_675, aod_at_440; double aod_at_550_expected, aod_at_550; - //double angstrom_675_440_expected, angstrom_675_440; aod_at_675 = 0.645283; aod_at_440 = 0.794593; @@ -466,16 +465,16 @@ bool AeronetHandler::_readObservations(LineDataFile &ascii_file) //angstrom_675_440_expected = 0.486381371; aod_at_550 = angstrom_power_interplation(aod_at_675,aod_at_440,675.,440.,550); if (! is_eq(aod_at_550, aod_at_550_expected)) - mlog << Warning << "AeronetHandler::_readObservations() Check AOD at 550: " + mlog << Warning << "\n" << method_name << "Check AOD at 550: " << aod_at_550 << " (" << aod_at_550_expected << ")" << "\t440: " << aod_at_440 - << "\t675: " << aod_at_675 + << "\t675: " << aod_at_675 << "\n"; else - mlog << Debug(3) << "AeronetHandler::_readObservations() Confirmed AOD interpolation at 550: " + mlog << Debug(3) << method_name << "Confirmed AOD interpolation at 550: " << aod_at_550 << " (" << aod_at_550_expected << ")" << "\t440: " << aod_at_440 - << "\t675: " << aod_at_675 + << "\t675: " << aod_at_675 << "\n"; aod_at_675 = 0.669274; @@ -484,16 +483,16 @@ bool AeronetHandler::_readObservations(LineDataFile &ascii_file) //angstrom_675_440_expected = 0.526983959; aod_at_550 = angstrom_power_interplation(aod_at_675,aod_at_440,675.,440.,550); if (! is_eq(aod_at_550, aod_at_550_expected)) - mlog << Warning << "AeronetHandler::_readObservations() Check AOD at 550: " + mlog << Warning << "\n" << method_name << "Check AOD at 550: " << aod_at_550 << " (" << aod_at_550_expected << ")" << "\t440: " << aod_at_440 - << "\t675: " << aod_at_675 + << "\t675: " << aod_at_675 << "\n"; else - mlog << Debug(3) << "AeronetHandler::_readObservations() Confirmed AOD interpolation at 550: " + mlog << Debug(3) << method_name << "Confirmed AOD interpolation at 550: " << aod_at_550 << " (" << aod_at_550_expected << ")" << "\t440: " << aod_at_440 - << "\t675: " << aod_at_675 + << "\t675: " << aod_at_675 << "\n"; } return true; @@ -501,29 +500,44 @@ bool AeronetHandler::_readObservations(LineDataFile &ascii_file) //////////////////////////////////////////////////////////////////////// -time_t AeronetHandler::_getValidTime(const DataLine &data_line) const +time_t AeronetHandler::_getValidTime(const DataLine &data_line, int date_offset) const { // // Pull out the date information // - ConcatString date_string(data_line[0]); + string mday; + string mon; + string year; + int month = -1; + ConcatString date_string(data_line[date_offset]); + bool date_yyyymmm = false; StringArray dateTokens = date_string.split(":"); if (1 == dateTokens.n_elements()) { - mlog << Error << "\nAeronetHandler::_getValidTime -> " - << "Not supported date: \"" << date_string << "\".\n\n"; - return 0; + // Support "yyyy-MMM" + StringArray ymTokens = date_string.split("-"); + if (2 == ymTokens.n_elements()) { + year = ymTokens[0]; + month = month_name_to_m(ymTokens[1].c_str()); + if (0 < month && month <= 12) date_yyyymmm = true; + } + if (!date_yyyymmm) { + mlog << Error << "\nAeronetHandler::_getValidTime -> " + << "Not supported date: \"" << date_string << "\".\n\n"; + return 0; + } + } + else { + mday = dateTokens[0]; + mon = dateTokens[1]; + year = dateTokens[2]; } - - string mday = dateTokens[0]; - string mon = dateTokens[1]; - string year = dateTokens[2]; // // Pull out the time information // - ConcatString time_string(data_line[1]); + ConcatString time_string(data_line[date_offset+1]); StringArray timeTokens = time_string.split(":"); // @@ -534,13 +548,13 @@ time_t AeronetHandler::_getValidTime(const DataLine &data_line) const memset(&time_struct, 0, sizeof(time_struct)); time_struct.tm_year = atoi(year.c_str()) - 1900; - time_struct.tm_mon = atoi(mon.c_str()) - 1; - time_struct.tm_mday = atoi(mday.c_str()); - if (3 <= timeTokens.n_elements()) { + time_struct.tm_mon = ((0 < month) ? month : atoi(mon.c_str())) - 1; + time_struct.tm_mday = (0 < month) ? 1 : atoi(mday.c_str()); + if (3 <= timeTokens.n_elements() && !date_yyyymmm) { string hour = timeTokens[0]; string min = timeTokens[1]; string sec = timeTokens[2]; - + time_struct.tm_hour = atoi(hour.c_str()); time_struct.tm_min = atoi(min.c_str()); time_struct.tm_sec = atoi(sec.c_str()); @@ -556,6 +570,7 @@ bool AeronetHandler::_readHeaderInfo(LineDataFile &ascii_file) { DataLine data_line; data_line.set_delimiter(","); + string method_name = "AeronetHandler::_readHeaderInfo(() --> "; // // Skip the first two lines @@ -570,15 +585,16 @@ bool AeronetHandler::_readHeaderInfo(LineDataFile &ascii_file) if (format_version == 3) { _stationId = data_line[0]; - if (' ' == _stationId[0]) _stationId = _stationId.substr(1); - mlog << Debug(5) << " _stationId: [" << _stationId << "]\n"; + if (string::npos != _stationId.find(AERONET_V3_STR)) _stationId = SITE_MISSING; + else if (' ' == _stationId[0]) _stationId = _stationId.substr(1); + mlog << Debug(5) << method_name << " _stationId: [" << _stationId << "]\n"; // read lat/lon from https://aeronet.gsfc.nasa.gov/aeronet_locations_v3.txt return true; } if (!(ascii_file >> data_line)) { - mlog << Error << "\nAeronetHandler::_readHeaderInfo() -> " + mlog << Error << "\n" << method_name << "error reading station id line from input ASCII file \"" << ascii_file.filename() << "\"\n\n"; @@ -591,7 +607,7 @@ bool AeronetHandler::_readHeaderInfo(LineDataFile &ascii_file) if (data_line.n_items() != NUM_HDR_COLS) { - mlog << Error << "\nAeronetHandler::_readHeaderInfo() -> " + mlog << Error << "\n" << method_name << "AERONET file has incorrect number of columns (" << data_line.n_items() << ") in header line\n\n"; return false; @@ -644,7 +660,7 @@ double AeronetHandler::extract_height(string hdr_field) { bool with_unit = false; string tmp_height; StringArray hdr_names; - + hdr_names.parse_delim(hdr_field.c_str(), "_"); // AOD_1640nm-Total,AOD_1640nm-AOD,AOD_1640nm-Rayleigh,AOD_1640nm-O3, // AOD_1640nm-NO2,AOD_1640nm-CO2,AOD_1640nm-CH4,AOD_1640nm-WaterVapor @@ -681,10 +697,10 @@ double AeronetHandler::extract_height(string hdr_field) { tmp_hdr_names.parse_delim(WAVELENGTHS_INPUT_AOD_NAME.c_str(), "_"); token_count = tmp_hdr_names.n_elements(); } - + if (0 < token_count && token_count < hdr_names.n_elements()) height_str = hdr_names[hdr_names.n_elements()-1]; - + if (with_unit && height_str.length() > 2) height_str = height_str.substr(0, (height_str.length()-2)); } @@ -698,7 +714,7 @@ double AeronetHandler::extract_height(string hdr_field) { mlog << Warning << "AeronetHandler::extract_height() converted to 0 from (" << height_str << ")\n"; } } - + return height; } @@ -706,7 +722,7 @@ double AeronetHandler::extract_height(string hdr_field) { int AeronetHandler::get_header_count_v3(StringArray hdr_tokens) { int header_cnt = hdr_tokens.n_elements(); - + mlog << Debug(5) << "get_header_count_v3() " << header_cnt << "\n"; return header_cnt; } diff --git a/src/tools/other/ascii2nc/aeronet_handler.h b/src/tools/other/ascii2nc/aeronet_handler.h index 9a04de1dc6..34a47029ce 100644 --- a/src/tools/other/ascii2nc/aeronet_handler.h +++ b/src/tools/other/ascii2nc/aeronet_handler.h @@ -103,7 +103,7 @@ class AeronetHandler : public FileHandler // Get the observation valid time from the given observation line - time_t _getValidTime(const DataLine &data_line) const; + time_t _getValidTime(const DataLine &data_line, int date_offset) const; // Read the observations from the given file and add them to the // _observations vector. diff --git a/src/tools/other/ascii2nc/airnow_handler.cc b/src/tools/other/ascii2nc/airnow_handler.cc index fbc655f9dd..cc710c47f0 100644 --- a/src/tools/other/ascii2nc/airnow_handler.cc +++ b/src/tools/other/ascii2nc/airnow_handler.cc @@ -16,7 +16,7 @@ using namespace std; #include #include #include -#include +#include #include #include "vx_log.h" @@ -148,17 +148,17 @@ bool AirnowHandler::isFileType(LineDataFile &ascii_file) const // try delimeter "|" and check for DAILYV2 or HOURLY StringArray tokens = cstring.split("|"); - if (NUM_COLS_DAILYV2 == tokens.n_elements()) { + if (NUM_COLS_DAILYV2 == tokens.n()) { is_file_type = true; return is_file_type; - } else if (NUM_COLS_HOURLY == tokens.n_elements()) { + } else if (NUM_COLS_HOURLY == tokens.n()) { is_file_type = true; return is_file_type; } // try with "," delimiter and look for HOURLYABQ StringArray tokens2 = cstring.split(","); - if (NUM_COLS_HOURLYAQOBS == tokens2.n_elements()) { + if (NUM_COLS_HOURLYAQOBS == tokens2.n()) { is_file_type = true; return is_file_type; } @@ -177,20 +177,20 @@ void AirnowHandler::setFormatVersion(int version) { bool AirnowHandler::_readObservations(LineDataFile &ascii_file) { DataLine data_line; - string method_name = "AirnowHandler::_readObservations() "; + string method_name = "AirnowHandler::_readObservations() -> "; string header_type; string delimiter; int column_cnt; _initializeColumnPointers(); - + if (format_version == AIRNOW_FORMAT_VERSION_UNKNOWN) { // try to figure out which one it is and set it if (!_determineFileType(ascii_file)) { return false; } } - + if (format_version == AIRNOW_FORMAT_VERSION_DAILYV2) { doStripQuotes = false; _setDailyv2HeaderInfo(); @@ -200,8 +200,8 @@ bool AirnowHandler::_readObservations(LineDataFile &ascii_file) } else if (format_version == AIRNOW_FORMAT_VERSION_HOURLYAQOBS) { doStripQuotes = true; - if (!_readHeaderInfo(ascii_file)) - return false; + if (!_readHeaderInfo(ascii_file)) return false; + header_type = "AIRNOW_HOURLY_AQOBS"; delimiter = ","; column_cnt = NUM_COLS_HOURLYAQOBS; @@ -209,21 +209,19 @@ bool AirnowHandler::_readObservations(LineDataFile &ascii_file) else if (format_version == AIRNOW_FORMAT_VERSION_HOURLY) { doStripQuotes = false; _setHourlyHeaderInfo(); - if (!locations.initialize(monitoringSiteFileName)) { - return false; - } + if (!locations.initialize(monitoringSiteFileName)) return false; header_type = "AIRNOW_HOURLY"; delimiter = "|"; column_cnt = NUM_COLS_HOURLY; } else { - mlog << Error << method_name << " format value=" << format_version << " expect " - << AIRNOW_FORMAT_VERSION_DAILYV2 << " or " << AIRNOW_FORMAT_VERSION_HOURLYAQOBS - << " or " << AIRNOW_FORMAT_VERSION_HOURLY << "\n\n"; + mlog << Error << method_name + << "format value=" << format_version << " expect " + << AIRNOW_FORMAT_VERSION_DAILYV2 << " or " << AIRNOW_FORMAT_VERSION_HOURLYAQOBS + << " or " << AIRNOW_FORMAT_VERSION_HOURLY << "\n\n"; header_type = "AIRNOW_HOURLY"; return false; } - header_names.dump(cout); if (format_version == AIRNOW_FORMAT_VERSION_HOURLYAQOBS) { // need to process this a special way because of comma separated strings with @@ -237,8 +235,8 @@ bool AirnowHandler::_readObservations(LineDataFile &ascii_file) //////////////////////////////////////////////////////////////////////// bool AirnowHandler::_readObservationsStandard(LineDataFile &ascii_file, - int column_cnt, const string &delimiter, - const string &header_type) + int column_cnt, const string &delimiter, + const string &header_type) { DataLine data_line; // @@ -248,54 +246,57 @@ bool AirnowHandler::_readObservationsStandard(LineDataFile &ascii_file, data_line.set_delimiter(delimiter.c_str()); while (ascii_file >> data_line) { if (!_parseObservationLineStandard(data_line, ascii_file.filename(), - column_cnt, header_type)) { + column_cnt, header_type)) { bad_line_count++; } } return true; } - + //////////////////////////////////////////////////////////////////////// bool AirnowHandler::_parseObservationLineStandard(DataLine &data_line, - const string &filename, - int column_cnt, - const string &header_type) + const string &filename, + int column_cnt, + const string &header_type) { - string method_name = "AirnowHandler::_parseObservationLineStandard() "; + string method_name = "AirnowHandler::_parseObservationLineStandard() -> "; // // Make sure that the line contains the correct number of tokens // if (data_line.n_items() != column_cnt) { - mlog << Error << "\n" << method_name << "-> " - << "line number " << data_line.line_number() - << " does not have the correct number of columns " << data_line.n_items() - << " (" << column_cnt << "). Skipping this line in \"" - << filename << "\".\n\n"; + mlog << Error << "\n" << method_name + << "line number " << data_line.line_number() + << " does not have the correct number of columns " << data_line.n_items() + << " (" << column_cnt << "). Skipping this line in \"" + << filename << "\".\n\n"; return false; } time_t valid_time = _getValidTime(data_line); if (valid_time == 0) { - mlog << Error << "\n" << method_name << "-> " - << "line number " << data_line.line_number() - << " time could not be parsed, skipping this line in \"" - << filename << "\".\n\n"; + mlog << Error << "\n" << method_name + << "line number " << data_line.line_number() + << " time could not be parsed, skipping this line in \"" + << filename << "\".\n\n"; return false; } - + // fill in expected things double lat, lon, elev; string stationId = _extractColumn(data_line, stationIdPtr); string col; - + if (format_version == AIRNOW_FORMAT_VERSION_HOURLY) { + + // skip lines for which no location is found if (!locations.lookupLatLonElev(stationId, lat, lon, elev)) { - mlog << Warning << method_name << "-> " - << "Skipping line number " << data_line.line_number() - << " StationId " << stationId << " Not found in locations file\n"; - // skip this line + mlog << Warning << "\n" << method_name + << "Skipping line number " << data_line.line_number() + << " since StationId " << stationId << " not found in locations file (" + << monitoringSiteFileName << ")! Set the " << airnow_stations_env + << " environment variable to define an updated version.\n\n"; return false; } } else { @@ -313,33 +314,44 @@ bool AirnowHandler::_parseObservationLineStandard(DataLine &data_line, string varName; string units; double value; + int avgPeriodSec; int aqiValue; int aqiCategory; - double height_m = elev; - + if (format_version == AIRNOW_FORMAT_VERSION_DAILYV2) { - varName = _extractColumn(data_line, varnamePtr); - units = _extractColumn(data_line, unitsPtr); - col = _extractColumn(data_line, valuePtr); - value = atof(col.c_str()); - col = _extractColumn(data_line, aqiPtr); - aqiValue = atoi(col.c_str()); - col = _extractColumn(data_line, aqiCategoryPtr); - aqiCategory = atoi(col.c_str()); - // for now only the single variable is written as an observation + + varName = _extractColumn(data_line, varnamePtr); + units = _extractColumn(data_line, unitsPtr); + col = _extractColumn(data_line, valuePtr); + value = atof(col.c_str()); + col = _extractColumn(data_line, avgperiodPtr); + avgPeriodSec = atoi(col.c_str()) * 3600; + col = _extractColumn(data_line, aqiPtr); + aqiValue = atoi(col.c_str()); + col = _extractColumn(data_line, aqiCategoryPtr); + aqiCategory = atoi(col.c_str()); + + // add the observation _addObservations(Observation(header_type, stationId, valid_time, - lat, lon, elev, na_str, 0, - bad_data_double, height_m, - value, varName)); + lat, lon, elev, na_str, _getVarIndex(varName, units), + avgPeriodSec, bad_data_double, + value, varName)); + } else if (format_version == AIRNOW_FORMAT_VERSION_HOURLY) { - varName = _extractColumn(data_line, varnamePtr); - units = _extractColumn(data_line, unitsPtr); - col = _extractColumn(data_line, valuePtr); - value = atof(col.c_str()); + + varName = _extractColumn(data_line, varnamePtr); + units = _extractColumn(data_line, unitsPtr); + col = _extractColumn(data_line, valuePtr); + value = atof(col.c_str()); + + // averaging period is 1-hour + avgPeriodSec = 3600; + + // add the observation _addObservations(Observation(header_type, stationId, valid_time, - lat, lon, elev, na_str, 0, - bad_data_double, height_m, - value, varName)); + lat, lon, elev, na_str, _getVarIndex(varName, units), + avgPeriodSec, bad_data_double, + value, varName)); } return true; } @@ -347,10 +359,10 @@ bool AirnowHandler::_parseObservationLineStandard(DataLine &data_line, //////////////////////////////////////////////////////////////////////// bool AirnowHandler::_readObservationsHourlyAqobs(LineDataFile &ascii_file, - int column_cnt, const string &delimiter, - const string &header_type) + int column_cnt, const string &delimiter, + const string &header_type) { - string method_name = "AirnowHandler::_readObservationsHourlyAqobs() "; + string method_name = "AirnowHandler::_readObservationsHourlyAqobs() -> "; // // Process the observation lines @@ -367,7 +379,7 @@ bool AirnowHandler::_readObservationsHourlyAqobs(LineDataFile &ascii_file, while (ascii_file >> data_line) { lineNumber++; if (!_parseObservationLineAqobs(data_line[0], column_cnt, header_type, - lineNumber, ascii_file.filename())) { + lineNumber, ascii_file.filename())) { ++bad_line_count; } } @@ -377,12 +389,12 @@ bool AirnowHandler::_readObservationsHourlyAqobs(LineDataFile &ascii_file, //////////////////////////////////////////////////////////////////////// bool AirnowHandler::_parseObservationLineAqobs(const string &data_line, - int column_cnt, - const string &header_type, - int lineNumber, - const string &filename) + int column_cnt, + const string &header_type, + int lineNumber, + const string &filename) { - string method_name = "AirnowHandler::_parseObservationLineAqobs() "; + string method_name = "AirnowHandler::_parseObservationLineAqobs() -> "; bool ok = true; vector tokens = parseHourlyAqobsLine(data_line, ok); @@ -391,28 +403,28 @@ bool AirnowHandler::_parseObservationLineAqobs(const string &data_line, } if ((int)tokens.size() != column_cnt) { - mlog << Error << "\nAirnowHandler" << method_name << "-> " - << "line number " << lineNumber - << " does not have the correct number of columns " << tokens.size() - << " (" << column_cnt << "). Skipping this line in \"" - << filename << "\".\n\n"; + mlog << Error << "\nAirnowHandler" << method_name + << "line number " << lineNumber + << " does not have the correct number of columns " << tokens.size() + << " (" << column_cnt << "). Skipping this line in \"" + << filename << "\".\n\n"; // for now just skip this line return false; } time_t valid_time = _getValidTime(tokens); if (valid_time == 0) { - mlog << Error << "\n" << method_name << "-> " - << "line number " << lineNumber - << " time could not be parsed, skipping this line in \"" - << filename << "\".\n\n"; + mlog << Error << "\n" << method_name + << "line number " << lineNumber + << " time could not be parsed, skipping this line in \"" + << filename << "\".\n\n"; return false; } - + // fill in expected things double lat, lon, elev; string stationId = tokens[stationIdPtr]; string col; - + lat = atof(tokens[latPtr].c_str()); lon = atof(tokens[lonPtr].c_str()); if (elevPtr >= 0) { @@ -420,23 +432,23 @@ bool AirnowHandler::_parseObservationLineAqobs(const string &data_line, } else { elev = 0.0; } - + _addHourlyAqobsObs(tokens, header_type, stationId, valid_time, lat, lon, elev, - ozoneMeasuredPtr, ozoneAqiPtr, ozonePtr, ozoneUnitPtr, - hdr_hourlyaqobs_ozone); + ozoneMeasuredPtr, ozoneAqiPtr, ozonePtr, ozoneUnitPtr, + hdr_hourlyaqobs_ozone); _addHourlyAqobsObs(tokens, header_type, stationId, valid_time, lat, lon, elev, - pm10MeasuredPtr, pm10AqiPtr, pm10Ptr, pm10UnitPtr, - hdr_hourlyaqobs_pm10); + pm10MeasuredPtr, pm10AqiPtr, pm10Ptr, pm10UnitPtr, + hdr_hourlyaqobs_pm10); _addHourlyAqobsObs(tokens, header_type, stationId, valid_time, lat, lon, elev, - pm25MeasuredPtr, pm25AqiPtr, pm25Ptr, pm25UnitPtr, - hdr_hourlyaqobs_pm25); + pm25MeasuredPtr, pm25AqiPtr, pm25Ptr, pm25UnitPtr, + hdr_hourlyaqobs_pm25); _addHourlyAqobsObs(tokens, header_type, stationId, valid_time, lat, lon, elev, - no2MeasuredPtr, no2AqiPtr, no2Ptr, no2UnitPtr, - hdr_hourlyaqobs_no2); + no2MeasuredPtr, no2AqiPtr, no2Ptr, no2UnitPtr, + hdr_hourlyaqobs_no2); _addHourlyAqobsObs(tokens, header_type, stationId, valid_time, lat, lon, elev, - coPtr, coUnitPtr, hdr_hourlyaqobs_co); + coPtr, coUnitPtr, hdr_hourlyaqobs_co); _addHourlyAqobsObs(tokens, header_type, stationId, valid_time, lat, lon, elev, - so2Ptr, so2UnitPtr, hdr_hourlyaqobs_so2); + so2Ptr, so2UnitPtr, hdr_hourlyaqobs_so2); return true; } @@ -456,17 +468,17 @@ bool AirnowHandler::_determineFileType(LineDataFile &ascii_file) // try delimeter "|" and check for DAILYV2 or HOURLY StringArray tokens = cstring.split("|"); - if (NUM_COLS_DAILYV2 == tokens.n_elements()) { + if (NUM_COLS_DAILYV2 == tokens.n()) { format_version = AIRNOW_FORMAT_VERSION_DAILYV2; return true; - } else if (NUM_COLS_HOURLY == tokens.n_elements()) { + } else if (NUM_COLS_HOURLY == tokens.n()) { format_version = AIRNOW_FORMAT_VERSION_HOURLY; return true; } // try with "," delimiter and look for HOURLYABQ StringArray tokens2 = cstring.split(","); - if (NUM_COLS_HOURLYAQOBS == tokens2.n_elements()) { + if (NUM_COLS_HOURLYAQOBS == tokens2.n()) { format_version = AIRNOW_FORMAT_VERSION_HOURLYAQOBS; return true; } @@ -479,29 +491,32 @@ bool AirnowHandler::_determineFileType(LineDataFile &ascii_file) //////////////////////////////////////////////////////////////////////// void AirnowHandler::_addHourlyAqobsObs(const vector &data_line, const string &header_type, - const string &stationId, const time_t &valid_time, - double lat, double lon, double elev, - int measuredPtr, int aqiPtr, int valuePtr, - int unitPtr, const string &varname) + const string &stationId, const time_t &valid_time, + double lat, double lon, double elev, + int measuredPtr, int aqiPtr, int valuePtr, + int unitPtr, const string &varname) { string col; int status; int aqi; double value; string units; - double height_m = elev; - + + // averging period is 1-hour + int avgPeriodSec = 3600; + status = atoi(data_line[measuredPtr].c_str()); if (status == 1) { aqi = atoi(data_line[aqiPtr].c_str()); if (doubleOrMissing(data_line[valuePtr], value)) { - units = data_line[unitPtr]; // ignored for now - // for now only the single variable is written as an observation + units = data_line[unitPtr]; + + // add the observation _addObservations(Observation(header_type, stationId, valid_time, - lat, lon, elev, na_str, 0, - bad_data_double, height_m, - value, varname)); - } + lat, lon, elev, na_str, _getVarIndex(varname, units), + avgPeriodSec, bad_data_double, + value, varname)); + } } } @@ -509,35 +524,39 @@ void AirnowHandler::_addHourlyAqobsObs(const vector &data_line, const st void AirnowHandler::_addHourlyAqobsObs(const vector &data_line, const string &header_type, - const string &stationId, const time_t &valid_time, - double lat, double lon, double elev, - int valuePtr, int unitPtr, const string &varname) + const string &stationId, const time_t &valid_time, + double lat, double lon, double elev, + int valuePtr, int unitPtr, const string &varname) { string col; double value; string units; - double height_m = elev; - + + // averging period is 1-hour + int avgPeriodSec = 3600; + if (doubleOrMissing(data_line[valuePtr], value)) { - units = data_line[unitPtr]; // ignored for now + units = data_line[unitPtr]; + + // add the observation _addObservations(Observation(header_type, stationId, valid_time, - lat, lon, elev, na_str, 0, - bad_data_double, height_m, - value, varname)); + lat, lon, elev, na_str, _getVarIndex(varname, units), + avgPeriodSec, bad_data_double, + value, varname)); } } //////////////////////////////////////////////////////////////////////// time_t AirnowHandler::_getValidTime(const DataLine &data_line) const - + { // // Pull out the date information // if (datePtr < 0) { mlog << Error << "\nAirnowHandler::_getValidTime -> " - << "Date column pointer is not set\n\n"; + << "Date column pointer is not set\n\n"; return 0; } string dateStr = _extractColumn(data_line, datePtr); @@ -559,7 +578,7 @@ time_t AirnowHandler::_getValidTime(const vector &data_line) const // if (datePtr < 0) { mlog << Error << "\nAirnowHandler::_getValidTime -> " - << "Date column pointer is not set\n\n"; + << "Date column pointer is not set\n\n"; return 0; } string dateStr = data_line[datePtr]; @@ -578,12 +597,12 @@ time_t AirnowHandler::_getValidTime(const string &dateStr, const string &timeStr { string mon, mday, year; string hour, min, sec; - + ConcatString date_string(dateStr); StringArray dateTokens = date_string.split("/"); - if (1 == dateTokens.n_elements()) { + if (1 == dateTokens.n()) { mlog << Error << "\nAirnowHandler::_getValidTime -> " - << "Not supported date: \"" << date_string << "\".\n\n"; + << "Not supported date: \"" << date_string << "\".\n\n"; return 0; } mon = dateTokens[0]; @@ -595,7 +614,7 @@ time_t AirnowHandler::_getValidTime(const string &dateStr, const string &timeStr // assumptions are true year = "20" + year; } - + hour = "00"; min = "00"; sec = "00"; @@ -605,23 +624,23 @@ time_t AirnowHandler::_getValidTime(const string &dateStr, const string &timeStr // ConcatString time_string(timeStr); StringArray timeTokens = time_string.split(":"); - if (1 == timeTokens.n_elements()) { + if (1 == timeTokens.n()) { // assume its hour hour = timeTokens[0]; - } else if (2 == timeTokens.n_elements()) { + } else if (2 == timeTokens.n()) { hour = timeTokens[0]; min = timeTokens[1]; - } else if (3 == timeTokens.n_elements()) { + } else if (3 == timeTokens.n()) { hour = timeTokens[0]; min = timeTokens[1]; sec = timeTokens[2]; } else { mlog << Error << "\nAirnowHandler::_getValidTime -> " - << "Not supported time: \"" << time_string << "\".\n\n"; + << "Not supported time: \"" << time_string << "\".\n\n"; return 0; } } - + // // Set up the time structure // @@ -654,6 +673,7 @@ void AirnowHandler::_setDailyv2HeaderInfo() header_names.add(hdr_value); valuePtr = 5; header_names.add(hdr_dailyv2_ave_period); + avgperiodPtr = 6; header_names.add(hdr_source); header_names.add(hdr_dailyv2_aqi_value); aqiPtr = 8; @@ -745,9 +765,8 @@ bool AirnowHandler::_readHeaderInfo(LineDataFile &ascii_file) if (!(ascii_file >> data_line)) { mlog << Error << "\nAirnowHandler::_readHeaderInfo() -> " - << "error reading header line from input ASCII file \"" - << ascii_file.filename() << "\"\n\n"; - + << "error reading header line from input ASCII file \"" + << ascii_file.filename() << "\"\n\n"; return false; } @@ -767,7 +786,7 @@ bool AirnowHandler::_readHeaderInfo(LineDataFile &ascii_file) // reference strings, while also setting pointers to // the columns that are of interest // - + header_names.clear(); bool status = true; for (int i=0; i " - << "AIRNOW file has unknown header item " << s << "\n\n"; + << "AIRNOW file has unknown header item " << s << "\n\n"; status = false; } } @@ -849,6 +868,7 @@ void AirnowHandler::_initializeColumnPointers() varnamePtr = -1; unitsPtr = -1; valuePtr = -1; + avgperiodPtr = -1; aqiPtr = -1; aqiCategoryPtr = -1; timePtr = -1; @@ -875,7 +895,6 @@ void AirnowHandler::_initializeColumnPointers() so2UnitPtr = -1; } - //////////////////////////////////////////////////////////////////////// string AirnowHandler::_extractColumn(const DataLine &data_line, int ptr) const @@ -884,7 +903,8 @@ string AirnowHandler::_extractColumn(const DataLine &data_line, int ptr) const if (doStripQuotes) { c = remove_quotes(c); } - // if you see a '\r' at the end remove that + + // if you see a '\r' at the end remove that std::size_t i1 = c.find_last_of("\r"); if (i1 == string::npos) { return c; @@ -894,6 +914,37 @@ string AirnowHandler::_extractColumn(const DataLine &data_line, int ptr) const //////////////////////////////////////////////////////////////////////// +int AirnowHandler::_getVarIndex(const string &var_name, const string &units) +{ + int var_index = bad_data_int; + + // variable name already exists + if (obs_names.has(var_name, var_index)) { + + // print warning if the units change + if (units != obs_units[var_index]) { + mlog << Warning << "\nAirnowHandler::_getVarIndex() -> " + << "the units for observation variable \"" << var_name + << "\" changed from \"" << obs_units[var_index] + << "\" to \"" << units << "\"!\n\n"; + } + } + // add new variable name and units + else { + obs_names.add(var_name); + obs_units.add(units); + var_index = obs_names.n() - 1; + } + + return var_index; +} + +//////////////////////////////////////////////////////////////////////// +// +// Begin utility functions +// +//////////////////////////////////////////////////////////////////////// + string remove_quotes(string &s) { std::size_t i0, i1; @@ -943,8 +994,8 @@ vector parseHourlyAqobsLine(const string &asciiLine, bool &ok) i1 = remainder.find_first_of("\""); if (i1 == string::npos) { mlog << Warning << "\nparseHourlyAqobsLine -> " - << "line doesn't have matching double quotes\n" - << fullLine << "\n\n"; + << "line doesn't have matching double quotes\n" + << fullLine << "\n\n"; // skip this line ok = false; break; @@ -952,7 +1003,7 @@ vector parseHourlyAqobsLine(const string &asciiLine, bool &ok) string token = remainder.substr(0, i1); tokens.push_back(token); remainder = remainder.substr(i1+1); - } + } } return tokens; } diff --git a/src/tools/other/ascii2nc/airnow_handler.h b/src/tools/other/ascii2nc/airnow_handler.h index 956f837f20..89afa7a4ce 100644 --- a/src/tools/other/ascii2nc/airnow_handler.h +++ b/src/tools/other/ascii2nc/airnow_handler.h @@ -74,7 +74,7 @@ class AirnowHandler : public FileHandler static const int NUM_COLS_HOURLYAQOBS; static const int NUM_COLS_DAILYV2; -protected: +protected: ///////////////////////// // Protected constants // @@ -104,6 +104,7 @@ class AirnowHandler : public FileHandler int varnamePtr; int unitsPtr; int valuePtr; + int avgperiodPtr; int aqiPtr; int aqiCategoryPtr; @@ -130,7 +131,7 @@ class AirnowHandler : public FileHandler int coUnitPtr; int so2Ptr; int so2UnitPtr; - + string monitoringSiteFileName; AirnowLocations locations; @@ -150,7 +151,7 @@ class AirnowHandler : public FileHandler // _stationLat, _stationLon and _stationAlt values. bool _readHeaderInfo(LineDataFile &ascii_file); - + bool _determineFileType(LineDataFile &ascii_file); void _addHourlyAqobsObs(const vector &data_line, const string &header_type, @@ -169,7 +170,7 @@ class AirnowHandler : public FileHandler time_t _getValidTime(const DataLine &data_line) const; time_t _getValidTime(const string &dateStr, const string &timeStr) const; - + // Read the observations from the given file and add them to the // _observations vector. @@ -189,7 +190,7 @@ class AirnowHandler : public FileHandler void _initializeColumnPointers(); string _extractColumn(const DataLine &data_line, int ptr) const; - vector _parseHourlyAqobsLine(const string &asciiLine); + int _getVarIndex(const string &, const string &); }; diff --git a/src/tools/other/ascii2nc/file_handler.cc b/src/tools/other/ascii2nc/file_handler.cc index 492398c89b..b0d6266bf9 100644 --- a/src/tools/other/ascii2nc/file_handler.cc +++ b/src/tools/other/ascii2nc/file_handler.cc @@ -188,7 +188,7 @@ bool FileHandler::summarizeObs(const TimeSummaryInfo &summary_info) _dataSummarized = true; _summaryInfo = summary_info; StringArray summary_vnames = summary_obs.getObsNames(); - for (int idx=0; idxclose(); delete _ncFile; _ncFile = (NcFile *) 0; } @@ -258,7 +257,6 @@ bool FileHandler::_openNetcdf(const string &nc_filename) nc_point_obs.get_dim_counts(&obs_cnt, &hdr_cnt); nc_point_obs.init_netcdf(obs_cnt, hdr_cnt, _programName); - // // Initialize the header and observation record counters // @@ -340,8 +338,8 @@ bool FileHandler::_addObservations(const Observation &obs) bool FileHandler::_writeObservations() { - StringArray descs, units; - nc_point_obs.write_to_netcdf(obs_names, units, descs); + StringArray descs; + nc_point_obs.write_to_netcdf(obs_names, obs_units, descs); return true; } @@ -380,3 +378,5 @@ void FileHandler::debug_print_observations(vector< Observation > my_observation, << " GC/VarIdx: " << last_obs.getGribCode() << " Value: " << last_obs.getValue() << " HeaderType: " << last_obs.getHeaderType() << "\n"; } + +//////////////////////////////////////////////////////////////////////// diff --git a/src/tools/other/ascii2nc/file_handler.h b/src/tools/other/ascii2nc/file_handler.h index 495705b803..129d29db65 100644 --- a/src/tools/other/ascii2nc/file_handler.h +++ b/src/tools/other/ascii2nc/file_handler.h @@ -109,6 +109,7 @@ class FileHandler vector< Observation > _observations; bool use_var_id; StringArray obs_names; + StringArray obs_units; bool do_monitor; int start_time, end_time;