From 050a833a3d3c9406d99ebc1778241568bf7d3365 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Tue, 18 Jul 2023 09:28:01 -0600 Subject: [PATCH] Feature #1626 TCDiag updates to wrapper and basic use case (#2248) Co-authored-by: Jonathan Vigh --- .github/parm/use_case_groups.json | 5 + docs/Contributors_Guide/add_use_case.rst | 6 + docs/Users_Guide/quicksearch.rst | 3 + docs/Users_Guide/wrappers.rst | 2 +- .../met_tool_wrapper/TCDiag/TCDiag.py | 52 +++++- .../use_cases/met_tool_wrapper/TCGen/TCGen.py | 1 + .../TCMPRPlotter/TCMPRPlotter.py | 1 + .../TCPairs/TCPairs_extra_tropical.py | 1 + .../TCPairs/TCPairs_tropical.py | 1 + .../use_cases/met_tool_wrapper/TCRMW/TCRMW.py | 1 + .../met_tool_wrapper/TCStat/TCStat.py | 1 + ...GFS_obsGFS_FeatureRelative_SeriesByInit.py | 1 + ...GFS_obsGFS_FeatureRelative_SeriesByLead.py | 1 + ...riesByLead_PyEmbed_Multiple_Diagnostics.py | 1 + ...otter_fcstGFS_obsGFS_UserScript_ExtraTC.py | 1 + .../GridStat_fcstHAFS_obsTDR_NetCDF.py | 1 + .../Plotter_fcstGFS_obsGFS_ExtraTC.py | 1 + .../Plotter_fcstGFS_obsGFS_RPlotting.py | 1 + .../TCGen_fcstGFS_obsBDECK_2021season.py | 1 + ...at_fcstADECK_obsBDECK_ATCF_BasicExample.py | 1 + .../TCRMW_fcstGFS_fcstOnly_gonzalo.py | 1 + ...CII2NC_PointStat_fcstHAFS_obsFRD_NetCDF.py | 1 + .../wrappers/tc_diag/test_tc_diag_wrapper.py | 23 +-- internal/tests/use_cases/all_use_cases.txt | 2 +- metplus/wrappers/tc_diag_wrapper.py | 153 ++++++++++++------ parm/met_config/TCDiagConfig_wrapped | 2 +- .../met_tool_wrapper/TCDiag/TCDiag.conf | 54 ++++--- 27 files changed, 234 insertions(+), 85 deletions(-) diff --git a/.github/parm/use_case_groups.json b/.github/parm/use_case_groups.json index faf63b90a..1f8ac22d0 100644 --- a/.github/parm/use_case_groups.json +++ b/.github/parm/use_case_groups.json @@ -9,6 +9,11 @@ "index_list": "30-58", "run": false }, + { + "category": "met_tool_wrapper", + "index_list": "62", + "run": false + }, { "category": "air_quality_and_comp", "index_list": "0", diff --git a/docs/Contributors_Guide/add_use_case.rst b/docs/Contributors_Guide/add_use_case.rst index 8710145e0..b4ca716bb 100644 --- a/docs/Contributors_Guide/add_use_case.rst +++ b/docs/Contributors_Guide/add_use_case.rst @@ -327,6 +327,12 @@ file.grib2, run the following command:: wgrib2 file.grib2 | grep TMP | wgrib2 -i file.grib2 -grib_out subset.grib2 +The egrep command can be used for more complex subsetting of grib2 data. +Example: To create a file called subset.grib2 from file.grib2 that contains +PRMSL data and TMP data on 1000, 900, 800, 700, 500, and 100 mb levels:: + + wgrib2 file.grib2 -s | egrep '(:TMP:1000 mb:|:TMP:900 mb:|:TMP:800 mb:|:TMP:700 mb:|:TMP:500 mb:|:TMP:100 mb:|:PRMSL)' | wgrib2 -i file.grib2 -grib subset.grib2 + If the input data is in NetCDF format, the `ncks `_ tool can be used to subset the file(s). diff --git a/docs/Users_Guide/quicksearch.rst b/docs/Users_Guide/quicksearch.rst index b2a3b68fe..48303cdd5 100644 --- a/docs/Users_Guide/quicksearch.rst +++ b/docs/Users_Guide/quicksearch.rst @@ -10,6 +10,7 @@ METplus Quick Search for Use Cases Use Cases by MET Tool: +====================== .. only:: html @@ -163,6 +164,7 @@ Use Cases by METplus Feature: | `Runtime Frequency <../search.html?q=RuntimeFreqUseCase&check_keywords=yes&area=default>`_ | `Series by Initialization <../search.html?q=SeriesByInitUseCase&check_keywords=yes&area=default>`_ | `Series by Forecast Lead <../search.html?q=SeriesByLeadUseCase&check_keywords=yes&area=default>`_ + | `Tropical Cyclone <../search.html?q=TropicalCycloneUseCase&check_keywords=yes&area=default>`_ | `Validation of Models or Analyses <../search.html?q=ValidationUseCase&check_keywords=yes&area=default>`_ | `User Defined Script <../search.html?q=UserScriptUseCase&check_keywords=yes&area=default>`_ @@ -191,6 +193,7 @@ Use Cases by METplus Feature: | **Runtime Frequency**: *RuntimeFreqUseCase* | **Series by Initialization**: *SeriesByInitUseCase* | **Series by Forecast Lead**: *SeriesByLeadUseCase* + | **Tropical Cyclone**: *TropicalCycloneUseCase* | **Validation of Models or Analyses**: *ValidationUseCase* | **User Defined Script**: *UserScriptUseCase* diff --git a/docs/Users_Guide/wrappers.rst b/docs/Users_Guide/wrappers.rst index d5e386264..b90bfcf5e 100644 --- a/docs/Users_Guide/wrappers.rst +++ b/docs/Users_Guide/wrappers.rst @@ -7692,7 +7692,7 @@ TCDiag Description ----------- -Used to configure the MET tool TC-Diag. +The TC-Diag wrapper encapsulates the behavior of the MET `tc_diag `_ tool. It provides the infrastructure to compute diagnostics from model fields and tracks. It can be configured to run over a single intialization time, all of the initialization times for a given storm, or over many storms. Configuration also allows a user to select which domain(s) of the input model data to use in the diagnostics calculations, set which levels and variables will be used as well as details about the azimuth-range grid used for the calculations, and to control which output files are generated. Future functionality of the tc_diag tool, such as vortex removal, will also be configurable from this wrapper. METplus Configuration --------------------- diff --git a/docs/use_cases/met_tool_wrapper/TCDiag/TCDiag.py b/docs/use_cases/met_tool_wrapper/TCDiag/TCDiag.py index d2a8f0b75..d525cbc8f 100644 --- a/docs/use_cases/met_tool_wrapper/TCDiag/TCDiag.py +++ b/docs/use_cases/met_tool_wrapper/TCDiag/TCDiag.py @@ -5,18 +5,48 @@ met_tool_wrapper/TCDiag/TCDiag.conf """ +############################################################################## +# Overview +# -------------------- +# +# This use case illustrates the use of tc_diag tool, which is currently +# considered a beta-level release that lacks full functionality. +# The use case illustrates running the +# tc_diag tool for a tropical cyclone forecast case and generating +# intermediate NetCDF output files of the input model's data transformed +# onto an azimuth-range grid. When the full functionality of the +# tc_diag tool is released in MET v12.0.0, this use case will be also +# output environmental diagnostics computed from callable Python scripts. +# +# The diagnostics are computed on a range-azimuth grid that follows the +# projected storm track. For inputs, it uses 0.25 deg gridded GRIB files from the +# a retrospective reforecast of the Global Forecast System (GFS). For the track, it uses the +# GFS's predicted track to ensure that the model's simulated storm doesn't +# contaminate the diagnostics result as a result of the model's simulated +# storm being mistaken for environmental factors. (Note: +# a future version of the tc_diag tool will include removal of the model's vortex, +# allowing diagnostics to be computed along any arbitrarily defined track.) +# +# Novel aspects of this use case: +# * This is the first example use case to run the tc_diag tool. +# * Example of running for a single tropical cyclone forecast case from +# Tropical Storm Bret (2023) using GFS data. + ############################################################################## # Scientific Objective # -------------------- # -# TODO: Add content here +# Generate intermediate data files, in which the input model's data have been +# transformed to a range-azimuth grid, in preparation for further diagnostic +# calculations using Python-based routines. ############################################################################## # Datasets # -------- # -# **Forecast:** GFS FV3 -# **Track:** A Deck +# **Forecast:** GFS grib files +# +# **Track:** a-deck file (Automated Tropical Cyclone Forecast System format) # # **Location:** All of the input data required for this use case can be found # in the met_test sample data tarball. Click here to the METplus releases page @@ -26,6 +56,11 @@ # This tarball should be unpacked into the directory that you will set the # value of INPUT_BASE. See `Running METplus`_ section for more information. # +# **Data source:** Users may obtain real-time data from the deterministic GFS runs from +# NOAA's NOMADS server: +# https://nomads.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/gfs.YYYYMMDD/ZZ/atmos/ +# where YYYYMMDD is the date (4-digit year, 2-digit month, 2-digit day), +# ZZ is the initialization hour of the desired model cycle (00, 06, 12, 18). ############################################################################## # METplus Components @@ -42,8 +77,8 @@ # TCDiag is the only tool called in this example. It processes the following # run times: # -# **Init:** 2016-09-29- 00Z -# **Forecast lead:** 141, 143, and 147 hour +# **Init:** 2023-06-20 0000Z +# **Forecast lead:** 0, 6, and 12 hours # ############################################################################## @@ -67,7 +102,7 @@ # If there is a setting in the MET configuration file that is currently not supported by METplus you'd like to control, please refer to: # :ref:`Overriding Unsupported MET config file settings` # -# .. note:: See the :ref:`TCDiag MET Configuration` section of the User's Guide for more information on the environment variables used in the file below: +# .. note:: See the :ref:`TCDiag MET Configuration` section of the User's Guide for more information on the environment variables used in the file below: # # .. highlight:: bash # .. literalinclude:: ../../../../parm/met_config/TCDiagConfig_wrapped @@ -96,7 +131,8 @@ # Output for this use case will be found in met_tool_wrapper/TCDiag (relative to **OUTPUT_BASE**) # and will contain the following files: # -# * tc_diag_aal142016.nc +# * tc_diag_AL032023_GFSO_2023062012_cyl_grid_nest.nc +# * tc_diag_AL032023_GFSO_2023062012_cyl_grid_parent.nc # ############################################################################## @@ -105,8 +141,10 @@ # # .. note:: # +# * DiagnosticsUseCase # * TCDiagToolUseCase # * GRIB2FileUseCase +# * TropicalCycloneUseCase # # Navigate to the :ref:`quick-search` page to discover other similar use cases. # diff --git a/docs/use_cases/met_tool_wrapper/TCGen/TCGen.py b/docs/use_cases/met_tool_wrapper/TCGen/TCGen.py index 1580255aa..e3feba3fd 100644 --- a/docs/use_cases/met_tool_wrapper/TCGen/TCGen.py +++ b/docs/use_cases/met_tool_wrapper/TCGen/TCGen.py @@ -124,6 +124,7 @@ # # * TCGenToolUseCase # * DTCOrgUseCase +# * TropicalCycloneUseCase # # Navigate to the :ref:`quick-search` page to discover other similar use cases. # diff --git a/docs/use_cases/met_tool_wrapper/TCMPRPlotter/TCMPRPlotter.py b/docs/use_cases/met_tool_wrapper/TCMPRPlotter/TCMPRPlotter.py index 34f456e36..84a9ced20 100644 --- a/docs/use_cases/met_tool_wrapper/TCMPRPlotter/TCMPRPlotter.py +++ b/docs/use_cases/met_tool_wrapper/TCMPRPlotter/TCMPRPlotter.py @@ -110,6 +110,7 @@ # .. note:: # # * TCMPRPlotterUseCase +# * TropicalCycloneUseCase # # Navigate to the :ref:`quick-search` page to discover other similar use cases. # diff --git a/docs/use_cases/met_tool_wrapper/TCPairs/TCPairs_extra_tropical.py b/docs/use_cases/met_tool_wrapper/TCPairs/TCPairs_extra_tropical.py index c68543981..0913065d7 100644 --- a/docs/use_cases/met_tool_wrapper/TCPairs/TCPairs_extra_tropical.py +++ b/docs/use_cases/met_tool_wrapper/TCPairs/TCPairs_extra_tropical.py @@ -124,6 +124,7 @@ # # * TCPairsToolUseCase # * SBUOrgUseCase +# * TropicalCycloneUseCase # # Navigate to the :ref:`quick-search` page to discover other similar use cases. # diff --git a/docs/use_cases/met_tool_wrapper/TCPairs/TCPairs_tropical.py b/docs/use_cases/met_tool_wrapper/TCPairs/TCPairs_tropical.py index 9a9e174b6..e0cb0ab72 100644 --- a/docs/use_cases/met_tool_wrapper/TCPairs/TCPairs_tropical.py +++ b/docs/use_cases/met_tool_wrapper/TCPairs/TCPairs_tropical.py @@ -128,6 +128,7 @@ # # * TCPairsToolUseCase # * DTCOrgUseCase +# * TropicalCycloneUseCase # # Navigate to the :ref:`quick-search` page to discover other similar use cases. # diff --git a/docs/use_cases/met_tool_wrapper/TCRMW/TCRMW.py b/docs/use_cases/met_tool_wrapper/TCRMW/TCRMW.py index 8e7e468a2..d3d3bdf40 100644 --- a/docs/use_cases/met_tool_wrapper/TCRMW/TCRMW.py +++ b/docs/use_cases/met_tool_wrapper/TCRMW/TCRMW.py @@ -122,6 +122,7 @@ # # * TCRMWToolUseCase # * GRIB2FileUseCase +# * TropicalCycloneUseCase # # Navigate to the :ref:`quick-search` page to discover other similar use cases. # diff --git a/docs/use_cases/met_tool_wrapper/TCStat/TCStat.py b/docs/use_cases/met_tool_wrapper/TCStat/TCStat.py index 4c4d08d03..7386d9ee6 100644 --- a/docs/use_cases/met_tool_wrapper/TCStat/TCStat.py +++ b/docs/use_cases/met_tool_wrapper/TCStat/TCStat.py @@ -120,6 +120,7 @@ # .. note:: # # * TCStatToolUseCase +# * TropicalCycloneUseCase # # Navigate to the :ref:`quick-search` page to discover other similar use cases. # diff --git a/docs/use_cases/model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByInit.py b/docs/use_cases/model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByInit.py index 4d6266a92..0dce153a7 100644 --- a/docs/use_cases/model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByInit.py +++ b/docs/use_cases/model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByInit.py @@ -252,6 +252,7 @@ # * SBUOrgUseCase # * DiagnosticsUseCase # * RuntimeFreqUseCase +# * TropicalCycloneUseCase # # Navigate to the :ref:`quick-search` page to discover other similar use cases. # diff --git a/docs/use_cases/model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead.py b/docs/use_cases/model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead.py index e8bd1bbaf..eb93987e4 100644 --- a/docs/use_cases/model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead.py +++ b/docs/use_cases/model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead.py @@ -244,6 +244,7 @@ # * SBUOrgUseCase # * DiagnosticsUseCase # * RuntimeFreqUseCase +# * TropicalCycloneUseCase # # Navigate to the :ref:`quick-search` page to discover other similar use cases. # diff --git a/docs/use_cases/model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead_PyEmbed_Multiple_Diagnostics.py b/docs/use_cases/model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead_PyEmbed_Multiple_Diagnostics.py index f521ae983..52808805f 100644 --- a/docs/use_cases/model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead_PyEmbed_Multiple_Diagnostics.py +++ b/docs/use_cases/model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead_PyEmbed_Multiple_Diagnostics.py @@ -334,6 +334,7 @@ # * SBUOrgUseCase # * DiagnosticsUseCase # * RuntimeFreqUseCase +# * TropicalCycloneUseCase # # Navigate to the :ref:`quick-search` page to discover other similar use cases. # diff --git a/docs/use_cases/model_applications/tc_and_extra_tc/CyclonePlotter_fcstGFS_obsGFS_UserScript_ExtraTC.py b/docs/use_cases/model_applications/tc_and_extra_tc/CyclonePlotter_fcstGFS_obsGFS_UserScript_ExtraTC.py index ea52376bb..22edf8e33 100644 --- a/docs/use_cases/model_applications/tc_and_extra_tc/CyclonePlotter_fcstGFS_obsGFS_UserScript_ExtraTC.py +++ b/docs/use_cases/model_applications/tc_and_extra_tc/CyclonePlotter_fcstGFS_obsGFS_UserScript_ExtraTC.py @@ -162,6 +162,7 @@ # * TCPairsToolUseCase # * SBUOrgUseCase # * CyclonePlotterUseCase +# * TropicalCycloneUseCase # # Navigate to the :ref:`quick-search` page to discover other similar use cases. # diff --git a/docs/use_cases/model_applications/tc_and_extra_tc/GridStat_fcstHAFS_obsTDR_NetCDF.py b/docs/use_cases/model_applications/tc_and_extra_tc/GridStat_fcstHAFS_obsTDR_NetCDF.py index ad6205e92..3a19f8abd 100644 --- a/docs/use_cases/model_applications/tc_and_extra_tc/GridStat_fcstHAFS_obsTDR_NetCDF.py +++ b/docs/use_cases/model_applications/tc_and_extra_tc/GridStat_fcstHAFS_obsTDR_NetCDF.py @@ -152,6 +152,7 @@ # # * TCandExtraTCAppUseCase # * GridStatToolUseCase +# * TropicalCycloneUseCase # # Navigate to the :ref:`quick-search` page to discover other similar use cases. # diff --git a/docs/use_cases/model_applications/tc_and_extra_tc/Plotter_fcstGFS_obsGFS_ExtraTC.py b/docs/use_cases/model_applications/tc_and_extra_tc/Plotter_fcstGFS_obsGFS_ExtraTC.py index 8f10fbb48..a0fbb57a1 100644 --- a/docs/use_cases/model_applications/tc_and_extra_tc/Plotter_fcstGFS_obsGFS_ExtraTC.py +++ b/docs/use_cases/model_applications/tc_and_extra_tc/Plotter_fcstGFS_obsGFS_ExtraTC.py @@ -139,6 +139,7 @@ # * NOAAEMCOrgUseCase # * SBUOrgUseCase # * DTCOrgUseCase +# * TropicalCycloneUseCase # # Navigate to the :ref:`quick-search` page to discover other similar use cases. # diff --git a/docs/use_cases/model_applications/tc_and_extra_tc/Plotter_fcstGFS_obsGFS_RPlotting.py b/docs/use_cases/model_applications/tc_and_extra_tc/Plotter_fcstGFS_obsGFS_RPlotting.py index d2aaaa4e2..f7ddd8c09 100644 --- a/docs/use_cases/model_applications/tc_and_extra_tc/Plotter_fcstGFS_obsGFS_RPlotting.py +++ b/docs/use_cases/model_applications/tc_and_extra_tc/Plotter_fcstGFS_obsGFS_RPlotting.py @@ -157,6 +157,7 @@ # * MediumRangeAppUseCase # * SBUOrgUseCase # * DTCOrgUseCase +# * TropicalCycloneUseCase # # Navigate to the :ref:`quick-search` page to discover other similar use cases. # diff --git a/docs/use_cases/model_applications/tc_and_extra_tc/TCGen_fcstGFS_obsBDECK_2021season.py b/docs/use_cases/model_applications/tc_and_extra_tc/TCGen_fcstGFS_obsBDECK_2021season.py index 307fb4a6d..d66e08cb2 100644 --- a/docs/use_cases/model_applications/tc_and_extra_tc/TCGen_fcstGFS_obsBDECK_2021season.py +++ b/docs/use_cases/model_applications/tc_and_extra_tc/TCGen_fcstGFS_obsBDECK_2021season.py @@ -135,6 +135,7 @@ # .. note:: # # * TCGenToolUseCase +# * TropicalCycloneUseCase # # Navigate to the :ref:`quick-search` page to discover other similar use cases. # diff --git a/docs/use_cases/model_applications/tc_and_extra_tc/TCPairs_TCStat_fcstADECK_obsBDECK_ATCF_BasicExample.py b/docs/use_cases/model_applications/tc_and_extra_tc/TCPairs_TCStat_fcstADECK_obsBDECK_ATCF_BasicExample.py index 99df0976d..f60d40189 100644 --- a/docs/use_cases/model_applications/tc_and_extra_tc/TCPairs_TCStat_fcstADECK_obsBDECK_ATCF_BasicExample.py +++ b/docs/use_cases/model_applications/tc_and_extra_tc/TCPairs_TCStat_fcstADECK_obsBDECK_ATCF_BasicExample.py @@ -149,6 +149,7 @@ # # * TCPairsToolUseCase # * TCStatToolUseCase +# * TropicalCycloneUseCase # # Navigate to the :ref:`quick-search` page to discover other similar use cases. # diff --git a/docs/use_cases/model_applications/tc_and_extra_tc/TCRMW_fcstGFS_fcstOnly_gonzalo.py b/docs/use_cases/model_applications/tc_and_extra_tc/TCRMW_fcstGFS_fcstOnly_gonzalo.py index 8bc43a8c7..ccad94908 100644 --- a/docs/use_cases/model_applications/tc_and_extra_tc/TCRMW_fcstGFS_fcstOnly_gonzalo.py +++ b/docs/use_cases/model_applications/tc_and_extra_tc/TCRMW_fcstGFS_fcstOnly_gonzalo.py @@ -123,6 +123,7 @@ # # * TCRMWToolUseCase # * GRIB2FileUseCase +# * TropicalCycloneUseCase # # Navigate to the :ref:`quick-search` page to discover other similar use cases. # diff --git a/docs/use_cases/model_applications/tc_and_extra_tc/UserScript_ASCII2NC_PointStat_fcstHAFS_obsFRD_NetCDF.py b/docs/use_cases/model_applications/tc_and_extra_tc/UserScript_ASCII2NC_PointStat_fcstHAFS_obsFRD_NetCDF.py index 313bd0c8d..8b24b5a67 100644 --- a/docs/use_cases/model_applications/tc_and_extra_tc/UserScript_ASCII2NC_PointStat_fcstHAFS_obsFRD_NetCDF.py +++ b/docs/use_cases/model_applications/tc_and_extra_tc/UserScript_ASCII2NC_PointStat_fcstHAFS_obsFRD_NetCDF.py @@ -164,6 +164,7 @@ # * UserScriptUseCase # * PointStatToolUseCase # * ASCII2NCToolUseCase +# * TropicalCycloneUseCase # # Navigate to the :ref:`quick-search` page to discover other similar use cases. # diff --git a/internal/tests/pytests/wrappers/tc_diag/test_tc_diag_wrapper.py b/internal/tests/pytests/wrappers/tc_diag/test_tc_diag_wrapper.py index faddf14e2..d62db8b24 100644 --- a/internal/tests/pytests/wrappers/tc_diag/test_tc_diag_wrapper.py +++ b/internal/tests/pytests/wrappers/tc_diag/test_tc_diag_wrapper.py @@ -9,7 +9,7 @@ deck_template = 'aal14{date?fmt=%Y}_short.dat' input_template = 'gfs.subset.t00z.pgrb2.0p25.f*' -output_template = 'tc_diag_aal14{date?fmt=%Y}.nc' +output_template = '{date?fmt=%Y}' time_fmt = '%Y%m%d%H' run_times = ['2016092900'] @@ -24,6 +24,9 @@ '{ name="TMP"; level="P100"; }];' ) +input_domain = 'parent' +input_tech_id_list = 'GFSO' + def get_data_dir(config): return os.path.join(config.getdir('METPLUS_BASE'), @@ -46,9 +49,11 @@ def set_minimum_config_settings(config): config.set('config', 'TC_DIAG_CONFIG_FILE', '{PARM_BASE}/met_config/TCDiagConfig_wrapped') config.set('config', 'TC_DIAG_DECK_TEMPLATE', deck_template) - config.set('config', 'TC_DIAG_INPUT_TEMPLATE', input_template) + config.set('config', 'TC_DIAG_INPUT1_TEMPLATE', input_template) + config.set('config', 'TC_DIAG_INPUT1_DOMAIN', input_domain) + config.set('config', 'TC_DIAG_INPUT1_TECH_ID_LIST', input_tech_id_list) config.set('config', 'TC_DIAG_OUTPUT_DIR', - '{OUTPUT_BASE}/TCDiag/output') + '{OUTPUT_BASE}/tc_diag') config.set('config', 'TC_DIAG_OUTPUT_TEMPLATE', output_template) config.set('config', 'BOTH_VAR1_NAME', 'PRMSL') @@ -231,13 +236,11 @@ def test_tc_diag_run(metplus_config, config_overrides, config_file = wrapper.c_dict.get('CONFIG_FILE') out_dir = wrapper.c_dict.get('OUTPUT_DIR') - expected_cmds = [(f"{app_path} " - f"-deck {deck_dir}/aal142016_short.dat " - f"-data {file_list_file} " - f"-config {config_file} " - f"-out {out_dir}/tc_diag_aal142016.nc " - f"{verbosity}"), - ] + expected_cmds = [ + (f"{app_path} -deck {deck_dir}/aal142016_short.dat " + f"-data {input_domain} {input_tech_id_list} {file_list_file} " + f"-config {config_file} -outdir {out_dir}/2016/ {verbosity}"), + ] all_cmds = wrapper.run_all_times() print(f"ALL COMMANDS: {all_cmds}") diff --git a/internal/tests/use_cases/all_use_cases.txt b/internal/tests/use_cases/all_use_cases.txt index 3ddc50030..40f8a0b44 100644 --- a/internal/tests/use_cases/all_use_cases.txt +++ b/internal/tests/use_cases/all_use_cases.txt @@ -61,7 +61,7 @@ Category: met_tool_wrapper 59::IODA2NC::met_tool_wrapper/IODA2NC/IODA2NC.conf 60::PointStat_python_embedding_obs:: met_tool_wrapper/PointStat/PointStat_python_embedding_obs.conf 61::PlotPointObs:: met_tool_wrapper/PlotPointObs/PlotPointObs.conf -#62::TCDiag:: met_tool_wrapper/TCDiag/TCDiag.conf +62::TCDiag:: met_tool_wrapper/TCDiag/TCDiag.conf Category: air_quality_and_comp 0::EnsembleStat_fcstICAP_obsMODIS_aod::model_applications/air_quality_and_comp/EnsembleStat_fcstICAP_obsMODIS_aod.conf diff --git a/metplus/wrappers/tc_diag_wrapper.py b/metplus/wrappers/tc_diag_wrapper.py index e2c49a013..1b8ebd8a7 100755 --- a/metplus/wrappers/tc_diag_wrapper.py +++ b/metplus/wrappers/tc_diag_wrapper.py @@ -15,7 +15,8 @@ from ..util import time_util from . import RuntimeFreqWrapper from ..util import do_string_sub, skip_time, get_lead_sequence -from ..util import parse_var_list, sub_var_list +from ..util import parse_var_list, sub_var_list, getlist +from ..util import find_indices_in_config_section from ..util.met_config import add_met_config_dict_list '''!@namespace TCDiagWrapper @@ -85,29 +86,28 @@ def create_c_dict(self): self.log_error('Only RUN_ONCE_PER_INIT_OR_VALID is supported for ' 'TC_DIAG_RUNTIME_FREQ.') - # get the MET config file path or use default - c_dict['CONFIG_FILE'] = self.get_config_file('TCDiagConfig_wrapped') + # get command line arguments domain and tech id list for -data + self._read_data_inputs(c_dict) - c_dict['INPUT_DIR'] = self.config.getdir('TC_DIAG_INPUT_DIR', '') - c_dict['INPUT_TEMPLATE'] = self.config.getraw('config', - 'TC_DIAG_INPUT_TEMPLATE') - c_dict['INPUT_FILE_LIST'] = self.config.getraw( - 'config', 'TC_DIAG_INPUT_FILE_LIST' + # get -deck argument dir/template + c_dict['DECK_INPUT_DIR'] = self.config.getdir('TC_DIAG_DECK_INPUT_DIR', + '') + c_dict['DECK_INPUT_TEMPLATE'] = ( + self.config.getraw('config', + 'TC_DIAG_DECK_TEMPLATE') ) + # get output dir/template c_dict['OUTPUT_DIR'] = self.config.getdir('TC_DIAG_OUTPUT_DIR', '') c_dict['OUTPUT_TEMPLATE'] = ( self.config.getraw('config', 'TC_DIAG_OUTPUT_TEMPLATE') ) - c_dict['DECK_INPUT_DIR'] = self.config.getdir('TC_DIAG_DECK_INPUT_DIR', - '') - c_dict['DECK_INPUT_TEMPLATE'] = ( - self.config.getraw('config', - 'TC_DIAG_DECK_TEMPLATE') - ) + # get the MET config file path or use default + c_dict['CONFIG_FILE'] = self.get_config_file('TCDiagConfig_wrapped') + # get variables to set in wrapped MET config file self.add_met_config(name='model', data_type='list', metplus_configs=['TC_DIAG_MODEL', 'MODEL']) @@ -232,23 +232,78 @@ def create_c_dict(self): return c_dict + def _read_data_inputs(self, c_dict): + """! Parse the -data arguments from the METplusConfig object. + Sets c_dict DATA_INPUTS key with a list of dictionaries. + Each input should include domain, tech_id_list, and dir/template. + Logs error if any required variables are not set. + + @param c_dict dictionary to save values into + """ + # get template indices + indices = list( + find_indices_in_config_section(r'TC_DIAG_INPUT(\d+)_TEMPLATE$', + self.config, + index_index=1).keys() + ) + + # if no template indices were found, look for file list indices + if not indices: + indices = list( + find_indices_in_config_section(r'TC_DIAG_INPUT(\d+)_FILE_LIST$', + self.config, + index_index=1).keys() + ) + # error if no file list or template indices were found + if not indices: + self.log_error( + 'Must set TC_DIAG_INPUT_TEMPLATE/DOMAIN/TECH_ID_LIST' + ) + return + + c_dict['DATA_INPUTS'] = [] + for index in indices: + prefix = f'TC_DIAG_INPUT{index}_' + directory = self.config.getdir(f'{prefix}DIR') + template = self.config.getraw('config', f'{prefix}TEMPLATE') + + # get file list if template is not set + if template: + file_list = None + else: + file_list = self.config.getraw('config', f'{prefix}FILE_LIST') + + domain = self.config.getraw('config', f'{prefix}DOMAIN') + if not domain: + self.log_error(f'Must set {prefix}DOMAIN') + + tech_id_list = getlist( + self.config.getraw('config', f'{prefix}TECH_ID_LIST') + ) + if not tech_id_list: + self.log_error(f'Must set {prefix}TECH_ID_LIST') + + data_dict = { + 'template': template, + 'directory': directory, + 'file_list': file_list, + 'domain': domain, + 'tech_id_list': tech_id_list, + } + c_dict['DATA_INPUTS'].append(data_dict) + def get_command(self): cmd = self.app_path # add deck cmd += ' -deck ' + self.c_dict['DECK_FILE'] - # add input files - cmd += ' -data' - for infile in self.infiles: - cmd += ' ' + infile - # add arguments cmd += ' ' + ' '.join(self.args) # add output path out_path = self.get_output_path() - cmd += ' -out ' + out_path + cmd += ' -outdir ' + out_path # add verbosity cmd += ' -v ' + self.c_dict['VERBOSITY'] @@ -259,13 +314,15 @@ def run_at_time_once(self, time_info): Args: @param time_info dictionary containing timing information """ + self.clear() time_info = time_util.ti_calculate(time_info) + # get input files - if self.find_input_files(time_info) is None: + if not self.find_input_files(time_info): return # get output path - if not self.find_and_check_output_file(time_info): + if not self.find_and_check_output_file(time_info, is_directory=True): return # get field information to set in MET config @@ -281,12 +338,6 @@ def run_at_time_once(self, time_info): # set environment variables if using config file self.set_environment_variables(time_info) - # build command and run - cmd = self.get_command() - if cmd is None: - self.log_error("Could not generate command") - return - self.build() def set_data_field(self, time_info): @@ -327,47 +378,57 @@ def find_input_files(self, time_info): # get deck file deck_file = self.find_data(time_info, data_type='DECK') if not deck_file: - return None - + return False self.c_dict['DECK_FILE'] = deck_file + # get files and values for -data arguments lead_seq = get_lead_sequence(self.config, time_info) + for data_dict in self.c_dict['DATA_INPUTS']: + if not self._find_data_inputs(data_dict, lead_seq, time_info, + deck_file): + return False + return True - # get input files - if self.c_dict['INPUT_FILE_LIST']: - self.logger.debug("Explicit file list file: " - f"{self.c_dict['INPUT_FILE_LIST']}") - list_file = do_string_sub(self.c_dict['INPUT_FILE_LIST'], - **time_info) + def _find_data_inputs(self, data_dict, lead_seq, time_info, deck_file): + # check if file list file is set and use that instead of template/dir + input_file_list = data_dict['file_list'] + if input_file_list: + self.logger.debug(f"Explicit file list file: {input_file_list}") + list_file = do_string_sub(input_file_list, **time_info) if not os.path.exists(list_file): self.log_error(f'Could not find file list: {list_file}') - return None + return False else: - all_input_files = [] + # set c_dict variables that are used in find_data function + self.c_dict['INPUT_DIR'] = data_dict['directory'] + self.c_dict['INPUT_TEMPLATE'] = data_dict['template'] + all_input_files = [] for lead in lead_seq: - self.clear() - time_info['lead'] = lead - - time_info = time_util.ti_calculate(time_info) + time_info_lead = time_info.copy() + time_info_lead['lead'] = lead + time_info_lead = time_util.ti_calculate(time_info_lead) # get a list of the input data files, # write to an ascii file if there are more than one - input_files = self.find_data(time_info, return_list=True) + input_files = self.find_data(time_info_lead, return_list=True) if not input_files: continue all_input_files.extend(input_files) if not all_input_files: - return None + return False # create an ascii file with a list of the input files list_file = f"{os.path.basename(deck_file)}_data_files.txt" list_file = self.write_list_file(list_file, all_input_files) - self.infiles.append(list_file) - return self.infiles + # build argument with file list file, domain, and tech id list + domain = data_dict['domain'] + tech_ids = ','.join(data_dict['tech_id_list']) + self.args.append(f'-data {domain} {tech_ids} {list_file}') + return True def set_lead_list(self, time_info): self.env_var_dict['METPLUS_LEAD_LIST'] = '' diff --git a/parm/met_config/TCDiagConfig_wrapped b/parm/met_config/TCDiagConfig_wrapped index 43d3708c0..d20be1a55 100644 --- a/parm/met_config/TCDiagConfig_wrapped +++ b/parm/met_config/TCDiagConfig_wrapped @@ -182,7 +182,7 @@ tmp_dir = "${MET_TMP_DIR}"; //output_prefix = ${METPLUS_OUTPUT_PREFIX} -//version = "V11.0.0"; +//version = "V11.1.0"; //////////////////////////////////////////////////////////////////////////////// diff --git a/parm/use_cases/met_tool_wrapper/TCDiag/TCDiag.conf b/parm/use_cases/met_tool_wrapper/TCDiag/TCDiag.conf index 7f2bcc3b0..b489eec5b 100644 --- a/parm/use_cases/met_tool_wrapper/TCDiag/TCDiag.conf +++ b/parm/use_cases/met_tool_wrapper/TCDiag/TCDiag.conf @@ -27,10 +27,10 @@ PROCESS_LIST = TCDiag LOOP_BY = INIT INIT_TIME_FMT = %Y%m%d%H -INIT_BEG = 2016092900 -INIT_END = 2016092900 +INIT_BEG = 2023062012 +INIT_END = 2023062012 INIT_INCREMENT = 21600 - +LEAD_SEQ = 0, 6, 12 ### # File I/O @@ -38,13 +38,21 @@ INIT_INCREMENT = 21600 ### TC_DIAG_DECK_INPUT_DIR = {INPUT_BASE}/met_test/new/tc_data/adeck -TC_DIAG_DECK_TEMPLATE = aal14{date?fmt=%Y}_short.dat +TC_DIAG_DECK_TEMPLATE = subset.aal03{date?fmt=%Y}.dat + +TC_DIAG_INPUT1_DIR = {INPUT_BASE}/met_test/new/model_data/grib2/gfs +TC_DIAG_INPUT1_TEMPLATE = subset.gfs.t12z.pgrb2.0p50.f* +TC_DIAG_INPUT1_DOMAIN = parent +TC_DIAG_INPUT1_TECH_ID_LIST = AVNO + -TC_DIAG_INPUT_DIR = {INPUT_BASE}/met_test/new/model_data/grib2/gfs_fv3 -TC_DIAG_INPUT_TEMPLATE = gfs.subset.t00z.pgrb2.0p25.f* +TC_DIAG_INPUT2_DIR = {INPUT_BASE}/met_test/new/model_data/grib2/gfs +TC_DIAG_INPUT2_TEMPLATE = subset.gfs.t12z.pgrb2.0p50.f* +TC_DIAG_INPUT2_DOMAIN = nest +TC_DIAG_INPUT2_TECH_ID_LIST = AVNO -TC_DIAG_OUTPUT_DIR = {OUTPUT_BASE}/met_tool_wrapper/TCDiag -TC_DIAG_OUTPUT_TEMPLATE = tc_diag_aal14{date?fmt=%Y}.nc +TC_DIAG_OUTPUT_DIR = {OUTPUT_BASE}/tc_diag +TC_DIAG_OUTPUT_TEMPLATE = {date?fmt=%Y} ### @@ -53,10 +61,10 @@ TC_DIAG_OUTPUT_TEMPLATE = tc_diag_aal14{date?fmt=%Y}.nc ### BOTH_VAR1_NAME = PRMSL -BOTH_VAR1_LEVELS = L0 +BOTH_VAR1_LEVELS = Z0 BOTH_VAR2_NAME = TMP -BOTH_VAR2_LEVELS = P1000, P900, P800, P700, P500, P100 +BOTH_VAR2_LEVELS = P1000, P925, P850, P700, P500, P400, P300, P250, P200, P150, P100 ### @@ -64,17 +72,17 @@ BOTH_VAR2_LEVELS = P1000, P900, P800, P700, P500, P100 # https://metplus.readthedocs.io/en/latest/Users_Guide/wrappers.html#tcdiag ### -#LOG_TC_DIAG_VERBOSITY = 2 +LOG_TC_DIAG_VERBOSITY = 2 TC_DIAG_CONFIG_FILE = {PARM_BASE}/met_config/TCDiagConfig_wrapped -MODEL = fv3 +MODEL = GFSO -TC_DIAG_STORM_ID = AL142016 +TC_DIAG_STORM_ID = AL032023 TC_DIAG_BASIN = AL -TC_DIAG_CYCLONE = 14 +TC_DIAG_CYCLONE = 03 -#TC_DIAG_INIT_INCLUDE = +TC_DIAG_INIT_INCLUDE = {init?fmt=%Y%m%d%H} #TC_DIAG_VALID_BEG = #TC_DIAG_VALID_END = #TC_DIAG_VALID_INCLUDE_LIST = @@ -84,12 +92,18 @@ TC_DIAG_CYCLONE = 14 #TC_DIAG_DIAG_SCRIPT = -#TC_DIAG_DOMAIN_INFO1_DOMAIN = -#TC_DIAG_DOMAIN_INFO1_N_RANGE = -#TC_DIAG_DOMAIN_INFO1_N_AZIMUTH = -#TC_DIAG_DOMAIN_INFO1_DELTA_RANGE_KM = +TC_DIAG_DOMAIN_INFO1_DOMAIN = parent +TC_DIAG_DOMAIN_INFO1_N_RANGE = 150 +TC_DIAG_DOMAIN_INFO1_N_AZIMUTH = 8 +TC_DIAG_DOMAIN_INFO1_DELTA_RANGE_KM = 10.0 #TC_DIAG_DOMAIN_INFO1_DIAG_SCRIPT = +TC_DIAG_DOMAIN_INFO2_DOMAIN = nest +TC_DIAG_DOMAIN_INFO2_N_RANGE = 150 +TC_DIAG_DOMAIN_INFO2_N_AZIMUTH = 8 +TC_DIAG_DOMAIN_INFO2_DELTA_RANGE_KM = 2.0 + + #TC_DIAG_CENSOR_THRESH = #TC_DIAG_CENSOR_VAL = #TC_DIAG_CONVERT = @@ -117,3 +131,5 @@ TC_DIAG_CYCLONE = 14 #TC_DIAG_NC_DIAG_FLAG = #TC_DIAG_CIRA_DIAG_FLAG = #TC_DIAG_OUTPUT_PREFIX = + +#LOG_LEVEL=DEBUG \ No newline at end of file