diff --git a/.github/parm/use_case_groups.json b/.github/parm/use_case_groups.json index d7f55c334..9a9a0627b 100644 --- a/.github/parm/use_case_groups.json +++ b/.github/parm/use_case_groups.json @@ -174,6 +174,11 @@ "index_list": "12", "run": false }, + { + "category": "s2s", + "index_list": "13", + "run": false + }, { "category": "space_weather", "index_list": "0-1", diff --git a/docs/Contributors_Guide/add_use_case.rst b/docs/Contributors_Guide/add_use_case.rst index 8d02d21e3..58f8f7a8e 100644 --- a/docs/Contributors_Guide/add_use_case.rst +++ b/docs/Contributors_Guide/add_use_case.rst @@ -966,6 +966,12 @@ the use case category just in case something goes wrong:: version**, then simply remove the tarfile link:: unlink sample_data-${METPLUS_USE_CASE_CATEGORY}.tgz + +Remove old data (if applicable). + +If the pull request notes mention an old directory path that should be removed, +please remove that directory. Be careful not to remove any files that are +still needed. Create the new sample data tarfile. @@ -981,12 +987,6 @@ Create the new sample data tarfile. tar czf sample_data-${METPLUS_USE_CASE_CATEGORY}-${METPLUS_VERSION}.tgz met_test -Remove old data (if applicable) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -If the pull request notes mention an old directory path that should be removed, -please remove that directory. Be careful not to remove any files that are -still needed. Update the link in the develop directory if needed ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/docs/Contributors_Guide/index.rst b/docs/Contributors_Guide/index.rst index 2c7b89ca3..4dd87bb41 100644 --- a/docs/Contributors_Guide/index.rst +++ b/docs/Contributors_Guide/index.rst @@ -17,6 +17,7 @@ Contributor's Guide add_use_case continuous_integration documentation + user_support Indices and tables ================== diff --git a/docs/Contributors_Guide/user_support.rst b/docs/Contributors_Guide/user_support.rst new file mode 100644 index 000000000..adb927a59 --- /dev/null +++ b/docs/Contributors_Guide/user_support.rst @@ -0,0 +1,138 @@ +************ +User Support +************ + +Support for the METplus components is provided through the +`METplus GitHub Discussions Forum `_. +Discussions is a collaborative communication forum, where the METplus +community can ask and answer questions, gather feedback, and share tips and +tricks. Discussions are categorized to help community members find related +discussions and begin conversations in the right place. + + +Support Responsibilities +======================== + +Five staff members take turns monitoring Discussions each day of the week. +The responsibilities for each daily assignee are described below. + + +Review the Tips/Tricks Page +--------------------------- + +Review the `Tips/Tricks `_ +page. + +The Tips/Tricks page is for discussion only and does not have the ability to +have an *answer* assigned to it. Check for any new dicussions that have been +added by users that are actually questions. If there are, move the discussion +to a more appropriate category so that it can be found when filtering for +unanswered discussions. Please note that if the discussions have been +filtered by "Unanswered" and the Tips/Tricks category is selected, no results +will be found because these discussions are not answerable. + +If a new tip or trick has been added, please review the addition. Consider +adding any comments to add detail or clarify, and thank the user for their +contribution. + +Review the Unanswered Discussions +--------------------------------- + +Review the `unanswered discussions `_. + +* For new discussions that have been posted on the current day (i.e. since last midnight): + + * If the issue is time sensitive (i.e. related to an upcoming training series + session) or from a user with priority support, take action. See :ref:`actions_new_discussions`. + + * Otherwise, leave the discussion unanswered until the next day to allow the + user community time to respond. + +* For new discussions that have 0 repsonses and were posted the previous day: + + * Take action. See :ref:`actions_new_discussions`. + +* Remaining unanswered discussions: + + * Review the latest on discussion history to see if any additional action is + required. + + * If an obvious answer exists, consider marking it as the answer and + locking the discussion to prevent future posts. See + :ref:`lock_discussions`. + + * If the user has not responded in 7 or more days, consider following up to + see if the user's issue is resolved or if they need more assistance. + + * If unsure how to proceed, post in the User Support Slack channel of + dtc-metplus.slack.com tagging the person who has been assisting the user. + +Review Answered Discussions that are Unlocked +--------------------------------------------- + +Review `anwered discussions that are unlocked `_. + +If there is any recent activity on an **answered** question that indicates the +questioner is satisified with the response, click on the button to **lock** the +discussion to prevent future activity. See :ref:`lock_discussions`. + +.. _actions_new_discussions: + +Actions for Handling New Discussions +------------------------------------ + +* Check to see if the discussion should be reassigned from its current category + to some other category. + +* Review the labels and pick at least one of the blue ones ("component:" and/or + repository-specific - e.g. "MET:", "METplus:", etc.) to further categorize it. + +* Ensure the discussion is answered: + + * If you can answer it yourself, do so. + + * If you can answer it with help from someone, post to the User Support Slack + channel of dtc-metplus.slack.com, asking for advice. + + * If it would be better to have someone else answer, coordinate in the User + Support Slack channel of dtc-metplus.slack.com or comment on the discussion + with something like: + + *@ would be the best person to address your question + and should respond within a couple of days.* + + If it is obvious that test data will be needed, please ask the user to + send that data, pointing them to the + `How to Send Us Data `_ + discussions post and asking them to let you know once they have uploaded it. + +.. _lock_discussions: + + +How to Lock a Discussion +------------------------ + +If an obvious answer exists and it is apparent the questioner is satisified +with the response, consider posting something like the following before locking +the discussion: + +*At this point I'm going to LOCK this conversation to prevent future posts. Our team has decided that once discussions have been answered, and the answer has been confirmed, we'll lock them. We want to encourage users to ask new questions in new discussions rather than posting to old ones. Hopefully that'll make the questions/answers easier for other users to follow. So if/when more issues/questions arise, please feel free to start a new discussion.* + +If there is no answer selected and no indication that the questioner is +satisified with the response, follow up, requesting that the user mark +the best answer so we can lock the discussion. The following text may need +to be modified to match a specific situation, but this is a good starting +point as an example: + +*I'd like to mark this discussion as being answered and "lock it" to prevent future posts. That's how we encourage users to create new discussions for new questions. But I wanted to give you an opportunity to comment on it before doing so. Please feel free to select one of the responses as being the best answer to your original question. That'll help future users with similar questions find answers.* + +If following up on discussions where the user hasn't responded back in a while, +consider posting something like the following before locking the discussion, +filling in the MONTH and DAY below: + +*I wanted to check in with you about the initial discussion you submitted. We were able to provide some guidance for you on , but haven't heard back if the suggested actions helped. If it did, or if you found a different way of accomplishing your goal, we'd appreciate hearing from you. If you have additional questions on this, we can try to find a solution; otherwise we'll choose an answer and lock this discussion due to inactivity.* + +GitHub users with sufficient permissions in the METplus repository should see a +"Lock conversation" link on the right side navigation of each Discussions page. + + diff --git a/docs/Release_Guide/met_bugfix.rst b/docs/Release_Guide/met_bugfix.rst index 2046bc542..ea3861750 100644 --- a/docs/Release_Guide/met_bugfix.rst +++ b/docs/Release_Guide/met_bugfix.rst @@ -14,7 +14,6 @@ Create a new vX.Y.Z bugfix release from the main_vX.Y branch. .. include:: release_steps/merge_release_issue.rst .. include:: release_steps/create_release_on_github.rst .. include:: release_steps/create_release_extra.rst -.. include:: release_steps/met/attach_release_tarfile.rst .. include:: release_steps/met/update_dtc_website.rst .. include:: release_steps/met/confirm_zenodo.rst .. include:: release_steps/finalize_release_on_github_bugfix.rst diff --git a/docs/Release_Guide/met_development.rst b/docs/Release_Guide/met_development.rst index 48299ec4a..a37f672b5 100644 --- a/docs/Release_Guide/met_development.rst +++ b/docs/Release_Guide/met_development.rst @@ -14,6 +14,5 @@ Create a new vX.Y.Z-betaN or vX.Y.Z-rcN development release from the develop bra .. include:: release_steps/merge_release_issue.rst .. include:: release_steps/create_release_on_github.rst .. include:: release_steps/create_release_extra.rst -.. include:: release_steps/met/attach_release_tarfile.rst .. include:: release_steps/met/update_dtc_website.rst .. include:: release_steps/finalize_release_on_github_development.rst diff --git a/docs/Release_Guide/met_official.rst b/docs/Release_Guide/met_official.rst index 7bef52530..47b3c969a 100644 --- a/docs/Release_Guide/met_official.rst +++ b/docs/Release_Guide/met_official.rst @@ -17,7 +17,6 @@ Create a new vX.Y.Z official release from the develop branch. .. include:: release_steps/create_release_branch.rst .. include:: release_steps/create_release_on_github.rst .. include:: release_steps/create_release_extra.rst -.. include:: release_steps/met/attach_release_tarfile.rst .. include:: release_steps/met/update_dtc_website.rst .. include:: release_steps/finalize_release_on_github_official.rst .. include:: release_steps/update_docs_official.rst diff --git a/docs/Release_Guide/release_steps/met/attach_release_tarfile.rst b/docs/Release_Guide/release_steps/met/attach_release_tarfile.rst deleted file mode 100644 index 65d99d6b0..000000000 --- a/docs/Release_Guide/release_steps/met/attach_release_tarfile.rst +++ /dev/null @@ -1,11 +0,0 @@ -Attach Release Tarfile ----------------------- - -* The MET software is distributed as a tarfile, but not the one created by GitHub. On a project machine (e.g. kiowa), clone the MET repository and run a script to build the tarfile for the newly tagged release. - -.. parsed-literal:: - - git clone https://github.com/dtcenter/MET - MET/scripts/build/met_checkout_and_build.sh tag vX.Y.Z - -* Edit the vX.Y.Z release on GitHub by uploading the resulting tar file (met-X.Y.Z.YYYYMMDD.tar.gz) as a release asset. diff --git a/docs/Release_Guide/release_steps/met/update_version_bugfix.rst b/docs/Release_Guide/release_steps/met/update_version_bugfix.rst index 7320285e3..05184f526 100644 --- a/docs/Release_Guide/release_steps/met/update_version_bugfix.rst +++ b/docs/Release_Guide/release_steps/met/update_version_bugfix.rst @@ -3,8 +3,8 @@ Update Version Number * Update the version in the code and documentation: - * Update the *met_version* variable in *met/src/basic/vx_util/util_constants.h* which defines the version number written to the MET output files. + * Update the *met_version* variable in *src/basic/vx_util/util_constants.h* which defines the version number written to the MET output files. - * In *met/docs/conf.py*, update the *version*, *release_year*, and *release_date* variables for the documentation. + * In *docs/conf.py*, update the *version*, *release_year*, and *release_date* variables for the documentation. * DO NOT update the version number listed in the MET configuration files, add a new table file, or add a new test header file. diff --git a/docs/Release_Guide/release_steps/met/update_version_development.rst b/docs/Release_Guide/release_steps/met/update_version_development.rst index c52654555..6764bd468 100644 --- a/docs/Release_Guide/release_steps/met/update_version_development.rst +++ b/docs/Release_Guide/release_steps/met/update_version_development.rst @@ -7,4 +7,4 @@ Update Version Number * If the official release version has already been updated (e.g. beta2 and beyond), do the following: - * In *met/docs/conf.py*, update the *version*, *release_year*, and *release_date* variables for the documentation. + * In *docs/conf.py*, update the *version*, *release_year*, and *release_date* variables for the documentation. diff --git a/docs/Release_Guide/release_steps/met/update_version_official.rst b/docs/Release_Guide/release_steps/met/update_version_official.rst index 6c72a7d91..1c4ec4f8d 100644 --- a/docs/Release_Guide/release_steps/met/update_version_official.rst +++ b/docs/Release_Guide/release_steps/met/update_version_official.rst @@ -3,18 +3,18 @@ Update Version Number * Update the version in the code and documentation: - * If necessary, update the *met_version* variable in *met/src/basic/vx_util/util_constants.h* which defines the version number written to the MET output files. + * If necessary, update the *met_version* variable in *src/basic/vx_util/util_constants.h* which defines the version number written to the MET output files. - * In *met/docs/conf.py*, update the *version*, *release_year*, and *release_date* variables for the documentation. + * In *docs/conf.py*, update the *version*, *release_year*, and *release_date* variables for the documentation. * If necessary, update the version number listed in the MET configuration files: - * Default configuration files in *met/data/config*. + * Default configuration files in *data/config*. - * Sample configuration files in *met/scripts/config*. + * Sample configuration files in *scripts/config*. - * Test configuration files, searching recursively, in *test/config*. + * Test configuration files, searching recursively, in *internal/test_unit/config*. - * If necessary, add a new *met/data/table_files/met_header_columns_VX.Y.txt* defining the columns names for this version. + * If necessary, add a new *data/table_files/met_header_columns_VX.Y.txt* defining the columns names for this version. - * If necessary, add a new *test/hdr/met_X_Y.hdr* file defining the column names for this version for the test scripts. + * If necessary, add a new *internal/test_unit/hdr/met_X_Y.hdr* file defining the column names for this version for the test scripts. diff --git a/docs/Release_Guide/release_steps/update_release_notes_bugfix.rst b/docs/Release_Guide/release_steps/update_release_notes_bugfix.rst index 54882d105..e530beada 100644 --- a/docs/Release_Guide/release_steps/update_release_notes_bugfix.rst +++ b/docs/Release_Guide/release_steps/update_release_notes_bugfix.rst @@ -1,15 +1,26 @@ Update Release Notes -------------------- -You can refer to the GitHub Issues page to see what has changed for this +You can refer to the GitHub Project board to see what has changed for this release. Open the following URL in a browser: .. parsed-literal:: - https://github.com/dtcenter/|projectRepo|/issues + https://github.com/orgs/dtcenter/projects?type=beta -* Click on the Projects tab and select the project (under Repository) that - corresponds to the release you are creating. +* Click on the project that corresponds to support for the release, i.e. + |projectRepo| Version X.Y Support + +* Navigate to the "Closed Issues" tab. + **If this tab does not exist**, follow these instructions to create it: + + * Click on "+ New view" button on the far right side of the view tabs + * Click on "View " (where is an integer) and rename it to + "Closed Issues" + * Click on the down arrow next to the newly created view + * Click on "Search or filter this view" + * Enter the following info into the filter bar: **is:closed is:issue** + * Click on the down arrow next to the view and click "Save changes" * Update the release-notes.rst file found in the User's Guide directory. diff --git a/docs/Release_Guide/release_steps/update_release_notes_development.rst b/docs/Release_Guide/release_steps/update_release_notes_development.rst index 378b17fe0..8cc4021a6 100644 --- a/docs/Release_Guide/release_steps/update_release_notes_development.rst +++ b/docs/Release_Guide/release_steps/update_release_notes_development.rst @@ -1,15 +1,26 @@ Update Release Notes -------------------- -You can refer to the GitHub Issues page to see what has changed for this +You can refer to the GitHub Project board to see what has changed for this release. Open the following URL in a browser: .. parsed-literal:: - https://github.com/dtcenter/|projectRepo|/issues + https://github.com/orgs/dtcenter/projects?type=beta -* Click on the Projects tab and select the project (under Repository) that - corresponds to the release you are creating. +* Click on the project that corresponds to this release, i.e. + |projectRepo|-X.Y.Z-betaN + +* Navigate to the "Closed Issues" tab. + **If this tab does not exist**, follow these instructions to create it: + + * Click on "+ New view" button on the far right side of the view tabs + * Click on "View " (where is an integer) and rename it to + "Closed Issues" + * Click on the down arrow next to the newly created view + * Click on "Search or filter this view" + * Enter the following info into the filter bar: **is:closed is:issue** + * Click on the down arrow next to the view and click "Save changes" * Update the release-notes.rst file found in the User's Guide directory. diff --git a/docs/Release_Guide/release_steps/update_release_notes_official.rst b/docs/Release_Guide/release_steps/update_release_notes_official.rst index f3c384e7b..02e077b4f 100644 --- a/docs/Release_Guide/release_steps/update_release_notes_official.rst +++ b/docs/Release_Guide/release_steps/update_release_notes_official.rst @@ -1,15 +1,26 @@ Update Release Notes -------------------- -You can refer to the GitHub Issues page to see what has changed for this +You can refer to the GitHub Project board to see what has changed for this release. Open the following URL in a browser: .. parsed-literal:: - https://github.com/dtcenter/|projectRepo|/issues + https://github.com/orgs/dtcenter/projects?type=beta -* Click on the Projects tab and select the project (under Repository) that - corresponds to the release you are creating. +* Click on the project that corresponds to this release, i.e. + |projectRepo|-X.Y.Z-rcN + +* Navigate to the "Closed Issues" tab. + **If this tab does not exist**, follow these instructions to create it: + + * Click on "+ New view" button on the far right side of the view tabs + * Click on "View " (where is an integer) and rename it to + "Closed Issues" + * Click on the down arrow next to the newly created view + * Click on "Search or filter this view" + * Enter the following info into the filter bar: **is:closed is:issue** + * Click on the down arrow next to the view and click "Save changes" * Update the release-notes.rst file found in the User's Guide directory. diff --git a/docs/Users_Guide/glossary.rst b/docs/Users_Guide/glossary.rst index dcae47e5e..5bd8a3559 100644 --- a/docs/Users_Guide/glossary.rst +++ b/docs/Users_Guide/glossary.rst @@ -5655,7 +5655,22 @@ METplus Configuration Glossary | *Used by:* EnsembleStat ENSEMBLE_STAT_CLIMO_MEAN_FIELD - Specify the value for 'climo_mean.field' in the MET configuration file for EnsembleStat. + See: :term:`_CLIMO_MEAN_FIELD` + + | *Used by:* EnsembleStat + + ENSEMBLE_STAT_CLIMO_MEAN_VAR_NAME + See: :term:`_CLIMO_MEAN_VAR_NAME` + + | *Used by:* EnsembleStat + + ENSEMBLE_STAT_CLIMO_MEAN_VAR_LEVELS + See: :term:`_CLIMO_MEAN_VAR_LEVELS` + + | *Used by:* EnsembleStat + + ENSEMBLE_STAT_CLIMO_MEAN_VAR_OPTIONS + See: :term:`_CLIMO_MEAN_VAR_OPTIONS` | *Used by:* EnsembleStat @@ -5705,7 +5720,42 @@ METplus Configuration Glossary | *Used by:* EnsembleStat ENSEMBLE_STAT_CLIMO_STDEV_FIELD - Specify the value for 'climo_stdev.field' in the MET configuration file for EnsembleStat. + Specify the value for 'climo_stdev.field' in the MET configuration file + for EnsembleStat. + The value set here must include the proper formatting that is expected in + MET configuration file for specifying field information. + Example: {name="TMP"; level="(*,*)";} + To set the field information un-formatted, use the + :term:`ENSEMBLE_STAT_CLIMO_STDEV_VAR_NAME`, + :term:`ENSEMBLE_STAT_CLIMO_STDEV_VAR_LEVELS`, and + :term:`ENSEMBLE_STAT_CLIMO_STDEV_VAR_OPTIONS` variables. + + | *Used by:* EnsembleStat + + ENSEMBLE_STAT_CLIMO_STDEV_VAR_NAME + Specify the name of the nth field for 'climo_stdev.field' in the + MET configuration file for EnsembleStat. If any fields are set using this + variable, then :term:`ENSEMBLE_STAT_CLIMO_STDEV_FIELD` will be ignored. + See also :term:`ENSEMBLE_STAT_CLIMO_STDEV_VAR_LEVELS` + and :term:`ENSEMBLE_STAT_CLIMO_STDEV_VAR_OPTIONS`. + + | *Used by:* EnsembleStat + + ENSEMBLE_STAT_CLIMO_STDEV_VAR_LEVELS + Specify the level of the nth field for 'climo_stdev.field' in the + MET configuration file for EnsembleStat. If any fields are set using this + variable, then :term:`ENSEMBLE_STAT_CLIMO_STDEV_FIELD` will be ignored. + See also :term:`ENSEMBLE_STAT_CLIMO_STDEV_VAR_NAME` + and :term:`ENSEMBLE_STAT_CLIMO_STDEV_VAR_OPTIONS`. + + | *Used by:* EnsembleStat + + ENSEMBLE_STAT_CLIMO_STDEV_VAR_OPTIONS + Specify the extra options of the nth field for 'climo_stdev.field' in the + MET configuration file for EnsembleStat. If any fields are set using this + variable, then :term:`ENSEMBLE_STAT_CLIMO_STDEV_FIELD` will be ignored. + See also :term:`ENSEMBLE_STAT_CLIMO_STDEV_VAR_NAME` + and :term:`ENSEMBLE_STAT_CLIMO_STDEV_VAR_LEVELS`. | *Used by:* EnsembleStat @@ -6997,7 +7047,22 @@ METplus Configuration Glossary | *Used by:* PointStat POINT_STAT_CLIMO_MEAN_FIELD - Specify the value for 'climo_mean.field' in the MET configuration file for PointStat. + See: :term:`_CLIMO_MEAN_FIELD` + + | *Used by:* PointStat + + POINT_STAT_CLIMO_MEAN_VAR_NAME + See: :term:`_CLIMO_MEAN_VAR_NAME` + + | *Used by:* PointStat + + POINT_STAT_CLIMO_MEAN_VAR_LEVELS + See: :term:`_CLIMO_MEAN_VAR_LEVELS` + + | *Used by:* PointStat + + POINT_STAT_CLIMO_MEAN_VAR_OPTIONS + See: :term:`_CLIMO_MEAN_VAR_OPTIONS` | *Used by:* PointStat @@ -7047,7 +7112,42 @@ METplus Configuration Glossary | *Used by:* PointStat POINT_STAT_CLIMO_STDEV_FIELD - Specify the value for 'climo_stdev.field' in the MET configuration file for PointStat. + Specify the value for 'climo_stdev.field' in the MET configuration file + for PointStat. + The value set here must include the proper formatting that is expected in + MET configuration file for specifying field information. + Example: {name="TMP"; level="(*,*)";} + To set the field information un-formatted, use the + :term:`POINT_STAT_CLIMO_STDEV_VAR_NAME`, + :term:`POINT_STAT_CLIMO_STDEV_VAR_LEVELS`, and + :term:`POINT_STAT_CLIMO_STDEV_VAR_OPTIONS` variables. + + | *Used by:* PointStat + + POINT_STAT_CLIMO_STDEV_VAR_NAME + Specify the name of the nth field for 'climo_stdev.field' in the + MET configuration file for PointStat. If any fields are set using this + variable, then :term:`POINT_STAT_CLIMO_STDEV_FIELD` will be ignored. + See also :term:`POINT_STAT_CLIMO_STDEV_VAR_LEVELS` + and :term:`POINT_STAT_CLIMO_STDEV_VAR_OPTIONS`. + + | *Used by:* PointStat + + POINT_STAT_CLIMO_STDEV_VAR_LEVELS + Specify the level of the nth field for 'climo_stdev.field' in the + MET configuration file for PointStat. If any fields are set using this + variable, then :term:`POINT_STAT_CLIMO_STDEV_FIELD` will be ignored. + See also :term:`POINT_STAT_CLIMO_STDEV_VAR_NAME` + and :term:`POINT_STAT_CLIMO_STDEV_VAR_OPTIONS`. + + | *Used by:* PointStat + + POINT_STAT_CLIMO_STDEV_VAR_OPTIONS + Specify the extra options of the nth field for 'climo_stdev.field' in the + MET configuration file for PointStat. If any fields are set using this + variable, then :term:`POINT_STAT_CLIMO_STDEV_FIELD` will be ignored. + See also :term:`POINT_STAT_CLIMO_STDEV_VAR_NAME` + and :term:`POINT_STAT_CLIMO_STDEV_VAR_LEVELS`. | *Used by:* PointStat @@ -7127,7 +7227,22 @@ METplus Configuration Glossary | *Used by:* GridStat GRID_STAT_CLIMO_MEAN_FIELD - Specify the value for 'climo_mean.field' in the MET configuration file for GridStat. + See: :term:`_CLIMO_MEAN_FIELD` + + | *Used by:* GridStat + + GRID_STAT_CLIMO_MEAN_VAR_NAME + See: :term:`_CLIMO_MEAN_VAR_NAME` + + | *Used by:* GridStat + + GRID_STAT_CLIMO_MEAN_VAR_LEVELS + See: :term:`_CLIMO_MEAN_VAR_LEVELS` + + | *Used by:* GridStat + + GRID_STAT_CLIMO_MEAN_VAR_OPTIONS + See: :term:`_CLIMO_MEAN_VAR_OPTIONS` | *Used by:* GridStat @@ -7177,7 +7292,42 @@ METplus Configuration Glossary | *Used by:* GridStat GRID_STAT_CLIMO_STDEV_FIELD - Specify the value for 'climo_stdev.field' in the MET configuration file for GridStat. + Specify the value for 'climo_stdev.field' in the MET configuration file + for GridStat. + The value set here must include the proper formatting that is expected in + MET configuration file for specifying field information. + Example: {name="TMP"; level="(*,*)";} + To set the field information un-formatted, use the + :term:`GRID_STAT_CLIMO_STDEV_VAR_NAME`, + :term:`GRID_STAT_CLIMO_STDEV_VAR_LEVELS`, and + :term:`GRID_STAT_CLIMO_STDEV_VAR_OPTIONS` variables. + + | *Used by:* GridStat + + GRID_STAT_CLIMO_STDEV_VAR_NAME + Specify the name of the nth field for 'climo_stdev.field' in the + MET configuration file for GridStat. If any fields are set using this + variable, then :term:`GRID_STAT_CLIMO_STDEV_FIELD` will be ignored. + See also :term:`GRID_STAT_CLIMO_STDEV_VAR_LEVELS` + and :term:`GRID_STAT_CLIMO_STDEV_VAR_OPTIONS`. + + | *Used by:* GridStat + + GRID_STAT_CLIMO_STDEV_VAR_LEVELS + Specify the level of the nth field for 'climo_stdev.field' in the + MET configuration file for GridStat. If any fields are set using this + variable, then :term:`GRID_STAT_CLIMO_STDEV_FIELD` will be ignored. + See also :term:`GRID_STAT_CLIMO_STDEV_VAR_NAME` + and :term:`GRID_STAT_CLIMO_STDEV_VAR_OPTIONS`. + + | *Used by:* GridStat + + GRID_STAT_CLIMO_STDEV_VAR_OPTIONS + Specify the extra options of the nth field for 'climo_stdev.field' in the + MET configuration file for GridStat. If any fields are set using this + variable, then :term:`GRID_STAT_CLIMO_STDEV_FIELD` will be ignored. + See also :term:`GRID_STAT_CLIMO_STDEV_VAR_NAME` + and :term:`GRID_STAT_CLIMO_STDEV_VAR_LEVELS`. | *Used by:* GridStat @@ -7243,7 +7393,22 @@ METplus Configuration Glossary | *Used by:* SeriesAnalysis SERIES_ANALYSIS_CLIMO_MEAN_FIELD - Specify the value for 'climo_mean.field' in the MET configuration file for SeriesAnalysis. + See: :term:`_CLIMO_MEAN_FIELD` + + | *Used by:* SeriesAnalysis + + SERIES_ANALYSIS_CLIMO_MEAN_VAR_NAME + See: :term:`_CLIMO_MEAN_VAR_NAME` + + | *Used by:* SeriesAnalysis + + SERIES_ANALYSIS_CLIMO_MEAN_VAR_LEVELS + See: :term:`_CLIMO_MEAN_VAR_LEVELS` + + | *Used by:* SeriesAnalysis + + SERIES_ANALYSIS_CLIMO_MEAN_VAR_OPTIONS + See: :term:`_CLIMO_MEAN_VAR_OPTIONS` | *Used by:* SeriesAnalysis @@ -7293,7 +7458,42 @@ METplus Configuration Glossary | *Used by:* SeriesAnalysis SERIES_ANALYSIS_CLIMO_STDEV_FIELD - Specify the value for 'climo_stdev.field' in the MET configuration file for SeriesAnalysis. + Specify the value for 'climo_stdev.field' in the MET configuration file + for SeriesAnalysis. + The value set here must include the proper formatting that is expected in + MET configuration file for specifying field information. + Example: {name="TMP"; level="(*,*)";} + To set the field information un-formatted, use the + :term:`SERIES_ANALYSIS_CLIMO_STDEV_VAR_NAME`, + :term:`SERIES_ANALYSIS_CLIMO_STDEV_VAR_LEVELS`, and + :term:`SERIES_ANALYSIS_CLIMO_STDEV_VAR_OPTIONS` variables. + + | *Used by:* SeriesAnalysis + + SERIES_ANALYSIS_CLIMO_STDEV_VAR_NAME + Specify the name of the nth field for 'climo_stdev.field' in the + MET configuration file for SeriesAnalysis. If any fields are set using this + variable, then :term:`SERIES_ANALYSIS_CLIMO_STDEV_FIELD` will be ignored. + See also :term:`SERIES_ANALYSIS_CLIMO_STDEV_VAR_LEVELS` + and :term:`SERIES_ANALYSIS_CLIMO_STDEV_VAR_OPTIONS`. + + | *Used by:* SeriesAnalysis + + SERIES_ANALYSIS_CLIMO_STDEV_VAR_LEVELS + Specify the level of the nth field for 'climo_stdev.field' in the + MET configuration file for SeriesAnalysis. If any fields are set using this + variable, then :term:`SERIES_ANALYSIS_CLIMO_STDEV_FIELD` will be ignored. + See also :term:`SERIES_ANALYSIS_CLIMO_STDEV_VAR_NAME` + and :term:`SERIES_ANALYSIS_CLIMO_STDEV_VAR_OPTIONS`. + + | *Used by:* SeriesAnalysis + + SERIES_ANALYSIS_CLIMO_STDEV_VAR_OPTIONS + Specify the extra options of the nth field for 'climo_stdev.field' in the + MET configuration file for SeriesAnalysis. If any fields are set using this + variable, then :term:`SERIES_ANALYSIS_CLIMO_STDEV_FIELD` will be ignored. + See also :term:`SERIES_ANALYSIS_CLIMO_STDEV_VAR_NAME` + and :term:`SERIES_ANALYSIS_CLIMO_STDEV_VAR_LEVELS`. | *Used by:* SeriesAnalysis @@ -8275,10 +8475,23 @@ METplus Configuration Glossary | *Used by:* GenEnsProd GEN_ENS_PROD_CLIMO_MEAN_FIELD - Specify the value for 'climo_mean.field' in the MET configuration file for GenEnsProd. + See: :term:`_CLIMO_MEAN_FIELD` + + | *Used by:* GenEnsProd + + GEN_ENS_PROD_CLIMO_MEAN_VAR_NAME + See: :term:`_CLIMO_MEAN_VAR_NAME` | *Used by:* GenEnsProd + GEN_ENS_PROD_CLIMO_MEAN_VAR_LEVELS + See: :term:`_CLIMO_MEAN_VAR_LEVELS` + + | *Used by:* GenEnsProd + + GEN_ENS_PROD_CLIMO_MEAN_VAR_OPTIONS + See: :term:`_CLIMO_MEAN_VAR_OPTIONS` + GEN_ENS_PROD_CLIMO_MEAN_REGRID_METHOD Specify the value for 'climo_mean.regrid.method' in the MET configuration file for GenEnsProd. @@ -8325,10 +8538,43 @@ METplus Configuration Glossary | *Used by:* GenEnsProd GEN_ENS_PROD_CLIMO_STDEV_FIELD - Specify the value for 'climo_stdev.field' in the MET configuration file for GenEnsProd. + Specify the value for 'climo_stdev.field' in the MET configuration file + for GenEnsProd. + The value set here must include the proper formatting that is expected in + MET configuration file for specifying field information. + Example: {name="TMP"; level="(*,*)";} + To set the field information un-formatted, use the + :term:`GEN_ENS_PROD_CLIMO_STDEV_VAR_NAME`, + :term:`GEN_ENS_PROD_CLIMO_STDEV_VAR_LEVELS`, and + :term:`GEN_ENS_PROD_CLIMO_STDEV_VAR_OPTIONS` variables. | *Used by:* GenEnsProd + GEN_ENS_PROD_CLIMO_STDEV_VAR_NAME + Specify the name of the nth field for 'climo_stdev.field' in the + MET configuration file for GenEnsProd. If any fields are set using this + variable, then :term:`GEN_ENS_PROD_CLIMO_STDEV_FIELD` will be ignored. + See also :term:`GEN_ENS_PROD_CLIMO_STDEV_VAR_LEVELS` + and :term:`GEN_ENS_PROD_CLIMO_STDEV_VAR_OPTIONS`. + + | *Used by:* GenEnsProd + + GEN_ENS_PROD_CLIMO_STDEV_VAR_LEVELS + Specify the level of the nth field for 'climo_stdev.field' in the + MET configuration file for GenEnsProd. If any fields are set using this + variable, then :term:`GEN_ENS_PROD_CLIMO_STDEV_FIELD` will be ignored. + See also :term:`GEN_ENS_PROD_CLIMO_STDEV_VAR_NAME` + and :term:`GEN_ENS_PROD_CLIMO_STDEV_VAR_OPTIONS`. + + | *Used by:* GenEnsProd + + GEN_ENS_PROD_CLIMO_STDEV_VAR_OPTIONS + Specify the extra options of the nth field for 'climo_stdev.field' in the + MET configuration file for GenEnsProd. If any fields are set using this + variable, then :term:`GEN_ENS_PROD_CLIMO_STDEV_FIELD` will be ignored. + See also :term:`GEN_ENS_PROD_CLIMO_STDEV_VAR_NAME` + and :term:`GEN_ENS_PROD_CLIMO_STDEV_VAR_LEVELS`. + GEN_ENS_PROD_CLIMO_STDEV_REGRID_METHOD Specify the value for 'climo_stdev.regrid.method' in the MET configuration file for GenEnsProd. @@ -9259,3 +9505,103 @@ METplus Configuration Glossary Specify the value for 'eclv_points' in the MET configuration file for EnsembleStat. | *Used by:* EnsembleStat + + _CLIMO_MEAN_FIELD + Specify the value for 'climo_mean.field' in the MET configuration file for + i.e. EnsembleStat. + The value set here must include the proper formatting that is expected in + MET configuration file for specifying field information. + Example: {name="TMP"; level="(*,*)";} + To set the field information un-formatted, use the + :term:`_CLIMO_MEAN_VAR_NAME`, + :term:`_CLIMO_MEAN_VAR_LEVELS`, and + :term:`_CLIMO_MEAN_VAR_OPTIONS` variables. + + | *Used by:* Varies + + _CLIMO_MEAN_VAR_NAME + Specify the name of the nth field for 'climo_mean.field' in the + MET configuration file for i.e. EnsembleStat. + If any fields are set using this + variable, then :term:`_CLIMO_MEAN_FIELD` will be ignored. + See also :term:`_CLIMO_MEAN_VAR_LEVELS` + and :term:`_CLIMO_MEAN_VAR_OPTIONS`. + + | *Used by:* Varies + + _CLIMO_MEAN_VAR_LEVELS + Specify the level of the nth field for 'climo_mean.field' in the + MET configuration file for i.e. EnsembleStat. + If any fields are set using this variable, + then :term:`_CLIMO_MEAN_FIELD` will be ignored. + See also :term:`_CLIMO_MEAN_VAR_NAME` + and :term:`_CLIMO_MEAN_VAR_OPTIONS`. + + | *Used by:* Varies + + _CLIMO_MEAN_VAR_OPTIONS + Specify the extra options of the nth field for 'climo_mean.field' in the + MET configuration file for i.e. EnsembleStat. + If any fields are set using this variable, + then :term:`_CLIMO_MEAN_FIELD` will be ignored. + See also :term:`_CLIMO_MEAN_VAR_NAME` + and :term:`_CLIMO_MEAN_VAR_LEVELS`. + + | *Used by:* Varies + + _CLIMO_STDEV_FIELD + Specify the value for 'climo_stdev.field' in the MET configuration file for + i.e. EnsembleStat. + The value set here must include the proper formatting that is expected in + MET configuration file for specifying field information. + Example: {name="TMP"; level="(*,*)";} + To set the field information un-formatted, use the + :term:`_CLIMO_STDEV_VAR_NAME`, + :term:`_CLIMO_STDEV_VAR_LEVELS`, and + :term:`_CLIMO_STDEV_VAR_OPTIONS` variables. + + | *Used by:* Varies + + _CLIMO_STDEV_VAR_NAME + Specify the name of the nth field for 'climo_stdev.field' in the + MET configuration file for i.e. EnsembleStat. + If any fields are set using this + variable, then :term:`_CLIMO_STDEV_FIELD` will be ignored. + See also :term:`_CLIMO_STDEV_VAR_LEVELS` + and :term:`_CLIMO_STDEV_VAR_OPTIONS`. + + | *Used by:* Varies + + _CLIMO_STDEV_VAR_LEVELS + Specify the level of the nth field for 'climo_stdev.field' in the + MET configuration file for i.e. EnsembleStat. + If any fields are set using this variable, + then :term:`_CLIMO_STDEV_FIELD` will be ignored. + See also :term:`_CLIMO_STDEV_VAR_NAME` + and :term:`_CLIMO_STDEV_VAR_OPTIONS`. + + | *Used by:* Varies + + _CLIMO_STDEV_VAR_OPTIONS + Specify the extra options of the nth field for 'climo_stdev.field' in the + MET configuration file for i.e. EnsembleStat. + If any fields are set using this variable, + then :term:`_CLIMO_STDEV_FIELD` will be ignored. + See also :term:`_CLIMO_STDEV_VAR_NAME` + and :term:`_CLIMO_STDEV_VAR_LEVELS`. + + | *Used by:* Varies + + MODE_MASK_MISSING_FLAG + Specify the value for 'mask_missing_flag' in the MET configuration file for MODE. + + | *Used by:* MODE + + MODE_MULTIVAR_LOGIC + Specify the value for 'multivar_logic' in the MET configuration file + for MODE. If this variable is set, then multi-variate MODE will be run. + This means that more than 1 input file will be read and all of the fields + specified will be processed in a single call to MODE. See the MET User's + Guide for more information on multi-variate MODE. + + | *Used by:* MODE diff --git a/docs/Users_Guide/release-notes.rst b/docs/Users_Guide/release-notes.rst index 3443efde4..a670f9d3a 100644 --- a/docs/Users_Guide/release-notes.rst +++ b/docs/Users_Guide/release-notes.rst @@ -39,311 +39,37 @@ describes the bugfix, enhancement, or new feature: https://github.com/dtcenter/METplus/issues -METplus Version 4.1.0 Release Notes (2022-03-14) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +METplus Version 5.0.0 Beta 1 Release Notes (2022-06-22) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ * Enhancements: * General: - * **Add support for setting control members in EnsembleStat and GenEnsProd** (`#1236 `_) - * **Create an Amazon AMI containing all METplus components** (`#506 `_) - * Modify wrappers that use wrapped MET config files to default to parm/met_config versions if unset (`#931 `_) - * Add support for setting hss_ec_value in MET config files (`#951 `_) - * Added support for setting a dictionary value for time_summary.width (`#1252 `_) - * Properly handle list values that include square braces (`#1212 `_) - * Update wrapped MET config files to reference MET_TMP_DIR in tmp value (`#1101 `_) - * Add support for setting INIT_LIST and VALID_LIST for irregular time intervals (`#1286 `_) - * Support setting the OMP_NUM_THREADS environment variable (`#1320 `_) - * Add support for commonly changed MET config variables (`#896 `_) - * Prevent wildcard character from being used in output file path (`#1291 `_) - * Add support for setting file_type for fcst/obs for applications that process gridded data (`#1165 `_) - * Enhance logic for setting mask.poly to allow MET list characters (square braces and semi-colon) (`#966 `_) - * Add support for new climo_cdf.direct_prob flag (`#1392 `_) - * Implement various enhancements to climatology settings (`#1247 `_) - * Enhance logic to set climatology info for Python embedding (`#944 `_) - * Updated logic for handling _CLIMO_MEAN_FIELD variables for specifying climatology fields (`#1021 `_) - * Incorporate basic zonal and meridional means into METplus (`#1230 `_) - * Add support for explicitly setting file list file paths in wrappers that support multiple input files (`#1289 `_) - * Add support for setting -out argument in TCStat and StatAnalysis wrappers (`#1102 `_) - - * PointStat: - - * Make output_flag.orank configurable for Point-Stat (`#1103 `_) - * Added support for setting obs_quality_inc/exc in PointStat (`#1213 `_) - - * GridStat: - - * Add Grid-Stat configuration options for distance_map dictionary (`#1089 `_) - - * EnsembleStat: - - * Add support for setting grid_weight_flag in EnsembleStat (`#1369 `_) - * Fix logic to use fcst dictionary if ens dictionary is not set in EnsembleStat wrapper (`#1421 `_) - * Add support for probabilistic verification to the Ensemble-Stat wrapper (`#1464 `_) - - * GenEnsProd: - - * Add support for the normalize option to the Gen-Ens-Prod wrapper (`#1445 `_) - - * TCPairs: - - * Enhance TC-Pairs wrapper to make valid_inc, valid_exc, and write_valid configurable options (`#1069 `_) - * Improve logic of TCPairs wrapper (`#749 `_) - * Enhance TCPairs to loop by valid time and allow looping when LOOP_ORDER = processes (`#986 `_) - - * TCGen: - - * Enhance TCGen wrapper to add support for new configurations (`#1273 `_) - - * SeriesAnalysis: - - * **Enhance SeriesAnalysis wrapper to allow different field info values for each file in a list** (`#1166 `_) - * Add support for probability field threshold in SeriesAnalysis (`#875 `_) - - * RegridDataPlane: - - * **Add support for extra field options in RegridDataPlane wrapper** (`#924 `_) - - * PCPCombine: - - * Improve PCPCombine derive mode logic to skip lookback (`#928 `_) - * Add support for using filename templates for defining input level in PCPCombine (`#1062 `_) - * Add option to PCPCombine to force using 0 hr accum in subtract mode (`#1368 `_) - - * GenVxMask: - - * Update GenVxMask wrapper to require setting -type (`#960 `_) - - * UserScript: - - * **Enhance UserScript to get a list of files that match the run times instead of using a wildcard** (`#1002 `_) - - * ExtractTiles: - - * Enhance ExtractTiles using MTD input to properly match times (`#1285 `_) - - * TCMPRPlotter: - - * Improvements to TCMPRPlotter wrapper logging and output control (`#926 `_) - * Add option to TCMPRPlotter to pass in directory to tc_stat instead of individual files (`#1057 `_) - * Add option to pass in the input directory to TCMPRPlotter instead of finding all tcst files and passing the list (`#1084 `_) - - * CyclonePlotter: - - * Update CyclonePlotter for offline/HPC usage (`#933 `_) - * CyclonePlotter, create options to format output grid area to user-desired area (`#1091 `_) - * CyclonePlotter, connected lines run over the Prime Meridian (`#1000 `_) - - * Use Cases: - - * Add stat_analysis to the Blocking and Weather Regime processing (`#1001 `_) - * Modify user diagnostic feature relative use case to use MetPy Python package (`#759 `_) - * Reorganize the Cryosphere and Marine and Coastal use case categories into one group (`#1200 `_) - * Add harmonic pre-processing to the RMM use case (`#1019 `_) - - -* New Wrappers: - - * GenEnsProd (`#1180 `_, `#1266 `_) - * GFDLTracker (`#615 `_) - * IODA2NC (`#1203 `_) - - -* New Use Cases: - - * MET Tool Wrapper: - - * **PointStat: Python Embedding for Point Observations** (`#1490 `_) - * IODA2NC (`#1204 `_) - * GenEnsProd (`#1180 `_, `#1266 `_) - * GFDLTracker for TropicalCyclone (`#615 `_) - * GFDLTracker for TC Genesis (`#616 `_) - * GFDLTracker for Extra-TC Tracking (`#617 `_) - - - * Marine and Cryosphere: - - * GridStat_fcstRTOFS_obsOSTIA_iceCover (`#834 `_) - * Satellite verification of sea surface temperature (GHRSST) against RTOFS output (`#1004 `_) - * Satellite verification of sea surface salinity: SMOS vs RTOFS output (`#1116 `_) - * Satellite verification of sea surface salinity: AVISO vs RTOFS output HYCOM climo (`#1318 `_) - * Satellite verification of sea surface salinity: SMAP vs RTOFS output (`#1216 `_) - - - * Medium Range: - - * Feature Relative using MTD output for feature centroid lat/lon (`#641 `_) - - - * Precipitation: - - * Precipitation-type comparison across 3 models (`#1408 `_) - - - * Seasonal to Subseasonal (S2S): - - * UserScript_fcstGFS_obsERA_OMI (`#892 `_) - * UserScript_fcstGFS_obsERA_PhaseDiagram (`#1019 `_) - * UserScript_fcstGFS_obsERA_RMM (`#892 `_) - * RMM and OMI (driver scripts) (`#892 `_) - - - * Tropical Cyclone and Extra Tropical Cyclone (tc_and_extra_tc): - - * TC Verification Compare ADECK vs BDECK (`#911 `_) - * TCGen Verify Deterministic Genesis Forecasts and Probabilities from ATCF e-deck files (`#1274 `_) - + * **Enhance MODE wrapper to support multi-variate MODE** (`#1585 `_) + * **Allow FCST_IS_PROB variable setting specific to tool (FCST__IS_PROB)** (`#1586 `_) + * **Enhance climatology field settings to be consistent with fcst/obs field** (`#1599 `_) + * Update Hovmoeller Use case to use updated Hovmoeller plotting (`#1650 `_) * Bugfixes: - * Fix read of PB2NC_FILE_WINDOW_[BEGIN/END] configuration variables (`#1486 `_) - * Fix use of current field info in output prefix when using process list instances (`#1471 `_) - * Fix logic to create instances of other wrappers within wrappers to avoid modifying global configurations (`#1356 `_) - + * Add support for the {custom} loop string in the MODEL config variable (`#1382 `_) + * Fix PCPCombine extra options removal of semi-colon (`#1534 `_) + * Fix reset of arguments for some wrappers (i.e. GenEnsProd) after each run (`#1555 `_) + * Enhance METDbLoad Wrapper to find MODE .txt files (`#1608 `_) + * Add missing brackets around list variable values for StatAnalysis wrapper (`#1641 `_) + * Allow NA value for _CLIMO_[MEAN/STDEV]_DAY_INTERVAL (`#1653 `_) -* Documentation: +* New Wrappers: None - * Add list of METplus statistics to documentation (`#1049 `_) - * Update documentation to reference GitHub Discussions instead of MET Help (`#956 `_) - * Fix installation instructions in User's Guide (`#1067 `_) - * Add instructions to update old METplus configuration files that reference user-defined wrapped MET config files (`#1147 `_) - -* Internal: +* New Use Cases: None - * Improve approach to obtain additional python packages needed for some use cases (`#839 `_) - * Make updates to the Release Guide (`#935 `_) - * Clean up GitHub wiki broken links and out-of-date information (`#237 `_) - * Add option to override MET version used for automated tests (`#936 `_) - * Transition Community and Developer Support to Github Discussions (`#932 `_) - * Add documentation about the Release Guide and Verification Datasets Guide (`#874 `_) - * Create guidance for memory-intensive use cases, introduce Python memory profiler (`#1183 `_) - * Identify code throughout METplus components that are common utilities (`#799 `_) - * Add definitions to the Release Guide for the stages of the release cycle (`#934 `_) - * Document Continous Integration Functionality in the METplus Contributor's Guide (`#675 `_) - * Update Contributor's Guide for new removing/adding data protocols (`#1227 `_) - * Add recording of Python packages to Adding Use Cases documentation (`#1374 `_) - * Remove public-facing access to outdated use case categories (Cryosphere, marine_and_coastal) (`#1226 `_) - - -METplus Version 4.0.0 Release Notes (2021-05-10) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -* Bugfixes: - - * **Changed default values in wrapped MET config files to align with actual default values in MET config files** (:ref:`reconcile_default_values`) - * Fix bug causing GridStat fatal error (`#740 `_) - * Add support for comparing inputs using a mix of python embedding and non-embedding (`#684 `_) - * Fix quick search links (`#687 `_) - * Align the user guide with get_relativedelta() in time_util.py (`#579 `_) - * Fix CyclonePlotter cartopy mapping issues (`#850 `_, `#803 `_) - -* Enhancements: - - * **Rename master_metplus.py script to run_metplus.py** (`#794 `_) - * **Update setting of environment variables for MET config files to add support for all to METPLUS\_ vars** (`#768 `_) - * **Add support for many commonly changed MET config variables** (`#779 `_, `#755 `_, `#621 `_, `#620 `_) - * **Add support for a UserScript wrapper** (`#723 `_) - * **Create use case subdirectories** (`#751 `_) - * **Implement [INIT/VALID]EXCLUDE for time looping** (`#307 `_) - * **Add files to allow installation of METplus wrappers as a Python package (beta)** (`#282 `_) - * Generate PDF of User's Guide (`#551 `_) - * Add support for MET tc_gen changes in METplus (`#871 `_, (`#801 `_) - * Add support for 2 fields with same name and different levels in SeriesBy cases (`#852 `_) - * Enhance PCPCombine wrapper to be able to process multiple fields in one command (`#718 `_) - * Update TCStat config options and wrappers to filter data by excluding strings (`#857 `_) - * Support METplus to run from a driver script (`#569 `_) - * Refactor field info parsing to read once then substitute time info for each run time (`#880 `_) - * Enhance Python embedding logic to allow multiple level values (`#719 `_) - * Enhance Python embedding logic to allow multiple fcst and obs variable levels (`#708 `_) - * Add support for a group of files covering multiple run times for a single analysis in GridDiag (`#733 `_) - * Enhance ascii2nc python embedding script for TC dropsonde data (`#734 `_, `#731 `_) - * Support additional configuration variables in EnsembleStat (`#748 `_) - * Ensure backwards compatibility for MET config environment variables (`#760 `_) - * Combine configuration file sections into single config section (`#777 `_) - * Add support for skipping existing output files for all wrappers (`#711 `_) - * Add support for multiple instance of the same tool in the process list (`#670 `_) - * Add GFDL build support in build_components (`#614 `_) - * Decouple PCPCombine, RegridDataPlane, and GridStat wrappers behavior (`#602 `_) - * StatAnalysis run without filtering or config file (`#625 `_) - * Enhance User Diagnostic Feature Relative use case to Run Multiple Diagnostics (`#536 `_) - * Enhance PyEmbedIngest to run RegridDataPlane over Multiple Fields in One Call (`#549 `_) - * Filename templates that have other arguments besides a filename for python embedding fails (`#581 `_) - * Add more logging to tc_gen_wrapper (`#576 `_) - * Prevent crash when improperly formatted filename template is used (`#674 `_) - -* New Wrappers: - - * **PlotDataPlane** - * **UserScript** - * **METdbLoad** - -* New Use Cases: +* Documentation: - * Air Quality and Comp: EnsembleStat_fcstICAP_obsMODIS_aod - * Medium Range: UserScript_fcstGEFS_Difficulty_Index - * Convection Allowing Models: MODE_fcstFV3_obsGOES_BrightnessTemp - * Convection Allowing Models: MODE_fcstFV3_obsGOES_BrightnessTempObjs - * Convection Allowing Models: GridStat_fcstFV3_obsGOES_BrightnessTempDmap - * Data Assimilation: StatAnalysis_fcstHAFS_obsPrepBufr_JEDI_IODA_interface - * Medium Range: SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead_PyEmbed_Multiple_Diagnostics - * Precipitation: EnsembleStat_fcstWOFS_obsWOFS - * Seasonal to Subseasonal: TCGen_fcstGFSO_obsBDECKS_GDF_TDF - * Seasonal to Subseasonal: UserScript_fcstGFS_obsERA_Blocking - * Seasonal to Subseasonal: UserScript_obsERA_obsOnly_Blocking - * Seasonal to Subseasonal: UserScript_obsERA_obsOnly_WeatherRegime - * Seasonal to Subseasonal: UserScript_obsPrecip_obsOnly_Hovmoeller - * Seasonal to Subseasonal: UserScript_obsPrecip_obsOnly_CrossSpectraPlot - * TC and Extra TC: CyclonePlotter_fcstGFS_obsGFS_OPC - * TC and Extra TC: UserScript_ASCII2NC_PointStat_fcstHAFS_obsFRD_NetCDF - * TC and Extra TC: GridStat_fcstHAFS_obsTDR_NetCDF - * Marine and Coastal: PlotDataPlane_obsHYCOM_coordTripolar - * MET Tool Wrapper: METdbLoad/METdbLoad - * MET Tool Wrapper: PlotDataPlane/PlotDataPlane_grib1 - * MET Tool Wrapper: PlotDataPlane/PlotDataPlane_netcdf - * MET Tool Wrapper: PlotDataPlane/PlotDataPlane_python_embedding - * MET Tool Wrapper: GridStat/GridStat_python_embedding - * MET Tool Wrapper: PointStat/PointStat_python_embedding - * MET Tool Wrapper: MODE/MODE_python_embedding - * MET Tool Wrapper: PyEmbedIngest_multi_field_one_file + * Update documentation to include instructions to disable UserScript wrapper (`dtcenter/METplus-Internal#33 `_) * Internal: - * Append semi-colon to end of _OPTIONS variables if not found (`#707 `_) - * Ensure all wrappers follow the same conventions (`#76 `_) - * Refactor SeriesBy and ExtractTiles wrappers (`#310 `_) - * Refactor SeriesByLead wrapper (`#671 `_, `#76 `_) - * Add the pull request approval process steps to the Contributor's Guide (`#429 `_) - * Remove jlogger and postmsg (`#470 `_) - * Add unit tests for set_met_config_X functions in CommandBuilder (`#682 `_) - * Define a common set of GitHub labels that apply to all of the METplus component repos (`#690 `_) - * Transition from using Travis CI to GitHub Actions (`#721 `_) - * Improve workflow formatting in Contributers Guide (`#688 `_) - * Change INPUT_BASE to optional (`#679 `_) - * Refactor TCStat and ExtractTiles wrappers to conform to current standards - * Automate release date (`#665 `_) - * Add documentation for input verification datasets (`#662 `_) - * Add timing tests for Travis/Docker (`#649 `_) - * Set up encrypted credentials in Travis to push to DockerHub (`#634 `_) - * Add to User's Guide: using environment variables in METplus configuration files (`#594 `_) - * Cleanup version info (`#651 `_) - * Fix Travis tests for pull requests from forks (`#659 `_) - * Enhance the build_docker_images.sh script to support TravisCI updates (`#636 `_) - * Reorganize use case tests so users can add new cases easily (`#648 `_) - * Investigate how to add version selector to documentation (`#653 `_) - * Docker push pull image repository (`#639 `_) - * Tutorial Proofreading (`#534 `_) - * Update METplus data container logic to pull tarballs from dtcenter.org instead of GitHub release assets (`#613 `_) - * Convert Travis Docker files (automated builds) to use Dockerhub data volumes instead of tarballs (`#597 `_) - * Migrate from travis-ci.org to travis-ci.com (`#618 `_) - * Migrate Docker run commands to the METplus ci/jobs scripts/files (`#607 `_) - * Add stage to Travis to update or create data volumes when new sample data is available (`#633 `_) - * Docker data caching (`#623 `_) - * Tutorial testing on supported platforms (`#468 `_) - * Add additional Branch support to the Travis CI pipeline (`#478 `_) - * Change $DOCKER_WORK_DIR from /metplus to /root to be consistent with METplus tutorial (`#595 `_) - * Add all use_cases to automated tests (eg Travis) (`#571 `_) - * Add support to run METplus tests against multiple version of Python (`#483 `_) - * Enhanced testing to use Docker data volumes to supply truth data for output comparisons (`#567 `_) - * Update manage externals for beta5 versions (`#832 `_) - * Create a new METplus GitHub issue template for "New Use Case" (`#726 `_) + * Document GitHub Discussions procedure for the Contributor's Guide (`#1159 `_) + * Create a METplus "Release Guide" describing how to build releases for the METplus components (`#673 `_) + * Update documentation about viewing RTD URLs on branches (`#1512 `_) diff --git a/docs/Users_Guide/systemconfiguration.rst b/docs/Users_Guide/systemconfiguration.rst index 96f5061ca..eb78d81b0 100644 --- a/docs/Users_Guide/systemconfiguration.rst +++ b/docs/Users_Guide/systemconfiguration.rst @@ -2386,6 +2386,18 @@ obs_raw_plot.color_table | METplus Config: | :term:`MODE_MET_CONFIG_OVERRIDES` = obs_raw_plot = {color_table = "MET_BASE/colortables/mode_raw.ctable";} | +------------------+------------------------------------------------------------------------------------------------------------+ +mask_missing_flag +^^^^^^^^^^^^^^^^^ + ++------------------+---------------------------------------+ +| Old (Incorrect): | mask_missing_flag = BOTH; | ++------------------+---------------------------------------+ +| New (Correct): | mask_missing_flag = NONE; | ++------------------+---------------------------------------+ +| METplus Config: | :term:`MODE_MASK_MISSING_FLAG` = BOTH | ++------------------+---------------------------------------+ + + PB2NCConfig ----------- diff --git a/docs/Users_Guide/wrappers.rst b/docs/Users_Guide/wrappers.rst index 091549060..86fc2052c 100644 --- a/docs/Users_Guide/wrappers.rst +++ b/docs/Users_Guide/wrappers.rst @@ -231,6 +231,9 @@ METplus Configuration | :term:`ENSEMBLE_STAT_SKIP_CONST` | :term:`ENSEMBLE_STAT_OBS_ERROR_FLAG` | :term:`ENSEMBLE_STAT_CLIMO_MEAN_FILE_NAME` +| :term:`ENSEMBLE_STAT_CLIMO_MEAN_VAR_NAME` +| :term:`ENSEMBLE_STAT_CLIMO_MEAN_VAR_LEVELS` +| :term:`ENSEMBLE_STAT_CLIMO_MEAN_VAR_OPTIONS` | :term:`ENSEMBLE_STAT_CLIMO_MEAN_FIELD` | :term:`ENSEMBLE_STAT_CLIMO_MEAN_REGRID_METHOD` | :term:`ENSEMBLE_STAT_CLIMO_MEAN_REGRID_WIDTH` @@ -243,6 +246,9 @@ METplus Configuration | :term:`ENSEMBLE_STAT_CLIMO_MEAN_USE_FCST` | :term:`ENSEMBLE_STAT_CLIMO_MEAN_USE_OBS` | :term:`ENSEMBLE_STAT_CLIMO_STDEV_FILE_NAME` +| :term:`ENSEMBLE_STAT_CLIMO_STDEV_VAR_NAME` +| :term:`ENSEMBLE_STAT_CLIMO_STDEV_VAR_LEVELS` +| :term:`ENSEMBLE_STAT_CLIMO_STDEV_VAR_OPTIONS` | :term:`ENSEMBLE_STAT_CLIMO_STDEV_FIELD` | :term:`ENSEMBLE_STAT_CLIMO_STDEV_REGRID_METHOD` | :term:`ENSEMBLE_STAT_CLIMO_STDEV_REGRID_WIDTH` @@ -1118,6 +1124,9 @@ METplus Configuration | :term:`GEN_ENS_PROD_NMEP_SMOOTH_METHOD` | :term:`GEN_ENS_PROD_NMEP_SMOOTH_WIDTH` | :term:`GEN_ENS_PROD_CLIMO_MEAN_FILE_NAME` +| :term:`GEN_ENS_PROD_CLIMO_MEAN_VAR_NAME` +| :term:`GEN_ENS_PROD_CLIMO_MEAN_VAR_LEVELS` +| :term:`GEN_ENS_PROD_CLIMO_MEAN_VAR_OPTIONS` | :term:`GEN_ENS_PROD_CLIMO_MEAN_FIELD` | :term:`GEN_ENS_PROD_CLIMO_MEAN_REGRID_METHOD` | :term:`GEN_ENS_PROD_CLIMO_MEAN_REGRID_WIDTH` @@ -1130,6 +1139,9 @@ METplus Configuration | :term:`GEN_ENS_PROD_CLIMO_MEAN_USE_FCST` | :term:`GEN_ENS_PROD_CLIMO_MEAN_USE_OBS` | :term:`GEN_ENS_PROD_CLIMO_STDEV_FILE_NAME` +| :term:`GEN_ENS_PROD_CLIMO_STDEV_VAR_NAME` +| :term:`GEN_ENS_PROD_CLIMO_STDEV_VAR_LEVELS` +| :term:`GEN_ENS_PROD_CLIMO_STDEV_VAR_OPTIONS` | :term:`GEN_ENS_PROD_CLIMO_STDEV_FIELD` | :term:`GEN_ENS_PROD_CLIMO_STDEV_REGRID_METHOD` | :term:`GEN_ENS_PROD_CLIMO_STDEV_REGRID_WIDTH` @@ -2917,6 +2929,9 @@ METplus Configuration | :term:`FCST_GRID_STAT_FILE_TYPE` | :term:`OBS_GRID_STAT_FILE_TYPE` | :term:`GRID_STAT_CLIMO_MEAN_FILE_NAME` +| :term:`GRID_STAT_CLIMO_MEAN_VAR_NAME` +| :term:`GRID_STAT_CLIMO_MEAN_VAR_LEVELS` +| :term:`GRID_STAT_CLIMO_MEAN_VAR_OPTIONS` | :term:`GRID_STAT_CLIMO_MEAN_FIELD` | :term:`GRID_STAT_CLIMO_MEAN_REGRID_METHOD` | :term:`GRID_STAT_CLIMO_MEAN_REGRID_WIDTH` @@ -2929,6 +2944,9 @@ METplus Configuration | :term:`GRID_STAT_CLIMO_MEAN_USE_FCST` | :term:`GRID_STAT_CLIMO_MEAN_USE_OBS` | :term:`GRID_STAT_CLIMO_STDEV_FILE_NAME` +| :term:`GRID_STAT_CLIMO_STDEV_VAR_NAME` +| :term:`GRID_STAT_CLIMO_STDEV_VAR_LEVELS` +| :term:`GRID_STAT_CLIMO_STDEV_VAR_OPTIONS` | :term:`GRID_STAT_CLIMO_STDEV_FIELD` | :term:`GRID_STAT_CLIMO_STDEV_REGRID_METHOD` | :term:`GRID_STAT_CLIMO_STDEV_REGRID_WIDTH` @@ -4110,6 +4128,7 @@ METplus Configuration | :term:`MODE_NC_PAIRS_FLAG_OBJECT_ID` | :term:`MODE_NC_PAIRS_FLAG_CLUSTER_ID` | :term:`MODE_NC_PAIRS_FLAG_POLYLINES` +| :term:`MODE_MASK_MISSING_FLAG` | :term:`MODE_MATCH_FLAG` | :term:`MODE_MAX_CENTROID_DIST` | :term:`MODE_TOTAL_INTEREST_THRESH` @@ -4120,6 +4139,7 @@ METplus Configuration | :term:`MODE_CT_STATS_FLAG` | :term:`FCST_MODE_IS_PROB` | :term:`FCST_MODE_PROB_IN_GRIB_PDS` +| :term:`MODE_MULTIVAR_LOGIC` | :term:`FCST_MODE_VAR_NAME` (optional) | :term:`FCST_MODE_VAR_LEVELS` (optional) | :term:`FCST_MODE_VAR_THRESH` (optional) @@ -4232,6 +4252,17 @@ see :ref:`How METplus controls MET config file settings`. * - :term:`MODE_QUILT` - quilt +**${METPLUS_MULTIVAR_LOGIC}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`MODE_MULTIVAR_LOGIC` + - multivar_logic + **${METPLUS_FCST_FIELD}** .. list-table:: @@ -4541,6 +4572,17 @@ see :ref:`How METplus controls MET config file settings`. * - :term:`MODE_MASK_POLY_FLAG` - mask.poly_flag +**${METPLUS_MASK_MISSING_FLAG}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`MODE_MASK_MISSING_FLAG` + - mask_missing_flag + **${METPLUS_MATCH_FLAG}** .. list-table:: @@ -5432,6 +5474,9 @@ Configuration | :term:`POINT_STAT_INTERP_TYPE_METHOD` | :term:`POINT_STAT_INTERP_TYPE_WIDTH` | :term:`POINT_STAT_CLIMO_MEAN_FILE_NAME` +| :term:`POINT_STAT_CLIMO_MEAN_VAR_NAME` +| :term:`POINT_STAT_CLIMO_MEAN_VAR_LEVELS` +| :term:`POINT_STAT_CLIMO_MEAN_VAR_OPTIONS` | :term:`POINT_STAT_CLIMO_MEAN_FIELD` | :term:`POINT_STAT_CLIMO_MEAN_REGRID_METHOD` | :term:`POINT_STAT_CLIMO_MEAN_REGRID_WIDTH` @@ -5444,6 +5489,9 @@ Configuration | :term:`POINT_STAT_CLIMO_MEAN_USE_FCST` | :term:`POINT_STAT_CLIMO_MEAN_USE_OBS` | :term:`POINT_STAT_CLIMO_STDEV_FILE_NAME` +| :term:`POINT_STAT_CLIMO_STDEV_VAR_NAME` +| :term:`POINT_STAT_CLIMO_STDEV_VAR_LEVELS` +| :term:`POINT_STAT_CLIMO_STDEV_VAR_OPTIONS` | :term:`POINT_STAT_CLIMO_STDEV_FIELD` | :term:`POINT_STAT_CLIMO_STDEV_REGRID_METHOD` | :term:`POINT_STAT_CLIMO_STDEV_REGRID_WIDTH` @@ -6069,6 +6117,9 @@ METplus Configuration | :term:`SERIES_ANALYSIS_TC_STAT_INPUT_TEMPLATE` | :term:`SERIES_ANALYSIS_OUTPUT_TEMPLATE` | :term:`SERIES_ANALYSIS_CLIMO_MEAN_FILE_NAME` +| :term:`SERIES_ANALYSIS_CLIMO_MEAN_VAR_NAME` +| :term:`SERIES_ANALYSIS_CLIMO_MEAN_VAR_LEVELS` +| :term:`SERIES_ANALYSIS_CLIMO_MEAN_VAR_OPTIONS` | :term:`SERIES_ANALYSIS_CLIMO_MEAN_FIELD` | :term:`SERIES_ANALYSIS_CLIMO_MEAN_REGRID_METHOD` | :term:`SERIES_ANALYSIS_CLIMO_MEAN_REGRID_WIDTH` @@ -6082,6 +6133,9 @@ METplus Configuration | :term:`SERIES_ANALYSIS_CLIMO_MEAN_USE_FCST` | :term:`SERIES_ANALYSIS_CLIMO_MEAN_USE_OBS` | :term:`SERIES_ANALYSIS_CLIMO_STDEV_FILE_NAME` +| :term:`SERIES_ANALYSIS_CLIMO_STDEV_VAR_NAME` +| :term:`SERIES_ANALYSIS_CLIMO_STDEV_VAR_LEVELS` +| :term:`SERIES_ANALYSIS_CLIMO_STDEV_VAR_OPTIONS` | :term:`SERIES_ANALYSIS_CLIMO_STDEV_FIELD` | :term:`SERIES_ANALYSIS_CLIMO_STDEV_REGRID_METHOD` | :term:`SERIES_ANALYSIS_CLIMO_STDEV_REGRID_WIDTH` diff --git a/docs/_static/s2s-SeriesAnalysis_fcstCFSv2_obsGHCNCAMS_climoStandardized_MultiStatisticTool.png b/docs/_static/s2s-SeriesAnalysis_fcstCFSv2_obsGHCNCAMS_climoStandardized_MultiStatisticTool.png new file mode 100644 index 000000000..4de9c9f44 Binary files /dev/null and b/docs/_static/s2s-SeriesAnalysis_fcstCFSv2_obsGHCNCAMS_climoStandardized_MultiStatisticTool.png differ diff --git a/docs/use_cases/model_applications/s2s/SeriesAnalysis_fcstCFSv2_obsGHCNCAMS_climoStandardized_MultiStatisticTool.py b/docs/use_cases/model_applications/s2s/SeriesAnalysis_fcstCFSv2_obsGHCNCAMS_climoStandardized_MultiStatisticTool.py new file mode 100644 index 000000000..7be759ada --- /dev/null +++ b/docs/use_cases/model_applications/s2s/SeriesAnalysis_fcstCFSv2_obsGHCNCAMS_climoStandardized_MultiStatisticTool.py @@ -0,0 +1,163 @@ +""" +SeriesAnalysis: Standardize ensemble members and calculate probabilistic outputs +================================================================================ + +model_applications/s2s/SeriesAnalysis_fcstCFSv2_obsGHCNCAMS_climoStandardized_MultiStatisticTool.conf + +""" +############################################################################## +# Scientific Objective +# -------------------- +# This use case ingests a CFSv2 Ensemble forecast, with all ensemble members in a single file for a given year. +# 29 years of forecast ensembles are used to create climatologies for each ensemble member. These climatologies +# are then used to normalize each ensemble member via the Gen-Ens-Prod tool, allowing a meaningful comparison to +# the observation dataset, which is presented as normalized. The forecast to observation verification are completed across both the temporal and spatial. +# This use case highlights several important features within METplus; in particular, how to create climatologies for ensemble members using SeriesAnalysis, +# how those climatologies can be used by GenEnsProd to normalize each ensemble member to its corresponding climatology, +# and calculating probabilistic verfication on s2s data, which is a frequent request from climatological centers. + +############################################################################## +# Datasets +# --------------------- +# +# | **Forecast:** 29 CFSv2 Ensemble files, 2m temperature fields +# +# | **Observations:** GHCNCAMS, 2m temperature field +# +# +# | **Location:** All of the input data required for this use case can be found in the met_test sample data tarball. Click here to the METplus releases page and download sample data for the appropriate release: https://github.com/dtcenter/METplus/releases +# | This tarball should be unpacked into the directory that you will set the value of INPUT_BASE. See `Running METplus`_ section for more information. +# +# | **Data Source:** CPC + +############################################################################## +# METplus Components +# ------------------ +# +# This use case initially runs SeriesAnalysis 24 times, once for each member of the CFSv2 ensemble, across the entire 29 years for forecast data. +# The resulting 24 outputs are read in by GenEnsProd, which is called 29 times (once for each year). GenEnsProd uses the **normalize** option +# and the SeriesAnalysis outputs to normalize each of the ensemble members relative to its climatology (FBAR) and standard deviation (FSTDEV). +# The output from GenEnsProd are 29 files containing the uncalibrated probability forecasts for the lower tercile of January for each year. +# The final probability verification is done across the temporal scale in SeriesAnalysis, and the spatial scale in GridStat. + +############################################################################## +# METplus Workflow +# ---------------- +# +# This use case utilizes 29 years of forecast data, with 24 members in each ensemble forecast. +# The following boundary times are used for the entire script: +# +# | **Init Beg:** 1982-01-01 +# | **Init End:** 2010-01-02 +# +# Because the increment is 1 year, all January 1st from 1982 to 2010 are processed for a total of 29 years. +# + +############################################################################## +# METplus Configuration +# --------------------- +# +# METplus first loads all of the configuration files found in parm/metplus_config, +# then it loads any configuration files passed to METplus via the command line +# i.e. -c parm/use_cases/model_applications/s2s/SeriesAnalysis_fcstCFSv2_obsGHCNCAMS_climoStandardized_MultiStatisticTool.conf +# +# .. highlight:: bash +# .. literalinclude:: ../../../../parm/use_cases/model_applications/s2s/SeriesAnalysis_fcstCFSv2_obsGHCNCAMS_climoStandardized_MultiStatisticTool.conf +# + +############################################################################## +# MET Configuration +# --------------------- +# +# METplus sets environment variables based on the values in the METplus configuration file. These variables are referenced in the MET configuration file. **YOU SHOULD NOT SET ANY OF THESE ENVIRONMENT VARIABLES YOURSELF! THEY WILL BE OVERWRITTEN BY METPLUS WHEN IT CALLS THE MET TOOLS!** If there is a setting in the MET configuration file that is not controlled by an environment variable, you can add additional environment variables to be set only within the METplus environment using the [user_env_vars] section of the METplus configuration files. See the ‘User Defined Config’ section on the ‘System Configuration’ page of the METplus User’s Guide for more information. +# +# .. highlight:: bash +# .. literalinclude:: ../../../../parm/met_config/SeriesAnalysisConfig_wrapped +# .. literalinclude:: ../../../../parm/met_config/GenEnsProdConfig_wrapped +# .. literalinclude:: ../../../../parm/met_config/GridStatConfig_wrapped +# + +############################################################################## +# Running METplus +# --------------- +# +# This use case can be run two ways: +# +# 1) Passing in SeriesAnalysis_fcstCFSv2_obsGHCNCAMS_climoStandardized_MultiStatisticTool.conf then a user-specific system configuration file:: +# +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/s2s/SeriesAnalysis_fcstCFSv2_obsGHCNCAMS_climoStandardized_MultiStatisticTool.conf /path/to/user_system.conf +# +# 2) Modifying the configurations in parm/metplus_config, then passing in SeriesAnalysis_fcstCFSv2_obsGHCNCAMS_climoStandardized_MultiStatisticTool.conf:: +# +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/marine_and_cryosphere/SeriesAnalysis_fcstCFSv2_obsGHCNCAMS_climoStandardized_MultiStatisticTool.conf +# +# The former method is recommended. Whether you add them to a user-specific configuration file or modify the metplus_config files, the following variables must be set correctly: +# +# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). This is not required to run METplus, but it is required to run the examples in parm/use_cases +# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions +# * **MET_INSTALL_DIR** - Path to location where MET is installed locally +# +# Example User Configuration File:: +# +# [config] +# INPUT_BASE = /path/to/sample/input/data +# OUTPUT_BASE = /path/to/output/dir +# MET_INSTALL_DIR = /path/to/met-X.Y +# +# + +############################################################################## +# Expected Output +# --------------- +# +# A successful run will output the following both to the screen and to the logfile:: +# +# INFO: METplus has successfully finished running. +# +# Refer to the value set for **OUTPUT_BASE** to find where the output data was generated. +# Output for use case will be found in 4 distinct folders (relative to **OUTPUT_BASE**). +# The output from the first SeriesAnalysis call goes to **SA_run1** will contain the following files: +# +# * mem??_output.nc +# +# where ?? will be replaced by values corresponding to each of the ensemble members (0 through 23). +# The output for GenEnsProd goes into **GEP** and contains the following files: +# +# * gen_ens_prod_YYYY01_ens.nc +# +# where YYYY will be replaced by each year of the forecast data being processed (1982 through 2010). +# The output from the second SeriesAnalysis call goes to **SA_run2** and contains the following files: +# +# * 198201to201002_CFSv2_SA.nc +# +# Finally, the output from GridStat will be in **GridStat** and will contain 29 folders of the following format: +# +# * ????01 +# +# where ???? will correspond to each year of the forecast data being processed (1982 through 2010). +# Each of those folders will have the following files: +# +# * grid_stat_198201_000000L_19700101_000000V_pairs.nc +# * grid_stat_198201_000000L_19700101_000000V_pstd.txt +# * grid_stat_198201_000000L_19700101_000000V.stat +# + +############################################################################## +# Keywords +# -------- +# +# .. note:: +# +# * SeriesAnalysisUseCase +# * GenEnsProdUseCase +# * GridStatUseCase +# * ProbabilityVerificationUseCase +# * S2SAppUseCase +# * NETCDFFileUseCase +# +# Navigate to the :ref:`quick-search` page to discover other similar use cases. +# +# +# +# sphinx_gallery_thumbnail_path = '_static/s2s-SeriesAnalysis_fcstCFSv2_obsGHCNCAMS_climoStandardized_MultiStatisticTool.png' + diff --git a/docs/use_cases/model_applications/s2s/UserScript_obsPrecip_obsOnly_Hovmoeller.py b/docs/use_cases/model_applications/s2s/UserScript_obsPrecip_obsOnly_Hovmoeller.py index a72af7258..cbc5031dd 100644 --- a/docs/use_cases/model_applications/s2s/UserScript_obsPrecip_obsOnly_Hovmoeller.py +++ b/docs/use_cases/model_applications/s2s/UserScript_obsPrecip_obsOnly_Hovmoeller.py @@ -30,6 +30,15 @@ # This use case runs the UserScript wrapper tool to run a user provided script, # in this case, hovmoeller.py. # +# It also requires the METcalcpy and METplotpy source code to generate the plot. +# Clone the METcalcpy repository (https://github.com/dtcenter/METcalcpy) and the +# METplotpy repository (https://github.com/dtcenter/METplotpy) under the same base +# directory as the METPLUS_BASE directory so that the METplotpy, METcalcpy, and +# METplotpy directories are under the same base directory (i.e. if the METPLUS_BASE directory is +# /home/username/working/METplus, then clone the METcalcpy and METplotpy source +# code into the /home/username/working directory). +# + ############################################################################## # METplus Workflow @@ -48,7 +57,7 @@ # # METplus first loads all of the configuration files found in parm/metplus_config, # then it loads any configuration files passed to METplus via the command line -# with the -c option, i.e. -c parm/use_cases/model_applications/s2s/UserScript_obsPrecip_obsOnly_Hovmoeller.conf +# parm/use_cases/model_applications/s2s/UserScript_obsPrecip_obsOnly_Hovmoeller.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/s2s/UserScript_obsPrecip_obsOnly_Hovmoeller.conf @@ -78,14 +87,14 @@ # then a user-specific system configuration file:: # # run_metplus.py \ -# -c /path/to/METplus/parm/use_cases/model_applications/s2s/UserScript_obsPrecip_obsOnly_Hovmoeller.conf \ -# -c /path/to/user_system.conf +# /path/to/METplus/parm/use_cases/model_applications/s2s/UserScript_obsPrecip_obsOnly_Hovmoeller.conf \ +# /path/to/user_system.conf # # 2) Modifying the configurations in parm/metplus_config, then passing in UserScript_obsPrecip_obsOnly_Hovmoeller.conf:: # # run_metplus.py \ -# -c /path/to/METplus/parm/use_cases/model_applications/s2s/UserScript_obsPrecip_obsOnly_Hovmoeller.conf -# +# /path/to/METplus/parm/use_cases/model_applications/s2s/UserScript_obsPrecip_obsOnly_Hovmoeller.conf +# # The former method is recommended. Whether you add them to a user-specific configuration file or modify the metplus_config files, the following variables must be set correctly: # # * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). This is not required to run METplus, but it is required to run the examples in parm/use_cases diff --git a/internal_tests/pytests/command_builder/test_command_builder.py b/internal_tests/pytests/command_builder/test_command_builder.py index e0a79733a..ab2c62272 100644 --- a/internal_tests/pytests/command_builder/test_command_builder.py +++ b/internal_tests/pytests/command_builder/test_command_builder.py @@ -1,76 +1,12 @@ #!/usr/bin/env python3 -import os -import sys -import re -import logging -from collections import namedtuple -import produtil import pytest -import datetime -from metplus.wrappers.command_builder import CommandBuilder -from metplus.util import time_util -from metplus.util import METConfig +import os +import datetime -@pytest.mark.parametrize( - 'config_overrides, expected_value', [ - # 0 no climo variables set - ({}, ''), - # 1 file name set only - ({'FILE_NAME': '/mean/dir/gs_climo_{init?fmt=%Y%m%d%H}.tmpl'}, - '/mean/dir/gs_climo_{init?fmt=%Y%m%d%H}.tmpl'), - # 2 input template set - ({'INPUT_TEMPLATE': '/mean/dir/gs_climo_{init?fmt=%Y%m%d%H}.tmpl'}, - '/mean/dir/gs_climo_{init?fmt=%Y%m%d%H}.tmpl'), - # 3 input template and dir set - ({'INPUT_DIR': '/mean/dir', - 'INPUT_TEMPLATE': 'gs_climo_{init?fmt=%Y%m%d%H}.tmpl'}, - '/mean/dir/gs_climo_{init?fmt=%Y%m%d%H}.tmpl'), - # 4 input template and dir set multiple templates - ({'INPUT_DIR': '/mean/dir', - 'INPUT_TEMPLATE': 'gs_climo_1.tmpl, gs_climo_2.tmpl'}, - '/mean/dir/gs_climo_1.tmpl,/mean/dir/gs_climo_2.tmpl'), - # 5file name, input template and dir all set - ({'FILE_NAME': '/mean/dir/gs_climo_{init?fmt=%Y%m%d%H}.tmpl', - 'INPUT_DIR': '/mean/dir', - 'INPUT_TEMPLATE': 'gs_climo_1.tmpl, gs_climo_2.tmpl'}, - '/mean/dir/gs_climo_{init?fmt=%Y%m%d%H}.tmpl'), - # 6 input template is python embedding keyword and dir is set - ({'INPUT_DIR': '/mean/dir', - 'INPUT_TEMPLATE': 'PYTHON_NUMPY'}, - 'PYTHON_NUMPY'), - # 7 input template is python embedding keyword and dir is set - ({'INPUT_DIR': '/mean/dir', - 'INPUT_TEMPLATE': 'PYTHON_XARRAY'}, - 'PYTHON_XARRAY'), - ] -) -def test_read_climo_file_name(metplus_config, config_overrides, - expected_value): - # name of app used for testing to read/set config variables - app_name = 'grid_stat' - - # check mean and stdev climo variables - for climo_type in CommandBuilder.climo_types: - prefix = f'{app_name.upper()}_CLIMO_{climo_type.upper()}_' - - config = metplus_config() - - # set config values - for key, value in config_overrides.items(): - config.set('config', f'{prefix}{key}', value) - - cbw = CommandBuilder(config) - - # set app_name to grid_stat for testing - cbw.app_name = app_name - - cbw.read_climo_file_name(climo_type) - actual_value = cbw.config.getraw('config', - f'{prefix}FILE_NAME', - '') - assert actual_value == expected_value +from metplus.wrappers.command_builder import CommandBuilder +from metplus.util import ti_calculate # ------------------------ # test_find_data_no_dated @@ -93,7 +29,7 @@ def test_find_data_no_dated(metplus_config, data_type): task_info = {} task_info['valid'] = datetime.datetime.strptime("201802010000",'%Y%m%d%H%M') task_info['lead'] = 0 - time_info = time_util.ti_calculate(task_info) + time_info = ti_calculate(task_info) pcw.c_dict[f'{data_type}FILE_WINDOW_BEGIN'] = -3600 pcw.c_dict[f'{data_type}FILE_WINDOW_END'] = 3600 @@ -120,7 +56,7 @@ def test_find_data_not_a_path(metplus_config, data_type): task_info = {} task_info['valid'] = datetime.datetime.strptime("201802010000",'%Y%m%d%H%M') task_info['lead'] = 0 - time_info = time_util.ti_calculate(task_info) + time_info = ti_calculate(task_info) pcw.c_dict[f'{data_type}FILE_WINDOW_BEGIN'] = 0 pcw.c_dict[f'{data_type}FILE_WINDOW_END'] = 0 @@ -138,7 +74,7 @@ def test_find_obs_no_dated(metplus_config): task_info = {} task_info['valid'] = datetime.datetime.strptime("201802010000", '%Y%m%d%H%M') task_info['lead'] = 0 - time_info = time_util.ti_calculate(task_info) + time_info = ti_calculate(task_info) pcw.c_dict['OBS_FILE_WINDOW_BEGIN'] = -3600 pcw.c_dict['OBS_FILE_WINDOW_END'] = 3600 @@ -156,7 +92,7 @@ def test_find_obs_dated(metplus_config): task_info = {} task_info['valid'] = datetime.datetime.strptime("201802010000", '%Y%m%d%H%M') task_info['lead'] = 0 - time_info = time_util.ti_calculate(task_info) + time_info = ti_calculate(task_info) pcw.c_dict['OBS_FILE_WINDOW_BEGIN'] = -3600 pcw.c_dict['OBS_FILE_WINDOW_END'] = 3600 @@ -184,7 +120,7 @@ def test_find_obs_offset(metplus_config, offsets, expected_file, offset_seconds) task_info = {} task_info['valid'] = datetime.datetime.strptime("2020020112", '%Y%m%d%H') task_info['lead'] = 0 - time_info = time_util.ti_calculate(task_info) + time_info = ti_calculate(task_info) pcw.c_dict['OFFSETS'] = offsets pcw.c_dict['OBS_INPUT_DIR'] = pcw.config.getdir('METPLUS_BASE') + "/internal_tests/data/obs" @@ -209,7 +145,7 @@ def test_find_obs_dated_previous_day(metplus_config): task_info = {} task_info['valid'] = datetime.datetime.strptime("201802010000", '%Y%m%d%H%M') task_info['lead'] = 0 - time_info = time_util.ti_calculate(task_info) + time_info = ti_calculate(task_info) pcw.c_dict['OBS_INPUT_DIR'] = pcw.config.getdir('METPLUS_BASE')+"/internal_tests/data/obs" pcw.c_dict['OBS_INPUT_TEMPLATE'] = '{valid?fmt=%Y%m%d}/{valid?fmt=%Y%m%d}_{valid?fmt=%H%M}' @@ -227,7 +163,7 @@ def test_find_obs_dated_next_day(metplus_config): task_info = {} task_info['valid'] = datetime.datetime.strptime("201802012345", '%Y%m%d%H%M') task_info['lead'] = 0 - time_info = time_util.ti_calculate(task_info) + time_info = ti_calculate(task_info) pcw.c_dict['OBS_INPUT_DIR'] = pcw.config.getdir('METPLUS_BASE')+"/internal_tests/data/obs" pcw.c_dict['OBS_INPUT_TEMPLATE'] = '{valid?fmt=%Y%m%d}/{valid?fmt=%Y%m%d}_{valid?fmt=%H%M}' diff --git a/internal_tests/pytests/grid_stat/test_grid_stat_wrapper.py b/internal_tests/pytests/grid_stat/test_grid_stat_wrapper.py index 34385c23c..e29aeaae5 100644 --- a/internal_tests/pytests/grid_stat/test_grid_stat_wrapper.py +++ b/internal_tests/pytests/grid_stat/test_grid_stat_wrapper.py @@ -516,6 +516,9 @@ def test_handle_climo_file_variables(metplus_config, config_overrides, ({'GRID_STAT_CLIMO_MEAN_DAY_INTERVAL': '30', }, {'METPLUS_CLIMO_MEAN_DICT': 'climo_mean = {day_interval = 30;}'}), + ({'GRID_STAT_CLIMO_MEAN_DAY_INTERVAL': 'NA', }, + {'METPLUS_CLIMO_MEAN_DICT': 'climo_mean = {day_interval = NA;}'}), + ({'GRID_STAT_CLIMO_MEAN_HOUR_INTERVAL': '12', }, {'METPLUS_CLIMO_MEAN_DICT': 'climo_mean = {hour_interval = 12;}'}), diff --git a/internal_tests/pytests/met_config/test_met_config.py b/internal_tests/pytests/met_config/test_met_config.py index 1e2e5e5f3..0f990e678 100644 --- a/internal_tests/pytests/met_config/test_met_config.py +++ b/internal_tests/pytests/met_config/test_met_config.py @@ -3,6 +3,147 @@ import pytest from metplus.util.met_config import * +from metplus.util.met_config import _read_climo_file_name, _read_climo_field +from metplus.util import CLIMO_TYPES + +@pytest.mark.parametrize( + 'config_overrides, expected_value', [ + # 0 no relevant config set + ({}, ''), + # 1 _FIELD set + ({'APP_CLIMO__FIELD': '{name="TMP"; level="(*,*)";}'}, + '{name="TMP"; level="(*,*)";}'), + # 2 VAR1 name/level set + ({'APP_CLIMO__VAR1_NAME': 'TMP', + 'APP_CLIMO__VAR1_LEVELS': '"(*,*)"'}, + '{ name="TMP"; level="(*,*)"; }'), + # 3 VAR1/2 name/level set + ({'APP_CLIMO__VAR1_NAME': 'TMP', + 'APP_CLIMO__VAR1_LEVELS': '"(*,*)"', + 'APP_CLIMO__VAR2_NAME': 'PRES', + 'APP_CLIMO__VAR2_LEVELS': '"(0,*,*)"'}, + '{ name="TMP"; level="(*,*)"; },{ name="PRES"; level="(0,*,*)"; }'), + # 4 VAR1 name/level and FIELD set - prefer VAR + ({'APP_CLIMO__FIELD': '{name="TEMP"; level="(0,*,*)";}', + 'APP_CLIMO__VAR1_NAME': 'TMP', + 'APP_CLIMO__VAR1_LEVELS': '"(*,*)"'}, + '{ name="TMP"; level="(*,*)"; }'), + ] +) +def test_read_climo_field(metplus_config, config_overrides, expected_value): + app_name = 'app' + for climo_type in ('MEAN', 'STDEV'): + expected_var = f'{app_name}_CLIMO_{climo_type}_FIELD'.upper() + config = metplus_config() + + # set config values + for key, value in config_overrides.items(): + key_sub = key.replace('', climo_type) + value_sub = value.replace('', climo_type.lower()) + config.set('config', key_sub, value_sub) + + _read_climo_field(climo_type, config, app_name) + assert config.getraw('config', expected_var) == expected_value + +@pytest.mark.parametrize( + 'config_overrides, expected_value', [ + # 0 no relevant config set + ({}, ''), + # 1 file name single + ({'APP_CLIMO__FILE_NAME': 'some/file/path'}, + 'climo_ = {file_name = ["some/file/path"];}'), + # 2 file name multiple + ({'APP_CLIMO__FILE_NAME': 'some/file/path, other/path'}, + 'climo_ = {file_name = ["some/file/path", "other/path"];}'), + # 3 field single + ({'APP_CLIMO__FIELD': '{name="TMP"; level="(*,*)";}'}, + 'climo_ = {field = [{name="TMP"; level="(*,*)";}];}'), + # 4 field multiple + ({'APP_CLIMO__FIELD': ('{name="TMP"; level="(*,*)";},' + '{name="TEMP"; level="P500";}')}, + ('climo_ = {field = [{name="TMP"; level="(*,*)";}, ' + '{name="TEMP"; level="P500";}];}')), + # 5 use fcst no other climo_ + ({'APP_CLIMO__USE_FCST': 'TRUE'}, + 'climo_ = fcst;'), + # 6 use obs no other climo_ + ({'APP_CLIMO__USE_OBS': 'TRUE'}, + 'climo_ = obs;'), + # 7 use fcst with other climo_ + ({'APP_CLIMO__REGRID_METHOD': 'NEAREST', + 'APP_CLIMO__USE_FCST': 'TRUE'}, + 'climo_ = {regrid = {method = NEAREST;}}climo_ = fcst;'), + # 8 use obs with other climo_ + ({'APP_CLIMO__REGRID_METHOD': 'NEAREST', + 'APP_CLIMO__USE_OBS': 'TRUE'}, + 'climo_ = {regrid = {method = NEAREST;}}climo_ = obs;'), + # 9 regrid method + ({'APP_CLIMO__REGRID_METHOD': 'NEAREST', }, + 'climo_ = {regrid = {method = NEAREST;}}'), + # 10 regrid width + ({'APP_CLIMO__REGRID_WIDTH': '1', }, + 'climo_ = {regrid = {width = 1;}}'), + # 11 regrid vld_thresh + ({'APP_CLIMO__REGRID_VLD_THRESH': '0.5', }, + 'climo_ = {regrid = {vld_thresh = 0.5;}}'), + # 12 regrid shape + ({'APP_CLIMO__REGRID_SHAPE': 'SQUARE', }, + 'climo_ = {regrid = {shape = SQUARE;}}'), + # 13 time_interp_method + ({'APP_CLIMO__TIME_INTERP_METHOD': 'NEAREST', }, + 'climo_ = {time_interp_method = NEAREST;}'), + # 14 match_month + ({'APP_CLIMO__MATCH_MONTH': 'True', }, + 'climo_ = {match_month = TRUE;}'), + # 15 day_interval - int + ({'APP_CLIMO__DAY_INTERVAL': '30', }, + 'climo_ = {day_interval = 30;}'), + # 16 day_interval - NA + ({'APP_CLIMO__DAY_INTERVAL': 'NA', }, + 'climo_ = {day_interval = NA;}'), + # 17 hour_interval + ({'APP_CLIMO__HOUR_INTERVAL': '12', }, + 'climo_ = {hour_interval = 12;}'), + # 18 all + ({ + 'APP_CLIMO__FILE_NAME': '/some/climo_/file.txt', + 'APP_CLIMO__FIELD': '{name="CLM_NAME"; level="(0,0,*,*)";}', + 'APP_CLIMO__REGRID_METHOD': 'NEAREST', + 'APP_CLIMO__REGRID_WIDTH': '1', + 'APP_CLIMO__REGRID_VLD_THRESH': '0.5', + 'APP_CLIMO__REGRID_SHAPE': 'SQUARE', + 'APP_CLIMO__TIME_INTERP_METHOD': 'NEAREST', + 'APP_CLIMO__MATCH_MONTH': 'True', + 'APP_CLIMO__DAY_INTERVAL': '30', + 'APP_CLIMO__HOUR_INTERVAL': '12', + }, + ('climo_ = {file_name = ' + '["/some/climo_/file.txt"];' + 'field = [{name="CLM_NAME"; level="(0,0,*,*)";}];' + 'regrid = {method = NEAREST;width = 1;' + 'vld_thresh = 0.5;shape = SQUARE;}' + 'time_interp_method = NEAREST;' + 'match_month = TRUE;day_interval = 30;' + 'hour_interval = 12;}')), + ] +) +def test_handle_climo_dict(metplus_config, config_overrides, expected_value): + app_name = 'app' + for climo_type in ('MEAN', 'STDEV'): + expected_var = f'METPLUS_CLIMO_{climo_type}_DICT' + config = metplus_config() + output_dict = {} + + # set config values + for key, value in config_overrides.items(): + key_sub = key.replace('', climo_type) + value_sub = value.replace('', climo_type.lower()) + config.set('config', key_sub, value_sub) + + handle_climo_dict(config, app_name, output_dict) + print(output_dict) + expected_sub = expected_value.replace('', climo_type.lower()) + assert output_dict[expected_var] == expected_sub @pytest.mark.parametrize( 'name, data_type, mp_configs, extra_args', [ @@ -57,3 +198,57 @@ def test_set_met_config_function(data_type, expected_function): ) def test_format_regrid_to_grid(input, output): assert format_regrid_to_grid(input) == output + +@pytest.mark.parametrize( + 'config_overrides, expected_value', [ + # 0 no climo variables set + ({}, ''), + # 1 file name set only + ({'FILE_NAME': '/mean/dir/gs_climo_{init?fmt=%Y%m%d%H}.tmpl'}, + '/mean/dir/gs_climo_{init?fmt=%Y%m%d%H}.tmpl'), + # 2 input template set + ({'INPUT_TEMPLATE': '/mean/dir/gs_climo_{init?fmt=%Y%m%d%H}.tmpl'}, + '/mean/dir/gs_climo_{init?fmt=%Y%m%d%H}.tmpl'), + # 3 input template and dir set + ({'INPUT_DIR': '/mean/dir', + 'INPUT_TEMPLATE': 'gs_climo_{init?fmt=%Y%m%d%H}.tmpl'}, + '/mean/dir/gs_climo_{init?fmt=%Y%m%d%H}.tmpl'), + # 4 input template and dir set multiple templates + ({'INPUT_DIR': '/mean/dir', + 'INPUT_TEMPLATE': 'gs_climo_1.tmpl, gs_climo_2.tmpl'}, + '/mean/dir/gs_climo_1.tmpl,/mean/dir/gs_climo_2.tmpl'), + # 5file name, input template and dir all set + ({'FILE_NAME': '/mean/dir/gs_climo_{init?fmt=%Y%m%d%H}.tmpl', + 'INPUT_DIR': '/mean/dir', + 'INPUT_TEMPLATE': 'gs_climo_1.tmpl, gs_climo_2.tmpl'}, + '/mean/dir/gs_climo_{init?fmt=%Y%m%d%H}.tmpl'), + # 6 input template is python embedding keyword and dir is set + ({'INPUT_DIR': '/mean/dir', + 'INPUT_TEMPLATE': 'PYTHON_NUMPY'}, + 'PYTHON_NUMPY'), + # 7 input template is python embedding keyword and dir is set + ({'INPUT_DIR': '/mean/dir', + 'INPUT_TEMPLATE': 'PYTHON_XARRAY'}, + 'PYTHON_XARRAY'), + ] +) +def test_read_climo_file_name(metplus_config, config_overrides, + expected_value): + # name of app used for testing to read/set config variables + app_name = 'grid_stat' + + # check mean and stdev climo variables + for climo_type in CLIMO_TYPES: + prefix = f'{app_name.upper()}_CLIMO_{climo_type.upper()}_' + + config = metplus_config() + + # set config values + for key, value in config_overrides.items(): + config.set('config', f'{prefix}{key}', value) + + _read_climo_file_name(climo_type=climo_type, + config=config, + app_name=app_name) + actual_value = config.getraw('config', f'{prefix}FILE_NAME', '') + assert actual_value == expected_value diff --git a/internal_tests/pytests/mode/test_mode_wrapper.py b/internal_tests/pytests/mode/test_mode_wrapper.py index 439bdfed2..4dd1158c0 100644 --- a/internal_tests/pytests/mode/test_mode_wrapper.py +++ b/internal_tests/pytests/mode/test_mode_wrapper.py @@ -25,6 +25,12 @@ obs_fmt = (f'field = {{ name="{obs_name}"; ' f'level="{obs_level_no_quotes}"; }};') +fcst_multi_fmt = (f'field = [{{ name="{fcst_name}"; level="{fcst_level}"; }},' + f'{{ name="{fcst_name}"; level="{fcst_level}"; }}];') +obs_multi_fmt = (f'field = [{{ name="{obs_name}"; ' + f'level="{obs_level_no_quotes}"; }},' + f'{{ name="{obs_name}"; level="{obs_level_no_quotes}"; }}];') + def set_minimum_config_settings(config): # set config variables to prevent command from running and bypass check @@ -310,6 +316,9 @@ def set_minimum_config_settings(config): {'METPLUS_FCST_FILE_TYPE': 'file_type = NETCDF_PINT;'}), ({'MODE_OBS_FILE_TYPE': 'NETCDF_PINT', }, {'METPLUS_OBS_FILE_TYPE': 'file_type = NETCDF_PINT;'}), + ({'MODE_MASK_MISSING_FLAG': 'BOTH', }, + {'METPLUS_MASK_MISSING_FLAG': 'mask_missing_flag = BOTH;'}), + ] ) def test_mode_single_field(metplus_config, config_overrides, @@ -368,7 +377,75 @@ def test_mode_single_field(metplus_config, config_overrides, for (cmd, env_vars), expected_cmd in zip(all_cmds, expected_cmds): # ensure commands are generated as expected - assert(cmd == expected_cmd) + assert cmd == expected_cmd + + # check that environment variables were set properly + # including deprecated env vars (not in wrapper env var keys) + env_var_keys = (wrapper.WRAPPER_ENV_VAR_KEYS + + [name for name in expected_output + if name not in wrapper.WRAPPER_ENV_VAR_KEYS]) + for env_var_key in env_var_keys: + match = next((item for item in env_vars if + item.startswith(env_var_key)), None) + assert match is not None + value = match.split('=', 1)[1] + if env_var_key == 'METPLUS_FCST_FIELD': + assert value == fcst_fmt + elif env_var_key == 'METPLUS_OBS_FIELD': + assert value == obs_fmt + else: + assert expected_output.get(env_var_key, '') == value + +@pytest.mark.parametrize( + 'config_overrides, expected_output', [ + ({'MODE_MULTIVAR_LOGIC': '#1 && #2 && #3', }, + {'METPLUS_MULTIVAR_LOGIC': 'multivar_logic = "#1 && #2 && #3";'}), + ] +) +def test_mode_multi_variate(metplus_config, config_overrides, + expected_output): + config = metplus_config() + + # set config variables needed to run + set_minimum_config_settings(config) + + # change config values to reflect an expected multi-variate run + # multiple fields and input files + config.set('config', 'FCST_VAR2_NAME', fcst_name) + config.set('config', 'FCST_VAR2_LEVELS', fcst_level) + config.set('config', 'OBS_VAR2_NAME', obs_name) + config.set('config', 'OBS_VAR2_LEVELS', obs_level) + config.set('config', 'FCST_MODE_INPUT_TEMPLATE', + ('{init?fmt=%Y%m%d%H}/fcst_file_F{lead?fmt=%3H},' + '{init?fmt=%Y%m%d%H}/fcst_file_F{lead?fmt=%3H}')) + config.set('config', 'OBS_MODE_INPUT_TEMPLATE', + '{valid?fmt=%Y%m%d%H}/obs_file,{valid?fmt=%Y%m%d%H}/obs_file') + + wrapper = MODEWrapper(config) + assert wrapper.isOK + + app_path = os.path.join(config.getdir('MET_BIN_DIR'), wrapper.app_name) + verbosity = f"-v {wrapper.c_dict['VERBOSITY']}" + config_file = wrapper.c_dict.get('CONFIG_FILE') + out_dir = wrapper.c_dict.get('OUTPUT_DIR') + file_list_dir = os.path.join(config.getdir('STAGING_DIR'), 'file_lists') + + expected_cmds = [(f"{app_path} {verbosity} " + f"{file_list_dir}/20050807000000_12_mode_fcst.txt " + f"{file_list_dir}/20050807000000_12_mode_obs.txt " + f"{config_file} -outdir {out_dir}/2005080712"), + (f"{app_path} {verbosity} " + f"{file_list_dir}/20050807120000_12_mode_fcst.txt " + f"{file_list_dir}/20050807120000_12_mode_obs.txt " + f"{config_file} -outdir {out_dir}/2005080800"), + ] + + all_cmds = wrapper.run_all_times() + print(f"ALL COMMANDS: {all_cmds}") + + for (cmd, env_vars), expected_cmd in zip(all_cmds, expected_cmds): + # ensure commands are generated as expected + assert cmd == expected_cmd # check that environment variables were set properly # including deprecated env vars (not in wrapper env var keys) @@ -378,14 +455,15 @@ def test_mode_single_field(metplus_config, config_overrides, for env_var_key in env_var_keys: match = next((item for item in env_vars if item.startswith(env_var_key)), None) - assert(match is not None) + assert match is not None value = match.split('=', 1)[1] if env_var_key == 'METPLUS_FCST_FIELD': - assert(value == fcst_fmt) + assert value == fcst_multi_fmt elif env_var_key == 'METPLUS_OBS_FIELD': - assert (value == obs_fmt) + assert value == obs_multi_fmt else: - assert(expected_output.get(env_var_key, '') == value) + assert expected_output.get(env_var_key, '') == value + @pytest.mark.parametrize( 'config_name, env_var_name, met_name, var_type', [ @@ -450,7 +528,7 @@ def test_config_synonyms(metplus_config, config_name, env_var_name, expected_output = f'{met_name} = {out_value};' - assert(wrapper.env_var_dict[env_var_name] == expected_output) + assert wrapper.env_var_dict[env_var_name] == expected_output def test_get_config_file(metplus_config): fake_config_name = '/my/config/file' diff --git a/internal_tests/pytests/point2grid/test_point2grid.py b/internal_tests/pytests/point2grid/test_point2grid.py index f29a8888b..d6b213722 100644 --- a/internal_tests/pytests/point2grid/test_point2grid.py +++ b/internal_tests/pytests/point2grid/test_point2grid.py @@ -135,4 +135,44 @@ def test_set_command_line_arguments(metplus_config): wrap.args.clear() + wrap.c_dict['QC_FLAGS'] = 1 + + expected_args = ['-qc 1', + '-method UW_MEAN', + '-gaussian_dx 2', + '-gaussian_radius 3', + '-prob_cat_thresh 1', + '-vld_thresh 0.5', + ] + + wrap.set_command_line_arguments(time_info) + if wrap.args != expected_args: + test_passed = False + print("Test 5 failed") + print(f"ARGS: {wrap.args}") + print(f"EXP: {expected_args}") + + wrap.args.clear() + + wrap.c_dict['ADP'] = 'test.nc' + + expected_args = ['-qc 1', + '-adp test.nc', + '-method UW_MEAN', + '-gaussian_dx 2', + '-gaussian_radius 3', + '-prob_cat_thresh 1', + '-vld_thresh 0.5', + ] + + wrap.set_command_line_arguments(time_info) + if wrap.args != expected_args: + test_passed = False + print("Test 6 failed") + print(f"ARGS: {wrap.args}") + print(f"EXP: {expected_args}") + + wrap.args.clear() + + assert(test_passed) diff --git a/internal_tests/use_cases/all_use_cases.txt b/internal_tests/use_cases/all_use_cases.txt index 7bb1da0d3..81aa3b400 100644 --- a/internal_tests/use_cases/all_use_cases.txt +++ b/internal_tests/use_cases/all_use_cases.txt @@ -137,6 +137,7 @@ Category: s2s 10:: UserScript_obsERA_obsOnly_RMM:: model_applications/s2s/UserScript_obsERA_obsOnly_RMM.conf:: spacetime_env, metdatadb 11:: UserScript_fcstGFS_obsERA_WeatherRegime:: model_applications/s2s/UserScript_fcstGFS_obsERA_WeatherRegime.conf:: weatherregime_env,cartopy,metplus 12:: UserScript_obsERA_obsOnly_Stratosphere:: model_applications/s2s/UserScript_obsERA_obsOnly_Stratosphere.conf:: metplotpy_env,metdatadb +13::SeriesAnalysis_fcstCFSv2_obsGHCNCAMS_climoStandardized_MultiStatisticTool:: model_applications/s2s/SeriesAnalysis_fcstCFSv2_obsGHCNCAMS_climoStandardized_MultiStatisticTool.conf:: netcdf4_env Category: space_weather 0::GridStat_fcstGloTEC_obsGloTEC_vx7:: model_applications/space_weather/GridStat_fcstGloTEC_obsGloTEC_vx7.conf diff --git a/metplus/VERSION b/metplus/VERSION index 3d298f67a..c3a40d43b 100644 --- a/metplus/VERSION +++ b/metplus/VERSION @@ -1 +1 @@ -5.0.0-beta1-dev +5.0.0-beta2-dev diff --git a/metplus/util/constants.py b/metplus/util/constants.py index 02cb3e22e..7e7d9cd9f 100644 --- a/metplus/util/constants.py +++ b/metplus/util/constants.py @@ -1,2 +1,37 @@ -COMPRESSION_EXTENSIONS = ['.gz', '.bz2', '.zip'] +# supported file extensions that will automatically be uncompressed +COMPRESSION_EXTENSIONS = [ + '.gz', + '.bz2', + '.zip', +] +# Keywords of valid types of Python Embedding in MET tools +PYTHON_EMBEDDING_TYPES = [ + 'PYTHON_NUMPY', + 'PYTHON_XARRAY', + 'PYTHON_PANDAS', +] + +# types of climatology values that should be checked and set +CLIMO_TYPES = [ + 'MEAN', + 'STDEV', +] + +# comparison operators that the MET tools support +# key is an operator and value is the alphabetic equivalent +VALID_COMPARISONS = { + '>=': 'ge', + '>': 'gt', + '==': 'eq', + '!=': 'ne', + '<=': 'le', + '<': 'lt', +} + +# wrappers that do not run shell commands +# used to check that at least one command was generated if it should +NO_COMMAND_WRAPPERS = ( + 'Example', + 'CyclonePlotter', +) diff --git a/metplus/util/field_util.py b/metplus/util/field_util.py index 9f621f6af..1f1457a03 100644 --- a/metplus/util/field_util.py +++ b/metplus/util/field_util.py @@ -119,6 +119,45 @@ def get_field_info(c_dict, data_type='', v_name='', v_level='', v_thresh=None, return fields +def format_field_info(c_dict, var_info, data_type, add_curly_braces=True): + """! Format field information into format expected by MET config file + + @param c_dict config dictionary to read values + @param var_info dictionary of field info to format + @param data_type type of data to find i.e. FCST or OBS + @param add_curly_braces if True, add curly braces around each + field info string. If False, add single quotes around each + field info string (defaults to True) + @rtype string + @return Returns a list of formatted field information or a string + containing an error message if something went wrong + """ + dt_lower = data_type.lower() + return get_field_info(c_dict=c_dict, + data_type=data_type, + v_name=var_info.get(f'{dt_lower}_name'), + v_level=var_info.get(f'{dt_lower}_level'), + v_thresh=var_info.get(f'{dt_lower}_thresh'), + v_extra=var_info.get(f'{dt_lower}_extra'), + add_curly_braces=add_curly_braces, + ) + + +def format_all_field_info(c_dict, var_list, data_type, add_curly_braces=True): + formatted_list = [] + for var_info in var_list: + field_info = format_field_info(c_dict=c_dict, + var_info=var_info, + data_type=data_type, + add_curly_braces=add_curly_braces) + if not field_info: + return None + + formatted_list.extend(field_info) + + return ','.join(formatted_list) + + def _handle_grib_pds_field_info(v_name, v_level, thresh): """! Format field string to read probabilistic data from the PDS of a GRIB file. Thresholds are formatted using thresh_lo and thresh_hi syntax. diff --git a/metplus/util/met_config.py b/metplus/util/met_config.py index 2b99e5410..807cf10a3 100644 --- a/metplus/util/met_config.py +++ b/metplus/util/met_config.py @@ -4,11 +4,14 @@ """ import os +import re +from .constants import PYTHON_EMBEDDING_TYPES, CLIMO_TYPES from .string_manip import getlist from .met_util import get_threshold_via_regex, MISSING_DATA_VALUE from .string_manip import remove_quotes as util_remove_quotes -from .config_metplus import find_indices_in_config_section +from .config_metplus import find_indices_in_config_section, parse_var_list +from .field_util import format_all_field_info class METConfig: """! Stores information for a member of a MET config variables that @@ -109,6 +112,7 @@ def children(self, children): self._children = children + def get_wrapped_met_config_file(config, app_name, default_config_file=None): """! Get the MET config file path for the wrapper from the METplusConfig object. If unset, use the default value if provided. @@ -132,12 +136,16 @@ def get_wrapped_met_config_file(config, app_name, default_config_file=None): f"Using {default_config_path}") return default_config_path + def add_met_config_dict(config, app_name, output_dict, dict_name, items): - """! Read config variables for MET config dictionary and set - env_var_dict with formatted values + """! Read config variables for MET config dictionary and set output_dict + (typically CommandBuilder's env_var_dict) with formatted values - @params dict_name name of MET dictionary variable - @params items dictionary where the key is name of variable inside MET + @param config METplusConfig object to read + @param app_name name of application to find wrapper-specific variables + @param output_dict dictionary to save formatted output + @param dict_name name of MET dictionary variable + @param items dictionary where the key is name of variable inside MET dictionary and the value is info about the item (see parse_item_info function for more information) """ @@ -231,11 +239,13 @@ def add_met_config_dict(config, app_name, output_dict, dict_name, items): final_met_config, output_dict) + def add_met_config_item(config, item, output_dict, depth=0): """! Reads info from METConfig object, gets value from METplusConfig, and formats it based on the specifications. Sets value in output dictionary with key starting with METPLUS_. + @param config METplusConfig object to read @param item METConfig object to read and determine what to get @param output_dict dictionary to save formatted output @param depth counter to check if item being processed is nested within @@ -279,8 +289,20 @@ def add_met_config_item(config, item, output_dict, depth=0): c_dict_key=env_var_name, **item.extra_args) + def add_met_config_dict_list(config, app_name, output_dict, dict_name, dict_items): + """! Read METplusConfig and format MET config variables that are a list of + dictionaries. Sets value in output dict with key starting with METPLUS_. + + @param config METplusConfig object to read + @param app_name name of application to find wrapper-specific variables + @param output_dict dictionary to save formatted output + @param dict_name name of MET dictionary variable + @param dict_items dictionary where the key is name of variable inside MET + dictionary and the value is info about the item (see parse_item_info + function for more information) + """ search_string = f'{app_name}_{dict_name}'.upper() regex = r'^' + search_string + r'(\d+)_(\w+)$' indices = find_indices_in_config_section(regex, config, @@ -329,6 +351,7 @@ def add_met_config_dict_list(config, app_name, output_dict, dict_name, output_dict[f'METPLUS_{dict_name.upper()}_LIST'] = output_string return is_ok + def format_met_config(data_type, c_dict, name, keys=None): """! Return formatted variable named with any if they are set to a value. If none of the items are set, return empty string @@ -368,6 +391,7 @@ def format_met_config(data_type, c_dict, name, keys=None): output = f'{name} = {output}' return output + def set_met_config_function(item_type): """! Return function to use based on item type @@ -390,6 +414,7 @@ def set_met_config_function(item_type): else: raise ValueError(f"Invalid argument for item type: {item_type}") + def _get_config_or_default(mp_config_name, get_function, default=None): conf_value = '' @@ -412,6 +437,7 @@ def _get_config_or_default(mp_config_name, get_function, return conf_value + def set_met_config_list(config, c_dict, mp_config, met_config_name, c_dict_key=None, **kwargs): """! Get list from METplus configuration file and format it to be passed @@ -471,6 +497,7 @@ def set_met_config_list(config, c_dict, mp_config, met_config_name, return True + def set_met_config_string(config, c_dict, mp_config, met_config_name, c_dict_key=None, **kwargs): """! Get string from METplus configuration file and format it to be passed @@ -517,22 +544,24 @@ def set_met_config_string(config, c_dict, mp_config, met_config_name, c_dict[c_key] = conf_value return True + def set_met_config_number(config, c_dict, num_type, mp_config, met_config_name, c_dict_key=None, **kwargs): """! Get integer from METplus configuration file and format it to be passed into a MET configuration file. Set c_dict item with formatted string. - Args: - @param c_dict configuration dictionary to set - @param num_type type of number to get from config. If set to 'int', call - getint function. If not, call getfloat function. - @param mp_config METplus configuration variable name. Assumed to be - in the [config] section. Value can be a comma-separated list of items. - @param met_config_name name of MET configuration variable to set. Also used - to determine the key in c_dict to set (upper-case) if c_dict_key is None - @param c_dict_key optional argument to specify c_dict key to store result. If - set to None (default) then use upper-case of met_config_name - @param default (Optional) if set, use this value as default - if config is not set + + @param c_dict configuration dictionary to set + @param num_type type of number to get from config. If set to 'int', + call getint function. If not, call getfloat function. + @param mp_config METplus configuration variable name. Assumed to be + in the [config] section. Value can be a comma-separated list of items. + @param met_config_name name of MET configuration variable to set. + Also used to determine the key in c_dict to set (upper-case) if + c_dict_key is None. + @param c_dict_key optional argument to specify c_dict key to store + result. If set to None (default) then use upper-case of met_config_name + @param default (Optional) if set, use this value as default + if config is not set """ mp_config_name = config.get_mp_config_name(mp_config) if mp_config_name is None: @@ -559,6 +588,7 @@ def set_met_config_number(config, c_dict, num_type, mp_config, return True + def set_met_config_int(config, c_dict, mp_config_name, met_config_name, c_dict_key=None, **kwargs): return set_met_config_number(config, c_dict, 'int', @@ -567,6 +597,7 @@ def set_met_config_int(config, c_dict, mp_config_name, met_config_name, c_dict_key=c_dict_key, **kwargs) + def set_met_config_float(config, c_dict, mp_config_name, met_config_name, c_dict_key=None, **kwargs): return set_met_config_number(config, c_dict, 'float', @@ -575,6 +606,7 @@ def set_met_config_float(config, c_dict, mp_config_name, c_dict_key=c_dict_key, **kwargs) + def set_met_config_thresh(config, c_dict, mp_config, met_config_name, c_dict_key=None, **kwargs): mp_config_name = config.get_mp_config_name(mp_config) @@ -600,6 +632,7 @@ def set_met_config_thresh(config, c_dict, mp_config, met_config_name, c_dict[c_key] = out_value return True + def set_met_config_bool(config, c_dict, mp_config, met_config_name, c_dict_key=None, **kwargs): """! Get boolean from METplus configuration file and format it to be @@ -644,6 +677,7 @@ def set_met_config_bool(config, c_dict, mp_config, met_config_name, c_dict[c_key] = conf_value return True + def format_regrid_to_grid(to_grid): to_grid = to_grid.strip('"') if not to_grid: @@ -657,6 +691,7 @@ def format_regrid_to_grid(to_grid): return to_grid + def _parse_item_info(item_info): """! Parses info about a MET config dictionary item. The input can be a single string that is the data type of the item. It can also be @@ -691,6 +726,7 @@ def _parse_item_info(item_info): return data_type, extra, kids, nicknames + def _parse_extra_args(extra): """! Check string for extra option keywords and set them to True in dictionary if they are found. Supports 'remove_quotes', 'uppercase' @@ -714,3 +750,190 @@ def _parse_extra_args(extra): if extra_option in extra: extra_args[extra_option] = True return extra_args + + +def handle_climo_dict(config, app_name, output_dict): + """! Read climo mean/stdev variables with and set env_var_dict + appropriately. Handle previous environment variables that are used + by wrapped MET configs pre 4.0 (CLIMO_MEAN_FILE and CLIMO_STDEV_FILE) + + @param config METplusConfig object to read + @param app_name name of application to find wrapper-specific variables + @param output_dict dictionary to save formatted output + @returns False if config settings are invalid, otherwise True + """ + items = { + 'file_name': 'list', + 'field': ('list', 'remove_quotes'), + 'regrid': ('dict', '', { + 'method': ('string', 'uppercase,remove_quotes'), + 'width': 'int', + 'vld_thresh': 'float', + 'shape': ('string', 'uppercase,remove_quotes'), + }), + 'time_interp_method': ('string', 'remove_quotes,uppercase'), + 'match_month': ('bool', 'uppercase'), + 'day_interval': ('string', 'remove_quotes,uppercase'), + 'hour_interval': 'int', + 'file_type': ('string', 'remove_quotes'), + } + is_ok = True + for climo_type in CLIMO_TYPES: + dict_name = f'climo_{climo_type.lower()}' + + # make sure _FILE_NAME is set from INPUT_TEMPLATE/DIR if used + _read_climo_file_name(climo_type, config, app_name) + + # make sure _FIELD is set from VAR vars if used + _read_climo_field(climo_type, config, app_name) + + add_met_config_dict(config=config, app_name=app_name, + output_dict=output_dict, + dict_name=dict_name, items=items) + + # handle use_fcst or use_obs options for setting field list + if not _climo_use_fcst_or_obs_fields(dict_name, config, app_name, + output_dict): + is_ok = False + + # handle deprecated env vars CLIMO_MEAN_FILE and CLIMO_STDEV_FILE + # that are used by pre v4.0.0 wrapped MET config files + env_var_name = f'METPLUS_{dict_name.upper()}_DICT' + dict_value = output_dict.get(env_var_name, '') + match = re.match(r'.*file_name = \[([^\[\]]*)\];.*', dict_value) + if match: + file_name = match.group(1) + output_dict[f'{dict_name.upper()}_FILE'] = file_name + + return is_ok + + +def _read_climo_file_name(climo_type, config, app_name): + """! Check values for {APP}_CLIMO_{climo_type}_ variables FILE_NAME, + INPUT_TEMPLATE, and INPUT_DIR. If FILE_NAME is set, use it and warn + if the INPUT_TEMPLATE/DIR variables are also set. If FILE_NAME is not + set, read template and dir variables and format the values to set + FILE_NAME, i.e. the variables: + GRID_STAT_CLIMO_MEAN_INPUT_TEMPLATE = a, b + GRID_STAT_CLIMO_MEAN_INPUT_DIR = /some/dir + will set: + GRID_STAT_CLIMO_MEAN_FILE_NAME = /some/dir/a, some/dir/b + Used to support pre v4.0 variables. + + @param climo_type type of climo field (mean or stdev) + @param config METplusConfig object to read + @param app_name name of application to find wrapper-specific variables + """ + # prefix i.e. GRID_STAT_CLIMO_MEAN_ + prefix = f'{app_name.upper()}_CLIMO_{climo_type.upper()}_' + + input_dir = config.getdir_nocheck(f'{prefix}INPUT_DIR', '') + input_template = config.getraw('config', f'{prefix}INPUT_TEMPLATE', '') + file_name = config.getraw('config', f'{prefix}FILE_NAME', '') + + # if input template is not set, nothing to do + if not input_template: + return + + # if input template is set and file name is also set, + # warn and use file name values + if file_name: + config.logger.warning(f'Both {prefix}INPUT_TEMPLATE and ' + f'{prefix}FILE_NAME are set. Using ' + f'value set in {prefix}FILE_NAME ' + f'({file_name})') + return + + template_list_string = input_template + # if file name is not set but template is, set file name from template + # if dir is set and not python embedding, + # prepend it to each template in list + if input_dir and input_template not in PYTHON_EMBEDDING_TYPES: + template_list = getlist(input_template) + for index, template in enumerate(template_list): + template_list[index] = os.path.join(input_dir, template) + + # change formatted list back to string + template_list_string = ','.join(template_list) + + config.set('config', f'{prefix}FILE_NAME', template_list_string) + + +def _read_climo_field(climo_type, config, app_name): + """! Check values for {APP}_CLIMO_{climo_type}_ variable FIELD and + VAR_[NAME/LEVELS/OPTIONS]. If VAR variables are set, format those + values and set the FIELD variable in the METplusConfig. + If FIELD is also set warn that it the value will be replaced. If only + FIELD is set, use that value. + The variables: + GRID_STAT_CLIMO_MEAN_VAR1_NAME = TMP + GRID_STAT_CLIMO_MEAN_VAR1_LEVELS = "(*,*)" + will set: + GRID_STAT_CLIMO_MEAN_FIELD = {name="TMP"; level="(*,*)";} + Used to allow users to format the field info for climo fields in the same + way as other field info such as FCST or OBS. + + @param climo_type type of climo field (mean or stdev) + @param config METplusConfig object to read + @param app_name name of application to find wrapper-specific variables + """ + # prefix i.e. GRID_STAT_CLIMO_MEAN + prefix = f'{app_name.upper()}_CLIMO_{climo_type.upper()}' + + field = config.getraw('config', f'{prefix}_FIELD', '') + var_list = parse_var_list(config, data_type=prefix) + + # if field info is not set using VAR variables, nothing to do + if not var_list: + return + + # if FIELD is set and VAR is also set, warn and use VAR values + if field: + config.logger.warning(f'Both {prefix}_FIELD and ' + f'{prefix}_VAR_* variables are set. Using ' + f'values set in {prefix}_VAR_* variables') + + # format var list and set it to the _FIELD value + all_formatted = format_all_field_info(c_dict={}, + var_list=var_list, + data_type=prefix) + + config.set('config', f'{prefix}_FIELD', all_formatted) + + +def _climo_use_fcst_or_obs_fields(dict_name, config, app_name, output_dict): + """! If climo field is not explicitly set, check if config is set + to use forecast or observation fields. + + @param dict_name name of climo to check: climo_mean or climo_stdev + @param config METplusConfig object to read + @param app_name name of application to find wrapper-specific variables + @param output_dict dictionary to save formatted output + @returns False if config settings are invalid, otherwise True + """ + # if {APP}_CLIMO_[MEAN/STDEV]_FIELD is set, do nothing + field_conf = f'{app_name}_{dict_name}_FIELD'.upper() + if config.has_option('config', field_conf): + return True + + use_fcst_conf = f'{app_name}_{dict_name}_USE_FCST'.upper() + use_obs_conf = f'{app_name}_{dict_name}_USE_OBS'.upper() + + use_fcst = config.getbool('config', use_fcst_conf, False) + use_obs = config.getbool('config', use_obs_conf, False) + + # if both are set, report an error + if use_fcst and use_obs: + config.logger.error(f'Cannot set both {use_fcst_conf} and ' + f'{use_obs_conf} in config.') + return False + + # if neither are set, do nothing + if not use_fcst and not use_obs: + return True + + env_var_name = f'METPLUS_{dict_name.upper()}_DICT' + rvalue = 'fcst' if use_fcst else 'obs' + + output_dict[env_var_name] += f'{dict_name} = {rvalue};' + return True diff --git a/metplus/util/met_util.py b/metplus/util/met_util.py index abe99536f..9b093be6a 100644 --- a/metplus/util/met_util.py +++ b/metplus/util/met_util.py @@ -24,16 +24,6 @@ from .constants import * -PYTHON_EMBEDDING_TYPES = ['PYTHON_NUMPY', 'PYTHON_XARRAY', 'PYTHON_PANDAS'] - -valid_comparisons = {">=": "ge", - ">": "gt", - "==": "eq", - "!=": "ne", - "<=": "le", - "<": "lt", - } - # missing data value used to check if integer values are not set # we often check for None if a variable is not set, but 0 and None # have the same result in a test. 0 may be a valid integer value @@ -159,7 +149,12 @@ def run_metplus(config, process_list): return 1 # write out all commands and environment variables to file - write_all_commands(all_commands, config) + if not write_all_commands(all_commands, config): + # if process list contains any wrapper that should run commands, + # report an error if no commands were generated + if any([item[0] not in NO_COMMAND_WRAPPERS + for item in process_list]): + total_errors += 1 # compute total number of errors that occurred and output results for process in processes: @@ -217,10 +212,20 @@ def post_run_cleanup(config, app_name, total_errors): sys.exit(1) def write_all_commands(all_commands, config): + """! Write all commands that were run to a file in the log + directory. This includes the environment variables that + were set before each command. + + @param all_commands list of tuples with command run and + list of environment variables that were set + @param config METplusConfig object used to write log output + and get the log timestamp to name the output file + @returns False if no commands were provided, True otherwise + """ if not all_commands: - config.logger.debug("No commands were run. " + config.logger.error("No commands were run. " "Skip writing all_commands file") - return + return False log_timestamp = config.getstr('config', 'LOG_TIMESTAMP') filename = os.path.join(config.getdir('LOG_DIR'), @@ -234,6 +239,8 @@ def write_all_commands(all_commands, config): file_handle.write("COMMAND:\n") file_handle.write(f"{command}\n\n") + return True + def handle_tmp_dir(config): """! if env var MET_TMP_DIR is set, override config TMP_DIR with value if it differs from what is set @@ -837,7 +844,7 @@ def get_threshold_via_regex(thresh_string): # check each threshold for validity for thresh in thresh_split: found_match = False - for comp in list(valid_comparisons.keys())+list(valid_comparisons.values()): + for comp in list(VALID_COMPARISONS)+list(VALID_COMPARISONS.values()): # if valid, add to list of tuples # must be one of the valid comparison operators followed by # at least 1 digit or NA @@ -875,7 +882,7 @@ def comparison_to_letter_format(expression): convert, i.e. gt3 or <=5.4 @returns letter comparison operator, i.e. gt3 or le5.4 or None if invalid """ - for symbol_comp, letter_comp in valid_comparisons.items(): + for symbol_comp, letter_comp in VALID_COMPARISONS.items(): if letter_comp in expression or symbol_comp in expression: return expression.replace(symbol_comp, letter_comp) diff --git a/metplus/wrappers/command_builder.py b/metplus/wrappers/command_builder.py index c45206013..e41b51cf8 100755 --- a/metplus/wrappers/command_builder.py +++ b/metplus/wrappers/command_builder.py @@ -15,9 +15,10 @@ from datetime import datetime from abc import ABCMeta from inspect import getframeinfo, stack -import re from .command_runner import CommandRunner + +from ..util.constants import PYTHON_EMBEDDING_TYPES from ..util import getlist from ..util import met_util as util from ..util import do_string_sub, ti_calculate, get_seconds_from_string @@ -28,8 +29,8 @@ from ..util import get_custom_string_list from ..util import get_wrapped_met_config_file, add_met_config_item, format_met_config from ..util import remove_quotes -from ..util import get_field_info -from ..util.met_config import add_met_config_dict +from ..util import get_field_info, format_field_info +from ..util.met_config import add_met_config_dict, handle_climo_dict # pylint:disable=pointless-string-statement '''!@namespace CommandBuilder @@ -46,9 +47,6 @@ class CommandBuilder: """ __metaclass__ = ABCMeta - # types of climatology values that should be checked and set - climo_types = ['MEAN', 'STDEV'] - # name of variable to hold any MET config overrides MET_OVERRIDES_KEY = 'METPLUS_MET_CONFIG_OVERRIDES' @@ -165,8 +163,7 @@ def check_for_unused_env_vars(self): def create_c_dict(self): c_dict = dict() - # set skip if output exists to False for all wrappers - # wrappers that support this functionality can override this value + c_dict['VERBOSITY'] = self.config.getstr('config', 'LOG_MET_VERBOSITY', '2') @@ -1118,10 +1115,10 @@ def check_for_python_embedding(self, input_type, var_info): # if it is a python script, set file extension to show that and make sure *_INPUT_DATATYPE is a valid PYTHON_* string file_ext = 'python_embedding' data_type = self.c_dict.get(f'{input_type}_INPUT_DATATYPE', '') - if data_type not in util.PYTHON_EMBEDDING_TYPES: + if data_type not in PYTHON_EMBEDDING_TYPES: self.log_error(f"{input_type}_{self.app_name.upper()}_INPUT_DATATYPE ({data_type}) must be set to a valid Python Embedding type " f"if supplying a Python script as the {input_type}_VAR_NAME. Valid options: " - f"{','.join(util.PYTHON_EMBEDDING_TYPES)}") + f"{','.join(PYTHON_EMBEDDING_TYPES)}") return None # set file type string to be set in MET config file to specify Python Embedding is being used for this dataset @@ -1162,6 +1159,32 @@ def get_field_info(self, d_type='', v_name='', v_level='', v_thresh=None, return fields + def format_field_info(self, var_info, data_type, add_curly_braces=True): + """! Format field information into format expected by MET config file. + Calls utility function found in metplus.util.field_util. + This may eventually replace get_field_info. + + @param var_info dictionary of field info to format + @param data_type type of data to find i.e. FCST or OBS + @param add_curly_braces if True, add curly braces around each + field info string. If False, add single quotes around each + field info string (defaults to True) + @rtype string + @return Returns formatted field information or None on error + """ + fields = format_field_info(c_dict=self.c_dict, + var_info=var_info, + data_type=data_type, + add_curly_braces=add_curly_braces) + + # if value returned is a string, it is an error message, + # so log error and return None + if isinstance(fields, str): + self.log_error(fields) + return None + + return fields + def get_command(self): """! Builds the command to run the MET application @rtype string @@ -1378,122 +1401,10 @@ def handle_climo_dict(self): by wrapped MET configs pre 4.0 (CLIMO_MEAN_FILE and CLIMO_STDEV_FILE) """ - items = { - 'file_name': 'list', - 'field': ('list', 'remove_quotes'), - 'regrid': ('dict', '', { - 'method': ('string', 'uppercase,remove_quotes'), - 'width': 'int', - 'vld_thresh': 'float', - 'shape': ('string', 'uppercase,remove_quotes'), - }), - 'time_interp_method': ('string', 'remove_quotes,uppercase'), - 'match_month': ('bool', 'uppercase'), - 'day_interval': 'int', - 'hour_interval': 'int', - 'file_type': ('string', 'remove_quotes'), - } - for climo_type in self.climo_types: - dict_name = f'climo_{climo_type.lower()}' - - # make sure _FILE_NAME is set from INPUT_TEMPLATE/DIR if used - self.read_climo_file_name(climo_type) - - self.add_met_config_dict(dict_name, items) - - # handle use_fcst or use_obs options for setting field list - self.climo_use_fcst_or_obs_fields(dict_name) - - # handle deprecated env vars CLIMO_MEAN_FILE and CLIMO_STDEV_FILE - # that are used by pre v4.0.0 wrapped MET config files - env_var_name = f'METPLUS_{dict_name.upper()}_DICT' - dict_value = self.env_var_dict.get(env_var_name, '') - match = re.match(r'.*file_name = \[([^\[\]]*)\];.*', dict_value) - if match: - file_name = match.group(1) - self.env_var_dict[f'{dict_name.upper()}_FILE'] = file_name - - def read_climo_file_name(self, climo_type): - """! Check values for {APP}_CLIMO_{climo_type}_ variables FILE_NAME, - INPUT_TEMPLATE, and INPUT_DIR. If FILE_NAME is set, use it and warn - if the INPUT_TEMPLATE/DIR variables are also set. If FILE_NAME is not - set, read template and dir variables and format the values to set - FILE_NAME, i.e. the variables: - GRID_STAT_CLIMO_MEAN_INPUT_TEMPLATE = a, b - GRID_STAT_CLIMO_MEAN_INPUT_DIR = /some/dir - will set: - GRID_STAT_CLIMO_MEAN_FILE_NAME = /some/dir/a, some/dir/b - Used to support pre v4.0 variables. - - @param climo_type type of climo field (mean or stdev) - """ - # prefix i.e. GRID_STAT_CLIMO_MEAN_ - prefix = f'{self.app_name.upper()}_CLIMO_{climo_type.upper()}_' - - input_dir = self.config.getdir_nocheck(f'{prefix}INPUT_DIR', '') - input_template = self.config.getraw('config', - f'{prefix}INPUT_TEMPLATE', '') - file_name = self.config.getraw('config', - f'{prefix}FILE_NAME', '') - - # if input template is not set, nothing to do - if not input_template: - return - - # if input template is set and file name is also set, - # warn and use file name values - if file_name: - self.logger.warning(f'Both {prefix}INPUT_TEMPLATE and ' - f'{prefix}FILE_NAME are set. Using ' - f'value set in {prefix}FILE_NAME ' - f'({file_name})') - return - - template_list_string = input_template - # if file name is not set but template is, set file name from template - # if dir is set and not python embedding, - # prepend it to each template in list - if input_dir and input_template not in util.PYTHON_EMBEDDING_TYPES: - template_list = getlist(input_template) - for index, template in enumerate(template_list): - template_list[index] = os.path.join(input_dir, template) - - # change formatted list back to string - template_list_string = ','.join(template_list) - - self.config.set('config', f'{prefix}FILE_NAME', template_list_string) - - def climo_use_fcst_or_obs_fields(self, dict_name): - """! If climo field is not explicitly set, check if config is set - to use forecast or observation fields. - - @param dict_name name of climo to check: climo_mean or climo_stdev - """ - # if {APP}_CLIMO_[MEAN/STDEV]_FIELD is set, do nothing - field_conf = f'{self.app_name}_{dict_name}_FIELD'.upper() - if self.config.has_option('config', field_conf): - return - - use_fcst_conf = f'{self.app_name}_{dict_name}_USE_FCST'.upper() - use_obs_conf = f'{self.app_name}_{dict_name}_USE_OBS'.upper() - - use_fcst = self.config.getbool('config', use_fcst_conf, False) - use_obs = self.config.getbool('config', use_obs_conf, False) - - # if both are set, report an error - if use_fcst and use_obs: - self.log_error(f'Cannot set both {use_fcst_conf} and ' - f'{use_obs_conf} in config.') - return - - # if neither are set, do nothing - if not use_fcst and not use_obs: - return - - env_var_name = f'METPLUS_{dict_name.upper()}_DICT' - rvalue = 'fcst' if use_fcst else 'obs' - - self.env_var_dict[env_var_name] += f'{dict_name} = {rvalue};' + if not handle_climo_dict(config=self.config, + app_name=self.app_name, + output_dict=self.env_var_dict): + self.errors += 1 def get_wrapper_or_generic_config(self, generic_config_name): """! Check for config variable with _ prepended first. If set diff --git a/metplus/wrappers/compare_gridded_wrapper.py b/metplus/wrappers/compare_gridded_wrapper.py index d089e22b5..dca662e89 100755 --- a/metplus/wrappers/compare_gridded_wrapper.py +++ b/metplus/wrappers/compare_gridded_wrapper.py @@ -198,17 +198,11 @@ def run_at_time_one_field(self, time_info, var_info): self.infiles.extend(obs_path) # get field info field a single field to pass to the MET config file - fcst_field_list = self.get_field_info(v_level=var_info['fcst_level'], - v_thresh=var_info['fcst_thresh'], - v_name=var_info['fcst_name'], - v_extra=var_info['fcst_extra'], - d_type='FCST') - - obs_field_list = self.get_field_info(v_level=var_info['obs_level'], - v_thresh=var_info['obs_thresh'], - v_name=var_info['obs_name'], - v_extra=var_info['obs_extra'], - d_type='OBS') + fcst_field_list = self.format_field_info(var_info=var_info, + data_type='FCST') + + obs_field_list = self.format_field_info(var_info=var_info, + data_type='OBS') if fcst_field_list is None or obs_field_list is None: return @@ -237,7 +231,17 @@ def run_at_time_all_fields(self, time_info): if not model_path: return - self.infiles.extend(model_path) + # if there is more than 1 file, create file list file + if len(model_path) > 1: + list_filename = (f"{time_info['init_fmt']}_" + f"{time_info['lead_hours']}_" + f"{self.app_name}_fcst.txt") + model_path = self.write_list_file(list_filename, model_path) + else: + model_path = model_path[0] + + self.infiles.append(model_path) + # get observation to from first var compare obs_path, time_info = self.find_obs_offset(time_info, var_list[0], @@ -246,7 +250,16 @@ def run_at_time_all_fields(self, time_info): if obs_path is None: return - self.infiles.extend(obs_path) + # if there is more than 1 file, create file list file + if len(obs_path) > 1: + list_filename = (f"{time_info['init_fmt']}_" + f"{time_info['lead_hours']}_" + f"{self.app_name}_obs.txt") + obs_path = self.write_list_file(list_filename, obs_path) + else: + obs_path = obs_path[0] + + self.infiles.append(obs_path) fcst_field_list = [] obs_field_list = [] @@ -296,12 +309,6 @@ def process_fields(self, time_info): # set environment variables needed by MET config file self.set_environment_variables(time_info) - # check if METplus can generate the command successfully - cmd = self.get_command() - if cmd is None: - self.log_error("Could not generate command") - return - # run the MET command self.build() diff --git a/metplus/wrappers/mode_wrapper.py b/metplus/wrappers/mode_wrapper.py index e8e1f0f3a..bec9f67cf 100755 --- a/metplus/wrappers/mode_wrapper.py +++ b/metplus/wrappers/mode_wrapper.py @@ -48,6 +48,7 @@ class MODEWrapper(CompareGriddedWrapper): 'METPLUS_MASK_DICT', 'METPLUS_OUTPUT_PREFIX', 'METPLUS_GRID_RES', + 'METPLUS_MASK_MISSING_FLAG', 'METPLUS_MATCH_FLAG', 'METPLUS_WEIGHT_DICT', 'METPLUS_NC_PAIRS_FLAG_DICT', @@ -60,6 +61,7 @@ class MODEWrapper(CompareGriddedWrapper): 'METPLUS_CT_STATS_FLAG', 'METPLUS_FCST_FILE_TYPE', 'METPLUS_OBS_FILE_TYPE', + 'METPLUS_MULTIVAR_LOGIC', ] # handle deprecated env vars used pre v4.0.0 @@ -104,11 +106,9 @@ class MODEWrapper(CompareGriddedWrapper): } def __init__(self, config, instance=None): - # only set app variables if not already set by MTD (subclass) - if not hasattr(self, 'app_name'): - self.app_name = 'mode' - self.app_path = os.path.join(config.getdir('MET_BIN_DIR', ''), - self.app_name) + self.app_name = 'mode' + self.app_path = os.path.join(config.getdir('MET_BIN_DIR', ''), + self.app_name) super().__init__(config, instance=instance) def add_merge_config_file(self, time_info): @@ -120,43 +120,67 @@ def add_merge_config_file(self, time_info): def create_c_dict(self): c_dict = super().create_c_dict() + + # variable to substitute for wrapper name, i.e. MODE + tool = self.app_name.upper() + c_dict['VERBOSITY'] = self.config.getstr('config', - 'LOG_MODE_VERBOSITY', + f'LOG_{tool}_VERBOSITY', c_dict['VERBOSITY']) # get the MET config file path or use default c_dict['CONFIG_FILE'] = self.get_config_file('MODEConfig_wrapped') - c_dict['OBS_INPUT_DIR'] = \ - self.config.getdir('OBS_MODE_INPUT_DIR', '') - c_dict['OBS_INPUT_TEMPLATE'] = \ - self.config.getraw('filename_templates', - 'OBS_MODE_INPUT_TEMPLATE') + # MODE can only process a single pair of fcst/obs fields at a time + # unless it is a multi-variate MODE run + mv_logic = self.config.getstr('config', f'{tool}_MULTIVAR_LOGIC', '') + if not mv_logic: + c_dict['ONCE_PER_FIELD'] = True + c_dict['ALLOW_MULTIPLE_FILES'] = False + else: + c_dict['ONCE_PER_FIELD'] = False + c_dict['ALLOW_MULTIPLE_FILES'] = True + self.add_met_config(name='multivar_logic', data_type='string') + self.logger.info(f'{tool}_MULTIVAR_LOGIC was set, so running ' + 'multi-variate MODE') + + # observation input file info + c_dict['OBS_INPUT_DIR'] = ( + self.config.getdir(f'OBS_{tool}_INPUT_DIR', '') + ) + c_dict['OBS_INPUT_TEMPLATE'] = ( + self.config.getraw('config', f'OBS_{tool}_INPUT_TEMPLATE') + ) if not c_dict['OBS_INPUT_TEMPLATE']: - self.log_error('OBS_MODE_INPUT_TEMPLATE must be set') - - c_dict['OBS_INPUT_DATATYPE'] = \ - self.config.getstr('config', 'OBS_MODE_INPUT_DATATYPE', '') - c_dict['FCST_INPUT_DIR'] = \ - self.config.getdir('FCST_MODE_INPUT_DIR', '') - c_dict['FCST_INPUT_TEMPLATE'] = \ - self.config.getraw('filename_templates', - 'FCST_MODE_INPUT_TEMPLATE') + self.log_error(f'OBS_{tool}_INPUT_TEMPLATE must be set') + + c_dict['OBS_INPUT_DATATYPE'] = ( + self.config.getstr('config', f'OBS_{tool}_INPUT_DATATYPE', '') + ) + + # forecast input file info + c_dict['FCST_INPUT_DIR'] = ( + self.config.getdir(f'FCST_{tool}_INPUT_DIR', '') + ) + c_dict['FCST_INPUT_TEMPLATE'] = ( + self.config.getraw('config', f'FCST_{tool}_INPUT_TEMPLATE') + ) if not c_dict['FCST_INPUT_TEMPLATE']: - self.log_error('OBS_MODE_INPUT_TEMPLATE must be set') + self.log_error(f'FCST_{tool}_INPUT_TEMPLATE must be set') + + c_dict['FCST_INPUT_DATATYPE'] = ( + self.config.getstr('config', f'FCST_{tool}_INPUT_DATATYPE', '') + ) - c_dict['FCST_INPUT_DATATYPE'] = \ - self.config.getstr('config', 'FCST_MODE_INPUT_DATATYPE', '') - c_dict['OUTPUT_DIR'] = self.config.getdir('MODE_OUTPUT_DIR', '') + # output info + c_dict['OUTPUT_DIR'] = self.config.getdir(f'{tool}_OUTPUT_DIR', '') if not c_dict['OUTPUT_DIR']: - self.log_error('MODE_OUTPUT_DIR must be set') + self.log_error(f'{tool}_OUTPUT_DIR must be set') c_dict['OUTPUT_TEMPLATE'] = ( - self.config.getraw('config', - 'MODE_OUTPUT_TEMPLATE') + self.config.getraw('config', f'{tool}_OUTPUT_TEMPLATE') ) - c_dict['ONCE_PER_FIELD'] = True self.add_met_config(name='quilt', data_type='bool') @@ -175,9 +199,9 @@ def create_c_dict(self): name='conv_radius', data_type='list', env_var_name=f'METPLUS_{data_type}_CONV_RADIUS', - metplus_configs=[f'{data_type}_MODE_CONV_RADIUS', - f'MODE_{data_type}_CONV_RADIUS', - 'MODE_CONV_RADIUS' + metplus_configs=[f'{data_type}_{tool}_CONV_RADIUS', + f'{tool}_{data_type}_CONV_RADIUS', + f'{tool}_CONV_RADIUS' ], extra_args={ 'remove_quotes': True, @@ -189,9 +213,9 @@ def create_c_dict(self): name='conv_thresh', data_type='list', env_var_name=f'METPLUS_{data_type}_CONV_THRESH', - metplus_configs=[f'{data_type}_MODE_CONV_THRESH', - f'MODE_{data_type}_CONV_THRESH', - 'MODE_CONV_THRESH' + metplus_configs=[f'{data_type}_{tool}_CONV_THRESH', + f'{tool}_{data_type}_CONV_THRESH', + f'{tool}_CONV_THRESH' ], extra_args={ 'remove_quotes': True, @@ -202,9 +226,9 @@ def create_c_dict(self): name='merge_thresh', data_type='list', env_var_name=f'METPLUS_{data_type}_MERGE_THRESH', - metplus_configs=[f'{data_type}_MODE_MERGE_THRESH', - f'MODE_{data_type}_MERGE_THRESH', - 'MODE_MERGE_THRESH' + metplus_configs=[f'{data_type}_{tool}_MERGE_THRESH', + f'{tool}_{data_type}_MERGE_THRESH', + f'{tool}_MERGE_THRESH' ], extra_args={ 'remove_quotes': True, @@ -215,9 +239,9 @@ def create_c_dict(self): name='merge_flag', data_type='string', env_var_name=f'METPLUS_{data_type}_MERGE_FLAG', - metplus_configs=[f'{data_type}_MODE_MERGE_FLAG', - f'MODE_{data_type}_MERGE_FLAG', - 'MODE_MERGE_FLAG' + metplus_configs=[f'{data_type}_{tool}_MERGE_FLAG', + f'{tool}_{data_type}_MERGE_FLAG', + f'{tool}_MERGE_FLAG' ], extra_args={ 'remove_quotes': True, @@ -229,9 +253,9 @@ def create_c_dict(self): name='filter_attr_name', data_type='list', env_var_name=f'METPLUS_{data_type}_FILTER_ATTR_NAME', - metplus_configs=[f'{data_type}_MODE_FILTER_ATTR_NAME', - f'MODE_{data_type}_FILTER_ATTR_NAME', - 'MODE_FILTER_ATTR_NAME' + metplus_configs=[f'{data_type}_{tool}_FILTER_ATTR_NAME', + f'{tool}_{data_type}_FILTER_ATTR_NAME', + f'{tool}_FILTER_ATTR_NAME' ], ) @@ -239,9 +263,9 @@ def create_c_dict(self): name='filter_attr_thresh', data_type='list', env_var_name=f'METPLUS_{data_type}_FILTER_ATTR_THRESH', - metplus_configs=[f'{data_type}_MODE_FILTER_ATTR_THRESH', - f'MODE_{data_type}_FILTER_ATTR_THRESH', - 'MODE_FILTER_ATTR_THRESH' + metplus_configs=[f'{data_type}_{tool}_FILTER_ATTR_THRESH', + f'{tool}_{data_type}_FILTER_ATTR_THRESH', + f'{tool}_FILTER_ATTR_THRESH' ], extra_args={ 'remove_quotes': True, @@ -252,9 +276,9 @@ def create_c_dict(self): name='censor_thresh', data_type='list', env_var_name=f'METPLUS_{data_type}_CENSOR_THRESH', - metplus_configs=[f'{data_type}_MODE_CENSOR_THRESH', - f'MODE_{data_type}_CENSOR_THRESH', - 'MODE_CENSOR_THRESH' + metplus_configs=[f'{data_type}_{tool}_CENSOR_THRESH', + f'{tool}_{data_type}_CENSOR_THRESH', + f'{tool}_CENSOR_THRESH' ], extra_args={ 'remove_quotes': True, @@ -265,12 +289,12 @@ def create_c_dict(self): name='censor_val', data_type='list', env_var_name=f'METPLUS_{data_type}_CENSOR_VAL', - metplus_configs=[f'{data_type}_MODE_CENSOR_VAL', - f'MODE_{data_type}_CENSOR_VAL', - f'{data_type}_MODE_CENSOR_VALUE', - f'MODE_{data_type}_CENSOR_VALUE', - 'MODE_CENSOR_VAL', - 'MODE_CENSOR_VALUE', + metplus_configs=[f'{data_type}_{tool}_CENSOR_VAL', + f'{tool}_{data_type}_CENSOR_VAL', + f'{data_type}_{tool}_CENSOR_VALUE', + f'{tool}_{data_type}_CENSOR_VALUE', + f'{tool}_CENSOR_VAL', + f'{tool}_CENSOR_VALUE', ], extra_args={ 'remove_quotes': True, @@ -281,12 +305,12 @@ def create_c_dict(self): name='vld_thresh', data_type='float', env_var_name=f'METPLUS_{data_type}_VLD_THRESH', - metplus_configs=[f'{data_type}_MODE_VLD_THRESH', - f'MODE_{data_type}_VLD_THRESH', - f'{data_type}_MODE_VALID_THRESH', - f'MODE_{data_type}_VALID_THRESH', - 'MODE_VLD_THRESH', - 'MODE_VALID_THRESH' + metplus_configs=[f'{data_type}_{tool}_VLD_THRESH', + f'{tool}_{data_type}_VLD_THRESH', + f'{data_type}_{tool}_VALID_THRESH', + f'{tool}_{data_type}_VALID_THRESH', + f'{tool}_VLD_THRESH', + f'{tool}_VALID_THRESH' ], ) @@ -298,6 +322,15 @@ def create_c_dict(self): value = self.get_env_var_value(f'METPLUS_{data_type}_{name}') c_dict[f'{data_type}_{name}'] = value + self.add_met_config( + name='mask_missing_flag', + data_type='string', + extra_args={ + 'remove_quotes': True, + 'uppercase': True, + } + ) + self.add_met_config( name='match_flag', data_type='string', @@ -312,12 +345,12 @@ def create_c_dict(self): self.add_met_config(name='total_interest_thresh', data_type='float', - metplus_configs=['MODE_TOTAL_INTEREST_THRESH']) + metplus_configs=[f'{tool}_TOTAL_INTEREST_THRESH']) self.add_met_config( name='max_centroid_dist', data_type='string', - metplus_configs=['MODE_MAX_CENTROID_DIST'], + metplus_configs=[f'{tool}_MAX_CENTROID_DIST'], extra_args={ 'remove_quotes': True, 'default': defaults.get('MAX_CENTROID_DIST') @@ -327,7 +360,7 @@ def create_c_dict(self): name='centroid_dist', data_type='string', env_var_name='INTEREST_FUNCTION_CENTROID_DIST', - metplus_configs=['MODE_INTEREST_FUNCTION_CENTROID_DIST'], + metplus_configs=[f'{tool}_INTEREST_FUNCTION_CENTROID_DIST'], extra_args={ 'remove_quotes': True, 'default': defaults.get('INTEREST_FUNCTION_CENTROID_DIST') @@ -337,7 +370,7 @@ def create_c_dict(self): name='boundary_dist', data_type='string', env_var_name='INTEREST_FUNCTION_BOUNDARY_DIST', - metplus_configs=['MODE_INTEREST_FUNCTION_BOUNDARY_DIST'], + metplus_configs=[f'{tool}_INTEREST_FUNCTION_BOUNDARY_DIST'], extra_args={ 'remove_quotes': True, 'default': defaults.get('INTEREST_FUNCTION_BOUNDARY_DIST') @@ -347,63 +380,54 @@ def create_c_dict(self): name='convex_hull_dist', data_type='string', env_var_name='INTEREST_FUNCTION_CONVEX_HULL_DIST', - metplus_configs=['MODE_INTEREST_FUNCTION_CONVEX_HULL_DIST'], + metplus_configs=[f'{tool}_INTEREST_FUNCTION_CONVEX_HULL_DIST'], extra_args={ 'remove_quotes': True, 'default': defaults.get('INTEREST_FUNCTION_CONVEX_HULL_DIST') } ) - self.add_met_config(name='ps_plot_flag', - data_type='bool') + self.add_met_config(name='ps_plot_flag', data_type='bool') - self.add_met_config(name='ct_stats_flag', - data_type='bool') + self.add_met_config(name='ct_stats_flag', data_type='bool') self.add_met_config(name='file_type', data_type='string', env_var_name='FCST_FILE_TYPE', - metplus_configs=['MODE_FCST_FILE_TYPE', - 'FCST_MODE_FILE_TYPE', - 'MODE_FILE_TYPE'], + metplus_configs=[f'{tool}_FCST_FILE_TYPE', + f'FCST_{tool}_FILE_TYPE', + f'{tool}_FILE_TYPE'], extra_args={'remove_quotes': True, 'uppercase': True}) self.add_met_config(name='file_type', data_type='string', env_var_name='OBS_FILE_TYPE', - metplus_configs=['MODE_OBS_FILE_TYPE', - 'OBS_MODE_FILE_TYPE', - 'MODE_FILE_TYPE'], + metplus_configs=[f'{tool}_OBS_FILE_TYPE', + f'OBS_{tool}_FILE_TYPE', + f'{tool}_FILE_TYPE'], extra_args={'remove_quotes': True, 'uppercase': True}) - c_dict['ALLOW_MULTIPLE_FILES'] = False - c_dict['MERGE_CONFIG_FILE'] = ( - self.config.getraw('config', - 'MODE_MERGE_CONFIG_FILE', '') + self.config.getraw('config', f'{tool}_MERGE_CONFIG_FILE', '') ) - self.handle_mask(single_value=True, - get_flags=True) + self.handle_mask(single_value=True, get_flags=True) # handle setting VERIF_MASK for old wrapped MET config files self.add_met_config(name='poly', data_type='list', env_var_name='METPLUS_VERIF_MASK', - metplus_configs=['MODE_MASK_POLY', - 'MODE_POLY', - ('MODE_' + metplus_configs=[f'{tool}_MASK_POLY', + f'{tool}_POLY', + (f'{tool}_' 'VERIFICATION_MASK_TEMPLATE')], extra_args={'allow_empty': True}) self.env_var_dict['VERIF_MASK'] = ( self.get_env_var_value('METPLUS_VERIF_MASK').strip('[]') ) - - - return c_dict def set_environment_variables(self, time_info): @@ -452,16 +476,19 @@ def run_at_time_one_field(self, time_info, var_info): if obs_path is None: return - # loop over all variables and levels (and probability thresholds) and call the app for each - self.process_fields_one_thresh(time_info, var_info, model_path, obs_path) + # loop over all variables and levels (and probability thresholds) and + # call the app for each + self.process_fields_one_thresh(time_info, var_info, model_path, + obs_path) - def process_fields_one_thresh(self, time_info, var_info, model_path, obs_path): + def process_fields_one_thresh(self, time_info, var_info, model_path, + obs_path): """! For each threshold, set up environment variables and run mode - Args: - @param time_info dictionary containing timing information - @param var_info object containing variable information - @param model_path forecast file - @param obs_path observation file + + @param time_info dictionary containing timing information + @param var_info object containing variable information + @param model_path forecast file + @param obs_path observation file """ # if no thresholds are specified, run once fcst_thresh_list = [] diff --git a/metplus/wrappers/point2grid_wrapper.py b/metplus/wrappers/point2grid_wrapper.py index 267a77740..e21031f5f 100755 --- a/metplus/wrappers/point2grid_wrapper.py +++ b/metplus/wrappers/point2grid_wrapper.py @@ -238,10 +238,10 @@ def set_command_line_arguments(self, time_info): self.args.append(f"-field 'name=\"{input_field}\"; level=\"{input_level}\";'") if self.c_dict['QC_FLAGS']: - self.args.append("-qc") + self.args.append(f"-qc {self.c_dict['QC_FLAGS']}") if self.c_dict['ADP']: - self.args.append("-adp") + self.args.append(f"-adp {self.c_dict['ADP']}") if self.c_dict['REGRID_METHOD']: self.args.append(f"-method {self.c_dict['REGRID_METHOD']}") diff --git a/parm/met_config/MODEConfig_wrapped b/parm/met_config/MODEConfig_wrapped index 6a386c40a..0ddcdceb6 100644 --- a/parm/met_config/MODEConfig_wrapped +++ b/parm/met_config/MODEConfig_wrapped @@ -9,16 +9,19 @@ // // Output model name to be written // +// model = ${METPLUS_MODEL} // // Output description to be written // +// desc = ${METPLUS_DESC} // // Output observation type to be written // +// obtype = ${METPLUS_OBTYPE} //////////////////////////////////////////////////////////////////////////////// @@ -26,6 +29,7 @@ ${METPLUS_OBTYPE} // // Verification grid // +// regrid = { ${METPLUS_REGRID_DICT} //////////////////////////////////////////////////////////////////////////////// @@ -41,8 +45,15 @@ ${METPLUS_GRID_RES} // // Run all permutations of radius and threshold // +// quilt = ${METPLUS_QUILT} +// +// MODE Multivar boolean combination logic +// +//multivar_logic = +${METPLUS_MULTIVAR_LOGIC} + // // Forecast and observation fields to be verified // @@ -81,7 +92,8 @@ obs = { // // Handle missing data // -mask_missing_flag = BOTH; +// mask_missing_flag = +${METPLUS_MASK_MISSING_FLAG} // // Match objects between the forecast and observation fields @@ -183,14 +195,12 @@ fcst_raw_plot = { color_table = "MET_BASE/colortables/met_default.ctable"; plot_min = 0.0; plot_max = 0.0; - colorbar_spacing = 1; } obs_raw_plot = { color_table = "MET_BASE/colortables/met_default.ctable"; plot_min = 0.0; plot_max = 0.0; - colorbar_spacing = 1; } object_plot = { diff --git a/parm/use_cases/met_tool_wrapper/MODE/MODE.conf b/parm/use_cases/met_tool_wrapper/MODE/MODE.conf index 188ff00c6..fa243f518 100644 --- a/parm/use_cases/met_tool_wrapper/MODE/MODE.conf +++ b/parm/use_cases/met_tool_wrapper/MODE/MODE.conf @@ -38,6 +38,8 @@ MODE_CONFIG_FILE = {PARM_BASE}/met_config/MODEConfig_wrapped FCST_VAR1_NAME = RH FCST_VAR1_LEVELS = P500 +#MODE_MULTIVAR_LOGIC = + FCST_MODE_CONV_RADIUS = 5 FCST_MODE_CONV_THRESH = >=80.0 FCST_MODE_MERGE_THRESH = >=75.0 @@ -76,7 +78,6 @@ FCST_MODE_FILE_WINDOW_END = 0 OBS_MODE_FILE_WINDOW_BEGIN = 0 OBS_MODE_FILE_WINDOW_END = 0 - MODE_REGRID_TO_GRID = NONE #MODE_REGRID_METHOD = #MODE_REGRID_WIDTH = @@ -100,6 +101,7 @@ MODE_GRID_RES = 40 #MODE_MASK_POLY = #MODE_MASK_POLY_FLAG = +MODE_MASK_MISSING_FLAG = BOTH #MODE_MATCH_FLAG = #MODE_WEIGHT_CENTROID_DIST = diff --git a/parm/use_cases/met_tool_wrapper/MODE/MODE_python_embedding.conf b/parm/use_cases/met_tool_wrapper/MODE/MODE_python_embedding.conf index 8a3be0ebf..a9a5cb45a 100644 --- a/parm/use_cases/met_tool_wrapper/MODE/MODE_python_embedding.conf +++ b/parm/use_cases/met_tool_wrapper/MODE/MODE_python_embedding.conf @@ -56,6 +56,8 @@ MODE_CONFIG_FILE = {CONFIG_DIR}/MODEConfig_wrapped # See MET User's Guide for more information MODE_REGRID_TO_GRID = NONE +MODE_MASK_MISSING_FLAG = BOTH + MODE_OUTPUT_PREFIX = FCST_vs_OBS # Location of merge config file to pass to the MODE diff --git a/parm/use_cases/model_applications/climate/MODE_fcstCESM_obsGPCP_AsianMonsoonPrecip.conf b/parm/use_cases/model_applications/climate/MODE_fcstCESM_obsGPCP_AsianMonsoonPrecip.conf index 9f61d6108..cdf0c56d8 100644 --- a/parm/use_cases/model_applications/climate/MODE_fcstCESM_obsGPCP_AsianMonsoonPrecip.conf +++ b/parm/use_cases/model_applications/climate/MODE_fcstCESM_obsGPCP_AsianMonsoonPrecip.conf @@ -74,6 +74,8 @@ MODE_WEIGHT_ASPECT_DIFF = 1.0 MODE_REGRID_TO_GRID = FCST +MODE_MASK_MISSING_FLAG = BOTH + [dir] # Directory for CESM data FCST_MODE_INPUT_DIR = {INPUT_BASE}/model_applications/climate/CESM diff --git a/parm/use_cases/model_applications/convection_allowing_models/MODE_fcstFV3_obsGOES_BrightnessTemp.conf b/parm/use_cases/model_applications/convection_allowing_models/MODE_fcstFV3_obsGOES_BrightnessTemp.conf index f8381030d..a9f02d47d 100644 --- a/parm/use_cases/model_applications/convection_allowing_models/MODE_fcstFV3_obsGOES_BrightnessTemp.conf +++ b/parm/use_cases/model_applications/convection_allowing_models/MODE_fcstFV3_obsGOES_BrightnessTemp.conf @@ -39,6 +39,8 @@ MODE_MAX_CENTROID_DIST = 600.0/grid_res MODE_INTEREST_FUNCTION_CENTROID_DIST = ( ( 0.0, 1.0 ) ( 60.0/grid_res, 1.0 ) ( 450.0/grid_res, 0.0 ) ) +MODE_MASK_MISSING_FLAG = BOTH + MODE_FCST_CENSOR_THRESH = <=0 MODE_FCST_CENSOR_VAL = 9999 diff --git a/parm/use_cases/model_applications/convection_allowing_models/MODE_fcstFV3_obsGOES_BrightnessTempObjs.conf b/parm/use_cases/model_applications/convection_allowing_models/MODE_fcstFV3_obsGOES_BrightnessTempObjs.conf index aea3f16ad..9eb9200b1 100644 --- a/parm/use_cases/model_applications/convection_allowing_models/MODE_fcstFV3_obsGOES_BrightnessTempObjs.conf +++ b/parm/use_cases/model_applications/convection_allowing_models/MODE_fcstFV3_obsGOES_BrightnessTempObjs.conf @@ -62,6 +62,7 @@ MODE_FCST_CENSOR_VAL = 9999 MODE_OBS_CENSOR_THRESH = <=0 MODE_OBS_CENSOR_VAL = 9999 +MODE_MASK_MISSING_FLAG = BOTH MODE_MASK_POLY_FLAG = BOTH diff --git a/parm/use_cases/model_applications/convection_allowing_models/MODE_fcstHRRR_obsMRMS_Hail_GRIB2.conf b/parm/use_cases/model_applications/convection_allowing_models/MODE_fcstHRRR_obsMRMS_Hail_GRIB2.conf index d41834257..c236b9e6d 100644 --- a/parm/use_cases/model_applications/convection_allowing_models/MODE_fcstHRRR_obsMRMS_Hail_GRIB2.conf +++ b/parm/use_cases/model_applications/convection_allowing_models/MODE_fcstHRRR_obsMRMS_Hail_GRIB2.conf @@ -46,6 +46,8 @@ MODE_MATCH_FLAG = NO_MERGE MODE_MAX_CENTROID_DIST = 400.0/grid_res +MODE_MASK_MISSING_FLAG = BOTH + MODE_MASK_POLY_FLAG = BOTH MODE_WEIGHT_INTEN_PERC_VALUE = 99 diff --git a/parm/use_cases/model_applications/marine_and_cryosphere/GridStat_MODE_fcstIMS_obsNCEP_sea_ice.conf b/parm/use_cases/model_applications/marine_and_cryosphere/GridStat_MODE_fcstIMS_obsNCEP_sea_ice.conf index bb864122f..5c107dd41 100644 --- a/parm/use_cases/model_applications/marine_and_cryosphere/GridStat_MODE_fcstIMS_obsNCEP_sea_ice.conf +++ b/parm/use_cases/model_applications/marine_and_cryosphere/GridStat_MODE_fcstIMS_obsNCEP_sea_ice.conf @@ -110,6 +110,8 @@ MODE_GRID_RES = 12.7 MODE_OBS_CENSOR_THRESH = >1.00 && <=1.28, >1.28 MODE_OBS_CENSOR_VAL = 1.00 , -9999 +MODE_MASK_MISSING_FLAG = BOTH + MODE_MATCH_FLAG = NO_MERGE MODE_MASK_POLY_FLAG = BOTH diff --git a/parm/use_cases/model_applications/s2s/SeriesAnalysis_fcstCFSv2_obsGHCNCAMS_climoStandardized_MultiStatisticTool.conf b/parm/use_cases/model_applications/s2s/SeriesAnalysis_fcstCFSv2_obsGHCNCAMS_climoStandardized_MultiStatisticTool.conf new file mode 100644 index 000000000..d6e1f475c --- /dev/null +++ b/parm/use_cases/model_applications/s2s/SeriesAnalysis_fcstCFSv2_obsGHCNCAMS_climoStandardized_MultiStatisticTool.conf @@ -0,0 +1,251 @@ +[config] + +PROCESS_LIST = SeriesAnalysis, GenEnsProd, SeriesAnalysis(run_two), GridStat + +### +# Time Info +### + +LOOP_BY = INIT +INIT_TIME_FMT = %Y%m +INIT_BEG=198201 +INIT_END=201002 +INIT_INCREMENT = 1Y + +LEAD_SEQ = + +LOOP_ORDER = processes + +### +# SERIES_ANALYSIS FIELDINFO +### + +SERIES_ANALYSIS_RUNTIME_FREQ = RUN_ONCE +MODEL = CFSv2 +SERIES_ANALYSIS_CUSTOM_LOOP_LIST = 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23 + +BOTH_SERIES_ANALYSIS_VAR1_NAME = fcst +BOTH_SERIES_ANALYSIS_VAR1_LEVELS = "({custom},0,*,*)" +SERIES_ANALYSIS_FCST_FILE_TYPE = NETCDF_NCCF +SERIES_ANALYSIS_OBS_FILE_TYPE = NETCDF_NCCF +SERIES_ANALYSIS_OUTPUT_STATS_CNT = TOTAL, FBAR, FSTDEV +SERIES_ANALYSIS_BLOCK_SIZE = 0 + +### +# File I/O SERIES_ANALYSIS +### + +FCST_SERIES_ANALYSIS_INPUT_DIR = {INPUT_BASE}/model_applications/s2s/SeriesAnalysis_fcstCFSv2_obsGHCNCAMS_climoStandardized_MultiStatisticTool +FCST_SERIES_ANALYSIS_INPUT_TEMPLATE = CFSv2.tmp2m.{init?fmt=%Y%m}.fcst.nc + +OBS_SERIES_ANALYSIS_INPUT_DIR = {FCST_SERIES_ANALYSIS_INPUT_DIR} +OBS_SERIES_ANALYSIS_INPUT_TEMPLATE = {FCST_SERIES_ANALYSIS_INPUT_TEMPLATE} + +SERIES_ANALYSIS_OUTPUT_DIR = {OUTPUT_BASE}/SA_run1 +SERIES_ANALYSIS_OUTPUT_TEMPLATE = mem{custom?fmt=%s}_output.nc + +### +# File I/O Gen_Ens_Prod +### + +GEN_ENS_PROD_INPUT_DIR = {FCST_SERIES_ANALYSIS_INPUT_DIR} + +GEN_ENS_PROD_INPUT_TEMPLATE = {FCST_SERIES_ANALYSIS_INPUT_TEMPLATE} + +#GEN_ENS_PROD_CTRL_INPUT_DIR = +#GEN_ENS_PROD_CTRL_INPUT_TEMPLATE = + +GEN_ENS_PROD_N_MEMBERS = 24 + +GEN_ENS_PROD_OUTPUT_DIR = {OUTPUT_BASE}/GEP +GEN_ENS_PROD_OUTPUT_TEMPLATE = gen_ens_prod_{init?fmt=%Y%m}_ens.nc + +### +# Field Info +### + +ENS_VAR1_NAME = fcst +ENS_VAR1_LEVELS = "(MET_ENS_MEMBER_ID,0,*,*)" +ENS_VAR1_THRESH = <-0.43, >=-0.43&&<=0.43, >0.43 +ENS_FILE_TYPE = NETCDF_NCCF + +### +# GenEnsProd +### + +#LOG_GEN_ENS_PROD_VERBOSITY = 2 + +MODEL = CFSv2 +# GEN_ENS_PROD_DESC = NA + +#GEN_ENS_PROD_REGRID_TO_GRID = NONE +#GEN_ENS_PROD_REGRID_METHOD = NEAREST +#GEN_ENS_PROD_REGRID_WIDTH = 1 +#GEN_ENS_PROD_REGRID_VLD_THRESH = 0.5 +#GEN_ENS_PROD_REGRID_SHAPE = SQUARE + +#GEN_ENS_PROD_CENSOR_THRESH = +#GEN_ENS_PROD_CENSOR_VAL = +GEN_ENS_PROD_NORMALIZE = CLIMO_STD_ANOM +#GEN_ENS_PROD_CAT_THRESH = +#GEN_ENS_PROD_NC_VAR_STR = + +GEN_ENS_PROD_ENS_THRESH = 0.3 +GEN_ENS_PROD_VLD_THRESH = 0.3 + +#GEN_ENS_PROD_NBRHD_PROB_WIDTH = 5 +#GEN_ENS_PROD_NBRHD_PROB_SHAPE = CIRCLE +#GEN_ENS_PROD_NBRHD_PROB_VLD_THRESH = 0.0 + +#GEN_ENS_PROD_NMEP_SMOOTH_VLD_THRESH = 0.0 +#GEN_ENS_PROD_NMEP_SMOOTH_SHAPE = CIRCLE +#GEN_ENS_PROD_NMEP_SMOOTH_GAUSSIAN_DX = 81.27 +#GEN_ENS_PROD_NMEP_SMOOTH_GAUSSIAN_RADIUS = 120 +#GEN_ENS_PROD_NMEP_SMOOTH_METHOD = GAUSSIAN +#GEN_ENS_PROD_NMEP_SMOOTH_WIDTH = 1 + +GEN_ENS_PROD_CLIMO_MEAN_FILE_NAME = {SERIES_ANALYSIS_OUTPUT_DIR}/memMET_ENS_MEMBER_ID_output.nc +GEN_ENS_PROD_CLIMO_MEAN_FIELD = {name="series_cnt_FBAR"; level="(*,*)";} +#GEN_ENS_PROD_CLIMO_MEAN_REGRID_METHOD = +#GEN_ENS_PROD_CLIMO_MEAN_REGRID_WIDTH = +#GEN_ENS_PROD_CLIMO_MEAN_REGRID_VLD_THRESH = +#GEN_ENS_PROD_CLIMO_MEAN_REGRID_SHAPE = +#GEN_ENS_PROD_CLIMO_MEAN_TIME_INTERP_METHOD = +#GEN_ENS_PROD_CLIMO_MEAN_MATCH_MONTH = +#GEN_ENS_PROD_CLIMO_MEAN_DAY_INTERVAL = 31 +#GEN_ENS_PROD_CLIMO_MEAN_HOUR_INTERVAL = 6 + +GEN_ENS_PROD_CLIMO_STDEV_FILE_NAME = {SERIES_ANALYSIS_OUTPUT_DIR}/memMET_ENS_MEMBER_ID_output.nc +GEN_ENS_PROD_CLIMO_STDEV_FIELD = {name="series_cnt_FSTDEV"; level="(*,*)";} +#GEN_ENS_PROD_CLIMO_STDEV_REGRID_METHOD = +#GEN_ENS_PROD_CLIMO_STDEV_REGRID_WIDTH = +#GEN_ENS_PROD_CLIMO_STDEV_REGRID_VLD_THRESH = +#GEN_ENS_PROD_CLIMO_STDEV_REGRID_SHAPE = +#GEN_ENS_PROD_CLIMO_STDEV_TIME_INTERP_METHOD = +#GEN_ENS_PROD_CLIMO_STDEV_MATCH_MONTH = +#GEN_ENS_PROD_CLIMO_STDEV_DAY_INTERVAL = 31 +#GEN_ENS_PROD_CLIMO_STDEV_HOUR_INTERVAL = 6 + +GEN_ENS_PROD_ENSEMBLE_FLAG_LATLON = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_MEAN = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_STDEV = TRUE +#GEN_ENS_PROD_ENSEMBLE_FLAG_MINUS = TRUE +#GEN_ENS_PROD_ENSEMBLE_FLAG_PLUS = TRUE +#GEN_ENS_PROD_ENSEMBLE_FLAG_MIN = TRUE +#GEN_ENS_PROD_ENSEMBLE_FLAG_MAX = TRUE +#GEN_ENS_PROD_ENSEMBLE_FLAG_RANGE = TRUE +#GEN_ENS_PROD_ENSEMBLE_FLAG_VLD_COUNT = TRUE +GEN_ENS_PROD_ENSEMBLE_FLAG_FREQUENCY = TRUE +#GEN_ENS_PROD_ENSEMBLE_FLAG_NEP = FALSE +#GEN_ENS_PROD_ENSEMBLE_FLAG_NMEP = FALSE +#GEN_ENS_PROD_ENSEMBLE_FLAG_CLIMO = FALSE +#GEN_ENS_PROD_ENSEMBLE_FLAG_CLIMO_CDF = FALSE + +GEN_ENS_PROD_ENS_MEMBER_IDS = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23 +#GEN_ENS_PROD_CONTROL_ID = + +### +# File I/O Grid_Stat +### + +FCST_GRID_STAT_INPUT_DIR = {GEN_ENS_PROD_OUTPUT_DIR} +FCST_GRID_STAT_INPUT_TEMPLATE = {GEN_ENS_PROD_OUTPUT_TEMPLATE} + +OBS_GRID_STAT_INPUT_DIR = {INPUT_BASE}/model_applications/s2s/SeriesAnalysis_fcstCFSv2_obsGHCNCAMS_climoStandardized_MultiStatisticTool +OBS_GRID_STAT_INPUT_TEMPLATE = ghcn_cams.1x1.1982-2020.mon.nc + +GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE}/GridStat +GRID_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m} +GRID_STAT_OUTPUT_PREFIX = {init?fmt=%Y%m} + + +### +# Field Info GridStat +### + +FCST_GRID_STAT_VAR1_NAME = fcst_0_0_all_all_ENS_FREQ_lt-0.43 +FCST_GRID_STAT_VAR1_LEVELS = "(*,*)" +FCST_GRID_STAT_VAR1_THRESH = ==0.1 +FCST_GRID_STAT_IS_PROB = True + +OBS_GRID_STAT_VAR1_NAME = tmp2m +OBS_GRID_STAT_VAR1_LEVELS = "({init?fmt=%Y%m%d_%H%M%S},*,*)" +OBS_GRID_STAT_VAR1_THRESH = <=CDP33 +OBS_GRID_STAT_FILE_TYPE = NETCDF_NCCF + + +### +# Field Info for GridStat +### + +GRID_STAT_CLIMO_MEAN_FILE_NAME = {INPUT_BASE}/model_applications/s2s/SeriesAnalysis_fcstCFSv2_obsGHCNCAMS_climoStandardized_MultiStatisticTool/ghcn_cams.1x1.1982-2010.mon.clim.nc +GRID_STAT_CLIMO_MEAN_FIELD = {name="clim"; level="(0,*,*)";} +GRID_STAT_CLIMO_MEAN_FILE_TYPE = NETCDF_NCCF + + +GRID_STAT_CLIMO_STDEV_FILE_NAME = {INPUT_BASE}/model_applications/s2s/SeriesAnalysis_fcstCFSv2_obsGHCNCAMS_climoStandardized_MultiStatisticTool/ghcn_cams.1x1.1982-2010.mon.stddev.nc +GRID_STAT_CLIMO_STDEV_FIELD = {name="stddev"; level="(0,*,*)";} +GRID_STAT_CLIMO_STDEV_FILE_TYPE = NETCDF_NCCF + +GRID_STAT_REGRID_TO_GRID = FCST +GRID_STAT_OUTPUT_FLAG_PSTD = BOTH +GRID_STAT_NC_PAIRS_FLAG_APPLY_MASK = TRUE +GRID_STAT_NC_PAIRS_FLAG_RAW = TRUE + +[run_two] +### +# FILE I/O of SeriesAnalysis run_two +### + +FCST_SERIES_ANALYSIS_INPUT_DIR = {OUTPUT_BASE}/GEP +FCST_SERIES_ANALYSIS_INPUT_TEMPLATE = gen_ens_prod_{init?fmt=%Y%m}_ens.nc + +OBS_SERIES_ANALYSIS_INPUT_DIR = {INPUT_BASE}/model_applications/s2s/SeriesAnalysis_fcstCFSv2_obsGHCNCAMS_climoStandardized_MultiStatisticTool +OBS_SERIES_ANALYSIS_INPUT_TEMPLATE = ghcn_cams.1x1.1982-2020.mon.nc + +SERIES_ANALYSIS_OUTPUT_DIR = {OUTPUT_BASE}/SA_run2 +SERIES_ANALYSIS_OUTPUT_TEMPLATE = {INIT_BEG}to{INIT_END}_CFSv2_SA.nc + +### +# Field Info for SeriesAnalysis run_two +### +# +#These first entries are empty to override the intial SeriesAnalysis call +#SERIES_ANALYSIS_CUSTOM_LOOP_LIST = + +#BOTH_SERIES_ANALYSIS_VAR1_NAME = +#BOTH_SERIES_ANALYSIS_VAR1_LEVELS = + +FCST_SERIES_ANALYSIS_VAR1_NAME = fcst_0_0_all_all_ENS_FREQ_lt-0.43 +FCST_SERIES_ANALYSIS_VAR1_LEVELS = "(*,*)" + +FCST_CAT_THRESH = ==0.1 +FCST_IS_PROB = True + +OBS_SERIES_ANALYSIS_VAR1_NAME = tmp2m +OBS_SERIES_ANALYSIS_VAR1_LEVELS = "({init?fmt=%Y%m%d_%H%M%S},*,*)" +OBS_SERIES_ANALYSIS_CAT_THRESH = <=CDP33 + +OBS_FILE_TYPE = NETCDF_NCCF + +### +# SeriesAnalysis General for run_two +### + +SERIES_ANALYSIS_REGRID_TO_GRID = FCST +SERIES_ANALYSIS_OUTPUT_STATS_PSTD = TOTAL, BRIER, RELIABILITY, BRIERCL, BSS +SERIES_ANALYSIS_VLD_THRESH = 0.5 + +SERIES_ANALYSIS_BLOCK_SIZE = 0 + +SERIES_ANALYSIS_CLIMO_MEAN_FILE_NAME = {INPUT_BASE}/model_applications/s2s/SeriesAnalysis_fcstCFSv2_obsGHCNCAMS_climoStandardized_MultiStatisticTool/ghcn_cams.1x1.1982-2010.mon.clim.nc +SERIES_ANALYSIS_CLIMO_MEAN_FIELD = {name="clim"; level="(0,*,*)";} +SERIES_ANALYSIS_CLIMO_MEAN_FILE_TYPE = NETCDF_NCCF + + +SERIES_ANALYSIS_CLIMO_STDEV_FILE_NAME = {INPUT_BASE}/model_applications/s2s/SeriesAnalysis_fcstCFSv2_obsGHCNCAMS_climoStandardized_MultiStatisticTool/ghcn_cams.1x1.1982-2010.mon.stddev.nc +SERIES_ANALYSIS_CLIMO_STDEV_FIELD = {name="stddev"; level="(0,*,*)";} +SERIES_ANALYSIS_CLIMO_STDEV_FILE_TYPE = NETCDF_NCCF +SERIES_ANALYSIS_RUNTIME_FREQ = RUN_ONCE + +SERIES_ANALYSIS_RUN_ONCE_PER_STORM_ID = False diff --git a/parm/use_cases/model_applications/s2s/UserScript_obsPrecip_obsOnly_Hovmoeller.conf b/parm/use_cases/model_applications/s2s/UserScript_obsPrecip_obsOnly_Hovmoeller.conf index c729d81d6..ed4ef9111 100644 --- a/parm/use_cases/model_applications/s2s/UserScript_obsPrecip_obsOnly_Hovmoeller.conf +++ b/parm/use_cases/model_applications/s2s/UserScript_obsPrecip_obsOnly_Hovmoeller.conf @@ -19,8 +19,9 @@ LOG_FILE = "Hovmoeller_diagram.log" LOG_LEVEL = "INFO" -INPUT_FILE_NAME = {INPUT_BASE}/model_applications/s2s/UserScript_obsPrecip_obsOnly_Hovmoeller/precip.erai.sfc.1p0.2x.2014-2016.nc YAML_CONFIG_NAME = {PARM_BASE}/use_cases/model_applications/s2s/UserScript_obsPrecip_obsOnly_Hovmoeller/hovmoeller.yaml +INPUT_FILE_NAME = {INPUT_BASE}/model_applications/s2s/UserScript_obsPrecip_obsOnly_Hovmoeller/precip.erai.sfc.1p0.2x.2014-2016.nc + METPLOTPY_BASE = {METPLUS_BASE}/../METplotpy OUTPUT_DIR = {OUTPUT_BASE}/plots diff --git a/parm/use_cases/model_applications/s2s/UserScript_obsPrecip_obsOnly_Hovmoeller/hovmoeller.yaml b/parm/use_cases/model_applications/s2s/UserScript_obsPrecip_obsOnly_Hovmoeller/hovmoeller.yaml index a66945af7..33f977031 100644 --- a/parm/use_cases/model_applications/s2s/UserScript_obsPrecip_obsOnly_Hovmoeller/hovmoeller.yaml +++ b/parm/use_cases/model_applications/s2s/UserScript_obsPrecip_obsOnly_Hovmoeller/hovmoeller.yaml @@ -1,11 +1,14 @@ -height: 800 -width: 1200 +plot_filename: !ENV '${OUTPUT_DIR}/erai_precip2.png' +input_data_file: !ENV '${INPUT_FILE_NAME}' +plot_height: 800 +plot_width: 1200 title: ERAI Precipitation -font_size: 20 +xy_label_font_size: 20 +title_size: 20 -xaxis_title: Longitude -yaxis_title: Time +xaxis: Longitude +yaxis: Time date_start: 2016-01-01 date_end: 2016-03-31 @@ -21,5 +24,3 @@ contour_min: 0.2 contour_max: 1.6 contour_del: 0.2 colorscale: 'BuPu' -plot_filename: -- !ENV '${OUTPUT_DIR}/erai_precip2.png' diff --git a/parm/use_cases/model_applications/s2s/UserScript_obsPrecip_obsOnly_Hovmoeller/hovmoeller_diagram.py b/parm/use_cases/model_applications/s2s/UserScript_obsPrecip_obsOnly_Hovmoeller/hovmoeller_diagram.py index fcdf72c5d..22dd96727 100755 --- a/parm/use_cases/model_applications/s2s/UserScript_obsPrecip_obsOnly_Hovmoeller/hovmoeller_diagram.py +++ b/parm/use_cases/model_applications/s2s/UserScript_obsPrecip_obsOnly_Hovmoeller/hovmoeller_diagram.py @@ -5,86 +5,8 @@ """ import os -import sys -import logging -import yaml -import xarray as xr # http://xarray.pydata.org/ import metplotpy.plots.hovmoeller.hovmoeller as Hovmoeller -import metcalcpy.util.read_env_vars_in_config as readconfig - - -def main(): - """ - Use existing default Hovmoeller config file found int METplotpy to - create a default plot, using data in the METplus data store - """ - - """ - Read METplus config file paramaters - """ - input_file_name = os.environ.get("INPUT_FILE_NAME","precip.erai.sfc.1p0.2x.2014-2016.nc") - plot_config_file = os.getenv("YAML_CONFIG_NAME","hovmoeller.yaml") - input_file = input_file_name - - """ - Read Hovmoeller YAML configuration file - """ - try: - config = readconfig.parse_config(plot_config_file) - logging.info(config) - except yaml.YAMLError as exc: - logging.error(exc) - - - """ - Setup logging - """ - logfile = "Hovmoeller_diagram.log" - logging_level = os.environ.get("LOG_LEVEL","logging.INFO") - logging.basicConfig(stream=logfile, level=logging_level) - - # Plot and save difficulty index figures - """ - Read dataset - """ - try: - logging.info('Opening ' + input_file) - ds = xr.open_dataset(input_file) - except IOError as exc: - logging.error('Unable to open ' + input_file) - logging.error(exc) - sys.exit(1) - logging.debug(ds) - - data = ds[config['var_name']] - logging.debug(data) - - data = data.sel(time=slice(config['date_start'], config['date_end'])) - time = ds.time.sel(time=slice(config['date_start'], config['date_end'])) - lon = ds.lon - - data = data * config['unit_conversion'] - data.attrs['units'] = config['var_units'] - - plot_filename = config['plot_filename'][0] - print("Plot name",plot_filename) - logging.info(plot_filename) - - # create output directory if it does not exist - plot_dir = os.path.dirname(plot_filename) - if not os.path.exists(plot_dir): - logging.info(f"Creating output directory: {plot_dir}") - os.makedirs(plot_dir) - - custom_param_dict = {"plot_filename": plot_filename} - plot = Hovmoeller.Hovmoeller(custom_param_dict, time, lon, data) - plot.save_to_file() - - # check if output file exists since save_to_file doesn't return - # an error code on failure - if not os.path.exists(plot_filename): - logging.error(f"Could not create output file: {plot_filename}") - sys.exit(1) if __name__ == '__main__': - main() + config_file = os.environ.get('YAML_CONFIG_NAME') + Hovmoeller.main(config_file) diff --git a/scripts/docker/hooks/build b/scripts/docker/hooks/build index 378a386bb..472551c10 100644 --- a/scripts/docker/hooks/build +++ b/scripts/docker/hooks/build @@ -4,4 +4,14 @@ met_tag=`$(dirname $DOCKERFILE_PATH)/hooks/get_met_version` echo $met_tag -docker build -t $IMAGE_NAME --build-arg SOURCE_VERSION=$SOURCE_BRANCH --build-arg MET_TAG=$met_tag . + +MET_DOCKER_REPO=met +if [ "$met_tag" == "develop" ]; then + MET_DOCKER_REPO=met-dev +fi + +docker build -t $IMAGE_NAME \ + --build-arg SOURCE_VERSION=$SOURCE_BRANCH \ + --build-arg MET_TAG=$met_tag \ + --build-arg MET_DOCKER_REPO=$MET_DOCKER_REPO \ + .