From 8221969789d5ef014c9b5cf9831cfebf97c0f6d5 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 20 Apr 2022 12:42:17 -0600 Subject: [PATCH] Update Develop-ref after #1577 (#1579) * thru SPEED #1049 * thru S #1049 * thru T #1049 * thru U #1049 * thru V #1049 * thru V #1049 * thru Z #1049 * Update statistics_list.rst Tara is testing editing in UI * Update statistics_list.rst Updates through the A's * Update statistics_list.rst Cleaning up the A's * Update statistics_list.rst Standardizing MODE and MTD entries * Update statistics_list.rst Updating B's and C's * Update statistics_list.rst Testing adding TC-Stat and TCST to an entry * feature 1252 allow dictionary value for time_summary.width (#1253) * Update statistics_list.rst Clean up of a few A-Cs and then update of Ds * Update statistics_list.rst A few clean-ups and Es * Update statistics_list.rst Halfway through Fs... * feature 1213 obs_quality_inc/exc (#1260) * Feature 1203 ioda2nc (#1262) * Update statistics_list.rst A little clean up and the rest of Fs * Update statistics_list.rst G, H, I, Ks * Update statistics_list.rst L, M, Ns * Update statistics_list.rst A little clean-up and Os * Update statistics_list.rst A little clean-up and Os * Update statistics_list.rst Rs * Update statistics_list.rst S and Ts * Update statistics_list.rst The rest of the list * Update statistics_list.rst Removed Attr from Stat Type thru E * Update statistics_list.rst Remove Attr from Statistics Type through Gs * Update statistics_list.rst Remove Attr from Statistic Type through Rs * Update statistics_list.rst Remove Attr from Stat Type to the end * Update statistics_list.rst Cleaned up some Line Type typos * Update statistics_list.rst Still more Attr cleanup * Add default title for the new use case issue template. * Feature 1019 harmonic preprocessing (#1272) Co-authored-by: George McCabe <23407799+georgemccabe@users.noreply.github.com> * Feature 1266 gen ens prod missing ensembles (#1275) * Feature 1049 statistics list (#1271) * first attempt at a table #1049 * 2nd attempt, smaller, different formatting table #1049 * trying to get rid of git error message by adding this to the index #1049 * alignment #1049 * trying to text wrap in a table #1049 * attempting to create text wrapping in tables #1049 * removing role, trying to wrap text #1049 * removing html break, trying to wrap text #1049 * adding code to wrap text #1049 * removing code to wrap text #1049 * fixing naming typo #1049 * hitting returns in table #1049 * removing typo file, removing line breaks #1049 * attempting a simple table for text wrapping #1049 * attempting to bold header row in simple table #1049 * simple table line break attempt #1049 * simple table line break attempt #2 #1049 * simple table line break attempt #3 #1049 * trying another | #1049 * 2 trying another | #1049 * 3 trying another | #1049 * trying a blank line #1049 * trying reformating grid #1049 * 2 trying reformating grid #1049 * 3 trying reformating grid #1049 * 4 trying reformating grid #1049 * adding in one more table row #1049 * adding more text #1049 * trying glossary format #1049 * 2 trying glossary format #1049 * 3 trying glossary format #1049 * 4 trying glossary format #1049 * adding glossary format 2D objects #1049 * fixing glossary format 2D objects #1049 * adding glossary items #1049 * fixing glossary format #1049 * hopefully all examples are working #1049 * changing reference to tool * removing 2D object examples * removing the unwrapped table * adding ACC into the example * fixing formatting #1049 * fixing formatting take 2 #1049 * fixing formatting take 3 #1049 * bolding and language change #1049 * fixing spacing #1049 * fixing spacing again #1049 * changing title #1049 * trying superscript #1049 * superscript glossary #1049 * removing quotes #1049 * fixing spacing #1049 * Tara has decided to go with the glossary format. Removing table example. #1049 * Adding 2D objects #1049 * 2D objects fix #1049 * 2D objects fix attempt 2 #1049 * 2D objects fix attempt 3 #1049 * 2D objects fix attempt 4 #1049 * 2D objects fix attempt 5 #1049 * 2D objects fix attempt 7 #1049 * 2D objects fix attempt 8 #1049 * ABR added #1049 * adding remaining AB items to list #1049 * adding remaining ACC_ items to list #1049 * ADLAND & AFSS entries #1049 * A entries #1049 * fixing formatting #1049 * fixing formatting #2 #1049 * removing new entries to make it run #1049 * adding a couple back in #1049 * adding a couple more back in #1049 * tool key at the bottom #1049 * spacing changes #1049 * spacing changes another try #1049 * fixing spacing #1049 * fixing spacing attempt 2 #1049 * fixing spacing attempt 3 #1049 * through AREA items #1049 * AMODEL listed twice but it's not #1049 * AMODEL listed twice but it's not #2 #1049 * AMODEL added question marks so it will run #1049 * final A entries #1049 * naming the glossary statistics in hopes of not conflicting with original glossary #1049 * removing glossary statistics space #1049 * Testing multiple glossary names * Per #1067, working on attempts to have multiple glossaries with the same term * Per #1067, working on attempts to have multiple glossaries with the same term * Per #1067, working on attempts to have multiple glossaries with the same term * Per #1067, removing code with attempts at mutiple glossaries * Per #1067, adding back three question marks due to duplicate term * paring down list, adding a table for review * problems with table * still trying to get it to publish * still trying to get a second line in the table * still trying to get a second line in the table 2 * testing different formatting * testing different formatting * trying to comment out the glossary inner workings * formatting again * line breaks in table #1049 * adding in some more for an example #1049 * adding new entries in through AREA #1049 * fixing line breaks #1049 * fixing line breaks #1049 #2 * fixing line breaks and warning messages #1049 * fixing line breaks #1049 * adding AREA_RATIO through ASPECT_DIFF * fixing typo #1049 * fixing typo #1049 * fixing typo #1049 take 2 * adding AXIS_ANG to BCMSE #1049 * adding spacing #1049 * adding spacing removing commas #1049 * removing comma #1049 * BOUNDARY_DIST thru BSS_SMPL #1049 * BOUNDARY_DIST splitting across 2 lines #1049 * cleaning up typos #1049 * calibration thru centriod_dist #1049 * centriod_lat thru centroid_y #1049 * fixing spacing #1049 * removing test glossary #1049 * climo_mean thru crtk_err #1049 * fixing crtk_err spacing #1049 * fixing spacing #1049 * CSI to CURVATURE_Y #1049 * CURVATURE_X & Y spacing #1049 * DEV_CAT to DURATION_DIFF #1049 * EC_VALUE to F #1049 * F_RATE TO FBS #1049 * Fixing spacing #1049 * fcst_clus thru fcst_conv_radius #1049 * removing CTOP_PRS #1049 * fixing the order of tools for FBAR and FBIAS #1049 * fixing spacing #1049 * adding grid-stat to all point-stat entries #1049 * adding fixing spacing #1049 * adding fixing spacing take 2 #1049 * adding fixing spacing take 3 #1049 * adding fixing spacing take 4 #1049 * adding fixing spacing take 5 #1049 * adding fixing spacing take 6 #1049 * adding fixing spacing take 7 #1049 * fixing spacing with a period take 7 #1049 * first attempt fcst_ #1049 * fixing typos #1049 * thru FOBAR #1049 * thru end of F #1049 * g thru h #1049 * i thru intensity #s #1049 * fixing typos #1049 * capturing example for Julie #1049 * thru k #1049 * fixing typos #1049 * thru L #1049 * thru MG #1049 * thru N_ENS #1049 * thru all N #1049 * fixing FBIAS alignment #1049 * fixing ME and MSE alignment #1049 * fixing ME alignment take 2 #1049 * thru OBS_E #1049 * fixing alignment #1049 * fixing alignment n_thresh #1049 * thru OBS_thresh #1049 * thru O #1049 * fixing OOBAR formating #1049 * thru PR_CORR #1049 * commented lines out with line total info #1049 * commented lines out with line total info take 2 #1049 * thru R #1049 * thru SPEED #1049 * thru S #1049 * thru T #1049 * thru U #1049 * thru V #1049 * thru V #1049 * thru Z #1049 * Update statistics_list.rst Tara is testing editing in UI * Update statistics_list.rst Updates through the A's * Update statistics_list.rst Cleaning up the A's * Update statistics_list.rst Standardizing MODE and MTD entries * Update statistics_list.rst Updating B's and C's * Update statistics_list.rst Testing adding TC-Stat and TCST to an entry * Update statistics_list.rst Clean up of a few A-Cs and then update of Ds * Update statistics_list.rst A few clean-ups and Es * Update statistics_list.rst Halfway through Fs... * Update statistics_list.rst A little clean up and the rest of Fs * Update statistics_list.rst G, H, I, Ks * Update statistics_list.rst L, M, Ns * Update statistics_list.rst A little clean-up and Os * Update statistics_list.rst A little clean-up and Os * Update statistics_list.rst Rs * Update statistics_list.rst S and Ts * Update statistics_list.rst The rest of the list * Update statistics_list.rst Removed Attr from Stat Type thru E * Update statistics_list.rst Remove Attr from Statistics Type through Gs * Update statistics_list.rst Remove Attr from Statistic Type through Rs * Update statistics_list.rst Remove Attr from Stat Type to the end * Update statistics_list.rst Cleaned up some Line Type typos * Update statistics_list.rst Still more Attr cleanup Co-authored-by: Julie Prestopnik Co-authored-by: TaraJensen * creating a separate list for diagnostics, in progesss * removing diagnostics from statistics, in progess #1049 * adding diagnostics_list to TOC #1049 * separating lists * fixing typos #1049 * moving blank statistic type from statistics to diagnostics #1049 * Feature 1263 v4.1.0 beta4 (#1277) * update version to note development towards beta5 * fixing errors #1049 * updating table name #1049 * fixing typo #1049 * Feature 934 release stage doc (#1235) * Per #934 add stages of the METplus release cycle. * Per #934, adding link to descriptions of the release cycle in the User's Guide. * Per #934, made corrections * Per #934, changed Beta and Release Candidate (rc) from bold to subsubsections. * Update index.rst Co-authored-by: Julie Prestopnik * putting both tables into one chapter * removing diagnostics since it's going back into the statistics chapter #1049 * Delete diagnostics_list.rst This table will be added to the statistics page as a separate table. #1049 * test. breaking out directories A-B and C-D #1049 * testing with section names #1049 * alphbetical sections for statistics #1049 * making U-Z statistics list #1049 * alpabetical breaks for diagnostics list #1049 * trying to fix errors #1049 * trying to fix errors take 2 #1049 * trying to fix errors take 3 #1049 * Removed a unnecessary space #1049 * Making all METplus Names CAPITAL LETTERS #1049 * Feature 344 met util refactor (#1292) * removed deprecated sections from config examples * minor change to METplus release guide to add a link to the PDF of the User's Guide instead of downloading it and attaching it to the release * Feature 1285 extract tiles mtd times (#1315) * Feature 896 more met config (#1322) * removed incorrect search keyword * added workflow_dispatch event so workflow can be triggered by an external repository such as MET to test to ensure that changes from that repo will break anything in METplus * added another input argument for workflow_dispatch event * added job with name that shows the event name or the repository name if triggered by an external repository such as MET * GHA: add username that triggered external event to event info job name * added required input argument for external trigger that contains the commit hash of the push event that triggered in the other repo * change event info to show commit hash instead of username that merged the PR * changed input names to match names of event in repository that triggered workflow * feature 1320 OMP_NUM_THREADS (#1338) * Feature 1183 memory documentation (#1340) Co-authored-by: George McCabe <23407799+georgemccabe@users.noreply.github.com> * add email address of user who triggered push event to job name * Feature 1166 series analysis field info (#1353) * Feature 1116 usecase smos (#1348) Co-authored-by: Mrinal Biswas * turn off new use case from every push * feature 1236 Control Members in EnsembleStat and GenEnsProd (#1357) * added optional argument to change the directory to untar new input data into so the same Dockerfile can be used to add data for other METplus components such as MET * feature 1358 v4.1.0-beta5 release (#1359) * update version for next development cycle * Feature 1216 usecase smap (#1361) * Adding a conf file for SMAP * Adding a directory to host the read file * Removing temp file * Updated the valid dates to match Todd's code * Adding documentation for SMAP case * Updates the valis dates to match Todd's code * Removing a tmp file * Typo in file name * Updating the input RTOFS to have the init time instead of the valid time ii the file name * updated file paths, tesing * updated use case descriptions, rearranged use case group testing * put new use case into its own group so that the diff logic can evaluate marine_and_cryosphere:3. The truth data for 3-4 does not exist yet so the diff fails. Co-authored-by: Mrinal Biswas Co-authored-by: George McCabe <23407799+georgemccabe@users.noreply.github.com> * updated marine_and_cryo grouping * Feature 1230 stratosphere metrics (#1354) * Initial checkin for Meridial Mean use case * Changed the name and directories * Added some documentation * Fixed config variables * Issue 1230 Stratospheric metrics (zonal/meridional use case) put INPUT_BASE in the user_env_vars so the user doesn't need to set the INPUT_BASE environment in the current working shell * Issue #1230 remove import of metplotpy-this isn't called * Issue #1230_stratosphere_metric Add use case to the list of use cases to be run for testing * Issue 1230 Remove the INPUT_BASE from the user_env_vars section * Issue #1230 redundant files * Issue #1230 redundant files * Issue #1230 redundant files * Issue #120 replace INPUT_BASE with INPUT_FILE_NAME * Issue #1230 remove entry for INPUT_FILE_NAME, this goes in the system.conf * Update all_use_cases.txt Copy and pasted #11 from s2s use case for the Stratosphere use case but forgot to update the index to 12. * Update use_case_groups.json added use case #12 from s2s to test stratosphere use case * Issue #1230 put INPUT_FILE_NAME back under the user_env_vars * Issue #1230 forgot to include the filename * Issue #1230 clean up config file, remove uneccessary comments, group related entries in the user_env_vars * Issue #1230 remove extraneous and incorrect path to the input_filename setting * Removed pingouin dependency * Issue #1230 another cut and paste error fixed for the Stratosphere use case * Issue #1230 type in name, obs_Only should be obsOnly * Issue #1230 added the metdatadb to the env, code imports metdatadb and may require some of these dependencies * issue #1230 removed pingouin dependency from comment to reduce any confusion * issue #1230 turn off the test for the Stratosphere metrics use case * Added use case image Co-authored-by: Hank Fisher Co-authored-by: Minna Win Co-authored-by: bikegeek <3753118+bikegeek@users.noreply.github.com> Co-authored-by: Christina Kalb * Corrected spelling of occurrence in two places * Fixed misspelling of occurrence * Found and fixed two more misspellings of occurence * Changed version specific information to by X.Y.Z moved text indicating to click save to the bottom of the list. * Feature 1374 python packages (#1378) * Added documentation about updating spreadsheet of Python requirements * Changed references to master_metplus.py to run_metplus.py * Fixed formatting of section with run_metplus.py commands * Updated text for updating the spreadsheet * Removed a newly added section that wasn't needed Co-authored-by: Julie Prestopnik * feature 1368 PCPCombine use zero accum (#1381) * feature 1369 grid_weight_flag in EnsembleStat (#1379) * Per #1356, change how wrappers create instances of other wrappers to ensure that config settings for the created instance do not change values in METplusConfig used by the rest of the wrappers, ci-run-all-diff * Per #1356, remove config_overrides functionality in favor of using instances -- this prevents the issue where overrides for a given tool affect the global config settings. it also forces the configs to override for an instance to be put into another config section so that it will be available in the final conf, ci-run-all-diff * Per #1356, update documentation to reflect changes * Per #1356, fixed unit test to no longer use deprecated approach to overriding config variables * feature 1247 climatology settings (#1385) * feature 1356 isolate config (#1386) * Feature 675 Continuous Integration Documentation (#1409) Co-authored-by: johnhg * feature 1382 Filename templates in MODEL (#1414) * Feature 1289 explicit file list (#1387) * Per PR #1387 review, fix typo * Bugfix 1421 EnsembleStat use fcst fields in ens dictionary if ens is unset (#1422) * fixed missing path change for docker file path * removed variables that are not used * Feature 1408 use case ptype (#1425) * adding imagery, config file, updating internal tests * updated lists, auto testing * corrected old reference * Update use_case_groups.json * only trigger testing workflow for pull requests that are going into develop or main_* branches * Feature 1371 blocking wr (#1426) * turn off use cases for push events * Feature 1392 climo_cdf.direct_prob and SeriesAnalysis (#1396) * Changed event triggering rules for documentation workflow so that it will run for pull requests even if no files in the docs directory have changed. A recent pull request passed but develop failed after it was approved because files that were referenced in the use case documentation files had been deleted as part of the PR. * feature 1273 TCGen -edeck and -shape (#1424) * Feature 966 Fix mask.poly logic (#1430) * Feature 1030 total seconds documentation (#1428) Co-authored-by: George McCabe <23407799+georgemccabe@users.noreply.github.com> * Update UserScript_fcstGFS_obsERA_Blocking.py * Update UserScript_fcstGFS_obsERA_Blocking.py * Update UserScript_fcstGFS_obsERA_Blocking.py * Update UserScript_obsERA_obsOnly_Blocking.py * Update UserScript_fcstGFS_obsERA_WeatherRegime.py * Update UserScript_obsERA_obsOnly_WeatherRegime.py * Update UserScript_obsERA_obsOnly_WeatherRegime.py * Update UserScript_fcstGFS_obsERA_Blocking.py * Update UserScript_fcstGFS_obsERA_WeatherRegime.py * Update UserScript_obsERA_obsOnly_Blocking.py * Update UserScript_obsERA_obsOnly_WeatherRegime.py * Update UserScript_fcstGFS_obsERA_Blocking.py * Update UserScript_fcstGFS_obsERA_WeatherRegime.py * Update UserScript_obsERA_obsOnly_Blocking.py * Update UserScript_obsERA_obsOnly_WeatherRegime.py * Update UserScript_fcstGFS_obsERA_Blocking.py * Update UserScript_fcstGFS_obsERA_WeatherRegime.py * Update UserScript_fcstGFS_obsERA_WeatherRegime.py * Update UserScript_fcstGFS_obsERA_Blocking.py * Update UserScript_obsERA_obsOnly_Blocking.py * Update UserScript_obsERA_obsOnly_WeatherRegime.py * get branch name in job controls job * replace logic to update input data volumes with call to dtcenter/metplus-action-data-update action * feature 1431 remove obs_prepbufr_map (#1433) * Per #675, added images and updated content * added statsmodel package to metplotpy docker conda environment * Added fcst_file_type to two wrappers (#1437) Co-authored-by: Hank Fisher Co-authored-by: George McCabe <23407799+georgemccabe@users.noreply.github.com> * added release notes for tasks that have already been completed for beta6 * updated env var for METplotpy base to match changes made to the METplotpy repository * Updated METplotpy base env var to preemptively match changes from PR dtcenter/metplotpy#197. Once that PR is merged into develop, a test workflow will run that should fix this issue. * removed commented code that is no longer used * update the instructions for updating test input data to ensure volume_mount_directories file is updated in the upcoming version directory, i.e. v4.1, so that the Docker data volumes from the release will be created properly * removed duplicate script * updated script to copy volume_mount_directories file into new METplus version directory * added logic to skip files that are temporary copies of tar files that should not be added to new version directory * updated release guide instructions for creating METplus data directory for next development cycle * Feature 911 tc trackint (#1439) Co-authored-by: Mrinal Biswas Co-authored-by: George McCabe <23407799+georgemccabe@users.noreply.github.com> * added new use cases that are currently being reviewed to release notes * updated event info in contributor's guide * Feature 626 fix hovmoeller documentation (#1444) * Update UserScript_obsPrecip_obsOnly_Hovmoeller.py replace hovmoeller_diagram.conf with UserScript_obsPrecip_obsOnly_Hovmoeller.conf * Update UserScript_obsPrecip_obsOnly_Hovmoeller.py Removed a typo: UserScript This uses data from... Now: This uses data from... * Update UserScript_obsPrecip_obsOnly_Hovmoeller.py * Modified the PR template to add review of the source issue metadata ci-skip-all * feature_1403_python_components_table (#1446) * creating new METplus Components Python Requirements * adding a return to test * testing use case links * testing use case links #2 * testing use case links #3 * testing use case links #4 * Filling in some use cases * fixing indentation #1403 * fixing indentation #1403 take 2 * fixing indentation #1403 take 3 * cleaning up anything after # in the links #1403 * entries thru D #1403 * entries thru H #1403 * fixing indents #1403 * fixing indents #1403 take 2 * fixing web name #1403 * entries thru L #1403 * entry matplotlib #1403 * entry matplotlib #1403 marking duplicate names * entry metcalcpy #1403 * entry metcalcpy fix indents #1403 * entry matplotlib #1403 fixing duplicate names * entry metplotpy #1403 * entry metplotpy #1403 marking duplicate names * entry metplotpy #1403 fixing * entry thru ps #1403 * double names #1403 * character fix #1403 * adding in missing links #1403 * thru pyproj #1403 * problem with line 353 #1403 * thru pyproj #1403 typos * more duplicate names #1403 * line break #1403 * loose ends #1403 * adding >= to some of the versions #1403 * adding name to the Use Cases to avoid duplicate naming issues TEST #1403 * adding name to the Use Cases to avoid duplicate naming issues thru pyproj #1403 * adding thru pyresample #1403 * fixing pyresample typo #1403 * adding thru scikit-image #1403 * fixing line breaks#1403 * web links thru scikit-learn #1403 * web links thru scipy #1403 * web links thru sklearn #1403 * Duplicate web names. temp. fix #1403 * Duplicate web names fixed #1403 * Duplicate web names fixing #3 #1403 * Duplicate web names fixing #4 #1403 * Duplicate web names fixing #5 #1403 * table end #1403 * table loose ends #1403 * Removed unnecessary comma * table loose ends after pull request #1403 * Minor modifications to METplus Component column * Changed description of pyproj * Added link for last referenced use case * Attempting to fix doc warning Co-authored-by: jprestop * Feature 1318 usecase aviso (#1440) * Correct small typo. * Update pull_request_template.md * Added files for ReadTheDocs (#864) * Updated documentation links for ReadtheDocs * Update pull_request_template.md Added entry for completion date for pull request review. * Adding python embedding script to read AVISO, RTOFS, HYCOM and OSTIA data * Updated the script - working but only RMSE values match * Adding documentation for the AVISO case * Adding a conf file for the AVISO case * Updates to the codel to include model3, obs, climo3 but also has many extra print statements, netcdf output files which needs to be cleaned * Cleaning up the conf file and adding SAL1L2 to get the model_climo sdev and obs_climo sdev * Updating the file to fix the standard names and add the model3, climo3 and obs3 * Updating the file to fix the standard names and add the model3, climo3 and obs3 * Adding an image to the use case. Image Courtesy: EMC/NOAA website * added to use case list * updated case description * removed merge issues * caught second docs issue * removed other doc merge issues, turned off auto test * more merge comments sneaking in Co-authored-by: johnhg Co-authored-by: jprestop Co-authored-by: Mrinal Biswas * Revert "Feature 1318 usecase aviso (#1440)" (#1459) This reverts commit d3021deccbedb15bbb40599659c1bbd4f912002e. * Feature 1449 v4.1.0 beta6 (#1461) * Per #1449, updated version as specified in the Release Guide * Per #1449, updated release notes * Updated release date for beta6 release Co-authored-by: Julie Prestopnik * updated version number after beta release * Feature 1318 usecase aviso attempt 2 (#1473) Co-authored-by: johnhg Co-authored-by: jprestop Co-authored-by: Mrinal Biswas Co-authored-by: j-opatz * error if no input template is set for GridDiag wrapper * added comment for new logic, ci-run-diff * updated old section name to config, ci-run-all-diff * fixed bug with replacing wildcard characters with 'all' in output file paths * Revert "fixed bug with replacing wildcard characters with 'all' in output file paths" This reverts commit c11711e29d7451c3d996907d72fac93ca62192a5. * fixed bug with replacing wildcard characters with 'all' in output file paths, ci-run-all-diff * feature 1102 -out for TCStat and StatAnalysis (#1477) * feature 1464 EnsembleStat probabilistic additions (#1479) * feature 1445 normalize in GenEnsProd (#1478) * added logic to add quotes around grid value if they are not set in conf variable value, moved error check for unset grid to init step, ci-run-all-diff * bugfix 1486 PB2NC file window variables not read (#1487) * added build arg to override MET DockerHub repository so development version tests can use dtcenter/met-dev:develop * set MET DockerHub repo based on develop or stable version of METplus * run diff logic for push events to develop and main_v* branches so we can catch if changes to external repositories caused differences in the output * do not run diff logic on main_v* and develop branch push events -- diff logic is already running when external repository triggers workflow, which is what we need to test with the diff logic * Added description for Use Cases * updated script to set up new release input data directories -- use real path for all symbolic links for consistency, added usage statement * changed logic to error if new version directory to create already exists at all -- previous logic only errors if directory is not empty * Changed all filename template tags for datetime objects (init/valid/da_init/etc.) to use %H instead of %2H. We discovered that Python 3.6.8 interprets %2H as the same as %H (correctly substituting the hour), however Python 3.8.8 interprets %2H as 2H. The number in between the % and letter is not supported by strftime so it is not valid for these datetime values. The logic to zero-pad using this notation was written in the METplus string substitution logic and only applies to lead and level template tags. ci-run-all-diff * feature 1274 TCGen use case using -shape/-genesis (#1492) Co-authored-by: Kathryn Newman Co-authored-by: j-opatz <59586397+j-opatz@users.noreply.github.com> * feature 1490 Python Embedding for Point Observations (#1491) * update use case groupings for automated tests * removed extra comma typo, ci-run-all-diff * added missing characters in commmands * Feature 1403 python components table (#1497) Co-authored-by: Lisa Goodrich Co-authored-by: Julie Prestopnik * testing new statistics provided by Tara #1049 * testing new statistics #2 #1049 * New statistics from Tara #1049 * New statistics from Tara small corrections #1049 * New statistics from Tara small corrections #2 #1049 * New statistics from Tara small moving to Diagnostics table #1049 * Fixing space-time, like that's possible #1049 * Fixing spacing #1049 * feature 1471 fix current field info in output_prefix (#1500) * Feature 1503 v4.1.0 rc1 (#1504) * update version after rc1 release to reflect development towards 4.1.0 release * Corrected documentation title name * bugfix 1486 pb2nc window variables (#1507) * feature 1443 document probabilistic field settings (#1508) Co-authored-by: johnhg * feature 943 document referencing current level (#1509) * feature 1510 v4.1.0 Release (#1511) * updated release guide to reflect updated process to trigger reference branch GHA workflow for an official release * update version to reflect development towards 5.0.0-beta1 release * Feature 673 MET (#1517) * set MET docker repo to dtcenter/met if forcing a specific MET tag to use since that is where the stable release versions are stored on DockerHub * pass variable to tell docker build script that an external workflow triggered the run and use -lite if so to ensure that MET docker image that was created by the triggering MET workflow is used * echo output variables from job control job so they can be reviewed in the GHA log output * minor fix for consistency in logs, ci-skip-all * Feature 1399 usecase cable (#1538) * Adding initial code to read AOML Cable Transport and RTOFS cable transport variables * Updating the conf file (no stat analysis yet) and updates to the read file * Updating the conf file to add STAT_ANALYSIS * Updates to the code- adding stats * Updated code--runs now but needs to follow User Script conventions * Changed directory name based on MET conventions * Updating the read file and adding a conf file * Added logic to count the number of directories in the RTOFS dir * Updated to run with UserScript conf * update version for official release * Adding a yaml file to specify start_date * Added the yaml file and cleaned up the conf file * Added logic to read the yaml file, cleaned up the script * Updating the confile to add output dir, input template and clean up * Removing yaml, getting vDate from conf file, adding output file * Adding the doc file for the cable use case * Adding image courtesy NOAA * Adding more documentation * Adding more info * Updated docs * Correcting the python embedding script * Updating to see if the error goes away..no clue * Fixing the doc error * modifed docs, testing * updated permissions for script * turned off always testing * Updated version number * Update use_case_groups.json * Removed erroneous file * Apply suggestions from code review * Update .github/parm/use_case_groups.json Co-authored-by: George McCabe <23407799+georgemccabe@users.noreply.github.com> * Update docs/use_cases/model_applications/marine_and_cryosphere/UserScript_fcstRTOFS_obsAOML_calcTransport.py Co-authored-by: George McCabe <23407799+georgemccabe@users.noreply.github.com> * Update docs/use_cases/model_applications/marine_and_cryosphere/UserScript_fcstRTOFS_obsAOML_calcTransport.py Co-authored-by: George McCabe <23407799+georgemccabe@users.noreply.github.com> * Update docs/use_cases/model_applications/marine_and_cryosphere/UserScript_fcstRTOFS_obsAOML_calcTransport.py Co-authored-by: George McCabe <23407799+georgemccabe@users.noreply.github.com> Co-authored-by: Mrinal Biswas Co-authored-by: George McCabe <23407799+georgemccabe@users.noreply.github.com> * Update UserScript_fcstGFS_obsERA_WeatherRegime.conf Fixed a title typo * Feature component links (#1518) * Feature 770 3rd person (#1564) * changing from 2nd to 3rd person and fixing some spacing #770 * missed we and corrected #770 * typos, spacing etc #770 * spelling, italics and bolding #770 * removing you changing to 3rd person #770 * changing to 3rd person #770 * 3rd person already done. Fixing formatting, typos etc. #770 * 3rd person already done. Fixing formatting #770 * changing to 3rd person, fixing typos, spacing, etc. #770 * changing to 3rd person, fixing typos, grammar and punctuation. #770 * google doc corrections. #770 * clean up #770 * removing | #770 * typo #770 * Fixing TOC and coding block #770 * fixing note #770 * one 3rd person fix. Updating bolding and italics for file names and directories #770 * Modified Python Code Analyis Tools * Modified Python Code Analyis Tools * Removed link for PROCESS_LIST. If we can't have an internal link, we do not want one in this case. * fixing space #770 * fixing conflict #2 #770 * fixing typos #770 * fixing typos #2 #770 * directory formatting #2 #770 * Changed sub process to subprocess * Modified usage of user because this text is for developers. Made other minor changes. Co-authored-by: Julie Prestopnik * bugfix 1555 develop clear args properly (#1562) * Fixed bug in classification text file (#1568) Co-authored-by: Lisa Goodrich Co-authored-by: TaraJensen Co-authored-by: johnhg Co-authored-by: Christina Kalb Co-authored-by: lisagoodrich <33230218+lisagoodrich@users.noreply.github.com> Co-authored-by: Julie Prestopnik Co-authored-by: jprestop Co-authored-by: j-opatz <59586397+j-opatz@users.noreply.github.com> Co-authored-by: Mrinal Biswas Co-authored-by: j-opatz Co-authored-by: bikegeek Co-authored-by: Hank Fisher Co-authored-by: Hank Fisher Co-authored-by: Minna Win Co-authored-by: bikegeek <3753118+bikegeek@users.noreply.github.com> Co-authored-by: Christina Kalb Co-authored-by: Hank Fisher Co-authored-by: Mrinal Biswas Co-authored-by: Kathryn Newman --- docs/Contributors_Guide/basic_components.rst | 104 +++++--- docs/Contributors_Guide/coding_standards.rst | 109 +++++--- docs/Contributors_Guide/conda_env.rst | 50 ++-- docs/Contributors_Guide/create_wrapper.rst | 105 ++++++-- docs/Contributors_Guide/deprecation.rst | 40 ++- docs/Contributors_Guide/github_workflow.rst | 242 ++++++++++-------- docs/Contributors_Guide/testing.rst | 41 ++- docs/index.rst | 21 +- .../arw-fer-gep1/d01_2009123112_04800.grib | 0 .../arw-fer-gep5/d01_2009123112_04800.grib | 0 .../arw-sch-gep2/d01_2009123112_04800.grib | 0 .../arw-sch-gep6/d01_2009123112_04800.grib | 0 .../arw-tom-gep3/d01_2009123112_04800.grib | 0 .../arw-tom-gep7/d01_2009123112_04800.grib | 0 .../arw-fer-gep1/d01_2009123118_04800.grib | 0 .../arw-fer-gep5/d01_2009123118_04800.grib | 0 .../arw-sch-gep2/d01_2009123118_04800.grib | 0 .../arw-sch-gep6/d01_2009123118_04800.grib | 0 .../arw-tom-gep3/d01_2009123118_04800.grib | 0 .../arw-tom-gep7/d01_2009123118_04800.grib | 0 .../gen_ens_prod/test_gen_ens_prod_wrapper.py | 43 ++-- .../pytests/tc_gen/test_tc_gen_wrapper.py | 13 +- .../pytests/user_script/test_user_script.py | 28 +- metplus/util/met_util.py | 6 +- metplus/wrappers/command_builder.py | 11 +- metplus/wrappers/compare_gridded_wrapper.py | 3 +- metplus/wrappers/pb2nc_wrapper.py | 4 +- metplus/wrappers/runtime_freq_wrapper.py | 63 +++-- metplus/wrappers/tc_gen_wrapper.py | 1 + metplus/wrappers/tcrmw_wrapper.py | 1 + ...erScript_fcstGFS_obsERA_WeatherRegime.conf | 10 +- .../WeatherRegime_driver.py | 26 +- 32 files changed, 574 insertions(+), 347 deletions(-) create mode 100644 internal_tests/data/ens/2009123112/arw-fer-gep1/d01_2009123112_04800.grib create mode 100644 internal_tests/data/ens/2009123112/arw-fer-gep5/d01_2009123112_04800.grib create mode 100644 internal_tests/data/ens/2009123112/arw-sch-gep2/d01_2009123112_04800.grib create mode 100644 internal_tests/data/ens/2009123112/arw-sch-gep6/d01_2009123112_04800.grib create mode 100644 internal_tests/data/ens/2009123112/arw-tom-gep3/d01_2009123112_04800.grib create mode 100644 internal_tests/data/ens/2009123112/arw-tom-gep7/d01_2009123112_04800.grib create mode 100644 internal_tests/data/ens/2009123118/arw-fer-gep1/d01_2009123118_04800.grib create mode 100644 internal_tests/data/ens/2009123118/arw-fer-gep5/d01_2009123118_04800.grib create mode 100644 internal_tests/data/ens/2009123118/arw-sch-gep2/d01_2009123118_04800.grib create mode 100644 internal_tests/data/ens/2009123118/arw-sch-gep6/d01_2009123118_04800.grib create mode 100644 internal_tests/data/ens/2009123118/arw-tom-gep3/d01_2009123118_04800.grib create mode 100644 internal_tests/data/ens/2009123118/arw-tom-gep7/d01_2009123118_04800.grib diff --git a/docs/Contributors_Guide/basic_components.rst b/docs/Contributors_Guide/basic_components.rst index 487d89a66..c386c03f7 100644 --- a/docs/Contributors_Guide/basic_components.rst +++ b/docs/Contributors_Guide/basic_components.rst @@ -44,7 +44,20 @@ found in metplus/wrappers/ascii2nc_wrapper.py. The class name should always be the item that is passed into the METplus configuration variable list PROCESS_LIST with 'Wrapper' at the end. -CommandBuilder's initialization function sets default values for instance variables, initializes the CommandRunner object (used to execute shell commands), and calls the create_c_dict() function. This function is found in CommandBuilder but it is also implemented by most (eventually all) wrappers. The wrapper implementations start off by calling the parent's version of create_c_dict using super(), then adding additional dictionary items that are specific to that wrapper and finally returning the dictionary that was created. If possible, all of the calls to the 'get' functions of the cMETplusConfig object should be found in the create_c_dict function. This allows the configuration values to be referenced throughout the wrapper without the redundantly referencing the wrapper name (i.e. ASCII2NC_INPUT_DIR can be referenced as INPUT_DIR in ASCII2NC since we already it pertains to ASCII2NC) It also makes it easier to see which configuration variables are used in each wrapper. +CommandBuilder's initialization function sets default values for instance +variables, initializes the CommandRunner object (used to execute shell +commands), and calls the create_c_dict() function. This function is found +in CommandBuilder but it is also implemented by most (eventually all) +wrappers. The wrapper implementations start off by calling the parent's +version of create_c_dict using super(), then adding additional dictionary +items that are specific to that wrapper and finally returning the dictionary +that was created. If possible, all of the calls to the 'get' functions of the +cMETplusConfig object should be found in the create_c_dict function. This +allows the configuration values to be referenced throughout the wrapper +without the redundantly referencing the wrapper name (i.e. ASCII2NC_INPUT_DIR +can be referenced as INPUT_DIR in ASCII2NC since it already pertains to +ASCII2NC) It also makes it easier to see which configuration variables are +used in each wrapper. create_c_dict (ExampleWrapper):: @@ -85,10 +98,13 @@ isOK class variable isOK is defined in CommandBuilder (ush/command_builder.py). Its function is to note a failed process while not stopping a parent process. -Instead of instantly exiting a larger wrapper script once one sub process has failed we -want all of the processes to attempt to be executed and then note which ones failed. +Instead of instantly exiting a larger wrapper script once one subprocess has +failed this allows all of the processes to attempt to be executed and +then note which ones failed. -At the end of the wrapper initialization step, all isOK=false will be collected and reported. Execution of the wrappers will not occur unless all wrappers in the process list are initialized correctly. +At the end of the wrapper initialization step, all isOK=false will be +collected and reported. Execution of the wrappers will not occur unless all +wrappers in the process list are initialized correctly. .. code-block:: python @@ -127,38 +143,57 @@ See ush/pb2nc_wrapper.py for an example. run_all_times function ====================== -run_all_times loops over a series of times calling run_at_time for one process for each time -Defined in CommandBuilder but overridden in a wrappers that process all of the data from every run time at once. +run_all_times loops over a series of times calling run_at_time for one +process for each time. Defined in CommandBuilder but overridden in +wrappers that process all of the data from every run time at once. -See SeriesByLeadWrapper (ush/series_by_lead_wrapper.py) for an example of overridding the function +See SeriesByLeadWrapper (ush/series_by_lead_wrapper.py) for an example of +overriding the function. get_command function ==================== -get_command assembles a MET command with arguments that can be run via the shell or the wrapper. -It is defined in CommandBuilder but is overridden in most wrappers because the command line arguments differ for each MET tool. +get_command assembles a MET command with arguments that can be run via the +shell or the wrapper. +It is defined in CommandBuilder but is overridden in most wrappers because +the command line arguments differ for each MET tool. set_environment_variables function ================================== -Uses add_env_var function (CommandBuilder) to set any shell environment variables that MET or other METplus wrappers -need to be set. This allows a wrapper to pass information into a MET configuration file. The MET config file refers to the environment variables. -This is currently only set in wrappers that use MET config files, but the other wrappers will also need to set environment variables -that are needed to be set in the environment when running, such as MET_TMP_DIR and MET_PYTHON_EXE. +Uses add_env_var function (CommandBuilder) to set any shell environment +variables that MET or other METplus wrappers +need to be set. This allows a wrapper to pass information into a MET +configuration file. The MET config file refers to the environment variables. +This is currently only set in wrappers that use MET config files, but the +other wrappers will also need to set environment variables +that are needed to be set in the environment when running, such as +MET_TMP_DIR and MET_PYTHON_EXE. find_data/find_model/find_obs functions (in CommandBuilder) =========================================================== -find_* uses the c_dict directory templates and then queries the file system to find the files you are looking for -uses c_dict dictionary items [FCST/OBS]_FILE_WINDOW_[BEGIN/END], [FCST/OBS]_INPUT_[DIR/TEMPLATE], etc. -If [FCST/OBS]_FILE_WINDOW_[BEGIN/END] are non-zero, these functions will list all files under [FCST/OBS]_INPUT_DIR and use [FCST/OBS]_INPUT_TEMPLATE to extract out time information from each file to determine which files within the file window range should be used. Some tools allow multiple files to be selected. If a tool does not allow multiple files, the file closest to the valid time is returned. If multiple files are the same distance from the valid time, the first file that was found is used. -If a wrapper can read in multiple files, the c_dict item 'ALLOW_MULTIPLE_FILES' should be set to True. +These find_* functions use the c_dict directory templates, queries +the file system to find files, and use c_dict dictionary items +like [FCST/OBS]_FILE_WINDOW_[BEGIN/END], [FCST/OBS]_INPUT_[DIR/TEMPLATE], +etc. +If [FCST/OBS]_FILE_WINDOW_[BEGIN/END] are non-zero, these functions will +list all files under [FCST/OBS]_INPUT_DIR and use [FCST/OBS]_INPUT_TEMPLATE +to extract out time information from each file to determine which files +within the file window range should be used. Some tools allow multiple +files to be selected. If a tool does not allow multiple files, the file +closest to the valid time is returned. If multiple files are the same +distance from the valid time, the first file that was found is used. +If a wrapper can be read in multiple files, the c_dict item +'ALLOW_MULTIPLE_FILES' should be set to True. do_string_sub function ====================== -do_string_sub is found in ush/string_template_substitution.py and is the critical function for substituting the placeholder -values in templates with the actual values needed for running a particular wrapper +do_string_sub is found in ush/string_template_substitution.py and is the +critical function for substituting the placeholder +values in templates with the actual values needed for running a particular +wrapper tc_pairs_wrapper has a good example @@ -172,13 +207,17 @@ tc_pairs_wrapper has a good example bdeck_glob = os.path.join(self.c_dict['BDECK_DIR'], bdeck_file) -time_info is a dictionary of current run time information that can be substituted into the template. See the 'Time Utilities' section for more information. +time_info is a dictionary of current run time information that can be +substituted into the template. See the 'Time Utilities' section for more +information. Time Utilities ============== -time_util is a collection of functions to handle the idosyncracies of working with valid, initialization and observation times. -METplus creates a dictionary containing the current time and either init or valid time:: +time_util is a collection of functions to handle the idiosyncrasies of working +with valid, initialization and observation times. +METplus creates a dictionary containing the current time and either init or +valid time:: input_dict = {} input_dict['now'] = clock_time_obj @@ -188,7 +227,9 @@ METplus creates a dictionary containing the current time and either init or vali else: input_dict['valid'] = loop_time -The forecast lead is also set if provided ('lead'). This dictionary is passed into time_util's ti_calculate function, which determines the other time values that were not provided:: +The forecast lead is also set if provided ('lead'). This dictionary is +passed into time_util's ti_calculate function, which determines the other +time values that were not provided:: >>> import time_util >>> import datetime @@ -196,16 +237,18 @@ The forecast lead is also set if provided ('lead'). This dictionary is passed in >>> time_util.ti_calculate(input_dict) {'lead': 10800, 'offset': 0, 'init': datetime.datetime(1987, 2, 1, 6, 0), 'valid': datetime.datetime(1987, 2, 1, 9, 0), 'loop_by': 'init', 'da_init': datetime.datetime(1987, 2, 1, 9, 0), 'init_fmt': '19870201060000', 'da_init_fmt': '19870201090000', 'valid_fmt': '19870201090000', 'lead_string': '3 hours', 'lead_hours': 3, 'lead_minutes': 180, 'lead_seconds': 10800, 'offset_hours': 0, 'date': datetime.datetime(1987, 2, 1, 9, 0), 'cycle': datetime.datetime(1987, 2, 1, 9, 0)} -Items that will be parsed from the input dictionary are: now, init, valid, lead, lead_seconds, lead_minutes, lead_hours, offset, offset_hours, da_init +Items that will be parsed from the input dictionary are: now, init, valid, +lead, lead_seconds, lead_minutes, lead_hours, offset, offset_hours, da_init -pcp_combine uses a variety of time_util functions like ti_calculate and ti_get_lead_string +pcp_combine uses a variety of time_util functions like ti_calculate and +ti_get_lead_string Adding Support for MET Configuration Variables ============================================== The METplus wrappers utilize environment variables to override values in the MET configuration files. There are functions in CommandBuilder that can be -used to easily add support for override MET configuration variables that did +used to easily add support for overriding MET configuration variables that were not previously supported in METplus configuration files. There is a utility that can be used to easily see what changes are needed to @@ -243,7 +286,7 @@ should be set. * name: Name of the variable to set, i.e. model * data_type: Type of variable. Valid options are int, string, list, float, - bool, and thresh + bool, and thresh. * metplus_configs: List of METplus configuration variable names that should be checked. Variable names are checked in order that they appear in the list. If any of the variables are set in the config object, then the value will be @@ -305,11 +348,12 @@ data type, extra info, children, and nicknames. set by the METplus config variable GRID_STAT_MASK_POLY. However, in older versions of the METplus wrappers, the variable used was GRID_STAT_VERIFICATION_MASK_TEMPLATE. To preserve support for this name, the - nickname can be set to [f'{self.app_name.upper()}_VERIFICATION_MASK_TEMPLATE'] and the old variable + nickname can be set to + [f'{self.app_name.upper()}_VERIFICATION_MASK_TEMPLATE'] and the old variable will be checked if GRID_STAT_MASK_POLY is not set. Values must be set to None to preserve the order. -For example, if you need to define a nickname but no extra info or children, +For example, to define a nickname but no extra info or children, then use: ('string', None, None, ['NICKNAME1]). If a complex MET configuration dictionary is used by multiple MET tools, then @@ -326,7 +370,7 @@ CompareGriddedWrapper and is used by GridStat, PointStat, and EnsembleStat:: This function handles setting the climo_cdf dictionary. The METplus config variable that fits the format {APP_NAME}_{DICTIONARY_NAME}_{VARIABLE_NAME}, i.e. GRID_STAT_CLIMO_CDF_CDF_BINS for GridStat's climo_cdf.cdf_bins, is -quieried first. However, this default name is a little redundant, so adding +queried first. However, this default name is a little redundant, so adding the nickname 'GRID_STAT_CLIMO_CDF_BINS' allows the user to set the variable GRID_STAT_CLIMO_CDF_BINS instead. diff --git a/docs/Contributors_Guide/coding_standards.rst b/docs/Contributors_Guide/coding_standards.rst index 675880c0f..99bac0442 100644 --- a/docs/Contributors_Guide/coding_standards.rst +++ b/docs/Contributors_Guide/coding_standards.rst @@ -1,7 +1,8 @@ .. _codingstandards: +**************** Coding Standards -================ +**************** @@ -14,81 +15,105 @@ Coding Standards `Sphinx `_ for documentation * **NOTE: Please do not use f-strings in the run_metplus.py file so that the Python version check can notify the user of the incorrect version. Using Python 3.5 or earlier will output the SyntaxError from the f-string instead of the useful error message.** -Python code analysis tools --------------------------- +Python Code Analysis Tools +========================== -Static: +Static Tools +------------ pylint ^^^^^^ -:: - pip install pylint or conda install pylint +`pylint `_ is a tool that checks +for errors in Python code, tries to enforce a coding standard and looks for code +smells. - pylint.org +To `install pylint `_ +the following can be run: - checks for code errors +.. code-block:: ini + + pip install pylint + +or + +.. code-block:: ini + + conda install pylint + + +To check for errors as well as PEP-8 style code, run: + +.. code-block:: ini + + pylint pep8 + +replacing with the name of the file to analyze. - pylint pep8 code-to-analyze will check for errors as well as PEP-8 style code pyflakes ^^^^^^^^ -:: +`pyflakes `_ is a simple program which +checks Python source files for errors. Pyflakes analyzes programs and +detects various errors. It works by parsing the source file, not importing +it, so it is safe to use on modules with side effects. It’s also much faster. - pip install pyflakes or conda install pyflakes - parses code rather than importing code, therefore OK to use on modules with side-effects +To install pyflakes the following can be run: - checks for code errors +.. code-block:: ini - faster than pylint + pip install pyflakes - https://pypi.python.org/pypi/pyflakes +or - flake8 is wrapper to pyflakes, performs PEP-8 style checking in addition to error checking - http://flake8.pycqa.org/en/latest/index.html#quickstart +.. code-block:: ini -vulture -^^^^^^^ + conda install pyflakes -:: - checks for unused imports, variables, methods, classed ie "dead code" +`flake8 `_ is wrapper +to pyflakes, performs PEP-8 style checking in addition to error checking. - pip install vulture or conda install vulture +vulture +^^^^^^^ - https://pypi.python.org/pypi/vulture +`vulture `_ finds unused code in Python +programs and is useful for cleaning up and finding errors in large code bases. +It checks for unused imports, variables, methods, and classes. -Dynamic (run-time): +To install vulture the following can be run: +.. code-block:: ini -cpde-coverage analysis -^^^^^^^^^^^^^^^^^^^^^^ + pip install vulture -Useful when running unit tests to determine whether tests are executing all possible branches, loops, etc. +or -figleaf -^^^^^^^ +.. code-block:: ini -http://darcs.idyll.org/~t/projects/figleaf/doc/ -Checking for God objects and God methods: + conda install vulture -(from Chapter 4 of "How to Make Mistakes in Python", Mike Pirnat) -:: +Dynamic (run-time) Tools +------------------------ - find . -name "*.py" -exec wc -l {} \; | sort -g -r - for all Python source files, order by size - anything over 1000 lines is worth investigating (as general rule of thumb) +Code Coverage Analysis +^^^^^^^^^^^^^^^^^^^^^^ - grep "^class " mybigmodule.py |wc -l - counts the number of classes defined in mybigmodule.py +Code coverage analysis tools are useful when running unit tests to determine +whether tests are executing all possible branches, loops, etc. - grep "\sdef " mybigmodule.py |wc -l +**Examples:** - counts the number of methods defined within a class or other function (ie at some level of indentation) in mybigmodule.py +`Coverage.py `_: A free tool for +monitoring the coverage of your Python apps, monitoring every bit of your code +to find what was executed and what was not. - try this if the above doesn't work: grep "def " mybigmodule.py |wc -l +`pytest-cov `_: A free language plug-in +to produce a coverage report of your app. - A high ratio of methods to classes warrants investigation (what constitutes a high ratio- 10:1, 5:1???) +`figleaf `_: A code coverage analysis +tool intended to be to be a minimal replacement of 'coverage.py' that supports +more configurable coverage gathering and reporting. diff --git a/docs/Contributors_Guide/conda_env.rst b/docs/Contributors_Guide/conda_env.rst index 8aed31a25..4a732bc92 100644 --- a/docs/Contributors_Guide/conda_env.rst +++ b/docs/Contributors_Guide/conda_env.rst @@ -8,13 +8,13 @@ _______________________________________________________________________ If the host (i.e. the computer on which the METplus and MET tools are running) doesn't already have all the necessary packages installed, it is -recommend that METplus be run within a conda environment. This enables +recommended that METplus be run within a conda environment. This enables the user to manage dependencies and to isolate the METplus version of Python from other Python applications. To replicate the packages used in the development of METplus, use the -environment.yml file to recreate the Python environment used by METplus. -This file is found at the top-level dirctory of the +**environment.yml** file to recreate the Python environment used by METplus. +This file is found at the top-level directory of the METplus source code: *METplus/environment.yml* @@ -30,18 +30,19 @@ _______________________________________________________________________ (this is a minimal installer that contains the package manager, *conda* that has Python 3 and other dependent packages.) - If it doesn't exist on the computer, download it `here `_. + If it doesn't exist on the computer, download it + `here `_. -| - 2. Select the appropriate Python 3.7 (or newer) version for the operating system and computer's architecture. - Follow the `installation instructions `_. + Follow the + `installation instructions `_. .. note:: - It may be worth considering changing the default installation location to a partition with more space. + It may be worth considering changing the default installation location + to a partition with more space. From the command line, enter: @@ -49,9 +50,8 @@ _______________________________________________________________________ df -h - to see how much space is available on the disk partitions on the computer/host. - -| + to see how much space is available on the disk partitions on the + computer/host. 3. When queried whether to run initialization: @@ -91,21 +91,21 @@ to end the application. .. note:: - The user's prompt may vary, based on how the system administrator set up the account. + The user's prompt may vary, based on how the system administrator + set up the account. - | 2. Recreate the Python 3.x environment used for METplus via the - environment.yml file found in the repository under the METplus directory: + **environment.yml** file found in the repository under the METplus + directory: .. code-block:: ini conda env create -f environment.yml This may take a few minutes to install all the packages specified - in the environment.yml file. + in the **environment.yml** file. -| 3. Activate the environment. From the (base) prompt, enter: @@ -113,15 +113,14 @@ to end the application. conda activate name-of-env - where name-of-env is found at the top of the environment.yml file + where name-of-env is found at the top of the **environment.yml** file e.g. *mini3.6.3* 4. Run the METplus applications in this conda env. - | - + 5. When finished, deactivate the environment: .. code-block:: ini @@ -142,8 +141,6 @@ conda env", follow these instructions to start running METplus : .. note:: The prompt may vary, based on how the sys admin set up the account. - -| 2. Activate the environment. From the (base) prompt, enter: @@ -168,8 +165,9 @@ conda env", follow these instructions to start running METplus : Optional: Checking for missing packages and mismatched version ______________________________________________________________ -To check for missing and mismatched packages run the check_python.py script -in the METplus directory: + +To check for missing and mismatched packages run the **check_python.py** +script in the METplus directory: .. code-block:: ini @@ -177,16 +175,16 @@ in the METplus directory: The results are sent to stdout (screen) and three files are created: - * actual.txt: + * **actual.txt**: * A list of Python packages that are on the host system - * missing_packages.txt: + * **missing_packages.txt**: * A list of Python packages needed for METplus but not found on the host system - * mismatched.txt: + * **mismatched.txt**: * A list of Python packages on the host system but with different version than what is used by METplus diff --git a/docs/Contributors_Guide/create_wrapper.rst b/docs/Contributors_Guide/create_wrapper.rst index bf016d3aa..9ce44ecee 100644 --- a/docs/Contributors_Guide/create_wrapper.rst +++ b/docs/Contributors_Guide/create_wrapper.rst @@ -7,10 +7,10 @@ Naming File Name ^^^^^^^^^ -Create the new wrapper in the METplus/metplus/wrappers directory and +Create the new wrapper in the *METplus/metplus/wrappers* directory and name it to reflect the wrapper's function, e.g.: new_tool_wrapper.py is a wrapper around an application named "new_tool." -You can copy example_wrapper.py to start. +Copy the **example_wrapper.py** to start the process. Class Name ^^^^^^^^^^ @@ -19,10 +19,10 @@ The name of the class should match the wrapper's function without underscores and with the first letter of each word capitalized followed by "Wrapper." For example, the new_tool wrapper would be named **NewToolWrapper**. -Add entry to LOWER_TO_WRAPPER_NAME dictionary +Add Entry to LOWER_TO_WRAPPER_NAME Dictionary --------------------------------------------- -In metplus/util/doc_util.py, add entries to the LOWER_TO_WRAPPER_NAME +In *metplus/util/doc_util.py*, add entries to the LOWER_TO_WRAPPER_NAME dictionary so that the wrapper can be found in the PROCESS_LIST even if it is formatted differently. The key should be the wrapper name in all lower-case letters without any underscores. The value should be the class name @@ -65,7 +65,7 @@ Naming ^^^^^^ Rename the class to match the wrapper's class from the above sections. -Most wrappers should be a sub-class of the CommandBuilder wrapper:: +Most wrappers should be a subclass of the CommandBuilder wrapper:: class NewToolWrapper(CommandBuilder) @@ -83,15 +83,15 @@ Parent Class ^^^^^^^^^^^^ If the new tool falls under one of the existing tool categories, -then you can make the tool a subclass of one of those classes. +then make the tool a subclass of one of the existing classes. This should only be done if the functions in the parent class are needed -by the new wrapper. If you are unsure, then use CommandBuilder. +by the new wrapper. When in doubt, use the CommandBuilder. Init Function ^^^^^^^^^^^^^ Modify the init function to initialize NewTool from its base class -to set the self.app_name variable to name of the application. +to set the self.app_name variable to the name of the application. If the application is a MET tool, then set self.app_path to the full path of the tool under **MET_BIN_DIR**. See the Basic Components :ref:`bc_init_function` section for more information:: @@ -195,41 +195,82 @@ Run Functions If the wrapper will not loop and process for each forecast lead, put the logic to build the command in the run_at_time method. -* It is recommended to divide up the logic into components, as illustrated above, the make the code more readable and easier to test. - -* The function self.set_environment_variables should be called by all wrappers even if the MET tool does not have a config file. This is done to set environment variables that MET expects to be set when running, such as MET_TMP_DIR and MET_PYTHON_EXE. If no environment variables need to be set specific to the wrapper, you do not need to write your own implementation of the function in the wrapper. You can call the implementation of the function from CommandBuilder, which sets the environment variables defined in the [user_env_vars] section of the configuration file and outputs DEBUG logs for each environment variable that has been set in the wrapper. MET_TMP_DIR is automatically set for each wrapper. - -* Once you have provided all the necessary information to create the MET command, call self.build_and_run_command(). This calls self.get_command() to assemble the command and verify that the command your wrapper generated contains all of the required arguments. You may need to override get_command() in your wrapper if your MET application is different from the example. For instance, some MET tools require flags such as -f to precede the input filename. You can override get_command in the wrapper to prepend the required flag to the filename in your constructed MET command. - -* Call self.clear() at the beginning of each loop iteration that tries to build/run a MET command to prevent inadvertently reusing/re-running commands that were previously created. - -* To allow your use case to use your wrapper, assign the wrapper name to PROCESS_LIST:: +* It is recommended to divide up the logic into components, as illustrated + above, to make the code more readable and easier to test. + +* The function self.set_environment_variables should be called by all + wrappers even if the MET tool does not have a config file. This is done + to set environment variables that MET expects to be set when running, such + as MET_TMP_DIR and MET_PYTHON_EXE. If no environment variables need to be + set specific to the wrapper, then no + implementation of the function in the wrapper needs to be written. + Call the + implementation of the function from CommandBuilder, which sets the + environment variables defined in the [user_env_vars] section of the + configuration file and outputs DEBUG logs for each environment variable + that has been set in the wrapper. MET_TMP_DIR is automatically set for + each wrapper. + +* Once all the necessary information has been provided to create the MET + command, call self.build_and_run_command(). This calls self.get_command() + to assemble the command and verify that the command wrapper generated + contains all of the required arguments. The get_command() in the wrapper + may need to be overridden if the MET application is different from + the example. + For instance, some MET tools require flags such as -f to + precede the input filename. The get_command function in the wrapper can be + overwritten to prepend the required flag to the filename in the + constructed MET command. + +* Call self.clear() at the beginning of each loop iteration that tries to + build/run a MET command to prevent inadvertently reusing/re-running + commands that were previously created. + +* To allow the use case to use the specific wrapper, assign the wrapper name to + PROCESS_LIST:: [config] PROCESS_LIST = NewExample .. note:: - Do not include the text "Wrapper" at the end of your wrapper name. - The PROCESS_LIST is located under the [config] section header in your use case and/or example configuration file. + Do not include the text "Wrapper" at the end of the wrapper name. + + Each value must match an existing wrapper name without the ‘Wrapper' + suffix. The PROCESS_LIST :numref:`Process_list` is located under the + [config] section header in the + use case and/or example configuration file. -* Add a section to the Python Wrappers page of the documentation with information about the new tool including a list of all METplus configuration variables that can be used. +* Add a section to the Python Wrappers page of the documentation with + information about the new tool including a list of all METplus + configuration variables that can be used. -* Add an entry for each METplus configuration variable added to the wrapper to the METplus Configuration Glossary. Each configuration variable should be the MET tool name in all caps i.e. GRID_STAT followed by the variable name. MET tool names generally have underscores between words unless there is a number in the name. Examples below:: +* Add an entry for each METplus configuration variable added to the wrapper + to the METplus Configuration Glossary. Each configuration variable should + be the MET tool name in all caps i.e. GRID_STAT followed by the variable + name. MET tool names generally have underscores between words unless there + is a number in the name. Examples below:: GRID_STAT_PROB_THRESH REGRID_DATA_PLANE_METHOD POINT2GRID_QC_FLAGS -* Create a directory named after the new wrapper to hold the use case configuration files in the met_tool_wrapper directory that users can run to try out the new wrapper. In the corresponding directory under docs/use_cases, be sure to include a .py file that contains the documentation for that use case and a README file to create a header for the documentation page. +* Create a directory named after the new wrapper to hold the use case + configuration files in the met_tool_wrapper directory that users can run + to try out the new wrapper. In the corresponding directory under + docs/use_cases, be sure to include a .py file that contains the + documentation for that use case and a README file to create a header for + the documentation page. -Your use case/example configuration file is located in a directory structure like the following:: +This new uuse case/example configuration file is located in a directory structure +like the following:: METplus/parm/use_cases/met_tool_wrapper/NewTool/NewTool.conf METplus/docs/use_cases/met_tool_wrapper/NewTool/NewTool.py METplus/docs/use_cases/met_tool_wrapper/NewTool/README.md -Note the documentation file is in METplus/docs while the use case conf file is in METplus/parm +Note the documentation file is in METplus/docs while the use case conf file +is in METplus/parm. Refer to the :ref:`basic_components_of_wrappers` section of the Contributor's Guide for more information on what should be added. @@ -237,10 +278,18 @@ Guide for more information on what should be added. Documentation ------------- -* Add a section for the new wrapper in the 'Python Wrappers' section of the User's Guide. This includes a list of all configuration variables specific to this wrapper. +* Add a section for the new wrapper in the 'Python Wrappers' section of the + User's Guide. This includes a list of all configuration variables specific + to this wrapper. -* Add all new configuration variables to the 'METplus Configuration Glossary' section of the User's Guide +* Add all new configuration variables to the 'METplus Configuration Glossary' + section of the User's Guide. -* Add any relevant new keywords to the 'METplus Quick Search for Use Cases' section of the User's Guide. +* Add any relevant new keywords to the 'METplus Quick Search for Use Cases' + section of the User's Guide. -* Create Sphinx documentation files for each new use case (under docs/use_cases). There should be at least one use case in the docs/use_cases/met_tool_wrapper subdirectory for the new wrapper (more if it can be configured in diffferent ways that should be shown in an example). Be sure to add a README.rst file for the header. +* Create Sphinx documentation files for each new use case + (under *docs/use_cases*). There should be at least one use case in the + *docs/use_cases/met_tool_wrapper* subdirectory for the new wrapper (more if + it can be configured in different ways that should be shown in an example). + Be sure to add a **README.rst** file for the header. diff --git a/docs/Contributors_Guide/deprecation.rst b/docs/Contributors_Guide/deprecation.rst index f85bd4ba7..491baef9e 100644 --- a/docs/Contributors_Guide/deprecation.rst +++ b/docs/Contributors_Guide/deprecation.rst @@ -1,10 +1,9 @@ Deprecating an Old Config Variable ================================== -If a config variable changes names, we need to be able to alert the -user that they need to update the config files if they are using a -deprecated variable. - +If a config variable changes names, an alert is needed to let +the user know that they need to update the config files if they +are using a deprecated variable. Example ------- @@ -20,28 +19,25 @@ changed to WGRIB2 The new variable is set to wgrib2 in the default config file -(parm/metplus_config/defaults.conf). If the user is still using -WGRIB2_EXE to set to /usr/local/bin/wgrib2, this value will not be +(*parm/metplus_config/defaults.conf*). If the user is still using +WGRIB2_EXE to set to */usr/local/bin/wgrib2*, this value will not be read and the user will have no way to know that they are setting the wrong variable and it is using WGRIB2 = wgrib2. - check_for_deprecated_config() ----------------------------- -In met_util.py there is a function called +In **met_util.py** there is a function called check_for_deprecated_config. It contains a dictionary of dictionaries called deprecated_dict that specifies the old config name, the section it was found in, and a suggested alternative (None if no alternative exists). - - **Example 1** :: 'WGRIB2_EXE' : {'sec' : 'exe', 'alt' : 'WGRIB2'} -this says that WGRIB2_EXE was found in the [exe] section and should +This says that WGRIB2_EXE was found in the [exe] section and should be replaced with WGRIB2. **Example 2** @@ -49,7 +45,7 @@ be replaced with WGRIB2. 'PREPBUFR_DIR_REGEX' : {'sec' : 'regex_pattern', 'alt' : None} -this says that [regex_pattern] PREPBUFR_DIR_REGEX is no longer used +This says that [regex_pattern] PREPBUFR_DIR_REGEX is no longer used and there is no alternative (because the wrapper uses filename templates instead of regex now). @@ -58,22 +54,24 @@ If any of these old variables are found in any config file passed to METplus by the user, an error report will be displayed with the old variables and suggested new ones if applicable. -If we want to support an old config variable but warn the users that -they should still update because it will be phased out in the future, -you can add the ‘req’ item to the dictionary and set it to False. It -will warn the user but not stop execution. If this is done, you need -to be sure to modify the code to check for the new config and if it is -not set, check for the old config as well. - +If support for an old config variable is temporarily needed, the +user should be warned to update their config file because the +variable will be phased out in the future. In this case, add the +‘req’ item to the dictionary and set it to False. This will provide +a warning to the user but will not stop the execution of the code. +If this is done, be sure to modify the code to check for the new +config variable, and if it is not set, check the old config variable +to see if it is set. **Example** :: 'LOOP_METHOD' : {'sec' : 'config', 'alt' : 'LOOP_ORDER', 'req' : False} -this says that [config] LOOP_METHOD is deprecated and you +This says that [config] LOOP_METHOD is deprecated and the user should use LOOP_ORDER, but it is not required to change -immediately. If you do this, you should check for LOOP_ORDER and then +immediately. If this is done, it is important to +check for LOOP_ORDER and then check for LOOP_METHOD if it is not set. In run_metplus.py: diff --git a/docs/Contributors_Guide/github_workflow.rst b/docs/Contributors_Guide/github_workflow.rst index 153433b65..dd748a672 100644 --- a/docs/Contributors_Guide/github_workflow.rst +++ b/docs/Contributors_Guide/github_workflow.rst @@ -13,7 +13,7 @@ where new or updated code is created on a 'feature' branch that is based on the `dtcenter/METplus GitHub 'develop' branch `_. From James McCreight (WRF-Hydro team) this is a good write-up on -`best practices for collaboration on GitHub `_ +`best practices for collaboration on GitHub `_. The feature branch is named after the corresponding GitHub issue: @@ -29,9 +29,10 @@ which includes data tarballs for use in running use cases. Sequence of Events - Contributing Code -------------------------------------- -*Pre-requisite:* +*Prerequisite:* -The user must set up a GitHub repository account if one does not already have exhist. +The user must set up a GitHub repository account if one does not already +exist. Log into the account. For more information about GitHub accounts, please refer to the GitHub Documentation on `GitHub accounts `_. @@ -43,24 +44,26 @@ A contributor to METplus will do the following: 1. Create a GitHub Issue to track the new contribution. -2. Fork the dtcenter/METplus repository. +2. Fork the *dtcenter/METplus* repository. -3. Clone the fork to local repository. +3. Clone the fork to the local repository. -4. Set upstream remote (to assist in keeping upstream and local repositories synchronized). +4. Set upstream remote (to assist in keeping upstream and local repositories + synchronized). 5. Generate a feature branch from the 'develop' branch for new development. 6. Make changes to code in the feature branch. -7. Commit changes to feature branch (limit one change per commit). +7. Commit changes to the feature branch (limit one change per commit). 8. Push the feature branch to GitHub. -9. Open a pull request from feature branch to original repo (from which the original - branch was forked, in step 2 above). +9. Open a pull request from feature branch to original repo (from which + the original branch was forked, in step 2 above). -10. Clean up after pull request has been merged by an authorized METplus developer. +10. Clean up after pull request has been merged by an authorized METplus + developer. @@ -79,27 +82,30 @@ Create a GitHub Issue that reflects what needs to be done Fork the dtcenter/METplus repository ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -* Retrieve a copy of the source code by forking the dtcenter/METplus repository - into the user's own GitHub repository. Click on the **Fork** button in the upper right - hand side of the `METplus repository `_. +* Retrieve a copy of the source code by forking the *dtcenter/METplus* + repository into the user's own GitHub repository. Click on the **Fork** + button in the upper right hand side of the + `METplus repository `_. * Click on the appropriate GitHub account when presented with the pop-up window - with the question 'Where should we fork METplus?'. + with the question 'Where should we fork METplus?' -* The web page will refresh to the GitHub repository at, for example: +* The web page will refresh to the GitHub repository. For example: .. code-block:: ini https://github.com//METplus - where ** is replaced with the user's GitHub username. - An entire copy of the dtcenter/METplus Github repository is now in the User's area. + Where ** is replaced with the user's GitHub username, + without the angle brackets <>. + An entire copy of the *dtcenter/METplus* Github repository is now in the + user's area. Clone the fork to a local repository ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -* Copy the source code to the directory where you will be doing your work. +* Copy the source code to the working directory. * Change directories to a working directory. From the command line, enter the following: @@ -108,7 +114,8 @@ Clone the fork to a local repository git clone https://github.com//METplus - replacing with your GitHub username. + replacing with the user's GitHub username and without the + angle brackets <>. * Change directories to the METplus directory: @@ -116,25 +123,26 @@ Clone the fork to a local repository cd METplus - Now you are in your local METplus repository. + This is the local METplus repository. Set upstream remote ^^^^^^^^^^^^^^^^^^^ -* Add a remote named origin to the clone of your local Git repository, which - will allow you to push changes to the repository you forked in step 1. +* Add a remote named origin to the clone of the local Git repository, which + will allow changes to be pushed to the repository that was forked above. .. code-block:: ini git remote add upstream https://github.com/dtcenter/METplus -* To verify that the upstream and origin are correct, at the command line enter: +* To verify that the upstream and origin are correct, at the command + line enter: .. code-block:: ini git remote -v - You should see something like the following: + Something like the following will be output: .. code-block:: ini @@ -143,35 +151,37 @@ Set upstream remote upstream https://github.com/dtcenter/METplus (fetch) upstream https://github.com/dtcenter/METplus (push) - where is your GitHub username. + where is the user's GitHub username without the + angle brackets <>. Generate a feature branch from the 'develop' branch for new development ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -* Create a feature branch in the dtcenter/METplus GitHub repository following this naming convention +* Create a feature branch in the *dtcenter/METplus* GitHub repository + following this naming convention: -| *feature__* -| + *feature__* - For example, for GitHub issue #777 that creates new wrapper xyz, the feature branch would be named: + For example, for GitHub issue #777 that creates new wrapper xyz, the + feature branch would be named: -| *feature_777_wrapper_xyz* -| + *feature_777_wrapper_xyz* -* Create your feature branch based off the develop branch + +* Create the feature branch based off the develop branch: .. code-block:: ini git checkout develop -* Verify that you are currently working in the develop branch by running +* Verify the current development branch is active by running: .. code-block:: ini git branch - You should see something like the following: + Something like the following will be output: .. code-block:: ini @@ -186,15 +196,15 @@ Generate a feature branch from the 'develop' branch for new development git checkout -b feature_777_wrapper_xyz - replacing *feature_777_wrapper_xyz* with your feature branch name. + replacing *feature_777_wrapper_xyz* with the user's feature branch name. -* Verify that you are working in the correct branch by running: +* Verify that the user is working in the correct branch by running: .. code-block:: ini git branch - You should see something like the following: + Something like the following will be output: .. code-block:: ini @@ -202,7 +212,8 @@ Generate a feature branch from the 'develop' branch for new development main_v3.1 * feature_777_wrapper_xyz - where the asterisk, "*", indicates which branch is currently in use/checked out. + where the asterisk, "*", indicates which branch is currently in + use/checked out. Make changes to code in the feature branch @@ -211,15 +222,18 @@ Make changes to code in the feature branch Users should make changes to their feature branch and commit changes to their local repository. -* Create code following the coding standards in the :ref:`codingstandards` section of - the Contributor's Guide. +* Create code following the coding standards in the + :ref:`codingstandards` section of the Contributor's Guide. -* Provide some tests for your code using the pytest framework, provide user documentation +* Provide some tests for the code using the pytest framework, provide + user documentation describing what the code does, and provide any necessary data. -* Keep your fork in sync. While working, it is highly likely that changes are occurring in - the original repository. This may impact your work. Regularly use the following commands - to keep your fork in sync with the original repository. +* Keep the fork in sync with the original repository. While working, it is + highly likely that changes are occurring in + the original repository, which could have an impact. + Regularly use the following commands + to keep the fork in sync with the original repository. .. code-block:: ini @@ -228,36 +242,43 @@ local repository. git merge origin develop The first command pulls changes from the original repository (the - `METplus GitHub repository `_ that you see when you - run *git remote -v* and that you set to upstream in step 4 above). The second command - pushes those changes to your forked repository. The third command will merge the local + `METplus GitHub repository `_ + that is output when + running *git remote -v* and that was set to upstream in the + "Set upstream remote" section above). + The second command pushes those changes to the forked repository. + The third command will merge the local develop branch into the feature branch. Commit changes to feature branch ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -* Commit any new files by running the following. Run the *'git add'* command only if this file is - newly created and does not yet exist in your repository. +* Commit any new files by running the following. Run the *'git add'* + command only if this file is + newly created and does not yet exist in the repository. .. code-block:: ini git add git commit - replacing with the filename. + replacing with the filename and removing the angle brackets (<>). - A popup window will appear, where you will enter a description of this commit, using the - editor you selected when you set up your Git account. Please refer to the + A popup window will appear. Enter a description about this commit, using the + editor the user selected when the Git account was set up. + Please refer to the `Git Setup `_ - documentation on configuring your Git account. + documentation on configuring a Git account. - For the first line of your commit comment, enter a brief description, such as the GitHub - Issue number and a brief description. On the second and subsequent lines, provide a - detailed description of the changes/additions you made. + For the first line of the commit comment, enter a brief description, + such as the GitHub + Issue number and a brief description. On the second and subsequent lines, + provide a detailed description of the changes/additions that were made. - **Note**: It is a best practice to commit one change per commit, rather than wait - until you have multiple changes to include in one commit. + **Note**: It is a best practice to commit one change per commit, + rather than wait + until there are multiple changes to include in one commit. Push the feature branch to GitHub ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -268,43 +289,50 @@ Push the feature branch to GitHub git push origin - replacing *feature_777_wrapper_xyz* with your feature branch name, to push your changes to - the origin (i.e. to your *https://github.com//METplus* repository) + replacing ** with the feature branch name + to push the changes to + the origin (i.e. to the *https://github.com//METplus* + repository). Open a pull request using a browser -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -* To request to have your changes be incorporated into the remote repository - (i.e. the `GitHub METplus repository `_). +* To request to have the changes be incorporated into the remote repository + (i.e. the + `GitHub METplus repository `_). -* An authorized METplus developer will need to approve the request and then merge your files - into the repository's develop branch. The develop branch will be used to create a future - METplus release. +* An authorized METplus developer will need to approve the request and + then merge the files into the repository's develop branch. + The develop branch will be used to create a future METplus release. -* In your browser, navigate to *https://github.com//METplus* replacing - with your GitHub username. +* In the browser, navigate to https://github.com//METplus + replacing + with the user's GitHub username and no angle brackets <>. -* Click on the green 'Compare & pull request' button +* Click on the green 'Compare & pull request' button. * A web page with four grey buttons should appear: - * On the left-most button (for setting the base repository), make sure you have selected - 'base repository:dtcenter/METplus' + * On the left-most button (for setting the base repository), + make sure the + 'base repository:dtcenter/METplus' is selected. - * For the base button, make sure you have selected 'base:develop' + * For the base button, make sure to select 'base:develop'. - * For the head repository button, make sure you have selected - 'head repository:/METplus' where is your GitHub - account name. + * For the head repository button, make sure to select + 'head repository:/METplus' + with the appropriate replacement for + . - * For the compare button, make sure you have selected 'compare:' - where corresponds to the feature branch where you have been - making your changes (e.g. feature_777_wrapper_xyz). + * For the compare button, make sure to select + 'compare:' + where corresponds to the feature branch + where the changes have been made (e.g. feature_777_wrapper_xyz). - * In the 'write' window, follow the directions and fill in the template - add any additional comments/details. When filling in the template, - users will need to "Define the PR metadata, as permissions allow. - Select: **Reviewer(s), Project(s)**, and **Milestone**" When selecting a + * In the 'write' window, follow the directions and fill in the template. + Add any additional comments/details. When filling in the template, + be sure to "Define the PR metadata, as permissions allow. + Select: **Reviewer(s), Project(s)**, and **Milestone**". When selecting a reviewer, internal contributors submitting pull requests should select the appropriate reviewer(s) and let the reviewer know that the pull request has been assigned to them. If external contributors are unsure @@ -316,7 +344,7 @@ Open a pull request using a browser request' button. * An authorized METplus developer will accept the pull request (if - everything meets acceptance criteria) and merge your code into the remote + everything meets acceptance criteria) and merge the code into the remote repository's develop branch. Approve a pull request using a browser @@ -368,14 +396,16 @@ Creating a pull request .. figure:: figure/insert_suggestion.png - Click on the icon of a paper with +/- to “Insert a Suggestion”. The line - will be quoted and the reviewer can enter their suggestion below. Then, click on + Click on the icon of a paper with +/- to “Insert a Suggestion”. + The line + will be quoted and the reviewer can enter their suggestion below. + Then, click on the “Add Single Comment” button, so that the requestor will get an email letting them know the reviewer has made a suggested change. b. Or, a reviewer can edit the file directly on the web by clicking on the - “...” icon (three dots) in the right hand corner next to - “Viewed” icon and select “Edit file”. + “...” icon (three dots) in the right hand corner next to the + “Viewed” icon and selecting “Edit file”. .. figure:: figure/how_to_edit_file.png @@ -402,9 +432,9 @@ Creating a pull request .. figure:: figure/review_approve_changes.png - A reviewer should click on: "Review changes", add comments to - the "Write box", and select either "Comment", "Approve", - or "Request Changes", and then click on "Submit Review". + A reviewer should click on: "Review changes", add comments to + the "Write box", and select either "Comment", "Approve", + or "Request Changes", and then click on "Submit Review". 12. Once the recommended testing is complete and any necessary changes have been made, approve the request. @@ -423,7 +453,8 @@ There are three merge methods to choose from: "Create a merge commit", combined into one and a clean history is retained. Click on the chosen merge method. -After merging, the requestor can then decide whether or not to delete the branch. +After merging, the requestor can then decide whether or not to delete +the branch. .. figure:: figure/delete_branch.png @@ -434,44 +465,47 @@ button should be selected and the corresponding GitHub issue should be closed. Clean up after a successfully merged pull request ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -* After an authorized METplus developer has accepted your changes and merged - them into the develop repository, update your local clone by pulling changes - from the original repository's (i.e. the `METplus develop branch `_): +* After an authorized METplus developer has accepted the changes and merged + them into the develop repository, update the local clone by pulling changes + from the original repository's (i.e. the + `METplus develop branch `_): -* Checkout your develop branch +* Checkout the develop branch: .. code-block:: ini git checkout develop -* Verify that you are now working from the develop branch +* Verify that the develop branch is now active: .. code-block:: ini git branch -* Merge changes from the upstream develop branch with your local develop branch +* Merge changes from the upstream develop branch with the local develop branch: .. code-block:: ini git pull upstream develop -* Your local cloned repository should now have all the latest changes from the +* The local cloned repository should now have all the latest changes from the original repository's develop branch. - Now you can delete your feature branch: + Now the feature branch can be deleted: .. code-block:: ini *git branch -D * *git push --delete origin * - where is your feature branch name, e.g. feature_777_wrapper_xyz + where is the feature branch name, e.g. feature_777_wrapper_xyz. - You can verify that your feature branch has been successfully removed/deleted - via your web browser. Navigate to *https://github.com//METplus*, - replacing with your GitHub username, and under the 'Branch' - pulldown menu, you should no longer find your feature branch as a selection. + Verify that the feature branch has been successfully removed/deleted + via the web browser. Navigate to + https://github.com//METplus, + replacing appropriately. Under the 'Branch' + pulldown menu, the feature branch name should no longer be seen + as an option. diff --git a/docs/Contributors_Guide/testing.rst b/docs/Contributors_Guide/testing.rst index c04c21a24..3db6f6ff7 100644 --- a/docs/Contributors_Guide/testing.rst +++ b/docs/Contributors_Guide/testing.rst @@ -1,25 +1,46 @@ Testing ======= -Test scripts are found in the GitHub repository in the internal_tests directory. +Test scripts are found in the GitHub repository in the internal_tests +directory. Unit Tests ---------- -Unit tests are run with pytest. They are found in the 'pytests' directory. Each tool has its own sub-directory containing its test files. +Unit tests are run with pytest. They are found in the *pytests* directory. +Each tool has its own subdirectory containing its test files. -run_pytests.sh is a bash script that can be run to execute all of the pytests. A report will be output showing which pytest categories failed. -When running on a new computer, you must create a minimum_pytest..sh file to be able to run the script. This file contains information about the local environment so that the tests can run. +**run_pytests.sh** is a bash script that can be run to execute all of the +pytests. A report will be output showing which pytest categories failed. +When running on a new computer, a +**minimum_pytest..sh** +file must be created to be able to run the script. This file contains +information about the local environment so that the tests can run. Use Case Tests -------------- -Use case tests are run via a Python script called test_use_cases.py, found in the use_cases directory. -Eventually the running of these tests will be automated using an external tool, such as GitHub Actions or Travis CI. -The script contains a list of use cases that are found in the repository. For each computer that will run the use cases, a metplus_test_env..sh file must exist to set local configurations. -All of the use cases can be run by executing the script run_test_use_cases.sh. The use case test script will output the results into a directory such as /d1//test-use-case-b, defined in the environment file. -If /d1//test-use-case-b already exists, its content will be copied over to /d1//test-use-case-a. If data is found in If /d1//test-use-case-b already exists, its content will be copied over th /d1//test-use-case-a, the script will prompt the user to remove those files. -Once the tests have finished running, the output found in the two directories can be compared to see what has changed. Suggested commands to run to compare the output will be shown on the screen after completion of the script. +Use case tests are run via a Python script called **test_use_cases.py**, +found in the *use_cases* directory. +Eventually the running of these tests will be automated using an external +tool, such as GitHub Actions or Travis CI. +The script contains a list of use cases that are found in the repository. +For each computer that will run the use cases, a +**metplus_test_env..sh** file must exist to set local configurations. +All of the use cases can be run by executing the script +**run_test_use_cases.sh**. The use case test script will output the results +into a directory such as */d1//test-use-case-b*, defined in the +environment file. +If */d1//test-use-case-b* already exists, its content will be copied +over to */d1//test-use-case-a*. If data is found in +the */d1//test-use-case-b* directory already exists, its content +will be copied +over to the */d1//test-use-case-a* directory, the script will prompt +the user to remove those files. +Once the tests have finished running, the output found in the two +directories can be compared to see what has changed. Suggested commands +to run to compare the output will be shown on the screen after completion +of the script. To see which files and directories are only found in one run:: diff --git a/docs/index.rst b/docs/index.rst index c454a4e58..21582943a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -47,24 +47,25 @@ wrappers to provide low-level automation and examples, also called use-cases. A description of each tool along with some ancillary repositories are as follows: -* **MET** - core statistical tool that matches up grids with either gridded +* **METplus** (`repo `__, `docs `__) + + * **wrappers** - suite of Python-based wrappers that provide low-level automation of the METplus software components listed below + * **use-cases** - example configuration files and sample data to demonstrate scientific applications of the METplus wrappers + +* **MET** (`repo `__, `docs `__) - core statistical tool that matches up grids with either gridded analyses or point observations and applies configurable methods to compute statistics and diagnostics -* **METviewer** - core database and display system intended for deep analysis +* **METviewer** (`repo `__, `docs `__) - core database and display system intended for deep analysis of MET output -* **METexpress** - core database and display system intended for quick +* **METexpress** (`repo `__, `docs `__) - core database and display system intended for quick analysis via pre-defined queries of MET output -* **METplus wrappers** - suite of Python-based wrappers that provide - low-level automation of MET tools and newly developed plotting capability -* **METplus use-cases** - configuration files and sample data to show how to - invoke METplus wrappers to make using MET tools easier and reproducible -* **METcalcpy** - suite of Python-based scripts to be used by other +* **METcalcpy** (`repo `__, `docs `__) - suite of Python-based scripts to be used by other components of METplus tools for statistical aggregation, event equalization, and other analysis needs -* **METplotpy** - suite of Python-based scripts to plot MET output, +* **METplotpy** (`repo `__, `docs `__) - suite of Python-based scripts to plot MET output, and in come cases provide additional post-processing of output prior to plotting -* **METdatadb** - database to store MET output and to be used by both +* **METdatadb** (`repo `__, `docs `__) - database to store MET output and to be used by both METviewer and METexpress The umbrella repository will be brought together by using a software package diff --git a/internal_tests/data/ens/2009123112/arw-fer-gep1/d01_2009123112_04800.grib b/internal_tests/data/ens/2009123112/arw-fer-gep1/d01_2009123112_04800.grib new file mode 100644 index 000000000..e69de29bb diff --git a/internal_tests/data/ens/2009123112/arw-fer-gep5/d01_2009123112_04800.grib b/internal_tests/data/ens/2009123112/arw-fer-gep5/d01_2009123112_04800.grib new file mode 100644 index 000000000..e69de29bb diff --git a/internal_tests/data/ens/2009123112/arw-sch-gep2/d01_2009123112_04800.grib b/internal_tests/data/ens/2009123112/arw-sch-gep2/d01_2009123112_04800.grib new file mode 100644 index 000000000..e69de29bb diff --git a/internal_tests/data/ens/2009123112/arw-sch-gep6/d01_2009123112_04800.grib b/internal_tests/data/ens/2009123112/arw-sch-gep6/d01_2009123112_04800.grib new file mode 100644 index 000000000..e69de29bb diff --git a/internal_tests/data/ens/2009123112/arw-tom-gep3/d01_2009123112_04800.grib b/internal_tests/data/ens/2009123112/arw-tom-gep3/d01_2009123112_04800.grib new file mode 100644 index 000000000..e69de29bb diff --git a/internal_tests/data/ens/2009123112/arw-tom-gep7/d01_2009123112_04800.grib b/internal_tests/data/ens/2009123112/arw-tom-gep7/d01_2009123112_04800.grib new file mode 100644 index 000000000..e69de29bb diff --git a/internal_tests/data/ens/2009123118/arw-fer-gep1/d01_2009123118_04800.grib b/internal_tests/data/ens/2009123118/arw-fer-gep1/d01_2009123118_04800.grib new file mode 100644 index 000000000..e69de29bb diff --git a/internal_tests/data/ens/2009123118/arw-fer-gep5/d01_2009123118_04800.grib b/internal_tests/data/ens/2009123118/arw-fer-gep5/d01_2009123118_04800.grib new file mode 100644 index 000000000..e69de29bb diff --git a/internal_tests/data/ens/2009123118/arw-sch-gep2/d01_2009123118_04800.grib b/internal_tests/data/ens/2009123118/arw-sch-gep2/d01_2009123118_04800.grib new file mode 100644 index 000000000..e69de29bb diff --git a/internal_tests/data/ens/2009123118/arw-sch-gep6/d01_2009123118_04800.grib b/internal_tests/data/ens/2009123118/arw-sch-gep6/d01_2009123118_04800.grib new file mode 100644 index 000000000..e69de29bb diff --git a/internal_tests/data/ens/2009123118/arw-tom-gep3/d01_2009123118_04800.grib b/internal_tests/data/ens/2009123118/arw-tom-gep3/d01_2009123118_04800.grib new file mode 100644 index 000000000..e69de29bb diff --git a/internal_tests/data/ens/2009123118/arw-tom-gep7/d01_2009123118_04800.grib b/internal_tests/data/ens/2009123118/arw-tom-gep7/d01_2009123118_04800.grib new file mode 100644 index 000000000..e69de29bb diff --git a/internal_tests/pytests/gen_ens_prod/test_gen_ens_prod_wrapper.py b/internal_tests/pytests/gen_ens_prod/test_gen_ens_prod_wrapper.py index 408a10d13..d9c264834 100644 --- a/internal_tests/pytests/gen_ens_prod/test_gen_ens_prod_wrapper.py +++ b/internal_tests/pytests/gen_ens_prod/test_gen_ens_prod_wrapper.py @@ -34,7 +34,7 @@ def set_minimum_config_settings(config): config.set('config', 'INIT_BEG', run_times[0]) config.set('config', 'INIT_END', run_times[-1]) config.set('config', 'INIT_INCREMENT', '6H') - config.set('config', 'LEAD_SEQ', '24H') + config.set('config', 'LEAD_SEQ', '24H, 48H') config.set('config', 'LOOP_ORDER', 'times') config.set('config', 'GEN_ENS_PROD_CONFIG_FILE', '{PARM_BASE}/met_config/GenEnsProdConfig_wrapped') @@ -385,19 +385,32 @@ def test_gen_ens_prod_single_field(metplus_config, config_overrides, 'file_lists') config_file = wrapper.c_dict.get('CONFIG_FILE') out_dir = wrapper.c_dict.get('OUTPUT_DIR') - expected_cmds = [(f"{app_path} {verbosity} -ens " - f"{file_list_dir}/20091231120000_24_gen_ens_prod.txt " - "-out " - f"{out_dir}/gen_ens_prod_20100101_120000V_ens.nc " - f"-config {config_file} -ctrl " - f"{input_dir}/2009123112/arw-tom-gep3/d01_2009123112_02400.grib"), - (f"{app_path} {verbosity} -ens " - f"{file_list_dir}/20091231180000_24_gen_ens_prod.txt " - "-out " - f"{out_dir}/gen_ens_prod_20100101_180000V_ens.nc " - f"-config {config_file} -ctrl " - f"{input_dir}/2009123118/arw-tom-gep3/d01_2009123118_02400.grib"), - ] + expected_cmds = [ + (f"{app_path} {verbosity} -ens " + f"{file_list_dir}/20091231120000_24_gen_ens_prod.txt " + "-out " + f"{out_dir}/gen_ens_prod_20100101_120000V_ens.nc " + f"-config {config_file} -ctrl " + f"{input_dir}/2009123112/arw-tom-gep3/d01_2009123112_02400.grib"), + (f"{app_path} {verbosity} -ens " + f"{file_list_dir}/20091231120000_48_gen_ens_prod.txt " + "-out " + f"{out_dir}/gen_ens_prod_20100102_120000V_ens.nc " + f"-config {config_file} -ctrl " + f"{input_dir}/2009123112/arw-tom-gep3/d01_2009123112_04800.grib"), + (f"{app_path} {verbosity} -ens " + f"{file_list_dir}/20091231180000_24_gen_ens_prod.txt " + "-out " + f"{out_dir}/gen_ens_prod_20100101_180000V_ens.nc " + f"-config {config_file} -ctrl " + f"{input_dir}/2009123118/arw-tom-gep3/d01_2009123118_02400.grib"), + (f"{app_path} {verbosity} -ens " + f"{file_list_dir}/20091231180000_48_gen_ens_prod.txt " + "-out " + f"{out_dir}/gen_ens_prod_20100102_180000V_ens.nc " + f"-config {config_file} -ctrl " + f"{input_dir}/2009123118/arw-tom-gep3/d01_2009123118_04800.grib"), + ] all_cmds = wrapper.run_all_times() print(f"ALL COMMANDS: {all_cmds}") @@ -472,7 +485,7 @@ def test_gen_ens_prod_fill_missing(metplus_config, config_overrides, os.remove(file_list_file) all_cmds = wrapper.run_all_times() - assert len(all_cmds) == 1 + assert len(all_cmds) == 2 with open(file_list_file, 'r') as file_handle: actual_num_files = len(file_handle.read().splitlines()) - 1 diff --git a/internal_tests/pytests/tc_gen/test_tc_gen_wrapper.py b/internal_tests/pytests/tc_gen/test_tc_gen_wrapper.py index 0bbc7654b..575fd2bf6 100644 --- a/internal_tests/pytests/tc_gen/test_tc_gen_wrapper.py +++ b/internal_tests/pytests/tc_gen/test_tc_gen_wrapper.py @@ -329,11 +329,14 @@ def test_tc_gen(metplus_config, config_overrides, env_var_values): config.set('config', 'TC_GEN_SHAPE_INPUT_DIR', shape_dir) config.set('config', 'TC_GEN_SHAPE_INPUT_TEMPLATE', shape_template) config.set('config', 'TC_GEN_OUTPUT_DIR', '{OUTPUT_BASE}/TCGen/output') - config.set('config', 'TC_GEN_OUTPUT_TEMPLATE', 'tc_gen_{init?fmt=%Y}') + config.set('config', 'TC_GEN_OUTPUT_TEMPLATE', + 'tc_gen_{custom}_{init?fmt=%Y}') config.set('config', 'TC_GEN_CONFIG_FILE', '{PARM_BASE}/met_config/TCGenConfig_wrapped') + config.set('config', 'TC_GEN_CUSTOM_LOOP_LIST', 'a, b') + # set config variable overrides for key, value in config_overrides.items(): config.set('config', key, value) @@ -368,7 +371,13 @@ def test_tc_gen(metplus_config, config_overrides, env_var_values): f"-edeck {edeck_path} " f"-shape {shape_path} " f"-track {track_path} " - f"-config {config_file} -out {out_dir}/tc_gen_2016"), + f"-config {config_file} -out {out_dir}/tc_gen_a_2016"), + (f"{app_path} {verbosity} " + f"-genesis {genesis_path} " + f"-edeck {edeck_path} " + f"-shape {shape_path} " + f"-track {track_path} " + f"-config {config_file} -out {out_dir}/tc_gen_b_2016"), ] all_cmds = wrapper.run_all_times() diff --git a/internal_tests/pytests/user_script/test_user_script.py b/internal_tests/pytests/user_script/test_user_script.py index 4c79d50f2..de69d7634 100644 --- a/internal_tests/pytests/user_script/test_user_script.py +++ b/internal_tests/pytests/user_script/test_user_script.py @@ -321,6 +321,30 @@ def set_run_type_info(config, run_type): 'echo init_20141026093015_valid_20141101093015_lead_144.nc', 'echo init_20141025093015_valid_20141101093015_lead_168.nc', ]), + # run once custom loop list + ({'USER_SCRIPT_RUNTIME_FREQ': 'RUN_ONCE', + 'USER_SCRIPT_COMMAND': 'echo {custom}', + 'USER_SCRIPT_CUSTOM_LOOP_LIST': 'a,b'}, + None, + ['echo a', 'echo b']), + # run per valid custom loop list + ({'USER_SCRIPT_RUNTIME_FREQ': 'RUN_ONCE_PER_INIT_OR_VALID', + 'USER_SCRIPT_COMMAND': 'echo {custom}', + 'USER_SCRIPT_CUSTOM_LOOP_LIST': 'a,b'}, + ['VALID'], + ['echo a'] * 3 + ['echo b'] * 3), + # run per init custom loop list + ({'USER_SCRIPT_RUNTIME_FREQ': 'RUN_ONCE_PER_INIT_OR_VALID', + 'USER_SCRIPT_COMMAND': 'echo {custom}', + 'USER_SCRIPT_CUSTOM_LOOP_LIST': 'a,b'}, + ['INIT'], + ['echo a'] * 3 + ['echo b'] * 3), + # run all init/lead sequence - simple + ({'USER_SCRIPT_RUNTIME_FREQ': 'RUN_ONCE_FOR_EACH', + 'USER_SCRIPT_COMMAND': 'echo {custom}', + 'USER_SCRIPT_CUSTOM_LOOP_LIST': 'a,b'}, + ['LEAD_SEQ', 'INIT'], + ['echo a'] * 12 + ['echo b'] * 12), ] ) def test_run_user_script_all_times(metplus_config, input_configs, @@ -340,7 +364,7 @@ def test_run_user_script_all_times(metplus_config, input_configs, all_commands = wrapper.run_all_times() if not all_commands: - assert(False) + assert False clock_time = datetime.strptime(config.getstr('config', 'CLOCK_TIME'), '%Y%m%d%H%M%S') @@ -349,4 +373,4 @@ def test_run_user_script_all_times(metplus_config, input_configs, expected_cmd = sub_clock_time(expected_cmd, clock_time) print(f" ACTUAL:{actual_cmd}") print(f"EXPECTED:{expected_cmd}") - assert(actual_cmd == expected_cmd) + assert actual_cmd == expected_cmd diff --git a/metplus/util/met_util.py b/metplus/util/met_util.py index 7a65110f2..4ceb861d2 100644 --- a/metplus/util/met_util.py +++ b/metplus/util/met_util.py @@ -388,11 +388,12 @@ def is_loop_by_init(config): return None -def loop_over_times_and_call(config, processes): +def loop_over_times_and_call(config, processes, custom=None): """! Loop over all run times and call wrappers listed in config @param config METplusConfig object @param processes list of CommandBuilder subclass objects (Wrappers) to call + @param custom (optional) custom loop string value @returns list of tuples with all commands run and the environment variables that were set for each """ @@ -410,7 +411,8 @@ def loop_over_times_and_call(config, processes): log_runtime_banner(config, time_input, process) add_to_time_input(time_input, - instance=process.instance) + instance=process.instance, + custom=custom) process.clear() process.run_at_time(time_input) diff --git a/metplus/wrappers/command_builder.py b/metplus/wrappers/command_builder.py index c0c7d0475..bec31ba31 100755 --- a/metplus/wrappers/command_builder.py +++ b/metplus/wrappers/command_builder.py @@ -1373,10 +1373,13 @@ def run_at_time(self, input_dict): 'wrapper. Cannot run with LOOP_ORDER = times') return None - def run_all_times(self): - """!Loop over time range specified in conf file and - call METplus wrapper for each time""" - return util.loop_over_times_and_call(self.config, self) + def run_all_times(self, custom=None): + """! Loop over time range specified in conf file and + call METplus wrapper for each time + + @param custom (optional) custom loop string value + """ + return util.loop_over_times_and_call(self.config, self, custom=custom) @staticmethod def format_met_config_dict(c_dict, name, keys=None): diff --git a/metplus/wrappers/compare_gridded_wrapper.py b/metplus/wrappers/compare_gridded_wrapper.py index d8b60fb81..0a9583167 100755 --- a/metplus/wrappers/compare_gridded_wrapper.py +++ b/metplus/wrappers/compare_gridded_wrapper.py @@ -169,8 +169,6 @@ def run_at_time_once(self, time_info): Args: @param time_info dictionary containing timing information """ - self.clear() - var_list = util.sub_var_list(self.c_dict['VAR_LIST_TEMP'], time_info) @@ -191,6 +189,7 @@ def run_at_time_once(self, time_info): if var_list: self.c_dict['CURRENT_VAR_INFO'] = var_list[0] + self.clear() self.run_at_time_all_fields(time_info) def run_at_time_one_field(self, time_info, var_info): diff --git a/metplus/wrappers/pb2nc_wrapper.py b/metplus/wrappers/pb2nc_wrapper.py index 411c4c629..922d7a7b6 100755 --- a/metplus/wrappers/pb2nc_wrapper.py +++ b/metplus/wrappers/pb2nc_wrapper.py @@ -274,14 +274,12 @@ def run_at_time(self, input_dict): input_dict['custom'] = custom_string # Run for given init/valid time and forecast lead combination + self.clear() self.run_at_time_once(input_dict) def run_at_time_once(self, input_dict): """!Find files needed to run pb2nc and run if found""" - # clear out information set from previous run - self.clear() - # look for input files to process time_info = self.find_input_files(input_dict) diff --git a/metplus/wrappers/runtime_freq_wrapper.py b/metplus/wrappers/runtime_freq_wrapper.py index adfaec06a..8488f2072 100755 --- a/metplus/wrappers/runtime_freq_wrapper.py +++ b/metplus/wrappers/runtime_freq_wrapper.py @@ -138,7 +138,7 @@ def run_all_times_custom(self, custom): elif runtime_freq == 'RUN_ONCE_PER_LEAD': self.run_once_per_lead(custom) elif runtime_freq == 'RUN_ONCE_FOR_EACH': - self.all_commands = super().run_all_times() + self.all_commands = super().run_all_times(custom) def run_once(self, custom): self.logger.debug("Running once for all files") @@ -178,6 +178,7 @@ def run_once_per_init_or_valid(self, custom): time_input['lead'] = '*' + self.clear() if not self.run_at_time_once(time_input): success = False @@ -204,6 +205,7 @@ def run_once_per_lead(self, custom): time_input['init'] = '*' time_input['valid'] = '*' + self.clear() if not self.run_at_time_once(time_input): success = False @@ -216,42 +218,37 @@ def run_at_time(self, input_dict): @param input_dict dictionary containing time information """ - for custom_string in self.c_dict['CUSTOM_LOOP_LIST']: - if custom_string: - self.logger.info(f"Processing custom string: {custom_string}") - - input_dict['custom'] = custom_string - - # loop of forecast leads and process each - lead_seq = get_lead_sequence(self.config, input_dict) - for lead in lead_seq: - input_dict['lead'] = lead - - # set current lead time config and environment variables - time_info = time_util.ti_calculate(input_dict) - - self.logger.info( - f"Processing forecast lead {time_info['lead_string']}" - ) + # loop of forecast leads and process each + lead_seq = get_lead_sequence(self.config, input_dict) + for lead in lead_seq: + input_dict['lead'] = lead - if skip_time(time_info, self.c_dict.get('SKIP_TIMES', {})): - self.logger.debug('Skipping run time') - continue + # set current lead time config and environment variables + time_info = time_util.ti_calculate(input_dict) - # since run_all_times was not called (LOOP_BY=times) then - # get files for current run time - file_dict = self.get_files_from_time(time_info) - all_files = [] - if file_dict: - if isinstance(file_dict, list): - all_files = file_dict - else: - all_files = [file_dict] + self.logger.info( + f"Processing forecast lead {time_info['lead_string']}" + ) - self.c_dict['ALL_FILES'] = all_files + if skip_time(time_info, self.c_dict.get('SKIP_TIMES', {})): + self.logger.debug('Skipping run time') + continue - # Run for given init/valid time and forecast lead combination - self.run_at_time_once(time_info) + # since run_all_times was not called (LOOP_BY=times) then + # get files for current run time + file_dict = self.get_files_from_time(time_info) + all_files = [] + if file_dict: + if isinstance(file_dict, list): + all_files = file_dict + else: + all_files = [file_dict] + + self.c_dict['ALL_FILES'] = all_files + + # Run for given init/valid time and forecast lead combination + self.clear() + self.run_at_time_once(time_info) def get_all_files(self, custom=None): """! Get all files that can be processed with the app. diff --git a/metplus/wrappers/tc_gen_wrapper.py b/metplus/wrappers/tc_gen_wrapper.py index c24c5e23d..11b825675 100755 --- a/metplus/wrappers/tc_gen_wrapper.py +++ b/metplus/wrappers/tc_gen_wrapper.py @@ -368,6 +368,7 @@ def run_at_time(self, input_dict): self.logger.debug('Skipping run time') continue + self.clear() self.run_at_time_once(time_info) def run_at_time_once(self, time_info): diff --git a/metplus/wrappers/tcrmw_wrapper.py b/metplus/wrappers/tcrmw_wrapper.py index c7cc1a23a..f116a6faf 100755 --- a/metplus/wrappers/tcrmw_wrapper.py +++ b/metplus/wrappers/tcrmw_wrapper.py @@ -225,6 +225,7 @@ def run_at_time(self, input_dict): self.logger.debug('Skipping run time') continue + self.clear() self.run_at_time_once(time_info) def run_at_time_once(self, time_info): diff --git a/parm/use_cases/model_applications/s2s/UserScript_fcstGFS_obsERA_WeatherRegime.conf b/parm/use_cases/model_applications/s2s/UserScript_fcstGFS_obsERA_WeatherRegime.conf index d06418294..02f48837c 100644 --- a/parm/use_cases/model_applications/s2s/UserScript_fcstGFS_obsERA_WeatherRegime.conf +++ b/parm/use_cases/model_applications/s2s/UserScript_fcstGFS_obsERA_WeatherRegime.conf @@ -179,9 +179,13 @@ FCST_WR_FREQ = {OBS_WR_FREQ} # These variables control reordering the forecast weather regime to match the # observations if their orders are different -# It is recommended to set this to False if this is the first time running the +# REORDER_FCST_MANUAL will use the order in FCST_ORDER, whereas REORDER_FCST will +# use a pattern correlation to reorder +# It is recommended to set REORDER_FCST_MANUAL to False if this is the first time running the # case REORDER_FCST = True +REORDER_FCST_MANUAL = False +#Order to use if REORDER_FCST_MANUAL = True; will be ignored if REORER_FCST_MANUAL = False FCST_ORDER = 1,3,4,2,5,6 # Type, name and directory of Output File for weather regime classification @@ -213,9 +217,9 @@ FCST_KMEANS_PLOT_OUTPUT_NAME = fcst_kmeans KMEANS_PLOT_LEVELS = -80, -70, -60, -50, -40, -30, -20, -10, 0, 10, 20, 30, 40, 50, 60, 70, 80 # Frequency Plot title and output file name -OBS_FREQ_PLOT_TITLE = ERA Seasonal Cycle of WR Days/Week (1979-2017) +OBS_FREQ_PLOT_TITLE = ERA Seasonal Cycle of WR Days/Week (2000-2017) OBS_FREQ_PLOT_OUTPUT_NAME = obs_freq -FCST_FREQ_PLOT_TITLE = GFS Seasonal Cycle of WR Days/Week (1979-2017) +FCST_FREQ_PLOT_TITLE = GFS Seasonal Cycle of WR Days/Week (2000-2017) FCST_FREQ_PLOT_OUTPUT_NAME = fcst_freq # MPR file information diff --git a/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/WeatherRegime_driver.py b/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/WeatherRegime_driver.py index e91a3082a..66ddff000 100755 --- a/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/WeatherRegime_driver.py +++ b/parm/use_cases/model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime/WeatherRegime_driver.py @@ -6,7 +6,7 @@ import warnings from metcalcpy.contributed.blocking_weather_regime.WeatherRegime import WeatherRegimeCalculation -from metcalcpy.contributed.blocking_weather_regime.Blocking_WeatherRegime_util import parse_steps, read_nc_met, write_mpr_file, reorder_fcst_regimes +from metcalcpy.contributed.blocking_weather_regime.Blocking_WeatherRegime_util import parse_steps, read_nc_met, write_mpr_file, reorder_fcst_regimes,reorder_fcst_regimes_correlate from metplotpy.contributed.weather_regime import plot_weather_regime as pwr @@ -16,7 +16,8 @@ def main(): if not steps_list_obs and not steps_list_fcst: warnings.warn('No processing steps requested for either the model or observations,') - warnings.warn('No data will be processed') + warnings.warn(' nothing will be run') + warnings.warn('Set FCST_STEPS and/or OBS_STEPS in the [user_env_vars] section to process data') ###################################################################### @@ -129,21 +130,22 @@ def main(): if ("KMEANS" in steps_list_obs): print('Running Obs K Means') kmeans_obs,wrnum_obs,perc_obs,wrc_obs= steps_obs.run_K_means(z500_detrend_2d_obs,timedict_obs,z500_obs.shape) + steps_obs.write_K_means_file(timedict_obs,wrc_obs) if ("KMEANS" in steps_list_fcst): print('Running Forecast K Means') kmeans_fcst,wrnum_fcst,perc_fcst,wrc_fcst = steps_fcst.run_K_means(z500_detrend_2d_fcst,timedict_fcst, z500_fcst.shape) - - if ("KMEANS" in steps_list_obs) and ("KMEANS" in steps_list_fcst): - # Check to see if reordering the data so that the weather regime patterns match between - # the forecast and observations, is needed - #TODO: make this automated based on spatial correlations reorder_fcst = os.environ.get('REORDER_FCST','False').lower() - fcst_order_str = os.environ['FCST_ORDER'].split(',') - fcst_order = [int(fo) for fo in fcst_order_str] - if reorder_fcst == 'true': + reorder_fcst_manual = os.environ.get('REORDER_FCST_MANUAL','False').lower() + if (reorder_fcst == 'true') and ("KMEANS" in steps_list_obs): + kmeans_fcst,perc_fcst,wrc_fcst = reorder_fcst_regimes_correlate(kmeans_obs,kmeans_fcst,perc_fcst,wrc_fcst,wrnum_fcst) + if reorder_fcst_manual == 'true': + fcst_order_str = os.environ['FCST_ORDER'].split(',') + fcst_order = [int(fo) for fo in fcst_order_str] kmeans_fcst,perc_fcst,wrc_fcst = reorder_fcst_regimes(kmeans_fcst,perc_fcst,wrc_fcst,wrnum_fcst,fcst_order) + steps_fcst.write_K_means_file(timedict_fcst,wrc_fcst) + # Write matched pair output for weather regime classification modname = os.environ.get('MODEL_NAME','GFS') @@ -177,9 +179,13 @@ def main(): if ("TIMEFREQ" in steps_list_obs): + if not ("KMEANS" in steps_list_obs): + raise Exception('Must run observed Kmeans before running frequencies.') wrfreq_obs,dlen_obs,ts_diff_obs = steps_obs.compute_wr_freq(wrc_obs) if ("TIMEFREQ" in steps_list_fcst): + if not ("KMEANS" in steps_list_fcst): + raise Exception('Must run forecast Kmeans before running frequencies.') wrfreq_fcst,dlen_fcst,ts_diff_fcst = steps_fcst.compute_wr_freq(wrc_fcst) if ("TIMEFREQ" in steps_list_obs) and ("TIMEFREQ" in steps_list_fcst):