Merge branch 'develop' into feature_2540_point2grid_config

.coveragerc

@@ -3,3 +3,4 @@ relative_files = True
 source = metplus
 omit =
   metplus/wrappers/cyclone_plotter_wrapper.py
+  metplus/produtil/**

.github/parm/use_case_groups.json

@@ -276,6 +276,12 @@
     "disabled": true,
     "run": false
   },
+  {
+    "category": "short_range",
+    "index_list": "15",
+    "disabled": true,
+    "run":false
+  },
   {
     "category": "space_weather",
     "index_list": "0-1",

docs/Contributors_Guide/add_use_case.rst

@@ -169,6 +169,7 @@ Use Cases that Exceed GitHub Actions Disk Space
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 - *model_applications/s2s_stratosphere/UserScript_fcstGFS_obsERA_StratospherePolar*
 - *model_applications/s2s_stratosphere/UserScript_fcstGFS_obsERA_StratosphereQBO*
+- *model_applications/short_range/MODEMultivar_fcstRRFS_obsGOES_MRMS_BrightnessTemp_Lightning*
 
 .. _use_case_documentation:
 

docs/Users_Guide/quicksearch.rst

@@ -156,6 +156,7 @@ Use Cases by METplus Feature:
    | `METplotpy <../search.html?q=METplotpyUseCase&check_keywords=yes&area=default>`_
    | `MET_PYTHON_EXE Environment Variable  <../search.html?q=MET_PYTHON_EXEUseCase&check_keywords=yes&area=default>`_
    | `Multiple Conf File Use <../search.html?q=MultiConfUseCase&check_keywords=yes&area=default>`_
+   | `MultivariateMODE <../search.html?q=MvMODEUseCase&check_keywords=yes&area=default>`_
    | `Observation Time Summary <../search.html?q=ObsTimeSummaryUseCase&check_keywords=yes&area=default>`_
    | `Observation Uncertainty <../search.html?q=ObsUncertaintyUseCase&check_keywords=yes&area=default>`_
    | `Python Embedding Ingest <../search.html?q=PyEmbedIngestToolUseCase&check_keywords=yes&area=default>`_
@@ -187,6 +188,7 @@ Use Cases by METplus Feature:
    | **METplotpy**: *METplotpyUseCase*
    | **MET_PYTHON_EXE Environment Variable**: *MET_PYTHON_EXEUseCase*
    | **Multiple Conf File Use**: *MultiConfUseCase*
+   | **MultivariateMODE**: *MvMODEUseCase*
    | **Observation Time Summary**: *ObsTimeSummaryUseCase*
    | **Observation Uncertainty**: *ObsUncertaintyUseCase*
    | **Python Embedding Ingest**: *PyEmbedIngestToolUseCase*

docs/use_cases/model_applications/short_range/MODEMultivar_fcstRRFS_obsGOES_MRMS_BrightnessTemp_Lightning.py

@@ -0,0 +1,179 @@
+"""
+MODEMultivar: Create objects of brightness temps and radar reflectivity  
+=======================================================================
+
+model_applications/
+short_range/
+MODEMultivar_fcstRRFS_obsGOES_MRMS_BrightnessTemp_Lightning.conf
+
+"""
+############################################################################## 
+# .. contents::
+#    :depth: 1
+#    :local:
+#    :backlinks: none
+
+##############################################################################
+# Scientific Objective
+# --------------------
+#
+# This use case identifies convective objects, which are defined by
+# the intersection of: 1) satellite infrared brightness temperature < 235 K and 
+# 2) radar reflectivity > 40 dBZ.
+# Satellite brightness temperatures are used in conjunction with radar reflectivity 
+# to capture both the cloud top (satellite) and in-cloud (radar) characteristics. 
+# Convective objects are also defined as lightning thresholds exceeding the 10th percentile. 
+# A percentile threshold is used for lightning data as RRFS lightning has units 
+# which are “non-dimensional” and therefore cannot be directly compared to the
+# Geostationary Lightning Mapper. 
+
+##############################################################################
+# Version Added
+# -------------
+#
+# METplus Version 6.0
+
+##############################################################################
+# Datasets
+# --------
+#
+# **Forecast:** Rapid Refresh Forecast System (RRFS) 3km resolution, 
+# channel 13 brightness temperature,
+# composite reflectivity, and lightning strike density
+#
+# **Observation:** Geostationary Operational Environmental Satellites (GOES) 3km resolution, 
+# channel 13 brightness temperature; 
+# Multi-radar Multi-sensor (MRMS) 3km resolution, composite reflectivity; 
+# GOES Global Lightning Mapper (GLM) 3km resolution, flash_extent_density
+# 
+# **Climatology:** None
+#
+# **Location:** All of the input data required for this use case can be found in a sample data 
+# tarball. Each use case category will have one or more sample data tarballs. It is only 
+# necessary to download the tarball with the use case’s dataset and not the entire 
+# collection of sample data. Click here to access the METplus releases page and download 
+# sample data for the appropriate release: https://github.com/dtcenter/METplus/releases
+# This tarball should be unpacked into the directory that you will set the value of 
+# INPUT_BASE. See :ref:`running-metplus` for more information.
+
+##############################################################################
+# METplus Components
+# ------------------
+#
+# The only tool this use case calls is MODE, which will identify super-objects
+# by intersection of the multiple variable fields.
+# 
+
+##############################################################################
+# METplus Workflow
+# ----------------
+#
+# | **Beginning Time (INIT_BEG):** 2024-01-09 05:00 UTC
+# | **End Time (INIT_END):** 2024-01-09 05:00 UTC
+# | **Increment between beginning and end times (VALID_INCREMENT):** 1 Hour
+# | **Sequence of forecast leads to process (LEAD_SEQ):** 9,10
+#
+# This use case runs twice, once for each forecast lead time provided. It 
+# creates objects valid at 14UTC and 15UTC from 09 January 2024 are compared to 
+# the 9h and 10h forecasts initialized at 05UTC on 9 January 2024.
+# Convective objects are identified with thresholds of
+# satellite brightness temperature < 235 K and radar reflectivity > 40 dBZ, 
+# or lightning > 10th percentile.
+# In this use case, MODE super-object intensity statistics are output for both 
+# radar reflectivity and lightning. Using the MODE_MULTIVAR_INTENSITY_FLAG, 
+# users can control for which variables super object intensity statistics will be output. 
+# If all are set to False, then no intensity information will be output
+# and only statistics relative to the super-object geometry will be available.
+
+
+
+##############################################################################
+# METplus Configuration
+# ---------------------
+#
+# METplus first loads all of the configuration files found in parm/metplus_config,
+# then it loads any configuration files passed to METplus via the command line, i.e.
+# parm/use_cases/model_applications/short_range/MODEMultivar_fcstRRFS_obsGOES_MRMS_BrightnessTemp_Lightning.conf
+#
+# .. highlight:: bash
+# .. literalinclude:: ../../../../parm/use_cases/model_applications/short_range/MODEMultivar_fcstRRFS_obsGOES_MRMS_BrightnessTemp_Lightning.conf
+
+##############################################################################
+# MET Configuration
+# ---------------------
+#
+# METplus sets environment variables based on user settings in the METplus configuration file. 
+# See :ref:`How METplus controls MET config file settings<metplus-control-met>` for more details. 
+#
+# **YOU SHOULD NOT SET ANY OF THESE ENVIRONMENT VARIABLES YOURSELF! THEY WILL BE OVERWRITTEN BY METPLUS WHEN IT CALLS THE MET TOOLS!**
+#
+# If there is a setting in the MET configuration file that is currently not supported by METplus you'd like to control, please refer to:
+# :ref:`Overriding Unsupported MET config file settings<met-config-overrides>`
+#
+# .. dropdown:: MODEConfig_wrapped
+#
+#   .. literalinclude:: ../../../../parm/met_config/MODEConfig_wrapped
+
+##############################################################################
+# Python Embedding
+# ----------------
+#
+# This use case does not use any Python Embedding.
+
+##############################################################################
+# Python Scripting
+# ----------------
+#
+# This use case does not use any Python Scripting.
+
+##############################################################################
+# Running METplus
+# ---------------
+#
+# Pass the use case configuration file to the run_metplus.py script
+# along with any user-specific system configuration files if desired::
+#
+#    run_metplus.py /path/to/METplus/parm/use_cases/model_applications/short_range/MODEMultivar_fcstRRFS_obsGOES_MRMS_BrightnessTemp_Lightning.conf /path/to/user_system.conf
+#
+# See :ref:`running-metplus` for more information.
+
+##############################################################################
+# Expected Output
+# ---------------
+#
+# A successful run will output the following both to the screen and to the logfile::
+#
+#   INFO: METplus has successfully finished running.
+#
+# Refer to the value set for **OUTPUT_BASE** to find where the output data was generated.
+# Output for this use case will be found in 
+# {OUTPUT_BASE}/model_applications/short_range/MODEMultivar_fcstRRFS_obsGOES_MRMS_BrightnessTemp_Lightning/f??,
+# where the '??' characters will reflect the two forecast leads (09 and 10).
+# Each of these directories will contain the following files with their appropriate
+# verification times::
+#
+#  * mode_Fcst_LTNG_entireatmosphere_all_all_Obs_flash_extent_density_all_all_RRFS_or_ANALYSIS_090000L_20240109_140000V_000000A_cts.txt
+#  * mode_Fcst_LTNG_entireatmosphere_all_all_Obs_flash_extent_density_all_all_RRFS_or_ANALYSIS_090000L_20240109_140000V_000000A_obj.nc
+#  * mode_Fcst_LTNG_entireatmosphere_all_all_Obs_flash_extent_density_all_all_RRFS_or_ANALYSIS_090000L_20240109_140000V_000000A_obj.txt
+#  * mode_Fcst_LTNG_entireatmosphere_all_all_Obs_flash_extent_density_all_all_RRFS_or_ANALYSIS_090000L_20240109_140000V_000000A.ps
+#  * mode_Fcst_REFC_entireatmosphere_consideredasinglelayer_all_all_Obs_MergedReflectivityQCComposite_all_all_RRFS_or_ANALYSIS_090000L_20240109_140000V_000000A_cts.txt
+#  * mode_Fcst_REFC_entireatmosphere_consideredasinglelayer_all_all_Obs_MergedReflectivityQCComposite_all_all_RRFS_or_ANALYSIS_090000L_20240109_140000V_000000A_obj.nc
+#  * mode_Fcst_REFC_entireatmosphere_consideredasinglelayer_all_all_Obs_MergedReflectivityQCComposite_all_all_RRFS_or_ANALYSIS_090000L_20240109_140000V_000000A_obj.txt
+#  * mode_Fcst_REFC_entireatmosphere_consideredasinglelayer_all_all_Obs_MergedReflectivityQCComposite_all_all_RRFS_or_ANALYSIS_090000L_20240109_140000V_000000A.ps
+
+##############################################################################
+# Keywords
+# --------
+#
+# .. note::
+#
+#   * MODEToolUseCase 
+#   * ShortRangeAppUseCase
+#   * NetCDFFileUseCase
+#   * MvMODEUseCase
+#
+#   Navigate to the :ref:`quick-search` page to discover other similar use cases.
+#
+#
+# sphinx_gallery_thumbnail_path = '_static/short_range-MODEMultivar_fcstRRFS_obsGOES_MRMS_BrightnessTemp_Lightning.png'
+#

internal/scripts/docker/Dockerfile

@@ -58,5 +58,5 @@ RUN if [ ${OBTAIN_SOURCE_CODE} != "none" ]; then \
   sed -i 's|MET_INSTALL_DIR = /path/to|MET_INSTALL_DIR = /usr/local|g' parm/metplus_config/*.conf; \
   sed -i 's|OUTPUT_BASE = /path/to|OUTPUT_BASE = /data/output|g' parm/metplus_config/*.conf; \
   sed -i 's|INPUT_BASE = /path/to|INPUT_BASE = /data/input/METplus_Data|g' parm/metplus_config/*.conf; \
-  python3 setup.py install; \
+  pip install .; \
 fi

internal/scripts/sonarqube/sonar-project.properties

@@ -3,8 +3,8 @@ sonar.projectKey=METplus
 sonar.projectName=METplus
 sonar.projectVersion=SONAR_PROJECT_VERSION
 sonar.branch.name=SONAR_BRANCH_NAME
-sonar.sources=docs,internal,manage_externals,metplus,parm,produtil,ush
-sonar.coverage.exclusions=internal/tests/**,parm/**,internal/scripts/**,manage_externals/**,docs/**,produtil/**,ush/**,metplus/wrappers/cyclone_plotter_wrapper.py
+sonar.sources=docs,internal,manage_externals,metplus,parm,ush
+sonar.coverage.exclusions=internal/tests/**,parm/**,metplus/parm/**,internal/scripts/**,manage_externals/**,docs/**,metplus/produtil/**,ush/**,metplus/wrappers/cyclone_plotter_wrapper.py
 sonar.python.coverage.reportPaths=coverage.xml
 sonar.sourceEncoding=UTF-8
 

internal/tests/pytests/util/run_util/test_run_util.py

@@ -4,7 +4,6 @@
 import os
 import re
 
-import produtil
 import metplus.util.run_util as ru
 import metplus.util.wrapper_init as wi
 from metplus.wrappers.ensemble_stat_wrapper import EnsembleStatWrapper

internal/tests/use_cases/all_use_cases.txt

@@ -184,7 +184,7 @@ Category: short_range
 12::UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalCrossSection::model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalCrossSection.conf:: metplotpy_env
 13::MODEMultivar_fcstHRRR_obsMRMS_HRRRanl::model_applications/short_range/MODEMultivar_fcstHRRR_obsMRMS_HRRRanl.conf
 14::UserScript_fcstRRFS_fcstOnly_Reformat_Aggregate_Plot::model_applications/short_range/UserScript_fcstRRFS_fcstOnly_Reformat_Aggregate_Plot.conf:: metdataio, metcalcpy, metplotpy,mp_analysis_env
-
+15::MODEMultivar_fcstRRFS_obsGOES_MRMS_BrightnessTemp_Lightning::model_applications/short_range/MODEMultivar_fcstRRFS_obsGOES_MRMS_BrightnessTemp_Lightning.conf
 
 Category: space_weather
 0::GridStat_fcstGloTEC_obsGloTEC_vx7:: model_applications/space_weather/GridStat_fcstGloTEC_obsGloTEC_vx7.conf

metplus/parm

@@ -0,0 +1 @@
+../parm

produtil/config.py → metplus/produtil/config.py

@@ -11,19 +11,18 @@
 # decides what symbols are imported by "from produtil.config import *"
 __all__=['from_file','confwalker','ProdConfig','ENVIRONMENT','ProdTask']
 
-import collections,re,string,os,logging,threading
+import collections,re,os,logging,threading
 import os.path,sys
 import datetime
-import produtil.fileop, produtil.datastore
-import produtil.numerics, produtil.log
+import metplus.produtil.fileop as fileop
 
 import configparser
 from configparser import ConfigParser
 from io import StringIO
 
-from produtil.datastore import Datastore,Task
+from metplus.produtil.datastore import Datastore,Task
 
-from produtil.numerics import to_datetime
+from metplus.produtil.numerics import to_datetime, to_datetime_rel, fcst_hr_min
 from string import Formatter
 from configparser import NoOptionError,NoSectionError
 
@@ -379,7 +378,7 @@ def get_value(self,key,args,kwargs):
                 v=ap6.strftime(ANL_P6_KEYS[key])
             elif '__ftime' in kwargs and '__atime' in kwargs and \
                     key in TIME_DIFF_KEYS:
-                (ihours,iminutes)=produtil.numerics.fcst_hr_min(
+                (ihours,iminutes)=fcst_hr_min(
                     kwargs['__ftime'],kwargs['__atime'])
                 if key=='fahr':
                     v=int(ihours)
@@ -702,7 +701,7 @@ def from_args(self,args=None,allow_files=True,allow_options=True,
             elif not os.path.isfile(path):
                 logger.error(path+': conf file is not a regular file.')
                 sys.exit(2)
-            elif not produtil.fileop.isnonempty(path) and verbose:
+            elif not fileop.isnonempty(path) and verbose:
                 logger.warning(
                     path+': conf file is empty.  Will continue anyway.')
             if verbose: logger.info('Conf input: '+repr(path))
@@ -823,7 +822,7 @@ def getdatastore(self):
         with self:
             if self._datastore is None:
                 dsfile=self.getstr('config','datastore')
-                self._datastore=produtil.datastore.Datastore(dsfile,
+                self._datastore=Datastore(dsfile,
                     logger=self.log('datastore'))
             return self._datastore
 
@@ -1001,7 +1000,7 @@ def makedirs(self,*args):
         with self:
             dirs=[self.getstr('dir',arg) for arg in args]
         for makeme in dirs:
-            produtil.fileop.makedirs(makeme)
+            fileop.makedirs(makeme)
     def keys(self,sec):
         """!get options in a section
 
@@ -1107,13 +1106,13 @@ def timestrinterp(self,sec,string,ftime=None,atime=None,**kwargs):
         @param atime the analysis time or None
         @param kwargs more variables for string expansion"""
         if atime is not None:
-            atime=produtil.numerics.to_datetime(atime)
+            atime=to_datetime(atime)
         else:
             atime=self.cycle
         if ftime is None:
             ftime=atime
         else:
-            ftime=produtil.numerics.to_datetime_rel(ftime,atime)
+            ftime=to_datetime_rel(ftime,atime)
         with self:
             return self._time_formatter.format(string,__section=sec,
                 __key='__string__',__depth=0,__conf=self._conf,ENV=ENVIRONMENT,
@@ -1343,7 +1342,7 @@ def getbool(self,sec,opt,default=None,badtypeok=False,morevars=None,taskvars=Non
 
 ########################################################################
 
-class ProdTask(produtil.datastore.Task):
+class ProdTask(Task):
     """!A subclass of produtil.datastore.Task that provides a variety
     of convenience functions related to unix conf files and
     logging."""