From c7fa90a012dbe6365a9609a89d22eefccc4cc23c Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Wed, 20 Nov 2024 19:28:47 +0000 Subject: [PATCH 01/39] Update and add scripts for analysis stats capabilities --- env/HERA.env | 5 + jobs/JGLOBAL_ANALYSIS_STATS | 38 +++++++ parm/config/gfs/config.anlstat | 16 +++ parm/config/gfs/stat/aero/jcb-base.yaml.j2 | 24 +++++ parm/gdas/anlstat_jedi_config.yaml.j2 | 8 ++ scripts/exglobal_analysis_stats.py | 27 +++++ sorc/link_workflow.sh | 1 + ush/python/pygfs/task/stat_analysis.py | 115 +++++++++++++++++++++ workflow/applications/applications.py | 1 + workflow/rocoto/tasks.py | 1 + 10 files changed, 236 insertions(+) create mode 100644 jobs/JGLOBAL_ANALYSIS_STATS create mode 100644 parm/config/gfs/config.anlstat create mode 100644 parm/config/gfs/stat/aero/jcb-base.yaml.j2 create mode 100644 parm/gdas/anlstat_jedi_config.yaml.j2 create mode 100644 scripts/exglobal_analysis_stats.py create mode 100644 ush/python/pygfs/task/stat_analysis.py diff --git a/env/HERA.env b/env/HERA.env index f10bfcc537..7c27579db9 100755 --- a/env/HERA.env +++ b/env/HERA.env @@ -105,6 +105,11 @@ elif [[ "${step}" = "atmanlfv3inc" ]]; then export NTHREADS_ATMANLFV3INC=${NTHREADSmax} export APRUN_ATMANLFV3INC="${APRUN_default} --cpus-per-task=${NTHREADS_ATMANLFV3INC}" +elif [[ "${step}" = "anlstat" ]]; then + + export NTHREADS_ANLSTAT=${NTHREADSmax} + export APRUN_ANLSTAT="${APRUN_default} --cpus-per-task=${NTHREADS_ANLSTAT}" + elif [[ "${step}" = "prepobsaero" ]]; then export NTHREADS_PREPOBSAERO=${NTHREADS1} diff --git a/jobs/JGLOBAL_ANALYSIS_STATS b/jobs/JGLOBAL_ANALYSIS_STATS new file mode 100644 index 0000000000..537fc03122 --- /dev/null +++ b/jobs/JGLOBAL_ANALYSIS_STATS @@ -0,0 +1,38 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" +source "${HOMEgfs}/ush/jjob_header.sh" -e "anlstat" -c "base anlstat" + +############################################## +# Set variables used in the script +############################################## + + +############################################## +# Begin JOB SPECIFIC work +############################################## + +# Generate COM variables from templates +YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_OBS COM_CHEM_ANALYSIS + + +############################################################### +# Run relevant script + +EXSCRIPT=${ANLSTATSPY:-${SCRgfs}/exglobal_analysis_stats.py} +${EXSCRIPT} +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + +############################################## +# End JOB SPECIFIC work +############################################## + +############################################## +# Final processing +############################################## +if [[ -e "${pgmout}" ]] ; then + cat "${pgmout}" +fi + +exit 0 \ No newline at end of file diff --git a/parm/config/gfs/config.anlstat b/parm/config/gfs/config.anlstat new file mode 100644 index 0000000000..5c58eda542 --- /dev/null +++ b/parm/config/gfs/config.anlstat @@ -0,0 +1,16 @@ +#!/bin/bash -x + +########## config.anlstat ########## +# Analysis Stat + +echo "BEGIN: config.anlstat" + +# Get task specific resources +source "${EXPDIR}/config.resources" anlstat + +export JEDI_CONFIG_YAML="${PARMgfs}/gdas/anlstat_jedi_config.yaml.j2" +export JCB_BASE_YAML="${PARMgfs}/gdas/stat/aero/jcb-base.yaml.j2" +export JCB_ALGO_YAML="${PARMgfs}/gdas/jcb-algorithms/anlstat.yaml.j2" +export JEDIEXE=${HOMEgfs}/sorc/gdas.cd/build/bin/ioda-stats.x + +echo "END: config.anlstat" \ No newline at end of file diff --git a/parm/config/gfs/stat/aero/jcb-base.yaml.j2 b/parm/config/gfs/stat/aero/jcb-base.yaml.j2 new file mode 100644 index 0000000000..f14a2664fd --- /dev/null +++ b/parm/config/gfs/stat/aero/jcb-base.yaml.j2 @@ -0,0 +1,24 @@ +# Search path for model and obs for JCB +# ------------------------------------- +algorithm_path: "{{PARMgfs}}/gdas/jcb-algorithms" +app_path_algorithm: "{{PARMgfs}}/gdas/jcb-gdas/algorithm/obstats/aero" + +window_begin: "{{ STAT_WINDOW_BEGIN | to_isotime }}" +window_YMDH: "{{ STAT_WINDOW_BEGIN | to_YMDH }}" +window_length: "{{ STAT_WINDOW_LENGTH }}" + +# Inputted list of ob spaces +# -------------------------- +obspaces: {{ OBSPACES_LIST }} + +# Obspace variable things +# ----------------------- +aero_obsdatain_path: "{{ DATA }}" +aero_obsdatatin_simulated_variables: ['aerosolOpticalDepth'] +aero_obsdatain_observed_variables: ['aerosolOpticalDepth'] + +# Variables +# --------- +aero_variables: ['aerosolOpticalDepth'] +aero_file_groups: ['bkgmob', 'bkgmob1'] +aero_file_qc_groups: ['EffectiveQC0', 'EffectiveQC1'] \ No newline at end of file diff --git a/parm/gdas/anlstat_jedi_config.yaml.j2 b/parm/gdas/anlstat_jedi_config.yaml.j2 new file mode 100644 index 0000000000..bbefbaac2c --- /dev/null +++ b/parm/gdas/anlstat_jedi_config.yaml.j2 @@ -0,0 +1,8 @@ +statanl: + rundir: '{{ DATA }}' + exe_src: '{{ JEDIEXE }}' + mpi_cmd: '{{ APRUN_ANLSTAT }}' + jedi_args: None + jcb_base_yaml: '{{ PARMgfs }}/gdas/stat/aero/jcb-base.yaml.j2' + jcb_algo_yaml: '{{ JCB_ALGO_YAML }}' + jcb_algo: 'anlstat' \ No newline at end of file diff --git a/scripts/exglobal_analysis_stats.py b/scripts/exglobal_analysis_stats.py new file mode 100644 index 0000000000..cc00985fcc --- /dev/null +++ b/scripts/exglobal_analysis_stats.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 + +# exglobal_stat_analysis.py +# This script creates an StatAnalysis class +# and runs the initialize method +# which create and stage the runtime directory +# and create the YAML configuration +# for a global stat analysis +import os + +from wxflow import Logger, cast_strdict_as_dtypedict +from pygfs.task.stat_analysis import StatAnalysis + +# Initialize root logger +logger = Logger(level='DEBUG', colored_log=True) + + +if __name__ == '__main__': + + # Take configuration from environment and cast it as python dictionary + config = cast_strdict_as_dtypedict(os.environ) + + # Instantiate the atm analysis task + StatAnl = StatAnalysis(config) + + # Initialize JEDI variational analysis + StatAnl.initialize() \ No newline at end of file diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index b35b7ff35a..9f3678bc51 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -356,6 +356,7 @@ if [[ -d "${HOMEgfs}/sorc/gdas.cd/build" ]]; then "gdassoca_obsstats.x" \ "gdasapp_land_ensrecenter.x" \ "bufr2ioda.x" \ + "ioda-stats.x"\ "calcfIMS.exe" \ "apply_incr.exe" ) for gdasexe in "${JEDI_EXE[@]}"; do diff --git a/ush/python/pygfs/task/stat_analysis.py b/ush/python/pygfs/task/stat_analysis.py new file mode 100644 index 0000000000..c731c8038c --- /dev/null +++ b/ush/python/pygfs/task/stat_analysis.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 + +import os +import glob +import gzip +import tarfile +import yaml +from logging import getLogger +from pprint import pformat +from typing import Optional, Dict, Any + +from wxflow import (AttrDict, + FileHandler, + add_to_datetime, to_fv3time, to_timedelta, to_YMDH, + Task, + parse_j2yaml, save_as_yaml, + logit) +from pygfs.jedi import Jedi + +logger = getLogger(__name__.split('.')[-1]) + + +class StatAnalysis(Task): + """ + Class for JEDI-based global stat analysis tasks + """ + @logit(logger, name="StatAnalysis") + def __init__(self, config: Dict[str, Any]): + """ + Constructor global stat analysis task + This method will construct a global stat analysis task. + This includes: + - extending the task_config attribute AttrDict to include parameters required for this task + - instantiate the Jedi attribute objects + Parameters + ---------- + config: Dict + dictionary object containing task configuration + Returns + ---------- + None + """ + super().__init__(config) + + _res = int(self.task_config.CASE[1:]) + _window_begin = add_to_datetime(self.task_config.current_cycle, -to_timedelta(f"{self.task_config.assim_freq}H") / 2) + + # Create a local dictionary that is repeatedly used across this class + local_dict = AttrDict( + { + 'npx_ges': _res + 1, + 'npy_ges': _res + 1, + 'npz_ges': self.task_config.LEVS - 1, + 'npz': self.task_config.LEVS - 1, + 'npz_anl': self.task_config.LEVS - 1, + 'STAT_WINDOW_BEGIN': _window_begin, + 'STAT_WINDOW_LENGTH': f"PT{self.task_config.assim_freq}H", + 'OPREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", + 'APREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", + 'GPREFIX': f"gdas.t{self.task_config.previous_cycle.hour:02d}z." + } + ) + + # Extend task_config with local_dict + self.task_config = AttrDict(**self.task_config, **local_dict) + + # Create dictionary of Jedi objects + self.jedi_dict = Jedi.get_jedi_dict(self.task_config.JEDI_CONFIG_YAML, self.task_config) + + @logit(logger) + def initialize(self) -> None: + """ + Initialize a global stat analysis + This method will initialize a global stat analysis. + This includes: + - initialize JEDI applications + - copying stat files + Parameters + ---------- + None + Returns + ---------- + None + """ + logger.info(f"Copying files to {self.task_config.DATA}/stats") + + # Copy stat files to DATA path + aerostat = os.path.join(self.task_config.COM_CHEM_ANALYSIS, f"{self.task_config['APREFIX']}aerostat") + dest = os.path.join(self.task_config.DATA, "aerostats") + statlist = [[aerostat, dest]] + FileHandler({'copy': statlist}).sync() + + # Open tar file + logger.info(f"Open tarred stat file in {dest}") + with tarfile.open(dest, "r") as tar: + # Extract all files to the current directory + tar.extractall() + + # Gunzip .nc files + logger.info("Gunzip files from tar file") + gz_files = glob.glob(os.path.join(self.task_config.DATA, "*gz")) + + for diagfile in gz_files: + with gzip.open(diagfile, 'rb') as f_in: + with open(diagfile[:-3], 'wb') as f_out: + f_out.write(f_in.read()) + + # Get list of .nc4 files + obs_space_paths = glob.glob(os.path.join(self.task_config.DATA, "*.nc4")) + + self.task_config.OBSPACES_LIST = ['_'.join(os.path.basename(path).split('_')[1:3]) for path in obs_space_paths] + + # initialize JEDI application + logger.info(f"Initializing JEDI variational DA application") + self.jedi_dict['statanl'].initialize(self.task_config) \ No newline at end of file diff --git a/workflow/applications/applications.py b/workflow/applications/applications.py index ecd320d708..4ff5ca9aa5 100644 --- a/workflow/applications/applications.py +++ b/workflow/applications/applications.py @@ -55,6 +55,7 @@ def __init__(self, conf: Configuration) -> None: self.do_verfozn = base.get('DO_VERFOZN', True) self.do_verfrad = base.get('DO_VERFRAD', True) self.do_vminmon = base.get('DO_VMINMON', True) + self.do_anlstat = base.get('DO_ANLSTAT', True) self.do_tracker = base.get('DO_TRACKER', True) self.do_genesis = base.get('DO_GENESIS', True) self.do_genesis_fsu = base.get('DO_GENESIS_FSU', False) diff --git a/workflow/rocoto/tasks.py b/workflow/rocoto/tasks.py index df56f90718..05b3c5caea 100644 --- a/workflow/rocoto/tasks.py +++ b/workflow/rocoto/tasks.py @@ -21,6 +21,7 @@ class Tasks: 'atmensanlinit', 'atmensanlobs', 'atmensanlsol', 'atmensanlletkf', 'atmensanlfv3inc', 'atmensanlfinal', 'aeroanlinit', 'aeroanlvar', 'aeroanlfinal', 'aeroanlgenb', 'snowanl', 'esnowrecen', + 'anlstat', 'fcst', 'atmanlupp', 'atmanlprod', 'atmupp', 'goesupp', 'atmos_prod', 'ocean_prod', 'ice_prod', From b154521469bb881c48bfa9e2148d07c95153f8ed Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Wed, 20 Nov 2024 19:47:36 +0000 Subject: [PATCH 02/39] pynorms --- scripts/exglobal_analysis_stats.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/exglobal_analysis_stats.py b/scripts/exglobal_analysis_stats.py index cc00985fcc..72979cdba2 100644 --- a/scripts/exglobal_analysis_stats.py +++ b/scripts/exglobal_analysis_stats.py @@ -24,4 +24,4 @@ StatAnl = StatAnalysis(config) # Initialize JEDI variational analysis - StatAnl.initialize() \ No newline at end of file + StatAnl.initialize() From e3d015adbb578dff92bf17c18f8f6ce32def0f16 Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Wed, 20 Nov 2024 19:58:38 +0000 Subject: [PATCH 03/39] pynorms --- ush/python/pygfs/task/stat_analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/stat_analysis.py b/ush/python/pygfs/task/stat_analysis.py index c731c8038c..156e95c07a 100644 --- a/ush/python/pygfs/task/stat_analysis.py +++ b/ush/python/pygfs/task/stat_analysis.py @@ -112,4 +112,4 @@ def initialize(self) -> None: # initialize JEDI application logger.info(f"Initializing JEDI variational DA application") - self.jedi_dict['statanl'].initialize(self.task_config) \ No newline at end of file + self.jedi_dict['statanl'].initialize(self.task_config) From f745d3df565181ec5aa3976b1259b1e740f3c583 Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Thu, 21 Nov 2024 20:35:43 +0000 Subject: [PATCH 04/39] update missed files --- jobs/rocoto/anlstat.sh | 18 +++++++++++++ parm/config/gfs/config.base | 8 ++++++ parm/config/gfs/config.resources | 10 +++++++- workflow/applications/gfs_cycled.py | 9 +++++++ workflow/rocoto/gfs_tasks.py | 40 +++++++++++++++++++++++++++++ 5 files changed, 84 insertions(+), 1 deletion(-) create mode 100644 jobs/rocoto/anlstat.sh diff --git a/jobs/rocoto/anlstat.sh b/jobs/rocoto/anlstat.sh new file mode 100644 index 0000000000..ac7d8af16a --- /dev/null +++ b/jobs/rocoto/anlstat.sh @@ -0,0 +1,18 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" + +############################################################### +# Source UFSDA workflow modules +. "${HOMEgfs}/ush/load_ufsda_modules.sh" +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + +export job="anlstat" +export jobid="${job}.$$" + +############################################################### +# Execute the JJOB +"${HOMEgfs}/jobs/JGLOBAL_ANALYSIS_STATS" +status=$? +exit "${status}" diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index 4f702f9668..1a6b267da6 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -474,6 +474,14 @@ if [[ ${DO_JEDIATMVAR} = "YES" ]]; then export DO_VERFOZN="NO" # Ozone data assimilation monitoring export DO_VERFRAD="NO" # Radiance data assimilation monitoring export DO_VMINMON="NO" # GSI minimization monitoring + export DO_ANLSTAT="YES" # JEDI-based analysis statistics +else + export DO_VERFOZN="YES" # Ozone data assimilation monitoring + export DO_VERFRAD="YES" # Radiance data assimilation monitoring + export DO_VMINMON="YES" # GSI minimization monitoring + if [[ ${DO_AERO} = "YES" || ${DO_JEDIOCNVAR} = "YES" || ${DO_JEDISNOWDA} = "YES " ]]; then + export DO_ANLSTAT="YES" # JEDI-based analysis statistics + fi fi # If starting ICs that are not at cycle hour diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index cddd1643fd..7d2122d420 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -17,7 +17,7 @@ if (( $# != 1 )); then echo "atmensanlinit atmensanlobs atmensanlsol atmensanlletkf atmensanlfv3inc atmensanlfinal" echo "snowanl esnowrecen" echo "prepobsaero aeroanlinit aeroanlvar aeroanlfinal aeroanlgenb" - echo "anal sfcanl analcalc analdiag fcst echgres" + echo "anal sfcanl analcalc analdiag anlstat fcst echgres" echo "upp atmos_products" echo "tracker genesis genesis_fsu" echo "verfozn verfrad vminmon fit2obs metp arch cleanup" @@ -717,6 +717,14 @@ case ${step} in memory="48GB" ;; + "anlstat") + walltime="00:30:00" + ntasks=1 + threads_per_task=1 + tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + memory="24GB" + ;; + "sfcanl") walltime="00:20:00" ntasks=${ntiles:-6} diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index e85e8b159f..2c4d33787e 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -75,6 +75,9 @@ def _get_app_configs(self): if self.do_vminmon: configs += ['vminmon'] + if self.do_anlstat: + configs += ['anlstat'] + if self.do_tracker: configs += ['tracker'] @@ -212,6 +215,9 @@ def get_task_names(self): if self.do_vminmon: gdas_tasks += ['vminmon'] + if self.do_anlstat: + gdas_tasks += ['anlstat'] + if self.do_gempak: gdas_tasks += ['gempak', 'gempakmetancdc'] @@ -246,6 +252,9 @@ def get_task_names(self): if self.do_vminmon: gfs_tasks += ['vminmon'] + if self.do_anlstat: + gfs_tasks += ['anlstat'] + if self.do_tracker: gfs_tasks += ['tracker'] diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 461241450e..b3e2466768 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -1738,6 +1738,40 @@ def vminmon(self): return task + def anlstat(self): + deps = [] + if self.app_config.do_jediatmvar: + dep_dict = {'type': 'task', 'name': f'{self.run}atmanlfinal'} + deps.append(rocoto.add_dependency(dep_dict)) + if self.app_config.do_jediocnvar: + dep_dict = {'type': 'task', 'name': f'{self.run}ocnanalpost'} + deps.append(rocoto.add_dependency(dep_dict)) + if self.app_config.do_jedisnowda: + dep_dict = {'type': 'task', 'name': f'{self.run}snowanl'} + deps.append(rocoto.add_dependency(dep_dict)) + if self.app_config.do_aero: + dep_dict = {'type': 'task', 'name': f'{self.run}aeroanlfinal'} + deps.append(rocoto.add_dependency(dep_dict)) + + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + + resources = self.get_resource('anlstat') + task_name = f'{self.run}anlstat' + task_dict = {'task_name': task_name, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars, + 'cycledef': self.run.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/anlstat.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' + } + + task = rocoto.create_task(task_dict) + + return task + def tracker(self): deps = [] dep_dict = {'type': 'metatask', 'name': f'{self.run}_atmos_prod'} @@ -2245,6 +2279,9 @@ def arch(self): if self.app_config.do_vminmon: dep_dict = {'type': 'task', 'name': f'{self.run}_vminmon'} deps.append(rocoto.add_dependency(dep_dict)) + if self.app_config.do_anlstat: + dep_dict = {'type': 'task', 'name': f'{self.run}anlstat'} + deps.append(rocoto.add_dependency(dep_dict)) elif self.run in ['gdas']: dep_dict = {'type': 'task', 'name': f'{self.run}_atmanlprod'} deps.append(rocoto.add_dependency(dep_dict)) @@ -2260,6 +2297,9 @@ def arch(self): if self.app_config.do_vminmon: dep_dict = {'type': 'task', 'name': f'{self.run}_vminmon'} deps.append(rocoto.add_dependency(dep_dict)) + if self.app_config.do_anlstat: + dep_dict = {'type': 'task', 'name': f'{self.run}anlstat'} + deps.append(rocoto.add_dependency(dep_dict)) if self.run in ['gfs'] and self.app_config.do_tracker: dep_dict = {'type': 'task', 'name': f'{self.run}_tracker'} deps.append(rocoto.add_dependency(dep_dict)) From c04eb6a60cb739d1415bf815e856bfc16918b12c Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Tue, 26 Nov 2024 12:37:27 +0000 Subject: [PATCH 05/39] update task names to include underscores --- workflow/rocoto/gfs_tasks.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index b3e2466768..a7506289ca 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -1741,22 +1741,22 @@ def vminmon(self): def anlstat(self): deps = [] if self.app_config.do_jediatmvar: - dep_dict = {'type': 'task', 'name': f'{self.run}atmanlfinal'} + dep_dict = {'type': 'task', 'name': f'{self.run}_atmanlfinal'} deps.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_jediocnvar: - dep_dict = {'type': 'task', 'name': f'{self.run}ocnanalpost'} + dep_dict = {'type': 'task', 'name': f'{self.run}_ocnanalpost'} deps.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_jedisnowda: - dep_dict = {'type': 'task', 'name': f'{self.run}snowanl'} + dep_dict = {'type': 'task', 'name': f'{self.run}_snowanl'} deps.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_aero: - dep_dict = {'type': 'task', 'name': f'{self.run}aeroanlfinal'} + dep_dict = {'type': 'task', 'name': f'{self.run}_aeroanlfinal'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('anlstat') - task_name = f'{self.run}anlstat' + task_name = f'{self.run}_anlstat' task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, @@ -2280,7 +2280,7 @@ def arch(self): dep_dict = {'type': 'task', 'name': f'{self.run}_vminmon'} deps.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_anlstat: - dep_dict = {'type': 'task', 'name': f'{self.run}anlstat'} + dep_dict = {'type': 'task', 'name': f'{self.run}_anlstat'} deps.append(rocoto.add_dependency(dep_dict)) elif self.run in ['gdas']: dep_dict = {'type': 'task', 'name': f'{self.run}_atmanlprod'} @@ -2298,7 +2298,7 @@ def arch(self): dep_dict = {'type': 'task', 'name': f'{self.run}_vminmon'} deps.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_anlstat: - dep_dict = {'type': 'task', 'name': f'{self.run}anlstat'} + dep_dict = {'type': 'task', 'name': f'{self.run}_anlstat'} deps.append(rocoto.add_dependency(dep_dict)) if self.run in ['gfs'] and self.app_config.do_tracker: dep_dict = {'type': 'task', 'name': f'{self.run}_tracker'} From cccd6704f92e7f41e93747e07b2f7380bdd0f7c2 Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Tue, 26 Nov 2024 13:21:28 +0000 Subject: [PATCH 06/39] solved issue of job not running, files in wrong directory --- jobs/JGLOBAL_ANALYSIS_STATS | 0 jobs/rocoto/anlstat.sh | 0 parm/{config/gfs => gdas}/stat/aero/jcb-base.yaml.j2 | 0 scripts/exglobal_analysis_stats.py | 0 4 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 jobs/JGLOBAL_ANALYSIS_STATS mode change 100644 => 100755 jobs/rocoto/anlstat.sh rename parm/{config/gfs => gdas}/stat/aero/jcb-base.yaml.j2 (100%) mode change 100644 => 100755 scripts/exglobal_analysis_stats.py diff --git a/jobs/JGLOBAL_ANALYSIS_STATS b/jobs/JGLOBAL_ANALYSIS_STATS old mode 100644 new mode 100755 diff --git a/jobs/rocoto/anlstat.sh b/jobs/rocoto/anlstat.sh old mode 100644 new mode 100755 diff --git a/parm/config/gfs/stat/aero/jcb-base.yaml.j2 b/parm/gdas/stat/aero/jcb-base.yaml.j2 similarity index 100% rename from parm/config/gfs/stat/aero/jcb-base.yaml.j2 rename to parm/gdas/stat/aero/jcb-base.yaml.j2 diff --git a/scripts/exglobal_analysis_stats.py b/scripts/exglobal_analysis_stats.py old mode 100644 new mode 100755 From cfb9f2556e1188643c8429f254ee7e798125d88e Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Tue, 26 Nov 2024 19:31:08 +0000 Subject: [PATCH 07/39] update to include execute and finalize --- parm/config/gfs/config.anlstat | 1 + parm/gdas/anlstat_jedi_config.yaml.j2 | 4 +- parm/gdas/stat/aero/jcb-base.yaml.j2 | 8 +++- scripts/exglobal_analysis_stats.py | 2 + ush/python/pygfs/task/stat_analysis.py | 54 +++++++++++++++++++++++++- 5 files changed, 64 insertions(+), 5 deletions(-) diff --git a/parm/config/gfs/config.anlstat b/parm/config/gfs/config.anlstat index 5c58eda542..3a0d99b2d5 100644 --- a/parm/config/gfs/config.anlstat +++ b/parm/config/gfs/config.anlstat @@ -12,5 +12,6 @@ export JEDI_CONFIG_YAML="${PARMgfs}/gdas/anlstat_jedi_config.yaml.j2" export JCB_BASE_YAML="${PARMgfs}/gdas/stat/aero/jcb-base.yaml.j2" export JCB_ALGO_YAML="${PARMgfs}/gdas/jcb-algorithms/anlstat.yaml.j2" export JEDIEXE=${HOMEgfs}/sorc/gdas.cd/build/bin/ioda-stats.x +export STAT_OUTDIR="/scratch1/NCEPDEV/da/Kevin.Dougherty/tmp_outdir/" echo "END: config.anlstat" \ No newline at end of file diff --git a/parm/gdas/anlstat_jedi_config.yaml.j2 b/parm/gdas/anlstat_jedi_config.yaml.j2 index bbefbaac2c..a0150cb479 100644 --- a/parm/gdas/anlstat_jedi_config.yaml.j2 +++ b/parm/gdas/anlstat_jedi_config.yaml.j2 @@ -1,8 +1,8 @@ -statanl: +anlstat: rundir: '{{ DATA }}' exe_src: '{{ JEDIEXE }}' mpi_cmd: '{{ APRUN_ANLSTAT }}' - jedi_args: None + # jedi_args: None jcb_base_yaml: '{{ PARMgfs }}/gdas/stat/aero/jcb-base.yaml.j2' jcb_algo_yaml: '{{ JCB_ALGO_YAML }}' jcb_algo: 'anlstat' \ No newline at end of file diff --git a/parm/gdas/stat/aero/jcb-base.yaml.j2 b/parm/gdas/stat/aero/jcb-base.yaml.j2 index f14a2664fd..84e48756cd 100644 --- a/parm/gdas/stat/aero/jcb-base.yaml.j2 +++ b/parm/gdas/stat/aero/jcb-base.yaml.j2 @@ -3,9 +3,15 @@ algorithm_path: "{{PARMgfs}}/gdas/jcb-algorithms" app_path_algorithm: "{{PARMgfs}}/gdas/jcb-gdas/algorithm/obstats/aero" +# Assimilation window +# ------------------- window_begin: "{{ STAT_WINDOW_BEGIN | to_isotime }}" window_YMDH: "{{ STAT_WINDOW_BEGIN | to_YMDH }}" window_length: "{{ STAT_WINDOW_LENGTH }}" +bound_to_include: begin + +stat_current_cycle_iso: "{{ current_cycle | to_isotime }}" +stat_current_cycle_YMDH: "{{ current_cycle | to_YMDH }}" # Inputted list of ob spaces # -------------------------- @@ -20,5 +26,5 @@ aero_obsdatain_observed_variables: ['aerosolOpticalDepth'] # Variables # --------- aero_variables: ['aerosolOpticalDepth'] -aero_file_groups: ['bkgmob', 'bkgmob1'] +aero_file_groups: ['ombg', 'oman'] aero_file_qc_groups: ['EffectiveQC0', 'EffectiveQC1'] \ No newline at end of file diff --git a/scripts/exglobal_analysis_stats.py b/scripts/exglobal_analysis_stats.py index 72979cdba2..805d0c855d 100755 --- a/scripts/exglobal_analysis_stats.py +++ b/scripts/exglobal_analysis_stats.py @@ -25,3 +25,5 @@ # Initialize JEDI variational analysis StatAnl.initialize() + StatAnl.execute('anlstat') + StatAnl.finalize() diff --git a/ush/python/pygfs/task/stat_analysis.py b/ush/python/pygfs/task/stat_analysis.py index 156e95c07a..f26e437cf3 100644 --- a/ush/python/pygfs/task/stat_analysis.py +++ b/ush/python/pygfs/task/stat_analysis.py @@ -65,7 +65,8 @@ def __init__(self, config: Dict[str, Any]): self.task_config = AttrDict(**self.task_config, **local_dict) # Create dictionary of Jedi objects - self.jedi_dict = Jedi.get_jedi_dict(self.task_config.JEDI_CONFIG_YAML, self.task_config) + expected_keys = ['anlstat'] + self.jedi_dict = Jedi.get_jedi_dict(self.task_config.JEDI_CONFIG_YAML, self.task_config, expected_keys) @logit(logger) def initialize(self) -> None: @@ -112,4 +113,53 @@ def initialize(self) -> None: # initialize JEDI application logger.info(f"Initializing JEDI variational DA application") - self.jedi_dict['statanl'].initialize(self.task_config) + self.jedi_dict['anlstat'].initialize(self.task_config) + + @logit(logger) + def execute(self, jedi_dict_key: str) -> None: + """Execute JEDI application of atm analysis + + Parameters + ---------- + jedi_dict_key + key specifying particular Jedi object in self.jedi_dict + + Returns + ---------- + None + """ + + self.jedi_dict[jedi_dict_key].execute() + + @logit(logger) + def finalize(self) -> None: + """Finalize a global atm analysis + + This method will finalize a global atm analysis using JEDI. + This includes: + - tar output diag files and place in ROTDIR + - copy the generated YAML file from initialize to the ROTDIR + - copy the updated bias correction files to ROTDIR + + Parameters + ---------- + None + + Returns + ---------- + None + """ + + # get list of output diag files + diags = glob.glob(os.path.join(self.task_config.DATA, '*output_aod.nc')) + + logger.debug(f"diag files: {diags}") + + for diagfile in diags: + outfile = os.path.basename(diagfile) + dest = os.path.join(self.task_config.STAT_OUTDIR, f'{outfile}') + logger.debug(f"copying {diagfile} to {dest}") + diag_copy = { + 'copy': [[diagfile, dest]] + } + FileHandler(diag_copy).sync() From 899b20baf29d0caba45f605acdfce56f92ef5950 Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Thu, 12 Dec 2024 19:59:48 +0000 Subject: [PATCH 08/39] updated scripts to that run successfully --- jobs/JGLOBAL_ANALYSIS_STATS | 1 + parm/config/gfs/config.anlstat | 2 +- parm/gdas/anlstat_jedi_config.yaml.j2 | 14 ++++- scripts/exglobal_analysis_stats.py | 16 +++-- ush/python/pygfs/task/stat_analysis.py | 81 +++++++++++++------------- 5 files changed, 68 insertions(+), 46 deletions(-) diff --git a/jobs/JGLOBAL_ANALYSIS_STATS b/jobs/JGLOBAL_ANALYSIS_STATS index 537fc03122..cd7ce2e752 100755 --- a/jobs/JGLOBAL_ANALYSIS_STATS +++ b/jobs/JGLOBAL_ANALYSIS_STATS @@ -15,6 +15,7 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "anlstat" -c "base anlstat" # Generate COM variables from templates YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_OBS COM_CHEM_ANALYSIS +YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_SNOW_ANALYSIS COM_CONF ############################################################### # Run relevant script diff --git a/parm/config/gfs/config.anlstat b/parm/config/gfs/config.anlstat index 3a0d99b2d5..e7831bd565 100644 --- a/parm/config/gfs/config.anlstat +++ b/parm/config/gfs/config.anlstat @@ -12,6 +12,6 @@ export JEDI_CONFIG_YAML="${PARMgfs}/gdas/anlstat_jedi_config.yaml.j2" export JCB_BASE_YAML="${PARMgfs}/gdas/stat/aero/jcb-base.yaml.j2" export JCB_ALGO_YAML="${PARMgfs}/gdas/jcb-algorithms/anlstat.yaml.j2" export JEDIEXE=${HOMEgfs}/sorc/gdas.cd/build/bin/ioda-stats.x -export STAT_OUTDIR="/scratch1/NCEPDEV/da/Kevin.Dougherty/tmp_outdir/" +export STAT_OUTDIR="${COMOUT}/${net}/${run.yyyymmdd}/${hh}/products/stats/" echo "END: config.anlstat" \ No newline at end of file diff --git a/parm/gdas/anlstat_jedi_config.yaml.j2 b/parm/gdas/anlstat_jedi_config.yaml.j2 index a0150cb479..0c0fe70dda 100644 --- a/parm/gdas/anlstat_jedi_config.yaml.j2 +++ b/parm/gdas/anlstat_jedi_config.yaml.j2 @@ -1,8 +1,20 @@ -anlstat: +aero: rundir: '{{ DATA }}' exe_src: '{{ JEDIEXE }}' mpi_cmd: '{{ APRUN_ANLSTAT }}' + stat_file_path: '{{ COM_CHEM_ANALYSIS }}' + stat_file_name: 'aerostat' # jedi_args: None jcb_base_yaml: '{{ PARMgfs }}/gdas/stat/aero/jcb-base.yaml.j2' jcb_algo_yaml: '{{ JCB_ALGO_YAML }}' + jcb_algo: 'anlstat' +snow: + rundir: '{{ DATA }}' + exe_src: '{{ JEDIEXE }}' + mpi_cmd: '{{ APRUN_ANLSTAT }}' + stat_file_path: '{{ COM_SNOW_ANALYSIS }}' + stat_file_name: 'snowstat.tgz' + # jedi_args: None + jcb_base_yaml: '{{ PARMgfs }}/gdas/stat/snow/jcb-base.yaml.j2' + jcb_algo_yaml: '{{ JCB_ALGO_YAML }}' jcb_algo: 'anlstat' \ No newline at end of file diff --git a/scripts/exglobal_analysis_stats.py b/scripts/exglobal_analysis_stats.py index 805d0c855d..533b7d47dc 100755 --- a/scripts/exglobal_analysis_stats.py +++ b/scripts/exglobal_analysis_stats.py @@ -2,8 +2,8 @@ # exglobal_stat_analysis.py # This script creates an StatAnalysis class -# and runs the initialize method -# which create and stage the runtime directory +# and runs the initialize, execute, and finalize +# methods which create and stage the runtime directory # and create the YAML configuration # for a global stat analysis import os @@ -23,7 +23,15 @@ # Instantiate the atm analysis task StatAnl = StatAnalysis(config) + # Create list based on DA components + StatAnl.task_config['STAT_OBS'] = [] + if StatAnl.task_config.DO_AERO: + StatAnl.task_config['STAT_OBS'].append('aero') + if StatAnl.task_config.DO_JEDISNOWDA: + StatAnl.task_config['STAT_OBS'].append('snow') + # Initialize JEDI variational analysis StatAnl.initialize() - StatAnl.execute('anlstat') - StatAnl.finalize() + for ob in StatAnl.task_config.STAT_OBS: + StatAnl.execute(ob) + StatAnl.finalize(ob) diff --git a/ush/python/pygfs/task/stat_analysis.py b/ush/python/pygfs/task/stat_analysis.py index f26e437cf3..fb9e7af12b 100644 --- a/ush/python/pygfs/task/stat_analysis.py +++ b/ush/python/pygfs/task/stat_analysis.py @@ -11,9 +11,9 @@ from wxflow import (AttrDict, FileHandler, - add_to_datetime, to_fv3time, to_timedelta, to_YMDH, + add_to_datetime, to_timedelta, Task, - parse_j2yaml, save_as_yaml, + parse_j2yaml, logit) from pygfs.jedi import Jedi @@ -64,10 +64,6 @@ def __init__(self, config: Dict[str, Any]): # Extend task_config with local_dict self.task_config = AttrDict(**self.task_config, **local_dict) - # Create dictionary of Jedi objects - expected_keys = ['anlstat'] - self.jedi_dict = Jedi.get_jedi_dict(self.task_config.JEDI_CONFIG_YAML, self.task_config, expected_keys) - @logit(logger) def initialize(self) -> None: """ @@ -83,41 +79,49 @@ def initialize(self) -> None: ---------- None """ + # Create dictionary of Jedi objects + expected_keys = self.task_config.STAT_OBS + self.jedi_dict = Jedi.get_jedi_dict(self.task_config.JEDI_CONFIG_YAML, self.task_config, expected_keys) + logger.info(f"Copying files to {self.task_config.DATA}/stats") - # Copy stat files to DATA path - aerostat = os.path.join(self.task_config.COM_CHEM_ANALYSIS, f"{self.task_config['APREFIX']}aerostat") - dest = os.path.join(self.task_config.DATA, "aerostats") - statlist = [[aerostat, dest]] - FileHandler({'copy': statlist}).sync() + for OB in self.task_config.STAT_OBS: + # Parse JEDI analysis stat jinja file + obs_dict = parse_j2yaml(self.task_config.JEDI_CONFIG_YAML, self.task_config) - # Open tar file - logger.info(f"Open tarred stat file in {dest}") - with tarfile.open(dest, "r") as tar: - # Extract all files to the current directory - tar.extractall() + # Copy stat files to DATA path + instat_files = os.path.join(obs_dict[OB]['stat_file_path'], f"{self.task_config['APREFIX']}{obs_dict[OB]['stat_file_name']}") + dest = os.path.join(self.task_config.DATA, obs_dict[OB]['stat_file_name']) + statlist = [[instat_files, dest]] + FileHandler({'copy': statlist}).sync() - # Gunzip .nc files - logger.info("Gunzip files from tar file") - gz_files = glob.glob(os.path.join(self.task_config.DATA, "*gz")) + # Open tar file + logger.info(f"Open tarred stat file in {dest}") + with tarfile.open(dest, "r") as tar: + # Extract all files to the current directory + tar.extractall() - for diagfile in gz_files: - with gzip.open(diagfile, 'rb') as f_in: - with open(diagfile[:-3], 'wb') as f_out: - f_out.write(f_in.read()) + # Gunzip .nc files + logger.info("Gunzip files from tar file") + gz_files = glob.glob(os.path.join(self.task_config.DATA, "*gz")) - # Get list of .nc4 files - obs_space_paths = glob.glob(os.path.join(self.task_config.DATA, "*.nc4")) + for diagfile in gz_files: + with gzip.open(diagfile, 'rb') as f_in: + with open(diagfile[:-3], 'wb') as f_out: + f_out.write(f_in.read()) - self.task_config.OBSPACES_LIST = ['_'.join(os.path.basename(path).split('_')[1:3]) for path in obs_space_paths] + # Get list of .nc4 files + obs_space_paths = glob.glob(os.path.join(self.task_config.DATA, "*.nc4")) - # initialize JEDI application - logger.info(f"Initializing JEDI variational DA application") - self.jedi_dict['anlstat'].initialize(self.task_config) + self.task_config.OBSPACES_LIST = ['_'.join(os.path.basename(path).split('_')[1:3]) for path in obs_space_paths] + + # initialize JEDI application + logger.info(f"Initializing JEDI variational DA application") + self.jedi_dict[OB].initialize(self.task_config) @logit(logger) def execute(self, jedi_dict_key: str) -> None: - """Execute JEDI application of atm analysis + """Execute JEDI application of stat analysis Parameters ---------- @@ -132,18 +136,17 @@ def execute(self, jedi_dict_key: str) -> None: self.jedi_dict[jedi_dict_key].execute() @logit(logger) - def finalize(self) -> None: - """Finalize a global atm analysis + def finalize(self, jedi_dict_key: str) -> None: + """Finalize a statistic analysis - This method will finalize a global atm analysis using JEDI. + This method will finalize a statistic analysis using JEDI. This includes: - - tar output diag files and place in ROTDIR - - copy the generated YAML file from initialize to the ROTDIR - - copy the updated bias correction files to ROTDIR + - copying stat files to specified outdir Parameters ---------- - None + jedi_dict_key + key specifying particular Jedi object in self.jedi_dict Returns ---------- @@ -153,11 +156,9 @@ def finalize(self) -> None: # get list of output diag files diags = glob.glob(os.path.join(self.task_config.DATA, '*output_aod.nc')) - logger.debug(f"diag files: {diags}") - for diagfile in diags: outfile = os.path.basename(diagfile) - dest = os.path.join(self.task_config.STAT_OUTDIR, f'{outfile}') + dest = os.path.join(f'{self.task_config.STAT_OUTDIR}/{jedi_dict_key}/', f'{outfile}') logger.debug(f"copying {diagfile} to {dest}") diag_copy = { 'copy': [[diagfile, dest]] From 3d09d46a4fc3ddf8e60715d64c17668b29f1ca2e Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Mon, 16 Dec 2024 15:15:23 +0000 Subject: [PATCH 09/39] merge develop --- parm/config/gfs/config.base | 3 --- workflow/applications/applications.py | 27 +++++++++++++++++++++++++++ workflow/applications/gfs_cycled.py | 3 +++ workflow/rocoto/gfs_tasks.py | 23 +++++++++++++++++++++++ 4 files changed, 53 insertions(+), 3 deletions(-) diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index 06e5aad07e..795dc0e4eb 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -75,9 +75,6 @@ export DO_NPOESS="@DO_NPOESS@" # NPOESS products export DO_TRACKER="@DO_TRACKER@" # Hurricane track verification export DO_GENESIS="@DO_GENESIS@" # Cyclone genesis verification export DO_GENESIS_FSU="@DO_GENESIS_FSU@" # Cyclone genesis verification (FSU) -export DO_VERFOZN="YES" # Ozone data assimilation monitoring -export DO_VERFRAD="YES" # Radiance data assimilation monitoring -export DO_VMINMON="YES" # GSI minimization monitoring export DO_MOS="NO" # GFS Model Output Statistics - Only supported on WCOSS2 # NO for retrospective parallel; YES for real-time parallel diff --git a/workflow/applications/applications.py b/workflow/applications/applications.py index a5a4c2094d..dbc1c82673 100644 --- a/workflow/applications/applications.py +++ b/workflow/applications/applications.py @@ -40,6 +40,7 @@ def __init__(self, conf: Configuration) -> None: f'Valid application modes are:\n' f'{", ".join(self.VALID_MODES)}\n') +<<<<<<< HEAD self.net = base['NET'] self.model_app = base.get('APP', 'ATM') self.do_atm = base.get('DO_ATM', True) @@ -64,6 +65,32 @@ def __init__(self, conf: Configuration) -> None: self.do_goes = base.get('DO_GOES', False) self.do_mos = base.get('DO_MOS', False) self.do_extractvars = base.get('DO_EXTRACTVARS', False) +======= + self.net = _base['NET'] + self.model_app = _base.get('APP', 'ATM') + self.do_atm = _base.get('DO_ATM', True) + self.do_wave = _base.get('DO_WAVE', False) + self.do_wave_bnd = _base.get('DOBNDPNT_WAVE', False) + self.do_ocean = _base.get('DO_OCN', False) + self.do_ice = _base.get('DO_ICE', False) + self.do_aero = _base.get('DO_AERO', False) + self.do_prep_obs_aero = _base.get('DO_PREP_OBS_AERO', False) + self.do_bufrsnd = _base.get('DO_BUFRSND', False) + self.do_gempak = _base.get('DO_GEMPAK', False) + self.do_awips = _base.get('DO_AWIPS', False) + self.do_verfozn = _base.get('DO_VERFOZN', True) + self.do_verfrad = _base.get('DO_VERFRAD', True) + self.do_vminmon = _base.get('DO_VMINMON', True) + self.do_anlstat = _base.get('DO_ANLSTAT', False) + self.do_tracker = _base.get('DO_TRACKER', True) + self.do_genesis = _base.get('DO_GENESIS', True) + self.do_genesis_fsu = _base.get('DO_GENESIS_FSU', False) + self.do_metp = _base.get('DO_METP', False) + self.do_upp = not _base.get('WRITE_DOPOST', True) + self.do_goes = _base.get('DO_GOES', False) + self.do_mos = _base.get('DO_MOS', False) + self.do_extractvars = _base.get('DO_EXTRACTVARS', False) +>>>>>>> fbe784df (Stubs for anlstat job) self.do_hpssarch = base.get('HPSSARCH', False) diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index 983acb1493..0f9e420b65 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -274,6 +274,9 @@ def get_task_names(self): if self.do_vminmon: gfs_tasks += ['vminmon'] + + if self.do_anlstat: + gfs_tasks += ['anlstat'] if self.do_anlstat: gfs_tasks += ['anlstat'] diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 12b0a78a54..2c7ba9d2b0 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -1756,6 +1756,7 @@ def vminmon(self): def anlstat(self): deps = [] if self.app_config.do_jediatmvar: +<<<<<<< HEAD dep_dict = {'type': 'task', 'name': f'{self.run}_atmanlfinal'} deps.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_jediocnvar: @@ -1766,12 +1767,28 @@ def anlstat(self): deps.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_aero: dep_dict = {'type': 'task', 'name': f'{self.run}_aeroanlfinal'} +======= + dep_dict = {'type': 'task', 'name': f'{self.run}atmanlfinal'} + deps.append(rocoto.add_dependency(dep_dict)) + if self.do_jediocnvar: + dep_dict = {'type': 'task', 'name': f'{self.run}ocnanalpost'} + deps.append(rocoto.add_dependency(dep_dict)) + if self.do_jedisnowda: + dep_dict = {'type': 'task', 'name': f'{self.run}snowanl'} + deps.append(rocoto.add_dependency(dep_dict)) + if self.do_aero: + dep_dict = {'type': 'task', 'name': f'{self.run}aeroanlfinal'} +>>>>>>> fbe784df (Stubs for anlstat job) deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('anlstat') +<<<<<<< HEAD task_name = f'{self.run}_anlstat' +======= + task_name = f'{self.run}anlstat' +>>>>>>> fbe784df (Stubs for anlstat job) task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, @@ -2297,6 +2314,9 @@ def arch(self): if self.app_config.do_anlstat: dep_dict = {'type': 'task', 'name': f'{self.run}_anlstat'} deps.append(rocoto.add_dependency(dep_dict)) + if self.app_config.do_anlstat: + dep_dict = {'type': 'task', 'name': f'{self.run}anlstat'} + deps.append(rocoto.add_dependency(dep_dict)) elif self.run in ['gdas']: dep_dict = {'type': 'task', 'name': f'{self.run}_atmanlprod'} deps.append(rocoto.add_dependency(dep_dict)) @@ -2315,6 +2335,9 @@ def arch(self): if self.app_config.do_anlstat: dep_dict = {'type': 'task', 'name': f'{self.run}_anlstat'} deps.append(rocoto.add_dependency(dep_dict)) + if self.app_config.do_anlstat: + dep_dict = {'type': 'task', 'name': f'{self.run}anlstat'} + deps.append(rocoto.add_dependency(dep_dict)) if self.run in ['gfs'] and self.app_config.do_tracker: dep_dict = {'type': 'task', 'name': f'{self.run}_tracker'} deps.append(rocoto.add_dependency(dep_dict)) From 840153a67cea639f539bfaf347c00ea9c659a150 Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Mon, 16 Dec 2024 15:28:43 +0000 Subject: [PATCH 10/39] update files --- workflow/applications/gfs_cycled.py | 3 +++ workflow/rocoto/gfs_tasks.py | 16 ++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index 0f9e420b65..958cee36e8 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -277,9 +277,12 @@ def get_task_names(self): if self.do_anlstat: gfs_tasks += ['anlstat'] +<<<<<<< HEAD if self.do_anlstat: gfs_tasks += ['anlstat'] +======= +>>>>>>> fbe784df (Stubs for anlstat job) if self.do_tracker: gfs_tasks += ['tracker'] diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 2c7ba9d2b0..bcb56e25eb 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -1756,6 +1756,7 @@ def vminmon(self): def anlstat(self): deps = [] if self.app_config.do_jediatmvar: +<<<<<<< HEAD <<<<<<< HEAD dep_dict = {'type': 'task', 'name': f'{self.run}_atmanlfinal'} deps.append(rocoto.add_dependency(dep_dict)) @@ -1768,6 +1769,8 @@ def anlstat(self): if self.app_config.do_aero: dep_dict = {'type': 'task', 'name': f'{self.run}_aeroanlfinal'} ======= +======= +>>>>>>> fbe784df (Stubs for anlstat job) dep_dict = {'type': 'task', 'name': f'{self.run}atmanlfinal'} deps.append(rocoto.add_dependency(dep_dict)) if self.do_jediocnvar: @@ -1778,14 +1781,21 @@ def anlstat(self): deps.append(rocoto.add_dependency(dep_dict)) if self.do_aero: dep_dict = {'type': 'task', 'name': f'{self.run}aeroanlfinal'} +<<<<<<< HEAD +>>>>>>> fbe784df (Stubs for anlstat job) +======= >>>>>>> fbe784df (Stubs for anlstat job) deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('anlstat') +<<<<<<< HEAD <<<<<<< HEAD task_name = f'{self.run}_anlstat' +======= + task_name = f'{self.run}anlstat' +>>>>>>> fbe784df (Stubs for anlstat job) ======= task_name = f'{self.run}anlstat' >>>>>>> fbe784df (Stubs for anlstat job) @@ -2317,6 +2327,9 @@ def arch(self): if self.app_config.do_anlstat: dep_dict = {'type': 'task', 'name': f'{self.run}anlstat'} deps.append(rocoto.add_dependency(dep_dict)) + if self.app_config.do_anlstat: + dep_dict = {'type': 'task', 'name': f'{self.run}anlstat'} + deps.append(rocoto.add_dependency(dep_dict)) elif self.run in ['gdas']: dep_dict = {'type': 'task', 'name': f'{self.run}_atmanlprod'} deps.append(rocoto.add_dependency(dep_dict)) @@ -2338,6 +2351,9 @@ def arch(self): if self.app_config.do_anlstat: dep_dict = {'type': 'task', 'name': f'{self.run}anlstat'} deps.append(rocoto.add_dependency(dep_dict)) + if self.app_config.do_anlstat: + dep_dict = {'type': 'task', 'name': f'{self.run}anlstat'} + deps.append(rocoto.add_dependency(dep_dict)) if self.run in ['gfs'] and self.app_config.do_tracker: dep_dict = {'type': 'task', 'name': f'{self.run}_tracker'} deps.append(rocoto.add_dependency(dep_dict)) From ae48429632e99fb7eda46ca9351c1ab2d5a70f1b Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Mon, 16 Dec 2024 15:56:21 +0000 Subject: [PATCH 11/39] fix merge conflicts --- workflow/applications/gfs_cycled.py | 3 +++ workflow/rocoto/gfs_tasks.py | 35 +++++++---------------------- 2 files changed, 11 insertions(+), 27 deletions(-) diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index 958cee36e8..3d2dba2ea9 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -277,11 +277,14 @@ def get_task_names(self): if self.do_anlstat: gfs_tasks += ['anlstat'] +<<<<<<< HEAD <<<<<<< HEAD if self.do_anlstat: gfs_tasks += ['anlstat'] ======= +>>>>>>> fbe784df (Stubs for anlstat job) +======= >>>>>>> fbe784df (Stubs for anlstat job) if self.do_tracker: diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index bcb56e25eb..217d7c9e05 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -1756,8 +1756,6 @@ def vminmon(self): def anlstat(self): deps = [] if self.app_config.do_jediatmvar: -<<<<<<< HEAD -<<<<<<< HEAD dep_dict = {'type': 'task', 'name': f'{self.run}_atmanlfinal'} deps.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_jediocnvar: @@ -1768,37 +1766,14 @@ def anlstat(self): deps.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_aero: dep_dict = {'type': 'task', 'name': f'{self.run}_aeroanlfinal'} -======= -======= ->>>>>>> fbe784df (Stubs for anlstat job) - dep_dict = {'type': 'task', 'name': f'{self.run}atmanlfinal'} - deps.append(rocoto.add_dependency(dep_dict)) - if self.do_jediocnvar: - dep_dict = {'type': 'task', 'name': f'{self.run}ocnanalpost'} - deps.append(rocoto.add_dependency(dep_dict)) - if self.do_jedisnowda: - dep_dict = {'type': 'task', 'name': f'{self.run}snowanl'} - deps.append(rocoto.add_dependency(dep_dict)) - if self.do_aero: - dep_dict = {'type': 'task', 'name': f'{self.run}aeroanlfinal'} -<<<<<<< HEAD ->>>>>>> fbe784df (Stubs for anlstat job) -======= ->>>>>>> fbe784df (Stubs for anlstat job) + deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('anlstat') -<<<<<<< HEAD -<<<<<<< HEAD task_name = f'{self.run}_anlstat' -======= - task_name = f'{self.run}anlstat' ->>>>>>> fbe784df (Stubs for anlstat job) -======= - task_name = f'{self.run}anlstat' ->>>>>>> fbe784df (Stubs for anlstat job) + task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, @@ -2330,6 +2305,9 @@ def arch(self): if self.app_config.do_anlstat: dep_dict = {'type': 'task', 'name': f'{self.run}anlstat'} deps.append(rocoto.add_dependency(dep_dict)) + if self.app_config.do_anlstat: + dep_dict = {'type': 'task', 'name': f'{self.run}anlstat'} + deps.append(rocoto.add_dependency(dep_dict)) elif self.run in ['gdas']: dep_dict = {'type': 'task', 'name': f'{self.run}_atmanlprod'} deps.append(rocoto.add_dependency(dep_dict)) @@ -2354,6 +2332,9 @@ def arch(self): if self.app_config.do_anlstat: dep_dict = {'type': 'task', 'name': f'{self.run}anlstat'} deps.append(rocoto.add_dependency(dep_dict)) + if self.app_config.do_anlstat: + dep_dict = {'type': 'task', 'name': f'{self.run}anlstat'} + deps.append(rocoto.add_dependency(dep_dict)) if self.run in ['gfs'] and self.app_config.do_tracker: dep_dict = {'type': 'task', 'name': f'{self.run}_tracker'} deps.append(rocoto.add_dependency(dep_dict)) From ba89a3fc95e85a54af7d4b09c0f125732744fad5 Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Mon, 16 Dec 2024 16:56:57 +0000 Subject: [PATCH 12/39] Revert "updated scripts to that run successfully" This reverts commit 899b20baf29d0caba45f605acdfce56f92ef5950. --- jobs/JGLOBAL_ANALYSIS_STATS | 1 - parm/config/gfs/config.anlstat | 2 +- parm/gdas/anlstat_jedi_config.yaml.j2 | 14 +---- scripts/exglobal_analysis_stats.py | 16 ++--- ush/python/pygfs/task/stat_analysis.py | 81 +++++++++++++------------- 5 files changed, 46 insertions(+), 68 deletions(-) diff --git a/jobs/JGLOBAL_ANALYSIS_STATS b/jobs/JGLOBAL_ANALYSIS_STATS index cd7ce2e752..537fc03122 100755 --- a/jobs/JGLOBAL_ANALYSIS_STATS +++ b/jobs/JGLOBAL_ANALYSIS_STATS @@ -15,7 +15,6 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "anlstat" -c "base anlstat" # Generate COM variables from templates YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_OBS COM_CHEM_ANALYSIS -YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_SNOW_ANALYSIS COM_CONF ############################################################### # Run relevant script diff --git a/parm/config/gfs/config.anlstat b/parm/config/gfs/config.anlstat index e7831bd565..3a0d99b2d5 100644 --- a/parm/config/gfs/config.anlstat +++ b/parm/config/gfs/config.anlstat @@ -12,6 +12,6 @@ export JEDI_CONFIG_YAML="${PARMgfs}/gdas/anlstat_jedi_config.yaml.j2" export JCB_BASE_YAML="${PARMgfs}/gdas/stat/aero/jcb-base.yaml.j2" export JCB_ALGO_YAML="${PARMgfs}/gdas/jcb-algorithms/anlstat.yaml.j2" export JEDIEXE=${HOMEgfs}/sorc/gdas.cd/build/bin/ioda-stats.x -export STAT_OUTDIR="${COMOUT}/${net}/${run.yyyymmdd}/${hh}/products/stats/" +export STAT_OUTDIR="/scratch1/NCEPDEV/da/Kevin.Dougherty/tmp_outdir/" echo "END: config.anlstat" \ No newline at end of file diff --git a/parm/gdas/anlstat_jedi_config.yaml.j2 b/parm/gdas/anlstat_jedi_config.yaml.j2 index 0c0fe70dda..a0150cb479 100644 --- a/parm/gdas/anlstat_jedi_config.yaml.j2 +++ b/parm/gdas/anlstat_jedi_config.yaml.j2 @@ -1,20 +1,8 @@ -aero: +anlstat: rundir: '{{ DATA }}' exe_src: '{{ JEDIEXE }}' mpi_cmd: '{{ APRUN_ANLSTAT }}' - stat_file_path: '{{ COM_CHEM_ANALYSIS }}' - stat_file_name: 'aerostat' # jedi_args: None jcb_base_yaml: '{{ PARMgfs }}/gdas/stat/aero/jcb-base.yaml.j2' jcb_algo_yaml: '{{ JCB_ALGO_YAML }}' - jcb_algo: 'anlstat' -snow: - rundir: '{{ DATA }}' - exe_src: '{{ JEDIEXE }}' - mpi_cmd: '{{ APRUN_ANLSTAT }}' - stat_file_path: '{{ COM_SNOW_ANALYSIS }}' - stat_file_name: 'snowstat.tgz' - # jedi_args: None - jcb_base_yaml: '{{ PARMgfs }}/gdas/stat/snow/jcb-base.yaml.j2' - jcb_algo_yaml: '{{ JCB_ALGO_YAML }}' jcb_algo: 'anlstat' \ No newline at end of file diff --git a/scripts/exglobal_analysis_stats.py b/scripts/exglobal_analysis_stats.py index 533b7d47dc..805d0c855d 100755 --- a/scripts/exglobal_analysis_stats.py +++ b/scripts/exglobal_analysis_stats.py @@ -2,8 +2,8 @@ # exglobal_stat_analysis.py # This script creates an StatAnalysis class -# and runs the initialize, execute, and finalize -# methods which create and stage the runtime directory +# and runs the initialize method +# which create and stage the runtime directory # and create the YAML configuration # for a global stat analysis import os @@ -23,15 +23,7 @@ # Instantiate the atm analysis task StatAnl = StatAnalysis(config) - # Create list based on DA components - StatAnl.task_config['STAT_OBS'] = [] - if StatAnl.task_config.DO_AERO: - StatAnl.task_config['STAT_OBS'].append('aero') - if StatAnl.task_config.DO_JEDISNOWDA: - StatAnl.task_config['STAT_OBS'].append('snow') - # Initialize JEDI variational analysis StatAnl.initialize() - for ob in StatAnl.task_config.STAT_OBS: - StatAnl.execute(ob) - StatAnl.finalize(ob) + StatAnl.execute('anlstat') + StatAnl.finalize() diff --git a/ush/python/pygfs/task/stat_analysis.py b/ush/python/pygfs/task/stat_analysis.py index fb9e7af12b..f26e437cf3 100644 --- a/ush/python/pygfs/task/stat_analysis.py +++ b/ush/python/pygfs/task/stat_analysis.py @@ -11,9 +11,9 @@ from wxflow import (AttrDict, FileHandler, - add_to_datetime, to_timedelta, + add_to_datetime, to_fv3time, to_timedelta, to_YMDH, Task, - parse_j2yaml, + parse_j2yaml, save_as_yaml, logit) from pygfs.jedi import Jedi @@ -64,6 +64,10 @@ def __init__(self, config: Dict[str, Any]): # Extend task_config with local_dict self.task_config = AttrDict(**self.task_config, **local_dict) + # Create dictionary of Jedi objects + expected_keys = ['anlstat'] + self.jedi_dict = Jedi.get_jedi_dict(self.task_config.JEDI_CONFIG_YAML, self.task_config, expected_keys) + @logit(logger) def initialize(self) -> None: """ @@ -79,49 +83,41 @@ def initialize(self) -> None: ---------- None """ - # Create dictionary of Jedi objects - expected_keys = self.task_config.STAT_OBS - self.jedi_dict = Jedi.get_jedi_dict(self.task_config.JEDI_CONFIG_YAML, self.task_config, expected_keys) - logger.info(f"Copying files to {self.task_config.DATA}/stats") - for OB in self.task_config.STAT_OBS: - # Parse JEDI analysis stat jinja file - obs_dict = parse_j2yaml(self.task_config.JEDI_CONFIG_YAML, self.task_config) + # Copy stat files to DATA path + aerostat = os.path.join(self.task_config.COM_CHEM_ANALYSIS, f"{self.task_config['APREFIX']}aerostat") + dest = os.path.join(self.task_config.DATA, "aerostats") + statlist = [[aerostat, dest]] + FileHandler({'copy': statlist}).sync() - # Copy stat files to DATA path - instat_files = os.path.join(obs_dict[OB]['stat_file_path'], f"{self.task_config['APREFIX']}{obs_dict[OB]['stat_file_name']}") - dest = os.path.join(self.task_config.DATA, obs_dict[OB]['stat_file_name']) - statlist = [[instat_files, dest]] - FileHandler({'copy': statlist}).sync() + # Open tar file + logger.info(f"Open tarred stat file in {dest}") + with tarfile.open(dest, "r") as tar: + # Extract all files to the current directory + tar.extractall() - # Open tar file - logger.info(f"Open tarred stat file in {dest}") - with tarfile.open(dest, "r") as tar: - # Extract all files to the current directory - tar.extractall() + # Gunzip .nc files + logger.info("Gunzip files from tar file") + gz_files = glob.glob(os.path.join(self.task_config.DATA, "*gz")) - # Gunzip .nc files - logger.info("Gunzip files from tar file") - gz_files = glob.glob(os.path.join(self.task_config.DATA, "*gz")) + for diagfile in gz_files: + with gzip.open(diagfile, 'rb') as f_in: + with open(diagfile[:-3], 'wb') as f_out: + f_out.write(f_in.read()) - for diagfile in gz_files: - with gzip.open(diagfile, 'rb') as f_in: - with open(diagfile[:-3], 'wb') as f_out: - f_out.write(f_in.read()) + # Get list of .nc4 files + obs_space_paths = glob.glob(os.path.join(self.task_config.DATA, "*.nc4")) - # Get list of .nc4 files - obs_space_paths = glob.glob(os.path.join(self.task_config.DATA, "*.nc4")) + self.task_config.OBSPACES_LIST = ['_'.join(os.path.basename(path).split('_')[1:3]) for path in obs_space_paths] - self.task_config.OBSPACES_LIST = ['_'.join(os.path.basename(path).split('_')[1:3]) for path in obs_space_paths] - - # initialize JEDI application - logger.info(f"Initializing JEDI variational DA application") - self.jedi_dict[OB].initialize(self.task_config) + # initialize JEDI application + logger.info(f"Initializing JEDI variational DA application") + self.jedi_dict['anlstat'].initialize(self.task_config) @logit(logger) def execute(self, jedi_dict_key: str) -> None: - """Execute JEDI application of stat analysis + """Execute JEDI application of atm analysis Parameters ---------- @@ -136,17 +132,18 @@ def execute(self, jedi_dict_key: str) -> None: self.jedi_dict[jedi_dict_key].execute() @logit(logger) - def finalize(self, jedi_dict_key: str) -> None: - """Finalize a statistic analysis + def finalize(self) -> None: + """Finalize a global atm analysis - This method will finalize a statistic analysis using JEDI. + This method will finalize a global atm analysis using JEDI. This includes: - - copying stat files to specified outdir + - tar output diag files and place in ROTDIR + - copy the generated YAML file from initialize to the ROTDIR + - copy the updated bias correction files to ROTDIR Parameters ---------- - jedi_dict_key - key specifying particular Jedi object in self.jedi_dict + None Returns ---------- @@ -156,9 +153,11 @@ def finalize(self, jedi_dict_key: str) -> None: # get list of output diag files diags = glob.glob(os.path.join(self.task_config.DATA, '*output_aod.nc')) + logger.debug(f"diag files: {diags}") + for diagfile in diags: outfile = os.path.basename(diagfile) - dest = os.path.join(f'{self.task_config.STAT_OUTDIR}/{jedi_dict_key}/', f'{outfile}') + dest = os.path.join(self.task_config.STAT_OUTDIR, f'{outfile}') logger.debug(f"copying {diagfile} to {dest}") diag_copy = { 'copy': [[diagfile, dest]] From 5b8e7b175c308ec435eedccd45b03eddfb0a17b8 Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Thu, 12 Dec 2024 19:59:48 +0000 Subject: [PATCH 13/39] updated scripts to that run successfully --- jobs/JGLOBAL_ANALYSIS_STATS | 1 + parm/config/gfs/config.anlstat | 2 +- parm/gdas/anlstat_jedi_config.yaml.j2 | 14 ++++- scripts/exglobal_analysis_stats.py | 16 +++-- ush/python/pygfs/task/stat_analysis.py | 81 +++++++++++++------------- 5 files changed, 68 insertions(+), 46 deletions(-) diff --git a/jobs/JGLOBAL_ANALYSIS_STATS b/jobs/JGLOBAL_ANALYSIS_STATS index 537fc03122..cd7ce2e752 100755 --- a/jobs/JGLOBAL_ANALYSIS_STATS +++ b/jobs/JGLOBAL_ANALYSIS_STATS @@ -15,6 +15,7 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "anlstat" -c "base anlstat" # Generate COM variables from templates YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_OBS COM_CHEM_ANALYSIS +YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_SNOW_ANALYSIS COM_CONF ############################################################### # Run relevant script diff --git a/parm/config/gfs/config.anlstat b/parm/config/gfs/config.anlstat index 3a0d99b2d5..e7831bd565 100644 --- a/parm/config/gfs/config.anlstat +++ b/parm/config/gfs/config.anlstat @@ -12,6 +12,6 @@ export JEDI_CONFIG_YAML="${PARMgfs}/gdas/anlstat_jedi_config.yaml.j2" export JCB_BASE_YAML="${PARMgfs}/gdas/stat/aero/jcb-base.yaml.j2" export JCB_ALGO_YAML="${PARMgfs}/gdas/jcb-algorithms/anlstat.yaml.j2" export JEDIEXE=${HOMEgfs}/sorc/gdas.cd/build/bin/ioda-stats.x -export STAT_OUTDIR="/scratch1/NCEPDEV/da/Kevin.Dougherty/tmp_outdir/" +export STAT_OUTDIR="${COMOUT}/${net}/${run.yyyymmdd}/${hh}/products/stats/" echo "END: config.anlstat" \ No newline at end of file diff --git a/parm/gdas/anlstat_jedi_config.yaml.j2 b/parm/gdas/anlstat_jedi_config.yaml.j2 index a0150cb479..0c0fe70dda 100644 --- a/parm/gdas/anlstat_jedi_config.yaml.j2 +++ b/parm/gdas/anlstat_jedi_config.yaml.j2 @@ -1,8 +1,20 @@ -anlstat: +aero: rundir: '{{ DATA }}' exe_src: '{{ JEDIEXE }}' mpi_cmd: '{{ APRUN_ANLSTAT }}' + stat_file_path: '{{ COM_CHEM_ANALYSIS }}' + stat_file_name: 'aerostat' # jedi_args: None jcb_base_yaml: '{{ PARMgfs }}/gdas/stat/aero/jcb-base.yaml.j2' jcb_algo_yaml: '{{ JCB_ALGO_YAML }}' + jcb_algo: 'anlstat' +snow: + rundir: '{{ DATA }}' + exe_src: '{{ JEDIEXE }}' + mpi_cmd: '{{ APRUN_ANLSTAT }}' + stat_file_path: '{{ COM_SNOW_ANALYSIS }}' + stat_file_name: 'snowstat.tgz' + # jedi_args: None + jcb_base_yaml: '{{ PARMgfs }}/gdas/stat/snow/jcb-base.yaml.j2' + jcb_algo_yaml: '{{ JCB_ALGO_YAML }}' jcb_algo: 'anlstat' \ No newline at end of file diff --git a/scripts/exglobal_analysis_stats.py b/scripts/exglobal_analysis_stats.py index 805d0c855d..533b7d47dc 100755 --- a/scripts/exglobal_analysis_stats.py +++ b/scripts/exglobal_analysis_stats.py @@ -2,8 +2,8 @@ # exglobal_stat_analysis.py # This script creates an StatAnalysis class -# and runs the initialize method -# which create and stage the runtime directory +# and runs the initialize, execute, and finalize +# methods which create and stage the runtime directory # and create the YAML configuration # for a global stat analysis import os @@ -23,7 +23,15 @@ # Instantiate the atm analysis task StatAnl = StatAnalysis(config) + # Create list based on DA components + StatAnl.task_config['STAT_OBS'] = [] + if StatAnl.task_config.DO_AERO: + StatAnl.task_config['STAT_OBS'].append('aero') + if StatAnl.task_config.DO_JEDISNOWDA: + StatAnl.task_config['STAT_OBS'].append('snow') + # Initialize JEDI variational analysis StatAnl.initialize() - StatAnl.execute('anlstat') - StatAnl.finalize() + for ob in StatAnl.task_config.STAT_OBS: + StatAnl.execute(ob) + StatAnl.finalize(ob) diff --git a/ush/python/pygfs/task/stat_analysis.py b/ush/python/pygfs/task/stat_analysis.py index f26e437cf3..fb9e7af12b 100644 --- a/ush/python/pygfs/task/stat_analysis.py +++ b/ush/python/pygfs/task/stat_analysis.py @@ -11,9 +11,9 @@ from wxflow import (AttrDict, FileHandler, - add_to_datetime, to_fv3time, to_timedelta, to_YMDH, + add_to_datetime, to_timedelta, Task, - parse_j2yaml, save_as_yaml, + parse_j2yaml, logit) from pygfs.jedi import Jedi @@ -64,10 +64,6 @@ def __init__(self, config: Dict[str, Any]): # Extend task_config with local_dict self.task_config = AttrDict(**self.task_config, **local_dict) - # Create dictionary of Jedi objects - expected_keys = ['anlstat'] - self.jedi_dict = Jedi.get_jedi_dict(self.task_config.JEDI_CONFIG_YAML, self.task_config, expected_keys) - @logit(logger) def initialize(self) -> None: """ @@ -83,41 +79,49 @@ def initialize(self) -> None: ---------- None """ + # Create dictionary of Jedi objects + expected_keys = self.task_config.STAT_OBS + self.jedi_dict = Jedi.get_jedi_dict(self.task_config.JEDI_CONFIG_YAML, self.task_config, expected_keys) + logger.info(f"Copying files to {self.task_config.DATA}/stats") - # Copy stat files to DATA path - aerostat = os.path.join(self.task_config.COM_CHEM_ANALYSIS, f"{self.task_config['APREFIX']}aerostat") - dest = os.path.join(self.task_config.DATA, "aerostats") - statlist = [[aerostat, dest]] - FileHandler({'copy': statlist}).sync() + for OB in self.task_config.STAT_OBS: + # Parse JEDI analysis stat jinja file + obs_dict = parse_j2yaml(self.task_config.JEDI_CONFIG_YAML, self.task_config) - # Open tar file - logger.info(f"Open tarred stat file in {dest}") - with tarfile.open(dest, "r") as tar: - # Extract all files to the current directory - tar.extractall() + # Copy stat files to DATA path + instat_files = os.path.join(obs_dict[OB]['stat_file_path'], f"{self.task_config['APREFIX']}{obs_dict[OB]['stat_file_name']}") + dest = os.path.join(self.task_config.DATA, obs_dict[OB]['stat_file_name']) + statlist = [[instat_files, dest]] + FileHandler({'copy': statlist}).sync() - # Gunzip .nc files - logger.info("Gunzip files from tar file") - gz_files = glob.glob(os.path.join(self.task_config.DATA, "*gz")) + # Open tar file + logger.info(f"Open tarred stat file in {dest}") + with tarfile.open(dest, "r") as tar: + # Extract all files to the current directory + tar.extractall() - for diagfile in gz_files: - with gzip.open(diagfile, 'rb') as f_in: - with open(diagfile[:-3], 'wb') as f_out: - f_out.write(f_in.read()) + # Gunzip .nc files + logger.info("Gunzip files from tar file") + gz_files = glob.glob(os.path.join(self.task_config.DATA, "*gz")) - # Get list of .nc4 files - obs_space_paths = glob.glob(os.path.join(self.task_config.DATA, "*.nc4")) + for diagfile in gz_files: + with gzip.open(diagfile, 'rb') as f_in: + with open(diagfile[:-3], 'wb') as f_out: + f_out.write(f_in.read()) - self.task_config.OBSPACES_LIST = ['_'.join(os.path.basename(path).split('_')[1:3]) for path in obs_space_paths] + # Get list of .nc4 files + obs_space_paths = glob.glob(os.path.join(self.task_config.DATA, "*.nc4")) - # initialize JEDI application - logger.info(f"Initializing JEDI variational DA application") - self.jedi_dict['anlstat'].initialize(self.task_config) + self.task_config.OBSPACES_LIST = ['_'.join(os.path.basename(path).split('_')[1:3]) for path in obs_space_paths] + + # initialize JEDI application + logger.info(f"Initializing JEDI variational DA application") + self.jedi_dict[OB].initialize(self.task_config) @logit(logger) def execute(self, jedi_dict_key: str) -> None: - """Execute JEDI application of atm analysis + """Execute JEDI application of stat analysis Parameters ---------- @@ -132,18 +136,17 @@ def execute(self, jedi_dict_key: str) -> None: self.jedi_dict[jedi_dict_key].execute() @logit(logger) - def finalize(self) -> None: - """Finalize a global atm analysis + def finalize(self, jedi_dict_key: str) -> None: + """Finalize a statistic analysis - This method will finalize a global atm analysis using JEDI. + This method will finalize a statistic analysis using JEDI. This includes: - - tar output diag files and place in ROTDIR - - copy the generated YAML file from initialize to the ROTDIR - - copy the updated bias correction files to ROTDIR + - copying stat files to specified outdir Parameters ---------- - None + jedi_dict_key + key specifying particular Jedi object in self.jedi_dict Returns ---------- @@ -153,11 +156,9 @@ def finalize(self) -> None: # get list of output diag files diags = glob.glob(os.path.join(self.task_config.DATA, '*output_aod.nc')) - logger.debug(f"diag files: {diags}") - for diagfile in diags: outfile = os.path.basename(diagfile) - dest = os.path.join(self.task_config.STAT_OUTDIR, f'{outfile}') + dest = os.path.join(f'{self.task_config.STAT_OUTDIR}/{jedi_dict_key}/', f'{outfile}') logger.debug(f"copying {diagfile} to {dest}") diag_copy = { 'copy': [[diagfile, dest]] From 6bcd080bb92a1804c47dee74f6742d7c996fc137 Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Mon, 16 Dec 2024 17:06:20 +0000 Subject: [PATCH 14/39] reverting commits that were causing issues --- parm/config/gfs/config.base | 3 +++ workflow/applications/applications.py | 27 --------------------------- workflow/applications/gfs_cycled.py | 9 --------- workflow/rocoto/gfs_tasks.py | 20 -------------------- 4 files changed, 3 insertions(+), 56 deletions(-) diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index 795dc0e4eb..06e5aad07e 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -75,6 +75,9 @@ export DO_NPOESS="@DO_NPOESS@" # NPOESS products export DO_TRACKER="@DO_TRACKER@" # Hurricane track verification export DO_GENESIS="@DO_GENESIS@" # Cyclone genesis verification export DO_GENESIS_FSU="@DO_GENESIS_FSU@" # Cyclone genesis verification (FSU) +export DO_VERFOZN="YES" # Ozone data assimilation monitoring +export DO_VERFRAD="YES" # Radiance data assimilation monitoring +export DO_VMINMON="YES" # GSI minimization monitoring export DO_MOS="NO" # GFS Model Output Statistics - Only supported on WCOSS2 # NO for retrospective parallel; YES for real-time parallel diff --git a/workflow/applications/applications.py b/workflow/applications/applications.py index dbc1c82673..a5a4c2094d 100644 --- a/workflow/applications/applications.py +++ b/workflow/applications/applications.py @@ -40,7 +40,6 @@ def __init__(self, conf: Configuration) -> None: f'Valid application modes are:\n' f'{", ".join(self.VALID_MODES)}\n') -<<<<<<< HEAD self.net = base['NET'] self.model_app = base.get('APP', 'ATM') self.do_atm = base.get('DO_ATM', True) @@ -65,32 +64,6 @@ def __init__(self, conf: Configuration) -> None: self.do_goes = base.get('DO_GOES', False) self.do_mos = base.get('DO_MOS', False) self.do_extractvars = base.get('DO_EXTRACTVARS', False) -======= - self.net = _base['NET'] - self.model_app = _base.get('APP', 'ATM') - self.do_atm = _base.get('DO_ATM', True) - self.do_wave = _base.get('DO_WAVE', False) - self.do_wave_bnd = _base.get('DOBNDPNT_WAVE', False) - self.do_ocean = _base.get('DO_OCN', False) - self.do_ice = _base.get('DO_ICE', False) - self.do_aero = _base.get('DO_AERO', False) - self.do_prep_obs_aero = _base.get('DO_PREP_OBS_AERO', False) - self.do_bufrsnd = _base.get('DO_BUFRSND', False) - self.do_gempak = _base.get('DO_GEMPAK', False) - self.do_awips = _base.get('DO_AWIPS', False) - self.do_verfozn = _base.get('DO_VERFOZN', True) - self.do_verfrad = _base.get('DO_VERFRAD', True) - self.do_vminmon = _base.get('DO_VMINMON', True) - self.do_anlstat = _base.get('DO_ANLSTAT', False) - self.do_tracker = _base.get('DO_TRACKER', True) - self.do_genesis = _base.get('DO_GENESIS', True) - self.do_genesis_fsu = _base.get('DO_GENESIS_FSU', False) - self.do_metp = _base.get('DO_METP', False) - self.do_upp = not _base.get('WRITE_DOPOST', True) - self.do_goes = _base.get('DO_GOES', False) - self.do_mos = _base.get('DO_MOS', False) - self.do_extractvars = _base.get('DO_EXTRACTVARS', False) ->>>>>>> fbe784df (Stubs for anlstat job) self.do_hpssarch = base.get('HPSSARCH', False) diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index 3d2dba2ea9..983acb1493 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -274,18 +274,9 @@ def get_task_names(self): if self.do_vminmon: gfs_tasks += ['vminmon'] - - if self.do_anlstat: - gfs_tasks += ['anlstat'] -<<<<<<< HEAD -<<<<<<< HEAD if self.do_anlstat: gfs_tasks += ['anlstat'] -======= ->>>>>>> fbe784df (Stubs for anlstat job) -======= ->>>>>>> fbe784df (Stubs for anlstat job) if self.do_tracker: gfs_tasks += ['tracker'] diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 217d7c9e05..12b0a78a54 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -1766,14 +1766,12 @@ def anlstat(self): deps.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_aero: dep_dict = {'type': 'task', 'name': f'{self.run}_aeroanlfinal'} - deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('anlstat') task_name = f'{self.run}_anlstat' - task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, @@ -2299,15 +2297,6 @@ def arch(self): if self.app_config.do_anlstat: dep_dict = {'type': 'task', 'name': f'{self.run}_anlstat'} deps.append(rocoto.add_dependency(dep_dict)) - if self.app_config.do_anlstat: - dep_dict = {'type': 'task', 'name': f'{self.run}anlstat'} - deps.append(rocoto.add_dependency(dep_dict)) - if self.app_config.do_anlstat: - dep_dict = {'type': 'task', 'name': f'{self.run}anlstat'} - deps.append(rocoto.add_dependency(dep_dict)) - if self.app_config.do_anlstat: - dep_dict = {'type': 'task', 'name': f'{self.run}anlstat'} - deps.append(rocoto.add_dependency(dep_dict)) elif self.run in ['gdas']: dep_dict = {'type': 'task', 'name': f'{self.run}_atmanlprod'} deps.append(rocoto.add_dependency(dep_dict)) @@ -2326,15 +2315,6 @@ def arch(self): if self.app_config.do_anlstat: dep_dict = {'type': 'task', 'name': f'{self.run}_anlstat'} deps.append(rocoto.add_dependency(dep_dict)) - if self.app_config.do_anlstat: - dep_dict = {'type': 'task', 'name': f'{self.run}anlstat'} - deps.append(rocoto.add_dependency(dep_dict)) - if self.app_config.do_anlstat: - dep_dict = {'type': 'task', 'name': f'{self.run}anlstat'} - deps.append(rocoto.add_dependency(dep_dict)) - if self.app_config.do_anlstat: - dep_dict = {'type': 'task', 'name': f'{self.run}anlstat'} - deps.append(rocoto.add_dependency(dep_dict)) if self.run in ['gfs'] and self.app_config.do_tracker: dep_dict = {'type': 'task', 'name': f'{self.run}_tracker'} deps.append(rocoto.add_dependency(dep_dict)) From 76e60b1312a70083d7ec9c9e901686b4f358c968 Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Mon, 16 Dec 2024 17:42:20 +0000 Subject: [PATCH 15/39] fix descrepancies with workflow/applications files --- workflow/applications/applications.py | 58 +-------- workflow/applications/gfs_cycled.py | 176 +------------------------- 2 files changed, 8 insertions(+), 226 deletions(-) diff --git a/workflow/applications/applications.py b/workflow/applications/applications.py index a5a4c2094d..4e09224488 100644 --- a/workflow/applications/applications.py +++ b/workflow/applications/applications.py @@ -41,60 +41,7 @@ def __init__(self, conf: Configuration) -> None: f'{", ".join(self.VALID_MODES)}\n') self.net = base['NET'] - self.model_app = base.get('APP', 'ATM') - self.do_atm = base.get('DO_ATM', True) - self.do_wave = base.get('DO_WAVE', False) - self.do_wave_bnd = base.get('DOBNDPNT_WAVE', False) - self.do_ocean = base.get('DO_OCN', False) - self.do_ice = base.get('DO_ICE', False) - self.do_aero = base.get('DO_AERO', False) - self.do_prep_obs_aero = base.get('DO_PREP_OBS_AERO', False) - self.do_bufrsnd = base.get('DO_BUFRSND', False) - self.do_gempak = base.get('DO_GEMPAK', False) - self.do_awips = base.get('DO_AWIPS', False) - self.do_verfozn = base.get('DO_VERFOZN', True) - self.do_verfrad = base.get('DO_VERFRAD', True) - self.do_vminmon = base.get('DO_VMINMON', True) - self.do_anlstat = base.get('DO_ANLSTAT', True) - self.do_tracker = base.get('DO_TRACKER', True) - self.do_genesis = base.get('DO_GENESIS', True) - self.do_genesis_fsu = base.get('DO_GENESIS_FSU', False) - self.do_metp = base.get('DO_METP', False) - self.do_upp = not base.get('WRITE_DOPOST', True) - self.do_goes = base.get('DO_GOES', False) - self.do_mos = base.get('DO_MOS', False) - self.do_extractvars = base.get('DO_EXTRACTVARS', False) - - self.do_hpssarch = base.get('HPSSARCH', False) - - self.nens = base.get('NMEM_ENS', 0) - self.fcst_segments = base.get('FCST_SEGMENTS', None) - self.interval_gfs = to_timedelta(f"{base.get('INTERVAL_GFS')}H") - - if not AppConfig.is_monotonic(self.fcst_segments): - raise ValueError(f'Forecast segments do not increase monotonically: {",".join(self.fcst_segments)}') - - self.wave_runs = None - if self.do_wave: - wave_run = base.get('WAVE_RUN', 'BOTH').lower() - if wave_run in ['both']: - self.wave_runs = ['gfs', 'gdas'] - elif wave_run in ['gfs', 'gdas']: - self.wave_runs = [wave_run] - - self.aero_anl_runs = None - self.aero_fcst_runs = None - if self.do_aero: - aero_anl_run = base.get('AERO_ANL_RUN', 'BOTH').lower() - if aero_anl_run in ['both']: - self.aero_anl_runs = ['gfs', 'gdas'] - elif aero_anl_run in ['gfs', 'gdas']: - self.aero_anl_runs = [aero_anl_run] - aero_fcst_run = base.get('AERO_FCST_RUN', None).lower() - if aero_fcst_run in ['both']: - self.aero_fcst_runs = ['gfs', 'gdas'] - elif aero_fcst_run in ['gfs', 'gdas']: - self.aero_fcst_runs = [aero_fcst_run] + print(f"Generating the XML for a {self.mode}_{self.net} case") def _init_finalize(self, conf: Configuration): ''' @@ -135,6 +82,7 @@ def _get_run_options(self, conf: Configuration) -> Dict[str, Any]: run_options[run]['do_verfozn'] = run_base.get('DO_VERFOZN', True) run_options[run]['do_verfrad'] = run_base.get('DO_VERFRAD', True) run_options[run]['do_vminmon'] = run_base.get('DO_VMINMON', True) + run_options[run]['do_anlstat'] = run_base.get('DO_ANLSTAT', True) run_options[run]['do_tracker'] = run_base.get('DO_TRACKER', True) run_options[run]['do_genesis'] = run_base.get('DO_GENESIS', True) run_options[run]['do_genesis_fsu'] = run_base.get('DO_GENESIS_FSU', False) @@ -258,4 +206,4 @@ def is_monotonic(test_list: List, check_decreasing: bool = False) -> bool: if check_decreasing: return all(x > y for x, y in zip(test_list, test_list[1:])) else: - return all(x < y for x, y in zip(test_list, test_list[1:])) + return all(x < y for x, y in zip(test_list, test_list[1:])) \ No newline at end of file diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index 983acb1493..f162c1436d 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -98,10 +98,10 @@ def _get_app_configs(self, run): if options['do_vminmon']: configs += ['vminmon'] - if self.do_anlstat: + if options['do_anlstat']: configs += ['anlstat'] - if self.do_tracker: + if options['do_tracker']: configs += ['tracker'] if options['do_genesis']: @@ -163,174 +163,8 @@ def get_task_names(self): This is the place where that order is set. """ - gdas_gfs_common_tasks_before_fcst = ['prep'] - gdas_gfs_common_cleanup_tasks = ['arch', 'cleanup'] - - if self.do_jediatmvar: - gdas_gfs_common_tasks_before_fcst += ['prepatmiodaobs', 'atmanlinit', 'atmanlvar', 'atmanlfv3inc', 'atmanlfinal'] - else: - gdas_gfs_common_tasks_before_fcst += ['anal'] - - if self.do_jediocnvar: - gdas_gfs_common_tasks_before_fcst += ['prepoceanobs', 'marineanlinit', 'marinebmat', 'marineanlvar'] - if self.do_hybvar: - gdas_gfs_common_tasks_before_fcst += ['marineanlletkf', 'ocnanalecen'] - gdas_gfs_common_tasks_before_fcst += ['marineanlchkpt', 'marineanlfinal'] - if self.do_vrfy_oceanda: - gdas_gfs_common_tasks_before_fcst += ['ocnanalvrfy'] - - gdas_gfs_common_tasks_before_fcst += ['sfcanl', 'analcalc'] - - if self.do_jedisnowda: - gdas_gfs_common_tasks_before_fcst += ['snowanl'] - - wave_prep_tasks = ['waveinit', 'waveprep'] - wave_bndpnt_tasks = ['wavepostbndpnt', 'wavepostbndpntbll'] - wave_post_tasks = ['wavepostsbs', 'wavepostpnt'] - - hybrid_tasks = [] - hybrid_after_eupd_tasks = [] - if self.do_hybvar: - if self.do_jediatmens: - hybrid_tasks += ['atmensanlinit', 'atmensanlfv3inc', 'atmensanlfinal', 'echgres'] - hybrid_tasks += ['atmensanlobs', 'atmensanlsol'] if self.lobsdiag_forenkf else ['atmensanlletkf'] - else: - hybrid_tasks += ['eobs', 'eupd', 'echgres'] - hybrid_tasks += ['ediag'] if self.lobsdiag_forenkf else ['eomg'] - if self.do_jedisnowda: - hybrid_tasks += ['esnowrecen'] - hybrid_after_eupd_tasks += ['stage_ic', 'ecen', 'esfc', 'efcs', 'epos', 'earc', 'cleanup'] - - # Collect all "gdas" cycle tasks - gdas_tasks = gdas_gfs_common_tasks_before_fcst.copy() - - if not self.do_jediatmvar: - gdas_tasks += ['analdiag'] - - if self.do_wave and 'gdas' in self.wave_runs: - gdas_tasks += wave_prep_tasks - - if self.do_aero and 'gdas' in self.aero_anl_runs: - gdas_tasks += ['aeroanlgenb', 'aeroanlinit', 'aeroanlvar', 'aeroanlfinal'] - if self.do_prep_obs_aero: - gdas_tasks += ['prepobsaero'] - - gdas_tasks += ['stage_ic', 'atmanlupp', 'atmanlprod', 'fcst'] - - if self.do_upp: - gdas_tasks += ['atmupp'] - gdas_tasks += ['atmos_prod'] - - if self.do_wave and 'gdas' in self.wave_runs: - if self.do_wave_bnd: - gdas_tasks += wave_bndpnt_tasks - gdas_tasks += wave_post_tasks - - if self.do_fit2obs: - gdas_tasks += ['fit2obs'] - - if self.do_verfozn: - gdas_tasks += ['verfozn'] - - if self.do_verfrad: - gdas_tasks += ['verfrad'] - - if self.do_vminmon: - gdas_tasks += ['vminmon'] - - if self.do_anlstat: - gdas_tasks += ['anlstat'] - - if self.do_gempak: - gdas_tasks += ['gempak', 'gempakmetancdc'] - - gdas_tasks += gdas_gfs_common_cleanup_tasks - - # Collect "gfs" cycle tasks - gfs_tasks = gdas_gfs_common_tasks_before_fcst.copy() - - if self.do_wave and 'gfs' in self.wave_runs: - gfs_tasks += wave_prep_tasks - - if self.do_aero and 'gfs' in self.aero_anl_runs: - gfs_tasks += ['aeroanlinit', 'aeroanlvar', 'aeroanlfinal'] - if self.do_prep_obs_aero: - gfs_tasks += ['prepobsaero'] - - gfs_tasks += ['atmanlupp', 'atmanlprod', 'fcst'] - - if self.do_ocean: - gfs_tasks += ['ocean_prod'] - - if self.do_ice: - gfs_tasks += ['ice_prod'] - - if self.do_upp: - gfs_tasks += ['atmupp'] - gfs_tasks += ['atmos_prod'] - - if self.do_goes: - gfs_tasks += ['goesupp'] - - if self.do_vminmon: - gfs_tasks += ['vminmon'] - - if self.do_anlstat: - gfs_tasks += ['anlstat'] - - if self.do_tracker: - gfs_tasks += ['tracker'] - - if self.do_genesis: - gfs_tasks += ['genesis'] - - if self.do_genesis_fsu: - gfs_tasks += ['genesis_fsu'] - - if self.do_metp: - gfs_tasks += ['metp'] - - if self.do_wave and 'gfs' in self.wave_runs: - if self.do_wave_bnd: - gfs_tasks += wave_bndpnt_tasks - gfs_tasks += wave_post_tasks - if self.do_gempak: - gfs_tasks += ['wavegempak'] - if self.do_awips: - gfs_tasks += ['waveawipsbulls', 'waveawipsgridded'] - - if self.do_bufrsnd: - gfs_tasks += ['postsnd'] - - if self.do_gempak: - gfs_tasks += ['gempak'] - gfs_tasks += ['gempakmeta'] - gfs_tasks += ['gempakncdcupapgif'] - if self.do_goes: - gfs_tasks += ['npoess_pgrb2_0p5deg'] - gfs_tasks += ['gempakpgrb2spec'] - - if self.do_awips: - gfs_tasks += ['awips_20km_1p0deg', 'fbwind'] - - if self.do_mos: - gfs_tasks += ['mos_stn_prep', 'mos_grd_prep', 'mos_ext_stn_prep', 'mos_ext_grd_prep', - 'mos_stn_fcst', 'mos_grd_fcst', 'mos_ext_stn_fcst', 'mos_ext_grd_fcst', - 'mos_stn_prdgen', 'mos_grd_prdgen', 'mos_ext_stn_prdgen', 'mos_ext_grd_prdgen', - 'mos_wx_prdgen', 'mos_wx_ext_prdgen'] - - gfs_tasks += gdas_gfs_common_cleanup_tasks - - tasks = dict() - tasks['gdas'] = gdas_tasks - - if self.do_hybvar and 'gdas' in self.eupd_runs: - enkfgdas_tasks = hybrid_tasks + hybrid_after_eupd_tasks - tasks['enkfgdas'] = enkfgdas_tasks - - # Add RUN=gfs tasks if running early cycle - if self.interval_gfs > to_timedelta("0H"): - tasks['gfs'] = gfs_tasks + # Start with a dictionary of empty task lists for each valid run + task_names = {run: [] for run in self.runs} for run in self.runs: options = self.run_options[run] @@ -489,4 +323,4 @@ def get_task_names(self): task_names[run] += ['stage_ic', 'ecen', 'esfc', 'efcs', 'epos', 'earc', 'cleanup'] - return task_names + return task_names \ No newline at end of file From 332cedeaf41c7a7da5a368a6fc59fe699c865b34 Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Mon, 16 Dec 2024 17:43:45 +0000 Subject: [PATCH 16/39] pycodestyle --- workflow/applications/applications.py | 3 ++- workflow/applications/gfs_cycled.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/workflow/applications/applications.py b/workflow/applications/applications.py index 4e09224488..d887bd933a 100644 --- a/workflow/applications/applications.py +++ b/workflow/applications/applications.py @@ -206,4 +206,5 @@ def is_monotonic(test_list: List, check_decreasing: bool = False) -> bool: if check_decreasing: return all(x > y for x, y in zip(test_list, test_list[1:])) else: - return all(x < y for x, y in zip(test_list, test_list[1:])) \ No newline at end of file + return all(x < y for x, y in zip(test_list, test_list[1:])) + \ No newline at end of file diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index f162c1436d..2868874052 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -323,4 +323,4 @@ def get_task_names(self): task_names[run] += ['stage_ic', 'ecen', 'esfc', 'efcs', 'epos', 'earc', 'cleanup'] - return task_names \ No newline at end of file + return task_names From 74c78883a5a3042b4ef9606f6fca35591ce56d97 Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Mon, 16 Dec 2024 17:45:26 +0000 Subject: [PATCH 17/39] pynorms --- workflow/applications/applications.py | 1 - 1 file changed, 1 deletion(-) diff --git a/workflow/applications/applications.py b/workflow/applications/applications.py index d887bd933a..f100264583 100644 --- a/workflow/applications/applications.py +++ b/workflow/applications/applications.py @@ -207,4 +207,3 @@ def is_monotonic(test_list: List, check_decreasing: bool = False) -> bool: return all(x > y for x, y in zip(test_list, test_list[1:])) else: return all(x < y for x, y in zip(test_list, test_list[1:])) - \ No newline at end of file From 6687c41c08ddfd24c1316bf0b0ef668ffa2a1c78 Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Tue, 17 Dec 2024 16:15:27 +0000 Subject: [PATCH 18/39] add snow jcb-base yaml --- parm/gdas/stat/snow/jcb-base.yaml.j2 | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 parm/gdas/stat/snow/jcb-base.yaml.j2 diff --git a/parm/gdas/stat/snow/jcb-base.yaml.j2 b/parm/gdas/stat/snow/jcb-base.yaml.j2 new file mode 100644 index 0000000000..461010d738 --- /dev/null +++ b/parm/gdas/stat/snow/jcb-base.yaml.j2 @@ -0,0 +1,22 @@ +# Search path for model and obs for JCB +# ------------------------------------- +algorithm_path: "{{PARMgfs}}/gdas/jcb-algorithms" +app_path_algorithm: "{{PARMgfs}}/gdas/jcb-gdas/algorithm/obstats/snow" + +# Assimilation window +# ------------------- +window_begin: "{{ STAT_WINDOW_BEGIN | to_isotime }}" +window_YMDH: "{{ STAT_WINDOW_BEGIN | to_YMDH }}" +window_length: "{{ STAT_WINDOW_LENGTH }}" +bound_to_include: begin + +stat_current_cycle_iso: "{{ current_cycle | to_isotime }}" +stat_current_cycle_YMDH: "{{ current_cycle | to_YMDH }}" + +# Inputted list of ob spaces +# -------------------------- +obspaces: {{ OBSPACES_LIST }} + +# Obspace variable things +# ----------------------- +snow_obsdatain_path: "{{ DATA }}" \ No newline at end of file From cb4a7035c2abaa3e79761aeab3b78de97ad496a1 Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Wed, 18 Dec 2024 14:15:54 +0000 Subject: [PATCH 19/39] update aero jcb-base yaml to no longer include variables --- parm/gdas/stat/aero/jcb-base.yaml.j2 | 8 -------- 1 file changed, 8 deletions(-) diff --git a/parm/gdas/stat/aero/jcb-base.yaml.j2 b/parm/gdas/stat/aero/jcb-base.yaml.j2 index 84e48756cd..ddc5bcdbc7 100644 --- a/parm/gdas/stat/aero/jcb-base.yaml.j2 +++ b/parm/gdas/stat/aero/jcb-base.yaml.j2 @@ -20,11 +20,3 @@ obspaces: {{ OBSPACES_LIST }} # Obspace variable things # ----------------------- aero_obsdatain_path: "{{ DATA }}" -aero_obsdatatin_simulated_variables: ['aerosolOpticalDepth'] -aero_obsdatain_observed_variables: ['aerosolOpticalDepth'] - -# Variables -# --------- -aero_variables: ['aerosolOpticalDepth'] -aero_file_groups: ['ombg', 'oman'] -aero_file_qc_groups: ['EffectiveQC0', 'EffectiveQC1'] \ No newline at end of file From d0a7a3d13bc1aa2dc2483a248b573b9b9bf7b67f Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Fri, 20 Dec 2024 16:26:48 +0000 Subject: [PATCH 20/39] small bug fixes --- workflow/rocoto/gfs_tasks.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 12b0a78a54..6217e866b0 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -2294,7 +2294,7 @@ def arch(self): if self.options['do_vminmon']: dep_dict = {'type': 'task', 'name': f'{self.run}_vminmon'} deps.append(rocoto.add_dependency(dep_dict)) - if self.app_config.do_anlstat: + if self.options['do_anlstat']: dep_dict = {'type': 'task', 'name': f'{self.run}_anlstat'} deps.append(rocoto.add_dependency(dep_dict)) elif self.run in ['gdas']: @@ -2312,10 +2312,10 @@ def arch(self): if self.options['do_vminmon']: dep_dict = {'type': 'task', 'name': f'{self.run}_vminmon'} deps.append(rocoto.add_dependency(dep_dict)) - if self.app_config.do_anlstat: + if self.options['do_anlstat']: dep_dict = {'type': 'task', 'name': f'{self.run}_anlstat'} deps.append(rocoto.add_dependency(dep_dict)) - if self.run in ['gfs'] and self.app_config.do_tracker: + if self.run in ['gfs'] and self.options['do_tracker']: dep_dict = {'type': 'task', 'name': f'{self.run}_tracker'} deps.append(rocoto.add_dependency(dep_dict)) if self.run in ['gfs'] and self.options['do_genesis']: From 0c1ba04dfdd9a6255d6c255607bff8d3d15d1267 Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Wed, 8 Jan 2025 19:57:38 +0000 Subject: [PATCH 21/39] update gfs_cycled.py --- workflow/applications/gfs_cycled.py | 473 ++++++++++++++-------------- 1 file changed, 230 insertions(+), 243 deletions(-) diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index b6bfd429e3..1325aff664 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -1,6 +1,7 @@ -from applications.applications import AppConfig from typing import Dict, Any -from wxflow import Configuration +from applications.applications import AppConfig +from wxflow import Configuration, to_timedelta +from datetime import timedelta class GFSCycledAppConfig(AppConfig): @@ -10,140 +11,116 @@ class GFSCycledAppConfig(AppConfig): def __init__(self, conf: Configuration): super().__init__(conf) - # Re-read config.base without RUN specified to get the basic settings for - # cycled cases to be able to determine valid runs base = conf.parse_config('config.base') - - self.ens_runs = [] - - if base.get('DOHYBVAR', False): - ens_run = base.get('EUPD_CYC', 'gdas').lower() - if ens_run in ['both']: - self.ens_runs = ['gfs', 'gdas'] - elif ens_run in ['gfs', 'gdas']: - self.ens_runs = [ens_run] - - # Now construct self.runs the desired XML order (gdas, enkfgdas, gfs, enkfgfs) - self.runs = ["gdas"] # We always have a 'gdas' run - self.runs.append('enkfgdas') if 'gdas' in self.ens_runs else 0 - self.runs.append("gfs") if base['INTERVAL_GFS'] > 0 else 0 - self.runs.append('enkfgfs') if 'gfs' in self.ens_runs and "gfs" in self.runs else 0 - - def _get_run_options(self, conf: Configuration) -> Dict[str, Any]: - - run_options = super()._get_run_options(conf) - - for run in self.runs: - base = conf.parse_config('config.base', RUN=run) - - run_options[run]['do_hybvar'] = base.get('DOHYBVAR', False) - run_options[run]['do_hybvar_ocn'] = base.get('DOHYBVAR_OCN', False) - run_options[run]['nens'] = base.get('NMEM_ENS', 0) - if run_options[run]['do_hybvar']: - run_options[run]['lobsdiag_forenkf'] = base.get('lobsdiag_forenkf', False) - - run_options[run]['do_fit2obs'] = base.get('DO_FIT2OBS', True) - run_options[run]['do_jediatmvar'] = base.get('DO_JEDIATMVAR', False) - run_options[run]['do_jediatmens'] = base.get('DO_JEDIATMENS', False) - run_options[run]['do_jediocnvar'] = base.get('DO_JEDIOCNVAR', False) - run_options[run]['do_jedisnowda'] = base.get('DO_JEDISNOWDA', False) - run_options[run]['do_mergensst'] = base.get('DO_MERGENSST', False) - run_options[run]['do_vrfy_oceanda'] = base.get('DO_VRFY_OCEANDA', False) - - return run_options - - def _get_app_configs(self, run): + self.do_hybvar = base.get('DOHYBVAR', False) + self.do_fit2obs = base.get('DO_FIT2OBS', True) + self.do_jediatmvar = base.get('DO_JEDIATMVAR', False) + self.do_jediatmens = base.get('DO_JEDIATMENS', False) + self.do_jediocnvar = base.get('DO_JEDIOCNVAR', False) + self.do_jedisnowda = base.get('DO_JEDISNOWDA', False) + self.do_mergensst = base.get('DO_MERGENSST', False) + self.do_vrfy_oceanda = base.get('DO_VRFY_OCEANDA', False) + + self.lobsdiag_forenkf = False + self.eupd_runs = None + if self.do_hybvar: + self.lobsdiag_forenkf = base.get('lobsdiag_forenkf', False) + eupd_run = base.get('EUPD_CYC', 'gdas').lower() + if eupd_run in ['both']: + self.eupd_runs = ['gfs', 'gdas'] + elif eupd_run in ['gfs', 'gdas']: + self.eupd_runs = [eupd_run] + + def _get_app_configs(self): """ - Returns the config files that are involved in the cycled app + Returns the config_files that are involved in the cycled app """ - options = self.run_options[run] configs = ['prep'] - if options['do_jediatmvar']: + if self.do_jediatmvar: configs += ['prepatmiodaobs', 'atmanlinit', 'atmanlvar', 'atmanlfv3inc', 'atmanlfinal'] else: configs += ['anal', 'analdiag'] - if options['do_jediocnvar']: + if self.do_jediocnvar: configs += ['prepoceanobs', 'marineanlinit', 'marinebmat', 'marineanlvar'] - if options['do_hybvar']: + if self.do_hybvar: configs += ['marineanlletkf', 'ocnanalecen'] configs += ['marineanlchkpt', 'marineanlfinal'] - if options['do_vrfy_oceanda']: + if self.do_vrfy_oceanda: configs += ['ocnanalvrfy'] - if options['do_ocean'] or options['do_ice']: + if self.do_ocean or self.do_ice: configs += ['oceanice_products'] configs += ['stage_ic', 'sfcanl', 'analcalc', 'fcst', 'upp', 'atmos_products', 'arch', 'cleanup'] - if options['do_hybvar']: - if options['do_jediatmens']: - configs += ['atmensanlinit', 'atmensanlobs', 'atmensanlsol', - 'atmensanlletkf', 'atmensanlfv3inc', 'atmensanlfinal'] + if self.do_hybvar: + if self.do_jediatmens: + configs += ['atmensanlinit', 'atmensanlobs', 'atmensanlsol', 'atmensanlletkf', 'atmensanlfv3inc', 'atmensanlfinal'] else: configs += ['eobs', 'eomg', 'ediag', 'eupd'] configs += ['ecen', 'esfc', 'efcs', 'echgres', 'epos', 'earc'] - if options['do_fit2obs']: + if self.do_fit2obs: configs += ['fit2obs'] - if options['do_verfozn']: + if self.do_verfozn: configs += ['verfozn'] - if options['do_verfrad']: + if self.do_verfrad: configs += ['verfrad'] - if options['do_vminmon']: + if self.do_vminmon: configs += ['vminmon'] - - if options['do_anlstat']: + + if self.do_anlstat: configs += ['anlstat'] - if options['do_tracker']: + if self.do_tracker: configs += ['tracker'] - if options['do_genesis']: + if self.do_genesis: configs += ['genesis'] - if options['do_genesis_fsu']: + if self.do_genesis_fsu: configs += ['genesis_fsu'] - if options['do_metp']: + if self.do_metp: configs += ['metp'] - if options['do_gempak']: + if self.do_gempak: configs += ['gempak'] - if options['do_goes']: + if self.do_goes: configs += ['npoess'] - if options['do_bufrsnd']: + if self.do_bufrsnd: configs += ['postsnd'] - if options['do_awips']: + if self.do_awips: configs += ['awips'] - if options['do_wave']: + if self.do_wave: configs += ['waveinit', 'waveprep', 'wavepostsbs', 'wavepostpnt'] - if options['do_wave_bnd']: + if self.do_wave_bnd: configs += ['wavepostbndpnt', 'wavepostbndpntbll'] - if options['do_gempak']: + if self.do_gempak: configs += ['wavegempak'] - if options['do_awips']: + if self.do_awips: configs += ['waveawipsbulls', 'waveawipsgridded'] - if options['do_aero_anl']: + if self.do_aero: configs += ['aeroanlgenb', 'aeroanlinit', 'aeroanlvar', 'aeroanlfinal'] - if options['do_prep_obs_aero']: + if self.do_prep_obs_aero: configs += ['prepobsaero'] - if options['do_jedisnowda']: + if self.do_jedisnowda: configs += ['snowanl'] - if options['do_hybvar']: - configs += ['esnowanl'] + if self.do_hybvar: + configs += ['esnowrecen'] - if options['do_mos']: + if self.do_mos: configs += ['mos_stn_prep', 'mos_grd_prep', 'mos_ext_stn_prep', 'mos_ext_grd_prep', 'mos_stn_fcst', 'mos_grd_fcst', 'mos_ext_stn_fcst', 'mos_ext_grd_fcst', 'mos_stn_prdgen', 'mos_grd_prdgen', 'mos_ext_stn_prdgen', 'mos_ext_grd_prdgen', @@ -158,171 +135,181 @@ def _update_base(base_in): def get_task_names(self): """ - Get the task names for each valid run in this cycled configuration. - NOTE: The order of the task names matters in the XML. - This is the place where that order is set. + Get the task names for all the tasks in the cycled application. + Note that the order of the task names matters in the XML. + This is the place where that order is set. """ - # Start with a dictionary of empty task lists for each valid run - task_names = {run: [] for run in self.runs} + gdas_gfs_common_tasks_before_fcst = ['prep'] + gdas_gfs_common_cleanup_tasks = ['arch', 'cleanup'] + + if self.do_jediatmvar: + gdas_gfs_common_tasks_before_fcst += ['prepatmiodaobs', 'atmanlinit', 'atmanlvar', 'atmanlfv3inc', 'atmanlfinal'] + else: + gdas_gfs_common_tasks_before_fcst += ['anal'] + + if self.do_jediocnvar: + gdas_gfs_common_tasks_before_fcst += ['prepoceanobs', 'marineanlinit', 'marinebmat', 'marineanlvar'] + if self.do_hybvar: + gdas_gfs_common_tasks_before_fcst += ['marineanlletkf', 'ocnanalecen'] + gdas_gfs_common_tasks_before_fcst += ['marineanlchkpt', 'marineanlfinal'] + if self.do_vrfy_oceanda: + gdas_gfs_common_tasks_before_fcst += ['ocnanalvrfy'] + + gdas_gfs_common_tasks_before_fcst += ['sfcanl', 'analcalc'] + + if self.do_jedisnowda: + gdas_gfs_common_tasks_before_fcst += ['snowanl'] + + wave_prep_tasks = ['waveinit', 'waveprep'] + wave_bndpnt_tasks = ['wavepostbndpnt', 'wavepostbndpntbll'] + wave_post_tasks = ['wavepostsbs', 'wavepostpnt'] + + hybrid_tasks = [] + hybrid_after_eupd_tasks = [] + if self.do_hybvar: + if self.do_jediatmens: + hybrid_tasks += ['atmensanlinit', 'atmensanlfv3inc', 'atmensanlfinal', 'echgres'] + hybrid_tasks += ['atmensanlobs', 'atmensanlsol'] if self.lobsdiag_forenkf else ['atmensanlletkf'] + else: + hybrid_tasks += ['eobs', 'eupd', 'echgres'] + hybrid_tasks += ['ediag'] if self.lobsdiag_forenkf else ['eomg'] + if self.do_jedisnowda: + hybrid_tasks += ['esnowrecen'] + hybrid_after_eupd_tasks += ['stage_ic', 'ecen', 'esfc', 'efcs', 'epos', 'earc', 'cleanup'] + + # Collect all "gdas" cycle tasks + gdas_tasks = gdas_gfs_common_tasks_before_fcst.copy() + + if not self.do_jediatmvar: + gdas_tasks += ['analdiag'] + + if self.do_wave and 'gdas' in self.wave_runs: + gdas_tasks += wave_prep_tasks + + if self.do_aero and 'gdas' in self.aero_anl_runs: + gdas_tasks += ['aeroanlgenb', 'aeroanlinit', 'aeroanlvar', 'aeroanlfinal'] + if self.do_prep_obs_aero: + gdas_tasks += ['prepobsaero'] + + gdas_tasks += ['stage_ic', 'atmanlupp', 'atmanlprod', 'fcst'] + + if self.do_upp: + gdas_tasks += ['atmupp'] + gdas_tasks += ['atmos_prod'] + + if self.do_wave and 'gdas' in self.wave_runs: + if self.do_wave_bnd: + gdas_tasks += wave_bndpnt_tasks + gdas_tasks += wave_post_tasks + + if self.do_fit2obs: + gdas_tasks += ['fit2obs'] + + if self.do_verfozn: + gdas_tasks += ['verfozn'] + + if self.do_verfrad: + gdas_tasks += ['verfrad'] + + if self.do_vminmon: + gdas_tasks += ['vminmon'] + + if self.do_gempak: + gdas_tasks += ['gempak', 'gempakmetancdc'] + + gdas_tasks += gdas_gfs_common_cleanup_tasks + + # Collect "gfs" cycle tasks + gfs_tasks = gdas_gfs_common_tasks_before_fcst.copy() + + if self.do_wave and 'gfs' in self.wave_runs: + gfs_tasks += wave_prep_tasks + + if self.do_aero and 'gfs' in self.aero_anl_runs: + gfs_tasks += ['aeroanlinit', 'aeroanlvar', 'aeroanlfinal'] + if self.do_prep_obs_aero: + gfs_tasks += ['prepobsaero'] + + gfs_tasks += ['atmanlupp', 'atmanlprod', 'fcst'] + + if self.do_ocean: + gfs_tasks += ['ocean_prod'] + + if self.do_ice: + gfs_tasks += ['ice_prod'] + + if self.do_upp: + gfs_tasks += ['atmupp'] + gfs_tasks += ['atmos_prod'] + + if self.do_goes: + gfs_tasks += ['goesupp'] + + if self.do_vminmon: + gfs_tasks += ['vminmon'] + + if self.do_anlstat: + gfs_tasks += ['anlstat'] + + if self.do_tracker: + gfs_tasks += ['tracker'] + + if self.do_genesis: + gfs_tasks += ['genesis'] + + if self.do_genesis_fsu: + gfs_tasks += ['genesis_fsu'] + + if self.do_metp: + gfs_tasks += ['metp'] + + if self.do_wave and 'gfs' in self.wave_runs: + if self.do_wave_bnd: + gfs_tasks += wave_bndpnt_tasks + gfs_tasks += wave_post_tasks + if self.do_gempak: + gfs_tasks += ['wavegempak'] + if self.do_awips: + gfs_tasks += ['waveawipsbulls', 'waveawipsgridded'] + + if self.do_bufrsnd: + gfs_tasks += ['postsnd'] + + if self.do_gempak: + gfs_tasks += ['gempak'] + gfs_tasks += ['gempakmeta'] + gfs_tasks += ['gempakncdcupapgif'] + if self.do_goes: + gfs_tasks += ['npoess_pgrb2_0p5deg'] + gfs_tasks += ['gempakpgrb2spec'] + + if self.do_awips: + gfs_tasks += ['awips_20km_1p0deg', 'fbwind'] + + if self.do_mos: + gfs_tasks += ['mos_stn_prep', 'mos_grd_prep', 'mos_ext_stn_prep', 'mos_ext_grd_prep', + 'mos_stn_fcst', 'mos_grd_fcst', 'mos_ext_stn_fcst', 'mos_ext_grd_fcst', + 'mos_stn_prdgen', 'mos_grd_prdgen', 'mos_ext_stn_prdgen', 'mos_ext_grd_prdgen', + 'mos_wx_prdgen', 'mos_wx_ext_prdgen'] + + gfs_tasks += gdas_gfs_common_cleanup_tasks + + tasks = dict() + tasks['gdas'] = gdas_tasks + + if self.do_hybvar and 'gdas' in self.eupd_runs: + enkfgdas_tasks = hybrid_tasks + hybrid_after_eupd_tasks + tasks['enkfgdas'] = enkfgdas_tasks + + # Add RUN=gfs tasks if running early cycle + if self.interval_gfs > to_timedelta("0H"): + tasks['gfs'] = gfs_tasks - for run in self.runs: - options = self.run_options[run] + if self.do_hybvar and 'gfs' in self.eupd_runs: + enkfgfs_tasks = hybrid_tasks + hybrid_after_eupd_tasks + enkfgfs_tasks.remove("echgres") + enkfgfs_tasks.remove("esnowrecen") + tasks['enkfgfs'] = enkfgfs_tasks - # Common gdas and gfs tasks before fcst - if run in ['gdas', 'gfs']: - task_names[run] += ['prep'] - if options['do_jediatmvar']: - task_names[run] += ['prepatmiodaobs', 'atmanlinit', 'atmanlvar', 'atmanlfv3inc', 'atmanlfinal'] - else: - task_names[run] += ['anal'] - - if options['do_jediocnvar']: - task_names[run] += ['prepoceanobs', 'marineanlinit', 'marinebmat', 'marineanlvar'] - if options['do_hybvar']: - task_names[run] += ['marineanlletkf', 'ocnanalecen'] - task_names[run] += ['marineanlchkpt', 'marineanlfinal'] - if options['do_vrfy_oceanda']: - task_names[run] += ['ocnanalvrfy'] - - task_names[run] += ['sfcanl', 'analcalc'] - - if options['do_jedisnowda']: - task_names[run] += ['snowanl'] - - wave_prep_tasks = ['waveinit', 'waveprep'] - wave_bndpnt_tasks = ['wavepostbndpnt', 'wavepostbndpntbll'] - wave_post_tasks = ['wavepostsbs', 'wavepostpnt'] - - # gdas- and gfs-specific analysis tasks - if run == 'gdas': - if not options['do_jediatmvar']: - task_names[run] += ['analdiag'] - - if options['do_wave']: - task_names[run] += wave_prep_tasks - - if options['do_aero_anl']: - task_names[run] += ['aeroanlgenb'] - - else: - if options['do_wave']: - task_names[run] += wave_prep_tasks - - if options['do_aero_anl']: - task_names[run] += ['aeroanlinit', 'aeroanlvar', 'aeroanlfinal'] - - if options['do_prep_obs_aero']: - task_names[run] += ['prepobsaero'] - - # Staging is gdas-specific - if run == 'gdas': - task_names[run] += ['stage_ic'] - - task_names[run] += ['atmanlupp', 'atmanlprod', 'fcst'] - - # gfs-specific products - if run == 'gfs': - if options['do_ocean']: - task_names[run] += ['ocean_prod'] - - if options['do_ice']: - task_names[run] += ['ice_prod'] - - if options['do_upp']: - task_names[run] += ['atmupp'] - task_names[run] += ['atmos_prod'] - - # GOES post-processing (gfs only) - if run == 'gfs': - if options['do_goes']: - task_names[run] += ['goesupp'] - - # Only fit to obs and verify ozone and radiance during gdas cycles - if run == "gdas": - if options['do_fit2obs']: - task_names[run] += ['fit2obs'] - if options['do_verfozn']: - task_names[run] += ['verfozn'] - if options['do_verfrad']: - task_names[run] += ['verfrad'] - - if options['do_vminmon']: - task_names[run] += ['vminmon'] - - # gfs-only verification/tracking - if run == 'gfs': - if options['do_tracker']: - task_names[run] += ['tracker'] - - if options['do_genesis']: - task_names[run] += ['genesis'] - - if options['do_genesis_fsu']: - task_names[run] += ['genesis_fsu'] - - if options['do_metp']: - task_names[run] += ['metp'] - - if options['do_wave']: - if options['do_wave_bnd']: - task_names[run] += wave_bndpnt_tasks - task_names[run] += wave_post_tasks - # wave gempak and awips jobs are gfs-specific - if run == 'gfs': - if options['do_gempak']: - task_names[run] += ['wavegempak'] - if options['do_awips']: - task_names[run] += ['waveawipsbulls', 'waveawipsgridded'] - - # gdas- and gfs-specific downstream products - if run == 'gdas': - if options['do_gempak']: - task_names[run] += ['gempak', 'gempakmetancdc'] - else: - if options['do_bufrsnd']: - task_names[run] += ['postsnd'] - - if options['do_gempak']: - task_names[run] += ['gempak'] - task_names[run] += ['gempakmeta'] - task_names[run] += ['gempakncdcupapgif'] - if options['do_goes']: - task_names[run] += ['npoess_pgrb2_0p5deg'] - task_names[run] += ['gempakpgrb2spec'] - - if options['do_awips']: - task_names[run] += ['awips_20km_1p0deg', 'fbwind'] - - if options['do_mos']: - task_names[run] += ['mos_stn_prep', 'mos_grd_prep', 'mos_ext_stn_prep', 'mos_ext_grd_prep', - 'mos_stn_fcst', 'mos_grd_fcst', 'mos_ext_stn_fcst', 'mos_ext_grd_fcst', - 'mos_stn_prdgen', 'mos_grd_prdgen', 'mos_ext_stn_prdgen', - 'mos_ext_grd_prdgen', 'mos_wx_prdgen', 'mos_wx_ext_prdgen'] - - # Last two items - task_names[run] += ['arch', 'cleanup'] - - # Ensemble tasks - elif 'enkf' in run: - - if options['do_jediatmens']: - task_names[run] += ['atmensanlinit', 'atmensanlfv3inc', 'atmensanlfinal'] - # Only run echgres for the gdas cycle - task_names[run] += ['echgres'] if 'gdas' in run else 0 - if options['lobsdiag_forenkf']: - task_names[run] += ['atmensanlobs', 'atmensanlsol'] - else: - task_names[run] += ['atmensanlletkf'] - - else: - task_names[run] += ['eobs', 'eupd'] - task_names[run].append('echgres') if 'gdas' in run else 0 - task_names[run] += ['ediag'] if options['lobsdiag_forenkf'] else ['eomg'] - task_names[run].append('esnowanl') if options['do_jedisnowda'] and 'gdas' in run else 0 - - task_names[run].append('efcs') if 'gdas' in run else 0 - task_names[run].append('epos') if 'gdas' in run else 0 - task_names[run] += ['stage_ic', 'ecen', 'esfc', 'earc', 'cleanup'] - - return task_names + return tasks \ No newline at end of file From bec7451039be6afeb5e3c4aa3ea74bea4fc9c00c Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Wed, 8 Jan 2025 20:01:13 +0000 Subject: [PATCH 22/39] update applications.py --- workflow/applications/applications.py | 153 ++++++++++++++------------ 1 file changed, 83 insertions(+), 70 deletions(-) diff --git a/workflow/applications/applications.py b/workflow/applications/applications.py index f100264583..df49c96af0 100644 --- a/workflow/applications/applications.py +++ b/workflow/applications/applications.py @@ -1,8 +1,9 @@ #!/usr/bin/env python3 from typing import Dict, List, Any +from datetime import timedelta from hosts import Host -from wxflow import Configuration +from wxflow import Configuration, to_timedelta from abc import ABC, ABCMeta, abstractmethod __all__ = ['AppConfig'] @@ -30,84 +31,95 @@ def __init__(self, conf: Configuration) -> None: self.scheduler = Host().scheduler - # Get the most basic settings from config.base to determine - # experiment type ({NET}_{MODE}) base = conf.parse_config('config.base') self.mode = base['MODE'] + if self.mode not in self.VALID_MODES: raise NotImplementedError(f'{self.mode} is not a valid application mode.\n' f'Valid application modes are:\n' f'{", ".join(self.VALID_MODES)}\n') self.net = base['NET'] - print(f"Generating the XML for a {self.mode}_{self.net} case") + self.model_app = base.get('APP', 'ATM') + self.do_atm = base.get('DO_ATM', True) + self.do_wave = base.get('DO_WAVE', False) + self.do_wave_bnd = base.get('DOBNDPNT_WAVE', False) + self.do_ocean = base.get('DO_OCN', False) + self.do_ice = base.get('DO_ICE', False) + self.do_aero = base.get('DO_AERO', False) + self.do_prep_obs_aero = base.get('DO_PREP_OBS_AERO', False) + self.do_bufrsnd = base.get('DO_BUFRSND', False) + self.do_gempak = base.get('DO_GEMPAK', False) + self.do_awips = base.get('DO_AWIPS', False) + self.do_verfozn = base.get('DO_VERFOZN', True) + self.do_verfrad = base.get('DO_VERFRAD', True) + self.do_vminmon = base.get('DO_VMINMON', True) + self.do_anlstat = base.get('DO_ANLSTAT', True) + self.do_tracker = base.get('DO_TRACKER', True) + self.do_genesis = base.get('DO_GENESIS', True) + self.do_genesis_fsu = base.get('DO_GENESIS_FSU', False) + self.do_metp = base.get('DO_METP', False) + self.do_upp = not base.get('WRITE_DOPOST', True) + self.do_goes = base.get('DO_GOES', False) + self.do_mos = base.get('DO_MOS', False) + self.do_extractvars = base.get('DO_EXTRACTVARS', False) + + self.do_hpssarch = base.get('HPSSARCH', False) + + self.nens = base.get('NMEM_ENS', 0) + self.fcst_segments = base.get('FCST_SEGMENTS', None) + self.interval_gfs = to_timedelta(f"{base.get('INTERVAL_GFS')}H") + + if not AppConfig.is_monotonic(self.fcst_segments): + raise ValueError(f'Forecast segments do not increase monotonically: {",".join(self.fcst_segments)}') + + self.wave_runs = None + if self.do_wave: + wave_run = base.get('WAVE_RUN', 'BOTH').lower() + if wave_run in ['both']: + self.wave_runs = ['gfs', 'gdas'] + elif wave_run in ['gfs', 'gdas']: + self.wave_runs = [wave_run] + + self.aero_anl_runs = None + self.aero_fcst_runs = None + if self.do_aero: + aero_anl_run = base.get('AERO_ANL_RUN', 'BOTH').lower() + if aero_anl_run in ['both']: + self.aero_anl_runs = ['gfs', 'gdas'] + elif aero_anl_run in ['gfs', 'gdas']: + self.aero_anl_runs = [aero_anl_run] + aero_fcst_run = base.get('AERO_FCST_RUN', None).lower() + if aero_fcst_run in ['both']: + self.aero_fcst_runs = ['gfs', 'gdas'] + elif aero_fcst_run in ['gfs', 'gdas']: + self.aero_fcst_runs = [aero_fcst_run] def _init_finalize(self, conf: Configuration): - ''' - Finalize object initialization calling subclass methods - ''' + print("Finalizing initialize") - # Get run-, net-, and mode-based options - self.run_options = self._get_run_options(conf) + # Get a list of all possible config_files that would be part of the application + self.configs_names = self._get_app_configs() - # Get task names and runs for the application - self.task_names = self.get_task_names() + # Source the config files for the jobs in the application without specifying a RUN + self.configs = {'_no_run': self._source_configs(conf)} - # Initialize the configs and model_apps dictionaries - self.configs = dict.fromkeys(self.runs) + # Update the base config dictionary based on application + self.configs['_no_run']['base'] = self._update_base(self.configs['_no_run']['base']) - # Now configure the experiment for each valid run - for run in self.runs: - self.configs[run] = self._source_configs(conf, run=run, log=False) + # Save base in the internal state since it is often needed + base = self.configs['_no_run']['base'] - def _get_run_options(self, conf: Configuration) -> Dict[str, Any]: - ''' - Determine the do_* and APP options for each RUN by sourcing config.base - for each RUN and collecting the flags into self.run_options. Note that - this method is overloaded so additional NET- and MODE-dependent flags - can be set. - ''' + # Get task names for the application + self.task_names = self.get_task_names() - run_options = {run: {} for run in dict.fromkeys(self.runs)} - for run in self.runs: - # Read config.base with RUN specified - run_base = conf.parse_config('config.base', RUN=run) - - run_options[run]['app'] = run_base.get('APP', 'ATM') - run_options[run]['do_wave_bnd'] = run_base.get('DOBNDPNT_WAVE', False) - run_options[run]['do_bufrsnd'] = run_base.get('DO_BUFRSND', False) - run_options[run]['do_gempak'] = run_base.get('DO_GEMPAK', False) - run_options[run]['do_awips'] = run_base.get('DO_AWIPS', False) - run_options[run]['do_verfozn'] = run_base.get('DO_VERFOZN', True) - run_options[run]['do_verfrad'] = run_base.get('DO_VERFRAD', True) - run_options[run]['do_vminmon'] = run_base.get('DO_VMINMON', True) - run_options[run]['do_anlstat'] = run_base.get('DO_ANLSTAT', True) - run_options[run]['do_tracker'] = run_base.get('DO_TRACKER', True) - run_options[run]['do_genesis'] = run_base.get('DO_GENESIS', True) - run_options[run]['do_genesis_fsu'] = run_base.get('DO_GENESIS_FSU', False) - run_options[run]['do_metp'] = run_base.get('DO_METP', False) - run_options[run]['do_upp'] = not run_base.get('WRITE_DOPOST', True) - run_options[run]['do_goes'] = run_base.get('DO_GOES', False) - run_options[run]['do_mos'] = run_base.get('DO_MOS', False) - run_options[run]['do_extractvars'] = run_base.get('DO_EXTRACTVARS', False) - - run_options[run]['do_atm'] = run_base.get('DO_ATM', True) - run_options[run]['do_wave'] = run_base.get('DO_WAVE', False) - run_options[run]['do_ocean'] = run_base.get('DO_OCN', False) - run_options[run]['do_ice'] = run_base.get('DO_ICE', False) - run_options[run]['do_prep_obs_aero'] = run_base.get('DO_PREP_OBS_AERO', False) - run_options[run]['do_aero_anl'] = run_base.get('DO_AERO_ANL', False) - run_options[run]['do_aero_fcst'] = run_base.get('DO_AERO_FCST', False) - - run_options[run]['do_hpssarch'] = run_base.get('HPSSARCH', False) - run_options[run]['fcst_segments'] = run_base.get('FCST_SEGMENTS', None) - - if not AppConfig.is_monotonic(run_options[run]['fcst_segments']): - raise ValueError(f'Forecast segments do not increase monotonically: {",".join(self.fcst_segments)}') - - # Return the dictionary of run options - return run_options + # Finally, source the configuration files for each valid `RUN` + for run in self.task_names.keys(): + self.configs[run] = self._source_configs(conf, run=run, log=False) + + # Update the base config dictionary based on application and RUN + self.configs[run]['base'] = self._update_base(self.configs[run]['base']) @abstractmethod def _get_app_configs(self): @@ -139,12 +151,13 @@ def _source_configs(self, conf: Configuration, run: str = "gfs", log: bool = Tru Every config depends on "config.base" """ - # Include config.base by its lonesome and update it - configs = {'base': conf.parse_config('config.base', RUN=run)} - configs['base'] = self._update_base(configs['base']) + configs = dict() + + # Return config.base as well + configs['base'] = conf.parse_config('config.base', RUN=run) # Source the list of all config_files involved in the application - for config in self._get_app_configs(run): + for config in self.configs_names: # All must source config.base first files = ['config.base'] @@ -170,9 +183,9 @@ def _source_configs(self, conf: Configuration, run: str = "gfs", log: bool = Tru return configs @abstractmethod - def get_task_names(self, run: str) -> Dict[str, List[str]]: + def get_task_names(self, run="_no_run") -> Dict[str, List[str]]: ''' - Create a list of valid RUNs and a dict of task names for each RUN valid for the configuation. + Create a list of task names for each RUN valid for the configuation. Parameters ---------- @@ -180,7 +193,7 @@ def get_task_names(self, run: str) -> Dict[str, List[str]]: Returns ------- - Dict[str, List[str]]: Lists of all tasks for each RUN. + Dict[str, List[str]]: Lists of tasks for each RUN. ''' pass @@ -206,4 +219,4 @@ def is_monotonic(test_list: List, check_decreasing: bool = False) -> bool: if check_decreasing: return all(x > y for x, y in zip(test_list, test_list[1:])) else: - return all(x < y for x, y in zip(test_list, test_list[1:])) + return all(x < y for x, y in zip(test_list, test_list[1:])) \ No newline at end of file From 3e506907a111aa8cfc3a3fcdb629c95dedeb1257 Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Wed, 8 Jan 2025 20:12:12 +0000 Subject: [PATCH 23/39] update gfs_cycled.py correctly --- workflow/applications/gfs_cycled.py | 476 ++++++++++++++-------------- 1 file changed, 246 insertions(+), 230 deletions(-) diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index 1325aff664..f317dbd320 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -1,7 +1,6 @@ -from typing import Dict, Any from applications.applications import AppConfig -from wxflow import Configuration, to_timedelta -from datetime import timedelta +from typing import Dict, Any +from wxflow import Configuration class GFSCycledAppConfig(AppConfig): @@ -11,116 +10,140 @@ class GFSCycledAppConfig(AppConfig): def __init__(self, conf: Configuration): super().__init__(conf) + # Re-read config.base without RUN specified to get the basic settings for + # cycled cases to be able to determine valid runs base = conf.parse_config('config.base') - self.do_hybvar = base.get('DOHYBVAR', False) - self.do_fit2obs = base.get('DO_FIT2OBS', True) - self.do_jediatmvar = base.get('DO_JEDIATMVAR', False) - self.do_jediatmens = base.get('DO_JEDIATMENS', False) - self.do_jediocnvar = base.get('DO_JEDIOCNVAR', False) - self.do_jedisnowda = base.get('DO_JEDISNOWDA', False) - self.do_mergensst = base.get('DO_MERGENSST', False) - self.do_vrfy_oceanda = base.get('DO_VRFY_OCEANDA', False) - - self.lobsdiag_forenkf = False - self.eupd_runs = None - if self.do_hybvar: - self.lobsdiag_forenkf = base.get('lobsdiag_forenkf', False) - eupd_run = base.get('EUPD_CYC', 'gdas').lower() - if eupd_run in ['both']: - self.eupd_runs = ['gfs', 'gdas'] - elif eupd_run in ['gfs', 'gdas']: - self.eupd_runs = [eupd_run] - - def _get_app_configs(self): + + self.ens_runs = [] + + if base.get('DOHYBVAR', False): + ens_run = base.get('EUPD_CYC', 'gdas').lower() + if ens_run in ['both']: + self.ens_runs = ['gfs', 'gdas'] + elif ens_run in ['gfs', 'gdas']: + self.ens_runs = [ens_run] + + # Now construct self.runs the desired XML order (gdas, enkfgdas, gfs, enkfgfs) + self.runs = ["gdas"] # We always have a 'gdas' run + self.runs.append('enkfgdas') if 'gdas' in self.ens_runs else 0 + self.runs.append("gfs") if base['INTERVAL_GFS'] > 0 else 0 + self.runs.append('enkfgfs') if 'gfs' in self.ens_runs and "gfs" in self.runs else 0 + + def _get_run_options(self, conf: Configuration) -> Dict[str, Any]: + + run_options = super()._get_run_options(conf) + + for run in self.runs: + base = conf.parse_config('config.base', RUN=run) + + run_options[run]['do_hybvar'] = base.get('DOHYBVAR', False) + run_options[run]['do_hybvar_ocn'] = base.get('DOHYBVAR_OCN', False) + run_options[run]['nens'] = base.get('NMEM_ENS', 0) + if run_options[run]['do_hybvar']: + run_options[run]['lobsdiag_forenkf'] = base.get('lobsdiag_forenkf', False) + + run_options[run]['do_fit2obs'] = base.get('DO_FIT2OBS', True) + run_options[run]['do_jediatmvar'] = base.get('DO_JEDIATMVAR', False) + run_options[run]['do_jediatmens'] = base.get('DO_JEDIATMENS', False) + run_options[run]['do_jediocnvar'] = base.get('DO_JEDIOCNVAR', False) + run_options[run]['do_jedisnowda'] = base.get('DO_JEDISNOWDA', False) + run_options[run]['do_mergensst'] = base.get('DO_MERGENSST', False) + run_options[run]['do_vrfy_oceanda'] = base.get('DO_VRFY_OCEANDA', False) + + return run_options + + def _get_app_configs(self, run): """ - Returns the config_files that are involved in the cycled app + Returns the config files that are involved in the cycled app """ + options = self.run_options[run] configs = ['prep'] - if self.do_jediatmvar: + if options['do_jediatmvar']: configs += ['prepatmiodaobs', 'atmanlinit', 'atmanlvar', 'atmanlfv3inc', 'atmanlfinal'] else: configs += ['anal', 'analdiag'] - if self.do_jediocnvar: + if options['do_jediocnvar']: configs += ['prepoceanobs', 'marineanlinit', 'marinebmat', 'marineanlvar'] - if self.do_hybvar: + if options['do_hybvar']: configs += ['marineanlletkf', 'ocnanalecen'] configs += ['marineanlchkpt', 'marineanlfinal'] - if self.do_vrfy_oceanda: + if options['do_vrfy_oceanda']: configs += ['ocnanalvrfy'] - if self.do_ocean or self.do_ice: + if options['do_ocean'] or options['do_ice']: configs += ['oceanice_products'] configs += ['stage_ic', 'sfcanl', 'analcalc', 'fcst', 'upp', 'atmos_products', 'arch', 'cleanup'] - if self.do_hybvar: - if self.do_jediatmens: - configs += ['atmensanlinit', 'atmensanlobs', 'atmensanlsol', 'atmensanlletkf', 'atmensanlfv3inc', 'atmensanlfinal'] + if options['do_hybvar']: + if options['do_jediatmens']: + configs += ['atmensanlinit', 'atmensanlobs', 'atmensanlsol', + 'atmensanlletkf', 'atmensanlfv3inc', 'atmensanlfinal'] else: configs += ['eobs', 'eomg', 'ediag', 'eupd'] configs += ['ecen', 'esfc', 'efcs', 'echgres', 'epos', 'earc'] - if self.do_fit2obs: + if options['do_fit2obs']: configs += ['fit2obs'] - if self.do_verfozn: + if options['do_verfozn']: configs += ['verfozn'] - if self.do_verfrad: + if options['do_verfrad']: configs += ['verfrad'] - if self.do_vminmon: + if options['do_vminmon']: configs += ['vminmon'] - - if self.do_anlstat: + + if options['do_anlstat']: configs += ['anlstat'] - if self.do_tracker: + if options['do_tracker']: configs += ['tracker'] - if self.do_genesis: + if options['do_genesis']: configs += ['genesis'] - if self.do_genesis_fsu: + if options['do_genesis_fsu']: configs += ['genesis_fsu'] - if self.do_metp: + if options['do_metp']: configs += ['metp'] - if self.do_gempak: + if options['do_gempak']: configs += ['gempak'] - if self.do_goes: + if options['do_goes']: configs += ['npoess'] - if self.do_bufrsnd: + if options['do_bufrsnd']: configs += ['postsnd'] - if self.do_awips: + if options['do_awips']: configs += ['awips'] - if self.do_wave: + if options['do_wave']: configs += ['waveinit', 'waveprep', 'wavepostsbs', 'wavepostpnt'] - if self.do_wave_bnd: + if options['do_wave_bnd']: configs += ['wavepostbndpnt', 'wavepostbndpntbll'] - if self.do_gempak: + if options['do_gempak']: configs += ['wavegempak'] - if self.do_awips: + if options['do_awips']: configs += ['waveawipsbulls', 'waveawipsgridded'] - if self.do_aero: + if options['do_aero_anl']: configs += ['aeroanlgenb', 'aeroanlinit', 'aeroanlvar', 'aeroanlfinal'] - if self.do_prep_obs_aero: + if options['do_prep_obs_aero']: configs += ['prepobsaero'] - if self.do_jedisnowda: + if options['do_jedisnowda']: configs += ['snowanl'] - if self.do_hybvar: - configs += ['esnowrecen'] + if options['do_hybvar']: + configs += ['esnowanl'] - if self.do_mos: + if options['do_mos']: configs += ['mos_stn_prep', 'mos_grd_prep', 'mos_ext_stn_prep', 'mos_ext_grd_prep', 'mos_stn_fcst', 'mos_grd_fcst', 'mos_ext_stn_fcst', 'mos_ext_grd_fcst', 'mos_stn_prdgen', 'mos_grd_prdgen', 'mos_ext_stn_prdgen', 'mos_ext_grd_prdgen', @@ -135,181 +158,174 @@ def _update_base(base_in): def get_task_names(self): """ - Get the task names for all the tasks in the cycled application. - Note that the order of the task names matters in the XML. - This is the place where that order is set. + Get the task names for each valid run in this cycled configuration. + NOTE: The order of the task names matters in the XML. + This is the place where that order is set. """ - gdas_gfs_common_tasks_before_fcst = ['prep'] - gdas_gfs_common_cleanup_tasks = ['arch', 'cleanup'] - - if self.do_jediatmvar: - gdas_gfs_common_tasks_before_fcst += ['prepatmiodaobs', 'atmanlinit', 'atmanlvar', 'atmanlfv3inc', 'atmanlfinal'] - else: - gdas_gfs_common_tasks_before_fcst += ['anal'] - - if self.do_jediocnvar: - gdas_gfs_common_tasks_before_fcst += ['prepoceanobs', 'marineanlinit', 'marinebmat', 'marineanlvar'] - if self.do_hybvar: - gdas_gfs_common_tasks_before_fcst += ['marineanlletkf', 'ocnanalecen'] - gdas_gfs_common_tasks_before_fcst += ['marineanlchkpt', 'marineanlfinal'] - if self.do_vrfy_oceanda: - gdas_gfs_common_tasks_before_fcst += ['ocnanalvrfy'] - - gdas_gfs_common_tasks_before_fcst += ['sfcanl', 'analcalc'] - - if self.do_jedisnowda: - gdas_gfs_common_tasks_before_fcst += ['snowanl'] - - wave_prep_tasks = ['waveinit', 'waveprep'] - wave_bndpnt_tasks = ['wavepostbndpnt', 'wavepostbndpntbll'] - wave_post_tasks = ['wavepostsbs', 'wavepostpnt'] - - hybrid_tasks = [] - hybrid_after_eupd_tasks = [] - if self.do_hybvar: - if self.do_jediatmens: - hybrid_tasks += ['atmensanlinit', 'atmensanlfv3inc', 'atmensanlfinal', 'echgres'] - hybrid_tasks += ['atmensanlobs', 'atmensanlsol'] if self.lobsdiag_forenkf else ['atmensanlletkf'] - else: - hybrid_tasks += ['eobs', 'eupd', 'echgres'] - hybrid_tasks += ['ediag'] if self.lobsdiag_forenkf else ['eomg'] - if self.do_jedisnowda: - hybrid_tasks += ['esnowrecen'] - hybrid_after_eupd_tasks += ['stage_ic', 'ecen', 'esfc', 'efcs', 'epos', 'earc', 'cleanup'] - - # Collect all "gdas" cycle tasks - gdas_tasks = gdas_gfs_common_tasks_before_fcst.copy() - - if not self.do_jediatmvar: - gdas_tasks += ['analdiag'] - - if self.do_wave and 'gdas' in self.wave_runs: - gdas_tasks += wave_prep_tasks - - if self.do_aero and 'gdas' in self.aero_anl_runs: - gdas_tasks += ['aeroanlgenb', 'aeroanlinit', 'aeroanlvar', 'aeroanlfinal'] - if self.do_prep_obs_aero: - gdas_tasks += ['prepobsaero'] - - gdas_tasks += ['stage_ic', 'atmanlupp', 'atmanlprod', 'fcst'] - - if self.do_upp: - gdas_tasks += ['atmupp'] - gdas_tasks += ['atmos_prod'] - - if self.do_wave and 'gdas' in self.wave_runs: - if self.do_wave_bnd: - gdas_tasks += wave_bndpnt_tasks - gdas_tasks += wave_post_tasks - - if self.do_fit2obs: - gdas_tasks += ['fit2obs'] - - if self.do_verfozn: - gdas_tasks += ['verfozn'] - - if self.do_verfrad: - gdas_tasks += ['verfrad'] - - if self.do_vminmon: - gdas_tasks += ['vminmon'] - - if self.do_gempak: - gdas_tasks += ['gempak', 'gempakmetancdc'] - - gdas_tasks += gdas_gfs_common_cleanup_tasks - - # Collect "gfs" cycle tasks - gfs_tasks = gdas_gfs_common_tasks_before_fcst.copy() - - if self.do_wave and 'gfs' in self.wave_runs: - gfs_tasks += wave_prep_tasks - - if self.do_aero and 'gfs' in self.aero_anl_runs: - gfs_tasks += ['aeroanlinit', 'aeroanlvar', 'aeroanlfinal'] - if self.do_prep_obs_aero: - gfs_tasks += ['prepobsaero'] - - gfs_tasks += ['atmanlupp', 'atmanlprod', 'fcst'] - - if self.do_ocean: - gfs_tasks += ['ocean_prod'] - - if self.do_ice: - gfs_tasks += ['ice_prod'] - - if self.do_upp: - gfs_tasks += ['atmupp'] - gfs_tasks += ['atmos_prod'] - - if self.do_goes: - gfs_tasks += ['goesupp'] - - if self.do_vminmon: - gfs_tasks += ['vminmon'] - - if self.do_anlstat: - gfs_tasks += ['anlstat'] - - if self.do_tracker: - gfs_tasks += ['tracker'] - - if self.do_genesis: - gfs_tasks += ['genesis'] - - if self.do_genesis_fsu: - gfs_tasks += ['genesis_fsu'] - - if self.do_metp: - gfs_tasks += ['metp'] - - if self.do_wave and 'gfs' in self.wave_runs: - if self.do_wave_bnd: - gfs_tasks += wave_bndpnt_tasks - gfs_tasks += wave_post_tasks - if self.do_gempak: - gfs_tasks += ['wavegempak'] - if self.do_awips: - gfs_tasks += ['waveawipsbulls', 'waveawipsgridded'] - - if self.do_bufrsnd: - gfs_tasks += ['postsnd'] - - if self.do_gempak: - gfs_tasks += ['gempak'] - gfs_tasks += ['gempakmeta'] - gfs_tasks += ['gempakncdcupapgif'] - if self.do_goes: - gfs_tasks += ['npoess_pgrb2_0p5deg'] - gfs_tasks += ['gempakpgrb2spec'] - - if self.do_awips: - gfs_tasks += ['awips_20km_1p0deg', 'fbwind'] - - if self.do_mos: - gfs_tasks += ['mos_stn_prep', 'mos_grd_prep', 'mos_ext_stn_prep', 'mos_ext_grd_prep', - 'mos_stn_fcst', 'mos_grd_fcst', 'mos_ext_stn_fcst', 'mos_ext_grd_fcst', - 'mos_stn_prdgen', 'mos_grd_prdgen', 'mos_ext_stn_prdgen', 'mos_ext_grd_prdgen', - 'mos_wx_prdgen', 'mos_wx_ext_prdgen'] - - gfs_tasks += gdas_gfs_common_cleanup_tasks - - tasks = dict() - tasks['gdas'] = gdas_tasks - - if self.do_hybvar and 'gdas' in self.eupd_runs: - enkfgdas_tasks = hybrid_tasks + hybrid_after_eupd_tasks - tasks['enkfgdas'] = enkfgdas_tasks - - # Add RUN=gfs tasks if running early cycle - if self.interval_gfs > to_timedelta("0H"): - tasks['gfs'] = gfs_tasks + # Start with a dictionary of empty task lists for each valid run + task_names = {run: [] for run in self.runs} - if self.do_hybvar and 'gfs' in self.eupd_runs: - enkfgfs_tasks = hybrid_tasks + hybrid_after_eupd_tasks - enkfgfs_tasks.remove("echgres") - enkfgfs_tasks.remove("esnowrecen") - tasks['enkfgfs'] = enkfgfs_tasks + for run in self.runs: + options = self.run_options[run] - return tasks \ No newline at end of file + # Common gdas and gfs tasks before fcst + if run in ['gdas', 'gfs']: + task_names[run] += ['prep'] + if options['do_jediatmvar']: + task_names[run] += ['prepatmiodaobs', 'atmanlinit', 'atmanlvar', 'atmanlfv3inc', 'atmanlfinal'] + else: + task_names[run] += ['anal'] + + if options['do_jediocnvar']: + task_names[run] += ['prepoceanobs', 'marineanlinit', 'marinebmat', 'marineanlvar'] + if options['do_hybvar']: + task_names[run] += ['marineanlletkf', 'ocnanalecen'] + task_names[run] += ['marineanlchkpt', 'marineanlfinal'] + if options['do_vrfy_oceanda']: + task_names[run] += ['ocnanalvrfy'] + + task_names[run] += ['sfcanl', 'analcalc'] + + if options['do_jedisnowda']: + task_names[run] += ['snowanl'] + + wave_prep_tasks = ['waveinit', 'waveprep'] + wave_bndpnt_tasks = ['wavepostbndpnt', 'wavepostbndpntbll'] + wave_post_tasks = ['wavepostsbs', 'wavepostpnt'] + + # gdas- and gfs-specific analysis tasks + if run == 'gdas': + if not options['do_jediatmvar']: + task_names[run] += ['analdiag'] + + if options['do_wave']: + task_names[run] += wave_prep_tasks + + if options['do_aero_anl']: + task_names[run] += ['aeroanlgenb'] + + else: + if options['do_wave']: + task_names[run] += wave_prep_tasks + + if options['do_aero_anl']: + task_names[run] += ['aeroanlinit', 'aeroanlvar', 'aeroanlfinal'] + + if options['do_prep_obs_aero']: + task_names[run] += ['prepobsaero'] + + # Staging is gdas-specific + if run == 'gdas': + task_names[run] += ['stage_ic'] + + task_names[run] += ['atmanlupp', 'atmanlprod', 'fcst'] + + # gfs-specific products + if run == 'gfs': + if options['do_ocean']: + task_names[run] += ['ocean_prod'] + + if options['do_ice']: + task_names[run] += ['ice_prod'] + + if options['do_upp']: + task_names[run] += ['atmupp'] + task_names[run] += ['atmos_prod'] + + # GOES post-processing (gfs only) + if run == 'gfs': + if options['do_goes']: + task_names[run] += ['goesupp'] + + # Only fit to obs and verify ozone and radiance during gdas cycles + if run == "gdas": + if options['do_fit2obs']: + task_names[run] += ['fit2obs'] + if options['do_verfozn']: + task_names[run] += ['verfozn'] + if options['do_verfrad']: + task_names[run] += ['verfrad'] + + if options['do_vminmon']: + task_names[run] += ['vminmon'] + + if options['do_anlstat']: + task_names[run] += ['anlstat'] + + # gfs-only verification/tracking + if run == 'gfs': + if options['do_tracker']: + task_names[run] += ['tracker'] + + if options['do_genesis']: + task_names[run] += ['genesis'] + + if options['do_genesis_fsu']: + task_names[run] += ['genesis_fsu'] + + if options['do_metp']: + task_names[run] += ['metp'] + + if options['do_wave']: + if options['do_wave_bnd']: + task_names[run] += wave_bndpnt_tasks + task_names[run] += wave_post_tasks + # wave gempak and awips jobs are gfs-specific + if run == 'gfs': + if options['do_gempak']: + task_names[run] += ['wavegempak'] + if options['do_awips']: + task_names[run] += ['waveawipsbulls', 'waveawipsgridded'] + + # gdas- and gfs-specific downstream products + if run == 'gdas': + if options['do_gempak']: + task_names[run] += ['gempak', 'gempakmetancdc'] + else: + if options['do_bufrsnd']: + task_names[run] += ['postsnd'] + + if options['do_gempak']: + task_names[run] += ['gempak'] + task_names[run] += ['gempakmeta'] + task_names[run] += ['gempakncdcupapgif'] + if options['do_goes']: + task_names[run] += ['npoess_pgrb2_0p5deg'] + task_names[run] += ['gempakpgrb2spec'] + + if options['do_awips']: + task_names[run] += ['awips_20km_1p0deg', 'fbwind'] + + if options['do_mos']: + task_names[run] += ['mos_stn_prep', 'mos_grd_prep', 'mos_ext_stn_prep', 'mos_ext_grd_prep', + 'mos_stn_fcst', 'mos_grd_fcst', 'mos_ext_stn_fcst', 'mos_ext_grd_fcst', + 'mos_stn_prdgen', 'mos_grd_prdgen', 'mos_ext_stn_prdgen', + 'mos_ext_grd_prdgen', 'mos_wx_prdgen', 'mos_wx_ext_prdgen'] + + # Last two items + task_names[run] += ['arch', 'cleanup'] + + # Ensemble tasks + elif 'enkf' in run: + + if options['do_jediatmens']: + task_names[run] += ['atmensanlinit', 'atmensanlfv3inc', 'atmensanlfinal'] + # Only run echgres for the gdas cycle + task_names[run] += ['echgres'] if 'gdas' in run else 0 + if options['lobsdiag_forenkf']: + task_names[run] += ['atmensanlobs', 'atmensanlsol'] + else: + task_names[run] += ['atmensanlletkf'] + + else: + task_names[run] += ['eobs', 'eupd'] + task_names[run].append('echgres') if 'gdas' in run else 0 + task_names[run] += ['ediag'] if options['lobsdiag_forenkf'] else ['eomg'] + task_names[run].append('esnowanl') if options['do_jedisnowda'] and 'gdas' in run else 0 + + task_names[run].append('efcs') if 'gdas' in run else 0 + task_names[run].append('epos') if 'gdas' in run else 0 + task_names[run] += ['stage_ic', 'ecen', 'esfc', 'earc', 'cleanup'] + + return task_names \ No newline at end of file From fff3e76227e39e2d10fd7a08da32ec28be9bf84a Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Wed, 8 Jan 2025 20:24:57 +0000 Subject: [PATCH 24/39] update workflow tasks correctly --- workflow/applications/applications.py | 151 ++++++++++++-------------- workflow/rocoto/tasks.py | 7 +- 2 files changed, 72 insertions(+), 86 deletions(-) diff --git a/workflow/applications/applications.py b/workflow/applications/applications.py index df49c96af0..4e09224488 100644 --- a/workflow/applications/applications.py +++ b/workflow/applications/applications.py @@ -1,9 +1,8 @@ #!/usr/bin/env python3 from typing import Dict, List, Any -from datetime import timedelta from hosts import Host -from wxflow import Configuration, to_timedelta +from wxflow import Configuration from abc import ABC, ABCMeta, abstractmethod __all__ = ['AppConfig'] @@ -31,95 +30,84 @@ def __init__(self, conf: Configuration) -> None: self.scheduler = Host().scheduler + # Get the most basic settings from config.base to determine + # experiment type ({NET}_{MODE}) base = conf.parse_config('config.base') self.mode = base['MODE'] - if self.mode not in self.VALID_MODES: raise NotImplementedError(f'{self.mode} is not a valid application mode.\n' f'Valid application modes are:\n' f'{", ".join(self.VALID_MODES)}\n') self.net = base['NET'] - self.model_app = base.get('APP', 'ATM') - self.do_atm = base.get('DO_ATM', True) - self.do_wave = base.get('DO_WAVE', False) - self.do_wave_bnd = base.get('DOBNDPNT_WAVE', False) - self.do_ocean = base.get('DO_OCN', False) - self.do_ice = base.get('DO_ICE', False) - self.do_aero = base.get('DO_AERO', False) - self.do_prep_obs_aero = base.get('DO_PREP_OBS_AERO', False) - self.do_bufrsnd = base.get('DO_BUFRSND', False) - self.do_gempak = base.get('DO_GEMPAK', False) - self.do_awips = base.get('DO_AWIPS', False) - self.do_verfozn = base.get('DO_VERFOZN', True) - self.do_verfrad = base.get('DO_VERFRAD', True) - self.do_vminmon = base.get('DO_VMINMON', True) - self.do_anlstat = base.get('DO_ANLSTAT', True) - self.do_tracker = base.get('DO_TRACKER', True) - self.do_genesis = base.get('DO_GENESIS', True) - self.do_genesis_fsu = base.get('DO_GENESIS_FSU', False) - self.do_metp = base.get('DO_METP', False) - self.do_upp = not base.get('WRITE_DOPOST', True) - self.do_goes = base.get('DO_GOES', False) - self.do_mos = base.get('DO_MOS', False) - self.do_extractvars = base.get('DO_EXTRACTVARS', False) - - self.do_hpssarch = base.get('HPSSARCH', False) - - self.nens = base.get('NMEM_ENS', 0) - self.fcst_segments = base.get('FCST_SEGMENTS', None) - self.interval_gfs = to_timedelta(f"{base.get('INTERVAL_GFS')}H") - - if not AppConfig.is_monotonic(self.fcst_segments): - raise ValueError(f'Forecast segments do not increase monotonically: {",".join(self.fcst_segments)}') - - self.wave_runs = None - if self.do_wave: - wave_run = base.get('WAVE_RUN', 'BOTH').lower() - if wave_run in ['both']: - self.wave_runs = ['gfs', 'gdas'] - elif wave_run in ['gfs', 'gdas']: - self.wave_runs = [wave_run] - - self.aero_anl_runs = None - self.aero_fcst_runs = None - if self.do_aero: - aero_anl_run = base.get('AERO_ANL_RUN', 'BOTH').lower() - if aero_anl_run in ['both']: - self.aero_anl_runs = ['gfs', 'gdas'] - elif aero_anl_run in ['gfs', 'gdas']: - self.aero_anl_runs = [aero_anl_run] - aero_fcst_run = base.get('AERO_FCST_RUN', None).lower() - if aero_fcst_run in ['both']: - self.aero_fcst_runs = ['gfs', 'gdas'] - elif aero_fcst_run in ['gfs', 'gdas']: - self.aero_fcst_runs = [aero_fcst_run] + print(f"Generating the XML for a {self.mode}_{self.net} case") def _init_finalize(self, conf: Configuration): - print("Finalizing initialize") - - # Get a list of all possible config_files that would be part of the application - self.configs_names = self._get_app_configs() - - # Source the config files for the jobs in the application without specifying a RUN - self.configs = {'_no_run': self._source_configs(conf)} - - # Update the base config dictionary based on application - self.configs['_no_run']['base'] = self._update_base(self.configs['_no_run']['base']) + ''' + Finalize object initialization calling subclass methods + ''' - # Save base in the internal state since it is often needed - base = self.configs['_no_run']['base'] + # Get run-, net-, and mode-based options + self.run_options = self._get_run_options(conf) - # Get task names for the application + # Get task names and runs for the application self.task_names = self.get_task_names() - # Finally, source the configuration files for each valid `RUN` - for run in self.task_names.keys(): + # Initialize the configs and model_apps dictionaries + self.configs = dict.fromkeys(self.runs) + + # Now configure the experiment for each valid run + for run in self.runs: self.configs[run] = self._source_configs(conf, run=run, log=False) - # Update the base config dictionary based on application and RUN - self.configs[run]['base'] = self._update_base(self.configs[run]['base']) + def _get_run_options(self, conf: Configuration) -> Dict[str, Any]: + ''' + Determine the do_* and APP options for each RUN by sourcing config.base + for each RUN and collecting the flags into self.run_options. Note that + this method is overloaded so additional NET- and MODE-dependent flags + can be set. + ''' + + run_options = {run: {} for run in dict.fromkeys(self.runs)} + for run in self.runs: + # Read config.base with RUN specified + run_base = conf.parse_config('config.base', RUN=run) + + run_options[run]['app'] = run_base.get('APP', 'ATM') + run_options[run]['do_wave_bnd'] = run_base.get('DOBNDPNT_WAVE', False) + run_options[run]['do_bufrsnd'] = run_base.get('DO_BUFRSND', False) + run_options[run]['do_gempak'] = run_base.get('DO_GEMPAK', False) + run_options[run]['do_awips'] = run_base.get('DO_AWIPS', False) + run_options[run]['do_verfozn'] = run_base.get('DO_VERFOZN', True) + run_options[run]['do_verfrad'] = run_base.get('DO_VERFRAD', True) + run_options[run]['do_vminmon'] = run_base.get('DO_VMINMON', True) + run_options[run]['do_anlstat'] = run_base.get('DO_ANLSTAT', True) + run_options[run]['do_tracker'] = run_base.get('DO_TRACKER', True) + run_options[run]['do_genesis'] = run_base.get('DO_GENESIS', True) + run_options[run]['do_genesis_fsu'] = run_base.get('DO_GENESIS_FSU', False) + run_options[run]['do_metp'] = run_base.get('DO_METP', False) + run_options[run]['do_upp'] = not run_base.get('WRITE_DOPOST', True) + run_options[run]['do_goes'] = run_base.get('DO_GOES', False) + run_options[run]['do_mos'] = run_base.get('DO_MOS', False) + run_options[run]['do_extractvars'] = run_base.get('DO_EXTRACTVARS', False) + + run_options[run]['do_atm'] = run_base.get('DO_ATM', True) + run_options[run]['do_wave'] = run_base.get('DO_WAVE', False) + run_options[run]['do_ocean'] = run_base.get('DO_OCN', False) + run_options[run]['do_ice'] = run_base.get('DO_ICE', False) + run_options[run]['do_prep_obs_aero'] = run_base.get('DO_PREP_OBS_AERO', False) + run_options[run]['do_aero_anl'] = run_base.get('DO_AERO_ANL', False) + run_options[run]['do_aero_fcst'] = run_base.get('DO_AERO_FCST', False) + + run_options[run]['do_hpssarch'] = run_base.get('HPSSARCH', False) + run_options[run]['fcst_segments'] = run_base.get('FCST_SEGMENTS', None) + + if not AppConfig.is_monotonic(run_options[run]['fcst_segments']): + raise ValueError(f'Forecast segments do not increase monotonically: {",".join(self.fcst_segments)}') + + # Return the dictionary of run options + return run_options @abstractmethod def _get_app_configs(self): @@ -151,13 +139,12 @@ def _source_configs(self, conf: Configuration, run: str = "gfs", log: bool = Tru Every config depends on "config.base" """ - configs = dict() - - # Return config.base as well - configs['base'] = conf.parse_config('config.base', RUN=run) + # Include config.base by its lonesome and update it + configs = {'base': conf.parse_config('config.base', RUN=run)} + configs['base'] = self._update_base(configs['base']) # Source the list of all config_files involved in the application - for config in self.configs_names: + for config in self._get_app_configs(run): # All must source config.base first files = ['config.base'] @@ -183,9 +170,9 @@ def _source_configs(self, conf: Configuration, run: str = "gfs", log: bool = Tru return configs @abstractmethod - def get_task_names(self, run="_no_run") -> Dict[str, List[str]]: + def get_task_names(self, run: str) -> Dict[str, List[str]]: ''' - Create a list of task names for each RUN valid for the configuation. + Create a list of valid RUNs and a dict of task names for each RUN valid for the configuation. Parameters ---------- @@ -193,7 +180,7 @@ def get_task_names(self, run="_no_run") -> Dict[str, List[str]]: Returns ------- - Dict[str, List[str]]: Lists of tasks for each RUN. + Dict[str, List[str]]: Lists of all tasks for each RUN. ''' pass diff --git a/workflow/rocoto/tasks.py b/workflow/rocoto/tasks.py index 68217dfc87..6cff2d0ea1 100644 --- a/workflow/rocoto/tasks.py +++ b/workflow/rocoto/tasks.py @@ -20,12 +20,11 @@ class Tasks: 'eobs', 'eomg', 'epos', 'esfc', 'eupd', 'atmensanlinit', 'atmensanlobs', 'atmensanlsol', 'atmensanlletkf', 'atmensanlfv3inc', 'atmensanlfinal', 'aeroanlinit', 'aeroanlvar', 'aeroanlfinal', 'aeroanlgenb', - 'snowanl', 'esnowrecen', - 'anlstat', + 'snowanl', 'esnowanl', 'fcst', 'atmanlupp', 'atmanlprod', 'atmupp', 'goesupp', 'atmos_prod', 'ocean_prod', 'ice_prod', - 'verfozn', 'verfrad', 'vminmon', + 'verfozn', 'verfrad', 'vminmon', 'anlstat', 'metp', 'tracker', 'genesis', 'genesis_fsu', 'postsnd', 'awips_20km_1p0deg', 'fbwind', @@ -257,4 +256,4 @@ def get_task(self, task_name, *args, **kwargs): except AttributeError: raise AttributeError(f'"{task_name}" is not a valid task.\n' f'Valid tasks are:\n' - f'{", ".join(Tasks.VALID_TASKS)}') + f'{", ".join(Tasks.VALID_TASKS)}') \ No newline at end of file From 32b8a7ab55e7f77f6f87bb9be3523e4ac6175ec9 Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Mon, 13 Jan 2025 15:53:52 +0000 Subject: [PATCH 25/39] update gfs tasks --- workflow/rocoto/gfs_tasks.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 87147aa254..eda7622e3f 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -1755,16 +1755,16 @@ def vminmon(self): def anlstat(self): deps = [] - if self.app_config.do_jediatmvar: + if self.options['do_jediatmvar']: dep_dict = {'type': 'task', 'name': f'{self.run}_atmanlfinal'} deps.append(rocoto.add_dependency(dep_dict)) - if self.app_config.do_jediocnvar: + if self.options['do_jediocnvar']: dep_dict = {'type': 'task', 'name': f'{self.run}_ocnanalpost'} deps.append(rocoto.add_dependency(dep_dict)) - if self.app_config.do_jedisnowda: + if self.options['do_jedisnowda']: dep_dict = {'type': 'task', 'name': f'{self.run}_snowanl'} deps.append(rocoto.add_dependency(dep_dict)) - if self.app_config.do_aero: + if self.options['do_aero_anl']: dep_dict = {'type': 'task', 'name': f'{self.run}_aeroanlfinal'} deps.append(rocoto.add_dependency(dep_dict)) From 9fe7e203cd293dab5421779a9d9e2864bf9ee979 Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Mon, 27 Jan 2025 16:14:40 +0000 Subject: [PATCH 26/39] updates to python scripts to handle snow ob spaces --- ush/python/pygfs/task/stat_analysis.py | 37 +++++++++++++++++++------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/ush/python/pygfs/task/stat_analysis.py b/ush/python/pygfs/task/stat_analysis.py index fb9e7af12b..2b665a849b 100644 --- a/ush/python/pygfs/task/stat_analysis.py +++ b/ush/python/pygfs/task/stat_analysis.py @@ -60,7 +60,6 @@ def __init__(self, config: Dict[str, Any]): 'GPREFIX': f"gdas.t{self.task_config.previous_cycle.hour:02d}z." } ) - # Extend task_config with local_dict self.task_config = AttrDict(**self.task_config, **local_dict) @@ -80,18 +79,27 @@ def initialize(self) -> None: None """ # Create dictionary of Jedi objects - expected_keys = self.task_config.STAT_OBS + # Expected keys are what must be included from the JEDI config file. We can + # then loop through ob space list from scripts/exglobal_analysis_stats.py + expected_keys = ['aero', 'snow'] self.jedi_dict = Jedi.get_jedi_dict(self.task_config.JEDI_CONFIG_YAML, self.task_config, expected_keys) logger.info(f"Copying files to {self.task_config.DATA}/stats") + # Loop through ob space list for OB in self.task_config.STAT_OBS: + logger.info(f"Working on current observation: {OB}") + # Parse JEDI analysis stat jinja file obs_dict = parse_j2yaml(self.task_config.JEDI_CONFIG_YAML, self.task_config) # Copy stat files to DATA path instat_files = os.path.join(obs_dict[OB]['stat_file_path'], f"{self.task_config['APREFIX']}{obs_dict[OB]['stat_file_name']}") - dest = os.path.join(self.task_config.DATA, obs_dict[OB]['stat_file_name']) + ob_dir_str = f"{self.task_config.DATA}" + f"/{OB}" + os.mkdir(ob_dir_str) + + dest = os.path.join(ob_dir_str, obs_dict[OB]['stat_file_name']) + logger.info(f"Copying {instat_files} to {dest} ...") statlist = [[instat_files, dest]] FileHandler({'copy': statlist}).sync() @@ -99,24 +107,35 @@ def initialize(self) -> None: logger.info(f"Open tarred stat file in {dest}") with tarfile.open(dest, "r") as tar: # Extract all files to the current directory - tar.extractall() + tar.extractall(path=f'{ob_dir_str}') # Gunzip .nc files logger.info("Gunzip files from tar file") - gz_files = glob.glob(os.path.join(self.task_config.DATA, "*gz")) + gz_files = glob.glob(os.path.join(ob_dir_str, "*.gz")) + logger.info(f"Gunzip files: {gz_files}") for diagfile in gz_files: + output_file = os.path.join(ob_dir_str, os.path.basename(diagfile)[:-3]) with gzip.open(diagfile, 'rb') as f_in: - with open(diagfile[:-3], 'wb') as f_out: + with open(output_file, 'wb') as f_out: f_out.write(f_in.read()) # Get list of .nc4 files - obs_space_paths = glob.glob(os.path.join(self.task_config.DATA, "*.nc4")) + # obs_space_paths = glob.glob(os.path.join(ob_dir_str, "*.{nc,nc4}")) # THIS SHOULD WORK BUT ISNT, glob patterns introduced in Python 3.9 + nc_paths = glob.glob(os.path.join(ob_dir_str, "*.nc")) + nc4_paths = glob.glob(os.path.join(ob_dir_str, "*.nc4")) + obs_space_paths = nc_paths + nc4_paths + # Temporary. Create condition check here for available jcb algorithms? + if OB == 'snow': + obs_space_paths = glob.glob(os.path.join(ob_dir_str, "diag_ims_snow_*.nc")) + + # This grabs the obspace string from the .nc4 files, however not all are perfect. Need solution. self.task_config.OBSPACES_LIST = ['_'.join(os.path.basename(path).split('_')[1:3]) for path in obs_space_paths] # initialize JEDI application logger.info(f"Initializing JEDI variational DA application") + logger.info(f"{self.jedi_dict[OB]}") self.jedi_dict[OB].initialize(self.task_config) @logit(logger) @@ -132,7 +151,7 @@ def execute(self, jedi_dict_key: str) -> None: ---------- None """ - + logger.info(f"In execute. {self.jedi_dict[jedi_dict_key]}") self.jedi_dict[jedi_dict_key].execute() @logit(logger) @@ -154,7 +173,7 @@ def finalize(self, jedi_dict_key: str) -> None: """ # get list of output diag files - diags = glob.glob(os.path.join(self.task_config.DATA, '*output_aod.nc')) + diags = glob.glob(os.path.join(self.task_config.DATA, '*output_*.nc')) for diagfile in diags: outfile = os.path.basename(diagfile) From 1454c0fb57917c5503fb07b215fae9d3b34fa5bb Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Mon, 27 Jan 2025 16:56:48 +0000 Subject: [PATCH 27/39] fix pycodestyle issues --- workflow/applications/applications.py | 2 +- workflow/applications/gfs_cycled.py | 2 +- workflow/rocoto/tasks.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/workflow/applications/applications.py b/workflow/applications/applications.py index 4e09224488..f100264583 100644 --- a/workflow/applications/applications.py +++ b/workflow/applications/applications.py @@ -206,4 +206,4 @@ def is_monotonic(test_list: List, check_decreasing: bool = False) -> bool: if check_decreasing: return all(x > y for x, y in zip(test_list, test_list[1:])) else: - return all(x < y for x, y in zip(test_list, test_list[1:])) \ No newline at end of file + return all(x < y for x, y in zip(test_list, test_list[1:])) diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index f317dbd320..70511caa9d 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -328,4 +328,4 @@ def get_task_names(self): task_names[run].append('epos') if 'gdas' in run else 0 task_names[run] += ['stage_ic', 'ecen', 'esfc', 'earc', 'cleanup'] - return task_names \ No newline at end of file + return task_names diff --git a/workflow/rocoto/tasks.py b/workflow/rocoto/tasks.py index 6cff2d0ea1..514ecd22b6 100644 --- a/workflow/rocoto/tasks.py +++ b/workflow/rocoto/tasks.py @@ -256,4 +256,4 @@ def get_task(self, task_name, *args, **kwargs): except AttributeError: raise AttributeError(f'"{task_name}" is not a valid task.\n' f'Valid tasks are:\n' - f'{", ".join(Tasks.VALID_TASKS)}') \ No newline at end of file + f'{", ".join(Tasks.VALID_TASKS)}') From e05e241efba8a04e7e022c5fc52d926829b1c926 Mon Sep 17 00:00:00 2001 From: Kevin Dougherty <69815622+kevindougherty-noaa@users.noreply.github.com> Date: Tue, 28 Jan 2025 14:01:39 -0500 Subject: [PATCH 28/39] Update jobs/JGLOBAL_ANALYSIS_STATS Co-authored-by: Cory Martin --- jobs/JGLOBAL_ANALYSIS_STATS | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/jobs/JGLOBAL_ANALYSIS_STATS b/jobs/JGLOBAL_ANALYSIS_STATS index cd7ce2e752..4e851c1110 100755 --- a/jobs/JGLOBAL_ANALYSIS_STATS +++ b/jobs/JGLOBAL_ANALYSIS_STATS @@ -13,9 +13,11 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "anlstat" -c "base anlstat" ############################################## # Generate COM variables from templates -YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_OBS COM_CHEM_ANALYSIS - -YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_SNOW_ANALYSIS COM_CONF +YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COMIN_OBS COMOUT_CONF \ + COMIN_ATMOS_ANALYSIS \ + COMIN_MARINE_ANALYSIS \ + COMIN_CHEM_ANALYSIS \ + COMIN_SNOW_ANALYSIS ############################################################### # Run relevant script From 6cfcc48fb969e2a027f307417a01eb92b8a7e022 Mon Sep 17 00:00:00 2001 From: Kevin Dougherty <69815622+kevindougherty-noaa@users.noreply.github.com> Date: Tue, 28 Jan 2025 14:23:14 -0500 Subject: [PATCH 29/39] Update scripts/exglobal_analysis_stats.py Co-authored-by: Cory Martin --- scripts/exglobal_analysis_stats.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/exglobal_analysis_stats.py b/scripts/exglobal_analysis_stats.py index 533b7d47dc..6dc5aa89a4 100755 --- a/scripts/exglobal_analysis_stats.py +++ b/scripts/exglobal_analysis_stats.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # exglobal_stat_analysis.py -# This script creates an StatAnalysis class +# This script creates a StatAnalysis class # and runs the initialize, execute, and finalize # methods which create and stage the runtime directory # and create the YAML configuration From 7d4b1619ead4acc52279a5f2207689d301585cc7 Mon Sep 17 00:00:00 2001 From: Kevin Dougherty <69815622+kevindougherty-noaa@users.noreply.github.com> Date: Tue, 28 Jan 2025 14:23:31 -0500 Subject: [PATCH 30/39] Update scripts/exglobal_analysis_stats.py Co-authored-by: Cory Martin --- scripts/exglobal_analysis_stats.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/exglobal_analysis_stats.py b/scripts/exglobal_analysis_stats.py index 6dc5aa89a4..1039817213 100755 --- a/scripts/exglobal_analysis_stats.py +++ b/scripts/exglobal_analysis_stats.py @@ -5,7 +5,7 @@ # and runs the initialize, execute, and finalize # methods which create and stage the runtime directory # and create the YAML configuration -# for a global stat analysis +# to produce summary statistics from the analysis import os from wxflow import Logger, cast_strdict_as_dtypedict From 388c58b081d7ab396b3193826a29595bc10a2964 Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Tue, 28 Jan 2025 19:50:21 +0000 Subject: [PATCH 31/39] basic updated suggestions i.e. variable changes, EOL, etc. --- jobs/JGLOBAL_ANALYSIS_STATS | 2 +- parm/gdas/anlstat_jedi_config.yaml.j2 | 6 +++--- parm/gdas/stat/snow/jcb-base.yaml.j2 | 2 +- scripts/exglobal_analysis_stats.py | 8 ++++---- ush/python/pygfs/task/stat_analysis.py | 22 ++++++++-------------- 5 files changed, 17 insertions(+), 23 deletions(-) diff --git a/jobs/JGLOBAL_ANALYSIS_STATS b/jobs/JGLOBAL_ANALYSIS_STATS index cd7ce2e752..826c59c271 100755 --- a/jobs/JGLOBAL_ANALYSIS_STATS +++ b/jobs/JGLOBAL_ANALYSIS_STATS @@ -36,4 +36,4 @@ if [[ -e "${pgmout}" ]] ; then cat "${pgmout}" fi -exit 0 \ No newline at end of file +exit 0 diff --git a/parm/gdas/anlstat_jedi_config.yaml.j2 b/parm/gdas/anlstat_jedi_config.yaml.j2 index 0c0fe70dda..b7fa76aa7b 100644 --- a/parm/gdas/anlstat_jedi_config.yaml.j2 +++ b/parm/gdas/anlstat_jedi_config.yaml.j2 @@ -2,7 +2,7 @@ aero: rundir: '{{ DATA }}' exe_src: '{{ JEDIEXE }}' mpi_cmd: '{{ APRUN_ANLSTAT }}' - stat_file_path: '{{ COM_CHEM_ANALYSIS }}' + stat_file_path: '{{ COMIN_CHEM_ANALYSIS }}' stat_file_name: 'aerostat' # jedi_args: None jcb_base_yaml: '{{ PARMgfs }}/gdas/stat/aero/jcb-base.yaml.j2' @@ -12,9 +12,9 @@ snow: rundir: '{{ DATA }}' exe_src: '{{ JEDIEXE }}' mpi_cmd: '{{ APRUN_ANLSTAT }}' - stat_file_path: '{{ COM_SNOW_ANALYSIS }}' + stat_file_path: '{{ COMIN_SNOW_ANALYSIS }}' stat_file_name: 'snowstat.tgz' # jedi_args: None jcb_base_yaml: '{{ PARMgfs }}/gdas/stat/snow/jcb-base.yaml.j2' jcb_algo_yaml: '{{ JCB_ALGO_YAML }}' - jcb_algo: 'anlstat' \ No newline at end of file + jcb_algo: 'anlstat' diff --git a/parm/gdas/stat/snow/jcb-base.yaml.j2 b/parm/gdas/stat/snow/jcb-base.yaml.j2 index 461010d738..780295a239 100644 --- a/parm/gdas/stat/snow/jcb-base.yaml.j2 +++ b/parm/gdas/stat/snow/jcb-base.yaml.j2 @@ -19,4 +19,4 @@ obspaces: {{ OBSPACES_LIST }} # Obspace variable things # ----------------------- -snow_obsdatain_path: "{{ DATA }}" \ No newline at end of file +snow_obsdatain_path: "{{ DATA }}" diff --git a/scripts/exglobal_analysis_stats.py b/scripts/exglobal_analysis_stats.py index 533b7d47dc..a4a3ff55b9 100755 --- a/scripts/exglobal_analysis_stats.py +++ b/scripts/exglobal_analysis_stats.py @@ -24,7 +24,7 @@ StatAnl = StatAnalysis(config) # Create list based on DA components - StatAnl.task_config['STAT_OBS'] = [] + StatAnl.task_config['STAT_ANALYSES'] = [] if StatAnl.task_config.DO_AERO: StatAnl.task_config['STAT_OBS'].append('aero') if StatAnl.task_config.DO_JEDISNOWDA: @@ -32,6 +32,6 @@ # Initialize JEDI variational analysis StatAnl.initialize() - for ob in StatAnl.task_config.STAT_OBS: - StatAnl.execute(ob) - StatAnl.finalize(ob) + for anl in StatAnl.task_config.STAT_OBS: + StatAnl.execute(anl) + StatAnl.finalize(anl) diff --git a/ush/python/pygfs/task/stat_analysis.py b/ush/python/pygfs/task/stat_analysis.py index 2b665a849b..ab2a14c22f 100644 --- a/ush/python/pygfs/task/stat_analysis.py +++ b/ush/python/pygfs/task/stat_analysis.py @@ -42,17 +42,11 @@ def __init__(self, config: Dict[str, Any]): """ super().__init__(config) - _res = int(self.task_config.CASE[1:]) _window_begin = add_to_datetime(self.task_config.current_cycle, -to_timedelta(f"{self.task_config.assim_freq}H") / 2) # Create a local dictionary that is repeatedly used across this class local_dict = AttrDict( { - 'npx_ges': _res + 1, - 'npy_ges': _res + 1, - 'npz_ges': self.task_config.LEVS - 1, - 'npz': self.task_config.LEVS - 1, - 'npz_anl': self.task_config.LEVS - 1, 'STAT_WINDOW_BEGIN': _window_begin, 'STAT_WINDOW_LENGTH': f"PT{self.task_config.assim_freq}H", 'OPREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", @@ -87,18 +81,18 @@ def initialize(self) -> None: logger.info(f"Copying files to {self.task_config.DATA}/stats") # Loop through ob space list - for OB in self.task_config.STAT_OBS: - logger.info(f"Working on current observation: {OB}") + for ANL in self.task_config.STAT_ANALYSES: + logger.info(f"Working on current observation: {ANL}") # Parse JEDI analysis stat jinja file obs_dict = parse_j2yaml(self.task_config.JEDI_CONFIG_YAML, self.task_config) # Copy stat files to DATA path - instat_files = os.path.join(obs_dict[OB]['stat_file_path'], f"{self.task_config['APREFIX']}{obs_dict[OB]['stat_file_name']}") - ob_dir_str = f"{self.task_config.DATA}" + f"/{OB}" + instat_files = os.path.join(obs_dict[ANL]['stat_file_path'], f"{self.task_config['APREFIX']}{obs_dict[ANL]['stat_file_name']}") + ob_dir_str = f"{self.task_config.DATA}" + f"/{ANL}" os.mkdir(ob_dir_str) - dest = os.path.join(ob_dir_str, obs_dict[OB]['stat_file_name']) + dest = os.path.join(ob_dir_str, obs_dict[ANL]['stat_file_name']) logger.info(f"Copying {instat_files} to {dest} ...") statlist = [[instat_files, dest]] FileHandler({'copy': statlist}).sync() @@ -127,7 +121,7 @@ def initialize(self) -> None: obs_space_paths = nc_paths + nc4_paths # Temporary. Create condition check here for available jcb algorithms? - if OB == 'snow': + if ANL == 'snow': obs_space_paths = glob.glob(os.path.join(ob_dir_str, "diag_ims_snow_*.nc")) # This grabs the obspace string from the .nc4 files, however not all are perfect. Need solution. @@ -135,8 +129,8 @@ def initialize(self) -> None: # initialize JEDI application logger.info(f"Initializing JEDI variational DA application") - logger.info(f"{self.jedi_dict[OB]}") - self.jedi_dict[OB].initialize(self.task_config) + logger.info(f"{self.jedi_dict[ANL]}") + self.jedi_dict[ANL].initialize(self.task_config) @logit(logger) def execute(self, jedi_dict_key: str) -> None: From 4055c992d34da9109cc3ac1129ecc14a49436b7c Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Wed, 29 Jan 2025 17:37:45 +0000 Subject: [PATCH 32/39] added checks to python scripts. fixed formatting of job --- jobs/JGLOBAL_ANALYSIS_STATS | 8 ++++---- ush/python/pygfs/task/stat_analysis.py | 16 ++++++++++++++-- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/jobs/JGLOBAL_ANALYSIS_STATS b/jobs/JGLOBAL_ANALYSIS_STATS index dad84bf285..168bc89413 100755 --- a/jobs/JGLOBAL_ANALYSIS_STATS +++ b/jobs/JGLOBAL_ANALYSIS_STATS @@ -14,10 +14,10 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "anlstat" -c "base anlstat" # Generate COM variables from templates YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COMIN_OBS COMOUT_CONF \ - COMIN_ATMOS_ANALYSIS \ - COMIN_MARINE_ANALYSIS \ - COMIN_CHEM_ANALYSIS \ - COMIN_SNOW_ANALYSIS + COMIN_ATMOS_ANALYSIS \ + COMIN_MARINE_ANALYSIS \ + COMIN_CHEM_ANALYSIS \ + COMIN_SNOW_ANALYSIS ############################################################### # Run relevant script diff --git a/ush/python/pygfs/task/stat_analysis.py b/ush/python/pygfs/task/stat_analysis.py index ab2a14c22f..35cfdce965 100644 --- a/ush/python/pygfs/task/stat_analysis.py +++ b/ush/python/pygfs/task/stat_analysis.py @@ -100,12 +100,24 @@ def initialize(self) -> None: # Open tar file logger.info(f"Open tarred stat file in {dest}") with tarfile.open(dest, "r") as tar: + # Check if tar file is empty + if not tar.getnames(): + logger.warning(f"WARNING. The tar file {dest} is empty. No files to extract.") + logger.warning("Moving to next analysis ...") + continue # Skip current analysis and move to next # Extract all files to the current directory tar.extractall(path=f'{ob_dir_str}') # Gunzip .nc files logger.info("Gunzip files from tar file") gz_files = glob.glob(os.path.join(ob_dir_str, "*.gz")) + + # Check if gunzip files exist + if not gz_files: + logger.warning("WARNING. No .gz files to extract.") + logger.warning("Moving to next analysis ...") + continue # Skip current analysis and move to next + logger.info(f"Gunzip files: {gz_files}") for diagfile in gz_files: @@ -150,9 +162,9 @@ def execute(self, jedi_dict_key: str) -> None: @logit(logger) def finalize(self, jedi_dict_key: str) -> None: - """Finalize a statistic analysis + """Finalize the statistic analysis job. - This method will finalize a statistic analysis using JEDI. + This method will finalize the statistic analysis job using JEDI. This includes: - copying stat files to specified outdir From f8f6a7bede25db2c5fdb4556092b3856be0f6392 Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Fri, 14 Feb 2025 15:16:44 +0000 Subject: [PATCH 33/39] update to utilize config file in driver python script --- jobs/JGLOBAL_ANALYSIS_STATS | 11 +-- parm/config/gfs/config.anlstat | 3 +- parm/config/gfs/yaml/stat_base_config.yaml.j2 | 14 ++++ parm/gdas/anlstat_jedi_config.yaml.j2 | 2 + scripts/exglobal_analysis_stats.py | 6 +- ush/python/pygfs/task/stat_analysis.py | 83 +++++++++---------- 6 files changed, 65 insertions(+), 54 deletions(-) create mode 100644 parm/config/gfs/yaml/stat_base_config.yaml.j2 diff --git a/jobs/JGLOBAL_ANALYSIS_STATS b/jobs/JGLOBAL_ANALYSIS_STATS index 168bc89413..9c419e3462 100755 --- a/jobs/JGLOBAL_ANALYSIS_STATS +++ b/jobs/JGLOBAL_ANALYSIS_STATS @@ -13,11 +13,12 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "anlstat" -c "base anlstat" ############################################## # Generate COM variables from templates -YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COMIN_OBS COMOUT_CONF \ - COMIN_ATMOS_ANALYSIS \ - COMIN_MARINE_ANALYSIS \ - COMIN_CHEM_ANALYSIS \ - COMIN_SNOW_ANALYSIS +YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COMIN_OBS:COM_OBS_TMPL \ + COMOUT_CONF:COM_CONF_TMPL \ + COMIN_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL \ + COMIN_OCEAN_ANALYSIS:COM_OCEAN_ANALYSIS_TMPL \ + COMIN_CHEM_ANALYSIS:COM_CHEM_ANALYSIS_TMPL \ + COMIN_SNOW_ANALYSIS:COM_SNOW_ANALYSIS_TMPL ############################################################### # Run relevant script diff --git a/parm/config/gfs/config.anlstat b/parm/config/gfs/config.anlstat index e7831bd565..4a5fbb8d61 100644 --- a/parm/config/gfs/config.anlstat +++ b/parm/config/gfs/config.anlstat @@ -9,9 +9,10 @@ echo "BEGIN: config.anlstat" source "${EXPDIR}/config.resources" anlstat export JEDI_CONFIG_YAML="${PARMgfs}/gdas/anlstat_jedi_config.yaml.j2" +export STAT_BASE_CONFIG_YAML="${PARMgfs}/config/gfs/yaml/stat_base_config.yaml.j2" export JCB_BASE_YAML="${PARMgfs}/gdas/stat/aero/jcb-base.yaml.j2" export JCB_ALGO_YAML="${PARMgfs}/gdas/jcb-algorithms/anlstat.yaml.j2" export JEDIEXE=${HOMEgfs}/sorc/gdas.cd/build/bin/ioda-stats.x export STAT_OUTDIR="${COMOUT}/${net}/${run.yyyymmdd}/${hh}/products/stats/" -echo "END: config.anlstat" \ No newline at end of file +echo "END: config.anlstat" diff --git a/parm/config/gfs/yaml/stat_base_config.yaml.j2 b/parm/config/gfs/yaml/stat_base_config.yaml.j2 new file mode 100644 index 0000000000..4cbfaad5d0 --- /dev/null +++ b/parm/config/gfs/yaml/stat_base_config.yaml.j2 @@ -0,0 +1,14 @@ +aero: + ob spaces: + - name: viirs_npp + input file: "diag_viirs_npp_{{ current_cycle | to_YMDH }}.nc4" + output file: "viirs_npp_{{ current_cycle | to_YMDH }}_output_aod.nc" + - name: viirs_n20 + input file: "diag_viirs_n20_{{ current_cycle | to_YMDH }}.nc4" + output file: "viirs_n20_{{ current_cycle | to_YMDH }}_output_aod.nc" + +snow: + ob spaces: + - name: ims_snow + input file: "diag_ims_snow_{{ current_cycle | to_YMDH }}.nc" + output file: "ims_snow_{{ current_cycle | to_YMDH }}_output_snow.nc" diff --git a/parm/gdas/anlstat_jedi_config.yaml.j2 b/parm/gdas/anlstat_jedi_config.yaml.j2 index b7fa76aa7b..2962f37c01 100644 --- a/parm/gdas/anlstat_jedi_config.yaml.j2 +++ b/parm/gdas/anlstat_jedi_config.yaml.j2 @@ -8,6 +8,7 @@ aero: jcb_base_yaml: '{{ PARMgfs }}/gdas/stat/aero/jcb-base.yaml.j2' jcb_algo_yaml: '{{ JCB_ALGO_YAML }}' jcb_algo: 'anlstat' + base_config: '{{ STAT_BASE_CONFIG_YAML }}' snow: rundir: '{{ DATA }}' exe_src: '{{ JEDIEXE }}' @@ -18,3 +19,4 @@ snow: jcb_base_yaml: '{{ PARMgfs }}/gdas/stat/snow/jcb-base.yaml.j2' jcb_algo_yaml: '{{ JCB_ALGO_YAML }}' jcb_algo: 'anlstat' + base_config: '{{ STAT_BASE_CONFIG_YAML }}' diff --git a/scripts/exglobal_analysis_stats.py b/scripts/exglobal_analysis_stats.py index 52415bebb7..a44194862c 100755 --- a/scripts/exglobal_analysis_stats.py +++ b/scripts/exglobal_analysis_stats.py @@ -26,12 +26,12 @@ # Create list based on DA components StatAnl.task_config['STAT_ANALYSES'] = [] if StatAnl.task_config.DO_AERO: - StatAnl.task_config['STAT_OBS'].append('aero') + StatAnl.task_config['STAT_ANALYSES'].append('aero') if StatAnl.task_config.DO_JEDISNOWDA: - StatAnl.task_config['STAT_OBS'].append('snow') + StatAnl.task_config['STAT_ANALYSES'].append('snow') # Initialize JEDI variational analysis StatAnl.initialize() - for anl in StatAnl.task_config.STAT_OBS: + for anl in StatAnl.task_config.STAT_ANALYSES: StatAnl.execute(anl) StatAnl.finalize(anl) diff --git a/ush/python/pygfs/task/stat_analysis.py b/ush/python/pygfs/task/stat_analysis.py index 35cfdce965..76960ef978 100644 --- a/ush/python/pygfs/task/stat_analysis.py +++ b/ush/python/pygfs/task/stat_analysis.py @@ -80,19 +80,19 @@ def initialize(self) -> None: logger.info(f"Copying files to {self.task_config.DATA}/stats") - # Loop through ob space list - for ANL in self.task_config.STAT_ANALYSES: - logger.info(f"Working on current observation: {ANL}") + # Parse JEDI analysis stat jinja file + self.obs_dict = parse_j2yaml(self.task_config.JEDI_CONFIG_YAML, self.task_config) - # Parse JEDI analysis stat jinja file - obs_dict = parse_j2yaml(self.task_config.JEDI_CONFIG_YAML, self.task_config) + # Loop through a copy of ob space list + for ANL in self.task_config.STAT_ANALYSES[:]: + logger.info(f"Working on current observation: {ANL}") # Copy stat files to DATA path - instat_files = os.path.join(obs_dict[ANL]['stat_file_path'], f"{self.task_config['APREFIX']}{obs_dict[ANL]['stat_file_name']}") + instat_files = os.path.join(self.obs_dict[ANL]['stat_file_path'], f"{self.task_config['APREFIX']}{self.obs_dict[ANL]['stat_file_name']}") ob_dir_str = f"{self.task_config.DATA}" + f"/{ANL}" os.mkdir(ob_dir_str) - dest = os.path.join(ob_dir_str, obs_dict[ANL]['stat_file_name']) + dest = os.path.join(ob_dir_str, self.obs_dict[ANL]['stat_file_name']) logger.info(f"Copying {instat_files} to {dest} ...") statlist = [[instat_files, dest]] FileHandler({'copy': statlist}).sync() @@ -104,44 +104,38 @@ def initialize(self) -> None: if not tar.getnames(): logger.warning(f"WARNING. The tar file {dest} is empty. No files to extract.") logger.warning("Moving to next analysis ...") - continue # Skip current analysis and move to next + # Remove ANL from STAT_ANALYSES and move to next + self.task_config.STAT_ANALYSES.remove(ANL) + logger.info(f"current analysis list: {self.task_config.STAT_ANALYSES}") + continue # Extract all files to the current directory tar.extractall(path=f'{ob_dir_str}') - # Gunzip .nc files - logger.info("Gunzip files from tar file") - gz_files = glob.glob(os.path.join(ob_dir_str, "*.gz")) - - # Check if gunzip files exist - if not gz_files: - logger.warning("WARNING. No .gz files to extract.") - logger.warning("Moving to next analysis ...") - continue # Skip current analysis and move to next - - logger.info(f"Gunzip files: {gz_files}") - - for diagfile in gz_files: - output_file = os.path.join(ob_dir_str, os.path.basename(diagfile)[:-3]) - with gzip.open(diagfile, 'rb') as f_in: - with open(output_file, 'wb') as f_out: - f_out.write(f_in.read()) - - # Get list of .nc4 files - # obs_space_paths = glob.glob(os.path.join(ob_dir_str, "*.{nc,nc4}")) # THIS SHOULD WORK BUT ISNT, glob patterns introduced in Python 3.9 - nc_paths = glob.glob(os.path.join(ob_dir_str, "*.nc")) - nc4_paths = glob.glob(os.path.join(ob_dir_str, "*.nc4")) - obs_space_paths = nc_paths + nc4_paths - - # Temporary. Create condition check here for available jcb algorithms? - if ANL == 'snow': - obs_space_paths = glob.glob(os.path.join(ob_dir_str, "diag_ims_snow_*.nc")) - - # This grabs the obspace string from the .nc4 files, however not all are perfect. Need solution. - self.task_config.OBSPACES_LIST = ['_'.join(os.path.basename(path).split('_')[1:3]) for path in obs_space_paths] + # Extract info from stat config file + analysis_config_dict = parse_j2yaml(self.obs_dict[ANL]['base_config'], self.task_config) + + self.task_config.OBSPACES_LIST = [] + for analysis_dict in analysis_config_dict[ANL]['ob spaces']: + # Gunzip .nc files + logger.info("Gunzip files from tar file") + gz_file = os.path.join(ob_dir_str, (analysis_dict['input file'] + ".gz")) + + # Check if the file exists + if os.path.exists(gz_file): + output_file = os.path.join(ob_dir_str, analysis_dict['input file']) + # Open the .gz file + with gzip.open(gz_file, 'rb') as f_in: + with open(output_file, 'wb') as f_out: + f_out.write(f_in.read()) + else: + logger.warning("WARNING. No .gz files to extract.") + logger.warning("Moving to next analysis ...") + continue # Skip current analysis and move to next + self.task_config.OBSPACES_LIST.append(analysis_dict['name']) + # initialize JEDI application logger.info(f"Initializing JEDI variational DA application") - logger.info(f"{self.jedi_dict[ANL]}") self.jedi_dict[ANL].initialize(self.task_config) @logit(logger) @@ -157,7 +151,7 @@ def execute(self, jedi_dict_key: str) -> None: ---------- None """ - logger.info(f"In execute. {self.jedi_dict[jedi_dict_key]}") + self.jedi_dict[jedi_dict_key].execute() @logit(logger) @@ -178,12 +172,11 @@ def finalize(self, jedi_dict_key: str) -> None: None """ - # get list of output diag files - diags = glob.glob(os.path.join(self.task_config.DATA, '*output_*.nc')) + analysis_config_dict = parse_j2yaml(self.obs_dict[jedi_dict_key]['base_config'], self.task_config) - for diagfile in diags: - outfile = os.path.basename(diagfile) - dest = os.path.join(f'{self.task_config.STAT_OUTDIR}/{jedi_dict_key}/', f'{outfile}') + for analysis_dict in analysis_config_dict[jedi_dict_key]['ob spaces']: + diagfile = os.path.join(self.task_config.DATA, analysis_dict['output file']) + dest = os.path.join(f'{self.task_config.STAT_OUTDIR}/{jedi_dict_key}/', f"{analysis_dict['output file']}") logger.debug(f"copying {diagfile} to {dest}") diag_copy = { 'copy': [[diagfile, dest]] From b53a9d8be6008f16273ccce639f957e26b0549a4 Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Fri, 14 Feb 2025 15:18:04 +0000 Subject: [PATCH 34/39] pycodestyle --- ush/python/pygfs/task/stat_analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/stat_analysis.py b/ush/python/pygfs/task/stat_analysis.py index 76960ef978..114134167b 100644 --- a/ush/python/pygfs/task/stat_analysis.py +++ b/ush/python/pygfs/task/stat_analysis.py @@ -133,7 +133,7 @@ def initialize(self) -> None: continue # Skip current analysis and move to next self.task_config.OBSPACES_LIST.append(analysis_dict['name']) - + # initialize JEDI application logger.info(f"Initializing JEDI variational DA application") self.jedi_dict[ANL].initialize(self.task_config) From f75140c78abcec81d568241a7c9cd314ac745864 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Fri, 14 Feb 2025 20:07:54 +0000 Subject: [PATCH 35/39] remove STAT_OUTDIR from shell, need to still do it in py --- jobs/JGLOBAL_ANALYSIS_STATS | 8 ++++++-- parm/config/gfs/config.anlstat | 1 - parm/config/gfs/config.com | 5 +++++ 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/jobs/JGLOBAL_ANALYSIS_STATS b/jobs/JGLOBAL_ANALYSIS_STATS index 9c419e3462..43db184664 100755 --- a/jobs/JGLOBAL_ANALYSIS_STATS +++ b/jobs/JGLOBAL_ANALYSIS_STATS @@ -14,11 +14,15 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "anlstat" -c "base anlstat" # Generate COM variables from templates YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COMIN_OBS:COM_OBS_TMPL \ - COMOUT_CONF:COM_CONF_TMPL \ COMIN_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL \ COMIN_OCEAN_ANALYSIS:COM_OCEAN_ANALYSIS_TMPL \ COMIN_CHEM_ANALYSIS:COM_CHEM_ANALYSIS_TMPL \ - COMIN_SNOW_ANALYSIS:COM_SNOW_ANALYSIS_TMPL + COMIN_SNOW_ANALYSIS:COM_SNOW_ANALYSIS_TMPL \ + COMOUT_CONF:COM_CONF_TMPL \ + COMOUT_ATMOS_ANLMON_TMPL:COM_ATMOS_ANLMON_TMPL \ + COMOUT_OCEAN_ANLMON_TMPL:COM_OCEAN_ANLMON_TMPL \ + COMOUT_CHEM_ANLMON_TMPL:COM_CHEM_ANLMON_TMPL \ + COMOUT_SNOW_ANLMON_TMPL:COM_SNOW_ANLMON_TMPL ############################################################### # Run relevant script diff --git a/parm/config/gfs/config.anlstat b/parm/config/gfs/config.anlstat index 4a5fbb8d61..01da7ea2f0 100644 --- a/parm/config/gfs/config.anlstat +++ b/parm/config/gfs/config.anlstat @@ -13,6 +13,5 @@ export STAT_BASE_CONFIG_YAML="${PARMgfs}/config/gfs/yaml/stat_base_config.yaml.j export JCB_BASE_YAML="${PARMgfs}/gdas/stat/aero/jcb-base.yaml.j2" export JCB_ALGO_YAML="${PARMgfs}/gdas/jcb-algorithms/anlstat.yaml.j2" export JEDIEXE=${HOMEgfs}/sorc/gdas.cd/build/bin/ioda-stats.x -export STAT_OUTDIR="${COMOUT}/${net}/${run.yyyymmdd}/${hh}/products/stats/" echo "END: config.anlstat" diff --git a/parm/config/gfs/config.com b/parm/config/gfs/config.com index 8b6da376f3..54acdbccd7 100644 --- a/parm/config/gfs/config.com +++ b/parm/config/gfs/config.com @@ -55,6 +55,7 @@ declare -rx COM_ATMOS_INPUT_TMPL=${COM_BASE}'/model/atmos/input' declare -rx COM_ATMOS_RESTART_TMPL=${COM_BASE}'/model/atmos/restart' declare -rx COM_ATMOS_ANALYSIS_TMPL=${COM_BASE}'/analysis/atmos' declare -rx COM_SNOW_ANALYSIS_TMPL=${COM_BASE}'/analysis/snow' +declare -rx COM_SNOW_ANLMON_TMPL=${COM_BASE}'/products/snow/anlmon' declare -rx COM_ATMOS_HISTORY_TMPL=${COM_BASE}'/model/atmos/history' declare -rx COM_ATMOS_MASTER_TMPL=${COM_BASE}'/model/atmos/master' declare -rx COM_ATMOS_GRIB_TMPL=${COM_BASE}'/products/atmos/grib2' @@ -68,6 +69,7 @@ declare -rx COM_ATMOS_IMAGERY_TMPL=${COM_BASE}'/products/atmos/imagery' declare -rx COM_ATMOS_OZNMON_TMPL=${COM_BASE}'/products/atmos/oznmon' declare -rx COM_ATMOS_RADMON_TMPL=${COM_BASE}'/products/atmos/radmon' declare -rx COM_ATMOS_MINMON_TMPL=${COM_BASE}'/products/atmos/minmon' +declare -rx COM_ATMOS_ANLMON_TMPL=${COM_BASE}'/products/atmos/anlmon' declare -rx COM_ATMOS_WMO_TMPL=${COM_BASE}'/products/atmos/wmo' declare -rx COM_WAVE_RESTART_TMPL=${COM_BASE}'/model/wave/restart' @@ -83,6 +85,7 @@ declare -rx COM_OCEAN_HISTORY_TMPL=${COM_BASE}'/model/ocean/history' declare -rx COM_OCEAN_RESTART_TMPL=${COM_BASE}'/model/ocean/restart' declare -rx COM_OCEAN_INPUT_TMPL=${COM_BASE}'/model/ocean/input' declare -rx COM_OCEAN_ANALYSIS_TMPL=${COM_BASE}'/analysis/ocean' +declare -rx COM_OCEAN_ANLMON_TMPL=${COM_BASE}'/products/ocean/anlmon' declare -rx COM_OCEAN_LETKF_TMPL=${COM_BASE}'/analysis/ocean/letkf' declare -rx COM_OCEAN_BMATRIX_TMPL=${COM_BASE}'/bmatrix/ocean' declare -rx COM_OCEAN_NETCDF_TMPL=${COM_BASE}'/products/ocean/netcdf' @@ -91,6 +94,7 @@ declare -rx COM_OCEAN_GRIB_GRID_TMPL=${COM_OCEAN_GRIB_TMPL}'/${GRID}' declare -rx COM_ICE_ANALYSIS_TMPL=${COM_BASE}'/analysis/ice' declare -rx COM_ICE_LETKF_TMPL=${COM_BASE}'/analysis/ice/letkf' +declare -rx COM_ICE_ANLMON_TMPL=${COM_BASE}'/products/ice/anlmon' declare -rx COM_ICE_BMATRIX_TMPL=${COM_BASE}'/bmatrix/ice' declare -rx COM_ICE_INPUT_TMPL=${COM_BASE}'/model/ice/input' declare -rx COM_ICE_HISTORY_TMPL=${COM_BASE}'/model/ice/history' @@ -102,5 +106,6 @@ declare -rx COM_ICE_GRIB_GRID_TMPL=${COM_ICE_GRIB_TMPL}'/${GRID}' declare -rx COM_CHEM_HISTORY_TMPL=${COM_BASE}'/model/chem/history' declare -rx COM_CHEM_ANALYSIS_TMPL=${COM_BASE}'/analysis/chem' declare -rx COM_CHEM_BMAT_TMPL=${COM_CHEM_ANALYSIS_TMPL}'/bmatrix' +declare -rx COM_CHEM_ANLMON_TMPL=${COM_BASE}'/products/chem/anlmon' declare -rx COM_MED_RESTART_TMPL=${COM_BASE}'/model/med/restart' From 06763148f1172109d92aa6e47e2463f3da07e827 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Fri, 14 Feb 2025 20:31:29 +0000 Subject: [PATCH 36/39] use different names for the output directory --- jobs/JGLOBAL_ANALYSIS_STATS | 8 ++++---- ush/python/pygfs/task/stat_analysis.py | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/jobs/JGLOBAL_ANALYSIS_STATS b/jobs/JGLOBAL_ANALYSIS_STATS index 43db184664..590940e68d 100755 --- a/jobs/JGLOBAL_ANALYSIS_STATS +++ b/jobs/JGLOBAL_ANALYSIS_STATS @@ -19,10 +19,10 @@ YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COMIN_OBS:COM_OBS_TMPL \ COMIN_CHEM_ANALYSIS:COM_CHEM_ANALYSIS_TMPL \ COMIN_SNOW_ANALYSIS:COM_SNOW_ANALYSIS_TMPL \ COMOUT_CONF:COM_CONF_TMPL \ - COMOUT_ATMOS_ANLMON_TMPL:COM_ATMOS_ANLMON_TMPL \ - COMOUT_OCEAN_ANLMON_TMPL:COM_OCEAN_ANLMON_TMPL \ - COMOUT_CHEM_ANLMON_TMPL:COM_CHEM_ANLMON_TMPL \ - COMOUT_SNOW_ANLMON_TMPL:COM_SNOW_ANLMON_TMPL + COMOUT_ATMOS_ANLMON:COM_ATMOS_ANLMON_TMPL \ + COMOUT_OCEAN_ANLMON:COM_OCEAN_ANLMON_TMPL \ + COMOUT_CHEM_ANLMON:COM_CHEM_ANLMON_TMPL \ + COMOUT_SNOW_ANLMON:COM_SNOW_ANLMON_TMPL ############################################################### # Run relevant script diff --git a/ush/python/pygfs/task/stat_analysis.py b/ush/python/pygfs/task/stat_analysis.py index 114134167b..94d2a654f9 100644 --- a/ush/python/pygfs/task/stat_analysis.py +++ b/ush/python/pygfs/task/stat_analysis.py @@ -176,7 +176,8 @@ def finalize(self, jedi_dict_key: str) -> None: for analysis_dict in analysis_config_dict[jedi_dict_key]['ob spaces']: diagfile = os.path.join(self.task_config.DATA, analysis_dict['output file']) - dest = os.path.join(f'{self.task_config.STAT_OUTDIR}/{jedi_dict_key}/', f"{analysis_dict['output file']}") + outdir = self.task_config['COMOUT_' + jedi_dict_key.upper() + '_ANLMON'] + dest = os.path.join(outdir, f"{analysis_dict['output file']}") logger.debug(f"copying {diagfile} to {dest}") diag_copy = { 'copy': [[diagfile, dest]] From d8994cde36023e7001cafefbc0195b1d87224843 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Fri, 14 Feb 2025 20:33:15 +0000 Subject: [PATCH 37/39] a todo note --- ush/python/pygfs/task/stat_analysis.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ush/python/pygfs/task/stat_analysis.py b/ush/python/pygfs/task/stat_analysis.py index 94d2a654f9..7419ff8d44 100644 --- a/ush/python/pygfs/task/stat_analysis.py +++ b/ush/python/pygfs/task/stat_analysis.py @@ -177,6 +177,7 @@ def finalize(self, jedi_dict_key: str) -> None: for analysis_dict in analysis_config_dict[jedi_dict_key]['ob spaces']: diagfile = os.path.join(self.task_config.DATA, analysis_dict['output file']) outdir = self.task_config['COMOUT_' + jedi_dict_key.upper() + '_ANLMON'] + # TODO need to create outdir if it does not exist, can do that here or in the j-job dest = os.path.join(outdir, f"{analysis_dict['output file']}") logger.debug(f"copying {diagfile} to {dest}") diag_copy = { From e1f04f3e8df001b0f993a071832ad624f9a12925 Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Wed, 19 Feb 2025 18:20:40 +0000 Subject: [PATCH 38/39] minor change and add outdir check --- scripts/exglobal_analysis_stats.py | 2 +- ush/python/pygfs/task/stat_analysis.py | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/scripts/exglobal_analysis_stats.py b/scripts/exglobal_analysis_stats.py index a44194862c..f4c198999e 100755 --- a/scripts/exglobal_analysis_stats.py +++ b/scripts/exglobal_analysis_stats.py @@ -25,7 +25,7 @@ # Create list based on DA components StatAnl.task_config['STAT_ANALYSES'] = [] - if StatAnl.task_config.DO_AERO: + if StatAnl.task_config.DO_AERO_ANL: StatAnl.task_config['STAT_ANALYSES'].append('aero') if StatAnl.task_config.DO_JEDISNOWDA: StatAnl.task_config['STAT_ANALYSES'].append('snow') diff --git a/ush/python/pygfs/task/stat_analysis.py b/ush/python/pygfs/task/stat_analysis.py index 7419ff8d44..9f02f6b142 100644 --- a/ush/python/pygfs/task/stat_analysis.py +++ b/ush/python/pygfs/task/stat_analysis.py @@ -177,7 +177,12 @@ def finalize(self, jedi_dict_key: str) -> None: for analysis_dict in analysis_config_dict[jedi_dict_key]['ob spaces']: diagfile = os.path.join(self.task_config.DATA, analysis_dict['output file']) outdir = self.task_config['COMOUT_' + jedi_dict_key.upper() + '_ANLMON'] - # TODO need to create outdir if it does not exist, can do that here or in the j-job + + # Check if the directory exists; if not, create it + if not os.path.exists(outdir): + os.makedirs(outdir) + logger.info(f"Created directory: {outdir}") + dest = os.path.join(outdir, f"{analysis_dict['output file']}") logger.debug(f"copying {diagfile} to {dest}") diag_copy = { From 54c6946434d8e217a2a4790397f70744a1407436 Mon Sep 17 00:00:00 2001 From: Kevin Dougherty Date: Tue, 25 Feb 2025 14:22:49 +0000 Subject: [PATCH 39/39] minor updates with name changes and new aero tar file --- jobs/JGLOBAL_ANALYSIS_STATS | 4 ++-- parm/config/gfs/yaml/stat_base_config.yaml.j2 | 4 ++-- parm/gdas/anlstat_jedi_config.yaml.j2 | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/jobs/JGLOBAL_ANALYSIS_STATS b/jobs/JGLOBAL_ANALYSIS_STATS index 590940e68d..98d6262677 100755 --- a/jobs/JGLOBAL_ANALYSIS_STATS +++ b/jobs/JGLOBAL_ANALYSIS_STATS @@ -16,12 +16,12 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "anlstat" -c "base anlstat" YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COMIN_OBS:COM_OBS_TMPL \ COMIN_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL \ COMIN_OCEAN_ANALYSIS:COM_OCEAN_ANALYSIS_TMPL \ - COMIN_CHEM_ANALYSIS:COM_CHEM_ANALYSIS_TMPL \ + COMIN_AERO_ANALYSIS:COM_CHEM_ANALYSIS_TMPL \ COMIN_SNOW_ANALYSIS:COM_SNOW_ANALYSIS_TMPL \ COMOUT_CONF:COM_CONF_TMPL \ COMOUT_ATMOS_ANLMON:COM_ATMOS_ANLMON_TMPL \ COMOUT_OCEAN_ANLMON:COM_OCEAN_ANLMON_TMPL \ - COMOUT_CHEM_ANLMON:COM_CHEM_ANLMON_TMPL \ + COMOUT_AERO_ANLMON:COM_CHEM_ANLMON_TMPL \ COMOUT_SNOW_ANLMON:COM_SNOW_ANLMON_TMPL ############################################################### diff --git a/parm/config/gfs/yaml/stat_base_config.yaml.j2 b/parm/config/gfs/yaml/stat_base_config.yaml.j2 index 4cbfaad5d0..28d0198dd8 100644 --- a/parm/config/gfs/yaml/stat_base_config.yaml.j2 +++ b/parm/config/gfs/yaml/stat_base_config.yaml.j2 @@ -1,10 +1,10 @@ aero: ob spaces: - name: viirs_npp - input file: "diag_viirs_npp_{{ current_cycle | to_YMDH }}.nc4" + input file: "diag_viirs_npp_aod_{{ current_cycle | to_YMDH }}.nc" output file: "viirs_npp_{{ current_cycle | to_YMDH }}_output_aod.nc" - name: viirs_n20 - input file: "diag_viirs_n20_{{ current_cycle | to_YMDH }}.nc4" + input file: "diag_viirs_n20_aod_{{ current_cycle | to_YMDH }}.nc" output file: "viirs_n20_{{ current_cycle | to_YMDH }}_output_aod.nc" snow: diff --git a/parm/gdas/anlstat_jedi_config.yaml.j2 b/parm/gdas/anlstat_jedi_config.yaml.j2 index 2962f37c01..db7c177715 100644 --- a/parm/gdas/anlstat_jedi_config.yaml.j2 +++ b/parm/gdas/anlstat_jedi_config.yaml.j2 @@ -2,8 +2,8 @@ aero: rundir: '{{ DATA }}' exe_src: '{{ JEDIEXE }}' mpi_cmd: '{{ APRUN_ANLSTAT }}' - stat_file_path: '{{ COMIN_CHEM_ANALYSIS }}' - stat_file_name: 'aerostat' + stat_file_path: '{{ COMIN_AERO_ANALYSIS }}' + stat_file_name: 'aerostat.tgz' # jedi_args: None jcb_base_yaml: '{{ PARMgfs }}/gdas/stat/aero/jcb-base.yaml.j2' jcb_algo_yaml: '{{ JCB_ALGO_YAML }}'