Stat Analysis Implementation for global-workflow

env/HERA.env

@@ -105,6 +105,11 @@ elif [[ "${step}" = "atmanlfv3inc" ]]; then
     export NTHREADS_ATMANLFV3INC=${NTHREADSmax}
     export APRUN_ATMANLFV3INC="${APRUN_default} --cpus-per-task=${NTHREADS_ATMANLFV3INC}"
 
+elif [[ "${step}" = "anlstat" ]]; then
+
+    export NTHREADS_ANLSTAT=${NTHREADSmax}
+    export APRUN_ANLSTAT="${APRUN_default} --cpus-per-task=${NTHREADS_ANLSTAT}"
+
 elif [[ "${step}" = "prepobsaero" ]]; then
 
     export NTHREADS_PREPOBSAERO=${NTHREADS1}

jobs/JGLOBAL_ANALYSIS_STATS

@@ -0,0 +1,42 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+source "${HOMEgfs}/ush/jjob_header.sh" -e "anlstat" -c "base anlstat"
+
+##############################################
+# Set variables used in the script
+##############################################
+
+
+##############################################
+# Begin JOB SPECIFIC work
+##############################################
+
+# Generate COM variables from templates
+YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COMIN_OBS:COM_OBS_TMPL \
+                                           COMOUT_CONF:COM_CONF_TMPL \
+                                           COMIN_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL \
+                                           COMIN_OCEAN_ANALYSIS:COM_OCEAN_ANALYSIS_TMPL \
+                                           COMIN_CHEM_ANALYSIS:COM_CHEM_ANALYSIS_TMPL \
+                                           COMIN_SNOW_ANALYSIS:COM_SNOW_ANALYSIS_TMPL
+
+###############################################################
+# Run relevant script
+
+EXSCRIPT=${ANLSTATSPY:-${SCRgfs}/exglobal_analysis_stats.py}
+${EXSCRIPT}
+status=$?
+[[ ${status} -ne 0 ]] && exit "${status}"
+
+##############################################
+# End JOB SPECIFIC work
+##############################################
+
+##############################################
+# Final processing
+##############################################
+if [[ -e "${pgmout}" ]] ; then
+  cat "${pgmout}"
+fi
+
+exit 0

jobs/rocoto/anlstat.sh

@@ -0,0 +1,18 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+
+###############################################################
+# Source UFSDA workflow modules
+. "${HOMEgfs}/ush/load_ufsda_modules.sh"
+status=$?
+[[ ${status} -ne 0 ]] && exit "${status}"
+
+export job="anlstat"
+export jobid="${job}.$$"
+
+###############################################################
+# Execute the JJOB
+"${HOMEgfs}/jobs/JGLOBAL_ANALYSIS_STATS"
+status=$?
+exit "${status}"

parm/config/gfs/config.anlstat

@@ -0,0 +1,18 @@
+#!/bin/bash -x
+
+########## config.anlstat ##########
+# Analysis Stat
+
+echo "BEGIN: config.anlstat"
+
+# Get task specific resources
+source "${EXPDIR}/config.resources" anlstat
+
+export JEDI_CONFIG_YAML="${PARMgfs}/gdas/anlstat_jedi_config.yaml.j2"
+export STAT_BASE_CONFIG_YAML="${PARMgfs}/config/gfs/yaml/stat_base_config.yaml.j2"
+export JCB_BASE_YAML="${PARMgfs}/gdas/stat/aero/jcb-base.yaml.j2"
+export JCB_ALGO_YAML="${PARMgfs}/gdas/jcb-algorithms/anlstat.yaml.j2"
+export JEDIEXE=${HOMEgfs}/sorc/gdas.cd/build/bin/ioda-stats.x
+export STAT_OUTDIR="${COMOUT}/${net}/${run.yyyymmdd}/${hh}/products/stats/"
+
+echo "END: config.anlstat"

parm/config/gfs/config.base

@@ -495,6 +495,14 @@ if [[ ${DO_JEDIATMVAR} = "YES" ]]; then
    export DO_VERFOZN="NO"     # Ozone data assimilation monitoring
    export DO_VERFRAD="NO"     # Radiance data assimilation monitoring
    export DO_VMINMON="NO"     # GSI minimization monitoring
+   export DO_ANLSTAT="YES"    # JEDI-based analysis statistics
+else
+   export DO_VERFOZN="YES"    # Ozone data assimilation monitoring
+   export DO_VERFRAD="YES"    # Radiance data assimilation monitoring
+   export DO_VMINMON="YES"    # GSI minimization monitoring
+   if [[ ${DO_AERO} = "YES" || ${DO_JEDIOCNVAR} = "YES" || ${DO_JEDISNOWDA} = "YES " ]]; then
+     export DO_ANLSTAT="YES"  # JEDI-based analysis statistics
+   fi
 fi
 
 # If starting ICs that are not at cycle hour

parm/config/gfs/config.resources

@@ -17,7 +17,7 @@ if (( $# != 1 )); then
     echo "atmensanlinit atmensanlobs atmensanlsol atmensanlletkf atmensanlfv3inc atmensanlfinal"
     echo "snowanl esnowanl"
     echo "prepobsaero aeroanlinit aeroanlvar aeroanlfinal aeroanlgenb"
-    echo "anal sfcanl analcalc analdiag fcst echgres"
+    echo "anal sfcanl analcalc analdiag anlstat fcst echgres"
     echo "upp atmos_products"
     echo "tracker genesis genesis_fsu"
     echo "verfozn verfrad vminmon fit2obs metp arch cleanup"
@@ -717,6 +717,14 @@ case ${step} in
     memory="48GB"
     ;;
 
+  "anlstat")
+    walltime="00:30:00"
+    ntasks=1
+    threads_per_task=1
+    tasks_per_node=$(( max_tasks_per_node / threads_per_task ))
+    memory="24GB"
+    ;;
+
   "sfcanl")
     walltime="00:20:00"
     ntasks=${ntiles:-6}

parm/config/gfs/yaml/stat_base_config.yaml.j2

@@ -0,0 +1,14 @@
+aero:
+  ob spaces:
+  - name: viirs_npp
+    input file: "diag_viirs_npp_{{ current_cycle | to_YMDH }}.nc4"
+    output file: "viirs_npp_{{ current_cycle | to_YMDH }}_output_aod.nc"
+  - name: viirs_n20
+    input file: "diag_viirs_n20_{{ current_cycle | to_YMDH }}.nc4"
+    output file: "viirs_n20_{{ current_cycle | to_YMDH }}_output_aod.nc"
+
+snow:
+  ob spaces:
+  - name: ims_snow
+    input file: "diag_ims_snow_{{ current_cycle | to_YMDH }}.nc"
+    output file: "ims_snow_{{ current_cycle | to_YMDH }}_output_snow.nc"

parm/gdas/anlstat_jedi_config.yaml.j2

@@ -0,0 +1,22 @@
+aero:
+  rundir: '{{ DATA }}'
+  exe_src: '{{ JEDIEXE }}'
+  mpi_cmd: '{{ APRUN_ANLSTAT }}'
+  stat_file_path: '{{ COMIN_CHEM_ANALYSIS }}'
+  stat_file_name: 'aerostat'
+  # jedi_args: None
+  jcb_base_yaml: '{{ PARMgfs }}/gdas/stat/aero/jcb-base.yaml.j2'
+  jcb_algo_yaml: '{{ JCB_ALGO_YAML }}'
+  jcb_algo: 'anlstat'
+  base_config: '{{ STAT_BASE_CONFIG_YAML }}'
+snow:
+  rundir: '{{ DATA }}'
+  exe_src: '{{ JEDIEXE }}'
+  mpi_cmd: '{{ APRUN_ANLSTAT }}'
+  stat_file_path: '{{ COMIN_SNOW_ANALYSIS }}'
+  stat_file_name: 'snowstat.tgz'
+  # jedi_args: None
+  jcb_base_yaml: '{{ PARMgfs }}/gdas/stat/snow/jcb-base.yaml.j2'
+  jcb_algo_yaml: '{{ JCB_ALGO_YAML }}'
+  jcb_algo: 'anlstat'
+  base_config: '{{ STAT_BASE_CONFIG_YAML }}'

parm/gdas/stat/aero/jcb-base.yaml.j2

@@ -0,0 +1,22 @@
+# Search path for model and obs for JCB
+# -------------------------------------
+algorithm_path: "{{PARMgfs}}/gdas/jcb-algorithms"
+app_path_algorithm: "{{PARMgfs}}/gdas/jcb-gdas/algorithm/obstats/aero"
+
+# Assimilation window
+# -------------------
+window_begin: "{{ STAT_WINDOW_BEGIN | to_isotime }}"
+window_YMDH: "{{ STAT_WINDOW_BEGIN | to_YMDH }}"
+window_length: "{{ STAT_WINDOW_LENGTH }}"
+bound_to_include: begin
+
+stat_current_cycle_iso: "{{ current_cycle | to_isotime }}"
+stat_current_cycle_YMDH: "{{ current_cycle | to_YMDH }}"
+
+# Inputted list of ob spaces
+# --------------------------
+obspaces: {{ OBSPACES_LIST }}
+
+# Obspace variable things
+# -----------------------
+aero_obsdatain_path: "{{ DATA }}"

parm/gdas/stat/snow/jcb-base.yaml.j2

@@ -0,0 +1,22 @@
+# Search path for model and obs for JCB
+# -------------------------------------
+algorithm_path: "{{PARMgfs}}/gdas/jcb-algorithms"
+app_path_algorithm: "{{PARMgfs}}/gdas/jcb-gdas/algorithm/obstats/snow"
+
+# Assimilation window
+# -------------------
+window_begin: "{{ STAT_WINDOW_BEGIN | to_isotime }}"
+window_YMDH: "{{ STAT_WINDOW_BEGIN | to_YMDH }}"
+window_length: "{{ STAT_WINDOW_LENGTH }}"
+bound_to_include: begin
+
+stat_current_cycle_iso: "{{ current_cycle | to_isotime }}"
+stat_current_cycle_YMDH: "{{ current_cycle | to_YMDH }}"
+
+# Inputted list of ob spaces
+# --------------------------
+obspaces: {{ OBSPACES_LIST }}
+
+# Obspace variable things
+# -----------------------
+snow_obsdatain_path: "{{ DATA }}"

scripts/exglobal_analysis_stats.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+
+# exglobal_stat_analysis.py
+# This script creates a StatAnalysis class
+# and runs the initialize, execute, and finalize
+# methods which create and stage the runtime directory
+# and create the YAML configuration
+# to produce summary statistics from the analysis
+import os
+
+from wxflow import Logger, cast_strdict_as_dtypedict
+from pygfs.task.stat_analysis import StatAnalysis
+
+# Initialize root logger
+logger = Logger(level='DEBUG', colored_log=True)
+
+
+if __name__ == '__main__':
+
+    # Take configuration from environment and cast it as python dictionary
+    config = cast_strdict_as_dtypedict(os.environ)
+
+    # Instantiate the atm analysis task
+    StatAnl = StatAnalysis(config)
+
+    # Create list based on DA components
+    StatAnl.task_config['STAT_ANALYSES'] = []
+    if StatAnl.task_config.DO_AERO:
+        StatAnl.task_config['STAT_ANALYSES'].append('aero')
+    if StatAnl.task_config.DO_JEDISNOWDA:
+        StatAnl.task_config['STAT_ANALYSES'].append('snow')
+
+    # Initialize JEDI variational analysis
+    StatAnl.initialize()
+    for anl in StatAnl.task_config.STAT_ANALYSES:
+        StatAnl.execute(anl)
+        StatAnl.finalize(anl)