Add tests to convert sat bias files and save them to R2D2; make calls to R2D2 more generic

test/CMakeLists.txt

@@ -1,7 +1,7 @@
 ##### get test data from EMC FTP server
 # set URL and hash
 set(URL "https://ftp.emc.ncep.noaa.gov/static_files/public/GDASApp")
-set(SHA "2b547a5f489130edc240db04a08034d15787c15732871526b1873f0a27e491a4")
+set(SHA "0ecc05f647c27110a32c4181a92bdfc2c0474ed5035bd379a63add39429368c0")
 string(SUBSTRING ${SHA} 0 6 SHORTSHA)
 set(TAR "gdasapp-fix-${SHORTSHA}.tgz")
 # download test files
@@ -27,6 +27,8 @@ list(APPEND test_data
   ${CMAKE_CURRENT_BINARY_DIR}/testdata/prof.nc
   ${CMAKE_CURRENT_BINARY_DIR}/testdata/icec.nc
   ${CMAKE_CURRENT_BINARY_DIR}/testdata/icefb.nc
+  ${CMAKE_CURRENT_BINARY_DIR}/testdata/gdas.t00z.abias
+  ${CMAKE_CURRENT_BINARY_DIR}/testdata/gdas.t00z.abias_pc
 )
 
 # install
@@ -79,10 +81,9 @@ add_test(NAME test_gdasapp_jedi_increment_to_fv3
 add_test(NAME test_gdasapp_generate_yaml
          COMMAND ${Python3_EXECUTABLE} ${PROJECT_SOURCE_DIR}/test/test_generate_yaml.py --template ${PROJECT_BINARY_DIR}/test/testinput/generate_yaml.yaml --output ${PROJECT_BINARY_DIR}/test/testoutput/generate_yaml.yaml
          WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/)
-set_tests_properties(
-         test_gdasapp_generate_yaml
-PROPERTIES
-         ENVIRONMENT "PYTHONPATH=${PROJECT_BINARY_DIR}/ush:$ENV{PYTHONPATH}"
+set_tests_properties(test_gdasapp_generate_yaml
+    PROPERTIES
+        ENVIRONMENT "PYTHONPATH=${PROJECT_BINARY_DIR}/ush:$ENV{PYTHONPATH}"
 )
 
 # soca tests

test/atm/CMakeLists.txt

@@ -10,4 +10,19 @@ if(BUILD_GDASBUNDLE)
            COMMAND ${PROJECT_BINARY_DIR}/bin/bufr2ioda.x ${PROJECT_BINARY_DIR}/test/testinput/bufr_adpsfc.yaml
            WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/)
 
+  # test generate UFO CRTM sat bias files from GSI sat bias files
+  add_test(NAME test_gdasapp_convert_gsi_satbias
+           COMMAND ${PROJECT_SOURCE_DIR}/test/atm/test_convert_gsi_satbias.sh ${PROJECT_SOURCE_DIR} ${PROJECT_BINARY_DIR}
+           WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/)
+
+  # test saving the output of the previous test to an R2D2 database
+  add_test(NAME test_gdasapp_store_gsi_satbias
+           COMMAND ${PROJECT_SOURCE_DIR}/test/atm/test_store_gsi_satbias.py
+           WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/)
+  set_tests_properties(test_gdasapp_store_gsi_satbias
+      PROPERTIES
+          DEPENDS "test_gdasapp_convert_gsi_satbias"
+          ENVIRONMENT "PYTHONPATH=${PROJECT_BINARY_DIR}/ush:$ENV{PYTHONPATH}"
+  )
+
 endif(BUILD_GDASBUNDLE)

test/atm/test_convert_gsi_satbias.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+set -ex
+# test script to run GSI to UFO sat bias converter and produce files
+srcdir=$1
+builddir=$2
+
+# stage a testrun directory for this test
+testrun=$builddir/test/testrun/satbias_conv
+mkdir -p $testrun
+
+# copy GSI input files
+# the converter assumes a directory structure like we have for GDAS/GFS
+mkdir -p $testrun/input/gdas.20220401/00/atmos
+
+cp $builddir/test/testdata/gdas.t00z.abias* $testrun/input/gdas.20220401/00/atmos/.
+
+# create the YAML file as input
+cat > $builddir/test/testinput/run_satbias_conv.yaml << EOF
+start time: 2022-04-01T00:00:00Z
+end time: 2022-04-01T00:00:00Z
+assim_freq: 6
+dump: gdas
+gsi_bc_root: $testrun/input
+ufo_bc_root: $testrun/out
+work_root: $testrun/work
+satbias2ioda: $builddir/bin/satbias2ioda.x
+EOF
+
+# run the script
+python3 $srcdir/ush/run_satbias_conv.py --config $builddir/test/testinput/run_satbias_conv.yaml
+
+# copy output to testoutput
+testout=$builddir/test/testoutput
+mkdir -p $testout/satbias
+cp -rf $testrun/out/* $testout/satbias/.
+
+# clean up run directory
+rm -rf $testrun

test/atm/test_store_gsi_satbias.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+import os
+from r2d2 import store
+from solo.configuration import Configuration
+from solo.date import date_sequence
+import yaml
+import ufsda.r2d2
+
+
+def store_satbias(yaml_file):
+    config = Configuration(yaml_file)
+    ufsda.r2d2.store(config)
+
+
+if __name__ == "__main__":
+    # create a dummy R2D2 config YAML
+    r2d2_config = {
+        'databases':
+            {'test': {'cache_fetch': False,
+                      'class': 'LocalDB',
+                      'root': os.path.join(os.getcwd(), 'r2d2-test')}, },
+        'fetch_order': ['test'],
+        'store_order': ['test'],
+    }
+    with open(os.path.join(os.getcwd(), 'testinput', 'r2d2_config_test.yaml'), 'w') as f:
+        yaml.dump(r2d2_config, f, sort_keys=False, default_flow_style=False)
+    os.environ['R2D2_CONFIG'] = os.path.join(os.getcwd(), 'testinput', 'r2d2_config_test.yaml')
+    # create YAML for satbias
+    satbias = {
+        'start': '2022-04-01T00:00:00Z',
+        'end': '2022-04-01T00:00:00Z',
+        'step': 'PT6H',
+        'source_dir': os.path.join(os.getcwd(), 'testoutput', 'satbias'),
+        'source_file_fmt': '{source_dir}/{dump}.{year}{month}{day}/{hour}/atmos/{obs_type}_satbias.nc4',
+        'type': 'bc',
+        'file_type': 'satbias',
+        'database': 'test',
+        'provider': 'gsi',
+        'experiment': 'gdasapp',
+        'obs_types': [
+            'amsua_metop-b',
+            'atms_npp',
+            'cris-fsr_npp',
+            'iasi_metop-c',
+        ],
+    }
+    satbias_yaml = os.path.join(os.getcwd(), 'testinput', 'r2d2_store_satbias.yaml')
+    with open(satbias_yaml, 'w') as f:
+        yaml.dump(satbias, f, sort_keys=False, default_flow_style=False)
+    # create YAML for tlapse
+    tlapse = {
+        'start': '2022-04-01T00:00:00Z',
+        'end': '2022-04-01T00:00:00Z',
+        'step': 'PT6H',
+        'source_dir': os.path.join(os.getcwd(), 'testoutput', 'satbias'),
+        'source_file_fmt': '{source_dir}/{dump}.{year}{month}{day}/{hour}/atmos/{obs_type}_tlapmean.txt',
+        'type': 'bc',
+        'file_type': 'tlapse',
+        'database': 'test',
+        'provider': 'gsi',
+        'experiment': 'gdasapp',
+        'obs_types': [
+            'amsua_metop-b',
+            'atms_npp',
+            'cris-fsr_npp',
+            'iasi_metop-c',
+        ],
+    }
+    tlapse_yaml = os.path.join(os.getcwd(), 'testinput', 'r2d2_store_tlapse.yaml')
+    with open(tlapse_yaml, 'w') as f:
+        yaml.dump(tlapse, f, sort_keys=False, default_flow_style=False)
+    # call the store commands
+    store_satbias(satbias_yaml)
+    store_satbias(tlapse_yaml)

test/soca/CMakeLists.txt

@@ -10,4 +10,5 @@ ecbuild_add_test(
       WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obs
       ENVIRONMENT
         OBS_DIR=${OBS_DIR}
+        PYTHONPATH=${PROJECT_BINARY_DIR}/ush:$ENV{PYTHONPATH}
 )

test/soca/create_obsdb.py

@@ -2,41 +2,15 @@
 import argparse
 import os
 import shutil
-from r2d2 import store
 from solo.configuration import Configuration
 from solo.date import date_sequence
 import yaml
+import ufsda.r2d2
 
 
 def store_obs(yaml_file):
     config = Configuration(yaml_file)
-    dates = date_sequence(config.start, config.end, config.step)
-    obs_types = config.obs_types
-    provider = config.provider
-    experiment = config.experiment
-    database = config.database
-    type = config.type
-    source_dir = config.source_dir
-    step = config.step
-
-    for date in dates:
-        day = str(date).split('T')[0]
-        year = day[0:4]
-        month = day[4:6]
-        day = day[6:8]
-        for obs_type in obs_types:
-            obs_prefix = obs_type.split('_')[0]
-            store(
-                provider=provider,
-                type=type,
-                experiment=experiment,
-                database=database,
-                date=date,
-                obs_type=obs_type,
-                time_window=step,
-                source_file=f'{source_dir}/{obs_type}_{year}{month}{day}.nc4',
-                ignore_missing=True,
-            )
+    ufsda.r2d2.store(config)
 
 
 if __name__ == "__main__":
@@ -68,10 +42,11 @@ def store_obs(yaml_file):
     shutil.copyfile(os.path.join(obsdir, 'icefb.nc'), 'icefb_GDR_20180415.nc4', follow_symlinks=True)
 
     # Create the test R2D2 database
-    obsstore = {'start': '20180415',
-                'end': '20180415',
+    obsstore = {'start': '2018-04-15T00:00:00Z',
+                'end': '2018-04-15T00:00:00Z',
                 'step': 'P1D',
                 'source_dir': '.',
+                'source_file_fmt': '{source_dir}/{obs_type}_{year}{month}{day}.nc4',
                 'type': 'ob',
                 'database': 'shared',
                 'provider': 'gdasapp',

ush/ufsda/__init__.py

@@ -1,4 +1,5 @@
 from .disk_utils import *
 from .ufs_yaml import gen_yaml, parse_config
 import ufsda.stage
+import ufsda.r2d2
 import ufsda.post

ush/ufsda/r2d2.py

@@ -0,0 +1,48 @@
+import r2d2
+from solo.configuration import Configuration
+from solo.date import date_sequence, Hour
+
+
+def store(config):
+    times = date_sequence(config.start, config.end, config.step)
+    obs_types = config.obs_types
+    provider = config.provider
+    experiment = config.experiment
+    database = config.database
+    type = config.type
+    file_type = config.get('file_type', None)
+    source_dir = config.source_dir
+    source_file_fmt = config.source_file_fmt
+    step = config.step
+    dump = config.get('dump', 'gdas')
+
+    for time in times:
+        year = Hour(time).format('%Y')
+        month = Hour(time).format('%m')
+        day = Hour(time).format('%d')
+        hour = Hour(time).format('%H')
+        for obs_type in obs_types:
+            if type == 'bc':
+                r2d2.store(
+                    provider=provider,
+                    type=type,
+                    file_type=file_type,
+                    experiment=experiment,
+                    database=database,
+                    date=time,
+                    obs_type=obs_type,
+                    source_file=eval(f"f'{source_file_fmt}'"),
+                    ignore_missing=True,
+                )
+            else:
+                r2d2.store(
+                    provider=provider,
+                    type=type,
+                    experiment=experiment,
+                    database=database,
+                    date=time,
+                    obs_type=obs_type,
+                    time_window=step,
+                    source_file=eval(f"f'{source_file_fmt}'"),
+                    ignore_missing=True,
+                )