Merge branch 'develop' into bugfix/filter_topo

Fixes ufs-community#105

README.md

@@ -19,11 +19,12 @@ emcsfc_ice_blend | George Gayno
 emcsfc_snow2mdl | George Gayno
 fre-nctools | GFDL progammer
 fvcom_tools | David Wright, University of Michigan, Ming Hu, GSD/AMB
+gblevents | Hang Lei
+gdas_init | George Gayno
 global_cycle | George Gayno, Shrinivas Moorthi, Mark Iredell, Xu Li, Hang Lei
 grid_tools | R. J. Purser (regional_esg_grid), Tom Black/Ben Blake (shave.fd), Gerard Ketefian (global_equiv_resol), Tsukasa Fujita, JMA (pmat2), GFDL programmer (topo filtering code).
 orog_mask_tools | Ning Wang, Jordan Alpert, Shan Sun and Ning Wang
 sfc_climo_gen | George Gayno
-gblevents | Hang Lei
 vcoord_gen | Fanglin Yang, Mark Iredell
 
 UFS_UTILS Code managers: George Gayno, Kyle Gerheiser, Jeff Beck, Larissa Reames
@@ -70,6 +71,7 @@ directory):
 - emcsfc_snow2mdl
 - fre-nctools
 - fvcom_tools
+- gblevents
 - global_cycle
 - grid_tools
 - orog_mask_tools

VERSION

@@ -1 +1 @@
-1.5.0
+1.6.0

cmake/LibMPI.cmake

@@ -154,7 +154,11 @@ function (platform_name RETURN_VARIABLE)
 
 
     set (${RETURN_VARIABLE} "stampede" PARENT_SCOPE)
-
+
+  elseif (SITENAME MATCHES "^s4-submit.ssec.wisc.edu")
+
+    set (${RETURN_VARIABLE} "s4" PARENT_SCOPE)
+
   else ()
 
     set (${RETURN_VARIABLE} "unknown" PARENT_SCOPE)

docs/source/chgres_cube.rst

@@ -382,6 +382,7 @@ Column 2: Name of the variable in the output “coldstart” files. Unimplemente
 Column 3: Behavior when the code can't find the variable in the input file. Options are:
 
       * "skip": Don't write to the output file.
+      * "intrp": LnP interpolation to missing levels. No extrapolation allowd.
       * "set_to_fill": Set to user-specified field value (see column 4).
       * "stop": Force an exception and stop code execution. Use this if you absolutely require a field to be present.
 

fix/link_fixdirs.sh

@@ -9,20 +9,20 @@ machine=${2}
 if [ $# -lt 2 ]; then
     set +x
     echo '***ERROR*** must specify two arguements: (1) RUN_ENVIR, (2) machine'
-    echo ' Syntax: link_fv3gfs.sh ( nco | emc ) ( cray | dell | hera  | jet | orion )'
+    echo ' Syntax: link_fv3gfs.sh ( nco | emc ) ( cray | dell | hera  | jet | orion | s4 )'
     exit 1
 fi
 
 if [ $RUN_ENVIR != emc -a $RUN_ENVIR != nco ]; then
     set +x
     echo '***ERROR*** unsupported run environment'
-    echo 'Syntax: link_fv3gfs.sh ( nco | emc ) ( cray | dell | hera | jet | orion )'
+    echo 'Syntax: link_fv3gfs.sh ( nco | emc ) ( cray | dell | hera | jet | orion | s4 )'
     exit 1
 fi
-if [ $machine != cray -a $machine != hera -a $machine != dell -a $machine != jet -a $machine != orion ]; then
+if [ $machine != cray -a $machine != hera -a $machine != dell -a $machine != jet -a $machine != orion -a $machine != s4 ]; then
     set +x
     echo '***ERROR*** unsupported machine'
-    echo 'Syntax: link_fv3gfs.sh ( nco | emc ) ( cray | dell | hera | jet | orion )'
+    echo 'Syntax: link_fv3gfs.sh ( nco | emc ) ( cray | dell | hera | jet | orion | s4 )'
     exit 1
 fi
 
@@ -45,6 +45,8 @@ elif [ $machine = "jet" ]; then
     FIX_DIR="/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix"
 elif [ $machine = "orion" ]; then
     FIX_DIR="/work/noaa/global/glopara/fix"
+elif [ $machine = "s4" ]; then
+    FIX_DIR="/data/prod/glopara/fix"
 fi
 
 for dir in fix_am fix_fv3 fix_orog fix_fv3_gmted2010 fix_sfc_climo; do

modulefiles/build.jet.intel

@@ -2,6 +2,7 @@
 ## Build and run module for Jet
 #############################################################
 module load cmake/3.16.1
+module load hpss
 
 module use /lfs4/HFIP/hfv3gfs/nwprod/hpc-stack/libs/modulefiles/stack
 module load hpc/1.1.0
@@ -23,3 +24,4 @@ module load sfcio/1.4.1
 module load nemsio/2.5.2
 module load g2/3.4.1
 module load wgrib2/2.0.8
+module load prod_util/1.2.2

modulefiles/build.s4.intel

@@ -0,0 +1,28 @@
+#%Module#####################################################
+## Build and run module for S4
+#############################################################
+
+module load license_intel/S4
+module use /data/prod/hpc-stack/modulefiles/stack
+module load hpc/1.1.0
+module load hpc-intel/18.0.4
+module load hpc-impi/18.0.4
+
+module load bacio/2.4.1
+module load g2/3.4.1
+module load ip/3.3.3
+module load nemsio/2.5.2
+module load sp/2.3.3
+module load w3nco/2.4.1
+module load sfcio/1.4.1
+module load sigio/2.3.2
+module load wgrib2/2.0.8
+
+module load jasper/2.0.22
+module load zlib/1.2.11
+module load png/1.6.35
+
+module load hdf5/1.10.6
+module load netcdf/4.7.4
+module load nccmp/1.8.7.0
+module load esmf/8_1_0_beta_snapshot_27

parm/varmap_tables/GFSphys_var_map.txt

@@ -1,11 +1,11 @@
 dzdt                dzdt      set_to_fill   0        D
-sphum               sphum     set_to_fill   1E-7     T
-liq_wat             liq_wat   set_to_fill   0        T
-o3mr                o3mr      set_to_fill   1E-7     T
-ice_wat             ice_wat   set_to_fill   0        T
-rainwat             rainwat   set_to_fill   0        T
-snowwat             snowwat   set_to_fill   0        T
-graupel             graupel   set_to_fill   0        T
+sphum               sphum     intrp         1E-7     T
+liq_wat             liq_wat   intrp         0        T
+o3mr                o3mr      intrp         1E-7     T
+ice_wat             ice_wat   intrp         0        T
+rainwat             rainwat   intrp         0        T
+snowwat             snowwat   intrp         0        T
+graupel             graupel   intrp         0        T
 vtype               vtype     skip          0        S
 sotype              stype     skip          0        S
 vfrac               vfrac     skip          0        S

parm/varmap_tables/GSDphys_noaero_var_map.txt

@@ -1,16 +1,16 @@
 dzdt                dzdt      set_to_fill   0         D
 delta_p             delp      skip          0         D
-sphum               sphum     set_to_fill   1E-7      T
-liq_wat             liq_wat   set_to_fill   0         T
-o3mr                o3mr      set_to_fill   1E-7      T
-rainwat             rainwat   set_to_fill   0         T
-ice_wat             ice_wat   set_to_fill   0         T
-snowwat             snowwat   set_to_fill   0         T
-graupel             graupel   set_to_fill   0         T
-ice_nc              ice_nc    set_to_fill   -1.0      T
-rain_nc             rain_nc   set_to_fill   -1.0      T
-water_nc            water_nc  set_to_fill   -1.0      T
-sgs_tke             sgs_tke   set_to_fill   0         T
+sphum               sphum     intrp         1E-7      T
+liq_wat             liq_wat   intrp         0         T
+o3mr                o3mr      intrp         1E-7      T
+rainwat             rainwat   intrp         0         T
+ice_wat             ice_wat   intrp         0         T
+snowwat             snowwat   intrp         0         T
+graupel             graupel   intrp         0         T
+ice_nc              ice_nc    intrp         -1.0      T
+rain_nc             rain_nc   intrp         -1.0      T
+water_nc            water_nc  intrp         -1.0      T
+sgs_tke             sgs_tke   intrp         0         T
 vtype               vtype     skip          0         S
 sotype              stype     skip          0         S
 vfrac               vfrac     skip          0         S

parm/varmap_tables/GSDphys_var_map.txt

@@ -1,18 +1,18 @@
 dzdt                dzdt      set_to_fill   0         D
 delta_p             delp      skip          0         D
-sphum               sphum     set_to_fill   1E-7      T
-liq_wat             liq_wat   set_to_fill   0         T
-o3mr                o3mr      set_to_fill   1E-7      T
-rainwat             rainwat   set_to_fill   0         T
-ice_wat             ice_wat   set_to_fill   0         T
-snowwat             snowwat   set_to_fill   0         T
-graupel             graupel   set_to_fill   0         T
-ice_nc              ice_nc    set_to_fill   -1.0      T
-rain_nc             rain_nc   set_to_fill   -1.0      T
-water_nc            water_nc  set_to_fill   -1.0      T
-liq_aero            liq_aero  set_to_fill   0         T
-ice_aero            ice_aero  set_to_fill   0         T
-sgs_tke             sgs_tke   set_to_fill   0         T
+sphum               sphum     intrp         1E-7      T
+liq_wat             liq_wat   intrp         0         T
+o3mr                o3mr      intrp         1E-7      T
+rainwat             rainwat   intrp         0         T
+ice_wat             ice_wat   intrp         0         T
+snowwat             snowwat   intrp         0         T
+graupel             graupel   intrp         0         T
+ice_nc              ice_nc    intrp         -1.0      T
+rain_nc             rain_nc   intrp         -1.0      T
+water_nc            water_nc  intrp         -1.0      T
+liq_aero            liq_aero  intrp         0         T
+ice_aero            ice_aero  intrp         0         T
+sgs_tke             sgs_tke   intrp         0         T
 vtype               vtype     skip          0         S
 sotype              stype     skip          0         S
 vfrac               vfrac     skip          0         S

parm/varmap_tables/README

@@ -14,6 +14,7 @@ Column 2: Name the code will use to save the variable in the output file. Unimpl
 Comumn 3: Behavior when the code can't find the variable in the input file. Options are:
 	  "skip": Don't write to output file.
 	  "set_to_fill": Set to user-specified field value (see column 4).
+	  "intrp": LnP interpolation to missing levels. No extrapolation allowd.
 	  "stop": Force an exception and stop code execution. Use this if you absolutely
 		require a field to be present.
 Column 4: If column 3 = "set_to_fill", then this value is used to fill in all points 

reg_tests/chgres_cube/25km.conus.gfs.pbgrib2.sh

@@ -0,0 +1,86 @@
+#!/bin/bash
+
+#-----------------------------------------------------------------------------
+# Invoke chgres to create 25-km CONUS coldstart files using GFS 
+# PGRIB2+BGRIB2 data as input.  The coldstart files are then compared to
+# baseline files using the 'nccmp' utility.  This script is run by the
+# This script is run by the machine specific driver script.
+#-----------------------------------------------------------------------------
+
+set -x
+
+export DATA=$OUTDIR/25km_conus_gfs_pbgrib2
+rm -fr $DATA
+
+export CRES=403
+export KMRES=25km
+export FIXfv3=${HOMEreg}/fix/RRFS_CONUS_${KMRES}
+export FIXsfc=${FIXfv3}/fix_sfc
+export COMIN=${HOMEreg}/input_data/gfs.pbgrib2
+
+export GRIB2_FILE_INPUT=gfs.t18z.pgrb2.0p25.f006
+export VCOORD_FILE=${HOMEufs}/fix/fix_am/global_hyblev.l64.txt
+export VARMAP_FILE=${HOMEufs}/parm/varmap_tables/GFSphys_var_map.txt
+export INPUT_TYPE='grib2'
+export CONVERT_NST=".false."
+export OROG_FILES_TARGET_GRID="C403_oro_data.tile7.halo4.nc"
+export REGIONAL=1
+export HALO_BLEND=0
+export HALO_BNDY=4 
+export CDATE=2021062718
+
+export OMP_NUM_THREADS_CH=${OMP_NUM_THREADS:-1}
+
+#-----------------------------------------------------------------------------
+# Invoke chgres program.
+#-----------------------------------------------------------------------------
+
+echo "Starting at: " `date`
+
+${HOMEufs}/ush/chgres_cube.sh
+
+iret=$?
+if [ $iret -ne 0 ]; then
+  set +x
+  echo "<<< 25-km CONUS GFS PGRIB2+BGRIB2 TEST FAILED. >>>"
+  exit $iret
+fi
+
+echo "Ending at: " `date`
+
+#-----------------------------------------------------------------------------
+# Compare output from chgres to baseline set of data.
+#
+# orion's nccmp utility does not work with the netcdf
+# required to run ufs_utils.  So swap it.
+#-----------------------------------------------------------------------------
+
+machine=${machine:-NULL}
+if [ $machine == 'orion' ]; then
+  module unload netcdfp/4.7.4.release
+  module load netcdf/4.7.2
+fi
+
+cd $DATA
+
+test_failed=0
+for files in *.nc
+do
+  if [ -f $files ]; then
+    echo CHECK $files
+    $NCCMP -dmfqS $files $HOMEreg/baseline_data/25km_conus_gfs_pbgrib2/$files
+    iret=$?
+    if [ $iret -ne 0 ]; then
+      test_failed=1
+    fi
+  fi
+done
+
+set +x
+if [ $test_failed -ne 0 ]; then
+  echo "<<< 25-KM CONUS GFS PGRIB2+BGRIB2 TEST FAILED. >>>"
+else
+  echo "<<< 25-KM CONUS GFS PGRIB2+BGRIB2 TEST PASSED. >>>"
+fi
+
+exit 0

reg_tests/chgres_cube/driver.hera.sh

@@ -193,13 +193,22 @@ export OMP_NUM_THREADS=1   # should match cpus-per-task
 TEST15=$(sbatch --parsable --ntasks-per-node=12 --nodes=1 -t 0:15:00 -A $PROJECT_CODE -q $QUEUE -J c96.fv3.netcdf2wam \
       -o $LOG_FILE -e $LOG_FILE ./c96.fv3.netcdf2wam.sh)
 
+#-----------------------------------------------------------------------------
+# Initialize CONUS 25-KM USING  GFS PGRIB2+BGRIB2 files.
+#-----------------------------------------------------------------------------
+
+LOG_FILE=consistency.log16
+export OMP_NUM_THREADS=1   # should match cpus-per-task
+TEST16=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 -t 0:05:00 -A $PROJECT_CODE -q $QUEUE -J 25km.conus.gfs.pbgrib2.conus \
+      -o $LOG_FILE -e $LOG_FILE ./25km.conus.gfs.pbgrib2.sh)
+
 #-----------------------------------------------------------------------------
 # Create summary log.
 #-----------------------------------------------------------------------------
 LOG_FILE=consistency.log
 sbatch --nodes=1 -t 0:01:00 -A $PROJECT_CODE -J chgres_summary -o $LOG_FILE -e $LOG_FILE \
       --open-mode=append -q $QUEUE -d\
-      afterok:$TEST1:$TEST2:$TEST3:$TEST4:$TEST5:$TEST6:$TEST7:$TEST8:$TEST9:$TEST10:$TEST11:$TEST12:$TEST13:$TEST14:$TEST15 << EOF
+      afterok:$TEST1:$TEST2:$TEST3:$TEST4:$TEST5:$TEST6:$TEST7:$TEST8:$TEST9:$TEST10:$TEST11:$TEST12:$TEST13:$TEST14:$TEST15:$TEST16 << EOF
 #!/bin/bash
 grep -a '<<<' $LOG_FILE*  > $SUM_FILE
 EOF

reg_tests/chgres_cube/driver.jet.sh

@@ -194,14 +194,23 @@ export OMP_NUM_THREADS=1   # should match cpus-per-task
 TEST15=$(sbatch --parsable --partition=xjet --ntasks-per-node=6 --nodes=2 -t 0:15:00 -A $PROJECT_CODE -q $QUEUE -J c96.fv3.netcdf2wam \
       --exclusive -o $LOG_FILE -e $LOG_FILE ./c96.fv3.netcdf2wam.sh)
 
+#-----------------------------------------------------------------------------
+# Initialize CONUS 25-KM USING GFS PGRIB2+BGRIB2 files.
+#-----------------------------------------------------------------------------
+
+LOG_FILE=consistency.log16
+export OMP_NUM_THREADS=1   # should match cpus-per-task
+TEST16=$(sbatch --parsable --partition=xjet --ntasks-per-node=6 --nodes=1 -t 0:05:00 -A $PROJECT_CODE -q $QUEUE -J 25km.conus.gfs.pbgrib2.conus \
+      --exclusive -o $LOG_FILE -e $LOG_FILE ./25km.conus.gfs.pbgrib2.sh)
+
 #-----------------------------------------------------------------------------
 # Create summary log.
 #-----------------------------------------------------------------------------
 
 LOG_FILE=consistency.log
 sbatch --partition=xjet --nodes=1  -t 0:01:00 -A $PROJECT_CODE -J chgres_summary -o $LOG_FILE -e $LOG_FILE \
        --open-mode=append -q $QUEUE -d\
-       afterok:$TEST1:$TEST2:$TEST3:$TEST4:$TEST5:$TEST6:$TEST7:$TEST8:$TEST9:$TEST10:$TEST11:$TEST12:$TEST13:$TEST14 << EOF
+       afterok:$TEST1:$TEST2:$TEST3:$TEST4:$TEST5:$TEST6:$TEST7:$TEST8:$TEST9:$TEST10:$TEST11:$TEST12:$TEST13:$TEST14:$TEST15:$TEST16 << EOF
 #!/bin/bash
 grep -a '<<<' $LOG_FILE*  > $SUM_FILE
 EOF

reg_tests/chgres_cube/driver.orion.sh

@@ -197,13 +197,22 @@ export OMP_NUM_THREADS=1   # should match cpus-per-task
 TEST15=$(sbatch --parsable --ntasks-per-node=12 --nodes=1 -t 0:10:00 -A $PROJECT_CODE -q $QUEUE -J 13km.na.gfs.ncei.grib2 \
       --open-mode=append -o $LOG_FILE15 -e $LOG_FILE15 ./13km.na.gfs.ncei.grib2.sh)
 
+#-----------------------------------------------------------------------------
+# Initialize CONUS 25-KM USING GFS PGRIB2+BGRIB2 files.
+#-----------------------------------------------------------------------------
+
+LOG_FILE16=${LOG_FILE}16
+export OMP_NUM_THREADS=1   # should match cpus-per-task
+TEST16=$(sbatch --parsable --ntasks-per-node=12 --nodes=1 -t 0:10:00 -A $PROJECT_CODE -q $QUEUE -J 25km.conus.gfs.pbgrib2 \
+      --open-mode=append -o $LOG_FILE16 -e $LOG_FILE16 ./25km.conus.gfs.pbgrib2.sh)
+
 #-----------------------------------------------------------------------------
 # Create summary log.
 #-----------------------------------------------------------------------------
 
 sbatch --nodes=1 -t 0:01:00 -A $PROJECT_CODE -J chgres_summary -o $LOG_FILE -e $LOG_FILE \
        --open-mode=append -q $QUEUE \
-       -d afterok:$TEST1:$TEST2:$TEST3:$TEST4:$TEST5:$TEST6:$TEST7:$TEST8:$TEST9:$TEST10:$TEST11:$TEST12:$TEST13:$TEST14:$TEST15 << EOF
+       -d afterok:$TEST1:$TEST2:$TEST3:$TEST4:$TEST5:$TEST6:$TEST7:$TEST8:$TEST9:$TEST10:$TEST11:$TEST12:$TEST13:$TEST14:$TEST15:$TEST16 << EOF
 #!/bin/bash
 grep -a '<<<' ${LOG_FILE}*  > $SUM_FILE
 EOF

reg_tests/chgres_cube/driver.wcoss_cray.sh

@@ -204,6 +204,16 @@ export APRUN="aprun -j 1 -n 12 -N 6 -d ${OMP_NUM_THREADS} -cc depth"
 bsub -e $LOG_FILE15 -o $LOG_FILE15 -q $QUEUE -P $PROJECT_CODE -J chgres15 -M 1000 -W 0:15 -extsched 'CRAYLINUX[]' \
         "export NODES=2; $PWD/c96.fv3.netcdf2wam.sh"
 
+#-----------------------------------------------------------------------------
+# Initialize CONUS 25-KM USING GFS PGRIB2+BGRIB2 files.
+#-----------------------------------------------------------------------------
+
+LOG_FILE16=${LOG_FILE}16
+export OMP_NUM_THREADS=1
+export APRUN="aprun -j 1 -n 6 -N 6 -d ${OMP_NUM_THREADS} -cc depth"
+bsub -e $LOG_FILE16 -o $LOG_FILE16 -q $QUEUE -P $PROJECT_CODE -J chgres16 -M 1000 -W 0:05 -extsched 'CRAYLINUX[]' \
+        "export NODES=1; $PWD/25km.conus.gfs.pbgrib2.sh"
+
 #-----------------------------------------------------------------------------
 # Create summary log.
 #-----------------------------------------------------------------------------