Merge branch 'develop' into feature/sfcsub_ccpp

Fixes ufs-community#424

.github/workflows/esmf-8.0.1_jasper-2.0.22_nceplibs-1.3.0.yml

@@ -46,6 +46,7 @@ jobs:
           sudo apt-get install libpng-dev
           sudo apt-get install libjpeg-dev
         elif [[ ${{ matrix.os }} == "macos-10.15" ]]; then
+          brew update
           brew install doxygen
           brew install mpich
           brew install netcdf

.github/workflows/esmf-8.0.1_jasper-2.0.22_nceplibs-develop.yml

@@ -36,6 +36,7 @@ jobs:
 
     - name: install-dependencies
       run: |
+        echo "/home/runner/mpich/bin" >> $GITHUB_PATH
         if [[ ${{ matrix.os }} == "ubuntu-20.04" ]]; then
           sudo apt-get update
           sudo apt-get install libmpich-dev
@@ -45,6 +46,7 @@ jobs:
           sudo apt-get install libjpeg-dev
         elif [[ ${{ matrix.os }} == "macos-10.15" ]]; then
           echo "$HOME/mpich/bin" >> $GITHUB_PATH
+          brew update
           brew install doxygen
           brew install mpich
           brew install netcdf

CMakeLists.txt

@@ -46,7 +46,7 @@ endif()
 
 # Find packages.
 find_package(NetCDF REQUIRED C Fortran)
-find_package(MPI REQUIRED)
+find_package(MPI REQUIRED C Fortran)
 find_package(ESMF MODULE REQUIRED)
 
 if(OPENMP)

README.md

@@ -103,6 +103,12 @@ documentation build, as well as some markdown files which are part of
 the documentation. It also contains (in the source subdirectory) the
 ReadTheDocs documentation files.
 
+## References
+
+Gayno G., Beck J., Carson L., [Pre-Processing:
+chgres_cube](./docs/20201105-0945a-pre-processing-chgres-cube-gayno-final.pdf),
+UFS MRW App Training, 5 November 2020.
+
 ## Disclaimer
 
 The United States Department of Commerce (DOC) GitHub project code is

driver_scripts/driver_grid.cray.sh

@@ -68,6 +68,8 @@ module list
 
 export gtype=uniform           # 'uniform', 'stretch', 'nest', 
                                # 'regional_gfdl', 'regional_esg'
+export make_gsl_orog=false     # 'true' if user needs 'oro' files for GSL
+                               # orographic drag suite
 if [ $gtype = uniform ]; then
   export res=96
   export add_lake=false        # Add lake frac and depth to orography data.

driver_scripts/driver_grid.dell.sh

@@ -70,6 +70,8 @@ module list
 
 export gtype=uniform           # 'uniform', 'stretch', 'nest', 
                                # 'regional_gfdl', 'regional_esg'
+export make_gsl_orog=false     # 'true' if user needs 'oro' files for GSL
+                               # orographic drag suite
 if [ $gtype = uniform ]; then
   export res=96
   export add_lake=false        # Add lake frac and depth to orography data.

driver_scripts/driver_grid.hera.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 #SBATCH -J fv3_grid_driver
-#SBATCH -A fv3-cpu
+#SBATCH -A wrfruc
 #SBATCH --open-mode=truncate
 #SBATCH -o log.fv3_grid_driver
 #SBATCH -e log.fv3_grid_driver
@@ -71,6 +71,8 @@ module list
 
 export gtype=uniform           # 'uniform', 'stretch', 'nest', 
                                # 'regional_gfdl', 'regional_esg'
+export make_gsl_orog=false     # 'true' if user needs 'oro' files for GSL
+                               # orographic drag suite
 if [ $gtype = uniform ]; then
   export res=96
   export add_lake=false        # Add lake frac and depth to orography data.
@@ -92,7 +94,7 @@ elif [ $gtype = nest ] || [ $gtype = regional_gfdl ]; then
   export jstart_nest=331       # Starting j-direction index of nest grid in parent tile supergrid
   export iend_nest=1402        # Ending i-direction index of nest grid in parent tile supergrid
   export jend_nest=1194        # Ending j-direction index of nest grid in parent tile supergrid
-  export halo=3                # Lateral boundary halo
+  export halo=4                # Lateral boundary halo
 elif [ $gtype = regional_esg ] ; then
   export res=-999              # equivalent resolution is computed
   export target_lon=-97.5      # Center longitude of grid
@@ -105,7 +107,7 @@ elif [ $gtype = regional_esg ] ; then
                                # direction is related to delx as follows:
                                #    distance = 2*delx*(circumf_Earth/360 deg)
   export dely=0.0585           # Grid spacing (in degrees) in the 'j' direction.
-  export halo=3                # number of row/cols for halo
+  export halo=4                # number of row/cols for halo
 fi
 
 #-----------------------------------------------------------------------

driver_scripts/driver_grid.jet.sh

@@ -71,6 +71,8 @@ module list
 
 export gtype=uniform           # 'uniform', 'stretch', 'nest', 
                                # 'regional_gfdl', 'regional_esg'
+export make_gsl_orog=false     # 'true' if user needs 'oro' files for GSL
+                               # orographic drag suite
 if [ $gtype = uniform ]; then
   export res=96
   export add_lake=false        # Add lake frac and depth to orography data.

driver_scripts/driver_grid.orion.sh

@@ -70,6 +70,8 @@ module list
 
 export gtype=uniform           # 'uniform', 'stretch', 'nest', 
                                # 'regional_gfdl', 'regional_esg'
+export make_gsl_orog=false     # 'true' if user needs 'oro' files for GSL
+                               # orographic drag suite
 if [ $gtype = uniform ]; then
   export res=96
   export add_lake=false        # Add lake frac and depth to orography data.

reg_tests/chgres_cube/driver.hera.sh

@@ -31,9 +31,11 @@ module use ../../modulefiles
 module load build.$target.$compiler
 module list
 
-export OUTDIR=/scratch2/NCEPDEV/stmp1/$LOGNAME/chgres_reg_tests
-PROJECT_CODE="fv3-cpu"
-QUEUE="batch"
+export OUTDIR="${WORK_DIR:-/scratch2/NCEPDEV/stmp1/$LOGNAME}"
+export OUTDIR="${OUTDIR}/reg-tests/chgres-cube"
+
+PROJECT_CODE="${PROJECT_CODE:-fv3-cpu}"
+QUEUE="${QUEUE:-batch}"
 
 #-----------------------------------------------------------------------------
 # Should not have to change anything below here.  HOMEufs is the root

reg_tests/chgres_cube/driver.jet.sh

@@ -29,9 +29,11 @@ module use ../../modulefiles
 module load build.$target.intel
 module list
 
-export OUTDIR=/lfs4/HFIP/emcda/$LOGNAME/stmp/chgres_reg_tests
-PROJECT_CODE="hfv3gfs"
-QUEUE="debug"
+export OUTDIR="${WORK_DIR:-/lfs4/HFIP/emcda/$LOGNAME/stmp}"
+export OUTDIR="${OUTDIR}/reg-tests/chgres-cube"
+
+PROJECT_CODE="${PROJECT_CODE:-hfv3gfs}"
+QUEUE="${QUEUE:-debug}"
 
 #-----------------------------------------------------------------------------
 # Should not have to change anything below here.  HOMEufs is the root

reg_tests/chgres_cube/driver.orion.sh

@@ -29,9 +29,13 @@ module use ../../modulefiles
 module load build.$target.intel
 module list
 
-export OUTDIR=/work/noaa/stmp/$LOGNAME/chgres_reg_tests
-PROJECT_CODE="fv3-cpu"
-QUEUE="debug"
+ulimit -s unlimited
+
+export OUTDIR="${WORK_DIR:-/work/noaa/stmp/$LOGNAME}"
+export OUTDIR="${OUTDIR}/reg-tests/chgres-cube"
+
+PROJECT_CODE="${PROJECT_CODE:-fv3-cpu}"
+QUEUE="${QUEUE:-debug}"
 
 #-----------------------------------------------------------------------------
 # Should not have to change anything below here.  HOMEufs is the root

reg_tests/chgres_cube/driver.cray.sh → reg_tests/chgres_cube/driver.wcoss_cray.sh

@@ -27,9 +27,11 @@ module use ../../modulefiles
 module load build.$target.intel
 module list
 
-export OUTDIR=/gpfs/hps3/stmp/$LOGNAME/chgres_reg_tests
-QUEUE="debug"
-PROJECT_CODE="GFS-DEV"
+export OUTDIR="${WORK_DIR:-/gpfs/hps3/stmp/$LOGNAME}"
+export OUTDIR="${OUTDIR}/reg-tests/chgres-cube"
+
+QUEUE="${QUEUE:-debug}"
+PROJECT_CODE="${PROJECT_CODE:-GFS-DEV}"
 
 #-----------------------------------------------------------------------------
 # Should not have to change anything below here.  HOMEufs is the root

reg_tests/chgres_cube/driver.dell.sh → reg_tests/chgres_cube/driver.wcoss_dell_p3.sh

@@ -27,9 +27,11 @@ module use ../../modulefiles
 module load build.$target.intel
 module list
 
-export OUTDIR=/gpfs/dell1/stmp/$LOGNAME/chgres_reg_tests
-QUEUE="debug"
-PROJECT_CODE="GFS-DEV"
+export OUTDIR="${WORK_DIR:-/gpfs/dell1/stmp/$LOGNAME}"
+export OUTDIR="${OUTDIR}/reg-tests/chgres-cube"
+
+QUEUE="${QUEUE:-debug}"
+PROJECT_CODE="${PROJECT_CODE:-GFS-DEV}"
 
 #-----------------------------------------------------------------------------
 # Should not have to change anything below here.  HOMEufs is the root

reg_tests/global_cycle/driver.hera.sh

@@ -36,7 +36,8 @@ module use ../../modulefiles
 module load build.$target.$compiler
 module list
 
-export DATA=/scratch2/NCEPDEV/stmp1/$LOGNAME/reg_tests.cycle
+export DATA="${WORK_DIR:-/scratch2/NCEPDEV/stmp1/$LOGNAME}"
+export DATA="${DATA}/reg-tests/global-cycle"
 
 #-----------------------------------------------------------------------------
 # Should not have to change anything below.

reg_tests/global_cycle/driver.jet.sh

@@ -35,7 +35,8 @@ module use ../../modulefiles
 module load build.$target.intel
 module list
 
-export DATA=/lfs4/HFIP/emcda/$LOGNAME/stmp/reg_tests.cycle
+export DATA="${WORK_DIR:-/lfs4/HFIP/emcda/$LOGNAME/stmp}"
+export DATA="${DATA}/reg-tests/global-cycle"
 
 #-----------------------------------------------------------------------------
 # Should not have to change anything below.

reg_tests/global_cycle/driver.orion.sh

@@ -34,7 +34,10 @@ module use ../../modulefiles
 module load build.$target.intel
 module list
 
-export DATA=/work/noaa/stmp/$LOGNAME/reg_tests.cycle
+ulimit -s unlimited
+
+export DATA="${WORK_DIR:-/work/noaa/stmp/$LOGNAME}"
+export DATA="${DATA}/reg-tests/global-cycle"
 
 #-----------------------------------------------------------------------------
 # Should not have to change anything below.

reg_tests/global_cycle/driver.cray.sh → reg_tests/global_cycle/driver.wcoss_cray.sh

@@ -32,7 +32,8 @@ module use ../../modulefiles
 module load build.$target.intel
 module list
 
-export DATA=/gpfs/hps3/stmp/$LOGNAME/reg_tests.cycle
+export DATA="${WORK_DIR:-/gpfs/hps3/stmp/$LOGNAME}"
+export DATA="${DATA}/reg-tests/global-cycle"
 
 #-----------------------------------------------------------------------------
 # Should not have to change anything below.

reg_tests/global_cycle/driver.dell.sh → reg_tests/global_cycle/driver.wcoss_dell_p3.sh

@@ -37,7 +37,8 @@ module use ../../modulefiles
 module load build.$target.intel
 module list
 
-export DATA=/gpfs/dell1/stmp/$LOGNAME/reg_tests.cycle
+export DATA="${WORK_DIR:-/gpfs/dell1/stmp/$LOGNAME}"
+export DATA="${DATA}/reg-tests/global-cycle"
 
 #-----------------------------------------------------------------------------
 # Should not have to change anything below.

reg_tests/grid_gen/driver.hera.sh

@@ -30,9 +30,10 @@ module list
 
 set -x
 
-export WORK_DIR=/scratch2/NCEPDEV/stmp1/$LOGNAME/reg_tests.grid
-QUEUE="batch"
-PROJECT_CODE="fv3-cpu"
+export WORK_DIR="${WORK_DIR:-/scratch2/NCEPDEV/stmp1/$LOGNAME}"
+export WORK_DIR="${WORK_DIR}/reg-tests/grid-gen"
+QUEUE="${QUEUE:-batch}"
+PROJECT_CODE="${PROJECT_CODE:-fv3-cpu}"
 
 #-----------------------------------------------------------------------------
 # Should not have to change anything below here.
@@ -75,12 +76,19 @@ TEST2=$(sbatch --parsable --ntasks-per-node=24 --nodes=1 -t 0:10:00 -A $PROJECT_
 TEST3=$(sbatch --parsable --ntasks-per-node=24 --nodes=1 -t 0:10:00 -A $PROJECT_CODE -q $QUEUE -J esg.regional \
       -o $LOG_FILE -e $LOG_FILE -d afterok:$TEST2 ./esg.regional.sh)
 
+#-----------------------------------------------------------------------------
+# Regional GSL gravity wave drag test.
+#-----------------------------------------------------------------------------
+
+TEST4=$(sbatch --parsable --ntasks-per-node=24 --nodes=1 -t 0:10:00 -A $PROJECT_CODE -q $QUEUE -J reg.gsl.gwd \
+      -o $LOG_FILE -e $LOG_FILE -d afterok:$TEST3 ./regional.gsl.gwd.sh)
+
 #-----------------------------------------------------------------------------
 # Create summary log.
 #-----------------------------------------------------------------------------
 
 sbatch --nodes=1 -t 0:01:00 -A $PROJECT_CODE -J grid_summary -o $LOG_FILE -e $LOG_FILE \
-       --open-mode=append -q $QUEUE -d afterok:$TEST3 << EOF
+       --open-mode=append -q $QUEUE -d afterok:$TEST4 << EOF
 #!/bin/bash
 grep -a '<<<' $LOG_FILE  > $SUM_FILE
 EOF

reg_tests/grid_gen/driver.jet.sh

@@ -28,9 +28,10 @@ module list
 
 set -x
 
-QUEUE="windfall"
-PROJECT_CODE="emcda"
-export WORK_DIR=/lfs4/HFIP/emcda/$LOGNAME/stmp/reg_tests.grid
+QUEUE="${QUEUE:-windfall}"
+PROJECT_CODE="${PROJECT_CODE:-emcda}"
+export WORK_DIR="${WORK_DIR:-/lfs4/HFIP/emcda/$LOGNAME/stmp}"
+export WORK_DIR="${WORK_DIR}/reg-tests/grid-gen"
 
 #-----------------------------------------------------------------------------
 # Should not have to change anything below here.
@@ -73,12 +74,19 @@ TEST2=$(sbatch --parsable --ntasks-per-node=24 --nodes=1 -t 0:10:00 -A $PROJECT_
 TEST3=$(sbatch --parsable --ntasks-per-node=24 --nodes=1 -t 0:10:00 -A $PROJECT_CODE -q $QUEUE -J esg.regional \
       --partition=xjet -o $LOG_FILE -e $LOG_FILE -d afterok:$TEST2 ./esg.regional.sh)
 
+#-----------------------------------------------------------------------------
+# Regional GSL gravity wave drag.
+#-----------------------------------------------------------------------------
+
+TEST4=$(sbatch --parsable --ntasks-per-node=24 --nodes=1 -t 0:10:00 -A $PROJECT_CODE -q $QUEUE -J reg.gsl.gwd \
+      --partition=xjet -o $LOG_FILE -e $LOG_FILE -d afterok:$TEST3 ./regional.gsl.gwd.sh)
+
 #-----------------------------------------------------------------------------
 # Create summary log.
 #-----------------------------------------------------------------------------
 
 sbatch --partition=xjet --nodes=1  -t 0:01:00 -A $PROJECT_CODE -J grid_summary -o $LOG_FILE -e $LOG_FILE \
-       --open-mode=append -q $QUEUE -d afterok:$TEST3 << EOF
+       --open-mode=append -q $QUEUE -d afterok:$TEST4 << EOF
 #!/bin/bash
 grep -a '<<<' $LOG_FILE  > $SUM_FILE
 EOF

reg_tests/grid_gen/driver.orion.sh

@@ -27,10 +27,12 @@ module load build.$target.intel
 module list
 
 set -x
+ulimit -s unlimited
 
-export WORK_DIR=/work/noaa/stmp/$LOGNAME/reg_tests.grid
-QUEUE="batch"
-PROJECT_CODE="fv3-cpu"
+export WORK_DIR="${WORK_DIR:-/work/noaa/stmp/$LOGNAME}"
+export WORK_DIR="${WORK_DIR}/reg-tests/grid-gen"
+QUEUE="${QUEUE:-batch}"
+PROJECT_CODE=${PROJECT_CODE:-fv3-cpu}
 
 #-----------------------------------------------------------------------------
 # Should not have to change anything below here.
@@ -69,12 +71,19 @@ TEST2=$(sbatch --parsable --ntasks-per-node=24 --nodes=1 -t 0:10:00 -A $PROJECT_
 TEST3=$(sbatch --parsable --ntasks-per-node=24 --nodes=1 -t 0:10:00 -A $PROJECT_CODE -q $QUEUE -J esg.regional \
       --open-mode=append -o $LOG_FILE -e $LOG_FILE -d afterok:$TEST2 ./esg.regional.sh)
 
+#-----------------------------------------------------------------------------
+# Regional grid with GSL gravity wave drag fields.
+#-----------------------------------------------------------------------------
+
+TEST4=$(sbatch --parsable --ntasks-per-node=24 --nodes=1 -t 0:10:00 -A $PROJECT_CODE -q $QUEUE -J reg.gsl.gwd \
+      --open-mode=append -o $LOG_FILE -e $LOG_FILE -d afterok:$TEST3 ./regional.gsl.gwd.sh)
+
 #-----------------------------------------------------------------------------
 # Create summary log.
 #-----------------------------------------------------------------------------
 
 sbatch --nodes=1 -t 0:01:00 -A $PROJECT_CODE -J grid_summary -o $LOG_FILE -e $LOG_FILE \
-       --open-mode=append -q $QUEUE -d afterok:$TEST3 << EOF
+       --open-mode=append -q $QUEUE -d afterok:$TEST4 << EOF
 #!/bin/bash
 grep -a '<<<' $LOG_FILE  > $SUM_FILE
 EOF

reg_tests/grid_gen/driver.cray.sh → reg_tests/grid_gen/driver.wcoss_cray.sh

@@ -26,9 +26,10 @@ module list
 
 set -x
 
-QUEUE="debug"
-PROJECT_CODE="GFS-DEV"
-export WORK_DIR=/gpfs/hps3/stmp/$LOGNAME/reg_tests.grid
+QUEUE="${QUEUE:-debug}"
+PROJECT_CODE="${PROJECT_CODE:-GFS-DEV}"
+export WORK_DIR="${WORK_DIR:-/gpfs/hps3/stmp/$LOGNAME}"
+export WORK_DIR="${WORK_DIR}/reg-tests/grid-gen"
 
 #-----------------------------------------------------------------------------
 # Should not have to change anything below here.
@@ -73,10 +74,17 @@ bsub -e $LOG_FILE -o $LOG_FILE -q $QUEUE -P $PROJECT_CODE -J gfdl.regional -W 0:
 bsub -e $LOG_FILE -o $LOG_FILE -q $QUEUE -P $PROJECT_CODE -J esg.regional -W 0:10 -M 2400 \
         -w 'ended(gfdl.regional)' -extsched 'CRAYLINUX[]' "export NODES=1; $PWD/esg.regional.sh"
 
+#-----------------------------------------------------------------------------
+# Regional GSL gravity wave drag.
+#-----------------------------------------------------------------------------
+
+bsub -e $LOG_FILE -o $LOG_FILE -q $QUEUE -P $PROJECT_CODE -J reg.gsl.gwd -W 0:08 -M 2400 \
+        -w 'ended(esg.regional)' -extsched 'CRAYLINUX[]' "export NODES=1; $PWD/regional.gsl.gwd.sh"
+
 #-----------------------------------------------------------------------------
 # Create summary log.
 #-----------------------------------------------------------------------------
 
-bsub -o $LOG_FILE -q $QUEUE -P $PROJECT_CODE -J summary -R "rusage[mem=100]" -W 0:01 -w 'ended(esg.regional)' "grep -a '<<<' $LOG_FILE >> $SUM_FILE"
+bsub -o $LOG_FILE -q $QUEUE -P $PROJECT_CODE -J summary -R "rusage[mem=100]" -W 0:01 -w 'ended(reg.gsl.gwd)' "grep -a '<<<' $LOG_FILE >> $SUM_FILE"
 
 exit