Port repository to Orion

Updates to compile repo on Orion.  

New scripts to run regression tests on Orion.

New script to run the grid creation step on Orion.

Update link_fixdirs.sh to link to Orion fixed directories.

Remove some references to Tide/Gyre.

See #96 and #134 for more details.

driver_scripts/driver_grid.orion.sh

@@ -0,0 +1,133 @@
+#!/bin/bash
+
+#SBATCH -J fv3_grid_driver
+#SBATCH -A fv3-cpu
+#SBATCH --open-mode=truncate
+#SBATCH -o log.fv3_grid_driver
+#SBATCH -e log.fv3_grid_driver
+#SBATCH --nodes=1 --ntasks-per-node=24
+#SBATCH -q debug
+#SBATCH -t 00:30:00
+
+#-----------------------------------------------------------------------
+# Driver script to create a cubic-sphere based model grid on Orion.
+#
+# Produces the following files (netcdf, each tile in separate file):
+#   1) 'mosaic' and 'grid' files containing lat/lon and other
+#      records that describe the model grid.
+#   2) 'oro' files containing land mask, terrain and gravity
+#      wave drag fields.
+#   3) Surface climo fields, such as soil type, vegetation
+#      greenness and albedo.
+#
+# Note: The sfc_climo_gen program only runs with an
+#       mpi task count that is a multiple of six.  This is
+#       an ESMF library requirement.  Large grids may require
+#       tasks spread across multiple nodes.  The orography code
+#       benefits from threads.
+#
+# To run, do the following:
+#
+#   1) Set "C" resolution, "res" - Example: res=96.
+#   2) Set grid type ("gtype").  Valid choices are
+#         "uniform"  - global uniform grid
+#         "stretch"  - global stretched grid
+#         "nest"     - global stretched grid with nest
+#         "regional_gfdl" - stand-alone gfdl regional grid
+#         "regional_esg"  - stand-alone extended Schmidt
+#                           gnomonic (esg) regional grid
+#   3) For "stretch" and "nest" grids, set the stretching factor -
+#       "stretch_fac", and center lat/lon of highest resolution
+#      tile - "target_lat" and "target_lon".
+#   4) For "nest" grids, set the refinement ratio - "refine_ratio",
+#      the starting/ending i/j index location within the parent
+#      tile - "istart_nest", "jstart_nest", "iend_nest", "jend_nest"
+#   5) For "regional_gfdl" grids, set the "halo".  Default is three
+#      rows/columns.
+#   6) For "regional_esg" grids, set center lat/lon of grid,
+#      - "target_lat/lon" - the i/j dimensions - "i/jdim", the
+#      x/y grid spacing - "delx/y", and halo.
+#   7) Set working directory - TMPDIR - and path to the repository
+#      clone - home_dir.
+#   8) Submit script: "sbatch $script".
+#   9) All files will be placed in "out_dir".
+#
+#-----------------------------------------------------------------------
+
+set -x
+
+source ../sorc/machine-setup.sh > /dev/null 2>&1
+source ../modulefiles/build.$target
+module list
+
+#-----------------------------------------------------------------------
+# Set grid specs here.
+#-----------------------------------------------------------------------
+
+export gtype=uniform   # 'uniform', 'stretch', 'nest', 
+                       # 'regional_gfdl', 'regional_esg'
+
+if [ $gtype = uniform ]; then
+  export res=96
+elif [ $gtype = stretch ]; then
+  export res=96
+  export stretch_fac=1.5       # Stretching factor for the grid
+  export target_lon=-97.5      # Center longitude of the highest resolution tile
+  export target_lat=35.5       # Center latitude of the highest resolution tile
+elif [ $gtype = nest ] || [ $gtype = regional_gfdl ]; then
+  export res=96
+  export stretch_fac=1.5       # Stretching factor for the grid
+  export target_lon=-97.5      # Center longitude of the highest resolution tile
+  export target_lat=35.5       # Center latitude of the highest resolution tile
+  export refine_ratio=3        # The refinement ratio
+  export istart_nest=27        # Starting i-direction index of nest grid in parent tile supergrid
+  export jstart_nest=37        # Starting j-direction index of nest grid in parent tile supergrid
+  export iend_nest=166         # Ending i-direction index of nest grid in parent tile supergrid
+  export jend_nest=164         # Ending j-direction index of nest grid in parent tile supergrid
+  export halo=3                # Lateral boundary halo
+elif [ $gtype = regional_esg ] ; then
+  export res=-999              # equivalent res is computed.
+  export target_lon=-97.5      # Center longitude of grid
+  export target_lat=35.5       # Center latitude of grid
+  export idim=301              # Dimension of grid in 'i' direction
+  export jdim=200              # Dimension of grid in 'j' direction
+  export delx=0.0585           # Grid spacing (in degrees) in the 'i' direction
+                               # on the SUPERGRID (which has twice the resolution of
+                               # the model grid).  The physical grid spacing in the 'i'
+                               # direction is related to delx as follows:
+                               #    distance = 2*delx*(circumf_Earth/360 deg)
+  export dely=0.0585           # Grid spacing (in degrees) in the 'j' direction.
+  export halo=3                # number of row/cols for halo
+fi
+
+#-----------------------------------------------------------------------
+# Check paths.  
+#   home_dir - location of repository.
+#   TMPDIR   - working directory.
+#   out_dir  - where files will be placed upon completion. 
+#-----------------------------------------------------------------------
+
+export home_dir=$SLURM_SUBMIT_DIR/..
+export TMPDIR=/work/noaa/stmp/$LOGNAME/fv3_grid.$gtype
+export out_dir=/work/noaa/stmp/$LOGNAME/my_grids
+
+#-----------------------------------------------------------------------
+# Should not need to change anything below here.
+#-----------------------------------------------------------------------
+
+export APRUN=time
+export APRUN_SFC=srun
+export OMP_NUM_THREADS=24
+export OMP_STACKSIZE=2048m
+export machine=ORION
+
+ulimit -a
+ulimit -s 199000000
+
+#-----------------------------------------------------------------------
+# Start script.
+#-----------------------------------------------------------------------
+
+$home_dir/ush/fv3gfs_driver_grid.sh
+
+exit

fix/link_fixdirs.sh

@@ -9,20 +9,20 @@ machine=${2}
 if [ $# -lt 2 ]; then
     set +x
     echo '***ERROR*** must specify two arguements: (1) RUN_ENVIR, (2) machine'
-    echo ' Syntax: link_fv3gfs.sh ( nco | emc ) ( cray | dell | hera  | jet )'
+    echo ' Syntax: link_fv3gfs.sh ( nco | emc ) ( cray | dell | hera  | jet | orion )'
     exit 1
 fi
 
 if [ $RUN_ENVIR != emc -a $RUN_ENVIR != nco ]; then
     set +x
     echo '***ERROR*** unsupported run environment'
-    echo 'Syntax: link_fv3gfs.sh ( nco | emc ) ( cray | dell | hera | jet )'
+    echo 'Syntax: link_fv3gfs.sh ( nco | emc ) ( cray | dell | hera | jet | orion )'
     exit 1
 fi
-if [ $machine != cray -a $machine != hera -a $machine != dell -a $machine != jet ]; then
+if [ $machine != cray -a $machine != hera -a $machine != dell -a $machine != jet -a $machine != orion ]; then
     set +x
     echo '***ERROR*** unsupported machine'
-    echo 'Syntax: link_fv3gfs.sh ( nco | emc ) ( cray | dell | hera | jet )'
+    echo 'Syntax: link_fv3gfs.sh ( nco | emc ) ( cray | dell | hera | jet | orion )'
     exit 1
 fi
 
@@ -43,6 +43,8 @@ elif [ $machine = "hera" ]; then
     FIX_DIR="/scratch1/NCEPDEV/global/glopara/fix"
 elif [ $machine = "jet" ]; then
     FIX_DIR="/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix"
+elif [ $machine = "orion" ]; then
+    FIX_DIR="/work/noaa/global/glopara/fix"
 fi
 for dir in fix_am fix_fv3 fix_orog fix_fv3_gmted2010 fix_sfc_climo; do
     [[ -d $dir ]] && rm -rf $dir

modulefiles/build.orion

@@ -10,22 +10,19 @@ module use -a /apps/contrib/NCEPLIBS/orion/modulefiles
 module load w3nco/2.1.0
 module load nemsio/2.3.0
 module load bacio/2.2.0
-module load sp/2.1.0
 module load sfcio/1.2.0
 module load sigio/2.2.0
 module load gfsio/1.2.0
-module load w3emc/2.5.0
+module load w3emc/2.4.0
 module load ip/3.1.0
 module load nemsiogfs/2.3.0
 module load landsfcutil/2.2.0
-
-# George V's version.
-module use -a /apps/contrib/NCEPLIBS/lib/modulefiles
-module load g2-intel-sandybridge/2.5.0
-
+module load g2/3.1.1
+module load sp/2.0.3
 export Jasper_ROOT="/apps/jasper-1.900.1"
 
-module load netcdf/4.7.2-parallel
-module load esmf/8.0.0_ParallelNetCDF
+module use -a /apps/contrib/NCEPLIBS/lib/modulefiles
+module load netcdfp/4.7.4.release
+module load esmflocal/8_0_0.release
 
 export WGRIB2_ROOT="/work/noaa/da/ggayno/save/wgrib2"

modulefiles/module-setup.sh.inc

@@ -45,12 +45,6 @@ elif [[ -d /gpfs/hps && -e /etc/SuSE-release ]] ; then
     module use /gpfs/hps/nco/ops/nwprod/modulefiles
     module use /gpfs/hps/nco/ops/nwprod/lib/modulefiles
     module use /usrx/local/prod/modulefiles
-elif [[ -d /dcom && -d /hwrf ]] ; then
-    # We are on NOAA Tide or Gyre
-    if ( ! eval module help > /dev/null 2>&1 ) ; then
-        source /usrx/local/Modules/default/init/$__ms_shell
-    fi
-    module purge
 elif [[ -L /usrx && "$( readlink /usrx 2> /dev/null )" =~ dell ]] ; then
     # We are on NOAA Mars or Venus
     if ( ! eval module help > /dev/null 2>&1 ) ; then

reg_tests/chgres_cube/c192.fv3.history.sh

@@ -50,8 +50,17 @@ echo "Ending at: " `date`
 
 #-----------------------------------------------------------------------------
 # Compare output from chgres to baseline set of data.
+#
+# orion's nccmp utility does not work with the netcdf
+# required to run ufs_utils.  So swap it.
 #-----------------------------------------------------------------------------
 
+machine=${machine:-NULL}
+if [ $machine == 'orion' ]; then
+  module unload netcdfp/4.7.4.release
+  module load netcdf/4.7.2
+fi
+
 cd $DATA
 
 test_failed=0

reg_tests/chgres_cube/c192.gfs.grib2.sh

@@ -45,8 +45,17 @@ echo "Ending at: " `date`
 
 #-----------------------------------------------------------------------------
 # Compare output from chgres to baseline set of data.
+#
+# orion's nccmp utility does not work with the netcdf
+# required to run ufs_utils.  So swap it.
 #-----------------------------------------------------------------------------
 
+machine=${machine:-NULL}
+if [ $machine == 'orion' ]; then
+  module unload netcdfp/4.7.4.release
+  module load netcdf/4.7.2
+fi
+
 cd $DATA
 
 test_failed=0

reg_tests/chgres_cube/c96.fv3.nemsio.sh

@@ -41,8 +41,17 @@ echo "Ending at: " `date`
 
 #-----------------------------------------------------------------------------
 # Compare output from chgres to baseline set of data.
+#
+# orion's nccmp utility does not work with the netcdf
+# required to run ufs_utils.  So swap it.
 #-----------------------------------------------------------------------------
 
+machine=${machine:-NULL}
+if [ $machine == 'orion' ]; then
+  module unload netcdfp/4.7.4.release
+  module load netcdf/4.7.2
+fi
+
 cd $DATA
 
 test_failed=0

reg_tests/chgres_cube/c96.fv3.netcdf.sh

@@ -42,8 +42,17 @@ echo "Ending at: " `date`
 
 #-----------------------------------------------------------------------------
 # Compare output from chgres to baseline set of data.
+#
+# orion's nccmp utility does not work with the netcdf
+# required to run ufs_utils.  So swap it.
 #-----------------------------------------------------------------------------
 
+machine=${machine:-NULL}
+if [ $machine == 'orion' ]; then
+  module unload netcdfp/4.7.4.release
+  module load netcdf/4.7.2
+fi
+
 cd $DATA
 
 test_failed=0

reg_tests/chgres_cube/c96.fv3.restart.sh

@@ -51,8 +51,17 @@ echo "Ending at: " `date`
 
 #-----------------------------------------------------------------------------
 # Compare output from chgres to baseline set of data.
+#
+# orion's nccmp utility does not work with the netcdf
+# required to run ufs_utils.  So swap it.
 #-----------------------------------------------------------------------------
 
+machine=${machine:-NULL}
+if [ $machine == 'orion' ]; then
+  module unload netcdfp/4.7.4.release
+  module load netcdf/4.7.2
+fi
+
 cd $DATA
 
 test_failed=0

reg_tests/chgres_cube/c96.gfs.nemsio.sh

@@ -45,8 +45,17 @@ echo "Ending at: " `date`
 
 #-----------------------------------------------------------------------------
 # Compare output from chgres to baseline set of data.
+#
+# orion's nccmp utility does not work with the netcdf
+# required to run ufs_utils.  So swap it.
 #-----------------------------------------------------------------------------
 
+machine=${machine:-NULL}
+if [ $machine == 'orion' ]; then
+  module unload netcdfp/4.7.4.release
+  module load netcdf/4.7.2
+fi
+
 cd $DATA
 
 test_failed=0

reg_tests/chgres_cube/c96.gfs.sigio.sh

@@ -9,6 +9,12 @@
 
 set -x
 
+# Orion won't let me set the ulimit in the driver script.  Set it here.
+machine=${machine:-NULL}
+if [ $machine == 'orion' ]; then
+  ulimit -s 199000000
+fi
+
 export DATA=$OUTDIR/c96_gfs_sigio
 rm -fr $DATA
 
@@ -41,8 +47,16 @@ echo "Ending at: " `date`
 
 #-----------------------------------------------------------------------------
 # Compare output from chgres to baseline set of data.
+#
+# orion's nccmp utility does not work with the netcdf
+# required to run ufs_utils.  So swap it.
 #-----------------------------------------------------------------------------
 
+if [ $machine == 'orion' ]; then
+  module unload netcdfp/4.7.4.release
+  module load netcdf/4.7.2
+fi
+
 cd $DATA
 
 test_failed=0

reg_tests/chgres_cube/c96.regional.sh

@@ -47,8 +47,17 @@ echo "Ending at: " `date`
 
 #-----------------------------------------------------------------------------
 # Compare output from chgres to baseline set of data.
+#
+# orion's nccmp utility does not work with the netcdf
+# required to run ufs_utils.  So swap it.
 #-----------------------------------------------------------------------------
 
+machine=${machine:-NULL}
+if [ $machine == 'orion' ]; then
+  module unload netcdfp/4.7.4.release
+  module load netcdf/4.7.2
+fi
+
 cd $DATA
 
 test_failed=0