diff --git a/driver_scripts/driver_grid.theia.sh b/driver_scripts/driver_grid.hera.sh similarity index 93% rename from driver_scripts/driver_grid.theia.sh rename to driver_scripts/driver_grid.hera.sh index 88096511d..ad93e30e7 100644 --- a/driver_scripts/driver_grid.theia.sh +++ b/driver_scripts/driver_grid.hera.sh @@ -10,7 +10,7 @@ #SBATCH -t 00:30:00 #----------------------------------------------------------------------- -# Driver script to create a cubic-sphere based model grid on Theia. +# Driver script to create a cubic-sphere based model grid on Hera. # # Produces the following files (netcdf, each tile in separate file): # 1) 'mosaic' and 'grid' files containing lat/lon and other @@ -53,10 +53,10 @@ set -x . /apps/lmod/lmod/init/sh module purge -module load intel/16.1.150 -module load impi -module load hdf5/1.8.14 -module load netcdf/4.3.0 +module load intel/18.0.5.274 +module load impi/2018.0.4 +module load hdf5/1.10.5 +module load netcdf/4.7.0 module list #----------------------------------------------------------------------- @@ -90,8 +90,8 @@ fi #----------------------------------------------------------------------- export home_dir=$SLURM_SUBMIT_DIR/.. -export TMPDIR=/scratch3/NCEPDEV/stmp1/$LOGNAME/fv3_grid.$gtype -export out_dir=/scratch3/NCEPDEV/stmp1/$LOGNAME/C${res} +export TMPDIR=/scratch2/NCEPDEV/stmp1/$LOGNAME/fv3_grid.$gtype +export out_dir=/scratch2/NCEPDEV/stmp1/$LOGNAME/C${res} #----------------------------------------------------------------------- # Should not need to change anything below here. @@ -101,7 +101,7 @@ export APRUN=time export APRUN_SFC=srun export OMP_NUM_THREADS=24 export OMP_STACKSIZE=2048m -export machine=THEIA +export machine=HERA ulimit -a ulimit -s unlimited diff --git a/modulefiles/chgres_cube.theia b/modulefiles/chgres_cube.hera similarity index 60% rename from modulefiles/chgres_cube.theia rename to modulefiles/chgres_cube.hera index 8e948725d..238198dca 100644 --- a/modulefiles/chgres_cube.theia +++ b/modulefiles/chgres_cube.hera @@ -1,29 +1,28 @@ #%Module##################################################### -## chgres build module for Theia +## Build module for Hera ############################################################# -module load intel/18.1.163 -module load impi/5.1.1.109 -module load netcdf/4.3.0 +module load intel/18.0.5.274 +module load impi/2018.0.4 -module use /scratch4/NCEPDEV/nems/noscrub/emc.nemspara/soft/modulefiles -module load esmf/8.0.0bs20 +module use -a $MOD_PATH +module load w3nco/2.0.6 +module load nemsio/2.2.3 +module load bacio/2.0.2 +module load sp/2.0.2 +module load sfcio/1.1.0 +module load sigio/2.1.0 -module use -a /scratch3/NCEPDEV/nwprod/lib/modulefiles -module load w3nco/v2.0.6 -module load wgrib2/2.0.8 -module load nemsio/v2.2.3 -module load bacio/v2.0.2 -module load sp/v2.0.2 -module load sfcio/v1.0.0 -module load sigio/v2.0.1 +export NCEPLIBS=/scratch1/NCEPDEV/global/gwv/l819/lib +module use -a $NCEPLIBS/modulefiles +module load esmflocal/8_0_48b.netcdf47 -export FCOMP=mpiifort -export FFLAGS="-O3 -fp-model precise -g -traceback -r8 -i4 -qopenmp -convert big_endian -assume byterecl" +module load wgrib2/2.0.8 export WGRIB2API_LIB="/apps/wgrib2/2.0.8/intel/18.0.3.222/lib/libwgrib2_api.a" export WGRIB2API_INC="/apps/wgrib2/2.0.8/intel/18.0.3.222/lib" export WGRIB2_LIB="/apps/wgrib2/2.0.8/intel/18.0.3.222/lib/libwgrib2.a" - +export FCOMP=mpiifort +export FFLAGS="-O3 -fp-model precise -g -traceback -r8 -i4 -qopenmp -convert big_endian -assume byterecl" # for debugging #export FFLAGS="-O0 -g -traceback -r8 -i4 -qopenmp -convert big_endian -check bounds -warn unused -assume byterecl" diff --git a/modulefiles/fv3gfs/fre-nctools.theia b/modulefiles/fv3gfs/fre-nctools.hera similarity index 59% rename from modulefiles/fv3gfs/fre-nctools.theia rename to modulefiles/fv3gfs/fre-nctools.hera index c1b678411..e035db231 100644 --- a/modulefiles/fv3gfs/fre-nctools.theia +++ b/modulefiles/fv3gfs/fre-nctools.hera @@ -1,7 +1,9 @@ #%Module##################################################### ## Module file for fre-nctools ############################################################# -module load intel/14.0.2 -module load impi/4.1.3.048 -module load netcdf/4.3.0 -module load hdf5/1.8.14 + +module load intel/18.0.5.274 +module load impi/2018.0.4 + +module load netcdf/4.7.0 +module load hdf5/1.10.5 diff --git a/modulefiles/fv3gfs/global_chgres.hera b/modulefiles/fv3gfs/global_chgres.hera new file mode 100644 index 000000000..b91ac5e09 --- /dev/null +++ b/modulefiles/fv3gfs/global_chgres.hera @@ -0,0 +1,29 @@ +#%Module##################################################### +## global_chgres component - hera +############################################################# + +# Loading Intel Compiler Suite +module load intel/18.0.5.274 +module load impi/2018.0.4 + +module load netcdf/4.7.0 +module load hdf5/1.10.5 + +# Loding nceplibs modules +module use -a $MOD_PATH +module load sigio/2.1.0 +module load w3nco/2.0.6 +module load w3emc/2.3.0 +module load sp/2.0.2 +module load bacio/2.0.2 +module load nemsio/2.2.3 +module load nemsiogfs/2.2.0 +module load ip/3.0.1 +module load sfcio/1.1.0 +module load gfsio/1.1.0 +module load landsfcutil/2.1.0 + +export NETCDF_INCLUDE="-I${NETCDF}/include" +export NETCDF_LDFLAGS_F="-L${NETCDF}/lib -lnetcdf -lnetcdff -L${HDF5}/lib -lhdf5 -lhdf5_fortran" + +set FCMP ifort diff --git a/modulefiles/fv3gfs/global_chgres.theia b/modulefiles/fv3gfs/global_chgres.theia deleted file mode 100644 index fa764895f..000000000 --- a/modulefiles/fv3gfs/global_chgres.theia +++ /dev/null @@ -1,31 +0,0 @@ -#%Module##################################################### -## global_chgres component - theia -############################################################# - -# Loading Intel Compiler Suite -module load intel/14.0.2 -module load impi/5.1.2.150 - -# Loding nceplibs modules -module use -a $MOD_PATH -module load sigio/v2.0.1 -module load w3nco/v2.0.6 -module load w3emc/v2.2.0 -module load sp/v2.0.2 -module load bacio/v2.0.2 -module load nemsio/v2.2.3 -module load nemsiogfs/v2.0.1 -export NEMSIOGFS_INC=/scratch4/NCEPDEV/global/save/glopara/svn/nceplibs/branches/nemsiogfsv2.0.1/include/nemsiogfs -export NEMSIOGFS_LIB=/scratch4/NCEPDEV/global/save/glopara/svn/nceplibs/branches/nemsiogfsv2.0.1/libnemsiogfs.a - -module load ip/v3.0.0 -module load sfcio/v1.0.0 -module load gfsio/v1.1.0 -module load landsfcutil/v2.1.0 -module load netcdf/4.3.0 -module load hdf5/1.8.14 - -export NETCDF_INCLUDE="-I${NETCDF}/include" -export NETCDF_LDFLAGS_F="-L${NETCDF}/lib -lnetcdf -lnetcdff -L${HDF5}/lib -lhdf5 -lhdf5_fortran" - -set FCMP ifort diff --git a/modulefiles/fv3gfs/global_chgres.wcoss b/modulefiles/fv3gfs/global_chgres.wcoss index 80e60faaf..1d30465a5 100644 --- a/modulefiles/fv3gfs/global_chgres.wcoss +++ b/modulefiles/fv3gfs/global_chgres.wcoss @@ -3,7 +3,7 @@ ############################################################# # Loading Intel Compiler Suite -module load ics/14.0.1 +module load ics/15.0.6 # Loading nceplibs modules module load sigio/v2.1.0 diff --git a/modulefiles/fv3gfs/global_cycle.theia b/modulefiles/fv3gfs/global_cycle.hera similarity index 55% rename from modulefiles/fv3gfs/global_cycle.theia rename to modulefiles/fv3gfs/global_cycle.hera index 2f0f82cec..5f3afa8d5 100644 --- a/modulefiles/fv3gfs/global_cycle.theia +++ b/modulefiles/fv3gfs/global_cycle.hera @@ -1,20 +1,21 @@ #%Module##################################################### -## global_cycle component - theia +## global_cycle component - hera ############################################################# # Loading Intel Compiler Suite -module load intel/16.1.150 -module load impi +module load intel/18.0.5.274 +module load impi/2018.0.4 -# Loding nceplibs modules -module use -a /scratch3/NCEPDEV/nwprod/lib/modulefiles -module load w3nco/v2.0.6 -module load sp/v2.0.2 -module load bacio/v2.0.1 -module load ip/v3.0.0 -module load netcdf/4.3.0 -module load hdf5/1.8.14 +module load netcdf/4.7.0 +module load hdf5/1.10.5 export NETCDF_INCLUDE="-I${NETCDF}/include" export NETCDF_LDFLAGS_F="-L${NETCDF}/lib -lnetcdf -lnetcdff -L${HDF5}/lib -lhdf5 -lhdf5_fortran" +# Loding nceplibs modules +module use -a $MOD_PATH +module load w3nco/2.0.6 +module load sp/2.0.2 +module load bacio/2.0.2 +module load ip/3.0.1 + export FCMP=mpiifort diff --git a/modulefiles/fv3gfs/nst_tf_chg.theia b/modulefiles/fv3gfs/nst_tf_chg.hera similarity index 65% rename from modulefiles/fv3gfs/nst_tf_chg.theia rename to modulefiles/fv3gfs/nst_tf_chg.hera index 9a8829df8..2bbd3a802 100644 --- a/modulefiles/fv3gfs/nst_tf_chg.theia +++ b/modulefiles/fv3gfs/nst_tf_chg.hera @@ -1,17 +1,17 @@ #%Module##################################################### -## gaussian_sfcanl build module for Theia +## Build module for Hera ############################################################# # Loading Intel Compiler Suite -module load intel/15.1.133 +module load intel/18.0.5.274 # Loding nceplibs modules -module use -a /scratch3/NCEPDEV/nwprod/lib/modulefiles -module load w3nco/v2.0.6 -module load bacio/v2.0.2 -module load nemsio/v2.2.3 -module load netcdf/4.3.0 -module load hdf5/1.8.14 +module use -a $MOD_PATH +module load w3nco/2.0.6 +module load bacio/2.0.2 +module load nemsio/2.2.3 +module load netcdf/4.6.1 +module load hdf5/1.10.4 export NETCDF_INCLUDE="-I${NETCDF}/include" export NETCDF_LDFLAGS_F="-L${NETCDF}/lib -lnetcdf -lnetcdff -L${HDF5}/lib -lhdf5 -lhdf5_fortran" diff --git a/modulefiles/fv3gfs/orog.theia b/modulefiles/fv3gfs/orog.hera similarity index 54% rename from modulefiles/fv3gfs/orog.theia rename to modulefiles/fv3gfs/orog.hera index b0543289b..0780cece2 100644 --- a/modulefiles/fv3gfs/orog.theia +++ b/modulefiles/fv3gfs/orog.hera @@ -2,15 +2,14 @@ ## Module file for orog ############################################################# # Loading Intel Compiler Suite -module unload intel -module load intel/16.1.150 +module load intel/18.0.5.274 -module load netcdf/4.3.0 +module load netcdf/4.7.0 # Loding nceplibs modules module use -a $MOD_PATH -module load ip/v2.0.0 -module load sp/v2.0.2 -module load w3emc/v2.2.0 -module load w3nco/v2.0.6 -module load bacio/v2.0.2 +module load ip/3.0.1 +module load sp/2.0.2 +module load w3emc/2.3.0 +module load w3nco/2.0.6 +module load bacio/2.0.2 diff --git a/modulefiles/module-setup.csh.inc b/modulefiles/module-setup.csh.inc deleted file mode 100644 index a93b21a3c..000000000 --- a/modulefiles/module-setup.csh.inc +++ /dev/null @@ -1,89 +0,0 @@ -set __ms_shell=csh - -eval "if ( -d / ) set __ms_shell=tcsh" - -if ( { test -d /lfs3 } ) then - if ( ! { module help >& /dev/null } ) then - source /apps/lmod/lmod/init/$__ms_shell - endif - module purge -else if ( { test -d /scratch3 } ) then - # We are on NOAA Theia - if ( ! { module help >& /dev/null } ) then - source /apps/lmod/lmod/init/$__ms_shell - endif - module purge -else if ( { test -d /gpfs/hps -a -e /etc/SuSE-release } ) then - # We are on NOAA Luna or Surge - if ( ! { module help >& /dev/null } ) then - source /opt/modules/default/init/$__ms_shell - endif - module purge - module purge - unset _LMFILES_ - unset LOADEDMODULES - module use /opt/modulefiles - module use /opt/cray/ari/modulefiles - module use /opt/cray/craype/default/alt-modulefiles - module use /opt/cray/alt-modulefiles - module use /gpfs/hps/nco/ops/nwprod/modulefiles - module use /gpfs/hps/nco/ops/nwprod/lib/modulefiles - module use /usrx/local/prod/modulefiles -else if ( { test -d /dcom -a -d /hwrf } ) then - # We are on NOAA Tide or Gyre - if ( ! { module help >& /dev/null } ) then - source /usrx/local/Modules/default/init/$__ms_shell - endif - module purge -else if ( { test -L /usrx && sh -c "readlink /usrx 2> /dev/null | grep dell" } ) then - # We are on WCOSS Mars or Venus - if ( ! { module help >& /dev/null } ) then - source /usrx/local/prod/lmod/lmod/init/$__ms_shell - endif - module purge -else if ( { test -d /glade } ) then - # We are on NCAR Yellowstone - if ( ! { module help >& /dev/null } ) then - source /usr/share/Modules/init/$__ms_shell - endif - module purge -else if ( { test -d /lustre -a -d /ncrc } ) then - # We are on GAEA. - if ( ! { module help >& /dev/null } ) then - # We cannot simply load the module command. The GAEA - # /etc/csh.login modifies a number of module-related variables - # before loading the module command. Without those variables, - # the module command fails. Hence we actually have to source - # /etc/csh.login here. - source /etc/csh.login - set __ms_source_etc_csh_login=yes - else - set __ms_source_etc_csh_login=no - endif - module purge - unsetenv _LMFILES_ - unsetenv _LMFILES_000 - unsetenv _LMFILES_001 - unsetenv LOADEDMODULES - module load modules - if ( { test -d /opt/cray/ari/modulefiles } ) then - module use -a /opt/cray/ari/modulefiles - endif - if ( { test -d /opt/cray/pe/ari/modulefiles } ) then - module use -a /opt/cray/pe/ari/modulefiles - endif - if ( { test -d /opt/cray/pe/craype/default/modulefiles } ) then - module use -a /opt/cray/pe/craype/default/modulefiles - endif - setenv NCEPLIBS /lustre/f1/pdata/ncep_shared/NCEPLIBS/lib - if ( { test -d /lustre/f1/pdata/ncep_shared/NCEPLIBS/lib } ) then - module use $NCEPLIBS/modulefiles - endif - if ( "$__ms_source_etc_csh_login" == yes ) then - source /etc/csh.login - unset __ms_source_etc_csh_login - endif -else - # Workaround for csh limitation. Use sh to print to stderr. - sh -c 'echo WARNING: UNKNOWN PLATFORM 1>&2' -endif diff --git a/modulefiles/module-setup.sh.inc b/modulefiles/module-setup.sh.inc index 6815b1a46..d4faa4f64 100644 --- a/modulefiles/module-setup.sh.inc +++ b/modulefiles/module-setup.sh.inc @@ -22,8 +22,8 @@ if [[ -d /lfs3 ]] ; then source /apps/lmod/lmod/init/$__ms_shell fi module purge -elif [[ -d /scratch3 ]] ; then - # We are on NOAA Theia +elif [[ -d /scratch1 ]] ; then + # We are on NOAA Hera if ( ! eval module help > /dev/null 2>&1 ) ; then source /apps/lmod/lmod/init/$__ms_shell fi diff --git a/modulefiles/module_base.hera b/modulefiles/module_base.hera new file mode 100644 index 000000000..07e78cfc9 --- /dev/null +++ b/modulefiles/module_base.hera @@ -0,0 +1,11 @@ +#%Module###################################################################### +## +## FV3GFS prerequisites +## + +module load intel/18.0.5.274 +module load impi/2018.0.4 +module load hpss/hpss +module load netcdf/4.7.0 +module load nco/4.7.0 +module load hdf5/1.10.5 diff --git a/modulefiles/module_base.theia b/modulefiles/module_base.theia deleted file mode 100644 index 13fad8fd5..000000000 --- a/modulefiles/module_base.theia +++ /dev/null @@ -1,19 +0,0 @@ -#%Module###################################################################### -## -## FV3GFS prerequisites -## - -module load intel/16.1.150 -module load impi/5.1.2.150 -module load wgrib2 -module load hpss -module load netcdf -module load nco -module load hdf5/1.8.14 -module load gempak/6.7.0 - -module use /scratch3/NCEPDEV/nwprod/lib/modulefiles -module load EnvVars/v1.0.2 - -module use /scratch4/NCEPDEV/nems/noscrub/emc.nemspara/soft/modulefiles -module load prod_util diff --git a/modulefiles/module_nemsutil.theia b/modulefiles/module_nemsutil.hera similarity index 66% rename from modulefiles/module_nemsutil.theia rename to modulefiles/module_nemsutil.hera index 45685118e..4d1bfd5c4 100644 --- a/modulefiles/module_nemsutil.theia +++ b/modulefiles/module_nemsutil.hera @@ -3,13 +3,13 @@ ############################################################# # Loading Intel Compiler Suite -module load intel/14.0.2 -module load impi/5.1.2.150 +module load intel/18.0.5.274 +module load impi/2018.0.4 # Loding nceplibs modules module use -a $MOD_PATH -module load w3nco/v2.0.6 -module load bacio/v2.0.1 -module load nemsio/v2.2.1 +module load w3nco/2.0.6 +module load bacio/2.0.2 +module load nemsio/2.2.3 export FCMP=ifort diff --git a/modulefiles/modulefile.global_emcsfc_ice_blend.hera b/modulefiles/modulefile.global_emcsfc_ice_blend.hera new file mode 100644 index 000000000..2305392cf --- /dev/null +++ b/modulefiles/modulefile.global_emcsfc_ice_blend.hera @@ -0,0 +1,17 @@ +#%Module##################################################### +## emcsfc_ice_blend build module for Hera +############################################################# + +module load intel/18.0.5.274 +export FCOMP=ifort +export FFLAGS="-O0 -i4" + +# load ncep library modules + +module use -a $MOD_PATH +module load w3nco/2.0.6 +module load bacio/2.0.2 +module load jasper/1.900.1 +module load z/1.2.11 +module load png/1.2.44 +module load g2/2.5.0 diff --git a/modulefiles/modulefile.global_emcsfc_ice_blend.theia b/modulefiles/modulefile.global_emcsfc_ice_blend.theia deleted file mode 100644 index d42fb6ac8..000000000 --- a/modulefiles/modulefile.global_emcsfc_ice_blend.theia +++ /dev/null @@ -1,17 +0,0 @@ -#%Module##################################################### -## emcsfc_ice_blend build module for Theia -############################################################# - -module load intel/14.0.2 -export FCOMP=ifort -export FFLAGS="-O0 -i4" - -# load ncep library modules - -module use -a /scratch3/NCEPDEV/nwprod/lib/modulefiles -module load w3nco/v2.0.6 -module load bacio/v2.0.1 -module load jasper -module load z -module load png -module load g2/v2.5.0 diff --git a/modulefiles/modulefile.global_emcsfc_snow2mdl.hera b/modulefiles/modulefile.global_emcsfc_snow2mdl.hera new file mode 100644 index 000000000..193f6247d --- /dev/null +++ b/modulefiles/modulefile.global_emcsfc_snow2mdl.hera @@ -0,0 +1,23 @@ +#%Module##################################################### +## emcsfc_snow2mdl build module for Hera +############################################################# + +# load intel compiler + +module load intel/18.0.5.274 + +# load ncep library modules + +module use -a $MOD_PATH +module load ip/3.0.1 +module load sp/2.0.2 +module load w3nco/2.0.6 +module load bacio/2.0.2 +module load landsfcutil/2.1.0 +module load jasper/1.900.1 +module load z/1.2.11 +module load png/1.2.44 +module load g2/2.5.0 + +export FCOMP=ifort +export FFLAGS="-O0 -r8 -i4 -FR -I${IP_INCd} -qopenmp -convert big_endian -assume byterecl" diff --git a/modulefiles/modulefile.global_emcsfc_snow2mdl.theia b/modulefiles/modulefile.global_emcsfc_snow2mdl.theia deleted file mode 100644 index b364fbe47..000000000 --- a/modulefiles/modulefile.global_emcsfc_snow2mdl.theia +++ /dev/null @@ -1,23 +0,0 @@ -#%Module##################################################### -## emcsfc_snow2mdl build module for Theia -############################################################# - -# load intel compiler - -module load intel/14.0.2 - -# load ncep library modules - -module use -a /scratch3/NCEPDEV/nwprod/lib/modulefiles -module load ip/v3.0.0 -module load sp/v2.0.2 -module load w3nco/v2.0.6 -module load bacio/v2.0.1 -module load jasper -module load z -module load png -module load g2/v2.5.0 -module load landsfcutil/v2.1.0 - -export FCOMP=ifort -export FFLAGS="-O0 -r8 -i4 -FR -I${IP_INCd} -openmp -convert big_endian -assume byterecl" diff --git a/modulefiles/modulefile.sfc_climo_gen.theia b/modulefiles/modulefile.sfc_climo_gen.hera similarity index 50% rename from modulefiles/modulefile.sfc_climo_gen.theia rename to modulefiles/modulefile.sfc_climo_gen.hera index 39e306c5e..93aff0a2e 100644 --- a/modulefiles/modulefile.sfc_climo_gen.theia +++ b/modulefiles/modulefile.sfc_climo_gen.hera @@ -1,12 +1,13 @@ #%Module##################################################### -## Module file for sfc_climo_gen +## Module file for sfc_climo_gen (Hera) ############################################################# -module load intel/15.1.133 -module load impi/5.1.1.109 -module load netcdf/4.3.0 -module use /scratch4/NCEPDEV/nems/noscrub/emc.nemspara/soft/modulefiles -module load esmf/7.1.0r +module load intel/18.0.5.274 +module load impi/2018.0.4 + +export NCEPLIBS=/scratch1/NCEPDEV/global/gwv/l819/lib +module use -a $NCEPLIBS/modulefiles +module load esmflocal/8_0_48b.netcdf47 export FCOMP=mpiifort export FFLAGS="-O3 -fp-model=precise -g -traceback -r8 -i4 -convert big_endian" diff --git a/reg_tests/chgres_cube/README b/reg_tests/chgres_cube/README new file mode 100644 index 000000000..49551cd28 --- /dev/null +++ b/reg_tests/chgres_cube/README @@ -0,0 +1,23 @@ +This series of regression tests must be run before any +changes to the chgres_cube program are merged to the +main line of development. Any test failures must +be resolved or explained before merging. + +To run the regression tests: + +1) Build chgres_cube program. Go to ./sorc and + invoke 'build_chgres_cube.sh' with no arguments. +2) Invoke driver script for your machine. See script + prolog for details. Supported machines are: + - Hera (driver.hera.sh) + - WCOSS-Cray (driver.cray.sh) + - WCOSS-Dell (driver.dell.sh) + - Jet (driver.jet.sh) + +A series of daisy-chained tests will run. A test fails +if the output files differ from the baseline set of files +as determined by the 'nccmp' utility. + +Log output from the tests will be in "regression.log". + +A summary of results will be in "summary.log". diff --git a/reg_tests/chgres_cube/c192.fv3.history.sh b/reg_tests/chgres_cube/c192.fv3.history.sh new file mode 100755 index 000000000..72776b833 --- /dev/null +++ b/reg_tests/chgres_cube/c192.fv3.history.sh @@ -0,0 +1,76 @@ +#!/bin/bash + +#----------------------------------------------------------------------------- +# Invoke chgres to create C192 coldstart files using FV3 tiled history files +# as input. The coldstart files are then compared to baseline files +# using the 'nccmp' utility. This script is run by the machine specific +# driver script. +#----------------------------------------------------------------------------- + +set -x + +export DATA=$OUTDIR/c192_fv3_history +rm -fr $DATA + +export CRES=192 +export FIXfv3=${HOMEreg}/fix/C192 +export COMIN=${HOMEreg}/input_data/fv3.history + +# Pay attention to the quotes. Dont start/end with double quote. +export ATM_FILES_INPUT='dynf000.tile1.nc","dynf000.tile2.nc","dynf000.tile3.nc","dynf000.tile4.nc","dynf000.tile5.nc","dynf000.tile6.nc' +export SFC_FILES_INPUT='phyf000.tile1.nc","phyf000.tile2.nc","phyf000.tile3.nc","phyf000.tile4.nc","phyf000.tile5.nc","phyf000.tile6.nc' +export VCOORD_FILE=${HOMEufs}/fix/fix_am/global_hyblev.l64.txt +export INPUT_TYPE='history' +export MOSAIC_FILE_INPUT_GRID="${HOMEreg}/fix/C96/C96_mosaic.nc" +export OROG_DIR_INPUT_GRID=${HOMEreg}/fix/C96 +export OROG_FILES_INPUT_GRID='C96_oro_data.tile1.nc","C96_oro_data.tile2.nc","C96_oro_data.tile3.nc","C96_oro_data.tile4.nc","C96_oro_data.tile5.nc","C96_oro_data.tile6.nc' +export TRACERS_TARGET='"sphum","liq_wat","o3mr"' +export TRACERS_INPUT='"spfh","clwmr","o3mr"' + +export CDATE=2018100300 + +export OMP_NUM_THREADS_CY=1 + +#----------------------------------------------------------------------------- +# Invoke chgres program. +#----------------------------------------------------------------------------- + +echo "Starting at: " `date` + +${HOMEufs}/ush/chgres_cube.sh + +iret=$? +if [ $iret -ne 0 ]; then + echo "<<< C96 FV3 GAUSSIAN NEMSIO TEST FAILED. <<<" + exit $iret +fi + +echo "Ending at: " `date` + +#----------------------------------------------------------------------------- +# Compare output from chgres to baseline set of data. +#----------------------------------------------------------------------------- + +cd $DATA + +test_failed=0 +for files in *.nc +do + if [ -f $files ]; then + echo CHECK $files + $NCCMP -dmfqS $files $HOMEreg/baseline_data/c192_fv3_history/$files + iret=$? + if [ $iret -ne 0 ]; then + test_failed=1 + fi + fi +done + +set +x +if [ $test_failed -ne 0 ]; then + echo "<<< C192 FV3 HISTORY TEST FAILED. >>>" +else + echo "<<< C192 FV3 HISTORY TEST PASSED. >>>" +fi + +exit 0 diff --git a/reg_tests/chgres_cube/c96.fv3.nemsio.sh b/reg_tests/chgres_cube/c96.fv3.nemsio.sh new file mode 100755 index 000000000..a056eabed --- /dev/null +++ b/reg_tests/chgres_cube/c96.fv3.nemsio.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +#----------------------------------------------------------------------------- +# Invoke chgres to create C96 coldstart files using FV3 gaussian nemsio files +# as input. The coldstart files are then compared to baseline files +# using the 'nccmp' utility. This script is run by the machine specific +# driver script. +#----------------------------------------------------------------------------- + +set -x + +export DATA=$OUTDIR/c96_fv3_nemsio +rm -fr $DATA + +export FIXfv3=${HOMEreg}/fix/C96 +export COMIN=${HOMEreg}/input_data/fv3.nemsio +export ATM_FILES_INPUT=gfs.t12z.atmf000.nemsio +export SFC_FILES_INPUT=gfs.t12z.sfcf000.nemsio +export VCOORD_FILE=${HOMEufs}/fix/fix_am/global_hyblev.l64.txt + +export CDATE=2019070412 + +export OMP_NUM_THREADS_CY=1 + +#----------------------------------------------------------------------------- +# Invoke chgres program. +#----------------------------------------------------------------------------- + +echo "Starting at: " `date` + +${HOMEufs}/ush/chgres_cube.sh + +iret=$? +if [ $iret -ne 0 ]; then + echo "<<< C96 FV3 GAUSSIAN NEMSIO TEST FAILED. <<<" + exit $iret +fi + +echo "Ending at: " `date` + +#----------------------------------------------------------------------------- +# Compare output from chgres to baseline set of data. +#----------------------------------------------------------------------------- + +cd $DATA + +test_failed=0 +for files in *.nc +do + if [ -f $files ]; then + echo CHECK $files + $NCCMP -dmfqS $files $HOMEreg/baseline_data/c96_fv3_nemsio/$files + iret=$? + if [ $iret -ne 0 ]; then + test_failed=1 + fi + fi +done + +set +x +if [ $test_failed -ne 0 ]; then + echo "<<< C96 FV3 GAUSSIAN NEMSIO TEST FAILED. >>>" +else + echo "<<< C96 FV3 GAUSSIAN NEMSIO TEST PASSED. >>>" +fi + +exit 0 diff --git a/reg_tests/chgres_cube/c96.fv3.restart.sh b/reg_tests/chgres_cube/c96.fv3.restart.sh new file mode 100755 index 000000000..e16f5f273 --- /dev/null +++ b/reg_tests/chgres_cube/c96.fv3.restart.sh @@ -0,0 +1,77 @@ +#!/bin/bash + +#----------------------------------------------------------------------------- +# Invoke chgres to create C96 coldstart files using FV3 tiled restart files +# as input. The coldstart files are then compared to baseline files +# using the 'nccmp' utility. This script is run by the machine specific +# driver script. +#----------------------------------------------------------------------------- + +set -x + +export DATA=$OUTDIR/c96_fv3_restart +rm -fr $DATA + +export FIXfv3=${HOMEreg}/fix/C96 +export COMIN=${HOMEreg}/input_data/fv3.restart +export VCOORD_FILE=${HOMEufs}/fix/fix_am/global_hyblev.l64.txt +export INPUT_TYPE='restart' +export MOSAIC_FILE_INPUT_GRID="${HOMEreg}/fix/C384/C384_mosaic.nc" +export OROG_DIR_INPUT_GRID=${HOMEreg}/fix/C384 + +# Note: no double quotes at the beginning or end. +export OROG_FILES_INPUT_GRID='C384_oro_data.tile1.nc","C384_oro_data.tile2.nc","C384_oro_data.tile3.nc","C384_oro_data.tile4.nc","C384_oro_data.tile5.nc","C384_oro_data.tile6.nc' +export ATM_CORE_FILES_INPUT='20190706.120000.fv_core.res.tile1.nc","20190706.120000.fv_core.res.tile2.nc","20190706.120000.fv_core.res.tile3.nc","20190706.120000.fv_core.res.tile4.nc","20190706.120000.fv_core.res.tile5.nc","20190706.120000.fv_core.res.tile6.nc","20190706.120000.fv_core.res.nc' +export ATM_TRACER_FILES_INPUT='20190706.120000.fv_tracer.res.tile1.nc","20190706.120000.fv_tracer.res.tile2.nc","20190706.120000.fv_tracer.res.tile3.nc","20190706.120000.fv_tracer.res.tile4.nc","20190706.120000.fv_tracer.res.tile5.nc","20190706.120000.fv_tracer.res.tile6.nc' +export SFC_FILES_INPUT='20190706.120000.sfc_data.tile1.nc","20190706.120000.sfc_data.tile2.nc","20190706.120000.sfc_data.tile3.nc","20190706.120000.sfc_data.tile4.nc","20190706.120000.sfc_data.tile5.nc","20190706.120000.sfc_data.tile6.nc' + +export TRACERS_TARGET='"sphum","liq_wat","o3mr","ice_wat","rainwat","snowwat","graupel"' +export TRACERS_INPUT='"sphum","liq_wat","o3mr","ice_wat","rainwat","snowwat","graupel"' + +export CDATE=2019070612 + +export OMP_NUM_THREADS_CY=1 + +#----------------------------------------------------------------------------- +# Invoke chgres program. +#----------------------------------------------------------------------------- + +echo "Starting at: " `date` + +${HOMEufs}/ush/chgres_cube.sh + +iret=$? +if [ $iret -ne 0 ]; then + echo "<<< C96 FV3 GAUSSIAN NEMSIO TEST FAILED. <<<" + exit $iret +fi + +echo "Ending at: " `date` + +#----------------------------------------------------------------------------- +# Compare output from chgres to baseline set of data. +#----------------------------------------------------------------------------- + +cd $DATA + +test_failed=0 +for files in *.nc +do + if [ -f $files ]; then + echo CHECK $files + $NCCMP -dmfqS $files $HOMEreg/baseline_data/c96_fv3_restart/$files + iret=$? + if [ $iret -ne 0 ]; then + test_failed=1 + fi + fi +done + +set +x +if [ $test_failed -ne 0 ]; then + echo "<<< C96 FV3 RESTART TEST FAILED. >>>" +else + echo "<<< C96 FV3 RESTART TEST PASSED. >>>" +fi + +exit 0 diff --git a/reg_tests/chgres_cube/c96.gfs.nemsio.sh b/reg_tests/chgres_cube/c96.gfs.nemsio.sh new file mode 100755 index 000000000..a3d3a4d59 --- /dev/null +++ b/reg_tests/chgres_cube/c96.gfs.nemsio.sh @@ -0,0 +1,71 @@ +#!/bin/bash + +#----------------------------------------------------------------------------- +# Invoke chgres to create C96 coldstart files using spectral GFS nemsio data +# as input. The coldstart files are then compared to baseline files +# using the 'nccmp' utility. This script is run by the machine specific +# driver script. +#----------------------------------------------------------------------------- + +set -x + +export DATA=$OUTDIR/c96_gfs_nemsio +rm -fr $DATA + +export FIXfv3=${HOMEreg}/fix/C96 +export COMIN=${HOMEreg}/input_data/gfs.nemsio +export ATM_FILES_INPUT=gfnanl.gdas.2017071700 +export SFC_FILES_INPUT=sfnanl.gdas.2017071700 +export NST_FILES_INPUT=nsnanl.gdas.2017071700 +export VCOORD_FILE=${HOMEufs}/fix/fix_am/global_hyblev.l64.txt +export INPUT_TYPE="gfs_gaussian" + +# dont start/end with double quotes. +export TRACERS_TARGET='"sphum","liq_wat","o3mr"' +export TRACERS_INPUT='"spfh","clwmr","o3mr"' +export CDATE=2017071700 +export OMP_NUM_THREADS_CY=1 + +#----------------------------------------------------------------------------- +# Invoke chgres program. +#----------------------------------------------------------------------------- + +echo "Starting at: " `date` + +${HOMEufs}/ush/chgres_cube.sh + +iret=$? +if [ $iret -ne 0 ]; then + echo "<<< C96 GFS GAUSSIAN NEMSIO TEST FAILED. <<<" + exit $iret +fi + +echo "Ending at: " `date` + +#----------------------------------------------------------------------------- +# Compare output from chgres to baseline set of data. +#----------------------------------------------------------------------------- + +cd $DATA + +test_failed=0 +for files in *.nc +do + if [ -f $files ]; then + echo CHECK $files + $NCCMP -dmfqS $files $HOMEreg/baseline_data/c96_gfs_nemsio/$files + iret=$? + if [ $iret -ne 0 ]; then + test_failed=1 + fi + fi +done + +set +x +if [ $test_failed -ne 0 ]; then + echo "<<< C96 GFS GAUSSIAN NEMSIO TEST FAILED. >>>" +else + echo "<<< C96 GFS GAUSSIAN NEMSIO TEST PASSED. >>>" +fi + +exit 0 diff --git a/reg_tests/chgres_cube/c96.gfs.sigio.sh b/reg_tests/chgres_cube/c96.gfs.sigio.sh new file mode 100755 index 000000000..c2ea31e52 --- /dev/null +++ b/reg_tests/chgres_cube/c96.gfs.sigio.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +#----------------------------------------------------------------------------- +# Invoke chgres to create C96 coldstart files using GFS sigio/sfcio files +# as input. The coldstart files are then compared to baseline files +# using the 'nccmp' utility. This script is run by the machine specific +# driver script. +#----------------------------------------------------------------------------- + +set -x + +export DATA=$OUTDIR/c96_gfs_sigio +rm -fr $DATA + +export FIXfv3=${HOMEreg}/fix/C96 +export COMIN=${HOMEreg}/input_data/gfs.sigio +export ATM_FILES_INPUT=gdas.t00z.sanl +export SFC_FILES_INPUT=gdas.t00z.sfcanl +export CONVERT_NST='.false.' +export VCOORD_FILE=${HOMEufs}/fix/fix_am/global_hyblev.l64.txt +export INPUT_TYPE="gfs_spectral" + +# dont start/end with double quotes +export TRACERS_TARGET='"sphum","o3mr","liq_wat"' +export TRACERS_INPUT='"spfh","o3mr","clwmr"' +export CDATE=2017071700 +export OMP_NUM_THREADS_CY=6 + +echo "Starting at: " `date` + +${HOMEufs}/ush/chgres_cube.sh + +iret=$? +if [ $iret -ne 0 ]; then + echo "<<< C96 GFS SIGIO TEST FAILED. <<<" + exit $iret +fi + +echo "Ending at: " `date` + +#----------------------------------------------------------------------------- +# Compare output from chgres to baseline set of data. +#----------------------------------------------------------------------------- + +cd $DATA + +test_failed=0 +for files in *.nc +do + if [ -f $files ]; then + echo CHECK $files + $NCCMP -dmfqS $files $HOMEreg/baseline_data/c96_gfs_sigio/$files + iret=$? + if [ $iret -ne 0 ]; then + test_failed=1 + fi + fi +done + +set +x +if [ $test_failed -ne 0 ]; then + echo "<<< C96 GFS SIGIO TEST FAILED. >>>" +else + echo "<<< C96 GFS SIGIO TEST PASSED. >>>" +fi + +exit 0 diff --git a/reg_tests/chgres_cube/c96.regional.sh b/reg_tests/chgres_cube/c96.regional.sh new file mode 100755 index 000000000..51a201f19 --- /dev/null +++ b/reg_tests/chgres_cube/c96.regional.sh @@ -0,0 +1,73 @@ +#!/bin/bash + +#----------------------------------------------------------------------------- +# Invoke chgres to create C96 regional coldstart files, including a lateral +# boundary file, using FV3 gaussian nemsio files as input. The +# coldstart files are then compared to baseline files using the 'nccmp' +# utility. +# +# This script is run by the machine specific driver script. +#----------------------------------------------------------------------------- + +set -x + +export DATA=$OUTDIR/c96_regional +rm -fr $DATA + +export FIXfv3=${HOMEreg}/fix/C96.regional +export OROG_FILES_TARGET_GRID="C96_oro_data.tile7.nc" +export COMIN=${HOMEreg}/input_data/fv3.nemsio +export ATM_FILES_INPUT=gfs.t12z.atmf000.nemsio +export SFC_FILES_INPUT=gfs.t12z.sfcf000.nemsio +export VCOORD_FILE=${HOMEufs}/fix/fix_am/global_hyblev.l64.txt +export REGIONAL=1 +export HALO_BLEND=0 +export HALO_BNDY=4 + +export CDATE=2019070412 + +export OMP_NUM_THREADS_CY=1 + +#----------------------------------------------------------------------------- +# Invoke chgres program. +#----------------------------------------------------------------------------- + +echo "Starting at: " `date` + +${HOMEufs}/ush/chgres_cube.sh + +iret=$? +if [ $iret -ne 0 ]; then + echo "<<< C96 REGIONAL TEST FAILED. <<<" + exit $iret +fi + +echo "Ending at: " `date` + +#----------------------------------------------------------------------------- +# Compare output from chgres to baseline set of data. +#----------------------------------------------------------------------------- + +cd $DATA + +test_failed=0 +for files in *.nc +do + if [ -f $files ]; then + echo CHECK $files + $NCCMP -dmfqS $files $HOMEreg/baseline_data/c96_regional/$files + iret=$? + if [ $iret -ne 0 ]; then + test_failed=1 + fi + fi +done + +set +x +if [ $test_failed -ne 0 ]; then + echo "<<< C96 REGIONAL TEST FAILED. >>>" +else + echo "<<< C96 REGIONAL TEST PASSED. >>>" +fi + +exit 0 diff --git a/reg_tests/chgres_cube/driver.cray.sh b/reg_tests/chgres_cube/driver.cray.sh new file mode 100755 index 000000000..e6f073256 --- /dev/null +++ b/reg_tests/chgres_cube/driver.cray.sh @@ -0,0 +1,127 @@ +#!/bin/bash + +#----------------------------------------------------------------------------- +# +# Run the chgres_cube regression tests on WCOSS-Cray. +# +# Set OUTDIR to your working directory. Set the PROJECT_CODE and QUEUE as +# appropriate. +# +# Invoke the script with no arguments. A series of daily- +# chained jobs will be submitted. To check the queue, type: "bjobs". +# +# The run output will be stored in OUTDIR. Log output from the suite +# will be in LOG_FILE. Once the suite has completed, a summary is +# placed in SUM_FILE. +# +# A test fails when its output does not match the baseline files as +# determined by the "nccmp" utility. The baseline files are stored in +# HOMEreg. +# +#----------------------------------------------------------------------------- + +set -x + +module purge +module load PrgEnv-intel/5.2.56 +module rm intel +module load intel/16.3.210 +module load cray-mpich/7.2.0 +module load craype-haswell +module load cray-netcdf +module load xt-lsfhpc/9.1.3 +module list + +export OUTDIR=/gpfs/hps3/stmp/$LOGNAME/chgres_reg_tests +QUEUE="debug" +PROJECT_CODE="FV3GFS-T2O" + +#----------------------------------------------------------------------------- +# Should not have to change anything below here. HOMEufs is the root +# directory of your UFS_UTILS clone. HOMEreg contains the input data +# and baseline data for each test. +#----------------------------------------------------------------------------- + +export HOMEufs=$PWD/../.. + +export HOMEreg=/gpfs/hps3/emc/global/noscrub/George.Gayno/ufs_utils.git/reg_tests/chgres_cube + +LOG_FILE=regression.log + +SUM_FILE=summary.log + +rm -f $LOG_FILE $SUM_FILE + +export NCCMP=/gpfs/hps3/emc/global/noscrub/George.Gayno/util/netcdf/nccmp + +export OMP_STACKSIZE=1024M + +export KMP_AFFINITY=disabled + +#----------------------------------------------------------------------------- +# Initialize C96 using FV3 warm restart files. +#----------------------------------------------------------------------------- + +export OMP_NUM_THREADS=1 +export APRUN="aprun -j 1 -n 6 -N 6 -d ${OMP_NUM_THREADS} -cc depth" +export INPUT_DATA=${HOMEreg}/input_data/fv3.restart +bsub -e $LOG_FILE -o $LOG_FILE -q $QUEUE -P $PROJECT_CODE -J c96.fv3.restart -M 1000 -W 0:15 -extsched 'CRAYLINUX[]' \ + "export NODES=1; $PWD/c96.fv3.restart.sh" + +#----------------------------------------------------------------------------- +# Initialize C192 using FV3 tiled history files. +#----------------------------------------------------------------------------- + +export OMP_NUM_THREADS=1 +export APRUN="aprun -j 1 -n 6 -N 6 -d ${OMP_NUM_THREADS} -cc depth" +export INPUT_DATA=${HOMEreg}/input_data/fv3.history +bsub -e $LOG_FILE -o $LOG_FILE -q $QUEUE -P $PROJECT_CODE -J c192.fv3.history -M 1000 -W 0:15 -extsched 'CRAYLINUX[]' \ + -w 'ended(c96.fv3.restart)' "export NODES=1; $PWD/c192.fv3.history.sh" + +#----------------------------------------------------------------------------- +# Initialize C96 using FV3 gaussian nemsio files. +#----------------------------------------------------------------------------- + +export OMP_NUM_THREADS=1 +export APRUN="aprun -j 1 -n 6 -N 6 -d ${OMP_NUM_THREADS} -cc depth" +export INPUT_DATA=${HOMEreg}/input_data/fv3.nemsio +bsub -e $LOG_FILE -o $LOG_FILE -q $QUEUE -P $PROJECT_CODE -J c96.fv3.nemsio -M 1000 -W 0:15 -extsched 'CRAYLINUX[]' \ + -w 'ended(c192.fv3.history)' "export NODES=1; $PWD/c96.fv3.nemsio.sh" + +#----------------------------------------------------------------------------- +# Initialize C96 using spectral GFS sigio/sfcio files. +#----------------------------------------------------------------------------- + +export OMP_NUM_THREADS=4 +export APRUN="aprun -j 1 -n 6 -N 6 -d ${OMP_NUM_THREADS} -cc depth" +export INPUT_DATA=${HOMEreg}/input_data/gfs.sigio +bsub -e $LOG_FILE -o $LOG_FILE -q $QUEUE -P $PROJECT_CODE -J c96.gfs.sigio -M 1000 -W 0:15 -extsched 'CRAYLINUX[]' \ + -w 'ended(c96.fv3.nemsio)' "export NODES=1; $PWD/c96.gfs.sigio.sh" + +#----------------------------------------------------------------------------- +# Initialize C96 using spectral GFS gaussian nemsio files. +#----------------------------------------------------------------------------- + +export OMP_NUM_THREADS=1 +export APRUN="aprun -j 1 -n 6 -N 6 -d ${OMP_NUM_THREADS} -cc depth" +export INPUT_DATA=${HOMEreg}/input_data/gfs.nemsio +bsub -e $LOG_FILE -o $LOG_FILE -q $QUEUE -P $PROJECT_CODE -J c96.gfs.nemsio -M 1000 -W 0:15 -extsched 'CRAYLINUX[]' \ + -w 'ended(c96.gfs.sigio)' "export NODES=1; $PWD/c96.gfs.nemsio.sh" + +#----------------------------------------------------------------------------- +# Initialize regional C96 using FV3 gaussian nemsio files. +#----------------------------------------------------------------------------- + +export OMP_NUM_THREADS=1 +export APRUN="aprun -j 1 -n 6 -N 6 -d ${OMP_NUM_THREADS} -cc depth" +export INPUT_DATA=${HOMEreg}/input_data/fv3.nemsio +bsub -e $LOG_FILE -o $LOG_FILE -q $QUEUE -P $PROJECT_CODE -J c96.regional -M 1000 -W 0:15 -extsched 'CRAYLINUX[]' \ + -w 'ended(c96.gfs.nemsio)' "export NODES=1; $PWD/c96.regional.sh" + +#----------------------------------------------------------------------------- +# Create summary log. +#----------------------------------------------------------------------------- + +bsub -o $LOG_FILE -q $QUEUE -P $PROJECT_CODE -J summary -R "rusage[mem=100]" -W 0:01 -w 'ended(c96.regional)' "grep -a '<<<' $LOG_FILE >> $SUM_FILE" + +exit diff --git a/reg_tests/chgres_cube/driver.dell.sh b/reg_tests/chgres_cube/driver.dell.sh new file mode 100755 index 000000000..e9369e2f9 --- /dev/null +++ b/reg_tests/chgres_cube/driver.dell.sh @@ -0,0 +1,121 @@ +#!/bin/bash + +#----------------------------------------------------------------------------- +# +# Run the chgres_cube regression tests on WCOSS-Dell. +# +# Set OUTDIR to your working directory. Set the PROJECT_CODE and QUEUE +# as appropriate. +# +# Invoke the script as follows with no arguments. A series of daily- +# chained jobs will be submitted. To check the queue, type: "bjobs". +# +# The run output will be stored in OUTDIR. Log output from the suite +# will be in LOG_FILE. Once the suite has completed, a summary is +# placed in SUM_FILE. +# +# A test fails when its output does not match the baseline files as +# determined by the "nccmp" utility. The baseline files are stored in +# HOMEreg. +# +#----------------------------------------------------------------------------- + +set -x + +module purge +module load EnvVars/1.0.2 +module load ips/18.0.1.163 +module load impi/18.0.1 +module load lsf/10.1 +module use /usrx/local/dev/modulefiles +module load NetCDF/4.5.0 +module list + +export OUTDIR=/gpfs/dell1/stmp/$LOGNAME/chgres_reg_tests +QUEUE="debug" +PROJECT_CODE="FV3GFS-T2O" + +#----------------------------------------------------------------------------- +# Should not have to change anything below here. HOMEufs is the root +# directory of your UFS_UTILS clone. HOMEreg contains the input data +# and baseline data for each test. +#----------------------------------------------------------------------------- + +export HOMEufs=$PWD/../.. + +export HOMEreg=/gpfs/dell2/emc/modeling/noscrub/George.Gayno/ufs_utils.git/reg_tests/chgres_cube + +LOG_FILE=regression.log + +SUM_FILE=summary.log + +rm -f $LOG_FILE $SUM_FILE + +export NCCMP=/gpfs/dell2/emc/modeling/noscrub/George.Gayno/util/nccmp/nccmp-1.8.5.0/src/nccmp + +export OMP_STACKSIZE=1024M + +export APRUN=mpirun + +#----------------------------------------------------------------------------- +# Initialize C96 using FV3 warm restart files. +#----------------------------------------------------------------------------- + +export OMP_NUM_THREADS=1 +export INPUT_DATA=${HOMEreg}/input_data/fv3.restart +bsub -e $LOG_FILE -o $LOG_FILE -q $QUEUE -P $PROJECT_CODE -J c96.fv3.restart -W 0:15 -x -n 6 \ + -R "span[ptile=6]" -R "affinity[core(${OMP_NUM_THREADS}):distribute=balance]" "$PWD/c96.fv3.restart.sh" + +#----------------------------------------------------------------------------- +# Initialize C192 using FV3 tiled history files. +#----------------------------------------------------------------------------- + +export OMP_NUM_THREADS=1 +export INPUT_DATA=${HOMEreg}/input_data/fv3.history +bsub -e $LOG_FILE -o $LOG_FILE -q $QUEUE -P $PROJECT_CODE -J c192.fv3.history -W 0:15 -x -n 6 -w 'ended(c96.fv3.restart)' \ + -R "span[ptile=6]" -R "affinity[core(${OMP_NUM_THREADS}):distribute=balance]" "$PWD/c192.fv3.history.sh" + +#----------------------------------------------------------------------------- +# Initialize C96 using FV3 gaussian nemsio files. +#----------------------------------------------------------------------------- + +export OMP_NUM_THREADS=1 +export INPUT_DATA=${HOMEreg}/input_data/fv3.nemsio +bsub -e $LOG_FILE -o $LOG_FILE -q $QUEUE -P $PROJECT_CODE -J c96.fv3.nemsio -W 0:15 -x -n 6 -w 'ended(c192.fv3.history)' \ + -R "span[ptile=6]" -R "affinity[core(${OMP_NUM_THREADS}):distribute=balance]" "$PWD/c96.fv3.nemsio.sh" + +#----------------------------------------------------------------------------- +# Initialize C96 using spectral GFS sigio/sfcio files. +#----------------------------------------------------------------------------- + +export OMP_NUM_THREADS=4 +export INPUT_DATA=${HOMEreg}/input_data/gfs.sigio +bsub -e $LOG_FILE -o $LOG_FILE -q $QUEUE -P $PROJECT_CODE -J c96.gfs.sigio -W 0:15 -x -n 6 -w 'ended(c96.fv3.nemsio)' \ + -R "span[ptile=6]" -R "affinity[core(${OMP_NUM_THREADS}):distribute=balance]" "$PWD/c96.gfs.sigio.sh" + +#----------------------------------------------------------------------------- +# Initialize C96 using spectral GFS gaussian nemsio files. +#----------------------------------------------------------------------------- + +export OMP_NUM_THREADS=1 +export INPUT_DATA=${HOMEreg}/input_data/gfs.nemsio +bsub -e $LOG_FILE -o $LOG_FILE -q $QUEUE -P $PROJECT_CODE -J c96.gfs.nemsio -W 0:15 -x -n 6 -w 'ended(c96.gfs.sigio)' \ + -R "span[ptile=6]" -R "affinity[core(${OMP_NUM_THREADS}):distribute=balance]" "$PWD/c96.gfs.nemsio.sh" + +#----------------------------------------------------------------------------- +# Initialize regional C96 using FV3 gaussian nemsio files. +#----------------------------------------------------------------------------- + +export OMP_NUM_THREADS=1 +export INPUT_DATA=${HOMEreg}/input_data/fv3.nemsio +bsub -e $LOG_FILE -o $LOG_FILE -q $QUEUE -P $PROJECT_CODE -J c96.regional -W 0:15 -x -n 6 -w 'ended(c96.gfs.nemsio)' \ + -R "span[ptile=6]" -R "affinity[core(${OMP_NUM_THREADS}):distribute=balance]" "$PWD/c96.regional.sh" + +#----------------------------------------------------------------------------- +# Create summary log. +#----------------------------------------------------------------------------- + +bsub -o $LOG_FILE -q $QUEUE -P $PROJECT_CODE -J summary -R "affinity[core(1)]" -R "rusage[mem=100]" -W 0:01 \ + -w 'ended(c96.regional)' "grep -a '<<<' $LOG_FILE >> $SUM_FILE" + +exit diff --git a/reg_tests/chgres_cube/driver.hera.sh b/reg_tests/chgres_cube/driver.hera.sh new file mode 100755 index 000000000..ba58b3b15 --- /dev/null +++ b/reg_tests/chgres_cube/driver.hera.sh @@ -0,0 +1,112 @@ +#!/bin/bash + +#----------------------------------------------------------------------------- +# +# Run the chgres_cube regression tests on Hera. +# +# Set OUTDIR to your working directory. Set the PROJECT_CODE and QUEUE +# as appropriate. To see which projects you are authorized to use, +# type "account_params". +# +# Invoke the script with no arguments. A series of daily-chained +# regression tests will be submitted. To check the queue, type: +# "squeue -u USERNAME". +# +# The run output will be stored in OUTDIR. Log output from the suite +# will be in LOG_FILE. Once the suite has completed, a summary is +# placed in SUM_FILE. +# +# A test fails when its output does not match the baseline files as +# determined by the "nccmp" utility. The baseline files are stored in +# HOMEreg. +# +#----------------------------------------------------------------------------- + +set -x + +source /apps/lmod/lmod/init/sh +module purge +module load intel/18.0.5.274 +module load impi/2018.0.4 +module load netcdf/4.7.0 +module list + +export OUTDIR=/scratch2/NCEPDEV/stmp1/$LOGNAME/chgres_reg_tests +PROJECT_CODE="fv3-cpu" +QUEUE="debug" + +#----------------------------------------------------------------------------- +# Should not have to change anything below here. HOMEufs is the root +# directory of your UFS_UTILS clone. HOMEreg contains the input data +# and baseline data for each test. +#----------------------------------------------------------------------------- + +export HOMEufs=$PWD/../.. + +export HOMEreg=/scratch1/NCEPDEV/da/George.Gayno/noscrub/reg_tests/chgres_cube + +export NCCMP=/apps/nccmp/1.8.5/intel/18.0.3.051/bin/nccmp + +LOG_FILE=regression.log +SUM_FILE=summary.log +rm -f $LOG_FILE $SUM_FILE + +export OMP_STACKSIZE=1024M + +export APRUN=srun + +rm -fr $OUTDIR + +#----------------------------------------------------------------------------- +# Initialize C96 using FV3 warm restart files. +#----------------------------------------------------------------------------- + +TEST1=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 -t 0:15:00 -A $PROJECT_CODE -q $QUEUE -J c96.fv3.restart \ + -o $LOG_FILE -e $LOG_FILE ./c96.fv3.restart.sh) + +#----------------------------------------------------------------------------- +# Initialize C192 using FV3 tiled history files. +#----------------------------------------------------------------------------- + +TEST2=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 -t 0:15:00 -A $PROJECT_CODE -q $QUEUE -J c192.fv3.history \ + -o $LOG_FILE -e $LOG_FILE -d afterok:$TEST1 ./c192.fv3.history.sh) + +#----------------------------------------------------------------------------- +# Initialize C96 using FV3 gaussian nemsio files. +#----------------------------------------------------------------------------- + +TEST3=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 -t 0:15:00 -A $PROJECT_CODE -q $QUEUE -J c96.fv3.nemsio \ + -o $LOG_FILE -e $LOG_FILE -d afterok:$TEST2 ./c96.fv3.nemsio.sh) + +#----------------------------------------------------------------------------- +# Initialize C96 using spectral GFS sigio/sfcio files. +#----------------------------------------------------------------------------- + +TEST4=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 -t 0:15:00 -A $PROJECT_CODE -q $QUEUE -J c96.gfs.sigio \ + -o $LOG_FILE -e $LOG_FILE -d afterok:$TEST3 ./c96.gfs.sigio.sh) + +#----------------------------------------------------------------------------- +# Initialize C96 using spectral GFS gaussian nemsio files. +#----------------------------------------------------------------------------- + +TEST5=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 -t 0:15:00 -A $PROJECT_CODE -q $QUEUE -J c96.gfs.nemsio \ + -o $LOG_FILE -e $LOG_FILE -d afterok:$TEST4 ./c96.gfs.nemsio.sh) + +#----------------------------------------------------------------------------- +# Initialize regional C96 using FV3 gaussian nemsio files. +#----------------------------------------------------------------------------- + +TEST6=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 -t 0:15:00 -A $PROJECT_CODE -q $QUEUE -J c96.regional \ + -o $LOG_FILE -e $LOG_FILE -d afterok:$TEST5 ./c96.regional.sh) + +#----------------------------------------------------------------------------- +# Create summary log. +#----------------------------------------------------------------------------- + +sbatch --nodes=1 -t 0:01:00 -A $PROJECT_CODE -J chgres_summary -o $LOG_FILE -e $LOG_FILE \ + --open-mode=append -q $QUEUE -d afterok:$TEST6 << EOF +#!/bin/sh +grep -a '<<<' $LOG_FILE > $SUM_FILE +EOF + +exit 0 diff --git a/reg_tests/chgres_cube/driver.jet.sh b/reg_tests/chgres_cube/driver.jet.sh new file mode 100755 index 000000000..6f76db74c --- /dev/null +++ b/reg_tests/chgres_cube/driver.jet.sh @@ -0,0 +1,114 @@ +#!/bin/bash + +#----------------------------------------------------------------------------- +# +# Run the chgres_cube regression tests on JET. +# +# Set OUTDIR to your working directory. Set the PROJECT_CODE and QUEUE +# as appropriate. To see which projects you are authorized to use, +# type "account_params". +# +# Invoke the script with no arguments. A series of daily- +# chained jobs will be submitted. To check the queue, type: +# "squeue -u USERNAME". +# +# The run output will be stored in OUTDIR. Log output from the suite +# will be in LOG_FILE. Once the suite has completed, a summary is +# placed in SUM_FILE. +# +# A test fails when its output does not match the baseline files as +# determined by the "nccmp" utility. The baseline files are stored in +# HOMEreg. +# +#----------------------------------------------------------------------------- + +set -x + +source /apps/lmod/lmod/init/sh +module purge +module load intel/15.0.3.187 +module load impi/2018.4.274 +module load szip +module load hdf5 +module load netcdf/4.2.1.1 +module list + +export OUTDIR=/mnt/lfs3/projects/emcda/$LOGNAME/stmp/chgres_reg_tests +PROJECT_CODE="emcda" +QUEUE="windfall" + +#----------------------------------------------------------------------------- +# Should not have to change anything below here. HOMEufs is the root +# directory of your UFS_UTILS clone. HOMEreg contains the input data +# and baseline data for each test. +#----------------------------------------------------------------------------- + +export HOMEufs=$PWD/../.. + +export HOMEreg=/mnt/lfs3/projects/emcda/George.Gayno/reg_tests/chgres_cube + +export NCCMP=/apps/nccmp/1.8.2.1/intel/18.0.3.222/bin/nccmp + +LOG_FILE=regression.log +SUM_FILE=summary.log +rm -f $LOG_FILE $SUM_FILE + +export OMP_STACKSIZE=1024M + +export APRUN=srun + +rm -fr $OUTDIR + +#----------------------------------------------------------------------------- +# Initialize C96 using FV3 warm restart files. +#----------------------------------------------------------------------------- + +TEST1=$(sbatch --parsable --partition=xjet --nodes=1 --ntasks-per-node=6 -t 0:10:00 -A $PROJECT_CODE -q $QUEUE -J c96.fv3.restart \ + -o $LOG_FILE -e $LOG_FILE ./c96.fv3.restart.sh) + +#----------------------------------------------------------------------------- +# Initialize C192 using FV3 tiled history files. +#----------------------------------------------------------------------------- + +TEST2=$(sbatch --parsable --partition=xjet --nodes=1 --ntasks-per-node=6 -t 0:10:00 -A $PROJECT_CODE -q $QUEUE -J c192.fv3.history \ + -o $LOG_FILE -e $LOG_FILE -d afterok:$TEST1 ./c192.fv3.history.sh) + +#----------------------------------------------------------------------------- +# Initialize C96 using FV3 gaussian nemsio files. +#----------------------------------------------------------------------------- + +TEST3=$(sbatch --parsable --partition=xjet --nodes=1 --ntasks-per-node=6 -t 0:10:00 -A $PROJECT_CODE -q $QUEUE -J c96.fv3.nemsio \ + -o $LOG_FILE -e $LOG_FILE -d afterok:$TEST2 ./c96.fv3.nemsio.sh) + +#----------------------------------------------------------------------------- +# Initialize C96 using spectral GFS sigio/sfcio files. +#----------------------------------------------------------------------------- + +TEST4=$(sbatch --parsable --partition=xjet --nodes=2 --ntasks-per-node=3 --cpus-per-task=6 -t 0:15:00 \ + -A $PROJECT_CODE -q $QUEUE -J c96.gfs.sigio -o $LOG_FILE -e $LOG_FILE -d afterok:$TEST3 ./c96.gfs.sigio.sh) + +#----------------------------------------------------------------------------- +# Initialize C96 using spectral GFS gaussian nemsio files. +#----------------------------------------------------------------------------- + +TEST5=$(sbatch --parsable --partition=xjet --nodes=1 --ntasks-per-node=6 -t 0:10:00 -A $PROJECT_CODE -q $QUEUE -J c96.gfs.nemsio \ + -o $LOG_FILE -e $LOG_FILE -d afterok:$TEST4 ./c96.gfs.nemsio.sh) + +#----------------------------------------------------------------------------- +# Initialize regional C96 using FV3 gaussian nemsio files. +#----------------------------------------------------------------------------- + +TEST6=$(sbatch --parsable --partition=xjet --nodes=1 --ntasks-per-node=6 -t 0:10:00 -A $PROJECT_CODE -q $QUEUE -J c96.regional \ + -o $LOG_FILE -e $LOG_FILE -d afterok:$TEST5 ./c96.regional.sh) + +#----------------------------------------------------------------------------- +# Create summary log. +#----------------------------------------------------------------------------- + +sbatch --partition=xjet --nodes=1 -t 0:01:00 -A $PROJECT_CODE -J chgres_summary -o $LOG_FILE -e $LOG_FILE \ + --open-mode=append -q $QUEUE -d afterok:$TEST6 << EOF +#!/bin/sh +grep -a '<<<' $LOG_FILE > $SUM_FILE +EOF + +exit 0 diff --git a/sorc/build_chgres.sh b/sorc/build_chgres.sh index 210ebae36..7dcb4291f 100755 --- a/sorc/build_chgres.sh +++ b/sorc/build_chgres.sh @@ -6,7 +6,7 @@ cwd=`pwd` USE_PREINST_LIBS=${USE_PREINST_LIBS:-"true"} if [ $USE_PREINST_LIBS = true ]; then - export MOD_PATH=/scratch3/NCEPDEV/nwprod/lib/modulefiles + export MOD_PATH source ../modulefiles/fv3gfs/global_chgres.$target > /dev/null 2>&1 else export MOD_PATH=${cwd}/lib/modulefiles @@ -32,8 +32,8 @@ export FCMP95=$FCMP export FFLAGSM="-i4 -O3 -r8 -convert big_endian -fp-model precise" export RECURS= -export LDFLAGSM=${LDFLAGSM:-"-openmp -auto"} -export OMPFLAGM=${OMPFLAGM:-"-openmp -auto"} +export LDFLAGSM=${LDFLAGSM:-"-qopenmp -auto"} +export OMPFLAGM=${OMPFLAGM:-"-qopenmp -auto"} export INCS="-I${SIGIO_INC4} -I${SFCIO_INC4} -I${LANDSFCUTIL_INCd} \ -I${NEMSIO_INC} -I${NEMSIOGFS_INC} -I${GFSIO_INC4} -I${IP_INCd} ${NETCDF_INCLUDE}" diff --git a/sorc/build_chgres_cube.sh b/sorc/build_chgres_cube.sh index 412c337b7..752429b47 100755 --- a/sorc/build_chgres_cube.sh +++ b/sorc/build_chgres_cube.sh @@ -6,7 +6,7 @@ cwd=`pwd` USE_PREINST_LIBS=${USE_PREINST_LIBS:-"true"} if [ $USE_PREINST_LIBS = true ]; then - export MOD_PATH=/scratch3/NCEPDEV/nwprod/lib/modulefiles + export MOD_PATH source ../modulefiles/chgres_cube.$target > /dev/null 2>&1 else export MOD_PATH=${cwd}/lib/modulefiles diff --git a/sorc/build_cycle.sh b/sorc/build_cycle.sh index b721bf840..75e80cc29 100755 --- a/sorc/build_cycle.sh +++ b/sorc/build_cycle.sh @@ -6,7 +6,7 @@ cwd=`pwd` USE_PREINST_LIBS=${USE_PREINST_LIBS:-"true"} if [ $USE_PREINST_LIBS = true ]; then - export MOD_PATH=/scratch3/NCEPDEV/nwprod/lib/modulefiles + export MOD_PATH source ../modulefiles/fv3gfs/global_cycle.$target > /dev/null 2>&1 else export MOD_PATH=${cwd}/lib/modulefiles diff --git a/sorc/build_emcsfc.sh b/sorc/build_emcsfc.sh index c2c600b12..233c40b26 100755 --- a/sorc/build_emcsfc.sh +++ b/sorc/build_emcsfc.sh @@ -14,6 +14,7 @@ do module purge USE_PREINST_LIBS=${USE_PREINST_LIBS:-"true"} if [ $USE_PREINST_LIBS = true ]; then + export MOD_PATH source ../modulefiles/modulefile.global_${prog}.${target} > /dev/null 2>&1 else export MOD_PATH=${cwd}/lib/modulefiles diff --git a/sorc/build_nems_util.sh b/sorc/build_nems_util.sh index d46f2c584..3f2f1498c 100755 --- a/sorc/build_nems_util.sh +++ b/sorc/build_nems_util.sh @@ -6,7 +6,7 @@ cwd=`pwd` USE_PREINST_LIBS=${USE_PREINST_LIBS:-"true"} if [ $USE_PREINST_LIBS = true ]; then - export MOD_PATH=/scratch3/NCEPDEV/nwprod/lib/modulefiles + export MOD_PATH source ../modulefiles/module_nemsutil.$target > /dev/null 2>&1 else export MOD_PATH=${cwd}/lib/modulefiles @@ -22,7 +22,7 @@ if [ ! -d "../exec" ]; then mkdir ../exec fi -for prog in nemsio_get.fd nemsio_cvt.fd nemsio_read.fd nemsio_chgdate.fd ;do +for prog in nemsio_get.fd nemsio_read.fd nemsio_chgdate.fd mkgfsnemsioctl.fd ;do cd ${cwd}/${prog} make -f makefile done diff --git a/sorc/build_nst_tf_chg.sh b/sorc/build_nst_tf_chg.sh index ef063f5f7..5ba9ec958 100755 --- a/sorc/build_nst_tf_chg.sh +++ b/sorc/build_nst_tf_chg.sh @@ -7,7 +7,7 @@ cwd=`pwd` module purge USE_PREINST_LIBS=${USE_PREINST_LIBS:-"true"} if [ $USE_PREINST_LIBS = true ]; then - export MOD_PATH=/scratch3/NCEPDEV/nwprod/lib/modulefiles + export MOD_PATH source ../modulefiles/fv3gfs/nst_tf_chg.$target > /dev/null 2>&1 else export MOD_PATH=${cwd}/lib/modulefiles diff --git a/sorc/build_orog.sh b/sorc/build_orog.sh index a35846696..c1cc33a81 100755 --- a/sorc/build_orog.sh +++ b/sorc/build_orog.sh @@ -10,7 +10,7 @@ cwd=`pwd` USE_PREINST_LIBS=${USE_PREINST_LIBS:-"true"} if [ $USE_PREINST_LIBS = true ]; then - export MOD_PATH=/scratch3/NCEPDEV/nwprod/lib/modulefiles + export MOD_PATH source ../modulefiles/fv3gfs/orog.$target > /dev/null 2>&1 else export MOD_PATH=${cwd}/lib/modulefiles @@ -43,7 +43,7 @@ elif [ $target = jet ]; then INCS="-I${NETCDF}/include" export LIBSM="${BACIO_LIB4} ${W3NCO_LIBd} ${IP_LIBd} ${SP_LIBd} -L${NETCDF}/lib -lnetcdff -lnetcdf" export FFLAGSM="-O3 -g -traceback -r8 -convert big_endian -fp-model precise -assume byterecl ${INCS}" -elif [ $target = theia ]; then +elif [ $target = hera ]; then INCS="-I${NETCDF}/include" export LIBSM="${BACIO_LIB4} ${W3NCO_LIBd} ${IP_LIBd} ${SP_LIBd} -L${NETCDF}/lib -lnetcdff -lnetcdf" export FFLAGSM="-O3 -g -traceback -r8 -convert big_endian -fp-model precise -assume byterecl ${INCS}" diff --git a/sorc/build_sfc_climo_gen.sh b/sorc/build_sfc_climo_gen.sh index 237169f2f..cf253d358 100755 --- a/sorc/build_sfc_climo_gen.sh +++ b/sorc/build_sfc_climo_gen.sh @@ -12,6 +12,7 @@ fi module purge USE_PREINST_LIBS=${USE_PREINST_LIBS:-"true"} if [ $USE_PREINST_LIBS = true ]; then + export MOD_PATH source ../modulefiles/modulefile.sfc_climo_gen.${target} > /dev/null 2>&1 else export MOD_PATH=${cwd}/lib/modulefiles diff --git a/sorc/chgres_cube.fd/atmosphere.F90 b/sorc/chgres_cube.fd/atmosphere.F90 index 246c39b0d..70a95b176 100644 --- a/sorc/chgres_cube.fd/atmosphere.F90 +++ b/sorc/chgres_cube.fd/atmosphere.F90 @@ -185,7 +185,7 @@ subroutine atmosphere_driver(localpet) call ESMF_FieldRegridStore(temp_input_grid, & temp_b4adj_target_grid, & - polemethod=ESMF_POLEMETHOD_NONE, & + polemethod=ESMF_POLEMETHOD_ALLAVG, & srctermprocessing=isrctermprocessing, & extrapmethod=ESMF_EXTRAPMETHOD_NEAREST_STOD, & routehandle=regrid_bl, & @@ -337,7 +337,7 @@ subroutine atmosphere_driver(localpet) print*,"- CALL FieldRegridStore FOR 3D-WIND WEST EDGE." call ESMF_FieldRegridStore(wind_target_grid, & wind_w_target_grid, & - polemethod=ESMF_POLEMETHOD_NONE, & + polemethod=ESMF_POLEMETHOD_ALLAVG, & srctermprocessing=isrctermprocessing, & extrapMethod=ESMF_EXTRAPMETHOD_NEAREST_STOD, & routehandle=regrid_bl, & @@ -364,7 +364,7 @@ subroutine atmosphere_driver(localpet) print*,"- CALL FieldRegridStore FOR 3D-WIND SOUTH EDGE." call ESMF_FieldRegridStore(wind_target_grid, & wind_s_target_grid, & - polemethod=ESMF_POLEMETHOD_NONE, & + polemethod=ESMF_POLEMETHOD_ALLAVG, & srctermprocessing=isrctermprocessing, & extrapMethod=ESMF_EXTRAPMETHOD_NEAREST_STOD, & routehandle=regrid_bl, & diff --git a/sorc/chgres_cube.fd/surface.F90 b/sorc/chgres_cube.fd/surface.F90 index bd9e7c8af..4b6ea37da 100644 --- a/sorc/chgres_cube.fd/surface.F90 +++ b/sorc/chgres_cube.fd/surface.F90 @@ -376,7 +376,7 @@ subroutine interp(localpet) print*,"- CALL FieldRegridStore FOR NON-MASKED BILINEAR INTERPOLATION." call ESMF_FieldRegridStore(t2m_input_grid, & t2m_target_grid, & - polemethod=ESMF_POLEMETHOD_NONE, & + polemethod=ESMF_POLEMETHOD_ALLAVG, & srctermprocessing=isrctermprocessing, & routehandle=regrid_bl_no_mask, & regridmethod=method, rc=rc) diff --git a/sorc/fre-nctools.fd/tools/filter_topo/make.csh_theia b/sorc/fre-nctools.fd/tools/filter_topo/make.csh_hera similarity index 81% rename from sorc/fre-nctools.fd/tools/filter_topo/make.csh_theia rename to sorc/fre-nctools.fd/tools/filter_topo/make.csh_hera index 563538c4c..fb43802b0 100755 --- a/sorc/fre-nctools.fd/tools/filter_topo/make.csh_theia +++ b/sorc/fre-nctools.fd/tools/filter_topo/make.csh_hera @@ -1,6 +1,6 @@ #!/bin/csh -f source $MODULESHOME/init/csh -source ${PWD}/../../../../modulefiles/fv3gfs/fre-nctools.theia +source ${PWD}/../../../../modulefiles/fv3gfs/fre-nctools.hera ifort -o filter_topo -I${NETCDF}/include -fltconsistency -fno-alias -stack_temps -safe_cray_ptr -ftz -assume byterecl -g -O2 -i4 -real_size 64 -traceback filter_topo.F90 -L${NETCDF}/lib -L${HDF5}/lib -lnetcdf -lnetcdff -lhdf5_hl -lhdf5 diff --git a/sorc/fre-nctools.fd/tools/make_hgrid/env.theia b/sorc/fre-nctools.fd/tools/fregrid/env.hera similarity index 85% rename from sorc/fre-nctools.fd/tools/make_hgrid/env.theia rename to sorc/fre-nctools.fd/tools/fregrid/env.hera index 6c090e2e2..a4ca56ba4 100644 --- a/sorc/fre-nctools.fd/tools/make_hgrid/env.theia +++ b/sorc/fre-nctools.fd/tools/fregrid/env.hera @@ -1,4 +1,4 @@ -# theia +# hera MPICC := mpiicc CC := icc STATIC := diff --git a/sorc/fre-nctools.fd/tools/fregrid/env.wcoss_dell_p3 b/sorc/fre-nctools.fd/tools/fregrid/env.wcoss_dell_p3 index 834db6e7e..07f52aac8 100644 --- a/sorc/fre-nctools.fd/tools/fregrid/env.wcoss_dell_p3 +++ b/sorc/fre-nctools.fd/tools/fregrid/env.wcoss_dell_p3 @@ -1,4 +1,4 @@ -# theia +# Dell MPICC := mpiicc CC := icc -STATIC := \ No newline at end of file +STATIC := diff --git a/sorc/fre-nctools.fd/tools/fregrid/fre-nctools.mk_theia b/sorc/fre-nctools.fd/tools/fregrid/fre-nctools.mk_hera similarity index 100% rename from sorc/fre-nctools.fd/tools/fregrid/fre-nctools.mk_theia rename to sorc/fre-nctools.fd/tools/fregrid/fre-nctools.mk_hera diff --git a/sorc/fre-nctools.fd/tools/make_solo_mosaic/env.theia b/sorc/fre-nctools.fd/tools/make_hgrid/env.hera similarity index 85% rename from sorc/fre-nctools.fd/tools/make_solo_mosaic/env.theia rename to sorc/fre-nctools.fd/tools/make_hgrid/env.hera index 6c090e2e2..a4ca56ba4 100644 --- a/sorc/fre-nctools.fd/tools/make_solo_mosaic/env.theia +++ b/sorc/fre-nctools.fd/tools/make_hgrid/env.hera @@ -1,4 +1,4 @@ -# theia +# hera MPICC := mpiicc CC := icc STATIC := diff --git a/sorc/fre-nctools.fd/tools/make_hgrid/fre-nctools.mk_theia b/sorc/fre-nctools.fd/tools/make_hgrid/fre-nctools.mk_hera similarity index 100% rename from sorc/fre-nctools.fd/tools/make_hgrid/fre-nctools.mk_theia rename to sorc/fre-nctools.fd/tools/make_hgrid/fre-nctools.mk_hera diff --git a/sorc/fre-nctools.fd/tools/fregrid/env.theia b/sorc/fre-nctools.fd/tools/make_solo_mosaic/env.hera similarity index 64% rename from sorc/fre-nctools.fd/tools/fregrid/env.theia rename to sorc/fre-nctools.fd/tools/make_solo_mosaic/env.hera index 834db6e7e..a4ca56ba4 100644 --- a/sorc/fre-nctools.fd/tools/fregrid/env.theia +++ b/sorc/fre-nctools.fd/tools/make_solo_mosaic/env.hera @@ -1,4 +1,4 @@ -# theia +# hera MPICC := mpiicc CC := icc -STATIC := \ No newline at end of file +STATIC := diff --git a/sorc/fre-nctools.fd/tools/make_solo_mosaic/env.wcoss_dell_p3 b/sorc/fre-nctools.fd/tools/make_solo_mosaic/env.wcoss_dell_p3 index 6c090e2e2..07f52aac8 100644 --- a/sorc/fre-nctools.fd/tools/make_solo_mosaic/env.wcoss_dell_p3 +++ b/sorc/fre-nctools.fd/tools/make_solo_mosaic/env.wcoss_dell_p3 @@ -1,4 +1,4 @@ -# theia +# Dell MPICC := mpiicc CC := icc STATIC := diff --git a/sorc/fre-nctools.fd/tools/make_solo_mosaic/fre-nctools.mk_theia b/sorc/fre-nctools.fd/tools/make_solo_mosaic/fre-nctools.mk_hera similarity index 100% rename from sorc/fre-nctools.fd/tools/make_solo_mosaic/fre-nctools.mk_theia rename to sorc/fre-nctools.fd/tools/make_solo_mosaic/fre-nctools.mk_hera diff --git a/sorc/fre-nctools.fd/tools/shave.fd/build_shave b/sorc/fre-nctools.fd/tools/shave.fd/build_shave index 81aa8469d..18cdf07aa 100755 --- a/sorc/fre-nctools.fd/tools/shave.fd/build_shave +++ b/sorc/fre-nctools.fd/tools/shave.fd/build_shave @@ -15,7 +15,7 @@ case $1 in "jet" ) FCMP=ifort FFLAGS="-O3 -fp-model precise -I$NETCDF/include -L$NETCDF/lib -lnetcdff -lnetcdf" ;; - "theia" ) + "hera" ) FCMP=ifort FFLAGS="-O3 -fp-model precise -I$NETCDF/include -L$NETCDF/lib -lnetcdff -lnetcdf" ;; *) diff --git a/sorc/global_chgres.fd/sfcsub.F b/sorc/global_chgres.fd/sfcsub.F index b14271cd0..4fbabab8f 100644 --- a/sorc/global_chgres.fd/sfcsub.F +++ b/sorc/global_chgres.fd/sfcsub.F @@ -2697,12 +2697,13 @@ subroutine fixrdg(lugb,idim,jdim,fngrib, real (kind=kind_io8), allocatable :: data8(:) real (kind=kind_io4), allocatable :: data4(:) ! - logical*1 lbms(mdata) + logical*1, allocatable :: lbms(:) ! integer kpds(200),kgds(200) integer jpds(200),jgds(200), kpds0(200) ! allocate(data8(1:idim*jdim)) + allocate(lbms(1:mdata)) kpds = 0 kgds = 0 jpds = 0 @@ -2792,6 +2793,7 @@ subroutine fixrdg(lugb,idim,jdim,fngrib, endif ! deallocate(data8) + deallocate(lbms) return end subroutine getarea(kgds,dlat,dlon,rslat,rnlat,wlon,elon,ijordr @@ -6078,17 +6080,24 @@ subroutine setrmsk(kpds5,slmask,igaul,jgaul,wlon,rnlat, ! ijmax = imax*jmax rslmsk = 0. +! TG3 MODS BEGIN + if(kpds5 == kpdtsf .and. imax == 138 .and. jmax == 116 + & .and. kpds4 == 128) then +! print*,'turn off setrmsk for tg3' + lmask = .false. + + elseif(kpds5 == kpdtsf) then +! TG3 MODS END ! ! surface temperature ! - if(kpds5.eq.kpdtsf) then -! lmask=.false. + lmask = .false. call ga2la(slmask,igaul,jgaul,rslmsk,imax,jmax,wlon,rnlat &, rlnout, rltout, gaus, blno, blto) ! &, dlon, dlat, gaus, blno, blto) - crit=0.5 + crit = 0.5 call rof01(rslmsk,ijmax,'ge',crit) - lmask=.true. + lmask = .true. ! ! bucket soil wetness ! @@ -6096,16 +6105,16 @@ subroutine setrmsk(kpds5,slmask,igaul,jgaul,wlon,rnlat, call ga2la(slmask,igaul,jgaul,rslmsk,imax,jmax,wlon,rnlat &, rlnout, rltout, gaus, blno, blto) ! &, dlon, dlat, gaus, blno, blto) - crit=0.5 + crit = 0.5 call rof01(rslmsk,ijmax,'ge',crit) - lmask=.true. + lmask = .true. ! write(6,*) 'wet rslmsk' ! znnt=1. ! call nntprt(rslmsk,ijmax,znnt) ! ! snow depth ! - elseif(kpds5.eq.kpdsnd) then + elseif(kpds5 == kpdsnd) then if(kpds4 == 192) then ! use the bitmap rslmsk = 0. do j = 1, jmax @@ -6962,51 +6971,51 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, ! ! get tsf climatology for the begining of the forecast ! - if (fh .gt. 0.0) then + if (fh > 0.0) then !cbosu if (me == 0) print*,'bosu fh gt 0' - iy4=iy - if(iy.lt.101) iy4=1900+iy4 - fha=0 - ida=0 - jda=0 -! fha(2)=nint(fh) - ida(1)=iy - ida(2)=im - ida(3)=id - ida(5)=ih + iy4 = iy + if (iy < 101) iy4 = 1900 + iy4 + fha = 0 + ida = 0 + jda = 0 +! fha(2) = nint(fh) + ida(1) = iy + ida(2) = im + ida(3) = id + ida(5) = ih call w3kind(w3kindreal,w3kindint) if(w3kindreal == 4) then - fha4=fha + fha4 = fha call w3movdat(fha4,ida,jda) else call w3movdat(fha,ida,jda) endif - jy=jda(1) - jm=jda(2) - jd=jda(3) - jh=jda(5) - if (me .eq. 0) write(6,*) ' forecast jy,jm,jd,jh', - & jy,jm,jd,jh + jy = jda(1) + jm = jda(2) + jd = jda(3) + jh = jda(5) + if (me == 0) write(6,*) ' forecast jy,jm,jd,jh', + & jy,jm,jd,jh jdow = 0 jdoy = 0 jday = 0 call w3doxdat(jda,jdow,jdoy,jday) - rjday=jdoy+jda(5)/24. - if(rjday.lt.dayhf(1)) rjday=rjday+365. + rjday = jdoy + jda(5) / 24. + if(rjday < dayhf(1)) rjday = rjday + 365. ! - if (me .eq. 0) write(6,*) 'forecast jy,jm,jd,jh=',jy,jm,jd,jh + if (me == 0) write(6,*) 'forecast jy,jm,jd,jh=',jy,jm,jd,jh ! ! for monthly mean climatology ! monend = 12 do mm=1,monend - mmm=mm - mmp=mm+1 - if(rjday.ge.dayhf(mmm).and.rjday.lt.dayhf(mmp)) then - mon1=mmm - mon2=mmp + mmm = mm + mmp = mm + 1 + if(rjday >= dayhf(mmm) .and. rjday < dayhf(mmp)) then + mon1 = mmm + mon2 = mmp go to 10 endif enddo @@ -7014,17 +7023,18 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, call abort 10 continue wei1m = (dayhf(mon2)-rjday)/(dayhf(mon2)-dayhf(mon1)) - wei2m = (rjday-dayhf(mon1))/(dayhf(mon2)-dayhf(mon1)) - if(mon2.eq.13) mon2=1 - if (me .eq. 0) print *,'rjday,mon1,mon2,wei1m,wei2m=', - & rjday,mon1,mon2,wei1m,wei2m + wei2m = 1.0 - wei1m +! wei2m = (rjday-dayhf(mon1))/(dayhf(mon2)-dayhf(mon1)) + if (mon2 == 13) mon2 = 1 + if (me == 0) print *,'rjday,mon1,mon2,wei1m,wei2m=', + & rjday,mon1,mon2,wei1m,wei2m ! ! read monthly mean climatology of tsf ! kpd7 = -1 do nn=1,2 mon = mon1 - if (nn .eq. 2) mon = mon2 + if (nn == 2) mon = mon2 call fixrdc(lugb,fntsfc,kpdtsf,kpd7,mon,slmask, & tsf(1,nn),len,iret &, imsk, jmsk, slmskh, gaus,blno, blto @@ -7041,8 +7051,8 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, ! ! compute current jy,jm,jd,jh of forecast and the day of the year ! - iy4=iy - if(iy.lt.101) iy4=1900+iy4 + iy4 = iy + if (iy < 101) iy4=1900+iy4 fha = 0 ida = 0 jda = 0 @@ -7052,8 +7062,8 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, ida(3) = id ida(5) = ih call w3kind(w3kindreal,w3kindint) - if(w3kindreal==4) then - fha4=fha + if(w3kindreal == 4) then + fha4 = fha call w3movdat(fha4,ida,jda) else call w3movdat(fha,ida,jda) @@ -7068,44 +7078,45 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, jdoy = 0 jday = 0 call w3doxdat(jda,jdow,jdoy,jday) - rjday = jdoy+jda(5)/24. - if(rjday.lt.dayhf(1)) rjday=rjday+365. + rjday = jdoy + jda(5) / 24. + if(rjday < dayhf(1)) rjday = rjday + 365. - if (me .eq. 0) write(6,*) ' forecast jy,jm,jd,jh,rjday=', - & jy,jm,jd,jh,rjday + if (me == 0) write(6,*) ' forecast jy,jm,jd,jh,rjday=', + & jy,jm,jd,jh,rjday ! - if (me .eq. 0) write(6,*) 'forecast jy,jm,jd,jh=',jy,jm,jd,jh + if (me == 0) write(6,*) 'forecast jy,jm,jd,jh=',jy,jm,jd,jh ! ! for monthly mean climatology ! monend = 12 do mm=1,monend - mmm=mm - mmp=mm+1 - if(rjday.ge.dayhf(mmm).and.rjday.lt.dayhf(mmp)) then - mon1=mmm - mon2=mmp + mmm = mm + mmp = mm + 1 + if(rjday >= dayhf(mmm) .and. rjday < dayhf(mmp)) then + mon1 = mmm + mon2 = mmp go to 20 endif enddo print *,'wrong rjday',rjday call abort 20 continue - wei1m=(dayhf(mon2)-rjday)/(dayhf(mon2)-dayhf(mon1)) - wei2m=(rjday-dayhf(mon1))/(dayhf(mon2)-dayhf(mon1)) - if(mon2.eq.13) mon2=1 - if (me .eq. 0) print *,'rjday,mon1,mon2,wei1m,wei2m=', - & rjday,mon1,mon2,wei1m,wei2m + wei1m = (dayhf(mon2)-rjday)/(dayhf(mon2)-dayhf(mon1)) + wei2m = 1.0 - wei1m +! wei2m = (rjday-dayhf(mon1))/(dayhf(mon2)-dayhf(mon1)) + if (mon2 == 13) mon2 = 1 + if (me == 0) print *,'rjday,mon1,mon2,wei1m,wei2m=', + & rjday,mon1,mon2,wei1m,wei2m ! ! for seasonal mean climatology ! monend = 4 is = im/3 + 1 - if (is.eq.5) is = 1 + if (is == 5) is = 1 do mm=1,monend mmm = mm*3 - 2 mmp = (mm+1)*3 - 2 - if(rjday.ge.dayhf(mmm).and.rjday.lt.dayhf(mmp)) then + if(rjday >= dayhf(mmm) .and. rjday < dayhf(mmp)) then sea1 = mmm sea2 = mmp go to 30 @@ -7115,20 +7126,21 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, call abort 30 continue wei1s = (dayhf(sea2)-rjday)/(dayhf(sea2)-dayhf(sea1)) - wei2s = (rjday-dayhf(sea1))/(dayhf(sea2)-dayhf(sea1)) - if(sea2.eq.13) sea2=1 - if (me .eq. 0) print *,'rjday,sea1,sea2,wei1s,wei2s=', - & rjday,sea1,sea2,wei1s,wei2s + wei2s = 1.0 - wei1s +! wei2s = (rjday-dayhf(sea1))/(dayhf(sea2)-dayhf(sea1)) + if (sea2 == 13) sea2 = 1 + if (me == 0) print *,'rjday,sea1,sea2,wei1s,wei2s=', + & rjday,sea1,sea2,wei1s,wei2s ! ! for summer and winter values (maximum and minimum). ! monend = 2 is = im/6 + 1 - if (is.eq.3) is = 1 + if (is == 3) is = 1 do mm=1,monend mmm = mm*6 - 5 mmp = (mm+1)*6 - 5 - if(rjday.ge.dayhf(mmm).and.rjday.lt.dayhf(mmp)) then + if(rjday >= dayhf(mmm) .and. rjday < dayhf(mmp)) then hyr1 = mmm hyr2 = mmp go to 31 @@ -7138,10 +7150,11 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, call abort 31 continue wei1y = (dayhf(hyr2)-rjday)/(dayhf(hyr2)-dayhf(hyr1)) - wei2y = (rjday-dayhf(hyr1))/(dayhf(hyr2)-dayhf(hyr1)) - if(hyr2.eq.13) hyr2=1 - if (me .eq. 0) print *,'rjday,hyr1,hyr2,wei1y,wei2y=', - & rjday,hyr1,hyr2,wei1y,wei2y + wei2y = 1.0 - wei1y +! wei2y = (rjday-dayhf(hyr1))/(dayhf(hyr2)-dayhf(hyr1)) + if (hyr2 == 13) hyr2 = 1 + if (me == 0) print *,'rjday,hyr1,hyr2,wei1y,wei2y=', + & rjday,hyr1,hyr2,wei1y,wei2y ! ! start reading in climatology and interpolate to the date ! @@ -7541,7 +7554,7 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, ! mon1s = mon1 ; mon2s = mon2 ; sea1s = sea1 ; sea2s = sea2 ! - if (me .eq. 0) print *,' mon1s=',mon1s,' mon2s=',mon2s + if (me == 0) print *,' mon1s=',mon1s,' mon2s=',mon2s &,' sea1s=',sea1s,' sea2s=',sea2s ! k1 = 1 ; k2 = 2 @@ -7599,11 +7612,11 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, ! seasonal mean climatology ! isx = sea2/3 + 1 - if (isx .eq. 5) isx = 1 - if(isx.eq.1) kpd9 = 12 - if(isx.eq.2) kpd9 = 3 - if(isx.eq.3) kpd9 = 6 - if(isx.eq.4) kpd9 = 9 + if (isx == 5) isx = 1 + if (isx == 1) kpd9 = 12 + if (isx == 2) kpd9 = 3 + if (isx == 3) kpd9 = 6 + if (isx == 4) kpd9 = 9 ! ! albedo ! there are four albedo fields in this version: @@ -7639,7 +7652,7 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, if (me == 0) print*,'bosu 2nd time in clima for month ', & mon, k1,k2 if ( index(fnalbc, "tileX.nc") == 0) then ! grib file - kpd7=-1 + kpd7 = -1 do k = 1, 4 call fixrdc(lugb,fnalbc,kpdalb(k),kpd7,mon,slmask, & alb(1,k,nn),len,iret @@ -7656,7 +7669,7 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, ! ! tsf at the current time t ! - kpd7=-1 + kpd7 = -1 call fixrdc(lugb,fntsfc,kpdtsf,kpd7,mon,slmask, & tsf(1,nn),len,iret &, imsk, jmsk, slmskh, gaus,blno, blto @@ -7664,13 +7677,13 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, ! ! soil wetness ! - if(fnwetc(1:8).ne.' ') then + if (fnwetc(1:8).ne.' ') then kpd7=-1 call fixrdc(lugb,fnwetc,kpdwet,kpd7,mon,slmask, & wet(1,nn),len,iret &, imsk, jmsk, slmskh, gaus,blno, blto &, outlat, outlon, me) - elseif(fnsmcc(1:8).ne.' ') then + elseif (fnsmcc(1:8).ne.' ') then if (index(fnsmcc,'global_soilmcpc.1x1.grb') /= 0) then ! the old climo data kpd7=-1 call fixrdc(lugb,fnsmcc,kpdsmc,kpd7,mon,slmask, @@ -7712,13 +7725,13 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, ! ! sea ice ! - kpd7=-1 - if(fnacnc(1:8).ne.' ') then + kpd7 = -1 + if (fnacnc(1:8).ne.' ') then call fixrdc(lugb,fnacnc,kpdacn,kpd7,mon,slmask, & acn(1,nn),len,iret &, imsk, jmsk, slmskh, gaus,blno, blto &, outlat, outlon, me) - elseif(fnaisc(1:8).ne.' ') then + elseif (fnaisc(1:8).ne.' ') then call fixrdc(lugb,fnaisc,kpdais,kpd7,mon,slmask, & ais(1,nn),len,iret &, imsk, jmsk, slmskh, gaus,blno, blto @@ -7738,7 +7751,7 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, ! ! snow cover ! - if(fnscvc(1:8).ne.' ') then + if (fnscvc(1:8).ne.' ') then kpd7=-1 call fixrdc(lugb,fnscvc,kpdscv,kpd7,mon,slmask, & scv(1,nn),len,iret @@ -7749,7 +7762,7 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, ! ! surface roughness ! - if(fnzorc(1:3) == 'sib') then + if (fnzorc(1:3) == 'sib') then if (me == 0) then write(6,*) 'roughness length to be set from sib veg type' endif @@ -7767,7 +7780,7 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, ! ! vegetation cover ! - if(fnvegc(1:8).ne.' ') then + if (fnvegc(1:8) .ne. ' ') then if ( index(fnvegc, "tileX.nc") == 0) then ! grib file kpd7=-1 call fixrdc(lugb,fnvegc,kpdveg,kpd7,mon,slmask, @@ -7789,35 +7802,35 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, ! when chosen, set the z0 based on the vegetation type. ! for this option to work, namelist variable fnvetc must be ! set to point at the proper vegetation type file. - if(fnzorc(1:3) == 'sib') then - if(fnvetc(1:4) == ' ') then + if (fnzorc(1:3) == 'sib') then + if (fnvetc(1:4) == ' ') then if (me==0) write(6,*) "must choose sib veg type climo file" call abort endif zorclm = 0.0 do i=1,len - ivtyp=nint(vet(i)) + ivtyp = nint(vet(i)) if (ivtyp >= 1 .and. ivtyp <= 13) then zorclm(i) = z0_sib(ivtyp) endif enddo elseif(fnzorc(1:4) == 'igbp') then - if(fnvetc(1:4) == ' ') then - if (me==0) write(6,*) "must choose igbp veg type climo file" + if (fnvetc(1:4) == ' ') then + if (me == 0) write(6,*) "must choose igbp veg type climo file" call abort endif zorclm = 0.0 do i=1,len - ivtyp=nint(vet(i)) + ivtyp = nint(vet(i)) if (ivtyp >= 1 .and. ivtyp <= 20) then z0_season(1) = z0_igbp_min(ivtyp) z0_season(7) = z0_igbp_max(ivtyp) - if(outlat(i) < 0.0)then + if (outlat(i) < 0.0) then zorclm(i) = wei1y * z0_season(hyr2) + - & wei2y *z0_season(hyr1) + & wei2y * z0_season(hyr1) else zorclm(i) = wei1y * z0_season(hyr1) + - & wei2y *z0_season(hyr2) + & wei2y * z0_season(hyr2) endif endif enddo @@ -8148,7 +8161,7 @@ subroutine fixrdc(lugb,fngrib,kpds5,kpds7,mon,slmask, real (kind=kind_io8), allocatable :: rlngrb(:), rltgrb(:) ! logical lmask, yr2kc, gaus, ijordr - logical*1 lbms(mdata) + logical*1, allocatable :: lbms(:) ! integer, intent(in) :: kpds7 integer kpds(1000),kgds(1000) @@ -8156,6 +8169,7 @@ subroutine fixrdc(lugb,fngrib,kpds5,kpds7,mon,slmask, real (kind=kind_io8) outlat(len), outlon(len) ! allocate(data8(1:mdata)) + allocate(lbms(mdata)) ! ! integer imax_sv, jmax_sv, wlon_sv, rnlat_sv, kpds1_sv ! date imax_sv/0/, jmax_sv/0/, wlon_sv/999.0/, rnlat_sv/999.0/ @@ -8291,6 +8305,7 @@ subroutine fixrdc(lugb,fngrib,kpds5,kpds7,mon,slmask, call baclose(lugb,iret) ! deallocate(data8) + deallocate(lbms) return end subroutine fixrdc subroutine fixrda(lugb,fngrib,kpds5,slmask, diff --git a/sorc/global_cycle.fd/cycle.f90 b/sorc/global_cycle.fd/cycle.f90 old mode 100755 new mode 100644 index c5a7123c6..5aa800f9b --- a/sorc/global_cycle.fd/cycle.f90 +++ b/sorc/global_cycle.fd/cycle.f90 @@ -227,7 +227,8 @@ SUBROUTINE SFCDRV(LUGB,IDIM,JDIM,LENSFC,LSOIL,DELTSFC, & REAL, ALLOCATABLE :: SLIFCS_FG(:) TYPE(NSST_DATA) :: NSST - + real, dimension(idim,jdim) :: tf_clm,tf_trd,sal_clm + real, dimension(lensfc) :: tf_clm_tile,tf_trd_tile,sal_clm_tile !-------------------------------------------------------------------------------- ! GSI_FILE is the path/name of the gaussian GSI file which contains NSST ! increments. @@ -461,9 +462,27 @@ SUBROUTINE SFCDRV(LUGB,IDIM,JDIM,LENSFC,LSOIL,DELTSFC, & ELSE PRINT* PRINT*,"ADJUST TREF FROM GSI INCREMENT" +! +! Get tf climatology at the time +! + call get_tf_clm(rla,rlo,jdim,idim,iy,im,id,ih,tf_clm,tf_trd) + tf_clm_tile(:) = reshape(tf_clm, (/lensfc/) ) + tf_trd_tile(:) = reshape(tf_trd, (/lensfc/) ) +! +! Get salinity climatology at the time +! + call get_sal_clm(rla,rlo,jdim,idim,iy,im,id,ih,sal_clm) + sal_clm_tile(:) = reshape(sal_clm, (/lensfc/) ) +! +! read tf analysis increment generated by GSI +! CALL READ_GSI_DATA(GSI_FILE) - CALL ADJUST_NSST(RLA,RLO,SLIFCS,SLIFCS_FG,TSFFCS,SITFCS,STCFCS, & - NSST,LENSFC,LSOIL,IDIM,JDIM,ZSEA1,ZSEA2,IM,ID,DELTSFC) +! +! update foundation & surface temperature for NSST +! + CALL ADJUST_NSST(RLA,RLO,SLIFCS,SLIFCS_FG,TSFFCS,SITFCS,SICFCS,STCFCS, & + NSST,LENSFC,LSOIL,IDIM,JDIM,ZSEA1,ZSEA2,IM,ID,DELTSFC, & + tf_clm_tile,tf_trd_tile,sal_clm_tile) ENDIF ENDIF @@ -508,8 +527,9 @@ SUBROUTINE SFCDRV(LUGB,IDIM,JDIM,LENSFC,LSOIL,DELTSFC, & END SUBROUTINE SFCDRV SUBROUTINE ADJUST_NSST(RLA,RLO,SLMSK_TILE,SLMSK_FG_TILE,SKINT_TILE,& - SICET_TILE,SOILT_TILE,NSST,LENSFC,LSOIL, & - IDIM,JDIM,ZSEA1,ZSEA2,MON,DAY,DELTSFC) + SICET_TILE,sice_tile,SOILT_TILE,NSST,LENSFC,LSOIL, & + IDIM,JDIM,ZSEA1,ZSEA2,MON,DAY,DELTSFC, & + tf_clm_tile,tf_trd_tile,sal_clm_tile) !-------------------------------------------------------------------------------- ! READ IN GSI FILE WITH THE UPDATED TREF INCREMENTS (ON THE GAUSSIAN @@ -529,14 +549,14 @@ SUBROUTINE ADJUST_NSST(RLA,RLO,SLMSK_TILE,SLMSK_FG_TILE,SKINT_TILE,& INTEGER, INTENT(IN) :: LENSFC, LSOIL, IDIM, JDIM, MON, DAY REAL, INTENT(IN) :: SLMSK_TILE(LENSFC), SLMSK_FG_TILE(LENSFC) + real, intent(in) :: tf_clm_tile(lensfc),tf_trd_tile(lensfc),sal_clm_tile(lensfc) REAL, INTENT(IN) :: ZSEA1, ZSEA2, DELTSFC REAL, INTENT(INOUT) :: RLA(LENSFC), RLO(LENSFC), SKINT_TILE(LENSFC) - REAL, INTENT(INOUT) :: SICET_TILE(LENSFC), SOILT_TILE(LENSFC,LSOIL) + REAL, INTENT(INOUT) :: SICET_TILE(LENSFC),sice_tile(lensfc),SOILT_TILE(LENSFC,LSOIL) TYPE(NSST_DATA) :: NSST - REAL, PARAMETER :: TFREEZ=271.21 - REAL, PARAMETER :: TMAX=313.0 + REAL, PARAMETER :: TMAX=313.0,tzero=273.16 INTEGER :: IOPT, NRET, KGDS_GAUS(200) INTEGER :: IGAUS, JGAUS, IJ, II, JJ, III, JJJ, KRAD @@ -545,11 +565,16 @@ SUBROUTINE ADJUST_NSST(RLA,RLO,SLMSK_TILE,SLMSK_FG_TILE,SKINT_TILE,& INTEGER :: ITILE, JTILE INTEGER :: MAX_SEARCH, J INTEGER :: IGAUSP1, JGAUSP1 + integer :: nintp,nsearched,nice,nland + integer :: nfill,nfill_tice,nfill_clm + integer :: nset_thaw,nset_thaw_s,nset_thaw_i,nset_thaw_c + INTEGER, ALLOCATABLE :: ID1(:,:), ID2(:,:), JDC(:,:) LOGICAL :: IS_ICE - REAL :: TREF_SAVE, WSUM + real :: tfreez + REAL :: TREF_SAVE,WSUM,tf_ice,tf_thaw REAL :: FILL, DTZM, GAUS_RES_KM, DTREF REAL, ALLOCATABLE :: XPTS(:), YPTS(:), LATS(:), LONS(:) REAL, ALLOCATABLE :: DUM2D(:,:), LATS_RAD(:), LONS_RAD(:) @@ -646,6 +671,20 @@ SUBROUTINE ADJUST_NSST(RLA,RLO,SLMSK_TILE,SLMSK_FG_TILE,SKINT_TILE,& PRINT*,'MAXIMUM SEARCH IS ',MAX_SEARCH, ' GAUSSIAN POINTS.' PRINT* +! +! Initialize variables for counts statitics to be zeros +! + nintp = 0 + nset_thaw = 0 + nset_thaw_s = 0 + nset_thaw_i = 0 + nset_thaw_c = 0 + nsearched = 0 + nfill = 0 + nfill_tice = 0 + nfill_clm = 0 + nice = 0 + nland = 0 !---------------------------------------------------------------------- ! TREF INCREMENT WILL BE OUTPUT. INITIALIZE TO ZERO. !---------------------------------------------------------------------- @@ -657,31 +696,50 @@ SUBROUTINE ADJUST_NSST(RLA,RLO,SLMSK_TILE,SLMSK_FG_TILE,SKINT_TILE,& MASK_TILE = NINT(SLMSK_TILE(IJ)) MASK_FG_TILE = NINT(SLMSK_FG_TILE(IJ)) +! +! when sea ice exists, get salinity dependent water temperature +! + tf_ice = tfreez(sal_clm_tile(ij)) + tzero !---------------------------------------------------------------------- -! THESE ARE ICE POINTS. SET TREF TO FREEZING. +! SKIP LAND POINTS. NSST NOT APPLIED AT LAND. !---------------------------------------------------------------------- - IF (MASK_TILE == 2) THEN - NSST%TREF(IJ) = TFREEZ - CYCLE IJ_LOOP + IF (MASK_TILE == 1) THEN + nland = nland + 1 + CYCLE IJ_LOOP ENDIF -!---------------------------------------------------------------------- -! IF THE MODEL POINT WAS ICE COVERED, BUT IS NOW OPEN WATER, SET -! TREF TO FREEZING, XZ TO '30' AND ALL OTHER FIELDS TO ZERO. -!---------------------------------------------------------------------- +! +! these are ice points. set tref to tf_ice and update tmpsfc. +! + if (mask_tile == 2) then + nsst%tref(ij)=tf_ice ! water part tmp set + skint_tile(ij)=(1.0-sice_tile(ij))*nsst%tref(ij)+sice_tile(ij)*sicet_tile(ij) + nice = nice + 1 + cycle ij_loop + endif - IF (MASK_FG_TILE == 2 .AND. MASK_TILE == 0) THEN - CALL NSST_WATER_RESET(NSST,IJ,TFREEZ) - CYCLE IJ_LOOP - ENDIF +! +! Get i,j index on array of (idim,jdim) from known ij +! + JTILE = (IJ-1) / IDIM + 1 + ITILE = MOD(IJ,IDIM) + IF (ITILE==0) ITILE = IDIM !---------------------------------------------------------------------- -! SKIP LAND POINTS. NSST NOT APPLIED AT LAND. +! IF THE MODEL POINT WAS ICE COVERED, BUT IS NOW OPEN WATER, SET +! TREF TO searched adjascent open water onea, if failed the search, set to +! weighted average of tf_ice and tf_clm. For NSST vars, set xz TO '30' AND ALL OTHER FIELDS TO ZERO. !---------------------------------------------------------------------- - IF (MASK_TILE == 1) THEN - CYCLE IJ_LOOP + IF (MASK_FG_TILE == 2 .AND. MASK_TILE == 0) THEN +! +! set background for the thaw (just melted water) situation +! + call tf_thaw_set(nsst%tref,nint(slmsk_fg_tile),itile,jtile,tf_ice,tf_clm_tile(ij),tf_thaw,idim,jdim, & + nset_thaw_s,nset_thaw_i,nset_thaw_c) + call nsst_water_reset(nsst,ij,tf_thaw) + nset_thaw = nset_thaw + 1 ENDIF !---------------------------------------------------------------------- @@ -692,11 +750,6 @@ SUBROUTINE ADJUST_NSST(RLA,RLO,SLMSK_TILE,SLMSK_FG_TILE,SKINT_TILE,& ! IT IS SIMPLY A FILLER VALUE. THESE FIELDS ARE NOT USED AT ! OPEN WATER POINTS. !---------------------------------------------------------------------- - - JTILE = (IJ-1) / IDIM + 1 - ITILE = MOD(IJ,IDIM) - IF (ITILE==0) ITILE = IDIM - !---------------------------------------------------------------------- ! SEE IF ANY OF THE NEAREST GSI POINTS MASK AREA OPEN WATER. ! IF SO, APPLY NSST INCREMENT USING BILINEAR INTERPOLATION. @@ -735,11 +788,12 @@ SUBROUTINE ADJUST_NSST(RLA,RLO,SLMSK_TILE,SLMSK_FG_TILE,SKINT_TILE,& WSUM = WSUM + S2C(ITILE,JTILE,4) ENDIF + nintp = nintp + 1 DTREF = DTREF / WSUM TREF_SAVE = NSST%TREF(IJ) NSST%TREF(IJ) = NSST%TREF(IJ) + DTREF - NSST%TREF(IJ) = MAX(NSST%TREF(IJ), TFREEZ) + NSST%TREF(IJ) = MAX(NSST%TREF(IJ), tf_ice) NSST%TREF(IJ) = MIN(NSST%TREF(IJ), TMAX) NSST%TFINC(IJ) = NSST%TREF(IJ) - TREF_SAVE @@ -747,7 +801,7 @@ SUBROUTINE ADJUST_NSST(RLA,RLO,SLMSK_TILE,SLMSK_FG_TILE,SKINT_TILE,& NSST%Z_C(IJ),ZSEA1,ZSEA2,DTZM) SKINT_TILE(IJ) = NSST%TREF(IJ) + DTZM - SKINT_TILE(IJ) = MAX(SKINT_TILE(IJ), TFREEZ) + SKINT_TILE(IJ) = MAX(SKINT_TILE(IJ), tf_ice) SKINT_TILE(IJ) = MIN(SKINT_TILE(IJ), TMAX) SICET_TILE(IJ) = SKINT_TILE(IJ) @@ -796,12 +850,13 @@ SUBROUTINE ADJUST_NSST(RLA,RLO,SLMSK_TILE,SLMSK_FG_TILE,SKINT_TILE,& IF (SLMSK_GAUS(III,JJJ) == 0) THEN - PRINT*,'MISMATCH AT TILE POINT ',ITILE,JTILE - PRINT*,'UPDATE TREF USING GSI INCREMENT AT ',III,JJJ,DTREF_GAUS(III,JJJ) +! PRINT*,'MISMATCH AT TILE POINT ',ITILE,JTILE +! PRINT*,'UPDATE TREF USING GSI INCREMENT AT ',III,JJJ,DTREF_GAUS(III,JJJ) + nsearched = nsearched + 1 TREF_SAVE = NSST%TREF(IJ) NSST%TREF(IJ ) = NSST%TREF(IJ) + DTREF_GAUS(III,JJJ) - NSST%TREF(IJ) = MAX(NSST%TREF(IJ), TFREEZ) + NSST%TREF(IJ) = MAX(NSST%TREF(IJ), tf_ice) NSST%TREF(IJ) = MIN(NSST%TREF(IJ), TMAX) NSST%TFINC(IJ) = NSST%TREF(IJ) - TREF_SAVE @@ -809,7 +864,7 @@ SUBROUTINE ADJUST_NSST(RLA,RLO,SLMSK_TILE,SLMSK_FG_TILE,SKINT_TILE,& NSST%Z_C(IJ),ZSEA1,ZSEA2,DTZM) SKINT_TILE(IJ) = NSST%TREF(IJ) + DTZM - SKINT_TILE(IJ) = MAX(SKINT_TILE(IJ), TFREEZ) + SKINT_TILE(IJ) = MAX(SKINT_TILE(IJ), tf_ice) SKINT_TILE(IJ) = MIN(SKINT_TILE(IJ), TMAX) SICET_TILE(IJ) = SKINT_TILE(IJ) @@ -832,26 +887,28 @@ SUBROUTINE ADJUST_NSST(RLA,RLO,SLMSK_TILE,SLMSK_FG_TILE,SKINT_TILE,& ! ELSE UPDATE TREF BASED ON THE ANNUAL SST CYCLE. !---------------------------------------------------------------------- - PRINT*,'WARNING !!!!!! SEARCH FAILED AT TILE POINT ',ITILE,JTILE +! PRINT*,'WARNING !!!!!! SEARCH FAILED AT TILE POINT ',ITILE,JTILE + nfill = nfill + 1 IF (IS_ICE) THEN - NSST%TREF(IJ) = TFREEZ - PRINT*,"NEARBY ICE. SET TREF TO FREEZING" + NSST%TREF(IJ) = tf_ice +! PRINT*,"NEARBY ICE. SET TREF TO FREEZING" + nfill_tice = nfill_tice + 1 ELSE - CALL CLIMO_TREND(RLA(IJ),MON,DAY,DELTSFC,DTREF) TREF_SAVE = NSST%TREF(IJ) - NSST%TREF(IJ) = NSST%TREF(IJ) + DTREF - NSST%TREF(IJ) = MAX(NSST%TREF(IJ), TFREEZ) + NSST%TREF(IJ) = NSST%TREF(IJ) + TF_TRD_TILE(IJ) + NSST%TREF(IJ) = MAX(NSST%TREF(IJ), tf_ice) NSST%TREF(IJ) = MIN(NSST%TREF(IJ), TMAX) NSST%TFINC(IJ) = NSST%TREF(IJ) - TREF_SAVE - PRINT*,'UPDATE TREF FROM SST CLIMO ',DTREF +! PRINT*,'UPDATE TREF FROM SST CLIMO ',DTREF + nfill_clm = nfill_clm + 1 ENDIF CALL DTZM_POINT(NSST%XT(IJ),NSST%XZ(IJ),NSST%DT_COOL(IJ), & NSST%Z_C(IJ),ZSEA1,ZSEA2,DTZM) SKINT_TILE(IJ) = NSST%TREF(IJ) + DTZM - SKINT_TILE(IJ) = MAX(SKINT_TILE(IJ), TFREEZ) + SKINT_TILE(IJ) = MAX(SKINT_TILE(IJ), tf_ice) SKINT_TILE(IJ) = MIN(SKINT_TILE(IJ), TMAX) SICET_TILE(IJ) = SKINT_TILE(IJ) @@ -861,6 +918,14 @@ SUBROUTINE ADJUST_NSST(RLA,RLO,SLMSK_TILE,SLMSK_FG_TILE,SKINT_TILE,& ENDDO IJ_LOOP + write(*,'(a)') 'statistics of grids number processed for tile : ' + write(*,'(a,I8)') ' nintp = ',nintp + write(*,'(a,4I8)') 'nset_thaw,nset_thaw_s,nset_thaw_i,nset_thaw_c =',nset_thaw,nset_thaw_s,nset_thaw_i,nset_thaw_c + write(*,'(a,I8)') ' nsearched = ',nsearched + write(*,'(a,3I6)') ' nfill,nfill_tice,nfill_clm = ',nfill,nfill_tice,nfill_clm + write(*,'(a,I8)') ' nice = ',nice + write(*,'(a,I8)') ' nland = ',nland + DEALLOCATE(ID1, ID2, JDC, S2C) END SUBROUTINE ADJUST_NSST @@ -1095,44 +1160,6 @@ SUBROUTINE DTZM_POINT(XT,XZ,DT_COOL,ZC,Z1,Z2,DTZM) END SUBROUTINE DTZM_POINT - SUBROUTINE NSST_WATER_RESET(NSST,IJ,TFREEZ) - -!------------------------------------------------------------------- -! IF THE FIRST GUESS WAS SEA ICE, BUT THE ANALYSIS IS OPEN WATER, -! RESET ALL NSST VARIABLES. -!------------------------------------------------------------------- - - USE READ_WRITE_DATA, ONLY : NSST_DATA - - IMPLICIT NONE - - INTEGER, INTENT(IN) :: IJ - - REAL, INTENT(IN) :: TFREEZ - - TYPE(NSST_DATA), INTENT(INOUT) :: NSST - - NSST%C_0(IJ) = 0.0 - NSST%C_D(IJ) = 0.0 - NSST%D_CONV(IJ) = 0.0 - NSST%DT_COOL(IJ) = 0.0 - NSST%IFD(IJ) = 0.0 - NSST%QRAIN(IJ) = 0.0 - NSST%TREF(IJ) = TFREEZ - NSST%W_0(IJ) = 0.0 - NSST%W_D(IJ) = 0.0 - NSST%XS(IJ) = 0.0 - NSST%XT(IJ) = 0.0 - NSST%XTTS(IJ) = 0.0 - NSST%XU(IJ) = 0.0 - NSST%XV(IJ) = 0.0 - NSST%XZ(IJ) = 30.0 - NSST%XZTS(IJ) = 0.0 - NSST%Z_C(IJ) = 0.0 - NSST%ZM(IJ) = 0.0 - - END SUBROUTINE NSST_WATER_RESET - SUBROUTINE REMAP_COEF( is, ie, js, je,& im, jm, lon, lat, id1, id2, jdc, s2c, agrid ) @@ -1217,3 +1244,502 @@ SUBROUTINE REMAP_COEF( is, ie, js, je,& 5000 continue ! j-loop END SUBROUTINE REMAP_COEF + + subroutine tf_thaw_set(tf_ij,mask_ij,itile,jtile,tice,tclm,tf_thaw,nx,ny, & + nset_thaw_s,nset_thaw_i,nset_thaw_c) +! +! set a vakue to tf background for the thaw (just melted water) situation +! +!Input/output: +! tf : Foundation temperature background on FV3 native grids +! mask_ij : mask of the tile (FV3 native grids) +! itile,jtile : location index of the tile +! tice : water temperature (calulated with a salinity formula) +! tclm : SST climatology valid at the analysis time + + real, dimension(nx*ny), intent(in) :: tf_ij + integer, dimension(nx*ny), intent(in) :: mask_ij + real, intent(in) :: tice,tclm + integer, intent(in) :: itile,jtile,nx,ny + real, intent(out) :: tf_thaw + integer, intent(inout) :: nset_thaw_s,nset_thaw_i,nset_thaw_c +! Local + real, parameter :: bmiss = -999.0 + real, dimension(nx,ny) :: tf + integer, dimension(nx,ny) :: mask + integer :: krad,max_search,istart,iend,jstart,jend + integer :: ii,jj,iii,jjj + logical :: is_ice + + max_search = 2 + + mask(:,:) = reshape(mask_ij,(/nx,ny/) ) + tf(:,:) = reshape(tf_ij,(/nx,ny/) ) + + tf_thaw = bmiss + + do krad = 1, max_search + + istart = itile - krad + iend = itile + krad + jstart = jtile - krad + jend = jtile + krad + + do jj = jstart, jend + do ii = istart, iend + + + if ((jj == jstart) .or. (jj == jend) .or. & + (ii == istart) .or. (ii == iend)) then + + if ((jj >= 1) .and. (jj <= ny)) then + jjj = jj + if (ii <= 0) then + iii = nx + ii + else if (ii >= (nx+1)) then + iii = ii - nx + else + iii = ii + endif + +!---------------------------------------------------------------------- +! SEE IF NEARBY POINTS ARE SEA ICE. IF THEY ARE, AND THE SEARCH FOR +! A GAUSSIAN GRID OPEN WATER POINT FAILS, THEN TREF WILL BE SET TO +! FREEZING BELOW. +!---------------------------------------------------------------------- + if (krad <= 2 .and. mask(iii,jjj) == 2) is_ice = .true. + + if (mask(iii,jjj) == 0) then + tf_thaw = tf(iii,jjj) + nset_thaw_s = nset_thaw_s + 1 + write(*,'(a,I4,2F9.3)') 'nset_thaw_s,tf(iii,jjj),tclm : ',nset_thaw_s,tf(iii,jjj),tclm + go to 100 + endif ! tile mask is open water + + endif + endif + enddo + enddo + enddo ! krad loop + + 100 continue + + if ( tf_thaw == bmiss ) then + if (is_ice) then + tf_thaw = tice + nset_thaw_i = nset_thaw_i + 1 + write(*,'(a,I4,F9.3)') 'nset_thaw_i,tf_ice : ',nset_thaw_i,tice + else + tf_thaw = 0.8*tice+0.2*tclm + nset_thaw_c = nset_thaw_c + 1 + write(*,'(a,I4,2F9.3)') 'nset_thaw_c,tf_ice,tclm : ',nset_thaw_c,tice,tclm + endif + endif + + end subroutine tf_thaw_set + + subroutine nsst_water_reset(nsst,ij,tf_thaw) + +!------------------------------------------------------------------- +! if the first guess was sea ice, but the analysis is open water, +! reset all nsst variables. +!------------------------------------------------------------------- + + use read_write_data, only : nsst_data + implicit none + + integer, intent(in) :: ij + + real, intent(in) :: tf_thaw + + type(nsst_data), intent(inout) :: nsst + + nsst%c_0(ij) = 0.0 + nsst%c_d(ij) = 0.0 + nsst%d_conv(ij) = 0.0 + nsst%dt_cool(ij) = 0.0 + nsst%ifd(ij) = 0.0 + nsst%qrain(ij) = 0.0 + nsst%tref(ij) = tf_thaw + nsst%w_0(ij) = 0.0 + nsst%w_d(ij) = 0.0 + nsst%xs(ij) = 0.0 + nsst%xt(ij) = 0.0 + nsst%xtts(ij) = 0.0 + nsst%xu(ij) = 0.0 + nsst%xv(ij) = 0.0 + nsst%xz(ij) = 30.0 + nsst%xzts(ij) = 0.0 + nsst%z_c(ij) = 0.0 + nsst%zm(ij) = 0.0 + + end subroutine nsst_water_reset + +subroutine get_tf_clm(xlats_ij,xlons_ij,ny,nx,iy,im,id,ih,tf_clm,tf_trd) +! +! abstract: get sst climatology at the valid time (atime) and target resolution (nx,ny) +! +! input & output +! + use read_write_data, only : get_tf_clm_dim + + implicit none + + real, dimension(nx*ny), intent(in) :: xlats_ij ! latitudes of target grids (nx*ny) + real, dimension(nx*ny), intent(in) :: xlons_ij ! latitudes of target grids (nx*ny) + real, dimension(nx,ny), intent(out) :: tf_clm ! sst climatology valid at atime (nx,ny) + real, dimension(nx,ny), intent(out) :: tf_trd ! 6-hourly sst climatology tendency valid at atime (nx,ny) + integer, intent(in) :: iy,im,id,ih,nx,ny +! local declare + real, allocatable, dimension(:,:) :: tf_clm0 ! sst climatology at the valid time (nxc,nyc) + real, allocatable, dimension(:,:) :: tf_trd0 ! 6-hourly sst climatology tendency valid at atime (nxc,nyc) + real, allocatable, dimension(:) :: cxlats ! latitudes of sst climatology + real, allocatable, dimension(:) :: cxlons ! longitudes of sst climatology + + real, dimension(nx*ny) :: tf_clm_ij ! sst climatology at target grids (nx*ny) + real, dimension(nx*ny) :: tf_trd_ij ! 6-hourly sst climatology tendency + real :: wei1,wei2 + integer :: nxc,nyc,mon1,mon2,i,j + character (len=6), parameter :: fin_tf_clm='sstclm' ! sst climatology file name +! +! get which two months used and their weights from atime +! + call get_tim_wei(iy,im,id,ih,mon1,mon2,wei1,wei2) +! +! get the dimensions of the sst climatology & allocate the related arrays +! + call get_tf_clm_dim(fin_tf_clm,nyc,nxc) + allocate( tf_clm0(nxc,nyc),tf_trd0(nxc,nyc),cxlats(nyc),cxlons(nxc) ) +! +! get tf_clm at the analysis time from monthly climatology & cxlats, cxlons +! + call get_tf_clm_ta(tf_clm0,tf_trd0,cxlats,cxlons,nyc,nxc,mon1,mon2,wei1,wei2) +! +! get tf_clm (nx by ny lat/lon) valid at atime +! + if ( nx == nxc .and. ny == nyc ) then + tf_clm(:,:) = tf_clm0(:,:) + tf_trd(:,:) = tf_trd0(:,:) +! write(*,'(a,2f9.3)') 'same dimensions, tf_clm, min : ',minval(tf_clm),maxval(tf_clm) + else +! write(*,'(a,4i8)') 'different dimensions,nx,ny,nxc,nyc : ',nx,ny,nxc,nyc + call intp_tile(tf_clm0, cxlats, cxlons, nyc, nxc, & + tf_clm_ij,xlats_ij,xlons_ij,ny, nx) + call intp_tile(tf_trd0, cxlats, cxlons, nyc, nxc, & + tf_trd_ij,xlats_ij,xlons_ij,ny, nx) +! write(*,'(a,2f9.3)') 'tf_clm0, min, max : ',minval(tf_clm0),maxval(tf_clm0) + + tf_clm(:,:) = reshape (tf_clm_ij, (/nx,ny/) ) + tf_trd(:,:) = reshape (tf_trd_ij, (/nx,ny/) ) + endif + +end subroutine get_tf_clm + +subroutine get_tf_clm_ta(tf_clm_ta,tf_clm_trend,xlats,xlons,nlat,nlon,mon1,mon2,wei1,wei2) +!$$$ +! abstract: get tf/sst climatology & the trend at analysis time +! created by xu li, march, 2019 + + use read_write_data, only : read_tf_clim_grb + implicit none + +! input + integer, intent(in) :: nlat,nlon,mon1,mon2 + real, intent(in) :: wei1,wei2 +! output + real, dimension(nlon,nlat), intent(out) :: tf_clm_ta,tf_clm_trend + real, dimension(nlat), intent(out) :: xlats + real, dimension(nlon), intent(out) :: xlons + +!input/output data file names + character (len=6), parameter :: fin_tf_clm='sstclm' + +! local declare + real, dimension(nlon,nlat) :: tf_clm1,tf_clm2 + +! +! read in rtg sst climatology without bitmap (surface mask) for mon1 and mon2 +! + call read_tf_clim_grb(trim(fin_tf_clm),tf_clm1,xlats,xlons,nlat,nlon,mon1) + call read_tf_clim_grb(trim(fin_tf_clm),tf_clm2,xlats,xlons,nlat,nlon,mon2) +! +! tf_clim at the analysis time +! + tf_clm_ta(:,:) = wei1*tf_clm1(:,:)+wei2*tf_clm2(:,:) +! +! tf_clim trend at the analysis time (month to month) +! + tf_clm_trend(:,:) = (tf_clm2(:,:)-tf_clm1(:,:))/120.0 + + write(*,'(a,2f9.3)') 'tf_clm_ta, min, max : ',minval(tf_clm_ta),maxval(tf_clm_ta) + write(*,'(a,2f9.3)') 'tf_clm_trend, min, max : ',minval(tf_clm_trend),maxval(tf_clm_trend) + end subroutine get_tf_clm_ta + +subroutine get_sal_clm(xlats_ij,xlons_ij,ny,nx,iy,im,id,ih,sal_clm) +! +! abstract: get salinity climatology at the valid time (atime) and target resolution (nx,ny) +! +! input & output +! + use read_write_data, only : get_dim_nc + implicit none + + real, dimension(nx*ny), intent(in) :: xlats_ij ! latitudes of target grids (nx*ny) + real, dimension(nx*ny), intent(in) :: xlons_ij ! latitudes of target grids (nx*ny) + real, dimension(nx,ny), intent(out) :: sal_clm ! salinity climatology valid at atime (nx,ny) + integer, intent(in) :: iy,im,id,ih,nx,ny +! local declare + real, allocatable, dimension(:,:) :: sal_clm0 ! salinity climatology at the valid time + real, allocatable, dimension(:) :: cxlats ! latitudes of sst climatology + real, allocatable, dimension(:) :: cxlons ! longitudes of sst climatology + + real, dimension(nx*ny) :: sal_clm_ij ! salinity climatology at target grids (nx*ny) + real :: wei1,wei2 + integer :: nxc,nyc,mon1,mon2,i,j + character (len=6), parameter :: fin_sal_clm='salclm' ! salinity climatology file name +! +! get which two months used and their weights from atime +! + call get_tim_wei(iy,im,id,ih,mon1,mon2,wei1,wei2) +! +! get the dimensions of the sst climatology & allocate the related arrays +! + call get_dim_nc(fin_sal_clm,nyc,nxc) + allocate( sal_clm0(nxc,nyc),cxlats(nyc),cxlons(nxc) ) +! +! get sal_clm at the analysis time from monthly climatology & cxlats, cxlons +! + call get_sal_clm_ta(sal_clm0,cxlats,cxlons,nyc,nxc,mon1,mon2,wei1,wei2) +! +! get sal_clm (nx by ny lat/lon) valid at atime +! + if ( nx == nxc .and. ny == nyc ) then + sal_clm(:,:) = sal_clm0(:,:) +! write(*,'(a,2f9.3)') 'same dimensions, sal_clm, min : ',minval(sal_clm),maxval(sal_clm) + else +! write(*,'(a,4i8)') 'different dimensions,nx,ny,nxc,nyc : ',nx,ny,nxc,nyc + call intp_tile(sal_clm0, cxlats, cxlons, nyc, nxc, & + sal_clm_ij,xlats_ij,xlons_ij,ny, nx) +! write(*,'(a,2f9.3)') 'sal_clm0, min, max : ',minval(sal_clm0),maxval(sal_clm0) +! write(*,'(a,2f9.3)') 'done with intp_tile for sal_clm, min, max : ',minval(sal_clm_ij),maxval(sal_clm_ij) + + sal_clm(:,:) = reshape (sal_clm_ij, (/nx,ny/) ) + endif + +end subroutine get_sal_clm + +subroutine get_sal_clm_ta(sal_clm_ta,xlats,xlons,nlat,nlon,mon1,mon2,wei1,wei2) +!$$$ +! abstract: get sal climatology at analysis time +! created by xu li, march, 2019 + + use read_write_data, only : read_salclm_gfs_nc + implicit none + +! input + integer, intent(in) :: nlat,nlon,mon1,mon2 + real, intent(in) :: wei1,wei2 +! output + real, dimension(nlon,nlat), intent(out) :: sal_clm_ta + real, dimension(nlat), intent(out) :: xlats + real, dimension(nlon), intent(out) :: xlons + +!input/output data file names + character (len=6), parameter :: fin_sal_clm='salclm' + +! local declare + real, dimension(nlon,nlat) :: sal_clm1,sal_clm2 + +! +! read in rtg sst climatology without bitmap (surface mask) for mon1 and mon2 +! + call read_salclm_gfs_nc(trim(fin_sal_clm),sal_clm1,xlats,xlons,nlat,nlon,mon1) + call read_salclm_gfs_nc(trim(fin_sal_clm),sal_clm2,xlats,xlons,nlat,nlon,mon2) +! +! sal_clim at the analysis time +! + sal_clm_ta(:,:) = wei1*sal_clm1(:,:)+wei2*sal_clm2(:,:) + write(*,'(a,2f9.3)') 'sal_clm_ta, min, max : ',minval(sal_clm_ta),maxval(sal_clm_ta) + end subroutine get_sal_clm_ta + +subroutine intp_tile(tf_lalo,dlats_lalo,dlons_lalo,jdim_lalo,idim_lalo, & + tf_tile,xlats_tile,xlons_tile,jdim_tile,idim_tile) +!-------------------------------------------------------------------------------- +! abstract: interpolate lon/lat grid to fv3 native grid (tf_lalo => tf_tile) for +! not dependent on mask +!-------------------------------------------------------------------------------- + +! input +! +! tf_lalo : (idim_lalo,idim_lalo) tf at lat/lon regular grid +! dlats_lalo : (jdim_lalo) latitudes along y direction of lat/lon regular grid points +! dlons_lalo : (idim_lalo) longitudes along x direction of lat/lon regular grid points +! jdim_lalo : number of y dimension of tf_lalo +! idim_lalo : number of x dimension of tf_lalo +! xlats_tile : (jdim_tile*idim_tile) latitudes of all tile grid points +! xlons_tile : (jdim_tile*idim_tile) longitudes of all tile grid points +! jdim_tile : number of y dimension of tf_tile +! idim_tile : number of x dimension of tf_tile +! +! output +! +! tf_tile : (jdim_tile*idim_tile) tf at cubed sphere grid + + implicit none + +! input/output + real, dimension(idim_lalo,jdim_lalo), intent(in) :: tf_lalo + real, dimension(jdim_lalo), intent(in) :: dlats_lalo + real, dimension(idim_lalo), intent(in) :: dlons_lalo + real, dimension(jdim_tile*idim_tile), intent(in) :: xlats_tile + real, dimension(jdim_tile*idim_tile), intent(in) :: xlons_tile + integer, intent(in) :: jdim_lalo,idim_lalo,jdim_tile,idim_tile + real, dimension(jdim_tile*idim_tile), intent(out) :: tf_tile + +! local + real, parameter :: deg2rad=3.1415926/180.0 + real, dimension(jdim_lalo) :: xlats_lalo + real, dimension(idim_lalo) :: xlons_lalo + real :: tf,wsum,res_km + integer :: itile,jtile + integer :: ii,jj,ij,iii,jjj + integer :: ilalo,jlalo,ilalop1,jlalop1 + integer :: istart,iend,jstart,jend,krad + + integer, allocatable, dimension(:,:) :: id1,id2,jdc + real, allocatable, dimension(:,:,:) :: agrid,s2c + + print* + print*,'interpolate from lat/lon grids to any one grid with known lat/lon' + + xlats_lalo = dlats_lalo*deg2rad + xlons_lalo = dlons_lalo*deg2rad + + allocate(agrid(idim_tile,jdim_tile,2)) + agrid(:,:,1) = reshape (xlons_tile, (/idim_tile,jdim_tile/) ) + agrid(:,:,2) = reshape (xlats_tile, (/idim_tile,jdim_tile/) ) + agrid = agrid*deg2rad + + allocate(id1(idim_tile,jdim_tile)) + allocate(id2(idim_tile,jdim_tile)) + allocate(jdc(idim_tile,jdim_tile)) + allocate(s2c(idim_tile,jdim_tile,4)) + +!---------------------------------------------------------------------- +! compute bilinear weights for each model point from the nearest +! four lalo points. does not account for mask. that +! happens later. +!---------------------------------------------------------------------- + + call remap_coef( 1, idim_tile, 1, jdim_tile, idim_lalo, jdim_lalo, & + xlons_lalo, xlats_lalo, id1, id2, jdc, s2c, agrid ) + + do ij = 1, jdim_tile*idim_tile + + jtile = (ij-1)/idim_tile + 1 + itile = mod(ij,idim_tile) + if (itile==0) itile = idim_tile + + ilalo = id1(itile,jtile) + ilalop1 = id2(itile,jtile) + jlalo = jdc(itile,jtile) + jlalop1 = jdc(itile,jtile) + 1 + + wsum = s2c(itile,jtile,1) + s2c(itile,jtile,2) + & + s2c(itile,jtile,3) + s2c(itile,jtile,4) + + tf_tile(ij) = ( s2c(itile,jtile,1)*tf_lalo(ilalo,jlalo) + & + s2c(itile,jtile,2)*tf_lalo(ilalop1,jlalo) + & + s2c(itile,jtile,3)*tf_lalo(ilalop1,jlalop1) + & + s2c(itile,jtile,4)*tf_lalo(ilalo,jlalop1) )/wsum + enddo + + deallocate(id1, id2, jdc, s2c) + +end subroutine intp_tile + +subroutine get_tim_wei(iy,im,id,ih,mon1,mon2,wei1,wei2) +!$$$ +! abstract: get two months and weights for monthly climatology +! created by xu li, march, 2019 + + implicit none + +! input + integer, intent(in) :: iy,im,id,ih +! output + integer, intent(out) :: mon1,mon2 + real, intent(out) :: wei1,wei2 + +! local declare + real :: rjday + integer :: mon,monend,monm,monp,jdow,jdoy,jday + integer :: jda(8) +! +!dayhf : julian day of the middle of each month +! + real, dimension(13) :: dayhf + data dayhf/15.5,45.0,74.5,105.0,135.5,166.0,196.5,227.5,258.0,288.5,319.0,349.5,380.5/ + +! 15, 44, 73.5, 104, 134.5, 165, 195.5, 226.5, 257, 287.5, 318.5, 349 ! from +! woa05 + + jda=0 + jda(1)=iy + jda(2)=im + jda(3)=id + jda(5)=ih + jdow = 0 + jdoy = 0 + jday = 0 + call w3doxdat(jda,jdow,jdoy,jday) + rjday=jdoy+jda(5)/24. + if(rjday.lt.dayhf(1)) rjday=rjday+365. + + monend = 12 + do mon = 1, monend + monm = mon + monp = mon + 1 + if( rjday >= dayhf(monm) .and. rjday < dayhf(monp) ) then + mon1 = monm + mon2 = monp + go to 10 + endif + enddo + + print *,'wrong rjday',rjday + call abort + 10 continue + + wei1 = (dayhf(mon2)-rjday)/(dayhf(mon2)-dayhf(mon1)) + wei2 = (rjday-dayhf(mon1))/(dayhf(mon2)-dayhf(mon1)) + + if( mon2 == 13 ) mon2=1 + + write(*,'(a,2i4,3f9.3)') 'mon1,mon2,rjday,wei1,wei2=',mon1,mon2,rjday,wei1,wei2 + + end subroutine get_tim_wei + + real function tfreez(salinity) +!Constants taken from Gill, 1982. +!Author: Robert Grumbine +!LAST MODIFIED: 21 September 1994. + + implicit none + + REAL salinity,sal + REAL a1, a2, a3 + PARAMETER (a1 = -0.0575) + PARAMETER (a2 = 1.710523E-3) + PARAMETER (a3 = -2.154996E-4) + + IF (salinity .LT. 0.) THEN + sal = 0. + ELSE + sal = salinity + ENDIF + tfreez = sal*(a1+a2*SQRT(sal)+a3*sal) + + return + end + diff --git a/sorc/global_cycle.fd/read_write_data.f90 b/sorc/global_cycle.fd/read_write_data.f90 index 02a2c7b34..4bbac032b 100644 --- a/sorc/global_cycle.fd/read_write_data.f90 +++ b/sorc/global_cycle.fd/read_write_data.f90 @@ -38,6 +38,8 @@ MODULE READ_WRITE_DATA PUBLIC :: READ_GSI_DATA PUBLIC :: READ_LAT_LON_OROG PUBLIC :: WRITE_DATA + public :: read_tf_clim_grb,get_tf_clm_dim + public :: read_salclm_gfs_nc,get_dim_nc CONTAINS @@ -1499,4 +1501,358 @@ SUBROUTINE READ_DATA(TSFFCS,SMCFCS,SNOFCS,STCFCS, & END SUBROUTINE READ_DATA +subroutine read_tf_clim_grb(file_sst,sst,rlats_sst,rlons_sst,mlat_sst,mlon_sst,mon) + +! . . . . +! abstrac: read_tf_clim_grb : read grib1 sst analysis +! prgmmr: xu li org: np23 date: 2019-03-13 +! +! abstract: read sst analysis (grib format) and save it as expanded and transposed array +! +! subroutine rdgrbsst must be compiled with the ncep w3 library +! and the bacio library. +! +! +! program history log: +! +! input argument list: +! file_sst - file name of grib sst file +! mon - month number +! mlat_sst,mlon_sst +! output: +! rlats_sst +! rlons_sst +! sst +! +! argument list defined by this reading: +! sst - sst field +! note: (1) the data is stored from north to south originally in grib format, +! but is stored from south to north with this reading routine +! nlat_sst - latitudinal dimension of sst +! nlon_sst - longitudinal dimension of sst +! xsst0 - latitude of origin +! ysst0 - longitude of origin +! dres - lat/lon increment +! +! call subs: getgbh, getgb +! +! attributes: +! language: f90 +! +!$$$ + implicit none + + include "mpif.h" + +! declare passed variables and arrays + character(*) , intent(in ) :: file_sst + integer , intent(in ) :: mon,mlat_sst,mlon_sst + real, dimension(mlat_sst) , intent( out) :: rlats_sst + real, dimension(mlon_sst) , intent( out) :: rlons_sst + real, dimension(mlon_sst,mlat_sst) , intent( out) :: sst + +! declare local parameters + integer,parameter:: lu_sst = 21 ! fortran unit number of grib sst file + real, parameter :: deg2rad = 3.141593/180.0 + +! declare local variables and arrays + logical(1), allocatable, dimension(:) :: lb + + integer :: nlat_sst,nlon_sst + integer :: iret,ni,nj + integer :: mscan,kb1 + integer :: jincdir,i,iincdir,kb2,kb3,kf,kg,k,j,jf + integer, dimension(22):: jgds,kgds + integer, dimension(25):: jpds,kpds + + real :: xsst0,ysst0,dres + real, allocatable, dimension(:) :: f + +!************+****************************************************************************** +! +! open sst analysis file (grib) + write(*,*) ' sstclm : ',file_sst + call baopenr(lu_sst,trim(file_sst),iret) + if (iret /= 0 ) then + write(6,*)'read_tf_clm_grb: ***error*** opening sst file' + CALL MPI_ABORT(MPI_COMM_WORLD, 111) + endif + +! define sst variables for read + j=-1 + jpds=-1 + jgds=-1 + jpds(5)=11 ! sst variable + jpds(6)=1 ! surface + jpds(9) = mon + call getgbh(lu_sst,0,j,jpds,jgds,kg,kf,k,kpds,kgds,iret) + + nlat_sst = kgds(3) ! number points on longitude circle (360) + nlon_sst = kgds(2) ! number points on latitude circle (720) + +! write(*,*) 'nlat_sst, nlon_sst, mon : ',nlat_sst, nlon_sst, mon +! write(*,*) 'mlat_sst, mlon_sst : ',mlat_sst, mlon_sst + +! allocate arrays + allocate(lb(nlat_sst*nlon_sst)) + allocate(f(nlat_sst*nlon_sst)) + jf=nlat_sst*nlon_sst + +! read in the analysis + call getgb(lu_sst,0,jf,j,jpds,jgds,kf,k,kpds,kgds,lb,f,iret) + if (iret /= 0) then + write(6,*)'read_tf_clm_grb: ***error*** reading sst analysis data record' + deallocate(lb,f) + CALL MPI_ABORT(MPI_COMM_WORLD, 111) + endif + + if ( (nlat_sst /= mlat_sst) .or. (nlon_sst /= mlon_sst) ) then + write(6,*)'read_rtg_org: inconsistent dimensions. mlat_sst,mlon_sst=',& + mlat_sst,mlon_sst,' -versus- nlat_sst,nlon_sst=',nlat_sst,nlon_sst + deallocate(lb,f) + CALL MPI_ABORT(MPI_COMM_WORLD, 111) + endif + +! +! get xlats and xlons +! + dres = 180.0/real(nlat_sst) + ysst0 = 0.5*dres-90.0 + xsst0 = 0.5*dres + +! get lat_sst & lon_sst + do j = 1, nlat_sst + rlats_sst(j) = ysst0 + real(j-1)*dres + enddo + + do i = 1, nlon_sst + rlons_sst(i) = (xsst0 + real(i-1)*dres) + enddo + +! load dimensions and grid specs. check for unusual values + ni=kgds(2) ! 720 for 0.5 x 0.5 + nj=kgds(3) ! 360 for 0.5 x 0.5 resolution + + mscan=kgds(11) + kb1=ibits(mscan,7,1) ! i scan direction + kb2=ibits(mscan,6,1) ! j scan direction + kb3=ibits(mscan,5,1) ! (i,j) or (j,i) + +! get i and j scanning directions from kb1 and kb2. +! 0 yields +1, 1 yields -1. +1 is west to east, -1 is east to west. + iincdir = 1-kb1*2 + +! 0 yields -1, 1 yields +1. +1 is south to north, -1 is north to south. + jincdir = kb2*2 - 1 + +! write(*,*) 'read_tf_clim_grb,iincdir,jincdir : ',iincdir,jincdir + do k=1,kf + +! kb3 from scan mode indicates if i points are consecutive +! or if j points are consecutive + if(kb3==0)then ! (i,j) + i=(ni+1)*kb1+(mod(k-1,ni)+1)*iincdir + j=(nj+1)*(1-kb2)+jincdir*((k-1)/ni+1) + else ! (j,i) + j=(nj+1)*(1-kb2)+(mod(k-1,nj)+1)*jincdir + i=(ni+1)*kb1+iincdir*((k-1)/nj+1) + endif + sst (i,j) = f(k) + end do + + deallocate(lb,f) + + call baclose(lu_sst,iret) + if (iret /= 0 ) then + write(6,*)'read_tf_clm_grb: ***error*** close sst file' + CALL MPI_ABORT(MPI_COMM_WORLD, 121) + endif + +end subroutine read_tf_clim_grb + +subroutine get_tf_clm_dim(file_sst,mlat_sst,mlon_sst) +! . . . . +! abstract: get_tf_clm_dim : get dimension of rtg sst climatology +! prgmmr: xu li org: np23 date: 2019-03-13 +! +! +! program history log: +! +! input argument list: +! file_sst - file name of grib sst file +! output +! mlat_sst,mlon_sst +! +! call subs: getgbh +! +! attributes: +! language: f90 +! machine: ibm rs/6000 sp +! +!$$$ + implicit none + + include "mpif.h" + +! declare passed variables and arrays + character(*) , intent(in ) :: file_sst + integer , intent(out) :: mlat_sst,mlon_sst + +! declare local parameters + integer,parameter:: lu_sst = 21 ! fortran unit number of grib sst file + + integer :: iret + integer :: mscan,kb1 + integer :: kf,kg,k,j + integer, dimension(22):: jgds,kgds + integer, dimension(25):: jpds,kpds + +!************+****************************************************************************** +! +! open sst analysis file (grib) + call baopenr(lu_sst,trim(file_sst),iret) + if (iret /= 0 ) then + write(6,*)'get_tf_clm_dim: ***error*** opening sst file' + CALL MPI_ABORT(MPI_COMM_WORLD, 111) + endif + +! define sst variables for read + j=-1 + jpds=-1 + jgds=-1 + jpds(5)=11 ! sst variable + jpds(6)=1 ! surface + jpds(9) = 1 + call getgbh(lu_sst,0,j,jpds,jgds,kg,kf,k,kpds,kgds,iret) + + mlat_sst = kgds(3) ! number points on longitude circle (360) + mlon_sst = kgds(2) ! number points on latitude circle (720) + + write(*,*) 'mlat_sst, mlon_sst : ',mlat_sst, mlon_sst + + call baclose(lu_sst,iret) + if (iret /= 0 ) then + write(6,*)'get_tf_clm_dim: ***error*** close sst file' + CALL MPI_ABORT(MPI_COMM_WORLD, 121) + endif +end subroutine get_tf_clm_dim + +subroutine read_salclm_gfs_nc(filename,sal,xlats,xlons,nlat,nlon,itime) +! abstract: read woa05 salinity monthly climatology (netcdf) + use netcdf + implicit none + + ! This is the name of the data file we will read. + character (len=*), intent(in) :: filename + integer, intent(in) :: nlat,nlon + integer, intent(in) :: itime + real, dimension(nlat), intent(out) :: xlats + real, dimension(nlon), intent(out) :: xlons + real, dimension(nlon,nlat), intent(out) :: sal +! Local variables + integer :: ncid,ntime + + integer, parameter :: ndims = 3 + character (len = *), parameter :: lat_name = "latitude" + character (len = *), parameter :: lon_name = "longitude" + character (len = *), parameter :: t_name = "time" + character (len = *), parameter :: sal_name="sal" + integer :: no_fill,fill_value + integer :: time_varid,lon_varid, lat_varid, z_varid, sal_varid + + ! The start and count arrays will tell the netCDF library where to read our data. + integer, dimension(ndims) :: start, count + + character (len = *), parameter :: units = "units" + character (len = *), parameter :: sal_units = "psu" + ! PSU (Practical SalinitUnit). 1 PSU = 1g/kg + character (len = *), parameter :: time_units = "months" + character (len = *), parameter :: lat_units = "degrees_north" + character (len = *), parameter :: lon_units = "degrees_east" + + integer :: missv +! Loop indices + integer :: i,j + +! Open the file. + call nc_check( nf90_open(filename, nf90_nowrite, ncid) ) + +! Get the varids of time, latitude, longitude & depth coordinate variables. + call nc_check( nf90_inq_varid(ncid, t_name, time_varid) ) + call nc_check( nf90_inq_varid(ncid, lat_name, lat_varid) ) + call nc_check( nf90_inq_varid(ncid, lon_name, lon_varid) ) + +! Read the time, latitude and longitude data. +! call nc_check( nf90_get_var(ncid, time_varid, ntime) ) + call nc_check( nf90_get_var(ncid, lat_varid, xlats) ) + call nc_check( nf90_get_var(ncid, lon_varid, xlons) ) + +! Get the varids of the sal netCDF variables. + call nc_check( nf90_inq_varid(ncid, sal_name,sal_varid) ) + +! Read 1 record of nlat*nlon values, starting at the beginning +! of the record (the (1, 1, 1, rec) element in the netCDF file). + start = (/ 1, 1, itime /) + count = (/ nlon, nlat, 1 /) + +! write(*,*) 'read_salclm_gfs_nc itime : ',itime +! Read the sal data from the file, one record at a time. + call nc_check( nf90_get_var(ncid, sal_varid, sal, start, count) ) + +! Close the file. This frees up any internal netCDF resources +! associated with the file. + call nc_check( nf90_close(ncid) ) + +! If we got this far, everything worked as expected. Yipee! +! print *,"*** SUCCESS reading file ", filename, "!" + +end subroutine read_salclm_gfs_nc + +subroutine get_dim_nc(filename,nlat,nlon) +! abstract: get dimensions of sal array + use netcdf + implicit none + + character (len=*), intent(in) :: filename + integer, intent(out) :: nlat,nlon +! Local variables + character (len = *), parameter :: lat_name = "latitude" + character (len = *), parameter :: lon_name = "longitude" + integer :: ncid + integer :: LatDimID,LonDimID + +! Open the file. + call nc_check( nf90_open(filename, nf90_nowrite, ncid) ) + +! Get dimensions + call nc_check( nf90_inq_dimid(ncid,lat_name,LatDimID) ) + call nc_check( nf90_inq_dimid(ncid,lon_name,LonDimID) ) + call nc_check( nf90_inquire_dimension(ncid,LatDimID,len=nlat) ) + call nc_check( nf90_inquire_dimension(ncid,LonDimID,len=nlon) ) + +! write(*,'(a,1x,a6,2I8)') 'get_dim_nc, file, nlat, nlon : ',filename,nlat,nlon + +! Close the file. This frees up any internal netCDF resources +! associated with the file. + call nc_check( nf90_close(ncid) ) + +! If we got this far, everything worked as expected. Yipee! +! print *,"*** SUCCESS get dimensions from nc file ", filename, "!" + +end subroutine get_dim_nc + +subroutine nc_check(status) + + use netcdf + + include "mpif.h" + + integer, intent ( in) :: status + + if(status /= nf90_noerr) then + print *, trim(nf90_strerror(status)) + CALL MPI_ABORT(MPI_COMM_WORLD, 122) + end if +end subroutine nc_check + END MODULE READ_WRITE_DATA diff --git a/sorc/global_cycle.fd/sfcsub.F b/sorc/global_cycle.fd/sfcsub.F index b14271cd0..4fbabab8f 100644 --- a/sorc/global_cycle.fd/sfcsub.F +++ b/sorc/global_cycle.fd/sfcsub.F @@ -2697,12 +2697,13 @@ subroutine fixrdg(lugb,idim,jdim,fngrib, real (kind=kind_io8), allocatable :: data8(:) real (kind=kind_io4), allocatable :: data4(:) ! - logical*1 lbms(mdata) + logical*1, allocatable :: lbms(:) ! integer kpds(200),kgds(200) integer jpds(200),jgds(200), kpds0(200) ! allocate(data8(1:idim*jdim)) + allocate(lbms(1:mdata)) kpds = 0 kgds = 0 jpds = 0 @@ -2792,6 +2793,7 @@ subroutine fixrdg(lugb,idim,jdim,fngrib, endif ! deallocate(data8) + deallocate(lbms) return end subroutine getarea(kgds,dlat,dlon,rslat,rnlat,wlon,elon,ijordr @@ -6078,17 +6080,24 @@ subroutine setrmsk(kpds5,slmask,igaul,jgaul,wlon,rnlat, ! ijmax = imax*jmax rslmsk = 0. +! TG3 MODS BEGIN + if(kpds5 == kpdtsf .and. imax == 138 .and. jmax == 116 + & .and. kpds4 == 128) then +! print*,'turn off setrmsk for tg3' + lmask = .false. + + elseif(kpds5 == kpdtsf) then +! TG3 MODS END ! ! surface temperature ! - if(kpds5.eq.kpdtsf) then -! lmask=.false. + lmask = .false. call ga2la(slmask,igaul,jgaul,rslmsk,imax,jmax,wlon,rnlat &, rlnout, rltout, gaus, blno, blto) ! &, dlon, dlat, gaus, blno, blto) - crit=0.5 + crit = 0.5 call rof01(rslmsk,ijmax,'ge',crit) - lmask=.true. + lmask = .true. ! ! bucket soil wetness ! @@ -6096,16 +6105,16 @@ subroutine setrmsk(kpds5,slmask,igaul,jgaul,wlon,rnlat, call ga2la(slmask,igaul,jgaul,rslmsk,imax,jmax,wlon,rnlat &, rlnout, rltout, gaus, blno, blto) ! &, dlon, dlat, gaus, blno, blto) - crit=0.5 + crit = 0.5 call rof01(rslmsk,ijmax,'ge',crit) - lmask=.true. + lmask = .true. ! write(6,*) 'wet rslmsk' ! znnt=1. ! call nntprt(rslmsk,ijmax,znnt) ! ! snow depth ! - elseif(kpds5.eq.kpdsnd) then + elseif(kpds5 == kpdsnd) then if(kpds4 == 192) then ! use the bitmap rslmsk = 0. do j = 1, jmax @@ -6962,51 +6971,51 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, ! ! get tsf climatology for the begining of the forecast ! - if (fh .gt. 0.0) then + if (fh > 0.0) then !cbosu if (me == 0) print*,'bosu fh gt 0' - iy4=iy - if(iy.lt.101) iy4=1900+iy4 - fha=0 - ida=0 - jda=0 -! fha(2)=nint(fh) - ida(1)=iy - ida(2)=im - ida(3)=id - ida(5)=ih + iy4 = iy + if (iy < 101) iy4 = 1900 + iy4 + fha = 0 + ida = 0 + jda = 0 +! fha(2) = nint(fh) + ida(1) = iy + ida(2) = im + ida(3) = id + ida(5) = ih call w3kind(w3kindreal,w3kindint) if(w3kindreal == 4) then - fha4=fha + fha4 = fha call w3movdat(fha4,ida,jda) else call w3movdat(fha,ida,jda) endif - jy=jda(1) - jm=jda(2) - jd=jda(3) - jh=jda(5) - if (me .eq. 0) write(6,*) ' forecast jy,jm,jd,jh', - & jy,jm,jd,jh + jy = jda(1) + jm = jda(2) + jd = jda(3) + jh = jda(5) + if (me == 0) write(6,*) ' forecast jy,jm,jd,jh', + & jy,jm,jd,jh jdow = 0 jdoy = 0 jday = 0 call w3doxdat(jda,jdow,jdoy,jday) - rjday=jdoy+jda(5)/24. - if(rjday.lt.dayhf(1)) rjday=rjday+365. + rjday = jdoy + jda(5) / 24. + if(rjday < dayhf(1)) rjday = rjday + 365. ! - if (me .eq. 0) write(6,*) 'forecast jy,jm,jd,jh=',jy,jm,jd,jh + if (me == 0) write(6,*) 'forecast jy,jm,jd,jh=',jy,jm,jd,jh ! ! for monthly mean climatology ! monend = 12 do mm=1,monend - mmm=mm - mmp=mm+1 - if(rjday.ge.dayhf(mmm).and.rjday.lt.dayhf(mmp)) then - mon1=mmm - mon2=mmp + mmm = mm + mmp = mm + 1 + if(rjday >= dayhf(mmm) .and. rjday < dayhf(mmp)) then + mon1 = mmm + mon2 = mmp go to 10 endif enddo @@ -7014,17 +7023,18 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, call abort 10 continue wei1m = (dayhf(mon2)-rjday)/(dayhf(mon2)-dayhf(mon1)) - wei2m = (rjday-dayhf(mon1))/(dayhf(mon2)-dayhf(mon1)) - if(mon2.eq.13) mon2=1 - if (me .eq. 0) print *,'rjday,mon1,mon2,wei1m,wei2m=', - & rjday,mon1,mon2,wei1m,wei2m + wei2m = 1.0 - wei1m +! wei2m = (rjday-dayhf(mon1))/(dayhf(mon2)-dayhf(mon1)) + if (mon2 == 13) mon2 = 1 + if (me == 0) print *,'rjday,mon1,mon2,wei1m,wei2m=', + & rjday,mon1,mon2,wei1m,wei2m ! ! read monthly mean climatology of tsf ! kpd7 = -1 do nn=1,2 mon = mon1 - if (nn .eq. 2) mon = mon2 + if (nn == 2) mon = mon2 call fixrdc(lugb,fntsfc,kpdtsf,kpd7,mon,slmask, & tsf(1,nn),len,iret &, imsk, jmsk, slmskh, gaus,blno, blto @@ -7041,8 +7051,8 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, ! ! compute current jy,jm,jd,jh of forecast and the day of the year ! - iy4=iy - if(iy.lt.101) iy4=1900+iy4 + iy4 = iy + if (iy < 101) iy4=1900+iy4 fha = 0 ida = 0 jda = 0 @@ -7052,8 +7062,8 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, ida(3) = id ida(5) = ih call w3kind(w3kindreal,w3kindint) - if(w3kindreal==4) then - fha4=fha + if(w3kindreal == 4) then + fha4 = fha call w3movdat(fha4,ida,jda) else call w3movdat(fha,ida,jda) @@ -7068,44 +7078,45 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, jdoy = 0 jday = 0 call w3doxdat(jda,jdow,jdoy,jday) - rjday = jdoy+jda(5)/24. - if(rjday.lt.dayhf(1)) rjday=rjday+365. + rjday = jdoy + jda(5) / 24. + if(rjday < dayhf(1)) rjday = rjday + 365. - if (me .eq. 0) write(6,*) ' forecast jy,jm,jd,jh,rjday=', - & jy,jm,jd,jh,rjday + if (me == 0) write(6,*) ' forecast jy,jm,jd,jh,rjday=', + & jy,jm,jd,jh,rjday ! - if (me .eq. 0) write(6,*) 'forecast jy,jm,jd,jh=',jy,jm,jd,jh + if (me == 0) write(6,*) 'forecast jy,jm,jd,jh=',jy,jm,jd,jh ! ! for monthly mean climatology ! monend = 12 do mm=1,monend - mmm=mm - mmp=mm+1 - if(rjday.ge.dayhf(mmm).and.rjday.lt.dayhf(mmp)) then - mon1=mmm - mon2=mmp + mmm = mm + mmp = mm + 1 + if(rjday >= dayhf(mmm) .and. rjday < dayhf(mmp)) then + mon1 = mmm + mon2 = mmp go to 20 endif enddo print *,'wrong rjday',rjday call abort 20 continue - wei1m=(dayhf(mon2)-rjday)/(dayhf(mon2)-dayhf(mon1)) - wei2m=(rjday-dayhf(mon1))/(dayhf(mon2)-dayhf(mon1)) - if(mon2.eq.13) mon2=1 - if (me .eq. 0) print *,'rjday,mon1,mon2,wei1m,wei2m=', - & rjday,mon1,mon2,wei1m,wei2m + wei1m = (dayhf(mon2)-rjday)/(dayhf(mon2)-dayhf(mon1)) + wei2m = 1.0 - wei1m +! wei2m = (rjday-dayhf(mon1))/(dayhf(mon2)-dayhf(mon1)) + if (mon2 == 13) mon2 = 1 + if (me == 0) print *,'rjday,mon1,mon2,wei1m,wei2m=', + & rjday,mon1,mon2,wei1m,wei2m ! ! for seasonal mean climatology ! monend = 4 is = im/3 + 1 - if (is.eq.5) is = 1 + if (is == 5) is = 1 do mm=1,monend mmm = mm*3 - 2 mmp = (mm+1)*3 - 2 - if(rjday.ge.dayhf(mmm).and.rjday.lt.dayhf(mmp)) then + if(rjday >= dayhf(mmm) .and. rjday < dayhf(mmp)) then sea1 = mmm sea2 = mmp go to 30 @@ -7115,20 +7126,21 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, call abort 30 continue wei1s = (dayhf(sea2)-rjday)/(dayhf(sea2)-dayhf(sea1)) - wei2s = (rjday-dayhf(sea1))/(dayhf(sea2)-dayhf(sea1)) - if(sea2.eq.13) sea2=1 - if (me .eq. 0) print *,'rjday,sea1,sea2,wei1s,wei2s=', - & rjday,sea1,sea2,wei1s,wei2s + wei2s = 1.0 - wei1s +! wei2s = (rjday-dayhf(sea1))/(dayhf(sea2)-dayhf(sea1)) + if (sea2 == 13) sea2 = 1 + if (me == 0) print *,'rjday,sea1,sea2,wei1s,wei2s=', + & rjday,sea1,sea2,wei1s,wei2s ! ! for summer and winter values (maximum and minimum). ! monend = 2 is = im/6 + 1 - if (is.eq.3) is = 1 + if (is == 3) is = 1 do mm=1,monend mmm = mm*6 - 5 mmp = (mm+1)*6 - 5 - if(rjday.ge.dayhf(mmm).and.rjday.lt.dayhf(mmp)) then + if(rjday >= dayhf(mmm) .and. rjday < dayhf(mmp)) then hyr1 = mmm hyr2 = mmp go to 31 @@ -7138,10 +7150,11 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, call abort 31 continue wei1y = (dayhf(hyr2)-rjday)/(dayhf(hyr2)-dayhf(hyr1)) - wei2y = (rjday-dayhf(hyr1))/(dayhf(hyr2)-dayhf(hyr1)) - if(hyr2.eq.13) hyr2=1 - if (me .eq. 0) print *,'rjday,hyr1,hyr2,wei1y,wei2y=', - & rjday,hyr1,hyr2,wei1y,wei2y + wei2y = 1.0 - wei1y +! wei2y = (rjday-dayhf(hyr1))/(dayhf(hyr2)-dayhf(hyr1)) + if (hyr2 == 13) hyr2 = 1 + if (me == 0) print *,'rjday,hyr1,hyr2,wei1y,wei2y=', + & rjday,hyr1,hyr2,wei1y,wei2y ! ! start reading in climatology and interpolate to the date ! @@ -7541,7 +7554,7 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, ! mon1s = mon1 ; mon2s = mon2 ; sea1s = sea1 ; sea2s = sea2 ! - if (me .eq. 0) print *,' mon1s=',mon1s,' mon2s=',mon2s + if (me == 0) print *,' mon1s=',mon1s,' mon2s=',mon2s &,' sea1s=',sea1s,' sea2s=',sea2s ! k1 = 1 ; k2 = 2 @@ -7599,11 +7612,11 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, ! seasonal mean climatology ! isx = sea2/3 + 1 - if (isx .eq. 5) isx = 1 - if(isx.eq.1) kpd9 = 12 - if(isx.eq.2) kpd9 = 3 - if(isx.eq.3) kpd9 = 6 - if(isx.eq.4) kpd9 = 9 + if (isx == 5) isx = 1 + if (isx == 1) kpd9 = 12 + if (isx == 2) kpd9 = 3 + if (isx == 3) kpd9 = 6 + if (isx == 4) kpd9 = 9 ! ! albedo ! there are four albedo fields in this version: @@ -7639,7 +7652,7 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, if (me == 0) print*,'bosu 2nd time in clima for month ', & mon, k1,k2 if ( index(fnalbc, "tileX.nc") == 0) then ! grib file - kpd7=-1 + kpd7 = -1 do k = 1, 4 call fixrdc(lugb,fnalbc,kpdalb(k),kpd7,mon,slmask, & alb(1,k,nn),len,iret @@ -7656,7 +7669,7 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, ! ! tsf at the current time t ! - kpd7=-1 + kpd7 = -1 call fixrdc(lugb,fntsfc,kpdtsf,kpd7,mon,slmask, & tsf(1,nn),len,iret &, imsk, jmsk, slmskh, gaus,blno, blto @@ -7664,13 +7677,13 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, ! ! soil wetness ! - if(fnwetc(1:8).ne.' ') then + if (fnwetc(1:8).ne.' ') then kpd7=-1 call fixrdc(lugb,fnwetc,kpdwet,kpd7,mon,slmask, & wet(1,nn),len,iret &, imsk, jmsk, slmskh, gaus,blno, blto &, outlat, outlon, me) - elseif(fnsmcc(1:8).ne.' ') then + elseif (fnsmcc(1:8).ne.' ') then if (index(fnsmcc,'global_soilmcpc.1x1.grb') /= 0) then ! the old climo data kpd7=-1 call fixrdc(lugb,fnsmcc,kpdsmc,kpd7,mon,slmask, @@ -7712,13 +7725,13 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, ! ! sea ice ! - kpd7=-1 - if(fnacnc(1:8).ne.' ') then + kpd7 = -1 + if (fnacnc(1:8).ne.' ') then call fixrdc(lugb,fnacnc,kpdacn,kpd7,mon,slmask, & acn(1,nn),len,iret &, imsk, jmsk, slmskh, gaus,blno, blto &, outlat, outlon, me) - elseif(fnaisc(1:8).ne.' ') then + elseif (fnaisc(1:8).ne.' ') then call fixrdc(lugb,fnaisc,kpdais,kpd7,mon,slmask, & ais(1,nn),len,iret &, imsk, jmsk, slmskh, gaus,blno, blto @@ -7738,7 +7751,7 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, ! ! snow cover ! - if(fnscvc(1:8).ne.' ') then + if (fnscvc(1:8).ne.' ') then kpd7=-1 call fixrdc(lugb,fnscvc,kpdscv,kpd7,mon,slmask, & scv(1,nn),len,iret @@ -7749,7 +7762,7 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, ! ! surface roughness ! - if(fnzorc(1:3) == 'sib') then + if (fnzorc(1:3) == 'sib') then if (me == 0) then write(6,*) 'roughness length to be set from sib veg type' endif @@ -7767,7 +7780,7 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, ! ! vegetation cover ! - if(fnvegc(1:8).ne.' ') then + if (fnvegc(1:8) .ne. ' ') then if ( index(fnvegc, "tileX.nc") == 0) then ! grib file kpd7=-1 call fixrdc(lugb,fnvegc,kpdveg,kpd7,mon,slmask, @@ -7789,35 +7802,35 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, ! when chosen, set the z0 based on the vegetation type. ! for this option to work, namelist variable fnvetc must be ! set to point at the proper vegetation type file. - if(fnzorc(1:3) == 'sib') then - if(fnvetc(1:4) == ' ') then + if (fnzorc(1:3) == 'sib') then + if (fnvetc(1:4) == ' ') then if (me==0) write(6,*) "must choose sib veg type climo file" call abort endif zorclm = 0.0 do i=1,len - ivtyp=nint(vet(i)) + ivtyp = nint(vet(i)) if (ivtyp >= 1 .and. ivtyp <= 13) then zorclm(i) = z0_sib(ivtyp) endif enddo elseif(fnzorc(1:4) == 'igbp') then - if(fnvetc(1:4) == ' ') then - if (me==0) write(6,*) "must choose igbp veg type climo file" + if (fnvetc(1:4) == ' ') then + if (me == 0) write(6,*) "must choose igbp veg type climo file" call abort endif zorclm = 0.0 do i=1,len - ivtyp=nint(vet(i)) + ivtyp = nint(vet(i)) if (ivtyp >= 1 .and. ivtyp <= 20) then z0_season(1) = z0_igbp_min(ivtyp) z0_season(7) = z0_igbp_max(ivtyp) - if(outlat(i) < 0.0)then + if (outlat(i) < 0.0) then zorclm(i) = wei1y * z0_season(hyr2) + - & wei2y *z0_season(hyr1) + & wei2y * z0_season(hyr1) else zorclm(i) = wei1y * z0_season(hyr1) + - & wei2y *z0_season(hyr2) + & wei2y * z0_season(hyr2) endif endif enddo @@ -8148,7 +8161,7 @@ subroutine fixrdc(lugb,fngrib,kpds5,kpds7,mon,slmask, real (kind=kind_io8), allocatable :: rlngrb(:), rltgrb(:) ! logical lmask, yr2kc, gaus, ijordr - logical*1 lbms(mdata) + logical*1, allocatable :: lbms(:) ! integer, intent(in) :: kpds7 integer kpds(1000),kgds(1000) @@ -8156,6 +8169,7 @@ subroutine fixrdc(lugb,fngrib,kpds5,kpds7,mon,slmask, real (kind=kind_io8) outlat(len), outlon(len) ! allocate(data8(1:mdata)) + allocate(lbms(mdata)) ! ! integer imax_sv, jmax_sv, wlon_sv, rnlat_sv, kpds1_sv ! date imax_sv/0/, jmax_sv/0/, wlon_sv/999.0/, rnlat_sv/999.0/ @@ -8291,6 +8305,7 @@ subroutine fixrdc(lugb,fngrib,kpds5,kpds7,mon,slmask, call baclose(lugb,iret) ! deallocate(data8) + deallocate(lbms) return end subroutine fixrdc subroutine fixrda(lugb,fngrib,kpds5,slmask, diff --git a/sorc/link_fixdirs.sh b/sorc/link_fixdirs.sh index d2b4fdc8c..d5fbc1cde 100755 --- a/sorc/link_fixdirs.sh +++ b/sorc/link_fixdirs.sh @@ -9,20 +9,20 @@ machine=${2} if [ $# -lt 2 ]; then set +x echo '***ERROR*** must specify two arguements: (1) RUN_ENVIR, (2) machine' - echo ' Syntax: link_fv3gfs.sh ( nco | emc ) ( cray | dell | theia | jet )' + echo ' Syntax: link_fv3gfs.sh ( nco | emc ) ( cray | dell | hera | jet )' exit 1 fi if [ $RUN_ENVIR != emc -a $RUN_ENVIR != nco ]; then set +x echo '***ERROR*** unsupported run environment' - echo 'Syntax: link_fv3gfs.sh ( nco | emc ) ( cray | dell | theia | jet )' + echo 'Syntax: link_fv3gfs.sh ( nco | emc ) ( cray | dell | hera | jet )' exit 1 fi -if [ $machine != cray -a $machine != theia -a $machine != dell -a $machine != jet ]; then +if [ $machine != cray -a $machine != hera -a $machine != dell -a $machine != jet ]; then set +x echo '***ERROR*** unsupported machine' - echo 'Syntax: link_fv3gfs.sh ( nco | emc ) ( cray | dell | theia | jet )' + echo 'Syntax: link_fv3gfs.sh ( nco | emc ) ( cray | dell | hera | jet )' exit 1 fi @@ -39,23 +39,15 @@ if [ $machine == "cray" ]; then FIX_DIR="/gpfs/hps3/emc/global/noscrub/emc.glopara/git/fv3gfs/fix" elif [ $machine = "dell" ]; then FIX_DIR="/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/fv3gfs/fix" -elif [ $machine = "theia" ]; then - FIX_DIR="/scratch4/NCEPDEV/global/save/glopara/git/fv3gfs/fix" +elif [ $machine = "hera" ]; then + FIX_DIR="/scratch1/NCEPDEV/global/glopara/fix" elif [ $machine = "jet" ]; then FIX_DIR="/lfs3/projects/hfv3gfs/glopara/git/fv3gfs/fix" fi cd ${pwd}/../fix ||exit 8 -for dir in fix_am fix_fv3 fix_orog fix_fv3_gmted2010 ; do +for dir in fix_am fix_fv3 fix_orog fix_fv3_gmted2010 fix_sfc_climo; do [[ -d $dir ]] && rm -rf $dir $LINK $FIX_DIR/$dir . done -if [ $machine == "cray" ] || [ $machine = "dell" ]; then - $LINK /gpfs/dell2/emc/modeling/noscrub/George.Gayno/landutil.git/climo_fields_netcdf ./fix_sfc_climo -elif [ $machine = "theia" ]; then - $LINK /scratch4/NCEPDEV/da/noscrub/George.Gayno/climo_fields_netcdf ./fix_sfc_climo -elif [ $machine = "jet" ]; then - $LINK /mnt/lfs3/projects/emcda/George.Gayno/climo_fields_netcdf ./fix_sfc_climo -fi - exit 0 diff --git a/sorc/machine-setup.sh b/sorc/machine-setup.sh index 252f9d955..9bc1be6da 100644 --- a/sorc/machine-setup.sh +++ b/sorc/machine-setup.sh @@ -28,16 +28,15 @@ if [[ -d /lfs3 ]] ; then target=jet module purge module use /mnt/lfs3/projects/hfv3gfs/nwprod/lib/modulefiles -elif [[ -d /scratch3 ]] ; then - # We are on NOAA Theia +elif [[ -d /scratch1 ]] ; then + # We are on NOAA Hera if ( ! eval module help > /dev/null 2>&1 ) ; then echo load the module command 1>&2 source /apps/lmod/lmod/init/$__ms_shell fi - target=theia + target=hera module purge - module use /scratch3/NCEPDEV/nwprod/modulefiles/ - module use /scratch3/NCEPDEV/nwprod/lib/modulefiles + MOD_PATH=/scratch2/NCEPDEV/nwprod/NCEPLIBS/modulefiles elif [[ -d /gpfs/hps && -e /etc/SuSE-release ]] ; then # We are on NOAA Luna or Surge if ( ! eval module help > /dev/null 2>&1 ) ; then diff --git a/sorc/mkgfsnemsioctl.fd/makefile b/sorc/mkgfsnemsioctl.fd/makefile new file mode 100755 index 000000000..351cda92d --- /dev/null +++ b/sorc/mkgfsnemsioctl.fd/makefile @@ -0,0 +1,10 @@ +SHELL =/bin/ksh +EXEC =../../exec/mkgfsnemsioctl +FOPTS = -O -FR -I$(NEMSIO_INC) +LOPTS = $(OPTS) +LIBS = $(NEMSIO_LIB) $(BACIO_LIB4) $(W3NCO_LIBd) +OBJS = mkgfsnemsioctl.o +SRCS = mkgfsnemsioctl.f +# ************************************************************************* +all: $(SRCS) + $(FCMP) $(FOPTS) $(LOPTS) ${SRCS} -o $(EXEC) $(LIBS) diff --git a/sorc/mkgfsnemsioctl.fd/mkgfsnemsioctl.f b/sorc/mkgfsnemsioctl.fd/mkgfsnemsioctl.f new file mode 100644 index 000000000..cdc9efc90 --- /dev/null +++ b/sorc/mkgfsnemsioctl.fd/mkgfsnemsioctl.f @@ -0,0 +1,535 @@ +!- - - - -- - -- - -- - -- - - -- - -- -- - -- - -- - - -- - - - -- - -- +! the program create NEMS GFS sfc ctl file +! +! Mar 12, 2015 Jun Wang +! - - - - -- - -- - -- - -- - - -- - -- -- - -- - -- - - -- - - - -- - -- +! + program main + use nemsio_module + implicit none +! +!--------------------------------------------------------------------------- + type(nemsio_gfile) :: gfile + character(255) cin,cindx + real,allocatable :: data(:,:) +!--------------------------------------------------------------------------- +!--- nemsio meta data + integer nrec,im,jm,lm,idate(7),nfhour,tlmeta,nsoil,fieldsize + real(4),allocatable :: lat1(:),lat(:),lon1(:) + real(8),allocatable :: slat(:) + character(16),allocatable:: recname(:),reclevtyp(:) + integer,allocatable:: reclev(:) +!--------------------------------------------------------------------------- +!--- local vars + character(3) cmon + character(16) reclevtyp_sht + character(32) ctldate,varname + character(35) sweep_blanks + real(8) radi + real lon_stt,lat_stt,lon_intl,lat_intl + integer i,n,j,krec,iret,io_unit,idrt,nmeta +!--------------------------------------------------------------------------- +!--------------------------------------------------------------------------- +! + call nemsio_init(iret=iret) + if(iret/=0) print *,'ERROR: nemsio_init ' +! +!--------------------------------------------------------------------------- +!*** read nemsio grd header info +!--------------------------------------------------------------------------- +!--- open gfile for reading + call getarg(1,cin) + print *,'filename is cin=',trim(cin) + if(trim(cin)=='') then + print *,'usage: mknemsioctl input_nemsio_file_name' + stop + endif + call getarg(2,cindx) + if(trim(cindx)=='') then + cindx=trim(cin)//'.ctl' + endif + + call nemsio_open(gfile,trim(cin),'READ',iret=iret) + if(iret/=0) print *,'Error: open nemsio file,',trim(cin),' iret=',iret + + call nemsio_getfilehead(gfile,iret=iret,nrec=nrec,dimx=im,dimy=jm, & + dimz=lm,idate=idate,nfhour=nfhour,nmeta=nmeta, & + nsoil=nsoil,idrt=idrt,tlmeta=tlmeta) +! + fieldsize=im*jm + allocate(recname(nrec),reclevtyp(nrec),reclev(nrec)) + allocate(lat1(fieldsize),lat(jm),lon1(fieldsize)) + call nemsio_getfilehead(gfile,iret=iret,recname=recname, & + reclevtyp=reclevtyp,reclev=reclev,lat=lat1,lon=lon1) +! + call nemsio_close(gfile,iret=iret) +! + call nemsio_finalize() +! +!--------------------------------------------------------------------------- +!****** write .ctl file +!--------------------------------------------------------------------------- +! +!-- get date + call cmonth(idate(2),cmon) + write(ctldate,'(i2.2,a,i2.2,a3,i4.4)')idate(4),'Z',idate(3) & + ,cmon,idate(1) +!-- get Gaussian grid + if(idrt==4) then +!-- data has lat/lon info + if (nmeta>=8) then + lon_stt=lon1(1) + lon_intl=lon1(2)-lon1(1) + do j=1,jm + lat(j) = lat1(1+(j-1)*im) + enddo + else + call splat8(idrt,jm,slat) + radi=180.0/(4.*atan(1.0)) + do j=1,jm + lat(j) = asin(slat(j)) * radi + enddo + lon_stt=0. + lon_intl=360./real(jm) + endif + elseif(idrt==0) then + if(nmeta>=8) then + lon_stt=lon1(1) + lon_intl=lon1(2)-lon1(1) + lat_stt=-abs(lat1(1)) + lat_intl=abs(lat1(1+im)-lat1(1)) +!! print*, "im=",im,' jm=',jm +!! print*, "lon1=",lon1 +!! print*, "lat_intl=",lat_intl +!! print*, "lat1=",lat1 + else + if (mod(jm,2) == 0) then + lat_intl=180.0/real(jm) + lat_stt=-90.0+0.5*lat_intl + else + lat_intl=180.0/real(jm-1) + lat_stt=-90.0 + endif + lon_intl=360./real(im) + lon_stt=0.5*lon_intl + endif + endif + + io_unit=650 + open(io_unit,file=trim(cindx),form='formatted') +! + write(io_unit,105)trim(cin) + write(io_unit,106) + + if (idrt == 4 )then + write(io_unit,107) + elseif(idrt==0) then + if(nmeta>=8) then + if (lat1(1)>lat1(im+1)) then + write(io_unit,107) + else + write(io_unit,1077) + endif + else + write(io_unit,1077) + endif + endif + + write(io_unit,108)tlmeta + write(io_unit,109) + write(io_unit,111)im,lon_stt,lon_intl + if (idrt==4) then + write(io_unit,112)jm + write(io_unit,113)(lat(i),i=jm,1,-1) + else + write(io_unit,120)jm,lat_stt,lat_intl + endif + write(io_unit,114)lm + if(nfhour/=0) then + write(io_unit,115)1,trim(ctldate),nfhour + else + write(io_unit,116)1,trim(ctldate) + endif + +! + 105 FORMAT('dset ^',A) + 106 FORMAT('undef 9.99E+20') + 107 FORMAT('options big_endian sequential yrev') + 1077 FORMAT('options big_endian sequential') + 108 FORMAT('fileheader',I12.0) + 109 FORMAT('title gfs nemsioi file') + + 111 FORMAT('xdef ',I6,' linear ',f9.6,' ',f9.6) + 112 FORMAT('ydef ',I6,' levels') + 113 FORMAT(10f11.6) + + 114 FORMAT('zdef ',I6,' linear 1 1') + 115 FORMAT('tdef ',I6,' linear ',A12,' ',I6,'hr') + 116 FORMAT('tdef ',I6,' linear ',A12,' 1yr') +! + 120 FORMAT('ydef ',I6,' linear ',f13.6,' ',f9.6) +! + krec=0 + do n=1,nrec + if(reclev(n)==1) then + krec=krec+1 + endif + enddo + + WRITE(IO_UNIT,'(A,I6)')'VARS ',krec + + n=1 + do while (n<=nrec) + reclevtyp_sht=reclevtyp(n) + if(trim(reclevtyp_sht) == "convect-cld bot") then + reclevtyp_sht="cvb" + elseif (trim(reclevtyp_sht) == "convect-cld top") then + reclevtyp_sht="cvt" + elseif (trim(reclevtyp_sht) == "high cld bot") then + reclevtyp_sht="hcb" + elseif (trim(reclevtyp_sht) == "high cld top") then + reclevtyp_sht="hct" + elseif (trim(reclevtyp_sht) == "mid cld bot") then + reclevtyp_sht="mcb" + elseif (trim(reclevtyp_sht) == "mid cld top") then + reclevtyp_sht="mct" + elseif (trim(reclevtyp_sht) == "low cld bot") then + reclevtyp_sht="lcb" + elseif (trim(reclevtyp_sht) == "low cld top") then + reclevtyp_sht="lct" + elseif (trim(reclevtyp_sht) == "convect-cld laye") then + reclevtyp_sht="cvcl" + elseif (trim(reclevtyp_sht) == "bndary-layer cld") then + reclevtyp_sht="bdrlc" + elseif (trim(reclevtyp_sht) == "atmos col") then + reclevtyp_sht="acol" + elseif (trim(reclevtyp_sht) == "high cld lay") then + reclevtyp_sht="hcl" + elseif (trim(reclevtyp_sht) == "mid cld lay") then + reclevtyp_sht="mcl" + elseif (trim(reclevtyp_sht) == "low cld lay") then + reclevtyp_sht="lcl" + elseif (trim(reclevtyp_sht) == "2 m above gnd") then + reclevtyp_sht="2m" + elseif (trim(reclevtyp_sht) == "10 m above gnd") then + reclevtyp_sht="10m" + endif + varname=sweep_blanks(trim(recname(n))//trim(reclevtyp_sht)) + if(trim(reclevtyp(n))=='mid layer') then + write(io_unit,'(a16,i3,a)')varname,lm,' 99 model layer' + n=n+lm + elseif(trim(reclevtyp(n))=='soil layer') then + write(io_unit,'(a16,i3,a)')varname,nsoil,' 99 soil layer' + n=n+nsoil + elseif(trim(reclevtyp(n))=='2 m above gnd') then + recname(n)=trim(recname(n))//'2m' + write(io_unit,'(a16,a7,a)')varname,' 0 99 ',trim(reclevtyp(n)) + n=n+1 + elseif(trim(reclevtyp(n))=='10 m above gnd') then + recname(n)=trim(recname(n))//'10m' + write(io_unit,'(a16,a7,a)')varname,' 0 99 ',trim(reclevtyp(n)) + n=n+1 + else + write(io_unit,'(a16,a7,a)')varname,' 0 99 ',trim(reclevtyp(n)) + n=n+1 + endif + enddo + + write(io_unit,'(A8)')'endvars' + close(io_unit) + +!--------------------------------------------------------------------------- +!****** clean up +!--------------------------------------------------------------------------- + deallocate(recname,reclevtyp,reclev,lat,lat1,lon1) +!--------------------------------------------------------------------------- +! +! - - - - -- - -- - -- - -- - - -- - -- -- - -- - -- - - -- - - - -- - -- + stop + + end program +! - - - - -- - -- - -- - -- - - -- - -- -- - -- - -- - - -- - - - -- - -- + SUBROUTINE CMONTH(IMON,CMON) +! +!----------------------------------------------------------------------- +!*** Convert month +!----------------------------------------------------------------------- +! + INTEGER,INTENT(IN) :: IMON + CHARACTER(LEN=3) :: CMON +! +!----------------------------------------------------------------------- +! + SELECT CASE (IMON) + CASE(1) + CMON='Jan' + CASE(2) + CMON='Feb' + CASE(3) + CMON='Mar' + CASE(4) + CMON='Apr' + CASE(5) + CMON='May' + CASE(6) + CMON='Jun' + CASE(7) + CMON='Jul' + CASE(8) + CMON='Aug' + CASE(9) + CMON='Sep' + CASE(10) + CMON='Oct' + CASE(11) + CMON='Nov' + CASE(12) + CMON='Dec' + END SELECT +! +!----------------------------------------------------------------------- +! + END SUBROUTINE CMONTH + +! + sUBROUTINE splat4(IDRT,JMAX,ASLAT) +! + implicit none + integer,intent(in) :: idrt,jmax + real(4),intent(out) :: ASLAT(JMAX) + INTEGER,PARAMETER:: KD=SELECTED_REAL_KIND(15,45) + REAL(KIND=KD):: PK(JMAX/2),PKM1(JMAX/2),PKM2(JMAX/2) + REAL(KIND=KD):: ASLATD(JMAX/2),SP,SPMAX,EPS=10.d0*EPSILON(SP) + integer,PARAMETER:: JZ=50 + REAL(8) BZ(JZ) + DATA BZ / 2.4048255577d0, 5.5200781103d0, & + 8.6537279129d0, 11.7915344391d0, 14.9309177086d0, 18.0710639679d0, & + 21.2116366299d0, 24.3524715308d0, 27.4934791320d0, 30.6346064684d0, & + 33.7758202136d0, 36.9170983537d0, 40.0584257646d0, 43.1997917132d0, & + 46.3411883717d0, 49.4826098974d0, 52.6240518411d0, 55.7655107550d0, & + 58.9069839261d0, 62.0484691902d0, 65.1899648002d0, 68.3314693299d0, & + 71.4729816036d0, 74.6145006437d0, 77.7560256304d0, 80.8975558711d0, & + 84.0390907769d0, 87.1806298436d0, 90.3221726372d0, 93.4637187819d0, & + 96.6052679510d0, 99.7468198587d0, 102.888374254d0, 106.029930916d0, & + 109.171489649d0, 112.313050280d0, 115.454612653d0, 118.596176630d0, & + 121.737742088d0, 124.879308913d0, 128.020877005d0, 131.162446275d0, & + 134.304016638d0, 137.445588020d0, 140.587160352d0, 143.728733573d0, & + 146.870307625d0, 150.011882457d0, 153.153458019d0, 156.295034268d0 / + REAL(8):: DLT,D1=1.d0 + INTEGER(4):: JHE,JHO,J0=0 + real(8),PARAMETER :: PI=3.14159265358979d0,C=(1.d0-(2.d0/PI)**2)*0.25d0 + real(8) r + integer jh,js,n,j +!C - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +!C GAUSSIAN LATITUDES + IF(IDRT.EQ.4) THEN + JH=JMAX/2 + JHE=(JMAX+1)/2 + R=1.d0/SQRT((JMAX+0.5d0)**2+C) + DO J=1,MIN(JH,JZ) + ASLATD(J)=COS(BZ(J)*R) + ENDDO + DO J=JZ+1,JH + ASLATD(J)=COS((BZ(JZ)+(J-JZ)*PI)*R) + ENDDO + SPMAX=1.d0 + DO WHILE(SPMAX.GT.EPS) + SPMAX=0.d0 + DO J=1,JH + PKM1(J)=1.d0 + PK(J)=ASLATD(J) + ENDDO + DO N=2,JMAX + DO J=1,JH + PKM2(J)=PKM1(J) + PKM1(J)=PK(J) + PK(J)=((2*N-1)*ASLATD(J)*PKM1(J)-(N-1)*PKM2(J))/N + ENDDO + ENDDO + DO J=1,JH + SP=PK(J)*(1.d0-ASLATD(J)**2)/(JMAX*(PKM1(J)-ASLATD(J)*PK(J))) + ASLATD(J)=ASLATD(J)-SP + SPMAX=MAX(SPMAX,ABS(SP)) + ENDDO + ENDDO +!CDIR$ IVDEP + DO J=1,JH + ASLAT(J)=ASLATD(J) + ASLAT(JMAX+1-J)=-ASLAT(J) + ENDDO + IF(JHE.GT.JH) THEN + ASLAT(JHE)=0.d0 + ENDIF +!C - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +!C EQUALLY-SPACED LATITUDES INCLUDING POLES + ELSEIF(IDRT.EQ.0) THEN + JH=JMAX/2 + JHE=(JMAX+1)/2 + JHO=JHE-1 + DLT=PI/(JMAX-1) + ASLAT(1)=1.d0 + DO J=2,JH + ASLAT(J)=COS((J-1)*DLT) + ENDDO +!CDIR$ IVDEP + DO J=1,JH + ASLAT(JMAX+1-J)=-ASLAT(J) + ENDDO + IF(JHE.GT.JH) THEN + ASLAT(JHE)=0.d0 + ENDIF +!C - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +!C EQUALLY-SPACED LATITUDES EXCLUDING POLES + ELSEIF(IDRT.EQ.256) THEN + JH=JMAX/2 + JHE=(JMAX+1)/2 + JHO=JHE + DLT=PI/JMAX + ASLAT(1)=1.d0 + DO J=1,JH + ASLAT(J)=COS((J-0.5)*DLT) + ENDDO +!CDIR$ IVDEP + DO J=1,JH + ASLAT(JMAX+1-J)=-ASLAT(J) + ENDDO + IF(JHE.GT.JH) THEN + ASLAT(JHE)=0.d0 + ENDIF + ENDIF +!C - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + end subroutine splat4 +! +!---------------------------------------------------------------------- + SUBROUTINE splat8(IDRT,JMAX,ASLAT) +!$$$ + implicit none + integer,intent(in) :: idrt,jmax + real(8),intent(out) :: ASLAT(JMAX) + INTEGER,PARAMETER:: KD=SELECTED_REAL_KIND(15,45) + REAL(KIND=KD):: PK(JMAX/2),PKM1(JMAX/2),PKM2(JMAX/2) + REAL(KIND=KD):: ASLATD(JMAX/2),SP,SPMAX,EPS=10.d0*EPSILON(SP) + integer,PARAMETER:: JZ=50 + REAL(8) BZ(JZ) + DATA BZ / 2.4048255577d0, 5.5200781103d0, & + 8.6537279129d0, 11.7915344391d0, 14.9309177086d0, 18.0710639679d0, & + 21.2116366299d0, 24.3524715308d0, 27.4934791320d0, 30.6346064684d0, & + 33.7758202136d0, 36.9170983537d0, 40.0584257646d0, 43.1997917132d0, & + 46.3411883717d0, 49.4826098974d0, 52.6240518411d0, 55.7655107550d0, & + 58.9069839261d0, 62.0484691902d0, 65.1899648002d0, 68.3314693299d0, & + 71.4729816036d0, 74.6145006437d0, 77.7560256304d0, 80.8975558711d0, & + 84.0390907769d0, 87.1806298436d0, 90.3221726372d0, 93.4637187819d0, & + 96.6052679510d0, 99.7468198587d0, 102.888374254d0, 106.029930916d0, & + 109.171489649d0, 112.313050280d0, 115.454612653d0, 118.596176630d0, & + 121.737742088d0, 124.879308913d0, 128.020877005d0, 131.162446275d0, & + 134.304016638d0, 137.445588020d0, 140.587160352d0, 143.728733573d0, & + 146.870307625d0, 150.011882457d0, 153.153458019d0, 156.295034268d0 / + REAL(8):: DLT,D1=1.d0 + INTEGER(4):: JHE,JHO,J0=0 + real(8),PARAMETER :: PI=3.14159265358979d0,C=(1.d0-(2.d0/PI)**2)*0.25d0 + real(8) r + integer jh,js,n,j +!C - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +!C GAUSSIAN LATITUDES + IF(IDRT.EQ.4) THEN + JH=JMAX/2 + JHE=(JMAX+1)/2 + R=1.d0/SQRT((JMAX+0.5d0)**2+C) + DO J=1,MIN(JH,JZ) + ASLATD(J)=COS(BZ(J)*R) + ENDDO + DO J=JZ+1,JH + ASLATD(J)=COS((BZ(JZ)+(J-JZ)*PI)*R) + ENDDO + SPMAX=1.d0 + DO WHILE(SPMAX.GT.EPS) + SPMAX=0.d0 + DO J=1,JH + PKM1(J)=1.d0 + PK(J)=ASLATD(J) + ENDDO + DO N=2,JMAX + DO J=1,JH + PKM2(J)=PKM1(J) + PKM1(J)=PK(J) + PK(J)=((2*N-1)*ASLATD(J)*PKM1(J)-(N-1)*PKM2(J))/N + ENDDO + ENDDO + DO J=1,JH + SP=PK(J)*(1.d0-ASLATD(J)**2)/(JMAX*(PKM1(J)-ASLATD(J)*PK(J))) + ASLATD(J)=ASLATD(J)-SP + SPMAX=MAX(SPMAX,ABS(SP)) + ENDDO + ENDDO +!CDIR$ IVDEP + DO J=1,JH + ASLAT(J)=ASLATD(J) + ASLAT(JMAX+1-J)=-ASLAT(J) + ENDDO + IF(JHE.GT.JH) THEN + ASLAT(JHE)=0.d0 + ENDIF +!C - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +!C EQUALLY-SPACED LATITUDES INCLUDING POLES + ELSEIF(IDRT.EQ.0) THEN + JH=JMAX/2 + JHE=(JMAX+1)/2 + JHO=JHE-1 + DLT=PI/(JMAX-1) + ASLAT(1)=1.d0 + DO J=2,JH + ASLAT(J)=COS((J-1)*DLT) + ENDDO +!CDIR$ IVDEP + DO J=1,JH + ASLAT(JMAX+1-J)=-ASLAT(J) + ENDDO + IF(JHE.GT.JH) THEN + ASLAT(JHE)=0.d0 + ENDIF +!C - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +!C EQUALLY-SPACED LATITUDES EXCLUDING POLES + ELSEIF(IDRT.EQ.256) THEN + JH=JMAX/2 + JHE=(JMAX+1)/2 + JHO=JHE + DLT=PI/JMAX + ASLAT(1)=1.d0 + DO J=1,JH + ASLAT(J)=COS((J-0.5d0)*DLT) + ENDDO +!DIR$ IVDEP + DO J=1,JH + ASLAT(JMAX+1-J)=-ASLAT(J) + ENDDO + IF(JHE.GT.JH) THEN + ASLAT(JHE)=0.d0 + ENDIF + ENDIF +! - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + end subroutine splat8 +!----------------------------------------------------------------------- + + character(35) function sweep_blanks(in_str) + +! + implicit none +! + character(*), intent(in) :: in_str + character(35) :: out_str + character :: ch + integer :: j + + out_str = " " + do j=1, len_trim(in_str) + ! get j-th char + ch = in_str(j:j) + if (ch .eq. "-") then + out_str = trim(out_str) // "_" + else if (ch .ne. " ") then + out_str = trim(out_str) // ch + endif + sweep_blanks = out_str + end do + end function sweep_blanks + diff --git a/sorc/nemsio_cvt.fd/makefile b/sorc/nemsio_cvt.fd/makefile deleted file mode 100755 index 1fb2143db..000000000 --- a/sorc/nemsio_cvt.fd/makefile +++ /dev/null @@ -1,12 +0,0 @@ -SHELL =/bin/sh -EXEC =../../exec/nemsio_cvt -FOPTS = -O -FR -I$(NEMSIO_INC) -LOPTS = $(OPTS) -LIBS = $(NEMSIO_LIB) $(BACIO_LIB4) $(W3NCO_LIBd) -OBJS = nemsio_cvt -SRCS = nemsio_cvt.f -# ************************************************************************* - all: $(OBJS) - - $(OBJS): $(SRCS) - $(FCMP) $(FOPTS) $(LOPTS) $(SRCS) -o $(EXEC) $(LIBS) diff --git a/sorc/nemsio_cvt.fd/nemsio_cvt.f b/sorc/nemsio_cvt.fd/nemsio_cvt.f deleted file mode 100644 index c3c1ad116..000000000 --- a/sorc/nemsio_cvt.fd/nemsio_cvt.f +++ /dev/null @@ -1,300 +0,0 @@ -!- - - - -- - -- - -- - -- - - -- - -- -- - -- - -- - - -- - - - -- - -- -! the program convert a nemsio file to the opposite endian -! - - - - -- - -- - -- - -- - - -- - -- -- - -- - -- - - -- - - - -- - -! -- - program main -! - use nemsio_module - implicit none -! - type(nemsio_gfile) :: gfile,gfilew -! - character(255) cin,cout - character(8) gdatatype,lcnv - logical datareal4 - real,allocatable :: tmp(:) - real(8),allocatable :: tmp8(:) - character(16) file_endian,file_endianw -!--------------------------------------------------------------------------- -!--- nemsio meta data - integer i,nrec,fieldsize,iret -!--------------------------------------------------------------------------- -! -!--- initialize - call nemsio_init(iret=iret) -! -!--- open gfile for reading - call getarg(1,cin) - call getarg(2,lcnv) - if(cin=='') then - print *,'Usage: nemsio_cvt $FILE_NAME ' - stop - endif - if(lcnv=='')lcnv='YES' -! print *,'cin=',trim(cin),' lcnv=',trim(lcnv) -! - call nemsio_open(gfile,trim(cin),'read',iret=iret) - if(iret/=0) print *,'3b:: after open read, ',trim(cin), ' iret=',iret -! -!to set file header gfilew for write - call set_gfile(gfile,lcnv,cin,cout,gfilew,gdatatype,fieldsize,nrec, & - file_endian,file_endianw) -! -!if just check file endian, not convert - if(lcnv == 'NO') then - stop - endif -! - if(gdatatype(1:4)=="bin4".or.gdatatype(1:4)=="grib") then - allocate(tmp(fieldsize)) - datareal4=.true. - else if(gdatatype(1:4)=="bin8") then - allocate(tmp8(fieldsize)) - datareal4=.false. - endif -! - do i=1,nrec - if(datareal4) then - call nemsio_readrec(gfile,i,data=tmp,iret=iret) - call nemsio_writerec(gfilew,i,data=tmp,iret=iret) - else - call nemsio_readrec(gfile,i,data=tmp8,iret=iret) - call nemsio_writerec(gfilew,i,data=tmp8,iret=iret) - endif - - enddo -! -!--- close nemsio file - call nemsio_close(gfile,iret=iret) - if ( iret .ne.0) print *,'open input file,iret=',iret - call nemsio_close(gfilew,iret=iret) - if ( iret .ne.0) print *,'open output file,iret=',iret - -!!--------------------------------------------------------------------------- - if(allocated(tmp))deallocate(tmp) - if(allocated(tmp8))deallocate(tmp8) -! -!--------------------------------------------------------------------------- -! - call nemsio_finalize() -! - print *,'File ',trim(cin),' in ',trim(file_endian), & - ' is converted to ', trim(cout),' in ',trim(file_endianw) -! -! - - - - -- - -- - -- - -- - - -- - -- -- - -- - -- - - -- - - - -- - -! -- - stop - - end program - -!############################################################################ -! - subroutine set_gfile(gfile,lcnv,cin,cout,gfilew,gdatatype,fieldsize, & - nrec,file_endian,file_endianw) -! - use nemsio_module - implicit none -! - type(nemsio_gfile),intent(in) :: gfile - character(*),intent(in) :: lcnv,cin - character(*),intent(out) :: cout - type(nemsio_gfile),intent(inout) :: gfilew - character(8),intent(out) :: gdatatype - integer,intent(out) :: fieldsize,nrec - character(*),intent(out) :: file_endian,file_endianw -! -! local vars - integer im,jm,lm,ntrac,nframe,nmetavari,nmetavarr,nmeta, & - nmetavarl,nmetavarc,nmetavarr8,nmetaaryi,nmetaaryr, & - nmetaaryl,nmetaaryc,nmetaaryr8,iret - integer,allocatable :: reclev(:),varival(:),aryilen(:),aryrlen(:), & - aryllen(:),aryclen(:),aryr8len(:),aryival(:,:) - real,allocatable :: lat(:),lon(:),dx(:),dy(:),cpi(:),ri(:), & - vcoord(:,:,:),varrval(:),aryrval(:,:) - real(8),allocatable :: varr8val(:),aryr8val(:,:) - logical extrameta - logical,allocatable :: varlval(:),arylval(:,:) - character(16),allocatable :: recname(:),reclevtyp(:),variname(:), & - varrname(:),varlname(:),varcname(:),varr8name(:), & - aryiname(:),aryrname(:),arylname(:),arycname(:), & - aryr8name(:),varcval(:),arycval(:,:) -! - gfilew=gfile -! - call nemsio_getfilehead(gfile,dimx=im,dimy=jm,dimz=lm,nrec=nrec, & - nmeta=nmeta,gdatatype=gdatatype, & - nframe=nframe,ntrac=ntrac,file_endian=file_endian, & - extrameta=extrameta,nmetavari=nmetavari,nmetavarr=nmetavarr,& - nmetavarl=nmetavarl,nmetavarc=nmetavarc,nmetavarr8=nmetavarr8,& - nmetaaryi=nmetaaryi,nmetaaryr=nmetaaryr,nmetaaryl=nmetaaryl, & - nmetaaryc=nmetaaryc,nmetaaryr8=nmetaaryr8) -! -! print *,'in set_gfile,im=',im,'jm=',jm,'lm=',lm,'nrec=',nrec, & -! 'nmeta=',nmeta,'gdatatype=',gdatatype,'file_endian=',file_endian, & -! 'ntrac=',ntrac,'extrameta=',extrameta - -! - if(trim(file_endian)=='big_endian') then - file_endianw='little_endian' - gdatatype=trim(gdatatype(1:4))//"_le" - cout=trim(cin)//"_le" - elseif(trim(file_endian)=='little_endian') then - file_endianw='big_endian' - gdatatype=trim(gdatatype(1:4))//"_be" - cout=trim(cin)//"_be" - else - print *,'wrong endian in input file: ',trim(file_endian) - endif - if(lcnv == 'NO') then - print *,'File ',trim(cin),' is in ',trim(file_endian) - return - endif -! -!set array - fieldsize=(im+2*nframe)*(jm+2*nframe) - allocate(recname(nrec),reclevtyp(nrec),reclev(nrec)) - allocate(lon(fieldsize),lat(fieldsize),dx(fieldsize),dy(fieldsize)) - allocate(cpi(ntrac+1),ri(ntrac+1)) - allocate(vcoord(lm+1,3,2)) -! - if(nmeta>=5) then - call nemsio_getfilehead(gfile,iret=iret, recname=recname, & - reclevtyp=reclevtyp, reclev=reclev) - if(iret/=0) print *,'nemsio_cnv: cant get record infor!' - endif -! - if(nmeta>=6) then - call nemsio_getfilehead(gfile,iret=iret, vcoord=vcoord) - if(iret/=0) print *,'nemsio_cnv: cant get vcoord!' - endif -! - if(nmeta>=8) then - call nemsio_getfilehead(gfile,iret=iret, lon=lon,lat=lat) - if(iret/=0) print *,'nemsio_cnv: cant get lat/lon!' - endif -! - if(nmeta>=10) then - call nemsio_getfilehead(gfile,iret=iret, dx=dx,dy=dy) - if(iret/=0) print *,'nemsio_cnv: cant get dx/dy!' - endif -! - if(nmeta>=12) then - call nemsio_getfilehead(gfile,iret=iret, cpi=cpi,ri=ri) - if(iret/=0) print *,'nemsio_cnv: cant get cpi/ri!' - endif -! -! get user defined header vars - if(extrameta) then -! - if(nmetavari>0) then - allocate(variname(nmetavari),varival(nmetavari)) - call nemsio_getfilehead(gfile,iret=iret,variname=variname, & - varival=varival) - endif -! - if(nmetavarr>0) then - allocate(varrname(nmetavarr),varrval(nmetavarr)) - call nemsio_getfilehead(gfile,iret=iret,varrname=varrname, & - varrval=varrval) - endif -! - if(nmetavarl>0) then - allocate(varlname(nmetavarl),varlval(nmetavarl)) - call nemsio_getfilehead(gfile,iret=iret,varlname=varlname, & - varlval=varlval) - endif -! - if(nmetavarc>0) then - allocate(varcname(nmetavarc),varcval(nmetavarc)) - call nemsio_getfilehead(gfile,iret=iret,varcname=varcname, & - varcval=varcval) - endif -! - if(nmetavarr8>0) then - allocate(varr8name(nmetavarr8),varr8val(nmetavarr8)) - call nemsio_getfilehead(gfile,iret=iret,varr8name=varr8name, & - varr8val=varr8val) - endif -! - if(nmetaaryi>0) then - allocate(aryiname(nmetaaryi),aryilen(nmetaaryi)) - call nemsio_getfilehead(gfile,iret=iret,aryiname=aryiname, & - aryilen=aryilen) - allocate(aryival(maxval(aryilen),nmetaaryi)) - call nemsio_getfilehead(gfile,iret=iret,aryival=aryival) - endif -! - if(nmetaaryr>0) then - allocate(aryrname(nmetaaryr),aryrlen(nmetaaryr)) - call nemsio_getfilehead(gfile,iret=iret,aryrname=aryrname, & - aryrlen=aryrlen) - allocate(aryrval(maxval(aryrlen),nmetaaryr)) - call nemsio_getfilehead(gfile,iret=iret,aryrval=aryrval) - endif -! - if(nmetaaryl>0) then - allocate(arylname(nmetaaryl),aryllen(nmetaaryl)) - call nemsio_getfilehead(gfile,iret=iret,arylname=arylname, & - aryllen=aryllen) - allocate(arylval(maxval(aryllen),nmetaaryl)) - call nemsio_getfilehead(gfile,iret=iret,arylval=arylval) - endif -! - if(nmetaaryc>0) then - allocate(arycname(nmetaaryc),aryclen(nmetaaryc)) - call nemsio_getfilehead(gfile,iret=iret,arycname=arycname, & - aryclen=aryclen) - allocate(arycval(maxval(aryclen),nmetaaryc)) - call nemsio_getfilehead(gfile,iret=iret,arycval=arycval) - endif -! - if(nmetaaryr8>0) then - allocate(aryr8name(nmetaaryr8),aryr8len(nmetaaryr8)) - call nemsio_getfilehead(gfile,iret=iret,aryr8name=aryr8name, & - aryr8len=aryr8len) - allocate(aryr8val(maxval(aryr8len),nmetaaryr8)) - call nemsio_getfilehead(gfile,iret=iret,aryr8val=aryr8val) - endif -! - endif -! - call nemsio_open(gfilew,trim(cout),'write',iret=iret, & - gdatatype=gdatatype, & - recname=recname,reclevtyp=reclevtyp,reclev=reclev, & - lat=lat,lon=lon,cpi=cpi,ri=ri,extrameta=extrameta, & - nmetavari=nmetavari,nmetavarr=nmetavarr,nmetavarl=nmetavarl,& - nmetavarc=nmetavarc,nmetavarr8=nmetavarr8, & - nmetaaryi=nmetaaryi,nmetaaryr=nmetaaryr,nmetaaryl=nmetaaryl,& - nmetaaryc=nmetaaryc,nmetaaryr8=nmetaaryr8, & - variname=variname,varival=varival, & - varrname=varrname,varrval=varrval, & - varlname=varlname,varlval=varlval, & - varcname=varcname,varcval=varcval, & - varr8name=varr8name,varr8val=varr8val, & - aryiname=aryiname,aryival=aryival, & - aryrname=aryrname,aryrval=aryrval, & - arylname=arylname,arylval=arylval, & - arycname=arycname,arycval=arycval, & - aryr8name=aryiname,aryr8val=aryr8val) - if(iret.ne.0) then - print *,'ERROR: after open, file=',trim(cout),'fiel_endian=', & - trim(file_endian) - endif -! - if(allocated(recname)) deallocate(recname,reclevtyp,reclev) - if(allocated(vcoord)) deallocate(vcoord) - if(allocated(lon)) deallocate(lon,lat) - if(allocated(dy)) deallocate(dx,dy) - if(allocated(ri)) deallocate(cpi,ri) - if(allocated(variname))deallocate(variname,varival) - if(allocated(varrname))deallocate(varrname,varrval) - if(allocated(varlname))deallocate(varlname,varlval) - if(allocated(varcname))deallocate(varcname,varcval) - if(allocated(varr8name))deallocate(varr8name,varr8val) - if(allocated(aryiname))deallocate(aryiname,aryilen,aryival) - if(allocated(aryrname))deallocate(aryrname,aryrlen,aryrval) - if(allocated(arylname))deallocate(arylname,aryllen,arylval) - if(allocated(arycname))deallocate(arycname,aryclen,arycval) - if(allocated(aryr8name))deallocate(aryr8name,aryr8len,aryr8val) -! - end subroutine set_gfile diff --git a/sorc/sfc_climo_gen.fd/interp.F90 b/sorc/sfc_climo_gen.fd/interp.F90 index 880f2e7d4..a61450e80 100644 --- a/sorc/sfc_climo_gen.fd/interp.F90 +++ b/sorc/sfc_climo_gen.fd/interp.F90 @@ -50,6 +50,7 @@ subroutine interp(localpet, method, input_file) type(esmf_regridmethod_flag),intent(in) :: method type(esmf_field) :: data_field_src type(esmf_routehandle) :: regrid_data + type(esmf_polemethod_flag) :: pole print*,"- CALL FieldCreate FOR SOURCE GRID DATA." data_field_src = ESMF_FieldCreate(grid_src, & @@ -121,13 +122,19 @@ subroutine interp(localpet, method, input_file) if (record == 1) then + if (method == ESMF_REGRIDMETHOD_BILINEAR) then + pole = ESMF_POLEMETHOD_ALLAVG + else + pole = ESMF_POLEMETHOD_NONE + endif + print*,"- CALL FieldRegridStore." nullify(unmapped_ptr) call ESMF_FieldRegridStore(data_field_src, & data_field_mdl, & srcmaskvalues=(/0/), & dstmaskvalues=(/0/), & - polemethod=ESMF_POLEMETHOD_NONE, & + polemethod=pole, & unmappedaction=ESMF_UNMAPPEDACTION_IGNORE, & normtype=ESMF_NORMTYPE_FRACAREA, & srctermprocessing=isrctermprocessing, & diff --git a/ush/chgres_cube.sh b/ush/chgres_cube.sh new file mode 100755 index 000000000..26ab7fc19 --- /dev/null +++ b/ush/chgres_cube.sh @@ -0,0 +1,222 @@ +#!/bin/bash + +#---------------------------------------------------------------------------- +# Script name: chgres_cube.sh +# +# Abstract: Run the chgres program to initialize an FV3 run. +# +# See comments for variable definitions and setup information. +#---------------------------------------------------------------------------- + +set -x + +#---------------------------------------------------------------------------- +# Resolution of target grid. +#---------------------------------------------------------------------------- + +CRES=${CRES:-96} + +#---------------------------------------------------------------------------- +# Set up environment paths. +# +# HOMEufs - Location of root ufs_utils directory. +# EXECufs - Location of ufs_utils executable directory. +# FIXufs - Location of ufs_utils root fixed data directory. +# FIXfv3 - Location of target grid orography and 'grid' files. +# FIXsfc - Location of target grid surface climatological files. +# FIXam - Location of vertical coordinate definition file for target grid. +#---------------------------------------------------------------------------- + +ufs_ver=${ufs_ver:-v1.0.0} +envir=${envir:-prod} +NWROOT=${NWROOT:-/nw${envir}} +HOMEufs=${HOMEufs:-${NWROOT}/ufs_util.${ufs_ver}} +EXECufs=${EXECufs:-$HOMEufs/exec} +FIXufs=${FIXufs:-$HOMEufs/fix} +FIXfv3=${FIXfv3:-$FIXufs/fix_fv3_gmted2010/C${CRES}} +FIXsfc=${FIXsfc:-$FIXfv3/fix_sfc} +FIXam=${FIXam:-$FIXufs/fix_am} + +#---------------------------------------------------------------------------- +# CDATE - YYMMDDHH of your run. +#---------------------------------------------------------------------------- + +CDATE=${CDATE:?} +iy=$(echo $CDATE|cut -c1-4) +im=$(echo $CDATE|cut -c5-6) +id=$(echo $CDATE|cut -c7-8) +ih=$(echo $CDATE|cut -c9-10) + +#---------------------------------------------------------------------------- +# Variables for stand-alone regional grids. +# +# REGIONAL - Set to 1 to create remove halo and create lateral boundary +# file. Set to 2 for lateral boundary file only. Set to +# 0 for non-regional grids. +# HALO_BNDY - Number of rows/cols for lateral boundaries. +# HALO_BLEND - Number of rows/cols for blending zone. +#---------------------------------------------------------------------------- + +REGIONAL=${REGIONAL:-0} +HALO_BNDY=${HALO_BNDY:-0} +HALO_BLEND=${HALO_BLEND:-0} + +#---------------------------------------------------------------------------- +# INPUT_TYPE - Input data type. 'history' for tiled fv3 history files. +# 'restart' for tiled fv3 warm restart files. 'gfs_gaussian' +# for spectral gfs nemsio files. 'gfs_spectral' for +# for spectral gfs sigio/sfcio files. 'gaussian' for fv3 +# gaussian nemsio files. +# +# MOSAIC_FILE_INPUT_GRID - Path/Name of mosaic file for input grid. Only +# used for 'history' and 'restart' INPUT_TYPE. +# Set to NULL otherwise. +# +# OROG_DIR_INPUT_GRID - Location of orography and grid files for input grid. +# Only used for 'history' and 'restart' INPUT_TYPE. +# Set to NULL otherwise. +# +# OROG_FILES_INPUT_GRID - List of orography files for input grid. Only +# used for 'history' and 'restart' INPUT_TYPE. +# Set to NULL otherwise. +#---------------------------------------------------------------------------- + +INPUT_TYPE=${INPUT_TYPE:-"gaussian"} +MOSAIC_FILE_INPUT_GRID=${MOSAIC_FILE_INPUT_GRID:-NULL} +OROG_DIR_INPUT_GRID=${OROG_DIR_INPUT_GRID:-NULL} +OROG_FILES_INPUT_GRID=${OROG_FILES_INPUT_GRID:-NULL} + +#---------------------------------------------------------------------------- +# COMIN - Location of input data +# CONVERT_ATM - Convert atmospheric fields when true +# CONVERT_SFC - Convert surface fields when true +# CONVERT_NST - Convert nst fields when true +#---------------------------------------------------------------------------- + +CONVERT_ATM=${CONVERT_ATM:-.true.} +CONVERT_SFC=${CONVERT_SFC:-.true.} +CONVERT_NST=${CONVERT_NST:-.true.} + +COMIN=${COMIN:-$PWD} + +#---------------------------------------------------------------------------- +# ATM_FILES_INPUT - Input atmospheric data file(s). Not used for 'restart' +# INPUT_TYPE. +# +# ATM_CORE_FILES - Input atmospheric core files. Used for 'restart' +# INPUT_TYPE only. The first six entries are the tiled +# files. The seventh is the file containing the +# vertical coord definition. +# +# ATM_TRACER_FILES_INPUT - Input atmospheric tracer files for each tile. +# Used for 'restart' INPUT_TYPE only. +# +# SFC_FILES_INPUT - Input surface data file(s). +# +# NST_FILES_INPUT - Input nst data file. 'gfs_gaussian' INPUT_TYPE only. +# +# TRACERS_INPUT - List of input atmospheric tracer records to be processed. +#---------------------------------------------------------------------------- + +ATM_FILES_INPUT=${ATM_FILES_INPUT:-NULL} +ATM_CORE_FILES_INPUT=${ATM_CORE_FILES_INPUT:-NULL} +ATM_TRACER_FILES_INPUT=${ATM_TRACER_FILES_INPUT:-NULL} +SFC_FILES_INPUT=${SFC_FILES_INPUT:-NULL} +NST_FILES_INPUT=${NST_FILES_INPUT:-NULL} +TRACERS_INPUT=${TRACERS_INPUT:-'"spfh","clwmr","o3mr","icmr","rwmr","snmr","grle"'} + +#---------------------------------------------------------------------------- +# TRACERS_TARGET - List of target tracer records. Must corresponde with +# with TRACERS_INPUT. +# +# VCOORD_FILE - File containing vertical coordinate defintion for target +# grid. +# +# MOSAIC FILE_TARGET_GRID - Mosaic file for target grid (include path). +# The associated 'grid' files assumed to be in +# FIXfv3. +# +# OROG_FILES_TARGET_GRID - Orography file(s) for target grid. Assumed to +# be located in FIXfv3. +#---------------------------------------------------------------------------- + +TRACERS_TARGET=${TRACERS_TARGET:-'"sphum","liq_wat","o3mr","ice_wat","rainwat","snowwat","graupel"'} + +VCOORD_FILE=${VCOORD_FILE:-${FIXam}/global_hyblev.l65.txt} + +MOSAIC_FILE_TARGET_GRID=${MOSAIC_FILE_TARGET_GRID:-${FIXfv3}/C${CRES}_mosaic.nc} + +OROG_FILES_TARGET_GRID=${OROG_FILES_TARGET_GRID:-NULL} +if [ $OROG_FILES_TARGET_GRID == NULL ]; then + OROG_FILES_TARGET_GRID='C'${CRES}'_oro_data.tile1.nc","C'${CRES}'_oro_data.tile2.nc"' + OROG_FILES_TARGET_GRID=${OROG_FILES_TARGET_GRID}',"C'${CRES}'_oro_data.tile3.nc","C'${CRES}'_oro_data.tile4.nc"' + OROG_FILES_TARGET_GRID=${OROG_FILES_TARGET_GRID}',"C'${CRES}'_oro_data.tile5.nc","C'${CRES}'_oro_data.tile6.nc' +fi + +#---------------------------------------------------------------------------- +# APRUN - machine specific command to run program. +# CHGRESEXEC - program executable. +# OMP_NUM_THREADS - threads most useful for 'gfs_spectral' INPUT_TYPE. +# DATA - working directory. +# PGMOUT - standard output file +# PGMERR - standard error file +# REDOUT - standard output redirect +# REDERR - standard error redirect +#---------------------------------------------------------------------------- + +APRUN=${APRUN:-time} +CHGRESEXEC=${CHGRESEXEC:-${EXECufs}/chgres_cube.exe} + +export OMP_NUM_THREADS=${OMP_NUM_THREADS_CY:-1} + +PGMOUT=${PGMOUT:-${pgmout:-'&1'}} +PGMERR=${PGMERR:-${pgmerr:-'&2'}} +REDOUT=${REDOUT:-'1>'} +REDERR=${REDERR:-'2>'} + +DATA=${DATA:-$PWD/chgres} +mkdir -p $DATA +cd $DATA || exit 99 + +rm -f ./fort.41 + +cat << EOF > ./fort.41 + &config + mosaic_file_target_grid="${MOSAIC_FILE_TARGET_GRID}" + fix_dir_target_grid="${FIXsfc}" + orog_dir_target_grid="${FIXfv3}" + orog_files_target_grid="${OROG_FILES_TARGET_GRID}" + vcoord_file_target_grid="${VCOORD_FILE}" + mosaic_file_input_grid="${MOSAIC_FILE_INPUT_GRID}" + orog_dir_input_grid="${OROG_DIR_INPUT_GRID}" + orog_files_input_grid="${OROG_FILES_INPUT_GRID}" + data_dir_input_grid="${COMIN}" + atm_files_input_grid="${ATM_FILES_INPUT}" + atm_core_files_input_grid="${ATM_CORE_FILES_INPUT}" + atm_tracer_files_input_grid="${ATM_TRACER_FILES_INPUT}" + sfc_files_input_grid="${SFC_FILES_INPUT}" + nst_files_input_grid="${NST_FILES_INPUT}" + cycle_mon=$im + cycle_day=$id + cycle_hour=$ih + convert_atm=$CONVERT_ATM + convert_sfc=$CONVERT_SFC + convert_nst=$CONVERT_NST + input_type="${INPUT_TYPE}" + tracers=$TRACERS_TARGET + tracers_input=$TRACERS_INPUT + regional=$REGIONAL + halo_bndy=$HALO_BNDY + halo_blend=$HALO_BLEND + / +EOF + +$APRUN $CHGRESEXEC $REDOUT$PGMOUT $REDERR$PGMERR + +iret=$? +if [ $iret -ne 0 ]; then + echo "FATAL ERROR RUNNING CHGRES" + exit $iret +fi + +exit diff --git a/ush/fv3gfs_chgres.sh b/ush/fv3gfs_chgres.sh index a169abf1a..a7f02dcc7 100755 --- a/ush/fv3gfs_chgres.sh +++ b/ush/fv3gfs_chgres.sh @@ -12,7 +12,7 @@ # - Uncomment the job cards for your machine. Ensure the # 'machine' variable is uncommented. Ensure job cards for # the other machines are prefixed by "##". -# - Check the job card account. On Theia, this is +# - Check the job card account. On Hera, this is # "#SBATCH -q $account". On WCOSS, this is "#BSUB -P $account". # You may not have permissions to run under the default account. # - Make sure you have linked the 'fixed' directories using @@ -30,7 +30,7 @@ # CDATE - year/month/day/hour of your experiment. # HOMEgfs - Path to your checkout repository. # - Start the script as follows: -# Theia: "sbatch fv3gfs_chgres.sh" +# Hera: "sbatch fv3gfs_chgres.sh" # WCOSS Dell and Cray: "cat fv3gfs_chgres.sh | bsub" # #------------------------------------------------------------------------ @@ -59,7 +59,7 @@ export machine=WCOSS_DELL_P3 ##BSUB -extsched 'CRAYLINUX[]' #export machine=WCOSS_C -#---- THEIA JOBCARD +#---- Hera JOBCARD #---- Submit as: sbatch $script ##SBATCH -J fv3_chgres_driver ##SBATCH -A fv3-cpu @@ -68,13 +68,13 @@ export machine=WCOSS_DELL_P3 ##SBATCH -e log.chgres ##SBATCH --nodes=1 ##SBATCH -q batch -##SBATCH -t 02:00:00 -#export machine=THEIA +##SBATCH -t 03:00:00 +#export machine=HERA set -x export PSLOT=fv3test -export CDUMP=gdas +export CDUMP=gfs export CASE_HIGH=C768 export CASE_ENKF=C384 export CDATE=2019061000 @@ -87,10 +87,10 @@ elif [ $machine = WCOSS_DELL_P3 ]; then export HOMEgfs=$LS_SUBCWD/.. export PTMP=${PTMP:-"/gpfs/dell2/ptmp/$USER"} export zero_bias_dir=/gpfs/hps3/emc/global/noscrub/emc.glopara/ICS/bias_zero -elif [ $machine = THEIA ]; then +elif [ $machine = HERA ]; then export HOMEgfs=$SLURM_SUBMIT_DIR/.. - export PTMP=${PTMP:-"/scratch3/NCEPDEV/stmp1/$USER"} - export zero_bias_dir=/scratch4/NCEPDEV/global/save/glopara/git/bias_zero + export PTMP=${PTMP:-"/scratch2/NCEPDEV/stmp1/$USER"} + export zero_bias_dir=/scratch1/NCEPDEV/global/glopara/bias_zero else echo "Please define HOMEgfs and PTMP for your machine. exit" exit @@ -132,45 +132,28 @@ status=$? [[ $status -ne 0 ]] && exit $status if [ $machine = WCOSS_C ]; then - # . $MODULESHOME/init/sh 2>>/dev/null - # module load prod_util prod_envir hpss 2>>/dev/null - # module load PrgEnv-intel 2>>/dev/null export KMP_AFFINITY=disabled export OMP_NUM_THREADS_CH=24 export APRUNC="aprun -n 1 -N 1 -j 1 -d $OMP_NUM_THREADS_CH -cc depth" export APRUNTF='aprun -q -j1 -n1 -N1 -d1 -cc depth' - export SUB=/u/emc.glopara/bin/sub_wcoss_c + export SUB=$LS_SUBCWD/../util/sub_wcoss_c export ACCOUNT=FV3GFS-T2O export QUEUE=dev export QUEUE_TRANS=dev_transfer elif [ $machine = WCOSS_DELL_P3 ]; then - # . /usrx/local/prod/lmod/lmod/init/sh 2>>/dev/null - # module purge 2>>/dev/null - # module load EnvVars/1.0.2 2>>/dev/null - # module load lsf/10.1 2>>/dev/null - # module load ips/18.0.1.163 2>>/dev/null - # module load impi/18.0.1 2>>/dev/null - # module load prod_util/1.1.0 2>>/dev/null - # module load prod_envir/1.0.2 2>>/dev/null - # module load HPSS/5.0.2.5 2>>/dev/null - # module load NetCDF/4.5.0 2>>/dev/null - # module load HDF5-serial/1.10.1 2>>/dev/null export OMP_NUM_THREADS_CH=14 export KMP_AFFINITY=disabled export APRUNC="time" export APRUNTF="time" - export SUB=/u/emc.glopara/bin/sub_wcoss_d + export SUB=$LS_SUBCWD/../util/sub_wcoss_d export ACCOUNT=FV3GFS-T2O export QUEUE=dev export QUEUE_TRANS=dev_transfer -elif [ $machine = THEIA ]; then - # source $HOMEgfs/sorc/machine-setup.sh - # module use -a /scratch3/NCEPDEV/nwprod/lib/modulefiles - # module load netcdf hdf5/1.8.14 2>>/dev/null +elif [ $machine = HERA ]; then export APRUNC=time export APRUNTF=time export OMP_NUM_THREADS_CH=24 - export SUB=/home/glopara/bin/sub_slurm + export SUB=$SLURM_SUBMIT_DIR/../util/sub_slurm export ACCOUNT=fv3-cpu export QUEUE=batch export QUEUE_TRANS=batch @@ -421,19 +404,6 @@ cat > read_hpss.sh <>/dev/null - #export machine=$machine - #if [ $machine = WCOSS_C ]; then - # . $MODULESHOME/init/sh 2>>/dev/null - # module load hpss 2>>/dev/null - #elif [ $machine = WCOSS_DELL_P3 ]; then - # . /usrx/local/prod/lmod/lmod/init/sh 2>>/dev/null - # module load HPSS/5.0.2.5 2>>/dev/null - #elif [ $machine = THEIA ]; then - # source $HOMEgfs/sorc/machine-setup.sh 2>>/dev/null - # module use -a /scratch3/NCEPDEV/nwprod/lib/modulefiles 2>>/dev/null - # module load hpss 2>>/dev/null - #fi - cd $INIDIR htar -xvf $HPSSPATH/$tarball_enkf_atm if [ $CDATE -le 2017072000 ] && [ $CDATE -gt 2014050100 ]; then diff --git a/ush/global_chgres.sh b/ush/global_chgres.sh index fd280ce18..d2770edea 100755 --- a/ush/global_chgres.sh +++ b/ush/global_chgres.sh @@ -300,7 +300,7 @@ FNVMNC=${FNVMNC:-${FIXam}/global_shdmin.0.144x0.144.grb} FNVMXC=${FNVMXC:-${FIXam}/global_shdmax.0.144x0.144.grb} FNSLPC=${FNSLPC:-${FIXam}/global_slope.1x1.grb} FNABSC=${FNABSC:-${FIXam}/global_snoalb.1x1.grb} -FNMSKH=${FNMSKH:-${FIXam}/seaice_newland.grb} +FNMSKH=${FNMSKH:-${FIXam}/global_slmask.t1534.3072.1536.grb} LANDICE_OPT=${LANDICE_OPT:-2} CLIMO_FIELDS_OPT=${CLIMO_FIELDS_OPT:-3} SOILTYPE_INP=${SOILTYPE_INP:-"zobler"} diff --git a/ush/global_chgres_driver.sh b/ush/global_chgres_driver.sh index 01c02d518..e81997c08 100755 --- a/ush/global_chgres_driver.sh +++ b/ush/global_chgres_driver.sh @@ -191,7 +191,7 @@ export FNVEGC=${FIXam}/global_vegfrac.0.144.decpercent.grb export FNVMNC=${FIXam}/global_shdmin.0.144x0.144.grb export FNVMXC=${FIXam}/global_shdmax.0.144x0.144.grb export FNSLPC=${FIXam}/global_slope.1x1.grb -export FNMSKH=${FIXam}/seaice_newland.grb +export FNMSKH=${FIXam}/global_slmask.t1534.3072.1536.grb export FNSMCC=$FIXam/global_soilmgldas.statsgo.t${JCAP_CASE}.${LONB_SFC}.${LATB_SFC}.grb export FNSOTC=$FIXam/global_soiltype.statsgo.t${JCAP_CASE}.${LONB_SFC}.${LATB_SFC}.rg.grb export FNVETC=$FIXam/global_vegtype.igbp.t${JCAP_CASE}.${LONB_SFC}.${LATB_SFC}.rg.grb diff --git a/ush/global_cycle.sh b/ush/global_cycle.sh index cd7396c5a..477253681 100755 --- a/ush/global_cycle.sh +++ b/ush/global_cycle.sh @@ -59,6 +59,8 @@ # Defaults to ${FIXam}/global_maxice.2x2.grb # FNTSFC Input SST climatology GRIB file. # Defaults to ${FIXam}/RTGSST.1982.2012.monthly.clim.grb +# FNSALC Input Salinity climatology netcdf file. +# Defaults to ${FIXam}/global_salclm.t1534.3072.1536.nc # FNSNOC Input snow climatology GRIB file. # Defaults to ${FIXam}/global_snoclim.1.875.grb # FNZORC Input roughness climatology. @@ -178,6 +180,7 @@ # fixed data : $FNGLAC # $FNMXIC # $FNTSFC +# $FNSALC # $FNSNOC # $FNZORC # $FNALBC @@ -272,6 +275,7 @@ MAX_TASKS_CY=${MAX_TASKS_CY:-99999} FNGLAC=${FNGLAC:-${FIXam}/global_glacier.2x2.grb} FNMXIC=${FNMXIC:-${FIXam}/global_maxice.2x2.grb} FNTSFC=${FNTSFC:-${FIXam}/RTGSST.1982.2012.monthly.clim.grb} +FNSALC=${FNSALC:-${FIXam}/global_salclm.t1534.3072.1536.nc} FNSNOC=${FNSNOC:-${FIXam}/global_snoclim.1.875.grb} FNZORC=${FNZORC:-igbp} FNALBC2=${FNALBC2:-${FIXam}/global_albedo4.1x1.grb} @@ -286,7 +290,7 @@ FNABSC=${FNABSC:-${FIXam}/global_mxsnoalb.uariz.t$JCAP_CASE.$LONB_CASE.$LATB_CAS FNVMNC=${FNVMNC:-${FIXam}/global_shdmin.0.144x0.144.grb} FNVMXC=${FNVMXC:-${FIXam}/global_shdmax.0.144x0.144.grb} FNSLPC=${FNSLPC:-${FIXam}/global_slope.1x1.grb} -FNMSKH=${FNMSKH:-${FIXam}/seaice_newland.grb} +FNMSKH=${FNMSKH:-${FIXam}/global_slmask.t1534.3072.1536.grb} GSI_FILE=${GSI_FILE:-"NULL"} FNTSFA=${FNTSFA:-${COMIN}/${PREINP}sstgrb${SUFINP}} FNACNA=${FNACNA:-${COMIN}/${PREINP}engicegrb${SUFINP}} @@ -315,6 +319,9 @@ fi cd $DATA||exit 99 [[ -d $COMOUT ]]||mkdir -p $COMOUT +ln -fs $FNTSFC sstclm +ln -fs $FNSALC salclm + # If the appropriate resolution fix file is not present, use the highest resolution available (T1534) [[ ! -f $FNALBC ]] && FNALBC="$FIXam/global_snowfree_albedo.bosu.t1534.3072.1536.rg.grb" [[ ! -f $FNVETC ]] && FNVETC="$FIXam/global_vegtype.igbp.t1534.3072.1536.rg.grb" diff --git a/ush/load_fv3gfs_modules.sh b/ush/load_fv3gfs_modules.sh index e48f1af47..8972b6575 100755 --- a/ush/load_fv3gfs_modules.sh +++ b/ush/load_fv3gfs_modules.sh @@ -15,9 +15,9 @@ module use "$HOMEgfs/modulefiles" if [[ -d /lfs3 ]] ; then # We are on NOAA Jet module load module_base.jet -elif [[ -d /scratch3 ]] ; then - # We are on NOAA Theia - module load module_base.theia +elif [[ -d /scratch1 ]] ; then + # We are on NOAA Hera + module load module_base.hera elif [[ -d /gpfs/hps && -e /etc/SuSE-release ]] ; then # We are on NOAA Luna or Surge module load module_base.wcoss_c diff --git a/util/sub_slurm b/util/sub_slurm new file mode 100755 index 000000000..97e70c9e2 --- /dev/null +++ b/util/sub_slurm @@ -0,0 +1,144 @@ +#!/bin/ksh +set -x +usage="\ +Usage: $0 [options] executable [args] + where the options are: + -a account account (default: none) + -b binding run smt binding or not (default:NO) + -d dirin initial directory (default: cwd) + -e envars copy comma-separated environment variables + -g group group name + -i append standard input to command file + -j jobname specify jobname (default: executable basename) + -m machine machine on which to run (default: current) + -n write command file to stdout rather than submitting it + -o output specify output file (default: jobname.out) + -p procs[/nodes[/ppreq] + number of MPI tasks and optional nodes or Bblocking and + ppreq option (N or S) (defaults: serial, Bunlimited, S) + -q queue[/qpreq] queue name and optional requirement, e.g. dev/P + (defaults: 1 if serial or dev if parallel and none) + (queue 3 or 4 is dev or prod with twice tasks over ip) + (options: P=parallel, B=bigmem, b=batch) + -r rmem[/rcpu] resources memory and cpus/task (default: '1024 mb', 1) + -t timew wall time limit in [[hh:]mm:]ss format (default: 900) + -u userid userid to run under (default: self) + -v verbose mode + -w when when to run, in yyyymmddhh[mm], +hh[mm], thh[mm], or + Thh[mm] (full, incremental, today or tomorrow) format + (default: now) +Function: This command submits a job to the batch queue." +subcmd="$*" +stdin=NO +nosub=NO +account="" +binding="NO" +dirin="" +envars="para" +group="" +jobname="" +machine="" +output="" +procs=0 +nodes="" +ppreq="N" +queue="" +qpreq="" +rmem="1200" +rcpu="1" +timew="900" +userid="" +verbose=NO +when="" +while getopts a:b:d:e:g:ij:m:no:p:q:r:t:u:vw: opt;do + case $opt in + a) account="$OPTARG";; + b) binding="$OPTARG";; + d) dirin="$OPTARG";; + e) envars="$OPTARG";; + g) group="$OPTARG";; + i) stdin=YES;; + j) jobname=$OPTARG;; + m) machine="$OPTARG";; + n) nosub=YES;; + o) output=$OPTARG;; + p) procs=$(echo $OPTARG/|cut -d/ -f1);nodes=$(echo $OPTARG/|cut -d/ -f2);ppreq=$(echo $OPTARG/|cut -d/ -f3);; + q) queue=$(echo $OPTARG/|cut -d/ -f1);qpreq=$(echo $OPTARG/|cut -d/ -f2);; + r) rmem=$(echo $OPTARG/|cut -d/ -f1);rcpu=$(echo $OPTARG/|cut -d/ -f2);; + t) timew=$OPTARG;; + u) userid=$OPTARG;; + v) verbose=YES;; + w) when=$OPTARG;; + \?) echo $0: invalid option >&2;echo "$usage" >&2;exit 1;; + esac +done +shift $(($OPTIND-1)) +if [[ $# -eq 0 ]];then + echo $0: missing executable name >&2;echo "$usage" >&2;exit 1 +fi +exec=$1 +if [[ ! -s $exec ]]&&which $exec >/dev/null 2>&1;then + exec=$(which $exec) +fi +shift +args="$*" +bn=$(basename $exec) +jobname=${jobname:-$bn} +output=${output:-$jobname.out} +myuser=$LOGNAME +myhost=$(hostname) + +DATA=${DATA:-/scratch4/NCEPDEV/stmp4/$LOGNAME/sub} +mkdir -p $DATA + +partition=${partition:-service} +queue=${queue:-batch} +timew=${timew:-01:20:00} +task_node=${procs:-12} +ntasks=$((nodes*task_node)) +envars=$envars + +#export TZ=GMT +export TZ="America/New_York" +cfile=$DATA/sub$$ +> $cfile +echo "#!/bin/ksh " >> $cfile +echo "#SBATCH -A $account" >> $cfile +echo "#SBATCH -o $output" >> $cfile +echo "#SBATCH -e $output" >> $cfile +echo "#SBATCH -J $jobname" >> $cfile +echo "#SBATCH -q $queue" >> $cfile +echo "#SBATCH -p $partition" >> $cfile +##echo "#SBATCH -v $envars" >> $cfile +echo "#SBATCH --nodes=$nodes --ntasks=$ntasks" >> $cfile +echo "#SBATCH -t $timew" >> $cfile +echo "/bin/ksh --login -x $exec $args" >> $cfile + +if [[ $stdin = YES ]];then + cat +fi >>$cfile +if [[ $nosub = YES ]];then + cat $cfile + exit +elif [[ $verbose = YES ]];then + set -x + cat $cfile +fi + +sbatch=${sbatch:-/apps/slurm/default/bin/sbatch} + +ofile=$DATA/subout$$ +>$ofile +chmod 777 $ofile +$sbatch $cfile >$ofile +rc=$? +cat $ofile +if [[ -w $SUBLOG ]];then + jobn=$(grep -i submitted $ofile|head -n1|cut -d\" -f2) +# date -u +"%Y%m%d%H%M%S : $subcmd : $jobn" >>$SUBLOG + date +"%Y%m%d%H%M%S : $subcmd : $jobn" >>$SUBLOG +fi +#exit +#rm $cfile $ofile +[[ $MKDATA = YES ]] && rmdir $DATA +exit $rc diff --git a/util/sub_wcoss_c b/util/sub_wcoss_c new file mode 100755 index 000000000..7bfaeeab7 --- /dev/null +++ b/util/sub_wcoss_c @@ -0,0 +1,277 @@ +#!/bin/ksh +set -x +# +# May 28, 2013 - Shrinivas Moorthi :now updated for lsf9.1.1 - should handle coupled case also +# +usage="\ +Usage: $0 [options] executable [args] + where the options are: + -a account account (default: none) + -b binding run smt binding or not (default:NO) + -d dirin initial directory (default: cwd) + -e envars copy comma-separated environment variables + -g group group name + -i append standard input to command file + -j jobname specify jobname (default: executable basename) +# -m machine machine on which to run (default: current) + -m mpiver mpi version (poe or intelmpi) (default: poe) + -n write command file to stdout rather than submitting it + -o output specify output file (default: jobname.out) + -p procs[/nodes[/ppreq] + number of MPI tasks and optional nodes or Bblocking and + ppreq option (N or S) (defaults: serial, Bunlimited, S) + -q queue[/qpreq] queue name and optional requirement, e.g. dev/P + (defaults: 1 if serial or dev if parallel and none) + (queue 3 or 4 is dev or prod with twice tasks over ip) + (options: P=parallel, B=bigmem, b=batch) + -r rmem[/rcpu[/pe_node] resources memory and cpus/task and cores per node (default: '1024 mb', 1, and 16) + -t timew wall time limit in [[hh:]mm:]ss format (default: 900) + -u userid userid to run under (default: self) + -v verbose mode + -w when when to run, in yyyymmddhh[mm], +hh[mm], thh[mm], or + Thh[mm] (full, incremental, today or tomorrow) format + (default: now) + + You can also export variables and + \"INHERIT_ENV\" (default:-\"YES\") - Set this variable to \"NO\" and + export it if you do not want the next job to inherit current job + environment. + + Other environmental variables which can be exported from outside are: + + \"KMP_STACKSIZE\" (default:-\"1024m\") + \"MP_EUIDEVELOP\" (default:-\"NULL\") + \"F_UFMTENDIAN\" (default:-\"NULL\") + \"MPICH_ALLTOALL_THROTTLE\" (default:-\"NULL\") + \"MP_SINGLE_THREAD\" (default:-\"NULL\") + \"MP_EAGER_LIMIT\" (default:-\"NULL\") + \"MP_USE_BULK_XFER\" (default:-\"NULL\") + \"MP_COLLECTIVE_OFFLOAD\" (default:-\"NULL\") + \"MP_SHARED_MEMORY\" (default:-\"NULL\") + \"MP_MPILIB\" (default:-\"NULL\") + \"MP_LABELIO\" (default:-\"NULL\") + \"MP_STDOUTMODE\" (default:-\"NULL\") + \"DATA\" (default:-\"/stmp/$LOGNAME/sub\" + - deleted at the end if created) + +Function: This command submits a job to the batch queue." +subcmd="$*" +stdin=NO +nosub=NO +account="" +binding="NO" +dirin="" +envars="" +group="" +jobname="" +#machine="" +mpiver="" +output="" +procs=0 +nodes="" +ppreq="NONE" +queue="" +qpreq="" +rmem="1024" +rcpu="1" +pe_node=${pe_node:-16} +timew="900" +userid="" +verbose=NO +when="" +while getopts a:b:d:e:g:ij:m:no:p:q:r:t:u:vw: opt;do + case $opt in + a) account="$OPTARG";; + b) binding="$OPTARG";; + d) dirin="$OPTARG";; + e) envars="$OPTARG";; + g) group="$OPTARG";; + i) stdin=YES;; + j) jobname=$OPTARG;; +# m) machine="$OPTARG";; + m) mpiver="$OPTARG";; + n) nosub=YES;; + o) output=$OPTARG;; + p) procs=$(echo $OPTARG/|cut -d/ -f1);nodes=$(echo $OPTARG/|cut -d/ -f2);ppreq=$(echo $OPTARG/|cut -d/ -f3);; + q) queue=$(echo $OPTARG/|cut -d/ -f1);qpreq=$(echo $OPTARG/|cut -d/ -f2);; + r) rmem=$(echo $OPTARG/|cut -d/ -f1);rcpu=$(echo $OPTARG/|cut -d/ -f2);pe_node=$(echo $OPTARG/|cut -d/ -f3);; + t) timew=$OPTARG;; + u) userid=$OPTARG;; + v) verbose=YES;; + w) when=$OPTARG;; + \?) echo $0: invalid option >&2;echo "$usage" >&2;exit 1;; + esac +done +shift $(($OPTIND-1)) +if [[ $# -eq 0 ]];then + echo $0: missing executable name >&2;echo "$usage" >&2;exit 1 +fi +exec=$1 +if [[ ! -s $exec ]]&&which $exec >/dev/null 2>&1;then + exec=$(which $exec) +fi +shift +args="$*" +bn=$(basename $exec) +jobname=${jobname:-$bn} +#machine=${machine:-""} +output=${output:-$jobname.out} +myuser=$LOGNAME +myhost=$(hostname) +#mpiver=${mpiver:-poe} +envars=$envars + +#DATA=/lustre/fs/scratch/$LOGNAME/stmp +NDATE=${NDATE:-/gpfs/hps/nco/ops/nwprod/prod_util.v1.0.5/exec/ndate} +pext=${pext:-""} +#PTMP=${PREPTMP:-""}/ptmp${pext:-""} +PTMP=${PTMP:-/gpfs/hps/ptmp} +DATA=${DATA:-$PTMP/$LOGNAME/sub} +if [ -s $DATA ] ; then + MKDATA=NO +else + mkdir -p $DATA + MKDATA=YES +fi +dirin=${dirin:-$(pwd)} + +queue=${queue:-dev} +timew=${timew:-01:20} +timew=$(echo $timew |cut -d: -f1):$(echo $timew |cut -d: -f2) +threads=${rcpu:-1} +nthreads=$threads + + +max_core=${max_core:-24} +task_node=${pe_node:-${task_node:-$max_core}} + +export INHERIT_ENV=${INHERIT_ENV:-YES} +if [ $nodes -eq 1 ] ; then + task_node=$procs +fi +tot_size=$procs +max_tasks=$((max_core*nodes)) +#tot_size=$procs +if [ $((task_node*threads)) -gt $max_core ]; then + core_typ=cpu + nthreads=$threads + threads=$((2*max_core/task_node)) + echo "Hyper-threading is used - setting core_typ=$corei_typ" +fi +export core_typ=${core_typ:-core} + +export OMP_STACKSIZE=${OMP_STACKSIZE:-1024m} +export KMP_AFFINITY=${KMP_AFFINITY:-disabled} + +export TZ=GMT +cfile=$DATA/sub$$ +> $cfile + +if [ $INHERIT_ENV = YES ] ; then + echo "#!/bin/ksh" >> $cfile +else + echo "#!/bin/sh --login" >> $cfile + echo "#BSUB -L /bin/sh" >> $cfile +fi +echo "#BSUB -P $account" >> $cfile +echo "#BSUB -e $output" >> $cfile +echo "#BSUB -o $output" >> $cfile +echo "#BSUB -J $jobname" >> $cfile +#echo "#BSUB -network type=sn_all:mode=US" >> $cfile +echo "#BSUB -q $queue" >> $cfile +echo "#BSUB -W $timew" >> $cfile +##echo "#BSUB -cwd $dirin" >> $cfile +#echo "#BSUB -n $tot_size" >> $cfile +if [ $queue = dev_transfer ] ; then + echo "#BSUB -R rusage[mem=$rmem]" >> $cfile +fi +if [ $queue != dev_transfer ] ; then + echo "#BSUB -M $rmem" >> $cfile + echo "#BSUB -extsched 'CRAYLINUX[]' -R '1*{select[craylinux && !vnode]} + $max_tasks*{select[craylinux && vnode]span[ptile=24] cu[type=cabinet]}'" >> $cfile +fi + +if [[ -n $when ]];then + whena=$when + if [[ $when = +* ]];then + hr=$(echo $when|cut -c2-3) + mn=$(echo $when|cut -c4-5) + [[ -n $mn ]] || mn=00 + now=$(date -u +"%Y%m%d%H%M") + ((mn+=$(echo $now|cut -c11-12))) + [[ $mn -ge 60 ]] && ((hr+=1)) && ((mn-=60)) + [[ $mn -lt 10 ]] && mn=0$mn + whena=$($NDATE +$hr $(echo $now|cut -c1-10))$mn + elif [[ $when = t* ]];then + hr=$(echo $when|cut -c2-3) + mn=$(echo $when|cut -c4-5) + [[ -n $mn ]] || mn=00 + now=$(date -u +"%Y%m%d") + whena=$now$hr$mn + elif [[ $when = T* ]];then + hr=$(echo $when|cut -c2-3) + mn=$(echo $when|cut -c4-5) + [[ -n $mn ]] || mn=00 + now=$(date -u +"%Y%m%d%H") + whena=$($NDATE +24 $now|cut -c1-8)$hr$mn + fi + yr=$(echo $whena|cut -c1-4) + mo=$(echo $whena|cut -c5-6) + dy=$(echo $whena|cut -c7-8) + hr=$(echo $whena|cut -c9-10) + mn=$(echo $whena|cut -c11-12) + [[ -n $mn ]] || mn=00 + echo "#BSUB -b $yr:$mo:$dy:$hr:$mn" >> $cfile +fi + +#echo "source ~${LOGNAME}/.profile" >> $cfile +#echo "ulimit -s unlimited" >> $cfile +#if [ ${MP_EUIDEVICE:-NULL} = sn_all ] ; then +#echo "#BSUB -network \"type=sn_all:mode=US\" " >> $cfile +#fi +#if [ ${MP_EULIB:-NULL} != NULL ] ; then +#echo "export MP_EUILIB=$MP_EUILIB" >> $cfile +#fi + +if [ ${KMP_AFFINITY:-NULL} != NULL ] ; then + echo "export KMP_AFFINITY=$KMP_AFFINITY" >> $cfile +fi + +echo "export OMP_STACKSIZE=$OMP_STACKSIZE" >> $cfile + +if [ ${MP_LABELIO:-NULL} != NULL ] ; then + echo "export MP_LABELIO=$MP_LABELIO" >> $cfile +fi +if [ ${MP_STDOUTMODE:-NULL} != NULL ] ; then + echo "export MP_STDOUTMODE=$MP_STDOUTMODE " >> $cfile +fi +for var in $(eval echo $envars | tr , ' ') ; do + echo "export $var" >> $cfile +done +echo "$exec" >> $cfile +echo "export OMP_NUM_THREADS=$nthreads" >> $cfile + +if [[ $stdin = YES ]];then + cat +fi >>$cfile +if [[ $nosub = YES ]];then + cat $cfile + exit +elif [[ $verbose = YES ]];then + set -x + cat $cfile +fi +bsub=${bsub:-$LSF_BINDIR/bsub} + +ofile=$DATA/subout$$ +>$ofile +chmod 777 $ofile +$bsub < $cfile +rc=$? +cat $ofile +if [[ -w $SUBLOG ]];then + jobn=$(grep -i submitted $ofile|head -n1|cut -d\" -f2) + date +"%Y%m%d%H%M%S : $subcmd : $jobn" >>$SUBLOG +fi +#rm $cfile $ofile +#[[ $MKDATA = YES ]] && rmdir $DATA +exit $rc diff --git a/util/sub_wcoss_d b/util/sub_wcoss_d new file mode 100755 index 000000000..77f0e4da8 --- /dev/null +++ b/util/sub_wcoss_d @@ -0,0 +1,247 @@ +#!/bin/ksh +set -x +# +# May 28, 2013 - Shrinivas Moorthi :now updated for lsf9.1.1 - should handle coupled case also +# +usage="\ +Usage: $0 [options] executable [args] + where the options are: + -a account account (default: none) + -b binding run smt binding or not (default:NO) + -d dirin initial directory (default: cwd) + -e envars copy comma-separated environment variables + -g group group name + -i append standard input to command file + -j jobname specify jobname (default: executable basename) +# -m machine machine on which to run (default: current) + -m mpiver mpi version (poe or intelmpi) (default: poe) + -n write command file to stdout rather than submitting it + -o output specify output file (default: jobname.out) + -p procs[/nodes[/ppreq] + number of MPI tasks and optional nodes or Bblocking and + ppreq option (N or S) (defaults: serial, Bunlimited, S) + -q queue[/qpreq] queue name and optional requirement, e.g. dev/P + (defaults: 1 if serial or dev if parallel and none) + (queue 3 or 4 is dev or prod with twice tasks over ip) + (options: P=parallel, B=bigmem, b=batch) + -r rmem[/rcpu[/pe_node] resources memory and cpus/task and cores per node (default: '1024 mb', 1, and 16) + -t timew wall time limit in [[hh:]mm:]ss format (default: 900) + -u userid userid to run under (default: self) + -v verbose mode + -w when when to run, in yyyymmddhh[mm], +hh[mm], thh[mm], or + Thh[mm] (full, incremental, today or tomorrow) format + (default: now) + + You can also export variables and + \"INHERIT_ENV\" (default:-\"YES\") - Set this variable to \"NO\" and + export it if you do not want the next job to inherit current job + environment. + + Other environmental variables which can be exported from outside are: + + \"KMP_STACKSIZE\" (default:-\"1024m\") + \"MP_EUIDEVELOP\" (default:-\"NULL\") + \"F_UFMTENDIAN\" (default:-\"NULL\") + \"MPICH_ALLTOALL_THROTTLE\" (default:-\"NULL\") + \"MP_SINGLE_THREAD\" (default:-\"NULL\") + \"MP_EAGER_LIMIT\" (default:-\"NULL\") + \"MP_USE_BULK_XFER\" (default:-\"NULL\") + \"MP_COLLECTIVE_OFFLOAD\" (default:-\"NULL\") + \"MP_SHARED_MEMORY\" (default:-\"NULL\") + \"MP_MPILIB\" (default:-\"NULL\") + \"MP_LABELIO\" (default:-\"NULL\") + \"MP_STDOUTMODE\" (default:-\"NULL\") + \"DATA\" (default:-\"/stmp/$LOGNAME/sub\" + - deleted at the end if created) + +Function: This command submits a job to the batch queue." +subcmd="$*" +stdin=NO +nosub=NO +account="" +binding="NO" +dirin="" +envars="" +group="" +jobname="" +#machine="" +mpiver="" +output="" +procs=0 +nodes="" +ppreq="NONE" +queue="" +qpreq="" +rmem="1024" +rcpu="1" +pe_node=${pe_node:-16} +timew="900" +userid="" +verbose=NO +when="" +while getopts a:b:d:e:g:ij:m:no:p:q:r:t:u:vw: opt;do + case $opt in + a) account="$OPTARG";; + b) binding="$OPTARG";; + d) dirin="$OPTARG";; + e) envars="$OPTARG";; + g) group="$OPTARG";; + i) stdin=YES;; + j) jobname=$OPTARG;; + m) mpiver="$OPTARG";; + n) nosub=YES;; + o) output=$OPTARG;; + p) procs=$(echo $OPTARG/|cut -d/ -f1);nodes=$(echo $OPTARG/|cut -d/ -f2);ppreq=$(echo $OPTARG/|cut -d/ -f3);; + q) queue=$(echo $OPTARG/|cut -d/ -f1);qpreq=$(echo $OPTARG/|cut -d/ -f2);; + r) rmem=$(echo $OPTARG/|cut -d/ -f1);rcpu=$(echo $OPTARG/|cut -d/ -f2);pe_node=$(echo $OPTARG/|cut -d/ -f3);; + t) timew=$OPTARG;; + u) userid=$OPTARG;; + v) verbose=YES;; + w) when=$OPTARG;; + \?) echo $0: invalid option >&2;echo "$usage" >&2;exit 1;; + esac +done +shift $(($OPTIND-1)) +if [[ $# -eq 0 ]];then + echo $0: missing executable name >&2;echo "$usage" >&2;exit 1 +fi +exec=$1 +if [[ ! -s $exec ]]&&which $exec >/dev/null 2>&1;then + exec=$(which $exec) +fi +shift +args="$*" +bn=$(basename $exec) +jobname=${jobname:-$bn} +#machine=${machine:-""} +output=${output:-$jobname.out} +myuser=$LOGNAME +myhost=$(hostname) +envars=$envars + +#DATA=/lustre/fs/scratch/$LOGNAME/stmp +NDATE=${NDATE:-/gpfs/hps/nco/ops/nwprod/prod_util.v1.0.5/exec/ndate} +pext=${pext:-""} +PTMP=/gpfs/dell2/ptmp +DATA=${DATA:-$PTMP/$LOGNAME/sub} +if [ -s $DATA ] ; then + MKDATA=NO +else + mkdir -p $DATA + MKDATA=YES +fi +dirin=${dirin:-$(pwd)} + +queue=${queue:-dev} +timew=${timew:-01:20} +timew=$(echo $timew |cut -d: -f1):$(echo $timew |cut -d: -f2) +threads=${rcpu:-1} +nthreads=$threads + +max_core=${max_core:-28} +task_node=${pe_node:-${task_node:-$max_core}} + +export INHERIT_ENV=${INHERIT_ENV:-YES} +if [ $nodes -eq 1 ] ; then + task_node=$procs +fi +tot_size=$procs +max_tasks=$((max_core*nodes)) +if [ $((task_node*threads)) -gt $max_core ]; then + core_typ=cpu + nthreads=$threads + threads=$((2*max_core/task_node)) + echo "Hyper-threading is used - setting core_typ=$corei_typ" +fi +core_typ=${core_typ:-core} + +export OMP_STACKSIZE=${OMP_STACKSIZE:-1024m} +export KMP_AFFINITY=${KMP_AFFINITY:-disabled} + +export TZ=GMT +cfile=$DATA/sub$$ +> $cfile + +echo "#!/bin/bash" >> $cfile +echo "#BSUB -P $account" >> $cfile +echo "#BSUB -e $output" >> $cfile +echo "#BSUB -o $output" >> $cfile +echo "#BSUB -J $jobname" >> $cfile +echo "#BSUB -q $queue" >> $cfile +echo "#BSUB -W $timew" >> $cfile +echo "#BSUB -n $tot_size" >> $cfile +echo "#BSUB -R span[ptile=$task_node]" >> $cfile +echo "#BSUB -R affinity[core($nthreads)]" >> $cfile +echo "#BSUB -M $rmem" >> $cfile + + +if [[ -n $when ]];then + whena=$when + if [[ $when = +* ]];then + hr=$(echo $when|cut -c2-3) + mn=$(echo $when|cut -c4-5) + [[ -n $mn ]] || mn=00 + now=$(date -u +"%Y%m%d%H%M") + ((mn+=$(echo $now|cut -c11-12))) + [[ $mn -ge 60 ]] && ((hr+=1)) && ((mn-=60)) + [[ $mn -lt 10 ]] && mn=0$mn + whena=$($NDATE +$hr $(echo $now|cut -c1-10))$mn + elif [[ $when = t* ]];then + hr=$(echo $when|cut -c2-3) + mn=$(echo $when|cut -c4-5) + [[ -n $mn ]] || mn=00 + now=$(date -u +"%Y%m%d") + whena=$now$hr$mn + elif [[ $when = T* ]];then + hr=$(echo $when|cut -c2-3) + mn=$(echo $when|cut -c4-5) + [[ -n $mn ]] || mn=00 + now=$(date -u +"%Y%m%d%H") + whena=$($NDATE +24 $now|cut -c1-8)$hr$mn + fi + yr=$(echo $whena|cut -c1-4) + mo=$(echo $whena|cut -c5-6) + dy=$(echo $whena|cut -c7-8) + hr=$(echo $whena|cut -c9-10) + mn=$(echo $whena|cut -c11-12) + [[ -n $mn ]] || mn=00 + echo "#BSUB -b $yr:$mo:$dy:$hr:$mn" >> $cfile +fi + +if [ ${KMP_AFFINITY:-NULL} != NULL ] ; then + echo "export KMP_AFFINITY=$KMP_AFFINITY" >> $cfile +fi +echo "export OMP_STACKSIZE=$OMP_STACKSIZE" >> $cfile + + +for var in $(eval echo $envars | tr , ' ') ; do + echo "export $var" >> $cfile +done +echo "$exec" >> $cfile +echo "export OMP_NUM_THREADS=$nthreads" >> $cfile + +if [[ $stdin = YES ]];then + cat +fi >>$cfile +if [[ $nosub = YES ]];then + cat $cfile + exit +elif [[ $verbose = YES ]];then + set -x + cat $cfile +fi +bsub=${bsub:-$LSF_BINDIR/bsub} + +ofile=$DATA/subout$$ +>$ofile +chmod 777 $ofile +$bsub < $cfile +rc=$? +cat $ofile +if [[ -w $SUBLOG ]];then + jobn=$(grep -i submitted $ofile|head -n1|cut -d\" -f2) + date +"%Y%m%d%H%M%S : $subcmd : $jobn" >>$SUBLOG +fi +##rm $cfile $ofile +#[[ $MKDATA = YES ]] && rmdir $DATA +exit $rc