From 8592b81e0c2f93f814c3779f57c39610c4f2f670 Mon Sep 17 00:00:00 2001 From: Larissa Reames Date: Wed, 21 Aug 2019 20:51:07 +0000 Subject: [PATCH 01/34] chgres_cube_grib2_atmos: This commit references #66268 dzdt is now properly converted to sigma coordinates and flipped vertically for non-native grib2 files --- sorc/chgres_cube.fd/input_data.F90 | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/sorc/chgres_cube.fd/input_data.F90 b/sorc/chgres_cube.fd/input_data.F90 index 0cb754860..110b90cb7 100644 --- a/sorc/chgres_cube.fd/input_data.F90 +++ b/sorc/chgres_cube.fd/input_data.F90 @@ -2322,7 +2322,7 @@ subroutine read_input_atm_grib2_file(localpet) tracers(n)=tracers_default(i) enddo - allocate(atm(num_tracers+4)) + allocate(atm(num_tracers+5)) if (localpet==0) print*, "NUMBER OF TRACERS IN FILE = ", num_tracers if (localpet == 0) print*,"- CALL FieldCreate FOR INPUT GRID SURFACE PRESSURE." @@ -2513,7 +2513,7 @@ subroutine read_input_atm_grib2_file(localpet) vname = ":DZDT:" do vlev = 1, lev_input iret = grb2_inq(the_file,inv_file,vname,slevs(vlev),data2=dummy2d) - if (iret <= 0) then + if (iret <= 0 ) then print*,"DZDT not available at level ", trim(slevs(vlev)), " so checking for VVEL" vname = ":VVEL:" iret = grb2_inq(the_file,inv_file,vname,slevs(vlev),data2=dummy2d) @@ -2522,7 +2522,7 @@ subroutine read_input_atm_grib2_file(localpet) if (iret <= 0) then call handle_grib_error(vname, slevs(vlev),method,value,varnum,iret,var=dummy2d) if (iret==1) then ! missing_var_method == skip - exit + cycle endif else conv_omega = .true. @@ -2600,18 +2600,24 @@ subroutine read_input_atm_grib2_file(localpet) farrayPtr=atm(4)%var, rc=rc) if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & call error_handler("IN FieldGet", rc) + + if (localpet == 0) print*,"- CALL FieldGet FOR W" + call ESMF_FieldGet(dzdt_input_grid, & + farrayPtr=atm(5)%var, rc=rc) + if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & + call error_handler("IN FieldGet", rc) if (localpet == 0) print*,"- CALL FieldGet FOR TRACERS." do i=1,num_tracers call ESMF_FieldGet(tracers_input_grid(i), & - farrayPtr=atm(i+4)%var, rc=rc) + farrayPtr=atm(i+5)%var, rc=rc) if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & call error_handler("IN FieldGet", rc) if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & call error_handler("IN FieldGet", rc) end do - call iso2sig(rlevs,vcoord,lev_input,levp1_input,psptr,atm,clb,cub,4+num_tracers, iret) + call iso2sig(rlevs,vcoord,lev_input,levp1_input,psptr,atm,clb,cub,5+num_tracers, iret) deallocate(vcoord) else From d6c9b3186b890ecbf596fb45eb666cd6f7632d75 Mon Sep 17 00:00:00 2001 From: LarissaReames-NOAA <52886575+LarissaReames-NOAA@users.noreply.github.com> Date: Fri, 23 Aug 2019 09:21:06 -0500 Subject: [PATCH 02/34] chgres_cube_grib2_atmos: This commit references #66268 (#8) dzdt is now properly converted to sigma coordinates and flipped vertically for non-native grib2 files --- sorc/chgres_cube.fd/input_data.F90 | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/sorc/chgres_cube.fd/input_data.F90 b/sorc/chgres_cube.fd/input_data.F90 index 0cb754860..110b90cb7 100644 --- a/sorc/chgres_cube.fd/input_data.F90 +++ b/sorc/chgres_cube.fd/input_data.F90 @@ -2322,7 +2322,7 @@ subroutine read_input_atm_grib2_file(localpet) tracers(n)=tracers_default(i) enddo - allocate(atm(num_tracers+4)) + allocate(atm(num_tracers+5)) if (localpet==0) print*, "NUMBER OF TRACERS IN FILE = ", num_tracers if (localpet == 0) print*,"- CALL FieldCreate FOR INPUT GRID SURFACE PRESSURE." @@ -2513,7 +2513,7 @@ subroutine read_input_atm_grib2_file(localpet) vname = ":DZDT:" do vlev = 1, lev_input iret = grb2_inq(the_file,inv_file,vname,slevs(vlev),data2=dummy2d) - if (iret <= 0) then + if (iret <= 0 ) then print*,"DZDT not available at level ", trim(slevs(vlev)), " so checking for VVEL" vname = ":VVEL:" iret = grb2_inq(the_file,inv_file,vname,slevs(vlev),data2=dummy2d) @@ -2522,7 +2522,7 @@ subroutine read_input_atm_grib2_file(localpet) if (iret <= 0) then call handle_grib_error(vname, slevs(vlev),method,value,varnum,iret,var=dummy2d) if (iret==1) then ! missing_var_method == skip - exit + cycle endif else conv_omega = .true. @@ -2600,18 +2600,24 @@ subroutine read_input_atm_grib2_file(localpet) farrayPtr=atm(4)%var, rc=rc) if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & call error_handler("IN FieldGet", rc) + + if (localpet == 0) print*,"- CALL FieldGet FOR W" + call ESMF_FieldGet(dzdt_input_grid, & + farrayPtr=atm(5)%var, rc=rc) + if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & + call error_handler("IN FieldGet", rc) if (localpet == 0) print*,"- CALL FieldGet FOR TRACERS." do i=1,num_tracers call ESMF_FieldGet(tracers_input_grid(i), & - farrayPtr=atm(i+4)%var, rc=rc) + farrayPtr=atm(i+5)%var, rc=rc) if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & call error_handler("IN FieldGet", rc) if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & call error_handler("IN FieldGet", rc) end do - call iso2sig(rlevs,vcoord,lev_input,levp1_input,psptr,atm,clb,cub,4+num_tracers, iret) + call iso2sig(rlevs,vcoord,lev_input,levp1_input,psptr,atm,clb,cub,5+num_tracers, iret) deallocate(vcoord) else From 8aa3b631adfd1542fd00b97af8786befd796850d Mon Sep 17 00:00:00 2001 From: George Gayno Date: Wed, 4 Sep 2019 15:35:36 +0000 Subject: [PATCH 03/34] feature/hera_port This commit references #1. Update build of 'fre-nctools', 'nst_tf_chg', 'nems_util' and 'orog' programs for Hera. Remove references to Theia. --- .../{fre-nctools.theia => fre-nctools.hera} | 10 ++++++---- .../fv3gfs/{nst_tf_chg.theia => nst_tf_chg.hera} | 16 ++++++++-------- modulefiles/fv3gfs/{orog.theia => orog.hera} | 15 +++++++-------- ...odule_nemsutil.theia => module_nemsutil.hera} | 10 +++++----- sorc/build_nems_util.sh | 2 +- sorc/build_nst_tf_chg.sh | 2 +- sorc/build_orog.sh | 4 ++-- .../{make.csh_theia => make.csh_hera} | 2 +- .../{make_hgrid/env.theia => fregrid/env.hera} | 2 +- .../tools/fregrid/env.wcoss_dell_p3 | 4 ++-- ...{fre-nctools.mk_theia => fre-nctools.mk_hera} | 0 .../env.theia => make_hgrid/env.hera} | 2 +- ...{fre-nctools.mk_theia => fre-nctools.mk_hera} | 0 .../env.theia => make_solo_mosaic/env.hera} | 4 ++-- .../tools/make_solo_mosaic/env.wcoss_dell_p3 | 2 +- ...{fre-nctools.mk_theia => fre-nctools.mk_hera} | 0 sorc/fre-nctools.fd/tools/shave.fd/build_shave | 2 +- sorc/machine-setup.sh | 9 ++++----- 18 files changed, 43 insertions(+), 43 deletions(-) rename modulefiles/fv3gfs/{fre-nctools.theia => fre-nctools.hera} (59%) rename modulefiles/fv3gfs/{nst_tf_chg.theia => nst_tf_chg.hera} (65%) rename modulefiles/fv3gfs/{orog.theia => orog.hera} (54%) rename modulefiles/{module_nemsutil.theia => module_nemsutil.hera} (66%) rename sorc/fre-nctools.fd/tools/filter_topo/{make.csh_theia => make.csh_hera} (81%) rename sorc/fre-nctools.fd/tools/{make_hgrid/env.theia => fregrid/env.hera} (85%) rename sorc/fre-nctools.fd/tools/fregrid/{fre-nctools.mk_theia => fre-nctools.mk_hera} (100%) rename sorc/fre-nctools.fd/tools/{make_solo_mosaic/env.theia => make_hgrid/env.hera} (85%) rename sorc/fre-nctools.fd/tools/make_hgrid/{fre-nctools.mk_theia => fre-nctools.mk_hera} (100%) rename sorc/fre-nctools.fd/tools/{fregrid/env.theia => make_solo_mosaic/env.hera} (64%) rename sorc/fre-nctools.fd/tools/make_solo_mosaic/{fre-nctools.mk_theia => fre-nctools.mk_hera} (100%) diff --git a/modulefiles/fv3gfs/fre-nctools.theia b/modulefiles/fv3gfs/fre-nctools.hera similarity index 59% rename from modulefiles/fv3gfs/fre-nctools.theia rename to modulefiles/fv3gfs/fre-nctools.hera index c1b678411..4ebd982e8 100644 --- a/modulefiles/fv3gfs/fre-nctools.theia +++ b/modulefiles/fv3gfs/fre-nctools.hera @@ -1,7 +1,9 @@ #%Module##################################################### ## Module file for fre-nctools ############################################################# -module load intel/14.0.2 -module load impi/4.1.3.048 -module load netcdf/4.3.0 -module load hdf5/1.8.14 + +module load intel/18.0.5.274 +module load impi/2018.0.4 + +module load netcdf/4.6.1 +module load hdf5/1.10.4 diff --git a/modulefiles/fv3gfs/nst_tf_chg.theia b/modulefiles/fv3gfs/nst_tf_chg.hera similarity index 65% rename from modulefiles/fv3gfs/nst_tf_chg.theia rename to modulefiles/fv3gfs/nst_tf_chg.hera index 9a8829df8..db82e1884 100644 --- a/modulefiles/fv3gfs/nst_tf_chg.theia +++ b/modulefiles/fv3gfs/nst_tf_chg.hera @@ -1,17 +1,17 @@ #%Module##################################################### -## gaussian_sfcanl build module for Theia +## Build module for Hera ############################################################# # Loading Intel Compiler Suite -module load intel/15.1.133 +module load intel/18.0.5.274 # Loding nceplibs modules -module use -a /scratch3/NCEPDEV/nwprod/lib/modulefiles -module load w3nco/v2.0.6 -module load bacio/v2.0.2 -module load nemsio/v2.2.3 -module load netcdf/4.3.0 -module load hdf5/1.8.14 +module use -a $MOD_PATH +module load w3nco/2.0.6 +module load bacio/2.0.2 +module load nemsio/2.2.5 +module load netcdf/4.6.1 +module load hdf5/1.10.4 export NETCDF_INCLUDE="-I${NETCDF}/include" export NETCDF_LDFLAGS_F="-L${NETCDF}/lib -lnetcdf -lnetcdff -L${HDF5}/lib -lhdf5 -lhdf5_fortran" diff --git a/modulefiles/fv3gfs/orog.theia b/modulefiles/fv3gfs/orog.hera similarity index 54% rename from modulefiles/fv3gfs/orog.theia rename to modulefiles/fv3gfs/orog.hera index b0543289b..f21a1118e 100644 --- a/modulefiles/fv3gfs/orog.theia +++ b/modulefiles/fv3gfs/orog.hera @@ -2,15 +2,14 @@ ## Module file for orog ############################################################# # Loading Intel Compiler Suite -module unload intel -module load intel/16.1.150 +module load intel/18.0.5.274 -module load netcdf/4.3.0 +module load netcdf/4.6.1 # Loding nceplibs modules module use -a $MOD_PATH -module load ip/v2.0.0 -module load sp/v2.0.2 -module load w3emc/v2.2.0 -module load w3nco/v2.0.6 -module load bacio/v2.0.2 +module load ip/3.0.1 +module load sp/2.0.2 +module load w3emc/2.3.0 +module load w3nco/2.0.6 +module load bacio/2.0.2 diff --git a/modulefiles/module_nemsutil.theia b/modulefiles/module_nemsutil.hera similarity index 66% rename from modulefiles/module_nemsutil.theia rename to modulefiles/module_nemsutil.hera index 45685118e..dc3b2a99f 100644 --- a/modulefiles/module_nemsutil.theia +++ b/modulefiles/module_nemsutil.hera @@ -3,13 +3,13 @@ ############################################################# # Loading Intel Compiler Suite -module load intel/14.0.2 -module load impi/5.1.2.150 +module load intel/18.0.5.274 +module load impi/2018.0.4 # Loding nceplibs modules module use -a $MOD_PATH -module load w3nco/v2.0.6 -module load bacio/v2.0.1 -module load nemsio/v2.2.1 +module load w3nco/2.0.6 +module load bacio/2.0.2 +module load nemsio/2.2.5 export FCMP=ifort diff --git a/sorc/build_nems_util.sh b/sorc/build_nems_util.sh index d46f2c584..e7a5b8b2c 100755 --- a/sorc/build_nems_util.sh +++ b/sorc/build_nems_util.sh @@ -6,7 +6,7 @@ cwd=`pwd` USE_PREINST_LIBS=${USE_PREINST_LIBS:-"true"} if [ $USE_PREINST_LIBS = true ]; then - export MOD_PATH=/scratch3/NCEPDEV/nwprod/lib/modulefiles + export MOD_PATH source ../modulefiles/module_nemsutil.$target > /dev/null 2>&1 else export MOD_PATH=${cwd}/lib/modulefiles diff --git a/sorc/build_nst_tf_chg.sh b/sorc/build_nst_tf_chg.sh index ef063f5f7..5ba9ec958 100755 --- a/sorc/build_nst_tf_chg.sh +++ b/sorc/build_nst_tf_chg.sh @@ -7,7 +7,7 @@ cwd=`pwd` module purge USE_PREINST_LIBS=${USE_PREINST_LIBS:-"true"} if [ $USE_PREINST_LIBS = true ]; then - export MOD_PATH=/scratch3/NCEPDEV/nwprod/lib/modulefiles + export MOD_PATH source ../modulefiles/fv3gfs/nst_tf_chg.$target > /dev/null 2>&1 else export MOD_PATH=${cwd}/lib/modulefiles diff --git a/sorc/build_orog.sh b/sorc/build_orog.sh index a35846696..c1cc33a81 100755 --- a/sorc/build_orog.sh +++ b/sorc/build_orog.sh @@ -10,7 +10,7 @@ cwd=`pwd` USE_PREINST_LIBS=${USE_PREINST_LIBS:-"true"} if [ $USE_PREINST_LIBS = true ]; then - export MOD_PATH=/scratch3/NCEPDEV/nwprod/lib/modulefiles + export MOD_PATH source ../modulefiles/fv3gfs/orog.$target > /dev/null 2>&1 else export MOD_PATH=${cwd}/lib/modulefiles @@ -43,7 +43,7 @@ elif [ $target = jet ]; then INCS="-I${NETCDF}/include" export LIBSM="${BACIO_LIB4} ${W3NCO_LIBd} ${IP_LIBd} ${SP_LIBd} -L${NETCDF}/lib -lnetcdff -lnetcdf" export FFLAGSM="-O3 -g -traceback -r8 -convert big_endian -fp-model precise -assume byterecl ${INCS}" -elif [ $target = theia ]; then +elif [ $target = hera ]; then INCS="-I${NETCDF}/include" export LIBSM="${BACIO_LIB4} ${W3NCO_LIBd} ${IP_LIBd} ${SP_LIBd} -L${NETCDF}/lib -lnetcdff -lnetcdf" export FFLAGSM="-O3 -g -traceback -r8 -convert big_endian -fp-model precise -assume byterecl ${INCS}" diff --git a/sorc/fre-nctools.fd/tools/filter_topo/make.csh_theia b/sorc/fre-nctools.fd/tools/filter_topo/make.csh_hera similarity index 81% rename from sorc/fre-nctools.fd/tools/filter_topo/make.csh_theia rename to sorc/fre-nctools.fd/tools/filter_topo/make.csh_hera index 563538c4c..fb43802b0 100755 --- a/sorc/fre-nctools.fd/tools/filter_topo/make.csh_theia +++ b/sorc/fre-nctools.fd/tools/filter_topo/make.csh_hera @@ -1,6 +1,6 @@ #!/bin/csh -f source $MODULESHOME/init/csh -source ${PWD}/../../../../modulefiles/fv3gfs/fre-nctools.theia +source ${PWD}/../../../../modulefiles/fv3gfs/fre-nctools.hera ifort -o filter_topo -I${NETCDF}/include -fltconsistency -fno-alias -stack_temps -safe_cray_ptr -ftz -assume byterecl -g -O2 -i4 -real_size 64 -traceback filter_topo.F90 -L${NETCDF}/lib -L${HDF5}/lib -lnetcdf -lnetcdff -lhdf5_hl -lhdf5 diff --git a/sorc/fre-nctools.fd/tools/make_hgrid/env.theia b/sorc/fre-nctools.fd/tools/fregrid/env.hera similarity index 85% rename from sorc/fre-nctools.fd/tools/make_hgrid/env.theia rename to sorc/fre-nctools.fd/tools/fregrid/env.hera index 6c090e2e2..a4ca56ba4 100644 --- a/sorc/fre-nctools.fd/tools/make_hgrid/env.theia +++ b/sorc/fre-nctools.fd/tools/fregrid/env.hera @@ -1,4 +1,4 @@ -# theia +# hera MPICC := mpiicc CC := icc STATIC := diff --git a/sorc/fre-nctools.fd/tools/fregrid/env.wcoss_dell_p3 b/sorc/fre-nctools.fd/tools/fregrid/env.wcoss_dell_p3 index 834db6e7e..07f52aac8 100644 --- a/sorc/fre-nctools.fd/tools/fregrid/env.wcoss_dell_p3 +++ b/sorc/fre-nctools.fd/tools/fregrid/env.wcoss_dell_p3 @@ -1,4 +1,4 @@ -# theia +# Dell MPICC := mpiicc CC := icc -STATIC := \ No newline at end of file +STATIC := diff --git a/sorc/fre-nctools.fd/tools/fregrid/fre-nctools.mk_theia b/sorc/fre-nctools.fd/tools/fregrid/fre-nctools.mk_hera similarity index 100% rename from sorc/fre-nctools.fd/tools/fregrid/fre-nctools.mk_theia rename to sorc/fre-nctools.fd/tools/fregrid/fre-nctools.mk_hera diff --git a/sorc/fre-nctools.fd/tools/make_solo_mosaic/env.theia b/sorc/fre-nctools.fd/tools/make_hgrid/env.hera similarity index 85% rename from sorc/fre-nctools.fd/tools/make_solo_mosaic/env.theia rename to sorc/fre-nctools.fd/tools/make_hgrid/env.hera index 6c090e2e2..a4ca56ba4 100644 --- a/sorc/fre-nctools.fd/tools/make_solo_mosaic/env.theia +++ b/sorc/fre-nctools.fd/tools/make_hgrid/env.hera @@ -1,4 +1,4 @@ -# theia +# hera MPICC := mpiicc CC := icc STATIC := diff --git a/sorc/fre-nctools.fd/tools/make_hgrid/fre-nctools.mk_theia b/sorc/fre-nctools.fd/tools/make_hgrid/fre-nctools.mk_hera similarity index 100% rename from sorc/fre-nctools.fd/tools/make_hgrid/fre-nctools.mk_theia rename to sorc/fre-nctools.fd/tools/make_hgrid/fre-nctools.mk_hera diff --git a/sorc/fre-nctools.fd/tools/fregrid/env.theia b/sorc/fre-nctools.fd/tools/make_solo_mosaic/env.hera similarity index 64% rename from sorc/fre-nctools.fd/tools/fregrid/env.theia rename to sorc/fre-nctools.fd/tools/make_solo_mosaic/env.hera index 834db6e7e..a4ca56ba4 100644 --- a/sorc/fre-nctools.fd/tools/fregrid/env.theia +++ b/sorc/fre-nctools.fd/tools/make_solo_mosaic/env.hera @@ -1,4 +1,4 @@ -# theia +# hera MPICC := mpiicc CC := icc -STATIC := \ No newline at end of file +STATIC := diff --git a/sorc/fre-nctools.fd/tools/make_solo_mosaic/env.wcoss_dell_p3 b/sorc/fre-nctools.fd/tools/make_solo_mosaic/env.wcoss_dell_p3 index 6c090e2e2..07f52aac8 100644 --- a/sorc/fre-nctools.fd/tools/make_solo_mosaic/env.wcoss_dell_p3 +++ b/sorc/fre-nctools.fd/tools/make_solo_mosaic/env.wcoss_dell_p3 @@ -1,4 +1,4 @@ -# theia +# Dell MPICC := mpiicc CC := icc STATIC := diff --git a/sorc/fre-nctools.fd/tools/make_solo_mosaic/fre-nctools.mk_theia b/sorc/fre-nctools.fd/tools/make_solo_mosaic/fre-nctools.mk_hera similarity index 100% rename from sorc/fre-nctools.fd/tools/make_solo_mosaic/fre-nctools.mk_theia rename to sorc/fre-nctools.fd/tools/make_solo_mosaic/fre-nctools.mk_hera diff --git a/sorc/fre-nctools.fd/tools/shave.fd/build_shave b/sorc/fre-nctools.fd/tools/shave.fd/build_shave index 81aa8469d..18cdf07aa 100755 --- a/sorc/fre-nctools.fd/tools/shave.fd/build_shave +++ b/sorc/fre-nctools.fd/tools/shave.fd/build_shave @@ -15,7 +15,7 @@ case $1 in "jet" ) FCMP=ifort FFLAGS="-O3 -fp-model precise -I$NETCDF/include -L$NETCDF/lib -lnetcdff -lnetcdf" ;; - "theia" ) + "hera" ) FCMP=ifort FFLAGS="-O3 -fp-model precise -I$NETCDF/include -L$NETCDF/lib -lnetcdff -lnetcdf" ;; *) diff --git a/sorc/machine-setup.sh b/sorc/machine-setup.sh index ed796f86b..d54235b60 100644 --- a/sorc/machine-setup.sh +++ b/sorc/machine-setup.sh @@ -28,16 +28,15 @@ if [[ -d /lfs3 ]] ; then target=jet module purge module use /mnt/lfs3/projects/hfv3gfs/nwprod/lib/modulefiles -elif [[ -d /scratch3 ]] ; then - # We are on NOAA Theia +elif [[ -d /scratch1 ]] ; then + # We are on NOAA Hera if ( ! eval module help > /dev/null 2>&1 ) ; then echo load the module command 1>&2 source /apps/lmod/lmod/init/$__ms_shell fi - target=theia + target=hera module purge - module use /scratch3/NCEPDEV/nwprod/modulefiles/ - module use /scratch3/NCEPDEV/nwprod/lib/modulefiles + MOD_PATH=/scratch2/NCEPDEV/nwprod/NCEPLIBS/modulefiles elif [[ -d /gpfs/hps && -e /etc/SuSE-release ]] ; then # We are on NOAA Luna or Surge if ( ! eval module help > /dev/null 2>&1 ) ; then From a38b232614da697384c106e0aadfd845f51125e3 Mon Sep 17 00:00:00 2001 From: George Gayno Date: Wed, 4 Sep 2019 19:08:24 +0000 Subject: [PATCH 04/34] feature/hera_port This commit references #1. Update 'emcsfc' and 'sfc_climo_gen' build for Hera. Add better logic to 'machine-setup.sh' to determine if on Hera. --- .../modulefile.global_emcsfc_ice_blend.hera | 20 ++++++++++++++ .../modulefile.global_emcsfc_ice_blend.theia | 17 ------------ .../modulefile.global_emcsfc_snow2mdl.hera | 26 +++++++++++++++++++ .../modulefile.global_emcsfc_snow2mdl.theia | 23 ---------------- ...en.theia => modulefile.sfc_climo_gen.hera} | 11 ++++---- sorc/build_emcsfc.sh | 1 + sorc/build_sfc_climo_gen.sh | 1 + sorc/machine-setup.sh | 2 +- 8 files changed, 55 insertions(+), 46 deletions(-) create mode 100644 modulefiles/modulefile.global_emcsfc_ice_blend.hera delete mode 100644 modulefiles/modulefile.global_emcsfc_ice_blend.theia create mode 100644 modulefiles/modulefile.global_emcsfc_snow2mdl.hera delete mode 100644 modulefiles/modulefile.global_emcsfc_snow2mdl.theia rename modulefiles/{modulefile.sfc_climo_gen.theia => modulefile.sfc_climo_gen.hera} (57%) diff --git a/modulefiles/modulefile.global_emcsfc_ice_blend.hera b/modulefiles/modulefile.global_emcsfc_ice_blend.hera new file mode 100644 index 000000000..b7fd0765b --- /dev/null +++ b/modulefiles/modulefile.global_emcsfc_ice_blend.hera @@ -0,0 +1,20 @@ +#%Module##################################################### +## emcsfc_ice_blend build module for Hera +############################################################# + +module load intel/18.0.5.274 +export FCOMP=ifort +export FFLAGS="-O0 -i4" + +# load ncep library modules + +module use -a $MOD_PATH +module load w3nco/2.0.6 +module load bacio/2.0.2 + +export NCEPLIBS=/scratch1/NCEPDEV/global/gwv/l819/lib +module use -a $NCEPLIBS/modulefiles +module load jasper-intel-sandybridge/1.900.1 +module load zlib-intel-sandybridge/1.2.7 +module load png-intel-sandybridge/1.2.59 +module load g2-intel-sandybridge/2.5.0 diff --git a/modulefiles/modulefile.global_emcsfc_ice_blend.theia b/modulefiles/modulefile.global_emcsfc_ice_blend.theia deleted file mode 100644 index d42fb6ac8..000000000 --- a/modulefiles/modulefile.global_emcsfc_ice_blend.theia +++ /dev/null @@ -1,17 +0,0 @@ -#%Module##################################################### -## emcsfc_ice_blend build module for Theia -############################################################# - -module load intel/14.0.2 -export FCOMP=ifort -export FFLAGS="-O0 -i4" - -# load ncep library modules - -module use -a /scratch3/NCEPDEV/nwprod/lib/modulefiles -module load w3nco/v2.0.6 -module load bacio/v2.0.1 -module load jasper -module load z -module load png -module load g2/v2.5.0 diff --git a/modulefiles/modulefile.global_emcsfc_snow2mdl.hera b/modulefiles/modulefile.global_emcsfc_snow2mdl.hera new file mode 100644 index 000000000..8aab4cf1a --- /dev/null +++ b/modulefiles/modulefile.global_emcsfc_snow2mdl.hera @@ -0,0 +1,26 @@ +#%Module##################################################### +## emcsfc_snow2mdl build module for Hera +############################################################# + +# load intel compiler + +module load intel/18.0.5.274 + +# load ncep library modules + +module use -a $MOD_PATH +module load ip/3.0.1 +module load sp/2.0.2 +module load w3nco/2.0.6 +module load bacio/2.0.2 +module load landsfcutil/2.1.0 + +export NCEPLIBS=/scratch1/NCEPDEV/global/gwv/l819/lib +module use -a $NCEPLIBS/modulefiles +module load jasper-intel-sandybridge/1.900.1 +module load zlib-intel-sandybridge/1.2.7 +module load png-intel-sandybridge/1.2.59 +module load g2-intel-sandybridge/2.5.0 + +export FCOMP=ifort +export FFLAGS="-O0 -r8 -i4 -FR -I${IP_INCd} -qopenmp -convert big_endian -assume byterecl" diff --git a/modulefiles/modulefile.global_emcsfc_snow2mdl.theia b/modulefiles/modulefile.global_emcsfc_snow2mdl.theia deleted file mode 100644 index b364fbe47..000000000 --- a/modulefiles/modulefile.global_emcsfc_snow2mdl.theia +++ /dev/null @@ -1,23 +0,0 @@ -#%Module##################################################### -## emcsfc_snow2mdl build module for Theia -############################################################# - -# load intel compiler - -module load intel/14.0.2 - -# load ncep library modules - -module use -a /scratch3/NCEPDEV/nwprod/lib/modulefiles -module load ip/v3.0.0 -module load sp/v2.0.2 -module load w3nco/v2.0.6 -module load bacio/v2.0.1 -module load jasper -module load z -module load png -module load g2/v2.5.0 -module load landsfcutil/v2.1.0 - -export FCOMP=ifort -export FFLAGS="-O0 -r8 -i4 -FR -I${IP_INCd} -openmp -convert big_endian -assume byterecl" diff --git a/modulefiles/modulefile.sfc_climo_gen.theia b/modulefiles/modulefile.sfc_climo_gen.hera similarity index 57% rename from modulefiles/modulefile.sfc_climo_gen.theia rename to modulefiles/modulefile.sfc_climo_gen.hera index 39e306c5e..8d6faa63d 100644 --- a/modulefiles/modulefile.sfc_climo_gen.theia +++ b/modulefiles/modulefile.sfc_climo_gen.hera @@ -1,12 +1,13 @@ #%Module##################################################### -## Module file for sfc_climo_gen +## Module file for sfc_climo_gen (Hera) ############################################################# -module load intel/15.1.133 -module load impi/5.1.1.109 -module load netcdf/4.3.0 -module use /scratch4/NCEPDEV/nems/noscrub/emc.nemspara/soft/modulefiles +module load intel/18.0.5.274 +module load impi/2018.0.4 module load esmf/7.1.0r +module use -a $MOD_PATH +module load netcdf/4.6.1 + export FCOMP=mpiifort export FFLAGS="-O3 -fp-model=precise -g -traceback -r8 -i4 -convert big_endian" diff --git a/sorc/build_emcsfc.sh b/sorc/build_emcsfc.sh index c2c600b12..233c40b26 100755 --- a/sorc/build_emcsfc.sh +++ b/sorc/build_emcsfc.sh @@ -14,6 +14,7 @@ do module purge USE_PREINST_LIBS=${USE_PREINST_LIBS:-"true"} if [ $USE_PREINST_LIBS = true ]; then + export MOD_PATH source ../modulefiles/modulefile.global_${prog}.${target} > /dev/null 2>&1 else export MOD_PATH=${cwd}/lib/modulefiles diff --git a/sorc/build_sfc_climo_gen.sh b/sorc/build_sfc_climo_gen.sh index 237169f2f..cf253d358 100755 --- a/sorc/build_sfc_climo_gen.sh +++ b/sorc/build_sfc_climo_gen.sh @@ -12,6 +12,7 @@ fi module purge USE_PREINST_LIBS=${USE_PREINST_LIBS:-"true"} if [ $USE_PREINST_LIBS = true ]; then + export MOD_PATH source ../modulefiles/modulefile.sfc_climo_gen.${target} > /dev/null 2>&1 else export MOD_PATH=${cwd}/lib/modulefiles diff --git a/sorc/machine-setup.sh b/sorc/machine-setup.sh index d54235b60..d9a3b29d1 100644 --- a/sorc/machine-setup.sh +++ b/sorc/machine-setup.sh @@ -28,7 +28,7 @@ if [[ -d /lfs3 ]] ; then target=jet module purge module use /mnt/lfs3/projects/hfv3gfs/nwprod/lib/modulefiles -elif [[ -d /scratch1 ]] ; then +elif [[ `hostname -a` == *"hera.fairmont.rdhpcs.noaa.gov"* ]] ; then # We are on NOAA Hera if ( ! eval module help > /dev/null 2>&1 ) ; then echo load the module command 1>&2 From 01c6f1d529633c36f1f0947614ea4a773c53eeb1 Mon Sep 17 00:00:00 2001 From: George Gayno Date: Wed, 4 Sep 2019 20:02:17 +0000 Subject: [PATCH 05/34] feature/hera_port This commit references issue #1. Update 'chgres', 'chgres_cube' and 'global_cycle' to build on Hera. All programs can now be compiled on Hera. --- modulefiles/chgres_cube.hera | 24 ++++++++++++++ modulefiles/chgres_cube.theia | 23 -------------- modulefiles/fv3gfs/global_chgres.hera | 29 +++++++++++++++++ modulefiles/fv3gfs/global_chgres.theia | 31 ------------------- .../{global_cycle.theia => global_cycle.hera} | 23 +++++++------- sorc/build_chgres.sh | 6 ++-- sorc/build_chgres_cube.sh | 2 +- sorc/build_cycle.sh | 2 +- 8 files changed, 70 insertions(+), 70 deletions(-) create mode 100644 modulefiles/chgres_cube.hera delete mode 100644 modulefiles/chgres_cube.theia create mode 100644 modulefiles/fv3gfs/global_chgres.hera delete mode 100644 modulefiles/fv3gfs/global_chgres.theia rename modulefiles/fv3gfs/{global_cycle.theia => global_cycle.hera} (55%) diff --git a/modulefiles/chgres_cube.hera b/modulefiles/chgres_cube.hera new file mode 100644 index 000000000..16ce8d125 --- /dev/null +++ b/modulefiles/chgres_cube.hera @@ -0,0 +1,24 @@ +#%Module##################################################### +## Build module for Hera +############################################################# + +module load intel/18.0.5.274 +module load impi/2018.0.4 +module load netcdf/4.6.1 + +module use -a $MOD_PATH +module load w3nco/2.0.6 +module load nemsio/2.2.5 +module load bacio/2.0.2 +module load sp/2.0.2 +module load sfcio/1.1.0 +module load sigio/2.1.0 + +export NCEPLIBS=/scratch1/NCEPDEV/global/gwv/l819/lib +module use -a $NCEPLIBS/modulefiles +module load esmflocal/8_0_44b + +export FCOMP=mpiifort +export FFLAGS="-O3 -fp-model precise -g -traceback -r8 -i4 -qopenmp -convert big_endian -assume byterecl" +# for debugging +#export FFLAGS="-O0 -g -traceback -r8 -i4 -qopenmp -convert big_endian -check bounds -warn unused -assume byterecl" diff --git a/modulefiles/chgres_cube.theia b/modulefiles/chgres_cube.theia deleted file mode 100644 index 2b3548ba0..000000000 --- a/modulefiles/chgres_cube.theia +++ /dev/null @@ -1,23 +0,0 @@ -#%Module##################################################### -## chgres build module for Theia -############################################################# - -module load intel/15.1.133 -module load impi/5.1.1.109 -module load netcdf/4.3.0 - -module use /scratch4/NCEPDEV/nems/noscrub/emc.nemspara/soft/modulefiles -module load esmf/8.0.0bs20 - -module use -a /scratch3/NCEPDEV/nwprod/lib/modulefiles -module load w3nco -module load nemsio/v2.2.3 -module load bacio/v2.0.1 -module load sp/v2.0.2 -module load sfcio/v1.0.0 -module load sigio/v2.0.1 - -export FCOMP=mpiifort -export FFLAGS="-O3 -fp-model precise -g -traceback -r8 -i4 -qopenmp -convert big_endian -assume byterecl" -# for debugging -#export FFLAGS="-O0 -g -traceback -r8 -i4 -qopenmp -convert big_endian -check bounds -warn unused -assume byterecl" diff --git a/modulefiles/fv3gfs/global_chgres.hera b/modulefiles/fv3gfs/global_chgres.hera new file mode 100644 index 000000000..256d094ba --- /dev/null +++ b/modulefiles/fv3gfs/global_chgres.hera @@ -0,0 +1,29 @@ +#%Module##################################################### +## global_chgres component - hera +############################################################# + +# Loading Intel Compiler Suite +module load intel/18.0.5.274 +module load impi/2018.0.4 + +module load netcdf/4.6.1 +module load hdf5/1.10.4 + +# Loding nceplibs modules +module use -a $MOD_PATH +module load sigio/2.1.0 +module load w3nco/2.0.6 +module load w3emc/2.3.0 +module load sp/2.0.2 +module load bacio/2.0.2 +module load nemsio/2.2.5 +module load nemsiogfs/2.2.0 +module load ip/3.0.1 +module load sfcio/1.1.0 +module load gfsio/1.1.0 +module load landsfcutil/2.1.0 + +export NETCDF_INCLUDE="-I${NETCDF}/include" +export NETCDF_LDFLAGS_F="-L${NETCDF}/lib -lnetcdf -lnetcdff -L${HDF5}/lib -lhdf5 -lhdf5_fortran" + +set FCMP ifort diff --git a/modulefiles/fv3gfs/global_chgres.theia b/modulefiles/fv3gfs/global_chgres.theia deleted file mode 100644 index fa764895f..000000000 --- a/modulefiles/fv3gfs/global_chgres.theia +++ /dev/null @@ -1,31 +0,0 @@ -#%Module##################################################### -## global_chgres component - theia -############################################################# - -# Loading Intel Compiler Suite -module load intel/14.0.2 -module load impi/5.1.2.150 - -# Loding nceplibs modules -module use -a $MOD_PATH -module load sigio/v2.0.1 -module load w3nco/v2.0.6 -module load w3emc/v2.2.0 -module load sp/v2.0.2 -module load bacio/v2.0.2 -module load nemsio/v2.2.3 -module load nemsiogfs/v2.0.1 -export NEMSIOGFS_INC=/scratch4/NCEPDEV/global/save/glopara/svn/nceplibs/branches/nemsiogfsv2.0.1/include/nemsiogfs -export NEMSIOGFS_LIB=/scratch4/NCEPDEV/global/save/glopara/svn/nceplibs/branches/nemsiogfsv2.0.1/libnemsiogfs.a - -module load ip/v3.0.0 -module load sfcio/v1.0.0 -module load gfsio/v1.1.0 -module load landsfcutil/v2.1.0 -module load netcdf/4.3.0 -module load hdf5/1.8.14 - -export NETCDF_INCLUDE="-I${NETCDF}/include" -export NETCDF_LDFLAGS_F="-L${NETCDF}/lib -lnetcdf -lnetcdff -L${HDF5}/lib -lhdf5 -lhdf5_fortran" - -set FCMP ifort diff --git a/modulefiles/fv3gfs/global_cycle.theia b/modulefiles/fv3gfs/global_cycle.hera similarity index 55% rename from modulefiles/fv3gfs/global_cycle.theia rename to modulefiles/fv3gfs/global_cycle.hera index 2f0f82cec..25ccb9841 100644 --- a/modulefiles/fv3gfs/global_cycle.theia +++ b/modulefiles/fv3gfs/global_cycle.hera @@ -1,20 +1,21 @@ #%Module##################################################### -## global_cycle component - theia +## global_cycle component - hera ############################################################# # Loading Intel Compiler Suite -module load intel/16.1.150 -module load impi +module load intel/18.0.5.274 +module load impi/2018.0.4 -# Loding nceplibs modules -module use -a /scratch3/NCEPDEV/nwprod/lib/modulefiles -module load w3nco/v2.0.6 -module load sp/v2.0.2 -module load bacio/v2.0.1 -module load ip/v3.0.0 -module load netcdf/4.3.0 -module load hdf5/1.8.14 +module load netcdf/4.6.1 +module load hdf5/1.10.4 export NETCDF_INCLUDE="-I${NETCDF}/include" export NETCDF_LDFLAGS_F="-L${NETCDF}/lib -lnetcdf -lnetcdff -L${HDF5}/lib -lhdf5 -lhdf5_fortran" +# Loding nceplibs modules +module use -a $MOD_PATH +module load w3nco/2.0.6 +module load sp/2.0.2 +module load bacio/2.0.2 +module load ip/3.0.1 + export FCMP=mpiifort diff --git a/sorc/build_chgres.sh b/sorc/build_chgres.sh index 210ebae36..7dcb4291f 100755 --- a/sorc/build_chgres.sh +++ b/sorc/build_chgres.sh @@ -6,7 +6,7 @@ cwd=`pwd` USE_PREINST_LIBS=${USE_PREINST_LIBS:-"true"} if [ $USE_PREINST_LIBS = true ]; then - export MOD_PATH=/scratch3/NCEPDEV/nwprod/lib/modulefiles + export MOD_PATH source ../modulefiles/fv3gfs/global_chgres.$target > /dev/null 2>&1 else export MOD_PATH=${cwd}/lib/modulefiles @@ -32,8 +32,8 @@ export FCMP95=$FCMP export FFLAGSM="-i4 -O3 -r8 -convert big_endian -fp-model precise" export RECURS= -export LDFLAGSM=${LDFLAGSM:-"-openmp -auto"} -export OMPFLAGM=${OMPFLAGM:-"-openmp -auto"} +export LDFLAGSM=${LDFLAGSM:-"-qopenmp -auto"} +export OMPFLAGM=${OMPFLAGM:-"-qopenmp -auto"} export INCS="-I${SIGIO_INC4} -I${SFCIO_INC4} -I${LANDSFCUTIL_INCd} \ -I${NEMSIO_INC} -I${NEMSIOGFS_INC} -I${GFSIO_INC4} -I${IP_INCd} ${NETCDF_INCLUDE}" diff --git a/sorc/build_chgres_cube.sh b/sorc/build_chgres_cube.sh index 412c337b7..752429b47 100755 --- a/sorc/build_chgres_cube.sh +++ b/sorc/build_chgres_cube.sh @@ -6,7 +6,7 @@ cwd=`pwd` USE_PREINST_LIBS=${USE_PREINST_LIBS:-"true"} if [ $USE_PREINST_LIBS = true ]; then - export MOD_PATH=/scratch3/NCEPDEV/nwprod/lib/modulefiles + export MOD_PATH source ../modulefiles/chgres_cube.$target > /dev/null 2>&1 else export MOD_PATH=${cwd}/lib/modulefiles diff --git a/sorc/build_cycle.sh b/sorc/build_cycle.sh index b721bf840..75e80cc29 100755 --- a/sorc/build_cycle.sh +++ b/sorc/build_cycle.sh @@ -6,7 +6,7 @@ cwd=`pwd` USE_PREINST_LIBS=${USE_PREINST_LIBS:-"true"} if [ $USE_PREINST_LIBS = true ]; then - export MOD_PATH=/scratch3/NCEPDEV/nwprod/lib/modulefiles + export MOD_PATH source ../modulefiles/fv3gfs/global_cycle.$target > /dev/null 2>&1 else export MOD_PATH=${cwd}/lib/modulefiles From 5a0e912b3d4d3d524b799fe9f9f3c839a79e3cbf Mon Sep 17 00:00:00 2001 From: George Gayno Date: Wed, 4 Sep 2019 20:47:01 +0000 Subject: [PATCH 06/34] feature/hera_port This commit references issue #1. Update to intel v15 for chgres on phase 1/2. This is required because other machines use newer versions that require the -qopenmp flag. v14 only recognized -openmp. --- modulefiles/fv3gfs/global_chgres.wcoss | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modulefiles/fv3gfs/global_chgres.wcoss b/modulefiles/fv3gfs/global_chgres.wcoss index 80e60faaf..1d30465a5 100644 --- a/modulefiles/fv3gfs/global_chgres.wcoss +++ b/modulefiles/fv3gfs/global_chgres.wcoss @@ -3,7 +3,7 @@ ############################################################# # Loading Intel Compiler Suite -module load ics/14.0.1 +module load ics/15.0.6 # Loading nceplibs modules module load sigio/v2.1.0 From ec327b8e0b7c82543deba4391859fabc5cb03009 Mon Sep 17 00:00:00 2001 From: George Gayno Date: Thu, 5 Sep 2019 14:57:25 +0000 Subject: [PATCH 07/34] feature/hera_port This commit references issue #1. Update "link_fixdirs.sh" for Hera. Remove references to Theia. --- sorc/link_fixdirs.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/sorc/link_fixdirs.sh b/sorc/link_fixdirs.sh index d2b4fdc8c..eef200fcd 100755 --- a/sorc/link_fixdirs.sh +++ b/sorc/link_fixdirs.sh @@ -9,20 +9,20 @@ machine=${2} if [ $# -lt 2 ]; then set +x echo '***ERROR*** must specify two arguements: (1) RUN_ENVIR, (2) machine' - echo ' Syntax: link_fv3gfs.sh ( nco | emc ) ( cray | dell | theia | jet )' + echo ' Syntax: link_fv3gfs.sh ( nco | emc ) ( cray | dell | hera | jet )' exit 1 fi if [ $RUN_ENVIR != emc -a $RUN_ENVIR != nco ]; then set +x echo '***ERROR*** unsupported run environment' - echo 'Syntax: link_fv3gfs.sh ( nco | emc ) ( cray | dell | theia | jet )' + echo 'Syntax: link_fv3gfs.sh ( nco | emc ) ( cray | dell | hera | jet )' exit 1 fi -if [ $machine != cray -a $machine != theia -a $machine != dell -a $machine != jet ]; then +if [ $machine != cray -a $machine != hera -a $machine != dell -a $machine != jet ]; then set +x echo '***ERROR*** unsupported machine' - echo 'Syntax: link_fv3gfs.sh ( nco | emc ) ( cray | dell | theia | jet )' + echo 'Syntax: link_fv3gfs.sh ( nco | emc ) ( cray | dell | hera | jet )' exit 1 fi @@ -39,8 +39,8 @@ if [ $machine == "cray" ]; then FIX_DIR="/gpfs/hps3/emc/global/noscrub/emc.glopara/git/fv3gfs/fix" elif [ $machine = "dell" ]; then FIX_DIR="/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/fv3gfs/fix" -elif [ $machine = "theia" ]; then - FIX_DIR="/scratch4/NCEPDEV/global/save/glopara/git/fv3gfs/fix" +elif [ $machine = "hera" ]; then + FIX_DIR="/scratch1/NCEPDEV/global/glopara/fix" elif [ $machine = "jet" ]; then FIX_DIR="/lfs3/projects/hfv3gfs/glopara/git/fv3gfs/fix" fi @@ -52,8 +52,8 @@ done if [ $machine == "cray" ] || [ $machine = "dell" ]; then $LINK /gpfs/dell2/emc/modeling/noscrub/George.Gayno/landutil.git/climo_fields_netcdf ./fix_sfc_climo -elif [ $machine = "theia" ]; then - $LINK /scratch4/NCEPDEV/da/noscrub/George.Gayno/climo_fields_netcdf ./fix_sfc_climo +elif [ $machine = "hera" ]; then + $LINK /scratch1/NCEPDEV/da/George.Gayno/ufs_utils.git/climo_fields_netcdf ./fix_sfc_climo elif [ $machine = "jet" ]; then $LINK /mnt/lfs3/projects/emcda/George.Gayno/climo_fields_netcdf ./fix_sfc_climo fi From 947145c434c5f72cc3b4afa9ff7bc9eb6c7e29cf Mon Sep 17 00:00:00 2001 From: GeorgeGayno-NOAA <52789452+GeorgeGayno-NOAA@users.noreply.github.com> Date: Fri, 6 Sep 2019 10:29:16 -0400 Subject: [PATCH 08/34] feature/nemsio_cvt This commit references #2. (#9) Remove 'nemsio_cvt' source code and adjust build script accordingly. --- sorc/build_nems_util.sh | 2 +- sorc/nemsio_cvt.fd/makefile | 12 -- sorc/nemsio_cvt.fd/nemsio_cvt.f | 300 -------------------------------- 3 files changed, 1 insertion(+), 313 deletions(-) delete mode 100755 sorc/nemsio_cvt.fd/makefile delete mode 100644 sorc/nemsio_cvt.fd/nemsio_cvt.f diff --git a/sorc/build_nems_util.sh b/sorc/build_nems_util.sh index d46f2c584..8bf8330ab 100755 --- a/sorc/build_nems_util.sh +++ b/sorc/build_nems_util.sh @@ -22,7 +22,7 @@ if [ ! -d "../exec" ]; then mkdir ../exec fi -for prog in nemsio_get.fd nemsio_cvt.fd nemsio_read.fd nemsio_chgdate.fd ;do +for prog in nemsio_get.fd nemsio_read.fd nemsio_chgdate.fd ;do cd ${cwd}/${prog} make -f makefile done diff --git a/sorc/nemsio_cvt.fd/makefile b/sorc/nemsio_cvt.fd/makefile deleted file mode 100755 index 1fb2143db..000000000 --- a/sorc/nemsio_cvt.fd/makefile +++ /dev/null @@ -1,12 +0,0 @@ -SHELL =/bin/sh -EXEC =../../exec/nemsio_cvt -FOPTS = -O -FR -I$(NEMSIO_INC) -LOPTS = $(OPTS) -LIBS = $(NEMSIO_LIB) $(BACIO_LIB4) $(W3NCO_LIBd) -OBJS = nemsio_cvt -SRCS = nemsio_cvt.f -# ************************************************************************* - all: $(OBJS) - - $(OBJS): $(SRCS) - $(FCMP) $(FOPTS) $(LOPTS) $(SRCS) -o $(EXEC) $(LIBS) diff --git a/sorc/nemsio_cvt.fd/nemsio_cvt.f b/sorc/nemsio_cvt.fd/nemsio_cvt.f deleted file mode 100644 index c3c1ad116..000000000 --- a/sorc/nemsio_cvt.fd/nemsio_cvt.f +++ /dev/null @@ -1,300 +0,0 @@ -!- - - - -- - -- - -- - -- - - -- - -- -- - -- - -- - - -- - - - -- - -- -! the program convert a nemsio file to the opposite endian -! - - - - -- - -- - -- - -- - - -- - -- -- - -- - -- - - -- - - - -- - -! -- - program main -! - use nemsio_module - implicit none -! - type(nemsio_gfile) :: gfile,gfilew -! - character(255) cin,cout - character(8) gdatatype,lcnv - logical datareal4 - real,allocatable :: tmp(:) - real(8),allocatable :: tmp8(:) - character(16) file_endian,file_endianw -!--------------------------------------------------------------------------- -!--- nemsio meta data - integer i,nrec,fieldsize,iret -!--------------------------------------------------------------------------- -! -!--- initialize - call nemsio_init(iret=iret) -! -!--- open gfile for reading - call getarg(1,cin) - call getarg(2,lcnv) - if(cin=='') then - print *,'Usage: nemsio_cvt $FILE_NAME ' - stop - endif - if(lcnv=='')lcnv='YES' -! print *,'cin=',trim(cin),' lcnv=',trim(lcnv) -! - call nemsio_open(gfile,trim(cin),'read',iret=iret) - if(iret/=0) print *,'3b:: after open read, ',trim(cin), ' iret=',iret -! -!to set file header gfilew for write - call set_gfile(gfile,lcnv,cin,cout,gfilew,gdatatype,fieldsize,nrec, & - file_endian,file_endianw) -! -!if just check file endian, not convert - if(lcnv == 'NO') then - stop - endif -! - if(gdatatype(1:4)=="bin4".or.gdatatype(1:4)=="grib") then - allocate(tmp(fieldsize)) - datareal4=.true. - else if(gdatatype(1:4)=="bin8") then - allocate(tmp8(fieldsize)) - datareal4=.false. - endif -! - do i=1,nrec - if(datareal4) then - call nemsio_readrec(gfile,i,data=tmp,iret=iret) - call nemsio_writerec(gfilew,i,data=tmp,iret=iret) - else - call nemsio_readrec(gfile,i,data=tmp8,iret=iret) - call nemsio_writerec(gfilew,i,data=tmp8,iret=iret) - endif - - enddo -! -!--- close nemsio file - call nemsio_close(gfile,iret=iret) - if ( iret .ne.0) print *,'open input file,iret=',iret - call nemsio_close(gfilew,iret=iret) - if ( iret .ne.0) print *,'open output file,iret=',iret - -!!--------------------------------------------------------------------------- - if(allocated(tmp))deallocate(tmp) - if(allocated(tmp8))deallocate(tmp8) -! -!--------------------------------------------------------------------------- -! - call nemsio_finalize() -! - print *,'File ',trim(cin),' in ',trim(file_endian), & - ' is converted to ', trim(cout),' in ',trim(file_endianw) -! -! - - - - -- - -- - -- - -- - - -- - -- -- - -- - -- - - -- - - - -- - -! -- - stop - - end program - -!############################################################################ -! - subroutine set_gfile(gfile,lcnv,cin,cout,gfilew,gdatatype,fieldsize, & - nrec,file_endian,file_endianw) -! - use nemsio_module - implicit none -! - type(nemsio_gfile),intent(in) :: gfile - character(*),intent(in) :: lcnv,cin - character(*),intent(out) :: cout - type(nemsio_gfile),intent(inout) :: gfilew - character(8),intent(out) :: gdatatype - integer,intent(out) :: fieldsize,nrec - character(*),intent(out) :: file_endian,file_endianw -! -! local vars - integer im,jm,lm,ntrac,nframe,nmetavari,nmetavarr,nmeta, & - nmetavarl,nmetavarc,nmetavarr8,nmetaaryi,nmetaaryr, & - nmetaaryl,nmetaaryc,nmetaaryr8,iret - integer,allocatable :: reclev(:),varival(:),aryilen(:),aryrlen(:), & - aryllen(:),aryclen(:),aryr8len(:),aryival(:,:) - real,allocatable :: lat(:),lon(:),dx(:),dy(:),cpi(:),ri(:), & - vcoord(:,:,:),varrval(:),aryrval(:,:) - real(8),allocatable :: varr8val(:),aryr8val(:,:) - logical extrameta - logical,allocatable :: varlval(:),arylval(:,:) - character(16),allocatable :: recname(:),reclevtyp(:),variname(:), & - varrname(:),varlname(:),varcname(:),varr8name(:), & - aryiname(:),aryrname(:),arylname(:),arycname(:), & - aryr8name(:),varcval(:),arycval(:,:) -! - gfilew=gfile -! - call nemsio_getfilehead(gfile,dimx=im,dimy=jm,dimz=lm,nrec=nrec, & - nmeta=nmeta,gdatatype=gdatatype, & - nframe=nframe,ntrac=ntrac,file_endian=file_endian, & - extrameta=extrameta,nmetavari=nmetavari,nmetavarr=nmetavarr,& - nmetavarl=nmetavarl,nmetavarc=nmetavarc,nmetavarr8=nmetavarr8,& - nmetaaryi=nmetaaryi,nmetaaryr=nmetaaryr,nmetaaryl=nmetaaryl, & - nmetaaryc=nmetaaryc,nmetaaryr8=nmetaaryr8) -! -! print *,'in set_gfile,im=',im,'jm=',jm,'lm=',lm,'nrec=',nrec, & -! 'nmeta=',nmeta,'gdatatype=',gdatatype,'file_endian=',file_endian, & -! 'ntrac=',ntrac,'extrameta=',extrameta - -! - if(trim(file_endian)=='big_endian') then - file_endianw='little_endian' - gdatatype=trim(gdatatype(1:4))//"_le" - cout=trim(cin)//"_le" - elseif(trim(file_endian)=='little_endian') then - file_endianw='big_endian' - gdatatype=trim(gdatatype(1:4))//"_be" - cout=trim(cin)//"_be" - else - print *,'wrong endian in input file: ',trim(file_endian) - endif - if(lcnv == 'NO') then - print *,'File ',trim(cin),' is in ',trim(file_endian) - return - endif -! -!set array - fieldsize=(im+2*nframe)*(jm+2*nframe) - allocate(recname(nrec),reclevtyp(nrec),reclev(nrec)) - allocate(lon(fieldsize),lat(fieldsize),dx(fieldsize),dy(fieldsize)) - allocate(cpi(ntrac+1),ri(ntrac+1)) - allocate(vcoord(lm+1,3,2)) -! - if(nmeta>=5) then - call nemsio_getfilehead(gfile,iret=iret, recname=recname, & - reclevtyp=reclevtyp, reclev=reclev) - if(iret/=0) print *,'nemsio_cnv: cant get record infor!' - endif -! - if(nmeta>=6) then - call nemsio_getfilehead(gfile,iret=iret, vcoord=vcoord) - if(iret/=0) print *,'nemsio_cnv: cant get vcoord!' - endif -! - if(nmeta>=8) then - call nemsio_getfilehead(gfile,iret=iret, lon=lon,lat=lat) - if(iret/=0) print *,'nemsio_cnv: cant get lat/lon!' - endif -! - if(nmeta>=10) then - call nemsio_getfilehead(gfile,iret=iret, dx=dx,dy=dy) - if(iret/=0) print *,'nemsio_cnv: cant get dx/dy!' - endif -! - if(nmeta>=12) then - call nemsio_getfilehead(gfile,iret=iret, cpi=cpi,ri=ri) - if(iret/=0) print *,'nemsio_cnv: cant get cpi/ri!' - endif -! -! get user defined header vars - if(extrameta) then -! - if(nmetavari>0) then - allocate(variname(nmetavari),varival(nmetavari)) - call nemsio_getfilehead(gfile,iret=iret,variname=variname, & - varival=varival) - endif -! - if(nmetavarr>0) then - allocate(varrname(nmetavarr),varrval(nmetavarr)) - call nemsio_getfilehead(gfile,iret=iret,varrname=varrname, & - varrval=varrval) - endif -! - if(nmetavarl>0) then - allocate(varlname(nmetavarl),varlval(nmetavarl)) - call nemsio_getfilehead(gfile,iret=iret,varlname=varlname, & - varlval=varlval) - endif -! - if(nmetavarc>0) then - allocate(varcname(nmetavarc),varcval(nmetavarc)) - call nemsio_getfilehead(gfile,iret=iret,varcname=varcname, & - varcval=varcval) - endif -! - if(nmetavarr8>0) then - allocate(varr8name(nmetavarr8),varr8val(nmetavarr8)) - call nemsio_getfilehead(gfile,iret=iret,varr8name=varr8name, & - varr8val=varr8val) - endif -! - if(nmetaaryi>0) then - allocate(aryiname(nmetaaryi),aryilen(nmetaaryi)) - call nemsio_getfilehead(gfile,iret=iret,aryiname=aryiname, & - aryilen=aryilen) - allocate(aryival(maxval(aryilen),nmetaaryi)) - call nemsio_getfilehead(gfile,iret=iret,aryival=aryival) - endif -! - if(nmetaaryr>0) then - allocate(aryrname(nmetaaryr),aryrlen(nmetaaryr)) - call nemsio_getfilehead(gfile,iret=iret,aryrname=aryrname, & - aryrlen=aryrlen) - allocate(aryrval(maxval(aryrlen),nmetaaryr)) - call nemsio_getfilehead(gfile,iret=iret,aryrval=aryrval) - endif -! - if(nmetaaryl>0) then - allocate(arylname(nmetaaryl),aryllen(nmetaaryl)) - call nemsio_getfilehead(gfile,iret=iret,arylname=arylname, & - aryllen=aryllen) - allocate(arylval(maxval(aryllen),nmetaaryl)) - call nemsio_getfilehead(gfile,iret=iret,arylval=arylval) - endif -! - if(nmetaaryc>0) then - allocate(arycname(nmetaaryc),aryclen(nmetaaryc)) - call nemsio_getfilehead(gfile,iret=iret,arycname=arycname, & - aryclen=aryclen) - allocate(arycval(maxval(aryclen),nmetaaryc)) - call nemsio_getfilehead(gfile,iret=iret,arycval=arycval) - endif -! - if(nmetaaryr8>0) then - allocate(aryr8name(nmetaaryr8),aryr8len(nmetaaryr8)) - call nemsio_getfilehead(gfile,iret=iret,aryr8name=aryr8name, & - aryr8len=aryr8len) - allocate(aryr8val(maxval(aryr8len),nmetaaryr8)) - call nemsio_getfilehead(gfile,iret=iret,aryr8val=aryr8val) - endif -! - endif -! - call nemsio_open(gfilew,trim(cout),'write',iret=iret, & - gdatatype=gdatatype, & - recname=recname,reclevtyp=reclevtyp,reclev=reclev, & - lat=lat,lon=lon,cpi=cpi,ri=ri,extrameta=extrameta, & - nmetavari=nmetavari,nmetavarr=nmetavarr,nmetavarl=nmetavarl,& - nmetavarc=nmetavarc,nmetavarr8=nmetavarr8, & - nmetaaryi=nmetaaryi,nmetaaryr=nmetaaryr,nmetaaryl=nmetaaryl,& - nmetaaryc=nmetaaryc,nmetaaryr8=nmetaaryr8, & - variname=variname,varival=varival, & - varrname=varrname,varrval=varrval, & - varlname=varlname,varlval=varlval, & - varcname=varcname,varcval=varcval, & - varr8name=varr8name,varr8val=varr8val, & - aryiname=aryiname,aryival=aryival, & - aryrname=aryrname,aryrval=aryrval, & - arylname=arylname,arylval=arylval, & - arycname=arycname,arycval=arycval, & - aryr8name=aryiname,aryr8val=aryr8val) - if(iret.ne.0) then - print *,'ERROR: after open, file=',trim(cout),'fiel_endian=', & - trim(file_endian) - endif -! - if(allocated(recname)) deallocate(recname,reclevtyp,reclev) - if(allocated(vcoord)) deallocate(vcoord) - if(allocated(lon)) deallocate(lon,lat) - if(allocated(dy)) deallocate(dx,dy) - if(allocated(ri)) deallocate(cpi,ri) - if(allocated(variname))deallocate(variname,varival) - if(allocated(varrname))deallocate(varrname,varrval) - if(allocated(varlname))deallocate(varlname,varlval) - if(allocated(varcname))deallocate(varcname,varcval) - if(allocated(varr8name))deallocate(varr8name,varr8val) - if(allocated(aryiname))deallocate(aryiname,aryilen,aryival) - if(allocated(aryrname))deallocate(aryrname,aryrlen,aryrval) - if(allocated(arylname))deallocate(arylname,aryllen,arylval) - if(allocated(arycname))deallocate(arycname,aryclen,arycval) - if(allocated(aryr8name))deallocate(aryr8name,aryr8len,aryr8val) -! - end subroutine set_gfile From bae8fea78a4dd92f33dc9e60ee7e68033d5f84ee Mon Sep 17 00:00:00 2001 From: George Gayno Date: Fri, 6 Sep 2019 20:40:28 +0000 Subject: [PATCH 09/34] feature/hera_port This commit references #1. Create new grid driver script for Hera. Remove Theia version. Update sfc_climo_gen build module to point to George V's copy of esmf. Other users could not compile using the other esmf module. --- driver_scripts/{driver_grid.theia.sh => driver_grid.hera.sh} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename driver_scripts/{driver_grid.theia.sh => driver_grid.hera.sh} (100%) diff --git a/driver_scripts/driver_grid.theia.sh b/driver_scripts/driver_grid.hera.sh similarity index 100% rename from driver_scripts/driver_grid.theia.sh rename to driver_scripts/driver_grid.hera.sh From e44ff25309a48efb38aa71c611ca5c8416f891e9 Mon Sep 17 00:00:00 2001 From: George Gayno Date: Fri, 6 Sep 2019 20:44:24 +0000 Subject: [PATCH 10/34] feature/hera_port This commit references #1. --- driver_scripts/driver_grid.hera.sh | 16 ++++++++-------- modulefiles/modulefile.sfc_climo_gen.hera | 5 ++++- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/driver_scripts/driver_grid.hera.sh b/driver_scripts/driver_grid.hera.sh index 88096511d..e61d08a3e 100644 --- a/driver_scripts/driver_grid.hera.sh +++ b/driver_scripts/driver_grid.hera.sh @@ -10,7 +10,7 @@ #SBATCH -t 00:30:00 #----------------------------------------------------------------------- -# Driver script to create a cubic-sphere based model grid on Theia. +# Driver script to create a cubic-sphere based model grid on Hera. # # Produces the following files (netcdf, each tile in separate file): # 1) 'mosaic' and 'grid' files containing lat/lon and other @@ -53,10 +53,10 @@ set -x . /apps/lmod/lmod/init/sh module purge -module load intel/16.1.150 -module load impi -module load hdf5/1.8.14 -module load netcdf/4.3.0 +module load intel/18.0.5.274 +module load impi/2018.0.4 +module load hdf5/1.10.4 +module load netcdf/4.6.1 module list #----------------------------------------------------------------------- @@ -90,8 +90,8 @@ fi #----------------------------------------------------------------------- export home_dir=$SLURM_SUBMIT_DIR/.. -export TMPDIR=/scratch3/NCEPDEV/stmp1/$LOGNAME/fv3_grid.$gtype -export out_dir=/scratch3/NCEPDEV/stmp1/$LOGNAME/C${res} +export TMPDIR=/scratch2/NCEPDEV/stmp1/$LOGNAME/fv3_grid.$gtype +export out_dir=/scratch2/NCEPDEV/stmp1/$LOGNAME/C${res} #----------------------------------------------------------------------- # Should not need to change anything below here. @@ -101,7 +101,7 @@ export APRUN=time export APRUN_SFC=srun export OMP_NUM_THREADS=24 export OMP_STACKSIZE=2048m -export machine=THEIA +export machine=HERA ulimit -a ulimit -s unlimited diff --git a/modulefiles/modulefile.sfc_climo_gen.hera b/modulefiles/modulefile.sfc_climo_gen.hera index 8d6faa63d..d17cb63cb 100644 --- a/modulefiles/modulefile.sfc_climo_gen.hera +++ b/modulefiles/modulefile.sfc_climo_gen.hera @@ -4,10 +4,13 @@ module load intel/18.0.5.274 module load impi/2018.0.4 -module load esmf/7.1.0r module use -a $MOD_PATH module load netcdf/4.6.1 +export NCEPLIBS=/scratch1/NCEPDEV/global/gwv/l819/lib +module use -a $NCEPLIBS/modulefiles +module load esmflocal/7_1_0r + export FCOMP=mpiifort export FFLAGS="-O3 -fp-model=precise -g -traceback -r8 -i4 -convert big_endian" From 00de9545fc9847bec4152cb561d74cd2807a06ee Mon Sep 17 00:00:00 2001 From: George Gayno Date: Mon, 9 Sep 2019 16:39:37 +0000 Subject: [PATCH 11/34] feature/hera_port This commit references #1. Point to George V's copy of the nesmio library instead of the 'official' version. The nemsio utilities did not work with the 'official' version - they return bad istat (-41) when reading. Either the official version was compiled wrong or as a newer version (v2.5) it may be incompatible with the programs. --- modulefiles/chgres_cube.hera | 3 ++- modulefiles/fv3gfs/global_chgres.hera | 6 +++++- modulefiles/module_nemsutil.hera | 6 +++++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/modulefiles/chgres_cube.hera b/modulefiles/chgres_cube.hera index 16ce8d125..2469c4a24 100644 --- a/modulefiles/chgres_cube.hera +++ b/modulefiles/chgres_cube.hera @@ -8,7 +8,7 @@ module load netcdf/4.6.1 module use -a $MOD_PATH module load w3nco/2.0.6 -module load nemsio/2.2.5 +###module load nemsio/2.2.5 module load bacio/2.0.2 module load sp/2.0.2 module load sfcio/1.1.0 @@ -17,6 +17,7 @@ module load sigio/2.1.0 export NCEPLIBS=/scratch1/NCEPDEV/global/gwv/l819/lib module use -a $NCEPLIBS/modulefiles module load esmflocal/8_0_44b +module load nemsio-intel-sandybridge/2.2.2 export FCOMP=mpiifort export FFLAGS="-O3 -fp-model precise -g -traceback -r8 -i4 -qopenmp -convert big_endian -assume byterecl" diff --git a/modulefiles/fv3gfs/global_chgres.hera b/modulefiles/fv3gfs/global_chgres.hera index 256d094ba..88bfd1af9 100644 --- a/modulefiles/fv3gfs/global_chgres.hera +++ b/modulefiles/fv3gfs/global_chgres.hera @@ -16,7 +16,7 @@ module load w3nco/2.0.6 module load w3emc/2.3.0 module load sp/2.0.2 module load bacio/2.0.2 -module load nemsio/2.2.5 +###module load nemsio/2.2.5 module load nemsiogfs/2.2.0 module load ip/3.0.1 module load sfcio/1.1.0 @@ -26,4 +26,8 @@ module load landsfcutil/2.1.0 export NETCDF_INCLUDE="-I${NETCDF}/include" export NETCDF_LDFLAGS_F="-L${NETCDF}/lib -lnetcdf -lnetcdff -L${HDF5}/lib -lhdf5 -lhdf5_fortran" +export NCEPLIBS=/scratch1/NCEPDEV/global/gwv/l819/lib +module use -a $NCEPLIBS/modulefiles +module load nemsio-intel-sandybridge/2.2.2 + set FCMP ifort diff --git a/modulefiles/module_nemsutil.hera b/modulefiles/module_nemsutil.hera index dc3b2a99f..9db5ff74f 100644 --- a/modulefiles/module_nemsutil.hera +++ b/modulefiles/module_nemsutil.hera @@ -10,6 +10,10 @@ module load impi/2018.0.4 module use -a $MOD_PATH module load w3nco/2.0.6 module load bacio/2.0.2 -module load nemsio/2.2.5 +#module load nemsio/2.2.5 + +export NCEPLIBS=/scratch1/NCEPDEV/global/gwv/l819/lib +module use -a $NCEPLIBS/modulefiles +module load nemsio-intel-sandybridge/2.2.2 export FCMP=ifort From 3241cfab03c9ac6094490feb61711834733038b9 Mon Sep 17 00:00:00 2001 From: George Gayno Date: Mon, 9 Sep 2019 20:07:15 +0000 Subject: [PATCH 12/34] feature/hera_port This commit references #1. Dexin compiled nemsio v2.2.3 on Hera and it was successfully tested with the nemsio utilities. Switch to v2.2.3 for all programs. --- modulefiles/chgres_cube.hera | 3 +-- modulefiles/fv3gfs/global_chgres.hera | 6 +----- modulefiles/fv3gfs/nst_tf_chg.hera | 2 +- modulefiles/module_nemsutil.hera | 6 +----- 4 files changed, 4 insertions(+), 13 deletions(-) diff --git a/modulefiles/chgres_cube.hera b/modulefiles/chgres_cube.hera index 2469c4a24..edd787d32 100644 --- a/modulefiles/chgres_cube.hera +++ b/modulefiles/chgres_cube.hera @@ -8,7 +8,7 @@ module load netcdf/4.6.1 module use -a $MOD_PATH module load w3nco/2.0.6 -###module load nemsio/2.2.5 +module load nemsio/2.2.3 module load bacio/2.0.2 module load sp/2.0.2 module load sfcio/1.1.0 @@ -17,7 +17,6 @@ module load sigio/2.1.0 export NCEPLIBS=/scratch1/NCEPDEV/global/gwv/l819/lib module use -a $NCEPLIBS/modulefiles module load esmflocal/8_0_44b -module load nemsio-intel-sandybridge/2.2.2 export FCOMP=mpiifort export FFLAGS="-O3 -fp-model precise -g -traceback -r8 -i4 -qopenmp -convert big_endian -assume byterecl" diff --git a/modulefiles/fv3gfs/global_chgres.hera b/modulefiles/fv3gfs/global_chgres.hera index 88bfd1af9..90a618ad1 100644 --- a/modulefiles/fv3gfs/global_chgres.hera +++ b/modulefiles/fv3gfs/global_chgres.hera @@ -16,7 +16,7 @@ module load w3nco/2.0.6 module load w3emc/2.3.0 module load sp/2.0.2 module load bacio/2.0.2 -###module load nemsio/2.2.5 +module load nemsio/2.2.3 module load nemsiogfs/2.2.0 module load ip/3.0.1 module load sfcio/1.1.0 @@ -26,8 +26,4 @@ module load landsfcutil/2.1.0 export NETCDF_INCLUDE="-I${NETCDF}/include" export NETCDF_LDFLAGS_F="-L${NETCDF}/lib -lnetcdf -lnetcdff -L${HDF5}/lib -lhdf5 -lhdf5_fortran" -export NCEPLIBS=/scratch1/NCEPDEV/global/gwv/l819/lib -module use -a $NCEPLIBS/modulefiles -module load nemsio-intel-sandybridge/2.2.2 - set FCMP ifort diff --git a/modulefiles/fv3gfs/nst_tf_chg.hera b/modulefiles/fv3gfs/nst_tf_chg.hera index db82e1884..2bbd3a802 100644 --- a/modulefiles/fv3gfs/nst_tf_chg.hera +++ b/modulefiles/fv3gfs/nst_tf_chg.hera @@ -9,7 +9,7 @@ module load intel/18.0.5.274 module use -a $MOD_PATH module load w3nco/2.0.6 module load bacio/2.0.2 -module load nemsio/2.2.5 +module load nemsio/2.2.3 module load netcdf/4.6.1 module load hdf5/1.10.4 export NETCDF_INCLUDE="-I${NETCDF}/include" diff --git a/modulefiles/module_nemsutil.hera b/modulefiles/module_nemsutil.hera index 9db5ff74f..4d1bfd5c4 100644 --- a/modulefiles/module_nemsutil.hera +++ b/modulefiles/module_nemsutil.hera @@ -10,10 +10,6 @@ module load impi/2018.0.4 module use -a $MOD_PATH module load w3nco/2.0.6 module load bacio/2.0.2 -#module load nemsio/2.2.5 - -export NCEPLIBS=/scratch1/NCEPDEV/global/gwv/l819/lib -module use -a $NCEPLIBS/modulefiles -module load nemsio-intel-sandybridge/2.2.2 +module load nemsio/2.2.3 export FCMP=ifort From 2044db3e166d504a79f62dfcf139c71e1a007c84 Mon Sep 17 00:00:00 2001 From: George Gayno Date: Mon, 9 Sep 2019 20:09:38 +0000 Subject: [PATCH 13/34] feature/hera_port This commit references #1. Baseline the batch submission wrappers per request of global workflow group. These are used by the chgres script. Previously, they resided in the glopara home directories on each machine. --- util/sub_slurm | 144 ++++++++++++++++++++++++ util/sub_wcoss_c | 277 +++++++++++++++++++++++++++++++++++++++++++++++ util/sub_wcoss_d | 247 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 668 insertions(+) create mode 100755 util/sub_slurm create mode 100755 util/sub_wcoss_c create mode 100755 util/sub_wcoss_d diff --git a/util/sub_slurm b/util/sub_slurm new file mode 100755 index 000000000..97e70c9e2 --- /dev/null +++ b/util/sub_slurm @@ -0,0 +1,144 @@ +#!/bin/ksh +set -x +usage="\ +Usage: $0 [options] executable [args] + where the options are: + -a account account (default: none) + -b binding run smt binding or not (default:NO) + -d dirin initial directory (default: cwd) + -e envars copy comma-separated environment variables + -g group group name + -i append standard input to command file + -j jobname specify jobname (default: executable basename) + -m machine machine on which to run (default: current) + -n write command file to stdout rather than submitting it + -o output specify output file (default: jobname.out) + -p procs[/nodes[/ppreq] + number of MPI tasks and optional nodes or Bblocking and + ppreq option (N or S) (defaults: serial, Bunlimited, S) + -q queue[/qpreq] queue name and optional requirement, e.g. dev/P + (defaults: 1 if serial or dev if parallel and none) + (queue 3 or 4 is dev or prod with twice tasks over ip) + (options: P=parallel, B=bigmem, b=batch) + -r rmem[/rcpu] resources memory and cpus/task (default: '1024 mb', 1) + -t timew wall time limit in [[hh:]mm:]ss format (default: 900) + -u userid userid to run under (default: self) + -v verbose mode + -w when when to run, in yyyymmddhh[mm], +hh[mm], thh[mm], or + Thh[mm] (full, incremental, today or tomorrow) format + (default: now) +Function: This command submits a job to the batch queue." +subcmd="$*" +stdin=NO +nosub=NO +account="" +binding="NO" +dirin="" +envars="para" +group="" +jobname="" +machine="" +output="" +procs=0 +nodes="" +ppreq="N" +queue="" +qpreq="" +rmem="1200" +rcpu="1" +timew="900" +userid="" +verbose=NO +when="" +while getopts a:b:d:e:g:ij:m:no:p:q:r:t:u:vw: opt;do + case $opt in + a) account="$OPTARG";; + b) binding="$OPTARG";; + d) dirin="$OPTARG";; + e) envars="$OPTARG";; + g) group="$OPTARG";; + i) stdin=YES;; + j) jobname=$OPTARG;; + m) machine="$OPTARG";; + n) nosub=YES;; + o) output=$OPTARG;; + p) procs=$(echo $OPTARG/|cut -d/ -f1);nodes=$(echo $OPTARG/|cut -d/ -f2);ppreq=$(echo $OPTARG/|cut -d/ -f3);; + q) queue=$(echo $OPTARG/|cut -d/ -f1);qpreq=$(echo $OPTARG/|cut -d/ -f2);; + r) rmem=$(echo $OPTARG/|cut -d/ -f1);rcpu=$(echo $OPTARG/|cut -d/ -f2);; + t) timew=$OPTARG;; + u) userid=$OPTARG;; + v) verbose=YES;; + w) when=$OPTARG;; + \?) echo $0: invalid option >&2;echo "$usage" >&2;exit 1;; + esac +done +shift $(($OPTIND-1)) +if [[ $# -eq 0 ]];then + echo $0: missing executable name >&2;echo "$usage" >&2;exit 1 +fi +exec=$1 +if [[ ! -s $exec ]]&&which $exec >/dev/null 2>&1;then + exec=$(which $exec) +fi +shift +args="$*" +bn=$(basename $exec) +jobname=${jobname:-$bn} +output=${output:-$jobname.out} +myuser=$LOGNAME +myhost=$(hostname) + +DATA=${DATA:-/scratch4/NCEPDEV/stmp4/$LOGNAME/sub} +mkdir -p $DATA + +partition=${partition:-service} +queue=${queue:-batch} +timew=${timew:-01:20:00} +task_node=${procs:-12} +ntasks=$((nodes*task_node)) +envars=$envars + +#export TZ=GMT +export TZ="America/New_York" +cfile=$DATA/sub$$ +> $cfile +echo "#!/bin/ksh " >> $cfile +echo "#SBATCH -A $account" >> $cfile +echo "#SBATCH -o $output" >> $cfile +echo "#SBATCH -e $output" >> $cfile +echo "#SBATCH -J $jobname" >> $cfile +echo "#SBATCH -q $queue" >> $cfile +echo "#SBATCH -p $partition" >> $cfile +##echo "#SBATCH -v $envars" >> $cfile +echo "#SBATCH --nodes=$nodes --ntasks=$ntasks" >> $cfile +echo "#SBATCH -t $timew" >> $cfile +echo "/bin/ksh --login -x $exec $args" >> $cfile + +if [[ $stdin = YES ]];then + cat +fi >>$cfile +if [[ $nosub = YES ]];then + cat $cfile + exit +elif [[ $verbose = YES ]];then + set -x + cat $cfile +fi + +sbatch=${sbatch:-/apps/slurm/default/bin/sbatch} + +ofile=$DATA/subout$$ +>$ofile +chmod 777 $ofile +$sbatch $cfile >$ofile +rc=$? +cat $ofile +if [[ -w $SUBLOG ]];then + jobn=$(grep -i submitted $ofile|head -n1|cut -d\" -f2) +# date -u +"%Y%m%d%H%M%S : $subcmd : $jobn" >>$SUBLOG + date +"%Y%m%d%H%M%S : $subcmd : $jobn" >>$SUBLOG +fi +#exit +#rm $cfile $ofile +[[ $MKDATA = YES ]] && rmdir $DATA +exit $rc diff --git a/util/sub_wcoss_c b/util/sub_wcoss_c new file mode 100755 index 000000000..7bfaeeab7 --- /dev/null +++ b/util/sub_wcoss_c @@ -0,0 +1,277 @@ +#!/bin/ksh +set -x +# +# May 28, 2013 - Shrinivas Moorthi :now updated for lsf9.1.1 - should handle coupled case also +# +usage="\ +Usage: $0 [options] executable [args] + where the options are: + -a account account (default: none) + -b binding run smt binding or not (default:NO) + -d dirin initial directory (default: cwd) + -e envars copy comma-separated environment variables + -g group group name + -i append standard input to command file + -j jobname specify jobname (default: executable basename) +# -m machine machine on which to run (default: current) + -m mpiver mpi version (poe or intelmpi) (default: poe) + -n write command file to stdout rather than submitting it + -o output specify output file (default: jobname.out) + -p procs[/nodes[/ppreq] + number of MPI tasks and optional nodes or Bblocking and + ppreq option (N or S) (defaults: serial, Bunlimited, S) + -q queue[/qpreq] queue name and optional requirement, e.g. dev/P + (defaults: 1 if serial or dev if parallel and none) + (queue 3 or 4 is dev or prod with twice tasks over ip) + (options: P=parallel, B=bigmem, b=batch) + -r rmem[/rcpu[/pe_node] resources memory and cpus/task and cores per node (default: '1024 mb', 1, and 16) + -t timew wall time limit in [[hh:]mm:]ss format (default: 900) + -u userid userid to run under (default: self) + -v verbose mode + -w when when to run, in yyyymmddhh[mm], +hh[mm], thh[mm], or + Thh[mm] (full, incremental, today or tomorrow) format + (default: now) + + You can also export variables and + \"INHERIT_ENV\" (default:-\"YES\") - Set this variable to \"NO\" and + export it if you do not want the next job to inherit current job + environment. + + Other environmental variables which can be exported from outside are: + + \"KMP_STACKSIZE\" (default:-\"1024m\") + \"MP_EUIDEVELOP\" (default:-\"NULL\") + \"F_UFMTENDIAN\" (default:-\"NULL\") + \"MPICH_ALLTOALL_THROTTLE\" (default:-\"NULL\") + \"MP_SINGLE_THREAD\" (default:-\"NULL\") + \"MP_EAGER_LIMIT\" (default:-\"NULL\") + \"MP_USE_BULK_XFER\" (default:-\"NULL\") + \"MP_COLLECTIVE_OFFLOAD\" (default:-\"NULL\") + \"MP_SHARED_MEMORY\" (default:-\"NULL\") + \"MP_MPILIB\" (default:-\"NULL\") + \"MP_LABELIO\" (default:-\"NULL\") + \"MP_STDOUTMODE\" (default:-\"NULL\") + \"DATA\" (default:-\"/stmp/$LOGNAME/sub\" + - deleted at the end if created) + +Function: This command submits a job to the batch queue." +subcmd="$*" +stdin=NO +nosub=NO +account="" +binding="NO" +dirin="" +envars="" +group="" +jobname="" +#machine="" +mpiver="" +output="" +procs=0 +nodes="" +ppreq="NONE" +queue="" +qpreq="" +rmem="1024" +rcpu="1" +pe_node=${pe_node:-16} +timew="900" +userid="" +verbose=NO +when="" +while getopts a:b:d:e:g:ij:m:no:p:q:r:t:u:vw: opt;do + case $opt in + a) account="$OPTARG";; + b) binding="$OPTARG";; + d) dirin="$OPTARG";; + e) envars="$OPTARG";; + g) group="$OPTARG";; + i) stdin=YES;; + j) jobname=$OPTARG;; +# m) machine="$OPTARG";; + m) mpiver="$OPTARG";; + n) nosub=YES;; + o) output=$OPTARG;; + p) procs=$(echo $OPTARG/|cut -d/ -f1);nodes=$(echo $OPTARG/|cut -d/ -f2);ppreq=$(echo $OPTARG/|cut -d/ -f3);; + q) queue=$(echo $OPTARG/|cut -d/ -f1);qpreq=$(echo $OPTARG/|cut -d/ -f2);; + r) rmem=$(echo $OPTARG/|cut -d/ -f1);rcpu=$(echo $OPTARG/|cut -d/ -f2);pe_node=$(echo $OPTARG/|cut -d/ -f3);; + t) timew=$OPTARG;; + u) userid=$OPTARG;; + v) verbose=YES;; + w) when=$OPTARG;; + \?) echo $0: invalid option >&2;echo "$usage" >&2;exit 1;; + esac +done +shift $(($OPTIND-1)) +if [[ $# -eq 0 ]];then + echo $0: missing executable name >&2;echo "$usage" >&2;exit 1 +fi +exec=$1 +if [[ ! -s $exec ]]&&which $exec >/dev/null 2>&1;then + exec=$(which $exec) +fi +shift +args="$*" +bn=$(basename $exec) +jobname=${jobname:-$bn} +#machine=${machine:-""} +output=${output:-$jobname.out} +myuser=$LOGNAME +myhost=$(hostname) +#mpiver=${mpiver:-poe} +envars=$envars + +#DATA=/lustre/fs/scratch/$LOGNAME/stmp +NDATE=${NDATE:-/gpfs/hps/nco/ops/nwprod/prod_util.v1.0.5/exec/ndate} +pext=${pext:-""} +#PTMP=${PREPTMP:-""}/ptmp${pext:-""} +PTMP=${PTMP:-/gpfs/hps/ptmp} +DATA=${DATA:-$PTMP/$LOGNAME/sub} +if [ -s $DATA ] ; then + MKDATA=NO +else + mkdir -p $DATA + MKDATA=YES +fi +dirin=${dirin:-$(pwd)} + +queue=${queue:-dev} +timew=${timew:-01:20} +timew=$(echo $timew |cut -d: -f1):$(echo $timew |cut -d: -f2) +threads=${rcpu:-1} +nthreads=$threads + + +max_core=${max_core:-24} +task_node=${pe_node:-${task_node:-$max_core}} + +export INHERIT_ENV=${INHERIT_ENV:-YES} +if [ $nodes -eq 1 ] ; then + task_node=$procs +fi +tot_size=$procs +max_tasks=$((max_core*nodes)) +#tot_size=$procs +if [ $((task_node*threads)) -gt $max_core ]; then + core_typ=cpu + nthreads=$threads + threads=$((2*max_core/task_node)) + echo "Hyper-threading is used - setting core_typ=$corei_typ" +fi +export core_typ=${core_typ:-core} + +export OMP_STACKSIZE=${OMP_STACKSIZE:-1024m} +export KMP_AFFINITY=${KMP_AFFINITY:-disabled} + +export TZ=GMT +cfile=$DATA/sub$$ +> $cfile + +if [ $INHERIT_ENV = YES ] ; then + echo "#!/bin/ksh" >> $cfile +else + echo "#!/bin/sh --login" >> $cfile + echo "#BSUB -L /bin/sh" >> $cfile +fi +echo "#BSUB -P $account" >> $cfile +echo "#BSUB -e $output" >> $cfile +echo "#BSUB -o $output" >> $cfile +echo "#BSUB -J $jobname" >> $cfile +#echo "#BSUB -network type=sn_all:mode=US" >> $cfile +echo "#BSUB -q $queue" >> $cfile +echo "#BSUB -W $timew" >> $cfile +##echo "#BSUB -cwd $dirin" >> $cfile +#echo "#BSUB -n $tot_size" >> $cfile +if [ $queue = dev_transfer ] ; then + echo "#BSUB -R rusage[mem=$rmem]" >> $cfile +fi +if [ $queue != dev_transfer ] ; then + echo "#BSUB -M $rmem" >> $cfile + echo "#BSUB -extsched 'CRAYLINUX[]' -R '1*{select[craylinux && !vnode]} + $max_tasks*{select[craylinux && vnode]span[ptile=24] cu[type=cabinet]}'" >> $cfile +fi + +if [[ -n $when ]];then + whena=$when + if [[ $when = +* ]];then + hr=$(echo $when|cut -c2-3) + mn=$(echo $when|cut -c4-5) + [[ -n $mn ]] || mn=00 + now=$(date -u +"%Y%m%d%H%M") + ((mn+=$(echo $now|cut -c11-12))) + [[ $mn -ge 60 ]] && ((hr+=1)) && ((mn-=60)) + [[ $mn -lt 10 ]] && mn=0$mn + whena=$($NDATE +$hr $(echo $now|cut -c1-10))$mn + elif [[ $when = t* ]];then + hr=$(echo $when|cut -c2-3) + mn=$(echo $when|cut -c4-5) + [[ -n $mn ]] || mn=00 + now=$(date -u +"%Y%m%d") + whena=$now$hr$mn + elif [[ $when = T* ]];then + hr=$(echo $when|cut -c2-3) + mn=$(echo $when|cut -c4-5) + [[ -n $mn ]] || mn=00 + now=$(date -u +"%Y%m%d%H") + whena=$($NDATE +24 $now|cut -c1-8)$hr$mn + fi + yr=$(echo $whena|cut -c1-4) + mo=$(echo $whena|cut -c5-6) + dy=$(echo $whena|cut -c7-8) + hr=$(echo $whena|cut -c9-10) + mn=$(echo $whena|cut -c11-12) + [[ -n $mn ]] || mn=00 + echo "#BSUB -b $yr:$mo:$dy:$hr:$mn" >> $cfile +fi + +#echo "source ~${LOGNAME}/.profile" >> $cfile +#echo "ulimit -s unlimited" >> $cfile +#if [ ${MP_EUIDEVICE:-NULL} = sn_all ] ; then +#echo "#BSUB -network \"type=sn_all:mode=US\" " >> $cfile +#fi +#if [ ${MP_EULIB:-NULL} != NULL ] ; then +#echo "export MP_EUILIB=$MP_EUILIB" >> $cfile +#fi + +if [ ${KMP_AFFINITY:-NULL} != NULL ] ; then + echo "export KMP_AFFINITY=$KMP_AFFINITY" >> $cfile +fi + +echo "export OMP_STACKSIZE=$OMP_STACKSIZE" >> $cfile + +if [ ${MP_LABELIO:-NULL} != NULL ] ; then + echo "export MP_LABELIO=$MP_LABELIO" >> $cfile +fi +if [ ${MP_STDOUTMODE:-NULL} != NULL ] ; then + echo "export MP_STDOUTMODE=$MP_STDOUTMODE " >> $cfile +fi +for var in $(eval echo $envars | tr , ' ') ; do + echo "export $var" >> $cfile +done +echo "$exec" >> $cfile +echo "export OMP_NUM_THREADS=$nthreads" >> $cfile + +if [[ $stdin = YES ]];then + cat +fi >>$cfile +if [[ $nosub = YES ]];then + cat $cfile + exit +elif [[ $verbose = YES ]];then + set -x + cat $cfile +fi +bsub=${bsub:-$LSF_BINDIR/bsub} + +ofile=$DATA/subout$$ +>$ofile +chmod 777 $ofile +$bsub < $cfile +rc=$? +cat $ofile +if [[ -w $SUBLOG ]];then + jobn=$(grep -i submitted $ofile|head -n1|cut -d\" -f2) + date +"%Y%m%d%H%M%S : $subcmd : $jobn" >>$SUBLOG +fi +#rm $cfile $ofile +#[[ $MKDATA = YES ]] && rmdir $DATA +exit $rc diff --git a/util/sub_wcoss_d b/util/sub_wcoss_d new file mode 100755 index 000000000..77f0e4da8 --- /dev/null +++ b/util/sub_wcoss_d @@ -0,0 +1,247 @@ +#!/bin/ksh +set -x +# +# May 28, 2013 - Shrinivas Moorthi :now updated for lsf9.1.1 - should handle coupled case also +# +usage="\ +Usage: $0 [options] executable [args] + where the options are: + -a account account (default: none) + -b binding run smt binding or not (default:NO) + -d dirin initial directory (default: cwd) + -e envars copy comma-separated environment variables + -g group group name + -i append standard input to command file + -j jobname specify jobname (default: executable basename) +# -m machine machine on which to run (default: current) + -m mpiver mpi version (poe or intelmpi) (default: poe) + -n write command file to stdout rather than submitting it + -o output specify output file (default: jobname.out) + -p procs[/nodes[/ppreq] + number of MPI tasks and optional nodes or Bblocking and + ppreq option (N or S) (defaults: serial, Bunlimited, S) + -q queue[/qpreq] queue name and optional requirement, e.g. dev/P + (defaults: 1 if serial or dev if parallel and none) + (queue 3 or 4 is dev or prod with twice tasks over ip) + (options: P=parallel, B=bigmem, b=batch) + -r rmem[/rcpu[/pe_node] resources memory and cpus/task and cores per node (default: '1024 mb', 1, and 16) + -t timew wall time limit in [[hh:]mm:]ss format (default: 900) + -u userid userid to run under (default: self) + -v verbose mode + -w when when to run, in yyyymmddhh[mm], +hh[mm], thh[mm], or + Thh[mm] (full, incremental, today or tomorrow) format + (default: now) + + You can also export variables and + \"INHERIT_ENV\" (default:-\"YES\") - Set this variable to \"NO\" and + export it if you do not want the next job to inherit current job + environment. + + Other environmental variables which can be exported from outside are: + + \"KMP_STACKSIZE\" (default:-\"1024m\") + \"MP_EUIDEVELOP\" (default:-\"NULL\") + \"F_UFMTENDIAN\" (default:-\"NULL\") + \"MPICH_ALLTOALL_THROTTLE\" (default:-\"NULL\") + \"MP_SINGLE_THREAD\" (default:-\"NULL\") + \"MP_EAGER_LIMIT\" (default:-\"NULL\") + \"MP_USE_BULK_XFER\" (default:-\"NULL\") + \"MP_COLLECTIVE_OFFLOAD\" (default:-\"NULL\") + \"MP_SHARED_MEMORY\" (default:-\"NULL\") + \"MP_MPILIB\" (default:-\"NULL\") + \"MP_LABELIO\" (default:-\"NULL\") + \"MP_STDOUTMODE\" (default:-\"NULL\") + \"DATA\" (default:-\"/stmp/$LOGNAME/sub\" + - deleted at the end if created) + +Function: This command submits a job to the batch queue." +subcmd="$*" +stdin=NO +nosub=NO +account="" +binding="NO" +dirin="" +envars="" +group="" +jobname="" +#machine="" +mpiver="" +output="" +procs=0 +nodes="" +ppreq="NONE" +queue="" +qpreq="" +rmem="1024" +rcpu="1" +pe_node=${pe_node:-16} +timew="900" +userid="" +verbose=NO +when="" +while getopts a:b:d:e:g:ij:m:no:p:q:r:t:u:vw: opt;do + case $opt in + a) account="$OPTARG";; + b) binding="$OPTARG";; + d) dirin="$OPTARG";; + e) envars="$OPTARG";; + g) group="$OPTARG";; + i) stdin=YES;; + j) jobname=$OPTARG;; + m) mpiver="$OPTARG";; + n) nosub=YES;; + o) output=$OPTARG;; + p) procs=$(echo $OPTARG/|cut -d/ -f1);nodes=$(echo $OPTARG/|cut -d/ -f2);ppreq=$(echo $OPTARG/|cut -d/ -f3);; + q) queue=$(echo $OPTARG/|cut -d/ -f1);qpreq=$(echo $OPTARG/|cut -d/ -f2);; + r) rmem=$(echo $OPTARG/|cut -d/ -f1);rcpu=$(echo $OPTARG/|cut -d/ -f2);pe_node=$(echo $OPTARG/|cut -d/ -f3);; + t) timew=$OPTARG;; + u) userid=$OPTARG;; + v) verbose=YES;; + w) when=$OPTARG;; + \?) echo $0: invalid option >&2;echo "$usage" >&2;exit 1;; + esac +done +shift $(($OPTIND-1)) +if [[ $# -eq 0 ]];then + echo $0: missing executable name >&2;echo "$usage" >&2;exit 1 +fi +exec=$1 +if [[ ! -s $exec ]]&&which $exec >/dev/null 2>&1;then + exec=$(which $exec) +fi +shift +args="$*" +bn=$(basename $exec) +jobname=${jobname:-$bn} +#machine=${machine:-""} +output=${output:-$jobname.out} +myuser=$LOGNAME +myhost=$(hostname) +envars=$envars + +#DATA=/lustre/fs/scratch/$LOGNAME/stmp +NDATE=${NDATE:-/gpfs/hps/nco/ops/nwprod/prod_util.v1.0.5/exec/ndate} +pext=${pext:-""} +PTMP=/gpfs/dell2/ptmp +DATA=${DATA:-$PTMP/$LOGNAME/sub} +if [ -s $DATA ] ; then + MKDATA=NO +else + mkdir -p $DATA + MKDATA=YES +fi +dirin=${dirin:-$(pwd)} + +queue=${queue:-dev} +timew=${timew:-01:20} +timew=$(echo $timew |cut -d: -f1):$(echo $timew |cut -d: -f2) +threads=${rcpu:-1} +nthreads=$threads + +max_core=${max_core:-28} +task_node=${pe_node:-${task_node:-$max_core}} + +export INHERIT_ENV=${INHERIT_ENV:-YES} +if [ $nodes -eq 1 ] ; then + task_node=$procs +fi +tot_size=$procs +max_tasks=$((max_core*nodes)) +if [ $((task_node*threads)) -gt $max_core ]; then + core_typ=cpu + nthreads=$threads + threads=$((2*max_core/task_node)) + echo "Hyper-threading is used - setting core_typ=$corei_typ" +fi +core_typ=${core_typ:-core} + +export OMP_STACKSIZE=${OMP_STACKSIZE:-1024m} +export KMP_AFFINITY=${KMP_AFFINITY:-disabled} + +export TZ=GMT +cfile=$DATA/sub$$ +> $cfile + +echo "#!/bin/bash" >> $cfile +echo "#BSUB -P $account" >> $cfile +echo "#BSUB -e $output" >> $cfile +echo "#BSUB -o $output" >> $cfile +echo "#BSUB -J $jobname" >> $cfile +echo "#BSUB -q $queue" >> $cfile +echo "#BSUB -W $timew" >> $cfile +echo "#BSUB -n $tot_size" >> $cfile +echo "#BSUB -R span[ptile=$task_node]" >> $cfile +echo "#BSUB -R affinity[core($nthreads)]" >> $cfile +echo "#BSUB -M $rmem" >> $cfile + + +if [[ -n $when ]];then + whena=$when + if [[ $when = +* ]];then + hr=$(echo $when|cut -c2-3) + mn=$(echo $when|cut -c4-5) + [[ -n $mn ]] || mn=00 + now=$(date -u +"%Y%m%d%H%M") + ((mn+=$(echo $now|cut -c11-12))) + [[ $mn -ge 60 ]] && ((hr+=1)) && ((mn-=60)) + [[ $mn -lt 10 ]] && mn=0$mn + whena=$($NDATE +$hr $(echo $now|cut -c1-10))$mn + elif [[ $when = t* ]];then + hr=$(echo $when|cut -c2-3) + mn=$(echo $when|cut -c4-5) + [[ -n $mn ]] || mn=00 + now=$(date -u +"%Y%m%d") + whena=$now$hr$mn + elif [[ $when = T* ]];then + hr=$(echo $when|cut -c2-3) + mn=$(echo $when|cut -c4-5) + [[ -n $mn ]] || mn=00 + now=$(date -u +"%Y%m%d%H") + whena=$($NDATE +24 $now|cut -c1-8)$hr$mn + fi + yr=$(echo $whena|cut -c1-4) + mo=$(echo $whena|cut -c5-6) + dy=$(echo $whena|cut -c7-8) + hr=$(echo $whena|cut -c9-10) + mn=$(echo $whena|cut -c11-12) + [[ -n $mn ]] || mn=00 + echo "#BSUB -b $yr:$mo:$dy:$hr:$mn" >> $cfile +fi + +if [ ${KMP_AFFINITY:-NULL} != NULL ] ; then + echo "export KMP_AFFINITY=$KMP_AFFINITY" >> $cfile +fi +echo "export OMP_STACKSIZE=$OMP_STACKSIZE" >> $cfile + + +for var in $(eval echo $envars | tr , ' ') ; do + echo "export $var" >> $cfile +done +echo "$exec" >> $cfile +echo "export OMP_NUM_THREADS=$nthreads" >> $cfile + +if [[ $stdin = YES ]];then + cat +fi >>$cfile +if [[ $nosub = YES ]];then + cat $cfile + exit +elif [[ $verbose = YES ]];then + set -x + cat $cfile +fi +bsub=${bsub:-$LSF_BINDIR/bsub} + +ofile=$DATA/subout$$ +>$ofile +chmod 777 $ofile +$bsub < $cfile +rc=$? +cat $ofile +if [[ -w $SUBLOG ]];then + jobn=$(grep -i submitted $ofile|head -n1|cut -d\" -f2) + date +"%Y%m%d%H%M%S : $subcmd : $jobn" >>$SUBLOG +fi +##rm $cfile $ofile +#[[ $MKDATA = YES ]] && rmdir $DATA +exit $rc From e5c508d9dc16981212b3ab2d2286dd09e53181ad Mon Sep 17 00:00:00 2001 From: George Gayno Date: Tue, 10 Sep 2019 21:36:58 +0000 Subject: [PATCH 14/34] feature/hera_port This commit references #1. Updates to run 'fv3gfs_chgres.sh' on Hera. --- modulefiles/module_base.hera | 11 ++++++++ modulefiles/module_base.theia | 19 ------------- ush/fv3gfs_chgres.sh | 53 +++++++++++++++++------------------ ush/load_fv3gfs_modules.sh | 6 ++-- 4 files changed, 39 insertions(+), 50 deletions(-) create mode 100644 modulefiles/module_base.hera delete mode 100644 modulefiles/module_base.theia diff --git a/modulefiles/module_base.hera b/modulefiles/module_base.hera new file mode 100644 index 000000000..dae1d895d --- /dev/null +++ b/modulefiles/module_base.hera @@ -0,0 +1,11 @@ +#%Module###################################################################### +## +## FV3GFS prerequisites +## + +module load intel/18.0.5.274 +module load impi/2018.0.4 +module load hpss/hpss +module load netcdf/4.6.1 +module load nco/4.7.0 +module load hdf5/1.10.4 diff --git a/modulefiles/module_base.theia b/modulefiles/module_base.theia deleted file mode 100644 index 13fad8fd5..000000000 --- a/modulefiles/module_base.theia +++ /dev/null @@ -1,19 +0,0 @@ -#%Module###################################################################### -## -## FV3GFS prerequisites -## - -module load intel/16.1.150 -module load impi/5.1.2.150 -module load wgrib2 -module load hpss -module load netcdf -module load nco -module load hdf5/1.8.14 -module load gempak/6.7.0 - -module use /scratch3/NCEPDEV/nwprod/lib/modulefiles -module load EnvVars/v1.0.2 - -module use /scratch4/NCEPDEV/nems/noscrub/emc.nemspara/soft/modulefiles -module load prod_util diff --git a/ush/fv3gfs_chgres.sh b/ush/fv3gfs_chgres.sh index a169abf1a..188af291c 100755 --- a/ush/fv3gfs_chgres.sh +++ b/ush/fv3gfs_chgres.sh @@ -36,16 +36,16 @@ #------------------------------------------------------------------------ #----WCOSS_DELL JOBCARD -#BSUB -P FV3GFS-T2O -#BSUB -o log.chgres.%J -#BSUB -e log.chgres.%J -#BSUB -J fv3_chgres -#BSUB -q dev -#BSUB -M 6000 -#BSUB -W 10:00 -#BSUB -R span[ptile=14] -#BSUB -n 14 -export machine=WCOSS_DELL_P3 +##BSUB -P FV3GFS-T2O +##BSUB -o log.chgres.%J +##BSUB -e log.chgres.%J +##BSUB -J fv3_chgres +##BSUB -q dev +##BSUB -M 6000 +##BSUB -W 10:00 +##BSUB -R span[ptile=14] +##BSUB -n 14 +#export machine=WCOSS_DELL_P3 #----WCOSS_CRAY JOBCARD ##BSUB -L /bin/sh @@ -59,22 +59,22 @@ export machine=WCOSS_DELL_P3 ##BSUB -extsched 'CRAYLINUX[]' #export machine=WCOSS_C -#---- THEIA JOBCARD +#---- Hera JOBCARD #---- Submit as: sbatch $script -##SBATCH -J fv3_chgres_driver -##SBATCH -A fv3-cpu -##SBATCH --open-mode=truncate -##SBATCH -o log.chgres -##SBATCH -e log.chgres -##SBATCH --nodes=1 -##SBATCH -q batch -##SBATCH -t 02:00:00 -#export machine=THEIA +#SBATCH -J fv3_chgres_driver +#SBATCH -A fv3-cpu +#SBATCH --open-mode=truncate +#SBATCH -o log.chgres +#SBATCH -e log.chgres +#SBATCH --nodes=1 +#SBATCH -q batch +#SBATCH -t 02:00:00 +export machine=HERA set -x export PSLOT=fv3test -export CDUMP=gdas +export CDUMP=gfs export CASE_HIGH=C768 export CASE_ENKF=C384 export CDATE=2019061000 @@ -87,9 +87,9 @@ elif [ $machine = WCOSS_DELL_P3 ]; then export HOMEgfs=$LS_SUBCWD/.. export PTMP=${PTMP:-"/gpfs/dell2/ptmp/$USER"} export zero_bias_dir=/gpfs/hps3/emc/global/noscrub/emc.glopara/ICS/bias_zero -elif [ $machine = THEIA ]; then +elif [ $machine = HERA ]; then export HOMEgfs=$SLURM_SUBMIT_DIR/.. - export PTMP=${PTMP:-"/scratch3/NCEPDEV/stmp1/$USER"} + export PTMP=${PTMP:-"/scratch2/NCEPDEV/stmp1/$USER"} export zero_bias_dir=/scratch4/NCEPDEV/global/save/glopara/git/bias_zero else echo "Please define HOMEgfs and PTMP for your machine. exit" @@ -163,14 +163,11 @@ elif [ $machine = WCOSS_DELL_P3 ]; then export ACCOUNT=FV3GFS-T2O export QUEUE=dev export QUEUE_TRANS=dev_transfer -elif [ $machine = THEIA ]; then - # source $HOMEgfs/sorc/machine-setup.sh - # module use -a /scratch3/NCEPDEV/nwprod/lib/modulefiles - # module load netcdf hdf5/1.8.14 2>>/dev/null +elif [ $machine = HERA ]; then export APRUNC=time export APRUNTF=time export OMP_NUM_THREADS_CH=24 - export SUB=/home/glopara/bin/sub_slurm + export SUB=$SLURM_SUBMIT_DIR/../util/sub_slurm export ACCOUNT=fv3-cpu export QUEUE=batch export QUEUE_TRANS=batch diff --git a/ush/load_fv3gfs_modules.sh b/ush/load_fv3gfs_modules.sh index e48f1af47..d44e2fe35 100755 --- a/ush/load_fv3gfs_modules.sh +++ b/ush/load_fv3gfs_modules.sh @@ -15,9 +15,9 @@ module use "$HOMEgfs/modulefiles" if [[ -d /lfs3 ]] ; then # We are on NOAA Jet module load module_base.jet -elif [[ -d /scratch3 ]] ; then - # We are on NOAA Theia - module load module_base.theia +elif [[ `hostname -a` == *"hera.fairmont.rdhpcs.noaa.gov"* ]] ; then + # We are on NOAA Hera + module load module_base.hera elif [[ -d /gpfs/hps && -e /etc/SuSE-release ]] ; then # We are on NOAA Luna or Surge module load module_base.wcoss_c From b1951d154f02389fb6d9f1d1a38cf637b2a6e76d Mon Sep 17 00:00:00 2001 From: George Gayno Date: Wed, 11 Sep 2019 13:13:11 +0000 Subject: [PATCH 15/34] feature/hera_port This commit references #1. Remove unused "module-setup.csh.inc" Update "ush/fv3gfs_chgres.sh" to point to new location for 'sub_wcoss' scripts. Remove some commented out code. Update "module-setup.sh.inc" for Hera. --- modulefiles/module-setup.csh.inc | 89 -------------------------------- modulefiles/module-setup.sh.inc | 4 +- ush/fv3gfs_chgres.sh | 73 +++++++++----------------- 3 files changed, 25 insertions(+), 141 deletions(-) delete mode 100644 modulefiles/module-setup.csh.inc diff --git a/modulefiles/module-setup.csh.inc b/modulefiles/module-setup.csh.inc deleted file mode 100644 index a93b21a3c..000000000 --- a/modulefiles/module-setup.csh.inc +++ /dev/null @@ -1,89 +0,0 @@ -set __ms_shell=csh - -eval "if ( -d / ) set __ms_shell=tcsh" - -if ( { test -d /lfs3 } ) then - if ( ! { module help >& /dev/null } ) then - source /apps/lmod/lmod/init/$__ms_shell - endif - module purge -else if ( { test -d /scratch3 } ) then - # We are on NOAA Theia - if ( ! { module help >& /dev/null } ) then - source /apps/lmod/lmod/init/$__ms_shell - endif - module purge -else if ( { test -d /gpfs/hps -a -e /etc/SuSE-release } ) then - # We are on NOAA Luna or Surge - if ( ! { module help >& /dev/null } ) then - source /opt/modules/default/init/$__ms_shell - endif - module purge - module purge - unset _LMFILES_ - unset LOADEDMODULES - module use /opt/modulefiles - module use /opt/cray/ari/modulefiles - module use /opt/cray/craype/default/alt-modulefiles - module use /opt/cray/alt-modulefiles - module use /gpfs/hps/nco/ops/nwprod/modulefiles - module use /gpfs/hps/nco/ops/nwprod/lib/modulefiles - module use /usrx/local/prod/modulefiles -else if ( { test -d /dcom -a -d /hwrf } ) then - # We are on NOAA Tide or Gyre - if ( ! { module help >& /dev/null } ) then - source /usrx/local/Modules/default/init/$__ms_shell - endif - module purge -else if ( { test -L /usrx && sh -c "readlink /usrx 2> /dev/null | grep dell" } ) then - # We are on WCOSS Mars or Venus - if ( ! { module help >& /dev/null } ) then - source /usrx/local/prod/lmod/lmod/init/$__ms_shell - endif - module purge -else if ( { test -d /glade } ) then - # We are on NCAR Yellowstone - if ( ! { module help >& /dev/null } ) then - source /usr/share/Modules/init/$__ms_shell - endif - module purge -else if ( { test -d /lustre -a -d /ncrc } ) then - # We are on GAEA. - if ( ! { module help >& /dev/null } ) then - # We cannot simply load the module command. The GAEA - # /etc/csh.login modifies a number of module-related variables - # before loading the module command. Without those variables, - # the module command fails. Hence we actually have to source - # /etc/csh.login here. - source /etc/csh.login - set __ms_source_etc_csh_login=yes - else - set __ms_source_etc_csh_login=no - endif - module purge - unsetenv _LMFILES_ - unsetenv _LMFILES_000 - unsetenv _LMFILES_001 - unsetenv LOADEDMODULES - module load modules - if ( { test -d /opt/cray/ari/modulefiles } ) then - module use -a /opt/cray/ari/modulefiles - endif - if ( { test -d /opt/cray/pe/ari/modulefiles } ) then - module use -a /opt/cray/pe/ari/modulefiles - endif - if ( { test -d /opt/cray/pe/craype/default/modulefiles } ) then - module use -a /opt/cray/pe/craype/default/modulefiles - endif - setenv NCEPLIBS /lustre/f1/pdata/ncep_shared/NCEPLIBS/lib - if ( { test -d /lustre/f1/pdata/ncep_shared/NCEPLIBS/lib } ) then - module use $NCEPLIBS/modulefiles - endif - if ( "$__ms_source_etc_csh_login" == yes ) then - source /etc/csh.login - unset __ms_source_etc_csh_login - endif -else - # Workaround for csh limitation. Use sh to print to stderr. - sh -c 'echo WARNING: UNKNOWN PLATFORM 1>&2' -endif diff --git a/modulefiles/module-setup.sh.inc b/modulefiles/module-setup.sh.inc index 6815b1a46..bb9c1f0a1 100644 --- a/modulefiles/module-setup.sh.inc +++ b/modulefiles/module-setup.sh.inc @@ -22,8 +22,8 @@ if [[ -d /lfs3 ]] ; then source /apps/lmod/lmod/init/$__ms_shell fi module purge -elif [[ -d /scratch3 ]] ; then - # We are on NOAA Theia +elif [[ `hostname -a` == *"hera.fairmont.rdhpcs.noaa.gov"* ]] ; then + # We are on NOAA Hera if ( ! eval module help > /dev/null 2>&1 ) ; then source /apps/lmod/lmod/init/$__ms_shell fi diff --git a/ush/fv3gfs_chgres.sh b/ush/fv3gfs_chgres.sh index 188af291c..0f795d0b5 100755 --- a/ush/fv3gfs_chgres.sh +++ b/ush/fv3gfs_chgres.sh @@ -12,7 +12,7 @@ # - Uncomment the job cards for your machine. Ensure the # 'machine' variable is uncommented. Ensure job cards for # the other machines are prefixed by "##". -# - Check the job card account. On Theia, this is +# - Check the job card account. On Hera, this is # "#SBATCH -q $account". On WCOSS, this is "#BSUB -P $account". # You may not have permissions to run under the default account. # - Make sure you have linked the 'fixed' directories using @@ -30,22 +30,22 @@ # CDATE - year/month/day/hour of your experiment. # HOMEgfs - Path to your checkout repository. # - Start the script as follows: -# Theia: "sbatch fv3gfs_chgres.sh" +# Hera: "sbatch fv3gfs_chgres.sh" # WCOSS Dell and Cray: "cat fv3gfs_chgres.sh | bsub" # #------------------------------------------------------------------------ #----WCOSS_DELL JOBCARD -##BSUB -P FV3GFS-T2O -##BSUB -o log.chgres.%J -##BSUB -e log.chgres.%J -##BSUB -J fv3_chgres -##BSUB -q dev -##BSUB -M 6000 -##BSUB -W 10:00 -##BSUB -R span[ptile=14] -##BSUB -n 14 -#export machine=WCOSS_DELL_P3 +#BSUB -P FV3GFS-T2O +#BSUB -o log.chgres.%J +#BSUB -e log.chgres.%J +#BSUB -J fv3_chgres +#BSUB -q dev +#BSUB -M 6000 +#BSUB -W 10:00 +#BSUB -R span[ptile=14] +#BSUB -n 14 +export machine=WCOSS_DELL_P3 #----WCOSS_CRAY JOBCARD ##BSUB -L /bin/sh @@ -61,15 +61,15 @@ #---- Hera JOBCARD #---- Submit as: sbatch $script -#SBATCH -J fv3_chgres_driver -#SBATCH -A fv3-cpu -#SBATCH --open-mode=truncate -#SBATCH -o log.chgres -#SBATCH -e log.chgres -#SBATCH --nodes=1 -#SBATCH -q batch -#SBATCH -t 02:00:00 -export machine=HERA +##SBATCH -J fv3_chgres_driver +##SBATCH -A fv3-cpu +##SBATCH --open-mode=truncate +##SBATCH -o log.chgres +##SBATCH -e log.chgres +##SBATCH --nodes=1 +##SBATCH -q batch +##SBATCH -t 02:00:00 +#export machine=HERA set -x @@ -132,34 +132,20 @@ status=$? [[ $status -ne 0 ]] && exit $status if [ $machine = WCOSS_C ]; then - # . $MODULESHOME/init/sh 2>>/dev/null - # module load prod_util prod_envir hpss 2>>/dev/null - # module load PrgEnv-intel 2>>/dev/null export KMP_AFFINITY=disabled export OMP_NUM_THREADS_CH=24 export APRUNC="aprun -n 1 -N 1 -j 1 -d $OMP_NUM_THREADS_CH -cc depth" export APRUNTF='aprun -q -j1 -n1 -N1 -d1 -cc depth' - export SUB=/u/emc.glopara/bin/sub_wcoss_c + export SUB=$LS_SUBCWD/../util/sub_wcoss_c export ACCOUNT=FV3GFS-T2O export QUEUE=dev export QUEUE_TRANS=dev_transfer elif [ $machine = WCOSS_DELL_P3 ]; then - # . /usrx/local/prod/lmod/lmod/init/sh 2>>/dev/null - # module purge 2>>/dev/null - # module load EnvVars/1.0.2 2>>/dev/null - # module load lsf/10.1 2>>/dev/null - # module load ips/18.0.1.163 2>>/dev/null - # module load impi/18.0.1 2>>/dev/null - # module load prod_util/1.1.0 2>>/dev/null - # module load prod_envir/1.0.2 2>>/dev/null - # module load HPSS/5.0.2.5 2>>/dev/null - # module load NetCDF/4.5.0 2>>/dev/null - # module load HDF5-serial/1.10.1 2>>/dev/null export OMP_NUM_THREADS_CH=14 export KMP_AFFINITY=disabled export APRUNC="time" export APRUNTF="time" - export SUB=/u/emc.glopara/bin/sub_wcoss_d + export SUB=$LS_SUBCWD/../util/sub_wcoss_d export ACCOUNT=FV3GFS-T2O export QUEUE=dev export QUEUE_TRANS=dev_transfer @@ -418,19 +404,6 @@ cat > read_hpss.sh <>/dev/null - #export machine=$machine - #if [ $machine = WCOSS_C ]; then - # . $MODULESHOME/init/sh 2>>/dev/null - # module load hpss 2>>/dev/null - #elif [ $machine = WCOSS_DELL_P3 ]; then - # . /usrx/local/prod/lmod/lmod/init/sh 2>>/dev/null - # module load HPSS/5.0.2.5 2>>/dev/null - #elif [ $machine = THEIA ]; then - # source $HOMEgfs/sorc/machine-setup.sh 2>>/dev/null - # module use -a /scratch3/NCEPDEV/nwprod/lib/modulefiles 2>>/dev/null - # module load hpss 2>>/dev/null - #fi - cd $INIDIR htar -xvf $HPSSPATH/$tarball_enkf_atm if [ $CDATE -le 2017072000 ] && [ $CDATE -gt 2014050100 ]; then From ac49bb5c8eb4935c5a6811633fbe71fd267bb730 Mon Sep 17 00:00:00 2001 From: Larissa Reames Date: Wed, 11 Sep 2019 21:12:52 +0000 Subject: [PATCH 16/34] chgres_cube_grib2_atmos: This commit references #66268 Fixed a new indexing error in creating the staggered grid coordinates. Also removed the call to iso2sig when data is in isobaric coordinates. 3d pressure array is now generated from pressure levels read from file. --- sorc/chgres_cube.fd/input_data.F90 | 10 +++++++++- sorc/chgres_cube.fd/model_grid.F90 | 24 ++++++++++++++++++++---- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/sorc/chgres_cube.fd/input_data.F90 b/sorc/chgres_cube.fd/input_data.F90 index 110b90cb7..ac145c540 100644 --- a/sorc/chgres_cube.fd/input_data.F90 +++ b/sorc/chgres_cube.fd/input_data.F90 @@ -2617,7 +2617,15 @@ subroutine read_input_atm_grib2_file(localpet) call error_handler("IN FieldGet", rc) end do - call iso2sig(rlevs,vcoord,lev_input,levp1_input,psptr,atm,clb,cub,5+num_tracers, iret) + !call iso2sig(rlevs,vcoord,lev_input,levp1_input,psptr,atm,clb,cub,5+num_tracers, iret) + do i = clb(1),cub(1) + do j = clb(2),cub(2) + atm(1)%var(i,j,:) = rlevs(lev_input:1:-1) + do n = 2,num_tracers+5 + atm(n)%var(i,j,:) = atm(n)%var(i,j,lev_input:1:-1) + end do + end do + end do deallocate(vcoord) else diff --git a/sorc/chgres_cube.fd/model_grid.F90 b/sorc/chgres_cube.fd/model_grid.F90 index ab75540d8..555b1a920 100644 --- a/sorc/chgres_cube.fd/model_grid.F90 +++ b/sorc/chgres_cube.fd/model_grid.F90 @@ -663,6 +663,7 @@ subroutine define_input_grid_gfs_grib2(localpet, npets) enddo deltalon = abs(longitude(2,1)-longitude(1,1)) + if(localpet==0) print*, "deltalon = ", deltalon print*,"- CALL FieldScatter FOR INPUT GRID LONGITUDE." call ESMF_FieldScatter(longitude_input_grid, longitude, rootpet=0, rc=rc) @@ -707,7 +708,9 @@ subroutine define_input_grid_gfs_grib2(localpet, npets) lat_src_ptr(i,j) = latitude(i,j) enddo enddo - + if(localpet==0) print*, "lon first = ", lon_src_ptr(1:10,1) + if(localpet==0) print*, "lat first = ", lat_src_ptr(1,1:10) + print*,"- CALL GridAddCoord FOR INPUT GRID." call ESMF_GridAddCoord(input_grid, & staggerloc=ESMF_STAGGERLOC_CORNER, rc=rc) @@ -782,7 +785,7 @@ subroutine define_input_grid_grib2(localpet, npets) lon_target(i_target,j_target) !lat_corners(ip1_input,jp1_input), & !lon_corners(ip1_input,jp1_input) - real(esmf_kind_r8) :: deltalon, dx + real(esmf_kind_r8) :: deltalon, dx, deltalat integer :: ncid,id_var, id_dim real(esmf_kind_r8), pointer :: lat_src_ptr(:,:), lon_src_ptr(:,:) character(len=10000) :: cmdline_msg, temp_msg, temp_msg2 @@ -1010,10 +1013,23 @@ subroutine define_input_grid_grib2(localpet, npets) if (trim(input_grid_type) == "latlon") then deltalon = abs(longitude_one_tile(2,1)-longitude_one_tile(1,1)) + deltalat = abs(latitude_one_tile(1,2) - latitude_one_tile(1,1)) do j = clb(2), cub(2) do i = clb(1), cub(1) - lon_src_ptr(i,j) = longitude_one_tile(i,1) - (0.5_esmf_kind_r8*deltalon) - lat_src_ptr(i,j) = 0.5_esmf_kind_r8 * (latitude_one_tile(i,j-1)+ latitude_one_tile(i,j)) + + if (i == ip1_input) then + lon_src_ptr(i,j) = longitude_one_tile(i-1,1)+(0.5_esmf_kind_r8*deltalon) + else + lon_src_ptr(i,j) = longitude_one_tile(i,1) - (0.5_esmf_kind_r8*deltalon) + endif + + if (j == jp1_input) then + lat_src_ptr(i,j) = latitude_one_tile(1,j-1)+(0.5_esmf_kind_r8*deltalat) + else + lat_src_ptr(i,j) = latitude_one_tile(1,j) - (0.5_esmf_kind_r8*deltalat) + endif + + !lat_src_ptr(i,j) = 0.5_esmf_kind_r8 * (latitude_one_tile(i,j-1)+ latitude_one_tile(i,j)) enddo enddo else From 009496c50ebf7f7a68a3952c31ae247e6c23b9c6 Mon Sep 17 00:00:00 2001 From: LarissaReames-NOAA <52886575+LarissaReames-NOAA@users.noreply.github.com> Date: Thu, 12 Sep 2019 08:10:28 -0500 Subject: [PATCH 17/34] Feature/chgres cube grib2 atmos (#13) * chgres_cube_grib2_atmos: This commit references #66268 dzdt is now properly converted to sigma coordinates and flipped vertically for non-native grib2 files * chgres_cube_grib2_atmos: This commit references #66268 Fixed a new indexing error in creating the staggered grid coordinates. Also removed the call to iso2sig when data is in isobaric coordinates. 3d pressure array is now generated from pressure levels read from file. --- sorc/chgres_cube.fd/input_data.F90 | 11 ++++++++++- sorc/chgres_cube.fd/model_grid.F90 | 24 ++++++++++++++++++++---- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/sorc/chgres_cube.fd/input_data.F90 b/sorc/chgres_cube.fd/input_data.F90 index 110b90cb7..09f3962fa 100644 --- a/sorc/chgres_cube.fd/input_data.F90 +++ b/sorc/chgres_cube.fd/input_data.F90 @@ -2617,7 +2617,16 @@ subroutine read_input_atm_grib2_file(localpet) call error_handler("IN FieldGet", rc) end do - call iso2sig(rlevs,vcoord,lev_input,levp1_input,psptr,atm,clb,cub,5+num_tracers, iret) + !call iso2sig(rlevs,vcoord,lev_input,levp1_input,psptr,atm,clb,cub,5+num_tracers, iret) + do i = clb(1),cub(1) + do j = clb(2),cub(2) + atm(1)%var(i,j,:) = rlevs(lev_input:1:-1) + do n = 2,num_tracers+5 + atm(n)%var(i,j,:) = atm(n)%var(i,j,lev_input:1:-1) + end do + end do + end do + deallocate(vcoord) else diff --git a/sorc/chgres_cube.fd/model_grid.F90 b/sorc/chgres_cube.fd/model_grid.F90 index ab75540d8..555b1a920 100644 --- a/sorc/chgres_cube.fd/model_grid.F90 +++ b/sorc/chgres_cube.fd/model_grid.F90 @@ -663,6 +663,7 @@ subroutine define_input_grid_gfs_grib2(localpet, npets) enddo deltalon = abs(longitude(2,1)-longitude(1,1)) + if(localpet==0) print*, "deltalon = ", deltalon print*,"- CALL FieldScatter FOR INPUT GRID LONGITUDE." call ESMF_FieldScatter(longitude_input_grid, longitude, rootpet=0, rc=rc) @@ -707,7 +708,9 @@ subroutine define_input_grid_gfs_grib2(localpet, npets) lat_src_ptr(i,j) = latitude(i,j) enddo enddo - + if(localpet==0) print*, "lon first = ", lon_src_ptr(1:10,1) + if(localpet==0) print*, "lat first = ", lat_src_ptr(1,1:10) + print*,"- CALL GridAddCoord FOR INPUT GRID." call ESMF_GridAddCoord(input_grid, & staggerloc=ESMF_STAGGERLOC_CORNER, rc=rc) @@ -782,7 +785,7 @@ subroutine define_input_grid_grib2(localpet, npets) lon_target(i_target,j_target) !lat_corners(ip1_input,jp1_input), & !lon_corners(ip1_input,jp1_input) - real(esmf_kind_r8) :: deltalon, dx + real(esmf_kind_r8) :: deltalon, dx, deltalat integer :: ncid,id_var, id_dim real(esmf_kind_r8), pointer :: lat_src_ptr(:,:), lon_src_ptr(:,:) character(len=10000) :: cmdline_msg, temp_msg, temp_msg2 @@ -1010,10 +1013,23 @@ subroutine define_input_grid_grib2(localpet, npets) if (trim(input_grid_type) == "latlon") then deltalon = abs(longitude_one_tile(2,1)-longitude_one_tile(1,1)) + deltalat = abs(latitude_one_tile(1,2) - latitude_one_tile(1,1)) do j = clb(2), cub(2) do i = clb(1), cub(1) - lon_src_ptr(i,j) = longitude_one_tile(i,1) - (0.5_esmf_kind_r8*deltalon) - lat_src_ptr(i,j) = 0.5_esmf_kind_r8 * (latitude_one_tile(i,j-1)+ latitude_one_tile(i,j)) + + if (i == ip1_input) then + lon_src_ptr(i,j) = longitude_one_tile(i-1,1)+(0.5_esmf_kind_r8*deltalon) + else + lon_src_ptr(i,j) = longitude_one_tile(i,1) - (0.5_esmf_kind_r8*deltalon) + endif + + if (j == jp1_input) then + lat_src_ptr(i,j) = latitude_one_tile(1,j-1)+(0.5_esmf_kind_r8*deltalat) + else + lat_src_ptr(i,j) = latitude_one_tile(1,j) - (0.5_esmf_kind_r8*deltalat) + endif + + !lat_src_ptr(i,j) = 0.5_esmf_kind_r8 * (latitude_one_tile(i,j-1)+ latitude_one_tile(i,j)) enddo enddo else From 7dbe65c72534ef0e02038ebbee87e0abc666071e Mon Sep 17 00:00:00 2001 From: George Gayno Date: Thu, 12 Sep 2019 21:33:06 +0000 Subject: [PATCH 18/34] feature/hera_port: This commit references #1. Update location of zero bia file on hera. --- ush/fv3gfs_chgres.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/fv3gfs_chgres.sh b/ush/fv3gfs_chgres.sh index 0f795d0b5..ac2b6e39a 100755 --- a/ush/fv3gfs_chgres.sh +++ b/ush/fv3gfs_chgres.sh @@ -90,7 +90,7 @@ elif [ $machine = WCOSS_DELL_P3 ]; then elif [ $machine = HERA ]; then export HOMEgfs=$SLURM_SUBMIT_DIR/.. export PTMP=${PTMP:-"/scratch2/NCEPDEV/stmp1/$USER"} - export zero_bias_dir=/scratch4/NCEPDEV/global/save/glopara/git/bias_zero + export zero_bias_dir=/scratch1/NCEPDEV/global/glopara/bias_zero else echo "Please define HOMEgfs and PTMP for your machine. exit" exit From ba9d9913ccfdc93f1ad4241474f04589e79aabc9 Mon Sep 17 00:00:00 2001 From: George Gayno Date: Fri, 13 Sep 2019 13:25:59 +0000 Subject: [PATCH 19/34] feature/hera_port: This commit references #1. Kate moved the climo files for program "sfc_climo_gen" to the standard location along with the other fixed files. Update "sorc/link_fixdirs.sh" accordingly. --- sorc/link_fixdirs.sh | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/sorc/link_fixdirs.sh b/sorc/link_fixdirs.sh index eef200fcd..d5fbc1cde 100755 --- a/sorc/link_fixdirs.sh +++ b/sorc/link_fixdirs.sh @@ -45,17 +45,9 @@ elif [ $machine = "jet" ]; then FIX_DIR="/lfs3/projects/hfv3gfs/glopara/git/fv3gfs/fix" fi cd ${pwd}/../fix ||exit 8 -for dir in fix_am fix_fv3 fix_orog fix_fv3_gmted2010 ; do +for dir in fix_am fix_fv3 fix_orog fix_fv3_gmted2010 fix_sfc_climo; do [[ -d $dir ]] && rm -rf $dir $LINK $FIX_DIR/$dir . done -if [ $machine == "cray" ] || [ $machine = "dell" ]; then - $LINK /gpfs/dell2/emc/modeling/noscrub/George.Gayno/landutil.git/climo_fields_netcdf ./fix_sfc_climo -elif [ $machine = "hera" ]; then - $LINK /scratch1/NCEPDEV/da/George.Gayno/ufs_utils.git/climo_fields_netcdf ./fix_sfc_climo -elif [ $machine = "jet" ]; then - $LINK /mnt/lfs3/projects/emcda/George.Gayno/climo_fields_netcdf ./fix_sfc_climo -fi - exit 0 From d88bf1c073660bf3bd2f4d7da32994247246faf9 Mon Sep 17 00:00:00 2001 From: George Gayno Date: Fri, 13 Sep 2019 14:26:06 +0000 Subject: [PATCH 20/34] feature/hera_port: This commit references #1. Add the "mkgfsnemsioctl" utility from the global-workflow repository. --- sorc/build_nems_util.sh | 2 +- sorc/mkgfsnemsioctl.fd/makefile | 10 + sorc/mkgfsnemsioctl.fd/mkgfsnemsioctl.f | 535 ++++++++++++++++++++++++ 3 files changed, 546 insertions(+), 1 deletion(-) create mode 100755 sorc/mkgfsnemsioctl.fd/makefile create mode 100644 sorc/mkgfsnemsioctl.fd/mkgfsnemsioctl.f diff --git a/sorc/build_nems_util.sh b/sorc/build_nems_util.sh index 795f1f056..3f2f1498c 100755 --- a/sorc/build_nems_util.sh +++ b/sorc/build_nems_util.sh @@ -22,7 +22,7 @@ if [ ! -d "../exec" ]; then mkdir ../exec fi -for prog in nemsio_get.fd nemsio_read.fd nemsio_chgdate.fd ;do +for prog in nemsio_get.fd nemsio_read.fd nemsio_chgdate.fd mkgfsnemsioctl.fd ;do cd ${cwd}/${prog} make -f makefile done diff --git a/sorc/mkgfsnemsioctl.fd/makefile b/sorc/mkgfsnemsioctl.fd/makefile new file mode 100755 index 000000000..351cda92d --- /dev/null +++ b/sorc/mkgfsnemsioctl.fd/makefile @@ -0,0 +1,10 @@ +SHELL =/bin/ksh +EXEC =../../exec/mkgfsnemsioctl +FOPTS = -O -FR -I$(NEMSIO_INC) +LOPTS = $(OPTS) +LIBS = $(NEMSIO_LIB) $(BACIO_LIB4) $(W3NCO_LIBd) +OBJS = mkgfsnemsioctl.o +SRCS = mkgfsnemsioctl.f +# ************************************************************************* +all: $(SRCS) + $(FCMP) $(FOPTS) $(LOPTS) ${SRCS} -o $(EXEC) $(LIBS) diff --git a/sorc/mkgfsnemsioctl.fd/mkgfsnemsioctl.f b/sorc/mkgfsnemsioctl.fd/mkgfsnemsioctl.f new file mode 100644 index 000000000..cdc9efc90 --- /dev/null +++ b/sorc/mkgfsnemsioctl.fd/mkgfsnemsioctl.f @@ -0,0 +1,535 @@ +!- - - - -- - -- - -- - -- - - -- - -- -- - -- - -- - - -- - - - -- - -- +! the program create NEMS GFS sfc ctl file +! +! Mar 12, 2015 Jun Wang +! - - - - -- - -- - -- - -- - - -- - -- -- - -- - -- - - -- - - - -- - -- +! + program main + use nemsio_module + implicit none +! +!--------------------------------------------------------------------------- + type(nemsio_gfile) :: gfile + character(255) cin,cindx + real,allocatable :: data(:,:) +!--------------------------------------------------------------------------- +!--- nemsio meta data + integer nrec,im,jm,lm,idate(7),nfhour,tlmeta,nsoil,fieldsize + real(4),allocatable :: lat1(:),lat(:),lon1(:) + real(8),allocatable :: slat(:) + character(16),allocatable:: recname(:),reclevtyp(:) + integer,allocatable:: reclev(:) +!--------------------------------------------------------------------------- +!--- local vars + character(3) cmon + character(16) reclevtyp_sht + character(32) ctldate,varname + character(35) sweep_blanks + real(8) radi + real lon_stt,lat_stt,lon_intl,lat_intl + integer i,n,j,krec,iret,io_unit,idrt,nmeta +!--------------------------------------------------------------------------- +!--------------------------------------------------------------------------- +! + call nemsio_init(iret=iret) + if(iret/=0) print *,'ERROR: nemsio_init ' +! +!--------------------------------------------------------------------------- +!*** read nemsio grd header info +!--------------------------------------------------------------------------- +!--- open gfile for reading + call getarg(1,cin) + print *,'filename is cin=',trim(cin) + if(trim(cin)=='') then + print *,'usage: mknemsioctl input_nemsio_file_name' + stop + endif + call getarg(2,cindx) + if(trim(cindx)=='') then + cindx=trim(cin)//'.ctl' + endif + + call nemsio_open(gfile,trim(cin),'READ',iret=iret) + if(iret/=0) print *,'Error: open nemsio file,',trim(cin),' iret=',iret + + call nemsio_getfilehead(gfile,iret=iret,nrec=nrec,dimx=im,dimy=jm, & + dimz=lm,idate=idate,nfhour=nfhour,nmeta=nmeta, & + nsoil=nsoil,idrt=idrt,tlmeta=tlmeta) +! + fieldsize=im*jm + allocate(recname(nrec),reclevtyp(nrec),reclev(nrec)) + allocate(lat1(fieldsize),lat(jm),lon1(fieldsize)) + call nemsio_getfilehead(gfile,iret=iret,recname=recname, & + reclevtyp=reclevtyp,reclev=reclev,lat=lat1,lon=lon1) +! + call nemsio_close(gfile,iret=iret) +! + call nemsio_finalize() +! +!--------------------------------------------------------------------------- +!****** write .ctl file +!--------------------------------------------------------------------------- +! +!-- get date + call cmonth(idate(2),cmon) + write(ctldate,'(i2.2,a,i2.2,a3,i4.4)')idate(4),'Z',idate(3) & + ,cmon,idate(1) +!-- get Gaussian grid + if(idrt==4) then +!-- data has lat/lon info + if (nmeta>=8) then + lon_stt=lon1(1) + lon_intl=lon1(2)-lon1(1) + do j=1,jm + lat(j) = lat1(1+(j-1)*im) + enddo + else + call splat8(idrt,jm,slat) + radi=180.0/(4.*atan(1.0)) + do j=1,jm + lat(j) = asin(slat(j)) * radi + enddo + lon_stt=0. + lon_intl=360./real(jm) + endif + elseif(idrt==0) then + if(nmeta>=8) then + lon_stt=lon1(1) + lon_intl=lon1(2)-lon1(1) + lat_stt=-abs(lat1(1)) + lat_intl=abs(lat1(1+im)-lat1(1)) +!! print*, "im=",im,' jm=',jm +!! print*, "lon1=",lon1 +!! print*, "lat_intl=",lat_intl +!! print*, "lat1=",lat1 + else + if (mod(jm,2) == 0) then + lat_intl=180.0/real(jm) + lat_stt=-90.0+0.5*lat_intl + else + lat_intl=180.0/real(jm-1) + lat_stt=-90.0 + endif + lon_intl=360./real(im) + lon_stt=0.5*lon_intl + endif + endif + + io_unit=650 + open(io_unit,file=trim(cindx),form='formatted') +! + write(io_unit,105)trim(cin) + write(io_unit,106) + + if (idrt == 4 )then + write(io_unit,107) + elseif(idrt==0) then + if(nmeta>=8) then + if (lat1(1)>lat1(im+1)) then + write(io_unit,107) + else + write(io_unit,1077) + endif + else + write(io_unit,1077) + endif + endif + + write(io_unit,108)tlmeta + write(io_unit,109) + write(io_unit,111)im,lon_stt,lon_intl + if (idrt==4) then + write(io_unit,112)jm + write(io_unit,113)(lat(i),i=jm,1,-1) + else + write(io_unit,120)jm,lat_stt,lat_intl + endif + write(io_unit,114)lm + if(nfhour/=0) then + write(io_unit,115)1,trim(ctldate),nfhour + else + write(io_unit,116)1,trim(ctldate) + endif + +! + 105 FORMAT('dset ^',A) + 106 FORMAT('undef 9.99E+20') + 107 FORMAT('options big_endian sequential yrev') + 1077 FORMAT('options big_endian sequential') + 108 FORMAT('fileheader',I12.0) + 109 FORMAT('title gfs nemsioi file') + + 111 FORMAT('xdef ',I6,' linear ',f9.6,' ',f9.6) + 112 FORMAT('ydef ',I6,' levels') + 113 FORMAT(10f11.6) + + 114 FORMAT('zdef ',I6,' linear 1 1') + 115 FORMAT('tdef ',I6,' linear ',A12,' ',I6,'hr') + 116 FORMAT('tdef ',I6,' linear ',A12,' 1yr') +! + 120 FORMAT('ydef ',I6,' linear ',f13.6,' ',f9.6) +! + krec=0 + do n=1,nrec + if(reclev(n)==1) then + krec=krec+1 + endif + enddo + + WRITE(IO_UNIT,'(A,I6)')'VARS ',krec + + n=1 + do while (n<=nrec) + reclevtyp_sht=reclevtyp(n) + if(trim(reclevtyp_sht) == "convect-cld bot") then + reclevtyp_sht="cvb" + elseif (trim(reclevtyp_sht) == "convect-cld top") then + reclevtyp_sht="cvt" + elseif (trim(reclevtyp_sht) == "high cld bot") then + reclevtyp_sht="hcb" + elseif (trim(reclevtyp_sht) == "high cld top") then + reclevtyp_sht="hct" + elseif (trim(reclevtyp_sht) == "mid cld bot") then + reclevtyp_sht="mcb" + elseif (trim(reclevtyp_sht) == "mid cld top") then + reclevtyp_sht="mct" + elseif (trim(reclevtyp_sht) == "low cld bot") then + reclevtyp_sht="lcb" + elseif (trim(reclevtyp_sht) == "low cld top") then + reclevtyp_sht="lct" + elseif (trim(reclevtyp_sht) == "convect-cld laye") then + reclevtyp_sht="cvcl" + elseif (trim(reclevtyp_sht) == "bndary-layer cld") then + reclevtyp_sht="bdrlc" + elseif (trim(reclevtyp_sht) == "atmos col") then + reclevtyp_sht="acol" + elseif (trim(reclevtyp_sht) == "high cld lay") then + reclevtyp_sht="hcl" + elseif (trim(reclevtyp_sht) == "mid cld lay") then + reclevtyp_sht="mcl" + elseif (trim(reclevtyp_sht) == "low cld lay") then + reclevtyp_sht="lcl" + elseif (trim(reclevtyp_sht) == "2 m above gnd") then + reclevtyp_sht="2m" + elseif (trim(reclevtyp_sht) == "10 m above gnd") then + reclevtyp_sht="10m" + endif + varname=sweep_blanks(trim(recname(n))//trim(reclevtyp_sht)) + if(trim(reclevtyp(n))=='mid layer') then + write(io_unit,'(a16,i3,a)')varname,lm,' 99 model layer' + n=n+lm + elseif(trim(reclevtyp(n))=='soil layer') then + write(io_unit,'(a16,i3,a)')varname,nsoil,' 99 soil layer' + n=n+nsoil + elseif(trim(reclevtyp(n))=='2 m above gnd') then + recname(n)=trim(recname(n))//'2m' + write(io_unit,'(a16,a7,a)')varname,' 0 99 ',trim(reclevtyp(n)) + n=n+1 + elseif(trim(reclevtyp(n))=='10 m above gnd') then + recname(n)=trim(recname(n))//'10m' + write(io_unit,'(a16,a7,a)')varname,' 0 99 ',trim(reclevtyp(n)) + n=n+1 + else + write(io_unit,'(a16,a7,a)')varname,' 0 99 ',trim(reclevtyp(n)) + n=n+1 + endif + enddo + + write(io_unit,'(A8)')'endvars' + close(io_unit) + +!--------------------------------------------------------------------------- +!****** clean up +!--------------------------------------------------------------------------- + deallocate(recname,reclevtyp,reclev,lat,lat1,lon1) +!--------------------------------------------------------------------------- +! +! - - - - -- - -- - -- - -- - - -- - -- -- - -- - -- - - -- - - - -- - -- + stop + + end program +! - - - - -- - -- - -- - -- - - -- - -- -- - -- - -- - - -- - - - -- - -- + SUBROUTINE CMONTH(IMON,CMON) +! +!----------------------------------------------------------------------- +!*** Convert month +!----------------------------------------------------------------------- +! + INTEGER,INTENT(IN) :: IMON + CHARACTER(LEN=3) :: CMON +! +!----------------------------------------------------------------------- +! + SELECT CASE (IMON) + CASE(1) + CMON='Jan' + CASE(2) + CMON='Feb' + CASE(3) + CMON='Mar' + CASE(4) + CMON='Apr' + CASE(5) + CMON='May' + CASE(6) + CMON='Jun' + CASE(7) + CMON='Jul' + CASE(8) + CMON='Aug' + CASE(9) + CMON='Sep' + CASE(10) + CMON='Oct' + CASE(11) + CMON='Nov' + CASE(12) + CMON='Dec' + END SELECT +! +!----------------------------------------------------------------------- +! + END SUBROUTINE CMONTH + +! + sUBROUTINE splat4(IDRT,JMAX,ASLAT) +! + implicit none + integer,intent(in) :: idrt,jmax + real(4),intent(out) :: ASLAT(JMAX) + INTEGER,PARAMETER:: KD=SELECTED_REAL_KIND(15,45) + REAL(KIND=KD):: PK(JMAX/2),PKM1(JMAX/2),PKM2(JMAX/2) + REAL(KIND=KD):: ASLATD(JMAX/2),SP,SPMAX,EPS=10.d0*EPSILON(SP) + integer,PARAMETER:: JZ=50 + REAL(8) BZ(JZ) + DATA BZ / 2.4048255577d0, 5.5200781103d0, & + 8.6537279129d0, 11.7915344391d0, 14.9309177086d0, 18.0710639679d0, & + 21.2116366299d0, 24.3524715308d0, 27.4934791320d0, 30.6346064684d0, & + 33.7758202136d0, 36.9170983537d0, 40.0584257646d0, 43.1997917132d0, & + 46.3411883717d0, 49.4826098974d0, 52.6240518411d0, 55.7655107550d0, & + 58.9069839261d0, 62.0484691902d0, 65.1899648002d0, 68.3314693299d0, & + 71.4729816036d0, 74.6145006437d0, 77.7560256304d0, 80.8975558711d0, & + 84.0390907769d0, 87.1806298436d0, 90.3221726372d0, 93.4637187819d0, & + 96.6052679510d0, 99.7468198587d0, 102.888374254d0, 106.029930916d0, & + 109.171489649d0, 112.313050280d0, 115.454612653d0, 118.596176630d0, & + 121.737742088d0, 124.879308913d0, 128.020877005d0, 131.162446275d0, & + 134.304016638d0, 137.445588020d0, 140.587160352d0, 143.728733573d0, & + 146.870307625d0, 150.011882457d0, 153.153458019d0, 156.295034268d0 / + REAL(8):: DLT,D1=1.d0 + INTEGER(4):: JHE,JHO,J0=0 + real(8),PARAMETER :: PI=3.14159265358979d0,C=(1.d0-(2.d0/PI)**2)*0.25d0 + real(8) r + integer jh,js,n,j +!C - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +!C GAUSSIAN LATITUDES + IF(IDRT.EQ.4) THEN + JH=JMAX/2 + JHE=(JMAX+1)/2 + R=1.d0/SQRT((JMAX+0.5d0)**2+C) + DO J=1,MIN(JH,JZ) + ASLATD(J)=COS(BZ(J)*R) + ENDDO + DO J=JZ+1,JH + ASLATD(J)=COS((BZ(JZ)+(J-JZ)*PI)*R) + ENDDO + SPMAX=1.d0 + DO WHILE(SPMAX.GT.EPS) + SPMAX=0.d0 + DO J=1,JH + PKM1(J)=1.d0 + PK(J)=ASLATD(J) + ENDDO + DO N=2,JMAX + DO J=1,JH + PKM2(J)=PKM1(J) + PKM1(J)=PK(J) + PK(J)=((2*N-1)*ASLATD(J)*PKM1(J)-(N-1)*PKM2(J))/N + ENDDO + ENDDO + DO J=1,JH + SP=PK(J)*(1.d0-ASLATD(J)**2)/(JMAX*(PKM1(J)-ASLATD(J)*PK(J))) + ASLATD(J)=ASLATD(J)-SP + SPMAX=MAX(SPMAX,ABS(SP)) + ENDDO + ENDDO +!CDIR$ IVDEP + DO J=1,JH + ASLAT(J)=ASLATD(J) + ASLAT(JMAX+1-J)=-ASLAT(J) + ENDDO + IF(JHE.GT.JH) THEN + ASLAT(JHE)=0.d0 + ENDIF +!C - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +!C EQUALLY-SPACED LATITUDES INCLUDING POLES + ELSEIF(IDRT.EQ.0) THEN + JH=JMAX/2 + JHE=(JMAX+1)/2 + JHO=JHE-1 + DLT=PI/(JMAX-1) + ASLAT(1)=1.d0 + DO J=2,JH + ASLAT(J)=COS((J-1)*DLT) + ENDDO +!CDIR$ IVDEP + DO J=1,JH + ASLAT(JMAX+1-J)=-ASLAT(J) + ENDDO + IF(JHE.GT.JH) THEN + ASLAT(JHE)=0.d0 + ENDIF +!C - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +!C EQUALLY-SPACED LATITUDES EXCLUDING POLES + ELSEIF(IDRT.EQ.256) THEN + JH=JMAX/2 + JHE=(JMAX+1)/2 + JHO=JHE + DLT=PI/JMAX + ASLAT(1)=1.d0 + DO J=1,JH + ASLAT(J)=COS((J-0.5)*DLT) + ENDDO +!CDIR$ IVDEP + DO J=1,JH + ASLAT(JMAX+1-J)=-ASLAT(J) + ENDDO + IF(JHE.GT.JH) THEN + ASLAT(JHE)=0.d0 + ENDIF + ENDIF +!C - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + end subroutine splat4 +! +!---------------------------------------------------------------------- + SUBROUTINE splat8(IDRT,JMAX,ASLAT) +!$$$ + implicit none + integer,intent(in) :: idrt,jmax + real(8),intent(out) :: ASLAT(JMAX) + INTEGER,PARAMETER:: KD=SELECTED_REAL_KIND(15,45) + REAL(KIND=KD):: PK(JMAX/2),PKM1(JMAX/2),PKM2(JMAX/2) + REAL(KIND=KD):: ASLATD(JMAX/2),SP,SPMAX,EPS=10.d0*EPSILON(SP) + integer,PARAMETER:: JZ=50 + REAL(8) BZ(JZ) + DATA BZ / 2.4048255577d0, 5.5200781103d0, & + 8.6537279129d0, 11.7915344391d0, 14.9309177086d0, 18.0710639679d0, & + 21.2116366299d0, 24.3524715308d0, 27.4934791320d0, 30.6346064684d0, & + 33.7758202136d0, 36.9170983537d0, 40.0584257646d0, 43.1997917132d0, & + 46.3411883717d0, 49.4826098974d0, 52.6240518411d0, 55.7655107550d0, & + 58.9069839261d0, 62.0484691902d0, 65.1899648002d0, 68.3314693299d0, & + 71.4729816036d0, 74.6145006437d0, 77.7560256304d0, 80.8975558711d0, & + 84.0390907769d0, 87.1806298436d0, 90.3221726372d0, 93.4637187819d0, & + 96.6052679510d0, 99.7468198587d0, 102.888374254d0, 106.029930916d0, & + 109.171489649d0, 112.313050280d0, 115.454612653d0, 118.596176630d0, & + 121.737742088d0, 124.879308913d0, 128.020877005d0, 131.162446275d0, & + 134.304016638d0, 137.445588020d0, 140.587160352d0, 143.728733573d0, & + 146.870307625d0, 150.011882457d0, 153.153458019d0, 156.295034268d0 / + REAL(8):: DLT,D1=1.d0 + INTEGER(4):: JHE,JHO,J0=0 + real(8),PARAMETER :: PI=3.14159265358979d0,C=(1.d0-(2.d0/PI)**2)*0.25d0 + real(8) r + integer jh,js,n,j +!C - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +!C GAUSSIAN LATITUDES + IF(IDRT.EQ.4) THEN + JH=JMAX/2 + JHE=(JMAX+1)/2 + R=1.d0/SQRT((JMAX+0.5d0)**2+C) + DO J=1,MIN(JH,JZ) + ASLATD(J)=COS(BZ(J)*R) + ENDDO + DO J=JZ+1,JH + ASLATD(J)=COS((BZ(JZ)+(J-JZ)*PI)*R) + ENDDO + SPMAX=1.d0 + DO WHILE(SPMAX.GT.EPS) + SPMAX=0.d0 + DO J=1,JH + PKM1(J)=1.d0 + PK(J)=ASLATD(J) + ENDDO + DO N=2,JMAX + DO J=1,JH + PKM2(J)=PKM1(J) + PKM1(J)=PK(J) + PK(J)=((2*N-1)*ASLATD(J)*PKM1(J)-(N-1)*PKM2(J))/N + ENDDO + ENDDO + DO J=1,JH + SP=PK(J)*(1.d0-ASLATD(J)**2)/(JMAX*(PKM1(J)-ASLATD(J)*PK(J))) + ASLATD(J)=ASLATD(J)-SP + SPMAX=MAX(SPMAX,ABS(SP)) + ENDDO + ENDDO +!CDIR$ IVDEP + DO J=1,JH + ASLAT(J)=ASLATD(J) + ASLAT(JMAX+1-J)=-ASLAT(J) + ENDDO + IF(JHE.GT.JH) THEN + ASLAT(JHE)=0.d0 + ENDIF +!C - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +!C EQUALLY-SPACED LATITUDES INCLUDING POLES + ELSEIF(IDRT.EQ.0) THEN + JH=JMAX/2 + JHE=(JMAX+1)/2 + JHO=JHE-1 + DLT=PI/(JMAX-1) + ASLAT(1)=1.d0 + DO J=2,JH + ASLAT(J)=COS((J-1)*DLT) + ENDDO +!CDIR$ IVDEP + DO J=1,JH + ASLAT(JMAX+1-J)=-ASLAT(J) + ENDDO + IF(JHE.GT.JH) THEN + ASLAT(JHE)=0.d0 + ENDIF +!C - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +!C EQUALLY-SPACED LATITUDES EXCLUDING POLES + ELSEIF(IDRT.EQ.256) THEN + JH=JMAX/2 + JHE=(JMAX+1)/2 + JHO=JHE + DLT=PI/JMAX + ASLAT(1)=1.d0 + DO J=1,JH + ASLAT(J)=COS((J-0.5d0)*DLT) + ENDDO +!DIR$ IVDEP + DO J=1,JH + ASLAT(JMAX+1-J)=-ASLAT(J) + ENDDO + IF(JHE.GT.JH) THEN + ASLAT(JHE)=0.d0 + ENDIF + ENDIF +! - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + end subroutine splat8 +!----------------------------------------------------------------------- + + character(35) function sweep_blanks(in_str) + +! + implicit none +! + character(*), intent(in) :: in_str + character(35) :: out_str + character :: ch + integer :: j + + out_str = " " + do j=1, len_trim(in_str) + ! get j-th char + ch = in_str(j:j) + if (ch .eq. "-") then + out_str = trim(out_str) // "_" + else if (ch .ne. " ") then + out_str = trim(out_str) // ch + endif + sweep_blanks = out_str + end do + end function sweep_blanks + From ca07277b6ac2fa0d275f6b50c38a12b3773211c0 Mon Sep 17 00:00:00 2001 From: Fanglin Yang Date: Mon, 16 Sep 2019 13:41:19 -0400 Subject: [PATCH 21/34] change default FNMSKH to high-res masks (#16) This merge references #14 set FNMSKH=${FNMSKH:-${FIXam}/global_slmask.t1534.3072.1536.grb} instead of FNMSKH=${FNMSKH:-${FIXam}/seaice_newland.grb} to use high-res land-sea mask in global_chgres.sh and global_cycle.sh for interpolation of climatological SST etc. This is needed to reduce lake temperature biases for GFS.v16 development. --- ush/global_chgres.sh | 2 +- ush/global_chgres_driver.sh | 2 +- ush/global_cycle.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ush/global_chgres.sh b/ush/global_chgres.sh index fd280ce18..d2770edea 100755 --- a/ush/global_chgres.sh +++ b/ush/global_chgres.sh @@ -300,7 +300,7 @@ FNVMNC=${FNVMNC:-${FIXam}/global_shdmin.0.144x0.144.grb} FNVMXC=${FNVMXC:-${FIXam}/global_shdmax.0.144x0.144.grb} FNSLPC=${FNSLPC:-${FIXam}/global_slope.1x1.grb} FNABSC=${FNABSC:-${FIXam}/global_snoalb.1x1.grb} -FNMSKH=${FNMSKH:-${FIXam}/seaice_newland.grb} +FNMSKH=${FNMSKH:-${FIXam}/global_slmask.t1534.3072.1536.grb} LANDICE_OPT=${LANDICE_OPT:-2} CLIMO_FIELDS_OPT=${CLIMO_FIELDS_OPT:-3} SOILTYPE_INP=${SOILTYPE_INP:-"zobler"} diff --git a/ush/global_chgres_driver.sh b/ush/global_chgres_driver.sh index 01c02d518..e81997c08 100755 --- a/ush/global_chgres_driver.sh +++ b/ush/global_chgres_driver.sh @@ -191,7 +191,7 @@ export FNVEGC=${FIXam}/global_vegfrac.0.144.decpercent.grb export FNVMNC=${FIXam}/global_shdmin.0.144x0.144.grb export FNVMXC=${FIXam}/global_shdmax.0.144x0.144.grb export FNSLPC=${FIXam}/global_slope.1x1.grb -export FNMSKH=${FIXam}/seaice_newland.grb +export FNMSKH=${FIXam}/global_slmask.t1534.3072.1536.grb export FNSMCC=$FIXam/global_soilmgldas.statsgo.t${JCAP_CASE}.${LONB_SFC}.${LATB_SFC}.grb export FNSOTC=$FIXam/global_soiltype.statsgo.t${JCAP_CASE}.${LONB_SFC}.${LATB_SFC}.rg.grb export FNVETC=$FIXam/global_vegtype.igbp.t${JCAP_CASE}.${LONB_SFC}.${LATB_SFC}.rg.grb diff --git a/ush/global_cycle.sh b/ush/global_cycle.sh index cd7396c5a..a0cc22943 100755 --- a/ush/global_cycle.sh +++ b/ush/global_cycle.sh @@ -286,7 +286,7 @@ FNABSC=${FNABSC:-${FIXam}/global_mxsnoalb.uariz.t$JCAP_CASE.$LONB_CASE.$LATB_CAS FNVMNC=${FNVMNC:-${FIXam}/global_shdmin.0.144x0.144.grb} FNVMXC=${FNVMXC:-${FIXam}/global_shdmax.0.144x0.144.grb} FNSLPC=${FNSLPC:-${FIXam}/global_slope.1x1.grb} -FNMSKH=${FNMSKH:-${FIXam}/seaice_newland.grb} +FNMSKH=${FNMSKH:-${FIXam}/global_slmask.t1534.3072.1536.grb} GSI_FILE=${GSI_FILE:-"NULL"} FNTSFA=${FNTSFA:-${COMIN}/${PREINP}sstgrb${SUFINP}} FNACNA=${FNACNA:-${COMIN}/${PREINP}engicegrb${SUFINP}} From 5d68083a35f666a31f1011a6a6beb94526ba83ff Mon Sep 17 00:00:00 2001 From: George Gayno Date: Wed, 18 Sep 2019 20:30:17 +0000 Subject: [PATCH 22/34] feature/hera_port: This commit references #1. Detect if running on Hera by checking for the existence of the /scratch1 directory per recommendation of the original designer. Previously, hostname was used. There is some concern that using 'hostname' will not work on the compute nodes (although I did not see a problem with my tests). --- modulefiles/module-setup.sh.inc | 2 +- sorc/machine-setup.sh | 2 +- ush/load_fv3gfs_modules.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/modulefiles/module-setup.sh.inc b/modulefiles/module-setup.sh.inc index bb9c1f0a1..d4faa4f64 100644 --- a/modulefiles/module-setup.sh.inc +++ b/modulefiles/module-setup.sh.inc @@ -22,7 +22,7 @@ if [[ -d /lfs3 ]] ; then source /apps/lmod/lmod/init/$__ms_shell fi module purge -elif [[ `hostname -a` == *"hera.fairmont.rdhpcs.noaa.gov"* ]] ; then +elif [[ -d /scratch1 ]] ; then # We are on NOAA Hera if ( ! eval module help > /dev/null 2>&1 ) ; then source /apps/lmod/lmod/init/$__ms_shell diff --git a/sorc/machine-setup.sh b/sorc/machine-setup.sh index d9a3b29d1..d54235b60 100644 --- a/sorc/machine-setup.sh +++ b/sorc/machine-setup.sh @@ -28,7 +28,7 @@ if [[ -d /lfs3 ]] ; then target=jet module purge module use /mnt/lfs3/projects/hfv3gfs/nwprod/lib/modulefiles -elif [[ `hostname -a` == *"hera.fairmont.rdhpcs.noaa.gov"* ]] ; then +elif [[ -d /scratch1 ]] ; then # We are on NOAA Hera if ( ! eval module help > /dev/null 2>&1 ) ; then echo load the module command 1>&2 diff --git a/ush/load_fv3gfs_modules.sh b/ush/load_fv3gfs_modules.sh index d44e2fe35..8972b6575 100755 --- a/ush/load_fv3gfs_modules.sh +++ b/ush/load_fv3gfs_modules.sh @@ -15,7 +15,7 @@ module use "$HOMEgfs/modulefiles" if [[ -d /lfs3 ]] ; then # We are on NOAA Jet module load module_base.jet -elif [[ `hostname -a` == *"hera.fairmont.rdhpcs.noaa.gov"* ]] ; then +elif [[ -d /scratch1 ]] ; then # We are on NOAA Hera module load module_base.hera elif [[ -d /gpfs/hps && -e /etc/SuSE-release ]] ; then From de48ad147cbed8457c363ca6a5b7ce8dea47120b Mon Sep 17 00:00:00 2001 From: George Gayno Date: Thu, 19 Sep 2019 15:30:21 +0000 Subject: [PATCH 23/34] feature/hera_port: This commit references #1. ./ush/fv3gfs_chgres.sh - increase Hera wall time to 3 hours. --- ush/fv3gfs_chgres.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/fv3gfs_chgres.sh b/ush/fv3gfs_chgres.sh index ac2b6e39a..a7f02dcc7 100755 --- a/ush/fv3gfs_chgres.sh +++ b/ush/fv3gfs_chgres.sh @@ -68,7 +68,7 @@ export machine=WCOSS_DELL_P3 ##SBATCH -e log.chgres ##SBATCH --nodes=1 ##SBATCH -q batch -##SBATCH -t 02:00:00 +##SBATCH -t 03:00:00 #export machine=HERA set -x From 829b1db9147ba911a96722f3d416c52138f7ee76 Mon Sep 17 00:00:00 2001 From: George Gayno Date: Mon, 23 Sep 2019 20:13:26 +0000 Subject: [PATCH 24/34] feature/hera_port: This commit references #1. Update serial chgres to use netcdf 4.7.0 and hdf 1.10.5 (on hera) per request of global workflow group. --- modulefiles/fv3gfs/global_chgres.hera | 4 ++-- modulefiles/module_base.hera | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modulefiles/fv3gfs/global_chgres.hera b/modulefiles/fv3gfs/global_chgres.hera index 90a618ad1..b91ac5e09 100644 --- a/modulefiles/fv3gfs/global_chgres.hera +++ b/modulefiles/fv3gfs/global_chgres.hera @@ -6,8 +6,8 @@ module load intel/18.0.5.274 module load impi/2018.0.4 -module load netcdf/4.6.1 -module load hdf5/1.10.4 +module load netcdf/4.7.0 +module load hdf5/1.10.5 # Loding nceplibs modules module use -a $MOD_PATH diff --git a/modulefiles/module_base.hera b/modulefiles/module_base.hera index dae1d895d..07e78cfc9 100644 --- a/modulefiles/module_base.hera +++ b/modulefiles/module_base.hera @@ -6,6 +6,6 @@ module load intel/18.0.5.274 module load impi/2018.0.4 module load hpss/hpss -module load netcdf/4.6.1 +module load netcdf/4.7.0 module load nco/4.7.0 -module load hdf5/1.10.4 +module load hdf5/1.10.5 From 573bd482531047f4aec2c59161719968763493ea Mon Sep 17 00:00:00 2001 From: George Gayno Date: Thu, 26 Sep 2019 18:26:13 +0000 Subject: [PATCH 25/34] feature/hera_port: This commit references #1. Hera build modules - Remove unneeded module loads. Point to standard location for libraries instead of George V's home directory. Update cycle program to use netcdf 4.7 per request of global workflow group. --- modulefiles/chgres_cube.hera | 1 - modulefiles/fv3gfs/global_cycle.hera | 4 ++-- modulefiles/modulefile.global_emcsfc_ice_blend.hera | 11 ++++------- modulefiles/modulefile.global_emcsfc_snow2mdl.hera | 11 ++++------- modulefiles/modulefile.sfc_climo_gen.hera | 8 ++------ 5 files changed, 12 insertions(+), 23 deletions(-) diff --git a/modulefiles/chgres_cube.hera b/modulefiles/chgres_cube.hera index edd787d32..9e7fcb753 100644 --- a/modulefiles/chgres_cube.hera +++ b/modulefiles/chgres_cube.hera @@ -4,7 +4,6 @@ module load intel/18.0.5.274 module load impi/2018.0.4 -module load netcdf/4.6.1 module use -a $MOD_PATH module load w3nco/2.0.6 diff --git a/modulefiles/fv3gfs/global_cycle.hera b/modulefiles/fv3gfs/global_cycle.hera index 25ccb9841..5f3afa8d5 100644 --- a/modulefiles/fv3gfs/global_cycle.hera +++ b/modulefiles/fv3gfs/global_cycle.hera @@ -6,8 +6,8 @@ module load intel/18.0.5.274 module load impi/2018.0.4 -module load netcdf/4.6.1 -module load hdf5/1.10.4 +module load netcdf/4.7.0 +module load hdf5/1.10.5 export NETCDF_INCLUDE="-I${NETCDF}/include" export NETCDF_LDFLAGS_F="-L${NETCDF}/lib -lnetcdf -lnetcdff -L${HDF5}/lib -lhdf5 -lhdf5_fortran" diff --git a/modulefiles/modulefile.global_emcsfc_ice_blend.hera b/modulefiles/modulefile.global_emcsfc_ice_blend.hera index b7fd0765b..2305392cf 100644 --- a/modulefiles/modulefile.global_emcsfc_ice_blend.hera +++ b/modulefiles/modulefile.global_emcsfc_ice_blend.hera @@ -11,10 +11,7 @@ export FFLAGS="-O0 -i4" module use -a $MOD_PATH module load w3nco/2.0.6 module load bacio/2.0.2 - -export NCEPLIBS=/scratch1/NCEPDEV/global/gwv/l819/lib -module use -a $NCEPLIBS/modulefiles -module load jasper-intel-sandybridge/1.900.1 -module load zlib-intel-sandybridge/1.2.7 -module load png-intel-sandybridge/1.2.59 -module load g2-intel-sandybridge/2.5.0 +module load jasper/1.900.1 +module load z/1.2.11 +module load png/1.2.44 +module load g2/2.5.0 diff --git a/modulefiles/modulefile.global_emcsfc_snow2mdl.hera b/modulefiles/modulefile.global_emcsfc_snow2mdl.hera index 8aab4cf1a..193f6247d 100644 --- a/modulefiles/modulefile.global_emcsfc_snow2mdl.hera +++ b/modulefiles/modulefile.global_emcsfc_snow2mdl.hera @@ -14,13 +14,10 @@ module load sp/2.0.2 module load w3nco/2.0.6 module load bacio/2.0.2 module load landsfcutil/2.1.0 - -export NCEPLIBS=/scratch1/NCEPDEV/global/gwv/l819/lib -module use -a $NCEPLIBS/modulefiles -module load jasper-intel-sandybridge/1.900.1 -module load zlib-intel-sandybridge/1.2.7 -module load png-intel-sandybridge/1.2.59 -module load g2-intel-sandybridge/2.5.0 +module load jasper/1.900.1 +module load z/1.2.11 +module load png/1.2.44 +module load g2/2.5.0 export FCOMP=ifort export FFLAGS="-O0 -r8 -i4 -FR -I${IP_INCd} -qopenmp -convert big_endian -assume byterecl" diff --git a/modulefiles/modulefile.sfc_climo_gen.hera b/modulefiles/modulefile.sfc_climo_gen.hera index d17cb63cb..ecea921b4 100644 --- a/modulefiles/modulefile.sfc_climo_gen.hera +++ b/modulefiles/modulefile.sfc_climo_gen.hera @@ -5,12 +5,8 @@ module load intel/18.0.5.274 module load impi/2018.0.4 -module use -a $MOD_PATH -module load netcdf/4.6.1 - -export NCEPLIBS=/scratch1/NCEPDEV/global/gwv/l819/lib -module use -a $NCEPLIBS/modulefiles -module load esmflocal/7_1_0r +module use -a /apps/modules/modulefamilies/intel_impi +module load esmf/7.1.0r export FCOMP=mpiifort export FFLAGS="-O3 -fp-model=precise -g -traceback -r8 -i4 -convert big_endian" From 5149a8835685eced2437c6c3481bd1dfa8f6b611 Mon Sep 17 00:00:00 2001 From: George Gayno Date: Mon, 30 Sep 2019 15:39:20 +0000 Subject: [PATCH 26/34] feature/hera_port: This commit references #1. Update remaining programs to use netcdf 4.7 per request of global workflow group. For esmf, use gwv's copy of version 8 compiled with netcdf 4.7. --- driver_scripts/driver_grid.hera.sh | 4 ++-- modulefiles/chgres_cube.hera | 2 +- modulefiles/fv3gfs/fre-nctools.hera | 4 ++-- modulefiles/fv3gfs/orog.hera | 2 +- modulefiles/modulefile.sfc_climo_gen.hera | 5 +++-- 5 files changed, 9 insertions(+), 8 deletions(-) diff --git a/driver_scripts/driver_grid.hera.sh b/driver_scripts/driver_grid.hera.sh index e61d08a3e..ad93e30e7 100644 --- a/driver_scripts/driver_grid.hera.sh +++ b/driver_scripts/driver_grid.hera.sh @@ -55,8 +55,8 @@ set -x module purge module load intel/18.0.5.274 module load impi/2018.0.4 -module load hdf5/1.10.4 -module load netcdf/4.6.1 +module load hdf5/1.10.5 +module load netcdf/4.7.0 module list #----------------------------------------------------------------------- diff --git a/modulefiles/chgres_cube.hera b/modulefiles/chgres_cube.hera index 9e7fcb753..72529a510 100644 --- a/modulefiles/chgres_cube.hera +++ b/modulefiles/chgres_cube.hera @@ -15,7 +15,7 @@ module load sigio/2.1.0 export NCEPLIBS=/scratch1/NCEPDEV/global/gwv/l819/lib module use -a $NCEPLIBS/modulefiles -module load esmflocal/8_0_44b +module load esmflocal/8_0_48b.netcdf47 export FCOMP=mpiifort export FFLAGS="-O3 -fp-model precise -g -traceback -r8 -i4 -qopenmp -convert big_endian -assume byterecl" diff --git a/modulefiles/fv3gfs/fre-nctools.hera b/modulefiles/fv3gfs/fre-nctools.hera index 4ebd982e8..e035db231 100644 --- a/modulefiles/fv3gfs/fre-nctools.hera +++ b/modulefiles/fv3gfs/fre-nctools.hera @@ -5,5 +5,5 @@ module load intel/18.0.5.274 module load impi/2018.0.4 -module load netcdf/4.6.1 -module load hdf5/1.10.4 +module load netcdf/4.7.0 +module load hdf5/1.10.5 diff --git a/modulefiles/fv3gfs/orog.hera b/modulefiles/fv3gfs/orog.hera index f21a1118e..0780cece2 100644 --- a/modulefiles/fv3gfs/orog.hera +++ b/modulefiles/fv3gfs/orog.hera @@ -4,7 +4,7 @@ # Loading Intel Compiler Suite module load intel/18.0.5.274 -module load netcdf/4.6.1 +module load netcdf/4.7.0 # Loding nceplibs modules module use -a $MOD_PATH diff --git a/modulefiles/modulefile.sfc_climo_gen.hera b/modulefiles/modulefile.sfc_climo_gen.hera index ecea921b4..93aff0a2e 100644 --- a/modulefiles/modulefile.sfc_climo_gen.hera +++ b/modulefiles/modulefile.sfc_climo_gen.hera @@ -5,8 +5,9 @@ module load intel/18.0.5.274 module load impi/2018.0.4 -module use -a /apps/modules/modulefamilies/intel_impi -module load esmf/7.1.0r +export NCEPLIBS=/scratch1/NCEPDEV/global/gwv/l819/lib +module use -a $NCEPLIBS/modulefiles +module load esmflocal/8_0_48b.netcdf47 export FCOMP=mpiifort export FFLAGS="-O3 -fp-model=precise -g -traceback -r8 -i4 -convert big_endian" From adf493debae792147859ab2e314d7bd7276857a6 Mon Sep 17 00:00:00 2001 From: GeorgeGayno-NOAA <52789452+GeorgeGayno-NOAA@users.noreply.github.com> Date: Wed, 2 Oct 2019 09:39:25 -0400 Subject: [PATCH 27/34] Feature/chgres cube reg tests (#21) develop: Create a set of regression tests for chgres_cube program. This commit references issue #5 . --- reg_tests/chgres_cube/README | 23 +++ reg_tests/chgres_cube/c192.fv3.history.sh | 76 ++++++++ reg_tests/chgres_cube/c96.fv3.nemsio.sh | 67 +++++++ reg_tests/chgres_cube/c96.fv3.restart.sh | 77 ++++++++ reg_tests/chgres_cube/c96.gfs.nemsio.sh | 71 +++++++ reg_tests/chgres_cube/c96.gfs.sigio.sh | 67 +++++++ reg_tests/chgres_cube/c96.regional.sh | 73 +++++++ reg_tests/chgres_cube/driver.cray.sh | 127 +++++++++++++ reg_tests/chgres_cube/driver.dell.sh | 121 ++++++++++++ reg_tests/chgres_cube/driver.hera.sh | 112 +++++++++++ reg_tests/chgres_cube/driver.jet.sh | 114 +++++++++++ ush/chgres_cube.sh | 222 ++++++++++++++++++++++ 12 files changed, 1150 insertions(+) create mode 100644 reg_tests/chgres_cube/README create mode 100755 reg_tests/chgres_cube/c192.fv3.history.sh create mode 100755 reg_tests/chgres_cube/c96.fv3.nemsio.sh create mode 100755 reg_tests/chgres_cube/c96.fv3.restart.sh create mode 100755 reg_tests/chgres_cube/c96.gfs.nemsio.sh create mode 100755 reg_tests/chgres_cube/c96.gfs.sigio.sh create mode 100755 reg_tests/chgres_cube/c96.regional.sh create mode 100755 reg_tests/chgres_cube/driver.cray.sh create mode 100755 reg_tests/chgres_cube/driver.dell.sh create mode 100755 reg_tests/chgres_cube/driver.hera.sh create mode 100755 reg_tests/chgres_cube/driver.jet.sh create mode 100755 ush/chgres_cube.sh diff --git a/reg_tests/chgres_cube/README b/reg_tests/chgres_cube/README new file mode 100644 index 000000000..49551cd28 --- /dev/null +++ b/reg_tests/chgres_cube/README @@ -0,0 +1,23 @@ +This series of regression tests must be run before any +changes to the chgres_cube program are merged to the +main line of development. Any test failures must +be resolved or explained before merging. + +To run the regression tests: + +1) Build chgres_cube program. Go to ./sorc and + invoke 'build_chgres_cube.sh' with no arguments. +2) Invoke driver script for your machine. See script + prolog for details. Supported machines are: + - Hera (driver.hera.sh) + - WCOSS-Cray (driver.cray.sh) + - WCOSS-Dell (driver.dell.sh) + - Jet (driver.jet.sh) + +A series of daisy-chained tests will run. A test fails +if the output files differ from the baseline set of files +as determined by the 'nccmp' utility. + +Log output from the tests will be in "regression.log". + +A summary of results will be in "summary.log". diff --git a/reg_tests/chgres_cube/c192.fv3.history.sh b/reg_tests/chgres_cube/c192.fv3.history.sh new file mode 100755 index 000000000..72776b833 --- /dev/null +++ b/reg_tests/chgres_cube/c192.fv3.history.sh @@ -0,0 +1,76 @@ +#!/bin/bash + +#----------------------------------------------------------------------------- +# Invoke chgres to create C192 coldstart files using FV3 tiled history files +# as input. The coldstart files are then compared to baseline files +# using the 'nccmp' utility. This script is run by the machine specific +# driver script. +#----------------------------------------------------------------------------- + +set -x + +export DATA=$OUTDIR/c192_fv3_history +rm -fr $DATA + +export CRES=192 +export FIXfv3=${HOMEreg}/fix/C192 +export COMIN=${HOMEreg}/input_data/fv3.history + +# Pay attention to the quotes. Dont start/end with double quote. +export ATM_FILES_INPUT='dynf000.tile1.nc","dynf000.tile2.nc","dynf000.tile3.nc","dynf000.tile4.nc","dynf000.tile5.nc","dynf000.tile6.nc' +export SFC_FILES_INPUT='phyf000.tile1.nc","phyf000.tile2.nc","phyf000.tile3.nc","phyf000.tile4.nc","phyf000.tile5.nc","phyf000.tile6.nc' +export VCOORD_FILE=${HOMEufs}/fix/fix_am/global_hyblev.l64.txt +export INPUT_TYPE='history' +export MOSAIC_FILE_INPUT_GRID="${HOMEreg}/fix/C96/C96_mosaic.nc" +export OROG_DIR_INPUT_GRID=${HOMEreg}/fix/C96 +export OROG_FILES_INPUT_GRID='C96_oro_data.tile1.nc","C96_oro_data.tile2.nc","C96_oro_data.tile3.nc","C96_oro_data.tile4.nc","C96_oro_data.tile5.nc","C96_oro_data.tile6.nc' +export TRACERS_TARGET='"sphum","liq_wat","o3mr"' +export TRACERS_INPUT='"spfh","clwmr","o3mr"' + +export CDATE=2018100300 + +export OMP_NUM_THREADS_CY=1 + +#----------------------------------------------------------------------------- +# Invoke chgres program. +#----------------------------------------------------------------------------- + +echo "Starting at: " `date` + +${HOMEufs}/ush/chgres_cube.sh + +iret=$? +if [ $iret -ne 0 ]; then + echo "<<< C96 FV3 GAUSSIAN NEMSIO TEST FAILED. <<<" + exit $iret +fi + +echo "Ending at: " `date` + +#----------------------------------------------------------------------------- +# Compare output from chgres to baseline set of data. +#----------------------------------------------------------------------------- + +cd $DATA + +test_failed=0 +for files in *.nc +do + if [ -f $files ]; then + echo CHECK $files + $NCCMP -dmfqS $files $HOMEreg/baseline_data/c192_fv3_history/$files + iret=$? + if [ $iret -ne 0 ]; then + test_failed=1 + fi + fi +done + +set +x +if [ $test_failed -ne 0 ]; then + echo "<<< C192 FV3 HISTORY TEST FAILED. >>>" +else + echo "<<< C192 FV3 HISTORY TEST PASSED. >>>" +fi + +exit 0 diff --git a/reg_tests/chgres_cube/c96.fv3.nemsio.sh b/reg_tests/chgres_cube/c96.fv3.nemsio.sh new file mode 100755 index 000000000..a056eabed --- /dev/null +++ b/reg_tests/chgres_cube/c96.fv3.nemsio.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +#----------------------------------------------------------------------------- +# Invoke chgres to create C96 coldstart files using FV3 gaussian nemsio files +# as input. The coldstart files are then compared to baseline files +# using the 'nccmp' utility. This script is run by the machine specific +# driver script. +#----------------------------------------------------------------------------- + +set -x + +export DATA=$OUTDIR/c96_fv3_nemsio +rm -fr $DATA + +export FIXfv3=${HOMEreg}/fix/C96 +export COMIN=${HOMEreg}/input_data/fv3.nemsio +export ATM_FILES_INPUT=gfs.t12z.atmf000.nemsio +export SFC_FILES_INPUT=gfs.t12z.sfcf000.nemsio +export VCOORD_FILE=${HOMEufs}/fix/fix_am/global_hyblev.l64.txt + +export CDATE=2019070412 + +export OMP_NUM_THREADS_CY=1 + +#----------------------------------------------------------------------------- +# Invoke chgres program. +#----------------------------------------------------------------------------- + +echo "Starting at: " `date` + +${HOMEufs}/ush/chgres_cube.sh + +iret=$? +if [ $iret -ne 0 ]; then + echo "<<< C96 FV3 GAUSSIAN NEMSIO TEST FAILED. <<<" + exit $iret +fi + +echo "Ending at: " `date` + +#----------------------------------------------------------------------------- +# Compare output from chgres to baseline set of data. +#----------------------------------------------------------------------------- + +cd $DATA + +test_failed=0 +for files in *.nc +do + if [ -f $files ]; then + echo CHECK $files + $NCCMP -dmfqS $files $HOMEreg/baseline_data/c96_fv3_nemsio/$files + iret=$? + if [ $iret -ne 0 ]; then + test_failed=1 + fi + fi +done + +set +x +if [ $test_failed -ne 0 ]; then + echo "<<< C96 FV3 GAUSSIAN NEMSIO TEST FAILED. >>>" +else + echo "<<< C96 FV3 GAUSSIAN NEMSIO TEST PASSED. >>>" +fi + +exit 0 diff --git a/reg_tests/chgres_cube/c96.fv3.restart.sh b/reg_tests/chgres_cube/c96.fv3.restart.sh new file mode 100755 index 000000000..e16f5f273 --- /dev/null +++ b/reg_tests/chgres_cube/c96.fv3.restart.sh @@ -0,0 +1,77 @@ +#!/bin/bash + +#----------------------------------------------------------------------------- +# Invoke chgres to create C96 coldstart files using FV3 tiled restart files +# as input. The coldstart files are then compared to baseline files +# using the 'nccmp' utility. This script is run by the machine specific +# driver script. +#----------------------------------------------------------------------------- + +set -x + +export DATA=$OUTDIR/c96_fv3_restart +rm -fr $DATA + +export FIXfv3=${HOMEreg}/fix/C96 +export COMIN=${HOMEreg}/input_data/fv3.restart +export VCOORD_FILE=${HOMEufs}/fix/fix_am/global_hyblev.l64.txt +export INPUT_TYPE='restart' +export MOSAIC_FILE_INPUT_GRID="${HOMEreg}/fix/C384/C384_mosaic.nc" +export OROG_DIR_INPUT_GRID=${HOMEreg}/fix/C384 + +# Note: no double quotes at the beginning or end. +export OROG_FILES_INPUT_GRID='C384_oro_data.tile1.nc","C384_oro_data.tile2.nc","C384_oro_data.tile3.nc","C384_oro_data.tile4.nc","C384_oro_data.tile5.nc","C384_oro_data.tile6.nc' +export ATM_CORE_FILES_INPUT='20190706.120000.fv_core.res.tile1.nc","20190706.120000.fv_core.res.tile2.nc","20190706.120000.fv_core.res.tile3.nc","20190706.120000.fv_core.res.tile4.nc","20190706.120000.fv_core.res.tile5.nc","20190706.120000.fv_core.res.tile6.nc","20190706.120000.fv_core.res.nc' +export ATM_TRACER_FILES_INPUT='20190706.120000.fv_tracer.res.tile1.nc","20190706.120000.fv_tracer.res.tile2.nc","20190706.120000.fv_tracer.res.tile3.nc","20190706.120000.fv_tracer.res.tile4.nc","20190706.120000.fv_tracer.res.tile5.nc","20190706.120000.fv_tracer.res.tile6.nc' +export SFC_FILES_INPUT='20190706.120000.sfc_data.tile1.nc","20190706.120000.sfc_data.tile2.nc","20190706.120000.sfc_data.tile3.nc","20190706.120000.sfc_data.tile4.nc","20190706.120000.sfc_data.tile5.nc","20190706.120000.sfc_data.tile6.nc' + +export TRACERS_TARGET='"sphum","liq_wat","o3mr","ice_wat","rainwat","snowwat","graupel"' +export TRACERS_INPUT='"sphum","liq_wat","o3mr","ice_wat","rainwat","snowwat","graupel"' + +export CDATE=2019070612 + +export OMP_NUM_THREADS_CY=1 + +#----------------------------------------------------------------------------- +# Invoke chgres program. +#----------------------------------------------------------------------------- + +echo "Starting at: " `date` + +${HOMEufs}/ush/chgres_cube.sh + +iret=$? +if [ $iret -ne 0 ]; then + echo "<<< C96 FV3 GAUSSIAN NEMSIO TEST FAILED. <<<" + exit $iret +fi + +echo "Ending at: " `date` + +#----------------------------------------------------------------------------- +# Compare output from chgres to baseline set of data. +#----------------------------------------------------------------------------- + +cd $DATA + +test_failed=0 +for files in *.nc +do + if [ -f $files ]; then + echo CHECK $files + $NCCMP -dmfqS $files $HOMEreg/baseline_data/c96_fv3_restart/$files + iret=$? + if [ $iret -ne 0 ]; then + test_failed=1 + fi + fi +done + +set +x +if [ $test_failed -ne 0 ]; then + echo "<<< C96 FV3 RESTART TEST FAILED. >>>" +else + echo "<<< C96 FV3 RESTART TEST PASSED. >>>" +fi + +exit 0 diff --git a/reg_tests/chgres_cube/c96.gfs.nemsio.sh b/reg_tests/chgres_cube/c96.gfs.nemsio.sh new file mode 100755 index 000000000..a3d3a4d59 --- /dev/null +++ b/reg_tests/chgres_cube/c96.gfs.nemsio.sh @@ -0,0 +1,71 @@ +#!/bin/bash + +#----------------------------------------------------------------------------- +# Invoke chgres to create C96 coldstart files using spectral GFS nemsio data +# as input. The coldstart files are then compared to baseline files +# using the 'nccmp' utility. This script is run by the machine specific +# driver script. +#----------------------------------------------------------------------------- + +set -x + +export DATA=$OUTDIR/c96_gfs_nemsio +rm -fr $DATA + +export FIXfv3=${HOMEreg}/fix/C96 +export COMIN=${HOMEreg}/input_data/gfs.nemsio +export ATM_FILES_INPUT=gfnanl.gdas.2017071700 +export SFC_FILES_INPUT=sfnanl.gdas.2017071700 +export NST_FILES_INPUT=nsnanl.gdas.2017071700 +export VCOORD_FILE=${HOMEufs}/fix/fix_am/global_hyblev.l64.txt +export INPUT_TYPE="gfs_gaussian" + +# dont start/end with double quotes. +export TRACERS_TARGET='"sphum","liq_wat","o3mr"' +export TRACERS_INPUT='"spfh","clwmr","o3mr"' +export CDATE=2017071700 +export OMP_NUM_THREADS_CY=1 + +#----------------------------------------------------------------------------- +# Invoke chgres program. +#----------------------------------------------------------------------------- + +echo "Starting at: " `date` + +${HOMEufs}/ush/chgres_cube.sh + +iret=$? +if [ $iret -ne 0 ]; then + echo "<<< C96 GFS GAUSSIAN NEMSIO TEST FAILED. <<<" + exit $iret +fi + +echo "Ending at: " `date` + +#----------------------------------------------------------------------------- +# Compare output from chgres to baseline set of data. +#----------------------------------------------------------------------------- + +cd $DATA + +test_failed=0 +for files in *.nc +do + if [ -f $files ]; then + echo CHECK $files + $NCCMP -dmfqS $files $HOMEreg/baseline_data/c96_gfs_nemsio/$files + iret=$? + if [ $iret -ne 0 ]; then + test_failed=1 + fi + fi +done + +set +x +if [ $test_failed -ne 0 ]; then + echo "<<< C96 GFS GAUSSIAN NEMSIO TEST FAILED. >>>" +else + echo "<<< C96 GFS GAUSSIAN NEMSIO TEST PASSED. >>>" +fi + +exit 0 diff --git a/reg_tests/chgres_cube/c96.gfs.sigio.sh b/reg_tests/chgres_cube/c96.gfs.sigio.sh new file mode 100755 index 000000000..c2ea31e52 --- /dev/null +++ b/reg_tests/chgres_cube/c96.gfs.sigio.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +#----------------------------------------------------------------------------- +# Invoke chgres to create C96 coldstart files using GFS sigio/sfcio files +# as input. The coldstart files are then compared to baseline files +# using the 'nccmp' utility. This script is run by the machine specific +# driver script. +#----------------------------------------------------------------------------- + +set -x + +export DATA=$OUTDIR/c96_gfs_sigio +rm -fr $DATA + +export FIXfv3=${HOMEreg}/fix/C96 +export COMIN=${HOMEreg}/input_data/gfs.sigio +export ATM_FILES_INPUT=gdas.t00z.sanl +export SFC_FILES_INPUT=gdas.t00z.sfcanl +export CONVERT_NST='.false.' +export VCOORD_FILE=${HOMEufs}/fix/fix_am/global_hyblev.l64.txt +export INPUT_TYPE="gfs_spectral" + +# dont start/end with double quotes +export TRACERS_TARGET='"sphum","o3mr","liq_wat"' +export TRACERS_INPUT='"spfh","o3mr","clwmr"' +export CDATE=2017071700 +export OMP_NUM_THREADS_CY=6 + +echo "Starting at: " `date` + +${HOMEufs}/ush/chgres_cube.sh + +iret=$? +if [ $iret -ne 0 ]; then + echo "<<< C96 GFS SIGIO TEST FAILED. <<<" + exit $iret +fi + +echo "Ending at: " `date` + +#----------------------------------------------------------------------------- +# Compare output from chgres to baseline set of data. +#----------------------------------------------------------------------------- + +cd $DATA + +test_failed=0 +for files in *.nc +do + if [ -f $files ]; then + echo CHECK $files + $NCCMP -dmfqS $files $HOMEreg/baseline_data/c96_gfs_sigio/$files + iret=$? + if [ $iret -ne 0 ]; then + test_failed=1 + fi + fi +done + +set +x +if [ $test_failed -ne 0 ]; then + echo "<<< C96 GFS SIGIO TEST FAILED. >>>" +else + echo "<<< C96 GFS SIGIO TEST PASSED. >>>" +fi + +exit 0 diff --git a/reg_tests/chgres_cube/c96.regional.sh b/reg_tests/chgres_cube/c96.regional.sh new file mode 100755 index 000000000..51a201f19 --- /dev/null +++ b/reg_tests/chgres_cube/c96.regional.sh @@ -0,0 +1,73 @@ +#!/bin/bash + +#----------------------------------------------------------------------------- +# Invoke chgres to create C96 regional coldstart files, including a lateral +# boundary file, using FV3 gaussian nemsio files as input. The +# coldstart files are then compared to baseline files using the 'nccmp' +# utility. +# +# This script is run by the machine specific driver script. +#----------------------------------------------------------------------------- + +set -x + +export DATA=$OUTDIR/c96_regional +rm -fr $DATA + +export FIXfv3=${HOMEreg}/fix/C96.regional +export OROG_FILES_TARGET_GRID="C96_oro_data.tile7.nc" +export COMIN=${HOMEreg}/input_data/fv3.nemsio +export ATM_FILES_INPUT=gfs.t12z.atmf000.nemsio +export SFC_FILES_INPUT=gfs.t12z.sfcf000.nemsio +export VCOORD_FILE=${HOMEufs}/fix/fix_am/global_hyblev.l64.txt +export REGIONAL=1 +export HALO_BLEND=0 +export HALO_BNDY=4 + +export CDATE=2019070412 + +export OMP_NUM_THREADS_CY=1 + +#----------------------------------------------------------------------------- +# Invoke chgres program. +#----------------------------------------------------------------------------- + +echo "Starting at: " `date` + +${HOMEufs}/ush/chgres_cube.sh + +iret=$? +if [ $iret -ne 0 ]; then + echo "<<< C96 REGIONAL TEST FAILED. <<<" + exit $iret +fi + +echo "Ending at: " `date` + +#----------------------------------------------------------------------------- +# Compare output from chgres to baseline set of data. +#----------------------------------------------------------------------------- + +cd $DATA + +test_failed=0 +for files in *.nc +do + if [ -f $files ]; then + echo CHECK $files + $NCCMP -dmfqS $files $HOMEreg/baseline_data/c96_regional/$files + iret=$? + if [ $iret -ne 0 ]; then + test_failed=1 + fi + fi +done + +set +x +if [ $test_failed -ne 0 ]; then + echo "<<< C96 REGIONAL TEST FAILED. >>>" +else + echo "<<< C96 REGIONAL TEST PASSED. >>>" +fi + +exit 0 diff --git a/reg_tests/chgres_cube/driver.cray.sh b/reg_tests/chgres_cube/driver.cray.sh new file mode 100755 index 000000000..e6f073256 --- /dev/null +++ b/reg_tests/chgres_cube/driver.cray.sh @@ -0,0 +1,127 @@ +#!/bin/bash + +#----------------------------------------------------------------------------- +# +# Run the chgres_cube regression tests on WCOSS-Cray. +# +# Set OUTDIR to your working directory. Set the PROJECT_CODE and QUEUE as +# appropriate. +# +# Invoke the script with no arguments. A series of daily- +# chained jobs will be submitted. To check the queue, type: "bjobs". +# +# The run output will be stored in OUTDIR. Log output from the suite +# will be in LOG_FILE. Once the suite has completed, a summary is +# placed in SUM_FILE. +# +# A test fails when its output does not match the baseline files as +# determined by the "nccmp" utility. The baseline files are stored in +# HOMEreg. +# +#----------------------------------------------------------------------------- + +set -x + +module purge +module load PrgEnv-intel/5.2.56 +module rm intel +module load intel/16.3.210 +module load cray-mpich/7.2.0 +module load craype-haswell +module load cray-netcdf +module load xt-lsfhpc/9.1.3 +module list + +export OUTDIR=/gpfs/hps3/stmp/$LOGNAME/chgres_reg_tests +QUEUE="debug" +PROJECT_CODE="FV3GFS-T2O" + +#----------------------------------------------------------------------------- +# Should not have to change anything below here. HOMEufs is the root +# directory of your UFS_UTILS clone. HOMEreg contains the input data +# and baseline data for each test. +#----------------------------------------------------------------------------- + +export HOMEufs=$PWD/../.. + +export HOMEreg=/gpfs/hps3/emc/global/noscrub/George.Gayno/ufs_utils.git/reg_tests/chgres_cube + +LOG_FILE=regression.log + +SUM_FILE=summary.log + +rm -f $LOG_FILE $SUM_FILE + +export NCCMP=/gpfs/hps3/emc/global/noscrub/George.Gayno/util/netcdf/nccmp + +export OMP_STACKSIZE=1024M + +export KMP_AFFINITY=disabled + +#----------------------------------------------------------------------------- +# Initialize C96 using FV3 warm restart files. +#----------------------------------------------------------------------------- + +export OMP_NUM_THREADS=1 +export APRUN="aprun -j 1 -n 6 -N 6 -d ${OMP_NUM_THREADS} -cc depth" +export INPUT_DATA=${HOMEreg}/input_data/fv3.restart +bsub -e $LOG_FILE -o $LOG_FILE -q $QUEUE -P $PROJECT_CODE -J c96.fv3.restart -M 1000 -W 0:15 -extsched 'CRAYLINUX[]' \ + "export NODES=1; $PWD/c96.fv3.restart.sh" + +#----------------------------------------------------------------------------- +# Initialize C192 using FV3 tiled history files. +#----------------------------------------------------------------------------- + +export OMP_NUM_THREADS=1 +export APRUN="aprun -j 1 -n 6 -N 6 -d ${OMP_NUM_THREADS} -cc depth" +export INPUT_DATA=${HOMEreg}/input_data/fv3.history +bsub -e $LOG_FILE -o $LOG_FILE -q $QUEUE -P $PROJECT_CODE -J c192.fv3.history -M 1000 -W 0:15 -extsched 'CRAYLINUX[]' \ + -w 'ended(c96.fv3.restart)' "export NODES=1; $PWD/c192.fv3.history.sh" + +#----------------------------------------------------------------------------- +# Initialize C96 using FV3 gaussian nemsio files. +#----------------------------------------------------------------------------- + +export OMP_NUM_THREADS=1 +export APRUN="aprun -j 1 -n 6 -N 6 -d ${OMP_NUM_THREADS} -cc depth" +export INPUT_DATA=${HOMEreg}/input_data/fv3.nemsio +bsub -e $LOG_FILE -o $LOG_FILE -q $QUEUE -P $PROJECT_CODE -J c96.fv3.nemsio -M 1000 -W 0:15 -extsched 'CRAYLINUX[]' \ + -w 'ended(c192.fv3.history)' "export NODES=1; $PWD/c96.fv3.nemsio.sh" + +#----------------------------------------------------------------------------- +# Initialize C96 using spectral GFS sigio/sfcio files. +#----------------------------------------------------------------------------- + +export OMP_NUM_THREADS=4 +export APRUN="aprun -j 1 -n 6 -N 6 -d ${OMP_NUM_THREADS} -cc depth" +export INPUT_DATA=${HOMEreg}/input_data/gfs.sigio +bsub -e $LOG_FILE -o $LOG_FILE -q $QUEUE -P $PROJECT_CODE -J c96.gfs.sigio -M 1000 -W 0:15 -extsched 'CRAYLINUX[]' \ + -w 'ended(c96.fv3.nemsio)' "export NODES=1; $PWD/c96.gfs.sigio.sh" + +#----------------------------------------------------------------------------- +# Initialize C96 using spectral GFS gaussian nemsio files. +#----------------------------------------------------------------------------- + +export OMP_NUM_THREADS=1 +export APRUN="aprun -j 1 -n 6 -N 6 -d ${OMP_NUM_THREADS} -cc depth" +export INPUT_DATA=${HOMEreg}/input_data/gfs.nemsio +bsub -e $LOG_FILE -o $LOG_FILE -q $QUEUE -P $PROJECT_CODE -J c96.gfs.nemsio -M 1000 -W 0:15 -extsched 'CRAYLINUX[]' \ + -w 'ended(c96.gfs.sigio)' "export NODES=1; $PWD/c96.gfs.nemsio.sh" + +#----------------------------------------------------------------------------- +# Initialize regional C96 using FV3 gaussian nemsio files. +#----------------------------------------------------------------------------- + +export OMP_NUM_THREADS=1 +export APRUN="aprun -j 1 -n 6 -N 6 -d ${OMP_NUM_THREADS} -cc depth" +export INPUT_DATA=${HOMEreg}/input_data/fv3.nemsio +bsub -e $LOG_FILE -o $LOG_FILE -q $QUEUE -P $PROJECT_CODE -J c96.regional -M 1000 -W 0:15 -extsched 'CRAYLINUX[]' \ + -w 'ended(c96.gfs.nemsio)' "export NODES=1; $PWD/c96.regional.sh" + +#----------------------------------------------------------------------------- +# Create summary log. +#----------------------------------------------------------------------------- + +bsub -o $LOG_FILE -q $QUEUE -P $PROJECT_CODE -J summary -R "rusage[mem=100]" -W 0:01 -w 'ended(c96.regional)' "grep -a '<<<' $LOG_FILE >> $SUM_FILE" + +exit diff --git a/reg_tests/chgres_cube/driver.dell.sh b/reg_tests/chgres_cube/driver.dell.sh new file mode 100755 index 000000000..e9369e2f9 --- /dev/null +++ b/reg_tests/chgres_cube/driver.dell.sh @@ -0,0 +1,121 @@ +#!/bin/bash + +#----------------------------------------------------------------------------- +# +# Run the chgres_cube regression tests on WCOSS-Dell. +# +# Set OUTDIR to your working directory. Set the PROJECT_CODE and QUEUE +# as appropriate. +# +# Invoke the script as follows with no arguments. A series of daily- +# chained jobs will be submitted. To check the queue, type: "bjobs". +# +# The run output will be stored in OUTDIR. Log output from the suite +# will be in LOG_FILE. Once the suite has completed, a summary is +# placed in SUM_FILE. +# +# A test fails when its output does not match the baseline files as +# determined by the "nccmp" utility. The baseline files are stored in +# HOMEreg. +# +#----------------------------------------------------------------------------- + +set -x + +module purge +module load EnvVars/1.0.2 +module load ips/18.0.1.163 +module load impi/18.0.1 +module load lsf/10.1 +module use /usrx/local/dev/modulefiles +module load NetCDF/4.5.0 +module list + +export OUTDIR=/gpfs/dell1/stmp/$LOGNAME/chgres_reg_tests +QUEUE="debug" +PROJECT_CODE="FV3GFS-T2O" + +#----------------------------------------------------------------------------- +# Should not have to change anything below here. HOMEufs is the root +# directory of your UFS_UTILS clone. HOMEreg contains the input data +# and baseline data for each test. +#----------------------------------------------------------------------------- + +export HOMEufs=$PWD/../.. + +export HOMEreg=/gpfs/dell2/emc/modeling/noscrub/George.Gayno/ufs_utils.git/reg_tests/chgres_cube + +LOG_FILE=regression.log + +SUM_FILE=summary.log + +rm -f $LOG_FILE $SUM_FILE + +export NCCMP=/gpfs/dell2/emc/modeling/noscrub/George.Gayno/util/nccmp/nccmp-1.8.5.0/src/nccmp + +export OMP_STACKSIZE=1024M + +export APRUN=mpirun + +#----------------------------------------------------------------------------- +# Initialize C96 using FV3 warm restart files. +#----------------------------------------------------------------------------- + +export OMP_NUM_THREADS=1 +export INPUT_DATA=${HOMEreg}/input_data/fv3.restart +bsub -e $LOG_FILE -o $LOG_FILE -q $QUEUE -P $PROJECT_CODE -J c96.fv3.restart -W 0:15 -x -n 6 \ + -R "span[ptile=6]" -R "affinity[core(${OMP_NUM_THREADS}):distribute=balance]" "$PWD/c96.fv3.restart.sh" + +#----------------------------------------------------------------------------- +# Initialize C192 using FV3 tiled history files. +#----------------------------------------------------------------------------- + +export OMP_NUM_THREADS=1 +export INPUT_DATA=${HOMEreg}/input_data/fv3.history +bsub -e $LOG_FILE -o $LOG_FILE -q $QUEUE -P $PROJECT_CODE -J c192.fv3.history -W 0:15 -x -n 6 -w 'ended(c96.fv3.restart)' \ + -R "span[ptile=6]" -R "affinity[core(${OMP_NUM_THREADS}):distribute=balance]" "$PWD/c192.fv3.history.sh" + +#----------------------------------------------------------------------------- +# Initialize C96 using FV3 gaussian nemsio files. +#----------------------------------------------------------------------------- + +export OMP_NUM_THREADS=1 +export INPUT_DATA=${HOMEreg}/input_data/fv3.nemsio +bsub -e $LOG_FILE -o $LOG_FILE -q $QUEUE -P $PROJECT_CODE -J c96.fv3.nemsio -W 0:15 -x -n 6 -w 'ended(c192.fv3.history)' \ + -R "span[ptile=6]" -R "affinity[core(${OMP_NUM_THREADS}):distribute=balance]" "$PWD/c96.fv3.nemsio.sh" + +#----------------------------------------------------------------------------- +# Initialize C96 using spectral GFS sigio/sfcio files. +#----------------------------------------------------------------------------- + +export OMP_NUM_THREADS=4 +export INPUT_DATA=${HOMEreg}/input_data/gfs.sigio +bsub -e $LOG_FILE -o $LOG_FILE -q $QUEUE -P $PROJECT_CODE -J c96.gfs.sigio -W 0:15 -x -n 6 -w 'ended(c96.fv3.nemsio)' \ + -R "span[ptile=6]" -R "affinity[core(${OMP_NUM_THREADS}):distribute=balance]" "$PWD/c96.gfs.sigio.sh" + +#----------------------------------------------------------------------------- +# Initialize C96 using spectral GFS gaussian nemsio files. +#----------------------------------------------------------------------------- + +export OMP_NUM_THREADS=1 +export INPUT_DATA=${HOMEreg}/input_data/gfs.nemsio +bsub -e $LOG_FILE -o $LOG_FILE -q $QUEUE -P $PROJECT_CODE -J c96.gfs.nemsio -W 0:15 -x -n 6 -w 'ended(c96.gfs.sigio)' \ + -R "span[ptile=6]" -R "affinity[core(${OMP_NUM_THREADS}):distribute=balance]" "$PWD/c96.gfs.nemsio.sh" + +#----------------------------------------------------------------------------- +# Initialize regional C96 using FV3 gaussian nemsio files. +#----------------------------------------------------------------------------- + +export OMP_NUM_THREADS=1 +export INPUT_DATA=${HOMEreg}/input_data/fv3.nemsio +bsub -e $LOG_FILE -o $LOG_FILE -q $QUEUE -P $PROJECT_CODE -J c96.regional -W 0:15 -x -n 6 -w 'ended(c96.gfs.nemsio)' \ + -R "span[ptile=6]" -R "affinity[core(${OMP_NUM_THREADS}):distribute=balance]" "$PWD/c96.regional.sh" + +#----------------------------------------------------------------------------- +# Create summary log. +#----------------------------------------------------------------------------- + +bsub -o $LOG_FILE -q $QUEUE -P $PROJECT_CODE -J summary -R "affinity[core(1)]" -R "rusage[mem=100]" -W 0:01 \ + -w 'ended(c96.regional)' "grep -a '<<<' $LOG_FILE >> $SUM_FILE" + +exit diff --git a/reg_tests/chgres_cube/driver.hera.sh b/reg_tests/chgres_cube/driver.hera.sh new file mode 100755 index 000000000..ba58b3b15 --- /dev/null +++ b/reg_tests/chgres_cube/driver.hera.sh @@ -0,0 +1,112 @@ +#!/bin/bash + +#----------------------------------------------------------------------------- +# +# Run the chgres_cube regression tests on Hera. +# +# Set OUTDIR to your working directory. Set the PROJECT_CODE and QUEUE +# as appropriate. To see which projects you are authorized to use, +# type "account_params". +# +# Invoke the script with no arguments. A series of daily-chained +# regression tests will be submitted. To check the queue, type: +# "squeue -u USERNAME". +# +# The run output will be stored in OUTDIR. Log output from the suite +# will be in LOG_FILE. Once the suite has completed, a summary is +# placed in SUM_FILE. +# +# A test fails when its output does not match the baseline files as +# determined by the "nccmp" utility. The baseline files are stored in +# HOMEreg. +# +#----------------------------------------------------------------------------- + +set -x + +source /apps/lmod/lmod/init/sh +module purge +module load intel/18.0.5.274 +module load impi/2018.0.4 +module load netcdf/4.7.0 +module list + +export OUTDIR=/scratch2/NCEPDEV/stmp1/$LOGNAME/chgres_reg_tests +PROJECT_CODE="fv3-cpu" +QUEUE="debug" + +#----------------------------------------------------------------------------- +# Should not have to change anything below here. HOMEufs is the root +# directory of your UFS_UTILS clone. HOMEreg contains the input data +# and baseline data for each test. +#----------------------------------------------------------------------------- + +export HOMEufs=$PWD/../.. + +export HOMEreg=/scratch1/NCEPDEV/da/George.Gayno/noscrub/reg_tests/chgres_cube + +export NCCMP=/apps/nccmp/1.8.5/intel/18.0.3.051/bin/nccmp + +LOG_FILE=regression.log +SUM_FILE=summary.log +rm -f $LOG_FILE $SUM_FILE + +export OMP_STACKSIZE=1024M + +export APRUN=srun + +rm -fr $OUTDIR + +#----------------------------------------------------------------------------- +# Initialize C96 using FV3 warm restart files. +#----------------------------------------------------------------------------- + +TEST1=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 -t 0:15:00 -A $PROJECT_CODE -q $QUEUE -J c96.fv3.restart \ + -o $LOG_FILE -e $LOG_FILE ./c96.fv3.restart.sh) + +#----------------------------------------------------------------------------- +# Initialize C192 using FV3 tiled history files. +#----------------------------------------------------------------------------- + +TEST2=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 -t 0:15:00 -A $PROJECT_CODE -q $QUEUE -J c192.fv3.history \ + -o $LOG_FILE -e $LOG_FILE -d afterok:$TEST1 ./c192.fv3.history.sh) + +#----------------------------------------------------------------------------- +# Initialize C96 using FV3 gaussian nemsio files. +#----------------------------------------------------------------------------- + +TEST3=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 -t 0:15:00 -A $PROJECT_CODE -q $QUEUE -J c96.fv3.nemsio \ + -o $LOG_FILE -e $LOG_FILE -d afterok:$TEST2 ./c96.fv3.nemsio.sh) + +#----------------------------------------------------------------------------- +# Initialize C96 using spectral GFS sigio/sfcio files. +#----------------------------------------------------------------------------- + +TEST4=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 -t 0:15:00 -A $PROJECT_CODE -q $QUEUE -J c96.gfs.sigio \ + -o $LOG_FILE -e $LOG_FILE -d afterok:$TEST3 ./c96.gfs.sigio.sh) + +#----------------------------------------------------------------------------- +# Initialize C96 using spectral GFS gaussian nemsio files. +#----------------------------------------------------------------------------- + +TEST5=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 -t 0:15:00 -A $PROJECT_CODE -q $QUEUE -J c96.gfs.nemsio \ + -o $LOG_FILE -e $LOG_FILE -d afterok:$TEST4 ./c96.gfs.nemsio.sh) + +#----------------------------------------------------------------------------- +# Initialize regional C96 using FV3 gaussian nemsio files. +#----------------------------------------------------------------------------- + +TEST6=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 -t 0:15:00 -A $PROJECT_CODE -q $QUEUE -J c96.regional \ + -o $LOG_FILE -e $LOG_FILE -d afterok:$TEST5 ./c96.regional.sh) + +#----------------------------------------------------------------------------- +# Create summary log. +#----------------------------------------------------------------------------- + +sbatch --nodes=1 -t 0:01:00 -A $PROJECT_CODE -J chgres_summary -o $LOG_FILE -e $LOG_FILE \ + --open-mode=append -q $QUEUE -d afterok:$TEST6 << EOF +#!/bin/sh +grep -a '<<<' $LOG_FILE > $SUM_FILE +EOF + +exit 0 diff --git a/reg_tests/chgres_cube/driver.jet.sh b/reg_tests/chgres_cube/driver.jet.sh new file mode 100755 index 000000000..6f76db74c --- /dev/null +++ b/reg_tests/chgres_cube/driver.jet.sh @@ -0,0 +1,114 @@ +#!/bin/bash + +#----------------------------------------------------------------------------- +# +# Run the chgres_cube regression tests on JET. +# +# Set OUTDIR to your working directory. Set the PROJECT_CODE and QUEUE +# as appropriate. To see which projects you are authorized to use, +# type "account_params". +# +# Invoke the script with no arguments. A series of daily- +# chained jobs will be submitted. To check the queue, type: +# "squeue -u USERNAME". +# +# The run output will be stored in OUTDIR. Log output from the suite +# will be in LOG_FILE. Once the suite has completed, a summary is +# placed in SUM_FILE. +# +# A test fails when its output does not match the baseline files as +# determined by the "nccmp" utility. The baseline files are stored in +# HOMEreg. +# +#----------------------------------------------------------------------------- + +set -x + +source /apps/lmod/lmod/init/sh +module purge +module load intel/15.0.3.187 +module load impi/2018.4.274 +module load szip +module load hdf5 +module load netcdf/4.2.1.1 +module list + +export OUTDIR=/mnt/lfs3/projects/emcda/$LOGNAME/stmp/chgres_reg_tests +PROJECT_CODE="emcda" +QUEUE="windfall" + +#----------------------------------------------------------------------------- +# Should not have to change anything below here. HOMEufs is the root +# directory of your UFS_UTILS clone. HOMEreg contains the input data +# and baseline data for each test. +#----------------------------------------------------------------------------- + +export HOMEufs=$PWD/../.. + +export HOMEreg=/mnt/lfs3/projects/emcda/George.Gayno/reg_tests/chgres_cube + +export NCCMP=/apps/nccmp/1.8.2.1/intel/18.0.3.222/bin/nccmp + +LOG_FILE=regression.log +SUM_FILE=summary.log +rm -f $LOG_FILE $SUM_FILE + +export OMP_STACKSIZE=1024M + +export APRUN=srun + +rm -fr $OUTDIR + +#----------------------------------------------------------------------------- +# Initialize C96 using FV3 warm restart files. +#----------------------------------------------------------------------------- + +TEST1=$(sbatch --parsable --partition=xjet --nodes=1 --ntasks-per-node=6 -t 0:10:00 -A $PROJECT_CODE -q $QUEUE -J c96.fv3.restart \ + -o $LOG_FILE -e $LOG_FILE ./c96.fv3.restart.sh) + +#----------------------------------------------------------------------------- +# Initialize C192 using FV3 tiled history files. +#----------------------------------------------------------------------------- + +TEST2=$(sbatch --parsable --partition=xjet --nodes=1 --ntasks-per-node=6 -t 0:10:00 -A $PROJECT_CODE -q $QUEUE -J c192.fv3.history \ + -o $LOG_FILE -e $LOG_FILE -d afterok:$TEST1 ./c192.fv3.history.sh) + +#----------------------------------------------------------------------------- +# Initialize C96 using FV3 gaussian nemsio files. +#----------------------------------------------------------------------------- + +TEST3=$(sbatch --parsable --partition=xjet --nodes=1 --ntasks-per-node=6 -t 0:10:00 -A $PROJECT_CODE -q $QUEUE -J c96.fv3.nemsio \ + -o $LOG_FILE -e $LOG_FILE -d afterok:$TEST2 ./c96.fv3.nemsio.sh) + +#----------------------------------------------------------------------------- +# Initialize C96 using spectral GFS sigio/sfcio files. +#----------------------------------------------------------------------------- + +TEST4=$(sbatch --parsable --partition=xjet --nodes=2 --ntasks-per-node=3 --cpus-per-task=6 -t 0:15:00 \ + -A $PROJECT_CODE -q $QUEUE -J c96.gfs.sigio -o $LOG_FILE -e $LOG_FILE -d afterok:$TEST3 ./c96.gfs.sigio.sh) + +#----------------------------------------------------------------------------- +# Initialize C96 using spectral GFS gaussian nemsio files. +#----------------------------------------------------------------------------- + +TEST5=$(sbatch --parsable --partition=xjet --nodes=1 --ntasks-per-node=6 -t 0:10:00 -A $PROJECT_CODE -q $QUEUE -J c96.gfs.nemsio \ + -o $LOG_FILE -e $LOG_FILE -d afterok:$TEST4 ./c96.gfs.nemsio.sh) + +#----------------------------------------------------------------------------- +# Initialize regional C96 using FV3 gaussian nemsio files. +#----------------------------------------------------------------------------- + +TEST6=$(sbatch --parsable --partition=xjet --nodes=1 --ntasks-per-node=6 -t 0:10:00 -A $PROJECT_CODE -q $QUEUE -J c96.regional \ + -o $LOG_FILE -e $LOG_FILE -d afterok:$TEST5 ./c96.regional.sh) + +#----------------------------------------------------------------------------- +# Create summary log. +#----------------------------------------------------------------------------- + +sbatch --partition=xjet --nodes=1 -t 0:01:00 -A $PROJECT_CODE -J chgres_summary -o $LOG_FILE -e $LOG_FILE \ + --open-mode=append -q $QUEUE -d afterok:$TEST6 << EOF +#!/bin/sh +grep -a '<<<' $LOG_FILE > $SUM_FILE +EOF + +exit 0 diff --git a/ush/chgres_cube.sh b/ush/chgres_cube.sh new file mode 100755 index 000000000..26ab7fc19 --- /dev/null +++ b/ush/chgres_cube.sh @@ -0,0 +1,222 @@ +#!/bin/bash + +#---------------------------------------------------------------------------- +# Script name: chgres_cube.sh +# +# Abstract: Run the chgres program to initialize an FV3 run. +# +# See comments for variable definitions and setup information. +#---------------------------------------------------------------------------- + +set -x + +#---------------------------------------------------------------------------- +# Resolution of target grid. +#---------------------------------------------------------------------------- + +CRES=${CRES:-96} + +#---------------------------------------------------------------------------- +# Set up environment paths. +# +# HOMEufs - Location of root ufs_utils directory. +# EXECufs - Location of ufs_utils executable directory. +# FIXufs - Location of ufs_utils root fixed data directory. +# FIXfv3 - Location of target grid orography and 'grid' files. +# FIXsfc - Location of target grid surface climatological files. +# FIXam - Location of vertical coordinate definition file for target grid. +#---------------------------------------------------------------------------- + +ufs_ver=${ufs_ver:-v1.0.0} +envir=${envir:-prod} +NWROOT=${NWROOT:-/nw${envir}} +HOMEufs=${HOMEufs:-${NWROOT}/ufs_util.${ufs_ver}} +EXECufs=${EXECufs:-$HOMEufs/exec} +FIXufs=${FIXufs:-$HOMEufs/fix} +FIXfv3=${FIXfv3:-$FIXufs/fix_fv3_gmted2010/C${CRES}} +FIXsfc=${FIXsfc:-$FIXfv3/fix_sfc} +FIXam=${FIXam:-$FIXufs/fix_am} + +#---------------------------------------------------------------------------- +# CDATE - YYMMDDHH of your run. +#---------------------------------------------------------------------------- + +CDATE=${CDATE:?} +iy=$(echo $CDATE|cut -c1-4) +im=$(echo $CDATE|cut -c5-6) +id=$(echo $CDATE|cut -c7-8) +ih=$(echo $CDATE|cut -c9-10) + +#---------------------------------------------------------------------------- +# Variables for stand-alone regional grids. +# +# REGIONAL - Set to 1 to create remove halo and create lateral boundary +# file. Set to 2 for lateral boundary file only. Set to +# 0 for non-regional grids. +# HALO_BNDY - Number of rows/cols for lateral boundaries. +# HALO_BLEND - Number of rows/cols for blending zone. +#---------------------------------------------------------------------------- + +REGIONAL=${REGIONAL:-0} +HALO_BNDY=${HALO_BNDY:-0} +HALO_BLEND=${HALO_BLEND:-0} + +#---------------------------------------------------------------------------- +# INPUT_TYPE - Input data type. 'history' for tiled fv3 history files. +# 'restart' for tiled fv3 warm restart files. 'gfs_gaussian' +# for spectral gfs nemsio files. 'gfs_spectral' for +# for spectral gfs sigio/sfcio files. 'gaussian' for fv3 +# gaussian nemsio files. +# +# MOSAIC_FILE_INPUT_GRID - Path/Name of mosaic file for input grid. Only +# used for 'history' and 'restart' INPUT_TYPE. +# Set to NULL otherwise. +# +# OROG_DIR_INPUT_GRID - Location of orography and grid files for input grid. +# Only used for 'history' and 'restart' INPUT_TYPE. +# Set to NULL otherwise. +# +# OROG_FILES_INPUT_GRID - List of orography files for input grid. Only +# used for 'history' and 'restart' INPUT_TYPE. +# Set to NULL otherwise. +#---------------------------------------------------------------------------- + +INPUT_TYPE=${INPUT_TYPE:-"gaussian"} +MOSAIC_FILE_INPUT_GRID=${MOSAIC_FILE_INPUT_GRID:-NULL} +OROG_DIR_INPUT_GRID=${OROG_DIR_INPUT_GRID:-NULL} +OROG_FILES_INPUT_GRID=${OROG_FILES_INPUT_GRID:-NULL} + +#---------------------------------------------------------------------------- +# COMIN - Location of input data +# CONVERT_ATM - Convert atmospheric fields when true +# CONVERT_SFC - Convert surface fields when true +# CONVERT_NST - Convert nst fields when true +#---------------------------------------------------------------------------- + +CONVERT_ATM=${CONVERT_ATM:-.true.} +CONVERT_SFC=${CONVERT_SFC:-.true.} +CONVERT_NST=${CONVERT_NST:-.true.} + +COMIN=${COMIN:-$PWD} + +#---------------------------------------------------------------------------- +# ATM_FILES_INPUT - Input atmospheric data file(s). Not used for 'restart' +# INPUT_TYPE. +# +# ATM_CORE_FILES - Input atmospheric core files. Used for 'restart' +# INPUT_TYPE only. The first six entries are the tiled +# files. The seventh is the file containing the +# vertical coord definition. +# +# ATM_TRACER_FILES_INPUT - Input atmospheric tracer files for each tile. +# Used for 'restart' INPUT_TYPE only. +# +# SFC_FILES_INPUT - Input surface data file(s). +# +# NST_FILES_INPUT - Input nst data file. 'gfs_gaussian' INPUT_TYPE only. +# +# TRACERS_INPUT - List of input atmospheric tracer records to be processed. +#---------------------------------------------------------------------------- + +ATM_FILES_INPUT=${ATM_FILES_INPUT:-NULL} +ATM_CORE_FILES_INPUT=${ATM_CORE_FILES_INPUT:-NULL} +ATM_TRACER_FILES_INPUT=${ATM_TRACER_FILES_INPUT:-NULL} +SFC_FILES_INPUT=${SFC_FILES_INPUT:-NULL} +NST_FILES_INPUT=${NST_FILES_INPUT:-NULL} +TRACERS_INPUT=${TRACERS_INPUT:-'"spfh","clwmr","o3mr","icmr","rwmr","snmr","grle"'} + +#---------------------------------------------------------------------------- +# TRACERS_TARGET - List of target tracer records. Must corresponde with +# with TRACERS_INPUT. +# +# VCOORD_FILE - File containing vertical coordinate defintion for target +# grid. +# +# MOSAIC FILE_TARGET_GRID - Mosaic file for target grid (include path). +# The associated 'grid' files assumed to be in +# FIXfv3. +# +# OROG_FILES_TARGET_GRID - Orography file(s) for target grid. Assumed to +# be located in FIXfv3. +#---------------------------------------------------------------------------- + +TRACERS_TARGET=${TRACERS_TARGET:-'"sphum","liq_wat","o3mr","ice_wat","rainwat","snowwat","graupel"'} + +VCOORD_FILE=${VCOORD_FILE:-${FIXam}/global_hyblev.l65.txt} + +MOSAIC_FILE_TARGET_GRID=${MOSAIC_FILE_TARGET_GRID:-${FIXfv3}/C${CRES}_mosaic.nc} + +OROG_FILES_TARGET_GRID=${OROG_FILES_TARGET_GRID:-NULL} +if [ $OROG_FILES_TARGET_GRID == NULL ]; then + OROG_FILES_TARGET_GRID='C'${CRES}'_oro_data.tile1.nc","C'${CRES}'_oro_data.tile2.nc"' + OROG_FILES_TARGET_GRID=${OROG_FILES_TARGET_GRID}',"C'${CRES}'_oro_data.tile3.nc","C'${CRES}'_oro_data.tile4.nc"' + OROG_FILES_TARGET_GRID=${OROG_FILES_TARGET_GRID}',"C'${CRES}'_oro_data.tile5.nc","C'${CRES}'_oro_data.tile6.nc' +fi + +#---------------------------------------------------------------------------- +# APRUN - machine specific command to run program. +# CHGRESEXEC - program executable. +# OMP_NUM_THREADS - threads most useful for 'gfs_spectral' INPUT_TYPE. +# DATA - working directory. +# PGMOUT - standard output file +# PGMERR - standard error file +# REDOUT - standard output redirect +# REDERR - standard error redirect +#---------------------------------------------------------------------------- + +APRUN=${APRUN:-time} +CHGRESEXEC=${CHGRESEXEC:-${EXECufs}/chgres_cube.exe} + +export OMP_NUM_THREADS=${OMP_NUM_THREADS_CY:-1} + +PGMOUT=${PGMOUT:-${pgmout:-'&1'}} +PGMERR=${PGMERR:-${pgmerr:-'&2'}} +REDOUT=${REDOUT:-'1>'} +REDERR=${REDERR:-'2>'} + +DATA=${DATA:-$PWD/chgres} +mkdir -p $DATA +cd $DATA || exit 99 + +rm -f ./fort.41 + +cat << EOF > ./fort.41 + &config + mosaic_file_target_grid="${MOSAIC_FILE_TARGET_GRID}" + fix_dir_target_grid="${FIXsfc}" + orog_dir_target_grid="${FIXfv3}" + orog_files_target_grid="${OROG_FILES_TARGET_GRID}" + vcoord_file_target_grid="${VCOORD_FILE}" + mosaic_file_input_grid="${MOSAIC_FILE_INPUT_GRID}" + orog_dir_input_grid="${OROG_DIR_INPUT_GRID}" + orog_files_input_grid="${OROG_FILES_INPUT_GRID}" + data_dir_input_grid="${COMIN}" + atm_files_input_grid="${ATM_FILES_INPUT}" + atm_core_files_input_grid="${ATM_CORE_FILES_INPUT}" + atm_tracer_files_input_grid="${ATM_TRACER_FILES_INPUT}" + sfc_files_input_grid="${SFC_FILES_INPUT}" + nst_files_input_grid="${NST_FILES_INPUT}" + cycle_mon=$im + cycle_day=$id + cycle_hour=$ih + convert_atm=$CONVERT_ATM + convert_sfc=$CONVERT_SFC + convert_nst=$CONVERT_NST + input_type="${INPUT_TYPE}" + tracers=$TRACERS_TARGET + tracers_input=$TRACERS_INPUT + regional=$REGIONAL + halo_bndy=$HALO_BNDY + halo_blend=$HALO_BLEND + / +EOF + +$APRUN $CHGRESEXEC $REDOUT$PGMOUT $REDERR$PGMERR + +iret=$? +if [ $iret -ne 0 ]; then + echo "FATAL ERROR RUNNING CHGRES" + exit $iret +fi + +exit From 445bb677ec2693bc788895e4b1bb8bb77db8890e Mon Sep 17 00:00:00 2001 From: GeorgeGayno-NOAA <52789452+GeorgeGayno-NOAA@users.noreply.github.com> Date: Thu, 10 Oct 2019 08:33:47 -0400 Subject: [PATCH 28/34] Feature/hera port (#25) develop: This commit references #1 Update repository to run on Hera, the replacement for Theia. Remove all references to Theia. --- ...iver_grid.theia.sh => driver_grid.hera.sh} | 16 +- modulefiles/chgres_cube.hera | 23 + modulefiles/chgres_cube.theia | 23 - .../{fre-nctools.theia => fre-nctools.hera} | 10 +- modulefiles/fv3gfs/global_chgres.hera | 29 + modulefiles/fv3gfs/global_chgres.theia | 31 - modulefiles/fv3gfs/global_chgres.wcoss | 2 +- .../{global_cycle.theia => global_cycle.hera} | 23 +- .../{nst_tf_chg.theia => nst_tf_chg.hera} | 16 +- modulefiles/fv3gfs/{orog.theia => orog.hera} | 15 +- modulefiles/module-setup.csh.inc | 89 --- modulefiles/module-setup.sh.inc | 4 +- modulefiles/module_base.hera | 11 + modulefiles/module_base.theia | 19 - ...le_nemsutil.theia => module_nemsutil.hera} | 10 +- .../modulefile.global_emcsfc_ice_blend.hera | 17 + .../modulefile.global_emcsfc_ice_blend.theia | 17 - .../modulefile.global_emcsfc_snow2mdl.hera | 23 + .../modulefile.global_emcsfc_snow2mdl.theia | 23 - ...en.theia => modulefile.sfc_climo_gen.hera} | 13 +- sorc/build_chgres.sh | 6 +- sorc/build_chgres_cube.sh | 2 +- sorc/build_cycle.sh | 2 +- sorc/build_emcsfc.sh | 1 + sorc/build_nems_util.sh | 4 +- sorc/build_nst_tf_chg.sh | 2 +- sorc/build_orog.sh | 4 +- sorc/build_sfc_climo_gen.sh | 1 + .../{make.csh_theia => make.csh_hera} | 2 +- .../env.theia => fregrid/env.hera} | 2 +- .../tools/fregrid/env.wcoss_dell_p3 | 4 +- ...e-nctools.mk_theia => fre-nctools.mk_hera} | 0 .../env.theia => make_hgrid/env.hera} | 2 +- ...e-nctools.mk_theia => fre-nctools.mk_hera} | 0 .../env.theia => make_solo_mosaic/env.hera} | 4 +- .../tools/make_solo_mosaic/env.wcoss_dell_p3 | 2 +- ...e-nctools.mk_theia => fre-nctools.mk_hera} | 0 .../fre-nctools.fd/tools/shave.fd/build_shave | 2 +- sorc/link_fixdirs.sh | 22 +- sorc/machine-setup.sh | 9 +- sorc/mkgfsnemsioctl.fd/makefile | 10 + sorc/mkgfsnemsioctl.fd/mkgfsnemsioctl.f | 535 ++++++++++++++++++ ush/fv3gfs_chgres.sh | 56 +- ush/load_fv3gfs_modules.sh | 6 +- util/sub_slurm | 144 +++++ util/sub_wcoss_c | 277 +++++++++ util/sub_wcoss_d | 247 ++++++++ 47 files changed, 1420 insertions(+), 340 deletions(-) rename driver_scripts/{driver_grid.theia.sh => driver_grid.hera.sh} (93%) create mode 100644 modulefiles/chgres_cube.hera delete mode 100644 modulefiles/chgres_cube.theia rename modulefiles/fv3gfs/{fre-nctools.theia => fre-nctools.hera} (59%) create mode 100644 modulefiles/fv3gfs/global_chgres.hera delete mode 100644 modulefiles/fv3gfs/global_chgres.theia rename modulefiles/fv3gfs/{global_cycle.theia => global_cycle.hera} (55%) rename modulefiles/fv3gfs/{nst_tf_chg.theia => nst_tf_chg.hera} (65%) rename modulefiles/fv3gfs/{orog.theia => orog.hera} (54%) delete mode 100644 modulefiles/module-setup.csh.inc create mode 100644 modulefiles/module_base.hera delete mode 100644 modulefiles/module_base.theia rename modulefiles/{module_nemsutil.theia => module_nemsutil.hera} (66%) create mode 100644 modulefiles/modulefile.global_emcsfc_ice_blend.hera delete mode 100644 modulefiles/modulefile.global_emcsfc_ice_blend.theia create mode 100644 modulefiles/modulefile.global_emcsfc_snow2mdl.hera delete mode 100644 modulefiles/modulefile.global_emcsfc_snow2mdl.theia rename modulefiles/{modulefile.sfc_climo_gen.theia => modulefile.sfc_climo_gen.hera} (50%) rename sorc/fre-nctools.fd/tools/filter_topo/{make.csh_theia => make.csh_hera} (81%) rename sorc/fre-nctools.fd/tools/{make_hgrid/env.theia => fregrid/env.hera} (85%) rename sorc/fre-nctools.fd/tools/fregrid/{fre-nctools.mk_theia => fre-nctools.mk_hera} (100%) rename sorc/fre-nctools.fd/tools/{make_solo_mosaic/env.theia => make_hgrid/env.hera} (85%) rename sorc/fre-nctools.fd/tools/make_hgrid/{fre-nctools.mk_theia => fre-nctools.mk_hera} (100%) rename sorc/fre-nctools.fd/tools/{fregrid/env.theia => make_solo_mosaic/env.hera} (64%) rename sorc/fre-nctools.fd/tools/make_solo_mosaic/{fre-nctools.mk_theia => fre-nctools.mk_hera} (100%) create mode 100755 sorc/mkgfsnemsioctl.fd/makefile create mode 100644 sorc/mkgfsnemsioctl.fd/mkgfsnemsioctl.f create mode 100755 util/sub_slurm create mode 100755 util/sub_wcoss_c create mode 100755 util/sub_wcoss_d diff --git a/driver_scripts/driver_grid.theia.sh b/driver_scripts/driver_grid.hera.sh similarity index 93% rename from driver_scripts/driver_grid.theia.sh rename to driver_scripts/driver_grid.hera.sh index 88096511d..ad93e30e7 100644 --- a/driver_scripts/driver_grid.theia.sh +++ b/driver_scripts/driver_grid.hera.sh @@ -10,7 +10,7 @@ #SBATCH -t 00:30:00 #----------------------------------------------------------------------- -# Driver script to create a cubic-sphere based model grid on Theia. +# Driver script to create a cubic-sphere based model grid on Hera. # # Produces the following files (netcdf, each tile in separate file): # 1) 'mosaic' and 'grid' files containing lat/lon and other @@ -53,10 +53,10 @@ set -x . /apps/lmod/lmod/init/sh module purge -module load intel/16.1.150 -module load impi -module load hdf5/1.8.14 -module load netcdf/4.3.0 +module load intel/18.0.5.274 +module load impi/2018.0.4 +module load hdf5/1.10.5 +module load netcdf/4.7.0 module list #----------------------------------------------------------------------- @@ -90,8 +90,8 @@ fi #----------------------------------------------------------------------- export home_dir=$SLURM_SUBMIT_DIR/.. -export TMPDIR=/scratch3/NCEPDEV/stmp1/$LOGNAME/fv3_grid.$gtype -export out_dir=/scratch3/NCEPDEV/stmp1/$LOGNAME/C${res} +export TMPDIR=/scratch2/NCEPDEV/stmp1/$LOGNAME/fv3_grid.$gtype +export out_dir=/scratch2/NCEPDEV/stmp1/$LOGNAME/C${res} #----------------------------------------------------------------------- # Should not need to change anything below here. @@ -101,7 +101,7 @@ export APRUN=time export APRUN_SFC=srun export OMP_NUM_THREADS=24 export OMP_STACKSIZE=2048m -export machine=THEIA +export machine=HERA ulimit -a ulimit -s unlimited diff --git a/modulefiles/chgres_cube.hera b/modulefiles/chgres_cube.hera new file mode 100644 index 000000000..72529a510 --- /dev/null +++ b/modulefiles/chgres_cube.hera @@ -0,0 +1,23 @@ +#%Module##################################################### +## Build module for Hera +############################################################# + +module load intel/18.0.5.274 +module load impi/2018.0.4 + +module use -a $MOD_PATH +module load w3nco/2.0.6 +module load nemsio/2.2.3 +module load bacio/2.0.2 +module load sp/2.0.2 +module load sfcio/1.1.0 +module load sigio/2.1.0 + +export NCEPLIBS=/scratch1/NCEPDEV/global/gwv/l819/lib +module use -a $NCEPLIBS/modulefiles +module load esmflocal/8_0_48b.netcdf47 + +export FCOMP=mpiifort +export FFLAGS="-O3 -fp-model precise -g -traceback -r8 -i4 -qopenmp -convert big_endian -assume byterecl" +# for debugging +#export FFLAGS="-O0 -g -traceback -r8 -i4 -qopenmp -convert big_endian -check bounds -warn unused -assume byterecl" diff --git a/modulefiles/chgres_cube.theia b/modulefiles/chgres_cube.theia deleted file mode 100644 index 2b3548ba0..000000000 --- a/modulefiles/chgres_cube.theia +++ /dev/null @@ -1,23 +0,0 @@ -#%Module##################################################### -## chgres build module for Theia -############################################################# - -module load intel/15.1.133 -module load impi/5.1.1.109 -module load netcdf/4.3.0 - -module use /scratch4/NCEPDEV/nems/noscrub/emc.nemspara/soft/modulefiles -module load esmf/8.0.0bs20 - -module use -a /scratch3/NCEPDEV/nwprod/lib/modulefiles -module load w3nco -module load nemsio/v2.2.3 -module load bacio/v2.0.1 -module load sp/v2.0.2 -module load sfcio/v1.0.0 -module load sigio/v2.0.1 - -export FCOMP=mpiifort -export FFLAGS="-O3 -fp-model precise -g -traceback -r8 -i4 -qopenmp -convert big_endian -assume byterecl" -# for debugging -#export FFLAGS="-O0 -g -traceback -r8 -i4 -qopenmp -convert big_endian -check bounds -warn unused -assume byterecl" diff --git a/modulefiles/fv3gfs/fre-nctools.theia b/modulefiles/fv3gfs/fre-nctools.hera similarity index 59% rename from modulefiles/fv3gfs/fre-nctools.theia rename to modulefiles/fv3gfs/fre-nctools.hera index c1b678411..e035db231 100644 --- a/modulefiles/fv3gfs/fre-nctools.theia +++ b/modulefiles/fv3gfs/fre-nctools.hera @@ -1,7 +1,9 @@ #%Module##################################################### ## Module file for fre-nctools ############################################################# -module load intel/14.0.2 -module load impi/4.1.3.048 -module load netcdf/4.3.0 -module load hdf5/1.8.14 + +module load intel/18.0.5.274 +module load impi/2018.0.4 + +module load netcdf/4.7.0 +module load hdf5/1.10.5 diff --git a/modulefiles/fv3gfs/global_chgres.hera b/modulefiles/fv3gfs/global_chgres.hera new file mode 100644 index 000000000..b91ac5e09 --- /dev/null +++ b/modulefiles/fv3gfs/global_chgres.hera @@ -0,0 +1,29 @@ +#%Module##################################################### +## global_chgres component - hera +############################################################# + +# Loading Intel Compiler Suite +module load intel/18.0.5.274 +module load impi/2018.0.4 + +module load netcdf/4.7.0 +module load hdf5/1.10.5 + +# Loding nceplibs modules +module use -a $MOD_PATH +module load sigio/2.1.0 +module load w3nco/2.0.6 +module load w3emc/2.3.0 +module load sp/2.0.2 +module load bacio/2.0.2 +module load nemsio/2.2.3 +module load nemsiogfs/2.2.0 +module load ip/3.0.1 +module load sfcio/1.1.0 +module load gfsio/1.1.0 +module load landsfcutil/2.1.0 + +export NETCDF_INCLUDE="-I${NETCDF}/include" +export NETCDF_LDFLAGS_F="-L${NETCDF}/lib -lnetcdf -lnetcdff -L${HDF5}/lib -lhdf5 -lhdf5_fortran" + +set FCMP ifort diff --git a/modulefiles/fv3gfs/global_chgres.theia b/modulefiles/fv3gfs/global_chgres.theia deleted file mode 100644 index fa764895f..000000000 --- a/modulefiles/fv3gfs/global_chgres.theia +++ /dev/null @@ -1,31 +0,0 @@ -#%Module##################################################### -## global_chgres component - theia -############################################################# - -# Loading Intel Compiler Suite -module load intel/14.0.2 -module load impi/5.1.2.150 - -# Loding nceplibs modules -module use -a $MOD_PATH -module load sigio/v2.0.1 -module load w3nco/v2.0.6 -module load w3emc/v2.2.0 -module load sp/v2.0.2 -module load bacio/v2.0.2 -module load nemsio/v2.2.3 -module load nemsiogfs/v2.0.1 -export NEMSIOGFS_INC=/scratch4/NCEPDEV/global/save/glopara/svn/nceplibs/branches/nemsiogfsv2.0.1/include/nemsiogfs -export NEMSIOGFS_LIB=/scratch4/NCEPDEV/global/save/glopara/svn/nceplibs/branches/nemsiogfsv2.0.1/libnemsiogfs.a - -module load ip/v3.0.0 -module load sfcio/v1.0.0 -module load gfsio/v1.1.0 -module load landsfcutil/v2.1.0 -module load netcdf/4.3.0 -module load hdf5/1.8.14 - -export NETCDF_INCLUDE="-I${NETCDF}/include" -export NETCDF_LDFLAGS_F="-L${NETCDF}/lib -lnetcdf -lnetcdff -L${HDF5}/lib -lhdf5 -lhdf5_fortran" - -set FCMP ifort diff --git a/modulefiles/fv3gfs/global_chgres.wcoss b/modulefiles/fv3gfs/global_chgres.wcoss index 80e60faaf..1d30465a5 100644 --- a/modulefiles/fv3gfs/global_chgres.wcoss +++ b/modulefiles/fv3gfs/global_chgres.wcoss @@ -3,7 +3,7 @@ ############################################################# # Loading Intel Compiler Suite -module load ics/14.0.1 +module load ics/15.0.6 # Loading nceplibs modules module load sigio/v2.1.0 diff --git a/modulefiles/fv3gfs/global_cycle.theia b/modulefiles/fv3gfs/global_cycle.hera similarity index 55% rename from modulefiles/fv3gfs/global_cycle.theia rename to modulefiles/fv3gfs/global_cycle.hera index 2f0f82cec..5f3afa8d5 100644 --- a/modulefiles/fv3gfs/global_cycle.theia +++ b/modulefiles/fv3gfs/global_cycle.hera @@ -1,20 +1,21 @@ #%Module##################################################### -## global_cycle component - theia +## global_cycle component - hera ############################################################# # Loading Intel Compiler Suite -module load intel/16.1.150 -module load impi +module load intel/18.0.5.274 +module load impi/2018.0.4 -# Loding nceplibs modules -module use -a /scratch3/NCEPDEV/nwprod/lib/modulefiles -module load w3nco/v2.0.6 -module load sp/v2.0.2 -module load bacio/v2.0.1 -module load ip/v3.0.0 -module load netcdf/4.3.0 -module load hdf5/1.8.14 +module load netcdf/4.7.0 +module load hdf5/1.10.5 export NETCDF_INCLUDE="-I${NETCDF}/include" export NETCDF_LDFLAGS_F="-L${NETCDF}/lib -lnetcdf -lnetcdff -L${HDF5}/lib -lhdf5 -lhdf5_fortran" +# Loding nceplibs modules +module use -a $MOD_PATH +module load w3nco/2.0.6 +module load sp/2.0.2 +module load bacio/2.0.2 +module load ip/3.0.1 + export FCMP=mpiifort diff --git a/modulefiles/fv3gfs/nst_tf_chg.theia b/modulefiles/fv3gfs/nst_tf_chg.hera similarity index 65% rename from modulefiles/fv3gfs/nst_tf_chg.theia rename to modulefiles/fv3gfs/nst_tf_chg.hera index 9a8829df8..2bbd3a802 100644 --- a/modulefiles/fv3gfs/nst_tf_chg.theia +++ b/modulefiles/fv3gfs/nst_tf_chg.hera @@ -1,17 +1,17 @@ #%Module##################################################### -## gaussian_sfcanl build module for Theia +## Build module for Hera ############################################################# # Loading Intel Compiler Suite -module load intel/15.1.133 +module load intel/18.0.5.274 # Loding nceplibs modules -module use -a /scratch3/NCEPDEV/nwprod/lib/modulefiles -module load w3nco/v2.0.6 -module load bacio/v2.0.2 -module load nemsio/v2.2.3 -module load netcdf/4.3.0 -module load hdf5/1.8.14 +module use -a $MOD_PATH +module load w3nco/2.0.6 +module load bacio/2.0.2 +module load nemsio/2.2.3 +module load netcdf/4.6.1 +module load hdf5/1.10.4 export NETCDF_INCLUDE="-I${NETCDF}/include" export NETCDF_LDFLAGS_F="-L${NETCDF}/lib -lnetcdf -lnetcdff -L${HDF5}/lib -lhdf5 -lhdf5_fortran" diff --git a/modulefiles/fv3gfs/orog.theia b/modulefiles/fv3gfs/orog.hera similarity index 54% rename from modulefiles/fv3gfs/orog.theia rename to modulefiles/fv3gfs/orog.hera index b0543289b..0780cece2 100644 --- a/modulefiles/fv3gfs/orog.theia +++ b/modulefiles/fv3gfs/orog.hera @@ -2,15 +2,14 @@ ## Module file for orog ############################################################# # Loading Intel Compiler Suite -module unload intel -module load intel/16.1.150 +module load intel/18.0.5.274 -module load netcdf/4.3.0 +module load netcdf/4.7.0 # Loding nceplibs modules module use -a $MOD_PATH -module load ip/v2.0.0 -module load sp/v2.0.2 -module load w3emc/v2.2.0 -module load w3nco/v2.0.6 -module load bacio/v2.0.2 +module load ip/3.0.1 +module load sp/2.0.2 +module load w3emc/2.3.0 +module load w3nco/2.0.6 +module load bacio/2.0.2 diff --git a/modulefiles/module-setup.csh.inc b/modulefiles/module-setup.csh.inc deleted file mode 100644 index a93b21a3c..000000000 --- a/modulefiles/module-setup.csh.inc +++ /dev/null @@ -1,89 +0,0 @@ -set __ms_shell=csh - -eval "if ( -d / ) set __ms_shell=tcsh" - -if ( { test -d /lfs3 } ) then - if ( ! { module help >& /dev/null } ) then - source /apps/lmod/lmod/init/$__ms_shell - endif - module purge -else if ( { test -d /scratch3 } ) then - # We are on NOAA Theia - if ( ! { module help >& /dev/null } ) then - source /apps/lmod/lmod/init/$__ms_shell - endif - module purge -else if ( { test -d /gpfs/hps -a -e /etc/SuSE-release } ) then - # We are on NOAA Luna or Surge - if ( ! { module help >& /dev/null } ) then - source /opt/modules/default/init/$__ms_shell - endif - module purge - module purge - unset _LMFILES_ - unset LOADEDMODULES - module use /opt/modulefiles - module use /opt/cray/ari/modulefiles - module use /opt/cray/craype/default/alt-modulefiles - module use /opt/cray/alt-modulefiles - module use /gpfs/hps/nco/ops/nwprod/modulefiles - module use /gpfs/hps/nco/ops/nwprod/lib/modulefiles - module use /usrx/local/prod/modulefiles -else if ( { test -d /dcom -a -d /hwrf } ) then - # We are on NOAA Tide or Gyre - if ( ! { module help >& /dev/null } ) then - source /usrx/local/Modules/default/init/$__ms_shell - endif - module purge -else if ( { test -L /usrx && sh -c "readlink /usrx 2> /dev/null | grep dell" } ) then - # We are on WCOSS Mars or Venus - if ( ! { module help >& /dev/null } ) then - source /usrx/local/prod/lmod/lmod/init/$__ms_shell - endif - module purge -else if ( { test -d /glade } ) then - # We are on NCAR Yellowstone - if ( ! { module help >& /dev/null } ) then - source /usr/share/Modules/init/$__ms_shell - endif - module purge -else if ( { test -d /lustre -a -d /ncrc } ) then - # We are on GAEA. - if ( ! { module help >& /dev/null } ) then - # We cannot simply load the module command. The GAEA - # /etc/csh.login modifies a number of module-related variables - # before loading the module command. Without those variables, - # the module command fails. Hence we actually have to source - # /etc/csh.login here. - source /etc/csh.login - set __ms_source_etc_csh_login=yes - else - set __ms_source_etc_csh_login=no - endif - module purge - unsetenv _LMFILES_ - unsetenv _LMFILES_000 - unsetenv _LMFILES_001 - unsetenv LOADEDMODULES - module load modules - if ( { test -d /opt/cray/ari/modulefiles } ) then - module use -a /opt/cray/ari/modulefiles - endif - if ( { test -d /opt/cray/pe/ari/modulefiles } ) then - module use -a /opt/cray/pe/ari/modulefiles - endif - if ( { test -d /opt/cray/pe/craype/default/modulefiles } ) then - module use -a /opt/cray/pe/craype/default/modulefiles - endif - setenv NCEPLIBS /lustre/f1/pdata/ncep_shared/NCEPLIBS/lib - if ( { test -d /lustre/f1/pdata/ncep_shared/NCEPLIBS/lib } ) then - module use $NCEPLIBS/modulefiles - endif - if ( "$__ms_source_etc_csh_login" == yes ) then - source /etc/csh.login - unset __ms_source_etc_csh_login - endif -else - # Workaround for csh limitation. Use sh to print to stderr. - sh -c 'echo WARNING: UNKNOWN PLATFORM 1>&2' -endif diff --git a/modulefiles/module-setup.sh.inc b/modulefiles/module-setup.sh.inc index 6815b1a46..d4faa4f64 100644 --- a/modulefiles/module-setup.sh.inc +++ b/modulefiles/module-setup.sh.inc @@ -22,8 +22,8 @@ if [[ -d /lfs3 ]] ; then source /apps/lmod/lmod/init/$__ms_shell fi module purge -elif [[ -d /scratch3 ]] ; then - # We are on NOAA Theia +elif [[ -d /scratch1 ]] ; then + # We are on NOAA Hera if ( ! eval module help > /dev/null 2>&1 ) ; then source /apps/lmod/lmod/init/$__ms_shell fi diff --git a/modulefiles/module_base.hera b/modulefiles/module_base.hera new file mode 100644 index 000000000..07e78cfc9 --- /dev/null +++ b/modulefiles/module_base.hera @@ -0,0 +1,11 @@ +#%Module###################################################################### +## +## FV3GFS prerequisites +## + +module load intel/18.0.5.274 +module load impi/2018.0.4 +module load hpss/hpss +module load netcdf/4.7.0 +module load nco/4.7.0 +module load hdf5/1.10.5 diff --git a/modulefiles/module_base.theia b/modulefiles/module_base.theia deleted file mode 100644 index 13fad8fd5..000000000 --- a/modulefiles/module_base.theia +++ /dev/null @@ -1,19 +0,0 @@ -#%Module###################################################################### -## -## FV3GFS prerequisites -## - -module load intel/16.1.150 -module load impi/5.1.2.150 -module load wgrib2 -module load hpss -module load netcdf -module load nco -module load hdf5/1.8.14 -module load gempak/6.7.0 - -module use /scratch3/NCEPDEV/nwprod/lib/modulefiles -module load EnvVars/v1.0.2 - -module use /scratch4/NCEPDEV/nems/noscrub/emc.nemspara/soft/modulefiles -module load prod_util diff --git a/modulefiles/module_nemsutil.theia b/modulefiles/module_nemsutil.hera similarity index 66% rename from modulefiles/module_nemsutil.theia rename to modulefiles/module_nemsutil.hera index 45685118e..4d1bfd5c4 100644 --- a/modulefiles/module_nemsutil.theia +++ b/modulefiles/module_nemsutil.hera @@ -3,13 +3,13 @@ ############################################################# # Loading Intel Compiler Suite -module load intel/14.0.2 -module load impi/5.1.2.150 +module load intel/18.0.5.274 +module load impi/2018.0.4 # Loding nceplibs modules module use -a $MOD_PATH -module load w3nco/v2.0.6 -module load bacio/v2.0.1 -module load nemsio/v2.2.1 +module load w3nco/2.0.6 +module load bacio/2.0.2 +module load nemsio/2.2.3 export FCMP=ifort diff --git a/modulefiles/modulefile.global_emcsfc_ice_blend.hera b/modulefiles/modulefile.global_emcsfc_ice_blend.hera new file mode 100644 index 000000000..2305392cf --- /dev/null +++ b/modulefiles/modulefile.global_emcsfc_ice_blend.hera @@ -0,0 +1,17 @@ +#%Module##################################################### +## emcsfc_ice_blend build module for Hera +############################################################# + +module load intel/18.0.5.274 +export FCOMP=ifort +export FFLAGS="-O0 -i4" + +# load ncep library modules + +module use -a $MOD_PATH +module load w3nco/2.0.6 +module load bacio/2.0.2 +module load jasper/1.900.1 +module load z/1.2.11 +module load png/1.2.44 +module load g2/2.5.0 diff --git a/modulefiles/modulefile.global_emcsfc_ice_blend.theia b/modulefiles/modulefile.global_emcsfc_ice_blend.theia deleted file mode 100644 index d42fb6ac8..000000000 --- a/modulefiles/modulefile.global_emcsfc_ice_blend.theia +++ /dev/null @@ -1,17 +0,0 @@ -#%Module##################################################### -## emcsfc_ice_blend build module for Theia -############################################################# - -module load intel/14.0.2 -export FCOMP=ifort -export FFLAGS="-O0 -i4" - -# load ncep library modules - -module use -a /scratch3/NCEPDEV/nwprod/lib/modulefiles -module load w3nco/v2.0.6 -module load bacio/v2.0.1 -module load jasper -module load z -module load png -module load g2/v2.5.0 diff --git a/modulefiles/modulefile.global_emcsfc_snow2mdl.hera b/modulefiles/modulefile.global_emcsfc_snow2mdl.hera new file mode 100644 index 000000000..193f6247d --- /dev/null +++ b/modulefiles/modulefile.global_emcsfc_snow2mdl.hera @@ -0,0 +1,23 @@ +#%Module##################################################### +## emcsfc_snow2mdl build module for Hera +############################################################# + +# load intel compiler + +module load intel/18.0.5.274 + +# load ncep library modules + +module use -a $MOD_PATH +module load ip/3.0.1 +module load sp/2.0.2 +module load w3nco/2.0.6 +module load bacio/2.0.2 +module load landsfcutil/2.1.0 +module load jasper/1.900.1 +module load z/1.2.11 +module load png/1.2.44 +module load g2/2.5.0 + +export FCOMP=ifort +export FFLAGS="-O0 -r8 -i4 -FR -I${IP_INCd} -qopenmp -convert big_endian -assume byterecl" diff --git a/modulefiles/modulefile.global_emcsfc_snow2mdl.theia b/modulefiles/modulefile.global_emcsfc_snow2mdl.theia deleted file mode 100644 index b364fbe47..000000000 --- a/modulefiles/modulefile.global_emcsfc_snow2mdl.theia +++ /dev/null @@ -1,23 +0,0 @@ -#%Module##################################################### -## emcsfc_snow2mdl build module for Theia -############################################################# - -# load intel compiler - -module load intel/14.0.2 - -# load ncep library modules - -module use -a /scratch3/NCEPDEV/nwprod/lib/modulefiles -module load ip/v3.0.0 -module load sp/v2.0.2 -module load w3nco/v2.0.6 -module load bacio/v2.0.1 -module load jasper -module load z -module load png -module load g2/v2.5.0 -module load landsfcutil/v2.1.0 - -export FCOMP=ifort -export FFLAGS="-O0 -r8 -i4 -FR -I${IP_INCd} -openmp -convert big_endian -assume byterecl" diff --git a/modulefiles/modulefile.sfc_climo_gen.theia b/modulefiles/modulefile.sfc_climo_gen.hera similarity index 50% rename from modulefiles/modulefile.sfc_climo_gen.theia rename to modulefiles/modulefile.sfc_climo_gen.hera index 39e306c5e..93aff0a2e 100644 --- a/modulefiles/modulefile.sfc_climo_gen.theia +++ b/modulefiles/modulefile.sfc_climo_gen.hera @@ -1,12 +1,13 @@ #%Module##################################################### -## Module file for sfc_climo_gen +## Module file for sfc_climo_gen (Hera) ############################################################# -module load intel/15.1.133 -module load impi/5.1.1.109 -module load netcdf/4.3.0 -module use /scratch4/NCEPDEV/nems/noscrub/emc.nemspara/soft/modulefiles -module load esmf/7.1.0r +module load intel/18.0.5.274 +module load impi/2018.0.4 + +export NCEPLIBS=/scratch1/NCEPDEV/global/gwv/l819/lib +module use -a $NCEPLIBS/modulefiles +module load esmflocal/8_0_48b.netcdf47 export FCOMP=mpiifort export FFLAGS="-O3 -fp-model=precise -g -traceback -r8 -i4 -convert big_endian" diff --git a/sorc/build_chgres.sh b/sorc/build_chgres.sh index 210ebae36..7dcb4291f 100755 --- a/sorc/build_chgres.sh +++ b/sorc/build_chgres.sh @@ -6,7 +6,7 @@ cwd=`pwd` USE_PREINST_LIBS=${USE_PREINST_LIBS:-"true"} if [ $USE_PREINST_LIBS = true ]; then - export MOD_PATH=/scratch3/NCEPDEV/nwprod/lib/modulefiles + export MOD_PATH source ../modulefiles/fv3gfs/global_chgres.$target > /dev/null 2>&1 else export MOD_PATH=${cwd}/lib/modulefiles @@ -32,8 +32,8 @@ export FCMP95=$FCMP export FFLAGSM="-i4 -O3 -r8 -convert big_endian -fp-model precise" export RECURS= -export LDFLAGSM=${LDFLAGSM:-"-openmp -auto"} -export OMPFLAGM=${OMPFLAGM:-"-openmp -auto"} +export LDFLAGSM=${LDFLAGSM:-"-qopenmp -auto"} +export OMPFLAGM=${OMPFLAGM:-"-qopenmp -auto"} export INCS="-I${SIGIO_INC4} -I${SFCIO_INC4} -I${LANDSFCUTIL_INCd} \ -I${NEMSIO_INC} -I${NEMSIOGFS_INC} -I${GFSIO_INC4} -I${IP_INCd} ${NETCDF_INCLUDE}" diff --git a/sorc/build_chgres_cube.sh b/sorc/build_chgres_cube.sh index 412c337b7..752429b47 100755 --- a/sorc/build_chgres_cube.sh +++ b/sorc/build_chgres_cube.sh @@ -6,7 +6,7 @@ cwd=`pwd` USE_PREINST_LIBS=${USE_PREINST_LIBS:-"true"} if [ $USE_PREINST_LIBS = true ]; then - export MOD_PATH=/scratch3/NCEPDEV/nwprod/lib/modulefiles + export MOD_PATH source ../modulefiles/chgres_cube.$target > /dev/null 2>&1 else export MOD_PATH=${cwd}/lib/modulefiles diff --git a/sorc/build_cycle.sh b/sorc/build_cycle.sh index b721bf840..75e80cc29 100755 --- a/sorc/build_cycle.sh +++ b/sorc/build_cycle.sh @@ -6,7 +6,7 @@ cwd=`pwd` USE_PREINST_LIBS=${USE_PREINST_LIBS:-"true"} if [ $USE_PREINST_LIBS = true ]; then - export MOD_PATH=/scratch3/NCEPDEV/nwprod/lib/modulefiles + export MOD_PATH source ../modulefiles/fv3gfs/global_cycle.$target > /dev/null 2>&1 else export MOD_PATH=${cwd}/lib/modulefiles diff --git a/sorc/build_emcsfc.sh b/sorc/build_emcsfc.sh index c2c600b12..233c40b26 100755 --- a/sorc/build_emcsfc.sh +++ b/sorc/build_emcsfc.sh @@ -14,6 +14,7 @@ do module purge USE_PREINST_LIBS=${USE_PREINST_LIBS:-"true"} if [ $USE_PREINST_LIBS = true ]; then + export MOD_PATH source ../modulefiles/modulefile.global_${prog}.${target} > /dev/null 2>&1 else export MOD_PATH=${cwd}/lib/modulefiles diff --git a/sorc/build_nems_util.sh b/sorc/build_nems_util.sh index 8bf8330ab..3f2f1498c 100755 --- a/sorc/build_nems_util.sh +++ b/sorc/build_nems_util.sh @@ -6,7 +6,7 @@ cwd=`pwd` USE_PREINST_LIBS=${USE_PREINST_LIBS:-"true"} if [ $USE_PREINST_LIBS = true ]; then - export MOD_PATH=/scratch3/NCEPDEV/nwprod/lib/modulefiles + export MOD_PATH source ../modulefiles/module_nemsutil.$target > /dev/null 2>&1 else export MOD_PATH=${cwd}/lib/modulefiles @@ -22,7 +22,7 @@ if [ ! -d "../exec" ]; then mkdir ../exec fi -for prog in nemsio_get.fd nemsio_read.fd nemsio_chgdate.fd ;do +for prog in nemsio_get.fd nemsio_read.fd nemsio_chgdate.fd mkgfsnemsioctl.fd ;do cd ${cwd}/${prog} make -f makefile done diff --git a/sorc/build_nst_tf_chg.sh b/sorc/build_nst_tf_chg.sh index ef063f5f7..5ba9ec958 100755 --- a/sorc/build_nst_tf_chg.sh +++ b/sorc/build_nst_tf_chg.sh @@ -7,7 +7,7 @@ cwd=`pwd` module purge USE_PREINST_LIBS=${USE_PREINST_LIBS:-"true"} if [ $USE_PREINST_LIBS = true ]; then - export MOD_PATH=/scratch3/NCEPDEV/nwprod/lib/modulefiles + export MOD_PATH source ../modulefiles/fv3gfs/nst_tf_chg.$target > /dev/null 2>&1 else export MOD_PATH=${cwd}/lib/modulefiles diff --git a/sorc/build_orog.sh b/sorc/build_orog.sh index a35846696..c1cc33a81 100755 --- a/sorc/build_orog.sh +++ b/sorc/build_orog.sh @@ -10,7 +10,7 @@ cwd=`pwd` USE_PREINST_LIBS=${USE_PREINST_LIBS:-"true"} if [ $USE_PREINST_LIBS = true ]; then - export MOD_PATH=/scratch3/NCEPDEV/nwprod/lib/modulefiles + export MOD_PATH source ../modulefiles/fv3gfs/orog.$target > /dev/null 2>&1 else export MOD_PATH=${cwd}/lib/modulefiles @@ -43,7 +43,7 @@ elif [ $target = jet ]; then INCS="-I${NETCDF}/include" export LIBSM="${BACIO_LIB4} ${W3NCO_LIBd} ${IP_LIBd} ${SP_LIBd} -L${NETCDF}/lib -lnetcdff -lnetcdf" export FFLAGSM="-O3 -g -traceback -r8 -convert big_endian -fp-model precise -assume byterecl ${INCS}" -elif [ $target = theia ]; then +elif [ $target = hera ]; then INCS="-I${NETCDF}/include" export LIBSM="${BACIO_LIB4} ${W3NCO_LIBd} ${IP_LIBd} ${SP_LIBd} -L${NETCDF}/lib -lnetcdff -lnetcdf" export FFLAGSM="-O3 -g -traceback -r8 -convert big_endian -fp-model precise -assume byterecl ${INCS}" diff --git a/sorc/build_sfc_climo_gen.sh b/sorc/build_sfc_climo_gen.sh index 237169f2f..cf253d358 100755 --- a/sorc/build_sfc_climo_gen.sh +++ b/sorc/build_sfc_climo_gen.sh @@ -12,6 +12,7 @@ fi module purge USE_PREINST_LIBS=${USE_PREINST_LIBS:-"true"} if [ $USE_PREINST_LIBS = true ]; then + export MOD_PATH source ../modulefiles/modulefile.sfc_climo_gen.${target} > /dev/null 2>&1 else export MOD_PATH=${cwd}/lib/modulefiles diff --git a/sorc/fre-nctools.fd/tools/filter_topo/make.csh_theia b/sorc/fre-nctools.fd/tools/filter_topo/make.csh_hera similarity index 81% rename from sorc/fre-nctools.fd/tools/filter_topo/make.csh_theia rename to sorc/fre-nctools.fd/tools/filter_topo/make.csh_hera index 563538c4c..fb43802b0 100755 --- a/sorc/fre-nctools.fd/tools/filter_topo/make.csh_theia +++ b/sorc/fre-nctools.fd/tools/filter_topo/make.csh_hera @@ -1,6 +1,6 @@ #!/bin/csh -f source $MODULESHOME/init/csh -source ${PWD}/../../../../modulefiles/fv3gfs/fre-nctools.theia +source ${PWD}/../../../../modulefiles/fv3gfs/fre-nctools.hera ifort -o filter_topo -I${NETCDF}/include -fltconsistency -fno-alias -stack_temps -safe_cray_ptr -ftz -assume byterecl -g -O2 -i4 -real_size 64 -traceback filter_topo.F90 -L${NETCDF}/lib -L${HDF5}/lib -lnetcdf -lnetcdff -lhdf5_hl -lhdf5 diff --git a/sorc/fre-nctools.fd/tools/make_hgrid/env.theia b/sorc/fre-nctools.fd/tools/fregrid/env.hera similarity index 85% rename from sorc/fre-nctools.fd/tools/make_hgrid/env.theia rename to sorc/fre-nctools.fd/tools/fregrid/env.hera index 6c090e2e2..a4ca56ba4 100644 --- a/sorc/fre-nctools.fd/tools/make_hgrid/env.theia +++ b/sorc/fre-nctools.fd/tools/fregrid/env.hera @@ -1,4 +1,4 @@ -# theia +# hera MPICC := mpiicc CC := icc STATIC := diff --git a/sorc/fre-nctools.fd/tools/fregrid/env.wcoss_dell_p3 b/sorc/fre-nctools.fd/tools/fregrid/env.wcoss_dell_p3 index 834db6e7e..07f52aac8 100644 --- a/sorc/fre-nctools.fd/tools/fregrid/env.wcoss_dell_p3 +++ b/sorc/fre-nctools.fd/tools/fregrid/env.wcoss_dell_p3 @@ -1,4 +1,4 @@ -# theia +# Dell MPICC := mpiicc CC := icc -STATIC := \ No newline at end of file +STATIC := diff --git a/sorc/fre-nctools.fd/tools/fregrid/fre-nctools.mk_theia b/sorc/fre-nctools.fd/tools/fregrid/fre-nctools.mk_hera similarity index 100% rename from sorc/fre-nctools.fd/tools/fregrid/fre-nctools.mk_theia rename to sorc/fre-nctools.fd/tools/fregrid/fre-nctools.mk_hera diff --git a/sorc/fre-nctools.fd/tools/make_solo_mosaic/env.theia b/sorc/fre-nctools.fd/tools/make_hgrid/env.hera similarity index 85% rename from sorc/fre-nctools.fd/tools/make_solo_mosaic/env.theia rename to sorc/fre-nctools.fd/tools/make_hgrid/env.hera index 6c090e2e2..a4ca56ba4 100644 --- a/sorc/fre-nctools.fd/tools/make_solo_mosaic/env.theia +++ b/sorc/fre-nctools.fd/tools/make_hgrid/env.hera @@ -1,4 +1,4 @@ -# theia +# hera MPICC := mpiicc CC := icc STATIC := diff --git a/sorc/fre-nctools.fd/tools/make_hgrid/fre-nctools.mk_theia b/sorc/fre-nctools.fd/tools/make_hgrid/fre-nctools.mk_hera similarity index 100% rename from sorc/fre-nctools.fd/tools/make_hgrid/fre-nctools.mk_theia rename to sorc/fre-nctools.fd/tools/make_hgrid/fre-nctools.mk_hera diff --git a/sorc/fre-nctools.fd/tools/fregrid/env.theia b/sorc/fre-nctools.fd/tools/make_solo_mosaic/env.hera similarity index 64% rename from sorc/fre-nctools.fd/tools/fregrid/env.theia rename to sorc/fre-nctools.fd/tools/make_solo_mosaic/env.hera index 834db6e7e..a4ca56ba4 100644 --- a/sorc/fre-nctools.fd/tools/fregrid/env.theia +++ b/sorc/fre-nctools.fd/tools/make_solo_mosaic/env.hera @@ -1,4 +1,4 @@ -# theia +# hera MPICC := mpiicc CC := icc -STATIC := \ No newline at end of file +STATIC := diff --git a/sorc/fre-nctools.fd/tools/make_solo_mosaic/env.wcoss_dell_p3 b/sorc/fre-nctools.fd/tools/make_solo_mosaic/env.wcoss_dell_p3 index 6c090e2e2..07f52aac8 100644 --- a/sorc/fre-nctools.fd/tools/make_solo_mosaic/env.wcoss_dell_p3 +++ b/sorc/fre-nctools.fd/tools/make_solo_mosaic/env.wcoss_dell_p3 @@ -1,4 +1,4 @@ -# theia +# Dell MPICC := mpiicc CC := icc STATIC := diff --git a/sorc/fre-nctools.fd/tools/make_solo_mosaic/fre-nctools.mk_theia b/sorc/fre-nctools.fd/tools/make_solo_mosaic/fre-nctools.mk_hera similarity index 100% rename from sorc/fre-nctools.fd/tools/make_solo_mosaic/fre-nctools.mk_theia rename to sorc/fre-nctools.fd/tools/make_solo_mosaic/fre-nctools.mk_hera diff --git a/sorc/fre-nctools.fd/tools/shave.fd/build_shave b/sorc/fre-nctools.fd/tools/shave.fd/build_shave index 81aa8469d..18cdf07aa 100755 --- a/sorc/fre-nctools.fd/tools/shave.fd/build_shave +++ b/sorc/fre-nctools.fd/tools/shave.fd/build_shave @@ -15,7 +15,7 @@ case $1 in "jet" ) FCMP=ifort FFLAGS="-O3 -fp-model precise -I$NETCDF/include -L$NETCDF/lib -lnetcdff -lnetcdf" ;; - "theia" ) + "hera" ) FCMP=ifort FFLAGS="-O3 -fp-model precise -I$NETCDF/include -L$NETCDF/lib -lnetcdff -lnetcdf" ;; *) diff --git a/sorc/link_fixdirs.sh b/sorc/link_fixdirs.sh index d2b4fdc8c..d5fbc1cde 100755 --- a/sorc/link_fixdirs.sh +++ b/sorc/link_fixdirs.sh @@ -9,20 +9,20 @@ machine=${2} if [ $# -lt 2 ]; then set +x echo '***ERROR*** must specify two arguements: (1) RUN_ENVIR, (2) machine' - echo ' Syntax: link_fv3gfs.sh ( nco | emc ) ( cray | dell | theia | jet )' + echo ' Syntax: link_fv3gfs.sh ( nco | emc ) ( cray | dell | hera | jet )' exit 1 fi if [ $RUN_ENVIR != emc -a $RUN_ENVIR != nco ]; then set +x echo '***ERROR*** unsupported run environment' - echo 'Syntax: link_fv3gfs.sh ( nco | emc ) ( cray | dell | theia | jet )' + echo 'Syntax: link_fv3gfs.sh ( nco | emc ) ( cray | dell | hera | jet )' exit 1 fi -if [ $machine != cray -a $machine != theia -a $machine != dell -a $machine != jet ]; then +if [ $machine != cray -a $machine != hera -a $machine != dell -a $machine != jet ]; then set +x echo '***ERROR*** unsupported machine' - echo 'Syntax: link_fv3gfs.sh ( nco | emc ) ( cray | dell | theia | jet )' + echo 'Syntax: link_fv3gfs.sh ( nco | emc ) ( cray | dell | hera | jet )' exit 1 fi @@ -39,23 +39,15 @@ if [ $machine == "cray" ]; then FIX_DIR="/gpfs/hps3/emc/global/noscrub/emc.glopara/git/fv3gfs/fix" elif [ $machine = "dell" ]; then FIX_DIR="/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/fv3gfs/fix" -elif [ $machine = "theia" ]; then - FIX_DIR="/scratch4/NCEPDEV/global/save/glopara/git/fv3gfs/fix" +elif [ $machine = "hera" ]; then + FIX_DIR="/scratch1/NCEPDEV/global/glopara/fix" elif [ $machine = "jet" ]; then FIX_DIR="/lfs3/projects/hfv3gfs/glopara/git/fv3gfs/fix" fi cd ${pwd}/../fix ||exit 8 -for dir in fix_am fix_fv3 fix_orog fix_fv3_gmted2010 ; do +for dir in fix_am fix_fv3 fix_orog fix_fv3_gmted2010 fix_sfc_climo; do [[ -d $dir ]] && rm -rf $dir $LINK $FIX_DIR/$dir . done -if [ $machine == "cray" ] || [ $machine = "dell" ]; then - $LINK /gpfs/dell2/emc/modeling/noscrub/George.Gayno/landutil.git/climo_fields_netcdf ./fix_sfc_climo -elif [ $machine = "theia" ]; then - $LINK /scratch4/NCEPDEV/da/noscrub/George.Gayno/climo_fields_netcdf ./fix_sfc_climo -elif [ $machine = "jet" ]; then - $LINK /mnt/lfs3/projects/emcda/George.Gayno/climo_fields_netcdf ./fix_sfc_climo -fi - exit 0 diff --git a/sorc/machine-setup.sh b/sorc/machine-setup.sh index ed796f86b..d54235b60 100644 --- a/sorc/machine-setup.sh +++ b/sorc/machine-setup.sh @@ -28,16 +28,15 @@ if [[ -d /lfs3 ]] ; then target=jet module purge module use /mnt/lfs3/projects/hfv3gfs/nwprod/lib/modulefiles -elif [[ -d /scratch3 ]] ; then - # We are on NOAA Theia +elif [[ -d /scratch1 ]] ; then + # We are on NOAA Hera if ( ! eval module help > /dev/null 2>&1 ) ; then echo load the module command 1>&2 source /apps/lmod/lmod/init/$__ms_shell fi - target=theia + target=hera module purge - module use /scratch3/NCEPDEV/nwprod/modulefiles/ - module use /scratch3/NCEPDEV/nwprod/lib/modulefiles + MOD_PATH=/scratch2/NCEPDEV/nwprod/NCEPLIBS/modulefiles elif [[ -d /gpfs/hps && -e /etc/SuSE-release ]] ; then # We are on NOAA Luna or Surge if ( ! eval module help > /dev/null 2>&1 ) ; then diff --git a/sorc/mkgfsnemsioctl.fd/makefile b/sorc/mkgfsnemsioctl.fd/makefile new file mode 100755 index 000000000..351cda92d --- /dev/null +++ b/sorc/mkgfsnemsioctl.fd/makefile @@ -0,0 +1,10 @@ +SHELL =/bin/ksh +EXEC =../../exec/mkgfsnemsioctl +FOPTS = -O -FR -I$(NEMSIO_INC) +LOPTS = $(OPTS) +LIBS = $(NEMSIO_LIB) $(BACIO_LIB4) $(W3NCO_LIBd) +OBJS = mkgfsnemsioctl.o +SRCS = mkgfsnemsioctl.f +# ************************************************************************* +all: $(SRCS) + $(FCMP) $(FOPTS) $(LOPTS) ${SRCS} -o $(EXEC) $(LIBS) diff --git a/sorc/mkgfsnemsioctl.fd/mkgfsnemsioctl.f b/sorc/mkgfsnemsioctl.fd/mkgfsnemsioctl.f new file mode 100644 index 000000000..cdc9efc90 --- /dev/null +++ b/sorc/mkgfsnemsioctl.fd/mkgfsnemsioctl.f @@ -0,0 +1,535 @@ +!- - - - -- - -- - -- - -- - - -- - -- -- - -- - -- - - -- - - - -- - -- +! the program create NEMS GFS sfc ctl file +! +! Mar 12, 2015 Jun Wang +! - - - - -- - -- - -- - -- - - -- - -- -- - -- - -- - - -- - - - -- - -- +! + program main + use nemsio_module + implicit none +! +!--------------------------------------------------------------------------- + type(nemsio_gfile) :: gfile + character(255) cin,cindx + real,allocatable :: data(:,:) +!--------------------------------------------------------------------------- +!--- nemsio meta data + integer nrec,im,jm,lm,idate(7),nfhour,tlmeta,nsoil,fieldsize + real(4),allocatable :: lat1(:),lat(:),lon1(:) + real(8),allocatable :: slat(:) + character(16),allocatable:: recname(:),reclevtyp(:) + integer,allocatable:: reclev(:) +!--------------------------------------------------------------------------- +!--- local vars + character(3) cmon + character(16) reclevtyp_sht + character(32) ctldate,varname + character(35) sweep_blanks + real(8) radi + real lon_stt,lat_stt,lon_intl,lat_intl + integer i,n,j,krec,iret,io_unit,idrt,nmeta +!--------------------------------------------------------------------------- +!--------------------------------------------------------------------------- +! + call nemsio_init(iret=iret) + if(iret/=0) print *,'ERROR: nemsio_init ' +! +!--------------------------------------------------------------------------- +!*** read nemsio grd header info +!--------------------------------------------------------------------------- +!--- open gfile for reading + call getarg(1,cin) + print *,'filename is cin=',trim(cin) + if(trim(cin)=='') then + print *,'usage: mknemsioctl input_nemsio_file_name' + stop + endif + call getarg(2,cindx) + if(trim(cindx)=='') then + cindx=trim(cin)//'.ctl' + endif + + call nemsio_open(gfile,trim(cin),'READ',iret=iret) + if(iret/=0) print *,'Error: open nemsio file,',trim(cin),' iret=',iret + + call nemsio_getfilehead(gfile,iret=iret,nrec=nrec,dimx=im,dimy=jm, & + dimz=lm,idate=idate,nfhour=nfhour,nmeta=nmeta, & + nsoil=nsoil,idrt=idrt,tlmeta=tlmeta) +! + fieldsize=im*jm + allocate(recname(nrec),reclevtyp(nrec),reclev(nrec)) + allocate(lat1(fieldsize),lat(jm),lon1(fieldsize)) + call nemsio_getfilehead(gfile,iret=iret,recname=recname, & + reclevtyp=reclevtyp,reclev=reclev,lat=lat1,lon=lon1) +! + call nemsio_close(gfile,iret=iret) +! + call nemsio_finalize() +! +!--------------------------------------------------------------------------- +!****** write .ctl file +!--------------------------------------------------------------------------- +! +!-- get date + call cmonth(idate(2),cmon) + write(ctldate,'(i2.2,a,i2.2,a3,i4.4)')idate(4),'Z',idate(3) & + ,cmon,idate(1) +!-- get Gaussian grid + if(idrt==4) then +!-- data has lat/lon info + if (nmeta>=8) then + lon_stt=lon1(1) + lon_intl=lon1(2)-lon1(1) + do j=1,jm + lat(j) = lat1(1+(j-1)*im) + enddo + else + call splat8(idrt,jm,slat) + radi=180.0/(4.*atan(1.0)) + do j=1,jm + lat(j) = asin(slat(j)) * radi + enddo + lon_stt=0. + lon_intl=360./real(jm) + endif + elseif(idrt==0) then + if(nmeta>=8) then + lon_stt=lon1(1) + lon_intl=lon1(2)-lon1(1) + lat_stt=-abs(lat1(1)) + lat_intl=abs(lat1(1+im)-lat1(1)) +!! print*, "im=",im,' jm=',jm +!! print*, "lon1=",lon1 +!! print*, "lat_intl=",lat_intl +!! print*, "lat1=",lat1 + else + if (mod(jm,2) == 0) then + lat_intl=180.0/real(jm) + lat_stt=-90.0+0.5*lat_intl + else + lat_intl=180.0/real(jm-1) + lat_stt=-90.0 + endif + lon_intl=360./real(im) + lon_stt=0.5*lon_intl + endif + endif + + io_unit=650 + open(io_unit,file=trim(cindx),form='formatted') +! + write(io_unit,105)trim(cin) + write(io_unit,106) + + if (idrt == 4 )then + write(io_unit,107) + elseif(idrt==0) then + if(nmeta>=8) then + if (lat1(1)>lat1(im+1)) then + write(io_unit,107) + else + write(io_unit,1077) + endif + else + write(io_unit,1077) + endif + endif + + write(io_unit,108)tlmeta + write(io_unit,109) + write(io_unit,111)im,lon_stt,lon_intl + if (idrt==4) then + write(io_unit,112)jm + write(io_unit,113)(lat(i),i=jm,1,-1) + else + write(io_unit,120)jm,lat_stt,lat_intl + endif + write(io_unit,114)lm + if(nfhour/=0) then + write(io_unit,115)1,trim(ctldate),nfhour + else + write(io_unit,116)1,trim(ctldate) + endif + +! + 105 FORMAT('dset ^',A) + 106 FORMAT('undef 9.99E+20') + 107 FORMAT('options big_endian sequential yrev') + 1077 FORMAT('options big_endian sequential') + 108 FORMAT('fileheader',I12.0) + 109 FORMAT('title gfs nemsioi file') + + 111 FORMAT('xdef ',I6,' linear ',f9.6,' ',f9.6) + 112 FORMAT('ydef ',I6,' levels') + 113 FORMAT(10f11.6) + + 114 FORMAT('zdef ',I6,' linear 1 1') + 115 FORMAT('tdef ',I6,' linear ',A12,' ',I6,'hr') + 116 FORMAT('tdef ',I6,' linear ',A12,' 1yr') +! + 120 FORMAT('ydef ',I6,' linear ',f13.6,' ',f9.6) +! + krec=0 + do n=1,nrec + if(reclev(n)==1) then + krec=krec+1 + endif + enddo + + WRITE(IO_UNIT,'(A,I6)')'VARS ',krec + + n=1 + do while (n<=nrec) + reclevtyp_sht=reclevtyp(n) + if(trim(reclevtyp_sht) == "convect-cld bot") then + reclevtyp_sht="cvb" + elseif (trim(reclevtyp_sht) == "convect-cld top") then + reclevtyp_sht="cvt" + elseif (trim(reclevtyp_sht) == "high cld bot") then + reclevtyp_sht="hcb" + elseif (trim(reclevtyp_sht) == "high cld top") then + reclevtyp_sht="hct" + elseif (trim(reclevtyp_sht) == "mid cld bot") then + reclevtyp_sht="mcb" + elseif (trim(reclevtyp_sht) == "mid cld top") then + reclevtyp_sht="mct" + elseif (trim(reclevtyp_sht) == "low cld bot") then + reclevtyp_sht="lcb" + elseif (trim(reclevtyp_sht) == "low cld top") then + reclevtyp_sht="lct" + elseif (trim(reclevtyp_sht) == "convect-cld laye") then + reclevtyp_sht="cvcl" + elseif (trim(reclevtyp_sht) == "bndary-layer cld") then + reclevtyp_sht="bdrlc" + elseif (trim(reclevtyp_sht) == "atmos col") then + reclevtyp_sht="acol" + elseif (trim(reclevtyp_sht) == "high cld lay") then + reclevtyp_sht="hcl" + elseif (trim(reclevtyp_sht) == "mid cld lay") then + reclevtyp_sht="mcl" + elseif (trim(reclevtyp_sht) == "low cld lay") then + reclevtyp_sht="lcl" + elseif (trim(reclevtyp_sht) == "2 m above gnd") then + reclevtyp_sht="2m" + elseif (trim(reclevtyp_sht) == "10 m above gnd") then + reclevtyp_sht="10m" + endif + varname=sweep_blanks(trim(recname(n))//trim(reclevtyp_sht)) + if(trim(reclevtyp(n))=='mid layer') then + write(io_unit,'(a16,i3,a)')varname,lm,' 99 model layer' + n=n+lm + elseif(trim(reclevtyp(n))=='soil layer') then + write(io_unit,'(a16,i3,a)')varname,nsoil,' 99 soil layer' + n=n+nsoil + elseif(trim(reclevtyp(n))=='2 m above gnd') then + recname(n)=trim(recname(n))//'2m' + write(io_unit,'(a16,a7,a)')varname,' 0 99 ',trim(reclevtyp(n)) + n=n+1 + elseif(trim(reclevtyp(n))=='10 m above gnd') then + recname(n)=trim(recname(n))//'10m' + write(io_unit,'(a16,a7,a)')varname,' 0 99 ',trim(reclevtyp(n)) + n=n+1 + else + write(io_unit,'(a16,a7,a)')varname,' 0 99 ',trim(reclevtyp(n)) + n=n+1 + endif + enddo + + write(io_unit,'(A8)')'endvars' + close(io_unit) + +!--------------------------------------------------------------------------- +!****** clean up +!--------------------------------------------------------------------------- + deallocate(recname,reclevtyp,reclev,lat,lat1,lon1) +!--------------------------------------------------------------------------- +! +! - - - - -- - -- - -- - -- - - -- - -- -- - -- - -- - - -- - - - -- - -- + stop + + end program +! - - - - -- - -- - -- - -- - - -- - -- -- - -- - -- - - -- - - - -- - -- + SUBROUTINE CMONTH(IMON,CMON) +! +!----------------------------------------------------------------------- +!*** Convert month +!----------------------------------------------------------------------- +! + INTEGER,INTENT(IN) :: IMON + CHARACTER(LEN=3) :: CMON +! +!----------------------------------------------------------------------- +! + SELECT CASE (IMON) + CASE(1) + CMON='Jan' + CASE(2) + CMON='Feb' + CASE(3) + CMON='Mar' + CASE(4) + CMON='Apr' + CASE(5) + CMON='May' + CASE(6) + CMON='Jun' + CASE(7) + CMON='Jul' + CASE(8) + CMON='Aug' + CASE(9) + CMON='Sep' + CASE(10) + CMON='Oct' + CASE(11) + CMON='Nov' + CASE(12) + CMON='Dec' + END SELECT +! +!----------------------------------------------------------------------- +! + END SUBROUTINE CMONTH + +! + sUBROUTINE splat4(IDRT,JMAX,ASLAT) +! + implicit none + integer,intent(in) :: idrt,jmax + real(4),intent(out) :: ASLAT(JMAX) + INTEGER,PARAMETER:: KD=SELECTED_REAL_KIND(15,45) + REAL(KIND=KD):: PK(JMAX/2),PKM1(JMAX/2),PKM2(JMAX/2) + REAL(KIND=KD):: ASLATD(JMAX/2),SP,SPMAX,EPS=10.d0*EPSILON(SP) + integer,PARAMETER:: JZ=50 + REAL(8) BZ(JZ) + DATA BZ / 2.4048255577d0, 5.5200781103d0, & + 8.6537279129d0, 11.7915344391d0, 14.9309177086d0, 18.0710639679d0, & + 21.2116366299d0, 24.3524715308d0, 27.4934791320d0, 30.6346064684d0, & + 33.7758202136d0, 36.9170983537d0, 40.0584257646d0, 43.1997917132d0, & + 46.3411883717d0, 49.4826098974d0, 52.6240518411d0, 55.7655107550d0, & + 58.9069839261d0, 62.0484691902d0, 65.1899648002d0, 68.3314693299d0, & + 71.4729816036d0, 74.6145006437d0, 77.7560256304d0, 80.8975558711d0, & + 84.0390907769d0, 87.1806298436d0, 90.3221726372d0, 93.4637187819d0, & + 96.6052679510d0, 99.7468198587d0, 102.888374254d0, 106.029930916d0, & + 109.171489649d0, 112.313050280d0, 115.454612653d0, 118.596176630d0, & + 121.737742088d0, 124.879308913d0, 128.020877005d0, 131.162446275d0, & + 134.304016638d0, 137.445588020d0, 140.587160352d0, 143.728733573d0, & + 146.870307625d0, 150.011882457d0, 153.153458019d0, 156.295034268d0 / + REAL(8):: DLT,D1=1.d0 + INTEGER(4):: JHE,JHO,J0=0 + real(8),PARAMETER :: PI=3.14159265358979d0,C=(1.d0-(2.d0/PI)**2)*0.25d0 + real(8) r + integer jh,js,n,j +!C - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +!C GAUSSIAN LATITUDES + IF(IDRT.EQ.4) THEN + JH=JMAX/2 + JHE=(JMAX+1)/2 + R=1.d0/SQRT((JMAX+0.5d0)**2+C) + DO J=1,MIN(JH,JZ) + ASLATD(J)=COS(BZ(J)*R) + ENDDO + DO J=JZ+1,JH + ASLATD(J)=COS((BZ(JZ)+(J-JZ)*PI)*R) + ENDDO + SPMAX=1.d0 + DO WHILE(SPMAX.GT.EPS) + SPMAX=0.d0 + DO J=1,JH + PKM1(J)=1.d0 + PK(J)=ASLATD(J) + ENDDO + DO N=2,JMAX + DO J=1,JH + PKM2(J)=PKM1(J) + PKM1(J)=PK(J) + PK(J)=((2*N-1)*ASLATD(J)*PKM1(J)-(N-1)*PKM2(J))/N + ENDDO + ENDDO + DO J=1,JH + SP=PK(J)*(1.d0-ASLATD(J)**2)/(JMAX*(PKM1(J)-ASLATD(J)*PK(J))) + ASLATD(J)=ASLATD(J)-SP + SPMAX=MAX(SPMAX,ABS(SP)) + ENDDO + ENDDO +!CDIR$ IVDEP + DO J=1,JH + ASLAT(J)=ASLATD(J) + ASLAT(JMAX+1-J)=-ASLAT(J) + ENDDO + IF(JHE.GT.JH) THEN + ASLAT(JHE)=0.d0 + ENDIF +!C - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +!C EQUALLY-SPACED LATITUDES INCLUDING POLES + ELSEIF(IDRT.EQ.0) THEN + JH=JMAX/2 + JHE=(JMAX+1)/2 + JHO=JHE-1 + DLT=PI/(JMAX-1) + ASLAT(1)=1.d0 + DO J=2,JH + ASLAT(J)=COS((J-1)*DLT) + ENDDO +!CDIR$ IVDEP + DO J=1,JH + ASLAT(JMAX+1-J)=-ASLAT(J) + ENDDO + IF(JHE.GT.JH) THEN + ASLAT(JHE)=0.d0 + ENDIF +!C - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +!C EQUALLY-SPACED LATITUDES EXCLUDING POLES + ELSEIF(IDRT.EQ.256) THEN + JH=JMAX/2 + JHE=(JMAX+1)/2 + JHO=JHE + DLT=PI/JMAX + ASLAT(1)=1.d0 + DO J=1,JH + ASLAT(J)=COS((J-0.5)*DLT) + ENDDO +!CDIR$ IVDEP + DO J=1,JH + ASLAT(JMAX+1-J)=-ASLAT(J) + ENDDO + IF(JHE.GT.JH) THEN + ASLAT(JHE)=0.d0 + ENDIF + ENDIF +!C - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + end subroutine splat4 +! +!---------------------------------------------------------------------- + SUBROUTINE splat8(IDRT,JMAX,ASLAT) +!$$$ + implicit none + integer,intent(in) :: idrt,jmax + real(8),intent(out) :: ASLAT(JMAX) + INTEGER,PARAMETER:: KD=SELECTED_REAL_KIND(15,45) + REAL(KIND=KD):: PK(JMAX/2),PKM1(JMAX/2),PKM2(JMAX/2) + REAL(KIND=KD):: ASLATD(JMAX/2),SP,SPMAX,EPS=10.d0*EPSILON(SP) + integer,PARAMETER:: JZ=50 + REAL(8) BZ(JZ) + DATA BZ / 2.4048255577d0, 5.5200781103d0, & + 8.6537279129d0, 11.7915344391d0, 14.9309177086d0, 18.0710639679d0, & + 21.2116366299d0, 24.3524715308d0, 27.4934791320d0, 30.6346064684d0, & + 33.7758202136d0, 36.9170983537d0, 40.0584257646d0, 43.1997917132d0, & + 46.3411883717d0, 49.4826098974d0, 52.6240518411d0, 55.7655107550d0, & + 58.9069839261d0, 62.0484691902d0, 65.1899648002d0, 68.3314693299d0, & + 71.4729816036d0, 74.6145006437d0, 77.7560256304d0, 80.8975558711d0, & + 84.0390907769d0, 87.1806298436d0, 90.3221726372d0, 93.4637187819d0, & + 96.6052679510d0, 99.7468198587d0, 102.888374254d0, 106.029930916d0, & + 109.171489649d0, 112.313050280d0, 115.454612653d0, 118.596176630d0, & + 121.737742088d0, 124.879308913d0, 128.020877005d0, 131.162446275d0, & + 134.304016638d0, 137.445588020d0, 140.587160352d0, 143.728733573d0, & + 146.870307625d0, 150.011882457d0, 153.153458019d0, 156.295034268d0 / + REAL(8):: DLT,D1=1.d0 + INTEGER(4):: JHE,JHO,J0=0 + real(8),PARAMETER :: PI=3.14159265358979d0,C=(1.d0-(2.d0/PI)**2)*0.25d0 + real(8) r + integer jh,js,n,j +!C - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +!C GAUSSIAN LATITUDES + IF(IDRT.EQ.4) THEN + JH=JMAX/2 + JHE=(JMAX+1)/2 + R=1.d0/SQRT((JMAX+0.5d0)**2+C) + DO J=1,MIN(JH,JZ) + ASLATD(J)=COS(BZ(J)*R) + ENDDO + DO J=JZ+1,JH + ASLATD(J)=COS((BZ(JZ)+(J-JZ)*PI)*R) + ENDDO + SPMAX=1.d0 + DO WHILE(SPMAX.GT.EPS) + SPMAX=0.d0 + DO J=1,JH + PKM1(J)=1.d0 + PK(J)=ASLATD(J) + ENDDO + DO N=2,JMAX + DO J=1,JH + PKM2(J)=PKM1(J) + PKM1(J)=PK(J) + PK(J)=((2*N-1)*ASLATD(J)*PKM1(J)-(N-1)*PKM2(J))/N + ENDDO + ENDDO + DO J=1,JH + SP=PK(J)*(1.d0-ASLATD(J)**2)/(JMAX*(PKM1(J)-ASLATD(J)*PK(J))) + ASLATD(J)=ASLATD(J)-SP + SPMAX=MAX(SPMAX,ABS(SP)) + ENDDO + ENDDO +!CDIR$ IVDEP + DO J=1,JH + ASLAT(J)=ASLATD(J) + ASLAT(JMAX+1-J)=-ASLAT(J) + ENDDO + IF(JHE.GT.JH) THEN + ASLAT(JHE)=0.d0 + ENDIF +!C - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +!C EQUALLY-SPACED LATITUDES INCLUDING POLES + ELSEIF(IDRT.EQ.0) THEN + JH=JMAX/2 + JHE=(JMAX+1)/2 + JHO=JHE-1 + DLT=PI/(JMAX-1) + ASLAT(1)=1.d0 + DO J=2,JH + ASLAT(J)=COS((J-1)*DLT) + ENDDO +!CDIR$ IVDEP + DO J=1,JH + ASLAT(JMAX+1-J)=-ASLAT(J) + ENDDO + IF(JHE.GT.JH) THEN + ASLAT(JHE)=0.d0 + ENDIF +!C - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +!C EQUALLY-SPACED LATITUDES EXCLUDING POLES + ELSEIF(IDRT.EQ.256) THEN + JH=JMAX/2 + JHE=(JMAX+1)/2 + JHO=JHE + DLT=PI/JMAX + ASLAT(1)=1.d0 + DO J=1,JH + ASLAT(J)=COS((J-0.5d0)*DLT) + ENDDO +!DIR$ IVDEP + DO J=1,JH + ASLAT(JMAX+1-J)=-ASLAT(J) + ENDDO + IF(JHE.GT.JH) THEN + ASLAT(JHE)=0.d0 + ENDIF + ENDIF +! - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + end subroutine splat8 +!----------------------------------------------------------------------- + + character(35) function sweep_blanks(in_str) + +! + implicit none +! + character(*), intent(in) :: in_str + character(35) :: out_str + character :: ch + integer :: j + + out_str = " " + do j=1, len_trim(in_str) + ! get j-th char + ch = in_str(j:j) + if (ch .eq. "-") then + out_str = trim(out_str) // "_" + else if (ch .ne. " ") then + out_str = trim(out_str) // ch + endif + sweep_blanks = out_str + end do + end function sweep_blanks + diff --git a/ush/fv3gfs_chgres.sh b/ush/fv3gfs_chgres.sh index a169abf1a..a7f02dcc7 100755 --- a/ush/fv3gfs_chgres.sh +++ b/ush/fv3gfs_chgres.sh @@ -12,7 +12,7 @@ # - Uncomment the job cards for your machine. Ensure the # 'machine' variable is uncommented. Ensure job cards for # the other machines are prefixed by "##". -# - Check the job card account. On Theia, this is +# - Check the job card account. On Hera, this is # "#SBATCH -q $account". On WCOSS, this is "#BSUB -P $account". # You may not have permissions to run under the default account. # - Make sure you have linked the 'fixed' directories using @@ -30,7 +30,7 @@ # CDATE - year/month/day/hour of your experiment. # HOMEgfs - Path to your checkout repository. # - Start the script as follows: -# Theia: "sbatch fv3gfs_chgres.sh" +# Hera: "sbatch fv3gfs_chgres.sh" # WCOSS Dell and Cray: "cat fv3gfs_chgres.sh | bsub" # #------------------------------------------------------------------------ @@ -59,7 +59,7 @@ export machine=WCOSS_DELL_P3 ##BSUB -extsched 'CRAYLINUX[]' #export machine=WCOSS_C -#---- THEIA JOBCARD +#---- Hera JOBCARD #---- Submit as: sbatch $script ##SBATCH -J fv3_chgres_driver ##SBATCH -A fv3-cpu @@ -68,13 +68,13 @@ export machine=WCOSS_DELL_P3 ##SBATCH -e log.chgres ##SBATCH --nodes=1 ##SBATCH -q batch -##SBATCH -t 02:00:00 -#export machine=THEIA +##SBATCH -t 03:00:00 +#export machine=HERA set -x export PSLOT=fv3test -export CDUMP=gdas +export CDUMP=gfs export CASE_HIGH=C768 export CASE_ENKF=C384 export CDATE=2019061000 @@ -87,10 +87,10 @@ elif [ $machine = WCOSS_DELL_P3 ]; then export HOMEgfs=$LS_SUBCWD/.. export PTMP=${PTMP:-"/gpfs/dell2/ptmp/$USER"} export zero_bias_dir=/gpfs/hps3/emc/global/noscrub/emc.glopara/ICS/bias_zero -elif [ $machine = THEIA ]; then +elif [ $machine = HERA ]; then export HOMEgfs=$SLURM_SUBMIT_DIR/.. - export PTMP=${PTMP:-"/scratch3/NCEPDEV/stmp1/$USER"} - export zero_bias_dir=/scratch4/NCEPDEV/global/save/glopara/git/bias_zero + export PTMP=${PTMP:-"/scratch2/NCEPDEV/stmp1/$USER"} + export zero_bias_dir=/scratch1/NCEPDEV/global/glopara/bias_zero else echo "Please define HOMEgfs and PTMP for your machine. exit" exit @@ -132,45 +132,28 @@ status=$? [[ $status -ne 0 ]] && exit $status if [ $machine = WCOSS_C ]; then - # . $MODULESHOME/init/sh 2>>/dev/null - # module load prod_util prod_envir hpss 2>>/dev/null - # module load PrgEnv-intel 2>>/dev/null export KMP_AFFINITY=disabled export OMP_NUM_THREADS_CH=24 export APRUNC="aprun -n 1 -N 1 -j 1 -d $OMP_NUM_THREADS_CH -cc depth" export APRUNTF='aprun -q -j1 -n1 -N1 -d1 -cc depth' - export SUB=/u/emc.glopara/bin/sub_wcoss_c + export SUB=$LS_SUBCWD/../util/sub_wcoss_c export ACCOUNT=FV3GFS-T2O export QUEUE=dev export QUEUE_TRANS=dev_transfer elif [ $machine = WCOSS_DELL_P3 ]; then - # . /usrx/local/prod/lmod/lmod/init/sh 2>>/dev/null - # module purge 2>>/dev/null - # module load EnvVars/1.0.2 2>>/dev/null - # module load lsf/10.1 2>>/dev/null - # module load ips/18.0.1.163 2>>/dev/null - # module load impi/18.0.1 2>>/dev/null - # module load prod_util/1.1.0 2>>/dev/null - # module load prod_envir/1.0.2 2>>/dev/null - # module load HPSS/5.0.2.5 2>>/dev/null - # module load NetCDF/4.5.0 2>>/dev/null - # module load HDF5-serial/1.10.1 2>>/dev/null export OMP_NUM_THREADS_CH=14 export KMP_AFFINITY=disabled export APRUNC="time" export APRUNTF="time" - export SUB=/u/emc.glopara/bin/sub_wcoss_d + export SUB=$LS_SUBCWD/../util/sub_wcoss_d export ACCOUNT=FV3GFS-T2O export QUEUE=dev export QUEUE_TRANS=dev_transfer -elif [ $machine = THEIA ]; then - # source $HOMEgfs/sorc/machine-setup.sh - # module use -a /scratch3/NCEPDEV/nwprod/lib/modulefiles - # module load netcdf hdf5/1.8.14 2>>/dev/null +elif [ $machine = HERA ]; then export APRUNC=time export APRUNTF=time export OMP_NUM_THREADS_CH=24 - export SUB=/home/glopara/bin/sub_slurm + export SUB=$SLURM_SUBMIT_DIR/../util/sub_slurm export ACCOUNT=fv3-cpu export QUEUE=batch export QUEUE_TRANS=batch @@ -421,19 +404,6 @@ cat > read_hpss.sh <>/dev/null - #export machine=$machine - #if [ $machine = WCOSS_C ]; then - # . $MODULESHOME/init/sh 2>>/dev/null - # module load hpss 2>>/dev/null - #elif [ $machine = WCOSS_DELL_P3 ]; then - # . /usrx/local/prod/lmod/lmod/init/sh 2>>/dev/null - # module load HPSS/5.0.2.5 2>>/dev/null - #elif [ $machine = THEIA ]; then - # source $HOMEgfs/sorc/machine-setup.sh 2>>/dev/null - # module use -a /scratch3/NCEPDEV/nwprod/lib/modulefiles 2>>/dev/null - # module load hpss 2>>/dev/null - #fi - cd $INIDIR htar -xvf $HPSSPATH/$tarball_enkf_atm if [ $CDATE -le 2017072000 ] && [ $CDATE -gt 2014050100 ]; then diff --git a/ush/load_fv3gfs_modules.sh b/ush/load_fv3gfs_modules.sh index e48f1af47..8972b6575 100755 --- a/ush/load_fv3gfs_modules.sh +++ b/ush/load_fv3gfs_modules.sh @@ -15,9 +15,9 @@ module use "$HOMEgfs/modulefiles" if [[ -d /lfs3 ]] ; then # We are on NOAA Jet module load module_base.jet -elif [[ -d /scratch3 ]] ; then - # We are on NOAA Theia - module load module_base.theia +elif [[ -d /scratch1 ]] ; then + # We are on NOAA Hera + module load module_base.hera elif [[ -d /gpfs/hps && -e /etc/SuSE-release ]] ; then # We are on NOAA Luna or Surge module load module_base.wcoss_c diff --git a/util/sub_slurm b/util/sub_slurm new file mode 100755 index 000000000..97e70c9e2 --- /dev/null +++ b/util/sub_slurm @@ -0,0 +1,144 @@ +#!/bin/ksh +set -x +usage="\ +Usage: $0 [options] executable [args] + where the options are: + -a account account (default: none) + -b binding run smt binding or not (default:NO) + -d dirin initial directory (default: cwd) + -e envars copy comma-separated environment variables + -g group group name + -i append standard input to command file + -j jobname specify jobname (default: executable basename) + -m machine machine on which to run (default: current) + -n write command file to stdout rather than submitting it + -o output specify output file (default: jobname.out) + -p procs[/nodes[/ppreq] + number of MPI tasks and optional nodes or Bblocking and + ppreq option (N or S) (defaults: serial, Bunlimited, S) + -q queue[/qpreq] queue name and optional requirement, e.g. dev/P + (defaults: 1 if serial or dev if parallel and none) + (queue 3 or 4 is dev or prod with twice tasks over ip) + (options: P=parallel, B=bigmem, b=batch) + -r rmem[/rcpu] resources memory and cpus/task (default: '1024 mb', 1) + -t timew wall time limit in [[hh:]mm:]ss format (default: 900) + -u userid userid to run under (default: self) + -v verbose mode + -w when when to run, in yyyymmddhh[mm], +hh[mm], thh[mm], or + Thh[mm] (full, incremental, today or tomorrow) format + (default: now) +Function: This command submits a job to the batch queue." +subcmd="$*" +stdin=NO +nosub=NO +account="" +binding="NO" +dirin="" +envars="para" +group="" +jobname="" +machine="" +output="" +procs=0 +nodes="" +ppreq="N" +queue="" +qpreq="" +rmem="1200" +rcpu="1" +timew="900" +userid="" +verbose=NO +when="" +while getopts a:b:d:e:g:ij:m:no:p:q:r:t:u:vw: opt;do + case $opt in + a) account="$OPTARG";; + b) binding="$OPTARG";; + d) dirin="$OPTARG";; + e) envars="$OPTARG";; + g) group="$OPTARG";; + i) stdin=YES;; + j) jobname=$OPTARG;; + m) machine="$OPTARG";; + n) nosub=YES;; + o) output=$OPTARG;; + p) procs=$(echo $OPTARG/|cut -d/ -f1);nodes=$(echo $OPTARG/|cut -d/ -f2);ppreq=$(echo $OPTARG/|cut -d/ -f3);; + q) queue=$(echo $OPTARG/|cut -d/ -f1);qpreq=$(echo $OPTARG/|cut -d/ -f2);; + r) rmem=$(echo $OPTARG/|cut -d/ -f1);rcpu=$(echo $OPTARG/|cut -d/ -f2);; + t) timew=$OPTARG;; + u) userid=$OPTARG;; + v) verbose=YES;; + w) when=$OPTARG;; + \?) echo $0: invalid option >&2;echo "$usage" >&2;exit 1;; + esac +done +shift $(($OPTIND-1)) +if [[ $# -eq 0 ]];then + echo $0: missing executable name >&2;echo "$usage" >&2;exit 1 +fi +exec=$1 +if [[ ! -s $exec ]]&&which $exec >/dev/null 2>&1;then + exec=$(which $exec) +fi +shift +args="$*" +bn=$(basename $exec) +jobname=${jobname:-$bn} +output=${output:-$jobname.out} +myuser=$LOGNAME +myhost=$(hostname) + +DATA=${DATA:-/scratch4/NCEPDEV/stmp4/$LOGNAME/sub} +mkdir -p $DATA + +partition=${partition:-service} +queue=${queue:-batch} +timew=${timew:-01:20:00} +task_node=${procs:-12} +ntasks=$((nodes*task_node)) +envars=$envars + +#export TZ=GMT +export TZ="America/New_York" +cfile=$DATA/sub$$ +> $cfile +echo "#!/bin/ksh " >> $cfile +echo "#SBATCH -A $account" >> $cfile +echo "#SBATCH -o $output" >> $cfile +echo "#SBATCH -e $output" >> $cfile +echo "#SBATCH -J $jobname" >> $cfile +echo "#SBATCH -q $queue" >> $cfile +echo "#SBATCH -p $partition" >> $cfile +##echo "#SBATCH -v $envars" >> $cfile +echo "#SBATCH --nodes=$nodes --ntasks=$ntasks" >> $cfile +echo "#SBATCH -t $timew" >> $cfile +echo "/bin/ksh --login -x $exec $args" >> $cfile + +if [[ $stdin = YES ]];then + cat +fi >>$cfile +if [[ $nosub = YES ]];then + cat $cfile + exit +elif [[ $verbose = YES ]];then + set -x + cat $cfile +fi + +sbatch=${sbatch:-/apps/slurm/default/bin/sbatch} + +ofile=$DATA/subout$$ +>$ofile +chmod 777 $ofile +$sbatch $cfile >$ofile +rc=$? +cat $ofile +if [[ -w $SUBLOG ]];then + jobn=$(grep -i submitted $ofile|head -n1|cut -d\" -f2) +# date -u +"%Y%m%d%H%M%S : $subcmd : $jobn" >>$SUBLOG + date +"%Y%m%d%H%M%S : $subcmd : $jobn" >>$SUBLOG +fi +#exit +#rm $cfile $ofile +[[ $MKDATA = YES ]] && rmdir $DATA +exit $rc diff --git a/util/sub_wcoss_c b/util/sub_wcoss_c new file mode 100755 index 000000000..7bfaeeab7 --- /dev/null +++ b/util/sub_wcoss_c @@ -0,0 +1,277 @@ +#!/bin/ksh +set -x +# +# May 28, 2013 - Shrinivas Moorthi :now updated for lsf9.1.1 - should handle coupled case also +# +usage="\ +Usage: $0 [options] executable [args] + where the options are: + -a account account (default: none) + -b binding run smt binding or not (default:NO) + -d dirin initial directory (default: cwd) + -e envars copy comma-separated environment variables + -g group group name + -i append standard input to command file + -j jobname specify jobname (default: executable basename) +# -m machine machine on which to run (default: current) + -m mpiver mpi version (poe or intelmpi) (default: poe) + -n write command file to stdout rather than submitting it + -o output specify output file (default: jobname.out) + -p procs[/nodes[/ppreq] + number of MPI tasks and optional nodes or Bblocking and + ppreq option (N or S) (defaults: serial, Bunlimited, S) + -q queue[/qpreq] queue name and optional requirement, e.g. dev/P + (defaults: 1 if serial or dev if parallel and none) + (queue 3 or 4 is dev or prod with twice tasks over ip) + (options: P=parallel, B=bigmem, b=batch) + -r rmem[/rcpu[/pe_node] resources memory and cpus/task and cores per node (default: '1024 mb', 1, and 16) + -t timew wall time limit in [[hh:]mm:]ss format (default: 900) + -u userid userid to run under (default: self) + -v verbose mode + -w when when to run, in yyyymmddhh[mm], +hh[mm], thh[mm], or + Thh[mm] (full, incremental, today or tomorrow) format + (default: now) + + You can also export variables and + \"INHERIT_ENV\" (default:-\"YES\") - Set this variable to \"NO\" and + export it if you do not want the next job to inherit current job + environment. + + Other environmental variables which can be exported from outside are: + + \"KMP_STACKSIZE\" (default:-\"1024m\") + \"MP_EUIDEVELOP\" (default:-\"NULL\") + \"F_UFMTENDIAN\" (default:-\"NULL\") + \"MPICH_ALLTOALL_THROTTLE\" (default:-\"NULL\") + \"MP_SINGLE_THREAD\" (default:-\"NULL\") + \"MP_EAGER_LIMIT\" (default:-\"NULL\") + \"MP_USE_BULK_XFER\" (default:-\"NULL\") + \"MP_COLLECTIVE_OFFLOAD\" (default:-\"NULL\") + \"MP_SHARED_MEMORY\" (default:-\"NULL\") + \"MP_MPILIB\" (default:-\"NULL\") + \"MP_LABELIO\" (default:-\"NULL\") + \"MP_STDOUTMODE\" (default:-\"NULL\") + \"DATA\" (default:-\"/stmp/$LOGNAME/sub\" + - deleted at the end if created) + +Function: This command submits a job to the batch queue." +subcmd="$*" +stdin=NO +nosub=NO +account="" +binding="NO" +dirin="" +envars="" +group="" +jobname="" +#machine="" +mpiver="" +output="" +procs=0 +nodes="" +ppreq="NONE" +queue="" +qpreq="" +rmem="1024" +rcpu="1" +pe_node=${pe_node:-16} +timew="900" +userid="" +verbose=NO +when="" +while getopts a:b:d:e:g:ij:m:no:p:q:r:t:u:vw: opt;do + case $opt in + a) account="$OPTARG";; + b) binding="$OPTARG";; + d) dirin="$OPTARG";; + e) envars="$OPTARG";; + g) group="$OPTARG";; + i) stdin=YES;; + j) jobname=$OPTARG;; +# m) machine="$OPTARG";; + m) mpiver="$OPTARG";; + n) nosub=YES;; + o) output=$OPTARG;; + p) procs=$(echo $OPTARG/|cut -d/ -f1);nodes=$(echo $OPTARG/|cut -d/ -f2);ppreq=$(echo $OPTARG/|cut -d/ -f3);; + q) queue=$(echo $OPTARG/|cut -d/ -f1);qpreq=$(echo $OPTARG/|cut -d/ -f2);; + r) rmem=$(echo $OPTARG/|cut -d/ -f1);rcpu=$(echo $OPTARG/|cut -d/ -f2);pe_node=$(echo $OPTARG/|cut -d/ -f3);; + t) timew=$OPTARG;; + u) userid=$OPTARG;; + v) verbose=YES;; + w) when=$OPTARG;; + \?) echo $0: invalid option >&2;echo "$usage" >&2;exit 1;; + esac +done +shift $(($OPTIND-1)) +if [[ $# -eq 0 ]];then + echo $0: missing executable name >&2;echo "$usage" >&2;exit 1 +fi +exec=$1 +if [[ ! -s $exec ]]&&which $exec >/dev/null 2>&1;then + exec=$(which $exec) +fi +shift +args="$*" +bn=$(basename $exec) +jobname=${jobname:-$bn} +#machine=${machine:-""} +output=${output:-$jobname.out} +myuser=$LOGNAME +myhost=$(hostname) +#mpiver=${mpiver:-poe} +envars=$envars + +#DATA=/lustre/fs/scratch/$LOGNAME/stmp +NDATE=${NDATE:-/gpfs/hps/nco/ops/nwprod/prod_util.v1.0.5/exec/ndate} +pext=${pext:-""} +#PTMP=${PREPTMP:-""}/ptmp${pext:-""} +PTMP=${PTMP:-/gpfs/hps/ptmp} +DATA=${DATA:-$PTMP/$LOGNAME/sub} +if [ -s $DATA ] ; then + MKDATA=NO +else + mkdir -p $DATA + MKDATA=YES +fi +dirin=${dirin:-$(pwd)} + +queue=${queue:-dev} +timew=${timew:-01:20} +timew=$(echo $timew |cut -d: -f1):$(echo $timew |cut -d: -f2) +threads=${rcpu:-1} +nthreads=$threads + + +max_core=${max_core:-24} +task_node=${pe_node:-${task_node:-$max_core}} + +export INHERIT_ENV=${INHERIT_ENV:-YES} +if [ $nodes -eq 1 ] ; then + task_node=$procs +fi +tot_size=$procs +max_tasks=$((max_core*nodes)) +#tot_size=$procs +if [ $((task_node*threads)) -gt $max_core ]; then + core_typ=cpu + nthreads=$threads + threads=$((2*max_core/task_node)) + echo "Hyper-threading is used - setting core_typ=$corei_typ" +fi +export core_typ=${core_typ:-core} + +export OMP_STACKSIZE=${OMP_STACKSIZE:-1024m} +export KMP_AFFINITY=${KMP_AFFINITY:-disabled} + +export TZ=GMT +cfile=$DATA/sub$$ +> $cfile + +if [ $INHERIT_ENV = YES ] ; then + echo "#!/bin/ksh" >> $cfile +else + echo "#!/bin/sh --login" >> $cfile + echo "#BSUB -L /bin/sh" >> $cfile +fi +echo "#BSUB -P $account" >> $cfile +echo "#BSUB -e $output" >> $cfile +echo "#BSUB -o $output" >> $cfile +echo "#BSUB -J $jobname" >> $cfile +#echo "#BSUB -network type=sn_all:mode=US" >> $cfile +echo "#BSUB -q $queue" >> $cfile +echo "#BSUB -W $timew" >> $cfile +##echo "#BSUB -cwd $dirin" >> $cfile +#echo "#BSUB -n $tot_size" >> $cfile +if [ $queue = dev_transfer ] ; then + echo "#BSUB -R rusage[mem=$rmem]" >> $cfile +fi +if [ $queue != dev_transfer ] ; then + echo "#BSUB -M $rmem" >> $cfile + echo "#BSUB -extsched 'CRAYLINUX[]' -R '1*{select[craylinux && !vnode]} + $max_tasks*{select[craylinux && vnode]span[ptile=24] cu[type=cabinet]}'" >> $cfile +fi + +if [[ -n $when ]];then + whena=$when + if [[ $when = +* ]];then + hr=$(echo $when|cut -c2-3) + mn=$(echo $when|cut -c4-5) + [[ -n $mn ]] || mn=00 + now=$(date -u +"%Y%m%d%H%M") + ((mn+=$(echo $now|cut -c11-12))) + [[ $mn -ge 60 ]] && ((hr+=1)) && ((mn-=60)) + [[ $mn -lt 10 ]] && mn=0$mn + whena=$($NDATE +$hr $(echo $now|cut -c1-10))$mn + elif [[ $when = t* ]];then + hr=$(echo $when|cut -c2-3) + mn=$(echo $when|cut -c4-5) + [[ -n $mn ]] || mn=00 + now=$(date -u +"%Y%m%d") + whena=$now$hr$mn + elif [[ $when = T* ]];then + hr=$(echo $when|cut -c2-3) + mn=$(echo $when|cut -c4-5) + [[ -n $mn ]] || mn=00 + now=$(date -u +"%Y%m%d%H") + whena=$($NDATE +24 $now|cut -c1-8)$hr$mn + fi + yr=$(echo $whena|cut -c1-4) + mo=$(echo $whena|cut -c5-6) + dy=$(echo $whena|cut -c7-8) + hr=$(echo $whena|cut -c9-10) + mn=$(echo $whena|cut -c11-12) + [[ -n $mn ]] || mn=00 + echo "#BSUB -b $yr:$mo:$dy:$hr:$mn" >> $cfile +fi + +#echo "source ~${LOGNAME}/.profile" >> $cfile +#echo "ulimit -s unlimited" >> $cfile +#if [ ${MP_EUIDEVICE:-NULL} = sn_all ] ; then +#echo "#BSUB -network \"type=sn_all:mode=US\" " >> $cfile +#fi +#if [ ${MP_EULIB:-NULL} != NULL ] ; then +#echo "export MP_EUILIB=$MP_EUILIB" >> $cfile +#fi + +if [ ${KMP_AFFINITY:-NULL} != NULL ] ; then + echo "export KMP_AFFINITY=$KMP_AFFINITY" >> $cfile +fi + +echo "export OMP_STACKSIZE=$OMP_STACKSIZE" >> $cfile + +if [ ${MP_LABELIO:-NULL} != NULL ] ; then + echo "export MP_LABELIO=$MP_LABELIO" >> $cfile +fi +if [ ${MP_STDOUTMODE:-NULL} != NULL ] ; then + echo "export MP_STDOUTMODE=$MP_STDOUTMODE " >> $cfile +fi +for var in $(eval echo $envars | tr , ' ') ; do + echo "export $var" >> $cfile +done +echo "$exec" >> $cfile +echo "export OMP_NUM_THREADS=$nthreads" >> $cfile + +if [[ $stdin = YES ]];then + cat +fi >>$cfile +if [[ $nosub = YES ]];then + cat $cfile + exit +elif [[ $verbose = YES ]];then + set -x + cat $cfile +fi +bsub=${bsub:-$LSF_BINDIR/bsub} + +ofile=$DATA/subout$$ +>$ofile +chmod 777 $ofile +$bsub < $cfile +rc=$? +cat $ofile +if [[ -w $SUBLOG ]];then + jobn=$(grep -i submitted $ofile|head -n1|cut -d\" -f2) + date +"%Y%m%d%H%M%S : $subcmd : $jobn" >>$SUBLOG +fi +#rm $cfile $ofile +#[[ $MKDATA = YES ]] && rmdir $DATA +exit $rc diff --git a/util/sub_wcoss_d b/util/sub_wcoss_d new file mode 100755 index 000000000..77f0e4da8 --- /dev/null +++ b/util/sub_wcoss_d @@ -0,0 +1,247 @@ +#!/bin/ksh +set -x +# +# May 28, 2013 - Shrinivas Moorthi :now updated for lsf9.1.1 - should handle coupled case also +# +usage="\ +Usage: $0 [options] executable [args] + where the options are: + -a account account (default: none) + -b binding run smt binding or not (default:NO) + -d dirin initial directory (default: cwd) + -e envars copy comma-separated environment variables + -g group group name + -i append standard input to command file + -j jobname specify jobname (default: executable basename) +# -m machine machine on which to run (default: current) + -m mpiver mpi version (poe or intelmpi) (default: poe) + -n write command file to stdout rather than submitting it + -o output specify output file (default: jobname.out) + -p procs[/nodes[/ppreq] + number of MPI tasks and optional nodes or Bblocking and + ppreq option (N or S) (defaults: serial, Bunlimited, S) + -q queue[/qpreq] queue name and optional requirement, e.g. dev/P + (defaults: 1 if serial or dev if parallel and none) + (queue 3 or 4 is dev or prod with twice tasks over ip) + (options: P=parallel, B=bigmem, b=batch) + -r rmem[/rcpu[/pe_node] resources memory and cpus/task and cores per node (default: '1024 mb', 1, and 16) + -t timew wall time limit in [[hh:]mm:]ss format (default: 900) + -u userid userid to run under (default: self) + -v verbose mode + -w when when to run, in yyyymmddhh[mm], +hh[mm], thh[mm], or + Thh[mm] (full, incremental, today or tomorrow) format + (default: now) + + You can also export variables and + \"INHERIT_ENV\" (default:-\"YES\") - Set this variable to \"NO\" and + export it if you do not want the next job to inherit current job + environment. + + Other environmental variables which can be exported from outside are: + + \"KMP_STACKSIZE\" (default:-\"1024m\") + \"MP_EUIDEVELOP\" (default:-\"NULL\") + \"F_UFMTENDIAN\" (default:-\"NULL\") + \"MPICH_ALLTOALL_THROTTLE\" (default:-\"NULL\") + \"MP_SINGLE_THREAD\" (default:-\"NULL\") + \"MP_EAGER_LIMIT\" (default:-\"NULL\") + \"MP_USE_BULK_XFER\" (default:-\"NULL\") + \"MP_COLLECTIVE_OFFLOAD\" (default:-\"NULL\") + \"MP_SHARED_MEMORY\" (default:-\"NULL\") + \"MP_MPILIB\" (default:-\"NULL\") + \"MP_LABELIO\" (default:-\"NULL\") + \"MP_STDOUTMODE\" (default:-\"NULL\") + \"DATA\" (default:-\"/stmp/$LOGNAME/sub\" + - deleted at the end if created) + +Function: This command submits a job to the batch queue." +subcmd="$*" +stdin=NO +nosub=NO +account="" +binding="NO" +dirin="" +envars="" +group="" +jobname="" +#machine="" +mpiver="" +output="" +procs=0 +nodes="" +ppreq="NONE" +queue="" +qpreq="" +rmem="1024" +rcpu="1" +pe_node=${pe_node:-16} +timew="900" +userid="" +verbose=NO +when="" +while getopts a:b:d:e:g:ij:m:no:p:q:r:t:u:vw: opt;do + case $opt in + a) account="$OPTARG";; + b) binding="$OPTARG";; + d) dirin="$OPTARG";; + e) envars="$OPTARG";; + g) group="$OPTARG";; + i) stdin=YES;; + j) jobname=$OPTARG;; + m) mpiver="$OPTARG";; + n) nosub=YES;; + o) output=$OPTARG;; + p) procs=$(echo $OPTARG/|cut -d/ -f1);nodes=$(echo $OPTARG/|cut -d/ -f2);ppreq=$(echo $OPTARG/|cut -d/ -f3);; + q) queue=$(echo $OPTARG/|cut -d/ -f1);qpreq=$(echo $OPTARG/|cut -d/ -f2);; + r) rmem=$(echo $OPTARG/|cut -d/ -f1);rcpu=$(echo $OPTARG/|cut -d/ -f2);pe_node=$(echo $OPTARG/|cut -d/ -f3);; + t) timew=$OPTARG;; + u) userid=$OPTARG;; + v) verbose=YES;; + w) when=$OPTARG;; + \?) echo $0: invalid option >&2;echo "$usage" >&2;exit 1;; + esac +done +shift $(($OPTIND-1)) +if [[ $# -eq 0 ]];then + echo $0: missing executable name >&2;echo "$usage" >&2;exit 1 +fi +exec=$1 +if [[ ! -s $exec ]]&&which $exec >/dev/null 2>&1;then + exec=$(which $exec) +fi +shift +args="$*" +bn=$(basename $exec) +jobname=${jobname:-$bn} +#machine=${machine:-""} +output=${output:-$jobname.out} +myuser=$LOGNAME +myhost=$(hostname) +envars=$envars + +#DATA=/lustre/fs/scratch/$LOGNAME/stmp +NDATE=${NDATE:-/gpfs/hps/nco/ops/nwprod/prod_util.v1.0.5/exec/ndate} +pext=${pext:-""} +PTMP=/gpfs/dell2/ptmp +DATA=${DATA:-$PTMP/$LOGNAME/sub} +if [ -s $DATA ] ; then + MKDATA=NO +else + mkdir -p $DATA + MKDATA=YES +fi +dirin=${dirin:-$(pwd)} + +queue=${queue:-dev} +timew=${timew:-01:20} +timew=$(echo $timew |cut -d: -f1):$(echo $timew |cut -d: -f2) +threads=${rcpu:-1} +nthreads=$threads + +max_core=${max_core:-28} +task_node=${pe_node:-${task_node:-$max_core}} + +export INHERIT_ENV=${INHERIT_ENV:-YES} +if [ $nodes -eq 1 ] ; then + task_node=$procs +fi +tot_size=$procs +max_tasks=$((max_core*nodes)) +if [ $((task_node*threads)) -gt $max_core ]; then + core_typ=cpu + nthreads=$threads + threads=$((2*max_core/task_node)) + echo "Hyper-threading is used - setting core_typ=$corei_typ" +fi +core_typ=${core_typ:-core} + +export OMP_STACKSIZE=${OMP_STACKSIZE:-1024m} +export KMP_AFFINITY=${KMP_AFFINITY:-disabled} + +export TZ=GMT +cfile=$DATA/sub$$ +> $cfile + +echo "#!/bin/bash" >> $cfile +echo "#BSUB -P $account" >> $cfile +echo "#BSUB -e $output" >> $cfile +echo "#BSUB -o $output" >> $cfile +echo "#BSUB -J $jobname" >> $cfile +echo "#BSUB -q $queue" >> $cfile +echo "#BSUB -W $timew" >> $cfile +echo "#BSUB -n $tot_size" >> $cfile +echo "#BSUB -R span[ptile=$task_node]" >> $cfile +echo "#BSUB -R affinity[core($nthreads)]" >> $cfile +echo "#BSUB -M $rmem" >> $cfile + + +if [[ -n $when ]];then + whena=$when + if [[ $when = +* ]];then + hr=$(echo $when|cut -c2-3) + mn=$(echo $when|cut -c4-5) + [[ -n $mn ]] || mn=00 + now=$(date -u +"%Y%m%d%H%M") + ((mn+=$(echo $now|cut -c11-12))) + [[ $mn -ge 60 ]] && ((hr+=1)) && ((mn-=60)) + [[ $mn -lt 10 ]] && mn=0$mn + whena=$($NDATE +$hr $(echo $now|cut -c1-10))$mn + elif [[ $when = t* ]];then + hr=$(echo $when|cut -c2-3) + mn=$(echo $when|cut -c4-5) + [[ -n $mn ]] || mn=00 + now=$(date -u +"%Y%m%d") + whena=$now$hr$mn + elif [[ $when = T* ]];then + hr=$(echo $when|cut -c2-3) + mn=$(echo $when|cut -c4-5) + [[ -n $mn ]] || mn=00 + now=$(date -u +"%Y%m%d%H") + whena=$($NDATE +24 $now|cut -c1-8)$hr$mn + fi + yr=$(echo $whena|cut -c1-4) + mo=$(echo $whena|cut -c5-6) + dy=$(echo $whena|cut -c7-8) + hr=$(echo $whena|cut -c9-10) + mn=$(echo $whena|cut -c11-12) + [[ -n $mn ]] || mn=00 + echo "#BSUB -b $yr:$mo:$dy:$hr:$mn" >> $cfile +fi + +if [ ${KMP_AFFINITY:-NULL} != NULL ] ; then + echo "export KMP_AFFINITY=$KMP_AFFINITY" >> $cfile +fi +echo "export OMP_STACKSIZE=$OMP_STACKSIZE" >> $cfile + + +for var in $(eval echo $envars | tr , ' ') ; do + echo "export $var" >> $cfile +done +echo "$exec" >> $cfile +echo "export OMP_NUM_THREADS=$nthreads" >> $cfile + +if [[ $stdin = YES ]];then + cat +fi >>$cfile +if [[ $nosub = YES ]];then + cat $cfile + exit +elif [[ $verbose = YES ]];then + set -x + cat $cfile +fi +bsub=${bsub:-$LSF_BINDIR/bsub} + +ofile=$DATA/subout$$ +>$ofile +chmod 777 $ofile +$bsub < $cfile +rc=$? +cat $ofile +if [[ -w $SUBLOG ]];then + jobn=$(grep -i submitted $ofile|head -n1|cut -d\" -f2) + date +"%Y%m%d%H%M%S : $subcmd : $jobn" >>$SUBLOG +fi +##rm $cfile $ofile +#[[ $MKDATA = YES ]] && rmdir $DATA +exit $rc From 1b7699418e592d6f46fe20f0cf777e5ce9c3ca58 Mon Sep 17 00:00:00 2001 From: XuLi-NOAA <55100838+XuLi-NOAA@users.noreply.github.com> Date: Fri, 11 Oct 2019 12:44:39 -0400 Subject: [PATCH 29/34] A few sea ice related issues in the NSST analysis: Issue #18 (#20) develop: This commit references issue #18. Updates to global_cycle: Tf analysis for the grids with sea ice done using a salinity dependent formula (instead of using 271.2 K). Update sfcsub.F to the latest version used by the forecast model (includes elimination of masked interpolation for substrate temperature and some updates to Fortran 90 standards). The global_cycle.sh script was updated to reference a new global salinity dataset. Updates to global_chgres - Update sfcsub.F to use same version as global_cycle. --- sorc/global_chgres.fd/sfcsub.F | 207 +++---- sorc/global_cycle.fd/cycle.f90 | 688 ++++++++++++++++++++--- sorc/global_cycle.fd/read_write_data.f90 | 356 ++++++++++++ sorc/global_cycle.fd/sfcsub.F | 207 +++---- ush/global_cycle.sh | 7 + 5 files changed, 1192 insertions(+), 273 deletions(-) mode change 100755 => 100644 sorc/global_cycle.fd/cycle.f90 diff --git a/sorc/global_chgres.fd/sfcsub.F b/sorc/global_chgres.fd/sfcsub.F index b14271cd0..4fbabab8f 100644 --- a/sorc/global_chgres.fd/sfcsub.F +++ b/sorc/global_chgres.fd/sfcsub.F @@ -2697,12 +2697,13 @@ subroutine fixrdg(lugb,idim,jdim,fngrib, real (kind=kind_io8), allocatable :: data8(:) real (kind=kind_io4), allocatable :: data4(:) ! - logical*1 lbms(mdata) + logical*1, allocatable :: lbms(:) ! integer kpds(200),kgds(200) integer jpds(200),jgds(200), kpds0(200) ! allocate(data8(1:idim*jdim)) + allocate(lbms(1:mdata)) kpds = 0 kgds = 0 jpds = 0 @@ -2792,6 +2793,7 @@ subroutine fixrdg(lugb,idim,jdim,fngrib, endif ! deallocate(data8) + deallocate(lbms) return end subroutine getarea(kgds,dlat,dlon,rslat,rnlat,wlon,elon,ijordr @@ -6078,17 +6080,24 @@ subroutine setrmsk(kpds5,slmask,igaul,jgaul,wlon,rnlat, ! ijmax = imax*jmax rslmsk = 0. +! TG3 MODS BEGIN + if(kpds5 == kpdtsf .and. imax == 138 .and. jmax == 116 + & .and. kpds4 == 128) then +! print*,'turn off setrmsk for tg3' + lmask = .false. + + elseif(kpds5 == kpdtsf) then +! TG3 MODS END ! ! surface temperature ! - if(kpds5.eq.kpdtsf) then -! lmask=.false. + lmask = .false. call ga2la(slmask,igaul,jgaul,rslmsk,imax,jmax,wlon,rnlat &, rlnout, rltout, gaus, blno, blto) ! &, dlon, dlat, gaus, blno, blto) - crit=0.5 + crit = 0.5 call rof01(rslmsk,ijmax,'ge',crit) - lmask=.true. + lmask = .true. ! ! bucket soil wetness ! @@ -6096,16 +6105,16 @@ subroutine setrmsk(kpds5,slmask,igaul,jgaul,wlon,rnlat, call ga2la(slmask,igaul,jgaul,rslmsk,imax,jmax,wlon,rnlat &, rlnout, rltout, gaus, blno, blto) ! &, dlon, dlat, gaus, blno, blto) - crit=0.5 + crit = 0.5 call rof01(rslmsk,ijmax,'ge',crit) - lmask=.true. + lmask = .true. ! write(6,*) 'wet rslmsk' ! znnt=1. ! call nntprt(rslmsk,ijmax,znnt) ! ! snow depth ! - elseif(kpds5.eq.kpdsnd) then + elseif(kpds5 == kpdsnd) then if(kpds4 == 192) then ! use the bitmap rslmsk = 0. do j = 1, jmax @@ -6962,51 +6971,51 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, ! ! get tsf climatology for the begining of the forecast ! - if (fh .gt. 0.0) then + if (fh > 0.0) then !cbosu if (me == 0) print*,'bosu fh gt 0' - iy4=iy - if(iy.lt.101) iy4=1900+iy4 - fha=0 - ida=0 - jda=0 -! fha(2)=nint(fh) - ida(1)=iy - ida(2)=im - ida(3)=id - ida(5)=ih + iy4 = iy + if (iy < 101) iy4 = 1900 + iy4 + fha = 0 + ida = 0 + jda = 0 +! fha(2) = nint(fh) + ida(1) = iy + ida(2) = im + ida(3) = id + ida(5) = ih call w3kind(w3kindreal,w3kindint) if(w3kindreal == 4) then - fha4=fha + fha4 = fha call w3movdat(fha4,ida,jda) else call w3movdat(fha,ida,jda) endif - jy=jda(1) - jm=jda(2) - jd=jda(3) - jh=jda(5) - if (me .eq. 0) write(6,*) ' forecast jy,jm,jd,jh', - & jy,jm,jd,jh + jy = jda(1) + jm = jda(2) + jd = jda(3) + jh = jda(5) + if (me == 0) write(6,*) ' forecast jy,jm,jd,jh', + & jy,jm,jd,jh jdow = 0 jdoy = 0 jday = 0 call w3doxdat(jda,jdow,jdoy,jday) - rjday=jdoy+jda(5)/24. - if(rjday.lt.dayhf(1)) rjday=rjday+365. + rjday = jdoy + jda(5) / 24. + if(rjday < dayhf(1)) rjday = rjday + 365. ! - if (me .eq. 0) write(6,*) 'forecast jy,jm,jd,jh=',jy,jm,jd,jh + if (me == 0) write(6,*) 'forecast jy,jm,jd,jh=',jy,jm,jd,jh ! ! for monthly mean climatology ! monend = 12 do mm=1,monend - mmm=mm - mmp=mm+1 - if(rjday.ge.dayhf(mmm).and.rjday.lt.dayhf(mmp)) then - mon1=mmm - mon2=mmp + mmm = mm + mmp = mm + 1 + if(rjday >= dayhf(mmm) .and. rjday < dayhf(mmp)) then + mon1 = mmm + mon2 = mmp go to 10 endif enddo @@ -7014,17 +7023,18 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, call abort 10 continue wei1m = (dayhf(mon2)-rjday)/(dayhf(mon2)-dayhf(mon1)) - wei2m = (rjday-dayhf(mon1))/(dayhf(mon2)-dayhf(mon1)) - if(mon2.eq.13) mon2=1 - if (me .eq. 0) print *,'rjday,mon1,mon2,wei1m,wei2m=', - & rjday,mon1,mon2,wei1m,wei2m + wei2m = 1.0 - wei1m +! wei2m = (rjday-dayhf(mon1))/(dayhf(mon2)-dayhf(mon1)) + if (mon2 == 13) mon2 = 1 + if (me == 0) print *,'rjday,mon1,mon2,wei1m,wei2m=', + & rjday,mon1,mon2,wei1m,wei2m ! ! read monthly mean climatology of tsf ! kpd7 = -1 do nn=1,2 mon = mon1 - if (nn .eq. 2) mon = mon2 + if (nn == 2) mon = mon2 call fixrdc(lugb,fntsfc,kpdtsf,kpd7,mon,slmask, & tsf(1,nn),len,iret &, imsk, jmsk, slmskh, gaus,blno, blto @@ -7041,8 +7051,8 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, ! ! compute current jy,jm,jd,jh of forecast and the day of the year ! - iy4=iy - if(iy.lt.101) iy4=1900+iy4 + iy4 = iy + if (iy < 101) iy4=1900+iy4 fha = 0 ida = 0 jda = 0 @@ -7052,8 +7062,8 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, ida(3) = id ida(5) = ih call w3kind(w3kindreal,w3kindint) - if(w3kindreal==4) then - fha4=fha + if(w3kindreal == 4) then + fha4 = fha call w3movdat(fha4,ida,jda) else call w3movdat(fha,ida,jda) @@ -7068,44 +7078,45 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, jdoy = 0 jday = 0 call w3doxdat(jda,jdow,jdoy,jday) - rjday = jdoy+jda(5)/24. - if(rjday.lt.dayhf(1)) rjday=rjday+365. + rjday = jdoy + jda(5) / 24. + if(rjday < dayhf(1)) rjday = rjday + 365. - if (me .eq. 0) write(6,*) ' forecast jy,jm,jd,jh,rjday=', - & jy,jm,jd,jh,rjday + if (me == 0) write(6,*) ' forecast jy,jm,jd,jh,rjday=', + & jy,jm,jd,jh,rjday ! - if (me .eq. 0) write(6,*) 'forecast jy,jm,jd,jh=',jy,jm,jd,jh + if (me == 0) write(6,*) 'forecast jy,jm,jd,jh=',jy,jm,jd,jh ! ! for monthly mean climatology ! monend = 12 do mm=1,monend - mmm=mm - mmp=mm+1 - if(rjday.ge.dayhf(mmm).and.rjday.lt.dayhf(mmp)) then - mon1=mmm - mon2=mmp + mmm = mm + mmp = mm + 1 + if(rjday >= dayhf(mmm) .and. rjday < dayhf(mmp)) then + mon1 = mmm + mon2 = mmp go to 20 endif enddo print *,'wrong rjday',rjday call abort 20 continue - wei1m=(dayhf(mon2)-rjday)/(dayhf(mon2)-dayhf(mon1)) - wei2m=(rjday-dayhf(mon1))/(dayhf(mon2)-dayhf(mon1)) - if(mon2.eq.13) mon2=1 - if (me .eq. 0) print *,'rjday,mon1,mon2,wei1m,wei2m=', - & rjday,mon1,mon2,wei1m,wei2m + wei1m = (dayhf(mon2)-rjday)/(dayhf(mon2)-dayhf(mon1)) + wei2m = 1.0 - wei1m +! wei2m = (rjday-dayhf(mon1))/(dayhf(mon2)-dayhf(mon1)) + if (mon2 == 13) mon2 = 1 + if (me == 0) print *,'rjday,mon1,mon2,wei1m,wei2m=', + & rjday,mon1,mon2,wei1m,wei2m ! ! for seasonal mean climatology ! monend = 4 is = im/3 + 1 - if (is.eq.5) is = 1 + if (is == 5) is = 1 do mm=1,monend mmm = mm*3 - 2 mmp = (mm+1)*3 - 2 - if(rjday.ge.dayhf(mmm).and.rjday.lt.dayhf(mmp)) then + if(rjday >= dayhf(mmm) .and. rjday < dayhf(mmp)) then sea1 = mmm sea2 = mmp go to 30 @@ -7115,20 +7126,21 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, call abort 30 continue wei1s = (dayhf(sea2)-rjday)/(dayhf(sea2)-dayhf(sea1)) - wei2s = (rjday-dayhf(sea1))/(dayhf(sea2)-dayhf(sea1)) - if(sea2.eq.13) sea2=1 - if (me .eq. 0) print *,'rjday,sea1,sea2,wei1s,wei2s=', - & rjday,sea1,sea2,wei1s,wei2s + wei2s = 1.0 - wei1s +! wei2s = (rjday-dayhf(sea1))/(dayhf(sea2)-dayhf(sea1)) + if (sea2 == 13) sea2 = 1 + if (me == 0) print *,'rjday,sea1,sea2,wei1s,wei2s=', + & rjday,sea1,sea2,wei1s,wei2s ! ! for summer and winter values (maximum and minimum). ! monend = 2 is = im/6 + 1 - if (is.eq.3) is = 1 + if (is == 3) is = 1 do mm=1,monend mmm = mm*6 - 5 mmp = (mm+1)*6 - 5 - if(rjday.ge.dayhf(mmm).and.rjday.lt.dayhf(mmp)) then + if(rjday >= dayhf(mmm) .and. rjday < dayhf(mmp)) then hyr1 = mmm hyr2 = mmp go to 31 @@ -7138,10 +7150,11 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, call abort 31 continue wei1y = (dayhf(hyr2)-rjday)/(dayhf(hyr2)-dayhf(hyr1)) - wei2y = (rjday-dayhf(hyr1))/(dayhf(hyr2)-dayhf(hyr1)) - if(hyr2.eq.13) hyr2=1 - if (me .eq. 0) print *,'rjday,hyr1,hyr2,wei1y,wei2y=', - & rjday,hyr1,hyr2,wei1y,wei2y + wei2y = 1.0 - wei1y +! wei2y = (rjday-dayhf(hyr1))/(dayhf(hyr2)-dayhf(hyr1)) + if (hyr2 == 13) hyr2 = 1 + if (me == 0) print *,'rjday,hyr1,hyr2,wei1y,wei2y=', + & rjday,hyr1,hyr2,wei1y,wei2y ! ! start reading in climatology and interpolate to the date ! @@ -7541,7 +7554,7 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, ! mon1s = mon1 ; mon2s = mon2 ; sea1s = sea1 ; sea2s = sea2 ! - if (me .eq. 0) print *,' mon1s=',mon1s,' mon2s=',mon2s + if (me == 0) print *,' mon1s=',mon1s,' mon2s=',mon2s &,' sea1s=',sea1s,' sea2s=',sea2s ! k1 = 1 ; k2 = 2 @@ -7599,11 +7612,11 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, ! seasonal mean climatology ! isx = sea2/3 + 1 - if (isx .eq. 5) isx = 1 - if(isx.eq.1) kpd9 = 12 - if(isx.eq.2) kpd9 = 3 - if(isx.eq.3) kpd9 = 6 - if(isx.eq.4) kpd9 = 9 + if (isx == 5) isx = 1 + if (isx == 1) kpd9 = 12 + if (isx == 2) kpd9 = 3 + if (isx == 3) kpd9 = 6 + if (isx == 4) kpd9 = 9 ! ! albedo ! there are four albedo fields in this version: @@ -7639,7 +7652,7 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, if (me == 0) print*,'bosu 2nd time in clima for month ', & mon, k1,k2 if ( index(fnalbc, "tileX.nc") == 0) then ! grib file - kpd7=-1 + kpd7 = -1 do k = 1, 4 call fixrdc(lugb,fnalbc,kpdalb(k),kpd7,mon,slmask, & alb(1,k,nn),len,iret @@ -7656,7 +7669,7 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, ! ! tsf at the current time t ! - kpd7=-1 + kpd7 = -1 call fixrdc(lugb,fntsfc,kpdtsf,kpd7,mon,slmask, & tsf(1,nn),len,iret &, imsk, jmsk, slmskh, gaus,blno, blto @@ -7664,13 +7677,13 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, ! ! soil wetness ! - if(fnwetc(1:8).ne.' ') then + if (fnwetc(1:8).ne.' ') then kpd7=-1 call fixrdc(lugb,fnwetc,kpdwet,kpd7,mon,slmask, & wet(1,nn),len,iret &, imsk, jmsk, slmskh, gaus,blno, blto &, outlat, outlon, me) - elseif(fnsmcc(1:8).ne.' ') then + elseif (fnsmcc(1:8).ne.' ') then if (index(fnsmcc,'global_soilmcpc.1x1.grb') /= 0) then ! the old climo data kpd7=-1 call fixrdc(lugb,fnsmcc,kpdsmc,kpd7,mon,slmask, @@ -7712,13 +7725,13 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, ! ! sea ice ! - kpd7=-1 - if(fnacnc(1:8).ne.' ') then + kpd7 = -1 + if (fnacnc(1:8).ne.' ') then call fixrdc(lugb,fnacnc,kpdacn,kpd7,mon,slmask, & acn(1,nn),len,iret &, imsk, jmsk, slmskh, gaus,blno, blto &, outlat, outlon, me) - elseif(fnaisc(1:8).ne.' ') then + elseif (fnaisc(1:8).ne.' ') then call fixrdc(lugb,fnaisc,kpdais,kpd7,mon,slmask, & ais(1,nn),len,iret &, imsk, jmsk, slmskh, gaus,blno, blto @@ -7738,7 +7751,7 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, ! ! snow cover ! - if(fnscvc(1:8).ne.' ') then + if (fnscvc(1:8).ne.' ') then kpd7=-1 call fixrdc(lugb,fnscvc,kpdscv,kpd7,mon,slmask, & scv(1,nn),len,iret @@ -7749,7 +7762,7 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, ! ! surface roughness ! - if(fnzorc(1:3) == 'sib') then + if (fnzorc(1:3) == 'sib') then if (me == 0) then write(6,*) 'roughness length to be set from sib veg type' endif @@ -7767,7 +7780,7 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, ! ! vegetation cover ! - if(fnvegc(1:8).ne.' ') then + if (fnvegc(1:8) .ne. ' ') then if ( index(fnvegc, "tileX.nc") == 0) then ! grib file kpd7=-1 call fixrdc(lugb,fnvegc,kpdveg,kpd7,mon,slmask, @@ -7789,35 +7802,35 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, ! when chosen, set the z0 based on the vegetation type. ! for this option to work, namelist variable fnvetc must be ! set to point at the proper vegetation type file. - if(fnzorc(1:3) == 'sib') then - if(fnvetc(1:4) == ' ') then + if (fnzorc(1:3) == 'sib') then + if (fnvetc(1:4) == ' ') then if (me==0) write(6,*) "must choose sib veg type climo file" call abort endif zorclm = 0.0 do i=1,len - ivtyp=nint(vet(i)) + ivtyp = nint(vet(i)) if (ivtyp >= 1 .and. ivtyp <= 13) then zorclm(i) = z0_sib(ivtyp) endif enddo elseif(fnzorc(1:4) == 'igbp') then - if(fnvetc(1:4) == ' ') then - if (me==0) write(6,*) "must choose igbp veg type climo file" + if (fnvetc(1:4) == ' ') then + if (me == 0) write(6,*) "must choose igbp veg type climo file" call abort endif zorclm = 0.0 do i=1,len - ivtyp=nint(vet(i)) + ivtyp = nint(vet(i)) if (ivtyp >= 1 .and. ivtyp <= 20) then z0_season(1) = z0_igbp_min(ivtyp) z0_season(7) = z0_igbp_max(ivtyp) - if(outlat(i) < 0.0)then + if (outlat(i) < 0.0) then zorclm(i) = wei1y * z0_season(hyr2) + - & wei2y *z0_season(hyr1) + & wei2y * z0_season(hyr1) else zorclm(i) = wei1y * z0_season(hyr1) + - & wei2y *z0_season(hyr2) + & wei2y * z0_season(hyr2) endif endif enddo @@ -8148,7 +8161,7 @@ subroutine fixrdc(lugb,fngrib,kpds5,kpds7,mon,slmask, real (kind=kind_io8), allocatable :: rlngrb(:), rltgrb(:) ! logical lmask, yr2kc, gaus, ijordr - logical*1 lbms(mdata) + logical*1, allocatable :: lbms(:) ! integer, intent(in) :: kpds7 integer kpds(1000),kgds(1000) @@ -8156,6 +8169,7 @@ subroutine fixrdc(lugb,fngrib,kpds5,kpds7,mon,slmask, real (kind=kind_io8) outlat(len), outlon(len) ! allocate(data8(1:mdata)) + allocate(lbms(mdata)) ! ! integer imax_sv, jmax_sv, wlon_sv, rnlat_sv, kpds1_sv ! date imax_sv/0/, jmax_sv/0/, wlon_sv/999.0/, rnlat_sv/999.0/ @@ -8291,6 +8305,7 @@ subroutine fixrdc(lugb,fngrib,kpds5,kpds7,mon,slmask, call baclose(lugb,iret) ! deallocate(data8) + deallocate(lbms) return end subroutine fixrdc subroutine fixrda(lugb,fngrib,kpds5,slmask, diff --git a/sorc/global_cycle.fd/cycle.f90 b/sorc/global_cycle.fd/cycle.f90 old mode 100755 new mode 100644 index c5a7123c6..5aa800f9b --- a/sorc/global_cycle.fd/cycle.f90 +++ b/sorc/global_cycle.fd/cycle.f90 @@ -227,7 +227,8 @@ SUBROUTINE SFCDRV(LUGB,IDIM,JDIM,LENSFC,LSOIL,DELTSFC, & REAL, ALLOCATABLE :: SLIFCS_FG(:) TYPE(NSST_DATA) :: NSST - + real, dimension(idim,jdim) :: tf_clm,tf_trd,sal_clm + real, dimension(lensfc) :: tf_clm_tile,tf_trd_tile,sal_clm_tile !-------------------------------------------------------------------------------- ! GSI_FILE is the path/name of the gaussian GSI file which contains NSST ! increments. @@ -461,9 +462,27 @@ SUBROUTINE SFCDRV(LUGB,IDIM,JDIM,LENSFC,LSOIL,DELTSFC, & ELSE PRINT* PRINT*,"ADJUST TREF FROM GSI INCREMENT" +! +! Get tf climatology at the time +! + call get_tf_clm(rla,rlo,jdim,idim,iy,im,id,ih,tf_clm,tf_trd) + tf_clm_tile(:) = reshape(tf_clm, (/lensfc/) ) + tf_trd_tile(:) = reshape(tf_trd, (/lensfc/) ) +! +! Get salinity climatology at the time +! + call get_sal_clm(rla,rlo,jdim,idim,iy,im,id,ih,sal_clm) + sal_clm_tile(:) = reshape(sal_clm, (/lensfc/) ) +! +! read tf analysis increment generated by GSI +! CALL READ_GSI_DATA(GSI_FILE) - CALL ADJUST_NSST(RLA,RLO,SLIFCS,SLIFCS_FG,TSFFCS,SITFCS,STCFCS, & - NSST,LENSFC,LSOIL,IDIM,JDIM,ZSEA1,ZSEA2,IM,ID,DELTSFC) +! +! update foundation & surface temperature for NSST +! + CALL ADJUST_NSST(RLA,RLO,SLIFCS,SLIFCS_FG,TSFFCS,SITFCS,SICFCS,STCFCS, & + NSST,LENSFC,LSOIL,IDIM,JDIM,ZSEA1,ZSEA2,IM,ID,DELTSFC, & + tf_clm_tile,tf_trd_tile,sal_clm_tile) ENDIF ENDIF @@ -508,8 +527,9 @@ SUBROUTINE SFCDRV(LUGB,IDIM,JDIM,LENSFC,LSOIL,DELTSFC, & END SUBROUTINE SFCDRV SUBROUTINE ADJUST_NSST(RLA,RLO,SLMSK_TILE,SLMSK_FG_TILE,SKINT_TILE,& - SICET_TILE,SOILT_TILE,NSST,LENSFC,LSOIL, & - IDIM,JDIM,ZSEA1,ZSEA2,MON,DAY,DELTSFC) + SICET_TILE,sice_tile,SOILT_TILE,NSST,LENSFC,LSOIL, & + IDIM,JDIM,ZSEA1,ZSEA2,MON,DAY,DELTSFC, & + tf_clm_tile,tf_trd_tile,sal_clm_tile) !-------------------------------------------------------------------------------- ! READ IN GSI FILE WITH THE UPDATED TREF INCREMENTS (ON THE GAUSSIAN @@ -529,14 +549,14 @@ SUBROUTINE ADJUST_NSST(RLA,RLO,SLMSK_TILE,SLMSK_FG_TILE,SKINT_TILE,& INTEGER, INTENT(IN) :: LENSFC, LSOIL, IDIM, JDIM, MON, DAY REAL, INTENT(IN) :: SLMSK_TILE(LENSFC), SLMSK_FG_TILE(LENSFC) + real, intent(in) :: tf_clm_tile(lensfc),tf_trd_tile(lensfc),sal_clm_tile(lensfc) REAL, INTENT(IN) :: ZSEA1, ZSEA2, DELTSFC REAL, INTENT(INOUT) :: RLA(LENSFC), RLO(LENSFC), SKINT_TILE(LENSFC) - REAL, INTENT(INOUT) :: SICET_TILE(LENSFC), SOILT_TILE(LENSFC,LSOIL) + REAL, INTENT(INOUT) :: SICET_TILE(LENSFC),sice_tile(lensfc),SOILT_TILE(LENSFC,LSOIL) TYPE(NSST_DATA) :: NSST - REAL, PARAMETER :: TFREEZ=271.21 - REAL, PARAMETER :: TMAX=313.0 + REAL, PARAMETER :: TMAX=313.0,tzero=273.16 INTEGER :: IOPT, NRET, KGDS_GAUS(200) INTEGER :: IGAUS, JGAUS, IJ, II, JJ, III, JJJ, KRAD @@ -545,11 +565,16 @@ SUBROUTINE ADJUST_NSST(RLA,RLO,SLMSK_TILE,SLMSK_FG_TILE,SKINT_TILE,& INTEGER :: ITILE, JTILE INTEGER :: MAX_SEARCH, J INTEGER :: IGAUSP1, JGAUSP1 + integer :: nintp,nsearched,nice,nland + integer :: nfill,nfill_tice,nfill_clm + integer :: nset_thaw,nset_thaw_s,nset_thaw_i,nset_thaw_c + INTEGER, ALLOCATABLE :: ID1(:,:), ID2(:,:), JDC(:,:) LOGICAL :: IS_ICE - REAL :: TREF_SAVE, WSUM + real :: tfreez + REAL :: TREF_SAVE,WSUM,tf_ice,tf_thaw REAL :: FILL, DTZM, GAUS_RES_KM, DTREF REAL, ALLOCATABLE :: XPTS(:), YPTS(:), LATS(:), LONS(:) REAL, ALLOCATABLE :: DUM2D(:,:), LATS_RAD(:), LONS_RAD(:) @@ -646,6 +671,20 @@ SUBROUTINE ADJUST_NSST(RLA,RLO,SLMSK_TILE,SLMSK_FG_TILE,SKINT_TILE,& PRINT*,'MAXIMUM SEARCH IS ',MAX_SEARCH, ' GAUSSIAN POINTS.' PRINT* +! +! Initialize variables for counts statitics to be zeros +! + nintp = 0 + nset_thaw = 0 + nset_thaw_s = 0 + nset_thaw_i = 0 + nset_thaw_c = 0 + nsearched = 0 + nfill = 0 + nfill_tice = 0 + nfill_clm = 0 + nice = 0 + nland = 0 !---------------------------------------------------------------------- ! TREF INCREMENT WILL BE OUTPUT. INITIALIZE TO ZERO. !---------------------------------------------------------------------- @@ -657,31 +696,50 @@ SUBROUTINE ADJUST_NSST(RLA,RLO,SLMSK_TILE,SLMSK_FG_TILE,SKINT_TILE,& MASK_TILE = NINT(SLMSK_TILE(IJ)) MASK_FG_TILE = NINT(SLMSK_FG_TILE(IJ)) +! +! when sea ice exists, get salinity dependent water temperature +! + tf_ice = tfreez(sal_clm_tile(ij)) + tzero !---------------------------------------------------------------------- -! THESE ARE ICE POINTS. SET TREF TO FREEZING. +! SKIP LAND POINTS. NSST NOT APPLIED AT LAND. !---------------------------------------------------------------------- - IF (MASK_TILE == 2) THEN - NSST%TREF(IJ) = TFREEZ - CYCLE IJ_LOOP + IF (MASK_TILE == 1) THEN + nland = nland + 1 + CYCLE IJ_LOOP ENDIF -!---------------------------------------------------------------------- -! IF THE MODEL POINT WAS ICE COVERED, BUT IS NOW OPEN WATER, SET -! TREF TO FREEZING, XZ TO '30' AND ALL OTHER FIELDS TO ZERO. -!---------------------------------------------------------------------- +! +! these are ice points. set tref to tf_ice and update tmpsfc. +! + if (mask_tile == 2) then + nsst%tref(ij)=tf_ice ! water part tmp set + skint_tile(ij)=(1.0-sice_tile(ij))*nsst%tref(ij)+sice_tile(ij)*sicet_tile(ij) + nice = nice + 1 + cycle ij_loop + endif - IF (MASK_FG_TILE == 2 .AND. MASK_TILE == 0) THEN - CALL NSST_WATER_RESET(NSST,IJ,TFREEZ) - CYCLE IJ_LOOP - ENDIF +! +! Get i,j index on array of (idim,jdim) from known ij +! + JTILE = (IJ-1) / IDIM + 1 + ITILE = MOD(IJ,IDIM) + IF (ITILE==0) ITILE = IDIM !---------------------------------------------------------------------- -! SKIP LAND POINTS. NSST NOT APPLIED AT LAND. +! IF THE MODEL POINT WAS ICE COVERED, BUT IS NOW OPEN WATER, SET +! TREF TO searched adjascent open water onea, if failed the search, set to +! weighted average of tf_ice and tf_clm. For NSST vars, set xz TO '30' AND ALL OTHER FIELDS TO ZERO. !---------------------------------------------------------------------- - IF (MASK_TILE == 1) THEN - CYCLE IJ_LOOP + IF (MASK_FG_TILE == 2 .AND. MASK_TILE == 0) THEN +! +! set background for the thaw (just melted water) situation +! + call tf_thaw_set(nsst%tref,nint(slmsk_fg_tile),itile,jtile,tf_ice,tf_clm_tile(ij),tf_thaw,idim,jdim, & + nset_thaw_s,nset_thaw_i,nset_thaw_c) + call nsst_water_reset(nsst,ij,tf_thaw) + nset_thaw = nset_thaw + 1 ENDIF !---------------------------------------------------------------------- @@ -692,11 +750,6 @@ SUBROUTINE ADJUST_NSST(RLA,RLO,SLMSK_TILE,SLMSK_FG_TILE,SKINT_TILE,& ! IT IS SIMPLY A FILLER VALUE. THESE FIELDS ARE NOT USED AT ! OPEN WATER POINTS. !---------------------------------------------------------------------- - - JTILE = (IJ-1) / IDIM + 1 - ITILE = MOD(IJ,IDIM) - IF (ITILE==0) ITILE = IDIM - !---------------------------------------------------------------------- ! SEE IF ANY OF THE NEAREST GSI POINTS MASK AREA OPEN WATER. ! IF SO, APPLY NSST INCREMENT USING BILINEAR INTERPOLATION. @@ -735,11 +788,12 @@ SUBROUTINE ADJUST_NSST(RLA,RLO,SLMSK_TILE,SLMSK_FG_TILE,SKINT_TILE,& WSUM = WSUM + S2C(ITILE,JTILE,4) ENDIF + nintp = nintp + 1 DTREF = DTREF / WSUM TREF_SAVE = NSST%TREF(IJ) NSST%TREF(IJ) = NSST%TREF(IJ) + DTREF - NSST%TREF(IJ) = MAX(NSST%TREF(IJ), TFREEZ) + NSST%TREF(IJ) = MAX(NSST%TREF(IJ), tf_ice) NSST%TREF(IJ) = MIN(NSST%TREF(IJ), TMAX) NSST%TFINC(IJ) = NSST%TREF(IJ) - TREF_SAVE @@ -747,7 +801,7 @@ SUBROUTINE ADJUST_NSST(RLA,RLO,SLMSK_TILE,SLMSK_FG_TILE,SKINT_TILE,& NSST%Z_C(IJ),ZSEA1,ZSEA2,DTZM) SKINT_TILE(IJ) = NSST%TREF(IJ) + DTZM - SKINT_TILE(IJ) = MAX(SKINT_TILE(IJ), TFREEZ) + SKINT_TILE(IJ) = MAX(SKINT_TILE(IJ), tf_ice) SKINT_TILE(IJ) = MIN(SKINT_TILE(IJ), TMAX) SICET_TILE(IJ) = SKINT_TILE(IJ) @@ -796,12 +850,13 @@ SUBROUTINE ADJUST_NSST(RLA,RLO,SLMSK_TILE,SLMSK_FG_TILE,SKINT_TILE,& IF (SLMSK_GAUS(III,JJJ) == 0) THEN - PRINT*,'MISMATCH AT TILE POINT ',ITILE,JTILE - PRINT*,'UPDATE TREF USING GSI INCREMENT AT ',III,JJJ,DTREF_GAUS(III,JJJ) +! PRINT*,'MISMATCH AT TILE POINT ',ITILE,JTILE +! PRINT*,'UPDATE TREF USING GSI INCREMENT AT ',III,JJJ,DTREF_GAUS(III,JJJ) + nsearched = nsearched + 1 TREF_SAVE = NSST%TREF(IJ) NSST%TREF(IJ ) = NSST%TREF(IJ) + DTREF_GAUS(III,JJJ) - NSST%TREF(IJ) = MAX(NSST%TREF(IJ), TFREEZ) + NSST%TREF(IJ) = MAX(NSST%TREF(IJ), tf_ice) NSST%TREF(IJ) = MIN(NSST%TREF(IJ), TMAX) NSST%TFINC(IJ) = NSST%TREF(IJ) - TREF_SAVE @@ -809,7 +864,7 @@ SUBROUTINE ADJUST_NSST(RLA,RLO,SLMSK_TILE,SLMSK_FG_TILE,SKINT_TILE,& NSST%Z_C(IJ),ZSEA1,ZSEA2,DTZM) SKINT_TILE(IJ) = NSST%TREF(IJ) + DTZM - SKINT_TILE(IJ) = MAX(SKINT_TILE(IJ), TFREEZ) + SKINT_TILE(IJ) = MAX(SKINT_TILE(IJ), tf_ice) SKINT_TILE(IJ) = MIN(SKINT_TILE(IJ), TMAX) SICET_TILE(IJ) = SKINT_TILE(IJ) @@ -832,26 +887,28 @@ SUBROUTINE ADJUST_NSST(RLA,RLO,SLMSK_TILE,SLMSK_FG_TILE,SKINT_TILE,& ! ELSE UPDATE TREF BASED ON THE ANNUAL SST CYCLE. !---------------------------------------------------------------------- - PRINT*,'WARNING !!!!!! SEARCH FAILED AT TILE POINT ',ITILE,JTILE +! PRINT*,'WARNING !!!!!! SEARCH FAILED AT TILE POINT ',ITILE,JTILE + nfill = nfill + 1 IF (IS_ICE) THEN - NSST%TREF(IJ) = TFREEZ - PRINT*,"NEARBY ICE. SET TREF TO FREEZING" + NSST%TREF(IJ) = tf_ice +! PRINT*,"NEARBY ICE. SET TREF TO FREEZING" + nfill_tice = nfill_tice + 1 ELSE - CALL CLIMO_TREND(RLA(IJ),MON,DAY,DELTSFC,DTREF) TREF_SAVE = NSST%TREF(IJ) - NSST%TREF(IJ) = NSST%TREF(IJ) + DTREF - NSST%TREF(IJ) = MAX(NSST%TREF(IJ), TFREEZ) + NSST%TREF(IJ) = NSST%TREF(IJ) + TF_TRD_TILE(IJ) + NSST%TREF(IJ) = MAX(NSST%TREF(IJ), tf_ice) NSST%TREF(IJ) = MIN(NSST%TREF(IJ), TMAX) NSST%TFINC(IJ) = NSST%TREF(IJ) - TREF_SAVE - PRINT*,'UPDATE TREF FROM SST CLIMO ',DTREF +! PRINT*,'UPDATE TREF FROM SST CLIMO ',DTREF + nfill_clm = nfill_clm + 1 ENDIF CALL DTZM_POINT(NSST%XT(IJ),NSST%XZ(IJ),NSST%DT_COOL(IJ), & NSST%Z_C(IJ),ZSEA1,ZSEA2,DTZM) SKINT_TILE(IJ) = NSST%TREF(IJ) + DTZM - SKINT_TILE(IJ) = MAX(SKINT_TILE(IJ), TFREEZ) + SKINT_TILE(IJ) = MAX(SKINT_TILE(IJ), tf_ice) SKINT_TILE(IJ) = MIN(SKINT_TILE(IJ), TMAX) SICET_TILE(IJ) = SKINT_TILE(IJ) @@ -861,6 +918,14 @@ SUBROUTINE ADJUST_NSST(RLA,RLO,SLMSK_TILE,SLMSK_FG_TILE,SKINT_TILE,& ENDDO IJ_LOOP + write(*,'(a)') 'statistics of grids number processed for tile : ' + write(*,'(a,I8)') ' nintp = ',nintp + write(*,'(a,4I8)') 'nset_thaw,nset_thaw_s,nset_thaw_i,nset_thaw_c =',nset_thaw,nset_thaw_s,nset_thaw_i,nset_thaw_c + write(*,'(a,I8)') ' nsearched = ',nsearched + write(*,'(a,3I6)') ' nfill,nfill_tice,nfill_clm = ',nfill,nfill_tice,nfill_clm + write(*,'(a,I8)') ' nice = ',nice + write(*,'(a,I8)') ' nland = ',nland + DEALLOCATE(ID1, ID2, JDC, S2C) END SUBROUTINE ADJUST_NSST @@ -1095,44 +1160,6 @@ SUBROUTINE DTZM_POINT(XT,XZ,DT_COOL,ZC,Z1,Z2,DTZM) END SUBROUTINE DTZM_POINT - SUBROUTINE NSST_WATER_RESET(NSST,IJ,TFREEZ) - -!------------------------------------------------------------------- -! IF THE FIRST GUESS WAS SEA ICE, BUT THE ANALYSIS IS OPEN WATER, -! RESET ALL NSST VARIABLES. -!------------------------------------------------------------------- - - USE READ_WRITE_DATA, ONLY : NSST_DATA - - IMPLICIT NONE - - INTEGER, INTENT(IN) :: IJ - - REAL, INTENT(IN) :: TFREEZ - - TYPE(NSST_DATA), INTENT(INOUT) :: NSST - - NSST%C_0(IJ) = 0.0 - NSST%C_D(IJ) = 0.0 - NSST%D_CONV(IJ) = 0.0 - NSST%DT_COOL(IJ) = 0.0 - NSST%IFD(IJ) = 0.0 - NSST%QRAIN(IJ) = 0.0 - NSST%TREF(IJ) = TFREEZ - NSST%W_0(IJ) = 0.0 - NSST%W_D(IJ) = 0.0 - NSST%XS(IJ) = 0.0 - NSST%XT(IJ) = 0.0 - NSST%XTTS(IJ) = 0.0 - NSST%XU(IJ) = 0.0 - NSST%XV(IJ) = 0.0 - NSST%XZ(IJ) = 30.0 - NSST%XZTS(IJ) = 0.0 - NSST%Z_C(IJ) = 0.0 - NSST%ZM(IJ) = 0.0 - - END SUBROUTINE NSST_WATER_RESET - SUBROUTINE REMAP_COEF( is, ie, js, je,& im, jm, lon, lat, id1, id2, jdc, s2c, agrid ) @@ -1217,3 +1244,502 @@ SUBROUTINE REMAP_COEF( is, ie, js, je,& 5000 continue ! j-loop END SUBROUTINE REMAP_COEF + + subroutine tf_thaw_set(tf_ij,mask_ij,itile,jtile,tice,tclm,tf_thaw,nx,ny, & + nset_thaw_s,nset_thaw_i,nset_thaw_c) +! +! set a vakue to tf background for the thaw (just melted water) situation +! +!Input/output: +! tf : Foundation temperature background on FV3 native grids +! mask_ij : mask of the tile (FV3 native grids) +! itile,jtile : location index of the tile +! tice : water temperature (calulated with a salinity formula) +! tclm : SST climatology valid at the analysis time + + real, dimension(nx*ny), intent(in) :: tf_ij + integer, dimension(nx*ny), intent(in) :: mask_ij + real, intent(in) :: tice,tclm + integer, intent(in) :: itile,jtile,nx,ny + real, intent(out) :: tf_thaw + integer, intent(inout) :: nset_thaw_s,nset_thaw_i,nset_thaw_c +! Local + real, parameter :: bmiss = -999.0 + real, dimension(nx,ny) :: tf + integer, dimension(nx,ny) :: mask + integer :: krad,max_search,istart,iend,jstart,jend + integer :: ii,jj,iii,jjj + logical :: is_ice + + max_search = 2 + + mask(:,:) = reshape(mask_ij,(/nx,ny/) ) + tf(:,:) = reshape(tf_ij,(/nx,ny/) ) + + tf_thaw = bmiss + + do krad = 1, max_search + + istart = itile - krad + iend = itile + krad + jstart = jtile - krad + jend = jtile + krad + + do jj = jstart, jend + do ii = istart, iend + + + if ((jj == jstart) .or. (jj == jend) .or. & + (ii == istart) .or. (ii == iend)) then + + if ((jj >= 1) .and. (jj <= ny)) then + jjj = jj + if (ii <= 0) then + iii = nx + ii + else if (ii >= (nx+1)) then + iii = ii - nx + else + iii = ii + endif + +!---------------------------------------------------------------------- +! SEE IF NEARBY POINTS ARE SEA ICE. IF THEY ARE, AND THE SEARCH FOR +! A GAUSSIAN GRID OPEN WATER POINT FAILS, THEN TREF WILL BE SET TO +! FREEZING BELOW. +!---------------------------------------------------------------------- + if (krad <= 2 .and. mask(iii,jjj) == 2) is_ice = .true. + + if (mask(iii,jjj) == 0) then + tf_thaw = tf(iii,jjj) + nset_thaw_s = nset_thaw_s + 1 + write(*,'(a,I4,2F9.3)') 'nset_thaw_s,tf(iii,jjj),tclm : ',nset_thaw_s,tf(iii,jjj),tclm + go to 100 + endif ! tile mask is open water + + endif + endif + enddo + enddo + enddo ! krad loop + + 100 continue + + if ( tf_thaw == bmiss ) then + if (is_ice) then + tf_thaw = tice + nset_thaw_i = nset_thaw_i + 1 + write(*,'(a,I4,F9.3)') 'nset_thaw_i,tf_ice : ',nset_thaw_i,tice + else + tf_thaw = 0.8*tice+0.2*tclm + nset_thaw_c = nset_thaw_c + 1 + write(*,'(a,I4,2F9.3)') 'nset_thaw_c,tf_ice,tclm : ',nset_thaw_c,tice,tclm + endif + endif + + end subroutine tf_thaw_set + + subroutine nsst_water_reset(nsst,ij,tf_thaw) + +!------------------------------------------------------------------- +! if the first guess was sea ice, but the analysis is open water, +! reset all nsst variables. +!------------------------------------------------------------------- + + use read_write_data, only : nsst_data + implicit none + + integer, intent(in) :: ij + + real, intent(in) :: tf_thaw + + type(nsst_data), intent(inout) :: nsst + + nsst%c_0(ij) = 0.0 + nsst%c_d(ij) = 0.0 + nsst%d_conv(ij) = 0.0 + nsst%dt_cool(ij) = 0.0 + nsst%ifd(ij) = 0.0 + nsst%qrain(ij) = 0.0 + nsst%tref(ij) = tf_thaw + nsst%w_0(ij) = 0.0 + nsst%w_d(ij) = 0.0 + nsst%xs(ij) = 0.0 + nsst%xt(ij) = 0.0 + nsst%xtts(ij) = 0.0 + nsst%xu(ij) = 0.0 + nsst%xv(ij) = 0.0 + nsst%xz(ij) = 30.0 + nsst%xzts(ij) = 0.0 + nsst%z_c(ij) = 0.0 + nsst%zm(ij) = 0.0 + + end subroutine nsst_water_reset + +subroutine get_tf_clm(xlats_ij,xlons_ij,ny,nx,iy,im,id,ih,tf_clm,tf_trd) +! +! abstract: get sst climatology at the valid time (atime) and target resolution (nx,ny) +! +! input & output +! + use read_write_data, only : get_tf_clm_dim + + implicit none + + real, dimension(nx*ny), intent(in) :: xlats_ij ! latitudes of target grids (nx*ny) + real, dimension(nx*ny), intent(in) :: xlons_ij ! latitudes of target grids (nx*ny) + real, dimension(nx,ny), intent(out) :: tf_clm ! sst climatology valid at atime (nx,ny) + real, dimension(nx,ny), intent(out) :: tf_trd ! 6-hourly sst climatology tendency valid at atime (nx,ny) + integer, intent(in) :: iy,im,id,ih,nx,ny +! local declare + real, allocatable, dimension(:,:) :: tf_clm0 ! sst climatology at the valid time (nxc,nyc) + real, allocatable, dimension(:,:) :: tf_trd0 ! 6-hourly sst climatology tendency valid at atime (nxc,nyc) + real, allocatable, dimension(:) :: cxlats ! latitudes of sst climatology + real, allocatable, dimension(:) :: cxlons ! longitudes of sst climatology + + real, dimension(nx*ny) :: tf_clm_ij ! sst climatology at target grids (nx*ny) + real, dimension(nx*ny) :: tf_trd_ij ! 6-hourly sst climatology tendency + real :: wei1,wei2 + integer :: nxc,nyc,mon1,mon2,i,j + character (len=6), parameter :: fin_tf_clm='sstclm' ! sst climatology file name +! +! get which two months used and their weights from atime +! + call get_tim_wei(iy,im,id,ih,mon1,mon2,wei1,wei2) +! +! get the dimensions of the sst climatology & allocate the related arrays +! + call get_tf_clm_dim(fin_tf_clm,nyc,nxc) + allocate( tf_clm0(nxc,nyc),tf_trd0(nxc,nyc),cxlats(nyc),cxlons(nxc) ) +! +! get tf_clm at the analysis time from monthly climatology & cxlats, cxlons +! + call get_tf_clm_ta(tf_clm0,tf_trd0,cxlats,cxlons,nyc,nxc,mon1,mon2,wei1,wei2) +! +! get tf_clm (nx by ny lat/lon) valid at atime +! + if ( nx == nxc .and. ny == nyc ) then + tf_clm(:,:) = tf_clm0(:,:) + tf_trd(:,:) = tf_trd0(:,:) +! write(*,'(a,2f9.3)') 'same dimensions, tf_clm, min : ',minval(tf_clm),maxval(tf_clm) + else +! write(*,'(a,4i8)') 'different dimensions,nx,ny,nxc,nyc : ',nx,ny,nxc,nyc + call intp_tile(tf_clm0, cxlats, cxlons, nyc, nxc, & + tf_clm_ij,xlats_ij,xlons_ij,ny, nx) + call intp_tile(tf_trd0, cxlats, cxlons, nyc, nxc, & + tf_trd_ij,xlats_ij,xlons_ij,ny, nx) +! write(*,'(a,2f9.3)') 'tf_clm0, min, max : ',minval(tf_clm0),maxval(tf_clm0) + + tf_clm(:,:) = reshape (tf_clm_ij, (/nx,ny/) ) + tf_trd(:,:) = reshape (tf_trd_ij, (/nx,ny/) ) + endif + +end subroutine get_tf_clm + +subroutine get_tf_clm_ta(tf_clm_ta,tf_clm_trend,xlats,xlons,nlat,nlon,mon1,mon2,wei1,wei2) +!$$$ +! abstract: get tf/sst climatology & the trend at analysis time +! created by xu li, march, 2019 + + use read_write_data, only : read_tf_clim_grb + implicit none + +! input + integer, intent(in) :: nlat,nlon,mon1,mon2 + real, intent(in) :: wei1,wei2 +! output + real, dimension(nlon,nlat), intent(out) :: tf_clm_ta,tf_clm_trend + real, dimension(nlat), intent(out) :: xlats + real, dimension(nlon), intent(out) :: xlons + +!input/output data file names + character (len=6), parameter :: fin_tf_clm='sstclm' + +! local declare + real, dimension(nlon,nlat) :: tf_clm1,tf_clm2 + +! +! read in rtg sst climatology without bitmap (surface mask) for mon1 and mon2 +! + call read_tf_clim_grb(trim(fin_tf_clm),tf_clm1,xlats,xlons,nlat,nlon,mon1) + call read_tf_clim_grb(trim(fin_tf_clm),tf_clm2,xlats,xlons,nlat,nlon,mon2) +! +! tf_clim at the analysis time +! + tf_clm_ta(:,:) = wei1*tf_clm1(:,:)+wei2*tf_clm2(:,:) +! +! tf_clim trend at the analysis time (month to month) +! + tf_clm_trend(:,:) = (tf_clm2(:,:)-tf_clm1(:,:))/120.0 + + write(*,'(a,2f9.3)') 'tf_clm_ta, min, max : ',minval(tf_clm_ta),maxval(tf_clm_ta) + write(*,'(a,2f9.3)') 'tf_clm_trend, min, max : ',minval(tf_clm_trend),maxval(tf_clm_trend) + end subroutine get_tf_clm_ta + +subroutine get_sal_clm(xlats_ij,xlons_ij,ny,nx,iy,im,id,ih,sal_clm) +! +! abstract: get salinity climatology at the valid time (atime) and target resolution (nx,ny) +! +! input & output +! + use read_write_data, only : get_dim_nc + implicit none + + real, dimension(nx*ny), intent(in) :: xlats_ij ! latitudes of target grids (nx*ny) + real, dimension(nx*ny), intent(in) :: xlons_ij ! latitudes of target grids (nx*ny) + real, dimension(nx,ny), intent(out) :: sal_clm ! salinity climatology valid at atime (nx,ny) + integer, intent(in) :: iy,im,id,ih,nx,ny +! local declare + real, allocatable, dimension(:,:) :: sal_clm0 ! salinity climatology at the valid time + real, allocatable, dimension(:) :: cxlats ! latitudes of sst climatology + real, allocatable, dimension(:) :: cxlons ! longitudes of sst climatology + + real, dimension(nx*ny) :: sal_clm_ij ! salinity climatology at target grids (nx*ny) + real :: wei1,wei2 + integer :: nxc,nyc,mon1,mon2,i,j + character (len=6), parameter :: fin_sal_clm='salclm' ! salinity climatology file name +! +! get which two months used and their weights from atime +! + call get_tim_wei(iy,im,id,ih,mon1,mon2,wei1,wei2) +! +! get the dimensions of the sst climatology & allocate the related arrays +! + call get_dim_nc(fin_sal_clm,nyc,nxc) + allocate( sal_clm0(nxc,nyc),cxlats(nyc),cxlons(nxc) ) +! +! get sal_clm at the analysis time from monthly climatology & cxlats, cxlons +! + call get_sal_clm_ta(sal_clm0,cxlats,cxlons,nyc,nxc,mon1,mon2,wei1,wei2) +! +! get sal_clm (nx by ny lat/lon) valid at atime +! + if ( nx == nxc .and. ny == nyc ) then + sal_clm(:,:) = sal_clm0(:,:) +! write(*,'(a,2f9.3)') 'same dimensions, sal_clm, min : ',minval(sal_clm),maxval(sal_clm) + else +! write(*,'(a,4i8)') 'different dimensions,nx,ny,nxc,nyc : ',nx,ny,nxc,nyc + call intp_tile(sal_clm0, cxlats, cxlons, nyc, nxc, & + sal_clm_ij,xlats_ij,xlons_ij,ny, nx) +! write(*,'(a,2f9.3)') 'sal_clm0, min, max : ',minval(sal_clm0),maxval(sal_clm0) +! write(*,'(a,2f9.3)') 'done with intp_tile for sal_clm, min, max : ',minval(sal_clm_ij),maxval(sal_clm_ij) + + sal_clm(:,:) = reshape (sal_clm_ij, (/nx,ny/) ) + endif + +end subroutine get_sal_clm + +subroutine get_sal_clm_ta(sal_clm_ta,xlats,xlons,nlat,nlon,mon1,mon2,wei1,wei2) +!$$$ +! abstract: get sal climatology at analysis time +! created by xu li, march, 2019 + + use read_write_data, only : read_salclm_gfs_nc + implicit none + +! input + integer, intent(in) :: nlat,nlon,mon1,mon2 + real, intent(in) :: wei1,wei2 +! output + real, dimension(nlon,nlat), intent(out) :: sal_clm_ta + real, dimension(nlat), intent(out) :: xlats + real, dimension(nlon), intent(out) :: xlons + +!input/output data file names + character (len=6), parameter :: fin_sal_clm='salclm' + +! local declare + real, dimension(nlon,nlat) :: sal_clm1,sal_clm2 + +! +! read in rtg sst climatology without bitmap (surface mask) for mon1 and mon2 +! + call read_salclm_gfs_nc(trim(fin_sal_clm),sal_clm1,xlats,xlons,nlat,nlon,mon1) + call read_salclm_gfs_nc(trim(fin_sal_clm),sal_clm2,xlats,xlons,nlat,nlon,mon2) +! +! sal_clim at the analysis time +! + sal_clm_ta(:,:) = wei1*sal_clm1(:,:)+wei2*sal_clm2(:,:) + write(*,'(a,2f9.3)') 'sal_clm_ta, min, max : ',minval(sal_clm_ta),maxval(sal_clm_ta) + end subroutine get_sal_clm_ta + +subroutine intp_tile(tf_lalo,dlats_lalo,dlons_lalo,jdim_lalo,idim_lalo, & + tf_tile,xlats_tile,xlons_tile,jdim_tile,idim_tile) +!-------------------------------------------------------------------------------- +! abstract: interpolate lon/lat grid to fv3 native grid (tf_lalo => tf_tile) for +! not dependent on mask +!-------------------------------------------------------------------------------- + +! input +! +! tf_lalo : (idim_lalo,idim_lalo) tf at lat/lon regular grid +! dlats_lalo : (jdim_lalo) latitudes along y direction of lat/lon regular grid points +! dlons_lalo : (idim_lalo) longitudes along x direction of lat/lon regular grid points +! jdim_lalo : number of y dimension of tf_lalo +! idim_lalo : number of x dimension of tf_lalo +! xlats_tile : (jdim_tile*idim_tile) latitudes of all tile grid points +! xlons_tile : (jdim_tile*idim_tile) longitudes of all tile grid points +! jdim_tile : number of y dimension of tf_tile +! idim_tile : number of x dimension of tf_tile +! +! output +! +! tf_tile : (jdim_tile*idim_tile) tf at cubed sphere grid + + implicit none + +! input/output + real, dimension(idim_lalo,jdim_lalo), intent(in) :: tf_lalo + real, dimension(jdim_lalo), intent(in) :: dlats_lalo + real, dimension(idim_lalo), intent(in) :: dlons_lalo + real, dimension(jdim_tile*idim_tile), intent(in) :: xlats_tile + real, dimension(jdim_tile*idim_tile), intent(in) :: xlons_tile + integer, intent(in) :: jdim_lalo,idim_lalo,jdim_tile,idim_tile + real, dimension(jdim_tile*idim_tile), intent(out) :: tf_tile + +! local + real, parameter :: deg2rad=3.1415926/180.0 + real, dimension(jdim_lalo) :: xlats_lalo + real, dimension(idim_lalo) :: xlons_lalo + real :: tf,wsum,res_km + integer :: itile,jtile + integer :: ii,jj,ij,iii,jjj + integer :: ilalo,jlalo,ilalop1,jlalop1 + integer :: istart,iend,jstart,jend,krad + + integer, allocatable, dimension(:,:) :: id1,id2,jdc + real, allocatable, dimension(:,:,:) :: agrid,s2c + + print* + print*,'interpolate from lat/lon grids to any one grid with known lat/lon' + + xlats_lalo = dlats_lalo*deg2rad + xlons_lalo = dlons_lalo*deg2rad + + allocate(agrid(idim_tile,jdim_tile,2)) + agrid(:,:,1) = reshape (xlons_tile, (/idim_tile,jdim_tile/) ) + agrid(:,:,2) = reshape (xlats_tile, (/idim_tile,jdim_tile/) ) + agrid = agrid*deg2rad + + allocate(id1(idim_tile,jdim_tile)) + allocate(id2(idim_tile,jdim_tile)) + allocate(jdc(idim_tile,jdim_tile)) + allocate(s2c(idim_tile,jdim_tile,4)) + +!---------------------------------------------------------------------- +! compute bilinear weights for each model point from the nearest +! four lalo points. does not account for mask. that +! happens later. +!---------------------------------------------------------------------- + + call remap_coef( 1, idim_tile, 1, jdim_tile, idim_lalo, jdim_lalo, & + xlons_lalo, xlats_lalo, id1, id2, jdc, s2c, agrid ) + + do ij = 1, jdim_tile*idim_tile + + jtile = (ij-1)/idim_tile + 1 + itile = mod(ij,idim_tile) + if (itile==0) itile = idim_tile + + ilalo = id1(itile,jtile) + ilalop1 = id2(itile,jtile) + jlalo = jdc(itile,jtile) + jlalop1 = jdc(itile,jtile) + 1 + + wsum = s2c(itile,jtile,1) + s2c(itile,jtile,2) + & + s2c(itile,jtile,3) + s2c(itile,jtile,4) + + tf_tile(ij) = ( s2c(itile,jtile,1)*tf_lalo(ilalo,jlalo) + & + s2c(itile,jtile,2)*tf_lalo(ilalop1,jlalo) + & + s2c(itile,jtile,3)*tf_lalo(ilalop1,jlalop1) + & + s2c(itile,jtile,4)*tf_lalo(ilalo,jlalop1) )/wsum + enddo + + deallocate(id1, id2, jdc, s2c) + +end subroutine intp_tile + +subroutine get_tim_wei(iy,im,id,ih,mon1,mon2,wei1,wei2) +!$$$ +! abstract: get two months and weights for monthly climatology +! created by xu li, march, 2019 + + implicit none + +! input + integer, intent(in) :: iy,im,id,ih +! output + integer, intent(out) :: mon1,mon2 + real, intent(out) :: wei1,wei2 + +! local declare + real :: rjday + integer :: mon,monend,monm,monp,jdow,jdoy,jday + integer :: jda(8) +! +!dayhf : julian day of the middle of each month +! + real, dimension(13) :: dayhf + data dayhf/15.5,45.0,74.5,105.0,135.5,166.0,196.5,227.5,258.0,288.5,319.0,349.5,380.5/ + +! 15, 44, 73.5, 104, 134.5, 165, 195.5, 226.5, 257, 287.5, 318.5, 349 ! from +! woa05 + + jda=0 + jda(1)=iy + jda(2)=im + jda(3)=id + jda(5)=ih + jdow = 0 + jdoy = 0 + jday = 0 + call w3doxdat(jda,jdow,jdoy,jday) + rjday=jdoy+jda(5)/24. + if(rjday.lt.dayhf(1)) rjday=rjday+365. + + monend = 12 + do mon = 1, monend + monm = mon + monp = mon + 1 + if( rjday >= dayhf(monm) .and. rjday < dayhf(monp) ) then + mon1 = monm + mon2 = monp + go to 10 + endif + enddo + + print *,'wrong rjday',rjday + call abort + 10 continue + + wei1 = (dayhf(mon2)-rjday)/(dayhf(mon2)-dayhf(mon1)) + wei2 = (rjday-dayhf(mon1))/(dayhf(mon2)-dayhf(mon1)) + + if( mon2 == 13 ) mon2=1 + + write(*,'(a,2i4,3f9.3)') 'mon1,mon2,rjday,wei1,wei2=',mon1,mon2,rjday,wei1,wei2 + + end subroutine get_tim_wei + + real function tfreez(salinity) +!Constants taken from Gill, 1982. +!Author: Robert Grumbine +!LAST MODIFIED: 21 September 1994. + + implicit none + + REAL salinity,sal + REAL a1, a2, a3 + PARAMETER (a1 = -0.0575) + PARAMETER (a2 = 1.710523E-3) + PARAMETER (a3 = -2.154996E-4) + + IF (salinity .LT. 0.) THEN + sal = 0. + ELSE + sal = salinity + ENDIF + tfreez = sal*(a1+a2*SQRT(sal)+a3*sal) + + return + end + diff --git a/sorc/global_cycle.fd/read_write_data.f90 b/sorc/global_cycle.fd/read_write_data.f90 index 02a2c7b34..4bbac032b 100644 --- a/sorc/global_cycle.fd/read_write_data.f90 +++ b/sorc/global_cycle.fd/read_write_data.f90 @@ -38,6 +38,8 @@ MODULE READ_WRITE_DATA PUBLIC :: READ_GSI_DATA PUBLIC :: READ_LAT_LON_OROG PUBLIC :: WRITE_DATA + public :: read_tf_clim_grb,get_tf_clm_dim + public :: read_salclm_gfs_nc,get_dim_nc CONTAINS @@ -1499,4 +1501,358 @@ SUBROUTINE READ_DATA(TSFFCS,SMCFCS,SNOFCS,STCFCS, & END SUBROUTINE READ_DATA +subroutine read_tf_clim_grb(file_sst,sst,rlats_sst,rlons_sst,mlat_sst,mlon_sst,mon) + +! . . . . +! abstrac: read_tf_clim_grb : read grib1 sst analysis +! prgmmr: xu li org: np23 date: 2019-03-13 +! +! abstract: read sst analysis (grib format) and save it as expanded and transposed array +! +! subroutine rdgrbsst must be compiled with the ncep w3 library +! and the bacio library. +! +! +! program history log: +! +! input argument list: +! file_sst - file name of grib sst file +! mon - month number +! mlat_sst,mlon_sst +! output: +! rlats_sst +! rlons_sst +! sst +! +! argument list defined by this reading: +! sst - sst field +! note: (1) the data is stored from north to south originally in grib format, +! but is stored from south to north with this reading routine +! nlat_sst - latitudinal dimension of sst +! nlon_sst - longitudinal dimension of sst +! xsst0 - latitude of origin +! ysst0 - longitude of origin +! dres - lat/lon increment +! +! call subs: getgbh, getgb +! +! attributes: +! language: f90 +! +!$$$ + implicit none + + include "mpif.h" + +! declare passed variables and arrays + character(*) , intent(in ) :: file_sst + integer , intent(in ) :: mon,mlat_sst,mlon_sst + real, dimension(mlat_sst) , intent( out) :: rlats_sst + real, dimension(mlon_sst) , intent( out) :: rlons_sst + real, dimension(mlon_sst,mlat_sst) , intent( out) :: sst + +! declare local parameters + integer,parameter:: lu_sst = 21 ! fortran unit number of grib sst file + real, parameter :: deg2rad = 3.141593/180.0 + +! declare local variables and arrays + logical(1), allocatable, dimension(:) :: lb + + integer :: nlat_sst,nlon_sst + integer :: iret,ni,nj + integer :: mscan,kb1 + integer :: jincdir,i,iincdir,kb2,kb3,kf,kg,k,j,jf + integer, dimension(22):: jgds,kgds + integer, dimension(25):: jpds,kpds + + real :: xsst0,ysst0,dres + real, allocatable, dimension(:) :: f + +!************+****************************************************************************** +! +! open sst analysis file (grib) + write(*,*) ' sstclm : ',file_sst + call baopenr(lu_sst,trim(file_sst),iret) + if (iret /= 0 ) then + write(6,*)'read_tf_clm_grb: ***error*** opening sst file' + CALL MPI_ABORT(MPI_COMM_WORLD, 111) + endif + +! define sst variables for read + j=-1 + jpds=-1 + jgds=-1 + jpds(5)=11 ! sst variable + jpds(6)=1 ! surface + jpds(9) = mon + call getgbh(lu_sst,0,j,jpds,jgds,kg,kf,k,kpds,kgds,iret) + + nlat_sst = kgds(3) ! number points on longitude circle (360) + nlon_sst = kgds(2) ! number points on latitude circle (720) + +! write(*,*) 'nlat_sst, nlon_sst, mon : ',nlat_sst, nlon_sst, mon +! write(*,*) 'mlat_sst, mlon_sst : ',mlat_sst, mlon_sst + +! allocate arrays + allocate(lb(nlat_sst*nlon_sst)) + allocate(f(nlat_sst*nlon_sst)) + jf=nlat_sst*nlon_sst + +! read in the analysis + call getgb(lu_sst,0,jf,j,jpds,jgds,kf,k,kpds,kgds,lb,f,iret) + if (iret /= 0) then + write(6,*)'read_tf_clm_grb: ***error*** reading sst analysis data record' + deallocate(lb,f) + CALL MPI_ABORT(MPI_COMM_WORLD, 111) + endif + + if ( (nlat_sst /= mlat_sst) .or. (nlon_sst /= mlon_sst) ) then + write(6,*)'read_rtg_org: inconsistent dimensions. mlat_sst,mlon_sst=',& + mlat_sst,mlon_sst,' -versus- nlat_sst,nlon_sst=',nlat_sst,nlon_sst + deallocate(lb,f) + CALL MPI_ABORT(MPI_COMM_WORLD, 111) + endif + +! +! get xlats and xlons +! + dres = 180.0/real(nlat_sst) + ysst0 = 0.5*dres-90.0 + xsst0 = 0.5*dres + +! get lat_sst & lon_sst + do j = 1, nlat_sst + rlats_sst(j) = ysst0 + real(j-1)*dres + enddo + + do i = 1, nlon_sst + rlons_sst(i) = (xsst0 + real(i-1)*dres) + enddo + +! load dimensions and grid specs. check for unusual values + ni=kgds(2) ! 720 for 0.5 x 0.5 + nj=kgds(3) ! 360 for 0.5 x 0.5 resolution + + mscan=kgds(11) + kb1=ibits(mscan,7,1) ! i scan direction + kb2=ibits(mscan,6,1) ! j scan direction + kb3=ibits(mscan,5,1) ! (i,j) or (j,i) + +! get i and j scanning directions from kb1 and kb2. +! 0 yields +1, 1 yields -1. +1 is west to east, -1 is east to west. + iincdir = 1-kb1*2 + +! 0 yields -1, 1 yields +1. +1 is south to north, -1 is north to south. + jincdir = kb2*2 - 1 + +! write(*,*) 'read_tf_clim_grb,iincdir,jincdir : ',iincdir,jincdir + do k=1,kf + +! kb3 from scan mode indicates if i points are consecutive +! or if j points are consecutive + if(kb3==0)then ! (i,j) + i=(ni+1)*kb1+(mod(k-1,ni)+1)*iincdir + j=(nj+1)*(1-kb2)+jincdir*((k-1)/ni+1) + else ! (j,i) + j=(nj+1)*(1-kb2)+(mod(k-1,nj)+1)*jincdir + i=(ni+1)*kb1+iincdir*((k-1)/nj+1) + endif + sst (i,j) = f(k) + end do + + deallocate(lb,f) + + call baclose(lu_sst,iret) + if (iret /= 0 ) then + write(6,*)'read_tf_clm_grb: ***error*** close sst file' + CALL MPI_ABORT(MPI_COMM_WORLD, 121) + endif + +end subroutine read_tf_clim_grb + +subroutine get_tf_clm_dim(file_sst,mlat_sst,mlon_sst) +! . . . . +! abstract: get_tf_clm_dim : get dimension of rtg sst climatology +! prgmmr: xu li org: np23 date: 2019-03-13 +! +! +! program history log: +! +! input argument list: +! file_sst - file name of grib sst file +! output +! mlat_sst,mlon_sst +! +! call subs: getgbh +! +! attributes: +! language: f90 +! machine: ibm rs/6000 sp +! +!$$$ + implicit none + + include "mpif.h" + +! declare passed variables and arrays + character(*) , intent(in ) :: file_sst + integer , intent(out) :: mlat_sst,mlon_sst + +! declare local parameters + integer,parameter:: lu_sst = 21 ! fortran unit number of grib sst file + + integer :: iret + integer :: mscan,kb1 + integer :: kf,kg,k,j + integer, dimension(22):: jgds,kgds + integer, dimension(25):: jpds,kpds + +!************+****************************************************************************** +! +! open sst analysis file (grib) + call baopenr(lu_sst,trim(file_sst),iret) + if (iret /= 0 ) then + write(6,*)'get_tf_clm_dim: ***error*** opening sst file' + CALL MPI_ABORT(MPI_COMM_WORLD, 111) + endif + +! define sst variables for read + j=-1 + jpds=-1 + jgds=-1 + jpds(5)=11 ! sst variable + jpds(6)=1 ! surface + jpds(9) = 1 + call getgbh(lu_sst,0,j,jpds,jgds,kg,kf,k,kpds,kgds,iret) + + mlat_sst = kgds(3) ! number points on longitude circle (360) + mlon_sst = kgds(2) ! number points on latitude circle (720) + + write(*,*) 'mlat_sst, mlon_sst : ',mlat_sst, mlon_sst + + call baclose(lu_sst,iret) + if (iret /= 0 ) then + write(6,*)'get_tf_clm_dim: ***error*** close sst file' + CALL MPI_ABORT(MPI_COMM_WORLD, 121) + endif +end subroutine get_tf_clm_dim + +subroutine read_salclm_gfs_nc(filename,sal,xlats,xlons,nlat,nlon,itime) +! abstract: read woa05 salinity monthly climatology (netcdf) + use netcdf + implicit none + + ! This is the name of the data file we will read. + character (len=*), intent(in) :: filename + integer, intent(in) :: nlat,nlon + integer, intent(in) :: itime + real, dimension(nlat), intent(out) :: xlats + real, dimension(nlon), intent(out) :: xlons + real, dimension(nlon,nlat), intent(out) :: sal +! Local variables + integer :: ncid,ntime + + integer, parameter :: ndims = 3 + character (len = *), parameter :: lat_name = "latitude" + character (len = *), parameter :: lon_name = "longitude" + character (len = *), parameter :: t_name = "time" + character (len = *), parameter :: sal_name="sal" + integer :: no_fill,fill_value + integer :: time_varid,lon_varid, lat_varid, z_varid, sal_varid + + ! The start and count arrays will tell the netCDF library where to read our data. + integer, dimension(ndims) :: start, count + + character (len = *), parameter :: units = "units" + character (len = *), parameter :: sal_units = "psu" + ! PSU (Practical SalinitUnit). 1 PSU = 1g/kg + character (len = *), parameter :: time_units = "months" + character (len = *), parameter :: lat_units = "degrees_north" + character (len = *), parameter :: lon_units = "degrees_east" + + integer :: missv +! Loop indices + integer :: i,j + +! Open the file. + call nc_check( nf90_open(filename, nf90_nowrite, ncid) ) + +! Get the varids of time, latitude, longitude & depth coordinate variables. + call nc_check( nf90_inq_varid(ncid, t_name, time_varid) ) + call nc_check( nf90_inq_varid(ncid, lat_name, lat_varid) ) + call nc_check( nf90_inq_varid(ncid, lon_name, lon_varid) ) + +! Read the time, latitude and longitude data. +! call nc_check( nf90_get_var(ncid, time_varid, ntime) ) + call nc_check( nf90_get_var(ncid, lat_varid, xlats) ) + call nc_check( nf90_get_var(ncid, lon_varid, xlons) ) + +! Get the varids of the sal netCDF variables. + call nc_check( nf90_inq_varid(ncid, sal_name,sal_varid) ) + +! Read 1 record of nlat*nlon values, starting at the beginning +! of the record (the (1, 1, 1, rec) element in the netCDF file). + start = (/ 1, 1, itime /) + count = (/ nlon, nlat, 1 /) + +! write(*,*) 'read_salclm_gfs_nc itime : ',itime +! Read the sal data from the file, one record at a time. + call nc_check( nf90_get_var(ncid, sal_varid, sal, start, count) ) + +! Close the file. This frees up any internal netCDF resources +! associated with the file. + call nc_check( nf90_close(ncid) ) + +! If we got this far, everything worked as expected. Yipee! +! print *,"*** SUCCESS reading file ", filename, "!" + +end subroutine read_salclm_gfs_nc + +subroutine get_dim_nc(filename,nlat,nlon) +! abstract: get dimensions of sal array + use netcdf + implicit none + + character (len=*), intent(in) :: filename + integer, intent(out) :: nlat,nlon +! Local variables + character (len = *), parameter :: lat_name = "latitude" + character (len = *), parameter :: lon_name = "longitude" + integer :: ncid + integer :: LatDimID,LonDimID + +! Open the file. + call nc_check( nf90_open(filename, nf90_nowrite, ncid) ) + +! Get dimensions + call nc_check( nf90_inq_dimid(ncid,lat_name,LatDimID) ) + call nc_check( nf90_inq_dimid(ncid,lon_name,LonDimID) ) + call nc_check( nf90_inquire_dimension(ncid,LatDimID,len=nlat) ) + call nc_check( nf90_inquire_dimension(ncid,LonDimID,len=nlon) ) + +! write(*,'(a,1x,a6,2I8)') 'get_dim_nc, file, nlat, nlon : ',filename,nlat,nlon + +! Close the file. This frees up any internal netCDF resources +! associated with the file. + call nc_check( nf90_close(ncid) ) + +! If we got this far, everything worked as expected. Yipee! +! print *,"*** SUCCESS get dimensions from nc file ", filename, "!" + +end subroutine get_dim_nc + +subroutine nc_check(status) + + use netcdf + + include "mpif.h" + + integer, intent ( in) :: status + + if(status /= nf90_noerr) then + print *, trim(nf90_strerror(status)) + CALL MPI_ABORT(MPI_COMM_WORLD, 122) + end if +end subroutine nc_check + END MODULE READ_WRITE_DATA diff --git a/sorc/global_cycle.fd/sfcsub.F b/sorc/global_cycle.fd/sfcsub.F index b14271cd0..4fbabab8f 100644 --- a/sorc/global_cycle.fd/sfcsub.F +++ b/sorc/global_cycle.fd/sfcsub.F @@ -2697,12 +2697,13 @@ subroutine fixrdg(lugb,idim,jdim,fngrib, real (kind=kind_io8), allocatable :: data8(:) real (kind=kind_io4), allocatable :: data4(:) ! - logical*1 lbms(mdata) + logical*1, allocatable :: lbms(:) ! integer kpds(200),kgds(200) integer jpds(200),jgds(200), kpds0(200) ! allocate(data8(1:idim*jdim)) + allocate(lbms(1:mdata)) kpds = 0 kgds = 0 jpds = 0 @@ -2792,6 +2793,7 @@ subroutine fixrdg(lugb,idim,jdim,fngrib, endif ! deallocate(data8) + deallocate(lbms) return end subroutine getarea(kgds,dlat,dlon,rslat,rnlat,wlon,elon,ijordr @@ -6078,17 +6080,24 @@ subroutine setrmsk(kpds5,slmask,igaul,jgaul,wlon,rnlat, ! ijmax = imax*jmax rslmsk = 0. +! TG3 MODS BEGIN + if(kpds5 == kpdtsf .and. imax == 138 .and. jmax == 116 + & .and. kpds4 == 128) then +! print*,'turn off setrmsk for tg3' + lmask = .false. + + elseif(kpds5 == kpdtsf) then +! TG3 MODS END ! ! surface temperature ! - if(kpds5.eq.kpdtsf) then -! lmask=.false. + lmask = .false. call ga2la(slmask,igaul,jgaul,rslmsk,imax,jmax,wlon,rnlat &, rlnout, rltout, gaus, blno, blto) ! &, dlon, dlat, gaus, blno, blto) - crit=0.5 + crit = 0.5 call rof01(rslmsk,ijmax,'ge',crit) - lmask=.true. + lmask = .true. ! ! bucket soil wetness ! @@ -6096,16 +6105,16 @@ subroutine setrmsk(kpds5,slmask,igaul,jgaul,wlon,rnlat, call ga2la(slmask,igaul,jgaul,rslmsk,imax,jmax,wlon,rnlat &, rlnout, rltout, gaus, blno, blto) ! &, dlon, dlat, gaus, blno, blto) - crit=0.5 + crit = 0.5 call rof01(rslmsk,ijmax,'ge',crit) - lmask=.true. + lmask = .true. ! write(6,*) 'wet rslmsk' ! znnt=1. ! call nntprt(rslmsk,ijmax,znnt) ! ! snow depth ! - elseif(kpds5.eq.kpdsnd) then + elseif(kpds5 == kpdsnd) then if(kpds4 == 192) then ! use the bitmap rslmsk = 0. do j = 1, jmax @@ -6962,51 +6971,51 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, ! ! get tsf climatology for the begining of the forecast ! - if (fh .gt. 0.0) then + if (fh > 0.0) then !cbosu if (me == 0) print*,'bosu fh gt 0' - iy4=iy - if(iy.lt.101) iy4=1900+iy4 - fha=0 - ida=0 - jda=0 -! fha(2)=nint(fh) - ida(1)=iy - ida(2)=im - ida(3)=id - ida(5)=ih + iy4 = iy + if (iy < 101) iy4 = 1900 + iy4 + fha = 0 + ida = 0 + jda = 0 +! fha(2) = nint(fh) + ida(1) = iy + ida(2) = im + ida(3) = id + ida(5) = ih call w3kind(w3kindreal,w3kindint) if(w3kindreal == 4) then - fha4=fha + fha4 = fha call w3movdat(fha4,ida,jda) else call w3movdat(fha,ida,jda) endif - jy=jda(1) - jm=jda(2) - jd=jda(3) - jh=jda(5) - if (me .eq. 0) write(6,*) ' forecast jy,jm,jd,jh', - & jy,jm,jd,jh + jy = jda(1) + jm = jda(2) + jd = jda(3) + jh = jda(5) + if (me == 0) write(6,*) ' forecast jy,jm,jd,jh', + & jy,jm,jd,jh jdow = 0 jdoy = 0 jday = 0 call w3doxdat(jda,jdow,jdoy,jday) - rjday=jdoy+jda(5)/24. - if(rjday.lt.dayhf(1)) rjday=rjday+365. + rjday = jdoy + jda(5) / 24. + if(rjday < dayhf(1)) rjday = rjday + 365. ! - if (me .eq. 0) write(6,*) 'forecast jy,jm,jd,jh=',jy,jm,jd,jh + if (me == 0) write(6,*) 'forecast jy,jm,jd,jh=',jy,jm,jd,jh ! ! for monthly mean climatology ! monend = 12 do mm=1,monend - mmm=mm - mmp=mm+1 - if(rjday.ge.dayhf(mmm).and.rjday.lt.dayhf(mmp)) then - mon1=mmm - mon2=mmp + mmm = mm + mmp = mm + 1 + if(rjday >= dayhf(mmm) .and. rjday < dayhf(mmp)) then + mon1 = mmm + mon2 = mmp go to 10 endif enddo @@ -7014,17 +7023,18 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, call abort 10 continue wei1m = (dayhf(mon2)-rjday)/(dayhf(mon2)-dayhf(mon1)) - wei2m = (rjday-dayhf(mon1))/(dayhf(mon2)-dayhf(mon1)) - if(mon2.eq.13) mon2=1 - if (me .eq. 0) print *,'rjday,mon1,mon2,wei1m,wei2m=', - & rjday,mon1,mon2,wei1m,wei2m + wei2m = 1.0 - wei1m +! wei2m = (rjday-dayhf(mon1))/(dayhf(mon2)-dayhf(mon1)) + if (mon2 == 13) mon2 = 1 + if (me == 0) print *,'rjday,mon1,mon2,wei1m,wei2m=', + & rjday,mon1,mon2,wei1m,wei2m ! ! read monthly mean climatology of tsf ! kpd7 = -1 do nn=1,2 mon = mon1 - if (nn .eq. 2) mon = mon2 + if (nn == 2) mon = mon2 call fixrdc(lugb,fntsfc,kpdtsf,kpd7,mon,slmask, & tsf(1,nn),len,iret &, imsk, jmsk, slmskh, gaus,blno, blto @@ -7041,8 +7051,8 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, ! ! compute current jy,jm,jd,jh of forecast and the day of the year ! - iy4=iy - if(iy.lt.101) iy4=1900+iy4 + iy4 = iy + if (iy < 101) iy4=1900+iy4 fha = 0 ida = 0 jda = 0 @@ -7052,8 +7062,8 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, ida(3) = id ida(5) = ih call w3kind(w3kindreal,w3kindint) - if(w3kindreal==4) then - fha4=fha + if(w3kindreal == 4) then + fha4 = fha call w3movdat(fha4,ida,jda) else call w3movdat(fha,ida,jda) @@ -7068,44 +7078,45 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, jdoy = 0 jday = 0 call w3doxdat(jda,jdow,jdoy,jday) - rjday = jdoy+jda(5)/24. - if(rjday.lt.dayhf(1)) rjday=rjday+365. + rjday = jdoy + jda(5) / 24. + if(rjday < dayhf(1)) rjday = rjday + 365. - if (me .eq. 0) write(6,*) ' forecast jy,jm,jd,jh,rjday=', - & jy,jm,jd,jh,rjday + if (me == 0) write(6,*) ' forecast jy,jm,jd,jh,rjday=', + & jy,jm,jd,jh,rjday ! - if (me .eq. 0) write(6,*) 'forecast jy,jm,jd,jh=',jy,jm,jd,jh + if (me == 0) write(6,*) 'forecast jy,jm,jd,jh=',jy,jm,jd,jh ! ! for monthly mean climatology ! monend = 12 do mm=1,monend - mmm=mm - mmp=mm+1 - if(rjday.ge.dayhf(mmm).and.rjday.lt.dayhf(mmp)) then - mon1=mmm - mon2=mmp + mmm = mm + mmp = mm + 1 + if(rjday >= dayhf(mmm) .and. rjday < dayhf(mmp)) then + mon1 = mmm + mon2 = mmp go to 20 endif enddo print *,'wrong rjday',rjday call abort 20 continue - wei1m=(dayhf(mon2)-rjday)/(dayhf(mon2)-dayhf(mon1)) - wei2m=(rjday-dayhf(mon1))/(dayhf(mon2)-dayhf(mon1)) - if(mon2.eq.13) mon2=1 - if (me .eq. 0) print *,'rjday,mon1,mon2,wei1m,wei2m=', - & rjday,mon1,mon2,wei1m,wei2m + wei1m = (dayhf(mon2)-rjday)/(dayhf(mon2)-dayhf(mon1)) + wei2m = 1.0 - wei1m +! wei2m = (rjday-dayhf(mon1))/(dayhf(mon2)-dayhf(mon1)) + if (mon2 == 13) mon2 = 1 + if (me == 0) print *,'rjday,mon1,mon2,wei1m,wei2m=', + & rjday,mon1,mon2,wei1m,wei2m ! ! for seasonal mean climatology ! monend = 4 is = im/3 + 1 - if (is.eq.5) is = 1 + if (is == 5) is = 1 do mm=1,monend mmm = mm*3 - 2 mmp = (mm+1)*3 - 2 - if(rjday.ge.dayhf(mmm).and.rjday.lt.dayhf(mmp)) then + if(rjday >= dayhf(mmm) .and. rjday < dayhf(mmp)) then sea1 = mmm sea2 = mmp go to 30 @@ -7115,20 +7126,21 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, call abort 30 continue wei1s = (dayhf(sea2)-rjday)/(dayhf(sea2)-dayhf(sea1)) - wei2s = (rjday-dayhf(sea1))/(dayhf(sea2)-dayhf(sea1)) - if(sea2.eq.13) sea2=1 - if (me .eq. 0) print *,'rjday,sea1,sea2,wei1s,wei2s=', - & rjday,sea1,sea2,wei1s,wei2s + wei2s = 1.0 - wei1s +! wei2s = (rjday-dayhf(sea1))/(dayhf(sea2)-dayhf(sea1)) + if (sea2 == 13) sea2 = 1 + if (me == 0) print *,'rjday,sea1,sea2,wei1s,wei2s=', + & rjday,sea1,sea2,wei1s,wei2s ! ! for summer and winter values (maximum and minimum). ! monend = 2 is = im/6 + 1 - if (is.eq.3) is = 1 + if (is == 3) is = 1 do mm=1,monend mmm = mm*6 - 5 mmp = (mm+1)*6 - 5 - if(rjday.ge.dayhf(mmm).and.rjday.lt.dayhf(mmp)) then + if(rjday >= dayhf(mmm) .and. rjday < dayhf(mmp)) then hyr1 = mmm hyr2 = mmp go to 31 @@ -7138,10 +7150,11 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, call abort 31 continue wei1y = (dayhf(hyr2)-rjday)/(dayhf(hyr2)-dayhf(hyr1)) - wei2y = (rjday-dayhf(hyr1))/(dayhf(hyr2)-dayhf(hyr1)) - if(hyr2.eq.13) hyr2=1 - if (me .eq. 0) print *,'rjday,hyr1,hyr2,wei1y,wei2y=', - & rjday,hyr1,hyr2,wei1y,wei2y + wei2y = 1.0 - wei1y +! wei2y = (rjday-dayhf(hyr1))/(dayhf(hyr2)-dayhf(hyr1)) + if (hyr2 == 13) hyr2 = 1 + if (me == 0) print *,'rjday,hyr1,hyr2,wei1y,wei2y=', + & rjday,hyr1,hyr2,wei1y,wei2y ! ! start reading in climatology and interpolate to the date ! @@ -7541,7 +7554,7 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, ! mon1s = mon1 ; mon2s = mon2 ; sea1s = sea1 ; sea2s = sea2 ! - if (me .eq. 0) print *,' mon1s=',mon1s,' mon2s=',mon2s + if (me == 0) print *,' mon1s=',mon1s,' mon2s=',mon2s &,' sea1s=',sea1s,' sea2s=',sea2s ! k1 = 1 ; k2 = 2 @@ -7599,11 +7612,11 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, ! seasonal mean climatology ! isx = sea2/3 + 1 - if (isx .eq. 5) isx = 1 - if(isx.eq.1) kpd9 = 12 - if(isx.eq.2) kpd9 = 3 - if(isx.eq.3) kpd9 = 6 - if(isx.eq.4) kpd9 = 9 + if (isx == 5) isx = 1 + if (isx == 1) kpd9 = 12 + if (isx == 2) kpd9 = 3 + if (isx == 3) kpd9 = 6 + if (isx == 4) kpd9 = 9 ! ! albedo ! there are four albedo fields in this version: @@ -7639,7 +7652,7 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, if (me == 0) print*,'bosu 2nd time in clima for month ', & mon, k1,k2 if ( index(fnalbc, "tileX.nc") == 0) then ! grib file - kpd7=-1 + kpd7 = -1 do k = 1, 4 call fixrdc(lugb,fnalbc,kpdalb(k),kpd7,mon,slmask, & alb(1,k,nn),len,iret @@ -7656,7 +7669,7 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, ! ! tsf at the current time t ! - kpd7=-1 + kpd7 = -1 call fixrdc(lugb,fntsfc,kpdtsf,kpd7,mon,slmask, & tsf(1,nn),len,iret &, imsk, jmsk, slmskh, gaus,blno, blto @@ -7664,13 +7677,13 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, ! ! soil wetness ! - if(fnwetc(1:8).ne.' ') then + if (fnwetc(1:8).ne.' ') then kpd7=-1 call fixrdc(lugb,fnwetc,kpdwet,kpd7,mon,slmask, & wet(1,nn),len,iret &, imsk, jmsk, slmskh, gaus,blno, blto &, outlat, outlon, me) - elseif(fnsmcc(1:8).ne.' ') then + elseif (fnsmcc(1:8).ne.' ') then if (index(fnsmcc,'global_soilmcpc.1x1.grb') /= 0) then ! the old climo data kpd7=-1 call fixrdc(lugb,fnsmcc,kpdsmc,kpd7,mon,slmask, @@ -7712,13 +7725,13 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, ! ! sea ice ! - kpd7=-1 - if(fnacnc(1:8).ne.' ') then + kpd7 = -1 + if (fnacnc(1:8).ne.' ') then call fixrdc(lugb,fnacnc,kpdacn,kpd7,mon,slmask, & acn(1,nn),len,iret &, imsk, jmsk, slmskh, gaus,blno, blto &, outlat, outlon, me) - elseif(fnaisc(1:8).ne.' ') then + elseif (fnaisc(1:8).ne.' ') then call fixrdc(lugb,fnaisc,kpdais,kpd7,mon,slmask, & ais(1,nn),len,iret &, imsk, jmsk, slmskh, gaus,blno, blto @@ -7738,7 +7751,7 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, ! ! snow cover ! - if(fnscvc(1:8).ne.' ') then + if (fnscvc(1:8).ne.' ') then kpd7=-1 call fixrdc(lugb,fnscvc,kpdscv,kpd7,mon,slmask, & scv(1,nn),len,iret @@ -7749,7 +7762,7 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, ! ! surface roughness ! - if(fnzorc(1:3) == 'sib') then + if (fnzorc(1:3) == 'sib') then if (me == 0) then write(6,*) 'roughness length to be set from sib veg type' endif @@ -7767,7 +7780,7 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, ! ! vegetation cover ! - if(fnvegc(1:8).ne.' ') then + if (fnvegc(1:8) .ne. ' ') then if ( index(fnvegc, "tileX.nc") == 0) then ! grib file kpd7=-1 call fixrdc(lugb,fnvegc,kpdveg,kpd7,mon,slmask, @@ -7789,35 +7802,35 @@ subroutine clima(lugb,iy,im,id,ih,fh,len,lsoil, ! when chosen, set the z0 based on the vegetation type. ! for this option to work, namelist variable fnvetc must be ! set to point at the proper vegetation type file. - if(fnzorc(1:3) == 'sib') then - if(fnvetc(1:4) == ' ') then + if (fnzorc(1:3) == 'sib') then + if (fnvetc(1:4) == ' ') then if (me==0) write(6,*) "must choose sib veg type climo file" call abort endif zorclm = 0.0 do i=1,len - ivtyp=nint(vet(i)) + ivtyp = nint(vet(i)) if (ivtyp >= 1 .and. ivtyp <= 13) then zorclm(i) = z0_sib(ivtyp) endif enddo elseif(fnzorc(1:4) == 'igbp') then - if(fnvetc(1:4) == ' ') then - if (me==0) write(6,*) "must choose igbp veg type climo file" + if (fnvetc(1:4) == ' ') then + if (me == 0) write(6,*) "must choose igbp veg type climo file" call abort endif zorclm = 0.0 do i=1,len - ivtyp=nint(vet(i)) + ivtyp = nint(vet(i)) if (ivtyp >= 1 .and. ivtyp <= 20) then z0_season(1) = z0_igbp_min(ivtyp) z0_season(7) = z0_igbp_max(ivtyp) - if(outlat(i) < 0.0)then + if (outlat(i) < 0.0) then zorclm(i) = wei1y * z0_season(hyr2) + - & wei2y *z0_season(hyr1) + & wei2y * z0_season(hyr1) else zorclm(i) = wei1y * z0_season(hyr1) + - & wei2y *z0_season(hyr2) + & wei2y * z0_season(hyr2) endif endif enddo @@ -8148,7 +8161,7 @@ subroutine fixrdc(lugb,fngrib,kpds5,kpds7,mon,slmask, real (kind=kind_io8), allocatable :: rlngrb(:), rltgrb(:) ! logical lmask, yr2kc, gaus, ijordr - logical*1 lbms(mdata) + logical*1, allocatable :: lbms(:) ! integer, intent(in) :: kpds7 integer kpds(1000),kgds(1000) @@ -8156,6 +8169,7 @@ subroutine fixrdc(lugb,fngrib,kpds5,kpds7,mon,slmask, real (kind=kind_io8) outlat(len), outlon(len) ! allocate(data8(1:mdata)) + allocate(lbms(mdata)) ! ! integer imax_sv, jmax_sv, wlon_sv, rnlat_sv, kpds1_sv ! date imax_sv/0/, jmax_sv/0/, wlon_sv/999.0/, rnlat_sv/999.0/ @@ -8291,6 +8305,7 @@ subroutine fixrdc(lugb,fngrib,kpds5,kpds7,mon,slmask, call baclose(lugb,iret) ! deallocate(data8) + deallocate(lbms) return end subroutine fixrdc subroutine fixrda(lugb,fngrib,kpds5,slmask, diff --git a/ush/global_cycle.sh b/ush/global_cycle.sh index a0cc22943..477253681 100755 --- a/ush/global_cycle.sh +++ b/ush/global_cycle.sh @@ -59,6 +59,8 @@ # Defaults to ${FIXam}/global_maxice.2x2.grb # FNTSFC Input SST climatology GRIB file. # Defaults to ${FIXam}/RTGSST.1982.2012.monthly.clim.grb +# FNSALC Input Salinity climatology netcdf file. +# Defaults to ${FIXam}/global_salclm.t1534.3072.1536.nc # FNSNOC Input snow climatology GRIB file. # Defaults to ${FIXam}/global_snoclim.1.875.grb # FNZORC Input roughness climatology. @@ -178,6 +180,7 @@ # fixed data : $FNGLAC # $FNMXIC # $FNTSFC +# $FNSALC # $FNSNOC # $FNZORC # $FNALBC @@ -272,6 +275,7 @@ MAX_TASKS_CY=${MAX_TASKS_CY:-99999} FNGLAC=${FNGLAC:-${FIXam}/global_glacier.2x2.grb} FNMXIC=${FNMXIC:-${FIXam}/global_maxice.2x2.grb} FNTSFC=${FNTSFC:-${FIXam}/RTGSST.1982.2012.monthly.clim.grb} +FNSALC=${FNSALC:-${FIXam}/global_salclm.t1534.3072.1536.nc} FNSNOC=${FNSNOC:-${FIXam}/global_snoclim.1.875.grb} FNZORC=${FNZORC:-igbp} FNALBC2=${FNALBC2:-${FIXam}/global_albedo4.1x1.grb} @@ -315,6 +319,9 @@ fi cd $DATA||exit 99 [[ -d $COMOUT ]]||mkdir -p $COMOUT +ln -fs $FNTSFC sstclm +ln -fs $FNSALC salclm + # If the appropriate resolution fix file is not present, use the highest resolution available (T1534) [[ ! -f $FNALBC ]] && FNALBC="$FIXam/global_snowfree_albedo.bosu.t1534.3072.1536.rg.grb" [[ ! -f $FNVETC ]] && FNVETC="$FIXam/global_vegtype.igbp.t1534.3072.1536.rg.grb" From 4643e2c7e1c393e24a45a0965c367ef31a84a8fe Mon Sep 17 00:00:00 2001 From: GeorgeGayno-NOAA <52789452+GeorgeGayno-NOAA@users.noreply.github.com> Date: Tue, 15 Oct 2019 09:57:55 -0400 Subject: [PATCH 30/34] develop: This commit references #23. Update sfc_climo_gen and chgres_cube programs: For regrid options other than conservative and neighbor, change pole method from NONE to ALLAVG. This prevents interpolation problems near the poles. --- sorc/chgres_cube.fd/atmosphere.F90 | 6 +++--- sorc/chgres_cube.fd/surface.F90 | 2 +- sorc/sfc_climo_gen.fd/interp.F90 | 9 ++++++++- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/sorc/chgres_cube.fd/atmosphere.F90 b/sorc/chgres_cube.fd/atmosphere.F90 index 067362c04..0e22b551e 100644 --- a/sorc/chgres_cube.fd/atmosphere.F90 +++ b/sorc/chgres_cube.fd/atmosphere.F90 @@ -182,7 +182,7 @@ subroutine atmosphere_driver(localpet) call ESMF_FieldRegridStore(temp_input_grid, & temp_b4adj_target_grid, & - polemethod=ESMF_POLEMETHOD_NONE, & + polemethod=ESMF_POLEMETHOD_ALLAVG, & srctermprocessing=isrctermprocessing, & extrapmethod=ESMF_EXTRAPMETHOD_NEAREST_STOD, & routehandle=regrid_bl, & @@ -333,7 +333,7 @@ subroutine atmosphere_driver(localpet) print*,"- CALL FieldRegridStore FOR 3D-WIND WEST EDGE." call ESMF_FieldRegridStore(wind_target_grid, & wind_w_target_grid, & - polemethod=ESMF_POLEMETHOD_NONE, & + polemethod=ESMF_POLEMETHOD_ALLAVG, & srctermprocessing=isrctermprocessing, & extrapMethod=ESMF_EXTRAPMETHOD_NEAREST_STOD, & routehandle=regrid_bl, & @@ -360,7 +360,7 @@ subroutine atmosphere_driver(localpet) print*,"- CALL FieldRegridStore FOR 3D-WIND SOUTH EDGE." call ESMF_FieldRegridStore(wind_target_grid, & wind_s_target_grid, & - polemethod=ESMF_POLEMETHOD_NONE, & + polemethod=ESMF_POLEMETHOD_ALLAVG, & srctermprocessing=isrctermprocessing, & extrapMethod=ESMF_EXTRAPMETHOD_NEAREST_STOD, & routehandle=regrid_bl, & diff --git a/sorc/chgres_cube.fd/surface.F90 b/sorc/chgres_cube.fd/surface.F90 index bd9e7c8af..4b6ea37da 100644 --- a/sorc/chgres_cube.fd/surface.F90 +++ b/sorc/chgres_cube.fd/surface.F90 @@ -376,7 +376,7 @@ subroutine interp(localpet) print*,"- CALL FieldRegridStore FOR NON-MASKED BILINEAR INTERPOLATION." call ESMF_FieldRegridStore(t2m_input_grid, & t2m_target_grid, & - polemethod=ESMF_POLEMETHOD_NONE, & + polemethod=ESMF_POLEMETHOD_ALLAVG, & srctermprocessing=isrctermprocessing, & routehandle=regrid_bl_no_mask, & regridmethod=method, rc=rc) diff --git a/sorc/sfc_climo_gen.fd/interp.F90 b/sorc/sfc_climo_gen.fd/interp.F90 index 880f2e7d4..a61450e80 100644 --- a/sorc/sfc_climo_gen.fd/interp.F90 +++ b/sorc/sfc_climo_gen.fd/interp.F90 @@ -50,6 +50,7 @@ subroutine interp(localpet, method, input_file) type(esmf_regridmethod_flag),intent(in) :: method type(esmf_field) :: data_field_src type(esmf_routehandle) :: regrid_data + type(esmf_polemethod_flag) :: pole print*,"- CALL FieldCreate FOR SOURCE GRID DATA." data_field_src = ESMF_FieldCreate(grid_src, & @@ -121,13 +122,19 @@ subroutine interp(localpet, method, input_file) if (record == 1) then + if (method == ESMF_REGRIDMETHOD_BILINEAR) then + pole = ESMF_POLEMETHOD_ALLAVG + else + pole = ESMF_POLEMETHOD_NONE + endif + print*,"- CALL FieldRegridStore." nullify(unmapped_ptr) call ESMF_FieldRegridStore(data_field_src, & data_field_mdl, & srcmaskvalues=(/0/), & dstmaskvalues=(/0/), & - polemethod=ESMF_POLEMETHOD_NONE, & + polemethod=pole, & unmappedaction=ESMF_UNMAPPEDACTION_IGNORE, & normtype=ESMF_NORMTYPE_FRACAREA, & srctermprocessing=isrctermprocessing, & From b2ab11872de088ec93cd6215aa26e86c68456245 Mon Sep 17 00:00:00 2001 From: GeorgeGayno-NOAA <52789452+GeorgeGayno-NOAA@users.noreply.github.com> Date: Thu, 24 Oct 2019 15:18:02 -0400 Subject: [PATCH 31/34] release/v1.1.0: This commit references #26. (#28) develop: This commit references issue #26. Merge minor change to fv3gfs_chgres.sh script from 'release/v1.1.0'. --- ush/fv3gfs_chgres.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/fv3gfs_chgres.sh b/ush/fv3gfs_chgres.sh index a7f02dcc7..8e22ba648 100755 --- a/ush/fv3gfs_chgres.sh +++ b/ush/fv3gfs_chgres.sh @@ -13,7 +13,7 @@ # 'machine' variable is uncommented. Ensure job cards for # the other machines are prefixed by "##". # - Check the job card account. On Hera, this is -# "#SBATCH -q $account". On WCOSS, this is "#BSUB -P $account". +# "#SBATCH -A $account". On WCOSS, this is "#BSUB -P $account". # You may not have permissions to run under the default account. # - Make sure you have linked the 'fixed' directories using # sorc/link_fixdirs.sh. From 2104affa7384905f0215658c36d63bf260fa9cf8 Mon Sep 17 00:00:00 2001 From: GeorgeGayno-NOAA <52789452+GeorgeGayno-NOAA@users.noreply.github.com> Date: Mon, 28 Oct 2019 12:44:41 -0400 Subject: [PATCH 32/34] chgres_cube snow fix (#30) develop: This commit references #27. Update chgres_cube - Initialize all surface variables to a missing flag after the FieldCreate step. In routine qc_check explicitly set snow to zero at open water and ice depth to zero at non-ice points. --- reg_tests/chgres_cube/driver.jet.sh | 4 +- sorc/chgres_cube.fd/surface.F90 | 243 +++++++++++++++++++++++++--- 2 files changed, 221 insertions(+), 26 deletions(-) diff --git a/reg_tests/chgres_cube/driver.jet.sh b/reg_tests/chgres_cube/driver.jet.sh index 6f76db74c..b256b64c3 100755 --- a/reg_tests/chgres_cube/driver.jet.sh +++ b/reg_tests/chgres_cube/driver.jet.sh @@ -34,8 +34,8 @@ module load netcdf/4.2.1.1 module list export OUTDIR=/mnt/lfs3/projects/emcda/$LOGNAME/stmp/chgres_reg_tests -PROJECT_CODE="emcda" -QUEUE="windfall" +PROJECT_CODE="hfv3gfs" +QUEUE="debug" #----------------------------------------------------------------------------- # Should not have to change anything below here. HOMEufs is the root diff --git a/sorc/chgres_cube.fd/surface.F90 b/sorc/chgres_cube.fd/surface.F90 index 4b6ea37da..d68f704a9 100644 --- a/sorc/chgres_cube.fd/surface.F90 +++ b/sorc/chgres_cube.fd/surface.F90 @@ -2723,6 +2723,7 @@ subroutine qc_check real(esmf_kind_r8), pointer :: seaice_skint_ptr(:,:) real(esmf_kind_r8), pointer :: skint_ptr(:,:) real(esmf_kind_r8), pointer :: fice_ptr(:,:) + real(esmf_kind_r8), pointer :: hice_ptr(:,:) print*,"- CALL FieldGet FOR TARGET GRID LAND-SEA MASK." call ESMF_FieldGet(landmask_target_grid, & @@ -2732,11 +2733,7 @@ subroutine qc_check if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & call error_handler("IN FieldGet", rc) -!--------------------------------------------------------------------------------------------- -! Set slope type flag value at non-land points. -!--------------------------------------------------------------------------------------------- - - print*,"- CALL FieldGet FOR TARGET GRID SLOPE TYPE." + print*,"- SET NON-LAND FLAG FOR TARGET GRID SLOPE TYPE." call ESMF_FieldGet(slope_type_target_grid, & farrayPtr=data_ptr, rc=rc) if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & @@ -2748,7 +2745,7 @@ subroutine qc_check enddo enddo - print*,"- CALL FieldGet FOR TARGET GRID SOIL TYPE." + print*,"- SET NON-LAND FLAG FOR TARGET GRID SOIL TYPE." call ESMF_FieldGet(soil_type_target_grid, & farrayPtr=data_ptr, rc=rc) if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & @@ -2760,7 +2757,7 @@ subroutine qc_check enddo enddo - print*,"- CALL FieldGet FOR TARGET GRID VEGETATION TYPE." + print*,"- SET NON-LAND FLAG FOR TARGET GRID VEGETATION TYPE." call ESMF_FieldGet(veg_type_target_grid, & farrayPtr=veg_type_ptr, rc=rc) if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & @@ -2772,7 +2769,7 @@ subroutine qc_check enddo enddo - print*,"- CALL FieldGet FOR TARGET GRID ALVSF." + print*,"- SET TARGET GRID ALVSF AT NON-LAND." call ESMF_FieldGet(alvsf_target_grid, & farrayPtr=data_ptr, rc=rc) if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & @@ -2784,7 +2781,7 @@ subroutine qc_check enddo enddo - print*,"- CALL FieldGet FOR TARGET GRID ALVWF." + print*,"- SET TARGET GRID ALVWF AT NON-LAND." call ESMF_FieldGet(alvwf_target_grid, & farrayPtr=data_ptr, rc=rc) if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & @@ -2796,7 +2793,7 @@ subroutine qc_check enddo enddo - print*,"- CALL FieldGet FOR TARGET GRID ALNSF." + print*,"- SET TARGET GRID ALNSF AT NON-LAND." call ESMF_FieldGet(alnsf_target_grid, & farrayPtr=data_ptr, rc=rc) if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & @@ -2808,7 +2805,7 @@ subroutine qc_check enddo enddo - print*,"- CALL FieldGet FOR TARGET GRID ALNWF." + print*,"- SET TARGET GRID ALNWF AT NON-LAND." call ESMF_FieldGet(alnwf_target_grid, & farrayPtr=data_ptr, rc=rc) if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & @@ -2820,7 +2817,7 @@ subroutine qc_check enddo enddo - print*,"- CALL FieldGet FOR TARGET GRID FACSF." + print*,"- SET NON-LAND FLAG FOR TARGET GRID FACSF." call ESMF_FieldGet(facsf_target_grid, & farrayPtr=data_ptr, rc=rc) if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & @@ -2832,7 +2829,7 @@ subroutine qc_check enddo enddo - print*,"- CALL FieldGet FOR TARGET GRID FAWSF." + print*,"- SET NON-LAND FLAG FOR TARGET GRID FACSF." call ESMF_FieldGet(facwf_target_grid, & farrayPtr=data_ptr, rc=rc) if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & @@ -2844,7 +2841,7 @@ subroutine qc_check enddo enddo - print*,"- CALL FieldGet FOR TARGET GRID MAXIMUM GREENNESS." + print*,"- SET NON-LAND FLAG FOR TARGET GRID MAXIMUM GREENNESS." call ESMF_FieldGet(max_veg_greenness_target_grid, & farrayPtr=data_ptr, rc=rc) if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & @@ -2856,7 +2853,7 @@ subroutine qc_check enddo enddo - print*,"- CALL FieldGet FOR TARGET GRID MINIMUM GREENNESS." + print*,"- SET NON-LAND FLAG FOR TARGET GRID MINIMUM GREENNESS." call ESMF_FieldGet(min_veg_greenness_target_grid, & farrayPtr=data_ptr, rc=rc) if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & @@ -2868,7 +2865,7 @@ subroutine qc_check enddo enddo - print*,"- CALL FieldGet FOR TARGET GRID VEGETATION GREENNESS." + print*,"- SET NON-LAND FLAG FOR TARGET GRID VEGETATION GREENNESS." call ESMF_FieldGet(veg_greenness_target_grid, & farrayPtr=veg_greenness_ptr, rc=rc) if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & @@ -2880,7 +2877,7 @@ subroutine qc_check enddo enddo - print*,"- CALL FieldGet FOR TARGET GRID MAX SNOW ALBEDO." + print*,"- SET NON-LAND FLAG FOR TARGET GRID MAX SNOW ALBEDO." call ESMF_FieldGet(mxsno_albedo_target_grid, & farrayPtr=data_ptr, rc=rc) if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & @@ -2892,7 +2889,7 @@ subroutine qc_check enddo enddo - print*,"- CALL FieldGet FOR TARGET GRID CANOPY MOISTURE CONTENT." + print*,"- ZERO OUT TARGET GRID CANOPY MOISTURE CONTENT WHERE NO PLANTS." call ESMF_FieldGet(canopy_mc_target_grid, & farrayPtr=data_ptr, rc=rc) if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & @@ -2910,7 +2907,7 @@ subroutine qc_check if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & call error_handler("IN FieldGet", rc) - print*,"- CALL FieldGet FOR TARGET GRID SKIN TEMP." + print*,"- SET TARGET GRID SKIN TEMP AT ICE POINTS." call ESMF_FieldGet(skin_temp_target_grid, & farrayPtr=skint_ptr, rc=rc) if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & @@ -2922,6 +2919,12 @@ subroutine qc_check if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & call error_handler("IN FieldGet", rc) + print*,"- SET TARGET GRID SEA ICE DEPTH TO ZERO AT NON-ICE POINTS." + call ESMF_FieldGet(seaice_depth_target_grid, & + farrayPtr=hice_ptr, rc=rc) + if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & + call error_handler("IN FieldGet", rc) + do j = clb(2), cub(2) do i = clb(1), cub(1) if (fice_ptr(i,j) > 0.0) then @@ -2929,11 +2932,12 @@ subroutine qc_check ( (1.0 - fice_ptr(i,j)) * frz_ice ) else seaice_skint_ptr(i,j) = skint_ptr(i,j) + hice_ptr(i,j) = 0.0 endif enddo enddo - print*,"- CALL FieldGet FOR TARGET GRID SUBSTRATE TEMP." + print*,"- SET TARGET GRID SUBSTRATE TEMP AT ICE." call ESMF_FieldGet(substrate_temp_target_grid, & farrayPtr=data_ptr, rc=rc) if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & @@ -2949,13 +2953,41 @@ subroutine qc_check enddo enddo - print*,"- CALL FieldGet FOR TARGET GRID TOTAL SOIL MOISTURE." + print*,"- ZERO OUT TARGET GRID SNOW DEPTH AT OPEN WATER." + call ESMF_FieldGet(snow_depth_target_grid, & + farrayPtr=data_ptr, rc=rc) + if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & + call error_handler("IN FieldGet", rc) + + do j = clb(2), cub(2) + do i = clb(1), cub(1) + if (landmask_ptr(i,j) == 0) then ! open water + data_ptr(i,j) = 0.0 + end if + enddo + enddo + + print*,"- ZERO OUT TARGET GRID SNOW LIQ AT OPEN WATER." + call ESMF_FieldGet(snow_liq_equiv_target_grid, & + farrayPtr=data_ptr, rc=rc) + if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & + call error_handler("IN FieldGet", rc) + + do j = clb(2), cub(2) + do i = clb(1), cub(1) + if (landmask_ptr(i,j) == 0) then ! open water + data_ptr(i,j) = 0.0 + endif + enddo + enddo + + print*,"- SET NON-LAND FLAG VALUE FOR TARGET GRID TOTAL SOIL MOISTURE." call ESMF_FieldGet(soilm_tot_target_grid, & farrayPtr=soilmt_ptr, rc=rc) if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & call error_handler("IN FieldGet", rc) - print*,"- CALL FieldGet FOR TARGET GRID LIQUID SOIL MOISTURE." + print*,"- SET NON-LAND FLAG VALUE FOR TARGET GRID LIQUID SOIL MOISTURE." call ESMF_FieldGet(soilm_liq_target_grid, & farrayPtr=soilml_ptr, rc=rc) if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & @@ -2971,7 +3003,7 @@ subroutine qc_check enddo enddo - print*,"- CALL FieldGet FOR TARGET GRID SOIL TEMPERATURE." + print*,"- SET OPEN WATER FLAG FOR TARGET GRID SOIL TEMPERATURE." call ESMF_FieldGet(soil_temp_target_grid, & farrayPtr=data3d_ptr, rc=rc) if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & @@ -3206,7 +3238,10 @@ subroutine create_surface_esmf_fields implicit none - integer :: rc + integer :: rc + + real(esmf_kind_r8), pointer :: target_ptr(:,:), target_ptr_3d(:,:,:) + real :: init_val = -999.9 print*,"- CALL FieldCreate FOR TARGET GRID T2M." t2m_target_grid = ESMF_FieldCreate(target_grid, & @@ -3215,6 +3250,14 @@ subroutine create_surface_esmf_fields if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & call error_handler("IN FieldCreate", rc) + print*,"- INITIALIZE TARGET grid t2m." + call ESMF_FieldGet(t2m_target_grid, & + farrayPtr=target_ptr, rc=rc) + if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & + call error_handler("IN FieldGet", rc) + + target_ptr = init_val + print*,"- CALL FieldCreate FOR TARGET GRID Q2M." q2m_target_grid = ESMF_FieldCreate(target_grid, & typekind=ESMF_TYPEKIND_R8, & @@ -3222,6 +3265,14 @@ subroutine create_surface_esmf_fields if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & call error_handler("IN FieldCreate", rc) + print*,"- INITIALIZE TARGET grid q2m." + call ESMF_FieldGet(q2m_target_grid, & + farrayPtr=target_ptr, rc=rc) + if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & + call error_handler("IN FieldGet", rc) + + target_ptr = init_val + print*,"- CALL FieldCreate FOR TARGET GRID TPRCP." tprcp_target_grid = ESMF_FieldCreate(target_grid, & typekind=ESMF_TYPEKIND_R8, & @@ -3229,6 +3280,14 @@ subroutine create_surface_esmf_fields if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & call error_handler("IN FieldCreate", rc) + print*,"- INITIALIZE TARGET grid tprcp." + call ESMF_FieldGet(tprcp_target_grid, & + farrayPtr=target_ptr, rc=rc) + if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & + call error_handler("IN FieldGet", rc) + + target_ptr = init_val + print*,"- CALL FieldCreate FOR TARGET GRID F10M." f10m_target_grid = ESMF_FieldCreate(target_grid, & typekind=ESMF_TYPEKIND_R8, & @@ -3236,6 +3295,14 @@ subroutine create_surface_esmf_fields if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & call error_handler("IN FieldCreate", rc) + print*,"- INITIALIZE TARGET grid f10m." + call ESMF_FieldGet(f10m_target_grid, & + farrayPtr=target_ptr, rc=rc) + if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & + call error_handler("IN FieldGet", rc) + + target_ptr = init_val + print*,"- CALL FieldCreate FOR TARGET GRID FFMM." ffmm_target_grid = ESMF_FieldCreate(target_grid, & typekind=ESMF_TYPEKIND_R8, & @@ -3243,6 +3310,14 @@ subroutine create_surface_esmf_fields if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & call error_handler("IN FieldCreate", rc) + print*,"- INITIALIZE TARGET grid ffmm." + call ESMF_FieldGet(ffmm_target_grid, & + farrayPtr=target_ptr, rc=rc) + if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & + call error_handler("IN FieldGet", rc) + + target_ptr = init_val + print*,"- CALL FieldCreate FOR TARGET GRID USTAR." ustar_target_grid = ESMF_FieldCreate(target_grid, & typekind=ESMF_TYPEKIND_R8, & @@ -3250,6 +3325,14 @@ subroutine create_surface_esmf_fields if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & call error_handler("IN FieldCreate", rc) + print*,"- INITIALIZE TARGET grid ustar." + call ESMF_FieldGet(ustar_target_grid, & + farrayPtr=target_ptr, rc=rc) + if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & + call error_handler("IN FieldGet", rc) + + target_ptr = init_val + print*,"- CALL FieldCreate FOR TARGET GRID SNOW LIQ EQUIV." snow_liq_equiv_target_grid = ESMF_FieldCreate(target_grid, & typekind=ESMF_TYPEKIND_R8, & @@ -3257,6 +3340,14 @@ subroutine create_surface_esmf_fields if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & call error_handler("IN FieldCreate", rc) + print*,"- INITIALIZE TARGET grid snow liq equiv." + call ESMF_FieldGet(snow_liq_equiv_target_grid, & + farrayPtr=target_ptr, rc=rc) + if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & + call error_handler("IN FieldGet", rc) + + target_ptr = init_val + print*,"- CALL FieldCreate FOR TARGET GRID SNOW DEPTH." snow_depth_target_grid = ESMF_FieldCreate(target_grid, & typekind=ESMF_TYPEKIND_R8, & @@ -3264,6 +3355,14 @@ subroutine create_surface_esmf_fields if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & call error_handler("IN FieldCreate", rc) + print*,"- INITIALIZE TARGET grid snow depth." + call ESMF_FieldGet(snow_depth_target_grid, & + farrayPtr=target_ptr, rc=rc) + if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & + call error_handler("IN FieldGet", rc) + + target_ptr = init_val + print*,"- CALL FieldCreate FOR TARGET GRID SEA ICE FRACTION." seaice_fract_target_grid = ESMF_FieldCreate(target_grid, & typekind=ESMF_TYPEKIND_R8, & @@ -3271,6 +3370,14 @@ subroutine create_surface_esmf_fields if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & call error_handler("IN FieldCreate", rc) + print*,"- INITIALIZE TARGET grid sea ice fraction." + call ESMF_FieldGet(seaice_fract_target_grid, & + farrayPtr=target_ptr, rc=rc) + if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & + call error_handler("IN FieldGet", rc) + + target_ptr = init_val + print*,"- CALL FieldCreate FOR TARGET GRID SEA ICE DEPTH." seaice_depth_target_grid = ESMF_FieldCreate(target_grid, & typekind=ESMF_TYPEKIND_R8, & @@ -3278,6 +3385,14 @@ subroutine create_surface_esmf_fields if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & call error_handler("IN FieldCreate", rc) + print*,"- INITIALIZE TARGET sea ice depth." + call ESMF_FieldGet(seaice_depth_target_grid, & + farrayPtr=target_ptr, rc=rc) + if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & + call error_handler("IN FieldGet", rc) + + target_ptr = init_val + print*,"- CALL FieldCreate FOR TARGET GRID SEA ICE SKIN TEMP." seaice_skin_temp_target_grid = ESMF_FieldCreate(target_grid, & typekind=ESMF_TYPEKIND_R8, & @@ -3285,6 +3400,14 @@ subroutine create_surface_esmf_fields if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & call error_handler("IN FieldCreate", rc) + print*,"- INITIALIZE TARGET sea ice skin temp." + call ESMF_FieldGet(seaice_skin_temp_target_grid, & + farrayPtr=target_ptr, rc=rc) + if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & + call error_handler("IN FieldGet", rc) + + target_ptr = init_val + print*,"- CALL FieldCreate FOR TARGET GRID SRFLAG." srflag_target_grid = ESMF_FieldCreate(target_grid, & typekind=ESMF_TYPEKIND_R8, & @@ -3292,6 +3415,14 @@ subroutine create_surface_esmf_fields if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & call error_handler("IN FieldCreate", rc) + print*,"- INITIALIZE TARGET srflag." + call ESMF_FieldGet(srflag_target_grid, & + farrayPtr=target_ptr, rc=rc) + if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & + call error_handler("IN FieldGet", rc) + + target_ptr = init_val + print*,"- CALL FieldCreate FOR TARGET GRID SKIN TEMPERATURE." skin_temp_target_grid = ESMF_FieldCreate(target_grid, & typekind=ESMF_TYPEKIND_R8, & @@ -3299,6 +3430,14 @@ subroutine create_surface_esmf_fields if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & call error_handler("IN FieldCreate", rc) + print*,"- INITIALIZE TARGET grid skin temp." + call ESMF_FieldGet(skin_temp_target_grid, & + farrayPtr=target_ptr, rc=rc) + if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & + call error_handler("IN FieldGet", rc) + + target_ptr = init_val + print*,"- CALL FieldCreate FOR TARGET GRID CANOPY MOISTURE CONTENT." canopy_mc_target_grid = ESMF_FieldCreate(target_grid, & typekind=ESMF_TYPEKIND_R8, & @@ -3306,6 +3445,14 @@ subroutine create_surface_esmf_fields if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & call error_handler("IN FieldCreate", rc) + print*,"- INITIALIZE TARGET grid canopy moisture." + call ESMF_FieldGet(canopy_mc_target_grid, & + farrayPtr=target_ptr, rc=rc) + if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & + call error_handler("IN FieldGet", rc) + + target_ptr = init_val + print*,"- CALL FieldCreate FOR TARGET GRID Z0." z0_target_grid = ESMF_FieldCreate(target_grid, & typekind=ESMF_TYPEKIND_R8, & @@ -3313,6 +3460,14 @@ subroutine create_surface_esmf_fields if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & call error_handler("IN FieldCreate", rc) + print*,"- INITIALIZE TARGET grid z0." + call ESMF_FieldGet(z0_target_grid, & + farrayPtr=target_ptr, rc=rc) + if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & + call error_handler("IN FieldGet", rc) + + target_ptr = init_val + print*,"- CALL FieldCreate FOR INTERPOLATED TARGET GRID TERRAIN." terrain_from_input_grid = ESMF_FieldCreate(target_grid, & typekind=ESMF_TYPEKIND_R8, & @@ -3320,6 +3475,14 @@ subroutine create_surface_esmf_fields if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & call error_handler("IN FieldCreate", rc) + print*,"- INITIALIZE TARGET grid interpolated terrain." + call ESMF_FieldGet(terrain_from_input_grid, & + farrayPtr=target_ptr, rc=rc) + if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & + call error_handler("IN FieldGet", rc) + + target_ptr = init_val + print*,"- CALL FieldCreate FOR INTERPOLATED TARGET GRID SOIL TYPE." soil_type_from_input_grid = ESMF_FieldCreate(target_grid, & typekind=ESMF_TYPEKIND_R8, & @@ -3327,6 +3490,14 @@ subroutine create_surface_esmf_fields if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & call error_handler("IN FieldCreate", rc) + print*,"- INITIALIZE TARGET grid soil type" + call ESMF_FieldGet(soil_type_from_input_grid, & + farrayPtr=target_ptr, rc=rc) + if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & + call error_handler("IN FieldGet", rc) + + target_ptr = init_val + print*,"- CALL FieldCreate FOR TARGET GRID SOIL TEMPERATURE." soil_temp_target_grid = ESMF_FieldCreate(target_grid, & typekind=ESMF_TYPEKIND_R8, & @@ -3336,6 +3507,14 @@ subroutine create_surface_esmf_fields if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & call error_handler("IN FieldCreate", rc) + print*,"- INITIALIZE TARGET grid soil temp" + call ESMF_FieldGet(soil_temp_target_grid, & + farrayPtr=target_ptr_3d, rc=rc) + if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & + call error_handler("IN FieldGet", rc) + + target_ptr_3d = init_val + print*,"- CALL FieldCreate FOR TARGET GRID TOTAL SOIL MOISTURE." soilm_tot_target_grid = ESMF_FieldCreate(target_grid, & typekind=ESMF_TYPEKIND_R8, & @@ -3345,6 +3524,14 @@ subroutine create_surface_esmf_fields if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & call error_handler("IN FieldCreate", rc) + print*,"- INITIALIZE TARGET grid soil moist" + call ESMF_FieldGet(soilm_tot_target_grid, & + farrayPtr=target_ptr_3d, rc=rc) + if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & + call error_handler("IN FieldGet", rc) + + target_ptr_3d = init_val + print*,"- CALL FieldCreate FOR TARGET GRID LIQUID SOIL MOISTURE." soilm_liq_target_grid = ESMF_FieldCreate(target_grid, & typekind=ESMF_TYPEKIND_R8, & @@ -3354,6 +3541,14 @@ subroutine create_surface_esmf_fields if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & call error_handler("IN FieldCreate", rc) + print*,"- INITIALIZE TARGET grid soil liq" + call ESMF_FieldGet(soilm_liq_target_grid, & + farrayPtr=target_ptr_3d, rc=rc) + if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & + call error_handler("IN FieldGet", rc) + + target_ptr_3d = init_val + end subroutine create_surface_esmf_fields subroutine create_nst_esmf_fields From ecec8efe54becab595685a6d4d777e9bffa91fbb Mon Sep 17 00:00:00 2001 From: Jili Dong Date: Thu, 31 Oct 2019 17:22:22 +0000 Subject: [PATCH 33/34] re-order grib2 isobaric levels --- sorc/chgres_cube.fd/input_data.F90 | 44 +++++++++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/sorc/chgres_cube.fd/input_data.F90 b/sorc/chgres_cube.fd/input_data.F90 index 09f3962fa..7ed646cc6 100644 --- a/sorc/chgres_cube.fd/input_data.F90 +++ b/sorc/chgres_cube.fd/input_data.F90 @@ -2211,7 +2211,7 @@ subroutine read_input_atm_grib2_file(localpet) type(atmdata), allocatable :: atm(:) tracers(:) = "NULL" - trac_names_grib = (/":SPFH:",":CLWMR:", "O3MR",":CICE:", ":RWMR:",":SNMR:",":GRLE:", & + trac_names_grib = (/":SPFH:",":CLMR:", "O3MR",":CICE:", ":RWMR:",":SNMR:",":GRLE:", & ":TCDC:", ":NCCICE:",":SPNCR:", ":NCONCD:",":PMTF:",":PMTC:",":TKE:"/) trac_names_vmap = (/"sphum", "liq_wat","o3mr","ice_wat", & "rainwat", "snowwat", "graupel", "cld_amt", "ice_nc", & @@ -2278,6 +2278,17 @@ subroutine read_input_atm_grib2_file(localpet) if (localpet==0) print*, "LEVEL = ", slevs(i) enddo +! Jili Dong add sort to re-order isobaric levels + call quicksort(rlevs,1,lev_input) + + do i = 1,lev_input + write(slevs(i),"(I5)") int(rlevs(i)/100.0) + slevs(i) = ":"//trim(adjustl(slevs(i)))//" mb:" + if (localpet==0) print*, "level after sort = ",slevs(i) + enddo +! Jili Dong add sort to re-order isobaric levels + + allocate(vcoord(levp1_input,2)) if (localpet == 0) print*,"- READ VERTICAL COORDINATE INFO." if (localpet == 0) print*, metadata @@ -5266,4 +5277,35 @@ subroutine cleanup_input_sfc_data end subroutine cleanup_input_sfc_data +! Jili Dong add sort subroutine +! quicksort.f -*-f90-*- +! Author: t-nissie +! License: GPLv3 +! Gist: https://gist.github.com/t-nissie/479f0f16966925fa29ea +!! +recursive subroutine quicksort(a, first, last) + implicit none + real*8 a(*), x, t + integer first, last + integer i, j + + x = a( (first+last) / 2 ) + i = first + j = last + do + do while (a(i) < x) + i=i+1 + end do + do while (x < a(j)) + j=j-1 + end do + if (i >= j) exit + t = a(i); a(i) = a(j); a(j) = t + i=i+1 + j=j-1 + end do + if (first < i-1) call quicksort(a, first, i-1) + if (j+1 < last) call quicksort(a, j+1, last) +end subroutine quicksort + end module input_data From 332c584b66440c367a5d1e8e4108745ede32157c Mon Sep 17 00:00:00 2001 From: Jili Dong Date: Thu, 31 Oct 2019 17:28:51 +0000 Subject: [PATCH 34/34] keep grib2 CLWMR name consistent with origional --- sorc/chgres_cube.fd/input_data.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/chgres_cube.fd/input_data.F90 b/sorc/chgres_cube.fd/input_data.F90 index 7ed646cc6..a2ff6e527 100644 --- a/sorc/chgres_cube.fd/input_data.F90 +++ b/sorc/chgres_cube.fd/input_data.F90 @@ -2211,7 +2211,7 @@ subroutine read_input_atm_grib2_file(localpet) type(atmdata), allocatable :: atm(:) tracers(:) = "NULL" - trac_names_grib = (/":SPFH:",":CLMR:", "O3MR",":CICE:", ":RWMR:",":SNMR:",":GRLE:", & + trac_names_grib = (/":SPFH:",":CLWMR:", "O3MR",":CICE:", ":RWMR:",":SNMR:",":GRLE:", & ":TCDC:", ":NCCICE:",":SPNCR:", ":NCONCD:",":PMTF:",":PMTC:",":TKE:"/) trac_names_vmap = (/"sphum", "liq_wat","o3mr","ice_wat", & "rainwat", "snowwat", "graupel", "cld_amt", "ice_nc", &