diff --git a/driver_scripts/driver_grid.jet.sh b/driver_scripts/driver_grid.jet.sh index 0f7ca77b0..7035b33f7 100644 --- a/driver_scripts/driver_grid.jet.sh +++ b/driver_scripts/driver_grid.jet.sh @@ -54,7 +54,7 @@ set -x . /apps/lmod/lmod/init/sh module purge -module load intel/15.0.3.187 +module load intel/18.0.5.274 module load impi/2018.4.274 module load szip module load hdf5 diff --git a/modulefiles/chgres_cube.jet b/modulefiles/chgres_cube.jet index 3fc899f96..afc371978 100644 --- a/modulefiles/chgres_cube.jet +++ b/modulefiles/chgres_cube.jet @@ -2,28 +2,21 @@ ## chgres build module for Jet ############################################################# -module use /mnt/lfs3/projects/hfv3gfs/gwv/ljtjet/lib/modulefiles - - module load intel/18.0.5.274 module load impi/2018.4.274 module load szip/2.1 module load hdf5/1.8.9 module load netcdf/4.2.1.1 -# -# -export NCEPLIBS="/mnt/lfs3/projects/hfv3gfs/gwv/ljtjet/lib" - -module use /mnt/lfs3/projects/hfv3gfs/gwv/ltmp2/lib//modulefiles -module load esmflocal/8.1.21b +module load w3nco/v2.0.6 +module load sp/v2.0.2 +module load bacio/v2.0.2 +module load sigio/v2.1.0 +module load sfcio/v1.0.0 +module load nemsio/v2.2.3 -module load w3nco-intel-sandybridge/2.0.6 -module load sp-intel-sandybridge/2.0.2 -module load bacio-intel-sandybridge/2.0.2 -module load sigio-intel-sandybridge/2.0.1 -module load sfcio-intel-sandybridge/1.0.0 -module load nemsio-intel-sandybridge/2.2.2 +module use /mnt/lfs3/projects/hfv3gfs/gwv/ljtjet/lib/modulefiles +module load esmflocal/ESMF_8_0_0_beta_snapshot_21 export FCOMP=mpiifort export FFLAGS="-O3 -fp-model precise -g -traceback -r8 -i4 -qopenmp -convert big_endian -assume byterecl" diff --git a/modulefiles/fv3gfs/global_chgres.jet b/modulefiles/fv3gfs/global_chgres.jet index 98c3bf282..088023563 100644 --- a/modulefiles/fv3gfs/global_chgres.jet +++ b/modulefiles/fv3gfs/global_chgres.jet @@ -3,7 +3,7 @@ ############################################################# module purge -module load intel/15.0.3.187 +module load intel/18.0.5.274 module load szip/2.1 module load hdf5/1.8.9 module load netcdf/4.2.1.1 @@ -13,8 +13,8 @@ module load sp/v2.0.2 module load w3emc/v2.2.0 module load w3nco/v2.0.6 module load bacio/v2.0.2 -module load sigio/v2.0.1 -module load nemsio/v2.2.2 +module load sigio/v2.1.0 +module load nemsio/v2.2.3 module load nemsiogfs/v2.0.1 module load sfcio/v1.0.0 module load gfsio/v1.1.0 diff --git a/modulefiles/fv3gfs/global_cycle.jet b/modulefiles/fv3gfs/global_cycle.jet index 6702cdf03..fc12b53f8 100644 --- a/modulefiles/fv3gfs/global_cycle.jet +++ b/modulefiles/fv3gfs/global_cycle.jet @@ -3,12 +3,12 @@ ############################################################# # Loading Intel Compiler Suite -module load intel/15.0.3.187 -module load impi/5.0.3.048 +module load intel/18.0.5.274 +module load impi/2018.4.274 module load w3nco/v2.0.6 module load sp/v2.0.2 -module load bacio/v2.0.1 +module load bacio/v2.0.2 module load ip/v3.0.0 module load szip module load hdf5/1.8.9 diff --git a/modulefiles/fv3gfs/nst_tf_chg.jet b/modulefiles/fv3gfs/nst_tf_chg.jet index a66e1f644..67bd8a087 100644 --- a/modulefiles/fv3gfs/nst_tf_chg.jet +++ b/modulefiles/fv3gfs/nst_tf_chg.jet @@ -3,11 +3,11 @@ ############################################################# # Loading Intel Compiler Suite -module load intel/15.0.3.187 +module load intel/18.0.5.274 module load w3nco/v2.0.6 module load bacio/v2.0.2 -module load nemsio/v2.2.2 +module load nemsio/v2.2.3 module load szip module load hdf5/1.8.9 module load netcdf/4.2.1.1 diff --git a/modulefiles/fv3gfs/orog.jet b/modulefiles/fv3gfs/orog.jet index 5b067744d..893457136 100644 --- a/modulefiles/fv3gfs/orog.jet +++ b/modulefiles/fv3gfs/orog.jet @@ -5,7 +5,7 @@ # module purge -module load intel/15.0.3.187 +module load intel/18.0.5.274 module load szip/2.1 module load hdf5/1.8.9 module load netcdf/4.2.1.1 @@ -13,6 +13,6 @@ module load netcdf/4.2.1.1 # Loding nceplibs modules module load ip/v2.0.0 module load sp/v2.0.2 -module load w3emc/v2.2.0 +module load w3emc/v2.3.0 module load w3nco/v2.0.6 module load bacio/v2.0.2 diff --git a/modulefiles/module_base.jet b/modulefiles/module_base.jet index c6c767eed..00e997aee 100644 --- a/modulefiles/module_base.jet +++ b/modulefiles/module_base.jet @@ -1,3 +1,4 @@ +#%Module############################################################# ## ## nems prerequisites ## @@ -6,19 +7,15 @@ proc ModulesHelp {} { puts stderr "The prerequisites for compiling or running FV3 on Jet. " } -module load newdefaults intel/15.0.3.187 impi/5.1.3.181 szip hdf5 netcdf4/4.2.1.1 +module load intel/15.0.3.187 impi/2018.4.274 szip hdf5 netcdf4/4.2.1.1 -module use /lfs3/projects/hfv3gfs/nwprod/lib/modulefiles +module use /lfs3/projects/hfv3gfs/nwprod/NCEPLIBS/modulefiles module load bacio/v2.0.2 module load sp/v2.0.2 module load ip/v2.0.0 module load w3nco/v2.0.6 -module load w3emc/v2.2.0 -module load nemsio/v2.2.2 - -#set NCEPLIBS $::env(NCEPLIBS) -#module use $NCEPLIBS/modulefiles -#module load esmf/7.1.0r_impi +module load w3emc/v2.3.0 +module load nemsio/v2.2.3 module use /lfs3/projects/hwrf-vd/soft/modulefiles module load prod_util @@ -29,4 +26,3 @@ module load hpss module load mpiserial module load gempak/7.4.2 - diff --git a/modulefiles/module_nemsutil.jet b/modulefiles/module_nemsutil.jet index 7f6bb028b..c1ff57408 100644 --- a/modulefiles/module_nemsutil.jet +++ b/modulefiles/module_nemsutil.jet @@ -3,12 +3,12 @@ ############################################################# # Loading Intel Compiler Suite -module load intel/15.0.3.187 +module load intel/18.0.5.274 module load impi/5.0.3.048 # Loding nceplibs modules module load w3nco/v2.0.6 -module load bacio/v2.0.1 -module load nemsio/v2.2.1 +module load bacio/v2.0.2 +module load nemsio/v2.2.3 export FCMP=ifort diff --git a/modulefiles/modulefile.global_emcsfc_ice_blend.jet b/modulefiles/modulefile.global_emcsfc_ice_blend.jet index 37f4ca900..2e094dc03 100644 --- a/modulefiles/modulefile.global_emcsfc_ice_blend.jet +++ b/modulefiles/modulefile.global_emcsfc_ice_blend.jet @@ -2,15 +2,15 @@ ## emcsfc_ice_blend build module for Jet ############################################################# -module load intel/15.0.3.187 +module load intel/18.0.5.274 export FCOMP=ifort export FFLAGS="-O0 -i4" # load ncep library modules module load w3nco/v2.0.6 -module load bacio/v2.0.1 +module load bacio/v2.0.2 module load jasper/v1.900.1 module load z/v1.2.6 module load png/v1.2.44 -module load g2/v2.5.0 +module load g2/v3.1.0 diff --git a/modulefiles/modulefile.global_emcsfc_snow2mdl.jet b/modulefiles/modulefile.global_emcsfc_snow2mdl.jet index b1f6a6fd6..0b88ae7c0 100644 --- a/modulefiles/modulefile.global_emcsfc_snow2mdl.jet +++ b/modulefiles/modulefile.global_emcsfc_snow2mdl.jet @@ -4,19 +4,19 @@ # load intel compiler -module load intel/15.0.3.187 +module load intel/18.0.5.274 # load ncep library modules module load ip/v3.0.0 module load sp/v2.0.2 module load w3nco/v2.0.6 -module load bacio/v2.0.1 +module load bacio/v2.0.2 module load jasper/v1.900.1 module load z/v1.2.6 module load png/v1.2.44 -module load g2/v2.5.0 +module load g2/v3.1.0 module load landsfcutil/v2.1.0 export FCOMP=ifort -export FFLAGS="-O0 -r8 -i4 -FR -I${IP_INCd} -openmp -convert big_endian -assume byterecl" +export FFLAGS="-O0 -r8 -i4 -FR -I${IP_INCd} -qopenmp -convert big_endian -assume byterecl" diff --git a/modulefiles/modulefile.sfc_climo_gen.jet b/modulefiles/modulefile.sfc_climo_gen.jet index 108dfcf2d..4969d85c6 100644 --- a/modulefiles/modulefile.sfc_climo_gen.jet +++ b/modulefiles/modulefile.sfc_climo_gen.jet @@ -4,7 +4,7 @@ module use /mnt/lfs3/projects/hfv3gfs/gwv/ljtjet/lib/modulefiles -module load intel/15.0.3.187 +module load intel/18.0.5.274 module load impi/2018.4.274 module load szip module load hdf5 diff --git a/reg_tests/chgres_cube/driver.cray.sh b/reg_tests/chgres_cube/driver.cray.sh index e6f073256..005812149 100755 --- a/reg_tests/chgres_cube/driver.cray.sh +++ b/reg_tests/chgres_cube/driver.cray.sh @@ -34,7 +34,7 @@ module list export OUTDIR=/gpfs/hps3/stmp/$LOGNAME/chgres_reg_tests QUEUE="debug" -PROJECT_CODE="FV3GFS-T2O" +PROJECT_CODE="GFS-DEV" #----------------------------------------------------------------------------- # Should not have to change anything below here. HOMEufs is the root diff --git a/reg_tests/chgres_cube/driver.dell.sh b/reg_tests/chgres_cube/driver.dell.sh index e9369e2f9..a8fcb7cf2 100755 --- a/reg_tests/chgres_cube/driver.dell.sh +++ b/reg_tests/chgres_cube/driver.dell.sh @@ -33,7 +33,7 @@ module list export OUTDIR=/gpfs/dell1/stmp/$LOGNAME/chgres_reg_tests QUEUE="debug" -PROJECT_CODE="FV3GFS-T2O" +PROJECT_CODE="GFS-DEV" #----------------------------------------------------------------------------- # Should not have to change anything below here. HOMEufs is the root diff --git a/reg_tests/chgres_cube/driver.jet.sh b/reg_tests/chgres_cube/driver.jet.sh index b256b64c3..7fe60ee71 100755 --- a/reg_tests/chgres_cube/driver.jet.sh +++ b/reg_tests/chgres_cube/driver.jet.sh @@ -26,7 +26,7 @@ set -x source /apps/lmod/lmod/init/sh module purge -module load intel/15.0.3.187 +module load intel/18.0.5.274 module load impi/2018.4.274 module load szip module load hdf5 diff --git a/sorc/chgres_cube.fd/input_data.F90 b/sorc/chgres_cube.fd/input_data.F90 index f6a823a88..865117cb6 100644 --- a/sorc/chgres_cube.fd/input_data.F90 +++ b/sorc/chgres_cube.fd/input_data.F90 @@ -1624,11 +1624,14 @@ subroutine read_input_atm_restart_file(localpet) enddo if (localpet < num_tiles_input_grid) then - error=nf90_inq_varid(ncid, 'W', id_var) - call netcdf_err(error, 'reading field id' ) - error=nf90_get_var(ncid, id_var, data_one_tile_3d) - call netcdf_err(error, 'reading field' ) - data_one_tile_3d(:,:,1:lev_input) = data_one_tile_3d(:,:,lev_input:1:-1) +! error=nf90_inq_varid(ncid, 'W', id_var) +! call netcdf_err(error, 'reading field id' ) +! error=nf90_get_var(ncid, id_var, data_one_tile_3d) +! call netcdf_err(error, 'reading field' ) +! data_one_tile_3d(:,:,1:lev_input) = data_one_tile_3d(:,:,lev_input:1:-1) + +! Using 'w' from restart files has caused problems. Set to zero. + data_one_tile_3d = 0.0_8 endif do tile = 1, num_tiles_input_grid @@ -1958,12 +1961,16 @@ subroutine read_input_atm_history_file(localpet) endif if (localpet < num_tiles_input_grid) then - print*,"- READ VERTICAL VELOCITY." - error=nf90_inq_varid(ncid, 'dzdt', id_var) - call netcdf_err(error, 'reading field id' ) - error=nf90_get_var(ncid, id_var, data_one_tile_3d) - call netcdf_err(error, 'reading field' ) - data_one_tile_3d(:,:,1:lev_input) = data_one_tile_3d(:,:,lev_input:1:-1) +! print*,"- READ VERTICAL VELOCITY." +! error=nf90_inq_varid(ncid, 'dzdt', id_var) +! call netcdf_err(error, 'reading field id' ) +! error=nf90_get_var(ncid, id_var, data_one_tile_3d) +! call netcdf_err(error, 'reading field' ) +! data_one_tile_3d(:,:,1:lev_input) = data_one_tile_3d(:,:,lev_input:1:-1) + +! Using w from the tiled history files has caused problems. +! Set to zero. + data_one_tile_3d = 0.0_8 endif do tile = 1, num_tiles_input_grid diff --git a/sorc/chgres_cube.fd/model_grid.F90 b/sorc/chgres_cube.fd/model_grid.F90 index 7591617b8..c0b25ee89 100644 --- a/sorc/chgres_cube.fd/model_grid.F90 +++ b/sorc/chgres_cube.fd/model_grid.F90 @@ -167,6 +167,8 @@ subroutine define_input_grid_gaussian(localpet, npets) real(esmf_kind_r8), allocatable :: longitude(:,:) real(esmf_kind_r8), pointer :: lat_src_ptr(:,:) real(esmf_kind_r8), pointer :: lon_src_ptr(:,:) + real(esmf_kind_r8), pointer :: lat_corner_src_ptr(:,:) + real(esmf_kind_r8), pointer :: lon_corner_src_ptr(:,:) real(esmf_kind_r8) :: deltalon real(esmf_kind_r8), allocatable :: slat(:), wlat(:) @@ -326,22 +328,22 @@ subroutine define_input_grid_gaussian(localpet, npets) call error_handler("IN GridAddCoord", rc) print*,"- CALL GridGetCoord FOR INPUT GRID X-COORD." - nullify(lon_src_ptr) + nullify(lon_corner_src_ptr) call ESMF_GridGetCoord(input_grid, & staggerLoc=ESMF_STAGGERLOC_CORNER, & coordDim=1, & - farrayPtr=lon_src_ptr, rc=rc) + farrayPtr=lon_corner_src_ptr, rc=rc) if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & call error_handler("IN GridGetCoord", rc) print*,"- CALL GridGetCoord FOR INPUT GRID Y-COORD." - nullify(lat_src_ptr) + nullify(lat_corner_src_ptr) call ESMF_GridGetCoord(input_grid, & staggerLoc=ESMF_STAGGERLOC_CORNER, & coordDim=2, & computationalLBound=clb, & computationalUBound=cub, & - farrayPtr=lat_src_ptr, rc=rc) + farrayPtr=lat_corner_src_ptr, rc=rc) if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__line__,file=__file__)) & call error_handler("IN GridGetCoord", rc) @@ -349,17 +351,17 @@ subroutine define_input_grid_gaussian(localpet, npets) do j = clb(2), cub(2) do i = clb(1), cub(1) - lon_src_ptr(i,j) = longitude(i,1) - (0.5_esmf_kind_r8*deltalon) - if (lon_src_ptr(i,j) > 360.0_esmf_kind_r8) lon_src_ptr(i,j) = lon_src_ptr(i,j) - 360.0_esmf_kind_r8 + lon_corner_src_ptr(i,j) = longitude(i,1) - (0.5_esmf_kind_r8*deltalon) + if (lon_corner_src_ptr(i,j) > 360.0_esmf_kind_r8) lon_corner_src_ptr(i,j) = lon_corner_src_ptr(i,j) - 360.0_esmf_kind_r8 if (j == 1) then - lat_src_ptr(i,j) = 90.0_esmf_kind_r8 + lat_corner_src_ptr(i,j) = 90.0_esmf_kind_r8 cycle endif if (j == jp1_input) then - lat_src_ptr(i,j) = -90.0_esmf_kind_r8 + lat_corner_src_ptr(i,j) = -90.0_esmf_kind_r8 cycle endif - lat_src_ptr(i,j) = 0.5_esmf_kind_r8 * (latitude(i,j-1)+ latitude(i,j)) + lat_corner_src_ptr(i,j) = 0.5_esmf_kind_r8 * (latitude(i,j-1)+ latitude(i,j)) enddo enddo diff --git a/sorc/chgres_cube.fd/surface.F90 b/sorc/chgres_cube.fd/surface.F90 index a00bcc923..3f833ba74 100644 --- a/sorc/chgres_cube.fd/surface.F90 +++ b/sorc/chgres_cube.fd/surface.F90 @@ -619,8 +619,11 @@ subroutine interp(localpet) print*,'bf ice repl, ice, slmsk_target(14,86) = ', data_one_tile(14,86), mask_target_one_tile(14,86) do j = 1, j_target do i = 1, i_target - if (data_one_tile(i,j) < 0.15) data_one_tile(i,j) = 0.0 - if (data_one_tile(i,j) >= 0.15) mask_target_one_tile(i,j) = 2 + if (data_one_tile(i,j) > 1.0_esmf_kind_r8) then + data_one_tile(i,j) = 1.0_esmf_kind_r8 + endif + if (data_one_tile(i,j) < 0.15_esmf_kind_r8) data_one_tile(i,j) = 0.0_esmf_kind_r8 + if (data_one_tile(i,j) >= 0.15_esmf_kind_r8) mask_target_one_tile(i,j) = 2 enddo enddo print*,'af ice repl, ice, slmsk_target(14,86) = ', data_one_tile(14,86),mask_target_one_tile(14,86) diff --git a/sorc/machine-setup.sh b/sorc/machine-setup.sh index 9bc1be6da..48b2ee1bf 100644 --- a/sorc/machine-setup.sh +++ b/sorc/machine-setup.sh @@ -27,7 +27,7 @@ if [[ -d /lfs3 ]] ; then fi target=jet module purge - module use /mnt/lfs3/projects/hfv3gfs/nwprod/lib/modulefiles + module use /lfs3/projects/hfv3gfs/nwprod/NCEPLIBS/modulefiles elif [[ -d /scratch1 ]] ; then # We are on NOAA Hera if ( ! eval module help > /dev/null 2>&1 ) ; then diff --git a/ush/fv3gfs_driver_grid.sh b/ush/fv3gfs_driver_grid.sh index 838552139..3fd806c67 100755 --- a/ush/fv3gfs_driver_grid.sh +++ b/ush/fv3gfs_driver_grid.sh @@ -185,6 +185,10 @@ if [ $gtype = uniform ] || [ $gtype = stretch ] || [ $gtype = nest ]; then echo set -x $script_dir/fv3gfs_make_orog.sh $res $tile $grid_dir $orog_dir $script_dir $topo $TMPDIR + err=$? + if [ $err != 0 ]; then + exit $err + fi tile=$(( $tile + 1 )) done fi @@ -322,6 +326,10 @@ elif [ $gtype = regional ]; then echo set -x $script_dir/fv3gfs_make_orog.sh $res $tile $grid_dir $orog_dir $script_dir $topo $TMPDIR + err=$? + if [ $err != 0 ]; then + exit $err + fi fi set +x