Skip to content

Commit

Permalink
Update the GDAS initialization scripts to process GFS v16 data. (#531)
Browse files Browse the repository at this point in the history
Update to process OPS GFS v16 data or GFS v16 parallel data.

For all input data sources, add option to process the GFS or the GDAS 
CDUMP.  In other words, process for a GFS free forecast or process 
for a GDAS/ENKF cycle

Fixes #102
  • Loading branch information
GeorgeGayno-NOAA authored Jun 3, 2021
1 parent ebc60b8 commit f09acb9
Show file tree
Hide file tree
Showing 15 changed files with 1,106 additions and 282 deletions.
99 changes: 65 additions & 34 deletions util/gdas_init/config
Original file line number Diff line number Diff line change
Expand Up @@ -15,41 +15,54 @@
#
# Variable definitions:
# --------------------
# EXTRACT_DIR - directory where data extracted from HPSS
# is stored.
# EXTRACT_DIR - Directory where data extracted from HPSS
# is stored.
# EXTRACT_DATA - Set to 'yes' to extract data from HPSS.
# If data has been extracted and is located
# in EXTRACT_DIR, set to 'no'.
# RUN_CHGRES - To run chgres, set to 'yes'. To extract
# data only, set to 'no'.
# yy/mm/dd/hh - The year/month/day/hour of your desired
# experiment. Currently, does not support
# pre-ENKF GFS data, prior to
# 2012 May 21 00z.
# LEVS - Number of hybrid levels plus 1. To
# run with 64 levels, set LEVS to 65.
# CRES_HIRES - Resolution of the hires component of
# your experiment.
# CRES_ENKF - Resolution of the enkf component of the
# your experiment.
# UFS_DIR - Location of your checked out UFS_UTILS
# repo.
# OUTDIR - Directory where data output from chgres is stored.
#
# RUN_CHGRES - To run chgres, set to 'yes'. To extract
# data only, set to 'no'.
# yy/mm/dd/hh - The year/month/day/hour of your desired
# experiment. Currently, does not support
# pre-ENKF GFS data, prior to
# 2012 May 21 00z. Use two digits.
# LEVS - Number of hybrid levels plus 1. To
# run with 64 levels, set LEVS to 65.
# CRES_HIRES - Resolution of the hires component of
# your experiment.
# CRES_ENKF - Resolution of the enkf component of the
# your experiment.
# UFS_DIR - Location of your checked out UFS_UTILS
# repo.
# OUTDIR - Directory where the coldstart data output
# from chgres is stored.
# CDUMP - When 'gdas', will process gdas and enkf
# members. When 'gfs', will process gfs
# member for running free forecast only.
# use_v16retro - When 'yes', use v16 retro parallel data.
# The retro parallel tarballs can be missing
# or incomplete. So this option may not
# always work. Contact [email protected]
# if you encounter problems.
#
#-----------------------------------------------------------

EXTRACT_DIR=/gpfs/dell1/stmp/$USER/gdas.init/input
EXTRACT_DATA=yes

RUN_CHGRES=yes

yy=2017
mm=07
dd=19
hh=18
yy=2021
mm=03
dd=21
hh=06

use_v16retro=no

LEVS=65

CDUMP=gdas

CRES_HIRES=C192
CRES_ENKF=C96

Expand All @@ -61,20 +74,38 @@ OUTDIR=/gpfs/dell1/stmp/$USER/gdas.init/output
# Dont touch anything below here.
#---------------------------------------------------------

gfs_ver=v15
if [ "$use_v16retro" = "yes" ]; then

gfs_ver=v16retro

else

gfs_ver=v16

# No ENKF data prior to 2012/05/21/00z
if [ $yy$mm$dd$hh -lt 2012052100 ]; then
set +x
echo FATAL ERROR: SCRIPTS DO NOT SUPPORT OLD GFS DATA
exit 2
elif [ $yy$mm$dd$hh -lt 2016051000 ]; then
gfs_ver=v12
elif [ $yy$mm$dd$hh -lt 2017072000 ]; then
gfs_ver=v13
elif [ $yy$mm$dd$hh -lt 2019061200 ]; then
gfs_ver=v14
if [ $yy$mm$dd$hh -lt 2012052100 ]; then
set +x
echo FATAL ERROR: SCRIPTS DO NOT SUPPORT OLD GFS DATA
exit 2
elif [ $yy$mm$dd$hh -lt 2016051000 ]; then
gfs_ver=v12
elif [ $yy$mm$dd$hh -lt 2017072000 ]; then
gfs_ver=v13
elif [ $yy$mm$dd$hh -lt 2019061200 ]; then
gfs_ver=v14
elif [ $yy$mm$dd$hh -lt 2021032100 ]; then
gfs_ver=v15
# The way the v16 switch over was done, there is no complete
# set of v16 or v15 data for 2021032100. And although
# v16 was officially implemented 2021032212, the v16 prod
# tarballs were archived starting 2021032106.
elif [ $yy$mm$dd$hh -lt 2021032106 ]; then
set +x
echo FATAL ERROR: NO V15 OR V16 DATA FOR 2021032100
exit 1
fi

fi

export EXTRACT_DIR yy mm dd hh UFS_DIR OUTDIR CRES_HIRES CRES_ENKF RUN_CHGRES
export EXTRACT_DIR yy mm dd hh UFS_DIR OUTDIR CRES_HIRES CRES_ENKF
export LEVS gfs_ver
144 changes: 96 additions & 48 deletions util/gdas_init/driver.cray.sh
Original file line number Diff line number Diff line change
Expand Up @@ -29,38 +29,50 @@ if [ $EXTRACT_DATA == yes ]; then

case $gfs_ver in
v12 | v13)
bsub -o log.data.hires -e log.data.hires -q $QUEUE -P $PROJECT_CODE -J get.data.hires -W $WALLT \
-R "rusage[mem=$MEM]" "./get_pre-v14.data.sh hires"
bsub -o log.data.enkf -e log.data.enkf -q $QUEUE -P $PROJECT_CODE -J get.data.enkf -W $WALLT \
-R "rusage[mem=$MEM]" "./get_pre-v14.data.sh enkf"
bsub -o log.data.$CDUMP -e log.data.$CDUMP -q $QUEUE -P $PROJECT_CODE -J get.data.$CDUMP -W $WALLT \
-R "rusage[mem=$MEM]" "./get_pre-v14.data.sh ${CDUMP}"
if [ "$CDUMP" = "gdas" ] ; then
bsub -o log.data.enkf -e log.data.enkf -q $QUEUE -P $PROJECT_CODE -J get.data.enkf -W $WALLT \
-R "rusage[mem=$MEM]" "./get_pre-v14.data.sh enkf"
fi
DEPEND="-w ended(get.data.*)"
;;
v14)
bsub -o log.data.hires -e log.data.hires -q $QUEUE -P $PROJECT_CODE -J get.data.hires -W $WALLT \
-R "rusage[mem=$MEM]" "./get_v14.data.sh hires"
bsub -o log.data.enkf -e log.data.enkf -q $QUEUE -P $PROJECT_CODE -J get.data.enkf -W $WALLT \
-R "rusage[mem=$MEM]" "./get_v14.data.sh enkf"
bsub -o log.data.$CDUMP -e log.data.$CDUMP -q $QUEUE -P $PROJECT_CODE -J get.data.$CDUMP -W $WALLT \
-R "rusage[mem=$MEM]" "./get_v14.data.sh ${CDUMP}"
if [ "$CDUMP" = "gdas" ] ; then
bsub -o log.data.enkf -e log.data.enkf -q $QUEUE -P $PROJECT_CODE -J get.data.enkf -W $WALLT \
-R "rusage[mem=$MEM]" "./get_v14.data.sh enkf"
fi
DEPEND="-w ended(get.data.*)"
;;
v15)
bsub -o log.data.hires -e log.data.hires -q $QUEUE -P $PROJECT_CODE -J get.data.hires -W $WALLT \
-R "rusage[mem=$MEM]" "./get_v15.data.sh hires"
bsub -o log.data.grp1 -e log.data.grp1 -q $QUEUE -P $PROJECT_CODE -J get.data.enkf1 -W $WALLT \
-R "rusage[mem=$MEM]" "./get_v15.data.sh grp1"
bsub -o log.data.grp2 -e log.data.grp2 -q $QUEUE -P $PROJECT_CODE -J get.data.enkf2 -W $WALLT \
-R "rusage[mem=$MEM]" "./get_v15.data.sh grp2"
bsub -o log.data.grp3 -e log.data.grp3 -q $QUEUE -P $PROJECT_CODE -J get.data.enkf3 -W $WALLT \
-R "rusage[mem=$MEM]" "./get_v15.data.sh grp3"
bsub -o log.data.grp4 -e log.data.grp4 -q $QUEUE -P $PROJECT_CODE -J get.data.enkf4 -W $WALLT \
-R "rusage[mem=$MEM]" "./get_v15.data.sh grp4"
bsub -o log.data.grp5 -e log.data.grp5 -q $QUEUE -P $PROJECT_CODE -J get.data.enkf5 -W $WALLT \
-R "rusage[mem=$MEM]" "./get_v15.data.sh grp5"
bsub -o log.data.grp6 -e log.data.grp6 -q $QUEUE -P $PROJECT_CODE -J get.data.enkf6 -W $WALLT \
-R "rusage[mem=$MEM]" "./get_v15.data.sh grp6"
bsub -o log.data.grp7 -e log.data.grp7 -q $QUEUE -P $PROJECT_CODE -J get.data.enkf7 -W $WALLT \
-R "rusage[mem=$MEM]" "./get_v15.data.sh grp7"
bsub -o log.data.grp8 -e log.data.grp8 -q $QUEUE -P $PROJECT_CODE -J get.data.enkf8 -W $WALLT \
-R "rusage[mem=$MEM]" "./get_v15.data.sh grp8"
bsub -o log.data.${CDUMP} -e log.data.${CDUMP} -q $QUEUE -P $PROJECT_CODE -J get.data.${CDUMP} -W $WALLT \
-R "rusage[mem=$MEM]" "./get_v15.data.sh ${CDUMP}"
if [ "$CDUMP" = "gdas" ] ; then
for group in grp1 grp2 grp3 grp4 grp5 grp6 grp7 grp8
do
bsub -o log.data.${group} -e log.data.${group} -q $QUEUE -P $PROJECT_CODE -J get.data.${group} -W $WALLT \
-R "rusage[mem=$MEM]" "./get_v15.data.sh ${group}"
done
fi
DEPEND="-w ended(get.data.*)"
;;
v16retro)
bsub -o log.data.$CDUMP -e log.data.$CDUMP -q $QUEUE -P $PROJECT_CODE -J get.data.$CDUMP -W $WALLT \
-R "rusage[mem=$MEM]" "./get_v16retro.data.sh ${CDUMP}"
DEPEND="-w ended(get.data.${CDUMP})"
;;
v16)
bsub -o log.data.${CDUMP} -e log.data.${CDUMP} -q $QUEUE -P $PROJECT_CODE -J get.data.${CDUMP} -W $WALLT \
-R "rusage[mem=$MEM]" "./get_v16.data.sh ${CDUMP}"
if [ "$CDUMP" = "gdas" ] ; then
for group in grp1 grp2 grp3 grp4 grp5 grp6 grp7 grp8
do
bsub -o log.data.${group} -e log.data.${group} -q $QUEUE -P $PROJECT_CODE -J get.data.${group} -W $WALLT \
-R "rusage[mem=$MEM]" "./get_v16.data.sh ${group}"
done
fi
DEPEND="-w ended(get.data.*)"
;;
esac
Expand All @@ -74,9 +86,8 @@ fi
if [ $RUN_CHGRES == yes ]; then
MEM=2000
QUEUE=dev
MEMBER=hires
WALLT="0:15"
NUM_NODES=1
NUM_NODES=2
case $gfs_ver in
v12 | v13)
export OMP_NUM_THREADS=2
Expand All @@ -86,7 +97,7 @@ if [ $RUN_CHGRES == yes ]; then
export OMP_NUM_THREADS=1
;;
esac
export APRUN="aprun -j 1 -n 12 -N 12 -d ${OMP_NUM_THREADS} -cc depth"
export APRUN="aprun -j 1 -n 24 -N 12 -d ${OMP_NUM_THREADS} -cc depth"
if [ $CRES_HIRES == 'C768' ] ; then
WALLT="0:20"
NUM_NODES=3
Expand All @@ -98,30 +109,58 @@ if [ $RUN_CHGRES == yes ]; then
fi
case $gfs_ver in
v12 | v13)
bsub -e log.${MEMBER} -o log.${MEMBER} -q $QUEUE -P $PROJECT_CODE -J chgres_${MEMBER} -M $MEM -W $WALLT \
-extsched 'CRAYLINUX[]' $DEPEND "export NODES=$NUM_NODES; ./run_pre-v14.chgres.sh ${MEMBER}"
bsub -e log.${CDUMP} -o log.${CDUMP} -q $QUEUE -P $PROJECT_CODE -J chgres_${CDUMP} -M $MEM -W $WALLT \
-extsched 'CRAYLINUX[]' $DEPEND "export NODES=$NUM_NODES; ./run_pre-v14.chgres.sh ${CDUMP}"
;;
v14)
bsub -e log.${MEMBER} -o log.${MEMBER} -q $QUEUE -P $PROJECT_CODE -J chgres_${MEMBER} -M $MEM -W $WALLT \
-extsched 'CRAYLINUX[]' $DEPEND "export NODES=$NUM_NODES; ./run_v14.chgres.sh ${MEMBER}"
bsub -e log.${CDUMP} -o log.${CDUMP} -q $QUEUE -P $PROJECT_CODE -J chgres_${CDUMP} -M $MEM -W $WALLT \
-extsched 'CRAYLINUX[]' $DEPEND "export NODES=$NUM_NODES; ./run_v14.chgres.sh ${CDUMP}"
;;
v15)
bsub -e log.${MEMBER} -o log.${MEMBER} -q $QUEUE -P $PROJECT_CODE -J chgres_${MEMBER} -M $MEM -W $WALLT \
-extsched 'CRAYLINUX[]' $DEPEND "export NODES=$NUM_NODES; ./run_v15.chgres.sh ${MEMBER}"
if [ "$CDUMP" = "gdas" ]; then
bsub -e log.${CDUMP} -o log.${CDUMP} -q $QUEUE -P $PROJECT_CODE -J chgres_${CDUMP} -M $MEM -W $WALLT \
-extsched 'CRAYLINUX[]' $DEPEND "export NODES=$NUM_NODES; ./run_v15.chgres.sh ${CDUMP}"
else
bsub -e log.${CDUMP} -o log.${CDUMP} -q $QUEUE -P $PROJECT_CODE -J chgres_${CDUMP} -M $MEM -W $WALLT \
-extsched 'CRAYLINUX[]' $DEPEND "export NODES=$NUM_NODES; ./run_v15.chgres.gfs.sh"
fi
;;
v16retro)
if [ "$CDUMP" = "gdas" ] ; then
bsub -e log.gdas -o log.gdas -q $QUEUE -P $PROJECT_CODE -J chgres_gdas -M $MEM -W $WALLT \
-extsched 'CRAYLINUX[]' $DEPEND "export NODES=$NUM_NODES; ./run_v16retro.chgres.sh hires"
else
bsub -e log.${CDUMP} -o log.${CDUMP} -q $QUEUE -P $PROJECT_CODE -J chgres_${CDUMP} -M $MEM -W $WALLT \
-extsched 'CRAYLINUX[]' $DEPEND "export NODES=$NUM_NODES; ./run_v16.chgres.sh ${CDUMP}"
fi
;;
v16)
bsub -e log.${CDUMP} -o log.${CDUMP} -q $QUEUE -P $PROJECT_CODE -J chgres_${CDUMP} -M $MEM -W $WALLT \
-extsched 'CRAYLINUX[]' $DEPEND "export NODES=$NUM_NODES; ./run_v16.chgres.sh ${CDUMP}"
;;
esac

WALLT="0:15"
NUM_NODES=1
export APRUN="aprun -j 1 -n 12 -N 12 -d ${OMP_NUM_THREADS} -cc depth"
MEMBER=1
while [ $MEMBER -le 80 ]; do
if [ $MEMBER -lt 10 ]; then
MEMBER_CH="00${MEMBER}"
if [ "$CDUMP" = 'gdas' ]; then

WALLT="0:15"
NUM_NODES=1
export APRUN="aprun -j 1 -n 12 -N 12 -d ${OMP_NUM_THREADS} -cc depth"

if [ "$gfs_ver" = "v16retro" ]; then

bsub -e log.enkf -o log.enkf -q $QUEUE -P $PROJECT_CODE -J chgres_enkf -M $MEM -W $WALLT \
-extsched 'CRAYLINUX[]' $DEPEND "export NODES=$NUM_NODES; ./run_v16retro.chgres.sh enkf"

else
MEMBER_CH="0${MEMBER}"
fi
case $gfs_ver in

MEMBER=1
while [ $MEMBER -le 80 ]; do
if [ $MEMBER -lt 10 ]; then
MEMBER_CH="00${MEMBER}"
else
MEMBER_CH="0${MEMBER}"
fi
case $gfs_ver in
v12 | v13)
bsub -e log.${MEMBER_CH} -o log.${MEMBER_CH} -q $QUEUE -P $PROJECT_CODE -J chgres_${MEMBER_CH} -M $MEM -W $WALLT \
-extsched 'CRAYLINUX[]' $DEPEND "export NODES=$NUM_NODES; ./run_pre-v14.chgres.sh ${MEMBER_CH}"
Expand All @@ -134,7 +173,16 @@ if [ $RUN_CHGRES == yes ]; then
bsub -e log.${MEMBER_CH} -o log.${MEMBER_CH} -q $QUEUE -P $PROJECT_CODE -J chgres_${MEMBER_CH} -M $MEM -W $WALLT \
-extsched 'CRAYLINUX[]' $DEPEND "export NODES=$NUM_NODES; ./run_v15.chgres.sh ${MEMBER_CH}"
;;
esac
MEMBER=$(( $MEMBER + 1 ))
done
v16)
bsub -e log.${MEMBER_CH} -o log.${MEMBER_CH} -q $QUEUE -P $PROJECT_CODE -J chgres_${MEMBER_CH} -M $MEM -W $WALLT \
-extsched 'CRAYLINUX[]' $DEPEND "export NODES=$NUM_NODES; ./run_v16.chgres.sh ${MEMBER_CH}"
;;
esac
MEMBER=$(( $MEMBER + 1 ))
done

fi # is this v16 retro?

fi

fi
Loading

0 comments on commit f09acb9

Please sign in to comment.