Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

A few miscellaneous minor fixes to regression test scripts #2301

Closed
wants to merge 7 commits into from
+39 −68
Closed

A few miscellaneous minor fixes to regression test scripts #2301

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
49857ec
Print 'ECFLOW Tasks Remaining' message only when the number of tasks …
DusanJovic-NOAA May 16, 2024
13139a7
Remove explicit 'set -e'. Use -v command line option for verbose mode
DusanJovic-NOAA May 16, 2024
2ca46a6
Remove 'Computing PET bounds and tasks' message
DusanJovic-NOAA May 16, 2024
5da94e0
Uncomment new_compile=true in rocoto_create_compile_task
DusanJovic-NOAA May 16, 2024
314a340
Fix bugs with log generation script (#40)
BrianCurtis-NOAA May 29, 2024
04cdfeb
Merge remote-tracking branch 'origin/develop' into random_rt_fixes
DusanJovic-NOAA May 29, 2024
31d2e01
Removed unneded colonifnco
DusanJovic-NOAA May 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 16 additions & 31 deletions tests/rt.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@ usage() {
echo " -v verbose output"
echo " -w for weekly_test, skip comparing baseline results"
echo
set -x
}

[[ $# -eq 0 ]] && usage
Expand Down Expand Up @@ -155,11 +154,11 @@ update_rtconf() {

generate_log() {
echo "rt.sh: Generating Regression Testing Log..."
set -x
COMPILE_COUNTER=0
FAILED_COMPILES=()
TEST_COUNTER=0
FAILED_TESTS=()
SKIPPED_TESTS=()
FAILED_TEST_ID=()
FAILED_COMPILE_LOGS=()
FAILED_TEST_LOGS=()
Expand Down Expand Up @@ -255,14 +254,14 @@ EOF
COMPILE_RESULT="FAILED: UNABLE TO START COMPILE"
FAIL_LOG="N/A"
elif [[ -f fail_compile_${COMPILE_ID} ]]; then
COMPILE_RESULT="FAILED: UNABLE TO COMPILE"
COMPILE_RESULT="FAILED: UNABLE TO FINISH COMPILE"
FAIL_LOG="${LOG_DIR}/compile_${COMPILE_ID}.log"
if grep -q "quota" "${LOG_DIR}/compile_${COMPILE_ID}.log"; then
COMPILE_RESULT="FAILED: DISK QUOTA ISSUE"
FAIL_LOG="${LOG_DIR}/compile_${COMPILE_ID}.log"
elif grep -q "timeout" "${LOG_DIR}/compile_${COMPILE_ID}.log"; then
COMPILE_RESULT="FAILED: TEST TIMED OUT"
FAIL_LOG="${LOG_DIR}/compile_${COMPILE_ID}.log"
elif grep -q "TIME LIMIT" "${RUNDIR_ROOT}/compile_${COMPILE_ID}/err"; then
COMPILE_RESULT="FAILED: COMPILE TIMED OUT"
FAIL_LOG="${RUNDIR_ROOT}/compile_${COMPILE_ID}/err"
fi
else
COMPILE_RESULT="PASS"
Expand Down Expand Up @@ -330,21 +329,25 @@ EOF
RT_TEST_TIME=""
RT_TEST_MEM=""
if [[ ${CREATE_BASELINE} == true && ${GEN_BASELINE} != "baseline" ]]; then
TEST_RESULT="SKIPPED (TEST DOES NOT GENERATE BASELINE)"
TEST_RESULT="SKIPPED: TEST DOES NOT GENERATE BASELINE"
SKIPPED_TESTS+=("TEST ${TEST_NAME}_${COMPILER}: ${TEST_RESULT}")
elif [[ ${COMPILE_RESULT} =~ FAILED ]]; then
TEST_RESULT="SKIPPED: ASSOCIATED COMPILE FAILED"
SKIPPED_TESTS+=("TEST ${TEST_NAME}_${COMPILER}: ${TEST_RESULT}")
elif [[ ! -f "${LOG_DIR}/run_${TEST_NAME}_${COMPILER}.log" ]]; then
TEST_RESULT="FAILED: UNABLE TO START RUN"
TEST_RESULT="FAILED: UNABLE TO START TEST"
FAIL_LOG="N/A"
elif [[ -f fail_test_${TEST_NAME}_${COMPILER} ]]; then
if [[ -f "${LOG_DIR}/rt_${TEST_NAME}_${COMPILER}.log" ]]; then
if grep -q "FAIL" "${LOG_DIR}/rt_${TEST_NAME}_${COMPILER}.log"; then
TEST_RESULT="FAILED: UNABLE TO RUN COMPARISON"
TEST_RESULT="FAILED: UNABLE TO COMPLETE COMPARISON"
FAIL_LOG="${LOG_DIR}/run_${TEST_NAME}_${COMPILER}.log"
# We need to catch a "PASS" in rt_*.log even if a fail_test_* files exists
# I am not sure why this can happen.
elif grep -q "PASS" "${LOG_DIR}/rt_${TEST_NAME}_${COMPILER}.log"; then
TEST_RESULT="PASS"
else
TEST_RESULT="FAILED: BASELINE COMPARISON"
TEST_RESULT="FAILED: UNSUCCESSFUL BASELINE COMPARISON"
FAIL_LOG="${LOG_DIR}/rt_${TEST_NAME}_${COMPILER}.log"
fi
else
Expand All @@ -354,9 +357,9 @@ EOF
if grep -q "quota" "${LOG_DIR}/run_${TEST_NAME}_${COMPILER}.log"; then
TEST_RESULT="FAILED: DISK QUOTA ISSUE"
FAIL_LOG="${LOG_DIR}/run_${TEST_NAME}_${COMPILER}.log"
elif grep -q "timeout" "${LOG_DIR}/run_${TEST_NAME}_${COMPILER}.log"; then
elif grep -q "TIME LIMIT" "${RUNDIR_ROOT}/${TEST_NAME}_${COMPILER}/err"; then
TEST_RESULT="FAILED: TEST TIMED OUT"
FAIL_LOG="${LOG_DIR}/run_${TEST_NAME}_${COMPILER}.log"
FAIL_LOG="${RUNDIR_ROOT}/${TEST_NAME}_${COMPILER}/err"
fi
else
TEST_RESULT="PASS"
Expand Down Expand Up @@ -399,7 +402,7 @@ Starting Date/Time: ${TEST_START_TIME}
Ending Date/Time: ${TEST_END_TIME}
Total Time: ${elapsed_time}
Compiles Completed: $((COMPILE_COUNTER-${#FAILED_COMPILES[@]}))/${COMPILE_COUNTER}
Tests Completed: $((TEST_COUNTER-${#FAILED_TESTS[@]}))/${TEST_COUNTER}
Tests Completed: $((TEST_COUNTER-${#FAILED_TESTS[@]}-${#SKIPPED_TESTS[@]}))/${TEST_COUNTER}
EOF
# PRINT FAILED COMPILES
if [[ "${#FAILED_COMPILES[@]}" -ne "0" ]]; then
Expand Down Expand Up @@ -674,14 +677,10 @@ done
#B&N not run together
[[ ${NEW_BASELINES_FILE} != '' && ${RUN_SINGLE_TEST} == true ]] && die "-b and -n options cannot be used at the same time"

[[ -o xtrace ]] && set_x='set -x' || set_x='set +x'

if [[ ${RTVERBOSE} == true ]]; then
set -x
fi

[[ -o xtrace ]] && set_x='set -x' || set_x='set +x'

if [[ -z "${ACCNR}" ]]; then
echo "Please use -a <account> to set group account to use on HPC"
exit 1
Expand All @@ -694,12 +693,10 @@ echo "Account: ${ACCNR}"
case ${MACHINE_ID} in
wcoss2|acorn)
echo "rt.sh: Setting up WCOSS2/Acorn"
set -x
if [[ "${ECFLOW:-false}" == true ]] ; then
module load ecflow/5.6.0.13
fi
module load intel/19.1.3.304 python/3.8.6
export colonifnco=":output" # hack

DISKNM="/lfs/h2/emc/nems/noscrub/emc.nems/RT"
QUEUE="dev"
Expand All @@ -714,7 +711,6 @@ case ${MACHINE_ID} in
;;
gaea)
echo "rt.sh: Setting up gaea..."
set -x
if [[ "${ROCOTO:-false}" == true ]] ; then
module use /ncrc/proj/epic/rocoto/modulefiles
module load rocoto
Expand Down Expand Up @@ -747,7 +743,6 @@ case ${MACHINE_ID} in
;;
hera)
echo "rt.sh: Setting up hera..."
set -x
if [[ "${ROCOTO:-false}" == true ]] ; then
module load rocoto
ROCOTO_SCHEDULER=slurm
Expand All @@ -770,7 +765,6 @@ case ${MACHINE_ID} in
;;
orion)
echo "rt.sh: Setting up orion..."
set -x
module load git/2.28.0
module load gcc/10.2.0
module load python/3.9.2
Expand Down Expand Up @@ -800,7 +794,6 @@ case ${MACHINE_ID} in
;;
hercules)
echo "rt.sh: Setting up hercules..."
set -x
if [[ "${ROCOTO:-false}" == true ]] ; then
module load contrib rocoto
ROCOTO_SCHEDULER="slurm"
Expand Down Expand Up @@ -828,7 +821,6 @@ case ${MACHINE_ID} in
;;
jet)
echo "rt.sh: Setting up jet..."
set -x
CurJetOS=$(lsb_release -is)
echo "=======Running on ${CurJetOS}======="
if [[ ${CurJetOS} == "CentOS" ]]; then
Expand Down Expand Up @@ -861,7 +853,6 @@ case ${MACHINE_ID} in
;;
s4)
echo "rt.sh: Setting up s4..."
set -x
if [[ "${ROCOTO:-false}" == true ]] ; then
module load rocoto/1.3.2
ROCOTO_SCHEDULER=slurm
Expand Down Expand Up @@ -892,7 +883,6 @@ case ${MACHINE_ID} in
;;
derecho)
echo "rt.sh: Setting up derecho..."
set -x
if [[ "${ROCOTO:-false}" == true ]] ; then
module use /glade/work/epicufsrt/contrib/derecho/rocoto/modulefiles
module load rocoto
Expand Down Expand Up @@ -930,7 +920,6 @@ case ${MACHINE_ID} in
;;
stampede)
echo "rt.sh: Setting up stampede..."
set -x
export PYTHONPATH=
if [[ "${ECFLOW:-false}" == true ]] ; then
ECFLOW_START=
Expand All @@ -948,7 +937,6 @@ case ${MACHINE_ID} in
;;
expanse)
echo "rt.sh: Setting up expanse..."
set -x
export PYTHONPATH=

if [[ "${ECFLOW:-false}" == true ]] ; then
Expand All @@ -965,7 +953,6 @@ case ${MACHINE_ID} in
;;
noaacloud)
echo "rt.sh: Setting up noaacloud..."
set -x
export PATH="/contrib/EPIC/bin:${PATH}"
module use /apps/modules/modulefiles

Expand All @@ -987,7 +974,6 @@ case ${MACHINE_ID} in
die "Unknown machine ID, please edit detect_machine.sh file"
;;
esac
eval "${set_x}"

mkdir -p "${STMP}/${USER}"

Expand Down Expand Up @@ -1374,5 +1360,4 @@ fi

## Lets verify all tests were run and that they passed
generate_log
eval "${set_x}"
echo "******Regression Testing Script Completed******"
Loading
Loading