diff --git a/qa/L0_backend_python/argument_validation/test.sh b/qa/L0_backend_python/argument_validation/test.sh index f47abb8485..31edaf12ea 100755 --- a/qa/L0_backend_python/argument_validation/test.sh +++ b/qa/L0_backend_python/argument_validation/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -26,14 +26,14 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. CLIENT_PY=../python_unittest.py -CLIENT_LOG="./client.log" +CLIENT_LOG="./arg_validation_client.log" EXPECTED_NUM_TESTS="1" TEST_RESULT_FILE='test_results.txt' TRITON_DIR=${TRITON_DIR:="/opt/tritonserver"} SERVER=${TRITON_DIR}/bin/tritonserver BACKEND_DIR=${TRITON_DIR}/backends SERVER_ARGS="--model-repository=`pwd`/models --backend-directory=${BACKEND_DIR} --log-verbose=1" -SERVER_LOG="./inference_server.log" +SERVER_LOG="./arg_validation_server.log" RET=0 source ../../common/util.sh @@ -75,4 +75,6 @@ else echo -e "\n***\n*** Argument validation test PASSED. \n***" fi +collect_artifacts_from_subdir + exit $RET diff --git a/qa/L0_backend_python/bls/test.sh b/qa/L0_backend_python/bls/test.sh index 3d87cf7b65..e6ecc78166 100755 --- a/qa/L0_backend_python/bls/test.sh +++ b/qa/L0_backend_python/bls/test.sh @@ -26,7 +26,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. CLIENT_PY=../python_unittest.py -CLIENT_LOG="./client.log" +CLIENT_LOG="./bls_client.log" EXPECTED_NUM_TESTS="1" TEST_RESULT_FILE='test_results.txt' source ../../common/util.sh @@ -35,9 +35,10 @@ TRITON_DIR=${TRITON_DIR:="/opt/tritonserver"} SERVER=${TRITON_DIR}/bin/tritonserver BACKEND_DIR=${TRITON_DIR}/backends SERVER_ARGS="--model-repository=`pwd`/models --backend-directory=${BACKEND_DIR} --log-verbose=1" -SERVER_LOG="./inference_server.log" RET=0 +# This variable is used to print out the correct server log for each sub-test. +SUB_TEST_RET=0 rm -fr *.log ./models *.txt pip3 uninstall -y torch @@ -104,6 +105,7 @@ cp -r ${DATADIR}/qa_model_repository/libtorch_nobatch_float32_float32_float32/ . for TRIAL in non_decoupled decoupled ; do export BLS_KIND=$TRIAL + SERVER_LOG="./bls_$TRIAL.inference_server.log" run_server if [ "$SERVER_PID" == "0" ]; then @@ -120,12 +122,14 @@ for TRIAL in non_decoupled decoupled ; do echo -e "\n***\n*** 'bls' $BLS_KIND test FAILED. \n***" cat $CLIENT_LOG RET=1 + SUB_TEST_RET=1 else check_test_results $TEST_RESULT_FILE $EXPECTED_NUM_TESTS if [ $? -ne 0 ]; then cat $CLIENT_LOG echo -e "\n***\n*** Test Result Verification Failed\n***" RET=1 + SUB_TEST_RET=1 fi fi @@ -135,12 +139,14 @@ for TRIAL in non_decoupled decoupled ; do echo -e "\n***\n*** 'bls_memory' $BLS_KIND test FAILED. \n***" cat $CLIENT_LOG RET=1 + SUB_TEST_RET=1 else check_test_results $TEST_RESULT_FILE $EXPECTED_NUM_TESTS if [ $? -ne 0 ]; then cat $CLIENT_LOG echo -e "\n***\n*** Test Result Verification Failed\n***" RET=1 + SUB_TEST_RET=1 fi fi @@ -150,12 +156,14 @@ for TRIAL in non_decoupled decoupled ; do echo -e "\n***\n*** 'bls_async_memory' $BLS_KIND test FAILED. \n***" cat $CLIENT_LOG RET=1 + SUB_TEST_RET=1 else check_test_results $TEST_RESULT_FILE $EXPECTED_NUM_TESTS if [ $? -ne 0 ]; then cat $CLIENT_LOG echo -e "\n***\n*** Test Result Verification Failed\n***" RET=1 + SUB_TEST_RET=1 fi fi @@ -165,12 +173,14 @@ for TRIAL in non_decoupled decoupled ; do echo -e "\n***\n*** 'bls_async' $BLS_KIND test FAILED. \n***" cat $CLIENT_LOG RET=1 + SUB_TEST_RET=1 else check_test_results $TEST_RESULT_FILE $EXPECTED_NUM_TESTS if [ $? -ne 0 ]; then cat $CLIENT_LOG echo -e "\n***\n*** Test Result Verification Failed\n***" RET=1 + SUB_TEST_RET=1 fi fi @@ -178,6 +188,11 @@ for TRIAL in non_decoupled decoupled ; do kill $SERVER_PID wait $SERVER_PID + + if [ $SUB_TEST_RET -eq 1 ]; then + cat $CLIENT_LOG + cat $SERVER_LOG + fi done # Test error handling when BLS is used in "initialize" or "finalize" function @@ -188,11 +203,13 @@ mkdir -p models/bls_init_error/1/ cp ../../python_models/bls_init_error/model.py models/bls_init_error/1/ cp ../../python_models/bls_init_error/config.pbtxt models/bls_init_error SERVER_LOG="./bls_init_error_server.log" +SUB_TEST_RET=0 run_server if [ "$SERVER_PID" != "0" ]; then echo -e "*** FAILED: unexpected success starting $SERVER" >> $CLIENT_LOG RET=1 + SUB_TEST_RET=1 kill $SERVER_PID wait $SERVER_PID else @@ -201,14 +218,21 @@ else else echo -e "Not found \"$ERROR_MESSAGE\"" >> $CLIENT_LOG RET=1 + SUB_TEST_RET=1 fi fi +if [ $SUB_TEST_RET -eq 1 ]; then + cat $CLIENT_LOG + cat $SERVER_LOG +fi + rm -fr ./models mkdir -p models/bls_finalize_error/1/ cp ../../python_models/bls_finalize_error/model.py models/bls_finalize_error/1/ cp ../../python_models/bls_finalize_error/config.pbtxt models/bls_finalize_error/ SERVER_LOG="./bls_finalize_error_server.log" +SUB_TEST_RET=0 run_server if [ "$SERVER_PID" == "0" ]; then @@ -225,9 +249,16 @@ if grep "$ERROR_MESSAGE" $SERVER_LOG; then else echo -e "Not found \"$ERROR_MESSAGE\"" >> $CLIENT_LOG RET=1 + SUB_TEST_RET=1 +fi + +if [ $SUB_TEST_RET -eq 1 ]; then + cat $CLIENT_LOG + cat $SERVER_LOG fi # Test model loading API with BLS +SUB_TEST_RET=0 rm -fr ./models mkdir -p models/bls_model_loading/1/ cp ../../python_models/bls_model_loading/model.py models/bls_model_loading/1/ @@ -254,6 +285,7 @@ set -e if [ "$code" == "400" ]; then echo -e "\n***\n*** Failed to load model '${MODEL_NAME}'\n***" RET=1 + SUB_TEST_RET=1 fi set +e @@ -263,12 +295,14 @@ if [ $? -ne 0 ]; then echo -e "\n***\n*** 'bls_model_loading' test FAILED. \n***" cat $CLIENT_LOG RET=1 + SUB_TEST_RET=1 else check_test_results $TEST_RESULT_FILE $EXPECTED_NUM_TESTS if [ $? -ne 0 ]; then cat $CLIENT_LOG echo -e "\n***\n*** Test Result Verification Failed\n***" RET=1 + SUB_TEST_RET=1 fi fi @@ -277,6 +311,11 @@ set -e kill $SERVER_PID wait $SERVER_PID +if [ $SUB_TEST_RET -eq 1 ]; then + cat $CLIENT_LOG + cat $SERVER_LOG +fi + # Test model loading API with BLS warmup (cd models/bls_model_loading && \ echo "model_warmup [{" >> config.pbtxt && \ @@ -300,6 +339,7 @@ wait $SERVER_PID echo " }" >> config.pbtxt && \ echo "}]" >> config.pbtxt ) +SUB_TEST_RET=0 SERVER_LOG="./bls_model_loading_server_warmup.log" run_server if [ "$SERVER_PID" == "0" ]; then @@ -314,17 +354,23 @@ set -e if [ "$code" == "400" ]; then echo -e "\n***\n*** Failed to load model '${MODEL_NAME}'\n***" RET=1 + SUB_TEST_RET=1 fi kill $SERVER_PID wait $SERVER_PID -if [ $RET -eq 1 ]; then +if [ $SUB_TEST_RET -eq 1 ]; then cat $CLIENT_LOG cat $SERVER_LOG +fi + +if [ $RET -eq 1 ]; then echo -e "\n***\n*** BLS test FAILED. \n***" else echo -e "\n***\n*** BLS test PASSED. \n***" fi +collect_artifacts_from_subdir + exit $RET diff --git a/qa/L0_backend_python/custom_metrics/test.sh b/qa/L0_backend_python/custom_metrics/test.sh index 149f5e5d56..00200aff27 100755 --- a/qa/L0_backend_python/custom_metrics/test.sh +++ b/qa/L0_backend_python/custom_metrics/test.sh @@ -26,7 +26,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. CLIENT_PY=../python_unittest.py -CLIENT_LOG="./client.log" +CLIENT_LOG="./custom_metrics_client.log" EXPECTED_NUM_TESTS="1" TEST_RESULT_FILE='test_results.txt' source ../../common/util.sh @@ -35,7 +35,7 @@ TRITON_DIR=${TRITON_DIR:="/opt/tritonserver"} SERVER=${TRITON_DIR}/bin/tritonserver BACKEND_DIR=${TRITON_DIR}/backends SERVER_ARGS="--model-repository=`pwd`/models --backend-directory=${BACKEND_DIR} --log-verbose=1" -SERVER_LOG="./inference_server.log" +SERVER_LOG="./custom_metrics_server.log" RET=0 rm -fr *.log ./models *.txt @@ -82,4 +82,6 @@ else echo -e "\n***\n*** Custom Metrics test PASSED. \n***" fi +collect_artifacts_from_subdir + exit $RET diff --git a/qa/L0_backend_python/decoupled/test.sh b/qa/L0_backend_python/decoupled/test.sh index 0e316c8452..b4fa4ffe75 100755 --- a/qa/L0_backend_python/decoupled/test.sh +++ b/qa/L0_backend_python/decoupled/test.sh @@ -26,14 +26,14 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. CLIENT_PY=./decoupled_test.py -CLIENT_LOG="./client.log" +CLIENT_LOG="./decoupled_client.log" EXPECTED_NUM_TESTS="5" TEST_RESULT_FILE='test_results.txt' TRITON_DIR=${TRITON_DIR:="/opt/tritonserver"} SERVER=${TRITON_DIR}/bin/tritonserver BACKEND_DIR=${TRITON_DIR}/backends SERVER_ARGS="--model-repository=`pwd`/models --backend-directory=${BACKEND_DIR} --log-verbose=1" -SERVER_LOG="./inference_server.log" +SERVER_LOG="./decoupled_server.log" pip3 uninstall -y torch pip3 install torch==1.13.0+cu117 -f https://download.pytorch.org/whl/torch_stable.html @@ -118,4 +118,6 @@ else echo -e "\n***\n*** Decoupled test PASSED. \n***" fi +collect_artifacts_from_subdir + exit $RET diff --git a/qa/L0_backend_python/ensemble/test.sh b/qa/L0_backend_python/ensemble/test.sh index cd1018733b..961ff16e5b 100755 --- a/qa/L0_backend_python/ensemble/test.sh +++ b/qa/L0_backend_python/ensemble/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -26,7 +26,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. CLIENT_PY=./lifecycle_test.py -CLIENT_LOG="./client.log" +CLIENT_LOG="./ensemble_client.log" EXPECTED_NUM_TESTS="1" TEST_RESULT_FILE='test_results.txt' source ../common.sh @@ -36,7 +36,7 @@ TRITON_DIR=${TRITON_DIR:="/opt/tritonserver"} SERVER=${TRITON_DIR}/bin/tritonserver BACKEND_DIR=${TRITON_DIR}/backends SERVER_ARGS="--model-repository=`pwd`/models --backend-directory=${BACKEND_DIR} --log-verbose=1" -SERVER_LOG="./inference_server.log" +SERVER_LOG="./ensemble_server.log" RET=0 rm -rf models/ $CLIENT_LOG @@ -114,4 +114,6 @@ else echo -e "\n***\n*** Ensemble test PASSED. \n***" fi +collect_artifacts_from_subdir + exit $RET diff --git a/qa/L0_backend_python/env/test.sh b/qa/L0_backend_python/env/test.sh index aeba2691cf..d5bbe8e95f 100755 --- a/qa/L0_backend_python/env/test.sh +++ b/qa/L0_backend_python/env/test.sh @@ -25,7 +25,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -CLIENT_LOG="./client.log" +CLIENT_LOG="./env_client.log" source ../common.sh source ../../common/util.sh @@ -33,7 +33,7 @@ SERVER=/opt/tritonserver/bin/tritonserver BASE_SERVER_ARGS="--model-repository=`pwd`/models --log-verbose=1 --disable-auto-complete-config" PYTHON_BACKEND_BRANCH=$PYTHON_BACKEND_REPO_TAG SERVER_ARGS=$BASE_SERVER_ARGS -SERVER_LOG="./inference_server.log" +SERVER_LOG="./env_server.log" RET=0 @@ -315,4 +315,6 @@ else echo -e "\n***\n*** Env Manager Test FAILED.\n***" fi +collect_artifacts_from_subdir + exit $RET diff --git a/qa/L0_backend_python/examples/test.sh b/qa/L0_backend_python/examples/test.sh index 19acd4f127..e7c3fe45ac 100755 --- a/qa/L0_backend_python/examples/test.sh +++ b/qa/L0_backend_python/examples/test.sh @@ -32,7 +32,7 @@ TRITON_DIR=${TRITON_DIR:="/opt/tritonserver"} SERVER=${TRITON_DIR}/bin/tritonserver BACKEND_DIR=${TRITON_DIR}/backends SERVER_ARGS="--model-repository=`pwd`/python_backend/models --backend-directory=${BACKEND_DIR} --log-verbose=1" -SERVER_LOG="./inference_server.log" +SERVER_LOG="./examples_server.log" RET=0 rm -fr *.log python_backend/ @@ -56,7 +56,7 @@ git clone https://github.com/triton-inference-server/python_backend -b $PYTHON_B cd python_backend # Example 1 -CLIENT_LOG="./add_sub_client.log" +CLIENT_LOG="./examples_add_sub_client.log" mkdir -p models/add_sub/1/ cp examples/add_sub/model.py models/add_sub/1/model.py cp examples/add_sub/config.pbtxt models/add_sub/config.pbtxt @@ -86,7 +86,7 @@ kill $SERVER_PID wait $SERVER_PID # Example 2 -CLIENT_LOG="./pytorch_client.log" +CLIENT_LOG="./examples_pytorch_client.log" mkdir -p models/pytorch/1/ cp examples/pytorch/model.py models/pytorch/1/model.py cp examples/pytorch/config.pbtxt models/pytorch/config.pbtxt @@ -120,7 +120,7 @@ wait $SERVER_PID # JAX AddSub # JAX is not supported on Jetson if [ "$TEST_JETSON" == "0" ]; then - CLIENT_LOG="./jax_client.log" + CLIENT_LOG="./examples_jax_client.log" mkdir -p models/jax/1/ cp examples/jax/model.py models/jax/1/model.py cp examples/jax/config.pbtxt models/jax/config.pbtxt @@ -153,7 +153,7 @@ fi # Example 4 # BLS Sync -CLIENT_LOG="./sync_client.log" +CLIENT_LOG="./examples_sync_client.log" mkdir -p models/bls_sync/1 cp examples/bls/sync_model.py models/bls_sync/1/model.py cp examples/bls/sync_config.pbtxt models/bls_sync/config.pbtxt @@ -185,7 +185,7 @@ wait $SERVER_PID # Example 5 # Decoupled Repeat -CLIENT_LOG="./repeat_client.log" +CLIENT_LOG="./examples_repeat_client.log" mkdir -p models/repeat_int32/1/ cp examples/decoupled/repeat_model.py models/repeat_int32/1/model.py cp examples/decoupled/repeat_config.pbtxt models/repeat_int32/config.pbtxt @@ -217,7 +217,7 @@ wait $SERVER_PID # Example 6 # Decoupled Square -CLIENT_LOG="./square_client.log" +CLIENT_LOG="./examples_square_client.log" mkdir -p models/square_int32/1/ cp examples/decoupled/square_model.py models/square_int32/1/model.py cp examples/decoupled/square_config.pbtxt models/square_int32/config.pbtxt @@ -253,7 +253,7 @@ wait $SERVER_PID # Having multiple python versions lead to build issues. # Anaconda is not officially supported on Jetson. if [ "$TEST_JETSON" == "0" ]; then - CLIENT_LOG="./async_client.log" + CLIENT_LOG="./examples_async_client.log" mkdir -p models/bls_async/1 cp examples/bls/async_model.py models/bls_async/1/model.py cp examples/bls/async_config.pbtxt models/bls_async/config.pbtxt @@ -285,7 +285,7 @@ if [ "$TEST_JETSON" == "0" ]; then fi # Auto Complete Model Configuration Example -CLIENT_LOG="./auto_complete_client.log" +CLIENT_LOG="./examples_auto_complete_client.log" mkdir -p models/nobatch_auto_complete/1/ mkdir -p models/batch_auto_complete/1/ cp examples/auto_complete/nobatch_model.py models/nobatch_auto_complete/1/model.py @@ -319,7 +319,7 @@ kill $SERVER_PID wait $SERVER_PID # BLS Decoupled Sync -CLIENT_LOG="./bls_decoupled_sync_client.log" +CLIENT_LOG="./examples_bls_decoupled_sync_client.log" mkdir -p models/bls_decoupled_sync/1 cp examples/bls_decoupled/sync_model.py models/bls_decoupled_sync/1/model.py cp examples/bls_decoupled/sync_config.pbtxt models/bls_decoupled_sync/config.pbtxt @@ -350,7 +350,7 @@ wait $SERVER_PID # BLS Decoupled Async if [ "$TEST_JETSON" == "0" ]; then - CLIENT_LOG="./bls_decoupled_async_client.log" + CLIENT_LOG="./examples_bls_decoupled_async_client.log" mkdir -p models/bls_decoupled_async/1 cp examples/bls_decoupled/async_model.py models/bls_decoupled_async/1/model.py cp examples/bls_decoupled/async_config.pbtxt models/bls_decoupled_async/config.pbtxt @@ -384,7 +384,7 @@ fi # Example 7 # Model Instance Kind -CLIENT_LOG="./model_instance_kind.log" +CLIENT_LOG="./examples_model_instance_kind.log" mkdir -p models/resnet50/1 cp examples/instance_kind/model.py models/resnet50/1/ cp examples/instance_kind/config.pbtxt models/resnet50/ @@ -414,7 +414,7 @@ kill $SERVER_PID wait $SERVER_PID # Custom Metrics -CLIENT_LOG="./custom_metrics_client.log" +CLIENT_LOG="./examples_custom_metrics_client.log" mkdir -p models/custom_metrics/1 cp examples/custom_metrics/model.py models/custom_metrics/1/model.py cp examples/custom_metrics/config.pbtxt models/custom_metrics/config.pbtxt @@ -450,4 +450,6 @@ else echo -e "\n***\n*** Example verification test FAILED.\n***" fi +collect_artifacts_from_subdir + exit $RET diff --git a/qa/L0_backend_python/io/test.sh b/qa/L0_backend_python/io/test.sh index 9096038b94..e64ec18eb4 100755 --- a/qa/L0_backend_python/io/test.sh +++ b/qa/L0_backend_python/io/test.sh @@ -26,7 +26,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. UNITTEST_PY=./io_test.py -CLIENT_LOG="./client.log" +CLIENT_LOG="./io_client.log" EXPECTED_NUM_TESTS="1" TEST_RESULT_FILE='test_results.txt' source ../common.sh @@ -37,7 +37,7 @@ SERVER=${TRITON_DIR}/bin/tritonserver BACKEND_DIR=${TRITON_DIR}/backends SERVER_ARGS="--model-repository=`pwd`/models --backend-directory=${BACKEND_DIR} --log-verbose=1" -SERVER_LOG="./inference_server.log" +SERVER_LOG="./io_server.log" RET=0 rm -fr *.log ./models @@ -172,4 +172,6 @@ else echo -e "\n***\n*** IO test FAILED.\n***" fi +collect_artifacts_from_subdir + exit $RET diff --git a/qa/L0_backend_python/lifecycle/test.sh b/qa/L0_backend_python/lifecycle/test.sh index c1ab6baf92..80646e2b78 100755 --- a/qa/L0_backend_python/lifecycle/test.sh +++ b/qa/L0_backend_python/lifecycle/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -25,7 +25,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -CLIENT_LOG="./client.log" +CLIENT_LOG="./lifecycle_client.log" EXPECTED_NUM_TESTS="3" TEST_RESULT_FILE='test_results.txt' source ../common.sh @@ -35,7 +35,7 @@ TRITON_DIR=${TRITON_DIR:="/opt/tritonserver"} SERVER=${TRITON_DIR}/bin/tritonserver BACKEND_DIR=${TRITON_DIR}/backends SERVER_ARGS="--model-repository=`pwd`/models --backend-directory=${BACKEND_DIR} --log-verbose=1" -SERVER_LOG="./inference_server.log" +SERVER_LOG="./lifecycle_server.log" RET=0 rm -fr *.log ./models @@ -205,4 +205,6 @@ else echo -e "\n***\n*** Lifecycle test PASSED. \n***" fi +collect_artifacts_from_subdir + exit $RET diff --git a/qa/L0_backend_python/logging/test.sh b/qa/L0_backend_python/logging/test.sh index 369d28d0b9..9ccd42cc0f 100755 --- a/qa/L0_backend_python/logging/test.sh +++ b/qa/L0_backend_python/logging/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -25,10 +25,10 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -CLIENT_LOG="client.log" +CLIENT_LOG="logging_client.log" TEST_RESULT_FILE="test_results.txt" LOG_TEST="logging_test.py" -SERVER_LOG="./server.log" +SERVER_LOG="./logging_server.log" REPO_VERSION=${NVIDIA_TRITON_SERVER_VERSION} if [ "$#" -ge 1 ]; then @@ -135,8 +135,6 @@ verify_log_counts 4 0 rm -f *.log #Run Server Enabling Verbose Messages -SERVER_ARGS="--model-repository=$MODELSDIR --backend-directory=${BACKEND_DIR}" -SERVER_LOG="./server.log" run_server if [ "$SERVER_PID" == "0" ]; then echo -e "\n***\n*** Failed to start $SERVER\n***" @@ -179,8 +177,6 @@ verify_log_counts 4 3 rm -f *.log #Run Server Enabling Verbose Messages -SERVER_ARGS="--model-repository=$MODELSDIR --backend-directory=${BACKEND_DIR}" -SERVER_LOG="./server.log" run_server if [ "$SERVER_PID" == "0" ]; then echo -e "\n***\n*** Failed to start $SERVER\n***" @@ -232,5 +228,6 @@ else echo -e "\n***\n*** Logging test FAILED. \n***" fi +collect_artifacts_from_subdir exit $RET diff --git a/qa/L0_backend_python/model_control/test.sh b/qa/L0_backend_python/model_control/test.sh index 63fabd8bd2..13ff09784c 100755 --- a/qa/L0_backend_python/model_control/test.sh +++ b/qa/L0_backend_python/model_control/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -25,14 +25,14 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -CLIENT_LOG="./client.log" +CLIENT_LOG="./model_control_client.log" EXPECTED_NUM_TESTS="1" TEST_RESULT_FILE='test_results.txt' TRITON_DIR=${TRITON_DIR:="/opt/tritonserver"} SERVER=${TRITON_DIR}/bin/tritonserver BACKEND_DIR=${TRITON_DIR}/backends SERVER_ARGS="--model-repository=`pwd`/models --model-control-mode=explicit --backend-directory=${BACKEND_DIR} --log-verbose=1" -SERVER_LOG="./inference_server.log" +SERVER_LOG="./model_control_server.log" RET=0 rm -fr *.log ./models @@ -77,3 +77,7 @@ if [ $RET -eq 1 ]; then else echo -e "\n***\n*** model_control_test PASSED. \n***" fi + +collect_artifacts_from_subdir + +exit $RET diff --git a/qa/L0_backend_python/restart/test.sh b/qa/L0_backend_python/restart/test.sh index 64c80332ac..0b08c496bc 100755 --- a/qa/L0_backend_python/restart/test.sh +++ b/qa/L0_backend_python/restart/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -25,13 +25,13 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -CLIENT_LOG="./client.log" +CLIENT_LOG="./restart_client.log" EXPECTED_NUM_TESTS="7" TRITON_DIR=${TRITON_DIR:="/opt/tritonserver"} SERVER=${TRITON_DIR}/bin/tritonserver BACKEND_DIR=${TRITON_DIR}/backends SERVER_ARGS="--model-repository=`pwd`/models --backend-directory=${BACKEND_DIR} --log-verbose=1" -SERVER_LOG="./inference_server.log" +SERVER_LOG="./restart_server.log" source ../../common/util.sh source ../common.sh @@ -126,5 +126,6 @@ else echo -e "\n***\n*** Restart test PASSED. \n***" fi -exit $RET +collect_artifacts_from_subdir +exit $RET diff --git a/qa/common/util.sh b/qa/common/util.sh index 9dfdc7a389..43cd757890 100755 --- a/qa/common/util.sh +++ b/qa/common/util.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2018-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2018-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -467,3 +467,9 @@ function kill_servers () { wait ${!server_pid[$i]} done } + +# Collect all logs and core dumps and copy them to an upper-level directory for +# proper capture on the CI. +function collect_artifacts_from_subdir () { + cp *.*log* core* ../ || true +}