From a5971f0e5f479a329ce9974f8ebeb051f2efa010 Mon Sep 17 00:00:00 2001 From: George Wu Date: Fri, 4 Nov 2022 10:48:17 -0700 Subject: [PATCH 01/49] test TRT 8.5 GA --- .../linux-gpu-tensorrt-ci-pipeline.yml | 6 +- ...kerfile.manylinux2014_cuda11_6_tensorrt8_5 | 184 ++++++++++++++++++ 2 files changed, 187 insertions(+), 3 deletions(-) create mode 100644 tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cuda11_6_tensorrt8_5 diff --git a/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml index 77a541f05c824..11c5a92c24c7d 100644 --- a/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml @@ -22,10 +22,10 @@ jobs: - template: templates/get-docker-image-steps.yml parameters: - Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cuda11_6_tensorrt8_4 + Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cuda11_6_tensorrt8_5 Context: tools/ci_build/github/linux/docker DockerBuildArgs: "--network=host --build-arg POLICY=manylinux2014 --build-arg PLATFORM=x86_64 --build-arg DEVTOOLSET_ROOTPATH=/opt/rh/devtoolset-11/root --build-arg PREPEND_PATH=/opt/rh/devtoolset-11/root/usr/bin: --build-arg LD_LIBRARY_PATH_ARG=/opt/rh/devtoolset-11/root/usr/lib64:/opt/rh/devtoolset-11/root/usr/lib:/opt/rh/devtoolset-11/root/usr/lib64/dyninst:/opt/rh/devtoolset-11/root/usr/lib/dyninst:/usr/local/lib64 --build-arg BUILD_UID=$( id -u )" - Repository: onnxruntimetensorrt84gpubuild + Repository: onnxruntimetensorrt85gpubuild - task: CmdLine@2 inputs: @@ -39,7 +39,7 @@ jobs: -e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \ -e NIGHTLY_BUILD \ -e BUILD_BUILDNUMBER \ - onnxruntimetensorrt84gpubuild \ + onnxruntimetensorrt85gpubuild \ /opt/python/cp38-cp38/bin/python3 /onnxruntime_src/tools/ci_build/build.py \ --build_dir /build --cmake_generator Ninja \ --config Release --update --build \ diff --git a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cuda11_6_tensorrt8_5 b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cuda11_6_tensorrt8_5 new file mode 100644 index 0000000000000..5deedc71d0a4d --- /dev/null +++ b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cuda11_6_tensorrt8_5 @@ -0,0 +1,184 @@ +ARG BASEIMAGE=nvidia/cuda:11.6.1-cudnn8-devel-centos7 +ARG POLICY=manylinux2014 +ARG PLATFORM=x86_64 +ARG DEVTOOLSET_ROOTPATH= +ARG LD_LIBRARY_PATH_ARG= +ARG PREPEND_PATH= + +#We need CUDA, TensorRT and manylinux. But the CUDA Toolkit End User License Agreement says NVIDIA CUDA Driver Libraries(libcuda.so, libnvidia-ptxjitcompiler.so) are only distributable in applications that meet this criteria: +#1. The application was developed starting from a NVIDIA CUDA container obtained from Docker Hub or the NVIDIA GPU Cloud, and +#2. The resulting application is packaged as a Docker container and distributed to users on Docker Hub or the NVIDIA GPU Cloud only. +#So we use CUDA as the base image then add manylinux and TensorRT on top of it. + +#Build manylinux2014 docker image begin +FROM $BASEIMAGE AS runtime_base +ARG POLICY +ARG PLATFORM +ARG DEVTOOLSET_ROOTPATH +ARG LD_LIBRARY_PATH_ARG +ARG PREPEND_PATH +LABEL maintainer="The ManyLinux project" + +ENV AUDITWHEEL_POLICY=${POLICY} AUDITWHEEL_ARCH=${PLATFORM} AUDITWHEEL_PLAT=${POLICY}_${PLATFORM} +ENV LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8 LANGUAGE=en_US.UTF-8 +ENV DEVTOOLSET_ROOTPATH=${DEVTOOLSET_ROOTPATH} +ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH_ARG} +ENV PATH=${PREPEND_PATH}${PATH} +ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig + +# first copy the fixup mirrors script, keep the script around +COPY build_scripts/fixup-mirrors.sh /usr/local/sbin/fixup-mirrors + +# setup entrypoint, this will wrap commands with `linux32` with i686 images +COPY build_scripts/install-entrypoint.sh \ + build_scripts/update-system-packages.sh \ + build_scripts/build_utils.sh \ + /build_scripts/ + +RUN /build_scripts/install-entrypoint.sh && rm -rf /build_scripts +COPY manylinux-entrypoint /usr/local/bin/manylinux-entrypoint +ENTRYPOINT ["manylinux-entrypoint"] + +COPY build_scripts/install-runtime-packages.sh \ + build_scripts/update-system-packages.sh \ + build_scripts/build_utils.sh \ + /build_scripts/ +RUN manylinux-entrypoint /build_scripts/install-runtime-packages.sh && rm -rf /build_scripts/ + +COPY build_scripts/build_utils.sh /build_scripts/ + +COPY build_scripts/install-autoconf.sh /build_scripts/ +RUN export AUTOCONF_ROOT=autoconf-2.71 && \ + export AUTOCONF_HASH=431075ad0bf529ef13cb41e9042c542381103e80015686222b8a9d4abef42a1c && \ + export AUTOCONF_DOWNLOAD_URL=http://ftp.gnu.org/gnu/autoconf && \ + manylinux-entrypoint /build_scripts/install-autoconf.sh + +COPY build_scripts/install-automake.sh /build_scripts/ +RUN export AUTOMAKE_ROOT=automake-1.16.5 && \ + export AUTOMAKE_HASH=07bd24ad08a64bc17250ce09ec56e921d6343903943e99ccf63bbf0705e34605 && \ + export AUTOMAKE_DOWNLOAD_URL=http://ftp.gnu.org/gnu/automake && \ + manylinux-entrypoint /build_scripts/install-automake.sh + +COPY build_scripts/install-libtool.sh /build_scripts/ +RUN export LIBTOOL_ROOT=libtool-2.4.7 && \ + export LIBTOOL_HASH=04e96c2404ea70c590c546eba4202a4e12722c640016c12b9b2f1ce3d481e9a8 && \ + export LIBTOOL_DOWNLOAD_URL=http://ftp.gnu.org/gnu/libtool && \ + manylinux-entrypoint /build_scripts/install-libtool.sh + +COPY build_scripts/install-libxcrypt.sh /build_scripts/ +RUN export LIBXCRYPT_VERSION=4.4.28 && \ + export LIBXCRYPT_HASH=db7e37901969cb1d1e8020cb73a991ef81e48e31ea5b76a101862c806426b457 && \ + export LIBXCRYPT_DOWNLOAD_URL=https://github.com/besser82/libxcrypt/archive && \ + export PERL_ROOT=perl-5.34.0 && \ + export PERL_HASH=551efc818b968b05216024fb0b727ef2ad4c100f8cb6b43fab615fa78ae5be9a && \ + export PERL_DOWNLOAD_URL=https://www.cpan.org/src/5.0 && \ + manylinux-entrypoint /build_scripts/install-libxcrypt.sh + +FROM runtime_base AS build_base +COPY build_scripts/install-build-packages.sh /build_scripts/ +RUN manylinux-entrypoint /build_scripts/install-build-packages.sh + + +FROM build_base AS build_git +COPY build_scripts/build-git.sh /build_scripts/ +RUN export GIT_ROOT=git-2.36.2 && \ + export GIT_HASH=6dc2cdea5fb23d823ba4871cc23222c1db31dfbb6d6c6ff74c4128700df57c68 && \ + export GIT_DOWNLOAD_URL=https://www.kernel.org/pub/software/scm/git && \ + manylinux-entrypoint /build_scripts/build-git.sh + + +FROM build_base AS build_cpython +COPY build_scripts/build-sqlite3.sh /build_scripts/ +RUN export SQLITE_AUTOCONF_ROOT=sqlite-autoconf-3390200 && \ + export SQLITE_AUTOCONF_HASH=852be8a6183a17ba47cee0bbff7400b7aa5affd283bf3beefc34fcd088a239de && \ + export SQLITE_AUTOCONF_DOWNLOAD_URL=https://www.sqlite.org/2022 && \ + manylinux-entrypoint /build_scripts/build-sqlite3.sh + +COPY build_scripts/build-openssl.sh /build_scripts/ +RUN export OPENSSL_ROOT=openssl-1.1.1q && \ + export OPENSSL_HASH=d7939ce614029cdff0b6c20f0e2e5703158a489a72b2507b8bd51bf8c8fd10ca && \ + export OPENSSL_DOWNLOAD_URL=https://www.openssl.org/source && \ + manylinux-entrypoint /build_scripts/build-openssl.sh + +COPY build_scripts/build-cpython.sh /build_scripts/ + + +FROM build_cpython AS build_cpython37 +COPY build_scripts/cpython-pubkeys.txt /build_scripts/cpython-pubkeys.txt +RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.7.13 + + +FROM build_cpython AS build_cpython38 +COPY build_scripts/ambv-pubkey.txt /build_scripts/cpython-pubkeys.txt +RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.8.13 + + +FROM build_cpython AS build_cpython39 +COPY build_scripts/ambv-pubkey.txt /build_scripts/cpython-pubkeys.txt +RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.9.13 + + +FROM build_cpython AS build_cpython310 +COPY build_scripts/cpython-pubkey-310-311.txt /build_scripts/cpython-pubkeys.txt +RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.10.5 + +FROM build_cpython AS build_cpython311 +COPY build_scripts/cpython-pubkey-310-311.txt /build_scripts/cpython-pubkeys.txt +RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.11.0b5 + +FROM build_cpython AS all_python +COPY build_scripts/install-pypy.sh \ + build_scripts/pypy.sha256 \ + build_scripts/finalize-python.sh \ + /build_scripts/ +RUN manylinux-entrypoint /build_scripts/install-pypy.sh 3.7 7.3.9 +RUN manylinux-entrypoint /build_scripts/install-pypy.sh 3.8 7.3.9 +RUN manylinux-entrypoint /build_scripts/install-pypy.sh 3.9 7.3.9 +COPY --from=build_cpython37 /opt/_internal /opt/_internal/ +COPY --from=build_cpython38 /opt/_internal /opt/_internal/ +COPY --from=build_cpython39 /opt/_internal /opt/_internal/ +COPY --from=build_cpython310 /opt/_internal /opt/_internal/ +COPY --from=build_cpython311 /opt/_internal /opt/_internal/ +RUN manylinux-entrypoint /build_scripts/finalize-python.sh + + +FROM runtime_base +COPY --from=build_git /manylinux-rootfs / +COPY --from=build_cpython /manylinux-rootfs / +COPY --from=all_python /opt/_internal /opt/_internal/ +COPY build_scripts/finalize.sh \ + build_scripts/update-system-packages.sh \ + build_scripts/python-tag-abi-tag.py \ + build_scripts/requirements3.7.txt \ + build_scripts/requirements3.8.txt \ + build_scripts/requirements3.9.txt \ + build_scripts/requirements3.10.txt \ + build_scripts/requirements3.11.txt \ + build_scripts/requirements-base-tools.txt \ + /build_scripts/ +COPY build_scripts/requirements-tools/* /build_scripts/requirements-tools/ +RUN manylinux-entrypoint /build_scripts/finalize.sh && rm -rf /build_scripts + +ENV SSL_CERT_FILE=/opt/_internal/certs.pem + +CMD ["/bin/bash"] + +#Build manylinux2014 docker image end + +#Install TensorRT 8.5.1.7 +#RUN yum install -y wget +RUN v="8.5.1-1.cuda11.8" &&\ + yum-config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel7/x86_64/cuda-rhel7.repo &&\ + yum -y install libnvinfer8-${v} libnvparsers8-${v} libnvonnxparsers8-${v} libnvinfer-plugin8-${v} \ + libnvinfer-devel-${v} libnvparsers-devel-${v} libnvonnxparsers-devel-${v} libnvinfer-plugin-devel-${v} + +#Add our own dependencies +ADD scripts /tmp/scripts +RUN cd /tmp/scripts && /tmp/scripts/manylinux/install_centos.sh && /tmp/scripts/manylinux/install_deps.sh && rm -rf /tmp/scripts + +ARG BUILD_UID=1001 +ARG BUILD_USER=onnxruntimedev +RUN adduser --uid $BUILD_UID $BUILD_USER +WORKDIR /home/$BUILD_USER +USER $BUILD_USER +ENV PATH /usr/local/gradle/bin:/usr/local/dotnet:$PATH From 4d11ee8cdea3cfb021bf3e01329b505d0429e666 Mon Sep 17 00:00:00 2001 From: George Wu Date: Tue, 8 Nov 2022 17:28:42 -0800 Subject: [PATCH 02/49] update onnx-tensorrt submodule to 8.5-GA --- .gitmodules | 2 +- cmake/external/onnx-tensorrt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 109570c9f1bff..916ad337c3376 100644 --- a/.gitmodules +++ b/.gitmodules @@ -65,7 +65,7 @@ [submodule "cmake/external/onnx-tensorrt"] path = cmake/external/onnx-tensorrt url = https://github.com/onnx/onnx-tensorrt.git - branch = 8.4-GA + branch = 8.5-GA [submodule "cmake/external/pthreadpool"] path = cmake/external/pthreadpool url = https://github.com/Maratyszcza/pthreadpool.git diff --git a/cmake/external/onnx-tensorrt b/cmake/external/onnx-tensorrt index 87c7a70688fd9..3b008c466bcb7 160000 --- a/cmake/external/onnx-tensorrt +++ b/cmake/external/onnx-tensorrt @@ -1 +1 @@ -Subproject commit 87c7a70688fd98fb355b8976f41425b40e4fe52f +Subproject commit 3b008c466bcb7375aaf5cabf51b289fd34d40c44 From fa2a58af286083c9ea6e2c9620fe5b9e958183ff Mon Sep 17 00:00:00 2001 From: George Wu Date: Mon, 14 Nov 2022 10:55:52 -0800 Subject: [PATCH 03/49] test builtin parser --- .../github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml index 11c5a92c24c7d..1e32b115fdd72 100644 --- a/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml @@ -49,7 +49,7 @@ jobs: --build_wheel \ --enable_onnx_tests --use_cuda --cuda_version=11.6 --cuda_home=/usr/local/cuda-11.6 --cudnn_home=/usr/local/cuda-11.6 \ --enable_pybind --build_java \ - --use_tensorrt --tensorrt_placeholder_builder --tensorrt_home /usr \ + --use_tensorrt --use_tensorrt_builtin_parser --tensorrt_placeholder_builder --tensorrt_home /usr \ --cmake_extra_defines CMAKE_CUDA_HOST_COMPILER=/opt/rh/devtoolset-11/root/usr/bin/cc CMAKE_CUDA_ARCHITECTURES=75 workingDirectory: $(Build.SourcesDirectory) @@ -120,7 +120,7 @@ jobs: --build_wheel \ --enable_onnx_tests --use_cuda --cuda_version=11.6 --cuda_home=/usr/local/cuda-11.6 --cudnn_home=/usr/local/cuda-11.6 \ --enable_pybind --build_java \ - --use_tensorrt --tensorrt_placeholder_builder --tensorrt_home /usr --ctest_path '' \ + --use_tensorrt --use_tensorrt_builtin_parser --tensorrt_placeholder_builder --tensorrt_home /usr --ctest_path '' \ --cmake_extra_defines CMAKE_CUDA_HOST_COMPILER=/opt/rh/devtoolset-11/root/usr/bin/cc CMAKE_CUDA_ARCHITECTURES=75 - template: templates/clean-agent-build-directory-step.yml From ce021a22782d230033622d98c8bc386dafe2f45a Mon Sep 17 00:00:00 2001 From: George Wu Date: Tue, 15 Nov 2022 15:27:16 -0800 Subject: [PATCH 04/49] try OSS parser again --- .../github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml index 1e32b115fdd72..11c5a92c24c7d 100644 --- a/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml @@ -49,7 +49,7 @@ jobs: --build_wheel \ --enable_onnx_tests --use_cuda --cuda_version=11.6 --cuda_home=/usr/local/cuda-11.6 --cudnn_home=/usr/local/cuda-11.6 \ --enable_pybind --build_java \ - --use_tensorrt --use_tensorrt_builtin_parser --tensorrt_placeholder_builder --tensorrt_home /usr \ + --use_tensorrt --tensorrt_placeholder_builder --tensorrt_home /usr \ --cmake_extra_defines CMAKE_CUDA_HOST_COMPILER=/opt/rh/devtoolset-11/root/usr/bin/cc CMAKE_CUDA_ARCHITECTURES=75 workingDirectory: $(Build.SourcesDirectory) @@ -120,7 +120,7 @@ jobs: --build_wheel \ --enable_onnx_tests --use_cuda --cuda_version=11.6 --cuda_home=/usr/local/cuda-11.6 --cudnn_home=/usr/local/cuda-11.6 \ --enable_pybind --build_java \ - --use_tensorrt --use_tensorrt_builtin_parser --tensorrt_placeholder_builder --tensorrt_home /usr --ctest_path '' \ + --use_tensorrt --tensorrt_placeholder_builder --tensorrt_home /usr --ctest_path '' \ --cmake_extra_defines CMAKE_CUDA_HOST_COMPILER=/opt/rh/devtoolset-11/root/usr/bin/cc CMAKE_CUDA_ARCHITECTURES=75 - template: templates/clean-agent-build-directory-step.yml From b82948d16c6501daf44c10444f87ca6e8fcb679d Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Fri, 18 Nov 2022 20:28:32 +0000 Subject: [PATCH 05/49] add back --gpus all --- .../github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml index 11c5a92c24c7d..06cd18f8c5eeb 100644 --- a/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml @@ -30,7 +30,7 @@ jobs: - task: CmdLine@2 inputs: script: | - docker run -e CC=/opt/rh/devtoolset-11/root/usr/bin/cc -e CXX=/opt/rh/devtoolset-11/root/usr/bin/c++ -e CFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e CXXFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" --rm \ + docker run --gpus all -e CC=/opt/rh/devtoolset-11/root/usr/bin/cc -e CXX=/opt/rh/devtoolset-11/root/usr/bin/c++ -e CFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e CXXFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" --rm \ --volume /data/onnx:/data/onnx:ro \ --volume $(Build.SourcesDirectory):/onnxruntime_src \ --volume $(Build.BinariesDirectory):/build \ From 8c1bb7fff25768e6675a186a887f263e53291b82 Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Mon, 21 Nov 2022 18:27:24 +0000 Subject: [PATCH 06/49] Revert to the state where build and test are running in container --- .../linux-gpu-tensorrt-ci-pipeline.yml | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml index 06cd18f8c5eeb..bbe62c1e200c8 100644 --- a/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml @@ -14,7 +14,7 @@ jobs: ALLOW_RELEASED_ONNX_OPSET_ONLY: '1' workspace: clean: all - pool: Linux-CPU-2019 + pool: onnxruntime-tensorrt-linuxbuild-T4 steps: - checkout: self clean: true @@ -30,7 +30,7 @@ jobs: - task: CmdLine@2 inputs: script: | - docker run --gpus all -e CC=/opt/rh/devtoolset-11/root/usr/bin/cc -e CXX=/opt/rh/devtoolset-11/root/usr/bin/c++ -e CFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e CXXFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" --rm \ + docker run -e CC=/opt/rh/devtoolset-11/root/usr/bin/cc -e CXX=/opt/rh/devtoolset-11/root/usr/bin/c++ -e CFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e CXXFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" --rm \ --volume /data/onnx:/data/onnx:ro \ --volume $(Build.SourcesDirectory):/onnxruntime_src \ --volume $(Build.BinariesDirectory):/build \ @@ -42,7 +42,7 @@ jobs: onnxruntimetensorrt85gpubuild \ /opt/python/cp38-cp38/bin/python3 /onnxruntime_src/tools/ci_build/build.py \ --build_dir /build --cmake_generator Ninja \ - --config Release --update --build \ + --config Release \ --skip_submodule_sync \ --build_shared_lib \ --parallel \ @@ -52,6 +52,15 @@ jobs: --use_tensorrt --tensorrt_placeholder_builder --tensorrt_home /usr \ --cmake_extra_defines CMAKE_CUDA_HOST_COMPILER=/opt/rh/devtoolset-11/root/usr/bin/cc CMAKE_CUDA_ARCHITECTURES=75 workingDirectory: $(Build.SourcesDirectory) + - task: PublishTestResults@2 + displayName: 'Publish unit test results' + inputs: + testResultsFiles: '**/*.results.xml' + searchFolder: '$(Build.BinariesDirectory)' + testRunTitle: 'Unit Test Run' + condition: succeededOrFailed() + + - template: templates/clean-agent-build-directory-step.yml - task: CmdLine@2 inputs: From 0dd11291e46456ed0532d272298f659c5cdf531e Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Mon, 21 Nov 2022 18:30:35 +0000 Subject: [PATCH 07/49] Revert to the state where build and test are running in container (cont.) --- .../linux-gpu-tensorrt-ci-pipeline.yml | 72 ------------------- 1 file changed, 72 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml index bbe62c1e200c8..ac9609f9cc930 100644 --- a/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml @@ -61,75 +61,3 @@ jobs: condition: succeededOrFailed() - template: templates/clean-agent-build-directory-step.yml - - - task: CmdLine@2 - inputs: - script: | - rm -rf $(Build.BinariesDirectory)/Release/onnxruntime $(Build.BinariesDirectory)/Release/pybind11 $(Build.BinariesDirectory)/Release/models $(Build.BinariesDirectory)/Release/_deps $(Build.BinariesDirectory)/Release/CMakeFiles - cd $(Build.BinariesDirectory)/Release - find -executable -type f > $(Build.BinariesDirectory)/Release/perms.txt - - - task: PublishPipelineArtifact@0 - displayName: 'Publish Pipeline Artifact' - inputs: - artifactName: 'drop-linux' - targetPath: '$(Build.BinariesDirectory)/Release' - - - task: PublishTestResults@2 - displayName: 'Publish unit test results' - inputs: - testResultsFiles: '**/*.results.xml' - searchFolder: '$(Build.BinariesDirectory)' - testRunTitle: 'Unit Test Run' - condition: succeededOrFailed() - - - template: templates/clean-agent-build-directory-step.yml - -- job: Linux_Test - timeoutInMinutes: 120 - variables: - skipComponentGovernanceDetection: true - workspace: - clean: all - pool: onnxruntime-tensorrt-linuxbuild-T4 - dependsOn: - - Linux_Build - steps: - - task: DownloadPipelineArtifact@2 - displayName: 'Download Pipeline Artifact' - inputs: - buildType: 'current' - artifactName: 'drop-linux' - targetPath: '$(Build.BinariesDirectory)/Release' - - - task: CmdLine@2 - inputs: - script: | - set -e -x - # We assume the machine doesn't have gcc and python development header files - sudo rm -f /build /onnxruntime_src - sudo ln -s $(Build.SourcesDirectory) /onnxruntime_src - python3 -m pip uninstall -y ort-nightly-gpu ort-nightly onnxruntime onnxruntime-gpu onnxruntime-training onnxruntime-directml ort-nightly-directml onnx -qq - cp $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/scripts/manylinux/requirements.txt $(Build.BinariesDirectory)/requirements.txt - # Test ORT with the latest ONNX release. - sed -i "s/git+http:\/\/github\.com\/onnx\/onnx.*/onnx/" $(Build.BinariesDirectory)/requirements.txt - python3 -m pip install -r $(Build.BinariesDirectory)/requirements.txt - python3 -m pip install $(Build.BinariesDirectory)/Release/dist/*.whl - ln -s /data/models $(Build.BinariesDirectory) - cd $(Build.BinariesDirectory)/Release - # Restore file permissions - xargs -a $(Build.BinariesDirectory)/Release/perms.txt chmod a+x - cd /tmp - python3 $(Build.SourcesDirectory)/tools/ci_build/build.py \ - --build_dir $(Build.BinariesDirectory) --cmake_generator Ninja \ - --config Release --test \ - --skip_submodule_sync \ - --build_shared_lib \ - --parallel \ - --build_wheel \ - --enable_onnx_tests --use_cuda --cuda_version=11.6 --cuda_home=/usr/local/cuda-11.6 --cudnn_home=/usr/local/cuda-11.6 \ - --enable_pybind --build_java \ - --use_tensorrt --tensorrt_placeholder_builder --tensorrt_home /usr --ctest_path '' \ - --cmake_extra_defines CMAKE_CUDA_HOST_COMPILER=/opt/rh/devtoolset-11/root/usr/bin/cc CMAKE_CUDA_ARCHITECTURES=75 - - - template: templates/clean-agent-build-directory-step.yml From 1514ff9cb4e10cba710a54bf024b3136940bc549 Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Mon, 21 Nov 2022 18:34:57 +0000 Subject: [PATCH 08/49] Revert to the state where build and test are running in container (cont.) --- .../github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml index ac9609f9cc930..37fbc3f9b7dd8 100644 --- a/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml @@ -52,12 +52,13 @@ jobs: --use_tensorrt --tensorrt_placeholder_builder --tensorrt_home /usr \ --cmake_extra_defines CMAKE_CUDA_HOST_COMPILER=/opt/rh/devtoolset-11/root/usr/bin/cc CMAKE_CUDA_ARCHITECTURES=75 workingDirectory: $(Build.SourcesDirectory) + - task: PublishTestResults@2 displayName: 'Publish unit test results' inputs: testResultsFiles: '**/*.results.xml' searchFolder: '$(Build.BinariesDirectory)' - testRunTitle: 'Unit Test Run' + testRunTitle: 'Unit Test Run' condition: succeededOrFailed() - template: templates/clean-agent-build-directory-step.yml From 42666d39a4148958367a82c4ab89b293b2ffceaa Mon Sep 17 00:00:00 2001 From: Chi Lo <54722500+chilo-ms@users.noreply.github.com> Date: Mon, 21 Nov 2022 13:59:52 -0800 Subject: [PATCH 09/49] Update linux-gpu-tensorrt-ci-pipeline.yml add back "--gpus all" --- .../github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml index 37fbc3f9b7dd8..0a9bf19143a93 100644 --- a/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml @@ -30,7 +30,7 @@ jobs: - task: CmdLine@2 inputs: script: | - docker run -e CC=/opt/rh/devtoolset-11/root/usr/bin/cc -e CXX=/opt/rh/devtoolset-11/root/usr/bin/c++ -e CFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e CXXFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" --rm \ + docker run --gpus all -e CC=/opt/rh/devtoolset-11/root/usr/bin/cc -e CXX=/opt/rh/devtoolset-11/root/usr/bin/c++ -e CFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e CXXFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" --rm \ --volume /data/onnx:/data/onnx:ro \ --volume $(Build.SourcesDirectory):/onnxruntime_src \ --volume $(Build.BinariesDirectory):/build \ From de0f435da33ff15a1920ed8512a8b2134dc29ac8 Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Tue, 22 Nov 2022 00:48:38 +0000 Subject: [PATCH 10/49] skip tests for known issues --- onnxruntime/test/contrib_ops/quantize_ops_test.cc | 8 ++++---- onnxruntime/test/contrib_ops/tensor_op_test.cc | 4 ++-- .../test/providers/cpu/generator/random_test.cc | 3 ++- .../test/providers/cpu/math/element_wise_ops_test.cc | 4 ++-- onnxruntime/test/providers/cpu/nn/shrink_test.cc | 2 +- .../test/providers/cpu/tensor/quantize_linear_test.cc | 10 +++++----- 6 files changed, 16 insertions(+), 15 deletions(-) diff --git a/onnxruntime/test/contrib_ops/quantize_ops_test.cc b/onnxruntime/test/contrib_ops/quantize_ops_test.cc index af0b1a2dbbaf7..e2afffcc384ff 100644 --- a/onnxruntime/test/contrib_ops/quantize_ops_test.cc +++ b/onnxruntime/test/contrib_ops/quantize_ops_test.cc @@ -48,7 +48,7 @@ TEST(DequantizeLinearOpTest, DequantizeLinear_per_tensor_half_uint8) { test.AddInput("x_scale", {}, ToFloat16({2.0f})); test.AddInput("x_zero_point", {}, {128}); test.AddOutput("y", dims, ToFloat16({-256.0f, -250.0f, 0.0f, 254.0f})); - test.Run(); + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT doesn't support support UINT8 for quantization } // scalar zero & scale with int8 @@ -270,7 +270,7 @@ TEST(QuantizeLinearContribOpTest, QuantizeLinear_per_tensor_half_uint8) { 255, 0, 255, 0, 255, 0}); - test.Run(); + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT doesn't support support UINT8 for quantization } TEST(QuantizeLinearContribOpTest, QuantizeLinear_per_tensor_half_int8) { @@ -317,7 +317,7 @@ TEST(QuantizeLinearContribOpTest, QuantizeLinear_per_channel) { {0, 2, 3, 255, 0, 1, 2, 255, 0, 0, 1, 250}); - test.Run(); + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT doesn't support support UINT8 for quantization } // quantize with broadcasting and negative axis (-2 resolves to axis 0) @@ -335,7 +335,7 @@ TEST(QuantizeLinearContribOpTest, QuantizeLinear_per_channel_negative_axis) { {0, 2, 3, 255, 0, 1, 2, 255, 0, 0, 1, 250}); - test.Run(); + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT doesn't support support UINT8 for quantization } } // namespace test } // namespace onnxruntime diff --git a/onnxruntime/test/contrib_ops/tensor_op_test.cc b/onnxruntime/test/contrib_ops/tensor_op_test.cc index 6ed563909560c..96a2a39748c56 100644 --- a/onnxruntime/test/contrib_ops/tensor_op_test.cc +++ b/onnxruntime/test/contrib_ops/tensor_op_test.cc @@ -120,7 +120,7 @@ void MeanVarianceNormalizationAcrossChannels(bool across_channels, bool normaliz test.AddAttribute("normalize_variance", normalize_variance ? one : zero); test.AddInput("input", {N, C, H, W}, X); test.AddOutput("output", {N, C, H, W}, result); - test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kOpenVINOExecutionProvider}); //OpenVINO doesn't support MVN operator below opset 9 + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kOpenVINOExecutionProvider, kTensorrtExecutionProvider}); //OpenVINO doesn't support MVN operator below opset 9. TensorRT doesn't support opset 8 of MVN operator. } void MeanVarianceNormalizationPerChannel(bool across_channels, bool normalize_variance) { @@ -187,7 +187,7 @@ void MeanVarianceNormalizationPerChannel(bool across_channels, bool normalize_va test.AddAttribute("normalize_variance", normalize_variance ? one : zero); test.AddInput("input", {N, C, H, W}, X); test.AddOutput("output", {N, C, H, W}, result); - test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kOpenVINOExecutionProvider}); //OpenVINO doesn't support MVN operator below opset 9 + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kOpenVINOExecutionProvider, kTensorrtExecutionProvider}); //OpenVINO doesn't support MVN operator below opset 9. TensorRT doesn't support opset 8 of MVN operator. } TEST(MVNContribOpTest, MeanVarianceNormalizationCPUTest_Version1_TO_8) { diff --git a/onnxruntime/test/providers/cpu/generator/random_test.cc b/onnxruntime/test/providers/cpu/generator/random_test.cc index e3101297b25f3..85ad297cab814 100644 --- a/onnxruntime/test/providers/cpu/generator/random_test.cc +++ b/onnxruntime/test/providers/cpu/generator/random_test.cc @@ -71,7 +71,8 @@ void RunRandomNormalLike3DFloat(bool infer_dtype = false) { test.AddOutput("Y", dims, expected_output); - test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCudaExecutionProvider, kRocmExecutionProvider}); + // TensorRT does not support manual seed overrides and there will be result mismatch + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCudaExecutionProvider, kRocmExecutionProvider, kTensorrtExecutionProvider}); } TEST(Random, RandomNormalLike3DDouble) { diff --git a/onnxruntime/test/providers/cpu/math/element_wise_ops_test.cc b/onnxruntime/test/providers/cpu/math/element_wise_ops_test.cc index b2e6e75c751e2..006758ad6bc45 100644 --- a/onnxruntime/test/providers/cpu/math/element_wise_ops_test.cc +++ b/onnxruntime/test/providers/cpu/math/element_wise_ops_test.cc @@ -2338,7 +2338,7 @@ TEST(ModOpTest, Int8_mixed_sign) { test.AddInput("Y", {6}, {2, -3, 8, -2, 3, 5}); test.AddOutput("Z", {6}, {0, -2, 5, 0, 2, 3}); - test.Run(); + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); // For TensorRT running in these in INT8 quantization scales are needed, so skip it now } TEST(ModOpTest, Int8_mixed_sign_fmod) { @@ -2349,7 +2349,7 @@ TEST(ModOpTest, Int8_mixed_sign_fmod) { test.AddInput("Y", {6}, {2, -3, 8, -2, 3, 5}); test.AddOutput("Z", {6}, {0, 1, 5, 0, -1, 3}); - test.Run(); + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); // For TensorRT running in these in INT8 quantization scales are needed, so skip it now } TEST(ModOpTest, UInt8_mod) { diff --git a/onnxruntime/test/providers/cpu/nn/shrink_test.cc b/onnxruntime/test/providers/cpu/nn/shrink_test.cc index 709f6a16d2277..4a1e18db21d47 100644 --- a/onnxruntime/test/providers/cpu/nn/shrink_test.cc +++ b/onnxruntime/test/providers/cpu/nn/shrink_test.cc @@ -97,7 +97,7 @@ const std::vector ConvertFloatToMLFloat16(const std::vector& f TEST(MathOpTest, ShrinkInt8Type) { const auto& test_cases = GenerateSignedTestCases(); - RunShrinkTest(test_cases); + RunShrinkTest(test_cases, {kTensorrtExecutionProvider}); // For TensorRT running in these in INT8 quantization scales are needed, so skip it now } TEST(MathOpTest, ShrinkUint8Type) { diff --git a/onnxruntime/test/providers/cpu/tensor/quantize_linear_test.cc b/onnxruntime/test/providers/cpu/tensor/quantize_linear_test.cc index 63b911805c36c..da4c2d6491d62 100644 --- a/onnxruntime/test/providers/cpu/tensor/quantize_linear_test.cc +++ b/onnxruntime/test/providers/cpu/tensor/quantize_linear_test.cc @@ -296,7 +296,7 @@ TEST(QuantizeLinearOpTest, 2D) { {0, 0, 1, 250, 0, 0, 1, 250, 0, 0, 1, 250}); - test.Run(); + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT doesn't support support UINT8 for quantization } // quantize with scalar data @@ -306,7 +306,7 @@ TEST(QuantizeLinearOpTest, Scalar) { test.AddInput("y_scale", {}, {2.0f}); test.AddInput("y_zero_point", {}, {128}); test.AddOutput("y", {}, {130}); - test.Run(); + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT doesn't support support UINT8 for quantization } // quantize with scalar data @@ -315,7 +315,7 @@ TEST(QuantizeLinearOpTest, DISABLED_QuantizeLinear_Without_Zero_Point) { test.AddInput("x", {}, {3}); test.AddInput("y_scale", {}, {2.0f}); test.AddOutput("y", {}, {2}); - test.Run(); + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT doesn't support support UINT8 for quantization } TEST(QuantizeLinearOpTest, Per_Channel_Axis_Default) { @@ -331,7 +331,7 @@ TEST(QuantizeLinearOpTest, Per_Channel_Axis_Default) { {64, 101, 127, 177, 65, 100, 128, 182, 66, 102, 128, 187}); - test.Run(); + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT doesn't support support UINT8 for quantization } TEST(QuantizeLinearOpTest, Per_Channel_Axis_0) { @@ -348,7 +348,7 @@ TEST(QuantizeLinearOpTest, Per_Channel_Axis_0) { {0, 2, 3, 255, 0, 1, 2, 255, 0, 0, 1, 250}); - test.Run(); + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT doesn't support support UINT8 for quantization } // quantize with per-channel and negative axis (-2 resolves to axis 0) From 5a217de5aec1f5c0605f7a4080ae5ca74cbadaee Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Tue, 22 Nov 2022 02:51:45 +0000 Subject: [PATCH 11/49] skip tests for known issues --- onnxruntime/test/contrib_ops/quantize_ops_test.cc | 4 ++-- onnxruntime/test/providers/cpu/tensor/quantize_linear_test.cc | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/onnxruntime/test/contrib_ops/quantize_ops_test.cc b/onnxruntime/test/contrib_ops/quantize_ops_test.cc index e2afffcc384ff..5b01544d4bb2b 100644 --- a/onnxruntime/test/contrib_ops/quantize_ops_test.cc +++ b/onnxruntime/test/contrib_ops/quantize_ops_test.cc @@ -48,7 +48,7 @@ TEST(DequantizeLinearOpTest, DequantizeLinear_per_tensor_half_uint8) { test.AddInput("x_scale", {}, ToFloat16({2.0f})); test.AddInput("x_zero_point", {}, {128}); test.AddOutput("y", dims, ToFloat16({-256.0f, -250.0f, 0.0f, 254.0f})); - test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT doesn't support support UINT8 for quantization + test.Run(); } // scalar zero & scale with int8 @@ -203,7 +203,7 @@ void TestQuantizeLinearPerTensorFloatUint8(bool use_initializer_except_x) { 255, 0, 255, 0, 255, 0}); - test.Run(); + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT doesn't support support UINT8 for quantization } TEST(QuantizeLinearContribOpTest, QuantizeLinear_per_tensor_float_uint8) { diff --git a/onnxruntime/test/providers/cpu/tensor/quantize_linear_test.cc b/onnxruntime/test/providers/cpu/tensor/quantize_linear_test.cc index da4c2d6491d62..0b286f628582f 100644 --- a/onnxruntime/test/providers/cpu/tensor/quantize_linear_test.cc +++ b/onnxruntime/test/providers/cpu/tensor/quantize_linear_test.cc @@ -228,7 +228,7 @@ TEST(QuantizeLinearOpTest, Uint8) { test.AddInput("y_scale", {}, {2.0f}); test.AddInput("y_zero_point", {}, {128}); test.AddOutput("y", dims, {128, 129, 130, 255, 1, 0}); - test.Run(); + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT doesn't support support UINT8 for quantization } // quantize with scalar zero point and scale @@ -366,7 +366,7 @@ TEST(QuantizeLinearOpTest, Per_Channel_Axis_neg) { {0, 2, 3, 255, 0, 1, 2, 255, 0, 0, 1, 250}); - test.Run(); + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT doesn't support support UINT8 for quantization } } // namespace test From 4f5ef220ff4319fb4a5dd19e2d2ebfba7165e314 Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Tue, 6 Dec 2022 23:02:52 +0000 Subject: [PATCH 12/49] Update TRT Windows CI ymal --- .../github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml index 0aba08795f314..5c551a0ca19f0 100644 --- a/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml @@ -48,7 +48,7 @@ jobs: displayName: 'Generate cmake config' inputs: scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' - arguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --build_wheel --enable_onnx_tests --use_tensorrt --tensorrt_placeholder_builder --tensorrt_home="C:\local\TensorRT-8.4.1.5.Windows10.x86_64.cuda-11.6.cudnn8.4" --cuda_version=11.6 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.6" --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=75' + arguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --build_wheel --enable_onnx_tests --use_tensorrt --tensorrt_placeholder_builder --tensorrt_home="C:\local\TensorRT-8.5.1.7.Windows10.x86_64.cuda-11.8.cudnn8.6" --cuda_version=11.6 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.6" --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=75' workingDirectory: '$(Build.BinariesDirectory)' - task: VSBuild@1 @@ -78,7 +78,7 @@ jobs: del wheel_filename_file python.exe -m pip install -q --upgrade %WHEEL_FILENAME% set PATH=$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig);%PATH% - python $(Build.SourcesDirectory)\tools\ci_build\build.py --config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --test --cmake_generator "Visual Studio 16 2019" --build_wheel --enable_onnx_tests --use_tensorrt --tensorrt_placeholder_builder --tensorrt_home="C:\local\TensorRT-8.4.1.5.Windows10.x86_64.cuda-11.6.cudnn8.4" --cuda_version=11.6 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.6" --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=75 + python $(Build.SourcesDirectory)\tools\ci_build\build.py --config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --test --cmake_generator "Visual Studio 16 2019" --build_wheel --enable_onnx_tests --use_tensorrt --tensorrt_placeholder_builder --tensorrt_home="C:\local\TensorRT-8.5.1.7.Windows10.x86_64.cuda-11.8.cudnn8.6" --cuda_version=11.6 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.6" --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=75 workingDirectory: '$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig)' displayName: 'Run tests' From 9288869dfb13a465ae2fa778949a04cbacc7ee50 Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Tue, 6 Dec 2022 23:29:36 +0000 Subject: [PATCH 13/49] update CI ymals --- .../c-api-noopenmp-packaging-pipelines.yml | 12 ++++++------ .../linux-gpu-tensorrt-packaging-pipeline.yml | 2 +- .../templates/py-packaging-selectable-stage.yml | 8 ++++---- .../azure-pipelines/templates/py-packaging-stage.yml | 10 +++++----- tools/ci_build/github/windows/setup_env_gpu.bat | 2 +- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml b/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml index 289fa7a8cdc0b..fdb46e777db58 100644 --- a/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml +++ b/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml @@ -93,7 +93,7 @@ jobs: - template: templates/win-ci.yml parameters: - ort_build_pool_name: 'onnxruntime-gpu-tensorrt8-winbuild-t4' + ort_build_pool_name: 'onnxruntime-tensorrt8-winbuild-T4' DoCompliance: ${{ parameters.DoCompliance }} DoEsrp: ${{ parameters.DoEsrp }} job_name_suffix: gpu @@ -108,7 +108,7 @@ jobs: - template: templates/win-ci.yml parameters: - ort_build_pool_name: 'onnxruntime-gpu-tensorrt8-winbuild-t4' + ort_build_pool_name: 'onnxruntime-tensorrt8-winbuild-T4' DoCompliance: ${{ parameters.DoCompliance }} DoEsrp: ${{ parameters.DoEsrp }} job_name_suffix: tensorrt @@ -116,7 +116,7 @@ jobs: buildArch: x64 msbuildPlatform: x64 packageName: x64-tensorrt - buildparameter: --use_tensorrt --tensorrt_home="C:\local\TensorRT-8.4.1.5.Windows10.x86_64.cuda-11.6.cudnn8.4" --cuda_version=11.6 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.6" --enable_onnx_tests --enable_wcos --build_java --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=37;52;60;61;70;75;80" + buildparameter: --use_tensorrt --tensorrt_home="C:\local\TensorRT-8.5.1.7.Windows10.x86_64.cuda-11.8.cudnn8.6" --cuda_version=11.6 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.6" --enable_onnx_tests --enable_wcos --build_java --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=37;52;60;61;70;75;80" runTests: ${{ parameters.RunOnnxRuntimeTests }} buildJava: true java_artifact_id: onnxruntime_gpu @@ -294,11 +294,11 @@ jobs: Steps: - script: | tools/ci_build/get_docker_image.py \ - --dockerfile tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cuda11_6_tensorrt8_4 \ + --dockerfile tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cuda11_6_tensorrt8_5 \ --context tools/ci_build/github/linux/docker \ --docker-build-args "--network=host --build-arg POLICY=manylinux2014 --build-arg PLATFORM=x86_64 --build-arg DEVTOOLSET_ROOTPATH=/opt/rh/devtoolset-11/root --build-arg PREPEND_PATH=/opt/rh/devtoolset-11/root/usr/bin: --build-arg LD_LIBRARY_PATH_ARG=/opt/rh/devtoolset-11/root/usr/lib64:/opt/rh/devtoolset-11/root/usr/lib:/opt/rh/devtoolset-11/root/usr/lib64/dyninst:/opt/rh/devtoolset-11/root/usr/lib/dyninst:/usr/local/lib64 --build-arg BUILD_UID=$( id -u )" \ --container-registry onnxruntimebuildcache \ - --repository onnxruntimecuda116xtrt84build + --repository onnxruntimecuda116xtrt85build displayName: "Getonnxruntimecuda116xtrt84build image for tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cuda11_6_tensorrt8_4" workingDirectory: $(Build.SourcesDirectory)/onnxruntime ContainerRegistry: onnxruntimebuildcache @@ -351,7 +351,7 @@ jobs: inputs: script: | docker run --gpus all -e CC=/opt/rh/devtoolset-11/root/usr/bin/cc -e CXX=/opt/rh/devtoolset-11/root/usr/bin/c++ -e CFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e CXXFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e NVIDIA_VISIBLE_DEVICES=all --rm --volume $(Build.SourcesDirectory):/src_dir \ - --volume $(Build.ArtifactStagingDirectory):/artifact_src -e NIGHTLY_BUILD onnxruntimecuda116xtrt84build \ + --volume $(Build.ArtifactStagingDirectory):/artifact_src -e NIGHTLY_BUILD onnxruntimecuda116xtrt85build \ /src_dir/onnxruntime-inference-examples/c_cxx/squeezenet/run_capi_application.sh -o /src_dir/onnxruntime -p /artifact_src/onnxruntime-linux-x64-gpu-$(OnnxRuntimeVersion).tgz -w /src_dir/onnxruntime-inference-examples/c_cxx/squeezenet workingDirectory: '$(Build.ArtifactStagingDirectory)' diff --git a/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-packaging-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-packaging-pipeline.yml index f78e741584f6f..b6844710a4806 100644 --- a/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-packaging-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-packaging-pipeline.yml @@ -30,7 +30,7 @@ jobs: submodules: recursive - template: templates/get-docker-image-steps.yml parameters: - Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cuda11_6_tensorrt8_4 + Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cuda11_6_tensorrt8_5 Context: tools/ci_build/github/linux/docker DockerBuildArgs: "--network=host --build-arg POLICY=manylinux2014 --build-arg PLATFORM=x86_64 --build-arg DEVTOOLSET_ROOTPATH=/opt/rh/devtoolset-11/root --build-arg PREPEND_PATH=/opt/rh/devtoolset-11/root/usr/bin: --build-arg LD_LIBRARY_PATH_ARG=/opt/rh/devtoolset-11/root/usr/lib64:/opt/rh/devtoolset-11/root/usr/lib:/opt/rh/devtoolset-11/root/usr/lib64/dyninst:/opt/rh/devtoolset-11/root/usr/lib/dyninst:/usr/local/lib64 --build-arg BUILD_UID=$( id -u )" Repository: onnxruntimecuda114xtrt82build diff --git a/tools/ci_build/github/azure-pipelines/templates/py-packaging-selectable-stage.yml b/tools/ci_build/github/azure-pipelines/templates/py-packaging-selectable-stage.yml index cbe3ddadcbe57..14721c22c1854 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-packaging-selectable-stage.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-packaging-selectable-stage.yml @@ -304,10 +304,10 @@ stages: - template: get-docker-image-steps.yml parameters: - Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cuda11_6_tensorrt8_4 + Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cuda11_6_tensorrt8_5 Context: tools/ci_build/github/linux/docker DockerBuildArgs: "--network=host --build-arg POLICY=manylinux2014 --build-arg PLATFORM=x86_64 --build-arg DEVTOOLSET_ROOTPATH=/opt/rh/devtoolset-11/root --build-arg PREPEND_PATH=/opt/rh/devtoolset-11/root/usr/bin: --build-arg LD_LIBRARY_PATH_ARG=/opt/rh/devtoolset-11/root/usr/lib64:/opt/rh/devtoolset-11/root/usr/lib:/opt/rh/devtoolset-11/root/usr/lib64/dyninst:/opt/rh/devtoolset-11/root/usr/lib/dyninst:/usr/local/lib64 --build-arg BUILD_UID=$( id -u )" - Repository: onnxruntimecuda116xtrt84build + Repository: onnxruntimecuda116xtrt85build - task: CmdLine@2 displayName: 'Build Python Wheel' @@ -321,7 +321,7 @@ stages: --volume /data/models:/build/models:ro \ --volume $HOME/.onnx:/home/onnxruntimedev/.onnx \ -e BUILD_BUILDNUMBER \ - onnxruntimecuda116xtrt84build \ + onnxruntimecuda116xtrt85build \ $(PythonManylinuxDir)/bin/python3 /onnxruntime_src/tools/ci_build/build.py \ --build_dir /build --cmake_generator Ninja \ --config Release --update --build \ @@ -389,7 +389,7 @@ stages: variables: CUDA_VERSION: '11.6' buildArch: x64 - EpBuildFlags: --use_tensorrt --tensorrt_home="C:\local\TensorRT-8.4.1.5.Windows10.x86_64.cuda-11.6.cudnn8.4" --cuda_version=$(CUDA_VERSION) --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v$(CUDA_VERSION)" --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=37;50;52;60;61;70;75;80" + EpBuildFlags: --use_tensorrt --tensorrt_home="C:\local\TensorRT-8.5.1.7.Windows10.x86_64.cuda-11.8.cudnn8.6" --cuda_version=$(CUDA_VERSION) --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v$(CUDA_VERSION)" --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=37;50;52;60;61;70;75;80" EnvSetupScript: setup_env_gpu.bat EP_NAME: gpu strategy: diff --git a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml index b9c81739bd620..1503f7a3ee048 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml @@ -265,7 +265,7 @@ stages: parameters: MACHINE_POOL: 'onnxruntime-gpu-winbuild-t4' PYTHON_VERSION: '3.7' - EP_BUILD_FLAGS: --use_tensorrt --tensorrt_home="C:\local\TensorRT-8.4.1.5.Windows10.x86_64.cuda-11.6.cudnn8.4" --cuda_version=11.6 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.6" --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=37;50;52;60;61;70;75;80" + EP_BUILD_FLAGS: --use_tensorrt --tensorrt_home="C:\local\TensorRT-8.5.1.7.Windows10.x86_64.cuda-11.8.cudnn8.6" --cuda_version=11.6 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.6" --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=37;50;52;60;61;70;75;80" ENV_SETUP_SCRIPT: setup_env_gpu.bat EP_NAME: gpu @@ -273,7 +273,7 @@ stages: parameters: MACHINE_POOL: 'onnxruntime-gpu-winbuild-t4' PYTHON_VERSION: '3.8' - EP_BUILD_FLAGS: --use_tensorrt --tensorrt_home="C:\local\TensorRT-8.4.1.5.Windows10.x86_64.cuda-11.6.cudnn8.4" --cuda_version=11.6 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.6" --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=37;50;52;60;61;70;75;80" + EP_BUILD_FLAGS: --use_tensorrt --tensorrt_home="C:\local\TensorRT-8.5.1.7.Windows10.x86_64.cuda-11.8.cudnn8.6" --cuda_version=11.6 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.6" --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=37;50;52;60;61;70;75;80" ENV_SETUP_SCRIPT: setup_env_gpu.bat EP_NAME: gpu @@ -281,7 +281,7 @@ stages: parameters: MACHINE_POOL: 'onnxruntime-gpu-winbuild-t4' PYTHON_VERSION: '3.9' - EP_BUILD_FLAGS: --use_tensorrt --tensorrt_home="C:\local\TensorRT-8.4.1.5.Windows10.x86_64.cuda-11.6.cudnn8.4" --cuda_version=11.6 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.6" --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=37;50;52;60;61;70;75;80" + EP_BUILD_FLAGS: --use_tensorrt --tensorrt_home="C:\local\TensorRT-8.5.1.7.Windows10.x86_64.cuda-11.8.cudnn8.6" --cuda_version=11.6 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.6" --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=37;50;52;60;61;70;75;80" ENV_SETUP_SCRIPT: setup_env_gpu.bat EP_NAME: gpu @@ -289,7 +289,7 @@ stages: parameters: MACHINE_POOL: 'onnxruntime-gpu-winbuild-t4' PYTHON_VERSION: '3.10' - EP_BUILD_FLAGS: --use_tensorrt --tensorrt_home="C:\local\TensorRT-8.4.1.5.Windows10.x86_64.cuda-11.6.cudnn8.4" --cuda_version=11.6 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.6" --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=37;50;52;60;61;70;75;80" + EP_BUILD_FLAGS: --use_tensorrt --tensorrt_home="C:\local\TensorRT-8.5.1.7.Windows10.x86_64.cuda-11.8.cudnn8.6" --cuda_version=11.6 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.6" --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=37;50;52;60;61;70;75;80" ENV_SETUP_SCRIPT: setup_env_gpu.bat EP_NAME: gpu @@ -502,4 +502,4 @@ stages: - template: py-linux-gpu.yml parameters: arch: 'x86_64' - machine_pool: 'Linux-CPU' \ No newline at end of file + machine_pool: 'Linux-CPU' diff --git a/tools/ci_build/github/windows/setup_env_gpu.bat b/tools/ci_build/github/windows/setup_env_gpu.bat index faaff0387d56b..1ef315aee722f 100644 --- a/tools/ci_build/github/windows/setup_env_gpu.bat +++ b/tools/ci_build/github/windows/setup_env_gpu.bat @@ -1,5 +1,5 @@ REM Copyright (c) Microsoft Corporation. All rights reserved. REM Licensed under the MIT License. -set PATH=C:\local\TensorRT-8.4.1.5.Windows10.x86_64.cuda-11.6.cudnn8.4\lib;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.6\bin;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.6\extras\CUPTI\lib64;C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Current\Bin;%PATH% +set PATH=C:\local\TensorRT-8.5.1.7.Windows10.x86_64.cuda-11.8.cudnn8.6\lib;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.6\bin;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.6\extras\CUPTI\lib64;C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Current\Bin;%PATH% set GRADLE_OPTS=-Dorg.gradle.daemon=false From 331a94714deffdd68eca775345e0f7cc9782b26d Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Wed, 7 Dec 2022 07:15:51 +0000 Subject: [PATCH 14/49] use original pool --- .../azure-pipelines/c-api-noopenmp-packaging-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml b/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml index b5fda8d309360..328824fdefa59 100644 --- a/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml +++ b/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml @@ -93,7 +93,7 @@ jobs: - template: templates/win-ci.yml parameters: - ort_build_pool_name: 'onnxruntime-tensorrt8-winbuild-T4' + ort_build_pool_name: 'onnxruntime-gpu-tensorrt8-winbuild-t4' DoCompliance: ${{ parameters.DoCompliance }} DoEsrp: ${{ parameters.DoEsrp }} job_name_suffix: gpu @@ -108,7 +108,7 @@ jobs: - template: templates/win-ci.yml parameters: - ort_build_pool_name: 'onnxruntime-tensorrt8-winbuild-T4' + ort_build_pool_name: 'onnxruntime-gpu-tensorrt8-winbuild-t4' DoCompliance: ${{ parameters.DoCompliance }} DoEsrp: ${{ parameters.DoEsrp }} job_name_suffix: tensorrt From f59bd5992d62fbc9e47e001e8e024d2966ce274b Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Wed, 7 Dec 2022 19:35:27 +0000 Subject: [PATCH 15/49] add placeholder flag for package pipelines --- .../azure-pipelines/c-api-noopenmp-packaging-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml b/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml index 328824fdefa59..3d7d1440b6b67 100644 --- a/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml +++ b/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml @@ -116,7 +116,7 @@ jobs: buildArch: x64 msbuildPlatform: x64 packageName: x64-tensorrt - buildparameter: --use_tensorrt --tensorrt_home="C:\local\TensorRT-8.5.1.7.Windows10.x86_64.cuda-11.8.cudnn8.6" --cuda_version=11.6 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.6" --enable_onnx_tests --enable_wcos --build_java --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=37;52;60;61;70;75;80" + buildparameter: --use_tensorrt --tensorrt_home="C:\local\TensorRT-8.5.1.7.Windows10.x86_64.cuda-11.8.cudnn8.6" --tensorrt_placeholder_builder --cuda_version=11.6 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.6" --enable_onnx_tests --enable_wcos --build_java --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=37;52;60;61;70;75;80" runTests: ${{ parameters.RunOnnxRuntimeTests }} buildJava: true java_artifact_id: onnxruntime_gpu From 26d9c844576547c8bd694debe892191e7488ac1b Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Wed, 7 Dec 2022 20:05:57 +0000 Subject: [PATCH 16/49] increase timeout for TRT EP --- tools/ci_build/build.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index a6e6b4cdc715b..de15414303422 100644 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -1881,7 +1881,10 @@ def run_onnxruntime_tests(args, source_dir, ctest_path, build_dir, configs): run_subprocess([os.path.join(cwd, exe)], cwd=cwd, dll_path=dll_path) else: - ctest_cmd = [ctest_path, "--build-config", config, "--verbose", "--timeout", "10800"] + if args.use_tensorrt: + ctest_cmd = [ctest_path, "--build-config", config, "--verbose", "--timeout", "18000"] + else: + ctest_cmd = [ctest_path, "--build-config", config, "--verbose", "--timeout", "10800"] run_subprocess(ctest_cmd, cwd=cwd, dll_path=dll_path) if args.enable_pybind: From 50d583a1cd3d45ef62a2fd93b00deb4b7988814b Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Wed, 7 Dec 2022 22:42:03 +0000 Subject: [PATCH 17/49] revert increase timeout --- tools/ci_build/build.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index de15414303422..a6e6b4cdc715b 100644 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -1881,10 +1881,7 @@ def run_onnxruntime_tests(args, source_dir, ctest_path, build_dir, configs): run_subprocess([os.path.join(cwd, exe)], cwd=cwd, dll_path=dll_path) else: - if args.use_tensorrt: - ctest_cmd = [ctest_path, "--build-config", config, "--verbose", "--timeout", "18000"] - else: - ctest_cmd = [ctest_path, "--build-config", config, "--verbose", "--timeout", "10800"] + ctest_cmd = [ctest_path, "--build-config", config, "--verbose", "--timeout", "10800"] run_subprocess(ctest_cmd, cwd=cwd, dll_path=dll_path) if args.enable_pybind: From 74940802fc82f239fe534759e47e7e14ec226e88 Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Wed, 7 Dec 2022 23:41:23 +0000 Subject: [PATCH 18/49] add back timeout --- tools/ci_build/build.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index a6e6b4cdc715b..de15414303422 100644 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -1881,7 +1881,10 @@ def run_onnxruntime_tests(args, source_dir, ctest_path, build_dir, configs): run_subprocess([os.path.join(cwd, exe)], cwd=cwd, dll_path=dll_path) else: - ctest_cmd = [ctest_path, "--build-config", config, "--verbose", "--timeout", "10800"] + if args.use_tensorrt: + ctest_cmd = [ctest_path, "--build-config", config, "--verbose", "--timeout", "18000"] + else: + ctest_cmd = [ctest_path, "--build-config", config, "--verbose", "--timeout", "10800"] run_subprocess(ctest_cmd, cwd=cwd, dll_path=dll_path) if args.enable_pybind: From c59a4211e5807a02d7a47029d9e7440ed3933548 Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Wed, 7 Dec 2022 23:43:04 +0000 Subject: [PATCH 19/49] remove place holder since it still causes application deadlock --- .../azure-pipelines/c-api-noopenmp-packaging-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml b/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml index 3d7d1440b6b67..328824fdefa59 100644 --- a/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml +++ b/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml @@ -116,7 +116,7 @@ jobs: buildArch: x64 msbuildPlatform: x64 packageName: x64-tensorrt - buildparameter: --use_tensorrt --tensorrt_home="C:\local\TensorRT-8.5.1.7.Windows10.x86_64.cuda-11.8.cudnn8.6" --tensorrt_placeholder_builder --cuda_version=11.6 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.6" --enable_onnx_tests --enable_wcos --build_java --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=37;52;60;61;70;75;80" + buildparameter: --use_tensorrt --tensorrt_home="C:\local\TensorRT-8.5.1.7.Windows10.x86_64.cuda-11.8.cudnn8.6" --cuda_version=11.6 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.6" --enable_onnx_tests --enable_wcos --build_java --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=37;52;60;61;70;75;80" runTests: ${{ parameters.RunOnnxRuntimeTests }} buildJava: true java_artifact_id: onnxruntime_gpu From cade3ab5abd6d52613cb46591b7be10945612037 Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Thu, 8 Dec 2022 19:33:06 +0000 Subject: [PATCH 20/49] increase timeout to 10 hours --- tools/ci_build/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index de15414303422..09b6e2a305c96 100644 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -1882,7 +1882,7 @@ def run_onnxruntime_tests(args, source_dir, ctest_path, build_dir, configs): else: if args.use_tensorrt: - ctest_cmd = [ctest_path, "--build-config", config, "--verbose", "--timeout", "18000"] + ctest_cmd = [ctest_path, "--build-config", config, "--verbose", "--timeout", "36000"] else: ctest_cmd = [ctest_path, "--build-config", config, "--verbose", "--timeout", "10800"] run_subprocess(ctest_cmd, cwd=cwd, dll_path=dll_path) From 10611cbaceb13b2c5d4f683aa55b7f25bd644b91 Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Thu, 8 Dec 2022 20:36:48 +0000 Subject: [PATCH 21/49] update deps.txt --- cmake/deps.txt | 5 ++--- .../github/azure-pipelines/templates/download-deps.yml | 6 +++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/cmake/deps.txt b/cmake/deps.txt index 1cc6ed26dcd95..32d57e35181b8 100644 --- a/cmake/deps.txt +++ b/cmake/deps.txt @@ -24,8 +24,7 @@ microsoft_wil;https://github.com/microsoft/wil/archive/5f4caba4e7a9017816e47becd mimalloc;https://github.com/microsoft/mimalloc/archive/refs/tags/v2.0.3.zip;e4f37b93b2da78a5816c2495603a4188d316214b mp11;https://github.com/boostorg/mp11/archive/refs/tags/boost-1.79.0.zip;c8f04e378535ededbe5af52c8f969d2dedbe73d5 onnx;https://github.com/onnx/onnx/archive/5a5f8a5935762397aa68429b5493084ff970f774.zip;edc8e1338c02f3ab222f3d803a24e17608c13895 -#Branch name: 8.4-GA -onnx_tensorrt;https://github.com/onnx/onnx-tensorrt/archive/87c7a70688fd98fb355b8976f41425b40e4fe52f.zip;b97d112d9d6efa180c9b94e05268f2ff3294a534 +onnx_tensorrt;https://github.com/onnx/onnx-tensorrt/archive/369d6676423c2a6dbf4a5665c4b5010240d99d3c.zip;62119892edfb78689061790140c439b111491275 protobuf;https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.18.3.zip;b95bf7e9de9c2249b6c1f2ca556ace49999e90bd psimd;https://github.com/Maratyszcza/psimd/archive/072586a71b55b7f8c584153d223e95687148a900.zip;1f5454b01f06f9656b77e4a5e2e31d7422487013 pthreadpool;https://github.com/Maratyszcza/pthreadpool/archive/1787867f6183f056420e532eec640cba25efafea.zip;e43e80781560c5ab404a4da20f34d846f5f5d101 @@ -33,4 +32,4 @@ pybind11;https://github.com/pybind/pybind11/archive/refs/tags/v2.10.1.zip;769b6a pytorch_cpuinfo;https://github.com/pytorch/cpuinfo/archive/5916273f79a21551890fd3d56fc5375a78d1598d.zip;2be4d2ae321fada97cb39eaf4eeba5f8c85597cf re2;https://github.com/google/re2/archive/refs/tags/2022-06-01.zip;aa77313b76e91b531ee7f3e45f004c6a502a5374 safeint;https://github.com/dcleblanc/SafeInt/archive/ff15c6ada150a5018c5ef2172401cb4529eac9c0.zip;913a4046e5274d329af2806cb53194f617d8c0ab -tensorboard;https://github.com/tensorflow/tensorboard/archive/373eb09e4c5d2b3cc2493f0949dc4be6b6a45e81.zip;67b833913605a4f3f499894ab11528a702c2b381 \ No newline at end of file +tensorboard;https://github.com/tensorflow/tensorboard/archive/373eb09e4c5d2b3cc2493f0949dc4be6b6a45e81.zip;67b833913605a4f3f499894ab11528a702c2b381 diff --git a/tools/ci_build/github/azure-pipelines/templates/download-deps.yml b/tools/ci_build/github/azure-pipelines/templates/download-deps.yml index 03693edf7cf7e..5cbe400e9f455 100644 --- a/tools/ci_build/github/azure-pipelines/templates/download-deps.yml +++ b/tools/ci_build/github/azure-pipelines/templates/download-deps.yml @@ -11,7 +11,7 @@ steps: packageType: upack feed: '/7424c8e4-5c62-490e-95c4-79446f31017c' definition: '517c4f6f-5437-4392-a70d-4f15ec5be2f0' - version: 1.0.11 + version: 1.0.15 downloadPath: $(Build.BinariesDirectory)/deps # The private ADO project @@ -22,7 +22,7 @@ steps: packageType: upack feed: '/4c7631f5-24c0-4307-8822-1aa8f180c325' definition: 'fd9dd5ad-b73e-4678-890e-edcf680dbc1a' - version: 1.0.11 + version: 1.0.15 downloadPath: $(Build.BinariesDirectory)/deps -# You can add more ADO accounts at here. \ No newline at end of file +# You can add more ADO accounts at here. From 948279a89266229fbd356bb716f02796b07964b6 Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Thu, 8 Dec 2022 20:39:48 +0000 Subject: [PATCH 22/49] remove increased time since merging the main --- tools/ci_build/build.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index 09b6e2a305c96..a6e6b4cdc715b 100644 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -1881,10 +1881,7 @@ def run_onnxruntime_tests(args, source_dir, ctest_path, build_dir, configs): run_subprocess([os.path.join(cwd, exe)], cwd=cwd, dll_path=dll_path) else: - if args.use_tensorrt: - ctest_cmd = [ctest_path, "--build-config", config, "--verbose", "--timeout", "36000"] - else: - ctest_cmd = [ctest_path, "--build-config", config, "--verbose", "--timeout", "10800"] + ctest_cmd = [ctest_path, "--build-config", config, "--verbose", "--timeout", "10800"] run_subprocess(ctest_cmd, cwd=cwd, dll_path=dll_path) if args.enable_pybind: From a21b306bfa5da3230366d3a98d65162e85ff6c71 Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Thu, 8 Dec 2022 22:18:27 +0000 Subject: [PATCH 23/49] fix bug --- cmake/external/onnx-tensorrt | 1 - 1 file changed, 1 deletion(-) delete mode 160000 cmake/external/onnx-tensorrt diff --git a/cmake/external/onnx-tensorrt b/cmake/external/onnx-tensorrt deleted file mode 160000 index 87c7a70688fd9..0000000000000 --- a/cmake/external/onnx-tensorrt +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 87c7a70688fd98fb355b8976f41425b40e4fe52f From ff83678a0a9da3895f755a0a1e4f4e60c8b83ba3 Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Fri, 9 Dec 2022 19:29:21 +0000 Subject: [PATCH 24/49] include https://github.com/microsoft/onnxruntime/pull/13918 to fix compile issue --- cgmanifests/generate_cgmanifest.py | 3 ++- cmake/onnxruntime_providers.cmake | 12 +++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/cgmanifests/generate_cgmanifest.py b/cgmanifests/generate_cgmanifest.py index a85516011a402..bd2c85762115e 100644 --- a/cgmanifests/generate_cgmanifest.py +++ b/cgmanifests/generate_cgmanifest.py @@ -59,6 +59,7 @@ def add_github_dep(name, parsed_url): if dep not in git_deps: git_deps[dep] = name else: + # TODO: support urls like: https://github.com/onnx/onnx-tensorrt/archive/refs/tags/release/7.1.zip if len(segments) == 5: tag = PurePosixPath(segments[4]).stem if tag.endswith(".tar"): @@ -72,6 +73,7 @@ def add_github_dep(name, parsed_url): return # Make a REST call to convert to tag to a git commit url = "https://api.github.com/repos/%s/%s/git/refs/tags/%s" % (org_name, repo_name, tag) + print("requesting %s ..." % url) res = requests.get(url, auth=(args.username, args.token)) response_json = res.json() tag_object = response_json["object"] @@ -148,7 +150,6 @@ def normalize_path_separators(path): "submodule", "foreach", "--quiet", - "--recursive", "'{}' '{}' $toplevel/$sm_path".format( normalize_path_separators(sys.executable), normalize_path_separators(os.path.join(SCRIPT_DIR, "print_submodule_info.py")), diff --git a/cmake/onnxruntime_providers.cmake b/cmake/onnxruntime_providers.cmake index 99c985f9ee98e..0d8d7207adb1f 100644 --- a/cmake/onnxruntime_providers.cmake +++ b/cmake/onnxruntime_providers.cmake @@ -635,10 +635,12 @@ if (onnxruntime_USE_TENSORRT) FetchContent_Declare( onnx_tensorrt URL ${DEP_URL_onnx_tensorrt} - URL_HASH SHA1=${DEP_SHA1_onnx_tensorrt} + URL_HASH SHA1=${DEP_SHA1_onnx_tensorrt} ) - FetchContent_MakeAvailable(onnx_tensorrt) - include_directories(${onnx_tensorrt_SOURCE_DIR}) + # The onnx_tensorrt repo contains a test program, getSupportedAPITest, which doesn't support Windows. It uses + # unistd.h. So we must exclude it from our build. onnxruntime_fetchcontent_makeavailable is for the purpose. + onnxruntime_fetchcontent_makeavailable(onnx_tensorrt) + include_directories(${onnx_tensorrt_SOURCE_DIR}) set(CMAKE_CXX_FLAGS ${OLD_CMAKE_CXX_FLAGS}) if ( CMAKE_COMPILER_IS_GNUCC ) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-parameter") @@ -895,6 +897,7 @@ if (onnxruntime_USE_COREML) "${ONNXRUNTIME_ROOT}/core/providers/coreml/model/model.h" "${ONNXRUNTIME_ROOT}/core/providers/coreml/model/model.mm" "${ONNXRUNTIME_ROOT}/core/providers/coreml/model/host_utils.h" + ) "${ONNXRUNTIME_ROOT}/core/providers/coreml/model/host_utils.mm" ) endif() @@ -1125,7 +1128,6 @@ if (onnxruntime_USE_DML) target_compile_definitions(onnxruntime_providers_dml PRIVATE ONNX_NAMESPACE=onnx ONNX_ML LOTUS_LOG_THRESHOLD=2 LOTUS_ENABLE_STDERR_LOGGING PLATFORM_WINDOWS - ) target_compile_definitions(onnxruntime_providers_dml PRIVATE UNICODE _UNICODE NOMINMAX) if (MSVC) target_compile_definitions(onnxruntime_providers_dml PRIVATE _SILENCE_CXX17_ITERATOR_BASE_CLASS_DEPRECATION_WARNING) @@ -1479,7 +1481,7 @@ if (onnxruntime_USE_TVM) # wd4100: identifier' : unreferenced formal parameter # wd4127: conditional expression is constant # wd4244: conversion from 'int' to 'char', possible loss of data - # TODO: 4244 should not be disabled + # TODO: 4244 should not be disabled target_compile_options(onnxruntime_providers_tvm PRIVATE "/wd4100" "/wd4127" "/wd4244") else() target_compile_options(onnxruntime_providers_tvm PRIVATE "-Wno-error=type-limits") From ea0c76315af3580ce9e74fe62e40c6ad4d90d519 Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Fri, 9 Dec 2022 19:36:33 +0000 Subject: [PATCH 25/49] add comment to deps.txt --- cmake/deps.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/cmake/deps.txt b/cmake/deps.txt index 32d57e35181b8..ee487872f9246 100644 --- a/cmake/deps.txt +++ b/cmake/deps.txt @@ -24,6 +24,7 @@ microsoft_wil;https://github.com/microsoft/wil/archive/5f4caba4e7a9017816e47becd mimalloc;https://github.com/microsoft/mimalloc/archive/refs/tags/v2.0.3.zip;e4f37b93b2da78a5816c2495603a4188d316214b mp11;https://github.com/boostorg/mp11/archive/refs/tags/boost-1.79.0.zip;c8f04e378535ededbe5af52c8f969d2dedbe73d5 onnx;https://github.com/onnx/onnx/archive/5a5f8a5935762397aa68429b5493084ff970f774.zip;edc8e1338c02f3ab222f3d803a24e17608c13895 +#use the commit where it's several commits after 8.5-GA branch (https://github.com/onnx/onnx-tensorrt/commit/369d6676423c2a6dbf4a5665c4b5010240d99d3c) onnx_tensorrt;https://github.com/onnx/onnx-tensorrt/archive/369d6676423c2a6dbf4a5665c4b5010240d99d3c.zip;62119892edfb78689061790140c439b111491275 protobuf;https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.18.3.zip;b95bf7e9de9c2249b6c1f2ca556ace49999e90bd psimd;https://github.com/Maratyszcza/psimd/archive/072586a71b55b7f8c584153d223e95687148a900.zip;1f5454b01f06f9656b77e4a5e2e31d7422487013 From 01741c1f26e383635bdf7e4e043fea3b1a8931e3 Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Fri, 9 Dec 2022 19:56:25 +0000 Subject: [PATCH 26/49] fix bug --- cmake/onnxruntime_providers.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/onnxruntime_providers.cmake b/cmake/onnxruntime_providers.cmake index 0d8d7207adb1f..b80e75e8c5342 100644 --- a/cmake/onnxruntime_providers.cmake +++ b/cmake/onnxruntime_providers.cmake @@ -897,7 +897,6 @@ if (onnxruntime_USE_COREML) "${ONNXRUNTIME_ROOT}/core/providers/coreml/model/model.h" "${ONNXRUNTIME_ROOT}/core/providers/coreml/model/model.mm" "${ONNXRUNTIME_ROOT}/core/providers/coreml/model/host_utils.h" - ) "${ONNXRUNTIME_ROOT}/core/providers/coreml/model/host_utils.mm" ) endif() @@ -1128,6 +1127,7 @@ if (onnxruntime_USE_DML) target_compile_definitions(onnxruntime_providers_dml PRIVATE ONNX_NAMESPACE=onnx ONNX_ML LOTUS_LOG_THRESHOLD=2 LOTUS_ENABLE_STDERR_LOGGING PLATFORM_WINDOWS + ) target_compile_definitions(onnxruntime_providers_dml PRIVATE UNICODE _UNICODE NOMINMAX) if (MSVC) target_compile_definitions(onnxruntime_providers_dml PRIVATE _SILENCE_CXX17_ITERATOR_BASE_CLASS_DEPRECATION_WARNING) From c32458c7a5b3b76a58af15a92062a7404cb84978 Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Fri, 9 Dec 2022 21:56:06 +0000 Subject: [PATCH 27/49] increase timeout --- tools/ci_build/build.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index a6e6b4cdc715b..5aea69fd6bbf8 100644 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -1881,7 +1881,14 @@ def run_onnxruntime_tests(args, source_dir, ctest_path, build_dir, configs): run_subprocess([os.path.join(cwd, exe)], cwd=cwd, dll_path=dll_path) else: - ctest_cmd = [ctest_path, "--build-config", config, "--verbose", "--timeout", "10800"] + if args.use_tensorrt: + # Using --tensorrt_placeholder_builder can help reduce CI testing time. + # However, in C-API application test in package pipelines, with this flag will cause ORT TRT to deadlock, + # so we remove this flag for package pipelines. + # For package pipelines with TRT 8.5 GA, we observe increased testing time, hence increase the timeout as workaround. + ctest_cmd = [ctest_path, "--build-config", config, "--verbose", "--timeout", "36000"] + else: + ctest_cmd = [ctest_path, "--build-config", config, "--verbose", "--timeout", "10800"] run_subprocess(ctest_cmd, cwd=cwd, dll_path=dll_path) if args.enable_pybind: From 2bcdef458937e6f39e80a71a409c12c3d57865be Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Fri, 9 Dec 2022 22:14:24 +0000 Subject: [PATCH 28/49] fix python format --- tools/ci_build/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index 5aea69fd6bbf8..5d745ebe1a44c 100644 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -1883,7 +1883,7 @@ def run_onnxruntime_tests(args, source_dir, ctest_path, build_dir, configs): else: if args.use_tensorrt: # Using --tensorrt_placeholder_builder can help reduce CI testing time. - # However, in C-API application test in package pipelines, with this flag will cause ORT TRT to deadlock, + # However, in C-API application test in package pipelines, with this flag will cause ORT TRT to deadlock, # so we remove this flag for package pipelines. # For package pipelines with TRT 8.5 GA, we observe increased testing time, hence increase the timeout as workaround. ctest_cmd = [ctest_path, "--build-config", config, "--verbose", "--timeout", "36000"] From 9c199c1bcb40ebc8f7005e8cbeef3d52e62807d0 Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Fri, 9 Dec 2022 22:37:13 +0000 Subject: [PATCH 29/49] format compliance --- tools/ci_build/build.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index 5d745ebe1a44c..9444878f800e8 100644 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -1883,9 +1883,10 @@ def run_onnxruntime_tests(args, source_dir, ctest_path, build_dir, configs): else: if args.use_tensorrt: # Using --tensorrt_placeholder_builder can help reduce CI testing time. - # However, in C-API application test in package pipelines, with this flag will cause ORT TRT to deadlock, - # so we remove this flag for package pipelines. - # For package pipelines with TRT 8.5 GA, we observe increased testing time, hence increase the timeout as workaround. + # However, in C-API application test in package pipelines, with this flag will cause + # ORT TRT to deadlock, so we remove this flag for package pipelines. + # For package pipelines with TRT 8.5 GA, we observe increased testing time, hence + # increase the timeout as workaround. ctest_cmd = [ctest_path, "--build-config", config, "--verbose", "--timeout", "36000"] else: ctest_cmd = [ctest_path, "--build-config", config, "--verbose", "--timeout", "10800"] From e1d6aafa2c41d26f03e3f54dbd5bbc3981c1c2d4 Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Sat, 10 Dec 2022 06:23:50 +0000 Subject: [PATCH 30/49] increase timeout for package pipeline --- .../azure-pipelines/c-api-noopenmp-packaging-pipelines.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml b/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml index 328824fdefa59..3c1d9ff142b4c 100644 --- a/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml +++ b/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml @@ -120,6 +120,7 @@ jobs: runTests: ${{ parameters.RunOnnxRuntimeTests }} buildJava: true java_artifact_id: onnxruntime_gpu + timeoutInMinutes: 600 - job: Jar_Packaging_GPU workspace: From af7d16992f7600c5c2902e5464b43d8c8c2111ff Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Sat, 10 Dec 2022 06:31:11 +0000 Subject: [PATCH 31/49] fix bug for increasing timeout for package pipeline --- tools/ci_build/github/azure-pipelines/templates/win-ci.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/ci_build/github/azure-pipelines/templates/win-ci.yml b/tools/ci_build/github/azure-pipelines/templates/win-ci.yml index 59c0c53fd8e39..8e49ad5607c3f 100644 --- a/tools/ci_build/github/azure-pipelines/templates/win-ci.yml +++ b/tools/ci_build/github/azure-pipelines/templates/win-ci.yml @@ -52,12 +52,16 @@ parameters: type: string default: 'onnxruntime' +- name: timeoutInMinutes + type: integer + default: 300 + jobs: - job: Windows_Packaging_${{ parameters.job_name_suffix }} workspace: clean: all pool: ${{ parameters.ort_build_pool_name }} - timeoutInMinutes: 300 + timeoutInMinutes: ${{ parameters.timeoutInMinutes }} steps: - checkout: self From 35ab89707dec8263ed29d11f2cb9e03668ef9f45 Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Sat, 10 Dec 2022 06:33:21 +0000 Subject: [PATCH 32/49] fix bug for increasing timeout for package pipeline --- tools/ci_build/github/azure-pipelines/templates/win-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/github/azure-pipelines/templates/win-ci.yml b/tools/ci_build/github/azure-pipelines/templates/win-ci.yml index 8e49ad5607c3f..5d93fb352835b 100644 --- a/tools/ci_build/github/azure-pipelines/templates/win-ci.yml +++ b/tools/ci_build/github/azure-pipelines/templates/win-ci.yml @@ -53,7 +53,7 @@ parameters: default: 'onnxruntime' - name: timeoutInMinutes - type: integer + type: number default: 300 jobs: From a1fded41736e98644a37056ba53213ba7f37f044 Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Sat, 10 Dec 2022 23:57:52 +0000 Subject: [PATCH 33/49] test CUDA_MODULE_LOADING=LAZY --- tools/ci_build/github/windows/setup_env_gpu.bat | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/ci_build/github/windows/setup_env_gpu.bat b/tools/ci_build/github/windows/setup_env_gpu.bat index 1ef315aee722f..57adb11d9056b 100644 --- a/tools/ci_build/github/windows/setup_env_gpu.bat +++ b/tools/ci_build/github/windows/setup_env_gpu.bat @@ -3,3 +3,4 @@ REM Licensed under the MIT License. set PATH=C:\local\TensorRT-8.5.1.7.Windows10.x86_64.cuda-11.8.cudnn8.6\lib;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.6\bin;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.6\extras\CUPTI\lib64;C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Current\Bin;%PATH% set GRADLE_OPTS=-Dorg.gradle.daemon=false +set CUDA_MODULE_LOADING=LAZY From a62f6e19e1c947c2d6d7b8c1006885e93dbe1387 Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Sun, 11 Dec 2022 07:36:59 +0000 Subject: [PATCH 34/49] skip unnecessay and time-consuming unittests for TRT EP --- cmake/onnxruntime_unittests.cmake | 9 +++++++-- tools/ci_build/build.py | 2 ++ .../c-api-noopenmp-packaging-pipelines.yml | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/cmake/onnxruntime_unittests.cmake b/cmake/onnxruntime_unittests.cmake index 43e79a2dd6e88..e66f4dff9061a 100644 --- a/cmake/onnxruntime_unittests.cmake +++ b/cmake/onnxruntime_unittests.cmake @@ -713,8 +713,13 @@ if (onnxruntime_USE_TENSORRT) # # The test names of model tests were using sequential number in the past. # This PR https://github.com/microsoft/onnxruntime/pull/10220 (Please see ExpandModelName function in model_tests.cc for more details) - # made test name contain the "ep" and "model path" information, so we can easily filter the tests using cuda ep or other ep with *cpu__* or *xxx__*. - list(APPEND test_all_args "--gtest_filter=-*cpu__*:*cuda__*" ) + # made test name contain the "ep" and "model path" information, so we can easily filter the tests using cuda ep or other ep with *cpu_* or *xxx_*. + if (onnxruntime_SKIP_UNNECESSARY_TENSORRT_UNITTESTS) + list(APPEND test_all_args "--gtest_filter=-*cpu_*:*cuda_*:*ContribOpTest*:*QuantGemmTest*:*QLinearConvTest*:*MurmurHash3OpTest*" ) + else() + list(APPEND test_all_args "--gtest_filter=-*cpu_*:*cuda_*" ) + endif() + endif () AddTest( diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index 9444878f800e8..5a677bb9c1c08 100644 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -491,6 +491,7 @@ def convert_arg_line_to_args(self, arg_line): "--tensorrt_placeholder_builder", action="store_true", help="Instantiate Placeholder TensorRT Builder" ) parser.add_argument("--tensorrt_home", help="Path to TensorRT installation dir") + parser.add_argument("--skip_unnecessary_tensorrt_unittests", action="store_true", help="Skip unnecessary and time-consuming unittests for TensorRT EP") parser.add_argument("--use_migraphx", action="store_true", help="Build with MIGraphX") parser.add_argument("--migraphx_home", help="Path to MIGraphX installation dir") parser.add_argument("--use_full_protobuf", action="store_true", help="Use the full protobuf library") @@ -876,6 +877,7 @@ def generate_build_tree( "-Donnxruntime_ENABLE_MICROSOFT_INTERNAL=" + ("ON" if args.enable_msinternal else "OFF"), "-Donnxruntime_USE_VITISAI=" + ("ON" if args.use_vitisai else "OFF"), "-Donnxruntime_USE_TENSORRT=" + ("ON" if args.use_tensorrt else "OFF"), + "-Donnxruntime_SKIP_UNNECESSARY_TENSORRT_UNITTESTS=" + ("ON" if args.skip_unnecessary_tensorrt_unittests else "OFF"), "-Donnxruntime_USE_TENSORRT_BUILTIN_PARSER=" + ("ON" if args.use_tensorrt_builtin_parser else "OFF"), "-Donnxruntime_TENSORRT_PLACEHOLDER_BUILDER=" + ("ON" if args.tensorrt_placeholder_builder else "OFF"), # set vars for TVM diff --git a/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml b/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml index 3c1d9ff142b4c..40da6fbea4ca1 100644 --- a/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml +++ b/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml @@ -116,7 +116,7 @@ jobs: buildArch: x64 msbuildPlatform: x64 packageName: x64-tensorrt - buildparameter: --use_tensorrt --tensorrt_home="C:\local\TensorRT-8.5.1.7.Windows10.x86_64.cuda-11.8.cudnn8.6" --cuda_version=11.6 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.6" --enable_onnx_tests --enable_wcos --build_java --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=37;52;60;61;70;75;80" + buildparameter: --use_tensorrt --skip_unnecessary_tensorrt_unittests --tensorrt_home="C:\local\TensorRT-8.5.1.7.Windows10.x86_64.cuda-11.8.cudnn8.6" --cuda_version=11.6 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.6" --enable_onnx_tests --enable_wcos --build_java --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=37;52;60;61;70;75;80" runTests: ${{ parameters.RunOnnxRuntimeTests }} buildJava: true java_artifact_id: onnxruntime_gpu From d7c7ba7f54b75cb2cf61b6df4a9465fc42881263 Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Sun, 11 Dec 2022 17:33:05 +0000 Subject: [PATCH 35/49] only run TRT related tests --- cmake/onnxruntime_unittests.cmake | 17 ++++++++++------- tools/ci_build/build.py | 9 +++++++-- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/cmake/onnxruntime_unittests.cmake b/cmake/onnxruntime_unittests.cmake index e66f4dff9061a..67e190c999f15 100644 --- a/cmake/onnxruntime_unittests.cmake +++ b/cmake/onnxruntime_unittests.cmake @@ -708,15 +708,18 @@ endif() set(test_all_args) if (onnxruntime_USE_TENSORRT) - # TRT EP CI takes much longer time when updating to TRT 8.2 - # So, we only run trt ep and exclude other eps to reduce CI test time. - # - # The test names of model tests were using sequential number in the past. - # This PR https://github.com/microsoft/onnxruntime/pull/10220 (Please see ExpandModelName function in model_tests.cc for more details) - # made test name contain the "ep" and "model path" information, so we can easily filter the tests using cuda ep or other ep with *cpu_* or *xxx_*. if (onnxruntime_SKIP_UNNECESSARY_TENSORRT_UNITTESTS) - list(APPEND test_all_args "--gtest_filter=-*cpu_*:*cuda_*:*ContribOpTest*:*QuantGemmTest*:*QLinearConvTest*:*MurmurHash3OpTest*" ) + # TRT EP package pipelines takes much longer time to run tests with TRT 8.5. We don't use place holder to reduce testing time due to application test deadlock. + # Therefore we only run TRT EP related tests. + list(APPEND test_all_args "--gtest_filter=*tensorrt_*:*TensorrtExecutionProviderTest*" ) + #list(APPEND test_all_args "--gtest_filter=-*cpu_*:*cuda_*:*ContribOpTest*:*QuantGemmTest*:*QLinearConvTest*:*MurmurHash3OpTest*:*PadOpTest*:*QLinearConvTest*" ) else() + # TRT EP CI takes much longer time when updating to TRT 8.2 + # So, we only run trt ep and exclude other eps to reduce CI test time. + # + # The test names of model tests were using sequential number in the past. + # This PR https://github.com/microsoft/onnxruntime/pull/10220 (Please see ExpandModelName function in model_tests.cc for more details) + # made test name contain the "ep" and "model path" information, so we can easily filter the tests using cuda ep or other ep with *cpu_* or *xxx_*. list(APPEND test_all_args "--gtest_filter=-*cpu_*:*cuda_*" ) endif() diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index 5a677bb9c1c08..155f187170506 100644 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -491,7 +491,11 @@ def convert_arg_line_to_args(self, arg_line): "--tensorrt_placeholder_builder", action="store_true", help="Instantiate Placeholder TensorRT Builder" ) parser.add_argument("--tensorrt_home", help="Path to TensorRT installation dir") - parser.add_argument("--skip_unnecessary_tensorrt_unittests", action="store_true", help="Skip unnecessary and time-consuming unittests for TensorRT EP") + parser.add_argument( + "--skip_unnecessary_tensorrt_unittests", + action="store_true", + help="Skip unnecessary and time-consuming unittests for TensorRT EP", + ) parser.add_argument("--use_migraphx", action="store_true", help="Build with MIGraphX") parser.add_argument("--migraphx_home", help="Path to MIGraphX installation dir") parser.add_argument("--use_full_protobuf", action="store_true", help="Use the full protobuf library") @@ -877,7 +881,8 @@ def generate_build_tree( "-Donnxruntime_ENABLE_MICROSOFT_INTERNAL=" + ("ON" if args.enable_msinternal else "OFF"), "-Donnxruntime_USE_VITISAI=" + ("ON" if args.use_vitisai else "OFF"), "-Donnxruntime_USE_TENSORRT=" + ("ON" if args.use_tensorrt else "OFF"), - "-Donnxruntime_SKIP_UNNECESSARY_TENSORRT_UNITTESTS=" + ("ON" if args.skip_unnecessary_tensorrt_unittests else "OFF"), ++ "-Donnxruntime_SKIP_UNNECESSARY_TENSORRT_UNITTESTS=" ++ + ("ON" if args.skip_unnecessary_tensorrt_unittests else "OFF"), "-Donnxruntime_USE_TENSORRT_BUILTIN_PARSER=" + ("ON" if args.use_tensorrt_builtin_parser else "OFF"), "-Donnxruntime_TENSORRT_PLACEHOLDER_BUILDER=" + ("ON" if args.tensorrt_placeholder_builder else "OFF"), # set vars for TVM From 0e89046b1a82cc0d537fa10e46178fb40d2f434b Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Sun, 11 Dec 2022 17:42:41 +0000 Subject: [PATCH 36/49] fix format bug --- tools/ci_build/build.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index 155f187170506..70b6c32b0f406 100644 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -881,8 +881,8 @@ def generate_build_tree( "-Donnxruntime_ENABLE_MICROSOFT_INTERNAL=" + ("ON" if args.enable_msinternal else "OFF"), "-Donnxruntime_USE_VITISAI=" + ("ON" if args.use_vitisai else "OFF"), "-Donnxruntime_USE_TENSORRT=" + ("ON" if args.use_tensorrt else "OFF"), -+ "-Donnxruntime_SKIP_UNNECESSARY_TENSORRT_UNITTESTS=" -+ + ("ON" if args.skip_unnecessary_tensorrt_unittests else "OFF"), + "-Donnxruntime_SKIP_UNNECESSARY_TENSORRT_UNITTESTS=" + + ("ON" if args.skip_unnecessary_tensorrt_unittests else "OFF"), "-Donnxruntime_USE_TENSORRT_BUILTIN_PARSER=" + ("ON" if args.use_tensorrt_builtin_parser else "OFF"), "-Donnxruntime_TENSORRT_PLACEHOLDER_BUILDER=" + ("ON" if args.tensorrt_placeholder_builder else "OFF"), # set vars for TVM From 63d664b7f24c1d27533c4b3a5ac0d6e619c26056 Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Mon, 12 Dec 2022 02:00:55 +0000 Subject: [PATCH 37/49] rename flag --- cmake/onnxruntime_unittests.cmake | 6 +++--- tools/ci_build/build.py | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cmake/onnxruntime_unittests.cmake b/cmake/onnxruntime_unittests.cmake index 67e190c999f15..aa224d753a997 100644 --- a/cmake/onnxruntime_unittests.cmake +++ b/cmake/onnxruntime_unittests.cmake @@ -708,9 +708,9 @@ endif() set(test_all_args) if (onnxruntime_USE_TENSORRT) - if (onnxruntime_SKIP_UNNECESSARY_TENSORRT_UNITTESTS) - # TRT EP package pipelines takes much longer time to run tests with TRT 8.5. We don't use place holder to reduce testing time due to application test deadlock. - # Therefore we only run TRT EP related tests. + if (onnxruntime_SKIP_AND_PERFORM_FILTERED_TENSORRT_TESTS) + # TRT EP package pipelines takes much longer time to run tests with TRT 8.5. We can't use placeholder to reduce testing time due to application test deadlock. + # Therefore we only run filtered TRT EP tests. list(APPEND test_all_args "--gtest_filter=*tensorrt_*:*TensorrtExecutionProviderTest*" ) #list(APPEND test_all_args "--gtest_filter=-*cpu_*:*cuda_*:*ContribOpTest*:*QuantGemmTest*:*QLinearConvTest*:*MurmurHash3OpTest*:*PadOpTest*:*QLinearConvTest*" ) else() diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index 70b6c32b0f406..c7e1845d5b808 100644 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -492,9 +492,9 @@ def convert_arg_line_to_args(self, arg_line): ) parser.add_argument("--tensorrt_home", help="Path to TensorRT installation dir") parser.add_argument( - "--skip_unnecessary_tensorrt_unittests", + "--skip_and_perform_filtered_tensorrt_tests", action="store_true", - help="Skip unnecessary and time-consuming unittests for TensorRT EP", + help="Skip time-consuming and only perform filtered tests for TensorRT EP", ) parser.add_argument("--use_migraphx", action="store_true", help="Build with MIGraphX") parser.add_argument("--migraphx_home", help="Path to MIGraphX installation dir") @@ -881,8 +881,8 @@ def generate_build_tree( "-Donnxruntime_ENABLE_MICROSOFT_INTERNAL=" + ("ON" if args.enable_msinternal else "OFF"), "-Donnxruntime_USE_VITISAI=" + ("ON" if args.use_vitisai else "OFF"), "-Donnxruntime_USE_TENSORRT=" + ("ON" if args.use_tensorrt else "OFF"), - "-Donnxruntime_SKIP_UNNECESSARY_TENSORRT_UNITTESTS=" - + ("ON" if args.skip_unnecessary_tensorrt_unittests else "OFF"), + "-Donnxruntime_SKIP_AND_PERFORM_FILTERED_TENSORRT_TESTS=" + + ("ON" if args.skip_and_perform_filtered_tensorrt_tests else "OFF"), "-Donnxruntime_USE_TENSORRT_BUILTIN_PARSER=" + ("ON" if args.use_tensorrt_builtin_parser else "OFF"), "-Donnxruntime_TENSORRT_PLACEHOLDER_BUILDER=" + ("ON" if args.tensorrt_placeholder_builder else "OFF"), # set vars for TVM From f390402fa692d1ca8f53fb9ecf2e1766afb251d9 Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Mon, 12 Dec 2022 02:06:14 +0000 Subject: [PATCH 38/49] remove timeout for TRT EP unittests --- tools/ci_build/build.py | 10 +--------- .../github/azure-pipelines/templates/win-ci.yml | 6 +----- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index c7e1845d5b808..b7c118db8ea5c 100644 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -1888,15 +1888,7 @@ def run_onnxruntime_tests(args, source_dir, ctest_path, build_dir, configs): run_subprocess([os.path.join(cwd, exe)], cwd=cwd, dll_path=dll_path) else: - if args.use_tensorrt: - # Using --tensorrt_placeholder_builder can help reduce CI testing time. - # However, in C-API application test in package pipelines, with this flag will cause - # ORT TRT to deadlock, so we remove this flag for package pipelines. - # For package pipelines with TRT 8.5 GA, we observe increased testing time, hence - # increase the timeout as workaround. - ctest_cmd = [ctest_path, "--build-config", config, "--verbose", "--timeout", "36000"] - else: - ctest_cmd = [ctest_path, "--build-config", config, "--verbose", "--timeout", "10800"] + ctest_cmd = [ctest_path, "--build-config", config, "--verbose", "--timeout", "10800"] run_subprocess(ctest_cmd, cwd=cwd, dll_path=dll_path) if args.enable_pybind: diff --git a/tools/ci_build/github/azure-pipelines/templates/win-ci.yml b/tools/ci_build/github/azure-pipelines/templates/win-ci.yml index 5d93fb352835b..27f7054813ff4 100644 --- a/tools/ci_build/github/azure-pipelines/templates/win-ci.yml +++ b/tools/ci_build/github/azure-pipelines/templates/win-ci.yml @@ -52,16 +52,12 @@ parameters: type: string default: 'onnxruntime' -- name: timeoutInMinutes - type: number - default: 300 - jobs: - job: Windows_Packaging_${{ parameters.job_name_suffix }} workspace: clean: all pool: ${{ parameters.ort_build_pool_name }} - timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + timeoutInMinutes: 300 steps: - checkout: self From 5fc64cac5d59011d555b24fc09e945dd34b6e67f Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Mon, 12 Dec 2022 02:12:45 +0000 Subject: [PATCH 39/49] remove timeout for TRT EP unittests --- .../azure-pipelines/c-api-noopenmp-packaging-pipelines.yml | 1 - tools/ci_build/github/azure-pipelines/templates/win-ci.yml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml b/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml index 40da6fbea4ca1..c7ac90b0da213 100644 --- a/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml +++ b/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml @@ -120,7 +120,6 @@ jobs: runTests: ${{ parameters.RunOnnxRuntimeTests }} buildJava: true java_artifact_id: onnxruntime_gpu - timeoutInMinutes: 600 - job: Jar_Packaging_GPU workspace: diff --git a/tools/ci_build/github/azure-pipelines/templates/win-ci.yml b/tools/ci_build/github/azure-pipelines/templates/win-ci.yml index 27f7054813ff4..59c0c53fd8e39 100644 --- a/tools/ci_build/github/azure-pipelines/templates/win-ci.yml +++ b/tools/ci_build/github/azure-pipelines/templates/win-ci.yml @@ -57,7 +57,7 @@ jobs: workspace: clean: all pool: ${{ parameters.ort_build_pool_name }} - timeoutInMinutes: 300 + timeoutInMinutes: 300 steps: - checkout: self From ed16a9b32fb6c3784fcd55bfcef5a457866543d9 Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Mon, 12 Dec 2022 02:28:05 +0000 Subject: [PATCH 40/49] add --skip_and_perform_filtered_tensorrt_tests to package pipeline --- .../azure-pipelines/c-api-noopenmp-packaging-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml b/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml index c7ac90b0da213..79c9b3cb2c975 100644 --- a/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml +++ b/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml @@ -116,7 +116,7 @@ jobs: buildArch: x64 msbuildPlatform: x64 packageName: x64-tensorrt - buildparameter: --use_tensorrt --skip_unnecessary_tensorrt_unittests --tensorrt_home="C:\local\TensorRT-8.5.1.7.Windows10.x86_64.cuda-11.8.cudnn8.6" --cuda_version=11.6 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.6" --enable_onnx_tests --enable_wcos --build_java --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=37;52;60;61;70;75;80" + buildparameter: --use_tensorrt --skip_and_perform_filtered_tensorrt_tests --tensorrt_home="C:\local\TensorRT-8.5.1.7.Windows10.x86_64.cuda-11.8.cudnn8.6" --cuda_version=11.6 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.6" --enable_onnx_tests --enable_wcos --build_java --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=37;52;60;61;70;75;80" runTests: ${{ parameters.RunOnnxRuntimeTests }} buildJava: true java_artifact_id: onnxruntime_gpu From 61fdf47d5a1f591fe8547fc10a48b2874c48e4cd Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Mon, 12 Dec 2022 20:30:22 +0000 Subject: [PATCH 41/49] make timeout configurable --- tools/ci_build/build.py | 6 ++- .../c-api-noopenmp-packaging-pipelines.yml | 51 ++++++++++++++----- .../azure-pipelines/templates/win-ci.yml | 6 ++- 3 files changed, 47 insertions(+), 16 deletions(-) diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index b7c118db8ea5c..e32de9bc5f2b2 100644 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -1888,7 +1888,11 @@ def run_onnxruntime_tests(args, source_dir, ctest_path, build_dir, configs): run_subprocess([os.path.join(cwd, exe)], cwd=cwd, dll_path=dll_path) else: - ctest_cmd = [ctest_path, "--build-config", config, "--verbose", "--timeout", "10800"] + if args.use_tensorrt and not args.skip_and_perform_filtered_tensorrt_tests + # TensorRT 8.5 needs more time to run tests on Windows + ctest_cmd = [ctest_path, "--build-config", config, "--verbose", "--timeout", "72000"] + else: + ctest_cmd = [ctest_path, "--build-config", config, "--verbose", "--timeout", "10800"] run_subprocess(ctest_cmd, cwd=cwd, dll_path=dll_path) if args.enable_pybind: diff --git a/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml b/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml index 79c9b3cb2c975..fcb5ef0525d20 100644 --- a/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml +++ b/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml @@ -4,6 +4,11 @@ parameters: type: boolean default: true +- name: UseIncreasedTimeoutForTests + displayName: Increase timeout for tests? Set it to false if you are doing an Onnx Runtime release. + type: boolean + default: false + - name: DoCompliance displayName: Run Compliance Tasks? type: boolean @@ -106,20 +111,38 @@ jobs: buildJava: true java_artifact_id: onnxruntime_gpu -- template: templates/win-ci.yml - parameters: - ort_build_pool_name: 'onnxruntime-gpu-tensorrt8-winbuild-t4' - DoCompliance: ${{ parameters.DoCompliance }} - DoEsrp: ${{ parameters.DoEsrp }} - job_name_suffix: tensorrt - EnvSetupScript: setup_env_gpu.bat - buildArch: x64 - msbuildPlatform: x64 - packageName: x64-tensorrt - buildparameter: --use_tensorrt --skip_and_perform_filtered_tensorrt_tests --tensorrt_home="C:\local\TensorRT-8.5.1.7.Windows10.x86_64.cuda-11.8.cudnn8.6" --cuda_version=11.6 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.6" --enable_onnx_tests --enable_wcos --build_java --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=37;52;60;61;70;75;80" - runTests: ${{ parameters.RunOnnxRuntimeTests }} - buildJava: true - java_artifact_id: onnxruntime_gpu +- ${{ if eq(parameters['UseIncreasedTimeoutForTests'], 'true') }}: + - template: templates/win-ci.yml + parameters: + ort_build_pool_name: 'onnxruntime-gpu-tensorrt8-winbuild-t4' + DoCompliance: ${{ parameters.DoCompliance }} + DoEsrp: ${{ parameters.DoEsrp }} + job_name_suffix: tensorrt + EnvSetupScript: setup_env_gpu.bat + buildArch: x64 + msbuildPlatform: x64 + packageName: x64-tensorrt + buildparameter: --use_tensorrt --tensorrt_home="C:\local\TensorRT-8.5.1.7.Windows10.x86_64.cuda-11.8.cudnn8.6" --cuda_version=11.6 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.6" --enable_onnx_tests --enable_wcos --build_java --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=37;52;60;61;70;75;80" + runTests: ${{ parameters.RunOnnxRuntimeTests }} + buildJava: true + java_artifact_id: onnxruntime_gpu + timeoutInMinutes: 1200 +${{ else }}: + - template: templates/win-ci.yml + parameters: + ort_build_pool_name: 'onnxruntime-gpu-tensorrt8-winbuild-t4' + DoCompliance: ${{ parameters.DoCompliance }} + DoEsrp: ${{ parameters.DoEsrp }} + job_name_suffix: tensorrt + EnvSetupScript: setup_env_gpu.bat + buildArch: x64 + msbuildPlatform: x64 + packageName: x64-tensorrt + buildparameter: --use_tensorrt --skip_and_perform_filtered_tensorrt_tests --tensorrt_home="C:\local\TensorRT-8.5.1.7.Windows10.x86_64.cuda-11.8.cudnn8.6" --cuda_version=11.6 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.6" --enable_onnx_tests --enable_wcos --build_java --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=37;52;60;61;70;75;80" + runTests: ${{ parameters.RunOnnxRuntimeTests }} + buildJava: true + java_artifact_id: onnxruntime_gpu + timeoutInMinutes: 600 - job: Jar_Packaging_GPU workspace: diff --git a/tools/ci_build/github/azure-pipelines/templates/win-ci.yml b/tools/ci_build/github/azure-pipelines/templates/win-ci.yml index 59c0c53fd8e39..0d478b1843970 100644 --- a/tools/ci_build/github/azure-pipelines/templates/win-ci.yml +++ b/tools/ci_build/github/azure-pipelines/templates/win-ci.yml @@ -52,12 +52,16 @@ parameters: type: string default: 'onnxruntime' +- name: timeoutInMinutes + type: number + default: 300 + jobs: - job: Windows_Packaging_${{ parameters.job_name_suffix }} workspace: clean: all pool: ${{ parameters.ort_build_pool_name }} - timeoutInMinutes: 300 + timeoutInMinutes: ${{ parameters.timeoutInMinutes }} steps: - checkout: self From 11c9d2956724713831c8c5ab1c41d908b7b726a5 Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Mon, 12 Dec 2022 20:33:21 +0000 Subject: [PATCH 42/49] make timeout configurable (cont.) --- .../azure-pipelines/c-api-noopenmp-packaging-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml b/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml index fcb5ef0525d20..788294f48c55d 100644 --- a/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml +++ b/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml @@ -127,7 +127,7 @@ jobs: buildJava: true java_artifact_id: onnxruntime_gpu timeoutInMinutes: 1200 -${{ else }}: +- ${{ if eq(parameters['UseIncreasedTimeoutForTests'], 'false') }}: - template: templates/win-ci.yml parameters: ort_build_pool_name: 'onnxruntime-gpu-tensorrt8-winbuild-t4' From c3376d937971b798620bf3f2fcdbdc76b1594bb7 Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Mon, 12 Dec 2022 22:06:41 +0000 Subject: [PATCH 43/49] make timeout configurable --- tools/ci_build/build.py | 6 +-- .../c-api-noopenmp-packaging-pipelines.yml | 47 ++++++------------- .../azure-pipelines/templates/win-ci.yml | 19 +++++++- 3 files changed, 35 insertions(+), 37 deletions(-) diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index e32de9bc5f2b2..4cacd24bfccae 100644 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -492,7 +492,7 @@ def convert_arg_line_to_args(self, arg_line): ) parser.add_argument("--tensorrt_home", help="Path to TensorRT installation dir") parser.add_argument( - "--skip_and_perform_filtered_tensorrt_tests", + "--skip_and_perform_filtered_tests", action="store_true", help="Skip time-consuming and only perform filtered tests for TensorRT EP", ) @@ -882,7 +882,7 @@ def generate_build_tree( "-Donnxruntime_USE_VITISAI=" + ("ON" if args.use_vitisai else "OFF"), "-Donnxruntime_USE_TENSORRT=" + ("ON" if args.use_tensorrt else "OFF"), "-Donnxruntime_SKIP_AND_PERFORM_FILTERED_TENSORRT_TESTS=" - + ("ON" if args.skip_and_perform_filtered_tensorrt_tests else "OFF"), + + ("ON" if args.skip_and_perform_filtered_tests else "OFF"), "-Donnxruntime_USE_TENSORRT_BUILTIN_PARSER=" + ("ON" if args.use_tensorrt_builtin_parser else "OFF"), "-Donnxruntime_TENSORRT_PLACEHOLDER_BUILDER=" + ("ON" if args.tensorrt_placeholder_builder else "OFF"), # set vars for TVM @@ -1888,7 +1888,7 @@ def run_onnxruntime_tests(args, source_dir, ctest_path, build_dir, configs): run_subprocess([os.path.join(cwd, exe)], cwd=cwd, dll_path=dll_path) else: - if args.use_tensorrt and not args.skip_and_perform_filtered_tensorrt_tests + if args.use_tensorrt and not args.skip_and_perform_filtered_tests # TensorRT 8.5 needs more time to run tests on Windows ctest_cmd = [ctest_path, "--build-config", config, "--verbose", "--timeout", "72000"] else: diff --git a/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml b/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml index 788294f48c55d..b13bd25e61d47 100644 --- a/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml +++ b/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml @@ -111,38 +111,21 @@ jobs: buildJava: true java_artifact_id: onnxruntime_gpu -- ${{ if eq(parameters['UseIncreasedTimeoutForTests'], 'true') }}: - - template: templates/win-ci.yml - parameters: - ort_build_pool_name: 'onnxruntime-gpu-tensorrt8-winbuild-t4' - DoCompliance: ${{ parameters.DoCompliance }} - DoEsrp: ${{ parameters.DoEsrp }} - job_name_suffix: tensorrt - EnvSetupScript: setup_env_gpu.bat - buildArch: x64 - msbuildPlatform: x64 - packageName: x64-tensorrt - buildparameter: --use_tensorrt --tensorrt_home="C:\local\TensorRT-8.5.1.7.Windows10.x86_64.cuda-11.8.cudnn8.6" --cuda_version=11.6 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.6" --enable_onnx_tests --enable_wcos --build_java --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=37;52;60;61;70;75;80" - runTests: ${{ parameters.RunOnnxRuntimeTests }} - buildJava: true - java_artifact_id: onnxruntime_gpu - timeoutInMinutes: 1200 -- ${{ if eq(parameters['UseIncreasedTimeoutForTests'], 'false') }}: - - template: templates/win-ci.yml - parameters: - ort_build_pool_name: 'onnxruntime-gpu-tensorrt8-winbuild-t4' - DoCompliance: ${{ parameters.DoCompliance }} - DoEsrp: ${{ parameters.DoEsrp }} - job_name_suffix: tensorrt - EnvSetupScript: setup_env_gpu.bat - buildArch: x64 - msbuildPlatform: x64 - packageName: x64-tensorrt - buildparameter: --use_tensorrt --skip_and_perform_filtered_tensorrt_tests --tensorrt_home="C:\local\TensorRT-8.5.1.7.Windows10.x86_64.cuda-11.8.cudnn8.6" --cuda_version=11.6 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.6" --enable_onnx_tests --enable_wcos --build_java --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=37;52;60;61;70;75;80" - runTests: ${{ parameters.RunOnnxRuntimeTests }} - buildJava: true - java_artifact_id: onnxruntime_gpu - timeoutInMinutes: 600 +- template: templates/win-ci.yml + parameters: + ort_build_pool_name: 'onnxruntime-gpu-tensorrt8-winbuild-t4' + DoCompliance: ${{ parameters.DoCompliance }} + DoEsrp: ${{ parameters.DoEsrp }} + job_name_suffix: tensorrt + EnvSetupScript: setup_env_gpu.bat + buildArch: x64 + msbuildPlatform: x64 + packageName: x64-tensorrt + buildparameter: --use_tensorrt --tensorrt_home="C:\local\TensorRT-8.5.1.7.Windows10.x86_64.cuda-11.8.cudnn8.6" --cuda_version=11.6 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.6" --enable_onnx_tests --enable_wcos --build_java --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=37;52;60;61;70;75;80" + runTests: ${{ parameters.RunOnnxRuntimeTests }} + buildJava: true + java_artifact_id: onnxruntime_gpu + UseIncreasedTimeoutForTests: ${{ parameters.UseIncreasedTimeoutForTests }} - job: Jar_Packaging_GPU workspace: diff --git a/tools/ci_build/github/azure-pipelines/templates/win-ci.yml b/tools/ci_build/github/azure-pipelines/templates/win-ci.yml index 0d478b1843970..45e996e66ea9e 100644 --- a/tools/ci_build/github/azure-pipelines/templates/win-ci.yml +++ b/tools/ci_build/github/azure-pipelines/templates/win-ci.yml @@ -52,6 +52,11 @@ parameters: type: string default: 'onnxruntime' +- name: UseIncreasedTimeoutForTests + displayName: Increase timeout for tests? Set it to false if you are doing an Onnx Runtime release. + type: boolean + default: false + - name: timeoutInMinutes type: number default: 300 @@ -61,7 +66,17 @@ jobs: workspace: clean: all pool: ${{ parameters.ort_build_pool_name }} - timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + ${{ if eq(parameters['UseIncreasedTimeoutForTests'], 'true') }}: + timeoutInMinutes: 1200 + ${{ else }}: + timeoutInMinutes: 300 + + variables: + - name: increasedTimeoutArg + ${{ if eq(parameters['UseIncreasedTimeoutForTests'], 'true') }}: + value: "" + ${{ else }}: + value: "--skip_and_perform_filtered_tests" steps: - checkout: self @@ -110,7 +125,7 @@ jobs: displayName: 'Generate cmake config' inputs: scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' - arguments: '--config RelWithDebInfo --enable_lto --disable_rtti --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --enable_onnx_tests $(TelemetryOption) ${{ parameters.buildparameter }}' + arguments: '--config RelWithDebInfo --enable_lto --disable_rtti --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --enable_onnx_tests $(TelemetryOption) ${{ parameters.buildparameter }} ${{ parameters.increasedTimeoutArg }}' workingDirectory: '$(Build.BinariesDirectory)' - task: VSBuild@1 From ba4b59ec325de296c979b34f354eae3fab92e246 Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Mon, 12 Dec 2022 22:12:26 +0000 Subject: [PATCH 44/49] make timeout configurable --- tools/ci_build/github/azure-pipelines/templates/win-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/templates/win-ci.yml b/tools/ci_build/github/azure-pipelines/templates/win-ci.yml index 45e996e66ea9e..bfe41b2d34e03 100644 --- a/tools/ci_build/github/azure-pipelines/templates/win-ci.yml +++ b/tools/ci_build/github/azure-pipelines/templates/win-ci.yml @@ -72,7 +72,7 @@ jobs: timeoutInMinutes: 300 variables: - - name: increasedTimeoutArg + - name: skipAndPerformFilteredTests ${{ if eq(parameters['UseIncreasedTimeoutForTests'], 'true') }}: value: "" ${{ else }}: @@ -125,7 +125,7 @@ jobs: displayName: 'Generate cmake config' inputs: scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' - arguments: '--config RelWithDebInfo --enable_lto --disable_rtti --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --enable_onnx_tests $(TelemetryOption) ${{ parameters.buildparameter }} ${{ parameters.increasedTimeoutArg }}' + arguments: '--config RelWithDebInfo --enable_lto --disable_rtti --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --enable_onnx_tests $(TelemetryOption) ${{ parameters.buildparameter }} $(skipAndPerformFilteredTests)' workingDirectory: '$(Build.BinariesDirectory)' - task: VSBuild@1 From af57f18e31572477821bdebc1acb717a3cf5011b Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Mon, 12 Dec 2022 22:20:51 +0000 Subject: [PATCH 45/49] make timeout configurable --- .../github/azure-pipelines/templates/win-ci.yml | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/templates/win-ci.yml b/tools/ci_build/github/azure-pipelines/templates/win-ci.yml index bfe41b2d34e03..d2a0af084c723 100644 --- a/tools/ci_build/github/azure-pipelines/templates/win-ci.yml +++ b/tools/ci_build/github/azure-pipelines/templates/win-ci.yml @@ -57,10 +57,6 @@ parameters: type: boolean default: false -- name: timeoutInMinutes - type: number - default: 300 - jobs: - job: Windows_Packaging_${{ parameters.job_name_suffix }} workspace: @@ -71,13 +67,6 @@ jobs: ${{ else }}: timeoutInMinutes: 300 - variables: - - name: skipAndPerformFilteredTests - ${{ if eq(parameters['UseIncreasedTimeoutForTests'], 'true') }}: - value: "" - ${{ else }}: - value: "--skip_and_perform_filtered_tests" - steps: - checkout: self clean: true @@ -125,7 +114,10 @@ jobs: displayName: 'Generate cmake config' inputs: scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' - arguments: '--config RelWithDebInfo --enable_lto --disable_rtti --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --enable_onnx_tests $(TelemetryOption) ${{ parameters.buildparameter }} $(skipAndPerformFilteredTests)' + ${{ if eq(parameters['UseIncreasedTimeoutForTests'], 'true') }}: + arguments: '--config RelWithDebInfo --enable_lto --disable_rtti --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --enable_onnx_tests $(TelemetryOption) ${{ parameters.buildparameter }}' + ${{ else }}: + arguments: '--config RelWithDebInfo --enable_lto --disable_rtti --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --enable_onnx_tests $(TelemetryOption) ${{ parameters.buildparameter }} --skip_and_perform_filtered_tests' workingDirectory: '$(Build.BinariesDirectory)' - task: VSBuild@1 From 68c185dfa266d4cf04585d9629a1f6e552c9680c Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Mon, 12 Dec 2022 22:23:14 +0000 Subject: [PATCH 46/49] make timeout configurable (fix bug) --- tools/ci_build/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index 4cacd24bfccae..23cf10277f38a 100644 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -1888,7 +1888,7 @@ def run_onnxruntime_tests(args, source_dir, ctest_path, build_dir, configs): run_subprocess([os.path.join(cwd, exe)], cwd=cwd, dll_path=dll_path) else: - if args.use_tensorrt and not args.skip_and_perform_filtered_tests + if args.use_tensorrt and not args.skip_and_perform_filtered_tests: # TensorRT 8.5 needs more time to run tests on Windows ctest_cmd = [ctest_path, "--build-config", config, "--verbose", "--timeout", "72000"] else: From f3ccdd6bc33b99162aefc134f9a56648d00325a6 Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Tue, 13 Dec 2022 04:58:41 +0000 Subject: [PATCH 47/49] refactor --- tools/ci_build/build.py | 11 +++++------ .../github/azure-pipelines/templates/win-ci.yml | 4 ++-- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index 23cf10277f38a..47a5b5892d766 100644 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -491,6 +491,9 @@ def convert_arg_line_to_args(self, arg_line): "--tensorrt_placeholder_builder", action="store_true", help="Instantiate Placeholder TensorRT Builder" ) parser.add_argument("--tensorrt_home", help="Path to TensorRT installation dir") + parser.add_argument( + "--test_all_timeout", default="10800", help="Set timeout for onnxruntime_test_all" + ) parser.add_argument( "--skip_and_perform_filtered_tests", action="store_true", @@ -882,7 +885,7 @@ def generate_build_tree( "-Donnxruntime_USE_VITISAI=" + ("ON" if args.use_vitisai else "OFF"), "-Donnxruntime_USE_TENSORRT=" + ("ON" if args.use_tensorrt else "OFF"), "-Donnxruntime_SKIP_AND_PERFORM_FILTERED_TENSORRT_TESTS=" - + ("ON" if args.skip_and_perform_filtered_tests else "OFF"), + + ("ON" if args.test_all_timeout is "10800" else "OFF"), "-Donnxruntime_USE_TENSORRT_BUILTIN_PARSER=" + ("ON" if args.use_tensorrt_builtin_parser else "OFF"), "-Donnxruntime_TENSORRT_PLACEHOLDER_BUILDER=" + ("ON" if args.tensorrt_placeholder_builder else "OFF"), # set vars for TVM @@ -1888,11 +1891,7 @@ def run_onnxruntime_tests(args, source_dir, ctest_path, build_dir, configs): run_subprocess([os.path.join(cwd, exe)], cwd=cwd, dll_path=dll_path) else: - if args.use_tensorrt and not args.skip_and_perform_filtered_tests: - # TensorRT 8.5 needs more time to run tests on Windows - ctest_cmd = [ctest_path, "--build-config", config, "--verbose", "--timeout", "72000"] - else: - ctest_cmd = [ctest_path, "--build-config", config, "--verbose", "--timeout", "10800"] + ctest_cmd = [ctest_path, "--build-config", config, "--verbose", "--timeout", args.test_all_timeout] run_subprocess(ctest_cmd, cwd=cwd, dll_path=dll_path) if args.enable_pybind: diff --git a/tools/ci_build/github/azure-pipelines/templates/win-ci.yml b/tools/ci_build/github/azure-pipelines/templates/win-ci.yml index d2a0af084c723..dad7d23f57b24 100644 --- a/tools/ci_build/github/azure-pipelines/templates/win-ci.yml +++ b/tools/ci_build/github/azure-pipelines/templates/win-ci.yml @@ -115,9 +115,9 @@ jobs: inputs: scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' ${{ if eq(parameters['UseIncreasedTimeoutForTests'], 'true') }}: - arguments: '--config RelWithDebInfo --enable_lto --disable_rtti --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --enable_onnx_tests $(TelemetryOption) ${{ parameters.buildparameter }}' + arguments: '--config RelWithDebInfo --enable_lto --disable_rtti --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --enable_onnx_tests $(TelemetryOption) ${{ parameters.buildparameter }} --test_all_timeout 72000' ${{ else }}: - arguments: '--config RelWithDebInfo --enable_lto --disable_rtti --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --enable_onnx_tests $(TelemetryOption) ${{ parameters.buildparameter }} --skip_and_perform_filtered_tests' + arguments: '--config RelWithDebInfo --enable_lto --disable_rtti --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --enable_onnx_tests $(TelemetryOption) ${{ parameters.buildparameter }} ' workingDirectory: '$(Build.BinariesDirectory)' - task: VSBuild@1 From 8b63162f2fa378baf78fa1830de7a5cb4ee4f993 Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Tue, 13 Dec 2022 05:02:33 +0000 Subject: [PATCH 48/49] refactor --- tools/ci_build/build.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index 47a5b5892d766..7979442e9049f 100644 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -491,9 +491,7 @@ def convert_arg_line_to_args(self, arg_line): "--tensorrt_placeholder_builder", action="store_true", help="Instantiate Placeholder TensorRT Builder" ) parser.add_argument("--tensorrt_home", help="Path to TensorRT installation dir") - parser.add_argument( - "--test_all_timeout", default="10800", help="Set timeout for onnxruntime_test_all" - ) + parser.add_argument("--test_all_timeout", default="10800", help="Set timeout for onnxruntime_test_all") parser.add_argument( "--skip_and_perform_filtered_tests", action="store_true", From ffce45cbf91b9f7fad38580ca44ff3bb1701f844 Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Tue, 13 Dec 2022 06:36:33 +0000 Subject: [PATCH 49/49] fix for flake8 error --- tools/ci_build/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index 7979442e9049f..62901a876f2f1 100644 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -883,7 +883,7 @@ def generate_build_tree( "-Donnxruntime_USE_VITISAI=" + ("ON" if args.use_vitisai else "OFF"), "-Donnxruntime_USE_TENSORRT=" + ("ON" if args.use_tensorrt else "OFF"), "-Donnxruntime_SKIP_AND_PERFORM_FILTERED_TENSORRT_TESTS=" - + ("ON" if args.test_all_timeout is "10800" else "OFF"), + + ("ON" if args.test_all_timeout == "10800" else "OFF"), "-Donnxruntime_USE_TENSORRT_BUILTIN_PARSER=" + ("ON" if args.use_tensorrt_builtin_parser else "OFF"), "-Donnxruntime_TENSORRT_PLACEHOLDER_BUILDER=" + ("ON" if args.tensorrt_placeholder_builder else "OFF"), # set vars for TVM