From b564fcbad41d0133c355046eb99c7a28e01d6be7 Mon Sep 17 00:00:00 2001 From: David Yastremsky Date: Mon, 13 Mar 2023 12:38:48 -0700 Subject: [PATCH 01/11] For Jetson, do not build compute_90 arch --- tools/gen_ort_dockerfile.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tools/gen_ort_dockerfile.py b/tools/gen_ort_dockerfile.py index dd90249..2756f86 100755 --- a/tools/gen_ort_dockerfile.py +++ b/tools/gen_ort_dockerfile.py @@ -214,10 +214,19 @@ def dockerfile_for_linux(output_file): ep_flags += ' --tensorrt_home "{}"'.format(FLAGS.tensorrt_home) if FLAGS.ort_openvino is not None: ep_flags += ' --use_openvino CPU_FP32' + + # DLIS-4658: Once Jetson build supports high enough level of CUDA, include compute_90 for Jetson. + cuda_archs="52;60;61;70;75;80;86" + if target_platform() != 'jetpack': + cuda_archs+=";90" df += ''' WORKDIR /workspace/onnxruntime -ARG COMMON_BUILD_ARGS="--config ${ONNXRUNTIME_BUILD_CONFIG} --skip_submodule_sync --parallel --build_shared_lib --build_dir /workspace/build --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES='52;60;61;70;75;80;86;90' " +ARG COMMON_BUILD_ARGS="--config ${ONNXRUNTIME_BUILD_CONFIG} --skip_submodule_sync --parallel --build_shared_lib \ + --build_dir /workspace/build --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES='{}' " +'''.format(cuda_archs) + +if target_platform() != 'jetpack': df += ''' ''' # Remove version info from libonnxruntime.so From c8c412e0b83109bda57a3b654efe200a831f3b15 Mon Sep 17 00:00:00 2001 From: David Yastremsky Date: Mon, 13 Mar 2023 14:12:40 -0700 Subject: [PATCH 02/11] Remove unnecessary statement --- tools/gen_ort_dockerfile.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tools/gen_ort_dockerfile.py b/tools/gen_ort_dockerfile.py index 2756f86..f729240 100755 --- a/tools/gen_ort_dockerfile.py +++ b/tools/gen_ort_dockerfile.py @@ -226,9 +226,7 @@ def dockerfile_for_linux(output_file): --build_dir /workspace/build --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES='{}' " '''.format(cuda_archs) -if target_platform() != 'jetpack': df += ''' -''' - + df += ''' # Remove version info from libonnxruntime.so # This makes it possible to replace ort binaries in released triton containers # for experimentation, without having to build triton-ort backend. From 50cc1e316961b53f6d23ebcec6a8d4285495a8ee Mon Sep 17 00:00:00 2001 From: David Yastremsky Date: Mon, 13 Mar 2023 16:09:20 -0700 Subject: [PATCH 03/11] Fix quote --- tools/gen_ort_dockerfile.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tools/gen_ort_dockerfile.py b/tools/gen_ort_dockerfile.py index f729240..4b41c10 100755 --- a/tools/gen_ort_dockerfile.py +++ b/tools/gen_ort_dockerfile.py @@ -227,10 +227,9 @@ def dockerfile_for_linux(output_file): '''.format(cuda_archs) df += ''' - # Remove version info from libonnxruntime.so - # This makes it possible to replace ort binaries in released triton containers - # for experimentation, without having to build triton-ort backend. - df += ''' +# Remove version info from libonnxruntime.so +# This makes it possible to replace ort binaries in released triton containers +# for experimentation, without having to build triton-ort backend. RUN sed -i 's/VERS_%s//' tools/ci_build/gen_def.py && (sed -i 's/% VERSION_STRING//' tools/ci_build/gen_def.py) RUN sed -i 's/set_target_properties(onnxruntime PROPERTIES VERSION ${ORT_VERSION})//' cmake/onnxruntime.cmake ''' From dd395e8e6ecfda09fc89be8331c27637cffe38c7 Mon Sep 17 00:00:00 2001 From: David Yastremsky Date: Mon, 13 Mar 2023 16:10:32 -0700 Subject: [PATCH 04/11] Move comment outside of quote --- tools/gen_ort_dockerfile.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/gen_ort_dockerfile.py b/tools/gen_ort_dockerfile.py index 4b41c10..a0a9b23 100755 --- a/tools/gen_ort_dockerfile.py +++ b/tools/gen_ort_dockerfile.py @@ -226,10 +226,10 @@ def dockerfile_for_linux(output_file): --build_dir /workspace/build --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES='{}' " '''.format(cuda_archs) + # Remove version info from libonnxruntime.so + # This makes it possible to replace ort binaries in released triton containers + # for experimentation, without having to build triton-ort backend. df += ''' -# Remove version info from libonnxruntime.so -# This makes it possible to replace ort binaries in released triton containers -# for experimentation, without having to build triton-ort backend. RUN sed -i 's/VERS_%s//' tools/ci_build/gen_def.py && (sed -i 's/% VERSION_STRING//' tools/ci_build/gen_def.py) RUN sed -i 's/set_target_properties(onnxruntime PROPERTIES VERSION ${ORT_VERSION})//' cmake/onnxruntime.cmake ''' From 024d0bbaa9b190c8c7c60cf171ece9ce1f784c22 Mon Sep 17 00:00:00 2001 From: David Yastremsky Date: Mon, 13 Mar 2023 17:31:01 -0700 Subject: [PATCH 05/11] Auto-format --- tools/gen_ort_dockerfile.py | 59 +++++++++++++++++++++---------------- 1 file changed, 33 insertions(+), 26 deletions(-) diff --git a/tools/gen_ort_dockerfile.py b/tools/gen_ort_dockerfile.py index a0a9b23..30bdd15 100755 --- a/tools/gen_ort_dockerfile.py +++ b/tools/gen_ort_dockerfile.py @@ -34,15 +34,16 @@ ORT_TO_TRTPARSER_VERSION_MAP = { '1.9.0': ( - '8.2', # TensorRT version - 'release/8.2-GA' # ONNX-Tensorrt parser version + '8.2', # TensorRT version + 'release/8.2-GA' # ONNX-Tensorrt parser version ), '1.10.0': ( - '8.2', # TensorRT version - 'release/8.2-GA' # ONNX-Tensorrt parser version + '8.2', # TensorRT version + 'release/8.2-GA' # ONNX-Tensorrt parser version ) } + def target_platform(): if FLAGS.target_platform is not None: return FLAGS.target_platform @@ -147,10 +148,11 @@ def dockerfile_for_linux(output_file): wget ${INTEL_COMPUTE_RUNTIME_URL}/intel-ocloc_19.41.14441_amd64.deb && \ dpkg -i *.deb && rm -rf *.deb ''' - ## TEMPORARY: Using the tensorrt-8.0 branch until ORT 1.9 release to enable ORT backend with TRT 8.0 support. - # For ORT versions 1.8.0 and below the behavior will remain same. For ORT version 1.8.1 we will - # use tensorrt-8.0 branch instead of using rel-1.8.1 - # From ORT 1.9 onwards we will switch back to using rel-* branches + +## TEMPORARY: Using the tensorrt-8.0 branch until ORT 1.9 release to enable ORT backend with TRT 8.0 support. +# For ORT versions 1.8.0 and below the behavior will remain same. For ORT version 1.8.1 we will +# use tensorrt-8.0 branch instead of using rel-1.8.1 +# From ORT 1.9 onwards we will switch back to using rel-* branches if FLAGS.ort_version == "1.8.1": df += ''' # @@ -214,11 +216,11 @@ def dockerfile_for_linux(output_file): ep_flags += ' --tensorrt_home "{}"'.format(FLAGS.tensorrt_home) if FLAGS.ort_openvino is not None: ep_flags += ' --use_openvino CPU_FP32' - + # DLIS-4658: Once Jetson build supports high enough level of CUDA, include compute_90 for Jetson. - cuda_archs="52;60;61;70;75;80;86" + cuda_archs = "52;60;61;70;75;80;86" if target_platform() != 'jetpack': - cuda_archs+=";90" + cuda_archs += ";90" df += ''' WORKDIR /workspace/onnxruntime @@ -276,7 +278,6 @@ def dockerfile_for_linux(output_file): /opt/onnxruntime/lib ''' - if FLAGS.ort_tensorrt: df += ''' # TensorRT specific headers and libraries @@ -444,10 +445,10 @@ def dockerfile_for_windows(output_file): def preprocess_gpu_flags(): - if target_platform() == 'windows': + if target_platform() == 'windows': # Default to CUDA based on CUDA_PATH envvar and TensorRT in # C:/tensorrt - if 'CUDA_PATH'in os.environ: + if 'CUDA_PATH' in os.environ: if FLAGS.cuda_home is None: FLAGS.cuda_home = os.environ['CUDA_PATH'] elif FLAGS.cuda_home != os.environ['CUDA_PATH']: @@ -467,14 +468,16 @@ def preprocess_gpu_flags(): print("warning: --cuda-version does not match CUDA_PATH envvar") if (FLAGS.cuda_home is None) or (FLAGS.cuda_version is None): - print("error: windows build requires --cuda-version and --cuda-home") + print( + "error: windows build requires --cuda-version and --cuda-home") if FLAGS.tensorrt_home is None: FLAGS.tensorrt_home = '/tensorrt' else: - if 'CUDNN_VERSION'in os.environ: + if 'CUDNN_VERSION' in os.environ: version = None - m = re.match(r'([0-9]\.[0-9])\.[0-9]\.[0-9]', os.environ['CUDNN_VERSION']) + m = re.match(r'([0-9]\.[0-9])\.[0-9]\.[0-9]', + os.environ['CUDNN_VERSION']) if m: version = m.group(1) if FLAGS.cudnn_home is None: @@ -511,7 +514,7 @@ def preprocess_gpu_flags(): help='Enable GPU support') parser.add_argument('--ort-build-config', type=str, - default ="Release", + default="Release", choices=["Debug", "Release", "RelWithDebInfo"], help='ORT build configuration.') parser.add_argument( @@ -534,10 +537,12 @@ def preprocess_gpu_flags(): type=str, required=False, help='Home directory for CUDNN.') - parser.add_argument('--ort-openvino', - type=str, - required=False, - help='Enable OpenVino execution provider using specified OpenVINO version.') + parser.add_argument( + '--ort-openvino', + type=str, + required=False, + help= + 'Enable OpenVino execution provider using specified OpenVINO version.') parser.add_argument('--ort-tensorrt', action="store_true", required=False, @@ -563,11 +568,13 @@ def preprocess_gpu_flags(): # if the tag is empty - check whether there is an entry in the ORT_TO_TRTPARSER_VERSION_MAP # map corresponding to ort version + trt version combo. If yes then use it # otherwise we leave it empty and use the defaults from ort - if FLAGS.onnx_tensorrt_tag == "" and FLAGS.ort_version in ORT_TO_TRTPARSER_VERSION_MAP.keys(): + if FLAGS.onnx_tensorrt_tag == "" and FLAGS.ort_version in ORT_TO_TRTPARSER_VERSION_MAP.keys( + ): trt_version = re.match(r'^[0-9]+\.[0-9]+', FLAGS.trt_version) - if trt_version and trt_version.group(0) == ORT_TO_TRTPARSER_VERSION_MAP[FLAGS.ort_version][0]: - FLAGS.onnx_tensorrt_tag = ORT_TO_TRTPARSER_VERSION_MAP[FLAGS.ort_version][1] - + if trt_version and trt_version.group(0) == ORT_TO_TRTPARSER_VERSION_MAP[ + FLAGS.ort_version][0]: + FLAGS.onnx_tensorrt_tag = ORT_TO_TRTPARSER_VERSION_MAP[ + FLAGS.ort_version][1] if target_platform() == 'windows': # OpenVINO EP not yet supported for windows build From 000c0353958a41e05fcedf8f2ee3a16c5914f6da Mon Sep 17 00:00:00 2001 From: David Yastremsky Date: Mon, 13 Mar 2023 17:37:17 -0700 Subject: [PATCH 06/11] Escape brace for gen Bash script --- tools/gen_ort_dockerfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/gen_ort_dockerfile.py b/tools/gen_ort_dockerfile.py index 30bdd15..ee95334 100755 --- a/tools/gen_ort_dockerfile.py +++ b/tools/gen_ort_dockerfile.py @@ -224,7 +224,7 @@ def dockerfile_for_linux(output_file): df += ''' WORKDIR /workspace/onnxruntime -ARG COMMON_BUILD_ARGS="--config ${ONNXRUNTIME_BUILD_CONFIG} --skip_submodule_sync --parallel --build_shared_lib \ +ARG COMMON_BUILD_ARGS="--config ${{ONNXRUNTIME_BUILD_CONFIG}} --skip_submodule_sync --parallel --build_shared_lib \ --build_dir /workspace/build --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES='{}' " '''.format(cuda_archs) From cecf2271bc72dc3ad64bc490bda51d20546e7d7c Mon Sep 17 00:00:00 2001 From: David Yastremsky Date: Mon, 13 Mar 2023 20:31:12 -0700 Subject: [PATCH 07/11] Add platform variable use to ORT build --- CMakeLists.txt | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 72d01a1..31edebd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -81,6 +81,9 @@ project(tritononnxruntimebackend LANGUAGES C CXX) # - If you want to disable GPU usage, set TRITON_ENABLE_GPU=OFF. # This will make builds with CUDA and TensorRT flags to fail. # +# - If you want to optionally set the platform rather than rely on it being detected, +# set TRITON_BUILD_PLATFORM equal to Ubuntu, Windows, or Jetpack. +# option(TRITON_ENABLE_GPU "Enable GPU support in backend" ON) option(TRITON_ENABLE_STATS "Include statistics collections in backend" ON) option(TRITON_ENABLE_ONNXRUNTIME_TENSORRT @@ -94,6 +97,7 @@ set(TRITON_BUILD_ONNXRUNTIME_OPENVINO_VERSION "" CACHE STRING "ONNXRuntime OpenV set(TRITON_BUILD_CUDA_VERSION "" CACHE STRING "Version of CUDA install") set(TRITON_BUILD_CUDA_HOME "" CACHE PATH "Path to CUDA install") set(TRITON_BUILD_CUDNN_HOME "" CACHE PATH "Path to CUDNN install") +set(TRITON_BUILD_PLATFORM "" CACHE STRING "Platform of build") set(TRITON_BUILD_TENSORRT_HOME "" CACHE PATH "Path to TensorRT install") set(TRITON_ONNXRUNTIME_INCLUDE_PATHS "" CACHE PATH "Paths to ONNXRuntime includes") set(TRITON_ONNX_TENSORRT_REPO_TAG "" CACHE STRING "Tag for onnx-tensorrt repo") @@ -334,6 +338,9 @@ if(TRITON_ONNXRUNTIME_DOCKER_BUILD) if(${TRITON_ENABLE_ONNXRUNTIME_OPENVINO}) set(_GEN_FLAGS ${_GEN_FLAGS} "--ort-openvino=${TRITON_BUILD_ONNXRUNTIME_OPENVINO_VERSION}") endif() # TRITON_ENABLE_ONNXRUNTIME_OPENVINO + if(${TRITON_BUILD_PLATFORM}) + set(_GEN_FLAGS ${_GEN_FLAGS} "--target-platform=${TRITON_BUILD_PLATFORM}") + endif() # TRITON_BUILD_PLATFORM set(ENABLE_GPU_EXTRA_ARGS "") if(${TRITON_ENABLE_GPU}) From 9882d491f2cd0e391db953d23457c002ef1d307a Mon Sep 17 00:00:00 2001 From: David Yastremsky Date: Mon, 13 Mar 2023 22:13:37 -0700 Subject: [PATCH 08/11] Fix if statement --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 31edebd..2072c7a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -343,7 +343,7 @@ if(TRITON_ONNXRUNTIME_DOCKER_BUILD) endif() # TRITON_BUILD_PLATFORM set(ENABLE_GPU_EXTRA_ARGS "") - if(${TRITON_ENABLE_GPU}) + if(NOT ${TRITON_ENABLE_GPU} STREQUAL "") set(ENABLE_GPU_EXTRA_ARGS "--enable-gpu") endif() # TRITON_ENABLE_GPU From 0fe97b0e02f323edc23a3bf4bec95a732e36fe36 Mon Sep 17 00:00:00 2001 From: David Yastremsky Date: Mon, 13 Mar 2023 20:31:30 -0700 Subject: [PATCH 09/11] Format --- tools/gen_ort_dockerfile.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/gen_ort_dockerfile.py b/tools/gen_ort_dockerfile.py index ee95334..21218d1 100755 --- a/tools/gen_ort_dockerfile.py +++ b/tools/gen_ort_dockerfile.py @@ -149,6 +149,7 @@ def dockerfile_for_linux(output_file): dpkg -i *.deb && rm -rf *.deb ''' + ## TEMPORARY: Using the tensorrt-8.0 branch until ORT 1.9 release to enable ORT backend with TRT 8.0 support. # For ORT versions 1.8.0 and below the behavior will remain same. For ORT version 1.8.1 we will # use tensorrt-8.0 branch instead of using rel-1.8.1 From 2885c6edbcc3bcf8673549236e199c16dd1fa3c0 Mon Sep 17 00:00:00 2001 From: David Yastremsky Date: Tue, 14 Mar 2023 11:57:43 -0700 Subject: [PATCH 10/11] Fix flag check --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2072c7a..ba12eb6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -338,12 +338,12 @@ if(TRITON_ONNXRUNTIME_DOCKER_BUILD) if(${TRITON_ENABLE_ONNXRUNTIME_OPENVINO}) set(_GEN_FLAGS ${_GEN_FLAGS} "--ort-openvino=${TRITON_BUILD_ONNXRUNTIME_OPENVINO_VERSION}") endif() # TRITON_ENABLE_ONNXRUNTIME_OPENVINO - if(${TRITON_BUILD_PLATFORM}) + if(NOT ${TRITON_BUILD_PLATFORM} STREQUAL "") set(_GEN_FLAGS ${_GEN_FLAGS} "--target-platform=${TRITON_BUILD_PLATFORM}") endif() # TRITON_BUILD_PLATFORM set(ENABLE_GPU_EXTRA_ARGS "") - if(NOT ${TRITON_ENABLE_GPU} STREQUAL "") + if(${TRITON_ENABLE_GPU}) set(ENABLE_GPU_EXTRA_ARGS "--enable-gpu") endif() # TRITON_ENABLE_GPU From a3d14f3668eb606db03ac008f1bd2259025822b0 Mon Sep 17 00:00:00 2001 From: David Yastremsky Date: Mon, 3 Apr 2023 15:11:59 -0700 Subject: [PATCH 11/11] Specify CUDA version required --- tools/gen_ort_dockerfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/gen_ort_dockerfile.py b/tools/gen_ort_dockerfile.py index 21218d1..5aabbee 100755 --- a/tools/gen_ort_dockerfile.py +++ b/tools/gen_ort_dockerfile.py @@ -218,7 +218,7 @@ def dockerfile_for_linux(output_file): if FLAGS.ort_openvino is not None: ep_flags += ' --use_openvino CPU_FP32' - # DLIS-4658: Once Jetson build supports high enough level of CUDA, include compute_90 for Jetson. + # DLIS-4658: Once Jetson build supports CUDA 11.8+, include compute_90 for Jetson. cuda_archs = "52;60;61;70;75;80;86" if target_platform() != 'jetpack': cuda_archs += ";90"