Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

revert: "Change TensorRT-LLM (#7143)" #7341

Merged
merged 9 commits into from
Jun 17, 2024
Merged

revert: "Change TensorRT-LLM (#7143)" #7341

Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
302c4d4
Revert "Change TensorRT-LLM (#7143)"
krishung5 Jun 12, 2024
933d7e9
Fix version for setuptools
krishung5 Jun 12, 2024
5caed7f
Fix version for grpcio-tools
krishung5 Jun 12, 2024
41a4912
Add -j 18 flag to avoid OOM during building TRTLLM
krishung5 Jun 13, 2024
16e7f1d
Merge remote-tracking branch 'origin/r24.06' into krish-r24.06
krishung5 Jun 13, 2024
5504a6c
Reduce the num of thread to avoid OOM during build
krishung5 Jun 13, 2024
7807b34
Use flag -j8
krishung5 Jun 13, 2024
017898a
Update flags
krishung5 Jun 13, 2024
9899937
Use -j18
krishung5 Jun 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 29 additions & 1 deletion build.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -819,7 +819,21 @@ def fastertransformer_cmake_args():


def tensorrtllm_cmake_args(images):
cargs = []
cmake_script.cmd("apt-get update && apt-get install -y libcudnn8-dev && ldconfig")
cmake_script.cmd(
"python3 ../tensorrt_llm/scripts/build_wheel.py --trt_root /usr/local/tensorrt -i -c -j 18"
)
cargs = [
cmake_backend_arg(
"tensorrtllm",
"TRT_LIB_DIR",
None,
"${TRT_ROOT}/targets/${ARCH}-linux-gnu/lib",
),
cmake_backend_arg(
"tensorrtllm", "TRT_INCLUDE_DIR", None, "${TRT_ROOT}/include"
),
]
cargs.append(cmake_backend_enable("tensorrtllm", "USE_CXX11_ABI", True))
return cargs

Expand Down Expand Up @@ -1094,10 +1108,19 @@ def create_dockerfile_linux(
RUN python3 -m pip install --upgrade pip \\
&& pip3 install transformers

# Drop the static libs
RUN ARCH="$(uname -i)" \\
&& rm -f ${TRT_ROOT}/targets/${ARCH}-linux-gnu/lib/libnvinfer*.a \\
${TRT_ROOT}/targets/${ARCH}-linux-gnu/lib/libnvonnxparser_*.a

# Install TensorRT-LLM
RUN python3 -m pip install /opt/tritonserver/backends/tensorrtllm/tensorrt_llm-*.whl -U --pre --extra-index-url https://pypi.nvidia.com \\
&& rm -fv /opt/tritonserver/backends/tensorrtllm/tensorrt_llm-*.whl
RUN find /usr -name libtensorrt_llm.so -exec dirname {} \; > /etc/ld.so.conf.d/tensorrt-llm.conf
RUN find /opt/tritonserver -name libtritonserver.so -exec dirname {} \; > /etc/ld.so.conf.d/triton-tensorrtllm-worker.conf

RUN pip3 install setuptools==69.5.1 grpcio-tools==1.64.0

ENV LD_LIBRARY_PATH=/usr/local/tensorrt/lib/:/opt/tritonserver/backends/tensorrtllm:$LD_LIBRARY_PATH
"""
with open(os.path.join(ddir, dockerfile_name), "w") as dfile:
Expand Down Expand Up @@ -1702,6 +1725,11 @@ def tensorrtllm_postbuild(cmake_script, repo_install_dir, tensorrtllm_be_dir):
# TODO: Update the CMakeLists.txt of TRT-LLM backend to install the artifacts to the correct location
cmake_destination_dir = os.path.join(repo_install_dir, "backends/tensorrtllm")
cmake_script.mkdir(cmake_destination_dir)
# Copy over the TRT-LLM wheel for later installation
cmake_script.cp(
os.path.join(tensorrtllm_be_dir, "tensorrt_llm", "build", "tensorrt_llm-*.whl"),
cmake_destination_dir,
)

# Copy over the TRT-LLM backend libraries
cmake_script.cp(
Expand Down
Loading