Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update build.py to build vLLM backend #6394

Merged
merged 16 commits into from
Oct 8, 2023
Merged
76 changes: 65 additions & 11 deletions build.py
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@
"py310_23.1.0-1", # Conda version
"9.1.0.1", # TRT version for building TRT-LLM backend
"12.2", # CUDA version for building TRT-LLM backend
"0.2.0", # vLLM version
)
}

Expand Down Expand Up @@ -1332,6 +1333,16 @@ def dockerfile_prepare_container_linux(argmap, backends, enable_gpu, target_mach
argmap["TRT_LLM_TRT_VERSION"], argmap["TRT_LLM_CUDA_VERSION"]
)

if "vllm" in backends:
# [DLIS-5606] Build Conda environment for vLLM backend
# Remove Pip install once vLLM backend moves to Conda environment.
df += """
# vLLM needed for vLLM backend
RUN pip3 install vllm=={}
""".format(
TRITON_VERSION_MAP[FLAGS.version][9]
)

df += """
WORKDIR /opt/tritonserver
RUN rm -fr /opt/tritonserver/*
Expand Down Expand Up @@ -1839,6 +1850,39 @@ def backend_build(
cmake_script.blankln()


def backend_clone(
be,
clone_script,
tag,
build_dir,
install_dir,
github_organization,
):
clone_script.commentln(8)
clone_script.comment(f"'{be}' backend")
clone_script.comment("Delete this section to remove backend from build")
clone_script.comment()
clone_script.mkdir(build_dir)
clone_script.cwd(build_dir)
clone_script.gitclone(backend_repo(be), tag, be, github_organization)
tanmayv25 marked this conversation as resolved.
Show resolved Hide resolved

repo_target_dir = os.path.join(install_dir, "backends")
clone_script.mkdir(repo_target_dir)
backend_dir = os.path.join(repo_target_dir, be)
clone_script.rmdir(backend_dir)
clone_script.mkdir(backend_dir)

clone_script.cp(
os.path.join(build_dir, be, "src", "model.py"),
backend_dir,
)

clone_script.comment()
clone_script.comment(f"end '{be}' backend")
clone_script.commentln(8)
clone_script.blankln()


def repo_agent_build(
ra, cmake_script, build_dir, install_dir, repoagent_repo, repoagents
):
Expand Down Expand Up @@ -2700,17 +2744,27 @@ def enable_all():
else:
github_organization = FLAGS.github_organization

backend_build(
be,
cmake_script,
backends[be],
script_build_dir,
script_install_dir,
github_organization,
images,
components,
library_paths,
)
if be == "vllm":
backend_clone(
be,
cmake_script,
backends[be],
script_build_dir,
script_install_dir,
github_organization,
)
else:
backend_build(
be,
cmake_script,
backends[be],
script_build_dir,
script_install_dir,
github_organization,
images,
components,
library_paths,
)

# Commands to build each repo agent...
for ra in repoagents:
Expand Down