From 0c034befe79a35d601a85f0e12e3cd6355874686 Mon Sep 17 00:00:00 2001 From: dhuangnm Date: Mon, 10 Jun 2024 12:13:36 -0400 Subject: [PATCH 01/13] githash --- CMakeLists.txt | 10 +++++++--- cmake/dep.cmake | 6 ++++++ csrc/pybind.cpp | 9 +++++++++ vllm/__init__.py | 5 +++++ 4 files changed, 27 insertions(+), 3 deletions(-) create mode 100644 cmake/dep.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 35846fd1cfa99..01ab079a2ecc7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,6 +8,7 @@ message(STATUS "Build type: ${CMAKE_BUILD_TYPE}") message(STATUS "Target device: ${VLLM_TARGET_DEVICE}") include(${CMAKE_CURRENT_LIST_DIR}/cmake/utils.cmake) +include(${CMAKE_CURRENT_LIST_DIR}/cmake/dep.cmake) # # Supported python versions. These versions will be searched in order, the @@ -215,7 +216,8 @@ define_gpu_extension_target( COMPILE_FLAGS ${VLLM_GPU_FLAGS} ARCHITECTURES ${VLLM_GPU_ARCHES} INCLUDE_DIRECTORIES ${CUTLASS_INCLUDE_DIR};${CUTLASS_TOOLS_UTIL_INCLUDE_DIR} - WITH_SOABI) + WITH_SOABI + cmake_git_version_tracking) # # _moe_C extension @@ -232,7 +234,8 @@ define_gpu_extension_target( SOURCES ${VLLM_MOE_EXT_SRC} COMPILE_FLAGS ${VLLM_GPU_FLAGS} ARCHITECTURES ${VLLM_GPU_ARCHES} - WITH_SOABI) + WITH_SOABI + cmake_git_version_tracking) # # _punica_C extension @@ -283,7 +286,8 @@ if (VLLM_PUNICA_GPU_ARCHES) SOURCES ${VLLM_PUNICA_EXT_SRC} COMPILE_FLAGS ${VLLM_PUNICA_GPU_FLAGS} ARCHITECTURES ${VLLM_PUNICA_GPU_ARCHES} - WITH_SOABI) + WITH_SOABI + cmake_git_version_tracking) else() message(WARNING "Unable to create _punica_C target because none of the " "requested architectures (${VLLM_GPU_ARCHES}) are supported, i.e. >= 8.0") diff --git a/cmake/dep.cmake b/cmake/dep.cmake new file mode 100644 index 0000000000000..006fa6904aac7 --- /dev/null +++ b/cmake/dep.cmake @@ -0,0 +1,6 @@ +include(FetchContent) +FetchContent_Declare(cmake_git_version_tracking + GIT_REPOSITORY https://github.com/andrew-hardin/cmake-git-version-tracking.git + GIT_TAG 6c0cb87edd029ddfb403a8e24577c144a03605a6 +) +FetchContent_MakeAvailable(cmake_git_version_tracking) diff --git a/csrc/pybind.cpp b/csrc/pybind.cpp index f5b4865506568..e4e5dfdaf116f 100644 --- a/csrc/pybind.cpp +++ b/csrc/pybind.cpp @@ -2,11 +2,20 @@ #include "cuda_utils.h" #include "ops.h" #include +#include + +std::string githash() { return std::string{git::CommitSHA1()}; } PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { // vLLM custom ops pybind11::module ops = m.def_submodule("ops", "vLLM custom operators"); + // githash + ops.def( + "githash", + &githash, + "nm-vllm git hash"); + // Attention ops ops.def( "paged_attention_v1", diff --git a/vllm/__init__.py b/vllm/__init__.py index eb137ce9442e9..dcc16491e3577 100644 --- a/vllm/__init__.py +++ b/vllm/__init__.py @@ -14,7 +14,12 @@ # UPSTREAM SYNC: use the current downstream. __version__ = "0.5.0" +def githash(): + import torch + return torch.ops.githash() + __all__ = [ + "githash", "LLM", "ModelRegistry", "SamplingParams", From 9f38095f0519d41322cf201bb95ca82b42116791 Mon Sep 17 00:00:00 2001 From: dhuangnm Date: Mon, 10 Jun 2024 15:56:14 -0400 Subject: [PATCH 02/13] fix issue and add more places --- CMakeLists.txt | 6 +++--- csrc/cpu/pybind.cpp | 9 +++++++++ csrc/punica/punica_pybind.cpp | 5 ++++- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 01ab079a2ecc7..b419ae8104da1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -217,7 +217,7 @@ define_gpu_extension_target( ARCHITECTURES ${VLLM_GPU_ARCHES} INCLUDE_DIRECTORIES ${CUTLASS_INCLUDE_DIR};${CUTLASS_TOOLS_UTIL_INCLUDE_DIR} WITH_SOABI - cmake_git_version_tracking) + LIBRARIES cmake_git_version_tracking) # # _moe_C extension @@ -235,7 +235,7 @@ define_gpu_extension_target( COMPILE_FLAGS ${VLLM_GPU_FLAGS} ARCHITECTURES ${VLLM_GPU_ARCHES} WITH_SOABI - cmake_git_version_tracking) + LIBRARIES cmake_git_version_tracking) # # _punica_C extension @@ -287,7 +287,7 @@ if (VLLM_PUNICA_GPU_ARCHES) COMPILE_FLAGS ${VLLM_PUNICA_GPU_FLAGS} ARCHITECTURES ${VLLM_PUNICA_GPU_ARCHES} WITH_SOABI - cmake_git_version_tracking) + LIBRARIES cmake_git_version_tracking) else() message(WARNING "Unable to create _punica_C target because none of the " "requested architectures (${VLLM_GPU_ARCHES}) are supported, i.e. >= 8.0") diff --git a/csrc/cpu/pybind.cpp b/csrc/cpu/pybind.cpp index bba044087f37c..00822ce406e06 100644 --- a/csrc/cpu/pybind.cpp +++ b/csrc/cpu/pybind.cpp @@ -2,11 +2,20 @@ #include "cuda_utils.h" #include "ops.h" #include +#include + +std::string githash() { return std::string{git::CommitSHA1()}; } PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { // vLLM custom ops pybind11::module ops = m.def_submodule("ops", "vLLM custom operators"); + // githash + ops.def( + "githash", + &githash, + "nm-vllm git hash"); + // Attention ops ops.def( "paged_attention_v1", diff --git a/csrc/punica/punica_pybind.cpp b/csrc/punica/punica_pybind.cpp index 9490ad59cdd5f..a1e63e3955c3a 100644 --- a/csrc/punica/punica_pybind.cpp +++ b/csrc/punica/punica_pybind.cpp @@ -1,12 +1,15 @@ #include - +#include #include "punica_ops.h" //====== pybind ====== #define DEFINE_pybind(name) m.def(#name, &name, #name); +std::string githash() { return std::string{git::CommitSHA1()}; } + PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("githash", &githash, "nm-vllm git hash"); m.def("dispatch_bgmv", &dispatch_bgmv, "dispatch_bgmv"); m.def("dispatch_bgmv_low_level", &dispatch_bgmv_low_level, "dispatch_bgmv_low_level"); From 3ffcd0b3468ee579177ff1c4f96b4e3a4053c1be Mon Sep 17 00:00:00 2001 From: dhuangnm Date: Tue, 11 Jun 2024 09:57:22 -0400 Subject: [PATCH 03/13] fix clang error --- csrc/cpu/pybind.cpp | 5 +---- csrc/pybind.cpp | 5 +---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/csrc/cpu/pybind.cpp b/csrc/cpu/pybind.cpp index 00822ce406e06..7ba7b7b8c12bd 100644 --- a/csrc/cpu/pybind.cpp +++ b/csrc/cpu/pybind.cpp @@ -11,10 +11,7 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { pybind11::module ops = m.def_submodule("ops", "vLLM custom operators"); // githash - ops.def( - "githash", - &githash, - "nm-vllm git hash"); + ops.def("githash", &githash, "Show nm-vllm git hash."); // Attention ops ops.def( diff --git a/csrc/pybind.cpp b/csrc/pybind.cpp index e4e5dfdaf116f..10dd00d4beeda 100644 --- a/csrc/pybind.cpp +++ b/csrc/pybind.cpp @@ -11,10 +11,7 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { pybind11::module ops = m.def_submodule("ops", "vLLM custom operators"); // githash - ops.def( - "githash", - &githash, - "nm-vllm git hash"); + ops.def("githash", &githash, "Show nm-vllm git hash."); // Attention ops ops.def( From d975432d31acb6fca43aa913bc06cc6a36199904 Mon Sep 17 00:00:00 2001 From: dhuangnm Date: Tue, 11 Jun 2024 10:12:23 -0400 Subject: [PATCH 04/13] fix namespace error --- vllm/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vllm/__init__.py b/vllm/__init__.py index dcc16491e3577..f5af4db222dea 100644 --- a/vllm/__init__.py +++ b/vllm/__init__.py @@ -15,8 +15,8 @@ __version__ = "0.5.0" def githash(): - import torch - return torch.ops.githash() + from torch.ops._C import ops + return ops.githash() __all__ = [ "githash", From 106796861914146372aba9386aeff9361edfb34d Mon Sep 17 00:00:00 2001 From: dhuangnm Date: Tue, 11 Jun 2024 10:27:21 -0400 Subject: [PATCH 05/13] fix yapf error --- vllm/__init__.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/vllm/__init__.py b/vllm/__init__.py index f5af4db222dea..1efa6a4a9503e 100644 --- a/vllm/__init__.py +++ b/vllm/__init__.py @@ -14,10 +14,6 @@ # UPSTREAM SYNC: use the current downstream. __version__ = "0.5.0" -def githash(): - from torch.ops._C import ops - return ops.githash() - __all__ = [ "githash", "LLM", @@ -34,3 +30,8 @@ def githash(): "initialize_ray_cluster", "PoolingParams", ] + + +def githash(): + from torch.ops._C import ops + return ops.githash() From cb7170e153ee656a9952c31ec97cb1c53fc20c1e Mon Sep 17 00:00:00 2001 From: dhuangnm Date: Tue, 11 Jun 2024 15:38:01 -0400 Subject: [PATCH 06/13] clean up and add git hash to collect_env.py --- collect_env.py | 8 ++++++++ vllm/__init__.py | 4 ++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/collect_env.py b/collect_env.py index c89f8c64eddcb..c1c54d424ec15 100644 --- a/collect_env.py +++ b/collect_env.py @@ -22,6 +22,7 @@ SystemEnv = namedtuple( 'SystemEnv', [ + 'vllm_git_hash', 'torch_version', 'is_debug_build', 'cuda_compiled_version', @@ -138,6 +139,11 @@ def get_conda_packages(run_lambda, patterns=None): for name in patterns)) +def get_vllm_git_hash(): + import vllm + return vllm.githash() + + def get_gcc_version(run_lambda): return run_and_parse_first_match(run_lambda, 'gcc --version', r'gcc (.*)') @@ -536,6 +542,7 @@ def get_version_or_na(cfg, prefix): gpu_topo = get_gpu_topo(run_lambda) return SystemEnv( + vllm_git_hash=get_vllm_git_hash(), torch_version=version_str, is_debug_build=debug_mode_str, python_version='{} ({}-bit runtime)'.format( @@ -583,6 +590,7 @@ def get_version_or_na(cfg, prefix): CMake version: {cmake_version} Libc version: {libc_version} +vllm git hash: {vllm_git_hash} Python version: {python_version} Python platform: {python_platform} Is CUDA available: {is_cuda_available} diff --git a/vllm/__init__.py b/vllm/__init__.py index 1efa6a4a9503e..64eb4f65d917b 100644 --- a/vllm/__init__.py +++ b/vllm/__init__.py @@ -33,5 +33,5 @@ def githash(): - from torch.ops._C import ops - return ops.githash() + from vllm._C import ops as vllm_ops + return vllm_ops.githash() From bc8830394d9b52a48f1684de18084ee58b882f8c Mon Sep 17 00:00:00 2001 From: dhuangnm Date: Tue, 11 Jun 2024 22:15:08 -0400 Subject: [PATCH 07/13] fix import issue due to local vllm directory --- collect_env.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/collect_env.py b/collect_env.py index c1c54d424ec15..c9e7d05595452 100644 --- a/collect_env.py +++ b/collect_env.py @@ -15,6 +15,8 @@ try: import torch TORCH_AVAILABLE = True + installed_path = os.path.dirname(torch.__file__) + sys.path.insert(0, os.path.dirname(installed_path)) except (ImportError, NameError, AttributeError, OSError): TORCH_AVAILABLE = False @@ -140,8 +142,11 @@ def get_conda_packages(run_lambda, patterns=None): def get_vllm_git_hash(): - import vllm - return vllm.githash() + try: + import vllm + return vllm.githash() + except ImportError: + return 'N/A' def get_gcc_version(run_lambda): From e4c1d6f70a75359d070b134ac3c5f374a323fd14 Mon Sep 17 00:00:00 2001 From: dhuangnm Date: Tue, 11 Jun 2024 22:57:11 -0400 Subject: [PATCH 08/13] move git hash to where vLLM info locates --- collect_env.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/collect_env.py b/collect_env.py index c9e7d05595452..4bbb9240a7762 100644 --- a/collect_env.py +++ b/collect_env.py @@ -595,7 +595,6 @@ def get_version_or_na(cfg, prefix): CMake version: {cmake_version} Libc version: {libc_version} -vllm git hash: {vllm_git_hash} Python version: {python_version} Python platform: {python_platform} Is CUDA available: {is_cuda_available} @@ -620,6 +619,7 @@ def get_version_or_na(cfg, prefix): ROCM Version: {rocm_version} Neuron SDK Version: {neuron_sdk_version} vLLM Version: {vllm_version} +vLLM Git Hash: {vllm_git_hash} vLLM Build Flags: {vllm_build_flags} GPU Topology: From 84e1f1c16af20e9f0c2f545039a66f96fb2673ca Mon Sep 17 00:00:00 2001 From: dhuangnm Date: Wed, 12 Jun 2024 09:26:56 -0400 Subject: [PATCH 09/13] punica is not needed --- csrc/punica/punica_pybind.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/csrc/punica/punica_pybind.cpp b/csrc/punica/punica_pybind.cpp index a1e63e3955c3a..4f0537a3ef1ca 100644 --- a/csrc/punica/punica_pybind.cpp +++ b/csrc/punica/punica_pybind.cpp @@ -1,15 +1,11 @@ #include -#include #include "punica_ops.h" //====== pybind ====== #define DEFINE_pybind(name) m.def(#name, &name, #name); -std::string githash() { return std::string{git::CommitSHA1()}; } - PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { - m.def("githash", &githash, "nm-vllm git hash"); m.def("dispatch_bgmv", &dispatch_bgmv, "dispatch_bgmv"); m.def("dispatch_bgmv_low_level", &dispatch_bgmv_low_level, "dispatch_bgmv_low_level"); From 91b5bb02a76404e29bc745f725bae5e13c3f5766 Mon Sep 17 00:00:00 2001 From: dhuangnm Date: Wed, 12 Jun 2024 09:29:34 -0400 Subject: [PATCH 10/13] restore file --- csrc/punica/punica_pybind.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/csrc/punica/punica_pybind.cpp b/csrc/punica/punica_pybind.cpp index 4f0537a3ef1ca..9490ad59cdd5f 100644 --- a/csrc/punica/punica_pybind.cpp +++ b/csrc/punica/punica_pybind.cpp @@ -1,4 +1,5 @@ #include + #include "punica_ops.h" //====== pybind ====== From 0e9fd2213754650051188b7f0064d9975efa6a2b Mon Sep 17 00:00:00 2001 From: dhuangnm Date: Mon, 17 Jun 2024 09:46:18 -0400 Subject: [PATCH 11/13] fix issue --- CMakeLists.txt | 1 + vllm/__init__.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 94bfe8385e00b..fab73ebc4a101 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,6 +8,7 @@ message(STATUS "Build type: ${CMAKE_BUILD_TYPE}") message(STATUS "Target device: ${VLLM_TARGET_DEVICE}") include(${CMAKE_CURRENT_LIST_DIR}/cmake/utils.cmake) +include(${CMAKE_CURRENT_LIST_DIR}/cmake/dep.cmake) # # Supported python versions. These versions will be searched in order, the diff --git a/vllm/__init__.py b/vllm/__init__.py index 16e23ce55315e..84dc6399e83ae 100644 --- a/vllm/__init__.py +++ b/vllm/__init__.py @@ -37,5 +37,5 @@ def githash(): - from vllm._C import ops as vllm_ops - return vllm_ops.githash() + import torch + return torch.ops._C.githash() From ec1715a452635d36348752f918bdb6f77cdf0bbe Mon Sep 17 00:00:00 2001 From: dhuangnm Date: Mon, 17 Jun 2024 12:27:58 -0400 Subject: [PATCH 12/13] address comment --- collect_env.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/collect_env.py b/collect_env.py index 95c05af223897..5a9ed4822c8b8 100644 --- a/collect_env.py +++ b/collect_env.py @@ -148,7 +148,7 @@ def get_vllm_git_hash(): import vllm return vllm.githash() except ImportError: - return 'N/A' + return 'None' def get_gcc_version(run_lambda): From 11127281ee4d64c088fc633ef783ed9f6b5ab100 Mon Sep 17 00:00:00 2001 From: dhuangnm Date: Mon, 17 Jun 2024 22:20:26 -0400 Subject: [PATCH 13/13] return None --- collect_env.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/collect_env.py b/collect_env.py index 5a9ed4822c8b8..0dbeaf5f8ad66 100644 --- a/collect_env.py +++ b/collect_env.py @@ -148,7 +148,7 @@ def get_vllm_git_hash(): import vllm return vllm.githash() except ImportError: - return 'None' + return None def get_gcc_version(run_lambda):