Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: L0_sequence_batcher_cudashm #7852

Merged
merged 6 commits into from
Dec 4, 2024
Merged

fix: L0_sequence_batcher_cudashm #7852

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
d63dd85
bumping c++ version in custom sequence test backend
oandreeva-nv Nov 21, 2024
b25c8ea
checking cudaMemCopy
oandreeva-nv Nov 25, 2024
f4d6083
testing removinf explicit export of CUDA_VISIBLE_DEVICES
oandreeva-nv Nov 26, 2024
f7c5360
testing setting CUDA_VISIBLE_DEVICES by logic
oandreeva-nv Dec 2, 2024
62850e3
changing cudaMemcpy to cudaMemcpyAsync
oandreeva-nv Dec 4, 2024
34bdcb4
Restoring cuda visible devices
oandreeva-nv Dec 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/test/sequence/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Copyright 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# Copyright 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
Expand Down Expand Up @@ -43,7 +43,7 @@ add_library(
TritonSequenceBackend::triton-sequence-backend ALIAS triton-sequence-backend
)

target_compile_features(triton-sequence-backend PRIVATE cxx_std_11)
target_compile_features(triton-sequence-backend PRIVATE cxx_std_17)
target_compile_options(
triton-sequence-backend PRIVATE
$<$<OR:$<CXX_COMPILER_ID:Clang>,$<CXX_COMPILER_ID:AppleClang>,$<CXX_COMPILER_ID:GNU>>:
Expand Down
26 changes: 19 additions & 7 deletions src/test/sequence/src/sequence.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
// Copyright (c) 2021-2024, NVIDIA CORPORATION. All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
Expand Down Expand Up @@ -847,9 +847,15 @@ TRITONBACKEND_ModelInstanceExecute(
if (input_memory_type == TRITONSERVER_MEMORY_GPU) {
ipbuffer_vec.resize(input_element_cnt);
ipbuffer_int = ipbuffer_vec.data();
cudaMemcpy(
const_cast<int32_t*>(ipbuffer_int), input_buffer, input_byte_size,
cudaMemcpyDeviceToHost);
LOG_IF_CUDA_ERROR(
cudaMemcpyAsync(
const_cast<int32_t*>(ipbuffer_int), input_buffer, input_byte_size,
cudaMemcpyDeviceToHost, instance_state->CudaStream()),
"failed to copy buffer from Device to Host");

LOG_IF_CUDA_ERROR(
cudaStreamSynchronize(instance_state->CudaStream()),
"failed to perform synchronization on cuda stream");
} else {
ipbuffer_int = reinterpret_cast<const int32_t*>(input_buffer);
}
Expand Down Expand Up @@ -939,9 +945,15 @@ TRITONBACKEND_ModelInstanceExecute(
}

if (output_memory_type == TRITONSERVER_MEMORY_GPU) {
cudaMemcpy(
output_buffer, const_cast<int32_t*>(obuffer_int),
buffer_byte_size, cudaMemcpyHostToDevice);
LOG_IF_CUDA_ERROR(
cudaMemcpyAsync(
output_buffer, const_cast<int32_t*>(obuffer_int),
buffer_byte_size, cudaMemcpyHostToDevice,
instance_state->CudaStream()),
"failed to copy buffer from Device to Host");
LOG_IF_CUDA_ERROR(
cudaStreamSynchronize(instance_state->CudaStream()),
"failed to perform synchronization on cuda stream");
}
}
}
Expand Down
Loading