Skip to content

Commit

Permalink
[Bug] [V1] Try fetching stop_reason from EngineOutput before checking…
Browse files Browse the repository at this point in the history
… the request (#13108)
  • Loading branch information
bnellnm authored Feb 12, 2025
1 parent f1042e8 commit f4d97e4
Showing 1 changed file with 6 additions and 4 deletions.
10 changes: 6 additions & 4 deletions vllm/v1/engine/output_processor.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

import asyncio
from dataclasses import dataclass
from typing import Dict, List, Optional
from typing import Dict, List, Optional, Union

from vllm.outputs import RequestOutput
from vllm.sampling_params import RequestOutputKind
Expand Down Expand Up @@ -164,6 +164,7 @@ def process_outputs(

new_token_ids = engine_core_output.new_token_ids
finish_reason = engine_core_output.finish_reason
stop_reason = engine_core_output.stop_reason

# TODO(andy): prompt logprobs + chunked prefill can
# result in engine core returning an output for a
Expand All @@ -181,9 +182,10 @@ def process_outputs(

# 2) Detokenize the token ids into text and check for stop
# strings.
stop_reason = req_state.detokenizer.update(new_token_ids)
if stop_reason:
stop_string = req_state.detokenizer.update(new_token_ids)
if stop_string and finish_reason != FinishReason.STOP:
finish_reason = FinishReason.STOP
stop_reason = stop_string

# 3) Compute sample and prompt logprobs for request,
# if required.
Expand Down Expand Up @@ -250,7 +252,7 @@ def _make_request_output(
request_state: RequestState,
new_token_ids: List[int],
finish_reason: Optional[FinishReason],
stop_reason: Optional[str],
stop_reason: Union[int, str, None],
) -> Optional[RequestOutput]:

finished = finish_reason is not None
Expand Down

0 comments on commit f4d97e4

Please sign in to comment.