Fixes assertion failure in prefix caching: the lora index mapping sho…

…uld respect prefix_len (vllm-project#2688) Signed-off-by: Tao He <[email protected]>
xjpang · Feb 20, 2024 · 3f4374a · 3f4374a
1 parent a90d068
commit 3f4374a
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/vllm/worker/model_runner.py b/vllm/worker/model_runner.py
@@ -142,10 +142,10 @@ def _prepare_prompt(
             if lora_id > 0:
                 lora_requests.add(seq_group_metadata.lora_request)
 
-            lora_index_mapping.append([lora_id] * prompt_len)
+            lora_index_mapping.append([lora_id] * (prompt_len - prefix_len))
             lora_prompt_mapping.extend(
                 [lora_id] *
-                (prompt_len
+                (prompt_len - prefix_len
                  if seq_group_metadata.sampling_params.prompt_logprobs else 1))
 
             if seq_group_metadata.block_tables is None: