From 0d028cc4f094da8c2fb122ba8440597f49fd8077 Mon Sep 17 00:00:00 2001
From: Arpan Tripathi <tripathiarpan20@gmail.com>
Date: Wed, 2 Oct 2024 15:18:23 +0100
Subject: [PATCH] cherry picked PR #8870

---
 vllm/worker/enc_dec_model_runner.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/vllm/worker/enc_dec_model_runner.py b/vllm/worker/enc_dec_model_runner.py
index bd716ac3e7ec3..4679524c76173 100644
--- a/vllm/worker/enc_dec_model_runner.py
+++ b/vllm/worker/enc_dec_model_runner.py
@@ -268,11 +268,13 @@ def prepare_model_input(
             encoder_input_positions=encoder_input_positions_tensor,
         )
 
+        generators = self.get_generators(finished_requests_ids)
         sampling_metadata = SamplingMetadata.prepare(seq_group_metadata_list,
                                                      model_input.seq_lens,
                                                      model_input.query_lens,
                                                      self.device,
-                                                     self.pin_memory)
+                                                     self.pin_memory,
+                                                     generators=generators)
         is_prompt = (seq_group_metadata_list[0].is_prompt
                      if seq_group_metadata_list else None)
         return dataclasses.replace(model_input,