Fix error when tp > 1 (vllm-project#2644)

Co-authored-by: zhaoyang-star <[email protected]>
neuralmagic · Feb 13, 2024 · dcfc90b · dcfc90b
1 parent 18fb250
commit dcfc90b
Showing 1 changed file with 2 additions and 3 deletions.
diff --git a/vllm/engine/llm_engine.py b/vllm/engine/llm_engine.py
@@ -237,7 +237,6 @@ def _init_workers_ray(self, placement_group: "PlacementGroup",
         model_config = copy.deepcopy(self.model_config)
         parallel_config = copy.deepcopy(self.parallel_config)
         scheduler_config = copy.deepcopy(self.scheduler_config)
-        cache_config = copy.deepcopy(self.cache_config)
 
         for rank, (worker, (node_id,
                             _)) in enumerate(zip(self.workers,
@@ -253,7 +252,7 @@ def _init_workers_ray(self, placement_group: "PlacementGroup",
                     rank,
                     distributed_init_method,
                     lora_config=self.lora_config,
-                    cache_config=cache_config,
+                    kv_cache_dtype=self.cache_config.cache_dtype,
                 ))
 
         driver_rank = 0
@@ -266,7 +265,7 @@ def _init_workers_ray(self, placement_group: "PlacementGroup",
             driver_rank,
             distributed_init_method,
             lora_config=self.lora_config,
-            cache_config=cache_config,
+            kv_cache_dtype=self.cache_config.cache_dtype,
             is_driver_worker=True,
         )