From 3cce04b52eb2f68cb0d84b5ffab4a55953af4f3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniele=20Trifir=C3=B2?= Date: Wed, 24 Apr 2024 14:44:08 +0200 Subject: [PATCH] grpc_server: fix tokenizer group usage --- vllm/entrypoints/grpc/grpc_server.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/vllm/entrypoints/grpc/grpc_server.py b/vllm/entrypoints/grpc/grpc_server.py index f4450e175dc24..c8ebc2fa6b214 100644 --- a/vllm/entrypoints/grpc/grpc_server.py +++ b/vllm/entrypoints/grpc/grpc_server.py @@ -107,7 +107,6 @@ def __init__(self, engine: AsyncLLMEngine, args: argparse.Namespace): self.engine: AsyncLLMEngine = engine # These set in _post_init() - self.tokenizer_group: BaseTokenizerGroup = None self.tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast] = None self.config: ModelConfig = None @@ -116,9 +115,13 @@ def __init__(self, engine: AsyncLLMEngine, args: argparse.Namespace): self.skip_special_tokens = not args.output_special_tokens self.default_include_stop_seqs = args.default_include_stop_seqs + @property + def tokenizer_group(self) -> BaseTokenizerGroup: + return self.engine.engine + + async def _post_init(self): self.config = await self.engine.get_model_config() - self.tokenizer_group = await self.engine.get_tokenizer_group() self.tokenizer = await self.engine.get_tokenizer() # Swap in the special TGIS stats logger