diff --git a/functionary/openai_types.py b/functionary/openai_types.py index b382c61..c346d4d 100644 --- a/functionary/openai_types.py +++ b/functionary/openai_types.py @@ -120,7 +120,7 @@ class ChatCompletionRequest(BaseModel): tools: Optional[List[Tool]] = None function_call: Optional[Union[str, Function]] = None tool_choice: Optional[Union[str, Tool]] = None - temperature: Optional[float] = 0.6 + temperature: Union[Optional[float], str] = 0.6 top_p: Optional[float] = 1.0 n: Optional[int] = 1 max_tokens: Optional[int] = 512 diff --git a/functionary/sglang_inference.py b/functionary/sglang_inference.py index 1d87a0b..b45cb64 100644 --- a/functionary/sglang_inference.py +++ b/functionary/sglang_inference.py @@ -570,6 +570,9 @@ async def v1_chat_completions( """ request_json = await raw_request.json() request = ChatCompletionRequest(**request_json) + if type(request.temperature) is not float: + request.temperature = 1e-5 + tokenizer = ( tokenizer_manager.tokenizer if tokenizer_manager