fix temperature types

MeetKai · Jan 8, 2025 · e9ec966 · e9ec966
1 parent dfd08d3
commit e9ec966
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 1 deletion.
diff --git a/functionary/openai_types.py b/functionary/openai_types.py
@@ -120,7 +120,7 @@ class ChatCompletionRequest(BaseModel):
     tools: Optional[List[Tool]] = None
     function_call: Optional[Union[str, Function]] = None
     tool_choice: Optional[Union[str, Tool]] = None
-    temperature: Optional[float] = 0.6
+    temperature: Union[Optional[float], str] = 0.6
     top_p: Optional[float] = 1.0
     n: Optional[int] = 1
     max_tokens: Optional[int] = 512

diff --git a/functionary/sglang_inference.py b/functionary/sglang_inference.py
@@ -570,6 +570,9 @@ async def v1_chat_completions(
     """
     request_json = await raw_request.json()
     request = ChatCompletionRequest(**request_json)
+    if type(request.temperature) is not float:
+        request.temperature = 1e-5
+
     tokenizer = (
         tokenizer_manager.tokenizer
         if tokenizer_manager