stream generator failed.. using openai compat api(s) ie: requesty.ai, openrouter ..

[quantumalchemy]

letta [0.6.41]
Trying to use openai api services like
requesty.ai, openrouter .. using latest function call llms like qwq 32B always generating
"stream generator failed." errors

config: using the sdk:
agent = client.agents.create(
name="thedude", 
memory_blocks=[
	CreateBlock(
		label="human",
		value="Name: The Dude",
	),
	CreateBlock(
		label="persona",
		value="You are a helpful assistant",
	),
],
llm_config=LlmConfig(
    model="nebius/Qwen/QwQ-32B",
	model_endpoint_type="openai",
	model_endpoint="https://router.requesty.ai/v1",
	api_key=os.getenv("OPENAI_API_KEY"),
	context_window=16384
),
embedding_config=EmbeddingConfig(embedding_endpoint_type='hugging-face', embedding_endpoint='https://embeddings.memgpt.ai', embedding_model='letta-free', embedding_dim=1024, embedding_chunk_size=300),
)

ERROR LOG:

File "/app/letta/llm_api/llm_api_tools.py", line 61, in wrapper

return func(*args, **kwargs)

       ^^^^^^^^^^^^^^^^^^^^^

File "/app/letta/llm_api/llm_api_tools.py", line 205, in create

response = openai_chat_completions_process_stream(

           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

File "/app/letta/llm_api/openai.py", line 387, in openai_chat_completions_process_stream

raise e

File "/app/letta/llm_api/openai.py", line 255, in openai_chat_completions_process_stream

for chat_completion_chunk in openai_chat_completions_request_stream(

File "/app/letta/llm_api/openai.py", line 424, in openai_chat_completions_request_stream

stream = client.chat.completions.create(**data)

         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

File "/app/.venv/lib/python3.11/site-packages/openai/_utils/_utils.py", line 279, in wrapper

return func(*args, **kwargs)

       ^^^^^^^^^^^^^^^^^^^^^

File "/app/.venv/lib/python3.11/site-packages/openai/resources/chat/completions/completions.py", line 914, in create

return self._post(

       ^^^^^^^^^^^

File "/app/.venv/lib/python3.11/site-packages/openai/_base_client.py", line 1242, in post

return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))

                       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

File "/app/.venv/lib/python3.11/site-packages/openai/_base_client.py", line 919, in request

return self._request(

       ^^^^^^^^^^^^^^

File "/app/.venv/lib/python3.11/site-packages/openai/_base_client.py", line 1023, in _request

raise self._make_status_error_from_response(err.response) from None

openai.BadRequestError: Error code: 400 - {'error': {'message': ''}}

None

Traceback (most recent call last):

File "/app/letta/server/rest_api/utils.py", line 77, in sse_async_generator

usage = await usage_task

        ^^^^^^^^^^^^^^^^

File "/usr/lib/python3.11/asyncio/threads.py", line 25, in to_thread

return await loop.run_in_executor(None, func_call)

       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

File "/usr/lib/python3.11/concurrent/futures/thread.py", line 58, in run

result = self.fn(*self.args, **self.kwargs)

         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

File "/app/letta/server/server.py", line 716, in send_messages

return self._step(

       ^^^^^^^^^^^

File "/app/letta/server/server.py", line 399, in _step

usage_stats = letta_agent.step(

              ^^^^^^^^^^^^^^^^^

File "/app/letta/tracing.py", line 188, in sync_wrapper

return func(*args, **kwargs)

       ^^^^^^^^^^^^^^^^^^^^^

File "/app/letta/agent.py", line 779, in step

step_response = self.inner_step(

                ^^^^^^^^^^^^^^^^

File "/app/letta/agent.py", line 1055, in inner_step

raise e

File "/app/letta/agent.py", line 892, in inner_step

response = self._get_ai_reply(

           ^^^^^^^^^^^^^^^^^^^

File "/app/letta/tracing.py", line 188, in sync_wrapper

return func(*args, **kwargs)

       ^^^^^^^^^^^^^^^^^^^^^

File "/app/letta/agent.py", line 455, in _get_ai_reply

raise e

File "/app/letta/agent.py", line 414, in _get_ai_reply

response = create(

           ^^^^^^^

File "/app/letta/tracing.py", line 188, in sync_wrapper

return func(*args, **kwargs)

       ^^^^^^^^^^^^^^^^^^^^^

File "/app/letta/llm_api/llm_api_tools.py", line 118, in wrapper

raise e

File "/app/letta/llm_api/llm_api_tools.py", line 61, in wrapper

return func(*args, **kwargs)

       ^^^^^^^^^^^^^^^^^^^^^

File "/app/letta/llm_api/llm_api_tools.py", line 205, in create

response = openai_chat_completions_process_stream(

           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

File "/app/letta/llm_api/openai.py", line 387, in openai_chat_completions_process_stream

raise e

File "/app/letta/llm_api/openai.py", line 255, in openai_chat_completions_process_stream

for chat_completion_chunk in openai_chat_completions_request_stream(

File "/app/letta/llm_api/openai.py", line 424, in openai_chat_completions_request_stream

stream = client.chat.completions.create(**data)

         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

File "/app/.venv/lib/python3.11/site-packages/openai/_utils/_utils.py", line 279, in wrapper

return func(*args, **kwargs)

       ^^^^^^^^^^^^^^^^^^^^^

File "/app/.venv/lib/python3.11/site-packages/openai/resources/chat/completions/completions.py", line 914, in create

return self._post(

       ^^^^^^^^^^^

File "/app/.venv/lib/python3.11/site-packages/openai/_base_client.py", line 1242, in post

return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))

                       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

File "/app/.venv/lib/python3.11/site-packages/openai/_base_client.py", line 919, in request

return self._request(

       ^^^^^^^^^^^^^^

File "/app/.venv/lib/python3.11/site-packages/openai/_base_client.py", line 1023, in _request

raise self._make_status_error_from_response(err.response) from None

openai.BadRequestError: Error code: 400 - {'error': {'message': ''}}

/app/letta/server/rest_api/utils.py:133: UserWarning: SSE stream generator failed: Error code: 400 - {'error': {'message': ''}}

warnings.warn(f"SSE stream generator failed: {e}")

Letta.letta.server.rest_api.utils - ERROR - Caught unexpected Exception: Error code: 400 - {'error': {'message': ''}}

maybe these 'services' are not supporting streaming?, tool use, function calls? anyone successfully using any llms from requesty.ai, ?