From 3dc95b64c9a3f0a9e3bc60e74034ee0290e45e0d Mon Sep 17 00:00:00 2001 From: Din Date: Sat, 25 Jan 2025 20:23:51 +0500 Subject: [PATCH] fix(openai): add response id attribute --- .../instrumentation/openai/shared/__init__.py | 3 ++- .../openai/shared/chat_wrappers.py | 5 +++-- .../openai/shared/completion_wrappers.py | 6 +++--- .../openai/v1/assistant_wrappers.py | 1 + .../openai/v1/event_handler_wrapper.py | 8 ++++++-- .../tests/traces/test_assistant.py | 18 ++++++++++++++++++ .../tests/traces/test_azure.py | 4 ++++ .../tests/traces/test_chat.py | 11 +++++++++-- .../tests/traces/test_completions.py | 7 +++++++ .../tests/traces/test_functions.py | 5 +++++ .../tests/traces/test_structured.py | 4 ++++ .../tests/traces/test_vision.py | 2 ++ 12 files changed, 64 insertions(+), 10 deletions(-) diff --git a/packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/shared/__init__.py b/packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/shared/__init__.py index c562bc0d3..e54902ac5 100644 --- a/packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/shared/__init__.py +++ b/packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/shared/__init__.py @@ -11,6 +11,7 @@ from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator from opentelemetry.instrumentation.openai.shared.config import Config +from opentelemetry.semconv._incubating.attributes.gen_ai_attributes import GEN_AI_RESPONSE_ID from opentelemetry.semconv_ai import SpanAttributes from opentelemetry.instrumentation.openai.utils import ( dont_throw, @@ -149,6 +150,7 @@ def _set_response_attributes(span, response): return _set_span_attribute(span, SpanAttributes.LLM_RESPONSE_MODEL, response.get("model")) + _set_span_attribute(span, GEN_AI_RESPONSE_ID, response.get("id")) _set_span_attribute( span, @@ -156,7 +158,6 @@ def _set_response_attributes(span, response): response.get("system_fingerprint"), ) _log_prompt_filter(span, response) - usage = response.get("usage") if not usage: return diff --git a/packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/shared/chat_wrappers.py b/packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/shared/chat_wrappers.py index a407e4349..5d06b068a 100644 --- a/packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/shared/chat_wrappers.py +++ b/packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/shared/chat_wrappers.py @@ -700,7 +700,7 @@ def _build_from_streaming_response( start_time=None, request_kwargs=None, ): - complete_response = {"choices": [], "model": ""} + complete_response = {"choices": [], "model": "", "id": ""} first_token = True time_of_first_token = start_time # will be updated when first token is received @@ -767,7 +767,7 @@ async def _abuild_from_streaming_response( start_time=None, request_kwargs=None, ): - complete_response = {"choices": [], "model": ""} + complete_response = {"choices": [], "model": "", "id": ""} first_token = True time_of_first_token = start_time # will be updated when first token is received @@ -826,6 +826,7 @@ def _accumulate_stream_items(item, complete_response): item = model_as_dict(item) complete_response["model"] = item.get("model") + complete_response["id"] = item.get("id") # prompt filter results if item.get("prompt_filter_results"): diff --git a/packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/shared/completion_wrappers.py b/packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/shared/completion_wrappers.py index 8ce53c777..5922c2c22 100644 --- a/packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/shared/completion_wrappers.py +++ b/packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/shared/completion_wrappers.py @@ -142,7 +142,7 @@ def _set_completions(span, choices): @dont_throw def _build_from_streaming_response(span, request_kwargs, response): - complete_response = {"choices": [], "model": ""} + complete_response = {"choices": [], "model": "", "id": ""} for item in response: yield item _accumulate_streaming_response(complete_response, item) @@ -160,7 +160,7 @@ def _build_from_streaming_response(span, request_kwargs, response): @dont_throw async def _abuild_from_streaming_response(span, request_kwargs, response): - complete_response = {"choices": [], "model": ""} + complete_response = {"choices": [], "model": "", "id": ""} async for item in response: yield item _accumulate_streaming_response(complete_response, item) @@ -215,7 +215,7 @@ def _accumulate_streaming_response(complete_response, item): item = model_as_dict(item) complete_response["model"] = item.get("model") - + complete_response["id"] = item.get("id") for choice in item.get("choices"): index = choice.get("index") if len(complete_response.get("choices")) <= index: diff --git a/packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/v1/assistant_wrappers.py b/packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/v1/assistant_wrappers.py index 8a140100c..dfd3d0e8c 100644 --- a/packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/v1/assistant_wrappers.py +++ b/packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/v1/assistant_wrappers.py @@ -151,6 +151,7 @@ def messages_list_wrapper(tracer, wrapped, instance, args, kwargs): _set_span_attribute( span, f"{prefix}.content", content[0].get("text").get("value") ) + _set_span_attribute(span, f"gen_ai.response.{i}.id", msg.get("id")) if run.get("usage"): usage_dict = model_as_dict(run.get("usage")) diff --git a/packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/v1/event_handler_wrapper.py b/packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/v1/event_handler_wrapper.py index acd0e004a..50a3602c8 100644 --- a/packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/v1/event_handler_wrapper.py +++ b/packages/opentelemetry-instrumentation-openai/opentelemetry/instrumentation/openai/v1/event_handler_wrapper.py @@ -81,7 +81,13 @@ def on_message_delta(self, delta, snapshot): @override def on_message_done(self, message): + _set_span_attribute( + self._span, + f"gen_ai.response.{self._current_text_index}.id", + message.id, + ) self._original_handler.on_message_done(message) + self._current_text_index += 1 @override def on_text_created(self, text): @@ -105,8 +111,6 @@ def on_text_done(self, text): text.value, ) - self._current_text_index += 1 - @override def on_image_file_done(self, image_file): self._original_handler.on_image_file_done(image_file) diff --git a/packages/opentelemetry-instrumentation-openai/tests/traces/test_assistant.py b/packages/opentelemetry-instrumentation-openai/tests/traces/test_assistant.py index 571b3aa68..c6ca8f99f 100644 --- a/packages/opentelemetry-instrumentation-openai/tests/traces/test_assistant.py +++ b/packages/opentelemetry-instrumentation-openai/tests/traces/test_assistant.py @@ -82,6 +82,10 @@ def test_new_assistant(exporter, openai_client, assistant): open_ai_span.attributes[f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.role"] == message.role ) + assert ( + open_ai_span.attributes[f"gen_ai.response.{idx}.id"] + == message.id + ) @pytest.mark.vcr @@ -134,6 +138,10 @@ def test_new_assistant_with_polling(exporter, openai_client, assistant): == message.content[0].text.value ) assert open_ai_span.attributes[f"gen_ai.completion.{idx}.role"] == message.role + assert ( + open_ai_span.attributes[f"gen_ai.response.{idx}.id"] + == message.id + ) @pytest.mark.vcr @@ -202,6 +210,10 @@ def test_existing_assistant(exporter, openai_client): open_ai_span.attributes[f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.role"] == message.role ) + assert ( + open_ai_span.attributes[f"gen_ai.response.{idx}.id"] + == message.id + ) @pytest.mark.vcr @@ -272,6 +284,9 @@ def on_text_delta(self, delta, snapshot): open_ai_span.attributes[f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.role"] == "assistant" ) + assert ( + open_ai_span.attributes[f"gen_ai.response.{idx}.id"].startswith("msg") + ) @pytest.mark.vcr @@ -341,3 +356,6 @@ def on_text_delta(self, delta, snapshot): open_ai_span.attributes[f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.role"] == "assistant" ) + assert ( + open_ai_span.attributes[f"gen_ai.response.{idx}.id"].startswith("msg_") + ) diff --git a/packages/opentelemetry-instrumentation-openai/tests/traces/test_azure.py b/packages/opentelemetry-instrumentation-openai/tests/traces/test_azure.py index 28a4dce7f..04d3e5bc6 100644 --- a/packages/opentelemetry-instrumentation-openai/tests/traces/test_azure.py +++ b/packages/opentelemetry-instrumentation-openai/tests/traces/test_azure.py @@ -29,6 +29,7 @@ def test_chat(exporter, azure_openai_client): == "https://traceloop-stg.openai.azure.com//openai/" ) assert open_ai_span.attributes.get(SpanAttributes.LLM_IS_STREAMING) is False + assert open_ai_span.attributes.get("gen_ai.response.id") == "chatcmpl-9HpbZPf84KZFiQG6fdY0KVtIwHyIa" @pytest.mark.vcr @@ -57,6 +58,7 @@ def test_chat_content_filtering(exporter, azure_openai_client): == "https://traceloop-stg.openai.azure.com//openai/" ) assert open_ai_span.attributes.get(SpanAttributes.LLM_IS_STREAMING) is False + assert open_ai_span.attributes.get("gen_ai.response.id") == "chatcmpl-9HpyGSWv1hoKdGaUaiFhfxzTEVlZo" content_filter_json = open_ai_span.attributes.get( f"{SpanAttributes.LLM_COMPLETIONS}.0.content_filter_results" @@ -153,6 +155,7 @@ def test_chat_streaming(exporter, azure_openai_client): prompt_filter_results[0]["content_filter_results"]["self_harm"]["filtered"] is False ) + assert open_ai_span.attributes.get("gen_ai.response.id") == "chatcmpl-9HpbaAXyt0cAnlWvI8kUAFpZt5jyQ" @pytest.mark.vcr @@ -192,3 +195,4 @@ async def test_chat_async_streaming(exporter, async_azure_openai_client): events = open_ai_span.events assert len(events) == chunk_count + assert open_ai_span.attributes.get("gen_ai.response.id") == "chatcmpl-9HpbbsSaH8U6amSDAwdA2WzMeDdLB" diff --git a/packages/opentelemetry-instrumentation-openai/tests/traces/test_chat.py b/packages/opentelemetry-instrumentation-openai/tests/traces/test_chat.py index 3d190fc75..177be9390 100644 --- a/packages/opentelemetry-instrumentation-openai/tests/traces/test_chat.py +++ b/packages/opentelemetry-instrumentation-openai/tests/traces/test_chat.py @@ -39,6 +39,7 @@ def test_chat(exporter, openai_client): == "fp_2b778c6b35" ) assert open_ai_span.attributes.get(SpanAttributes.LLM_IS_STREAMING) is False + assert open_ai_span.attributes.get("gen_ai.response.id") == "chatcmpl-908MD9ivBBLb6EaIjlqwFokntayQK" @pytest.mark.vcr @@ -93,6 +94,7 @@ def test_chat_tool_calls(exporter, openai_client): assert ( open_ai_span.attributes[f"{SpanAttributes.LLM_PROMPTS}.1.tool_call_id"] == "1" ) + assert open_ai_span.attributes.get("gen_ai.response.id") == "chatcmpl-9gKNZbUWSC4s2Uh2QfVV7PYiqWIuH" @pytest.mark.vcr @@ -147,6 +149,7 @@ def test_chat_pydantic_based_tool_calls(exporter, openai_client): assert ( open_ai_span.attributes[f"{SpanAttributes.LLM_PROMPTS}.1.tool_call_id"] == "1" ) + assert open_ai_span.attributes.get("gen_ai.response.id") == "chatcmpl-9lvGJKrBUPeJjHi3KKSEbGfcfomOP" @pytest.mark.vcr @@ -190,6 +193,7 @@ def test_chat_streaming(exporter, openai_client): total_tokens = open_ai_span.attributes.get(SpanAttributes.LLM_USAGE_TOTAL_TOKENS) assert completion_tokens and prompt_tokens and total_tokens assert completion_tokens + prompt_tokens == total_tokens + assert open_ai_span.attributes.get("gen_ai.response.id") == "chatcmpl-908MECg5dMyTTbJEltubwQXeeWlBA" @pytest.mark.vcr @@ -233,6 +237,7 @@ async def test_chat_async_streaming(exporter, async_openai_client): total_tokens = open_ai_span.attributes.get(SpanAttributes.LLM_USAGE_TOTAL_TOKENS) assert completion_tokens and prompt_tokens and total_tokens assert completion_tokens + prompt_tokens == total_tokens + assert open_ai_span.attributes.get("gen_ai.response.id") == "chatcmpl-9AGW3t9akkLW9f5f93B7mOhiqhNMC" @pytest.mark.vcr @@ -249,6 +254,8 @@ def test_with_asyncio_run(exporter, async_openai_client): assert [span.name for span in spans] == [ "openai.chat", ] + open_ai_span = spans[0] + assert open_ai_span.attributes.get("gen_ai.response.id") == "chatcmpl-ANnyEsyt6uxfIIA7lcPLH95lKcEeK" @pytest.mark.vcr @@ -266,7 +273,7 @@ def test_chat_context_propagation(exporter, vllm_openai_client): "openai.chat", ] open_ai_span = spans[0] - + assert open_ai_span.attributes.get("gen_ai.response.id") == "chat-43f4347c3299481e9704ab77439fbdb8" args, kwargs = send_spy.mock.call_args request = args[0] @@ -289,7 +296,7 @@ async def test_chat_async_context_propagation(exporter, async_vllm_openai_client "openai.chat", ] open_ai_span = spans[0] - + assert open_ai_span.attributes.get("gen_ai.response.id") == "chat-4db07f02ecae49cbafe1d359db1650df" args, kwargs = send_spy.mock.call_args request = args[0] diff --git a/packages/opentelemetry-instrumentation-openai/tests/traces/test_completions.py b/packages/opentelemetry-instrumentation-openai/tests/traces/test_completions.py index ace71d0c6..183b5cf42 100644 --- a/packages/opentelemetry-instrumentation-openai/tests/traces/test_completions.py +++ b/packages/opentelemetry-instrumentation-openai/tests/traces/test_completions.py @@ -29,6 +29,7 @@ def test_completion(exporter, openai_client): == "https://api.openai.com/v1/" ) assert open_ai_span.attributes.get(SpanAttributes.LLM_IS_STREAMING) is False + assert open_ai_span.attributes.get("gen_ai.response.id") == "cmpl-8wq42D1Socatcl1rCmgYZOFX7dFZw" @pytest.mark.vcr @@ -49,6 +50,7 @@ async def test_async_completion(exporter, async_openai_client): == "Tell me a joke about opentelemetry" ) assert open_ai_span.attributes.get(f"{SpanAttributes.LLM_COMPLETIONS}.0.content") + assert open_ai_span.attributes.get("gen_ai.response.id") == "cmpl-8wq43c8U5ZZCQBX5lrSpsANwcd3OF" @pytest.mark.vcr @@ -68,6 +70,7 @@ def test_completion_langchain_style(exporter, openai_client): == "Tell me a joke about opentelemetry" ) assert open_ai_span.attributes.get(f"{SpanAttributes.LLM_COMPLETIONS}.0.content") + assert open_ai_span.attributes.get("gen_ai.response.id") == "cmpl-8wq43QD6R2WqfxXLpYsRvSAIn9LB9" @pytest.mark.vcr @@ -115,6 +118,7 @@ def test_completion_streaming(exporter, openai_client): ) assert completion_tokens and prompt_tokens and total_tokens assert completion_tokens + prompt_tokens == total_tokens + assert open_ai_span.attributes.get("gen_ai.response.id") == "cmpl-8wq44ev1DvyhsBfm1hNwxfv6Dltco" finally: # unset env if original_value is None: @@ -149,6 +153,7 @@ async def test_async_completion_streaming(exporter, async_openai_client): open_ai_span.attributes.get(SpanAttributes.LLM_OPENAI_API_BASE) == "https://api.openai.com/v1/" ) + assert open_ai_span.attributes.get("gen_ai.response.id") == "cmpl-8wq44uFYuGm6kNe44ntRwluggKZFY" @pytest.mark.vcr @@ -172,6 +177,7 @@ def test_completion_context_propagation(exporter, vllm_openai_client): request = args[0] assert_request_contains_tracecontext(request, openai_span) + assert openai_span.attributes.get("gen_ai.response.id") == "cmpl-2996bf68f7f142fa817bdd32af678df9" @pytest.mark.vcr @@ -195,3 +201,4 @@ async def test_async_completion_context_propagation(exporter, async_vllm_openai_ request = args[0] assert_request_contains_tracecontext(request, openai_span) + assert openai_span.attributes.get("gen_ai.response.id") == "cmpl-4acc6171f6c34008af07ca8490da3b95" diff --git a/packages/opentelemetry-instrumentation-openai/tests/traces/test_functions.py b/packages/opentelemetry-instrumentation-openai/tests/traces/test_functions.py index 12f926ec9..c523374f9 100644 --- a/packages/opentelemetry-instrumentation-openai/tests/traces/test_functions.py +++ b/packages/opentelemetry-instrumentation-openai/tests/traces/test_functions.py @@ -75,6 +75,7 @@ def test_open_ai_function_calls(exporter, openai_client): open_ai_span.attributes[SpanAttributes.LLM_OPENAI_API_BASE] == "https://api.openai.com/v1/" ) + assert open_ai_span.attributes.get("gen_ai.response.id") == "chatcmpl-8wq4AUDD36geK9Za8cccowhObkV9H" @pytest.mark.vcr @@ -112,6 +113,7 @@ def test_open_ai_function_calls_tools(exporter, openai_client, openai_tools): open_ai_span.attributes[SpanAttributes.LLM_OPENAI_API_BASE] == "https://api.openai.com/v1/" ) + assert open_ai_span.attributes.get("gen_ai.response.id") == "chatcmpl-934OqhoorTmk1VnovIRXQCPk8PUTd" @pytest.mark.vcr @@ -158,6 +160,7 @@ async def test_open_ai_function_calls_tools_streaming( ) == '{"location":"San Francisco, CA"}' ) + assert open_ai_span.attributes.get("gen_ai.response.id") == "chatcmpl-9g4TmLd49mPoD6c0EnGlhNAp8b0on" @pytest.mark.vcr @@ -221,6 +224,7 @@ def test_open_ai_function_calls_tools_parallel(exporter, openai_client, openai_t ) == '{"location": "Boston"}' ) + assert open_ai_span.attributes.get("gen_ai.response.id") == "chatcmpl-9g4cZhrW9CsqihSvXslk0EUtjASsO" @pytest.mark.vcr @@ -288,3 +292,4 @@ async def test_open_ai_function_calls_tools_streaming_parallel( ) == '{"location": "Boston"}' ) + assert open_ai_span.attributes.get("gen_ai.response.id") == "chatcmpl-9g58noIjRkOeNNxfFsFfcNjhXlul7" diff --git a/packages/opentelemetry-instrumentation-openai/tests/traces/test_structured.py b/packages/opentelemetry-instrumentation-openai/tests/traces/test_structured.py index 4c8da1d6d..a6b9d2cc2 100644 --- a/packages/opentelemetry-instrumentation-openai/tests/traces/test_structured.py +++ b/packages/opentelemetry-instrumentation-openai/tests/traces/test_structured.py @@ -31,6 +31,7 @@ def test_parsed_completion(exporter, openai_client): open_ai_span.attributes[f"{SpanAttributes.LLM_PROMPTS}.0.content"] == "Tell me a joke about opentelemetry" ) + assert open_ai_span.attributes.get("gen_ai.response.id") == "chatcmpl-AGC1gNoe1Zyq9yZicdhLc85lmt2Ep" @pytest.mark.vcr @@ -52,6 +53,7 @@ def test_parsed_refused_completion(exporter, openai_client): open_ai_span.attributes[f"{SpanAttributes.LLM_COMPLETIONS}.0.refusal"] == "I'm very sorry, but I can't assist with that request." ) + assert open_ai_span.attributes.get("gen_ai.response.id") == "chatcmpl-AGky8KFDbg6f5fF4qLtsBredIjZZh" @pytest.mark.vcr @@ -78,6 +80,7 @@ async def test_async_parsed_completion(exporter, async_openai_client): open_ai_span.attributes[f"{SpanAttributes.LLM_PROMPTS}.0.content"] == "Tell me a joke about opentelemetry" ) + assert open_ai_span.attributes.get("gen_ai.response.id") == "chatcmpl-AGC1iysV7rZ0qZ510vbeKVTNxSOHB" @pytest.mark.vcr @@ -100,3 +103,4 @@ async def test_async_parsed_refused_completion(exporter, async_openai_client): open_ai_span.attributes[f"{SpanAttributes.LLM_COMPLETIONS}.0.refusal"] == "I'm very sorry, but I can't assist with that request." ) + assert open_ai_span.attributes.get("gen_ai.response.id") == "chatcmpl-AGkyFJGzZPUGAAEDJJuOS3idKvD3G" diff --git a/packages/opentelemetry-instrumentation-openai/tests/traces/test_vision.py b/packages/opentelemetry-instrumentation-openai/tests/traces/test_vision.py index e949a347b..ef6c3587d 100644 --- a/packages/opentelemetry-instrumentation-openai/tests/traces/test_vision.py +++ b/packages/opentelemetry-instrumentation-openai/tests/traces/test_vision.py @@ -48,6 +48,7 @@ def test_vision(exporter, openai_client): open_ai_span.attributes[SpanAttributes.LLM_OPENAI_API_BASE] == "https://api.openai.com/v1/" ) + assert open_ai_span.attributes.get("gen_ai.response.id") == "chatcmpl-8wq4EsSXTQC0JbGzob3SBHg6pS7Tt" @pytest.mark.vcr @@ -103,3 +104,4 @@ def test_vision_base64(exporter, openai_client): open_ai_span.attributes[SpanAttributes.LLM_OPENAI_API_BASE] == "https://api.openai.com/v1/" ) + assert open_ai_span.attributes.get("gen_ai.response.id") == "chatcmpl-AC7YAG2uy8c4VfbqJp4QkdHc5PDZ4"