diff --git a/eng/packages/General.props b/eng/packages/General.props index 78a6353abc3..6872259407d 100644 --- a/eng/packages/General.props +++ b/eng/packages/General.props @@ -11,7 +11,7 @@ - + diff --git a/eng/packages/TestOnly.props b/eng/packages/TestOnly.props index 8c4c858de83..bd7fe7d01d3 100644 --- a/eng/packages/TestOnly.props +++ b/eng/packages/TestOnly.props @@ -2,7 +2,7 @@ - + diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIAssistantClient.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIAssistantClient.cs index b0f36e43313..b820fde3134 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIAssistantClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIAssistantClient.cs @@ -266,7 +266,8 @@ strictObj is bool strictValue ? { List messageContents = []; - if (chatMessage.Role == ChatRole.System) + if (chatMessage.Role == ChatRole.System || + chatMessage.Role == OpenAIModelMappers.ChatRoleDeveloper) { instructions ??= new(); foreach (var textContent in chatMessage.Contents.OfType()) diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIChatClient.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIChatClient.cs index 6e3aa019c77..acd73142dcf 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIChatClient.cs @@ -110,7 +110,7 @@ public async Task CompleteAsync( // Make the call to OpenAI. var response = await _chatClient.CompleteChatAsync(openAIChatMessages, openAIOptions, cancellationToken).ConfigureAwait(false); - return OpenAIModelMappers.FromOpenAIChatCompletion(response.Value, options); + return OpenAIModelMappers.FromOpenAIChatCompletion(response.Value, options, openAIOptions); } /// diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIModelMapper.ChatCompletion.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIModelMapper.ChatCompletion.cs index 2612980b34f..e7eba21e4a8 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIModelMapper.ChatCompletion.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIModelMapper.ChatCompletion.cs @@ -13,11 +13,14 @@ using Microsoft.Shared.Diagnostics; using OpenAI.Chat; +#pragma warning disable CA1308 // Normalize strings to uppercase +#pragma warning disable CA1859 // Use concrete types when possible for improved performance #pragma warning disable SA1204 // Static elements should appear before instance elements #pragma warning disable S103 // Lines should not be too long -#pragma warning disable CA1859 // Use concrete types when possible for improved performance #pragma warning disable S1067 // Expressions should not be too complex +#pragma warning disable S2178 // Short-circuit logic should be used in boolean contexts #pragma warning disable S3440 // Variables should not be checked against the values they're about to be assigned +#pragma warning disable EA0011 // Consider removing unnecessary conditional access operator (?) namespace Microsoft.Extensions.AI; @@ -70,7 +73,7 @@ public static OpenAI.Chat.ChatCompletion ToOpenAIChatCompletion(ChatCompletion c usage: chatTokenUsage); } - public static ChatCompletion FromOpenAIChatCompletion(OpenAI.Chat.ChatCompletion openAICompletion, ChatOptions? options) + public static ChatCompletion FromOpenAIChatCompletion(OpenAI.Chat.ChatCompletion openAICompletion, ChatOptions? options, ChatCompletionOptions chatCompletionOptions) { _ = Throw.IfNull(openAICompletion); @@ -90,6 +93,37 @@ public static ChatCompletion FromOpenAIChatCompletion(OpenAI.Chat.ChatCompletion } } + // Output audio is handled separately from message content parts. + if (openAICompletion.OutputAudio is ChatOutputAudio audio) + { + string mimeType = chatCompletionOptions?.AudioOptions?.OutputAudioFormat.ToString()?.ToLowerInvariant() switch + { + "opus" => "audio/opus", + "aac" => "audio/aac", + "flac" => "audio/flac", + "wav" => "audio/wav", + "pcm" => "audio/pcm", + "mp3" or _ => "audio/mpeg", + }; + + var dc = new DataContent(audio.AudioBytes.ToMemory(), mimeType) + { + AdditionalProperties = new() { [nameof(audio.ExpiresAt)] = audio.ExpiresAt }, + }; + + if (audio.Id is string id) + { + dc.AdditionalProperties[nameof(audio.Id)] = id; + } + + if (audio.Transcript is string transcript) + { + dc.AdditionalProperties[nameof(audio.Transcript)] = transcript; + } + + returnMessage.Contents.Add(dc); + } + // Also manufacture function calling content items from any tool calls in the response. if (options?.Tools is { Count: > 0 }) { @@ -108,11 +142,11 @@ public static ChatCompletion FromOpenAIChatCompletion(OpenAI.Chat.ChatCompletion // Wrap the content in a ChatCompletion to return. var completion = new ChatCompletion([returnMessage]) { - RawRepresentation = openAICompletion, CompletionId = openAICompletion.Id, CreatedAt = openAICompletion.CreatedAt, - ModelId = openAICompletion.Model, FinishReason = FromOpenAIFinishReason(openAICompletion.FinishReason), + ModelId = openAICompletion.Model, + RawRepresentation = openAICompletion, }; if (openAICompletion.Usage is ChatTokenUsage tokenUsage) @@ -265,6 +299,16 @@ public static ChatCompletionOptions ToOpenAIOptions(ChatOptions? options) if (options.AdditionalProperties is { Count: > 0 } additionalProperties) { + if (additionalProperties.TryGetValue(nameof(result.AllowParallelToolCalls), out bool allowParallelToolCalls)) + { + result.AllowParallelToolCalls = allowParallelToolCalls; + } + + if (additionalProperties.TryGetValue(nameof(result.AudioOptions), out ChatAudioOptions? audioOptions)) + { + result.AudioOptions = audioOptions; + } + if (additionalProperties.TryGetValue(nameof(result.EndUserId), out string? endUserId)) { result.EndUserId = endUserId; @@ -283,28 +327,38 @@ public static ChatCompletionOptions ToOpenAIOptions(ChatOptions? options) } } - if (additionalProperties.TryGetValue(nameof(result.AllowParallelToolCalls), out bool allowParallelToolCalls)) + if (additionalProperties.TryGetValue(nameof(result.Metadata), out IDictionary? metadata)) { - result.AllowParallelToolCalls = allowParallelToolCalls; + foreach (KeyValuePair kvp in metadata) + { + result.Metadata[kvp.Key] = kvp.Value; + } } - if (additionalProperties.TryGetValue(nameof(result.TopLogProbabilityCount), out int topLogProbabilityCountInt)) + if (additionalProperties.TryGetValue(nameof(result.OutputPrediction), out ChatOutputPrediction? outputPrediction)) { - result.TopLogProbabilityCount = topLogProbabilityCountInt; + result.OutputPrediction = outputPrediction; } - if (additionalProperties.TryGetValue(nameof(result.Metadata), out IDictionary? metadata)) + if (additionalProperties.TryGetValue(nameof(result.ReasoningEffortLevel), out ChatReasoningEffortLevel reasoningEffortLevel)) { - foreach (KeyValuePair kvp in metadata) - { - result.Metadata[kvp.Key] = kvp.Value; - } + result.ReasoningEffortLevel = reasoningEffortLevel; + } + + if (additionalProperties.TryGetValue(nameof(result.ResponseModalities), out ChatResponseModalities responseModalities)) + { + result.ResponseModalities = responseModalities; } if (additionalProperties.TryGetValue(nameof(result.StoredOutputEnabled), out bool storeOutputEnabled)) { result.StoredOutputEnabled = storeOutputEnabled; } + + if (additionalProperties.TryGetValue(nameof(result.TopLogProbabilityCount), out int topLogProbabilityCountInt)) + { + result.TopLogProbabilityCount = topLogProbabilityCountInt; + } } if (options.Tools is { Count: > 0 } tools) @@ -420,26 +474,22 @@ private static UsageDetails FromOpenAIUsage(ChatTokenUsage tokenUsage) AdditionalCounts = [], }; + var counts = destination.AdditionalCounts; + if (tokenUsage.InputTokenDetails is ChatInputTokenUsageDetails inputDetails) { - destination.AdditionalCounts.Add( - $"{nameof(ChatTokenUsage.InputTokenDetails)}.{nameof(ChatInputTokenUsageDetails.AudioTokenCount)}", - inputDetails.AudioTokenCount); - - destination.AdditionalCounts.Add( - $"{nameof(ChatTokenUsage.InputTokenDetails)}.{nameof(ChatInputTokenUsageDetails.CachedTokenCount)}", - inputDetails.CachedTokenCount); + const string InputDetails = nameof(ChatTokenUsage.InputTokenDetails); + counts.Add($"{InputDetails}.{nameof(ChatInputTokenUsageDetails.AudioTokenCount)}", inputDetails.AudioTokenCount); + counts.Add($"{InputDetails}.{nameof(ChatInputTokenUsageDetails.CachedTokenCount)}", inputDetails.CachedTokenCount); } if (tokenUsage.OutputTokenDetails is ChatOutputTokenUsageDetails outputDetails) { - destination.AdditionalCounts.Add( - $"{nameof(ChatTokenUsage.OutputTokenDetails)}.{nameof(ChatOutputTokenUsageDetails.AudioTokenCount)}", - outputDetails.AudioTokenCount); - - destination.AdditionalCounts.Add( - $"{nameof(ChatTokenUsage.OutputTokenDetails)}.{nameof(ChatOutputTokenUsageDetails.ReasoningTokenCount)}", - outputDetails.ReasoningTokenCount); + const string OutputDetails = nameof(ChatTokenUsage.OutputTokenDetails); + counts.Add($"{OutputDetails}.{nameof(ChatOutputTokenUsageDetails.ReasoningTokenCount)}", outputDetails.ReasoningTokenCount); + counts.Add($"{OutputDetails}.{nameof(ChatOutputTokenUsageDetails.AudioTokenCount)}", outputDetails.AudioTokenCount); + counts.Add($"{OutputDetails}.{nameof(ChatOutputTokenUsageDetails.AcceptedPredictionTokenCount)}", outputDetails.AcceptedPredictionTokenCount); + counts.Add($"{OutputDetails}.{nameof(ChatOutputTokenUsageDetails.RejectedPredictionTokenCount)}", outputDetails.RejectedPredictionTokenCount); } return destination; @@ -452,34 +502,26 @@ private static ChatTokenUsage ToOpenAIUsage(UsageDetails usageDetails) if (usageDetails.AdditionalCounts is { Count: > 0 } additionalCounts) { - int? inputAudioTokenCount = additionalCounts.TryGetValue( - $"{nameof(ChatTokenUsage.InputTokenDetails)}.{nameof(ChatInputTokenUsageDetails.AudioTokenCount)}", - out int value) ? value : null; - - int? inputCachedTokenCount = additionalCounts.TryGetValue( - $"{nameof(ChatTokenUsage.InputTokenDetails)}.{nameof(ChatInputTokenUsageDetails.CachedTokenCount)}", - out value) ? value : null; - - int? outputAudioTokenCount = additionalCounts.TryGetValue( - $"{nameof(ChatTokenUsage.OutputTokenDetails)}.{nameof(ChatOutputTokenUsageDetails.AudioTokenCount)}", - out value) ? value : null; - - int? outputReasoningTokenCount = additionalCounts.TryGetValue( - $"{nameof(ChatTokenUsage.OutputTokenDetails)}.{nameof(ChatOutputTokenUsageDetails.ReasoningTokenCount)}", - out value) ? value : null; - - if (inputAudioTokenCount is not null || inputCachedTokenCount is not null) + const string InputDetails = nameof(ChatTokenUsage.InputTokenDetails); + if (additionalCounts.TryGetValue($"{InputDetails}.{nameof(ChatInputTokenUsageDetails.AudioTokenCount)}", out int inputAudioTokenCount) | + additionalCounts.TryGetValue($"{InputDetails}.{nameof(ChatInputTokenUsageDetails.CachedTokenCount)}", out int inputCachedTokenCount)) { inputTokenUsageDetails = OpenAIChatModelFactory.ChatInputTokenUsageDetails( - audioTokenCount: inputAudioTokenCount ?? 0, - cachedTokenCount: inputCachedTokenCount ?? 0); + audioTokenCount: inputAudioTokenCount, + cachedTokenCount: inputCachedTokenCount); } - if (outputAudioTokenCount is not null || outputReasoningTokenCount is not null) + const string OutputDetails = nameof(ChatTokenUsage.OutputTokenDetails); + if (additionalCounts.TryGetValue($"{OutputDetails}.{nameof(ChatOutputTokenUsageDetails.ReasoningTokenCount)}", out int outputReasoningTokenCount) | + additionalCounts.TryGetValue($"{OutputDetails}.{nameof(ChatOutputTokenUsageDetails.AudioTokenCount)}", out int outputAudioTokenCount) | + additionalCounts.TryGetValue($"{OutputDetails}.{nameof(ChatOutputTokenUsageDetails.AcceptedPredictionTokenCount)}", out int outputAcceptedPredictionCount) | + additionalCounts.TryGetValue($"{OutputDetails}.{nameof(ChatOutputTokenUsageDetails.RejectedPredictionTokenCount)}", out int outputRejectedPredictionCount)) { outputTokenUsageDetails = OpenAIChatModelFactory.ChatOutputTokenUsageDetails( - audioTokenCount: outputAudioTokenCount ?? 0, - reasoningTokenCount: outputReasoningTokenCount ?? 0); + reasoningTokenCount: outputReasoningTokenCount, + audioTokenCount: outputAudioTokenCount, + acceptedPredictionTokenCount: outputAcceptedPredictionCount, + rejectedPredictionTokenCount: outputRejectedPredictionCount); } } @@ -505,6 +547,7 @@ private static ChatRole FromOpenAIChatRole(ChatMessageRole role) => ChatMessageRole.User => ChatRole.User, ChatMessageRole.Assistant => ChatRole.Assistant, ChatMessageRole.Tool => ChatRole.Tool, + ChatMessageRole.Developer => ChatRoleDeveloper, _ => new ChatRole(role.ToString()), }; @@ -515,7 +558,9 @@ private static ChatRole FromOpenAIChatRole(ChatMessageRole role) => role == ChatRole.System ? ChatMessageRole.System : role == ChatRole.User ? ChatMessageRole.User : role == ChatRole.Assistant ? ChatMessageRole.Assistant : - role == ChatRole.Tool ? ChatMessageRole.Tool : ChatMessageRole.User; + role == ChatRole.Tool ? ChatMessageRole.Tool : + role == OpenAIModelMappers.ChatRoleDeveloper ? ChatMessageRole.Developer : + ChatMessageRole.User; /// Creates an from a . /// The content part to convert into a content. diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIModelMapper.ChatMessage.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIModelMapper.ChatMessage.cs index 9294e2137e7..399ca5484f0 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIModelMapper.ChatMessage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIModelMapper.ChatMessage.cs @@ -14,6 +14,8 @@ namespace Microsoft.Extensions.AI; internal static partial class OpenAIModelMappers { + public static ChatRole ChatRoleDeveloper { get; } = new ChatRole("developer"); + public static OpenAIChatCompletionRequest FromOpenAIChatCompletionRequest(OpenAI.Chat.ChatCompletionOptions chatCompletionOptions) { ChatOptions chatOptions = FromOpenAIOptions(chatCompletionOptions); @@ -45,6 +47,15 @@ public static IEnumerable FromOpenAIChatMessages(IEnumerable FromOpenAIChatMessages(IEnumerable ToOpenAIChatContent(IList parts.Add(ChatMessageContentPart.CreateImagePart(new Uri(uri))); } + break; + + case DataContent dataContent when dataContent.MediaTypeStartsWith("audio/") && dataContent.Data.HasValue: + var audioData = BinaryData.FromBytes(dataContent.Data.Value); + if (dataContent.MediaTypeStartsWith("audio/mpeg")) + { + parts.Add(ChatMessageContentPart.CreateInputAudioPart(audioData, ChatInputAudioFormat.Mp3)); + } + else if (dataContent.MediaTypeStartsWith("audio/wav")) + { + parts.Add(ChatMessageContentPart.CreateInputAudioPart(audioData, ChatInputAudioFormat.Wav)); + } + break; } } diff --git a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIChatClientTests.cs b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIChatClientTests.cs index 9238b4fbba0..2aaaeec9e0d 100644 --- a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIChatClientTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIChatClientTests.cs @@ -142,7 +142,12 @@ public void GetService_ChatClient_SuccessfullyReturnsUnderlyingClient() public async Task BasicRequestResponse_NonStreaming() { const string Input = """ - {"messages":[{"role":"user","content":"hello"}],"model":"gpt-4o-mini","max_completion_tokens":10,"temperature":0.5} + { + "temperature":0.5, + "messages":[{"role":"user","content":"hello"}], + "model":"gpt-4o-mini", + "max_completion_tokens":10 + } """; const string Output = """ @@ -205,8 +210,10 @@ public async Task BasicRequestResponse_NonStreaming() { { "InputTokenDetails.AudioTokenCount", 0 }, { "InputTokenDetails.CachedTokenCount", 13 }, + { "OutputTokenDetails.ReasoningTokenCount", 90 }, { "OutputTokenDetails.AudioTokenCount", 0 }, - { "OutputTokenDetails.ReasoningTokenCount", 90 } + { "OutputTokenDetails.AcceptedPredictionTokenCount", 0 }, + { "OutputTokenDetails.RejectedPredictionTokenCount", 0 }, }, response.Usage.AdditionalCounts); Assert.NotNull(response.AdditionalProperties); @@ -217,7 +224,14 @@ public async Task BasicRequestResponse_NonStreaming() public async Task BasicRequestResponse_Streaming() { const string Input = """ - {"messages":[{"role":"user","content":"hello"}],"model":"gpt-4o-mini","max_completion_tokens":20,"stream":true,"stream_options":{"include_usage":true},"temperature":0.5} + { + "temperature":0.5, + "messages":[{"role":"user","content":"hello"}], + "model":"gpt-4o-mini", + "stream":true, + "stream_options":{"include_usage":true}, + "max_completion_tokens":20 + } """; const string Output = """ @@ -288,8 +302,10 @@ public async Task BasicRequestResponse_Streaming() { { "InputTokenDetails.AudioTokenCount", 123 }, { "InputTokenDetails.CachedTokenCount", 5 }, - { "OutputTokenDetails.AudioTokenCount", 456 }, { "OutputTokenDetails.ReasoningTokenCount", 90 }, + { "OutputTokenDetails.AudioTokenCount", 456 }, + { "OutputTokenDetails.AcceptedPredictionTokenCount", 0 }, + { "OutputTokenDetails.RejectedPredictionTokenCount", 0 }, }, usage.Details.AdditionalCounts); } @@ -297,15 +313,17 @@ public async Task BasicRequestResponse_Streaming() public async Task NonStronglyTypedOptions_AllSent() { const string Input = """ - {"messages":[{"role":"user","content":"hello"}], - "model":"gpt-4o-mini", - "store":true, - "metadata":{"something":"else"}, - "logit_bias":{"12":34}, - "logprobs":true, - "top_logprobs":42, - "parallel_tool_calls":false, - "user":"12345"} + { + "messages":[{"role":"user","content":"hello"}], + "model":"gpt-4o-mini", + "logprobs":true, + "top_logprobs":42, + "logit_bias":{"12":34}, + "parallel_tool_calls":false, + "user":"12345", + "metadata":{"something":"else"}, + "store":true + } """; const string Output = """ @@ -352,6 +370,9 @@ public async Task MultipleMessages_NonStreaming() { const string Input = """ { + "frequency_penalty": 0.75, + "presence_penalty": 0.5, + "temperature": 0.25, "messages": [ { "role": "system", @@ -371,13 +392,8 @@ public async Task MultipleMessages_NonStreaming() } ], "model": "gpt-4o-mini", - "frequency_penalty": 0.75, - "presence_penalty": 0.5, - "seed":42, - "stop": [ - "great" - ], - "temperature": 0.25 + "stop": ["great"], + "seed": 42 } """; @@ -454,8 +470,10 @@ public async Task MultipleMessages_NonStreaming() { { "InputTokenDetails.AudioTokenCount", 123 }, { "InputTokenDetails.CachedTokenCount", 13 }, - { "OutputTokenDetails.AudioTokenCount", 456 }, { "OutputTokenDetails.ReasoningTokenCount", 90 }, + { "OutputTokenDetails.AudioTokenCount", 456 }, + { "OutputTokenDetails.AcceptedPredictionTokenCount", 0 }, + { "OutputTokenDetails.RejectedPredictionTokenCount", 0 }, }, response.Usage.AdditionalCounts); Assert.NotNull(response.AdditionalProperties); @@ -552,8 +570,10 @@ public async Task MultiPartSystemMessage_NonStreaming() { { "InputTokenDetails.AudioTokenCount", 0 }, { "InputTokenDetails.CachedTokenCount", 13 }, + { "OutputTokenDetails.ReasoningTokenCount", 90 }, { "OutputTokenDetails.AudioTokenCount", 0 }, - { "OutputTokenDetails.ReasoningTokenCount", 90 } + { "OutputTokenDetails.AcceptedPredictionTokenCount", 0 }, + { "OutputTokenDetails.RejectedPredictionTokenCount", 0 }, }, response.Usage.AdditionalCounts); Assert.NotNull(response.AdditionalProperties); @@ -651,8 +671,10 @@ public async Task EmptyAssistantMessage_NonStreaming() { { "InputTokenDetails.AudioTokenCount", 0 }, { "InputTokenDetails.CachedTokenCount", 13 }, + { "OutputTokenDetails.ReasoningTokenCount", 90 }, { "OutputTokenDetails.AudioTokenCount", 0 }, - { "OutputTokenDetails.ReasoningTokenCount", 90 } + { "OutputTokenDetails.AcceptedPredictionTokenCount", 0 }, + { "OutputTokenDetails.RejectedPredictionTokenCount", 0 }, }, response.Usage.AdditionalCounts); Assert.NotNull(response.AdditionalProperties); @@ -664,16 +686,8 @@ public async Task FunctionCallContent_NonStreaming() { const string Input = """ { - "messages": [ - { - "role": "user", - "content": "How old is Alice?" - } - ], - "model": "gpt-4o-mini", "tools": [ { - "type": "function", "function": { "description": "Gets the age of the specified person.", "name": "GetPersonAge", @@ -689,9 +703,17 @@ public async Task FunctionCallContent_NonStreaming() } } } - } + }, + "type": "function" } ], + "messages": [ + { + "role": "user", + "content": "How old is Alice?" + } + ], + "model": "gpt-4o-mini", "tool_choice": "auto" } """; @@ -763,8 +785,10 @@ public async Task FunctionCallContent_NonStreaming() { { "InputTokenDetails.AudioTokenCount", 0 }, { "InputTokenDetails.CachedTokenCount", 13 }, + { "OutputTokenDetails.ReasoningTokenCount", 90 }, { "OutputTokenDetails.AudioTokenCount", 0 }, - { "OutputTokenDetails.ReasoningTokenCount", 90 } + { "OutputTokenDetails.AcceptedPredictionTokenCount", 0 }, + { "OutputTokenDetails.RejectedPredictionTokenCount", 0 }, }, response.Usage.AdditionalCounts); Assert.Single(response.Choices); @@ -782,20 +806,8 @@ public async Task FunctionCallContent_Streaming() { const string Input = """ { - "messages": [ - { - "role": "user", - "content": "How old is Alice?" - } - ], - "model": "gpt-4o-mini", - "stream": true, - "stream_options": { - "include_usage": true - }, "tools": [ { - "type": "function", "function": { "description": "Gets the age of the specified person.", "name": "GetPersonAge", @@ -811,9 +823,21 @@ public async Task FunctionCallContent_Streaming() } } } - } + }, + "type": "function" } ], + "messages": [ + { + "role": "user", + "content": "How old is Alice?" + } + ], + "model": "gpt-4o-mini", + "stream": true, + "stream_options": { + "include_usage": true + }, "tool_choice": "auto" } """; @@ -883,8 +907,10 @@ public async Task FunctionCallContent_Streaming() { { "InputTokenDetails.AudioTokenCount", 0 }, { "InputTokenDetails.CachedTokenCount", 0 }, + { "OutputTokenDetails.ReasoningTokenCount", 90 }, { "OutputTokenDetails.AudioTokenCount", 0 }, - { "OutputTokenDetails.ReasoningTokenCount", 90 } + { "OutputTokenDetails.AcceptedPredictionTokenCount", 0 }, + { "OutputTokenDetails.RejectedPredictionTokenCount", 0 }, }, usage.Details.AdditionalCounts); } @@ -908,19 +934,19 @@ public async Task AssistantMessageWithBothToolsAndContent_NonStreaming() "tool_calls": [ { "id": "12345", - "type": "function", "function": { "name": "SayHello", "arguments": "null" - } + }, + "type": "function" }, { "id": "12346", - "type": "function", "function": { "name": "SayHi", "arguments": "null" - } + }, + "type": "function" } ] }, @@ -1022,8 +1048,10 @@ public async Task AssistantMessageWithBothToolsAndContent_NonStreaming() { { "InputTokenDetails.AudioTokenCount", 0 }, { "InputTokenDetails.CachedTokenCount", 20 }, + { "OutputTokenDetails.ReasoningTokenCount", 90 }, { "OutputTokenDetails.AudioTokenCount", 0 }, - { "OutputTokenDetails.ReasoningTokenCount", 90 } + { "OutputTokenDetails.AcceptedPredictionTokenCount", 0 }, + { "OutputTokenDetails.RejectedPredictionTokenCount", 0 }, }, response.Usage.AdditionalCounts); Assert.NotNull(response.AdditionalProperties); diff --git a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeTests.cs b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeTests.cs index 32e4d059a51..2ce567482bb 100644 --- a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeTests.cs @@ -72,18 +72,13 @@ public async Task HandleToolCallsAsync_RejectsNulls() { var conversationSession = (RealtimeConversationSession)default!; - // Null RealtimeConversationSession - await Assert.ThrowsAsync(() => conversationSession.HandleToolCallsAsync( - new TestConversationUpdate(), [])); + // There's currently no way to create a ConversationUpdate instance from outside of the OpenAI + // library, so we can't validate behavior when a valid ConversationUpdate instance is passed in. // Null ConversationUpdate using var session = TestRealtimeConversationSession.CreateTestInstance(); await Assert.ThrowsAsync(() => conversationSession.HandleToolCallsAsync( null!, [])); - - // Null tools - await Assert.ThrowsAsync(() => conversationSession.HandleToolCallsAsync( - new TestConversationUpdate(), null!)); } [Description("This is a description")] @@ -110,12 +105,4 @@ public static TestRealtimeConversationSession CreateTestInstance() new Uri("http://endpoint"), credential); } } - - private class TestConversationUpdate : ConversationUpdate - { - public TestConversationUpdate() - : base("eventId") - { - } - } } diff --git a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAISerializationTests.cs b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAISerializationTests.cs index 71ec7bd7f5f..22627731b16 100644 --- a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAISerializationTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAISerializationTests.cs @@ -514,24 +514,42 @@ public static async Task SerializeCompletion_SingleChoice() AssertJsonEqual(""" { "id": "chatcmpl-ADx3PvAnCwJg0woha4pYsBTi3ZpOI", + "model": "gpt-4o-mini-2024-07-18", + "system_fingerprint": "fp_f85bea6784", + "usage": { + "completion_tokens": 9, + "prompt_tokens": 8, + "total_tokens": 17, + "completion_tokens_details": { + "reasoning_tokens": 90, + "audio_tokens": 2, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + }, + "prompt_tokens_details": { + "audio_tokens": 1, + "cached_tokens": 13 + } + }, + "object": "chat.completion", "choices": [ { "finish_reason": "stop", "index": 0, "message": { - "content": "Hello! How can I assist you today?", "refusal": null, "tool_calls": [ { "id": "callId", - "type": "function", "function": { "name": "MyCoolFunc", "arguments": "{\r\n \u0022arg1\u0022: 42,\r\n \u0022arg2\u0022: \u0022str\u0022\r\n}" - } + }, + "type": "function" } ], - "role": "assistant" + "role": "assistant", + "content": "Hello! How can I assist you today?" }, "logprobs": { "content": [], @@ -539,23 +557,7 @@ public static async Task SerializeCompletion_SingleChoice() } } ], - "created": 1727888631, - "model": "gpt-4o-mini-2024-07-18", - "system_fingerprint": "fp_f85bea6784", - "object": "chat.completion", - "usage": { - "completion_tokens": 9, - "prompt_tokens": 8, - "total_tokens": 17, - "completion_tokens_details": { - "audio_tokens": 2, - "reasoning_tokens": 90 - }, - "prompt_tokens_details": { - "audio_tokens": 1, - "cached_tokens": 13 - } - } + "created": 1727888631 } """, result); } @@ -647,15 +649,15 @@ static async IAsyncEnumerable CreateStreamingComp string result = Encoding.UTF8.GetString(stream.ToArray()); AssertSseEqual(""" - data: {"id":"chatcmpl-ADymNiWWeqCJqHNFXiI1QtRcLuXcl","choices":[{"delta":{"content":"Streaming update 0","tool_calls":[],"role":"assistant"},"logprobs":{"content":[],"refusal":[]},"index":0}],"created":1727888631,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","object":"chat.completion.chunk"} + data: {"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","object":"chat.completion.chunk","id":"chatcmpl-ADymNiWWeqCJqHNFXiI1QtRcLuXcl","choices":[{"delta":{"tool_calls":[],"role":"assistant","content":"Streaming update 0"},"logprobs":{"content":[],"refusal":[]},"index":0}],"created":1727888631} - data: {"id":"chatcmpl-ADymNiWWeqCJqHNFXiI1QtRcLuXcl","choices":[{"delta":{"content":"Streaming update 1","tool_calls":[],"role":"assistant"},"logprobs":{"content":[],"refusal":[]},"index":0}],"created":1727888631,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","object":"chat.completion.chunk"} + data: {"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","object":"chat.completion.chunk","id":"chatcmpl-ADymNiWWeqCJqHNFXiI1QtRcLuXcl","choices":[{"delta":{"tool_calls":[],"role":"assistant","content":"Streaming update 1"},"logprobs":{"content":[],"refusal":[]},"index":0}],"created":1727888631} - data: {"id":"chatcmpl-ADymNiWWeqCJqHNFXiI1QtRcLuXcl","choices":[{"delta":{"content":"Streaming update 2","tool_calls":[{"index":0,"id":"callId","type":"function","function":{"name":"MyCoolFunc","arguments":"{\r\n \u0022arg1\u0022: 42,\r\n \u0022arg2\u0022: \u0022str\u0022\r\n}"}}],"role":"assistant"},"logprobs":{"content":[],"refusal":[]},"index":0}],"created":1727888631,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","object":"chat.completion.chunk"} + data: {"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","object":"chat.completion.chunk","id":"chatcmpl-ADymNiWWeqCJqHNFXiI1QtRcLuXcl","choices":[{"delta":{"tool_calls":[{"index":0,"function":{"name":"MyCoolFunc","arguments":"{\r\n \u0022arg1\u0022: 42,\r\n \u0022arg2\u0022: \u0022str\u0022\r\n}"},"type":"function","id":"callId"}],"role":"assistant","content":"Streaming update 2"},"logprobs":{"content":[],"refusal":[]},"index":0}],"created":1727888631} - data: {"id":"chatcmpl-ADymNiWWeqCJqHNFXiI1QtRcLuXcl","choices":[{"delta":{"content":"Streaming update 3","tool_calls":[],"role":"assistant"},"logprobs":{"content":[],"refusal":[]},"index":0}],"created":1727888631,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","object":"chat.completion.chunk"} + data: {"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","object":"chat.completion.chunk","id":"chatcmpl-ADymNiWWeqCJqHNFXiI1QtRcLuXcl","choices":[{"delta":{"tool_calls":[],"role":"assistant","content":"Streaming update 3"},"logprobs":{"content":[],"refusal":[]},"index":0}],"created":1727888631} - data: {"id":"chatcmpl-ADymNiWWeqCJqHNFXiI1QtRcLuXcl","choices":[{"delta":{"content":"Streaming update 4","tool_calls":[],"role":"assistant"},"logprobs":{"content":[],"refusal":[]},"finish_reason":"stop","index":0}],"created":1727888631,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","object":"chat.completion.chunk","usage":{"completion_tokens":9,"prompt_tokens":8,"total_tokens":17,"completion_tokens_details":{"audio_tokens":2,"reasoning_tokens":90},"prompt_tokens_details":{"audio_tokens":1,"cached_tokens":13}}} + data: {"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","object":"chat.completion.chunk","id":"chatcmpl-ADymNiWWeqCJqHNFXiI1QtRcLuXcl","choices":[{"delta":{"tool_calls":[],"role":"assistant","content":"Streaming update 4"},"logprobs":{"content":[],"refusal":[]},"finish_reason":"stop","index":0}],"created":1727888631,"usage":{"completion_tokens":9,"prompt_tokens":8,"total_tokens":17,"completion_tokens_details":{"reasoning_tokens":90,"audio_tokens":2,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens_details":{"audio_tokens":1,"cached_tokens":13}}} data: [DONE]