From 98dc467f6ace1f2d891a22d52f17b76a9f6274ad Mon Sep 17 00:00:00 2001 From: Steve Sanderson Date: Mon, 18 Nov 2024 11:17:05 -0500 Subject: [PATCH 1/8] Add OpenAIRealtimeExtensions with ToConversationFunctionTool --- eng/packages/General.props | 2 +- .../JsonContext.cs | 16 +++++ .../Microsoft.Extensions.AI.OpenAI.csproj | 2 +- .../OpenAIChatClient.cs | 14 +--- .../OpenAIRealtimeExtensions.cs | 53 ++++++++++++++ .../OpenAIRealtimeTests.cs | 71 +++++++++++++++++++ 6 files changed, 145 insertions(+), 13 deletions(-) create mode 100644 src/Libraries/Microsoft.Extensions.AI.OpenAI/JsonContext.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeExtensions.cs create mode 100644 test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeTests.cs diff --git a/eng/packages/General.props b/eng/packages/General.props index 9c54a2351ab..ff2c3010128 100644 --- a/eng/packages/General.props +++ b/eng/packages/General.props @@ -11,7 +11,7 @@ - + diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/JsonContext.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/JsonContext.cs new file mode 100644 index 00000000000..9cd075e1d04 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/JsonContext.cs @@ -0,0 +1,16 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Microsoft.Extensions.AI; + +/// Source-generated JSON type information. +[JsonSourceGenerationOptions(JsonSerializerDefaults.Web, + UseStringEnumConverter = true, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + WriteIndented = true)] +[JsonSerializable(typeof(OpenAIChatClient.OpenAIChatToolJson))] +[JsonSerializable(typeof(OpenAIRealtimeExtensions.ConversationFunctionToolParametersSchema))] +internal sealed partial class OpenAIJsonContext : JsonSerializerContext; diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/Microsoft.Extensions.AI.OpenAI.csproj b/src/Libraries/Microsoft.Extensions.AI.OpenAI/Microsoft.Extensions.AI.OpenAI.csproj index f2e2e9c0f52..1d400389af0 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/Microsoft.Extensions.AI.OpenAI.csproj +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/Microsoft.Extensions.AI.OpenAI.csproj @@ -15,7 +15,7 @@ $(TargetFrameworks);netstandard2.0 - $(NoWarn);CA1063;CA1508;CA2227;SA1316;S1121;S3358;EA0002 + $(NoWarn);CA1063;CA1508;CA2227;SA1316;S1121;S3358;EA0002;OPENAI002 true true diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIChatClient.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIChatClient.cs index 90329a9b593..05bd801ac09 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIChatClient.cs @@ -24,7 +24,7 @@ namespace Microsoft.Extensions.AI; /// Represents an for an OpenAI or . -public sealed partial class OpenAIChatClient : IChatClient +public sealed class OpenAIChatClient : IChatClient { private static readonly JsonElement _defaultParameterSchema = JsonDocument.Parse("{}").RootElement; @@ -513,14 +513,14 @@ strictObj is bool strictValue ? } resultParameters = BinaryData.FromBytes( - JsonSerializer.SerializeToUtf8Bytes(tool, JsonContext.Default.OpenAIChatToolJson)); + JsonSerializer.SerializeToUtf8Bytes(tool, OpenAIJsonContext.Default.OpenAIChatToolJson)); } return ChatTool.CreateFunctionTool(aiFunction.Metadata.Name, aiFunction.Metadata.Description, resultParameters, strict); } /// Used to create the JSON payload for an OpenAI chat tool description. - private sealed class OpenAIChatToolJson + internal sealed class OpenAIChatToolJson { /// Gets a singleton JSON data for empty parameters. Optimization for the reasonably common case of a parameterless function. public static BinaryData ZeroFunctionParametersSchema { get; } = new("""{"type":"object","required":[],"properties":{}}"""u8.ToArray()); @@ -681,12 +681,4 @@ private static FunctionCallContent ParseCallContentFromBinaryData(BinaryData ut8 FunctionCallContent.CreateFromParsedArguments(ut8Json, callId, name, argumentParser: static json => JsonSerializer.Deserialize(json, (JsonTypeInfo>)AIJsonUtilities.DefaultOptions.GetTypeInfo(typeof(IDictionary)))!); - - /// Source-generated JSON type information. - [JsonSourceGenerationOptions(JsonSerializerDefaults.Web, - UseStringEnumConverter = true, - DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, - WriteIndented = true)] - [JsonSerializable(typeof(OpenAIChatToolJson))] - private sealed partial class JsonContext : JsonSerializerContext; } diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeExtensions.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeExtensions.cs new file mode 100644 index 00000000000..4ccba79c528 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeExtensions.cs @@ -0,0 +1,53 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using Microsoft.Shared.Diagnostics; +using OpenAI.RealtimeConversation; + +namespace Microsoft.Extensions.AI; + +/// +/// Provides extension methods for working with and related types. +/// +public static class OpenAIRealtimeExtensions +{ + /// + /// Converts a into a so that + /// it can be used with . + /// + /// A that can be used with . + public static ConversationFunctionTool ToConversationFunctionTool(this AIFunction aiFunction) + { + _ = Throw.IfNull(aiFunction); + + var parametersSchema = new ConversationFunctionToolParametersSchema + { + Type = "object", + Properties = aiFunction.Metadata.Parameters + .Where(p => p.Schema is JsonElement) + .ToDictionary(p => p.Name, p => (JsonElement)p.Schema!), + Required = aiFunction.Metadata.Parameters + .Where(p => p.IsRequired) + .Select(p => p.Name), + }; + + return new ConversationFunctionTool + { + Name = aiFunction.Metadata.Name, + Description = aiFunction.Metadata.Description, + Parameters = new BinaryData(JsonSerializer.SerializeToUtf8Bytes( + parametersSchema, OpenAIJsonContext.Default.ConversationFunctionToolParametersSchema)) + }; + } + + internal sealed class ConversationFunctionToolParametersSchema + { + public string? Type { get; set; } + public IDictionary? Properties { get; set; } + public IEnumerable? Required { get; set; } + } +} diff --git a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeTests.cs b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeTests.cs new file mode 100644 index 00000000000..4e4c071e78b --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeTests.cs @@ -0,0 +1,71 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.ComponentModel; +using Xunit; + +namespace Microsoft.Extensions.AI; + +public class OpenAIRealtimeTests +{ + [Fact] + public void ConvertsAIFunctionToConversationFunctionTool_Basics() + { + var input = AIFunctionFactory.Create(() => { }, "MyFunction", "MyDescription"); + var result = input.ToConversationFunctionTool(); + + Assert.Equal("MyFunction", result.Name); + Assert.Equal("MyDescription", result.Description); + } + + [Fact] + public void ConvertsAIFunctionToConversationFunctionTool_Parameters() + { + var input = AIFunctionFactory.Create(MyFunction); + var result = input.ToConversationFunctionTool(); + + Assert.Equal(nameof(MyFunction), result.Name); + Assert.Equal("This is a description", result.Description); + Assert.Equal(""" + { + "type": "object", + "properties": { + "a": { + "type": "integer" + }, + "b": { + "description": "Another param", + "type": "string" + }, + "c": { + "type": "object", + "properties": { + "a": { + "type": "integer" + } + }, + "additionalProperties": false, + "required": [ + "a" + ], + "default": "null" + } + }, + "required": [ + "a", + "b" + ] + } + """, result.Parameters.ToString()); + } + + [Description("This is a description")] + private MyType MyFunction(int a, [Description("Another param")] string b, MyType? c = null) + => throw new NotSupportedException(); + + public class MyType + { + public int A { get; set; } + } +} From 0cde5ff2296349345fac66020141d242eda629d5 Mon Sep 17 00:00:00 2001 From: Steve Sanderson Date: Mon, 18 Nov 2024 16:15:00 -0500 Subject: [PATCH 2/8] Rename file --- .../{JsonContext.cs => OpenAIJsonContext.cs} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/Libraries/Microsoft.Extensions.AI.OpenAI/{JsonContext.cs => OpenAIJsonContext.cs} (100%) diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/JsonContext.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIJsonContext.cs similarity index 100% rename from src/Libraries/Microsoft.Extensions.AI.OpenAI/JsonContext.cs rename to src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIJsonContext.cs From fb708ba81b3abdb8db1c8780884c95213bcfb899 Mon Sep 17 00:00:00 2001 From: Steve Sanderson Date: Mon, 18 Nov 2024 16:47:11 -0500 Subject: [PATCH 3/8] Adding HandleToolCallsAsync --- .../OpenAIRealtimeExtensions.cs | 93 +++++++++++++++++++ ...icrosoft.Extensions.AI.OpenAI.Tests.csproj | 1 + .../OpenAIRealtimeTests.cs | 19 ++++ 3 files changed, 113 insertions(+) diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeExtensions.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeExtensions.cs index 4ccba79c528..37f5fcd496f 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeExtensions.cs @@ -5,6 +5,9 @@ using System.Collections.Generic; using System.Linq; using System.Text.Json; +using System.Text.Json.Serialization.Metadata; +using System.Threading; +using System.Threading.Tasks; using Microsoft.Shared.Diagnostics; using OpenAI.RealtimeConversation; @@ -44,6 +47,96 @@ public static ConversationFunctionTool ToConversationFunctionTool(this AIFunctio }; } + /// + /// Handles tool calls. + /// + /// If the represents a tool call, calls the corresponding tool and + /// adds the result to the . + /// + /// If the represents the end of a response, checks if this was due + /// to a tool call and if so, instructs the to begin responding to it. + /// + /// The . + /// The being processed. + /// The available tools. + /// An optional flag specifying whether to disclose detailed exception information to the model. The default value is . + /// An optional that controls JSON handling. + /// An optional . + /// A that represents the completion of processing, including invoking any asynchronous tools. + public static async Task HandleToolCallsAsync( + this RealtimeConversationSession session, + ConversationUpdate update, + IReadOnlyList tools, + bool? detailedErrors = false, + JsonSerializerOptions? jsonSerializerOptions = null, + CancellationToken cancellationToken = default) + { + _ = Throw.IfNull(session); + _ = Throw.IfNull(update); + _ = Throw.IfNull(tools); + + if (update is ConversationItemStreamingFinishedUpdate itemFinished) + { + // If we need to call a tool to update the model, do so + if (!string.IsNullOrEmpty(itemFinished.FunctionName) + && await itemFinished.GetFunctionCallOutputAsync(tools, detailedErrors, jsonSerializerOptions, cancellationToken).ConfigureAwait(false) is { } output) + { + await session.AddItemAsync(output, cancellationToken).ConfigureAwait(false); + } + } + else if (update is ConversationResponseFinishedUpdate responseFinished) + { + // If we added one or more function call results, instruct the model to respond to them + if (responseFinished.CreatedItems.Any(item => !string.IsNullOrEmpty(item.FunctionName))) + { + await session!.StartResponseAsync(cancellationToken).ConfigureAwait(false); + } + } + } + + private static async Task GetFunctionCallOutputAsync( + this ConversationItemStreamingFinishedUpdate update, + IReadOnlyList tools, + bool? detailedErrors = false, + JsonSerializerOptions? jsonSerializerOptions = null, + CancellationToken cancellationToken = default) + { + if (!string.IsNullOrEmpty(update.FunctionName) + && tools.FirstOrDefault(t => t.Metadata.Name == update.FunctionName) is AIFunction aiFunction) + { + var jsonOptions = jsonSerializerOptions ?? AIJsonUtilities.DefaultOptions; + + var functionCallContent = FunctionCallContent.CreateFromParsedArguments( + update.FunctionCallArguments, update.FunctionCallId, update.FunctionName, + argumentParser: json => JsonSerializer.Deserialize(json, + (JsonTypeInfo>)jsonOptions.GetTypeInfo(typeof(IDictionary)))!); + + try + { + var result = await aiFunction.InvokeAsync(functionCallContent.Arguments, cancellationToken).ConfigureAwait(false); + var resultJson = JsonSerializer.Serialize(result, jsonOptions.GetTypeInfo(typeof(object))); + return ConversationItem.CreateFunctionCallOutput(update.FunctionCallId, resultJson); + } + catch (JsonException) + { + return ConversationItem.CreateFunctionCallOutput(update.FunctionCallId, "Invalid JSON"); + } + catch (Exception e) when (!cancellationToken.IsCancellationRequested) + { + var message = "Error calling tool"; + + if (detailedErrors == true) + { + message += $": {e.Message}"; + } + + return ConversationItem.CreateFunctionCallOutput(update.FunctionCallId, message); + } + } + + return null; + } + internal sealed class ConversationFunctionToolParametersSchema { public string? Type { get; set; } diff --git a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/Microsoft.Extensions.AI.OpenAI.Tests.csproj b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/Microsoft.Extensions.AI.OpenAI.Tests.csproj index 0ef40e12df3..1f078b2cd67 100644 --- a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/Microsoft.Extensions.AI.OpenAI.Tests.csproj +++ b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/Microsoft.Extensions.AI.OpenAI.Tests.csproj @@ -2,6 +2,7 @@ Microsoft.Extensions.AI Unit tests for Microsoft.Extensions.AI.OpenAI + $(NoWarn);OPENAI002 diff --git a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeTests.cs b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeTests.cs index 4e4c071e78b..29eae48d7a3 100644 --- a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeTests.cs @@ -3,6 +3,8 @@ using System; using System.ComponentModel; +using System.Threading.Tasks; +using OpenAI.RealtimeConversation; using Xunit; namespace Microsoft.Extensions.AI; @@ -60,6 +62,15 @@ public void ConvertsAIFunctionToConversationFunctionTool_Parameters() """, result.Parameters.ToString()); } + [Fact] + public async Task HandleToolCallsAsync_RejectsNulls() + { + var conversationSession = (RealtimeConversationSession)default!; + + await Assert.ThrowsAsync(() => conversationSession.HandleToolCallsAsync( + new TestConversationUpdate(), [])); + } + [Description("This is a description")] private MyType MyFunction(int a, [Description("Another param")] string b, MyType? c = null) => throw new NotSupportedException(); @@ -68,4 +79,12 @@ public class MyType { public int A { get; set; } } + + private class TestConversationUpdate : ConversationUpdate + { + public TestConversationUpdate() + : base("eventId") + { + } + } } From 04aab41075c42a6eaea4204b96684f7672a7d431 Mon Sep 17 00:00:00 2001 From: Steve Sanderson Date: Mon, 18 Nov 2024 16:52:15 -0500 Subject: [PATCH 4/8] Update OpenAIRealtimeTests.cs --- .../OpenAIRealtimeTests.cs | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeTests.cs b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeTests.cs index 29eae48d7a3..85b00d9bde6 100644 --- a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeTests.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; +using System.ClientModel; using System.ComponentModel; using System.Threading.Tasks; using OpenAI.RealtimeConversation; @@ -67,8 +68,18 @@ public async Task HandleToolCallsAsync_RejectsNulls() { var conversationSession = (RealtimeConversationSession)default!; + // Null RealtimeConversationSession await Assert.ThrowsAsync(() => conversationSession.HandleToolCallsAsync( new TestConversationUpdate(), [])); + + // Null ConversationUpdate + using var session = TestRealtimeConversationSession.CreateTestInstance(); + await Assert.ThrowsAsync(() => conversationSession.HandleToolCallsAsync( + null!, [])); + + // Null tools + await Assert.ThrowsAsync(() => conversationSession.HandleToolCallsAsync( + new TestConversationUpdate(), null!)); } [Description("This is a description")] @@ -80,6 +91,22 @@ public class MyType public int A { get; set; } } + private class TestRealtimeConversationSession : RealtimeConversationSession + { + protected internal TestRealtimeConversationSession(RealtimeConversationClient parentClient, Uri endpoint, ApiKeyCredential credential) + : base(parentClient, endpoint, credential) + { + } + + public static TestRealtimeConversationSession CreateTestInstance() + { + var credential = new ApiKeyCredential("key"); + return new TestRealtimeConversationSession( + new RealtimeConversationClient("model", credential), + new Uri("http://endpoint"), credential); + } + } + private class TestConversationUpdate : ConversationUpdate { public TestConversationUpdate() From cff5f9b3546f5841071eb59dded324d7c0b500c8 Mon Sep 17 00:00:00 2001 From: Steve Sanderson Date: Mon, 18 Nov 2024 17:27:44 -0500 Subject: [PATCH 5/8] Start on integration tests --- eng/packages/TestOnly.props | 2 +- ...icrosoft.Extensions.AI.OpenAI.Tests.csproj | 1 + .../OpenAIRealtimeIntegrationTests.cs | 52 +++++++++++++++++++ .../OpenAIRealtimeTests.cs | 4 ++ 4 files changed, 58 insertions(+), 1 deletion(-) create mode 100644 test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeIntegrationTests.cs diff --git a/eng/packages/TestOnly.props b/eng/packages/TestOnly.props index d9802530ed3..6443d61c224 100644 --- a/eng/packages/TestOnly.props +++ b/eng/packages/TestOnly.props @@ -2,7 +2,7 @@ - + diff --git a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/Microsoft.Extensions.AI.OpenAI.Tests.csproj b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/Microsoft.Extensions.AI.OpenAI.Tests.csproj index 1f078b2cd67..66412bfeace 100644 --- a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/Microsoft.Extensions.AI.OpenAI.Tests.csproj +++ b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/Microsoft.Extensions.AI.OpenAI.Tests.csproj @@ -7,6 +7,7 @@ true + true diff --git a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeIntegrationTests.cs b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeIntegrationTests.cs new file mode 100644 index 00000000000..b77a50fec29 --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeIntegrationTests.cs @@ -0,0 +1,52 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Diagnostics.CodeAnalysis; +using System.Threading.Tasks; +using Azure.AI.OpenAI; +using Microsoft.TestUtilities; +using OpenAI.RealtimeConversation; +using Xunit; + +namespace Microsoft.Extensions.AI; + +public class OpenAIRealtimeIntegrationTests +{ + private RealtimeConversationClient? _conversationClient; + + public OpenAIRealtimeIntegrationTests() + { + _conversationClient = CreateConversationClient(); + } + + [Fact] + public async Task CanPerformFunctionCall() + { + SkipIfNotEnabled(); + + using var conversation = await _conversationClient.StartConversationSessionAsync(); + Assert.NotNull(conversation); + } + + [MemberNotNull(nameof(_conversationClient))] + protected void SkipIfNotEnabled() + { + if (_conversationClient is null) + { + throw new SkipTestException("Client is not enabled."); + } + } + + private static RealtimeConversationClient? CreateConversationClient() + { + var realtimeModel = Environment.GetEnvironmentVariable("OPENAI_REALTIME_MODEL"); + if (string.IsNullOrEmpty(realtimeModel)) + { + return null; + } + + var openAiClient = (AzureOpenAIClient?)IntegrationTestHelpers.GetOpenAIClient(); + return openAiClient?.GetRealtimeConversationClient(realtimeModel); + } +} diff --git a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeTests.cs b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeTests.cs index 85b00d9bde6..32e4d059a51 100644 --- a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeTests.cs @@ -10,6 +10,10 @@ namespace Microsoft.Extensions.AI; +// Note that we're limited on ability to unit-test OpenAIRealtimeExtension, because some of the +// OpenAI types it uses (e.g., ConversationItemStreamingFinishedUpdate) can't be instantiated or +// subclassed from outside. We will mostly have to rely on integration tests for now. + public class OpenAIRealtimeTests { [Fact] From 852adcfd56dd9a6c851b28525b95898eaa571e76 Mon Sep 17 00:00:00 2001 From: Steve Sanderson Date: Mon, 18 Nov 2024 18:04:27 -0500 Subject: [PATCH 6/8] Add integration test --- .../OpenAIRealtimeIntegrationTests.cs | 66 ++++++++++++++++++- 1 file changed, 64 insertions(+), 2 deletions(-) diff --git a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeIntegrationTests.cs b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeIntegrationTests.cs index b77a50fec29..9f581a17fbd 100644 --- a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeIntegrationTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeIntegrationTests.cs @@ -2,7 +2,10 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; +using System.Collections.Generic; +using System.ComponentModel; using System.Diagnostics.CodeAnalysis; +using System.Linq; using System.Threading.Tasks; using Azure.AI.OpenAI; using Microsoft.TestUtilities; @@ -25,8 +28,67 @@ public async Task CanPerformFunctionCall() { SkipIfNotEnabled(); - using var conversation = await _conversationClient.StartConversationSessionAsync(); - Assert.NotNull(conversation); + var roomCapacityTool = AIFunctionFactory.Create(GetRoomCapacity); + var sessionOptions = new ConversationSessionOptions + { + Instructions = "You help with booking appointments", + Tools = { roomCapacityTool.ToConversationFunctionTool() }, + ContentModalities = ConversationContentModalities.Text, + }; + + using var session = await _conversationClient.StartConversationSessionAsync(); + await session.ConfigureSessionAsync(sessionOptions); + + await foreach (var update in session.ReceiveUpdatesAsync()) + { + switch (update) + { + case ConversationSessionStartedUpdate: + await session.AddItemAsync( + ConversationItem.CreateUserMessage([""" + What type of room can hold the most people? + Reply with the full name of the biggest venue and its capacity only. + Do not mention the other venues. + """])); + await session.StartResponseAsync(); + break; + + case ConversationResponseFinishedUpdate responseFinished: + var content = responseFinished.CreatedItems + .SelectMany(i => i.MessageContentParts ?? []) + .OfType() + .FirstOrDefault(); + if (content is not null) + { + Assert.Contains("VehicleAssemblyBuilding", content.Text.Replace(" ", string.Empty)); + Assert.Contains("12000", content.Text.Replace(",", string.Empty)); + return; + } + + break; + } + + await session.HandleToolCallsAsync(update, [roomCapacityTool]); + } + } + + [Description("Returns the number of people that can fit in a room.")] + private static int GetRoomCapacity(RoomType roomType) + { + return roomType switch + { + RoomType.ShuttleSimulator => throw new InvalidOperationException("No longer available"), + RoomType.NorthAtlantisLawn => 450, + RoomType.VehicleAssemblyBuilding => 12000, + _ => throw new NotSupportedException($"Unknown room type: {roomType}"), + }; + } + + private enum RoomType + { + ShuttleSimulator, + NorthAtlantisLawn, + VehicleAssemblyBuilding, } [MemberNotNull(nameof(_conversationClient))] From 4b24e969a2dae5af2be0f12de75e7ef760ee44a2 Mon Sep 17 00:00:00 2001 From: Steve Sanderson Date: Mon, 18 Nov 2024 18:06:06 -0500 Subject: [PATCH 7/8] Skip properly --- .../OpenAIRealtimeIntegrationTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeIntegrationTests.cs b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeIntegrationTests.cs index 9f581a17fbd..46b9fac7cab 100644 --- a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeIntegrationTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIRealtimeIntegrationTests.cs @@ -23,7 +23,7 @@ public OpenAIRealtimeIntegrationTests() _conversationClient = CreateConversationClient(); } - [Fact] + [ConditionalFact] public async Task CanPerformFunctionCall() { SkipIfNotEnabled(); From 90319d284bba484fd2ec742f5184666e0be95b29 Mon Sep 17 00:00:00 2001 From: Steve Sanderson Date: Tue, 19 Nov 2024 15:27:56 -0500 Subject: [PATCH 8/8] Use {} as fallback parameter schema --- .../OpenAIRealtimeExtensions.cs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeExtensions.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeExtensions.cs index 37f5fcd496f..c47cfc52c21 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeExtensions.cs @@ -18,6 +18,8 @@ namespace Microsoft.Extensions.AI; /// public static class OpenAIRealtimeExtensions { + private static readonly JsonElement _defaultParameterSchema = JsonDocument.Parse("{}").RootElement; + /// /// Converts a into a so that /// it can be used with . @@ -31,8 +33,7 @@ public static ConversationFunctionTool ToConversationFunctionTool(this AIFunctio { Type = "object", Properties = aiFunction.Metadata.Parameters - .Where(p => p.Schema is JsonElement) - .ToDictionary(p => p.Name, p => (JsonElement)p.Schema!), + .ToDictionary(p => p.Name, GetParameterSchema), Required = aiFunction.Metadata.Parameters .Where(p => p.IsRequired) .Select(p => p.Name), @@ -94,6 +95,15 @@ public static async Task HandleToolCallsAsync( } } + private static JsonElement GetParameterSchema(AIFunctionParameterMetadata parameterMetadata) + { + return parameterMetadata switch + { + { Schema: JsonElement jsonElement } => jsonElement, + _ => _defaultParameterSchema, + }; + } + private static async Task GetFunctionCallOutputAsync( this ConversationItemStreamingFinishedUpdate update, IReadOnlyList tools,