From d2efdbaa7ce80288426bc5724f3ba6f4719cd507 Mon Sep 17 00:00:00 2001 From: Stephen Toub Date: Mon, 2 Dec 2024 10:10:30 -0500 Subject: [PATCH 1/3] Update otel chat client / embedding generator for 1.29 Also address feedback to include additional properties as tags. --- .../ChatCompletion/ChatClientMetadata.cs | 14 +++++- ...StreamingChatCompletionUpdateExtensions.cs | 16 +++++++ .../Embeddings/EmbeddingGeneratorMetadata.cs | 19 +++++++- .../ChatCompletion/OpenTelemetryChatClient.cs | 31 ++++++++++++- .../OpenTelemetryEmbeddingGenerator.cs | 44 ++++++++++++++++--- .../OpenTelemetryConsts.cs | 4 +- .../OpenTelemetryChatClientTests.cs | 19 ++++++++ 7 files changed, 136 insertions(+), 11 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatClientMetadata.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatClientMetadata.cs index d21d3b20585..406b9768dd7 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatClientMetadata.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatClientMetadata.cs @@ -9,7 +9,10 @@ namespace Microsoft.Extensions.AI; public class ChatClientMetadata { /// Initializes a new instance of the class. - /// The name of the chat completion provider, if applicable. + /// + /// The name of the chat completion provider, if applicable. Where possible, this should map to the + /// appropriate name defined in the OpenTelemetry Semantic Conventions for Generative AI systems. + /// /// The URL for accessing the chat completion provider, if applicable. /// The ID of the chat completion model used, if applicable. public ChatClientMetadata(string? providerName = null, Uri? providerUri = null, string? modelId = null) @@ -20,12 +23,19 @@ public ChatClientMetadata(string? providerName = null, Uri? providerUri = null, } /// Gets the name of the chat completion provider. + /// + /// Where possible, this maps to the appropriate name defined in the + /// OpenTelemetry Semantic Conventions for Generative AI systems. + /// public string? ProviderName { get; } /// Gets the URL for accessing the chat completion provider. public Uri? ProviderUri { get; } /// Gets the ID of the model used by this chat completion provider. - /// This value can be null if either the name is unknown or there are multiple possible models associated with this instance. + /// + /// This value can be null if either the name is unknown or there are multiple possible models associated with this instance. + /// An individual request may override this value via . + /// public string? ModelId { get; } } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/StreamingChatCompletionUpdateExtensions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/StreamingChatCompletionUpdateExtensions.cs index 928b9366a27..b70d7471b80 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/StreamingChatCompletionUpdateExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/StreamingChatCompletionUpdateExtensions.cs @@ -136,17 +136,33 @@ private static void AddMessagesToCompletion(Dictionary message { if (messages.Count <= 1) { + // Add the single message if there is one. foreach (var entry in messages) { AddMessage(completion, coalesceContent, entry); } + + // In the vast majority case where there's only one choice, promote any additional properties + // from the single message to the chat completion, making them more discoverable and more similar + // to how they're typically surfaced from non-streaming services. + if (completion.Choices.Count == 1 && + completion.Choices[0].AdditionalProperties is { } messageProps) + { + completion.Choices[0].AdditionalProperties = null; + completion.AdditionalProperties = messageProps; + } } else { + // Add all of the messages, sorted by choice index. foreach (var entry in messages.OrderBy(entry => entry.Key)) { AddMessage(completion, coalesceContent, entry); } + + // If there are multiple choices, we don't promote additional properties from the individual messages. + // At a minimum, we'd want to know which choice the additional properties applied to, and if there were + // conflicting values across the choices, it would be unclear which one should be used. } static void AddMessage(ChatCompletion completion, bool coalesceContent, KeyValuePair entry) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Embeddings/EmbeddingGeneratorMetadata.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Embeddings/EmbeddingGeneratorMetadata.cs index 0f2f7b23af5..a3f5181648b 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Embeddings/EmbeddingGeneratorMetadata.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Embeddings/EmbeddingGeneratorMetadata.cs @@ -9,7 +9,11 @@ namespace Microsoft.Extensions.AI; public class EmbeddingGeneratorMetadata { /// Initializes a new instance of the class. - /// The name of the embedding generation provider, if applicable. + + /// + /// The name of the embedding generation provider, if applicable. Where possible, this should map to the + /// appropriate name defined in the OpenTelemetry Semantic Conventions for Generative AI systems. + /// /// The URL for accessing the embedding generation provider, if applicable. /// The ID of the embedding generation model used, if applicable. /// The number of dimensions in vectors produced by this generator, if applicable. @@ -22,15 +26,26 @@ public EmbeddingGeneratorMetadata(string? providerName = null, Uri? providerUri } /// Gets the name of the embedding generation provider. + /// + /// Where possible, this maps to the appropriate name defined in the + /// OpenTelemetry Semantic Conventions for Generative AI systems. + /// public string? ProviderName { get; } /// Gets the URL for accessing the embedding generation provider. public Uri? ProviderUri { get; } /// Gets the ID of the model used by this embedding generation provider. - /// This value can be null if either the name is unknown or there are multiple possible models associated with this instance. + /// + /// This value can be null if either the name is unknown or there are multiple possible models associated with this instance. + /// An individual request may override this value via . + /// public string? ModelId { get; } /// Gets the number of dimensions in the embeddings produced by this instance. + /// + /// This value can be null if either the number of dimensions is unknown or there are multiple possible lengths associated with this instance. + /// An individual request may override this value via . + /// public int? Dimensions { get; } } diff --git a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/OpenTelemetryChatClient.cs b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/OpenTelemetryChatClient.cs index 193006780a2..9da805932f2 100644 --- a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/OpenTelemetryChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/OpenTelemetryChatClient.cs @@ -23,7 +23,7 @@ namespace Microsoft.Extensions.AI; /// Represents a delegating chat client that implements the OpenTelemetry Semantic Conventions for Generative AI systems. /// -/// The draft specification this follows is available at . +/// This class provides an implementation of the Semantic Conventions for Generative AI systems v1.29, defined at . /// The specification is still experimental and subject to change; as such, the telemetry output by this client is also subject to change. /// public sealed partial class OpenTelemetryChatClient : DelegatingChatClient @@ -288,6 +288,19 @@ public override async IAsyncEnumerable CompleteSt { _ = activity.AddTag(OpenTelemetryConsts.GenAI.Request.PerProvider(_system, "seed"), seed); } + + if (options.AdditionalProperties is { } props) + { + // Log all additional request options as per-provider tags. This is non-normative, but it covers cases where + // there's a per-provider specification in a best-effort manner (e.g. gen_ai.openai.request.service_tier), + // and more generally cases where there's additional useful information to be logged. + foreach (KeyValuePair prop in props) + { + _ = activity.AddTag( + OpenTelemetryConsts.GenAI.Request.PerProvider(_system, JsonNamingPolicy.SnakeCaseLower.ConvertName(prop.Key)), + prop.Value); + } + } } } } @@ -375,6 +388,22 @@ private void TraceCompletion( { _ = activity.AddTag(OpenTelemetryConsts.GenAI.Response.OutputTokens, outputTokens); } + + if (_system is not null) + { + // Log all additional response properties as per-provider tags. This is non-normative, but it covers cases where + // there's a per-provider specification in a best-effort manner (e.g. gen_ai.openai.response.system_fingerprint), + // and more generally cases where there's additional useful information to be logged. + if (completion.AdditionalProperties is { } props) + { + foreach (KeyValuePair prop in props) + { + _ = activity.AddTag( + OpenTelemetryConsts.GenAI.Response.PerProvider(_system, JsonNamingPolicy.SnakeCaseLower.ConvertName(prop.Key)), + prop.Value); + } + } + } } } diff --git a/src/Libraries/Microsoft.Extensions.AI/Embeddings/OpenTelemetryEmbeddingGenerator.cs b/src/Libraries/Microsoft.Extensions.AI/Embeddings/OpenTelemetryEmbeddingGenerator.cs index 09f762d33d0..8bb38bf2e07 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Embeddings/OpenTelemetryEmbeddingGenerator.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Embeddings/OpenTelemetryEmbeddingGenerator.cs @@ -6,6 +6,7 @@ using System.Diagnostics; using System.Diagnostics.Metrics; using System.Linq; +using System.Text.Json; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; @@ -15,8 +16,8 @@ namespace Microsoft.Extensions.AI; /// Represents a delegating embedding generator that implements the OpenTelemetry Semantic Conventions for Generative AI systems. /// -/// The draft specification this follows is available at . -/// The specification is still experimental and subject to change; as such, the telemetry output by this generator is also subject to change. +/// This class provides an implementation of the Semantic Conventions for Generative AI systems v1.29, defined at . +/// The specification is still experimental and subject to change; as such, the telemetry output by this client is also subject to change. /// /// The type of input used to produce embeddings. /// The type of embedding generated. @@ -29,6 +30,7 @@ public sealed class OpenTelemetryEmbeddingGenerator : Delega private readonly Histogram _tokenUsageHistogram; private readonly Histogram _operationDurationHistogram; + private readonly string? _system; private readonly string? _modelId; private readonly string? _modelProvider; private readonly string? _endpointAddress; @@ -49,6 +51,7 @@ public OpenTelemetryEmbeddingGenerator(IEmbeddingGenerator i Debug.Assert(innerGenerator is not null, "Should have been validated by the base ctor."); EmbeddingGeneratorMetadata metadata = innerGenerator!.Metadata; + _system = metadata.ProviderName; _modelId = metadata.ModelId; _modelProvider = metadata.ProviderName; _endpointAddress = metadata.ProviderUri?.GetLeftPart(UriPartial.Path); @@ -126,11 +129,11 @@ protected override void Dispose(bool disposing) string? modelId = options?.ModelId ?? _modelId; activity = _activitySource.StartActivity( - string.IsNullOrWhiteSpace(modelId) ? OpenTelemetryConsts.GenAI.Embed : $"{OpenTelemetryConsts.GenAI.Embed} {modelId}", + string.IsNullOrWhiteSpace(modelId) ? OpenTelemetryConsts.GenAI.Embeddings : $"{OpenTelemetryConsts.GenAI.Embeddings} {modelId}", ActivityKind.Client, default(ActivityContext), [ - new(OpenTelemetryConsts.GenAI.Operation.Name, OpenTelemetryConsts.GenAI.Embed), + new(OpenTelemetryConsts.GenAI.Operation.Name, OpenTelemetryConsts.GenAI.Embeddings), new(OpenTelemetryConsts.GenAI.Request.Model, modelId), new(OpenTelemetryConsts.GenAI.SystemName, _modelProvider), ]); @@ -148,6 +151,23 @@ protected override void Dispose(bool disposing) { _ = activity.AddTag(OpenTelemetryConsts.GenAI.Request.EmbeddingDimensions, dimensions); } + + if (options is not null && + _system is not null) + { + // Log all additional request options as per-provider tags. This is non-normative, but it covers cases where + // there's a per-provider specification in a best-effort manner (e.g. gen_ai.openai.request.service_tier), + // and more generally cases where there's additional useful information to be logged. + if (options.AdditionalProperties is { } props) + { + foreach (KeyValuePair prop in props) + { + _ = activity.AddTag( + OpenTelemetryConsts.GenAI.Request.PerProvider(_system, JsonNamingPolicy.SnakeCaseLower.ConvertName(prop.Key)), + prop.Value); + } + } + } } } @@ -212,12 +232,26 @@ private void TraceCompletion( { _ = activity.AddTag(OpenTelemetryConsts.GenAI.Response.Model, responseModelId); } + + // Log all additional response properties as per-provider tags. This is non-normative, but it covers cases where + // there's a per-provider specification in a best-effort manner (e.g. gen_ai.openai.response.system_fingerprint), + // and more generally cases where there's additional useful information to be logged. + if (_system is not null && + embeddings?.AdditionalProperties is { } props) + { + foreach (KeyValuePair prop in props) + { + _ = activity.AddTag( + OpenTelemetryConsts.GenAI.Response.PerProvider(_system, JsonNamingPolicy.SnakeCaseLower.ConvertName(prop.Key)), + prop.Value); + } + } } } private void AddMetricTags(ref TagList tags, string? requestModelId, string? responseModelId) { - tags.Add(OpenTelemetryConsts.GenAI.Operation.Name, OpenTelemetryConsts.GenAI.Embed); + tags.Add(OpenTelemetryConsts.GenAI.Operation.Name, OpenTelemetryConsts.GenAI.Embeddings); if (requestModelId is not null) { diff --git a/src/Libraries/Microsoft.Extensions.AI/OpenTelemetryConsts.cs b/src/Libraries/Microsoft.Extensions.AI/OpenTelemetryConsts.cs index 27a543705ba..4c40c04c236 100644 --- a/src/Libraries/Microsoft.Extensions.AI/OpenTelemetryConsts.cs +++ b/src/Libraries/Microsoft.Extensions.AI/OpenTelemetryConsts.cs @@ -30,7 +30,7 @@ public static class GenAI public const string SystemName = "gen_ai.system"; public const string Chat = "chat"; - public const string Embed = "embed"; + public const string Embeddings = "embeddings"; public static class Assistant { @@ -81,6 +81,8 @@ public static class Response public const string InputTokens = "gen_ai.response.input_tokens"; public const string Model = "gen_ai.response.model"; public const string OutputTokens = "gen_ai.response.output_tokens"; + + public static string PerProvider(string providerName, string parameterName) => $"gen_ai.{providerName}.response.{parameterName}"; } public static class System diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/OpenTelemetryChatClientTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/OpenTelemetryChatClientTests.cs index 3d7d05f981a..e7123db6ee0 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/OpenTelemetryChatClientTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/OpenTelemetryChatClientTests.cs @@ -49,6 +49,11 @@ public async Task ExpectedInformationLogged_Async(bool enableSensitiveData, bool OutputTokenCount = 20, TotalTokenCount = 42, }, + AdditionalProperties = new() + { + ["system_fingerprint"] = "abcdefgh", + ["AndSomethingElse"] = "value2", + }, }; }, CompleteStreamingAsyncCallback = CallbackAsync, @@ -83,6 +88,11 @@ async static IAsyncEnumerable CallbackAsync( OutputTokenCount = 20, TotalTokenCount = 42, })], + AdditionalProperties = new() + { + ["system_fingerprint"] = "abcdefgh", + ["AndSomethingElse"] = "value2", + }, }; } @@ -116,6 +126,11 @@ async static IAsyncEnumerable CallbackAsync( ResponseFormat = ChatResponseFormat.Json, Temperature = 6.0f, StopSequences = ["hello", "world"], + AdditionalProperties = new() + { + ["service_tier"] = "value1", + ["SomethingElse"] = "value2", + }, }; if (streaming) @@ -149,11 +164,15 @@ async static IAsyncEnumerable CallbackAsync( Assert.Equal(7, activity.GetTagItem("gen_ai.request.top_k")); Assert.Equal(123, activity.GetTagItem("gen_ai.request.max_tokens")); Assert.Equal("""["hello", "world"]""", activity.GetTagItem("gen_ai.request.stop_sequences")); + Assert.Equal("value1", activity.GetTagItem("gen_ai.testservice.request.service_tier")); + Assert.Equal("value2", activity.GetTagItem("gen_ai.testservice.request.something_else")); Assert.Equal("id123", activity.GetTagItem("gen_ai.response.id")); Assert.Equal("""["stop"]""", activity.GetTagItem("gen_ai.response.finish_reasons")); Assert.Equal(10, activity.GetTagItem("gen_ai.response.input_tokens")); Assert.Equal(20, activity.GetTagItem("gen_ai.response.output_tokens")); + Assert.Equal("abcdefgh", activity.GetTagItem("gen_ai.testservice.response.system_fingerprint")); + Assert.Equal("value2", activity.GetTagItem("gen_ai.testservice.response.and_something_else")); Assert.True(activity.Duration.TotalMilliseconds > 0); From e429073f1ea91231820c77f673b277a97e691638 Mon Sep 17 00:00:00 2001 From: Stephen Toub Date: Mon, 2 Dec 2024 22:15:37 -0500 Subject: [PATCH 2/3] Increase code coverage --- ...soft.Extensions.AI.AzureAIInference.csproj | 2 +- .../Microsoft.Extensions.AI.csproj | 2 +- .../TestEmbeddingGenerator.cs | 2 +- .../ChatCompletion/ChatClientBuilderTest.cs | 18 ++++ .../OpenTelemetryChatClientTests.cs | 4 +- .../OpenTelemetryEmbeddingGeneratorTests.cs | 91 +++++++++++++++++++ 6 files changed, 115 insertions(+), 4 deletions(-) create mode 100644 test/Libraries/Microsoft.Extensions.AI.Tests/Embeddings/OpenTelemetryEmbeddingGeneratorTests.cs diff --git a/src/Libraries/Microsoft.Extensions.AI.AzureAIInference/Microsoft.Extensions.AI.AzureAIInference.csproj b/src/Libraries/Microsoft.Extensions.AI.AzureAIInference/Microsoft.Extensions.AI.AzureAIInference.csproj index 6896a186d0a..919fa9b751f 100644 --- a/src/Libraries/Microsoft.Extensions.AI.AzureAIInference/Microsoft.Extensions.AI.AzureAIInference.csproj +++ b/src/Libraries/Microsoft.Extensions.AI.AzureAIInference/Microsoft.Extensions.AI.AzureAIInference.csproj @@ -9,7 +9,7 @@ preview true - 83 + 91 0 diff --git a/src/Libraries/Microsoft.Extensions.AI/Microsoft.Extensions.AI.csproj b/src/Libraries/Microsoft.Extensions.AI/Microsoft.Extensions.AI.csproj index a3bed483c44..7bb57e95a3b 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Microsoft.Extensions.AI.csproj +++ b/src/Libraries/Microsoft.Extensions.AI/Microsoft.Extensions.AI.csproj @@ -11,7 +11,7 @@ preview true - 83 + 88 0 diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/TestEmbeddingGenerator.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/TestEmbeddingGenerator.cs index 7438edc752e..0765dc60a2b 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/TestEmbeddingGenerator.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/TestEmbeddingGenerator.cs @@ -10,7 +10,7 @@ namespace Microsoft.Extensions.AI; public sealed class TestEmbeddingGenerator : IEmbeddingGenerator> { - public EmbeddingGeneratorMetadata Metadata { get; } = new(); + public EmbeddingGeneratorMetadata Metadata { get; set; } = new(); public Func, EmbeddingGenerationOptions?, CancellationToken, Task>>>? GenerateAsyncCallback { get; set; } diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/ChatClientBuilderTest.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/ChatClientBuilderTest.cs index c9d09db9836..c39c6b8c2b7 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/ChatClientBuilderTest.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/ChatClientBuilderTest.cs @@ -78,6 +78,24 @@ public void DoesNotAllowFactoriesToReturnNull() Assert.Contains("entry at index 0", ex.Message); } + [Fact] + public void UsesEmptyServiceProviderWhenNoServicesProvided() + { + using var innerClient = new TestChatClient(); + ChatClientBuilder builder = new(innerClient); + builder.Use((innerClient, serviceProvider) => + { + Assert.Null(serviceProvider.GetService(typeof(object))); + + var keyedServiceProvider = Assert.IsAssignableFrom(serviceProvider); + Assert.Null(keyedServiceProvider.GetKeyedService(typeof(object), "key")); + Assert.Throws(() => keyedServiceProvider.GetRequiredKeyedService(typeof(object), "key")); + + return innerClient; + }); + builder.Build(); + } + private sealed class InnerClientCapturingChatClient(string name, IChatClient innerClient) : DelegatingChatClient(innerClient) { #pragma warning disable S3604 // False positive: Member initializer values should not be redundant diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/OpenTelemetryChatClientTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/OpenTelemetryChatClientTests.cs index e7123db6ee0..1d99cb60731 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/OpenTelemetryChatClientTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/OpenTelemetryChatClientTests.cs @@ -96,7 +96,7 @@ async static IAsyncEnumerable CallbackAsync( }; } - var chatClient = innerClient + using var chatClient = innerClient .AsBuilder() .UseOpenTelemetry(loggerFactory, sourceName, configure: instance => { @@ -125,6 +125,7 @@ async static IAsyncEnumerable CallbackAsync( PresencePenalty = 5.0f, ResponseFormat = ChatResponseFormat.Json, Temperature = 6.0f, + Seed = 42, StopSequences = ["hello", "world"], AdditionalProperties = new() { @@ -166,6 +167,7 @@ async static IAsyncEnumerable CallbackAsync( Assert.Equal("""["hello", "world"]""", activity.GetTagItem("gen_ai.request.stop_sequences")); Assert.Equal("value1", activity.GetTagItem("gen_ai.testservice.request.service_tier")); Assert.Equal("value2", activity.GetTagItem("gen_ai.testservice.request.something_else")); + Assert.Equal(42L, activity.GetTagItem("gen_ai.testservice.request.seed")); Assert.Equal("id123", activity.GetTagItem("gen_ai.response.id")); Assert.Equal("""["stop"]""", activity.GetTagItem("gen_ai.response.finish_reasons")); diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Embeddings/OpenTelemetryEmbeddingGeneratorTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Embeddings/OpenTelemetryEmbeddingGeneratorTests.cs new file mode 100644 index 00000000000..e5dc014d6aa --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Embeddings/OpenTelemetryEmbeddingGeneratorTests.cs @@ -0,0 +1,91 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Testing; +using OpenTelemetry.Trace; +using Xunit; + +namespace Microsoft.Extensions.AI; + +public class OpenTelemetryEmbeddingGeneratorTests +{ + [Fact] + public async Task ExpectedInformationLogged_Async() + { + var sourceName = Guid.NewGuid().ToString(); + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + var collector = new FakeLogCollector(); + using ILoggerFactory loggerFactory = LoggerFactory.Create(b => b.AddProvider(new FakeLoggerProvider(collector))); + + using var innerGenerator = new TestEmbeddingGenerator + { + Metadata = new("testservice", new Uri("http://localhost:12345/something"), "amazingmodel", 384), + GenerateAsyncCallback = async (values, options, cancellationToken) => + { + await Task.Yield(); + return new GeneratedEmbeddings>([new Embedding(new float[] { 1, 2, 3 })]) + { + Usage = new() + { + InputTokenCount = 10, + TotalTokenCount = 10, + }, + AdditionalProperties = new() + { + ["system_fingerprint"] = "abcdefgh", + ["AndSomethingElse"] = "value2", + } + }; + }, + }; + + using var generator = innerGenerator + .AsBuilder() + .UseOpenTelemetry(loggerFactory, sourceName) + .Build(); + + var options = new EmbeddingGenerationOptions + { + ModelId = "replacementmodel", + AdditionalProperties = new() + { + ["service_tier"] = "value1", + ["SomethingElse"] = "value2", + }, + }; + + await generator.GenerateEmbeddingVectorAsync("hello", options); + + var activity = Assert.Single(activities); + + Assert.NotNull(activity.Id); + Assert.NotEmpty(activity.Id); + + Assert.Equal("http://localhost:12345/something", activity.GetTagItem("server.address")); + Assert.Equal(12345, (int)activity.GetTagItem("server.port")!); + + Assert.Equal("embeddings replacementmodel", activity.DisplayName); + Assert.Equal("testservice", activity.GetTagItem("gen_ai.system")); + + Assert.Equal("replacementmodel", activity.GetTagItem("gen_ai.request.model")); + Assert.Equal("value1", activity.GetTagItem("gen_ai.testservice.request.service_tier")); + Assert.Equal("value2", activity.GetTagItem("gen_ai.testservice.request.something_else")); + + Assert.Equal(10, activity.GetTagItem("gen_ai.response.input_tokens")); + Assert.Equal("abcdefgh", activity.GetTagItem("gen_ai.testservice.response.system_fingerprint")); + Assert.Equal("value2", activity.GetTagItem("gen_ai.testservice.response.and_something_else")); + + Assert.True(activity.Duration.TotalMilliseconds > 0); + } +} From c048554a6fe4972fef16e74291ab606f43e3b966 Mon Sep 17 00:00:00 2001 From: Stephen Toub Date: Wed, 4 Dec 2024 09:54:57 -0500 Subject: [PATCH 3/3] Bump Microsoft.Extensions.AI.Abstractions min coverage --- .../Microsoft.Extensions.AI.Abstractions.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.csproj b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.csproj index 4d7e314a0e4..756ec27adc4 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.csproj +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.csproj @@ -9,7 +9,7 @@ preview true - 83 + 84 0