add image embedding to azure index (#271)

## Purpose  * ... ## Does this introduce a breaking change?  ``` [ ] Yes [ ] No ``` ## Pull Request Type What kind of change does this Pull Request introduce?  ``` [ ] Bugfix [ ] Feature [ ] Code style update (formatting, local variables) [ ] Refactoring (no functional changes, no api changes) [ ] Documentation content changes [ ] Other... Please describe: ``` ## How to Test * Get the code ``` git clone [repo-address] cd [repo-name] git checkout [branch-name] npm install ``` * Test the code  ``` ``` ## What to Check Verify that the following are valid * ... ## Other Information  --------- Co-authored-by: David Pine <[email protected]>
Azure-Samples · Feb 6, 2024 · f5b168f · f5b168f
1 parent 9543b54
commit f5b168f
Show file tree

Hide file tree

Showing 37 changed files with 739 additions and 100 deletions.
diff --git a/app/backend/Extensions/ServiceCollectionExtensions.cs b/app/backend/Extensions/ServiceCollectionExtensions.cs
@@ -27,7 +27,7 @@ internal static IServiceCollection AddAzureServices(this IServiceCollection serv
             return sp.GetRequiredService<BlobServiceClient>().GetBlobContainerClient(azureStorageContainer);
         });
 
-        services.AddSingleton<IDocumentService, AzureDocumentService>(sp =>
+        services.AddSingleton<ISearchService, AzureSearchService>(sp =>
         {
             var config = sp.GetRequiredService<IConfiguration>();
             var azureSearchServiceEndpoint = config["AzureSearchServiceEndpoint"];
@@ -39,7 +39,7 @@ internal static IServiceCollection AddAzureServices(this IServiceCollection serv
             var searchClient = new SearchClient(
                                new Uri(azureSearchServiceEndpoint), azureSearchIndex, s_azureCredential);
 
-            return new AzureDocumentService(searchClient);
+            return new AzureSearchService(searchClient);
         });
 
         services.AddSingleton<DocumentAnalysisClient>(sp =>

diff --git a/app/backend/Services/ReadRetrieveReadChatService.cs b/app/backend/Services/ReadRetrieveReadChatService.cs
@@ -4,12 +4,12 @@ namespace MinimalApi.Services;
 
 public class ReadRetrieveReadChatService
 {
-    private readonly IDocumentService _searchClient;
+    private readonly ISearchService _searchClient;
     private readonly IKernel _kernel;
     private readonly IConfiguration _configuration;
 
     public ReadRetrieveReadChatService(
-        IDocumentService searchClient,
+        ISearchService searchClient,
         OpenAIClient client,
         IConfiguration configuration)
     {
@@ -158,6 +158,7 @@ Return the follow-up question as a json string list.
         }
         return new ApproachResponse(
             DataPoints: documentContentList,
+            Images: null,
             Answer: ans,
             Thoughts: thoughts,
             CitationBaseUrl: _configuration.ToCitationBaseUrl());

diff --git a/app/frontend/Services/ApiClient.cs b/app/frontend/Services/ApiClient.cs
@@ -125,6 +125,7 @@ private async Task<AnswerResult<TRequest>> PostRequestAsync<TRequest>(
                 $"HTTP {(int)response.StatusCode} : {response.ReasonPhrase ?? "☹️ Unknown error..."}",
                 null,
                 [],
+                null,
                 "Unable to retrieve valid response from the server.");
 
             return result with

diff --git a/app/functions/EmbedFunctions/Program.cs b/app/functions/EmbedFunctions/Program.cs
@@ -65,7 +65,17 @@ uri is not null
             var documentClient = provider.GetRequiredService<DocumentAnalysisClient>();
             var logger = provider.GetRequiredService<ILogger<AzureSearchEmbedService>>();
 
-            return new AzureSearchEmbedService(openAIClient, embeddingModelName, searchClient, searchIndexName, searchIndexClient, documentClient, blobContainerClient, logger);
+            return new AzureSearchEmbedService(
+                openAIClient: openAIClient,
+                embeddingModelName: embeddingModelName,
+                searchClient: searchClient,
+                searchIndexName: searchIndexName,
+                searchIndexClient: searchIndexClient,
+                documentAnalysisClient: documentClient,
+                corpusContainerClient: blobContainerClient,
+                computerVisionService: null,
+                includeImageEmbeddingsField: false,
+                logger: logger);
         });
     })
     .ConfigureFunctionsWorkerDefaults()

diff --git a/app/functions/EmbedFunctions/Services/EmbeddingAggregateService.cs b/app/functions/EmbedFunctions/Services/EmbeddingAggregateService.cs
@@ -14,7 +14,7 @@ internal async Task EmbedBlobAsync(Stream blobStream, string blobName)
             var embeddingType = GetEmbeddingType();
             var embedService = embedServiceFactory.GetEmbedService(embeddingType);
 
-            var result = await embedService.EmbedBlobAsync(blobStream, blobName);
+            var result = await embedService.EmbedPDFBlobAsync(blobStream, blobName);
 
             var status = result switch
             {

diff --git a/app/functions/EmbedFunctions/Services/IEmbedService.cs b/app/functions/EmbedFunctions/Services/IEmbedService.cs
diff --git a/app/functions/EmbedFunctions/Services/MilvusEmbedService.cs b/app/functions/EmbedFunctions/Services/MilvusEmbedService.cs
@@ -1,8 +1,24 @@
 // Copyright (c) Microsoft. All rights reserved.
 
+
 namespace EmbedFunctions.Services;
 
 internal sealed class MilvusEmbedService : IEmbedService
 {
-    public Task<bool> EmbedBlobAsync(Stream blobStream, string blobName) => throw new NotImplementedException();
+    public Task CreateSearchIndexAsync(string searchIndexName, CancellationToken ct = default)
+    {
+        throw new NotImplementedException();
+    }
+
+    public Task<bool> EmbedImageBlobAsync(Stream imageStream, string imageName, CancellationToken ct = default)
+    {
+        throw new NotImplementedException();
+    }
+
+    public Task<bool> EmbedPDFBlobAsync(Stream blobStream, string blobName) => throw new NotImplementedException();
+
+    public Task EnsureSearchIndexAsync(string searchIndexName, CancellationToken ct = default)
+    {
+        throw new NotImplementedException();
+    }
 }
diff --git a/app/functions/EmbedFunctions/Services/PineconeEmbedService.cs b/app/functions/EmbedFunctions/Services/PineconeEmbedService.cs
@@ -1,9 +1,25 @@
 // Copyright (c) Microsoft. All rights reserved.
 
+
 namespace EmbedFunctions.Services;
 
 internal sealed class PineconeEmbedService : IEmbedService
 {
-    public Task<bool> EmbedBlobAsync(Stream blobStream, string blobName) => throw new NotImplementedException(
+    public Task CreateSearchIndexAsync(string searchIndexName, CancellationToken ct = default)
+    {
+        throw new NotImplementedException();
+    }
+
+    public Task<bool> EmbedImageBlobAsync(Stream imageStream, string imageName, CancellationToken ct = default)
+    {
+        throw new NotImplementedException();
+    }
+
+    public Task<bool> EmbedPDFBlobAsync(Stream blobStream, string blobName) => throw new NotImplementedException(
             "Pinecone embedding isn't implemented.");
+
+    public Task EnsureSearchIndexAsync(string searchIndexName, CancellationToken ct = default)
+    {
+        throw new NotImplementedException();
+    }
 }
diff --git a/app/functions/EmbedFunctions/Services/QdrantEmbedService.cs b/app/functions/EmbedFunctions/Services/QdrantEmbedService.cs
@@ -1,8 +1,24 @@
 // Copyright (c) Microsoft. All rights reserved.
 
+
 namespace EmbedFunctions.Services;
 
 internal sealed class QdrantEmbedService : IEmbedService
 {
-    public Task<bool> EmbedBlobAsync(Stream blobStream, string blobName) => throw new NotImplementedException();
+    public Task CreateSearchIndexAsync(string searchIndexName, CancellationToken ct = default)
+    {
+        throw new NotImplementedException();
+    }
+
+    public Task<bool> EmbedImageBlobAsync(Stream imageStream, string imageName, CancellationToken ct = default)
+    {
+        throw new NotImplementedException();
+    }
+
+    public Task<bool> EmbedPDFBlobAsync(Stream blobStream, string blobName) => throw new NotImplementedException();
+
+    public Task EnsureSearchIndexAsync(string searchIndexName, CancellationToken ct = default)
+    {
+        throw new NotImplementedException();
+    }
 }
diff --git a/app/prepdocs/PrepareDocs/AppOptions.cs b/app/prepdocs/PrepareDocs/AppOptions.cs
@@ -16,6 +16,7 @@ internal record class AppOptions(
     bool Remove,
     bool RemoveAll,
     string? FormRecognizerServiceEndpoint,
+    string? ComputerVisionServiceEndpoint,
     bool Verbose,
     IConsole Console) : AppConsole(Console);
 

diff --git a/app/prepdocs/PrepareDocs/PrepareDocs.csproj b/app/prepdocs/PrepareDocs/PrepareDocs.csproj
@@ -20,7 +20,7 @@
   </ItemGroup>
 
   <ItemGroup>
-    <ProjectReference Include="..\..\functions\EmbedFunctions\EmbedFunctions.csproj" />
+    <ProjectReference Include="..\..\shared\Shared\Shared.csproj" />
   </ItemGroup>
 
 </Project>
diff --git a/app/prepdocs/PrepareDocs/Program.Clients.cs b/app/prepdocs/PrepareDocs/Program.Clients.cs
@@ -1,8 +1,5 @@
 // Copyright (c) Microsoft. All rights reserved.
 
-
-using EmbedFunctions.Services;
-
 internal static partial class Program
 {
     private static BlobContainerClient? s_corpusContainerClient;
@@ -30,8 +27,19 @@ private static Task<AzureSearchEmbedService> GetAzureSearchEmbedService(AppOptio
             var openAIClient = await GetAzureOpenAIClientAsync(o);
             var embeddingModelName = o.EmbeddingModelName ?? throw new ArgumentNullException(nameof(o.EmbeddingModelName));
             var searchIndexName = o.SearchIndexName ?? throw new ArgumentNullException(nameof(o.SearchIndexName));
-
-            return new AzureSearchEmbedService(openAIClient, embeddingModelName, searchClient, searchIndexName, searchIndexClient, documentClient, blobContainerClient, null);
+            var computerVisionService = await GetComputerVisionServiceAsync(o);
+
+            return new AzureSearchEmbedService(
+                openAIClient: openAIClient,
+                embeddingModelName: embeddingModelName,
+                searchClient: searchClient,
+                searchIndexName: searchIndexName,
+                searchIndexClient: searchIndexClient,
+                documentAnalysisClient: documentClient,
+                corpusContainerClient: blobContainerClient,
+                computerVisionService: computerVisionService,
+                includeImageEmbeddingsField: computerVisionService != null,
+                logger: null);
         });
 
     private static Task<BlobContainerClient> GetCorpusBlobContainerClientAsync(AppOptions options) =>
@@ -139,6 +147,20 @@ private static Task<SearchClient> GetSearchClientAsync(AppOptions options) =>
             return s_searchClient;
         });
 
+    private static Task<IComputerVisionService?> GetComputerVisionServiceAsync(AppOptions options) =>
+        GetLazyClientAsync<IComputerVisionService?>(options, s_openAILock, async o =>
+        {
+            await Task.CompletedTask;
+            var endpoint = o.ComputerVisionServiceEndpoint;
+
+            if (string.IsNullOrEmpty(endpoint))
+            {
+                return null;
+            }
+
+            return new AzureComputerVisionService(new HttpClient(), endpoint, DefaultCredential);
+        });
+
     private static Task<OpenAIClient> GetAzureOpenAIClientAsync(AppOptions options) =>
        GetLazyClientAsync<OpenAIClient>(options, s_openAILock, async o =>
        {

diff --git a/app/prepdocs/PrepareDocs/Program.Options.cs b/app/prepdocs/PrepareDocs/Program.Options.cs
@@ -41,6 +41,9 @@ internal static partial class Program
     private static readonly Option<string> s_formRecognizerServiceEndpoint =
         new(name: "--formrecognizerendpoint", description: "Optional. The Azure Form Recognizer service endpoint which will be used to extract text, tables and layout from the documents (must exist already)");
 
+    private static readonly Option<string> s_computerVisionServiceEndpoint =
+        new(name: "--computervisionendpoint", description: "Optional. The Azure Computer Vision service endpoint which will be used to vectorize image and query");
+
     private static readonly Option<bool> s_verbose =
        new(aliases: new[] { "--verbose", "-v" }, description: "Verbose output");
 
@@ -49,11 +52,23 @@ internal static partial class Program
         Prepare documents by extracting content from PDFs, splitting content into sections,
         uploading to blob storage, and indexing in a search index.
         """)
-    {
-        s_files, s_category, s_skipBlobs, s_storageEndpoint,
-        s_container, s_tenantId, s_searchService, s_searchIndexName, s_azureOpenAIService, s_embeddingModelName,
-        s_remove, s_removeAll, s_formRecognizerServiceEndpoint, s_verbose
-    };
+        {
+            s_files,
+            s_category,
+            s_skipBlobs,
+            s_storageEndpoint,
+            s_container,
+            s_tenantId,
+            s_searchService,
+            s_searchIndexName,
+            s_azureOpenAIService,
+            s_embeddingModelName,
+            s_remove,
+            s_removeAll,
+            s_formRecognizerServiceEndpoint,
+            s_computerVisionServiceEndpoint,
+            s_verbose,
+        };
 
     private static AppOptions GetParsedAppOptions(InvocationContext context) => new(
             Files: context.ParseResult.GetValueForArgument(s_files),
@@ -69,6 +84,7 @@ internal static partial class Program
             Remove: context.ParseResult.GetValueForOption(s_remove),
             RemoveAll: context.ParseResult.GetValueForOption(s_removeAll),
             FormRecognizerServiceEndpoint: context.ParseResult.GetValueForOption(s_formRecognizerServiceEndpoint),
+            ComputerVisionServiceEndpoint: context.ParseResult.GetValueForOption(s_computerVisionServiceEndpoint),
             Verbose: context.ParseResult.GetValueForOption(s_verbose),
             Console: context.Console);
 }
diff --git a/app/prepdocs/PrepareDocs/Program.cs b/app/prepdocs/PrepareDocs/Program.cs
@@ -1,6 +1,6 @@
 // Copyright (c) Microsoft. All rights reserved.
 
-using EmbedFunctions.Services;
+using System.Diagnostics;
 
 s_rootCommand.SetHandler(
     async (context) =>
@@ -15,7 +15,6 @@
         {
             var searchIndexName = options.SearchIndexName ?? throw new ArgumentNullException(nameof(options.SearchIndexName));
             var embedService = await GetAzureSearchEmbedService(options);
-
             await embedService.EnsureSearchIndexAsync(options.SearchIndexName);
 
             Matcher matcher = new();
@@ -190,19 +189,26 @@ static async ValueTask UploadBlobsAndCreateIndexAsync(
                 // revert stream position
                 stream.Position = 0;
 
-                await embeddingService.EmbedBlobAsync(stream, documentName);
+                await embeddingService.EmbedPDFBlobAsync(stream, documentName);
             }
             finally
             {
                 File.Delete(tempFileName);
             }
         }
     }
+    // if it's an img (end with .png/.jpg/.jpeg), upload it to blob storage and embed it.
+    else if (Path.GetExtension(fileName).Equals(".png", StringComparison.OrdinalIgnoreCase) ||
+        Path.GetExtension(fileName).Equals(".jpg", StringComparison.OrdinalIgnoreCase) ||
+        Path.GetExtension(fileName).Equals(".jpeg", StringComparison.OrdinalIgnoreCase))
+    {
+        await embeddingService.EmbedImageBlobAsync(File.OpenRead(fileName), fileName);
+    }
     else
     {
         var blobName = BlobNameFromFilePage(fileName);
         await UploadBlobAsync(fileName, blobName, container);
-        await embeddingService.EmbedBlobAsync(File.OpenRead(fileName), blobName);
+        await embeddingService.EmbedPDFBlobAsync(File.OpenRead(fileName), blobName);
     }
 }
 

diff --git a/app/shared/Shared/Models/ApproachResponse.cs b/app/shared/Shared/Models/ApproachResponse.cs
@@ -3,9 +3,13 @@
 namespace Shared.Models;
 
 public record SupportingContentRecord(string Title, string Content);
+
+public record SupportingImageRecord(string Title, string Url);
+
 public record ApproachResponse(
     string Answer,
     string? Thoughts,
-    SupportingContentRecord[] DataPoints, // title, content
+    SupportingContentRecord[]? DataPoints, // title, content
+    SupportingImageRecord[]? Images, // title, url
     string CitationBaseUrl,
     string? Error = null);
diff --git a/app/shared/Shared/Services/AzureComputerVisionService.cs b/app/shared/Shared/Services/AzureComputerVisionService.cs
@@ -5,8 +5,10 @@
 using System.Text.Json;
 using Azure.Core;
 
-public class AzureComputerVisionService(HttpClient client, string endPoint, TokenCredential tokenCredential)
+public class AzureComputerVisionService(HttpClient client, string endPoint, TokenCredential tokenCredential) : IComputerVisionService
 {
+    public int Dimension => 1024;
+
     // add virtual keyword to make it mockable
     public async Task<ImageEmbeddingResponse> VectorizeImageAsync(string imagePathOrUrl, CancellationToken ct = default)
     {
-Original file line number
+Diff line change
@@ Expand Up @@
                     $"HTTP {(int)response.StatusCode} : {response.ReasonPhrase ?? "☹️ Unknown error..."}",
                     null,
                     [],
+                    null,
                     "Unable to retrieve valid response from the server.");
                 return result with
@@ Expand Down @@