Add Postgres support to KM Service (#219)

## Motivation and Context (Why the change? What's the scenario?) Allow to run the service with Postgres. Allow to run the service with connectors developed in separate repositories. ## High level description (Approach, Design) * Change Service dependencies, use nugets instead of using direct references to latest code in main. * Add Postgres dependency to Service project * Extend `dotnet run setup` to allow choosing Postgres option Postgres package: https://www.nuget.org/packages/Microsoft.KernelMemory.Postgres Postgres package src: https://github.com/microsoft/kernel-memory-postgres
microsoft · Dec 24, 2023 · 6c96994 · 6c96994
1 parent 296aee8
commit 6c96994
Show file tree

Hide file tree

Showing 21 changed files with 1,085 additions and 16 deletions.
diff --git a/Directory.Packages.props b/Directory.Packages.props
@@ -10,7 +10,7 @@
     <PackageVersion Include="Azure.Storage.Queues" Version="12.17.1"/>
     <PackageVersion Include="ClosedXML" Version="0.102.1"/>
     <PackageVersion Include="DocumentFormat.OpenXml" Version="2.20.0"/>
-    <PackageVersion Include="HtmlAgilityPack" Version="1.11.55"/>
+    <PackageVersion Include="HtmlAgilityPack" Version="1.11.57"/>
     <PackageVersion Include="LLamaSharp" Version="0.8.1"/>
     <PackageVersion Include="LLamaSharp.Backend.Cpu" Version="0.8.1"/>
     <PackageVersion Include="LLamaSharp.Backend.Cuda12" Version="0.8.1"/>

diff --git a/KernelMemory.sln b/KernelMemory.sln
@@ -183,6 +183,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ServiceFunctionalTests", "s
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ManualTests", "service\tests\ManualTests\ManualTests.csproj", "{F325396A-8320-45CE-9DC9-8A679B2E78A4}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FunctionalTests3P", "service\tests\FunctionalTests3P\FunctionalTests3P.csproj", "{C1F052ED-4667-49D8-AA7D-21F8039EE7C7}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -227,6 +229,7 @@ Global
 		{EDFDA12E-BA10-4A00-BBE7-1C10A0B0F1A6} = {0A43C65C-6007-4BB4-B3FE-8D439FC91841}
 		{BCD0255F-2F35-4B6D-BBF2-2FB43B6F95FA} = {5E7DD43D-B5E7-4827-B57D-447E5B428589}
 		{F325396A-8320-45CE-9DC9-8A679B2E78A4} = {5E7DD43D-B5E7-4827-B57D-447E5B428589}
+		{C1F052ED-4667-49D8-AA7D-21F8039EE7C7} = {5E7DD43D-B5E7-4827-B57D-447E5B428589}
 	EndGlobalSection
 	GlobalSection(ProjectConfigurationPlatforms) = postSolution
 		{8A9FA587-7EBA-4D43-BE47-38D798B1C74C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
@@ -341,5 +344,9 @@ Global
 		{F325396A-8320-45CE-9DC9-8A679B2E78A4}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{F325396A-8320-45CE-9DC9-8A679B2E78A4}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{F325396A-8320-45CE-9DC9-8A679B2E78A4}.Release|Any CPU.Build.0 = Release|Any CPU
+		{C1F052ED-4667-49D8-AA7D-21F8039EE7C7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{C1F052ED-4667-49D8-AA7D-21F8039EE7C7}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{C1F052ED-4667-49D8-AA7D-21F8039EE7C7}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{C1F052ED-4667-49D8-AA7D-21F8039EE7C7}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 EndGlobal
diff --git a/examples/002-dotnet-SemanticKernelPlugin/002-dotnet-SemanticKernelPlugin.csproj b/examples/002-dotnet-SemanticKernelPlugin/002-dotnet-SemanticKernelPlugin.csproj
@@ -13,7 +13,7 @@
 
     <ItemGroup>
         <PackageReference Include="Microsoft.SemanticKernel" Version="1.0.1" />
-        <PackageReference Include="Microsoft.KernelMemory.SemanticKernelPlugin" Version="0.23.231218.1" />
+        <PackageReference Include="Microsoft.KernelMemory.SemanticKernelPlugin" Version="0.23.231224.1" />
     </ItemGroup>
 
     <ItemGroup>

diff --git a/examples/203-dotnet-using-core-nuget/203-dotnet-using-core-nuget.csproj b/examples/203-dotnet-using-core-nuget/203-dotnet-using-core-nuget.csproj
@@ -10,7 +10,7 @@
     </PropertyGroup>
 
     <ItemGroup>
-        <PackageReference Include="Microsoft.KernelMemory.Core" Version="0.23.231218.1" />
+        <PackageReference Include="Microsoft.KernelMemory.Core" Version="0.23.231224.1" />
     </ItemGroup>
 
     <ItemGroup>

diff --git a/service/Service/Service.csproj b/service/Service/Service.csproj
@@ -4,16 +4,60 @@
         <TargetFramework>net7.0</TargetFramework>
         <AssemblyName>Microsoft.KernelMemory.ServiceAssembly</AssemblyName>
         <RootNamespace>Microsoft.KernelMemory.Service</RootNamespace>
+        <ManagePackageVersionsCentrally>false</ManagePackageVersionsCentrally>
         <UserSecretsId>5ee045b0-aea3-4f08-8d31-32d1a6f8fed0</UserSecretsId>
         <NoWarn>$(NoWarn);CA2007,CA1724,CA2254,CA1031,CS1591,CA1861,CA1303,SKEXP0001</NoWarn>
     </PropertyGroup>
 
+    <!--
+    This dependency is needed only to generate appsetting.development.json when testing
+    the code in GitHub for quick demo, e.g. running
+
+    "dotnet run setup"
+
+
+    You can safely remove this reference, removing also this code from Program.cs:
+
+        if (new[] { "setup", "-setup" }.Contains(args.FirstOrDefault(), StringComparer.OrdinalIgnoreCase))
+        {
+            Main.InteractiveSetup(cfgService: true);
+        }
+    -->
     <ItemGroup>
-        <ProjectReference Include="..\Abstractions\Abstractions.csproj"/>
-        <ProjectReference Include="..\Core\Core.csproj"/>
         <ProjectReference Include="..\..\tools\InteractiveSetup\InteractiveSetup.csproj"/>
     </ItemGroup>
 
-    <Import Project="../code-analysis.props"/>
+    <ItemGroup>
+        <PackageReference Include="Microsoft.KernelMemory.Core" Version="0.23.231224.1" />
+        <PackageReference Include="Microsoft.KernelMemory.Postgres" Version="0.3.231224.1" />
+    </ItemGroup>
+
+    <!-- Code Analysis -->
+    <ItemGroup>
+        <PackageReference Include="Microsoft.CodeAnalysis.Analyzers" Version="3.3.4">
+            <PrivateAssets>all</PrivateAssets>
+            <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+        </PackageReference>
+        <PackageReference Include="Microsoft.CodeAnalysis.CSharp.CodeStyle" Version="4.8.0">
+            <PrivateAssets>all</PrivateAssets>
+            <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+        </PackageReference>
+        <PackageReference Include="Microsoft.CodeAnalysis.NetAnalyzers" Version="8.0.0">
+            <PrivateAssets>all</PrivateAssets>
+            <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+        </PackageReference>
+        <PackageReference Include="Microsoft.VisualStudio.Threading.Analyzers" Version="17.8.14">
+            <PrivateAssets>all</PrivateAssets>
+            <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+        </PackageReference>
+        <PackageReference Include="Roslynator.CodeAnalysis.Analyzers" Version="4.7.0">
+            <PrivateAssets>all</PrivateAssets>
+            <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+        </PackageReference>
+        <PackageReference Include="Roslynator.Formatting.Analyzers" Version="4.7.0">
+            <PrivateAssets>all</PrivateAssets>
+            <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+        </PackageReference>
+    </ItemGroup>
 
 </Project>
diff --git a/service/Service/ServiceConfiguration.cs b/service/Service/ServiceConfiguration.cs
@@ -17,6 +17,7 @@
 using Microsoft.KernelMemory.Pipeline.Queue.AzureQueues;
 using Microsoft.KernelMemory.Pipeline.Queue.DevTools;
 using Microsoft.KernelMemory.Pipeline.Queue.RabbitMq;
+using Microsoft.KernelMemory.Postgres;
 
 namespace Microsoft.KernelMemory.Service;
 
@@ -277,6 +278,16 @@ private void ConfigureIngestionMemoryDb(IKernelMemoryBuilder builder)
         {
             switch (type)
             {
+                default:
+                    throw new ConfigurationException(
+                        $"Unknown Memory DB option '{type}'. " +
+                        "To use a custom Memory DB, set the configuration value to an empty string, " +
+                        "and inject the custom implementation using `IKernelMemoryBuilder.WithCustomMemoryDb(...)`");
+
+                case "":
+                    // NOOP - allow custom implementations, via WithCustomMemoryDb()
+                    break;
+
                 case string x when x.Equals("AzureAISearch", StringComparison.OrdinalIgnoreCase):
                 {
                     var instance = this.GetServiceInstance<IMemoryDb>(builder,
@@ -295,6 +306,15 @@ private void ConfigureIngestionMemoryDb(IKernelMemoryBuilder builder)
                     break;
                 }
 
+                case string x when x.Equals("Postgres", StringComparison.OrdinalIgnoreCase):
+                {
+                    var instance = this.GetServiceInstance<IMemoryDb>(builder,
+                        s => s.AddPostgresAsMemoryDb(this.GetServiceConfig<PostgresConfig>("Postgres"))
+                    );
+                    builder.AddIngestionMemoryDb(instance);
+                    break;
+                }
+
                 case string x when x.Equals("SimpleVectorDb", StringComparison.OrdinalIgnoreCase):
                 {
                     var instance = this.GetServiceInstance<IMemoryDb>(builder,
@@ -312,10 +332,6 @@ private void ConfigureIngestionMemoryDb(IKernelMemoryBuilder builder)
                     builder.AddIngestionMemoryDb(instance);
                     break;
                 }
-
-                default:
-                    // NOOP - allow custom implementations, via WithCustomMemoryDb()
-                    break;
             }
         }
     }
@@ -359,6 +375,10 @@ private void ConfigureRetrievalMemoryDb(IKernelMemoryBuilder builder)
                 builder.Services.AddQdrantAsMemoryDb(this.GetServiceConfig<QdrantConfig>("Qdrant"));
                 break;
 
+            case string x when x.Equals("Postgres", StringComparison.OrdinalIgnoreCase):
+                builder.Services.AddPostgresAsMemoryDb(this.GetServiceConfig<PostgresConfig>("Postgres"));
+                break;
+
             case string x when x.Equals("SimpleVectorDb", StringComparison.OrdinalIgnoreCase):
                 builder.Services.AddSimpleVectorDbAsMemoryDb(this.GetServiceConfig<SimpleVectorDbConfig>("SimpleVectorDb"));
                 break;

diff --git a/service/Service/appsettings.json b/service/Service/appsettings.json
@@ -57,7 +57,7 @@
       "EmbeddingGeneratorTypes": [
       ],
       // Vectors can be written to multiple storages, e.g. for data migration, A/B testing, etc.
-      // "AzureAISearch", "Qdrant", "SimpleVectorDb"
+      // "AzureAISearch", "Qdrant", "Postgres", "SimpleVectorDb"
       "MemoryDbTypes": [
         "AzureAISearch"
       ],
@@ -78,7 +78,7 @@
       // "AzureOpenAIEmbedding" or "OpenAI"
       // This is the generator registered for `ITextEmbeddingGeneration` dependency injection.
       "EmbeddingGeneratorType": "",
-      // "AzureAISearch", "Qdrant", "SimpleVectorDb"
+      // "AzureAISearch", "Qdrant", "Postgres", "SimpleVectorDb"
       "MemoryDbType": "AzureAISearch",
       // Search client settings
       "SearchClient": {
@@ -168,6 +168,13 @@
         "Endpoint": "http://127.0.0.1:6333",
         "APIKey": "",
       },
+      "Postgres": {
+        // Postgres instance connection string
+        "ConnectionString": "Host=localhost;Port=5432;Username=user;Password=",
+        // Mandatory prefix to add to the name of table managed by KM,
+        // e.g. to exclude other tables in the same schema.
+        "TableNamePrefix": "km-"
+      },
       "AzureOpenAIText": {
         // "ApiKey" or "AzureIdentity"
         // AzureIdentity: use automatic AAD authentication mechanism. You can test locally

diff --git a/service/tests/FunctionalTests/TestHelpers/BaseTestCase.cs b/service/tests/FunctionalTests/TestHelpers/BaseTestCase.cs
@@ -23,6 +23,7 @@ public abstract class BaseTestCase : IDisposable
     protected IConfiguration QdrantConfiguration => this.ServiceConfiguration.GetSection("Qdrant");
     protected IConfiguration AzureAISearchConfiguration => this.ServiceConfiguration.GetSection("AzureAISearch");
 
+    // IMPORTANT: install Xunit.DependencyInjection package
     protected BaseTestCase(IConfiguration cfg, ITestOutputHelper output)
     {
         this._cfg = cfg;