Self upgrade old db records (#216)

Introduce schema version on DB records, to handle breaking changes. Add logic for in-memory upgrade, to support old record schemas. Fixes #215 Fixes #212 Upgrade examples to latest nugets.
microsoft · Dec 20, 2023 · 46ff3ad · 46ff3ad
1 parent fa1d2ef
commit 46ff3ad
Show file tree

Hide file tree

Showing 47 changed files with 978 additions and 270 deletions.
diff --git a/.editorconfig b/.editorconfig
@@ -17,9 +17,16 @@ indent_size = 2
 indent_size = 2
 
 # XML config files
-[*.{props,csproj}]
+[*.{csproj}]
 indent_size = 4
 
+# XML config files
+[*.props]
+indent_size = 4
+
+[Directory.Packages.props]
+indent_size = 2
+
 # YAML config files
 [*.{yml,yaml}]
 tab_width = 2

diff --git a/Directory.Packages.props b/Directory.Packages.props
@@ -1,63 +1,66 @@
 <Project>
-    <PropertyGroup>
-        <ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
-    </PropertyGroup>
-    <ItemGroup>
-        <PackageVersion Include="Azure.AI.FormRecognizer" Version="4.1.0"/>
-        <PackageVersion Include="Azure.Identity" Version="1.10.4"/>
-        <PackageVersion Include="Azure.Search.Documents" Version="11.5.1"/>
-        <PackageVersion Include="Azure.Storage.Blobs" Version="12.19.1"/>
-        <PackageVersion Include="Azure.Storage.Queues" Version="12.17.1"/>
-        <PackageVersion Include="ClosedXML" Version="0.102.1"/>
-        <PackageVersion Include="DocumentFormat.OpenXml" Version="2.20.0"/>
-        <PackageVersion Include="HtmlAgilityPack" Version="1.11.54"/>
-        <PackageVersion Include="LLamaSharp" Version="0.8.1"/>
-        <PackageVersion Include="LLamaSharp.Backend.Cpu" Version="0.8.1"/>
-        <PackageVersion Include="LLamaSharp.Backend.Cuda12" Version="0.8.1"/>
-        <PackageVersion Include="Microsoft.Bcl.AsyncInterfaces" Version="8.0.0"/>
-        <PackageVersion Include="Microsoft.Extensions.Configuration" Version="8.0.0"/>
-        <PackageVersion Include="Microsoft.Extensions.Configuration.Json" Version="8.0.0"/>
-        <PackageVersion Include="Microsoft.Extensions.DependencyInjection" Version="8.0.0"/>
-        <PackageVersion Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="8.0.0"/>
-        <PackageVersion Include="Microsoft.Extensions.Hosting" Version="8.0.0"/>
-        <PackageVersion Include="Microsoft.Extensions.Logging" Version="8.0.0"/>
-        <PackageVersion Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.0"/>
-        <PackageVersion Include="Microsoft.Extensions.Logging.TraceSource" Version="8.0.0"/>
-        <PackageVersion Include="PdfPig" Version="0.1.8"/>
-        <PackageVersion Include="Polly.Core" Version="8.2.0"/>
-        <PackageVersion Include="RabbitMQ.Client" Version="6.7.0"/>
-        <PackageVersion Include="Swashbuckle.AspNetCore" Version="6.5.0"/>
-        <PackageVersion Include="System.Linq.Async" Version="6.0.1"/>
-        <PackageVersion Include="System.Memory.Data" Version="8.0.0"/>
-        <PackageVersion Include="System.Numerics.Tensors" Version="8.0.0"/>
-        <PackageVersion Include="System.Text.Json" Version="8.0.0"/>
-    </ItemGroup>
-    <!-- Semantic Kernel -->
-    <ItemGroup>
-        <PackageVersion Include="Microsoft.SemanticKernel.Abstractions" Version="1.0.1"/>
-        <PackageVersion Include="Microsoft.SemanticKernel.Connectors.OpenAI" Version="1.0.1"/>
-    </ItemGroup>
-    <!-- Documentation -->
-    <ItemGroup>
-        <PackageVersion Include="Microsoft.SourceLink.GitHub" Version="8.0.0"/>
-    </ItemGroup>
-    <!-- Code Analysis -->
-    <ItemGroup>
-        <PackageVersion Include="Microsoft.CodeAnalysis.Analyzers" Version="3.3.4"/>
-        <PackageVersion Include="Microsoft.CodeAnalysis.CSharp.CodeStyle" Version="4.8.0"/>
-        <PackageVersion Include="Microsoft.CodeAnalysis.NetAnalyzers" Version="8.0.0"/>
-        <PackageVersion Include="Microsoft.VisualStudio.Threading.Analyzers" Version="17.8.14"/>
-        <PackageVersion Include="Roslynator.CodeAnalysis.Analyzers" Version="4.7.0"/>
-        <PackageVersion Include="Roslynator.Formatting.Analyzers" Version="4.7.0"/>
-    </ItemGroup>
-    <!-- Tests -->
-    <ItemGroup>
-        <PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.8.0"/>
-        <PackageVersion Include="Xunit.DependencyInjection" Version="8.9.0"/>
-        <PackageVersion Include="Xunit.DependencyInjection.Logging" Version="8.1.0"/>
-        <PackageVersion Include="coverlet.collector" Version="6.0.0"/>
-        <PackageVersion Include="xunit" Version="2.6.2"/>
-        <PackageVersion Include="xunit.abstractions" Version="2.0.3"/>
-        <PackageVersion Include="xunit.runner.visualstudio" Version="2.5.4"/>
-    </ItemGroup>
+  <PropertyGroup>
+    <ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
+  </PropertyGroup>
+  <ItemGroup>
+    <PackageVersion Include="Azure.AI.FormRecognizer" Version="4.1.0"/>
+    <PackageVersion Include="Azure.Identity" Version="1.10.4"/>
+    <PackageVersion Include="Azure.Search.Documents" Version="11.5.1"/>
+    <PackageVersion Include="Azure.Storage.Blobs" Version="12.19.1"/>
+    <PackageVersion Include="Azure.Storage.Queues" Version="12.17.1"/>
+    <PackageVersion Include="ClosedXML" Version="0.102.1"/>
+    <PackageVersion Include="DocumentFormat.OpenXml" Version="2.20.0"/>
+    <PackageVersion Include="HtmlAgilityPack" Version="1.11.55"/>
+    <PackageVersion Include="LLamaSharp" Version="0.8.1"/>
+    <PackageVersion Include="LLamaSharp.Backend.Cpu" Version="0.8.1"/>
+    <PackageVersion Include="LLamaSharp.Backend.Cuda12" Version="0.8.1"/>
+    <PackageVersion Include="Microsoft.Bcl.AsyncInterfaces" Version="8.0.0"/>
+    <PackageVersion Include="Microsoft.Extensions.Configuration" Version="8.0.0"/>
+    <PackageVersion Include="Microsoft.Extensions.Configuration.Json" Version="8.0.0"/>
+    <PackageVersion Include="Microsoft.Extensions.DependencyInjection" Version="8.0.0"/>
+    <PackageVersion Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="8.0.0"/>
+    <PackageVersion Include="Microsoft.Extensions.Hosting" Version="8.0.0"/>
+    <PackageVersion Include="Microsoft.Extensions.Logging" Version="8.0.0"/>
+    <PackageVersion Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.0"/>
+    <PackageVersion Include="Microsoft.Extensions.Logging.TraceSource" Version="8.0.0"/>
+    <PackageVersion Include="PdfPig" Version="0.1.8"/>
+    <PackageVersion Include="Polly.Core" Version="8.2.0"/>
+    <PackageVersion Include="RabbitMQ.Client" Version="6.8.1"/>
+    <PackageVersion Include="Swashbuckle.AspNetCore" Version="6.5.0"/>
+    <PackageVersion Include="System.Linq.Async" Version="6.0.1"/>
+    <PackageVersion Include="System.Memory.Data" Version="8.0.0"/>
+    <PackageVersion Include="System.Numerics.Tensors" Version="8.0.0"/>
+    <PackageVersion Include="System.Text.Json" Version="8.0.0"/>
+  </ItemGroup>
+  <!-- Semantic Kernel -->
+  <ItemGroup>
+    <PackageVersion Include="Microsoft.SemanticKernel.Abstractions" Version="1.0.1"/>
+    <PackageVersion Include="Microsoft.SemanticKernel.Connectors.OpenAI" Version="1.0.1"/>
+  </ItemGroup>
+  <!-- Documentation -->
+  <ItemGroup>
+    <PackageVersion Include="Microsoft.SourceLink.GitHub" Version="8.0.0"/>
+  </ItemGroup>
+  <!-- Code Analysis -->
+  <ItemGroup>
+    <PackageVersion Include="Microsoft.CodeAnalysis.Analyzers" Version="3.3.4"/>
+    <PackageVersion Include="Microsoft.CodeAnalysis.CSharp.CodeStyle" Version="4.8.0"/>
+    <PackageVersion Include="Microsoft.CodeAnalysis.NetAnalyzers" Version="8.0.0"/>
+    <PackageVersion Include="Microsoft.VisualStudio.Threading.Analyzers" Version="17.8.14"/>
+    <PackageVersion Include="Roslynator.CodeAnalysis.Analyzers" Version="4.7.0"/>
+    <PackageVersion Include="Roslynator.Formatting.Analyzers" Version="4.7.0"/>
+  </ItemGroup>
+  <!-- Tests -->
+  <ItemGroup>
+    <PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.8.0"/>
+    <PackageVersion Include="Xunit.DependencyInjection" Version="8.9.1"/>
+    <PackageVersion Include="Xunit.DependencyInjection.Logging" Version="8.1.0"/>
+    <PackageVersion Include="coverlet.collector" Version="6.0.0"/>
+    <PackageVersion Include="xunit" Version="2.6.3"/>
+    <PackageVersion Include="xunit.abstractions" Version="2.0.3"/>
+    <PackageVersion Include="xunit.runner.visualstudio" Version="2.5.5">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageVersion>
+  </ItemGroup>
 </Project>
diff --git a/KernelMemory.sln b/KernelMemory.sln
@@ -67,10 +67,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "UnitTests", "service\tests\
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FunctionalTests", "service\tests\FunctionalTests\FunctionalTests.csproj", "{3AAD973E-2FDC-4C89-94BA-4F6671C2F23A}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "azure-ai-search-tests", "service\tests\azure-ai-search-tests\azure-ai-search-tests.csproj", "{4A56D84B-1D8B-447B-9D9A-E36A2AE44806}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "qdrant-tests", "service\tests\qdrant-tests\qdrant-tests.csproj", "{8A90555B-C28B-401B-960A-619DE9A87C4C}"
-EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "InteractiveSetup", "tools\InteractiveSetup\InteractiveSetup.csproj", "{C03BC9FE-5301-43EE-B912-1363F430DBA0}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SemanticKernelPlugin", "clients\dotnet\SemanticKernelPlugin\SemanticKernelPlugin.csproj", "{F7609330-E97E-422C-8983-EC501B2DDC52}"
@@ -183,6 +179,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "105-dotnet-serverless-llama
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "106-dotnet-retrieve-synthetics", "examples\106-dotnet-retrieve-synthetics\106-dotnet-retrieve-synthetics.csproj", "{EDFDA12E-BA10-4A00-BBE7-1C10A0B0F1A6}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ServiceFunctionalTests", "service\tests\ServiceFunctionalTests\ServiceFunctionalTests.csproj", "{BCD0255F-2F35-4B6D-BBF2-2FB43B6F95FA}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ManualTests", "service\tests\ManualTests\ManualTests.csproj", "{F325396A-8320-45CE-9DC9-8A679B2E78A4}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -201,8 +201,6 @@ Global
 		{C95AECD6-8EF0-48FB-9BB4-317059748CF5} = {9DD64A4A-FB76-4DF8-8CFD-4F3A49BE065B}
 		{3AAD973E-2FDC-4C89-94BA-4F6671C2F23A} = {5E7DD43D-B5E7-4827-B57D-447E5B428589}
 		{4AE9DB51-2A22-4FB3-A780-838738566D18} = {5E7DD43D-B5E7-4827-B57D-447E5B428589}
-		{4A56D84B-1D8B-447B-9D9A-E36A2AE44806} = {5E7DD43D-B5E7-4827-B57D-447E5B428589}
-		{8A90555B-C28B-401B-960A-619DE9A87C4C} = {5E7DD43D-B5E7-4827-B57D-447E5B428589}
 		{C03BC9FE-5301-43EE-B912-1363F430DBA0} = {CA49F1A1-C3FA-4E99-ACB3-D7FF33D47976}
 		{F7609330-E97E-422C-8983-EC501B2DDC52} = {371BB479-AA1C-41CB-BF07-24C363601289}
 		{D04A01C0-EF1B-49B2-B6AB-8AC635566E6A} = {371BB479-AA1C-41CB-BF07-24C363601289}
@@ -227,6 +225,8 @@ Global
 		{9202380E-D793-4C3C-89B3-3BE790925029} = {0A43C65C-6007-4BB4-B3FE-8D439FC91841}
 		{E762B1C6-4B5B-487A-BF8C-C6870D6EACB7} = {0A43C65C-6007-4BB4-B3FE-8D439FC91841}
 		{EDFDA12E-BA10-4A00-BBE7-1C10A0B0F1A6} = {0A43C65C-6007-4BB4-B3FE-8D439FC91841}
+		{BCD0255F-2F35-4B6D-BBF2-2FB43B6F95FA} = {5E7DD43D-B5E7-4827-B57D-447E5B428589}
+		{F325396A-8320-45CE-9DC9-8A679B2E78A4} = {5E7DD43D-B5E7-4827-B57D-447E5B428589}
 	EndGlobalSection
 	GlobalSection(ProjectConfigurationPlatforms) = postSolution
 		{8A9FA587-7EBA-4D43-BE47-38D798B1C74C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
@@ -241,14 +241,6 @@ Global
 		{1071A8B6-ED76-4E46-A291-E0563B9C4575}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{1071A8B6-ED76-4E46-A291-E0563B9C4575}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{1071A8B6-ED76-4E46-A291-E0563B9C4575}.Release|Any CPU.Build.0 = Release|Any CPU
-		{4A56D84B-1D8B-447B-9D9A-E36A2AE44806}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{4A56D84B-1D8B-447B-9D9A-E36A2AE44806}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{4A56D84B-1D8B-447B-9D9A-E36A2AE44806}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{4A56D84B-1D8B-447B-9D9A-E36A2AE44806}.Release|Any CPU.Build.0 = Release|Any CPU
-		{8A90555B-C28B-401B-960A-619DE9A87C4C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{8A90555B-C28B-401B-960A-619DE9A87C4C}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{8A90555B-C28B-401B-960A-619DE9A87C4C}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{8A90555B-C28B-401B-960A-619DE9A87C4C}.Release|Any CPU.Build.0 = Release|Any CPU
 		{67F37386-9D93-4591-98A1-32FD4C3AD8BA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{67F37386-9D93-4591-98A1-32FD4C3AD8BA}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{67F37386-9D93-4591-98A1-32FD4C3AD8BA}.Release|Any CPU.ActiveCfg = Release|Any CPU
@@ -341,5 +333,13 @@ Global
 		{EDFDA12E-BA10-4A00-BBE7-1C10A0B0F1A6}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{EDFDA12E-BA10-4A00-BBE7-1C10A0B0F1A6}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{EDFDA12E-BA10-4A00-BBE7-1C10A0B0F1A6}.Release|Any CPU.Build.0 = Release|Any CPU
+		{BCD0255F-2F35-4B6D-BBF2-2FB43B6F95FA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{BCD0255F-2F35-4B6D-BBF2-2FB43B6F95FA}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{BCD0255F-2F35-4B6D-BBF2-2FB43B6F95FA}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{BCD0255F-2F35-4B6D-BBF2-2FB43B6F95FA}.Release|Any CPU.Build.0 = Release|Any CPU
+		{F325396A-8320-45CE-9DC9-8A679B2E78A4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{F325396A-8320-45CE-9DC9-8A679B2E78A4}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{F325396A-8320-45CE-9DC9-8A679B2E78A4}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{F325396A-8320-45CE-9DC9-8A679B2E78A4}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 EndGlobal
diff --git a/examples/000-notebooks/001-upload-and-ask.ipynb b/examples/000-notebooks/001-upload-and-ask.ipynb
@@ -26,15 +26,15 @@
     {
      "data": {
       "text/html": [
-       "<div><div></div><div></div><div><strong>Installed Packages</strong><ul><li><span>Microsoft.KernelMemory.Core, 0.20.231211.3</span></li></ul></div></div>"
+       "<div><div></div><div></div><div><strong>Installed Packages</strong><ul><li><span>Microsoft.KernelMemory.Core, 0.23.231218.1</span></li></ul></div></div>"
       ]
      },
      "metadata": {},
      "output_type": "display_data"
     }
    ],
    "source": [
-    "#r \"nuget: Microsoft.KernelMemory.Core, 0.20.231211.3\"\n",
+    "#r \"nuget: Microsoft.KernelMemory.Core, 0.23.231218.1\"\n",
     "\n",
     "using Microsoft.KernelMemory;"
    ]
@@ -89,6 +89,11 @@
     "In this demo we use OpenAI to calculate embeddings and generate text, and the default storage settings, with content and embeddings kept in a volatile memory automatically deleted after the execution."
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": []
+  },
   {
    "cell_type": "code",
    "execution_count": 3,
@@ -118,7 +123,9 @@
       "info: Microsoft.KernelMemory.Handlers.DeleteDocumentHandler[0]\n",
       "      Handler 'private_delete_document' ready\n",
       "info: Microsoft.KernelMemory.Handlers.DeleteIndexHandler[0]\n",
-      "      Handler 'private_delete_index' ready\n"
+      "      Handler 'private_delete_index' ready\n",
+      "info: Microsoft.KernelMemory.Handlers.DeleteGeneratedFilesHandler[0]\n",
+      "      Handler 'delete_generated_files' ready\n"
      ]
     }
    ],
@@ -178,7 +185,7 @@
      "text": [
       "Question: What's Semantic Kernel?\n",
       "\n",
-      "Answer: Semantic Kernel is a lightweight SDK (Software Development Kit) developed by Microsoft. It enables the integration of AI Large Language Models (LLMs) with conventional programming languages. The SDK combines natural language semantic functions, traditional code native functions, and embeddings-based memory to add value and unlock new potential in applications with AI. Semantic Kernel supports prompt templating, function chaining, vectorized memory, and intelligent planning capabilities. It encapsulates several design patterns from the latest AI research, allowing developers to infuse their applications with features like prompt chaining, recursive reasoning, summarization, zero/few-shot learning, contextual memory, long-term memory, embeddings, semantic indexing, planning, retrieval-augmented generation, and accessing external knowledge stores. Semantic Kernel is open-source and aims to enable developers to build AI-first apps faster.\n"
+      "Answer: Semantic Kernel is a lightweight SDK (Software Development Kit) developed by Microsoft. It enables the integration of AI Large Language Models (LLMs) with conventional programming languages. The SDK combines natural language semantic functions, traditional code native functions, and embeddings-based memory to add value and unlock new potential in applications with AI. Semantic Kernel supports prompt templating, function chaining, vectorized memory, and intelligent planning capabilities. It encapsulates several design patterns from the latest AI research, allowing developers to infuse their applications with features like prompt chaining, recursive reasoning, summarization, zero/few-shot learning, contextual memory, long-term memory, embeddings, semantic indexing, planning, retrieval-augmented generation, and accessing external knowledge stores. Semantic Kernel is available for use with C# and Python programming languages. It is an open-source project, and developers are invited to contribute to its development through GitHub discussions, opening issues, sending pull requests, and joining the Discord community.\n"
      ]
     }
    ],
@@ -215,7 +222,7 @@
      "text": [
       "Sources:\n",
       "\n",
-      "  - sample-SK-Readme.pdf  - doc001/ecb6d46eb039411193996b6cc802b11e [Friday, December 8, 2023]\n"
+      "  - sample-SK-Readme.pdf  - default/doc001/a6c00dad170140178e805c9a0c267a7f [Tuesday, December 19, 2023]\n"
      ]
     }
    ],