diff --git a/.editorconfig b/.editorconfig index 326491041..84c1efc1c 100644 --- a/.editorconfig +++ b/.editorconfig @@ -17,9 +17,16 @@ indent_size = 2 indent_size = 2 # XML config files -[*.{props,csproj}] +[*.{csproj}] indent_size = 4 +# XML config files +[*.props] +indent_size = 4 + +[Directory.Packages.props] +indent_size = 2 + # YAML config files [*.{yml,yaml}] tab_width = 2 diff --git a/Directory.Packages.props b/Directory.Packages.props index 44997eb42..9a4c785f3 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -1,63 +1,66 @@ - - true - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + \ No newline at end of file diff --git a/KernelMemory.sln b/KernelMemory.sln index e4384051b..be1c3058d 100644 --- a/KernelMemory.sln +++ b/KernelMemory.sln @@ -67,10 +67,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "UnitTests", "service\tests\ EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FunctionalTests", "service\tests\FunctionalTests\FunctionalTests.csproj", "{3AAD973E-2FDC-4C89-94BA-4F6671C2F23A}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "azure-ai-search-tests", "service\tests\azure-ai-search-tests\azure-ai-search-tests.csproj", "{4A56D84B-1D8B-447B-9D9A-E36A2AE44806}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "qdrant-tests", "service\tests\qdrant-tests\qdrant-tests.csproj", "{8A90555B-C28B-401B-960A-619DE9A87C4C}" -EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "InteractiveSetup", "tools\InteractiveSetup\InteractiveSetup.csproj", "{C03BC9FE-5301-43EE-B912-1363F430DBA0}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SemanticKernelPlugin", "clients\dotnet\SemanticKernelPlugin\SemanticKernelPlugin.csproj", "{F7609330-E97E-422C-8983-EC501B2DDC52}" @@ -183,6 +179,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "105-dotnet-serverless-llama EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "106-dotnet-retrieve-synthetics", "examples\106-dotnet-retrieve-synthetics\106-dotnet-retrieve-synthetics.csproj", "{EDFDA12E-BA10-4A00-BBE7-1C10A0B0F1A6}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ServiceFunctionalTests", "service\tests\ServiceFunctionalTests\ServiceFunctionalTests.csproj", "{BCD0255F-2F35-4B6D-BBF2-2FB43B6F95FA}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ManualTests", "service\tests\ManualTests\ManualTests.csproj", "{F325396A-8320-45CE-9DC9-8A679B2E78A4}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -201,8 +201,6 @@ Global {C95AECD6-8EF0-48FB-9BB4-317059748CF5} = {9DD64A4A-FB76-4DF8-8CFD-4F3A49BE065B} {3AAD973E-2FDC-4C89-94BA-4F6671C2F23A} = {5E7DD43D-B5E7-4827-B57D-447E5B428589} {4AE9DB51-2A22-4FB3-A780-838738566D18} = {5E7DD43D-B5E7-4827-B57D-447E5B428589} - {4A56D84B-1D8B-447B-9D9A-E36A2AE44806} = {5E7DD43D-B5E7-4827-B57D-447E5B428589} - {8A90555B-C28B-401B-960A-619DE9A87C4C} = {5E7DD43D-B5E7-4827-B57D-447E5B428589} {C03BC9FE-5301-43EE-B912-1363F430DBA0} = {CA49F1A1-C3FA-4E99-ACB3-D7FF33D47976} {F7609330-E97E-422C-8983-EC501B2DDC52} = {371BB479-AA1C-41CB-BF07-24C363601289} {D04A01C0-EF1B-49B2-B6AB-8AC635566E6A} = {371BB479-AA1C-41CB-BF07-24C363601289} @@ -227,6 +225,8 @@ Global {9202380E-D793-4C3C-89B3-3BE790925029} = {0A43C65C-6007-4BB4-B3FE-8D439FC91841} {E762B1C6-4B5B-487A-BF8C-C6870D6EACB7} = {0A43C65C-6007-4BB4-B3FE-8D439FC91841} {EDFDA12E-BA10-4A00-BBE7-1C10A0B0F1A6} = {0A43C65C-6007-4BB4-B3FE-8D439FC91841} + {BCD0255F-2F35-4B6D-BBF2-2FB43B6F95FA} = {5E7DD43D-B5E7-4827-B57D-447E5B428589} + {F325396A-8320-45CE-9DC9-8A679B2E78A4} = {5E7DD43D-B5E7-4827-B57D-447E5B428589} EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution {8A9FA587-7EBA-4D43-BE47-38D798B1C74C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU @@ -241,14 +241,6 @@ Global {1071A8B6-ED76-4E46-A291-E0563B9C4575}.Debug|Any CPU.Build.0 = Debug|Any CPU {1071A8B6-ED76-4E46-A291-E0563B9C4575}.Release|Any CPU.ActiveCfg = Release|Any CPU {1071A8B6-ED76-4E46-A291-E0563B9C4575}.Release|Any CPU.Build.0 = Release|Any CPU - {4A56D84B-1D8B-447B-9D9A-E36A2AE44806}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {4A56D84B-1D8B-447B-9D9A-E36A2AE44806}.Debug|Any CPU.Build.0 = Debug|Any CPU - {4A56D84B-1D8B-447B-9D9A-E36A2AE44806}.Release|Any CPU.ActiveCfg = Release|Any CPU - {4A56D84B-1D8B-447B-9D9A-E36A2AE44806}.Release|Any CPU.Build.0 = Release|Any CPU - {8A90555B-C28B-401B-960A-619DE9A87C4C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {8A90555B-C28B-401B-960A-619DE9A87C4C}.Debug|Any CPU.Build.0 = Debug|Any CPU - {8A90555B-C28B-401B-960A-619DE9A87C4C}.Release|Any CPU.ActiveCfg = Release|Any CPU - {8A90555B-C28B-401B-960A-619DE9A87C4C}.Release|Any CPU.Build.0 = Release|Any CPU {67F37386-9D93-4591-98A1-32FD4C3AD8BA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {67F37386-9D93-4591-98A1-32FD4C3AD8BA}.Debug|Any CPU.Build.0 = Debug|Any CPU {67F37386-9D93-4591-98A1-32FD4C3AD8BA}.Release|Any CPU.ActiveCfg = Release|Any CPU @@ -341,5 +333,13 @@ Global {EDFDA12E-BA10-4A00-BBE7-1C10A0B0F1A6}.Debug|Any CPU.Build.0 = Debug|Any CPU {EDFDA12E-BA10-4A00-BBE7-1C10A0B0F1A6}.Release|Any CPU.ActiveCfg = Release|Any CPU {EDFDA12E-BA10-4A00-BBE7-1C10A0B0F1A6}.Release|Any CPU.Build.0 = Release|Any CPU + {BCD0255F-2F35-4B6D-BBF2-2FB43B6F95FA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {BCD0255F-2F35-4B6D-BBF2-2FB43B6F95FA}.Debug|Any CPU.Build.0 = Debug|Any CPU + {BCD0255F-2F35-4B6D-BBF2-2FB43B6F95FA}.Release|Any CPU.ActiveCfg = Release|Any CPU + {BCD0255F-2F35-4B6D-BBF2-2FB43B6F95FA}.Release|Any CPU.Build.0 = Release|Any CPU + {F325396A-8320-45CE-9DC9-8A679B2E78A4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F325396A-8320-45CE-9DC9-8A679B2E78A4}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F325396A-8320-45CE-9DC9-8A679B2E78A4}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F325396A-8320-45CE-9DC9-8A679B2E78A4}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection EndGlobal diff --git a/examples/000-notebooks/001-upload-and-ask.ipynb b/examples/000-notebooks/001-upload-and-ask.ipynb index c652c539f..89ffac53e 100644 --- a/examples/000-notebooks/001-upload-and-ask.ipynb +++ b/examples/000-notebooks/001-upload-and-ask.ipynb @@ -26,7 +26,7 @@ { "data": { "text/html": [ - "
Installed Packages
" + "
Installed Packages
" ] }, "metadata": {}, @@ -34,7 +34,7 @@ } ], "source": [ - "#r \"nuget: Microsoft.KernelMemory.Core, 0.20.231211.3\"\n", + "#r \"nuget: Microsoft.KernelMemory.Core, 0.23.231218.1\"\n", "\n", "using Microsoft.KernelMemory;" ] @@ -89,6 +89,11 @@ "In this demo we use OpenAI to calculate embeddings and generate text, and the default storage settings, with content and embeddings kept in a volatile memory automatically deleted after the execution." ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + }, { "cell_type": "code", "execution_count": 3, @@ -118,7 +123,9 @@ "info: Microsoft.KernelMemory.Handlers.DeleteDocumentHandler[0]\n", " Handler 'private_delete_document' ready\n", "info: Microsoft.KernelMemory.Handlers.DeleteIndexHandler[0]\n", - " Handler 'private_delete_index' ready\n" + " Handler 'private_delete_index' ready\n", + "info: Microsoft.KernelMemory.Handlers.DeleteGeneratedFilesHandler[0]\n", + " Handler 'delete_generated_files' ready\n" ] } ], @@ -178,7 +185,7 @@ "text": [ "Question: What's Semantic Kernel?\n", "\n", - "Answer: Semantic Kernel is a lightweight SDK (Software Development Kit) developed by Microsoft. It enables the integration of AI Large Language Models (LLMs) with conventional programming languages. The SDK combines natural language semantic functions, traditional code native functions, and embeddings-based memory to add value and unlock new potential in applications with AI. Semantic Kernel supports prompt templating, function chaining, vectorized memory, and intelligent planning capabilities. It encapsulates several design patterns from the latest AI research, allowing developers to infuse their applications with features like prompt chaining, recursive reasoning, summarization, zero/few-shot learning, contextual memory, long-term memory, embeddings, semantic indexing, planning, retrieval-augmented generation, and accessing external knowledge stores. Semantic Kernel is open-source and aims to enable developers to build AI-first apps faster.\n" + "Answer: Semantic Kernel is a lightweight SDK (Software Development Kit) developed by Microsoft. It enables the integration of AI Large Language Models (LLMs) with conventional programming languages. The SDK combines natural language semantic functions, traditional code native functions, and embeddings-based memory to add value and unlock new potential in applications with AI. Semantic Kernel supports prompt templating, function chaining, vectorized memory, and intelligent planning capabilities. It encapsulates several design patterns from the latest AI research, allowing developers to infuse their applications with features like prompt chaining, recursive reasoning, summarization, zero/few-shot learning, contextual memory, long-term memory, embeddings, semantic indexing, planning, retrieval-augmented generation, and accessing external knowledge stores. Semantic Kernel is available for use with C# and Python programming languages. It is an open-source project, and developers are invited to contribute to its development through GitHub discussions, opening issues, sending pull requests, and joining the Discord community.\n" ] } ], @@ -215,7 +222,7 @@ "text": [ "Sources:\n", "\n", - " - sample-SK-Readme.pdf - doc001/ecb6d46eb039411193996b6cc802b11e [Friday, December 8, 2023]\n" + " - sample-SK-Readme.pdf - default/doc001/a6c00dad170140178e805c9a0c267a7f [Tuesday, December 19, 2023]\n" ] } ], diff --git a/examples/000-notebooks/002-semantic-kernel-plugin.ipynb b/examples/000-notebooks/002-semantic-kernel-plugin.ipynb index 9868dbe91..a599524c6 100644 --- a/examples/000-notebooks/002-semantic-kernel-plugin.ipynb +++ b/examples/000-notebooks/002-semantic-kernel-plugin.ipynb @@ -13,7 +13,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 15, "metadata": { "dotnet_interactive": { "language": "csharp" @@ -26,7 +26,7 @@ { "data": { "text/html": [ - "
Installed Packages
" + "
Installed Packages
" ] }, "metadata": {}, @@ -34,8 +34,8 @@ } ], "source": [ - "#r \"nuget: Microsoft.SemanticKernel, 1.0.0-rc3\"\n", - "#r \"nuget: Microsoft.KernelMemory.SemanticKernelPlugin, 0.20.231211.3\"\n", + "#r \"nuget: Microsoft.SemanticKernel, 1.0.1\"\n", + "#r \"nuget: Microsoft.KernelMemory.SemanticKernelPlugin, 0.23.231218.1\"\n", "\n", "using Microsoft.SemanticKernel;\n", "using Microsoft.KernelMemory;" @@ -58,7 +58,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 16, "metadata": { "dotnet_interactive": { "language": "csharp" @@ -94,7 +94,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 17, "metadata": { "dotnet_interactive": { "language": "csharp" @@ -113,7 +113,7 @@ } ], "source": [ - "var kernel = new KernelBuilder()\n", + "var kernel = Kernel.CreateBuilder()\n", " .AddOpenAIChatCompletion(\n", " modelId: \"gpt-3.5-turbo\",\n", " apiKey: env[\"OPENAI_API_KEY\"])\n", @@ -137,7 +137,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 18, "metadata": { "dotnet_interactive": { "language": "csharp" @@ -198,7 +198,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 19, "metadata": { "dotnet_interactive": { "language": "csharp" @@ -248,7 +248,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 20, "metadata": { "dotnet_interactive": { "language": "csharp" @@ -292,7 +292,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 21, "metadata": { "dotnet_interactive": { "language": "csharp" @@ -306,7 +306,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "🚀 NASA has invited media to see the new test version of the Orion spacecraft and recovery hardware.\n" + "💡 NASA has invited media to see a new test version of their Orion spacecraft.\n" ] } ], @@ -319,7 +319,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 22, "metadata": { "dotnet_interactive": { "language": "csharp" @@ -333,7 +333,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "🌌 Orion is a constellation named after a Greek hunter. It is visible in winter and has a spacecraft named after it.\n" + "🌌 Orion is a constellation in the winter sky and NASA's Artemis II mission spacecraft.\n" ] } ], diff --git a/examples/000-notebooks/003-summarizing-documents.ipynb b/examples/000-notebooks/003-summarizing-documents.ipynb index 1d9ae5959..c7d11318a 100644 --- a/examples/000-notebooks/003-summarizing-documents.ipynb +++ b/examples/000-notebooks/003-summarizing-documents.ipynb @@ -40,7 +40,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 1, "metadata": { "dotnet_interactive": { "language": "csharp" @@ -76,7 +76,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 2, "metadata": { "dotnet_interactive": { "language": "csharp" @@ -89,7 +89,7 @@ { "data": { "text/html": [ - "
Installed Packages
" + "
Installed Packages
" ] }, "metadata": {}, @@ -99,12 +99,28 @@ "name": "stdout", "output_type": "stream", "text": [ + "info: Microsoft.KernelMemory.Handlers.TextExtractionHandler[0]\n", + " Handler 'extract' ready\n", + "info: Microsoft.KernelMemory.Handlers.TextPartitioningHandler[0]\n", + " Handler 'partition' ready\n", + "info: Microsoft.KernelMemory.Handlers.SummarizationHandler[0]\n", + " Handler 'summarize' ready\n", + "info: Microsoft.KernelMemory.Handlers.GenerateEmbeddingsHandler[0]\n", + " Handler 'gen_embeddings' ready, 1 embedding generators\n", + "info: Microsoft.KernelMemory.Handlers.SaveRecordsHandler[0]\n", + " Handler save_records ready, 1 vector storages\n", + "info: Microsoft.KernelMemory.Handlers.DeleteDocumentHandler[0]\n", + " Handler 'private_delete_document' ready\n", + "info: Microsoft.KernelMemory.Handlers.DeleteIndexHandler[0]\n", + " Handler 'private_delete_index' ready\n", + "info: Microsoft.KernelMemory.Handlers.DeleteGeneratedFilesHandler[0]\n", + " Handler 'delete_generated_files' ready\n", "Kernel Memory ready.\n" ] } ], "source": [ - "#r \"nuget: Microsoft.KernelMemory.Core, 0.20.231211.3\"\n", + "#r \"nuget: Microsoft.KernelMemory.Core, 0.23.231218.1\"\n", "\n", "using Microsoft.KernelMemory;\n", "\n", @@ -129,7 +145,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 3, "metadata": { "dotnet_interactive": { "language": "csharp" @@ -162,7 +178,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 4, "metadata": { "dotnet_interactive": { "language": "csharp" diff --git a/examples/002-dotnet-SemanticKernelPlugin/002-dotnet-SemanticKernelPlugin.csproj b/examples/002-dotnet-SemanticKernelPlugin/002-dotnet-SemanticKernelPlugin.csproj index 0e2c77faa..dc7747981 100644 --- a/examples/002-dotnet-SemanticKernelPlugin/002-dotnet-SemanticKernelPlugin.csproj +++ b/examples/002-dotnet-SemanticKernelPlugin/002-dotnet-SemanticKernelPlugin.csproj @@ -12,8 +12,8 @@ - - + + diff --git a/examples/002-dotnet-SemanticKernelPlugin/Program.cs b/examples/002-dotnet-SemanticKernelPlugin/Program.cs index a172ef4a1..96412859a 100644 --- a/examples/002-dotnet-SemanticKernelPlugin/Program.cs +++ b/examples/002-dotnet-SemanticKernelPlugin/Program.cs @@ -17,7 +17,7 @@ public static async Task Main() // Usual code to create an instance of SK, using Azure OpenAI. // You can use any LLM, replacing `WithAzureChatCompletionService` with other LLM options. - var builder = new KernelBuilder(); + var builder = Kernel.CreateBuilder(); builder // For OpenAI: //.AddOpenAIChatCompletion( diff --git a/examples/200-dotnet-nl2sql/nl2sql.harness/Nl2Sql.Harness.csproj b/examples/200-dotnet-nl2sql/nl2sql.harness/Nl2Sql.Harness.csproj index 3c7321fe6..66afbf41a 100644 --- a/examples/200-dotnet-nl2sql/nl2sql.harness/Nl2Sql.Harness.csproj +++ b/examples/200-dotnet-nl2sql/nl2sql.harness/Nl2Sql.Harness.csproj @@ -20,8 +20,8 @@ - - + + all diff --git a/examples/203-dotnet-using-core-nuget/203-dotnet-using-core-nuget.csproj b/examples/203-dotnet-using-core-nuget/203-dotnet-using-core-nuget.csproj index d0980baa4..bca8e8d71 100644 --- a/examples/203-dotnet-using-core-nuget/203-dotnet-using-core-nuget.csproj +++ b/examples/203-dotnet-using-core-nuget/203-dotnet-using-core-nuget.csproj @@ -10,7 +10,7 @@ - + diff --git a/service/Abstractions/Constants.cs b/service/Abstractions/Constants.cs index 4ce094660..adff1fd05 100644 --- a/service/Abstractions/Constants.cs +++ b/service/Abstractions/Constants.cs @@ -37,6 +37,7 @@ public static class Constants public const string TagsSyntheticSummary = "summary"; // Properties stored inside the payload + public const string ReservedPayloadSchemaVersionField = "schema"; public const string ReservedPayloadTextField = "text"; public const string ReservedPayloadFileNameField = "file"; public const string ReservedPayloadUrlField = "url"; diff --git a/service/Abstractions/MemoryStorage/MemoryRecord.cs b/service/Abstractions/MemoryStorage/MemoryRecord.cs index 0d48ed5cf..c021a1fb2 100644 --- a/service/Abstractions/MemoryStorage/MemoryRecord.cs +++ b/service/Abstractions/MemoryStorage/MemoryRecord.cs @@ -5,8 +5,18 @@ namespace Microsoft.KernelMemory.MemoryStorage; +// ReSharper disable ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract public class MemoryRecord { + // Memory Db Record schema versioning - Introduced after version 0.23.231218.1 + private const string SchemaVersionZero = ""; + private const string SchemaVersion20231218A = "20231218A"; + private const string CurrentSchemaVersion = SchemaVersion20231218A; + + // Internal data + private TagCollection _tags = new(); + private Dictionary _payload = new(); + /// /// Unique record ID /// @@ -37,7 +47,19 @@ public class MemoryRecord /// [JsonPropertyName("tags")] [JsonPropertyOrder(2)] - public TagCollection Tags { get; set; } = new(); + public TagCollection Tags + { + get + { + if (this.UpgradeRequired()) { this.Upgrade(); } + + return this._tags; + } + set + { + this._tags = value; + } + } /// /// Optional Non-Searchable payload processed client side. @@ -54,5 +76,70 @@ public class MemoryRecord /// [JsonPropertyName("payload")] [JsonPropertyOrder(3)] - public Dictionary Payload { get; set; } = new(); + public Dictionary Payload + { + get + { + if (this.UpgradeRequired()) { this.Upgrade(); } + + return this._payload; + } + set + { + this._payload = value; + } + } + + /// + /// Check if the current state requires an upgrade + /// + private bool UpgradeRequired() + { + if (this._payload == null) { return true; } + + if (!this._payload.TryGetValue(Constants.ReservedPayloadSchemaVersionField, out object? versionValue)) + { + return true; + } + + return (versionValue == null || versionValue.ToString() != CurrentSchemaVersion); + } + +#pragma warning disable CA1820 // readability + /// + /// Upgrade the record to the latest schema + /// + private void Upgrade() + { + if (this._payload == null) { this._payload = new(); } + + if (this._tags == null) { this._tags = new(); } + + string version = SchemaVersionZero; + if (this._payload.TryGetValue(Constants.ReservedPayloadSchemaVersionField, out object? versionValue)) + { + version = versionValue == null ? string.Empty : versionValue.ToString(); + } + + // Upgrade to "20231218A" + if (version == SchemaVersionZero) + { + if (!this._payload.ContainsKey(Constants.ReservedPayloadUrlField)) + { + this._payload[Constants.ReservedPayloadUrlField] = string.Empty; + } + + version = SchemaVersion20231218A; + this._payload[Constants.ReservedPayloadSchemaVersionField] = SchemaVersion20231218A; + } + + // if (version == SchemaVersion20231218A) + // { + // Nothing to do, this is the latest version + // Add future upgrade logic here if required + // } + + this._payload[Constants.ReservedPayloadSchemaVersionField] = CurrentSchemaVersion; + } +#pragma warning restore CA1820 } diff --git a/service/Core/Handlers/SaveRecordsHandler.cs b/service/Core/Handlers/SaveRecordsHandler.cs index f03a9ff7c..00f590981 100644 --- a/service/Core/Handlers/SaveRecordsHandler.cs +++ b/service/Core/Handlers/SaveRecordsHandler.cs @@ -288,12 +288,6 @@ private static MemoryRecord PrepareRecord( * FILE DETAILS */ - // Original file name, useful for context, e.g. "NASA-August-2043.pdf" - record.Payload.Add(Constants.ReservedPayloadFileNameField, fileName); - - // Web page URL, used when importing from a URL (the file name is not useful in that case) - record.Payload.Add(Constants.ReservedPayloadUrlField, url); - // File type, e.g. "application/pdf" - Can be used for filtering by file type record.Tags.Add(Constants.ReservedFileTypeTag, pipeline.GetFile(fileId).MimeType); @@ -301,14 +295,20 @@ private static MemoryRecord PrepareRecord( // Can be used for filtering and deletions/purge record.Tags.Add(Constants.ReservedFileIdTag, fileId); + // Original file name, useful for context, e.g. "NASA-August-2043.pdf" + record.Payload[Constants.ReservedPayloadFileNameField] = fileName; + + // Web page URL, used when importing from a URL (the file name is not useful in that case) + record.Payload[Constants.ReservedPayloadUrlField] = url; + /* * PARTITION DETAILS */ record.Vector = partitionEmbedding; - record.Payload.Add(Constants.ReservedPayloadTextField, partitionContent); - record.Payload.Add(Constants.ReservedPayloadVectorProviderField, embeddingGeneratorProvider); - record.Payload.Add(Constants.ReservedPayloadVectorGeneratorField, embeddingGeneratorName); + record.Payload[Constants.ReservedPayloadTextField] = partitionContent; + record.Payload[Constants.ReservedPayloadVectorProviderField] = embeddingGeneratorProvider; + record.Payload[Constants.ReservedPayloadVectorGeneratorField] = embeddingGeneratorName; // Partition ID. Filtering used for purge. record.Tags.Add(Constants.ReservedFilePartitionTag, partitionFileId); @@ -317,7 +317,7 @@ private static MemoryRecord PrepareRecord( * TIMESTAMP and USER TAGS */ - record.Payload.Add(Constants.ReservedPayloadLastUpdateField, DateTimeOffset.UtcNow.ToString("s")); + record.Payload[Constants.ReservedPayloadLastUpdateField] = DateTimeOffset.UtcNow.ToString("s"); tags.CopyTo(record.Tags); diff --git a/service/Core/MemoryStorage/DevTools/SimpleTextDb.cs b/service/Core/MemoryStorage/DevTools/SimpleTextDb.cs index 0eab15819..3434f0348 100644 --- a/service/Core/MemoryStorage/DevTools/SimpleTextDb.cs +++ b/service/Core/MemoryStorage/DevTools/SimpleTextDb.cs @@ -84,7 +84,7 @@ public async Task UpsertAsync(string index, MemoryRecord record, Cancell var list = this.GetListAsync(index, filters, limit, withEmbeddings, cancellationToken); var records = new Dictionary(); - await foreach (MemoryRecord r in list.WithCancellation(cancellationToken).ConfigureAwait(false)) + await foreach (MemoryRecord r in list.ConfigureAwait(false)) { records[r.Id] = r; } diff --git a/service/tests/FunctionalTests/TestHelpers/BaseTestCase.cs b/service/tests/FunctionalTests/TestHelpers/BaseTestCase.cs index 5dcffe46c..ab37e96f5 100644 --- a/service/tests/FunctionalTests/TestHelpers/BaseTestCase.cs +++ b/service/tests/FunctionalTests/TestHelpers/BaseTestCase.cs @@ -30,13 +30,6 @@ protected BaseTestCase(IConfiguration cfg, ITestOutputHelper output) Console.SetOut(this._output); } - protected IKernelMemory GetMemoryWebClient() - { - string endpoint = this.Configuration.GetSection("ServiceAuthorization").GetValue("Endpoint", "http://127.0.0.1:9001/")!; - string? apiKey = this.Configuration.GetSection("ServiceAuthorization").GetValue("AccessKey"); - return new MemoryWebClient(endpoint, apiKey: apiKey); - } - protected IKernelMemory GetServerlessMemory(string memoryType) { var openAIKey = this.OpenAIConfiguration.GetValue("APIKey") diff --git a/service/tests/azure-ai-search-tests/Program.cs b/service/tests/ManualTests/AzureAiSearch/Program.cs similarity index 99% rename from service/tests/azure-ai-search-tests/Program.cs rename to service/tests/ManualTests/AzureAiSearch/Program.cs index 62accf56e..d474c511b 100644 --- a/service/tests/azure-ai-search-tests/Program.cs +++ b/service/tests/ManualTests/AzureAiSearch/Program.cs @@ -24,7 +24,7 @@ using Microsoft.KernelMemory.MemoryStorage; using Microsoft.KernelMemory.MemoryStorage.AzureAISearch; -namespace AzureAISearchTests; +namespace AzureAISearch; public static class Program { @@ -45,7 +45,7 @@ public static class Program private static readonly string s_endpoint = Env.Var("SEARCH_ENDPOINT")!; private static readonly string s_apiKey = Env.Var("SEARCH_KEY")!; - public static async Task Main(string[] args) + public static async Task RunAsync() { // Azure AI Search service client s_adminClient = new SearchIndexClient( diff --git a/service/tests/qdrant-tests/qdrant-tests.csproj b/service/tests/ManualTests/ManualTests.csproj similarity index 89% rename from service/tests/qdrant-tests/qdrant-tests.csproj rename to service/tests/ManualTests/ManualTests.csproj index ad9139773..5e471c5c9 100644 --- a/service/tests/qdrant-tests/qdrant-tests.csproj +++ b/service/tests/ManualTests/ManualTests.csproj @@ -15,6 +15,10 @@ + + + + diff --git a/service/tests/ManualTests/Program.cs b/service/tests/ManualTests/Program.cs new file mode 100644 index 000000000..f661364d9 --- /dev/null +++ b/service/tests/ManualTests/Program.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace ManualTests; + +public static class Program +{ + public static async Task Main() + { + await AzureAISearch.Program.RunAsync(); + await Qdrant.Program.RunAsync(); + } +} diff --git a/service/tests/qdrant-tests/FakeEmbeddingGenerator.cs b/service/tests/ManualTests/Qdrant/FakeEmbeddingGenerator.cs similarity index 100% rename from service/tests/qdrant-tests/FakeEmbeddingGenerator.cs rename to service/tests/ManualTests/Qdrant/FakeEmbeddingGenerator.cs diff --git a/service/tests/ManualTests/Qdrant/Program.cs b/service/tests/ManualTests/Qdrant/Program.cs new file mode 100644 index 000000000..c939e2ae7 --- /dev/null +++ b/service/tests/ManualTests/Qdrant/Program.cs @@ -0,0 +1,111 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.KernelMemory; +using Microsoft.KernelMemory.MemoryStorage; +using Microsoft.KernelMemory.MemoryStorage.Qdrant; + +namespace Qdrant; + +public static class Program +{ + public static async Task RunAsync() + { + var embeddingGenerator = new FakeEmbeddingGenerator(); + + var config = new QdrantConfig + { + Endpoint = "http://127.0.0.1:6333", + APIKey = "" + }; + + var memory = new QdrantMemory(config, embeddingGenerator); + + Console.WriteLine("===== DELETE INDEX ====="); + + await memory.DeleteIndexAsync("test"); + + Console.WriteLine("===== CREATE INDEX ====="); + + await memory.CreateIndexAsync("test", 5); + + Console.WriteLine("===== INSERT RECORD 1 ====="); + + const string Text = "test1"; + var embedding = new[] { 0f, 0, 1, 0, 1 }; + embeddingGenerator.Mock(Text, embedding); + + var memoryRecord1 = new MemoryRecord + { + Id = "memory 1", + Vector = embedding, + Tags = new TagCollection { { "updated", "no" }, { "type", "email" } }, + Payload = new Dictionary() + }; + + var id1 = await memory.UpsertAsync("test", memoryRecord1); + Console.WriteLine($"Insert 1: {id1} {memoryRecord1.Id}"); + + Console.WriteLine("===== INSERT RECORD 2 ====="); + + var memoryRecord2 = new MemoryRecord + { + Id = "memory two", + Vector = new[] { 0f, 0, 1, 0, 1 }, + Tags = new TagCollection { { "type", "news" } }, + Payload = new Dictionary() + }; + + var id2 = await memory.UpsertAsync("test", memoryRecord2); + Console.WriteLine($"Insert 2: {id2} {memoryRecord2.Id}"); + + Console.WriteLine("===== UPDATE RECORD 2 ====="); + memoryRecord2.Tags.Add("updated", "yes"); + id2 = await memory.UpsertAsync("test", memoryRecord2); + Console.WriteLine($"Update 2: {id2} {memoryRecord2.Id}"); + + Console.WriteLine("===== SEARCH 1 ====="); + + var similarList = memory.GetSimilarListAsync("test", text: Text, + limit: 10, withEmbeddings: true); + await foreach ((MemoryRecord, double) record in similarList) + { + Console.WriteLine(record.Item1.Id); + Console.WriteLine(" tags: " + record.Item1.Tags.Count); + Console.WriteLine(" size: " + record.Item1.Vector.Length); + } + + Console.WriteLine("===== SEARCH 2 ====="); + + similarList = memory.GetSimilarListAsync("test", text: Text, + limit: 10, withEmbeddings: true, filters: new List { MemoryFilters.ByTag("type", "email") }); + await foreach ((MemoryRecord, double) record in similarList) + { + Console.WriteLine(record.Item1.Id); + Console.WriteLine(" type: " + record.Item1.Tags["type"].First()); + } + + Console.WriteLine("===== LIST ====="); + + var list = memory.GetListAsync("test", limit: 10, withEmbeddings: false); + await foreach (MemoryRecord record in list) + { + Console.WriteLine(record.Id); + Console.WriteLine(" type: " + record.Tags["type"].First()); + } + + Console.WriteLine("===== DELETE ====="); + + await memory.DeleteAsync("test", new MemoryRecord { Id = "memory 1" }); + + Console.WriteLine("===== LIST AFTER DELETE ====="); + + list = memory.GetListAsync("test", limit: 10, withEmbeddings: false); + await foreach (MemoryRecord record in list) + { + Console.WriteLine(record.Id); + Console.WriteLine(" type: " + record.Tags["type"].First()); + } + + Console.WriteLine("== Done =="); + } +} diff --git a/service/tests/qdrant-tests/appsettings.json b/service/tests/ManualTests/appsettings.json similarity index 100% rename from service/tests/qdrant-tests/appsettings.json rename to service/tests/ManualTests/appsettings.json diff --git a/service/tests/FunctionalTests/Service/DocumentUploadTest.cs b/service/tests/ServiceFunctionalTests/DocumentUploadTest.cs similarity index 100% rename from service/tests/FunctionalTests/Service/DocumentUploadTest.cs rename to service/tests/ServiceFunctionalTests/DocumentUploadTest.cs diff --git a/service/tests/FunctionalTests/Service/FilteringTest.cs b/service/tests/ServiceFunctionalTests/FilteringTest.cs similarity index 100% rename from service/tests/FunctionalTests/Service/FilteringTest.cs rename to service/tests/ServiceFunctionalTests/FilteringTest.cs diff --git a/service/tests/ServiceFunctionalTests/Fixtures/ANWC-image-for-OCR.jpg b/service/tests/ServiceFunctionalTests/Fixtures/ANWC-image-for-OCR.jpg new file mode 100644 index 000000000..066896fc8 Binary files /dev/null and b/service/tests/ServiceFunctionalTests/Fixtures/ANWC-image-for-OCR.jpg differ diff --git a/service/tests/ServiceFunctionalTests/Fixtures/Doc1.txt b/service/tests/ServiceFunctionalTests/Fixtures/Doc1.txt new file mode 100644 index 000000000..e69de29bb diff --git a/service/tests/ServiceFunctionalTests/Fixtures/Documents/Doc1.txt b/service/tests/ServiceFunctionalTests/Fixtures/Documents/Doc1.txt new file mode 100644 index 000000000..e69de29bb diff --git a/service/tests/ServiceFunctionalTests/Fixtures/Documents/Doc2.txt b/service/tests/ServiceFunctionalTests/Fixtures/Documents/Doc2.txt new file mode 100644 index 000000000..e69de29bb diff --git a/service/tests/ServiceFunctionalTests/Fixtures/Documents/Sales/Doc1.txt b/service/tests/ServiceFunctionalTests/Fixtures/Documents/Sales/Doc1.txt new file mode 100644 index 000000000..e69de29bb diff --git a/service/tests/ServiceFunctionalTests/Fixtures/Documents/Sales/Doc2.txt b/service/tests/ServiceFunctionalTests/Fixtures/Documents/Sales/Doc2.txt new file mode 100644 index 000000000..e69de29bb diff --git a/service/tests/ServiceFunctionalTests/Fixtures/Documents/Support/Doc1.txt b/service/tests/ServiceFunctionalTests/Fixtures/Documents/Support/Doc1.txt new file mode 100644 index 000000000..e69de29bb diff --git a/service/tests/ServiceFunctionalTests/Fixtures/Documents/Support/Doc2.txt b/service/tests/ServiceFunctionalTests/Fixtures/Documents/Support/Doc2.txt new file mode 100644 index 000000000..e69de29bb diff --git a/service/tests/ServiceFunctionalTests/Fixtures/Documents/Support/Doc3.txt b/service/tests/ServiceFunctionalTests/Fixtures/Documents/Support/Doc3.txt new file mode 100644 index 000000000..e69de29bb diff --git a/service/tests/FunctionalTests/Service/ImageOCRTest.cs b/service/tests/ServiceFunctionalTests/ImageOCRTest.cs similarity index 100% rename from service/tests/FunctionalTests/Service/ImageOCRTest.cs rename to service/tests/ServiceFunctionalTests/ImageOCRTest.cs diff --git a/service/tests/FunctionalTests/Service/ImportFilesTest.cs b/service/tests/ServiceFunctionalTests/ImportFilesTest.cs similarity index 100% rename from service/tests/FunctionalTests/Service/ImportFilesTest.cs rename to service/tests/ServiceFunctionalTests/ImportFilesTest.cs diff --git a/service/tests/FunctionalTests/Service/IndexListTest.cs b/service/tests/ServiceFunctionalTests/IndexListTest.cs similarity index 100% rename from service/tests/FunctionalTests/Service/IndexListTest.cs rename to service/tests/ServiceFunctionalTests/IndexListTest.cs diff --git a/service/tests/FunctionalTests/Service/PluginTest.cs b/service/tests/ServiceFunctionalTests/PluginTest.cs similarity index 100% rename from service/tests/FunctionalTests/Service/PluginTest.cs rename to service/tests/ServiceFunctionalTests/PluginTest.cs diff --git a/service/tests/ServiceFunctionalTests/ServiceFunctionalTests.csproj b/service/tests/ServiceFunctionalTests/ServiceFunctionalTests.csproj new file mode 100644 index 000000000..68c2a5514 --- /dev/null +++ b/service/tests/ServiceFunctionalTests/ServiceFunctionalTests.csproj @@ -0,0 +1,42 @@ + + + + net7.0 + true + Microsoft.ServiceFunctionalTests + enable + false + 5ee045b0-aea3-4f08-8d31-32d1a6f8fed0 + $(NoWarn);IDE1006,CA1303,CA1307,CA1859,CA2007,CA2201,CS1591,CA1861,SKEXP0001 + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + Always + + + + diff --git a/service/tests/ServiceFunctionalTests/TestHelpers/BaseTestCase.cs b/service/tests/ServiceFunctionalTests/TestHelpers/BaseTestCase.cs new file mode 100644 index 000000000..136a416b2 --- /dev/null +++ b/service/tests/ServiceFunctionalTests/TestHelpers/BaseTestCase.cs @@ -0,0 +1,68 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Reflection; +using Microsoft.KernelMemory; +using Xunit.Abstractions; + +namespace FunctionalTests.TestHelpers; + +public abstract class BaseTestCase : IDisposable +{ + protected const string NotFound = "INFO NOT FOUND"; + + private readonly IConfiguration _cfg; + private readonly RedirectConsole _output; + + protected IConfiguration Configuration => this._cfg; + + protected BaseTestCase(IConfiguration cfg, ITestOutputHelper output) + { + this._cfg = cfg; + this._output = new RedirectConsole(output); + Console.SetOut(this._output); + } + + protected IKernelMemory GetMemoryWebClient() + { + string endpoint = this.Configuration.GetSection("ServiceAuthorization").GetValue("Endpoint", "http://127.0.0.1:9001/")!; + string? apiKey = this.Configuration.GetSection("ServiceAuthorization").GetValue("AccessKey"); + return new MemoryWebClient(endpoint, apiKey: apiKey); + } + + // Find the "Fixtures" directory (inside the project, requires source code) + protected string? FindFixturesDir() + { + // start from the location of the executing assembly, and traverse up max 5 levels + var path = Path.GetDirectoryName(Path.GetFullPath(Assembly.GetExecutingAssembly().Location)); + for (var i = 0; i < 5; i++) + { + Console.WriteLine($"Checking '{path}'"); + var test = Path.Join(path, "Fixtures"); + if (Directory.Exists(test)) { return test; } + + // up one level + path = Path.GetDirectoryName(path); + } + + return null; + } + + public void Dispose() + { + this.Dispose(true); + GC.SuppressFinalize(this); + } + + protected virtual void Dispose(bool disposing) + { + if (disposing) + { + this._output.Dispose(); + } + } + + protected void Log(string text) + { + this._output.WriteLine(text); + } +} diff --git a/service/tests/ServiceFunctionalTests/TestHelpers/RedirectConsole.cs b/service/tests/ServiceFunctionalTests/TestHelpers/RedirectConsole.cs new file mode 100644 index 000000000..f19dd44db --- /dev/null +++ b/service/tests/ServiceFunctionalTests/TestHelpers/RedirectConsole.cs @@ -0,0 +1,321 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Diagnostics.CodeAnalysis; +using System.Globalization; +using System.Text; +using Xunit.Abstractions; + +namespace FunctionalTests.TestHelpers; + +internal sealed class RedirectConsole : TextWriter +{ + private readonly ITestOutputHelper _output; + + public override IFormatProvider FormatProvider => CultureInfo.CurrentCulture; + + public override Encoding Encoding { get; } = Encoding.Default; + + public RedirectConsole(ITestOutputHelper output) + { + this._output = output; + } + + public override void Write(string? value) + { + this.Text(value); + } + + public override void WriteLine(string? value) + { + this.Line(value); + } + + public override void Write(char value) + { + this.Text($"{value}"); + } + + public override void WriteLine(char value) + { + this.Line($"{value}"); + } + + public override void Write(char[]? buffer) + { + if (buffer == null || buffer.Length == 0) { return; } + + var s = new StringBuilder(); + foreach (var c in buffer) { s.Append(c); } + + this.Text(s.ToString()); + } + + public override void WriteLine(char[]? buffer) + { + if (buffer == null) + { + this.Line(); + return; + } + + var s = new StringBuilder(); + foreach (var c in buffer) { s.Append(c); } + + this.Line(s.ToString()); + } + + public override void Write(char[] buffer, int index, int count) + { + if (buffer.Length == 0 || count <= 0 || index < 0 || buffer.Length - index < count) + { + return; + } + + var s = new StringBuilder(); + for (int i = 0; i < count; i++) + { + s.Append(buffer[index + i]); + } + + this.Text(s.ToString()); + } + + public override void WriteLine(char[] buffer, int index, int count) + { + if (count <= 0 || index < 0 || buffer.Length - index < count) + { + this.Line(); + return; + } + + var s = new StringBuilder(); + for (int i = 0; i < count; i++) + { + s.Append(buffer[index + i]); + } + + this.Line(s.ToString()); + } + + public override void Write(ReadOnlySpan buffer) + { + if (buffer == null || buffer.Length == 0) { return; } + + var s = new StringBuilder(); + foreach (var c in buffer) { s.Append(c); } + + this.Text(s.ToString()); + } + + public override void WriteLine(ReadOnlySpan buffer) + { + if (buffer == null) + { + this.Line(); + return; + } + + var s = new StringBuilder(); + foreach (var c in buffer) { s.Append(c); } + + this.Line(s.ToString()); + } + + public override void Write(StringBuilder? buffer) + { + if (buffer == null || buffer.Length == 0) { return; } + + this.Text(buffer.ToString()); + } + + public override void WriteLine(StringBuilder? buffer) + { + if (buffer == null) + { + this.Line(); + return; + } + + this.Line(buffer.ToString()); + } + + public override void Write(bool value) + { + this.Text(value ? "True" : "False"); + } + + public override void WriteLine(bool value) + { + this.Line(value ? "True" : "False"); + } + + public override void Write(int value) + { + this.Text(value.ToString(this.FormatProvider)); + } + + public override void WriteLine(int value) + { + this.Line(value.ToString(this.FormatProvider)); + } + + public override void Write(uint value) + { + this.Text(value.ToString(this.FormatProvider)); + } + + public override void WriteLine(uint value) + { + this.Line(value.ToString(this.FormatProvider)); + } + + public override void Write(long value) + { + this.Text(value.ToString(this.FormatProvider)); + } + + public override void WriteLine(long value) + { + this.Line(value.ToString(this.FormatProvider)); + } + + public override void Write(ulong value) + { + this.Text(value.ToString(this.FormatProvider)); + } + + public override void WriteLine(ulong value) + { + this.Line(value.ToString(this.FormatProvider)); + } + + public override void Write(float value) + { + this.Text(value.ToString(this.FormatProvider)); + } + + public override void WriteLine(float value) + { + this.Line(value.ToString(this.FormatProvider)); + } + + public override void Write(double value) + { + this.Text(value.ToString(this.FormatProvider)); + } + + public override void WriteLine(double value) + { + this.Line(value.ToString(this.FormatProvider)); + } + + public override void Write(decimal value) + { + this.Text(value.ToString(this.FormatProvider)); + } + + public override void WriteLine(decimal value) + { + this.Line(value.ToString(this.FormatProvider)); + } + + public override void Write(object? value) + { + if (value != null) + { + if (value is IFormattable f) + { + this.Text(f.ToString(null, this.FormatProvider)); + } + else + { + this.Text(value.ToString()); + } + } + } + + public override void WriteLine(object? value) + { + if (value != null) + { + if (value is IFormattable f) + { + this.Line(f.ToString(null, this.FormatProvider)); + } + else + { + this.Line(value.ToString()); + } + } + else + { + this.Line(); + } + } + + public override void Write([StringSyntax(StringSyntaxAttribute.CompositeFormat)] string format, object? arg0) + { + this.Text(string.Format(this.FormatProvider, format, arg0)); + } + + public override void WriteLine([StringSyntax(StringSyntaxAttribute.CompositeFormat)] string format, object? arg0) + { + this.Line(string.Format(this.FormatProvider, format, arg0)); + } + + public override void Write([StringSyntax(StringSyntaxAttribute.CompositeFormat)] string format, object? arg0, object? arg1) + { + this.Text(string.Format(this.FormatProvider, format, arg0, arg1)); + } + + public override void WriteLine([StringSyntax(StringSyntaxAttribute.CompositeFormat)] string format, object? arg0, object? arg1) + { + this.Line(string.Format(this.FormatProvider, format, arg0, arg1)); + } + + public override void Write([StringSyntax(StringSyntaxAttribute.CompositeFormat)] string format, object? arg0, object? arg1, object? arg2) + { + this.Text(string.Format(this.FormatProvider, format, arg0, arg1, arg2)); + } + + public override void WriteLine([StringSyntax(StringSyntaxAttribute.CompositeFormat)] string format, object? arg0, object? arg1, object? arg2) + { + this.Line(string.Format(this.FormatProvider, format, arg0, arg1, arg2)); + } + + public override void Write([StringSyntax(StringSyntaxAttribute.CompositeFormat)] string format, params object?[] arg) + { + this.Text(string.Format(this.FormatProvider, format, arg)); + } + + public override void WriteLine([StringSyntax(StringSyntaxAttribute.CompositeFormat)] string format, params object?[] arg) + { + this.Line(string.Format(this.FormatProvider, format, arg)); + } + + private void Text(string? s) + { + if (string.IsNullOrEmpty(s)) { return; } + + try + { + this._output.WriteLine(s); + } + catch (InvalidOperationException e) when (e.Message.Contains("no currently active test", StringComparison.OrdinalIgnoreCase)) + { + // NOOP: Xunit thread out of scope + } + } + + private void Line(string? s = null) + { + try + { + this._output.WriteLine(s); + } + catch (InvalidOperationException e) when (e.Message.Contains("no currently active test", StringComparison.OrdinalIgnoreCase)) + { + // NOOP: Xunit thread out of scope + } + } +} diff --git a/service/tests/ServiceFunctionalTests/TestHelpers/Startup.cs b/service/tests/ServiceFunctionalTests/TestHelpers/Startup.cs new file mode 100644 index 000000000..e500feca5 --- /dev/null +++ b/service/tests/ServiceFunctionalTests/TestHelpers/Startup.cs @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft. All rights reserved. + +// IMPORTANT: this file must be at the root of the namespace + +namespace FunctionalTests; + +/// +/// IMPORTANT: this file must be at the root of the namespace +/// +public class Startup +{ + public void ConfigureHost(IHostBuilder hostBuilder) + { + var config = new ConfigurationBuilder() + .AddJsonFile("appsettings.json", optional: true) + .AddJsonFile("appsettings.development.json", optional: true) + .AddUserSecrets() + .AddEnvironmentVariables() + .Build(); + + hostBuilder.ConfigureHostConfiguration(builder => builder.AddConfiguration(config)); + } +} diff --git a/service/tests/ServiceFunctionalTests/TestHelpers/Usings.cs b/service/tests/ServiceFunctionalTests/TestHelpers/Usings.cs new file mode 100644 index 000000000..38b0ca7bb --- /dev/null +++ b/service/tests/ServiceFunctionalTests/TestHelpers/Usings.cs @@ -0,0 +1,3 @@ +// Copyright (c) Microsoft. All rights reserved. + +global using Xunit; diff --git a/service/tests/ServiceFunctionalTests/appsettings.json b/service/tests/ServiceFunctionalTests/appsettings.json new file mode 100644 index 000000000..bc4b048e6 --- /dev/null +++ b/service/tests/ServiceFunctionalTests/appsettings.json @@ -0,0 +1,42 @@ +{ + "ServiceAuthorization": { + "Endpoint": "http://127.0.0.1:9001/", + "AccessKey": "", + }, + "Services": { + "SimpleVectorDb": { + // Options: "Disk", "Volatile" + "StorageType": "Volatile", + "Directory": "tmp-vectors" + }, + "AzureAISearch": { + // "ApiKey" or "AzureIdentity". For other options see . + // AzureIdentity: use automatic AAD authentication mechanism. You can test locally + // using the env vars AZURE_TENANT_ID, AZURE_CLIENT_ID, AZURE_CLIENT_SECRET. + "Auth": "AzureIdentity", + "Endpoint": "https://<...>", + "APIKey": "", + }, + "Qdrant": { + "Endpoint": "http://127.0.0.1:6333", + "APIKey": "", + }, + "OpenAI": { + "APIKey": "", + }, + "LlamaSharp": { + // path to file, e.g. "llama-2-7b-chat.Q6_K.gguf" + "ModelPath": "", + // Max number of tokens supported by the model + "MaxTokenTotal": 4096 + // Optional parameters + // "GpuLayerCount": 32, + // "Seed": 1337, + }, + }, + "Logging": { + "LogLevel": { + "Default": "Information" + } + } +} \ No newline at end of file diff --git a/service/tests/ServiceFunctionalTests/file1-NASA-news.pdf b/service/tests/ServiceFunctionalTests/file1-NASA-news.pdf new file mode 100644 index 000000000..92c129799 Binary files /dev/null and b/service/tests/ServiceFunctionalTests/file1-NASA-news.pdf differ diff --git a/service/tests/UnitTests/MemoryStorage/MemoryRecordTest.cs b/service/tests/UnitTests/MemoryStorage/MemoryRecordTest.cs new file mode 100644 index 000000000..4413f2f5f --- /dev/null +++ b/service/tests/UnitTests/MemoryStorage/MemoryRecordTest.cs @@ -0,0 +1,109 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Text.Json; +using Microsoft.KernelMemory.MemoryStorage; +using UnitTests.TestHelpers; +using Xunit.Abstractions; + +namespace UnitTests.MemoryStorage; + +public class MemoryRecordTest : BaseTestCase +{ + public MemoryRecordTest(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public void ItCanBeSerialized() + { + // Arrange + var record = new MemoryRecord(); + + // Act + string serialized = JsonSerializer.Serialize(record); + var actual = JsonSerializer.Deserialize(serialized); + + // Assert + Assert.NotNull(actual); + Assert.Equal(record.Id, actual.Id); + Assert.Equal(JsonSerializer.Serialize(record.Vector), JsonSerializer.Serialize(actual.Vector)); + + // Arrange + record.Id = "123"; + record.Tags["foo"] = new List { "bar" }; + record.Payload["bar"] = "foo"; + + // Act + serialized = JsonSerializer.Serialize(record); + actual = JsonSerializer.Deserialize(serialized); + + // Assert + Assert.NotNull(actual); + Assert.Equal("123", actual.Id); + Assert.Equal("bar", actual.Tags["foo"].First()); + Assert.Equal("foo", actual.Payload["bar"].ToString()); + } + + [Fact] + public void ItSupportsSchemaVersioning() + { + // This constant should never change + Assert.Equal("schema", Microsoft.KernelMemory.Constants.ReservedPayloadSchemaVersionField); + + // Arrange + var record = new MemoryRecord(); + Assert.True(record.Payload.ContainsKey("schema")); + + // Act + string serialized = JsonSerializer.Serialize(record); + var actual = JsonSerializer.Deserialize(serialized); + + // Assert + Assert.NotNull(actual); + Assert.True(actual.Payload.ContainsKey("schema")); + Assert.Equal("20231218A", record.Payload["schema"]); + } + + [Fact] + public void ItSelfUpgrades() + { + // Arrange + var record1 = new MemoryRecord(); + var record2 = new MemoryRecord(); + + // Act + record1.Payload.Remove("schema"); + record2.Payload["schema"] = ""; + + // Assert + Assert.Equal("20231218A", record1.Payload["schema"]); + Assert.Equal("20231218A", record2.Payload["schema"]); + + // Act + record1.Payload.Remove(Microsoft.KernelMemory.Constants.ReservedPayloadUrlField); + record2.Payload[Microsoft.KernelMemory.Constants.ReservedPayloadUrlField] = "foo"; + record1.Payload.Remove("schema"); + record2.Payload.Remove("schema"); + + // Assert - the default value is added + Assert.Equal("", record1.Payload[Microsoft.KernelMemory.Constants.ReservedPayloadUrlField]); + + // Assert - the value persists even if an upgrade occurred + Assert.Equal("foo", record2.Payload[Microsoft.KernelMemory.Constants.ReservedPayloadUrlField]); + } + + [Fact] + public void ItAllowsToRemoveKeys() + { + // Arrange + var record1 = new MemoryRecord(); + var record2 = new MemoryRecord(); + + // Act - Note: the Url value is removed after the internal upgrade occurs + record1.Payload.Remove("schema"); + record1.Payload.Remove(Microsoft.KernelMemory.Constants.ReservedPayloadUrlField); + + // Assert - The upgrade should not occur hence the key should not exist + Assert.False(record1.Payload.ContainsKey(Microsoft.KernelMemory.Constants.ReservedPayloadUrlField)); + } +} diff --git a/service/tests/azure-ai-search-tests/appsettings.json b/service/tests/azure-ai-search-tests/appsettings.json deleted file mode 100644 index bcdd3fc2c..000000000 --- a/service/tests/azure-ai-search-tests/appsettings.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "Logging": { - "LogLevel": { - "Default": "Warning", - "Microsoft.AspNetCore": "Warning" - } - } -} diff --git a/service/tests/azure-ai-search-tests/azure-ai-search-tests.csproj b/service/tests/azure-ai-search-tests/azure-ai-search-tests.csproj deleted file mode 100644 index e24ad5853..000000000 --- a/service/tests/azure-ai-search-tests/azure-ai-search-tests.csproj +++ /dev/null @@ -1,30 +0,0 @@ - - - - Exe - net6.0 - enable - enable - enable - false - 5ee045b0-aea3-4f08-8d31-32d1a6f8fed0 - $(NoWarn);CA1050,CA2007,CA1826,CA1303,CA1307,CS1591 - AzureAISearchTests - - - - - - - - - - - - - - Always - - - - diff --git a/service/tests/qdrant-tests/Program.cs b/service/tests/qdrant-tests/Program.cs deleted file mode 100644 index 3b55ecc83..000000000 --- a/service/tests/qdrant-tests/Program.cs +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using Microsoft.KernelMemory; -using Microsoft.KernelMemory.MemoryStorage; -using Microsoft.KernelMemory.MemoryStorage.Qdrant; - -var embeddingGenerator = new FakeEmbeddingGenerator(); - -var config = new QdrantConfig -{ - Endpoint = "http://127.0.0.1:6333", - APIKey = "" -}; - -var memory = new QdrantMemory(config, embeddingGenerator); - -Console.WriteLine("===== DELETE INDEX ====="); - -await memory.DeleteIndexAsync("test"); - -Console.WriteLine("===== CREATE INDEX ====="); - -await memory.CreateIndexAsync("test", 5); - -Console.WriteLine("===== INSERT RECORD 1 ====="); - -const string Text = "test1"; -var embedding = new[] { 0f, 0, 1, 0, 1 }; -embeddingGenerator.Mock(Text, embedding); - -var memoryRecord1 = new MemoryRecord -{ - Id = "memory 1", - Vector = embedding, - Tags = new TagCollection { { "updated", "no" }, { "type", "email" } }, - Payload = new Dictionary() -}; - -var id1 = await memory.UpsertAsync("test", memoryRecord1); -Console.WriteLine($"Insert 1: {id1} {memoryRecord1.Id}"); - -Console.WriteLine("===== INSERT RECORD 2 ====="); - -var memoryRecord2 = new MemoryRecord -{ - Id = "memory two", - Vector = new[] { 0f, 0, 1, 0, 1 }, - Tags = new TagCollection { { "type", "news" } }, - Payload = new Dictionary() -}; - -var id2 = await memory.UpsertAsync("test", memoryRecord2); -Console.WriteLine($"Insert 2: {id2} {memoryRecord2.Id}"); - -Console.WriteLine("===== UPDATE RECORD 2 ====="); -memoryRecord2.Tags.Add("updated", "yes"); -id2 = await memory.UpsertAsync("test", memoryRecord2); -Console.WriteLine($"Update 2: {id2} {memoryRecord2.Id}"); - -Console.WriteLine("===== SEARCH 1 ====="); - -var similarList = memory.GetSimilarListAsync("test", text: Text, - limit: 10, withEmbeddings: true); -await foreach ((MemoryRecord, double) record in similarList) -{ - Console.WriteLine(record.Item1.Id); - Console.WriteLine(" tags: " + record.Item1.Tags.Count); - Console.WriteLine(" size: " + record.Item1.Vector.Length); -} - -Console.WriteLine("===== SEARCH 2 ====="); - -similarList = memory.GetSimilarListAsync("test", text: Text, - limit: 10, withEmbeddings: true, filters: new List { MemoryFilters.ByTag("type", "email") }); -await foreach ((MemoryRecord, double) record in similarList) -{ - Console.WriteLine(record.Item1.Id); - Console.WriteLine(" type: " + record.Item1.Tags["type"].First()); -} - -Console.WriteLine("===== LIST ====="); - -var list = memory.GetListAsync("test", limit: 10, withEmbeddings: false); -await foreach (MemoryRecord record in list) -{ - Console.WriteLine(record.Id); - Console.WriteLine(" type: " + record.Tags["type"].First()); -} - -Console.WriteLine("===== DELETE ====="); - -await memory.DeleteAsync("test", new MemoryRecord { Id = "memory 1" }); - -Console.WriteLine("===== LIST AFTER DELETE ====="); - -list = memory.GetListAsync("test", limit: 10, withEmbeddings: false); -await foreach (MemoryRecord record in list) -{ - Console.WriteLine(record.Id); - Console.WriteLine(" type: " + record.Tags["type"].First()); -} - -Console.WriteLine("== Done ==");