-
Notifications
You must be signed in to change notification settings - Fork 3.5k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
.Net: Added implementation of SQLite connector for new memory design (#…
…9164) ### Motivation and Context <!-- Thank you for your contribution to the semantic-kernel repo! Please help reviewers and future users, providing the following information: 1. Why is this change required? 2. What problem does it solve? 3. What scenario does it contribute to? 4. If it fixes an open issue, please link to the issue here. --> Related: #8089 In this PR: - Implemented `IVectorStore` - Implemented `IVectorizedSearch` - Implemented `IVectorStoreRecordCollection<TKey, TRecord>` - SQLite default record mapper - SQLite generic data model mapper - `Options` classes - Extension methods for DI - Integration tests - Unit tests (in progress) ### Contribution Checklist <!-- Before submitting this PR, please make sure: --> - [x] The code builds clean without any errors or warnings - [x] The PR follows the [SK Contribution Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md) and the [pre-submission formatting script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts) raises no violations - [x] All unit tests pass, and I have added new tests where possible - [x] I didn't break anyone 😄 --------- Co-authored-by: Mark Wallace <[email protected]> Co-authored-by: westey <[email protected]> Co-authored-by: Weihan Li <[email protected]> Co-authored-by: Roger Barreto <[email protected]>
- Loading branch information
5 people
authored
Oct 17, 2024
1 parent
dbf1819
commit e9499b6
Showing
38 changed files
with
5,204 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
20 changes: 20 additions & 0 deletions
20
dotnet/src/Connectors/Connectors.Memory.Sqlite/Conditions/SqliteWhereCondition.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,20 @@ | ||
| // Copyright (c) Microsoft. All rights reserved. | ||
|
|
||
| using System.Collections.Generic; | ||
|
|
||
| namespace Microsoft.SemanticKernel.Connectors.Sqlite; | ||
|
|
||
| internal abstract class SqliteWhereCondition(string operand, List<object> values) | ||
| { | ||
| public string Operand { get; set; } = operand; | ||
|
|
||
| public List<object> Values { get; set; } = values; | ||
|
|
||
| public string? TableName { get; set; } | ||
|
|
||
| public abstract string BuildQuery(List<string> parameterNames); | ||
|
|
||
| protected string GetOperand() => !string.IsNullOrWhiteSpace(this.TableName) ? | ||
| $"{this.TableName}.{this.Operand}" : | ||
| this.Operand; | ||
| } |
18 changes: 18 additions & 0 deletions
18
dotnet/src/Connectors/Connectors.Memory.Sqlite/Conditions/SqliteWhereEqualsCondition.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,18 @@ | ||
| // Copyright (c) Microsoft. All rights reserved. | ||
|
|
||
| using System.Collections.Generic; | ||
|
|
||
| namespace Microsoft.SemanticKernel.Connectors.Sqlite; | ||
|
|
||
| internal sealed class SqliteWhereEqualsCondition(string operand, object value) | ||
| : SqliteWhereCondition(operand, [value]) | ||
| { | ||
| public override string BuildQuery(List<string> parameterNames) | ||
| { | ||
| const string EqualsOperator = "="; | ||
|
|
||
| Verify.True(parameterNames.Count > 0, $"Cannot build '{nameof(SqliteWhereEqualsCondition)}' condition without parameter name."); | ||
|
|
||
| return $"{this.GetOperand()} {EqualsOperator} {parameterNames[0]}"; | ||
| } | ||
| } |
18 changes: 18 additions & 0 deletions
18
dotnet/src/Connectors/Connectors.Memory.Sqlite/Conditions/SqliteWhereInCondition.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,18 @@ | ||
| // Copyright (c) Microsoft. All rights reserved. | ||
|
|
||
| using System.Collections.Generic; | ||
|
|
||
| namespace Microsoft.SemanticKernel.Connectors.Sqlite; | ||
|
|
||
| internal sealed class SqliteWhereInCondition(string operand, List<object> values) | ||
| : SqliteWhereCondition(operand, values) | ||
| { | ||
| public override string BuildQuery(List<string> parameterNames) | ||
| { | ||
| const string InOperator = "IN"; | ||
|
|
||
| Verify.True(parameterNames.Count > 0, $"Cannot build '{nameof(SqliteWhereInCondition)}' condition without parameter names."); | ||
|
|
||
| return $"{this.GetOperand()} {InOperator} ({string.Join(", ", parameterNames)})"; | ||
| } | ||
| } |
18 changes: 18 additions & 0 deletions
18
dotnet/src/Connectors/Connectors.Memory.Sqlite/Conditions/SqliteWhereMatchCondition.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,18 @@ | ||
| // Copyright (c) Microsoft. All rights reserved. | ||
|
|
||
| using System.Collections.Generic; | ||
|
|
||
| namespace Microsoft.SemanticKernel.Connectors.Sqlite; | ||
|
|
||
| internal sealed class SqliteWhereMatchCondition(string operand, object value) | ||
| : SqliteWhereCondition(operand, [value]) | ||
| { | ||
| public override string BuildQuery(List<string> parameterNames) | ||
| { | ||
| const string MatchOperator = "MATCH"; | ||
|
|
||
| Verify.True(parameterNames.Count > 0, $"Cannot build '{nameof(SqliteWhereMatchCondition)}' condition without parameter name."); | ||
|
|
||
| return $"{this.GetOperand()} {MatchOperator} {parameterNames[0]}"; | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
27 changes: 27 additions & 0 deletions
27
dotnet/src/Connectors/Connectors.Memory.Sqlite/ISqliteVectorStoreRecordCollectionFactory.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,27 @@ | ||
| // Copyright (c) Microsoft. All rights reserved. | ||
|
|
||
| using System.Data.Common; | ||
| using Microsoft.Extensions.VectorData; | ||
|
|
||
| namespace Microsoft.SemanticKernel.Connectors.Sqlite; | ||
|
|
||
| /// <summary> | ||
| /// Interface for constructing <see cref="IVectorStoreRecordCollection{TKey, TRecord}"/> SQLite instances when using <see cref="IVectorStore"/> to retrieve these. | ||
| /// </summary> | ||
| public interface ISqliteVectorStoreRecordCollectionFactory | ||
| { | ||
| /// <summary> | ||
| /// Constructs a new instance of the <see cref="IVectorStoreRecordCollection{TKey, TRecord}"/>. | ||
| /// </summary> | ||
| /// <typeparam name="TKey">The data type of the record key.</typeparam> | ||
| /// <typeparam name="TRecord">The data model to use for adding, updating and retrieving data from storage.</typeparam> | ||
| /// <param name="connection"><see cref="DbConnection"/> that will be used to manage the data in SQLite.</param> | ||
| /// <param name="name">The name of the collection to connect to.</param> | ||
| /// <param name="vectorStoreRecordDefinition">An optional record definition that defines the schema of the record type. If not present, attributes on <typeparamref name="TRecord"/> will be used.</param> | ||
| /// <returns>The new instance of <see cref="IVectorStoreRecordCollection{TKey, TRecord}"/>.</returns> | ||
| IVectorStoreRecordCollection<TKey, TRecord> CreateVectorStoreRecordCollection<TKey, TRecord>( | ||
| DbConnection connection, | ||
| string name, | ||
| VectorStoreRecordDefinition? vectorStoreRecordDefinition) | ||
| where TKey : notnull; | ||
| } |
22 changes: 22 additions & 0 deletions
22
dotnet/src/Connectors/Connectors.Memory.Sqlite/SqliteColumn.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,22 @@ | ||
| // Copyright (c) Microsoft. All rights reserved. | ||
|
|
||
| using System.Collections.Generic; | ||
|
|
||
| namespace Microsoft.SemanticKernel.Connectors.Sqlite; | ||
|
|
||
| /// <summary> | ||
| /// Representation of SQLite column. | ||
| /// </summary> | ||
| internal sealed class SqliteColumn( | ||
| string name, | ||
| string type, | ||
| bool isPrimary) | ||
| { | ||
| public string Name { get; set; } = name; | ||
|
|
||
| public string Type { get; set; } = type; | ||
|
|
||
| public bool IsPrimary { get; set; } = isPrimary; | ||
|
|
||
| public Dictionary<string, object>? Configuration { get; set; } | ||
| } |
54 changes: 54 additions & 0 deletions
54
dotnet/src/Connectors/Connectors.Memory.Sqlite/SqliteConstants.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,54 @@ | ||
| // Copyright (c) Microsoft. All rights reserved. | ||
|
|
||
| using System; | ||
| using System.Collections.Generic; | ||
|
|
||
| namespace Microsoft.SemanticKernel.Connectors.Sqlite; | ||
|
|
||
| internal static class SqliteConstants | ||
| { | ||
| /// <summary> | ||
| /// SQLite extension name for vector search. | ||
| /// More information here: <see href="https://github.com/asg017/sqlite-vec"/>. | ||
| /// </summary> | ||
| public const string VectorSearchExtensionName = "vec0"; | ||
|
|
||
| /// <summary>A <see cref="HashSet{T}"/> of types that a key on the provided model may have.</summary> | ||
| public static readonly HashSet<Type> SupportedKeyTypes = | ||
| [ | ||
| typeof(ulong), | ||
| typeof(string) | ||
| ]; | ||
|
|
||
| /// <summary>A <see cref="HashSet{T}"/> of types that data properties on the provided model may have.</summary> | ||
| public static readonly HashSet<Type> SupportedDataTypes = | ||
| [ | ||
| typeof(int), | ||
| typeof(int?), | ||
| typeof(long), | ||
| typeof(long?), | ||
| typeof(ulong), | ||
| typeof(ulong?), | ||
| typeof(short), | ||
| typeof(short?), | ||
| typeof(ushort), | ||
| typeof(ushort?), | ||
| typeof(string), | ||
| typeof(bool), | ||
| typeof(bool?), | ||
| typeof(float), | ||
| typeof(float?), | ||
| typeof(double), | ||
| typeof(double?), | ||
| typeof(decimal), | ||
| typeof(decimal?), | ||
| typeof(byte[]), | ||
| ]; | ||
|
|
||
| /// <summary>A <see cref="HashSet{T}"/> of types that vector properties on the provided model may have.</summary> | ||
| public static readonly HashSet<Type> SupportedVectorTypes = | ||
| [ | ||
| typeof(ReadOnlyMemory<float>), | ||
| typeof(ReadOnlyMemory<float>?) | ||
| ]; | ||
| } |
152 changes: 152 additions & 0 deletions
152
dotnet/src/Connectors/Connectors.Memory.Sqlite/SqliteGenericDataModelMapper.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,152 @@ | ||
| // Copyright (c) Microsoft. All rights reserved. | ||
|
|
||
| using System; | ||
| using System.Collections.Generic; | ||
| using Microsoft.Extensions.VectorData; | ||
|
|
||
| namespace Microsoft.SemanticKernel.Connectors.Sqlite; | ||
|
|
||
| /// <summary> | ||
| /// A mapper that maps between the generic Semantic Kernel data model and the model that the data is stored under, within SQLite. | ||
| /// </summary> | ||
| internal sealed class SqliteGenericDataModelMapper : | ||
| IVectorStoreRecordMapper<VectorStoreGenericDataModel<ulong>, Dictionary<string, object?>>, | ||
| IVectorStoreRecordMapper<VectorStoreGenericDataModel<string>, Dictionary<string, object?>> | ||
| { | ||
| /// <summary><see cref="VectorStoreRecordPropertyReader"/> with helpers for reading vector store model properties and their attributes.</summary> | ||
| private readonly VectorStoreRecordPropertyReader _propertyReader; | ||
|
|
||
| /// <summary> | ||
| /// Initializes a new instance of the <see cref="SqliteGenericDataModelMapper"/> class. | ||
| /// </summary> | ||
| /// <param name="propertyReader">A <see cref="VectorStoreRecordDefinition"/> that defines the schema of the data in the database.</param> | ||
| public SqliteGenericDataModelMapper(VectorStoreRecordPropertyReader propertyReader) | ||
| { | ||
| Verify.NotNull(propertyReader); | ||
|
|
||
| this._propertyReader = propertyReader; | ||
|
|
||
| // Validate property types. | ||
| this._propertyReader.VerifyDataProperties(SqliteConstants.SupportedDataTypes, supportEnumerable: false); | ||
| this._propertyReader.VerifyVectorProperties(SqliteConstants.SupportedVectorTypes); | ||
| } | ||
|
|
||
| #region Implementation of IVectorStoreRecordMapper<VectorStoreGenericDataModel<string>, Dictionary<string, object?>> | ||
|
|
||
| public Dictionary<string, object?> MapFromDataToStorageModel(VectorStoreGenericDataModel<string> dataModel) | ||
| { | ||
| return this.InternalMapFromDataToStorageModel(dataModel); | ||
| } | ||
|
|
||
| public VectorStoreGenericDataModel<string> MapFromStorageToDataModel(Dictionary<string, object?> storageModel, StorageToDataModelMapperOptions options) | ||
| { | ||
| return this.InternalMapFromStorageToDataModel<string>(storageModel, options); | ||
| } | ||
|
|
||
| #endregion | ||
|
|
||
| #region Implementation of IVectorStoreRecordMapper<VectorStoreGenericDataModel<ulong>, Dictionary<string, object?>> | ||
|
|
||
| public Dictionary<string, object?> MapFromDataToStorageModel(VectorStoreGenericDataModel<ulong> dataModel) | ||
| { | ||
| return this.InternalMapFromDataToStorageModel(dataModel); | ||
| } | ||
|
|
||
| VectorStoreGenericDataModel<ulong> IVectorStoreRecordMapper<VectorStoreGenericDataModel<ulong>, Dictionary<string, object?>>.MapFromStorageToDataModel(Dictionary<string, object?> storageModel, StorageToDataModelMapperOptions options) | ||
| { | ||
| return this.InternalMapFromStorageToDataModel<ulong>(storageModel, options); | ||
| } | ||
|
|
||
| #endregion | ||
|
|
||
| #region private | ||
|
|
||
| private Dictionary<string, object?> InternalMapFromDataToStorageModel<TKey>(VectorStoreGenericDataModel<TKey> dataModel) | ||
| where TKey : notnull | ||
| { | ||
| var properties = new Dictionary<string, object?> | ||
| { | ||
| // Add key property | ||
| { this._propertyReader.KeyPropertyStoragePropertyName, dataModel.Key } | ||
| }; | ||
|
|
||
| // Add data properties | ||
| if (dataModel.Data is not null) | ||
| { | ||
| foreach (var property in this._propertyReader.DataProperties) | ||
| { | ||
| if (dataModel.Data.TryGetValue(property.DataModelPropertyName, out var dataValue)) | ||
| { | ||
| properties.Add(this._propertyReader.GetStoragePropertyName(property.DataModelPropertyName), dataValue); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| // Add vector properties | ||
| if (dataModel.Vectors is not null) | ||
| { | ||
| foreach (var property in this._propertyReader.VectorProperties) | ||
| { | ||
| if (dataModel.Vectors.TryGetValue(property.DataModelPropertyName, out var vectorValue)) | ||
| { | ||
| object? result = null; | ||
|
|
||
| if (vectorValue is not null) | ||
| { | ||
| var vector = (ReadOnlyMemory<float>)vectorValue; | ||
| result = SqliteVectorStoreRecordPropertyMapping.MapVectorForStorageModel(vector); | ||
| } | ||
|
|
||
| properties.Add(this._propertyReader.GetStoragePropertyName(property.DataModelPropertyName), result); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| return properties; | ||
| } | ||
|
|
||
| private VectorStoreGenericDataModel<TKey> InternalMapFromStorageToDataModel<TKey>(Dictionary<string, object?> storageModel, StorageToDataModelMapperOptions options) | ||
| where TKey : notnull | ||
| { | ||
| TKey key; | ||
| var dataProperties = new Dictionary<string, object?>(); | ||
| var vectorProperties = new Dictionary<string, object?>(); | ||
|
|
||
| // Process key property. | ||
| if (storageModel.TryGetValue(this._propertyReader.KeyPropertyStoragePropertyName, out var keyObject) && keyObject is not null) | ||
| { | ||
| key = (TKey)keyObject; | ||
| } | ||
| else | ||
| { | ||
| throw new VectorStoreRecordMappingException("No key property was found in the record retrieved from storage."); | ||
| } | ||
|
|
||
| // Process data properties. | ||
| foreach (var property in this._propertyReader.DataProperties) | ||
| { | ||
| if (storageModel.TryGetValue(this._propertyReader.GetStoragePropertyName(property.DataModelPropertyName), out var dataValue)) | ||
| { | ||
| dataProperties.Add(property.DataModelPropertyName, dataValue); | ||
| } | ||
| } | ||
|
|
||
| // Process vector properties | ||
| if (options.IncludeVectors) | ||
| { | ||
| foreach (var property in this._propertyReader.VectorProperties) | ||
| { | ||
| if (storageModel.TryGetValue(this._propertyReader.GetStoragePropertyName(property.DataModelPropertyName), out var vectorValue) && | ||
| vectorValue is byte[] vectorBytes) | ||
| { | ||
| var vector = SqliteVectorStoreRecordPropertyMapping.MapVectorForDataModel(vectorBytes); | ||
| vectorProperties.Add(property.DataModelPropertyName, vector); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| return new VectorStoreGenericDataModel<TKey>(key) { Data = dataProperties, Vectors = vectorProperties }; | ||
| } | ||
|
|
||
| #endregion | ||
| } |
Oops, something went wrong.