forked from microsoft/semantic-kernel
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Adds a WeaviateMemoryStore, which allows for memories to be stored in Weaviate.
- Loading branch information
Showing
36 changed files
with
1,689 additions
and
2 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
30 changes: 30 additions & 0 deletions
30
dotnet/src/Connectors/Connectors.Memory.Weaviate/Connectors.Memory.Weaviate.csproj
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,30 @@ | ||
| <Project Sdk="Microsoft.NET.Sdk"> | ||
|
|
||
| <PropertyGroup> | ||
| <!-- THIS PROPERTY GROUP MUST COME FIRST --> | ||
| <AssemblyName>Microsoft.SemanticKernel.Connectors.Memory.Weaviate</AssemblyName> | ||
| <RootNamespace>$(AssemblyName)</RootNamespace> | ||
| <TargetFramework>netstandard2.0</TargetFramework> | ||
| </PropertyGroup> | ||
|
|
||
| <!-- IMPORT NUGET PACKAGE SHARED PROPERTIES --> | ||
| <Import Project="$(RepoRoot)/dotnet/nuget/nuget-package.props" /> | ||
|
|
||
| <PropertyGroup> | ||
| <!-- NuGet Package Settings --> | ||
| <Title>Semantic Kernel - Weaviate Connector</Title> | ||
| <Description>Weaviate connector for Semantic Kernel skills and semantic memory</Description> | ||
| </PropertyGroup> | ||
|
|
||
| <ItemGroup> | ||
| <PackageReference Include="System.Text.Json" /> | ||
| </ItemGroup> | ||
|
|
||
| <ItemGroup> | ||
| <ProjectReference Include="..\..\SemanticKernel\SemanticKernel.csproj" /> | ||
| </ItemGroup> | ||
|
|
||
| <ItemGroup> | ||
| <Compile Include="$(RepoRoot)/dotnet/src/InternalUtilities/Linq/AsyncEnumerable.cs"/> | ||
| </ItemGroup> | ||
| </Project> |
23 changes: 23 additions & 0 deletions
23
dotnet/src/Connectors/Connectors.Memory.Weaviate/Diagnostics/NullableAttributes.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,23 @@ | ||
| // Copyright (c) Microsoft. All rights reserved. | ||
|
|
||
| // Licensed to the .NET Foundation under one or more agreements. | ||
| // The .NET Foundation licenses this file to you under the MIT license. | ||
| // See the LICENSE file in the project root for more information. | ||
|
|
||
| // This was copied from https://github.com/dotnet/runtime/blob/39b9607807f29e48cae4652cd74735182b31182e/src/libraries/System.Private.CoreLib/src/System/Diagnostics/CodeAnalysis/NullableAttributes.cs | ||
| // and updated to have the scope of the attributes be internal. | ||
|
|
||
| #pragma warning disable IDE0130 // Namespace does not match folder structure | ||
| // ReSharper disable once CheckNamespace | ||
| namespace System.Diagnostics.CodeAnalysis; | ||
| #pragma warning restore IDE0130 | ||
|
|
||
| #if !NETCOREAPP | ||
|
|
||
| /// <summary>Specifies that an output will not be null even if the corresponding type allows it.</summary> | ||
| [AttributeUsage(AttributeTargets.Field | AttributeTargets.Parameter | AttributeTargets.Property | AttributeTargets.ReturnValue, Inherited = false)] | ||
| internal sealed class NotNullAttribute : Attribute | ||
| { | ||
| } | ||
|
|
||
| #endif |
36 changes: 36 additions & 0 deletions
36
dotnet/src/Connectors/Connectors.Memory.Weaviate/Diagnostics/Verify.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,36 @@ | ||
| // Copyright (c) Microsoft. All rights reserved. | ||
|
|
||
| using System; | ||
| using System.Diagnostics.CodeAnalysis; | ||
| using System.Runtime.CompilerServices; | ||
|
|
||
| namespace Microsoft.SemanticKernel.Connectors.Memory.Weaviate.Diagnostics; | ||
|
|
||
| internal static class Verify | ||
| { | ||
| [MethodImpl(MethodImplOptions.AggressiveInlining)] | ||
| internal static void NotNull([NotNull] object? obj, string message) | ||
| { | ||
| if (obj != null) { return; } | ||
|
|
||
| throw new ArgumentNullException(null, message); | ||
| } | ||
|
|
||
| [MethodImpl(MethodImplOptions.AggressiveInlining)] | ||
| internal static void NotNullOrEmpty([NotNull] string? str, string message) | ||
| { | ||
| NotNull(str, message); | ||
| if (!string.IsNullOrWhiteSpace(str)) { return; } | ||
|
|
||
| throw new ArgumentOutOfRangeException(message); | ||
| } | ||
|
|
||
| [MethodImpl(MethodImplOptions.AggressiveInlining)] | ||
| internal static void ArgNotNullOrEmpty([NotNull] string? str, string paramName, [CallerMemberName] string? caller = default) | ||
| { | ||
| NotNull(str, paramName); | ||
| if (!string.IsNullOrWhiteSpace(str)) { return; } | ||
|
|
||
| throw new ArgumentException(paramName, $"Parameter {paramName} cannot be empty." + (!string.IsNullOrEmpty(caller) ? $"({caller})" : string.Empty)); | ||
| } | ||
| } |
134 changes: 134 additions & 0 deletions
134
dotnet/src/Connectors/Connectors.Memory.Weaviate/Diagnostics/WeaviateMemoryException.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,134 @@ | ||
| // Copyright (c) Microsoft. All rights reserved. | ||
|
|
||
| using System; | ||
| using Microsoft.SemanticKernel.Diagnostics; | ||
|
|
||
| namespace Microsoft.SemanticKernel.Connectors.Memory.Weaviate.Diagnostics; | ||
|
|
||
| #pragma warning disable RCS1194 // Implement exception constructors | ||
|
|
||
| /// <summary> | ||
| /// Exception thrown for errors related to the Weaviate connector. | ||
| /// </summary> | ||
| public class WeaviateMemoryException : SKException | ||
| { | ||
| /// <summary> | ||
| /// Initializes a new instance of the <see cref="WeaviateMemoryException"/> class with a provided error code. | ||
| /// </summary> | ||
| /// <param name="errorCode">The error code.</param> | ||
| public WeaviateMemoryException(ErrorCodes errorCode) | ||
| : this(errorCode, message: null, innerException: null) | ||
| { | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Initializes a new instance of the <see cref="WeaviateMemoryException"/> class with a provided error code and message. | ||
| /// </summary> | ||
| /// <param name="errorCode">The error code.</param> | ||
| /// <param name="message">The exception message.</param> | ||
| public WeaviateMemoryException(ErrorCodes errorCode, string? message) | ||
| : this(errorCode, message, innerException: null) | ||
| { | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Initializes a new instance of the <see cref="WeaviateMemoryException"/> class with a provided error code and inner exception. | ||
| /// </summary> | ||
| /// <param name="errorCode">The error code.</param> | ||
| /// <param name="innerException">The exception that is the cause of the current exception.</param> | ||
| public WeaviateMemoryException(ErrorCodes errorCode, Exception? innerException) | ||
| : this(errorCode, message: null, innerException) | ||
| { | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Initializes a new instance of the <see cref="WeaviateMemoryException"/> class with a provided error code, message, and inner exception. | ||
| /// </summary> | ||
| /// <param name="errorCode">The error code.</param> | ||
| /// <param name="message">A string that describes the error.</param> | ||
| /// <param name="innerException">The exception that is the cause of the current exception.</param> | ||
| public WeaviateMemoryException(ErrorCodes errorCode, string? message, Exception? innerException) | ||
| : base(GetDefaultMessage(errorCode, message, innerException), innerException) | ||
| { | ||
| this.ErrorCode = errorCode; | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Gets the error code for this exception. | ||
| /// </summary> | ||
| public ErrorCodes ErrorCode { get; } | ||
|
|
||
| /// <summary>Translate the error code into a default message.</summary> | ||
| private static string GetDefaultMessage(ErrorCodes errorCode, string? message, Exception? innerException) | ||
| { | ||
| if (message is not null) | ||
| { | ||
| return message; | ||
| } | ||
|
|
||
| string description = errorCode switch | ||
| { | ||
| ErrorCodes.FailedToUpsertVectors => "Failed to upsert vectors", | ||
| ErrorCodes.FailedToGetVectorData => "Failed to get vector data", | ||
| ErrorCodes.FailedToRemoveVectorData => "Failed to remove vector data", | ||
| ErrorCodes.CollectionNameConflict => "Naming conflict for the collection name", | ||
| ErrorCodes.FailedToCreateCollection => "Failed to create the collection", | ||
| ErrorCodes.FailedToDeleteCollection => "Failed to delete the collection", | ||
| ErrorCodes.FailedToListCollections => "Failed to list collections", | ||
| ErrorCodes.FailedToGetClass => "Failed to get class", | ||
| _ => $"Unknown error ({errorCode:G})", | ||
| }; | ||
|
|
||
| return innerException is not null ? $"{description}: {innerException.Message}" : description; | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Error codes for the Weaviate connector exceptions. | ||
| /// </summary> | ||
| public enum ErrorCodes | ||
| { | ||
| /// <summary> | ||
| /// Failed to upsert the vector. | ||
| /// </summary> | ||
| FailedToUpsertVectors, | ||
|
|
||
| /// <summary> | ||
| /// Failed to get vector data from Weaviate. | ||
| /// </summary> | ||
| FailedToGetVectorData, | ||
|
|
||
| /// <summary> | ||
| /// Failed to remove vector data from Weaviate. | ||
| /// </summary> | ||
| FailedToRemoveVectorData, | ||
|
|
||
| /// <summary> | ||
| /// Failed to create a collection. | ||
| /// </summary> | ||
| FailedToCreateCollection, | ||
|
|
||
| // ReSharper disable once CommentTypo | ||
| /// <summary> | ||
| /// Naming conflict for the collection name. | ||
| /// For example a collectionName of '__this_collection' and 'this_collection' are | ||
| /// both transformed to the class name of SKthiscollection - even though | ||
| /// semantic kernel would consider them as unique collection names. | ||
| /// </summary> | ||
| CollectionNameConflict, | ||
|
|
||
| /// <summary> | ||
| /// Failed to delete a collection. | ||
| /// </summary> | ||
| FailedToDeleteCollection, | ||
|
|
||
| /// <summary> | ||
| /// Failed to list collections. | ||
| /// </summary> | ||
| FailedToListCollections, | ||
|
|
||
| /// <summary> | ||
| /// Failed to get a Weaviate class. | ||
| /// </summary> | ||
| FailedToGetClass | ||
| } | ||
| } |
66 changes: 66 additions & 0 deletions
66
dotnet/src/Connectors/Connectors.Memory.Weaviate/Http/ApiSchema/BatchRequest.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,66 @@ | ||
| // Copyright (c) Microsoft. All rights reserved. | ||
|
|
||
| using System.Collections.Generic; | ||
| using System.Linq; | ||
| using System.Net.Http; | ||
| using Microsoft.SemanticKernel.Connectors.Memory.Weaviate.Model; | ||
| using Microsoft.SemanticKernel.Memory; | ||
|
|
||
| namespace Microsoft.SemanticKernel.Connectors.Memory.Weaviate.Http.ApiSchema; | ||
|
|
||
| internal sealed class BatchRequest | ||
| { | ||
| private readonly string _class; | ||
|
|
||
| private BatchRequest(string @class) | ||
| { | ||
| this._class = @class; | ||
| this.Objects = new(); | ||
| } | ||
|
|
||
| // ReSharper disable once UnusedMember.Global | ||
| public string[] Fields { get; } = { "ALL" }; | ||
|
|
||
| // ReSharper disable once MemberCanBePrivate.Global | ||
| // ReSharper disable once CollectionNeverQueried.Global | ||
| public List<WeaviateObject> Objects { get; set; } | ||
|
|
||
| public static BatchRequest Create(string @class) | ||
| { | ||
| return new(@class); | ||
| } | ||
|
|
||
| public void Add(MemoryRecord record) | ||
| { | ||
| record.Key = ToWeaviateFriendlyId(record.Metadata.Id); | ||
|
|
||
| WeaviateObject weaviateObject = new() | ||
| { | ||
| Class = this._class, | ||
| Id = record.Key, | ||
| Vector = record.Embedding.Vector.ToArray(), | ||
| Properties = new() | ||
| { | ||
| { "sk_timestamp", record.Timestamp! }, | ||
| { "sk_id", record.Metadata.Id }, | ||
| { "sk_description", record.Metadata.Description }, | ||
| { "sk_text", record.Metadata.Text }, | ||
| { "sk_additional_metadata", record.Metadata.AdditionalMetadata } | ||
| } | ||
| }; | ||
|
|
||
| this.Objects.Add(weaviateObject); | ||
| } | ||
|
|
||
| private static string ToWeaviateFriendlyId(string id) | ||
| { | ||
| return $"{id.Trim().Replace(' ', '-').Replace('/', '_').Replace('\\', '_').Replace('?', '_').Replace('#', '_')}"; | ||
| } | ||
|
|
||
| public HttpRequestMessage Build() | ||
| { | ||
| return HttpRequest.CreatePostRequest( | ||
| "batch/objects", | ||
| this); | ||
| } | ||
| } |
25 changes: 25 additions & 0 deletions
25
dotnet/src/Connectors/Connectors.Memory.Weaviate/Http/ApiSchema/BatchResponse.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,25 @@ | ||
| // Copyright (c) Microsoft. All rights reserved. | ||
|
|
||
| using System; | ||
| using System.Text.Json.Serialization; | ||
| using Microsoft.SemanticKernel.Connectors.Memory.Weaviate.Http.JsonConverter; | ||
| using Microsoft.SemanticKernel.Connectors.Memory.Weaviate.Model; | ||
|
|
||
| namespace Microsoft.SemanticKernel.Connectors.Memory.Weaviate.Http.ApiSchema; | ||
|
|
||
| // ReSharper disable once ClassNeverInstantiated.Global | ||
| #pragma warning disable CA1812 // 'BatchResponse' is an internal class that is apparently never instantiated. If so, remove the code from the assembly. If this class is intended to contain only static members, make it 'static' (Module in Visual Basic). | ||
| internal sealed class BatchResponse : WeaviateObject | ||
| #pragma warning restore CA1812 // 'BatchResponse' is an internal class that is apparently never instantiated. If so, remove the code from the assembly. If this class is intended to contain only static members, make it 'static' (Module in Visual Basic). | ||
| { | ||
| public Deprecation[]? Deprecations { get; set; } | ||
| public ObjectResponseResult? Result { get; set; } | ||
|
|
||
| [JsonConverter(typeof(UnixSecondsDateTimeJsonConverter))] | ||
| [JsonPropertyName("creationTimeUnix")] | ||
| public DateTime? CreationTime { get; set; } | ||
|
|
||
| [JsonConverter(typeof(UnixSecondsDateTimeJsonConverter))] | ||
| [JsonPropertyName("lastUpdateTimeUnix")] | ||
| public DateTime? LastUpdateTime { get; set; } | ||
| } |
Oops, something went wrong.