forked from microsoft/semantic-kernel
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Pinecone memory connector (microsoft#1065)
Pinecone vector database implementation Original PR: microsoft#770 --------- Co-authored-by: Kevdome3000 <[email protected]> Co-authored-by: tech <[email protected]> Co-authored-by: Shawn Callegari <[email protected]> Co-authored-by: Abby Harrison <[email protected]>
- Loading branch information
5 people
authored
May 18, 2023
1 parent
120e400
commit ca8d614
Showing
44 changed files
with
4,727 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
27 changes: 27 additions & 0 deletions
27
dotnet/src/Connectors/Connectors.Memory.Pinecone/Connectors.Memory.Pinecone.csproj
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,27 @@ | ||
| <Project Sdk="Microsoft.NET.Sdk"> | ||
|
|
||
| <PropertyGroup> | ||
| <!-- THIS PROPERTY GROUP MUST COME FIRST --> | ||
| <AssemblyName>Microsoft.SemanticKernel.Connectors.Memory.Pinecone</AssemblyName> | ||
| <RootNamespace>$(AssemblyName)</RootNamespace> | ||
| <TargetFramework>netstandard2.0</TargetFramework> | ||
| </PropertyGroup> | ||
|
|
||
| <!-- IMPORT NUGET PACKAGE SHARED PROPERTIES --> | ||
| <Import Project="$(RepoRoot)/dotnet/nuget/nuget-package.props" /> | ||
|
|
||
| <PropertyGroup> | ||
| <!-- NuGet Package Settings --> | ||
| <Title>Semantic Kernel - Pinecone Connector</Title> | ||
| <Description>Pinecone connector for Semantic Kernel skills and semantic memory</Description> | ||
| </PropertyGroup> | ||
|
|
||
| <ItemGroup> | ||
| <PackageReference Include="System.Text.Json" /> | ||
| </ItemGroup> | ||
|
|
||
| <ItemGroup> | ||
| <ProjectReference Include="..\..\SemanticKernel\SemanticKernel.csproj" /> | ||
| </ItemGroup> | ||
|
|
||
| </Project> |
69 changes: 69 additions & 0 deletions
69
dotnet/src/Connectors/Connectors.Memory.Pinecone/Http/ApiSchema/ConfigureIndexRequest.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,69 @@ | ||
| // Copyright (c) Microsoft. All rights reserved. | ||
|
|
||
| using System.Net.Http; | ||
| using System.Text.Json.Serialization; | ||
| using Microsoft.SemanticKernel.Connectors.Memory.Pinecone.Model; | ||
|
|
||
| namespace Microsoft.SemanticKernel.Connectors.Memory.Pinecone.Http.ApiSchema; | ||
|
|
||
| /// <summary> | ||
| /// This operation specifies the pod type and number of replicas for an index. | ||
| /// See https://docs.pinecone.io/reference/configure_index | ||
| /// </summary> | ||
| internal sealed class ConfigureIndexRequest | ||
| { | ||
| public string IndexName { get; set; } | ||
|
|
||
| /// <summary> | ||
| /// Gets or Sets PodType | ||
| /// </summary> | ||
| [JsonPropertyName("pod_type")] | ||
| public PodType PodType { get; set; } | ||
|
|
||
| /// <summary> | ||
| /// The desired number of replicas for the index. | ||
| /// </summary> | ||
| /// <value>The desired number of replicas for the index.</value> | ||
| [JsonPropertyName("replicas")] | ||
| public int Replicas { get; set; } | ||
|
|
||
| public static ConfigureIndexRequest Create(string indexName) | ||
| { | ||
| return new ConfigureIndexRequest(indexName); | ||
| } | ||
|
|
||
| public ConfigureIndexRequest WithPodType(PodType podType) | ||
| { | ||
| this.PodType = podType; | ||
| return this; | ||
| } | ||
|
|
||
| public ConfigureIndexRequest NumberOfReplicas(int replicas) | ||
| { | ||
| this.Replicas = replicas; | ||
| return this; | ||
| } | ||
|
|
||
| public HttpRequestMessage Build() | ||
| { | ||
| HttpRequestMessage? request = HttpRequest.CreatePatchRequest( | ||
| $"/databases/{this.IndexName}", this); | ||
|
|
||
| request.Headers.Add("accept", "text/plain"); | ||
|
|
||
| return request; | ||
| } | ||
|
|
||
| #region private ================================================================================ | ||
|
|
||
| /// <summary> | ||
| /// Initializes a new instance of the <see cref="ConfigureIndexRequest" /> class. | ||
| /// </summary> | ||
| /// <param name="indexName"> Name of the index.</param> | ||
| private ConfigureIndexRequest(string indexName) | ||
| { | ||
| this.IndexName = indexName; | ||
| } | ||
|
|
||
| #endregion | ||
| } |
38 changes: 38 additions & 0 deletions
38
dotnet/src/Connectors/Connectors.Memory.Pinecone/Http/ApiSchema/DeleteIndexRequest.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,38 @@ | ||
| // Copyright (c) Microsoft. All rights reserved. | ||
|
|
||
| using System.Net.Http; | ||
|
|
||
| namespace Microsoft.SemanticKernel.Connectors.Memory.Pinecone.Http.ApiSchema; | ||
|
|
||
| /// <summary> | ||
| /// Deletes an index and all its data. | ||
| /// See https://docs.pinecone.io/reference/delete_index | ||
| /// </summary> | ||
| internal sealed class DeleteIndexRequest | ||
| { | ||
| public static DeleteIndexRequest Create(string indexName) | ||
| { | ||
| return new DeleteIndexRequest(indexName); | ||
| } | ||
|
|
||
| public HttpRequestMessage Build() | ||
| { | ||
| HttpRequestMessage request = HttpRequest.CreateDeleteRequest( | ||
| $"/databases/{this._indexName}"); | ||
|
|
||
| request.Headers.Add("accept", "text/plain"); | ||
|
|
||
| return request; | ||
| } | ||
|
|
||
| #region private ================================================================================ | ||
|
|
||
| private readonly string _indexName; | ||
|
|
||
| private DeleteIndexRequest(string indexName) | ||
| { | ||
| this._indexName = indexName; | ||
| } | ||
|
|
||
| #endregion | ||
| } |
147 changes: 147 additions & 0 deletions
147
dotnet/src/Connectors/Connectors.Memory.Pinecone/Http/ApiSchema/DeleteRequest.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,147 @@ | ||
| // Copyright (c) Microsoft. All rights reserved. | ||
|
|
||
| using System.Collections.Generic; | ||
| using System.Linq; | ||
| using System.Net.Http; | ||
| using System.Text; | ||
| using System.Text.Json.Serialization; | ||
|
|
||
| namespace Microsoft.SemanticKernel.Connectors.Memory.Pinecone.Http.ApiSchema; | ||
|
|
||
| /// <summary> | ||
| /// DeleteRequest | ||
| /// See https://docs.pinecone.io/reference/delete_post | ||
| /// </summary> | ||
| internal sealed class DeleteRequest | ||
| { | ||
| /// <summary> | ||
| /// The ids of the vectors to delete | ||
| /// </summary> | ||
| [JsonPropertyName("ids")] | ||
| public IEnumerable<string>? Ids { get; set; } | ||
|
|
||
| /// <summary> | ||
| /// Whether to delete all vectors | ||
| /// </summary> | ||
| [JsonPropertyName("deleteAll")] | ||
| [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] | ||
| public bool? DeleteAll { get; set; } | ||
|
|
||
| /// <summary> | ||
| /// The namespace to delete vectors from | ||
| /// </summary> | ||
| [JsonPropertyName("namespace")] | ||
| [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] | ||
| public string? Namespace { get; set; } | ||
|
|
||
| /// <summary> | ||
| /// If this parameter is present, the operation only affects vectors that satisfy the filter. See https://www.pinecone.io/docs/metadata-filtering/. | ||
| /// </summary> | ||
| [JsonPropertyName("filter")] | ||
| [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] | ||
| public Dictionary<string, object>? Filter { get; set; } | ||
|
|
||
| public static DeleteRequest GetDeleteAllVectorsRequest() | ||
| { | ||
| return new DeleteRequest(true); | ||
| } | ||
|
|
||
| public static DeleteRequest ClearNamespace(string indexNamespace) | ||
| { | ||
| return new DeleteRequest(true) | ||
| { | ||
| Namespace = indexNamespace | ||
| }; | ||
| } | ||
|
|
||
| public static DeleteRequest DeleteVectors(IEnumerable<string>? ids) | ||
| { | ||
| return new DeleteRequest(ids); | ||
| } | ||
|
|
||
| public DeleteRequest FilterBy(Dictionary<string, object>? filter) | ||
| { | ||
| this.Filter = filter; | ||
| return this; | ||
| } | ||
|
|
||
| public DeleteRequest FromNamespace(string? indexNamespace) | ||
| { | ||
| this.Namespace = indexNamespace; | ||
| return this; | ||
| } | ||
|
|
||
| public DeleteRequest Clear(bool deleteAll) | ||
| { | ||
| this.DeleteAll = deleteAll; | ||
| return this; | ||
| } | ||
|
|
||
| public HttpRequestMessage Build() | ||
| { | ||
| if (this.Filter != null) | ||
| { | ||
| this.Filter = PineconeUtils.ConvertFilterToPineconeFilter(this.Filter); | ||
| } | ||
|
|
||
| HttpRequestMessage? request = HttpRequest.CreatePostRequest( | ||
| "/vectors/delete", | ||
| this); | ||
|
|
||
| request.Headers.Add("accept", "application/json"); | ||
|
|
||
| return request; | ||
| } | ||
|
|
||
| /// <inheritdoc /> | ||
| public override string ToString() | ||
| { | ||
| var sb = new StringBuilder(); | ||
|
|
||
| sb.Append("DeleteRequest: "); | ||
|
|
||
| if (this.Ids != null) | ||
| { | ||
| sb.Append($"Deleting {this.Ids.Count()} vectors, {string.Join(", ", this.Ids)},"); | ||
| } | ||
|
|
||
| if (this.DeleteAll != null) | ||
| { | ||
| sb.Append("Deleting All vectors,"); | ||
| } | ||
|
|
||
| if (this.Namespace != null) | ||
| { | ||
| sb.Append($"From Namespace: {this.Namespace}, "); | ||
| } | ||
|
|
||
| if (this.Filter == null) | ||
| { | ||
| return sb.ToString(); | ||
| } | ||
|
|
||
| sb.Append("With Filter: "); | ||
|
|
||
| foreach (var pair in this.Filter) | ||
| { | ||
| sb.Append($"{pair.Key}={pair.Value}, "); | ||
| } | ||
|
|
||
| return sb.ToString(); | ||
| } | ||
|
|
||
| #region private ================================================================================ | ||
|
|
||
| private DeleteRequest(IEnumerable<string>? ids) | ||
| { | ||
| this.Ids = ids ?? new List<string>(); | ||
| } | ||
|
|
||
| private DeleteRequest(bool clear) | ||
| { | ||
| this.Ids = new List<string>(); | ||
| this.DeleteAll = clear; | ||
| } | ||
|
|
||
| #endregion | ||
| } |
41 changes: 41 additions & 0 deletions
41
dotnet/src/Connectors/Connectors.Memory.Pinecone/Http/ApiSchema/DescribeIndexRequest.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,41 @@ | ||
| // Copyright (c) Microsoft. All rights reserved. | ||
|
|
||
| using System.Net.Http; | ||
|
|
||
| namespace Microsoft.SemanticKernel.Connectors.Memory.Pinecone.Http.ApiSchema; | ||
|
|
||
| /// <summary> | ||
| /// Get information about an index. | ||
| /// See https://docs.pinecone.io/reference/describe_index | ||
| /// </summary> | ||
| internal sealed class DescribeIndexRequest | ||
| { | ||
| /// <summary> | ||
| /// The unique name of an index. | ||
| /// </summary> | ||
| public string IndexName { get; } | ||
|
|
||
| public static DescribeIndexRequest Create(string indexName) | ||
| { | ||
| return new DescribeIndexRequest(indexName); | ||
| } | ||
|
|
||
| public HttpRequestMessage Build() | ||
| { | ||
| HttpRequestMessage? request = HttpRequest.CreateGetRequest( | ||
| $"/databases/{this.IndexName}"); | ||
|
|
||
| request.Headers.Add("accept", "application/json"); | ||
|
|
||
| return request; | ||
| } | ||
|
|
||
| #region private ================================================================================ | ||
|
|
||
| private DescribeIndexRequest(string indexName) | ||
| { | ||
| this.IndexName = indexName; | ||
| } | ||
|
|
||
| #endregion | ||
| } |
Oops, something went wrong.