Update SharpVector.OpenAI to 2.0.0 with save/load functionality support

CHANGELOG.md

@@ -5,11 +5,11 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## v2.0.0 (In Progress)
+## v2.0.0
 
 Added:
 
-- Add data persistence capability to save/load from a file or to/from a `Stream`
+- Add data persistence capability to save/load from a file or to/from a `Stream` (Both SharpVector and SharpVector.OpenAI)
 - Add Chinese language/character support
 
 Breaking Change:

src/Build5Nines.SharpVector.OpenAI/OpenAIMemoryVectorDatabaseBase.cs

@@ -3,13 +3,15 @@
 using Build5Nines.SharpVector.VectorStore;
 using System.Collections.Concurrent;
 using OpenAI.Embeddings;
+using System.IO.Compression;
+using System.Text.Json;
 
 namespace Build5Nines.SharpVector.OpenAI;
 
 public abstract class OpenAIMemoryVectorDatabaseBase<TId, TMetadata, TVectorStore, TIdGenerator, TVectorComparer>
     : IVectorDatabase<TId, TMetadata>
     where TId : notnull
-    where TVectorStore : IVectorStore<TId, TMetadata>
+    where TVectorStore : IVectorStore<TId, TMetadata, string>
     where TIdGenerator : IIdGenerator<TId>, new()
     where TVectorComparer : IVectorComparer, new()
 {
@@ -76,7 +78,7 @@ public IEnumerable<TId> GetIds()
     /// <param name="id"></param>
     /// <returns></returns>
     /// <exception cref="KeyNotFoundException"></exception>
-    public IVectorTextItem<TMetadata> GetText(TId id)
+    public IVectorTextItem<string, TMetadata> GetText(TId id)
     {
         return VectorStore.Get(id);
     }
@@ -86,7 +88,7 @@ public IVectorTextItem<TMetadata> GetText(TId id)
     /// </summary>
     /// <param name="id"></param>
     /// <exception cref="KeyNotFoundException"></exception>
-    public IVectorTextItem<TMetadata> DeleteText(TId id)
+    public IVectorTextItem<string, TMetadata> DeleteText(TId id)
     {
         return VectorStore.Delete(id);
     }
@@ -193,7 +195,7 @@ public void UpdateTextAndMetadata(TId id, string text, TMetadata metadata)
     /// <param name="topN">The highest number of results to show.</param>
     /// <param name="threshold">The similarity threshold. Only return items greater or equal to the threshold. Null returns all.</param>
     /// <returns></returns>
-    public IVectorTextResult<TMetadata> Search(string queryText, float? threshold = null, int pageIndex = 0, int? pageCount = null)
+    public IVectorTextResult<string, TMetadata> Search(string queryText, float? threshold = null, int pageIndex = 0, int? pageCount = null)
     {
         return SearchAsync(queryText, threshold, pageIndex, pageCount).Result;
     }
@@ -206,7 +208,7 @@ public IVectorTextResult<TMetadata> Search(string queryText, float? threshold =
     /// <param name="pageIndex">The page index of the search results. Default is 0.</param>
     /// <param name="pageCount">The number of search results per page. Default is Null and returns all results.</param>
     /// <returns></returns>
-    public async Task<IVectorTextResult<TMetadata>> SearchAsync(string queryText, float? threshold = null, int pageIndex = 0, int? pageCount = null)
+    public async Task<IVectorTextResult<string, TMetadata>> SearchAsync(string queryText, float? threshold = null, int pageIndex = 0, int? pageCount = null)
     {
         var similarities = await CalculateVectorComparisonAsync(queryText, threshold);
 
@@ -260,10 +262,7 @@ public virtual async Task SerializeToJsonStreamAsync(Stream stream)
         var streamVectorStore = new MemoryStream();
         var streamVocabularyStore = new MemoryStream();
 
-        var taskVectorStore = VectorStore.SerializeToJsonStreamAsync(streamVectorStore);
-        var taskVocabularyStore = VectorStore.VocabularyStore.SerializeToJsonStreamAsync(streamVocabularyStore);
-
-        await Task.WhenAll(taskVectorStore, taskVocabularyStore);
+        await VectorStore.SerializeToJsonStreamAsync(streamVectorStore);
 
         using (var archive = new ZipArchive(stream, ZipArchiveMode.Create, true))
         {
@@ -330,19 +329,22 @@ public virtual async Task DeserializeFromJsonStreamAsync(Stream stream)
 
                     var databaseInfo = JsonSerializer.Deserialize<DatabaseInfo>(databaseInfoJson);
 
+                    string SupportedVersion = "1.0.0";
+                    string SupportedSchema = "Build5Nines.SharpVector";
+
                     if (databaseInfo == null)
                     {
                         throw new DatabaseFileInfoException("Database info entry is null.");
                     }
 
-                    if (databaseInfo.Schema != DatabaseInfo.SupportedSchema)
+                    if (databaseInfo.Schema != SupportedSchema)
                     {
-                        throw new DatabaseFileSchemaException($"The database schema does not match the expected schema (Expected: {DatabaseInfo.SupportedSchema} - Actual: {databaseInfo.Schema})."); 
+                        throw new DatabaseFileSchemaException($"The database schema does not match the expected schema (Expected: {SupportedSchema} - Actual: {databaseInfo.Schema})."); 
                     }
 
-                    if (databaseInfo.Version != DatabaseInfo.SupportedVersion)
+                    if (databaseInfo.Version != SupportedVersion)
                     {
-                        throw new DatabaseFileVersionException($"The database version does not match the expected version (Expected: {DatabaseInfo.SupportedVersion} - Actual: {databaseInfo.Version}).");
+                        throw new DatabaseFileVersionException($"The database version does not match the expected version (Expected: {SupportedVersion} - Actual: {databaseInfo.Version}).");
                     }
 
                     if (databaseInfo.ClassType != this.GetType().FullName)

src/OpenAIConsoleTest/Program.cs

@@ -77,15 +77,12 @@ await Parallel.ForEachAsync(movies.EnumerateArray(), async (movie, cancellationT
     Console.WriteLine(string.Empty);
 
     if (newPrompt != null) {
-        IVectorTextResult<string> result;
-
         var timer = new Stopwatch();
         timer.Start();
 
-
         var pageSize = 3;
         // result = await vdb.Search(newPrompt,
-        result = await vdb.SearchAsync(newPrompt,
+        var result = await vdb.SearchAsync(newPrompt,
             threshold: 0.001f, // 0.2f, // Cosine Similarity - Only return results with similarity greater than this threshold
             // threshold: (float)1.4f, // Euclidean Distance - Only return results with distance less than this threshold
 

src/SharpVectorOpenAITest/BasicOpenAIMemoryVectorDatabaseTest.cs

@@ -29,5 +29,14 @@ public void TestInitialization()
             Assert.IsNotNull(_database);
         }
 
+        [TestMethod]
+        public async Task Test_SaveLoad_01()
+        {
+            var filename = "openai_test_saveload_01.b59vdb";
+            await _database.SaveToFileAsync(filename);
+
+            await _database.LoadFromFileAsync(filename);
+        }
+
     }
 }