From 63db2cb06e53c6873303abe316619eb955f5af5d Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Thu, 19 Dec 2024 17:12:52 +0000 Subject: [PATCH 01/31] add ServerObjectManagerFactory --- .../Serialisation/V2/SerializeProcessFactory.cs | 11 ++++------- .../Serialisation/V2/ServerObjectManagerFactory.cs | 13 +++++++++++++ 2 files changed, 17 insertions(+), 7 deletions(-) create mode 100644 src/Speckle.Sdk/Serialisation/V2/ServerObjectManagerFactory.cs diff --git a/src/Speckle.Sdk/Serialisation/V2/SerializeProcessFactory.cs b/src/Speckle.Sdk/Serialisation/V2/SerializeProcessFactory.cs index 36cc08af..119e1a18 100644 --- a/src/Speckle.Sdk/Serialisation/V2/SerializeProcessFactory.cs +++ b/src/Speckle.Sdk/Serialisation/V2/SerializeProcessFactory.cs @@ -1,5 +1,3 @@ -using Speckle.Sdk.Helpers; -using Speckle.Sdk.Logging; using Speckle.Sdk.Serialisation.V2.Receive; using Speckle.Sdk.Serialisation.V2.Send; using Speckle.Sdk.SQLite; @@ -31,12 +29,11 @@ public ISerializeProcess CreateSerializeProcess( } public class SerializeProcessFactory( - ISpeckleHttp speckleHttp, - ISdkActivityFactory activityFactory, IBaseChildFinder baseChildFinder, IObjectSerializerFactory objectSerializerFactory, IObjectDeserializerFactory objectDeserializerFactory, - ISqLiteJsonCacheManagerFactory sqLiteJsonCacheManagerFactory + ISqLiteJsonCacheManagerFactory sqLiteJsonCacheManagerFactory, + IServerObjectManagerFactory serverObjectManagerFactory ) : ISerializeProcessFactory { public ISerializeProcess CreateSerializeProcess( @@ -48,7 +45,7 @@ public ISerializeProcess CreateSerializeProcess( ) { var sqLiteJsonCacheManager = sqLiteJsonCacheManagerFactory.CreateFromStream(streamId); - var serverObjectManager = new ServerObjectManager(speckleHttp, activityFactory, url, streamId, authorizationToken); + var serverObjectManager = serverObjectManagerFactory.Create(url, streamId, authorizationToken); return new SerializeProcess( progress, sqLiteJsonCacheManager, @@ -85,7 +82,7 @@ public IDeserializeProcess CreateDeserializeProcess( ) { var sqLiteJsonCacheManager = sqLiteJsonCacheManagerFactory.CreateFromStream(streamId); - var serverObjectManager = new ServerObjectManager(speckleHttp, activityFactory, url, streamId, authorizationToken); + var serverObjectManager = serverObjectManagerFactory.Create(url, streamId, authorizationToken); var objectLoader = new ObjectLoader(sqLiteJsonCacheManager, serverObjectManager, progress); return new DeserializeProcess(progress, objectLoader, objectDeserializerFactory, options); diff --git a/src/Speckle.Sdk/Serialisation/V2/ServerObjectManagerFactory.cs b/src/Speckle.Sdk/Serialisation/V2/ServerObjectManagerFactory.cs new file mode 100644 index 00000000..071a62b5 --- /dev/null +++ b/src/Speckle.Sdk/Serialisation/V2/ServerObjectManagerFactory.cs @@ -0,0 +1,13 @@ +using Speckle.InterfaceGenerator; +using Speckle.Sdk.Helpers; +using Speckle.Sdk.Logging; + +namespace Speckle.Sdk.Serialisation.V2; + +[GenerateAutoInterface] +public class ServerObjectManagerFactory(ISpeckleHttp speckleHttp, ISdkActivityFactory activityFactory) + : IServerObjectManagerFactory +{ + public IServerObjectManager Create(Uri url, string streamId, string? authorizationToken, int timeoutSeconds = 120) => + new ServerObjectManager(speckleHttp, activityFactory, url, streamId, authorizationToken, timeoutSeconds); +} From 1b84c6cea76a628f4657bc830abe9195bdbfca17 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Fri, 20 Dec 2024 09:26:36 +0000 Subject: [PATCH 02/31] add usage of a command pool --- src/Speckle.Sdk/SQLite/CacheDbCommandPool.cs | 103 ++++++++++ src/Speckle.Sdk/SQLite/CacheDbCommands.cs | 35 ++++ .../SQLite/SQLiteJsonCacheManager.cs | 180 +++++++++--------- 3 files changed, 233 insertions(+), 85 deletions(-) create mode 100644 src/Speckle.Sdk/SQLite/CacheDbCommandPool.cs create mode 100644 src/Speckle.Sdk/SQLite/CacheDbCommands.cs diff --git a/src/Speckle.Sdk/SQLite/CacheDbCommandPool.cs b/src/Speckle.Sdk/SQLite/CacheDbCommandPool.cs new file mode 100644 index 00000000..3144aa2e --- /dev/null +++ b/src/Speckle.Sdk/SQLite/CacheDbCommandPool.cs @@ -0,0 +1,103 @@ +using System.Collections.Concurrent; +using Microsoft.Data.Sqlite; + +namespace Speckle.Sdk.SQLite; + +//inspired by https://github.com/neosmart/SqliteCache/blob/master/SqliteCache/DbCommandPool.cs +public sealed class CacheDbCommandPool : IDisposable +{ + private const int INITIAL_CONCURRENCY = 4; + private readonly ConcurrentBag[] _commands = new ConcurrentBag[CacheDbCommands.Count]; + private readonly ConcurrentBag _connections = new(); + private readonly string _connectionString; + + public CacheDbCommandPool(string connectionString) + { + _connectionString = connectionString; + for (int i = 0; i < _commands.Length; ++i) + { + _commands[i] = new ConcurrentBag(); + } + for (int i = 0; i < INITIAL_CONCURRENCY; ++i) + { + var connection = new SqliteConnection(_connectionString); + connection.Open(); + _connections.Add(connection); + } + } + + public void Use(CacheOperation type, Action handler) + { + Use( + type, + (cmd) => + { + handler(cmd); + return true; + } + ); + } + + public T Use(Func handler) + { + if (!_connections.TryTake(out var db)) + { + db = new SqliteConnection(_connectionString); + db.Open(); + } + + try + { + return handler(db); + } + finally + { + _connections.Add(db); + } + } + + public T Use(CacheOperation type, Func handler) + { + return Use( + (conn) => + { + var pool = _commands[(int)type]; + if (!pool.TryTake(out var command)) + { +#pragma warning disable CA2100 + command = new SqliteCommand(CacheDbCommands.Commands[(int)type], conn); +#pragma warning restore CA2100 + } + + try + { + command.Connection = conn; + return handler(command); + } + finally + { + command.Connection = null; + command.Parameters.Clear(); + pool.Add(command); + } + } + ); + } + + public void Dispose() + { + foreach (var pool in _commands) + { + while (pool.TryTake(out var cmd)) + { + cmd.Dispose(); + } + } + + foreach (var conn in _connections) + { + conn.Close(); + conn.Dispose(); + } + } +} diff --git a/src/Speckle.Sdk/SQLite/CacheDbCommands.cs b/src/Speckle.Sdk/SQLite/CacheDbCommands.cs new file mode 100644 index 00000000..ee3aae47 --- /dev/null +++ b/src/Speckle.Sdk/SQLite/CacheDbCommands.cs @@ -0,0 +1,35 @@ +namespace Speckle.Sdk.SQLite; + +public enum CacheOperation +{ + InsertOrIgnore, + InsertOrReplace, + Has, + Get, + Delete, + GetAll, + BulkInsertOrIgnore, +} + +public static class CacheDbCommands +{ + public static readonly string[] Commands; + public static readonly int Count = Enum.GetValues(typeof(CacheOperation)).Length; + +#pragma warning disable CA1810 + static CacheDbCommands() +#pragma warning restore CA1810 + { + Commands = new string[Count]; + + Commands[(int)CacheOperation.InsertOrIgnore] = + "INSERT OR IGNORE INTO objects(hash, content) VALUES(@hash, @content)"; + Commands[(int)CacheOperation.InsertOrReplace] = "REPLACE INTO objects(hash, content) VALUES(@hash, @content)"; + Commands[(int)CacheOperation.Has] = "SELECT 1 FROM objects WHERE hash = @hash LIMIT 1"; + Commands[(int)CacheOperation.Get] = "SELECT * FROM objects WHERE hash = @hash LIMIT 1"; + Commands[(int)CacheOperation.Delete] = "DELETE FROM objects WHERE hash = @hash"; + Commands[(int)CacheOperation.GetAll] = "SELECT * FROM objects"; + + Commands[(int)CacheOperation.BulkInsertOrIgnore] = "INSERT OR IGNORE INTO objects (hash, content) VALUES "; + } +} diff --git a/src/Speckle.Sdk/SQLite/SQLiteJsonCacheManager.cs b/src/Speckle.Sdk/SQLite/SQLiteJsonCacheManager.cs index 8ffffddd..f0bf82e4 100644 --- a/src/Speckle.Sdk/SQLite/SQLiteJsonCacheManager.cs +++ b/src/Speckle.Sdk/SQLite/SQLiteJsonCacheManager.cs @@ -1,19 +1,24 @@ +using System.Text; using Microsoft.Data.Sqlite; using Speckle.InterfaceGenerator; namespace Speckle.Sdk.SQLite; [GenerateAutoInterface] -public class SqLiteJsonCacheManager : ISqLiteJsonCacheManager +public sealed class SqLiteJsonCacheManager : ISqLiteJsonCacheManager, IDisposable { private readonly string _connectionString; + private readonly CacheDbCommandPool _pool; public SqLiteJsonCacheManager(string rootPath) { _connectionString = $"Data Source={rootPath};"; Initialize(); + _pool = new CacheDbCommandPool(_connectionString); } + public void Dispose() => _pool.Dispose(); + private void Initialize() { // NOTE: used for creating partioned object tables. @@ -59,98 +64,103 @@ content TEXT cmd4.ExecuteNonQuery(); } - public IEnumerable GetAllObjects() - { - using var c = new SqliteConnection(_connectionString); - c.Open(); - using var command = new SqliteCommand("SELECT * FROM objects", c); - - using var reader = command.ExecuteReader(); - while (reader.Read()) - { - yield return reader.GetString(1); - } - } - - public void DeleteObject(string id) - { - using var c = new SqliteConnection(_connectionString); - c.Open(); - using var command = new SqliteCommand("DELETE FROM objects WHERE hash = @hash", c); - command.Parameters.AddWithValue("@hash", id); - command.ExecuteNonQuery(); - } - - public string? GetObject(string id) - { - using var c = new SqliteConnection(_connectionString); - c.Open(); - using var command = new SqliteCommand("SELECT * FROM objects WHERE hash = @hash LIMIT 1 ", c); - command.Parameters.AddWithValue("@hash", id); - using var reader = command.ExecuteReader(); - if (reader.Read()) - { - return reader.GetString(1); - } - - return null; // pass on the duty of null checks to consumers - } + public IReadOnlyCollection GetAllObjects() => + _pool.Use( + CacheOperation.GetAll, + command => + { + var list = new HashSet(); + using var reader = command.ExecuteReader(); + while (reader.Read()) + { + list.Add(reader.GetString(1)); + } + return list; + } + ); + + public void DeleteObject(string id) => + _pool.Use( + CacheOperation.Delete, + command => + { + command.Parameters.AddWithValue("@hash", id); + command.ExecuteNonQuery(); + } + ); + + public string? GetObject(string id) => + _pool.Use( + CacheOperation.Get, + command => + { + command.Parameters.AddWithValue("@hash", id); + return (string?)command.ExecuteScalar(); + } + ); //This does an insert or ignores if already exists - public void SaveObject(string id, string json) - { - using var c = new SqliteConnection(_connectionString); - c.Open(); - const string COMMAND_TEXT = "INSERT OR IGNORE INTO objects(hash, content) VALUES(@hash, @content)"; - - using var command = new SqliteCommand(COMMAND_TEXT, c); - command.Parameters.AddWithValue("@hash", id); - command.Parameters.AddWithValue("@content", json); - command.ExecuteNonQuery(); - } + public void SaveObject(string id, string json) => + _pool.Use( + CacheOperation.InsertOrIgnore, + command => + { + command.Parameters.AddWithValue("@hash", id); + command.Parameters.AddWithValue("@content", json); + command.ExecuteNonQuery(); + } + ); //This does an insert or replaces if already exists - public void UpdateObject(string id, string json) - { - using var c = new SqliteConnection(_connectionString); - c.Open(); - const string COMMAND_TEXT = "REPLACE INTO objects(hash, content) VALUES(@hash, @content)"; - using var command = new SqliteCommand(COMMAND_TEXT, c); - command.Parameters.AddWithValue("@hash", id); - command.Parameters.AddWithValue("@content", json); - command.ExecuteNonQuery(); - } - - public void SaveObjects(IEnumerable<(string id, string json)> items) + public void UpdateObject(string id, string json) => + _pool.Use( + CacheOperation.InsertOrReplace, + command => + { + command.Parameters.AddWithValue("@hash", id); + command.Parameters.AddWithValue("@content", json); + command.ExecuteNonQuery(); + } + ); + + public void SaveObjects(IEnumerable<(string id, string json)> items) => + _pool.Use( + CacheOperation.BulkInsertOrIgnore, + cmd => + { + CreateBulkInsert(cmd, items); + return cmd.ExecuteNonQuery(); + } + ); + + private void CreateBulkInsert(SqliteCommand cmd, IEnumerable<(string id, string json)> items) { - using var c = new SqliteConnection(_connectionString); - c.Open(); - using var t = c.BeginTransaction(); - const string COMMAND_TEXT = "INSERT OR IGNORE INTO objects(hash, content) VALUES(@hash, @content)"; - - using var command = new SqliteCommand(COMMAND_TEXT, c); - command.Transaction = t; - var idParam = command.Parameters.Add("@hash", SqliteType.Text); - var jsonParam = command.Parameters.Add("@content", SqliteType.Text); + StringBuilder sb = new(); + sb.AppendLine(CacheDbCommands.Commands[(int)CacheOperation.BulkInsertOrIgnore]); + int i = 0; foreach (var (id, json) in items) { - idParam.Value = id; - jsonParam.Value = json; - command.ExecuteNonQuery(); + sb.Append($"(@key{i}, @value{i},"); + cmd.Parameters.AddWithValue($"@key{i}", id); + cmd.Parameters.AddWithValue($"@value{i}", json); + i++; } - t.Commit(); + sb.Remove(sb.Length - 1, 1); + sb.Append(';'); +#pragma warning disable CA2100 + cmd.CommandText = sb.ToString(); +#pragma warning restore CA2100 } - public bool HasObject(string objectId) - { - using var c = new SqliteConnection(_connectionString); - c.Open(); - const string COMMAND_TEXT = "SELECT 1 FROM objects WHERE hash = @hash LIMIT 1 "; - using var command = new SqliteCommand(COMMAND_TEXT, c); - command.Parameters.AddWithValue("@hash", objectId); - - using var reader = command.ExecuteReader(); - bool rowFound = reader.Read(); - return rowFound; - } + public bool HasObject(string objectId) => + _pool.Use( + CacheOperation.Has, + command => + { + command.Parameters.AddWithValue("@hash", objectId); + using var reader = command.ExecuteReader(); + bool rowFound = reader.Read(); + return rowFound; + } + ); } From cf0dfe8ef86d7e87bad51f89340e2ab94c741971 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Fri, 20 Dec 2024 10:01:30 +0000 Subject: [PATCH 03/31] add more disposal --- .../SQLite/SQLiteJsonCacheManager.cs | 8 ++++-- .../V2/DummySendServerObjectManager.cs | 6 +++-- .../V2/Receive/DeserializeProcess.cs | 5 ++++ .../Serialisation/V2/Receive/ObjectLoader.cs | 3 +++ .../Serialisation/V2/Send/BaseItem.cs | 19 ++++++++++++++ .../Serialisation/V2/Send/SerializeProcess.cs | 22 +++------------- .../V2/SerializeProcessFactory.cs | 25 +++---------------- 7 files changed, 44 insertions(+), 44 deletions(-) create mode 100644 src/Speckle.Sdk/Serialisation/V2/Send/BaseItem.cs diff --git a/src/Speckle.Sdk/SQLite/SQLiteJsonCacheManager.cs b/src/Speckle.Sdk/SQLite/SQLiteJsonCacheManager.cs index f0bf82e4..fb9c7644 100644 --- a/src/Speckle.Sdk/SQLite/SQLiteJsonCacheManager.cs +++ b/src/Speckle.Sdk/SQLite/SQLiteJsonCacheManager.cs @@ -1,11 +1,13 @@ using System.Text; using Microsoft.Data.Sqlite; using Speckle.InterfaceGenerator; +using Speckle.Sdk.Dependencies; namespace Speckle.Sdk.SQLite; +public partial interface ISqLiteJsonCacheManager : IDisposable; [GenerateAutoInterface] -public sealed class SqLiteJsonCacheManager : ISqLiteJsonCacheManager, IDisposable +public sealed class SqLiteJsonCacheManager : ISqLiteJsonCacheManager { private readonly string _connectionString; private readonly CacheDbCommandPool _pool; @@ -17,6 +19,7 @@ public SqLiteJsonCacheManager(string rootPath) _pool = new CacheDbCommandPool(_connectionString); } + [AutoInterfaceIgnore] public void Dispose() => _pool.Dispose(); private void Initialize() @@ -135,7 +138,7 @@ public void SaveObjects(IEnumerable<(string id, string json)> items) => private void CreateBulkInsert(SqliteCommand cmd, IEnumerable<(string id, string json)> items) { - StringBuilder sb = new(); + StringBuilder sb = Pools.StringBuilders.Get(); sb.AppendLine(CacheDbCommands.Commands[(int)CacheOperation.BulkInsertOrIgnore]); int i = 0; foreach (var (id, json) in items) @@ -150,6 +153,7 @@ private void CreateBulkInsert(SqliteCommand cmd, IEnumerable<(string id, string #pragma warning disable CA2100 cmd.CommandText = sb.ToString(); #pragma warning restore CA2100 + Pools.StringBuilders.Return(sb); } public bool HasObject(string objectId) => diff --git a/src/Speckle.Sdk/Serialisation/V2/DummySendServerObjectManager.cs b/src/Speckle.Sdk/Serialisation/V2/DummySendServerObjectManager.cs index 827ed34a..626f9516 100644 --- a/src/Speckle.Sdk/Serialisation/V2/DummySendServerObjectManager.cs +++ b/src/Speckle.Sdk/Serialisation/V2/DummySendServerObjectManager.cs @@ -5,9 +5,11 @@ namespace Speckle.Sdk.Serialisation.V2; -public class DummySqLiteJsonCacheManager : ISqLiteJsonCacheManager +public sealed class DummySqLiteJsonCacheManager : ISqLiteJsonCacheManager { - public IEnumerable GetAllObjects() => throw new NotImplementedException(); + public void Dispose() { } + + public IReadOnlyCollection GetAllObjects() => throw new NotImplementedException(); public void DeleteObject(string id) => throw new NotImplementedException(); diff --git a/src/Speckle.Sdk/Serialisation/V2/Receive/DeserializeProcess.cs b/src/Speckle.Sdk/Serialisation/V2/Receive/DeserializeProcess.cs index 87aa22a3..14281d18 100644 --- a/src/Speckle.Sdk/Serialisation/V2/Receive/DeserializeProcess.cs +++ b/src/Speckle.Sdk/Serialisation/V2/Receive/DeserializeProcess.cs @@ -13,6 +13,7 @@ public record DeserializeProcessOptions( bool SkipInvalidConverts = false ); +public partial interface IDeserializeProcess : IDisposable; [GenerateAutoInterface] public sealed class DeserializeProcess( IProgress? progress, @@ -29,6 +30,10 @@ public sealed class DeserializeProcess( public IReadOnlyDictionary BaseCache => _baseCache; public long Total { get; private set; } + + [AutoInterfaceIgnore] + public void Dispose() => objectLoader.Dispose(); + public async Task Deserialize(string rootId, CancellationToken cancellationToken) { diff --git a/src/Speckle.Sdk/Serialisation/V2/Receive/ObjectLoader.cs b/src/Speckle.Sdk/Serialisation/V2/Receive/ObjectLoader.cs index 7201c4d0..bb885144 100644 --- a/src/Speckle.Sdk/Serialisation/V2/Receive/ObjectLoader.cs +++ b/src/Speckle.Sdk/Serialisation/V2/Receive/ObjectLoader.cs @@ -9,6 +9,7 @@ namespace Speckle.Sdk.Serialisation.V2.Receive; +public partial interface IObjectLoader : IDisposable; [GenerateAutoInterface] public sealed class ObjectLoader( ISqLiteJsonCacheManager sqLiteJsonCacheManager, @@ -20,6 +21,8 @@ public sealed class ObjectLoader( private long _checkCache; private long _cached; private DeserializeProcessOptions _options = new(false); + [AutoInterfaceIgnore] + public void Dispose() => sqLiteJsonCacheManager.Dispose(); public async Task<(string, IReadOnlyCollection)> GetAndCache( string rootId, diff --git a/src/Speckle.Sdk/Serialisation/V2/Send/BaseItem.cs b/src/Speckle.Sdk/Serialisation/V2/Send/BaseItem.cs new file mode 100644 index 00000000..38f2922f --- /dev/null +++ b/src/Speckle.Sdk/Serialisation/V2/Send/BaseItem.cs @@ -0,0 +1,19 @@ +using System.Text; + +namespace Speckle.Sdk.Serialisation.V2.Send; + +public readonly record struct BaseItem(Id Id, Json Json, bool NeedsStorage, Dictionary? Closures) : IHasSize +{ + public int Size { get; } = Encoding.UTF8.GetByteCount(Json.Value); + + public bool Equals(BaseItem? other) + { + if (other is null) + { + return false; + } + return string.Equals(Id.Value, other.Value.Id.Value, StringComparison.OrdinalIgnoreCase); + } + + public override int GetHashCode() => Id.GetHashCode(); +} diff --git a/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs b/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs index 564e47f7..5ed133a8 100644 --- a/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs +++ b/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs @@ -1,5 +1,4 @@ using System.Collections.Concurrent; -using System.Text; using Speckle.InterfaceGenerator; using Speckle.Sdk.Common; using Speckle.Sdk.Dependencies; @@ -25,24 +24,9 @@ public readonly record struct SerializeProcessResults( IReadOnlyDictionary ConvertedReferences ); -public readonly record struct BaseItem(Id Id, Json Json, bool NeedsStorage, Closures? Closures) : IHasSize -{ - public int Size { get; } = Encoding.UTF8.GetByteCount(Json.Value); - - public bool Equals(BaseItem? other) - { - if (other is null) - { - return false; - } - return string.Equals(Id.Value, other.Value.Id.Value, StringComparison.OrdinalIgnoreCase); - } - - public override int GetHashCode() => Id.GetHashCode(); -} - +public partial interface ISerializeProcess : IDisposable; [GenerateAutoInterface] -public class SerializeProcess( +public sealed class SerializeProcess( IProgress? progress, ISqLiteJsonCacheManager sqLiteJsonCacheManager, IServerObjectManager serverObjectManager, @@ -64,6 +48,8 @@ public class SerializeProcess( private long _uploaded; private long _cached; + [AutoInterfaceIgnore] + public void Dispose() => sqLiteJsonCacheManager.Dispose(); public async Task Serialize(Base root, CancellationToken cancellationToken) { diff --git a/src/Speckle.Sdk/Serialisation/V2/SerializeProcessFactory.cs b/src/Speckle.Sdk/Serialisation/V2/SerializeProcessFactory.cs index 119e1a18..68615ec5 100644 --- a/src/Speckle.Sdk/Serialisation/V2/SerializeProcessFactory.cs +++ b/src/Speckle.Sdk/Serialisation/V2/SerializeProcessFactory.cs @@ -21,11 +21,6 @@ IDeserializeProcess CreateDeserializeProcess( IProgress? progress, DeserializeProcessOptions? options = null ); - - public ISerializeProcess CreateSerializeProcess( - SerializeProcessOptions? options = null, - IProgress? progress = null - ); } public class SerializeProcessFactory( @@ -56,23 +51,6 @@ public ISerializeProcess CreateSerializeProcess( ); } - public ISerializeProcess CreateSerializeProcess( - SerializeProcessOptions? options = null, - IProgress? progress = null - ) - { - var sqLiteJsonCacheManager = new DummySqLiteJsonCacheManager(); - var serverObjectManager = new DummySendServerObjectManager(); - return new SerializeProcess( - progress, - sqLiteJsonCacheManager, - serverObjectManager, - baseChildFinder, - objectSerializerFactory, - options - ); - } - public IDeserializeProcess CreateDeserializeProcess( Uri url, string streamId, @@ -84,7 +62,10 @@ public IDeserializeProcess CreateDeserializeProcess( var sqLiteJsonCacheManager = sqLiteJsonCacheManagerFactory.CreateFromStream(streamId); var serverObjectManager = serverObjectManagerFactory.Create(url, streamId, authorizationToken); +#pragma warning disable CA2000 + //owned by process, refactor later var objectLoader = new ObjectLoader(sqLiteJsonCacheManager, serverObjectManager, progress); +#pragma warning restore CA2000 return new DeserializeProcess(progress, objectLoader, objectDeserializerFactory, options); } } From 3b50c857fb9a9da3fbf8193d36b149d2e0d2cb72 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Fri, 20 Dec 2024 10:01:46 +0000 Subject: [PATCH 04/31] save saving increase --- src/Speckle.Sdk.Dependencies/Serialization/ChannelLoader.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Speckle.Sdk.Dependencies/Serialization/ChannelLoader.cs b/src/Speckle.Sdk.Dependencies/Serialization/ChannelLoader.cs index e9d6e729..5305adf1 100644 --- a/src/Speckle.Sdk.Dependencies/Serialization/ChannelLoader.cs +++ b/src/Speckle.Sdk.Dependencies/Serialization/ChannelLoader.cs @@ -9,7 +9,7 @@ public abstract class ChannelLoader private static readonly TimeSpan HTTP_BATCH_TIMEOUT = TimeSpan.FromSeconds(2); private static readonly int MAX_READ_CACHE_PARALLELISM = Environment.ProcessorCount; private const int MAX_SAVE_CACHE_BATCH = 200; - private const int MAX_SAVE_CACHE_PARALLELISM = 1; + private const int MAX_SAVE_CACHE_PARALLELISM = 4; protected async Task GetAndCache(IEnumerable allChildrenIds, CancellationToken cancellationToken = default) => await allChildrenIds From 13082d930b5e06a42200d1afa996e3da86f3a5da Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Fri, 20 Dec 2024 10:30:09 +0000 Subject: [PATCH 05/31] fix tests --- tests/Speckle.Sdk.Serialization.Testing/Program.cs | 7 ++----- .../Speckle.Sdk.Serialization.Tests/DetachedTests.cs | 11 ++++++----- .../DummySqLiteReceiveManager.cs | 3 ++- .../DummySqLiteSendManager.cs | 4 +++- .../ExplicitInterfaceTests.cs | 2 +- .../SerializationTests.cs | 8 +++++--- .../Benchmarks/GeneralDeserializerTest.cs | 2 +- 7 files changed, 20 insertions(+), 17 deletions(-) diff --git a/tests/Speckle.Sdk.Serialization.Testing/Program.cs b/tests/Speckle.Sdk.Serialization.Testing/Program.cs index b807f795..73f30a9a 100644 --- a/tests/Speckle.Sdk.Serialization.Testing/Program.cs +++ b/tests/Speckle.Sdk.Serialization.Testing/Program.cs @@ -3,9 +3,7 @@ using Microsoft.Extensions.DependencyInjection; using Speckle.Sdk; using Speckle.Sdk.Credentials; -using Speckle.Sdk.Helpers; using Speckle.Sdk.Host; -using Speckle.Sdk.Logging; using Speckle.Sdk.Models; using Speckle.Sdk.Serialisation.V2; using Speckle.Sdk.Serialisation.V2.Receive; @@ -43,12 +41,11 @@ var progress = new Progress(true); var factory = new SerializeProcessFactory( - serviceProvider.GetRequiredService(), - serviceProvider.GetRequiredService(), new BaseChildFinder(new BasePropertyGatherer()), new ObjectSerializerFactory(new BasePropertyGatherer()), new ObjectDeserializerFactory(), - serviceProvider.GetRequiredService() + serviceProvider.GetRequiredService(), + serviceProvider.GetRequiredService() ); var process = factory.CreateDeserializeProcess(new Uri(url), streamId, token, progress, new(skipCacheReceive)); var @base = await process.Deserialize(rootId, default).ConfigureAwait(false); diff --git a/tests/Speckle.Sdk.Serialization.Tests/DetachedTests.cs b/tests/Speckle.Sdk.Serialization.Tests/DetachedTests.cs index 07d02914..4ef63ebb 100644 --- a/tests/Speckle.Sdk.Serialization.Tests/DetachedTests.cs +++ b/tests/Speckle.Sdk.Serialization.Tests/DetachedTests.cs @@ -67,7 +67,7 @@ public async Task CanSerialize_New_Detached() var objects = new Dictionary(); - var process2 = new SerializeProcess( + using var process2 = new SerializeProcess( null, new DummySendCacheManager(objects), new DummyServerObjectManager(), @@ -259,7 +259,7 @@ public async Task CanSerialize_New_Detached2() var objects = new Dictionary(); - var process2 = new SerializeProcess( + using var process2 = new SerializeProcess( null, new DummySendCacheManager(objects), new DummyServerObjectManager(), @@ -332,7 +332,7 @@ public async Task CanSerialize_New_Detached_With_DataChunks() var objects = new Dictionary(); - var process2 = new SerializeProcess( + using var process2 = new SerializeProcess( null, new DummySendCacheManager(objects), new DummyServerObjectManager(), @@ -413,7 +413,7 @@ public async Task CanSerialize_New_Detached_With_DataChunks2() var objects = new Dictionary(); - var process2 = new SerializeProcess( + using var process2 = new SerializeProcess( null, new DummySendCacheManager(objects), new DummyServerObjectManager(), @@ -529,7 +529,8 @@ CancellationToken cancellationToken public class DummySendCacheManager(Dictionary objects) : ISqLiteJsonCacheManager { - public IEnumerable GetAllObjects() => throw new NotImplementedException(); + public void Dispose() { } + public IReadOnlyCollection GetAllObjects() => throw new NotImplementedException(); public void DeleteObject(string id) => throw new NotImplementedException(); diff --git a/tests/Speckle.Sdk.Serialization.Tests/DummySqLiteReceiveManager.cs b/tests/Speckle.Sdk.Serialization.Tests/DummySqLiteReceiveManager.cs index 42961d6b..99258fe8 100644 --- a/tests/Speckle.Sdk.Serialization.Tests/DummySqLiteReceiveManager.cs +++ b/tests/Speckle.Sdk.Serialization.Tests/DummySqLiteReceiveManager.cs @@ -4,7 +4,8 @@ namespace Speckle.Sdk.Serialization.Tests; public class DummySqLiteReceiveManager(Dictionary savedObjects) : ISqLiteJsonCacheManager { - public IEnumerable GetAllObjects() => throw new NotImplementedException(); + public void Dispose() { } + public IReadOnlyCollection GetAllObjects() => throw new NotImplementedException(); public void DeleteObject(string id) => throw new NotImplementedException(); diff --git a/tests/Speckle.Sdk.Serialization.Tests/DummySqLiteSendManager.cs b/tests/Speckle.Sdk.Serialization.Tests/DummySqLiteSendManager.cs index ce1dae8c..882704ff 100644 --- a/tests/Speckle.Sdk.Serialization.Tests/DummySqLiteSendManager.cs +++ b/tests/Speckle.Sdk.Serialization.Tests/DummySqLiteSendManager.cs @@ -14,7 +14,9 @@ public class DummySqLiteSendManager : ISqLiteJsonCacheManager public bool HasObject(string objectId) => throw new NotImplementedException(); - public IEnumerable GetAllObjects() => throw new NotImplementedException(); + public IReadOnlyCollection GetAllObjects() => throw new NotImplementedException(); public void DeleteObject(string id) => throw new NotImplementedException(); + + public void Dispose() { } } diff --git a/tests/Speckle.Sdk.Serialization.Tests/ExplicitInterfaceTests.cs b/tests/Speckle.Sdk.Serialization.Tests/ExplicitInterfaceTests.cs index d82eef21..e90293de 100644 --- a/tests/Speckle.Sdk.Serialization.Tests/ExplicitInterfaceTests.cs +++ b/tests/Speckle.Sdk.Serialization.Tests/ExplicitInterfaceTests.cs @@ -21,7 +21,7 @@ public async Task Test_Json() var testClass = new TestClass() { RegularProperty = "Hello" }; var objects = new Dictionary(); - var process2 = new SerializeProcess( + using var process2 = new SerializeProcess( null, new DummySendCacheManager(objects), new DummyServerObjectManager(), diff --git a/tests/Speckle.Sdk.Serialization.Tests/SerializationTests.cs b/tests/Speckle.Sdk.Serialization.Tests/SerializationTests.cs index 49648b4a..23f24e49 100644 --- a/tests/Speckle.Sdk.Serialization.Tests/SerializationTests.cs +++ b/tests/Speckle.Sdk.Serialization.Tests/SerializationTests.cs @@ -33,6 +33,7 @@ CancellationToken cancellationToken } public string? LoadId(string id) => null; + public void Dispose() { } } private readonly Assembly _assembly = Assembly.GetExecutingAssembly(); @@ -103,6 +104,7 @@ CancellationToken cancellationToken } public string? LoadId(string id) => idToObject.GetValueOrDefault(id); + public void Dispose() { } } [Test] @@ -154,7 +156,7 @@ public async Task Basic_Namespace_Validation_New(string fileName) var fullName = _assembly.GetManifestResourceNames().Single(x => x.EndsWith(fileName)); var json = await ReadJson(fullName); var closures = ReadAsObjects(json); - var process = new DeserializeProcess(null, new TestObjectLoader(closures), new ObjectDeserializerFactory()); + using var process = new DeserializeProcess(null, new TestObjectLoader(closures), new ObjectDeserializerFactory()); await process.Deserialize("3416d3fe01c9196115514c4a2f41617b", default); foreach (var (id, objJson) in closures) { @@ -251,13 +253,13 @@ public async Task Roundtrip_Test_New(string fileName, string rootId, int oldCoun new DummyReceiveServerObjectManager(closure), null ); - var process = new DeserializeProcess(null, o, new ObjectDeserializerFactory(), new(true)); + using var process = new DeserializeProcess(null, o, new ObjectDeserializerFactory(), new(true)); var root = await process.Deserialize(rootId, default); process.BaseCache.Count.ShouldBe(oldCount); process.Total.ShouldBe(oldCount); var newIdToJson = new ConcurrentDictionary(); - var serializeProcess = new SerializeProcess( + using var serializeProcess = new SerializeProcess( null, new DummySqLiteSendManager(), new DummySendServerObjectManager(newIdToJson), diff --git a/tests/Speckle.Sdk.Tests.Performance/Benchmarks/GeneralDeserializerTest.cs b/tests/Speckle.Sdk.Tests.Performance/Benchmarks/GeneralDeserializerTest.cs index 6ffc63ea..59fc7b33 100644 --- a/tests/Speckle.Sdk.Tests.Performance/Benchmarks/GeneralDeserializerTest.cs +++ b/tests/Speckle.Sdk.Tests.Performance/Benchmarks/GeneralDeserializerTest.cs @@ -57,7 +57,7 @@ public async Task RunTest_New() null ); var o = new ObjectLoader(sqlite, serverObjects, null); - var process = new DeserializeProcess(null, o, new ObjectDeserializerFactory(), new(skipCache)); + using var process = new DeserializeProcess(null, o, new ObjectDeserializerFactory(), new(skipCache)); return await process.Deserialize(rootId, default).ConfigureAwait(false); } From 5481150979739d967db4302f616353554576ee21 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Fri, 20 Dec 2024 10:40:42 +0000 Subject: [PATCH 06/31] fixes --- src/Speckle.Sdk.Dependencies/Serialization/ChannelSaver.cs | 2 +- src/Speckle.Sdk/SQLite/CacheDbCommands.cs | 2 +- src/Speckle.Sdk/SQLite/SQLiteJsonCacheManager.cs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Speckle.Sdk.Dependencies/Serialization/ChannelSaver.cs b/src/Speckle.Sdk.Dependencies/Serialization/ChannelSaver.cs index 4f974eda..865484c1 100644 --- a/src/Speckle.Sdk.Dependencies/Serialization/ChannelSaver.cs +++ b/src/Speckle.Sdk.Dependencies/Serialization/ChannelSaver.cs @@ -12,7 +12,7 @@ public abstract class ChannelSaver private static readonly TimeSpan HTTP_BATCH_TIMEOUT = TimeSpan.FromSeconds(2); private const int MAX_PARALLELISM_HTTP = 4; private const int HTTP_CAPACITY = 50; - private const int MAX_CACHE_WRITE_PARALLELISM = 1; + private const int MAX_CACHE_WRITE_PARALLELISM = 4; private const int MAX_CACHE_BATCH = 200; private bool _enabled; diff --git a/src/Speckle.Sdk/SQLite/CacheDbCommands.cs b/src/Speckle.Sdk/SQLite/CacheDbCommands.cs index ee3aae47..a56e4fbb 100644 --- a/src/Speckle.Sdk/SQLite/CacheDbCommands.cs +++ b/src/Speckle.Sdk/SQLite/CacheDbCommands.cs @@ -26,7 +26,7 @@ static CacheDbCommands() "INSERT OR IGNORE INTO objects(hash, content) VALUES(@hash, @content)"; Commands[(int)CacheOperation.InsertOrReplace] = "REPLACE INTO objects(hash, content) VALUES(@hash, @content)"; Commands[(int)CacheOperation.Has] = "SELECT 1 FROM objects WHERE hash = @hash LIMIT 1"; - Commands[(int)CacheOperation.Get] = "SELECT * FROM objects WHERE hash = @hash LIMIT 1"; + Commands[(int)CacheOperation.Get] = "SELECT content FROM objects WHERE hash = @hash LIMIT 1"; Commands[(int)CacheOperation.Delete] = "DELETE FROM objects WHERE hash = @hash"; Commands[(int)CacheOperation.GetAll] = "SELECT * FROM objects"; diff --git a/src/Speckle.Sdk/SQLite/SQLiteJsonCacheManager.cs b/src/Speckle.Sdk/SQLite/SQLiteJsonCacheManager.cs index fb9c7644..e9bd2225 100644 --- a/src/Speckle.Sdk/SQLite/SQLiteJsonCacheManager.cs +++ b/src/Speckle.Sdk/SQLite/SQLiteJsonCacheManager.cs @@ -143,7 +143,7 @@ private void CreateBulkInsert(SqliteCommand cmd, IEnumerable<(string id, string int i = 0; foreach (var (id, json) in items) { - sb.Append($"(@key{i}, @value{i},"); + sb.Append($"(@key{i}, @value{i}),"); cmd.Parameters.AddWithValue($"@key{i}", id); cmd.Parameters.AddWithValue($"@value{i}", json); i++; From 42f21364bd70e703926d3d4912559e79457536ff Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Fri, 20 Dec 2024 10:47:22 +0000 Subject: [PATCH 07/31] push out concurrency and disposablity --- src/Speckle.Sdk/Credentials/AccountManager.cs | 9 ++++++++- src/Speckle.Sdk/SQLite/CacheDbCommandPool.cs | 5 ++--- src/Speckle.Sdk/SQLite/SQLiteJsonCacheManager.cs | 4 ++-- src/Speckle.Sdk/SQLite/SqLiteJsonCacheManagerFactory.cs | 7 ++++--- 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/src/Speckle.Sdk/Credentials/AccountManager.cs b/src/Speckle.Sdk/Credentials/AccountManager.cs index e8784ee1..2af283d8 100644 --- a/src/Speckle.Sdk/Credentials/AccountManager.cs +++ b/src/Speckle.Sdk/Credentials/AccountManager.cs @@ -21,11 +21,12 @@ namespace Speckle.Sdk.Credentials; +public partial interface IAccountManager : IDisposable; /// /// Manage accounts locally for desktop applications. /// [GenerateAutoInterface] -public class AccountManager( +public sealed class AccountManager( ISpeckleApplication application, ILogger logger, ISpeckleHttp speckleHttp, @@ -40,6 +41,12 @@ ISqLiteJsonCacheManagerFactory sqLiteJsonCacheManagerFactory "AccountAddFlow" ); + [AutoInterfaceIgnore] + public void Dispose() + { + _accountStorage.Dispose(); + _accountAddLockStorage.Dispose(); + } /// /// Gets the basic information about a server. /// diff --git a/src/Speckle.Sdk/SQLite/CacheDbCommandPool.cs b/src/Speckle.Sdk/SQLite/CacheDbCommandPool.cs index 3144aa2e..0242240d 100644 --- a/src/Speckle.Sdk/SQLite/CacheDbCommandPool.cs +++ b/src/Speckle.Sdk/SQLite/CacheDbCommandPool.cs @@ -6,19 +6,18 @@ namespace Speckle.Sdk.SQLite; //inspired by https://github.com/neosmart/SqliteCache/blob/master/SqliteCache/DbCommandPool.cs public sealed class CacheDbCommandPool : IDisposable { - private const int INITIAL_CONCURRENCY = 4; private readonly ConcurrentBag[] _commands = new ConcurrentBag[CacheDbCommands.Count]; private readonly ConcurrentBag _connections = new(); private readonly string _connectionString; - public CacheDbCommandPool(string connectionString) + public CacheDbCommandPool(string connectionString, int concurrency) { _connectionString = connectionString; for (int i = 0; i < _commands.Length; ++i) { _commands[i] = new ConcurrentBag(); } - for (int i = 0; i < INITIAL_CONCURRENCY; ++i) + for (int i = 0; i < concurrency; ++i) { var connection = new SqliteConnection(_connectionString); connection.Open(); diff --git a/src/Speckle.Sdk/SQLite/SQLiteJsonCacheManager.cs b/src/Speckle.Sdk/SQLite/SQLiteJsonCacheManager.cs index e9bd2225..7e4789e2 100644 --- a/src/Speckle.Sdk/SQLite/SQLiteJsonCacheManager.cs +++ b/src/Speckle.Sdk/SQLite/SQLiteJsonCacheManager.cs @@ -12,11 +12,11 @@ public sealed class SqLiteJsonCacheManager : ISqLiteJsonCacheManager private readonly string _connectionString; private readonly CacheDbCommandPool _pool; - public SqLiteJsonCacheManager(string rootPath) + public SqLiteJsonCacheManager(string rootPath, int concurrency) { _connectionString = $"Data Source={rootPath};"; Initialize(); - _pool = new CacheDbCommandPool(_connectionString); + _pool = new CacheDbCommandPool(_connectionString, concurrency); } [AutoInterfaceIgnore] diff --git a/src/Speckle.Sdk/SQLite/SqLiteJsonCacheManagerFactory.cs b/src/Speckle.Sdk/SQLite/SqLiteJsonCacheManagerFactory.cs index c73a28a7..ef546bd2 100644 --- a/src/Speckle.Sdk/SQLite/SqLiteJsonCacheManagerFactory.cs +++ b/src/Speckle.Sdk/SQLite/SqLiteJsonCacheManagerFactory.cs @@ -7,10 +7,11 @@ namespace Speckle.Sdk.SQLite; [GenerateAutoInterface] public class SqLiteJsonCacheManagerFactory : ISqLiteJsonCacheManagerFactory { - private ISqLiteJsonCacheManager Create(string path) => new SqLiteJsonCacheManager(path); + public const int INITIAL_CONCURRENCY = 4; + private ISqLiteJsonCacheManager Create(string path, int concurrency) => new SqLiteJsonCacheManager(path, concurrency); public ISqLiteJsonCacheManager CreateForUser(string scope) => - Create(Path.Combine(SpecklePathProvider.UserApplicationDataPath(), "Speckle", $"{scope}.db")); + Create(Path.Combine(SpecklePathProvider.UserApplicationDataPath(), "Speckle", $"{scope}.db"), 1); - public ISqLiteJsonCacheManager CreateFromStream(string streamId) => Create(SqlitePaths.GetDBPath(streamId)); + public ISqLiteJsonCacheManager CreateFromStream(string streamId) => Create(SqlitePaths.GetDBPath(streamId), INITIAL_CONCURRENCY); } From db9b9ec7fdb6eb7cdd9a7c5fda355763e2b344a4 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Mon, 30 Dec 2024 12:58:20 +0000 Subject: [PATCH 08/31] Add a custom task scheduler --- .../V2/Send/PriorityScheduler.cs | 39 +++++++++++++++++++ .../Serialisation/V2/Send/SerializeProcess.cs | 23 +++++++++-- 2 files changed, 58 insertions(+), 4 deletions(-) create mode 100644 src/Speckle.Sdk/Serialisation/V2/Send/PriorityScheduler.cs diff --git a/src/Speckle.Sdk/Serialisation/V2/Send/PriorityScheduler.cs b/src/Speckle.Sdk/Serialisation/V2/Send/PriorityScheduler.cs new file mode 100644 index 00000000..2e5babf0 --- /dev/null +++ b/src/Speckle.Sdk/Serialisation/V2/Send/PriorityScheduler.cs @@ -0,0 +1,39 @@ +using System.Collections.Concurrent; + +namespace Speckle.Sdk.Serialisation.V2.Send; + +public sealed class PriorityScheduler(ThreadPriority priority) : TaskScheduler, IDisposable +{ + private readonly BlockingCollection _tasks = new BlockingCollection(); + private Thread[]? _threads; + private readonly int _maximumConcurrencyLevel = Math.Max(1, Environment.ProcessorCount); + + public void Dispose() => _tasks.Dispose(); + + public override int MaximumConcurrencyLevel => _maximumConcurrencyLevel; + + protected override IEnumerable GetScheduledTasks() => _tasks; + + protected override void QueueTask(Task task) + { + _tasks.Add(task); + + if (_threads == null) + { + _threads = new Thread[_maximumConcurrencyLevel]; + for (int i = 0; i < _threads.Length; i++) + { + _threads[i] = new Thread(() => + { + foreach (Task t in _tasks.GetConsumingEnumerable()) + { + TryExecuteTask(t); + } + }) { Name = $"PriorityScheduler: {i}", Priority = priority, IsBackground = true }; + _threads[i].Start(); + } + } + } + + protected override bool TryExecuteTaskInline(Task task, bool taskWasPreviouslyQueued) => false; // we might not want to execute task that should schedule as high or low priority inline +} diff --git a/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs b/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs index 564e47f7..ba26d929 100644 --- a/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs +++ b/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs @@ -51,6 +51,10 @@ public class SerializeProcess( SerializeProcessOptions? options = null ) : ChannelSaver, ISerializeProcess { + private readonly PriorityScheduler _countScheduler = new PriorityScheduler(ThreadPriority.Highest); + private readonly PriorityScheduler _writeScheduler = new PriorityScheduler(ThreadPriority.Normal); + + private readonly SerializeProcessOptions _options = options ?? new(false, false, false, false); private readonly ConcurrentDictionary _objectReferences = new(); @@ -65,6 +69,14 @@ public class SerializeProcess( private long _uploaded; private long _cached; + [AutoInterfaceIgnore] + public void Dispose() + { + sqLiteJsonCacheManager.Dispose(); + _countScheduler.Dispose(); + _writeScheduler.Dispose(); + } + public async Task Serialize(Base root, CancellationToken cancellationToken) { var channelTask = Start(_options.EnableServerSending, _options.EnableCacheSaving, cancellationToken); @@ -74,8 +86,8 @@ public async Task Serialize(Base root, CancellationToke findTotalObjectsTask = Task.Factory.StartNew( () => TraverseTotal(root), default, - TaskCreationOptions.LongRunning, - TaskScheduler.Default + TaskCreationOptions.AttachedToParent, + _countScheduler ); } @@ -107,7 +119,7 @@ private async Task> Traverse(Base obj, bool isEnd, Canc () => Traverse(tmp, false, cancellationToken), cancellationToken, TaskCreationOptions.AttachedToParent, - TaskScheduler.Default + _writeScheduler ) .Unwrap(); tasks.Add(t); @@ -136,7 +148,10 @@ private async Task> Traverse(Base obj, bool isEnd, Canc { if (item.NeedsStorage) { - await Save(item, cancellationToken).ConfigureAwait(false); + var saving = Task.Factory.StartNew(async () => + await Save(item, cancellationToken).ConfigureAwait(false), + cancellationToken, TaskCreationOptions.AttachedToParent, _writeScheduler); + await saving.Unwrap().ConfigureAwait(false); } if (!currentClosures.ContainsKey(item.Id)) From 5c75d7dec117b776eb0b403866b0ca4078a38794 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Tue, 31 Dec 2024 12:33:59 +0000 Subject: [PATCH 09/31] Better usage, don't wait to enqueue to save to channels --- src/Speckle.Sdk.Dependencies/Pools.cs | 19 ++++- .../Serialization/ChannelSaver.cs | 78 ++++++------------- .../V2/Send/PriorityScheduler.cs | 18 +++-- .../Serialisation/V2/Send/SerializeProcess.cs | 56 +++++++------ 4 files changed, 85 insertions(+), 86 deletions(-) diff --git a/src/Speckle.Sdk.Dependencies/Pools.cs b/src/Speckle.Sdk.Dependencies/Pools.cs index 86bfe7c1..4774a16b 100644 --- a/src/Speckle.Sdk.Dependencies/Pools.cs +++ b/src/Speckle.Sdk.Dependencies/Pools.cs @@ -1,4 +1,5 @@ -using System.Text; +using System.Collections.Concurrent; +using System.Text; using Microsoft.Extensions.ObjectPool; namespace Speckle.Sdk.Dependencies; @@ -33,6 +34,19 @@ public bool Return(Dictionary obj) } } + private sealed class ObjectConcurrentDictionaryPolicy + : IPooledObjectPolicy> + where TKey : notnull + { + public ConcurrentDictionary Create() => new(Environment.ProcessorCount, 50); + + public bool Return(ConcurrentDictionary obj) + { + obj.Clear(); + return true; + } + } + private sealed class ObjectListPolicy : IPooledObjectPolicy> { public List Create() => new(50); @@ -48,4 +62,7 @@ public bool Return(List obj) public static Pool> CreateDictionaryPool() where TKey : notnull => new(new ObjectDictionaryPolicy()); + + public static Pool> CreateConcurrentictionaryPool() + where TKey : notnull => new(new ObjectConcurrentDictionaryPolicy()); } diff --git a/src/Speckle.Sdk.Dependencies/Serialization/ChannelSaver.cs b/src/Speckle.Sdk.Dependencies/Serialization/ChannelSaver.cs index 4f974eda..c5221a4b 100644 --- a/src/Speckle.Sdk.Dependencies/Serialization/ChannelSaver.cs +++ b/src/Speckle.Sdk.Dependencies/Serialization/ChannelSaver.cs @@ -7,20 +7,18 @@ namespace Speckle.Sdk.Dependencies.Serialization; public abstract class ChannelSaver where T : IHasSize { - private const int SEND_CAPACITY = 50; + private const int SEND_CAPACITY = 500; private const int HTTP_SEND_CHUNK_SIZE = 25_000_000; //bytes private static readonly TimeSpan HTTP_BATCH_TIMEOUT = TimeSpan.FromSeconds(2); private const int MAX_PARALLELISM_HTTP = 4; - private const int HTTP_CAPACITY = 50; - private const int MAX_CACHE_WRITE_PARALLELISM = 1; + private const int HTTP_CAPACITY = 500; + private const int MAX_CACHE_WRITE_PARALLELISM = 4; private const int MAX_CACHE_BATCH = 200; - private bool _enabled; - private readonly Channel _checkCacheChannel = Channel.CreateBounded( new BoundedChannelOptions(SEND_CAPACITY) { - AllowSynchronousContinuations = true, + AllowSynchronousContinuations = false, Capacity = SEND_CAPACITY, SingleWriter = false, SingleReader = false, @@ -29,59 +27,31 @@ public abstract class ChannelSaver _ => throw new NotImplementedException("Dropping items not supported.") ); - public Task Start( - bool enableServerSending = true, - bool enableCacheSaving = true, - CancellationToken cancellationToken = default - ) - { - ValueTask t = new(Task.FromResult(0L)); - if (enableServerSending) - { - _enabled = true; - var tChannelReader = _checkCacheChannel - .Reader.BatchBySize(HTTP_SEND_CHUNK_SIZE) - .WithTimeout(HTTP_BATCH_TIMEOUT) - .PipeAsync( - MAX_PARALLELISM_HTTP, - async x => await SendToServer(x, cancellationToken).ConfigureAwait(false), - HTTP_CAPACITY, - false, - cancellationToken - ); - if (enableCacheSaving) - { - t = new( - tChannelReader - .Join() - .Batch(MAX_CACHE_BATCH) - .WithTimeout(HTTP_BATCH_TIMEOUT) - .ReadAllConcurrently(MAX_CACHE_WRITE_PARALLELISM, SaveToCache, cancellationToken) - ); - } - else - { - t = tChannelReader.ReadUntilCancelledAsync(cancellationToken, (list, l) => new ValueTask()); - } - } - - return t.AsTask(); - } - - public async ValueTask Save(T item, CancellationToken cancellationToken = default) - { - if (_enabled) - { - await _checkCacheChannel.Writer.WriteAsync(item, cancellationToken).ConfigureAwait(false); - } - } + public Task Start(CancellationToken cancellationToken = default) => + _checkCacheChannel + .Reader.BatchBySize(HTTP_SEND_CHUNK_SIZE) + .WithTimeout(HTTP_BATCH_TIMEOUT) + .PipeAsync( + MAX_PARALLELISM_HTTP, + async x => await SendToServer(x, cancellationToken).ConfigureAwait(false), + HTTP_CAPACITY, + false, + cancellationToken + ) + .Join() + .Batch(MAX_CACHE_BATCH) + .WithTimeout(HTTP_BATCH_TIMEOUT) + .ReadAllConcurrently(MAX_CACHE_WRITE_PARALLELISM, SaveToCache, cancellationToken); + + public ValueTask Save(T item, CancellationToken cancellationToken = default) => + _checkCacheChannel.Writer.WriteAsync(item, cancellationToken); public abstract Task> SendToServer(List batch, CancellationToken cancellationToken); - public ValueTask Done() + public Task Done() { _checkCacheChannel.Writer.Complete(); - return new(Task.CompletedTask); + return Task.CompletedTask; } public abstract void SaveToCache(List item); diff --git a/src/Speckle.Sdk/Serialisation/V2/Send/PriorityScheduler.cs b/src/Speckle.Sdk/Serialisation/V2/Send/PriorityScheduler.cs index 2e5babf0..a7aaac0c 100644 --- a/src/Speckle.Sdk/Serialisation/V2/Send/PriorityScheduler.cs +++ b/src/Speckle.Sdk/Serialisation/V2/Send/PriorityScheduler.cs @@ -2,15 +2,16 @@ namespace Speckle.Sdk.Serialisation.V2.Send; -public sealed class PriorityScheduler(ThreadPriority priority) : TaskScheduler, IDisposable +public sealed class PriorityScheduler(ThreadPriority priority, int maximumConcurrencyLevel) + : TaskScheduler, + IDisposable { - private readonly BlockingCollection _tasks = new BlockingCollection(); + private readonly BlockingCollection _tasks = new(); private Thread[]? _threads; - private readonly int _maximumConcurrencyLevel = Math.Max(1, Environment.ProcessorCount); public void Dispose() => _tasks.Dispose(); - public override int MaximumConcurrencyLevel => _maximumConcurrencyLevel; + public override int MaximumConcurrencyLevel => maximumConcurrencyLevel; protected override IEnumerable GetScheduledTasks() => _tasks; @@ -20,7 +21,7 @@ protected override void QueueTask(Task task) if (_threads == null) { - _threads = new Thread[_maximumConcurrencyLevel]; + _threads = new Thread[maximumConcurrencyLevel]; for (int i = 0; i < _threads.Length; i++) { _threads[i] = new Thread(() => @@ -29,7 +30,12 @@ protected override void QueueTask(Task task) { TryExecuteTask(t); } - }) { Name = $"PriorityScheduler: {i}", Priority = priority, IsBackground = true }; + }) + { + Name = $"{priority}: {i}", + Priority = priority, + IsBackground = true, + }; _threads[i].Start(); } } diff --git a/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs b/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs index ba26d929..8073012b 100644 --- a/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs +++ b/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs @@ -15,9 +15,7 @@ public record SerializeProcessOptions( bool SkipCacheRead = false, bool SkipCacheWrite = false, bool SkipServer = false, - bool SkipFindTotalObjects = false, - bool EnableServerSending = true, - bool EnableCacheSaving = true + bool SkipFindTotalObjects = false ); public readonly record struct SerializeProcessResults( @@ -41,8 +39,10 @@ public bool Equals(BaseItem? other) public override int GetHashCode() => Id.GetHashCode(); } +public partial interface ISerializeProcess : IDisposable; + [GenerateAutoInterface] -public class SerializeProcess( +public sealed class SerializeProcess( IProgress? progress, ISqLiteJsonCacheManager sqLiteJsonCacheManager, IServerObjectManager serverObjectManager, @@ -51,15 +51,21 @@ public class SerializeProcess( SerializeProcessOptions? options = null ) : ChannelSaver, ISerializeProcess { - private readonly PriorityScheduler _countScheduler = new PriorityScheduler(ThreadPriority.Highest); - private readonly PriorityScheduler _writeScheduler = new PriorityScheduler(ThreadPriority.Normal); - - + private readonly PriorityScheduler _highest = new( ThreadPriority.Highest, 2); + private readonly PriorityScheduler _belowNormal = new( + ThreadPriority.BelowNormal, + Environment.ProcessorCount * 3 + ); + private readonly SerializeProcessOptions _options = options ?? new(false, false, false, false); private readonly ConcurrentDictionary _objectReferences = new(); private readonly Pool> _pool = Pools.CreateListPool<(Id, Json, Closures)>(); - private readonly Pool> _childClosurePool = Pools.CreateDictionaryPool(); + private readonly Pool> _currentClosurePool = Pools.CreateDictionaryPool(); + private readonly Pool> _childClosurePool = Pools.CreateConcurrentictionaryPool< + Id, + NodeInfo + >(); private long _objectCount; private long _objectsFound; @@ -72,22 +78,21 @@ public class SerializeProcess( [AutoInterfaceIgnore] public void Dispose() { - sqLiteJsonCacheManager.Dispose(); - _countScheduler.Dispose(); - _writeScheduler.Dispose(); + _highest.Dispose(); + _belowNormal.Dispose(); } public async Task Serialize(Base root, CancellationToken cancellationToken) { - var channelTask = Start(_options.EnableServerSending, _options.EnableCacheSaving, cancellationToken); + var channelTask = Start(cancellationToken); var findTotalObjectsTask = Task.CompletedTask; if (!_options.SkipFindTotalObjects) { findTotalObjectsTask = Task.Factory.StartNew( () => TraverseTotal(root), default, - TaskCreationOptions.AttachedToParent, - _countScheduler + TaskCreationOptions.AttachedToParent | TaskCreationOptions.PreferFairness, + _highest ); } @@ -118,40 +123,41 @@ private async Task> Traverse(Base obj, bool isEnd, Canc .Factory.StartNew( () => Traverse(tmp, false, cancellationToken), cancellationToken, - TaskCreationOptions.AttachedToParent, - _writeScheduler + TaskCreationOptions.AttachedToParent | TaskCreationOptions.PreferFairness, + _belowNormal ) .Unwrap(); tasks.Add(t); } + Dictionary[] taskClosures = []; if (tasks.Count > 0) { - await Task.WhenAll(tasks).ConfigureAwait(false); + taskClosures = await Task.WhenAll(tasks).ConfigureAwait(false); } var childClosures = _childClosurePool.Get(); - foreach (var t in tasks) + foreach (var childClosure in taskClosures) { - var childClosure = t.Result; foreach (var kvp in childClosure) { childClosures[kvp.Key] = kvp.Value; } + _currentClosurePool.Return(childClosure); } var items = Serialise(obj, childClosures, cancellationToken); - var currentClosures = new Dictionary(); + var currentClosures = _currentClosurePool.Get(); Interlocked.Increment(ref _objectCount); - progress?.Report(new(ProgressEvent.FromCacheOrSerialized, _objectCount, _objectsFound)); + progress?.Report(new(ProgressEvent.FromCacheOrSerialized, _objectCount, Math.Max(_objectCount, _objectsFound))); foreach (var item in items) { if (item.NeedsStorage) { - var saving = Task.Factory.StartNew(async () => + //just await enqueuing + await Task.Factory.StartNew(async () => await Save(item, cancellationToken).ConfigureAwait(false), - cancellationToken, TaskCreationOptions.AttachedToParent, _writeScheduler); - await saving.Unwrap().ConfigureAwait(false); + cancellationToken, TaskCreationOptions.DenyChildAttach, _highest); } if (!currentClosures.ContainsKey(item.Id)) From f2c29d7658c1ecf607174444810f493405a75188 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Tue, 31 Dec 2024 14:31:30 +0000 Subject: [PATCH 10/31] Completely pre-cal batch size to avoid spinning issues --- .../Serialization/ChannelExtensions.cs | 2 +- .../Serialization/ChannelSaver.cs | 2 +- .../SizeBatchingChannelReader.cs | 50 +++++++++++-------- .../Serialisation/V2/Send/SerializeProcess.cs | 22 ++++---- 4 files changed, 43 insertions(+), 33 deletions(-) diff --git a/src/Speckle.Sdk.Dependencies/Serialization/ChannelExtensions.cs b/src/Speckle.Sdk.Dependencies/Serialization/ChannelExtensions.cs index cb56850c..a03a7337 100644 --- a/src/Speckle.Sdk.Dependencies/Serialization/ChannelExtensions.cs +++ b/src/Speckle.Sdk.Dependencies/Serialization/ChannelExtensions.cs @@ -5,7 +5,7 @@ namespace Speckle.Sdk.Serialisation.V2.Send; public static class ChannelExtensions { - public static BatchingChannelReader> BatchBySize( + public static BatchingChannelReader> BatchBySize( this ChannelReader source, int batchSize, bool singleReader = false, diff --git a/src/Speckle.Sdk.Dependencies/Serialization/ChannelSaver.cs b/src/Speckle.Sdk.Dependencies/Serialization/ChannelSaver.cs index c5221a4b..038de829 100644 --- a/src/Speckle.Sdk.Dependencies/Serialization/ChannelSaver.cs +++ b/src/Speckle.Sdk.Dependencies/Serialization/ChannelSaver.cs @@ -46,7 +46,7 @@ public Task Start(CancellationToken cancellationToken = default) => public ValueTask Save(T item, CancellationToken cancellationToken = default) => _checkCacheChannel.Writer.WriteAsync(item, cancellationToken); - public abstract Task> SendToServer(List batch, CancellationToken cancellationToken); + public abstract Task> SendToServer(Batch batch, CancellationToken cancellationToken); public Task Done() { diff --git a/src/Speckle.Sdk.Dependencies/Serialization/SizeBatchingChannelReader.cs b/src/Speckle.Sdk.Dependencies/Serialization/SizeBatchingChannelReader.cs index 9c668547..b05c9657 100644 --- a/src/Speckle.Sdk.Dependencies/Serialization/SizeBatchingChannelReader.cs +++ b/src/Speckle.Sdk.Dependencies/Serialization/SizeBatchingChannelReader.cs @@ -8,34 +8,44 @@ public interface IHasSize int Size { get; } } +public class Batch(int capacity) : IHasSize + where T : IHasSize +{ +#pragma warning disable IDE0032 + private readonly List _items = new(capacity); + private int _batchSize; +#pragma warning restore IDE0032 + + public void Add(T item) + { + _items.Add(item); + _batchSize += item.Size; + } + + public void TrimExcess() + { + _items.TrimExcess(); + _batchSize = _items.Sum(x => x.Size); + } + + public int Size => _batchSize; + public List Items => _items; +} + + public class SizeBatchingChannelReader( ChannelReader source, int batchSize, bool singleReader, bool syncCont = false -) : BatchingChannelReader>(source, batchSize, singleReader, syncCont) +) : BatchingChannelReader>(source, batchSize, singleReader, syncCont) where T : IHasSize { - private readonly int _batchSize = batchSize; + protected override Batch CreateBatch(int capacity) => new(capacity); - protected override List CreateBatch(int capacity) => new(); + protected override void TrimBatch(Batch batch) => batch.TrimExcess(); - protected override void TrimBatch(List batch) => batch.TrimExcess(); + protected override void AddBatchItem(Batch batch, T item) => batch.Add(item); - protected override void AddBatchItem(List batch, T item) => batch.Add(item); - - protected override int GetBatchSize(List batch) - { - int size = 0; - foreach (T item in batch) - { - size += item.Size; - } - - if (size >= _batchSize) - { - return _batchSize; - } - return size; - } + protected override int GetBatchSize(Batch batch) => batch.Size; } diff --git a/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs b/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs index 8073012b..fc6dd9e8 100644 --- a/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs +++ b/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs @@ -84,7 +84,7 @@ public void Dispose() public async Task Serialize(Base root, CancellationToken cancellationToken) { - var channelTask = Start(cancellationToken); + var channelTask = Start( cancellationToken); var findTotalObjectsTask = Task.CompletedTask; if (!_options.SkipFindTotalObjects) { @@ -133,7 +133,7 @@ private async Task> Traverse(Base obj, bool isEnd, Canc Dictionary[] taskClosures = []; if (tasks.Count > 0) { - taskClosures = await Task.WhenAll(tasks).ConfigureAwait(false); + taskClosures = await Task.WhenAll(tasks).ConfigureAwait(true); } var childClosures = _childClosurePool.Get(); foreach (var childClosure in taskClosures) @@ -157,7 +157,7 @@ private async Task> Traverse(Base obj, bool isEnd, Canc //just await enqueuing await Task.Factory.StartNew(async () => await Save(item, cancellationToken).ConfigureAwait(false), - cancellationToken, TaskCreationOptions.DenyChildAttach, _highest); + cancellationToken, TaskCreationOptions.DenyChildAttach, _highest).ConfigureAwait(true); } if (!currentClosures.ContainsKey(item.Id)) @@ -169,7 +169,7 @@ await Save(item, cancellationToken).ConfigureAwait(false), if (isEnd) { - await Done().ConfigureAwait(false); + await Done().ConfigureAwait(true); } return currentClosures; @@ -229,24 +229,24 @@ private BaseItem CheckCache(Id id, Json json, Dictionary closures) return new BaseItem(id, json, true, closures); } - public override async Task> SendToServer(List batch, CancellationToken cancellationToken) + public override async Task> SendToServer(Batch batch, CancellationToken cancellationToken) { - if (!_options.SkipServer && batch.Count != 0) + if (!_options.SkipServer && batch.Items.Count != 0) { - var objectBatch = batch.Distinct().ToList(); + var objectBatch = batch.Items.Distinct().ToList(); var hasObjects = await serverObjectManager .HasObjects(objectBatch.Select(x => x.Id.Value).Freeze(), cancellationToken) .ConfigureAwait(false); - objectBatch = batch.Where(x => !hasObjects[x.Id.Value]).ToList(); + objectBatch = batch.Items.Where(x => !hasObjects[x.Id.Value]).ToList(); if (objectBatch.Count != 0) { await serverObjectManager.UploadObjects(objectBatch, true, progress, cancellationToken).ConfigureAwait(false); - Interlocked.Exchange(ref _uploaded, _uploaded + batch.Count); + Interlocked.Exchange(ref _uploaded, _uploaded + batch.Items.Count); } - progress?.Report(new(ProgressEvent.UploadedObjects, _uploaded, null)); + return objectBatch; } - return batch; + return batch.Items; } public override void SaveToCache(List batch) From 52affdc1b7c63c889d20cfea50d3645664ae961c Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Tue, 31 Dec 2024 14:37:07 +0000 Subject: [PATCH 11/31] Try to fix cache counting --- src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs b/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs index fc6dd9e8..14d303c5 100644 --- a/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs +++ b/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs @@ -156,7 +156,10 @@ private async Task> Traverse(Base obj, bool isEnd, Canc { //just await enqueuing await Task.Factory.StartNew(async () => - await Save(item, cancellationToken).ConfigureAwait(false), + { + Interlocked.Increment(ref _objectsSerialized); + await Save(item, cancellationToken).ConfigureAwait(false); + }, cancellationToken, TaskCreationOptions.DenyChildAttach, _highest).ConfigureAwait(true); } @@ -197,7 +200,6 @@ CancellationToken cancellationToken try { items.AddRange(serializer2.Serialize(obj)); - Interlocked.Add(ref _objectsSerialized, items.Count); foreach (var kvp in serializer2.ObjectReferences) { _objectReferences.TryAdd(kvp.Key, kvp.Value); @@ -256,6 +258,7 @@ public override void SaveToCache(List batch) sqLiteJsonCacheManager.SaveObjects(batch.Select(x => (x.Id.Value, x.Json.Value))); Interlocked.Exchange(ref _cached, _cached + batch.Count); progress?.Report(new(ProgressEvent.CachedToLocal, _cached, _objectsSerialized)); + Console.WriteLine($"Saved {_cached} / {_objectsSerialized} objects to cache"); } } } From 550ee2b273c8044e46379b911c7b6e831bbd2973 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Tue, 31 Dec 2024 14:44:34 +0000 Subject: [PATCH 12/31] properly dispose things --- src/Speckle.Sdk/Api/Operations/Operations.Send.cs | 2 +- tests/Speckle.Sdk.Serialization.Testing/Program.cs | 2 +- tests/Speckle.Sdk.Serialization.Tests/DetachedTests.cs | 8 ++++---- .../ExplicitInterfaceTests.cs | 2 +- .../Speckle.Sdk.Serialization.Tests/SerializationTests.cs | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/Speckle.Sdk/Api/Operations/Operations.Send.cs b/src/Speckle.Sdk/Api/Operations/Operations.Send.cs index 94d6e270..3a2d1b36 100644 --- a/src/Speckle.Sdk/Api/Operations/Operations.Send.cs +++ b/src/Speckle.Sdk/Api/Operations/Operations.Send.cs @@ -25,7 +25,7 @@ public async Task Send2( try { - var process = serializeProcessFactory.CreateSerializeProcess(url, streamId, authorizationToken, onProgressAction); + using var process = serializeProcessFactory.CreateSerializeProcess(url, streamId, authorizationToken, onProgressAction); var results = await process.Serialize(value, cancellationToken).ConfigureAwait(false); receiveActivity?.SetStatus(SdkActivityStatusCode.Ok); diff --git a/tests/Speckle.Sdk.Serialization.Testing/Program.cs b/tests/Speckle.Sdk.Serialization.Testing/Program.cs index b807f795..b5a324f4 100644 --- a/tests/Speckle.Sdk.Serialization.Testing/Program.cs +++ b/tests/Speckle.Sdk.Serialization.Testing/Program.cs @@ -56,7 +56,7 @@ Console.ReadLine(); Console.WriteLine("Executing"); -var process2 = factory.CreateSerializeProcess( +using var process2 = factory.CreateSerializeProcess( new Uri(url), streamId, token, diff --git a/tests/Speckle.Sdk.Serialization.Tests/DetachedTests.cs b/tests/Speckle.Sdk.Serialization.Tests/DetachedTests.cs index 07d02914..37b92f02 100644 --- a/tests/Speckle.Sdk.Serialization.Tests/DetachedTests.cs +++ b/tests/Speckle.Sdk.Serialization.Tests/DetachedTests.cs @@ -67,7 +67,7 @@ public async Task CanSerialize_New_Detached() var objects = new Dictionary(); - var process2 = new SerializeProcess( + using var process2 = new SerializeProcess( null, new DummySendCacheManager(objects), new DummyServerObjectManager(), @@ -259,7 +259,7 @@ public async Task CanSerialize_New_Detached2() var objects = new Dictionary(); - var process2 = new SerializeProcess( + using var process2 = new SerializeProcess( null, new DummySendCacheManager(objects), new DummyServerObjectManager(), @@ -332,7 +332,7 @@ public async Task CanSerialize_New_Detached_With_DataChunks() var objects = new Dictionary(); - var process2 = new SerializeProcess( + using var process2 = new SerializeProcess( null, new DummySendCacheManager(objects), new DummyServerObjectManager(), @@ -413,7 +413,7 @@ public async Task CanSerialize_New_Detached_With_DataChunks2() var objects = new Dictionary(); - var process2 = new SerializeProcess( + using var process2 = new SerializeProcess( null, new DummySendCacheManager(objects), new DummyServerObjectManager(), diff --git a/tests/Speckle.Sdk.Serialization.Tests/ExplicitInterfaceTests.cs b/tests/Speckle.Sdk.Serialization.Tests/ExplicitInterfaceTests.cs index d82eef21..e90293de 100644 --- a/tests/Speckle.Sdk.Serialization.Tests/ExplicitInterfaceTests.cs +++ b/tests/Speckle.Sdk.Serialization.Tests/ExplicitInterfaceTests.cs @@ -21,7 +21,7 @@ public async Task Test_Json() var testClass = new TestClass() { RegularProperty = "Hello" }; var objects = new Dictionary(); - var process2 = new SerializeProcess( + using var process2 = new SerializeProcess( null, new DummySendCacheManager(objects), new DummyServerObjectManager(), diff --git a/tests/Speckle.Sdk.Serialization.Tests/SerializationTests.cs b/tests/Speckle.Sdk.Serialization.Tests/SerializationTests.cs index 49648b4a..aaf98d30 100644 --- a/tests/Speckle.Sdk.Serialization.Tests/SerializationTests.cs +++ b/tests/Speckle.Sdk.Serialization.Tests/SerializationTests.cs @@ -257,7 +257,7 @@ public async Task Roundtrip_Test_New(string fileName, string rootId, int oldCoun process.Total.ShouldBe(oldCount); var newIdToJson = new ConcurrentDictionary(); - var serializeProcess = new SerializeProcess( + using var serializeProcess = new SerializeProcess( null, new DummySqLiteSendManager(), new DummySendServerObjectManager(newIdToJson), From e1f1adefcd1cbd18dbd95528528fb9b6a418a129 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Tue, 31 Dec 2024 14:45:01 +0000 Subject: [PATCH 13/31] format --- .../SizeBatchingChannelReader.cs | 3 +-- .../Api/Operations/Operations.Send.cs | 7 ++++- .../V2/Send/PriorityScheduler.cs | 4 +-- .../Serialisation/V2/Send/SerializeProcess.cs | 27 ++++++++++--------- .../DetachedTests.cs | 4 +-- 5 files changed, 25 insertions(+), 20 deletions(-) diff --git a/src/Speckle.Sdk.Dependencies/Serialization/SizeBatchingChannelReader.cs b/src/Speckle.Sdk.Dependencies/Serialization/SizeBatchingChannelReader.cs index b05c9657..464cbb6d 100644 --- a/src/Speckle.Sdk.Dependencies/Serialization/SizeBatchingChannelReader.cs +++ b/src/Speckle.Sdk.Dependencies/Serialization/SizeBatchingChannelReader.cs @@ -21,7 +21,7 @@ public void Add(T item) _items.Add(item); _batchSize += item.Size; } - + public void TrimExcess() { _items.TrimExcess(); @@ -32,7 +32,6 @@ public void TrimExcess() public List Items => _items; } - public class SizeBatchingChannelReader( ChannelReader source, int batchSize, diff --git a/src/Speckle.Sdk/Api/Operations/Operations.Send.cs b/src/Speckle.Sdk/Api/Operations/Operations.Send.cs index 3a2d1b36..16636e70 100644 --- a/src/Speckle.Sdk/Api/Operations/Operations.Send.cs +++ b/src/Speckle.Sdk/Api/Operations/Operations.Send.cs @@ -25,7 +25,12 @@ public async Task Send2( try { - using var process = serializeProcessFactory.CreateSerializeProcess(url, streamId, authorizationToken, onProgressAction); + using var process = serializeProcessFactory.CreateSerializeProcess( + url, + streamId, + authorizationToken, + onProgressAction + ); var results = await process.Serialize(value, cancellationToken).ConfigureAwait(false); receiveActivity?.SetStatus(SdkActivityStatusCode.Ok); diff --git a/src/Speckle.Sdk/Serialisation/V2/Send/PriorityScheduler.cs b/src/Speckle.Sdk/Serialisation/V2/Send/PriorityScheduler.cs index a7aaac0c..3e254d1e 100644 --- a/src/Speckle.Sdk/Serialisation/V2/Send/PriorityScheduler.cs +++ b/src/Speckle.Sdk/Serialisation/V2/Send/PriorityScheduler.cs @@ -2,9 +2,7 @@ namespace Speckle.Sdk.Serialisation.V2.Send; -public sealed class PriorityScheduler(ThreadPriority priority, int maximumConcurrencyLevel) - : TaskScheduler, - IDisposable +public sealed class PriorityScheduler(ThreadPriority priority, int maximumConcurrencyLevel) : TaskScheduler, IDisposable { private readonly BlockingCollection _tasks = new(); private Thread[]? _threads; diff --git a/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs b/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs index 14d303c5..33bf74c3 100644 --- a/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs +++ b/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs @@ -51,11 +51,8 @@ public sealed class SerializeProcess( SerializeProcessOptions? options = null ) : ChannelSaver, ISerializeProcess { - private readonly PriorityScheduler _highest = new( ThreadPriority.Highest, 2); - private readonly PriorityScheduler _belowNormal = new( - ThreadPriority.BelowNormal, - Environment.ProcessorCount * 3 - ); + private readonly PriorityScheduler _highest = new(ThreadPriority.Highest, 2); + private readonly PriorityScheduler _belowNormal = new(ThreadPriority.BelowNormal, Environment.ProcessorCount * 3); private readonly SerializeProcessOptions _options = options ?? new(false, false, false, false); @@ -84,7 +81,7 @@ public void Dispose() public async Task Serialize(Base root, CancellationToken cancellationToken) { - var channelTask = Start( cancellationToken); + var channelTask = Start(cancellationToken); var findTotalObjectsTask = Task.CompletedTask; if (!_options.SkipFindTotalObjects) { @@ -155,12 +152,18 @@ private async Task> Traverse(Base obj, bool isEnd, Canc if (item.NeedsStorage) { //just await enqueuing - await Task.Factory.StartNew(async () => - { - Interlocked.Increment(ref _objectsSerialized); - await Save(item, cancellationToken).ConfigureAwait(false); - }, - cancellationToken, TaskCreationOptions.DenyChildAttach, _highest).ConfigureAwait(true); + await Task + .Factory.StartNew( + async () => + { + Interlocked.Increment(ref _objectsSerialized); + await Save(item, cancellationToken).ConfigureAwait(false); + }, + cancellationToken, + TaskCreationOptions.DenyChildAttach, + _highest + ) + .ConfigureAwait(true); } if (!currentClosures.ContainsKey(item.Id)) diff --git a/tests/Speckle.Sdk.Serialization.Tests/DetachedTests.cs b/tests/Speckle.Sdk.Serialization.Tests/DetachedTests.cs index 37b92f02..94a396e2 100644 --- a/tests/Speckle.Sdk.Serialization.Tests/DetachedTests.cs +++ b/tests/Speckle.Sdk.Serialization.Tests/DetachedTests.cs @@ -332,7 +332,7 @@ public async Task CanSerialize_New_Detached_With_DataChunks() var objects = new Dictionary(); - using var process2 = new SerializeProcess( + using var process2 = new SerializeProcess( null, new DummySendCacheManager(objects), new DummyServerObjectManager(), @@ -413,7 +413,7 @@ public async Task CanSerialize_New_Detached_With_DataChunks2() var objects = new Dictionary(); - using var process2 = new SerializeProcess( + using var process2 = new SerializeProcess( null, new DummySendCacheManager(objects), new DummyServerObjectManager(), From 747afa92764dd8df352d4f15a968097a506095ff Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Tue, 31 Dec 2024 14:48:55 +0000 Subject: [PATCH 14/31] clean up --- src/Speckle.Sdk.Dependencies/Pools.cs | 2 +- src/Speckle.Sdk.Dependencies/Serialization/ChannelSaver.cs | 2 +- src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs | 3 +-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/Speckle.Sdk.Dependencies/Pools.cs b/src/Speckle.Sdk.Dependencies/Pools.cs index 4774a16b..5f3451fb 100644 --- a/src/Speckle.Sdk.Dependencies/Pools.cs +++ b/src/Speckle.Sdk.Dependencies/Pools.cs @@ -63,6 +63,6 @@ public bool Return(List obj) public static Pool> CreateDictionaryPool() where TKey : notnull => new(new ObjectDictionaryPolicy()); - public static Pool> CreateConcurrentictionaryPool() + public static Pool> CreateConcurrentDictionaryPool() where TKey : notnull => new(new ObjectConcurrentDictionaryPolicy()); } diff --git a/src/Speckle.Sdk.Dependencies/Serialization/ChannelSaver.cs b/src/Speckle.Sdk.Dependencies/Serialization/ChannelSaver.cs index 038de829..97c00f10 100644 --- a/src/Speckle.Sdk.Dependencies/Serialization/ChannelSaver.cs +++ b/src/Speckle.Sdk.Dependencies/Serialization/ChannelSaver.cs @@ -18,7 +18,7 @@ public abstract class ChannelSaver private readonly Channel _checkCacheChannel = Channel.CreateBounded( new BoundedChannelOptions(SEND_CAPACITY) { - AllowSynchronousContinuations = false, + AllowSynchronousContinuations = true, Capacity = SEND_CAPACITY, SingleWriter = false, SingleReader = false, diff --git a/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs b/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs index 33bf74c3..08f53f6f 100644 --- a/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs +++ b/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs @@ -59,7 +59,7 @@ public sealed class SerializeProcess( private readonly ConcurrentDictionary _objectReferences = new(); private readonly Pool> _pool = Pools.CreateListPool<(Id, Json, Closures)>(); private readonly Pool> _currentClosurePool = Pools.CreateDictionaryPool(); - private readonly Pool> _childClosurePool = Pools.CreateConcurrentictionaryPool< + private readonly Pool> _childClosurePool = Pools.CreateConcurrentDictionaryPool< Id, NodeInfo >(); @@ -261,7 +261,6 @@ public override void SaveToCache(List batch) sqLiteJsonCacheManager.SaveObjects(batch.Select(x => (x.Id.Value, x.Json.Value))); Interlocked.Exchange(ref _cached, _cached + batch.Count); progress?.Report(new(ProgressEvent.CachedToLocal, _cached, _objectsSerialized)); - Console.WriteLine($"Saved {_cached} / {_objectsSerialized} objects to cache"); } } } From ad41e662f0e4bc67bca9334faf3415f246b830f9 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Tue, 31 Dec 2024 15:50:50 +0000 Subject: [PATCH 15/31] adjust count and save on current thread --- .../Serialisation/V2/Send/SerializeProcess.cs | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs b/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs index 08f53f6f..8f08173a 100644 --- a/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs +++ b/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs @@ -52,7 +52,7 @@ public sealed class SerializeProcess( ) : ChannelSaver, ISerializeProcess { private readonly PriorityScheduler _highest = new(ThreadPriority.Highest, 2); - private readonly PriorityScheduler _belowNormal = new(ThreadPriority.BelowNormal, Environment.ProcessorCount * 3); + private readonly PriorityScheduler _belowNormal = new(ThreadPriority.BelowNormal, Environment.ProcessorCount * 2); private readonly SerializeProcessOptions _options = options ?? new(false, false, false, false); @@ -151,19 +151,8 @@ private async Task> Traverse(Base obj, bool isEnd, Canc { if (item.NeedsStorage) { - //just await enqueuing - await Task - .Factory.StartNew( - async () => - { - Interlocked.Increment(ref _objectsSerialized); - await Save(item, cancellationToken).ConfigureAwait(false); - }, - cancellationToken, - TaskCreationOptions.DenyChildAttach, - _highest - ) - .ConfigureAwait(true); + Interlocked.Increment(ref _objectsSerialized); + await Save(item, cancellationToken).ConfigureAwait(true); } if (!currentClosures.ContainsKey(item.Id)) From c84e9ccef73166ff3bd08843f5f86008adfb2fc0 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Thu, 2 Jan 2025 09:24:23 +0000 Subject: [PATCH 16/31] move batch it's own file --- .../Serialization/Batch.cs | 25 +++++++++++++++++++ .../SizeBatchingChannelReader.cs | 24 ------------------ 2 files changed, 25 insertions(+), 24 deletions(-) create mode 100644 src/Speckle.Sdk.Dependencies/Serialization/Batch.cs diff --git a/src/Speckle.Sdk.Dependencies/Serialization/Batch.cs b/src/Speckle.Sdk.Dependencies/Serialization/Batch.cs new file mode 100644 index 00000000..368cfd5c --- /dev/null +++ b/src/Speckle.Sdk.Dependencies/Serialization/Batch.cs @@ -0,0 +1,25 @@ +namespace Speckle.Sdk.Serialisation.V2.Send; + +public class Batch(int capacity) : IHasSize + where T : IHasSize +{ +#pragma warning disable IDE0032 + private readonly List _items = new(capacity); + private int _batchSize; +#pragma warning restore IDE0032 + + public void Add(T item) + { + _items.Add(item); + _batchSize += item.Size; + } + + public void TrimExcess() + { + _items.TrimExcess(); + _batchSize = _items.Sum(x => x.Size); + } + + public int Size => _batchSize; + public List Items => _items; +} diff --git a/src/Speckle.Sdk.Dependencies/Serialization/SizeBatchingChannelReader.cs b/src/Speckle.Sdk.Dependencies/Serialization/SizeBatchingChannelReader.cs index 464cbb6d..6c161bee 100644 --- a/src/Speckle.Sdk.Dependencies/Serialization/SizeBatchingChannelReader.cs +++ b/src/Speckle.Sdk.Dependencies/Serialization/SizeBatchingChannelReader.cs @@ -8,30 +8,6 @@ public interface IHasSize int Size { get; } } -public class Batch(int capacity) : IHasSize - where T : IHasSize -{ -#pragma warning disable IDE0032 - private readonly List _items = new(capacity); - private int _batchSize; -#pragma warning restore IDE0032 - - public void Add(T item) - { - _items.Add(item); - _batchSize += item.Size; - } - - public void TrimExcess() - { - _items.TrimExcess(); - _batchSize = _items.Sum(x => x.Size); - } - - public int Size => _batchSize; - public List Items => _items; -} - public class SizeBatchingChannelReader( ChannelReader source, int batchSize, From 9e3c47b31179044017fccb02786e21083d4948f5 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Thu, 2 Jan 2025 09:25:09 +0000 Subject: [PATCH 17/31] update a few packages --- Directory.Packages.props | 7 ++-- build/packages.lock.json | 6 ++-- src/Speckle.Objects/packages.lock.json | 12 +++---- .../packages.lock.json | 36 +++++++++---------- src/Speckle.Sdk/packages.lock.json | 12 +++---- .../packages.lock.json | 6 ++-- .../packages.lock.json | 6 ++-- .../packages.lock.json | 6 ++-- .../packages.lock.json | 6 ++-- .../packages.lock.json | 6 ++-- .../Speckle.Sdk.Tests.Unit/packages.lock.json | 6 ++-- 11 files changed, 54 insertions(+), 55 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index d86ca569..6f1923c7 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -9,7 +9,7 @@ - + @@ -19,7 +19,7 @@ - + @@ -27,9 +27,8 @@ - - + diff --git a/build/packages.lock.json b/build/packages.lock.json index 0e580e7a..775329d9 100644 --- a/build/packages.lock.json +++ b/build/packages.lock.json @@ -32,9 +32,9 @@ }, "PolySharp": { "type": "Direct", - "requested": "[1.14.1, )", - "resolved": "1.14.1", - "contentHash": "mOOmFYwad3MIOL14VCjj02LljyF1GNw1wP0YVlxtcPvqdxjGGMNdNJJxHptlry3MOd8b40Flm8RPOM8JOlN2sQ==" + "requested": "[1.15.0, )", + "resolved": "1.15.0", + "contentHash": "FbU0El+EEjdpuIX4iDbeS7ki1uzpJPx8vbqOzEtqnl1GZeAGJfq+jCbxeJL2y0EPnUNk8dRnnqR2xnYXg9Tf+g==" }, "SimpleExec": { "type": "Direct", diff --git a/src/Speckle.Objects/packages.lock.json b/src/Speckle.Objects/packages.lock.json index e0eab833..74ea8aa1 100644 --- a/src/Speckle.Objects/packages.lock.json +++ b/src/Speckle.Objects/packages.lock.json @@ -29,9 +29,9 @@ }, "PolySharp": { "type": "Direct", - "requested": "[1.14.1, )", - "resolved": "1.14.1", - "contentHash": "mOOmFYwad3MIOL14VCjj02LljyF1GNw1wP0YVlxtcPvqdxjGGMNdNJJxHptlry3MOd8b40Flm8RPOM8JOlN2sQ==" + "requested": "[1.15.0, )", + "resolved": "1.15.0", + "contentHash": "FbU0El+EEjdpuIX4iDbeS7ki1uzpJPx8vbqOzEtqnl1GZeAGJfq+jCbxeJL2y0EPnUNk8dRnnqR2xnYXg9Tf+g==" }, "Speckle.InterfaceGenerator": { "type": "Direct", @@ -333,9 +333,9 @@ }, "PolySharp": { "type": "Direct", - "requested": "[1.14.1, )", - "resolved": "1.14.1", - "contentHash": "mOOmFYwad3MIOL14VCjj02LljyF1GNw1wP0YVlxtcPvqdxjGGMNdNJJxHptlry3MOd8b40Flm8RPOM8JOlN2sQ==" + "requested": "[1.15.0, )", + "resolved": "1.15.0", + "contentHash": "FbU0El+EEjdpuIX4iDbeS7ki1uzpJPx8vbqOzEtqnl1GZeAGJfq+jCbxeJL2y0EPnUNk8dRnnqR2xnYXg9Tf+g==" }, "Speckle.InterfaceGenerator": { "type": "Direct", diff --git a/src/Speckle.Sdk.Dependencies/packages.lock.json b/src/Speckle.Sdk.Dependencies/packages.lock.json index 80f29e97..38bc5fac 100644 --- a/src/Speckle.Sdk.Dependencies/packages.lock.json +++ b/src/Speckle.Sdk.Dependencies/packages.lock.json @@ -19,9 +19,9 @@ }, "Microsoft.Extensions.ObjectPool": { "type": "Direct", - "requested": "[8.0.10, )", - "resolved": "8.0.10", - "contentHash": "u7gAG7JgxF8VSJUGPSudAcPxOt+ymJKQCSxNRxiuKV+klCQbHljQR75SilpedCTfhPWDhtUwIJpnDVtspr9nMg==" + "requested": "[8.0.11, )", + "resolved": "8.0.11", + "contentHash": "6ApKcHNJigXBfZa6XlDQ8feJpq7SG1ogZXg6M4FiNzgd6irs3LUAzo0Pfn4F2ZI9liGnH1XIBR/OtSbZmJAV5w==" }, "Microsoft.SourceLink.GitHub": { "type": "Direct", @@ -44,9 +44,9 @@ }, "Open.ChannelExtensions": { "type": "Direct", - "requested": "[8.5.0, )", - "resolved": "8.5.0", - "contentHash": "dKD2iNfUYw+aOvwM2vCnD+q6JCtHiabkufKM1GateedRzcgv0RrtA4MoJI+7Y8N21R5A+wUA+j6P88g6mXPavA==", + "requested": "[8.6.0, )", + "resolved": "8.6.0", + "contentHash": "g5axz417bA6FXifJaBlB0l62gV7dYmknXx0n8lT/LSA3+7isaGMsOjJp5J+H/yXDRe4r+KZrE+bzQcs4Ets2kA==", "dependencies": { "Microsoft.Bcl.AsyncInterfaces": "8.0.0", "System.Collections.Immutable": "8.0.0", @@ -76,9 +76,9 @@ }, "PolySharp": { "type": "Direct", - "requested": "[1.14.1, )", - "resolved": "1.14.1", - "contentHash": "mOOmFYwad3MIOL14VCjj02LljyF1GNw1wP0YVlxtcPvqdxjGGMNdNJJxHptlry3MOd8b40Flm8RPOM8JOlN2sQ==" + "requested": "[1.15.0, )", + "resolved": "1.15.0", + "contentHash": "FbU0El+EEjdpuIX4iDbeS7ki1uzpJPx8vbqOzEtqnl1GZeAGJfq+jCbxeJL2y0EPnUNk8dRnnqR2xnYXg9Tf+g==" }, "Speckle.InterfaceGenerator": { "type": "Direct", @@ -185,9 +185,9 @@ }, "Microsoft.Extensions.ObjectPool": { "type": "Direct", - "requested": "[8.0.10, )", - "resolved": "8.0.10", - "contentHash": "u7gAG7JgxF8VSJUGPSudAcPxOt+ymJKQCSxNRxiuKV+klCQbHljQR75SilpedCTfhPWDhtUwIJpnDVtspr9nMg==" + "requested": "[8.0.11, )", + "resolved": "8.0.11", + "contentHash": "6ApKcHNJigXBfZa6XlDQ8feJpq7SG1ogZXg6M4FiNzgd6irs3LUAzo0Pfn4F2ZI9liGnH1XIBR/OtSbZmJAV5w==" }, "Microsoft.SourceLink.GitHub": { "type": "Direct", @@ -201,9 +201,9 @@ }, "Open.ChannelExtensions": { "type": "Direct", - "requested": "[8.5.0, )", - "resolved": "8.5.0", - "contentHash": "dKD2iNfUYw+aOvwM2vCnD+q6JCtHiabkufKM1GateedRzcgv0RrtA4MoJI+7Y8N21R5A+wUA+j6P88g6mXPavA==" + "requested": "[8.6.0, )", + "resolved": "8.6.0", + "contentHash": "g5axz417bA6FXifJaBlB0l62gV7dYmknXx0n8lT/LSA3+7isaGMsOjJp5J+H/yXDRe4r+KZrE+bzQcs4Ets2kA==" }, "Polly": { "type": "Direct", @@ -228,9 +228,9 @@ }, "PolySharp": { "type": "Direct", - "requested": "[1.14.1, )", - "resolved": "1.14.1", - "contentHash": "mOOmFYwad3MIOL14VCjj02LljyF1GNw1wP0YVlxtcPvqdxjGGMNdNJJxHptlry3MOd8b40Flm8RPOM8JOlN2sQ==" + "requested": "[1.15.0, )", + "resolved": "1.15.0", + "contentHash": "FbU0El+EEjdpuIX4iDbeS7ki1uzpJPx8vbqOzEtqnl1GZeAGJfq+jCbxeJL2y0EPnUNk8dRnnqR2xnYXg9Tf+g==" }, "Speckle.InterfaceGenerator": { "type": "Direct", diff --git a/src/Speckle.Sdk/packages.lock.json b/src/Speckle.Sdk/packages.lock.json index 03e98959..bd0ad6f2 100644 --- a/src/Speckle.Sdk/packages.lock.json +++ b/src/Speckle.Sdk/packages.lock.json @@ -83,9 +83,9 @@ }, "PolySharp": { "type": "Direct", - "requested": "[1.14.1, )", - "resolved": "1.14.1", - "contentHash": "mOOmFYwad3MIOL14VCjj02LljyF1GNw1wP0YVlxtcPvqdxjGGMNdNJJxHptlry3MOd8b40Flm8RPOM8JOlN2sQ==" + "requested": "[1.15.0, )", + "resolved": "1.15.0", + "contentHash": "FbU0El+EEjdpuIX4iDbeS7ki1uzpJPx8vbqOzEtqnl1GZeAGJfq+jCbxeJL2y0EPnUNk8dRnnqR2xnYXg9Tf+g==" }, "Speckle.DoubleNumerics": { "type": "Direct", @@ -364,9 +364,9 @@ }, "PolySharp": { "type": "Direct", - "requested": "[1.14.1, )", - "resolved": "1.14.1", - "contentHash": "mOOmFYwad3MIOL14VCjj02LljyF1GNw1wP0YVlxtcPvqdxjGGMNdNJJxHptlry3MOd8b40Flm8RPOM8JOlN2sQ==" + "requested": "[1.15.0, )", + "resolved": "1.15.0", + "contentHash": "FbU0El+EEjdpuIX4iDbeS7ki1uzpJPx8vbqOzEtqnl1GZeAGJfq+jCbxeJL2y0EPnUNk8dRnnqR2xnYXg9Tf+g==" }, "Speckle.DoubleNumerics": { "type": "Direct", diff --git a/tests/Speckle.Objects.Tests.Unit/packages.lock.json b/tests/Speckle.Objects.Tests.Unit/packages.lock.json index 973a8574..fcd06d85 100644 --- a/tests/Speckle.Objects.Tests.Unit/packages.lock.json +++ b/tests/Speckle.Objects.Tests.Unit/packages.lock.json @@ -48,9 +48,9 @@ }, "PolySharp": { "type": "Direct", - "requested": "[1.14.1, )", - "resolved": "1.14.1", - "contentHash": "mOOmFYwad3MIOL14VCjj02LljyF1GNw1wP0YVlxtcPvqdxjGGMNdNJJxHptlry3MOd8b40Flm8RPOM8JOlN2sQ==" + "requested": "[1.15.0, )", + "resolved": "1.15.0", + "contentHash": "FbU0El+EEjdpuIX4iDbeS7ki1uzpJPx8vbqOzEtqnl1GZeAGJfq+jCbxeJL2y0EPnUNk8dRnnqR2xnYXg9Tf+g==" }, "Shouldly": { "type": "Direct", diff --git a/tests/Speckle.Sdk.Serialization.Testing/packages.lock.json b/tests/Speckle.Sdk.Serialization.Testing/packages.lock.json index 6f507f12..544b5e42 100644 --- a/tests/Speckle.Sdk.Serialization.Testing/packages.lock.json +++ b/tests/Speckle.Sdk.Serialization.Testing/packages.lock.json @@ -29,9 +29,9 @@ }, "PolySharp": { "type": "Direct", - "requested": "[1.14.1, )", - "resolved": "1.14.1", - "contentHash": "mOOmFYwad3MIOL14VCjj02LljyF1GNw1wP0YVlxtcPvqdxjGGMNdNJJxHptlry3MOd8b40Flm8RPOM8JOlN2sQ==" + "requested": "[1.15.0, )", + "resolved": "1.15.0", + "contentHash": "FbU0El+EEjdpuIX4iDbeS7ki1uzpJPx8vbqOzEtqnl1GZeAGJfq+jCbxeJL2y0EPnUNk8dRnnqR2xnYXg9Tf+g==" }, "Speckle.InterfaceGenerator": { "type": "Direct", diff --git a/tests/Speckle.Sdk.Serialization.Tests/packages.lock.json b/tests/Speckle.Sdk.Serialization.Tests/packages.lock.json index 973a8574..fcd06d85 100644 --- a/tests/Speckle.Sdk.Serialization.Tests/packages.lock.json +++ b/tests/Speckle.Sdk.Serialization.Tests/packages.lock.json @@ -48,9 +48,9 @@ }, "PolySharp": { "type": "Direct", - "requested": "[1.14.1, )", - "resolved": "1.14.1", - "contentHash": "mOOmFYwad3MIOL14VCjj02LljyF1GNw1wP0YVlxtcPvqdxjGGMNdNJJxHptlry3MOd8b40Flm8RPOM8JOlN2sQ==" + "requested": "[1.15.0, )", + "resolved": "1.15.0", + "contentHash": "FbU0El+EEjdpuIX4iDbeS7ki1uzpJPx8vbqOzEtqnl1GZeAGJfq+jCbxeJL2y0EPnUNk8dRnnqR2xnYXg9Tf+g==" }, "Shouldly": { "type": "Direct", diff --git a/tests/Speckle.Sdk.Tests.Integration/packages.lock.json b/tests/Speckle.Sdk.Tests.Integration/packages.lock.json index 8fc495de..160e2842 100644 --- a/tests/Speckle.Sdk.Tests.Integration/packages.lock.json +++ b/tests/Speckle.Sdk.Tests.Integration/packages.lock.json @@ -48,9 +48,9 @@ }, "PolySharp": { "type": "Direct", - "requested": "[1.14.1, )", - "resolved": "1.14.1", - "contentHash": "mOOmFYwad3MIOL14VCjj02LljyF1GNw1wP0YVlxtcPvqdxjGGMNdNJJxHptlry3MOd8b40Flm8RPOM8JOlN2sQ==" + "requested": "[1.15.0, )", + "resolved": "1.15.0", + "contentHash": "FbU0El+EEjdpuIX4iDbeS7ki1uzpJPx8vbqOzEtqnl1GZeAGJfq+jCbxeJL2y0EPnUNk8dRnnqR2xnYXg9Tf+g==" }, "Shouldly": { "type": "Direct", diff --git a/tests/Speckle.Sdk.Tests.Performance/packages.lock.json b/tests/Speckle.Sdk.Tests.Performance/packages.lock.json index ee12cf74..d02dfe45 100644 --- a/tests/Speckle.Sdk.Tests.Performance/packages.lock.json +++ b/tests/Speckle.Sdk.Tests.Performance/packages.lock.json @@ -47,9 +47,9 @@ }, "PolySharp": { "type": "Direct", - "requested": "[1.14.1, )", - "resolved": "1.14.1", - "contentHash": "mOOmFYwad3MIOL14VCjj02LljyF1GNw1wP0YVlxtcPvqdxjGGMNdNJJxHptlry3MOd8b40Flm8RPOM8JOlN2sQ==" + "requested": "[1.15.0, )", + "resolved": "1.15.0", + "contentHash": "FbU0El+EEjdpuIX4iDbeS7ki1uzpJPx8vbqOzEtqnl1GZeAGJfq+jCbxeJL2y0EPnUNk8dRnnqR2xnYXg9Tf+g==" }, "Speckle.InterfaceGenerator": { "type": "Direct", diff --git a/tests/Speckle.Sdk.Tests.Unit/packages.lock.json b/tests/Speckle.Sdk.Tests.Unit/packages.lock.json index 5cb51559..7c77fd93 100644 --- a/tests/Speckle.Sdk.Tests.Unit/packages.lock.json +++ b/tests/Speckle.Sdk.Tests.Unit/packages.lock.json @@ -57,9 +57,9 @@ }, "PolySharp": { "type": "Direct", - "requested": "[1.14.1, )", - "resolved": "1.14.1", - "contentHash": "mOOmFYwad3MIOL14VCjj02LljyF1GNw1wP0YVlxtcPvqdxjGGMNdNJJxHptlry3MOd8b40Flm8RPOM8JOlN2sQ==" + "requested": "[1.15.0, )", + "resolved": "1.15.0", + "contentHash": "FbU0El+EEjdpuIX4iDbeS7ki1uzpJPx8vbqOzEtqnl1GZeAGJfq+jCbxeJL2y0EPnUNk8dRnnqR2xnYXg9Tf+g==" }, "Shouldly": { "type": "Direct", From 79d7ab9fe2843721ac9115bd552be1ae661d5c35 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Thu, 2 Jan 2025 09:40:26 +0000 Subject: [PATCH 18/31] fix build and add batch tests --- .../SizeBatchingChannelReader.cs | 10 ++++- .../Serialisation/BatchTests.cs | 44 +++++++++++++++++++ .../Serialisation/SimpleRoundTripTests.cs | 1 + 3 files changed, 53 insertions(+), 2 deletions(-) create mode 100644 tests/Speckle.Sdk.Tests.Unit/Serialisation/BatchTests.cs diff --git a/src/Speckle.Sdk.Dependencies/Serialization/SizeBatchingChannelReader.cs b/src/Speckle.Sdk.Dependencies/Serialization/SizeBatchingChannelReader.cs index 6c161bee..0f66b49b 100644 --- a/src/Speckle.Sdk.Dependencies/Serialization/SizeBatchingChannelReader.cs +++ b/src/Speckle.Sdk.Dependencies/Serialization/SizeBatchingChannelReader.cs @@ -13,12 +13,18 @@ public class SizeBatchingChannelReader( int batchSize, bool singleReader, bool syncCont = false -) : BatchingChannelReader>(source, batchSize, singleReader, syncCont) +) : BatchingChannelReader>(x => new(x), source, batchSize, singleReader, syncCont) where T : IHasSize { protected override Batch CreateBatch(int capacity) => new(capacity); - protected override void TrimBatch(Batch batch) => batch.TrimExcess(); + protected override void TrimBatch(ref Batch batch, bool isVerifiedFull) + { + if (!isVerifiedFull) + { + batch.TrimExcess(); + } + } protected override void AddBatchItem(Batch batch, T item) => batch.Add(item); diff --git a/tests/Speckle.Sdk.Tests.Unit/Serialisation/BatchTests.cs b/tests/Speckle.Sdk.Tests.Unit/Serialisation/BatchTests.cs new file mode 100644 index 00000000..e994288d --- /dev/null +++ b/tests/Speckle.Sdk.Tests.Unit/Serialisation/BatchTests.cs @@ -0,0 +1,44 @@ +using NUnit.Framework; +using Shouldly; +using Speckle.Sdk.Serialisation.V2.Send; + +namespace Speckle.Sdk.Tests.Unit.Serialisation; + +[TestFixture] +public class BatchTests +{ + private class BatchItem : IHasSize + { + public BatchItem(int size) + { + Size = size; + } + + public int Size { get; } + } + + [Test] + public void TestBatchSize_Calc() + { + var batch = new Batch(4); + batch.Add(new BatchItem(1)); + batch.Size.ShouldBe(1); + batch.Add(new BatchItem(2)); + batch.Size.ShouldBe(3); + } + + [Test] + public void TestBatchSize_Trim() + { + var batch = new Batch(4); + batch.Add(new BatchItem(1)); + batch.Add(new BatchItem(2)); + batch.Size.ShouldBe(3); + + batch.Items.Capacity.ShouldBe(4); + batch.TrimExcess(); + + batch.Items.Capacity.ShouldBe(2); + batch.Size.ShouldBe(3); + } +} diff --git a/tests/Speckle.Sdk.Tests.Unit/Serialisation/SimpleRoundTripTests.cs b/tests/Speckle.Sdk.Tests.Unit/Serialisation/SimpleRoundTripTests.cs index 3efc054f..b9cd0183 100644 --- a/tests/Speckle.Sdk.Tests.Unit/Serialisation/SimpleRoundTripTests.cs +++ b/tests/Speckle.Sdk.Tests.Unit/Serialisation/SimpleRoundTripTests.cs @@ -8,6 +8,7 @@ namespace Speckle.Sdk.Tests.Unit.Serialisation; +[TestFixture] public class SimpleRoundTripTests { private IOperations _operations; From b94252ba570f99e9d7b650995887bf28516cc3e2 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Thu, 2 Jan 2025 10:23:54 +0000 Subject: [PATCH 19/31] revert and format --- src/Speckle.Sdk/Credentials/AccountManager.cs | 2 ++ src/Speckle.Sdk/SQLite/SQLiteJsonCacheManager.cs | 1 + src/Speckle.Sdk/SQLite/SqLiteJsonCacheManagerFactory.cs | 6 ++++-- .../Serialisation/V2/Receive/DeserializeProcess.cs | 4 ++-- src/Speckle.Sdk/Serialisation/V2/Receive/ObjectLoader.cs | 2 ++ src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs | 2 ++ tests/Speckle.Sdk.Serialization.Tests/DetachedTests.cs | 3 ++- .../DummySqLiteReceiveManager.cs | 1 + .../DummySqLiteSendManager.cs | 2 +- tests/Speckle.Sdk.Serialization.Tests/SerializationTests.cs | 6 ++++-- 10 files changed, 21 insertions(+), 8 deletions(-) diff --git a/src/Speckle.Sdk/Credentials/AccountManager.cs b/src/Speckle.Sdk/Credentials/AccountManager.cs index 2af283d8..f8331d13 100644 --- a/src/Speckle.Sdk/Credentials/AccountManager.cs +++ b/src/Speckle.Sdk/Credentials/AccountManager.cs @@ -22,6 +22,7 @@ namespace Speckle.Sdk.Credentials; public partial interface IAccountManager : IDisposable; + /// /// Manage accounts locally for desktop applications. /// @@ -47,6 +48,7 @@ public void Dispose() _accountStorage.Dispose(); _accountAddLockStorage.Dispose(); } + /// /// Gets the basic information about a server. /// diff --git a/src/Speckle.Sdk/SQLite/SQLiteJsonCacheManager.cs b/src/Speckle.Sdk/SQLite/SQLiteJsonCacheManager.cs index 7e4789e2..36c03eb1 100644 --- a/src/Speckle.Sdk/SQLite/SQLiteJsonCacheManager.cs +++ b/src/Speckle.Sdk/SQLite/SQLiteJsonCacheManager.cs @@ -6,6 +6,7 @@ namespace Speckle.Sdk.SQLite; public partial interface ISqLiteJsonCacheManager : IDisposable; + [GenerateAutoInterface] public sealed class SqLiteJsonCacheManager : ISqLiteJsonCacheManager { diff --git a/src/Speckle.Sdk/SQLite/SqLiteJsonCacheManagerFactory.cs b/src/Speckle.Sdk/SQLite/SqLiteJsonCacheManagerFactory.cs index ef546bd2..f8fedca4 100644 --- a/src/Speckle.Sdk/SQLite/SqLiteJsonCacheManagerFactory.cs +++ b/src/Speckle.Sdk/SQLite/SqLiteJsonCacheManagerFactory.cs @@ -7,11 +7,13 @@ namespace Speckle.Sdk.SQLite; [GenerateAutoInterface] public class SqLiteJsonCacheManagerFactory : ISqLiteJsonCacheManagerFactory { - public const int INITIAL_CONCURRENCY = 4; + public const int INITIAL_CONCURRENCY = 4; + private ISqLiteJsonCacheManager Create(string path, int concurrency) => new SqLiteJsonCacheManager(path, concurrency); public ISqLiteJsonCacheManager CreateForUser(string scope) => Create(Path.Combine(SpecklePathProvider.UserApplicationDataPath(), "Speckle", $"{scope}.db"), 1); - public ISqLiteJsonCacheManager CreateFromStream(string streamId) => Create(SqlitePaths.GetDBPath(streamId), INITIAL_CONCURRENCY); + public ISqLiteJsonCacheManager CreateFromStream(string streamId) => + Create(SqlitePaths.GetDBPath(streamId), INITIAL_CONCURRENCY); } diff --git a/src/Speckle.Sdk/Serialisation/V2/Receive/DeserializeProcess.cs b/src/Speckle.Sdk/Serialisation/V2/Receive/DeserializeProcess.cs index 14281d18..5be948fd 100644 --- a/src/Speckle.Sdk/Serialisation/V2/Receive/DeserializeProcess.cs +++ b/src/Speckle.Sdk/Serialisation/V2/Receive/DeserializeProcess.cs @@ -14,6 +14,7 @@ public record DeserializeProcessOptions( ); public partial interface IDeserializeProcess : IDisposable; + [GenerateAutoInterface] public sealed class DeserializeProcess( IProgress? progress, @@ -30,11 +31,10 @@ public sealed class DeserializeProcess( public IReadOnlyDictionary BaseCache => _baseCache; public long Total { get; private set; } - + [AutoInterfaceIgnore] public void Dispose() => objectLoader.Dispose(); - public async Task Deserialize(string rootId, CancellationToken cancellationToken) { var (rootJson, childrenIds) = await objectLoader diff --git a/src/Speckle.Sdk/Serialisation/V2/Receive/ObjectLoader.cs b/src/Speckle.Sdk/Serialisation/V2/Receive/ObjectLoader.cs index bb885144..345190c5 100644 --- a/src/Speckle.Sdk/Serialisation/V2/Receive/ObjectLoader.cs +++ b/src/Speckle.Sdk/Serialisation/V2/Receive/ObjectLoader.cs @@ -10,6 +10,7 @@ namespace Speckle.Sdk.Serialisation.V2.Receive; public partial interface IObjectLoader : IDisposable; + [GenerateAutoInterface] public sealed class ObjectLoader( ISqLiteJsonCacheManager sqLiteJsonCacheManager, @@ -21,6 +22,7 @@ public sealed class ObjectLoader( private long _checkCache; private long _cached; private DeserializeProcessOptions _options = new(false); + [AutoInterfaceIgnore] public void Dispose() => sqLiteJsonCacheManager.Dispose(); diff --git a/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs b/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs index 5ed133a8..5e5ce152 100644 --- a/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs +++ b/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs @@ -25,6 +25,7 @@ IReadOnlyDictionary ConvertedReferences ); public partial interface ISerializeProcess : IDisposable; + [GenerateAutoInterface] public sealed class SerializeProcess( IProgress? progress, @@ -48,6 +49,7 @@ public sealed class SerializeProcess( private long _uploaded; private long _cached; + [AutoInterfaceIgnore] public void Dispose() => sqLiteJsonCacheManager.Dispose(); diff --git a/tests/Speckle.Sdk.Serialization.Tests/DetachedTests.cs b/tests/Speckle.Sdk.Serialization.Tests/DetachedTests.cs index 4ef63ebb..a60c1865 100644 --- a/tests/Speckle.Sdk.Serialization.Tests/DetachedTests.cs +++ b/tests/Speckle.Sdk.Serialization.Tests/DetachedTests.cs @@ -259,7 +259,7 @@ public async Task CanSerialize_New_Detached2() var objects = new Dictionary(); - using var process2 = new SerializeProcess( + using var process2 = new SerializeProcess( null, new DummySendCacheManager(objects), new DummyServerObjectManager(), @@ -530,6 +530,7 @@ CancellationToken cancellationToken public class DummySendCacheManager(Dictionary objects) : ISqLiteJsonCacheManager { public void Dispose() { } + public IReadOnlyCollection GetAllObjects() => throw new NotImplementedException(); public void DeleteObject(string id) => throw new NotImplementedException(); diff --git a/tests/Speckle.Sdk.Serialization.Tests/DummySqLiteReceiveManager.cs b/tests/Speckle.Sdk.Serialization.Tests/DummySqLiteReceiveManager.cs index 99258fe8..c2e60c7d 100644 --- a/tests/Speckle.Sdk.Serialization.Tests/DummySqLiteReceiveManager.cs +++ b/tests/Speckle.Sdk.Serialization.Tests/DummySqLiteReceiveManager.cs @@ -5,6 +5,7 @@ namespace Speckle.Sdk.Serialization.Tests; public class DummySqLiteReceiveManager(Dictionary savedObjects) : ISqLiteJsonCacheManager { public void Dispose() { } + public IReadOnlyCollection GetAllObjects() => throw new NotImplementedException(); public void DeleteObject(string id) => throw new NotImplementedException(); diff --git a/tests/Speckle.Sdk.Serialization.Tests/DummySqLiteSendManager.cs b/tests/Speckle.Sdk.Serialization.Tests/DummySqLiteSendManager.cs index 882704ff..e69a87b8 100644 --- a/tests/Speckle.Sdk.Serialization.Tests/DummySqLiteSendManager.cs +++ b/tests/Speckle.Sdk.Serialization.Tests/DummySqLiteSendManager.cs @@ -17,6 +17,6 @@ public class DummySqLiteSendManager : ISqLiteJsonCacheManager public IReadOnlyCollection GetAllObjects() => throw new NotImplementedException(); public void DeleteObject(string id) => throw new NotImplementedException(); - + public void Dispose() { } } diff --git a/tests/Speckle.Sdk.Serialization.Tests/SerializationTests.cs b/tests/Speckle.Sdk.Serialization.Tests/SerializationTests.cs index 23f24e49..076730a2 100644 --- a/tests/Speckle.Sdk.Serialization.Tests/SerializationTests.cs +++ b/tests/Speckle.Sdk.Serialization.Tests/SerializationTests.cs @@ -33,6 +33,7 @@ CancellationToken cancellationToken } public string? LoadId(string id) => null; + public void Dispose() { } } @@ -104,6 +105,7 @@ CancellationToken cancellationToken } public string? LoadId(string id) => idToObject.GetValueOrDefault(id); + public void Dispose() { } } @@ -156,7 +158,7 @@ public async Task Basic_Namespace_Validation_New(string fileName) var fullName = _assembly.GetManifestResourceNames().Single(x => x.EndsWith(fileName)); var json = await ReadJson(fullName); var closures = ReadAsObjects(json); - using var process = new DeserializeProcess(null, new TestObjectLoader(closures), new ObjectDeserializerFactory()); + using var process = new DeserializeProcess(null, new TestObjectLoader(closures), new ObjectDeserializerFactory()); await process.Deserialize("3416d3fe01c9196115514c4a2f41617b", default); foreach (var (id, objJson) in closures) { @@ -259,7 +261,7 @@ public async Task Roundtrip_Test_New(string fileName, string rootId, int oldCoun process.Total.ShouldBe(oldCount); var newIdToJson = new ConcurrentDictionary(); - using var serializeProcess = new SerializeProcess( + using var serializeProcess = new SerializeProcess( null, new DummySqLiteSendManager(), new DummySendServerObjectManager(newIdToJson), From f3f067eb913eb14e8d3cdce9e011d0303343c94c Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Thu, 2 Jan 2025 10:30:45 +0000 Subject: [PATCH 20/31] Revert "save saving increase" This reverts commit 3b50c857fb9a9da3fbf8193d36b149d2e0d2cb72. --- src/Speckle.Sdk.Dependencies/Serialization/ChannelLoader.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Speckle.Sdk.Dependencies/Serialization/ChannelLoader.cs b/src/Speckle.Sdk.Dependencies/Serialization/ChannelLoader.cs index 5305adf1..e9d6e729 100644 --- a/src/Speckle.Sdk.Dependencies/Serialization/ChannelLoader.cs +++ b/src/Speckle.Sdk.Dependencies/Serialization/ChannelLoader.cs @@ -9,7 +9,7 @@ public abstract class ChannelLoader private static readonly TimeSpan HTTP_BATCH_TIMEOUT = TimeSpan.FromSeconds(2); private static readonly int MAX_READ_CACHE_PARALLELISM = Environment.ProcessorCount; private const int MAX_SAVE_CACHE_BATCH = 200; - private const int MAX_SAVE_CACHE_PARALLELISM = 4; + private const int MAX_SAVE_CACHE_PARALLELISM = 1; protected async Task GetAndCache(IEnumerable allChildrenIds, CancellationToken cancellationToken = default) => await allChildrenIds From a96db42f16156abf7311c26a9307c4f60f0ccd08 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Thu, 2 Jan 2025 10:32:29 +0000 Subject: [PATCH 21/31] revert change --- src/Speckle.Sdk.Dependencies/Serialization/ChannelSaver.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Speckle.Sdk.Dependencies/Serialization/ChannelSaver.cs b/src/Speckle.Sdk.Dependencies/Serialization/ChannelSaver.cs index 865484c1..4f974eda 100644 --- a/src/Speckle.Sdk.Dependencies/Serialization/ChannelSaver.cs +++ b/src/Speckle.Sdk.Dependencies/Serialization/ChannelSaver.cs @@ -12,7 +12,7 @@ public abstract class ChannelSaver private static readonly TimeSpan HTTP_BATCH_TIMEOUT = TimeSpan.FromSeconds(2); private const int MAX_PARALLELISM_HTTP = 4; private const int HTTP_CAPACITY = 50; - private const int MAX_CACHE_WRITE_PARALLELISM = 4; + private const int MAX_CACHE_WRITE_PARALLELISM = 1; private const int MAX_CACHE_BATCH = 200; private bool _enabled; From f810fee0472017c4c63396e8a50c6c835ebe540f Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Thu, 2 Jan 2025 11:09:47 +0000 Subject: [PATCH 22/31] adjust and add tests --- src/Speckle.Sdk/Credentials/AccountManager.cs | 6 +- src/Speckle.Sdk/SQLite/CacheDbCommandPool.cs | 50 +++++------ src/Speckle.Sdk/SQLite/CacheDbCommands.cs | 2 +- .../SQLite/SQLiteJsonCacheManager.cs | 11 +-- .../SQLite/SqLiteJsonCacheManagerFactory.cs | 3 +- .../V2/DummySendServerObjectManager.cs | 2 +- .../DetachedTests.cs | 2 +- .../DummySqLiteReceiveManager.cs | 2 +- .../DummySqLiteSendManager.cs | 2 +- .../SQLite/SQLiteJsonCacheManagerTests.cs | 89 +++++++++++++++++++ 10 files changed, 127 insertions(+), 42 deletions(-) create mode 100644 tests/Speckle.Sdk.Tests.Unit/SQLite/SQLiteJsonCacheManagerTests.cs diff --git a/src/Speckle.Sdk/Credentials/AccountManager.cs b/src/Speckle.Sdk/Credentials/AccountManager.cs index f8331d13..41ac0271 100644 --- a/src/Speckle.Sdk/Credentials/AccountManager.cs +++ b/src/Speckle.Sdk/Credentials/AccountManager.cs @@ -330,7 +330,7 @@ public IEnumerable GetAccounts() { static bool IsInvalid(Account ac) => ac.userInfo == null || ac.serverInfo == null; - var sqlAccounts = _accountStorage.GetAllObjects().Select(x => JsonConvert.DeserializeObject(x)); + var sqlAccounts = _accountStorage.GetAllObjects().Select(x => JsonConvert.DeserializeObject(x.Item2)); var localAccounts = GetLocalAccounts(); foreach (var acc in sqlAccounts) @@ -651,7 +651,7 @@ private void TryLockAccountAddFlow(TimeSpan timespan) } // this uses the SQLite transport to store locks - var lockIds = _accountAddLockStorage.GetAllObjects().OrderByDescending(d => d).ToList(); + var lockIds = _accountAddLockStorage.GetAllObjects().Select(x => x.Item2).OrderByDescending(d => d).ToList(); var now = DateTime.Now; foreach (var l in lockIds) { @@ -683,7 +683,7 @@ private void UnlockAccountAddFlow() { s_isAddingAccount = false; // make sure all old locks are removed - foreach (var id in _accountAddLockStorage.GetAllObjects()) + foreach (var (id, _) in _accountAddLockStorage.GetAllObjects()) { _accountAddLockStorage.DeleteObject(id); } diff --git a/src/Speckle.Sdk/SQLite/CacheDbCommandPool.cs b/src/Speckle.Sdk/SQLite/CacheDbCommandPool.cs index 0242240d..500d2230 100644 --- a/src/Speckle.Sdk/SQLite/CacheDbCommandPool.cs +++ b/src/Speckle.Sdk/SQLite/CacheDbCommandPool.cs @@ -25,19 +25,17 @@ public CacheDbCommandPool(string connectionString, int concurrency) } } - public void Use(CacheOperation type, Action handler) - { - Use( + public void Use(CacheOperation type, Action handler) => + Use( type, - (cmd) => + cmd => { handler(cmd); return true; } ); - } - public T Use(Func handler) + private T Use(Func handler) { if (!_connections.TryTake(out var db)) { @@ -55,33 +53,29 @@ public T Use(Func handler) } } - public T Use(CacheOperation type, Func handler) - { - return Use( - (conn) => + public T Use(CacheOperation type, Func handler) => + Use(conn => + { + var pool = _commands[(int)type]; + if (!pool.TryTake(out var command)) { - var pool = _commands[(int)type]; - if (!pool.TryTake(out var command)) - { #pragma warning disable CA2100 - command = new SqliteCommand(CacheDbCommands.Commands[(int)type], conn); + command = new SqliteCommand(CacheDbCommands.Commands[(int)type], conn); #pragma warning restore CA2100 - } + } - try - { - command.Connection = conn; - return handler(command); - } - finally - { - command.Connection = null; - command.Parameters.Clear(); - pool.Add(command); - } + try + { + command.Connection = conn; + return handler(command); } - ); - } + finally + { + command.Connection = null; + command.Parameters.Clear(); + pool.Add(command); + } + }); public void Dispose() { diff --git a/src/Speckle.Sdk/SQLite/CacheDbCommands.cs b/src/Speckle.Sdk/SQLite/CacheDbCommands.cs index a56e4fbb..33a28771 100644 --- a/src/Speckle.Sdk/SQLite/CacheDbCommands.cs +++ b/src/Speckle.Sdk/SQLite/CacheDbCommands.cs @@ -28,7 +28,7 @@ static CacheDbCommands() Commands[(int)CacheOperation.Has] = "SELECT 1 FROM objects WHERE hash = @hash LIMIT 1"; Commands[(int)CacheOperation.Get] = "SELECT content FROM objects WHERE hash = @hash LIMIT 1"; Commands[(int)CacheOperation.Delete] = "DELETE FROM objects WHERE hash = @hash"; - Commands[(int)CacheOperation.GetAll] = "SELECT * FROM objects"; + Commands[(int)CacheOperation.GetAll] = "SELECT hash, content FROM objects"; Commands[(int)CacheOperation.BulkInsertOrIgnore] = "INSERT OR IGNORE INTO objects (hash, content) VALUES "; } diff --git a/src/Speckle.Sdk/SQLite/SQLiteJsonCacheManager.cs b/src/Speckle.Sdk/SQLite/SQLiteJsonCacheManager.cs index 36c03eb1..ef3d1549 100644 --- a/src/Speckle.Sdk/SQLite/SQLiteJsonCacheManager.cs +++ b/src/Speckle.Sdk/SQLite/SQLiteJsonCacheManager.cs @@ -13,9 +13,9 @@ public sealed class SqLiteJsonCacheManager : ISqLiteJsonCacheManager private readonly string _connectionString; private readonly CacheDbCommandPool _pool; - public SqLiteJsonCacheManager(string rootPath, int concurrency) + public SqLiteJsonCacheManager(string connectionString, int concurrency) { - _connectionString = $"Data Source={rootPath};"; + _connectionString = connectionString; Initialize(); _pool = new CacheDbCommandPool(_connectionString, concurrency); } @@ -66,18 +66,19 @@ content TEXT using SqliteCommand cmd4 = new("PRAGMA page_size = 32768;", c); cmd4.ExecuteNonQuery(); + c.Close(); } - public IReadOnlyCollection GetAllObjects() => + public IReadOnlyCollection<(string, string)> GetAllObjects() => _pool.Use( CacheOperation.GetAll, command => { - var list = new HashSet(); + var list = new HashSet<(string, string)>(); using var reader = command.ExecuteReader(); while (reader.Read()) { - list.Add(reader.GetString(1)); + list.Add((reader.GetString(0), reader.GetString(1))); } return list; } diff --git a/src/Speckle.Sdk/SQLite/SqLiteJsonCacheManagerFactory.cs b/src/Speckle.Sdk/SQLite/SqLiteJsonCacheManagerFactory.cs index f8fedca4..94eb9339 100644 --- a/src/Speckle.Sdk/SQLite/SqLiteJsonCacheManagerFactory.cs +++ b/src/Speckle.Sdk/SQLite/SqLiteJsonCacheManagerFactory.cs @@ -9,7 +9,8 @@ public class SqLiteJsonCacheManagerFactory : ISqLiteJsonCacheManagerFactory { public const int INITIAL_CONCURRENCY = 4; - private ISqLiteJsonCacheManager Create(string path, int concurrency) => new SqLiteJsonCacheManager(path, concurrency); + private ISqLiteJsonCacheManager Create(string path, int concurrency) => + new SqLiteJsonCacheManager($"Data Source={path};", concurrency); public ISqLiteJsonCacheManager CreateForUser(string scope) => Create(Path.Combine(SpecklePathProvider.UserApplicationDataPath(), "Speckle", $"{scope}.db"), 1); diff --git a/src/Speckle.Sdk/Serialisation/V2/DummySendServerObjectManager.cs b/src/Speckle.Sdk/Serialisation/V2/DummySendServerObjectManager.cs index 626f9516..26e50477 100644 --- a/src/Speckle.Sdk/Serialisation/V2/DummySendServerObjectManager.cs +++ b/src/Speckle.Sdk/Serialisation/V2/DummySendServerObjectManager.cs @@ -9,7 +9,7 @@ public sealed class DummySqLiteJsonCacheManager : ISqLiteJsonCacheManager { public void Dispose() { } - public IReadOnlyCollection GetAllObjects() => throw new NotImplementedException(); + public IReadOnlyCollection<(string, string)> GetAllObjects() => throw new NotImplementedException(); public void DeleteObject(string id) => throw new NotImplementedException(); diff --git a/tests/Speckle.Sdk.Serialization.Tests/DetachedTests.cs b/tests/Speckle.Sdk.Serialization.Tests/DetachedTests.cs index a60c1865..0448cb4a 100644 --- a/tests/Speckle.Sdk.Serialization.Tests/DetachedTests.cs +++ b/tests/Speckle.Sdk.Serialization.Tests/DetachedTests.cs @@ -531,7 +531,7 @@ public class DummySendCacheManager(Dictionary objects) : ISqLite { public void Dispose() { } - public IReadOnlyCollection GetAllObjects() => throw new NotImplementedException(); + public IReadOnlyCollection<(string, string)> GetAllObjects() => throw new NotImplementedException(); public void DeleteObject(string id) => throw new NotImplementedException(); diff --git a/tests/Speckle.Sdk.Serialization.Tests/DummySqLiteReceiveManager.cs b/tests/Speckle.Sdk.Serialization.Tests/DummySqLiteReceiveManager.cs index c2e60c7d..797eacd9 100644 --- a/tests/Speckle.Sdk.Serialization.Tests/DummySqLiteReceiveManager.cs +++ b/tests/Speckle.Sdk.Serialization.Tests/DummySqLiteReceiveManager.cs @@ -6,7 +6,7 @@ public class DummySqLiteReceiveManager(Dictionary savedObjects) { public void Dispose() { } - public IReadOnlyCollection GetAllObjects() => throw new NotImplementedException(); + public IReadOnlyCollection<(string, string)> GetAllObjects() => throw new NotImplementedException(); public void DeleteObject(string id) => throw new NotImplementedException(); diff --git a/tests/Speckle.Sdk.Serialization.Tests/DummySqLiteSendManager.cs b/tests/Speckle.Sdk.Serialization.Tests/DummySqLiteSendManager.cs index e69a87b8..0b93af62 100644 --- a/tests/Speckle.Sdk.Serialization.Tests/DummySqLiteSendManager.cs +++ b/tests/Speckle.Sdk.Serialization.Tests/DummySqLiteSendManager.cs @@ -14,7 +14,7 @@ public class DummySqLiteSendManager : ISqLiteJsonCacheManager public bool HasObject(string objectId) => throw new NotImplementedException(); - public IReadOnlyCollection GetAllObjects() => throw new NotImplementedException(); + public IReadOnlyCollection<(string, string)> GetAllObjects() => throw new NotImplementedException(); public void DeleteObject(string id) => throw new NotImplementedException(); diff --git a/tests/Speckle.Sdk.Tests.Unit/SQLite/SQLiteJsonCacheManagerTests.cs b/tests/Speckle.Sdk.Tests.Unit/SQLite/SQLiteJsonCacheManagerTests.cs new file mode 100644 index 00000000..4f479bb8 --- /dev/null +++ b/tests/Speckle.Sdk.Tests.Unit/SQLite/SQLiteJsonCacheManagerTests.cs @@ -0,0 +1,89 @@ +using Microsoft.Data.Sqlite; +using NUnit.Framework; +using Shouldly; +using Speckle.Sdk.Common; +using Speckle.Sdk.SQLite; + +namespace Speckle.Sdk.Tests.Unit.SQLite; + +[TestFixture] +public class SQLiteJsonCacheManagerTests +{ + private readonly string _basePath = $"{Guid.NewGuid()}.db"; + private string? _connectionString; + + [SetUp] + public void Setup() => _connectionString = $"Data Source={_basePath}.;"; + + [TearDown] + public void TearDown() + { + if (File.Exists(_basePath)) + { + SqliteConnection.ClearAllPools(); + GC.Collect(); + GC.WaitForPendingFinalizers(); + File.Delete(_basePath); + } + } + + [Test] + public void TestGetAll() + { + var data = new List<(string id, string json)>() { ("1", "1"), ("2", "2") }; + using var manager = new SqLiteJsonCacheManager(_connectionString.NotNull(), 2); + manager.SaveObjects(data); + var items = manager.GetAllObjects(); + items.Count.ShouldBe(data.Count); + foreach (var (id, json) in items) + { + data.Contains((id, json)).ShouldBeTrue(); + } + } + + [Test] + public void TestGet() + { + var data = new List<(string id, string json)>() { ("1", "1"), ("2", "2") }; + using var manager = new SqLiteJsonCacheManager(_connectionString.NotNull(), 2); + foreach (var d in data) + { + manager.SaveObject(d.id, d.json); + } + foreach (var d in data) + { + manager.SaveObject(d.id, d.json); + } + var items = manager.GetAllObjects(); + items.Count.ShouldBe(data.Count); + + var id1 = data[0].id; + var json1 = manager.GetObject(id1); + json1.ShouldBe(data[0].json); + manager.HasObject(id1).ShouldBeTrue(); + + manager.UpdateObject(id1, "3"); + json1 = manager.GetObject(id1); + json1.ShouldBe("3"); + manager.HasObject(id1).ShouldBeTrue(); + + manager.DeleteObject(id1); + json1 = manager.GetObject(id1); + json1.ShouldBeNull(); + manager.HasObject(id1).ShouldBeFalse(); + + manager.UpdateObject(id1, "3"); + json1 = manager.GetObject(id1); + json1.ShouldBe("3"); + manager.HasObject(id1).ShouldBeTrue(); + + var id2 = data[1].id; + var json2 = manager.GetObject(id2); + json2.ShouldBe(data[1].json); + manager.HasObject(id2).ShouldBeTrue(); + manager.DeleteObject(id2); + json2 = manager.GetObject(id2); + json2.ShouldBeNull(); + manager.HasObject(id2).ShouldBeFalse(); + } +} From c13f7a723be91a6679c75b6a929116f36affffcd Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Thu, 2 Jan 2025 11:15:20 +0000 Subject: [PATCH 23/31] Dispose sqlite manager properly --- src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs b/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs index bf6edc4e..aa73e14b 100644 --- a/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs +++ b/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs @@ -60,6 +60,7 @@ public void Dispose() { _highest.Dispose(); _belowNormal.Dispose(); + sqLiteJsonCacheManager.Dispose(); } public async Task Serialize(Base root, CancellationToken cancellationToken) From 429ce775866f8bd63867c7806d963b98beaec80d Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Thu, 2 Jan 2025 13:03:46 +0000 Subject: [PATCH 24/31] Make Batch a IMemoryOwner to allow for pooling --- src/Speckle.Sdk.Dependencies/Serialization/Batch.cs | 13 ++++++++++--- .../Serialization/ChannelExtensions.cs | 5 +++-- .../Serialization/ChannelSaver.cs | 10 ++++++++-- .../Serialization/SizeBatchingChannelReader.cs | 13 +++++++------ .../Serialisation/V2/Send/SerializeProcess.cs | 4 +--- 5 files changed, 29 insertions(+), 16 deletions(-) diff --git a/src/Speckle.Sdk.Dependencies/Serialization/Batch.cs b/src/Speckle.Sdk.Dependencies/Serialization/Batch.cs index 368cfd5c..6a2ebffc 100644 --- a/src/Speckle.Sdk.Dependencies/Serialization/Batch.cs +++ b/src/Speckle.Sdk.Dependencies/Serialization/Batch.cs @@ -1,10 +1,14 @@ -namespace Speckle.Sdk.Serialisation.V2.Send; +using System.Buffers; +using Speckle.Sdk.Dependencies; -public class Batch(int capacity) : IHasSize +namespace Speckle.Sdk.Serialisation.V2.Send; + +public sealed class Batch : IHasSize, IMemoryOwner where T : IHasSize { + private static readonly Pool> _pool = Pools.CreateListPool(); #pragma warning disable IDE0032 - private readonly List _items = new(capacity); + private readonly List _items = _pool.Get(); private int _batchSize; #pragma warning restore IDE0032 @@ -22,4 +26,7 @@ public void TrimExcess() public int Size => _batchSize; public List Items => _items; + public void Dispose() => _pool.Return(_items); + + public Memory Memory => new Memory(_items.ToArray()); } diff --git a/src/Speckle.Sdk.Dependencies/Serialization/ChannelExtensions.cs b/src/Speckle.Sdk.Dependencies/Serialization/ChannelExtensions.cs index a03a7337..c088e77e 100644 --- a/src/Speckle.Sdk.Dependencies/Serialization/ChannelExtensions.cs +++ b/src/Speckle.Sdk.Dependencies/Serialization/ChannelExtensions.cs @@ -1,11 +1,12 @@ -using System.Threading.Channels; +using System.Buffers; +using System.Threading.Channels; using Open.ChannelExtensions; namespace Speckle.Sdk.Serialisation.V2.Send; public static class ChannelExtensions { - public static BatchingChannelReader> BatchBySize( + public static BatchingChannelReader> BatchBySize( this ChannelReader source, int batchSize, bool singleReader = false, diff --git a/src/Speckle.Sdk.Dependencies/Serialization/ChannelSaver.cs b/src/Speckle.Sdk.Dependencies/Serialization/ChannelSaver.cs index 97c00f10..f9012c02 100644 --- a/src/Speckle.Sdk.Dependencies/Serialization/ChannelSaver.cs +++ b/src/Speckle.Sdk.Dependencies/Serialization/ChannelSaver.cs @@ -1,3 +1,4 @@ +using System.Buffers; using System.Threading.Channels; using Open.ChannelExtensions; using Speckle.Sdk.Serialisation.V2.Send; @@ -13,7 +14,7 @@ public abstract class ChannelSaver private const int MAX_PARALLELISM_HTTP = 4; private const int HTTP_CAPACITY = 500; private const int MAX_CACHE_WRITE_PARALLELISM = 4; - private const int MAX_CACHE_BATCH = 200; + private const int MAX_CACHE_BATCH = 500; private readonly Channel _checkCacheChannel = Channel.CreateBounded( new BoundedChannelOptions(SEND_CAPACITY) @@ -46,7 +47,12 @@ public Task Start(CancellationToken cancellationToken = default) => public ValueTask Save(T item, CancellationToken cancellationToken = default) => _checkCacheChannel.Writer.WriteAsync(item, cancellationToken); - public abstract Task> SendToServer(Batch batch, CancellationToken cancellationToken); + public async Task> SendToServer(IMemoryOwner batch, CancellationToken cancellationToken) + { + await SendToServer((Batch)batch, cancellationToken).ConfigureAwait(false); + return batch; + } + public abstract Task SendToServer(Batch batch, CancellationToken cancellationToken); public Task Done() { diff --git a/src/Speckle.Sdk.Dependencies/Serialization/SizeBatchingChannelReader.cs b/src/Speckle.Sdk.Dependencies/Serialization/SizeBatchingChannelReader.cs index 0f66b49b..e979aba9 100644 --- a/src/Speckle.Sdk.Dependencies/Serialization/SizeBatchingChannelReader.cs +++ b/src/Speckle.Sdk.Dependencies/Serialization/SizeBatchingChannelReader.cs @@ -1,3 +1,4 @@ +using System.Buffers; using System.Threading.Channels; using Open.ChannelExtensions; @@ -13,20 +14,20 @@ public class SizeBatchingChannelReader( int batchSize, bool singleReader, bool syncCont = false -) : BatchingChannelReader>(x => new(x), source, batchSize, singleReader, syncCont) +) : BatchingChannelReader>(x => new Batch(), source, batchSize, singleReader, syncCont) where T : IHasSize { - protected override Batch CreateBatch(int capacity) => new(capacity); + protected override IMemoryOwner CreateBatch(int capacity) => new Batch(); - protected override void TrimBatch(ref Batch batch, bool isVerifiedFull) + protected override void TrimBatch(ref IMemoryOwner batch, bool isVerifiedFull) { if (!isVerifiedFull) { - batch.TrimExcess(); + ((Batch)batch).TrimExcess(); } } - protected override void AddBatchItem(Batch batch, T item) => batch.Add(item); + protected override void AddBatchItem(IMemoryOwner batch, T item) => ((Batch)batch).Add(item); - protected override int GetBatchSize(Batch batch) => batch.Size; + protected override int GetBatchSize(IMemoryOwner batch) => ((Batch)batch).Size; } diff --git a/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs b/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs index aa73e14b..242fc5a0 100644 --- a/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs +++ b/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs @@ -207,7 +207,7 @@ private BaseItem CheckCache(Id id, Json json, Dictionary closures) return new BaseItem(id, json, true, closures); } - public override async Task> SendToServer(Batch batch, CancellationToken cancellationToken) + public override async Task SendToServer(Batch batch, CancellationToken cancellationToken) { if (!_options.SkipServer && batch.Items.Count != 0) { @@ -222,9 +222,7 @@ public override async Task> SendToServer(Batch batch, C Interlocked.Exchange(ref _uploaded, _uploaded + batch.Items.Count); } progress?.Report(new(ProgressEvent.UploadedObjects, _uploaded, null)); - return objectBatch; } - return batch.Items; } public override void SaveToCache(List batch) From 1c6d14bc6a5703ddbfeb61c44c41a931957e783c Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Thu, 2 Jan 2025 13:11:21 +0000 Subject: [PATCH 25/31] Fix tests --- src/Speckle.Sdk.Dependencies/Pools.cs | 8 +++++--- src/Speckle.Sdk.Dependencies/Serialization/Batch.cs | 3 ++- .../Serialization/ChannelSaver.cs | 1 + tests/Speckle.Sdk.Tests.Unit/Serialisation/BatchTests.cs | 7 ++++--- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/Speckle.Sdk.Dependencies/Pools.cs b/src/Speckle.Sdk.Dependencies/Pools.cs index 5f3451fb..990ffd05 100644 --- a/src/Speckle.Sdk.Dependencies/Pools.cs +++ b/src/Speckle.Sdk.Dependencies/Pools.cs @@ -6,6 +6,8 @@ namespace Speckle.Sdk.Dependencies; public static class Pools { + public const int DefaultCapacity = 50; + public static Pool> ObjectDictionaries { get; } = new(new ObjectDictionaryPolicy()); private sealed class ObjectDictionaryPolicy : IPooledObjectPolicy> @@ -25,7 +27,7 @@ public bool Return(Dictionary obj) private sealed class ObjectDictionaryPolicy : IPooledObjectPolicy> where TKey : notnull { - public Dictionary Create() => new(50); + public Dictionary Create() => new(DefaultCapacity); public bool Return(Dictionary obj) { @@ -38,7 +40,7 @@ private sealed class ObjectConcurrentDictionaryPolicy : IPooledObjectPolicy> where TKey : notnull { - public ConcurrentDictionary Create() => new(Environment.ProcessorCount, 50); + public ConcurrentDictionary Create() => new(Environment.ProcessorCount, DefaultCapacity); public bool Return(ConcurrentDictionary obj) { @@ -49,7 +51,7 @@ public bool Return(ConcurrentDictionary obj) private sealed class ObjectListPolicy : IPooledObjectPolicy> { - public List Create() => new(50); + public List Create() => new(DefaultCapacity); public bool Return(List obj) { diff --git a/src/Speckle.Sdk.Dependencies/Serialization/Batch.cs b/src/Speckle.Sdk.Dependencies/Serialization/Batch.cs index 6a2ebffc..fb6bb14c 100644 --- a/src/Speckle.Sdk.Dependencies/Serialization/Batch.cs +++ b/src/Speckle.Sdk.Dependencies/Serialization/Batch.cs @@ -26,7 +26,8 @@ public void TrimExcess() public int Size => _batchSize; public List Items => _items; + public void Dispose() => _pool.Return(_items); - public Memory Memory => new Memory(_items.ToArray()); + public Memory Memory => new(_items.ToArray()); } diff --git a/src/Speckle.Sdk.Dependencies/Serialization/ChannelSaver.cs b/src/Speckle.Sdk.Dependencies/Serialization/ChannelSaver.cs index f9012c02..99190998 100644 --- a/src/Speckle.Sdk.Dependencies/Serialization/ChannelSaver.cs +++ b/src/Speckle.Sdk.Dependencies/Serialization/ChannelSaver.cs @@ -52,6 +52,7 @@ public async Task> SendToServer(IMemoryOwner batch, Cancellat await SendToServer((Batch)batch, cancellationToken).ConfigureAwait(false); return batch; } + public abstract Task SendToServer(Batch batch, CancellationToken cancellationToken); public Task Done() diff --git a/tests/Speckle.Sdk.Tests.Unit/Serialisation/BatchTests.cs b/tests/Speckle.Sdk.Tests.Unit/Serialisation/BatchTests.cs index e994288d..6c6c81e5 100644 --- a/tests/Speckle.Sdk.Tests.Unit/Serialisation/BatchTests.cs +++ b/tests/Speckle.Sdk.Tests.Unit/Serialisation/BatchTests.cs @@ -1,5 +1,6 @@ using NUnit.Framework; using Shouldly; +using Speckle.Sdk.Dependencies; using Speckle.Sdk.Serialisation.V2.Send; namespace Speckle.Sdk.Tests.Unit.Serialisation; @@ -20,7 +21,7 @@ public BatchItem(int size) [Test] public void TestBatchSize_Calc() { - var batch = new Batch(4); + using var batch = new Batch(); batch.Add(new BatchItem(1)); batch.Size.ShouldBe(1); batch.Add(new BatchItem(2)); @@ -30,12 +31,12 @@ public void TestBatchSize_Calc() [Test] public void TestBatchSize_Trim() { - var batch = new Batch(4); + using var batch = new Batch(); batch.Add(new BatchItem(1)); batch.Add(new BatchItem(2)); batch.Size.ShouldBe(3); - batch.Items.Capacity.ShouldBe(4); + batch.Items.Capacity.ShouldBe(Pools.DefaultCapacity); batch.TrimExcess(); batch.Items.Capacity.ShouldBe(2); From 7f7645ad8d5ad74edd3ed4611b6a07f8516b7ef7 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Thu, 2 Jan 2025 13:11:43 +0000 Subject: [PATCH 26/31] Upgrade some deps --- Directory.Packages.props | 6 +-- .../packages.lock.json | 51 ++++++++++--------- 2 files changed, 29 insertions(+), 28 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 6f1923c7..f2acca28 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -9,7 +9,7 @@ - + @@ -19,7 +19,7 @@ - + @@ -27,7 +27,7 @@ - + diff --git a/src/Speckle.Sdk.Dependencies/packages.lock.json b/src/Speckle.Sdk.Dependencies/packages.lock.json index 38bc5fac..123ae1af 100644 --- a/src/Speckle.Sdk.Dependencies/packages.lock.json +++ b/src/Speckle.Sdk.Dependencies/packages.lock.json @@ -19,9 +19,9 @@ }, "Microsoft.Extensions.ObjectPool": { "type": "Direct", - "requested": "[8.0.11, )", - "resolved": "8.0.11", - "contentHash": "6ApKcHNJigXBfZa6XlDQ8feJpq7SG1ogZXg6M4FiNzgd6irs3LUAzo0Pfn4F2ZI9liGnH1XIBR/OtSbZmJAV5w==" + "requested": "[9.0.0, )", + "resolved": "9.0.0", + "contentHash": "UbsU/gYe4nv1DeqMXIVzDfNNek7Sk2kKuAOXL/Y+sLcAR0HwFUqzg1EPiU88jeHNe0g81aPvvHbvHarQr3r9IA==" }, "Microsoft.SourceLink.GitHub": { "type": "Direct", @@ -44,13 +44,13 @@ }, "Open.ChannelExtensions": { "type": "Direct", - "requested": "[8.6.0, )", - "resolved": "8.6.0", - "contentHash": "g5axz417bA6FXifJaBlB0l62gV7dYmknXx0n8lT/LSA3+7isaGMsOjJp5J+H/yXDRe4r+KZrE+bzQcs4Ets2kA==", + "requested": "[9.0.0, )", + "resolved": "9.0.0", + "contentHash": "DP+l5S6G46wcuY4I4kNXE+RDOmJr0DKuMienOdt0mMBN9z7vmLSC8YQbqCyb9i9LNjXj1tgCx5LyitJiRr/v7g==", "dependencies": { - "Microsoft.Bcl.AsyncInterfaces": "8.0.0", - "System.Collections.Immutable": "8.0.0", - "System.Threading.Channels": "8.0.0" + "Microsoft.Bcl.AsyncInterfaces": "9.0.0", + "System.Collections.Immutable": "9.0.0", + "System.Threading.Channels": "9.0.0" } }, "Polly": { @@ -88,10 +88,11 @@ }, "System.Threading.Channels": { "type": "Direct", - "requested": "[8.0.0, )", - "resolved": "8.0.0", - "contentHash": "CMaFr7v+57RW7uZfZkPExsPB6ljwzhjACWW1gfU35Y56rk72B/Wu+sTqxVmGSk4SFUlPc3cjeKND0zktziyjBA==", + "requested": "[9.0.0, )", + "resolved": "9.0.0", + "contentHash": "hzACdIf1C+4Dqos5ijV404b94+LqfIC8nfS3mNpCDFWowb1N3PNfJPopneq32ahWlDeyaPZJqjBk76YFR69Rpg==", "dependencies": { + "Microsoft.Bcl.AsyncInterfaces": "9.0.0", "System.Threading.Tasks.Extensions": "4.5.4" } }, @@ -122,8 +123,8 @@ }, "System.Collections.Immutable": { "type": "Transitive", - "resolved": "8.0.0", - "contentHash": "AurL6Y5BA1WotzlEvVaIDpqzpIPvYnnldxru8oXJU2yFxFUy3+pNXjXd1ymO+RA0rq0+590Q8gaz2l3Sr7fmqg==", + "resolved": "9.0.0", + "contentHash": "QhkXUl2gNrQtvPmtBTQHb0YsUrDiDQ2QS09YbtTTiSjGcf7NBqtYbrG/BE06zcBPCKEwQGzIv13IVdXNOSub2w==", "dependencies": { "System.Memory": "4.5.5", "System.Runtime.CompilerServices.Unsafe": "6.0.0" @@ -160,8 +161,8 @@ "Microsoft.Bcl.AsyncInterfaces": { "type": "CentralTransitive", "requested": "[5.0.0, )", - "resolved": "8.0.0", - "contentHash": "3WA9q9yVqJp222P3x1wYIGDAkpjAku0TMUaaQV22g6L67AI0LdOIrVS7Ht2vJfLHGSPVuqN94vIr15qn+HEkHw==", + "resolved": "9.0.0", + "contentHash": "owmu2Cr3IQ8yQiBleBHlGk8dSQ12oaF2e7TpzwJKEl4m84kkZJjEY1n33L67Y3zM5jPOjmmbdHjbfiL0RqcMRQ==", "dependencies": { "System.Threading.Tasks.Extensions": "4.5.4" } @@ -185,9 +186,9 @@ }, "Microsoft.Extensions.ObjectPool": { "type": "Direct", - "requested": "[8.0.11, )", - "resolved": "8.0.11", - "contentHash": "6ApKcHNJigXBfZa6XlDQ8feJpq7SG1ogZXg6M4FiNzgd6irs3LUAzo0Pfn4F2ZI9liGnH1XIBR/OtSbZmJAV5w==" + "requested": "[9.0.0, )", + "resolved": "9.0.0", + "contentHash": "UbsU/gYe4nv1DeqMXIVzDfNNek7Sk2kKuAOXL/Y+sLcAR0HwFUqzg1EPiU88jeHNe0g81aPvvHbvHarQr3r9IA==" }, "Microsoft.SourceLink.GitHub": { "type": "Direct", @@ -201,9 +202,9 @@ }, "Open.ChannelExtensions": { "type": "Direct", - "requested": "[8.6.0, )", - "resolved": "8.6.0", - "contentHash": "g5axz417bA6FXifJaBlB0l62gV7dYmknXx0n8lT/LSA3+7isaGMsOjJp5J+H/yXDRe4r+KZrE+bzQcs4Ets2kA==" + "requested": "[9.0.0, )", + "resolved": "9.0.0", + "contentHash": "DP+l5S6G46wcuY4I4kNXE+RDOmJr0DKuMienOdt0mMBN9z7vmLSC8YQbqCyb9i9LNjXj1tgCx5LyitJiRr/v7g==" }, "Polly": { "type": "Direct", @@ -240,9 +241,9 @@ }, "System.Threading.Channels": { "type": "Direct", - "requested": "[8.0.0, )", - "resolved": "8.0.0", - "contentHash": "CMaFr7v+57RW7uZfZkPExsPB6ljwzhjACWW1gfU35Y56rk72B/Wu+sTqxVmGSk4SFUlPc3cjeKND0zktziyjBA==" + "requested": "[9.0.0, )", + "resolved": "9.0.0", + "contentHash": "hzACdIf1C+4Dqos5ijV404b94+LqfIC8nfS3mNpCDFWowb1N3PNfJPopneq32ahWlDeyaPZJqjBk76YFR69Rpg==" }, "ILRepack": { "type": "Transitive", From b202f73653bc61fa1b4831e411a9857e56dc6704 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Thu, 2 Jan 2025 13:49:10 +0000 Subject: [PATCH 27/31] try to make tests more explicit --- .../SQLite/SQLiteJsonCacheManagerTests.cs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/Speckle.Sdk.Tests.Unit/SQLite/SQLiteJsonCacheManagerTests.cs b/tests/Speckle.Sdk.Tests.Unit/SQLite/SQLiteJsonCacheManagerTests.cs index 4f479bb8..961729a5 100644 --- a/tests/Speckle.Sdk.Tests.Unit/SQLite/SQLiteJsonCacheManagerTests.cs +++ b/tests/Speckle.Sdk.Tests.Unit/SQLite/SQLiteJsonCacheManagerTests.cs @@ -13,7 +13,7 @@ public class SQLiteJsonCacheManagerTests private string? _connectionString; [SetUp] - public void Setup() => _connectionString = $"Data Source={_basePath}.;"; + public void Setup() => _connectionString = $"Data Source={_basePath};"; [TearDown] public void TearDown() @@ -30,21 +30,23 @@ public void TearDown() [Test] public void TestGetAll() { - var data = new List<(string id, string json)>() { ("1", "1"), ("2", "2") }; + var data = new List<(string id, string json)>() { ("id1", "1"), ("id2", "2") }; using var manager = new SqLiteJsonCacheManager(_connectionString.NotNull(), 2); manager.SaveObjects(data); var items = manager.GetAllObjects(); items.Count.ShouldBe(data.Count); - foreach (var (id, json) in items) + var i = items.ToDictionary(); + foreach (var (id, json) in data) { - data.Contains((id, json)).ShouldBeTrue(); + i.TryGetValue(id, out var j).ShouldBeTrue(); + j.ShouldBe(json); } } [Test] public void TestGet() { - var data = new List<(string id, string json)>() { ("1", "1"), ("2", "2") }; + var data = new List<(string id, string json)>() { ("id1", "1"), ("id2", "2") }; using var manager = new SqLiteJsonCacheManager(_connectionString.NotNull(), 2); foreach (var d in data) { From 17a1398f32fc09f0b1807057e9806f8bdb104d17 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Fri, 3 Jan 2025 14:45:52 +0000 Subject: [PATCH 28/31] remove return value --- src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs b/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs index 6b08bb8c..242fc5a0 100644 --- a/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs +++ b/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs @@ -222,7 +222,6 @@ public override async Task SendToServer(Batch batch, CancellationToken Interlocked.Exchange(ref _uploaded, _uploaded + batch.Items.Count); } progress?.Report(new(ProgressEvent.UploadedObjects, _uploaded, null)); - return objectBatch; } } From 893a02c00c821e026a4750b997ee66cf45f04929 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Fri, 3 Jan 2025 15:19:40 +0000 Subject: [PATCH 29/31] Add detailed SqLiteJsonCacheException --- src/Speckle.Sdk/SQLite/CacheDbCommandPool.cs | 8 ++ .../SQLite/SqLiteJsonCacheException.cs | 30 +++++ src/Speckle.Sdk/SQLite/SqliteExceptions.cs | 104 ++++++++++++++++++ 3 files changed, 142 insertions(+) create mode 100644 src/Speckle.Sdk/SQLite/SqLiteJsonCacheException.cs create mode 100644 src/Speckle.Sdk/SQLite/SqliteExceptions.cs diff --git a/src/Speckle.Sdk/SQLite/CacheDbCommandPool.cs b/src/Speckle.Sdk/SQLite/CacheDbCommandPool.cs index 500d2230..e6a8ffc1 100644 --- a/src/Speckle.Sdk/SQLite/CacheDbCommandPool.cs +++ b/src/Speckle.Sdk/SQLite/CacheDbCommandPool.cs @@ -47,6 +47,10 @@ private T Use(Func handler) { return handler(db); } + catch (SqliteException se) + { + throw SqLiteJsonCacheException.Create(se); + } finally { _connections.Add(db); @@ -69,6 +73,10 @@ public T Use(CacheOperation type, Func handler) => command.Connection = conn; return handler(command); } + catch (SqliteException se) + { + throw SqLiteJsonCacheException.Create(se); + } finally { command.Connection = null; diff --git a/src/Speckle.Sdk/SQLite/SqLiteJsonCacheException.cs b/src/Speckle.Sdk/SQLite/SqLiteJsonCacheException.cs new file mode 100644 index 00000000..da5400c0 --- /dev/null +++ b/src/Speckle.Sdk/SQLite/SqLiteJsonCacheException.cs @@ -0,0 +1,30 @@ +using Microsoft.Data.Sqlite; + +namespace Speckle.Sdk.SQLite; + +public class SqLiteJsonCacheException : SpeckleException +{ + public SqLiteJsonCacheException() { } + + public SqLiteJsonCacheException(string message) + : base(message) { } + + public SqLiteJsonCacheException(string message, Exception inner) + : base(message, inner) { } + + public static SqLiteJsonCacheException Create(SqliteException inner) + { + if (!SqliteExceptions.SqliteErrorCodes.TryGetValue(inner.SqliteErrorCode, out string? errorMessage)) + { + errorMessage = $"An error occurred while executing a SQLite command: {inner.SqliteErrorCode}"; + } + if (!SqliteExceptions.SqliteErrorCodes.TryGetValue(inner.SqliteExtendedErrorCode, out string? detailedMessage)) + { + detailedMessage = $"Detail: {inner.SqliteExtendedErrorCode}"; + } + return new SqLiteJsonCacheException( + $"An error occured with the SQLite cache:{Environment.NewLine}{errorMessage}{Environment.NewLine}{detailedMessage}", + inner + ); + } +} diff --git a/src/Speckle.Sdk/SQLite/SqliteExceptions.cs b/src/Speckle.Sdk/SQLite/SqliteExceptions.cs new file mode 100644 index 00000000..aee95122 --- /dev/null +++ b/src/Speckle.Sdk/SQLite/SqliteExceptions.cs @@ -0,0 +1,104 @@ +namespace Speckle.Sdk.SQLite; + +internal static class SqliteExceptions +{ + public static readonly IReadOnlyDictionary SqliteErrorCodes = new Dictionary + { + { 0, "Successful result" }, + { 1, "Generic error" }, + { 2, "Internal logic error in SQLite" }, + { 3, "Access permission denied" }, + { 4, "Callback routine requested an abort" }, + { 5, "The database file is locked" }, + { 6, "A table in the database is locked" }, + { 7, "A malloc() failed" }, + { 8, "Attempt to write a readonly database" }, + { 9, "Operation terminated by sqlite3_interrupt()" }, + { 10, "Some kind of disk I/O error occurred" }, + { 11, "The database disk image is malformed" }, + { 12, "Unknown opcode in sqlite3_file_control()" }, + { 13, "Insertion failed because database is full" }, + { 14, "Unable to open the database file" }, + { 15, "Database lock protocol error" }, + { 16, "Internal use only" }, + { 17, "The database schema changed" }, + { 18, "String or BLOB exceeds size limit" }, + { 19, "Abort due to constraint violation" }, + { 20, "Data type mismatch" }, + { 21, "Library used incorrectly" }, + { 22, "Uses OS features not supported on host" }, + { 23, "Authorization denied" }, + { 24, "Not used" }, + { 25, "2nd parameter to sqlite3_bind out of range" }, + { 26, "File opened that is not a database file" }, + { 27, "Notifications from sqlite3_log()" }, + { 28, "Warnings from sqlite3_log()" }, + { 100, "sqlite3_step() has another row ready" }, + { 101, "sqlite3_step() has finished executing" }, + }; + + public static readonly IReadOnlyDictionary SqliteExtendedResultCodes = new Dictionary() + { + // Successful result codes + { 0, "SQLITE_OK: Successful result" }, + // General error + { 1, "SQLITE_ERROR: Generic error" }, + // I/O errors + { 10, "SQLITE_IOERR: Some kind of disk I/O error occurred" }, + { 266, "SQLITE_IOERR_READ: I/O error during read operation" }, + { 267, "SQLITE_IOERR_SHORT_READ: I/O error: short read" }, + { 778, "SQLITE_IOERR_WRITE: I/O error during write operation" }, + { 1034, "SQLITE_IOERR_FSYNC: I/O error during fsync()" }, + { 1290, "SQLITE_IOERR_DIR_FSYNC: I/O error during directory fsync()" }, + { 1546, "SQLITE_IOERR_TRUNCATE: I/O error during file truncate" }, + { 1802, "SQLITE_IOERR_FSTAT: I/O error during file stat" }, + { 2058, "SQLITE_IOERR_UNLOCK: I/O error during file unlock" }, + { 2314, "SQLITE_IOERR_RDLOCK: I/O error during read lock" }, + { 2570, "SQLITE_IOERR_DELETE: I/O error during file delete" }, + { 2826, "SQLITE_IOERR_BLOCKED: I/O error while blocked on lock" }, + { 3338, "SQLITE_IOERR_NOMEM: I/O error due to malloc failure" }, + { 3594, "SQLITE_IOERR_ACCESS: I/O error during access check" }, + { 3850, "SQLITE_IOERR_CHECKRESERVEDLOCK: I/O error during reserved lock check" }, + { 4106, "SQLITE_IOERR_LOCK: I/O error when acquiring lock" }, + { 4362, "SQLITE_IOERR_CLOSE: I/O error during file close" }, + { 4618, "SQLITE_IOERR_DIR_CLOSE: I/O error when closing dir" }, + { 4874, "SQLITE_IOERR_SHMOPEN: I/O error during shared memory open" }, + { 5130, "SQLITE_IOERR_SHMSIZE: I/O error during shared memory sizing" }, + { 5642, "SQLITE_IOERR_SEEK: I/O error during seek" }, + { 5898, "SQLITE_IOERR_DELETE_NOENT: I/O error attempting to delete file" }, + { 6154, "SQLITE_IOERR_MMAP: I/O error during memory map" }, + { 6410, "SQLITE_IOERR_GETTEMPPATH: I/O error during temp path creation" }, + { 6922, "SQLITE_IOERR_CONVPATH: I/O error converting path" }, + { 7178, "SQLITE_IOERR_VNODE: I/O error in virtual node" }, + { 7426, "SQLITE_IOERR_AUTH: I/O error due to authorization failure" }, + // Corrupt database errors + { 11, "SQLITE_CORRUPT: The database disk image is malformed" }, + { 267, "SQLITE_CORRUPT_VTAB: Column is corrupt in virtual table" }, + // Read-only errors + { 8, "SQLITE_READONLY: Attempt to write to a readonly database" }, + { 264, "SQLITE_READONLY_RECOVERY: Access denied due to recovery mode" }, + { 520, "SQLITE_READONLY_CANTLOCK: Could not lock the database" }, + { 776, "SQLITE_READONLY_ROLLBACK: Read-only during rollback operation" }, + { 1032, "SQLITE_READONLY_DBMOVED: Database file moved, causing read-only error" }, + // Constraint errors + { 19, "SQLITE_CONSTRAINT: Constraint violation" }, + { 275, "SQLITE_CONSTRAINT_UNIQUE: UNIQUE constraint failed" }, + { 531, "SQLITE_CONSTRAINT_FOREIGNKEY: FOREIGN KEY constraint failed" }, + { 787, "SQLITE_CONSTRAINT_NOTNULL: NOT NULL constraint failed" }, + { 1043, "SQLITE_CONSTRAINT_CHECK: CHECK constraint failed" }, + { 1299, "SQLITE_CONSTRAINT_PRIMARYKEY: PRIMARY KEY constraint failed" }, + { 1555, "SQLITE_CONSTRAINT_TRIGGER: Trigger caused constraint failure" }, + { 1803, "SQLITE_CONSTRAINT_ROWID: Row ID constraint violated" }, + { 2067, "SQLITE_CONSTRAINT_PINNED: Pinned constraint violation" }, + // CANTOPEN errors + { 14, "SQLITE_CANTOPEN: Cannot open database file" }, + { 1038, "SQLITE_CANTOPEN_NOTEMPDIR: Unable to open temporary directory" }, + { 1542, "SQLITE_CANTOPEN_ISDIR: Cannot open database file as it is a directory" }, + { 1806, "SQLITE_CANTOPEN_FULLPATH: Full filepath cannot be accessed" }, + { 4102, "SQLITE_CANTOPEN_CONVPATH: Cannot open database file due to conversion path error" }, + // Other + { 26, "SQLITE_NOTADB: File opened is not a database file" }, + { 100, "SQLITE_ROW: sqlite3_step() has another row ready" }, + { 101, "SQLITE_DONE: sqlite3_step() has finished executing" }, + }; +} From 60c2072ae1679f204ecf72459dfcda815e851892 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Fri, 3 Jan 2025 15:44:55 +0000 Subject: [PATCH 30/31] details changes --- .../SQLite/SqLiteJsonCacheException.cs | 9 +- src/Speckle.Sdk/SQLite/SqliteExceptions.cs | 125 +++++++++--------- 2 files changed, 71 insertions(+), 63 deletions(-) diff --git a/src/Speckle.Sdk/SQLite/SqLiteJsonCacheException.cs b/src/Speckle.Sdk/SQLite/SqLiteJsonCacheException.cs index da5400c0..b98de798 100644 --- a/src/Speckle.Sdk/SQLite/SqLiteJsonCacheException.cs +++ b/src/Speckle.Sdk/SQLite/SqLiteJsonCacheException.cs @@ -18,12 +18,17 @@ public static SqLiteJsonCacheException Create(SqliteException inner) { errorMessage = $"An error occurred while executing a SQLite command: {inner.SqliteErrorCode}"; } - if (!SqliteExceptions.SqliteErrorCodes.TryGetValue(inner.SqliteExtendedErrorCode, out string? detailedMessage)) + if ( + !SqliteExceptions.SqliteExtendedResultCodes.TryGetValue( + inner.SqliteExtendedErrorCode, + out string? detailedMessage + ) + ) { detailedMessage = $"Detail: {inner.SqliteExtendedErrorCode}"; } return new SqLiteJsonCacheException( - $"An error occured with the SQLite cache:{Environment.NewLine}{errorMessage}{Environment.NewLine}{detailedMessage}", + $"An error occured with the SQLite cache: {inner.Message}{Environment.NewLine}{errorMessage}{Environment.NewLine}{detailedMessage}", inner ); } diff --git a/src/Speckle.Sdk/SQLite/SqliteExceptions.cs b/src/Speckle.Sdk/SQLite/SqliteExceptions.cs index aee95122..eec7738c 100644 --- a/src/Speckle.Sdk/SQLite/SqliteExceptions.cs +++ b/src/Speckle.Sdk/SQLite/SqliteExceptions.cs @@ -39,66 +39,69 @@ internal static class SqliteExceptions public static readonly IReadOnlyDictionary SqliteExtendedResultCodes = new Dictionary() { - // Successful result codes - { 0, "SQLITE_OK: Successful result" }, - // General error - { 1, "SQLITE_ERROR: Generic error" }, - // I/O errors - { 10, "SQLITE_IOERR: Some kind of disk I/O error occurred" }, - { 266, "SQLITE_IOERR_READ: I/O error during read operation" }, - { 267, "SQLITE_IOERR_SHORT_READ: I/O error: short read" }, - { 778, "SQLITE_IOERR_WRITE: I/O error during write operation" }, - { 1034, "SQLITE_IOERR_FSYNC: I/O error during fsync()" }, - { 1290, "SQLITE_IOERR_DIR_FSYNC: I/O error during directory fsync()" }, - { 1546, "SQLITE_IOERR_TRUNCATE: I/O error during file truncate" }, - { 1802, "SQLITE_IOERR_FSTAT: I/O error during file stat" }, - { 2058, "SQLITE_IOERR_UNLOCK: I/O error during file unlock" }, - { 2314, "SQLITE_IOERR_RDLOCK: I/O error during read lock" }, - { 2570, "SQLITE_IOERR_DELETE: I/O error during file delete" }, - { 2826, "SQLITE_IOERR_BLOCKED: I/O error while blocked on lock" }, - { 3338, "SQLITE_IOERR_NOMEM: I/O error due to malloc failure" }, - { 3594, "SQLITE_IOERR_ACCESS: I/O error during access check" }, - { 3850, "SQLITE_IOERR_CHECKRESERVEDLOCK: I/O error during reserved lock check" }, - { 4106, "SQLITE_IOERR_LOCK: I/O error when acquiring lock" }, - { 4362, "SQLITE_IOERR_CLOSE: I/O error during file close" }, - { 4618, "SQLITE_IOERR_DIR_CLOSE: I/O error when closing dir" }, - { 4874, "SQLITE_IOERR_SHMOPEN: I/O error during shared memory open" }, - { 5130, "SQLITE_IOERR_SHMSIZE: I/O error during shared memory sizing" }, - { 5642, "SQLITE_IOERR_SEEK: I/O error during seek" }, - { 5898, "SQLITE_IOERR_DELETE_NOENT: I/O error attempting to delete file" }, - { 6154, "SQLITE_IOERR_MMAP: I/O error during memory map" }, - { 6410, "SQLITE_IOERR_GETTEMPPATH: I/O error during temp path creation" }, - { 6922, "SQLITE_IOERR_CONVPATH: I/O error converting path" }, - { 7178, "SQLITE_IOERR_VNODE: I/O error in virtual node" }, - { 7426, "SQLITE_IOERR_AUTH: I/O error due to authorization failure" }, - // Corrupt database errors - { 11, "SQLITE_CORRUPT: The database disk image is malformed" }, - { 267, "SQLITE_CORRUPT_VTAB: Column is corrupt in virtual table" }, - // Read-only errors - { 8, "SQLITE_READONLY: Attempt to write to a readonly database" }, - { 264, "SQLITE_READONLY_RECOVERY: Access denied due to recovery mode" }, - { 520, "SQLITE_READONLY_CANTLOCK: Could not lock the database" }, - { 776, "SQLITE_READONLY_ROLLBACK: Read-only during rollback operation" }, - { 1032, "SQLITE_READONLY_DBMOVED: Database file moved, causing read-only error" }, - // Constraint errors - { 19, "SQLITE_CONSTRAINT: Constraint violation" }, - { 275, "SQLITE_CONSTRAINT_UNIQUE: UNIQUE constraint failed" }, - { 531, "SQLITE_CONSTRAINT_FOREIGNKEY: FOREIGN KEY constraint failed" }, - { 787, "SQLITE_CONSTRAINT_NOTNULL: NOT NULL constraint failed" }, - { 1043, "SQLITE_CONSTRAINT_CHECK: CHECK constraint failed" }, - { 1299, "SQLITE_CONSTRAINT_PRIMARYKEY: PRIMARY KEY constraint failed" }, - { 1555, "SQLITE_CONSTRAINT_TRIGGER: Trigger caused constraint failure" }, - { 1803, "SQLITE_CONSTRAINT_ROWID: Row ID constraint violated" }, - { 2067, "SQLITE_CONSTRAINT_PINNED: Pinned constraint violation" }, - // CANTOPEN errors - { 14, "SQLITE_CANTOPEN: Cannot open database file" }, - { 1038, "SQLITE_CANTOPEN_NOTEMPDIR: Unable to open temporary directory" }, - { 1542, "SQLITE_CANTOPEN_ISDIR: Cannot open database file as it is a directory" }, - { 1806, "SQLITE_CANTOPEN_FULLPATH: Full filepath cannot be accessed" }, - { 4102, "SQLITE_CANTOPEN_CONVPATH: Cannot open database file due to conversion path error" }, - // Other - { 26, "SQLITE_NOTADB: File opened is not a database file" }, - { 100, "SQLITE_ROW: sqlite3_step() has another row ready" }, - { 101, "SQLITE_DONE: sqlite3_step() has finished executing" }, + { 516, "SQLITE_ABORT_ROLLBACK: A rollback occurred due to a constraint violation." }, + { 279, "SQLITE_AUTH_USER: Authorization denied by a user authentication callback." }, + { 261, "SQLITE_BUSY_RECOVERY: The database file is locked because another connection is recovering the WAL." }, + { 517, "SQLITE_BUSY_SNAPSHOT: The database file is locked because another connection has a conflicting snapshot." }, + { 773, "SQLITE_BUSY_TIMEOUT: A blocking operation was interrupted by a call to sqlite3_interrupt()." }, + { 1038, "SQLITE_CANTOPEN_CONVPATH: Unable to open the database file due to a conversion error." }, + { + 1294, + "SQLITE_CANTOPEN_DIRTYWAL: The database file cannot be opened because the Write-Ahead Log contains uncommitted changes." + }, + { 782, "SQLITE_CANTOPEN_FULLPATH: Unable to open the database file with the full pathname." }, + { 526, "SQLITE_CANTOPEN_ISDIR: The database file cannot be opened because it is a directory." }, + { + 270, + "SQLITE_CANTOPEN_NOTEMPDIR: Unable to open a temporary database file because a temporary directory is not available." + }, + { 1550, "SQLITE_CANTOPEN_SYMLINK: The database file cannot be opened because it is a symbolic link." }, + { 275, "SQLITE_CONSTRAINT_CHECK: A CHECK constraint failed." }, + { 531, "SQLITE_CONSTRAINT_COMMITHOOK: A commit hook caused the transaction to roll back." }, + { 3091, "SQLITE_CONSTRAINT_DATATYPE: A datatype mismatch occurred." }, + { 787, "SQLITE_CONSTRAINT_FOREIGNKEY: A foreign key constraint failed." }, + { 1043, "SQLITE_CONSTRAINT_FUNCTION: A function constraint failed." }, + { 1299, "SQLITE_CONSTRAINT_NOTNULL: A NOT NULL constraint failed." }, + { 1555, "SQLITE_CONSTRAINT_PRIMARYKEY: A PRIMARY KEY constraint failed." }, + { 1803, "SQLITE_CONSTRAINT_TRIGGER: A trigger constraint failed." }, + { 2059, "SQLITE_CONSTRAINT_UNIQUE: A UNIQUE constraint failed." }, + { 2315, "SQLITE_CONSTRAINT_VTAB: A virtual table constraint failed." }, + { 2571, "SQLITE_CONSTRAINT_ROWID: A rowid constraint failed." }, + { 1034, "SQLITE_IOERR_FSYNC: An I/O error occurred during the fsync() system call." }, + { 6410, "SQLITE_IOERR_GETTEMPPATH: An I/O error occurred while trying to get the temporary file path." }, + { 3850, "SQLITE_IOERR_LOCK: An I/O error occurred while trying to lock the database file." }, + { 6154, "SQLITE_IOERR_MMAP: An I/O error occurred during memory mapping." }, + { 3082, "SQLITE_IOERR_NOMEM: An I/O error occurred due to a memory allocation failure." }, + { 2314, "SQLITE_IOERR_RDLOCK: An I/O error occurred while trying to read-lock the database file." }, + { 266, "SQLITE_IOERR_READ: An I/O error occurred while reading from the database file." }, + { 7946, "SQLITE_IOERR_ROLLBACK_ATOMIC: An I/O error occurred during an atomic rollback." }, + { 5642, "SQLITE_IOERR_SEEK: An I/O error occurred while seeking in the database file." }, + { 5130, "SQLITE_IOERR_SHMLOCK: An I/O error occurred while locking a shared memory segment." }, + { 5386, "SQLITE_IOERR_SHMMAP: An I/O error occurred while mapping a shared memory segment." }, + { 4618, "SQLITE_IOERR_SHMOPEN: An I/O error occurred while opening a shared memory segment." }, + { 4874, "SQLITE_IOERR_SHMSIZE: An I/O error occurred while setting the size of a shared memory segment." }, + { 522, "SQLITE_IOERR_SHORT_READ: An I/O error occurred due to a short read." }, + { 1546, "SQLITE_IOERR_TRUNCATE: An I/O error occurred while truncating the database file." }, + { 2058, "SQLITE_IOERR_UNLOCK: An I/O error occurred while unlocking the database file." }, + { 6922, "SQLITE_IOERR_VNODE: A virtual node I/O error occurred." }, + { 778, "SQLITE_IOERR_WRITE: An I/O error occurred while writing to the database file." }, + { + 262, + "SQLITE_LOCKED_SHAREDCACHE: A write operation could not continue due to a conflict within the shared cache." + }, + { 518, "SQLITE_LOCKED_VTAB: A virtual table is locked." }, + { 539, "SQLITE_NOTICE_RECOVER_ROLLBACK: A rollback was performed to recover from a previous error." }, + { 283, "SQLITE_NOTICE_RECOVER_WAL: Recovery was performed from the Write-Ahead Log." }, + { 256, "SQLITE_OK_LOAD_PERMANENTLY: Operation completed successfully; the extension was loaded permanently." }, + { + 1288, + "SQLITE_READONLY_CANTINIT: Attempt to write to a read-only database failed because initialization is not allowed." + }, + { 520, "SQLITE_READONLY_CANTLOCK: A read-only database cannot be locked." }, + { 1032, "SQLITE_READONLY_DBMOVED: The database file has been moved, making it read-only." }, + { 1544, "SQLITE_READONLY_DIRECTORY: The database is read-only because it is a directory." }, + { 264, "SQLITE_READONLY_RECOVERY: The database is read-only due to recovery mode." }, + { 776, "SQLITE_READONLY_ROLLBACK: The database is read-only because a rollback is required." }, + { 284, "SQLITE_WARNING_AUTOINDEX: Automatic indexing is in use." }, }; } From b5060c40dbd71a02e06d262bebfb11ea1ff12924 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Thu, 9 Jan 2025 08:38:32 +0000 Subject: [PATCH 31/31] Add throwing tests --- .../SQLite/SQLiteJsonExceptionTests.cs | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 tests/Speckle.Sdk.Tests.Unit/SQLite/SQLiteJsonExceptionTests.cs diff --git a/tests/Speckle.Sdk.Tests.Unit/SQLite/SQLiteJsonExceptionTests.cs b/tests/Speckle.Sdk.Tests.Unit/SQLite/SQLiteJsonExceptionTests.cs new file mode 100644 index 00000000..a5d80fd6 --- /dev/null +++ b/tests/Speckle.Sdk.Tests.Unit/SQLite/SQLiteJsonExceptionTests.cs @@ -0,0 +1,27 @@ +using Microsoft.Data.Sqlite; +using NUnit.Framework; +using Speckle.Sdk.SQLite; + +namespace Speckle.Sdk.Tests.Unit.SQLite; + +[TestFixture] +public class SqLiteJsonCacheExceptionTests +{ + [Test] + public void ExpectedExceptionFires_Void() + { + using var pool = new CacheDbCommandPool("DataSource=:memory:", 1); + Assert.Throws( + () => pool.Use(CacheOperation.Get, new Action(_ => throw new SqliteException("test", 1, 1))) + ); + } + + [Test] + public void ExpectedExceptionFires_Return() + { + using var pool = new CacheDbCommandPool("DataSource=:memory:", 1); + Assert.Throws( + () => pool.Use(CacheOperation.Get, new Func(_ => throw new SqliteException("test", 1, 1))) + ); + } +}