diff --git a/src/Speckle.Sdk.Dependencies/Pools.cs b/src/Speckle.Sdk.Dependencies/Pools.cs index f0d8a4fb..7977e681 100644 --- a/src/Speckle.Sdk.Dependencies/Pools.cs +++ b/src/Speckle.Sdk.Dependencies/Pools.cs @@ -1,4 +1,5 @@ -using Microsoft.Extensions.ObjectPool; +using System.Text; +using Microsoft.Extensions.ObjectPool; namespace Speckle.Sdk.Dependencies; @@ -17,16 +18,6 @@ public bool Return(Dictionary obj) } } - public static Pool> ListString { get; } = new(new ListStringPolicy()); - - private sealed class ListStringPolicy : IPooledObjectPolicy> - { - public List Create() => new(20); - - public bool Return(List obj) - { - obj.Clear(); - return true; - } - } + public static Pool StringBuilders { get; } = + new(new StringBuilderPooledObjectPolicy() { MaximumRetainedCapacity = 100 * 1024 * 1024 }); } diff --git a/src/Speckle.Sdk.Dependencies/Serialization/ChannelExtensions.cs b/src/Speckle.Sdk.Dependencies/Serialization/ChannelExtensions.cs new file mode 100644 index 00000000..2154b26c --- /dev/null +++ b/src/Speckle.Sdk.Dependencies/Serialization/ChannelExtensions.cs @@ -0,0 +1,21 @@ +using System.Threading.Channels; +using Open.ChannelExtensions; +using Speckle.Sdk.Dependencies.Serialization; + +namespace Speckle.Sdk.Serialisation.V2.Send; + +public static class ChannelExtensions +{ + public static BatchingChannelReader> BatchBySize( + this ChannelReader source, + int batchSize, + bool singleReader = false, + bool allowSynchronousContinuations = false + ) => + new SizeBatchingChannelReader( + source ?? throw new ArgumentNullException(nameof(source)), + batchSize, + singleReader, + allowSynchronousContinuations + ); +} diff --git a/src/Speckle.Sdk.Dependencies/Serialization/ChannelLoader.cs b/src/Speckle.Sdk.Dependencies/Serialization/ChannelLoader.cs index 7d2ab39b..447440fd 100644 --- a/src/Speckle.Sdk.Dependencies/Serialization/ChannelLoader.cs +++ b/src/Speckle.Sdk.Dependencies/Serialization/ChannelLoader.cs @@ -7,23 +7,27 @@ public abstract class ChannelLoader private const int HTTP_GET_CHUNK_SIZE = 500; private const int MAX_PARALLELISM_HTTP = 4; private static readonly TimeSpan HTTP_BATCH_TIMEOUT = TimeSpan.FromSeconds(2); - private static readonly int MAX_CACHE_PARALLELISM = Environment.ProcessorCount; + private static readonly int MAX_READ_CACHE_PARALLELISM = Environment.ProcessorCount; + private const int MAX_SAVE_CACHE_BATCH = 200; + private const int MAX_SAVE_CACHE_PARALLELISM = 1; protected async Task GetAndCache(IEnumerable allChildrenIds, CancellationToken cancellationToken = default) => await allChildrenIds .ToChannel(cancellationToken: cancellationToken) - .Pipe(MAX_CACHE_PARALLELISM, CheckCache, cancellationToken: cancellationToken) + .Pipe(MAX_READ_CACHE_PARALLELISM, CheckCache, cancellationToken: cancellationToken) .Filter(x => x is not null) .Batch(HTTP_GET_CHUNK_SIZE) .WithTimeout(HTTP_BATCH_TIMEOUT) .PipeAsync(MAX_PARALLELISM_HTTP, async x => await Download(x).ConfigureAwait(false), -1, false, cancellationToken) .Join() - .ReadAllConcurrently(MAX_CACHE_PARALLELISM, SaveToCache, cancellationToken) + .Batch(MAX_SAVE_CACHE_BATCH) + .WithTimeout(HTTP_BATCH_TIMEOUT) + .ReadAllConcurrently(MAX_SAVE_CACHE_PARALLELISM, SaveToCache, cancellationToken) .ConfigureAwait(false); public abstract string? CheckCache(string id); public abstract Task> Download(List ids); - public abstract void SaveToCache(BaseItem x); + public abstract void SaveToCache(List x); } diff --git a/src/Speckle.Sdk.Dependencies/Serialization/ChannelSaver.cs b/src/Speckle.Sdk.Dependencies/Serialization/ChannelSaver.cs index e792ea38..fa161355 100644 --- a/src/Speckle.Sdk.Dependencies/Serialization/ChannelSaver.cs +++ b/src/Speckle.Sdk.Dependencies/Serialization/ChannelSaver.cs @@ -1,29 +1,33 @@ -using System.Threading.Channels; +using System.Text; +using System.Threading.Channels; using Open.ChannelExtensions; +using Speckle.Sdk.Serialisation.V2.Send; namespace Speckle.Sdk.Dependencies.Serialization; -public readonly record struct BaseItem(string Id, string Json, bool NeedsStorage); +public readonly record struct BaseItem(string Id, string Json, bool NeedsStorage) +{ + public int Size { get; } = Encoding.UTF8.GetByteCount(Json); +} public abstract class ChannelSaver { - private const int HTTP_SEND_CHUNK_SIZE = 500; + private const int HTTP_SEND_CHUNK_SIZE = 25_000_000; //bytes private static readonly TimeSpan HTTP_BATCH_TIMEOUT = TimeSpan.FromSeconds(2); private const int MAX_PARALLELISM_HTTP = 4; private const int MAX_CACHE_WRITE_PARALLELISM = 1; - private const int MAX_CACHE_BATCH = 100; - private const string DUMMY = "dummy"; + private const int MAX_CACHE_BATCH = 200; private readonly Channel _checkCacheChannel = Channel.CreateUnbounded(); - public Task Start(string streamId, CancellationToken cancellationToken = default) + public Task Start(CancellationToken cancellationToken = default) { var t = _checkCacheChannel - .Reader.Batch(HTTP_SEND_CHUNK_SIZE) + .Reader.BatchBySize(HTTP_SEND_CHUNK_SIZE) .WithTimeout(HTTP_BATCH_TIMEOUT) .PipeAsync( MAX_PARALLELISM_HTTP, - async x => await SendToServerInternal(streamId, x, cancellationToken).ConfigureAwait(false), + async x => await SendToServer(x, cancellationToken).ConfigureAwait(false), -1, false, cancellationToken @@ -31,52 +35,19 @@ public Task Start(string streamId, CancellationToken cancellationToken = default .Join() .Batch(MAX_CACHE_BATCH) .WithTimeout(HTTP_BATCH_TIMEOUT) - .ReadAllConcurrently(MAX_CACHE_WRITE_PARALLELISM, SaveToCacheInternal, cancellationToken); + .ReadAllConcurrently(MAX_CACHE_WRITE_PARALLELISM, SaveToCache, cancellationToken); return t; } public async Task Save(BaseItem item, CancellationToken cancellationToken = default) => await _checkCacheChannel.Writer.WriteAsync(item, cancellationToken).ConfigureAwait(false); - private async Task> SendToServerInternal( - string streamId, - List batch, - CancellationToken cancellationToken = default - ) - { - var ending = batch.Select(x => x.Id).Contains(DUMMY); - if (ending) - { - batch.RemoveAll(x => x.Id == DUMMY); - } - var results = await SendToServer(streamId, batch, cancellationToken).ConfigureAwait(false); - if (ending) - { - results.Add(new BaseItem(DUMMY, DUMMY, false)); - } - return results; - } - - public abstract Task> SendToServer( - string streamId, - List batch, - CancellationToken cancellationToken - ); - - public async Task Done() => await Save(new BaseItem(DUMMY, DUMMY, false)).ConfigureAwait(false); + public abstract Task> SendToServer(List batch, CancellationToken cancellationToken); - private void SaveToCacheInternal(List batch) + public Task Done() { - var ending = batch.Select(x => x.Id).Contains(DUMMY); - if (ending) - { - batch.RemoveAll(x => x.Id == DUMMY); - } - SaveToCache(batch); - if (ending) - { - _checkCacheChannel.Writer.Complete(); - } + _checkCacheChannel.Writer.Complete(); + return Task.CompletedTask; } public abstract void SaveToCache(List item); diff --git a/src/Speckle.Sdk.Dependencies/Serialization/SizeBatchingChannelReader.cs b/src/Speckle.Sdk.Dependencies/Serialization/SizeBatchingChannelReader.cs new file mode 100644 index 00000000..4bacb7a8 --- /dev/null +++ b/src/Speckle.Sdk.Dependencies/Serialization/SizeBatchingChannelReader.cs @@ -0,0 +1,36 @@ +using System.Threading.Channels; +using Open.ChannelExtensions; +using Speckle.Sdk.Dependencies.Serialization; + +namespace Speckle.Sdk.Serialisation.V2.Send; + +public class SizeBatchingChannelReader( + ChannelReader source, + int batchSize, + bool singleReader, + bool syncCont = false +) : BatchingChannelReader>(source, batchSize, singleReader, syncCont) +{ + private readonly int _batchSize = batchSize; + + protected override List CreateBatch(int capacity) => new(); + + protected override void TrimBatch(List batch) => batch.TrimExcess(); + + protected override void AddBatchItem(List batch, BaseItem item) => batch.Add(item); + + protected override int GetBatchSize(List batch) + { + int size = 0; + foreach (BaseItem item in batch) + { + size += item.Size; + } + + if (size >= _batchSize) + { + return _batchSize; + } + return size; + } +} diff --git a/src/Speckle.Sdk/Api/GraphQL/Inputs/VersionInputs.cs b/src/Speckle.Sdk/Api/GraphQL/Inputs/VersionInputs.cs index a9690796..af694dac 100644 --- a/src/Speckle.Sdk/Api/GraphQL/Inputs/VersionInputs.cs +++ b/src/Speckle.Sdk/Api/GraphQL/Inputs/VersionInputs.cs @@ -1,10 +1,10 @@ namespace Speckle.Sdk.Api.GraphQL.Inputs; -public sealed record UpdateVersionInput(string versionId, string? message); +public sealed record UpdateVersionInput(string versionId, string projectId, string? message); -public sealed record MoveVersionsInput(string targetModelName, IReadOnlyList versionIds); +public sealed record MoveVersionsInput(string projectId, string targetModelName, IReadOnlyList versionIds); -public sealed record DeleteVersionsInput(IReadOnlyList versionIds); +public sealed record DeleteVersionsInput(IReadOnlyList versionIds, string projectId); public sealed record CreateVersionInput( string objectId, diff --git a/src/Speckle.Sdk/Api/Operations/Operations.Receive.cs b/src/Speckle.Sdk/Api/Operations/Operations.Receive.cs index 5c516798..da5e64b7 100644 --- a/src/Speckle.Sdk/Api/Operations/Operations.Receive.cs +++ b/src/Speckle.Sdk/Api/Operations/Operations.Receive.cs @@ -2,8 +2,6 @@ using Speckle.Sdk.Logging; using Speckle.Sdk.Models; using Speckle.Sdk.Serialisation; -using Speckle.Sdk.Serialisation.V2; -using Speckle.Sdk.Serialisation.V2.Receive; using Speckle.Sdk.Transports; namespace Speckle.Sdk.Api; @@ -26,10 +24,12 @@ public async Task Receive2( try { - var sqliteTransport = new SQLiteReceiveCacheManager(streamId); - var serverObjects = new ServerObjectManager(speckleHttp, activityFactory, url, authorizationToken); - var o = new ObjectLoader(sqliteTransport, serverObjects, streamId, onProgressAction); - var process = new DeserializeProcess(onProgressAction, o); + var process = serializeProcessFactory.CreateDeserializeProcess( + url, + streamId, + authorizationToken, + onProgressAction + ); var result = await process.Deserialize(objectId, cancellationToken).ConfigureAwait(false); receiveActivity?.SetStatus(SdkActivityStatusCode.Ok); return result; diff --git a/src/Speckle.Sdk/Api/Operations/Operations.Send.cs b/src/Speckle.Sdk/Api/Operations/Operations.Send.cs index 82bdee14..93b09ed9 100644 --- a/src/Speckle.Sdk/Api/Operations/Operations.Send.cs +++ b/src/Speckle.Sdk/Api/Operations/Operations.Send.cs @@ -4,8 +4,6 @@ using Speckle.Sdk.Logging; using Speckle.Sdk.Models; using Speckle.Sdk.Serialisation; -using Speckle.Sdk.Serialisation.V2; -using Speckle.Sdk.Serialisation.V2.Send; using Speckle.Sdk.Transports; namespace Speckle.Sdk.Api; @@ -26,18 +24,8 @@ public partial class Operations try { - var sqliteTransport = new SQLiteSendCacheManager(streamId); - var serverObjects = new ServerObjectManager(speckleHttp, activityFactory, url, authorizationToken); - var process = new SerializeProcess( - onProgressAction, - sqliteTransport, - serverObjects, - speckleBaseChildFinder, - speckleBasePropertyGatherer - ); - var (rootObjId, convertedReferences) = await process - .Serialize(streamId, value, cancellationToken) - .ConfigureAwait(false); + var process = serializeProcessFactory.CreateSerializeProcess(url, streamId, authorizationToken, onProgressAction); + var (rootObjId, convertedReferences) = await process.Serialize(value, cancellationToken).ConfigureAwait(false); receiveActivity?.SetStatus(SdkActivityStatusCode.Ok); return new(rootObjId, convertedReferences); diff --git a/src/Speckle.Sdk/Api/Operations/Operations.cs b/src/Speckle.Sdk/Api/Operations/Operations.cs index ed41111e..bb0664f8 100644 --- a/src/Speckle.Sdk/Api/Operations/Operations.cs +++ b/src/Speckle.Sdk/Api/Operations/Operations.cs @@ -1,8 +1,7 @@ using Microsoft.Extensions.Logging; using Speckle.InterfaceGenerator; -using Speckle.Sdk.Helpers; using Speckle.Sdk.Logging; -using Speckle.Sdk.Serialisation.V2.Send; +using Speckle.Sdk.Serialisation.V2; namespace Speckle.Sdk.Api; @@ -14,9 +13,7 @@ namespace Speckle.Sdk.Api; [GenerateAutoInterface] public partial class Operations( ILogger logger, - ISpeckleHttp speckleHttp, ISdkActivityFactory activityFactory, ISdkMetricsFactory metricsFactory, - ISpeckleBaseChildFinder speckleBaseChildFinder, - ISpeckleBasePropertyGatherer speckleBasePropertyGatherer + ISerializeProcessFactory serializeProcessFactory ) : IOperations; diff --git a/src/Speckle.Sdk/Helpers/SerializerIdWriter.cs b/src/Speckle.Sdk/Helpers/SerializerIdWriter.cs index 5a8df3ac..b9be351d 100644 --- a/src/Speckle.Sdk/Helpers/SerializerIdWriter.cs +++ b/src/Speckle.Sdk/Helpers/SerializerIdWriter.cs @@ -1,4 +1,6 @@ -using Speckle.Newtonsoft.Json; +using System.Text; +using Speckle.Newtonsoft.Json; +using Speckle.Sdk.Dependencies; using Speckle.Sdk.Serialisation; namespace Speckle.Sdk.Helpers; @@ -9,12 +11,14 @@ public sealed class SerializerIdWriter : JsonWriter #pragma warning disable CA2213 private readonly JsonWriter _jsonIdWriter; private readonly StringWriter _idWriter; + private readonly StringBuilder _stringBuilder; #pragma warning restore CA2213 public SerializerIdWriter(JsonWriter jsonWriter) { _jsonWriter = jsonWriter; - _idWriter = new StringWriter(); + _stringBuilder = Pools.StringBuilders.Get(); + _idWriter = new StringWriter(_stringBuilder); _jsonIdWriter = SpeckleObjectSerializerPool.Instance.GetJsonTextWriter(_idWriter); } @@ -23,6 +27,7 @@ public SerializerIdWriter(JsonWriter jsonWriter) _jsonIdWriter.WriteEndObject(); _jsonIdWriter.Flush(); var json = _idWriter.ToString(); + Pools.StringBuilders.Return(_stringBuilder); return (json, _jsonWriter); } diff --git a/src/Speckle.Sdk/Serialisation/SpeckleObjectSerializer.cs b/src/Speckle.Sdk/Serialisation/SpeckleObjectSerializer.cs index dee490f5..b08e2a33 100644 --- a/src/Speckle.Sdk/Serialisation/SpeckleObjectSerializer.cs +++ b/src/Speckle.Sdk/Serialisation/SpeckleObjectSerializer.cs @@ -6,6 +6,7 @@ using Speckle.DoubleNumerics; using Speckle.Newtonsoft.Json; using Speckle.Sdk.Common; +using Speckle.Sdk.Dependencies; using Speckle.Sdk.Helpers; using Speckle.Sdk.Models; using Speckle.Sdk.Serialisation.Utilities; @@ -249,10 +250,12 @@ private void SerializeProperty( _parentClosures.Add(closure); } + var stringBuilder = Pools.StringBuilders.Get(); using var writer = new StringWriter(); using var jsonWriter = SpeckleObjectSerializerPool.Instance.GetJsonTextWriter(writer); string id = SerializeBaseObject(baseObj, jsonWriter, closure); var json = writer.ToString(); + Pools.StringBuilders.Return(stringBuilder); if (computeClosures || inheritedDetachInfo.IsDetachable || baseObj is Blob) { diff --git a/src/Speckle.Sdk/Serialisation/Utilities/ReferenceGenerator.cs b/src/Speckle.Sdk/Serialisation/Utilities/ReferenceGenerator.cs index 609a1955..bae3396a 100644 --- a/src/Speckle.Sdk/Serialisation/Utilities/ReferenceGenerator.cs +++ b/src/Speckle.Sdk/Serialisation/Utilities/ReferenceGenerator.cs @@ -3,7 +3,7 @@ public static class ReferenceGenerator { private const string REFERENCE_JSON_START = "{\"speckle_type\":\"reference\",\"referencedId\":\""; - private const string REFERENCE_JSON_END = "\",\"__closure\":null}"; + private const string REFERENCE_JSON_END = "\",\"__closure\":null}"; //TODO: remove null but ID calculation changes public static string CreateReference(string id) => REFERENCE_JSON_START + id + REFERENCE_JSON_END; } diff --git a/src/Speckle.Sdk/Serialisation/V2/Receive/DeserializeProcess.cs b/src/Speckle.Sdk/Serialisation/V2/Receive/DeserializeProcess.cs index c072f64f..1db2e0a4 100644 --- a/src/Speckle.Sdk/Serialisation/V2/Receive/DeserializeProcess.cs +++ b/src/Speckle.Sdk/Serialisation/V2/Receive/DeserializeProcess.cs @@ -1,21 +1,33 @@ using System.Collections.Concurrent; +using Speckle.InterfaceGenerator; using Speckle.Sdk.Models; using Speckle.Sdk.Serialisation.Utilities; using Speckle.Sdk.Transports; namespace Speckle.Sdk.Serialisation.V2.Receive; -public record DeserializeOptions(bool SkipCache); +public record DeserializeOptions( + bool SkipCache, + bool ThrowOnMissingReferences = true, + bool SkipInvalidConverts = false +); -public sealed class DeserializeProcess(IProgress? progress, IObjectLoader objectLoader) +[GenerateAutoInterface] +public sealed class DeserializeProcess( + IProgress? progress, + IObjectLoader objectLoader, + IObjectDeserializerFactory objectDeserializerFactory +) : IDeserializeProcess { - private readonly ConcurrentDictionary)> _closures = new(); private long _total; private DeserializeOptions _options = new(false); - public ConcurrentDictionary BaseCache { get; } = new(); + private readonly ConcurrentDictionary)> _closures = new(); + private readonly ConcurrentDictionary _baseCache = new(); private readonly ConcurrentDictionary _activeTasks = new(); + public IReadOnlyDictionary BaseCache => _baseCache; + public async Task Deserialize( string rootId, CancellationToken cancellationToken, @@ -28,14 +40,14 @@ public async Task Deserialize( .ConfigureAwait(false); _total = childrenIds.Count; _closures.TryAdd(rootId, (rootJson, childrenIds)); - progress?.Report(new(ProgressEvent.DeserializeObject, BaseCache.Count, childrenIds.Count)); + progress?.Report(new(ProgressEvent.DeserializeObject, _baseCache.Count, childrenIds.Count)); await Traverse(rootId, cancellationToken).ConfigureAwait(false); - return BaseCache[rootId]; + return _baseCache[rootId]; } private async Task Traverse(string id, CancellationToken cancellationToken) { - if (BaseCache.ContainsKey(id)) + if (_baseCache.ContainsKey(id)) { return; } @@ -43,7 +55,7 @@ private async Task Traverse(string id, CancellationToken cancellationToken) var tasks = new List(); foreach (var childId in childIds) { - if (BaseCache.ContainsKey(childId)) + if (_baseCache.ContainsKey(childId)) { continue; } @@ -75,10 +87,10 @@ private async Task Traverse(string id, CancellationToken cancellationToken) } //don't redo things if the id is decoded already in the cache - if (!BaseCache.ContainsKey(id)) + if (!_baseCache.ContainsKey(id)) { DecodeOrEnqueueChildren(id); - progress?.Report(new(ProgressEvent.DeserializeObject, BaseCache.Count, _total)); + progress?.Report(new(ProgressEvent.DeserializeObject, _baseCache.Count, _total)); } } @@ -101,13 +113,13 @@ private async Task Traverse(string id, CancellationToken cancellationToken) public void DecodeOrEnqueueChildren(string id) { - if (BaseCache.ContainsKey(id)) + if (_baseCache.ContainsKey(id)) { return; } (string json, _) = GetClosures(id); var @base = Deserialise(id, json); - BaseCache.TryAdd(id, @base); + _baseCache.TryAdd(id, @base); //remove from JSON cache because we've finally made the Base _closures.TryRemove(id, out _); _activeTasks.TryRemove(id, out _); @@ -115,11 +127,12 @@ public void DecodeOrEnqueueChildren(string id) private Base Deserialise(string id, string json) { - if (BaseCache.TryGetValue(id, out var baseObject)) + if (_baseCache.TryGetValue(id, out var baseObject)) { return baseObject; } - SpeckleObjectDeserializer2 deserializer = new(BaseCache, SpeckleObjectSerializerPool.Instance); + + var deserializer = objectDeserializerFactory.Create(_baseCache); return deserializer.Deserialize(json); } } diff --git a/src/Speckle.Sdk/Serialisation/V2/Receive/SpeckleObjectDeserializer2.cs b/src/Speckle.Sdk/Serialisation/V2/Receive/ObjectDeserializer.cs similarity index 96% rename from src/Speckle.Sdk/Serialisation/V2/Receive/SpeckleObjectDeserializer2.cs rename to src/Speckle.Sdk/Serialisation/V2/Receive/ObjectDeserializer.cs index 4404c6db..2b41c816 100644 --- a/src/Speckle.Sdk/Serialisation/V2/Receive/SpeckleObjectDeserializer2.cs +++ b/src/Speckle.Sdk/Serialisation/V2/Receive/ObjectDeserializer.cs @@ -1,4 +1,5 @@ using System.Numerics; +using Speckle.InterfaceGenerator; using Speckle.Newtonsoft.Json; using Speckle.Sdk.Common; using Speckle.Sdk.Dependencies; @@ -6,13 +7,12 @@ namespace Speckle.Sdk.Serialisation.V2.Receive; -public record DeserializedOptions(bool ThrowOnMissingReferences = true, bool SkipInvalidConverts = false); - -public sealed class SpeckleObjectDeserializer2( +[GenerateAutoInterface] +public sealed class ObjectDeserializer( IReadOnlyDictionary references, SpeckleObjectSerializerPool pool, - DeserializedOptions? options = null -) + DeserializeOptions? options = null +) : IObjectDeserializer { /// The JSON string of the object to be deserialized /// A typed object deserialized from the diff --git a/src/Speckle.Sdk/Serialisation/V2/Receive/ObjectDeserializerFactory.cs b/src/Speckle.Sdk/Serialisation/V2/Receive/ObjectDeserializerFactory.cs new file mode 100644 index 00000000..1b13836d --- /dev/null +++ b/src/Speckle.Sdk/Serialisation/V2/Receive/ObjectDeserializerFactory.cs @@ -0,0 +1,11 @@ +using Speckle.InterfaceGenerator; +using Speckle.Sdk.Models; + +namespace Speckle.Sdk.Serialisation.V2.Receive; + +[GenerateAutoInterface] +public class ObjectDeserializerFactory : IObjectDeserializerFactory +{ + public IObjectDeserializer Create(IReadOnlyDictionary references, DeserializeOptions? options = null) => + new ObjectDeserializer(references, SpeckleObjectSerializerPool.Instance, options); +} diff --git a/src/Speckle.Sdk/Serialisation/V2/Receive/ObjectLoader.cs b/src/Speckle.Sdk/Serialisation/V2/Receive/ObjectLoader.cs index 8f0d32b5..1fbc4578 100644 --- a/src/Speckle.Sdk/Serialisation/V2/Receive/ObjectLoader.cs +++ b/src/Speckle.Sdk/Serialisation/V2/Receive/ObjectLoader.cs @@ -10,7 +10,6 @@ namespace Speckle.Sdk.Serialisation.V2.Receive; public sealed class ObjectLoader( ISQLiteReceiveCacheManager sqliteReceiveCacheManager, IServerObjectManager serverObjectManager, - string streamId, IProgress? progress ) : ChannelLoader, IObjectLoader { @@ -38,7 +37,7 @@ CancellationToken cancellationToken } } rootJson = await serverObjectManager - .DownloadSingleObject(streamId, rootId, progress, cancellationToken) + .DownloadSingleObject(rootId, progress, cancellationToken) .NotNull() .ConfigureAwait(false); List allChildrenIds = ClosureParser @@ -76,12 +75,7 @@ public override async Task> Download(List ids) { var toCache = new List(); await foreach ( - var (id, json) in serverObjectManager.DownloadObjects( - streamId, - ids.Select(x => x.NotNull()).ToList(), - progress, - default - ) + var (id, json) in serverObjectManager.DownloadObjects(ids.Select(x => x.NotNull()).ToList(), progress, default) ) { toCache.Add(new(id, json, true)); @@ -97,12 +91,12 @@ public override async Task> Download(List ids) } [AutoInterfaceIgnore] - public override void SaveToCache(BaseItem x) + public override void SaveToCache(List batch) { if (!_options.SkipCache) { - sqliteReceiveCacheManager.SaveObject(x); - _cached++; + sqliteReceiveCacheManager.SaveObjects(batch); + Interlocked.Exchange(ref _cached, _cached + batch.Count); progress?.Report(new(ProgressEvent.CachedToLocal, _cached, _allChildrenCount)); } } diff --git a/src/Speckle.Sdk/Serialisation/V2/SQLiteReceiveCacheManager.cs b/src/Speckle.Sdk/Serialisation/V2/SQLiteReceiveCacheManager.cs index 4f380f99..6e629c9a 100644 --- a/src/Speckle.Sdk/Serialisation/V2/SQLiteReceiveCacheManager.cs +++ b/src/Speckle.Sdk/Serialisation/V2/SQLiteReceiveCacheManager.cs @@ -34,6 +34,26 @@ public void SaveObject(BaseItem item) command.ExecuteNonQuery(); } + public void SaveObjects(List items) + { + using var c = new SqliteConnection(ConnectionString); + c.Open(); + using var t = c.BeginTransaction(); + const string COMMAND_TEXT = "INSERT OR IGNORE INTO objects(hash, content) VALUES(@hash, @content)"; + + using var command = new SqliteCommand(COMMAND_TEXT, c); + command.Transaction = t; + var idParam = command.Parameters.Add("@hash", SqliteType.Text); + var jsonParam = command.Parameters.Add("@content", SqliteType.Text); + foreach (var item in items) + { + idParam.Value = item.Id; + jsonParam.Value = item.Json; + command.ExecuteNonQuery(); + } + t.Commit(); + } + public bool HasObject(string objectId) { using var c = new SqliteConnection(ConnectionString); diff --git a/src/Speckle.Sdk/Serialisation/V2/Send/SpeckleBaseChildFinder.cs b/src/Speckle.Sdk/Serialisation/V2/Send/BaseChildFinder.cs similarity index 84% rename from src/Speckle.Sdk/Serialisation/V2/Send/SpeckleBaseChildFinder.cs rename to src/Speckle.Sdk/Serialisation/V2/Send/BaseChildFinder.cs index 77d695fe..e9e1b149 100644 --- a/src/Speckle.Sdk/Serialisation/V2/Send/SpeckleBaseChildFinder.cs +++ b/src/Speckle.Sdk/Serialisation/V2/Send/BaseChildFinder.cs @@ -5,14 +5,14 @@ namespace Speckle.Sdk.Serialisation.V2.Send; [GenerateAutoInterface] -public class SpeckleBaseChildFinder(ISpeckleBasePropertyGatherer propertyGatherer) : ISpeckleBaseChildFinder +public class BaseChildFinder(IBasePropertyGatherer propertyGatherer) : IBaseChildFinder { public IEnumerable GetChildProperties(Base obj) => propertyGatherer.ExtractAllProperties(obj).Where(x => x.PropertyAttributeInfo.IsDetachable); public IEnumerable GetChildren(Base obj) { - var props = GetChildProperties(obj).ToList(); + var props = GetChildProperties(obj); foreach (var kvp in props) { if (kvp.Value is Base child) diff --git a/src/Speckle.Sdk/Serialisation/V2/Send/SpeckleBasePropertyGatherer.cs b/src/Speckle.Sdk/Serialisation/V2/Send/BasePropertyGatherer.cs similarity index 96% rename from src/Speckle.Sdk/Serialisation/V2/Send/SpeckleBasePropertyGatherer.cs rename to src/Speckle.Sdk/Serialisation/V2/Send/BasePropertyGatherer.cs index 1860904b..196c2164 100644 --- a/src/Speckle.Sdk/Serialisation/V2/Send/SpeckleBasePropertyGatherer.cs +++ b/src/Speckle.Sdk/Serialisation/V2/Send/BasePropertyGatherer.cs @@ -1,4 +1,4 @@ -using System.Collections.Concurrent; +using System.Collections.Concurrent; using System.Reflection; using Speckle.InterfaceGenerator; using Speckle.Newtonsoft.Json; @@ -11,7 +11,7 @@ namespace Speckle.Sdk.Serialisation.V2.Send; public readonly record struct Property(string Name, object? Value, PropertyAttributeInfo PropertyAttributeInfo); [GenerateAutoInterface] -public class SpeckleBasePropertyGatherer : ISpeckleBasePropertyGatherer +public class BasePropertyGatherer : IBasePropertyGatherer { private readonly ConcurrentDictionary> _typedPropertiesCache = new(); diff --git a/src/Speckle.Sdk/Serialisation/V2/Send/SpeckleObjectSerializer.cs b/src/Speckle.Sdk/Serialisation/V2/Send/ObjectSerializer.cs similarity index 78% rename from src/Speckle.Sdk/Serialisation/V2/Send/SpeckleObjectSerializer.cs rename to src/Speckle.Sdk/Serialisation/V2/Send/ObjectSerializer.cs index 7cab704a..a98b7c44 100644 --- a/src/Speckle.Sdk/Serialisation/V2/Send/SpeckleObjectSerializer.cs +++ b/src/Speckle.Sdk/Serialisation/V2/Send/ObjectSerializer.cs @@ -1,22 +1,28 @@ using System.Collections; +using System.Collections.Concurrent; using System.Drawing; using System.Globalization; using Speckle.DoubleNumerics; +using Speckle.InterfaceGenerator; using Speckle.Newtonsoft.Json; using Speckle.Sdk.Common; +using Speckle.Sdk.Dependencies; +using Speckle.Sdk.Dependencies.Serialization; using Speckle.Sdk.Helpers; using Speckle.Sdk.Models; using Speckle.Sdk.Serialisation.Utilities; namespace Speckle.Sdk.Serialisation.V2.Send; -public class SpeckleObjectSerializer2 +[GenerateAutoInterface] +public class ObjectSerializer : IObjectSerializer { private HashSet _parentObjects = new(); - private readonly List> _childclosures; + private readonly Dictionary _currentClosures = new(); + private readonly ConcurrentDictionary)> _baseCache; private readonly bool _trackDetachedChildren; - private readonly ISpeckleBasePropertyGatherer _propertyGatherer; + private readonly IBasePropertyGatherer _propertyGatherer; private readonly CancellationToken _cancellationToken; /// @@ -32,14 +38,14 @@ public class SpeckleObjectSerializer2 /// /// Whether to store all detachable objects while serializing. They can be retrieved via post serialization. /// - public SpeckleObjectSerializer2( - ISpeckleBasePropertyGatherer propertyGatherer, - List> childclosures, + public ObjectSerializer( + IBasePropertyGatherer propertyGatherer, + ConcurrentDictionary)> baseCache, bool trackDetachedChildren = false, CancellationToken cancellationToken = default ) { - _childclosures = childclosures; + _baseCache = baseCache; _propertyGatherer = propertyGatherer; _cancellationToken = cancellationToken; _trackDetachedChildren = trackDetachedChildren; @@ -55,8 +61,9 @@ public SpeckleObjectSerializer2( { try { - var result = SerializeBase(baseObj, true).NotNull(); - return [(result.Id.NotNull(), result.Json), .. _chunks]; + var item = SerializeBase(baseObj, true).NotNull(); + _baseCache.TryAdd(baseObj, (item.Json, _currentClosures)); + return [new(item.Id, item.Json), .. _chunks]; } catch (Exception ex) when (!ex.IsFatal() && ex is not OperationCanceledException) { @@ -71,12 +78,7 @@ public SpeckleObjectSerializer2( // `Preserialize` means transforming all objects into the final form that will appear in json, with basic .net objects // (primitives, lists and dictionaries with string keys) - private void SerializeProperty( - object? obj, - JsonWriter writer, - bool computeClosures = false, - PropertyAttributeInfo inheritedDetachInfo = default - ) + private void SerializeProperty(object? obj, JsonWriter writer, PropertyAttributeInfo inheritedDetachInfo = default) { _cancellationToken.ThrowIfCancellationRequested(); @@ -105,21 +107,13 @@ private void SerializeProperty( ["referencedId"] = r.referencedId, ["__closure"] = r.closure, }; - if (r.closure is not null) - { - foreach (var kvp in r.closure) - { - UpdateChildClosures(kvp.Key); - } - } - UpdateChildClosures(r.referencedId); SerializeProperty(ret, writer); break; case Base b: - var result = SerializeBase(b, computeClosures, inheritedDetachInfo); + var result = SerializeBase(b, false, inheritedDetachInfo); if (result is not null) { - writer.WriteRawValue(result.Json); + writer.WriteRawValue(result.Value.Json); } else { @@ -198,11 +192,7 @@ private void SerializeProperty( } } - private SerializationResult? SerializeBase( - Base baseObj, - bool computeClosures = false, - PropertyAttributeInfo inheritedDetachInfo = default - ) + private BaseItem? SerializeBase(Base baseObj, bool isRoot, PropertyAttributeInfo inheritedDetachInfo = default) { // handle circular references bool alreadySerialized = !_parentObjects.Add(baseObj); @@ -211,25 +201,25 @@ private void SerializeProperty( return null; } - Dictionary closure = new(); + Dictionary childClosures; string id; string json; - lock (_childclosures) + if (_baseCache.TryGetValue(baseObj, out var info)) { - if (computeClosures || inheritedDetachInfo.IsDetachable || baseObj is Blob) - { - _childclosures.Add(closure); - } - - using var writer = new StringWriter(); + id = baseObj.id; + childClosures = info.Item2; + json = info.Item1; + MergeClosures(_currentClosures, childClosures); + } + else + { + childClosures = isRoot ? _currentClosures : new(); + var sb = Pools.StringBuilders.Get(); + using var writer = new StringWriter(sb); using var jsonWriter = SpeckleObjectSerializerPool.Instance.GetJsonTextWriter(writer); - id = SerializeBaseObject(baseObj, jsonWriter, closure); + id = SerializeBaseObject(baseObj, jsonWriter, childClosures); json = writer.ToString(); - - if (computeClosures || inheritedDetachInfo.IsDetachable || baseObj is Blob) - { - _childclosures.RemoveAt(_childclosures.Count - 1); - } + Pools.StringBuilders.Return(sb); } _parentObjects.Remove(baseObj); @@ -245,8 +235,7 @@ private void SerializeProperty( if (inheritedDetachInfo.IsDetachable) { var json2 = ReferenceGenerator.CreateReference(id); - UpdateChildClosures(id); - + AddClosure(id); // add to obj refs to return if (baseObj.applicationId != null && _trackDetachedChildren) // && baseObj is not DataChunk && baseObj is not Abstract) // not needed, as data chunks will never have application ids, and abstract objs are not really used. { @@ -254,16 +243,16 @@ private void SerializeProperty( { referencedId = id, applicationId = baseObj.applicationId, - closure = closure, + closure = childClosures, }; } - _chunks.Add((id, json)); - return new(json2, null); + _chunks.Add(new(id, json)); + return new(id, json2, true); } - return new(json.NotNull(), id); + return new(id, json, true); } - private string SerializeBaseObject(Base baseObj, JsonWriter writer, IReadOnlyDictionary closure) + private string SerializeBaseObject(Base baseObj, JsonWriter writer, Dictionary closure) { if (baseObj is not Blob) { @@ -280,7 +269,7 @@ private string SerializeBaseObject(Base baseObj, JsonWriter writer, IReadOnlyDic } writer.WritePropertyName(prop.Name); - SerializeProperty(prop.Value, writer, prop.PropertyAttributeInfo); + SerializeOrChunkProperty(prop.Value, writer, prop.PropertyAttributeInfo); } string id; @@ -313,7 +302,7 @@ private string SerializeBaseObject(Base baseObj, JsonWriter writer, IReadOnlyDic return id; } - private void SerializeProperty(object? baseValue, JsonWriter jsonWriter, PropertyAttributeInfo detachInfo) + private void SerializeOrChunkProperty(object? baseValue, JsonWriter jsonWriter, PropertyAttributeInfo detachInfo) { if (baseValue is IEnumerable chunkableCollection && detachInfo.IsChunkable) { @@ -342,20 +331,13 @@ private void SerializeProperty(object? baseValue, JsonWriter jsonWriter, Propert SerializeProperty(baseValue, jsonWriter, inheritedDetachInfo: detachInfo); } - private void UpdateChildClosures(string objectId) + private static void MergeClosures(Dictionary current, Dictionary child) { - lock (_childclosures) + foreach (var closure in child) { - for (int i = 0; i < _childclosures.Count; i++) - { - int childDepth = _childclosures.Count - i; - if (!_childclosures[i].TryGetValue(objectId, out int currentValue)) - { - currentValue = childDepth; - } - - _childclosures[i][objectId] = Math.Min(currentValue, childDepth); - } + current[closure.Key] = 100; } } + + private void AddClosure(string id) => _currentClosures[id] = 100; } diff --git a/src/Speckle.Sdk/Serialisation/V2/Send/ObjectSerializerFactory.cs b/src/Speckle.Sdk/Serialisation/V2/Send/ObjectSerializerFactory.cs new file mode 100644 index 00000000..56a7185a --- /dev/null +++ b/src/Speckle.Sdk/Serialisation/V2/Send/ObjectSerializerFactory.cs @@ -0,0 +1,14 @@ +using System.Collections.Concurrent; +using Speckle.InterfaceGenerator; +using Speckle.Sdk.Models; + +namespace Speckle.Sdk.Serialisation.V2.Send; + +[GenerateAutoInterface] +public class ObjectSerializerFactory(IBasePropertyGatherer propertyGatherer) : IObjectSerializerFactory +{ + public IObjectSerializer Create( + ConcurrentDictionary)> baseCache, + CancellationToken cancellationToken + ) => new ObjectSerializer(propertyGatherer, baseCache, true, cancellationToken); +} diff --git a/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs b/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs index 3bb79240..a95fce7f 100644 --- a/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs +++ b/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs @@ -1,4 +1,5 @@ using System.Collections.Concurrent; +using Speckle.InterfaceGenerator; using Speckle.Sdk.Common; using Speckle.Sdk.Dependencies.Serialization; using Speckle.Sdk.Models; @@ -6,17 +7,19 @@ namespace Speckle.Sdk.Serialisation.V2.Send; -public record SerializeProcessOptions(bool SkipCache, bool SkipServer); +public record SerializeProcessOptions(bool SkipCacheRead, bool SkipCacheWrite, bool SkipServer); +[GenerateAutoInterface] public class SerializeProcess( IProgress? progress, ISQLiteSendCacheManager sqliteSendCacheManager, IServerObjectManager serverObjectManager, - ISpeckleBaseChildFinder speckleBaseChildFinder, - ISpeckleBasePropertyGatherer speckleBasePropertyGatherer -) : ChannelSaver + IBaseChildFinder baseChildFinder, + IObjectSerializerFactory objectSerializerFactory +) : ChannelSaver, ISerializeProcess { private readonly ConcurrentDictionary _jsonCache = new(); + private readonly ConcurrentDictionary)> _baseCache = new(); private readonly ConcurrentDictionary _objectReferences = new(); private long _totalFound; @@ -25,26 +28,25 @@ ISpeckleBasePropertyGatherer speckleBasePropertyGatherer private long _cached; private long _serialized; - private SerializeProcessOptions _options = new(false, false); + private SerializeProcessOptions _options = new(false, false, false); public async Task<(string rootObjId, IReadOnlyDictionary convertedReferences)> Serialize( - string streamId, Base root, CancellationToken cancellationToken, SerializeProcessOptions? options = null ) { _options = options ?? _options; - var channelTask = Start(streamId, cancellationToken); + var channelTask = Start(cancellationToken); await Traverse(root, true, cancellationToken).ConfigureAwait(false); await channelTask.ConfigureAwait(false); return (root.id, _objectReferences); } - private async Task>> Traverse(Base obj, bool isEnd, CancellationToken cancellationToken) + private async Task Traverse(Base obj, bool isEnd, CancellationToken cancellationToken) { - var tasks = new List>>>(); - foreach (var child in speckleBaseChildFinder.GetChildren(obj)) + var tasks = new List(); + foreach (var child in baseChildFinder.GetChildren(obj)) { Interlocked.Increment(ref _totalFound); progress?.Report(new(ProgressEvent.FindingChildren, _totalFound, null)); @@ -65,19 +67,8 @@ private async Task>> Traverse(Base obj, bool isEnd, { await Task.WhenAll(tasks).ConfigureAwait(false); } - var closures = tasks - .Select(t => t.Result) - .Aggregate( - new List>(), - (a, s) => - { - a.AddRange(s); - return a; - } - ) - .ToList(); - var items = Serialise(obj, closures); + var items = Serialise(obj, cancellationToken); foreach (var item in items) { Interlocked.Increment(ref _serialized); @@ -93,18 +84,17 @@ private async Task>> Traverse(Base obj, bool isEnd, { await Done().ConfigureAwait(false); } - return closures; } //leave this sync - private IEnumerable Serialise(Base obj, List> childClosures) + private IEnumerable Serialise(Base obj, CancellationToken cancellationToken) { if (obj.id != null && _jsonCache.ContainsKey(obj.id)) { yield break; } - if (!_options.SkipCache && obj.id != null) + if (!_options.SkipCacheRead && obj.id != null) { var cachedJson = sqliteSendCacheManager.GetObject(obj.id); if (cachedJson != null) @@ -116,7 +106,7 @@ private IEnumerable Serialise(Base obj, List> var id = obj.id; if (id is null || !_jsonCache.TryGetValue(id, out var json)) { - SpeckleObjectSerializer2 serializer2 = new(speckleBasePropertyGatherer, childClosures, true); + var serializer2 = objectSerializerFactory.Create(_baseCache, cancellationToken); var items = serializer2.Serialize(obj).ToList(); foreach (var kvp in serializer2.ObjectReferences) { @@ -149,7 +139,7 @@ private IEnumerable Serialise(Base obj, List> private BaseItem CheckCache(string id, string json) { - if (!_options.SkipCache) + if (!_options.SkipCacheRead) { var cachedJson = sqliteSendCacheManager.GetObject(id); if (cachedJson != null) @@ -160,38 +150,23 @@ private BaseItem CheckCache(string id, string json) return new BaseItem(id, json, true); } - public override async Task> SendToServer( - string streamId, - List batch, - CancellationToken cancellationToken - ) + public override async Task> SendToServer(List batch, CancellationToken cancellationToken) { - if (batch.Count == 0) + if (!_options.SkipServer && batch.Count != 0) { - progress?.Report(new(ProgressEvent.UploadedObjects, _uploaded, _totalToUpload)); - return batch; - } - - if (!_options.SkipServer) - { - await serverObjectManager.UploadObjects(streamId, batch, true, progress, cancellationToken).ConfigureAwait(false); + await serverObjectManager.UploadObjects(batch, true, progress, cancellationToken).ConfigureAwait(false); Interlocked.Exchange(ref _uploaded, _uploaded + batch.Count); progress?.Report(new(ProgressEvent.UploadedObjects, _uploaded, _totalToUpload)); } return batch; } - public override void SaveToCache(List items) + public override void SaveToCache(List batch) { - if (!_options.SkipCache) + if (!_options.SkipCacheWrite && batch.Count != 0) { - if (items.Count == 0) - { - progress?.Report(new(ProgressEvent.CachedToLocal, _cached, null)); - return; - } - sqliteSendCacheManager.SaveObjects(items); - Interlocked.Exchange(ref _cached, _cached + items.Count); + sqliteSendCacheManager.SaveObjects(batch); + Interlocked.Exchange(ref _cached, _cached + batch.Count); progress?.Report(new(ProgressEvent.CachedToLocal, _cached, null)); } } diff --git a/src/Speckle.Sdk/Serialisation/V2/SerializeProcessFactory.cs b/src/Speckle.Sdk/Serialisation/V2/SerializeProcessFactory.cs new file mode 100644 index 00000000..23a8c2bf --- /dev/null +++ b/src/Speckle.Sdk/Serialisation/V2/SerializeProcessFactory.cs @@ -0,0 +1,64 @@ +using Speckle.Sdk.Helpers; +using Speckle.Sdk.Logging; +using Speckle.Sdk.Serialisation.V2.Receive; +using Speckle.Sdk.Serialisation.V2.Send; +using Speckle.Sdk.Transports; + +namespace Speckle.Sdk.Serialisation.V2; + +public interface ISerializeProcessFactory +{ + ISerializeProcess CreateSerializeProcess( + Uri url, + string streamId, + string? authorizationToken, + IProgress? progress + ); + IDeserializeProcess CreateDeserializeProcess( + Uri url, + string streamId, + string? authorizationToken, + IProgress? progress + ); +} + +public class SerializeProcessFactory( + ISpeckleHttp speckleHttp, + ISdkActivityFactory activityFactory, + IBaseChildFinder baseChildFinder, + IObjectSerializerFactory objectSerializerFactory, + IObjectDeserializerFactory objectDeserializerFactory +) : ISerializeProcessFactory +{ + public ISerializeProcess CreateSerializeProcess( + Uri url, + string streamId, + string? authorizationToken, + IProgress? progress + ) + { + var sqliteSendCacheManager = new SQLiteSendCacheManager(streamId); + var serverObjectManager = new ServerObjectManager(speckleHttp, activityFactory, url, streamId, authorizationToken); + return new SerializeProcess( + progress, + sqliteSendCacheManager, + serverObjectManager, + baseChildFinder, + objectSerializerFactory + ); + } + + public IDeserializeProcess CreateDeserializeProcess( + Uri url, + string streamId, + string? authorizationToken, + IProgress? progress + ) + { + var sqliteSendCacheManager = new SQLiteReceiveCacheManager(streamId); + var serverObjectManager = new ServerObjectManager(speckleHttp, activityFactory, url, streamId, authorizationToken); + + var objectLoader = new ObjectLoader(sqliteSendCacheManager, serverObjectManager, progress); + return new DeserializeProcess(progress, objectLoader, objectDeserializerFactory); + } +} diff --git a/src/Speckle.Sdk/Serialisation/V2/ServerObjectManager.cs b/src/Speckle.Sdk/Serialisation/V2/ServerObjectManager.cs index 9f817d6c..f21ee85c 100644 --- a/src/Speckle.Sdk/Serialisation/V2/ServerObjectManager.cs +++ b/src/Speckle.Sdk/Serialisation/V2/ServerObjectManager.cs @@ -16,15 +16,17 @@ namespace Speckle.Sdk.Serialisation.V2; [GenerateAutoInterface] public class ServerObjectManager : IServerObjectManager { - private static readonly char[] s_separator = { '\t' }; + private static readonly char[] s_separator = ['\t']; private readonly ISdkActivityFactory _activityFactory; private readonly HttpClient _client; + private readonly string _streamId; public ServerObjectManager( ISpeckleHttp speckleHttp, ISdkActivityFactory activityFactory, Uri baseUri, + string streamId, string? authorizationToken, int timeoutSeconds = 120 ) @@ -36,10 +38,10 @@ public ServerObjectManager( authorizationToken: authorizationToken ); _client.BaseAddress = baseUri; + _streamId = streamId; } public async IAsyncEnumerable<(string, string)> DownloadObjects( - string streamId, IReadOnlyList objectIds, IProgress? progress, [EnumeratorCancellation] CancellationToken cancellationToken @@ -50,7 +52,7 @@ [EnumeratorCancellation] CancellationToken cancellationToken using var childrenHttpMessage = new HttpRequestMessage { - RequestUri = new Uri($"/api/getobjects/{streamId}", UriKind.Relative), + RequestUri = new Uri($"/api/getobjects/{_streamId}", UriKind.Relative), Method = HttpMethod.Post, }; @@ -73,7 +75,6 @@ [EnumeratorCancellation] CancellationToken cancellationToken } public async Task DownloadSingleObject( - string streamId, string objectId, IProgress? progress, CancellationToken cancellationToken @@ -85,7 +86,7 @@ CancellationToken cancellationToken // Get root object using var rootHttpMessage = new HttpRequestMessage { - RequestUri = new Uri($"/objects/{streamId}/{objectId}/single", UriKind.Relative), + RequestUri = new Uri($"/objects/{_streamId}/{objectId}/single", UriKind.Relative), Method = HttpMethod.Get, }; @@ -138,7 +139,6 @@ [EnumeratorCancellation] CancellationToken cancellationToken } public async Task> HasObjects( - string streamId, IReadOnlyList objectIds, CancellationToken cancellationToken ) @@ -150,7 +150,7 @@ CancellationToken cancellationToken string objectsPostParameter = JsonConvert.SerializeObject(objectIds); var payload = new Dictionary { { "objects", objectsPostParameter } }; string serializedPayload = JsonConvert.SerializeObject(payload); - var uri = new Uri($"/api/diff/{streamId}", UriKind.Relative); + var uri = new Uri($"/api/diff/{_streamId}", UriKind.Relative); using StringContent stringContent = new(serializedPayload, Encoding.UTF8, "application/json"); using HttpResponseMessage response = await _client @@ -167,7 +167,6 @@ CancellationToken cancellationToken } public async Task UploadObjects( - string streamId, IReadOnlyList objects, bool compressPayloads, IProgress? progress, @@ -177,7 +176,7 @@ CancellationToken cancellationToken cancellationToken.ThrowIfCancellationRequested(); using HttpRequestMessage message = - new() { RequestUri = new Uri($"/objects/{streamId}", UriKind.Relative), Method = HttpMethod.Post }; + new() { RequestUri = new Uri($"/objects/{_streamId}", UriKind.Relative), Method = HttpMethod.Post }; MultipartFormDataContent multipart = new(); diff --git a/tests/Speckle.Sdk.Serialization.Testing/DummyServerObjectManager.cs b/tests/Speckle.Sdk.Serialization.Testing/DummyServerObjectManager.cs index 2d1f4ebf..226238f6 100644 --- a/tests/Speckle.Sdk.Serialization.Testing/DummyServerObjectManager.cs +++ b/tests/Speckle.Sdk.Serialization.Testing/DummyServerObjectManager.cs @@ -8,27 +8,23 @@ namespace Speckle.Sdk.Serialization.Testing; public class DummyServerObjectManager : IServerObjectManager { public IAsyncEnumerable<(string, string)> DownloadObjects( - string streamId, IReadOnlyList objectIds, IProgress? progress, CancellationToken cancellationToken ) => throw new NotImplementedException(); public Task DownloadSingleObject( - string streamId, string objectId, IProgress? progress, CancellationToken cancellationToken ) => throw new NotImplementedException(); public Task> HasObjects( - string streamId, IReadOnlyList objectIds, CancellationToken cancellationToken ) => throw new NotImplementedException(); public Task UploadObjects( - string streamId, IReadOnlyList objects, bool compressPayloads, IProgress? progress, diff --git a/tests/Speckle.Sdk.Serialization.Testing/Program.cs b/tests/Speckle.Sdk.Serialization.Testing/Program.cs index 0faa7339..06501ba5 100644 --- a/tests/Speckle.Sdk.Serialization.Testing/Program.cs +++ b/tests/Speckle.Sdk.Serialization.Testing/Program.cs @@ -1,3 +1,4 @@ +#pragma warning disable CA1506 using System.Reflection; using Microsoft.Extensions.DependencyInjection; using Speckle.Sdk; @@ -11,23 +12,25 @@ using Speckle.Sdk.Serialisation.V2.Send; using Speckle.Sdk.Serialization.Testing; -const bool skipCache = false; +const bool skipCacheReceive = false; +const bool skipCacheSendCheck = true; +const bool skipCacheSendSave = false; TypeLoader.Reset(); TypeLoader.Initialize(typeof(Base).Assembly, Assembly.GetExecutingAssembly()); -/* var url = "https://latest.speckle.systems/projects/a3ac1b2706/models/59d3b0f3c6"; //small? var streamId = "a3ac1b2706"; -var rootId = "7d53bcf28c6696ecac8781684a0aa006";*/ +var rootId = "7d53bcf28c6696ecac8781684a0aa006"; + /* var url = "https://latest.speckle.systems/"; //other? var streamId = "368f598929"; var rootId = "67374cfe689c43ff8be12090af122244";*/ - +/* var url = "https://latest.speckle.systems/projects/2099ac4b5f/models/da511c4d1e"; //perf? var streamId = "2099ac4b5f"; -var rootId = "30fb4cbe6eb2202b9e7b4a4fcc3dd2b6"; +var rootId = "30fb4cbe6eb2202b9e7b4a4fcc3dd2b6";*/ var serviceCollection = new ServiceCollection(); serviceCollection.AddSpeckleSdk(HostApplications.Navisworks, HostAppVersion.v2023, "Test"); @@ -37,27 +40,25 @@ var token = serviceProvider.GetRequiredService().GetDefaultAccount()?.token; var progress = new Progress(true); -var sqliteTransport = new SQLiteReceiveCacheManager(streamId); -var serverObjects = new ServerObjectManager( + +var factory = new SerializeProcessFactory( serviceProvider.GetRequiredService(), serviceProvider.GetRequiredService(), - new Uri(url), - token + new BaseChildFinder(new BasePropertyGatherer()), + new ObjectSerializerFactory(new BasePropertyGatherer()), + new ObjectDeserializerFactory() ); -var o = new ObjectLoader(sqliteTransport, serverObjects, streamId, progress); -var process = new DeserializeProcess(progress, o); -var @base = await process.Deserialize(rootId, default, new(skipCache)).ConfigureAwait(false); +var process = factory.CreateDeserializeProcess(new Uri(url), streamId, token, progress); +var @base = await process.Deserialize(rootId, default, new(skipCacheReceive)).ConfigureAwait(false); + Console.WriteLine("Deserialized"); Console.ReadLine(); Console.WriteLine("Executing"); -var process2 = new SerializeProcess( - progress, - new SQLiteSendCacheManager(streamId), - new DummyServerObjectManager(), - new SpeckleBaseChildFinder(new SpeckleBasePropertyGatherer()), - new SpeckleBasePropertyGatherer() -); -await process2.Serialize(streamId, @base, default, new SerializeProcessOptions(skipCache, true)).ConfigureAwait(false); +var process2 = factory.CreateSerializeProcess(new Uri(url), streamId, token, progress); +await process2 + .Serialize(@base, default, new SerializeProcessOptions(skipCacheSendCheck, skipCacheSendSave, true)) + .ConfigureAwait(false); Console.WriteLine("Detach"); Console.ReadLine(); +#pragma warning restore CA1506 diff --git a/tests/Speckle.Sdk.Serialization.Tests/DetachedTests.cs b/tests/Speckle.Sdk.Serialization.Tests/DetachedTests.cs index 0606813c..5e727b5b 100644 --- a/tests/Speckle.Sdk.Serialization.Tests/DetachedTests.cs +++ b/tests/Speckle.Sdk.Serialization.Tests/DetachedTests.cs @@ -2,6 +2,7 @@ using System.Text; using NUnit.Framework; using Shouldly; +using Speckle.Newtonsoft.Json; using Speckle.Newtonsoft.Json.Linq; using Speckle.Objects.Geometry; using Speckle.Sdk.Dependencies.Serialization; @@ -47,7 +48,7 @@ public async Task CanSerialize_New_Detached() "dynamicProp": 123, "id": "9ff8efb13c62fa80f3d1c4519376ba13", "__closure": { - "d3dd4621b2f68c3058c2b9c023a9de19": 1 + "d3dd4621b2f68c3058c2b9c023a9de19": 100 } } """; @@ -70,12 +71,10 @@ public async Task CanSerialize_New_Detached() null, new DummySendCacheManager(objects), new DummyServerObjectManager(), - new SpeckleBaseChildFinder(new SpeckleBasePropertyGatherer()), - new SpeckleBasePropertyGatherer() + new BaseChildFinder(new BasePropertyGatherer()), + new ObjectSerializerFactory(new BasePropertyGatherer()) ); - await process2 - .Serialize(string.Empty, @base, default, new SerializeProcessOptions(false, true)) - .ConfigureAwait(false); + await process2.Serialize(@base, default, new SerializeProcessOptions(false, false, true)).ConfigureAwait(false); objects.Count.ShouldBe(2); objects.ContainsKey("9ff8efb13c62fa80f3d1c4519376ba13").ShouldBeTrue(); @@ -155,7 +154,7 @@ public void GetPropertiesExpected_Detached() @base.detachedProp = new SamplePropBase() { name = "detachedProp" }; @base.attachedProp = new SamplePropBase() { name = "attachedProp" }; - var children = new SpeckleBaseChildFinder(new SpeckleBasePropertyGatherer()).GetChildProperties(@base).ToList(); + var children = new BaseChildFinder(new BasePropertyGatherer()).GetChildProperties(@base).ToList(); children.Count.ShouldBe(4); children.First(x => x.Name == "detachedProp").PropertyAttributeInfo.IsDetachable.ShouldBeTrue(); @@ -174,7 +173,7 @@ public void GetPropertiesExpected_All() @base.detachedProp = new SamplePropBase() { name = "detachedProp" }; @base.attachedProp = new SamplePropBase() { name = "attachedProp" }; - var children = new SpeckleBasePropertyGatherer().ExtractAllProperties(@base).ToList(); + var children = new BasePropertyGatherer().ExtractAllProperties(@base).ToList(); children.Count.ShouldBe(9); children.First(x => x.Name == "dynamicProp").PropertyAttributeInfo.IsDetachable.ShouldBeFalse(); @@ -224,14 +223,14 @@ public async Task CanSerialize_New_Detached2() "dynamicProp": 123, "id": "fd4efeb8a036838c53ad1cf9e82b8992", "__closure": { - "8d27f5c7fac36d985d89bb6d6d8acddc": 3, - "4ba53b5e84e956fb076bc8b0a03ca879": 2, - "32a385e7ddeda810e037b21ab26381b7": 1, - "1afc694774efa5913d0077302cd37888": 3, - "045cbee36837d589b17f9d8483c90763": 2, - "c3858f47dd3e7a308a1b465375f1645f": 1, - "5b86b66b61c556ead500915b05852875": 2, - "027a7c5ffcf8d8efe432899c729a954c": 1 + "8d27f5c7fac36d985d89bb6d6d8acddc": 100, + "4ba53b5e84e956fb076bc8b0a03ca879": 100, + "32a385e7ddeda810e037b21ab26381b7": 100, + "1afc694774efa5913d0077302cd37888": 100, + "045cbee36837d589b17f9d8483c90763": 100, + "c3858f47dd3e7a308a1b465375f1645f": 100, + "5b86b66b61c556ead500915b05852875": 100, + "027a7c5ffcf8d8efe432899c729a954c": 100 } } """; @@ -263,15 +262,18 @@ public async Task CanSerialize_New_Detached2() null, new DummySendCacheManager(objects), new DummyServerObjectManager(), - new SpeckleBaseChildFinder(new SpeckleBasePropertyGatherer()), - new SpeckleBasePropertyGatherer() + new BaseChildFinder(new BasePropertyGatherer()), + new ObjectSerializerFactory(new BasePropertyGatherer()) ); var results = await process2 - .Serialize(string.Empty, @base, default, new SerializeProcessOptions(false, true)) + .Serialize(@base, default, new SerializeProcessOptions(false, false, true)) .ConfigureAwait(false); objects.Count.ShouldBe(9); - JToken.DeepEquals(JObject.Parse(root), JObject.Parse(objects["fd4efeb8a036838c53ad1cf9e82b8992"])).ShouldBeTrue(); + var x = JObject.Parse(objects["fd4efeb8a036838c53ad1cf9e82b8992"]); + var y = x.ToString(Formatting.Indented); + Console.WriteLine(y); + JToken.DeepEquals(JObject.Parse(root), x).ShouldBeTrue(); results.rootObjId.ShouldBe(@base.id); results.convertedReferences.Count.ShouldBe(2); @@ -333,27 +335,23 @@ public class SamplePropBase2 : Base public class DummyServerObjectManager : IServerObjectManager { public IAsyncEnumerable<(string, string)> DownloadObjects( - string streamId, IReadOnlyList objectIds, IProgress? progress, CancellationToken cancellationToken ) => throw new NotImplementedException(); public Task DownloadSingleObject( - string streamId, string objectId, IProgress? progress, CancellationToken cancellationToken ) => throw new NotImplementedException(); public Task> HasObjects( - string streamId, IReadOnlyList objectIds, CancellationToken cancellationToken ) => throw new NotImplementedException(); public Task UploadObjects( - string streamId, IReadOnlyList objects, bool compressPayloads, IProgress? progress, diff --git a/tests/Speckle.Sdk.Serialization.Tests/DummyReceiveServerObjectManager.cs b/tests/Speckle.Sdk.Serialization.Tests/DummyReceiveServerObjectManager.cs index ba43e279..213bc391 100644 --- a/tests/Speckle.Sdk.Serialization.Tests/DummyReceiveServerObjectManager.cs +++ b/tests/Speckle.Sdk.Serialization.Tests/DummyReceiveServerObjectManager.cs @@ -9,7 +9,6 @@ namespace Speckle.Sdk.Serialization.Tests; public class DummyReceiveServerObjectManager(Dictionary objects) : IServerObjectManager { public async IAsyncEnumerable<(string, string)> DownloadObjects( - string streamId, IReadOnlyList objectIds, IProgress? progress, [EnumeratorCancellation] CancellationToken cancellationToken @@ -23,7 +22,6 @@ [EnumeratorCancellation] CancellationToken cancellationToken } public async Task DownloadSingleObject( - string streamId, string objectId, IProgress? progress, CancellationToken cancellationToken @@ -34,13 +32,11 @@ CancellationToken cancellationToken } public Task> HasObjects( - string streamId, IReadOnlyList objectIds, CancellationToken cancellationToken ) => throw new NotImplementedException(); public Task UploadObjects( - string streamId, IReadOnlyList objects, bool compressPayloads, IProgress? progress, diff --git a/tests/Speckle.Sdk.Serialization.Tests/DummySendServerObjectManager.cs b/tests/Speckle.Sdk.Serialization.Tests/DummySendServerObjectManager.cs index b425da57..e2434282 100644 --- a/tests/Speckle.Sdk.Serialization.Tests/DummySendServerObjectManager.cs +++ b/tests/Speckle.Sdk.Serialization.Tests/DummySendServerObjectManager.cs @@ -11,30 +11,23 @@ namespace Speckle.Sdk.Serialization.Tests; public class DummySendServerObjectManager(ConcurrentDictionary savedObjects) : IServerObjectManager { public IAsyncEnumerable<(string, string)> DownloadObjects( - string streamId, IReadOnlyList objectIds, IProgress? progress, CancellationToken cancellationToken ) => throw new NotImplementedException(); public Task DownloadSingleObject( - string streamId, string objectId, IProgress? progress, CancellationToken cancellationToken ) => throw new NotImplementedException(); - public Task> HasObjects( - string streamId, - IReadOnlyList objectIds, - CancellationToken cancellationToken - ) + public Task> HasObjects(IReadOnlyList objectIds, CancellationToken cancellationToken) { return Task.FromResult(objectIds.ToDictionary(x => x, x => false)); } public Task UploadObjects( - string streamId, IReadOnlyList objects, bool compressPayloads, IProgress? progress, diff --git a/tests/Speckle.Sdk.Serialization.Tests/DummySqLiteReceiveManager.cs b/tests/Speckle.Sdk.Serialization.Tests/DummySqLiteReceiveManager.cs index 1b5c1ae2..abf2fe18 100644 --- a/tests/Speckle.Sdk.Serialization.Tests/DummySqLiteReceiveManager.cs +++ b/tests/Speckle.Sdk.Serialization.Tests/DummySqLiteReceiveManager.cs @@ -9,5 +9,7 @@ public class DummySqLiteReceiveManager(Dictionary savedObjects) public void SaveObject(BaseItem item) => throw new NotImplementedException(); + public void SaveObjects(List item) => throw new NotImplementedException(); + public bool HasObject(string objectId) => throw new NotImplementedException(); } diff --git a/tests/Speckle.Sdk.Serialization.Tests/SerializationTests.cs b/tests/Speckle.Sdk.Serialization.Tests/SerializationTests.cs index 392e370c..dee43346 100644 --- a/tests/Speckle.Sdk.Serialization.Tests/SerializationTests.cs +++ b/tests/Speckle.Sdk.Serialization.Tests/SerializationTests.cs @@ -146,7 +146,7 @@ public async Task Basic_Namespace_Validation_New(string fileName) var fullName = _assembly.GetManifestResourceNames().Single(x => x.EndsWith(fileName)); var json = await ReadJson(fullName); var closures = ReadAsObjects(json); - var process = new DeserializeProcess(null, new TestObjectLoader(closures)); + var process = new DeserializeProcess(null, new TestObjectLoader(closures), new ObjectDeserializerFactory()); await process.Deserialize("3416d3fe01c9196115514c4a2f41617b", default); foreach (var (id, objJson) in closures) { @@ -228,10 +228,9 @@ public async Task Roundtrip_Test_New(string fileName, string rootId, int count) var o = new ObjectLoader( new DummySqLiteReceiveManager(closure), new DummyReceiveServerObjectManager(closure), - string.Empty, null ); - var process = new DeserializeProcess(null, o); + var process = new DeserializeProcess(null, o, new ObjectDeserializerFactory()); var root = await process.Deserialize(rootId, default, new DeserializeOptions(true)); var newIdToJson = new ConcurrentDictionary(); @@ -239,15 +238,10 @@ public async Task Roundtrip_Test_New(string fileName, string rootId, int count) null, new DummySqLiteSendManager(), new DummySendServerObjectManager(newIdToJson), - new SpeckleBaseChildFinder(new SpeckleBasePropertyGatherer()), - new SpeckleBasePropertyGatherer() - ); - var (rootId2, _) = await serializeProcess.Serialize( - string.Empty, - root, - default, - new SerializeProcessOptions(true, false) + new BaseChildFinder(new BasePropertyGatherer()), + new ObjectSerializerFactory(new BasePropertyGatherer()) ); + var (rootId2, _) = await serializeProcess.Serialize(root, default, new SerializeProcessOptions(true, true, false)); rootId2.ShouldBe(root.id); newIdToJson.Count.ShouldBe(count); diff --git a/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/VersionResourceTests.cs b/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/VersionResourceTests.cs index a25badda..9b62f527 100644 --- a/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/VersionResourceTests.cs +++ b/tests/Speckle.Sdk.Tests.Integration/Api/GraphQL/Resources/VersionResourceTests.cs @@ -73,7 +73,7 @@ public async Task VersionUpdate() { const string NEW_MESSAGE = "MY new version message"; - UpdateVersionInput input = new(_version.id, NEW_MESSAGE); + UpdateVersionInput input = new(_version.id, _project.id, NEW_MESSAGE); Version updatedVersion = await Sut.Update(input); Assert.That(updatedVersion, Has.Property(nameof(Version.id)).EqualTo(_version.id)); @@ -84,7 +84,7 @@ public async Task VersionUpdate() [Test] public async Task VersionMoveToModel() { - MoveVersionsInput input = new(_model2.name, new[] { _version.id }); + MoveVersionsInput input = new(_project.id, _model2.name, [_version.id]); string id = await Sut.MoveToModel(input); Assert.That(id, Is.EqualTo(_model2.id)); Version movedVersion = await Sut.Get(_version.id, _project.id); @@ -97,7 +97,7 @@ public async Task VersionMoveToModel() [Test] public async Task VersionDelete() { - DeleteVersionsInput input = new(new[] { _version.id }); + DeleteVersionsInput input = new([_version.id], _project.id); bool response = await Sut.Delete(input); Assert.That(response, Is.True); diff --git a/tests/Speckle.Sdk.Tests.Performance/Benchmarks/GeneralDeserializerTest.cs b/tests/Speckle.Sdk.Tests.Performance/Benchmarks/GeneralDeserializerTest.cs index 194c59ee..79f121d0 100644 --- a/tests/Speckle.Sdk.Tests.Performance/Benchmarks/GeneralDeserializerTest.cs +++ b/tests/Speckle.Sdk.Tests.Performance/Benchmarks/GeneralDeserializerTest.cs @@ -51,10 +51,11 @@ public async Task RunTest_New() TestDataHelper.ServiceProvider.GetRequiredService(), TestDataHelper.ServiceProvider.GetRequiredService(), new Uri(url), + streamId, null ); - var o = new ObjectLoader(sqlite, serverObjects, streamId, null); - var process = new DeserializeProcess(null, o); + var o = new ObjectLoader(sqlite, serverObjects, null); + var process = new DeserializeProcess(null, o, new ObjectDeserializerFactory()); return await process.Deserialize(rootId, default, new(skipCache)).ConfigureAwait(false); }