diff --git a/tests/Speckle.Sdk.Serialization.Testing/DummyServerObjectManager.cs b/src/Speckle.Sdk/Serialisation/V2/DummySendServerObjectManager.cs similarity index 60% rename from tests/Speckle.Sdk.Serialization.Testing/DummyServerObjectManager.cs rename to src/Speckle.Sdk/Serialisation/V2/DummySendServerObjectManager.cs index 226238f6..e50ef75e 100644 --- a/tests/Speckle.Sdk.Serialization.Testing/DummyServerObjectManager.cs +++ b/src/Speckle.Sdk/Serialisation/V2/DummySendServerObjectManager.cs @@ -1,11 +1,26 @@ using System.Text; using Speckle.Sdk.Dependencies.Serialization; -using Speckle.Sdk.Serialisation.V2; +using Speckle.Sdk.SQLite; using Speckle.Sdk.Transports; -namespace Speckle.Sdk.Serialization.Testing; +namespace Speckle.Sdk.Serialisation.V2; -public class DummyServerObjectManager : IServerObjectManager +public class DummySqLiteJsonCacheManager : ISqLiteJsonCacheManager +{ + public IEnumerable GetAllObjects() => throw new NotImplementedException(); + + public void DeleteObject(string id) => throw new NotImplementedException(); + + public string? GetObject(string id) => throw new NotImplementedException(); + + public void SaveObject(string id, string json) => throw new NotImplementedException(); + + public void SaveObjects(IEnumerable<(string id, string json)> items) => throw new NotImplementedException(); + + public bool HasObject(string objectId) => throw new NotImplementedException(); +} + +public class DummySendServerObjectManager : IServerObjectManager { public IAsyncEnumerable<(string, string)> DownloadObjects( IReadOnlyList objectIds, diff --git a/src/Speckle.Sdk/Serialisation/V2/Receive/DeserializeProcess.cs b/src/Speckle.Sdk/Serialisation/V2/Receive/DeserializeProcess.cs index f1bb6c0e..87aa22a3 100644 --- a/src/Speckle.Sdk/Serialisation/V2/Receive/DeserializeProcess.cs +++ b/src/Speckle.Sdk/Serialisation/V2/Receive/DeserializeProcess.cs @@ -7,7 +7,7 @@ namespace Speckle.Sdk.Serialisation.V2.Receive; -public record DeserializeOptions( +public record DeserializeProcessOptions( bool SkipCache, bool ThrowOnMissingReferences = true, bool SkipInvalidConverts = false @@ -17,10 +17,11 @@ public record DeserializeOptions( public sealed class DeserializeProcess( IProgress? progress, IObjectLoader objectLoader, - IObjectDeserializerFactory objectDeserializerFactory + IObjectDeserializerFactory objectDeserializerFactory, + DeserializeProcessOptions? options = null ) : IDeserializeProcess { - private DeserializeOptions _options = new(false); + private readonly DeserializeProcessOptions _options = options ?? new(false); private readonly ConcurrentDictionary)> _closures = new(); private readonly ConcurrentDictionary _baseCache = new(); @@ -29,13 +30,8 @@ IObjectDeserializerFactory objectDeserializerFactory public IReadOnlyDictionary BaseCache => _baseCache; public long Total { get; private set; } - public async Task Deserialize( - string rootId, - CancellationToken cancellationToken, - DeserializeOptions? options = null - ) + public async Task Deserialize(string rootId, CancellationToken cancellationToken) { - _options = options ?? _options; var (rootJson, childrenIds) = await objectLoader .GetAndCache(rootId, _options, cancellationToken) .ConfigureAwait(false); diff --git a/src/Speckle.Sdk/Serialisation/V2/Receive/ObjectDeserializer.cs b/src/Speckle.Sdk/Serialisation/V2/Receive/ObjectDeserializer.cs index 2da8828d..85206b3d 100644 --- a/src/Speckle.Sdk/Serialisation/V2/Receive/ObjectDeserializer.cs +++ b/src/Speckle.Sdk/Serialisation/V2/Receive/ObjectDeserializer.cs @@ -13,7 +13,7 @@ public sealed class ObjectDeserializer( IReadOnlyCollection currentClosures, IReadOnlyDictionary references, SpeckleObjectSerializerPool pool, - DeserializeOptions? options = null + DeserializeProcessOptions? options = null ) : IObjectDeserializer { /// The JSON string of the object to be deserialized diff --git a/src/Speckle.Sdk/Serialisation/V2/Receive/ObjectDeserializerFactory.cs b/src/Speckle.Sdk/Serialisation/V2/Receive/ObjectDeserializerFactory.cs index f88d6a81..b81f40ce 100644 --- a/src/Speckle.Sdk/Serialisation/V2/Receive/ObjectDeserializerFactory.cs +++ b/src/Speckle.Sdk/Serialisation/V2/Receive/ObjectDeserializerFactory.cs @@ -10,6 +10,6 @@ public IObjectDeserializer Create( string currentId, IReadOnlyCollection currentClosures, IReadOnlyDictionary references, - DeserializeOptions? options = null + DeserializeProcessOptions? options = null ) => new ObjectDeserializer(currentId, currentClosures, references, SpeckleObjectSerializerPool.Instance, options); } diff --git a/src/Speckle.Sdk/Serialisation/V2/Receive/ObjectLoader.cs b/src/Speckle.Sdk/Serialisation/V2/Receive/ObjectLoader.cs index e278e3ce..b55e30ba 100644 --- a/src/Speckle.Sdk/Serialisation/V2/Receive/ObjectLoader.cs +++ b/src/Speckle.Sdk/Serialisation/V2/Receive/ObjectLoader.cs @@ -18,11 +18,11 @@ public sealed class ObjectLoader( private int? _allChildrenCount; private long _checkCache; private long _cached; - private DeserializeOptions _options = new(false); + private DeserializeProcessOptions _options = new(false); public async Task<(string, IReadOnlyCollection)> GetAndCache( string rootId, - DeserializeOptions options, + DeserializeProcessOptions options, CancellationToken cancellationToken ) { diff --git a/src/Speckle.Sdk/Serialisation/V2/Send/EmptyDictionary.cs b/src/Speckle.Sdk/Serialisation/V2/Send/EmptyDictionary.cs new file mode 100644 index 00000000..7059231a --- /dev/null +++ b/src/Speckle.Sdk/Serialisation/V2/Send/EmptyDictionary.cs @@ -0,0 +1,47 @@ +using System.Collections; +using System.Diagnostics.CodeAnalysis; + +namespace Speckle.Sdk.Serialisation.V2.Send; + +public class EmptyDictionary : IDictionary +{ + public IEnumerator> GetEnumerator() => throw new NotImplementedException(); + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + + public void Add(KeyValuePair item) { } + + public void Clear() => throw new NotImplementedException(); + + public bool Contains(KeyValuePair item) => false; + + public void CopyTo(KeyValuePair[] array, int arrayIndex) => throw new NotImplementedException(); + + public bool Remove(KeyValuePair item) => false; + + public int Count => 0; + public bool IsReadOnly => false; + + public void Add(TKey key, TValue value) { } + + public bool ContainsKey(TKey key) => false; + + public bool Remove(TKey key) => false; + + public bool TryGetValue(TKey key, [UnscopedRef] out TValue value) + { + value = default!; + return false; + } + + public TValue this[TKey key] + { +#pragma warning disable CA1065 + get => throw new NotImplementedException(); +#pragma warning restore CA1065 + set { } + } + + public ICollection Keys { get; } + public ICollection Values { get; } +} diff --git a/src/Speckle.Sdk/Serialisation/V2/Send/ObjectSerializer.cs b/src/Speckle.Sdk/Serialisation/V2/Send/ObjectSerializer.cs index 8ef4fe5f..b427094e 100644 --- a/src/Speckle.Sdk/Serialisation/V2/Send/ObjectSerializer.cs +++ b/src/Speckle.Sdk/Serialisation/V2/Send/ObjectSerializer.cs @@ -1,5 +1,4 @@ using System.Collections; -using System.Collections.Concurrent; using System.Drawing; using System.Globalization; using Speckle.DoubleNumerics; @@ -21,7 +20,7 @@ public class ObjectSerializer : IObjectSerializer { private HashSet _parentObjects = new(); private readonly Dictionary _currentClosures = new(); - private readonly ConcurrentDictionary _baseCache; + private readonly IDictionary _baseCache; private readonly bool _trackDetachedChildren; private readonly IBasePropertyGatherer _propertyGatherer; @@ -42,7 +41,7 @@ public class ObjectSerializer : IObjectSerializer /// public ObjectSerializer( IBasePropertyGatherer propertyGatherer, - ConcurrentDictionary baseCache, + IDictionary baseCache, bool trackDetachedChildren = false, CancellationToken cancellationToken = default ) @@ -61,16 +60,21 @@ public ObjectSerializer( { try { + (Id, Json) item; try { - var item = SerializeBase(baseObj, true).NotNull(); - _baseCache.TryAdd(baseObj, new(item.Item2, _currentClosures.Freeze())); - return [new(item.Item1, item.Item2), .. _chunks]; + item = SerializeBase(baseObj, true).NotNull(); } catch (Exception ex) when (!ex.IsFatal() && ex is not OperationCanceledException) { throw new SpeckleSerializeException($"Failed to extract (pre-serialize) properties from the {baseObj}", ex); } + _baseCache[baseObj] = new(item.Item2, _currentClosures); + yield return (item.Item1, item.Item2); + foreach (var chunk in _chunks) + { + yield return chunk; + } } finally { @@ -90,6 +94,25 @@ private void SerializeProperty(object? obj, JsonWriter writer, PropertyAttribute return; } + switch (obj) + { + case double d: + writer.WriteValue(d); + return; + case string d: + writer.WriteValue(d); + return; + case bool d: + writer.WriteValue(d); + return; + case int d: + writer.WriteValue(d); + return; + case long d: + writer.WriteValue(d); + return; + } + if (obj.GetType().IsPrimitive || obj is string) { writer.WriteValue(obj); diff --git a/src/Speckle.Sdk/Serialisation/V2/Send/ObjectSerializerFactory.cs b/src/Speckle.Sdk/Serialisation/V2/Send/ObjectSerializerFactory.cs index c244bff8..2fa75d05 100644 --- a/src/Speckle.Sdk/Serialisation/V2/Send/ObjectSerializerFactory.cs +++ b/src/Speckle.Sdk/Serialisation/V2/Send/ObjectSerializerFactory.cs @@ -1,4 +1,3 @@ -using System.Collections.Concurrent; using Speckle.InterfaceGenerator; using Speckle.Sdk.Models; @@ -7,8 +6,6 @@ namespace Speckle.Sdk.Serialisation.V2.Send; [GenerateAutoInterface] public class ObjectSerializerFactory(IBasePropertyGatherer propertyGatherer) : IObjectSerializerFactory { - public IObjectSerializer Create( - ConcurrentDictionary baseCache, - CancellationToken cancellationToken - ) => new ObjectSerializer(propertyGatherer, baseCache, true, cancellationToken); + public IObjectSerializer Create(IDictionary baseCache, CancellationToken cancellationToken) => + new ObjectSerializer(propertyGatherer, baseCache, true, cancellationToken); } diff --git a/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs b/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs index be07e58a..40186bea 100644 --- a/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs +++ b/src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs @@ -9,7 +9,7 @@ namespace Speckle.Sdk.Serialisation.V2.Send; -public record SerializeProcessOptions(bool SkipCacheRead, bool SkipCacheWrite, bool SkipServer); +public record SerializeProcessOptions(bool SkipCacheRead, bool SkipCacheWrite, bool SkipServer, bool CacheBases); public readonly record struct SerializeProcessResults( string RootId, @@ -22,11 +22,15 @@ public class SerializeProcess( ISqLiteJsonCacheManager sqLiteJsonCacheManager, IServerObjectManager serverObjectManager, IBaseChildFinder baseChildFinder, - IObjectSerializerFactory objectSerializerFactory + IObjectSerializerFactory objectSerializerFactory, + SerializeProcessOptions? options = null ) : ChannelSaver, ISerializeProcess { + private readonly SerializeProcessOptions _options = options ?? new(false, false, false, true); private readonly ConcurrentDictionary _jsonCache = new(); - private readonly ConcurrentDictionary _baseCache = new(); + + private readonly IDictionary _baseCache = + options?.CacheBases ?? true ? new ConcurrentDictionary() : new EmptyDictionary(); private readonly ConcurrentDictionary _objectReferences = new(); private long _totalFound; @@ -35,15 +39,8 @@ IObjectSerializerFactory objectSerializerFactory private long _cached; private long _serialized; - private SerializeProcessOptions _options = new(false, false, false); - - public async Task Serialize( - Base root, - CancellationToken cancellationToken, - SerializeProcessOptions? options = null - ) + public async Task Serialize(Base root, CancellationToken cancellationToken) { - _options = options ?? _options; var channelTask = Start(cancellationToken); await Traverse(root, true, cancellationToken).ConfigureAwait(false); await channelTask.ConfigureAwait(false); diff --git a/src/Speckle.Sdk/Serialisation/V2/SerializeProcessFactory.cs b/src/Speckle.Sdk/Serialisation/V2/SerializeProcessFactory.cs index 48346809..36cc08af 100644 --- a/src/Speckle.Sdk/Serialisation/V2/SerializeProcessFactory.cs +++ b/src/Speckle.Sdk/Serialisation/V2/SerializeProcessFactory.cs @@ -13,13 +13,20 @@ ISerializeProcess CreateSerializeProcess( Uri url, string streamId, string? authorizationToken, - IProgress? progress + IProgress? progress, + SerializeProcessOptions? options = null ); IDeserializeProcess CreateDeserializeProcess( Uri url, string streamId, string? authorizationToken, - IProgress? progress + IProgress? progress, + DeserializeProcessOptions? options = null + ); + + public ISerializeProcess CreateSerializeProcess( + SerializeProcessOptions? options = null, + IProgress? progress = null ); } @@ -36,7 +43,8 @@ public ISerializeProcess CreateSerializeProcess( Uri url, string streamId, string? authorizationToken, - IProgress? progress + IProgress? progress, + SerializeProcessOptions? options = null ) { var sqLiteJsonCacheManager = sqLiteJsonCacheManagerFactory.CreateFromStream(streamId); @@ -46,7 +54,25 @@ public ISerializeProcess CreateSerializeProcess( sqLiteJsonCacheManager, serverObjectManager, baseChildFinder, - objectSerializerFactory + objectSerializerFactory, + options + ); + } + + public ISerializeProcess CreateSerializeProcess( + SerializeProcessOptions? options = null, + IProgress? progress = null + ) + { + var sqLiteJsonCacheManager = new DummySqLiteJsonCacheManager(); + var serverObjectManager = new DummySendServerObjectManager(); + return new SerializeProcess( + progress, + sqLiteJsonCacheManager, + serverObjectManager, + baseChildFinder, + objectSerializerFactory, + options ); } @@ -54,13 +80,14 @@ public IDeserializeProcess CreateDeserializeProcess( Uri url, string streamId, string? authorizationToken, - IProgress? progress + IProgress? progress, + DeserializeProcessOptions? options = null ) { var sqLiteJsonCacheManager = sqLiteJsonCacheManagerFactory.CreateFromStream(streamId); var serverObjectManager = new ServerObjectManager(speckleHttp, activityFactory, url, streamId, authorizationToken); var objectLoader = new ObjectLoader(sqLiteJsonCacheManager, serverObjectManager, progress); - return new DeserializeProcess(progress, objectLoader, objectDeserializerFactory); + return new DeserializeProcess(progress, objectLoader, objectDeserializerFactory, options); } } diff --git a/tests/Speckle.Sdk.Serialization.Testing/Program.cs b/tests/Speckle.Sdk.Serialization.Testing/Program.cs index 74bbef0e..b807f795 100644 --- a/tests/Speckle.Sdk.Serialization.Testing/Program.cs +++ b/tests/Speckle.Sdk.Serialization.Testing/Program.cs @@ -50,16 +50,20 @@ new ObjectDeserializerFactory(), serviceProvider.GetRequiredService() ); -var process = factory.CreateDeserializeProcess(new Uri(url), streamId, token, progress); -var @base = await process.Deserialize(rootId, default, new(skipCacheReceive)).ConfigureAwait(false); +var process = factory.CreateDeserializeProcess(new Uri(url), streamId, token, progress, new(skipCacheReceive)); +var @base = await process.Deserialize(rootId, default).ConfigureAwait(false); Console.WriteLine("Deserialized"); Console.ReadLine(); Console.WriteLine("Executing"); -var process2 = factory.CreateSerializeProcess(new Uri(url), streamId, token, progress); -await process2 - .Serialize(@base, default, new SerializeProcessOptions(skipCacheSendCheck, skipCacheSendSave, true)) - .ConfigureAwait(false); +var process2 = factory.CreateSerializeProcess( + new Uri(url), + streamId, + token, + progress, + new SerializeProcessOptions(skipCacheSendCheck, skipCacheSendSave, true, true) +); +await process2.Serialize(@base, default).ConfigureAwait(false); Console.WriteLine("Detach"); Console.ReadLine(); #pragma warning restore CA1506 diff --git a/tests/Speckle.Sdk.Serialization.Tests/DetachedTests.cs b/tests/Speckle.Sdk.Serialization.Tests/DetachedTests.cs index f15d5088..dabdda56 100644 --- a/tests/Speckle.Sdk.Serialization.Tests/DetachedTests.cs +++ b/tests/Speckle.Sdk.Serialization.Tests/DetachedTests.cs @@ -26,7 +26,9 @@ public void Setup() } [Test(Description = "Checks that all typed properties (including obsolete ones) are returned")] - public async Task CanSerialize_New_Detached() + [TestCase(true)] + [TestCase(false)] + public async Task CanSerialize_New_Detached(bool cacheBases) { var expectedJson = """ { @@ -73,9 +75,10 @@ public async Task CanSerialize_New_Detached() new DummySendCacheManager(objects), new DummyServerObjectManager(), new BaseChildFinder(new BasePropertyGatherer()), - new ObjectSerializerFactory(new BasePropertyGatherer()) + new ObjectSerializerFactory(new BasePropertyGatherer()), + new SerializeProcessOptions(false, false, true, cacheBases) ); - await process2.Serialize(@base, default, new SerializeProcessOptions(false, false, true)).ConfigureAwait(false); + await process2.Serialize(@base, default).ConfigureAwait(false); objects.Count.ShouldBe(2); objects.ContainsKey("9ff8efb13c62fa80f3d1c4519376ba13").ShouldBeTrue(); @@ -190,7 +193,9 @@ public void GetPropertiesExpected_All() } [Test(Description = "Checks that all typed properties (including obsolete ones) are returned")] - public async Task CanSerialize_New_Detached2() + [TestCase(true)] + [TestCase(false)] + public async Task CanSerialize_New_Detached2(bool cacheBases) { var root = """ @@ -264,11 +269,10 @@ public async Task CanSerialize_New_Detached2() new DummySendCacheManager(objects), new DummyServerObjectManager(), new BaseChildFinder(new BasePropertyGatherer()), - new ObjectSerializerFactory(new BasePropertyGatherer()) + new ObjectSerializerFactory(new BasePropertyGatherer()), + new SerializeProcessOptions(false, false, true, cacheBases) ); - var results = await process2 - .Serialize(@base, default, new SerializeProcessOptions(false, false, true)) - .ConfigureAwait(false); + var results = await process2.Serialize(@base, default).ConfigureAwait(false); objects.Count.ShouldBe(9); var x = JObject.Parse(objects["fd4efeb8a036838c53ad1cf9e82b8992"]); diff --git a/tests/Speckle.Sdk.Serialization.Tests/SerializationTests.cs b/tests/Speckle.Sdk.Serialization.Tests/SerializationTests.cs index 99647474..70cd8ba4 100644 --- a/tests/Speckle.Sdk.Serialization.Tests/SerializationTests.cs +++ b/tests/Speckle.Sdk.Serialization.Tests/SerializationTests.cs @@ -24,7 +24,7 @@ private class TestLoader(string json) : IObjectLoader { public Task<(string, IReadOnlyCollection)> GetAndCache( string rootId, - DeserializeOptions? options, + DeserializeProcessOptions? options, CancellationToken cancellationToken ) { @@ -88,7 +88,7 @@ public class TestObjectLoader(Dictionary idToObject) : IObjectLo { public Task<(string, IReadOnlyCollection)> GetAndCache( string rootId, - DeserializeOptions? options, + DeserializeProcessOptions? options, CancellationToken cancellationToken ) { @@ -251,8 +251,8 @@ public async Task Roundtrip_Test_New(string fileName, string rootId, int oldCoun new DummyReceiveServerObjectManager(closure), null ); - var process = new DeserializeProcess(null, o, new ObjectDeserializerFactory()); - var root = await process.Deserialize(rootId, default, new DeserializeOptions(true)); + var process = new DeserializeProcess(null, o, new ObjectDeserializerFactory(), new(true)); + var root = await process.Deserialize(rootId, default); process.BaseCache.Count.ShouldBe(oldCount); process.Total.ShouldBe(oldCount); @@ -262,9 +262,10 @@ public async Task Roundtrip_Test_New(string fileName, string rootId, int oldCoun new DummySqLiteSendManager(), new DummySendServerObjectManager(newIdToJson), new BaseChildFinder(new BasePropertyGatherer()), - new ObjectSerializerFactory(new BasePropertyGatherer()) + new ObjectSerializerFactory(new BasePropertyGatherer()), + new SerializeProcessOptions(true, true, false, true) ); - var (rootId2, _) = await serializeProcess.Serialize(root, default, new SerializeProcessOptions(true, true, false)); + var (rootId2, _) = await serializeProcess.Serialize(root, default); rootId2.ShouldBe(root.id); newIdToJson.Count.ShouldBe(newCount); diff --git a/tests/Speckle.Sdk.Tests.Performance/Benchmarks/GeneralDeserializerTest.cs b/tests/Speckle.Sdk.Tests.Performance/Benchmarks/GeneralDeserializerTest.cs index 510889f6..6ffc63ea 100644 --- a/tests/Speckle.Sdk.Tests.Performance/Benchmarks/GeneralDeserializerTest.cs +++ b/tests/Speckle.Sdk.Tests.Performance/Benchmarks/GeneralDeserializerTest.cs @@ -57,8 +57,8 @@ public async Task RunTest_New() null ); var o = new ObjectLoader(sqlite, serverObjects, null); - var process = new DeserializeProcess(null, o, new ObjectDeserializerFactory()); - return await process.Deserialize(rootId, default, new(skipCache)).ConfigureAwait(false); + var process = new DeserializeProcess(null, o, new ObjectDeserializerFactory(), new(skipCache)); + return await process.Deserialize(rootId, default).ConfigureAwait(false); } /*