From dfe29616bf8e9021163443aa29af113de5006f15 Mon Sep 17 00:00:00 2001 From: Herman Schoenfeld Date: Fri, 16 Feb 2024 18:29:15 +1000 Subject: [PATCH] Lazily calculate merkle-root for merkleized collections --- .../StreamMapped/IStreamMappedCollection.cs | 12 +++ .../StreamMapped/IStreamMappedDictionary.cs | 6 +- .../StreamMapped/IStreamMappedList.cs | 5 +- .../IStreamMappedRecyclableList.cs | 4 +- .../StreamMapped/StreamMappedDictionary.cs | 4 +- .../StreamMapped/StreamMappedDictionaryCLK.cs | 2 +- .../StreamMapped/StreamMappedList.cs | 4 +- .../StreamMappedRecyclableList.cs | 4 +- src/Hydrogen/Merkle/StreamMappedMerkleList.cs | 2 + .../StreamMappedMerkleRecyclableList.cs | 2 + .../ObjectSpaces/Indexes/MerkleTreeIndex.cs | 6 +- .../ObjectSpaces/MetaData/MerkleTreeStore.cs | 82 ++++++++++++------- src/Hydrogen/ObjectSpaces/ObjectSpace.cs | 77 +++++++++++------ .../Transactions/TransactionalList.cs | 2 + 14 files changed, 135 insertions(+), 77 deletions(-) create mode 100644 src/Hydrogen/Collections/StreamMapped/IStreamMappedCollection.cs diff --git a/src/Hydrogen/Collections/StreamMapped/IStreamMappedCollection.cs b/src/Hydrogen/Collections/StreamMapped/IStreamMappedCollection.cs new file mode 100644 index 00000000..f39f962f --- /dev/null +++ b/src/Hydrogen/Collections/StreamMapped/IStreamMappedCollection.cs @@ -0,0 +1,12 @@ +using Hydrogen.ObjectSpaces; + +namespace Hydrogen; + +public interface IStreamMappedCollection { + ObjectContainer ObjectContainer { get; } +} + +public interface IStreamMappedCollection : IStreamMappedCollection { + new ObjectContainer ObjectContainer { get; } + +} \ No newline at end of file diff --git a/src/Hydrogen/Collections/StreamMapped/IStreamMappedDictionary.cs b/src/Hydrogen/Collections/StreamMapped/IStreamMappedDictionary.cs index 48693964..d5408763 100644 --- a/src/Hydrogen/Collections/StreamMapped/IStreamMappedDictionary.cs +++ b/src/Hydrogen/Collections/StreamMapped/IStreamMappedDictionary.cs @@ -8,14 +8,10 @@ using System; using System.Collections.Generic; -using Hydrogen.ObjectSpaces; - namespace Hydrogen; -public interface IStreamMappedDictionary : IDictionary, ILoadable, IDisposable { - - ObjectContainer ObjectContainer { get; } +public interface IStreamMappedDictionary : IDictionary, IStreamMappedCollection, ILoadable, IDisposable { TKey ReadKey(long index); diff --git a/src/Hydrogen/Collections/StreamMapped/IStreamMappedList.cs b/src/Hydrogen/Collections/StreamMapped/IStreamMappedList.cs index 947a2b71..873a970f 100644 --- a/src/Hydrogen/Collections/StreamMapped/IStreamMappedList.cs +++ b/src/Hydrogen/Collections/StreamMapped/IStreamMappedList.cs @@ -8,13 +8,10 @@ using System; using System.Collections.Generic; -using Hydrogen.ObjectSpaces; namespace Hydrogen; -public interface IStreamMappedList : IExtendedList, ILoadable, IDisposable { - - ObjectContainer ObjectContainer { get; } +public interface IStreamMappedList : IExtendedList, IStreamMappedCollection, ILoadable, IDisposable { IItemSerializer ItemSerializer { get; } diff --git a/src/Hydrogen/Collections/StreamMapped/IStreamMappedRecyclableList.cs b/src/Hydrogen/Collections/StreamMapped/IStreamMappedRecyclableList.cs index 0df44289..c74a08d3 100644 --- a/src/Hydrogen/Collections/StreamMapped/IStreamMappedRecyclableList.cs +++ b/src/Hydrogen/Collections/StreamMapped/IStreamMappedRecyclableList.cs @@ -12,9 +12,7 @@ namespace Hydrogen; -public interface IStreamMappedRecyclableList : IRecyclableList, ILoadable, IDisposable { - - ObjectContainer ObjectContainer { get; } +public interface IStreamMappedRecyclableList : IRecyclableList, IStreamMappedCollection, ILoadable, IDisposable { IItemSerializer ItemSerializer { get; } diff --git a/src/Hydrogen/Collections/StreamMapped/StreamMappedDictionary.cs b/src/Hydrogen/Collections/StreamMapped/StreamMappedDictionary.cs index f5836293..93b572c6 100644 --- a/src/Hydrogen/Collections/StreamMapped/StreamMappedDictionary.cs +++ b/src/Hydrogen/Collections/StreamMapped/StreamMappedDictionary.cs @@ -9,9 +9,7 @@ using System; using System.Collections.Generic; using System.Diagnostics; -using System.IO; using System.Linq; -using System.Runtime.CompilerServices; using System.Threading.Tasks; using Hydrogen.ObjectSpaces; @@ -47,7 +45,7 @@ internal StreamMappedDictionary(ObjectContainer objectContainer, IEqualityCompar } - ObjectContainer IStreamMappedDictionary.ObjectContainer => ObjectContainer; + ObjectContainer IStreamMappedCollection.ObjectContainer => ObjectContainer; public ObjectContainer> ObjectContainer { get; } diff --git a/src/Hydrogen/Collections/StreamMapped/StreamMappedDictionaryCLK.cs b/src/Hydrogen/Collections/StreamMapped/StreamMappedDictionaryCLK.cs index d779d598..666af3c4 100644 --- a/src/Hydrogen/Collections/StreamMapped/StreamMappedDictionaryCLK.cs +++ b/src/Hydrogen/Collections/StreamMapped/StreamMappedDictionaryCLK.cs @@ -47,7 +47,7 @@ internal StreamMappedDictionaryCLK(ObjectContainer objectContainer, IEqualityCom Load(); } - ObjectContainer IStreamMappedDictionary.ObjectContainer => ObjectContainer; + ObjectContainer IStreamMappedCollection.ObjectContainer => ObjectContainer; public ObjectContainer ObjectContainer { get; } diff --git a/src/Hydrogen/Collections/StreamMapped/StreamMappedList.cs b/src/Hydrogen/Collections/StreamMapped/StreamMappedList.cs index 5fa7f55b..b0359bdb 100644 --- a/src/Hydrogen/Collections/StreamMapped/StreamMappedList.cs +++ b/src/Hydrogen/Collections/StreamMapped/StreamMappedList.cs @@ -15,7 +15,7 @@ namespace Hydrogen; /// -/// A list whose items are persisted over a stream via an . +/// A list whose items are persisted over a stream via an . /// /// public class StreamMappedList : SingularListBase, IStreamMappedList { @@ -38,6 +38,8 @@ internal StreamMappedList(ObjectContainer container, IEqualityComparer ObjectContainer { get; } + ObjectContainer IStreamMappedCollection.ObjectContainer => ObjectContainer; + public IItemSerializer ItemSerializer => ObjectContainer.ItemSerializer; public IEqualityComparer ItemComparer { get; } diff --git a/src/Hydrogen/Collections/StreamMapped/StreamMappedRecyclableList.cs b/src/Hydrogen/Collections/StreamMapped/StreamMappedRecyclableList.cs index 09662e3b..60b6f186 100644 --- a/src/Hydrogen/Collections/StreamMapped/StreamMappedRecyclableList.cs +++ b/src/Hydrogen/Collections/StreamMapped/StreamMappedRecyclableList.cs @@ -41,6 +41,8 @@ public StreamMappedRecyclableList(ObjectContainer container, IEqualityCom public ObjectContainer ObjectContainer { get; } + ObjectContainer IStreamMappedCollection.ObjectContainer => ObjectContainer; + public IItemSerializer ItemSerializer => ObjectContainer.ItemSerializer; public IEqualityComparer ItemComparer { get; } @@ -149,5 +151,5 @@ protected override void ClearInternal() { _freeIndexStore.Stack.Clear(); } } - + } diff --git a/src/Hydrogen/Merkle/StreamMappedMerkleList.cs b/src/Hydrogen/Merkle/StreamMappedMerkleList.cs index d5fccedf..ccbe456e 100644 --- a/src/Hydrogen/Merkle/StreamMappedMerkleList.cs +++ b/src/Hydrogen/Merkle/StreamMappedMerkleList.cs @@ -63,6 +63,8 @@ internal StreamMappedMerkleList(IStreamMappedList streamMappedList, bool } public ObjectContainer ObjectContainer => InternalCollection.ObjectContainer; + + ObjectContainer IStreamMappedCollection.ObjectContainer => ObjectContainer; public IItemSerializer ItemSerializer => InternalCollection.ItemSerializer; diff --git a/src/Hydrogen/Merkle/StreamMappedMerkleRecyclableList.cs b/src/Hydrogen/Merkle/StreamMappedMerkleRecyclableList.cs index 76fe9de6..0261974f 100644 --- a/src/Hydrogen/Merkle/StreamMappedMerkleRecyclableList.cs +++ b/src/Hydrogen/Merkle/StreamMappedMerkleRecyclableList.cs @@ -68,6 +68,8 @@ internal StreamMappedMerkleRecyclableList(IStreamMappedRecyclableList str public ObjectContainer ObjectContainer => InternalCollection.ObjectContainer; + ObjectContainer IStreamMappedCollection.ObjectContainer => ObjectContainer; + public IItemSerializer ItemSerializer => InternalCollection.ItemSerializer; public IEqualityComparer ItemComparer => InternalCollection.ItemComparer; diff --git a/src/Hydrogen/ObjectSpaces/Indexes/MerkleTreeIndex.cs b/src/Hydrogen/ObjectSpaces/Indexes/MerkleTreeIndex.cs index bd5ade3b..c438a16d 100644 --- a/src/Hydrogen/ObjectSpaces/Indexes/MerkleTreeIndex.cs +++ b/src/Hydrogen/ObjectSpaces/Indexes/MerkleTreeIndex.cs @@ -8,6 +8,7 @@ using Hydrogen.Collections; using System; +using System.Collections.Generic; using System.ComponentModel; using System.IO; @@ -26,13 +27,12 @@ public MerkleTreeIndex( long reservedStreamIndex, Func itemDigestor, CHF chf - ) : base(objectContainer, new MerkleTreeStore(objectContainer, reservedStreamIndex, chf)) { + ) : base(objectContainer, new MerkleTreeStore(objectContainer, reservedStreamIndex, chf)) { _itemDigestor = itemDigestor; } public IMerkleTree MerkleTree => KeyStore.MerkleTree; - protected override void OnAdded(object item, long index) => KeyStore.Add(index, _itemDigestor(index)); @@ -40,6 +40,6 @@ protected override void OnInserted(object item, long index) => KeyStore.Insert(index, _itemDigestor(index)); protected override void OnUpdated(object item, long index) - => KeyStore.Update(index, _itemDigestor(index)); + => KeyStore.Update(index, _itemDigestor(index)); } diff --git a/src/Hydrogen/ObjectSpaces/MetaData/MerkleTreeStore.cs b/src/Hydrogen/ObjectSpaces/MetaData/MerkleTreeStore.cs index 8dc4d28c..15b895fb 100644 --- a/src/Hydrogen/ObjectSpaces/MetaData/MerkleTreeStore.cs +++ b/src/Hydrogen/ObjectSpaces/MetaData/MerkleTreeStore.cs @@ -18,44 +18,26 @@ namespace Hydrogen.ObjectSpaces; /// stores only the value part in the container, the keys are stored in these (mapped to a reserved stream). /// /// Unlike which automatically extracts the key from the item and stores it, this is used as a primary storage for the key itself. Thus it is not an index, it is a pure store. -/// internal class MerkleTreeStore : MetaDataStoreBase { private readonly CHF _hashAlgorithm; private IMerkleTree _readOnlyMerkleTree; private IDynamicMerkleTree _merkleTree; private StreamMappedProperty _merkleRootProperty; + private bool _dirtyRoot; // Migrate from MerkleTreeIndex stuff into here public MerkleTreeStore(ObjectContainer container, long reservedStreamIndex, CHF hashAlgorithm) : base(container, reservedStreamIndex) { _hashAlgorithm = hashAlgorithm; + _dirtyRoot = false; } public IMerkleTree MerkleTree => _readOnlyMerkleTree; - protected override void AttachInternal() { - var flatTreeData = new StreamMappedBuffer(Stream); - _merkleTree = new FlatMerkleTree(_hashAlgorithm, flatTreeData, Container.Count); - _readOnlyMerkleTree = new ContainerLockingMerkleTree(_merkleTree, Container); - var hashSize = Hashers.GetDigestSizeBytes(_hashAlgorithm); - using (Container.StreamContainer.EnterAccessScope()) { - _merkleRootProperty = Container.StreamContainer.Header.CreateExtensionProperty( - 0, - hashSize, - new ConstantSizeByteArraySerializer(hashSize).WithNullSubstitution(Hashers.ZeroHash(_hashAlgorithm)) - ); - } - } - - protected override void DetachInternal() { - _merkleTree = null; - _readOnlyMerkleTree = null; - _merkleRootProperty = null; - } - public override long Count => _merkleTree.Size.LeafCount; - public override byte[] Read(long index) => ReadBytes(index); + public override byte[] Read(long index) + => ReadBytes(index); public override byte[] ReadBytes(long index) => _merkleTree.Leafs.Read(index); @@ -63,55 +45,93 @@ public override byte[] ReadBytes(long index) public override void Add(long index, byte[] data) { using var _ = Container.EnterAccessScope(); _merkleTree.Leafs.Add(data); - _merkleRootProperty.Value = _merkleTree.Root; + _dirtyRoot = true; } public override void Update(long index, byte[] data) { using var _ = Container.EnterAccessScope(); _merkleTree.Leafs.Update(index, data); - _merkleRootProperty.Value = _merkleTree.Root; + _dirtyRoot = true; } public override void Insert(long index, byte[] data) { using var _ = Container.EnterAccessScope(); _merkleTree.Leafs.Insert(index, data); - _merkleRootProperty.Value = _merkleTree.Root; + _dirtyRoot = true; } public override void Remove(long index) { using var _ = Container.EnterAccessScope(); _merkleTree.Leafs.RemoveAt(index); - _merkleRootProperty.Value = _merkleTree.Root; + _dirtyRoot = true; } public override void Reap(long index) { using var _ = Container.EnterAccessScope(); var digest = Hashers.ZeroHash(_hashAlgorithm); _merkleTree.Leafs.Update(index, digest); - _merkleRootProperty.Value = _merkleTree.Root; + _dirtyRoot = true; } - public override void Clear() - => _merkleTree.Leafs.Clear(); + public override void Clear() { + _merkleTree.Leafs.Clear(); + _dirtyRoot = true; + } + + protected override void AttachInternal() { + var flatTreeData = new StreamMappedBuffer(Stream); + _merkleTree = new FlatMerkleTree(_hashAlgorithm, flatTreeData, Container.Count); + _readOnlyMerkleTree = new ContainerLockingMerkleTree(this, _merkleTree, Container); + var hashSize = Hashers.GetDigestSizeBytes(_hashAlgorithm); + using (Container.StreamContainer.EnterAccessScope()) { + _merkleRootProperty = Container.StreamContainer.Header.CreateExtensionProperty( + 0, + hashSize, + new ConstantSizeByteArraySerializer(hashSize).WithNullSubstitution(Hashers.ZeroHash(_hashAlgorithm)) + ); + } + } + + protected override void DetachInternal() { + // Ensure tree is calculated + _merkleTree = null; + _readOnlyMerkleTree = null; + _merkleRootProperty = null; + } + + internal void EnsureTreeCalculated() { + // TODO: future optimizations can be made by doing smart eliminations of operations from _unsavedChanges + // Example: Add(0, "alpha"), Update(0, "beta"), Add(1, "gamma"), Remove(1) -> Add(0, "beta") + + if (!_dirtyRoot) + return; + + _merkleRootProperty.Value = _merkleTree.Root; + } private class ContainerLockingMerkleTree : MerkleTreeDecorator { + private readonly MerkleTreeStore _merkleTreeStore; private readonly ObjectContainer _container; - public ContainerLockingMerkleTree(IMerkleTree internalMerkleTree, ObjectContainer container) : base(internalMerkleTree) { + + public ContainerLockingMerkleTree(MerkleTreeStore merkleTreeStore, IMerkleTree internalMerkleTree, ObjectContainer container) + : base(internalMerkleTree) { + _merkleTreeStore = merkleTreeStore; _container = container; } public override byte[] Root { get { using var _ = _container.EnterAccessScope(); + _merkleTreeStore.EnsureTreeCalculated(); return base.Root; } } public override ReadOnlySpan GetValue(MerkleCoordinate coordinate) { using var _ = _container.EnterAccessScope(); + _merkleTreeStore.EnsureTreeCalculated(); return base.GetValue(coordinate); } } - } \ No newline at end of file diff --git a/src/Hydrogen/ObjectSpaces/ObjectSpace.cs b/src/Hydrogen/ObjectSpaces/ObjectSpace.cs index 539f21f1..29311b2f 100644 --- a/src/Hydrogen/ObjectSpaces/ObjectSpace.cs +++ b/src/Hydrogen/ObjectSpaces/ObjectSpace.cs @@ -27,7 +27,7 @@ public class ObjectSpace : SyncLoadableBase, ISynchronizedObject, ITransactional private readonly StreamContainer _streamContainer; private readonly SerializerFactory _serializerFactory; private readonly ComparerFactory _comparerFactory; - private readonly IDictionary _collections; + private readonly IDictionary _collections; private readonly InstanceTracker _instanceTracker; private bool _loaded; @@ -44,17 +44,10 @@ public ObjectSpace(HydrogenFileDescriptor file, ObjectSpaceDefinition objectSpac file, accessMode.WithoutAutoLoad() ); - _fileStream.RollingBack += _ => { - foreach (var disposable in _collections.Values.Cast()) - disposable.Dispose(); - _collections.Clear(); - }; - _fileStream.RolledBack += _ => { - _instanceTracker.Clear(); - _streamContainer.Initialize(); - _loaded = false; - Load(); - }; + _fileStream.Committing += _ => OnCommitting(); + _fileStream.Committed += _ => OnCommitted(); + _fileStream.RollingBack += _ => OnRollingBack(); + _fileStream.RolledBack += _ => OnRolledBack(); _streamContainer = new StreamContainer( _fileStream, (int)file.ClusterSize, @@ -65,7 +58,7 @@ public ObjectSpace(HydrogenFileDescriptor file, ObjectSpaceDefinition objectSpac ); _loaded = false; - _collections = new Dictionary(); + _collections = new Dictionary(); _instanceTracker = new InstanceTracker(); if (accessMode.HasFlag(FileAccessMode.AutoLoad)) Load(); @@ -120,7 +113,6 @@ public bool TryGet(long index, out TItem item) { return true; } - public long Count() { // Get underlying stream mapped collection var objectList = GetList(); @@ -159,15 +151,6 @@ public void Delete(TItem item) { objectList.Recycle(index); } - private StreamMappedRecyclableList GetList() { - var itemType = typeof(TItem); - - if (!_collections.TryGetValue(itemType, out var dimension)) - throw new InvalidOperationException($"A container for type '{itemType.ToStringCS()}' was not registered"); - - return (StreamMappedRecyclableList)dimension; - } - public void Commit() => _fileStream.Commit(); public Task CommitAsync() => _fileStream.CommitAsync(); @@ -215,7 +198,8 @@ protected override void LoadInternal() { _loaded = true; } - protected virtual object BuildObjectList(ContainerDefinition containerDefinition, int containerIndex) { + + protected virtual IStreamMappedCollection BuildObjectList(ContainerDefinition containerDefinition, int containerIndex) { // Get the stream within the object space which will comprise the object container var containerStream = _streamContainer.Open(_streamContainer.Header.ReservedStreams + containerIndex, false, true); @@ -259,7 +243,7 @@ protected virtual object BuildObjectList(ContainerDefinition containerDefinition var comparer = _comparerFactory.GetEqualityComparer(containerDefinition.ObjectType); // construct the the object collection - var list = typeof(StreamMappedRecyclableList<>) + var list = (IStreamMappedCollection)typeof(StreamMappedRecyclableList<>) .MakeGenericType(containerDefinition.ObjectType) .ActivateWithCompatibleArgs( container, @@ -310,4 +294,47 @@ protected virtual IItemSerializer CreateItemSerializer(Type objectType) { protected virtual int SanitizeContainerClusterSize(int clusterSizeB) => Tools.Values.ClipValue(clusterSizeB, 256, 8192); + + protected virtual void OnCommitting() { + // TODO: potentially move this up to Consensus Space + + // need to ensure all merkle-trees are committed + foreach(var collection in _collections.Values) { + if (collection.ObjectContainer.TryFindAttachment(out var merkleTreeIndex)) { + // fetching the root ensures the stream-mapped merkle-tree is fully calculated + var root = merkleTreeIndex.MerkleTree.Root; + } + } + + // TODO: ensure the global merkle-tree is updated + // var truthSingularity = containerTree.MerkleTree.Root; + } + + protected virtual void OnCommitted() { + } + + protected virtual void OnRollingBack() { + // close all object containers when rolling back + foreach (var disposable in _collections.Values.Cast()) + disposable.Dispose(); + _collections.Clear(); + } + + protected virtual void OnRolledBack() { + // reload after rollback + _instanceTracker.Clear(); + _streamContainer.Initialize(); + _loaded = false; + Load(); + } + + private StreamMappedRecyclableList GetList() { + var itemType = typeof(TItem); + + if (!_collections.TryGetValue(itemType, out var dimension)) + throw new InvalidOperationException($"A container for type '{itemType.ToStringCS()}' was not registered"); + + return (StreamMappedRecyclableList)dimension; + } + } diff --git a/src/Hydrogen/Transactions/TransactionalList.cs b/src/Hydrogen/Transactions/TransactionalList.cs index 6561b0b3..2873bc91 100644 --- a/src/Hydrogen/Transactions/TransactionalList.cs +++ b/src/Hydrogen/Transactions/TransactionalList.cs @@ -128,6 +128,8 @@ public TransactionalList(IStreamMappedList streamMappedList, ITransactionalOb public ObjectContainer ObjectContainer => InternalCollection.ObjectContainer; + ObjectContainer IStreamMappedCollection.ObjectContainer => ObjectContainer; + public IItemSerializer ItemSerializer => InternalCollection.ItemSerializer; public IEqualityComparer ItemComparer => InternalCollection.ItemComparer;