Lazily calculate merkle-root for merkleized collections

src/Hydrogen/Collections/StreamMapped/IStreamMappedCollection.cs

@@ -0,0 +1,12 @@
+using Hydrogen.ObjectSpaces;
+
+namespace Hydrogen;
+
+public interface IStreamMappedCollection {
+	ObjectContainer ObjectContainer { get; }
+}
+
+public interface IStreamMappedCollection<TItem> : IStreamMappedCollection {
+	new ObjectContainer<TItem> ObjectContainer { get; }
+
+}

src/Hydrogen/Collections/StreamMapped/IStreamMappedDictionary.cs

@@ -8,14 +8,10 @@
 
 using System;
 using System.Collections.Generic;
-using Hydrogen.ObjectSpaces;
-
 
 namespace Hydrogen;
 
-public interface IStreamMappedDictionary<TKey, TValue> : IDictionary<TKey, TValue>, ILoadable, IDisposable {
-
-	ObjectContainer ObjectContainer { get; }
+public interface IStreamMappedDictionary<TKey, TValue> : IDictionary<TKey, TValue>, IStreamMappedCollection, ILoadable, IDisposable {
 
 	TKey ReadKey(long index);
 

src/Hydrogen/Collections/StreamMapped/IStreamMappedList.cs

@@ -8,13 +8,10 @@
 
 using System;
 using System.Collections.Generic;
-using Hydrogen.ObjectSpaces;
 
 namespace Hydrogen;
 
-public interface IStreamMappedList<TItem> : IExtendedList<TItem>, ILoadable, IDisposable {
-
-	ObjectContainer<TItem> ObjectContainer { get; }
+public interface IStreamMappedList<TItem> : IExtendedList<TItem>,  IStreamMappedCollection<TItem>, ILoadable, IDisposable {
 
 	IItemSerializer<TItem> ItemSerializer { get; }
 

src/Hydrogen/Collections/StreamMapped/IStreamMappedRecyclableList.cs

@@ -12,9 +12,7 @@
 
 namespace Hydrogen;
 
-public interface IStreamMappedRecyclableList<TItem> : IRecyclableList<TItem>, ILoadable, IDisposable {
-
-	ObjectContainer<TItem> ObjectContainer { get; }
+public interface IStreamMappedRecyclableList<TItem> : IRecyclableList<TItem>, IStreamMappedCollection<TItem>, ILoadable, IDisposable {
 
 	IItemSerializer<TItem> ItemSerializer { get; }
 

src/Hydrogen/Collections/StreamMapped/StreamMappedDictionary.cs

@@ -9,9 +9,7 @@
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
-using System.IO;
 using System.Linq;
-using System.Runtime.CompilerServices;
 using System.Threading.Tasks;
 using Hydrogen.ObjectSpaces;
 
@@ -47,7 +45,7 @@ internal StreamMappedDictionary(ObjectContainer objectContainer, IEqualityCompar
 	}
 
 
-	ObjectContainer IStreamMappedDictionary<TKey, TValue>.ObjectContainer => ObjectContainer;
+	ObjectContainer IStreamMappedCollection.ObjectContainer => ObjectContainer;
 
 	public ObjectContainer<KeyValuePair<TKey, TValue>> ObjectContainer { get; }
 

src/Hydrogen/Collections/StreamMapped/StreamMappedDictionaryCLK.cs

@@ -47,7 +47,7 @@ internal StreamMappedDictionaryCLK(ObjectContainer objectContainer, IEqualityCom
 			Load();
 	}
 
-	ObjectContainer IStreamMappedDictionary<TKey, TValue>.ObjectContainer => ObjectContainer;
+	ObjectContainer IStreamMappedCollection.ObjectContainer => ObjectContainer;
 
 	public ObjectContainer<TValue> ObjectContainer { get; }
 

src/Hydrogen/Collections/StreamMapped/StreamMappedList.cs

@@ -15,7 +15,7 @@
 namespace Hydrogen;
 
 /// <summary>
-/// A list whose items are persisted over a stream via an <see cref="IClusteredStorage"/>.
+/// A list whose items are persisted over a stream via an <see cref="ObjectContainer{TItem}"/>.
 /// </summary>
 /// <typeparam name="TItem"></typeparam>
 public class StreamMappedList<TItem> : SingularListBase<TItem>, IStreamMappedList<TItem> {
@@ -38,6 +38,8 @@ internal StreamMappedList(ObjectContainer<TItem> container, IEqualityComparer<TI
 
 	public ObjectContainer<TItem> ObjectContainer { get; }
 
+	ObjectContainer IStreamMappedCollection.ObjectContainer => ObjectContainer;
+
 	public IItemSerializer<TItem> ItemSerializer => ObjectContainer.ItemSerializer;
 
 	public IEqualityComparer<TItem> ItemComparer { get; }

src/Hydrogen/Collections/StreamMapped/StreamMappedRecyclableList.cs

@@ -41,6 +41,8 @@ public StreamMappedRecyclableList(ObjectContainer<TItem> container, IEqualityCom
 
 	public ObjectContainer<TItem> ObjectContainer { get; }
 
+	ObjectContainer IStreamMappedCollection.ObjectContainer => ObjectContainer;
+
 	public IItemSerializer<TItem> ItemSerializer => ObjectContainer.ItemSerializer;
 
 	public IEqualityComparer<TItem> ItemComparer { get; }
@@ -149,5 +151,5 @@ protected override void ClearInternal() {
 			_freeIndexStore.Stack.Clear();
 		}
 	}
-	
+
 }

src/Hydrogen/Merkle/StreamMappedMerkleList.cs

@@ -63,6 +63,8 @@ internal StreamMappedMerkleList(IStreamMappedList<TItem> streamMappedList, bool
 	}
 
 	public ObjectContainer<TItem> ObjectContainer => InternalCollection.ObjectContainer;
+
+	ObjectContainer IStreamMappedCollection.ObjectContainer => ObjectContainer;
 
 	public IItemSerializer<TItem> ItemSerializer => InternalCollection.ItemSerializer;
 

src/Hydrogen/Merkle/StreamMappedMerkleRecyclableList.cs

@@ -68,6 +68,8 @@ internal StreamMappedMerkleRecyclableList(IStreamMappedRecyclableList<TItem> str
 
 	public ObjectContainer<TItem> ObjectContainer => InternalCollection.ObjectContainer;
 
+	ObjectContainer IStreamMappedCollection.ObjectContainer => ObjectContainer;
+
 	public IItemSerializer<TItem> ItemSerializer => InternalCollection.ItemSerializer;
 
 	public IEqualityComparer<TItem> ItemComparer => InternalCollection.ItemComparer;

src/Hydrogen/ObjectSpaces/Indexes/MerkleTreeIndex.cs

@@ -8,6 +8,7 @@
 
 using Hydrogen.Collections;
 using System;
+using System.Collections.Generic;
 using System.ComponentModel;
 using System.IO;
 
@@ -26,20 +27,19 @@ public MerkleTreeIndex(
 		long reservedStreamIndex,
 		Func<long, byte[]> itemDigestor,
 		CHF chf
-	) : base(objectContainer,  new MerkleTreeStore(objectContainer, reservedStreamIndex, chf)) {
+	) : base(objectContainer, new MerkleTreeStore(objectContainer, reservedStreamIndex, chf)) {
 		_itemDigestor = itemDigestor;
 	}
 
 	public IMerkleTree MerkleTree => KeyStore.MerkleTree;
 
-
 	protected override void OnAdded(object item, long index) 
 		=> KeyStore.Add(index, _itemDigestor(index));
 
 	protected override void OnInserted(object item, long index) 
 		=> KeyStore.Insert(index, _itemDigestor(index));
 
 	protected override void OnUpdated(object item, long index) 
-		=> KeyStore.Update(index, _itemDigestor(index));
+		=> KeyStore.Update(index,  _itemDigestor(index));
 
 }

src/Hydrogen/ObjectSpaces/MetaData/MerkleTreeStore.cs

@@ -18,100 +18,120 @@ namespace Hydrogen.ObjectSpaces;
 /// stores only the value part in the container, the keys are stored in these (mapped to a reserved stream).
 /// </summary>
 /// <remarks>Unlike <see cref="KeyIndex{TItem,TKey}"/> which automatically extracts the key from the item and stores it, this is used as a primary storage for the key itself. Thus it is not an index, it is a pure store.</remarks>
-/// <typeparam name="TKey"></typeparam>
 internal class MerkleTreeStore : MetaDataStoreBase<byte[]> {
 	private readonly CHF _hashAlgorithm;
 	private IMerkleTree _readOnlyMerkleTree;
 	private IDynamicMerkleTree _merkleTree;
 	private StreamMappedProperty<byte[]> _merkleRootProperty;
+	private bool _dirtyRoot;
 
 	// Migrate from MerkleTreeIndex stuff into here
 	public MerkleTreeStore(ObjectContainer container, long reservedStreamIndex, CHF hashAlgorithm) 
 		: base(container, reservedStreamIndex) {
 		_hashAlgorithm = hashAlgorithm;
+		_dirtyRoot = false;
 	}
 
 	public IMerkleTree MerkleTree => _readOnlyMerkleTree;
 
-	protected override void AttachInternal() {
-		var flatTreeData = new StreamMappedBuffer(Stream);
-		_merkleTree = new FlatMerkleTree(_hashAlgorithm, flatTreeData, Container.Count);
-		_readOnlyMerkleTree = new ContainerLockingMerkleTree(_merkleTree, Container);
-		var hashSize = Hashers.GetDigestSizeBytes(_hashAlgorithm);
-		using (Container.StreamContainer.EnterAccessScope()) {
-			_merkleRootProperty = Container.StreamContainer.Header.CreateExtensionProperty(
-				0, 
-				hashSize, 
-				new ConstantSizeByteArraySerializer(hashSize).WithNullSubstitution(Hashers.ZeroHash(_hashAlgorithm))
-			);
-		}
-	}
-
-	protected override void DetachInternal() {
-		_merkleTree = null;
-		_readOnlyMerkleTree = null;
-		_merkleRootProperty = null;
-	}
-
 	public override long Count => _merkleTree.Size.LeafCount;
 
-	public override byte[] Read(long index) => ReadBytes(index);
+	public override byte[] Read(long index) 
+		=> ReadBytes(index);
 
 	public override byte[] ReadBytes(long index) 
 		=> _merkleTree.Leafs.Read(index);
 
 	public override void Add(long index, byte[] data) {
 		using var _ = Container.EnterAccessScope();
 		_merkleTree.Leafs.Add(data);
-		_merkleRootProperty.Value = _merkleTree.Root;
+		_dirtyRoot = true;
 	}
 
 	public override void Update(long index, byte[] data) {
 		using var _ = Container.EnterAccessScope();
 		_merkleTree.Leafs.Update(index, data);
-		_merkleRootProperty.Value = _merkleTree.Root;
+		_dirtyRoot = true;
 	}
 
 	public override void Insert(long index, byte[] data) {
 		using var _ = Container.EnterAccessScope();
 		_merkleTree.Leafs.Insert(index, data);
-		_merkleRootProperty.Value = _merkleTree.Root;
+		_dirtyRoot = true;
 	}
 
 	public override void Remove(long index) {
 		using var _ = Container.EnterAccessScope();
 		_merkleTree.Leafs.RemoveAt(index);
-		_merkleRootProperty.Value = _merkleTree.Root;
+		_dirtyRoot = true;
 	}
 
 	public override void Reap(long index) {
 		using var _ = Container.EnterAccessScope();
 		var digest = Hashers.ZeroHash(_hashAlgorithm);
 		_merkleTree.Leafs.Update(index, digest);
-		_merkleRootProperty.Value = _merkleTree.Root;
+		_dirtyRoot = true;
 	}
 
-	public override void Clear() 
-		=> _merkleTree.Leafs.Clear();
+	public override void Clear() {
+		 _merkleTree.Leafs.Clear();
+		 _dirtyRoot = true;
+	}
+
+	protected override void AttachInternal() {
+		var flatTreeData = new StreamMappedBuffer(Stream);
+		_merkleTree = new FlatMerkleTree(_hashAlgorithm, flatTreeData, Container.Count);
+		_readOnlyMerkleTree = new ContainerLockingMerkleTree(this, _merkleTree, Container);
+		var hashSize = Hashers.GetDigestSizeBytes(_hashAlgorithm);
+		using (Container.StreamContainer.EnterAccessScope()) {
+			_merkleRootProperty = Container.StreamContainer.Header.CreateExtensionProperty(
+				0, 
+				hashSize, 
+				new ConstantSizeByteArraySerializer(hashSize).WithNullSubstitution(Hashers.ZeroHash(_hashAlgorithm))
+			);
+		}
+	}
+
+	protected override void DetachInternal() {
+		// Ensure tree is calculated 
+		_merkleTree = null;
+		_readOnlyMerkleTree = null;
+		_merkleRootProperty = null;
+	}
+
+	internal void EnsureTreeCalculated() {
+		// TODO: future optimizations can be made by doing smart eliminations of operations from _unsavedChanges
+		// Example: Add(0, "alpha"), Update(0, "beta"), Add(1, "gamma"), Remove(1) -> Add(0, "beta")
+
+		if (!_dirtyRoot) 
+			return;
+
+		_merkleRootProperty.Value = _merkleTree.Root;
+	}
 
 	private class ContainerLockingMerkleTree : MerkleTreeDecorator  {
+		private readonly MerkleTreeStore _merkleTreeStore;
 		private readonly ObjectContainer _container;
-		public ContainerLockingMerkleTree(IMerkleTree internalMerkleTree, ObjectContainer container) : base(internalMerkleTree) {
+
+		public ContainerLockingMerkleTree(MerkleTreeStore merkleTreeStore, IMerkleTree internalMerkleTree, ObjectContainer container) 
+			: base(internalMerkleTree) {
+			_merkleTreeStore = merkleTreeStore;
 			_container = container;
 		}
 
 		public override byte[] Root { 
 			get {
 				using var _ = _container.EnterAccessScope();
+				_merkleTreeStore.EnsureTreeCalculated();
 				return base.Root;
 			}
 		}
 
 		public override ReadOnlySpan<byte> GetValue(MerkleCoordinate coordinate) {
 			using var _ = _container.EnterAccessScope();
+			_merkleTreeStore.EnsureTreeCalculated();
 			return base.GetValue(coordinate);
 		}
 	}
 
-
 }