Minor fix to MerkleTreeStore and added related unit tests

src/Hydrogen/Merkle/FlatMerkleTree.cs

@@ -29,6 +29,7 @@ namespace Hydrogen;
 public class FlatMerkleTree : IDynamicMerkleTree {
 	public const int DefaultLeafGrowth = 4096;
 	public const int DefaultMaxLeaf = 1 << 24;
+
 	private IBuffer _nodeBuffer;
 	private BitArray _dirtyNodes;
 	private readonly int _digestSize;

src/Hydrogen/ObjectSpaces/MetaData/MerkleTreeStore.cs

@@ -7,8 +7,7 @@
 // This notice must not be removed when duplicating this file or its contents, in whole or in part.
 
 using System;
-using System.Collections.Generic;
-using System.Linq;
+using System.Diagnostics;
 using Hydrogen.Collections;
 
 namespace Hydrogen.ObjectSpaces;
@@ -87,7 +86,7 @@ protected override void AttachInternal() {
 			_merkleRootProperty = Container.StreamContainer.Header.CreateExtensionProperty(
 				0, 
 				hashSize, 
-				new ConstantSizeByteArraySerializer(hashSize).WithNullSubstitution(Hashers.ZeroHash(_hashAlgorithm))
+				new ConstantSizeByteArraySerializer(hashSize).WithNullSubstitution(Hashers.ZeroHash(_hashAlgorithm), ByteArrayEqualityComparer.Instance)
 			);
 		}
 	}
@@ -99,14 +98,14 @@ protected override void DetachInternal() {
 		_merkleRootProperty = null;
 	}
 
-	internal void EnsureTreeCalculated() {
-		// TODO: future optimizations can be made by doing smart eliminations of operations from _unsavedChanges
-		// Example: Add(0, "alpha"), Update(0, "beta"), Add(1, "gamma"), Remove(1) -> Add(0, "beta")
+	private void EnsureTreeCalculated() {
+		// TODO: Guard ensure access scope is entered
 
 		if (!_dirtyRoot) 
 			return;
 
 		_merkleRootProperty.Value = _merkleTree.Root;
+		_dirtyRoot = true;
 	}
 
 	private class ContainerLockingMerkleTree : MerkleTreeDecorator  {
@@ -123,6 +122,8 @@ public override byte[] Root {
 			get {
 				using var _ = _container.EnterAccessScope();
 				_merkleTreeStore.EnsureTreeCalculated();
+				var root = base.Root;;
+				Debug.Assert(ByteArrayEqualityComparer.Instance.Equals(root, _merkleTreeStore._merkleRootProperty.Value));
 				return base.Root;
 			}
 		}

tests/Hydrogen.Tests/Collections/StreamMappedMerkle/StreamMappedMerkleListTests.cs

@@ -32,6 +32,38 @@ public void TestAdaptedScopes([Values(CHF.SHA2_256, CHF.Blake2b_128)] CHF chf) {
 		Assert.That(clusteredList.MerkleTree.Root, Is.EqualTo(MerkleTree.ComputeMerkleRoot(new[] { "alpha", "beta", "gamma", "delta", "epsilon" }, chf)));
 	}
 
+
+
+	[Test]
+	public void Special_Remove([Values(CHF.SHA2_256, CHF.Blake2b_128)] CHF chf) {
+		var memStream = new MemoryStream();
+		using var clusteredList = new StreamMappedMerkleList<string>(memStream, chf, 256, autoLoad: true);
+		clusteredList.Add("alpha");
+		clusteredList.Insert(0, "beta");
+		clusteredList.Add("gamma");
+		clusteredList.RemoveAt(1);
+		clusteredList.Add("delta");
+		clusteredList.RemoveAt(0);
+		clusteredList.RemoveAt(0);
+		Assert.That(clusteredList.MerkleTree.Root, Is.EqualTo(MerkleTree.ComputeMerkleRoot(new[] { "delta" }, chf)));
+	}
+
+
+	[Test]
+	public void Special_RemoveAll([Values(CHF.SHA2_256, CHF.Blake2b_128)] CHF chf) {
+		var memStream = new MemoryStream();
+		using var clusteredList = new StreamMappedMerkleList<string>(memStream, chf, 256, autoLoad: true);
+		clusteredList.Add("alpha");
+		clusteredList.Insert(0, "beta");
+		clusteredList.Add("gamma");
+		clusteredList.RemoveAt(1);
+		clusteredList.Add("delta");
+		clusteredList.RemoveAt(0);
+		clusteredList.RemoveAt(0);
+		clusteredList.RemoveAt(0);
+		Assert.That(clusteredList.MerkleTree.Root, Is.Null);
+	}
+
 	protected override IDisposable CreateMerkleList([Values(CHF.SHA2_256, CHF.Blake2b_128)] CHF chf, out IMerkleList<string> merkleList) {
 		var memStream = new MemoryStream();
 		var clusteredList = new StreamMappedMerkleList<string>(memStream, chf, DefaultClusterSize, autoLoad: true);

tests/Hydrogen.Tests/Collections/StreamMappedMerkle/StreamMappedMerkleRecyclableListTests.cs

@@ -10,10 +10,9 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
-using System.Text;
+using FastSerialization;
 using NUnit.Framework;
-using Hydrogen.NUnit;
-using Microsoft.CodeAnalysis.CSharp.Syntax;
+
 
 namespace Hydrogen.Tests;
 
@@ -22,10 +21,16 @@ namespace Hydrogen.Tests;
 public class StreamMappedMerkleRecyclableListTests : RecyclableListTestsBase {
 
 	protected override IDisposable CreateList<T>(IItemSerializer<T> serializer, IEqualityComparer<T> comparer, out IRecyclableList<T> list) {
+		var result = CreateList(CHF.SHA2_256, serializer, comparer, out var mlist);
+		list = mlist;
+		return result;
+	}
+
+	protected IDisposable CreateList<T>(CHF chf, IItemSerializer<T> serializer, IEqualityComparer<T> comparer, out StreamMappedMerkleRecyclableList<T> list) {
 		var stream = new MemoryStream();
 		var smrlist = new StreamMappedMerkleRecyclableList<T>(
 			stream,
-			CHF.SHA2_256,
+			chf,
 			32, 
 			serializer,
 			autoLoad: true
@@ -35,135 +40,164 @@ protected override IDisposable CreateList<T>(IItemSerializer<T> serializer, IEqu
 	}
 
 	[Test]
-	public void WalkThrough_CheckMerkleTree() {
+	public void WalkThrough_CheckMerkleTree([Values(CHF.SHA2_256, CHF.Blake2b_128)] CHF chf) {
 		var serializer = new StringSerializer();
-		var recycledHash = Hashers.ZeroHash(CHF.SHA2_256);
-		using var disposables = CreateList(serializer, StringComparer.InvariantCultureIgnoreCase, out var rlist);
+		var recycledHash = Hashers.ZeroHash(chf);
+		using var disposables = CreateList(chf, serializer, StringComparer.InvariantCultureIgnoreCase, out var list);
 
-		byte[] TreeHash(params string[] text) => MerkleTree.ComputeMerkleRoot(text.Select(x => x is not null ? Hashers.Hash(CHF.SHA2_256, serializer.SerializeBytesLE(x)) : recycledHash), CHF.SHA2_256);
+		byte[] TreeHash(params string[] text) => MerkleTree.ComputeMerkleRoot(text.Select(x => x is not null ? Hashers.Hash(chf, serializer.SerializeBytesLE(x)) : recycledHash), chf);
 
-		var mlist = (StreamMappedMerkleRecyclableList<string>)rlist;
 
 		// verify empty
-		Assert.That(rlist.Count, Is.EqualTo(0));
-		Assert.That(rlist.ListCount, Is.EqualTo(0));
-		Assert.That(rlist.RecycledCount, Is.EqualTo(0));
-		Assert.That(mlist.MerkleTree.Root, Is.Null);
+		Assert.That(list.Count, Is.EqualTo(0));
+		Assert.That(list.ListCount, Is.EqualTo(0));
+		Assert.That(list.RecycledCount, Is.EqualTo(0));
+		Assert.That(list.MerkleTree.Root, Is.Null);
 
 		// Enumerate empty
-		CollectionAssert.AreEqual(rlist, Array.Empty<string>());
+		CollectionAssert.AreEqual(list, Array.Empty<string>());
 
 		// add "A"
-		rlist.Add("A");
-		Assert.That(rlist.Count, Is.EqualTo(1));
-		Assert.That(rlist.ListCount, Is.EqualTo(1));
-		Assert.That(rlist.RecycledCount, Is.EqualTo(0));
-		Assert.That(mlist.MerkleTree.Root, Is.EqualTo(TreeHash("A")));
+		list.Add("A");
+		Assert.That(list.Count, Is.EqualTo(1));
+		Assert.That(list.ListCount, Is.EqualTo(1));
+		Assert.That(list.RecycledCount, Is.EqualTo(0));
+		Assert.That(list.MerkleTree.Root, Is.EqualTo(TreeHash("A")));
 
 		// try insert
-		Assert.That(() => rlist.Insert(0, "text"), Throws.InstanceOf<NotSupportedException>());
+		Assert.That(() => list.Insert(0, "text"), Throws.InstanceOf<NotSupportedException>());
 
 		// add "B" and "C"
-		rlist.AddRange(new[] { "B", "C" });
-		Assert.That(rlist.Count, Is.EqualTo(3));
-		Assert.That(rlist.ListCount, Is.EqualTo(3));
-		Assert.That(rlist.RecycledCount, Is.EqualTo(0));
-		Assert.That(mlist.MerkleTree.Root, Is.EqualTo(TreeHash("A", "B", "C")));
+		list.AddRange(new[] { "B", "C" });
+		Assert.That(list.Count, Is.EqualTo(3));
+		Assert.That(list.ListCount, Is.EqualTo(3));
+		Assert.That(list.RecycledCount, Is.EqualTo(0));
+		Assert.That(list.MerkleTree.Root, Is.EqualTo(TreeHash("A", "B", "C")));
 
 		// remove "B"
-		Assert.That(rlist.Remove("B"), Is.True);
-		Assert.That(rlist.Count, Is.EqualTo(2));
-		Assert.That(rlist.ListCount, Is.EqualTo(3));
-		Assert.That(rlist.RecycledCount, Is.EqualTo(1));
-		Assert.That(mlist.MerkleTree.Root, Is.EqualTo(TreeHash("A", null, "C")));
+		Assert.That(list.Remove("B"), Is.True);
+		Assert.That(list.Count, Is.EqualTo(2));
+		Assert.That(list.ListCount, Is.EqualTo(3));
+		Assert.That(list.RecycledCount, Is.EqualTo(1));
+		Assert.That(list.MerkleTree.Root, Is.EqualTo(TreeHash("A", null, "C")));
 
 		// Ensure B is not found
-		Assert.That(rlist.IndexOf("B"), Is.EqualTo(-1));
+		Assert.That(list.IndexOf("B"), Is.EqualTo(-1));
 
 		// try read recycled
-		Assert.That(() => rlist.Read(1), Throws.ArgumentException);
-		Assert.That(() => rlist.Update(1, "X"), Throws.ArgumentException);
-		Assert.That(() => rlist.RemoveAt(1), Throws.ArgumentException);
+		Assert.That(() => list.Read(1), Throws.ArgumentException);
+		Assert.That(() => list.Update(1, "X"), Throws.ArgumentException);
+		Assert.That(() => list.RemoveAt(1), Throws.ArgumentException);
 
 		// Enumerate 
-		CollectionAssert.AreEqual(rlist, new[] { "A", "C" });
-		Assert.That(rlist.Count, Is.EqualTo(2));
-		Assert.That(rlist.ListCount, Is.EqualTo(3));
-		Assert.That(rlist.RecycledCount, Is.EqualTo(1));
+		CollectionAssert.AreEqual(list, new[] { "A", "C" });
+		Assert.That(list.Count, Is.EqualTo(2));
+		Assert.That(list.ListCount, Is.EqualTo(3));
+		Assert.That(list.RecycledCount, Is.EqualTo(1));
 
 		// add "B1" (verify used recycled index)
-		rlist.Add("B1");
-		Assert.That(rlist.Count, Is.EqualTo(3));
-		Assert.That(rlist.ListCount, Is.EqualTo(3));
-		Assert.That(rlist.RecycledCount, Is.EqualTo(0));
-		Assert.That(rlist.IndexOf("B1"), Is.EqualTo(1));
-		Assert.That(rlist.Read(1), Is.EqualTo("B1"));
-		Assert.That(mlist.MerkleTree.Root, Is.EqualTo(TreeHash("A", "B1", "C")));
+		list.Add("B1");
+		Assert.That(list.Count, Is.EqualTo(3));
+		Assert.That(list.ListCount, Is.EqualTo(3));
+		Assert.That(list.RecycledCount, Is.EqualTo(0));
+		Assert.That(list.IndexOf("B1"), Is.EqualTo(1));
+		Assert.That(list.Read(1), Is.EqualTo("B1"));
+		Assert.That(list.MerkleTree.Root, Is.EqualTo(TreeHash("A", "B1", "C")));
 
 		// Update B1 to B2
-		rlist.Update(1, "B2");
-		Assert.That(rlist.IndexOf("B2"), Is.EqualTo(1));
-		Assert.That(rlist.Count, Is.EqualTo(3));
-		Assert.That(rlist.ListCount, Is.EqualTo(3));
-		Assert.That(rlist.RecycledCount, Is.EqualTo(0));
-		Assert.That(rlist.IndexOf("B2"), Is.EqualTo(1));
-		Assert.That(rlist.Read(1), Is.EqualTo("B2"));
-		Assert.That(mlist.MerkleTree.Root, Is.EqualTo(TreeHash("A", "B2", "C")));
+		list.Update(1, "B2");
+		Assert.That(list.IndexOf("B2"), Is.EqualTo(1));
+		Assert.That(list.Count, Is.EqualTo(3));
+		Assert.That(list.ListCount, Is.EqualTo(3));
+		Assert.That(list.RecycledCount, Is.EqualTo(0));
+		Assert.That(list.IndexOf("B2"), Is.EqualTo(1));
+		Assert.That(list.Read(1), Is.EqualTo("B2"));
+		Assert.That(list.MerkleTree.Root, Is.EqualTo(TreeHash("A", "B2", "C")));
 
 		// Enumeration check
-		CollectionAssert.AreEqual(rlist, new[] { "A", "B2", "C" });
+		CollectionAssert.AreEqual(list, new[] { "A", "B2", "C" });
 
 		// add another "A" (verify used new index)
-		rlist.Add("A");
-		Assert.That(rlist.ListCount, Is.EqualTo(4));
-		Assert.That(rlist.RecycledCount, Is.EqualTo(0));
-		Assert.That(rlist.Count, Is.EqualTo(4));
-		Assert.That(mlist.MerkleTree.Root, Is.EqualTo(TreeHash("A", "B2", "C", "A")));
+		list.Add("A");
+		Assert.That(list.ListCount, Is.EqualTo(4));
+		Assert.That(list.RecycledCount, Is.EqualTo(0));
+		Assert.That(list.Count, Is.EqualTo(4));
+		Assert.That(list.MerkleTree.Root, Is.EqualTo(TreeHash("A", "B2", "C", "A")));
 
 		// verify index of first "A"
-		Assert.That(rlist.IndexOf("A"), Is.EqualTo(0));
+		Assert.That(list.IndexOf("A"), Is.EqualTo(0));
 
 		// Remove first A
-		rlist.RemoveAt(0);
-		Assert.That(rlist.ListCount, Is.EqualTo(4));
-		Assert.That(rlist.RecycledCount, Is.EqualTo(1));
-		Assert.That(rlist.Count, Is.EqualTo(3));
-		Assert.That(mlist.MerkleTree.Root, Is.EqualTo(TreeHash(null, "B2", "C", "A")));
+		list.RemoveAt(0);
+		Assert.That(list.ListCount, Is.EqualTo(4));
+		Assert.That(list.RecycledCount, Is.EqualTo(1));
+		Assert.That(list.Count, Is.EqualTo(3));
+		Assert.That(list.MerkleTree.Root, Is.EqualTo(TreeHash(null, "B2", "C", "A")));
 
 		// Verify index of second "A"
-		Assert.That(rlist.IndexOf("A"), Is.EqualTo(3));
+		Assert.That(list.IndexOf("A"), Is.EqualTo(3));
 
 		// Remove second "A"
-		rlist.RemoveAt(3);
-		Assert.That(rlist.ListCount, Is.EqualTo(4));
-		Assert.That(rlist.RecycledCount, Is.EqualTo(2));
-		Assert.That(rlist.Count, Is.EqualTo(2));
-		Assert.That(mlist.MerkleTree.Root, Is.EqualTo(TreeHash(null, "B2", "C", null)));
+		list.RemoveAt(3);
+		Assert.That(list.ListCount, Is.EqualTo(4));
+		Assert.That(list.RecycledCount, Is.EqualTo(2));
+		Assert.That(list.Count, Is.EqualTo(2));
+		Assert.That(list.MerkleTree.Root, Is.EqualTo(TreeHash(null, "B2", "C", null)));
 
 		// Verify recycled indices
-		Assert.That(rlist.IsRecycled(0), Is.True);
-		Assert.That(rlist.IsRecycled(1), Is.False);
-		Assert.That(rlist.IsRecycled(2), Is.False);
-		Assert.That(rlist.IsRecycled(3), Is.True);
+		Assert.That(list.IsRecycled(0), Is.True);
+		Assert.That(list.IsRecycled(1), Is.False);
+		Assert.That(list.IsRecycled(2), Is.False);
+		Assert.That(list.IsRecycled(3), Is.True);
 
 		// Verify not "A" is found
-		Assert.That(rlist.IndexOf("A"), Is.EqualTo(-1));
-		Assert.That(rlist.Contains("A"), Is.False);
+		Assert.That(list.IndexOf("A"), Is.EqualTo(-1));
+		Assert.That(list.Contains("A"), Is.False);
 
 
 		// Verify index of B2 and C are (1, 2) respectively and other indices are recycled
-		Assert.That(rlist.IndexOf("B2"), Is.EqualTo(1));
-		Assert.That(rlist.IndexOf("C"), Is.EqualTo(2));
+		Assert.That(list.IndexOf("B2"), Is.EqualTo(1));
+		Assert.That(list.IndexOf("C"), Is.EqualTo(2));
 
 		// Enumerate "B2" and "C"
-		CollectionAssert.AreEqual(rlist, new[] { "B2", "C" });
+		CollectionAssert.AreEqual(list, new[] { "B2", "C" });
 
 		// Clear
-		rlist.Clear();
-		Assert.That(rlist.ListCount, Is.EqualTo(0));
-		Assert.That(rlist.RecycledCount, Is.EqualTo(0));
-		Assert.That(rlist.Count, Is.EqualTo(0));
-		Assert.That(mlist.MerkleTree.Root, Is.Null);
+		list.Clear();
+		Assert.That(list.ListCount, Is.EqualTo(0));
+		Assert.That(list.RecycledCount, Is.EqualTo(0));
+		Assert.That(list.Count, Is.EqualTo(0));
+		Assert.That(list.MerkleTree.Root, Is.Null);
+	}
+
+
+
+
+	[Test]
+	public void Special_Remove([Values(CHF.SHA2_256, CHF.Blake2b_128)] CHF chf) {
+		using var disposables = CreateList(chf, new StringSerializer(), StringComparer.InvariantCultureIgnoreCase, out var list);
+		list.Add("alpha");
+		list.Add("beta");
+		list.Add("gamma");
+		list.RemoveAt(0);
+		list.Add("delta");
+		list.RemoveAt(1);
+		list.RemoveAt(2);
+		Assert.That(list.MerkleTree.Root, Is.EqualTo(MerkleTree.ComputeMerkleRoot(new[] { "delta", null, null }, chf)));
 	}
 
+
+	[Test]
+	public void Special_RemoveAll([Values(CHF.SHA2_256, CHF.Blake2b_128)] CHF chf) {
+		using var disposables = CreateList(chf, new StringSerializer(), StringComparer.InvariantCultureIgnoreCase, out var list);
+		list.Add("alpha");
+		list.Add("beta");
+		list.Add("gamma");
+		list.RemoveAt(0);
+		list.Add("delta");
+		list.RemoveAt(0);
+		list.RemoveAt(1);
+		list.RemoveAt(2);
+		Assert.That(list.MerkleTree.Root, Is.EqualTo(MerkleTree.ComputeMerkleRoot(new string[] { null, null, null }, chf)));
+	}
 }