Object Spaces: added null strategy for indexes (and unit tests)

src/Hydrogen/Crypto/Hashing/Hashers.cs

@@ -52,7 +52,7 @@ public static int GetDigestSizeBytes(CHF algorithm)
 	public static byte[] Hash<TItem>(CHF algorithm, TItem item, IItemSerializer<TItem> serializer, Endianness endianness = HydrogenDefaults.Endianness) {
 		var bytes = serializer.SerializeToBytes(item, endianness);
 		return Hash(algorithm, bytes);
-}
+	}
 
 
 	[MethodImpl(MethodImplOptions.AggressiveInlining)]

src/Hydrogen/ObjectSpaces/Builder/ObjectSpaceDimensionBuilder.cs

@@ -60,28 +60,31 @@ public ObjectSpaceDimensionBuilder<T> WithIdentifier<TMember>(Expression<Func<T,
 			Type = ObjectSpaceDefinition.IndexType.Identifier,
 			Name = indexName ?? member.Name,
 			Member = member,
+			NullPolicy = IndexNullPolicy.ThrowOnNull
 		};
 		_indexes.Add(index);
 		return this;
 	}
 
-	public ObjectSpaceDimensionBuilder<T> WithIndexOn<TMember>(Expression<Func<T, TMember>> memberExpression, string indexName = null) {
+	public ObjectSpaceDimensionBuilder<T> WithIndexOn<TMember>(Expression<Func<T, TMember>> memberExpression, string indexName = null, IndexNullPolicy nullPolicy = IndexNullPolicy.IgnoreNull) {
 		var member = memberExpression.ToMember();
 		var index = new ObjectSpaceDefinition.IndexDefinition {
 			Type = ObjectSpaceDefinition.IndexType.Index,
 			Name = indexName ?? member.Name,
 			Member = member,
+			NullPolicy = nullPolicy
 		};
 		_indexes.Add(index);
 		return this;
 	}
 
-	public ObjectSpaceDimensionBuilder<T> WithUniqueIndexOn<TMember>(Expression<Func<T, TMember>> memberExpression, string indexName = null) {
+	public ObjectSpaceDimensionBuilder<T> WithUniqueIndexOn<TMember>(Expression<Func<T, TMember>> memberExpression, string indexName = null, IndexNullPolicy nullPolicy = IndexNullPolicy.IgnoreNull) {
 		var member = memberExpression.ToMember();
 		var index = new ObjectSpaceDefinition.IndexDefinition {
 			Type = ObjectSpaceDefinition.IndexType.UniqueKey,
 			Name = indexName ?? member.Name,
 			Member = member,
+			NullPolicy = nullPolicy
 		};
 		_indexes.Add(index);
 		return this;

src/Hydrogen/ObjectSpaces/ObjectSpace.cs

@@ -382,7 +382,7 @@ protected virtual IClusteredStreamsAttachment BuildIdentifier(ObjectStream dimen
 		return
 			keySerializer.IsConstantSize ?
 				IndexFactory.CreateUniqueMemberIndex(dimension, indexDefinition.Name, indexDefinition.Member, keySerializer, keyComparer) :
-				IndexFactory.CreateUniqueMemberChecksumIndex(dimension, indexDefinition.Name, indexDefinition.Member, keySerializer, null, null, keyComparer);
+				IndexFactory.CreateUniqueMemberChecksumIndex(dimension, indexDefinition.Name, indexDefinition.Member, keySerializer, null, null, keyComparer, indexDefinition.NullPolicy);
 	}
 
 	protected virtual IClusteredStreamsAttachment BuildUniqueKey(ObjectStream dimension, ObjectSpaceDefinition.DimensionDefinition dimensionDefinition, ObjectSpaceDefinition.IndexDefinition indexDefinition) {
@@ -391,7 +391,7 @@ protected virtual IClusteredStreamsAttachment BuildUniqueKey(ObjectStream dimens
 		return
 			keySerializer.IsConstantSize ?
 				IndexFactory.CreateUniqueMemberIndex(dimension, indexDefinition.Name, indexDefinition.Member, keySerializer, keyComparer) :
-				IndexFactory.CreateUniqueMemberChecksumIndex(dimension, indexDefinition.Name, indexDefinition.Member, keySerializer, null, null, keyComparer);
+				IndexFactory.CreateUniqueMemberChecksumIndex(dimension, indexDefinition.Name, indexDefinition.Member, keySerializer, null, null, keyComparer, indexDefinition.NullPolicy);
 	}
 
 	protected virtual IClusteredStreamsAttachment BuildIndex(ObjectStream dimension, ObjectSpaceDefinition.DimensionDefinition dimensionDefinition, ObjectSpaceDefinition.IndexDefinition indexDefinition) {
@@ -400,7 +400,7 @@ protected virtual IClusteredStreamsAttachment BuildIndex(ObjectStream dimension,
 		return
 			keySerializer.IsConstantSize ?
 				IndexFactory.CreateMemberIndex(dimension, indexDefinition.Name, indexDefinition.Member, keySerializer, keyComparer) :
-				IndexFactory.CreateMemberChecksumIndex(dimension, indexDefinition.Name, indexDefinition.Member, keySerializer, null, null, keyComparer);
+				IndexFactory.CreateMemberChecksumIndex(dimension, indexDefinition.Name, indexDefinition.Member, keySerializer, null, null, keyComparer, indexDefinition.NullPolicy);
 	}
 
 	protected virtual IClusteredStreamsAttachment BuildRecyclableIndexStore(ObjectStream dimension, ObjectSpaceDefinition.DimensionDefinition dimensionDefinition, ObjectSpaceDefinition.IndexDefinition indexDefinition) {

src/Hydrogen/ObjectSpaces/ObjectSpaceDefinition.cs

@@ -128,6 +128,8 @@ public class IndexDefinition {
 
 		public Member Member { get; set; }
 
+		public IndexNullPolicy NullPolicy { get; set; }
+
 	}
 
 	public enum IndexType {

src/Hydrogen/ObjectStream/Indexes/IndexFactory.cs

@@ -86,7 +86,8 @@ internal static IClusteredStreamsAttachment CreateMemberChecksumIndex(
 		IItemSerializer keySerializer = null, 
 		object keyChecksummer = null, 
 		object keyFetcher = null, 
-		object keyComparer = null
+		object keyComparer = null,
+		IndexNullPolicy indexNullPolicy = IndexNullPolicy.IgnoreNull
 	) {
 		Guard.Ensure(objectStream.GetType() == typeof(ObjectStream<>).MakeGenericType(member.DeclaringType));
 
@@ -101,7 +102,7 @@ internal static IClusteredStreamsAttachment CreateMemberChecksumIndex(
 			.GetMethod(nameof(CreateProjectionChecksumIndex), BindingFlags.NonPublic | BindingFlags.Static)
 			.MakeGenericMethod(member.DeclaringType, member.PropertyType);
 
-		return (IClusteredStreamsAttachment)method.Invoke(null, new object[] { genericContainer, indexName, projection, keySerializer, keyChecksummer, keyFetcher, keyComparer });
+		return (IClusteredStreamsAttachment)method.Invoke(null, new object[] { genericContainer, indexName, projection, keySerializer, keyChecksummer, keyFetcher, keyComparer, indexNullPolicy });
 
 	}
 
@@ -112,7 +113,9 @@ internal static ProjectionChecksumIndex<TItem, TKey> CreateProjectionChecksumInd
 		IItemSerializer<TKey> keySerializer = null, 
 		IItemChecksummer<TKey> keyChecksummer = null, 
 		Func<long, TKey> keyFetcher = null, 
-		IEqualityComparer<TKey> keyComparer= null) {
+		IEqualityComparer<TKey> keyComparer= null,
+		IndexNullPolicy indexNullPolicy = IndexNullPolicy.IgnoreNull
+	) {
 
 		keySerializer ??= ItemSerializer<TKey>.Default;
 		keyChecksummer ??= new ItemDigestor<TKey>(keySerializer, objectStream.Streams.Endianness);
@@ -124,7 +127,8 @@ internal static ProjectionChecksumIndex<TItem, TKey> CreateProjectionChecksumInd
 			projection,
 			keyChecksummer,
 			keyFetcher,
-			keyComparer
+			keyComparer,
+			indexNullPolicy
 		);
 
 		return keyChecksumKeyIndex;
@@ -141,7 +145,8 @@ internal static IClusteredStreamsAttachment CreateUniqueMemberChecksumIndex(
 		IItemSerializer keySerializer = null,
 		object keyChecksummer = null, 
 		object keyFetcher = null, 
-		object keyComparer = null
+		object keyComparer = null,
+		IndexNullPolicy indexNullPolicy = IndexNullPolicy.IgnoreNull
 	) {
 		Guard.Ensure(objectStream.GetType() == typeof(ObjectStream<>).MakeGenericType(member.DeclaringType));
 
@@ -156,7 +161,7 @@ internal static IClusteredStreamsAttachment CreateUniqueMemberChecksumIndex(
 			.GetMethod(nameof(CreateUniqueProjectionChecksumIndex), BindingFlags.NonPublic | BindingFlags.Static)
 			.MakeGenericMethod(member.DeclaringType, member.PropertyType);
 
-		return (IClusteredStreamsAttachment)method.Invoke(null, new object[] { genericContainer, indexName, projection, keySerializer, keyChecksummer, keyFetcher, keyComparer });
+		return (IClusteredStreamsAttachment)method.Invoke(null, new object[] { genericContainer, indexName, projection, keySerializer, keyChecksummer, keyFetcher, keyComparer, indexNullPolicy });
 
 	}
 
@@ -167,7 +172,8 @@ internal static UniqueProjectionChecksumIndex<TItem, TKey> CreateUniqueProjectio
 		IItemSerializer<TKey> keySerializer = null, 
 		IItemChecksummer<TKey> keyChecksummer = null, 
 		Func<long, TKey> keyFetcher = null, 
-		IEqualityComparer<TKey> keyComparer= null
+		IEqualityComparer<TKey> keyComparer = null,
+		IndexNullPolicy indexNullPolicy = IndexNullPolicy.IgnoreNull
 	) {
 		keySerializer ??= ItemSerializer<TKey>.Default;
 		keyChecksummer ??= new ItemDigestor<TKey>(keySerializer, objectStream.Streams.Endianness);
@@ -179,7 +185,8 @@ internal static UniqueProjectionChecksumIndex<TItem, TKey> CreateUniqueProjectio
 			projection,
 			keyChecksummer,
 			keyFetcher,
-			keyComparer
+			keyComparer,
+			indexNullPolicy
 		);
 
 		return uniqueKeyChecksumIndex;

src/Hydrogen/ObjectStream/Indexes/IndexNullPolicy.cs

@@ -0,0 +1,7 @@
+namespace Hydrogen;
+
+public enum IndexNullPolicy {
+	IgnoreNull,
+	IndexNullValue,
+	ThrowOnNull,
+}

src/Hydrogen/ObjectStream/Indexes/ProjectionChecksumIndex.cs

@@ -32,10 +32,12 @@ public ProjectionChecksumIndex(
 		Func<TItem, TProjection> projection,
 		IItemChecksummer<TProjection> projectionChecksummer,
 		Func<long, TProjection> projectionHydrator,
-		IEqualityComparer<TProjection> keyComparer
+		IEqualityComparer<TProjection> keyComparer,
+		IndexNullPolicy nullPolicy
 	) : base(
 			objectStream,
-			new IndexStorageAttachment<int>(objectStream.Streams, indexName, PrimitiveSerializer<int>.Instance, EqualityComparer<int>.Default)
+			new IndexStorageAttachment<int>(objectStream.Streams, indexName, PrimitiveSerializer<int>.Instance, EqualityComparer<int>.Default),
+			nullPolicy
 		) {
 		Guard.ArgumentNotNull(projection, nameof(projection));
 		Guard.ArgumentNotNull(projectionChecksummer, nameof(projectionChecksummer));
@@ -96,6 +98,8 @@ protected override void OnContainerCleared() {
 		Store.Attach();
 	}
 
+	protected override bool IsNullValue((TProjection, int) projection) => projection.Item1 is null;
+
 	private class ChecksumBasedLookup : ILookup<TProjection, long> {
 		private readonly ProjectionChecksumIndex<TItem, TProjection> _parent;
 

src/Hydrogen/ObjectStream/Indexes/ProjectionIndex.cs

@@ -21,9 +21,11 @@ internal sealed class ProjectionIndex<TItem, TProjection> : ProjectionIndexBase<
 	public ProjectionIndex(ObjectStream<TItem> objectStream, string indexName, Func<TItem, TProjection> projection, IItemSerializer<TProjection> projectionSerializer, IEqualityComparer<TProjection> projectionComparer)
 		: base(
 			objectStream,
-			new IndexStorageAttachment<TProjection>(objectStream.Streams, indexName, projectionSerializer, projectionComparer)
+			new IndexStorageAttachment<TProjection>(objectStream.Streams, indexName, projectionSerializer, projectionComparer),
+			IndexNullPolicy.ThrowOnNull
 		) {
 		Guard.ArgumentNotNull(projection, nameof(projection));
+		Guard.Argument(projectionSerializer.IsConstantSize, nameof(projectionSerializer), "Must be a constant size serializer");
 		_projection = projection;
 	}
 

src/Hydrogen/ObjectStream/Indexes/ProjectionIndexBase.cs

@@ -6,6 +6,8 @@
 //
 // This notice must not be removed when duplicating this file or its contents, in whole or in part.
 
+using System;
+
 namespace Hydrogen;
 
 /// <summary>
@@ -16,19 +18,25 @@ namespace Hydrogen;
 /// <typeparam name="TStore">Type of store used to store item member values</typeparam>
 public abstract class ProjectionIndexBase<TItem, TProjection, TStore> : IndexBase<TStore> where TStore : IClusteredStreamsAttachment {
 	protected new ObjectStream<TItem> Objects;
-	protected ProjectionIndexBase(ObjectStream<TItem> objectStream, TStore store)
+
+	protected ProjectionIndexBase(ObjectStream<TItem> objectStream, TStore store, IndexNullPolicy nullPolicy)
 		: base(objectStream, store) {
 
 		Guard.Ensure(!store.IsAttached, "Store must not be attached already");
 		Objects = (ObjectStream<TItem>)base.Objects;
+		NullPolicy = nullPolicy;
 	}
 
+	public IndexNullPolicy NullPolicy { get; }
+
 	public abstract TProjection ApplyProjection(TItem item);
 
 	protected sealed override void OnAdding(object item, long index) {
 		base.OnAdding(item, index);
 		var itemT = (TItem)item;
-		OnAdding(itemT, index, ApplyProjection(itemT));
+		var projection = ApplyProjection(itemT);
+		if (ApplyNullPolicy(itemT, projection))
+			OnAdding(itemT, index, projection);
 	}
 
 	protected sealed override void OnAdded(object item, long index) {
@@ -40,7 +48,9 @@ protected sealed override void OnAdded(object item, long index) {
 	protected sealed override void OnInserting(object item, long index) {
 		base.OnInserting(item, index);
 		var itemT = (TItem)item;
-		OnInserting(itemT, index, ApplyProjection(itemT));
+		var projection = ApplyProjection(itemT);
+		if (ApplyNullPolicy(itemT, projection))
+			OnInserting(itemT, index, projection);
 	}
 
 	protected sealed override void OnInserted(object item, long index) {
@@ -52,7 +62,9 @@ protected sealed override void OnInserted(object item, long index) {
 	protected sealed override void OnUpdating(object item, long index) {
 		base.OnUpdating(item, index);
 		var itemT = (TItem)item;
-		OnUpdating(itemT, index, ApplyProjection(itemT));
+		var projection = ApplyProjection(itemT);
+		if (ApplyNullPolicy(itemT, projection))
+			OnUpdating(itemT, index, projection);
 	}
 
 	protected sealed override void OnUpdated(object item, long index) {
@@ -85,4 +97,23 @@ protected override void OnRemoved(long index) {
 	protected override void OnReaped(long index) {
 	}
 
-}
+
+	protected virtual bool IsNullValue(TProjection projection) => projection is null;
+
+	protected virtual bool ApplyNullPolicy(TItem item, TProjection projection) {
+		if (!IsNullValue(projection)) 
+			return true;
+
+		switch(NullPolicy) {
+			case IndexNullPolicy.IgnoreNull:
+				return false;
+			case IndexNullPolicy.IndexNullValue:
+				return true;
+			case IndexNullPolicy.ThrowOnNull:
+				throw new InvalidOperationException($"Unable to apply index {AttachmentID} for {item.GetType().ToStringCS()} {item} as it resulted in NULL projection"); 
+			default:
+				throw new ArgumentOutOfRangeException();
+		}
+	}
+
+}

src/Hydrogen/ObjectStream/Indexes/UniqueProjectionChecksumIndex.cs

@@ -31,10 +31,12 @@ public UniqueProjectionChecksumIndex(
 		Func<TItem, TKey> projection,
 		IItemChecksummer<TKey> keyChecksummer,
 		Func<long, TKey> keyHydrator,
-		IEqualityComparer<TKey> keyComparer
+		IEqualityComparer<TKey> keyComparer,
+		IndexNullPolicy indexNullPolicy
 	) : base(
 			objectStream,
-			new IndexStorageAttachment<int>(objectStream.Streams, indexName, PrimitiveSerializer<int>.Instance, EqualityComparer<int>.Default)
+			new IndexStorageAttachment<int>(objectStream.Streams, indexName, PrimitiveSerializer<int>.Instance, EqualityComparer<int>.Default),
+			indexNullPolicy
 		) {
 		Guard.ArgumentNotNull(projection, nameof(projection));
 		Guard.ArgumentNotNull(keyChecksummer, nameof(keyChecksummer));
@@ -63,7 +65,7 @@ public override (TKey, int) ApplyProjection(TItem item) {
 
 	protected override void OnAdding(TItem item, long index, (TKey, int) keyChecksum) {
 		if (!IsUnique(keyChecksum, null, out var clashIndex)) 
-			throw new InvalidOperationException($"Unable to add {typeof(TItem).ToStringCS()} as a unique projection (checksummed) violation occurs with projected key '{keyChecksum.Item1}' ({keyChecksum.Item2}) with index {clashIndex}");
+			throw new InvalidOperationException($"Unable to add {typeof(TItem).ToStringCS()} as a unique projection (checksummed) violation occurs with projected key '{keyChecksum.Item1?.ToString() ?? "NULL"}' ({keyChecksum.Item2}) with index {clashIndex}");
 	}
 
 	protected override void OnAdded(TItem item, long index, (TKey, int) keyChecksum) {
@@ -73,7 +75,7 @@ protected override void OnAdded(TItem item, long index, (TKey, int) keyChecksum)
 
 	protected override void OnUpdating(TItem item, long index, (TKey, int) keyChecksum) {
 		if (!IsUnique(keyChecksum, index, out var clashIndex)) 
-			throw new InvalidOperationException($"Unable to update {typeof(TItem).ToStringCS()} as a unique projection (checksummed) violation occurs with projected key '{keyChecksum.Item1}' ({keyChecksum.Item2}) with index {clashIndex}");
+			throw new InvalidOperationException($"Unable to update {typeof(TItem).ToStringCS()} as a unique projection (checksummed) violation occurs with projected key '{keyChecksum.Item1?.ToString() ?? "NULL"}' ({keyChecksum.Item2}) with index {clashIndex}");
 	}
 
 	protected override void OnUpdated(TItem item, long index, (TKey, int) keyChecksum) {
@@ -83,7 +85,7 @@ protected override void OnUpdated(TItem item, long index, (TKey, int) keyChecksu
 
 	protected override void OnInserting(TItem item, long index, (TKey, int) keyChecksum) {
 		if (!IsUnique(keyChecksum, index, out var clashIndex)) 
-			throw new InvalidOperationException($"Unable to insert {typeof(TItem).ToStringCS()} as a unique projection (checksummed) violation occurs with projected key '{keyChecksum.Item1}' ({keyChecksum.Item2}) with index {clashIndex}");
+			throw new InvalidOperationException($"Unable to insert {typeof(TItem).ToStringCS()} as a unique projection (checksummed) violation occurs with projected key '{keyChecksum.Item1?.ToString() ?? "NULL"}' ({keyChecksum.Item2}) with index {clashIndex}");
 	}
 
 	protected override void OnInserted(TItem item, long index, (TKey, int) keyChecksum) {
@@ -108,6 +110,8 @@ protected override void OnContainerCleared() {
 		Store.Attach();
 	}
 
+	protected override bool IsNullValue((TKey, int) projection) => projection.Item1 is null;
+
 	private bool IsUnique((TKey, int) keyChecksum, long? exemptIndex, out long clashIndex) {
 		if (_keyDictionary.TryGetValue(keyChecksum, out var foundIndex)) {
 			if (foundIndex != exemptIndex) {

src/Hydrogen/ObjectStream/Indexes/UniqueProjectionIndex.cs

@@ -17,8 +17,11 @@ internal sealed class UniqueProjectionIndex<TItem, TKey> : ProjectionIndexBase<T
 	public UniqueProjectionIndex(ObjectStream<TItem> objectStream, string indexName, Func<TItem, TKey> projection, IItemSerializer<TKey> keySerializer, IEqualityComparer<TKey> keyComparer)
 		: base(
 			objectStream, 
-			new UniqueKeyStorageAttachment<TKey>(objectStream.Streams, indexName, keySerializer, keyComparer)
-		)  {
+			new UniqueKeyStorageAttachment<TKey>(objectStream.Streams, indexName, keySerializer, keyComparer),
+			IndexNullPolicy.ThrowOnNull
+		) {
+		Guard.ArgumentNotNull(projection, nameof(projection));
+		Guard.Argument(keySerializer.IsConstantSize, nameof(keySerializer), "Must be a constant size serializer");
 		_projection = projection;
 	}
 
@@ -34,7 +37,7 @@ public IReadOnlyDictionary<TKey, long> Values {
 
 	protected override void OnAdding(TItem item, long index, TKey key) {
 		if (!IsUnique(key, null, out var clashIndex)) 
-			throw new InvalidOperationException($"Unable to add {typeof(TItem).ToStringCS()} as a unique keyChecksum violation occurs with item at {clashIndex}");
+			throw new InvalidOperationException($"Unable to add {typeof(TItem).ToStringCS()} as a unique projection violation occurs with projected key '{key?.ToString() ?? "NULL"}' with index {clashIndex}");
 	}
 
 	protected override void OnAdded(TItem item, long index, TKey keyChecksum) {
@@ -43,7 +46,7 @@ protected override void OnAdded(TItem item, long index, TKey keyChecksum) {
 
 	protected override void OnUpdating(TItem item, long index, TKey key) {
 		if (!IsUnique(key, index, out var clashIndex)) 
-			throw new InvalidOperationException($"Unable to update {typeof(TItem).ToStringCS()} at {index} as a unique keyChecksum violation occurs with item at {clashIndex}");
+			throw new InvalidOperationException($"Unable to update {typeof(TItem).ToStringCS()} as a unique projection violation occurs with projected key '{key?.ToString() ?? "NULL"}' with index {clashIndex}");
 	}
 
 	protected override void OnUpdated(TItem item, long index, TKey keyChecksum) {
@@ -52,7 +55,7 @@ protected override void OnUpdated(TItem item, long index, TKey keyChecksum) {
 
 	protected override void OnInserting(TItem item, long index, TKey key) {
 		if (!IsUnique(key, index, out var clashIndex)) 
-			throw new InvalidOperationException($"Unable to insert {typeof(TItem).ToStringCS()} at {index} as a unique keyChecksum violation occurs with item at {clashIndex}");
+			throw new InvalidOperationException($"Unable to insert {typeof(TItem).ToStringCS()} as a unique projection violation occurs with projected key '{key?.ToString() ?? "NULL"}' with index {clashIndex}");
 	}
 
 	protected override void OnInserted(TItem item, long index, TKey keyChecksum) {