Serialization library major refactoring

- moved serializer assembly out of factor and into builder
- comprehensive polymorphic type support
- removed all "on-the-fly" registrations
- binary serializer serializes registrations
- lots of bug fixes
- far more robust

src/Hydrogen/Extensions/DecoratorExtensions.cs

@@ -5,6 +5,7 @@
 using System.ComponentModel;
 using System.IO;
 using System.Linq;
+using System.Reflection;
 
 namespace Hydrogen;
 
@@ -326,18 +327,63 @@ public static TransactionalStream<TStream> AsTransactional<TStream>(this TStream
 
 	#region Serializer
 
-	public static IItemSerializer<T> AsNullableSerializer<T>(this IItemSerializer<T> serializer)
-		=> (typeof(T).IsValueType || serializer.SupportsNull) ? serializer : new ReferenceSerializer<T>(serializer, ReferenceSerializerMode.SupportNull);
+	private static readonly MethodInfo _asUnwrappedGenericMethod = typeof(DecoratorExtensions).GetGenericMethod(nameof(AsUnwrapped), 1);
+	private static readonly MethodInfo _asWrappedGenericMethod1 = typeof(DecoratorExtensions).GetGenericMethods(nameof(AsWrapped), 1).FirstOrDefault(x => x.GetParameters().Length == 1);
+	private static readonly MethodInfo _asWrappedGenericMethod3 = typeof(DecoratorExtensions).GetGenericMethods(nameof(AsWrapped), 1).FirstOrDefault(x => x.GetParameters().Length == 3);
+	private static readonly MethodInfo _asReferenceSerializerMethod = typeof(DecoratorExtensions).GetGenericMethod(nameof(AsReferenceSerializer), 1);
+	private static readonly MethodInfo _asDereferencedSerializer = typeof(DecoratorExtensions).GetGenericMethod(nameof(AsDereferencedSerializer), 1);
+	private static readonly MethodInfo _asCastedSerializer = typeof(DecoratorExtensions).GetGenericMethod(nameof(AsCastedSerializer), 2);	
+
+
+	public static IItemSerializer AsWrapped(this IItemSerializer serializer) 
+		=> _asWrappedGenericMethod1.MakeGenericMethod(serializer.ItemType).Invoke(null, [serializer]) as IItemSerializer;
+
+	public static IItemSerializer AsWrapped(this IItemSerializer serializer, SerializerFactory factory, ReferenceSerializerMode referenceMode) 
+		=> _asWrappedGenericMethod3.MakeGenericMethod(serializer.ItemType).Invoke(null, [serializer, factory, referenceMode]) as IItemSerializer;
+
+	public static IItemSerializer AsUnwrapped(this IItemSerializer serializer) 
+		=> _asUnwrappedGenericMethod.MakeGenericMethod(serializer.ItemType).Invoke(null, [serializer]) as IItemSerializer;
+
+	public static IItemSerializer AsCastedSerializer(this IItemSerializer serializer, Type baseType)
+		=> _asCastedSerializer.MakeGenericMethod(serializer.ItemType, baseType).Invoke(null, [serializer]) as IItemSerializer;
 
 	public static IItemSerializer AsReferenceSerializer(this IItemSerializer serializer, ReferenceSerializerMode mode = ReferenceSerializerMode.Default)
-		=> (serializer.ItemType.IsValueType || serializer.SupportsNull) ? serializer : (IItemSerializer)typeof(ReferenceSerializer<>).MakeGenericType(serializer.ItemType).ActivateWithCompatibleArgs(serializer, mode);
-
-	public static IItemSerializer<T> AsReferenceSerializer<T>(this IItemSerializer<T> serializer, ReferenceSerializerMode mode = ReferenceSerializerMode.Default)
-		=> (typeof(T).IsValueType || serializer.SupportsNull) ? serializer : new ReferenceSerializer<T>(serializer, mode);
+		=> _asReferenceSerializerMethod.MakeGenericMethod(serializer.ItemType).Invoke(null, [serializer, mode]) as IItemSerializer;
 
 	public static IItemSerializer AsDereferencedSerializer(this IItemSerializer serializer)
-		=> serializer.GetType().IsSubtypeOfGenericType(typeof(ReferenceSerializer<>)) ? ((IItemSerializerDecorator)serializer).InternalSerializer : serializer;
+		=> _asDereferencedSerializer.MakeGenericMethod(serializer.ItemType).Invoke(null, [serializer]) as IItemSerializer;
+
+	public static IItemSerializer<T> AsWrapped<T>(this IItemSerializer<T> serializer)  
+		=> serializer.AsWrapped(SerializerFactory.Default, ReferenceSerializerMode.Default);
+
+	public static IItemSerializer<T> AsWrapped<T>(this IItemSerializer<T> serializer, SerializerFactory factory, ReferenceSerializerMode referenceMode)  {
+		var itemType = serializer.ItemType;
+		if (!itemType.IsSealed && serializer is not PolymorphicSerializer<T>) {
+			serializer = serializer.AsPolymorphicSerializer(factory);
+		}  if (!itemType.IsValueType) {
+			serializer = serializer.AsReferenceSerializer(referenceMode);
+		}
+		return serializer;
+	}
+
+	public static IItemSerializer<T> AsUnwrapped<T>(this IItemSerializer<T> serializer) 
+		=> serializer switch {
+			ReferenceSerializer<T> referenceSerializer => referenceSerializer.Internal.AsUnwrapped(),
+			PolymorphicSerializer<T> polymorphicSerializer when !typeof(T).IsAbstract => polymorphicSerializer.Factory.GetPureSerializer<T>(),
+			_ => serializer
+		};
+
+	public static IItemSerializer<TTo> AsCastedSerializer<TFrom, TTo>(this IItemSerializer<TFrom> serializer)
+		=> new CastedSerializer<TFrom, TTo>(serializer);
+
+	public static IItemSerializer<T> AsNullableSerializer<T>(this IItemSerializer<T> serializer)
+		=> (typeof(T).IsValueType || serializer.SupportsNull) ? serializer : new ReferenceSerializer<T>(serializer, ReferenceSerializerMode.SupportNull);
+
+	public static IItemSerializer<T> AsReferenceSerializer<T>(this IItemSerializer<T> serializer, ReferenceSerializerMode mode = ReferenceSerializerMode.Default)
+		=> typeof(T).IsValueType ? serializer : new ReferenceSerializer<T>(serializer, mode);
 
+	public static IItemSerializer<T> AsPolymorphicSerializer<T>(this IItemSerializer<T> serializer, SerializerFactory factory)
+		=> new PolymorphicSerializer<T>(factory, serializer);
 	public static IItemSerializer<T> AsDereferencedSerializer<T>(this IItemSerializer<T> serializer)
 		=> (serializer is ReferenceSerializer<T> referenceSerializer) ? referenceSerializer.Internal : serializer;
 

src/Hydrogen/Extensions/IEnumerableExtensions.cs

@@ -19,7 +19,7 @@ namespace Hydrogen;
 
 public static class IEnumerableExtensions {
 
-	public static (T[], T[]) SelectInto<T>(this IEnumerable<T> source, Predicate<T> predicate) {
+	public static (T[], T[]) SeparateBy<T>(this IEnumerable<T> source, Predicate<T> predicate) {
 		var group1 = new List<T>();
 		var group2 = new List<T>();
 		foreach(var item in source) 

src/Hydrogen/Extensions/TypeExtensions.cs

@@ -23,15 +23,50 @@ namespace Hydrogen;
 /// <remarks></remarks>
 public static class TypeExtensions {
 
+
+	/// <summary>
+	/// Gets generic arguments (and their generic arguments) transitively.
+	/// </summary>
+	public static Type[] GetGenericArgumentsTransitively(this Type type) {
+		var alreadyVisited = new HashSet<Type>();
+		return Get(type).ToArray();
+
+		IEnumerable<Type> Get(Type type) {
+			if (!alreadyVisited.Add(type))
+				yield break;
+
+			yield return type;
+				foreach(var genericArgument in type.GetGenericArguments()) 
+					foreach (var subType in Get(genericArgument))
+						yield return subType;
+
+		}
+	}
+
+	public static bool IsEnumOrNullableEnum(this Type type, out Type enumType) {
+		enumType = Nullable.GetUnderlyingType(type) ?? type;
+		return enumType.IsEnum;
+	}
+
+	public static MethodInfo GetGenericMethod(this Type type, string name, int genericArgs) {
+		var method = type.GetGenericMethods(name, genericArgs).FirstOrDefault();
+		if (method is null) 
+			throw new MissingMethodException(type.FullName, $"{name}<{Tools.Collection.Generate(() => ",").Take(genericArgs).ToDelimittedString(string.Empty)}>");
+		return method;
+	}
+
+	public static IEnumerable<MethodInfo> GetGenericMethods(this Type type, string name, int genericArgs)
+		=> type.GetMethods().Where(m => m.Name == name && m.IsGenericMethod && m.GetGenericArguments().Length == genericArgs);
+
 	public static bool IsCrossAssemblyType(this Type type) {
 		if (!type.IsGenericType)
 			return false;
 		var typeArgAssemblies = type.GenericTypeArguments.Select(x => x.Assembly).Distinct().ToArray();
 		return typeArgAssemblies.Length != 0 && !typeArgAssemblies.SequenceEqual([type.Assembly]);
 	}
-
 
 	public static bool IsAssignableTo(this Type type, [NotNullWhen(true)] Type? targetType) => targetType?.IsAssignableFrom(type) ?? false;
+
 	public static string ToStringCS(this Type type) {
 		if (!type.IsGenericType)
 			return type.Name;

src/Hydrogen/Misc/RecursiveDataType.cs

@@ -5,7 +5,7 @@
 namespace Hydrogen;
 
 /// <summary>
-/// A recursive data type is a data type for values that may contain other values of the same type. Data of recursive types is usually viewed as directed graphs.
+/// A recursive data type is a data type for values that may contain other values of the same type. Data of recursive types is usually viewed as directed acyclic graphs.
 /// A = 0
 /// A(B) = 1 0
 /// A(B(C)) = 1 1 0

src/Hydrogen/ObjectSpaces/Builder/ObjectSpaceBuilder.cs

@@ -216,7 +216,7 @@ public IObjectSpaceDimensionBuilder AddDimension(Type type, bool ignoreAnnotatio
 			if (type.TryGetCustomAttributeOfType<EqualityComparerAttribute>(false, out var equalityComparerAttribute))
 				dimensionBuilder.UsingEqualityComparer(equalityComparerAttribute.EqualityComparerType.ActivateWithCompatibleArgs());
 
-			foreach(var member in SerializerHelper.GetSerializableMembers(type)) {
+			foreach(var member in SerializerBuilder.GetSerializableMembers(type)) {
 				if (member.MemberInfo.TryGetCustomAttributeOfType<IdentityAttribute>(false, out var identityAttribute)) 
 					dimensionBuilder.WithIdentifier(member, identityAttribute.IndexName); 
 

src/Hydrogen/ObjectStream/ObjectStream.cs

@@ -151,11 +151,11 @@ internal ClusteredStream SaveItemAndReturnStream(long index, object item, Object
 				if (_preAllocateOptimization) {
 					// pre-setting the stream length before serialization improves performance since it avoids
 					// re-allocating fragmented stream on individual properties of the serialized item
-					var expectedSize = ItemSerializer.CalculateSize(item);
+					var expectedSize = ItemSerializer.PackedCalculateSize(item);
 					stream.SetLength(expectedSize);
 					ItemSerializer.Serialize(item, writer);
 				} else {
-					var byteLength = ItemSerializer.SerializeReturnSize(item, writer);
+					var byteLength = ItemSerializer.PackedSerializeReturnSize(item, writer);
 					stream.SetLength(byteLength);
 				}
 
@@ -183,7 +183,7 @@ internal ClusteredStream LoadItemAndReturnStream(long index, out object item) {
 			NotifyPreItemOperation(index, default, ObjectStreamOperationType.Read);
 			if (!stream.IsNull) {
 				using var reader = new EndianBinaryReader(EndianBitConverter.For(Streams.Endianness), stream);
-				item = ItemSerializer.Deserialize(reader);
+				item = ItemSerializer.PackedDeserialize(reader);
 				CheckItemType(item);
 			} else {
 				item = default;

src/Hydrogen/Protocol/Builder/ProtocolSerializerBuilder.cs

@@ -10,7 +10,7 @@ namespace Hydrogen;
 
 public class ProtocolSerializerBuilder<TBase> : ProtocolSerializerBuilderBase<TBase, ProtocolSerializerBuilder<TBase>> {
 
-	public ProtocolSerializerBuilder() : this(new PolymorphicSerializer<TBase>()) {
+	public ProtocolSerializerBuilder() : this(new PolymorphicSerializer<TBase>(SerializerFactory.Default)) {
 	}
 
 	internal ProtocolSerializerBuilder(PolymorphicSerializer<TBase> serializer)

src/Hydrogen/Protocol/ProtocolMode.cs

@@ -20,7 +20,7 @@ public ProtocolMode() {
 		RequestHandlers = new Dictionary<Type, IRequestHandler>();
 		ResponseHandlers = new MultiKeyDictionary<Type, Type, IResponseHandler>();
 		MessageGenerators = new Dictionary<Type, IMessageGenerator>();
-		MessageSerializer = new PolymorphicSerializer<object>();
+		MessageSerializer = new PolymorphicSerializer<object>(SerializerFactory.Default);
 	}
 
 	public int Number { get; init; }

src/Hydrogen/Serialization/Annotations/ReferenceModeAttribute.cs

@@ -1,4 +1,6 @@
 using System;
+using System.Reflection;
+using Hydrogen.Mapping;
 
 namespace Hydrogen;
 
@@ -21,4 +23,12 @@ public bool AllowExternalReference {
 	}
 
 	public ReferenceSerializerMode Mode { get; private set; } = ReferenceSerializerMode.Default;
+
+	public static ReferenceSerializerMode GetReferenceModeOrDefault(MemberInfo memberInfo)  {
+		var result = memberInfo.TryGetCustomAttributeOfType<ReferenceModeAttribute>(false, out var attr) ? attr.Mode : ReferenceSerializerMode.Default;
+		if (attr is not null && memberInfo is FieldInfo { FieldType.IsValueType: true } or PropertyInfo { PropertyType.IsValueType: true }) 
+			throw new InvalidOperationException($"Type member {memberInfo.DeclaringType.ToStringCS()}.{memberInfo.Name} incorrectly specifies a {nameof(ReferenceModeAttribute)} for a value-type");
+		return result;		
+
+	}
 }

src/Hydrogen/Serialization/Special/AssemblySerializer.cs → src/Hydrogen/Serialization/BaseTypes/AssemblySerializer.cs

@@ -15,11 +15,13 @@ namespace Hydrogen;
 public sealed class AssemblySerializer : ItemSerializerBase<Assembly> {
 
 	private readonly StringSerializer _fqnSerializer;
-	
-	public AssemblySerializer(SizeDescriptorStrategy sizeDescriptorStrategy = SizeDescriptorStrategy.UseCVarInt) {
+
+	public AssemblySerializer(SizeDescriptorStrategy sizeDescriptorStrategy  = SizeDescriptorStrategy.UseCVarInt) {
 		_fqnSerializer = new StringSerializer(Encoding.Unicode, sizeDescriptorStrategy);
 	}
 
+	public static AssemblySerializer Instance { get; } = new ();
+
 	public override long CalculateSize(SerializationContext context, Assembly item) 
 		=> _fqnSerializer.CalculateSize(item.FullName);
 

src/Hydrogen/Serialization/BaseTypes/KeyValuePairSerializer.cs

@@ -28,7 +28,7 @@ public class KeyValuePairSerializer<TKey, TValue> : ItemSerializerBase<KeyValueP
 	private readonly SizeDescriptorSerializer _sizeSerializer;
 	public KeyValuePairSerializer(IItemSerializer<TKey> keySerializer = null, IItemSerializer<TValue> valueSerializer = null, SizeDescriptorStrategy sizeDescriptorStrategy = SizeDescriptorStrategy.UseCVarInt) {
 		KeySerializer = keySerializer ?? ItemSerializer<TKey>.Default;
-		ValueSerializer = (valueSerializer ?? ItemSerializer<TValue>.Default).AsReferenceSerializer();
+		ValueSerializer = (valueSerializer ?? ItemSerializer<TValue>.Default);
 		_sizeSerializer = new SizeDescriptorSerializer(sizeDescriptorStrategy);
 	}
 

src/Hydrogen/Serialization/BaseTypes/ObjectSerializer.cs

@@ -0,0 +1,10 @@
+namespace Hydrogen;
+
+public sealed class ObjectSerializer : ItemSerializerDecorator<object> {
+
+	public ObjectSerializer(SerializerFactory factory)
+		: base(new ReferenceSerializer<object>(new PolymorphicSerializer<object>( factory, factory.GetPureSerializer<object>()))) {
+	}
+}
+
+

src/Hydrogen/Serialization/Special/TypeSerializer.cs → src/Hydrogen/Serialization/BaseTypes/TypeSerializer.cs

@@ -19,6 +19,8 @@ public TypeSerializer(SizeDescriptorStrategy sizeDescriptorStrategy = SizeDescri
 		_fqnSerializer = new StringSerializer(Encoding.Unicode, sizeDescriptorStrategy);	
 	}
 
+	public static TypeSerializer Instance { get; } = new ();
+
 	public override long CalculateSize(SerializationContext context, Type item) 
 		=> _fqnSerializer.CalculateSize(item.AssemblyQualifiedName);
 

src/Hydrogen/Serialization/BinarySerializer.cs

@@ -1,14 +1,101 @@
-namespace Hydrogen;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+
+namespace Hydrogen;
 
 /// <summary>
 /// Replacement for BinarySerializer deprecated in .NET 5.0 using Hydrogen's serialization framework.
 /// </summary>
-public sealed class BinarySerializer : ItemSerializerDecorator<object> {
+public sealed class BinarySerializer : ItemSerializerDecorator<object, ReferenceSerializer<object>> {
+	private readonly RegistrationSerializer _registrationsSerializer;
+	private readonly SerializerFactory _staticFactory;
+
+	private record struct FactoryRegistration(Type Type, long TypeCode);
+
 	public BinarySerializer() : this(SerializerFactory.Default) {
 	}
 
 	public BinarySerializer(SerializerFactory serializerFactory) 
-		: base(new PolymorphicSerializer<object>(serializerFactory)) {
+		: base(new ReferenceSerializer<object>(new PolymorphicSerializer<object>(serializerFactory))) {
+		_registrationsSerializer = new RegistrationSerializer();
+		_staticFactory = serializerFactory;
+	}
+
+
+	public override long CalculateSize(SerializationContext context, object item) {
+		context.SetEphemeralFactory(new SerializerFactory(_staticFactory));
+		try {
+			// calculate item size
+			var itemSize = base.CalculateSize(context, item);
+
+			// calculate registrations size
+			context.EphemeralFactory.FinishRegistrations();
+			var registrations = context.GetEphemeralRegistrations().Select(x => new FactoryRegistration(x.DataType, x.TypeCode));
+			var registrationsSize = _registrationsSerializer.CalculateSize(registrations); // don't use context
+
+			return registrationsSize + itemSize;
+		} finally {
+			context.ClearEphemeralFactory();
+		}
 	}
 
+	public override void Serialize(object item, EndianBinaryWriter writer, SerializationContext context) {
+		context.SetEphemeralFactory(new SerializerFactory(_staticFactory));
+		try {
+			// serialize item into temp buffer
+			using var memoryStream = new MemoryStream();
+			using var memoryWriter = new EndianBinaryWriter(writer.BitConverter, memoryStream);
+			base.Serialize(item, memoryWriter, context);
+			var buffer = memoryStream.ToArray();
+
+			// serialize registrations first
+			context.EphemeralFactory.FinishRegistrations();
+			var registrations = context.GetEphemeralRegistrations().Select(x => new FactoryRegistration(x.DataType, x.TypeCode));;
+			_registrationsSerializer.Serialize(registrations, writer);  // don't use context
+
+			// serialize item (buffer)
+			writer.Write(buffer);
+		} finally {
+			context.ClearEphemeralFactory();
+		}
+	}
+
+
+	public override object Deserialize(EndianBinaryReader reader, SerializationContext context) {
+
+		// Deserialize type registrations
+		var registrations = _registrationsSerializer.Deserialize(reader); // don't use context
+
+		// Rebuild ephemeral factory used for serialization
+		context.SetEphemeralFactory(new SerializerFactory(_staticFactory));
+
+		foreach(var registration in registrations.OrderBy(x => x.TypeCode)) {
+			if (!context.EphemeralFactory.TryGetRegistration(registration.TypeCode, out var factoryRegistration)) {
+				SerializerBuilder.FactoryAssemble(context.EphemeralFactory, registration.Type, true, registration.TypeCode);
+			} else {
+				// it's already registered (likely as a dependent of prior registration), need to check it matches what we expect
+				Guard.Ensure(registration.Type == factoryRegistration.DataType, $"Deserialization type-code mismatch for type-code {registration.TypeCode}  (expected type {registration.Type.ToStringCS()} but was {factoryRegistration.DataType.ToStringCS()})");
+			}
+		}
+		context.EphemeralFactory.FinishRegistrations();
+
+		// Deserialize item
+		var item = base.Deserialize(reader, context);
+
+		return item;
+	}
+
+
+
+	private class RegistrationSerializer : ProjectedSerializer<IEnumerable<(Type, long)>, IEnumerable<FactoryRegistration>> {
+		public RegistrationSerializer() 
+			: base(
+				  new TaggedTypeCollectionSerializer<long>(PrimitiveSerializer<long>.Instance, SizeDescriptorStrategy.UseCVarInt),
+				  x => x.Select(y => new FactoryRegistration(y.Item1, y.Item2)),
+				  x => x.Select(y => (y.Type, y.TypeCode))
+			) {
+		}
+	}
 }