Implementation of subtypes in Java

TODO:
- extend language subtype tests
- extend SerializeUtil tests in runtime
- update documentation in main README.md for Java extension
- update design notes and discussion

Author: mikir

compiler/extensions/java/freemarker/Subtype.java.ftl

@@ -0,0 +1,27 @@
+<#include "FileHeader.inc.ftl">
+<#include "DocComment.inc.ftl">
+<@standard_header generatorDescription, packageName/>
+
+<#if withCodeComments && docComments??>
+<@doc_comments docComments/>
+</#if>
+public final class ${name} implements zserio.runtime.ZserioSubtype
+{
+    @Override
+    public String getBaseTypePackage()
+    {
+        return <#if basePackageName?? && basePackageName != "">"${basePackageName}"<#else>null</#if>;
+    }
+
+    @Override
+    public String getBaseTypeName()
+    {
+        return "${baseTypeName}";
+    }
+
+    @Override
+    public Class<?> getBaseClass()
+    {
+        return ${baseClassName}.class;
+    }
+}

compiler/extensions/java/runtime/src/zserio/runtime/ZserioSubtype.java

@@ -0,0 +1,28 @@
+package zserio.runtime;
+
+/**
+ * Interface implemented by all subtypes generated by Zserio.
+ */
+public interface ZserioSubtype
+{
+    /**
+     * Gets the package of the base type which is resolved from the subtype.
+     *
+     * @return The package of the base type or null if the base type is simple type.
+     */
+    String getBaseTypePackage();
+
+    /**
+     * Gets the name of the base type which is resolved from the subtype.
+     *
+     * @return The name of the base type.
+     */
+    String getBaseTypeName();
+
+    /**
+     * Gets the class of the base type which is resolved from the subtype.
+     *
+     * @return The class of the base type or the class of boxing type if the base type is simple type.
+     */
+    Class<?> getBaseClass();
+}

compiler/extensions/java/runtime/src/zserio/runtime/io/SerializeUtil.java

@@ -13,6 +13,7 @@
 
 import zserio.runtime.ZserioEnum;
 import zserio.runtime.ZserioError;
+import zserio.runtime.ZserioSubtype;
 
 /**
  * Provides help methods for serialization and deserialization of generated objects.
@@ -71,7 +72,7 @@ public static <T extends Writer> BitBuffer serialize(T object)
      *
      * @return Generated object created from given bit buffer.
      */
-    public static <T> T deserialize(final Class<T> clazz, BitBuffer bitBuffer, Object... arguments)
+    public static <T> T deserialize(Class<T> clazz, BitBuffer bitBuffer, Object... arguments)
     {
         return deserializeFromBytes(clazz, bitBuffer.getBuffer(), arguments);
     }
@@ -136,7 +137,7 @@ public static <T extends Writer> byte[] serializeToBytes(T object)
      *
      * @return Generated object created from given byte array.
      */
-    public static <T> T deserializeFromBytes(final Class<T> clazz, byte[] buffer, Object... arguments)
+    public static <T> T deserializeFromBytes(Class<T> clazz, byte[] buffer, Object... arguments)
     {
         try (final BitStreamReader reader = new ByteArrayBitStreamReader(buffer))
         {
@@ -148,6 +149,42 @@ public static <T> T deserializeFromBytes(final Class<T> clazz, byte[] buffer, Ob
         }
     }
 
+    /**
+     * Deserializes byte array to the generated object given by the class name.
+     * <p>
+     * This method resolves subtypes to the base types before deserialization.
+     * <p>
+     * This method can potentially use all bits of the last byte even if not all of them were written during
+     * serialization (because there is no way how to specify exact number of bits). Thus, it could allow reading
+     * behind stream (possibly in case of damaged data).
+     * <p>
+     * Example:
+     * <blockquote><pre>
+     * import zserio.runtime.io.SerializeUtil;
+     *
+     * final SomeZserioObject object = new SomeZserioObject();
+     * final byte[] buffer = SerializeUtil.serializeToBytes(object);
+     * final SomeZserioObject readObject = SerializeUtil.deserializeFromBytes("SomeZserioObject", buffer);
+     * </pre></blockquote>
+     *
+     * @param className Class name of the generated object to deserialize.
+     * @param buffer Byte array which represents generated object in binary format.
+     * @param arguments Additional arguments needed for reader constructor (optional).
+     *
+     * @return Generated object created from given byte array.
+     */
+    public static Object deserializeFromBytes(String className, byte[] buffer, Object... arguments)
+    {
+        try (final BitStreamReader reader = new ByteArrayBitStreamReader(buffer))
+        {
+            return deserializeFromReader(className, reader, arguments);
+        }
+        catch (IOException exception)
+        {
+            throw new ZserioError("SerializeUtil: " + exception, exception);
+        }
+    }
+
     /**
      * Serializes generated object to the file using file name.
      * <p>
@@ -231,11 +268,39 @@ public static <T extends Writer> void serializeToFile(T object, File file)
      *
      * @return Generated object created from given file contents.
      */
-    public static <T> T deserializeFromFile(final Class<T> clazz, String fileName, Object... arguments)
+    public static <T> T deserializeFromFile(Class<T> clazz, String fileName, Object... arguments)
     {
         return deserializeFromFile(clazz, new File(fileName), arguments);
     }
 
+    /**
+     * Deserializes file to the generated object using class name and file name.
+     * <p>
+     * This method resolves subtypes to the base types before deserialization.
+     * <p>
+     * This is a convenient method for users to easily read given generated object from file.
+     * <p>
+     * Example:
+     * <blockquote><pre>
+     * import zserio.runtime.io.SerializeUtil;
+     *
+     * final String fileName = "FileName.bin";
+     * final SomeZserioObject object = new SomeZserioObject();
+     * SerializeUtil.serializeToFile(object, fileName);
+     * final SomeZserioObject readObject = SerializeUtil.deserializeFromFile("SomeZserioObject", fileName);
+     * </pre></blockquote>
+     *
+     * @param className Class name of the generated object to deserialize.
+     * @param fileName Name of the file which represents generated object in binary format.
+     * @param arguments Additional arguments needed for reader constructor (optional).
+     *
+     * @return Generated object created from given file contents.
+     */
+    public static Object deserializeFromFile(String className, String fileName, Object... arguments)
+    {
+        return deserializeFromFile(className, new File(fileName), arguments);
+    }
+
     /**
      * Deserializes file to the generated object.
      * <p>
@@ -258,7 +323,7 @@ public static <T> T deserializeFromFile(final Class<T> clazz, String fileName, O
      *
      * @return Generated object created from given file contents.
      */
-    public static <T> T deserializeFromFile(final Class<T> clazz, File file, Object... arguments)
+    public static <T> T deserializeFromFile(Class<T> clazz, File file, Object... arguments)
     {
         try
         {
@@ -274,15 +339,73 @@ public static <T> T deserializeFromFile(final Class<T> clazz, File file, Object.
         }
     }
 
+    /**
+     * Deserializes file to the generated object using class name.
+     * <p>
+     * This method resolves subtypes to the base types before deserialization.
+     * <p>
+     * This is a convenient method for users to easily read given generated object from file.
+     * <p>
+     * Example:
+     * <blockquote><pre>
+     * import zserio.runtime.io.SerializeUtil;
+     *
+     * final String fileName = "FileName.bin";
+     * final SomeZserioObject object = new SomeZserioObject();
+     * SerializeUtil.serializeToFile(object, fileName);
+     * final SomeZserioObject readObject = SerializeUtil.deserializeFromFile("SomeZserioObject", fileName);
+     * </pre></blockquote>
+     *
+     * @param className Class name of the generated object to deserialize.
+     * @param file File which represents generated object in binary format.
+     * @param arguments Additional arguments needed for reader constructor (optional).
+     *
+     * @return Generated object created from given file contents.
+     */
+    public static Object deserializeFromFile(String className, File file, Object... arguments)
+    {
+        try
+        {
+            final byte[] fileContent = Files.readAllBytes(file.toPath());
+            try (final BitStreamReader reader = new ByteArrayBitStreamReader(fileContent))
+            {
+                return deserializeFromReader(className, reader, arguments);
+            }
+        }
+        catch (IOException exception)
+        {
+            throw new ZserioError("SerializeUtil: " + exception, exception);
+        }
+    }
+
     private static <T extends Writer> void serializeToWriter(T object, BitStreamWriter writer)
             throws IOException
     {
         object.initializeOffsets(writer.getBitPosition());
         object.write(writer);
     }
 
-    private static <T> T deserializeFromReader(
-            final Class<T> clazz, BitStreamReader reader, Object... arguments)
+    private static Object deserializeFromReader(String className, BitStreamReader reader, Object... arguments)
+    {
+        try
+        {
+            Class<?> clazz = Class.forName(className);
+            if (clazz.isInstance(ZserioSubtype.class))
+            {
+                final Method baseClassMethod = clazz.getMethod("getBaseClass");
+                clazz = (Class<?>)baseClassMethod.invoke(null);
+            }
+
+            return deserializeFromReader(clazz, reader, arguments);
+        }
+        catch (ClassNotFoundException | NoSuchMethodException | SecurityException | IllegalAccessException |
+                IllegalArgumentException | InvocationTargetException exception)
+        {
+            throw new ZserioError("SerializeUtil: " + exception, exception);
+        }
+    }
+
+    private static <T> T deserializeFromReader(Class<T> clazz, BitStreamReader reader, Object... arguments)
     {
         try
         {

compiler/extensions/java/src/zserio/extension/java/JavaExtension.java

@@ -82,6 +82,7 @@ public void process(Root rootNode, ExtensionParameters parameters) throws Zserio
         emitters.add(new SqlTableEmitter(outputFileManager, javaParameters, packedTypesCollector));
         emitters.add(new ConstEmitter(outputFileManager, javaParameters, packedTypesCollector));
         emitters.add(new ServiceEmitter(outputFileManager, javaParameters, packedTypesCollector));
+        emitters.add(new SubtypeEmitter(outputFileManager, javaParameters, packedTypesCollector));
         emitters.add(new PubsubEmitter(outputFileManager, javaParameters, packedTypesCollector));
 
         // emit Java code

compiler/extensions/java/src/zserio/extension/java/JavaNativeMapper.java

@@ -22,6 +22,7 @@
 import zserio.ast.StdIntegerType;
 import zserio.ast.StringType;
 import zserio.ast.StructureType;
+import zserio.ast.Subtype;
 import zserio.ast.TypeInstantiation;
 import zserio.ast.TypeReference;
 import zserio.ast.UnionType;
@@ -55,6 +56,7 @@
 import zserio.extension.java.types.NativeSqlDatabaseType;
 import zserio.extension.java.types.NativeSqlTableType;
 import zserio.extension.java.types.NativeStringType;
+import zserio.extension.java.types.NativeSubtype;
 
 /**
  * Java native mapper.
@@ -148,6 +150,19 @@ public NativeIntegralType getJavaIntegralType(TypeInstantiation typeInstantiatio
         return (NativeIntegralType)nativeType;
     }
 
+    public JavaNativeType getNullableJavaType(TypeReference typeReference) throws ZserioExtensionException
+    {
+        final JavaNativeTypes javaTypes = getJavaTypes(typeReference);
+        final JavaNativeType nativeNullableType = javaTypes != null ? javaTypes.getNullableType() : null;
+        if (nativeNullableType == null)
+        {
+            throw new ZserioExtensionException(
+                    "Unhandled type '" + typeReference.getClass().getName() + "' in JavaNativeMapper!");
+        }
+
+        return nativeNullableType;
+    }
+
     public JavaNativeType getNullableJavaType(TypeInstantiation typeInstantiation)
             throws ZserioExtensionException
     {
@@ -329,6 +344,23 @@ public void visitUnionType(UnionType type)
             javaTypes = mapCompoundType(type);
         }
 
+        @Override
+        public void visitSubtype(Subtype type)
+        {
+            try
+            {
+                final JavaNativeType nativeBaseType = getJavaType(type.getBaseTypeReference());
+                final PackageName packageName = type.getPackage().getPackageName();
+                final String name = type.getName();
+                final JavaNativeType javaType = new NativeSubtype(packageName, name, nativeBaseType.isSimple());
+                javaTypes = new JavaNativeTypes(javaType);
+            }
+            catch (ZserioExtensionException exception)
+            {
+                thrownException = exception;
+            }
+        }
+
         @Override
         public void visitEnumType(EnumType type)
         {

compiler/extensions/java/src/zserio/extension/java/SubtypeEmitter.java

@@ -0,0 +1,28 @@
+package zserio.extension.java;
+
+import zserio.ast.Subtype;
+import zserio.extension.common.OutputFileManager;
+import zserio.extension.common.PackedTypesCollector;
+import zserio.extension.common.ZserioExtensionException;
+
+/**
+ * Emitter for subtypes.
+ */
+public final class SubtypeEmitter extends JavaDefaultEmitter
+{
+    public SubtypeEmitter(OutputFileManager outputFileManager, JavaExtensionParameters javaParameters,
+            PackedTypesCollector packedTypesCollector)
+    {
+        super(outputFileManager, javaParameters, packedTypesCollector);
+    }
+
+    @Override
+    public void beginSubtype(Subtype subtype) throws ZserioExtensionException
+    {
+        final SubtypeEmitterTemplateData templateData =
+                new SubtypeEmitterTemplateData(getTemplateDataContext(), subtype);
+        processTemplate(TEMPLATE_NAME, templateData, subtype.getPackage(), subtype.getName());
+    }
+
+    private static final String TEMPLATE_NAME = "Subtype.java.ftl";
+}

compiler/extensions/java/src/zserio/extension/java/SubtypeEmitterTemplateData.java

@@ -0,0 +1,47 @@
+package zserio.extension.java;
+
+import zserio.ast.Subtype;
+import zserio.ast.TypeReference;
+import zserio.extension.common.ZserioExtensionException;
+import zserio.extension.java.types.JavaNativeType;
+
+/**
+ * FreeMarker template data for SubtypeEmitter.
+ */
+public final class SubtypeEmitterTemplateData extends UserTypeTemplateData
+{
+    public SubtypeEmitterTemplateData(TemplateDataContext context, Subtype subtype)
+            throws ZserioExtensionException
+    {
+        super(context, subtype, subtype);
+
+        final TypeReference baseTypeReference = subtype.getBaseTypeReference();
+        final JavaNativeMapper javaNativeMapper = context.getJavaNativeMapper();
+
+        final JavaNativeType nullableBaseNativeType = javaNativeMapper.getNullableJavaType(baseTypeReference);
+        final JavaNativeType baseNativeType = javaNativeMapper.getJavaType(baseTypeReference);
+
+        basePackageName = JavaFullNameFormatter.getFullName(baseNativeType.getPackageName());
+        baseTypeName = baseNativeType.getName();
+        baseClassName = nullableBaseNativeType.getFullName();
+    }
+
+    public String getBasePackageName()
+    {
+        return basePackageName;
+    }
+
+    public String getBaseTypeName()
+    {
+        return baseTypeName;
+    }
+
+    public String getBaseClassName()
+    {
+        return baseClassName;
+    }
+
+    private final String basePackageName;
+    private final String baseTypeName;
+    private final String baseClassName;
+}