Dense layers with optionally no bias (#155)

* Add has_bias template parameter for DenseT

* Implementing other backends and tests

* More tests

* Apply clang-format

* Fixing Windows test

---------

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

RTNeural/ModelT.h

@@ -77,8 +77,8 @@ namespace modelt_detail
         json_parser::debug_print("Loading a no-op layer!", debug);
     }
 
-    template <typename T, int in_size, int out_size>
-    void loadLayer(DenseT<T, in_size, out_size>& dense, int& json_stream_idx, const nlohmann::json& l,
+    template <typename T, int in_size, int out_size, bool has_bias>
+    void loadLayer(DenseT<T, in_size, out_size, has_bias>& dense, int& json_stream_idx, const nlohmann::json& l,
         const std::string& type, int layerDims, bool debug)
     {
         using namespace json_parser;

RTNeural/common.h

@@ -29,6 +29,10 @@ constexpr T ceil_div(T num, T den)
 {
     return (num + den - 1) / den;
 }
+
+struct Empty
+{
+};
 } // namespace RTNEURAL_NAMESPACE
 
 #if RTNEURAL_USE_EIGEN

RTNeural/config.h

@@ -13,6 +13,19 @@
 #define RTNEURAL_DEFAULT_ALIGNMENT 16
 #endif
 
+#if defined(_MSVC_LANG)
+#define RTNEURAL_CPLUSPLUS _MSVC_LANG
+#elif defined(__cplusplus)
+#define RTNEURAL_CPLUSPLUS __cplusplus
+#endif
+
+#if defined(RTNEURAL_CPLUSPLUS) && RTNEURAL_CPLUSPLUS >= 201703L
+#define RTNEURAL_HAS_CPP17 1
+#define RTNEURAL_IF_CONSTEXPR if constexpr
+#else
+#define RTNEURAL_IF_CONSTEXPR if
+#endif
+
 /**
     Facilitate testing real-time safety with RealtimeSanitizer (RADSan)
 

RTNeural/dense/dense.h

@@ -49,7 +49,7 @@ class Dense1
 
 private:
     const int in_size;
-    T bias;
+    T bias {};
 
     T* weights;
 };
@@ -63,6 +63,8 @@ template <typename T>
 class Dense final : public Layer<T>
 {
 public:
+    static constexpr bool dense_has_bias = true;
+
     /** Constructs a dense layer for a given input and output size. */
     Dense(int in_size, int out_size)
         : Layer<T>(in_size, out_size)
@@ -157,14 +159,15 @@ class Dense final : public Layer<T>
  * Static implementation of a fully-connected (dense) layer,
  * with no activation.
  */
-template <typename T, int in_sizet, int out_sizet>
+template <typename T, int in_sizet, int out_sizet, bool has_bias = true>
 class DenseT
 {
     static constexpr auto weights_size = in_sizet * out_sizet;
 
 public:
     static constexpr auto in_size = in_sizet;
     static constexpr auto out_size = out_sizet;
+    static constexpr bool dense_has_bias = has_bias;
 
     DenseT()
     {
@@ -188,10 +191,19 @@ class DenseT
     RTNEURAL_REALTIME void reset() { }
 
     /** Performs forward propagation for this layer. */
-    RTNEURAL_REALTIME inline void forward(const T (&ins)[in_size]) noexcept
+    template <bool b = has_bias>
+    RTNEURAL_REALTIME inline typename std::enable_if<b>::type forward(const T (&ins)[in_size]) noexcept
+    {
+        for(int i = 0; i < out_size; ++i)
+            outs[i] = std::inner_product(ins, ins + in_size, &weights[i * in_size], bias[i]);
+    }
+
+    /** Performs forward propagation for this layer (no bias). */
+    template <bool b = has_bias>
+    RTNEURAL_REALTIME inline typename std::enable_if<!b>::type forward(const T (&ins)[in_size]) noexcept
     {
         for(int i = 0; i < out_size; ++i)
-            outs[i] = std::inner_product(ins, ins + in_size, &weights[i * in_size], (T)0) + bias[i];
+            outs[i] = std::inner_product(ins, ins + in_size, &weights[i * in_size], (T)0);
     }
 
     /**

RTNeural/dense/dense_eigen.h

@@ -2,6 +2,7 @@
 #define DENSEEIGEN_H_INCLUDED
 
 #include "../Layer.h"
+#include "../common.h"
 #include "../config.h"
 #include <Eigen/Dense>
 
@@ -16,6 +17,8 @@ template <typename T>
 class Dense : public Layer<T>
 {
 public:
+    static constexpr bool dense_has_bias = true;
+
     /** Constructs a dense layer for a given input and output size. */
     Dense(int in_size, int out_size)
         : Layer<T>(in_size, out_size)
@@ -118,23 +121,31 @@ class Dense : public Layer<T>
  * Static implementation of a fully-connected (dense) layer,
  * with no activation.
  */
-template <typename T, int in_sizet, int out_sizet>
+template <typename T, int in_sizet, int out_sizet, bool has_bias = true>
 class DenseT
 {
     using out_vec_type = Eigen::Matrix<T, out_sizet, 1>;
+#if RTNEURAL_HAS_CPP17
+    using in_vec_type = typename std::conditional<has_bias, Eigen::Matrix<T, in_sizet + 1, 1>, Empty>::type;
+#else
     using in_vec_type = Eigen::Matrix<T, in_sizet + 1, 1>;
-    using mat_type = Eigen::Matrix<T, out_sizet, in_sizet + 1>;
+#endif
+    using mat_type = Eigen::Matrix<T, out_sizet, in_sizet + (has_bias ? 1 : 0)>;
 
 public:
     static constexpr auto in_size = in_sizet;
     static constexpr auto out_size = out_sizet;
+    static constexpr bool dense_has_bias = has_bias;
 
     DenseT()
         : outs(outs_internal)
     {
         weights = mat_type::Zero();
-        ins_internal = in_vec_type::Zero();
-        ins_internal(in_size, 0) = (T)1;
+        RTNEURAL_IF_CONSTEXPR(has_bias)
+        {
+            ins_internal = in_vec_type::Zero();
+            ins_internal(in_size, 0) = (T)1;
+        }
         outs = out_vec_type::Zero();
     }
 
@@ -148,7 +159,8 @@ class DenseT
     RTNEURAL_REALTIME void reset() { }
 
     /** Performs forward propagation for this layer. */
-    RTNEURAL_REALTIME inline void forward(const Eigen::Matrix<T, in_size, 1>& ins) noexcept
+    template <bool b = has_bias>
+    RTNEURAL_REALTIME inline typename std::enable_if<b>::type forward(const Eigen::Matrix<T, in_size, 1>& ins) noexcept
     {
         for(int i = 0; i < in_size; ++i)
             ins_internal(i, 0) = ins(i, 0);
@@ -160,6 +172,13 @@ class DenseT
         outs.noalias() = weights * ins_internal;
     }
 
+    /** Performs forward propagation for this layer (no bias). */
+    template <bool b = has_bias>
+    RTNEURAL_REALTIME inline typename std::enable_if<!b>::type forward(const Eigen::Matrix<T, in_size, 1>& ins) noexcept
+    {
+        outs.noalias() = weights * ins;
+    }
+
     /**
      * Sets the layer weights from a given vector.
      *
@@ -190,7 +209,12 @@ class DenseT
      * Sets the layer bias from a given array of size
      * bias[out_size]
      */
-    RTNEURAL_REALTIME void setBias(const T* b)
+#if RTNEURAL_HAS_CPP17
+    template <bool b = has_bias>
+    RTNEURAL_REALTIME inline typename std::enable_if<b>::type setBias(const T* b)
+#else
+    RTNEURAL_REALTIME inline void setBias(const T* b)
+#endif
     {
         for(int i = 0; i < out_size; ++i)
             weights(i, in_size) = b[i];